From b36790bec991e9cd4d2dfcb93e6e5a45e0873797 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:26:30 -0400 Subject: [PATCH 001/631] cpufreq: apple: Add bounded T8152 E and P/M performance domains --- Documentation/arch/arm64/apple-m6-cpu.rst | 29 +++++++++ .../cpufreq/apple,cluster-cpufreq.yaml | 1 + arch/arm64/boot/dts/apple/t8152-j873g.dts | 59 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 Documentation/arch/arm64/apple-m6-cpu.rst diff --git a/Documentation/arch/arm64/apple-m6-cpu.rst b/Documentation/arch/arm64/apple-m6-cpu.rst new file mode 100644 index 00000000000000..046872e83e09a7 --- /dev/null +++ b/Documentation/arch/arm64/apple-m6-cpu.rst @@ -0,0 +1,29 @@ +M6 CPU bring-up +============== + +J873g has six E CPUs and a shared performance domain containing two P CPUs +and four M CPUs. The T8152 ACC request uses a five-bit hardware state, with +state 2 as the first ADT operating point. Do not use the older voltage or PLL +initialization sequences. + +Failure cases and qualification +------------------------------ + +Before testing, account for incorrect MPIDRs, incomplete spin-table release, +broken cross-CPU interrupts, a stalled architectural timer, mismatched OPP +indices, an unknown inherited request, a stuck busy bit, an unacknowledged +transition, and another agent changing the request. A transition failure must +latch until the policy is recreated. A requested frequency is not a measured +frequency. + +The initial tables expose only states 2 and 3. Higher states require an SMC +thermal policy; this port does not qualify them. The loader must hand over +within the advertised table. Keep firmware voltage, PLL and throttling +configuration intact. + +On physical J873g, capture the source revision, image and DTB hashes, boot log, +online CPU list, per-CPU workload completion, interrupt counters and both +cpufreq policies. Exercise both exposed states in each domain, compare timed +dependent workloads, restore the low states and check the kernel log. Save +the commands and a machine-readable receipt. Native m1n1 SMP results alone +do not qualify Linux scheduling. diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index 8e87c0f6d3d451..e4b1102553f720 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -36,6 +36,7 @@ properties: - const: apple,s5l8960x-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq - const: apple,t8140-cluster-cpufreq + - const: apple,t8152-cluster-cpufreq - items: - const: apple,t6020-cluster-cpufreq - const: apple,t8112-cluster-cpufreq diff --git a/arch/arm64/boot/dts/apple/t8152-j873g.dts b/arch/arm64/boot/dts/apple/t8152-j873g.dts index bd259e3f7e364a..6b5e14036c98af 100644 --- a/arch/arm64/boot/dts/apple/t8152-j873g.dts +++ b/arch/arm64/boot/dts/apple/t8152-j873g.dts @@ -62,6 +62,8 @@ reg = <0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu1: cpu@1 { @@ -70,6 +72,8 @@ reg = <0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu2: cpu@2 { @@ -78,6 +82,8 @@ reg = <0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu3: cpu@3 { @@ -86,6 +92,8 @@ reg = <0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu4: cpu@4 { @@ -94,6 +102,8 @@ reg = <0 0x4>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu5: cpu@5 { @@ -102,6 +112,8 @@ reg = <0 0x5>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_e_opp>; + performance-domains = <&m6_e_cpufreq>; }; cpu6: cpu@10100 { @@ -110,6 +122,8 @@ reg = <0 0x10100>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu7: cpu@10101 { @@ -118,6 +132,8 @@ reg = <0 0x10101>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu8: cpu@40102 { @@ -126,6 +142,8 @@ reg = <0 0x40102>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu9: cpu@40103 { @@ -134,6 +152,8 @@ reg = <0 0x40103>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu10: cpu@40104 { @@ -142,6 +162,8 @@ reg = <0 0x40104>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; cpu11: cpu@40105 { @@ -150,9 +172,35 @@ reg = <0 0x40105>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ + operating-points-v2 = <&m6_pm_opp>; + performance-domains = <&m6_pm_cpufreq>; }; }; + m6_e_opp: opp-table-e { + compatible = "operating-points-v2"; + opp-shared; + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + opp-1152000000 { + opp-hz = /bits/ 64 <1152000000>; + opp-level = <3>; + }; + }; + m6_pm_opp: opp-table-pm { + compatible = "operating-points-v2"; + opp-shared; + opp-1440000000 { + opp-hz = /bits/ 64 <1440000000>; + opp-level = <2>; + }; + opp-1728000000 { + opp-hz = /bits/ 64 <1728000000>; + opp-level = <3>; + }; + }; timer { compatible = "arm,armv8-timer"; interrupts = , @@ -178,6 +226,17 @@ reg-names = "core", "event"; }; + m6_e_cpufreq: performance-controller@210e20000 { + compatible = "apple,t8152-cluster-cpufreq"; + reg = <2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + m6_pm_cpufreq: performance-controller@211e20000 { + compatible = "apple,t8152-cluster-cpufreq"; + reg = <2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + dockchannel: serial@30812c000 { compatible = "apple,dockchannel-uart"; reg = <3 0x0812c000 0 0x4000>, From f1da0a655fa4441daed637e4e3e59d2b310c50ce Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 11:57:48 -0400 Subject: [PATCH 002/631] arm64: apple: preserve registers across WFI on T8142 --- Documentation/arch/arm64/apple-j813.rst | 30 ++++++++++++++++++++----- arch/arm64/include/asm/cputype.h | 4 ++++ arch/arm64/kernel/idle.c | 6 +++-- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index b2fa78079dd46b..28e13f9aafffc6 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -8,16 +8,36 @@ J813 is the 13-inch M5 MacBook Air (T8142, Mac17,3). This configuration provides an initial RAM-only boot path: one boot CPU, AICv3, architectural timers, DockChannel console and the framebuffer left by the boot firmware. -The device tree describes all ten CPU affinities so the loader can identify -the boot CPU. It does not qualify SMP. Keep secondary cores stopped in -m1n1 and use ``maxcpus=1`` for this milestone. The loader removes unavailable -CPUs and fills the release addresses of the remaining nodes. The P-core +The device tree describes all ten CPU affinities. For the initial one-CPU +baseline, keep secondary cores stopped in m1n1 and use ``maxcpus=1``. +The loader removes unavailable CPUs and fills the release addresses of the +remaining nodes. The P-core affinities are 0x10100 through 0x10103, rather than the 0x100 through 0x103 values in the Apple Device Tree. The CPU compatible strings identify T8142 explicitly. The firmware's reused ``sawtooth`` and ``everest`` names do not identify the M5 microarchitecture. +SMP bring-up +============ + +M5 secondary CPUs can return from WFI with cleared general-purpose registers. +Both M5 MIDRs (parts 0x062 and 0x063) select the Apple register-preserving idle +path. Without it, all ten CPUs can start and then panic in their idle tasks. + +SMP also requires the M5-capable m1n1 secondary-start path: dense PMGR CPU +masks, correct reset vectors and per-core stacks. Verify all ten cores there +before preparing the FDT, and use ``maxcpus=10`` for the Linux test. The +existing J813 DT and configuration support this handoff. + +On J813, this path reached the RAM shell with CPUs 0-9 online. A five-second +sleep advanced every CPU's architectural timer count by 1,251 interrupts. +Ten concurrent, individually pinned jobs each hashed 16 MiB of zeros to +``080acf35a507ac9849cfcba47dc2ad83e01b75663a516279c8b9d243b719643e``. +Verify affinity through ``Cpus_allowed_list`` and retain the console output, +kernel configuration hash and payload hashes when repeating the test. +Hotplug, deep sleep and KVM remain unqualified. + Bootloader requirements ======================= @@ -103,7 +123,7 @@ RAM boot. Preserve the complete console transcript and a hash manifest; an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. -This milestone does not establish SMP, CPU hotplug, KVM guest timers, +These tests do not establish CPU hotplug, KVM guest timers, cpufreq, suspend, native DCP, GPU acceleration, built-in input, USB, storage, networking or audio support. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 632f980ec55e56..809e39962fe231 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -155,6 +155,8 @@ #define APPLE_CPU_PART_M2_AVALANCHE_MAX 0x039 #define APPLE_CPU_PART_TAHITI_E 0x060 #define APPLE_CPU_PART_TAHITI_P 0x061 +#define APPLE_CPU_PART_T8142_E 0x062 +#define APPLE_CPU_PART_T8142_P 0x063 #define APPLE_CPU_PART_T8152_E 0x072 #define APPLE_CPU_PART_T8152_P 0x073 @@ -252,6 +254,8 @@ #define MIDR_APPLE_M2_AVALANCHE_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M2_AVALANCHE_MAX) #define MIDR_APPLE_TAHITI_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_E) #define MIDR_APPLE_TAHITI_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_TAHITI_P) +#define MIDR_APPLE_T8142_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_E) +#define MIDR_APPLE_T8142_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8142_P) #define MIDR_APPLE_T8152_E MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_E) #define MIDR_APPLE_T8152_P MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_T8152_P) #define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 972164e78874c4..14cde8592c86e4 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -41,9 +41,9 @@ early_param("idle", setup_idle); #ifdef CONFIG_ARCH_APPLE /* - * Apple T8140 (A18 Pro) secondary CPUs and T8152 (M6) CPUs can return from + * Apple T8140 (A18 Pro), T8142 (M5), and T8152 (M6) CPUs can return from * WFI with general-purpose registers cleared. T8140 also loses FP/SIMD - * state; conservatively preserve it on both parts. The key is enabled once + * state; conservatively preserve it on all three parts. The key is enabled once * at boot on those parts; everywhere else the check is a patched-out branch. */ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); @@ -51,6 +51,8 @@ static DEFINE_STATIC_KEY_FALSE(apple_wfi_loses_regs); static const struct midr_range apple_wfi_loses_regs_cpus[] __initconst = { MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_E), MIDR_ALL_VERSIONS(MIDR_APPLE_TAHITI_P), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_E), + MIDR_ALL_VERSIONS(MIDR_APPLE_T8142_P), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_E), MIDR_ALL_VERSIONS(MIDR_APPLE_T8152_P), {} From 1174b7eee3b4a61c8f38f406397d388a5ad27666 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 003/631] dt-bindings: cpufreq: apple: describe the T8142 cluster controller Add the M5 compatible for its five-bit P-state command interface. The controller retains firmware voltage and CLPC/PMP configuration. Signed-off-by: Acelogic --- .../devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index e4b1102553f720..d6618e159cf723 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -36,6 +36,7 @@ properties: - const: apple,s5l8960x-cluster-cpufreq - const: apple,s5l8960x-cluster-cpufreq - const: apple,t8140-cluster-cpufreq + - const: apple,t8142-cluster-cpufreq - const: apple,t8152-cluster-cpufreq - items: - const: apple,t6020-cluster-cpufreq From 5b40ecc8b038bdfd52dc92e0403e90bad845fb8c Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 004/631] arm64: delay: avoid register-losing WFIT on Apple M5 T8142 can clear registers when returning from WFIT, just as it can with WFI in the idle task. The second Linux cluster-frequency transition on J813 reached __const_udelay while polling the command register, then returned with a cleared callee-saved pointer and faulted in cpufreq. Use the existing counter delay loop on both T8142 core types. The idle task's register/FP flush path cannot be used by arbitrary kernel delays which may have live SIMD state. With this change, repeated E/P state sweeps and pinned integer workloads complete on J813, followed by timer progress and SHA-256 jobs on all ten CPUs. Other CPU models retain their existing delay paths. Signed-off-by: Acelogic --- arch/arm64/lib/delay.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/lib/delay.c b/arch/arm64/lib/delay.c index e278e060e78a9e..e0c5e615b5e5d2 100644 --- a/arch/arm64/lib/delay.c +++ b/arch/arm64/lib/delay.c @@ -14,6 +14,7 @@ #include #include +#include #define USECS_TO_CYCLES(time_usecs) \ xloops_to_cycles((time_usecs) * 0x10C7UL) @@ -42,6 +43,18 @@ void __delay(unsigned long cycles) { cycles_t start = __delay_cycles(); + /* + * T8142 can clear registers on WFIT just as it does on WFI. A delay + * may run with live kernel SIMD state, so the idle task's save/flush + * path is not applicable here. Keep the core awake for short delays. + */ + if (IS_ENABLED(CONFIG_ARCH_APPLE)) { + u32 midr = read_cpuid_id() & MIDR_CPU_MODEL_MASK; + + if (midr == MIDR_APPLE_T8142_E || midr == MIDR_APPLE_T8142_P) + goto counter_delay; + } + if (alternative_has_cap_unlikely(ARM64_HAS_WFXT)) { u64 end = start + cycles; @@ -60,6 +73,7 @@ void __delay(unsigned long cycles) wfe(); } +counter_delay: while ((__delay_cycles() - start) < cycles) cpu_relax(); } From e0dc472276159769238b6f29365619101642f2da Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 12:37:37 -0400 Subject: [PATCH 005/631] arm64: dts: apple: enable bounded J813 CPU frequency control Describe both shared cluster policies and enable the userspace governor in the J813 configuration. Expose only the states up to the observed firmware handoff: E state 2 at 972 MHz and P state 10 at 3516 MHz. State 1 is a 300 MHz floor before the firmware SRAM table entries. Dependent decrement-loop timing on J813 confirmed every exposed state's frequency. Document repeatable state sweeps and qualification limits. Signed-off-by: Acelogic --- Documentation/arch/arm64/apple-j813.rst | 52 +++++++++++- arch/arm64/boot/dts/apple/t8142-j813.dts | 104 +++++++++++++++++++++++ arch/arm64/configs/j813.config | 5 ++ 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 28e13f9aafffc6..eb618f4d26588a 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -38,6 +38,54 @@ Verify affinity through ``Cpus_allowed_list`` and retain the console output, kernel configuration hash and payload hashes when repeating the test. Hotplug, deep sleep and KVM remain unqualified. +CPU frequency bring-up +====================== + +The T8142 cluster controllers use five-bit P-state requests with busy and +command-state readback. The driver preserves firmware voltage, PLL and +CLPC/PMP configuration, and stops issuing requests after a timeout or an +unexpected command state. Unlike T8140, this initial J813 path does not +register a Linux SMC thermal policy. The DT exposes only states up to the +measured firmware handoff values; higher states remain unqualified. + +State 1 is 300 MHz on both clusters. The firmware SRAM voltage tables begin +at state 2: the E cluster exposes 300/972 MHz, and the P cluster exposes +300/1308/1608/1932/2220/2508/2820/3072/3300/3516 MHz. A dependent decrement +loop timed against CNTPCT_EL0 in m1n1 confirmed this mapping on J813. +``scaling_cur_freq`` reports the requested state, not an independent clock +measurement or a guarantee against firmware throttling. + +The configuration enables the userspace governor. On a fresh ten-core boot, +capture a sweep of both policies to the host serial log:: + + set -e + for p in /sys/devices/system/cpu/cpufreq/policy*; do + cat "$p/affected_cpus" + echo userspace > "$p/scaling_governor" + for round in 1 2; do + for f in $(cat "$p/scaling_available_frequencies"); do + echo "$f" > "$p/scaling_setspeed" + test "$(cat "$p/scaling_cur_freq")" = "$f" + echo "$p $f" + sleep 0.02 + done + done + cat "$p/cpuinfo_max_freq" > "$p/scaling_setspeed" + cat "$p/stats/total_trans" + done + dmesg + +Retain the transcript and payload hashes. Qualification also compared three +pairs of pinned, checksum-verified integer workloads at each cluster's +minimum and maximum states and checked every CPU's timer and pinned SHA-256 +job afterwards. Short frequency sweeps do not qualify sustained thermal +behavior, automatic governors, suspend or states above the handoff limit. + +T8142 can also lose registers on WFIT in delay loops. Frequency transition +polling exposed this as a cleared pointer in the cpufreq driver. T8142 uses +the existing counter delay loop, avoiding low-power waits with live kernel +register state. + Bootloader requirements ======================= @@ -78,7 +126,7 @@ cross-compiling with GCC from another architecture. With dtschema installed, check the affected bindings and DTB:: make ARCH=arm64 O="$out" CHECK_DTBS=y \ - DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml \ + DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:cpufreq/apple,cluster-cpufreq.yaml:opp/opp-v2-base.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml \ apple/t8142-j813.dtb Boot and qualification @@ -124,6 +172,6 @@ an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. These tests do not establish CPU hotplug, KVM guest timers, -cpufreq, suspend, native DCP, GPU acceleration, built-in input, USB, +sustained thermal behavior, suspend, native DCP, GPU acceleration, built-in input, USB, storage, networking or audio support. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index d56cd1dd2f560c..e47839a68dfe4d 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -58,6 +58,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x0>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -66,6 +68,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x1>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -74,6 +78,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x2>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -82,6 +88,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x3>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -90,6 +98,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x4>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -98,6 +108,8 @@ compatible = "apple,t8142-e"; device_type = "cpu"; reg = <0 0x5>; + operating-points-v2 = <&ecluster_opp>; + performance-domains = <&cpufreq_e>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -106,6 +118,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10100>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -114,6 +128,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10101>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -122,6 +138,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10102>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -130,6 +148,8 @@ compatible = "apple,t8142-p"; device_type = "cpu"; reg = <0 0x10103>; + operating-points-v2 = <&pcluster_opp>; + performance-domains = <&cpufreq_p>; enable-method = "spin-table"; cpu-release-addr = <0 0>; /* Filled by the loader. */ }; @@ -144,6 +164,78 @@ interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; }; + /* State 1 is 300 MHz; firmware SRAM OPP tables begin at state 2. */ + /* Cap each cluster at its measured J813 firmware handoff state. */ + ecluster_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-972000000 { + opp-hz = /bits/ 64 <972000000>; + opp-level = <2>; + }; + }; + + pcluster_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + + opp-300000000 { + opp-hz = /bits/ 64 <300000000>; + opp-level = <1>; + }; + + opp-1308000000 { + opp-hz = /bits/ 64 <1308000000>; + opp-level = <2>; + }; + + opp-1608000000 { + opp-hz = /bits/ 64 <1608000000>; + opp-level = <3>; + }; + + opp-1932000000 { + opp-hz = /bits/ 64 <1932000000>; + opp-level = <4>; + }; + + opp-2220000000 { + opp-hz = /bits/ 64 <2220000000>; + opp-level = <5>; + }; + + opp-2508000000 { + opp-hz = /bits/ 64 <2508000000>; + opp-level = <6>; + }; + + opp-2820000000 { + opp-hz = /bits/ 64 <2820000000>; + opp-level = <7>; + }; + + opp-3072000000 { + opp-hz = /bits/ 64 <3072000000>; + opp-level = <8>; + }; + + opp-3300000000 { + opp-hz = /bits/ 64 <3300000000>; + opp-level = <9>; + }; + + opp-3516000000 { + opp-hz = /bits/ 64 <3516000000>; + opp-level = <10>; + }; + }; + clkref: clock-ref { compatible = "fixed-clock"; #clock-cells = <0>; @@ -157,6 +249,18 @@ ranges; nonposted-mmio; + cpufreq_e: performance-controller@210e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x10e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p: performance-controller@211e20000 { + compatible = "apple,t8142-cluster-cpufreq"; + reg = <0x2 0x11e20000 0 0x1000>; + #performance-domain-cells = <0>; + }; + aic: interrupt-controller@381000000 { compatible = "apple,t8142-aic3", "apple,t8122-aic3"; interrupt-controller; diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index d8143807c2e3ee..f944f153962abc 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -6,6 +6,11 @@ CONFIG_ARM64_16K_PAGES=y CONFIG_ARM64_VA_BITS_47=y CONFIG_SMP=y CONFIG_NR_CPUS=16 +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y CONFIG_ARM64_VHE=y CONFIG_ARM64_SVE=y CONFIG_ARM64_PAN=y From 6214fd6a893ca4e1e65b17715a17ab837a5bd14c Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 006/631] dt-bindings: apple: describe J813 monitoring and input peripherals Describe the T8142 mailbox, SMC, MTP, DART and PWM resources. Add explicit contracts for runtime temperature discovery and firmware-owned trackpad reset without embedding sensor tables or calibration. Signed-off-by: Acelogic --- .../bindings/arm/apple/apple,pmgr.yaml | 1 + .../bindings/hwmon/apple,smc-hwmon.yaml | 22 +++++++++++ .../bindings/input/apple,dockchannel-hid.yaml | 38 ++++++++++++++++++- .../devicetree/bindings/iommu/apple,dart.yaml | 1 + .../bindings/mailbox/apple,mailbox.yaml | 1 + .../devicetree/bindings/mfd/apple,smc.yaml | 1 + .../bindings/power/apple,pmgr-pwrstate.yaml | 1 + .../bindings/pwm/apple,s5l-fpwm.yaml | 1 + .../bindings/soc/apple/apple,dockchannel.yaml | 1 + .../soc/apple/apple,rtk-helper-asc4.yaml | 2 + 10 files changed, 67 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml index 77d87d8c376e8a..f10b9cbfce597e 100644 --- a/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml +++ b/Documentation/devicetree/bindings/arm/apple/apple,pmgr.yaml @@ -43,6 +43,7 @@ properties: - apple,t8122-pmgr - apple,t8132-pmgr - apple,t8140-pmgr + - apple,t8142-pmgr - const: apple,t8103-pmgr - const: syscon - const: simple-mfd diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 076de649627c80..0c585686f2e9b9 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -40,6 +40,18 @@ properties: compatible: const: apple,smc-hwmon + apple,discover-temperatures: + type: boolean + description: + Enumerate readable, non-function temperature keys from the running SMC + instead of providing a board-specific temperature table. Only keys + beginning with T and reporting a four-byte floating-point value are + exposed. Labels are the key identifiers, without inferred locations. + Channels are reporting-only; their indices depend on the firmware and + must not be used by a static thermal policy. + + apple,cpu-thermal-policy: + type: boolean "#thermal-sensor-cells": const: 1 description: @@ -89,4 +101,14 @@ patternProperties: $ref: "#/$defs/sensor" unevaluatedProperties: false +allOf: + - if: + required: + - apple,discover-temperatures + then: + properties: + apple,cpu-thermal-policy: false + patternProperties: + "^temperature-": false + additionalProperties: false diff --git a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml index 3b82308e0b568c..924442134b2482 100644 --- a/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +++ b/Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml @@ -23,7 +23,9 @@ description: | trackpad itself and asks the host to pulse a reset line when it needs one. With power method 2 the host announces each power transition to the coprocessor, drives the reset line of the trackpad analog front end (AFE) - itself and then confirms the transition. + itself and then confirms the transition. On systems with firmware-managed + reset, the same notifications bracket a firmware-owned transition and the + host waits 50 ms during power-on without operating a GPIO. properties: compatible: @@ -140,7 +142,27 @@ properties: default: 1 description: The power sequencing protocol of the coprocessor firmware, as - described above. Power method 2 requires apple,afe-reset-gpios. + described above. Power method 2 requires apple,afe-reset-gpios + unless apple,firmware-managed-reset is present. + + apple,firmware-managed-reset: + type: boolean + description: + The coprocessor owns the trackpad reset signal. The host sends + power method 2 notifications without controlling an AFE GPIO. + Mutually exclusive with apple,afe-reset-gpios on the parent. + + allOf: + - if: + required: + - apple,firmware-managed-reset + then: + properties: + apple,power-method: + const: 2 + required: + - apple,power-method + - firmware-name stm: type: object @@ -171,6 +193,18 @@ required: additionalProperties: false +allOf: + - if: + properties: + multi-touch: + required: + - apple,firmware-managed-reset + required: + - multi-touch + then: + properties: + apple,afe-reset-gpios: false + examples: - | #include diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 7fd21f26df6f89..5a128e38d8f6f4 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -38,6 +38,7 @@ properties: - apple,t6020-dart - apple,t8122-dart - apple,t8140-dart + - apple,t8142-dart - const: apple,t8110-dart reg: diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 32fb3acb8c99fd..4d704442a923af 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -32,6 +32,7 @@ properties: - apple,t8112-asc-mailbox - apple,t8122-asc-mailbox - apple,t8140-asc-mailbox + - apple,t8142-asc-mailbox - apple,t6000-asc-mailbox - apple,t6020-asc-mailbox - const: apple,asc-mailbox-v4 diff --git a/Documentation/devicetree/bindings/mfd/apple,smc.yaml b/Documentation/devicetree/bindings/mfd/apple,smc.yaml index ed6b02086c33ca..31d1940f9d3dc0 100644 --- a/Documentation/devicetree/bindings/mfd/apple,smc.yaml +++ b/Documentation/devicetree/bindings/mfd/apple,smc.yaml @@ -21,6 +21,7 @@ properties: - apple,t6020-smc - apple,t8122-smc - apple,t8140-smc + - apple,t8142-smc - const: apple,t8103-smc - items: - enum: diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index 0e9e1fbcd3b100..d131441214db14 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -50,6 +50,7 @@ properties: - apple,t8122-pmgr-pwrstate - apple,t8132-pmgr-pwrstate - apple,t8140-pmgr-pwrstate + - apple,t8142-pmgr-pwrstate - const: apple,t8103-pmgr-pwrstate reg: diff --git a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml index ae8baa5f7bcdc4..3b3cad92b601fa 100644 --- a/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml +++ b/Documentation/devicetree/bindings/pwm/apple,s5l-fpwm.yaml @@ -19,6 +19,7 @@ properties: - apple,t8103-fpwm - apple,t8112-fpwm - apple,t8122-fpwm + - apple,t8142-fpwm - apple,t8132-fpwm - apple,t6000-fpwm - apple,t6020-fpwm diff --git a/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml index 40831b082bf28f..a3362df34e148b 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml @@ -28,6 +28,7 @@ properties: - apple,t8112-dockchannel - apple,t8122-dockchannel - apple,t8140-dockchannel + - apple,t8142-dockchannel - const: apple,dockchannel reg: diff --git a/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml index 6db0197b4e8647..da78694e374661 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml @@ -28,12 +28,14 @@ properties: - apple,t8112-mtp - apple,t8122-mtp - apple,t8140-mtp + - apple,t8142-mtp - enum: - apple,t6020-rtk-helper-asc4 - apple,t6030-rtk-helper-asc4 - apple,t8112-rtk-helper-asc4 - apple,t8122-rtk-helper-asc4 - apple,t8140-rtk-helper-asc4 + - apple,t8142-rtk-helper-asc4 - const: apple,mtp - const: apple,rtk-helper-asc4 From aad4733f8bdcaf5a275a2e50383985da81f4ad71 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 007/631] hwmon: macsmc: discover firmware temperature keys on opt-in boards Enumerate readable float temperature keys from the running SMC and label them by key identity. Keep dynamically assigned channels separate from static thermal policy, and leave existing board sensor definitions unchanged. Signed-off-by: Acelogic --- drivers/hwmon/macsmc-hwmon.c | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index c85430ad14e66f..fcf381aa2c1947 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -685,6 +685,66 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } +/* Discover reporting-only temperatures without a board-specific key table. */ +static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) +{ + struct apple_smc *smc = hwmon->smc; + struct macsmc_hwmon_sensor *sensor; + struct apple_smc_key_info info; + smc_key key, previous = 0; + u32 i; + int ret; + + if (!of_property_read_bool(hwmon->dev->of_node, "apple,discover-temperatures")) + return 0; + + /* Dynamic channel numbers must not replace a DT-described thermal policy. */ + if (hwmon->temp.count || + of_property_read_bool(hwmon->dev->of_node, "apple,cpu-thermal-policy")) + return -EINVAL; + + /* Bound both allocation and mailbox transactions for invalid firmware data. */ + if (!smc->key_count || smc->key_count > U16_MAX) + return -EINVAL; + + hwmon->temp.sensors = devm_kcalloc(hwmon->dev, smc->key_count, + sizeof(*sensor), GFP_KERNEL); + if (!hwmon->temp.sensors) + return -ENOMEM; + + for (i = 0; i < smc->key_count; i++) { + ret = apple_smc_get_key_by_index(smc, i, &key); + if (ret < 0) + return ret; + /* SMC key tables are strictly ordered; reject duplicates as well. */ + if (i && key <= previous) + return -EINVAL; + previous = key; + if ((key >> 24) < 'T') + continue; + if ((key >> 24) > 'T') + break; + + ret = apple_smc_get_key_info(smc, key, &info); + if (ret < 0) + return ret; + if (!(info.flags & APPLE_SMC_READABLE) || + (info.flags & APPLE_SMC_FUNCTION) || + info.type_code != __SMC_KEY('f', 'l', 't', ' ') || info.size != 4) + continue; + + sensor = &hwmon->temp.sensors[hwmon->temp.count++]; + sensor->macsmc_key = key; + sensor->info = info; + sensor->attrs = HWMON_T_INPUT | HWMON_T_LABEL; + /* Key identity only: do not invent locations or calibration. */ + snprintf(sensor->label, sizeof(sensor->label), "%c%c%c%c", + key >> 24, (key >> 16) & 0xff, (key >> 8) & 0xff, key & 0xff); + } + + return 0; +} + /* * A SoC die temperature sensor: a readable float temperature key, labelled * "SoC die ". @@ -1256,6 +1316,9 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) return ret; } + ret = macsmc_hwmon_discover_temperatures(hwmon); + if (ret) + return dev_err_probe(hwmon->dev, ret, "Temperature discovery failed\n"); if (macsmc_hwmon_is_m3()) { ret = macsmc_hwmon_create_pressure(hwmon, hwmon->dev->of_node); if (ret) From 0f6c8258cb603d107ac0e3d7c7b6c4fabf9c640b Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 008/631] HID: dockchannel: support firmware-managed M5 trackpad power Support power method 2 when the MTP firmware owns reset, reject conflicting GPIO ownership, and announce AP readiness before the T8142 MTP power acknowledgment. Preserve the existing sequence for other coprocessors and host-controlled trackpads. Signed-off-by: Acelogic --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 41 +++++++++++++++---- drivers/soc/apple/rtkit-helper.c | 5 ++- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index 527c00d26c46ca..bd529afd3d4a80 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -177,6 +177,7 @@ struct dchid_iface { u32 keyboard_layout_id; u32 power_method; + bool firmware_managed_reset; }; struct dockchannel_hid { @@ -217,6 +218,7 @@ static struct dchid_iface * dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) { u32 power_method = DCHID_POWER_METHOD_1; + bool firmware_managed_reset = false; struct device_node *of_node = NULL; struct dchid_iface *iface; @@ -240,13 +242,19 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) } of_property_read_u32(of_node, "apple,power-method", &power_method); + firmware_managed_reset = + of_property_read_bool(of_node, "apple,firmware-managed-reset"); + if (firmware_managed_reset && + (strcmp(name, "multi-touch") || power_method != DCHID_POWER_METHOD_2)) + goto err_put_node; if (power_method != DCHID_POWER_METHOD_1 && power_method != DCHID_POWER_METHOD_2) { dev_err(dchid->dev, "Unsupported power method %u for %s\n", power_method, name); goto err_put_node; } - if (power_method == DCHID_POWER_METHOD_2 && !dchid->afe_reset) { + if (power_method == DCHID_POWER_METHOD_2 && + !firmware_managed_reset && !dchid->afe_reset) { dev_err(dchid->dev, "Power method 2 for %s needs the AFE reset line\n", name); goto err_put_node; @@ -264,6 +272,7 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) iface->dchid = dchid; iface->of_node = of_node; iface->power_method = power_method; + iface->firmware_managed_reset = firmware_managed_reset; iface->out_report = -1; init_completion(&iface->out_complete); init_completion(&iface->ready); @@ -425,12 +434,18 @@ static int dchid_pm2_set_power(struct dchid_iface *iface, u8 state) if (ret < 0) return ret; - ret = gpiod_set_value_cansleep(dchid->afe_reset, - state == DCHID_POWER_STATE_OFF); - if (ret < 0) - return ret; - if (state == DCHID_POWER_STATE_OFF) - fsleep(50 * USEC_PER_MSEC); + if (iface->firmware_managed_reset) { + /* Firmware owns the reset; allow its power-on transition to settle. */ + if (state == DCHID_POWER_STATE_ON) + fsleep(50 * USEC_PER_MSEC); + } else { + ret = gpiod_set_value_cansleep(dchid->afe_reset, + state == DCHID_POWER_STATE_OFF); + if (ret < 0) + return ret; + if (state == DCHID_POWER_STATE_OFF) + fsleep(50 * USEC_PER_MSEC); + } dchid_pm2_command(iface->index, state, true, msg); return dchid_comm_cmd(dchid, msg, sizeof(msg)); @@ -527,6 +542,9 @@ static int dchid_request_gpio(struct dchid_iface *iface) char prop_name[MAX_GPIO_NAME + 16]; int ret; + if (iface->firmware_managed_reset) + return -EINVAL; + if (iface->gpio) return 0; @@ -1302,10 +1320,17 @@ static int dockchannel_hid_probe(struct platform_device *pdev) child = of_get_child_by_name(dev->of_node, "multi-touch"); if (child) { u32 power_method = DCHID_POWER_METHOD_1; + bool firmware_managed_reset; of_property_read_u32(child, "apple,power-method", &power_method); + firmware_managed_reset = + of_property_read_bool(child, "apple,firmware-managed-reset"); of_node_put(child); - if (power_method == DCHID_POWER_METHOD_2) { + if (firmware_managed_reset && + (power_method != DCHID_POWER_METHOD_2 || + of_property_present(dev->of_node, "apple,afe-reset-gpios"))) + return dev_err_probe(dev, -EINVAL, "Conflicting touch reset ownership\n"); + if (power_method == DCHID_POWER_METHOD_2 && !firmware_managed_reset) { dchid->afe_reset = devm_gpiod_get(dev, "apple,afe-reset", GPIOD_ASIS); if (IS_ERR(dchid->afe_reset)) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index ef0400e07d49cd..9235a44dd9b851 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -113,7 +113,10 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->asc_base + APPLE_ASC_CPU_CONTROL); /* Works for both wake and boot */ - ret = apple_rtkit_wake(helper->rtk); + if (of_device_is_compatible(dev->of_node, "apple,t8142-mtp")) + ret = apple_rtkit_wake_early_ap(helper->rtk); + else + ret = apple_rtkit_wake(helper->rtk); if (ret != 0) return dev_err_probe(dev, ret, "Failed to wake up coprocessor"); From 43c4bbde2873c073506536d7d68c0f17ff0c37c8 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 009/631] arm64: apple: enable J813 temperatures, input and keyboard backlight Describe the MTP, SMC and PWM resources needed by the existing generic drivers. Request trackpad firmware externally, obtain keyboard layout through the loader, and use runtime sensor discovery with a linear LED brightness scale. Signed-off-by: Acelogic --- .../dts/apple/t8142-j813-peripherals.dtsi | 137 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 2 + arch/arm64/configs/j813.config | 26 ++++ 3 files changed, 165 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi diff --git a/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi new file mode 100644 index 00000000000000..a6708b1c972dfd --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-peripherals.dtsi @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include + +/ { + aliases { + mtp = &mtp; + keyboard = &keyboard; + }; + + led-controller { + compatible = "pwm-leds"; + + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&{/soc} { + pmgr: power-management@380700000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <3 0x80700000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_fpwm0: power-controller@330 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "fpwm0"; + }; + }; + + fpwm0: pwm@3a5040000 { + compatible = "apple,t8142-fpwm", "apple,s5l-fpwm"; + reg = <3 0xa5040000 0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + }; + + smc_mbox: mailbox@38c608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x8c608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + smc: smc@38c600000 { + compatible = "apple,t8142-smc", "apple,t8103-smc"; + reg = <3 0x8c600000 0 0x4000>, <3 0x8de00000 0 0x120000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + hwmon { + compatible = "apple,smc-hwmon"; + apple,discover-temperatures; + }; + }; + + mtp: mtp@394600000 { + compatible = "apple,t8142-mtp", "apple,t8142-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + /* The second region is firmware SRAM, not the ASC control window. */ + reg = <3 0x94600000 0 0x4000>, <3 0x94c00000 0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + }; + + mtp_mbox: mailbox@394608000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <3 0x94608000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + mtp_dart: iommu@394800000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <3 0x94800000 0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + /* Keep host allocations clear of firmware's low IOVA mappings. */ + apple,dma-range = <0 0x02000000 0 0x10000000>; + }; + + mtp_dockchannel: fifo@394b14000 { + compatible = "apple,t8142-dockchannel", "apple,dockchannel"; + reg = <3 0x94b14000 0 0x1000>; + reg-names = "irq"; + interrupts = ; + ranges = <0 3 0x94b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x1000>, <0xc000 0x1000>, + <0x0000 0x1000>, <0x4000 0x1000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + + stm { }; + + multi-touch { + firmware-name = "apple/tpmtfw-j813.bin"; + apple,power-method = <2>; + apple,firmware-managed-reset; + }; + + /* Layout is filled by the loader using this machine's own data. */ + keyboard: keyboard { }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index e47839a68dfe4d..b0a8ac8339955a 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -290,3 +290,5 @@ }; }; }; + +#include "t8142-j813-peripherals.dtsi" diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index f944f153962abc..13b8116af07107 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -49,3 +49,29 @@ CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_LOG_BUF_SHIFT=20 CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y +CONFIG_PM=y +CONFIG_MAILBOX=y +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_APPLE_DART=y +CONFIG_MFD_MACSMC=y +CONFIG_HWMON=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_INPUT=y +CONFIG_INPUT_EVDEV=y +CONFIG_HID_SUPPORT=y +CONFIG_HID=y +CONFIG_HID_GENERIC=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_GPIOLIB=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_PWM=y +CONFIG_PWM=y +CONFIG_PWM_APPLE=y +CONFIG_FW_LOADER=y From 4413b2a422325d2cb3abdc4dd2f9afe407f704bd Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 18:48:20 -0400 Subject: [PATCH 010/631] selftests: apple: document and exercise J813 peripherals on hardware Document the external firmware contract and qualification limits. Add an explicitly manual RAM-boot check that records temperatures, restores backlight brightness and counts physical input events without retaining key codes or pointer coordinates. Signed-off-by: Acelogic --- Documentation/arch/arm64/apple-j813.rst | 85 ++++++++++++-- MAINTAINERS | 1 + tools/testing/selftests/Makefile | 1 + .../testing/selftests/drivers/apple/Makefile | 5 + .../drivers/apple/j813-peripherals.sh | 104 ++++++++++++++++++ 5 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 tools/testing/selftests/drivers/apple/Makefile create mode 100644 tools/testing/selftests/drivers/apple/j813-peripherals.sh diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index eb618f4d26588a..69d28107a9d09a 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -5,8 +5,9 @@ Apple J813 initial native Linux boot ===================================== J813 is the 13-inch M5 MacBook Air (T8142, Mac17,3). This configuration -provides an initial RAM-only boot path: one boot CPU, AICv3, architectural -timers, DockChannel console and the framebuffer left by the boot firmware. +provides a RAM-only boot path with AICv3, architectural timers, DockChannel +console and the framebuffer left by the boot firmware. The J813 tree also +describes SMC temperature monitoring, MTP input and the keyboard backlight. The device tree describes all ten CPU affinities. For the initial one-CPU baseline, keep secondary cores stopped in m1n1 and use ``maxcpus=1``. @@ -98,8 +99,8 @@ loader is not sufficient. Before entering Linux, the loader must: * Fill the firmware framebuffer's address, size, geometry and format before enabling it. * Prepare DMA protection only for devices enabled in the supplied FDT. The - initial tree exposes no AOP, MTP, PMP or ISP DMA consumers. Their firmware - protection state must remain intact during the native FDT handoff. + ``mtp`` alias enables the MTP DAPF handoff. Leave the protection state of + unrelated AOP, PMP and ISP firmware intact. * Leave the console, interrupt controller and display powered. This minimal DT does not describe their power domains. * Give U-Boot a T8142 memory map covering its MMIO and high DRAM addresses. @@ -119,14 +120,14 @@ On an arm64 Linux build host with the usual kernel build dependencies:: KCONFIG_ALLCONFIG=arch/arm64/configs/j813.config allnoconfig make ARCH=arm64 O="$out" -j"$(nproc)" W=1 Image apple/t8142-j813.dtb -The fragment is deliberately minimal and does not enable KVM or storage -drivers. Add ``CROSS_COMPILE=aarch64-linux-gnu-`` to both commands when +The fragment does not enable KVM or storage drivers. Add +``CROSS_COMPILE=aarch64-linux-gnu-`` to both commands when cross-compiling with GCC from another architecture. With dtschema installed, check the affected bindings and DTB:: make ARCH=arm64 O="$out" CHECK_DTBS=y \ - DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:cpufreq/apple,cluster-cpufreq.yaml:opp/opp-v2-base.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml \ + DT_SCHEMA_FILES=arm/apple.yaml:arm/cpus.yaml:cpufreq/apple,cluster-cpufreq.yaml:opp/opp-v2-base.yaml:interrupt-controller/apple,aic2.yaml:serial/apple,dockchannel-uart.yaml:serial/samsung_uart.yaml:display/simple-framebuffer.yaml:apple,pmgr.yaml:apple,smc-hwmon.yaml:apple,dockchannel-hid.yaml:apple,dart.yaml:apple,mailbox.yaml:apple,smc.yaml:apple,pmgr-pwrstate.yaml:apple,s5l-fpwm.yaml:apple,dockchannel.yaml:apple,rtk-helper-asc4.yaml \ apple/t8142-j813.dtb Boot and qualification @@ -171,7 +172,71 @@ RAM boot. Preserve the complete console transcript and a hash manifest; an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. -These tests do not establish CPU hotplug, KVM guest timers, -sustained thermal behavior, suspend, native DCP, GPU acceleration, built-in input, USB, -storage, networking or audio support. The disabled Samsung UART node +These boot tests do not establish CPU hotplug, KVM guest timers, +sustained thermal behavior, suspend, native DCP, GPU acceleration, USB, +storage, networking or audio support. Input and monitoring require their +separate checks below. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. + +Temperature monitoring and built-in input +======================================== + +Temperature monitoring enumerates the running SMC's readable, non-function +``T`` keys with four-byte floating-point values. It exposes their identifiers +through ``/sys/class/hwmon/*/temp*_label`` and millidegrees Celsius through +``temp*_input``. It does not embed a sensor inventory, calibration, inferred +sensor locations, trip temperatures or a new CPU cooling policy. Channel +numbers can change with firmware; identify channels by label. These keys can +include virtual or inactive channels; their number is not a count of physical +sensors, and a readable value does not establish a sensor's location. Read failures, +NaNs and infinities remain errors, rather than fabricated temperatures. +The existing J700 thermal provider and bounded J813 frequency states are +unchanged. Temperature reporting alone does not qualify sustained load. + +MTP provides keyboard and trackpad HID descriptors and identifiers at runtime. +The ``keyboard`` alias lets m1n1 fill ``hid-country-code`` and +``apple,keyboard-layout-id`` from this machine's own boot environment. The +source tree does not assume a US keyboard or contain another machine's layout. +The J813 trackpad uses power method 2 with firmware-owned reset; the host +does not operate an AFE GPIO. Other machines retain their existing sequence. + +The trackpad requires an externally supplied +``/lib/firmware/apple/tpmtfw-j813.bin``. The existing DockChannel firmware +container has a 20-byte little-endian header: magic ``0x46444948`` (``HIDF``), +version 1, header length, data length and interface-byte offset. See +``struct fw_header`` and ``dchid_get_firmware()`` for validation. An interface +offset of zero means no patching. The payload must +be prepared from firmware and any required calibration belonging to the +user's own machine. A filename is a lookup contract, not a firmware payload. +The file is supplied from outside the kernel tree, and any tool that +produces a valid container can provide it without changing the +driver. A missing or invalid file fails trackpad startup; it does not prevent +the keyboard or temperature monitor from registering. + +The loader must preserve the iBoot-loaded MTP image and SRAM, prepare its DMA +protection, and leave MTP ready for Linux's RTKit startup. A prior-stage MTP +client must complete a compatible shutdown before Linux starts; a partially +initialized or crashed coprocessor is not a supported handoff. + +The backlight uses the existing PWM and LED drivers. Its boot brightness is +retained when the initial PWM period is valid, otherwise it starts off. +Userspace controls ``/sys/class/leds/kbd_backlight/brightness`` +on a linear 0--255 scale. No captured brightness curve or factory calibration +is included. + +Physical verification +--------------------- + +Place the user's touch firmware in a private RAM initramfs, then run:: + + sh tools/testing/selftests/drivers/apple/j813-peripherals.sh 30 + +The check reads temperatures twice, exercises and restores backlight +brightness, and counts keyboard and trackpad events during a bounded physical +interaction window. It records no key codes or pointer coordinates. Press +and release keys and move/click the built-in trackpad during that window. +Confirm visible brightness changes separately; sysfs readback is not optical +proof. Preserve stdout, the source revision and SHA-256 hashes of the exact +Image, DTB, configuration, initramfs and bootloaders as the repeatable result. +Keep firmware and raw machine data private. Without physical events the input +check fails rather than crediting device registration as working input. diff --git a/MAINTAINERS b/MAINTAINERS index 1103f335b26c74..d9454a3d24a2f6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2643,6 +2643,7 @@ F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* +F: tools/testing/selftests/drivers/apple/ F: include/uapi/drm/asahi_drm.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 641a180fb35f41..ac5717ad1078c9 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -17,6 +17,7 @@ TARGETS += damon TARGETS += devices/error_logs TARGETS += devices/probe TARGETS += dmabuf-heaps +TARGETS += drivers/apple TARGETS += drivers/dma-buf TARGETS += drivers/ntsync TARGETS += drivers/s390x/uvdevice diff --git a/tools/testing/selftests/drivers/apple/Makefile b/tools/testing/selftests/drivers/apple/Makefile new file mode 100644 index 00000000000000..76bca086dde7a3 --- /dev/null +++ b/tools/testing/selftests/drivers/apple/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +# Manual physical test: requires user interaction and changes LED brightness. +TEST_FILES := j813-peripherals.sh + +include ../../lib.mk diff --git a/tools/testing/selftests/drivers/apple/j813-peripherals.sh b/tools/testing/selftests/drivers/apple/j813-peripherals.sh new file mode 100644 index 00000000000000..ecad597fe9113e --- /dev/null +++ b/tools/testing/selftests/drivers/apple/j813-peripherals.sh @@ -0,0 +1,104 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# Physical J813 check. Save stdout and the payload SHA-256 manifest together. +# Usage: sh j813-peripherals.sh [event-capture-seconds] +set -eu + +duration=${1:-30} +case "$duration" in *[!0-9]*|'') exit 2;; esac +[ "$duration" -ge 1 ] && [ "$duration" -le 600 ] || exit 2 +[ -r /proc/device-tree/compatible ] || exit 4 +tr '\000' '\n' < /proc/device-tree/compatible | grep -qx apple,j813 || exit 4 +[ "$(id -u)" -eq 0 ] || { echo 'SKIP: run as root'; exit 4; } + +work=$(mktemp -d) +led=/sys/class/leds/kbd_backlight +saved_brightness= +cleanup() +{ + if [ -n "$saved_brightness" ]; then + echo "$saved_brightness" > "$led/brightness" + fi + rm -rf "$work" +} +trap cleanup EXIT +trap 'exit 1' INT TERM + +echo J813_PERIPHERALS_BEGIN +uname -r +zcat /proc/config.gz | sha256sum +cat /sys/devices/system/cpu/online + +hwmon= +for h in /sys/class/hwmon/hwmon*; do + [ -r "$h/name" ] || continue + [ "$(cat "$h/name")" = macsmc_hwmon ] || continue + hwmon=$h +done +[ -n "$hwmon" ] || { echo 'FAIL: no SMC hwmon'; exit 1; } +for sample in 1 2; do + count=0 + for input in "$hwmon"/temp*_input; do + [ -r "$input" ] || continue + value=$(cat "$input") + label=$(cat "${input%_input}_label") + [ "$value" -ge -40000 ] && [ "$value" -le 125000 ] || exit 1 + printf 'temperature sample=%s key=%s millidegrees=%s\n' "$sample" "$label" "$value" + count=$((count + 1)) + done + [ "$count" -gt 0 ] || { echo 'FAIL: no temperature channels'; exit 1; } + sleep 1 +done +echo "PASS: $count readable SMC temperature channels" + +[ -r "$led/brightness" ] || { echo 'FAIL: no keyboard backlight'; exit 1; } +saved_brightness=$(cat "$led/brightness") +maximum=$(cat "$led/max_brightness") +for level in 0 $((maximum / 2)) "$maximum"; do + echo "$level" > "$led/brightness" + [ "$(cat "$led/brightness")" = "$level" ] || exit 1 + printf 'backlight requested=%s readback=%s\n' "$level" "$(cat "$led/brightness")" + sleep 5 +done +echo "$saved_brightness" > "$led/brightness" +echo 'PASS: backlight control and restore (visual confirmation is separate)' + +keyboard= +trackpad= +for e in /sys/class/input/event*; do + [ -r "$e/device/name" ] || continue + case "$(cat "$e/device/name")" in + 'Apple MTP keyboard') keyboard=/dev/input/${e##*/} ;; + 'Apple MTP multi-touch') trackpad=/dev/input/${e##*/} ;; + esac +done +[ -n "$keyboard" ] && [ -n "$trackpad" ] || { + echo 'FAIL: keyboard or trackpad missing'; exit 1; +} +echo "Move/click the built-in trackpad and press/release keys for $duration seconds." +# Count bytes only. Do not retain key codes, pointer coordinates, or raw events. +timeout "$duration" cat "$keyboard" | wc -c > "$work/keyboard" & +kp=$! +timeout "$duration" cat "$trackpad" | wc -c > "$work/trackpad" & +tp=$! +wait "$kp" +wait "$tp" +failed=0 +for name in keyboard trackpad; do + bytes=$(cat "$work/$name") + [ "$bytes" -gt 0 ] && [ $((bytes % 24)) -eq 0 ] || { + echo "FAIL: $name produced no complete arm64 input events" + failed=1 + continue + } + printf 'PASS: %s events=%s\n' "$name" "$((bytes / 24))" +done +dmesg > "$work/dmesg" +fault_pattern='Kernel panic|Oops:|(^|[[:space:]])SError([[:space:]:]|$)' +fault_pattern="$fault_pattern|(^|[[:space:]])BUG:|apple-dart .*:.*(fault|error)" +if grep -E "$fault_pattern" "$work/dmesg"; then + echo 'FAIL: kernel fault detected' + exit 1 +fi +[ "$failed" -eq 0 ] || exit 1 +echo J813_PERIPHERALS_PASS From d7a3ffe0f7d93834b3128cdf6c25e4f38c4bbfff Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:03:28 -0400 Subject: [PATCH 011/631] usb: add verified M5 J813 USB3 host support --- .../devicetree/bindings/phy/apple,atcphy.yaml | 1 + .../devicetree/bindings/spmi/apple,spmi.yaml | 1 + .../devicetree/bindings/usb/apple,dwc3.yaml | 1 + .../bindings/usb/apple,sn201202x.yaml | 8 +- arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi | 323 ++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + arch/arm64/configs/j813.config | 26 ++ drivers/phy/apple/atc.c | 38 ++- drivers/spmi/spmi-apple-controller.c | 62 +++- drivers/usb/typec/tipd/core.c | 6 +- drivers/usb/typec/tipd/spmi.c | 109 ++++-- drivers/usb/typec/tipd/tps6598x.h | 2 + 12 files changed, 517 insertions(+), 61 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index c42b9c56842967..24841218704009 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -78,6 +78,7 @@ properties: - const: apple,t8103-atcphy - const: apple,t8103-atcphy - const: apple,t8140-atcphy + - const: apple,t8142-atcphy reg: minItems: 5 diff --git a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml index f0f7d21d031cab..5a52f808b6f5b8 100644 --- a/Documentation/devicetree/bindings/spmi/apple,spmi.yaml +++ b/Documentation/devicetree/bindings/spmi/apple,spmi.yaml @@ -34,6 +34,7 @@ properties: - apple,t6000-spmi - apple,t8112-spmi - const: apple,spmi + - const: apple,t8142-spmi reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml index 44e8b33576ddac..9dc0adc535bea1 100644 --- a/Documentation/devicetree/bindings/usb/apple,dwc3.yaml +++ b/Documentation/devicetree/bindings/usb/apple,dwc3.yaml @@ -25,6 +25,7 @@ properties: - apple,t6020-dwc3 - apple,t8112-dwc3 - apple,t8140-dwc3 + - apple,t8142-dwc3 - const: apple,t8103-dwc3 - const: apple,t8103-dwc3 diff --git a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml index 133a71bbedb7cd..9b1a566fd14373 100644 --- a/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml +++ b/Documentation/devicetree/bindings/usb/apple,sn201202x.yaml @@ -15,6 +15,8 @@ description: | A variant of tps6598x controller also known as Apple ACE3 present on hardware with Apple SoCs starting from the M3. + The controller can be polled when interrupt resources are not supplied. + properties: compatible: enum: @@ -43,8 +45,10 @@ properties: required: - compatible - reg - - interrupts - - interrupt-names + +dependentRequired: + interrupts: [interrupt-names] + interrupt-names: [interrupts] additionalProperties: false diff --git a/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi new file mode 100644 index 00000000000000..e27e40c04e268d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-usb.dtsi @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +#include +#include + +/ { + aliases { + atcphy0 = &atcphy0; + usb0 = &dwc3_0; + atcphy1 = &atcphy1; + usb1 = &dwc3_1; + }; +}; + +&pmgr { + ps_atc0_common: power-controller@2c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-common"; + }; +}; + +&{/soc} { + pmgr_nub: power-management@388280000 { + compatible = "apple,t8142-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <3 0x88280000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + + ps_nub_spmi_a0: power-controller@70 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "nub-spmi-a0"; + }; + + ps_atc0_usb_aon: power-controller@c8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc0-usb-aon"; + }; + + ps_atc0_usb: power-controller@d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc0_common>, <&ps_atc0_usb_aon>; + label = "atc0-usb"; + }; + }; + + nub_spmi_a0: spmi@388908000 { + compatible = "apple,t8142-spmi"; + reg = <3 0x88908000 0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a0>; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-front"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec0_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + + port@1 { + reg = <1>; + typec0_ss: endpoint { + remote-endpoint = <&atcphy0_typec>; + }; + }; + }; + }; + }; + }; + + dwc3_0: usb@402280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x02280000 0 0xcd00>, <4 0x0228cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb>; + }; + }; + }; + }; + + dwc3_0_dart_0: iommu@402f00000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x02f00000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_0_dart_1: iommu@402f80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x02f80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy0: phy@403000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x03000000 0 0x4c000>, + <4 0x03050000 0 0x8000>, + <4 0x00000000 0 0x8000>, + <4 0x02a90000 0 0x4000>, + <4 0x02a84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy0_typec: endpoint { + remote-endpoint = <&typec0_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy0_usb: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; + }; +}; + +&pmgr { + ps_atc1_common: power-controller@2d0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-common"; + }; +}; + +&pmgr_nub { + ps_atc1_usb_aon: power-controller@d8 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd8 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + label = "atc1-usb-aon"; + }; + + ps_atc1_usb: power-controller@e0 { + compatible = "apple,t8142-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_atc1_common>, <&ps_atc1_usb_aon>; + label = "atc1-usb"; + }; +}; + +&nub_spmi_a0 { + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + + typec1: connector { + compatible = "usb-c-connector"; + label = "USB-C Left-rear"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec1_hs: endpoint { + remote-endpoint = <&dwc3_1_hs>; + }; + }; + + port@1 { + reg = <1>; + typec1_ss: endpoint { + remote-endpoint = <&atcphy1_typec>; + }; + }; + }; + }; + }; +}; + +&{/soc} { + dwc3_1: usb@40a280000 { + compatible = "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <4 0x0a280000 0 0xcd00>, <4 0x0a28cd00 0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + resets = <&atcphy1>; + phys = <&atcphy1 PHY_TYPE_USB2>, <&atcphy1 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_1_hs: endpoint { + remote-endpoint = <&typec1_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_1_ss: endpoint { + remote-endpoint = <&atcphy1_usb>; + }; + }; + }; + }; + + dwc3_1_dart_0: iommu@40af00000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x0af00000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_1_dart_1: iommu@40af80000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <4 0x0af80000 0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy1: phy@40b000000 { + compatible = "apple,t8142-atcphy"; + reg = <4 0x0b000000 0 0x4c000>, + <4 0x0b050000 0 0x8000>, + <4 0x08000000 0 0x8000>, + <4 0x0aa90000 0 0x4000>, + <4 0x0aa84000 0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc1_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy1_typec: endpoint { + remote-endpoint = <&typec1_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy1_usb: endpoint { + remote-endpoint = <&dwc3_1_ss>; + }; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index b0a8ac8339955a..be04fd061a8854 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -292,3 +292,4 @@ }; #include "t8142-j813-peripherals.dtsi" +#include "t8142-j813-usb.dtsi" diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index 13b8116af07107..57fb42b0295c51 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -75,3 +75,29 @@ CONFIG_LEDS_PWM=y CONFIG_PWM=y CONFIG_PWM_APPLE=y CONFIG_FW_LOADER=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_SPMI=y +CONFIG_SPMI_APPLE=y +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PLATFORM=y +CONFIG_USB_DWC3=y +CONFIG_USB_DWC3_HOST=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_USB_HID=y +CONFIG_BLOCK=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_USB_STORAGE=y +CONFIG_USB_UAS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_EFI_PARTITION=y +CONFIG_MSDOS_PARTITION=y +CONFIG_EXFAT_FS=y +CONFIG_NLS_UTF8=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_DEBUG_FS=y diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 4211e4baabb8dd..a2e4e45aaaf6f2 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -488,8 +488,10 @@ enum atcphy_lane_mode { #define PIPEHANDLER_OVERRIDE_VAL_PHY_STATUS BIT(4) #define PIPEHANDLER_MUX_CTRL 0x0c -#define PIPEHANDLER_MUX_CTRL_CLK GENMASK(5, 3) -#define PIPEHANDLER_MUX_CTRL_DATA GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_T8103 GENMASK(5, 3) +#define PIPEHANDLER_MUX_CTRL_DATA_T8103 GENMASK(2, 0) +#define PIPEHANDLER_MUX_CTRL_CLK_T8142 GENMASK(6, 4) +#define PIPEHANDLER_MUX_CTRL_DATA_T8142 GENMASK(1, 0) #define PIPEHANDLER_MUX_CTRL_CLK_OFF 0 #define PIPEHANDLER_MUX_CTRL_CLK_USB3 1 #define PIPEHANDLER_MUX_CTRL_CLK_USB4 2 @@ -621,6 +623,8 @@ struct atcphy_mode_configuration { * @aciophy_lane_mode: Lane mode register offset * @aciophy_crossbar: Crossbar register offset * @has_usb4: A USB4/Thunderbolt controller sits behind the PHY + * @pipe_mux_clk: PIPE clock selection field mask + * @pipe_mux_data: PIPE data selection field mask * @has_usb2phy_reg: The PHY has the secondary eUSB2 register bank (T8140) * @optional_tunables: The bootloader may leave out the common-a tunables (this * generation has none) and the SuperSpeed tunables; USB2 @@ -640,6 +644,8 @@ struct atcphy_hw { int aciophy_lane_mode; int aciophy_crossbar; bool has_usb4; + u32 pipe_mux_clk; + u32 pipe_mux_data; bool has_usb2phy_reg; bool optional_tunables; bool dp_t8122; @@ -1138,14 +1144,14 @@ static int atcphy_pipehandler_check(struct apple_atcphy *atcphy) static void atcphy_pipehandler_set_mux(struct apple_atcphy *atcphy, u32 data, u32 clk) { - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, PIPEHANDLER_MUX_CTRL_CLK_OFF)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_clk, + field_prep(atcphy->hw->pipe_mux_clk, PIPEHANDLER_MUX_CTRL_CLK_OFF)); udelay(10); - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_DATA, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_DATA, data)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_data, + field_prep(atcphy->hw->pipe_mux_data, data)); udelay(10); - mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CTRL_CLK, - FIELD_PREP(PIPEHANDLER_MUX_CTRL_CLK, clk)); + mask32(atcphy->regs.pipehandler + PIPEHANDLER_MUX_CTRL, atcphy->hw->pipe_mux_clk, + field_prep(atcphy->hw->pipe_mux_clk, clk)); udelay(10); } @@ -4217,6 +4223,8 @@ static const struct atcphy_hw atcphy_hw_t8103 = { .gen = ATCPHY_GENERATION_T8103, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8103, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8103, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, }; @@ -4224,6 +4232,8 @@ static const struct atcphy_hw atcphy_hw_t8122 = { .gen = ATCPHY_GENERATION_T8122, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb4 = true, .dp_t8122 = true, .park_pipe_unlocked = true, @@ -4235,13 +4245,25 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .gen = ATCPHY_GENERATION_T8122, .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb2phy_reg = true, .optional_tunables = true, }; +static const struct atcphy_hw atcphy_hw_t8142 = { + .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, + .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8142, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8142, + .has_usb4 = true, +}; + static const struct of_device_id atcphy_match[] = { { .compatible = "apple,t8103-atcphy", .data = &atcphy_hw_t8103 }, { .compatible = "apple,t8122-atcphy", .data = &atcphy_hw_t8122 }, + { .compatible = "apple,t8142-atcphy", .data = &atcphy_hw_t8142 }, { .compatible = "apple,t8140-atcphy", .data = &atcphy_hw_t8140 }, {}, }; diff --git a/drivers/spmi/spmi-apple-controller.c b/drivers/spmi/spmi-apple-controller.c index ec49d412a4b0e8..d31bcf33a8a359 100644 --- a/drivers/spmi/spmi-apple-controller.c +++ b/drivers/spmi/spmi-apple-controller.c @@ -52,8 +52,32 @@ #define REG_POLL_INTERVAL_US 10000 #define REG_POLL_TIMEOUT_US (REG_POLL_INTERVAL_US * 5) +struct apple_spmi_hw { + u32 status; + u32 command; + u32 reply; + u32 rx_empty; + bool legacy_irqs; +}; + +static const struct apple_spmi_hw apple_spmi_gen1 = { + .status = SPMI_STATUS_REG, + .command = SPMI_CMD_REG, + .reply = SPMI_RSP_REG, + .rx_empty = SPMI_RX_FIFO_EMPTY, + .legacy_irqs = true, +}; + +static const struct apple_spmi_hw apple_spmi_gen4 = { + .status = 0x200, + .command = 0x210, + .reply = 0x220, + .rx_empty = BIT(30), +}; + struct apple_spmi { void __iomem *regs; + const struct apple_spmi_hw *hw; struct mutex fifo_lock; struct completion fifo_rx; struct irq_domain *irqd; @@ -64,10 +88,6 @@ struct apple_spmi { bool prev_fail; }; -#define poll_reg(spmi, reg, val, cond) \ - readl_poll_timeout((spmi)->regs + (reg), (val), (cond), \ - REG_POLL_INTERVAL_US, REG_POLL_TIMEOUT_US) - static void apple_spmi_irq_ack_raw(struct apple_spmi *spmi, u32 irq) { u32 __iomem *reg = spmi->regs + SPMI_IRQ_ACK_BASE + (irq / 32) * 4; @@ -135,12 +155,14 @@ static int apple_spmi_wait_rx_not_empty(struct spmi_controller *ctrl) usecs_to_jiffies(REG_POLL_TIMEOUT_US)); if (!ret) ret = -ETIMEDOUT; - else if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) + else if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) ret = -EIO; else ret = 0; } else { - ret = poll_reg(spmi, SPMI_STATUS_REG, status, !(status & SPMI_RX_FIFO_EMPTY)); + ret = readl_poll_timeout(spmi->regs + spmi->hw->status, status, + !(status & spmi->hw->rx_empty), 10, + REG_POLL_TIMEOUT_US); } if (ret) { @@ -166,19 +188,22 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, guard(mutex)(&spmi->fifo_lock); if (spmi->prev_fail) { - writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + SPMI_RSP_REG); + /* Do not consume a late reply as the reply to a new command. */ + if (!spmi->hw->legacy_irqs) + return -EIO; + writel(SPMI_ACT_FIFO_FLUSH, spmi->regs + SPMI_ACT_REG); apple_spmi_irq_ack_raw(spmi, SPMI_IRQ_NOTIFY); spmi->prev_fail = false; } reinit_completion(&spmi->fifo_rx); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->command); while (i < len) { j = min_t(size_t, sizeof(spmi_cmd), len - i); spmi_cmd = 0; memcpy(&spmi_cmd, buf + i, j); - writel(spmi_cmd, spmi->regs + SPMI_CMD_REG); + writel(spmi_cmd, spmi->regs + spmi->hw->command); i += j; } @@ -186,23 +211,23 @@ static int spmi_raw_cmd(struct spmi_controller *ctrl, u8 opc, u8 sid, if (ret) return ret; - reply = readl(spmi->regs + SPMI_RSP_REG); + reply = readl(spmi->regs + spmi->hw->reply); /* Read SPMI data reply */ while (len_read < ilen) { - if (readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY) { + if (readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty) { spmi->prev_fail = true; dev_err_ratelimited(&ctrl->dev, "FIFO lacks reply data, controller stuck?\n"); return -EIO; } - rsp = readl(spmi->regs + SPMI_RSP_REG); + rsp = readl(spmi->regs + spmi->hw->reply); i = min_t(size_t, sizeof(spmi_cmd), ilen - len_read); memcpy(ibuf + len_read, &rsp, i); len_read += i; } - if (!(readl(spmi->regs + SPMI_STATUS_REG) & SPMI_RX_FIFO_EMPTY)) { + if (!(readl(spmi->regs + spmi->hw->status) & spmi->hw->rx_empty)) { dev_warn(&ctrl->dev, "FIFO has extra data\n"); spmi->prev_fail = true; } @@ -441,6 +466,9 @@ static int apple_spmi_probe(struct platform_device *pdev) return -ENOMEM; spmi = spmi_controller_get_drvdata(ctrl); + spmi->hw = device_get_match_data(&pdev->dev); + if (!spmi->hw) + return -EINVAL; mutex_init(&spmi->fifo_lock); init_completion(&spmi->fifo_rx); platform_set_drvdata(pdev, spmi); @@ -459,6 +487,9 @@ static int apple_spmi_probe(struct platform_device *pdev) if (spmi->irq < 0 && spmi->irq != -ENXIO) return spmi->irq; if (spmi->irq >= 0) { + if (!spmi->hw->legacy_irqs) + return dev_err_probe(&pdev->dev, -EOPNOTSUPP, + "Generation 4 interrupt layout is not supported\n"); ret = apple_spmi_init_irq(pdev, spmi, spmi->irq); if (ret) return ret; @@ -473,8 +504,9 @@ static int apple_spmi_probe(struct platform_device *pdev) } static const struct of_device_id apple_spmi_match_table[] = { - { .compatible = "apple,t8103-spmi", }, - { .compatible = "apple,spmi", }, + { .compatible = "apple,t8142-spmi", .data = &apple_spmi_gen4 }, + { .compatible = "apple,t8103-spmi", .data = &apple_spmi_gen1 }, + { .compatible = "apple,spmi", .data = &apple_spmi_gen1 }, {} }; MODULE_DEVICE_TABLE(of, apple_spmi_match_table); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index b228a7b363a084..41991349be5828 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -29,7 +29,6 @@ /* Register offsets */ #define TPS_REG_VID 0x00 -#define TPS_REG_MODE 0x03 #define TPS_REG_CMD1 0x08 #define TPS_REG_DATA1 0x09 #define TPS_REG_VERSION 0x0F @@ -938,7 +937,7 @@ static void cd321x_update_work(struct work_struct *work) * request is rejected with -EBUSY; treating that sink as the connector * owner would tear down the display carried by the first port. */ - if (cd321x->connector_fwnode && + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && dp_route_was_active && (!new_connected || was_disconnected || !dp_connected || !dp_hpd || dp_hpd_changed || dp_mode_changed)) { @@ -1033,7 +1032,8 @@ static void cd321x_update_work(struct work_struct *work) if (ret) goto retry; - if (cd321x->connector_fwnode && cd321x->display_route_active && dp_hpd) + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + cd321x->display_route_active && dp_hpd) drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_connected); power_supply_changed(tps->psy); diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index dc57ef88ca7fe6..3ff9f602c23b6d 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -24,6 +25,15 @@ static int regmap_sn201202x_select_reg(struct spmi_device *sdev, u8 reg) if (err) return err; + if (sn->polled) { + int poll_err; + + poll_err = read_poll_timeout(spmi_register_read, err, + err || val == reg, 100, 100000, + false, sdev, 0, &val); + return err ?: poll_err; + } + if (!wait_for_completion_timeout(&sn->select_completion, msecs_to_jiffies(100))) return -ETIMEDOUT; @@ -157,6 +167,39 @@ static const struct of_device_id sn201202x_of_match[] = { {} }; +static int sn201202x_wake(struct sn201202x *sn) +{ + unsigned long deadline; + u8 size; + int ret; + + reinit_completion(&sn->wake_completion); + ret = spmi_command_wakeup(sn->sdev); + if (ret) + return ret; + if (!sn->polled) + return wait_for_completion_timeout(&sn->wake_completion, + msecs_to_jiffies(100)) ? 0 : -ETIMEDOUT; + + /* A sleeping ACE3 ACKs commands but returns zeros until it is awake. */ + deadline = jiffies + msecs_to_jiffies(500); + do { + ret = regmap_sn201202x_select_reg(sn->sdev, TPS_REG_MODE); + if (ret == -ETIMEDOUT) + continue; + if (ret) + return ret; + ret = spmi_register_read(sn->sdev, 0x1f, &size); + if (ret) + return ret; + if (size) + return 0; + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + + return -ETIMEDOUT; +} + static int sn201202x_probe(struct spmi_device *device) { const struct of_device_id *match; @@ -181,46 +224,48 @@ static int sn201202x_probe(struct spmi_device *device) tps->dev = &device->dev; tps->data = data; - tps->irq = of_irq_get_byname(device->dev.of_node, "irq"); - if (tps->irq < 0) - return tps->irq; - irq_select = of_irq_get_byname(device->dev.of_node, "select"); - if (irq_select < 0) - return irq_select; - irq_sleep = of_irq_get_byname(device->dev.of_node, "sleep"); - if (irq_sleep < 0) - return irq_sleep; - irq_wake = of_irq_get_byname(device->dev.of_node, "wake"); - if (irq_wake < 0) - return irq_wake; - init_completion(&sn->select_completion); init_completion(&sn->sleep_completion); init_completion(&sn->wake_completion); - ret = devm_request_irq(&device->dev, irq_select, sn201202x_irq, - 0, NULL, &sn->select_completion); - if (ret) - return ret; - ret = devm_request_irq(&device->dev, irq_sleep, sn201202x_irq, - 0, NULL, &sn->sleep_completion); - if (ret) - return ret; - ret = devm_request_irq(&device->dev, irq_wake, sn201202x_irq, - 0, NULL, &sn->wake_completion); - if (ret) - return ret; + sn->polled = !of_property_present(device->dev.of_node, "interrupts") && + !of_property_present(device->dev.of_node, "interrupts-extended"); + if (!sn->polled) { + tps->irq = of_irq_get_byname(device->dev.of_node, "irq"); + if (tps->irq < 0) + return tps->irq; + irq_select = of_irq_get_byname(device->dev.of_node, "select"); + if (irq_select < 0) + return irq_select; + irq_sleep = of_irq_get_byname(device->dev.of_node, "sleep"); + if (irq_sleep < 0) + return irq_sleep; + irq_wake = of_irq_get_byname(device->dev.of_node, "wake"); + if (irq_wake < 0) + return irq_wake; + + ret = devm_request_irq(&device->dev, irq_select, sn201202x_irq, + 0, NULL, &sn->select_completion); + if (ret) + return ret; + ret = devm_request_irq(&device->dev, irq_sleep, sn201202x_irq, + 0, NULL, &sn->sleep_completion); + if (ret) + return ret; + ret = devm_request_irq(&device->dev, irq_wake, sn201202x_irq, + 0, NULL, &sn->wake_completion); + if (ret) + return ret; + } spmi_device_set_drvdata(device, tps); tps->regmap = devm_regmap_init_sn201202x(device, &tps6598x_regmap_config); if (IS_ERR(tps->regmap)) return PTR_ERR(tps->regmap); - ret = spmi_command_wakeup(device); + ret = sn201202x_wake(sn); if (ret) return ret; - if (!wait_for_completion_timeout(&sn->wake_completion, msecs_to_jiffies(100))) - return -ETIMEDOUT; ret = tipd_init(tps); if (ret) @@ -243,12 +288,9 @@ static int __maybe_unused sn201202x_resume(struct device *dev) struct sn201202x *sn = tps_to_sn(tps); int err; - reinit_completion(&sn->wake_completion); - err = spmi_command_wakeup(sn->sdev); + err = sn201202x_wake(sn); if (err) return err; - if (!wait_for_completion_timeout(&sn->wake_completion, msecs_to_jiffies(100))) - return -ETIMEDOUT; return tipd_resume(tps); } @@ -265,7 +307,8 @@ static int __maybe_unused sn201202x_suspend(struct device *dev) err = spmi_command_sleep(sn->sdev); if (err) goto out_resume; - if (!wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) { + if (!sn->polled && + !wait_for_completion_timeout(&sn->sleep_completion, msecs_to_jiffies(100))) { err = -ETIMEDOUT; goto out_resume; } diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 1239cd971f2499..50286e9d7ebcb3 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -19,6 +19,7 @@ #define __TPS6598X_H__ #include "cd321x-pm.h" +#define TPS_REG_MODE 0x03 #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) @@ -411,6 +412,7 @@ struct cd321x { struct sn201202x { struct cd321x cd; + bool polled; struct completion select_completion; struct completion sleep_completion; struct completion wake_completion; From 8675f6b3528b5377a2db663c28e47a53ab171cae Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:09:10 -0400 Subject: [PATCH 012/631] dt-bindings: thunderbolt: describe T8142 Type 7 NHI and CIO reset --- .../bindings/reset/apple,t8103-cio-reset.yaml | 6 ++++- .../thunderbolt/apple,t8103-usb4-acio.yaml | 1 + .../thunderbolt/apple,t8103-usb4-nhi.yaml | 24 ++++++++++++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml index 5f5be0636b369b..789e55fdaf384d 100644 --- a/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml +++ b/Documentation/devicetree/bindings/reset/apple,t8103-cio-reset.yaml @@ -19,13 +19,17 @@ description: | the PMGR MMIO region it is represented as a sub-node of the PMGR syscon with reg used as an offset into it. - In both cases the reset id is the CIO port index on the die. + On t8142, a standalone register page contains a shared request register + with alternating request and busy bits for four ports. + + In all cases the reset id is the CIO port index on the die. properties: compatible: oneOf: - enum: - apple,t8103-cio-reset + - apple,t8142-cio-reset - apple,t6000-cio-reset - items: - const: apple,t8112-cio-reset diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml index 7787adb6805ccb..33960878649c21 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-acio.yaml @@ -29,6 +29,7 @@ properties: compatible: oneOf: - const: apple,t8103-usb4-acio + - const: apple,t8142-usb4-acio - items: - enum: - apple,t6000-usb4-acio diff --git a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml index 4f95132cebf303..49b60aaa8deafb 100644 --- a/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml +++ b/Documentation/devicetree/bindings/thunderbolt/apple,t8103-usb4-nhi.yaml @@ -21,6 +21,7 @@ properties: compatible: oneOf: - const: apple,t8103-usb4-nhi + - const: apple,t8142-usb4-nhi - items: - enum: - apple,t6000-usb4-nhi @@ -32,6 +33,7 @@ properties: - const: apple,t8103-usb4-nhi reg: + minItems: 1 items: - description: NHI registers with the transmit and receive rings - description: @@ -39,6 +41,7 @@ properties: accepted by each receive ring reg-names: + minItems: 1 items: - const: nhi - const: pdf @@ -85,7 +88,8 @@ properties: all other rings which are only used for XDomain connections to other hosts. - T6030, T6031 and T8122 use 24 entries. These are most likely one DART + T8142 uses one DART stream per ring: TX streams 0-11 and RX streams 16-27. + T6030, T6031 and T8122 also use 24 entries. These are most likely one DART stream for each of the 12 transmit and 12 receive rings. apple,thunderbolt-drom: @@ -115,6 +119,23 @@ required: - apple,thunderbolt-drom allOf: + - if: + properties: + compatible: + contains: + const: apple,t8142-usb4-nhi + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + else: + properties: + reg: + minItems: 2 + reg-names: + minItems: 2 - if: properties: compatible: @@ -123,6 +144,7 @@ allOf: - apple,t6030-usb4-nhi - apple,t6031-usb4-nhi - apple,t8122-usb4-nhi + - apple,t8142-usb4-nhi then: properties: iommus: From 68069f3bec2d06873ee0f360c7e47405c138cc9f Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 21:09:11 -0400 Subject: [PATCH 013/631] thunderbolt: prepare M5 Type 7 controller support Handle the T8142 reset request layout, firmware ready state, and receive-ring PDF masks without the older external PDF aperture. Require loader tuning and matching ACIO/NHI generations. Add an opt-in build fragment; controller DT enablement and firmware startup remain separate qualification work. --- arch/arm64/configs/j813-usb4.config | 7 +++++ drivers/reset/reset-apple-cio.c | 46 +++++++++++++++++++++++++++-- drivers/thunderbolt/apple.c | 42 +++++++++++++++++++++----- 3 files changed, 86 insertions(+), 9 deletions(-) create mode 100644 arch/arm64/configs/j813-usb4.config diff --git a/arch/arm64/configs/j813-usb4.config b/arch/arm64/configs/j813-usb4.config new file mode 100644 index 00000000000000..d754400c6eccd5 --- /dev/null +++ b/arch/arm64/configs/j813-usb4.config @@ -0,0 +1,7 @@ +# Additional build coverage for M5 USB4 controller development. +# Merge with j813.config. This does not enable USB4 device-tree nodes. +CONFIG_PCI=y +CONFIG_PCIE_APPLE=y +CONFIG_USB4=y +CONFIG_USB4_APPLE_SOC=y +CONFIG_RESET_APPLE_CIO=y diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 5e818246ef32f2..252d57cd59bb1b 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -18,6 +18,7 @@ * t8103 0 0x84000 + 0x4000 * port BIT(0), rmw BIT(1) * t8112 0x63 0x8000 / 0x4000 BIT(0), rmw BIT(1) * t8122 0x62 0x8 BIT(slot) BIT(16 + slot) + * t8142 0x62 0x30 BIT(2 * port) BIT(2 * port + 1) * t6000 0 0x1a034 BIT(port) BIT(16 + port) * t6020 0 0xa02c BIT(port) BIT(16 + port) * t6030 0x4c 0x8 BIT(slot) BIT(16 + slot) @@ -38,8 +39,9 @@ * * t8132 and t8142 use a different, per-port layout and are not supported. * - * Every implementation polls the busy bits, stores the request, then polls - * again, with a 192 ms budget. The first poll's result is discarded. + * Reference firmware polls the busy bits, stores the request, then polls + * again, with a 192 ms budget. It discards the first poll's result. The + * t8142 implementation below instead refuses to issue a request while busy. */ #include @@ -62,6 +64,11 @@ #define T6000_CIO_CTRL_INIT_REQ(port) BIT(port) #define T6000_CIO_CTRL_INIT_BUSY(port) BIT(16 + (port)) +#define T8142_CIO_CTRL 0x30 +#define T8142_CIO_CTRL_INIT_REQ(port) BIT(2 * (port)) +#define T8142_CIO_CTRL_INIT_BUSY(port) BIT(2 * (port) + 1) +#define T8142_CIO_CTRL_BUSY_MASK 0xaa + struct apple_cio_reset; struct apple_cio_reset_variant { @@ -143,6 +150,37 @@ static int t6000_cio_deassert(struct apple_cio_reset *priv, unsigned long id) APPLE_CIO_RESET_TIMEOUT_US); } +static int t8142_cio_deassert(struct apple_cio_reset *priv, unsigned long id) +{ + u32 val; + int ret; + + guard(mutex)(&priv->lock); + + /* Serialize all four ports sharing the request register. */ + ret = regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + !(val & T8142_CIO_CTRL_BUSY_MASK), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); + if (ret) + return ret; + + ret = regmap_write(priv->regmap, T8142_CIO_CTRL, + T8142_CIO_CTRL_INIT_REQ(id)); + if (ret) + return ret; + + return regmap_read_poll_timeout(priv->regmap, T8142_CIO_CTRL, val, + !(val & T8142_CIO_CTRL_INIT_BUSY(id)), + APPLE_CIO_RESET_POLL_US, + APPLE_CIO_RESET_TIMEOUT_US); +} + +static const struct apple_cio_reset_variant apple_t8142_cio_reset = { + .nr_resets = 4, + .deassert = t8142_cio_deassert, +}; + static const struct apple_cio_reset_variant apple_t8103_cio_reset = { .nr_resets = 2, .deassert = t8103_cio_deassert, @@ -221,6 +259,10 @@ static int apple_cio_reset_probe(struct platform_device *pdev) } static const struct of_device_id apple_cio_reset_match[] = { + { + .compatible = "apple,t8142-cio-reset", + .data = &apple_t8142_cio_reset, + }, { .compatible = "apple,t8103-cio-reset", .data = &apple_t8103_cio_reset, diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 1f3b1ba568b86d..fd5c4eabcf5aaf 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -194,6 +194,7 @@ struct apple_dpin_binding; struct apple_cio { struct device *dev; + bool type7; struct device_node *np; const struct apple_cio_hw *hw; const struct apple_dpin_policy *dp; @@ -1151,7 +1152,9 @@ static void apple_nhi_ring_configure(struct tb_ring *ring, u32 flags, u32 e2e_fl sof_eof_mask = ring->sof_mask << 16 | ring->eof_mask; writel(sof_eof_mask, options + 4); - writel(sof_eof_mask, anhi->pdf_base + ring->hop * APPLE_CIO_NHI_PDF_STRIDE); + if (anhi->pdf_base) + writel(sof_eof_mask, + anhi->pdf_base + ring->hop * APPLE_CIO_NHI_PDF_STRIDE); } /* @@ -1906,6 +1909,12 @@ static int apple_nhi_probe(struct platform_device *pdev) int cap_apple; int ret = 0; + if (acio->type7 != + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-usb4-nhi")) { + ret = dev_err_probe(&pdev->dev, -EINVAL, "ACIO/NHI generation mismatch\n"); + goto err; + } + anhi = devm_kzalloc(&pdev->dev, sizeof(*anhi), GFP_KERNEL); if (!anhi) { ret = -ENOMEM; @@ -1921,6 +1930,13 @@ static int apple_nhi_probe(struct platform_device *pdev) INIT_DELAYED_WORK(&anhi->dp_aux_work, apple_dp_aux_work); platform_set_drvdata(pdev, anhi); + if (acio->type7 && + of_property_count_u32_elems(anhi->np, "apple,tunable-nhi") <= 0) { + ret = dev_err_probe(anhi->dev, -EINVAL, + "Missing bootloader NHI tuning\n"); + goto err; + } + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "nhi"); anhi->nhi_base = devm_ioremap_resource(&pdev->dev, res); if (IS_ERR(anhi->nhi_base)) { @@ -1935,12 +1951,15 @@ static int apple_nhi_probe(struct platform_device *pdev) } apple_tunable_apply(anhi->nhi_base, tunable); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pdf"); - anhi->pdf_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(anhi->pdf_base)) { - ret = dev_err_probe(&pdev->dev, PTR_ERR(anhi->pdf_base), - "Unable to map PDF regs\n"); - goto err; + /* Type 7 keeps the PDF masks in the receive ring registers only. */ + if (!acio->type7) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pdf"); + anhi->pdf_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(anhi->pdf_base)) { + ret = dev_err_probe(&pdev->dev, PTR_ERR(anhi->pdf_base), + "Unable to map PDF regs\n"); + goto err; + } } ret = apple_nhi_probe_irqs(anhi); @@ -2283,6 +2302,9 @@ static const struct dev_pm_ops apple_nhi_pm_ops = { }; static const struct of_device_id apple_nhi_match[] = { + { + .compatible = "apple,t8142-usb4-nhi", + }, { .compatible = "apple,t8103-usb4-nhi", }, @@ -3007,6 +3029,11 @@ static int apple_cio_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(acio->cpu_base), "Unable to map MxWrap cpu regs\n"); } + acio->type7 = of_device_is_compatible(acio->np, "apple,t8142-usb4-acio"); + if (acio->type7 && + of_property_count_u32_elems(acio->np, "apple,tunable-rc") <= 0) + return dev_err_probe(dev, -EINVAL, "Missing bootloader RC tuning\n"); + acio->pcie_tunnel_np = of_parse_phandle(dev->of_node, "apple,pcie-tunnel", 0); if (acio->pcie_tunnel_np) { @@ -3181,6 +3208,7 @@ static void apple_cio_remove(struct platform_device *pdev) static const struct of_device_id apple_acio_match[] = { { + .compatible = "apple,t8142-usb4-acio", .compatible = "apple,t6030-usb4-acio", .data = &apple_cio_t6030_hw, }, From dedeac37fa95fd5e76954d2d7b48c6134b9ec76a Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:50 -0400 Subject: [PATCH 014/631] drm: apple: support H17-generation DCP firmware on T8142 iBoot leaves the T8142 (M5) display coprocessor running with macOS 26 ("H17") firmware, mapped through locked DARTs. Add support for that firmware generation and attach to the live session: - Match apple,t8142-dcp and pin the H17 interface. Attach to the running coprocessor with the standard RTKit wake handshake; never stop, reset or reload it. - Add the H17 IOMFB method and callback tables, with T8142 method aliases, the nested-call reply handling, the AFK receive layout and the service-init keys this firmware uses, and the DCP bandwidth and clock requests answered from DT resources. - Take the panel dimensions from the firmware at startup and keep the mode the boot loader programmed. Only that mode is offered. - Skip teardown calls whose H17 method numbers are not established, and don't send AFK requests on H17, whose send format is unknown. Linux loads no firmware here, and no panel timing, geometry, power model or other calibration data is included. Optional properties the firmware asks for are answered "not found". Changes to shared code that alter behaviour (ACK reply buffers, framebuffer release, EPIC service-init keys) are limited to H17 firmware; v12.3 and v13.5 keep their existing paths. Co-authored-by: Dj <43033058+DjDeveloperr@users.noreply.github.com> --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/afk.c | 109 +++- drivers/gpu/drm/apple/afk.h | 26 +- drivers/gpu/drm/apple/dcp-internal.h | 36 ++ drivers/gpu/drm/apple/dcp-link.h | 83 +++ drivers/gpu/drm/apple/dcp.c | 186 +++++- drivers/gpu/drm/apple/iomfb.c | 109 +++- drivers/gpu/drm/apple/iomfb.h | 30 + drivers/gpu/drm/apple/iomfb_h17p.c | 239 ++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 22 + drivers/gpu/drm/apple/iomfb_template.c | 777 ++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.h | 126 +++- drivers/gpu/drm/apple/iomfb_v12_3.c | 1 + drivers/gpu/drm/apple/iomfb_v13_3.c | 1 + drivers/gpu/drm/apple/parser.c | 25 +- drivers/gpu/drm/apple/parser.h | 2 +- 16 files changed, 1731 insertions(+), 42 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.c create mode 100644 drivers/gpu/drm/apple/iomfb_h17p.h diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 2690627fcb4827..8f33959f281c57 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -16,6 +16,7 @@ appledrm-y += iomfb_v13_3.o appledrm-y += iomfb_v14_7_tpl.o appledrm-y += iomfb_v14_7.o iomfb_v14_7_link.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o +appledrm-y += iomfb_h17p.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 604b1972705e8a..dbc3753411480e 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -171,7 +171,7 @@ int afk_start(struct apple_dcp_afkep *ep) static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) { - u16 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(GETBUF_TAG, message); u64 reply; @@ -202,10 +202,10 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, struct afk_ringbuffer *bfr) { - u16 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; - u16 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; + u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; + u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, end; + u32 bufsz, hdrsz, end, block; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -219,7 +219,7 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - if (base >= ep->bfr_size) { + if (!ep->bfr || size < sizeof(__le32) || base >= ep->bfr_size) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", ep->endpoint, base, ep->bfr_size); @@ -235,11 +235,16 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, } bfr->hdr = ep->bfr + base; - bufsz = le32_to_cpu(bfr->hdr->bufsz); - if (bufsz + sizeof(*bfr->hdr) != size) { + dma_rmb(); + bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)bfr->hdr)); + hdrsz = size - bufsz; + block = hdrsz / AFK_RB_BLOCKS; + if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || + block < AFK_RB_BLOCK_MIN || !is_power_of_2(block) || + !IS_ALIGNED(bufsz, block) || bufsz < 2 * block) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring buffer size 0x%x != expected 0x%lx\n", - ep->endpoint, bufsz, sizeof(*bfr->hdr)); + "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", + ep->endpoint, bufsz, size, base, message); return; } if (bufsz < BIT(BLOCK_SHIFT)) { @@ -254,7 +259,7 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - bfr->buf = bfr->hdr + 1; + bfr->buf = (u8 *)bfr->hdr + hdrsz; bfr->bufsz = bufsz; bfr->ready = true; @@ -348,7 +353,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, ep->endpoint, name); return; } - ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit); + ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, + ep->dcp->fw_compat == DCP_FIRMWARE_H17P); if (ret) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: failed to extract init props: %d\n", @@ -621,15 +627,53 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr = (struct epic_hdr *)data; struct epic_sub_hdr *eshdr = (struct epic_sub_hdr *)(data + sizeof(*ehdr)); + struct epic_sub_hdr h17p_eshdr; + size_t hdr_len = sizeof(*ehdr) + sizeof(*eshdr); + u16 subtype; + u8 *payload; u16 subtype; u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); size_t payload_size; - if (data_size < sizeof(*ehdr) + sizeof(*eshdr)) { - dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", - ep->endpoint, data_size); - return; + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + const struct epic_sub_hdr_h17p *c = + (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); + + hdr_len = sizeof(*ehdr) + sizeof(*c); + if (data_size < hdr_len) + goto too_small; + + /* + * H17P multiplexes every service of an endpoint onto queue + * channel 0 and carries the real channel in the EPIC header + * instead: byte 0 is a sequence counter, byte 1 a flags byte + * and bytes 2-3 the channel. ep:20 announces "system" as + * channel 1 and "powerlog-service" as channel 3, then sends the + * mNits/uAmps/iDAC backlight reports on channel 3. Taking the + * queue channel at face value binds "system" to 0 and makes + * every later announce and report look like a duplicate. + */ + channel = le16_to_cpup((__le16 *)(data + 2)); + + /* + * Normalise into the wide sub-header the rest of this file + * expects, mapping H17P's announce subtype onto the canonical + * one so no downstream check has to know the difference. + */ + memset(&h17p_eshdr, 0, sizeof(h17p_eshdr)); + h17p_eshdr.category = c->category; + h17p_eshdr.type = cpu_to_le16(c->type == EPIC_SUBTYPE_ANNOUNCE_H17P ? + EPIC_SUBTYPE_ANNOUNCE : c->type); + h17p_eshdr.tag = cpu_to_le16(le32_to_cpu(c->tag)); + eshdr = &h17p_eshdr; } + + if (data_size < hdr_len) + goto too_small; + + subtype = le16_to_cpu(eshdr->type); + payload = data + hdr_len; + payload_size = data_size - hdr_len; payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); subtype = le16_to_cpu(eshdr->type); @@ -698,6 +742,11 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); + return; + +too_small: + dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", + ep->endpoint, data_size); } static bool afk_recv(struct apple_dcp_afkep *ep) @@ -712,8 +761,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - rptr = le32_to_cpu(ep->rxbfr.hdr->rptr); - wptr = le32_to_cpu(ep->rxbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->rxbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->rxbfr)); trace_afk_recv_rwptr_pre(ep, rptr, wptr); if (rptr == wptr) @@ -775,7 +824,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) */ memcpy(ep->rx_scratch, hdr->data, size); - rptr = ALIGN(rptr + sizeof(*hdr) + size, 1 << BLOCK_SHIFT); + rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block); if (WARN_ON(rptr > ep->rxbfr.bufsz)) rptr = 0; if (rptr == ep->rxbfr.bufsz) @@ -783,7 +832,7 @@ static bool afk_recv(struct apple_dcp_afkep *ep) dma_mb(); - ep->rxbfr.hdr->rptr = cpu_to_le32(rptr); + *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); afk_recv_handle(ep, channel, type, ep->rx_scratch, size); @@ -880,6 +929,22 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, size_t total_epic_size, total_size; int ret; + /* + * H17P frames EPIC messages differently (see afk_recv_handle()): an + * 8-byte sub-header, with the service channel carried in the EPIC + * header on queue channel 0. Only the receive side of that format is + * known, and the 24-byte framing below would be misparsed, so send + * nothing. The display path needs no AP-initiated EPIC traffic: the + * callers are optional (verbose firmware logging, EDID copy, audio, + * DPTX, debugfs) or replies to calls only DPTX services make. + */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + dev_dbg_once(ep->dcp->dev, + "AFK[ep:%02x]: not sending EPIC message on H17P\n", + ep->endpoint); + return -EOPNOTSUPP; + } + spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { ret = -ESHUTDOWN; @@ -887,8 +952,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, } dma_rmb(); - rptr = le32_to_cpu(ep->txbfr.hdr->rptr); - wptr = le32_to_cpu(ep->txbfr.hdr->wptr); + rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); + wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; total_size = sizeof(*hdr) + total_epic_size; @@ -1001,7 +1066,7 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, 1 << BLOCK_SHIFT); + wptr = ALIGN(wptr, ep->txbfr.block); if (wptr == ep->txbfr.bufsz) wptr = 0; trace_afk_send_rwptr_post(ep, rptr, wptr); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index f0a2ce2e70c676..90860b994985fa 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -201,6 +201,21 @@ struct epic_sub_hdr { __le32 inline_len; } __attribute__((packed)); +/* + * H17P compresses the EPIC sub-header from 24 bytes to 8 and drops the + * separate length/timestamp: the announce for the "system" service puts its + * name at data + 0x18, not data + 0x28. An announce carries type 0x11 and a + * standard-service report carries the usual 0xc0. + */ +struct epic_sub_hdr_h17p { + u8 type; + u8 category; + __le16 flags; + __le32 tag; +} __packed; + +#define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 + struct epic_cmd { __le32 retcode; __le64 rxbuf; @@ -244,12 +259,21 @@ enum epic_subtype { struct afk_ringbuffer { bool ready; - struct afk_ringbuffer_header *hdr; + void *hdr; + u32 block; /* control-block stride, 0x40 or 0x80 */ u32 rptr; void *buf; size_t bufsz; }; +static inline __le32 *afk_rb_field(struct afk_ringbuffer *bfr, unsigned int idx) +{ + return (__le32 *)((u8 *)bfr->hdr + idx * bfr->block); +} + +#define afk_rb_rptr(bfr) afk_rb_field(bfr, 1) +#define afk_rb_wptr(bfr) afk_rb_field(bfr, 2) + struct apple_dcp_afkep { struct apple_dcp *dcp; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index e5e40af46e233e..ee9b2477d0887c 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include "dcp-fabric-session.h" #include "dptxep.h" #include "iomfb.h" +#include "iomfb_h17p.h" #include "iomfb-state.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" @@ -51,6 +53,7 @@ enum dcp_firmware_version { * endpoints speak the 14.7 protocol (dcp_fw_is_14_7()). */ DCP_FIRMWARE_V_14_7_TEMPLATE, + DCP_FIRMWARE_H17P, }; enum { @@ -84,6 +87,8 @@ struct dcp_mem_descriptor { dma_addr_t dva; struct sg_table map; u64 reg; + /* live iommu mappings installed by dcpep_cb_map_piodma() */ + bool piodma_mapped; }; /* Limit on call stack depth (arbitrary). Some nesting is required */ @@ -106,7 +111,14 @@ struct dcp_channel { struct dcp_fb_reference { struct list_head head; struct drm_framebuffer *fb; + /* + * Id of the swap that unbinds @fb, i.e. the one that puts its + * replacement on screen. Only meaningful once @armed is set: the + * firmware assigns the id in the swap_start reply, which happens after + * the atomic commit that displaced @fb has already queued this entry. + */ u32 swap_id; + bool armed; }; #define MAX_NOTCH_HEIGHT 160 @@ -132,9 +144,21 @@ struct dcp_panel { bool has_mini_led; }; +enum dcp_iomfb_method_profile { + DCP_IOMFB_METHODS_DEFAULT, + DCP_IOMFB_METHODS_H17G, +}; + struct apple_dcp_hw_data { u32 num_dptx_ports; bool t6020_tunnel_flow; + enum dcp_iomfb_method_profile iomfb_method_profile; + /* + * The bootloader leaves the coprocessor running. Attach to it with a + * standard RTKit wake instead of starting or restarting the ASC. + */ + bool adopt_live_session; + enum dcp_firmware_version firmware_compat; }; /* TODO: move IOMFB members to its own struct */ @@ -146,6 +170,8 @@ struct apple_dcp { struct device *dev; struct platform_device *piodma; struct iommu_domain *iommu_dom; + /* which of the nine cumulative A031 notify-client states to send next */ + unsigned int a031_step; struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; @@ -215,6 +241,7 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; + struct clk *clk_194; /* DCP shared memory */ void *shmem; @@ -252,10 +279,13 @@ struct apple_dcp { struct dcp_swap_submit_req_v12_3 v12_3; struct dcp_swap_submit_req_v13_3 v13_3; struct dcp_swap_submit_req_v14_7_0 v14_7_0; + struct dcp_swap_submit_req_h17p h17p; } swap; /* swap id of the last completed swap */ u32 last_swap_id; + /* last_swap_id is only meaningful after the first swap completes */ + bool have_swap_complete; ktime_t swap_start; u64 swap_submit_timestamp; @@ -322,6 +352,12 @@ struct apple_dcp { * on the next successfully completed swap. */ struct list_head swapped_out_fbs; + /* + * Protects swapped_out_fbs, which is appended to from the DRM atomic + * commit (dcp_flush -> .atomic_flush) and armed/drained from the RTKit + * workqueue that runs the DCP callbacks. + */ + spinlock_t swapped_out_lock; struct dcp_brightness brightness; /* Workqueue for updating the initial brightness */ diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 6f27757be27953..2168d03dd53767 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -779,6 +779,89 @@ apple_dcp_h17p_rt_bw_encode_reply(struct apple_dcp_link_rpc_view *view, return -EINVAL; reply = view->output; +#include +#include +#include + +#include + +/* + * H17-generation DCP firmware uses the ordinary IOMFB transport once the + * INIT/SetShmem handshake is done: the same six fixed channels (CMD 0x00000, + * OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, CB 0x60000, + * OOBCB 0x68000), selected by the context field of the message. Records are + * allocated on each channel's stack at + * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) + * which is the rule dcp_packet_start() and DCP_PACKET_ALIGNMENT implement in + * iomfb.c. + * + * This header only describes the callback payloads whose layout differs from + * older firmware. + */ + +#define APPLE_DCP_H17P_CLOCK_ID_154 0x154 +#define APPLE_DCP_H17P_CLOCK_ID_194 0x194 + +struct apple_dcp_h17p_clock_request { + __le32 service; + __le32 clock_id; +} __packed; + +struct apple_dcp_h17p_rt_bw_config { + __le64 scratch; + __le64 clock_request; +} __packed; + +struct apple_dcp_h17p_rt_bw_request { + __le32 config_null; +} __packed; + +struct apple_dcp_h17p_rt_bw_reply { + struct apple_dcp_h17p_rt_bw_config config; + __le32 status; +} __packed; + +static_assert(sizeof(struct apple_dcp_h17p_clock_request) == 0x08); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); +static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); + +static inline int +apple_dcp_h17p_clock_rate(const struct apple_dcp_h17p_clock_request *request, + u64 rate_154, u64 rate_194, u64 *rate) +{ + if (!request || !rate) + return -EINVAL; + + switch (le32_to_cpu(request->clock_id)) { + case APPLE_DCP_H17P_CLOCK_ID_154: + *rate = rate_154; + return 0; + case APPLE_DCP_H17P_CLOCK_ID_194: + *rate = rate_194; + return 0; + default: + return -ENOENT; + } +} + +static inline void +apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, + u64 scratch, u64 clock_request) +{ + config->scratch = cpu_to_le64(scratch); + config->clock_request = cpu_to_le64(clock_request); +} + +/* D003: fill in the bandwidth config unless the firmware passed it as null */ +static inline int +apple_dcp_h17p_rt_bw_encode_reply(const struct apple_dcp_h17p_rt_bw_request *request, + struct apple_dcp_h17p_rt_bw_reply *reply, + u64 scratch, u64 clock_request) +{ + if (le32_to_cpu(request->config_null) & 1) + return -EINVAL; + apple_dcp_h17p_rt_bw_encode(&reply->config, scratch, clock_request); reply->status = cpu_to_le32(0); return 0; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 03a3737c9b2507..6d51199013e336 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -247,6 +248,25 @@ static void dcp_delayed_vblank(struct work_struct *work) * hotplug worker re-apply the active CRTC. */ static void dcp_swap_watchdog(struct work_struct *work) +static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) +{ + switch (endpoint) { + case AV_ENDPOINT: + return dcp->avep; + case SYSTEM_ENDPOINT: + return dcp->systemep; + case DISP0_ENDPOINT: + return dcp->ibootep; + case DPAVSERV_ENDPOINT: + return dcp->dcpavservep; + case DPTX_ENDPOINT: + return dcp->dptxep; + default: + return NULL; + } +} + +static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, @@ -282,6 +302,17 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) /* A failed T8122 dcpext: its firmware may still talk, nobody listens. */ if (READ_ONCE(dcp->ext_dead)) return; + /* + * H17P starts endpoints that have no AFK instance behind them (the + * remote allocator endpoint, for one). Log their messages rather than + * dereferencing a NULL afkep or warning about an unknown endpoint. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P && endpoint != IOMFB_ENDPOINT && + !dcp_afkep(dcp, endpoint)) { + dev_dbg_ratelimited(dcp->dev, "ep %#04x: %#llx\n", endpoint, + message); + return; + } switch (endpoint) { case IOMFB_ENDPOINT: @@ -329,9 +360,32 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ complete(&dcp->start_done); } +/* + * Check that every IOMMU page of [iova, iova + size) translates to the same + * offset from @phys as it has from @iova. + */ +static int dcp_check_iova_contiguous(struct iommu_domain *domain, + dma_addr_t iova, size_t size, + phys_addr_t phys) +{ + unsigned long pgsize = domain->pgsize_bitmap ? + 1UL << __ffs(domain->pgsize_bitmap) : PAGE_SIZE; + dma_addr_t end = iova + size; + dma_addr_t addr = iova; + + while (addr < end) { + if (iommu_iova_to_phys(domain, addr) != phys + (addr - iova)) + return -EINVAL; + addr = ALIGN_DOWN(addr, pgsize) + pgsize; + } + + return 0; +} + static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { struct apple_dcp *dcp = cookie; + int ret; if (bfr->iova) { struct iommu_domain *domain = @@ -346,6 +400,16 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) if (!phy_addr) return -ENOMEM; + /* memremap() below needs the whole buffer physically contiguous */ + ret = dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, + phy_addr); + if (ret) { + dev_err(dcp->dev, + "shmem_setup: iova %pad (%#zx bytes) is not physically contiguous\n", + &bfr->iova, bfr->size); + return ret; + } + // TODO: verify phy_addr, cache attribute bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); if (!bfr->buffer) @@ -1351,6 +1415,18 @@ int dcp_start(struct platform_device *pdev) return 0; } + /* + * The coprocessor start can fail (or be recovered from) without tearing + * the DRM side down, so this can be reached with no RTKit instance at + * all. Starting an endpoint then dereferences NULL inside + * apple_rtkit_start_ep() and takes the machine with it. + */ + if (!dcp->rtk || IS_ERR(dcp->rtk)) { + dev_err(dcp->dev, + "cannot start endpoints: RTKit is not initialised\n"); + return -ENODEV; + } + /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) @@ -1435,6 +1511,8 @@ static void _dcp_poweroff(struct apple_dcp *dcp) break; case DCP_FIRMWARE_V_14_7: iomfb_v14_7_poweroff(dcp); + case DCP_FIRMWARE_H17P: + iomfb_poweroff_h17p(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -1523,6 +1601,8 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) break; case DCP_FIRMWARE_V_14_7: iomfb_v14_7_poweroff(dcp); + case DCP_FIRMWARE_H17P: + iomfb_sleep_h17p(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -1611,6 +1691,8 @@ void dcp_poweron(struct platform_device *pdev) break; case DCP_FIRMWARE_V_14_7: iomfb_v14_7_poweron(dcp); + case DCP_FIRMWARE_H17P: + iomfb_poweron_h17p(dcp); break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -1877,18 +1959,66 @@ static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) return ret; } +/* + * Count the "disp-*" entries in reg-names. + * + * The positional form below assumes the reg layout is [coproc, disp-0 ...], + * i.e. exactly one non-disp register, and derives the count as + * num_resources - 1. DCPs with H17-generation firmware list a trailing + * "iop-vbar" register after the display apertures, which would inflate that + * count, bind the last disp_registers[] entry to iop-vbar and make the + * "apple,bw-scratch" disp_reg index check reject a valid device tree. + * + * Returns a negative errno when reg-names is absent, so callers fall back. + */ +static int dcp_count_disp_regs(struct device *dev) +{ + int n = of_property_count_strings(dev->of_node, "reg-names"); + const char *name; + int i, count = 0; + + if (n <= 0) + return -EINVAL; + + for (i = 0; i < n; ++i) { + if (of_property_read_string_index(dev->of_node, "reg-names", i, + &name)) + return -EINVAL; + if (!strncmp(name, "disp-", 5)) + count++; + } + + return count ? count : -EINVAL; +} + static int dcp_get_disp_regs(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); int count = pdev->num_resources - 1; int i, ret; + /* Device trees without reg-names keep the historical positional form. */ + if (!by_name) + count = pdev->num_resources - 1; + if (count <= 0 || count > MAX_DISP_REGISTERS) return -EINVAL; for (i = 0; i < count; ++i) { - dcp->disp_registers[i] = - platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + if (by_name) { + char name[8]; + + snprintf(name, sizeof(name), "disp-%d", i); + dcp->disp_registers[i] = platform_get_resource_byname( + pdev, IORESOURCE_MEM, name); + } else { + dcp->disp_registers[i] = platform_get_resource( + pdev, IORESOURCE_MEM, 1 + i); + } + if (!dcp->disp_registers[i]) { + dev_err(dcp->dev, "missing display register %d\n", i); + return -EINVAL; + } } /* load pmgr bandwidth scratch resource and offset */ @@ -1955,6 +2085,7 @@ static int dcp_read_fw_version(struct device *dev, const char *name, static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) { + const struct apple_dcp_hw_data *hw = of_device_get_match_data(dev); const struct dcp_v14_board *v14_board; char compat_str[DCP_FW_VERSION_STR_LEN]; char fw_str[DCP_FW_VERSION_STR_LEN]; @@ -2004,6 +2135,15 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) compat_str); return DCP_FIRMWARE_UNKNOWN; } + /* + * SoCs introduced with H17-generation firmware pin its interface; the + * loader may not recognise their boot firmware version. + */ + if (hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) + return hw->firmware_compat; + + /* firmware version is just informative */ + dcp_read_fw_version(dev, "apple,firmware-version", fw_str); /* The internal panels with a board record (T6030, J613): the same. */ v14_board = iomfb_v14_7_board(dev); @@ -2156,6 +2296,20 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat return PTR_ERR(dcp->coproc_reg); } + /* + * Display coprocessors running H17-generation firmware are left running + * by the bootloader with their firmware mapped through a locked DART. + * Linux attaches to that session with a standard RTKit INIT/HELLO + * exchange and must not stop or restart the ASC. + */ + if (dcp->hw.adopt_live_session) { + cpu_ctrl = readl_relaxed(dcp->coproc_reg + + APPLE_DCP_COPROC_CPU_CONTROL); + if (!(cpu_ctrl & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) + return dev_err_probe(dev, -EOPNOTSUPP, + "DCP was not left running by the bootloader\n"); + } + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -2241,6 +2395,10 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat if (IS_ERR(dcp->clk)) return dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); + dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); + if (IS_ERR(dcp->clk_194)) + return dev_err_probe(dev, PTR_ERR(dcp->clk_194), + "Unable to find clock 0x194\n"); bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry @@ -2251,6 +2409,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + spin_lock_init(&dcp->swapped_out_lock); dcp_ext_wait_pmp(dcp); @@ -2258,12 +2417,21 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + if (!dcp->hw.adopt_live_session) { + cpu_ctrl = + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + } dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) return dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); + if (dcp->hw.adopt_live_session) + dev_info(dev, "negotiating RTKit with the running DCP\n"); + ret = apple_rtkit_wake(dcp->rtk); if (ret) return dev_err_probe(dev, ret, @@ -2385,6 +2553,8 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) devm_clk_put(dev, dcp->clk); dcp->clk = NULL; + /* optional and possibly NULL, released with the bind's devres group */ + dcp->clk_194 = NULL; } static const struct component_ops dcp_comp_ops = { @@ -3049,6 +3219,17 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +/* + * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering + * and is left running by the bootloader. + */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8142 = { + .num_dptx_ports = 0, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17G, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { .num_dptx_ports = 0, }; @@ -3087,6 +3268,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t8122-dcp", .data = &apple_dcp_hw_t8122, }, { .compatible = "apple,t6031-dcp", .data = &apple_dcp_hw_t6031, }, { .compatible = "apple,t8122-dcpext", .data = &apple_dcp_hw_t8122_dcpext, }, + { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, {} diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 40f410828e1a5e..fdda5a02b08e64 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -162,6 +162,13 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca u32 in_len, u32 out_len, void *data, dcp_callback_t cb, void *cookie) { + struct dcp_method_entry resolved = *call; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && + call->tag_h17g[0]) + memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); + call = &resolved; + enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); @@ -396,15 +403,30 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || + !dcp->cb_handlers[tag]) { dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + /* + * Leaving a callback unanswered wedges the coprocessor: it + * waits for the ack forever and the outer call never returns. + * On H17P answer it with a zeroed output instead. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (hdr->out_len) + memset(out, 0, hdr->out_len); + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + + ALIGN(length, DCP_PACKET_ALIGNMENT); + dcp_ack(dcp, context); + } return; } - in = data + sizeof(*hdr); - out = in + hdr->in_len; - // TODO: verify that in_len and out_len match our prototypes // for now just clear the out data to have at least consistent results if (hdr->out_len) @@ -423,7 +445,8 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, { struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); - void *cookie; + bool h17p = dcp->fw_compat == DCP_FIRMWARE_H17P; + void *cookie, *out; dcp_callback_t cb; if (!ch) { @@ -431,6 +454,11 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, return; } + if (h17p && !ch->depth) { + dev_warn(dcp->dev, "ignoring ack on idle context %X\n", context); + return; + } + dcp_pop_depth(&ch->depth); cb = ch->callbacks[ch->depth]; @@ -439,8 +467,21 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, ch->callbacks[ch->depth] = NULL; ch->cookies[ch->depth] = NULL; + if (h17p) { + /* + * H17P acks carry no payload and offset zero, including for + * nested calls, so the output cannot be located from the ack + * message. It is in the original AP command record, whose + * address dcp_push() saved. + */ + out = ch->output[ch->depth]; + ch->output[ch->depth] = NULL; + } else { + out = data + sizeof(*header) + header->in_len; + } + if (cb) - cb(dcp, data + sizeof(*header) + header->in_len, cookie); + cb(dcp, out, cookie); } static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) @@ -766,6 +807,27 @@ bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, return lookup_mode_locked(dcp, mode, out); } +/* + * H17P keeps the mode the bootloader programmed and never sends + * set_digital_out_mode (see DCP_INHERIT_BOOT_MODE in iomfb_template.c), so a + * modeset to any other mode would silently not be applied. The bootloader + * brings the panel up in its native timing, which is the mode the firmware + * scores highest and enumerate_modes() marks preferred; offer only that one. + */ +static bool dcp_mode_settable(struct apple_dcp *dcp, + const struct drm_display_mode *mode) +{ + struct dcp_display_mode *dcp_mode = lookup_mode(dcp, mode); + + if (!dcp_mode) + return false; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; + + return true; +} + enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { @@ -815,6 +877,8 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_V_13_5: ret = iomfb_modeset_v13_3(dcp, crtc_state); break; + case DCP_FIRMWARE_H17P: + ret = iomfb_modeset_h17p(dcp, crtc_state); case DCP_FIRMWARE_V_26_6: ret = iomfb_v26_6_modeset(dcp, crtc_state); break; @@ -976,6 +1040,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) case DCP_FIRMWARE_V_13_5: iomfb_flush_v13_3(dcp, crtc, state); break; + case DCP_FIRMWARE_H17P: + iomfb_flush_h17p(dcp, crtc, state); case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_flush_v14_7_0(dcp, crtc, state); break; @@ -994,6 +1060,8 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_start_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_start_h17p(dcp); case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_start_v14_7_0(dcp); break; @@ -1014,9 +1082,19 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); - if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { + /* + * H17P reports its interface version (bits 63:48) and a + * firmware hash in the high half of the InitComplete word. + * The transport below it is unchanged. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + dev_info(dcp->dev, + "IOMFB: init complete %#llx (version %llu)\n", + message, + FIELD_GET(GENMASK_ULL(63, 48), message)); iomfb_start(dcp); - else if (type == IOMFB_MESSAGE_TYPE_MSG) + } else if (type == IOMFB_MESSAGE_TYPE_MSG) dcpep_got_msg(dcp, message); else dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); @@ -1027,12 +1105,25 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) dma_addr_t shmem_iova; int ret; + /* + * H17P firmware expects the remote allocator endpoint to be started + * before IOMFB. A firmware subsystem whose endpoint was never started + * has nothing answering it on the AP side, and without this one the + * display DART faults. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); + if (ret) + return ret; + } ret = apple_rtkit_start_ep(dcp->rtk, IOMFB_ENDPOINT); if (ret) return ret; dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, GFP_KERNEL); + if (!dcp->shmem) + return -ENOMEM; dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); @@ -1052,6 +1143,8 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_shutdown_v13_3(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_shutdown_h17p(dcp); case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_shutdown_v14_7_0(dcp); break; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 2f022a0320a219..57569d1679027d 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -156,6 +156,7 @@ enum dcpep_method { dcpep_flush_supports_power, dcpep_set_power_state, dcpep_first_client_open, + dcpep_register_dfb_surface, dcpep_set_parameter_dcp, dcpep_enable_disable_video_power_savings, dcpep_is_main_display, @@ -166,14 +167,43 @@ enum dcpep_method { iomfbep_last_client_close, iomfbep_abort_swaps_dcp, iomfbep_set_matrix, + /* + * H17P default-framebuffer queries, issued from inside + * start_hardware_boot (D121) between create_default_fb and + * setup_video_limits. + */ + dcpep_get_dfb_compression_info, + dcpep_get_dfb_info, + dcpep_get_dfb_layout, + dcpep_get_dfb_state, + dcpep_set_dfb_dimensions, + dcpep_commit_dfb_info, + dcpep_dfb_query, + dcpep_dfb_ready_query, + /* + * H17P pipe-configuration calls made before the first swap. Most + * names are unknown; they are identified by method number. + */ + dcpep_pipe_cfg_415, + dcpep_pipe_cfg_031, + dcpep_pipe_cfg_414, + dcpep_pipe_query_478, + dcpep_pipe_query_474, + dcpep_pipe_set_476, + dcpep_pipe_set_428, dcpep_num_methods }; #define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } +#define IOMFB_METHOD_H17(tag, h17g, name) [name] = { #name, \ + { tag[0], tag[1], tag[2], tag[3] }, \ + { h17g[0], h17g[1], h17g[2], h17g[3] } } struct dcp_method_entry { const char *name; char tag[4]; + /* H17G shares the H17P transport but changes selected method IDs. */ + char tag_h17g[4]; }; #define IOMFB_MAX_CB (1000) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c new file mode 100644 index 00000000000000..28f219b8be14ef --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -0,0 +1,239 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright The Asahi Linux Contributors */ + +#include "dcp-link.h" +#include "iomfb_h17p.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" + +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); +/* D590 swap_complete_ap_gated is 0x730 bytes on the wire. */ +static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ + IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), /* nested in D206 */ + IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), /* nested in D207 */ + IOMFB_METHOD("A385", dcpep_set_create_dfb), /* first call inside D121; (0,0) */ + IOMFB_METHOD("A386", iomfbep_a358_vi_set_temperature_hint), /* nested in D100 */ + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + /* + * swap_start moved A407 -> A406 on H17P, but swap_submit did not move + * with it: every swap is A406 followed by A408, A407 is unused, and the + * A408 response is 0x0c bytes (dcp_swap_submit_resp_h17p). + */ + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A442", dcpep_set_parameter_dcp), /* (0x28, 0x4) */ + IOMFB_METHOD("A446", dcpep_create_default_fb), /* follows A385 in D121 */ + /* + * The default-framebuffer block the firmware expects inside D121, + * straight after A446 and before A025. Sizes are (in, out). + */ + IOMFB_METHOD_H17("A035", "A039", dcpep_get_dfb_compression_info), /* (0x8, 0x8) */ + IOMFB_METHOD_H17("A034", "A038", dcpep_get_dfb_info), /* (0x2c, 0x2c) */ + IOMFB_METHOD("A103", dcpep_get_dfb_layout), /* (0xc, 0x8) */ + IOMFB_METHOD("A445", dcpep_get_dfb_state), /* (0x4, 0x8) */ + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), /* (0x8, 0) w,h */ + IOMFB_METHOD("A105", dcpep_commit_dfb_info), /* (0, 0) */ + IOMFB_METHOD_H17("A033", "A037", dcpep_dfb_query), /* (0, 0x4) */ + IOMFB_METHOD("A380", dcpep_dfb_ready_query), /* (0, 0x4) */ + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), /* (0xc, 0xc) zeros */ + /* update_notify_clients_dcp */ + IOMFB_METHOD_H17("A031", "A035", dcpep_pipe_cfg_031), /* (0x6c, 0) */ + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ + IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ + IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ + IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ + IOMFB_METHOD("A428", dcpep_pipe_set_428), /* (0x4, 0x4) 0x10000 */ + IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ + IOMFB_METHOD("A457", dcpep_first_client_open), /* (0, 0) */ + /* D561 (displayMinRefreshInterval) arrives while A465 is in flight */ + IOMFB_METHOD("A465", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A468", dcpep_flush_supports_power), /* follows A025 in D121, in = 1 */ + /* + * iomfbep_last_client_close and iomfbep_abort_swaps_dcp are deliberately + * absent: their H17P numbers are unverified, so the teardown paths skip + * them (DCP_HAS_CLIENT_TEARDOWN in iomfb_template.c). + */ + IOMFB_METHOD("A473", dcpep_set_power_state), /* out-of-band; out 0x8 */ + IOMFB_METHOD("A472", dcpep_register_dfb_surface), /* default FB surface, nested in D582 */ +}; + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + /* + * The bandwidth scratch and doorbell come from the DCP's DT node; the + * apple,bw-doorbell binding has no offset cell. + */ + u64 scratch = dcp->disp_bw_scratch_res.start + + dcp->disp_bw_scratch_offset; + u64 clock_request = dcp->disp_bw_doorbell_res.start; + + trace_iomfb_callback(dcp, tag, __func__); + if (apple_dcp_h17p_rt_bw_encode_reply(in, out, scratch, clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; +} + +static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u64 rate; + int ret; + + trace_iomfb_callback(dcp, tag, __func__); + ret = apple_dcp_h17p_clock_rate(request, clk_get_rate(dcp->clk), + clk_get_rate(dcp->clk_194), &rate); + if (ret) { + dev_warn(dcp->dev, "unknown display clock ID %#x\n", + le32_to_cpu(request->clock_id)); + rate = 0; + } + + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + +/* + * D400 (get_property) is left unhandled: the zeroed reply means "no such + * property". The firmware then runs without a panel power model, as it does + * for its other optional power properties. + */ + +/* + * H17P callback numbering. It is not a uniform shift of the v13.5 table: the + * service-creation block is D108..D113 exactly as on v13.5, and the swap + * completion is D590. A callback with no entry here is answered with a + * zeroed reply by dcpep_handle_cb(). + */ +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props_h17p, /* 0x54/0x50, identity scale */ + [100] = iomfbep_cb_match_pmu_service, /* match_pmu_service */ + [101] = trampoline_zero, + [102] = trampoline_nop, /* set_number_property */ + [104] = trampoline_nop, /* set_boolean_property */ + [107] = trampoline_nop, + [108] = trampoline_true, /* create_provider_service */ + [109] = trampoline_true, /* create_product_service */ + [110] = trampoline_true, /* create_PMU_service */ + [111] = trampoline_true, /* create_iomfb_service */ + [112] = trampoline_create_backlight_service, /* create_backlight_service */ + [113] = trampoline_true, /* create_nvram_service */ + [114] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [117] = trampoline_nop, /* set_idle_caching_state_ap */ + [118] = trampoline_zero, /* upload_trace_start */ + [119] = trampoline_zero, /* upload_trace_chunk */ + [120] = trampoline_zero, /* upload_trace_end */ + [121] = dcpep_cb_boot_1, /* start_hardware_boot */ + [122] = trampoline_false, /* is_dark_boot */ + [123] = trampoline_false, /* is_waking_from_hibernate */ + [124] = trampoline_zero, /* detect_fastsim */ + [125] = trampoline_read_edt_data, /* read_edt_data */ + [127] = trampoline_prop_start, /* setDCPAVPropStart */ + [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ + [129] = trampoline_prop_end, /* setDCPAVPropEnd */ + [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ + [201] = trampoline_map_piodma, /* map_buf */ + [202] = trampoline_unmap_piodma, /* unmap_buf */ + [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ + [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ + [208] = trampoline_nop, /* update_backlight_factor_prop */ + [209] = trampoline_get_time, /* get_calendar_time_ms */ + [300] = trampoline_pr_publish, + [401] = trampoline_get_uint_prop, /* get_uint_prop */ + [404] = trampoline_nop, /* set_uint_prop */ + [406] = trampoline_set_fx_prop, /* set_fx_prop */ + [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ + [411] = trampoline_map_reg, /* mapDeviceMemoryWithIndex */ + [413] = trampoline_true, /* setProperty */ + [414] = trampoline_sr_set_property_int, /* setProperty */ + [415] = trampoline_true, /* setProperty */ + [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [452] = trampoline_map_physical, /* prepare */ + [454] = trampoline_release_mem_desc, /* release_descriptor */ + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [572] = trampoline_zero, /* powerUpDART */ + [575] = trampoline_hotplug_h17p, + [576] = trampoline_nop, + [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ + [583] = trampoline_nop, /* clear_default_surface */ + [584] = trampoline_nop, /* swap_notify_gated */ + [585] = trampoline_swap_info_h17p, /* swap_info_notify_dispatch */ + [590] = trampoline_swap_complete, /* swap_complete_ap_gated */ + [592] = trampoline_swap_complete_intent_gated, /* swap_complete_intent_gated */ + [593] = trampoline_abort_swap_ap_gated, /* abort_swap_ap_gated */ + [594] = trampoline_enable_backlight_message_ap_gated, + [595] = trampoline_nop, /* setSystemConsoleMode */ + [597] = trampoline_false, /* isDFBAllocated */ + [598] = trampoline_false, + [599] = trampoline_nop, /* find_swap_function_gated */ +}; + +void iomfb_start_h17p(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h new file mode 100644 index 00000000000000..97c0da9e05b940 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright The Asahi Linux Contributors */ + +#ifndef __APPLE_IOMFB_H17P_H__ +#define __APPLE_IOMFB_H17P_H__ + +#include "version_utils.h" + +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) + +#include "iomfb_template.h" + +static_assert(sizeof(struct dcp_swap_h17p) == 0x58a); +static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); +static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); + +#undef DCP_FW_VER +#undef DCP_FW + +#endif /* __APPLE_IOMFB_H17P_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 7c1f6c76b22693..88f310e95ed930 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -72,6 +72,43 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, struct dcp_set_power_state_req, struct dcp_set_power_state_resp); +/* + * H17P firmware expects set_power_state (A473) on the out-of-band command + * channel. Sent on the ordinary command channel it is never answered, which + * strands the power transition and every later call behind it. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_POWER_OOB true +#else +#define DCP_POWER_OOB false +#endif + +/* + * H17P firmware is attached to a display the bootloader has already brought + * up, and it does not accept set_digital_out_mode (A413) for that display: + * the call fails (0x8000000b) and the timing generator never starts, so there + * is no vblank and no swap completion. Keep the mode the bootloader + * programmed, which is the panel's native mode, instead of requesting one. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_INHERIT_BOOT_MODE true +#else +#define DCP_INHERIT_BOOT_MODE false +#endif + +/* + * abort_swaps_dcp and last_client_close have no verified H17P method numbers, + * and the H17P table leaves them out. Rather than send the firmware a record + * it would decode as some other call, power-off and sleep skip both and go + * straight to set_power_state, whose number and sizes are known. Power-off + * still sends its clear swap first. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_HAS_CLIENT_TEARDOWN false +#else +#define DCP_HAS_CLIENT_TEARDOWN true +#endif + DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, struct dcp_set_digital_out_mode_req, u32); @@ -90,6 +127,68 @@ DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * Default-framebuffer queries issued from inside start_hardware_boot (D121). + * The getters take a zeroed input of the expected size; the firmware only + * checks the size. + */ +struct dcp_dfb_info_h17p { + u8 data[0x2c]; +} __packed; + +struct dcp_dfb_layout_req_h17p { + u8 data[0xc]; +} __packed; + +struct dcp_dfb_dimensions_h17p { + u32 width; + u32 height; +} __packed; + +DCP_THUNK_INOUT(dcp_get_dfb_compression_info, dcpep_get_dfb_compression_info, + u64, u64); +DCP_THUNK_INOUT(dcp_get_dfb_info, dcpep_get_dfb_info, + struct dcp_dfb_info_h17p, struct dcp_dfb_info_h17p); +DCP_THUNK_INOUT(dcp_get_dfb_layout, dcpep_get_dfb_layout, + struct dcp_dfb_layout_req_h17p, u64); +DCP_THUNK_INOUT(dcp_get_dfb_state, dcpep_get_dfb_state, u32, u64); +DCP_THUNK_IN(dcp_set_dfb_dimensions, dcpep_set_dfb_dimensions, + struct dcp_dfb_dimensions_h17p); +DCP_THUNK_VOID(dcp_commit_dfb_info, dcpep_commit_dfb_info); +DCP_THUNK_OUT(dcp_dfb_query, dcpep_dfb_query, u32); +DCP_THUNK_OUT(dcp_dfb_ready_query, dcpep_dfb_ready_query, u32); + +/* + * Pipe-configuration calls the firmware expects between the boot chain and + * the first swap. Most of them have no known name; they are identified by + * method number and wire size. + */ +struct dcp_pipe_cfg_415 { + u8 data[0xc]; +} __packed; + +/* + * A031 is update_notify_clients_dcp(u32[27]): an array of the DCP + * notification clients the AP wants delivered. Clients that are not + * registered here never send the callbacks they gate. + */ +struct dcp_pipe_cfg_031 { + __le32 clients[27]; +} __packed; +static_assert(sizeof(struct dcp_pipe_cfg_031) == 0x6c); + +DCP_THUNK_INOUT(dcp_pipe_cfg_415, dcpep_pipe_cfg_415, + struct dcp_pipe_cfg_415, struct dcp_pipe_cfg_415); +DCP_THUNK_IN(dcp_pipe_cfg_031, dcpep_pipe_cfg_031, struct dcp_pipe_cfg_031); +DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); +DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); +DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); +DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_INOUT(dcp_pipe_set_428, dcpep_pipe_set_428, u32, u32); +#endif + DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, @@ -121,6 +220,96 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * Bind every framebuffer displaced by the commit that produced this swap to + * that swap's id. + * + * The id only exists once the firmware answers swap_start, after the atomic + * commit has queued its entries, so they are queued unarmed and stamped here. + * Swap-start requests are serialized by the DCP command stack. + */ +static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + unsigned long flags; + + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { + if (entry->armed) + continue; + entry->swap_id = swap_id; + entry->armed = true; + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); +} + +/* + * Drop framebuffers whose unbinding swap has completed. + * + * Entries must be keyed to the swap that unbinds them, rather than the last + * swap that happened to have completed when the atomic commit ran. A later + * successful completion also retires entries whose own completion was lost; + * the signed difference keeps that comparison valid across a 32-bit wrap. + */ +static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) +{ + struct dcp_fb_reference *entry, *tmp; + unsigned long flags; + LIST_HEAD(done); + + /* + * last_swap_id / have_swap_complete are only ever touched by the RTKit + * worker, which is also the only caller of this function, so the lock + * is here for the list alone. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, flags); + while (!list_empty(&dcp->swapped_out_fbs)) { + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + + /* Newest entries; their swap has not been started yet. */ + if (!entry->armed) + break; + /* last_swap_id is 0 from kzalloc until a swap really lands. */ + if (!dcp->have_swap_complete) + break; + /* Hold the old framebuffer for one additional completed swap. */ + if ((s32)(dcp->last_swap_id - entry->swap_id) <= 0) + break; + + list_move_tail(&entry->head, &done); + } + spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); + + /* drm_framebuffer_put() can sleep, so never call it under the lock. */ + list_for_each_entry_safe(entry, tmp, &done, head) { + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} +#else +static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) { } + +static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) +{ + while (!list_empty(&dcp->swapped_out_fbs)) { + struct dcp_fb_reference *entry; + + entry = list_first_entry(&dcp->swapped_out_fbs, + struct dcp_fb_reference, head); + if (entry->swap_id == dcp->last_swap_id) + break; + if (entry->fb) + drm_framebuffer_put(entry->fb); + list_del(&entry->head); + kfree(entry); + } +} +#endif + static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { @@ -130,6 +319,10 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp_swap_watchdog_complete(dcp); dcp_drm_crtc_page_flip(dcp, now); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp->have_swap_complete = true; + dcp_release_swapped_out_fbs(dcp); +#endif if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -154,6 +347,94 @@ static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *o return false; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P asks the AP to make the default framebuffer surface (D582). The + * firmware expects the AP to register a surface from inside that callback, + * with a nested A472 carrying a 0x22c-byte surface descriptor, and only then + * to report success. Without it the firmware has no surface, programs the + * display engine from a null base and the piodma DART faults. + */ +struct dcp_h17p_dfb_surface_req { + u8 surface[0x22c]; + u8 surface_null; + u8 padding[3]; +} __packed; +static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); + +/* + * A472 has a 4-byte output; with out_len 0 the firmware has nowhere to write + * its result and stops answering. + */ +DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, + struct dcp_h17p_dfb_surface_req, u32); + +static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * Fill the default surface descriptor. Fields not set here are zero. The + * format at +0x0b is the fourcc 'b3a8', stored byte-reversed; +0x35 is the + * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose + * meaning is not known. + */ +static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) +{ + s[0x002] = 0x01; + memcpy(s + 0x00b, "8a3b", 4); + s[0x019] = 0x01; + s[0x01b] = 0x01; + s[0x01c] = 0x01; + s[0x02b] = 0x4d; + s[0x02c] = 0x01; + s[0x035] = 0x02; + s[0x051] = 0x01; + s[0x149] = 0x01; +} + +/* + * A471 (update_dfb) is never sent: on H17P it makes the coprocessor take an + * AXI read error. Scanout does not need it, since every swap carries its own + * surface. Just acknowledge the callback. + */ +static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_ack_dfb_surface(dcp); +} + +static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + struct dcp_h17p_dfb_surface_req *req; + + trace_iomfb_callback(dcp, tag, __func__); + + /* + * The request is too large for the stack this deep in the RTKit + * receive work (rx work -> iomfb_recv_msg -> dcpep_handle_cb -> here -> + * dcp_push). dcp_push() copies the payload into shared memory before + * returning, so a short-lived allocation is enough. + */ + req = kzalloc_obj(*req); + if (!req) + return true; + + dcp_fill_dfb_surface(dcp, req->surface); + + dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); + kfree(req); + + /* deferred ACK: the A472 completion reports success for us */ + return false; +} +#endif + static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_channel *ch = &dcp->ch_cb; @@ -299,12 +580,30 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, goto reject; } - return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* remembered so release_mem_desc() can tear a stale mapping down */ + memdesc->piodma_mapped = true; + + /* the H17P reply echoes the buffer index */ + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = memdesc->dva, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; +#endif reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); - return (struct dcp_map_buf_resp){ .ret = EINVAL }; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + .buffer = req->buffer, + .dva = 0, + }; +#else + return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; +#endif } static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, @@ -380,6 +679,16 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, size); resp.dva = memdesc->dva; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P firmware also needs the physical address: it loads the APT M3 + * microcode through it, and with a zero paddr the timing sequencer + * comes up and is powered straight back down. + */ + if (memdesc->map.sgl) + resp.paddr = sg_phys(memdesc->map.sgl); +#endif + return resp; } @@ -403,6 +712,23 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) memdesc = &dcp->memdesc[id]; size = ALIGN(memdesc->size, SZ_16K); + + /* + * Tear down any translation still standing before the pages go back to + * the allocator. The firmware is expected to unmap piodma first, but + * if it releases a descriptor without doing so we would hand the pages + * back while the DART still resolves them -- the display engine would + * then read whoever owns them next. Only the H17P map_piodma handler + * sets piodma_mapped, so older firmware is unaffected. + */ + if (memdesc->piodma_mapped) { + dev_warn(dcp->dev, + "releasing buffer %u while still mapped for piodma; unmapping first\n", + id); + iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + memdesc->piodma_mapped = false; + } + if (memdesc->buf) { dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; @@ -506,6 +832,17 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; } else { struct resource *rsrc = dcp->disp_registers[req->index]; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P reaches these apertures physically, not through its + * DART, and expects dva == addr in the reply. Trying to + * dma_map_resource() them instead fails and hands the firmware + * DMA_MAPPING_ERROR as a device address. + */ + dma_addr_t dva = rsrc->start; +#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), + DMA_BIDIRECTIONAL, 0); #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) dma_addr_t dva = dcpep_map_reg_dva(dcp, req->index, rsrc); WARN_ON(dva == DMA_MAPPING_ERROR); @@ -561,6 +898,19 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) return false; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * This length comes straight off the wire. A bogus value would turn + * the allocation below into a multi-gigabyte GFP_KERNEL request. + * Property blobs are a few KiB, so bound it. + */ + if (*length == 0 || *length > SZ_1M) { + dev_warn(dcp->dev, "rejecting %u byte property transfer\n", + *length); + return false; + } +#endif + if (!dcp_property_size_valid(*length)) return false; @@ -688,7 +1038,17 @@ static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); } -static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A380 right before set_display_refresh_properties */ + dcp_dfb_ready_query(dcp, false, boot_5b, NULL); +#else + boot_5b(dcp, out, cookie); +#endif +} + +static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u32 v_true = 1; @@ -698,6 +1058,16 @@ static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) #endif } +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A033 between flush_supports_power and late_init_signal */ + dcp_dfb_query(dcp, false, boot_4b, NULL); +#else + boot_4b(dcp, out, cookie); +#endif +} + static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) { u32 v_true = true; @@ -710,11 +1080,103 @@ static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) dcp_setup_video_limits(dcp, false, boot_3, NULL); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects the default framebuffer to be described between + * create_default_fb and setup_video_limits: + * get_dfb_compression_info -> get_dfb_info -> get_dfb_layout -> + * get_dfb_state -> set_dfb_dimensions(w, h) -> commit_dfb_info + * Without it the firmware never finishes its own default framebuffer setup. + */ +static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_commit_dfb_info(dcp, false, boot_2, NULL); +} + +static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_dimensions_h17p dim; + + /* + * get_dfb_state returns the default framebuffer dimensions, which are + * passed back unchanged to set_dfb_dimensions. + */ + memcpy(&dim, out, sizeof(dim)); + if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); + return; + } + dev_dbg(dcp->dev, "default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); +} + +static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); +} + +static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); +} + +static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_info_h17p info = {}; + + dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); +} + +static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) +{ + u64 zero = 0; + + dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); +} +#endif + static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp_create_default_fb(dcp, false, boot_1_6, NULL); +#else dcp_create_default_fb(dcp, false, boot_2, NULL); +#endif +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +static void d000_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); } +/* + * The firmware expects D000 to be answered only after a nested A033 + * (dfb_query), and then with 1. Defer the ACK until that call completes. + */ +static bool dcpep_cb_d000_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_dfb_query(dcp, false, d000_done, NULL); + return false; +} +#endif + /* Use special function signature to defer the ACK */ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -762,6 +1224,46 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) return rt_bw; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P's D006 is 0x54 in / 0x50 out, far wider than the 28-byte + * frame_sync_props of older firmware, so the fields that matter sit past + * anything the old struct could describe. + * + * The firmware expects zeros except for three u32 at 0x24, 0x28 and 0x2c, + * each 0x10000 -- 1.0 in 16.16 fixed point, the identity scale. An all-zero + * reply hands the pipe a zero scale, and it then programs a null surface + * base: the display DART faults at DVA 0 and the coprocessor takes an AXI + * error. + */ +#define DCP_FRAME_SYNC_IDENTITY 0x10000 + +struct dcp_set_frame_sync_props_req_h17p { + u8 unk[0x54]; +} __packed; + +struct dcp_set_frame_sync_props_resp_h17p { + u8 unk[0x50]; +} __packed; + +static struct dcp_set_frame_sync_props_resp_h17p +dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req_h17p *req) +{ + struct dcp_set_frame_sync_props_resp_h17p resp = {}; + static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(identity_at); i++) { + u32 v = DCP_FRAME_SYNC_IDENTITY; + + memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); + } + + return resp; +} +#endif + static struct dcp_set_frame_sync_props_resp dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, struct dcp_set_frame_sync_props_req *req) @@ -789,6 +1291,71 @@ static void release_swap_cookie(struct kref *ref) kfree(cookie); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_SWAP_FLAGS1 0x34865202ULL +#define DCP_SWAP_FLAGS2 0x4ULL + +/* + * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: + * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier + * than on every other firmware. With the older layout the firmware reads the + * format two bytes late and its surface check rejects the swap as an + * unsupported format. + * + * Shift each surface's tail down by two rather than fork the shared struct. + */ +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + unsigned int i; + + /* + * The firmware expects flags1 = 0x34865202 and flags2 = 4 (at +0x40 + * and +0x48) in every swap. + */ + req->swap.flags1 = DCP_SWAP_FLAGS1; + req->swap.flags2 = DCP_SWAP_FLAGS2; + + /* + * The H17P gap before swap_id carries two small counters the firmware + * expects to be 2 (at +0x50) and 3 (at +0x78). The gap starts at + * +0x50, so these are indices 0 and 0x28 of the blob. + */ + req->swap.h17p_pre_swap_id[0x00] = 2; + req->swap.h17p_pre_swap_id[0x28] = 3; + + for (i = 0; i < SWAP_SURFACES; i++) { + u8 *p = (u8 *)&req->surf[i]; + size_t n = sizeof(req->surf[i]); + + memmove(p + 1, p + 3, n - 3); + memset(p + n - 2, 0, 2); + } + + /* + * Everything from surf_iova onward also sits two bytes earlier on + * H17P: surf_iova[0] is at +0xe38 of the 0xe9c-byte record, not + * +0xe3a. Left in place, the firmware reads every IOVA two bytes off + * and the trailing null-flag block misaligned, so it faults on a bogus + * address or verifies layers that are not present. Shift the whole + * block down. + */ + { + u8 *r = (u8 *)req; + size_t n = sizeof(*req); + size_t off = offsetof(typeof(*req), surf_iova); + + static_assert(offsetof(typeof(*req), surf_iova) == 0xe3a, + "H17P source IOVA offset"); + static_assert(sizeof(*req) == 0xe9c, "H17P request size"); + memmove(r + off - 2, r + off, n - off); + memset(r + n - 2, 0, 2); + } +} +#else +static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } +#endif + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -823,12 +1390,14 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, { struct dcp_swap_start_resp *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); if (cookie) { struct dcp_swap_cookie *info = cookie; info->swap_id = resp->swap_id; } + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -883,9 +1452,19 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) kref_get(&cookie->refcount); if (dcp->main_display) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P takes set_display_device (A411) once, in the + * client-open sequence. Repeating it after the client is open + * is a call the firmware stops answering, which wedges the + * command channel and with it every later swap. + */ + dcp_on_set_power_state(dcp, NULL, cookie); +#else handle = 0; dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, cookie); +#endif } else { handle = 2; dcp_set_display_device(dcp, false, &handle, @@ -987,6 +1566,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) for (int l = 0; l < 5; l++) swap->surf2_null[l] = true; +#endif swap->unkU32Ptr_null = true; swap->unkU32out_null = true; #endif @@ -1165,7 +1745,19 @@ dcpep_cb_get_tiling_state(struct apple_dcp *dcp, static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P asks during start_ios_gated, before any mode is known, but + * dcp_has_panel() only becomes true once the panel dimensions have been + * parsed. An internal panel always has a backlight, so answer for the + * connector; answering false makes the firmware report that it could + * not find the backlight service. + */ + return dcp_has_panel(dcp) || + dcp->connector_type == DRM_MODE_CONNECTOR_eDP; +#else return dcp_has_panel(dcp); +#endif } TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); @@ -1208,6 +1800,12 @@ TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, struct dcp_set_frame_sync_props_req, struct dcp_set_frame_sync_props_resp); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props_h17p, + dcpep_cb_set_frame_sync_props_h17p, + struct dcp_set_frame_sync_props_req_h17p, + struct dcp_set_frame_sync_props_resp_h17p); +#endif TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); @@ -1259,8 +1857,10 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) struct dcp_swap_start_resp *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + dcp_arm_swapped_out_fbs(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } @@ -1294,6 +1894,19 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) } } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in + * 16.16 fixed point), the same identity value the D006 reply carries. + */ +static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 identity = 0x10000; + + dcp_pipe_set_428(dcp, false, &identity, do_swap, cookie); +} +#endif + static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, void *cookie) { @@ -1443,6 +2056,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) for (l = 0; l < 5; l++) req->surf2_null[l] = true; +#endif req->unkU32Ptr_null = true; req->unkU32out_null = true; #endif @@ -1481,10 +2095,27 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru struct dcp_fb_reference *entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (entry) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + unsigned long flags; + + entry->fb = old_state->fb; + /* + * Left unarmed: the id of the swap that + * unbinds this framebuffer is not known until + * the firmware answers swap_start. + */ + spin_lock_irqsave(&dcp->swapped_out_lock, + flags); + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + spin_unlock_irqrestore(&dcp->swapped_out_lock, + flags); +#else entry->fb = old_state->fb; entry->swap_id = dcp->last_swap_id; list_add_tail(&entry->head, &dcp->swapped_out_fbs); +#endif } drm_framebuffer_get(old_state->fb); } @@ -1577,7 +2208,11 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); +#else iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); +#endif } else do_swap(dcp, NULL, NULL); } @@ -1605,6 +2240,141 @@ static void init_3(struct apple_dcp *dcp, void *out, void *cookie) dcp_is_main_display(dcp, false, res_is_main_display, NULL); } +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * The client-open sequence H17P firmware expects before the first swap, in + * H17P method numbers (dcp_push() applies the H17G aliases): + * + * A445 A411 A445 A415 A450 A031 x9 A414 A478 A457 A474 A478 A476 A478 A412 + * + * A412 is not in the table: its result feeds res_is_main_display(), so it + * stays on the existing path. + */ +enum dcp_client_step_kind { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A415, DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, + DCP_STEP_A474, DCP_STEP_A476, +}; + +static const u8 dcp_client_open_seq[] = { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A445, DCP_STEP_A415, + DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, + DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, DCP_STEP_A474, + DCP_STEP_A478, DCP_STEP_A476, DCP_STEP_A478, +}; + +static void dcp_client_open_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned long idx = (unsigned long)cookie; + dcp_callback_t next = dcp_client_open_run; + void *ncookie = (void *)(idx + 1); + + if (idx >= ARRAY_SIZE(dcp_client_open_seq)) { + init_3(dcp, NULL, NULL); + return; + } + + switch (dcp_client_open_seq[idx]) { + case DCP_STEP_A445: { + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, next, ncookie); + break; + } + case DCP_STEP_A411: { + u32 handle = 0; + + dcp_set_display_device(dcp, false, &handle, next, ncookie); + break; + } + case DCP_STEP_A415: { + struct dcp_pipe_cfg_415 cfg = {}; + + dcp_pipe_cfg_415(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A450: { + u32 val = 0; + + dcp_enable_disable_video_power_savings(dcp, false, &val, next, + ncookie); + break; + } + case DCP_STEP_A031: { + /* + * Nine cumulative states: each call repeats the previous set + * of notification clients and adds one, so state n lists n + * clients. + */ + static const u8 notify_clients[9][8] = { + {}, + { 6 }, + { 6, 7 }, + { 6, 7, 8 }, + { 6, 7, 8, 10 }, + { 6, 7, 8, 10, 11 }, + { 6, 7, 8, 10, 11, 12 }, + { 6, 7, 8, 10, 11, 12, 25 }, + { 6, 7, 8, 10, 11, 12, 25, 26 }, + }; + struct dcp_pipe_cfg_031 cfg = {}; + unsigned int step = dcp->a031_step; + unsigned int i; + + if (step >= ARRAY_SIZE(notify_clients)) + step = ARRAY_SIZE(notify_clients) - 1; + for (i = 0; i < step; i++) + cfg.clients[i] = cpu_to_le32(notify_clients[step][i]); + dcp->a031_step++; + + dcp_pipe_cfg_031(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, next, ncookie); + break; + } + case DCP_STEP_A478: + dcp_pipe_query_478(dcp, false, next, ncookie); + break; + case DCP_STEP_A457: + dcp_first_client_open(dcp, false, next, ncookie); + break; + case DCP_STEP_A474: + dcp_pipe_query_474(dcp, false, next, ncookie); + break; + case DCP_STEP_A476: { + /* + * A476 takes a single u32, which the firmware expects to be 1. + * Its name is not known for certain: the shape (in 4, out 0) + * fits both set_dcp_clamshellstate() and + * set_has_frame_swap_function(). + */ + u32 state = 1; + + dcp_pipe_set_476(dcp, false, &state, next, ncookie); + break; + } + default: + dev_warn(dcp->dev, "unknown client-open step %u\n", + dcp_client_open_seq[idx]); + init_3(dcp, NULL, NULL); + break; + } +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + /* every client-open sequence starts from the empty A031 state */ + dcp->a031_step = 0; + dcp_client_open_run(dcp, NULL, (void *)0UL); +} +#else static void init_2(struct apple_dcp *dcp, void *out, void *cookie) { dcp_first_client_open(dcp, false, init_3, NULL); @@ -1615,6 +2385,7 @@ static void init_1(struct apple_dcp *dcp, void *out, void *cookie) u32 val = 0; dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); } +#endif static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) { diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 6f5824c8043f3b..2b4f476eabf403 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -28,6 +28,17 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_v14_7[0x48]; #endif +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P inserts 0x48 bytes between flags2 and the swap ID, which moves + * to +0x98. Every following field shifts with it: surf_ids +0x9c, + * src_rect +0xac, surf_flags +0xec, dst_rect +0x10c, + * swap_enabled/completed +0x14c/+0x150, bg_color +0x154. The gap holds + * two small u32 counters (at +0x50 and +0x78) that are not modelled. + */ + u8 h17p_pre_swap_id[0x48]; +#endif + u32 swap_id; u32 surf_ids[SWAP_SURFACES]; @@ -73,6 +84,13 @@ struct DCP_FW_NAME(dcp_swap) { #endif u64 unk_1; #endif +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * The H17P A408 request is 0xe9c bytes; with a 0x22c-byte surface, + * SWAP_SURFACES = 4 and no surf2 block, that leaves this tail. + */ + u8 h17p_tail[0xda]; +#endif } __packed; /* Information describing a surface */ @@ -93,8 +111,14 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u64 surf_iova[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u64 unk_u64_a[SWAP_SURFACES]; + /* + * H17P carries no surf2 block: its trailing u8 null-flag run is 8 bytes + * (swap_null + surf_null[4] + three tail bools). + */ +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) struct DCP_FW_NAME(dcp_surface) surf2[5]; u64 surf2_iova[5]; +#endif #endif u8 unkbool; u64 unkdouble; @@ -112,14 +136,18 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { u8 swap_null; u8 surf_null[SWAP_SURFACES]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 surf2_null[5]; +#endif #endif u8 unkoutbool_null; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u8 unkU32Ptr_null; u8 unkU32out_null; #endif +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; +#endif } __packed; struct DCP_FW_NAME(dcp_swap_submit_resp) { @@ -131,9 +159,104 @@ struct DCP_FW_NAME(dcp_swap_submit_resp) { u8 padding[3]; } __packed; -struct DCP_FW_NAME(dc_swap_complete_resp) { +/* + * A406 swap_start / A473 set_power_state. H17P shrank both requests: the + * 0x10-byte struct dcp_iouserclient became a bare u64 handle and the + * swap_start response no longer echoes it. swap_start is 0x10 in / 0x8 out, + * set_power_state 0x10 in. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +struct DCP_FW_NAME(dcp_swap_start_req) { + u32 swap_id; + u64 client; /* IOUserClient kernel VA */ + u8 client_null; + u8 padding[3]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; + u32 unkint; /* widened from u8 on H17P */ + u8 unkint_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_req)) == 0x10); +static_assert(sizeof(struct DCP_FW_NAME(dcp_swap_start_resp)) == 0x8); +static_assert(sizeof(struct DCP_FW_NAME(dcp_set_power_state_req)) == 0x10); +#else +struct DCP_FW_NAME(dcp_swap_start_req) { u32 swap_id; + struct dcp_iouserclient client; + u8 swap_id_null; + u8 client_null; + u8 padding[2]; +} __packed; + +struct DCP_FW_NAME(dcp_swap_start_resp) { + u32 swap_id; + struct dcp_iouserclient client; + u32 ret; +} __packed; + +struct DCP_FW_NAME(dcp_set_power_state_req) { + u64 unklong; u8 unkbool; + u8 unkint_null; + u8 padding[2]; +} __packed; +#endif + +/* + * D201 map_piodma reply. H17P declares a 16-byte output, not 20, laid out as + * { u32 buffer index, u64 dva, u32 pad } rather than { u64 vaddr, u64 dva, + * u32 ret }. With the old layout the DVA lands four bytes late and the + * display engine DMAs to a bogus address. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +struct DCP_FW_NAME(dcp_map_buf_resp) { + u32 buffer; + u64 dva; + u32 unk; +} __packed; +static_assert(sizeof(struct DCP_FW_NAME(dcp_map_buf_resp)) == 0x10); +#else +struct DCP_FW_NAME(dcp_map_buf_resp) { + u64 vaddr; + u64 dva; + u32 ret; +} __packed; +#endif + +struct DCP_FW_NAME(dc_swap_complete_resp) { + u32 swap_id; /* +0x000 */ + u8 unkbool; /* +0x004 */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P: swap_complete_ap_gated(u32, bool, SwapCompleteData const *, + * SwapInfoBlob const *, u32, bool) is 0x730 bytes on the wire, 89 more + * than the pre-H17P record: the SwapCompleteData grew from 8 to 0x22, + * swap_info became eight 0xe0-byte SwapInfoBlobs rather than one + * 0x6c5 blob, and the tail gained a flag and three pad bytes. Only + * swap_info_count of those blobs are populated; the rest are + * uninitialised DCP buffer memory. + */ + u32 hol_swap_ids[4]; /* +0x005 */ + u32 hol_arg; /* +0x015 */ + u32 hol_count; /* +0x019 */ + u8 hol_valid; /* +0x01d */ + u64 displayed_data; /* +0x01e, unaligned */ + u8 notify_displayed; /* +0x026 */ + u8 swap_info[8][0xe0]; /* +0x027, blob[0]+0x38 == swap_id */ + u32 swap_info_count; /* +0x727, max 8 */ + u8 completion_flag; /* +0x72b */ + u8 swap_data_null; /* +0x72c, 0 => data at +0x005 valid */ + u8 padding[3]; /* +0x72d, to sizeof == 0x730 */ +#else u64 swap_data; #if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) u8 swap_info[0x6c4]; @@ -142,6 +265,7 @@ struct DCP_FW_NAME(dc_swap_complete_resp) { #endif u32 unkint; u8 swap_info_null; +#endif } __packed; struct DCP_FW_NAME(dcp_map_reg_req) { diff --git a/drivers/gpu/drm/apple/iomfb_v12_3.c b/drivers/gpu/drm/apple/iomfb_v12_3.c index 092a3878562e40..cae54a1d1a2755 100644 --- a/drivers/gpu/drm/apple/iomfb_v12_3.c +++ b/drivers/gpu/drm/apple/iomfb_v12_3.c @@ -4,6 +4,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7_tpl.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_v13_3.c b/drivers/gpu/drm/apple/iomfb_v13_3.c index 05624058537f8d..f47505fbbdc310 100644 --- a/drivers/gpu/drm/apple/iomfb_v13_3.c +++ b/drivers/gpu/drm/apple/iomfb_v13_3.c @@ -4,6 +4,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7_tpl.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 689df138df9a14..20ced0f826ed3c 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -770,7 +770,7 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, } int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit) + const char **class, s64 *unit, bool h17p_keys) { int ret = 0; struct iterator it; @@ -789,19 +789,36 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, break; } - if (!strcmp(key, "EPICName")) { + /* + * H17P advertises its system service with lower-case keys -- + * "compartment", "name", "interface-id" -- where earlier + * firmware uses EPICName/EPICProviderClass/EPICUnit. Without + * them the service never registers, the firmware has no system + * service to ask and idles the panel, so no vblank and no swap + * ever completes. Only H17P callers accept them. If a + * dictionary carries both spellings, the later value wins and + * the earlier string is freed. + */ + if (!strcmp(key, "EPICName") || + (h17p_keys && !strcmp(key, "compartment"))) { + if (parsed_name) + kfree(*name); *name = parse_string(it.handle); if (IS_ERR(*name)) ret = PTR_ERR(*name); else parsed_name = true; - } else if (!strcmp(key, "EPICProviderClass")) { + } else if (!strcmp(key, "EPICProviderClass") || + (h17p_keys && !strcmp(key, "name"))) { + if (parsed_class) + kfree(*class); *class = parse_string(it.handle); if (IS_ERR(*class)) ret = PTR_ERR(*class); else parsed_class = true; - } else if (!strcmp(key, "EPICUnit")) { + } else if (!strcmp(key, "EPICUnit") || + (h17p_keys && !strcmp(key, "interface-id"))) { ret = parse_int(it.handle, unit); if (!ret) parsed_unit = true; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 18919ed777b307..0dfca73de4b4ef 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -135,7 +135,7 @@ int replace_modes(struct dcp_parse_ctx *handle, int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm, bool *backlight_control); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit); + const char **class, s64 *unit, bool h17p_keys); struct dcp_sound_format_mask { u64 formats; /* SNDRV_PCM_FMTBIT_* */ From 8590a9d3b6ca9d463f8b060ddfffd78e3fb9bd69 Mon Sep 17 00:00:00 2001 From: Acelogic Date: Fri, 2 Oct 2026 23:34:51 -0400 Subject: [PATCH 015/631] arm64: dts: apple: t8142-j813: add the internal display Describe the J813 display coprocessor (DCP), its mailbox and both display DARTs, with resources read from this machine's Apple Device Tree, plus the dcp/disp0/disp0_piodma aliases the loader uses to hand over the live firmware mappings. The DCP and display DART stay disabled until the loader has reserved those mappings and enables them. Enable the Apple DRM driver in the J813 configuration and document the loader contract and the tested result. Tested on J813 from RAM with public m1n1 and U-Boot plus their J813 display and MTP companion changes: DCP completed startup and drove the internal panel at 2560x1600 (notch hidden) through appledrmfb, DPMS off/on cycles completed, and a KDE Plasma Wayland session ran with the built-in keyboard and trackpad. Rendering is software-only. --- Documentation/arch/arm64/apple-j813.rst | 40 +++++++- .../boot/dts/apple/t8142-j813-display.dtsi | 95 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8142-j813.dts | 1 + arch/arm64/configs/j813.config | 4 + 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 arch/arm64/boot/dts/apple/t8142-j813-display.dtsi diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index 69d28107a9d09a..ff02f1cb91ac94 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -173,7 +173,7 @@ an Image build or successful loader exit alone is not a boot result. Review captures for device identifiers and boot entropy before publishing. These boot tests do not establish CPU hotplug, KVM guest timers, -sustained thermal behavior, suspend, native DCP, GPU acceleration, USB, +sustained thermal behavior, suspend, GPU acceleration, USB, storage, networking or audio support. Input and monitoring require their separate checks below. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. @@ -216,7 +216,9 @@ the keyboard or temperature monitor from registering. The loader must preserve the iBoot-loaded MTP image and SRAM, prepare its DMA protection, and leave MTP ready for Linux's RTKit startup. A prior-stage MTP client must complete a compatible shutdown before Linux starts; a partially -initialized or crashed coprocessor is not a supported handoff. +initialized or crashed coprocessor is not a supported handoff. On J813, U-Boot +must not start and stop MTP before Linux: Linux's later restart crashes the +firmware and takes the keyboard and trackpad down. The backlight uses the existing PWM and LED drivers. Its boot brightness is retained when the initial PWM period is valid, otherwise it starts off. @@ -240,3 +242,37 @@ proof. Preserve stdout, the source revision and SHA-256 hashes of the exact Image, DTB, configuration, initramfs and bootloaders as the repeatable result. Keep firmware and raw machine data private. Without physical events the input check fails rather than crediting device registration as working input. + +Native display +============== + +The internal panel is driven by the display coprocessor (DCP) through the +Apple DRM driver. iBoot leaves DCP running with H17-generation (macOS 26) +firmware mapped through locked DARTs. Linux attaches to that live session: +it sends the standard RTKit wake handshake and never stops, resets or reloads +the coprocessor. No firmware file is loaded by Linux. + +The loader must: + +* reserve every live DCP firmware segment and the boot framebuffer, and + export their IOVAs to the ``dcp``, ``disp0`` and ``disp0_piodma`` nodes; +* enable the display DART (the first ``iommus`` entry of ``disp0``) and the + ``dcp`` node only after doing so. + +The display DARTs keep the firmware's root page-table slots. Linux publishes +its own translations only into vacant slots. On ``apple,t8142-dart``, a +firmware slot is accepted when it already translates every page Linux mapped +there to the same physical address, and later Linux mappings into it must +match the firmware's translation. Firmware buffers DCP later asks Linux to +share are resolved through those firmware tables. + +The DCP firmware reports the panel dimensions at startup; the driver does not +embed a panel timing or geometry. The J813 panel reports 2560x1664; the driver +hides its 64-pixel notch area by default, so the mode is 2560x1600 at 60 Hz. +The primary plane scans out IOMFB surface 2. Backlight control and a cursor +plane are not provided yet. + +Boot the RAM payload as above with ``CONFIG_DRM_APPLE``. On J813 this reached +``DCP booted``, registered ``appledrmfb`` as the console framebuffer, and ran +a KDE Plasma Wayland session with the built-in keyboard and trackpad. The +desktop renders in software; GPU acceleration is separate work. diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi new file mode 100644 index 00000000000000..a50fa38fd9265f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J813 internal display pipeline: the DCP coprocessor and its DARTs. + * + * The bootloader leaves DCP running with its firmware mapped through + * locked DART tables. The loader reserves those mappings and enables the + * nodes below; Linux attaches to the running coprocessor without resetting + * it. Resources were read from this machine's Apple Device Tree. + */ + +/ { + aliases { + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + }; + + display: display-subsystem { + compatible = "apple,display-subsystem"; + /* disp0_dart must be first since it is locked */ + iommus = <&disp0_dart 0>; + /* generate phandle explicitly for use in loader */ + phandle = <&display>; + }; +}; + +&{/soc} { + pmgr_dcp: power-management@382720000 { + reg = <3 0x82720000 0 0x8>, <3 0x827a4000 0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + dcp_mbox: mailbox@286e08000 { + compatible = "apple,t8142-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <2 0x86e08000 0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + disp0_dart: iommu@286300000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <2 0x86300000 0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + status = "disabled"; + }; + + dcp_dart: iommu@286340000 { + compatible = "apple,t8142-dart", "apple,t8110-dart"; + reg = <2 0x86340000 0 0x20000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x10 0x0>; + }; + + dcp: dcp@286e00000 { + compatible = "apple,t8142-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + reg = <2 0x86e00000 0 0x4000>, + <2 0x84000000 0 0xb1c000>, + <2 0x86240000 0 0x10000>, + <2 0x863ac000 0 0x4000>, + <2 0x86800000 0 0x800000>, + <3 0x82700000 0 0x114000>, + <2 0x86850000 0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + /* Scan out the primary plane on IOMFB surface 2. */ + apple,iomfb-surfaces = <0 0 1 0>; + clocks = <&clkref>; + /* generate phandle explicitly for use in loader */ + phandle = <&dcp>; + status = "disabled"; + /* required bus properties for 'piodma' subdevice */ + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8142-j813.dts b/arch/arm64/boot/dts/apple/t8142-j813.dts index be04fd061a8854..20df27fa1bfe3b 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813.dts +++ b/arch/arm64/boot/dts/apple/t8142-j813.dts @@ -291,5 +291,6 @@ }; }; +#include "t8142-j813-display.dtsi" #include "t8142-j813-peripherals.dtsi" #include "t8142-j813-usb.dtsi" diff --git a/arch/arm64/configs/j813.config b/arch/arm64/configs/j813.config index 57fb42b0295c51..9091e74f3e9187 100644 --- a/arch/arm64/configs/j813.config +++ b/arch/arm64/configs/j813.config @@ -45,6 +45,10 @@ CONFIG_FB=y CONFIG_FB_DEVICE=y CONFIG_FB_SIMPLE=y CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_DRM=y +CONFIG_DRM_APPLE=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_FBDEV_EMULATION=y # CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER is not set CONFIG_LOG_BUF_SHIFT=20 CONFIG_IKCONFIG=y From eab410a09559158cd2121088e3483289491e0c81 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 016/631] mm: make VM operations aware of per-process mapping sizes Separate target-mm virtual geometry from native physical pages across faults, COW, reclaim, swap, GUP, VMAs and accounting. Supply explicit fragment interfaces and keep native defaults until the optional backend is enabled later in the series. Signed-off-by: Diljit Singh (cherry picked from commit 81fd8e0b21629d0fdcfd36bed3c684cf278b6113) --- arch/x86/kernel/uprobes.c | 6 +- fs/proc/array.c | 3 +- fs/proc/task_mmu.c | 295 ++++- fs/userfaultfd.c | 57 +- include/asm-generic/pgalloc.h | 17 +- include/asm-generic/tlb.h | 43 +- include/linux/binfmts.h | 6 + include/linux/coredump.h | 13 +- include/linux/highmem.h | 15 + include/linux/huge_mm.h | 4 + include/linux/khugepaged.h | 8 + include/linux/mm.h | 104 +- include/linux/mm_granule.h | 245 ++++ include/linux/mm_subpage.h | 277 ++++ include/linux/mm_types.h | 31 +- include/linux/mman.h | 11 + include/linux/mmu_context.h | 5 + include/linux/page_table_check.h | 4 + include/linux/pagemap.h | 5 +- include/linux/pgtable.h | 98 +- include/linux/rmap.h | 106 +- include/linux/sched/mm.h | 4 + include/linux/sched/signal.h | 2 +- include/linux/shmem_fs.h | 51 + include/linux/swap.h | 26 +- include/linux/syscalls.h | 5 + include/linux/uprobes.h | 1 + include/trace/events/kmem.h | 2 +- include/trace/events/mmap.h | 2 +- include/trace/events/oom.h | 9 +- include/uapi/linux/prctl.h | 18 + ipc/shm.c | 36 +- kernel/events/core.c | 83 +- kernel/events/internal.h | 37 +- kernel/events/ring_buffer.c | 47 +- kernel/events/uprobes.c | 212 ++- kernel/fork.c | 35 +- kernel/futex/core.c | 57 +- kernel/sched/fair.c | 6 +- kernel/sys.c | 52 +- kernel/trace/trace_events_user.c | 36 +- kernel/tsacct.c | 16 +- lib/vdso/datastore.c | 3 +- mm/Kconfig | 30 +- mm/damon/ops-common.c | 10 +- mm/filemap.c | 73 +- mm/gup.c | 590 ++++++++- mm/huge_memory.c | 11 +- mm/internal.h | 87 +- mm/khugepaged.c | 119 +- mm/ksm.c | 9 +- mm/madvise.c | 156 ++- mm/memory-failure.c | 45 + mm/memory.c | 2071 +++++++++++++++++++++++++----- mm/mempolicy.c | 57 +- mm/migrate.c | 85 +- mm/migrate_device.c | 2 +- mm/mincore.c | 43 +- mm/mlock.c | 42 +- mm/mm_init.c | 3 +- mm/mmap.c | 139 +- mm/mmu_gather.c | 67 +- mm/mprotect.c | 143 ++- mm/mremap.c | 89 +- mm/mseal.c | 4 +- mm/msync.c | 6 +- mm/oom_kill.c | 25 +- mm/page_idle.c | 7 +- mm/page_io.c | 61 +- mm/page_table_check.c | 142 +- mm/page_vma_mapped.c | 61 +- mm/pagewalk.c | 49 +- mm/pgtable-generic.c | 35 +- mm/process_vm_access.c | 96 +- mm/rmap.c | 519 +++++++- mm/shmem.c | 512 +++++++- mm/swap.h | 22 +- mm/swap_state.c | 6 +- mm/swapfile.c | 387 +++++- mm/truncate.c | 2 + mm/user-subpage-internal.h | 69 + mm/userfaultfd.c | 518 +++++++- mm/util.c | 106 +- mm/vma.c | 180 ++- mm/vma.h | 40 +- mm/vma_exec.c | 6 +- mm/vma_init.c | 5 +- mm/vmscan.c | 124 +- 88 files changed, 7671 insertions(+), 1275 deletions(-) create mode 100644 include/linux/mm_granule.h create mode 100644 include/linux/mm_subpage.h create mode 100644 mm/user-subpage-internal.h diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d38d..3155eed81c2ecb 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -926,13 +926,15 @@ static int is_call_insn(uprobe_opcode_t *insn) * Verification callback used by int3_update uprobe_write calls to make sure * the underlying instruction is as expected - either int3 or call. */ -static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, +static int verify_insn(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; uprobe_opcode_t old_opcode[5]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, page_offset - (vaddr - ctx->base), + (uprobe_opcode_t *) &old_opcode, 5); switch (ctx->expect) { case EXPECT_SWBP: diff --git a/fs/proc/array.c b/fs/proc/array.c index 479ea8cb4ef43d..5b45506942991e 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -610,7 +610,8 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, seq_put_decimal_ull(m, " ", 0); seq_put_decimal_ull(m, " ", start_time); seq_put_decimal_ull(m, " ", vsize); - seq_put_decimal_ull(m, " ", mm ? get_mm_rss(mm) : 0); + seq_put_decimal_ull(m, " ", mm ? + mm_pages_for_reader(mm, current->mm, get_mm_rss(mm)) : 0); seq_put_decimal_ull(m, " ", rsslim); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->start_code : 1) : 0); seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->end_code : 1) : 0); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 436e1c1d11f41e..8938ae67581001 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -33,7 +34,7 @@ #define SENTINEL_VMA_GATE -2 #define SEQ_PUT_DEC(str, val) \ - seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) + seq_put_decimal_ull_width(m, str, mm_pages_to_kb(mm, val), 8) void task_mem(struct seq_file *m, struct mm_struct *mm) { unsigned long text, lib, swap, anon, file, shmem; @@ -58,15 +59,18 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) hiwater_rss = mm->hiwater_rss; /* split executable areas between text and lib */ - text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); - text = min(text, mm->exec_vm << PAGE_SHIFT); - lib = (mm->exec_vm << PAGE_SHIFT) - text; + text = mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm)); + text = min(text, mm_pages_to_bytes(mm, mm->exec_vm)); + lib = (mm_pages_to_bytes(mm, mm->exec_vm)) - text; swap = get_mm_counter_sum(mm, MM_SWAPENTS); SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); - SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); - SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); + seq_put_decimal_ull_width(m, " kB\nVmLck:\t", + mm_locked_vm_bytes(mm) >> 10, 8); + seq_put_decimal_ull_width(m, " kB\nVmPin:\t", + atomic64_read(&mm->pinned_vm) << (PAGE_SHIFT - 10), 8); SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); @@ -88,20 +92,25 @@ void task_mem(struct seq_file *m, struct mm_struct *mm) unsigned long task_vsize(struct mm_struct *mm) { - return PAGE_SIZE * mm->total_vm; + return mm_pages_to_bytes(mm, mm->total_vm); } unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, unsigned long *text, unsigned long *data, unsigned long *resident) { - *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + + struct mm_struct *reader = current->mm; + unsigned long file = get_mm_counter_sum(mm, MM_FILEPAGES) + get_mm_counter_sum(mm, MM_SHMEMPAGES); - *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) - >> PAGE_SHIFT; - *data = mm->data_vm + mm->stack_vm; - *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); - return mm->total_vm; + unsigned long code = (mm_page_align(mm, mm->end_code) - + (mm->start_code & mm_page_mask(mm))) >> mm_page_shift(mm); + + *shared = mm_pages_for_reader(mm, reader, file); + *text = mm_pages_for_reader(mm, reader, code); + *data = mm_pages_for_reader(mm, reader, mm->data_vm + mm->stack_vm); + *resident = mm_pages_for_reader(mm, reader, + file + get_mm_counter_sum(mm, MM_ANONPAGES)); + return mm_pages_for_reader(mm, reader, mm->total_vm); } #ifdef CONFIG_NUMA @@ -475,7 +484,8 @@ show_map_vma(struct seq_file *m, struct vm_area_struct *vma) dev = inode->i_sb->s_dev; ino = inode->i_ino; - pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; + pgoff = (((loff_t)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); } start = vma->vm_start; @@ -717,7 +727,8 @@ static int do_procmap_query(struct mm_struct *mm, void __user *uarg) if (vma->vm_file) { const struct inode *inode = file_user_inode(vma->vm_file); - karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; + karg.vma_offset = (((__u64)vma->vm_pgoff) << PAGE_SHIFT) + + vma_subpage_offset(vma); karg.dev_major = MAJOR(inode->i_sb->s_dev); karg.dev_minor = MINOR(inode->i_sb->s_dev); karg.inode = inode->i_ino; @@ -857,6 +868,50 @@ const struct file_operations proc_pid_maps_operations = { */ #define PSS_SHIFT 12 +#if defined(CONFIG_MM_SUBPAGE) && (defined(CONFIG_PROC_PAGE_MONITOR) || defined(CONFIG_NUMA)) +/* Snapshot only: rmap and quarter counters can change in other address spaces. */ +static void file_quarter_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + int maps = folio_precise_page_mapcount(page_folio(page), page); + unsigned int small = mm_subpage_file_mapcounts(page, counts); + unsigned int native = maps > small ? maps - small : 0; + unsigned int i; + + for (i = 0; i < MM_SUBPAGES_PER_PAGE; i++) + counts->slots[i] += native; +} + +static unsigned int proc_leaf_mapcount(struct vm_area_struct *vma, pte_t pte, + struct page *page) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts; + unsigned int i, count = 0; + + if (!folio_test_anon(folio)) { + unsigned int first = mm_page_size(vma->vm_mm) < PAGE_SIZE ? + offset_in_page(pte_phys_mm(vma->vm_mm, pte)) / SZ_4K : 0; + unsigned int nr = mm_page_size(vma->vm_mm) / SZ_4K; + + file_quarter_mapcounts(page, &counts); + for (i = first; i < first + nr; i++) + count = max(count, counts.slots[i]); + return count; + } + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + count = mm_subpage_mapcount(slot); + mm_subpage_put(slot); + } + return count; + } + return folio_precise_page_mapcount(folio, page); +} + +#endif + #ifdef CONFIG_PROC_PAGE_MONITOR struct mem_size_stats { unsigned long resident; @@ -913,6 +968,73 @@ static void smaps_page_accumulate(struct mem_size_stats *mss, } } +#ifdef CONFIG_MM_SUBPAGE +static void smaps_account_file_extent(struct mem_size_stats *mss, struct page *page, + unsigned int offset, unsigned long size, bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + struct mm_subpage_mapcounts counts = {}; + unsigned int i; + + if (present) + file_quarter_mapcounts(page, &counts); + for (i = offset / SZ_4K; i < (offset + size) / SZ_4K; i++) { + unsigned long pss = SZ_4K << PSS_SHIFT; + + if (counts.slots[i] > 1) + pss /= counts.slots[i]; + smaps_page_accumulate(mss, folio, SZ_4K, pss, dirty, locked, + present && counts.slots[i] < 2); + } +} + +static void smaps_account_small(struct mem_size_stats *mss, struct vm_area_struct *vma, + struct page *page, pte_t pte, bool young, + bool dirty, bool locked, bool present) +{ + struct folio *folio = page_folio(page); + unsigned long size = mm_page_size(vma->vm_mm), pss = size << PSS_SHIFT; + unsigned int count = present && folio_test_anon(folio) ? + proc_leaf_mapcount(vma, pte, page) : 0; + + mss->resident += size; + if (folio_test_anon(folio)) { + mss->anonymous += size; + if (present && !dirty) { + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + + if (slot) { + if (mm_subpage_is_lazyfree(slot)) + mss->lazyfree += size; + mm_subpage_put(slot); + } + } + } + if (folio_test_ksm(folio)) + mss->ksm += size; + if (young || folio_test_young(folio) || folio_test_referenced(folio)) + mss->referenced += size; + if (present && !folio_test_anon(folio)) { + smaps_account_file_extent(mss, page, offset_in_page(pte_phys_mm(vma->vm_mm, pte)), + size, dirty, locked, present); + return; + } + if (count > 1) + pss /= count; + smaps_page_accumulate(mss, folio, size, pss, dirty, locked, present && count < 2); +} + +static bool smaps_account_file_quarters(struct mem_size_stats *mss, struct page *page, + bool dirty, bool locked) +{ + if (folio_test_anon(page_folio(page))) + return false; + smaps_account_file_extent(mss, page, 0, PAGE_SIZE, dirty, locked, true); + return true; +} + +#endif + static void smaps_account(struct mem_size_stats *mss, struct page *page, bool compound, bool young, bool dirty, bool locked, bool present) @@ -976,6 +1098,11 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page, for (i = 0; i < nr; i++, page++) { unsigned long pss = PAGE_SIZE << PSS_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (smaps_account_file_quarters(mss, page, dirty, locked)) + continue; +#endif + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { mapcount = folio_precise_page_mapcount(folio, page); exclusive = mapcount < 2; @@ -995,6 +1122,14 @@ static int smaps_pte_hole(unsigned long addr, unsigned long end, struct mem_size_stats *mss = walk->private; struct vm_area_struct *vma = walk->vma; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + u64 start = ((u64)pos.index << PAGE_SHIFT) + pos.offset; + + mss->swap += shmem_partial_swap_usage_bytes(vma->vm_file->f_mapping, + start, start + end - addr); + return 0; + } mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, linear_page_index(vma, addr), linear_page_index(vma, end)); @@ -1010,7 +1145,7 @@ static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) #ifdef CONFIG_SHMEM if (walk->ops->pte_hole) { /* depth is not used */ - smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); + smaps_pte_hole(addr, addr + mm_page_size(walk->mm), 0, walk); } #endif } @@ -1038,15 +1173,20 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (softleaf_is_swap(entry)) { int mapcount; - mss->swap += PAGE_SIZE; - mapcount = swp_swapcount(entry); + mss->swap += mm_page_size(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + mapcount = swap_subpage_count(entry, pte_swp_subpage_offset(ptent)); + else +#endif + mapcount = swp_swapcount(entry); if (mapcount >= 2) { - u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; + u64 pss_delta = (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; do_div(pss_delta, mapcount); mss->swap_pss += pss_delta; } else { - mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; + mss->swap_pss += (u64)mm_page_size(vma->vm_mm) << PSS_SHIFT; } } else if (softleaf_has_pfn(entry)) { if (softleaf_is_device_private(entry)) @@ -1058,6 +1198,12 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr, if (!page) return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + smaps_account_small(mss, vma, page, ptent, young, dirty, locked, present); + return; + } +#endif smaps_account(mss, page, false, young, dirty, locked, present); } @@ -1124,7 +1270,7 @@ static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) smaps_pte_entry(pte, addr, walk); pte_unmap_unlock(pte - 1, ptl); out: @@ -1712,7 +1858,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte++, addr += PAGE_SIZE) { + for (; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (cp->type == CLEAR_REFS_SOFT_DIRTY) { @@ -1855,9 +2001,6 @@ struct pagemapread { bool show_pfn; }; -#define PAGEMAP_WALK_SIZE (PMD_SIZE) -#define PAGEMAP_WALK_MASK (PMD_MASK) - #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) #define PM_PFRAME_BITS 55 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) @@ -1884,7 +2027,7 @@ static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) return 0; } -static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) +static inline bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) { if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) return folio_precise_page_mapcount(folio, page) == 1; @@ -1909,7 +2052,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, else hole_end = end; - for (; addr < hole_end; addr += PAGE_SIZE) { + for (; addr < hole_end; addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1921,7 +2064,7 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, /* Addresses in the VMA. */ if (vma->vm_flags & VM_SOFTDIRTY) pme = make_pme(0, PM_SOFT_DIRTY); - for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { + for (; addr < min(end, vma->vm_end); addr += mm_page_size(walk->mm)) { err = add_to_pagemap(&pme, pm); if (err) goto out; @@ -1931,6 +2074,16 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, return err; } +static bool pagemap_page_exclusive(struct vm_area_struct *vma, pte_t pte, + struct folio *folio, struct page *page) +{ +#ifdef CONFIG_MM_SUBPAGE + return proc_leaf_mapcount(vma, pte, page) == 1; +#else + return __folio_page_mapped_exclusively(folio, page); +#endif +} + static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -1943,7 +2096,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, if (pte_present(pte)) { if (pm->show_pfn) - frame = pte_pfn(pte); + frame = pte_phys_mm(vma->vm_mm, pte) >> mm_page_shift(vma->vm_mm); flags |= PM_PRESENT; page = vm_normal_page(vma, addr, pte); if (pte_soft_dirty(pte)) @@ -1969,6 +2122,12 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, offset = softleaf_to_pfn(entry); else offset = swp_offset(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && + (softleaf_is_swap(entry) || softleaf_has_pfn(entry))) + offset = (offset << (PAGE_SHIFT - mm_page_shift(vma->vm_mm))) + + (pte_swp_subpage_offset(pte) >> mm_page_shift(vma->vm_mm)); +#endif frame = swp_type(entry) | (offset << MAX_SWAPFILES_SHIFT); } @@ -1985,8 +2144,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, folio = page_folio(page); if (!folio_test_anon(folio)) flags |= PM_FILE; - if ((flags & PM_PRESENT) && - __folio_page_mapped_exclusively(folio, page)) + if ((flags & PM_PRESENT) && pagemap_page_exclusive(vma, pte, folio, page)) flags |= PM_MMAP_EXCLUSIVE; } @@ -2102,7 +2260,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return err; } - for (; addr < end; pte++, addr += PAGE_SIZE) { + for (; addr < end; pte++, addr += mm_page_size(walk->mm)) { pagemap_entry_t pme; pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); @@ -2236,7 +2394,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* do not disclose physical addresses: attack vector */ pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); - pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); + pm.len = (pmd_size_mm(mm) >> mm_page_shift(mm)); pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); ret = -ENOMEM; if (!pm.buffer) @@ -2248,16 +2406,16 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, /* watch out for wraparound */ start_vaddr = end_vaddr; - if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + if (svpfn <= (ULONG_MAX >> mm_page_shift(mm))) { unsigned long end; ret = mmap_read_lock_killable(mm); if (ret) goto out_free; - start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + start_vaddr = untagged_addr_remote(mm, svpfn << mm_page_shift(mm)); mmap_read_unlock(mm); - end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); + end = start_vaddr + ((count / PM_ENTRY_BYTES) << mm_page_shift(mm)); if (end >= start_vaddr && end < mm->task_size) end_vaddr = end; } @@ -2272,7 +2430,7 @@ static ssize_t pagemap_read(struct file *file, char __user *buf, unsigned long end; pm.pos = 0; - end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; + end = (start_vaddr + pmd_size_mm(mm)) & pmd_mask_mm(mm); /* overflow ? */ if (end < start_vaddr || end > end_vaddr) end = end_vaddr; @@ -2333,6 +2491,7 @@ static int pagemap_release(struct inode *inode, struct file *file) struct pagemap_scan_private { struct pm_scan_arg arg; + unsigned long page_size; unsigned long masks_of_interest, cur_vma_category; struct page_region *vec_buf; unsigned long vec_buf_len, vec_buf_index, found_pages; @@ -2559,7 +2718,7 @@ static void pagemap_scan_backout_range(struct pagemap_scan_private *p, else cur_buf->start = cur_buf->end = 0; - p->found_pages -= (end - addr) / PAGE_SIZE; + p->found_pages -= (end - addr) / p->page_size; } #endif @@ -2671,11 +2830,11 @@ static int pagemap_scan_output(unsigned long categories, categories &= p->arg.return_mask; - n_pages = (*end - addr) / PAGE_SIZE; + n_pages = (*end - addr) / p->page_size; if (check_add_overflow(p->found_pages, n_pages, &total_pages) || total_pages > p->arg.max_pages) { size_t n_too_much = total_pages - p->arg.max_pages; - *end -= n_too_much * PAGE_SIZE; + *end -= n_too_much * p->page_size; n_pages -= n_too_much; ret = -ENOSPC; } @@ -2769,7 +2928,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { /* Fast path for performing exclusive WP */ - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2778,7 +2937,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) start = addr; - flush_end = addr + PAGE_SIZE; + flush_end = addr + p->page_size; } goto flush_and_return; } @@ -2786,8 +2945,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if (!p->arg.category_anyof_mask && !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { - for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { - unsigned long next = addr + PAGE_SIZE; + for (addr = start; addr < end; pte++, addr += p->page_size) { + unsigned long next = addr + p->page_size; pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2807,11 +2966,11 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + for (addr = start; addr != end; pte++, addr += p->page_size) { pte_t ptent = ptep_get(pte); unsigned long categories = p->cur_vma_category | pagemap_page_category(p, vma, addr, ptent); - unsigned long next = addr + PAGE_SIZE; + unsigned long next = addr + p->page_size; if (!pagemap_scan_is_interesting_page(categories, p)) continue; @@ -3007,7 +3166,7 @@ static const struct mm_walk_ops pagemap_scan_ops = { .hugetlb_entry = pagemap_scan_hugetlb_entry, }; -static int pagemap_scan_get_args(struct pm_scan_arg *arg, +static int pagemap_scan_get_args(struct mm_struct *mm, struct pm_scan_arg *arg, unsigned long uarg) { if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) @@ -3028,7 +3187,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, arg->vec = untagged_addr((unsigned long)arg->vec); /* Validate memory pointers */ - if (!IS_ALIGNED(arg->start, PAGE_SIZE)) + if (!IS_ALIGNED(arg->start, mm_page_size(mm))) return -EINVAL; if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) return -EFAULT; @@ -3041,7 +3200,7 @@ static int pagemap_scan_get_args(struct pm_scan_arg *arg, return -EFAULT; /* Fixup default values */ - arg->end = ALIGN(arg->end, PAGE_SIZE); + arg->end = ALIGN(arg->end, mm_page_size(mm)); arg->walk_end = 0; if (!arg->max_pages) arg->max_pages = ULONG_MAX; @@ -3060,12 +3219,13 @@ static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, return 0; } -static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) +static int pagemap_scan_init_bounce_buffer(struct mm_struct *mm, + struct pagemap_scan_private *p) { if (!p->arg.vec_len) return 0; - p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, + p->vec_buf_len = min_t(size_t, pmd_size_mm(mm) >> mm_page_shift(mm), p->arg.vec_len); p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); if (!p->vec_buf) @@ -3106,18 +3266,18 @@ static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) { - struct pagemap_scan_private p = {0}; + struct pagemap_scan_private p = { .page_size = mm_page_size(mm) }; unsigned long walk_start; size_t n_ranges_out = 0; int ret; - ret = pagemap_scan_get_args(&p.arg, uarg); + ret = pagemap_scan_get_args(mm, &p.arg, uarg); if (ret) return ret; p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | p.arg.return_mask; - ret = pagemap_scan_init_bounce_buffer(&p); + ret = pagemap_scan_init_bounce_buffer(mm, &p); if (ret) return ret; @@ -3219,16 +3379,17 @@ struct numa_maps_private { struct numa_maps md; }; +static int numa_page_mapcount(struct page *page) +{ + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + return folio_precise_page_mapcount(page_folio(page), page); + return folio_average_page_mapcount(page_folio(page)); +} + static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, - unsigned long nr_pages) + unsigned long nr_pages, int count) { struct folio *folio = page_folio(page); - int count; - - if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) - count = folio_precise_page_mapcount(folio, page); - else - count = folio_average_page_mapcount(folio); md->pages += nr_pages; if (pte_dirty || folio_test_dirty(folio)) @@ -3318,7 +3479,7 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, page = can_gather_numa_stats_pmd(*pmd, vma, addr); if (page) gather_stats(page, md, pmd_dirty(*pmd), - HPAGE_PMD_SIZE/PAGE_SIZE); + HPAGE_PMD_SIZE/PAGE_SIZE, numa_page_mapcount(page)); spin_unlock(ptl); return 0; } @@ -3333,9 +3494,13 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, struct page *page = can_gather_numa_stats(ptent, vma, addr); if (!page) continue; - gather_stats(page, md, pte_dirty(ptent), 1); +#ifdef CONFIG_MM_SUBPAGE + gather_stats(page, md, pte_dirty(ptent), 1, proc_leaf_mapcount(vma, ptent, page)); +#else + gather_stats(page, md, pte_dirty(ptent), 1, numa_page_mapcount(page)); +#endif - } while (pte++, addr += PAGE_SIZE, addr != end); + } while (pte++, addr += mm_page_size(vma->vm_mm), addr != end); pte_unmap_unlock(orig_pte, ptl); cond_resched(); return 0; @@ -3357,7 +3522,7 @@ static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, page = pte_page(huge_pte); md = walk->private; - gather_stats(page, md, pte_dirty(huge_pte), 1); + gather_stats(page, md, pte_dirty(huge_pte), 1, numa_page_mapcount(page)); out: spin_unlock(ptl); return 0; diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index dba1172436b745..3174e3b6edb414 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -301,6 +301,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pmd_t *pmd, _pmd; pte_t *pte; pte_t ptent; + spinlock_t *ptl; bool ret; assert_fault_locked(vmf); @@ -308,13 +309,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) return true; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) return true; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) return true; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); again: _pmd = pmdp_get_lockless(pmd); if (pmd_none(_pmd)) @@ -331,7 +332,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, if (pmd_trans_huge(_pmd)) return !pmd_write(_pmd) && (reason & VM_UFFD_WP); - pte = pte_offset_map(pmd, address); + pte = pte_offset_map_ro_nolock(mm, pmd, address, &ptl); if (!pte) goto again; @@ -1251,13 +1252,11 @@ static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx, __wake_userfault(ctx, range); } -static __always_inline int validate_unaligned_range( +static __always_inline int validate_byte_range( struct mm_struct *mm, __u64 start, __u64 len) { __u64 task_size = mm->task_size; - if (len & ~PAGE_MASK) - return -EINVAL; if (!len) return -EINVAL; if (start >= task_size) @@ -1269,10 +1268,19 @@ static __always_inline int validate_unaligned_range( return 0; } +static __always_inline int validate_unaligned_range( + struct mm_struct *mm, __u64 start, __u64 len) +{ + if (len & ~mm_page_mask(mm)) + return -EINVAL; + + return validate_byte_range(mm, start, len); +} + static __always_inline int validate_range(struct mm_struct *mm, __u64 start, __u64 len) { - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; return validate_unaligned_range(mm, start, len); @@ -1428,6 +1436,13 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx, ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC : UFFD_API_RANGE_IOCTLS; + if (mm_page_size(mm) != PAGE_SIZE) { + ioctls_out &= (1ULL << _UFFDIO_WAKE) | (1ULL << _UFFDIO_WRITEPROTECT) | + (1ULL << _UFFDIO_CONTINUE) | (1ULL << _UFFDIO_COPY) | + (1ULL << _UFFDIO_ZEROPAGE) | (1ULL << _UFFDIO_POISON); + } + if (mm_page_size(mm) < PAGE_SIZE) + ioctls_out |= 1ULL << _UFFDIO_MOVE; /* * Declare the WP ioctl only if the WP mode is @@ -1645,8 +1660,12 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, sizeof(uffdio_copy)-sizeof(__s64))) goto out; - ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, - uffdio_copy.len); + /* + * COPY reads the pager's address space, which can have a different + * task size and granule from the registered target. Only the target + * range below requires a page-aligned length. + */ + ret = validate_byte_range(current->mm, uffdio_copy.src, uffdio_copy.len); if (ret) goto out; ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); @@ -2015,6 +2034,18 @@ static int userfaultfd_api(struct userfaultfd_ctx *ctx, /* report all available features and ioctls to userland */ uffdio_api.features = UFFD_API_FEATURES; + /* Base-page anonymous and shared shmem operations support alternative leaves. */ + if (mm_page_size(ctx->mm) != PAGE_SIZE) + uffdio_api.features &= UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_MISSING_SHMEM | UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP | + UFFD_FEATURE_EVENT_REMOVE | UFFD_FEATURE_EVENT_UNMAP | + UFFD_FEATURE_SIGBUS | UFFD_FEATURE_THREAD_ID | UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_WP_UNPOPULATED | UFFD_FEATURE_WP_ASYNC | UFFD_FEATURE_POISON; + + if (mm_page_size(ctx->mm) < PAGE_SIZE) + uffdio_api.features |= UFFD_FEATURE_MOVE; + #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR uffdio_api.features &= ~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM); @@ -2062,6 +2093,10 @@ static long userfaultfd_ioctl(struct file *file, unsigned cmd, if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx)) return -EINVAL; + /* Larger-than-native MOVE still requires contiguous-owner support. */ + if (cmd == UFFDIO_MOVE && mm_page_size(ctx->mm) > PAGE_SIZE) + return -EOPNOTSUPP; + switch(cmd) { case UFFDIO_API: ret = userfaultfd_api(ctx, arg); diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h index 57137d3ac15925..619fd3ed406340 100644 --- a/include/asm-generic/pgalloc.h +++ b/include/asm-generic/pgalloc.h @@ -81,7 +81,7 @@ static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp) return NULL; } - return ptdesc_page(ptdesc); + return pgtable_from_page(ptdesc_page(ptdesc)); } #define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__)) @@ -111,17 +111,18 @@ static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) * @mm: the mm_struct of the current context * @pte_page: the `struct page` referencing the ptdesc */ -static inline void pte_free(struct mm_struct *mm, struct page *pte_page) +#ifndef __HAVE_ARCH_PTE_FREE +static inline void pte_free(struct mm_struct *mm, pgtable_t pte_page) { - struct ptdesc *ptdesc = page_ptdesc(pte_page); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte_page)); pagetable_dtor_free(ptdesc); } +#endif #if CONFIG_PGTABLE_LEVELS > 2 -#ifndef __HAVE_ARCH_PMD_ALLOC_ONE /** * pmd_alloc_one - allocate memory for a PMD-level page table * @mm: the mm_struct of the current context @@ -133,7 +134,7 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page) * * Return: pointer to the allocated memory or %NULL on error */ -static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +static inline pmd_t *__pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) { struct ptdesc *ptdesc; gfp_t gfp = GFP_PGTABLE_USER; @@ -153,6 +154,12 @@ static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long ad return ptdesc_address(ptdesc); } + +#ifndef __HAVE_ARCH_PMD_ALLOC_ONE +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + return __pmd_alloc_one_noprof(mm, addr); +} #define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) #endif diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h index bdcc2778ac64f4..900bee13b40cb8 100644 --- a/include/asm-generic/tlb.h +++ b/include/asm-generic/tlb.h @@ -222,6 +222,15 @@ static inline void __tlb_remove_table(void *table) } #endif +/* Architectures may pass opaque table tokens instead of descriptors. */ +#ifndef tlb_table_is_ptdesc +static inline bool tlb_table_is_ptdesc(void *table) +{ + (void)table; + return true; +} +#endif + extern void tlb_remove_table(struct mmu_gather *tlb, void *table); #else /* !CONFIG_MMU_GATHER_TABLE_FREE */ @@ -295,6 +304,12 @@ extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, in bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap); +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage; +/* Consumes one slot reference; true requires flushing the full batch. */ +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage); +#endif + #ifdef CONFIG_SMP /* * This both sets 'delayed_rmap', and returns true. It would be an inline @@ -543,15 +558,15 @@ static inline void tlb_change_page_size(struct mmu_gather *tlb, static inline unsigned long tlb_get_unmap_shift(struct mmu_gather *tlb) { if (tlb->cleared_ptes) - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); if (tlb->cleared_pmds) - return PMD_SHIFT; + return pmd_shift_mm(tlb->mm); if (tlb->cleared_puds) - return PUD_SHIFT; + return pud_shift_mm(tlb->mm); if (tlb->cleared_p4ds) - return P4D_SHIFT; + return p4d_shift_mm(tlb->mm); - return PAGE_SHIFT; + return mm_page_shift(tlb->mm); } static inline unsigned long tlb_get_unmap_size(struct mmu_gather *tlb) @@ -658,7 +673,7 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u */ #define tlb_remove_tlb_entry(tlb, ptep, address) \ do { \ - tlb_flush_pte_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pte_range(tlb, address, mm_page_size((tlb)->mm)); \ __tlb_remove_tlb_entry(tlb, ptep, address); \ } while (0) @@ -672,13 +687,15 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, u static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, pte_t *ptep, unsigned int nr, unsigned long address) { - tlb_flush_pte_range(tlb, address, PAGE_SIZE * nr); + unsigned long page_size = mm_page_size(tlb->mm); + + tlb_flush_pte_range(tlb, address, page_size * nr); for (;;) { __tlb_remove_tlb_entry(tlb, ptep, address); if (--nr == 0) break; ptep++; - address += PAGE_SIZE; + address += page_size; } } @@ -739,13 +756,13 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, * architecture to do its own odd thing, not cause pain for others * http://lkml.kernel.org/r/CA+55aFzBggoXtNXQeng5d_mRoDnaMBE5Y+URs+PHR67nUpMtaw@mail.gmail.com * - * For now w.r.t page table cache, mark the range_size as PAGE_SIZE + * For now w.r.t page table cache, mark the range_size as one mm granule. */ #ifndef pte_free_tlb #define pte_free_tlb(tlb, ptep, address) \ do { \ - tlb_flush_pmd_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pmd_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pte_free_tlb(tlb, ptep, address); \ } while (0) @@ -754,7 +771,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pmd_free_tlb #define pmd_free_tlb(tlb, pmdp, address) \ do { \ - tlb_flush_pud_range(tlb, address, PAGE_SIZE); \ + tlb_flush_pud_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pmd_free_tlb(tlb, pmdp, address); \ } while (0) @@ -763,7 +780,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef pud_free_tlb #define pud_free_tlb(tlb, pudp, address) \ do { \ - tlb_flush_p4d_range(tlb, address, PAGE_SIZE); \ + tlb_flush_p4d_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __pud_free_tlb(tlb, pudp, address); \ } while (0) @@ -772,7 +789,7 @@ static inline void tlb_remove_tlb_entries(struct mmu_gather *tlb, #ifndef p4d_free_tlb #define p4d_free_tlb(tlb, pudp, address) \ do { \ - __tlb_adjust_range(tlb, address, PAGE_SIZE); \ + __tlb_adjust_range(tlb, address, mm_page_size((tlb)->mm)); \ tlb->freed_tables = 1; \ __p4d_free_tlb(tlb, pudp, address); \ } while (0) diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 65abd5ab8836cc..c242a5e9cc758f 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -93,6 +93,8 @@ struct linux_binfmt { #ifdef CONFIG_COREDUMP int (*core_dump)(struct coredump_params *cprm); unsigned long min_coredump; /* minimal dump size */ + /* Optional per-mm replacement for formats with a variable page ABI. */ + unsigned long (*min_coredump_size)(void); #endif } __randomize_layout; @@ -121,6 +123,10 @@ static inline void insert_binfmt(struct linux_binfmt *fmt) extern void unregister_binfmt(struct linux_binfmt *); +#ifdef CONFIG_MM_SUBPAGE +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift); +#endif + extern int __must_check remove_arg_zero(struct linux_binprm *); extern int begin_new_exec(struct linux_binprm * bprm); extern void setup_new_exec(struct linux_binprm * bprm); diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 68861da4cf7c2f..5f85a12c7f9985 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -12,10 +12,21 @@ struct core_vma_metadata { unsigned long start, end; vm_flags_t flags; unsigned long dump_size; - unsigned long pgoff; + unsigned long pgoff; /* File offset in the dumped mm's page units. */ struct file *file; }; +/* NT_FILE offsets are expressed in the dumped process's page units. */ +static inline unsigned long core_vma_pgoff(const struct vm_area_struct *vma) +{ + unsigned int shift = mm_page_shift(vma->vm_mm); + + if (shift > PAGE_SHIFT) + return vma->vm_pgoff >> (shift - PAGE_SHIFT); + return (vma->vm_pgoff << (PAGE_SHIFT - shift)) + + (vma_subpage_offset(vma) >> shift); +} + struct coredump_params { const kernel_siginfo_t *siginfo; struct file *file; diff --git a/include/linux/highmem.h b/include/linux/highmem.h index d7aac9de1c8a4e..37812575512be3 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -330,6 +330,21 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, } #endif +#ifndef vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio_order( + struct vm_area_struct *vma, unsigned long addr, unsigned int order) +{ + struct folio *folio; + + if (!order) + return vma_alloc_zeroed_movable_folio(vma, addr); + folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); + if (folio && user_alloc_needs_zeroing()) + clear_user_highpages(&folio->page, addr, folio_nr_pages(folio)); + return folio; +} +#endif + static inline void clear_highpage(struct page *page) { void *kaddr = kmap_local_page(page); diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 2949e5acff351b..1f7f729b87868c 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -3,6 +3,7 @@ #define _LINUX_HUGE_MM_H #include +#include #include /* only for vma_is_dax() */ #include @@ -331,6 +332,9 @@ struct thpsize { static inline bool vma_thp_disabled(struct vm_area_struct *vma, vm_flags_t vm_flags, bool forced_collapse) { + /* Native THP geometry and rmap cannot describe alternative user leaves. */ + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return true; /* Are THPs disabled for this VMA? */ if (vm_flags & VM_NOHUGEPAGE) return true; diff --git a/include/linux/khugepaged.h b/include/linux/khugepaged.h index d7a9053ff4fed1..1510fdb0480107 100644 --- a/include/linux/khugepaged.h +++ b/include/linux/khugepaged.h @@ -8,6 +8,8 @@ extern unsigned int khugepaged_max_ptes_none __read_mostly; #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern struct attribute_group khugepaged_attr_group; +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start); + extern int khugepaged_init(void); extern void khugepaged_destroy(void); extern int start_stop_khugepaged(void); @@ -32,6 +34,12 @@ static inline void khugepaged_exit(struct mm_struct *mm) __khugepaged_exit(mm); } #else /* CONFIG_TRANSPARENT_HUGEPAGE */ +static inline int collapse_file_user_page(struct mm_struct *mm, struct file *file, + pgoff_t start) +{ + return -EOPNOTSUPP; +} + static inline void khugepaged_fork(struct mm_struct *mm, struct mm_struct *oldmm) { } diff --git a/include/linux/mm.h b/include/linux/mm.h index f4481c338b36cb..6c01377914fe8c 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -750,6 +751,9 @@ struct vm_fault { }; struct page *cow_page; /* Page handler may use for COW fault */ +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *cow_subpage; /* Owns cow_page's selected fragment */ +#endif struct page *page; /* ->fault handlers should return a * page here, unless VM_FAULT_NOPAGE * is set (which is also implied by @@ -1615,7 +1619,7 @@ static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) { if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) return vma->vm_ops->pagesize(vma); - return PAGE_SIZE; + return mm_page_size(vma->vm_mm); } unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); @@ -2010,6 +2014,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa void set_pte_range(struct vm_fault *vmf, struct folio *folio, struct page *page, unsigned int nr, unsigned long addr); +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault); +#endif vm_fault_t finish_fault(struct vm_fault *vmf); #endif @@ -3143,10 +3151,13 @@ struct follow_pfnmap_args { * @address: the virtual address to walk * @write: if true, fail with -EFAULT unless the mapping is * writable + * @allow_subpage: caller handles exact byte offsets and user-leaf bounds; + * otherwise a non-native user granule is rejected */ struct vm_area_struct *vma; unsigned long address; bool write; + bool allow_subpage; /** * Internals: * @@ -3157,13 +3168,15 @@ struct follow_pfnmap_args { /** * Outputs: * - * @pfn: the PFN of the address - * @addr_mask: address mask covering pfn + * @pfn: native PFN containing the requested physical byte + * @offset: byte offset of that byte within the native physical page + * @addr_mask: virtual address mask of the leaf mapping * @pgprot: the pgprot_t of the mapping * @writable: whether the mapping is writable * @special: whether the mapping is a special mapping (real PFN maps) */ unsigned long pfn; + unsigned int offset; unsigned long addr_mask; pgprot_t pgprot; bool writable; @@ -3215,6 +3228,13 @@ static inline void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows) { } #endif +#ifdef CONFIG_MM_SUBPAGE +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize); +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t start, u64 end); +#else +static inline void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) { } +#endif + static inline void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen) { @@ -3443,7 +3463,8 @@ static inline void reset_mm_hiwater_rss(struct mm_struct *mm) static inline void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm) { - unsigned long hiwater_rss = get_mm_hiwater_rss(mm); + /* signal->maxrss/cmaxrss survive exec and child reaping: store kB. */ + unsigned long hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); if (*maxrss < hiwater_rss) *maxrss = hiwater_rss; @@ -3514,14 +3535,14 @@ static inline void mm_inc_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_add(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_puds(struct mm_struct *mm) { if (mm_pud_folded(mm)) return; - atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes); + atomic_long_sub(pud_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3542,14 +3563,14 @@ static inline void mm_inc_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_add(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_pmds(struct mm_struct *mm) { if (mm_pmd_folded(mm)) return; - atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes); + atomic_long_sub(pmd_table_bytes_mm(mm), &mm->pgtables_bytes); } #endif @@ -3566,12 +3587,12 @@ static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm) static inline void mm_inc_nr_ptes(struct mm_struct *mm) { - atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_add(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } static inline void mm_dec_nr_ptes(struct mm_struct *mm) { - atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes); + atomic_long_sub(pte_table_bytes_mm(mm), &mm->pgtables_bytes); } #else @@ -3594,32 +3615,40 @@ static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? - NULL : p4d_offset(pgd, address); + NULL : p4d_offset_mm(mm, pgd, address); } static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) { - return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? - NULL : pud_offset(p4d, address); + return (unlikely(p4d_none_mm(mm, *p4d)) && __pud_alloc(mm, p4d, address)) ? + NULL : pud_offset_mm(mm, p4d, address); } static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) { return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? - NULL: pmd_offset(pud, address); + NULL : pmd_offset_mm(mm, pud, address); } #endif /* CONFIG_MMU */ enum pt_flags { PT_kernel = PG_referenced, PT_reserved = PG_reserved, + PT_fragmented = PG_private, + PT_frag_defer = PG_active, /* High bits are used for zone/node/section */ }; static inline struct ptdesc *virt_to_ptdesc(const void *x) { - return page_ptdesc(virt_to_page(x)); + struct page *page = virt_to_page(x); + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* One hardware table may span multiple native allocator pages. */ + page = compound_head(page); +#endif + return page_ptdesc(page); } /** @@ -3836,12 +3865,25 @@ static inline bool pagetable_pte_ctor(struct mm_struct *mm, } pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp); +#else +#define __pte_offset_map_mm(mm, pmd, addr, pmdvalp) \ + __pte_offset_map(pmd, addr, pmdvalp) +#endif static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) { return __pte_offset_map(pmd, addr, NULL); } +static inline pte_t *pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + return __pte_offset_map_mm(mm, pmd, addr, NULL); +} + pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp); @@ -3859,7 +3901,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) #define pte_alloc_map(mm, pmd, address) \ - (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) + (pte_alloc(mm, pmd) ? NULL : pte_offset_map_mm(mm, pmd, address)) #define pte_alloc_map_lock(mm, pmd, address, ptlp) \ (pte_alloc(mm, pmd) ? \ @@ -3874,7 +3916,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, static inline struct page *pmd_pgtable_page(pmd_t *pmd) { unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); - return virt_to_page((void *)((unsigned long) pmd & mask)); + return ptdesc_page(virt_to_ptdesc((void *)((unsigned long)pmd & mask))); } static inline struct ptdesc *pmd_ptdesc(pmd_t *pmd) @@ -4160,6 +4202,12 @@ extern unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf); +#endif extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, bool unlock); @@ -4288,7 +4336,11 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) static inline unsigned long vma_last_pgoff(struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + return vma_page_offset_at(vma, vma->vm_end - 1).index; +#else return vma->vm_pgoff + vma_pages(vma) - 1; +#endif } static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) @@ -4541,6 +4593,9 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length); int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num); int map_kernel_pages_prepare(struct vm_area_desc *desc); @@ -4552,6 +4607,9 @@ int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num); vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, bool write); +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, bool mkwrite); vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr, @@ -4648,6 +4706,18 @@ static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma, return VM_FAULT_NOPAGE; } +/* Insert one user granule from an explicit offset within a native page. */ +#ifdef CONFIG_MM_SUBPAGE +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset); +#else +static inline vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, + unsigned long addr, struct page *page, unsigned int offset) +{ + return offset ? VM_FAULT_SIGBUS : vmf_insert_page(vma, addr, page); +} +#endif + #ifndef io_remap_pfn_range_pfn static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, unsigned long size) diff --git a/include/linux/mm_granule.h b/include/linux/mm_granule.h new file mode 100644 index 00000000000000..6e73216edcf96d --- /dev/null +++ b/include/linux/mm_granule.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_GRANULE_H +#define _LINUX_MM_GRANULE_H + +#include +#include +#include + +/* + * User virtual mapping granule. Never use these for physical PFNs, struct + * page/folio sizes, page-cache indices, or physical memory accounting. + * The granule is immutable once an mm has mappings or is visible to tasks. + */ +#ifndef arch_mm_page_shift +static inline unsigned int arch_mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return arch_mm_page_shift(mm); +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return 1UL << mm_page_shift(mm); +} + +/* A byte-range reference must fit both the user leaf and a native page. */ +static inline unsigned long mm_user_fragment_size(const struct mm_struct *mm) +{ + return mm_page_shift(mm) < PAGE_SHIFT ? mm_page_size(mm) : PAGE_SIZE; +} + +/* Native physical pages covered by one base user PTE, including fragments. */ +static inline unsigned long mm_pte_native_pages(const struct mm_struct *mm) +{ + unsigned long pages = mm_page_size(mm) >> PAGE_SHIFT; + + return pages ?: 1; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return ~(mm_page_size(mm) - 1); +} + +/* + * ASLR tunables count native-page bits. Retain their byte window when a + * process uses another granule, including the extra low user-page bits. + * native_mask must be a contiguous low-bit mask whose byte span fits ulong. + */ +static inline unsigned long mm_aslr_mask(const struct mm_struct *mm, + unsigned long native_mask) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift < PAGE_SHIFT) { + unsigned int extra = PAGE_SHIFT - shift; + + return (native_mask << extra) | ((1UL << extra) - 1); + } + return native_mask >> (shift - PAGE_SHIFT); +} + +/* Virtual-range accounting. Physical pages and cache indices stay native. */ +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> mm_page_shift(vma->vm_mm); +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long bytes) +{ + return (bytes + mm_page_size(mm) - 1) & mm_page_mask(mm); +} + +/* + * RSS and virtual counters use the target mm's user granule. Consumers + * comparing different mms or physical memory must convert explicitly. + */ +static inline unsigned long mm_pages_to_bytes(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << mm_page_shift(mm); +} + +static inline unsigned long mm_pages_to_kb(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - 10); +} + +/* Round up only when reducing precision; do not hide a partial user page. */ +static inline unsigned long mm_pages_to_shift(const struct mm_struct *mm, + unsigned long pages, unsigned int shift) +{ + unsigned int source = mm_page_shift(mm); + + if (source >= shift) + return pages << (source - shift); + return (pages >> (shift - source)) + + !!(pages & ((1UL << (shift - source)) - 1)); +} + +static inline unsigned long mm_pages_to_native(const struct mm_struct *mm, + unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, PAGE_SHIFT); +} + +/* A page-count proc ABI uses the reader's granule, including remote reads. */ +static inline unsigned long mm_pages_for_reader(const struct mm_struct *mm, + const struct mm_struct *reader, unsigned long pages) +{ + return mm_pages_to_shift(mm, pages, + reader ? mm_page_shift(reader) : PAGE_SHIFT); +} + +/* + * Time integrals need one unit across exec. Locked VM also uses this unit: + * it must represent both logical mappings and native physical charges exactly, + * including a single native-page pin in a larger-granule userspace mm. + */ +#ifdef CONFIG_MM_SUBPAGE +#define MM_ACCOUNT_SHIFT 12 +#else +#define MM_ACCOUNT_SHIFT PAGE_SHIFT +#endif + +static inline unsigned long mm_pages_to_account(const struct mm_struct *mm, + unsigned long pages) +{ + return pages << (mm_page_shift(mm) - MM_ACCOUNT_SHIFT); +} + +static inline unsigned long mm_locked_vm_bytes(const struct mm_struct *mm) +{ + return READ_ONCE(mm->locked_vm) << MM_ACCOUNT_SHIFT; +} + +/* Physical-page limits must include any partial native page of locked VM. */ +static inline unsigned long mm_locked_vm_native_pages(const struct mm_struct *mm) +{ + unsigned long units = READ_ONCE(mm->locked_vm); + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; + + return (units >> shift) + !!(units & ((1UL << shift) - 1)); +} + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +/* The start byte of a faulting granule must precede EOF. */ +static inline bool vma_file_offset_valid(const struct vm_area_struct *vma, + unsigned long address, loff_t size) +{ + struct vm_page_offset pos = vma_page_offset_at(vma, address); + pgoff_t end = size >> PAGE_SHIFT; + + return pos.index < end || + (pos.index == end && pos.offset < (size & ~PAGE_MASK)); +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + +#endif diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h new file mode 100644 index 00000000000000..13344e3a92384c --- /dev/null +++ b/include/linux/mm_subpage.h @@ -0,0 +1,277 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_MM_SUBPAGE_H +#define _LINUX_MM_SUBPAGE_H + +#include +#include +#include + +#define MM_SUBPAGE_SHIFT 12 +#define MM_SUBPAGE_SIZE (1UL << MM_SUBPAGE_SHIFT) +#define MM_SUBPAGES_PER_PAGE (PAGE_SIZE / MM_SUBPAGE_SIZE) + +struct mm_subpage_mapcounts { + unsigned int slots[MM_SUBPAGES_PER_PAGE]; +}; + +struct folio; +struct page; +struct mm_struct; +struct mm_subpage; +struct mm_subpage_pool; +struct page_ext_operations; +struct vm_area_struct; + +extern struct page_ext_operations mm_subpage_ext_ops; + +#ifdef CONFIG_MM_SUBPAGE +/* Per-native-page shmem validity; caller holds the containing folio lock. */ +unsigned long mm_subpage_shmem_missing(struct page *page); +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing); +#endif + +/* + * A pool is one caller-defined allocation/policy/charge domain. The caller + * supplies fresh, unmapped native folios using its normal NUMA and memcg + * allocation path. No mm, VMA, rmap, swap or folio-private field is repurposed. + * Fault paths transfer slot references to PTEs through the rmap helpers below. + * + * Each caller must hold a pool reference through pool operations. close() + * stops allocation and releases unused backing; it does not consume that + * reference. Existing slot references outlive close() and the originating mm. + */ +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp); +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift); +void mm_subpage_pool_get(struct mm_subpage_pool *pool); +void mm_subpage_pool_put(struct mm_subpage_pool *pool); +void mm_subpage_pool_close(struct mm_subpage_pool *pool); + +/* Success consumes the folio reference. On error the caller retains it. */ +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp); +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp); +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool); + +/* -EAGAIN requires more compatible backing; -ESHUTDOWN means pool closed. */ +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +/* Exact physical byte offset within a native folio; no other slot is substituted. */ +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset); +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); + +/* + * Reattach a slot to an uptodate, locked order-0 swap-cache folio, without + * zeroing or copying its bytes. Caller holds a native folio reference and + * retains it. The returned slot owns an additional reference. -EAGAIN means + * ownership changed concurrently; -ENOMEM means metadata allocation failed. + */ +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp); +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp); + +/* PTE lock held, one file-backed alternative leaf per call. Native mms are no-ops. */ +#ifdef CONFIG_MM_SUBPAGE +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size); +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size); +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_add(phys, mm_page_size(vma->vm_mm)); +} +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) +{ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + __mm_subpage_file_map_del(phys, mm_page_size(vma->vm_mm)); +} +/* Fill 4K coverage counts; return alternative leaf count. Statistics only. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, + struct mm_subpage_mapcounts *counts); +#else +static inline void mm_subpage_file_map_add(struct vm_area_struct *vma, phys_addr_t phys) { } +static inline void mm_subpage_file_map_del(struct vm_area_struct *vma, phys_addr_t phys) { } +#endif + +/* Both folios locked; preallocate destination metadata before removing PTEs. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst); +bool mm_subpage_migrating(struct folio *folio); +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage); +/* Infallible metadata reuse after prepare; caller still holds the folio lock. */ +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset); +void mm_subpage_migrate_finish(struct folio *folio); +#else +static inline int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + return 0; +} +static inline bool mm_subpage_migrating(struct folio *folio) +{ + return false; +} +static inline void mm_subpage_migrate_finish(struct folio *folio) { } +#endif + +/* + * get() requires a live reference. A mapping reference must not be put until + * its PTE is removed and TLB invalidation completes. Every pin/alias holds + * its own reference: the slot becomes reusable only after the final put. + * Every live slot reference also holds one reference to the native folio. + * copy() requires the caller to stabilise the source contents (as for COW). + * A failed data copy returns -EHWPOISON without publishing the destination. + */ +void mm_subpage_get(struct mm_subpage *subpage); +void mm_subpage_put(struct mm_subpage *subpage); +/* + * Acquire the live identity containing a 4K-aligned physical address, or NULL. + * Interior quarters of a larger identity return that same identity. + * The caller must stabilise the physical allocation, e.g. with a live PTE + * under its lock or a folio reference. A lockless walker additionally needs + * GUP's TLB/free ordering; RCU protects metadata, not arbitrary physical PFNs. + * This is not a mapping identity check: a lockless PTE reader must revalidate + * its PTE after acquisition, just as for speculative folio references. + */ +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys); +/* + * Optional shared sets for asynchronous consumers of native byte spans. + * add() requires the source slot identities to remain live for the call. + * The pointer being modified is exclusively owned by the caller; get() shares + * a read-only set, and mutation uses copy-on-write. Duplicate ranges add no + * references. prune() requires a stable, side-effect-free membership predicate. + * Allocation failure leaves the original set intact. put() accepts NULL. + */ +struct mm_subpage_refs; +#ifdef CONFIG_MM_SUBPAGE +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs); +void mm_subpage_refs_put(struct mm_subpage_refs *refs); +int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp); +int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp); +#else +static inline struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + return refs; +} +static inline void mm_subpage_refs_put(struct mm_subpage_refs *refs) { } +static inline int mm_subpage_refs_add(struct mm_subpage_refs **refs, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + return 0; +} +static inline int mm_subpage_refs_prune(struct mm_subpage_refs **refs, + bool (*keep)(const struct mm_subpage *, void *), void *context, gfp_t gfp) +{ + return 0; +} +#endif + +/* + * Anonymous rmap attachment: caller holds the folio lock and PTE lock, has + * prepared vma->anon_vma, and transfers an existing slot reference to its PTE. + * A folio may cover only one native linear index and anon_vma root; its slot + * offset must match the VMA's linear byte offset. Removal needs the PTE lock. + * These helpers do not install PTEs, update RSS, enqueue on LRU, or drop refs. + */ +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address); +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma); +/* Both PTE locks and source write_protect_seq held; both mappings become R/O. */ +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address); +/* + * Per-slot anonymous exclusivity. Setting requires the folio and PTE locks, + * and proof of a fresh private slot or an exclusive migration/swap entry. + * Sharing requires the PTE lock and fails while a typed pin exists. + */ +void mm_subpage_set_exclusive(struct mm_subpage *subpage); +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage); +/* + * Folio and PTE locked; caller proves one remaining swap PTE for this slot. + * Rmap has been attached, but its PTE is not published yet. Reject old GET + * references or present aliases before recovering lost exclusive ownership. + */ +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage); +/* Present PTE plus caller lookup ref; caller proves no swapped aliases. */ +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage); +/* PTE locked, caller holds one GET; retire the PTE/TLB before dropping its ref. */ +bool mm_subpage_can_zap(struct mm_subpage *subpage); +bool mm_subpage_try_share(struct mm_subpage *subpage); +/* Folio/PTE locked, caller owns a lookup reference. */ +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); +bool mm_subpage_discardable(struct mm_subpage *subpage); +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); +/* + * Caller holds a live slot reference and its PTE lock. Success takes an + * additional slot reference plus a native FOLL_PIN reference. The pin keeps + * the exact fragment occupied after unmap; release only with subpage_unpin. + * This is not a GUP walker and does not validate user addresses/permissions. + */ +int mm_subpage_pin(struct mm_subpage *subpage); +void mm_subpage_unpin(struct mm_subpage *subpage); +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage); +#ifdef CONFIG_MM_SUBPAGE +/* A referenced/pinned folio range; never infer offset from the user VA. */ +struct user_page_fragment { + struct folio *folio; + struct mm_subpage *subpage; + unsigned long offset; + unsigned int length; + bool pinned; +}; + +/* + * Takes/releases mmap_lock internally. Returns the number of records filled, + * or -errno if none. A positive partial result is valid; sum record lengths + * to advance the user address. Each record stops at the target mm's granule + * or a native physical-page boundary, whichever comes first. + * Caller owns the target mm lifetime. Release either kind of record with + * release_user_fragments(); copying a record does not acquire a reference. + * FOLL_LONGTERM pins migrate unpinnable backing before returning success. + * Device/PFN mappings are not accepted by this interface. + */ +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity); +/* + * Single-record GET with a stable VMA. Caller holds mmap_lock throughout; + * this function never releases it. Returns 1 or -errno (0 for zero length). + * FOLL_NOWAIT is not supported. The returned VMA is valid until unlock. + */ +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Single-record PIN under the caller's mmap_lock, retained throughout. + * Same stable-VMA/partial-record contract as GET above. FOLL_LONGTERM and + * FOLL_NOWAIT are rejected: long-term migration can require dropping locks. + */ +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma); +/* Current mm, one aligned fragment; sparse zero pages return a negative errno. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment); +/* Dirty a retained reference without releasing its allocation identity. */ +void mark_user_fragment_dirty(struct user_page_fragment *fragment); +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty); +#endif + +int mm_subpage_mapcount(const struct mm_subpage *subpage); +/* Borrowed folio: a raw folio reference alone does not keep a slot occupied. */ +struct folio *mm_subpage_folio(const struct mm_subpage *subpage); +unsigned int mm_subpage_shift(const struct mm_subpage *subpage); +unsigned int mm_subpage_size(const struct mm_subpage *subpage); +unsigned int mm_subpage_offset(const struct mm_subpage *subpage); +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage); + +#endif diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index a308e2c23b829f..7f201dda7a13ba 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -558,7 +558,8 @@ FOLIO_MATCH(compound_info, _head_3); * @__page_mapping: Aliases with page->mapping. Unused for page tables. * @pt_index: Used for s390 gmap. * @pt_mm: Used for x86 pgds. - * @pt_frag_refcount: For fragmented page table tracking. Powerpc only. + * @pt_frag_refcount: For fragmented page table tracking (powerpc, arm64). + * @pt_frag_shift: Immutable arm64 fragment granule, valid through RCU release. * @pt_share_count: Used for HugeTLB PMD page table share count. * @_pt_pad_2: Padding to ensure proper alignment. * @ptl: Lock for the page table. @@ -585,7 +586,15 @@ struct ptdesc { union { pgoff_t pt_index; struct mm_struct *pt_mm; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* arm64 is 64-bit: use the spare half of this existing word. */ + struct { + atomic_t pt_frag_refcount; + u8 pt_frag_shift; + }; +#else atomic_t pt_frag_refcount; +#endif #ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING atomic_t pt_share_count; #endif @@ -905,6 +914,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -992,6 +1004,9 @@ struct vm_area_struct { /* Information about our backing store: */ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE units */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int vm_subpage_offset; /* Byte offset within vm_pgoff. */ +#endif struct file * vm_file; /* File we map to (can be NULL). */ void * vm_private_data; /* was vm_pte (shared mem) */ @@ -1281,12 +1296,16 @@ struct mm_struct { unsigned int __percpu *futex_ref; #endif - unsigned long hiwater_rss; /* High-watermark of RSS usage */ - unsigned long hiwater_vm; /* High-water virtual memory usage */ + unsigned long hiwater_rss; /* Peak RSS in user granules */ + unsigned long hiwater_vm; /* Peak VM in user granules */ - unsigned long total_vm; /* Total pages mapped */ - unsigned long locked_vm; /* Pages that have PG_mlocked set */ - atomic64_t pinned_vm; /* Refcount permanently increased */ + unsigned long total_vm; /* Mapped user granules */ +#ifdef CONFIG_MM_SUBPAGE + /* VM_ACCOUNT granules; global commitment rounds this sum once. */ + unsigned long committed_user_pages; +#endif + unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ + atomic64_t pinned_vm; /* Pinned native physical pages */ unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */ unsigned long stack_vm; /* VM_STACK */ diff --git a/include/linux/mman.h b/include/linux/mman.h index 389521594c69c2..7b80c19f6aa5da 100644 --- a/include/linux/mman.h +++ b/include/linux/mman.h @@ -83,6 +83,17 @@ static inline void vm_unacct_memory(long pages) vm_acct_memory(-pages); } +/* VMA reservations use user granules; shmem/physical reservations use pages. */ +#ifdef CONFIG_MM_SUBPAGE +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages); +void mm_acct_memory(struct mm_struct *mm, long pages); +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages); +#else +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_acct_memory(mm, pages) vm_acct_memory(pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) +#endif + /* * Allow architectures to handle additional protection and flag bits. The * overriding macros must be defined in the arch-specific asm/mman.h file. diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index ed3dd0f3fe1946..a4260bc898205c 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -5,6 +5,11 @@ #include #include +/* Select immutable exec geometry before allocating any page tables. */ +#ifndef arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) { } +#endif + /* Architectures that care about IRQ state in switch_mm can override this. */ #ifndef switch_mm_irqs_off # define switch_mm_irqs_off switch_mm diff --git a/include/linux/page_table_check.h b/include/linux/page_table_check.h index 12268a32e8be15..3ee1096fb0e53f 100644 --- a/include/linux/page_table_check.h +++ b/include/linux/page_table_check.h @@ -12,6 +12,10 @@ extern struct static_key_true page_table_check_disabled; extern struct page_ext_operations page_table_check_ops; +#if IS_ENABLED(CONFIG_KUNIT) +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file); +#endif void __page_table_check_zero(struct page *page, unsigned int order); void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 31a848485ad9d9..9cde417795f002 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -1068,10 +1068,7 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { - pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - return pgoff; + return vma_page_offset_at(vma, address).index; } struct wait_page_key { diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index cdd68ed3ae1a90..f5ef44f25c52b9 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -12,11 +12,18 @@ #ifdef CONFIG_MMU #include +#include #include #include #include #include +/* pgtable_t is an opaque allocation token, not necessarily struct page *. */ +#ifndef pgtable_to_page +#define pgtable_to_page(table) (table) +#define pgtable_from_page(page) (page) +#endif + #if 5 - defined(__PAGETABLE_P4D_FOLDED) - defined(__PAGETABLE_PUD_FOLDED) - \ defined(__PAGETABLE_PMD_FOLDED) != CONFIG_PGTABLE_LEVELS #error CONFIG_PGTABLE_LEVELS is not consistent with __PAGETABLE_{P4D,PUD,PMD}_FOLDED @@ -155,6 +162,59 @@ static inline pgd_t *pgd_offset_pgd(pgd_t *pgd, unsigned long address) */ #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) +/* Explicit target-mm accessors; legacy accessors keep native geometry. */ +#ifndef pte_phys_mm +#define pte_phys_mm(mm, pte) PFN_PHYS(pte_pfn(pte)) +#endif +#ifndef phys_pte_mm +#define phys_pte_mm(mm, phys, prot) pfn_pte((phys) >> PAGE_SHIFT, prot) +#endif +#ifndef pte_pgprot_mm +#define pte_pgprot_mm(mm, pte) pte_pgprot(pte) +#endif +#ifndef pgd_offset_pgd_mm +#define pgd_offset_pgd_mm(mm, pgd, addr) pgd_offset_pgd(pgd, addr) +#endif +#ifndef p4d_offset_mm +#define p4d_offset_mm(mm, pgd, addr) p4d_offset(pgd, addr) +#endif +#ifndef pud_offset_mm +#define pud_offset_mm(mm, p4d, addr) pud_offset(p4d, addr) +#endif +#ifndef pmd_offset_mm +#define pmd_offset_mm(mm, pud, addr) pmd_offset(pud, addr) +#endif +#ifndef __pte_map_mm +#define __pte_map_mm(mm, pmd, addr) __pte_map(pmd, addr) +#endif +#ifndef pte_offset_kernel_mm +#define pte_offset_kernel_mm(mm, pmd, addr) pte_offset_kernel(pmd, addr) +#endif + +#ifndef pud_pgtable_mm +#define pud_pgtable_mm(mm, pud) pud_pgtable(pud) +#endif + +#ifndef pmd_size_mm +#define pmd_shift_mm(mm) PMD_SHIFT +#define pud_shift_mm(mm) PUD_SHIFT +#define p4d_shift_mm(mm) P4D_SHIFT +#define pmd_size_mm(mm) PMD_SIZE +#define pud_size_mm(mm) PUD_SIZE +#define p4d_size_mm(mm) P4D_SIZE +#define pgd_size_mm(mm) PGDIR_SIZE +#define pmd_mask_mm(mm) PMD_MASK +#define pud_mask_mm(mm) PUD_MASK +#define p4d_mask_mm(mm) P4D_MASK +#define pgd_mask_mm(mm) PGDIR_MASK +#endif + +#ifndef pte_table_bytes_mm +#define pte_table_bytes_mm(mm) (PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (PTRS_PER_PUD * sizeof(pud_t)) +#endif + /* * In many cases it is known that a virtual address is mapped at PMD or PTE * level, so instead of traversing all the page table levels, we can get a @@ -388,6 +448,10 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) #define pte_next_pfn(pte) pte_advance_pfn(pte, 1) +#ifndef pte_advance_pfn_mm +#define pte_advance_pfn_mm(mm, pte, nr) pte_advance_pfn(pte, nr) +#endif + #ifndef set_ptes /** * set_ptes - Map consecutive pages to a contiguous range of addresses. @@ -1465,6 +1529,13 @@ static inline void arch_swap_restore(swp_entry_t entry, struct folio *folio) * and any p?d_bad entries - reporting the error before resetting to none. * Do the tests inline, but report and clear the bad entry in mm/memory.c. */ +#ifndef pgd_addr_end_mm +#define pgd_addr_end_mm(mm, addr, end) pgd_addr_end(addr, end) +#define p4d_addr_end_mm(mm, addr, end) p4d_addr_end(addr, end) +#define pud_addr_end_mm(mm, addr, end) pud_addr_end(addr, end) +#define pmd_addr_end_mm(mm, addr, end) pmd_addr_end(addr, end) +#endif + void pgd_clear_bad(pgd_t *); #ifndef __PAGETABLE_P4D_FOLDED @@ -1492,6 +1563,25 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) return 0; } +#ifndef p4d_none_mm +#define p4d_none_mm(mm, p4d) p4d_none(p4d) +#define p4d_bad_mm(mm, p4d) p4d_bad(p4d) +#define p4d_present_mm(mm, p4d) p4d_present(p4d) +#define p4d_clear_mm(mm, p4d) p4d_clear(p4d) +#endif + +static inline int p4d_none_or_clear_bad_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (p4d_none_mm(mm, *p4d)) + return 1; + if (unlikely(p4d_bad_mm(mm, *p4d))) { + p4d_ERROR(*p4d); + p4d_clear_mm(mm, p4d); + return 1; + } + return 0; +} + static inline int p4d_none_or_clear_bad(p4d_t *p4d) { if (p4d_none(*p4d)) @@ -1979,11 +2069,15 @@ static inline int is_zero_pfn(unsigned long pfn) #define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) #else +#ifndef ARCH_ZERO_PAGE_SIZE +#define ARCH_ZERO_PAGE_SIZE PAGE_SIZE +#endif + static inline int is_zero_pfn(unsigned long pfn) { extern unsigned long zero_page_pfn; - return pfn == zero_page_pfn; + return pfn - zero_page_pfn < (ARCH_ZERO_PAGE_SIZE >> PAGE_SHIFT); } static inline unsigned long zero_pfn(unsigned long addr) @@ -1993,7 +2087,7 @@ static inline unsigned long zero_pfn(unsigned long addr) return zero_page_pfn; } -extern uint8_t empty_zero_page[PAGE_SIZE]; +extern uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE]; extern struct page *__zero_page; static inline struct page *_zero_page(unsigned long addr) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f0011..2bc2c986eb76e8 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -40,6 +40,10 @@ struct anon_vma { * anon_vma if they are the last user on release */ atomic_t refcount; +#ifdef CONFIG_MM_SUBPAGE + /* Immutable base-page requirement shared by this anonymous lineage. */ + unsigned int min_folio_order; +#endif /* * Count of child anon_vmas. Equals to the count of all anon_vmas that @@ -398,29 +402,40 @@ static __always_inline void __folio_rmap_sanity_checks(const struct folio *folio } /* - * rmap interfaces called when adding or removing pte of page + * Single-PTE interfaces cover the complete native-page span of a user leaf. + * Multi-page interfaces below retain explicit native physical-page counts. */ void folio_move_anon_rmap(struct folio *, struct vm_area_struct *); void folio_add_anon_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *, unsigned long address, rmap_t flags); -#define folio_add_anon_rmap_pte(folio, page, vma, address, flags) \ - folio_add_anon_rmap_ptes(folio, page, 1, vma, address, flags) +static inline void folio_add_anon_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma, unsigned long address, rmap_t flags) +{ + folio_add_anon_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), + vma, address, flags); +} void folio_add_anon_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_new_anon_rmap(struct folio *, struct vm_area_struct *, unsigned long address, rmap_t flags); void folio_add_file_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_add_file_rmap_pte(folio, page, vma) \ - folio_add_file_rmap_ptes(folio, page, 1, vma) +static inline void folio_add_file_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_add_file_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_add_file_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_add_file_rmap_pud(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_ptes(struct folio *, struct page *, int nr_pages, struct vm_area_struct *); -#define folio_remove_rmap_pte(folio, page, vma) \ - folio_remove_rmap_ptes(folio, page, 1, vma) +static inline void folio_remove_rmap_pte(struct folio *folio, struct page *page, + struct vm_area_struct *vma) +{ + folio_remove_rmap_ptes(folio, page, mm_pte_native_pages(vma->vm_mm), vma); +} void folio_remove_rmap_pmd(struct folio *, struct page *, struct vm_area_struct *); void folio_remove_rmap_pud(struct folio *, struct page *, @@ -541,7 +556,8 @@ static inline void folio_dup_file_rmap_ptes(struct folio *folio, static __always_inline void folio_dup_file_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma) { - __folio_dup_file_rmap(folio, page, 1, dst_vma, PGTABLE_LEVEL_PTE); + __folio_dup_file_rmap(folio, page, mm_pte_native_pages(dst_vma->vm_mm), + dst_vma, PGTABLE_LEVEL_PTE); } /** @@ -665,8 +681,8 @@ static __always_inline int folio_try_dup_anon_rmap_pte(struct folio *folio, struct page *page, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) { - return __folio_try_dup_anon_rmap(folio, page, 1, dst_vma, src_vma, - PGTABLE_LEVEL_PTE); + return __folio_try_dup_anon_rmap(folio, page, mm_pte_native_pages(src_vma->vm_mm), + dst_vma, src_vma, PGTABLE_LEVEL_PTE); } /** @@ -708,13 +724,16 @@ static inline int folio_try_dup_anon_rmap_pmd(struct folio *folio, static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, struct page *page, int nr_pages, enum pgtable_level level) { + int i, exclusive_pages = level == PGTABLE_LEVEL_PTE ? nr_pages : 1; + VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); VM_WARN_ON_FOLIO(!PageAnonExclusive(page), folio); __folio_rmap_sanity_checks(folio, page, nr_pages, level); /* device private folios cannot get pinned via GUP. */ if (unlikely(folio_is_device_private(folio))) { - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); return 0; } @@ -766,7 +785,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, if (unlikely(folio_maybe_dma_pinned(folio))) return -EBUSY; - ClearPageAnonExclusive(page); + for (i = 0; i < exclusive_pages; i++) + ClearPageAnonExclusive(page + i); /* * This is conceptually a smp_wmb() paired with the smp_rmb() in @@ -782,7 +802,8 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * mapped by a PTE possibly shared to prepare * for KSM or temporary unmapping * @folio: The folio to share a mapping of - * @page: The mapped exclusive page + * @page: The first native page covered by the exclusive user PTE + * @vma: The VMA defining the base user PTE granule * * The caller needs to hold the page table lock and has to have the page table * entries cleared/invalidated. @@ -799,9 +820,10 @@ static __always_inline int __folio_try_share_anon_rmap(struct folio *folio, * -EBUSY otherwise. */ static inline int folio_try_share_anon_rmap_pte(struct folio *folio, - struct page *page) + struct page *page, struct vm_area_struct *vma) { - return __folio_try_share_anon_rmap(folio, page, 1, PGTABLE_LEVEL_PTE); + return __folio_try_share_anon_rmap(folio, page, mm_pte_native_pages(vma->vm_mm), + PGTABLE_LEVEL_PTE); } /** @@ -855,12 +877,23 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, #define PVMW_SYNC (1 << 0) /* Look for migration entries rather than present PTEs */ #define PVMW_MIGRATION (1 << 1) +/* Match one present subpage PTE at address, including its physical offset. */ +#define PVMW_SUBPAGE (1 << 2) /* Result flags */ /* The page is mapped across page table boundary */ #define PVMW_PGTABLE_CROSSED (1 << 16) +/* One reverse-map callback, optionally restricted to one physical user leaf. */ +struct rmap_walk_range { + unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; + bool subpage; +#endif +}; + struct page_vma_mapped_walk { unsigned long pfn; unsigned long nr_pages; @@ -871,6 +904,10 @@ struct page_vma_mapped_walk { pte_t *pte; spinlock_t *ptl; unsigned int flags; +#ifdef CONFIG_MM_SUBPAGE + /* PVMW_SUBPAGE: byte offset within pfn; nr_pages must be one. */ + unsigned int subpage_offset; +#endif }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -883,6 +920,32 @@ struct page_vma_mapped_walk { .flags = _flags, \ } +static inline struct page_vma_mapped_walk folio_rmap_walk_init( + struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, unsigned int flags) +{ + struct page_vma_mapped_walk walk = { + .pfn = folio_pfn(folio), + .nr_pages = folio_nr_pages(folio), + .pgoff = folio_pgoff(folio), + .vma = vma, + .address = range.address, + .flags = flags, + }; + +#ifdef CONFIG_MM_SUBPAGE + if (range.subpage) { + walk.flags |= PVMW_SUBPAGE; + walk.subpage_offset = range.subpage_offset; + } +#endif + return walk; +} + +#define DEFINE_FOLIO_RMAP_WALK(name, _folio, _vma, _range, _flags) \ + struct page_vma_mapped_walk name = \ + folio_rmap_walk_init(_folio, _vma, _range, _flags) + static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) { /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */ @@ -943,7 +1006,7 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, * arg: passed to rmap_one() and invalid_vma() * try_lock: bail out if the rmap lock is contended * contended: indicate the rmap traversal bailed out due to lock contention - * rmap_one: executed on each vma where page is mapped + * rmap_one: executed for each mapped VMA range (possibly one user leaf) * done: for checking traversing termination condition * anon_lock: for getting anon_lock by optimized way rather than default * invalid_vma: for skipping uninterested vma @@ -957,7 +1020,7 @@ struct rmap_walk_control { * Otherwise, return true. */ bool (*rmap_one)(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg); + struct rmap_walk_range range, void *arg); int (*done)(struct folio *folio); struct anon_vma *(*anon_lock)(const struct folio *folio, struct rmap_walk_control *rwc); @@ -966,6 +1029,15 @@ struct rmap_walk_control { void rmap_walk(struct folio *folio, struct rmap_walk_control *rwc); void rmap_walk_locked(struct folio *folio, struct rmap_walk_control *rwc); +#ifdef CONFIG_MM_SUBPAGE +unsigned int folio_anon_min_order(struct folio *folio); +#else +static inline unsigned int folio_anon_min_order(struct folio *folio) +{ + return 0; +} +#endif + struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, struct rmap_walk_control *rwc); diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df58413..38d573026d7fb8 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -14,6 +14,10 @@ * Routines for handling mm_structs */ extern struct mm_struct *mm_alloc(void); +extern struct mm_struct *mm_alloc_exec(void); +extern struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift); +/* Return zero for native, an alternative shift, or a negative errno. */ +int arch_exec_page_size_shift(unsigned long size); /** * mmgrab() - Pin a &struct mm_struct. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 584ae88b435e9e..7af2aeb81e41d9 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -194,7 +194,7 @@ struct signal_struct { unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; unsigned long inblock, oublock, cinblock, coublock; - unsigned long maxrss, cmaxrss; + unsigned long maxrss, cmaxrss; /* kB, independent of each mm granule */ struct task_io_accounting ioac; /* diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 69b0177da15648..bf3a0129547001 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -32,6 +32,8 @@ struct swap_iocb; * isn't directly visible to userspace. */ #define SHMEM_F_MAPPING_FROZEN BIT(2) +/* This inode has used sub-native shared-page validity. */ +#define SHMEM_F_UFFD_SUBPAGE BIT(3) struct shmem_inode_info { spinlock_t lock; @@ -52,6 +54,10 @@ struct shmem_inode_info { pgoff_t fallocend; /* highest fallocate endindex */ unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Swapped missing 4K units, keyed by native cache index. */ + struct xarray uffd_missing; +#endif #ifdef CONFIG_TMPFS_QUOTA struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif @@ -113,13 +119,56 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts); #ifdef CONFIG_SHMEM +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order); bool shmem_mapping(const struct address_space *mapping); #else +static inline struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, + struct inode *inode, pgoff_t index, unsigned int order) +{ + return ERR_PTR(-EOPNOTSUPP); +} static inline bool shmem_mapping(const struct address_space *mapping) { return false; } #endif /* CONFIG_SHMEM */ +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +/* Return a locked cache folio and caller reference; new backing is all missing. */ +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new); +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new); +bool shmem_uffd_has_missing_native_page(struct folio *folio); +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length); +/* Resident masks require no allocation; inode metadata persists only in swap. */ +void shmem_uffd_copy_page(struct page *dst, struct page *src); +void shmem_uffd_migrate(struct folio *dst, struct folio *src); +/* Locked, uptodate cache folio. */ +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index); +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing); +/* Clear populated units; no allocation. Caller holds the folio lock. */ +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present); +/* Mark exactly the byte range exposed by a mapping; folio lock held. */ +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length); +/* i_pages lock held; deletion, not replacement by swap or migration. */ +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr); +#else +static inline void shmem_uffd_copy_page(struct page *dst, struct page *src) { } +static inline void shmem_uffd_migrate(struct folio *dst, struct folio *src) { } +static inline bool shmem_uffd_has_missing_native_page(struct folio *folio) { return false; } +static inline bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) { return false; } +static inline void shmem_uffd_mark_mapped(struct folio *folio, + unsigned long offset, unsigned long length) { } +static inline void shmem_uffd_forget(struct address_space *mapping, + pgoff_t start, unsigned long nr) { } +#endif + void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); @@ -160,6 +209,8 @@ static inline void shmem_uncharge(struct inode *inode, long pages) { } #endif +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end); extern unsigned long shmem_partial_swap_usage(struct address_space *mapping, pgoff_t start, pgoff_t end); diff --git a/include/linux/swap.h b/include/linux/swap.h index 7a09df6977a5f3..4e9136d3ae3f68 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -228,7 +228,7 @@ enum { */ #define SWAP_ENTRY_INVALID 0 -#ifdef CONFIG_THP_SWAP +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) #define SWAP_NR_ORDERS (PMD_ORDER + 1) #else #define SWAP_NR_ORDERS 1 @@ -443,6 +443,7 @@ extern int swp_swapcount(swp_entry_t entry); struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages); /* * If there is an existing swap slot reference (swap entry) and the caller @@ -455,6 +456,13 @@ sector_t swap_folio_sector(struct folio *folio); * a swap count > 1. See comments of folio_*_swap helpers for more info. */ int swap_dup_entry_direct(swp_entry_t entry); +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr); +#ifdef CONFIG_MM_SUBPAGE +/* Offset is bytes within one native swap entry; caller owns a PTE/cache ref. */ +int swap_subpage_dup(swp_entry_t entry, unsigned int offset); +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache); +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset); +#endif void swap_put_entries_direct(swp_entry_t entry, int nr); /* @@ -498,11 +506,27 @@ static inline void free_swap_cache(struct folio *folio) { } +#ifdef CONFIG_MM_SUBPAGE +static inline int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + return -EINVAL; +} +static inline void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) { } +static inline unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + return 0; +} +#endif static inline int swap_dup_entry_direct(swp_entry_t ent) { return 0; } +static inline int swap_dup_entries_direct(swp_entry_t ent, unsigned int nr) +{ + return 0; +} + static inline void swap_put_entries_direct(swp_entry_t ent, int nr) { } diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 4fb7291f54b62b..29beb44f370fe6 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1261,6 +1261,11 @@ static inline int ksys_fadvise64_64(int fd, loff_t offset, loff_t len, unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff); +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset); +#endif + ssize_t ksys_readahead(int fd, loff_t offset, size_t count); int ksys_ipc(unsigned int call, int first, unsigned long second, unsigned long third, void __user * ptr, long fifth); diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec8f..237b530ed600fd 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -192,6 +192,7 @@ struct uprobes_state { }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, + unsigned int page_offset, uprobe_opcode_t *insn, int nbytes, void *data); extern void __init uprobes_init(void); diff --git a/include/trace/events/kmem.h b/include/trace/events/kmem.h index cd7920c81f85ba..fefbf65d61441b 100644 --- a/include/trace/events/kmem.h +++ b/include/trace/events/kmem.h @@ -449,7 +449,7 @@ TRACE_EVENT(rss_stat, __entry->curr = current->mm == mm && !(current->flags & PF_KTHREAD); __entry->member = member; __entry->size = (percpu_counter_sum_positive(&mm->rss_stat[member]) - << PAGE_SHIFT); + << mm_page_shift(mm)); ), TP_printk("mm_id=%u curr=%d type=%s size=%ldB", diff --git a/include/trace/events/mmap.h b/include/trace/events/mmap.h index ee2843a5daef10..46df97560969d2 100644 --- a/include/trace/events/mmap.h +++ b/include/trace/events/mmap.h @@ -26,7 +26,7 @@ TRACE_EVENT(vm_unmapped_area, TP_fast_assign( __entry->addr = addr; - __entry->total_vm = current->mm->total_vm; + __entry->total_vm = mm_pages_to_native(current->mm, current->mm->total_vm); __entry->flags = info->flags; __entry->length = info->length; __entry->low_limit = info->low_limit; diff --git a/include/trace/events/oom.h b/include/trace/events/oom.h index 9f0a5d1482c448..d5321b6f01ffa3 100644 --- a/include/trace/events/oom.h +++ b/include/trace/events/oom.h @@ -7,7 +7,6 @@ #include #include -#define PG_COUNT_TO_KB(x) ((x) << (PAGE_SHIFT - 10)) TRACE_EVENT(oom_score_adj_update, @@ -93,10 +92,10 @@ TRACE_EVENT(mark_victim, TP_fast_assign( __entry->pid = task->pid; __assign_str(comm); - __entry->total_vm = PG_COUNT_TO_KB(task->mm->total_vm); - __entry->anon_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_ANONPAGES)); - __entry->file_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_FILEPAGES)); - __entry->shmem_rss = PG_COUNT_TO_KB(get_mm_counter(task->mm, MM_SHMEMPAGES)); + __entry->total_vm = mm_pages_to_kb(task->mm, task->mm->total_vm); + __entry->anon_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_ANONPAGES)); + __entry->file_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_FILEPAGES)); + __entry->shmem_rss = mm_pages_to_kb(task->mm, get_mm_counter(task->mm, MM_SHMEMPAGES)); __entry->uid = uid; __entry->pgtables = mm_pgtables_bytes(task->mm) >> 10; __entry->oom_score_adj = task->signal->oom_score_adj; diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 1a06e7feb7dc9b..6e08471fbe822f 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -421,4 +421,22 @@ struct prctl_mm_map { # define PR_CFI_DISABLE _BITUL(1) # define PR_CFI_LOCK _BITUL(2) +/* + * Aurora experimental ABI: private numbers, not upstream assignments. + * SET_EXEC selects the calling thread's next successful exec granule. + * Zero requests native pages; with an alternative DEFAULT in effect this + * installs an explicit one-shot native override. GET_EXEC returns zero when + * no override remains, or its size in bytes. Successful exec consumes it. + * SET_DEFAULT selects the fallback for subsequent execs; zero restores native. + * GET_DEFAULT returns the alternative size in bytes, or zero for native. + * Fork inherits both policies; exec preserves DEFAULT. Failed exec preserves + * both. Neither setter changes the live address space or other threads. + * AArch32 always selects 4K when supported, otherwise the native granule, + * independently of these preferences. Sizes are capability-checked. + */ +#define PR_AURORA_SET_EXEC_PAGE_SIZE 0x41555001 +#define PR_AURORA_GET_EXEC_PAGE_SIZE 0x41555002 +#define PR_AURORA_SET_DEFAULT_PAGE_SIZE 0x41555003 +#define PR_AURORA_GET_DEFAULT_PAGE_SIZE 0x41555004 + #endif /* _LINUX_PRCTL_H */ diff --git a/ipc/shm.c b/ipc/shm.c index b3e8a58e177d3b..2929b9243f8e89 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -51,6 +51,14 @@ #include "util.h" +#ifndef arch_shm_attach_align +static unsigned long arch_shm_attach_align(const struct mm_struct *mm, + unsigned long align) +{ + return align; +} +#endif + struct shmid_kernel /* private to the kernel */ { struct kern_ipc_perm shm_perm; @@ -1538,6 +1546,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (shmid < 0) goto out; + shmlba = arch_shm_attach_align(current->mm, shmlba); if (addr) { if (addr & (shmlba - 1)) { if (shmflg & SHM_RND) { @@ -1552,7 +1561,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto out; } else #ifndef __ARCH_FORCE_SHMLBA - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(current->mm)) #endif goto out; } @@ -1723,6 +1732,20 @@ COMPAT_SYSCALL_DEFINE3(shmat, int, shmid, compat_uptr_t, shmaddr, int, shmflg) } #endif +/* VMA splits retain native file indices plus a subpage byte offset. */ +#ifdef CONFIG_MMU +static bool shm_vma_matches_addr(struct vm_area_struct *vma, unsigned long addr) +{ + unsigned long offset; + + if (vma->vm_ops != &shm_vm_ops || vma->vm_start < addr) + return false; + offset = vma->vm_start - addr; + return (offset >> PAGE_SHIFT) == vma->vm_pgoff && + (offset & ~PAGE_MASK) == vma_subpage_offset(vma); +} +#endif + /* * detach and kill segment if marked destroyed. * The work is done in shm_close. @@ -1739,7 +1762,7 @@ long ksys_shmdt(char __user *shmaddr) VMA_ITERATOR(vmi, mm, addr); #endif - if (addr & ~PAGE_MASK) + if (addr & ~mm_page_mask(mm)) return retval; if (mmap_write_lock_killable(mm)) @@ -1774,8 +1797,7 @@ long ksys_shmdt(char __user *shmaddr) * a fragment created by mprotect() and/or munmap(), or it * otherwise it starts at this address with no hassles. */ - if ((vma->vm_ops == &shm_vm_ops) && - (vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) { + if (shm_vma_matches_addr(vma, addr)) { /* * Record the file of the shm segment being @@ -1804,12 +1826,10 @@ long ksys_shmdt(char __user *shmaddr) * could possibly have landed at. Also cast things to loff_t to * prevent overflows and make comparisons vs. equal-width types. */ - size = PAGE_ALIGN(size); + size = ALIGN(size, mm_page_size(mm)); while (vma && (loff_t)(vma->vm_end - addr) <= size) { /* finding a matching vma now does not alter retval */ - if ((vma->vm_ops == &shm_vm_ops) && - ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) && - (vma->vm_file == file)) { + if (shm_vma_matches_addr(vma, addr) && vma->vm_file == file) { do_vmi_align_munmap(&vmi, vma, mm, vma->vm_start, vma->vm_end, NULL, false); } diff --git a/kernel/events/core.c b/kernel/events/core.c index 1a23ae67c3cc8f..343dc1af5a3085 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -6812,7 +6812,7 @@ static void perf_event_init_userpage(struct perf_event *event) /* Allow new userspace to detect that bit 0 is deprecated */ userpg->cap_bit0_is_deprecated = 1; userpg->size = offsetof(struct perf_event_mmap_page, __reserved); - userpg->data_offset = PAGE_SIZE; + userpg->data_offset = perf_metadata_size(rb); userpg->data_size = perf_data_size(rb); unlock: @@ -6997,6 +6997,11 @@ typedef void (*mapped_f)(struct perf_event *event, struct mm_struct *mm); f; \ }) +static u64 perf_mmap_offset(struct vm_area_struct *vma) +{ + return ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); +} + static void perf_mmap_open(struct vm_area_struct *vma) { struct perf_event *event = vma->vm_file->private_data; @@ -7005,7 +7010,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) refcount_inc(&event->mmap_count); refcount_inc(&event->rb->mmap_count); - if (vma->vm_pgoff) + if (perf_mmap_offset(vma)) refcount_inc(&event->rb->aux_mmap_count); if (mapped) @@ -7039,7 +7044,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex * to avoid complications. */ - if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && + if (rb_has_aux(rb) && perf_mmap_offset(vma) == rb->aux_offset && refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { /* * Stop all AUX events that are writing to this buffer, @@ -7132,7 +7137,9 @@ static void perf_mmap_close(struct vm_area_struct *vma) static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf) { /* The first page is the user control page, others are read-only. */ - return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS; + return !perf_mmap_offset(vmf->vma) && + vmf->address - vmf->vma->vm_start < mm_page_size(vmf->vma->vm_mm) ? + 0 : VM_FAULT_SIGBUS; } static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr) @@ -7153,7 +7160,8 @@ static const struct vm_operations_struct perf_mmap_vmops = { static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) { - unsigned long nr_pages = vma_pages(vma); + unsigned long va = vma->vm_start; + bool aux = perf_mmap_offset(vma) != 0; int err = 0; unsigned long pagenum; @@ -7197,9 +7205,21 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) * perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of * this mapping is maintained correctly. */ - for (pagenum = 0; pagenum < nr_pages; pagenum++) { - unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; - struct page *page = perf_mmap_to_page(rb, vma->vm_pgoff + pagenum); + for (pagenum = 0; va < vma->vm_end; pagenum++) { + unsigned long size = min(PAGE_SIZE, vma->vm_end - va); + struct page *page; + + if (aux) { + if (pagenum >= rb->aux_nr_pages) { + err = -EINVAL; + break; + } + page = virt_to_page(rb->aux_pages[pagenum]); + } else { + page = perf_mmap_to_page(rb, pagenum); + if (!pagenum) + size = perf_metadata_size(rb); + } if (page == NULL) { err = -EINVAL; @@ -7207,16 +7227,17 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) } /* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */ - err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE, + err = remap_pfn_range(vma, va, page_to_pfn(page), size, vm_get_page_prot(vma->vm_flags & ~VM_SHARED)); if (err) break; + va += size; } #ifdef CONFIG_MMU /* Clear any partial mappings on error. */ if (err) - zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE); + zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); #endif return err; @@ -7269,7 +7290,7 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * { struct user_struct *user = rb->mmap_user; - atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, + atomic_long_sub(data_page_nr(rb) + 1 - rb->mmap_locked, &user->locked_vm); atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); } @@ -7277,7 +7298,9 @@ static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer * static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { - long extra = 0, user_extra = nr_pages; + unsigned int page_shift = mm_page_shift(vma->vm_mm); + unsigned long data_size; + long extra = 0, user_extra; struct perf_buffer *rb; int rb_flags = 0; @@ -7289,11 +7312,14 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, */ if (nr_pages != 0 && !is_power_of_2(nr_pages)) return -EINVAL; + data_size = nr_pages << page_shift; + /* Physical allocation and locked-memory accounting stay in native pages. */ + user_extra = DIV_ROUND_UP(data_size, PAGE_SIZE) + 1; WARN_ON_ONCE(event->ctx->parent_ctx); if (event->rb) { - if (data_page_nr(event->rb) != nr_pages) + if (perf_data_size(event->rb) != data_size) return -EINVAL; /* @@ -7329,7 +7355,7 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - rb = rb_alloc(nr_pages, + rb = rb_alloc(data_size, page_shift, event->attr.watermark ? event->attr.wakeup_watermark : 0, event->cpu, rb_flags); @@ -7350,9 +7376,10 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, return 0; } -static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, - unsigned long nr_pages) +static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event) { + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long nr_pages = size >> PAGE_SHIFT; long extra = 0, user_extra = nr_pages; u64 aux_offset, aux_size; struct perf_buffer *rb; @@ -7361,6 +7388,9 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, rb = event->rb; if (!rb) return -EINVAL; + /* PMU setup_aux() consumes whole native pages, independent of mmap layout. */ + if (!IS_ALIGNED(size, PAGE_SIZE)) + return -EINVAL; guard(mutex)(&rb->aux_mutex); @@ -7372,14 +7402,14 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, aux_offset = READ_ONCE(rb->user_page->aux_offset); aux_size = READ_ONCE(rb->user_page->aux_size); - if (aux_offset < perf_data_size(rb) + PAGE_SIZE) + if (aux_offset < perf_data_size(rb) + perf_metadata_size(rb)) return -EINVAL; - if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) + if (aux_offset != perf_mmap_offset(vma)) return -EINVAL; /* already mapped with a different offset */ - if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) + if (rb_has_aux(rb) && rb->aux_offset != aux_offset) return -EINVAL; if (aux_size != nr_pages * PAGE_SIZE) @@ -7409,7 +7439,7 @@ static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, if (vma->vm_flags & VM_WRITE) rb_flags |= RING_BUFFER_WRITABLE; - ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, + ret = rb_alloc_aux(rb, event, aux_offset, nr_pages, event->attr.aux_watermark, rb_flags); if (ret) { refcount_dec(&rb->mmap_count); @@ -7449,12 +7479,13 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) return ret; vma_size = vma->vm_end - vma->vm_start; - nr_pages = vma_size / PAGE_SIZE; + nr_pages = vma_size >> mm_page_shift(vma->vm_mm); if (nr_pages > INT_MAX) return -ENOMEM; - if (vma_size != PAGE_SIZE * nr_pages) + if (vma_size != mm_page_size(vma->vm_mm) * nr_pages || + mm_page_shift(vma->vm_mm) > PAGE_SHIFT) return -EINVAL; scoped_guard (mutex, &event->mmap_mutex) { @@ -7466,10 +7497,14 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) if (event->state <= PERF_EVENT_STATE_REVOKED) return -ENODEV; - if (vma->vm_pgoff == 0) + /* One ring has one immutable metadata/data boundary for all aliases. */ + if (event->rb && event->rb->mmap_page_shift != mm_page_shift(vma->vm_mm)) + return -EINVAL; + + if (!perf_mmap_offset(vma)) ret = perf_mmap_rb(vma, event, nr_pages); else - ret = perf_mmap_aux(vma, event, nr_pages); + ret = perf_mmap_aux(vma, event); if (ret) return ret; diff --git a/kernel/events/internal.h b/kernel/events/internal.h index c03c4f2eea5718..5238cfa3399b1a 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -15,9 +15,10 @@ struct perf_buffer { struct rcu_head rcu_head; #ifdef CONFIG_PERF_USE_VMALLOC struct work_struct work; - int page_order; /* allocation order */ #endif int nr_pages; /* nr of data pages */ + unsigned int data_page_shift; /* bytes per output chunk */ + unsigned int mmap_page_shift; /* userspace metadata size */ int overwrite; /* can overwrite itself */ int paused; /* can write into ring buffer */ @@ -44,7 +45,7 @@ struct perf_buffer { long aux_head; unsigned int aux_nest; long aux_wakeup; /* last aux_watermark boundary crossed by aux_head */ - unsigned long aux_pgoff; + u64 aux_offset; int aux_nr_pages; int aux_overwrite; refcount_t aux_mmap_count; @@ -80,10 +81,11 @@ static inline void rb_toggle_paused(struct perf_buffer *rb, bool pause) } extern struct perf_buffer * -rb_alloc(int nr_pages, long watermark, int cpu, int flags); +rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags); extern void perf_event_wakeup(struct perf_event *event); extern int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags); + u64 offset, int nr_pages, long watermark, int flags); extern void rb_free_aux(struct perf_buffer *rb); extern struct perf_buffer *ring_buffer_get(struct perf_event *event); extern void ring_buffer_put(struct perf_buffer *rb); @@ -99,34 +101,19 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, extern struct page * perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff); -#ifdef CONFIG_PERF_USE_VMALLOC -/* - * Back perf_mmap() with vmalloc memory. - * - * Required for architectures that have d-cache aliasing issues. - */ - -static inline int page_order(struct perf_buffer *rb) -{ - return rb->page_order; -} - -#else - -static inline int page_order(struct perf_buffer *rb) +static inline unsigned long perf_data_size(struct perf_buffer *rb) { - return 0; + return (unsigned long)rb->nr_pages << rb->data_page_shift; } -#endif static inline int data_page_nr(struct perf_buffer *rb) { - return rb->nr_pages << page_order(rb); + return DIV_ROUND_UP(perf_data_size(rb), PAGE_SIZE); } -static inline unsigned long perf_data_size(struct perf_buffer *rb) +static inline unsigned long perf_metadata_size(struct perf_buffer *rb) { - return rb->nr_pages << (PAGE_SHIFT + page_order(rb)); + return 1UL << rb->mmap_page_shift; } static inline unsigned long perf_aux_size(struct perf_buffer *rb) @@ -154,7 +141,7 @@ static inline unsigned long perf_aux_size(struct perf_buffer *rb) handle->page++; \ handle->page &= rb->nr_pages - 1; \ handle->addr = rb->data_pages[handle->page]; \ - handle->size = PAGE_SIZE << page_order(rb); \ + handle->size = 1UL << rb->data_page_shift; \ } \ } while (len && written == size); \ \ diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 9fe92161715e09..6c5de8fba83f23 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -238,7 +238,7 @@ __perf_output_begin(struct perf_output_handle *handle, if (unlikely(head - local_read(&rb->wakeup) > rb->watermark)) local_add(rb->watermark, &rb->wakeup); - page_shift = PAGE_SHIFT + page_order(rb); + page_shift = rb->data_page_shift; handle->page = (offset >> page_shift) & (rb->nr_pages - 1); offset &= (1UL << page_shift) - 1; @@ -677,7 +677,7 @@ static void __rb_free_aux(struct perf_buffer *rb) } int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, - pgoff_t pgoff, int nr_pages, long watermark, int flags) + u64 offset, int nr_pages, long watermark, int flags) { bool overwrite = !(flags & RING_BUFFER_WRITABLE); int node = (event->cpu == -1) ? -1 : cpu_to_node(event->cpu); @@ -784,7 +784,7 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, out: if (!ret) - rb->aux_pgoff = pgoff; + rb->aux_offset = offset; else __rb_free_aux(rb); @@ -803,8 +803,8 @@ void rb_free_aux(struct perf_buffer *rb) * Back perf_mmap() with regular GFP_KERNEL-0 pages. */ -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { if (pgoff > rb->nr_pages) return NULL; @@ -835,10 +835,12 @@ static void perf_mmap_free_page(void *addr) __free_page(page); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); int i, node; size = sizeof(struct perf_buffer); @@ -863,6 +865,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } rb->nr_pages = nr_pages; + rb->mmap_page_shift = mmap_page_shift; + /* A ring smaller than a native page still wraps at its requested size. */ + rb->data_page_shift = data_size && data_size < PAGE_SIZE ? + ilog2(data_size) : PAGE_SHIFT; ring_buffer_init(rb, watermark, flags); @@ -892,8 +898,8 @@ void rb_free(struct perf_buffer *rb) } #else -static struct page * -__perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) +struct page * +perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) { /* The '>' counts in the user page. */ if (pgoff > data_page_nr(rb)) @@ -917,10 +923,12 @@ void rb_free(struct perf_buffer *rb) schedule_work(&rb->work); } -struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) +struct perf_buffer *rb_alloc(unsigned long data_size, unsigned int mmap_page_shift, + long watermark, int cpu, int flags) { struct perf_buffer *rb; unsigned long size; + int nr_pages = DIV_ROUND_UP(data_size, PAGE_SIZE); void *all_buf; int node; @@ -940,9 +948,10 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) rb->user_page = all_buf; rb->data_pages[0] = all_buf + PAGE_SIZE; + rb->mmap_page_shift = mmap_page_shift; if (nr_pages) { rb->nr_pages = 1; - rb->page_order = ilog2(nr_pages); + rb->data_page_shift = ilog2(data_size); } ring_buffer_init(rb, watermark, flags); @@ -957,21 +966,3 @@ struct perf_buffer *rb_alloc(int nr_pages, long watermark, int cpu, int flags) } #endif - -struct page * -perf_mmap_to_page(struct perf_buffer *rb, unsigned long pgoff) -{ - if (rb->aux_nr_pages) { - /* above AUX space */ - if (pgoff > rb->aux_pgoff + rb->aux_nr_pages) - return NULL; - - /* AUX space */ - if (pgoff >= rb->aux_pgoff) { - int aux_pgoff = array_index_nospec(pgoff - rb->aux_pgoff, rb->aux_nr_pages); - return virt_to_page(rb->aux_pages[aux_pgoff]); - } - } - - return __perf_mmap_to_page(rb, pgoff); -} diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012cbe..640991b1c25cb8 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -30,6 +30,7 @@ #include #include /* check_stable_address_space */ #include +#include #include @@ -144,12 +145,14 @@ static bool valid_vma(struct vm_area_struct *vma, bool is_register) static unsigned long offset_to_vaddr(struct vm_area_struct *vma, loff_t offset) { - return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + return vma->vm_start + offset - ((loff_t)vma->vm_pgoff << PAGE_SHIFT) - + vma_subpage_offset(vma); } static loff_t vaddr_to_offset(struct vm_area_struct *vma, unsigned long vaddr) { - return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + (vaddr - vma->vm_start); + return ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma) + + (vaddr - vma->vm_start); } /** @@ -191,7 +194,8 @@ static void copy_to_page(struct page *page, unsigned long vaddr, const void *src kunmap_local(kaddr); } -static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t *insn, +static int verify_opcode(struct page *page, unsigned long vaddr, unsigned int page_offset, + uprobe_opcode_t *insn, int nbytes, void *data) { uprobe_opcode_t old_opcode; @@ -206,7 +210,7 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t * is a trap variant; uprobes always wins over any other (gdb) * breakpoint. */ - uprobe_copy_from_page(page, vaddr, &old_opcode, UPROBE_SWBP_INSN_SIZE); + uprobe_copy_from_page(page, page_offset, &old_opcode, UPROBE_SWBP_INSN_SIZE); is_swbp = is_swbp_insn(&old_opcode); if (is_swbp_insn(insn)) { @@ -307,14 +311,24 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) { void *kaddr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm) != PAGE_SIZE; +#endif int ret; short *ptr; if (!vaddr || !d) return -EINVAL; - ret = get_user_pages_remote(mm, vaddr, 1, - FOLL_WRITE, &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = get_user_fragment_vma_remote(mm, vaddr, sizeof(short), + FOLL_WRITE, &fragment, &vma); + else +#endif + ret = get_user_pages_remote(mm, vaddr, 1, FOLL_WRITE, &page, NULL); if (unlikely(ret <= 0)) { /* * We are asking for 1 page. If get_user_pages_remote() fails, @@ -323,8 +337,16 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) return ret == 0 ? -EBUSY : ret; } - kaddr = kmap_local_page(page); - ptr = kaddr + (vaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (vaddr & ~PAGE_MASK); + } if (unlikely(*ptr + d < 0)) { pr_warn("ref_ctr going negative. vaddr: 0x%lx, " @@ -337,7 +359,12 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d) ret = 0; out: kunmap_local(kaddr); - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, !ret); + else +#endif + put_page(page); return ret; } @@ -381,7 +408,8 @@ static int update_ref_ctr(struct uprobe *uprobe, struct mm_struct *mm, } static bool orig_page_is_identical(struct vm_area_struct *vma, - unsigned long vaddr, struct page *page, bool *pmd_mappable) + unsigned long vaddr, struct page *page, unsigned int page_offset, + bool *pmd_mappable) { const pgoff_t index = vaddr_to_offset(vma, vaddr) >> PAGE_SHIFT; struct folio *orig_folio = filemap_get_folio(vma->vm_file->f_mapping, @@ -393,19 +421,30 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, return false; orig_page = folio_file_page(orig_folio, index); - *pmd_mappable = folio_test_pmd_mappable(orig_folio); - identical = folio_test_uptodate(orig_folio) && - pages_identical(page, orig_page); + *pmd_mappable = mm_page_size(vma->vm_mm) == PAGE_SIZE && + folio_test_pmd_mappable(orig_folio); + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned int offset = vma_page_offset_at(vma, vaddr).offset; + void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); + + identical = folio_test_uptodate(orig_folio) && + !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); + kunmap_local(b); + kunmap_local(a); + } else { + identical = folio_test_uptodate(orig_folio) && + pages_identical(page, orig_page); + } folio_put(orig_folio); return identical; } static int __uprobe_write(struct vm_area_struct *vma, - struct folio_walk *fw, struct folio *folio, - unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, + struct folio_walk *fw, struct folio *folio, struct mm_subpage *slot, + unsigned int page_offset, unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, bool is_register) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; + const unsigned long vaddr = insn_vaddr & mm_page_mask(vma->vm_mm); bool pmd_mappable; /* For now, we'll only handle PTE-mapped folios. */ @@ -417,6 +456,12 @@ static int __uprobe_write(struct vm_area_struct *vma, * but the VMA might not be writable. */ if (!pte_write(fw->pte)) { +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_is_exclusive(slot)) + return -EFAULT; + } else +#endif if (!PageAnonExclusive(fw->page)) return -EFAULT; if (unlikely(userfaultfd_pte_wp(vma, fw->pte))) @@ -430,31 +475,49 @@ static int __uprobe_write(struct vm_area_struct *vma, */ flush_cache_page(vma, vaddr, pte_pfn(fw->pte)); fw->pte = ptep_clear_flush(vma, vaddr, fw->ptep); - copy_to_page(fw->page, insn_vaddr, insn, nbytes); + copy_to_page(fw->page, page_offset, insn, nbytes); /* * When unregistering, we may only zap a PTE if uffd is disabled and * there are no unexpected folio references ... */ - if (is_register || userfaultfd_missing(vma) || - (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1)) + if (is_register || userfaultfd_missing(vma)) + goto remap; +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + if (!mm_subpage_can_zap(slot)) + goto remap; + } else +#endif + if (folio_ref_count(folio) != folio_expected_ref_count(folio) + 1) goto remap; /* * ... and the mapped page is identical to the original page that * would get faulted in on next access. */ - if (!orig_page_is_identical(vma, vaddr, fw->page, &pmd_mappable)) + if (!orig_page_is_identical(vma, vaddr, fw->page, + page_offset - (insn_vaddr - vaddr), &pmd_mappable)) goto remap; dec_mm_counter(vma->vm_mm, MM_ANONPAGES); - folio_remove_rmap_pte(folio, fw->page, vma); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_remove_anon_rmap(slot, vma); + else +#endif + folio_remove_rmap_pte(folio, fw->page, vma); if (!folio_mapped(folio) && folio_test_swapcache(folio) && folio_trylock(folio)) { folio_free_swap(folio); folio_unlock(folio); } - folio_put(folio); +#ifdef CONFIG_MM_SUBPAGE + if (slot) + mm_subpage_put(slot); /* Retired PTE reference; caller still has its GET. */ + else +#endif + folio_put(folio); return pmd_mappable; remap: @@ -494,13 +557,24 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct vm_area_struct *vma, verify_opcode, is_register, true /* do_update_ref_ctr */, NULL); } +static void uprobe_put_folio(struct folio *folio, struct mm_subpage *slot) +{ +#ifdef CONFIG_MM_SUBPAGE + if (slot) { + mm_subpage_put(slot); + return; + } +#endif + folio_put(folio); +} + int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, const unsigned long insn_vaddr, uprobe_opcode_t *insn, int nbytes, uprobe_write_verify_t verify, bool is_register, bool do_update_ref_ctr, void *data) { - const unsigned long vaddr = insn_vaddr & PAGE_MASK; struct mm_struct *mm = vma->vm_mm; + const unsigned long vaddr = insn_vaddr & mm_page_mask(mm); struct uprobe *uprobe; int ret, ref_ctr_updated = 0; unsigned int gup_flags = FOLL_FORCE; @@ -508,7 +582,16 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, struct folio_walk fw; struct folio *folio; struct page *page; - + struct mm_subpage *slot = NULL; + unsigned int page_offset; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *found; +#endif + + /* Larger-than-native public mms are not enabled by the mixed-page ABI yet. */ + if (mm_page_size(mm) > PAGE_SIZE) + return -EOPNOTSUPP; uprobe = container_of(auprobe, struct uprobe, arch); if (WARN_ON_ONCE(!is_cow_mapping(vma->vm_flags))) @@ -526,14 +609,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; retry: - ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); - if (ret <= 0) - goto out; - folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = get_user_fragment_vma_remote(mm, vaddr, mm_page_size(mm), + gup_flags & ~FOLL_SPLIT_PMD, &fragment, &found); + if (ret <= 0) + goto out; + folio = fragment.folio; + slot = fragment.subpage; + page = folio_page(folio, fragment.offset >> PAGE_SHIFT); + page_offset = offset_in_page(fragment.offset) + insn_vaddr - vaddr; + } else +#endif + { + ret = get_user_pages_remote(mm, vaddr, 1, gup_flags, &page, NULL); + if (ret <= 0) + goto out; + folio = page_folio(page); + page_offset = offset_in_page(insn_vaddr); + } - ret = verify(page, insn_vaddr, insn, nbytes, data); + ret = verify(page, insn_vaddr, page_offset, insn, nbytes, data); if (ret <= 0) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -541,7 +639,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, if (do_update_ref_ctr && !ref_ctr_updated && uprobe->ref_ctr_offset) { ret = update_ref_ctr(uprobe, mm, is_register ? 1 : -1); if (ret) { - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -551,7 +649,7 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, ret = 0; if (unlikely(!folio_test_anon(folio) || folio_is_zone_device(folio))) { VM_WARN_ON_ONCE(is_register); - folio_put(folio); + uprobe_put_folio(folio, slot); goto out; } @@ -562,22 +660,29 @@ int uprobe_write(struct arch_uprobe *auprobe, struct vm_area_struct *vma, * be able to do it under PTL. */ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vaddr, vaddr + PAGE_SIZE); + vaddr, vaddr + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); } ret = -EAGAIN; /* Walk the page tables again, to perform the actual update. */ if (folio_walk_start(&fw, vma, vaddr, 0)) { - if (fw.page == page) - ret = __uprobe_write(vma, &fw, folio, insn_vaddr, insn, nbytes, is_register); + bool same = fw.page == page; +#ifdef CONFIG_MM_SUBPAGE + if (slot) + same &= fw.level == FW_LEVEL_PTE && + pte_phys_mm(mm, fw.pte) == mm_subpage_phys(slot); +#endif + if (same) + ret = __uprobe_write(vma, &fw, folio, slot, page_offset, + insn_vaddr, insn, nbytes, is_register); folio_walk_end(&fw, vma); } if (!is_register) mmu_notifier_invalidate_range_end(&range); - folio_put(folio); + uprobe_put_folio(folio, slot); switch (ret) { case -EFAULT: gup_flags |= FOLL_WRITE | FOLL_SPLIT_PMD; @@ -1211,7 +1316,9 @@ build_map_info(struct address_space *mapping, loff_t offset, bool is_register) again: i_mmap_lock_read(mapping); vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { - if (!valid_vma(vma, is_register)) + if (!valid_vma(vma, is_register) || + offset < vaddr_to_offset(vma, vma->vm_start) || + offset >= vaddr_to_offset(vma, vma->vm_end)) continue; if (!prev && !more) { @@ -1482,7 +1589,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm) file_inode(vma->vm_file) != uprobe->inode) continue; - offset = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + offset = vaddr_to_offset(vma, vma->vm_start); if (uprobe->offset < offset || uprobe->offset >= offset + vma->vm_end - vma->vm_start) continue; @@ -1696,8 +1803,14 @@ static const struct vm_special_mapping xol_mapping = { unsigned long __weak arch_uprobe_get_xol_area(void) { - /* Try to map as high as possible, this is only a hint. */ - return get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE, PAGE_SIZE, 0, 0); + unsigned long size = PAGE_SIZE; + unsigned long addr; + + /* XOL arch copy hooks use native offsets into the one backing page. */ + if (mm_page_size(current->mm) < PAGE_SIZE) + size += PAGE_SIZE - mm_page_size(current->mm); + addr = get_unmapped_area(NULL, TASK_SIZE - size, size, 0, 0); + return IS_ERR_VALUE(addr) ? addr : ALIGN(addr, PAGE_SIZE); } /* Slot allocation for XOL */ @@ -2413,6 +2526,23 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr) if (likely(result == 0)) goto out; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *vma; + void *kaddr; + + result = get_user_fragment_vma_remote(mm, vaddr, UPROBE_SWBP_INSN_SIZE, + FOLL_FORCE, &fragment, &vma); + if (result != 1) + return result < 0 ? result : -EFAULT; + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + memcpy(&opcode, kaddr, UPROBE_SWBP_INSN_SIZE); + kunmap_local(kaddr); + release_user_fragments(&fragment, 1, false); + goto out; + } +#endif result = get_user_pages(vaddr, 1, FOLL_FORCE, &page); if (result < 0) return result; @@ -2453,7 +2583,7 @@ static struct uprobe *find_active_uprobe_speculative(unsigned long bp_vaddr) if (!vm_file) return NULL; - offset = (loff_t)(vma->vm_pgoff << PAGE_SHIFT) + (bp_vaddr - vma->vm_start); + offset = vaddr_to_offset(vma, bp_vaddr); uprobe = find_uprobe_rcu(vm_file->f_inode, offset); if (!uprobe) return NULL; diff --git a/kernel/fork.c b/kernel/fork.c index 5a4623b058a20b..88788baaff964c 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -112,7 +112,7 @@ #include #include -#include +#include #include #include @@ -639,6 +639,12 @@ static void check_mm(struct mm_struct *mm) } } +#ifdef CONFIG_MM_SUBPAGE + if (mm->committed_user_pages) + pr_alert("BUG: non-zero user commitment on freeing mm: %lu\n", + mm->committed_user_pages); +#endif + if (mm_pgtables_bytes(mm)) pr_alert("BUG: non-zero pgtables_bytes on freeing mm: %ld\n", mm_pgtables_bytes(mm)); @@ -1089,6 +1095,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, INIT_LIST_HEAD(&mm->mmlist); mm_pgtables_bytes_init(mm); mm->map_count = 0; +#ifdef CONFIG_MM_SUBPAGE + mm->committed_user_pages = 0; +#endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); @@ -1159,7 +1168,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, /* * Allocate and initialize an mm_struct. */ -struct mm_struct *mm_alloc(void) +static struct mm_struct *__mm_alloc(bool for_exec, unsigned int page_shift) { struct mm_struct *mm; @@ -1168,10 +1177,32 @@ struct mm_struct *mm_alloc(void) return NULL; memset(mm, 0, sizeof(*mm)); + if (for_exec) + arch_mm_init_exec(mm, page_shift); return mm_init(mm, current, current_user_ns()); } + +struct mm_struct *mm_alloc(void) +{ + return __mm_alloc(false, 0); +} EXPORT_SYMBOL_IF_KUNIT(mm_alloc); +struct mm_struct *mm_alloc_exec(void) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + return __mm_alloc(true, current->exec_page_shift); +#else + return __mm_alloc(true, 0); +#endif +} + +/* The caller has validated the requested exec granule. */ +struct mm_struct *mm_alloc_exec_page_shift(unsigned int page_shift) +{ + return __mm_alloc(true, page_shift); +} + static inline void __mmput(struct mm_struct *mm) { VM_BUG_ON(atomic_read(&mm->mm_users)); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 7ffe6dc79b28e8..771c12983cd2a7 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include "futex.h" @@ -512,6 +513,44 @@ static u64 get_inode_sequence_number(struct inode *inode) } } +/* Keep a typed slot reference alive until the mapping identity is sampled. */ +struct futex_page_ref { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; +#endif +}; + +static int futex_get_page_ref(struct mm_struct *mm, unsigned long address, + unsigned long uaddr, unsigned int size, unsigned int flags, + struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + int ret = get_user_fragments_remote(mm, uaddr, size, flags, + &ref->fragment, 1); + + if (ret != 1) + return ret ?: -EFAULT; + ref->page = folio_page(ref->fragment.folio, + ref->fragment.offset >> PAGE_SHIFT); + return 1; + } +#endif + return get_user_pages_fast(address, 1, flags, &ref->page); +} + +static void futex_put_page_ref(struct futex_page_ref *ref) +{ +#ifdef CONFIG_MM_SUBPAGE + if (ref->fragment.folio) { + release_user_fragments(&ref->fragment, 1, false); + return; + } +#endif + put_page(ref->page); +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -545,6 +584,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; struct page *page; + struct futex_page_ref ref = {}; struct folio *folio; struct address_space *mapping; int node, err, size, ro = 0; @@ -629,13 +669,13 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (unlikely(should_fail_futex(true))) return -EFAULT; - err = get_user_pages_fast(address, 1, FOLL_WRITE, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, FOLL_WRITE, &ref); /* * If write access is not required (eg. FUTEX_WAIT), try * and get read-only access. */ if (err == -EFAULT && rw == FUTEX_READ) { - err = get_user_pages_fast(address, 1, 0, &page); + err = futex_get_page_ref(mm, address, (unsigned long)uaddr, size, 0, &ref); ro = 1; } if (err < 0) @@ -660,6 +700,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * filesystem-backed pages, the precise page is required as the * index of the page determines the key. */ + page = ref.page; folio = page_folio(page); mapping = READ_ONCE(folio->mapping); @@ -689,7 +730,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, folio_lock(folio); shmem_swizzled = folio_test_swapcache(folio) || folio->mapping; folio_unlock(folio); - folio_put(folio); + futex_put_page_ref(&ref); if (shmem_swizzled) goto again; @@ -739,7 +780,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, if (READ_ONCE(folio->mapping) != mapping) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } @@ -747,11 +788,15 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, inode = READ_ONCE(mapping->host); if (!inode) { rcu_read_unlock(); - folio_put(folio); + futex_put_page_ref(&ref); goto again; } +#ifdef CONFIG_MM_SUBPAGE + if (ref.fragment.folio) + key->both.offset = offset_in_page(ref.fragment.offset); +#endif key->both.offset |= FUT_OFF_INODE; /* inode-based key */ key->shared.i_seq = get_inode_sequence_number(inode); key->shared.pgoff = page_pgoff(folio, page); @@ -759,7 +804,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, } out: - folio_put(folio); + futex_put_page_ref(&ref); return err; } diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index abb76775aa7287..e06b05a08f7f4b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1694,7 +1694,7 @@ static unsigned int task_nr_scan_windows(struct task_struct *p) * on resident pages */ nr_scan_pages = MB_TO_PAGES(sysctl_numa_balancing_scan_size); - rss = get_mm_rss(p->mm); + rss = mm_pages_to_native(p->mm, get_mm_rss(p->mm)); if (!rss) rss = nr_scan_pages; @@ -3682,8 +3682,8 @@ static void task_numa_work(struct callback_head *work) * areas faster. */ if (nr_pte_updates) - pages -= (end - start) >> PAGE_SHIFT; - virtpages -= (end - start) >> PAGE_SHIFT; + pages -= DIV_ROUND_UP(end - start, PAGE_SIZE); + virtpages -= DIV_ROUND_UP(end - start, PAGE_SIZE); start = end; if (pages <= 0 || virtpages <= 0) diff --git a/kernel/sys.c b/kernel/sys.c index ec432b89e85406..09702e8f773ba0 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -1927,7 +1927,7 @@ void getrusage(struct task_struct *p, int who, struct rusage *r) } out_children: - r->ru_maxrss = maxrss * (PAGE_SIZE / 1024); /* convert pages to KBs */ + r->ru_maxrss = maxrss; /* signal and mm high-water maxima are in kB */ r->ru_utime = ns_to_kernel_old_timeval(utime); r->ru_stime = ns_to_kernel_old_timeval(stime); } @@ -2532,6 +2532,40 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +static int prctl_page_size_get(struct task_struct *task, bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + unsigned int shift = persistent ? task->default_exec_page_shift : + task->exec_page_shift; + + return shift ? 1U << shift : 0; +#else + return -EINVAL; +#endif +} + +static int prctl_page_size_set(struct task_struct *task, unsigned long size, + bool persistent) +{ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + int shift = arch_exec_page_size_shift(size); + + if (shift < 0) + return shift; + if (persistent) { + task->default_exec_page_shift = shift; + } else { + /* A native request overrides an inherited default for one exec. */ + if (!shift && task->default_exec_page_shift) + shift = PAGE_SHIFT; + task->exec_page_shift = shift; + } + return 0; +#else + return -EINVAL; +#endif +} + int __weak arch_prctl_mem_model_get(struct task_struct *t) { return -EINVAL; @@ -2555,6 +2589,22 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { + case PR_AURORA_GET_EXEC_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, false); + case PR_AURORA_SET_EXEC_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, false); + case PR_AURORA_GET_DEFAULT_PAGE_SIZE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_get(me, true); + case PR_AURORA_SET_DEFAULT_PAGE_SIZE: + if (arg3 || arg4 || arg5) + return -EINVAL; + return prctl_page_size_set(me, arg2, true); case PR_GET_MEM_MODEL: if (arg2 || arg3 || arg4 || arg5) return -EINVAL; diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 8c82ecb735f415..e8b622a8178cb7 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include "trace_dynevent.h" @@ -554,6 +555,11 @@ static int user_event_enabler_write(struct user_event_mm *mm, unsigned long uaddr = enabler->addr; unsigned long *ptr; struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment; + struct vm_area_struct *vma; + bool alternative = mm_page_size(mm->mm) != PAGE_SIZE; +#endif void *kaddr; int bit = ENABLE_BIT(enabler); int ret; @@ -573,8 +579,14 @@ static int user_event_enabler_write(struct user_event_mm *mm, align_addr_bit(&uaddr, &bit, ENABLE_BITOPS(enabler)); - ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + ret = pin_user_fragment_vma_remote(mm->mm, uaddr, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &vma); + else +#endif + ret = pin_user_pages_remote(mm->mm, uaddr, 1, FOLL_WRITE | FOLL_NOFAULT, + &page, NULL); if (unlikely(ret <= 0)) { if (!fixup_fault) @@ -586,8 +598,17 @@ static int user_event_enabler_write(struct user_event_mm *mm, return -EFAULT; } - kaddr = kmap_local_page(page); - ptr = kaddr + (uaddr & ~PAGE_MASK); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) { + /* align_addr_bit() keeps the complete atomic word in this fragment. */ + kaddr = kmap_local_folio(fragment.folio, fragment.offset); + ptr = kaddr; + } else +#endif + { + kaddr = kmap_local_page(page); + ptr = kaddr + (uaddr & ~PAGE_MASK); + } /* Update bit atomically, user tracers must be atomic as well */ if (enabler->event && enabler->event->status) @@ -596,7 +617,12 @@ static int user_event_enabler_write(struct user_event_mm *mm, clear_bit(bit, ptr); kunmap_local(kaddr); - unpin_user_pages_dirty_lock(&page, 1, true); +#ifdef CONFIG_MM_SUBPAGE + if (alternative) + release_user_fragments(&fragment, 1, true); + else +#endif + unpin_user_pages_dirty_lock(&page, 1, true); return 0; } diff --git a/kernel/tsacct.c b/kernel/tsacct.c index 5c153106e64271..5f59d071ab5655 100644 --- a/kernel/tsacct.c +++ b/kernel/tsacct.c @@ -91,16 +91,16 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p) { struct mm_struct *mm; - /* convert pages-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ - stats->coremem = p->acct_rss_mem1 * PAGE_SIZE; + /* convert account-units-nsec/1024 to Mbyte-usec, see __acct_update_integrals */ + stats->coremem = p->acct_rss_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->coremem, 1000 * KB); - stats->virtmem = p->acct_vm_mem1 * PAGE_SIZE; + stats->virtmem = p->acct_vm_mem1 * (1UL << MM_ACCOUNT_SHIFT); do_div(stats->virtmem, 1000 * KB); mm = get_task_mm(p); if (mm) { /* adjust to KB unit */ - stats->hiwater_rss = get_mm_hiwater_rss(mm) * PAGE_SIZE / KB; - stats->hiwater_vm = get_mm_hiwater_vm(mm) * PAGE_SIZE / KB; + stats->hiwater_rss = mm_pages_to_kb(mm, get_mm_hiwater_rss(mm)); + stats->hiwater_vm = mm_pages_to_kb(mm, get_mm_hiwater_vm(mm)); mmput(mm); } stats->read_char = p->ioac.rchar & KB_MASK; @@ -140,8 +140,10 @@ static void __acct_update_integrals(struct task_struct *tsk, * The final unit reported to userspace is Mbyte-usecs, * the rest of the math is done in xacct_add_tsk. */ - tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm) >> 10; - tsk->acct_vm_mem1 += delta * READ_ONCE(tsk->mm->total_vm) >> 10; + tsk->acct_rss_mem1 += delta * mm_pages_to_shift(tsk->mm, get_mm_rss(tsk->mm), + MM_ACCOUNT_SHIFT) >> 10; + tsk->acct_vm_mem1 += delta * mm_pages_to_shift(tsk->mm, READ_ONCE(tsk->mm->total_vm), + MM_ACCOUNT_SHIFT) >> 10; } /** diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index 17d37b82ebc685..d808ddcb94b01a 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -83,7 +83,8 @@ static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, vm_fault_t err; addr = vmf->address + VDSO_TIMENS_PAGE_OFFSET * PAGE_SIZE; - err = vmf_insert_page(vma, addr, page); + err = vmf_insert_page_offset(vma, addr, page, + vma_page_offset_at(vma, vmf->address).offset); if (unlikely(err & VM_FAULT_ERROR)) return err; page = timens_page; diff --git a/mm/Kconfig b/mm/Kconfig index e8bf1e9e6ad907..b743b2290b3091 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1164,6 +1164,28 @@ config IDLE_PAGE_TRACKING See Documentation/admin-guide/mm/idle_page_tracking.rst for more details. +config ARCH_HAS_MM_PGTABLE + bool + help + Architecture supplies target-mm page-table geometry and accessors. + This does not change the native physical allocation granule. + +config ARCH_HAS_USER_PAGE_SIZE + bool + help + Architecture supports selecting the user mapping granule at exec. + It supplies arch_exec_page_size_shift() to validate requests, and + integrates the selected granule into mm allocation and binary loading. + +config MM_SUBPAGE + bool + select PAGE_EXTENSION + select XARRAY_MULTI + help + Reference-counted user data slots smaller than a native folio. + Architectures must integrate rmap, faults, reclaim and pins before + exposing these backing primitives to userspace. + # Architectures which implement cpu_dcache_is_aliasing() to query # whether the data caches are aliased (VIVT or VIPT with dcache # aliasing) need to select this. @@ -1439,9 +1461,15 @@ config ARCH_HAS_USER_SHADOW_STACK config HAVE_ARCH_TLB_REMOVE_TABLE def_bool n +config ARCH_SUPPORTS_PT_RECLAIM + bool + help + The architecture's custom table-free callback supports PTE reclamation. + config PT_RECLAIM def_bool y - depends on MMU_GATHER_RCU_TABLE_FREE && !HAVE_ARCH_TLB_REMOVE_TABLE + depends on MMU_GATHER_RCU_TABLE_FREE + depends on !HAVE_ARCH_TLB_REMOVE_TABLE || ARCH_SUPPORTS_PT_RECLAIM help Try to reclaim empty user page table pages in paths other than munmap and exit_mmap path. diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index cca7014ab5f137..3d618828c2c8dd 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -155,9 +155,10 @@ int damon_cold_score(struct damon_ctx *c, struct damon_region *r, } static bool damon_folio_mkold_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); while (page_vma_mapped_walk(&pvmw)) { addr = pvmw.address; @@ -190,10 +191,11 @@ void damon_folio_mkold(struct folio *folio) } static bool damon_folio_young_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long addr = rmap.address; bool *accessed = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); pte_t pte; *accessed = false; diff --git a/mm/filemap.c b/mm/filemap.c index bbd400d893aa5e..14e2656b6a9ccc 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -225,6 +225,7 @@ void __filemap_remove_folio(struct folio *folio, void *shadow) trace_mm_filemap_delete_from_page_cache(folio); filemap_unaccount_folio(mapping, folio); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); page_cache_delete(mapping, folio, shadow); } @@ -308,6 +309,7 @@ static void page_cache_delete_batch(struct address_space *mapping, WARN_ON_ONCE(!folio_test_locked(folio)); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); folio->mapping = NULL; /* Leave folio->index set: truncation lookup relies on it */ @@ -827,6 +829,7 @@ void replace_page_cache_folio(struct folio *old, struct folio *new) mem_cgroup_replace_folio(old, new); xas_lock_irq(&xas); + shmem_uffd_migrate(new, old); xas_store(&xas, new); old->mapping = NULL; @@ -3387,7 +3390,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf) */ struct vm_area_struct *vma = vmf->vma; unsigned long start = vma->vm_pgoff; - unsigned long end = start + vma_pages(vma); + unsigned long end = vma_last_pgoff(vma) + 1; unsigned long ra_end; ra->order = exec_folio_order(); @@ -3529,7 +3532,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) bool mapping_locked = false; max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) return VM_FAULT_SIGBUS; trace_mm_filemap_fault(mapping, index); @@ -3633,7 +3641,12 @@ vm_fault_t filemap_fault(struct vm_fault *vmf) * We must recheck i_size under page lock. */ max_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - if (unlikely(index >= max_idx)) { + if (unlikely(index >= max_idx +#ifdef CONFIG_MM_SUBPAGE + || (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) +#endif + )) { folio_unlock(folio); folio_put(folio); return VM_FAULT_SIGBUS; @@ -3783,6 +3796,10 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf, } do { + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index + + folio_page_idx(folio, page + count), 0, PAGE_SIZE)) + goto skip; if (PageHWPoison(page + count)) goto skip; @@ -3847,6 +3864,9 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, vm_fault_t ret = 0; struct page *page = &folio->page; + if (userfaultfd_missing(vmf->vma) && + shmem_uffd_range_missing(folio, folio->index, 0, PAGE_SIZE)) + goto out; if (PageHWPoison(page)) goto out; @@ -3875,6 +3895,48 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t filemap_map_user_pages(struct vm_fault *vmf, + pgoff_t first, pgoff_t last) +{ + struct vm_area_struct *vma = vmf->vma; + struct address_space *mapping = vma->vm_file->f_mapping; + XA_STATE(xas, &mapping->i_pages, first); + struct folio *folio; + vm_fault_t ret = 0; + unsigned short miss = 0, saved; + + rcu_read_lock(); + folio = next_uptodate_folio(&xas, mapping, last); + if (!folio) + goto out; + if (pmd_none(*vmf->pmd) && vmf->prealloc_pte) + pmd_install(vma->vm_mm, vmf->pmd, &vmf->prealloc_pte); + vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, + vmf->address, &vmf->ptl); + if (!vmf->pte) { + folio_unlock(folio); + folio_put(folio); + goto out; + } + do { + ret |= map_file_user_pages(vmf, folio, first, last, false); + if (!folio_test_workingset(folio)) + miss += min(folio_next_index(folio) - 1, last) - + max(folio->index, first) + 1; + folio_unlock(folio); + folio_put(folio); + } while ((folio = next_uptodate_folio(&xas, mapping, last)) != NULL); + pte_unmap_unlock(vmf->pte, vmf->ptl); + trace_mm_filemap_map_pages(mapping, first, last); +out: + rcu_read_unlock(); + saved = READ_ONCE(vma->vm_file->f_ra.mmap_miss); + WRITE_ONCE(vma->vm_file->f_ra.mmap_miss, saved > miss ? saved - miss : 0); + return ret; +} +#endif + vm_fault_t filemap_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) { @@ -3890,6 +3952,11 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf, unsigned int nr_pages = 0, folio_type; unsigned short mmap_miss = 0, mmap_miss_saved; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return filemap_map_user_pages(vmf, start_pgoff, end_pgoff); +#endif + /* * Recalculate end_pgoff based on file_end before calling * next_uptodate_folio() to avoid races with concurrent diff --git a/mm/gup.c b/mm/gup.c index ad9ded39609cb6..8d9bbc80546fa2 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -1297,6 +1298,11 @@ static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, #endif } +#ifdef CONFIG_MM_SUBPAGE +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked); +#endif + /** * __get_user_pages() - pin user pages in memory * @mm: mm_struct of target mm @@ -1363,6 +1369,15 @@ static long __get_user_pages(struct mm_struct *mm, if (!nr_pages) return 0; +#ifdef CONFIG_MM_SUBPAGE + /* A prefault has no page-array ownership or physical-offset ambiguity. */ + if (mm_page_size(mm) != PAGE_SIZE) { + if (pages || (gup_flags & (FOLL_GET | FOLL_PIN))) + return -EOPNOTSUPP; + return prefault_subpage_range(mm, start, nr_pages, gup_flags, locked); + } +#endif + start = untagged_addr_remote(mm, start); VM_WARN_ON_ONCE(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); @@ -1721,7 +1736,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, */ if (likely(pages)) pages += ret; - start += ret << PAGE_SHIFT; + start += ret << mm_page_shift(mm); /* The lock was temporarily dropped, so we must unlock later */ must_unlock = true; @@ -1768,7 +1783,7 @@ static __always_inline long __get_user_pages_locked(struct mm_struct *mm, break; if (likely(pages)) pages++; - start += PAGE_SIZE; + start += mm_page_size(mm); } if (must_unlock && *locked) { /* @@ -1814,13 +1829,13 @@ long populate_vma_page_range(struct vm_area_struct *vma, unsigned long start, unsigned long end, int *locked) { struct mm_struct *mm = vma->vm_mm; - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int local_locked = 1; int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); VM_WARN_ON_ONCE_VMA(start < vma->vm_start, vma); VM_WARN_ON_ONCE_VMA(end > vma->vm_end, vma); mmap_assert_locked(mm); @@ -1887,12 +1902,12 @@ long populate_vma_page_range(struct vm_area_struct *vma, long faultin_page_range(struct mm_struct *mm, unsigned long start, unsigned long end, bool write, int *locked) { - unsigned long nr_pages = (end - start) / PAGE_SIZE; + unsigned long nr_pages = (end - start) / mm_page_size(mm); int gup_flags; long ret; - VM_WARN_ON_ONCE(!PAGE_ALIGNED(start)); - VM_WARN_ON_ONCE(!PAGE_ALIGNED(end)); + VM_WARN_ON_ONCE(!IS_ALIGNED(start, mm_page_size(mm))); + VM_WARN_ON_ONCE(!IS_ALIGNED(end, mm_page_size(mm))); mmap_assert_locked(mm); /* @@ -1968,7 +1983,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) } break; } - nend = nstart + ret * PAGE_SIZE; + nend = nstart + ret * mm_page_size(mm); ret = 0; } if (locked) @@ -2047,6 +2062,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; if (unlikely(size == 0)) @@ -2055,7 +2071,7 @@ size_t fault_in_writeable(char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_put_user(0, (char __user *)cur, out); out: user_write_access_end(); @@ -2116,6 +2132,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; struct mm_struct *mm = current->mm; bool unlocked = false; @@ -2125,7 +2142,7 @@ size_t fault_in_safe_writeable(const char __user *uaddr, size_t size) mmap_read_lock(mm); /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) if (fixup_user_fault(mm, cur, FAULT_FLAG_WRITE, &unlocked)) break; mmap_read_unlock(mm); @@ -2148,6 +2165,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) { const unsigned long start = (unsigned long)uaddr; const unsigned long end = start + size; + const unsigned long page_size = current->mm ? mm_page_size(current->mm) : PAGE_SIZE; unsigned long cur; volatile char c; @@ -2157,7 +2175,7 @@ size_t fault_in_readable(const char __user *uaddr, size_t size) return size; /* Stop once we overflow to 0. */ - for (cur = start; cur && cur < end; cur = PAGE_ALIGN_DOWN(cur + PAGE_SIZE)) + for (cur = start; cur && cur < end; cur = round_down(cur + page_size, page_size)) unsafe_get_user(c, (const char __user *)cur, out); out: user_read_access_end(); @@ -2211,11 +2229,18 @@ struct pages_or_folios { void **entries; }; bool has_folios; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; +#endif long nr_entries; }; static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) + return pofs->fragments[i].folio; +#endif if (pofs->has_folios) return pofs->folios[i]; return page_folio(pofs->pages[i]); @@ -2223,22 +2248,56 @@ static struct folio *pofs_get_folio(struct pages_or_folios *pofs, long i) static void pofs_clear_entry(struct pages_or_folios *pofs, long i) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + pofs->fragments[i] = (struct user_page_fragment) {}; + return; + } +#endif pofs->entries[i] = NULL; } static void pofs_unpin(struct pages_or_folios *pofs) { +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(pofs->fragments, pofs->nr_entries, false); + return; + } +#endif if (pofs->has_folios) unpin_folios(pofs->folios, pofs->nr_entries); else unpin_user_pages(pofs->pages, pofs->nr_entries); } +static void pofs_unpin_entry(struct pages_or_folios *pofs, long i) +{ +#ifdef CONFIG_MM_SUBPAGE + if (pofs->fragments) { + release_user_fragments(&pofs->fragments[i], 1, false); + return; + } +#endif + unpin_folio(pofs_get_folio(pofs, i)); +} + static struct folio *pofs_next_folio(struct folio *folio, struct pages_or_folios *pofs, long *index_ptr) { long i = *index_ptr + 1; +#ifdef CONFIG_MM_SUBPAGE + /* Several adjacent user quarters can pin the same native folio. */ + if (pofs->fragments) { + while (i < pofs->nr_entries && pofs->fragments[i].folio == folio) + i++; + if (i == pofs->nr_entries) + return NULL; + *index_ptr = i; + return pofs->fragments[i].folio; + } +#endif if (!pofs->has_folios && folio_test_large(folio)) { const unsigned long start_pfn = folio_pfn(folio); const unsigned long end_pfn = start_pfn + folio_nr_pages(folio); @@ -2333,9 +2392,9 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * convert the pin on the source folio to a normal * reference. */ - pofs_clear_entry(pofs, i); folio_get(folio); - gup_put_folio(folio, 1, FOLL_PIN); + pofs_unpin_entry(pofs, i); + pofs_clear_entry(pofs, i); if (migrate_device_coherent_folio(folio)) { ret = -EBUSY; @@ -2352,7 +2411,7 @@ migrate_longterm_unpinnable_folios(struct list_head *movable_folio_list, * calling folio_isolate_lru() which takes a reference so the * folio won't be freed if it's migrating. */ - unpin_folio(folio); + pofs_unpin_entry(pofs, i); pofs_clear_entry(pofs, i); } @@ -2458,6 +2517,23 @@ static long check_and_migrate_movable_folios(unsigned long nr_folios, } #endif /* CONFIG_MIGRATION */ +#ifdef CONFIG_MM_SUBPAGE +static long check_and_migrate_movable_fragments(unsigned long count, + struct user_page_fragment *fragments) +{ +#ifdef CONFIG_MIGRATION + struct pages_or_folios pofs = { + .fragments = fragments, + .nr_entries = count, + }; + + return check_and_migrate_movable_pages_or_folios(&pofs); +#else + return 0; +#endif +} +#endif + /* * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which * allows us to process the FOLL_LONGTERM flag. @@ -3133,6 +3209,11 @@ static unsigned long gup_fast(unsigned long start, unsigned long end, int nr_pinned = 0; unsigned seq; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return 0; +#endif + if (!IS_ENABLED(CONFIG_HAVE_GUP_FAST) || !gup_fast_permitted(start, end)) return 0; @@ -3355,6 +3436,464 @@ long pin_user_pages_remote(struct mm_struct *mm, } EXPORT_SYMBOL(pin_user_pages_remote); +#ifdef CONFIG_MM_SUBPAGE +/* 1: acquired, 0: fault required, -EMLINK: read pin needs private backing. */ +static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long addr, + unsigned int flags, struct user_page_fragment *fragment) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + struct folio *folio; + struct page *page; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *ptep, pte; + spinlock_t *ptl; + phys_addr_t phys; + int ret = 0; + bool file_ref = false; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pud_leaf(*pud) || pud_bad(*pud)) + return -EOPNOTSUPP; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd)) + return PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + if (pmd_leaf(*pmd) || pmd_bad(*pmd)) + return -EOPNOTSUPP; + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + pte = ptep_get(ptep); + if (pte_none(pte)) { + ret = PTR_ERR_OR_ZERO(no_page_table(vma, flags, addr)); + goto out; + } + if (!pte_present(pte) || (pte_protnone(pte) && !gup_can_follow_protnone(vma, flags))) + goto out; + phys = pte_phys_mm(mm, pte); + page = vm_normal_page(vma, addr, pte); + if (mm_page_size(mm) > PAGE_SIZE) { + unsigned long offset = (addr & (mm_page_size(mm) - 1)) & PAGE_MASK; + + phys += offset; + if (page) + page += offset >> PAGE_SHIFT; + } + if (!page) { + if (flags & FOLL_DUMP) { + ret = -EFAULT; + goto out; + } + if (!is_zero_pfn(pte_pfn(pte))) { + ret = -EOPNOTSUPP; + goto out; + } + if (flags & FOLL_WRITE) + goto out; + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + if (flags & FOLL_GET) + folio_get(folio); + } else { + folio = page_folio(page); + /* Coarse leaves use ordinary native-page ownership, not slots. */ + if (mm_page_size(mm) > PAGE_SIZE) { + if ((flags & FOLL_WRITE) && + !can_follow_write_pte(pte, page, vma, flags)) + goto out; + if (!pte_write(pte) && gup_must_unshare(vma, flags, page)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (!folio_test_anon(folio)) { + if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { + ret = -EOPNOTSUPP; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) + goto out; + /* Private non-anonymous read pins must first acquire COW ownership. */ + if ((flags & FOLL_PIN) && is_cow_mapping(vma->vm_flags)) { + ret = -EMLINK; + goto out; + } + ret = try_grab_folio(folio, 1, flags); + if (ret) + goto out; + file_ref = true; + goto acquired; + } + if (folio_test_ksm(folio)) { + ret = -EOPNOTSUPP; + goto out; + } + slot = mm_subpage_get_from_phys(phys); + if (!slot) { + ret = -EFAULT; + goto out; + } + if ((flags & FOLL_WRITE) && !pte_write(pte)) { + bool force = (flags & FOLL_FORCE) && + !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE | VM_WRITE)) && + (vma->vm_flags & VM_MAYWRITE) && mm_subpage_is_exclusive(slot) && + !pte_needs_soft_dirty_wp(vma, pte) && !userfaultfd_pte_wp(vma, pte); + + if (!force) + goto out; + } + if ((flags & FOLL_PIN) && !mm_subpage_is_exclusive(slot)) { + ret = flags & FOLL_WRITE ? 0 : -EMLINK; + goto out; + } + if (flags & FOLL_PIN) { + ret = mm_subpage_pin(slot); + if (ret) { + if (ret == -EAGAIN) + ret = 0; + goto out; + } + } + } +acquired: + if (slot && (flags & FOLL_WRITE)) + mm_subpage_cancel_lazyfree(slot); + ret = flags & FOLL_PIN ? arch_make_folio_accessible(folio) : 0; + if (ret) { + if (slot) + mm_subpage_unpin(slot); + else if (file_ref) + gup_put_folio(folio, 1, flags); + goto out; + } + if ((flags & FOLL_TOUCH) && (flags & FOLL_WRITE) && + !pte_dirty(pte) && !folio_test_dirty(folio)) + folio_mark_dirty(folio); + folio_mark_accessed(folio); + flush_anon_page(vma, &folio->page, addr); + flush_dcache_folio(folio); + fragment->folio = folio; + fragment->subpage = slot; + fragment->pinned = flags & FOLL_PIN; + if (flags & FOLL_GET) + slot = NULL; /* The returned record consumes the lookup reference. */ + fragment->offset = phys - PFN_PHYS(folio_pfn(folio)) + + (addr & (mm_user_fragment_size(mm) - 1)); + ret = 1; +out: + if (slot) + mm_subpage_put(slot); /* Lookup reference; success retains a separate pin. */ + pte_unmap_unlock(ptep, ptl); + return ret; +} + +/* No references escape: only ensure each user leaf can be followed. */ +static long prefault_subpage_range(struct mm_struct *mm, unsigned long start, + unsigned long nr_pages, unsigned int flags, int *locked) +{ + unsigned long done = 0; + long ret = 0; + + mmap_assert_locked(mm); + start = untagged_addr_remote(mm, start); + while (done < nr_pages) { + struct user_page_fragment unused; + struct vm_area_struct *vma = vma_lookup(mm, start); + + if (!vma) { + ret = flags & FOLL_MADV_POPULATE ? -ENOMEM : -EFAULT; + break; + } + ret = check_vma_flags(vma, flags); + if (ret) { + if (flags & FOLL_MADV_POPULATE) + ret = -EINVAL; + break; + } + if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) { + ret = flags & FOLL_MADV_POPULATE ? -EINVAL : -EOPNOTSUPP; + break; + } + if (gup_signal_pending(flags)) { + ret = -EINTR; + break; + } + cond_resched(); + ret = follow_subpage_fragment(vma, start, flags, &unused); + if (!ret) { + ret = faultin_page(vma, start, flags, false, locked); + if (!ret) + continue; + if (ret == -EBUSY || ret == -EAGAIN) + ret = 0; /* Preserve GUP's caller-managed retry contract. */ + break; + } + if (ret < 0) + break; + done++; + start += mm_page_size(mm); + } + if (ret == -EOPNOTSUPP && (flags & FOLL_MADV_POPULATE)) + ret = -EINVAL; + return done ? (long)done : ret; +} + +static long user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity, bool pin, + struct vm_area_struct **vma_out, bool dump) +{ + const unsigned int allowed = FOLL_WRITE | FOLL_FORCE | FOLL_NOFAULT | + FOLL_NOWAIT | FOLL_INTERRUPTIBLE | FOLL_LONGTERM | FOLL_ANON; + unsigned int internal = flags | FOLL_TOUCH | FOLL_REMOTE | + (pin ? FOLL_PIN : FOLL_GET) | (dump ? FOLL_DUMP : 0); + unsigned long end, count = 0; + int locked = 0; + long ret; + + if (!length) + return 0; + if ((flags & FOLL_LONGTERM) && !pin) + return -EINVAL; + if (!fragments || !capacity || capacity > LONG_MAX || (flags & ~allowed)) + return -EINVAL; + if (vma_out) { + mmap_assert_locked(mm); + *vma_out = NULL; + } else { + internal |= FOLL_UNLOCKABLE; + ret = mmap_read_lock_killable(mm); + if (ret) + return ret; + } + locked = 1; + start = untagged_addr_remote(mm, start); + if (check_add_overflow(start, length, &end)) { + ret = -EFAULT; + goto out; + } + if (pin) + mm_set_has_pinned_flag(mm); + while (start < end && count < capacity) { + unsigned long bytes = min(end - start, + mm_user_fragment_size(mm) - (start & (mm_user_fragment_size(mm) - 1))); + struct vm_area_struct *vma; + + if (!locked) { + ret = mmap_read_lock_killable(mm); + if (ret) + break; + locked = 1; + } + if (gup_signal_pending(internal)) { + ret = -EINTR; + break; + } + cond_resched(); + if (mm_page_size(mm) == PAGE_SIZE) { + struct page *page; + + if (dump) + ret = __get_user_pages_locked(mm, start, 1, &page, &locked, internal); + else if (pin) + ret = pin_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + else + ret = get_user_pages_remote(mm, start, 1, flags, &page, + vma_out ? NULL : &locked); + if (ret != 1) { + if (!ret) + ret = -EAGAIN; + break; + } + fragments[count].folio = page_folio(page); + fragments[count].subpage = NULL; + fragments[count].pinned = pin; + fragments[count].offset = folio_page_idx(page_folio(page), page) * PAGE_SIZE + + offset_in_page(start); + } else { + vma = gup_vma_lookup(mm, start); + ret = vma ? check_vma_flags(vma, internal) : -EFAULT; + if (ret) + break; + /* Huge and special/device mappings need separate integration. */ + if (is_vm_hugetlb_page(vma) || + (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP | VM_IO))) { + ret = -EOPNOTSUPP; + break; + } + ret = follow_subpage_fragment(vma, start, internal, &fragments[count]); + if (!ret || ret == -EMLINK) { + ret = faultin_page(vma, start, internal, ret == -EMLINK, &locked); + if (!ret) + continue; + if (!locked && (ret == -EBUSY || ret == -EAGAIN)) { + internal |= FOLL_TRIED; + continue; + } + if (ret == -EBUSY) + ret = -EAGAIN; + break; + } + if (ret < 0) + break; + } + if (vma_out) + *vma_out = vma_lookup(mm, start); + fragments[count++].length = bytes; + start += bytes; + internal &= ~FOLL_TRIED; + } +out: + if (locked && !vma_out) + mmap_read_unlock(mm); + return count ? (long)count : ret; +} + +long pin_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + unsigned int saved; + long ret, count; + + if (!(flags & FOLL_LONGTERM) || mm_page_size(mm) == PAGE_SIZE) + return user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + /* Match native long-term GUP: fault outside movable zones, migrate any + * existing unpinnable backing, then retry with fresh typed references. + */ + saved = memalloc_pin_save(); + do { + count = user_fragments_remote(mm, start, length, flags, fragments, + capacity, true, NULL, false); + if (count <= 0) { + ret = count; + break; + } + ret = check_and_migrate_movable_fragments(count, fragments); + } while (ret == -EAGAIN); + memalloc_pin_restore(saved); + return ret ? ret : count; +} +EXPORT_SYMBOL_GPL(pin_user_fragments_remote); + +long get_user_fragments_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragments, unsigned long capacity) +{ + return user_fragments_remote(mm, start, length, flags, fragments, capacity, + false, NULL, false); +} +EXPORT_SYMBOL_GPL(get_user_fragments_remote); + +int get_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + /* As with get_user_page_vma_remote(), preserve the caller's VMA lock. */ + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & FOLL_NOWAIT) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, false, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +int pin_user_fragment_vma_remote(struct mm_struct *mm, unsigned long start, + unsigned long length, unsigned int flags, + struct user_page_fragment *fragment, struct vm_area_struct **vma) +{ + int ret; + + if (!vma) + return -EINVAL; + *vma = NULL; + if (flags & (FOLL_NOWAIT | FOLL_LONGTERM)) + return -EINVAL; + ret = user_fragments_remote(mm, start, length, flags, fragment, 1, true, vma, false); + if (ret > 0 && !*vma) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + return ret; +} + +#ifdef CONFIG_ELF_CORE +/* Like get_dump_page(), but preserves the selected user-page ownership. */ +int get_dump_fragment(unsigned long addr, struct user_page_fragment *fragment) +{ + return user_fragments_remote(current->mm, addr, mm_user_fragment_size(current->mm), + FOLL_FORCE, fragment, 1, false, NULL, true); +} +#endif + +void mark_user_fragment_dirty(struct user_page_fragment *fragment) +{ + struct folio *folio = fragment->folio; + + if (!folio) + return; + if (!is_zero_folio(folio) && !folio_test_dirty(folio)) { + folio_lock(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + } + if (fragment->subpage) + mm_subpage_cancel_lazyfree(fragment->subpage); +} +EXPORT_SYMBOL_GPL(mark_user_fragment_dirty); + +void release_user_fragments(struct user_page_fragment *fragments, + unsigned long count, bool dirty) +{ + unsigned long i; + + for (i = 0; i < count; i++) { + struct user_page_fragment *fragment = &fragments[i]; + struct folio *folio = fragment->folio; + + if (!folio) + continue; + if (dirty) + mark_user_fragment_dirty(fragment); + if (fragment->subpage) { + if (fragment->pinned) + mm_subpage_unpin(fragment->subpage); + else + mm_subpage_put(fragment->subpage); + } else if (fragment->pinned) { + unpin_folio(folio); + } else { + folio_put(folio); + } + *fragment = (struct user_page_fragment) {}; + } +} +EXPORT_SYMBOL_GPL(release_user_fragments); +#endif + /** * pin_user_pages() - pin user pages in memory for use by other devices * @@ -3407,6 +3946,25 @@ long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages, } EXPORT_SYMBOL(pin_user_pages_unlocked); +/* Direct cache pins have no PTE fault to publish fine shmem exposure. */ +static int memfd_pin_folio(struct file *memfd, struct folio *folio) +{ + bool lock = IS_ENABLED(CONFIG_MM_SUBPAGE) && + IS_ENABLED(CONFIG_USERFAULTFD) && shmem_file(memfd); + int ret; + + /* Serialize both acquisition and exposure against partial hole punch. */ + if (lock) + folio_lock(folio); + ret = try_grab_folio(folio, 1, FOLL_PIN); + if (lock) { + if (!ret) + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); + folio_unlock(folio); + } + return ret; +} + /** * memfd_pin_folios() - pin folios associated with a memfd * @memfd: the memfd whose folios are to be pinned @@ -3492,7 +4050,7 @@ long memfd_pin_folios(struct file *memfd, loff_t start, loff_t end, for (i = 0; i < nr_found; i++) { folio = fbatch.folios[i]; - if (try_grab_folio(folio, 1, FOLL_PIN)) { + if (memfd_pin_folio(memfd, folio)) { folio_batch_release(&fbatch); ret = -EINVAL; goto err; diff --git a/mm/huge_memory.c b/mm/huge_memory.c index dd93f1b0c44a20..7ab369477fcc74 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2715,7 +2715,8 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto unlock; if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, - vma_is_single_threaded_private(vma))) + vma_is_single_threaded_private(vma), + folio_maybe_mapped_shared(pmd_folio(*pmd)))) goto unlock; } /* @@ -4118,6 +4119,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order, } anon_vma_lock_write(anon_vma); mapping = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (new_order < anon_vma->min_folio_order) { + ret = -EBUSY; + goto out_unlock; + } +#endif } else { unsigned int min_order; gfp_t gfp; @@ -4394,7 +4401,7 @@ int folio_split(struct folio *folio, unsigned int new_order, unsigned int min_order_for_split(struct folio *folio) { if (folio_test_anon(folio)) - return 0; + return folio_anon_min_order(folio); /* * If the folio got truncated, we don't know the previous mapping and diff --git a/mm/internal.h b/mm/internal.h index 5a2ddcf68e0b6d..5ad4e57daedc6a 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -305,6 +305,50 @@ static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) return pte_mkold(pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Batch contiguous user leaves without confusing their offsets with PFNs. */ +static inline unsigned int folio_subpage_pte_batch_flags(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *ptentp, unsigned int max_nr, fpb_t flags) +{ + struct mm_struct *mm = vma->vm_mm; + pte_t pte = *ptentp; + phys_addr_t offset = pte_phys_mm(mm, pte) - PFN_PHYS(folio_pfn(folio)); + pte_t expected; + unsigned int nr; + + VM_BUG_ON_FOLIO(offset >= folio_size(folio), folio); + max_nr = min_t(unsigned long, max_nr, (folio_size(folio) - offset) >> mm_page_shift(mm)); + VM_BUG_ON_FOLIO(!max_nr, folio); + nr = min(max_nr, pte_batch_hint(ptep, pte)); + expected = pte_advance_pfn_mm(mm, __pte_batch_clear_ignored(pte, flags), nr); + while (nr < max_nr) { + unsigned int batch; + + pte = ptep_get(ptep + nr); + if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected)) + break; + if ((flags & FPB_MERGE_WRITE) && pte_write(pte)) + *ptentp = pte_mkwrite(*ptentp, vma); + if (flags & FPB_MERGE_YOUNG_DIRTY) { + if (pte_young(pte)) + *ptentp = pte_mkyoung(*ptentp); + if (pte_dirty(pte)) + *ptentp = pte_mkdirty(*ptentp); + } + batch = min(max_nr - nr, pte_batch_hint(ptep + nr, pte)); + nr += batch; + expected = pte_advance_pfn_mm(mm, expected, batch); + } + return nr; +} + +static inline unsigned int folio_subpage_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, unsigned int max_nr, fpb_t flags) +{ + return folio_subpage_pte_batch_flags(vma, folio, ptep, &pte, max_nr, flags); +} +#endif + /** * folio_pte_batch_flags - detect a PTE batch for a large folio * @folio: The large folio to detect a PTE batch for. @@ -416,6 +460,10 @@ static inline pte_t pte_move_swp_offset(pte_t pte, long delta) if (pte_swp_uffd_wp(pte)) new = pte_swp_mkuffd_wp(new); +#ifdef CONFIG_MM_SUBPAGE + new = pte_swp_set_subpage_offset(new, pte_swp_subpage_offset(pte)); +#endif + return new; } @@ -1158,6 +1206,11 @@ folio_within_range(struct folio *folio, struct vm_area_struct *vma, end = vma->vm_end; pgoff = folio_pgoff(folio); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + addr = vma_address_at_offset(vma, (struct vm_page_offset) { .index = pgoff }); + return !IS_ERR_VALUE(addr) && addr >= start && addr <= end && + end - addr >= folio_size(folio); + } /* if folio start address is not in vma range */ if (!in_range(pgoff, vma->vm_pgoff, vma_pglen)) @@ -1240,6 +1293,8 @@ static inline unsigned long vma_address(const struct vm_area_struct *vma, if (pgoff >= vma->vm_pgoff) { address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + if (pgoff != vma->vm_pgoff) + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address >= vma->vm_end) address = -EFAULT; @@ -1262,15 +1317,24 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) pgoff_t pgoff; unsigned long address; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) + return min(pvmw->address + mm_page_size(vma->vm_mm), vma->vm_end); +#endif + /* Common case, plus ->pgoff is invalid for KSM */ - if (pvmw->nr_pages == 1) - return pvmw->address + PAGE_SIZE; + if (pvmw->nr_pages == 1 && mm_page_size(vma->vm_mm) >= PAGE_SIZE) + return min((pvmw->address & mm_page_mask(vma->vm_mm)) + + mm_page_size(vma->vm_mm), vma->vm_end); pgoff = pvmw->pgoff + pvmw->nr_pages; address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); + address -= vma_subpage_offset(vma); /* Check for address beyond vma (or wrapped through 0?) */ if (address < vma->vm_start || address > vma->vm_end) address = vma->vm_end; + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + address = min(mm_page_align(vma->vm_mm, address), vma->vm_end); return address; } @@ -1568,11 +1632,11 @@ static inline bool vma_is_single_threaded_private(struct vm_area_struct *vma) #ifdef CONFIG_NUMA_BALANCING bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded); + bool is_private_single_threaded, bool shared); #else static inline bool folio_can_map_prot_numa(struct folio *folio, - struct vm_area_struct *vma, bool is_private_single_threaded) + struct vm_area_struct *vma, bool is_private_single_threaded, bool shared) { return false; } @@ -1900,7 +1964,7 @@ static inline bool clear_flush_young_ptes_notify(struct vm_area_struct *vma, young = clear_flush_young_ptes(vma, addr, ptep, nr); young |= mmu_notifier_clear_flush_young(vma->vm_mm, addr, - addr + nr * PAGE_SIZE); + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1920,7 +1984,8 @@ static inline bool test_and_clear_young_ptes_notify(struct vm_area_struct *vma, bool young; young = test_and_clear_young_ptes(vma, addr, ptep, nr); - young |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + nr * PAGE_SIZE); + young |= mmu_notifier_clear_young(vma->vm_mm, addr, + addr + nr * mm_page_size(vma->vm_mm)); return young; } @@ -1952,4 +2017,14 @@ static inline int get_sysctl_max_map_count(void) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages); +#ifdef CONFIG_MM_SUBPAGE +struct iov_iter; +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write); +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf); +#endif + #endif /* __MM_INTERNAL_H */ diff --git a/mm/khugepaged.c b/mm/khugepaged.c index d6e04041f5dc11..ad4832f64fc306 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1066,28 +1066,31 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm, } static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm, - struct collapse_control *cc) + struct collapse_control *cc, unsigned int order) { gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() : GFP_TRANSHUGE); int node = collapse_find_target_node(cc); struct folio *folio; - folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask); + folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask); if (!folio) { *foliop = NULL; - count_vm_event(THP_COLLAPSE_ALLOC_FAILED); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC_FAILED); return SCAN_ALLOC_HUGE_PAGE_FAIL; } - count_vm_event(THP_COLLAPSE_ALLOC); + if (is_pmd_order(order)) + count_vm_event(THP_COLLAPSE_ALLOC); if (unlikely(mem_cgroup_charge(folio, mm, gfp))) { folio_put(folio); *foliop = NULL; return SCAN_CGROUP_CHARGE_FAIL; } - count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); + if (is_pmd_order(order)) + count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1); *foliop = folio; return SCAN_SUCCEED; @@ -1116,7 +1119,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a */ mmap_read_unlock(mm); - result = alloc_charge_folio(&folio, mm, cc); + result = alloc_charge_folio(&folio, mm, cc, HPAGE_PMD_ORDER); if (result != SCAN_SUCCEED) goto out_nolock; @@ -1880,22 +1883,33 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) * + unlock and free huge page; */ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, - struct file *file, pgoff_t start, struct collapse_control *cc) + struct file *file, pgoff_t start, struct collapse_control *cc, + unsigned int order) { struct address_space *mapping = file->f_mapping; struct page *dst; struct folio *folio, *tmp, *new_folio; - pgoff_t index = 0, end = start + HPAGE_PMD_NR; + unsigned long nr_pages = 1UL << order; + pgoff_t index = 0, end = start + nr_pages; LIST_HEAD(pagelist); - XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); + XA_STATE_ORDER(xas, &mapping->i_pages, start, order); enum scan_result result = SCAN_SUCCEED; int nr_none = 0; bool is_shmem = shmem_file(file); VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem); - VM_BUG_ON(start & (HPAGE_PMD_NR - 1)); + VM_BUG_ON(start & (nr_pages - 1)); - result = alloc_charge_folio(&new_folio, mm, cc); + if (is_pmd_order(order)) { + result = alloc_charge_folio(&new_folio, mm, cc, order); + } else { + /* Sub-PMD user leaves must respect the shmem inode's allocation policy. */ + new_folio = shmem_alloc_collapse_folio(mm, mapping->host, start, order); + if (IS_ERR(new_folio)) { + new_folio = NULL; + result = SCAN_ALLOC_HUGE_PAGE_FAIL; + } + } if (result != SCAN_SUCCEED) goto out; @@ -2024,7 +2038,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * we locked the first folio, then a THP might be there already. * This will be discovered on the first iteration. */ - if (is_pmd_order(folio_order(folio))) { + if (folio_order(folio) >= order) { result = SCAN_PTE_MAPPED_HUGEPAGE; goto out_unlock; } @@ -2149,6 +2163,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, while (index < folio->index) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2158,12 +2174,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, result = SCAN_COPY_MC; goto rollback; } + if (is_shmem) + shmem_uffd_copy_page(dst, folio_page(folio, i)); index++; dst++; } } while (index < end) { clear_highpage(dst); + if (is_shmem) + shmem_uffd_copy_page(dst, NULL); index++; dst++; } @@ -2205,8 +2225,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * not be able to observe any missing pages due to the * previously inserted retry entries. */ + /* + * A required user leaf populates its whole granule, as a native + * fault does. Only optional PMD collapse must preserve cache holes + * merely because another VMA has missing-page registration. + */ vma_interval_tree_foreach(vma, &mapping->i_mmap, start, end) { - if (userfaultfd_missing(vma)) { + if (is_pmd_order(order) && userfaultfd_missing(vma)) { result = SCAN_EXCEED_NONE_PTE; goto immap_locked; } @@ -2229,12 +2254,13 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, } if (is_shmem) { - lruvec_stat_mod_folio(new_folio, NR_SHMEM, HPAGE_PMD_NR); - lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, HPAGE_PMD_NR); - } else { - lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_SHMEM, nr_pages); + if (is_pmd_order(order)) + lruvec_stat_mod_folio(new_folio, NR_SHMEM_THPS, nr_pages); + } else if (is_pmd_order(order)) { + lruvec_stat_mod_folio(new_folio, NR_FILE_THPS, nr_pages); } - lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, HPAGE_PMD_NR); + lruvec_stat_mod_folio(new_folio, NR_FILE_PAGES, nr_pages); /* * Mark new_folio as uptodate before inserting it into the @@ -2242,14 +2268,14 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * unwritten page. */ folio_mark_uptodate(new_folio); - folio_ref_add(new_folio, HPAGE_PMD_NR - 1); + folio_ref_add(new_folio, nr_pages - 1); if (is_shmem) folio_mark_dirty(new_folio); folio_add_lru(new_folio); /* Join all the small entries into a single multi-index entry. */ - xas_set_order(&xas, start, HPAGE_PMD_ORDER); + xas_set_order(&xas, start, order); xas_store(&xas, new_folio); WARN_ON_ONCE(xas_error(&xas)); xas_unlock_irq(&xas); @@ -2258,9 +2284,11 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, * Remove pte page tables, so we can re-fault the page as huge. * If MADV_COLLAPSE, adjust result to call try_collapse_pte_mapped_thp(). */ - retract_page_tables(mapping, start); - if (cc && !cc->is_khugepaged) - result = SCAN_PTE_MAPPED_HUGEPAGE; + if (is_pmd_order(order)) { + retract_page_tables(mapping, start); + if (cc && !cc->is_khugepaged) + result = SCAN_PTE_MAPPED_HUGEPAGE; + } folio_unlock(new_folio); /* @@ -2317,7 +2345,7 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr, folio_put(new_folio); out: VM_BUG_ON(!list_empty(&pagelist)); - trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, HPAGE_PMD_NR, result); + trace_mm_khugepaged_collapse_file(mm, new_folio, index, addr, is_shmem, file, nr_pages, result); return result; } @@ -2420,7 +2448,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm, result = SCAN_EXCEED_NONE_PTE; count_vm_event(THP_SCAN_EXCEED_NONE_PTE); } else { - result = collapse_file(mm, addr, file, start, cc); + result = collapse_file(mm, addr, file, start, cc, HPAGE_PMD_ORDER); } } @@ -2831,6 +2859,47 @@ static int madvise_collapse_errno(enum scan_result r) } } +/* + * Promote the cache backing for one larger-than-native user leaf. The caller + * owns mm/file references; no folio or page-table lock may be held. Start is + * a native file index. This sub-PMD shmem path may run under the fault lock: + * it neither retracts page tables (which requires mmap write locking) nor + * invokes filesystem writeback. Source folios are trylocked and rolled back + * on contention. Ordinary PMD collapse retains its existing lock policy. + */ +int collapse_file_user_page(struct mm_struct *mm, struct file *file, pgoff_t start) +{ + struct collapse_control *cc; + unsigned int order; + enum scan_result result; + + if (mm_page_shift(mm) <= PAGE_SHIFT || !shmem_file(file)) + return -EOPNOTSUPP; + order = mm_page_shift(mm) - PAGE_SHIFT; + /* Shmem's order limit reflects THP policy, not an architectural leaf limit. */ + if (order >= HPAGE_PMD_ORDER || order > MAX_PAGECACHE_ORDER || + !IS_ALIGNED(start, 1UL << order) || start > ULONG_MAX - (1UL << order)) + return -EINVAL; + cc = kzalloc(sizeof(*cc), GFP_KERNEL); + if (!cc) + return -ENOMEM; + lru_add_drain_all(); + result = collapse_file(mm, 0, file, start, cc, order); + kfree(cc); + if (result == SCAN_SUCCEED || result == SCAN_PTE_MAPPED_HUGEPAGE) + return 0; + /* Retained references cannot be displaced by replacing their cache folio. */ + if (result == SCAN_PAGE_COUNT) + return -EBUSY; + if (result == SCAN_CGROUP_CHARGE_FAIL) + return -ENOMEM; + if (result == SCAN_COPY_MC) + return -EHWPOISON; + if (result == SCAN_TRUNCATED) + return -EAGAIN; + return madvise_collapse_errno(result); +} + int madvise_collapse(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool *lock_dropped) { diff --git a/mm/ksm.c b/mm/ksm.c index 7d5b76478f0b65..cd0398e9729410 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -804,6 +804,10 @@ static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item) struct folio_walk fw; struct folio *folio; + /* The KSM comparison and replacement paths still require native leaves. */ + if (mm_page_size(mm) != PAGE_SIZE) + return NULL; + mmap_read_lock(mm); vma = find_mergeable_vma(mm, addr); if (!vma) @@ -1338,7 +1342,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct folio *folio, /* See folio_try_share_anon_rmap_pte(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, &folio->page)) { + folio_try_share_anon_rmap_pte(folio, &folio->page, vma)) { set_pte_at(mm, pvmw.address, pvmw.pte, entry); goto out_unlock; } @@ -3206,7 +3210,8 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc) if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, addr, rwc->arg)) { + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = addr }, rwc->arg)) { anon_vma_unlock_read(anon_vma); return; } diff --git a/mm/madvise.c b/mm/madvise.c index 5665785e5ac383..1d15da9d1200d2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -193,7 +194,7 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start, spinlock_t *ptl; unsigned long addr; - for (addr = start; addr < end; addr += PAGE_SIZE) { + for (addr = start; addr < end; addr += mm_page_size(vma->vm_mm)) { pte_t pte; softleaf_t entry; struct folio *folio; @@ -236,7 +237,7 @@ static void shmem_swapin_range(struct vm_area_struct *vma, struct address_space *mapping) { XA_STATE(xas, &mapping->i_pages, linear_page_index(vma, start)); - pgoff_t end_index = linear_page_index(vma, end) - 1; + pgoff_t end_index = linear_page_index(vma, end - 1); struct folio *folio; struct swap_iocb *splug = NULL; @@ -252,8 +253,11 @@ static void shmem_swapin_range(struct vm_area_struct *vma, if (!softleaf_is_swap(entry)) continue; - addr = vma->vm_start + - ((xas.xa_index - vma->vm_pgoff) << PAGE_SHIFT); + /* The first cache page may start before this VMA's byte offset. */ + addr = xas.xa_index == linear_page_index(vma, start) ? start : + vma_address_at_offset(vma, (struct vm_page_offset) { + .index = xas.xa_index, + }); xas_pause(&xas); rcu_read_unlock(); @@ -318,7 +322,7 @@ static long madvise_willneed(struct madvise_behavior *madv_behavior) mark_mmap_lock_dropped(madv_behavior); get_file(file); offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); mmap_read_unlock(mm); vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED); fput(file); @@ -340,12 +344,18 @@ static inline bool can_do_file_pageout(struct vm_area_struct *vma) file_permission(vma->vm_file, MAY_WRITE) == 0; } -static inline int madvise_folio_pte_batch(unsigned long addr, unsigned long end, +static inline int madvise_folio_pte_batch(struct vm_area_struct *vma, + unsigned long addr, unsigned long end, struct folio *folio, pte_t *ptep, pte_t *ptentp) { - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, ptentp, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif return folio_pte_batch_flags(folio, NULL, ptep, ptentp, max_nr, FPB_MERGE_YOUNG_DIRTY); } @@ -376,7 +386,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, #ifdef CONFIG_TRANSPARENT_HUGEPAGE if (pmd_trans_huge(*pmd)) { pmd_t orig_pmd; - unsigned long next = pmd_addr_end(addr, end); + unsigned long next = pmd_addr_end_mm(mm, addr, end); tlb_change_page_size(tlb, HPAGE_PMD_SIZE); ptl = pmd_trans_huge_lock(pmd, vma); @@ -446,14 +456,14 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, regular_folio: #endif - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); restart: start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!start_pte) return 0; flush_tlb_batched_pending(mm); lazy_mmu_mode_enable(); - for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) { + for (; addr < end; pte += nr, addr += nr * mm_page_size(mm)) { nr = 1; ptent = ptep_get(pte); @@ -484,9 +494,10 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * fail to split a folio, leave it in place and advance to the * next pte in the range. */ - if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); - if (nr < folio_nr_pages(folio)) { + if (folio_test_large(folio) || mm_page_size(mm) != PAGE_SIZE) { + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); + if (folio_test_large(folio) && + nr < (folio_size(folio) >> mm_page_shift(mm))) { int err; if (folio_maybe_mapped_shared(folio)) @@ -521,7 +532,8 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, * number of pages, it must be exclusive. */ if (!folio_test_lru(folio) || - folio_mapcount(folio) != folio_nr_pages(folio)) + folio_mapcount(folio) != + (mm_page_size(mm) != PAGE_SIZE ? nr : folio_nr_pages(folio))) continue; if (pageout_anon_only_filter && !folio_test_anon(folio)) @@ -649,6 +661,72 @@ static long madvise_pageout(struct madvise_behavior *madv_behavior) return 0; } +#ifdef CONFIG_MM_SUBPAGE +static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + struct mm_struct *mm = vma->vm_mm; + struct mmu_gather *tlb = walk->private; + spinlock_t *ptl; + pte_t *base, *ptep; + long swapped = 0; + + base = ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!base) + return 0; + flush_tlb_batched_pending(mm); + for (; addr < end; addr += mm_page_size(mm), ptep++) { + pte_t pte = ptep_get(ptep); + struct mm_subpage *slot; + struct folio *folio; + struct page *page; + + if (!pte_present(pte)) { + softleaf_t entry = softleaf_from_pte(pte); + + if (softleaf_is_swap(entry)) { + swap_subpage_put(entry, + pte_swp_subpage_offset(pte), true); + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); + swapped--; + } + continue; + } + page = vm_normal_page(vma, addr, pte); + if (!page) + continue; + folio = page_folio(page); + if (!folio_test_anon(folio) || !folio_trylock(folio)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (!slot) + goto unlock; + if (folio_test_swapcache(folio) && !folio_free_swap(folio)) + goto put; + if (!mm_subpage_mark_lazyfree(slot)) + goto put; + clear_young_dirty_ptes(vma, addr, ptep, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + tlb_remove_tlb_entry(tlb, ptep, addr); + /* Partial hints must never clear a neighbor's dirty state. */ + if (mm_subpage_all_lazyfree(slot)) { + folio_clear_dirty(folio); + folio_mark_lazyfree(folio); + } +put: + mm_subpage_put(slot); +unlock: + folio_unlock(folio); + } + if (swapped) + add_mm_counter(mm, MM_SWAPENTS, swapped); + pte_unmap_unlock(base, ptl); + cond_resched(); + return 0; +} +#endif + static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) @@ -664,6 +742,10 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, unsigned long next; int nr, max_nr; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) + return madvise_free_subpage_range(pmd, addr, end, walk); +#endif next = pmd_addr_end(addr, end); if (pmd_trans_huge(*pmd)) if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) @@ -714,7 +796,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, * next pte in the range. */ if (folio_test_large(folio)) { - nr = madvise_folio_pte_batch(addr, end, folio, pte, &ptent); + nr = madvise_folio_pte_batch(vma, addr, end, folio, pte, &ptent); if (nr < folio_nr_pages(folio)) { int err; @@ -988,7 +1070,7 @@ static long madvise_populate(struct madvise_behavior *madv_behavior) return -ENOMEM; } } - start += pages * PAGE_SIZE; + start += pages * mm_page_size(mm); } return 0; } @@ -1022,7 +1104,7 @@ static long madvise_remove(struct madvise_behavior *madv_behavior) return -EACCES; offset = (loff_t)(start - vma->vm_start) - + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); /* * Filesystem's fallocate may need to take i_rwsem. We need to @@ -1182,7 +1264,7 @@ static long madvise_guard_install(struct madvise_behavior *madv_behavior) if (err == 0) { unsigned long nr_expected_pages = - PHYS_PFN(range->end - range->start); + (range->end - range->start) >> mm_page_shift(vma->vm_mm); VM_WARN_ON(nr_pages != nr_expected_pages); return 0; @@ -1833,16 +1915,17 @@ static void madvise_finish_tlb(struct madvise_behavior *madv_behavior) tlb_finish_mmu(madv_behavior->tlb); } -static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) +static bool is_valid_madvise(struct mm_struct *mm, unsigned long start, + size_t len_in, int behavior) { size_t len; if (!madvise_behavior_valid(behavior)) return false; - if (!PAGE_ALIGNED(start)) + if (!IS_ALIGNED(start, mm_page_size(mm))) return false; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) @@ -1865,14 +1948,29 @@ static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior) * operation. This function returns true in the cases, otherwise false. In * the former case we store an error on @err. */ -static bool madvise_should_skip(unsigned long start, size_t len_in, +static bool madvise_should_skip(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior, int *err) { - if (!is_valid_madvise(start, len_in, behavior)) { + if (!is_valid_madvise(mm, start, len_in, behavior)) { *err = -EINVAL; return true; } - if (start + PAGE_ALIGN(len_in) == start) { +#ifdef CONFIG_MM_SUBPAGE + /* These engines still assume whole native folios/tables. */ + if (mm_page_size(mm) != PAGE_SIZE) { + switch (behavior) { + case MADV_MERGEABLE: + case MADV_UNMERGEABLE: + case MADV_HUGEPAGE: + case MADV_COLLAPSE: + case MADV_HWPOISON: + case MADV_SOFT_OFFLINE: + *err = -EOPNOTSUPP; + return true; + } + } +#endif + if (start + mm_page_align(mm, len_in) == start) { *err = 0; return true; } @@ -1919,7 +2017,7 @@ static int madvise_do_behavior(unsigned long start, size_t len_in, } range->start = get_untagged_addr(madv_behavior->mm, start); - range->end = range->start + PAGE_ALIGN(len_in); + range->end = range->start + mm_page_align(madv_behavior->mm, len_in); blk_start_plug(&plug); if (is_madvise_populate(madv_behavior)) @@ -2012,7 +2110,7 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh .tlb = &tlb, }; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) return error; error = madvise_lock(&madv_behavior); if (error) @@ -2055,7 +2153,7 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter, size_t len_in = iter_iov_len(iter); int error; - if (madvise_should_skip(start, len_in, behavior, &error)) + if (madvise_should_skip(mm, start, len_in, behavior, &error)) ret = error; else ret = madvise_do_behavior(start, len_in, &madv_behavior); @@ -2185,9 +2283,9 @@ static int madvise_set_anon_name(struct mm_struct *mm, unsigned long start, .anon_name = anon_name, }; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = (len_in + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero */ if (len_in && !len) diff --git a/mm/memory-failure.c b/mm/memory-failure.c index d47aef256a3249..964c002533131a 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -67,6 +67,9 @@ #include "swap.h" #include "internal.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif static int sysctl_memory_failure_early_kill __read_mostly; @@ -539,6 +542,37 @@ struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) return find_early_kill_thread(tsk); } +#ifdef CONFIG_MM_SUBPAGE +struct subpage_collect_procs { + const struct page *page; + struct list_head *to_kill; + int force_early; +}; + +static bool collect_procs_subpage(struct folio *folio, struct vm_area_struct *vma, + struct rmap_walk_range range, void *arg) +{ + struct subpage_collect_procs *args = arg; + DEFINE_FOLIO_RMAP_WALK(walk, folio, vma, range, PVMW_SYNC); + struct task_struct *tsk; + unsigned long address; + + if (!page_vma_mapped_walk(&walk)) + return true; + address = walk.address; + page_vma_mapped_walk_done(&walk); + rcu_read_lock(); + for_each_process(tsk) { + struct task_struct *t = task_early_kill(tsk, args->force_early); + + if (t && t->mm == vma->vm_mm) + add_to_kill_anon_file(t, args->page, vma, args->to_kill, address); + } + rcu_read_unlock(); + return true; +} +#endif + /* * Collect processes when the error hit an anonymous page. */ @@ -549,6 +583,17 @@ static void collect_procs_anon(const struct folio *folio, struct task_struct *tsk; struct anon_vma *av; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + struct subpage_collect_procs args = { page, to_kill, force_early }; + struct rmap_walk_control rwc = { + .rmap_one = collect_procs_subpage, + .arg = &args, + }; + + if (!folio_test_large(folio) && + rmap_walk_subpages((struct folio *)folio, &rwc, false)) + return; +#endif av = folio_lock_anon_vma_read(folio, NULL); if (av == NULL) /* Not actually mapped anymore */ diff --git a/mm/memory.c b/mm/memory.c index a903d5c1859cb6..ade3dbadb409a4 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -87,6 +87,8 @@ #include #include "pgalloc-track.h" +#include + #include "internal.h" #include "swap.h" @@ -191,26 +193,26 @@ static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud, unsigned long start; start = addr; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_none_or_clear_bad(pmd)) continue; free_pte_range(tlb, pmd, addr); } while (pmd++, addr = next, addr != end); - start &= PUD_MASK; + start &= pud_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PUD_MASK; + ceiling &= pud_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pmd = pmd_offset(pud, start); + pmd = pmd_offset_mm(tlb->mm, pud, start); pud_clear(pud); pmd_free_tlb(tlb, pmd, start); mm_dec_nr_pmds(tlb->mm); @@ -225,27 +227,27 @@ static inline void free_pud_range(struct mmu_gather *tlb, p4d_t *p4d, unsigned long start; start = addr; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; free_pmd_range(tlb, pud, addr, next, floor, ceiling); } while (pud++, addr = next, addr != end); - start &= P4D_MASK; + start &= p4d_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= P4D_MASK; + ceiling &= p4d_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - pud = pud_offset(p4d, start); - p4d_clear(p4d); + pud = pud_offset_mm(tlb->mm, p4d, start); + p4d_clear_mm(tlb->mm, p4d); pud_free_tlb(tlb, pud, start); mm_dec_nr_puds(tlb->mm); } @@ -259,26 +261,26 @@ static inline void free_p4d_range(struct mmu_gather *tlb, pgd_t *pgd, unsigned long start; start = addr; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; free_pud_range(tlb, p4d, addr, next, floor, ceiling); } while (p4d++, addr = next, addr != end); - start &= PGDIR_MASK; + start &= pgd_mask_mm(tlb->mm); if (start < floor) return; if (ceiling) { - ceiling &= PGDIR_MASK; + ceiling &= pgd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) return; - p4d = p4d_offset(pgd, start); + p4d = p4d_offset_mm(tlb->mm, pgd, start); pgd_clear(pgd); p4d_free_tlb(tlb, p4d, start); } @@ -320,7 +322,7 @@ void free_pgd_range(struct mmu_gather *tlb, * Wherever addr is brought up or ceiling brought down, we must * be careful to reject "the opposite 0" before it confuses the * subsequent tests. But what about where end is brought down - * by PMD_SIZE below? no, end can't go down to 0 there. + * by one PMD span below? no, end can't go down to 0 there. * * Whereas we round start (addr) and ceiling down, by different * masks at different levels, in order to test whether a table @@ -328,19 +330,19 @@ void free_pgd_range(struct mmu_gather *tlb, * bother to round floor or end up - the tests don't need that. */ - addr &= PMD_MASK; + addr &= pmd_mask_mm(tlb->mm); if (addr < floor) { - addr += PMD_SIZE; + addr += pmd_size_mm(tlb->mm); if (!addr) return; } if (ceiling) { - ceiling &= PMD_MASK; + ceiling &= pmd_mask_mm(tlb->mm); if (!ceiling) return; } if (end - 1 > ceiling - 1) - end -= PMD_SIZE; + end -= pmd_size_mm(tlb->mm); if (addr > end - 1) return; /* @@ -350,7 +352,7 @@ void free_pgd_range(struct mmu_gather *tlb, tlb_change_page_size(tlb, PAGE_SIZE); pgd = pgd_offset(tlb->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; free_p4d_range(tlb, pgd, addr, next, floor, ceiling); @@ -539,16 +541,16 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = p4dp_get(p4dp); p4dv = p4d_val(p4d); - if (!p4d_present(p4d) || p4d_leaf(p4d)) { + if (!p4d_present_mm(mm, p4d) || p4d_leaf(p4d)) { pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv); return; } - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = pudp_get(pudp); pudv = pud_val(pud); @@ -557,7 +559,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add return; } - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = pmdp_get(pmdp); pmdv = pmd_val(pmd); @@ -950,7 +952,15 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, struct page *page; if (likely(softleaf_is_swap(entry))) { - if (swap_dup_entry_direct(entry) < 0) + int err; + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + err = swap_subpage_dup(entry, pte_swp_subpage_offset(orig_pte)); + else +#endif + err = swap_dup_entries_direct(entry, mm_pte_native_pages(src_mm)); + if (err < 0) return -EIO; /* make sure dst_mm is on swapoff's mmlist. */ @@ -982,6 +992,10 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, entry = make_readable_migration_entry( swp_offset(entry)); pte = softleaf_to_pte(entry); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) + pte = pte_swp_set_subpage_offset(pte, pte_swp_subpage_offset(orig_pte)); +#endif if (pte_swp_soft_dirty(orig_pte)) pte = pte_swp_mksoft_dirty(pte); if (pte_swp_uffd_wp(orig_pte)) @@ -1076,10 +1090,14 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma * over and copy the page & arm it. */ - if (copy_mc_user_highpage(&new_folio->page, page, addr, src_vma)) - return -EHWPOISON; + for (unsigned int i = 0; i < mm_pte_native_pages(src_vma->vm_mm); i++) { + if (copy_mc_user_highpage(folio_page(new_folio, i), page + i, + addr + i * PAGE_SIZE, src_vma)) + return -EHWPOISON; + } *prealloc = NULL; + folio_ref_add(new_folio, mm_pte_native_pages(dst_vma->vm_mm) - 1); __folio_mark_uptodate(new_folio); folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, dst_vma); @@ -1095,6 +1113,55 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return 0; } +#ifdef CONFIG_MM_SUBPAGE +/* One new native folio may publish several independently locked user leaves. */ +static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, + unsigned int mapped) +{ + folio_add_lru_vma(folio, vma); + if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + while (--mapped) + mlock_folio(folio); +} + +/* The destination slot and its charge/metadata were allocated outside PTLs. */ +static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, + pte_t source_pte, unsigned long addr, int *rss, + struct mm_subpage *source, struct mm_subpage **prealloc) +{ + struct mm_subpage *dest = *prealloc; + struct folio *new; + pte_t pte; + int ret; + + if (!dest) + return -EAGAIN; + new = mm_subpage_folio(dest); + ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), + &mm_subpage_folio(source)->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (ret) + return ret; + /* Fresh, unmapped backing is private to this preallocation. */ + if (WARN_ON_ONCE(!folio_trylock(new))) + return -EBUSY; + mm_subpage_set_exclusive(dest); + ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + if (!ret) { + subpage_add_new_lru(new, dst_vma, 1); + rss[MM_ANONPAGES]++; + pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); + pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); + if (userfaultfd_pte_wp(dst_vma, source_pte)) + pte = pte_mkuffd_wp(pte); + set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); + *prealloc = NULL; + } + folio_unlock(new); + return ret; +} +#endif + static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, int nr) @@ -1128,18 +1195,46 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma, static inline int copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, pte_t pte, unsigned long addr, - int max_nr, int *rss, struct folio **prealloc) + int max_nr, int *rss, struct folio **prealloc, + struct mm_subpage **subprealloc) { fpb_t flags = FPB_MERGE_WRITE; struct page *page; struct folio *folio; - int err, nr; + int err, nr, nr_native; + unsigned int span = mm_pte_native_pages(src_vma->vm_mm); page = vm_normal_page(src_vma, addr, pte); if (unlikely(!page)) goto copy_pte; folio = page_folio(page); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + struct mm_subpage *slot; + + slot = mm_subpage_get_from_phys(pte_phys_mm(src_vma->vm_mm, pte)); + if (WARN_ON_ONCE(!slot)) + return -EINVAL; + err = mm_subpage_dup_anon_rmap(slot, dst_vma, src_vma, addr); + if (err == -EBUSY) { + err = copy_pinned_subpage(dst_vma, dst_pte, pte, addr, + rss, slot, subprealloc); + mm_subpage_put(slot); + return err ? err : 1; + } + if (err) { + mm_subpage_put(slot); + return err; + } + /* The lookup reference becomes the child's PTE reference. */ + rss[MM_ANONPAGES]++; + goto copy_pte; + } + max_nr = 1; + } +#endif /* * If we likely have to copy, just don't bother with batching. Make @@ -1152,18 +1247,25 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma if (vma_soft_dirty_enabled(src_vma)) flags |= FPB_RESPECT_SOFT_DIRTY; - nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); - folio_ref_add(folio, nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_vma->vm_mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch_flags(src_vma, folio, src_pte, + &pte, max_nr, flags); + else +#endif + nr = folio_pte_batch_flags(folio, src_vma, src_pte, &pte, max_nr, flags); + nr_native = nr * span; + folio_ref_add(folio, nr_native); if (folio_test_anon(folio)) { if (unlikely(folio_try_dup_anon_rmap_ptes(folio, page, - nr, dst_vma, src_vma))) { - folio_ref_sub(folio, nr); + nr_native, dst_vma, src_vma))) { + folio_ref_sub(folio, nr_native); return -EAGAIN; } rss[MM_ANONPAGES] += nr; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { - folio_dup_file_rmap_ptes(folio, page, nr, dst_vma); + folio_dup_file_rmap_ptes(folio, page, nr_native, dst_vma); rss[mm_counter_file(folio)] += nr; } __copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, pte, @@ -1171,7 +1273,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma return nr; } - folio_get(folio); + folio_ref_add(folio, span); if (folio_test_anon(folio)) { /* * If this page may have been pinned by the parent process, @@ -1181,7 +1283,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma */ if (unlikely(folio_try_dup_anon_rmap_pte(folio, page, dst_vma, src_vma))) { /* Page may be pinned, we have to copy. */ - folio_put(folio); + folio_put_refs(folio, span); err = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, addr, rss, prealloc, page); return err ? err : 1; @@ -1189,6 +1291,7 @@ copy_present_ptes(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma rss[MM_ANONPAGES]++; VM_WARN_ON_FOLIO(PageAnonExclusive(page), folio); } else { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(src_vma->vm_mm, pte)); folio_dup_file_rmap_pte(folio, page, dst_vma); rss[mm_counter_file(folio)]++; } @@ -1202,11 +1305,12 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, struct vm_area_struct *vma, unsigned long addr, bool need_zero) { struct folio *new_folio; + unsigned int order = max_t(unsigned int, mm_page_shift(src_mm), PAGE_SHIFT) - PAGE_SHIFT; if (need_zero) - new_folio = vma_alloc_zeroed_movable_folio(vma, addr); + new_folio = vma_alloc_zeroed_movable_folio_order(vma, addr, order); else - new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr); + new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, order, vma, addr); if (!new_folio) return NULL; @@ -1220,6 +1324,34 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, return new_folio; } +#ifdef CONFIG_MM_SUBPAGE +static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + struct mm_subpage *slot = ERR_PTR(-ENOMEM); + struct folio *folio; + int ret; + + if (!pool) + return slot; + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) + goto out; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + slot = ERR_PTR(ret); + goto out; + } + slot = mm_subpage_alloc_at(pool, vma_page_offset_at(vma, addr).offset); +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} +#endif + static int copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, @@ -1236,6 +1368,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, int rss[NR_MM_COUNTERS]; softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; + struct mm_subpage *subprealloc = NULL; int nr; again: @@ -1316,16 +1449,23 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, WARN_ON_ONCE(ret != -ENOENT); } /* copy_present_ptes() will clear `*prealloc' if consumed */ - max_nr = (end - addr) / PAGE_SIZE; + max_nr = (end - addr) >> mm_page_shift(src_mm); ret = copy_present_ptes(dst_vma, src_vma, dst_pte, src_pte, - ptent, addr, max_nr, rss, &prealloc); + ptent, addr, max_nr, rss, &prealloc, &subprealloc); /* * If we need a pre-allocated page for this pte, drop the * locks, allocate, and try again. * If copy failed due to hwpoison in source page, break out. */ - if (unlikely(ret == -EAGAIN || ret == -EHWPOISON)) + if (unlikely(ret == -EAGAIN || ret == -EHWPOISON || + (IS_ENABLED(CONFIG_MM_SUBPAGE) && ret < 0))) break; +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) { + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif if (unlikely(prealloc)) { /* * pre-alloc page cannot be reused by next time so as @@ -1338,7 +1478,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } nr = ret; progress += 8 * nr; - } while (dst_pte += nr, src_pte += nr, addr += PAGE_SIZE * nr, + } while (dst_pte += nr, src_pte += nr, addr += mm_page_size(src_mm) * nr, addr != end); lazy_mmu_mode_disable(); @@ -1357,10 +1497,22 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EBUSY || unlikely(ret == -EHWPOISON)) { goto out; } else if (ret == -EAGAIN) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(src_mm) < PAGE_SIZE) { + subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + if (IS_ERR(subprealloc)) + return PTR_ERR(subprealloc); + ret = 0; + goto again; + } +#endif prealloc = folio_prealloc(src_mm, src_vma, addr, false); if (!prealloc) return -ENOMEM; } else if (ret < 0) { + /* Subpage ownership checks can return additional fatal errors. */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) + goto out; VM_WARN_ON_ONCE(1); } @@ -1370,6 +1522,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, if (addr != end) goto again; out: +#ifdef CONFIG_MM_SUBPAGE + if (subprealloc) + mm_subpage_put(subprealloc); +#endif if (unlikely(prealloc)) folio_put(prealloc); return ret; @@ -1388,9 +1544,9 @@ copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pmd = pmd_alloc(dst_mm, dst_pud, addr); if (!dst_pmd) return -ENOMEM; - src_pmd = pmd_offset(src_pud, addr); + src_pmd = pmd_offset_mm(src_vma->vm_mm, src_pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(src_vma->vm_mm, addr, end); if (pmd_is_huge(*src_pmd)) { int err; @@ -1425,9 +1581,9 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_pud = pud_alloc(dst_mm, dst_p4d, addr); if (!dst_pud) return -ENOMEM; - src_pud = pud_offset(src_p4d, addr); + src_pud = pud_offset_mm(src_vma->vm_mm, src_p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(src_vma->vm_mm, addr, end); if (pud_trans_huge(*src_pud)) { int err; @@ -1461,10 +1617,10 @@ copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, dst_p4d = p4d_alloc(dst_mm, dst_pgd, addr); if (!dst_p4d) return -ENOMEM; - src_p4d = p4d_offset(src_pgd, addr); + src_p4d = p4d_offset_mm(src_vma->vm_mm, src_pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(src_p4d)) + next = p4d_addr_end_mm(src_vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(src_vma->vm_mm, src_p4d)) continue; if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d, addr, next)) @@ -1516,6 +1672,9 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) bool is_cow; int ret; + if (mm_page_shift(dst_mm) != mm_page_shift(src_mm)) + return -EINVAL; + if (!vma_needs_copy(dst_vma, src_vma)) return 0; @@ -1549,7 +1708,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma) dst_pgd = pgd_offset(dst_mm, addr); src_pgd = pgd_offset(src_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(src_vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(src_pgd)) continue; if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd, @@ -1629,7 +1788,7 @@ zap_install_uffd_wp_if_needed(struct vm_area_struct *vma, if (--nr == 0) break; pte++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } return was_installed; @@ -1642,10 +1801,12 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, bool *force_flush, bool *force_break, bool *any_skipped) { struct mm_struct *mm = tlb->mm; + unsigned int nr_native = nr * mm_pte_native_pages(mm); bool delay_rmap = false; if (!folio_test_anon(folio)) { ptent = get_and_clear_full_ptes(mm, addr, pte, nr, tlb->fullmm); + mm_subpage_file_map_del(vma, pte_phys_mm(mm, ptent)); if (pte_dirty(ptent)) { folio_mark_dirty(folio); if (tlb_delay_rmap(tlb)) { @@ -1669,16 +1830,44 @@ static __always_inline void zap_present_folio_ptes(struct mmu_gather *tlb, nr, details, ptent); if (!delay_rmap) { - folio_remove_rmap_ptes(folio, page, nr, vma); + folio_remove_rmap_ptes(folio, page, nr_native, vma); if (unlikely(folio_mapcount(folio) < 0)) print_bad_pte(vma, addr, ptent, page); } - if (unlikely(__tlb_remove_folio_pages(tlb, page, nr, delay_rmap))) { + /* Gather payloads count native references, not logical PTEs. */ + if (unlikely(__tlb_remove_folio_pages(tlb, page, nr_native, delay_rmap))) { + *force_flush = true; + *force_break = true; + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void zap_present_subpage(struct mmu_gather *tlb, + struct vm_area_struct *vma, pte_t *pte, pte_t ptent, + unsigned long addr, int *rss, bool *force_flush, + bool *force_break, bool *any_skipped) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(tlb->mm, ptent)); + + /* All small-granule anonymous PTEs must carry slot ownership. */ + if (WARN_ON_ONCE(!slot)) { + *any_skipped = true; + return; + } + ptep_get_and_clear_full(tlb->mm, addr, pte, tlb->fullmm); + arch_check_zapped_pte(vma, ptent); + tlb_remove_tlb_entry(tlb, pte, addr); + mm_subpage_remove_anon_rmap(slot, vma); + rss[MM_ANONPAGES]--; + /* Drop the lookup ref, then transfer the PTE ref to the TLB batch. */ + mm_subpage_put(slot); + if (__tlb_remove_subpage(tlb, slot)) { *force_flush = true; *force_break = true; } } +#endif /* * Zap or skip at least one present PTE, trying to batch-process subsequent @@ -1716,12 +1905,30 @@ static inline int zap_present_ptes(struct mmu_gather *tlb, return 1; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + if (folio_test_anon(folio)) { + zap_present_subpage(tlb, vma, pte, ptent, addr, rss, + force_flush, force_break, any_skipped); + return 1; + } + /* Native folio batching still expresses physical PAGE_SIZE steps. */ + max_nr = 1; + } +#endif + /* * Make sure that the common "small folio" case is as fast as possible * by keeping the batching logic separate. */ if (unlikely(folio_test_large(folio) && max_nr != 1)) { - nr = folio_pte_batch(folio, pte, ptent, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) > PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, + max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pte, ptent, max_nr); zap_present_folio_ptes(tlb, vma, folio, page, pte, ptent, nr, addr, details, rss, force_flush, force_break, any_skipped); @@ -1764,9 +1971,20 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb, if (!should_zap_cows(details)) return 1; - nr = swap_pte_batch(pte, max_nr, ptent); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + swap_subpage_put(entry, pte_swp_subpage_offset(ptent), true); + } else +#endif + { + unsigned int span = mm_pte_native_pages(vma->vm_mm); + + /* Adjacent coarse PTEs advance by a full group of swap slots. */ + if (span == 1) + nr = swap_pte_batch(pte, max_nr, ptent); + swap_put_entries_direct(entry, nr * span); + } rss[MM_SWAPENTS] -= nr; - swap_put_entries_direct(entry, nr); } else if (softleaf_is_migration(entry)) { struct folio *folio = softleaf_to_folio(entry); @@ -1811,7 +2029,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, bool *any_skipped) { pte_t ptent = ptep_get(pte); - int max_nr = (end - addr) / PAGE_SIZE; + int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); int nr = 0; /* Skip all consecutive none ptes */ @@ -1825,7 +2043,7 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, if (!max_nr) return nr; pte += nr; - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); } if (pte_present(ptent)) @@ -1839,13 +2057,14 @@ static inline int do_zap_pte_range(struct mmu_gather *tlb, return nr; } -static bool pte_table_reclaim_possible(unsigned long start, unsigned long end, +static bool pte_table_reclaim_possible(struct mm_struct *mm, + unsigned long start, unsigned long end, struct zap_details *details) { if (!IS_ENABLED(CONFIG_PT_RECLAIM)) return false; /* Only zap if we are allowed to and cover the full page table. */ - return details && details->reclaim_pt && (end - start >= PMD_SIZE); + return details && details->reclaim_pt && (end - start >= pmd_size_mm(mm)); } static bool zap_empty_pte_table(struct mm_struct *mm, pmd_t *pmd, @@ -1877,7 +2096,7 @@ static bool zap_pte_table_if_empty(struct mm_struct *mm, pmd_t *pmd, if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); - for (i = 0, pte = start_pte; i < PTRS_PER_PTE; i++, pte++) { + for (i = 0, pte = start_pte; i < pte_table_bytes_mm(mm) / sizeof(pte_t); i++, pte++) { if (!pte_none(ptep_get(pte))) goto out_ptl; } @@ -1902,7 +2121,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, struct zap_details *details) { - bool can_reclaim_pt = pte_table_reclaim_possible(addr, end, details); + bool can_reclaim_pt = pte_table_reclaim_possible(tlb->mm, addr, end, details); bool force_flush = false, force_break = false; struct mm_struct *mm = tlb->mm; int rss[NR_MM_COUNTERS]; @@ -1915,7 +2134,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, int nr; retry: - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, mm_page_size(mm)); init_rss_vec(rss); start_pte = pte = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!pte) @@ -1936,11 +2155,11 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, if (any_skipped) can_reclaim_pt = false; if (unlikely(force_break)) { - addr += nr * PAGE_SIZE; + addr += nr * mm_page_size(vma->vm_mm); direct_reclaim = false; break; } - } while (pte += nr, addr += PAGE_SIZE * nr, addr != end); + } while (pte += nr, addr += mm_page_size(mm) * nr, addr != end); /* * Fast path: try to hold the pmd lock and unmap the PTE page. @@ -1997,9 +2216,9 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, pmd_t *pmd; unsigned long next; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(tlb->mm, pud, addr); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(tlb->mm, addr, end); if (pmd_is_huge(*pmd)) { if (next - addr != HPAGE_PMD_SIZE) __split_huge_pmd(vma, pmd, addr, false); @@ -2033,9 +2252,9 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, pud_t *pud; unsigned long next; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(tlb->mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(tlb->mm, addr, end); if (pud_trans_huge(*pud)) { if (next - addr != HPAGE_PUD_SIZE) split_huge_pud(vma, pud, addr); @@ -2061,10 +2280,10 @@ static inline unsigned long zap_p4d_range(struct mmu_gather *tlb, p4d_t *p4d; unsigned long next; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(tlb->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(tlb->mm, addr, end); + if (p4d_none_or_clear_bad_mm(tlb->mm, p4d)) continue; next = zap_pud_range(tlb, vma, p4d, addr, next, details); } while (p4d++, addr = next, addr != end); @@ -2103,7 +2322,7 @@ static void __zap_vma_range(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb_start_vma(tlb, vma); pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(tlb->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; next = zap_p4d_range(tlb, vma, pgd, addr, next, details); @@ -2350,7 +2569,7 @@ static int validate_page_before_insert(struct vm_area_struct *vma, static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, unsigned long addr, struct page *page, - pgprot_t prot, bool mkwrite) + pgprot_t prot, bool mkwrite, unsigned int offset) { struct folio *folio = page_folio(page); pte_t pteval = ptep_get(pte); @@ -2360,7 +2579,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, return -EBUSY; /* see insert_pfn(). */ - if (pte_pfn(pteval) != page_to_pfn(page)) { + if (pte_phys_mm(vma->vm_mm, pteval) != page_to_phys(page) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(pteval))); return -EFAULT; } @@ -2372,17 +2591,18 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } /* Ok, finally just insert the thing.. */ - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (unlikely(is_zero_folio(folio))) { pteval = pte_mkspecial(pteval); } else { folio_get(folio); - pteval = mk_pte(page, prot); + pteval = phys_pte_mm(vma->vm_mm, page_to_phys(page) + offset, prot); if (mkwrite) { pteval = pte_mkyoung(pteval); pteval = maybe_mkwrite(pte_mkdirty(pteval), vma); } inc_mm_counter(vma->vm_mm, mm_counter_file(folio)); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pteval)); folio_add_file_rmap_pte(folio, page, vma); } set_pte_at(vma->vm_mm, addr, pte, pteval); @@ -2390,7 +2610,7 @@ static int insert_page_into_pte_locked(struct vm_area_struct *vma, pte_t *pte, } static int insert_page(struct vm_area_struct *vma, unsigned long addr, - struct page *page, pgprot_t prot, bool mkwrite) + struct page *page, pgprot_t prot, bool mkwrite, unsigned int offset) { int retval; pte_t *pte; @@ -2404,83 +2624,120 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, if (!pte) goto out; retval = insert_page_into_pte_locked(vma, pte, addr, page, prot, - mkwrite); + mkwrite, offset); pte_unmap_unlock(pte, ptl); out: return retval; } -static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, - unsigned long addr, struct page *page, pgprot_t prot) +/* Keep lock batching in units of hardware leaves, backing in native pages. */ +static int insert_page_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long offset, + unsigned long *length, pgprot_t prot) { - int err; + struct mm_struct *mm = vma->vm_mm; + unsigned long remaining = *length; + unsigned long size = mm_page_size(mm); + int ret = 0; - err = validate_page_before_insert(vma, page); - if (err) - return err; - return insert_page_into_pte_locked(vma, pte, addr, page, prot, false); + while (remaining) { + unsigned long in_pmd = min(remaining, + pmd_size_mm(mm) - (addr & ~pmd_mask_mm(mm))); + pmd_t *pmd = walk_to_pmd(mm, addr); + + if (!pmd) { + ret = -EFAULT; + break; + } + if (pte_alloc(mm, pmd)) { + ret = -ENOMEM; + break; + } + while (in_pmd) { + unsigned int batch = min_t(unsigned long, in_pmd / size, 8); + unsigned int i; + spinlock_t *ptl; + pte_t *start = pte_offset_map_lock(mm, pmd, addr, &ptl); + + if (!start) { + ret = -EFAULT; + goto out; + } + for (i = 0; i < batch; i++) { + struct page *page = pages[offset >> PAGE_SHIFT]; + + ret = validate_page_before_insert(vma, page); + if (!ret) + ret = insert_page_into_pte_locked(vma, start + i, + addr, page, prot, false, offset_in_page(offset)); + if (ret) + break; + addr += size; + offset += size; + remaining -= size; + in_pmd -= size; + } + pte_unmap_unlock(start, ptl); + if (ret) + goto out; + } + } +out: + *length = remaining; + return ret; } -/* insert_pages() amortizes the cost of spinlock operations - * when inserting pages in a loop. - */ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot) { - pmd_t *pmd = NULL; - pte_t *start_pte, *pte; - spinlock_t *pte_lock; - struct mm_struct *const mm = vma->vm_mm; - unsigned long curr_page_idx = 0; - unsigned long remaining_pages_total = *num; - unsigned long pages_to_write_in_pmd; + unsigned long length; int ret; -more: - ret = -EFAULT; - pmd = walk_to_pmd(mm, addr); - if (!pmd) - goto out; - pages_to_write_in_pmd = min_t(unsigned long, - remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); - - /* Allocate the PTE if necessary; takes PMD lock once only. */ - ret = -ENOMEM; - if (pte_alloc(mm, pmd)) - goto out; + if (check_mul_overflow(*num, PAGE_SIZE, &length)) + return -EOVERFLOW; + ret = insert_page_range(vma, addr, pages, 0, &length, prot); + /* A partially inserted native page is still an incomplete input page. */ + *num = DIV_ROUND_UP(length, PAGE_SIZE); + return ret; +} - while (pages_to_write_in_pmd) { - int pte_idx = 0; - const int batch_size = min_t(int, pages_to_write_in_pmd, 8); +/** + * vm_insert_pages_range - map a byte range from a native-page array + * @vma: target VMA, held under the mmap write lock + * @addr: first user address + * @pages: native backing pages + * @nr_pages: size of @pages + * @offset: byte offset within the backing array + * @length: bytes to map + * + * Address, offset and length must match the target mm's granule. A failed + * insertion can leave a prefix mapped, as with vm_insert_pages(); the caller + * must unmap that prefix when abandoning the mapping. Physical ownership + * remains in native pages. Each installed user leaf acquires its own ref. + */ +int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long nr_pages, + unsigned long offset, unsigned long length) +{ + unsigned long bytes, end, size = mm_page_size(vma->vm_mm); - start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); - if (!start_pte) { - ret = -EFAULT; - goto out; - } - for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { - int err = insert_page_in_batch_locked(vma, pte, - addr, pages[curr_page_idx], prot); - if (unlikely(err)) { - pte_unmap_unlock(start_pte, pte_lock); - ret = err; - remaining_pages_total -= pte_idx; - goto out; - } - addr += PAGE_SIZE; - ++curr_page_idx; - } - pte_unmap_unlock(start_pte, pte_lock); - pages_to_write_in_pmd -= batch_size; - remaining_pages_total -= batch_size; + if (!IS_ALIGNED(addr | offset | length, size) || + check_mul_overflow(nr_pages, PAGE_SIZE, &bytes) || + offset > bytes || length > bytes - offset || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) + return -EINVAL; + if (!length) + return 0; + if (vma->vm_flags & VM_PFNMAP) + return -EINVAL; + if (!(vma->vm_flags & VM_MIXEDMAP)) { + mmap_assert_write_locked(vma->vm_mm); + vm_flags_set(vma, VM_MIXEDMAP); } - if (remaining_pages_total) - goto more; - ret = 0; -out: - *num = remaining_pages_total; - return ret; + return insert_page_range(vma, addr, pages, offset, &length, vma->vm_page_prot); } +EXPORT_SYMBOL_GPL(vm_insert_pages_range); /** * vm_insert_pages - insert multiple pages into user vma, batching the pmd lock. @@ -2501,9 +2758,12 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num) { const unsigned long nr_pages = *num; - const unsigned long end = addr + PAGE_SIZE * nr_pages; + unsigned long length, end; - if (!range_in_vma(vma, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(vma->vm_mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &length) || + check_add_overflow(addr, length, &end) || + !range_in_vma(vma, addr, end)) return -EFAULT; if (!(vma->vm_flags & VM_MIXEDMAP)) { VM_WARN_ON_ONCE(mmap_read_trylock(vma->vm_mm)); @@ -2528,8 +2788,10 @@ int map_kernel_pages_prepare(struct vm_area_desc *desc) } nr_pages = action->map_kernel.nr_pages; - end = addr + PAGE_SIZE * nr_pages; - if (!range_in_vma_desc(desc, addr, end)) + if (!IS_ALIGNED(addr, mm_page_size(desc->mm)) || + check_mul_overflow(nr_pages, PAGE_SIZE, &end) || + check_add_overflow(addr, end, &end) || + !range_in_vma_desc(desc, addr, end)) return -EFAULT; return 0; @@ -2588,10 +2850,35 @@ int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, BUG_ON(vma->vm_flags & VM_PFNMAP); vm_flags_set(vma, VM_MIXEDMAP); } - return insert_page(vma, addr, page, vma->vm_page_prot, false); + return insert_page(vma, addr, page, vma->vm_page_prot, false, 0); } EXPORT_SYMBOL(vm_insert_page); +#ifdef CONFIG_MM_SUBPAGE +/* Explicit physical offset; the VMA and supplied page retain native ownership. */ +vm_fault_t vmf_insert_page_offset(struct vm_area_struct *vma, unsigned long addr, + struct page *page, unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long size = mm_page_size(mm); + int err; + + if (!(vma->vm_flags & VM_MIXEDMAP) || (vma->vm_flags & VM_PFNMAP) || + (addr & (size - 1)) || (offset & (size - 1)) || + offset > PAGE_SIZE - size || addr < vma->vm_start || + addr >= vma->vm_end || vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + err = insert_page(vma, addr, page, vma->vm_page_prot, false, offset); + if (err == -ENOMEM) + return VM_FAULT_OOM; + if (err < 0 && err != -EBUSY) + return VM_FAULT_SIGBUS; + return VM_FAULT_NOPAGE; +} +EXPORT_SYMBOL_GPL(vmf_insert_page_offset); +#endif + + /* * __vm_map_pages - maps range of kernel pages into user vma * @vma: user vma to map to @@ -2606,20 +2893,22 @@ EXPORT_SYMBOL(vm_insert_page); * Return: 0 on success and error code otherwise. */ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, - unsigned long num, unsigned long offset) + unsigned long num, unsigned long offset, + unsigned int subpage_offset) { - unsigned long count = vma_pages(vma); - unsigned long uaddr = vma->vm_start; + unsigned long bytes, available; - /* Fail if the user requested offset is beyond the end of the object */ - if (offset >= num) + if (offset >= num || + check_mul_overflow(num - offset, PAGE_SIZE, &available) || + subpage_offset >= PAGE_SIZE || + check_mul_overflow(offset, PAGE_SIZE, &bytes) || + check_add_overflow(bytes, (unsigned long)subpage_offset, &bytes)) return -ENXIO; - - /* Fail if the user requested size exceeds available object size */ - if (count > num - offset) + if (vma->vm_end - vma->vm_start > + available - subpage_offset) return -ENXIO; - - return vm_insert_pages(vma, uaddr, pages + offset, &count); + return vm_insert_pages_range(vma, vma->vm_start, pages, num, bytes, + vma->vm_end - vma->vm_start); } /** @@ -2643,7 +2932,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, vma->vm_pgoff); + return __vm_map_pages(vma, pages, num, vma->vm_pgoff, vma_subpage_offset(vma)); } EXPORT_SYMBOL(vm_map_pages); @@ -2663,12 +2952,12 @@ EXPORT_SYMBOL(vm_map_pages); int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, unsigned long num) { - return __vm_map_pages(vma, pages, num, 0); + return __vm_map_pages(vma, pages, num, 0, 0); } EXPORT_SYMBOL(vm_map_pages_zero); static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, pgprot_t prot, bool mkwrite) + unsigned long pfn, pgprot_t prot, bool mkwrite, unsigned int offset) { struct mm_struct *mm = vma->vm_mm; pte_t *pte, entry; @@ -2690,7 +2979,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * allocation and mapping invalidation so just skip the * update. */ - if (pte_pfn(entry) != pfn) { + if (pte_phys_mm(mm, entry) != PFN_PHYS(pfn) + offset) { WARN_ON_ONCE(!is_zero_pfn(pte_pfn(entry))); goto out_unlock; } @@ -2703,7 +2992,7 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, } /* Ok, finally just insert the thing.. */ - entry = pte_mkspecial(pfn_pte(pfn, prot)); + entry = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(pfn) + offset, prot)); if (mkwrite) { entry = pte_mkyoung(entry); @@ -2765,9 +3054,9 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, +static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, - bool mkwrite) + bool mkwrite, unsigned int offset) { /* * Technically, architectures with pte_special can avoid all these @@ -2789,10 +3078,31 @@ vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, pfnmap_setup_cachemode_pfn(pfn, &pgprot); - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, offset); +} +vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot, + bool mkwrite) +{ + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, 0); } EXPORT_SYMBOL(vmf_insert_pfn_prot_mkwrite); +/* Explicit quarter offset; a native PFN alone cannot encode a 4K leaf. */ +vm_fault_t vmf_insert_pfn_prot_mkwrite_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, unsigned int offset, + pgprot_t pgprot, bool mkwrite) +{ + unsigned long size = mm_page_size(vma->vm_mm); + + if (!IS_ALIGNED(addr | offset, size) || offset > PAGE_SIZE - size || + addr < vma->vm_start || addr >= vma->vm_end || + vma->vm_end - addr < size) + return VM_FAULT_SIGBUS; + return __vmf_insert_pfn_prot_mkwrite(vma, addr, pfn, pgprot, mkwrite, offset); +} +EXPORT_SYMBOL_GPL(vmf_insert_pfn_prot_mkwrite_offset); + static bool vm_mixed_ok(struct vm_area_struct *vma, unsigned long pfn, bool mkwrite) { @@ -2840,9 +3150,9 @@ static vm_fault_t __vm_insert_mixed(struct vm_area_struct *vma, * result in pfn_t_has_page() == false. */ page = pfn_to_page(pfn); - err = insert_page(vma, addr, page, pgprot, mkwrite); + err = insert_page(vma, addr, page, pgprot, mkwrite, 0); } else { - return insert_pfn(vma, addr, pfn, pgprot, mkwrite); + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, 0); } if (err == -ENOMEM) @@ -2863,7 +3173,7 @@ vm_fault_t vmf_insert_page_mkwrite(struct vm_fault *vmf, struct page *page, if (addr < vmf->vma->vm_start || addr >= vmf->vma->vm_end) return VM_FAULT_SIGBUS; - err = insert_page(vmf->vma, addr, page, pgprot, write); + err = insert_page(vmf->vma, addr, page, pgprot, write, 0); if (err == -ENOMEM) return VM_FAULT_OOM; if (err < 0 && err != -EBUSY) @@ -2898,7 +3208,7 @@ vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma, */ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pte_t *pte, *mapped_pte; spinlock_t *ptl; @@ -2910,13 +3220,13 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, lazy_mmu_mode_enable(); do { BUG_ON(!pte_none(ptep_get(pte))); - if (!pfn_modify_allowed(pfn, prot)) { + if (!pfn_modify_allowed(PHYS_PFN(phys), prot)) { err = -EACCES; break; } - set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot))); - pfn++; - } while (pte++, addr += PAGE_SIZE, addr != end); + set_pte_at(mm, addr, pte, pte_mkspecial(phys_pte_mm(mm, phys, prot))); + phys += mm_page_size(mm); + } while (pte++, addr += mm_page_size(mm), addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(mapped_pte, ptl); return err; @@ -2924,67 +3234,67 @@ static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd, static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pmd_t *pmd; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; VM_BUG_ON(pmd_trans_huge(*pmd)); do { - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(mm, addr, end); err = remap_pte_range(mm, pmd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pmd++, addr = next, addr != end); return 0; } static inline int remap_pud_range(struct mm_struct *mm, p4d_t *p4d, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { pud_t *pud; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; pud = pud_alloc(mm, p4d, addr); if (!pud) return -ENOMEM; do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(mm, addr, end); err = remap_pmd_range(mm, pud, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pud++, addr = next, addr != end); return 0; } static inline int remap_p4d_range(struct mm_struct *mm, pgd_t *pgd, unsigned long addr, unsigned long end, - unsigned long pfn, pgprot_t prot) + phys_addr_t phys, pgprot_t prot) { p4d_t *p4d; unsigned long next; int err; - pfn -= addr >> PAGE_SHIFT; p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return -ENOMEM; do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(mm, addr, end); err = remap_pud_range(mm, p4d, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (p4d++, addr = next, addr != end); return 0; } @@ -3013,25 +3323,26 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad { pgd_t *pgd; unsigned long next; - unsigned long end = addr + PAGE_ALIGN(size); struct mm_struct *mm = vma->vm_mm; + unsigned long end = addr + mm_page_align(mm, size); + phys_addr_t phys = PFN_PHYS(pfn); int err; - if (WARN_ON_ONCE(!PAGE_ALIGNED(addr))) + if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) return -EINVAL; VM_WARN_ON_ONCE(!vma_test_all_mask(vma, VMA_REMAP_FLAGS)); BUG_ON(addr >= end); - pfn -= addr >> PAGE_SHIFT; pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(mm, addr, end); err = remap_p4d_range(mm, pgd, addr, next, - pfn + (addr >> PAGE_SHIFT), prot); + phys, prot); if (err) return err; + phys += next - addr; } while (pgd++, addr = next, addr != end); return 0; @@ -3093,7 +3404,7 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, struct pfnmap_track_ctx *ctx = NULL; int err; - size = PAGE_ALIGN(size); + size = mm_page_align(vma->vm_mm, size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3162,7 +3473,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size) { - const unsigned long end = addr + PAGE_ALIGN(size); + const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); int err; @@ -3540,15 +3851,14 @@ static inline int pte_unmap_same(struct vm_fault *vmf) * -EHWPOISON: copy failed due to hwpoison in source page * -EAGAIN: copied failed (some other reason) */ -static inline int __wp_page_copy_user(struct page *dst, struct page *src, - struct vm_fault *vmf) +static inline int __wp_native_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf, unsigned long addr) { int ret; void *kaddr; void __user *uaddr; struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; - unsigned long addr = vmf->address; if (likely(src)) { if (copy_mc_user_highpage(dst, src, addr, vma)) @@ -3638,6 +3948,22 @@ static inline int __wp_page_copy_user(struct page *dst, struct page *src, return ret; } +static int __wp_page_copy_user(struct page *dst, struct page *src, + struct vm_fault *vmf) +{ + unsigned int i, nr = mm_pte_native_pages(vmf->vma->vm_mm); + + for (i = 0; i < nr; i++) { + int ret = __wp_native_page_copy_user(dst + i, src ? src + i : NULL, + vmf, vmf->address + i * PAGE_SIZE); + + if (ret) + return ret; + kmsan_copy_page_meta(dst + i, src ? src + i : NULL); + } + return 0; +} + static gfp_t __get_fault_gfp_mask(struct vm_area_struct *vma) { struct file *vm_file = vma->vm_file; @@ -3883,14 +4209,13 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) delayacct_wpcopy_end(); return err == -EHWPOISON ? VM_FAULT_HWPOISON : 0; } - kmsan_copy_page_meta(&new_folio->page, vmf->page); } __folio_mark_uptodate(new_folio); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, - vmf->address & PAGE_MASK, - (vmf->address & PAGE_MASK) + PAGE_SIZE); + vmf->address & mm_page_mask(mm), + (vmf->address & mm_page_mask(mm)) + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); /* @@ -3927,6 +4252,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) * some TLBs while the old PTE remains in others. */ ptep_clear_flush(vma, vmf->address, vmf->pte); + folio_ref_add(new_folio, mm_pte_native_pages(mm) - 1); folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); @@ -3970,7 +4296,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) mmu_notifier_invalidate_range_end(&range); if (new_folio) - folio_put(new_folio); + folio_put_refs(new_folio, page_copied ? mm_pte_native_pages(mm) : 1); if (old_folio) { if (page_copied) free_swap_cache(old_folio); @@ -4092,7 +4418,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf, struct folio *folio) return ret; } -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#ifdef CONFIG_MM_ID static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { @@ -4158,18 +4484,18 @@ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, folio_unlock_large_mapcount(folio); return exclusive; } -#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ +#else /* !CONFIG_MM_ID */ static bool __wp_can_reuse_large_anon_folio(struct folio *folio, struct vm_area_struct *vma) { BUILD_BUG(); } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* CONFIG_MM_ID */ static bool wp_can_reuse_anon_folio(struct folio *folio, struct vm_area_struct *vma) { - if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && folio_test_large(folio)) + if (IS_ENABLED(CONFIG_MM_ID) && folio_test_large(folio)) return __wp_can_reuse_large_anon_folio(folio, vma); /* @@ -4207,67 +4533,238 @@ static bool wp_can_reuse_anon_folio(struct folio *folio, return true; } -/* - * This routine handles present pages, when - * * users try to write to a shared page (FAULT_FLAG_WRITE) - * * GUP wants to take a R/O pin on a possibly shared anonymous page - * (FAULT_FLAG_UNSHARE) - * - * It is done by copying the page to a new address and decrementing the - * shared-page counter for the old page. - * - * Note that this routine assumes that the protection checks have been - * done by the caller (the low-level page fault routine in most cases). - * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've - * done any necessary COW. - * - * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even - * though the page will change only once the write actually happens. This - * avoids a few races, and potentially makes it more efficient. - * - * We enter with non-exclusive mmap_lock (to exclude vma changes, - * but allow concurrent faults), with pte both mapped and locked. - * We return with mmap_lock still held, but pte unmapped and unlocked. - */ -static vm_fault_t do_wp_page(struct vm_fault *vmf) +#ifdef CONFIG_MM_SUBPAGE +/* Enter with the original PTE locked; a slot, not its native folio, is copied. */ +static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) __releases(vmf->ptl) { - const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; struct vm_area_struct *vma = vmf->vma; - struct folio *folio = NULL; - pte_t pte; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *source = NULL, *dest = NULL; + struct mm_subpage_pool *pool = NULL; + struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; + struct folio *new = NULL; + struct mmu_notifier_range range; + bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + unsigned int offset; + vm_fault_t ret = VM_FAULT_SIGBUS; + pte_t entry; + int err; - if (likely(!unshare)) { - if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { - if (!userfaultfd_wp_async(vma)) { - pte_unmap_unlock(vmf->pte, vmf->ptl); - return handle_userfault(vmf, VM_UFFD_WP); - } + if (old && folio_test_anon(old)) { + source = mm_subpage_get_from_phys(pte_phys_mm(mm, vmf->orig_pte)); + if (WARN_ON_ONCE(!source)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; + } + if (!mm_subpage_is_exclusive(source) && mm_subpage_mapcount(source) == 1 && + folio_trylock(old)) { + bool reusable = true; - /* - * Nothing needed (cache flush, TLB invalidations, - * etc.) because we're only removing the uffd-wp bit, - * which is completely invisible to the user. - */ - pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + if (folio_test_swapcache(old)) { + swp_entry_t entry = page_swap_entry(&old->page); - set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); - /* - * Update this to be prepared for following up CoW - * handling - */ - vmf->orig_pte = pte; + reusable = !swap_subpage_count(entry, mm_subpage_offset(source)) && + !(folio_test_writeback(old) && + (folio_swap_flags(old) & SWP_STABLE_WRITES)); + } + if (reusable) + mm_subpage_try_reuse_anon(source); + folio_unlock(old); } - - /* - * Userfaultfd write-protect can defer flushes. Ensure the TLB - * is flushed in this case before copying. - */ - if (unlikely(userfaultfd_wp(vmf->vma) && - mm_tlb_flush_pending(vmf->vma->vm_mm))) - flush_tlb_page(vmf->vma, vmf->address); + if (mm_subpage_is_exclusive(source)) { + if (unshare) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + } else { + folio_xchg_last_cpupid(old, (1 << LAST_CPUPID_SHIFT) - 1); + /* Exclusivity belongs to the slot, not PageAnonExclusive. */ + wp_page_reuse(vmf, NULL); + } + mm_subpage_put(source); + return 0; + } + } else if (old) { + folio_get(old); + } else if (!is_zero_pfn(pte_pfn(vmf->orig_pte))) { + /* Private device/PFN mappings need their own subpage contract. */ + pte_unmap_unlock(vmf->pte, vmf->ptl); + return VM_FAULT_SIGBUS; } - + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + delayacct_wpcopy_start(); + ret = vmf_anon_prepare(vmf); + if (ret) + goto out; + offset = vma_page_offset_at(vma, vmf->address).offset; + ret = VM_FAULT_OOM; + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + goto out; + new = folio_prealloc(mm, vma, vmf->address, false); + if (!new) + goto out; + err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); + if (err) { + folio_put(new); + goto out; + } + dest = source ? mm_subpage_copy_at(pool, source, offset) : + mm_subpage_alloc_at(pool, offset); + if (IS_ERR(dest)) { + if (PTR_ERR(dest) == -EHWPOISON) + ret = VM_FAULT_HWPOISON; + dest = NULL; + goto out; + } + if (old && !source) { + err = copy_user_subpage_range(&new->page, offset, vmf->page, + pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, + mm_page_size(mm)); + if (err) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + mm_subpage_pool_close(pool); + mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, + vmf->address, vmf->address + mm_page_size(mm)); + mmu_notifier_invalidate_range_start(&range); + /* Keep the folio alive after a concurrent zap can consume its PTE ref. */ + folio_get(new); + folio_lock(new); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + mm_subpage_set_exclusive(dest); + err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + goto unlock_pte; + } + entry = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(dest), vma->vm_page_prot)); + if (unshare) { + if (pte_soft_dirty(vmf->orig_pte)) + entry = pte_mksoft_dirty(entry); + if (pte_uffd_wp(vmf->orig_pte)) + entry = pte_mkuffd_wp(entry); + } else { + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + } + BUG_ON(unshare && pte_write(entry)); + flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); + /* Break-before-make also orders removal of the old slot's rmap. */ + ptep_clear_flush(vma, vmf->address, vmf->pte); + subpage_add_new_lru(new, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + if (source) { + mm_subpage_remove_anon_rmap(source, vma); + } else if (old) { + mm_subpage_file_map_del(vma, pte_phys_mm(mm, vmf->orig_pte)); + folio_remove_rmap_pte(old, vmf->page, vma); + dec_mm_counter(mm, mm_counter_file(old)); + inc_mm_counter(mm, MM_ANONPAGES); + } else { + ksm_might_unmap_zero_page(mm, vmf->orig_pte); + inc_mm_counter(mm, MM_ANONPAGES); + } + dest = NULL; /* The new PTE owns the reference. */ + copied = true; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(new); + folio_put(new); + mmu_notifier_invalidate_range_end(&range); +out: + if (pool) { + mm_subpage_pool_close(pool); + if (dest) + mm_subpage_put(dest); + mm_subpage_pool_put(pool); + } + if (source) { + if (copied) + mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ + mm_subpage_put(source); /* Source stabilisation reference. */ + } else if (old) { + if (copied) + folio_put(old); + folio_put(old); + } + delayacct_wpcopy_end(); + return ret; +} +#endif + +/* + * This routine handles present pages, when + * * users try to write to a shared page (FAULT_FLAG_WRITE) + * * GUP wants to take a R/O pin on a possibly shared anonymous page + * (FAULT_FLAG_UNSHARE) + * + * It is done by copying the page to a new address and decrementing the + * shared-page counter for the old page. + * + * Note that this routine assumes that the protection checks have been + * done by the caller (the low-level page fault routine in most cases). + * Thus, with FAULT_FLAG_WRITE, we can safely just mark it writable once we've + * done any necessary COW. + * + * In case of FAULT_FLAG_WRITE, we also mark the page dirty at this point even + * though the page will change only once the write actually happens. This + * avoids a few races, and potentially makes it more efficient. + * + * We enter with non-exclusive mmap_lock (to exclude vma changes, + * but allow concurrent faults), with pte both mapped and locked. + * We return with mmap_lock still held, but pte unmapped and unlocked. + */ +static vm_fault_t do_wp_page(struct vm_fault *vmf) + __releases(vmf->ptl) +{ + const bool unshare = vmf->flags & FAULT_FLAG_UNSHARE; + struct vm_area_struct *vma = vmf->vma; + struct folio *folio = NULL; + pte_t pte; + + if (likely(!unshare)) { + if (userfaultfd_pte_wp(vma, ptep_get(vmf->pte))) { + if (!userfaultfd_wp_async(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + return handle_userfault(vmf, VM_UFFD_WP); + } + + /* + * Nothing needed (cache flush, TLB invalidations, + * etc.) because we're only removing the uffd-wp bit, + * which is completely invisible to the user. + */ + pte = pte_clear_uffd_wp(ptep_get(vmf->pte)); + + set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte); + /* + * Update this to be prepared for following up CoW + * handling + */ + vmf->orig_pte = pte; + } + + /* + * Userfaultfd write-protect can defer flushes. Ensure the TLB + * is flushed in this case before copying. + */ + if (unlikely(userfaultfd_wp(vmf->vma) && + mm_tlb_flush_pending(vmf->vma->vm_mm))) + flush_tlb_page(vmf->vma, vmf->address); + } + vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte); if (vmf->page) @@ -4292,6 +4789,11 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) return wp_page_shared(vmf, folio); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return wp_subpage_copy(vmf); +#endif + /* * Private mapping: create an exclusive anonymous page copy if reuse * is impossible. We might miss VM_WRITE for FOLL_FORCE handling. @@ -4301,8 +4803,12 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) */ if (folio && folio_test_anon(folio) && (PageAnonExclusive(vmf->page) || wp_can_reuse_anon_folio(folio, vma))) { - if (!PageAnonExclusive(vmf->page)) - SetPageAnonExclusive(vmf->page); + if (!PageAnonExclusive(vmf->page)) { + unsigned int i, nr = mm_pte_native_pages(vma->vm_mm); + + for (i = 0; i < nr; i++) + SetPageAnonExclusive(vmf->page + i); + } if (unlikely(unshare)) { pte_unmap_unlock(vmf->pte, vmf->ptl); return 0; @@ -4340,12 +4846,116 @@ static inline void unmap_mapping_range_tree(struct rb_root_cached *root, start = vma->vm_start + ((start_idx - vma->vm_pgoff) << PAGE_SHIFT); size = (end_idx - start_idx) << PAGE_SHIFT; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + unsigned long end = vma->vm_end; + + start = vma_address(vma, first_index, last_index - first_index + 1); + /* The interval tree guarantees a nonempty intersection. */ + VM_BUG_ON_VMA(IS_ERR_VALUE(start), vma); + if (last_index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = last_index + 1, + }); + size = end - start; + } else if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + unsigned long granule = mm_page_size(vma->vm_mm); + unsigned long end = min(start + size, vma->vm_end); + + /* A native-page invalidation removes every overlapping leaf. */ + start = max(round_down(start, granule), vma->vm_start); + end = min(round_up(end, granule), vma->vm_end); + size = end - start; + } +#endif + tlb_gather_mmu(&tlb, vma->vm_mm); zap_vma_range_batched(&tlb, vma, start, size, details); tlb_finish_mmu(&tlb); } } +#ifdef CONFIG_MM_SUBPAGE +/* Unmap affected user leaves, including coarse aliases; preserve private COW. */ +void unmap_mapping_subpage_hole(struct address_space *mapping, loff_t first, u64 last) +{ + struct vm_area_struct *vma; + struct zap_details details = { .skip_cows = true }; + + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, + first >> PAGE_SHIFT, (last - 1) >> PAGE_SHIFT) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + u64 low = round_down((u64)first, granule), high = round_up(last, granule); + struct vm_page_offset begin = { low >> PAGE_SHIFT, low & ~PAGE_MASK }; + struct vm_page_offset finish = { high >> PAGE_SHIFT, high & ~PAGE_MASK }; + struct vm_page_offset vbegin = vma_page_offset_at(vma, vma->vm_start); + struct vm_page_offset vend = vma_page_offset_at(vma, vma->vm_end); + struct mmu_gather tlb; + + if (begin.index < vbegin.index || + (begin.index == vbegin.index && begin.offset <= vbegin.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, begin); + if (finish.index > vend.index || + (finish.index == vend.index && finish.offset >= vend.offset)) + end = vma->vm_end; + else + end = vma_address_at_offset(vma, finish); + if (IS_ERR_VALUE(start) || IS_ERR_VALUE(end) || start >= end) + continue; + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} + +/* + * Native truncation keeps its final partial page. Smaller-granule mms must + * additionally discard complete granules in that native page, including + * private COW copies. Call on both sides of cache truncation, as for the + * ordinary unmap_mapping_range() calls. + */ +void unmap_mapping_subpage_tail(struct address_space *mapping, loff_t newsize) +{ + struct vm_area_struct *vma; + struct zap_details details = { }; + pgoff_t index = newsize >> PAGE_SHIFT; + unsigned int offset = newsize & ~PAGE_MASK; + + if (!offset) + return; + i_mmap_lock_read(mapping); + vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) { + unsigned long start, end, granule = mm_page_size(vma->vm_mm); + struct vm_page_offset pos = { .index = index, .offset = round_up(offset, granule) }; + struct vm_page_offset vpos = vma_page_offset_at(vma, vma->vm_start); + struct mmu_gather tlb; + + if (pos.offset >= PAGE_SIZE) + continue; + if (pos.index < vpos.index || + (pos.index == vpos.index && pos.offset < vpos.offset)) + start = vma->vm_start; + else + start = vma_address_at_offset(vma, pos); + if (IS_ERR_VALUE(start)) + continue; + end = vma->vm_end; + if (index < vma_last_pgoff(vma)) + end = vma_address_at_offset(vma, (struct vm_page_offset) { + .index = index + 1, + }); + tlb_gather_mmu(&tlb, vma->vm_mm); + zap_vma_range_batched(&tlb, vma, start, end - start, &details); + tlb_finish_mmu(&tlb); + } + i_mmap_unlock_read(mapping); +} +#endif + /** * unmap_mapping_folio() - Unmap single folio from processes. * @folio: The locked folio to be unmapped. @@ -4772,6 +5382,134 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, } while (--nr_pages); } +#ifdef CONFIG_MM_SUBPAGE +/* The swap-cache index remains native-sized; the PTE retains its slot offset. */ +static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct swap_info_struct *si; + struct mm_subpage *slot = NULL; + struct folio *folio; + unsigned int offset = pte_swp_subpage_offset(vmf->orig_pte); + vm_fault_t ret; + pte_t pte; + int err; + + ret = vmf_anon_prepare(vmf); + if (ret) + return ret; + /* Swap encodes the physical slot, independent of its virtual position. */ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + folio = swap_cache_get_folio(entry); + if (folio) { + swap_update_readahead(folio, vma, vmf->address); + } else { + folio = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, vmf); + if (!folio) { + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte) { + if (pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret = VM_FAULT_OOM; + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + } + goto out_device; + } + ret = VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + ret |= folio_lock_or_retry(folio, vmf); + if (ret & VM_FAULT_RETRY) + goto out_folio; + if (!folio_matches_swap_entry(folio, entry)) + goto unlock_folio; + if (folio_order(folio) || !folio_test_uptodate(folio) || folio_test_ksm(folio)) { + ret = VM_FAULT_SIGBUS; + goto unlock_folio; + } + if (PageHWPoison(&folio->page)) { + ret = VM_FAULT_HWPOISON; + goto unlock_folio; + } + arch_swap_restore(folio_swap(entry, folio), folio); + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + slot = NULL; + if (err != -EAGAIN) + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + folio_throttle_swaprate(folio, GFP_KERNEL); + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (!pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto unlock_pte; + err = mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_pte; + } + /* + * A carried marker proves exclusivity directly. After the last fork + * sharer exits, recover it only with one swap reference to this exact + * quarter and no other mapping or escaped slot reference. Dropping a + * nonpresent PTE in zap flushes pending reclaim TLB invalidations first. + * Stable-write backends must still finish writeback before reuse. + */ + if (!(folio_test_writeback(folio) && data_race(si->flags & SWP_STABLE_WRITES))) { + if (pte_swp_exclusive(vmf->orig_pte)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + } + pte = pte_sw_mkyoung(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + swap_subpage_put(entry, offset, false); + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + slot = NULL; /* The restored PTE owns the slot reference. */ + vmf->orig_pte = pte; + if (should_try_to_free_swap(si, folio, vma, 2, vmf->flags)) + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); /* Drop the cache lookup, not the PTE reference. */ + put_swap_device(si); + if (vmf->flags & FAULT_FLAG_WRITE) { + ret |= do_wp_page(vmf); + return ret & VM_FAULT_ERROR ? ret & VM_FAULT_ERROR : ret; + } + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + return ret; +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + if (slot) + mm_subpage_put(slot); + folio_unlock(folio); +out_folio: + folio_put(folio); +out_device: + put_swap_device(si); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -4780,6 +5518,209 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry, * We return with the mmap_lock locked or unlocked in the same cases * as does filemap_fault(). */ +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SWAP) +/* One coarse PTE owns a naturally aligned group of native swap entries. */ +vm_fault_t do_swap_coarse_page(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + /* Arm64's largest base user page is 64K, its smallest native page 4K. */ + struct folio *cache[SZ_64K / PAGE_SIZE]; + unsigned int nr = mm_pte_native_pages(mm), covered = 0, count = 0, i; + swp_entry_t entry = softleaf_from_pte(vmf->orig_pte), next; + struct swap_info_struct *si; + struct folio *backing = NULL; + struct page *page; + unsigned int start; + bool copied = false, exclusive, mapped = false; + vm_fault_t ret = 0; + pte_t pte; + + vmf->pte = NULL; + if (WARN_ON_ONCE(nr <= 1 || nr > ARRAY_SIZE(cache) || + !IS_ALIGNED(swp_offset(entry), nr))) + return VM_FAULT_SIGBUS; + si = get_swap_device(entry); + if (!si) + return 0; + + while (covered < nr) { + struct folio *folio; + unsigned int take; + vm_fault_t wait; + + next = entry; + next.val += covered; + folio = swap_cache_get_folio(next); + if (!folio) { + folio = read_swap_cache_async(next, GFP_HIGHUSER_MOVABLE, vma, + vmf->address + covered * PAGE_SIZE, NULL); + if (!(ret & VM_FAULT_MAJOR)) { + ret |= VM_FAULT_MAJOR; + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(mm, PGMAJFAULT); + } + } + if (!folio) + goto oom; + if (!count) { + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + } else if (!folio_trylock(folio)) { + /* Drop earlier cache locks before waiting for this I/O or owner. */ + for (i = 0; i < count; i++) { + folio_unlock(cache[i]); + folio_put(cache[i]); + } + count = 0; + covered = 0; + wait = folio_lock_or_retry(folio, vmf); + if (wait & VM_FAULT_RETRY) { + folio_put(folio); + ret |= wait; + goto out; + } + folio_unlock(folio); + folio_put(folio); + cond_resched(); + continue; + } + cache[count++] = folio; + if (!folio_matches_swap_entry(folio, next)) + goto out; + if (!folio_test_uptodate(folio)) { + ret = VM_FAULT_SIGBUS; + goto out; + } + start = swp_offset(next) - swp_offset(folio->swap); + take = min(nr - covered, folio_nr_pages(folio) - start); + for (i = 0; i < take; i++) { + if (PageHWPoison(folio_page(folio, start + i))) { + ret = VM_FAULT_HWPOISON; + goto out; + } + } + arch_swap_restore(folio_swap(next, folio), folio); + covered += take; + } + + start = swp_offset(entry) - swp_offset(cache[0]->swap); + if (count == 1 && IS_ALIGNED(start, nr) && + (folio_test_anon(cache[0]) || folio_nr_pages(cache[0]) == nr)) { + backing = cache[0]; + page = folio_page(backing, start); + } else { + backing = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, ilog2(nr), vma, vmf->address); + if (!backing) + goto oom; + copied = true; + if (mem_cgroup_swapin_charge_folio(backing, mm, GFP_KERNEL, entry)) + goto oom; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int j, take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + for (j = 0; j < take; j++) { + struct page *dst = folio_page(backing, covered + j); + struct page *src = folio_page(cache[i], start + j); + + if (copy_mc_user_highpage(dst, src, + vmf->address + (covered + j) * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto out; + } + kmsan_copy_page_meta(dst, src); + } + covered += take; + } + folio_set_dirty(backing); + __folio_mark_uptodate(backing); + page = &backing->page; + } + + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte || !pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + goto out; + exclusive = copied || pte_swp_exclusive(vmf->orig_pte); + if (!copied && exclusive) { + check_swap_exclusive(backing, entry, nr); + if (folio_test_writeback(backing) && data_race(si->flags & SWP_STABLE_WRITES)) + exclusive = false; + } + pte = mk_pte(page, vma->vm_page_prot); + if (pte_swp_soft_dirty(vmf->orig_pte)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(vmf->orig_pte)) + pte = pte_mkuffd_wp(pte); + if (exclusive && (vma->vm_flags & VM_WRITE) && !userfaultfd_pte_wp(vma, pte) && + !pte_needs_soft_dirty_wp(vma, pte)) { + pte = pte_mkwrite(pte, vma); + if (vmf->flags & FAULT_FLAG_WRITE) { + pte = pte_mkdirty(pte); + vmf->flags &= ~FAULT_FLAG_WRITE; + } + } + folio_ref_add(backing, nr - copied); + if (!folio_test_anon(backing)) + folio_add_new_anon_rmap(backing, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + else + folio_add_anon_rmap_pte(backing, page, vma, vmf->address, + exclusive ? RMAP_EXCLUSIVE : RMAP_NONE); + if (copied) + folio_add_lru_vma(backing, vma); + inc_mm_counter(mm, MM_ANONPAGES); + dec_mm_counter(mm, MM_SWAPENTS); + flush_icache_pages(vma, page, nr); + vmf->orig_pte = pte; + set_pte_at(mm, vmf->address, vmf->pte, pte); + arch_do_swap_page_nr(mm, vma, vmf->address, pte, pte, 1); + mapped = true; + covered = 0; + for (i = 0; i < count; i++) { + unsigned int take; + + start = swp_offset(entry) + covered - swp_offset(cache[i]->swap); + take = min(nr - covered, folio_nr_pages(cache[i]) - start); + folio_put_swap_range(cache[i], folio_page(cache[i], start), take); + covered += take; + } + goto out; + +oom: + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (vmf->pte && pte_same(ptep_get(vmf->pte), vmf->orig_pte)) + ret |= VM_FAULT_OOM; +out: + if (copied && !mapped) + folio_put(backing); + for (i = 0; i < count; i++) { + if (mapped) + folio_free_swap(cache[i]); + folio_unlock(cache[i]); + folio_put(cache[i]); + } + if (mapped && (vmf->flags & FAULT_FLAG_WRITE)) { + ret |= do_wp_page(vmf); + vmf->pte = NULL; /* do_wp_page drops the PT lock. */ + } else if (mapped) { + update_mmu_cache(vma, vmf->address, vmf->pte); + } + if (vmf->pte) + pte_unmap_unlock(vmf->pte, vmf->ptl); + put_swap_device(si); + if (!mapped && !(ret & (VM_FAULT_ERROR | VM_FAULT_RETRY))) + cond_resched(); + return ret; +} +#endif + vm_fault_t do_swap_page(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; @@ -4854,6 +5795,15 @@ vm_fault_t do_swap_page(struct vm_fault *vmf) goto out; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_swap_subpage(vmf, entry); +#ifdef CONFIG_SWAP + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return do_swap_coarse_page(vmf); +#endif +#endif + /* Prevent swapoff from happening to us. */ si = get_swap_device(entry); if (unlikely(!si)) @@ -5291,6 +6241,7 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, bool uffd_wp) { const unsigned int nr_pages = folio_nr_pages(folio); + const unsigned int nr_ptes = folio_size(folio) >> mm_page_shift(vma->vm_mm); pte_t entry = folio_mk_pte(folio, vma->vm_page_prot); entry = pte_sw_mkyoung(entry); @@ -5303,8 +6254,8 @@ void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, folio_ref_add(folio, nr_pages - 1); folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); - set_ptes(vma->vm_mm, addr, pte, entry, nr_pages); - update_mmu_cache_range(NULL, vma, addr, pte, nr_pages); + set_ptes(vma->vm_mm, addr, pte, entry, nr_ptes); + update_mmu_cache_range(NULL, vma, addr, pte, nr_ptes); } static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, @@ -5313,10 +6264,123 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, const unsigned int order = folio_order(folio); map_anon_folio_pte_nopf(folio, pte, vma, addr, uffd_wp); - add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1L << order); + add_mm_counter(vma->vm_mm, MM_ANONPAGES, folio_size(folio) >> mm_page_shift(vma->vm_mm)); count_mthp_stat(order, MTHP_STAT_ANON_FAULT_ALLOC); } +#ifdef CONFIG_MM_SUBPAGE +static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + struct mm_subpage_pool *pool; + struct folio *folio; + unsigned long start, end, lower; + unsigned int first_offset, nr, target, i, mapped = 0; + vm_fault_t ret = VM_FAULT_OOM; + pte_t *first; + int err; + + /* One native linear index, clipped to the VMA and this PTE table. */ + lower = max(vma->vm_start, vmf->address & pmd_mask_mm(mm)); + start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); + end = vmf->address + min(PAGE_SIZE - pos.offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); + first_offset = pos.offset - (vmf->address - start); + nr = (end - start) >> mm_page_shift(mm); + target = (vmf->address - start) >> mm_page_shift(mm); + + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return VM_FAULT_OOM; + /* Supply initialises every byte once, including the unissued slots. */ + folio = folio_prealloc(mm, vma, vmf->address, false); + if (!folio) + goto release; + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto release; + } + for (i = 0; i < nr; i++) { + slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); + if (IS_ERR(slots[i])) { + slots[i] = NULL; + goto release; + } + } + + /* PTE references may disappear as soon as we drop the PTL. */ + folio_get(folio); + folio_lock(folio); + ret = 0; + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock_pte; + } + ret = check_stable_address_space(mm); + if (ret) + goto unlock_pte; + if (userfaultfd_missing(vma)) { + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; + folio_unlock(folio); + folio_put(folio); + ret = handle_userfault(vmf, VM_UFFD_MISSING); + goto release; + } + + first = vmf->pte - target; + for (i = 0; i < nr; i++) { + unsigned long address = start + i * mm_page_size(mm); + pte_t entry; + + /* Do not replace existing mappings or userfaultfd markers nearby. */ + if (i != target && !pte_none(ptep_get(first + i))) + continue; + mm_subpage_set_exclusive(slots[i]); + err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + if (WARN_ON_ONCE(err)) { + ret = VM_FAULT_SIGBUS; + break; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(pte_mkdirty(entry), vma); + if (i == target && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, address, first + i, entry); + update_mmu_cache(vma, address, first + i); + slots[i] = NULL; /* The PTE now owns the reference. */ + mapped++; + } + if (mapped) { + subpage_add_new_lru(folio, vma, mapped); + add_mm_counter(mm, MM_ANONPAGES, mapped); + count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); + } +unlock_pte: + pte_unmap_unlock(vmf->pte, vmf->ptl); + vmf->pte = NULL; +unlock_folio: + folio_unlock(folio); + folio_put(folio); +release: + mm_subpage_pool_close(pool); + for (i = 0; i < ARRAY_SIZE(slots); i++) + if (slots[i]) + mm_subpage_put(slots[i]); + mm_subpage_pool_put(pool); + return ret; +} +#endif + /* * We enter with non-exclusive mmap_lock (to exclude vma changes, * but allow concurrent faults), and pte mapped but not yet locked. @@ -5376,6 +6440,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return do_anonymous_subpage(vmf); +#endif /* Returns NULL on OOM or ERR_PTR(-EAGAIN) if we must retry the fault */ folio = alloc_anon_folio(vmf); if (IS_ERR(folio)) @@ -5383,8 +6451,8 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) if (!folio) goto oom; - nr_pages = folio_nr_pages(folio); - addr = ALIGN_DOWN(vmf->address, nr_pages * PAGE_SIZE); + nr_pages = folio_size(folio) >> mm_page_shift(vma->vm_mm); + addr = ALIGN_DOWN(vmf->address, folio_size(folio)); /* * The memory barrier inside __folio_mark_uptodate makes sure that @@ -5517,6 +6585,10 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa * PMD mappings if THPs are disabled. As we already have a THP, * behave as if we are forcing a collapse. */ + /* Match the native PTE fault policy: partial data exposes its zero subunits. */ + if (userfaultfd_missing(vma) && shmem_uffd_has_missing_native_page(folio)) + return ret; + if (thp_disabled_by_hw() || vma_thp_disabled(vma, vma->vm_flags, /* forced_collapse=*/ true)) return ret; @@ -5558,6 +6630,8 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); add_mm_counter(vma->vm_mm, mm_counter_file(folio), HPAGE_PMD_NR); + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + HPAGE_PMD_SIZE); folio_add_file_rmap_pmd(folio, page, vma); /* @@ -5620,6 +6694,8 @@ void set_pte_range(struct vm_fault *vmf, struct folio *folio, folio_add_new_anon_rmap(folio, vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(folio, vma); } else { + shmem_uffd_mark_mapped(folio, folio_page_idx(folio, page) << PAGE_SHIFT, + (unsigned long)nr << PAGE_SHIFT); folio_add_file_rmap_ptes(folio, page, nr, vma); } set_ptes(vma->vm_mm, addr, vmf->pte, entry, nr); @@ -5636,6 +6712,252 @@ static bool vmf_pte_changed(struct vm_fault *vmf) return !pte_none(ptep_get(vmf->pte)); } +#ifdef CONFIG_MM_SUBPAGE +static bool folio_user_range_valid(struct folio *folio, unsigned long offset, + unsigned long size) +{ + return offset <= folio_size(folio) && size <= folio_size(folio) - offset && + IS_ALIGNED(page_to_phys(&folio->page) + offset, size); +} + +/* + * Map the intersection of a locked cache folio, file-index range, VMA and + * the fault's PTE table. vmf->pte points at vmf->address and its lock is held. + * Each PTE owns a reference per covered native page (one for a subpage). + * The caller's reference is untouched, including when no PTE is installed. + */ +vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, + pgoff_t first, pgoff_t last, bool fault) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + unsigned long addr, end, table_start = vmf->address & pmd_mask_mm(mm); + unsigned long size = mm_page_size(mm); + loff_t file_size = i_size_read(vma->vm_file->f_mapping->host); + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + first = max(first, folio->index); + last = min(last, folio_next_index(folio) - 1); + if (first > last) + return 0; + addr = vma_address(vma, first, last - first + 1); + if (IS_ERR_VALUE(addr)) + return 0; + addr = ALIGN(max(addr, table_start), size); + end = pmd_addr_end_mm(mm, vmf->address, vma->vm_end); + if (userfaultfd_armed(vma)) { + addr = max(addr, vmf->address); + end = min(end, vmf->address + size); + } + for (; addr < end; addr += size) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + struct page *page; + unsigned long offset; + unsigned int i, nr = mm_pte_native_pages(mm); + bool poisoned = false; + pte_t entry, *ptep; + + if (pos.index > last || !vma_file_offset_valid(vma, addr, file_size)) + break; + offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + /* + * A coarse leaf cannot span separate cache folios or a partial + * requested index range, even if neighboring PFNs happen to match. + */ + if (nr - 1 > last - pos.index || + !folio_user_range_valid(folio, offset, size)) + continue; + /* A native cache folio can contain independently missing leaves. */ + if (userfaultfd_missing(vma) && + shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + continue; + page = folio_file_page(folio, pos.index); + ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + for (i = 0; i < nr; i++) + poisoned |= PageHWPoison(page + i); + if (poisoned) + continue; + if (!pte_none(ptep_get(ptep)) && + !(fault && addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + flush_icache_pages(vma, page, nr); + entry = phys_pte_mm(mm, page_to_phys(page) + pos.offset, vma->vm_page_prot); + if (addr != vmf->address && arch_wants_old_prefaulted_pte()) + entry = pte_mkold(entry); + else + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + else if (pte_write(entry) && folio_test_dirty(folio)) + entry = pte_mkdirty(entry); + if (fault && addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, nr); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + shmem_uffd_mark_mapped(folio, + ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset, size); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +/* + * A non-file fault callback supplies one native page, not a cache-indexed + * folio. In particular, VVAR callbacks may reorder native pages. Only map + * quarters of this supplied page; never infer adjacent pages from its folio. + */ +static vm_fault_t map_special_subpages(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct page *page = vmf->page; + struct folio *folio = page_folio(page); + unsigned int offset = vma_page_offset_at(vma, vmf->address).offset; + unsigned long size = mm_page_size(mm), addr, end; + vm_fault_t ret = 0; + unsigned int mapped = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (PageHWPoison(page)) + return 0; + addr = offset > vmf->address ? vma->vm_start : + max(vma->vm_start, vmf->address - offset); + addr = max(addr, vmf->address & pmd_mask_mm(mm)); + end = min(vmf->address + PAGE_SIZE - offset, + pmd_addr_end_mm(mm, vmf->address, vma->vm_end)); + if (userfaultfd_armed(vma)) { + addr = vmf->address; + end = vmf->address + size; + } + flush_icache_pages(vma, page, 1); + for (; addr < end; addr += size) { + unsigned int physical_offset = addr < vmf->address ? + offset - (vmf->address - addr) : offset + (addr - vmf->address); + pte_t *ptep = vmf->pte + ((long)(addr - vmf->address) >> mm_page_shift(mm)); + pte_t entry; + + if (!pte_none(ptep_get(ptep)) && + !(addr == vmf->address && !vmf_pte_changed(vmf))) + continue; + entry = phys_pte_mm(mm, page_to_phys(page) + physical_offset, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); + if (vmf->flags & FAULT_FLAG_WRITE) + entry = maybe_mkwrite(pte_mkdirty(entry), vma); + if (addr == vmf->address && vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_get(folio); + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, entry)); + folio_add_file_rmap_pte(folio, page, vma); + set_pte_at(mm, addr, ptep, entry); + update_mmu_cache_range(vmf, vma, addr, ptep, 1); + mapped++; + if (addr == vmf->address) + ret = VM_FAULT_NOPAGE; + } + add_mm_counter(mm, mm_counter_file(folio), mapped); + return ret; +} + +static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = vmf->cow_subpage; + bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); + struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + vm_fault_t ret; + pte_t entry; + + if (!(vma->vm_flags & VM_SHARED)) { + ret = check_stable_address_space(mm); + if (ret) + return ret; + } + if (pmd_none(*vmf->pmd)) { + if (vmf->prealloc_pte) + pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); + else if (pte_alloc(mm, vmf->pmd)) + return VM_FAULT_OOM; + } + /* The private destination is not yet visible through a PTE. */ + if (is_cow) { + folio_get(folio); + folio_lock(folio); + } + vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); + ret = VM_FAULT_NOPAGE; + if (!vmf->pte) + goto unlock_folio; + if (vmf_pte_changed(vmf)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); + goto unlock; + } + if (slot) { + mm_subpage_set_exclusive(slot); + if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + subpage_add_new_lru(folio, vma, 1); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + vmf->cow_subpage = NULL; + ret = 0; + } else if (is_cow) { + if (!folio_user_range_valid(folio, 0, mm_page_size(mm))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + entry = phys_pte_mm(mm, page_to_phys(&folio->page), vma->vm_page_prot); + entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); + if (vmf_orig_pte_uffd_wp(vmf)) + entry = pte_mkuffd_wp(entry); + folio_ref_add(folio, mm_pte_native_pages(mm) - 1); + folio_add_new_anon_rmap(folio, vma, vmf->address, RMAP_EXCLUSIVE); + folio_add_lru_vma(folio, vma); + set_pte_at(mm, vmf->address, vmf->pte, entry); + update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); + inc_mm_counter(mm, MM_ANONPAGES); + ret = 0; + } else { + if (vma->vm_file) + ret = map_file_user_pages(vmf, folio, folio->index, + folio_next_index(folio) - 1, true); + else if (mm_page_size(mm) < PAGE_SIZE) + ret = map_special_subpages(vmf); + else + ret = VM_FAULT_SIGBUS; + if (ret & VM_FAULT_NOPAGE) { + folio_put(folio); /* Consume the fault callback's reference. */ + ret = 0; + } else { + ret = VM_FAULT_SIGBUS; + } + } +unlock: + pte_unmap_unlock(vmf->pte, vmf->ptl); +unlock_folio: + if (is_cow) { + folio_unlock(folio); + folio_put(folio); + } + return ret; +} +#endif + /** * finish_fault - finish page fault once we have prepared the page to fault * @@ -5663,6 +6985,11 @@ vm_fault_t finish_fault(struct vm_fault *vmf) unsigned long addr; bool needs_fallback = false; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return finish_user_page_fault(vmf); +#endif + fallback: addr = vmf->address; @@ -5768,13 +7095,12 @@ vm_fault_t finish_fault(struct vm_fault *vmf) return ret; } -static unsigned long fault_around_pages __read_mostly = - 65536 >> PAGE_SHIFT; +static unsigned long fault_around_bytes __read_mostly = SZ_64K; #ifdef CONFIG_DEBUG_FS static int fault_around_bytes_get(void *data, u64 *val) { - *val = fault_around_pages << PAGE_SHIFT; + *val = READ_ONCE(fault_around_bytes); return 0; } @@ -5788,11 +7114,13 @@ static int fault_around_bytes_set(void *data, u64 val) return -EINVAL; /* - * The minimum value is 1 page, however this results in no fault-around - * at all. See should_fault_around(). + * Keep the policy in bytes: one native page can still contain several + * user leaves. The smallest supported leaf disables fault-around for + * every mm; larger values are compared with each mm's own granule. */ - val = max(val, PAGE_SIZE); - fault_around_pages = rounddown_pow_of_two(val) >> PAGE_SHIFT; + val = max_t(u64, val, IS_ENABLED(CONFIG_MM_SUBPAGE) ? + MM_SUBPAGE_SIZE : PAGE_SIZE); + WRITE_ONCE(fault_around_bytes, rounddown_pow_of_two(val)); return 0; } @@ -5819,24 +7147,42 @@ late_initcall(fault_around_debugfs); * This function doesn't cross VMA or page table boundaries, in order to call * map_pages() and acquire a PTE lock only once. * - * fault_around_pages defines how many pages we'll try to map. - * do_fault_around() expects it to be set to a power of two less than or equal - * to PTRS_PER_PTE. - * - * The virtual address of the area that we map is naturally aligned to - * fault_around_pages * PAGE_SIZE rounded down to the machine page size - * (and therefore to page order). This way it's easier to guarantee - * that we don't cross page table boundaries. + * fault_around_bytes is a power-of-two byte policy, independent of either + * physical or user granules. The native path converts it to native pages; + * the alternative path clamps it to the target mm's PTE-table coverage. + * Natural alignment keeps the window inside one page table. */ static vm_fault_t do_fault_around(struct vm_fault *vmf) { - pgoff_t nr_pages = READ_ONCE(fault_around_pages); + unsigned long configured_bytes = READ_ONCE(fault_around_bytes); + pgoff_t nr_pages = max(1UL, configured_bytes >> PAGE_SHIFT); pgoff_t pte_off = pte_index(vmf->address); /* The page offset of vmf->address within the VMA. */ pgoff_t vma_off = vmf->pgoff - vmf->vma->vm_pgoff; pgoff_t from_pte, to_pte; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) != PAGE_SIZE) { + struct vm_area_struct *vma = vmf->vma; + unsigned long bytes = min(configured_bytes, + pmd_size_mm(vma->vm_mm)); + unsigned long start = max(ALIGN_DOWN(vmf->address, bytes), vma->vm_start); + unsigned long end = min(ALIGN_DOWN(vmf->address, bytes) + bytes, vma->vm_end); + + if (pmd_none(*vmf->pmd)) { + vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); + if (!vmf->prealloc_pte) + return VM_FAULT_OOM; + } + rcu_read_lock(); + ret = vma->vm_ops->map_pages(vmf, vma_page_offset_at(vma, start).index, + vma_page_offset_at(vma, end - 1).index); + rcu_read_unlock(); + return ret; + } +#endif + /* The PTE offset of the start address, clamped to the VMA. */ from_pte = max(ALIGN_DOWN(pte_off, nr_pages), pte_off - min(pte_off, vma_off)); @@ -5871,7 +7217,7 @@ static inline bool should_fault_around(struct vm_fault *vmf) return false; /* A single page implies no faulting 'around' at all. */ - return fault_around_pages > 1; + return READ_ONCE(fault_around_bytes) > mm_page_size(vmf->vma->vm_mm); } static vm_fault_t do_read_fault(struct vm_fault *vmf) @@ -5918,10 +7264,26 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret) return ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + if (IS_ERR(vmf->cow_subpage)) { + vmf->cow_subpage = NULL; + return VM_FAULT_OOM; + } + folio = mm_subpage_folio(vmf->cow_subpage); + goto allocated; + } +#endif + folio = folio_prealloc(vma->vm_mm, vma, vmf->address, false); if (!folio) return VM_FAULT_OOM; +#ifdef CONFIG_MM_SUBPAGE +allocated: +#endif + vmf->cow_page = &folio->page; ret = __do_fault(vmf); @@ -5930,10 +7292,38 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (ret & VM_FAULT_DONE_COW) return ret; - if (copy_mc_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma)) { - ret = VM_FAULT_HWPOISON; +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + goto copied; + } +#endif + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE && + !folio_user_range_valid(page_folio(vmf->page), + folio_page_idx(page_folio(vmf->page), vmf->page) << PAGE_SHIFT, + mm_page_size(vma->vm_mm))) { + ret = VM_FAULT_SIGBUS; goto unlock; } +#endif + for (unsigned int i = 0; i < mm_pte_native_pages(vma->vm_mm); i++) { + if (copy_mc_user_highpage(vmf->cow_page + i, vmf->page + i, + vmf->address + i * PAGE_SIZE, vma)) { + ret = VM_FAULT_HWPOISON; + goto unlock; + } + } +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif + __folio_mark_uptodate(folio); ret |= finish_fault(vmf); @@ -5944,6 +7334,13 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto uncharge_out; return ret; uncharge_out: +#ifdef CONFIG_MM_SUBPAGE + if (vmf->cow_subpage) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + return ret; + } +#endif folio_put(folio); return ret; } @@ -6104,31 +7501,33 @@ static void numa_rebuild_single_mapping(struct vm_fault *vmf, struct vm_area_str update_mmu_cache_range(vmf, vma, fault_addr, fault_pte, 1); } -static void numa_rebuild_large_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, +static void numa_rebuild_folio_mapping(struct vm_fault *vmf, struct vm_area_struct *vma, struct folio *folio, pte_t fault_pte, bool ignore_writable, bool pte_write_upgrade) { - int nr = pte_pfn(fault_pte) - folio_pfn(folio); + struct mm_struct *mm = vma->vm_mm; + unsigned int page_shift = mm_page_shift(mm); + unsigned long page_size = 1UL << page_shift; + unsigned long offset = pte_phys_mm(mm, fault_pte) - PFN_PHYS(folio_pfn(folio)); unsigned long start, end, addr = vmf->address; - unsigned long addr_start = addr - (nr << PAGE_SHIFT); - unsigned long pt_start = ALIGN_DOWN(addr, PMD_SIZE); + unsigned long lower = max(addr & pmd_mask_mm(mm), vma->vm_start); + unsigned long upper = pmd_addr_end_mm(mm, addr, vma->vm_end); pte_t *start_ptep; - /* Stay within the VMA and within the page table. */ - start = max3(addr_start, pt_start, vma->vm_start); - end = min3(addr_start + folio_size(folio), pt_start + PMD_SIZE, - vma->vm_end); - start_ptep = vmf->pte - ((addr - start) >> PAGE_SHIFT); + /* Stay within the VMA, page table and physical owner without wrapping. */ + start = addr - min(offset, addr - lower); + end = addr + min(folio_size(folio) - offset, upper - addr); + start_ptep = vmf->pte - ((addr - start) >> page_shift); - /* Restore all PTEs' mapping of the large folio */ - for (addr = start; addr != end; start_ptep++, addr += PAGE_SIZE) { + /* Restore the owner's PTEs, including smaller leaves in a native folio. */ + for (addr = start; addr != end; start_ptep++, addr += page_size) { pte_t ptent = ptep_get(start_ptep); bool writable = false; if (!pte_present(ptent) || !pte_protnone(ptent)) continue; - if (pfn_folio(pte_pfn(ptent)) != folio) + if (pfn_folio(PHYS_PFN(pte_phys_mm(mm, ptent))) != folio) continue; if (!ignore_writable) { @@ -6221,8 +7620,8 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf) * Make it present again, depending on how arch implements * non-accessible ptes, some can allow access by kernel mode. */ - if (folio && folio_test_large(folio)) - numa_rebuild_large_mapping(vmf, vma, folio, pte, ignore_writable, + if (folio && (folio_test_large(folio) || mm_page_shift(vma->vm_mm) < PAGE_SHIFT)) + numa_rebuild_folio_mapping(vmf, vma, folio, pte, ignore_writable, pte_write_upgrade); else numa_rebuild_single_mapping(vmf, vma, vmf->address, vmf->pte, @@ -6455,7 +7854,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, { struct vm_fault vmf = { .vma = vma, - .address = address & PAGE_MASK, + .address = address & mm_page_mask(vma->vm_mm), .real_address = address, .flags = flags, .pgoff = linear_page_index(vma, address), @@ -6786,7 +8185,7 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) return -ENOMEM; spin_lock(&mm->page_table_lock); - if (!p4d_present(*p4d)) { + if (!p4d_present_mm(mm, *p4d)) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); @@ -6824,16 +8223,19 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) static inline int pfnmap_args_setup(struct follow_pfnmap_args *args, spinlock_t *lock, pte_t *ptep, - pgprot_t pgprot, unsigned long pfn_base, + pgprot_t pgprot, phys_addr_t phys_base, unsigned long addr_mask, bool writable, bool special) { + phys_addr_t phys = phys_base + (args->address & ~addr_mask); + if (!writable && args->write) return -EFAULT; args->lock = lock; args->ptep = ptep; - args->pfn = pfn_base + ((args->address & ~addr_mask) >> PAGE_SHIFT); + args->pfn = PHYS_PFN(phys); + args->offset = offset_in_page(phys); args->addr_mask = addr_mask; args->pgprot = pgprot; args->writable = writable; @@ -6885,8 +8287,14 @@ static inline void pfnmap_lockdep_assert(struct vm_area_struct *vma) * * This function must not be used to modify PTE content. * + * Legacy callers retain the native-page contract. A caller setting + * allow_subpage must use pfn plus offset to identify the requested physical + * byte and must limit access to the leaf described by addr_mask. This does + * not pin the native page or grant access to neighbouring user mappings. + * * Return: zero on success, -EFAULT if @args->write was set but the - * mapping is not writable, -EINVAL if there is no mapping at all. + * mapping is not writable, -EINVAL if there is no mapping at all, or + * -EOPNOTSUPP for a non-native granule without allow_subpage. */ int follow_pfnmap_start(struct follow_pfnmap_args *args) { @@ -6908,17 +8316,20 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) goto out; + /* A legacy PFN consumer cannot represent partial native-page ownership. */ + if (mm_page_size(mm) != PAGE_SIZE && !args->allow_subpage) + return -EOPNOTSUPP; retry: pgdp = pgd_offset(mm, address); if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) goto out; - p4dp = p4d_offset(pgdp, address); + p4dp = p4d_offset_mm(mm, pgdp, address); p4d = p4dp_get(p4dp); - if (p4d_none(p4d) || unlikely(p4d_bad(p4d))) + if (p4d_none_mm(mm, p4d) || unlikely(p4d_bad_mm(mm, p4d))) goto out; - pudp = pud_offset(p4dp, address); + pudp = pud_offset_mm(mm, p4dp, address); pud = pudp_get(pudp); if (!pud_present(pud)) goto out; @@ -6934,14 +8345,14 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pud_pgprot(pud), - pud_pfn(pud), PUD_MASK, pud_write(pud), + PFN_PHYS(pud_pfn(pud)), pud_mask_mm(mm), pud_write(pud), pud_special(pud)); if (r) spin_unlock(lock); return r; } - pmdp = pmd_offset(pudp, address); + pmdp = pmd_offset_mm(mm, pudp, address); pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) goto out; @@ -6957,7 +8368,7 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto retry; } r = pfnmap_args_setup(args, lock, NULL, pmd_pgprot(pmd), - pmd_pfn(pmd), PMD_MASK, pmd_write(pmd), + PFN_PHYS(pmd_pfn(pmd)), pmd_mask_mm(mm), pmd_write(pmd), pmd_special(pmd)); if (r) spin_unlock(lock); @@ -6970,8 +8381,8 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) pte = ptep_get(ptep); if (!pte_present(pte)) goto unlock; - r = pfnmap_args_setup(args, lock, ptep, pte_pgprot(pte), - pte_pfn(pte), PAGE_MASK, pte_write(pte), + r = pfnmap_args_setup(args, lock, ptep, pte_pgprot_mm(mm, pte), + pte_phys_mm(mm, pte), mm_page_mask(mm), pte_write(pte), pte_special(pte)); if (r) goto unlock; @@ -7018,20 +8429,30 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, resource_size_t phys_addr; pgprot_t prot = __pgprot(0); void __iomem *maddr; - int offset = offset_in_page(addr); + int offset; int ret = -EINVAL; bool writable; struct follow_pfnmap_args args = { .vma = vma, .address = addr, - .write = !!(write & FOLL_WRITE) + .write = !!(write & FOLL_WRITE), + .allow_subpage = true }; + if (len <= 0) + return 0; + if (addr < vma->vm_start || addr >= vma->vm_end) + return -EINVAL; + /* Each following leaf needs its own translation and permission check. */ + len = min_t(unsigned long, len, vma->vm_end - addr); + len = min_t(unsigned long, len, mm_page_size(vma->vm_mm) - + (addr & ~mm_page_mask(vma->vm_mm))); retry: if (follow_pfnmap_start(&args)) return -EINVAL; prot = args.pgprot; phys_addr = (resource_size_t)args.pfn << PAGE_SHIFT; + offset = args.offset; writable = args.writable; follow_pfnmap_end(&args); @@ -7043,7 +8464,8 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, goto out_unmap; if ((pgprot_val(prot) != pgprot_val(args.pgprot)) || - (phys_addr != (args.pfn << PAGE_SHIFT)) || + (phys_addr != ((resource_size_t)args.pfn << PAGE_SHIFT)) || + (offset != args.offset) || (writable != args.writable)) { follow_pfnmap_end(&args); iounmap(maddr); @@ -7089,8 +8511,19 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *maddr; struct folio *folio; struct vm_area_struct *vma = NULL; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + int got = get_user_fragment_vma_remote(mm, addr, len, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { /* We might need to expand the stack to access it */ @@ -7124,6 +8557,12 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, offset = addr & (PAGE_SIZE-1); if (bytes > PAGE_SIZE-offset) bytes = PAGE_SIZE-offset; +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + bytes = fragment.length; + } +#endif maddr = kmap_local_folio(folio, folio_page_idx(folio, page) * PAGE_SIZE); if (write) { @@ -7134,7 +8573,13 @@ static int __access_remote_vm(struct mm_struct *mm, unsigned long addr, copy_from_user_page(vma, page, addr, buf, maddr + offset, bytes); } - folio_release_kmap(folio, maddr); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + kunmap_local(maddr); + release_user_fragments(&fragment, 1, false); + } else +#endif + folio_release_kmap(folio, maddr); } len -= bytes; buf += bytes; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 4e4421b22b59f7..3b97c71c9d3cf5 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -78,6 +78,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -688,6 +689,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, struct folio *folio; struct queue_pages *qp = walk->private; unsigned long flags = qp->flags; + unsigned int page_shift = mm_page_shift(walk->mm); + unsigned long page_size = 1UL << page_shift; pte_t *pte, *mapped_pte; pte_t ptent; spinlock_t *ptl; @@ -705,8 +708,8 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; pte += nr, addr += nr * PAGE_SIZE) { - max_nr = (end - addr) >> PAGE_SHIFT; + for (; addr != end; pte += nr, addr += nr * page_size) { + max_nr = (end - addr) >> page_shift; nr = 1; ptent = ptep_get(pte); if (pte_none(ptent)) @@ -721,7 +724,11 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, folio = vm_normal_folio(vma, addr, ptent); if (!folio || folio_is_zone_device(folio)) continue; - if (folio_test_large(folio) && max_nr != 1) +#ifdef CONFIG_MM_SUBPAGE + if (page_shift < PAGE_SHIFT) + nr = folio_subpage_pte_batch(vma, folio, pte, ptent, max_nr, 0); +#endif + if (page_shift >= PAGE_SHIFT && folio_test_large(folio) && max_nr != 1) nr = folio_pte_batch(folio, pte, ptent, max_nr); /* * vm_normal_folio() filters out zero pages, but there might @@ -731,9 +738,10 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, continue; if (!queue_folio_required(folio, qp)) continue; - if (folio_test_large(folio)) { + if (folio_test_large(folio) || page_shift < PAGE_SHIFT) { /* - * A large folio can only be isolated from LRU once, + * A native folio backing smaller leaves, like a large + * folio, can only be isolated from LRU once, * but may be mapped by many PTEs (and Copy-On-Write may * intersperse PTEs of other, order 0, folios). This is * a common case, so don't mistake it for failure (but @@ -755,7 +763,7 @@ static int queue_folios_pte_range(pmd_t *pmd, unsigned long addr, if (!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) || !vma_migratable(vma) || !migrate_folio_add(folio, qp->pagelist, flags)) { - qp->nr_failed += nr; + qp->nr_failed += page_shift < PAGE_SHIFT ? 1 : nr; if (strictly_unmovable(flags)) break; } @@ -824,6 +832,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * @folio: The folio whose mapping considered for being made NUMA hintable * @vma: The VMA that the folio belongs to. * @is_private_single_threaded: Is this a single-threaded private VMA or not + * @shared: Is the mapped memory potentially shared between address spaces * * This function checks to see if the folio actually indicates that * we need to make the mapping one which causes a NUMA hinting fault, @@ -833,7 +842,7 @@ static int queue_folios_hugetlb(pte_t *pte, unsigned long hmask, * Return: True if the mapping of the folio needs to be changed, false otherwise. */ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, - bool is_private_single_threaded) + bool is_private_single_threaded, bool shared) { int nid; @@ -841,7 +850,7 @@ bool folio_can_map_prot_numa(struct folio *folio, struct vm_area_struct *vma, return false; /* Also skip shared copy-on-write folios */ - if (is_cow_mapping(vma->vm_flags) && folio_maybe_mapped_shared(folio)) + if (is_cow_mapping(vma->vm_flags) && shared) return false; /* Folios are pinned and can't be migrated */ @@ -1050,6 +1059,12 @@ static int mbind_range(struct vma_iterator *vmi, struct vm_area_struct *vma, vmstart = vma->vm_start; } + /* Shared policy is indexed by native page-cache allocation units. */ + if (vma_is_shmem(vma) && + (vma_page_offset_at(vma, vmstart).offset || + ((vmend - vmstart) & ~PAGE_MASK))) + return -EINVAL; + if (mpol_equal(vma->vm_policy, new_pol)) { *prev = vma; return 0; @@ -1135,6 +1150,18 @@ static int lookup_node(struct mm_struct *mm, unsigned long addr) struct page *p = NULL; int ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_shift(mm) != PAGE_SHIFT) { + struct user_page_fragment fragment; + + ret = get_user_fragments_remote(mm, addr, 1, 0, &fragment, 1); + if (ret > 0) { + ret = folio_nid(fragment.folio); + release_user_fragments(&fragment, 1, false); + } + return ret; + } +#endif ret = get_user_pages_fast(addr & PAGE_MASK, 1, 0, &p); if (ret > 0) { ret = page_to_nid(p); @@ -1502,13 +1529,13 @@ static long do_mbind(unsigned long start, unsigned long len, if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE)) return -EPERM; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; if (mode == MPOL_DEFAULT) flags &= ~MPOL_MF_STRICT; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -1768,7 +1795,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le VMA_ITERATOR(vmi, mm, start); start = untagged_addr(start); - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(mm)) return -EINVAL; /* * flags is used for future extension if any. @@ -1783,7 +1810,7 @@ SYSCALL_DEFINE4(set_mempolicy_home_node, unsigned long, start, unsigned long, le if (home_node >= MAX_NUMNODES || !node_online(home_node)) return -EINVAL; - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); end = start + len; if (end < start) @@ -2049,7 +2076,7 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, if (pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) { *ilx += vma->vm_pgoff >> order; - *ilx += (addr - vma->vm_start) >> (PAGE_SHIFT + order); + *ilx += (vma_page_offset_at(vma, addr).index - vma->vm_pgoff) >> order; } return pol; } @@ -3254,6 +3281,10 @@ int mpol_set_shared_policy(struct shared_policy *sp, struct sp_node *new = NULL; unsigned long sz = vma_pages(vma); + /* Never truncate a partial cache-page policy update to zero or a neighbour. */ + if (vma_subpage_offset(vma) || ((vma->vm_end - vma->vm_start) & ~PAGE_MASK)) + return -EINVAL; + if (pol) { new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, pol); if (!new) diff --git a/mm/migrate.c b/mm/migrate.c index 8a64291ab5b44c..795143e1b4cc44 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -14,6 +14,8 @@ */ #include +#include +#include #include #include #include @@ -340,14 +342,50 @@ struct rmap_walk_arg { bool map_unused_to_zeropage; }; +#ifdef CONFIG_MM_SUBPAGE +static void remove_migration_subpage(struct folio *folio, + struct page_vma_mapped_walk *pvmw, pte_t old) +{ + struct vm_area_struct *vma = pvmw->vma; + softleaf_t entry = softleaf_from_pte(old); + struct mm_subpage *slot; + pte_t pte; + int ret; + + slot = mm_subpage_migrate_restore(folio, pte_swp_subpage_offset(old)); + pte = phys_pte_mm(vma->vm_mm, mm_subpage_phys(slot), READ_ONCE(vma->vm_page_prot)); + if (!softleaf_is_migration_young(entry)) + pte = pte_mkold(pte); + if (folio_test_dirty(folio) && softleaf_is_migration_dirty(entry)) + pte = pte_mkdirty(pte); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + else + pte = pte_clear_soft_dirty(pte); + if (softleaf_is_migration_write(entry)) + pte = pte_mkwrite(pte, vma); + else if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + if (!softleaf_is_migration_read(entry)) + mm_subpage_set_exclusive(slot); + ret = mm_subpage_add_anon_rmap(slot, vma, pvmw->address); + VM_BUG_ON_FOLIO(ret, folio); + set_pte_at(vma->vm_mm, pvmw->address, pvmw->pte, pte); + if (READ_ONCE(vma->vm_flags) & VM_LOCKED) + mlock_drain_local(); + trace_remove_migration_pte(pvmw->address, pte_val(pte), 0); + update_mmu_cache(vma, pvmw->address, pvmw->pte); +} +#endif + /* * Restore a potential migration pte to a working pte entry */ static bool remove_migration_pte(struct folio *folio, - struct vm_area_struct *vma, unsigned long addr, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { struct rmap_walk_arg *rmap_walk_arg = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, rmap_walk_arg->folio, vma, addr, PVMW_SYNC | PVMW_MIGRATION); + DEFINE_FOLIO_RMAP_WALK(pvmw, rmap_walk_arg->folio, vma, rmap, PVMW_SYNC | PVMW_MIGRATION); while (page_vma_mapped_walk(&pvmw)) { rmap_t rmap_flags = RMAP_NONE; @@ -372,13 +410,29 @@ static bool remove_migration_pte(struct folio *folio, } #endif old_pte = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE && folio_test_anon(folio)) { + remove_migration_subpage(folio, &pvmw, old_pte); + continue; + } +#endif if (rmap_walk_arg->map_unused_to_zeropage && try_to_map_unused_to_zeropage(&pvmw, folio, old_pte, idx)) continue; - folio_get(folio); + /* Match the native-page references dropped by try_to_migrate_one(). */ + if (!folio_test_hugetlb(folio)) + folio_ref_add(folio, mm_pte_native_pages(vma->vm_mm)); + else + folio_get(folio); pte = mk_pte(new, READ_ONCE(vma->vm_page_prot)); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + pte = phys_pte_mm(vma->vm_mm, page_to_phys(new) + + pte_swp_subpage_offset(old_pte), READ_ONCE(vma->vm_page_prot)); +#endif + entry = softleaf_from_pte(old_pte); if (!softleaf_is_migration_young(entry)) pte = pte_mkold(pte); @@ -431,8 +485,10 @@ static bool remove_migration_pte(struct folio *folio, if (folio_test_anon(folio)) folio_add_anon_rmap_pte(folio, new, vma, pvmw.address, rmap_flags); - else + else { + mm_subpage_file_map_add(vma, pte_phys_mm(vma->vm_mm, pte)); folio_add_file_rmap_pte(folio, new, vma); + } set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (READ_ONCE(vma->vm_flags) & VM_LOCKED) @@ -640,10 +696,12 @@ static int __folio_migrate_mapping(struct address_space *mapping, folio_set_dirty(newfolio); } - if (folio_test_swapcache(folio)) + if (folio_test_swapcache(folio)) { __swap_cache_replace_folio(ci, folio, newfolio); - else + } else { + shmem_uffd_migrate(newfolio, folio); xas_store(&xas, newfolio); + } /* * Drop cache reference from old folio by unfreezing @@ -1170,8 +1228,10 @@ static void migrate_folio_undo_src(struct folio *src, /* Drop an anon_vma reference if we took one */ if (anon_vma) put_anon_vma(anon_vma); - if (locked) + if (locked) { + mm_subpage_migrate_finish(src); folio_unlock(src); + } if (ret) list_move_tail(&src->lru, ret); } @@ -1180,8 +1240,10 @@ static void migrate_folio_undo_src(struct folio *src, static void migrate_folio_undo_dst(struct folio *dst, bool locked, free_folio_t put_new_folio, unsigned long private) { - if (locked) + if (locked) { + mm_subpage_migrate_finish(dst); folio_unlock(dst); + } if (put_new_folio) put_new_folio(dst, private); else @@ -1306,6 +1368,11 @@ static int migrate_folio_unmap(new_folio_t get_new_folio, return 0; } + rc = mm_subpage_migrate_prepare(src, dst); + if (rc) + goto out; + rc = -EAGAIN; + /* * Corner case handling: * 1. When a new swap-cache page is read into, it is added to the LRU @@ -1411,6 +1478,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, remove_migration_ptes(src, dst, 0); out_unlock_both: + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); folio_unlock(dst); folio_set_owner_migrate_reason(dst, reason); /* diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 17074c2573b94c..bae35a327cf1bb 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -408,7 +408,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, if (anon_exclusive) { pte = ptep_clear_flush(vma, addr, ptep); - if (folio_try_share_anon_rmap_pte(folio, page)) { + if (folio_try_share_anon_rmap_pte(folio, page, vma)) { set_pte_at(mm, addr, ptep, pte); if (fault_folio != folio) folio_unlock(folio); diff --git a/mm/mincore.c b/mm/mincore.c index c8757c5085bf90..66458b6bb20915 100644 --- a/mm/mincore.c +++ b/mm/mincore.c @@ -48,7 +48,7 @@ static int mincore_hugetlb(pte_t *pte, unsigned long hmask, unsigned long addr, present = 1; } - for (; addr != end; vec++, addr += PAGE_SIZE) + for (; addr != end; vec++, addr += mm_page_size(walk->mm)) *vec = present; walk->private = vec; spin_unlock(ptl); @@ -135,15 +135,22 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t index) static int __mincore_unmapped_range(unsigned long addr, unsigned long end, struct vm_area_struct *vma, unsigned char *vec) { - unsigned long nr = (end - addr) >> PAGE_SHIFT; + unsigned long nr = (end - addr) >> mm_page_shift(vma->vm_mm); int i; if (vma->vm_file) { - pgoff_t pgoff; + unsigned int shift = mm_page_shift(vma->vm_mm); - pgoff = linear_page_index(vma, addr); - for (i = 0; i < nr; i++, pgoff++) - vec[i] = mincore_page(vma->vm_file->f_mapping, pgoff); + /* Cache indices are native; several user leaves can share one. */ + for (i = 0; i < nr; ) { + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long batch = min(nr - i, (PAGE_SIZE - pos.offset) >> shift); + unsigned char present = mincore_page(vma->vm_file->f_mapping, pos.index); + + memset(vec + i, present, batch); + i += batch; + addr += batch << shift; + } } else { for (i = 0; i < nr; i++) vec[i] = 0; @@ -167,7 +174,7 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct vm_area_struct *vma = walk->vma; pte_t *ptep; unsigned char *vec = walk->private; - int nr = (end - addr) >> PAGE_SHIFT; + int nr = (end - addr) >> mm_page_shift(walk->mm); int step, i; ptl = pmd_trans_huge_lock(pmd, vma); @@ -182,19 +189,19 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, walk->action = ACTION_AGAIN; return 0; } - for (; addr != end; ptep += step, addr += step * PAGE_SIZE) { + for (; addr != end; ptep += step, addr += step * mm_page_size(walk->mm)) { pte_t pte = ptep_get(ptep); step = 1; /* We need to do cache lookup too for markers */ if (pte_none(pte) || pte_is_marker(pte)) - __mincore_unmapped_range(addr, addr + PAGE_SIZE, + __mincore_unmapped_range(addr, addr + mm_page_size(walk->mm), vma, vec); else if (pte_present(pte)) { unsigned int batch = pte_batch_hint(ptep, pte); if (batch > 1) { - unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + unsigned int max_nr = (end - addr) >> mm_page_shift(walk->mm); step = min_t(unsigned int, batch, max_nr); } @@ -252,16 +259,16 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v vma = vma_lookup(current->mm, addr); if (!vma) return -ENOMEM; - end = min(vma->vm_end, addr + (pages << PAGE_SHIFT)); + end = min(vma->vm_end, addr + (pages << mm_page_shift(vma->vm_mm))); if (!can_do_mincore(vma)) { - unsigned long pages = DIV_ROUND_UP(end - addr, PAGE_SIZE); + unsigned long pages = DIV_ROUND_UP(end - addr, mm_page_size(vma->vm_mm)); memset(vec, 1, pages); return pages; } err = walk_page_range(vma->vm_mm, addr, end, &mincore_walk_ops, vec); if (err < 0) return err; - return (end - addr) >> PAGE_SHIFT; + return (end - addr) >> mm_page_shift(vma->vm_mm); } /* @@ -281,7 +288,7 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v * return values: * zero - success * -EFAULT - vec points to an illegal address - * -EINVAL - addr is not a multiple of PAGE_SIZE + * -EINVAL - addr is not aligned to the process page size * -ENOMEM - Addresses in the range [addr, addr + len] are * invalid for the address space of this process, or * specify one or more pages which are not currently @@ -298,7 +305,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, start = untagged_addr(start); /* Check the start address: needs to be page-aligned.. */ - if (unlikely(start & ~PAGE_MASK)) + if (unlikely(start & ~mm_page_mask(current->mm))) return -EINVAL; /* ..and we need to be passed a valid user-space range */ @@ -306,8 +313,8 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, return -ENOMEM; /* This also avoids any overflows on PAGE_ALIGN */ - pages = len >> PAGE_SHIFT; - pages += (offset_in_page(len)) != 0; + pages = len >> mm_page_shift(current->mm); + pages += (len & ~mm_page_mask(current->mm)) != 0; if (!access_ok(vec, pages)) return -EFAULT; @@ -334,7 +341,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len, } pages -= retval; vec += retval; - start += retval << PAGE_SHIFT; + start += retval << mm_page_shift(current->mm); retval = 0; } free_page((unsigned long) tmp); diff --git a/mm/mlock.c b/mm/mlock.c index 8c227fefa2df0e..a6b67d6b20cac9 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -304,15 +304,19 @@ void munlock_folio(struct folio *folio) local_unlock(&mlock_fbatch.lock); } -static inline unsigned int folio_mlock_step(struct folio *folio, +static inline unsigned int folio_mlock_step(struct vm_area_struct *vma, struct folio *folio, pte_t *pte, unsigned long addr, unsigned long end) { - unsigned int count = (end - addr) >> PAGE_SHIFT; + unsigned int count = (end - addr) >> mm_page_shift(vma->vm_mm); pte_t ptent = ptep_get(pte); if (!folio_test_large(folio)) return 1; + #ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch(vma, folio, pte, ptent, count, 0); +#endif return folio_pte_batch(folio, pte, ptent, count); } @@ -341,7 +345,7 @@ static inline bool allow_mlock_munlock(struct folio *folio, return false; /* folio is not fully mapped, skip mlock */ - if (step != folio_nr_pages(folio)) + if (step != (folio_size(folio) >> mm_page_shift(vma->vm_mm))) return false; return true; @@ -381,7 +385,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, return 0; } - for (pte = start_pte; addr != end; pte++, addr += PAGE_SIZE) { + for (pte = start_pte; addr != end; pte++, addr += mm_page_size(vma->vm_mm)) { ptent = ptep_get(pte); if (!pte_present(ptent)) continue; @@ -389,7 +393,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, if (!folio || folio_is_zone_device(folio)) continue; - step = folio_mlock_step(folio, pte, addr, end); + step = folio_mlock_step(vma, folio, pte, addr, end); if (!allow_mlock_munlock(folio, vma, start, end, step)) goto next_entry; @@ -400,7 +404,7 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, next_entry: pte += step - 1; - addr += (step - 1) << PAGE_SHIFT; + addr += (step - 1) << mm_page_shift(vma->vm_mm); } pte_unmap(start_pte); out: @@ -471,7 +475,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); const vma_flags_t old_vma_flags = vma->flags; struct mm_struct *mm = vma->vm_mm; - int nr_pages; + long nr_pages; int ret = 0; if (vma_flags_same_pair(&old_vma_flags, &new_vma_flags) || @@ -492,7 +496,7 @@ static int mlock_fixup(struct vma_iterator *vmi, struct vm_area_struct *vma, /* * Keep track of amount of locked VM. */ - nr_pages = (end - start) >> PAGE_SHIFT; + nr_pages = (end - start) >> MM_ACCOUNT_SHIFT; if (!vma_flags_test(&new_vma_flags, VMA_LOCKED_BIT)) nr_pages = -nr_pages; else if (vma_flags_test(&old_vma_flags, VMA_LOCKED_BIT)) @@ -524,8 +528,8 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, struct vm_area_struct *vma, *prev; VMA_ITERATOR(vmi, current->mm, start); - VM_BUG_ON(offset_in_page(start)); - VM_BUG_ON(len != PAGE_ALIGN(len)); + VM_BUG_ON((start & ~mm_page_mask(current->mm))); + VM_BUG_ON(len != mm_page_align(current->mm, len)); end = start + len; if (end < start) return -EINVAL; @@ -572,7 +576,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len, * vma pages, as return value. * Note deferred memory locking case(mlock2(,,MLOCK_ONFAULT) * is also counted. - * Return value: previously mlocked page counts + * Return value: previously mlocked MM_ACCOUNT_SHIFT units */ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, unsigned long start, size_t len) @@ -600,7 +604,7 @@ static unsigned long count_mm_mlocked_page_nr(struct mm_struct *mm, } } - return count >> PAGE_SHIFT; + return count >> MM_ACCOUNT_SHIFT; } /* @@ -626,12 +630,12 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla if (!can_do_mlock()) return -EPERM; - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; - locked = len >> PAGE_SHIFT; + lock_limit >>= MM_ACCOUNT_SHIFT; + locked = len >> MM_ACCOUNT_SHIFT; if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -686,8 +690,8 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) start = untagged_addr(start); - len = PAGE_ALIGN(len + (offset_in_page(start))); - start &= PAGE_MASK; + len = mm_page_align(current->mm, len + (start & ~mm_page_mask(current->mm))); + start &= mm_page_mask(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; @@ -761,7 +765,7 @@ SYSCALL_DEFINE1(mlockall, int, flags) return -EPERM; lock_limit = rlimit(RLIMIT_MEMLOCK); - lock_limit >>= PAGE_SHIFT; + lock_limit >>= mm_page_shift(current->mm); if (mmap_write_lock_killable(current->mm)) return -EINTR; diff --git a/mm/mm_init.c b/mm/mm_init.c index c7e498fc1c4c01..8d14ce83be36b9 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -57,7 +57,8 @@ unsigned long zero_page_pfn __ro_after_init; EXPORT_SYMBOL(zero_page_pfn); #ifndef __HAVE_COLOR_ZERO_PAGE -uint8_t empty_zero_page[PAGE_SIZE] __page_aligned_bss; +uint8_t empty_zero_page[ARCH_ZERO_PAGE_SIZE] __page_aligned_bss + __aligned(ARCH_ZERO_PAGE_SIZE); EXPORT_SYMBOL(empty_zero_page); struct page *__zero_page __ro_after_init; diff --git a/mm/mmap.c b/mm/mmap.c index 2311ae7c2ff45c..097161e3e61efd 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -151,8 +151,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) mm->end_data, mm->start_data)) goto out; - newbrk = PAGE_ALIGN(brk); - oldbrk = PAGE_ALIGN(mm->brk); + newbrk = mm_page_align(mm, brk); + oldbrk = mm_page_align(mm, mm->brk); if (oldbrk == newbrk) { mm->brk = brk; goto success; @@ -186,8 +186,8 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * expansion area */ vma_iter_init(&vmi, mm, oldbrk); - next = vma_find(&vmi, newbrk + PAGE_SIZE + stack_guard_gap); - if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) + next = vma_find(&vmi, newbrk + mm_page_size(mm) + stack_guard_gap); + if (next && newbrk + mm_page_size(mm) > vm_start_gap(next)) goto out; brkvma = vma_prev_limit(&vmi, mm->start_brk); @@ -218,12 +218,12 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) * If a hint addr is less than mmap_min_addr change hint to be as * low as possible but still greater than mmap_min_addr */ -static inline unsigned long round_hint_to_min(unsigned long hint) +static inline unsigned long round_hint_to_min(struct mm_struct *mm, unsigned long hint) { - hint &= PAGE_MASK; + hint &= mm_page_mask(mm); if (((void *)hint != NULL) && (hint < mmap_min_addr)) - return PAGE_ALIGN(mmap_min_addr); + return mm_page_align(mm, mmap_min_addr); return hint; } @@ -235,11 +235,11 @@ bool mlock_future_ok(const struct mm_struct *mm, bool is_vma_locked, if (!is_vma_locked || capable(CAP_IPC_LOCK)) return true; - locked_pages = bytes >> PAGE_SHIFT; + locked_pages = bytes >> MM_ACCOUNT_SHIFT; locked_pages += mm->locked_vm; limit_pages = rlimit(RLIMIT_MEMLOCK); - limit_pages >>= PAGE_SHIFT; + limit_pages >>= MM_ACCOUNT_SHIFT; return locked_pages <= limit_pages; } @@ -264,24 +264,25 @@ static inline u64 file_mmap_size_max(struct file *file, struct inode *inode) } static inline bool file_mmap_ok(struct file *file, struct inode *inode, - unsigned long pgoff, unsigned long len) + struct vm_page_offset pos, unsigned long len) { u64 maxsize = file_mmap_size_max(file, inode); if (maxsize && len > maxsize) return false; maxsize -= len; - if (pgoff > maxsize >> PAGE_SHIFT) + if (pos.index > maxsize >> PAGE_SHIFT || + (pos.index == maxsize >> PAGE_SHIFT && pos.offset > (maxsize & ~PAGE_MASK))) return false; return true; } /** - * do_mmap() - Perform a userland memory mapping into the current process + * do_mmap_internal() - Perform a userland memory mapping into the current process * address space of length @len with protection bits @prot, mmap flags @flags * (from which VMA flags will be inferred), and any additional VMA flags to * apply @vm_flags. If this is a file-backed mapping then the file is specified - * in @file and page offset into the file via @pgoff. + * in @file and its byte position via @pos. * * This function does not perform security checks on the file and assumes, if * @uf is non-NULL, the caller has provided a list head to track unmap events @@ -321,7 +322,7 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * @flags: Flags specifying how the mapping should be performed, see mmap (2) * for details. * @vm_flags: VMA flags which should be set by default, or 0 otherwise. - * @pgoff: Page offset into the @file if file-backed, should be 0 otherwise. + * @pos: Native cache index and byte offset; ignored for anonymous mappings. * @populate: A pointer to a value which will be set to 0 if no population of * the range is required, or the number of bytes to populate if it is. Must be * non-NULL. See mmap (2) for details as to under what circumstances population @@ -333,20 +334,22 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, * Returns: Either an error, or the address at which the requested mapping has * been performed. */ -unsigned long do_mmap(struct file *file, unsigned long addr, +static unsigned long do_mmap_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, vm_flags_t vm_flags, - unsigned long pgoff, unsigned long *populate, + struct vm_page_offset pos, unsigned long *populate, struct list_head *uf) { struct mm_struct *mm = current->mm; + unsigned long pgoff = pos.index; int pkey = 0; *populate = 0; mmap_assert_write_locked(mm); - if (!len) + if (!len || pos.offset >= PAGE_SIZE || + !IS_ALIGNED(pos.offset, mm_page_size(mm))) return -EINVAL; /* @@ -364,15 +367,15 @@ unsigned long do_mmap(struct file *file, unsigned long addr, flags |= MAP_FIXED; if (!(flags & MAP_FIXED)) - addr = round_hint_to_min(addr); + addr = round_hint_to_min(mm, addr); /* Careful about overflows.. */ - len = PAGE_ALIGN(len); + len = mm_page_align(mm, len); if (!len) return -ENOMEM; /* offset overflow? */ - if ((pgoff + (len >> PAGE_SHIFT)) < pgoff) + if (vm_page_offset_add(pos, len).index < pgoff) return -EOVERFLOW; /* Too many mappings? */ @@ -426,7 +429,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long flags_mask; int err; - if (!file_mmap_ok(file, inode, pgoff, len)) + if (!file_mmap_ok(file, inode, pos, len)) return -EOVERFLOW; flags_mask = LEGACY_MAP_MASK; @@ -500,6 +503,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Ignore pgoff. */ pgoff = 0; + pos.offset = 0; vm_flags |= VM_SHARED | VM_MAYSHARE; break; case MAP_DROPPABLE: @@ -537,6 +541,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, * Set pgoff according to addr for anon_vma. */ pgoff = addr >> PAGE_SHIFT; + pos.offset = addr & ~PAGE_MASK; break; default: return -EINVAL; @@ -557,7 +562,11 @@ unsigned long do_mmap(struct file *file, unsigned long addr, vm_flags |= VM_NORESERVE; } - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = mmap_region(file, addr, len, vm_flags, + (struct vm_page_offset) { + .index = pgoff, + .offset = pos.offset, + }, uf); if (!IS_ERR_VALUE(addr) && ((vm_flags & VM_LOCKED) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) @@ -565,9 +574,28 @@ unsigned long do_mmap(struct file *file, unsigned long addr, return addr; } -unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, +unsigned long do_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, + (struct vm_page_offset) { pgoff, 0 }, populate, uf); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long do_mmap_offset(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, struct vm_page_offset pos, unsigned long *populate, + struct list_head *uf) +{ + return do_mmap_internal(file, addr, len, prot, flags, vm_flags, pos, populate, uf); +} +#endif + +static unsigned long ksys_mmap_offset(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long pgoff) + unsigned long fd, unsigned long pgoff, unsigned int suboffset) { struct file *file = NULL; unsigned long retval; @@ -603,13 +631,34 @@ unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, return PTR_ERR(file); } - retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); + if (suboffset) + retval = vm_mmap(file, addr, len, prot, flags, + (pgoff << PAGE_SHIFT) | suboffset); + else + retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); out_fput: if (file) fput(file); return retval; } +unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) +{ + return ksys_mmap_offset(addr, len, prot, flags, fd, pgoff, 0); +} + +#ifdef CONFIG_MM_SUBPAGE +unsigned long ksys_mmap_user(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, unsigned long fd, unsigned long offset) +{ + if (!IS_ALIGNED(offset, mm_page_size(current->mm))) + return -EINVAL; + return ksys_mmap_offset(addr, len, prot, flags, fd, + offset >> PAGE_SHIFT, offset & ~PAGE_MASK); +} +#endif + SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, pgoff) @@ -703,7 +752,7 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, return addr; if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -754,7 +803,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, /* requesting a specific address */ if (addr) { - addr = PAGE_ALIGN(addr); + addr = mm_page_align(mm, addr); vma = find_vma_prev(mm, addr, &prev); if (mmap_end - len >= addr && addr >= mmap_min_addr && (!vma || addr + len <= vm_start_gap(vma)) && @@ -764,7 +813,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; - info.low_limit = PAGE_SIZE; + info.low_limit = mm_page_size(mm); info.high_limit = arch_get_mmap_base(addr, mm->mmap_base); info.start_gap = stack_guard_placement(vm_flags); if (filp && is_file_hugepages(filp)) @@ -777,7 +826,7 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, * can happen with large stack limits and large mmap() * allocations. */ - if (offset_in_page(addr)) { + if (addr & ~mm_page_mask(mm)) { VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = TASK_UNMAPPED_BASE; @@ -857,7 +906,7 @@ __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, if (addr > TASK_SIZE - len) return -ENOMEM; - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; error = security_mmap_addr(addr); @@ -962,7 +1011,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon { struct vm_area_struct *vma, *prev; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma_prev(mm, addr, &prev); if (vma && (vma->vm_start <= addr)) return vma; @@ -985,7 +1034,7 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned lon struct vm_area_struct *vma; unsigned long start; - addr &= PAGE_MASK; + addr &= mm_page_mask(mm); vma = find_vma(mm, addr); if (!vma) return NULL; @@ -1214,7 +1263,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec) LIST_HEAD(uf); VMA_ITERATOR(vmi, mm, addr); - len = PAGE_ALIGN(request); + len = mm_page_align(mm, request); if (len < request) return -ENOMEM; if (!len) @@ -1257,7 +1306,7 @@ unsigned long tear_down_vmas(struct mm_struct *mm, struct vma_iterator *vmi, vma_iter_set(vmi, vma->vm_end); do { if (vma->vm_flags & VM_ACCOUNT) - nr_accounted += vma_pages(vma); + nr_accounted += vma_user_pages(vma); vma_mark_detached(vma); remove_vma(vma); count++; @@ -1322,8 +1371,8 @@ void exit_mmap(struct mm_struct *mm) destroy: __mt_destroy(&mm->mm_mt); trace_exit_mmap(mm); + mm_unacct_memory(mm, nr_accounted); mmap_write_unlock(mm); - vm_unacct_memory(nr_accounted); } /* @@ -1333,19 +1382,19 @@ void exit_mmap(struct mm_struct *mm) bool may_expand_vm(struct mm_struct *mm, const vma_flags_t *vma_flags, unsigned long npages) { - if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT) + if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> mm_page_shift(mm)) return false; if (is_data_mapping_vma_flags(vma_flags) && - mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) { + mm->data_vm + npages > rlimit(RLIMIT_DATA) >> mm_page_shift(mm)) { /* Workaround for Valgrind */ if (rlimit(RLIMIT_DATA) == 0 && - mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT) + mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> mm_page_shift(mm)) return true; pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n", current->comm, current->pid, - (mm->data_vm + npages) << PAGE_SHIFT, + (mm->data_vm + npages) << mm_page_shift(mm), rlimit(RLIMIT_DATA), ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data"); @@ -1474,7 +1523,7 @@ static struct vm_area_struct *__install_special_mapping( if (ret) goto out; - vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT); + vm_stat_account(mm, vma->vm_flags, len >> mm_page_shift(mm)); perf_event_mmap(vma); @@ -1771,14 +1820,14 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) if (retval) goto loop_out; - vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); + vm_stat_account(mm, mpnt->vm_flags, -vma_user_pages(mpnt)); continue; } charge = 0; if (mpnt->vm_flags & VM_ACCOUNT) { - unsigned long len = vma_pages(mpnt); + unsigned long len = vma_user_pages(mpnt); - if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ + if (mm_account_memory(mm, oldmm, len)) /* sic */ goto fail_nomem; charge = len; } @@ -1890,7 +1939,7 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) flush_cache_mm(mm); unmap_region(&unmap); charge = tear_down_vmas(mm, &vmi, tmp, end); - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); } __mt_destroy(&mm->mm_mt); /* @@ -1916,6 +1965,6 @@ __latent_entropy int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) vm_area_free(tmp); fail_nomem: retval = -ENOMEM; - vm_unacct_memory(charge); + mm_unacct_memory(mm, charge); goto loop_out; } diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c index 3985d856de7f9b..42419a47a1880d 100644 --- a/mm/mmu_gather.c +++ b/mm/mmu_gather.c @@ -12,6 +12,9 @@ #include #include #include +#include + +#include "user-subpage-internal.h" #include @@ -100,6 +103,49 @@ void tlb_flush_rmaps(struct mmu_gather *tlb, struct vm_area_struct *vma) */ #define MAX_NR_FOLIOS_PER_FREE 512 +#ifdef CONFIG_MM_SUBPAGE +/* Private to gather batches: never pass this token to encoded_page_ptr(). */ +#define TLB_SUBPAGE_TAG BIT(2) + +static void tlb_free_pages_and_subpages(struct encoded_page **pages, unsigned int nr) +{ + unsigned int start = 0, i = 0; + + while (i < nr) { + unsigned long token = (unsigned long)pages[i]; + + if (token & TLB_SUBPAGE_TAG) { + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); + mm_subpage_put((struct mm_subpage *)(token & ~TLB_SUBPAGE_TAG)); + start = ++i; + } else { + /* The count payload is not a pointer and may have bit 2 set. */ + i += encoded_page_flags(pages[i]) & ENCODED_PAGE_BIT_NR_PAGES_NEXT ? 2 : 1; + } + } + if (i != start) + free_pages_and_swap_cache(pages + start, i - start); +} + +bool __tlb_remove_subpage(struct mmu_gather *tlb, struct mm_subpage *subpage) +{ + struct mmu_gather_batch *batch = tlb->active; + + BUILD_BUG_ON(__alignof__(struct page) < 8); + BUILD_BUG_ON(__alignof__(struct mm_subpage) < 8); + BUILD_BUG_ON(TLB_SUBPAGE_TAG & ENCODED_PAGE_BITS); + VM_BUG_ON(!tlb->end); + VM_BUG_ON(mm_page_shift(tlb->mm) != mm_subpage_shift(subpage)); + batch->encoded_pages[batch->nr++] = + (struct encoded_page *)((unsigned long)subpage | TLB_SUBPAGE_TAG); + /* Keep room for a following native page/count pair. */ + return batch->nr >= batch->max - 1 && !tlb_next_batch(tlb); +} +#else +#define tlb_free_pages_and_subpages free_pages_and_swap_cache +#endif + static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) { struct encoded_page **pages = batch->encoded_pages; @@ -135,7 +181,7 @@ static void __tlb_batch_free_encoded_pages(struct mmu_gather_batch *batch) } } - free_pages_and_swap_cache(pages, nr); + tlb_free_pages_and_subpages(pages, nr); pages += nr; batch->nr -= nr; @@ -174,7 +220,15 @@ static bool __tlb_remove_folio_pages_size(struct mmu_gather *tlb, #ifdef CONFIG_MMU_GATHER_PAGE_SIZE VM_WARN_ON(tlb->page_size != page_size); - VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + if (page_size > PAGE_SIZE && page_size == mm_page_size(tlb->mm)) { + /* A coarse base PTE owns one reference per covered native page. */ + unsigned long span = mm_pte_native_pages(tlb->mm); + + VM_WARN_ON_ONCE(!nr_pages || nr_pages % span || + !IS_ALIGNED(page_to_pfn(page), span)); + } else { + VM_WARN_ON_ONCE(nr_pages != 1 && page_size != PAGE_SIZE); + } VM_WARN_ON_ONCE(page_folio(page) != page_folio(page + nr_pages - 1)); #endif @@ -208,7 +262,7 @@ bool __tlb_remove_folio_pages(struct mmu_gather *tlb, struct page *page, unsigned int nr_pages, bool delay_rmap) { return __tlb_remove_folio_pages_size(tlb, page, nr_pages, delay_rmap, - PAGE_SIZE); + mm_page_size(tlb->mm)); } bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size) @@ -352,6 +406,13 @@ static inline void __tlb_remove_table_one(void *table) { struct ptdesc *ptdesc; + /* Preserve opaque architecture tokens through the RCU grace period. */ + if (!tlb_table_is_ptdesc(table)) { + tlb_remove_table_sync_rcu(); + __tlb_remove_table(table); + return; + } + ptdesc = table; call_rcu(&ptdesc->pt_rcu_head, __tlb_remove_table_one_rcu); } diff --git a/mm/mprotect.c b/mm/mprotect.c index 9cbf932b028cf9..0f5ea22bf4c907 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,31 @@ static bool maybe_change_pte_writable(struct vm_area_struct *vma, pte_t pte) return true; } +#ifdef CONFIG_MM_SUBPAGE +static bool subpage_pte_exclusive(struct vm_area_struct *vma, pte_t pte) +{ + struct mm_subpage *slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + bool exclusive; + + if (!slot) + return false; + exclusive = mm_subpage_is_exclusive(slot); + mm_subpage_put(slot); + return exclusive; +} + +static int subpage_exclusive_pte_batch(struct vm_area_struct *vma, pte_t *ptep, + int nr_ptes, bool exclusive) +{ + int nr; + + for (nr = 1; nr < nr_ptes; nr++) + if (subpage_pte_exclusive(vma, ptep_get(ptep + nr)) != exclusive) + break; + return nr; +} +#endif + static bool can_change_private_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte) { @@ -73,6 +99,10 @@ static bool can_change_private_pte_writable(struct vm_area_struct *vma, * any additional checks while holding the PT lock. */ page = vm_normal_page(vma, addr, pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return page && PageAnon(page) && subpage_pte_exclusive(vma, pte); +#endif return page && PageAnon(page) && PageAnonExclusive(page); } @@ -103,13 +133,20 @@ bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, return can_change_shared_pte_writable(vma, pte); } -static int mprotect_folio_pte_batch(struct folio *folio, pte_t *ptep, - pte_t pte, int max_nr_ptes, fpb_t flags) + +static __always_inline int mprotect_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t pte, int max_nr_ptes, + fpb_t flags, unsigned int page_shift) { /* No underlying folio, so cannot batch */ if (!folio) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (page_shift != PAGE_SHIFT) + return folio_subpage_pte_batch(vma, folio, ptep, pte, max_nr_ptes, flags); +#endif + if (!folio_test_large(folio)) return 1; @@ -125,17 +162,17 @@ static __always_inline void prot_commit_flush_ptes(struct vm_area_struct *vma, * Advance the position in the batch by idx; note that if idx > 0, * then the nr_ptes passed here is <= batch size - idx. */ - addr += idx * PAGE_SIZE; + addr += idx * mm_page_size(vma->vm_mm); ptep += idx; - oldpte = pte_advance_pfn(oldpte, idx); - ptent = pte_advance_pfn(ptent, idx); + oldpte = pte_advance_pfn_mm(vma->vm_mm, oldpte, idx); + ptent = pte_advance_pfn_mm(vma->vm_mm, ptent, idx); if (set_write) ptent = pte_mkwrite(ptent, vma); modify_prot_commit_ptes(vma, addr, ptep, oldpte, ptent, nr_ptes); if (pte_needs_flush(oldpte, ptent)) - tlb_flush_pte_range(tlb, addr, nr_ptes * PAGE_SIZE); + tlb_flush_pte_range(tlb, addr, nr_ptes * mm_page_size(vma->vm_mm)); } /* @@ -177,6 +214,25 @@ static __always_inline void commit_anon_folio_batch(struct vm_area_struct *vma, int sub_batch_idx = 0; int len; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + while (nr_ptes) { + expected_anon_exclusive = subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx)); + for (len = 1; len < nr_ptes; len++) + if (subpage_pte_exclusive(vma, + pte_advance_pfn_mm(vma->vm_mm, oldpte, sub_batch_idx + len)) != + expected_anon_exclusive) + break; + prot_commit_flush_ptes(vma, addr, ptep, oldpte, ptent, len, + sub_batch_idx, expected_anon_exclusive, tlb); + sub_batch_idx += len; + nr_ptes -= len; + } + return; + } +#endif + while (nr_ptes) { expected_anon_exclusive = PageAnonExclusive(first_page + sub_batch_idx); len = page_anon_exclusive_sub_batch(sub_batch_idx, nr_ptes, @@ -265,6 +321,11 @@ static long change_softleaf_pte(struct vm_area_struct *vma, newpte = oldpte; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + newpte = pte_swp_set_subpage_offset(newpte, pte_swp_subpage_offset(oldpte)); +#endif + if (uffd_wp) newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) @@ -320,6 +381,9 @@ static long change_pte_range(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, unsigned long cp_flags) { + /* The mm's mapping granule cannot change during the walk. */ + const unsigned int page_shift = mm_page_shift(vma->vm_mm); + const unsigned long page_size = 1UL << page_shift; pte_t *pte, oldpte; spinlock_t *ptl; long pages = 0; @@ -328,7 +392,7 @@ static long change_pte_range(struct mmu_gather *tlb, bool uffd_wp = cp_flags & MM_CP_UFFD_WP; int nr_ptes; - tlb_change_page_size(tlb, PAGE_SIZE); + tlb_change_page_size(tlb, page_size); pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); if (!pte) return -EAGAIN; @@ -343,7 +407,7 @@ static long change_pte_range(struct mmu_gather *tlb, oldpte = ptep_get(pte); if (pte_present(oldpte)) { const fpb_t flags = FPB_RESPECT_SOFT_DIRTY | FPB_RESPECT_WRITE; - int max_nr_ptes = (end - addr) >> PAGE_SHIFT; + int max_nr_ptes = (end - addr) >> page_shift; struct folio *folio = NULL; struct page *page; @@ -359,17 +423,32 @@ static long change_pte_range(struct mmu_gather *tlb, * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ - if (prot_numa && - !folio_can_map_prot_numa(folio, vma, - is_private_single_threaded)) { - - /* determine batch to skip */ - nr_ptes = mprotect_folio_pte_batch(folio, - pte, oldpte, max_nr_ptes, /* flags = */ 0); - continue; + if (prot_numa) { + bool shared = folio && folio_maybe_mapped_shared(folio); + +#ifdef CONFIG_MM_SUBPAGE + /* Multiple private leaves do not imply shared COW data. */ + if (folio && folio_test_anon(folio) && page_shift < PAGE_SHIFT) { + shared = !subpage_pte_exclusive(vma, oldpte); + max_nr_ptes = folio_subpage_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, flags); + /* COW sharing can differ within one physical owner. */ + max_nr_ptes = subpage_exclusive_pte_batch(vma, pte, + max_nr_ptes, !shared); + } +#endif + if (!folio_can_map_prot_numa(folio, vma, + is_private_single_threaded, shared)) { + /* determine batch to skip */ + nr_ptes = mprotect_folio_pte_batch(vma, folio, + pte, oldpte, max_nr_ptes, /* flags = */ 0, + page_shift); + continue; + } } - nr_ptes = mprotect_folio_pte_batch(folio, pte, oldpte, max_nr_ptes, flags); + nr_ptes = mprotect_folio_pte_batch(vma, folio, pte, oldpte, + max_nr_ptes, flags, page_shift); /* * Optimize for the small-folio common case by @@ -408,7 +487,7 @@ static long change_pte_range(struct mmu_gather *tlb, } else { pages += change_softleaf_pte(vma, addr, pte, oldpte, cp_flags); } - } while (pte += nr_ptes, addr += nr_ptes * PAGE_SIZE, addr != end); + } while (pte += nr_ptes, addr += nr_ptes * page_size, addr != end); lazy_mmu_mode_disable(); pte_unmap_unlock(pte - 1, ptl); @@ -487,12 +566,12 @@ static inline long change_pmd_range(struct mmu_gather *tlb, long pages = 0; unsigned long nr_huge_updates = 0; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { long ret; pmd_t _pmd; again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = change_pmd_prepare(vma, pmd, cp_flags); if (ret) { @@ -559,10 +638,10 @@ static inline long change_pud_range(struct mmu_gather *tlb, range.start = 0; - pudp = pud_offset(p4d, addr); + pudp = pud_offset_mm(vma->vm_mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pudp, pmd, addr, cp_flags); if (ret) { pages = ret; @@ -615,13 +694,13 @@ static inline long change_p4d_range(struct mmu_gather *tlb, unsigned long next; long pages = 0, ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); + next = p4d_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, p4d, pud, addr, cp_flags); if (ret) return ret; - if (p4d_none_or_clear_bad(p4d)) + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; pages += change_pud_range(tlb, vma, p4d, addr, next, newprot, cp_flags); @@ -643,7 +722,7 @@ static long change_protection_range(struct mmu_gather *tlb, pgd = pgd_offset(mm, addr); tlb_start_vma(tlb, vma); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); ret = change_prepare(vma, pgd, p4d, addr, cp_flags); if (ret) { pages = ret; @@ -729,7 +808,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, struct mm_struct *mm = vma->vm_mm; const vma_flags_t old_vma_flags = READ_ONCE(vma->flags); vma_flags_t new_vma_flags = legacy_to_vma_flags(newflags); - long nrpages = (end - start) >> PAGE_SHIFT; + long nrpages = (end - start) >> mm_page_shift(mm); unsigned int mm_cp_flags = 0; unsigned long charged = 0; int error; @@ -777,7 +856,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, VMA_ACCOUNT_BIT, VMA_WRITE_BIT, VMA_HUGETLB_BIT, VMA_SHARED_BIT, VMA_NORESERVE_BIT)) { charged = nrpages; - if (security_vm_enough_memory_mm(mm, charged)) + if (mm_account_memory(mm, mm, charged)) return -ENOMEM; vma_flags_set(&new_vma_flags, VMA_ACCOUNT_BIT); } @@ -808,7 +887,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, if (vma_flags_test(&old_vma_flags, VMA_ACCOUNT_BIT) && !vma_flags_test(&new_vma_flags, VMA_ACCOUNT_BIT)) - vm_unacct_memory(nrpages); + mm_unacct_memory(mm, nrpages); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major @@ -826,7 +905,7 @@ mprotect_fixup(struct vma_iterator *vmi, struct mmu_gather *tlb, return 0; fail: - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return error; } @@ -851,11 +930,11 @@ static int do_mprotect_pkey(unsigned long start, size_t len, if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; - if (start & ~PAGE_MASK) + if (start & ~mm_page_mask(current->mm)) return -EINVAL; if (!len) return 0; - len = PAGE_ALIGN(len); + len = mm_page_align(current->mm, len); end = start + len; if (end <= start) return -ENOMEM; diff --git a/mm/mremap.c b/mm/mremap.c index e9c8b1d05832be..97d5ebf9452852 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -82,11 +82,11 @@ static pud_t *get_old_pud(struct mm_struct *mm, unsigned long addr) if (pgd_none_or_clear_bad(pgd)) return NULL; - p4d = p4d_offset(pgd, addr); - if (p4d_none_or_clear_bad(p4d)) + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_or_clear_bad_mm(mm, p4d)) return NULL; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(mm, p4d, addr); if (pud_none_or_clear_bad(pud)) return NULL; @@ -102,7 +102,7 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) if (!pud) return NULL; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(mm, pud, addr); if (pmd_none(*pmd)) return NULL; @@ -183,6 +183,14 @@ static int mremap_folio_pte_batch(struct vm_area_struct *vma, unsigned long addr if (max_nr == 1) return 1; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + folio = vm_normal_folio(vma, addr, pte); + return folio ? folio_subpage_pte_batch(vma, folio, ptep, pte, + max_nr, FPB_RESPECT_WRITE) : 1; + } +#endif + /* Avoid expensive folio lookup if we stand no chance of benefit. */ if (pte_batch_hint(ptep, pte) == 1) return 1; @@ -262,12 +270,12 @@ static int move_ptes(struct pagetable_move_control *pmc, flush_tlb_batched_pending(vma->vm_mm); lazy_mmu_mode_enable(); - for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * PAGE_SIZE, - new_ptep += nr_ptes, new_addr += nr_ptes * PAGE_SIZE) { + for (; old_addr < old_end; old_ptep += nr_ptes, old_addr += nr_ptes * mm_page_size(mm), + new_ptep += nr_ptes, new_addr += nr_ptes * mm_page_size(mm)) { VM_WARN_ON_ONCE(!pte_none(*new_ptep)); nr_ptes = 1; - max_nr_ptes = (old_end - old_addr) >> PAGE_SHIFT; + max_nr_ptes = (old_end - old_addr) >> mm_page_shift(mm); old_pte = ptep_get(old_ptep); if (pte_none(old_pte)) continue; @@ -407,7 +415,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, VM_BUG_ON(!pmd_none(*new_pmd)); pmd_populate(mm, new_pmd, pmd_pgtable(pmd)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pmd_size_mm(mm)); out_unlock: if (new_ptl != old_ptl) spin_unlock(new_ptl); @@ -458,8 +466,8 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, VM_BUG_ON(!pud_none(*new_pud)); - pud_populate(mm, new_pud, pud_pgtable(pud)); - flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE); + pud_populate(mm, new_pud, pud_pgtable_mm(mm, pud)); + flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + pud_size_mm(mm)); if (new_ptl != old_ptl) spin_unlock(new_ptl); spin_unlock(old_ptl); @@ -549,13 +557,13 @@ static __always_inline unsigned long get_extent(enum pgt_entry entry, switch (entry) { case HPAGE_PMD: case NORMAL_PMD: - mask = PMD_MASK; - size = PMD_SIZE; + mask = pmd_mask_mm(pmc->old->vm_mm); + size = pmd_size_mm(pmc->old->vm_mm); break; case HPAGE_PUD: case NORMAL_PUD: - mask = PUD_MASK; - size = PUD_SIZE; + mask = pud_mask_mm(pmc->old->vm_mm); + size = pud_size_mm(pmc->old->vm_mm); break; default: BUILD_BUG(); @@ -811,7 +819,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) * If possible, realign addresses to PMD boundary for faster copy. * Only realign if the mremap copying hits a PMD boundary. */ - try_realign_addr(pmc, PMD_MASK); + try_realign_addr(pmc, pmd_mask_mm(mm)); flush_cache_range(pmc->old, pmc->old_addr, pmc->old_end); mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, mm, @@ -838,7 +846,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) /* We ignore and continue on error? */ continue; } - } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == PUD_SIZE) { + } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PUD) && extent == pud_size_mm(mm)) { if (move_pgt_entry(pmc, NORMAL_PUD, old_pud, new_pud)) continue; } @@ -857,7 +865,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) continue; split_huge_pmd(pmc->old, old_pmd, pmc->old_addr); } else if (IS_ENABLED(CONFIG_HAVE_MOVE_PMD) && - extent == PMD_SIZE) { + extent == pmd_size_mm(mm)) { /* * If the extent is PMD-sized, try to speed the move by * moving at the PMD level if possible. @@ -947,9 +955,7 @@ static unsigned long vrm_set_new_addr(struct vma_remap_struct *vrm) { struct vm_area_struct *vma = vrm->vma; unsigned long map_flags = 0; - /* Page Offset _into_ the VMA. */ - pgoff_t internal_pgoff = (vrm->addr - vma->vm_start) >> PAGE_SHIFT; - pgoff_t pgoff = vma->vm_pgoff + internal_pgoff; + pgoff_t pgoff = vma_page_offset_at(vma, vrm->addr).index; unsigned long new_addr = vrm_implies_new_addr(vrm) ? vrm->new_addr : 0; unsigned long res; @@ -985,13 +991,13 @@ static bool vrm_calc_charge(struct vma_remap_struct *vrm) * the length of the new one. Otherwise it's just the delta in size. */ if (vrm->flags & MREMAP_DONTUNMAP) - charged = vrm->new_len >> PAGE_SHIFT; + charged = vrm->new_len >> mm_page_shift(current->mm); else - charged = vrm->delta >> PAGE_SHIFT; + charged = vrm->delta >> mm_page_shift(current->mm); /* This accounts 'charged' pages of memory. */ - if (security_vm_enough_memory_mm(current->mm, charged)) + if (mm_account_memory(current->mm, current->mm, charged)) return false; vrm->charged = charged; @@ -1007,7 +1013,7 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) if (!(vrm->vma->vm_flags & VM_ACCOUNT)) return; - vm_unacct_memory(vrm->charged); + mm_unacct_memory(current->mm, vrm->charged); vrm->charged = 0; } @@ -1019,13 +1025,13 @@ static void vrm_uncharge(struct vma_remap_struct *vrm) static void vrm_stat_account(struct vma_remap_struct *vrm, unsigned long bytes) { - unsigned long pages = bytes >> PAGE_SHIFT; + unsigned long pages = bytes >> mm_page_shift(current->mm); struct mm_struct *mm = current->mm; struct vm_area_struct *vma = vrm->vma; vm_stat_account(mm, vma->vm_flags, pages); if (vma->vm_flags & VM_LOCKED) - mm->locked_vm += pages; + mm->locked_vm += bytes >> MM_ACCOUNT_SHIFT; } static bool __check_map_count_against_split(struct mm_struct *mm, @@ -1198,7 +1204,7 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) vrm->vmi_needs_invalidate = true; if (err) { /* OOM: unable to split vma, just get accounts right */ - vm_acct_memory(len >> PAGE_SHIFT); + mm_acct_memory(current->mm, len >> mm_page_shift(current->mm)); return; } @@ -1255,16 +1261,14 @@ static void unmap_source_vma(struct vma_remap_struct *vrm) static int copy_vma_and_data(struct vma_remap_struct *vrm, struct vm_area_struct **new_vma_ptr) { - unsigned long internal_offset = vrm->addr - vrm->vma->vm_start; - unsigned long internal_pgoff = internal_offset >> PAGE_SHIFT; - unsigned long new_pgoff = vrm->vma->vm_pgoff + internal_pgoff; + struct vm_page_offset pos = vma_page_offset_at(vrm->vma, vrm->addr); unsigned long moved_len; struct vm_area_struct *vma = vrm->vma; struct vm_area_struct *new_vma; int err = 0; PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); - new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, new_pgoff, + new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, pos, &pmc.need_rmap_locks); if (!new_vma) { vrm_uncharge(vrm); @@ -1473,7 +1477,7 @@ static unsigned long mremap_to(struct vma_remap_struct *vrm) /* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */ if (vrm->flags & MREMAP_DONTUNMAP) { vma_flags_t vma_flags = vrm->vma->flags; - unsigned long pages = vrm->old_len >> PAGE_SHIFT; + unsigned long pages = vrm->old_len >> mm_page_shift(current->mm); if (!may_expand_vm(mm, &vma_flags, pages)) return -ENOMEM; @@ -1495,7 +1499,7 @@ static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) if (find_vma_intersection(vma->vm_mm, vma->vm_end, end)) return 0; if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, - 0, MAP_FIXED) & ~PAGE_MASK) + 0, MAP_FIXED) & ~mm_page_mask(vma->vm_mm)) return 0; return 1; } @@ -1727,7 +1731,8 @@ static int check_prep_vma(struct vma_remap_struct *vrm) struct vm_area_struct *vma = vrm->vma; struct mm_struct *mm = current->mm; unsigned long addr = vrm->addr; - unsigned long old_len, new_len, pgoff; + unsigned long old_len, new_len; + struct vm_page_offset pos, end; if (!vma) return -EFAULT; @@ -1802,9 +1807,9 @@ static int check_prep_vma(struct vma_remap_struct *vrm) vrm->populate_expand = true; /* Need to be careful about a growing mapping */ - pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; - if (pgoff + (new_len >> PAGE_SHIFT) < pgoff) + pos = vma_page_offset_at(vma, addr); + end = vm_page_offset_add(pos, new_len); + if (end.index < pos.index) return -EINVAL; if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) @@ -1813,7 +1818,7 @@ static int check_prep_vma(struct vma_remap_struct *vrm) if (!mlock_future_ok(mm, vma->vm_flags & VM_LOCKED, vrm->delta)) return -EAGAIN; - if (!may_expand_vm(mm, &vma->flags, vrm->delta >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma->flags, vrm->delta >> mm_page_shift(current->mm))) return -ENOMEM; return 0; @@ -1834,7 +1839,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* Start address must be page-aligned. */ - if (offset_in_page(addr)) + if (addr & ~mm_page_mask(current->mm)) return -EINVAL; /* @@ -1858,7 +1863,7 @@ static unsigned long check_mremap_params(struct vma_remap_struct *vrm) return -EINVAL; /* The new address must be page-aligned. */ - if (offset_in_page(vrm->new_addr)) + if (vrm->new_addr & ~mm_page_mask(current->mm)) return -EINVAL; /* A fixed address implies a move. */ @@ -1971,8 +1976,8 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm) unsigned long res; bool failed; - vrm->old_len = PAGE_ALIGN(vrm->old_len); - vrm->new_len = PAGE_ALIGN(vrm->new_len); + vrm->old_len = mm_page_align(mm, vrm->old_len); + vrm->new_len = mm_page_align(mm, vrm->new_len); res = check_mremap_params(vrm); if (res) diff --git a/mm/mseal.c b/mm/mseal.c index e2093ae3d25cdc..62df360b43545b 100644 --- a/mm/mseal.c +++ b/mm/mseal.c @@ -151,10 +151,10 @@ int do_mseal(unsigned long start, size_t len_in, unsigned long flags) return -EINVAL; start = untagged_addr(start); - if (!PAGE_ALIGNED(start)) + if (start & ~mm_page_mask(mm)) return -EINVAL; - len = PAGE_ALIGN(len_in); + len = mm_page_align(mm, len_in); /* Check to see whether len was rounded up from small -ve to zero. */ if (len_in && !len) return -EINVAL; diff --git a/mm/msync.c b/mm/msync.c index ac4c9bfea2e7fa..6d9027ee47246a 100644 --- a/mm/msync.c +++ b/mm/msync.c @@ -41,12 +41,12 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) if (flags & ~(MS_ASYNC | MS_INVALIDATE | MS_SYNC)) goto out; - if (offset_in_page(start)) + if (start & ~mm_page_mask(mm)) goto out; if ((flags & MS_ASYNC) && (flags & MS_SYNC)) goto out; error = -ENOMEM; - len = (len + ~PAGE_MASK) & PAGE_MASK; + len = mm_page_align(mm, len); end = start + len; if (end < start) goto out; @@ -86,7 +86,7 @@ SYSCALL_DEFINE3(msync, unsigned long, start, size_t, len, int, flags) } file = vma->vm_file; fstart = (start - vma->vm_start) + - ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + ((loff_t)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); fend = fstart + (min(end, vma->vm_end) - start) - 1; start = vma->vm_end; if ((flags & MS_SYNC) && file && diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 5f372f6e26fa32..5345f43c10b6ff 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -225,7 +225,8 @@ long oom_badness(struct task_struct *p, unsigned long totalpages) * The baseline for the badness score is the proportion of RAM that each * task's rss, pagetable and swap space use. */ - points = get_mm_rss_sum(p->mm) + get_mm_counter_sum(p->mm, MM_SWAPENTS) + + points = mm_pages_to_native(p->mm, get_mm_rss_sum(p->mm) + + get_mm_counter_sum(p->mm, MM_SWAPENTS)) + mm_pgtables_bytes(p->mm) / PAGE_SIZE; task_unlock(p); @@ -399,10 +400,10 @@ static int dump_task(struct task_struct *p, void *arg) pr_info("[%7d] %5d %5d %8lu %8lu %8lu %8lu %9lu %8ld %8lu %5hd %s\n", task->pid, from_kuid(&init_user_ns, task_uid(task)), - task->tgid, task->mm->total_vm, get_mm_rss_sum(task->mm), - get_mm_counter_sum(task->mm, MM_ANONPAGES), get_mm_counter_sum(task->mm, MM_FILEPAGES), - get_mm_counter_sum(task->mm, MM_SHMEMPAGES), mm_pgtables_bytes(task->mm), - get_mm_counter_sum(task->mm, MM_SWAPENTS), + task->tgid, mm_pages_to_native(task->mm, task->mm->total_vm), mm_pages_to_native(task->mm, get_mm_rss_sum(task->mm)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_ANONPAGES)), mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_FILEPAGES)), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SHMEMPAGES)), mm_pgtables_bytes(task->mm), + mm_pages_to_native(task->mm, get_mm_counter_sum(task->mm, MM_SWAPENTS)), task->signal->oom_score_adj, task->comm); task_unlock(task); @@ -588,9 +589,9 @@ static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm) pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n", task_pid_nr(tsk), tsk->comm, - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES))); + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES))); out_finish: trace_finish_task_reaping(tsk->pid); out_unlock: @@ -943,10 +944,10 @@ static void __oom_kill_process(struct task_struct *victim, const char *message) do_send_sig_info(SIGKILL, SEND_SIG_PRIV, victim, PIDTYPE_TGID); mark_oom_victim(victim); pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, UID:%u pgtables:%lukB oom_score_adj:%d\n", - message, task_pid_nr(victim), victim->comm, K(mm->total_vm), - K(get_mm_counter_sum(mm, MM_ANONPAGES)), - K(get_mm_counter_sum(mm, MM_FILEPAGES)), - K(get_mm_counter_sum(mm, MM_SHMEMPAGES)), + message, task_pid_nr(victim), victim->comm, mm_pages_to_kb(mm, mm->total_vm), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_ANONPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_FILEPAGES)), + mm_pages_to_kb(mm, get_mm_counter_sum(mm, MM_SHMEMPAGES)), from_kuid(&init_user_ns, task_uid(victim)), mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj); task_unlock(victim); diff --git a/mm/page_idle.c b/mm/page_idle.c index 9c67cbac296598..8af46b0d8f7460 100644 --- a/mm/page_idle.c +++ b/mm/page_idle.c @@ -51,9 +51,10 @@ static struct folio *page_idle_get_folio(unsigned long pfn) static bool page_idle_clear_pte_refs_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long addr, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0); + unsigned long addr = rmap.address; + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool referenced = false; while (page_vma_mapped_walk(&pvmw)) { @@ -69,7 +70,7 @@ static bool page_idle_clear_pte_refs_one(struct folio *folio, */ if (likely(pte_present(ptep_get(pvmw.pte)))) referenced |= ptep_test_and_clear_young(vma, addr, pvmw.pte); - referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + PAGE_SIZE); + referenced |= mmu_notifier_clear_young(vma->vm_mm, addr, addr + mm_page_size(vma->vm_mm)); } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { pmd_t pmdval = pmdp_get(pvmw.pmd); diff --git a/mm/page_io.c b/mm/page_io.c index 70cea9e24d2fd3..290706f931bdcd 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "swap.h" static void __end_swap_bio_write(struct bio *bio) @@ -411,6 +412,58 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) *swap_plug = sio; } +static DEFINE_MUTEX(swap_extent_io_mutex); +static struct bio_set swap_extent_bioset; + +int swap_extent_io_init(void) +{ + int ret = 0; + + /* Swapon prepares the reserve before any reclaim can use this file. */ + mutex_lock(&swap_extent_io_mutex); + if (!bioset_initialized(&swap_extent_bioset)) + ret = bioset_init(&swap_extent_bioset, SWAP_CLUSTER_MAX, 0, + BIOSET_NEED_RESCUER); + mutex_unlock(&swap_extent_io_mutex); + return ret; +} + +/* + * A mandatory large base page cannot split to fit a swap-file extent. Chain + * each physical run to the original bio: its folio completion must wait for + * every run, including on partial I/O or allocation failure. + */ +static void swap_submit_bio(struct bio *bio, void *private) +{ + struct folio *folio = private; + struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); + swp_entry_t entry = folio->swap; + + if (!(READ_ONCE(sis->flags) & SWP_BLKDEV) && folio_test_large(folio)) { + while (true) { + unsigned long nr = bio->bi_iter.bi_size >> PAGE_SHIFT; + unsigned int bytes; + struct bio *split; + + bio->bi_iter.bi_sector = swap_extent_sector(entry, &nr); + bytes = nr << PAGE_SHIFT; + if (bytes == bio->bi_iter.bi_size) + break; + /* The unsubmitted parent may own a bio from fs_bio_set. */ + split = bio_split(bio, bytes >> 9, GFP_NOIO, &swap_extent_bioset); + if (IS_ERR(split)) { + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return; + } + bio_chain(split, bio); + submit_bio(split); + entry.val += nr; + } + } + submit_bio(bio); +} + static void swap_writepage_bdev_sync(struct folio *folio, struct swap_info_struct *sis) { @@ -427,7 +480,7 @@ static void swap_writepage_bdev_sync(struct folio *folio, folio_start_writeback(folio); folio_unlock(folio); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_write(&bio); } @@ -445,7 +498,7 @@ static void swap_writepage_bdev_async(struct folio *folio, count_swpout_vm_event(folio); folio_start_writeback(folio); folio_unlock(folio); - submit_bio(bio); + swap_submit_bio(bio, folio); } void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug) @@ -590,7 +643,7 @@ static void swap_read_folio_bdev_sync(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio_wait(&bio); + bio_await(&bio, folio, swap_submit_bio); __end_swap_bio_read(&bio); put_task_struct(current); } @@ -607,7 +660,7 @@ static void swap_read_folio_bdev_async(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - submit_bio(bio); + swap_submit_bio(bio, folio); } void swap_read_folio(struct folio *folio, struct swap_iocb **plug) diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 2403f5a1141058..dc6797a41daaa2 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -4,6 +4,7 @@ * Copyright (c) 2021, Google LLC. * Pasha Tatashin */ +#include #include #include #include @@ -13,11 +14,21 @@ #undef pr_fmt #define pr_fmt(fmt) "page_table_check: " fmt -struct page_table_check { +#ifdef CONFIG_MM_SUBPAGE +#define PAGE_TABLE_CHECK_SLOTS (PAGE_SIZE / SZ_4K) +#else +#define PAGE_TABLE_CHECK_SLOTS 1 +#endif + +struct page_table_check_slot { atomic_t anon_map_count; atomic_t file_map_count; }; +struct page_table_check { + struct page_table_check_slot slots[PAGE_TABLE_CHECK_SLOTS]; +}; + static bool __page_table_check_enabled __initdata = IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED); @@ -56,6 +67,74 @@ static struct page_table_check *get_page_table_check(struct page_ext *page_ext) return page_ext_data(page_ext, &page_table_check_ops); } +#if IS_ENABLED(CONFIG_KUNIT) +/* Diagnostic snapshot for private test allocations, not a synchronization API. */ +int page_table_check_get_counts(struct page *page, unsigned int offset, + int *anon, int *file) +{ + struct page_ext *ext; + struct page_table_check *ptc; + unsigned int slot = 0; + + if (static_branch_likely(&page_table_check_disabled)) + return -EOPNOTSUPP; + if (offset >= PAGE_SIZE) + return -EINVAL; +#ifdef CONFIG_MM_SUBPAGE + slot = offset / SZ_4K; +#endif + ext = page_ext_get(page); + if (!ext) + return -ENOENT; + ptc = get_page_table_check(ext); + *anon = atomic_read(&ptc->slots[slot].anon_map_count); + *file = atomic_read(&ptc->slots[slot].file_map_count); + page_ext_put(ext); + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(page_table_check_get_counts); +#endif + +static void page_table_check_update(struct page_table_check_slot *slot, + bool anon, bool clear, bool rw) +{ + if (anon) { + BUG_ON(atomic_read(&slot->file_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->anon_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->anon_map_count) > 1 && rw); + } else { + BUG_ON(atomic_read(&slot->anon_map_count)); + if (clear) + BUG_ON(atomic_dec_return(&slot->file_map_count) < 0); + else + BUG_ON(atomic_inc_return(&slot->file_map_count) < 0); + } +} + +#ifdef CONFIG_MM_SUBPAGE +static void page_table_check_subpage(phys_addr_t phys, unsigned long size, + bool clear, bool rw) +{ + unsigned long pfn = PHYS_PFN(phys); + struct page_table_check *ptc; + struct page_ext *ext; + struct page *page; + unsigned int first = offset_in_page(phys) / SZ_4K, i; + + if (!pfn_valid(pfn)) + return; + page = pfn_to_page(pfn); + BUG_ON(PageSlab(page)); + ext = page_ext_get(page); + ptc = get_page_table_check(ext); + for (i = first; i < first + size / SZ_4K; i++) + page_table_check_update(&ptc->slots[i], PageAnon(page), clear, rw); + page_ext_put(ext); +} +#endif + /* * An entry is removed from the page table, decrement the counters for that page * verify that it is of correct type and counters do not become negative. @@ -78,13 +157,8 @@ static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt) for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, true, false); } rcu_read_unlock(); } @@ -113,13 +187,8 @@ static void page_table_check_set(unsigned long pfn, unsigned long pgcnt, for_each_page_ext(page, pgcnt, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - if (anon) { - BUG_ON(atomic_read(&ptc->file_map_count)); - BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw); - } else { - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0); - } + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) + page_table_check_update(&ptc->slots[i], anon, false, rw); } rcu_read_unlock(); } @@ -139,8 +208,10 @@ void __page_table_check_zero(struct page *page, unsigned int order) for_each_page_ext(page, 1 << order, page_ext, iter) { struct page_table_check *ptc = get_page_table_check(page_ext); - BUG_ON(atomic_read(&ptc->anon_map_count)); - BUG_ON(atomic_read(&ptc->file_map_count)); + for (unsigned int i = 0; i < PAGE_TABLE_CHECK_SLOTS; i++) { + BUG_ON(atomic_read(&ptc->slots[i].anon_map_count)); + BUG_ON(atomic_read(&ptc->slots[i].file_map_count)); + } } rcu_read_unlock(); } @@ -151,8 +222,15 @@ void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr, if (&init_mm == mm) return; - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + page_table_check_subpage(pte_phys_mm(mm, pte), mm_page_size(mm), true, false); + return; + } +#endif + page_table_check_clear(pte_pfn(pte), mm_pte_native_pages(mm)); } EXPORT_SYMBOL(__page_table_check_pte_clear); @@ -218,9 +296,20 @@ void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr, page_table_check_pte_flags(pte); for (i = 0; i < nr; i++) - __page_table_check_pte_clear(mm, addr + PAGE_SIZE * i, ptep_get(ptep + i)); - if (pte_user_accessible_page(mm, addr, pte) && !pte_special(pte)) - page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); + __page_table_check_pte_clear(mm, addr + mm_page_size(mm) * i, ptep_get(ptep + i)); + if (!pte_user_accessible_page(mm, addr, pte) || pte_special(pte)) + return; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + phys_addr_t phys = pte_phys_mm(mm, pte); + + for (i = 0; i < nr; i++) + page_table_check_subpage(phys + i * mm_page_size(mm), mm_page_size(mm), false, + pte_write(pte)); + return; + } +#endif + page_table_check_set(pte_pfn(pte), nr * mm_pte_native_pages(mm), pte_write(pte)); } EXPORT_SYMBOL(__page_table_check_ptes_set); @@ -279,16 +368,17 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, return; if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { - pte_t *ptep = pte_offset_map(&pmd, addr); + pte_t *ptep = pte_offset_map_mm(mm, &pmd, addr); + unsigned long nr = pte_table_bytes_mm(mm) / sizeof(pte_t); unsigned long i; if (WARN_ON(!ptep)) return; - for (i = 0; i < PTRS_PER_PTE; i++) { + for (i = 0; i < nr; i++) { __page_table_check_pte_clear(mm, addr, ptep_get(ptep)); - addr += PAGE_SIZE; + addr += mm_page_size(mm); ptep++; } - pte_unmap(ptep - PTRS_PER_PTE); + pte_unmap(ptep - nr); } } diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index ccd1c622c8b24f..e3b0b64fbfba21 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -87,7 +87,7 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, * mapped at the @pvmw->pte * @pvmw: page_vma_mapped_walk struct, includes a pair pte and pfn range * for checking - * @pte_nr: the number of small pages described by @pvmw->pte. + * @pte_nr: the number of native physical pages covered by @pvmw->pte. * * page_vma_mapped_walk() found a place where pfn range is *potentially* * mapped. check_pte() has to validate this. @@ -109,6 +109,18 @@ static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) unsigned long pfn; pte_t ptent = ptep_get(pvmw->pte); +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (!(pvmw->flags & PVMW_MIGRATION)) + return pte_present(ptent) && + pte_phys_mm(pvmw->vma->vm_mm, ptent) == + PFN_PHYS(pvmw->pfn) + pvmw->subpage_offset; + if (pte_swp_subpage_offset(ptent) != pvmw->subpage_offset) + return false; + /* The common migration check below validates the native PFN. */ + } +#endif + if (pvmw->flags & PVMW_MIGRATION) { const softleaf_t entry = softleaf_from_pte(ptent); @@ -188,6 +200,18 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) pud_t *pud; pmd_t pmde; +#ifdef CONFIG_MM_SUBPAGE + if (pvmw->flags & PVMW_SUBPAGE) { + if (WARN_ON_ONCE(pvmw->nr_pages != 1 || + mm_page_size(mm) >= PAGE_SIZE || + !IS_ALIGNED(pvmw->address, mm_page_size(mm)) || + pvmw->subpage_offset >= PAGE_SIZE || + !IS_ALIGNED(pvmw->subpage_offset, mm_page_size(mm)) || + is_vm_hugetlb_page(vma))) + return not_found(pvmw); + } +#endif + /* The only possible pmd mapping has been handled on last iteration */ if (pvmw->pmd && !pvmw->pte) return not_found(pvmw); @@ -213,6 +237,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) return true; } + /* A query for an interior native page still visits one whole user leaf. */ + if (mm_page_size(mm) > PAGE_SIZE) + pvmw->address &= mm_page_mask(mm); end = vma_address_end(pvmw); if (pvmw->pte) goto next_pte; @@ -220,21 +247,21 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) do { pgd = pgd_offset(mm, pvmw->address); if (!pgd_present(*pgd)) { - step_forward(pvmw, PGDIR_SIZE); + step_forward(pvmw, pgd_size_mm(mm)); continue; } - p4d = p4d_offset(pgd, pvmw->address); - if (!p4d_present(*p4d)) { - step_forward(pvmw, P4D_SIZE); + p4d = p4d_offset_mm(mm, pgd, pvmw->address); + if (!p4d_present_mm(mm, *p4d)) { + step_forward(pvmw, p4d_size_mm(mm)); continue; } - pud = pud_offset(p4d, pvmw->address); + pud = pud_offset_mm(mm, p4d, pvmw->address); if (!pud_present(*pud)) { - step_forward(pvmw, PUD_SIZE); + step_forward(pvmw, pud_size_mm(mm)); continue; } - pvmw->pmd = pmd_offset(pud, pvmw->address); + pvmw->pmd = pmd_offset_mm(mm, pud, pvmw->address); /* * Make sure the pmd value isn't cached in a register by the * compiler and used as a stale value after we've observed a @@ -245,6 +272,9 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde) || pmd_is_device_private_entry(pmde))) { + /* Subpage PMD mappings need a distinct huge-page implementation. */ + if (mm_page_size(mm) != PAGE_SIZE) + return not_found(pvmw); pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; if (pmd_is_migration_entry(pmde)) { @@ -285,7 +315,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) (pvmw->nr_pages >= HPAGE_PMD_NR)) sync_with_folio_pmd_zap(mm, pvmw->pmd); - step_forward(pvmw, PMD_SIZE); + step_forward(pvmw, pmd_size_mm(mm)); continue; } if (!map_pte(pvmw, &pmde, &ptl)) { @@ -294,15 +324,15 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) goto next_pte; } this_pte: - if (check_pte(pvmw, 1)) + if (check_pte(pvmw, mm_pte_native_pages(mm))) return true; next_pte: do { - pvmw->address += PAGE_SIZE; + pvmw->address += mm_page_size(mm); if (pvmw->address >= end) return not_found(pvmw); /* Did we cross page table boundary? */ - if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) { + if ((pvmw->address & (pmd_size_mm(mm) - mm_page_size(mm))) == 0) { if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL; @@ -345,6 +375,7 @@ unsigned long page_mapped_in_vma(const struct page *page, struct vm_area_struct *vma) { const struct folio *folio = page_folio(page); + unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), .nr_pages = 1, @@ -352,13 +383,15 @@ unsigned long page_mapped_in_vma(const struct page *page, .flags = PVMW_SYNC, }; - pvmw.address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, page_pgoff(folio, page), 1); + pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; page_vma_mapped_walk_done(&pvmw); out: - return pvmw.address; + /* Report this native page's byte address, not the covering leaf start. */ + return address; } #endif diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 5d87c632a25507..c3216b4852399e 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -30,13 +30,14 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, unsigned long end, struct mm_walk *walk) { const struct mm_walk_ops *ops = walk->ops; + const unsigned long page_size = mm_page_size(walk->mm); int err = 0; for (;;) { if (ops->install_pte && pte_none(ptep_get(pte))) { pte_t new_pte; - err = ops->install_pte(addr, addr + PAGE_SIZE, &new_pte, + err = ops->install_pte(addr, addr + page_size, &new_pte, walk); if (err) break; @@ -46,13 +47,13 @@ static int walk_pte_range_inner(pte_t *pte, unsigned long addr, if (!WARN_ON_ONCE(walk->no_vma)) update_mmu_cache(walk->vma, addr, pte); } else { - err = ops->pte_entry(pte, addr, addr + PAGE_SIZE, walk); + err = ops->pte_entry(pte, addr, addr + page_size, walk); if (err) break; } - if (addr >= end - PAGE_SIZE) + if (addr >= end - page_size) break; - addr += PAGE_SIZE; + addr += page_size; pte++; } return err; @@ -67,15 +68,15 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (walk->no_vma) { /* - * pte_offset_map() might apply user-specific validation. + * pte_offset_map_mm() might apply user-specific validation. * Indeed, on x86_64 the pmd entries set up by init_espfix_ap() * fit its pmd_bad() check (_PAGE_NX set and _PAGE_RW clear), * and CONFIG_EFI_PGT_DUMP efi_mm goes so far as to walk them. */ if (walk->mm == &init_mm || addr >= TASK_SIZE) - pte = pte_offset_kernel(pmd, addr); + pte = pte_offset_kernel_mm(walk->mm, pmd, addr); else - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(walk->mm, pmd, addr); if (pte) { err = walk_pte_range_inner(pte, addr, end, walk); if (walk->mm != &init_mm && addr < TASK_SIZE) @@ -123,10 +124,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, return 0; } - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(walk->mm, pud, addr); do { again: - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(walk->mm, addr, end); if (pmd_none(*pmd)) { if (has_install) err = __pte_alloc(walk->mm, pmd); @@ -193,10 +194,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(2); - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(walk->mm, p4d, addr); do { again: - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(walk->mm, addr, end); if (pud_none(*pud)) { if (has_install) err = __pmd_alloc(walk->mm, pud, addr); @@ -258,10 +259,10 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, int err = 0; int depth = real_depth(1); - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(walk->mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) { + next = p4d_addr_end_mm(walk->mm, addr, end); + if (p4d_none_or_clear_bad_mm(walk->mm, p4d)) { if (has_install) err = __pud_alloc(walk->mm, p4d, addr); else if (ops->pte_hole) @@ -297,11 +298,11 @@ static int walk_pgd_range(unsigned long addr, unsigned long end, int err = 0; if (walk->pgd) - pgd = walk->pgd + pgd_index(addr); + pgd = pgd_offset_pgd_mm(walk->mm, walk->pgd, addr); else pgd = pgd_offset(walk->mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(walk->mm, addr, end); if (pgd_none_or_clear_bad(pgd)) { if (has_install) err = __p4d_alloc(walk->mm, pgd, addr); @@ -924,11 +925,11 @@ struct folio *folio_walk_start(struct folio_walk *fw, if (pgd_none_or_clear_bad(pgdp)) goto not_found; - p4dp = p4d_offset(pgdp, addr); - if (p4d_none_or_clear_bad(p4dp)) + p4dp = p4d_offset_mm(vma->vm_mm, pgdp, addr); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4dp)) goto not_found; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(vma->vm_mm, p4dp, addr); pud = pudp_get(pudp); if (pud_none(pud)) goto not_found; @@ -937,7 +938,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pud_lock(vma->vm_mm, pudp); pud = pudp_get(pudp); - entry_size = PUD_SIZE; + entry_size = pud_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PUD; fw->pudp = pudp; fw->pud = pud; @@ -959,7 +960,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, pmd_table: VM_WARN_ON_ONCE(!pud_present(pud) || pud_leaf(pud)); - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(vma->vm_mm, pudp, addr); pmd = pmdp_get_lockless(pmdp); if (pmd_none(pmd)) goto not_found; @@ -968,7 +969,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, ptl = pmd_lock(vma->vm_mm, pmdp); pmd = pmdp_get(pmdp); - entry_size = PMD_SIZE; + entry_size = pmd_size_mm(vma->vm_mm); fw->level = FW_LEVEL_PMD; fw->pmdp = pmdp; fw->pmd = pmd; @@ -1001,7 +1002,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, goto not_found; pte = ptep_get(ptep); - entry_size = PAGE_SIZE; + entry_size = mm_page_size(vma->vm_mm); fw->level = FW_LEVEL_PTE; fw->ptep = ptep; fw->pte = pte; @@ -1028,5 +1029,5 @@ struct folio *folio_walk_start(struct folio_walk *fw, else fw->page = NULL; fw->ptl = ptl; - return page_folio(page); + return page_folio(zeropage ? page : fw->page); } diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index b91b1a98029c7f..7a9eef9b4f5b7e 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -168,13 +168,15 @@ pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) { + struct page *page = pgtable_to_page(pgtable); + assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ if (!pmd_huge_pte(mm, pmdp)) - INIT_LIST_HEAD(&pgtable->lru); + INIT_LIST_HEAD(&page->lru); else - list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); + list_add(&page->lru, &pgtable_to_page(pmd_huge_pte(mm, pmdp))->lru); pmd_huge_pte(mm, pmdp) = pgtable; } #endif @@ -184,15 +186,17 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) { pgtable_t pgtable; + struct page *page, *next; assert_spin_locked(pmd_lockptr(mm, pmdp)); /* FIFO */ pgtable = pmd_huge_pte(mm, pmdp); - pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru, - struct page, lru); + page = pgtable_to_page(pgtable); + next = list_first_entry_or_null(&page->lru, struct page, lru); + pmd_huge_pte(mm, pmdp) = pgtable_from_page(next); if (pmd_huge_pte(mm, pmdp)) - list_del(&pgtable->lru); + list_del(&page->lru); return pgtable; } #endif @@ -282,7 +286,12 @@ static unsigned long pmdp_get_lockless_start(void) { return 0; } static void pmdp_get_lockless_end(unsigned long irqflags) { } #endif +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, pmd_t *pmdvalp) +#else pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +#endif { unsigned long irqflags; pmd_t pmdval; @@ -302,19 +311,27 @@ pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) pmd_clear_bad(pmd); goto nomap; } - return __pte_map(&pmdval, addr); + return __pte_map_mm(mm, &pmdval, addr); nomap: rcu_read_unlock(); return NULL; } +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +{ + /* NULL retains native geometry for callers not yet converted. */ + return __pte_offset_map_mm(NULL, pmd, addr, pmdvalp); +} +#endif + pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp) { pmd_t pmdval; pte_t *pte; - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (likely(pte)) *ptlp = pte_lockptr(mm, &pmdval); return pte; @@ -327,7 +344,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, pte_t *pte; VM_WARN_ON_ONCE(!pmdvalp); - pte = __pte_offset_map(pmd, addr, pmdvalp); + pte = __pte_offset_map_mm(mm, pmd, addr, pmdvalp); if (likely(pte)) *ptlp = pte_lockptr(mm, pmdvalp); return pte; @@ -397,7 +414,7 @@ pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, pmd_t pmdval; pte_t *pte; again: - pte = __pte_offset_map(pmd, addr, &pmdval); + pte = __pte_offset_map_mm(mm, pmd, addr, &pmdval); if (unlikely(!pte)) return pte; ptl = pte_lockptr(mm, &pmdval); diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index 656d3e88755b6f..f178b5fdb65890 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,8 @@ #include #include +#include "internal.h" + /** * process_vm_rw_pages - read/write pages from task specified * @pages: array of pointers to pages we want to copy @@ -135,6 +138,63 @@ static int process_vm_rw_single_vec(unsigned long addr, which lives on stack */ #define PVM_MAX_PP_ARRAY_COUNT 16 +#ifdef CONFIG_MM_SUBPAGE +/* Same pin/copy/dirty lifetime as the page-array path, with explicit offsets. */ +int process_vm_rw_fragments(struct mm_struct *mm, unsigned long addr, + unsigned long len, struct iov_iter *iter, bool write) +{ + struct user_page_fragment stack[PVM_MAX_PP_ARRAY_COUNT]; + struct user_page_fragment *fragments = stack; + unsigned long capacity, nr; + int ret = 0; + + if (!len || !iov_iter_count(iter)) + return 0; + capacity = 1 + ((len - 1) >> MM_SUBPAGE_SHIFT) + + ((((len - 1) & (MM_SUBPAGE_SIZE - 1)) + + (addr & (MM_SUBPAGE_SIZE - 1))) >> MM_SUBPAGE_SHIFT); + capacity = min_t(unsigned long, capacity, + PVM_MAX_KMALLOC_PAGES * PAGE_SIZE / sizeof(*fragments)); + if (capacity > ARRAY_SIZE(stack)) { + fragments = kmalloc_array(capacity, sizeof(*fragments), GFP_KERNEL); + if (!fragments) + return -ENOMEM; + } + while (len && iov_iter_count(iter)) { + long got = pin_user_fragments_remote(mm, addr, len, + write ? FOLL_WRITE : 0, fragments, capacity); + + if (got <= 0) { + ret = -EFAULT; + break; + } + for (nr = 0; nr < got && iov_iter_count(iter); nr++) { + struct user_page_fragment *f = &fragments[nr]; + struct page *page = folio_page(f->folio, f->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(f->offset); + size_t copied; + + if (write) + copied = copy_page_from_iter(page, offset, f->length, iter); + else + copied = copy_page_to_iter(page, offset, f->length, iter); + addr += copied; + len -= copied; + if (copied < f->length && iov_iter_count(iter)) { + ret = -EFAULT; + break; + } + } + release_user_fragments(fragments, got, write); + if (ret) + break; + } + if (fragments != stack) + kfree(fragments); + return ret; +} +#endif + /** * process_vm_rw_core - core of reading/writing pages from task specified * @pid: PID of process to read/write from/to @@ -182,17 +242,6 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (nr_pages == 0) return 0; - if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { - /* For reliability don't try to kmalloc more than - 2 pages worth */ - process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, - sizeof(struct page *)*nr_pages), - GFP_KERNEL); - - if (!process_pages) - return -ENOMEM; - } - /* Get process information */ task = find_get_task_by_vpid(pid); if (!task) { @@ -212,11 +261,35 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, goto put_task_struct; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) + rc = process_vm_rw_fragments(mm, (unsigned long)rvec[i].iov_base, + rvec[i].iov_len, iter, vm_write); + goto copied; + } +#endif + if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { + /* For reliability don't try to kmalloc more than + 2 pages worth */ + process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, + sizeof(struct page *)*nr_pages), + GFP_KERNEL); + + if (!process_pages) { + rc = -ENOMEM; + goto put_mm; + } + } + for (i = 0; i < riovcnt && iov_iter_count(iter) && !rc; i++) rc = process_vm_rw_single_vec( (unsigned long)rvec[i].iov_base, rvec[i].iov_len, iter, process_pages, mm, task, vm_write); +#ifdef CONFIG_MM_SUBPAGE +copied: +#endif /* copied = space before - space after */ total_len -= iov_iter_count(iter); @@ -226,6 +299,7 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (total_len) rc = total_len; +put_mm: mmput(mm); put_task_struct: diff --git a/mm/rmap.c b/mm/rmap.c index 99e1b3dc390b7f..795b568b9f7c5a 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -75,6 +75,7 @@ #include #include #include +#include #include @@ -83,17 +84,22 @@ #include "internal.h" #include "swap.h" +#include "user-subpage-internal.h" static struct kmem_cache *anon_vma_cachep; static struct kmem_cache *anon_vma_chain_cachep; -static inline struct anon_vma *anon_vma_alloc(void) +static inline struct anon_vma *anon_vma_alloc(struct vm_area_struct *vma) { struct anon_vma *anon_vma; anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); +#ifdef CONFIG_MM_SUBPAGE + anon_vma->min_folio_order = + max_t(unsigned int, mm_page_shift(vma->vm_mm), PAGE_SHIFT) - PAGE_SHIFT; +#endif anon_vma->num_children = 0; anon_vma->num_active_vmas = 0; anon_vma->parent = anon_vma; @@ -198,7 +204,7 @@ int __anon_vma_prepare(struct vm_area_struct *vma) anon_vma = find_mergeable_anon_vma(vma); allocated = NULL; if (!anon_vma) { - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (unlikely(!anon_vma)) goto out_enomem_free_avc; anon_vma->num_children++; /* self-parent link for new root */ @@ -388,7 +394,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) /* Drop inherited anon_vma, we'll reuse existing or allocate new. */ vma->anon_vma = NULL; - anon_vma = anon_vma_alloc(); + anon_vma = anon_vma_alloc(vma); if (!anon_vma) return -ENOMEM; avc = anon_vma_chain_alloc(GFP_KERNEL); @@ -622,6 +628,35 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +/* The caller retains the folio; splitting rechecks the order under its lock. */ +unsigned int folio_anon_min_order(struct folio *folio) +{ + struct anon_vma *anon_vma = NULL; + unsigned long mapping; + unsigned int order = 0; + + /* Also used by memory-failure callers before they lock the folio. */ + rcu_read_lock(); + mapping = (unsigned long)READ_ONCE(folio->mapping); + if ((mapping & FOLIO_MAPPING_FLAGS) != FOLIO_MAPPING_ANON || !folio_mapped(folio)) + goto out; + anon_vma = (struct anon_vma *)(mapping - FOLIO_MAPPING_ANON); + if (!atomic_inc_not_zero(&anon_vma->refcount)) { + anon_vma = NULL; + goto out; + } + /* Revalidate against unmap/reuse of the RCU-typesafe anon_vma slab. */ + if (folio_mapped(folio) && mapping == (unsigned long)READ_ONCE(folio->mapping)) + order = anon_vma->min_folio_order; +out: + rcu_read_unlock(); + if (anon_vma) + put_anon_vma(anon_vma); + return order; +} +#endif + /* * Similar to folio_get_anon_vma() except it locks the anon_vma. * @@ -856,6 +891,13 @@ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) unsigned long page_address_in_vma(const struct folio *folio, const struct page *page, const struct vm_area_struct *vma) { +#ifdef CONFIG_MM_SUBPAGE + unsigned long address; + + if (folio_test_anon(folio) && !folio_test_large(folio) && + mm_subpage_address_in_vma(folio, vma, &address)) + return address; +#endif if (folio_test_anon(folio)) { struct anon_vma *anon_vma = folio_anon_vma(folio); /* @@ -891,15 +933,15 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) if (!pgd_present(*pgd)) goto out; - p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + p4d = p4d_offset_mm(mm, pgd, address); + if (!p4d_present_mm(mm, *p4d)) goto out; - pud = pud_offset(p4d, address); + pud = pud_offset_mm(mm, p4d, address); if (!pud_present(*pud)) goto out; - pmd = pmd_offset(pud, address); + pmd = pmd_offset_mm(mm, pud, address); out: return pmd; } @@ -915,10 +957,11 @@ struct folio_referenced_arg { * arg: folio_referenced_arg will be passed */ static bool folio_referenced_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct folio_referenced_arg *pra = arg; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); int ptes = 0, referenced = 0; unsigned int nr; @@ -928,10 +971,11 @@ static bool folio_referenced_one(struct folio *folio, if (vma->vm_flags & VM_LOCKED) { ptes++; - pra->mapcount--; + pra->mapcount -= pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -966,11 +1010,18 @@ static bool folio_referenced_one(struct folio *folio, } if (pvmw.pte && folio_test_large(folio)) { - const unsigned long end_addr = pmd_addr_end(address, vma->vm_end); - const unsigned int max_nr = (end_addr - address) >> PAGE_SHIFT; + const unsigned long end_addr = pmd_addr_end_mm(vma->vm_mm, + address, vma->vm_end); + const unsigned int max_nr = (end_addr - address) >> mm_page_shift(vma->vm_mm); pte_t pteval = ptep_get(pvmw.pte); - nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + nr = folio_subpage_pte_batch(vma, folio, pvmw.pte, + pteval, max_nr, 0); + else +#endif + nr = folio_pte_batch(folio, pvmw.pte, pteval, max_nr); } /* @@ -994,19 +1045,19 @@ static bool folio_referenced_one(struct folio *folio, } ptes += nr; - pra->mapcount -= nr; + pra->mapcount -= nr * (pvmw.pte ? mm_pte_native_pages(vma->vm_mm) : 1); /* * If we are sure that we batched the entire folio, * we can just optimize and stop right here. */ - if (ptes == pvmw.nr_pages) { + if (ptes == (((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm))) { page_vma_mapped_walk_done(&pvmw); break; } /* Skip the batched PTEs */ pvmw.pte += nr - 1; - pvmw.address += (nr - 1) * PAGE_SIZE; + pvmw.address += (nr - 1) * mm_page_size(vma->vm_mm); } if (referenced) @@ -1178,9 +1229,9 @@ static int page_vma_mkclean_one(struct page_vma_mapped_walk *pvmw) } static bool page_mkclean_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, PVMW_SYNC); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, PVMW_SYNC); int *cleaned = arg; *cleaned += page_vma_mkclean_one(&pvmw); @@ -1229,8 +1280,9 @@ struct wrprotect_file_state { }; static bool mapping_wrprotect_range_one(struct folio *folio, - struct vm_area_struct *vma, unsigned long address, void *arg) + struct vm_area_struct *vma, struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct wrprotect_file_state *state = (struct wrprotect_file_state *)arg; struct page_vma_mapped_walk pvmw = { .pfn = state->pfn, @@ -1515,7 +1567,7 @@ static void __page_check_anon_rmap(const struct folio *folio, static __always_inline void __folio_add_anon_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, - unsigned long address, rmap_t flags, enum pgtable_level level) + unsigned long address, rmap_t flags, enum pgtable_level level, bool new) { int i; @@ -1572,7 +1624,7 @@ static __always_inline void __folio_add_anon_rmap(struct folio *folio, * Partially mapped folios can be split on reclaim and part outside * of mlocked VMA can be evicted or freed. */ - if (folio_nr_pages(folio) == nr_pages) + if (!new && folio_nr_pages(folio) == nr_pages) mlock_vma_folio(folio, vma); } @@ -1597,7 +1649,7 @@ void folio_add_anon_rmap_ptes(struct folio *folio, struct page *page, rmap_t flags) { __folio_add_anon_rmap(folio, page, nr_pages, vma, address, flags, - PGTABLE_LEVEL_PTE); + PGTABLE_LEVEL_PTE, false); } /** @@ -1618,7 +1670,7 @@ void folio_add_anon_rmap_pmd(struct folio *folio, struct page *page, { #ifdef CONFIG_TRANSPARENT_HUGEPAGE __folio_add_anon_rmap(folio, page, HPAGE_PMD_NR, vma, address, flags, - PGTABLE_LEVEL_PMD); + PGTABLE_LEVEL_PMD, false); #else WARN_ON_ONCE(true); #endif @@ -1697,6 +1749,112 @@ void folio_add_new_anon_rmap(struct folio *folio, struct vm_area_struct *vma, mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON, 1); } +#ifdef CONFIG_MM_SUBPAGE +static int __mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address, bool new) +{ + struct folio *folio = mm_subpage_folio(subpage); + int ret; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (mm_page_shift(vma->vm_mm) != mm_subpage_shift(subpage) || !vma->anon_vma || + address < vma->vm_start || address >= vma->vm_end || + !IS_ALIGNED(address, mm_subpage_size(subpage))) + return -EINVAL; + if (mm_subpage_anon_root(subpage)) { + if (mm_subpage_is_droppable(subpage) != !!(vma->vm_flags & VM_DROPPABLE)) + return -EINVAL; + if (!folio_test_anon(folio) || folio_test_ksm(folio) || + PageAnonExclusive(&folio->page)) + return -EINVAL; + } else { + bool was_anon = folio_test_anon(folio); + + /* Swap-cache folios can retain a stale, unmapped anon pointer. */ + if (folio_mapped(folio) || folio_test_ksm(folio) || + (folio->mapping && !was_anon) || + (was_anon && PageAnonExclusive(&folio->page))) + return -EINVAL; + /* Slot references can survive the last VMA in this root. */ + mm_subpage_hold_anon_root(subpage, vma->anon_vma->root, + vma->vm_flags & VM_DROPPABLE); + /* + * A restored folio is already visible in the swap cache. Other + * faults may set PG_waiters while we hold its lock; preserve those + * concurrent flag updates even when PG_swapbacked is already set. + */ + if (!(vma->vm_flags & VM_DROPPABLE)) + folio_set_swapbacked(folio); + /* Neighboring slots may acquire different fork/COW relationships. */ + __folio_set_anon(folio, vma, address, false); + if (!was_anon) + mod_mthp_stat(0, MTHP_STAT_NR_ANON, 1); + } + ret = mm_subpage_bind_rmap(subpage, vma, address); + if (ret) + return ret; + atomic_inc(&subpage->mapcount); + /* Identity was checked per slot, not against the folio's linear anchor. */ + __folio_add_rmap(folio, &folio->page, 1, vma, PGTABLE_LEVEL_PTE); + if (!new) + mlock_vma_folio(folio, vma); + return 0; +} + +int mm_subpage_add_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, false); +} + +/* Caller adds this fresh folio to LRU and accounts its locked leaves together. */ +int mm_subpage_add_new_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma, + unsigned long address) +{ + return __mm_subpage_add_anon_rmap(subpage, vma, address, true); +} + +void mm_subpage_remove_anon_rmap(struct mm_subpage *subpage, struct vm_area_struct *vma) +{ + struct folio *folio = mm_subpage_folio(subpage); + + VM_BUG_ON_FOLIO(atomic_read(&subpage->mapcount) <= 0, folio); + atomic_dec(&subpage->mapcount); + folio_remove_rmap_pte(folio, &folio->page, vma); +} + +int mm_subpage_dup_anon_rmap(struct mm_subpage *subpage, + struct vm_area_struct *dst, struct vm_area_struct *src, unsigned long address) +{ + struct folio *folio = mm_subpage_folio(subpage); + struct vm_page_offset pos; + int ret; + + mmap_assert_write_locked(src->vm_mm); + mmap_assert_write_locked(dst->vm_mm); + if (mm_page_shift(src->vm_mm) != mm_subpage_shift(subpage) || + mm_page_shift(dst->vm_mm) != mm_subpage_shift(subpage) || + !src->anon_vma || !dst->anon_vma || !folio_test_anon(folio) || + folio_test_ksm(folio) || PageAnonExclusive(&folio->page) || + address < dst->vm_start || address >= dst->vm_end || + address < src->vm_start || address >= src->vm_end) + return -EINVAL; + pos = vma_page_offset_at(dst, address); + /* Both mmap locks and the source PTE stabilize this slot's identity. */ + if (!mm_subpage_match_rmap(subpage, dst, address) || + src->anon_vma->root != dst->anon_vma->root || + !vm_page_offset_equal(pos, vma_page_offset_at(src, address))) + return -EINVAL; + /* A pin in a neighboring physical slot does not share this slot. */ + if (!mm_subpage_try_share(subpage)) + return -EBUSY; + ret = folio_try_dup_anon_rmap_pte(folio, &folio->page, dst, src); + if (!ret) + atomic_inc(&subpage->mapcount); + return ret; +} +#endif + static __always_inline void __folio_add_file_rmap(struct folio *folio, struct page *page, int nr_pages, struct vm_area_struct *vma, enum pgtable_level level) @@ -1949,14 +2107,20 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, struct vm_area_struct *vma = pvmw->vma; unsigned int max_nr; +#ifdef CONFIG_MM_SUBPAGE + /* Native PFN batching advances physical pages, not 4K file quarters. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return 1; +#endif + if (flags & TTU_HWPOISON) return 1; if (!folio_test_large(folio)) return 1; /* We may only batch within a single VMA and a single page table. */ - end_addr = pmd_addr_end(addr, vma->vm_end); - max_nr = (end_addr - addr) >> PAGE_SHIFT; + end_addr = pmd_addr_end_mm(vma->vm_mm, addr, vma->vm_end); + max_nr = (end_addr - addr) >> mm_page_shift(vma->vm_mm); /* We only support lazyfree or file folios batching for now ... */ if (folio_test_anon(folio) && folio_test_swapbacked(folio)) @@ -1978,20 +2142,116 @@ static inline unsigned int folio_unmap_pte_batch(struct folio *folio, FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY); } +#ifdef CONFIG_MM_SUBPAGE +/* Folio and PTE locks held. Each PTE takes one reference to a native swap entry. */ +static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), swapped; + swp_entry_t entry; + bool exclusive, ret = false; + + if (!pte_present(pte) || (flags & TTU_HWPOISON)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* A released quarter can be reused before its native folio is freed. */ + flush_tlb_range(vma, address, end); + smp_mb(); + if (((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) && + mm_subpage_discardable(slot) && + arch_unmap_one(mm, vma, address, pte) >= 0) { + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference. */ + ret = true; + goto out; + } + set_pte_at(mm, address, pvmw->pte, pte); + /* Retained references must not turn a droppable folio into swap data. */ + if (vma->vm_flags & VM_DROPPABLE) + goto out; + if (pte_dirty(pte)) { + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); + } + } + if (!folio_test_swapbacked(folio)) { + /* A write or retained reference cancelled whole-folio lazy reclaim. */ + folio_set_swapbacked(folio); + goto out; + } + if (!folio_test_swapcache(folio)) + goto out; + entry = page_swap_entry(&folio->page); + if (swap_subpage_dup(entry, mm_subpage_offset(slot)) < 0) + goto out; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + /* The PTE lock serializes fork and this slot's ownership transfer. */ + exclusive = mm_subpage_is_exclusive(slot); + if (arch_unmap_one(mm, vma, address, pte) < 0 || !mm_subpage_try_share(slot)) { + swap_subpage_put(entry, mm_subpage_offset(slot), false); + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (list_empty(&mm->mmlist)) { + spin_lock(&mmlist_lock); + if (list_empty(&mm->mmlist)) + list_add(&mm->mmlist, &init_mm.mmlist); + spin_unlock(&mmlist_lock); + } + swapped = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (exclusive) + swapped = pte_swp_mkexclusive(swapped); + if (pte_soft_dirty(pte)) + swapped = pte_swp_mksoft_dirty(swapped); + if (pte_uffd_wp(pte)) + swapped = pte_swp_mkuffd_wp(swapped); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + inc_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, address, pvmw->pte, swapped); + mm_subpage_remove_anon_rmap(slot, vma); + /* Swap-cache backing cannot be recycled while a deferred TLBI is pending. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + /* * @arg: enum ttu_flags will be passed to this argument */ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, ret = true; pte_t pteval; + phys_addr_t file_phys = 0; struct page *subpage; struct mmu_notifier_range range; enum ttu_flags flags = (enum ttu_flags)(long)arg; - unsigned long nr_pages = 1, end_addr; + unsigned long nr_pages = 1, end_addr, nr_refs; unsigned long pfn; unsigned long hsz = 0; int ptes = 0; @@ -2003,7 +2263,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2049,7 +2309,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, ret = false; /* Only mlock fully mapped pages */ - if (pvmw.pte && ptes != pvmw.nr_pages) + if (pvmw.pte && ptes != (folio_test_large(folio) ? + ((unsigned long)pvmw.nr_pages << PAGE_SHIFT) >> mm_page_shift(vma->vm_mm) : 1)) continue; /* @@ -2100,6 +2361,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * actually map pages. */ pteval = ptep_get(pvmw.pte); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_unmap_subpage(folio, vma, &pvmw, flags)) + goto walk_abort; + /* One native folio may cover four independently mapped slots. */ + continue; + } +#endif + if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { @@ -2110,6 +2380,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, } subpage = folio_page(folio, pfn - folio_pfn(folio)); + if (!folio_test_anon(folio)) + file_phys = pte_phys_mm(mm, ptep_get(pvmw.pte)); address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2167,7 +2439,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, folio_mark_dirty(folio); } else if (likely(pte_present(pteval))) { nr_pages = folio_unmap_pte_batch(folio, &pvmw, flags, pteval); - end_addr = address + nr_pages * PAGE_SIZE; + end_addr = address + nr_pages * mm_page_size(mm); flush_cache_range(vma, address, end_addr); /* Nuke the page table entry. */ @@ -2281,7 +2553,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, goto discard; } - if (folio_dup_swap(folio, subpage) < 0) { + if (folio_dup_swap_range(folio, subpage, mm_pte_native_pages(mm)) < 0) { set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2292,15 +2564,15 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, * so we'll not check/care. */ if (arch_unmap_one(mm, vma, address, pteval) < 0) { - folio_put_swap(folio, subpage); + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } /* See folio_try_share_anon_rmap(): clear PTE first. */ if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { - folio_put_swap(folio, subpage); + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { + folio_put_swap_range(folio, subpage, mm_pte_native_pages(mm)); set_pte_at(mm, address, pvmw.pte, pteval); goto walk_abort; } @@ -2342,20 +2614,26 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, add_mm_counter(mm, mm_counter_file(folio), -nr_pages); } discard: + nr_refs = nr_pages; + /* A base user leaf can own references to several native pages. */ + if (!folio_test_hugetlb(folio)) + nr_refs *= mm_pte_native_pages(mm); + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, file_phys); if (unlikely(folio_test_hugetlb(folio))) { hugetlb_remove_rmap(folio); } else { - folio_remove_rmap_ptes(folio, subpage, nr_pages, vma); + folio_remove_rmap_ptes(folio, subpage, nr_refs, vma); } if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put_refs(folio, nr_pages); + folio_put_refs(folio, nr_refs); /* * If we are sure that we batched the entire folio and cleared * all PTEs, we can just optimize and stop right here. */ - if (nr_pages == folio_nr_pages(folio)) + if (nr_pages == folio_nr_pages(folio) && mm_page_size(mm) == PAGE_SIZE) goto walk_done; continue; walk_abort: @@ -2412,11 +2690,72 @@ void try_to_unmap(struct folio *folio, enum ttu_flags flags) * If TTU_SPLIT_HUGE_PMD is specified any PMD mappings will be split into PTEs * containing migration entries. */ +#ifdef CONFIG_MM_SUBPAGE +static bool try_to_migrate_subpage(struct folio *folio, struct vm_area_struct *vma, + struct page_vma_mapped_walk *pvmw, enum ttu_flags flags) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long address = pvmw->address, end = address + mm_page_size(mm); + struct mm_subpage *slot; + pte_t pte = ptep_get(pvmw->pte), migration; + swp_entry_t entry; + bool ret = false; + + /* Other migration drivers must prepare ownership before removing PTEs. */ + if (!pte_present(pte) || !mm_subpage_migrating(folio) || + folio_test_hwpoison(folio) || folio_is_zone_device(folio)) + return false; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + if (WARN_ON_ONCE(!slot)) + return false; + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + if (should_defer_flush(mm, flags)) + set_tlb_ubc_flush_pending(mm, pte, address, end); + else + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + if (arch_unmap_one(mm, vma, address, pte) < 0) { + set_pte_at(mm, address, pvmw->pte, pte); + goto out; + } + if (pte_write(pte)) + entry = make_writable_migration_entry(folio_pfn(folio)); + else if (mm_subpage_is_exclusive(slot)) + entry = make_readable_exclusive_migration_entry(folio_pfn(folio)); + else + entry = make_readable_migration_entry(folio_pfn(folio)); + if (pte_young(pte)) + entry = make_migration_entry_young(entry); + if (pte_dirty(pte)) + entry = make_migration_entry_dirty(entry); + migration = pte_swp_set_subpage_offset(swp_entry_to_pte(entry), mm_subpage_offset(slot)); + if (pte_soft_dirty(pte)) + migration = pte_swp_mksoft_dirty(migration); + if (pte_uffd_wp(pte)) + migration = pte_swp_mkuffd_wp(migration); + set_pte_at(mm, address, pvmw->pte, migration); + trace_set_migration_pte(address, pte_val(migration), 0); + mm_subpage_migrate_save_lazyfree(slot); + mm_subpage_remove_anon_rmap(slot, vma); + if (vma->vm_flags & VM_LOCKED) + mlock_drain_local(); + /* The driver holds backing until its batched invalidation completes. */ + mm_subpage_put(slot); + ret = true; +out: + mm_subpage_put(slot); + return ret; +} +#endif + static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, - unsigned long address, void *arg) + struct rmap_walk_range rmap, void *arg) { + unsigned long address = rmap.address; struct mm_struct *mm = vma->vm_mm; - DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, address, 0); + DEFINE_FOLIO_RMAP_WALK(pvmw, folio, vma, rmap, 0); bool anon_exclusive, writable, ret = true; pte_t pteval; struct page *subpage; @@ -2424,6 +2763,11 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, enum ttu_flags flags = (enum ttu_flags)(long)arg; unsigned long pfn; unsigned long hsz = 0; + unsigned int nr_refs = 1; + + /* A base PTE owns one reference for each covered native page. */ + if (!folio_test_hugetlb(folio)) + nr_refs = mm_pte_native_pages(mm); /* * When racing against e.g. zap_pte_range() on another cpu, @@ -2432,7 +2776,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, * if page table locking is skipped: use TTU_SYNC to wait for that. */ if (flags & TTU_SYNC) - pvmw.flags = PVMW_SYNC; + pvmw.flags |= PVMW_SYNC; /* * For THP, we have to assume the worse case ie pmd for invalidation. @@ -2501,6 +2845,17 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); + +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE && folio_test_anon(folio)) { + if (!try_to_migrate_subpage(folio, vma, &pvmw, flags)) { + ret = false; + page_vma_mapped_walk_done(&pvmw); + break; + } + continue; + } +#endif /* * Handle PFN swap PTEs, such as device-exclusive ones, that * actually map pages. @@ -2586,7 +2941,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ pteval = ptep_get_and_clear(mm, address, pvmw.pte); - set_tlb_ubc_flush_pending(mm, pteval, address, address + PAGE_SIZE); + set_tlb_ubc_flush_pending(mm, pteval, address, address + mm_page_size(mm)); } else { pteval = ptep_clear_flush(vma, address, pvmw.pte); } @@ -2663,7 +3018,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, break; } } else if (anon_exclusive && - folio_try_share_anon_rmap_pte(folio, subpage)) { + folio_try_share_anon_rmap_pte(folio, subpage, vma)) { set_pte_at(mm, address, pvmw.pte, pteval); ret = false; page_vma_mapped_walk_done(&pvmw); @@ -2701,6 +3056,13 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, if (pte_swp_uffd_wp(pteval)) swp_pte = pte_swp_mkuffd_wp(swp_pte); } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) + swp_pte = pte_swp_set_subpage_offset(swp_pte, + pte_present(pteval) ? pte_phys_mm(mm, pteval) & ~PAGE_MASK : + pte_swp_subpage_offset(pteval)); +#endif + if (folio_test_hugetlb(folio)) set_huge_pte_at(mm, address, pvmw.pte, swp_pte, hsz); @@ -2714,13 +3076,15 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ } + if (!folio_test_anon(folio)) + mm_subpage_file_map_del(vma, pte_phys_mm(mm, pteval)); if (unlikely(folio_test_hugetlb(folio))) hugetlb_remove_rmap(folio); else folio_remove_rmap_pte(folio, subpage, vma); if (vma->vm_flags & VM_LOCKED) mlock_drain_local(); - folio_put(folio); + folio_put_refs(folio, nr_refs); } mmu_notifier_invalidate_range_end(&range); @@ -2951,6 +3315,61 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio, return anon_vma; } +#ifdef CONFIG_MM_SUBPAGE +noinline bool rmap_walk_subpages(struct folio *folio, + struct rmap_walk_control *rwc, bool locked) +{ + struct mm_subpage_rmap maps[MM_SUBPAGES_PER_PAGE]; + int i, nr = mm_subpage_snapshot_rmap(folio, maps); + + if (nr < 0) + return false; + /* Existing locked anonymous walks operate on large folios. */ + VM_BUG_ON_FOLIO(locked, folio); + for (i = 0; i < nr; i++) { + struct mm_subpage_rmap *map = &maps[i]; + struct anon_vma_chain *avc; + bool stop = false; + + if (!anon_vma_trylock_read(map->root)) { + if (rwc->try_lock) { + rwc->contended = true; + break; + } + anon_vma_lock_read(map->root); + } + anon_vma_interval_tree_foreach(avc, &map->root->rb_root, + map->pos.index, map->pos.index) { + struct vm_area_struct *vma = avc->vma; + unsigned long address = vma_address_at_offset(vma, map->pos); + struct rmap_walk_range range = { + .address = address, + .subpage = true, + .subpage_offset = map->offset, + }; + + /* Native-index overlap need not include this exact leaf. */ + if (address == -EFAULT) + continue; + cond_resched(); + if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) + continue; + if (!rwc->rmap_one(folio, vma, range, rwc->arg) || + (rwc->done && rwc->done(folio))) { + stop = true; + break; + } + } + anon_vma_unlock_read(map->root); + if (stop) + break; + } + for (i = 0; i < nr; i++) + put_anon_vma(maps[i].root); + return true; +} +#endif + /* * rmap_walk_anon - do something to anonymous page using the object-based * rmap method @@ -2973,6 +3392,10 @@ static void rmap_walk_anon(struct folio *folio, * to an anon_vma with different root. */ VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); +#ifdef CONFIG_MM_SUBPAGE + if (!folio_test_large(folio) && rmap_walk_subpages(folio, rwc, locked)) + return; +#endif if (locked) { anon_vma = folio_anon_vma(folio); @@ -2998,7 +3421,8 @@ static void rmap_walk_anon(struct folio *folio, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) break; if (rwc->done && rwc->done(folio)) break; @@ -3061,7 +3485,8 @@ static void __rmap_walk_file(struct folio *folio, struct address_space *mapping, if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; - if (!rwc->rmap_one(folio, vma, address, rwc->arg)) + if (!rwc->rmap_one(folio, vma, + (struct rmap_walk_range) { .address = address }, rwc->arg)) goto done; if (rwc->done && rwc->done(folio)) goto done; diff --git a/mm/shmem.c b/mm/shmem.c index 7b1ea9fb598f4a..705765dddf1d22 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -30,6 +30,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -875,6 +878,266 @@ static void shmem_update_stats(struct folio *folio, int nr_pages) lruvec_stat_mod_folio(folio, NR_SHMEM, nr_pages); } +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +#define SHMEM_UFFD_ALL_UNITS GENMASK(PAGE_SIZE / SZ_4K - 1, 0) + +/* + * Resident masks live in page_ext, allowing partial truncate to update them + * without allocation. All cache insertion/replacement paths initialize or + * transfer them under the folio lock. The inode XArray persists masks only + * for swapped indices. Its publication/removal additionally holds i_pages, + * before taking uffd_missing's XArray lock. + */ +static void shmem_uffd_enable(struct inode *inode) +{ + struct shmem_inode_info *info = SHMEM_I(inode); + + spin_lock(&info->lock); + info->flags |= SHMEM_F_UFFD_SUBPAGE; + spin_unlock(&info->lock); +} + +static unsigned long shmem_uffd_mask(unsigned int offset, unsigned long length) +{ + return GENMASK((offset + length - 1) / SZ_4K, offset / SZ_4K); +} + +unsigned long shmem_uffd_missing_mask(struct folio *folio, pgoff_t index) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); + VM_BUG_ON_FOLIO(!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)), folio); + return mm_subpage_shmem_missing(folio_page(folio, index - folio->index)); +} + +int shmem_uffd_set_missing(struct folio *folio, pgoff_t index, + unsigned long missing) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!folio_mapping(folio) || !shmem_mapping(folio_mapping(folio)) || + !folio_contains(folio, index) || + !folio_test_uptodate(folio) || (missing & ~SHMEM_UFFD_ALL_UNITS)) + return -EINVAL; + if (missing) + shmem_uffd_enable(folio->mapping->host); + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), missing); + return 0; +} + +bool shmem_uffd_has_missing_native_page(struct folio *folio) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long i; + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return false; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + /* Native PMD exposure is allowed when every native leaf has some data. */ + for (i = 0; i < folio_nr_pages(folio); i++) + if (mm_subpage_shmem_missing(folio_page(folio, i)) == SHMEM_UFFD_ALL_UNITS) + return true; + return false; +} + +bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, + unsigned int offset, unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + unsigned long mask; + + if (!mapping || !shmem_mapping(mapping)) + return false; + mask = shmem_uffd_mask(offset, length); + return (shmem_uffd_missing_mask(folio, index) & mask) == mask; +} + +/* A coarse alias with any populated units preserves them and exposes zero holes. */ +static unsigned long shmem_uffd_reason(struct vm_fault *vmf, unsigned long missing) +{ + struct vm_area_struct *vma = vmf->vma; + struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); + unsigned long mask = shmem_uffd_mask(pos.offset, mm_page_size(vma->vm_mm)); + + if ((missing & mask) == mask) + return userfaultfd_missing(vma) ? VM_UFFD_MISSING : 0; + return userfaultfd_minor(vma) ? VM_UFFD_MINOR : 0; +} + +void shmem_uffd_mark_present(struct folio *folio, pgoff_t index, + unsigned long present) +{ + unsigned long missing = shmem_uffd_missing_mask(folio, index); + + if (missing & present) + mm_subpage_shmem_set_missing(folio_page(folio, index - folio->index), + missing & ~present); +} + +void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, + unsigned long length) +{ + struct address_space *mapping = folio_mapping(folio); + + if (!mapping || !shmem_mapping(mapping) || + !(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(offset + length > folio_size(folio), folio); + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long size = min(length, PAGE_SIZE - low); + unsigned long mask = GENMASK((low + size - 1) / SZ_4K, low / SZ_4K); + + shmem_uffd_mark_present(folio, folio->index + (offset >> PAGE_SHIFT), mask); + offset += size; + length -= size; + } +} + +void shmem_uffd_forget(struct address_space *mapping, pgoff_t start, + unsigned long nr) +{ + struct xarray *xa; + unsigned long index; + void *entry; + + if (!shmem_mapping(mapping)) + return; + lockdep_assert_held(&mapping->i_pages.xa_lock); + xa = &SHMEM_I(mapping->host)->uffd_missing; + if (xa_empty(xa)) + return; + xa_lock(xa); + xa_for_each_range(xa, index, entry, start, start + nr - 1) + __xa_erase(xa, index); + xa_unlock(xa); +} + +/* New cache identity or swap-in: initialize every native page before unlock. */ +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + unsigned long i; + + lockdep_assert_held(&mapping->i_pages.xa_lock); + for (i = 0; i < folio_nr_pages(folio); i++) { + void *entry = from_swap ? xa_load(xa, folio->index + i) : NULL; + + mm_subpage_shmem_set_missing(folio_page(folio, i), + entry ? xa_to_value(entry) : 0); + } + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); +} + +/* + * Reserve persistence before committing swap. On failure the cache folio and + * its resident mask stay intact; reclaim may retry once memory is available. + */ +static int shmem_uffd_swap_prepare(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct xarray *xa = &SHMEM_I(mapping->host)->uffd_missing; + struct mem_cgroup *memcg, *saved; + unsigned long i; + int err = 0; + + if (!(READ_ONCE(SHMEM_I(mapping->host)->flags) & SHMEM_F_UFFD_SUBPAGE)) + return 0; + /* Reclaim may run outside the backing folio's charge domain. */ + rcu_read_lock(); + do { + memcg = folio_memcg(folio) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + for (i = 0; i < folio_nr_pages(folio); i++) { + unsigned long mask = mm_subpage_shmem_missing(folio_page(folio, i)); + + if (!mask) + continue; + err = xa_err(xa_store(xa, folio->index + i, xa_mk_value(mask), + GFP_NOWAIT | __GFP_ACCOUNT | __GFP_NOWARN)); + if (err) + break; + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + if (err) { + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); + } + return err; +} + +void shmem_uffd_copy_page(struct page *dst, struct page *src) +{ + mm_subpage_shmem_set_missing(dst, src ? mm_subpage_shmem_missing(src) : 0); +} + +void shmem_uffd_migrate(struct folio *dst, struct folio *src) +{ + unsigned long i; + + if (!shmem_mapping(src->mapping)) + return; + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + for (i = 0; i < folio_nr_pages(src); i++) + shmem_uffd_copy_page(folio_page(dst, i), folio_page(src, i)); +} + +/* Only fully removed 4K units become holes; sub-unit edge bytes stay valid. */ +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) +{ + loff_t first = max(start, folio_pos(folio)); + u64 last = min_t(u64, end, folio_next_pos(folio) - 1); + unsigned long offset, length; + bool pinned; + + if (first >= last || + (first == folio_pos(folio) && last == folio_next_pos(folio) - 1)) + return; + first = round_up(first, SZ_4K); + last = round_down(last + 1, SZ_4K); + if (first >= last) + return; + shmem_uffd_enable(folio->mapping->host); + /* Revoke covering coarse aliases before their holes can be refilled. */ + unmap_mapping_subpage_hole(folio->mapping, first, last); + /* Pair with fast GUP's pin-before-PTE-recheck barrier. */ + smp_mb(); + pinned = folio_maybe_dma_pinned(folio); + offset = first - folio_pos(folio); + length = last - first; + while (length) { + unsigned int low = offset & ~PAGE_MASK; + unsigned long nr = min(length, PAGE_SIZE - low); + struct page *page = folio_page(folio, offset >> PAGE_SHIFT); + unsigned long mask = shmem_uffd_mask(low, nr); + unsigned long missing = mm_subpage_shmem_missing(page); + + /* + * A pin survives PTE removal and can modify retained backing. + * Preserve native partial-hole semantics: zero it, but do not + * let a missing-page COPY overwrite a subsequent device write. + * Pins have folio granularity, so exposure is conservative when + * only a neighbouring range is pinned. Full-folio removal still + * detaches old pins from the file's new cache identity. + */ + mm_subpage_shmem_set_missing(page, pinned ? missing & ~mask : missing | mask); + offset += nr; + length -= nr; + } +} +#else +static void shmem_uffd_cache_init(struct folio *folio, bool from_swap) { } +static int shmem_uffd_swap_prepare(struct folio *folio) { return 0; } +static void shmem_uffd_partial_hole(struct folio *folio, loff_t start, uoff_t end) { } +#endif + /* * Somewhat like filemap_add_folio, but error if expected item has gone. */ @@ -922,6 +1185,7 @@ int shmem_add_to_page_cache(struct folio *folio, xas_store(&xas, folio); if (xas_error(&xas)) goto unlock; + shmem_uffd_cache_init(folio, expected != NULL); shmem_update_stats(folio, nr); mapping->nrpages += nr; unlock: @@ -976,8 +1240,10 @@ static long shmem_free_swap(struct address_space *mapping, base = round_down(xas.xa_index, nr_pages); if (base < index || base + nr_pages - 1 > end) nr_pages = 0; - else + else { + shmem_uffd_forget(mapping, base, nr_pages); xas_store(&xas, NULL); + } } xas_unlock_irq(&xas); @@ -1020,6 +1286,39 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, return swapped << PAGE_SHIFT; } +/* Byte-clipped variant for mappings smaller than a native cache page. */ +unsigned long shmem_partial_swap_usage_bytes(struct address_space *mapping, + u64 start, u64 end) +{ + XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT); + void *entry; + unsigned long swapped = 0, last; + + if (start >= end) + return 0; + last = (end - 1) >> PAGE_SHIFT; + rcu_read_lock(); + xas_for_each(&xas, entry, last) { + if (xas_retry(&xas, entry)) + continue; + if (xa_is_value(entry)) { + unsigned int order = xas_get_order(&xas); + u64 first = ((u64)xas.xa_index & ~((1UL << order) - 1)) << PAGE_SHIFT; + u64 stop = first + ((u64)PAGE_SIZE << order); + + swapped += min(stop, end) - max(first, start); + } + if (xas.xa_index == last) + break; + if (need_resched()) { + xas_pause(&xas); + cond_resched_rcu(); + } + } + rcu_read_unlock(); + return swapped; +} + /* * Determine (in bytes) how many of the shmem object's pages mapped by the * given vma is swapped out. @@ -1044,6 +1343,12 @@ unsigned long shmem_swap_usage(struct vm_area_struct *vma) */ if (!swapped) return 0; + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { + u64 start = ((u64)vma->vm_pgoff << PAGE_SHIFT) + vma_subpage_offset(vma); + + return shmem_partial_swap_usage_bytes(mapping, start, + start + vma->vm_end - vma->vm_start); + } if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) return swapped << PAGE_SHIFT; @@ -1101,6 +1406,12 @@ static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) return folio; } +static bool shmem_truncate_partial_folio(struct folio *folio, loff_t start, uoff_t end) +{ + shmem_uffd_partial_hole(folio, start, end); + return truncate_inode_partial_folio(folio, start, end); +} + /* * Remove range of pages and swap entries from page cache, and free them. * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. @@ -1164,7 +1475,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (folio) { same_folio = lend < folio_next_pos(folio); folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) { + if (!shmem_truncate_partial_folio(folio, lstart, lend)) { start = folio_next_index(folio); if (same_folio) end = folio->index; @@ -1178,7 +1489,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); if (folio) { folio_mark_dirty(folio); - if (!truncate_inode_partial_folio(folio, lstart, lend)) + if (!shmem_truncate_partial_folio(folio, lstart, lend)) end = folio->index; folio_unlock(folio); folio_put(folio); @@ -1247,7 +1558,7 @@ static void shmem_undo_range(struct inode *inode, loff_t lstart, uoff_t lend, if (!folio_test_large(folio)) { truncate_inode_folio(mapping, folio); - } else if (truncate_inode_partial_folio(folio, lstart, lend)) { + } else if (shmem_truncate_partial_folio(folio, lstart, lend)) { /* * If we split a page, reset the loop so * that we pick up the new sub pages. @@ -1355,6 +1666,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, } if (newsize <= oldsize) { loff_t holebegin = round_up(newsize, PAGE_SIZE); + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1362,6 +1674,7 @@ static int shmem_setattr(struct mnt_idmap *idmap, shmem_truncate_range(inode, newsize, (loff_t)-1); /* unmap again to remove racily COWed private pages */ + unmap_mapping_subpage_tail(inode->i_mapping, newsize); if (oldsize > holebegin) unmap_mapping_range(inode->i_mapping, holebegin, 0, 1); @@ -1677,6 +1990,8 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, folio_mark_uptodate(folio); } + if (shmem_uffd_swap_prepare(folio)) + goto redirty; if (!folio_alloc_swap(folio)) { bool first_swapped = shmem_recalc_inode(inode, 0, nr_pages); int error; @@ -1730,6 +2045,10 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, swap_cache_del_folio(folio); goto redirty; } + /* Allocation failed: no swap entry owns the reserved inode metadata. */ + xa_lock_irq(&mapping->i_pages); + shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); + xa_unlock_irq(&mapping->i_pages); if (nr_pages > 1) goto try_split; redirty: @@ -1934,6 +2253,23 @@ static struct folio *shmem_alloc_folio(gfp_t gfp, int order, return folio; } +/* Allocate unexposed promotion backing with the inode's zone/NUMA policy. */ +struct folio *shmem_alloc_collapse_folio(struct mm_struct *mm, struct inode *inode, + pgoff_t index, unsigned int order) +{ + gfp_t gfp = limit_gfp_mask(GFP_TRANSHUGE, mapping_gfp_mask(inode->i_mapping)); + struct folio *folio; + + folio = shmem_alloc_folio(gfp, order, SHMEM_I(inode), index); + if (!folio) + return ERR_PTR(-ENOMEM); + if (mem_cgroup_charge(folio, mm, gfp)) { + folio_put(folio); + return ERR_PTR(-ENOMEM); + } + return folio; +} + static struct folio *shmem_alloc_and_add_folio(struct vm_fault *vmf, gfp_t gfp, struct inode *inode, pgoff_t index, struct mm_struct *fault_mm, unsigned long orders) @@ -2484,13 +2820,36 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, fault_mm = vma ? vma->vm_mm : NULL; folio = filemap_get_entry(inode->i_mapping, index); - if (folio && vma && userfaultfd_minor(vma)) { + if (folio && vma && userfaultfd_minor(vma) && + !(READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { if (!xa_is_value(folio)) folio_put(folio); *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (xa_is_value(folio) && vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason; + void *partial; + + /* Cache identity and its persisted mask must describe the same entry. */ + xa_lock_irq(&inode->i_mapping->i_pages); + if (xa_load(&inode->i_mapping->i_pages, index) != folio) { + xa_unlock_irq(&inode->i_mapping->i_pages); + goto repeat; + } + partial = xa_load(&SHMEM_I(inode)->uffd_missing, index); + reason = shmem_uffd_reason(vmf, partial ? xa_to_value(partial) : 0); + xa_unlock_irq(&inode->i_mapping->i_pages); + if (reason) { + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif + if (xa_is_value(folio)) { error = shmem_swapin_folio(inode, index, &folio, sgp, gfp, vma, fault_type); @@ -2510,6 +2869,20 @@ static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, folio_put(folio); goto repeat; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + if (vma && userfaultfd_armed(vma) && + (READ_ONCE(SHMEM_I(inode)->flags) & SHMEM_F_UFFD_SUBPAGE)) { + unsigned long reason = shmem_uffd_reason(vmf, + shmem_uffd_missing_mask(folio, index)); + + if (reason) { + folio_unlock(folio); + folio_put(folio); + *fault_type = handle_userfault(vmf, reason); + return 0; + } + } +#endif if (sgp == SGP_WRITE) folio_mark_accessed(folio); if (folio_test_uptodate(folio)) @@ -2748,15 +3121,33 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) struct folio *folio = NULL; vm_fault_t ret = 0; int err; +#ifdef CONFIG_MM_SUBPAGE + bool promoted = false; + unsigned long user_size = mm_page_size(vmf->vma->vm_mm); + + /* Missing coarse PTEs require a blocking caller to prepare backing. */ + if (user_size > PAGE_SIZE && (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return VM_FAULT_RETRY; +#endif +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) + return VM_FAULT_SIGBUS; +#endif + +#ifdef CONFIG_MM_SUBPAGE +retry: +#endif /* * Trinity finds that probing a hole which tmpfs is punching can * prevent the hole-punch from ever completing: noted in i_private. */ if (unlikely(inode->i_private)) { - ret = shmem_falloc_wait(vmf, inode); - if (ret) - return ret; + vm_fault_t wait = shmem_falloc_wait(vmf, inode); + + if (wait) + return ret | wait; } WARN_ON_ONCE(vmf->page != NULL); @@ -2765,6 +3156,45 @@ static vm_fault_t shmem_fault(struct vm_fault *vmf) if (err) return vmf_error(err); if (folio) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vmf->vma->vm_mm) < PAGE_SIZE && + !vma_file_offset_valid(vmf->vma, vmf->address, i_size_read(inode))) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (user_size > PAGE_SIZE) { + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + unsigned long nr = user_size >> PAGE_SHIFT; + + if (pos.offset || !IS_ALIGNED(pos.index, nr)) { + folio_unlock(folio); + folio_put(folio); + return VM_FAULT_SIGBUS; + } + if (folio_size(folio) < user_size) { + folio_unlock(folio); + folio_put(folio); + folio = NULL; + /* Bound work if splitting races a successful promotion. */ + if (promoted) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + err = collapse_file_user_page(vmf->vma->vm_mm, + vmf->vma->vm_file, pos.index); + if (err == -EAGAIN) { + cond_resched(); + return ret | VM_FAULT_NOPAGE; + } + if (err) + return vmf_error(err); + promoted = true; + goto retry; + } + } +#endif + vmf->page = folio_file_page(folio, vmf->pgoff); ret |= VM_FAULT_LOCKED; } @@ -2899,7 +3329,7 @@ static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). */ *ilx = inode->i_ino; - index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; + index = vma_page_offset_at(vma, addr).index; return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); } @@ -3086,6 +3516,9 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, memset(info, 0, (char *)inode - (char *)info); INIT_LIST_HEAD_RCU(&info->xattrs); spin_lock_init(&info->lock); +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; info->flags = vma_flags_test(&flags, VMA_NORESERVE_BIT) @@ -3245,6 +3678,57 @@ static void shmem_mfill_filemap_remove(struct folio *folio, folio_unlock(folio); } +#ifdef CONFIG_MM_SUBPAGE +struct folio *shmem_uffd_prepare_folio(struct vm_area_struct *vma, + unsigned long address, bool *new) +{ + struct inode *inode = file_inode(vma->vm_file); + pgoff_t index = vma_page_offset_at(vma, address).index; + struct folio *folio; + int err; + + *new = false; + /* Must precede cache insertion: minor faults must wait for initialization. */ + shmem_uffd_enable(inode); +retry: + err = shmem_get_folio(inode, index, 0, &folio, SGP_NOALLOC); + if (!err) + return folio; + if (err != -ENOENT) + return ERR_PTR(err); + folio = shmem_mfill_folio_alloc(vma, address); + if (!folio) + return ERR_PTR(-ENOMEM); + clear_highpage(&folio->page); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + err = shmem_mfill_filemap_add(folio, vma, address); + if (err) { + folio_put(folio); + if (err == -EEXIST) + goto retry; + return ERR_PTR(err); + } + err = shmem_uffd_set_missing(folio, index, SHMEM_UFFD_ALL_UNITS); + if (err) { + shmem_mfill_filemap_remove(folio, vma); + folio_put(folio); + return ERR_PTR(err); + } + *new = true; + return folio; +} + +void shmem_uffd_abort_folio(struct folio *folio, struct vm_area_struct *vma, bool new) +{ + if (new) + shmem_mfill_filemap_remove(folio, vma); + else + folio_unlock(folio); + folio_put(folio); +} +#endif + static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff) { struct folio *folio; @@ -3331,6 +3815,7 @@ shmem_write_end(const struct kiocb *iocb, struct address_space *mapping, } folio_mark_uptodate(folio); } + shmem_uffd_mark_mapped(folio, offset_in_folio(folio, pos), copied); folio_mark_dirty(folio); folio_unlock(folio); folio_put(folio); @@ -5143,6 +5628,10 @@ static struct inode *shmem_alloc_inode(struct super_block *sb) info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); if (!info) return NULL; +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + /* Also cover inode-init failure before __shmem_get_inode() runs. */ + xa_init_flags(&info->uffd_missing, XA_FLAGS_ACCOUNT); +#endif return &info->vfs_inode; } @@ -5155,6 +5644,9 @@ static void shmem_free_in_core_inode(struct inode *inode) static void shmem_destroy_inode(struct inode *inode) { +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + xa_destroy(&SHMEM_I(inode)->uffd_missing); +#endif if (S_ISREG(inode->i_mode)) mpol_free_shared_policy(&SHMEM_I(inode)->policy); if (S_ISDIR(inode->i_mode)) @@ -5972,6 +6464,8 @@ struct folio *shmem_read_folio_gfp(struct address_space *mapping, if (error) return ERR_PTR(error); + /* A kernel caller receives and may modify the complete native folio. */ + shmem_uffd_mark_mapped(folio, 0, folio_size(folio)); folio_unlock(folio); return folio; #else diff --git a/mm/swap.h b/mm/swap.h index a77016f2423b90..5d064cab36f4c5 100644 --- a/mm/swap.h +++ b/mm/swap.h @@ -10,9 +10,12 @@ extern int page_cluster; #ifdef CONFIG_THP_SWAP #define SWAPFILE_CLUSTER HPAGE_PMD_NR -#define swap_entry_order(order) (order) #else #define SWAPFILE_CLUSTER 256 +#endif +#if defined(CONFIG_THP_SWAP) || defined(CONFIG_MM_SUBPAGE) +#define swap_entry_order(order) (order) +#else #define swap_entry_order(order) 0 #endif @@ -38,6 +41,9 @@ struct swap_cluster_info { u8 order; atomic_long_t __rcu *table; /* Swap table entries, see mm/swap_table.h */ unsigned int *extend_table; /* For large swap count, protected by ci->lock */ +#ifdef CONFIG_MM_SUBPAGE + unsigned int *subpage_counts; /* Per-quarter PTE refs, protected by ci->lock */ +#endif struct list_head list; }; @@ -207,6 +213,8 @@ extern int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp); */ int folio_alloc_swap(struct folio *folio); int folio_dup_swap(struct folio *folio, struct page *subpage); +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr); +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr); void folio_put_swap(struct folio *folio, struct page *subpage); /* For internal use */ @@ -216,6 +224,7 @@ extern void __swap_cluster_free_entries(struct swap_info_struct *si, /* linux/mm/page_io.c */ int sio_pool_init(void); +int swap_extent_io_init(void); struct swap_iocb; void swap_read_folio(struct folio *folio, struct swap_iocb **plug); void __swap_read_unplug(struct swap_iocb *plug); @@ -399,6 +408,17 @@ static inline void folio_put_swap(struct folio *folio, struct page *page) { } +static inline int folio_dup_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ + return -EINVAL; +} + +static inline void folio_put_swap_range(struct folio *folio, struct page *page, + unsigned int nr) +{ +} + static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) { } diff --git a/mm/swap_state.c b/mm/swap_state.c index 1415a5c54a43b2..db1285bc1880dd 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -427,6 +427,10 @@ void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, if (unlikely(folio_test_large(folio))) return; + /* Small-granule PTEs currently use native swap-cluster readahead. */ + if (vma && mm_page_size(vma->vm_mm) != PAGE_SIZE) + vma = NULL; + readahead = folio_test_clear_readahead(folio); if (vma && vma_ra) { unsigned long ra_val; @@ -896,7 +900,7 @@ struct folio *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, struct folio *folio; mpol = get_vma_policy(vmf->vma, vmf->address, 0, &ilx); - folio = swap_use_vma_readahead() ? + folio = swap_use_vma_readahead() && mm_page_size(vmf->vma->vm_mm) == PAGE_SIZE ? swap_vma_readahead(entry, gfp_mask, mpol, ilx, vmf) : swap_cluster_readahead(entry, gfp_mask, mpol, ilx); mpol_cond_put(mpol); diff --git a/mm/swapfile.c b/mm/swapfile.c index 74a1e324449dc6..e5faba70227c07 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -329,17 +330,22 @@ offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset) BUG(); } +/* The extent tree is immutable while the swap entry is owned. */ +sector_t swap_extent_sector(swp_entry_t entry, unsigned long *nr_pages) +{ + struct swap_info_struct *sis = __swap_entry_to_info(entry); + pgoff_t offset = swp_offset(entry); + struct swap_extent *se = offset_to_swap_extent(sis, offset); + + *nr_pages = min(*nr_pages, se->start_page + se->nr_pages - offset); + return (se->start_block + offset - se->start_page) << (PAGE_SHIFT - 9); +} + sector_t swap_folio_sector(struct folio *folio) { - struct swap_info_struct *sis = __swap_entry_to_info(folio->swap); - struct swap_extent *se; - sector_t sector; - pgoff_t offset; + unsigned long nr = folio_nr_pages(folio); - offset = swp_offset(folio->swap); - se = offset_to_swap_extent(sis, offset); - sector = se->start_block + (offset - se->start_page); - return sector << (PAGE_SHIFT - 9); + return swap_extent_sector(folio->swap, &nr); } /* @@ -479,6 +485,16 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci) lockdep_assert_held(&ci->lock); table = (void *)rcu_dereference_protected(ci->table, true); rcu_assign_pointer(ci->table, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (ci->subpage_counts) { + unsigned int i; + + for (i = 0; i < SWAPFILE_CLUSTER * (PAGE_SIZE / MM_SUBPAGE_SIZE); i++) + VM_WARN_ON_ONCE(ci->subpage_counts[i]); + kfree(ci->subpage_counts); + ci->subpage_counts = NULL; + } +#endif swap_table_free(table); } @@ -1067,6 +1083,19 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +{ + unsigned int flags; + + if (!folio || !folio_order(folio)) + return true; + flags = READ_ONCE(si->flags); + if (flags & SWP_BLKDEV) + return true; + /* Filesystem swap_rw implementations retain their existing contract. */ + return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); +} + /* * Try to allocate swap entries with specified order and try set a new * cluster for current CPU too. @@ -1078,11 +1107,7 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - /* - * Swapfile is not block device so unable - * to allocate large entries. - */ - if (order && !(si->flags & SWP_BLKDEV)) + if (!swap_folio_alloc_allowed(si, folio)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1361,6 +1386,10 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; + if (!swap_folio_alloc_allowed(si, folio)) { + put_swap_device(si); + return false; + } ci = swap_cluster_lock(si, offset); if (cluster_is_usable(ci, order)) { @@ -1543,7 +1572,7 @@ static void __swap_cluster_put_entry(struct swap_cluster_info *ci, */ static void swap_put_entries_cluster(struct swap_info_struct *si, pgoff_t offset, int nr, - bool reclaim_cache) + bool reclaim_cache, int subpage) { struct swap_cluster_info *ci; unsigned int ci_off, ci_end; @@ -1556,6 +1585,17 @@ static void swap_put_entries_cluster(struct swap_info_struct *si, ci = swap_cluster_lock(si, offset); ci_off = offset % SWAPFILE_CLUSTER; ci_end = ci_off + nr; +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_off * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + if (WARN_ON_ONCE(nr != 1 || !ci->subpage_counts || !ci->subpage_counts[index])) { + swap_cluster_unlock(ci); + return; + } + ci->subpage_counts[index]--; + } +#endif do { swp_tb = __swap_table_get(ci, ci_off); if (swp_tb_get_count(swp_tb) == 1) { @@ -1649,7 +1689,7 @@ static int __swap_cluster_dup_entry(struct swap_cluster_info *ci, * and failed to allocate an extended table, -EINVAL if any entry is bad entry. */ static int swap_dup_entries_cluster(struct swap_info_struct *si, - pgoff_t offset, int nr) + pgoff_t offset, int nr, int subpage) { int err; struct swap_cluster_info *ci; @@ -1659,6 +1699,25 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, ci_end = ci_start + nr; ci_off = ci_start; ci = swap_cluster_lock(si, offset); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) { + unsigned int index = ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage; + + VM_BUG_ON(nr != 1); + if (!ci->subpage_counts) + ci->subpage_counts = kcalloc(SWAPFILE_CLUSTER, + (PAGE_SIZE / MM_SUBPAGE_SIZE) * sizeof(*ci->subpage_counts), + GFP_ATOMIC | __GFP_NOWARN); + if (!ci->subpage_counts) { + err = -ENOMEM; + goto failed; + } + if (ci->subpage_counts[index] == UINT_MAX) { + err = -EOVERFLOW; + goto failed; + } + } +#endif restart: do { err = __swap_cluster_dup_entry(ci, ci_off); @@ -1673,6 +1732,10 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, goto failed; } } while (++ci_off < ci_end); +#ifdef CONFIG_MM_SUBPAGE + if (subpage >= 0) + ci->subpage_counts[ci_start * (PAGE_SIZE / MM_SUBPAGE_SIZE) + subpage]++; +#endif swap_cluster_unlock(ci); return 0; failed: @@ -1683,6 +1746,132 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si, return err; } +#ifdef CONFIG_MM_SUBPAGE +static bool swap_subpage_offset_valid(unsigned int offset) +{ + return offset < PAGE_SIZE && IS_ALIGNED(offset, MM_SUBPAGE_SIZE); +} + +int swap_subpage_dup(swp_entry_t entry, unsigned int offset) +{ + struct swap_info_struct *si = swap_entry_to_info(entry); + + if (WARN_ON_ONCE(!si || !swap_subpage_offset_valid(offset))) + return -EINVAL; + /* A locked swap-cache folio also permits the initial zero-to-one ref. */ + return swap_dup_entries_cluster(si, swp_offset(entry), 1, offset >> MM_SUBPAGE_SHIFT); +} + +void swap_subpage_put(swp_entry_t entry, unsigned int offset, bool reclaim_cache) +{ + struct swap_info_struct *si; + + if (WARN_ON_ONCE(!swap_subpage_offset_valid(offset))) + return; + si = get_swap_device(entry); + if (WARN_ON_ONCE(!si)) + return; + swap_put_entries_cluster(si, swp_offset(entry), 1, reclaim_cache, + offset >> MM_SUBPAGE_SHIFT); + put_swap_device(si); +} + +unsigned int swap_subpage_count(swp_entry_t entry, unsigned int offset) +{ + struct swap_cluster_info *ci; + struct swap_info_struct *si; + unsigned int count = 0; + + if (!swap_subpage_offset_valid(offset)) + return 0; + si = get_swap_device(entry); + if (!si) + return 0; + ci = swap_cluster_lock(si, swp_offset(entry)); + if (ci->subpage_counts) + count = ci->subpage_counts[swp_cluster_offset(entry) * + (PAGE_SIZE / MM_SUBPAGE_SIZE) + (offset >> MM_SUBPAGE_SHIFT)]; + swap_cluster_unlock(ci); + put_swap_device(si); + return count; +} +#endif + +/* + * Optional THPs can split if their order-specific cluster lists have no room. + * A mandatory base user page cannot: find a free aligned group without keeping + * the per-CPU or device allocation lock held over a device-wide scan. + */ +static void swap_alloc_required_from_device(struct swap_info_struct *si, + struct folio *folio) +{ + unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); + unsigned long start; + + if (!swap_folio_alloc_allowed(si, folio)) + return; + for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { + struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); + unsigned long offset, end = min(start + SWAPFILE_CLUSTER, si->max); + bool found = false; + + if (!(READ_ONCE(si->flags) & SWP_WRITEOK)) + break; + if (!spin_trylock(&ci->lock)) + goto next; + /* Order zero's usability test permits occupied clusters of any order. */ + if (!cluster_is_usable(ci, 0) || ci->count + nr > SWAPFILE_CLUSTER) + goto unlock; + for (offset = start; offset + nr <= end; offset += nr) { + bool need_reclaim = false; + + /* Only genuinely free slots qualify; retain every existing owner. */ + if (!cluster_scan_range(si, ci, offset, nr, &need_reclaim) || need_reclaim) + continue; + if (!__swap_cluster_alloc_entries(si, ci, folio, offset - start)) + break; + ci->order = order; + if (ci->count < SWAPFILE_CLUSTER && ci->flags == CLUSTER_FLAG_FRAG) { + /* Same flag, different order: relocate the list explicitly. */ + spin_lock(&si->lock); + list_move_tail(&ci->list, &si->frag_clusters[order]); + spin_unlock(&si->lock); + } else { + relocate_cluster(si, ci); + } + found = true; + break; + } +unlock: + swap_cluster_unlock(ci); + if (found) + return; +next: + cond_resched(); + } +} + +static void swap_alloc_required(struct folio *folio) +{ + struct swap_info_struct *devices[MAX_SWAPFILES], *si; + unsigned int count = 0, i; + + if (get_nr_swap_pages() < folio_nr_pages(folio)) + return; + /* Pin a bounded snapshot in the allocator's existing priority order. */ + spin_lock(&swap_avail_lock); + plist_for_each_entry(si, &swap_avail_head, avail_list) { + if (get_swap_device_info(si)) + devices[count++] = si; + } + spin_unlock(&swap_avail_lock); + for (i = 0; i < count; i++) { + if (!folio_test_swapcache(folio)) + swap_alloc_required_from_device(devices[i], folio); + put_swap_device(devices[i]); + } +} + /** * folio_alloc_swap - allocate swap space for a folio * @folio: folio we want to move to swap @@ -1697,16 +1886,17 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; + bool mandatory = order && folio_anon_min_order(folio); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); if (order) { /* - * Reject large allocation when THP_SWAP is disabled, - * the caller should split the folio and try again. + * Optional THP backing may split when THP_SWAP is disabled. + * A larger base user page must retain contiguous swap ownership. */ - if (!IS_ENABLED(CONFIG_THP_SWAP)) + if (!IS_ENABLED(CONFIG_THP_SWAP) && !mandatory) return -EAGAIN; /* @@ -1725,6 +1915,9 @@ int folio_alloc_swap(struct folio *folio) swap_alloc_slow(folio); local_unlock(&percpu_swap_cluster.lock); + if (mandatory && unlikely(!folio_test_swapcache(folio))) + swap_alloc_required(folio); + if (!order && unlikely(!folio_test_swapcache(folio))) { if (swap_sync_discard()) goto again; @@ -1757,6 +1950,24 @@ int folio_alloc_swap(struct folio *folio) * swap_put_entries_direct on its swap entry before this helper returns, or * the swap count may underflow. */ +int folio_dup_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + return swap_dup_entries_cluster(swap_entry_to_info(entry), swp_offset(entry), nr, -1); +} + +void folio_put_swap_range(struct folio *folio, struct page *page, unsigned int nr) +{ + swp_entry_t entry = page_swap_entry(page); + + VM_BUG_ON_FOLIO(!folio_test_locked(folio) || !folio_test_swapcache(folio), folio); + VM_BUG_ON_FOLIO(!nr || folio_page_idx(folio, page) + nr > folio_nr_pages(folio), folio); + swap_put_entries_cluster(__swap_entry_to_info(entry), swp_offset(entry), nr, false, -1); +} + int folio_dup_swap(struct folio *folio, struct page *subpage) { swp_entry_t entry = folio->swap; @@ -1771,7 +1982,7 @@ int folio_dup_swap(struct folio *folio, struct page *subpage) } return swap_dup_entries_cluster(swap_entry_to_info(entry), - swp_offset(entry), nr_pages); + swp_offset(entry), nr_pages, -1); } /** @@ -1797,7 +2008,7 @@ void folio_put_swap(struct folio *folio, struct page *subpage) nr_pages = 1; } - swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false); + swap_put_entries_cluster(si, swp_offset(entry), nr_pages, false, -1); } /* @@ -2070,7 +2281,7 @@ void swap_put_entries_direct(swp_entry_t entry, int nr) offset = start_offset; do { cluster_end = min(round_up(offset + 1, SWAPFILE_CLUSTER), end_offset); - swap_put_entries_cluster(si, offset, cluster_end - offset, true); + swap_put_entries_cluster(si, offset, cluster_end - offset, true, -1); offset = cluster_end; } while (offset < end_offset); out: @@ -2239,14 +2450,80 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) return pte_same(pte_swp_clear_flags(pte), swp_pte); } +#ifdef CONFIG_MM_SUBPAGE +/* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ +static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr, swp_entry_t entry, struct folio *folio, + unsigned int offset) +{ + struct mm_struct *mm = vma->vm_mm; + struct mm_subpage *slot = NULL; + spinlock_t *ptl; + pte_t *ptep, old, pte; + bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); + int ret = 0; + + if (folio_order(folio) || folio_test_ksm(folio)) + return -EOPNOTSUPP; + if (!poisoned) { + arch_swap_restore(folio_swap(entry, folio), folio); + /* Allocate metadata before taking the PTE spinlock. */ + slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); + if (IS_ERR(slot)) + return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + } + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + goto out; + old = ptep_get(ptep); + if (softleaf_from_pte(old).val != entry.val) + goto unlock; + /* A moved swap PTE retains its backing offset, not its VMA offset. */ + if (pte_swp_subpage_offset(old) != offset) + goto unlock; + if (poisoned) { + pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? + make_hwpoison_entry(&folio->page) : make_poisoned_swp_entry()); + } else { + ret = mm_subpage_add_anon_rmap(slot, vma, addr); + if (ret) + goto unlock; + /* unuse_pte_range() waited for writeback and holds the folio lock. */ + VM_BUG_ON_FOLIO(folio_test_writeback(folio), folio); + if (pte_swp_exclusive(old)) + mm_subpage_set_exclusive(slot); + else if (swap_subpage_count(entry, offset) == 1) + mm_subpage_try_reuse_swap(slot); + pte = pte_mkold(phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot)); + if (pte_swp_soft_dirty(old)) + pte = pte_mksoft_dirty(pte); + if (pte_swp_uffd_wp(old)) + pte = pte_mkuffd_wp(pte); + inc_mm_counter(mm, MM_ANONPAGES); + slot = NULL; + ret = 1; + } + dec_mm_counter(mm, MM_SWAPENTS); + set_pte_at(mm, addr, ptep, pte); + swap_subpage_put(entry, offset, false); +unlock: + pte_unmap_unlock(ptep, ptl); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + /* * No need to decide whether this PTE shares the swap entry with others, * just let do_wp_page work it out if a write is requested later - to * force COW, vm_page_prot omits write permission from any private vma. */ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct folio *folio) + unsigned long addr, pte_t orig_pte, struct folio *folio) { + swp_entry_t entry = softleaf_from_pte(orig_pte); struct page *page; struct folio *swapcache; spinlock_t *ptl; @@ -2261,6 +2538,12 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, if (!folio_matches_swap_entry(folio, entry)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + return unuse_subpage(vma, pmd, addr, entry, folio, + pte_swp_subpage_offset(orig_pte)); +#endif + swapcache = folio; folio = ksm_might_need_to_copy(folio, vma, addr); if (unlikely(!folio)) @@ -2366,7 +2649,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_t ptent; if (!pte++) { - pte = pte_offset_map(pmd, addr); + pte = pte_offset_map_mm(vma->vm_mm, pmd, addr); if (!pte) break; } @@ -2382,6 +2665,22 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, pte_unmap(pte); pte = NULL; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) { + struct vm_fault vmf = { + .vma = vma, .address = addr, .real_address = addr, + .pmd = pmd, .orig_pte = ptent, + }; + vm_fault_t fault = do_swap_coarse_page(&vmf); + + if (fault & VM_FAULT_OOM) + return -ENOMEM; + if (fault & VM_FAULT_ERROR) + return -EIO; + continue; + } +#endif + folio = swap_cache_get_folio(entry); if (!folio) { struct vm_fault vmf = { @@ -2404,7 +2703,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_lock(folio); folio_wait_writeback(folio); - ret = unuse_pte(vma, pmd, addr, entry, folio); + ret = unuse_pte(vma, pmd, addr, ptent, folio); if (ret < 0) { folio_unlock(folio); folio_put(folio); @@ -2414,7 +2713,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd, folio_free_swap(folio); folio_unlock(folio); folio_put(folio); - } while (addr += PAGE_SIZE, addr != end); + } while (addr += mm_page_size(vma->vm_mm), addr != end); if (pte) pte_unmap(pte); @@ -2429,10 +2728,10 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long next; int ret; - pmd = pmd_offset(pud, addr); + pmd = pmd_offset_mm(vma->vm_mm, pud, addr); do { cond_resched(); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(vma->vm_mm, addr, end); ret = unuse_pte_range(vma, pmd, addr, next, type); if (ret) return ret; @@ -2448,9 +2747,9 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d, unsigned long next; int ret; - pud = pud_offset(p4d, addr); + pud = pud_offset_mm(vma->vm_mm, p4d, addr); do { - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(vma->vm_mm, addr, end); if (pud_none_or_clear_bad(pud)) continue; ret = unuse_pmd_range(vma, pud, addr, next, type); @@ -2468,10 +2767,10 @@ static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long next; int ret; - p4d = p4d_offset(pgd, addr); + p4d = p4d_offset_mm(vma->vm_mm, pgd, addr); do { - next = p4d_addr_end(addr, end); - if (p4d_none_or_clear_bad(p4d)) + next = p4d_addr_end_mm(vma->vm_mm, addr, end); + if (p4d_none_or_clear_bad_mm(vma->vm_mm, p4d)) continue; ret = unuse_pud_range(vma, p4d, addr, next, type); if (ret) @@ -2491,7 +2790,7 @@ static int unuse_vma(struct vm_area_struct *vma, unsigned int type) pgd = pgd_offset(vma->vm_mm, addr); do { - next = pgd_addr_end(addr, end); + next = pgd_addr_end_mm(vma->vm_mm, addr, end); if (pgd_none_or_clear_bad(pgd)) continue; ret = unuse_p4d_range(vma, pgd, addr, next, type); @@ -2798,10 +3097,15 @@ static int setup_swap_extents(struct swap_info_struct *sis, destroy_swap_extents(sis, swap_file); return -ENOMEM; } - return ret; + } else { + ret = generic_swapfile_activate(sis, swap_file, span); } - - return generic_swapfile_activate(sis, swap_file, span); + if (ret >= 0 && sis->bdev && + !(sis->flags & SWP_FS_OPS) && swap_extent_io_init()) { + destroy_swap_extents(sis, swap_file); + return -ENOMEM; + } + return ret; } static void _enable_swap_info(struct swap_info_struct *si) @@ -3708,7 +4012,7 @@ void si_swapinfo(struct sysinfo *val) * Also the swap entry must have a count >= 1. Otherwise folio_dup_swap should * be used. */ -int swap_dup_entry_direct(swp_entry_t entry) +int swap_dup_entries_direct(swp_entry_t entry, unsigned int nr) { struct swap_info_struct *si; @@ -3725,7 +4029,14 @@ int swap_dup_entry_direct(swp_entry_t entry) */ VM_WARN_ON_ONCE(!swap_entry_swapped(si, entry)); - return swap_dup_entries_cluster(si, swp_offset(entry), 1); + if (WARN_ON_ONCE(!nr || swp_cluster_offset(entry) + nr > SWAPFILE_CLUSTER)) + return -EINVAL; + return swap_dup_entries_cluster(si, swp_offset(entry), nr, -1); +} + +int swap_dup_entry_direct(swp_entry_t entry) +{ + return swap_dup_entries_direct(entry, 1); } #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) diff --git a/mm/truncate.c b/mm/truncate.c index 12cc89f89afcf7..ca1830b647a975 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -794,8 +794,10 @@ void truncate_pagecache(struct inode *inode, loff_t newsize) * truncate_inode_pages finishes, hence the second * unmap_mapping_range call must be made for correctness. */ + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); truncate_inode_pages(mapping, newsize); + unmap_mapping_subpage_tail(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); } EXPORT_SYMBOL(truncate_pagecache); diff --git a/mm/user-subpage-internal.h b/mm/user-subpage-internal.h new file mode 100644 index 00000000000000..bddb5eeb2ca7be --- /dev/null +++ b/mm/user-subpage-internal.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _MM_USER_SUBPAGE_INTERNAL_H +#define _MM_USER_SUBPAGE_INTERNAL_H + +#include +#include +#include +#include + +struct mm_subpage_owner; +struct rmap_walk_control; +struct anon_vma; + +struct mm_subpage { + struct mm_subpage_owner *owner; + refcount_t refs; + atomic_t mapcount; + unsigned int index; + u8 shift; + unsigned int pins; + bool exclusive; + bool lazyfree; + bool rmap_bound; +}; + +/* Logical identity is independent of the physical slot after relocation. */ +struct mm_subpage_rmap { + struct anon_vma *root; + struct vm_page_offset pos; + unsigned int offset; +}; + +struct subpage_retired_root; +struct mm_subpage_move { + struct mm_subpage *slot; + struct mm_subpage_rmap map; + struct subpage_retired_root *retired; + bool swap; + bool root_changed; + bool committed; +}; + +/* Folio and both PTE locks stay held across prepare, commit and finish. */ +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap); +/* Present source PTE must be cleared and its TLB invalidated before commit. */ +int mm_subpage_move_commit(struct mm_subpage_move *move); +void mm_subpage_move_finish(struct mm_subpage_move *move); + +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address); +/* Return -1 for ordinary linear rmap; otherwise return referenced identities. */ +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps); +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address); +bool rmap_walk_subpages(struct folio *folio, struct rmap_walk_control *rwc, + bool locked); + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage); + +/* Folio lock held; called once before publishing anonymous folio mapping. */ +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable); +bool mm_subpage_is_droppable(const struct mm_subpage *subpage); + +#endif diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 01b151dca3085d..8d72d991b05e50 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -16,10 +16,19 @@ #include #include #include +#include +#include #include #include #include "internal.h" #include "swap.h" +#ifdef CONFIG_MM_SUBPAGE +#include "user-subpage-internal.h" +#endif + +struct mfill_shared_data { + struct folio *source; /* NULL requests a zero-fill. */ +}; struct mfill_state { struct userfaultfd_ctx *ctx; @@ -32,6 +41,10 @@ struct mfill_state { unsigned long src_addr; unsigned long dst_addr; pmd_t *pmd; + struct folio *copy_buffer; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *slots[MM_SUBPAGES_PER_PAGE]; +#endif }; static bool anon_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags) @@ -194,8 +207,24 @@ static void uffd_mfill_unlock(struct vm_area_struct *vma) } #endif +/* Reservations keep the uninstalled leaves private to this operation. */ +static void mfill_free_subpages(struct mfill_state *state) +{ +#ifdef CONFIG_MM_SUBPAGE + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->slots); i++) { + if (state->slots[i]) { + mm_subpage_put(state->slots[i]); + state->slots[i] = NULL; + } + } +#endif +} + static void mfill_put_vma(struct mfill_state *state) { + mfill_free_subpages(state); if (!state->vma) return; @@ -248,6 +277,7 @@ static int mfill_get_vma(struct mfill_state *state) if ((flags & MFILL_ATOMIC_WP) && !(dst_vma->vm_flags & VM_UFFD_WP)) goto out_unlock; + if (is_vm_hugetlb_page(dst_vma)) return 0; @@ -320,16 +350,11 @@ static int mfill_establish_pmd(struct mfill_state *state) static bool mfill_file_over_size(struct vm_area_struct *dst_vma, unsigned long dst_addr) { - struct inode *inode; - pgoff_t offset, max_off; - if (!dst_vma->vm_file) return false; - inode = dst_vma->vm_file->f_inode; - offset = linear_page_index(dst_vma, dst_addr); - max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); - return offset >= max_off; + return !vma_file_offset_valid(dst_vma, dst_addr, + i_size_read(file_inode(dst_vma->vm_file))); } /* @@ -341,7 +366,8 @@ static bool mfill_file_over_size(struct vm_area_struct *dst_vma, static int mfill_atomic_install_pte(pmd_t *dst_pmd, struct vm_area_struct *dst_vma, unsigned long dst_addr, struct page *page, - uffd_flags_t flags) + uffd_flags_t flags, + const struct mfill_shared_data *shared_data) { int ret; struct mm_struct *dst_mm = dst_vma->vm_mm; @@ -354,6 +380,11 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, pte_t dst_ptep; _dst_pte = mk_pte(page, dst_vma->vm_page_prot); +#ifdef CONFIG_MM_SUBPAGE + if (page_in_cache && mm_page_size(dst_mm) < PAGE_SIZE) + _dst_pte = phys_pte_mm(dst_mm, page_to_phys(page) + + vma_page_offset_at(dst_vma, dst_addr).offset, dst_vma->vm_page_prot); +#endif _dst_pte = pte_mkdirty(_dst_pte); if (page_in_cache && !vm_shared) writable = false; @@ -384,7 +415,39 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, if (!pte_none(dst_ptep) && !pte_is_uffd_marker(dst_ptep)) goto out_unlock; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (shared_data) { + struct vm_page_offset pos = vma_page_offset_at(dst_vma, dst_addr); + unsigned long offset = ((pos.index - folio->index) << PAGE_SHIFT) + pos.offset; + unsigned long size = mm_page_size(dst_mm); + void *dst, *src; + + if (PageHWPoison(page)) { + ret = -EIO; + goto out_unlock; + } + if (!shmem_uffd_range_missing(folio, pos.index, pos.offset, size)) + goto out_unlock; + dst = kmap_local_folio(folio, offset); + if (shared_data->source) { + src = kmap_local_folio(shared_data->source, 0); + memcpy(dst, src, size); + kunmap_local(src); + } else { + memset(dst, 0, size); + } + kunmap_local(dst); + flush_dcache_folio(folio); + __folio_mark_uptodate(folio); + folio_mark_dirty(folio); + } +#endif + if (page_in_cache) { + mm_subpage_file_map_add(dst_vma, pte_phys_mm(dst_mm, _dst_pte)); + shmem_uffd_mark_mapped(folio, + (folio_page_idx(folio, page) << PAGE_SHIFT) + + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_mm)); folio_add_file_rmap_pte(folio, page, dst_vma); } else { folio_add_new_anon_rmap(folio, dst_vma, dst_addr, RMAP_EXCLUSIVE); @@ -411,12 +474,13 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, return ret; } -static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) +static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr, + unsigned int offset, unsigned long len) { void *kaddr; int ret; - kaddr = kmap_local_folio(folio, 0); + kaddr = kmap_local_folio(folio, offset); /* * The read mmap_lock is held here. Despite the * mmap_lock being read recursive a deadlock is still @@ -433,8 +497,7 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr) * and retry the copy outside the mmap_lock. */ pagefault_disable(); - ret = copy_from_user(kaddr, (const void __user *) src_addr, - PAGE_SIZE); + ret = copy_from_user(kaddr, (const void __user *)src_addr, len); pagefault_enable(); kunmap_local(kaddr); @@ -507,8 +570,11 @@ DEFINE_FREE(retry_put, struct mfill_retry_state *, if (_T) mfill_retry_state_put(_T)); static int mfill_copy_folio_retry(struct mfill_state *mfill_state, - struct folio *folio) + struct folio *folio, unsigned int offset, + unsigned long len) { + struct vm_page_offset pos = vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr); struct mfill_retry_state retry_state = { 0 }; struct mfill_retry_state *for_free __free(retry_put) = &retry_state; unsigned long src_addr = mfill_state->src_addr; @@ -520,8 +586,8 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, /* retry copying with mm_lock dropped */ mfill_put_vma(mfill_state); - kaddr = kmap_local_folio(folio, 0); - err = copy_from_user(kaddr, (const void __user *) src_addr, PAGE_SIZE); + kaddr = kmap_local_folio(folio, offset); + err = copy_from_user(kaddr, (const void __user *) src_addr, len); kunmap_local(kaddr); if (unlikely(err)) return -EFAULT; @@ -533,7 +599,10 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, if (err) return err; - if (mfill_retry_state_changed(&retry_state, mfill_state->vma)) + if (mfill_retry_state_changed(&retry_state, mfill_state->vma) || + (mm_page_size(mfill_state->ctx->mm) < PAGE_SIZE && + !vm_page_offset_equal(pos, vma_page_offset_at(mfill_state->vma, + mfill_state->dst_addr)))) return -EAGAIN; err = mfill_establish_pmd(mfill_state); @@ -543,6 +612,28 @@ static int mfill_copy_folio_retry(struct mfill_state *mfill_state, return 0; } +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +/* Fill one missing leaf in new or retained shmem backing using staged data. */ +static int mfill_atomic_shmem_hole(struct mfill_state *state, + const struct mfill_shared_data *data) +{ + struct vm_page_offset pos; + struct folio *folio; + bool new; + int err; + + folio = shmem_uffd_prepare_folio(state->vma, state->dst_addr, &new); + if (IS_ERR(folio)) + return PTR_ERR(folio); + pos = vma_page_offset_at(state->vma, state->dst_addr); + err = mfill_atomic_install_pte(state->pmd, state->vma, state->dst_addr, + folio_file_page(folio, pos.index), state->flags, data); + if (err) + shmem_uffd_abort_folio(folio, state->vma, new); + return err; +} +#endif + static int __mfill_atomic_pte(struct mfill_state *state, const struct vm_uffd_ops *ops) { @@ -562,7 +653,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, return -ENOMEM; if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { - ret = mfill_copy_folio_locked(folio, src_addr); + ret = mfill_copy_folio_locked(folio, src_addr, 0, PAGE_SIZE); /* * Fallback to copy_from_user outside mmap_lock. * If retry is successful, mfill_copy_folio_locked() returns @@ -571,7 +662,7 @@ static int __mfill_atomic_pte(struct mfill_state *state, * will take care of unlocking if needed. */ if (unlikely(ret)) { - ret = mfill_copy_folio_retry(state, folio); + ret = mfill_copy_folio_retry(state, folio, 0, PAGE_SIZE); if (ret) goto err_folio_put; } @@ -590,12 +681,29 @@ static int __mfill_atomic_pte(struct mfill_state *state, if (ops->filemap_add) { ret = ops->filemap_add(folio, state->vma, state->dst_addr); +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + /* + * A pinned large shmem folio can survive a partial hole punch. + * Its cache entry exists while this native-sized range is missing. + * Reuse the prepared source and preserve all neighbouring bytes. + * Ordinary native insertion and non-shmem paths stay unchanged. + */ + if (ret == -EEXIST && (state->vma->vm_flags & VM_SHARED) && + shmem_file(state->vma->vm_file) && + (READ_ONCE(SHMEM_I(file_inode(state->vma->vm_file))->flags) & + SHMEM_F_UFFD_SUBPAGE)) { + struct mfill_shared_data data = { .source = folio }; + + ret = mfill_atomic_shmem_hole(state, &data); + goto err_folio_put; + } +#endif if (ret) goto err_folio_put; } ret = mfill_atomic_install_pte(state->pmd, state->vma, dst_addr, - &folio->page, flags); + &folio->page, flags, NULL); if (ret) goto err_filemap_remove; @@ -609,10 +717,184 @@ static int __mfill_atomic_pte(struct mfill_state *state, return ret; } +#ifdef CONFIG_MM_SUBPAGE +/* Pack only this request's leaves in one native linear index. */ +static struct mm_subpage *mfill_alloc_subpage(struct mfill_state *state) +{ + struct vm_area_struct *vma = state->vma; + struct mm_struct *mm = vma->vm_mm; + struct vm_page_offset pos = vma_page_offset_at(vma, state->dst_addr); + unsigned long size = mm_page_size(mm); + unsigned int first = pos.offset >> mm_page_shift(mm), i, nr; + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + int err = -ENOMEM; + + if (state->slots[first]) + goto take; + + mfill_free_subpages(state); + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + folio = anon_alloc_folio(vma, state->dst_addr); + if (!folio) + goto out; + folio_throttle_swaprate(folio, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (err) { + folio_put(folio); + goto out; + } + nr = min_t(unsigned long, PAGE_SIZE - pos.offset, + state->dst_start + state->len - state->dst_addr) / size; + for (i = first; i < first + nr; i++) { + slot = mm_subpage_alloc_at(pool, i * size); + if (IS_ERR(slot)) { + err = PTR_ERR(slot); + mfill_free_subpages(state); + goto out; + } + state->slots[i] = slot; + } +out: + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + if (err) + return ERR_PTR(err); +take: + slot = state->slots[first]; + state->slots[first] = NULL; + return slot; +} + +static int mfill_atomic_subpage(struct mfill_state *state) +{ + struct mm_struct *mm = state->ctx->mm; + struct mm_subpage *slot = mfill_alloc_subpage(state); + struct vm_area_struct *vma; + struct folio *folio; + pte_t entry, *pte; + spinlock_t *ptl; + bool fresh; + int err; + + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + folio_lock(folio); + /* A previous leaf may have entered reclaim between iterations. */ + err = -EAGAIN; + if (folio_test_swapcache(folio)) + goto unlock_folio; + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + err = mfill_copy_folio_locked(folio, state->src_addr, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) { + /* + * The faulting retry must not hold the folio lock, or write + * a folio whose already-visible neighbours can be reclaimed. + * Use fresh backing and discard the remaining reservations + * before dropping the VMA lock. + */ + folio_unlock(folio); + mm_subpage_put(slot); + mfill_free_subpages(state); + slot = mfill_alloc_subpage(state); + if (IS_ERR(slot)) + return PTR_ERR(slot); + folio = mm_subpage_folio(slot); + err = mfill_copy_folio_retry(state, folio, + mm_subpage_offset(slot), mm_page_size(mm)); + if (err) + goto put_slot; + folio_lock(folio); + } + } + /* The allocator zeroed every byte, including unissued neighbours. */ + __folio_mark_uptodate(folio); + vma = state->vma; + pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); + err = -EAGAIN; + if (!pte) + goto unlock_folio; + err = -EFAULT; + if (mfill_file_over_size(vma, state->dst_addr)) + goto unlock_pte; + err = -EEXIST; + if (!pte_none(ptep_get(pte)) && !pte_is_uffd_marker(ptep_get(pte))) + goto unlock_pte; + fresh = !folio_test_anon(folio); + mm_subpage_set_exclusive(slot); + err = mm_subpage_add_new_anon_rmap(slot, vma, state->dst_addr); + if (err) + goto unlock_pte; + if (fresh) + folio_add_lru_vma(folio, vma); + else if ((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) == VM_LOCKED) + mlock_folio(folio); + inc_mm_counter(mm, MM_ANONPAGES); + entry = phys_pte_mm(mm, mm_subpage_phys(slot), vma->vm_page_prot); + entry = pte_mkdirty(entry); + if (vma->vm_flags & VM_WRITE) + entry = pte_mkwrite(entry, vma); + if (state->flags & MFILL_ATOMIC_WP) + entry = pte_mkuffd_wp(entry); + set_pte_at(mm, state->dst_addr, pte, entry); + update_mmu_cache(vma, state->dst_addr, pte); + /* The PTE owns slot now; release the folio lock before dropping PTL. */ + folio_unlock(folio); + pte_unmap_unlock(pte, ptl); + return 0; +unlock_pte: + pte_unmap_unlock(pte, ptl); +unlock_folio: + folio_unlock(folio); +put_slot: + mm_subpage_put(slot); + return err; +} +#endif + +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) +static int mfill_atomic_shared_subpage(struct mfill_state *state) +{ + struct mfill_shared_data data = {}; + int err; + + if (uffd_flags_mode_is(state->flags, MFILL_ATOMIC_COPY)) { + if (!state->copy_buffer) + state->copy_buffer = folio_alloc(GFP_KERNEL_ACCOUNT, 0); + data.source = state->copy_buffer; + if (!data.source) + return -ENOMEM; + err = mfill_copy_folio_locked(data.source, state->src_addr, 0, + mm_page_size(state->ctx->mm)); + if (err) + err = mfill_copy_folio_retry(state, data.source, 0, + mm_page_size(state->ctx->mm)); + if (err) + return err; + } + return mfill_atomic_shmem_hole(state, &data); +} +#endif + static int mfill_atomic_pte_copy(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + /* * The normal page fault path for a MAP_PRIVATE mapping in a * file-backed VMA will invoke the fault, fill the hole in the file and @@ -632,6 +914,16 @@ static int mfill_atomic_pte_zeroed_folio(struct mfill_state *state) { const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(state->ctx->mm) < PAGE_SIZE) { +#ifdef CONFIG_SHMEM + if (state->vma->vm_flags & VM_SHARED) + return mfill_atomic_shared_subpage(state); +#endif + return mfill_atomic_subpage(state); + } +#endif + return __mfill_atomic_pte(state, ops); } @@ -695,6 +987,13 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) if (IS_ERR_OR_NULL(folio)) return -EFAULT; +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_SHMEM) + if (vma_is_shmem(dst_vma) && shmem_uffd_range_missing(folio, pgoff, + vma_page_offset_at(dst_vma, dst_addr).offset, mm_page_size(dst_vma->vm_mm))) { + ret = -EFAULT; + goto out_release; + } +#endif page = folio_file_page(folio, pgoff); if (PageHWPoison(page)) { ret = -EIO; @@ -702,7 +1001,7 @@ static int mfill_atomic_pte_continue(struct mfill_state *state) } ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, - page, flags); + page, flags, NULL); if (ret) goto out_release; @@ -964,8 +1263,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(ctx->mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(ctx->mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len <= src_start); @@ -998,9 +1297,9 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, cond_resched(); if (!err) { - state.dst_addr += PAGE_SIZE; - state.src_addr += PAGE_SIZE; - copied += PAGE_SIZE; + state.dst_addr += mm_page_size(ctx->mm); + state.src_addr += mm_page_size(ctx->mm); + copied += mm_page_size(ctx->mm); if (fatal_signal_pending(current)) err = -EINTR; @@ -1011,6 +1310,8 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx, mfill_put_vma(&state); out: + if (state.copy_buffer) + folio_put(state.copy_buffer); VM_WARN_ON_ONCE(copied < 0); VM_WARN_ON_ONCE(err > 0); VM_WARN_ON_ONCE(!copied && !err); @@ -1099,8 +1400,8 @@ int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, /* * Sanitize the command parameters: */ - VM_WARN_ON_ONCE(start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(start & ~mm_page_mask(dst_mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(dst_mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(start + len <= start); @@ -1303,7 +1604,102 @@ static long move_present_ptes(struct mm_struct *mm, return src_addr > src_start ? src_addr - src_start : err; } +#ifdef CONFIG_MM_SUBPAGE +static long move_present_subpage(struct mm_struct *mm, + struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, + unsigned long dst_addr, unsigned long src_addr, + pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst, pte_t orig_src, + pmd_t *dst_pmd, pmd_t dst_pmdval, + spinlock_t *dst_ptl, spinlock_t *src_ptl, struct folio *folio) +{ + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + pte_t pte; + long ret = -EAGAIN; + + flush_cache_range(src_vma, src_addr, src_addr + mm_page_size(mm)); + double_pt_lock(dst_ptl, src_ptl); + if (!is_pte_pages_stable(dst_pte, src_pte, orig_dst, orig_src, + dst_pmd, dst_pmdval)) + goto out; + ret = -EBUSY; + if (!folio_test_anon(folio) || folio_test_large(folio)) + goto out; + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, orig_src)); + if (!slot || mm_subpage_folio(slot) != folio) + goto out; + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, false); + if (ret) + goto out; + orig_src = ptep_clear_flush(src_vma, src_addr, src_pte); + ret = mm_subpage_move_commit(&move); + if (ret) { + set_pte_at(mm, src_addr, src_pte, orig_src); + } else { + pte = phys_pte_mm(mm, mm_subpage_phys(slot), dst_vma->vm_page_prot); + if (pgtable_supports_soft_dirty()) + pte = pte_mksoft_dirty(pte); + if (pte_dirty(orig_src)) + pte = pte_mkdirty(pte); + pte = pte_mkwrite(pte, dst_vma); + set_pte_at(mm, dst_addr, dst_pte, pte); + ret = mm_page_size(mm); + } + mm_subpage_move_finish(&move); +out: + if (slot) + mm_subpage_put(slot); + double_pt_unlock(dst_ptl, src_ptl); + return ret; +} + +/* Both PTLs and, if present, the swap-cache folio lock are held. */ +static int move_swap_subpage(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, + unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, + pte_t orig_src, struct folio *folio, swp_entry_t entry) +{ + unsigned int offset = pte_swp_subpage_offset(orig_src); + struct mm_subpage_move move; + struct mm_subpage *slot = NULL; + int ret = -EBUSY; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EFAULT; + if (swap_subpage_count(entry, offset) != 1) + return -EBUSY; + if (folio) { + slot = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + offset); + if (slot) { + ret = mm_subpage_move_prepare(&move, slot, src_vma, src_addr, + dst_vma, dst_addr, true); + if (ret) + goto out; + ret = mm_subpage_move_commit(&move); + if (ret) { + mm_subpage_move_finish(&move); + goto out; + } + } + } + /* No mapping count changes: the same-mm exclusive swap PTE is transferred. */ + orig_src = ptep_get_and_clear(mm, src_addr, src_pte); + if (pgtable_supports_soft_dirty()) + orig_src = pte_swp_mksoft_dirty(orig_src); + set_pte_at(mm, dst_addr, dst_pte, orig_src); + if (slot) + mm_subpage_move_finish(&move); + ret = mm_page_size(mm); +out: + if (slot) + mm_subpage_put(slot); + return ret; +} +#endif + static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, + struct vm_area_struct *src_vma, unsigned long dst_addr, unsigned long src_addr, pte_t *dst_pte, pte_t *src_pte, pte_t orig_dst_pte, pte_t orig_src_pte, @@ -1329,6 +1725,21 @@ static int move_swap_pte(struct mm_struct *mm, struct vm_area_struct *dst_vma, return -EAGAIN; } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + int ret; + + if (!src_folio && swap_cache_has_folio(entry)) { + double_pt_unlock(dst_ptl, src_ptl); + return -EAGAIN; + } + ret = move_swap_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_src_pte, src_folio, entry); + double_pt_unlock(dst_ptl, src_ptl); + return ret; + } +#endif + /* * The src_folio resides in the swapcache, requiring an update to its * index and mapping to align with the dst_vma, where a swap-in may @@ -1381,13 +1792,13 @@ static int move_zeropage_pte(struct mm_struct *mm, return -EAGAIN; } - zero_pte = pte_mkspecial(pfn_pte(zero_pfn(dst_addr), - dst_vma->vm_page_prot)); + zero_pte = pte_mkspecial(phys_pte_mm(mm, PFN_PHYS(zero_pfn(dst_addr)), + dst_vma->vm_page_prot)); ptep_clear_flush(src_vma, src_addr, src_pte); set_pte_at(mm, dst_addr, dst_pte, zero_pte); double_pt_unlock(dst_ptl, src_ptl); - return PAGE_SIZE; + return mm_page_size(mm); } @@ -1474,7 +1885,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) ret = -ENOENT; else /* nothing to do to move a hole */ - ret = PAGE_SIZE; + ret = mm_page_size(mm); goto out; } @@ -1514,7 +1925,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } folio = vm_normal_folio(src_vma, src_addr, orig_src_pte); - if (!folio || !PageAnonExclusive(&folio->page)) { + if (!folio || (mm_page_size(mm) >= PAGE_SIZE && + !PageAnonExclusive(&folio->page))) { spin_unlock(src_ptl); ret = -EBUSY; goto out; @@ -1554,6 +1966,15 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } } +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) < PAGE_SIZE) { + ret = move_present_subpage(mm, dst_vma, src_vma, dst_addr, src_addr, + dst_pte, src_pte, orig_dst_pte, orig_src_pte, + dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio); + goto out; + } +#endif + /* at this point we have src_folio locked */ if (folio_test_large(src_folio)) { /* split_folio() can block */ @@ -1592,7 +2013,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto out; } - if (!pte_swp_exclusive(orig_src_pte)) { + /* Smaller leaves validate per-slot swap and mapping counts under PTL. */ + if (mm_page_size(mm) >= PAGE_SIZE && !pte_swp_exclusive(orig_src_pte)) { ret = -EBUSY; goto out; } @@ -1635,7 +2057,7 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd goto retry; } } - ret = move_swap_pte(mm, dst_vma, dst_addr, src_addr, dst_pte, src_pte, + ret = move_swap_pte(mm, dst_vma, src_vma, dst_addr, src_addr, dst_pte, src_pte, orig_dst_pte, orig_src_pte, dst_pmd, dst_pmdval, dst_ptl, src_ptl, src_folio, si, entry); } @@ -1694,6 +2116,11 @@ static int validate_move_areas(struct userfaultfd_ctx *ctx, pgprot_val(src_vma->vm_page_prot) != pgprot_val(dst_vma->vm_page_prot)) return -EINVAL; + /* Native backing cannot mix droppable and swap-backed slot policies. */ + if (mm_page_size(src_vma->vm_mm) < PAGE_SIZE && + ((src_vma->vm_flags ^ dst_vma->vm_flags) & VM_DROPPABLE)) + return -EINVAL; + /* Only allow moving if both are mlocked or both aren't */ if ((src_vma->vm_flags & VM_LOCKED) != (dst_vma->vm_flags & VM_LOCKED)) return -EINVAL; @@ -1926,9 +2353,9 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ssize_t moved = 0; /* Sanitize the command parameters. */ - VM_WARN_ON_ONCE(src_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(dst_start & ~PAGE_MASK); - VM_WARN_ON_ONCE(len & ~PAGE_MASK); + VM_WARN_ON_ONCE(src_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(dst_start & ~mm_page_mask(mm)); + VM_WARN_ON_ONCE(len & ~mm_page_mask(mm)); /* Does the address range wrap, or is the span zero-sized? */ VM_WARN_ON_ONCE(src_start + len < src_start); @@ -2053,10 +2480,12 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, ret = move_pages_ptes(mm, dst_pmd, src_pmd, dst_vma, src_vma, dst_addr, src_addr, src_end - src_addr, mode); - if (ret < 0) + if (ret < 0) { err = ret; - else + } else { + err = 0; step_size = ret; + } } cond_resched(); @@ -2095,6 +2524,17 @@ bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags, { const struct vm_uffd_ops *ops = vma_uffd_ops(vma); + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) { + if (mm_page_size(vma->vm_mm) > PAGE_SIZE) + return false; + if (vma_is_anonymous(vma)) { + if (vm_flags & VM_UFFD_MINOR) + return false; + } else if (!vma_is_shmem(vma)) { + return false; + } + } + if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK)) return false; diff --git a/mm/util.c b/mm/util.c index 3b26f5bf920121..1f41dfdd3ad13e 100644 --- a/mm/util.c +++ b/mm/util.c @@ -344,21 +344,22 @@ unsigned long randomize_stack_top(unsigned long stack_top) if (current->flags & PF_RANDOMIZE) { random_variable = get_random_long(); - random_variable &= STACK_RND_MASK; - random_variable <<= PAGE_SHIFT; + random_variable &= mm_aslr_mask(current->mm, STACK_RND_MASK); + random_variable <<= mm_page_shift(current->mm); } #ifdef CONFIG_STACK_GROWSUP - return PAGE_ALIGN(stack_top) + random_variable; + return mm_page_align(current->mm, stack_top) + random_variable; #else - return PAGE_ALIGN(stack_top) - random_variable; + return mm_page_align(current->mm, stack_top) - random_variable; #endif } /** - * randomize_page - Generate a random, page aligned address + * randomize_page_shift - Generate a random, granule aligned address * @start: The smallest acceptable address the caller will take. * @range: The size of the area, starting at @start, within which the * random address must fall. + * @shift: Log2 of the required address alignment. * * If @start + @range would overflow, @range is capped. * @@ -368,22 +369,31 @@ unsigned long randomize_stack_top(unsigned long stack_top) * Return: A page aligned address within [start, start + range). On error, * @start is returned. */ -unsigned long randomize_page(unsigned long start, unsigned long range) +static unsigned long randomize_page_shift(unsigned long start, + unsigned long range, unsigned int shift) { - if (!PAGE_ALIGNED(start)) { - range -= PAGE_ALIGN(start) - start; - start = PAGE_ALIGN(start); + unsigned long page_size = 1UL << shift; + + if (!IS_ALIGNED(start, page_size)) { + range -= ALIGN(start, page_size) - start; + start = ALIGN(start, page_size); } if (start > ULONG_MAX - range) range = ULONG_MAX - start; - range >>= PAGE_SHIFT; + range >>= shift; if (range == 0) return start; - return start + (get_random_long() % range << PAGE_SHIFT); + return start + (get_random_long() % range << shift); +} + +/* Existing callers retain native-page alignment. */ +unsigned long randomize_page(unsigned long start, unsigned long range) +{ + return randomize_page_shift(start, range, PAGE_SHIFT); } #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT @@ -391,23 +401,25 @@ unsigned long __weak arch_randomize_brk(struct mm_struct *mm) { /* Is the current task 32bit ? */ if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task()) - return randomize_page(mm->brk, SZ_32M); + return randomize_page_shift(mm->brk, SZ_32M, mm_page_shift(mm)); - return randomize_page(mm->brk, SZ_1G); + return randomize_page_shift(mm->brk, SZ_1G, mm_page_shift(mm)); } unsigned long arch_mmap_rnd(void) { - unsigned long rnd; + unsigned int bits; + unsigned long mask; #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS if (is_compat_task()) - rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1); + bits = mmap_rnd_compat_bits; else #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */ - rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1); + bits = mmap_rnd_bits; - return rnd << PAGE_SHIFT; + mask = mm_aslr_mask(current->mm, (1UL << bits) - 1); + return (get_random_long() & mask) << mm_page_shift(current->mm); } static int mmap_is_legacy(const struct rlimit *rlim_stack) @@ -440,14 +452,15 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli * task. mmap_base starts directly below the stack and grows * downwards. */ - return PAGE_ALIGN_DOWN(mmap_upper_limit(rlim_stack) - rnd); + return (mmap_upper_limit(rlim_stack) - rnd) & mm_page_mask(current->mm); #else unsigned long gap = rlim_stack->rlim_cur; unsigned long pad = stack_guard_gap; /* Account for stack randomization if necessary */ if (current->flags & PF_RANDOMIZE) - pad += (STACK_RND_MASK << PAGE_SHIFT); + pad += mm_aslr_mask(current->mm, STACK_RND_MASK) << + mm_page_shift(current->mm); /* Values close to RLIM_INFINITY can overflow. */ if (gap + pad > gap) @@ -458,7 +471,7 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli else if (gap > MAX_GAP) gap = MAX_GAP; - return PAGE_ALIGN(STACK_TOP - gap - rnd); + return mm_page_align(current->mm, STACK_TOP - gap - rnd); #endif } @@ -491,7 +504,7 @@ EXPORT_SYMBOL_IF_KUNIT(arch_pick_mmap_layout); /** * __account_locked_vm - account locked pages to an mm's locked_vm * @mm: mm to account against - * @pages: number of pages to account + * @pages: number of native physical pages to account * @inc: %true if @pages should be considered positive, %false if not * @task: task used to check RLIMIT_MEMLOCK * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped @@ -507,27 +520,35 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim) { unsigned long locked_vm, limit; + unsigned int shift = PAGE_SHIFT - MM_ACCOUNT_SHIFT; int ret = 0; mmap_assert_write_locked(mm); + /* This exported API receives native physical-page counts. */ + if (pages > ULONG_MAX >> shift) + return -ENOMEM; + pages <<= shift; locked_vm = mm->locked_vm; if (inc) { if (!bypass_rlim) { - limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked_vm + pages > limit) + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + if (pages > limit || locked_vm > limit - pages) ret = -ENOMEM; } + if (!ret && pages > ULONG_MAX - locked_vm) + ret = -ENOMEM; if (!ret) mm->locked_vm = locked_vm + pages; } else { - WARN_ON_ONCE(pages > locked_vm); + if (WARN_ON_ONCE(pages > locked_vm)) + return -EINVAL; mm->locked_vm = locked_vm - pages; } pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, - (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, - locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + (void *)_RET_IP_, (inc) ? '+' : '-', pages << MM_ACCOUNT_SHIFT, + locked_vm << MM_ACCOUNT_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), ret ? " - exceeded" : ""); return ret; @@ -562,11 +583,11 @@ int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc) } EXPORT_SYMBOL_GPL(account_locked_vm); -unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, +static unsigned long vm_mmap_offset_internal(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, - unsigned long flag, unsigned long pgoff) + unsigned long flag, struct vm_page_offset pos) { - loff_t off = (loff_t)pgoff << PAGE_SHIFT; + loff_t off = ((loff_t)pos.index << PAGE_SHIFT) | pos.offset; unsigned long ret; struct mm_struct *mm = current->mm; unsigned long populate; @@ -578,8 +599,11 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, if (!ret) { if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate, - &uf); +#ifdef CONFIG_MM_SUBPAGE + ret = do_mmap_offset(file, addr, len, prot, flag, 0, pos, &populate, &uf); +#else + ret = do_mmap(file, addr, len, prot, flag, 0, pos.index, &populate, &uf); +#endif mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf); if (populate) @@ -588,6 +612,13 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, return ret; } +unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff) +{ + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { pgoff, 0 }); +} + /* * Perform a userland memory mapping into the current process address space. See * the comment for do_mmap() for more details on this operation in general. @@ -609,12 +640,13 @@ unsigned long vm_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset) { - if (unlikely(offset + PAGE_ALIGN(len) < offset)) + if (unlikely(offset + mm_page_align(current->mm, len) < offset)) return -EINVAL; - if (unlikely(offset_in_page(offset))) + if (unlikely(offset & ~mm_page_mask(current->mm))) return -EINVAL; - return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT); + return vm_mmap_offset_internal(file, addr, len, prot, flag, + (struct vm_page_offset) { offset >> PAGE_SHIFT, offset & ~PAGE_MASK }); } EXPORT_SYMBOL(vm_mmap); @@ -988,7 +1020,8 @@ int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin if (mm) { long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10); - allowed -= min_t(long, mm->total_vm / 32, reserve); + allowed -= min_t(long, + mm_pages_to_bytes(mm, mm->total_vm) >> (PAGE_SHIFT + 5), reserve); } if (percpu_counter_read_positive(&vm_committed_as) < allowed) @@ -1189,6 +1222,9 @@ void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; diff --git a/mm/vma.c b/mm/vma.c index d90791b00a7b81..4161f7739a10e6 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -13,7 +13,7 @@ struct mmap_state { unsigned long addr; unsigned long end; - pgoff_t pgoff; + struct vm_page_offset offset; unsigned long pglen; union { vm_flags_t vm_flags; @@ -42,19 +42,25 @@ struct mmap_state { bool file_doesnt_need_get :1; }; -#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pgoff_, vma_flags_, file_) \ +#define MMAP_STATE(name, mm_, vmi_, addr_, len_, pos_, vma_flags_, file_) \ struct mmap_state name = { \ .mm = mm_, \ .vmi = vmi_, \ .addr = addr_, \ .end = (addr_) + (len_), \ - .pgoff = pgoff_, \ - .pglen = PHYS_PFN(len_), \ + .offset = pos_, \ + .pglen = (len_) >> mm_page_shift(mm_), \ .vma_flags = vma_flags_, \ .file = file_, \ .page_prot = vma_get_page_prot(vma_flags_), \ } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_MMAP_SUBPAGE_STATE(map_) .subpage_offset = (map_)->offset.offset, +#else +#define VMG_MMAP_SUBPAGE_STATE(map_) +#endif + #define VMG_MMAP_STATE(name, map_, vma_) \ struct vma_merge_struct name = { \ .mm = (map_)->mm, \ @@ -62,7 +68,8 @@ struct mmap_state { .start = (map_)->addr, \ .end = (map_)->end, \ .vma_flags = (map_)->vma_flags, \ - .pgoff = (map_)->pgoff, \ + .pgoff = (map_)->offset.index, \ + VMG_MMAP_SUBPAGE_STATE(map_) \ .file = (map_)->file, \ .prev = (map_)->prev, \ .middle = vma_, \ @@ -197,11 +204,12 @@ static void init_multi_vma_prep(struct vma_prepare *vp, */ static bool can_vma_merge_before(struct vma_merge_struct *vmg) { - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + struct vm_page_offset end = vm_page_offset_add(vmg_page_offset(vmg), + vmg->end - vmg->start); if (is_mergeable_vma(vmg, /* merge_next = */ true) && is_mergeable_anon_vma(vmg, /* merge_next = */ true)) { - if (vmg->next->vm_pgoff == vmg->pgoff + pglen) + if (vm_page_offset_equal(vma_page_offset_at(vmg->next, vmg->next->vm_start), end)) return true; } @@ -221,7 +229,8 @@ static bool can_vma_merge_after(struct vma_merge_struct *vmg) { if (is_mergeable_vma(vmg, /* merge_next = */ false) && is_mergeable_anon_vma(vmg, /* merge_next = */ false)) { - if (vmg->prev->vm_pgoff + vma_pages(vmg->prev) == vmg->pgoff) + if (vm_page_offset_equal(vma_page_offset_at(vmg->prev, vmg->prev->vm_end), + vmg_page_offset(vmg))) return true; } return false; @@ -521,7 +530,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, new->vm_end = addr; } else { new->vm_start = addr; - new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT); + vma_set_page_offset(new, vma_page_offset_at(vma, addr)); } err = -ENOMEM; @@ -560,7 +569,7 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, if (new_below) { vma->vm_start = addr; - vma->vm_pgoff += (addr - new->vm_start) >> PAGE_SHIFT; + vma_set_page_offset(vma, vma_page_offset_at(new, addr)); } else { vma->vm_end = addr; } @@ -705,19 +714,21 @@ void validate_mm(struct mm_struct *mm) static void vmg_adjust_set_range(struct vma_merge_struct *vmg) { struct vm_area_struct *adjust; - pgoff_t pgoff; + struct vm_page_offset pos; if (vmg->__adjust_middle_start) { adjust = vmg->middle; - pgoff = adjust->vm_pgoff + PHYS_PFN(vmg->end - adjust->vm_start); + pos = vma_page_offset_at(adjust, vmg->end); } else if (vmg->__adjust_next_start) { adjust = vmg->next; - pgoff = adjust->vm_pgoff - PHYS_PFN(adjust->vm_start - vmg->end); + pos = vm_page_offset_sub(vma_page_offset_at(adjust, adjust->vm_start), + adjust->vm_start - vmg->end); } else { return; } - vma_set_range(adjust, vmg->end, adjust->vm_end, pgoff); + vma_set_range(adjust, vmg->end, adjust->vm_end, pos.index); + vma_set_page_offset(adjust, pos); } /* @@ -762,6 +773,7 @@ static int commit_merge(struct vma_merge_struct *vmg) vma_adjust_trans_huge(vma, vmg->start, vmg->end, vmg->__adjust_middle_start ? vmg->middle : NULL); vma_set_range(vma, vmg->start, vmg->end, vmg->pgoff); + vma_set_page_offset(vma, vmg_page_offset(vmg)); vmg_adjust_set_range(vmg); vma_iter_store_overwrite(vmg->vmi, vmg->target); @@ -931,7 +943,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( vmg->start = prev->vm_start; vmg->end = next->vm_end; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * We already ensured anon_vma compatibility above, so now it's @@ -950,7 +962,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( */ vmg->start = prev->vm_start; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); if (!vmg->__remove_middle) vmg->__adjust_middle_start = true; @@ -965,7 +977,7 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( * shrink/delete extend */ - pgoff_t pglen = PHYS_PFN(vmg->end - vmg->start); + unsigned long len = vmg->end - vmg->start; VM_WARN_ON_VMG(!merge_right, vmg); /* If we are offset into a VMA, then prev must be middle. */ @@ -973,13 +985,14 @@ static __must_check struct vm_area_struct *vma_merge_existing_range( if (vmg->__remove_middle) { vmg->end = next->vm_end; - vmg->pgoff = next->vm_pgoff - pglen; + vmg_set_page_offset(vmg, + vm_page_offset_sub(vma_page_offset_at(next, next->vm_start), len)); } else { /* We shrink middle and expand next. */ vmg->__adjust_next_start = true; vmg->start = middle->vm_start; vmg->end = start; - vmg->pgoff = middle->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(middle, middle->vm_start)); } err = dup_anon_vma(next, middle, &anon_dup); @@ -1088,7 +1101,7 @@ struct vm_area_struct *vma_merge_new_range(struct vma_merge_struct *vmg) if (can_merge_left) { vmg->start = prev->vm_start; vmg->target = prev; - vmg->pgoff = prev->vm_pgoff; + vmg_set_page_offset(vmg, vma_page_offset_at(prev, prev->vm_start)); /* * If this merge would result in removal of the next VMA but we @@ -1245,7 +1258,7 @@ int vma_expand(struct vma_merge_struct *vmg) * Returns: 0 on success, -ENOMEM otherwise */ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff) + unsigned long start, unsigned long end, struct vm_page_offset pos) { struct vma_prepare vp; @@ -1266,7 +1279,8 @@ int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_adjust_trans_huge(vma, start, end, NULL); vma_iter_clear(vmi); - vma_set_range(vma, start, end, pgoff); + vma_set_range(vma, start, end, pos.index); + vma_set_page_offset(vma, pos); vma_complete(&vp, vmi, vma->vm_mm); validate_mm(vma->vm_mm); return 0; @@ -1335,7 +1349,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mm = current->mm; mm->map_count -= vms->vma_count; - mm->locked_vm -= vms->locked_vm; + mm->locked_vm -= mm_pages_to_account(mm, vms->locked_vm); if (vms->unlock) mmap_write_downgrade(mm); @@ -1360,7 +1374,7 @@ static void vms_complete_munmap_vmas(struct vma_munmap_struct *vms, mas_for_each(mas_detach, vma, ULONG_MAX) remove_vma(vma); - vm_unacct_memory(vms->nr_accounted); + mm_unacct_memory(mm, vms->nr_accounted); validate_mm(mm); if (vms->unlock) mmap_read_unlock(mm); @@ -1456,7 +1470,7 @@ static int vms_gather_munmap_vmas(struct vma_munmap_struct *vms, goto munmap_gather_failed; vma_mark_detached(next); - nrpages = vma_pages(next); + nrpages = vma_user_pages(next); vms->nr_pages += nrpages; if (vma_test(next, VMA_LOCKED_BIT)) @@ -1634,10 +1648,10 @@ int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long end; struct vm_area_struct *vma; - if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start) + if ((start & ~mm_page_mask(mm)) || start > TASK_SIZE || len > TASK_SIZE-start) return -EINVAL; - end = start + PAGE_ALIGN(len); + end = start + mm_page_align(mm, len); if (end == start) return -EINVAL; @@ -1862,7 +1876,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) * prior to moving page table entries, to effect an mremap move. */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks) { struct vm_area_struct *vma = *vmap; @@ -1878,7 +1892,10 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, * to match new location, to increase its chance of merging. */ if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) { - pgoff = addr >> PAGE_SHIFT; + pos = (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, + .offset = addr & ~PAGE_MASK, + }; faulted_in_anon_vma = false; } @@ -1894,7 +1911,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, if (new_vma && new_vma->vm_start < addr + len) return NULL; /* should never get here */ - vmg.pgoff = pgoff; + vmg_set_page_offset(&vmg, pos); vmg.next = vma_iter_next_rewind(&vmi, NULL); new_vma = vma_merge_copied_range(&vmg); @@ -1924,7 +1941,8 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, new_vma = vm_area_dup(vma); if (!new_vma) goto out; - vma_set_range(new_vma, addr, addr + len, pgoff); + vma_set_range(new_vma, addr, addr + len, pos.index); + vma_set_page_offset(new_vma, pos); if (vma_dup_policy(vma, new_vma)) goto out_free_vma; if (anon_vma_clone(new_vma, vma, VMA_OP_REMAP)) @@ -1979,7 +1997,8 @@ static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct * mpol_equal(vma_policy(a), vma_policy(b)) && a->vm_file == b->vm_file && vma_flags_empty(&diff) && - b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT); + vm_page_offset_equal(vma_page_offset_at(a, b->vm_start), + vma_page_offset_at(b, b->vm_start)); } /* @@ -2398,7 +2417,10 @@ static void set_desc_from_map(struct vm_area_desc *desc, desc->start = map->addr; desc->end = map->end; - desc->pgoff = map->pgoff; + desc->pgoff = map->offset.index; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = map->offset.offset; +#endif desc->vm_file = map->file; desc->vma_flags = map->vma_flags; desc->page_prot = map->page_prot; @@ -2458,7 +2480,7 @@ static int __mmap_setup(struct mmap_state *map, struct vm_area_desc *desc, map->charged = map->pglen; map->charged -= vms->nr_accounted; if (map->charged) { - error = security_vm_enough_memory_mm(map->mm, map->charged); + error = mm_account_memory(map->mm, map->mm, map->charged); if (error) return error; } @@ -2549,7 +2571,8 @@ static int __mmap_new_vma(struct mmap_state *map, struct vm_area_struct **vmap, return -ENOMEM; vma_iter_config(vmi, map->addr, map->end); - vma_set_range(vma, map->addr, map->end, map->pgoff); + vma_set_range(vma, map->addr, map->end, map->offset.index); + vma_set_page_offset(vma, map->offset); vma->flags = map->vma_flags; vma->vm_page_prot = map->page_prot; @@ -2621,7 +2644,7 @@ static void __mmap_complete(struct mmap_state *map, struct vm_area_struct *vma) if (!vma_supports_mlock(vma)) vma_clear_flags_mask(vma, VMA_LOCKED_MASK); else - mm->locked_vm += map->pglen; + mm->locked_vm += mm_pages_to_account(mm, map->pglen); } if (vma->vm_file) @@ -2678,7 +2701,10 @@ static int call_mmap_prepare(struct mmap_state *map, return err; /* Update fields permitted to be changed. */ - map->pgoff = desc->pgoff; + map->offset.index = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + map->offset.offset = desc->subpage_offset; +#endif if (desc->vm_file != map->file) { map->file_doesnt_need_get = true; map->file = desc->vm_file; @@ -2689,6 +2715,12 @@ static int call_mmap_prepare(struct mmap_state *map, map->vm_ops = desc->vm_ops; map->vm_private_data = desc->private_data; +#ifdef CONFIG_MM_SUBPAGE + if (map->offset.offset >= PAGE_SIZE || + !IS_ALIGNED(map->offset.offset, mm_page_size(map->mm))) + return -EINVAL; +#endif + return 0; } @@ -2731,13 +2763,13 @@ static bool can_set_ksm_flags_early(struct mmap_state *map) static unsigned long __mmap_region(struct file *file, unsigned long addr, unsigned long len, vma_flags_t vma_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma = NULL; bool have_mmap_prepare = file && file->f_op->mmap_prepare; VMA_ITERATOR(vmi, mm, addr); - MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file); + MMAP_STATE(map, mm, &vmi, addr, len, pos, vma_flags, file); struct vm_area_desc desc = { .mm = mm, .file = file, @@ -2791,7 +2823,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /* Accounting was done by __mmap_setup(). */ unacct_error: if (map.charged) - vm_unacct_memory(map.charged); + mm_unacct_memory(mm, map.charged); abort_munmap: /* * This indicates that .mmap_prepare has set a new file, differing from @@ -2807,7 +2839,7 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, /** * mmap_region() - Actually perform the userland mapping of a VMA into * current->mm with known, aligned and overflow-checked @addr and @len, and - * correctly determined VMA flags @vm_flags and page offset @pgoff. + * correctly determined VMA flags @vm_flags and file byte position @pos. * * This is an internal memory management function, and should not be used * directly. @@ -2819,8 +2851,8 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, * @addr: The page-aligned address at which to perform the mapping. * @len: The page-aligned, non-zero, length of the mapping. * @vm_flags: The VMA flags which should be applied to the mapping. - * @pgoff: If @file is specified, the page offset into the file, if not then - * the virtual page offset in memory of the anonymous mapping. + * @pos: Native page-cache index and byte offset within it. For anonymous + * mappings, the position is derived from the virtual address. * @uf: Optionally, a pointer to a list head used for tracking userfaultfd unmap * events. * @@ -2829,13 +2861,15 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, */ unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, vm_flags_t vm_flags, - unsigned long pgoff, struct list_head *uf) + struct vm_page_offset pos, struct list_head *uf) { unsigned long ret; bool writable_file_mapping = false; const vma_flags_t vma_flags = legacy_to_vma_flags(vm_flags); mmap_assert_write_locked(current->mm); + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, mm_page_size(current->mm))) + return -EINVAL; /* Check to see if MDWE is applicable. */ if (map_deny_write_exec(&vma_flags, &vma_flags)) @@ -2854,7 +2888,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, writable_file_mapping = true; } - ret = __mmap_region(file, addr, len, vma_flags, pgoff, uf); + ret = __mmap_region(file, addr, len, vma_flags, pos, uf); /* Clear our write mapping regardless of error. */ if (writable_file_mapping) @@ -2892,13 +2926,13 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_flags_set_mask(&vma_flags, mm->def_vma_flags); vma_flags = ksm_vma_flags(mm, NULL, vma_flags); - if (!may_expand_vm(mm, &vma_flags, len >> PAGE_SHIFT)) + if (!may_expand_vm(mm, &vma_flags, len >> mm_page_shift(mm))) return -ENOMEM; if (mm->map_count > get_sysctl_max_map_count()) return -ENOMEM; - if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) + if (mm_account_memory(mm, mm, len >> mm_page_shift(mm))) return -ENOMEM; /* @@ -2908,6 +2942,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, if (vma && vma->vm_end == addr) { VMG_STATE(vmg, mm, vmi, addr, addr + len, vma_flags, PHYS_PFN(addr)); + vmg_set_page_offset(&vmg, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); + vmg.prev = vma; /* vmi is positioned at prev, which this mode expects. */ vmg.just_expand = true; @@ -2927,6 +2965,9 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, vma_set_anonymous(vma); vma_set_range(vma, addr, addr + len, addr >> PAGE_SHIFT); + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = addr >> PAGE_SHIFT, .offset = addr & ~PAGE_MASK, + }); vma->flags = vma_flags; vma->vm_page_prot = vm_get_page_prot(vma_flags_to_legacy(vma_flags)); vma_start_write(vma); @@ -2937,10 +2978,10 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, validate_mm(mm); out: perf_event_mmap(vma); - mm->total_vm += len >> PAGE_SHIFT; - mm->data_vm += len >> PAGE_SHIFT; + mm->total_vm += len >> mm_page_shift(mm); + mm->data_vm += len >> mm_page_shift(mm); if (vma_flags_test(&vma_flags, VMA_LOCKED_BIT)) - mm->locked_vm += (len >> PAGE_SHIFT); + mm->locked_vm += len >> MM_ACCOUNT_SHIFT; if (pgtable_supports_soft_dirty()) vma_set_flags(vma, VMA_SOFTDIRTY_BIT); return 0; @@ -2948,7 +2989,7 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *vma, mas_store_fail: vm_area_free(vma); unacct_fail: - vm_unacct_memory(len >> PAGE_SHIFT); + mm_unacct_memory(mm, len >> mm_page_shift(mm)); return -ENOMEM; } @@ -3084,7 +3125,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, /* mlock limit tests */ if (!mlock_future_ok(mm, vma_test(vma, VMA_LOCKED_BIT), - grow << PAGE_SHIFT)) + grow << mm_page_shift(mm))) return -ENOMEM; /* Check to ensure the stack will not grow into a hugetlb-only region */ @@ -3100,7 +3141,7 @@ static int acct_stack_growth(struct vm_area_struct *vma, * Overcommit.. This must be the final test, as it will * update security statistics. */ - if (security_vm_enough_memory_mm(mm, grow)) + if (mm_account_memory(mm, mm, grow)) return -ENOMEM; return 0; @@ -3125,10 +3166,10 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); /* Guard against exceeding limits of the address space. */ - address &= PAGE_MASK; - if (address >= (TASK_SIZE & PAGE_MASK)) + address &= mm_page_mask(mm); + if (address >= (TASK_SIZE & mm_page_mask(mm))) return -ENOMEM; - address += PAGE_SIZE; + address += mm_page_size(mm); /* Enforce stack_guard_gap */ gap_addr = address + stack_guard_gap; @@ -3167,14 +3208,14 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address) unsigned long size, grow; size = address - vma->vm_start; - grow = (address - vma->vm_end) >> PAGE_SHIFT; + grow = (address - vma->vm_end) >> mm_page_shift(mm); error = -ENOMEM; if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_end = address; @@ -3209,7 +3250,7 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) mmap_assert_write_locked(mm); - address &= PAGE_MASK; + address &= mm_page_mask(mm); if (address < mmap_min_addr || address < FIRST_USER_ADDRESS) return -EPERM; @@ -3244,20 +3285,22 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address) /* Somebody else might have raced and expanded it already */ if (address < vma->vm_start) { unsigned long size, grow; + struct vm_page_offset pos = vm_page_offset_sub( + vma_page_offset_at(vma, vma->vm_start), vma->vm_start - address); size = vma->vm_end - address; - grow = (vma->vm_start - address) >> PAGE_SHIFT; + grow = (vma->vm_start - address) >> mm_page_shift(mm); error = -ENOMEM; - if (grow <= vma->vm_pgoff) { + if (pos.index <= vma->vm_pgoff) { error = acct_stack_growth(vma, size, grow); if (!error) { if (vma_test(vma, VMA_LOCKED_BIT)) - mm->locked_vm += grow; + mm->locked_vm += mm_pages_to_account(mm, grow); vm_stat_account(mm, vma->vm_flags, grow); anon_vma_interval_tree_pre_update_vma(vma); vma->vm_start = address; - vma->vm_pgoff -= grow; + vma_set_page_offset(vma, pos); /* Overwrite old entry in mtree. */ vma_iter_store_overwrite(&vmi, vma); anon_vma_interval_tree_post_update_vma(vma); @@ -3296,13 +3339,13 @@ int __vm_munmap(unsigned long start, size_t len, bool unlock) */ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) { - unsigned long charged = vma_pages(vma); + unsigned long charged = vma_user_pages(vma); if (find_vma_intersection(mm, vma->vm_start, vma->vm_end)) return -ENOMEM; if (vma_test(vma, VMA_ACCOUNT_BIT) && - security_vm_enough_memory_mm(mm, charged)) + mm_account_memory(mm, mm, charged)) return -ENOMEM; /* @@ -3319,12 +3362,15 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma) */ if (vma_is_anonymous(vma)) { BUG_ON(vma->anon_vma); - vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = vma->vm_start >> PAGE_SHIFT, + .offset = vma->vm_start & ~PAGE_MASK, + }); } if (vma_link(mm, vma)) { if (vma_test(vma, VMA_ACCOUNT_BIT)) - vm_unacct_memory(charged); + mm_unacct_memory(mm, charged); return -ENOMEM; } diff --git a/mm/vma.h b/mm/vma.h index 8e4b61a7304c68..3d9c8ee4be05d1 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -97,6 +97,9 @@ struct vma_merge_struct { unsigned long start; unsigned long end; pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif union { /* Temporary while VMA flags are being converted. */ @@ -234,9 +237,34 @@ static inline bool vmg_nomem(struct vma_merge_struct *vmg) static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, unsigned long addr) { - return vma->vm_pgoff + PHYS_PFN(addr - vma->vm_start); + return vma_page_offset_at(vma, addr).index; +} + +static inline struct vm_page_offset vmg_page_offset(const struct vma_merge_struct *vmg) +{ + return (struct vm_page_offset) { + .index = vmg->pgoff, +#ifdef CONFIG_MM_SUBPAGE + .offset = vmg->subpage_offset, +#endif + }; +} + +static inline void vmg_set_page_offset(struct vma_merge_struct *vmg, struct vm_page_offset pos) +{ + vmg->pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vmg->subpage_offset = pos.offset; +#endif } +#ifdef CONFIG_MM_SUBPAGE +#define VMG_SUBPAGE_STATE(vma, start) \ + .subpage_offset = vma_page_offset_at(vma, start).offset, +#else +#define VMG_SUBPAGE_STATE(vma, start) +#endif + #define VMG_STATE(name, mm_, vmi_, start_, end_, vma_flags_, pgoff_) \ struct vma_merge_struct name = { \ .mm = mm_, \ @@ -259,6 +287,7 @@ static inline pgoff_t vma_pgoff_offset(struct vm_area_struct *vma, .end = end_, \ .vm_flags = vma_->vm_flags, \ .pgoff = vma_pgoff_offset(vma_, start_), \ + VMG_SUBPAGE_STATE(vma_, start_) \ .file = vma_->vm_file, \ .anon_vma = vma_->anon_vma, \ .policy = vma_policy(vma_), \ @@ -276,7 +305,7 @@ void validate_mm(struct mm_struct *mm); __must_check int vma_expand(struct vma_merge_struct *vmg); __must_check int vma_shrink(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long start, unsigned long end, pgoff_t pgoff); + unsigned long start, unsigned long end, struct vm_page_offset pos); static inline int vma_iter_store_gfp(struct vma_iterator *vmi, struct vm_area_struct *vma, gfp_t gfp) @@ -311,6 +340,9 @@ static inline void compat_set_vma_from_desc(struct vm_area_struct *vma, /* Mutable fields. Populated with initial state. */ vma->vm_pgoff = desc->pgoff; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = desc->subpage_offset; +#endif if (desc->vm_file != vma->vm_file) vma_set_file(vma, desc->vm_file); vma->flags = desc->vma_flags; @@ -448,7 +480,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *vma); struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, - unsigned long addr, unsigned long len, pgoff_t pgoff, + unsigned long addr, unsigned long len, struct vm_page_offset pos, bool *need_rmap_locks); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -460,7 +492,7 @@ int mm_take_all_locks(struct mm_struct *mm); void mm_drop_all_locks(struct mm_struct *mm); unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long len, vm_flags_t vm_flags, struct vm_page_offset pos, struct list_head *uf); int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, diff --git a/mm/vma_exec.c b/mm/vma_exec.c index 5cee8b7efa0f21..8394f45a911133 100644 --- a/mm/vma_exec.c +++ b/mm/vma_exec.c @@ -42,6 +42,8 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) struct mmu_gather tlb; PAGETABLE_MOVE(pmc, vma, vma, old_start, new_start, length); + vmg_set_page_offset(&vmg, vma_page_offset_at(vma, old_start)); + BUG_ON(new_start > new_end); /* @@ -89,7 +91,7 @@ int relocate_vma_down(struct vm_area_struct *vma, unsigned long shift) vma_prev(&vmi); /* Shrink the vma to just the new range */ - return vma_shrink(&vmi, vma, new_start, new_end, vma->vm_pgoff); + return vma_shrink(&vmi, vma, new_start, new_end, vma_page_offset_at(vma, vma->vm_start)); } /* @@ -138,7 +140,7 @@ int create_init_stack_vma(struct mm_struct *mm, struct vm_area_struct **vmap, */ VM_WARN_ON_ONCE(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; - vma->vm_start = vma->vm_end - PAGE_SIZE; + vma->vm_start = vma->vm_end - mm_page_size(mm); if (pgtable_supports_soft_dirty()) flags |= VM_SOFTDIRTY; vm_flags_init(vma, flags); diff --git a/mm/vma_init.c b/mm/vma_init.c index 3c0b65950510e0..a41ba35de73203 100644 --- a/mm/vma_init.c +++ b/mm/vma_init.c @@ -46,7 +46,10 @@ static void vm_area_init_from(const struct vm_area_struct *src, dest->vm_start = src->vm_start; dest->vm_end = src->vm_end; dest->anon_vma = src->anon_vma; - dest->vm_pgoff = src->vm_pgoff; + vma_set_page_offset(dest, (struct vm_page_offset) { + .index = src->vm_pgoff, + .offset = vma_subpage_offset(src), + }); dest->vm_file = src->vm_file; dest->vm_private_data = src->vm_private_data; vm_flags_init(dest, src->vm_flags); diff --git a/mm/vmscan.c b/mm/vmscan.c index 1af6c96d1bc14b..fe28f70fd51b17 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3462,6 +3462,46 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct folio *folio, } } +static unsigned long lru_pte_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pte_index(addr); + return (addr >> mm_page_shift(mm)) & (pte_table_bytes_mm(mm) / sizeof(pte_t) - 1); +} + +static unsigned long lru_pmd_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_page_size(mm) == PAGE_SIZE) + return pmd_index(addr); + return (addr >> pmd_shift_mm(mm)) & (pmd_table_bytes_mm(mm) / sizeof(pmd_t) - 1); +} + +static unsigned long lru_pud_entries(struct mm_struct *mm) +{ + return mm_pud_folded(mm) ? 1 : pud_table_bytes_mm(mm) / sizeof(pud_t); +} + +static unsigned long lru_pud_index(struct mm_struct *mm, unsigned long addr) +{ + if (mm_pud_folded(mm)) + return 0; + if (mm_page_size(mm) == PAGE_SIZE) + return pud_index(addr); + return (addr >> pud_shift_mm(mm)) & (lru_pud_entries(mm) - 1); +} + +static unsigned int lru_folio_pte_batch(struct vm_area_struct *vma, + struct folio *folio, pte_t *ptep, pte_t *entry, unsigned int max_nr) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(vma->vm_mm) != PAGE_SIZE) + return folio_subpage_pte_batch_flags(vma, folio, ptep, entry, + max_nr, FPB_MERGE_YOUNG_DIRTY); +#endif + return folio_pte_batch_flags(folio, NULL, ptep, entry, max_nr, + FPB_MERGE_YOUNG_DIRTY); +} + static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, struct mm_walk *args) { @@ -3481,7 +3521,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, unsigned int nr; pmd_t pmdval; - pte = pte_offset_map_rw_nolock(args->mm, pmd, start & PMD_MASK, &pmdval, &ptl); + pte = pte_offset_map_rw_nolock(args->mm, pmd, start & pmd_mask_mm(args->mm), &pmdval, &ptl); if (!pte) return false; @@ -3497,7 +3537,7 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, lazy_mmu_mode_enable(); restart: - for (i = pte_index(start), addr = start; addr != end; i += nr, addr += nr * PAGE_SIZE) { + for (i = lru_pte_index(args->mm, start), addr = start; addr != end; i += nr, addr += nr * mm_page_size(args->mm)) { unsigned long pfn; struct folio *folio; pte_t *cur_pte = pte + i; @@ -3516,10 +3556,9 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(args->mm); - nr = folio_pte_batch_flags(folio, NULL, cur_pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(args->vma, folio, cur_pte, &ptent, max_nr); total += nr - 1; walk->mm_stats[MM_LEAF_TOTAL] += nr - 1; } @@ -3544,7 +3583,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, walk_update_folio(walk, last, gen, dirty); last = NULL; - if (i < PTRS_PER_PTE && get_next_vma(PMD_MASK, PAGE_SIZE, args, &start, &end)) + if (i < pte_table_bytes_mm(args->mm) / sizeof(pte_t) && + get_next_vma(pmd_mask_mm(args->mm), mm_page_size(args->mm), args, &start, &end)) goto restart; lazy_mmu_mode_disable(); @@ -3576,13 +3616,13 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area return; } - i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first); + i = addr == -1 ? 0 : lru_pmd_index(args->mm, addr) - lru_pmd_index(args->mm, *first); if (i && i <= MIN_LRU_BATCH) { __set_bit(i - 1, bitmap); return; } - pmd = pmd_offset(pud, *first); + pmd = pmd_offset_mm(args->mm, pud, *first); ptl = pmd_lockptr(args->mm, pmd); if (!spin_trylock(ptl)) @@ -3595,7 +3635,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area struct folio *folio; /* don't round down the first address */ - addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; + addr = i ? (*first & pmd_mask_mm(args->mm)) + i * pmd_size_mm(args->mm) : *first; if (!pmd_present(pmd[i])) goto next; @@ -3661,14 +3701,14 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, * tables to avoid taking the PMD lock; the second, if necessary, takes * the PMD lock to clear the accessed bit in PMD entries. */ - pmd = pmd_offset(pud, start & PUD_MASK); + pmd = pmd_offset_mm(args->mm, pud, start & pud_mask_mm(args->mm)); restart: /* walk_pte_range() may call get_next_vma() */ vma = args->vma; - for (i = pmd_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pmd_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pmd_t val = pmdp_get_lockless(pmd + i); - next = pmd_addr_end(addr, end); + next = pmd_addr_end_mm(args->mm, addr, end); if (!pmd_present(val) || is_huge_zero_pmd(val)) { walk->mm_stats[MM_LEAF_TOTAL]++; @@ -3710,7 +3750,8 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first); - if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end)) + if (i < pmd_table_bytes_mm(args->mm) / sizeof(pmd_t) && + get_next_vma(pud_mask_mm(args->mm), pmd_size_mm(args->mm), args, &start, &end)) goto restart; } @@ -3725,12 +3766,12 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, VM_WARN_ON_ONCE(p4d_leaf(*p4d)); - pud = pud_offset(p4d, start & P4D_MASK); + pud = pud_offset_mm(args->mm, p4d, start & p4d_mask_mm(args->mm)); restart: - for (i = pud_index(start), addr = start; addr != end; i++, addr = next) { + for (i = lru_pud_index(args->mm, start), addr = start; addr != end; i++, addr = next) { pud_t val = pudp_get(pud + i); - next = pud_addr_end(addr, end); + next = pud_addr_end_mm(args->mm, addr, end); if (!pud_present(val) || WARN_ON_ONCE(pud_leaf(val))) continue; @@ -3738,15 +3779,16 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, walk_pmd_range(&val, addr, next, args); if (need_resched() || walk->batched >= MAX_LRU_BATCH) { - end = (addr | ~PUD_MASK) + 1; + end = (addr | ~pud_mask_mm(args->mm)) + 1; goto done; } } - if (i < PTRS_PER_PUD && get_next_vma(P4D_MASK, PUD_SIZE, args, &start, &end)) + if (i < lru_pud_entries(args->mm) && + get_next_vma(p4d_mask_mm(args->mm), pud_size_mm(args->mm), args, &start, &end)) goto restart; - end = round_up(end, P4D_SIZE); + end = round_up(end, p4d_size_mm(args->mm)); done: if (!end || !args->vma) return 1; @@ -4215,20 +4257,20 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) /* avoid taking the LRU lock under the PTL when possible */ walk = current->reclaim_state ? current->reclaim_state->mm_walk : NULL; - start = max(addr & PMD_MASK, vma->vm_start); - end = min(addr | ~PMD_MASK, vma->vm_end - 1) + 1; + start = max(addr & pmd_mask_mm(vma->vm_mm), vma->vm_start); + end = min(addr | ~pmd_mask_mm(vma->vm_mm), vma->vm_end - 1) + 1; - if (end - start == PAGE_SIZE) + if (end - start == mm_page_size(vma->vm_mm)) return true; - if (end - start > MIN_LRU_BATCH * PAGE_SIZE) { - if (addr - start < MIN_LRU_BATCH * PAGE_SIZE / 2) - end = start + MIN_LRU_BATCH * PAGE_SIZE; - else if (end - addr < MIN_LRU_BATCH * PAGE_SIZE / 2) - start = end - MIN_LRU_BATCH * PAGE_SIZE; + if (end - start > MIN_LRU_BATCH * mm_page_size(vma->vm_mm)) { + if (addr - start < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + end = start + MIN_LRU_BATCH * mm_page_size(vma->vm_mm); + else if (end - addr < MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2) + start = end - MIN_LRU_BATCH * mm_page_size(vma->vm_mm); else { - start = addr - MIN_LRU_BATCH * PAGE_SIZE / 2; - end = addr + MIN_LRU_BATCH * PAGE_SIZE / 2; + start = addr - MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; + end = addr + MIN_LRU_BATCH * mm_page_size(vma->vm_mm) / 2; } } @@ -4240,10 +4282,10 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) lazy_mmu_mode_enable(); - pte -= (addr - start) / PAGE_SIZE; + pte -= (addr - start) / mm_page_size(vma->vm_mm); for (i = 0, addr = start; addr != end; - i += nr, pte += nr, addr += nr * PAGE_SIZE) { + i += nr, pte += nr, addr += nr * mm_page_size(vma->vm_mm)) { unsigned long pfn; pte_t ptent = ptep_get(pte); @@ -4257,10 +4299,9 @@ bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr) continue; if (folio_test_large(folio)) { - const unsigned int max_nr = (end - addr) >> PAGE_SHIFT; + const unsigned int max_nr = (end - addr) >> mm_page_shift(vma->vm_mm); - nr = folio_pte_batch_flags(folio, NULL, pte, &ptent, - max_nr, FPB_MERGE_YOUNG_DIRTY); + nr = lru_folio_pte_batch(vma, folio, pte, &ptent, max_nr); } if (!test_and_clear_young_ptes_notify(vma, addr, pte, nr)) @@ -4634,6 +4675,16 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c sc->nr.unqueued_dirty += delta; } + /* + * Legacy memcg reclaim relies on shrink_folio_list() waiting for + * writeback when reclaim-marked folios cycle through the LRU. Moving + * all of them to a younger generation can exhaust the charge limit + * before I/O completes. Let the common path apply its GFP and mapping + * deadlock checks instead of hiding those folios from it. + */ + if (writeback && !writeback_throttling_sane(sc)) + return false; + /* waiting for writeback */ if (writeback || (type == LRU_GEN_FILE && dirty)) { gen = folio_inc_gen(lruvec, folio, true); @@ -4668,8 +4719,9 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca if (!folio_test_referenced(folio)) set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); - /* for shrink_folio_list() */ - folio_clear_reclaim(folio); + /* Preserve legacy memcg's repeated-writeback marker for shrink_folio_list(). */ + if (writeback_throttling_sane(sc) || !folio_test_writeback(folio)) + folio_clear_reclaim(folio); success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); From e3867f20404443c8dcec7f14f4c581406c3348dd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 017/631] mm: add retained subpage backing ownership Implement reference-counted user slots within native folios, including zeroing, mapping and pin ownership, migration and deferred lifetime. The backend remains disabled until its architecture integration. Signed-off-by: Diljit Singh (cherry picked from commit c1a5f4938d74e70de292adee44236d69f7b14a2b) --- mm/Makefile | 1 + mm/page_ext.c | 4 + mm/user-subpage.c | 1673 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1678 insertions(+) create mode 100644 mm/user-subpage.c diff --git a/mm/Makefile b/mm/Makefile index 8ad2ab08244ebe..f531742956b8e5 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -150,3 +150,4 @@ obj-$(CONFIG_SHRINKER_DEBUG) += shrinker_debug.o obj-$(CONFIG_EXECMEM) += execmem.o obj-$(CONFIG_TMPFS_QUOTA) += shmem_quota.o obj-$(CONFIG_LAZY_MMU_MODE_KUNIT_TEST) += tests/lazy_mmu_mode_kunit.o +obj-$(CONFIG_MM_SUBPAGE) += user-subpage.o diff --git a/mm/page_ext.c b/mm/page_ext.c index e2e92bd27ebdd0..a73aab719c0406 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -78,6 +79,9 @@ static struct page_ext_operations page_idle_ops __initdata = { #endif static struct page_ext_operations *page_ext_ops[] __initdata = { +#ifdef CONFIG_MM_SUBPAGE + &mm_subpage_ext_ops, +#endif #ifdef CONFIG_PAGE_OWNER &page_owner_ops, #endif diff --git a/mm/user-subpage.c b/mm/user-subpage.c new file mode 100644 index 00000000000000..133d75241485a1 --- /dev/null +++ b/mm/user-subpage.c @@ -0,0 +1,1673 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Reference-counted user data slots in native folios. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "user-subpage-internal.h" +#include "internal.h" + +#define SUBPAGES_PER_FOLIO (PAGE_SIZE / MM_SUBPAGE_SIZE) +static bool subpage_shift_valid(unsigned int shift) +{ + return shift >= MM_SUBPAGE_SHIFT && shift < PAGE_SHIFT; +} + +static unsigned long subpage_mask(unsigned int shift) +{ + return GENMASK((PAGE_SIZE >> shift) - 1, 0); +} + +static unsigned long offset_mask(unsigned int shift, unsigned int offset) +{ + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, 1U << shift)) + return 0; + return BIT(offset >> shift); +} + +struct mm_subpage_owner { + struct rcu_head rcu; + struct work_struct free_work; + struct mm_subpage_pool *pool; + struct folio *folio; + struct anon_vma *anon_root; + struct mm_subpage_owner *migration_target; + /* Allocated only when slots need independent logical identities. */ + struct mm_subpage_rmap *rmap; + struct list_head all; + struct list_head available; + unsigned long used; + unsigned long zeroed; + u8 shift; + bool detached; + bool migrating; + bool droppable; + struct mm_subpage slots[SUBPAGES_PER_FOLIO]; +}; + +struct mm_subpage_ext { + struct mm_subpage_owner __rcu *owner; + atomic_t file_maps[SUBPAGES_PER_FOLIO]; + atomic_t file_leaves; + /* Folio lock protects validity; fits the 16K/64K layout padding. */ + u16 shmem_missing; +}; +static_assert(SUBPAGES_PER_FOLIO == 1 || sizeof(struct mm_subpage_ext) == + ALIGN(sizeof(void *) + sizeof(atomic_t) * (SUBPAGES_PER_FOLIO + 1), + sizeof(void *))); +static_assert(SUBPAGES_PER_FOLIO <= sizeof(u16) * BITS_PER_BYTE); + +unsigned long mm_subpage_shmem_missing(struct page *page) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned long missing; + + /* A referenced native page keeps its initialized section online. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + missing = subext->shmem_missing; + page_ext_put(ext); + return missing; +} + +void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + + VM_BUG_ON(!ext); + VM_BUG_ON(missing & ~GENMASK(SUBPAGES_PER_FOLIO - 1, 0)); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + subext->shmem_missing = missing; + page_ext_put(ext); +} + +static bool __init subpage_ext_needed(void) +{ + return true; +} + +struct page_ext_operations mm_subpage_ext_ops = { + .size = sizeof(struct mm_subpage_ext), + .need = subpage_ext_needed, +}; + +/* Only alternative PTE mappings contribute; native rmap counts stay unchanged. */ +static void subpage_file_map_update(phys_addr_t phys, unsigned long size, int delta) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + unsigned int index, nr, i; + int count; + + VM_BUG_ON(!is_power_of_2(size) || size < MM_SUBPAGE_SIZE || + size >= PAGE_SIZE || !IS_ALIGNED(phys, size)); + ext = page_ext_from_phys(phys); + if (WARN_ON_ONCE(!ext)) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + index = offset_in_page(phys) >> MM_SUBPAGE_SHIFT; + nr = size >> MM_SUBPAGE_SHIFT; + for (i = index; i < index + nr; i++) { + count = atomic_add_return(delta, &subext->file_maps[i]); + VM_WARN_ON(count < 0); + } + /* Rmap counts one mapping, regardless of its coverage in 4K units. */ + count = atomic_add_return(delta, &subext->file_leaves); + VM_WARN_ON(count < 0); + page_ext_put(ext); +} + +void __mm_subpage_file_map_add(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, 1); +} + +void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size) +{ + subpage_file_map_update(phys, size, -1); +} + +/* Snapshot for statistics only, never an ownership or pinning decision. */ +unsigned int mm_subpage_file_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) +{ + struct page_ext *ext = page_ext_get(page); + struct mm_subpage_ext *subext; + unsigned int i, leaves; + + if (WARN_ON_ONCE(!ext)) { + memset(counts, 0, sizeof(*counts)); + return 0; + } + subext = page_ext_data(ext, &mm_subpage_ext_ops); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) + counts->slots[i] = max(atomic_read(&subext->file_maps[i]), 0); + leaves = max(atomic_read(&subext->file_leaves), 0); + page_ext_put(ext); + return leaves; +} + +static int owner_bind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + int ret; + + if (!ext) + return -EOPNOTSUPP; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + ret = cmpxchg(&subext->owner, NULL, owner) ? -EEXIST : 0; + page_ext_put(ext); + return ret; +} + +static void owner_unbind(struct mm_subpage_owner *owner) +{ + struct page_ext *ext = page_ext_get(&owner->folio->page); + struct mm_subpage_ext *subext; + + /* The allocated, referenced folio prevents its section going offline. */ + VM_BUG_ON(!ext); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + VM_BUG_ON(rcu_access_pointer(subext->owner) != owner); + rcu_assign_pointer(subext->owner, NULL); + page_ext_put(ext); +} + +struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) +{ + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + struct page_ext *ext; + unsigned int index; + + if (!IS_ALIGNED(phys, MM_SUBPAGE_SIZE)) + return NULL; + ext = page_ext_from_phys(phys); + if (!ext) + return NULL; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + index = owner ? offset_in_page(phys) >> owner->shift : 0; + /* Keep the native allocation alive before publishing another slot ref. */ + if (owner && folio_try_get(owner->folio)) { + if (refcount_inc_not_zero_acquire(&owner->slots[index].refs)) + slot = &owner->slots[index]; + else + folio_put(owner->folio); + } + page_ext_put(ext); + return slot; +} + +struct mm_subpage_pool { + struct rcu_head rcu; + spinlock_t lock; + refcount_t refs; + struct list_head owners; + struct list_head available; + struct mm_subpage_owner *unissued; + unsigned long backing_pages; + unsigned long all_slots; + u8 shift; + bool closed; +}; + +static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, + struct folio *folio) +{ + unsigned int i; + + owner->pool = pool; + owner->shift = pool->shift; + owner->folio = folio; + INIT_LIST_HEAD(&owner->available); + for (i = 0; i < (PAGE_SIZE >> pool->shift); i++) { + owner->slots[i].owner = owner; + owner->slots[i].index = i; + owner->slots[i].shift = pool->shift; + refcount_set(&owner->slots[i].refs, 0); + atomic_set(&owner->slots[i].mapcount, 0); + } +} + +struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int shift) +{ + struct mm_subpage_pool *pool; + + BUILD_BUG_ON(PAGE_SHIFT <= MM_SUBPAGE_SHIFT); + BUILD_BUG_ON(SUBPAGES_PER_FOLIO > BITS_PER_LONG); + if (!subpage_shift_valid(shift)) + return NULL; + pool = kzalloc(sizeof(*pool), gfp | __GFP_ACCOUNT); + if (!pool) + return NULL; + pool->shift = shift; + pool->all_slots = subpage_mask(shift); + spin_lock_init(&pool->lock); + refcount_set(&pool->refs, 1); + INIT_LIST_HEAD(&pool->owners); + INIT_LIST_HEAD(&pool->available); + return pool; +} + +struct mm_subpage_pool *mm_subpage_pool_create(gfp_t gfp) +{ + return mm_subpage_pool_create_granule(gfp, MM_SUBPAGE_SHIFT); +} + +void mm_subpage_pool_get(struct mm_subpage_pool *pool) +{ + refcount_inc(&pool->refs); +} + +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + if (!refcount_dec_and_test(&pool->refs)) + return; + WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + kfree_rcu(pool, rcu); +} + +static void owner_remove(struct mm_subpage_owner *owner) +{ + lockdep_assert_held(&owner->pool->lock); + owner->detached = true; + list_del(&owner->all); + list_del_init(&owner->available); + if (owner->pool->unissued == owner) + owner->pool->unissued = NULL; + owner->pool->backing_pages--; +} + +static void owner_free_rmap(struct mm_subpage_owner *owner) +{ + unsigned int i; + + if (!owner->rmap) + return; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->rmap[i].root) + put_anon_vma(owner->rmap[i].root); + kfree(owner->rmap); +} + +static void owner_free_metadata(struct mm_subpage_owner *owner) +{ + struct mm_subpage_pool *pool = owner->pool; + + if (owner->anon_root) + put_anon_vma(owner->anon_root); + owner_free_rmap(owner); + kfree_rcu(owner, rcu); + mm_subpage_pool_put(pool); +} + +static void owner_free_work(struct work_struct *work) +{ + owner_free_metadata(container_of(work, struct mm_subpage_owner, free_work)); +} + +static void owner_free(struct mm_subpage_owner *owner, bool put_folio) +{ + owner_unbind(owner); + if (put_folio) { + if (owner->anon_root) + free_swap_cache(owner->folio); + /* Last slot reference, or wholly unissued supply. */ + folio_put(owner->folio); + } + if (owner->rmap) { + /* Historical roots can have no live VMA; their final put may sleep. */ + INIT_WORK(&owner->free_work, owner_free_work); + queue_work(system_unbound_wq, &owner->free_work); + } else { + owner_free_metadata(owner); + } +} + +struct anon_vma *mm_subpage_anon_root(const struct mm_subpage *subpage) +{ + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + return subpage->owner->anon_root; +} + +void mm_subpage_hold_anon_root(struct mm_subpage *subpage, struct anon_vma *root, + bool droppable) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + VM_BUG_ON(owner->anon_root || root != root->root); + get_anon_vma(root); + owner->droppable = droppable; + owner->anon_root = root; +} + +static struct mm_subpage_rmap slot_rmap(struct mm_subpage *slot) +{ + struct mm_subpage_owner *owner = slot->owner; + + lockdep_assert_held(&owner->pool->lock); + if (owner->rmap) + return owner->rmap[slot->index]; + return (struct mm_subpage_rmap) { + .root = owner->anon_root, + .pos = { .index = owner->folio->index, + .offset = mm_subpage_offset(slot) }, + .offset = mm_subpage_offset(slot), + }; +} + +static bool rmap_matches(struct mm_subpage_rmap map, struct vm_area_struct *vma, + unsigned long address) +{ + return map.root == vma->anon_vma->root && + vm_page_offset_equal(map.pos, vma_page_offset_at(vma, address)); +} + +bool mm_subpage_match_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_pool *pool = slot->owner->pool; + unsigned long flags; + bool matches; + + spin_lock_irqsave(&pool->lock, flags); + matches = slot->rmap_bound && rmap_matches(slot_rmap(slot), vma, address); + spin_unlock_irqrestore(&pool->lock, flags); + return matches; +} + +static void *owner_kcalloc(struct mm_subpage_owner *owner, size_t nr, size_t size) +{ + struct mem_cgroup *memcg, *saved; + void *maps; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + maps = kcalloc(nr, size, GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + set_active_memcg(saved); + mem_cgroup_put(memcg); + return maps; +} + +struct subpage_retired_root { + struct work_struct work; + struct anon_vma *root; +}; + +static void subpage_put_retired_root(struct work_struct *work) +{ + struct subpage_retired_root *retired = + container_of(work, struct subpage_retired_root, work); + + put_anon_vma(retired->root); + kfree(retired); +} + +/* The folio lock serializes binding and sidecar creation against migration. */ +int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, + unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage_rmap *maps = NULL; + struct subpage_retired_root *retired = NULL; + unsigned long flags; + unsigned int i; + int ret = 0; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&pool->lock, flags); + if (slot->rmap_bound) { + ret = rmap_matches(slot_rmap(slot), vma, address) ? 0 : -EINVAL; + goto unlock; + } + if (!owner->rmap && !rmap_matches(slot_rmap(slot), vma, address)) { + spin_unlock_irqrestore(&pool->lock, flags); + /* The caller may also hold a PTE lock. Never enter reclaim here. */ + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + } + if (owner->rmap) { + struct mm_subpage_rmap *map = &owner->rmap[slot->index]; + + if (map->root && map->root != vma->anon_vma->root) { + /* A final anon_vma put may sleep; the caller can hold a PTL. */ + spin_unlock_irqrestore(&pool->lock, flags); + retired = owner_kcalloc(owner, 1, sizeof(*retired)); + if (!retired) + return -ENOMEM; + INIT_WORK(&retired->work, subpage_put_retired_root); + spin_lock_irqsave(&pool->lock, flags); + retired->root = map->root; + } + if (map->root != vma->anon_vma->root) { + get_anon_vma(vma->anon_vma->root); + map->root = vma->anon_vma->root; + } + map->pos = vma_page_offset_at(vma, address); + map->offset = mm_subpage_offset(slot); + } + slot->rmap_bound = true; +unlock: + spin_unlock_irqrestore(&pool->lock, flags); + if (retired) + queue_work(system_unbound_wq, &retired->work); + return ret; +} + +static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) +{ + lockdep_assert_held(&slot->owner->pool->lock); + return !slot->owner->migrating && !slot->pins && + atomic_read(&slot->mapcount) == (swap ? 0 : 1) && + (swap || (slot->rmap_bound && slot->exclusive)); +} + +int mm_subpage_move_prepare(struct mm_subpage_move *move, struct mm_subpage *slot, + struct vm_area_struct *src, unsigned long src_address, + struct vm_area_struct *dst, unsigned long dst_address, bool swap) +{ + struct mm_subpage_owner *owner = slot->owner; + struct mm_subpage_rmap *maps, old; + unsigned long flags; + unsigned int i; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + memset(move, 0, sizeof(*move)); + if (!src->anon_vma || !dst->anon_vma || !owner->anon_root || + mm_page_shift(src->vm_mm) != slot->shift || + mm_page_shift(dst->vm_mm) != slot->shift || + owner->droppable != !!(dst->vm_flags & VM_DROPPABLE)) + return -EINVAL; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!subpage_move_allowed(slot, swap) || + (slot->rmap_bound && !rmap_matches(slot_rmap(slot), src, src_address))) + goto unlock; + spin_unlock_irqrestore(&owner->pool->lock, flags); + + /* Allocate before touching either PTE. The folio lock serializes writers. */ + if (!owner->rmap) { + maps = owner_kcalloc(owner, PAGE_SIZE >> owner->shift, sizeof(*maps)); + if (!maps) + return -ENOMEM; + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + maps[i].offset = i << owner->shift; + if (!owner->slots[i].rmap_bound) + continue; + maps[i] = slot_rmap(&owner->slots[i]); + get_anon_vma(maps[i].root); + } + owner->rmap = maps; + spin_unlock_irqrestore(&owner->pool->lock, flags); + } + old = owner->rmap[slot->index]; + move->root_changed = old.root != dst->anon_vma->root; + if (move->root_changed && old.root) { + move->retired = owner_kcalloc(owner, 1, sizeof(*move->retired)); + if (!move->retired) + return -ENOMEM; + INIT_WORK(&move->retired->work, subpage_put_retired_root); + move->retired->root = old.root; + } + if (move->root_changed) + get_anon_vma(dst->anon_vma->root); + move->slot = slot; + move->swap = swap; + move->map = (struct mm_subpage_rmap) { + .root = dst->anon_vma->root, + .pos = vma_page_offset_at(dst, dst_address), + .offset = mm_subpage_offset(slot), + }; + return 0; +unlock: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +int mm_subpage_move_commit(struct mm_subpage_move *move) +{ + struct mm_subpage *slot = move->slot; + struct mm_subpage_owner *owner = slot->owner; + unsigned long flags; + int ret = -EBUSY; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + spin_lock_irqsave(&owner->pool->lock, flags); + /* Recheck pins after source-PTE clearing and the matching TLB barrier. */ + if (subpage_move_allowed(slot, move->swap)) { + owner->rmap[slot->index] = move->map; + slot->rmap_bound = true; + move->committed = true; + ret = 0; + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_move_finish(struct mm_subpage_move *move) +{ + if (move->committed) { + if (move->retired) + queue_work(system_unbound_wq, &move->retired->work); + } else { + /* The locked destination VMA still holds this root. */ + if (move->root_changed) + put_anon_vma(move->map.root); + kfree(move->retired); + } +} + +int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + int nr = -1; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return nr; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nr = 0; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound) + continue; + maps[nr] = owner->rmap[i]; + get_anon_vma(maps[nr].root); + nr++; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nr; +} + +/* A referenced folio plus a stable VMA suffices; no folio lock required. */ +bool mm_subpage_address_in_vma(const struct folio *folio, + const struct vm_area_struct *vma, unsigned long *address) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner; + unsigned long flags; + unsigned int i; + bool nonlinear = false; + + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->detached && owner->rmap) { + nonlinear = true; + *address = -EFAULT; + for (i = 0; vma->anon_vma && i < (PAGE_SIZE >> owner->shift); i++) { + if (!owner->slots[i].rmap_bound || + owner->rmap[i].root != vma->anon_vma->root) + continue; + *address = vma_address_at_offset(vma, owner->rmap[i].pos); + if (*address != -EFAULT) + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); +out: + page_ext_put(ext); + return nonlinear; +} + +bool mm_subpage_is_droppable(const struct mm_subpage *subpage) +{ + return subpage->owner->droppable; +} + +void mm_subpage_pool_close(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + /* At most one newly supplied owner has no issued slots. Close is O(1). */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + +static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, + unsigned long eligible) +{ + struct mm_subpage_owner *owner; + + lockdep_assert_held(&pool->lock); + list_for_each_entry(owner, &pool->available, available) + /* A nonpresent PTE may still need any of this folio's slot bytes. */ + if (!owner->migrating && (~owner->used & eligible) && + !folio_test_swapcache(owner->folio)) + return owner; + return NULL; +} + +static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long eligible, gfp_t gfp) +{ + struct mm_subpage_owner *owner; + unsigned long flags; + int ret = 0; + + /* Only an unexposed order-0 allocation can become a slot owner. */ + if (folio_order(folio) || folio_ref_count(folio) != 1 || + folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || + folio_test_private(folio) || folio_test_swapcache(folio)) + return -EINVAL; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) + return -ENOMEM; + owner_init(owner, pool, folio); + /* Reject duplicate ownership before touching any existing data. */ + ret = owner_bind(owner); + if (ret) { + kfree(owner); + return ret; + } + /* Initialise even unissued bytes before any future whole-folio operation. */ + clear_user_subpage_range(&folio->page, 0, PAGE_SIZE); + __folio_mark_uptodate(folio); + owner->zeroed = pool->all_slots; + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) + ret = -ESHUTDOWN; + else if (find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. */ + ret = -EEXIST; + else { + mm_subpage_pool_get(pool); + list_add_tail(&owner->all, &pool->owners); + list_add_tail(&owner->available, &pool->available); + pool->backing_pages++; + pool->unissued = owner; + } + spin_unlock_irqrestore(&pool->lock, flags); + if (ret) { + owner_unbind(owner); + kfree_rcu(owner, rcu); + } + return ret; +} + +int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, + gfp_t gfp) +{ + return pool_add_folio(pool, folio, pool->all_slots, gfp); +} + +int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int offset, gfp_t gfp) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return -EINVAL; + return pool_add_folio(pool, folio, eligible, gfp); +} + +unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) +{ + unsigned long pages, flags; + + spin_lock_irqsave(&pool->lock, flags); + pages = pool->backing_pages; + spin_unlock_irqrestore(&pool->lock, flags); + return pages; +} + +static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, + unsigned long eligible, bool *needs_zero) +{ + struct mm_subpage_owner *owner; + struct mm_subpage *subpage; + unsigned long flags; + unsigned int index; + + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + subpage = ERR_PTR(-ESHUTDOWN); + goto out; + } + owner = find_available(pool, eligible); + if (!owner) { + subpage = ERR_PTR(-EAGAIN); + goto out; + } + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + index = __ffs(~owner->used & eligible); + __set_bit(index, &owner->used); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); +out: + spin_unlock_irqrestore(&pool->lock, flags); + return subpage; +} + +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +{ + bool needs_zero; + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + + if (IS_ERR(subpage)) + return subpage; + /* Fresh supply was already zeroed as a whole; reused slots must be cleared. */ + if (needs_zero) + clear_user_subpage_range(&subpage->owner->folio->page, + mm_subpage_offset(subpage), mm_subpage_size(subpage)); + /* Data stores must precede publication in a PTE or another CPU's reference. */ + refcount_set_release(&subpage->refs, 1); + return subpage; +} + +struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots); +} + +struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_alloc(pool, eligible); +} + +static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + bool release; + + lockdep_assert_held(&pool->lock); + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + VM_BUG_ON(subpage->pins); + WRITE_ONCE(subpage->exclusive, false); + if (!owner->migrating) { + WRITE_ONCE(subpage->lazyfree, false); + subpage->rmap_bound = false; + } + __clear_bit(subpage->index, &owner->used); + release = !owner->used && !owner->migrating; + if (release) + owner_remove(owner); + else if (!pool->closed && !owner->migrating && list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (release) + owner_free(owner, true); + else + folio_put(folio); +} + +static void subpage_release(struct mm_subpage *subpage) +{ + unsigned long flags; + + spin_lock_irqsave(&subpage->owner->pool->lock, flags); + subpage_release_locked(subpage, flags); +} + +static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned long eligible) +{ + struct mm_subpage *dest; + int err; + + if (pool->shift != mm_subpage_shift(source)) + return ERR_PTR(-EINVAL); + dest = subpage_reserve(pool, eligible, NULL); + + if (IS_ERR(dest)) + return dest; + err = copy_user_subpage_range(&dest->owner->folio->page, mm_subpage_offset(dest), + &source->owner->folio->page, mm_subpage_offset(source), + mm_subpage_size(source)); + if (err) { + /* Never publish a reservation whose copy did not finish. */ + subpage_release(dest); + return ERR_PTR(err); + } + refcount_set_release(&dest->refs, 1); + return dest; +} + +struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots); +} + +struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + if (!eligible) + return ERR_PTR(-EINVAL); + return subpage_copy(pool, source, eligible); +} + +/* The page-extension RCU section also protects the pool until its lock is held. */ +static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, + unsigned int shift) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_pool *pool; + struct mm_subpage_ext *subext; + struct mm_subpage *slot = NULL; + unsigned int index; + unsigned long flags; + + if (!ext) + return ERR_PTR(-EOPNOTSUPP); + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + if ((shift && shift != owner->shift) || !offset_mask(owner->shift, offset)) { + slot = ERR_PTR(-EINVAL); + goto out; + } + index = offset >> owner->shift; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + slot = ERR_PTR(-EAGAIN); + goto unlock; + } + slot = &owner->slots[index]; + if (test_bit(index, &owner->used)) { + folio_get(folio); + if (!refcount_inc_not_zero_acquire(&slot->refs)) { + /* Its final put or initial publication has not finished. */ + folio_put(folio); + slot = ERR_PTR(-EAGAIN); + } + goto unlock; + } + VM_BUG_ON(refcount_read(&slot->refs) || atomic_read(&slot->mapcount)); + if (pool->unissued == owner) + pool->unissued = NULL; + else + folio_get(folio); + __set_bit(index, &owner->used); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + /* The folio lock and uptodate state stabilise the restored bytes. */ + refcount_set_release(&slot->refs, 1); +unlock: + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + return slot; +} + +struct mm_subpage *mm_subpage_restore_granule(struct folio *folio, unsigned int offset, + unsigned int shift, gfp_t gfp) +{ + struct mm_subpage_pool *pool; + struct mm_subpage_owner *owner; + struct mm_subpage *slot; + unsigned int index; + int err; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!subpage_shift_valid(shift) || !offset_mask(shift, offset) || folio_order(folio) || !folio_test_swapcache(folio) || + !folio_test_uptodate(folio) || folio_test_ksm(folio) || + (folio->mapping && !folio_test_anon(folio)) || + (folio_test_anon(folio) && PageAnonExclusive(&folio->page))) + return ERR_PTR(-EINVAL); + index = offset >> shift; + slot = restore_bound_slot(folio, offset, shift); + if (slot) + return slot; + + /* A reclaimed folio has no surviving owner; do not initialise its data. */ + pool = mm_subpage_pool_create_granule(gfp, shift); + if (!pool) + return ERR_PTR(-ENOMEM); + pool->closed = true; + owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); + if (!owner) { + mm_subpage_pool_put(pool); + return ERR_PTR(-ENOMEM); + } + owner_init(owner, pool, folio); + owner->used = BIT(index); + refcount_set(&owner->slots[index].refs, 1); + list_add(&owner->all, &pool->owners); + pool->backing_pages = 1; + folio_get(folio); + err = owner_bind(owner); + if (err) { + folio_put(folio); + list_del(&owner->all); + pool->backing_pages = 0; + kfree(owner); + mm_subpage_pool_put(pool); + return ERR_PTR(err == -EEXIST ? -EAGAIN : err); + } + /* The private closed pool's initial reference belongs to this owner. */ + return &owner->slots[index]; +} + +struct mm_subpage *mm_subpage_restore(struct folio *folio, unsigned int offset, gfp_t gfp) +{ + return mm_subpage_restore_granule(folio, offset, MM_SUBPAGE_SHIFT, gfp); +} + +/* Preserve the metadata charge when migration runs outside its owning task. */ +static struct mm_subpage_owner *owner_alloc_migration(struct mm_subpage_owner *owner) +{ + struct mem_cgroup *memcg, *saved; + struct mm_subpage_owner *new; + + rcu_read_lock(); + do { + memcg = mem_cgroup_from_virt(owner) ?: root_mem_cgroup; + } while (memcg && !mem_cgroup_tryget(memcg)); + rcu_read_unlock(); + saved = set_active_memcg(memcg); + /* Folio locks are held: do not recurse into reclaim or trigger memcg OOM. */ + new = kzalloc(sizeof(*new), GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (new && owner->rmap) { + new->rmap = kcalloc(PAGE_SIZE >> owner->shift, sizeof(*new->rmap), + GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT); + if (!new->rmap) { + kfree(new); + new = NULL; + } + } + set_active_memcg(saved); + mem_cgroup_put(memcg); + return new; +} + +/* A migration hold pins metadata, not the native folio: the driver pins it. */ +int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) +{ + struct page_ext *ext; + struct mm_subpage_ext *subext; + struct mm_subpage_owner *owner, *new; + struct mm_subpage_pool *pool; + unsigned long flags; + int ret = 0; + unsigned int i; + + VM_BUG_ON_FOLIO(!folio_test_locked(src), src); + VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); + ext = page_ext_get(&src->page); + if (!ext) + return 0; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) { + page_ext_put(ext); + return 0; + } + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->detached) { + owner = NULL; + } else if (src == dst || folio_order(src) || folio_order(dst) || + owner->migrating || pool->unissued == owner) { + ret = -EBUSY; + } else { + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (owner->slots[i].pins) + ret = -EBUSY; + if (!ret) + owner->migrating = true; + } + spin_unlock_irqrestore(&pool->lock, flags); + page_ext_put(ext); + if (!owner || ret) + return ret; + + /* Async migration must not recurse into reclaim while holding folio locks. */ + new = owner_alloc_migration(owner); + if (!new) { + ret = -ENOMEM; + goto undo; + } + owner_init(new, pool, dst); + new->migrating = true; + new->zeroed = owner->zeroed; + spin_lock_irqsave(&pool->lock, flags); + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) { + new->slots[i].rmap_bound = owner->slots[i].rmap_bound; + if (new->rmap) { + new->rmap[i] = owner->rmap[i]; + if (new->rmap[i].root) + get_anon_vma(new->rmap[i].root); + } + } + spin_unlock_irqrestore(&pool->lock, flags); + + new->droppable = owner->droppable; + new->anon_root = owner->anon_root; + if (new->anon_root) + get_anon_vma(new->anon_root); + mm_subpage_pool_get(pool); + spin_lock_irqsave(&pool->lock, flags); + list_add_tail(&new->all, &pool->owners); + pool->backing_pages++; + ret = owner_bind(new); + if (ret) + owner_remove(new); + else + owner->migration_target = new; + spin_unlock_irqrestore(&pool->lock, flags); + if (!ret) + return 0; + if (new->anon_root) + put_anon_vma(new->anon_root); + owner_free_rmap(new); + kfree(new); + mm_subpage_pool_put(pool); +undo: + mm_subpage_migrate_finish(src); + return ret; +} + +bool mm_subpage_migrating(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + bool active = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return false; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (owner) + active = READ_ONCE(owner->migrating); + page_ext_put(ext); + return active; +} + +/* Source folio/PTE locked: writable GETs cannot race this final snapshot. */ +void mm_subpage_migrate_save_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + + VM_BUG_ON_FOLIO(!owner->migrating || !owner->migration_target, owner->folio); + WRITE_ONCE(owner->migration_target->slots[subpage->index].lazyfree, + READ_ONCE(subpage->lazyfree)); +} + +struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int offset) +{ + struct mm_subpage *slot; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + VM_BUG_ON_FOLIO(!offset_mask(MM_SUBPAGE_SHIFT, offset) || !mm_subpage_migrating(folio), folio); + /* No allocation or data initialisation can fail during migration recovery. */ + slot = restore_bound_slot(folio, offset, 0); + VM_BUG_ON_FOLIO(IS_ERR_OR_NULL(slot), folio); + return slot; +} + +void mm_subpage_migrate_finish(struct folio *folio) +{ + struct page_ext *ext = page_ext_get(&folio->page); + struct mm_subpage_owner *owner; + struct mm_subpage_ext *subext; + struct mm_subpage_pool *pool; + unsigned long flags; + unsigned int i; + bool release = false; + + VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); + if (!ext) + return; + subext = page_ext_data(ext, &mm_subpage_ext_ops); + owner = rcu_dereference(subext->owner); + if (!owner) + goto out; + pool = owner->pool; + spin_lock_irqsave(&pool->lock, flags); + if (owner->migrating) { + VM_BUG_ON(owner->detached); + owner->migrating = false; + owner->migration_target = NULL; + for (i = 0; i < (PAGE_SIZE >> owner->shift); i++) + if (!(owner->used & BIT(i))) + owner->slots[i].rmap_bound = false; + release = !owner->used; + if (release) + owner_remove(owner); + else if (!pool->closed && owner->used != pool->all_slots && + list_empty(&owner->available)) + list_add(&owner->available, &pool->available); + } + spin_unlock_irqrestore(&pool->lock, flags); +out: + page_ext_put(ext); + if (release) + /* No slot references remain; only the migration driver pins backing. */ + owner_free(owner, false); +} + +void mm_subpage_get(struct mm_subpage *subpage) +{ + folio_get(subpage->owner->folio); + refcount_inc(&subpage->refs); +} + +void mm_subpage_put(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + struct mm_subpage_pool *pool = owner->pool; + struct folio *folio = owner->folio; + unsigned long flags; + + /* + * Snapshot the backing while our slot reference still retains owner. + * A non-final decrement can race the final put and metadata reclamation; + * only our physical folio reference survives that decrement. + */ + if (refcount_dec_and_lock_irqsave(&subpage->refs, &pool->lock, &flags)) + subpage_release_locked(subpage, flags); + else + folio_put(folio); +} + +void mm_subpage_set_exclusive(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(atomic_read(&subpage->mapcount) > 1 || !refcount_read(&subpage->refs)); + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); +} + +bool mm_subpage_is_exclusive(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->exclusive); +} + +static bool subpage_try_reuse(struct mm_subpage *subpage, unsigned int references) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool reusable; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + reusable = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == references && !subpage->pins; + if (reusable) + WRITE_ONCE(subpage->exclusive, true); + spin_unlock_irqrestore(&pool->lock, flags); + return reusable; +} + +bool mm_subpage_try_reuse_swap(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 1); +} + +bool mm_subpage_try_reuse_anon(struct mm_subpage *subpage) +{ + return subpage_try_reuse(subpage, 2); +} + +bool mm_subpage_can_zap(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = !subpage->owner->migrating && !subpage->pins && + atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_try_share(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool shared; + + spin_lock_irqsave(&pool->lock, flags); + shared = !subpage->pins; + if (shared) + WRITE_ONCE(subpage->exclusive, false); + spin_unlock_irqrestore(&pool->lock, flags); + return shared; +} + +bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->lazyfree); +} + +void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) +{ + WRITE_ONCE(subpage->lazyfree, false); +} + +bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + VM_BUG_ON_FOLIO(!folio_test_locked(subpage->owner->folio), subpage->owner->folio); + spin_lock_irqsave(&pool->lock, flags); + eligible = atomic_read(&subpage->mapcount) == 1 && + refcount_read(&subpage->refs) == 2 && !subpage->pins; + if (eligible) + WRITE_ONCE(subpage->lazyfree, true); + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +bool mm_subpage_all_lazyfree(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + unsigned int i; + bool all = true; + + spin_lock_irqsave(&owner->pool->lock, flags); + for (i = 0; i < SUBPAGES_PER_FOLIO; i++) { + if (!(owner->used & BIT(i))) + continue; + if (!READ_ONCE(owner->slots[i].lazyfree)) { + all = false; + break; + } + } + spin_unlock_irqrestore(&owner->pool->lock, flags); + return all; +} + +bool mm_subpage_discardable(struct mm_subpage *subpage) +{ + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + bool eligible; + + spin_lock_irqsave(&pool->lock, flags); + eligible = (subpage->owner->droppable || READ_ONCE(subpage->lazyfree)) && + !subpage->pins && + refcount_read(&subpage->refs) == atomic_read(&subpage->mapcount) + 1; + spin_unlock_irqrestore(&pool->lock, flags); + return eligible; +} + +int mm_subpage_pin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + int ret = -EAGAIN; + + spin_lock_irqsave(&owner->pool->lock, flags); + if (!owner->anon_root || owner->migrating || !subpage->exclusive || + !atomic_read(&subpage->mapcount)) + goto out; + ret = try_grab_folio(owner->folio, 1, FOLL_PIN); + if (ret) + goto out; + mm_subpage_get(subpage); + subpage->pins++; +out: + spin_unlock_irqrestore(&owner->pool->lock, flags); + return ret; +} + +void mm_subpage_unpin(struct mm_subpage *subpage) +{ + struct mm_subpage_owner *owner = subpage->owner; + unsigned long flags; + + spin_lock_irqsave(&owner->pool->lock, flags); + VM_BUG_ON(!subpage->pins || !subpage->exclusive); + subpage->pins--; + spin_unlock_irqrestore(&owner->pool->lock, flags); + unpin_folio(owner->folio); + mm_subpage_put(subpage); +} + +unsigned int mm_subpage_pincount(const struct mm_subpage *subpage) +{ + return READ_ONCE(subpage->pins); +} + +int mm_subpage_mapcount(const struct mm_subpage *subpage) +{ + return atomic_read(&subpage->mapcount); +} + +struct folio *mm_subpage_folio(const struct mm_subpage *subpage) +{ + return subpage->owner->folio; +} + +unsigned int mm_subpage_shift(const struct mm_subpage *subpage) +{ + return subpage->shift; +} + +unsigned int mm_subpage_size(const struct mm_subpage *subpage) +{ + return 1U << subpage->shift; +} + +unsigned int mm_subpage_offset(const struct mm_subpage *subpage) +{ + return subpage->index << subpage->shift; +} + +phys_addr_t mm_subpage_phys(const struct mm_subpage *subpage) +{ + return PFN_PHYS(folio_pfn(subpage->owner->folio)) + mm_subpage_offset(subpage); +} + +/* + * Round the mm's aggregate reservation, never each VMA: splitting a VMA + * must not create extra charges or refunds. mmap writers serialize updates; + * munmap may retain the lock downgraded while finishing its refund. + */ +static int mm_commit_delta(struct mm_struct *mm, long pages, + unsigned long *next, long *delta) +{ + unsigned long old = mm->committed_user_pages; + unsigned int shift = mm_page_shift(mm); + + mmap_assert_locked(mm); + if (pages < 0 && WARN_ON_ONCE(pages < -(long)old)) + return -EINVAL; + if (pages > 0 && (unsigned long)pages > LONG_MAX - old) + return -ENOMEM; + *next = old + pages; + if (shift > PAGE_SHIFT) { + unsigned int extra = shift - PAGE_SHIFT; + + /* A logical reservation may span several native commit units. */ + if (*next > (LONG_MAX >> extra) || old > (LONG_MAX >> extra)) + return -ENOMEM; + *delta = (long)(*next << extra) - (long)(old << extra); + } else { + unsigned long per_native = 1UL << (PAGE_SHIFT - shift); + + *delta = DIV_ROUND_UP(*next, per_native) - DIV_ROUND_UP(old, per_native); + } + return 0; +} + +int mm_account_memory(struct mm_struct *mm, struct mm_struct *policy_mm, long pages) +{ + unsigned long next; + long delta; + int ret; + + if (mm_page_size(mm) == PAGE_SIZE) + return security_vm_enough_memory_mm(policy_mm, pages); + ret = mm_commit_delta(mm, pages, &next, &delta); + if (ret) + return ret; + ret = security_vm_enough_memory_mm(policy_mm, delta); + if (!ret) + mm->committed_user_pages = next; + return ret; +} + +void mm_acct_memory(struct mm_struct *mm, long pages) +{ + unsigned long next; + long delta; + + if (mm_page_size(mm) == PAGE_SIZE) { + vm_acct_memory(pages); + return; + } + if (WARN_ON_ONCE(mm_commit_delta(mm, pages, &next, &delta))) + return; + vm_acct_memory(delta); + mm->committed_user_pages = next; +} + +void mm_unacct_memory(struct mm_struct *mm, unsigned long pages) +{ + mm_acct_memory(mm, -(long)pages); +} + +/* Shared ownership for consumers whose native page spans contain user slots. */ +struct mm_subpage_refs { + refcount_t refs; + unsigned int count; + unsigned int capacity; + struct mm_subpage *slots[]; +}; + +struct mm_subpage_refs *mm_subpage_refs_get(struct mm_subpage_refs *refs) +{ + if (refs) + refcount_inc(&refs->refs); + return refs; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_get); + +void mm_subpage_refs_put(struct mm_subpage_refs *refs) +{ + unsigned int i; + + if (!refs || !refcount_dec_and_test(&refs->refs)) + return; + for (i = 0; i < refs->count; i++) + mm_subpage_put(refs->slots[i]); + kfree(refs); +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_put); + +static struct mm_subpage_refs *subpage_refs_new(unsigned int capacity, gfp_t gfp) +{ + struct mm_subpage_refs *refs = kmalloc(struct_size(refs, slots, capacity), gfp); + + if (!refs) + return NULL; + refcount_set(&refs->refs, 1); + refs->count = 0; + refs->capacity = capacity; + return refs; +} + +static int subpage_refs_prepare(struct mm_subpage_refs **refsp, + unsigned int needed, gfp_t gfp) +{ + struct mm_subpage_refs *old = *refsp, *refs; + unsigned int i; + + if (old && refcount_read(&old->refs) == 1 && old->capacity >= needed) + return 0; + refs = subpage_refs_new(max(needed, 4U), gfp); + if (!refs) + return -ENOMEM; + if (old) { + for (i = 0; i < old->count; i++) { + mm_subpage_get(old->slots[i]); + refs->slots[refs->count++] = old->slots[i]; + } + } + *refsp = refs; + mm_subpage_refs_put(old); + return 0; +} + +int mm_subpage_refs_add(struct mm_subpage_refs **refsp, struct page *page, + unsigned int offset, unsigned int len, gfp_t gfp) +{ + struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE]; + struct mm_subpage_refs *refs = *refsp; + unsigned int pos, count = 0, i, j; + int ret = 0; + bool found = false; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + if (offset >= PAGE_SIZE || len > PAGE_SIZE - offset) + return -EINVAL; + for (pos = round_down(offset, MM_SUBPAGE_SIZE); pos < offset + len; + pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(page) + pos); + + if (!slot) { + /* A native anonymous allocation has no quarter owner. */ + ret = found ? -EFAULT : 0; + goto out; + } + found = true; + for (j = 0; refs && j < refs->count; j++) + if (refs->slots[j] == slot) + break; + if (refs && j < refs->count) { + mm_subpage_put(slot); + continue; + } + for (j = 0; j < count; j++) + if (slots[j] == slot) + break; + if (j < count) + mm_subpage_put(slot); + else + slots[count++] = slot; + } + if (!count) + return 0; + if (refs && refs->count > UINT_MAX - count) { + ret = -ENOMEM; + goto out; + } + ret = subpage_refs_prepare(refsp, (refs ? refs->count : 0) + count, gfp); + if (ret) + goto out; + refs = *refsp; + for (i = 0; i < count; i++) + refs->slots[refs->count++] = slots[i]; + return 0; +out: + for (i = 0; i < count; i++) + mm_subpage_put(slots[i]); + return ret; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_add); + +int mm_subpage_refs_prune(struct mm_subpage_refs **refsp, + bool (*keep)(const struct mm_subpage *, void *), + void *context, gfp_t gfp) +{ + struct mm_subpage_refs *refs = *refsp, *new; + unsigned int i, count = 0; + + if (!refs) + return 0; + for (i = 0; i < refs->count; i++) + count += keep(refs->slots[i], context); + if (count == refs->count) + return 0; + if (!count) { + *refsp = NULL; + mm_subpage_refs_put(refs); + return 0; + } + if (refcount_read(&refs->refs) == 1) { + for (i = 0, count = 0; i < refs->count; i++) { + if (keep(refs->slots[i], context)) + refs->slots[count++] = refs->slots[i]; + else + mm_subpage_put(refs->slots[i]); + } + refs->count = count; + return 0; + } + new = subpage_refs_new(count, gfp); + if (!new) + return -ENOMEM; + for (i = 0; i < refs->count; i++) { + if (!keep(refs->slots[i], context)) + continue; + mm_subpage_get(refs->slots[i]); + new->slots[new->count++] = refs->slots[i]; + } + *refsp = new; + mm_subpage_refs_put(refs); + return 0; +} +EXPORT_SYMBOL_GPL(mm_subpage_refs_prune); From 9d95e30254c4786ec9557ec33966d886fc9658bf Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 018/631] exec: stage arguments and ELF mappings at the selected user page size Resolve the per-thread exec preference before exposing the new address space. Transfer staged arguments across granules, bound temporary copies by their native mappings, and use the selected ABI for ELF and core dumps. Signed-off-by: Diljit Singh (cherry picked from commit c16faad6c36dd65a3cf47cbde2e89b7ac84e46a5) --- fs/binfmt_elf.c | 42 ++++++-- fs/coredump.c | 81 ++++++++++++--- fs/exec.c | 265 +++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 313 insertions(+), 75 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 16a56b6b3f6ca1..826235fd977c25 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -79,11 +79,11 @@ static int elf_core_dump(struct coredump_params *cprm); #define elf_core_dump NULL #endif -#if ELF_EXEC_PAGESIZE > PAGE_SIZE -#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE -#else -#define ELF_MIN_ALIGN PAGE_SIZE +#ifndef ELF_EXEC_PAGESIZE_MM +#define ELF_EXEC_PAGESIZE_MM(mm) ELF_EXEC_PAGESIZE #endif +#define ELF_MIN_ALIGN max_t(unsigned long, mm_page_size(current->mm), \ + ELF_EXEC_PAGESIZE_MM(current->mm)) #ifndef ELF_CORE_EFLAGS #define ELF_CORE_EFLAGS 0 @@ -93,12 +93,20 @@ static int elf_core_dump(struct coredump_params *cprm); #define ELF_PAGEOFFSET(_v) ((_v) & (ELF_MIN_ALIGN-1)) #define ELF_PAGEALIGN(_v) (((_v) + ELF_MIN_ALIGN - 1) & ~(ELF_MIN_ALIGN - 1)) +#ifdef CONFIG_COREDUMP +static unsigned long elf_min_coredump_size(void) +{ + return ELF_EXEC_PAGESIZE_MM(current->mm); +} +#endif + static struct linux_binfmt elf_format = { .module = THIS_MODULE, .load_binary = load_elf_binary, #ifdef CONFIG_COREDUMP .core_dump = elf_core_dump, .min_coredump = ELF_EXEC_PAGESIZE, + .min_coredump_size = elf_min_coredump_size, #endif }; @@ -248,7 +256,7 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, ARCH_DLINFO; #endif NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); - NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE); + NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE_MM(mm)); NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); NEW_AUX_ENT(AT_PHDR, phdr_addr); NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); @@ -622,6 +630,13 @@ static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp, #endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */ +#ifndef arch_bprm_prepare_mm +static inline int arch_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + return 0; +} +#endif + static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, bool has_interp, bool is_interp) { @@ -1006,6 +1021,11 @@ static int load_elf_binary(struct linux_binprm *bprm) if (retval) goto out_free_dentry; + retval = arch_bprm_prepare_mm(bprm, + elf_ex->e_ident[EI_CLASS] == ELFCLASS32); + if (retval) + goto out_free_dentry; + /* Flush all traces of the currently running executable */ retval = begin_new_exec(bprm); if (retval) @@ -1335,7 +1355,7 @@ static int load_elf_binary(struct linux_binprm *bprm) * leave a gap between .bss and brk. */ if (!brk_moved) - mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; + mm->brk = mm->start_brk = mm->brk + mm_page_size(mm); mm->brk = mm->start_brk = arch_randomize_brk(mm); brk_moved = true; @@ -1351,10 +1371,10 @@ static int load_elf_binary(struct linux_binprm *bprm) and some applications "depend" upon this behavior. Since we do not have the power to recompile these, we emulate the SVr4 behavior. Sigh. */ - error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, + error = vm_mmap(NULL, 0, mm_page_size(mm), PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); - retval = do_mseal(0, PAGE_SIZE, 0); + retval = do_mseal(0, mm_page_size(mm), 0); if (retval) pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", task_pid_nr(current), retval); @@ -1658,7 +1678,7 @@ static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm /* Now we know exact count of files, can store it */ data[0] = count; - data[1] = PAGE_SIZE; + data[1] = mm_page_size(current->mm); /* * Count usually is less than mm->map_count, * we need to move filenames down. @@ -2043,7 +2063,7 @@ static int elf_core_dump(struct coredump_params *cprm) offset += sz; } - dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); + dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE_MM(current->mm)); offset += cprm->vma_data_size; offset += elf_core_extra_data_size(cprm); @@ -2083,7 +2103,7 @@ static int elf_core_dump(struct coredump_params *cprm) phdr.p_flags |= PF_W; if (meta->flags & VM_EXEC) phdr.p_flags |= PF_X; - phdr.p_align = ELF_EXEC_PAGESIZE; + phdr.p_align = ELF_EXEC_PAGESIZE_MM(current->mm); if (!dump_emit(cprm, &phdr, sizeof(phdr))) goto end_coredump; diff --git a/fs/coredump.c b/fs/coredump.c index bb6fdb1f458e9b..6ab5f2e076698d 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -884,7 +885,8 @@ static bool coredump_file(struct core_name *cn, struct coredump_params *cprm, struct file *file __free(fput) = NULL; int open_flags = O_CREAT | O_WRONLY | O_NOFOLLOW | O_LARGEFILE | O_EXCL; - if (cprm->limit < binfmt->min_coredump) + if (cprm->limit < (binfmt->min_coredump_size ? + binfmt->min_coredump_size() : binfmt->min_coredump)) return false; if (coredump_force_suid_safe(cprm) && cn->corename[0] != '/') { @@ -1278,7 +1280,8 @@ void dump_skip(struct coredump_params *cprm, size_t nr) EXPORT_SYMBOL(dump_skip); #ifdef CONFIG_ELF_CORE -static int dump_emit_page(struct coredump_params *cprm, struct page *page) +static int dump_emit_page(struct coredump_params *cprm, struct page *page, + unsigned int offset, unsigned int size) { struct bio_vec bvec; struct iov_iter iter; @@ -1294,19 +1297,19 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) return 0; cprm->to_skip = 0; } - if (cprm->written + PAGE_SIZE > cprm->limit) + if (cprm->written + size > cprm->limit) return 0; if (dump_interrupted()) return 0; pos = file->f_pos; - bvec_set_page(&bvec, page, PAGE_SIZE, 0); - iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); + bvec_set_page(&bvec, page, size, offset); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); n = __kernel_write_iter(cprm->file, &iter, &pos); - if (n != PAGE_SIZE) + if (n != size) return 0; file->f_pos = pos; - cprm->written += PAGE_SIZE; - cprm->pos += PAGE_SIZE; + cprm->written += size; + cprm->pos += size; return 1; } @@ -1321,10 +1324,11 @@ static int dump_emit_page(struct coredump_params *cprm, struct page *page) #define dump_page_alloc() alloc_page(GFP_KERNEL) #define dump_page_free(x) __free_page(x) -static struct page *dump_page_copy(struct page *src, struct page *dst) +static struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { void *buf = kmap_local_page(src); - size_t left = copy_mc_to_kernel(page_address(dst), buf, PAGE_SIZE); + size_t left = copy_mc_to_kernel(page_address(dst) + offset, buf + offset, size); kunmap_local(buf); return left ? NULL : dst; } @@ -1334,12 +1338,51 @@ static struct page *dump_page_copy(struct page *src, struct page *dst) /* We just want to return non-NULL; it's never used. */ #define dump_page_alloc() ERR_PTR(-EINVAL) #define dump_page_free(x) ((void)(x)) -static inline struct page *dump_page_copy(struct page *src, struct page *dst) +static inline struct page *dump_page_copy(struct page *src, struct page *dst, + unsigned int offset, unsigned int size) { return src; } #endif +#ifdef CONFIG_MM_SUBPAGE +static int dump_subpage_range(struct coredump_params *cprm, unsigned long start, + unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm), addr; + struct page *scratch = dump_page_alloc(); + int ret = 0; + + if (!scratch) + return 0; + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + + if (get_dump_fragment(addr, &fragment) == 1) { + struct page *page = folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment.offset); + int emitted; + + emitted = dump_emit_page(cprm, + dump_page_copy(page, scratch, offset, size), offset, size); + release_user_fragments(&fragment, 1, false); + if (!emitted) + goto out; + } else { + dump_skip(cprm, size); + } + if (dump_interrupted()) + goto out; + cond_resched(); + } + ret = 1; +out: + dump_page_free(scratch); + return ret; +} +#endif + int dump_user_range(struct coredump_params *cprm, unsigned long start, unsigned long len) { @@ -1347,6 +1390,10 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, struct page *dump_page; int locked, ret; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return dump_subpage_range(cprm, start, len); +#endif dump_page = dump_page_alloc(); if (!dump_page) return 0; @@ -1375,7 +1422,8 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start, mmap_read_unlock(current->mm); locked = 0; } - int stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page)); + int stop = !dump_emit_page(cprm, + dump_page_copy(page, dump_page, 0, PAGE_SIZE), 0, PAGE_SIZE); put_page(page); if (stop) goto out; @@ -1652,9 +1700,10 @@ static unsigned long vma_dump_size(struct vm_area_struct *vma, * dump the first page to aid in determining what was mapped here. */ if (FILTER(ELF_HEADERS) && - vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) { + vma->vm_pgoff == 0 && !vma_subpage_offset(vma) && + (vma->vm_flags & VM_READ)) { if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0) - return PAGE_SIZE; + return min(mm_page_size(vma->vm_mm), vma->vm_end - vma->vm_start); /* * ELF libraries aren't always executable. @@ -1754,7 +1803,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) m->end = vma->vm_end; m->flags = vma->vm_flags; m->dump_size = vma_dump_size(vma, cprm->mm_flags); - m->pgoff = vma->vm_pgoff; + m->pgoff = core_vma_pgoff(vma); m->file = vma->vm_file; if (m->file) get_file(m->file); @@ -1773,7 +1822,7 @@ static bool dump_vma_snapshot(struct coredump_params *cprm) memcmp(elfmag, ELFMAG, SELFMAG) != 0) { m->dump_size = 0; } else { - m->dump_size = PAGE_SIZE; + m->dump_size = min(mm_page_size(mm), m->end - m->start); } } diff --git a/fs/exec.c b/fs/exec.c index 2eacc1441c59fb..cb5cda32f8e304 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +123,54 @@ bool path_noexec(const struct path *path) (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); } +/* A temporary argument mapping owns one native page or an exact subpage slot. */ +struct arg_page { + struct page *page; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage *subpage; + unsigned int offset; +#endif +}; + +static unsigned long bprm_page_size(const struct linux_binprm *bprm) +{ +#ifdef CONFIG_MMU + return mm_page_size(bprm->mm); +#else + return PAGE_SIZE; +#endif +} + +/* kmap_local_page() maps one native page, even for a larger user leaf. */ +static unsigned long bprm_copy_size(const struct linux_binprm *bprm) +{ + return min(bprm_page_size(bprm), PAGE_SIZE); +} + +static unsigned long bprm_copy_mask(const struct linux_binprm *bprm) +{ + return ~(bprm_copy_size(bprm) - 1); +} + +static unsigned long bprm_max_arg_strlen(const struct linux_binprm *bprm) +{ + return (MAX_ARG_STRLEN / PAGE_SIZE) * bprm_page_size(bprm); +} + +static unsigned int arg_page_offset(struct arg_page page) +{ +#ifdef CONFIG_MM_SUBPAGE + return page.offset; +#else + return 0; +#endif +} + +static void *map_arg_page(struct arg_page page) +{ + return kmap_local_page(page.page) + arg_page_offset(page); +} + #ifdef CONFIG_MMU /* * The nascent bprm->mm is not visible until exec_mmap() but it can @@ -141,10 +190,10 @@ static void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) add_mm_counter(mm, MM_ANONPAGES, diff); } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { - struct page *page; + struct arg_page page = {}; struct vm_area_struct *vma = bprm->vma; struct mm_struct *mm = bprm->mm; int ret; @@ -155,28 +204,47 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, * ahead of time. */ if (!mmap_read_lock_maybe_expand(mm, vma, pos, write)) - return NULL; + return page; /* * We are doing an exec(). 'current' is the process * doing the exec and 'mm' is the new process's mm. */ - ret = get_user_pages_remote(mm, pos, 1, - write ? FOLL_WRITE : 0, - &page, NULL); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + struct vm_area_struct *area; + + ret = get_user_fragment_vma_remote(mm, pos, 1, write ? FOLL_WRITE : 0, + &fragment, &area); + if (ret > 0) { + page.page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + page.subpage = fragment.subpage; + page.offset = offset_in_page(fragment.offset) & mm_page_mask(mm); + } + } else +#endif + ret = get_user_pages_remote(mm, pos, 1, + write ? FOLL_WRITE : 0, &page.page, NULL); mmap_read_unlock(mm); if (ret <= 0) - return NULL; + return page; if (write) - acct_arg_size(bprm, vma_pages(vma)); + acct_arg_size(bprm, mm_pages_for_reader(mm, current->mm, vma_user_pages(vma))); return page; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { - put_page(page); +#ifdef CONFIG_MM_SUBPAGE + if (page.subpage) { + mm_subpage_put(page.subpage); + return; + } +#endif + put_page(page.page); } static void free_arg_pages(struct linux_binprm *bprm) @@ -184,14 +252,14 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { - flush_cache_page(bprm->vma, pos, page_to_pfn(page)); + flush_cache_page(bprm->vma, pos, page_to_pfn(page.page)); } static bool valid_arg_len(struct linux_binprm *bprm, long len) { - return len <= MAX_ARG_STRLEN; + return len <= bprm_max_arg_strlen(bprm); } #else @@ -200,7 +268,7 @@ static inline void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) { } -static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, +static struct arg_page get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { struct page *page; @@ -209,14 +277,14 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, if (!page && write) { page = alloc_page(GFP_HIGHUSER|__GFP_ZERO); if (!page) - return NULL; + return (struct arg_page) {}; bprm->page[pos / PAGE_SIZE] = page; } - return page; + return (struct arg_page) { .page = page }; } -static void put_arg_page(struct page *page) +static void put_arg_page(struct arg_page page) { } @@ -237,7 +305,7 @@ static void free_arg_pages(struct linux_binprm *bprm) } static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, - struct page *page) + struct arg_page page) { } @@ -259,7 +327,7 @@ static int bprm_mm_init(struct linux_binprm *bprm) int err; struct mm_struct *mm = NULL; - bprm->mm = mm = mm_alloc(); + bprm->mm = mm = mm_alloc_exec(); err = -ENOMEM; if (!mm) goto err; @@ -288,6 +356,102 @@ static int bprm_mm_init(struct linux_binprm *bprm) return err; } +#ifdef CONFIG_MM_SUBPAGE +/* + * Final ELF selection may change the granule after argv, envp and script + * arguments have been staged. Table geometry is immutable: build a second mm + * and transfer bytes at the same temporary stack addresses before committing + * exec. Until success, the original bprm and its argument charge remain intact. + */ +int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift) +{ + struct linux_binprm replacement = {}; + struct mm_struct *old_mm; + unsigned long pos, end, new_top, string_len = 0; + unsigned long old_size = mm_user_fragment_size(bprm->mm), new_size; + int ret; + + if (mm_page_shift(bprm->mm) == page_shift) + return 0; + if (WARN_ON_ONCE(bprm->point_of_no_return)) + return -EINVAL; + + replacement.mm = mm_alloc_exec_page_shift(page_shift); + if (!replacement.mm) + return -ENOMEM; + ret = -EINVAL; + if (mm_page_shift(replacement.mm) != page_shift) + goto out; + new_size = mm_user_fragment_size(replacement.mm); + ret = create_init_stack_vma(replacement.mm, &replacement.vma, &new_top); + if (ret) + goto out; + end = bprm->vma->vm_end - sizeof(void *); + ret = -EINVAL; + if (new_top != end || bprm->p > end) + goto out; + + for (pos = bprm->p; pos < end;) { + struct arg_page src, dst; + unsigned long src_off = pos & (old_size - 1); + unsigned long dst_off = pos & (new_size - 1); + unsigned long len = min3(end - pos, old_size - src_off, + new_size - dst_off); + const char *from; + char *to; + + ret = -ERESTARTNOHAND; + if (fatal_signal_pending(current)) + goto out; + src = get_arg_page(bprm, pos, 0); + ret = -EFAULT; + if (!src.page) + goto out; + dst = get_arg_page(&replacement, pos, 1); + if (!dst.page) { + put_arg_page(src); + ret = -ENOMEM; + goto out; + } + from = map_arg_page(src); + to = map_arg_page(dst); + /* Recheck per-string limits if the selected granule shrank. */ + ret = 0; + for (unsigned long i = 0; i < len; i++) { + if (++string_len > bprm_max_arg_strlen(&replacement)) { + ret = -E2BIG; + break; + } + if (!from[src_off + i]) + string_len = 0; + } + if (!ret) + memcpy(to + dst_off, from + src_off, len); + kunmap_local(to); + kunmap_local(from); + flush_arg_page(&replacement, pos, dst); + put_arg_page(dst); + put_arg_page(src); + if (ret) + goto out; + pos += len; + cond_resched(); + } + + old_mm = bprm->mm; + acct_arg_size(bprm, 0); + bprm->mm = replacement.mm; + bprm->vma = replacement.vma; + bprm->vma_pages = replacement.vma_pages; + mmput(old_mm); + return 0; +out: + acct_arg_size(&replacement, 0); + mmput(replacement.mm); + return ret; +} +#endif + struct user_arg_ptr { #ifdef CONFIG_COMPAT bool is_compat; @@ -449,7 +613,7 @@ static int bprm_stack_limits(struct linux_binprm *bprm) static int copy_strings(int argc, struct user_arg_ptr argv, struct linux_binprm *bprm) { - struct page *kmapped_page = NULL; + struct arg_page kmapped_page = {}; char *kaddr = NULL; unsigned long kpos = 0; int ret; @@ -464,7 +628,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, if (IS_ERR(str)) goto out; - len = strnlen_user(str, MAX_ARG_STRLEN); + len = strnlen_user(str, bprm_max_arg_strlen(bprm)); if (!len) goto out; @@ -488,9 +652,9 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } cond_resched(); - offset = pos % PAGE_SIZE; + offset = pos & ~bprm_copy_mask(bprm); if (offset == 0) - offset = PAGE_SIZE; + offset = bprm_copy_size(bprm); bytes_to_copy = offset; if (bytes_to_copy > len) @@ -501,23 +665,23 @@ static int copy_strings(int argc, struct user_arg_ptr argv, str -= bytes_to_copy; len -= bytes_to_copy; - if (!kmapped_page || kpos != (pos & PAGE_MASK)) { - struct page *page; + if (!kmapped_page.page || kpos != (pos & bprm_copy_mask(bprm))) { + struct arg_page page; page = get_arg_page(bprm, pos, 1); - if (!page) { + if (!page.page) { ret = -E2BIG; goto out; } - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } kmapped_page = page; - kaddr = kmap_local_page(kmapped_page); - kpos = pos & PAGE_MASK; + kaddr = map_arg_page(kmapped_page); + kpos = pos & bprm_copy_mask(bprm); flush_arg_page(bprm, kpos, kmapped_page); } if (copy_from_user(kaddr+offset, str, bytes_to_copy)) { @@ -528,8 +692,8 @@ static int copy_strings(int argc, struct user_arg_ptr argv, } ret = 0; out: - if (kmapped_page) { - flush_dcache_page(kmapped_page); + if (kmapped_page.page) { + flush_dcache_page(kmapped_page.page); kunmap_local(kaddr); put_arg_page(kmapped_page); } @@ -541,7 +705,7 @@ static int copy_strings(int argc, struct user_arg_ptr argv, */ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) { - int len = strnlen(arg, MAX_ARG_STRLEN) + 1 /* terminating NUL */; + int len = strnlen(arg, bprm_max_arg_strlen(bprm)) + 1 /* terminating NUL */; unsigned long pos = bprm->p; if (len == 0) @@ -557,18 +721,19 @@ int copy_string_kernel(const char *arg, struct linux_binprm *bprm) while (len > 0) { unsigned int bytes_to_copy = min(len, - min_not_zero(offset_in_page(pos), PAGE_SIZE)); - struct page *page; + min_not_zero(pos & ~bprm_copy_mask(bprm), bprm_copy_size(bprm))); + struct arg_page page; pos -= bytes_to_copy; arg -= bytes_to_copy; len -= bytes_to_copy; page = get_arg_page(bprm, pos, 1); - if (!page) + if (!page.page) return -E2BIG; - flush_arg_page(bprm, pos & PAGE_MASK, page); - memcpy_to_page(page, offset_in_page(pos), arg, bytes_to_copy); + flush_arg_page(bprm, pos & bprm_copy_mask(bprm), page); + memcpy_to_page(page.page, arg_page_offset(page) + + (pos & ~bprm_copy_mask(bprm)), arg, bytes_to_copy); put_arg_page(page); } @@ -621,20 +786,20 @@ int setup_arg_pages(struct linux_binprm *bprm, /* Add space for stack randomization. */ if (current->flags & PF_RANDOMIZE) - stack_base += (STACK_RND_MASK << PAGE_SHIFT); + stack_base += (STACK_RND_MASK << mm_page_shift(mm)); /* Make sure we didn't let the argument array grow too large. */ if (vma->vm_end - vma->vm_start > stack_base) return -ENOMEM; - stack_base = PAGE_ALIGN(stack_top - stack_base); + stack_base = mm_page_align(mm, stack_top - stack_base); stack_shift = vma->vm_start - stack_base; mm->arg_start = bprm->p - stack_shift; bprm->p = vma->vm_end - stack_shift; #else stack_top = arch_align_stack(stack_top); - stack_top = PAGE_ALIGN(stack_top); + stack_top = mm_page_align(mm, stack_top); if (unlikely(stack_top < mmap_min_addr) || unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr)) @@ -702,7 +867,7 @@ int setup_arg_pages(struct linux_binprm *bprm, * Align this down to a page boundary as expand_stack * will align it up. */ - rlim_stack = bprm->rlim_stack.rlim_cur & PAGE_MASK; + rlim_stack = bprm->rlim_stack.rlim_cur & mm_page_mask(mm); stack_expand = min(rlim_stack, stack_size + stack_expand); @@ -1323,6 +1488,10 @@ void setup_new_exec(struct linux_binprm * bprm) arch_pick_mmap_layout(me->mm, &bprm->rlim_stack); + /* Consume the one-shot request only after committing to the new image. */ +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + me->exec_page_shift = 0; +#endif arch_setup_new_exec(); /* Set the new mm task size. We have to do that late because it may @@ -1616,25 +1785,25 @@ int remove_arg_zero(struct linux_binprm *bprm) { unsigned long offset; char *kaddr; - struct page *page; + struct arg_page page; if (!bprm->argc) return 0; do { - offset = bprm->p & ~PAGE_MASK; + offset = bprm->p & ~bprm_copy_mask(bprm); page = get_arg_page(bprm, bprm->p, 0); - if (!page) + if (!page.page) return -EFAULT; - kaddr = kmap_local_page(page); + kaddr = map_arg_page(page); - for (; offset < PAGE_SIZE && kaddr[offset]; + for (; offset < bprm_copy_size(bprm) && kaddr[offset]; offset++, bprm->p++) ; kunmap_local(kaddr); put_arg_page(page); - } while (offset == PAGE_SIZE); + } while (offset == bprm_copy_size(bprm)); bprm->p++; bprm->argc--; From d97e37b9b97ef46826e64ed6c563a4ab8851c14f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 019/631] block: preserve user fragment ownership across I/O Carry the backing offset and retained fragment reference through bio, splice and io_uring paths so a smaller user mapping is not treated as an entire native physical page. Signed-off-by: Diljit Singh (cherry picked from commit c5f12989feb7fadc77031ea60829bdc879112864) --- block/bio.c | 150 +++++++++++++++++++++++++++++++++ block/blk-map.c | 2 +- fs/splice.c | 172 ++++++++++++++++++++++++++++++++++++++ include/linux/bio.h | 8 ++ include/linux/blk_types.h | 4 + io_uring/memmap.c | 8 +- io_uring/rsrc.c | 124 +++++++++++++++++++++++++++ 7 files changed, 466 insertions(+), 2 deletions(-) diff --git a/block/bio.c b/block/bio.c index 5f10900b3f42a1..846417dec7d164 100644 --- a/block/bio.c +++ b/block/bio.c @@ -3,6 +3,7 @@ * Copyright (C) 2001 Jens Axboe */ #include +#include #include #include #include @@ -246,6 +247,10 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, #endif #ifdef CONFIG_BLK_DEV_INTEGRITY bio->bi_integrity = NULL; +#endif +#ifdef CONFIG_MM_SUBPAGE + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; #endif bio->bi_vcnt = 0; @@ -1159,6 +1164,18 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) { struct folio_iter fi; +#ifdef CONFIG_MM_SUBPAGE + if (bio->bi_user_fragments) { + release_user_fragments(bio->bi_user_fragments, + bio->bi_nr_user_fragments, mark_dirty); + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + bio->bi_nr_user_fragments = 0; + bio_clear_flag(bio, BIO_PAGE_PINNED); + return; + } +#endif + bio_for_each_folio_all(fi, bio) { size_t nr_pages; @@ -1220,6 +1237,134 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE +static bool bio_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && + mm_page_size(current->mm) < PAGE_SIZE; +} + +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) +{ + struct iov_iter walk = *iter; + unsigned int count = 0; + + if (!bio_subpage_iter(iter)) + return iov_iter_npages(iter, max_vecs); + while (iov_iter_count(&walk) && count < max_vecs) { + size_t len, offset, nr; + + iov_iter_advance(&walk, 0); + len = min(iter_iov_len(&walk), iov_iter_count(&walk)); + if (WARN_ON_ONCE(!len)) + return max_vecs; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + nr = (len >> MM_SUBPAGE_SHIFT) + + DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + if (nr >= max_vecs - count) + return max_vecs; + count += nr; + iov_iter_advance(&walk, len); + } + return count; +} +EXPORT_SYMBOL_GPL(bio_iov_user_vecs); + +/* Pins belong to the submitting bio, just as with ordinary BIO_PAGE_PINNED. + * Split/cloned bios borrow its vectors and complete before their owner does. + */ +static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, + unsigned int align_mask) +{ + unsigned int capacity = bio->bi_max_vecs, total = 0; + unsigned int flags = iov_iter_rw(iter) == READ ? FOLL_WRITE : 0; + int ret = 0; + + if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) + return -EINVAL; + bio->bi_user_fragments = kmalloc_array(capacity, + sizeof(*bio->bi_user_fragments), GFP_KERNEL); + if (!bio->bi_user_fragments) + return -ENOMEM; + while (iov_iter_count(iter) && bio->bi_nr_user_fragments < capacity && + total < BIO_MAX_SIZE) { + unsigned int i, bytes = 0; + long nr; + + iov_iter_advance(iter, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(iter), + min3(iter_iov_len(iter), iov_iter_count(iter), + (size_t)(BIO_MAX_SIZE - total)), flags, + bio->bi_user_fragments + bio->bi_nr_user_fragments, + capacity - bio->bi_nr_user_fragments); + if (nr <= 0) { + ret = nr ?: -EFAULT; + break; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments++]; + struct page *page = folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(fragment->offset); + struct bio_vec *prev = bio->bi_vcnt ? + &bio->bi_io_vec[bio->bi_vcnt - 1] : NULL; + + /* Coalesce physically adjacent quarters within one folio. */ + if (prev && page_folio(prev->bv_page) == fragment->folio && + page_to_phys(prev->bv_page) + prev->bv_offset + prev->bv_len == + page_to_phys(page) + offset) + prev->bv_len += fragment->length; + else + bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt++], page, + fragment->length, offset); + bytes += fragment->length; + } + iov_iter_advance(iter, bytes); + total += bytes; + } + /* Trim pins separately from vectors: several pins may share one bvec. */ + if (total & align_mask) { + unsigned int trim = total & align_mask, left = trim; + + while (left) { + struct user_page_fragment *fragment = + &bio->bi_user_fragments[bio->bi_nr_user_fragments - 1]; + unsigned int bytes = min(left, fragment->length); + + if (bytes == fragment->length) { + release_user_fragments(fragment, 1, false); + bio->bi_nr_user_fragments--; + } else { + fragment->length -= bytes; + } + left -= bytes; + } + left = trim; + while (left) { + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; + unsigned int bytes = min(left, bv->bv_len); + + bv->bv_len -= bytes; + if (!bv->bv_len) + bio->bi_vcnt--; + left -= bytes; + } + iov_iter_revert(iter, trim); + total -= trim; + } + bio->bi_iter.bi_size = total; + if (!total) { + kfree(bio->bi_user_fragments); + bio->bi_user_fragments = NULL; + return ret ?: -EFAULT; + } + bio_set_flag(bio, BIO_PAGE_PINNED); + return 0; +} +#endif + /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to @@ -1255,6 +1400,11 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_MM_SUBPAGE + if (bio_subpage_iter(iter)) + return bio_iov_iter_get_fragments(bio, iter, len_align_mask); +#endif + if (iov_iter_extract_will_pin(iter)) bio_set_flag(bio, BIO_PAGE_PINNED); if (bio->bi_bdev && blk_queue_pci_p2pdma(bio->bi_bdev->bd_disk->queue)) diff --git a/block/blk-map.c b/block/blk-map.c index 768549f19f97ec..b98eed276aea25 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -260,7 +260,7 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { - unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); + unsigned int nr_vecs = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); struct bio *bio; int ret; diff --git a/fs/splice.c b/fs/splice.c index 9d8f63e2fd1abe..4b707ce00e86dd 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -25,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -183,6 +185,79 @@ static void wakeup_pipe_readers(struct pipe_inode_info *pipe) kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); } +#ifdef CONFIG_MM_SUBPAGE +/* A shared lifetime for all user leaves coalesced into one native pipe page. */ +struct subpage_pipe_refs { + refcount_t refs; + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void subpage_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + if (refcount_dec_and_test(&refs->refs)) { + release_user_fragments(refs->fragments, refs->count, false); + kfree(refs); + } +} + +static bool subpage_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs = (void *)buf->private; + + return refcount_inc_not_zero(&refs->refs); +} + +static const struct pipe_buf_operations subpage_pipe_buf_ops = { + .release = subpage_pipe_buf_release, + .get = subpage_pipe_buf_get, + /* A gift of user leaves is not ownership of the containing native page. */ +}; + +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + struct subpage_pipe_refs *refs; + unsigned int pos, count = 0; + + if (!buf->len || buf->ops != &nosteal_pipe_buf_ops || + !folio_test_anon(page_folio(buf->page))) + return 0; + refs = kmalloc(struct_size(refs, fragments, PAGE_SIZE / MM_SUBPAGE_SIZE), GFP_KERNEL); + if (!refs) + return -ENOMEM; + for (pos = round_down(buf->offset, MM_SUBPAGE_SIZE); + pos < buf->offset + buf->len; pos += MM_SUBPAGE_SIZE) { + struct mm_subpage *slot = mm_subpage_get_from_phys(page_to_phys(buf->page) + pos); + + if (!slot) { + release_user_fragments(refs->fragments, count, false); + kfree(refs); + return count ? -EFAULT : 0; + } + refs->fragments[count++] = (struct user_page_fragment) { + .folio = page_folio(buf->page), + .subpage = slot, + .offset = pos, + .length = MM_SUBPAGE_SIZE, + }; + } + refcount_set(&refs->refs, 1); + refs->count = count; + /* Replace the incoming native reference with the typed references. */ + put_page(buf->page); + buf->ops = &subpage_pipe_buf_ops; + buf->private = (unsigned long)refs; + return 0; +} +#else +static int subpage_pipe_buf_retain(struct pipe_buffer *buf) +{ + return 0; +} +#endif + /** * splice_to_pipe - fill passed data into a pipe * @pipe: pipe to fill @@ -221,6 +296,16 @@ ssize_t splice_to_pipe(struct pipe_inode_info *pipe, buf->private = spd->partial[page_nr].private; buf->ops = spd->ops; buf->flags = 0; + { + int err = subpage_pipe_buf_retain(buf); + + if (err) { + buf->ops = NULL; + if (!ret) + ret = err; + break; + } + } head++; pipe->head = head; @@ -852,6 +937,16 @@ ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, continue; } +#ifdef CONFIG_MM_SUBPAGE + /* Other transports still need their downstream lifetime audit. */ + if (buf->ops == &subpage_pipe_buf_ops && + sock->ops->family != AF_UNIX && + sock->ops->family != AF_INET && + sock->ops->family != AF_INET6) { + ret = -EOPNOTSUPP; + break; + } +#endif seg = min_t(size_t, remain, buf->len); ret = pipe_buf_confirm(pipe, buf); @@ -1440,6 +1535,79 @@ static ssize_t __do_splice(struct file *in, loff_t __user *off_in, return ret; } +#ifdef CONFIG_MM_SUBPAGE +static ssize_t subpage_iter_to_pipe(struct iov_iter *from, + struct pipe_inode_info *pipe, unsigned int flags) +{ + struct user_page_fragment fragments[16]; + size_t total = 0; + long ret = 0; + + if (WARN_ON_ONCE(!user_backed_iter(from))) + return -EINVAL; + while (iov_iter_count(from)) { + unsigned int i = 0; + long count; + + /* Normalize empty iovecs before taking the next address. */ + iov_iter_advance(from, 0); + count = get_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min(iov_iter_count(from), iter_iov_len(from)), 0, + fragments, ARRAY_SIZE(fragments)); + if (count <= 0) { + ret = count; + break; + } + while (i < count) { + struct user_page_fragment *first = &fragments[i]; + struct page *page = folio_page(first->folio, first->offset >> PAGE_SHIFT); + unsigned int offset = offset_in_page(first->offset); + unsigned int length = first->length, nr = 1; + struct subpage_pipe_refs *refs; + struct pipe_buffer buf; + + /* Preserve native pipe capacity for contiguous packed quarters. */ + while (i + nr < count && length < PAGE_SIZE - offset) { + struct user_page_fragment *next = &fragments[i + nr]; + + if (folio_page(next->folio, next->offset >> PAGE_SHIFT) != page || + offset_in_page(next->offset) != offset + length) + break; + length += next->length; + nr++; + } + refs = kmalloc(struct_size(refs, fragments, nr), GFP_KERNEL); + if (!refs) { + ret = -ENOMEM; + break; + } + refcount_set(&refs->refs, 1); + refs->count = nr; + memcpy(refs->fragments, first, sizeof(*first) * nr); + buf = (struct pipe_buffer) { + .page = page, + .offset = offset, + .len = length, + .ops = &subpage_pipe_buf_ops, + .flags = flags, + .private = (unsigned long)refs, + }; + i += nr; /* add_to_pipe consumes these references even on failure. */ + ret = add_to_pipe(pipe, &buf); + if (ret < 0) + break; + iov_iter_advance(from, ret); + total += ret; + } + release_user_fragments(fragments + i, count - i, false); + if (ret < 0) + break; + } + return total ? total : ret; +} +#endif + static ssize_t iter_to_pipe(struct iov_iter *from, struct pipe_inode_info *pipe, unsigned int flags) @@ -1451,6 +1619,10 @@ static ssize_t iter_to_pipe(struct iov_iter *from, size_t total = 0; ssize_t ret = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return subpage_iter_to_pipe(from, pipe, flags); +#endif while (iov_iter_count(from)) { struct page *pages[16]; ssize_t left; diff --git a/include/linux/bio.h b/include/linux/bio.h index dc17780d6c1e36..714e44044373aa 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -389,11 +389,19 @@ static inline void bio_wouldblock_error(struct bio *bio) * pointed by @iter. If @iter is backed by bvec it's going to be reused * instead of allocating a new one. */ +#ifdef CONFIG_MM_SUBPAGE +int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs); +#endif + static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) { if (iov_iter_is_bvec(iter)) return 0; +#ifdef CONFIG_MM_SUBPAGE + return bio_iov_user_vecs(iter, max_segs); +#else return iov_iter_npages(iter, max_segs); +#endif } /** diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c09..cc9b1eeddd415b 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -267,6 +267,10 @@ struct bio { struct bio_integrity_payload *bi_integrity; /* data integrity */ #endif +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *bi_user_fragments; + unsigned short bi_nr_user_fragments; +#endif unsigned short bi_vcnt; /* how many bio_vec's */ /* diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 4f9b439319c466..72e123757bea4f 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -289,7 +289,10 @@ static int io_region_mmap(struct io_ring_ctx *ctx, unsigned long nr_pages = min(mr->nr_pages, max_pages); vm_flags_set(vma, VM_DONTEXPAND); - return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + if (mm_page_size(vma->vm_mm) == PAGE_SIZE) + return vm_insert_pages(vma, vma->vm_start, mr->pages, &nr_pages); + return vm_insert_pages_range(vma, vma->vm_start, mr->pages, nr_pages, 0, + min_t(unsigned long, vma->vm_end - vma->vm_start, nr_pages << PAGE_SHIFT)); } __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) @@ -301,6 +304,9 @@ __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma) struct io_mapped_region *region; void *ptr; + if (vma_subpage_offset(vma)) + return -EINVAL; + guard(mutex)(&ctx->mmap_lock); ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 2dfc26233e2865..33cc1e26884801 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -762,6 +763,95 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, return true; } +#ifdef CONFIG_MM_SUBPAGE +struct io_fragment_buffer { + unsigned int count; + struct user_page_fragment fragments[]; +}; + +static void io_release_fragment_buffer(void *priv) +{ + struct io_fragment_buffer *buffer = priv; + + release_user_fragments(buffer->fragments, buffer->count, false); + kvfree(buffer); +} + +static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, + struct iovec *iov) +{ + unsigned long size = mm_page_size(current->mm); + unsigned long first = (unsigned long)iov->iov_base & (size - 1); + unsigned int count = DIV_ROUND_UP(first + iov->iov_len, size); + struct io_fragment_buffer *buffer; + struct io_mapped_ubuf *imu = NULL; + struct io_rsrc_node *node; + unsigned long charged = 0; + unsigned int i; + struct xarray seen; + long nr; + int ret = -ENOMEM; + + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return ERR_PTR(-ENOMEM); + buffer = kvmalloc(struct_size(buffer, fragments, count), GFP_KERNEL_ACCOUNT); + if (!buffer) + goto free_node; + buffer->count = 0; + nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, + iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); + if (nr > 0) + buffer->count = nr; + if (nr != count) { + ret = nr < 0 ? nr : -EFAULT; + goto free_buffer; + } + imu = io_alloc_imu(ctx, count); + if (!imu) + goto free_buffer; + imu->nr_bvecs = count; + imu->acct_pages = 0; + /* Several user leaves can share native backing, including aliases. */ + xa_init(&seen); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &buffer->fragments[i]; + + ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); + if (!ret) + charged += folio_nr_pages(f->folio); + else if (ret != -EBUSY) + break; + bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), + f->length, offset_in_page(f->offset)); + } + xa_destroy(&seen); + if (ret && ret != -EBUSY) + goto free_imu; + ret = io_account_mem(ctx->user, ctx->mm_account, charged); + if (ret) + goto free_imu; + imu->acct_pages = charged; + imu->ubuf = (unsigned long)iov->iov_base; + imu->len = iov->iov_len; + imu->folio_shift = mm_page_shift(current->mm); + imu->release = io_release_fragment_buffer; + imu->priv = buffer; + imu->flags = 0; + imu->dir = IO_IMU_DEST | IO_IMU_SOURCE; + refcount_set(&imu->refs, 1); + node->buf = imu; + return node; +free_imu: + io_free_imu(ctx, imu); +free_buffer: + io_release_fragment_buffer(buffer); +free_node: + io_cache_free(&ctx->node_cache, node); + return ERR_PTR(ret); +} +#endif + static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, struct page **last_hpage) @@ -787,6 +877,11 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, if (ret) return ERR_PTR(ret); +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) < PAGE_SIZE) + return io_register_fragment_buffer(ctx, iov); +#endif + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) return ERR_PTR(-ENOMEM); @@ -1095,6 +1190,14 @@ static int io_import_fixed(int ddir, struct iov_iter *iter, bvec += seg_skip; offset &= folio_mask; } +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + /* Physical fragment offsets are independent of the user address. */ + size_t first = min_t(size_t, len, bvec->bv_len - offset); + + nr_segs = 1 + DIV_ROUND_UP(len - first, 1UL << imu->folio_shift); + } else +#endif nr_segs = (offset + len + bvec->bv_offset + folio_mask) >> imu->folio_shift; iov_iter_bvec(iter, ddir, bvec, nr_segs, len); iter->iov_offset = offset; @@ -1356,6 +1459,27 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, return -EOVERFLOW; offset = buf_addr - imu->ubuf; +#ifdef CONFIG_MM_SUBPAGE + if (imu->folio_shift < PAGE_SHIFT) { + unsigned int index; + unsigned long first = imu->ubuf & folio_mask; + + offset += first; + index = offset >> imu->folio_shift; + offset &= folio_mask; + if (!index) + offset -= first; + src_bvec = imu->bvec + index; + for (; iov_len; offset = 0, bvec_idx++, src_bvec++) { + size_t bytes = min_t(size_t, iov_len, src_bvec->bv_len - offset); + + bvec_set_page(&res_bvec[bvec_idx], src_bvec->bv_page, + bytes, src_bvec->bv_offset + offset); + iov_len -= bytes; + } + continue; + } +#endif /* * Only the first bvec can have non zero bv_offset, account it * here and work with full folios below. From ebbcde8a05438886a5c1ca55f37f8cf825b49581 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 020/631] net: retain user fragments in networking and BPF consumers Preserve fragment ownership and byte ranges in socket, TLS, BPF and vhost consumers of user memory. Signed-off-by: Diljit Singh (cherry picked from commit e6bf191ec8db185fcc908e5a5410497aac3a7543) --- drivers/vhost/vhost.c | 21 +++ include/linux/bpf.h | 22 +++ include/linux/skmsg.h | 40 +++++- include/net/tls.h | 7 + kernel/bpf/bpf_local_storage.c | 23 ++-- kernel/bpf/syscall.c | 128 ++++++++++++++++-- net/core/skbuff.c | 119 ++++++++++++++++- net/core/skmsg.c | 88 +++++++++++- net/ipv4/tcp_bpf.c | 5 + net/tls/tls_device.c | 22 +++ net/tls/tls_sw.c | 238 +++++++++++++++++++++++++++++++-- 11 files changed, 673 insertions(+), 40 deletions(-) diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index e4c44f2ea413ac..4c3577ce7b85b8 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -2436,6 +2437,26 @@ static int set_bit_to_user(int nr, void __user *addr) int bit = nr + (log % PAGE_SIZE) * 8; int r; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + unsigned long word = log & ~(sizeof(unsigned long) - 1); + + /* Pin the complete atomic word, which stays inside one leaf. + * A relocated leaf's physical offset need not match its VA. + */ + r = pin_user_fragments_remote(current->mm, word, + sizeof(unsigned long), FOLL_WRITE, &fragment, 1); + if (r != 1) + return r < 0 ? r : -EFAULT; + base = kmap_local_folio(fragment.folio, fragment.offset); + set_bit(nr + (log - word) * 8, base); + kunmap_local(base); + release_user_fragments(&fragment, 1, true); + return 0; + } +#endif + r = pin_user_pages_fast(log, 1, FOLL_WRITE, &page); if (r < 0) return r; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a2cf24d1688020..cfec2e3b66979a 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -263,6 +263,11 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; +#ifdef CONFIG_MM_SUBPAGE + /* Private ownership pointer appended to task-storage values with UPTRs. */ + u32 uptr_pins_off; + u32 uptr_extra_size; +#endif struct btf_field fields[]; }; @@ -563,6 +568,15 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src bpf_obj_memcpy(map->record, dst, src, map->value_size, true); } +static inline u32 bpf_obj_uptr_extra_size(const struct btf_record *rec) +{ +#ifdef CONFIG_MM_SUBPAGE + return IS_ERR_OR_NULL(rec) ? 0 : rec->uptr_extra_size; +#else + return 0; +#endif +} + static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) { unsigned long *src_uptr, *dst_uptr; @@ -580,6 +594,14 @@ static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, v dst_uptr = dst + field->offset; swap(*src_uptr, *dst_uptr); } +#ifdef CONFIG_MM_SUBPAGE + if (rec->uptr_extra_size) { + void **src_pins = src + rec->uptr_pins_off; + void **dst_pins = dst + rec->uptr_pins_off; + + swap(*src_pins, *dst_pins); + } +#endif } static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size) diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 937823856de59d..30406f8cee1167 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -41,6 +41,8 @@ struct sk_msg_sg { }; /* UAPI in filter.c depends on struct sk_msg_sg being first element. */ +struct mm_subpage_refs; + struct sk_msg { struct sk_msg_sg sg; void *data; @@ -49,6 +51,9 @@ struct sk_msg { u32 cork_bytes; u32 flags; struct sk_buff *skb; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif struct sock *sk_redir; struct sock *sk; struct list_head list; @@ -124,6 +129,33 @@ struct sk_psock { struct rcu_work rwork; }; +#ifdef CONFIG_MM_SUBPAGE +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len); +void sk_msg_subpages_get(struct sk_msg *msg); +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src); +void sk_msg_subpages_release(struct sk_msg *msg); +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) +{ + dst->subpage_refs = src->subpage_refs; + src->subpage_refs = NULL; +} +#else +static inline int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + return 0; +} +static inline void sk_msg_subpages_get(struct sk_msg *msg) { } +static inline void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) { } +static inline void sk_msg_subpages_release(struct sk_msg *msg) { } +static inline void sk_msg_subpages_transfer(struct sk_msg *dst, struct sk_msg *src) { } +#endif + +static inline void sk_msg_subpages_drop_empty(struct sk_msg *msg) +{ + if (!msg->sg.size) + sk_msg_subpages_release(msg); +} + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, @@ -259,10 +291,14 @@ static inline void sk_msg_compute_data_pointers(struct sk_msg *msg) } } -static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, +static inline int sk_msg_page_add(struct sk_msg *msg, struct page *page, u32 len, u32 offset) { struct scatterlist *sge; + int ret = sk_msg_subpages_add(msg, page, offset, len); + + if (ret) + return ret; get_page(page); sge = sk_msg_elem(msg, msg->sg.end); @@ -272,6 +308,7 @@ static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, __set_bit(msg->sg.end, msg->sg.copy); msg->sg.size += len; sk_msg_iter_next(msg, end); + return 0; } static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) @@ -420,6 +457,7 @@ static inline bool sk_psock_queue_empty(const struct sk_psock *psock) static inline void kfree_sk_msg(struct sk_msg *msg) { + sk_msg_subpages_release(msg); if (msg->skb) consume_skb(msg->skb); kfree(msg); diff --git a/include/net/tls.h b/include/net/tls.h index ebd2550280ae24..79d358a5fe8874 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -34,6 +34,7 @@ #ifndef _TLS_OFFLOAD_H #define _TLS_OFFLOAD_H +#include #include #include #include @@ -139,6 +140,9 @@ struct tls_sw_context_rx { struct tls_strparser strp; atomic_t decrypt_pending; +#ifdef CONFIG_MM_SUBPAGE + struct llist_head subpage_cleanup; +#endif struct sk_buff_head async_hold; struct wait_queue_head wq; }; @@ -148,6 +152,9 @@ struct tls_record_info { u32 end_seq; int len; int num_frags; +#ifdef CONFIG_MM_SUBPAGE + struct mm_subpage_refs *subpage_refs; +#endif skb_frag_t frags[MAX_SKB_FRAGS]; }; diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c index 6fc6a4b672b556..9abd4972966ffe 100644 --- a/kernel/bpf/bpf_local_storage.c +++ b/kernel/bpf/bpf_local_storage.c @@ -24,6 +24,11 @@ select_bucket(struct bpf_local_storage_map *smap, return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)]; } +static u32 bpf_selem_size(const struct bpf_local_storage_map *smap) +{ + return smap->elem_size + bpf_obj_uptr_extra_size(smap->map.record); +} + static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) { struct bpf_map *map = &smap->map; @@ -72,10 +77,10 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, { struct bpf_local_storage_elem *selem; - if (mem_charge(smap, owner, smap->elem_size)) + if (mem_charge(smap, owner, bpf_selem_size(smap))) return NULL; - selem = bpf_map_kmalloc_nolock(&smap->map, smap->elem_size, + selem = bpf_map_kmalloc_nolock(&smap->map, bpf_selem_size(smap), __GFP_ZERO, NUMA_NO_NODE); if (selem) { @@ -91,7 +96,7 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, return selem; } - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return NULL; } @@ -178,7 +183,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * bool free_local_storage, bool pin_owner) { void *owner = local_storage->owner; - u32 uncharge = smap->elem_size; + u32 uncharge = bpf_selem_size(smap); if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == SDATA(selem)) @@ -204,7 +209,7 @@ static void bpf_selem_unlink_storage_nolock_misc(struct bpf_local_storage_elem * /* local_storage->lock must be held and selem->local_storage == local_storage. * The caller must ensure selem->smap is still valid to be - * dereferenced for its smap->elem_size and smap->cache_idx. + * dereferenced for its element size and smap->cache_idx. */ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_elem *selem, @@ -234,7 +239,7 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap; smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); - local_storage->mem_charge += smap->elem_size; + local_storage->mem_charge += bpf_selem_size(smap); RCU_INIT_POINTER(selem->local_storage, local_storage); hlist_add_head_rcu(&selem->snode, &local_storage->list); @@ -577,7 +582,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, err = bpf_local_storage_alloc(owner, smap, selem); if (err) { bpf_selem_free(selem, true); - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); return ERR_PTR(err); } @@ -661,7 +666,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, free_selem: bpf_selem_free_list(&old_selem_free_list, false); if (alloc_selem) { - mem_uncharge(smap, owner, smap->elem_size); + mem_uncharge(smap, owner, bpf_selem_size(smap)); bpf_selem_free(alloc_selem, true); } return err ? ERR_PTR(err) : SDATA(selem); @@ -862,7 +867,7 @@ void bpf_local_storage_map_free(struct bpf_map *map, * exit immediately. * * However, while freeing the storage one still needs to access the - * smap->elem_size to do the uncharging in + * bpf_selem_size(smap) to do the uncharging in * bpf_selem_unlink_storage_nolock(). * * Hence, wait another rcu grace period for the storage to be freed. diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 96d1b797e1523c..c084c1cad03d85 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include @@ -164,6 +166,43 @@ static void maybe_wait_bpf_programs(struct bpf_map *map) synchronize_rcu_expedited(); } +#ifdef CONFIG_MM_SUBPAGE +struct bpf_uptr_pins { + struct work_struct work; + u32 count; + struct user_page_fragment fragments[]; +}; + +static void bpf_uptr_release_work(struct work_struct *work) +{ + struct bpf_uptr_pins *pins = container_of(work, struct bpf_uptr_pins, work); + + release_user_fragments(pins->fragments, pins->count, true); + kfree(pins); +} +#endif + +/* May run from an RCU callback. The bundle owns its pins independently of + * the map, task and value, so dirty release can wait for folio locks in work. + */ +static bool bpf_obj_release_uptr_pins(const struct btf_record *rec, void *obj) +{ +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins **owner, *pins; + + if (!bpf_obj_uptr_extra_size(rec)) + return false; + owner = obj + rec->uptr_pins_off; + pins = *owner; + if (pins) { + *owner = NULL; + queue_work(system_unbound_wq, &pins->work); + return true; + } +#endif + return false; +} + static void unpin_uptr_kaddr(void *kaddr) { if (kaddr) @@ -176,6 +215,9 @@ static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj) void **uptr_addr; int i; + if (bpf_obj_release_uptr_pins(rec, obj)) + return; + for (i = 0, field = rec->fields; i < cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -193,8 +235,12 @@ static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj) __bpf_obj_unpin_uptrs(rec, rec->cnt, obj); } -static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) +static int bpf_obj_pin_uptrs(struct bpf_map *map, void *obj) { + struct btf_record *rec = map->record; +#ifdef CONFIG_MM_SUBPAGE + struct bpf_uptr_pins *pins = NULL; +#endif const struct btf_field *field; const struct btf_type *t; unsigned long start, end; @@ -205,6 +251,18 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) if (!btf_record_has_field(rec, BPF_UPTR)) return 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + pins = bpf_map_kzalloc(map, struct_size(pins, fragments, rec->cnt), + GFP_KERNEL); + if (!pins) + return -ENOMEM; + INIT_WORK(&pins->work, bpf_uptr_release_work); + pins->count = rec->cnt; + *(struct bpf_uptr_pins **)(obj + rec->uptr_pins_off) = pins; + } +#endif + for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) { if (field->type != BPF_UPTR) continue; @@ -221,12 +279,35 @@ static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj) goto unpin_all; } - /* The uptr's struct cannot span across two pages */ - if ((start & PAGE_MASK) != (end & PAGE_MASK)) { + /* The object must fit in one page of the supplying process. */ + if ((start & mm_page_mask(current->mm)) != + (end & mm_page_mask(current->mm))) { err = -EOPNOTSUPP; goto unpin_all; } +#ifdef CONFIG_MM_SUBPAGE + if (pins) { + struct user_page_fragment *fragment = &pins->fragments[i]; + + err = pin_user_fragments_remote(current->mm, start, t->size, + FOLL_LONGTERM | FOLL_WRITE, fragment, 1); + if (err != 1) { + if (!err) + err = -EFAULT; + goto unpin_all; + } + if (fragment->length != t->size || + PageHighMem(folio_page(fragment->folio, + fragment->offset >> PAGE_SHIFT))) { + err = -EOPNOTSUPP; + goto unpin_all; + } + *uptr_addr = folio_address(fragment->folio) + fragment->offset; + continue; + } +#endif + err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page); if (err != 1) goto unpin_all; @@ -291,7 +372,7 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) { err = map->ops->map_push_elem(map, value, flags); } else { - err = bpf_obj_pin_uptrs(map->record, value); + err = bpf_obj_pin_uptrs(map, value); if (!err) { rcu_read_lock(); err = map->ops->map_update_elem(map, key, value, flags); @@ -815,10 +896,12 @@ void bpf_obj_cancel_fields(struct bpf_map *map, void *obj) void bpf_obj_free_fields(const struct btf_record *rec, void *obj) { const struct btf_field *fields; + bool fragment_pins; int i; if (IS_ERR_OR_NULL(rec)) return; + fragment_pins = bpf_obj_release_uptr_pins(rec, obj); fields = rec->fields; for (i = 0; i < rec->cnt; i++) { struct btf_struct_meta *pointee_struct_meta; @@ -859,8 +942,9 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) } break; case BPF_UPTR: - /* The caller ensured that no one is using the uptr */ - unpin_uptr_kaddr(*(void **)field_ptr); + /* The caller ensured that no one is using the uptr. */ + if (!fragment_pins) + unpin_uptr_kaddr(*(void **)field_ptr); break; case BPF_LIST_HEAD: if (WARN_ON_ONCE(rec->spin_lock_off < 0)) @@ -1350,6 +1434,14 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, if (ret < 0) goto free_map_tab; +#ifdef CONFIG_MM_SUBPAGE + if (btf_record_has_field(map->record, BPF_UPTR)) { + map->record->uptr_pins_off = ALIGN(map->value_size, sizeof(void *)); + map->record->uptr_extra_size = map->record->uptr_pins_off + + sizeof(void *) - map->value_size; + } +#endif + if (map->ops->map_check_btf) { ret = map->ops->map_check_btf(map, btf, key_type, value_type); if (ret < 0) @@ -1775,6 +1867,24 @@ static int map_lookup_elem(union bpf_attr *attr) #define BPF_MAP_UPDATE_ELEM_LAST_FIELD flags +static void *bpf_map_memdup_value(struct bpf_map *map, bpfptr_t src, u32 size) +{ + u32 extra = bpf_obj_uptr_extra_size(map->record); + void *value; + + if (!extra) + return kvmemdup_bpfptr(src, size); + value = kvmalloc(size + extra, GFP_USER); + if (!value) + return ERR_PTR(-ENOMEM); + memset(value + size, 0, extra); + if (copy_from_bpfptr(value, src, size)) { + kvfree(value); + return ERR_PTR(-EFAULT); + } + return value; +} + static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) { bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel); @@ -1808,7 +1918,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr) } value_size = bpf_map_value_size(map, attr->flags); - value = kvmemdup_bpfptr(uvalue, value_size); + value = bpf_map_memdup_value(map, uvalue, value_size); if (IS_ERR(value)) { err = PTR_ERR(value); goto free_key; @@ -2021,7 +2131,9 @@ int generic_map_update_batch(struct bpf_map *map, struct file *map_file, if (!key) return -ENOMEM; - value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN); + value = kvmalloc(value_size + bpf_obj_uptr_extra_size(map->record), + GFP_USER | __GFP_NOWARN | + (bpf_obj_uptr_extra_size(map->record) ? __GFP_ZERO : 0)); if (!value) { kvfree(key); return -ENOMEM; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a5cdcc0fc55e1c..0bc6467aae5d4b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -7347,8 +7348,113 @@ nodefer: kfree_skb_napi_cache(skb); kick_defer_list_purge(cpu); } +#ifdef CONFIG_MM_SUBPAGE +/* The ordinary frag page ref protects allocation, not an anonymous quarter. */ +struct skb_subpage_refs { + struct ubuf_info ubuf; + struct mm_subpage_refs *owner; +}; + +static void skb_subpage_complete(struct sk_buff *skb, struct ubuf_info *ubuf, + bool success) +{ + struct skb_subpage_refs *refs = container_of(ubuf, struct skb_subpage_refs, ubuf); + + if (!refcount_dec_and_test(&ubuf->refcnt)) + return; + mm_subpage_refs_put(refs->owner); + kfree(refs); +} + +static const struct ubuf_info_ops skb_subpage_ops = { + .complete = skb_subpage_complete, +}; + +static bool skb_contains_subpage(const struct sk_buff *skb, phys_addr_t phys, + unsigned int size, unsigned int depth) +{ + struct sk_buff *child; + unsigned int i; + + /* Match the networking recursion bound, conservatively keeping unknowns. */ + if (depth >= 24) + return true; + if (skb->head_frag && skb_headlen(skb)) { + phys_addr_t start = page_to_phys(virt_to_page(skb->data)) + offset_in_page(skb->data); + + if (start < phys + size && phys < start + skb_headlen(skb)) + return true; + } + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + struct page *page = skb_frag_page(frag); + phys_addr_t start; + + if (!page) + continue; + start = page_to_phys(page) + skb_frag_off(frag); + if (start < phys + size && phys < start + skb_frag_size(frag)) + return true; + } + skb_walk_frags(skb, child) + if (skb_contains_subpage(child, phys, size, depth + 1)) + return true; + return false; +} + +static bool skb_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + return skb_contains_subpage(context, mm_subpage_phys(slot), mm_subpage_size(slot), 0); +} + +/* Caller owns the writable skb and keeps the source slot identity alive. */ +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + struct skb_subpage_refs *refs, *new; + struct ubuf_info *old = skb_zcopy(skb); + struct mm_subpage *probe; + int ret; + + if (!len || folio_order(page_folio(page)) || !folio_test_anon(page_folio(page))) + return 0; + /* Preserve the allocation-free native anonymous path. */ + probe = mm_subpage_get_from_phys(page_to_phys(page) + round_down(off, MM_SUBPAGE_SIZE)); + if (!probe) + return 0; + mm_subpage_put(probe); + if (old && (skb_zcopy_is_nouarg(skb) || old->ops != &skb_subpage_ops)) + return -EOPNOTSUPP; + refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; + if (!refs || refcount_read(&old->refcnt) != 1) { + new = kzalloc(sizeof(*new), GFP_ATOMIC); + if (!new) + return -ENOMEM; + new->ubuf.ops = &skb_subpage_ops; + new->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; + refcount_set(&new->ubuf.refcnt, 1); + if (refs) + new->owner = mm_subpage_refs_get(refs->owner); + skb_zcopy_init(skb, &new->ubuf); + net_zcopy_put(old); + refs = new; + } + /* Repeated trim/append must not accumulate references to discarded data. */ + ret = mm_subpage_refs_prune(&refs->owner, skb_keeps_subpage, skb, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&refs->owner, page, off, len, GFP_ATOMIC); +} +#else +static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, + unsigned int off, unsigned int len) +{ + return 0; +} +#endif + static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, - size_t offset, size_t len) + size_t offset, size_t len, size_t progress) { const char *kaddr; __wsum csum; @@ -7356,7 +7462,8 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, kaddr = kmap_local_page(page); csum = csum_partial(kaddr + offset, len, 0); kunmap_local(kaddr); - skb->csum = csum_block_add(skb->csum, csum, skb->len); + /* skb->len is updated only after the entire iterator batch. */ + skb->csum = csum_block_add(skb->csum, csum, skb->len + progress); } /** @@ -7408,6 +7515,12 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, if (WARN_ON_ONCE(!sendpage_ok(page))) goto out; + ret = skb_splice_hold_subpages(skb, page, off, part); + if (ret < 0) { + iov_iter_revert(iter, len); + goto out; + } + ret = skb_append_pagefrags(skb, page, off, part, frag_limit); if (ret < 0) { @@ -7416,7 +7529,7 @@ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, } if (skb->ip_summed == CHECKSUM_NONE) - skb_splice_csum_page(skb, page, off, part); + skb_splice_csum_page(skb, page, off, part, spliced); off = 0; spliced += part; diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 30c3b9a2681c44..10eb255be0ffa4 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2017 - 2018 Covalent IO, Inc. http://covalent.io */ +#include +#include #include #include #include @@ -24,6 +26,65 @@ static bool sk_msg_try_coalesce_ok(struct sk_msg *msg, int elem_first_coalesce) return false; } +#ifdef CONFIG_MM_SUBPAGE +static bool sk_msg_keeps_subpage(const struct mm_subpage *slot, void *context) +{ + struct sk_msg *msg = context; + phys_addr_t phys = mm_subpage_phys(slot); + phys_addr_t end = phys + mm_subpage_size(slot); + u32 i = msg->sg.start; + + if (!msg->sg.size) + return false; + while (i != msg->sg.end) { + struct scatterlist *sg = &msg->sg.data[i]; + + if (sg->length) { + phys_addr_t start = page_to_phys(sg_page(sg)) + sg->offset; + + if (start < end && phys < start + sg->length) + return true; + } + sk_msg_iter_var_next(i); + } + return false; +} + +int sk_msg_subpages_add(struct sk_msg *msg, struct page *page, u32 offset, u32 len) +{ + int ret; + + ret = mm_subpage_refs_prune(&msg->subpage_refs, sk_msg_keeps_subpage, msg, GFP_ATOMIC); + if (ret) + return ret; + return mm_subpage_refs_add(&msg->subpage_refs, page, offset, len, GFP_ATOMIC); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_add); + +/* Called after a borrowed descriptor copy whose source remains live. */ +void sk_msg_subpages_get(struct sk_msg *msg) +{ + mm_subpage_refs_get(msg->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_get); + +void sk_msg_subpages_share(struct sk_msg *dst, const struct sk_msg *src) +{ + WARN_ON_ONCE(dst->subpage_refs); + dst->subpage_refs = mm_subpage_refs_get(src->subpage_refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_share); + +void sk_msg_subpages_release(struct sk_msg *msg) +{ + struct mm_subpage_refs *refs = msg->subpage_refs; + + msg->subpage_refs = NULL; + mm_subpage_refs_put(refs); +} +EXPORT_SYMBOL_GPL(sk_msg_subpages_release); +#endif + int sk_msg_alloc(struct sock *sk, struct sk_msg *msg, int len, int elem_first_coalesce) { @@ -88,7 +149,7 @@ EXPORT_SYMBOL_GPL(sk_msg_alloc); int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, u32 off, u32 len) { - int i = src->sg.start; + int i = src->sg.start, ret; struct scatterlist *sge = sk_msg_elem(src, i); struct scatterlist *sgd = NULL; u32 sge_len, sge_off; @@ -114,11 +175,16 @@ int sk_msg_clone(struct sock *sk, struct sk_msg *dst, struct sk_msg *src, if (sgd && (sg_page(sge) == sg_page(sgd)) && (sg_virt(sge) + off == sg_virt(sgd) + sgd->length)) { + ret = sk_msg_subpages_add(dst, sg_page(sge), sge->offset + off, sge_len); + if (ret) + return ret; sgd->length += sge_len; dst->sg.size += sge_len; } else if (!sk_msg_full(dst)) { sge_off = sge->offset + off; - sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + ret = sk_msg_page_add(dst, sg_page(sge), sge_len, sge_off); + if (ret) + return ret; } else { return -ENOSPC; } @@ -205,6 +271,7 @@ static int __sk_msg_free(struct sock *sk, struct sk_msg *msg, u32 i, sk_msg_check_to_free(msg, i, msg->sg.size); sge = sk_msg_elem(msg, i); } + sk_msg_subpages_release(msg); consume_skb(msg->skb); sk_msg_init(msg); return freed; @@ -248,6 +315,7 @@ static void __sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, sk_msg_check_to_free(msg, i, bytes); } msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); } void sk_msg_free_partial(struct sock *sk, struct sk_msg *msg, u32 bytes) @@ -267,6 +335,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) int trim = msg->sg.size - len; u32 i = msg->sg.end; + sk_msg_subpages_drop_empty(msg); if (trim <= 0) { WARN_ON(trim < 0); return; @@ -299,6 +368,7 @@ void sk_msg_trim(struct sock *sk, struct sk_msg *msg, int len) * does not require an update. */ if (!msg->sg.size) { + sk_msg_subpages_release(msg); msg->sg.curr = msg->sg.start; msg->sg.copybreak = 0; } else if (sk_msg_iter_dist(msg->sg.start, msg->sg.curr) >= @@ -334,11 +404,19 @@ int sk_msg_zerocopy_from_iter(struct sock *sk, struct iov_iter *from, goto out; } - bytes -= copied; - msg->sg.size += copied; - while (copied) { use = min_t(int, copied, PAGE_SIZE - offset); + ret = sk_msg_subpages_add(msg, pages[i], offset, use); + if (ret) { + unsigned int nr = DIV_ROUND_UP(offset + copied, PAGE_SIZE); + + while (nr--) + put_page(pages[i++]); + iov_iter_revert(from, copied); + goto out; + } + bytes -= use; + msg->sg.size += use; sg_set_page(&msg->sg.data[msg->sg.end], pages[i], use, offset); sg_unmark_end(&msg->sg.data[msg->sg.end]); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 18b45950d1accc..d082be17d76fc6 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -44,6 +44,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, return -ENOMEM; lock_sock(sk); + sk_msg_subpages_share(tmp, msg); tmp->sg.start = msg->sg.start; i = msg->sg.start; do { @@ -76,6 +77,7 @@ static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock, if (!ret) { msg->sg.start = i; + sk_msg_subpages_drop_empty(msg); if (!sk_psock_queue_msg(psock, tmp)) atomic_sub(copied, &sk->sk_rmem_alloc); sk_psock_data_ready(sk, psock); @@ -147,6 +149,7 @@ static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, break; } + sk_msg_subpages_drop_empty(msg); return 0; } @@ -434,6 +437,8 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, } } memcpy(psock->cork, msg, sizeof(*msg)); + if (psock->cork != msg) + sk_msg_subpages_transfer(psock->cork, msg); return 0; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 33c42f65364a22..10da3de523b93c 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -32,6 +32,8 @@ #include #include #include +#include +#include #include #include #include @@ -144,6 +146,10 @@ static void destroy_record(struct tls_record_info *record) for (i = 0; i < record->num_frags; i++) __skb_frag_unref(&record->frags[i], false); + +#ifdef CONFIG_MM_SUBPAGE + mm_subpage_refs_put(record->subpage_refs); +#endif kfree(record); } @@ -349,6 +355,9 @@ static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx, record = kmalloc_obj(*record); if (!record) return -ENOMEM; +#ifdef CONFIG_MM_SUBPAGE + record->subpage_refs = NULL; +#endif frag = &record->frags[0]; skb_frag_fill_page_desc(frag, pfrag->page, pfrag->offset, @@ -513,6 +522,15 @@ static int tls_push_data(struct sock *sk, goto handle_error; } +#ifdef CONFIG_MM_SUBPAGE + rc = mm_subpage_refs_add(&record->subpage_refs, zc_pfrag.page, + off, copy, GFP_KERNEL); + if (rc) { + iov_iter_revert(iter, copy); + goto handle_error; + } +#endif + zc_pfrag.offset = off; zc_pfrag.size = copy; tls_append_frag(record, &zc_pfrag, copy); @@ -1118,6 +1136,10 @@ int tls_set_device_offload(struct sock *sk) goto release_netdev; } + +#ifdef CONFIG_MM_SUBPAGE + start_marker_record->subpage_refs = NULL; +#endif offload_ctx = alloc_offload_ctx_tx(ctx); if (!offload_ctx) { rc = -ENOMEM; diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index ca0f657121a3b9..f4f708533f79b9 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -35,8 +35,11 @@ * SOFTWARE. */ +#include #include #include +#include +#include #include #include #include @@ -65,9 +68,36 @@ struct tls_decrypt_ctx { u8 aad[TLS_MAX_AAD_SIZE]; u8 tail; bool free_sgout; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + unsigned int nr_fragments; + bool fragments_dirty; + struct llist_node cleanup; + void *allocation; +#endif struct scatterlist sg[]; }; +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) +/* Explicit test-only async provider selection; normal cipher selection stays intact. */ +static bool user4k_test_async; +module_param_named(user4k_test_async, user4k_test_async, bool, 0400); +MODULE_PARM_DESC(user4k_test_async, "KUnit: select cryptd for TLS receive testing"); +static atomic_long_t user4k_rx_deferred = ATOMIC_LONG_INIT(0); +static atomic_long_t user4k_rx_released = ATOMIC_LONG_INIT(0); + +static int user4k_read_count(char *buffer, const struct kernel_param *param) +{ + return scnprintf(buffer, PAGE_SIZE, "%ld\n", atomic_long_read(param->arg)); +} + +static const struct kernel_param_ops user4k_count_ops = { + .get = user4k_read_count, +}; +module_param_cb(user4k_rx_deferred, &user4k_count_ops, &user4k_rx_deferred, 0444); +module_param_cb(user4k_rx_released, &user4k_count_ops, &user4k_rx_released, 0444); +#endif + noinline void tls_err_abort(struct sock *sk, int err) { WARN_ON_ONCE(err >= 0); @@ -184,6 +214,35 @@ static int tls_padding_length(struct tls_prot_info *prot, struct sk_buff *skb, return sub; } +#ifdef CONFIG_MM_SUBPAGE +static void tls_release_user_fragments(struct tls_decrypt_ctx *dctx) +{ + release_user_fragments(dctx->fragments, dctx->nr_fragments, dctx->fragments_dirty); + kfree(dctx->fragments); + dctx->fragments = NULL; + dctx->nr_fragments = 0; +} +#endif + +/* Dirty pin release can take folio locks, so it belongs to the waiting task. */ +static void tls_finish_subpage_decrypts(struct tls_sw_context_rx *ctx) +{ +#ifdef CONFIG_MM_SUBPAGE + struct tls_decrypt_ctx *dctx, *next; + struct llist_node *head = llist_del_all(&ctx->subpage_cleanup); + + llist_for_each_entry_safe(dctx, next, head, cleanup) { + void *allocation = dctx->allocation; + + tls_release_user_fragments(dctx); +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_released); +#endif + kfree(allocation); + } +#endif +} + static void tls_decrypt_done(void *data, int err) { struct aead_request *aead_req = data; @@ -196,6 +255,7 @@ static void tls_decrypt_done(void *data, int err) unsigned int pages; struct sock *sk; int aead_size; + bool deferred = false; /* If requests get too backlogged crypto API returns -EBUSY and calls * ->complete(-EINPROGRESS) immediately followed by ->complete(0) @@ -224,17 +284,29 @@ static void tls_decrypt_done(void *data, int err) tls_err_abort(sk, err); } - /* Free the destination pages if skb was not decrypted inplace */ - if (dctx->free_sgout) { - /* Skip the first S/G entry as it points to AAD */ - for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { - if (!sg) - break; - put_page(sg_page(sg)); +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + dctx->allocation = aead_req; +#if IS_ENABLED(CONFIG_KUNIT) + atomic_long_inc(&user4k_rx_deferred); +#endif + llist_add(&dctx->cleanup, &ctx->subpage_cleanup); + deferred = true; + } +#endif + if (!deferred) { + /* Free the destination pages if skb was not decrypted inplace */ + if (dctx->free_sgout) { + /* Skip the first S/G entry as it points to AAD */ + for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) { + if (!sg) + break; + put_page(sg_page(sg)); + } } - } - kfree(aead_req); + kfree(aead_req); + } if (atomic_dec_and_test(&ctx->decrypt_pending)) complete(&ctx->async_wait.completion); @@ -246,6 +318,7 @@ static int tls_decrypt_async_wait(struct tls_sw_context_rx *ctx) crypto_wait_req(-EINPROGRESS, &ctx->async_wait); atomic_inc(&ctx->decrypt_pending); + tls_finish_subpage_decrypts(ctx); __skb_queue_purge(&ctx->async_hold); return ctx->async_wait.err; } @@ -670,6 +743,7 @@ static int tls_split_open_record(struct sock *sk, struct tls_rec *from, msg_opl->sg.size = bytes; msg_npl = &new->msg_plaintext; + sk_msg_subpages_share(msg_npl, msg_opl); msg_npl->apply_bytes = apply; msg_npl->sg.size = orig_size - bytes; @@ -733,6 +807,7 @@ static void tls_merge_open_record(struct sock *sk, struct tls_rec *to, sk_msg_free(sk, &to->msg_encrypted); sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted); + sk_msg_subpages_release(msg_npl); kfree(from); } @@ -935,12 +1010,14 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir; memcpy(&msg_redir, msg, sizeof(*msg)); + sk_msg_subpages_get(&msg_redir); if (msg->apply_bytes < send) msg->apply_bytes = 0; else msg->apply_bytes -= send; sk_msg_return_zero(sk, msg, send); msg->sg.size -= send; + sk_msg_subpages_drop_empty(msg); release_sock(sk); err = tcp_bpf_sendmsg_redir(sk_redir, redir_ingress, &msg_redir, send, flags); @@ -956,6 +1033,7 @@ static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk, *copied -= sk_msg_free_nocharge(sk, &msg_redir); msg->sg.size = 0; } + sk_msg_subpages_release(&msg_redir); if (msg->sg.size == 0) tls_free_open_rec(sk); break; @@ -1036,7 +1114,14 @@ static int tls_sw_sendmsg_splice(struct sock *sk, struct msghdr *msg, return -EIO; } - sk_msg_page_add(msg_pl, page, part, off); + { + int ret = sk_msg_page_add(msg_pl, page, part, off); + + if (ret) { + iov_iter_revert(&msg->msg_iter, part); + return ret; + } + } msg_pl->sg.copybreak = 0; msg_pl->sg.curr = msg_pl->sg.end; sk_mem_charge(sk, part); @@ -1454,10 +1539,96 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, return 1; } +#ifdef CONFIG_MM_SUBPAGE +static bool tls_subpage_iter(const struct iov_iter *iter) +{ + return user_backed_iter(iter) && current->mm && mm_page_size(current->mm) < PAGE_SIZE; +} +#endif + +static int tls_iter_sg_capacity(struct iov_iter *iter, size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(iter)) { + struct iov_iter walk = *iter; + unsigned int count = 0; + + length = min(length, iov_iter_count(iter)); + while (length) { + size_t bytes, offset; + + iov_iter_advance(&walk, 0); + bytes = min(length, iter_iov_len(&walk)); + if (WARN_ON_ONCE(!bytes)) + return INT_MAX; + offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); + count += DIV_ROUND_UP(offset + bytes, MM_SUBPAGE_SIZE); + iov_iter_advance(&walk, bytes); + length -= bytes; + } + return count; + } +#endif + return iov_iter_npages_cap(iter, INT_MAX, length); +} + +#ifdef CONFIG_MM_SUBPAGE +static int tls_setup_subpage_iter(struct iov_iter *from, int length, int *pages_used, + struct scatterlist *to, int capacity, + struct tls_decrypt_ctx *dctx) +{ + unsigned int used = 0, total = 0; + int ret = 0; + + if (!length) + return 0; + if (WARN_ON_ONCE(*pages_used || capacity <= 0)) + return -EINVAL; + dctx->fragments = kmalloc_array(capacity, sizeof(*dctx->fragments), GFP_KERNEL); + if (!dctx->fragments) + return -ENOMEM; + while (length) { + long nr; + unsigned int i, bytes = 0; + + iov_iter_advance(from, 0); + nr = pin_user_fragments_remote(current->mm, + (unsigned long)iter_iov_addr(from), + min_t(size_t, length, iter_iov_len(from)), FOLL_WRITE, + dctx->fragments + used, capacity - used); + if (nr <= 0) { + ret = nr ?: -EFAULT; + goto out; + } + for (i = 0; i < nr; i++) { + struct user_page_fragment *fragment = &dctx->fragments[used]; + + sg_set_page(&to[used], + folio_page(fragment->folio, fragment->offset >> PAGE_SHIFT), + fragment->length, offset_in_page(fragment->offset)); + sg_unmark_end(&to[used]); + bytes += fragment->length; + used++; + } + dctx->nr_fragments = used; + iov_iter_advance(from, bytes); + length -= bytes; + total += bytes; + } + if (used) + sg_mark_end(&to[used - 1]); +out: + if (ret) + iov_iter_revert(from, total); + *pages_used = used; + return ret; +} +#endif + static int tls_setup_from_iter(struct iov_iter *from, int length, int *pages_used, struct scatterlist *to, - int to_max_pages) + int to_max_pages, struct tls_decrypt_ctx *dctx) { int rc = 0, i = 0, num_elem = *pages_used, maxpages; struct page *pages[MAX_SKB_FRAGS]; @@ -1465,6 +1636,10 @@ static int tls_setup_from_iter(struct iov_iter *from, ssize_t copied, use; size_t offset; +#ifdef CONFIG_MM_SUBPAGE + if (tls_subpage_iter(from)) + return tls_setup_subpage_iter(from, length, pages_used, to, to_max_pages, dctx); +#endif while (length > 0) { i = 0; maxpages = to_max_pages - num_elem; @@ -1582,7 +1757,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, if (out_iov) n_sgout = 1 + tail_pages + - iov_iter_npages_cap(out_iov, INT_MAX, data_len); + tls_iter_sg_capacity(out_iov, data_len); else n_sgout = sg_nents(out_sg); } else { @@ -1615,6 +1790,11 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, aead_req = (struct aead_request *)mem; dctx = (struct tls_decrypt_ctx *)(mem + aead_size); dctx->sk = sk; +#ifdef CONFIG_MM_SUBPAGE + dctx->fragments = NULL; + dctx->nr_fragments = 0; + dctx->fragments_dirty = false; +#endif sgin = &dctx->sg[0]; sgout = &dctx->sg[n_sgin]; @@ -1672,7 +1852,7 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, sg_set_buf(&sgout[0], dctx->aad, prot->aad_size); err = tls_setup_from_iter(out_iov, data_len, &pages, &sgout[1], - (n_sgout - 1 - tail_pages)); + (n_sgout - 1 - tail_pages), dctx); if (err < 0) goto exit_free_pages; @@ -1687,6 +1867,10 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, } dctx->free_sgout = !!pages; +#ifdef CONFIG_MM_SUBPAGE + /* An error from crypto may still leave writes in the destination. */ + dctx->fragments_dirty = true; +#endif /* Prepare and submit AEAD request */ err = tls_do_decryption(sk, sgin, sgout, dctx->iv, data_len + prot->tail_size, aead_req, darg); @@ -1715,6 +1899,12 @@ static int tls_decrypt_sg(struct sock *sk, struct iov_iter *out_iov, darg->tail = dctx->tail; exit_free_pages: +#ifdef CONFIG_MM_SUBPAGE + if (dctx->fragments) { + tls_release_user_fragments(dctx); + pages = 0; + } +#endif /* Release the pages in case iov was mapped to pages */ for (; pages > 0; pages--) put_page(sg_page(&sgout[pages])); @@ -2795,6 +2985,9 @@ static struct tls_sw_context_rx *init_ctx_rx(struct tls_context *ctx) crypto_init_wait(&sw_ctx_rx->async_wait); atomic_set(&sw_ctx_rx->decrypt_pending, 1); +#ifdef CONFIG_MM_SUBPAGE + init_llist_head(&sw_ctx_rx->subpage_cleanup); +#endif init_waitqueue_head(&sw_ctx_rx->wq); skb_queue_head_init(&sw_ctx_rx->rx_list); skb_queue_head_init(&sw_ctx_rx->async_hold); @@ -2903,7 +3096,24 @@ int tls_set_sw_offload(struct sock *sk, int tx, rec_seq = crypto_info_rec_seq(src_crypto_info, cipher_desc); if (!*aead) { - *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); +#if IS_ENABLED(CONFIG_KUNIT) && IS_ENABLED(CONFIG_MM_SUBPAGE) + if (!tx && user4k_test_async) { + struct crypto_aead *sync; + char name[CRYPTO_MAX_ALG_NAME]; + + /* cryptd instances are named for the concrete child driver. */ + sync = crypto_alloc_aead(cipher_desc->cipher_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(sync)) { + *aead = sync; + } else { + snprintf(name, sizeof(name), "cryptd(%s)", + crypto_tfm_alg_driver_name(crypto_aead_tfm(sync))); + *aead = crypto_alloc_aead(name, CRYPTO_ALG_ASYNC, CRYPTO_ALG_ASYNC); + crypto_free_aead(sync); + } + } else +#endif + *aead = crypto_alloc_aead(cipher_desc->cipher_name, 0, 0); if (IS_ERR(*aead)) { rc = PTR_ERR(*aead); *aead = NULL; From 887b5ec08a19f2e92f76fd7ffdeb0949f2be1b2a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 021/631] iommu: carry exact user fragments through device mappings Convert IOMMUFD, VFIO and GEM/DMA-BUF paths to preserve the backing offset, mapping span and ownership of smaller user pages. Signed-off-by: Diljit Singh (cherry picked from commit 25dc8bcddb0298f46cbe5d335f6b56388e43e198) --- drivers/dma-buf/dma-buf.c | 35 +- drivers/gpu/drm/drm_gem.c | 7 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 25 +- drivers/iommu/iommufd/Makefile | 2 + drivers/iommu/iommufd/device.c | 102 +++- drivers/iommu/iommufd/fragments.c | 421 +++++++++++++++ drivers/iommu/iommufd/io_pagetable.c | 15 +- drivers/iommu/iommufd/io_pagetable.h | 77 ++- drivers/iommu/iommufd/iommufd_private.h | 4 + drivers/iommu/iommufd/iova_bitmap.c | 131 ++++- drivers/iommu/iommufd/pages.c | 99 +++- drivers/iommu/iommufd/selftest.c | 61 ++- drivers/iommu/iommufd/viommu.c | 56 +- drivers/vfio/vfio_iommu_type1.c | 534 +++++++++++++------ drivers/vfio/vfio_iommu_type1_fragments.c | 609 ++++++++++++++++++++++ 15 files changed, 1913 insertions(+), 265 deletions(-) create mode 100644 drivers/iommu/iommufd/fragments.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments.c diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 5c5dc8672fb814..76082bc71f7eba 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -225,9 +225,23 @@ static struct file_system_type dma_buf_fs_type = { .kill_sb = kill_anon_super, }; +static int dma_buf_mmap_range_check(struct dma_buf *dmabuf, + struct vm_area_struct *vma, unsigned long pgoff, unsigned int suboffset) +{ + unsigned long offset; + + if (check_mul_overflow(pgoff, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)suboffset, &offset)) + return -EOVERFLOW; + if (offset > dmabuf->size || vma->vm_end - vma->vm_start > dmabuf->size - offset) + return -EINVAL; + return 0; +} + static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; + int ret; if (!is_dma_buf_file(file)) return -EINVAL; @@ -239,9 +253,9 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) return -EINVAL; /* check for overflowing the buffer's size */ - if (vma->vm_pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, vma->vm_pgoff, vma_subpage_offset(vma)); + if (ret) + return ret; DMA_BUF_TRACE(trace_dma_buf_mmap_internal, dmabuf); @@ -1602,6 +1616,8 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF"); int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, unsigned long pgoff) { + int ret; + if (WARN_ON(!dmabuf || !vma)) return -EINVAL; @@ -1609,18 +1625,13 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, if (!dmabuf->ops->mmap) return -EINVAL; - /* check for offset overflow */ - if (pgoff + vma_pages(vma) < pgoff) - return -EOVERFLOW; - - /* check for overflowing the buffer's size */ - if (pgoff + vma_pages(vma) > - dmabuf->size >> PAGE_SHIFT) - return -EINVAL; + ret = dma_buf_mmap_range_check(dmabuf, vma, pgoff, 0); + if (ret) + return ret; /* readjust the vma */ vma_set_file(vma, dmabuf->file); - vma->vm_pgoff = pgoff; + vma_set_page_offset(vma, (struct vm_page_offset) { .index = pgoff }); DMA_BUF_TRACE(trace_dma_buf_mmap, dmabuf); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 3b2448a3a9de85..2176ab547bfb6a 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1343,7 +1343,7 @@ unsigned long drm_gem_get_unmapped_area(struct file *filp, unsigned long uaddr, struct drm_gem_object *obj; unsigned long ret; - obj = drm_gem_object_lookup_at_offset(filp, pgoff, len >> PAGE_SHIFT); + obj = drm_gem_object_lookup_at_offset(filp, pgoff, DIV_ROUND_UP(len, PAGE_SIZE)); if (IS_ERR(obj)) obj = NULL; @@ -1379,8 +1379,11 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) struct drm_gem_object *obj; int ret; + /* GEM fake offsets identify an exact object start, not an interior byte. */ + if (vma_subpage_offset(vma)) + return -EINVAL; obj = drm_gem_object_lookup_at_offset(filp, vma->vm_pgoff, - vma_pages(vma)); + DIV_ROUND_UP(vma->vm_end - vma->vm_start, PAGE_SIZE)); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index fc05581d47772b..3b94e2ccdbd91e 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -723,6 +723,14 @@ EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) { return pgoff - drm_vma_node_start(&obj->vma_node); +/* GEM and PRIME add the object's fake offset; VMA splits retain it. + * Direct helper callers may instead supply an object-relative offset. + */ +static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) +{ + pgoff_t base = drm_vma_node_start(&obj->vma_node); + + return pgoff >= base ? pgoff - base : pgoff; } static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) @@ -763,7 +771,10 @@ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, vm_fault_t ret = VM_FAULT_FALLBACK; if (!order) { - ret = vmf_insert_pfn_mkwrite(vmf->vma, vmf->address, pfn, write); + struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); + + ret = vmf_insert_pfn_prot_mkwrite_offset(vmf->vma, vmf->address, + pfn, pos.offset, vmf->vma->vm_page_prot, write); #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP } else if (order == PMD_ORDER) { unsigned long paddr = pfn << PAGE_SHIFT; @@ -800,7 +811,7 @@ static vm_fault_t drm_gem_shmem_any_fault(struct vm_fault *vmf, unsigned int ord struct folio *folio; unsigned long pfn; - if (order && order != PMD_ORDER) + if (order && (order != PMD_ORDER || mm_page_size(vma->vm_mm) != PAGE_SIZE)) return VM_FAULT_FALLBACK; dma_resv_lock(obj->resv, NULL); @@ -960,6 +971,16 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (shmem->reclaimable_cpu_mappings && (!vma->vm_file || vma->vm_file->f_mapping != obj->dev->anon_inode->i_mapping)) return -EINVAL; + /* Validate bytes, including a partial native-page offset from PRIME. */ + { + pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); + unsigned long offset, length = vma->vm_end - vma->vm_start; + + if (check_mul_overflow(index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)vma_subpage_offset(vma), &offset) || + offset > obj->size || length > obj->size - offset) + return -EINVAL; + } dma_resv_lock(shmem->base.resv, NULL); if (shmem->reclaimable_cpu_mappings) { diff --git a/drivers/iommu/iommufd/Makefile b/drivers/iommu/iommufd/Makefile index 71d692c9a8f49d..a868155e208d53 100644 --- a/drivers/iommu/iommufd/Makefile +++ b/drivers/iommu/iommufd/Makefile @@ -10,6 +10,8 @@ iommufd-y := \ vfio_compat.o \ viommu.o +iommufd-$(CONFIG_MM_SUBPAGE) += fragments.o + iommufd-$(CONFIG_IOMMUFD_TEST) += selftest.o obj-$(CONFIG_IOMMUFD) += iommufd.o diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 2895e5370910fe..3ccb352bb3e506 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -1171,6 +1171,18 @@ struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx) return access; } +/* Physical byte-span consumers do not impose native-page IOVA alignment. */ +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx) +{ + struct iommufd_access *access = __iommufd_access_create(ictx); + + if (IS_ERR(access)) + return access; + access->iova_alignment = 1; + iommufd_object_finalize(ictx, &access->obj); + return access; +} + /** * iommufd_access_create - Create an iommufd_access * @ictx: iommufd file descriptor @@ -1404,6 +1416,12 @@ static bool check_area_prot(struct iopt_area *area, unsigned int flags) * ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However * smaller alignments have corner cases where this API can fail on otherwise * aligned iova. + * + * Alternative user granules are accepted only if each native output page has + * an aligned physical base and consecutive backing bytes within the requested + * range. The driver must access only @length bytes: a partial final page does + * not grant access to its remaining bytes. Fragment ownership stays in the IOAS + * until unpin; incompatible physical layouts return -EINVAL. */ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, @@ -1456,7 +1474,7 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, flags, internal); if (rc) goto err_remove; - out_pages += last_index - index + 1; + out_pages += (last - iter.cur_iova) / PAGE_SIZE + 1; } if (!iopt_area_contig_done(&iter)) { rc = -ENOENT; @@ -1485,6 +1503,88 @@ int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, } EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, "IOMMUFD"); +/** + * iommufd_access_pin_phys - Pin an internal contiguous physical byte span + * @access: Internal access created for byte-span pinning + * @iova: First IOVA byte + * @length: Exact byte length + * @out_phys: Physical address of the first byte, set only on success + * + * Retains native or fragment pin ownership and prevents IOAS unmap until the + * matching iommufd_access_unpin_pages(). No surrounding IOVA bytes are required. + * The caller must stop hardware access before unpinning or destroying access. + */ +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_area_contig_iter iter; + struct io_pagetable *iopt; + struct iopt_area *area; + unsigned long last_iova; + phys_addr_t base = 0; + int rc; + + if (!iommufd_access_is_internal(access) || !out_phys || !length) + return -EINVAL; + if (check_add_overflow(iova, length - 1, &last_iova)) + return -EOVERFLOW; + + mutex_lock(&access->ioas_lock); + if (!access->ioas) { + rc = -ENOENT; + goto out_unlock; + } + iopt = &access->ioas->iopt; + down_read(&iopt->iova_rwsem); + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) { + unsigned long last = min(last_iova, iopt_area_last_iova(area)); + phys_addr_t phys; + + if (area->prevent_access || !(area->iommu_prot & IOMMU_READ)) { + rc = -EPERM; + goto out_remove; + } + rc = iopt_area_add_phys_access(area, + iopt_area_start_byte(area, iter.cur_iova), + last - iter.cur_iova + 1, &phys); + if (rc) + goto out_remove; + if (iter.cur_iova == iova) { + base = phys; + } else if (phys != base + (iter.cur_iova - iova)) { + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, last), true); + rc = -EFAULT; + goto out_remove; + } + } + if (!iopt_area_contig_done(&iter)) { + rc = -ENOENT; + goto out_remove; + } + *out_phys = base; + up_read(&iopt->iova_rwsem); + rc = 0; + goto out_unlock; + +out_remove: + /* iter.cur_iova is the first byte whose access was not retained. */ + if (iter.cur_iova > iova) { + unsigned long end = iter.cur_iova - 1; + + iopt_for_each_contig_area(&iter, area, iopt, iova, end) + iopt_area_remove_access(area, + iopt_area_iova_to_index(area, iter.cur_iova), + iopt_area_iova_to_index(area, min(end, iopt_area_last_iova(area))), + true); + } + up_read(&iopt->iova_rwsem); +out_unlock: + mutex_unlock(&access->ioas_lock); + return rc; +} + /** * iommufd_access_rw - Read or write data under the iova * @access: IOAS access to act on diff --git a/drivers/iommu/iommufd/fragments.c b/drivers/iommu/iommufd/fragments.c new file mode 100644 index 00000000000000..5229605852133d --- /dev/null +++ b/drivers/iommu/iommufd/fragments.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Alternative user granules need an ownership record in addition to a physical + * address. A device translation cannot retain a subpage allocation identity. + * Keep those records until the final domain interval has gone away. The native + * PFN reader remains in pages.c; file and dma-buf suppliers retain its geometry. + */ +#include +#include +#include +#include + +#include "iommufd_private.h" +#include "io_pagetable.h" + +static phys_addr_t fragment_phys(const struct user_page_fragment *fragment) +{ + return PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset; +} + +/* Charge each native physical page once, even when several slots pin it. */ +static int fragment_store(struct iopt_pages *pages, unsigned long index, + struct user_page_fragment *fragment) +{ + unsigned long pfn = PHYS_PFN(fragment_phys(fragment)); + unsigned long count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + struct user_page_fragment *record; + int rc; + + record = kmemdup(fragment, sizeof(*record), GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + rc = xa_reserve(&pages->fragments, index, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_free; + rc = xa_err(xa_store(&pages->fragment_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (rc) + goto out_release; + /* xa_reserve() made this infallible. Ownership transfers here. */ + xa_store(&pages->fragments, index, record, GFP_NOWAIT); + if (!count) + pages->npinned++; + *fragment = (struct user_page_fragment) {}; + return 0; + +out_release: + xa_release(&pages->fragments, index); +out_free: + kfree(record); + return rc; +} + +/* No allocation on teardown, and callers have already unmapped all devices. */ +static void fragments_release_unused(struct iopt_pages *pages, + unsigned long first, unsigned long last) +{ + struct user_page_fragment *record; + unsigned long index = first; + + lockdep_assert_held(&pages->mutex); + xa_for_each_range(&pages->fragments, index, record, first, last) { + unsigned long pfn, count; + + if (interval_tree_iter_first(&pages->domains_itree, index, index) || + interval_tree_iter_first(&pages->access_itree, index, index)) + continue; + pfn = PHYS_PFN(fragment_phys(record)); + count = xa_to_value(xa_load(&pages->fragment_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&pages->fragment_pages, pfn); + pages->npinned--; + } else { + xa_store(&pages->fragment_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + xa_erase(&pages->fragments, index); + release_user_fragments(record, 1, pages->writable); + kfree(record); + cond_resched(); + } + if (pages->npinned < pages->last_npinned) + iopt_pages_update_pinned(pages, pages->last_npinned - pages->npinned, + false, NULL); +} + +/* Reuse existing records even if the original VA was unmapped or replaced. */ +static int fragments_acquire(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + unsigned long size = 1UL << pages->page_shift, index = first; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool mm_live = false; + int rc = 0; + + lockdep_assert_held(&pages->mutex); + while (index <= last) { + unsigned long next = index, end, i; + long count; + + if (xa_load(&pages->fragments, index)) { + index++; + continue; + } + if (!mm_live) { + if (!mmget_not_zero(pages->source_mm)) { + rc = -EFAULT; + goto out; + } + mm_live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + rc = -ENOMEM; + goto out; + } + } + /* Stop before any cached pin; never replace its ownership record. */ + end = xa_find(&pages->fragments, &next, last, XA_PRESENT) ? next : last + 1; + end = min(end, index + capacity); + count = pin_user_fragments_remote(pages->source_mm, + (unsigned long)pages->uptr + (index << pages->page_shift), + (end - index) << pages->page_shift, + FOLL_LONGTERM | (pages->writable ? FOLL_WRITE : 0), batch, capacity); + if (count <= 0) { + rc = count ?: -EFAULT; + goto out; + } + for (i = 0; i < count; i++) { + /* Each aligned logical index must have exactly one full pin. */ + if (WARN_ON(batch[i].length != size || + !IS_ALIGNED(fragment_phys(&batch[i]), size))) + rc = -EFAULT; + else + rc = fragment_store(pages, index, &batch[i]); + if (rc) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + index++; + } + cond_resched(); + } + if (pages->npinned != pages->last_npinned) { + if (iommufd_should_fail()) + rc = -ENOMEM; + else + rc = iopt_pages_update_pinned(pages, + pages->npinned - pages->last_npinned, true, NULL); + } +out: + kfree(batch); + if (mm_live) + mmput(pages->source_mm); + if (rc) + fragments_release_unused(pages, first, last); + return rc; +} + +static void fragments_unmap(struct iommu_domain *domain, unsigned long iova, + unsigned long length) +{ + if (length) + WARN_ON(iommu_unmap(domain, iova, length) != length); +} + +/* All pins are present. Merge physically consecutive fragments before mapping. */ +static int fragments_map(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long index = iopt_area_index(area), last = iopt_area_last_index(area); + unsigned long remaining = iopt_area_length(area), done = 0; + unsigned long offset = area->page_offset; + unsigned long min_size = 1UL << __ffs(domain->pgsize_bitmap); + int rc; + + while (remaining) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + phys_addr_t phys = fragment_phys(record) + offset; + unsigned long bytes = min(size - offset, remaining); + + index++; + /* Legacy splitting forbids mappings larger than a native page. */ + while (index <= last && bytes < remaining && + (!area->iopt->disable_large_pages || bytes < PAGE_SIZE)) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != phys + bytes) + break; + bytes += min(size, remaining - bytes); + index++; + } + /* Never widen a device mapping to include an unowned neighbour. */ + if (!IS_ALIGNED(phys | (iopt_area_iova(area) + done) | bytes, min_size)) { + rc = -EINVAL; + goto out_unmap; + } + rc = iommu_map(domain, iopt_area_iova(area) + done, phys, bytes, + area->iommu_prot, GFP_KERNEL_ACCOUNT); + if (rc) + goto out_unmap; + done += bytes; + remaining -= bytes; + offset = 0; + cond_resched(); + } + return 0; + +out_unmap: + fragments_unmap(domain, iopt_area_iova(area), done); + return rc; +} + +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + int rc; + + lockdep_assert_held(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + rc = fragments_map(area, pages, domain); + if (rc) + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages) +{ + struct iommu_domain *domain; + unsigned long index, failed; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, iopt_area_index(area), iopt_area_last_index(area)); + if (rc) + return rc; + xa_for_each(&area->iopt->domains, index, domain) { + rc = fragments_map(area, pages, domain); + if (rc) + goto out_unmap; + } + area->storage_domain = xa_load(&area->iopt->domains, 0); + interval_tree_insert(&area->pages_node, &pages->domains_itree); + return 0; + +out_unmap: + failed = index; + xa_for_each(&area->iopt->domains, index, domain) { + if (index == failed) + break; + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + } + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); + return rc; +} + +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain) +{ + lockdep_assert_held(&pages->mutex); + fragments_unmap(domain, iopt_area_iova(area), iopt_area_length(area)); + fragments_release_unused(pages, iopt_area_index(area), iopt_area_last_index(area)); +} + +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags) +{ + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long index, offset = start_byte & (size - 1); + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long bytes = min(size - offset, length); + void *addr = kmap_local_folio(record->folio, record->offset + offset); + + if (flags & IOMMUFD_ACCESS_RW_WRITE) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + data += bytes; + length -= bytes; + offset = 0; + } + fragments_release_unused(pages, first, last); + return 0; +} + +/* The retained typed records, not the output page array, own the pins. */ +static int fragments_register_access(struct iopt_area *area, unsigned long first, + unsigned long last, bool lock_area) +{ + struct iopt_pages *pages = area->pages; + struct iopt_pages_access *access; + struct interval_tree_node *node; + + lockdep_assert_held(&pages->mutex); + for (node = interval_tree_iter_first(&pages->access_itree, first, last); + node; node = interval_tree_iter_next(node, first, last)) { + if (node->start == first && node->last == last) { + access = container_of(node, struct iopt_pages_access, node); + access->users++; + goto out_count; + } + } + access = kzalloc_obj(*access, GFP_KERNEL_ACCOUNT); + if (!access) + return -ENOMEM; + access->node.start = first; + access->node.last = last; + access->users = 1; + interval_tree_insert(&access->node, &pages->access_itree); +out_count: + area->num_accesses++; + if (lock_area) + area->num_locks++; + return 0; +} + +/* Register a physical byte span without discarding fragment ownership. */ +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long size = 1UL << pages->page_shift; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + struct user_page_fragment *record; + unsigned long index; + phys_addr_t base; + int rc; + + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + record = xa_load(&pages->fragments, first); + base = fragment_phys(record); + for (index = first + 1; index <= last; index++) { + record = xa_load(&pages->fragments, index); + if (fragment_phys(record) != + base + ((index - first) << pages->page_shift)) { + rc = -EFAULT; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, true); + if (rc) + goto out_release; + *out_phys = base + (start_byte & (size - 1)); + return 0; + +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +/* + * A native page pointer has no fragment offset. Require an aligned physical + * base and consecutive fragments within each output page; different output + * pages may be scattered. Only the requested logical interval is retained. + */ +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area) +{ + struct iopt_pages *pages = area->pages; + unsigned long per_page = 1UL << (PAGE_SHIFT - pages->page_shift); + unsigned long index; + phys_addr_t base = 0; + int rc; + + if (first & (per_page - 1)) + return -EINVAL; + guard(mutex)(&pages->mutex); + rc = fragments_acquire(pages, first, last); + if (rc) + return rc; + for (index = first; index <= last; index++) { + struct user_page_fragment *record = xa_load(&pages->fragments, index); + unsigned long offset = (index - first) & (per_page - 1); + phys_addr_t phys = fragment_phys(record); + + if (!offset) { + base = phys; + if (!IS_ALIGNED(base, PAGE_SIZE)) { + rc = -EINVAL; + goto out_release; + } + *out_pages++ = pfn_to_page(PHYS_PFN(base)); + } else if (phys != base + (offset << pages->page_shift)) { + rc = -EINVAL; + goto out_release; + } + cond_resched(); + } + rc = fragments_register_access(area, first, last, lock_area); + if (!rc) + return 0; +out_release: + fragments_release_unused(pages, first, last); + return rc; +} + +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last) +{ + fragments_release_unused(pages, first, last); +} diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c index 24d4917105d942..bf8cabee53fb6c 100644 --- a/drivers/iommu/iommufd/io_pagetable.c +++ b/drivers/iommu/iommufd/io_pagetable.c @@ -217,7 +217,8 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, return -EPERM; area->iommu_prot = iommu_prot; - area->page_offset = start_byte % PAGE_SIZE; + area->page_shift = pages->page_shift; + area->page_offset = start_byte & ((1UL << area->page_shift) - 1); if (area->page_offset & (iopt->iova_alignment - 1)) return -EINVAL; @@ -225,10 +226,10 @@ static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area, if (check_add_overflow(iova, length - 1, &area->node.last)) return -EOVERFLOW; - area->pages_node.start = start_byte / PAGE_SIZE; + area->pages_node.start = start_byte >> area->page_shift; if (check_add_overflow(start_byte, length - 1, &area->pages_node.last)) return -EOVERFLOW; - area->pages_node.last = area->pages_node.last / PAGE_SIZE; + area->pages_node.last >>= area->page_shift; if (WARN_ON(area->pages_node.last >= pages->npages)) return -EOVERFLOW; @@ -572,7 +573,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, struct iommu_dirty_bitmap dirty; struct iova_bitmap_fn_arg arg; struct iova_bitmap *iter; - int ret = 0; + int ret; if (!ops || !ops->read_and_clear_dirty) return -EOPNOTSUPP; @@ -581,7 +582,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, bitmap->page_size, u64_to_user_ptr(bitmap->data)); if (IS_ERR(iter)) - return -ENOMEM; + return PTR_ERR(iter); iommu_dirty_bitmap_init(&dirty, iter, &gather); @@ -589,7 +590,7 @@ iommu_read_and_clear_dirty(struct iommu_domain *domain, arg.iopt = iopt; arg.domain = domain; arg.dirty = &dirty; - iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); + ret = iova_bitmap_for_each(iter, &arg, __iommu_read_and_clear_dirty); if (!(flags & IOMMU_DIRTY_NO_CLEAR)) iommu_iotlb_sync(domain, &gather); @@ -615,7 +616,7 @@ int iommufd_check_iova_range(struct io_pagetable *iopt, ((last_iova + 1) & (iommu_pgsize - 1))) return -EINVAL; - if (!bitmap->page_size) + if (!is_power_of_2(bitmap->page_size)) return -EINVAL; if ((bitmap->iova & (bitmap->page_size - 1)) || diff --git a/drivers/iommu/iommufd/io_pagetable.h b/drivers/iommu/iommufd/io_pagetable.h index 27e3e311d395b2..d15e51fee0c107 100644 --- a/drivers/iommu/iommufd/io_pagetable.h +++ b/drivers/iommu/iommufd/io_pagetable.h @@ -5,6 +5,7 @@ #ifndef __IO_PAGETABLE_H #define __IO_PAGETABLE_H +#include #include #include #include @@ -45,6 +46,7 @@ struct iopt_area { struct iommu_domain *storage_domain; /* How many bytes into the first page the area starts */ unsigned int page_offset; + u8 page_shift; /* IOMMU_READ, IOMMU_WRITE, etc */ int iommu_prot; bool prevent_access : 1; @@ -107,8 +109,8 @@ static inline size_t iopt_area_length(struct iopt_area *area) /* * Number of bytes from the start of the iopt_pages that the iova begins. - * iopt_area_start_byte() / PAGE_SIZE encodes the starting page index - * iopt_area_start_byte() % PAGE_SIZE encodes the offset within that page + * iopt_area_start_byte() >> area->page_shift encodes the starting page index + * iopt_area_start_byte() % (1UL << area->page_shift) encodes the offset within that page */ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, unsigned long iova) @@ -117,13 +119,13 @@ static inline unsigned long iopt_area_start_byte(struct iopt_area *area, WARN_ON(iova < iopt_area_iova(area) || iova > iopt_area_last_iova(area)); return (iova - iopt_area_iova(area)) + area->page_offset + - iopt_area_index(area) * PAGE_SIZE; + (iopt_area_index(area) << area->page_shift); } static inline unsigned long iopt_area_iova_to_index(struct iopt_area *area, unsigned long iova) { - return iopt_area_start_byte(area, iova) / PAGE_SIZE; + return iopt_area_start_byte(area, iova) >> area->page_shift; } #define __make_iopt_iter(name) \ @@ -214,7 +216,7 @@ struct iopt_pages_dmabuf { * io_pagetable's, through their iopt_area's, can share a single iopt_pages * which avoids multi-pinning and double accounting of page consumption. * - * indexes in this structure are measured in PAGE_SIZE units, are 0 based from + * Indexes use (1UL << page_shift) units and are 0 based from * the start of the uptr and extend to npages. pages are pinned dynamically * according to the intervals in the access_itree and domains_itree, npinned * records the current number of pages pinned. @@ -222,6 +224,8 @@ struct iopt_pages_dmabuf { struct iopt_pages { struct kref kref; struct mutex mutex; + /* Logical indices use page_shift; pin accounting remains native pages. */ + u8 page_shift; size_t npages; size_t npinned; size_t last_npinned; @@ -242,12 +246,73 @@ struct iopt_pages { u8 account_mode; struct xarray pinned_pfns; +#ifdef CONFIG_MM_SUBPAGE + /* Exact fragment pins, indexed independently of device translations. */ + struct xarray fragments; + /* Native physical PFN -> fragment count, for pin accounting. */ + struct xarray fragment_pages; +#endif /* Of iopt_pages_access::node */ struct rb_root_cached access_itree; /* Of iopt_area::pages_node */ struct rb_root_cached domains_itree; }; +static inline bool iopt_pages_use_fragments(struct iopt_pages *pages) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) && pages->type == IOPT_ADDRESS_USER && + mm_page_size(pages->source_mm) != PAGE_SIZE; +} + +#ifdef CONFIG_MM_SUBPAGE +int iopt_fragments_fill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_fill_domains(struct iopt_area *area, struct iopt_pages *pages); +void iopt_fragments_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, + struct iommu_domain *domain); +int iopt_fragments_add_access(struct iopt_area *area, unsigned long first, + unsigned long last, struct page **out_pages, + bool lock_area); +int iopt_fragments_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); +void iopt_fragments_unfill_access(struct iopt_pages *pages, unsigned long first, + unsigned long last); +int iopt_fragments_rw(struct iopt_pages *pages, unsigned long start_byte, + void *data, unsigned long length, unsigned int flags); +#else +static inline int iopt_fragments_fill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_fill_domains(struct iopt_area *area, + struct iopt_pages *pages) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_domain(struct iopt_area *area, + struct iopt_pages *pages, struct iommu_domain *domain) { } +static inline int iopt_fragments_add_access(struct iopt_area *area, + unsigned long first, unsigned long last, struct page **out_pages, + bool lock_area) +{ + return -EOPNOTSUPP; +} +static inline int iopt_fragments_add_phys_access(struct iopt_area *area, + unsigned long start_byte, unsigned long length, phys_addr_t *out_phys) +{ + return -EOPNOTSUPP; +} +static inline void iopt_fragments_unfill_access(struct iopt_pages *pages, + unsigned long first, unsigned long last) { } +static inline int iopt_fragments_rw(struct iopt_pages *pages, + unsigned long start_byte, void *data, unsigned long length, + unsigned int flags) +{ + return -EOPNOTSUPP; +} +#endif + static inline bool iopt_is_dmabuf(struct iopt_pages *pages) { if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) @@ -290,6 +355,8 @@ void iopt_pages_unfill_xarray(struct iopt_pages *pages, unsigned long start, int iopt_area_add_access(struct iopt_area *area, unsigned long start, unsigned long last, struct page **out_pages, unsigned int flags, bool lock_area); +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys); void iopt_area_remove_access(struct iopt_area *area, unsigned long start, unsigned long last, bool unlock_area); int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 43fbc5bed8de3f..d1a14e51df55e5 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -531,6 +531,10 @@ static inline bool iommufd_access_is_internal(struct iommufd_access *access) } struct iommufd_access *iommufd_access_create_internal(struct iommufd_ctx *ictx); +struct iommufd_access *iommufd_access_create_internal_phys(struct iommufd_ctx *ictx); +int iommufd_access_pin_phys(struct iommufd_access *access, unsigned long iova, + unsigned long length, phys_addr_t *out_phys); + static inline void iommufd_access_destroy_internal(struct iommufd_ctx *ictx, diff --git a/drivers/iommu/iommufd/iova_bitmap.c b/drivers/iommu/iommufd/iova_bitmap.c index dac3e657d498da..12e1dd035871a7 100644 --- a/drivers/iommu/iommufd/iova_bitmap.c +++ b/drivers/iommu/iommufd/iova_bitmap.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #define BITS_PER_PAGE (PAGE_SIZE * BITS_PER_BYTE) @@ -44,11 +45,16 @@ struct iova_bitmap_map { /* page offset of the first user page pinned */ unsigned long pgoff; - /* number of pages pinned */ + /* number of native pages or typed fragments pinned */ unsigned long npages; /* pinned pages representing the bitmap data */ struct page **pages; +#ifdef CONFIG_MM_SUBPAGE + /* Exact byte ranges, including offsets unrelated to the user VA. */ + struct user_page_fragment *fragments; + unsigned long pinned_bytes; +#endif }; /* @@ -63,7 +69,10 @@ struct iova_bitmap_map { * The bitmap object uses one base page to store all the pinned pages * pointers related to the bitmap. For sizeof(struct page*) == 8 it stores * 512 struct page pointers which, if the base page size is 4K, it means - * 2M of bitmap data is pinned at a time. If the iova_bitmap page size is + * 2M of bitmap data is pinned at a time. Alternative-granule processes use + * one native page of fragment descriptors instead, retaining each physical + * byte offset and bounding the window by the sum of fragment lengths. + * If the iova_bitmap page size is * also 4K then the range window to iterate is 64G. * * For example iterating on a total IOVA range of 4G..128G, it will walk @@ -116,6 +125,9 @@ struct iova_bitmap { /* length of the IOVA range for the whole bitmap */ size_t length; + + /* First lazy pin/write error during an iteration. */ + int error; }; /* @@ -192,6 +204,26 @@ static int iova_bitmap_get(struct iova_bitmap *bitmap) npages = min(npages + !!offset_in_page(addr), PAGE_SIZE / sizeof(struct page *)); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + unsigned long bytes = bitmap->mapped_total_index - bitmap->mapped_base_index; + unsigned long i; + + ret = pin_user_fragments_remote(current->mm, (unsigned long)addr, + bytes, FOLL_WRITE, mapped->fragments, + PAGE_SIZE / sizeof(*mapped->fragments)); + if (ret <= 0) + return -EFAULT; + mapped->npages = ret; + mapped->pinned_bytes = 0; + for (i = 0; i < mapped->npages; i++) + mapped->pinned_bytes += mapped->fragments[i].length; + mapped->pgoff = 0; + mapped->iova = iova_bitmap_mapped_iova(bitmap); + mapped->length = iova_bitmap_mapped_length(bitmap); + return 0; + } +#endif ret = pin_user_pages_fast((unsigned long)addr, npages, FOLL_WRITE, mapped->pages); if (ret <= 0) @@ -221,7 +253,13 @@ static void iova_bitmap_put(struct iova_bitmap *bitmap) struct iova_bitmap_map *mapped = &bitmap->mapped; if (mapped->npages) { - unpin_user_pages(mapped->pages, mapped->npages); +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + release_user_fragments(mapped->fragments, mapped->npages, true); + mapped->pinned_bytes = 0; + } else +#endif + unpin_user_pages_dirty_lock(mapped->pages, mapped->npages, true); mapped->npages = 0; } } @@ -259,6 +297,16 @@ struct iova_bitmap *iova_bitmap_alloc(unsigned long iova, size_t length, bitmap->iova = iova; bitmap->length = length; mapped->iova = iova; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + mapped->fragments = (void *)__get_free_page(GFP_KERNEL); + if (!mapped->fragments) { + rc = -ENOMEM; + goto err; + } + return bitmap; + } +#endif mapped->pages = (struct page **)__get_free_page(GFP_KERNEL); if (!mapped->pages) { rc = -ENOMEM; @@ -291,6 +339,10 @@ void iova_bitmap_free(struct iova_bitmap *bitmap) mapped->pages = NULL; } +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) + free_page((unsigned long)mapped->fragments); +#endif kfree(bitmap); } EXPORT_SYMBOL_NS_GPL(iova_bitmap_free, "IOMMUFD"); @@ -303,7 +355,12 @@ static unsigned long iova_bitmap_mapped_remaining(struct iova_bitmap *bitmap) { unsigned long remaining, bytes; - bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; +#ifdef CONFIG_MM_SUBPAGE + if (bitmap->mapped.fragments) + bytes = bitmap->mapped.pinned_bytes; + else +#endif + bytes = (bitmap->mapped.npages << PAGE_SHIFT) - bitmap->mapped.pgoff; remaining = bitmap->mapped_total_index - bitmap->mapped_base_index; remaining = min_t(unsigned long, remaining, @@ -356,16 +413,21 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, unsigned long iova) { unsigned long index; + int ret = -EINVAL; index = iova_bitmap_offset_to_index(bitmap, iova - bitmap->iova); if (index >= bitmap->mapped_total_index) - return -EINVAL; + goto out; bitmap->mapped_base_index = index; iova_bitmap_put(bitmap); /* Pin the next set of bitmap pages */ - return iova_bitmap_get(bitmap); + ret = iova_bitmap_get(bitmap); +out: + if (ret && !bitmap->error) + bitmap->error = ret; + return ret; } /** @@ -379,15 +441,58 @@ static int iova_bitmap_advance_to(struct iova_bitmap *bitmap, * mapped bitmap user pages into IOVA ranges to process. * * Return: 0 on success, and an error on failure either upon - * iteration or when the callback returns an error. + * iteration, a lazy bitmap write, or when the callback returns an error. */ int iova_bitmap_for_each(struct iova_bitmap *bitmap, void *opaque, iova_bitmap_fn_t fn) { - return fn(bitmap, bitmap->iova, bitmap->length, opaque); + int ret; + + bitmap->error = 0; + ret = fn(bitmap, bitmap->iova, bitmap->length, opaque); + return ret ?: bitmap->error; } EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); +#ifdef CONFIG_MM_SUBPAGE +static void iova_bitmap_set_fragments(struct iova_bitmap *bitmap, + unsigned long iova, size_t length) +{ + struct iova_bitmap_map *mapped = &bitmap->mapped; + unsigned long bit = (iova - bitmap->iova) >> mapped->pgshift; + unsigned long end = ((iova + length - 1 - bitmap->iova) >> mapped->pgshift) + 1; + + while (bit < end) { + unsigned long base, relative, i; + + iova = bitmap->iova + (bit << mapped->pgshift); + if (!iova_bitmap_mapped_range(mapped, iova, 1) && + iova_bitmap_advance_to(bitmap, iova)) + return; + base = bitmap->mapped_base_index * BITS_PER_BYTE; + relative = bit - base; + for (i = 0; i < mapped->npages && bit < end; i++) { + struct user_page_fragment *fragment = &mapped->fragments[i]; + unsigned long bits = fragment->length * BITS_PER_BYTE; + unsigned long count; + void *addr; + + if (relative >= bits) { + relative -= bits; + continue; + } + count = min(end - bit, bits - relative); + addr = kmap_local_folio(fragment->folio, fragment->offset & PAGE_MASK); + bitmap_set(addr, offset_in_page(fragment->offset) * BITS_PER_BYTE + + relative, count); + kunmap_local(addr); + bit += count; + relative = 0; + } + } +} +#endif + /** * iova_bitmap_set() - Records an IOVA range in bitmap * @bitmap: IOVA bitmap @@ -395,7 +500,8 @@ EXPORT_SYMBOL_NS_GPL(iova_bitmap_for_each, "IOMMUFD"); * @length: IOVA range length * * Set the bits corresponding to the range [iova .. iova+length-1] in - * the user bitmap. + * the user bitmap. A lazy pin failure is retained for the enclosing + * iova_bitmap_for_each() to report; successful prefix writes are not undone. * */ void iova_bitmap_set(struct iova_bitmap *bitmap, @@ -404,6 +510,13 @@ void iova_bitmap_set(struct iova_bitmap *bitmap, struct iova_bitmap_map *mapped = &bitmap->mapped; unsigned long cur_bit, last_bit, last_page_idx; +#ifdef CONFIG_MM_SUBPAGE + if (mapped->fragments) { + iova_bitmap_set_fragments(bitmap, iova, length); + return; + } +#endif + update_indexes: if (unlikely(!iova_bitmap_mapped_range(mapped, iova, length))) { /* diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 404f31d8f72916..0839878bfb67c5 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -207,7 +208,7 @@ static unsigned long iopt_area_index_to_iova(struct iopt_area *area, index -= iopt_area_index(area); if (index == 0) return iopt_area_iova(area); - return iopt_area_iova(area) - area->page_offset + index * PAGE_SIZE; + return iopt_area_iova(area) - area->page_offset + (index << area->page_shift); } static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, @@ -219,7 +220,7 @@ static unsigned long iopt_area_index_to_iova_last(struct iopt_area *area, if (index == iopt_area_last_index(area)) return iopt_area_last_iova(area); return iopt_area_iova(area) - area->page_offset + - (index - iopt_area_index(area) + 1) * PAGE_SIZE - 1; + ((index - iopt_area_index(area) + 1) << area->page_shift) - 1; } static void iommu_unmap_nofail(struct iommu_domain *domain, unsigned long iova, @@ -993,9 +994,11 @@ static int update_mm_locked_vm(struct iopt_pages *pages, unsigned long npages, } else if ((!user || (!user->upages && !user->ufolios)) && pages->source_mm != current->mm) { - if (!mmget_not_zero(pages->source_mm)) + if (mmget_not_zero(pages->source_mm)) + do_put = true; + else if (inc) return -EINVAL; - do_put = true; + /* mmgrab() keeps the lock and counters alive for final uncharge. */ } mmap_write_lock(pages->source_mm); @@ -1378,10 +1381,15 @@ static struct iopt_pages *iopt_alloc_pages(unsigned long start_byte, kref_init(&pages->kref); xa_init_flags(&pages->pinned_pfns, XA_FLAGS_ACCOUNT); +#ifdef CONFIG_MM_SUBPAGE + xa_init_flags(&pages->fragments, XA_FLAGS_ACCOUNT); + xa_init_flags(&pages->fragment_pages, XA_FLAGS_ACCOUNT); +#endif + pages->page_shift = PAGE_SHIFT; mutex_init(&pages->mutex); pages->source_mm = current->mm; mmgrab(pages->source_mm); - pages->npages = DIV_ROUND_UP(length + start_byte, PAGE_SIZE); + pages->npages = ((length - 1 + start_byte) >> PAGE_SHIFT) + 1; pages->access_itree = RB_ROOT_CACHED; pages->domains_itree = RB_ROOT_CACHED; pages->writable = writable; @@ -1400,8 +1408,10 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, { struct iopt_pages *pages; unsigned long end; + unsigned int shift = min_t(unsigned int, mm_page_shift(current->mm), PAGE_SHIFT); + unsigned long granule = 1UL << shift; void __user *uptr_down = - (void __user *)ALIGN_DOWN((uintptr_t)uptr, PAGE_SIZE); + (void __user *)ALIGN_DOWN((uintptr_t)uptr, granule); if (check_add_overflow((unsigned long)uptr, length, &end)) return ERR_PTR(-EOVERFLOW); @@ -1409,6 +1419,8 @@ struct iopt_pages *iopt_alloc_user_pages(void __user *uptr, pages = iopt_alloc_pages(uptr - uptr_down, length, writable); if (IS_ERR(pages)) return pages; + pages->page_shift = shift; + pages->npages = ((length - 1 + (uptr - uptr_down)) >> shift) + 1; pages->uptr = uptr_down; pages->type = IOPT_ADDRESS_USER; return pages; @@ -1656,6 +1668,12 @@ void iopt_release_pages(struct kref *kref) WARN_ON(!RB_EMPTY_ROOT(&pages->domains_itree.rb_root)); WARN_ON(pages->npinned); WARN_ON(!xa_empty(&pages->pinned_pfns)); +#ifdef CONFIG_MM_SUBPAGE + WARN_ON(!xa_empty(&pages->fragments)); + WARN_ON(!xa_empty(&pages->fragment_pages)); + xa_destroy(&pages->fragments); + xa_destroy(&pages->fragment_pages); +#endif if (iopt_is_dmabuf(pages) && pages->dmabuf.attach) { struct dma_buf *dmabuf = pages->dmabuf.attach->dmabuf; @@ -1832,6 +1850,9 @@ void iopt_area_unmap_domain(struct iopt_area *area, struct iommu_domain *domain) void iopt_area_unfill_domain(struct iopt_area *area, struct iopt_pages *pages, struct iommu_domain *domain) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_unfill_domain(area, pages, domain); + if (iopt_dmabuf_revoked(pages)) return; @@ -1855,6 +1876,9 @@ int iopt_area_fill_domain(struct iopt_area *area, struct iommu_domain *domain) lockdep_assert_held(&area->pages->mutex); + if (iopt_pages_use_fragments(area->pages)) + return iopt_fragments_fill_domain(area, area->pages, domain); + if (iopt_dmabuf_revoked(area->pages)) return 0; @@ -1916,6 +1940,9 @@ int iopt_area_fill_domains(struct iopt_area *area, struct iopt_pages *pages) if (xa_empty(&area->iopt->domains)) return 0; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_fill_domains(area, pages); + mutex_lock(&pages->mutex); if (iopt_is_dmabuf(pages)) { rc = iopt_dmabuf_track_all_domains(area, pages); @@ -2365,6 +2392,10 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, if (iopt_is_dmabuf(pages)) return -EINVAL; + if (iopt_pages_use_fragments(pages) && change_mm && + !(flags & IOMMUFD_ACCESS_RW_KTHREAD)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); + if (pages->type != IOPT_ADDRESS_USER) return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, @@ -2385,11 +2416,14 @@ int iopt_pages_rw_access(struct iopt_pages *pages, unsigned long start_byte, * ignore any pinning inconsistencies, unlike a real DMA path. */ if (change_mm) { - if (!mmget_not_zero(pages->source_mm)) + if (!mmget_not_zero(pages->source_mm)) { + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_rw(pages, start_byte, data, length, flags); return iopt_pages_rw_slow(pages, start_index, last_index, start_byte % PAGE_SIZE, data, length, flags); + } kthread_use_mm(pages->source_mm); } @@ -2454,6 +2488,11 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, if (iopt_is_dmabuf(pages)) return -EINVAL; + /* Preserve typed ownership and validate the native output geometry. */ + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_access(area, start_index, last_index, + out_pages, lock_area); + mutex_lock(&pages->mutex); access = iopt_pages_get_exact_access(pages, start_index, last_index); if (access) { @@ -2494,6 +2533,47 @@ int iopt_area_add_access(struct iopt_area *area, unsigned long start_index, return rc; } +/* Internal hardware queues consume a contiguous physical byte range. */ +int iopt_area_add_phys_access(struct iopt_area *area, unsigned long start_byte, + unsigned long length, phys_addr_t *out_phys) +{ + struct iopt_pages *pages = area->pages; + unsigned long first = start_byte >> pages->page_shift; + unsigned long last = (start_byte + length - 1) >> pages->page_shift; + unsigned long count = last - first + 1, i; + struct page **list; + phys_addr_t base; + int rc; + + if (iopt_is_dmabuf(pages)) + return -EINVAL; + if (iopt_pages_use_fragments(pages)) + return iopt_fragments_add_phys_access(area, start_byte, length, out_phys); + + /* Retain the native page supplier and its accounting/batching rules. */ + list = kvcalloc(count, sizeof(*list), GFP_KERNEL_ACCOUNT); + if (!list) + return -ENOMEM; + rc = iopt_area_add_access(area, first, last, list, 0, true); + if (rc) + goto out_free; + base = PFN_PHYS(page_to_pfn(list[0])); + for (i = 1; i < count; i++) { + if (page_to_pfn(list[i]) != page_to_pfn(list[0]) + i) { + rc = -EFAULT; + goto out_unpin; + } + } + *out_phys = base + offset_in_page(start_byte); + goto out_free; + +out_unpin: + iopt_area_remove_access(area, first, last, true); +out_free: + kvfree(list); + return rc; +} + /** * iopt_area_remove_access() - Release an in-kernel access for PFNs * @area: The source of PFNs @@ -2526,7 +2606,10 @@ void iopt_area_remove_access(struct iopt_area *area, unsigned long start_index, goto out_unlock; interval_tree_remove(&access->node, &pages->access_itree); - iopt_pages_unfill_xarray(pages, start_index, last_index); + if (iopt_pages_use_fragments(pages)) + iopt_fragments_unfill_access(pages, start_index, last_index); + else + iopt_pages_unfill_xarray(pages, start_index, last_index); kfree(access); out_unlock: mutex_unlock(&pages->mutex); diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index af07c642a5260e..146b1fefa0475f 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -1201,6 +1202,28 @@ static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, unsigned long pfn; long npages; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + struct user_page_fragment fragment; + phys_addr_t user_phys; + + npages = get_user_fragments_remote(current->mm, + (uintptr_t)uptr, page_size, 0, &fragment, 1); + if (npages != 1) { + rc = npages < 0 ? npages : -EFAULT; + goto out_put; + } + user_phys = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + io_phys = iommu_iova_to_phys(&mock->domain, iova); + rc = fragment.length == page_size && io_phys == user_phys ? 0 : -EINVAL; + release_user_fragments(&fragment, 1, false); + if (rc) + goto out_put; + iova += page_size; + uptr += page_size; + continue; + } +#endif npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, pages); if (npages < 0) { @@ -1506,9 +1529,39 @@ static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, /* Check that the pages in a page array match the pages in the user VA */ static int iommufd_test_check_pages(void __user *uptr, struct page **pages, - size_t npages) -{ - for (; npages; npages--) { + size_t length) +{ +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) { + unsigned long offset = 0; + + while (offset < length) { + struct user_page_fragment fragment; + unsigned long bytes = min_t(unsigned long, length - offset, + mm_user_fragment_size(current->mm) - + (((unsigned long)uptr + offset) & + (mm_user_fragment_size(current->mm) - 1))); + phys_addr_t actual, expected; + long rc; + + rc = get_user_fragments_remote(current->mm, + (unsigned long)uptr + offset, bytes, 0, &fragment, 1); + if (rc <= 0) + return rc ?: -EFAULT; + actual = PFN_PHYS(folio_pfn(fragment.folio)) + fragment.offset; + expected = PFN_PHYS(page_to_pfn(pages[offset / PAGE_SIZE])) + + offset_in_page(offset); + bytes = fragment.length; + release_user_fragments(&fragment, 1, false); + if (actual != expected || !bytes || + bytes > PAGE_SIZE - offset_in_page(offset)) + return -EBADE; + offset += bytes; + } + return 0; + } +#endif + for (size_t npages = DIV_ROUND_UP(length, PAGE_SIZE); npages; npages--) { struct page *tmp_pages[1]; long rc; @@ -1586,7 +1639,7 @@ static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, if (uptr) { rc = iommufd_test_check_pages( uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, - npages); + length + offset_in_page(iova)); if (rc) goto out_unaccess; } diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c index fc13cf4737ea3f..fa8296b15ef596 100644 --- a/drivers/iommu/iommufd/viommu.c +++ b/drivers/iommu/iommufd/viommu.c @@ -263,11 +263,7 @@ static void iommufd_hw_queue_destroy_access(struct iommufd_ctx *ictx, struct iommufd_access *access, u64 base_iova, size_t length) { - u64 aligned_iova = PAGE_ALIGN_DOWN(base_iova); - u64 offset = base_iova - aligned_iova; - - iommufd_access_unpin_pages(access, aligned_iova, - PAGE_ALIGN(length + offset)); + iommufd_access_unpin_pages(access, base_iova, length); iommufd_access_detach_internal(access); iommufd_access_destroy_internal(ictx, access); } @@ -300,66 +296,28 @@ static struct iommufd_access * iommufd_hw_queue_alloc_phys(struct iommu_hw_queue_alloc *cmd, struct iommufd_viommu *viommu, phys_addr_t *base_pa) { - u64 aligned_iova = PAGE_ALIGN_DOWN(cmd->nesting_parent_iova); - u64 offset = cmd->nesting_parent_iova - aligned_iova; struct iommufd_access *access; - struct page **pages; - size_t max_npages; - size_t length; - size_t i; int rc; - /* max_npages = DIV_ROUND_UP(offset + cmd->length, PAGE_SIZE) */ - if (check_add_overflow(offset, cmd->length, &length)) - return ERR_PTR(-ERANGE); - if (check_add_overflow(length, PAGE_SIZE - 1, &length)) + if (cmd->nesting_parent_iova > ULONG_MAX || cmd->length > ULONG_MAX) return ERR_PTR(-ERANGE); - max_npages = length / PAGE_SIZE; - /* length needs to be page aligned too */ - length = max_npages * PAGE_SIZE; - - /* - * Use kvcalloc() to avoid memory fragmentation for a large page array. - * Set __GFP_NOWARN to avoid syzkaller blowups - */ - pages = kvzalloc_objs(*pages, max_npages, GFP_KERNEL | __GFP_NOWARN); - if (!pages) - return ERR_PTR(-ENOMEM); - - access = iommufd_access_create_internal(viommu->ictx); - if (IS_ERR(access)) { - rc = PTR_ERR(access); - goto out_free; - } + access = iommufd_access_create_internal_phys(viommu->ictx); + if (IS_ERR(access)) + return access; rc = iommufd_access_attach_internal(access, viommu->hwpt->ioas); if (rc) goto out_destroy; - - rc = iommufd_access_pin_pages(access, aligned_iova, length, pages, 0); + rc = iommufd_access_pin_phys(access, cmd->nesting_parent_iova, + cmd->length, base_pa); if (rc) goto out_detach; - - /* Validate if the underlying physical pages are contiguous */ - for (i = 1; i < max_npages; i++) { - if (page_to_pfn(pages[i]) == page_to_pfn(pages[i - 1]) + 1) - continue; - rc = -EFAULT; - goto out_unpin; - } - - *base_pa = (page_to_pfn(pages[0]) << PAGE_SHIFT) + offset; - kvfree(pages); return access; -out_unpin: - iommufd_access_unpin_pages(access, aligned_iova, length); out_detach: iommufd_access_detach_internal(access); out_destroy: iommufd_access_destroy_internal(viommu->ictx, access); -out_free: - kvfree(pages); return ERR_PTR(rc); } diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index e6d3a2311a99b2..df079de9ee5de5 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -28,11 +28,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -84,6 +87,8 @@ struct vfio_domain { bool enforce_cache_coherency : 1; }; +struct vfio_fragment_provider; + struct vfio_dma { struct rb_node node; dma_addr_t iova; /* Device address */ @@ -99,6 +104,7 @@ struct vfio_dma { unsigned long *bitmap; struct mm_struct *mm; size_t locked_vm; + struct vfio_fragment_provider *fragments; }; struct vfio_batch { @@ -128,7 +134,9 @@ struct vfio_pfn { struct rb_node node; dma_addr_t iova; /* Device address */ unsigned long pfn; /* Host pfn */ + unsigned long length; /* Bytes retained in this DMA */ unsigned int ref_count; + bool native_pin; /* separately adopted legacy external PIN */ }; struct vfio_regions { @@ -152,6 +160,9 @@ struct vfio_regions { #define DIRTY_BITMAP_SIZE_MAX DIRTY_BITMAP_BYTES(DIRTY_BITMAP_PAGES_MAX) static int put_pfn(unsigned long pfn, int prot); +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn); +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length); static struct vfio_iommu_group* vfio_iommu_find_iommu_group(struct vfio_iommu *iommu, @@ -238,6 +249,12 @@ static void vfio_unlink_dma(struct vfio_iommu *iommu, struct vfio_dma *old) } +/* Software dirty granularity must survive domain attach/detach unchanged. */ +static size_t vfio_dirty_pgsize(struct vfio_iommu *iommu) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) ? SZ_4K : 1UL << __ffs(iommu->pgsize_bitmap); +} + static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) { uint64_t npages = dma->size / pgsize; @@ -272,14 +289,15 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) for (p = rb_first(&dma->pfn_list); p; p = rb_next(p)) { struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); - bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); + bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, + vpfn->length >> pgshift); } } static void vfio_iommu_populate_bitmap_full(struct vfio_iommu *iommu) { struct rb_node *n; - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); for (n = rb_first(&iommu->dma_list); n; n = rb_next(n)) { struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); @@ -383,7 +401,7 @@ static void vfio_unlink_pfn(struct vfio_dma *dma, struct vfio_pfn *old) } static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, - unsigned long pfn) + unsigned long pfn, unsigned long length) { struct vfio_pfn *vpfn; @@ -393,6 +411,7 @@ static int vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, vpfn->iova = iova; vpfn->pfn = pfn; + vpfn->length = length; vpfn->ref_count = 1; vfio_link_pfn(dma, vpfn); return 0; @@ -421,8 +440,21 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) vpfn->ref_count--; if (!vpfn->ref_count) { - ret = put_pfn(vpfn->pfn, dma->prot); - vfio_remove_from_pfn_list(dma, vpfn); + if (dma->fragments) { + unsigned long offset = vpfn->iova - dma->iova; + unsigned long length = vpfn->length; + + unsigned long pfn = vpfn->pfn; + bool native_pin = vpfn->native_pin; + + vfio_remove_from_pfn_list(dma, vpfn); + if (native_pin) + vfio_fragments_put_native_pin(dma, pfn); + vfio_fragments_release_range(dma, offset, length); + } else { + ret = put_pfn(vpfn->pfn, dma->prot); + vfio_remove_from_pfn_list(dma, vpfn); + } } return ret; } @@ -430,10 +462,19 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, bool lock_cap, long npage) { - int ret = mmap_write_lock_killable(mm); + int ret; - if (ret) - return ret; + if (!npage) + return 0; + + /* A released pin must be refunded even if the caller is interrupted. */ + if (npage < 0) { + mmap_write_lock(mm); + } else { + ret = mmap_write_lock_killable(mm); + if (ret) + return ret; + } ret = __account_locked_vm(mm, abs(npage), npage > 0, task, lock_cap); mmap_write_unlock(mm); @@ -443,25 +484,28 @@ static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) { struct mm_struct *mm; + bool get_mm = async && npage > 0; int ret; if (!npage) return 0; mm = dma->mm; - if (async && !mmget_not_zero(mm)) + if (get_mm && !mmget_not_zero(mm)) return -ESRCH; /* process exited */ ret = mm_lock_acct(dma->task, mm, dma->lock_cap, npage); if (!ret) dma->locked_vm += npage; - if (async) + if (get_mm) mmput(mm); return ret; } +#include "vfio_iommu_type1_fragments.c" + /* * Some mappings aren't backed by a struct page, for example an mmap'd * MMIO range for our own or another device. These use a different @@ -765,7 +809,8 @@ static long vfio_pin_pages_remote(struct vfio_dma *dma, unsigned long vaddr, */ if (acct_pages) { if (!dma->lock_cap && - mm->locked_vm + lock_acct + acct_pages > limit) { + mm_locked_vm_native_pages(mm) + + lock_acct + acct_pages > limit) { pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, limit << PAGE_SHIFT); ret = -ENOMEM; @@ -889,6 +934,78 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, return unlocked; } +/* A native page may contain independently owned fine-grained DMA areas. */ +static unsigned long vfio_unpin_external_range(struct vfio_iommu *iommu, + dma_addr_t start, unsigned long length, + bool do_accounting) +{ + unsigned long done = 0; + + while (done < length) { + dma_addr_t iova = start + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long bytes; + + if (!dma || iova < dma->iova) + break; + bytes = min_t(unsigned long, length - done, + PAGE_SIZE - offset_in_page(iova)); + bytes = min(bytes, dma->size - (iova - dma->iova)); + vfio_unpin_page_external(dma, iova, do_accounting); + done += bytes; + } + return done; +} + +static int vfio_pin_external_segment(struct vfio_iommu *iommu, + struct vfio_dma *dma, dma_addr_t iova, unsigned long length, + int prot, bool do_accounting, unsigned long *pfn) +{ + struct vfio_pfn *vpfn; + int ret; + + if ((dma->prot & prot) != prot) + return -EPERM; + vpfn = vfio_iova_get_vfio_pfn(dma, iova); + if (vpfn) { + if (vpfn->length != length) { + vfio_unpin_page_external(dma, iova, do_accounting); + return -EINVAL; + } + *pfn = vpfn->pfn; + } else if (dma->fragments) { + ret = vfio_fragments_external(dma, iova, length, pfn); + if (ret) + return ret; + } else { + /* Ordinary native mappings still use one native GUP per page. */ + if (length != PAGE_SIZE) + return -EINVAL; + ret = vfio_pin_page_external(dma, dma->vaddr + iova - dma->iova, + pfn, do_accounting); + if (ret) + return ret; + if (!pfn_valid(*pfn)) { + ret = -EINVAL; + goto put_native; + } + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); + if (ret) + goto put_native; + } + if (iommu->dirty_page_tracking) { + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, (iova - dma->iova) >> pgshift, + length >> pgshift); + } + return 0; +put_native: + if (put_pfn(*pfn, dma->prot) && do_accounting) + vfio_lock_acct(dma, -1, true); + return ret; +} + static int vfio_iommu_type1_pin_pages(void *iommu_data, struct iommu_group *iommu_group, dma_addr_t user_iova, @@ -897,119 +1014,72 @@ static int vfio_iommu_type1_pin_pages(void *iommu_data, { struct vfio_iommu *iommu = iommu_data; struct vfio_iommu_group *group; - int i, j, ret; - unsigned long remote_vaddr; - struct vfio_dma *dma; + unsigned long pinned = 0; bool do_accounting; dma_addr_t iova_end; size_t iova_size; + int i, ret; if (!iommu || !pages || npage <= 0) return -EINVAL; - - /* Supported for v2 version only */ + /* Callers recover the original byte offset from the returned page. */ + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (!iommu->v2) return -EACCES; - if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end)) return -EOVERFLOW; mutex_lock(&iommu->lock); - if (WARN_ONCE(iommu->vaddr_invalid_count, "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) { ret = -EBUSY; goto pin_done; } - - /* Fail if no dma_umap notifier is registered */ if (list_empty(&iommu->device_list)) { ret = -EINVAL; goto pin_done; } - - /* - * If iommu capable domain exist in the container then all pages are - * already pinned and accounted. Accounting should be done if there is no - * iommu capable domain in the container. - */ do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - unsigned long phys_pfn; - dma_addr_t iova; - struct vfio_pfn *vpfn; - - iova = user_iova + PAGE_SIZE * i; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) { - ret = -EINVAL; - goto pin_unwind; - } - - if ((dma->prot & prot) != prot) { - ret = -EPERM; - goto pin_unwind; - } - - vpfn = vfio_iova_get_vfio_pfn(dma, iova); - if (vpfn) { - pages[i] = pfn_to_page(vpfn->pfn); - continue; - } - - remote_vaddr = dma->vaddr + (iova - dma->iova); - ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, - do_accounting); - if (ret) - goto pin_unwind; - - if (!pfn_valid(phys_pfn)) { - ret = -EINVAL; - goto pin_unwind; - } - - ret = vfio_add_to_pfn_list(dma, iova, phys_pfn); - if (ret) { - if (put_pfn(phys_pfn, dma->prot) && do_accounting) - vfio_lock_acct(dma, -1, true); - goto pin_unwind; - } + unsigned long done = 0; - pages[i] = pfn_to_page(phys_pfn); + pages[i] = NULL; + while (done < PAGE_SIZE) { + dma_addr_t iova = user_iova + PAGE_SIZE * i + done; + struct vfio_dma *dma = vfio_find_dma(iommu, iova, 1); + unsigned long length, pfn; - if (iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); - - /* - * Bitmap populated with the smallest supported page - * size - */ - bitmap_set(dma->bitmap, - (iova - dma->iova) >> pgshift, 1); + if (!dma || iova < dma->iova) { + ret = -EINVAL; + goto pin_unwind; + } + length = min_t(unsigned long, PAGE_SIZE - done, + dma->size - (iova - dma->iova)); + ret = vfio_pin_external_segment(iommu, dma, iova, length, + prot, do_accounting, &pfn); + if (ret) + goto pin_unwind; + pinned += length; + if (pages[i] && page_to_pfn(pages[i]) != pfn) { + ret = -EINVAL; + goto pin_unwind; + } + pages[i] = pfn_to_page(pfn); + done += length; } } - ret = i; - + ret = npage; group = vfio_iommu_find_iommu_group(iommu, iommu_group); if (!group->pinned_page_dirty_scope) { group->pinned_page_dirty_scope = true; iommu->num_non_pinned_groups--; } - goto pin_done; - pin_unwind: - pages[i] = NULL; - for (j = 0; j < i; j++) { - dma_addr_t iova; - - iova = user_iova + PAGE_SIZE * j; - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - vfio_unpin_page_external(dma, iova, do_accounting); - pages[j] = NULL; - } + vfio_unpin_external_range(iommu, user_iova, pinned, do_accounting); + while (i >= 0) + pages[i--] = NULL; pin_done: mutex_unlock(&iommu->lock); return ret; @@ -1019,39 +1089,21 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, dma_addr_t user_iova, int npage) { struct vfio_iommu *iommu = iommu_data; - bool do_accounting; dma_addr_t iova_end; size_t iova_size; - int i; - - /* Supported for v2 version only */ - if (WARN_ON(!iommu->v2)) - return; + unsigned long unpinned; - if (WARN_ON(npage <= 0)) + if (WARN_ON(!iommu->v2) || WARN_ON(npage <= 0)) return; - + user_iova = ALIGN_DOWN(user_iova, PAGE_SIZE); if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || check_add_overflow(user_iova, iova_size - 1, &iova_end))) return; - mutex_lock(&iommu->lock); - - do_accounting = list_empty(&iommu->domain_list); - for (i = 0; i < npage; i++) { - dma_addr_t iova = user_iova + PAGE_SIZE * i; - struct vfio_dma *dma; - - dma = vfio_find_dma(iommu, iova, PAGE_SIZE); - if (!dma) - break; - - vfio_unpin_page_external(dma, iova, do_accounting); - } - + unpinned = vfio_unpin_external_range(iommu, user_iova, iova_size, + list_empty(&iommu->domain_list)); mutex_unlock(&iommu->lock); - - WARN_ON(i != npage); + WARN_ON(unpinned != iova_size); } static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, @@ -1155,6 +1207,10 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, if (!dma->size) return 0; + if (dma->fragments) { + vfio_fragments_unmap(iommu, dma); + return 0; + } if (list_empty(&iommu->domain_list)) return 0; @@ -1233,6 +1289,7 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) { WARN_ON(!RB_EMPTY_ROOT(&dma->pfn_list)); vfio_unmap_unpin(iommu, dma, true); + vfio_fragments_destroy(dma); vfio_unlink_dma(iommu, dma); put_task_struct(dma->task); mmdrop(dma->mm); @@ -1246,21 +1303,18 @@ static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *dma) static void vfio_update_pgsize_bitmap(struct vfio_iommu *iommu) { struct vfio_domain *domain; + unsigned long minimum = SZ_4K; iommu->pgsize_bitmap = ULONG_MAX; - - list_for_each_entry(domain, &iommu->domain_list, next) + list_for_each_entry(domain, &iommu->domain_list, next) { iommu->pgsize_bitmap &= domain->domain->pgsize_bitmap; - - /* - * In case the IOMMU supports page sizes smaller than PAGE_SIZE - * we pretend PAGE_SIZE is supported and hide sub-PAGE_SIZE sizes. - * That way the user will be able to map/unmap buffers whose size/ - * start address is aligned with PAGE_SIZE. Pinning code uses that - * granularity while iommu driver can use the sub-PAGE_SIZE size - * to map the buffer. - */ - if (iommu->pgsize_bitmap & ~PAGE_MASK) { + minimum = max(minimum, 1UL << __ffs(domain->domain->pgsize_bitmap)); + } + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) { + /* Every domain can compose its minimum pages into this common unit. */ + iommu->pgsize_bitmap &= ~(minimum - 1); + iommu->pgsize_bitmap |= minimum; + } else if (iommu->pgsize_bitmap & ~PAGE_MASK) { iommu->pgsize_bitmap &= PAGE_MASK; iommu->pgsize_bitmap |= PAGE_SIZE; } @@ -1367,6 +1421,8 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) { struct vfio_device *device; + dma_addr_t iova = dma->iova; + size_t size = dma->size; if (list_empty(&iommu->device_list)) return; @@ -1377,11 +1433,12 @@ static void vfio_notify_dma_unmap(struct vfio_iommu *iommu, * call back down to this code and try to obtain the iommu->lock we must * drop it. */ - mutex_lock(&iommu->device_list_lock); + /* The range is a snapshot: another unmap may free dma while unlocked. */ mutex_unlock(&iommu->lock); + mutex_lock(&iommu->device_list_lock); list_for_each_entry(device, &iommu->device_list, iommu_entry) - device->ops->dma_unmap(device, dma->iova, dma->size); + device->ops->dma_unmap(device, iova, size); mutex_unlock(&iommu->device_list_lock); mutex_lock(&iommu->lock); @@ -1437,11 +1494,10 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, /* When dirty tracking is enabled, allow only min supported pgsize */ if ((unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) && - (!iommu->dirty_page_tracking || (bitmap->pgsize != pgsize))) { + (!iommu->dirty_page_tracking || (bitmap->pgsize != vfio_dirty_pgsize(iommu)))) { goto unlock; } - WARN_ON((pgsize - 1) & PAGE_MASK); again: /* * vfio-iommu-type1 (v1) - User mappings were coalesced together to @@ -1530,7 +1586,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, if (unmap->flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { ret = update_user_bitmap(bitmap->data, iommu, dma, - iova, pgsize); + iova, vfio_dirty_pgsize(iommu)); if (ret) break; } @@ -1587,6 +1643,13 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; + if (dma->fragments) { + dma->size = map_size; + ret = vfio_fragments_map_all(iommu, dma); + if (ret) + vfio_remove_dma(iommu, dma); + return ret; + } vfio_batch_init(&batch); while (size) { @@ -1644,27 +1707,30 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, return list_empty(iova); } -static int vfio_change_dma_owner(struct vfio_dma *dma) +static int vfio_change_dma_mm_pages(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap, unsigned long pages) { - struct task_struct *task = current->group_leader; - struct mm_struct *mm = current->mm; - long npage = dma->locked_vm; - bool lock_cap; + long old_pages = dma->locked_vm; int ret; - if (mm == dma->mm) - return 0; - - lock_cap = capable(CAP_IPC_LOCK); - ret = mm_lock_acct(task, mm, lock_cap, npage); + if (mm == dma->mm) { + if (pages == old_pages) + return 0; + ret = mm_lock_acct(task, mm, lock_cap, (long)pages - old_pages); + if (!ret) + dma->locked_vm = pages; + return ret; + } + ret = mm_lock_acct(task, mm, lock_cap, pages); if (ret) return ret; - if (mmget_not_zero(dma->mm)) { - mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); - mmput(dma->mm); + /* Metadata adoption can deduplicate charges, but not original PIN refs. */ + ret = mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -old_pages); + if (ret) { + mm_lock_acct(task, mm, lock_cap, -(long)pages); + return ret; } - if (dma->task != task) { put_task_struct(dma->task); dma->task = get_task_struct(task); @@ -1673,9 +1739,40 @@ static int vfio_change_dma_owner(struct vfio_dma *dma) dma->mm = mm; mmgrab(dma->mm); dma->lock_cap = lock_cap; + dma->locked_vm = pages; return 0; } +static int vfio_change_dma_mm(struct vfio_dma *dma, struct task_struct *task, + struct mm_struct *mm, bool lock_cap) +{ + return vfio_change_dma_mm_pages(dma, task, mm, lock_cap, dma->locked_vm); +} + +static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, + unsigned long vaddr) +{ + struct vfio_fragment_provider *provider; + int ret; + + if (dma->fragments || (mm_page_size(current->mm) == PAGE_SIZE && + IS_ALIGNED(vaddr, PAGE_SIZE))) + return vfio_change_dma_mm(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK)); + + /* Prepare only metadata: failure must preserve every legacy pin/charge. */ + provider = vfio_fragments_prepare_native(iommu, dma); + if (IS_ERR(provider)) + return PTR_ERR(provider); + ret = vfio_change_dma_mm_pages(dma, current->group_leader, current->mm, + capable(CAP_IPC_LOCK), vfio_fragments_native_pages(provider)); + if (ret) + vfio_fragments_abort_native(provider); + else + vfio_fragments_commit_native(dma, provider); + return ret; +} + static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { @@ -1713,8 +1810,6 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); - WARN_ON((pgsize - 1) & PAGE_MASK); - if ((size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; @@ -1728,7 +1823,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, dma->size != size) { ret = -EINVAL; } else { - ret = vfio_change_dma_owner(dma); + ret = vfio_change_dma_owner(iommu, dma, vaddr); if (ret) goto out_unlock; dma->vaddr = vaddr; @@ -1781,6 +1876,11 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, /* Insert zero-sized and grow as we map chunks of it */ vfio_link_dma(iommu, dma); + ret = vfio_fragments_init(dma, size); + if (ret) { + vfio_remove_dma(iommu, dma); + goto out_unlock; + } /* Don't pin and map if container doesn't contain IOMMU capable domain*/ if (list_empty(&iommu->domain_list)) @@ -1789,7 +1889,7 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, ret = vfio_pin_map_dma(iommu, dma, size); if (!ret && iommu->dirty_page_tracking) { - ret = vfio_dma_bitmap_alloc(dma, pgsize); + ret = vfio_dma_bitmap_alloc(dma, vfio_dirty_pgsize(iommu)); if (ret) vfio_remove_dma(iommu, dma); } @@ -1822,6 +1922,14 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, size_t pos = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (!ret) + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + continue; + } while (pos < dma->size) { dma_addr_t iova = dma->iova + pos; @@ -1906,6 +2014,11 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); size_t pos = 0; + if (dma->fragments) { + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + continue; + } if (dma->iommu_mapped) { iommu_unmap(domain->domain, dma->iova, dma->size); continue; @@ -2442,6 +2555,10 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) long locked = 0, unlocked = 0; dma = rb_entry(n, struct vfio_dma, node); + if (dma->fragments) { + vfio_unmap_unpin(iommu, dma, true); + continue; + } unlocked += vfio_unmap_unpin(iommu, dma, false); p = rb_first(&dma->pfn_list); for (; p; p = rb_next(p)) { @@ -2646,7 +2763,8 @@ static void *vfio_iommu_type1_open(unsigned long arg) mutex_init(&iommu->lock); mutex_init(&iommu->device_list_lock); INIT_LIST_HEAD(&iommu->device_list); - iommu->pgsize_bitmap = PAGE_MASK; + iommu->pgsize_bitmap = IS_ENABLED(CONFIG_MM_SUBPAGE) ? + GENMASK(BITS_PER_LONG - 1, 12) : PAGE_MASK; INIT_LIST_HEAD(&iommu->emulated_iommu_groups); return iommu; @@ -2812,7 +2930,7 @@ static int vfio_iommu_migration_build_caps(struct vfio_iommu *iommu, cap_mig.flags = 0; /* support minimum pgsize */ - cap_mig.pgsize_bitmap = (size_t)1 << __ffs(iommu->pgsize_bitmap); + cap_mig.pgsize_bitmap = vfio_dirty_pgsize(iommu); cap_mig.max_dirty_bitmap_size = DIRTY_BITMAP_SIZE_MAX; return vfio_info_add_capability(caps, &cap_mig.header, sizeof(cap_mig)); @@ -2910,6 +3028,63 @@ static int vfio_iommu_type1_map_dma(struct vfio_iommu *iommu, return vfio_dma_do_map(iommu, &map); } +#ifdef CONFIG_COMPAT +struct compat_vfio_bitmap { + compat_u64 pgsize; + compat_u64 size; + compat_uptr_t data; +}; +#endif + +static size_t vfio_bitmap_size(void) +{ +#ifdef CONFIG_COMPAT + /* IA32 differs from AArch32/x32 in its u64 alignment and tail padding. */ + if (in_compat_syscall() && compat_need_64bit_alignment_fixup()) + return sizeof(struct compat_vfio_bitmap); +#endif + return sizeof(struct vfio_bitmap); +} + +static size_t vfio_dirty_bitmap_get_size(void) +{ + return offsetof(struct vfio_iommu_type1_dirty_bitmap_get, bitmap) + + vfio_bitmap_size(); +} + +static int vfio_bitmap_from_user(struct vfio_bitmap *bitmap, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + compat_uptr_t data; + unsigned long offset = offsetof(struct vfio_bitmap, data); + + /* Read only the pointer field, never its trailing compat padding. */ + if (copy_from_user(bitmap, user, offset) || + get_user(data, (compat_uptr_t __user *)(user + offset))) + return -EFAULT; + bitmap->data = compat_ptr(data); + return 0; + } +#endif + return copy_from_user(bitmap, user, sizeof(*bitmap)) ? -EFAULT : 0; +} + +static int vfio_dirty_bitmap_get_from_user( + struct vfio_iommu_type1_dirty_bitmap_get *range, void __user *user) +{ +#ifdef CONFIG_COMPAT + if (in_compat_syscall()) { + unsigned long offset = offsetof(typeof(*range), bitmap); + + if (copy_from_user(range, user, offset)) + return -EFAULT; + return vfio_bitmap_from_user(&range->bitmap, user + offset); + } +#endif + return copy_from_user(range, user, sizeof(*range)) ? -EFAULT : 0; +} + static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, unsigned long arg) { @@ -2937,13 +3112,12 @@ static int vfio_iommu_type1_unmap_dma(struct vfio_iommu *iommu, if (unmap.flags & VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP) { unsigned long pgshift; - if (unmap.argsz < (minsz + sizeof(bitmap))) + if (unmap.argsz < minsz + vfio_bitmap_size()) return -EINVAL; - if (copy_from_user(&bitmap, - (void __user *)(arg + minsz), - sizeof(bitmap))) - return -EFAULT; + ret = vfio_bitmap_from_user(&bitmap, (void __user *)(arg + minsz)); + if (ret) + return ret; if (!access_ok((void __user *)bitmap.data, bitmap.size)) return -EINVAL; @@ -2992,7 +3166,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t pgsize; mutex_lock(&iommu->lock); - pgsize = 1 << __ffs(iommu->pgsize_bitmap); + pgsize = vfio_dirty_pgsize(iommu); if (!iommu->dirty_page_tracking) { ret = vfio_dma_bitmap_alloc_all(iommu, pgsize); if (!ret) @@ -3015,12 +3189,12 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, size_t size, iommu_pgsize; dma_addr_t iova, iova_end; - if (!data_size || data_size < sizeof(range)) + if (data_size < vfio_dirty_bitmap_get_size()) return -EINVAL; - if (copy_from_user(&range, (void __user *)(arg + minsz), - sizeof(range))) - return -EFAULT; + ret = vfio_dirty_bitmap_get_from_user(&range, (void __user *)(arg + minsz)); + if (ret) + return ret; iova = range.iova; size = range.size; @@ -3046,7 +3220,7 @@ static int vfio_iommu_type1_dirty_pages(struct vfio_iommu *iommu, mutex_lock(&iommu->lock); - iommu_pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); + iommu_pgsize = vfio_dirty_pgsize(iommu); /* allow only smallest supported pgsize */ if (range.bitmap.pgsize != iommu_pgsize) { @@ -3112,11 +3286,12 @@ static void vfio_iommu_type1_register_device(void *iommu_data, * Holding both locks here allows avoiding the device_list_lock in * several fast paths. See vfio_notify_dma_unmap() */ - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_add(&vdev->iommu_entry, &iommu->device_list); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static void vfio_iommu_type1_unregister_device(void *iommu_data, @@ -3127,11 +3302,12 @@ static void vfio_iommu_type1_unregister_device(void *iommu_data, if (!vdev->ops->dma_unmap) return; - mutex_lock(&iommu->lock); + /* Unmap callbacks hold device_list_lock while reentering iommu->lock. */ mutex_lock(&iommu->device_list_lock); + mutex_lock(&iommu->lock); list_del(&vdev->iommu_entry); - mutex_unlock(&iommu->device_list_lock); mutex_unlock(&iommu->lock); + mutex_unlock(&iommu->device_list_lock); } static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, @@ -3156,6 +3332,22 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, return -EPERM; mm = dma->mm; + if (dma->fragments) { + int ret; + + if (!kthread && current->mm != mm) + return -EFAULT; + offset = user_iova - dma->iova; + count = min(count, dma->size - offset); + ret = vfio_fragments_rw(dma, offset, data, count, write, copied); + if (*copied && write && iommu->dirty_page_tracking) { + unsigned long shift = __ffs(vfio_dirty_pgsize(iommu)); + + bitmap_set(dma->bitmap, offset >> shift, + ((offset + *copied - 1) >> shift) - (offset >> shift) + 1); + } + return ret; + } if (!mmget_not_zero(mm)) return -EPERM; @@ -3175,7 +3367,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, *copied = copy_to_user((void __user *)vaddr, data, count) ? 0 : count; if (*copied && iommu->dirty_page_tracking) { - unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + unsigned long pgshift = __ffs(vfio_dirty_pgsize(iommu)); /* * Bitmap populated with the smallest supported page * size @@ -3281,3 +3473,13 @@ MODULE_VERSION(DRIVER_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); + +#ifdef CONFIG_VFIO_TYPE1_ACCOUNT_KUNIT_TEST +#include "vfio_iommu_type1_account_test.c" +#if defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_ARM64_USER4K_EXPERIMENTAL) +#include "vfio_iommu_type1_fragments_test.c" +#ifdef CONFIG_ARM64_USER4K_VFIO_IOCTL_TEST +#include "vfio_iommu_type1_ioctl_test.c" +#endif +#endif +#endif diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c new file mode 100644 index 00000000000000..22d46a1b805a7b --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -0,0 +1,609 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1: retained byte ranges, rather than IOMMU PFNs, own pins. */ +#ifdef CONFIG_MM_SUBPAGE +struct vfio_fragment { + struct rb_node node; + unsigned long start; + struct user_page_fragment pin; +}; + +struct vfio_fragment_provider { + struct rb_root records; + struct xarray physical_pages; + unsigned long npinned; +}; + +static phys_addr_t vfio_fragment_phys(const struct user_page_fragment *pin) +{ + return PFN_PHYS(folio_pfn(pin->folio)) + pin->offset; +} + +/* Returns a containing range, or the first range after the requested byte. */ +static struct vfio_fragment *vfio_fragment_find(struct vfio_dma *dma, + unsigned long offset) +{ + struct rb_node *node = dma->fragments->records.rb_node; + struct vfio_fragment *next = NULL; + + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + if (offset < record->start) { + next = record; + node = node->rb_left; + } else if (offset - record->start >= record->pin.length) { + node = node->rb_right; + } else { + return record; + } + } + return next; +} + +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + struct vfio_fragment_provider *provider; + + if (mm_page_size(dma->mm) == PAGE_SIZE && + IS_ALIGNED(dma->vaddr | dma->iova | length, PAGE_SIZE)) + return 0; + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return -ENOMEM; + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + dma->fragments = provider; + return 0; +} + +static int vfio_fragment_store(struct vfio_dma *dma, unsigned long start, + struct user_page_fragment *pin) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct rb_node **link = &provider->records.rb_node, *parent = NULL; + unsigned long pfn = PHYS_PFN(vfio_fragment_phys(pin)); + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct vfio_fragment *record; + int ret; + + record = kmalloc_obj(*record, GFP_KERNEL_ACCOUNT); + if (!record) + return -ENOMEM; + ret = xa_err(xa_store(&provider->physical_pages, pfn, xa_mk_value(count + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) { + kfree(record); + return ret; + } + while (*link) { + struct vfio_fragment *cur = rb_entry(*link, struct vfio_fragment, node); + + parent = *link; + link = start < cur->start ? &(*link)->rb_left : &(*link)->rb_right; + } + record->start = start; + record->pin = *pin; + rb_link_node(&record->node, parent, link); + rb_insert_color(&record->node, &provider->records); + if (!count) + provider->npinned++; + *pin = (struct user_page_fragment) {}; + return 0; +} + +/* These descriptors borrow existing native GUP pins until owner commit. */ +static int vfio_fragment_native_pin(unsigned long pfn, struct user_page_fragment *pin) +{ + struct page *page; + + if (!pfn_valid(pfn)) + return -EOPNOTSUPP; + page = pfn_to_page(pfn); + if (PageReserved(page)) + return -EOPNOTSUPP; + *pin = (struct user_page_fragment) { + .folio = page_folio(page), + .offset = page_to_phys(page) - PFN_PHYS(folio_pfn(page_folio(page))), + .length = PAGE_SIZE, + .pinned = true, + }; + return 0; +} + +/* No PIN, GET or account reference was acquired while preparing metadata. */ +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + while ((node = rb_first(&provider->records))) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + + rb_erase(node, &provider->records); + kfree(record); + } + xa_destroy(&provider->physical_pages); + kfree(provider); +} + +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_fragment_provider *provider; + struct vfio_dma stage = {}; + struct vfio_domain *domain = NULL; + struct rb_node *node; + int ret; + + if (mm_page_size(dma->mm) != PAGE_SIZE || dma->has_rsvd) + return ERR_PTR(-EOPNOTSUPP); + if (dma->iommu_mapped) { + if (list_empty(&iommu->domain_list)) + return ERR_PTR(-EFAULT); + domain = list_first_entry(&iommu->domain_list, struct vfio_domain, next); + } + provider = kzalloc_obj(*provider, GFP_KERNEL_ACCOUNT); + if (!provider) + return ERR_PTR(-ENOMEM); + provider->records = RB_ROOT; + xa_init(&provider->physical_pages); + stage.fragments = provider; + /* One descriptor owns each original native GUP pin, including aliases. */ + if (domain) { + for (unsigned long offset = 0; offset < dma->size; offset += PAGE_SIZE) { + phys_addr_t phys = iommu_iova_to_phys(domain->domain, dma->iova + offset); + struct user_page_fragment pin; + + if (!phys || !IS_ALIGNED(phys, PAGE_SIZE)) { + ret = -EFAULT; + goto out; + } + ret = vfio_fragment_native_pin(PHYS_PFN(phys), &pin); + if (!ret) + ret = vfio_fragment_store(&stage, offset, &pin); + if (ret) + goto out; + cond_resched(); + } + } + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + struct user_page_fragment pin; + + ret = vfio_fragment_native_pin(vpfn->pfn, &pin); + if (ret) + goto out; + if (!domain) { + /* Without a domain, the external pin is the cached buffer. */ + ret = vfio_fragment_store(&stage, vpfn->iova - dma->iova, &pin); + } else { + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, + vpfn->pfn)); + + /* The external GUP reference may identify a different page. */ + ret = xa_err(xa_store(&provider->physical_pages, vpfn->pfn, + xa_mk_value(count + 1), GFP_KERNEL_ACCOUNT)); + if (!ret && !count) + provider->npinned++; + } + if (ret) + goto out; + } + return provider; +out: + vfio_fragments_abort_native(provider); + return ERR_PTR(ret); +} + +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return provider->npinned; +} + +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ + struct rb_node *node; + + /* Mapped legacy external consumers own separate native GUP references. */ + if (dma->iommu_mapped) { + for (node = rb_first(&dma->pfn_list); node; node = rb_next(node)) + rb_entry(node, struct vfio_pfn, node)->native_pin = true; + } + dma->fragments = provider; +} + +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ + struct vfio_fragment_provider *provider = dma->fragments; + unsigned long count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + struct user_page_fragment pin; + + if (WARN_ON(!count || vfio_fragment_native_pin(pfn, &pin))) + return; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), GFP_NOWAIT); + } + release_user_fragments(&pin, 1, dma->prot & IOMMU_WRITE); + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static bool vfio_fragments_external_overlap(struct vfio_dma *dma, + dma_addr_t start, unsigned long length) +{ + struct rb_node *node = dma->pfn_list.rb_node; + + while (node) { + struct vfio_pfn *vpfn = rb_entry(node, struct vfio_pfn, node); + + if (start + length <= vpfn->iova) + node = node->rb_left; + else if (start >= vpfn->iova + vpfn->length) + node = node->rb_right; + else + return true; + } + return false; +} + +/* Device unmaps and their IOTLB synchronization must precede this call. */ +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + struct vfio_fragment_provider *provider = dma->fragments; + struct vfio_fragment *first = vfio_fragment_find(dma, start); + struct rb_node *node = first ? &first->node : NULL; + unsigned long end = start + length; + + if (dma->iommu_mapped) + return; + while (node) { + struct vfio_fragment *record = rb_entry(node, struct vfio_fragment, node); + dma_addr_t iova = dma->iova + record->start; + unsigned long pfn, count; + + if (record->start >= end) + break; + node = rb_next(node); + /* Native-page consumers retain all records overlapping their page. */ + if (vfio_fragments_external_overlap(dma, iova, record->pin.length)) + continue; + pfn = PHYS_PFN(vfio_fragment_phys(&record->pin)); + count = xa_to_value(xa_load(&provider->physical_pages, pfn)); + if (WARN_ON(!count)) + continue; + if (count == 1) { + xa_erase(&provider->physical_pages, pfn); + provider->npinned--; + } else { + xa_store(&provider->physical_pages, pfn, xa_mk_value(count - 1), + GFP_NOWAIT); + } + rb_erase(&record->node, &provider->records); + release_user_fragments(&record->pin, 1, dma->prot & IOMMU_WRITE); + kfree(record); + cond_resched(); + } + if (provider->npinned < dma->locked_vm) + vfio_lock_acct(dma, -(long)(dma->locked_vm - provider->npinned), true); +} + +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ + vfio_fragments_release_range(dma, 0, dma->size); +} + +/* Cached records survive source VA replacement and the last source mm user. */ +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + unsigned long end = start + length, original = start; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + struct user_page_fragment *batch = NULL; + bool live = false; + int ret = 0; + + while (start < end) { + struct vfio_fragment *record = vfio_fragment_find(dma, start); + unsigned long stop = end; + long count; + + if (record && record->start <= start) { + start = record->start + record->pin.length; + continue; + } + if (record) + stop = min(stop, record->start); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + goto out; + } + live = true; + batch = kmalloc(PAGE_SIZE, GFP_KERNEL_ACCOUNT); + if (!batch) { + ret = -ENOMEM; + goto out; + } + } + count = pin_user_fragments_remote(dma->mm, dma->vaddr + start, + stop - start, FOLL_LONGTERM | (dma->prot & IOMMU_WRITE ? FOLL_WRITE : 0), + batch, capacity); + if (count <= 0) { + ret = count ?: -EFAULT; + goto out; + } + for (unsigned long i = 0; i < count; i++) { + unsigned long bytes = batch[i].length; + + if (WARN_ON(!bytes || bytes > stop - start || + offset_in_page(vfio_fragment_phys(&batch[i])) > + PAGE_SIZE - bytes)) + ret = -EFAULT; + else + ret = vfio_fragment_store(dma, start, &batch[i]); + if (ret) { + release_user_fragments(batch + i, count - i, false); + goto out; + } + start += bytes; + } + cond_resched(); + } + if (dma->fragments->npinned > dma->locked_vm) + ret = vfio_lock_acct(dma, dma->fragments->npinned - dma->locked_vm, false); +out: + kfree(batch); + if (live) + mmput(dma->mm); + if (ret) + vfio_fragments_release_range(dma, original, length); + return ret; +} + +/* Merge physical byte runs while retaining each original pin identity. */ +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + unsigned long done = 0, min_size = 1UL << __ffs(domain->pgsize_bitmap); + int ret = 0; + + while (done < dma->size) { + struct vfio_fragment *record = vfio_fragment_find(dma, done); + unsigned long bytes, limit = dma->size - done; + phys_addr_t phys; + + if (WARN_ON(!record || record->start > done)) { + ret = -EFAULT; + goto out; + } + if (disable_hugepages) + limit = min(limit, PAGE_SIZE); + phys = vfio_fragment_phys(&record->pin) + done - record->start; + bytes = min(record->pin.length - (done - record->start), limit); + while (bytes < limit) { + record = vfio_fragment_find(dma, done + bytes); + if (!record || record->start != done + bytes || + vfio_fragment_phys(&record->pin) != phys + bytes) + break; + bytes += min_t(unsigned long, record->pin.length, limit - bytes); + } + if (!IS_ALIGNED((dma->iova + done) | phys | bytes, min_size)) { + ret = -EINVAL; + goto out; + } + ret = iommu_map(domain, dma->iova + done, phys, bytes, + dma->prot | IOMMU_CACHE, GFP_KERNEL_ACCOUNT); + if (ret) + goto out; + done += bytes; + cond_resched(); + } +out: + if (ret && done) + iommu_unmap(domain, dma->iova, done); + return ret; +} + +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + int ret; + + ret = vfio_fragments_acquire(dma, 0, dma->size); + if (ret) + return ret; + list_for_each_entry(domain, &iommu->domain_list, next) { + ret = vfio_fragments_map_domain(dma, domain->domain); + if (ret) + goto unwind; + } + dma->iommu_mapped = true; + return 0; +unwind: + list_for_each_entry_continue_reverse(domain, &iommu->domain_list, next) + iommu_unmap(domain->domain, dma->iova, dma->size); + vfio_fragments_release_unused(dma); + return ret; +} + +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + struct vfio_domain *domain; + + if (dma->iommu_mapped) { + list_for_each_entry(domain, &iommu->domain_list, next) + WARN_ON(iommu_unmap(domain->domain, dma->iova, dma->size) != dma->size); + dma->iommu_mapped = false; + } + vfio_fragments_release_unused(dma); +} + +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ + if (!dma->fragments) + return; + vfio_fragments_release_unused(dma); + if (WARN_ON(!RB_EMPTY_ROOT(&dma->fragments->records))) + return; + xa_destroy(&dma->fragments->physical_pages); + kfree(dma->fragments); + dma->fragments = NULL; +} + +/* The output page has no offset field: validate every byte and its native offset. */ +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + unsigned long first = iova - dma->iova, done = 0; + phys_addr_t phys = 0; + int ret; + + ret = vfio_fragments_acquire(dma, first, length); + if (ret) + return ret; + while (done < length) { + struct vfio_fragment *record = vfio_fragment_find(dma, first + done); + phys_addr_t next; + unsigned long offset; + + if (WARN_ON(!record || record->start > first + done)) { + ret = -EFAULT; + goto out; + } + offset = first + done - record->start; + next = vfio_fragment_phys(&record->pin) + offset; + if (!done) + phys = next; + if (offset_in_page(phys) != offset_in_page(iova) || next != phys + done) { + ret = -EINVAL; + goto out; + } + done += min_t(unsigned long, record->pin.length - offset, length - done); + } + *pfn = PHYS_PFN(phys); + ret = vfio_add_to_pfn_list(dma, iova, *pfn, length); +out: + if (ret) + vfio_fragments_release_range(dma, first, length); + return ret; +} + +/* CPU-only missing ranges take GET references, not new long-term DMA pins. */ +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + bool live = false; + int ret = 0; + + while (length) { + struct vfio_fragment *record = vfio_fragment_find(dma, offset); + struct user_page_fragment temp = {}, *pin; + unsigned long in_record = 0, bytes = length; + void *addr; + + if (record && record->start <= offset) { + pin = &record->pin; + in_record = offset - record->start; + } else { + long count; + + if (record) + bytes = min(bytes, record->start - offset); + if (!live) { + if (!mmget_not_zero(dma->mm)) { + ret = -EFAULT; + break; + } + live = true; + } + count = get_user_fragments_remote(dma->mm, dma->vaddr + offset, bytes, + write ? FOLL_WRITE : 0, &temp, 1); + if (count <= 0) { + ret = count ?: -EFAULT; + break; + } + pin = &temp; + } + bytes = min_t(unsigned long, bytes, pin->length - in_record); + addr = kmap_local_folio(pin->folio, pin->offset + in_record); + if (write) + memcpy(addr, data, bytes); + else + memcpy(data, addr, bytes); + kunmap_local(addr); + if (write && record && record->start <= offset) + mark_user_fragment_dirty(pin); + release_user_fragments(&temp, 1, write); + data += bytes; + offset += bytes; + length -= bytes; + *copied += bytes; + } + if (live) + mmput(dma->mm); + return ret; +} +#else +static struct vfio_fragment_provider * +vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) +{ +} +static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) +{ + return 0; +} +static void vfio_fragments_commit_native(struct vfio_dma *dma, + struct vfio_fragment_provider *provider) +{ +} +static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) +{ +} +static int vfio_fragments_init(struct vfio_dma *dma, size_t length) +{ + return 0; +} +static void vfio_fragments_release_range(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ +} +static void vfio_fragments_release_unused(struct vfio_dma *dma) +{ +} +static int vfio_fragments_acquire(struct vfio_dma *dma, unsigned long start, + unsigned long length) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_domain(struct vfio_dma *dma, struct iommu_domain *domain) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_map_all(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ + return -EOPNOTSUPP; +} +static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) +{ +} +static void vfio_fragments_destroy(struct vfio_dma *dma) +{ +} +static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, + unsigned long length, unsigned long *pfn) +{ + return -EOPNOTSUPP; +} +static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, + unsigned long length, bool write, size_t *copied) +{ + return -EOPNOTSUPP; +} +#endif From e46dbf82d7966ae9d0db6cd5089f327c6fc8822f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 022/631] arm64: support smaller userspace translation granules per mm Keep kernel mappings at their native granule while selecting TTBR0 geometry per address space. Add capability checks, table-fragment lifetime, TLB invalidation and exec selection for 4K on 16K/64K and 16K on 64K. The backend remains opt-in; private launch commands are experimental. Signed-off-by: Diljit Singh (cherry picked from commit d2bfa3d271d01d3204306eb4c8cb6875a1310e35) --- arch/arm64/Kconfig | 1 + arch/arm64/Kconfig.debug | 21 +++ arch/arm64/include/asm/cpucaps.h | 5 + arch/arm64/include/asm/elf.h | 9 ++ arch/arm64/include/asm/kernel-pgtable.h | 2 +- arch/arm64/include/asm/mmu.h | 40 ++++++ arch/arm64/include/asm/mmu_context.h | 61 +++++++- arch/arm64/include/asm/mte.h | 8 ++ arch/arm64/include/asm/page.h | 33 ++++- arch/arm64/include/asm/pgalloc.h | 35 ++++- arch/arm64/include/asm/pgtable-hwdef.h | 12 +- arch/arm64/include/asm/pgtable.h | 51 +++++-- arch/arm64/include/asm/shmparam.h | 6 + arch/arm64/include/asm/tlb.h | 27 +++- arch/arm64/include/asm/tlbflush.h | 124 +++++++++++------ arch/arm64/include/asm/user-pgalloc.h | 59 ++++++++ arch/arm64/include/asm/user-pgtable.h | 164 ++++++++++++++++++++++ arch/arm64/include/asm/user-pte.h | 116 ++++++++++++++++ arch/arm64/kernel/cpufeature.c | 83 ++++++++++- arch/arm64/kernel/pi/idreg-override.c | 32 +++++ arch/arm64/kernel/pi/map_kernel.c | 2 +- arch/arm64/kernel/process.c | 33 ++++- arch/arm64/kernel/sys.c | 5 + arch/arm64/kernel/sys32.c | 6 + arch/arm64/kernel/vmlinux.lds.S | 5 +- arch/arm64/lib/mte.S | 46 +++++++ arch/arm64/mm/Makefile | 1 + arch/arm64/mm/context.c | 43 ++++++ arch/arm64/mm/contpte.c | 143 +++++++++++-------- arch/arm64/mm/copypage.c | 52 +++++++ arch/arm64/mm/fault.c | 30 ++-- arch/arm64/mm/fixmap.c | 2 +- arch/arm64/mm/init.c | 5 +- arch/arm64/mm/mmu.c | 4 +- arch/arm64/mm/pgd.c | 41 +++++- arch/arm64/mm/proc.S | 16 +-- arch/arm64/mm/user4k-pgalloc.c | 176 ++++++++++++++++++++++++ arch/arm64/tools/cpucaps | 2 + 38 files changed, 1349 insertions(+), 152 deletions(-) create mode 100644 arch/arm64/include/asm/user-pgalloc.h create mode 100644 arch/arm64/include/asm/user-pgtable.h create mode 100644 arch/arm64/include/asm/user-pte.h create mode 100644 arch/arm64/mm/user4k-pgalloc.c diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index a582d841515373..31b402de5245b0 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -362,6 +362,7 @@ config FIX_EARLYCON_MEM config PGTABLE_LEVELS int + default 4 if ARM64_USER4K_EXPERIMENTAL default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36 default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 || ARM64_VA_BITS_52) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 265c4461031f4d..d15ccdeee21250 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -21,3 +21,24 @@ config ARM64_RELOC_TEST tristate "Relocation testing module" source "drivers/hwtracing/coresight/Kconfig" + +config ARM64_USER4K_EXPERIMENTAL + bool "Experimental mixed-granule userspace" + select ARCH_HAS_MM_PGTABLE + select ARCH_HAS_USER_PAGE_SIZE + select MM_SUBPAGE + select MM_ID + select HAVE_ARCH_TLB_REMOVE_TABLE + select ARCH_SUPPORTS_PT_RECLAIM + depends on (ARM64_16K_PAGES || ARM64_64K_PAGES) && ARM64_VA_BITS_48 && ARM64_PA_BITS_48 + depends on !ARM64_LPA2 && !NO_PAGE_MAPCOUNT + default n + help + Build the in-development MM support for alternative 4K user tables + on 16K/64K kernels and 16K user tables on 64K kernels. + Ordinary address spaces and kernel tables retain the native granule. An experimental + one-shot prctl selects a supported granule for the next exec. + This is a development ABI: several MM interfaces and device mappings + still require integration and validation. The private prctl numbers + are not upstream assignments or a stable ABI promise. + Say N for normal kernels. diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index d0d3cdd5763cab..fff3a9570ad659 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -19,6 +19,11 @@ cpucap_is_possible(const unsigned int cap) "cap must be < ARM64_NCAPS"); switch (cap) { + case ARM64_HAS_USER16K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + IS_ENABLED(CONFIG_ARM64_64K_PAGES); + case ARM64_HAS_USER4K_GRANULE: + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL); case ARM64_HAS_EPAN: return IS_ENABLED(CONFIG_ARM64_EPAN); case ARM64_SVE: diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index d2779d604c7be2..23320b49c32082 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -121,6 +121,9 @@ #define CORE_DUMP_USE_REGSET #define ELF_EXEC_PAGESIZE PAGE_SIZE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define ELF_EXEC_PAGESIZE_MM(mm) mm_page_size(mm) +#endif /* * This is the base location for PIE (ET_DYN with INTERP) loads. On @@ -181,6 +184,12 @@ do { \ NEW_AUX_ENT(AT_IGNORE, 0); \ } while (0) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +struct linux_binprm; +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat); +#define arch_bprm_prepare_mm arm64_bprm_prepare_mm +#endif + #define ARCH_HAS_SETUP_ADDITIONAL_PAGES struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index 229ee7976f6934..2bd73e773095c1 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -27,7 +27,7 @@ #endif #define SWAPPER_BLOCK_SIZE (UL(1) << SWAPPER_BLOCK_SHIFT) -#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) +#define SWAPPER_PGTABLE_LEVELS (ARM64_NATIVE_PGTABLE_LEVELS - SWAPPER_SKIP_LEVEL) #define INIT_IDMAP_PGTABLE_LEVELS (IDMAP_LEVELS - SWAPPER_SKIP_LEVEL) #define IDMAP_VA_BITS 48 diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 5e1211c540ab23..15943bec53392e 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -8,6 +8,10 @@ #include #define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */ +#define MMCF_USER_4K 0x2 /* experimental, immutable mm table geometry */ +#define MMCF_USER_16K 0x4 /* experimental, immutable mm table geometry */ +#define MMCF_USER_64K 0x8 /* internal geometry; larger user ABI not enabled */ +#define MMCF_USER_GRANULE (MMCF_USER_4K | MMCF_USER_16K | MMCF_USER_64K) #define USER_ASID_BIT 48 #define USER_ASID_FLAG (UL(1) << USER_ASID_BIT) @@ -16,6 +20,15 @@ #include #include +/* AArch32 falls back to the native ABI when 4K translation is unavailable. */ +static inline unsigned int arm64_exec_page_shift(bool compat, bool has_4k, + unsigned int native_shift, unsigned int requested_shift) +{ + if (compat) + return has_4k ? 12 : native_shift; + return requested_shift ?: native_shift; +} + typedef struct { atomic64_t id; #ifdef CONFIG_COMPAT @@ -25,8 +38,35 @@ typedef struct { void *vdso; unsigned long flags; u8 pkey_allocation_map; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + void *user4k_pt_frag[3]; +#endif } mm_context_t; +static inline bool arm64_context_user4k(const mm_context_t *context) +{ + return IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && + (context->flags & MMCF_USER_4K); +} + +static inline unsigned int arm64_context_page_shift(const mm_context_t *context) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL)) { + if (context->flags & MMCF_USER_4K) + return 12; + if (context->flags & MMCF_USER_16K) + return 14; + if (context->flags & MMCF_USER_64K) + return 16; + } + return PAGE_SHIFT; +} + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_page_shift(mm) arm64_context_page_shift(&(mm)->context) +#endif + /* * We use atomic64_read() here because the ASID for an 'mm_struct' can * be reallocated when scheduling one of its threads following a diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 803b6875815275..db27ff606555be 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -26,6 +26,17 @@ #include #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_mm_init_exec arch_mm_init_exec +static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) +{ + if (page_shift == 12) + mm->context.flags |= MMCF_USER_4K; + else if (page_shift == 14 && PAGE_SHIFT > 14) + mm->context.flags |= MMCF_USER_16K; +} +#endif + extern bool rodata_full; static inline void contextidr_thread_switch(struct task_struct *next) @@ -53,6 +64,29 @@ static inline void cpu_set_reserved_ttbr0(void) isb(); } +/* + * The caller has installed reserved_pg_dir in TTBR0 and excludes preemption. + * On a granule transition, complete that installation before changing TG0, + * then synchronise the new walk geometry before installing a user root. + * ASIDs remain unique across both granules; switching does not flush them. + */ +static inline void cpu_set_mm_tg0(struct mm_struct *mm) +{ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + unsigned long tcr = read_sysreg(tcr_el1); + unsigned int shift = mm_page_shift(mm); + unsigned long tg0 = shift == 12 ? TCR_TG0_4K : + (shift == 14 ? TCR_TG0_16K : TCR_TG0_64K); + + if ((tcr & TCR_EL1_TG0_MASK) == tg0) + return; + + isb(); + write_sysreg((tcr & ~TCR_EL1_TG0_MASK) | tg0, tcr_el1); + isb(); +#endif +} + void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) @@ -62,16 +96,21 @@ static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm) } /* - * Ensure TCR.T0SZ is set to the provided value. + * Set TCR.T0SZ and restore native TG0 for the idmap/transition tables. */ static inline void __cpu_set_tcr_t0sz(unsigned long t0sz) { unsigned long tcr = read_sysreg(tcr_el1); + unsigned long mask = TCR_EL1_T0SZ_MASK; - if ((tcr & TCR_EL1_T0SZ_MASK) == t0sz) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + mask |= TCR_EL1_TG0_MASK; + t0sz |= IS_ENABLED(CONFIG_ARM64_64K_PAGES) ? TCR_TG0_64K : TCR_TG0_16K; +#endif + if ((tcr & mask) == t0sz) return; - tcr &= ~TCR_EL1_T0SZ_MASK; + tcr &= ~mask; tcr |= t0sz; write_sysreg(tcr, tcr_el1); isb(); @@ -97,6 +136,8 @@ static inline void cpu_uninstall_idmap(void) local_flush_tlb_all(); __cpu_set_tcr_t0sz(TCR_T0SZ(vabits_actual)); + /* SW PAN will restore TTBR0 later, but already needs the mm's TG0. */ + cpu_set_mm_tg0(mm); if (mm != &init_mm && !system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); } @@ -168,6 +209,9 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) { atomic64_set(&mm->context.id, 0); refcount_set(&mm->context.pinned, 0); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + memset(mm->context.user4k_pt_frag, 0, sizeof(mm->context.user4k_pt_frag)); +#endif /* pkey 0 is the default, so always reserve it. */ mm->context.pkey_allocation_map = BIT(0); @@ -175,6 +219,16 @@ init_new_context(struct task_struct *tsk, struct mm_struct *mm) return 0; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); + +static inline void destroy_context(struct mm_struct *mm) +{ + arm64_user4k_pt_cache_destroy(mm); +} +#define destroy_context destroy_context +#endif + static inline void arch_dup_pkeys(struct mm_struct *oldmm, struct mm_struct *mm) { @@ -241,6 +295,7 @@ static inline void __switch_mm(struct mm_struct *next) */ if (next == &init_mm) { cpu_set_reserved_ttbr0(); + cpu_set_mm_tg0(next); return; } diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 7f7b97e099968e..890d49a4398b28 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -98,6 +98,8 @@ static inline bool try_page_mte_tagging(struct page *page) void mte_zero_clear_page_tags(void *addr); void mte_sync_tags(pte_t pte, unsigned int nr_pages); void mte_copy_page_tags(void *kto, const void *kfrom); +void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size); +void mte_clear_subpage_tags_range(void *addr, unsigned long size); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); void mte_cpu_setup(void); @@ -134,6 +136,12 @@ static inline void mte_sync_tags(pte_t pte, unsigned int nr_pages) static inline void mte_copy_page_tags(void *kto, const void *kfrom) { } +static inline void mte_copy_subpage_tags_range(void *kto, const void *kfrom, unsigned long size) +{ +} +static inline void mte_clear_subpage_tags_range(void *addr, unsigned long size) +{ +} static inline void mte_thread_init_user(void) { } diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 58200de8a221a6..96fa9d8ea23c57 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A shared zero mapping must cover the largest supported user granule. */ +#define ARCH_ZERO_PAGE_SIZE (1UL << 16) +#endif + #ifndef __ASSEMBLER__ #include /* for READ_IMPLIES_EXEC */ @@ -26,11 +31,28 @@ void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma); #define __HAVE_ARCH_COPY_USER_HIGHPAGE +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, + unsigned int size); +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, + unsigned int size); +void clear_user_subpage(struct page *page, unsigned int offset); +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset); +#endif + void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr); +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order); +#define vma_alloc_zeroed_movable_folio_order vma_alloc_zeroed_movable_folio_order +static inline struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, + unsigned long vaddr) +{ + return vma_alloc_zeroed_movable_folio_order(vma, vaddr, 0); +} #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); @@ -38,7 +60,14 @@ bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages); #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* A table may occupy a 4K fragment of a native 16K physical page. */ +typedef pte_t *pgtable_t; +#define pgtable_to_page(table) virt_to_page(table) +#define pgtable_from_page(page) ((page) ? (pgtable_t)page_address(page) : NULL) +#else typedef struct page *pgtable_t; +#endif int pfn_is_map_memory(unsigned long pfn); diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index 1b4509d3382c63..41b81203e575ca 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -15,7 +15,15 @@ #define __HAVE_ARCH_PGD_FREE #define __HAVE_ARCH_PUD_FREE +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define __HAVE_ARCH_PTE_ALLOC_ONE +#define __HAVE_ARCH_PTE_FREE +#define __HAVE_ARCH_PMD_ALLOC_ONE +#define __HAVE_ARCH_PMD_FREE +#define __HAVE_ARCH_PUD_ALLOC_ONE +#endif #include +#include #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) @@ -31,6 +39,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp) pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_AF; pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pud(pudp, __pud(__pa(pmdp) | pudval)); + return; + } +#endif __pud_populate(pudp, __pa(pmdp), pudval); } #else @@ -53,14 +67,24 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp) p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF; p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_p4d(p4dp, __p4d(__pa(pudp) | p4dval)); + return; + } +#endif __p4d_populate(p4dp, __pa(pudp), p4dval); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - if (!pgtable_l4_enabled()) + if (mm_pud_folded(mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + arm64_pgtable_free(pud); +#else __pud_free(mm, pud); +#endif } #else static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot) @@ -117,7 +141,16 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep) { VM_BUG_ON(mm == &init_mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_mm_alt_granule(mm)) { + set_pmd(pmdp, __pmd(__pa(ptep) | PMD_TYPE_TABLE | + PMD_TABLE_AF | PMD_TABLE_PXN)); + return; + } + __pmd_populate(pmdp, __pa(ptep), +#else __pmd_populate(pmdp, page_to_phys(ptep), +#endif PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN); } diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 72f31800c7030c..cb211cbcdb7fdf 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -28,8 +28,10 @@ * * which gets simplified as : */ +#define ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, page_shift) \ + (((va_bits) - PTDESC_ORDER - 1) / ((page_shift) - PTDESC_ORDER)) #define ARM64_HW_PGTABLE_LEVELS(va_bits) \ - (((va_bits) - PTDESC_ORDER - 1) / PTDESC_TABLE_SHIFT) + ARM64_HW_PGTABLE_LEVELS_FOR(va_bits, PAGE_SHIFT) /* * Size mapped by an entry at level n ( -1 <= n <= 3) @@ -44,7 +46,10 @@ * Rearranging it a bit we get : * (4 - n) * PTDESC_TABLE_SHIFT + PTDESC_ORDER */ -#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) (PTDESC_TABLE_SHIFT * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, page_shift) \ + (((page_shift) - PTDESC_ORDER) * (4 - (n)) + PTDESC_ORDER) +#define ARM64_HW_PGTABLE_LEVEL_SHIFT(n) \ + ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(n, PAGE_SHIFT) #define PTRS_PER_PTE (1 << PTDESC_TABLE_SHIFT) @@ -79,7 +84,8 @@ * PGDIR_SHIFT determines the size a top-level page table entry can map * (depending on the configuration, this level can be -1, 0, 1 or 2). */ -#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS) +#define ARM64_NATIVE_PGTABLE_LEVELS ARM64_HW_PGTABLE_LEVELS(VA_BITS) +#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - ARM64_NATIVE_PGTABLE_LEVELS) #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 4dfa42b7d05356..1ce97b836c8ac5 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -467,6 +467,8 @@ static inline pte_t pte_advance_pfn(pte_t pte, unsigned long nr) return pfn_pte(pte_pfn(pte) + nr, pte_pgprot(pte)); } +#include + /* * Hugetlb definitions. */ @@ -711,6 +713,23 @@ static inline void __set_ptes_anysz(struct mm_struct *mm, unsigned long addr, static inline void __set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned int nr) { + if (arm64_mm_alt_granule(mm)) { + unsigned int native_pages = DIV_ROUND_UP((pte_phys_mm(mm, pte) & ~PAGE_MASK) + + ((unsigned long)nr << mm_page_shift(mm)), PAGE_SIZE); + + page_table_check_ptes_set(mm, addr, ptep, pte, nr); + __sync_cache_and_tags(pte, native_pages); + for (;;) { + __check_safe_pte_update(mm, ptep, pte); + __set_pte_nosync(ptep, pte); + if (--nr == 0) + break; + ptep++; + pte = pte_advance_pfn_mm(mm, pte, 1); + } + __set_pte_complete(pte); + return; + } __set_ptes_anysz(mm, addr, ptep, pte, nr, PAGE_SIZE); } @@ -946,6 +965,8 @@ static inline pmd_t *pud_pgtable(pud_t pud) static __always_inline bool pgtable_l4_enabled(void) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return false; if (CONFIG_PGTABLE_LEVELS > 4 || !IS_ENABLED(CONFIG_ARM64_LPA2)) return true; if (!alternative_has_cap_likely(ARM64_ALWAYS_BOOT)) @@ -955,6 +976,8 @@ static __always_inline bool pgtable_l4_enabled(void) static inline bool mm_pud_folded(const struct mm_struct *mm) { + if (arm64_mm_alt_granule(mm)) + return mm_page_shift(mm) == 16; return !pgtable_l4_enabled(); } #define mm_pud_folded mm_pud_folded @@ -994,6 +1017,8 @@ static inline phys_addr_t p4d_page_paddr(p4d_t p4d) static inline pud_t *p4d_to_folded_pud(p4d_t *p4dp, unsigned long addr) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + return (pud_t *)p4dp; /* Ensure that 'p4dp' indexes a page table according to 'addr' */ VM_BUG_ON(((addr >> P4D_SHIFT) ^ ((u64)p4dp >> 3)) % PTRS_PER_P4D); @@ -1216,6 +1241,8 @@ p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr) #endif /* CONFIG_PGTABLE_LEVELS > 4 */ +#include + #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %016llx.\n", __FILE__, __LINE__, pgd_val(e)) @@ -1261,7 +1288,7 @@ static inline int __ptep_set_access_flags(struct vm_area_struct *vma, pte_t entry, int dirty) { return __ptep_set_access_flags_anysz(vma, address, ptep, entry, dirty, - PAGE_SIZE); + mm_page_size(vma->vm_mm)); } #ifdef CONFIG_TRANSPARENT_HUGEPAGE @@ -1389,7 +1416,7 @@ static inline void __clear_full_ptes(struct mm_struct *mm, unsigned long addr, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); } } @@ -1402,7 +1429,7 @@ static inline pte_t __get_and_clear_full_ptes(struct mm_struct *mm, pte = __ptep_get_and_clear(mm, addr, ptep); while (--nr) { ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(mm); tmp_pte = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(tmp_pte)) pte = pte_mkdirty(pte); @@ -1450,7 +1477,7 @@ static inline void __wrprotect_ptes(struct mm_struct *mm, unsigned long address, { unsigned int i; - for (i = 0; i < nr; i++, address += PAGE_SIZE, ptep++) + for (i = 0; i < nr; i++, address += mm_page_size(mm), ptep++) __ptep_set_wrprotect(mm, address, ptep); } @@ -1490,7 +1517,7 @@ static inline void __clear_young_dirty_ptes(struct vm_area_struct *vma, if (--nr == 0) break; ptep++; - addr += PAGE_SIZE; + addr += mm_page_size(vma->vm_mm); } } @@ -1516,9 +1543,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, * bits 0-1: present (must be zero) * bits 2: remember PG_anon_exclusive * bit 3: remember uffd-wp state + * bits 4-5: 4K slot offset within a native swap page (MM_SUBPAGE) * bits 6-10: swap type * bit 11: PTE_PRESENT_INVALID (must be zero) * bits 12-61: swap offset + * bits 62-63: high 4K slot offset bits (MM_SUBPAGE, up to 64K) */ #define __SWP_TYPE_SHIFT 6 #define __SWP_TYPE_BITS 5 @@ -1688,11 +1717,11 @@ static __always_inline void contpte_try_fold(struct mm_struct *mm, * We can't fold special mappings, because there is no associated folio. */ - const unsigned long contmask = CONT_PTES - 1; - bool valign = ((addr >> PAGE_SHIFT) & contmask) == contmask; + const unsigned long contmask = cont_ptes_mm(mm) - 1; + bool valign = ((addr >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(valign)) { - bool palign = (pte_pfn(pte) & contmask) == contmask; + bool palign = ((pte_phys_mm(mm, pte) >> mm_page_shift(mm)) & contmask) == contmask; if (unlikely(palign && pte_valid(pte) && !pte_cont(pte) && !pte_special(pte))) @@ -1713,7 +1742,11 @@ static inline unsigned int pte_batch_hint(pte_t *ptep, pte_t pte) if (!pte_valid_cont(pte)) return 1; - return CONT_PTES - (((unsigned long)ptep >> 3) & (CONT_PTES - 1)); + { + unsigned int nr = arm64_cont_ptes(arm64_ptep_page_shift(ptep)); + + return nr - (((unsigned long)ptep >> 3) & (nr - 1)); + } } /* diff --git a/arch/arm64/include/asm/shmparam.h b/arch/arm64/include/asm/shmparam.h index f920e22ec677cd..75120d3a71889b 100644 --- a/arch/arm64/include/asm/shmparam.h +++ b/arch/arm64/include/asm/shmparam.h @@ -12,6 +12,12 @@ */ #define COMPAT_SHMLBA (4 * PAGE_SIZE) +/* SHMLBA also aligns kernel allocations; scale only SysV attachments. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#define arch_shm_attach_align(mm, align) \ + (((align) >> PAGE_SHIFT) << mm_page_shift(mm)) +#endif + #include #endif /* __ASM_SHMPARAM_H */ diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 751bd57bc3bac5..21453f3125e0f9 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -10,6 +10,11 @@ #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +void __tlb_remove_table(void *token); +/* Fragment tokens do not carry a private descriptor callback head. */ +#define tlb_table_is_ptdesc(token) (!((unsigned long)(token) & 1UL)) +#endif #define tlb_flush tlb_flush static void tlb_flush(struct mmu_gather *tlb); @@ -76,8 +81,14 @@ static inline void tlb_flush(struct mmu_gather *tlb) static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { - struct ptdesc *ptdesc = page_ptdesc(pte); + struct ptdesc *ptdesc = page_ptdesc(pgtable_to_page(pte)); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pte | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } @@ -87,6 +98,12 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, { struct ptdesc *ptdesc = virt_to_ptdesc(pmdp); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pmdp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif @@ -97,9 +114,15 @@ static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, { struct ptdesc *ptdesc = virt_to_ptdesc(pudp); - if (!pgtable_l4_enabled()) + if (mm_pud_folded(tlb->mm)) return; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (test_bit(PT_fragmented, &ptdesc->pt_flags.f)) { + tlb_remove_table(tlb, (void *)((unsigned long)pudp | 1)); + return; + } +#endif tlb_remove_ptdesc(tlb, ptdesc); } #endif diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 57b4eda6a72bba..56062bf906cd41 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -56,8 +57,8 @@ }) /* - * Get translation granule of the system, which is decided by - * PAGE_SIZE. Used by TTL. + * Encode a translation granule for TTL and range operands. Native kernel + * and stage-2 callers use PAGE_SHIFT; userspace callers use their mm. * - 4KB : 1 * - 16KB : 2 * - 64KB : 3 @@ -66,20 +67,25 @@ #define TLBI_TTL_TG_16K 2 #define TLBI_TTL_TG_64K 3 -static inline unsigned long get_trans_granule(void) +static inline unsigned long get_trans_granule_shift(unsigned int page_shift) { - switch (PAGE_SIZE) { - case SZ_4K: + switch (page_shift) { + case 12: return TLBI_TTL_TG_4K; - case SZ_16K: + case 14: return TLBI_TTL_TG_16K; - case SZ_64K: + case 16: return TLBI_TTL_TG_64K; default: return 0; } } +static inline unsigned long get_trans_granule(void) +{ + return get_trans_granule_shift(PAGE_SHIFT); +} + #ifdef CONFIG_ARM64_ERRATUM_4193714 extern cpumask_t sme_active_cpus; @@ -175,13 +181,14 @@ static __always_inline void ipas2e1is(u64 arg) __tlbi(ipas2e1is, arg); } -static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, - u16 asid) +static __always_inline void __tlbi_level_asid_granule(tlbi_op op, u64 addr, + u32 level, u16 asid, + unsigned int page_shift) { u64 arg = __TLBI_VADDR(addr, asid); if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && level <= 3) { - u64 ttl = level | (get_trans_granule() << 2); + u64 ttl = level | (get_trans_granule_shift(page_shift) << 2); FIELD_MODIFY(TLBI_TTL_MASK, &arg, ttl); } @@ -189,6 +196,12 @@ static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, op(arg); } +static __always_inline void __tlbi_level_asid(tlbi_op op, u64 addr, u32 level, + u16 asid) +{ + __tlbi_level_asid_granule(op, addr, level, asid, PAGE_SHIFT); +} + static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) { __tlbi_level_asid(op, addr, level, 0); @@ -471,36 +484,45 @@ static __always_inline void ripas2e1is(u64 arg) __tlbi(ripas2e1is, arg); } -static __always_inline void __tlbi_range(tlbi_op op, u64 addr, - u16 asid, int scale, int num, - u32 level, bool lpa2) +static __always_inline void __tlbi_range_granule(tlbi_op op, u64 addr, + u16 asid, int scale, int num, + u32 level, bool lpa2, + unsigned int page_shift) { u64 arg = 0; - arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : PAGE_SHIFT)); + arg |= FIELD_PREP(TLBIR_BADDR_MASK, addr >> (lpa2 ? 16 : page_shift)); arg |= FIELD_PREP(TLBIR_TTL_MASK, level > 3 ? 0 : level); arg |= FIELD_PREP(TLBIR_NUM_MASK, num); arg |= FIELD_PREP(TLBIR_SCALE_MASK, scale); - arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule()); + arg |= FIELD_PREP(TLBIR_TG_MASK, get_trans_granule_shift(page_shift)); arg |= FIELD_PREP(TLBIR_ASID_MASK, asid); op(arg); } -static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, +/* + * Production callers supply the constant system_supports_tlb_range function, + * which inlines to the existing alternative branch inside the loop. Do not + * evaluate it into a bool outside the loop: that adds runtime flag handling + * to the native path. Recording tests supply constant true/false predicates. + */ +static __always_inline void __flush_tlb_range_op_granule(tlbi_op lop, tlbi_op rop, u64 start, size_t pages, u64 stride, u16 asid, - u32 level, bool lpa2) + u32 level, bool lpa2, + unsigned int page_shift, + bool (*range_supported)(void)) { - u64 addr = start, end = start + pages * PAGE_SIZE; + u64 addr = start, end = start + (pages << page_shift); int scale = 3; while (addr != end) { int num; - pages = (end - addr) >> PAGE_SHIFT; + pages = (end - addr) >> page_shift; - if (!system_supports_tlb_range() || pages == 1) + if (!range_supported() || pages == 1) goto invalidate_one; if (lpa2 && !IS_ALIGNED(addr, SZ_64K)) @@ -508,26 +530,41 @@ static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, num = __TLBI_RANGE_NUM(pages, scale); if (num >= 0) { - __tlbi_range(rop, addr, asid, scale, num, level, lpa2); - addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; + __tlbi_range_granule(rop, addr, asid, scale, num, level, + lpa2, page_shift); + addr += __TLBI_RANGE_PAGES(num, scale) << page_shift; } scale--; continue; invalidate_one: - __tlbi_level_asid(lop, addr, level, asid); + __tlbi_level_asid_granule(lop, addr, level, asid, page_shift); addr += stride; } } +static __always_inline void __flush_tlb_range_op(tlbi_op lop, tlbi_op rop, + u64 start, size_t pages, + u64 stride, u16 asid, + u32 level, bool lpa2) +{ + __flush_tlb_range_op_granule(lop, rop, start, pages, stride, asid, + level, lpa2, PAGE_SHIFT, system_supports_tlb_range); +} + #define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled()) #define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \ __flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled()) -static inline bool __flush_tlb_range_limit_excess(unsigned long pages, - unsigned long stride) +#define __flush_user_tlb_range_op(op, start, pages, stride, asid, level, shift) \ + __flush_tlb_range_op_granule(op, r##op, start, pages, stride, asid, \ + level, lpa2_is_enabled(), shift, system_supports_tlb_range) + +static inline bool __flush_tlb_range_limit_excess_granule(unsigned long pages, + unsigned long stride, + unsigned int page_shift) { /* * Assume that the worst case number of DVM ops required to flush a @@ -538,7 +575,13 @@ static inline bool __flush_tlb_range_limit_excess(unsigned long pages, if (system_supports_tlb_range()) return pages > MAX_TLBI_RANGE_PAGES; - return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT; + return pages >= (MAX_DVM_OPS * stride) >> page_shift; +} + +static inline bool __flush_tlb_range_limit_excess(unsigned long pages, + unsigned long stride) +{ + return __flush_tlb_range_limit_excess_granule(pages, stride, PAGE_SHIFT); } typedef unsigned __bitwise tlbf_t; @@ -565,10 +608,11 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; unsigned long asid, pages; + unsigned int page_shift = mm_page_shift(mm); - pages = (end - start) >> PAGE_SHIFT; + pages = (end - start) >> page_shift; - if (__flush_tlb_range_limit_excess(pages, stride)) { + if (__flush_tlb_range_limit_excess_granule(pages, stride, page_shift)) { flush_tlb_mm(mm); return; } @@ -582,20 +626,20 @@ static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma, switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) { case TLBF_NONE: - __flush_s1_tlb_range_op(vae1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vae1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOWALKCACHE: - __flush_s1_tlb_range_op(vale1is, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1is, start, pages, stride, + asid, tlb_level, page_shift); break; case TLBF_NOBROADCAST: /* Combination unused */ BUG(); break; case TLBF_NOWALKCACHE | TLBF_NOBROADCAST: - __flush_s1_tlb_range_op(vale1, start, pages, stride, - asid, tlb_level); + __flush_user_tlb_range_op(vale1, start, pages, stride, + asid, tlb_level, page_shift); break; } @@ -629,16 +673,18 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, * Set the tlb_level to TLBI_TTL_UNKNOWN because we can not get enough * information here. */ - __flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, TLBF_NONE); + __flush_tlb_range(vma, start, end, mm_page_size(vma->vm_mm), + TLBI_TTL_UNKNOWN, TLBF_NONE); } static inline void __flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr, tlbf_t flags) { - unsigned long start = round_down(uaddr, PAGE_SIZE); - unsigned long end = start + PAGE_SIZE; + unsigned long page_size = mm_page_size(vma->vm_mm); + unsigned long start = round_down(uaddr, page_size); + unsigned long end = start + page_size; - __do_flush_tlb_range(vma, start, end, PAGE_SIZE, 3, + __do_flush_tlb_range(vma, start, end, page_size, 3, TLBF_NOWALKCACHE | flags); } @@ -688,7 +734,7 @@ static inline void arch_tlbbatch_add_pending(struct arch_tlbflush_unmap_batch *b { struct vm_area_struct vma = { .vm_mm = mm, .vm_flags = 0 }; - __flush_tlb_range(&vma, start, end, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } diff --git a/arch/arm64/include/asm/user-pgalloc.h b/arch/arm64/include/asm/user-pgalloc.h new file mode 100644 index 00000000000000..3b4dbde0c77367 --- /dev/null +++ b/arch/arm64/include/asm/user-pgalloc.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGALLOC_H +#define __ASM_USER_PGALLOC_H + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +enum arm64_user4k_pt_level { + USER4K_PT_PTE, + USER4K_PT_PMD, + USER4K_PT_PUD, +}; + +/* Explicit geometry entry point; callers must keep each mm's granule fixed. */ +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift); +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level); +void arm64_user4k_pt_free(void *table); +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm); +void arm64_pgtable_free(void *table); +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table); + +static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PTE, mm_page_shift(mm)); + return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER); +} +#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__)) + +static inline void pte_free(struct mm_struct *mm, pgtable_t table) +{ + arm64_pgtable_free(table); +} + +static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PMD, mm_page_shift(mm)); + return __pmd_alloc_one_noprof(mm, addr); +} +#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__)) + +static inline void pmd_free(struct mm_struct *mm, pmd_t *table) +{ + arm64_pgtable_free(table); +} + +static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user_pt_alloc_granule(mm, USER4K_PT_PUD, mm_page_shift(mm)); + return __pud_alloc_one_noprof(mm, addr); +} +#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__)) + +#define pte_free_defer arm64_pte_free_defer + +#endif +#endif diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h new file mode 100644 index 00000000000000..b244f2387a7b47 --- /dev/null +++ b/arch/arm64/include/asm/user-pgtable.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PGTABLE_H +#define __ASM_USER_PGTABLE_H + +/* Included by asm/pgtable.h after the native accessors. */ +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + +/* Alternative tables use 48-bit VA; 64K folds the PUD level. */ +static inline unsigned int arm64_user_level_shift(const struct mm_struct *mm, + unsigned int level) +{ + unsigned int shift = mm_page_shift(mm); + + if (shift == 16 && !level) + level = 1; + return shift + (3 - level) * (shift - 3); +} + +static inline unsigned long arm64_user_table_index(const struct mm_struct *mm, + unsigned long addr, unsigned int level) +{ + unsigned int shift = arm64_user_level_shift(mm, level); + unsigned int bits = level ? mm_page_shift(mm) - 3 : 48 - shift; + + return (addr >> shift) & ((1UL << bits) - 1); +} + +#define p4d_none_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && !p4d_val(p4d)) : p4d_none(p4d)) +#define p4d_bad_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? (!mm_pud_folded(mm) && ((p4d_val(p4d) & P4D_TYPE_MASK) != P4D_TYPE_TABLE)) : p4d_bad(p4d)) +#define p4d_present_mm(mm, p4d) (!p4d_none_mm(mm, p4d)) +static inline void p4d_clear_mm(struct mm_struct *mm, p4d_t *p4d) +{ + if (arm64_mm_alt_granule(mm)) { + if (!mm_pud_folded(mm)) + set_p4d(p4d, __p4d(0)); + } else { + p4d_clear(p4d); + } +} +#define p4d_clear_mm p4d_clear_mm + + +/* pgtable_t preserves the fragment offset; pmd_page() remains its owner. */ +#define pmd_pgtable(pmd) ((pgtable_t)__va(pmd_val(pmd) & ARM64_USER4K_TABLE_ADDR_MASK)) + +static inline void *arm64_user4k_table(u64 descriptor) +{ + /* Unlike native __pXd_to_phys(), retain sub-native table address bits. */ + return __va(descriptor & ARM64_USER4K_TABLE_ADDR_MASK); +} + +static inline pgd_t *pgd_offset_pgd_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return pgd + arm64_user_table_index(mm, addr, 0); + return pgd + ((addr >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)); +} + +#define pgd_offset_pgd_mm pgd_offset_pgd_mm + +static inline pgd_t *pgd_offset_mm(struct mm_struct *mm, unsigned long addr) +{ + return pgd_offset_pgd_mm(mm, mm->pgd, addr); +} + +#define pgd_offset(mm, addr) pgd_offset_mm(mm, addr) + +static inline p4d_t *p4d_offset_mm(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (p4d_t *)pgd; + return p4d_offset(pgd, addr); +} + +#define p4d_offset_mm p4d_offset_mm + +static inline pud_t *pud_offset_mm(struct mm_struct *mm, p4d_t *p4d, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) { + if (mm_pud_folded(mm)) + return (pud_t *)p4d; + return (pud_t *)arm64_user4k_table(p4d_val(READ_ONCE(*p4d))) + + arm64_user_table_index(mm, addr, 1); + } + return pud_offset(p4d, addr); +} + +#define pud_offset_mm pud_offset_mm + +static inline pmd_t *pud_pgtable_mm(struct mm_struct *mm, pud_t pud) +{ + if (arm64_mm_alt_granule(mm)) + return arm64_user4k_table(pud_val(pud)); + return pud_pgtable(pud); +} + +#define pud_pgtable_mm pud_pgtable_mm + +static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pmd_t *)arm64_user4k_table(pud_val(READ_ONCE(*pud))) + + arm64_user_table_index(mm, addr, 2); + return (pmd_t *)pud_pgtable(READ_ONCE(*pud)) + + ((addr >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); +} + +#define pmd_offset_mm pmd_offset_mm + +static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr) +{ + if (arm64_mm_alt_granule(mm)) + return (pte_t *)arm64_user4k_table(pmd_val(READ_ONCE(*pmd))) + + arm64_user_table_index(mm, addr, 3); + return (pte_t *)pmd_page_vaddr(READ_ONCE(*pmd)) + + ((addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); +} + +#define pte_offset_kernel_mm pte_offset_kernel_mm +#define __pte_map_mm(mm, pmd, addr) pte_offset_kernel_mm(mm, pmd, addr) + +#define pmd_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 2) : PMD_SHIFT) +#define pud_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 1) : PUD_SHIFT) +#define p4d_shift_mm(mm) (arm64_mm_alt_granule(mm) ? arm64_user_level_shift(mm, 0) : P4D_SHIFT) +#define pmd_size_mm(mm) (1UL << pmd_shift_mm(mm)) +#define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) +#define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) +#define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) +#define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) +#define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) +#define pmd_mask_mm(mm) (~(pmd_size_mm(mm) - 1)) +#define pud_mask_mm(mm) (~(pud_size_mm(mm) - 1)) +#define p4d_mask_mm(mm) (~(p4d_size_mm(mm) - 1)) +#define pgd_mask_mm(mm) (~(pgd_size_mm(mm) - 1)) + +static inline unsigned long arm64_user4k_addr_end(unsigned long addr, + unsigned long end, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +/* Native fallbacks retain their architecture-specific folding semantics. */ +#define pgd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 0)) : pgd_addr_end(addr, end)) +#define p4d_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? (end) : p4d_addr_end(addr, end)) +#define pud_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 1)) : pud_addr_end(addr, end)) +#define pmd_addr_end_mm(mm, addr, end) \ + (arm64_mm_alt_granule(mm) ? arm64_user4k_addr_end(addr, end, arm64_user_level_shift(mm, 2)) : pmd_addr_end(addr, end)) + +#endif /* CONFIG_ARM64_USER4K_EXPERIMENTAL */ +#endif diff --git a/arch/arm64/include/asm/user-pte.h b/arch/arm64/include/asm/user-pte.h new file mode 100644 index 00000000000000..aa4c1e3a5aab96 --- /dev/null +++ b/arch/arm64/include/asm/user-pte.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_USER_PTE_H +#define __ASM_USER_PTE_H + +/* Included after native leaf accessors, before their batched setters. */ +#define ARM64_USER4K_TABLE_ADDR_MASK GENMASK_ULL(47, 12) + +static inline bool arm64_mm_user4k(const struct mm_struct *mm) +{ + return mm && arm64_context_user4k(&mm->context); +} + +static inline bool arm64_mm_alt_granule(const struct mm_struct *mm) +{ + return mm && arm64_context_page_shift(&mm->context) != PAGE_SHIFT; +} + +static inline u64 arm64_user_leaf_addr_mask(const struct mm_struct *mm) +{ + return ARM64_USER4K_TABLE_ADDR_MASK & mm_page_mask(mm); +} + +/* PFNs still identify native pages; leaf physical addresses identify slots. */ +static inline phys_addr_t pte_phys_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return pte_val(pte) & arm64_user_leaf_addr_mask(mm); + return __pte_to_phys(pte); +} +#define pte_phys_mm pte_phys_mm + +static inline pte_t phys_pte_mm(const struct mm_struct *mm, phys_addr_t phys, + pgprot_t prot) +{ + if (arm64_mm_alt_granule(mm)) { + VM_BUG_ON(phys & ~arm64_user_leaf_addr_mask(mm)); + return __pte(phys | pgprot_val(prot)); + } + return pfn_pte(phys >> PAGE_SHIFT, prot); +} +#define phys_pte_mm phys_pte_mm + +static inline pgprot_t pte_pgprot_mm(const struct mm_struct *mm, pte_t pte) +{ + if (arm64_mm_alt_granule(mm)) + return __pgprot(pte_val(pte) & ~arm64_user_leaf_addr_mask(mm)); + return pte_pgprot(pte); +} +#define pte_pgprot_mm pte_pgprot_mm + + +#ifdef CONFIG_MM_SUBPAGE +/* Slot bits belong to the PTE, not the native swap-cache index. */ +#define ARM64_SWP_SUBPAGE_MASK (GENMASK_ULL(5, 4) | GENMASK_ULL(63, 62)) + +static inline unsigned int pte_swp_subpage_offset(pte_t pte) +{ + return ((((pte_val(pte) >> 4) & 3) | + ((pte_val(pte) >> 60) & 12)) << 12); +} + +static inline pte_t pte_swp_set_subpage_offset(pte_t pte, unsigned int offset) +{ + VM_BUG_ON(!IS_ALIGNED(offset, SZ_4K) || offset >= PAGE_SIZE); + return __pte((pte_val(pte) & ~ARM64_SWP_SUBPAGE_MASK) | + (((u64)(offset >> 12) & 3) << 4) | + (((u64)(offset >> 12) & 12) << 60)); +} +#endif + +static inline pte_t pte_advance_pfn_mm(const struct mm_struct *mm, pte_t pte, + unsigned long nr) +{ + if (arm64_mm_alt_granule(mm)) + return phys_pte_mm(mm, pte_phys_mm(mm, pte) + (nr << mm_page_shift(mm)), + pte_pgprot_mm(mm, pte)); + return pte_advance_pfn(pte, nr); +} + +#define pte_advance_pfn_mm pte_advance_pfn_mm + +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +unsigned int arm64_ptep_page_shift(pte_t *ptep); +#else +static inline unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + return PAGE_SHIFT; +} +#endif + +static inline unsigned int arm64_cont_ptes(unsigned int shift) +{ + /* Architectural contiguous group lengths for 4K, 16K and 64K. */ + return shift == 12 ? 16 : shift == 14 ? 128 : 32; +} + +static inline unsigned int cont_ptes_mm(const struct mm_struct *mm) +{ + return arm64_cont_ptes(mm_page_shift(mm)); +} + +static inline unsigned long cont_pte_size_mm(const struct mm_struct *mm) +{ + return (unsigned long)cont_ptes_mm(mm) << mm_page_shift(mm); +} + +static inline unsigned long pte_cont_addr_end_mm(const struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + unsigned long size = cont_pte_size_mm(mm); + unsigned long boundary = (addr + size) & ~(size - 1); + + return boundary - 1 < end - 1 ? boundary : end; +} + +#endif diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 432d35b2714cd2..e30ef03e87b3ef 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1661,6 +1661,62 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) return cpufeature_matches(val, entry); } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Removal-only model for late-CPU verification in a disposable test VM. */ +static int user4k_deny_late_cpu __ro_after_init = -1; + +static int __init user4k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user4k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user4k_deny_late_cpu", user4k_deny_late_cpu_setup); +#ifdef CONFIG_ARM64_64K_PAGES +static int user16k_deny_late_cpu __ro_after_init = -1; + +static int __init user16k_deny_late_cpu_setup(char *str) +{ + int cpu; + + if (kstrtoint(str, 0, &cpu) || cpu <= 0 || cpu >= NR_CPUS) + return -EINVAL; + user16k_deny_late_cpu = cpu; + return 0; +} +early_param("arm64.user16k_deny_late_cpu", user16k_deny_late_cpu_setup); +#endif +#endif + +static bool has_user4k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user4k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool has_user16k_granule(const struct arm64_cpu_capabilities *entry, + int scope) +{ +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && + smp_processor_id() == user16k_deny_late_cpu) + return false; +#endif + return has_cpuid_feature(entry, scope); +} +#endif +#endif + const struct cpumask *system_32bit_el0_cpumask(void) { if (!system_supports_32bit_el0()) @@ -2391,8 +2447,11 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap) * clear the tags or set PG_mte_tagged. */ if (!cleared_zero_page) { + unsigned long offset; + cleared_zero_page = true; - mte_clear_page_tags(lm_alias(empty_zero_page)); + for (offset = 0; offset < sizeof(empty_zero_page); offset += PAGE_SIZE) + mte_clear_page_tags(lm_alias(empty_zero_page + offset)); } kasan_init_hw_tags_cpu(); @@ -2529,6 +2588,28 @@ test_has_gicv5_legacy(const struct arm64_cpu_capabilities *entry, int scope) } static const struct arm64_cpu_capabilities arm64_features[] = { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + { + .desc = "4K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER4K_GRANULE, + .matches = has_user4k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX) + }, +#ifdef CONFIG_ARM64_64K_PAGES + { + .desc = "16K userspace translation granule", + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .capability = ARM64_HAS_USER16K_GRANULE, + .matches = has_user16k_granule, + __ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN, + ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX) + }, +#endif +#endif { .capability = ARM64_ALWAYS_BOOT, .type = ARM64_CPUCAP_BOOT_CPU_FEATURE, diff --git a/arch/arm64/kernel/pi/idreg-override.c b/arch/arm64/kernel/pi/idreg-override.c index 1288766a68bf6c..622dc2034664cd 100644 --- a/arch/arm64/kernel/pi/idreg-override.c +++ b/arch/arm64/kernel/pi/idreg-override.c @@ -38,11 +38,32 @@ struct ftr_set_desc { #define FIELD(n, s, f) { .name = n, .shift = s, .width = 4, .filter = f } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) +/* VM capability-model test: only remove 4K support, never fabricate it. */ +static bool __init user4k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN4_NI; +} + +#ifdef CONFIG_ARM64_64K_PAGES +static bool __init user16k_granule_test_filter(u64 val) +{ + return val == ID_AA64MMFR0_EL1_TGRAN16_NI; +} +#endif +#endif + static const struct ftr_set_desc mmfr0 __prel64_initconst = { .name = "id_aa64mmfr0", .override = &id_aa64mmfr0_override, .fields = { FIELD("ecv", ID_AA64MMFR0_EL1_ECV_SHIFT, NULL), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && !defined(CONFIG_ARM64_4K_PAGES) + FIELD("tgran4", ID_AA64MMFR0_EL1_TGRAN4_SHIFT, user4k_granule_test_filter), +#ifdef CONFIG_ARM64_64K_PAGES + FIELD("tgran16", ID_AA64MMFR0_EL1_TGRAN16_SHIFT, user16k_granule_test_filter), +#endif +#endif {} }, }; @@ -59,11 +80,22 @@ static bool __init mmfr1_vh_filter(u64 val) val == 0); } +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) +/* VM-only coverage of deferred TTBR0 restore on CPUs which implement PAN. */ +static bool __init user4k_pan_test_filter(u64 val) +{ + return val == 0; +} +#endif + static const struct ftr_set_desc mmfr1 __prel64_initconst = { .name = "id_aa64mmfr1", .override = &id_aa64mmfr1_override, .fields = { FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter), +#if defined(CONFIG_ARM64_USER4K_KUNIT_TEST) && defined(CONFIG_ARM64_SW_TTBR0_PAN) + FIELD("pan", ID_AA64MMFR1_EL1_PAN_SHIFT, user4k_pan_test_filter), +#endif {} }, }; diff --git a/arch/arm64/kernel/pi/map_kernel.c b/arch/arm64/kernel/pi/map_kernel.c index a852264958c36b..c7dd0ba8d800e8 100644 --- a/arch/arm64/kernel/pi/map_kernel.c +++ b/arch/arm64/kernel/pi/map_kernel.c @@ -243,7 +243,7 @@ asmlinkage void __init early_map_kernel(u64 boot_status, phys_addr_t fdt) static char const chosen_str[] __initconst = "/chosen"; u64 va_base, pa_base = (u64)&_text; u64 kaslr_offset = pa_base % MIN_KIMG_ALIGN; - int root_level = 4 - CONFIG_PGTABLE_LEVELS; + int root_level = 4 - ARM64_NATIVE_PGTABLE_LEVELS; int va_bits = VA_BITS; int chosen; void *fdt_mapped = map_fdt(fdt); diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index ab1beffa8d7a44..a326135b164d4f 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -7,11 +7,13 @@ * Copyright (C) 2012 ARM Ltd. */ #include +#include #include #include #include #include #include +#include #include #include #include @@ -884,6 +886,33 @@ unsigned long arch_align_stack(unsigned long sp) return sp & ~0xf; } +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) +{ + unsigned int requested = current->exec_page_shift ?: + current->default_exec_page_shift; + unsigned int shift = arm64_exec_page_shift(compat, + cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE), PAGE_SHIFT, + requested); + + return bprm_set_page_shift(bprm, shift); +} + +/* Return zero for the native default, or a validated alternative shift. */ +int arch_exec_page_size_shift(unsigned long size) +{ + if (!size || size == PAGE_SIZE) + return 0; + if (size == SZ_4K) + return cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE) ? + 12 : -EOPNOTSUPP; + if (PAGE_SIZE > SZ_16K && size == SZ_16K) + return cpus_have_final_cap(ARM64_HAS_USER16K_GRANULE) ? + 14 : -EOPNOTSUPP; + return -EINVAL; +} +#endif + #ifdef CONFIG_COMPAT int compat_elf_check_arch(const struct elf32_hdr *hdr) { @@ -911,10 +940,10 @@ int compat_elf_check_arch(const struct elf32_hdr *hdr) */ void arch_setup_new_exec(void) { - unsigned long mmflags = 0; + unsigned long mmflags = current->mm->context.flags & MMCF_USER_GRANULE; if (is_compat_task()) { - mmflags = MMCF_AARCH32; + mmflags |= MMCF_AARCH32; /* * Restrict the CPU affinity mask for a 32-bit task so that diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index f08408b6e8263c..bdd2531554f270 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -22,6 +22,11 @@ SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off); +#endif + if (offset_in_page(off) != 0) return -EINVAL; diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 12a948f3a50437..ba66b65f506f4b 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -51,6 +52,11 @@ COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len, unsigned long, prot, unsigned long, flags, unsigned long, fd, unsigned long, off_4k) { +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return ksys_mmap_user(addr, len, prot, flags, fd, off_4k << 12); +#endif + if (off_4k & (~PAGE_MASK >> 12)) return -EINVAL; diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index e1ac876200a3d6..688b82c0cf30da 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -66,9 +66,10 @@ * We require that __hyp_bss_start and __bss_start are aligned, and enforce it * with an assertion. But the BSS_SECTION macro places an empty .sbss section * between them, which can in some cases cause the linker to misalign them. To - * work around the issue, force a page alignment for __bss_start. + * work around the issue, align __bss_start to the BSS section's alignment, + * which can exceed PAGE_SIZE for the mixed-granule zero page. */ -#define SBSS_ALIGN PAGE_SIZE +#define SBSS_ALIGN MAX(PAGE_SIZE, ALIGNOF(.bss)) #else /* CONFIG_KVM */ #define HYPERVISOR_EXTABLE #define HYPERVISOR_RODATA_SECTIONS diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 5018ac03b6bf39..c1ce370068cf44 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -80,6 +80,52 @@ SYM_FUNC_START(mte_copy_page_tags) ret SYM_FUNC_END(mte_copy_page_tags) +#ifdef CONFIG_MM_SUBPAGE +/* Copy one aligned power-of-two extent, preserving neighbouring tags. */ +SYM_FUNC_START(mte_copy_subpage_tags_range) + mov x6, x2 + add x2, x0, x2 + multitag_transfer_size x4, x5 + cmp x4, x6 + b.hi 2f +1: ldgm x3, [x1] + stgm x3, [x0] + add x0, x0, x4 + add x1, x1, x4 + cmp x0, x2 + b.ne 1b + ret +2: mov x3, xzr +3: ldg x3, [x1] + stg x3, [x0] + add x0, x0, #MTE_GRANULE_SIZE + add x1, x1, #MTE_GRANULE_SIZE + cmp x0, x2 + b.ne 3b + ret +SYM_FUNC_END(mte_copy_subpage_tags_range) + +/* Clear an aligned extent without changing data or neighbouring tags. */ +SYM_FUNC_START(mte_clear_subpage_tags_range) + mov x4, x1 + add x1, x0, x1 + multitag_transfer_size x2, x3 + cmp x2, x4 + b.hi 2f +1: stgm xzr, [x0] + add x0, x0, x2 + cmp x0, x1 + b.ne 1b + ret +2: mov x2, xzr +3: stg x2, [x0] + add x0, x0, #MTE_GRANULE_SIZE + cmp x0, x1 + b.ne 3b + ret +SYM_FUNC_END(mte_clear_subpage_tags_range) +#endif + /* * Read tags from a user buffer (one tag per byte) and set the corresponding * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c26489cf96cd90..32d18488e527c5 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n obj-$(CONFIG_KASAN) += kasan_init.o diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 0f4a28b874692d..cd3f56b9427083 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -38,6 +39,34 @@ static unsigned long *pinned_asid_map; #define ctxid2asid(asid) ((asid) & ~ASID_MASK) #define asid2ctxid(asid, genid) ((asid) | (genid)) +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST +/* Test-only capacity reduction; hardware ASID width and normal boots are unchanged. */ +static bool user4k_test_small_asids; + +static int __init user4k_test_asid_bits_setup(char *str) +{ + unsigned int bits; + + if (kstrtouint(str, 0, &bits) || bits != 8) + return -EINVAL; + user4k_test_small_asids = true; + return 0; +} +early_param("arm64.user4k_test_asid_bits", user4k_test_asid_bits_setup); + +/* Read-only evidence that an EL0 stress run actually crossed ASID rollover. */ +static int user4k_asid_generation_get(char *buffer, const struct kernel_param *kp) +{ + return sysfs_emit(buffer, "%llu\n", (unsigned long long)atomic64_read(&asid_generation)); +} + +static const struct kernel_param_ops user4k_asid_generation_ops = { + .get = user4k_asid_generation_get, +}; +module_param_cb(user4k_asid_generation, &user4k_asid_generation_ops, NULL, 0444); +MODULE_PARM_DESC(user4k_asid_generation, "Current ASID generation for mixed-granule VM tests"); +#endif + /* Get the ASIDBits supported by the current CPU */ static u32 get_cpu_asid_bits(void) { @@ -268,6 +297,13 @@ void check_and_switch_context(struct mm_struct *mm) */ if (!system_uses_ttbr0_pan()) cpu_switch_mm(mm->pgd, mm); +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + else { + /* uaccess/EL0 return restore only TTBRs, so select TG0 here. */ + cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); + } +#endif } unsigned long arm64_mm_context_get(struct mm_struct *mm) @@ -365,6 +401,7 @@ void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) ttbr1 |= FIELD_PREP(TTBRx_EL1_ASID_MASK, asid); cpu_set_reserved_ttbr0_nosync(); + cpu_set_mm_tg0(mm); write_sysreg(ttbr1, ttbr1_el1); write_sysreg(ttbr0, ttbr0_el1); isb(); @@ -401,6 +438,12 @@ arch_initcall(asids_update_limit); static int asids_init(void) { asid_bits = get_cpu_asid_bits(); +#ifdef CONFIG_ARM64_USER4K_KUNIT_TEST + if (user4k_test_small_asids) { + pr_info("user4k test: limiting ASID allocator to 8 bits\n"); + asid_bits = 8; + } +#endif atomic64_set(&asid_generation, ASID_FIRST_VERSION); asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); if (!asid_map) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 2de12656b4d8ba..f8faf1a99eeca7 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -8,6 +8,25 @@ #include #include +static inline unsigned int contpte_nr(pte_t *ptep) +{ + return arm64_cont_ptes(arm64_ptep_page_shift(ptep)); +} + +static inline phys_addr_t contpte_phys(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return pte_val(pte) & (ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1)); + return __pte_to_phys(pte); +} + +static inline pgprot_t contpte_prot(pte_t pte, unsigned int shift) +{ + if (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) && shift != PAGE_SHIFT) + return __pgprot(pte_val(pte) & ~(ARM64_USER4K_TABLE_ADDR_MASK & ~((1ULL << shift) - 1))); + return pte_pgprot(pte); +} + static inline bool mm_is_user(struct mm_struct *mm) { /* @@ -23,10 +42,10 @@ static inline bool mm_is_user(struct mm_struct *mm) static inline pte_t *contpte_align_down(pte_t *ptep) { - return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * CONT_PTES); + return PTR_ALIGN_DOWN(ptep, sizeof(*ptep) * contpte_nr(ptep)); } -static inline pte_t *contpte_align_addr_ptep(unsigned long *start, +static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long *start, unsigned long *end, pte_t *ptep, unsigned int nr) { @@ -36,10 +55,10 @@ static inline pte_t *contpte_align_addr_ptep(unsigned long *start, * single VMA and a single page table. */ if (pte_cont(__ptep_get(ptep + nr - 1))) - *end = ALIGN(*end, CONT_PTE_SIZE); + *end = ALIGN(*end, cont_pte_size_mm(mm)); if (pte_cont(__ptep_get(ptep))) { - *start = ALIGN_DOWN(*start, CONT_PTE_SIZE); + *start = ALIGN_DOWN(*start, cont_pte_size_mm(mm)); ptep = contpte_align_down(ptep); } @@ -54,11 +73,11 @@ static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < CONT_PTES) + if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); if (ptep + nr != contpte_align_down(ptep + nr)) { - unsigned long last_addr = addr + PAGE_SIZE * (nr - 1); + unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; contpte_try_unfold(mm, last_addr, last_ptep, @@ -75,10 +94,11 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, int i; start_ptep = ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); - pte = pfn_pte(ALIGN_DOWN(pte_pfn(pte), CONT_PTES), pte_pgprot(pte)); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + pte = phys_pte_mm(mm, ALIGN_DOWN(pte_phys_mm(mm, pte), cont_pte_size_mm(mm)), + pte_pgprot_mm(mm, pte)); - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) { + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) { pte_t ptent = __ptep_get_and_clear(mm, addr, ptep); if (pte_dirty(ptent)) @@ -225,10 +245,10 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, */ if (!system_supports_bbml2_noabort()) - __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, + __flush_tlb_range(&vma, start_addr, addr, mm_page_size(mm), 3, TLBF_NOWALKCACHE); - __set_ptes(mm, start_addr, start_ptep, pte, CONT_PTES); + __set_ptes(mm, start_addr, start_ptep, pte, cont_ptes_mm(mm)); } void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, @@ -269,24 +289,26 @@ void __contpte_try_fold(struct mm_struct *mm, unsigned long addr, page = pte_page(pte); folio = page_folio(page); folio_start = addr - (page - &folio->page) * PAGE_SIZE; + if (arm64_mm_alt_granule(mm)) + folio_start -= pte_phys_mm(mm, pte) & ~PAGE_MASK; folio_end = folio_start + folio_nr_pages(folio) * PAGE_SIZE; - cont_start = ALIGN_DOWN(addr, CONT_PTE_SIZE); - cont_end = cont_start + CONT_PTE_SIZE; + cont_start = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); + cont_end = cont_start + cont_pte_size_mm(mm); if (folio_start > cont_start || folio_end < cont_end) return; - pfn = ALIGN_DOWN(pte_pfn(pte), CONT_PTES); - prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); - expected_pte = pfn_pte(pfn, prot); + pfn = ALIGN_DOWN(pte_phys_mm(mm, pte) >> mm_page_shift(mm), cont_ptes_mm(mm)); + prot = pte_pgprot_mm(mm, pte_mkold(pte_mkclean(pte))); + expected_pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); orig_ptep = ptep; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { subpte = pte_mkold(pte_mkclean(__ptep_get(ptep))); if (!pte_same(subpte, expected_pte)) return; - expected_pte = pte_advance_pfn(expected_pte, 1); + expected_pte = pte_advance_pfn_mm(mm, expected_pte, 1); ptep++; } @@ -319,17 +341,18 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) * feet. */ + unsigned int nr = contpte_nr(ptep); pte_t pte; int i; ptep = contpte_align_down(ptep); - for (i = 0; i < CONT_PTES; i++, ptep++) { + for (i = 0; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_young(pte)) { orig_pte = pte_mkyoung(orig_pte); @@ -343,7 +366,7 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) orig_pte = pte_mkyoung(orig_pte); i++; ptep++; - for (; i < CONT_PTES; i++, ptep++) { + for (; i < nr; i++, ptep++) { pte = __ptep_get(ptep); if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); @@ -359,11 +382,12 @@ pte_t contpte_ptep_get(pte_t *ptep, pte_t orig_pte) EXPORT_SYMBOL_GPL(contpte_ptep_get); static inline bool contpte_is_consistent(pte_t pte, unsigned long pfn, + unsigned int shift, pgprot_t orig_prot) { - pgprot_t prot = pte_pgprot(pte_mkold(pte_mkclean(pte))); + pgprot_t prot = contpte_prot(pte_mkold(pte_mkclean(pte)), shift); - return pte_valid_cont(pte) && pte_pfn(pte) == pfn && + return pte_valid_cont(pte) && (contpte_phys(pte, shift) >> shift) == pfn && pgprot_val(prot) == pgprot_val(orig_prot); } @@ -386,6 +410,8 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) * because it is not part of a contpte range. */ + unsigned int shift = arm64_ptep_page_shift(orig_ptep); + unsigned int nr = arm64_cont_ptes(shift); pgprot_t orig_prot; unsigned long pfn; pte_t orig_pte; @@ -399,22 +425,22 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) if (!pte_valid_cont(orig_pte)) return orig_pte; - orig_prot = pte_pgprot(pte_mkold(pte_mkclean(orig_pte))); + orig_prot = contpte_prot(pte_mkold(pte_mkclean(orig_pte)), shift); ptep = contpte_align_down(orig_ptep); - pfn = pte_pfn(orig_pte) - (orig_ptep - ptep); + pfn = (contpte_phys(orig_pte, shift) >> shift) - (orig_ptep - ptep); - for (i = 0; i < CONT_PTES; i++, ptep++, pfn++) { + for (i = 0; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { orig_pte = pte_mkdirty(orig_pte); - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_young(pte)) { @@ -430,10 +456,10 @@ pte_t contpte_ptep_get_lockless(pte_t *orig_ptep) i++; ptep++; pfn++; - for (; i < CONT_PTES; i++, ptep++, pfn++) { + for (; i < nr; i++, ptep++, pfn++) { pte = __ptep_get(ptep); - if (!contpte_is_consistent(pte, pfn, orig_prot)) + if (!contpte_is_consistent(pte, pfn, shift, orig_prot)) goto retry; if (pte_dirty(pte)) { @@ -468,16 +494,16 @@ void contpte_set_ptes(struct mm_struct *mm, unsigned long addr, if (!mm_is_user(mm)) return __set_ptes(mm, addr, ptep, pte, nr); - end = addr + (nr << PAGE_SHIFT); - pfn = pte_pfn(pte); - prot = pte_pgprot(pte); + end = addr + (nr << mm_page_shift(mm)); + pfn = pte_phys_mm(mm, pte) >> mm_page_shift(mm); + prot = pte_pgprot_mm(mm, pte); do { - next = pte_cont_addr_end(addr, end); - nr = (next - addr) >> PAGE_SHIFT; - pte = pfn_pte(pfn, prot); + next = pte_cont_addr_end_mm(mm, addr, end); + nr = (next - addr) >> mm_page_shift(mm); + pte = phys_pte_mm(mm, pfn << mm_page_shift(mm), prot); - if (((addr | next | (pfn << PAGE_SHIFT)) & ~CONT_PTE_MASK) == 0) + if (((addr | next | (pfn << mm_page_shift(mm))) & (cont_pte_size_mm(mm) - 1)) == 0) pte = pte_mkcont(pte); else pte = pte_mknoncont(pte); @@ -512,6 +538,8 @@ EXPORT_SYMBOL_GPL(contpte_get_and_clear_full_ptes); bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + /* * ptep_clear_flush_young() technically requires us to clear the access * flag for a _single_ pte. However, the core-mm code actually tracks @@ -524,11 +552,11 @@ bool contpte_test_and_clear_young_ptes(struct vm_area_struct *vma, * of the same large folio in a single VMA and a single page table. */ - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); bool young = false; - ptep = contpte_align_addr_ptep(&addr, &end, ptep, nr); - for (; addr != end; ptep++, addr += PAGE_SIZE) + ptep = contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); + for (; addr != end; ptep++, addr += mm_page_size(mm)) young |= __ptep_test_and_clear_young(vma, addr, ptep); return young; @@ -538,19 +566,21 @@ EXPORT_SYMBOL_GPL(contpte_test_and_clear_young_ptes); bool contpte_clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { + struct mm_struct *mm = vma->vm_mm; + bool young; young = contpte_test_and_clear_young_ptes(vma, addr, ptep, nr); if (young) { - unsigned long end = addr + nr * PAGE_SIZE; + unsigned long end = addr + nr * mm_page_size(mm); - contpte_align_addr_ptep(&addr, &end, ptep, nr); + contpte_align_addr_ptep(vma->vm_mm, &addr, &end, ptep, nr); /* * See comment in __ptep_clear_flush_young(); same rationale for * eliding the trailing DSB applies here. */ - __flush_tlb_range(vma, addr, end, PAGE_SIZE, 3, + __flush_tlb_range(vma, addr, end, mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOSYNC); } @@ -580,6 +610,8 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr, cydp_t flags) { + struct mm_struct *mm = vma->vm_mm; + /* * We can safely clear access/dirty without needing to unfold from * the architectures perspective, even when contpte is set. If the @@ -591,14 +623,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * clearing access/dirty for the whole block. */ unsigned long start = addr; - unsigned long end = start + nr * PAGE_SIZE; + unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(&start, &end, ptep, nr); - __clear_young_dirty_ptes(vma, start, ptep, (end - start) / PAGE_SIZE, flags); + ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); + __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } EXPORT_SYMBOL_GPL(contpte_clear_young_dirty_ptes); -static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) +static bool contpte_all_subptes_match_access_flags(struct mm_struct *mm, + pte_t *ptep, pte_t entry) { pte_t *cont_ptep = contpte_align_down(ptep); /* @@ -609,7 +642,7 @@ static bool contpte_all_subptes_match_access_flags(pte_t *ptep, pte_t entry) pteval_t entry_cmp = pte_val(entry) & cmp_mask; int i; - for (i = 0; i < CONT_PTES; i++) { + for (i = 0; i < cont_ptes_mm(mm); i++) { pteval_t pte_cmp = pte_val(__ptep_get(cont_ptep + i)) & cmp_mask; if (pte_cmp != entry_cmp) @@ -623,6 +656,8 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t entry, int dirty) { + struct mm_struct *mm = vma->vm_mm; + unsigned long start_addr; pte_t orig_pte; int i; @@ -651,7 +686,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * no-op, and when any sub-PTE mismatches, proceed to update the whole * range. */ - if (contpte_all_subptes_match_access_flags(ptep, entry)) + if (contpte_all_subptes_match_access_flags(mm, ptep, entry)) return 0; /* @@ -672,7 +707,7 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * and instead flush the whole range at the end. */ ptep = contpte_align_down(ptep); - start_addr = addr = ALIGN_DOWN(addr, CONT_PTE_SIZE); + start_addr = addr = ALIGN_DOWN(addr, cont_pte_size_mm(mm)); /* * We are not advancing entry because __ptep_set_access_flags() @@ -680,13 +715,13 @@ int contpte_ptep_set_access_flags(struct vm_area_struct *vma, * for the whole contpte block and returned early, pte_same() * within __ptep_set_access_flags() is likely false. */ - for (i = 0; i < CONT_PTES; i++, ptep++, addr += PAGE_SIZE) + for (i = 0; i < cont_ptes_mm(mm); i++, ptep++, addr += mm_page_size(mm)) __ptep_set_access_flags(vma, addr, ptep, entry, 0); if (dirty) __flush_tlb_range(vma, start_addr, - start_addr + CONT_PTE_SIZE, - PAGE_SIZE, 3, + start_addr + cont_pte_size_mm(mm), + mm_page_size(mm), 3, TLBF_NOWALKCACHE | TLBF_NOBROADCAST); } else { __contpte_try_unfold(vma->vm_mm, addr, ptep, orig_pte); diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c index cd5912ba617b70..b81ccf42b22d19 100644 --- a/arch/arm64/mm/copypage.c +++ b/arch/arm64/mm/copypage.c @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -72,3 +73,54 @@ void copy_user_highpage(struct page *to, struct page *from, flush_dcache_page(to); } EXPORT_SYMBOL_GPL(copy_user_highpage); + +#ifdef CONFIG_MM_SUBPAGE +void clear_user_subpage_range(struct page *page, unsigned int offset, unsigned int size) +{ + void *addr = page_address(page) + offset; + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(offset, size) || offset > PAGE_SIZE - size); + memset(addr, 0, size); + if (system_supports_mte() && page_mte_tagged(page)) + mte_clear_subpage_tags_range(addr, size); + flush_dcache_page(page); +} + +int copy_user_subpage_range(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset, unsigned int size) +{ + void *kto = page_address(to), *kfrom = page_address(from); + + VM_BUG_ON(!is_power_of_2(size) || size < SZ_4K || size > PAGE_SIZE); + VM_BUG_ON(!IS_ALIGNED(to_offset, size) || to_offset > PAGE_SIZE - size); + VM_BUG_ON(!IS_ALIGNED(from_offset, size) || from_offset > PAGE_SIZE - size); + if (PageHWPoison(from) || copy_mc_to_kernel(kto + to_offset, kfrom + from_offset, size)) + return -EHWPOISON; + if (kasan_hw_tags_enabled()) + page_kasan_tag_reset(to); + if (system_supports_mte()) { + if (page_mte_tagged(from)) { + if (try_page_mte_tagging(to)) + mte_clear_page_tags(kto); + mte_copy_subpage_tags_range(kto + to_offset, kfrom + from_offset, size); + set_page_mte_tagged(to); + } else if (page_mte_tagged(to)) { + mte_clear_subpage_tags_range(kto + to_offset, size); + } + } + flush_dcache_page(to); + return 0; +} +/* Existing 4K fault and ownership callers keep their fixed-size contract. */ +void clear_user_subpage(struct page *page, unsigned int offset) +{ + clear_user_subpage_range(page, offset, SZ_4K); +} + +int copy_user_subpage(struct page *to, unsigned int to_offset, + struct page *from, unsigned int from_offset) +{ + return copy_user_subpage_range(to, to_offset, from, from_offset, SZ_4K); +} +#endif diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 739800835920c1..6a37063417c4ce 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -150,7 +150,7 @@ static void show_pte(unsigned long addr) } pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n", - mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, + mm == &init_mm ? "swapper" : "user", mm_page_size(mm) / SZ_1K, vabits_actual, mm_to_pgd_phys(mm)); pgdp = pgd_offset(mm, addr); pgd = READ_ONCE(*pgdp); @@ -165,25 +165,25 @@ static void show_pte(unsigned long addr) if (pgd_none(pgd) || pgd_bad(pgd)) break; - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_mm(mm, pgdp, addr); p4d = READ_ONCE(*p4dp); pr_cont(", p4d=%016llx", p4d_val(p4d)); - if (p4d_none(p4d) || p4d_bad(p4d)) + if (p4d_none_mm(mm, p4d) || p4d_bad_mm(mm, p4d)) break; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_mm(mm, p4dp, addr); pud = READ_ONCE(*pudp); pr_cont(", pud=%016llx", pud_val(pud)); if (pud_none(pud) || pud_bad(pud)) break; - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_mm(mm, pudp, addr); pmd = READ_ONCE(*pmdp); pr_cont(", pmd=%016llx", pmd_val(pmd)); if (pmd_none(pmd) || pmd_bad(pmd)) break; - ptep = pte_offset_map(pmdp, addr); + ptep = pte_offset_map_mm(mm, pmdp, addr); if (!ptep) break; @@ -241,19 +241,15 @@ int __ptep_set_access_flags_anysz(struct vm_area_struct *vma, * flush_tlb_fix_spurious_fault(). */ if (dirty) { - switch (pgsize) { - case PAGE_SIZE: + if (pgsize == mm_page_size(vma->vm_mm)) { level = 3; - break; - case PMD_SIZE: + } else if (pgsize == pmd_size_mm(vma->vm_mm)) { level = 2; - break; #ifndef __PAGETABLE_PMD_FOLDED - case PUD_SIZE: + } else if (pgsize == pud_size_mm(vma->vm_mm)) { level = 1; - break; #endif - default: + } else { level = TLBI_TTL_UNKNOWN; WARN_ON(1); } @@ -1002,8 +998,8 @@ NOKPROBE_SYMBOL(do_sp_pc_abort); /* * Used during anonymous page fault handling. */ -struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, - unsigned long vaddr) +struct folio *vma_alloc_zeroed_movable_folio_order(struct vm_area_struct *vma, + unsigned long vaddr, unsigned int order) { gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; @@ -1015,7 +1011,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma, if (vma->vm_flags & VM_MTE) flags |= __GFP_ZEROTAGS; - return vma_alloc_folio(flags, 0, vma, vaddr); + return vma_alloc_folio(flags, order, vma, vaddr); } bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages) diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c index c5c5425791da72..b670d27a898cc3 100644 --- a/arch/arm64/mm/fixmap.c +++ b/arch/arm64/mm/fixmap.c @@ -77,7 +77,7 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr, p4d_t p4d = READ_ONCE(*p4dp); pud_t *pudp; - if (CONFIG_PGTABLE_LEVELS > 3 && !p4d_none(p4d) && + if (pgtable_l4_enabled() && !p4d_none(p4d) && p4d_page_paddr(p4d) != __pa_symbol(bm_pud)) { /* * We only end up here if the kernel mapping and the fixmap diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 97987f850a33c3..1e5f314b542d5c 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -368,8 +368,9 @@ void __init arch_mm_preinit(void) * Selected page table levels should match when derived from * scratch using the virtual address range and page size. */ - BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != - CONFIG_PGTABLE_LEVELS); + BUILD_BUG_ON(CONFIG_PGTABLE_LEVELS != + (IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) ? 4 : + ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS))); if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { extern int sysctl_overcommit_memory; diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 09fd9da4eeed41..976d115d72f3f2 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -2253,8 +2253,8 @@ pte_t modify_prot_start_ptes(struct vm_area_struct *vma, unsigned long addr, * in cases where cpu is affected with errata #2645198. */ if (pte_accessible(vma->vm_mm, pte) && pte_user_exec(pte)) - __flush_tlb_range(vma, addr, nr * PAGE_SIZE, - PAGE_SIZE, 3, TLBF_NOWALKCACHE); + __flush_tlb_range(vma, addr, addr + nr * mm_page_size(vma->vm_mm), + mm_page_size(vma->vm_mm), 3, TLBF_NOWALKCACHE); } return pte; diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index bf5110b91e2faa..3b361b95322e1b 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -16,14 +16,19 @@ #include static struct kmem_cache *pgd_cache __ro_after_init; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +static struct kmem_cache *user4k_pgd_cache __ro_after_init; +static struct kmem_cache *user16k_pgd_cache __ro_after_init; +static struct kmem_cache *user64k_pgd_cache __ro_after_init; +#endif static bool pgdir_is_page_size(void) { if (PGD_SIZE == PAGE_SIZE) return true; - if (CONFIG_PGTABLE_LEVELS == 4) + if (ARM64_NATIVE_PGTABLE_LEVELS == 4) return !pgtable_l4_enabled(); - if (CONFIG_PGTABLE_LEVELS == 5) + if (ARM64_NATIVE_PGTABLE_LEVELS == 5) return !pgtable_l5_enabled(); return false; } @@ -32,6 +37,14 @@ pgd_t *pgd_alloc(struct mm_struct *mm) { gfp_t gfp = GFP_PGTABLE_USER; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) + return kmem_cache_alloc(user4k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) + return kmem_cache_alloc(user16k_pgd_cache, gfp); + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) + return kmem_cache_alloc(user64k_pgd_cache, gfp); +#endif if (pgdir_is_page_size()) return __pgd_alloc(mm, 0); else @@ -40,6 +53,20 @@ pgd_t *pgd_alloc(struct mm_struct *mm) void pgd_free(struct mm_struct *mm, pgd_t *pgd) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (arm64_context_user4k(&mm->context)) { + kmem_cache_free(user4k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 14) { + kmem_cache_free(user16k_pgd_cache, pgd); + return; + } + if (arm64_mm_alt_granule(mm) && mm_page_shift(mm) == 16) { + kmem_cache_free(user64k_pgd_cache, pgd); + return; + } +#endif if (pgdir_is_page_size()) __pgd_free(mm, pgd); else @@ -48,6 +75,16 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd) void __init pgtable_cache_init(void) { +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (PAGE_SHIFT < 16) + user64k_pgd_cache = kmem_cache_create("user64k_pgd_cache", SZ_64K, SZ_64K, + SLAB_PANIC, NULL); + if (PAGE_SHIFT != 14) + user16k_pgd_cache = kmem_cache_create("user16k_pgd_cache", SZ_16K, SZ_16K, + SLAB_PANIC, NULL); + user4k_pgd_cache = kmem_cache_create("user4k_pgd_cache", SZ_4K, SZ_4K, + SLAB_PANIC, NULL); +#endif if (pgdir_is_page_size()) return; diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be3720..7f947589a3b4d7 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -291,7 +291,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) cbnz cpu, __idmap_kpti_secondary -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 stp x29, x30, [sp, #-32]! mov x29, sp stp x19, x20, [sp, #16] @@ -352,7 +352,7 @@ alternative_else_nop_endif /* Set the flag to zero to indicate that we're all done */ str wzr, [flag_ptr] -#if CONFIG_PGTABLE_LEVELS > 4 +#if ARM64_NATIVE_PGTABLE_LEVELS > 4 ldp x19, x20, [sp, #16] ldp x29, x30, [sp], #32 #endif @@ -360,39 +360,39 @@ alternative_else_nop_endif .Lderef_pgd: /* P4D */ - .if CONFIG_PGTABLE_LEVELS > 4 + .if ARM64_NATIVE_PGTABLE_LEVELS > 4 p4d .req x30 pte_to_phys cur_p4dp, pgd kpti_map_pgtbl p4d, 0 kpti_mk_tbl_ng p4d, PTRS_PER_P4D b .Lnext_pgd - .else /* CONFIG_PGTABLE_LEVELS <= 4 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 4 */ p4d .req pgd .set .Lnext_p4d, .Lnext_pgd .endif .Lderef_p4d: /* PUD */ - .if CONFIG_PGTABLE_LEVELS > 3 + .if ARM64_NATIVE_PGTABLE_LEVELS > 3 pud .req x10 pte_to_phys cur_pudp, p4d kpti_map_pgtbl pud, 1 kpti_mk_tbl_ng pud, PTRS_PER_PUD b .Lnext_p4d - .else /* CONFIG_PGTABLE_LEVELS <= 3 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 3 */ pud .req pgd .set .Lnext_pud, .Lnext_pgd .endif .Lderef_pud: /* PMD */ - .if CONFIG_PGTABLE_LEVELS > 2 + .if ARM64_NATIVE_PGTABLE_LEVELS > 2 pmd .req x13 pte_to_phys cur_pmdp, pud kpti_map_pgtbl pmd, 2 kpti_mk_tbl_ng pmd, PTRS_PER_PMD b .Lnext_pud - .else /* CONFIG_PGTABLE_LEVELS <= 2 */ + .else /* ARM64_NATIVE_PGTABLE_LEVELS <= 2 */ pmd .req pgd .set .Lnext_pmd, .Lnext_pgd .endif diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c new file mode 100644 index 00000000000000..0bcaab5c3a8d28 --- /dev/null +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Hardware tables have immutable granule/owner metadata. Smaller tables share + * a native page; larger tables own a compound allocation. Only unissued slots + * in a shared native page are cached by the mm. + */ +#include +#include +#include +#include +#include + +unsigned int arm64_ptep_page_shift(pte_t *ptep) +{ + struct ptdesc *pt = virt_to_ptdesc(ptep); + + /* Table owners retain geometry through their last RCU reader. */ + return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; +} +EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); + +static void fragment_owner_free(struct ptdesc *pt) +{ + clear_bit(PT_fragmented, &pt->pt_flags.f); + pagetable_dtor_free(pt); +} + +static void fragment_owner_free_rcu(struct rcu_head *rcu) +{ + fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); +} + +static void fragment_put(struct ptdesc *pt, unsigned int nr) +{ + if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) + return; + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + void *table, *next; + bool constructed; + unsigned int frag_size, nr_fragments, order; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + frag_size = 1U << shift; + order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; + nr_fragments = (PAGE_SIZE << order) >> shift; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + + spin_lock(&mm->page_table_lock); + table = mm->context.user4k_pt_frag[level]; + if (table) { + /* An mm's geometry is immutable; never consume another granule. */ + if (virt_to_ptdesc(table)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + return NULL; + } + next = table + frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } + spin_unlock(&mm->page_table_lock); + if (table) + return table; + + pt = pagetable_alloc(GFP_PGTABLE_USER, order); + if (!pt) + return NULL; + switch (level) { + case USER4K_PT_PTE: + constructed = pagetable_pte_ctor(mm, pt); + break; + case USER4K_PT_PMD: + constructed = pagetable_pmd_ctor(mm, pt); + break; + case USER4K_PT_PUD: + pagetable_pud_ctor(pt); + constructed = true; + break; + default: + constructed = false; + } + if (!constructed) { + pagetable_free(pt); + return NULL; + } + pt->pt_frag_shift = shift; + set_bit(PT_fragmented, &pt->pt_flags.f); + atomic_set(&pt->pt_frag_refcount, 1); + table = ptdesc_address(pt); + spin_lock(&mm->page_table_lock); + if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } + /* A racing allocation may own the cache; this page then has one user. */ + spin_unlock(&mm->page_table_lock); + return table; +} + +void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level level) +{ + return arm64_user_pt_alloc_granule(mm, level, 12); +} + +void arm64_user4k_pt_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); + VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); + VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); + fragment_put(pt, 1); +} + +void arm64_pgtable_free(void *table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) + arm64_user4k_pt_free(table); + else { + VM_BUG_ON(!PAGE_ALIGNED(table)); + pagetable_dtor_free(pt); + } +} + +void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) +{ + unsigned int level; + + /* The mm is private or dying; no allocator may still access its cache. */ + for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { + void *next = mm->context.user4k_pt_frag[level]; + struct ptdesc *pt; + unsigned int issued, shift; + + if (!next) + continue; + mm->context.user4k_pt_frag[level] = NULL; + pt = virt_to_ptdesc(next); + shift = pt->pt_frag_shift; + issued = ((unsigned long)next & ~PAGE_MASK) >> shift; + fragment_put(pt, (PAGE_SIZE >> shift) - issued); + } +} + +void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) +{ + struct ptdesc *pt = virt_to_ptdesc(table); + + if (test_bit(PT_fragmented, &pt->pt_flags.f)) { + /* No freed slot is reused. Delay the final owner release for RCU. */ + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } else { + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + } +} + +void __tlb_remove_table(void *token) +{ + if ((unsigned long)token & 1) + arm64_user4k_pt_free((void *)((unsigned long)token & ~1UL)); + else + pagetable_dtor_free(token); +} diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index de92d63fdbc763..432df2ce9ddfd4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -69,6 +69,8 @@ HAS_TIDCP1 HAS_TLB_RANGE HAS_TSO_APPLE HAS_TSO_FIXED +HAS_USER4K_GRANULE +HAS_USER16K_GRANULE HAS_VA52 HAS_VIRT_HOST_EXTN HAS_WFXT From 64fbc0965210c003bc3065ce9f699925dadf639e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:22 -0400 Subject: [PATCH 023/631] kunit: cover mixed-page ownership and translation lifetimes Add internal MM, table, subpage and recorded VFIO tests with separate opt-in controls. Translation fixtures require a disposable guest and recorded device mappings do not establish physical DMA qualification. Signed-off-by: Diljit Singh (cherry picked from commit e86efe2f2aaec4012ba91165f5211ca8c9c7716c) --- arch/arm64/Kconfig.debug | 70 + arch/arm64/mm/Makefile | 2 + arch/arm64/mm/user4k-data-test.c | 1835 ++ arch/arm64/mm/user4k-test.c | 14237 ++++++++++++++++ arch/arm64/mm/user4k-tlb-test.c | 184 + drivers/vfio/Kconfig | 12 + drivers/vfio/vfio_iommu_type1_account_test.c | 192 + .../vfio/vfio_iommu_type1_fragments_test.c | 1177 ++ drivers/vfio/vfio_iommu_type1_ioctl_test.c | 305 + drivers/vfio/vfio_iommu_type1_test.h | 36 + 10 files changed, 18050 insertions(+) create mode 100644 arch/arm64/mm/user4k-data-test.c create mode 100644 arch/arm64/mm/user4k-test.c create mode 100644 arch/arm64/mm/user4k-tlb-test.c create mode 100644 drivers/vfio/vfio_iommu_type1_account_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_fragments_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_ioctl_test.c create mode 100644 drivers/vfio/vfio_iommu_type1_test.h diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index d15ccdeee21250..58d7a1dc98405b 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -42,3 +42,73 @@ config ARM64_USER4K_EXPERIMENTAL still require integration and validation. The private prctl numbers are not upstream assignments or a stable ABI promise. Say N for normal kernels. + +config ARM64_USER4K_KUNIT_TEST + bool "Test mixed userspace granules with larger kernel pages" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + depends on ARM64_PA_BITS_48 + select ARM64_USER4K_DATA_KUNIT_TEST + select IOMMUFD + select IOMMUFD_DRIVER + default n + help + Exercise independent TTBR0 translation granules, subpage permissions + and invalidation while TTBR1 retains the normal native kernel tables. + Hardware translation/uaccess fixtures require arm64.user4k_test=1 + as a second opt-in and are disabled with KASAN. Software MM fixtures + can run with KASAN without borrowing or replacing TTBR0. + Run only in a disposable test VM. This regression harness does not + establish hardware qualification or production ABI readiness. + Say N for normal kernels. + +config ARM64_USER4K_IOMMUFD_TEST + bool "IOMMUFD ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL + depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + select GENERIC_PT + select IOMMU_PT + select IOMMU_PT_AMDV1 + select IOMMUFD + select IOMMUFD_TEST + default n + help + Enable the IOMMUFD mock-device ioctls and generic AMDv1 page tables + for the mixed-granule userspace ioctl fixture. This exposes privileged + test interfaces; use only in a disposable test VM. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_ACCOUNT_TEST + bool "VFIO type1 pinning harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + select VFIO + select VFIO_GROUP + select VFIO_CONTAINER + select VFIO_IOMMU_TYPE1 + select VFIO_TYPE1_ACCOUNT_KUNIT_TEST + default n + help + Enable built-in VFIO type1 accounting and retained-fragment tests + in a disposable mixed-granule guest. Recorded IOMMU translations + do not establish physical-device DMA support. + Say N for normal kernels. + +config ARM64_USER4K_VFIO_IOCTL_TEST + bool "Private VFIO type1 ioctl harness for mixed userspace granules" + depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL + depends on RUNTIME_TESTING_MENU + select ARM64_USER4K_VFIO_ACCOUNT_TEST + default n + help + Expose a root-only disposable-guest misc device forwarding real EL0 + arguments to type1 ioctls and supplying recording IOMMU domains. + These privileged test controls perform no physical device DMA. + Say N for normal kernels. + +config ARM64_USER4K_DATA_KUNIT_TEST + bool "Test packed userspace allocation ownership" + depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL + default n + help + Exercise packed allocation, slot reuse, copying and deferred lifetime. + These tests use native kernel addresses without changing TTBR0, and + can run with KASAN even when the architectural tests are disabled. diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 32d18488e527c5..757f7eb07ccac6 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -12,6 +12,8 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_ARM64_MTE) += mteswap.o obj-$(CONFIG_ARM64_GCS) += gcs.o +obj-$(CONFIG_ARM64_USER4K_KUNIT_TEST) += user4k-test.o user4k-tlb-test.o +obj-$(CONFIG_ARM64_USER4K_DATA_KUNIT_TEST) += user4k-data-test.o obj-$(CONFIG_ARM64_USER4K_EXPERIMENTAL) += user4k-pgalloc.o KASAN_SANITIZE_physaddr.o += n diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c new file mode 100644 index 00000000000000..d7d550c9e14205 --- /dev/null +++ b/arch/arm64/mm/user4k-data-test.c @@ -0,0 +1,1835 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Backing ownership tests; these slots are not fault/rmap-owned mappings. */ +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_NET +#include +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +#include +#endif +#ifdef CONFIG_INET +#include +#include +#endif +#include +#endif +#include +#include + +#define DATA_SLOTS MM_SUBPAGES_PER_PAGE + +struct data_fixture { + struct mm_subpage_pool *pool; + struct mm_subpage *held[2 * DATA_SLOTS + 2]; +}; + +static void data_fixture_free(void *arg) +{ + struct data_fixture *fixture = arg; + unsigned int i; + + if (fixture->pool) + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < ARRAY_SIZE(fixture->held); i++) + if (fixture->held[i]) + mm_subpage_put(fixture->held[i]); + if (fixture->pool) + mm_subpage_pool_put(fixture->pool); +} + +static struct data_fixture *data_fixture_create_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + + if (!fixture) + return NULL; + fixture->pool = mm_subpage_pool_create_granule(GFP_KERNEL, shift); + if (!fixture->pool) + return NULL; + if (kunit_add_action_or_reset(test, data_fixture_free, fixture)) + return NULL; + return fixture; +} + +static struct data_fixture *data_fixture_create(struct kunit *test) +{ + return data_fixture_create_granule(test, MM_SUBPAGE_SHIFT); +} + +static void put_held(struct data_fixture *fixture, unsigned int index) +{ + mm_subpage_put(fixture->held[index]); + fixture->held[index] = NULL; +} + +static struct mm_subpage *hold_alloc(struct data_fixture *fixture, unsigned int index) +{ + struct mm_subpage *slot = mm_subpage_alloc(fixture->pool); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static int supply_backing(struct mm_subpage_pool *pool) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + /* Check that stale allocation contents never become exposed. */ + memset(folio_address(folio), 0xcc, PAGE_SIZE); + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static int supply_backing_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + int ret; + + if (!folio) + return -ENOMEM; + ret = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + if (ret) + folio_put(folio); + return ret; +} + +static struct mm_subpage *hold_alloc_at(struct data_fixture *fixture, unsigned int index, + unsigned int offset) +{ + struct mm_subpage *slot = mm_subpage_alloc_at(fixture->pool, offset); + + if (!IS_ERR(slot)) + fixture->held[index] = slot; + return slot; +} + +static void *slot_address(struct mm_subpage *slot) +{ + return folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot); +} + +static void user4k_data_reuse_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + KUNIT_EXPECT_PTR_EQ(test, folio, mm_subpage_folio(fixture->held[0])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + memset(slot_address(slot), 0x10 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + /* A pin, alias or deferred-TLB holder keeps this slot unavailable. */ + fixture->held[DATA_SLOTS] = fixture->held[1]; + mm_subpage_get(fixture->held[DATA_SLOTS]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS + 1); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)DATA_SLOTS - 1); + slot = hold_alloc(fixture, DATA_SLOTS + 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + if (i != 1) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i]), + 0x10 + i, SZ_4K), NULL); + } + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -ESHUTDOWN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -ESHUTDOWN); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_copy_test(struct kunit *test) +{ + struct data_fixture *source = data_fixture_create(test); + struct data_fixture *dest = data_fixture_create(test); + struct mm_subpage *copy; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, supply_backing(source->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(source, i)); + memset(slot_address(source->held[i]), 0x50 + i, SZ_4K); + } + source->held[DATA_SLOTS] = source->held[1]; + mm_subpage_get(source->held[DATA_SLOTS]); + copy = mm_subpage_copy(dest->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + dest->held[0] = copy; + KUNIT_EXPECT_NE(test, mm_subpage_phys(copy), mm_subpage_phys(source->held[1])); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xee, SZ_4K); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[i]), 0x50 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(source->pool, source->held[1])), -EAGAIN); + /* Copy into a free neighboring slot in the same physical owner. */ + put_held(source, 2); + copy = mm_subpage_copy(source->pool, source->held[1]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + source->held[2] = copy; + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(copy), mm_subpage_folio(source->held[1])); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), 2U * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x51, SZ_4K), NULL); + memset(slot_address(copy), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[1]), 0x51, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source->held[3]), 0x53, SZ_4K), NULL); +} + +static void user4k_data_lookup_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot, *found; + struct folio *folio; + phys_addr_t phys; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + phys = PFN_PHYS(folio_pfn(folio)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys), NULL); + if (mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL)) { + folio_put(folio); + KUNIT_FAIL(test, "failed to supply lookup backing"); + return; + } + /* A published owner does not expose uninitialised or unissued slots. */ + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + for (i = 0; i < DATA_SLOTS; i++) { + slot = hold_alloc(fixture, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0x90 + i, SZ_4K); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 1); + if (!i) + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, + GFP_KERNEL), -EEXIST); + found = mm_subpage_get_from_phys(phys + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, found); + fixture->held[i + DATA_SLOTS] = found; + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2 * (int)i + 2); + KUNIT_EXPECT_PTR_EQ(test, found, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K + 1), NULL); + } + /* A second pool must not zero or acquire already owned backing. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL), + -EINVAL); + for (i = 0; i < DATA_SLOTS; i++) { + put_held(fixture, i); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + DATA_SLOTS]), + 0x90 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(fixture->pool)), -EAGAIN); + put_held(fixture, DATA_SLOTS + 1); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + SZ_4K), NULL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[1]), 0, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + /* Keep the physical allocation stable while checking final unbinding. */ + folio_get(folio); + put_held(fixture, 1); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 1) + put_held(fixture, i + DATA_SLOTS); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_get_from_phys(phys + i * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static void user4k_data_exact_slot_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *source, *copy, *reused; + struct folio *first; + static const unsigned int neighbors[] = { 0, SZ_4K, 3 * SZ_4K }; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, PAGE_SIZE)), -EINVAL); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + source = hold_alloc_at(fixture, 0, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + first = mm_subpage_folio(source); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(source), (unsigned int)SZ_8K); + memset(slot_address(source), 0xa7, SZ_4K); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, 1)), -EINVAL); + /* Three free neighbors cannot satisfy a request for this occupied slot. */ + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy_at(fixture->pool, source, SZ_8K)), -EAGAIN); + KUNIT_EXPECT_EQ(test, supply_backing(fixture->pool), -EEXIST); + KUNIT_ASSERT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), 0); + KUNIT_EXPECT_EQ(test, supply_backing_at(fixture->pool, SZ_8K), -EEXIST); + copy = mm_subpage_copy_at(fixture->pool, source, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + fixture->held[1] = copy; + KUNIT_EXPECT_PTR_NE(test, mm_subpage_folio(copy), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(copy), (unsigned int)SZ_8K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0xa7, SZ_4K), NULL); + memset(slot_address(copy), 0x4b, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(source), 0xa7, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 2UL); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) { + struct mm_subpage *slot = hold_alloc_at(fixture, i + 2, neighbors[i]); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), first); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), neighbors[i]); + memset(slot_address(slot), 0x30 + i, SZ_4K); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + fixture->held[5] = source; + mm_subpage_get(source); + put_held(fixture, 0); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(fixture->pool, SZ_8K)), -EAGAIN); + put_held(fixture, 5); + reused = hold_alloc_at(fixture, 6, SZ_8K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, reused); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(reused), first); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(reused), 0, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(first), 4); + for (i = 0; i < ARRAY_SIZE(neighbors); i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(fixture->held[i + 2]), + 0x30 + i, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(copy), 0x4b, SZ_4K), NULL); + mm_subpage_pool_close(fixture->pool); + for (i = 2; i <= 4; i++) + put_held(fixture, i); + put_held(fixture, 6); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + put_held(fixture, 1); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_close_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct mm_subpage *slot; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + /* Closing with no issued slots must release that entire unused owner. */ + mm_subpage_pool_close(fixture->pool); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); + fixture = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + slot = hold_alloc(fixture, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memset(slot_address(slot), 0xa7, SZ_4K); + mm_subpage_pool_close(fixture->pool); + mm_subpage_pool_put(fixture->pool); + fixture->pool = NULL; + /* The originating pool/mm reference is gone; a child's slot remains live. */ + fixture->held[1] = slot; + mm_subpage_get(slot); + put_held(fixture, 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0xa7, SZ_4K), NULL); + put_held(fixture, 1); +} + +struct deferred_slot { + struct rcu_head rcu; + struct mm_subpage *slot; + struct completion done; +}; + +static void deferred_slot_put(struct rcu_head *rcu) +{ + struct deferred_slot *pending = container_of(rcu, struct deferred_slot, rcu); + + mm_subpage_put(pending->slot); + complete(&pending->done); +} + +static void data_rcu_barrier(void *unused) +{ + rcu_barrier(); +} + +static void user4k_data_deferred_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct deferred_slot *pending = kunit_kzalloc(test, sizeof(*pending), GFP_KERNEL); + struct mm_subpage *probe; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, pending); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + init_completion(&pending->done); + pending->slot = fixture->held[0]; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_rcu_barrier, NULL), 0); + mm_subpage_get(pending->slot); + rcu_read_lock(); + call_rcu(&pending->rcu, deferred_slot_put); + put_held(fixture, 0); + /* The delayed reference must prevent recycling before the grace period. */ + probe = mm_subpage_alloc(fixture->pool); + for (i = 1; i < DATA_SLOTS; i++) + put_held(fixture, i); + rcu_read_unlock(); + if (!IS_ERR(probe)) + fixture->held[DATA_SLOTS] = probe; + KUNIT_EXPECT_EQ(test, PTR_ERR(probe), -EAGAIN); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&pending->done, 5 * HZ), 0UL); + /* The callback retired the final live slot and the whole native owner. */ + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_backing_reject_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + folio = folio_alloc(GFP_KERNEL, 1); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + if (ret) + folio_put(folio); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_get(folio); + ret = mm_subpage_pool_add_folio(fixture->pool, folio, GFP_KERNEL); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 2); + folio_put(folio); + if (ret) + folio_put(folio); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct data_worker { + struct mm_subpage_pool *pool; + struct task_struct *task; + struct completion ready, go, done; + unsigned int id, checks; + int error; +}; + +static struct mm_subpage *alloc_with_backing(struct mm_subpage_pool *pool) +{ + unsigned int tries; + + for (tries = 0; tries < 128; tries++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + int ret; + + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + return slot; + ret = supply_backing(pool); + if (ret && ret != -EEXIST) + return ERR_PTR(ret); + cond_resched(); + } + return ERR_PTR(-EAGAIN); +} + +static int data_worker_run(void *arg) +{ + struct data_worker *worker = arg; + struct mm_subpage *slot = NULL; + unsigned int round; + u8 pattern = 0; + + for (round = 0; round < 64; round++) { + slot = alloc_with_backing(worker->pool); + if (IS_ERR(slot)) { + worker->error = PTR_ERR(slot); + slot = NULL; + break; + } + pattern = 1 + round + worker->id * 64; + memset(slot_address(slot), pattern, SZ_4K); + mm_subpage_get(slot); + mm_subpage_put(slot); + schedule_timeout_uninterruptible(1); + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) { + worker->error = -EIO; + break; + } + worker->checks++; + if (round != 63) { + mm_subpage_put(slot); + slot = NULL; + } + } + complete(&worker->ready); + wait_for_completion(&worker->go); + if (slot) { + if (memchr_inv(slot_address(slot), pattern, SZ_4K)) + worker->error = -EIO; + mm_subpage_put(slot); + } + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void data_worker_stop(void *arg) +{ + struct data_worker *worker = arg; + + complete_all(&worker->go); + kthread_stop(worker->task); +} + +static void user4k_data_concurrent_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct data_worker *workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].pool = fixture->pool; + workers[i].id = i; + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].done); + workers[i].task = kthread_run(data_worker_run, &workers[i], "user4k-data-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(workers[i].task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, data_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 1UL); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < 2; i++) + complete(&workers[i].go); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +struct lookup_worker { + struct task_struct *task; + struct completion observed; + phys_addr_t phys; + unsigned long checks; + int error; +}; + +static int lookup_worker_run(void *arg) +{ + struct lookup_worker *worker = arg; + bool observed = false; + + while (!kthread_should_stop()) { + struct mm_subpage *slot = mm_subpage_get_from_phys(worker->phys); + + if (slot) { + unsigned char *data = slot_address(slot); + unsigned char value = READ_ONCE(*data); + + if (mm_subpage_phys(slot) != worker->phys || + (value != 0x5a && value != 0xab) || + memchr_inv(data, value, SZ_4K)) + worker->error = -EINVAL; + worker->checks++; + mm_subpage_put(slot); + if (!observed) { + complete(&worker->observed); + observed = true; + } + if (worker->error) + break; + } + cond_resched(); + } + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void lookup_worker_stop(void *arg) +{ + struct lookup_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_data_lookup_race_test(struct kunit *test) +{ + struct data_fixture *fixture = data_fixture_create(test); + struct lookup_worker *worker = kunit_kzalloc(test, sizeof(*worker), GFP_KERNEL); + unsigned int copied = 0, i; + unsigned long deadline; + struct folio *folio; + struct mm_subpage *found; + + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_NOT_NULL(test, worker); + KUNIT_ASSERT_EQ(test, supply_backing(fixture->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(fixture, i)); + memset(slot_address(fixture->held[0]), 0x5a, SZ_4K); + memset(slot_address(fixture->held[1]), 0xab, SZ_4K); + worker->phys = mm_subpage_phys(fixture->held[2]); + put_held(fixture, 2); + init_completion(&worker->observed); + worker->task = kthread_run(lookup_worker_run, worker, "user4k-lookup"); + KUNIT_ASSERT_FALSE(test, IS_ERR(worker->task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, lookup_worker_stop, worker), 0); + if (num_online_cpus() > 1) { + unsigned int cpu = cpumask_next(cpumask_first(cpu_online_mask), cpu_online_mask); + + KUNIT_ASSERT_EQ(test, set_cpus_allowed_ptr(worker->task, cpumask_of(cpu)), 0); + } + /* Alternate complete contents; a reader must never see a partial copy. */ + deadline = jiffies + 30 * HZ; + while (copied < 2048 && time_before(jiffies, deadline)) { + struct mm_subpage *dest = mm_subpage_copy(fixture->pool, + fixture->held[copied & 1]); + bool observed = true; + phys_addr_t phys; + + if (PTR_ERR_OR_ZERO(dest) == -EAGAIN) { + /* Let a scheduled-out lookup holder retire its reference. */ + schedule_timeout_uninterruptible(1); + continue; + } + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dest); + phys = mm_subpage_phys(dest); + if (!copied) + observed = wait_for_completion_timeout(&worker->observed, 5 * HZ) != 0; + mm_subpage_put(dest); + KUNIT_ASSERT_TRUE(test, observed); + KUNIT_ASSERT_EQ(test, phys, worker->phys); + copied++; + cond_resched(); + } + /* Retire the owner while lookup is still running, then verify unbinding. */ + folio = mm_subpage_folio(fixture->held[0]); + folio_get(folio); + mm_subpage_pool_close(fixture->pool); + for (i = 0; i < DATA_SLOTS; i++) + if (i != 2) + put_held(fixture, i); + lookup_worker_stop(worker); + found = mm_subpage_get_from_phys(worker->phys); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + if (found) + mm_subpage_put(found); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + KUNIT_EXPECT_EQ(test, worker->error, 0); + KUNIT_EXPECT_GT(test, worker->checks, 0UL); + KUNIT_EXPECT_EQ(test, copied, 2048U); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(fixture->pool), 0UL); +} + +static void user4k_data_swapcache_reuse_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct mm_subpage *attempt; + struct folio *folio; + unsigned int i; + bool intact; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + memset(slot_address(f->held[i]), 0x60 + i, MM_SUBPAGE_SIZE); + } + folio = mm_subpage_folio(f->held[0]); + put_held(f, 1); + /* Synthetic cache membership only; no swap entry or I/O is fabricated. */ + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + attempt = mm_subpage_alloc_at(f->pool, MM_SUBPAGE_SIZE); + intact = !memchr_inv(folio_address(folio) + MM_SUBPAGE_SIZE, 0x61, MM_SUBPAGE_SIZE); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + /* Restore the real state before assertions can unwind the fixture. */ + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x60 + i, MM_SUBPAGE_SIZE), NULL); +} + +static void restore_folio_free(void *arg) +{ + struct folio *folio = arg; + + /* Data-only fixtures never create a real swap entry or mapping. */ + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_put(folio); +} + +static struct mm_subpage *restore_test_slot_granule(struct folio *folio, unsigned int offset, unsigned int shift) +{ + struct mm_subpage *slot; + + folio_lock(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + slot = mm_subpage_restore_granule(folio, offset, shift, GFP_KERNEL); + folio_clear_swapcache(folio); + __folio_clear_swapbacked(folio); + folio_unlock(folio); + return slot; +} + +static struct mm_subpage *restore_test_slot(struct folio *folio, unsigned int offset) +{ + return restore_test_slot_granule(folio, offset, MM_SUBPAGE_SHIFT); +} + +static void test_data_restore_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x70 + i, size); + } + folio = mm_subpage_folio(f->held[0]); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + put_held(f, 1); + slot = restore_test_slot_granule(folio, size, shift); + if (!IS_ERR(slot)) + f->held[1] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + slot = restore_test_slot_granule(folio, 0, shift); + if (!IS_ERR(slot)) + f->held[4] = slot; + KUNIT_ASSERT_PTR_EQ(test, slot, f->held[0]); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 6); + put_held(f, 4); + /* Retire the entire old owner, retaining only a native backing reference. */ + for (i = 0; i < 4; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + for (i = 0; i < 4; i++) { + slot = restore_test_slot_granule(folio, i * size, shift); + if (!IS_ERR(slot)) + f->held[i] = slot; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x70 + i, + size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 5); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, 1, shift)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, PAGE_SIZE, shift)), -EINVAL); + if (shift != MM_SUBPAGE_SHIFT) { + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot(folio, 0)), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(restore_test_slot_granule(folio, SZ_4K, shift)), -EINVAL); + } + +} + +static void user4k_data_restore_test(struct kunit *test) +{ + test_data_restore_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_restore_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_restore_granule(test, 14); +} + +struct restore_worker { + struct folio *folio; + struct task_struct *task; + struct completion done; + unsigned int checks; + int error; +}; + +static int restore_worker_run(void *arg) +{ + struct restore_worker *w = arg; + unsigned long deadline = jiffies + 20 * HZ; + + while (!kthread_should_stop() && w->checks < 1024 && time_before(jiffies, deadline)) { + struct mm_subpage *slot; + + folio_lock(w->folio); + slot = mm_subpage_restore(w->folio, MM_SUBPAGE_SIZE, GFP_KERNEL); + folio_unlock(w->folio); + if (PTR_ERR_OR_ZERO(slot) == -EAGAIN) { + cond_resched(); + continue; + } + if (IS_ERR(slot)) { + w->error = PTR_ERR(slot); + break; + } + if (memchr_inv(slot_address(slot), 0x83, MM_SUBPAGE_SIZE)) + w->error = -EIO; + /* Retirement deliberately occurs without the folio lock. */ + mm_subpage_put(slot); + if (w->error) + break; + w->checks++; + cond_resched(); + } + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void restore_worker_stop(void *arg) +{ + struct restore_worker *w = arg; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user4k_data_restore_race_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL, 0); + struct restore_worker *workers; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, folio), 0); + memset(folio_address(folio), 0x83, PAGE_SIZE); + __folio_mark_uptodate(folio); + __folio_set_swapbacked(folio); + folio_set_swapcache(folio); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].folio = folio; + init_completion(&workers[i].done); + workers[i].task = kthread_run(restore_worker_run, &workers[i], "user4k-restore-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "restore worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_worker_stop, &workers[i]), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 25 * HZ), 0UL); + restore_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_EQ(test, workers[i].checks, 1024U); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); +} + +static void test_data_migration_granule(struct kunit *test, unsigned int shift) +{ + unsigned int mode, size = 1U << shift; + + /* Success, rollback, all PTEs discarded, and partial-folio success. */ + for (mode = 0; mode < 4; mode++) { + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct folio *src, *dst, *target; + unsigned int i, nr = mode == 3 ? 3 : 4; + int ret, src_refs = 0, dst_refs = 0, blocked = 0; + bool unpublished = true, active = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, index * size)); + memset(slot_address(f->held[i]), 0x80 + index, size); + } + src = mm_subpage_folio(f->held[0]); + folio_get(src); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, src), 0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, restore_folio_free, dst), 0); + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + struct mm_subpage *probe; + + src_refs = folio_ref_count(src); + dst_refs = folio_ref_count(dst); + active = mm_subpage_migrating(src) && mm_subpage_migrating(dst); + probe = mm_subpage_alloc_at(f->pool, size); + blocked = PTR_ERR_OR_ZERO(probe); + if (!IS_ERR(probe)) + mm_subpage_put(probe); + /* Emulate migration entries: their old mapping refs are gone. */ + for (i = 0; i < nr; i++) + put_held(f, i); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(src))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + probe = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(dst))); + unpublished &= !probe; + if (probe) + mm_subpage_put(probe); + if (mode != 1 && mode != 2) { + copy_user_highpage(&dst->page, &src->page, 0, NULL); + __folio_mark_uptodate(dst); + } + target = mode == 1 ? src : dst; + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + f->held[i] = mm_subpage_migrate_restore(target, index * size); + } + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, active); + KUNIT_EXPECT_TRUE(test, unpublished); + KUNIT_EXPECT_EQ(test, src_refs, (int)nr + 1); + KUNIT_EXPECT_EQ(test, dst_refs, 1); + KUNIT_EXPECT_EQ(test, blocked, -EAGAIN); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), mode == 2 ? 0UL : 1UL); + KUNIT_EXPECT_EQ(test, folio_ref_count(src), mode == 1 ? (int)nr + 1 : 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(dst), mode == 0 || mode == 3 ? (int)nr + 1 : 1); + if (mode != 2) + for (i = 0; i < nr; i++) { + unsigned int index = mode == 3 && i ? i + 1 : i; + + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[i]), mode == 1 ? src : dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + 0x80 + index, size), NULL); + } + if (mode == 3) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 3, size)); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->held[3]), dst); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[3]), 0, + size), NULL); + } + } +} + +static void user4k_data_migration_owner_test(struct kunit *test) +{ + test_data_migration_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_migration_owner_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_migration_granule(test, 14); +} + +#ifdef CONFIG_ARM64_MTE +static void test_slot_tags_set(void *addr, unsigned int tag) +{ + unsigned long value = (unsigned long)tag << MTE_TAG_SHIFT; + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (value), "r" (addr + offset) : "memory"); +} + +static bool test_slot_tags_equal(void *addr, unsigned int expected) +{ + unsigned int offset; + + for (offset = 0; offset < MM_SUBPAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long value = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (value) : "r" (addr + offset) : "memory"); + if (((value >> MTE_TAG_SHIFT) & 0xf) != expected) + return false; + } + return true; +} +#endif + +static void user4k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + unsigned int i; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + f = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * MM_SUBPAGE_SIZE)); + folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&folio->page)); + mte_clear_page_tags(folio_address(folio)); + for (i = 0; i < 4; i++) { + memset(slot_address(f->held[i]), 0x40 + i, MM_SUBPAGE_SIZE); + test_slot_tags_set(slot_address(f->held[i]), i + 1); + } + set_page_mte_tagged(&folio->page); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, MM_SUBPAGE_SIZE)); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 0 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + put_held(f, 1); + f->held[1] = mm_subpage_copy_at(f->pool, f->held[0], MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[1])) { + f->held[1] = NULL; + KUNIT_FAIL(test, "tagged slot copy failed"); + return; + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(slot_address(f->held[i]), i == 1 ? 1 : i + 1)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0x40 : 0x40 + i, MM_SUBPAGE_SIZE), NULL); + } + /* A fresh destination must initialise tags outside the copied slot too. */ + KUNIT_ASSERT_EQ(test, supply_backing_at(f->pool, 3 * MM_SUBPAGE_SIZE), 0); + f->held[4] = mm_subpage_copy_at(f->pool, f->held[2], 3 * MM_SUBPAGE_SIZE); + if (IS_ERR(f->held[4])) { + f->held[4] = NULL; + KUNIT_FAIL(test, "fresh tagged slot copy failed"); + return; + } + folio = mm_subpage_folio(f->held[4]); + KUNIT_EXPECT_TRUE(test, page_mte_tagged(&folio->page)); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(folio_address(folio) + + i * MM_SUBPAGE_SIZE, i == 3 ? 3 : 0)); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void extent_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_copy_extent_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; + + for (unsigned int n = 0; n < ARRAY_SIZE(sizes); n++) { + struct folio *source, *dest; + unsigned int size = sizes[n], to_offset; + void *from, *to; + + if (size > PAGE_SIZE) + continue; + source = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, source); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, source), 0); + dest = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, dest), 0); + from = folio_address(source); + to = folio_address(dest); + to_offset = PAGE_SIZE - size; + for (unsigned int i = 0; i < DATA_SLOTS; i++) + memset(from + i * SZ_4K, 0x40 + i, SZ_4K); + for (unsigned int tagged = 0; tagged < 2; tagged++) { + memset(to, 0xac, PAGE_SIZE); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) { + if (try_page_mte_tagging(&dest->page)) + mte_clear_page_tags(to); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(to + i * SZ_4K, 9); + set_page_mte_tagged(&dest->page); + if (tagged) { + if (try_page_mte_tagging(&source->page)) + mte_clear_page_tags(from); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(from + i * SZ_4K, 1 + i % 15); + set_page_mte_tagged(&source->page); + } + } +#endif + KUNIT_ASSERT_EQ(test, copy_user_subpage_range(&dest->page, to_offset, + &source->page, 0, size), 0); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool copied = offset >= to_offset; + unsigned int source_index = copied ? (offset - to_offset) / SZ_4K : 0; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, + copied ? 0x40 + source_index : 0xac, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(from + offset, 0x40 + i, SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, + copied ? (tagged ? 1 + source_index % 15 : 0) : 9)); +#endif + } + clear_user_subpage_range(&dest->page, to_offset, size); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int offset = i * SZ_4K; + bool cleared = offset >= to_offset; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(to + offset, cleared ? 0 : 0xac, + SZ_4K), NULL); +#ifdef CONFIG_ARM64_MTE + if (system_supports_mte()) + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(to + offset, cleared ? 0 : 9)); +#endif + } + } + } +} + +#ifdef CONFIG_NET +struct skb_test_state { + struct folio *folio; +#ifdef CONFIG_INET + struct socket *socket; +#endif + struct sk_buff *skbs[5]; +}; + +static void skb_test_cleanup(void *arg) +{ + struct skb_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->skbs); i++) { + if (state->skbs[i]) + state->skbs[i]->sk = NULL; + kfree_skb(state->skbs[i]); + } +#ifdef CONFIG_INET + if (state->socket) + sock_release(state->socket); +#endif + /* This fixture marks ownership only; it installs no anonymous rmap. */ + if (state->folio) + state->folio->mapping = NULL; +} + +static void user4k_data_skb_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct iov_iter iter; + struct bio_vec bvec; + unsigned char *bytes = kunit_kmalloc(test, SZ_4K, GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_NOT_NULL(test, bytes); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + memset(slot_address(f->held[i]), 0x51 + i, SZ_4K); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + state->skbs[0]->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, skb_zcopy(state->skbs[0])); + state->skbs[1] = pskb_copy(state->skbs[0], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[1]); + /* Append a distinct quarter while the callback object has two owners. */ + bvec_set_page(&bvec, &state->folio->page, SZ_4K, 2 * SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(state->skbs[0], &iter, SZ_4K), (ssize_t)SZ_4K); + KUNIT_EXPECT_PTR_NE(test, skb_zcopy(state->skbs[0]), skb_zcopy(state->skbs[1])); + state->skbs[2] = skb_clone(state->skbs[1], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[2]); + state->skbs[3] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[3]); + skb_split(state->skbs[0], state->skbs[3], SZ_2K); + state->skbs[4] = skb_copy(state->skbs[3], GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[4]); + put_held(f, 1); + put_held(f, 2); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_EXPECT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + for (i = 0; i < 2; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[2], 0, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x52, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[3], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + for (i = 2; i < 4; i++) { + kfree_skb(state->skbs[i]); + state->skbs[i] = NULL; + } + /* A full data copy must not keep ownership of the original quarters. */ + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 2, 2 * SZ_4K)); + memset(slot_address(f->held[1]), 0xe1, SZ_4K); + memset(slot_address(f->held[2]), 0xe2, SZ_4K); + KUNIT_ASSERT_EQ(test, skb_copy_bits(state->skbs[4], SZ_2K, bytes, SZ_4K), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(bytes, 0x53, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} +#ifdef CONFIG_INET +static void test_data_skb_prune_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct sk_buff *skb; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + /* A live socket association prevents trim from condensing data into the head. */ + skb->sk = state->socket->sk; + skb->ip_summed = CHECKSUM_UNNECESSARY; + bvec_set_page(&bvec, &state->folio->page, 16, size - SZ_4K + 16); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, 16); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, 16), (ssize_t)16); + for (i = 0; i < 1024; i++) { + bvec_set_page(&bvec, &state->folio->page, size, (1 + i % 3) * size); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, size), (ssize_t)size); + /* Four fixture refs, two frag refs, two current quarter refs. */ + KUNIT_ASSERT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, pskb_trim(skb, 16), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + skb->sk = NULL; + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skb_prune_test(struct kunit *test) +{ + test_data_skb_prune_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skb_prune_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skb_prune_granule(test, 14); +} + +#endif + +static void user4k_data_skb_checksum_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct skb_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct bio_vec bvec[2]; + struct iov_iter iter; + unsigned char *bytes; + unsigned int i, owned; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skb_test_cleanup, state), 0); + bytes = folio_address(state->folio); + for (i = 0; i < PAGE_SIZE; i++) + bytes[i] = i * 13 + (i >> 12) * 31; + for (owned = 0; owned < 2; owned++) { + struct sk_buff *skb; + + state->folio->mapping = owned ? (void *)FOLIO_MAPPING_ANON : NULL; + state->skbs[0] = alloc_skb(0, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->skbs[0]); + skb = state->skbs[0]; + skb->ip_summed = CHECKSUM_NONE; + bvec_set_page(&bvec[0], &state->folio->page, SZ_4K - 13, 13); + bvec_set_page(&bvec[1], &state->folio->page, SZ_4K, SZ_4K); + iov_iter_bvec(&iter, ITER_SOURCE, bvec, 2, SZ_8K - 13); + KUNIT_ASSERT_EQ(test, skb_splice_from_iter(skb, &iter, SZ_8K - 13), + (ssize_t)(SZ_8K - 13)); + KUNIT_EXPECT_EQ(test, (__force u16)csum_fold(skb->csum), + (__force u16)csum_fold(skb_checksum(skb, 0, skb->len, 0))); + kfree_skb(skb); + state->skbs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + } +} + +#endif + +struct refs_test_state { + struct folio *folio; + struct mm_subpage_refs *refs[3]; +}; + +static void refs_test_cleanup(void *arg) +{ + struct refs_test_state *state = arg; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(state->refs); i++) + mm_subpage_refs_put(state->refs[i]); + if (state->folio) + state->folio->mapping = NULL; +} + +static bool refs_test_keep(const struct mm_subpage *slot, void *context) +{ + unsigned long mask = *(unsigned long *)context; + + return mask & BIT(mm_subpage_offset(slot) >> MM_SUBPAGE_SHIFT); +} + +static void user4k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f = data_fixture_create(test); + struct refs_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct page *page; + unsigned long mask; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < DATA_SLOTS; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_4K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Ownership fixture only; no anonymous rmap is installed. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_4K + 17, SZ_8K - 34, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[1], page, 3 * SZ_4K, SZ_4K, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[1]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 5); + mask = BIT(2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 3); + state->refs[2] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(1); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[2], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_NE(test, state->refs[0], state->refs[2]); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + KUNIT_EXPECT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, PAGE_SIZE, 1, GFP_KERNEL), -EINVAL); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 4); + mask = 0; + for (i = 1; i < 3; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[i], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_PTR_EQ(test, state->refs[i], NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 2); + put_held(f, 1); + attempt = mm_subpage_alloc_at(f->pool, SZ_4K); + if (!IS_ERR(attempt)) + f->held[1] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_4K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); + for (i = 0; i < 1024; i++) { + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 2 * (int)DATA_SLOTS); + mask = BIT(i % DATA_SLOTS); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS + 1); + } + mm_subpage_refs_put(state->refs[0]); + state->refs[0] = NULL; + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)DATA_SLOTS); +} + +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) +struct skmsg_test_state { + struct folio *folio; + struct socket *socket; + struct sk_msg messages[2]; + struct sk_msg borrowed; +}; + +static void skmsg_test_cleanup(void *arg) +{ + struct skmsg_test_state *state = arg; + unsigned int i; + + if (state->socket) { + for (i = 0; i < ARRAY_SIZE(state->messages); i++) + sk_msg_free(state->socket->sk, &state->messages[i]); + sk_msg_subpages_release(&state->borrowed); + sock_release(state->socket); + } + if (state->folio) + state->folio->mapping = NULL; +} + +static void test_data_skmsg_granule(struct kunit *test, unsigned int shift) +{ + struct data_fixture *f = data_fixture_create_granule(test, shift); + struct skmsg_test_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct mm_subpage *attempt; + struct sk_msg *src, *dst; + struct page *page; + struct sock *sk; + struct bio_vec bvec; + struct iov_iter iter; + unsigned int i, size = 1U << shift; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (i = 0; i < 4; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * size)); + memset(slot_address(f->held[i]), 0x60 + i, size); + } + state->folio = mm_subpage_folio(f->held[0]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, skmsg_test_cleanup, state), 0); + KUNIT_ASSERT_EQ(test, sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &state->socket), 0); + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + page = &state->folio->page; + sk = state->socket->sk; + src = &state->messages[0]; + dst = &state->messages[1]; + sk_msg_init(src); + sk_msg_init(dst); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, (size / 2) - 13, size + (size / 2) + 13), 0); + sk_mem_charge(sk, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 2 * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + KUNIT_ASSERT_EQ(test, sk_msg_clone(sk, dst, src, 0, size + (size / 2) - 13), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free_partial(sk, src, (size / 2) - 13); + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 11); + sk_msg_free(sk, dst); + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, size)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + for (i = 0; i < 1024; i++) { + unsigned int index = (i + 1) & 3; + + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, index * size), 0); + sk_mem_charge(sk, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 8); + sk_msg_free_partial(sk, src, size); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 7); + } + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + /* A borrowed descriptor copy has an independent ownership lifetime. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 0), 0); + sk_mem_charge(sk, size); + memcpy(&state->borrowed, src, sizeof(*src)); + sk_msg_subpages_get(&state->borrowed); + sk_msg_free(sk, src); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 5); + put_held(f, 0); + attempt = mm_subpage_alloc_at(f->pool, 0); + if (!IS_ERR(attempt)) + f->held[0] = attempt; + KUNIT_ASSERT_EQ(test, PTR_ERR_OR_ZERO(attempt), -EAGAIN); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(sg_virt(&state->borrowed.sg.data[0]), 0x60, size), NULL); + sk_msg_subpages_release(&state->borrowed); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 0, 0)); + /* Full transfer moves the set, and zero-copy iterator trim releases its references. */ + KUNIT_ASSERT_EQ(test, sk_msg_page_add(src, page, size, 3 * size), 0); + sk_mem_charge(sk, size); + sk_msg_xfer_full(dst, src); + KUNIT_EXPECT_EQ(test, src->sg.size, 0U); + sk_msg_free(sk, dst); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); + bvec_set_page(&bvec, page, size - 17, 3 * size + 17); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, size - 17); + KUNIT_ASSERT_EQ(test, sk_msg_zerocopy_from_iter(sk, &iter, src, size - 17), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 6); + sk_msg_trim(sk, src, 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), 4); +} + +static void user4k_data_skmsg_test(struct kunit *test) +{ + test_data_skmsg_granule(test, MM_SUBPAGE_SHIFT); +} + +static void user16k_data_skmsg_test(struct kunit *test) +{ + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_data_skmsg_granule(test, 14); +} +#endif + +static void user16k_data_owner_test(struct kunit *test) +{ + struct data_fixture *f, *dest, *small; + struct mm_subpage *slot, *found; + unsigned int count = PAGE_SIZE / SZ_16K; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + dest = data_fixture_create_granule(test, 14); + small = data_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(dest->pool), 0); + KUNIT_ASSERT_EQ(test, supply_backing(small->pool), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc(small, 0)); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_4K)), -EINVAL); + for (unsigned int i = 0; i < count; i++) { + slot = hold_alloc_at(f, i, i * SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_shift(slot), 14U); + KUNIT_EXPECT_EQ(test, mm_subpage_size(slot), (unsigned int)SZ_16K); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_16K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0, SZ_16K), NULL); + memset(slot_address(slot), 0x30 + i, SZ_16K); + for (unsigned int quarter = 0; quarter < 4; quarter++) { + found = mm_subpage_get_from_phys(mm_subpage_phys(slot) + quarter * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, found, slot); + mm_subpage_put(found); + } + } + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -EAGAIN); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(dest->pool, small->held[0])), -EINVAL); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_copy(small->pool, f->held[0])), -EINVAL); + slot = mm_subpage_copy_at(dest->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + dest->held[0] = slot; + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(slot), 0x30 + count - 1, SZ_16K), NULL); + /* An interior-byte lookup holds the whole identity across release/reuse. */ + f->held[count] = mm_subpage_get_from_phys(mm_subpage_phys(f->held[1]) + 3 * SZ_4K); + KUNIT_ASSERT_PTR_EQ(test, f->held[count], f->held[1]); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + put_held(f, count); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(slot_address(f->held[i]), + i == 1 ? 0 : 0x30 + i, SZ_16K), NULL); + mm_subpage_pool_close(f->pool); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc(f->pool)), -ESHUTDOWN); + for (unsigned int i = 0; i < count; i++) + put_held(f, i); + KUNIT_EXPECT_EQ(test, mm_subpage_pool_backing_pages(f->pool), 0UL); +} + +static void user16k_data_refset_test(struct kunit *test) +{ + struct data_fixture *f; + struct refs_test_state *state; + unsigned int count = PAGE_SIZE / SZ_16K; + unsigned long mask; + struct page *page; + + if (PAGE_SIZE <= SZ_16K) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, supply_backing(f->pool), 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + state->folio = mm_subpage_folio(f->held[0]); + page = &state->folio->page; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, refs_test_cleanup, state), 0); + /* Metadata-only ownership fixture: no VMA or rmap is published. */ + state->folio->mapping = (void *)FOLIO_MAPPING_ANON; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, + SZ_16K - 1024, 2048, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_add(&state->refs[0], page, 0, PAGE_SIZE, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2); + state->refs[1] = mm_subpage_refs_get(state->refs[0]); + mask = BIT(SZ_16K >> MM_SUBPAGE_SHIFT); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count * 2 + 1); + mask = 0; + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[0], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count + 1); + put_held(f, 1); + KUNIT_EXPECT_EQ(test, PTR_ERR(mm_subpage_alloc_at(f->pool, SZ_16K)), -EAGAIN); + KUNIT_ASSERT_EQ(test, mm_subpage_refs_prune(&state->refs[1], refs_test_keep, &mask, GFP_KERNEL), 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->folio), (int)count); +} + +static void user16k_data_mte_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct data_fixture *f; + struct folio *folio; + struct mm_subpage *copy; + unsigned int count = PAGE_SIZE / SZ_16K; + int ret; + + if (PAGE_SIZE <= SZ_16K || !system_supports_mte()) { + kunit_skip(test, "requires MTE and native backing larger than 16K"); + return; + } + f = data_fixture_create_granule(test, 14); + KUNIT_ASSERT_NOT_NULL(test, f); + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) + test_slot_tags_set(folio_address(folio) + i * SZ_4K, 9); + set_page_mte_tagged(&folio->page); + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned int i = 0; i < count; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, i, i * SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 0)); + memset(address, 0x50 + i, SZ_4K); + test_slot_tags_set(address, 1 + i % 15); + } + put_held(f, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, hold_alloc_at(f, 1, SZ_16K)); + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + bool cleared = i >= 4 && i < 8; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, cleared ? 0 : 1 + i % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, cleared ? 0 : 0x50 + i, SZ_4K), NULL); + } + put_held(f, 1); + copy = mm_subpage_copy_at(f->pool, f->held[count - 1], SZ_16K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, copy); + f->held[1] = copy; + for (unsigned int i = 0; i < DATA_SLOTS; i++) { + unsigned int source = i >= 4 && i < 8 ? i + DATA_SLOTS - 8 : i; + void *address = folio_address(folio) + i * SZ_4K; + + KUNIT_EXPECT_TRUE(test, test_slot_tags_equal(address, 1 + source % 15)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(address, 0x50 + source, SZ_4K), NULL); + } +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + +static void expect_file_coverage(struct kunit *test, struct folio *folio, + unsigned int leaves, unsigned int first, unsigned int tail, + unsigned int final) +{ + struct mm_subpage_mapcounts counts; + unsigned int i; + + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), leaves); + for (i = 0; i < DATA_SLOTS; i++) { + unsigned int expected = i == 0 ? first : + i == DATA_SLOTS - 1 ? final : i >= DATA_SLOTS - 4 ? tail : 0; + + KUNIT_EXPECT_EQ_MSG(test, counts.slots[i], expected, "coverage slot %u", i); + } +} + +static void user16k_file_coverage_test(struct kunit *test) +{ + struct folio *folio; + phys_addr_t base, tail, last; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative file coverage requires larger native backing"); + return; + } + folio = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, extent_folio_put, folio), 0); + base = PFN_PHYS(folio_pfn(folio)); + tail = base + PAGE_SIZE - SZ_16K; + last = base + PAGE_SIZE - SZ_4K; + expect_file_coverage(test, folio, 0, 0, 0, 0); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 0, 1, 1); + __mm_subpage_file_map_add(last, SZ_4K); + expect_file_coverage(test, folio, 2, 0, 1, 2); + __mm_subpage_file_map_add(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 0, 2, 3); + __mm_subpage_file_map_add(base, SZ_4K); + expect_file_coverage(test, folio, 4, 1, 2, 3); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 3, 1, 1, 2); + __mm_subpage_file_map_del(last, SZ_4K); + expect_file_coverage(test, folio, 2, 1, 1, 1); + __mm_subpage_file_map_del(tail, SZ_16K); + expect_file_coverage(test, folio, 1, 1, 0, 0); + __mm_subpage_file_map_del(base, SZ_4K); + expect_file_coverage(test, folio, 0, 0, 0, 0); +} + +static struct kunit_case user4k_data_cases[] = { + KUNIT_CASE(user4k_data_reuse_test), + KUNIT_CASE(user4k_data_refset_test), +#if IS_ENABLED(CONFIG_NET_SOCK_MSG) && IS_ENABLED(CONFIG_INET) + KUNIT_CASE(user4k_data_skmsg_test), + KUNIT_CASE(user16k_data_skmsg_test), +#endif +#ifdef CONFIG_NET + KUNIT_CASE(user4k_data_skb_test), + KUNIT_CASE(user4k_data_skb_checksum_test), + +#ifdef CONFIG_INET + KUNIT_CASE(user4k_data_skb_prune_test), + KUNIT_CASE(user16k_data_skb_prune_test), +#endif +#endif + KUNIT_CASE(user4k_data_mte_test), + KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user16k_data_owner_test), + KUNIT_CASE(user16k_file_coverage_test), + KUNIT_CASE(user16k_data_refset_test), + KUNIT_CASE(user16k_data_mte_test), + KUNIT_CASE(user4k_data_swapcache_reuse_test), + KUNIT_CASE(user4k_data_restore_test), + KUNIT_CASE(user16k_data_restore_test), + KUNIT_CASE(user4k_data_restore_race_test), + KUNIT_CASE(user4k_data_migration_owner_test), + KUNIT_CASE(user16k_data_migration_owner_test), + KUNIT_CASE(user4k_data_copy_test), + KUNIT_CASE(user4k_data_lookup_test), + KUNIT_CASE(user4k_data_exact_slot_test), + KUNIT_CASE(user4k_data_close_test), + KUNIT_CASE(user4k_data_deferred_test), + KUNIT_CASE(user4k_data_backing_reject_test), + KUNIT_CASE(user4k_data_concurrent_test), + KUNIT_CASE(user4k_data_lookup_race_test), + {} +}; + +static void user4k_data_suite_exit(struct kunit_suite *suite) +{ + /* Drain batched metadata frees before the disposable guest powers off. */ + kvfree_rcu_barrier(); +} + +static struct kunit_suite user4k_data_suite = { + .name = "arm64-user4k-data", + .test_cases = user4k_data_cases, + .suite_exit = user4k_data_suite_exit, +}; + +kunit_test_suite(user4k_data_suite); +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c new file mode 100644 index 00000000000000..e406b97ea6235d --- /dev/null +++ b/arch/arm64/mm/user4k-test.c @@ -0,0 +1,14237 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Architectural prerequisite for mixed-granule userspace. + * + * Packed architectural tables and isolated Linux mms are separate fixtures. + * PAGE_SIZE, struct page, all ordinary mms and TTBR1 retain native geometry. Only bounded + * kernel workers may borrow the isolated Linux mms. Never pass + * packed architectural tables to generic MM walkers or expose them to a task. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../../drivers/iommu/iommufd/iommufd_private.h" +#include "../../../drivers/iommu/iommufd/io_pagetable.h" +#include "../../../mm/internal.h" +#include "../../../mm/vma.h" +#include "../../../mm/swap.h" +#ifdef CONFIG_FUTEX +#include "../../../kernel/futex/futex.h" +#endif + +#define USER4K_SHIFT 12 +#define TEST_SLOTS MM_SUBPAGES_PER_PAGE +#define USER4K_SIZE BIT(USER4K_SHIFT) +#define TEST_VA (4UL * SZ_1M) +#define TEST_LEVELS 4 +#define PROBE_SLOTS 4 +#define TEST_ROUNDS 8 +#define TEST_PROT (PTE_TYPE_PAGE | PTE_USER | PTE_AF | PTE_NG | \ + PTE_SHARED | PTE_PXN | PTE_UXN | PTE_ATTRINDX(MT_NORMAL)) + +static bool user4k_test_enabled; + +static int __init user4k_test_setup(char *str) +{ + bool enabled; + int ret = kstrtobool(str, &enabled); + + if (ret) + return ret; + /* Software MM tests are sanitizable; raw/borrowed TTBR0 tests stay gated. */ + if (enabled && IS_ENABLED(CONFIG_KASAN)) + return -EINVAL; + user4k_test_enabled = enabled; + return 0; +} +early_param("arm64.user4k_test", user4k_test_setup); + +struct test_tables { + u64 *level[TEST_LEVELS]; + unsigned int shift; +}; + +struct test_result { + u64 native[PROBE_SLOTS]; + u64 read[PROBE_SLOTS]; + u64 write[PROBE_SLOTS]; + u64 replaced[PROBE_SLOTS]; + u64 replaced_read; + u64 kernel; + u64 tcr; + u64 ttbr1; + u64 restored_tcr; + u64 restored_ttbr0; +}; + +static void test_free_page(void *ptr) +{ + free_page((unsigned long)ptr); +} + +static void *test_alloc_page(struct kunit *test) +{ + void *ptr = (void *)get_zeroed_page(GFP_KERNEL); + + if (!ptr) + return NULL; + if (kunit_add_action_or_reset(test, test_free_page, ptr)) + return NULL; + return ptr; +} + +static void test_put_compound(void *ptr) +{ + put_page(ptr); +} + +static void *test_alloc_granule(struct kunit *test, unsigned int shift) +{ + struct page *page; + + if (shift <= PAGE_SHIFT) + return test_alloc_page(test); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO | __GFP_COMP, shift - PAGE_SHIFT); + if (!page) + return NULL; + if (kunit_add_action_or_reset(test, test_put_compound, page)) + return NULL; + return page_address(page); +} + +static unsigned int test_index(unsigned long va, unsigned int shift, + unsigned int level) +{ + unsigned int bits = shift - PTDESC_ORDER; + + return (va >> ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, shift)) & (BIT(bits) - 1); +} + +static int test_alloc_tables(struct kunit *test, struct test_tables *tables, + unsigned int shift) +{ + int level; + void *packed = NULL; + + tables->shift = shift; + if (shift == USER4K_SHIFT) { + /* Also test table address bits 13:12, not just leaf addresses. */ + packed = test_alloc_page(test); + if (!packed) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS; level++) { + tables->level[level] = packed ? packed + level * USER4K_SIZE : + test_alloc_page(test); + if (!tables->level[level]) + return -ENOMEM; + } + for (level = 0; level < TEST_LEVELS - 1; level++) { + unsigned int index = test_index(TEST_VA, shift, level); + + tables->level[level][index] = + virt_to_phys(tables->level[level + 1]) | PMD_TYPE_TABLE; + } + return 0; +} + +static void user4k_geometry_test(struct kunit *test) +{ + static const unsigned int shifts4k[] = { 39, 30, 21, 12 }; + static const unsigned int shifts16k[] = { 47, 36, 25, 14 }; + static const unsigned int shifts64k[] = { 55, 42, 29, 16 }; + unsigned int level; + + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 12), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 14), 4); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(48, 16), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(42, 16), 2); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(39, 12), 3); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVELS_FOR(47, 14), 3); + for (level = 0; level < TEST_LEVELS; level++) { + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 12), + shifts4k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT_FOR(level, 14), + shifts16k[level]); + KUNIT_EXPECT_EQ(test, ARM64_HW_PGTABLE_LEVEL_SHIFT(level), + PAGE_SHIFT == 16 ? shifts64k[level] : shifts16k[level]); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]), 12, level), 1U); + KUNIT_EXPECT_EQ(test, test_index(BIT(shifts4k[level]) - 1, 12, level), 0U); + } +} + +static void test_leaf_encoding(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + const phys_addr_t base = GENMASK_ULL(47, PAGE_SHIFT); + pgprot_t prot = __pgprot(TEST_PROT | PTE_RDONLY); + unsigned int i; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, mm); + pte = phys_pte_mm(mm, base, prot); + KUNIT_EXPECT_EQ(test, pte_val(pte), pte_val(pfn_pte(base >> PAGE_SHIFT, prot))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), base); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + mm->context.flags |= shift == 12 ? MMCF_USER_4K : MMCF_USER_16K; + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + phys_addr_t phys = base + i * (1UL << shift); + + pte = phys_pte_mm(mm, phys, prot); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_EQ(test, pte_pfn(pte), (unsigned long)(base >> PAGE_SHIFT)); + KUNIT_EXPECT_EQ(test, pgprot_val(pte_pgprot_mm(mm, pte)), pgprot_val(prot)); + if (i + 1 < (PAGE_SIZE >> shift)) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte_advance_pfn_mm(mm, pte, 1)), + phys + (1UL << shift)); + pte = pte_modify(pte, __pgprot(TEST_PROT)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_val(pte) & PTE_RDONLY); + } +} + +static void user4k_leaf_encoding_test(struct kunit *test) +{ + test_leaf_encoding(test, 12); +} + +static void user16k_leaf_encoding_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative leaves require larger native backing"); + return; + } + test_leaf_encoding(test, 14); +} + + +/* Caller masks all asynchronous exceptions; this function cannot sleep. */ +static void test_switch_ttbr0(u64 ttbr, u64 tcr) +{ + /* The zero root is valid for both tested granules. */ + cpu_set_reserved_ttbr0(); + local_flush_tlb_all(); + write_sysreg(tcr, tcr_el1); + isb(); + write_sysreg(ttbr, ttbr0_el1); + isb(); +} + +static u64 test_translate(unsigned long va, bool write) +{ + if (write) + asm volatile("at s1e0w, %0" : : "r" (va) : "memory"); + else + asm volatile("at s1e0r, %0" : : "r" (va) : "memory"); + isb(); + return read_sysreg_par(); +} + +#define MM_TEST_MAPPINGS 11 + +struct test_mm { + struct kunit *test; + struct mm_struct *mm; + unsigned long address[MM_TEST_MAPPINGS]; + phys_addr_t phys[MM_TEST_MAPPINGS]; +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl); + +static void test_mm_hardware(struct test_mm *ctx) +{ + struct kunit *test = ctx->test; + struct vm_area_struct vma = TLB_FLUSH_VMA(ctx->mm, 0); + struct arch_tlbflush_unmap_batch batch = {}; + pte_t *leaves[4]; + u64 invalidated[3][4]; + u64 result[MM_TEST_MAPPINGS], writes[MM_TEST_MAPPINGS]; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr, kernel; + u64 old_id = atomic64_read(&ctx->mm->context.id); + u64 restored_tcr, restored_ttbr0, retained_ttbr1; + unsigned long flags; + unsigned int i, step; + + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + spinlock_t *ptl; + + leaves[i] = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + if (leaves[i]) + pte_unmap_unlock(leaves[i], ptl); + KUNIT_ASSERT_NOT_NULL(test, leaves[i]); + } + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | TCR_EL1_EPD0)) | + TCR_TG0_4K | TCR_T0SZ(48); + test_switch_ttbr0(virt_to_phys(ctx->mm->pgd), tcr); + /* A1 selects the saved TTBR1 ASID for these private test translations. */ + atomic64_set(&ctx->mm->context.id, old_ttbr1 >> 48); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + result[i] = test_translate(ctx->address[i], false); + writes[i] = test_translate(ctx->address[i], true); + } + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + unsigned int first = step == 2 ? 3 : 1; + unsigned int last = step == 1 ? 3 : first + 1; + + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(0)); + switch (step) { + case 0: + __flush_tlb_page(&vma, ctx->address[first] + 1, + TLBF_NONOTIFY | TLBF_NOBROADCAST); + break; + case 1: + __flush_tlb_range(&vma, ctx->address[first], ctx->address[last], + SZ_4K, 3, TLBF_NONOTIFY); + break; + case 2: + arch_tlbbatch_add_pending(&batch, ctx->mm, ctx->address[first], + ctx->address[first] + SZ_4K); + arch_tlbbatch_flush(&batch); + break; + } + isb(); + for (i = 0; i < ARRAY_SIZE(leaves); i++) + invalidated[step][i] = test_translate(ctx->address[i], false); + for (i = first; i < last; i++) + WRITE_ONCE(*leaves[i], __pte(ctx->phys[i] | TEST_PROT)); + dsb(ishst); + isb(); + /* Repopulate before the next independent invalidation. */ + for (i = 0; i < ARRAY_SIZE(leaves); i++) + test_translate(ctx->address[i], false); + } + asm volatile("at s1e1r, %0" : : "r" (ctx->mm->pgd) : "memory"); + isb(); + kernel = read_sysreg_par(); + retained_ttbr1 = read_sysreg(ttbr1_el1); + test_switch_ttbr0(old_ttbr0, old_tcr); + restored_tcr = read_sysreg(tcr_el1); + restored_ttbr0 = read_sysreg(ttbr0_el1); + write_sysreg(old_par, par_el1); + isb(); + atomic64_set(&ctx->mm->context.id, old_id); + local_daif_restore(flags); + preempt_enable(); + + KUNIT_EXPECT_EQ(test, restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, retained_ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, kernel & SYS_PAR_EL1_PA, + virt_to_phys(ctx->mm->pgd) & SYS_PAR_EL1_PA); + for (step = 0; step < ARRAY_SIZE(invalidated); step++) { + for (i = 0; i < ARRAY_SIZE(leaves); i++) { + bool hole = step == 0 ? i == 1 : step == 1 ? i == 1 || i == 2 : i == 3; + + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_F, (u64)hole); + if (!hole) + KUNIT_EXPECT_EQ(test, invalidated[step][i] & SYS_PAR_EL1_PA, + ctx->phys[i]); + } + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, result[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, writes[i] & SYS_PAR_EL1_PA, ctx->phys[i]); + } +} + +struct test_walk { + struct test_mm *ctx; + unsigned long size; + unsigned long holes; + unsigned int present; +}; + +/* Callbacks run under the PTE lock; record errors without invoking KUnit. */ +static int test_walk_pte(pte_t *pte, unsigned long addr, unsigned long next, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + u64 value = pte_val(ptep_get(pte)); + unsigned int i; + + if (next - addr != state->size) + return -EINVAL; + if (!value) { + state->holes += next - addr; + return 0; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + if (state->ctx->address[i] != addr) + continue; + if (value != (state->ctx->phys[i] | TEST_PROT)) + return -EINVAL; + state->present++; + return 0; + } + return -EINVAL; +} + +static int test_walk_hole(unsigned long addr, unsigned long next, int depth, + struct mm_walk *walk) +{ + struct test_walk *state = walk->private; + + state->holes += next - addr; + return 0; +} + +static const struct mm_walk_ops test_walk_ops = { + .pte_entry = test_walk_pte, + .pte_hole = test_walk_hole, +}; + +static pte_t *test_mm_lookup(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl) +{ + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + if (pgd_none(*pgd) || pgd_bad(*pgd)) + return NULL; + p4d = p4d_offset_mm(mm, pgd, addr); + if (p4d_none_mm(mm, *p4d) || p4d_bad_mm(mm, *p4d)) + return NULL; + pud = pud_offset_mm(mm, p4d, addr); + if (pud_none(*pud) || pud_bad(*pud)) + return NULL; + pmd = pmd_offset_mm(mm, pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return NULL; + return pte_offset_map_lock(mm, pmd, addr, ptl); +} + +static void test_mm_destroy(void *ptr) +{ + struct test_mm *ctx = ptr; + struct mm_struct *mm = ctx->mm; + struct mmu_gather tlb; + unsigned int i; + + if (!mm) + return; + mmap_write_lock(mm); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + + if (pte) { + /* Test-owned physical pages, not faulted/rmap-owned pages. */ + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + /* Exercise the real generic gather/teardown path, not a test walker. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(ctx->test, mm_pgtables_bytes(mm), 0UL); + mmput(mm); + ctx->mm = NULL; +} + +static unsigned int test_mm_backing_pages(struct test_mm *ctx) +{ + struct page *owners[3 * MM_TEST_MAPPINGS]; + unsigned int count = 0, total = 0, i, j, level; + struct mm_struct *mm = ctx->mm; + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + p4d_t *p4d = p4d_offset_mm(mm, pgd_offset(mm, addr), addr); + pud_t *pud = pud_offset_mm(mm, p4d, addr); + pmd_t *pmd = pmd_offset_mm(mm, pud, addr); + pte_t *pte = pte_offset_kernel_mm(mm, pmd, addr); + struct page *pages[] = { virt_to_page(pud), virt_to_page(pmd), virt_to_page(pte) }; + + for (level = mm_pud_folded(mm) ? 1 : 0; level < ARRAY_SIZE(pages); level++) { + pages[level] = compound_head(pages[level]); + for (j = 0; j < count; j++) + if (owners[j] == pages[level]) + break; + if (j == count) { + owners[count++] = pages[level]; + total += 1U << compound_order(pages[level]); + } + } + } + return total; +} + +/* Only valid for a private, empty mm with no ASID or hardware references. */ +static int test_mm_select_granule(struct mm_struct *mm, unsigned int shift) +{ + pgd_t *native_pgd = mm->pgd, *new_pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + mm->context.flags |= flag; + new_pgd = pgd_alloc(mm); + mm->context.flags &= ~flag; + if (!new_pgd) + return -ENOMEM; + pgd_free(mm, native_pgd); + mm->pgd = new_pgd; + mm->context.flags |= flag; + return 0; +} + +static int test_mm_select4k(struct mm_struct *mm) +{ + return test_mm_select_granule(mm, 12); +} + +static void test_mm_tables(struct kunit *test, unsigned int shift, bool hardware) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + bool small = shift != PAGE_SHIFT; + unsigned int pmd_shift = small ? shift + (shift - 3) : PMD_SHIFT; + unsigned int pud_shift = small ? shift + 2 * (shift - 3) : PUD_SHIFT; + unsigned int pgd_shift = small ? (shift == 16 ? pud_shift : + shift + 3 * (shift - 3)) : PGDIR_SHIFT; + unsigned long size = 1UL << shift; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * size; + ctx->address[4] = (1UL << pmd_shift) - size; + ctx->address[5] = 1UL << pmd_shift; + ctx->address[6] = (1UL << pud_shift) - size; + ctx->address[7] = 1UL << pud_shift; + ctx->address[8] = (1UL << pgd_shift) - size; + ctx->address[9] = 1UL << pgd_shift; + ctx->address[10] = (1UL << 48) - size; + if (pud_shift == pgd_shift) { + ctx->address[6] -= size; + ctx->address[7] += size; + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + void *data = test_alloc_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->phys[i] = virt_to_phys(data) + (shift < PAGE_SHIFT ? (i % (PAGE_SIZE >> shift)) * size : 0); + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + ctx->mm = mm; + /* Register after the data allocations: detach tables before freeing data. */ + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), (unsigned int)PAGE_SHIFT); + if (small) { + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)mm->pgd & (size - 1), 0UL); + } + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), shift); + if (small) + KUNIT_EXPECT_EQ(test, mm_pud_folded(mm), shift == 16); + KUNIT_EXPECT_EQ(test, mm_page_size(mm), size); + KUNIT_EXPECT_EQ(test, mm_page_mask(mm), ~(size - 1)); + KUNIT_EXPECT_EQ(test, mm_page_shift(&init_mm), (unsigned int)PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, pmd_addr_end_mm(mm, (1UL << pmd_shift) - 1, + (1UL << pmd_shift) + size), + 1UL << pmd_shift); + KUNIT_EXPECT_EQ(test, pgd_addr_end_mm(mm, ULONG_MAX - size, 0), 0UL); + + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, p4d); + pud = pud_alloc(mm, p4d, addr); + KUNIT_ASSERT_NOT_NULL(test, pud); + pmd = pmd_alloc(mm, pud, addr); + KUNIT_ASSERT_NOT_NULL(test, pmd); + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + /* Do not assert while holding the spinlock/RCU section. */ + set_pte(pte, __pte(ctx->phys[i] | TEST_PROT)); + pte_unmap_unlock(pte, ptl); + } + /* Installed table bytes and physical backing (including cached slots) differ. */ + { + unsigned int shifts[] = { pmd_shift, pud_shift, pgd_shift }; + unsigned int tables = 0, owners = 0; + + for (unsigned int level = 0; level < (mm_pud_folded(mm) ? 2 : 3); level++) { + unsigned int unique = 0; + + for (unsigned int a = 0; a < MM_TEST_MAPPINGS; a++) { + unsigned int b; + + for (b = 0; b < a; b++) + if ((ctx->address[a] >> shifts[level]) == + (ctx->address[b] >> shifts[level])) + break; + unique += b == a; + } + tables += unique; + if (shift < PAGE_SHIFT) + owners += DIV_ROUND_UP(unique, PAGE_SIZE >> shift); + else + owners += unique << (shift - PAGE_SHIFT); + } + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), tables * size); + KUNIT_EXPECT_EQ(test, test_mm_backing_pages(ctx), owners); + } + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + { + struct vm_area_struct *vma = vm_area_alloc(mm); + struct test_walk state = { .ctx = ctx, .size = size }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = 0; + vma->vm_end = 1UL << 48; + mmap_write_lock(mm); + ret = walk_page_range_vma(vma, 0, 1UL << 48, &test_walk_ops, &state); + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state.present, MM_TEST_MAPPINGS); + KUNIT_EXPECT_EQ(test, state.holes, (1UL << 48) - MM_TEST_MAPPINGS * size); + } + if (hardware) + test_mm_hardware(ctx); + { + struct mmu_gather tlb; + unsigned long before = mm_pgtables_bytes(mm); + unsigned long boundary = 1UL << pmd_shift; + + mmap_write_lock(mm); + /* An adjacent VMA's unaligned ceiling must keep the shared table. */ + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary - size, 0, boundary - size); + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before); + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte; + + if (ctx->address[i] >= boundary) + continue; + pte = test_mm_lookup(mm, ctx->address[i], &ptl); + if (pte) { + pte_clear(mm, ctx->address[i], pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, mm); + free_pgd_range(&tlb, 0, boundary, 0, boundary); + tlb_finish_mmu(&tlb); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, mm_pgtables_bytes(mm), before - size); + /* The next table and the higher levels must still be walkable. */ + for (i = 0; i < MM_TEST_MAPPINGS; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, ctx->address[i], &ptl); + u64 value; + + if (ctx->address[i] < boundary) { + if (pte) + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_PTR_EQ(test, pte, NULL); + continue; + } + KUNIT_ASSERT_NOT_NULL(test, pte); + value = pte_val(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, value, ctx->phys[i] | TEST_PROT); + } + } + test_mm_destroy(ctx); +} + +static void user4k_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, 12, false); +} + +static void native_mm_tables_test(struct kunit *test) +{ + test_mm_tables(test, PAGE_SHIFT, false); +} + +static void user16k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative tables require larger native backing"); + return; + } + test_mm_tables(test, 14, false); +} + +static void user64k_mm_tables_test(struct kunit *test) +{ + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "64K is already the native geometry"); + return; + } + test_mm_tables(test, 16, false); +} + +static void test_free_vma(void *ptr) +{ + vm_area_free(ptr); +} + +static void user4k_anon_read_fault_test(struct kunit *test) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + struct vm_area_struct *vma; + vm_flags_t vm_flags = VM_READ | VM_MAYREAD; + unsigned int i, j; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(ctx->mm), 0); + for (i = 0; i < 4; i++) + ctx->address[i] = TEST_VA + i * SZ_4K; + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_anonymous(vma); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + vm_flags |= VM_NOHUGEPAGE; +#endif + vm_flags_init(vma, vm_flags); + vma->vm_page_prot = vm_get_page_prot(vm_flags); + /* This VMA is a private fault fixture, not inserted into the VMA tree. */ + for (i = 0; i < 4; i++) { + vm_fault_t ret; + + mmap_read_lock(ctx->mm); + ret = handle_mm_fault(vma, ctx->address[i] + 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(ctx->mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + for (j = 0; j < 4; j++) { + spinlock_t *ptl; + pte_t *ptep = test_mm_lookup(ctx->mm, ctx->address[j], &ptl); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + pte_unmap_unlock(ptep, ptl); + if (j > i) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(ctx->mm, MM_ANONPAGES), 0L); +} + +static void user4k_mm_hardware_test(struct kunit *test) +{ + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + test_mm_tables(test, 12, true); +} + +struct fragment_fixture { + struct mm_struct *mm; + void *table[TEST_SLOTS + 2]; + struct page *pinned[3]; +}; + +static void fragment_fixture_destroy(void *data) +{ + struct fragment_fixture *f = data; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(f->table); i++) + if (f->table[i]) + arm64_pgtable_free(f->table[i]); + if (f->mm) + mmput(f->mm); + rcu_barrier(); + for (i = 0; i < ARRAY_SIZE(f->pinned); i++) + if (f->pinned[i]) + put_page(f->pinned[i]); +} + +static void test_fragment_lifetime(struct kunit *test, unsigned int shift) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct ptdesc *pte_owner; + unsigned int i, slots = PAGE_SIZE >> shift; + unsigned long size = 1UL << shift; + unsigned int read_shift = 0; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + /* Direct allocator fixture: the mm's native root is never changed or used. */ + for (i = 0; i < slots + 2; i++) { + enum arm64_user4k_pt_level level = i < slots ? USER4K_PT_PTE : i - slots + 1; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, level, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], size)); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i]), shift); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(f->table[i] + size - sizeof(pte_t)), shift); + KUNIT_EXPECT_EQ(test, pte_batch_hint(f->table[i], __pte(TEST_PROT | PTE_CONT)), + IS_ENABLED(CONFIG_ARM64_CONTPTE) ? arm64_cont_ptes(shift) : 1U); + if (!i && PAGE_SHIFT == 16) { + void *cached = f->mm->context.user4k_pt_frag[level]; + + KUNIT_EXPECT_PTR_EQ(test, arm64_user_pt_alloc_granule(f->mm, level, + shift == 12 ? 14 : 12), NULL); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[level], cached); + } + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + if (i == 0 || i >= slots) { + unsigned int slot = i == 0 ? 0 : i - slots + 1; + + f->pinned[slot] = virt_to_page(f->table[i]); + get_page(f->pinned[slot]); + } else { + KUNIT_EXPECT_PTR_EQ(test, virt_to_page(f->table[i]), f->pinned[0]); + KUNIT_EXPECT_EQ(test, (unsigned long)f->table[i] - + (unsigned long)f->table[0], i * (unsigned long)size); + } + memset(f->table[i], 0x50 + i, size); + } + for (i = 0; i < slots + 2; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x50 + i, size), NULL); + memset(f->table[i], 0, size); + } + /* Pointer-only walkers must inspect all 128 entries of a 16K group. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0]; + unsigned int nr = arm64_cont_ptes(shift), j; + pte_t observed; + + for (j = 0; j < nr; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << shift))))); + ptes[nr - 1] = pte_mkyoung(pte_mkdirty(ptes[nr - 1])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(size - 1)), + (u64)(SZ_4M + size)); + memset(ptes, 0, size); + } + KUNIT_EXPECT_PTR_NE(test, f->pinned[0], f->pinned[1]); + KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); + pte_owner = page_ptdesc(f->pinned[0]); + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + arm64_pte_free_defer(f->mm, f->table[0]); + f->table[0] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + for (i = 1; i < slots - 1; i++) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + /* A pre-existing software walker must survive the last fragment release. */ + rcu_read_lock(); + arm64_pgtable_free(f->table[slots - 1]); + f->table[slots - 1] = NULL; + live = folio_test_pgtable(page_folio(f->pinned[0])); + read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, read_shift, shift); + rcu_barrier(); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[0]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[0]), 1); + for (i = slots; i < slots + 2; i++) { + struct ptdesc *pt = virt_to_ptdesc(f->table[i]); + + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); + KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + } + mmput(f->mm); + f->mm = NULL; + for (i = 1; i < 3; i++) { + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + } +} + +/* Direct table-owner test; no larger-than-native user mm is enabled here. */ +static void user64k_table_owner_test(struct kunit *test) +{ + struct fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + unsigned int i, offset, order = 16 - PAGE_SHIFT; + unsigned int shift; + bool live; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, fragment_fixture_destroy, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + for (i = 0; i < 3; i++) { + struct ptdesc *pt; + + f->table[i] = arm64_user_pt_alloc_granule(f->mm, i, 16); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + pt = virt_to_ptdesc(f->table[i]); + f->pinned[i] = ptdesc_page(pt); + get_page(f->pinned[i]); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED((unsigned long)f->table[i], SZ_64K)); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 1); + KUNIT_EXPECT_PTR_EQ(test, f->mm->context.user4k_pt_frag[i], NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, SZ_64K), NULL); + /* Include the last entry of every constituent native page. */ + for (offset = PAGE_SIZE - sizeof(pte_t); offset < SZ_64K; offset += PAGE_SIZE) { + void *entry = f->table[i] + offset; + + KUNIT_EXPECT_PTR_EQ(test, virt_to_ptdesc(entry), pt); + KUNIT_EXPECT_EQ(test, arm64_ptep_page_shift(entry), 16U); + KUNIT_EXPECT_EQ(test, pte_batch_hint(entry, __pte(TEST_PROT | PTE_CONT)), + 32U - ((offset / sizeof(pte_t)) % 32)); + if (i == USER4K_PT_PTE) + KUNIT_EXPECT_PTR_EQ(test, ptep_lockptr(f->mm, entry), + ptep_lockptr(f->mm, f->table[i])); + if (i == USER4K_PT_PMD) + KUNIT_EXPECT_PTR_EQ(test, pmd_lockptr(f->mm, entry), + pmd_lockptr(f->mm, f->table[i])); + } + memset(f->table[i], 0x51 + i, SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0x51 + i, SZ_64K), NULL); + memset(f->table[i], 0, SZ_64K); + } + /* Contiguous readers on the last native page must still use 64K geometry. */ + if (IS_ENABLED(CONFIG_ARM64_CONTPTE)) { + pte_t *ptes = f->table[0] + SZ_64K - 32 * sizeof(pte_t); + pte_t observed; + unsigned int j; + + for (j = 0; j < 32; j++) + ptes[j] = pte_mkold(pte_mkclean(__pte(TEST_PROT | PTE_CONT | + (SZ_4M + (j << 16))))); + ptes[31] = pte_mkyoung(pte_mkdirty(ptes[31])); + observed = ptep_get(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + observed = ptep_get_lockless(ptes + 1); + KUNIT_EXPECT_TRUE(test, pte_young(observed)); + KUNIT_EXPECT_TRUE(test, pte_dirty(observed)); + KUNIT_EXPECT_EQ(test, pte_val(observed) & + (ARM64_USER4K_TABLE_ADDR_MASK & ~(SZ_64K - 1UL)), + (u64)(SZ_4M + SZ_64K)); + memset(f->table[0], 0, SZ_64K); + } + rcu_read_lock(); + arm64_pte_free_defer(f->mm, f->table[0]); + live = folio_test_pgtable(page_folio(f->pinned[0])); + shift = arm64_ptep_page_shift(f->table[0] + SZ_64K - sizeof(pte_t)); + f->table[0] = NULL; + rcu_read_unlock(); + KUNIT_EXPECT_TRUE(test, live); + KUNIT_EXPECT_EQ(test, shift, 16U); + rcu_barrier(); + for (i = 0; i < 3; i++) { + if (f->table[i]) { + arm64_pgtable_free(f->table[i]); + f->table[i] = NULL; + } + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(page_folio(f->pinned[i]))); + KUNIT_EXPECT_EQ(test, page_count(f->pinned[i]), 1); + KUNIT_EXPECT_EQ(test, compound_order(f->pinned[i]), order); + } +} + +static void user4k_fragment_lifetime_test(struct kunit *test) +{ + test_fragment_lifetime(test, 12); +} + +static void user16k_fragment_lifetime_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K fragments require larger native backing"); + return; + } + test_fragment_lifetime(test, 14); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void test_put_mm(void *ptr) +{ + mmput(ptr); +} + +static void native_pgtable_deposit_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + pgtable_t first, second, out1, out2; + pmd_t *pmd; + spinlock_t *ptl; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_put_mm, mm), 0); + pmd = pmd_alloc_one(mm, 0); + KUNIT_ASSERT_NOT_NULL(test, pmd); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, pmd), 0); + first = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, first), 0); + second = pte_alloc_one(mm); + KUNIT_ASSERT_NOT_NULL(test, second); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, arm64_pgtable_free, second), 0); + ptl = pmd_lock(mm, pmd); + pgtable_trans_huge_deposit(mm, pmd, first); + pgtable_trans_huge_deposit(mm, pmd, second); + out1 = pgtable_trans_huge_withdraw(mm, pmd); + out2 = pgtable_trans_huge_withdraw(mm, pmd); + spin_unlock(ptl); + KUNIT_EXPECT_PTR_EQ(test, out1, second); + KUNIT_EXPECT_PTR_EQ(test, out2, first); +} +#endif + +static void test_data_slot_put(void *ptr) +{ + mm_subpage_put(ptr); +} + +static void test_data_pool_free(void *ptr) +{ + mm_subpage_pool_close(ptr); + mm_subpage_pool_put(ptr); +} + +static bool test_alloc_data_slots(struct kunit *test, struct test_mm *ctx) +{ + struct mm_subpage_pool *pool = mm_subpage_pool_create(GFP_KERNEL); + struct folio *folio; + unsigned int i; + int ret; + + if (!pool) + return false; + if (kunit_add_action_or_reset(test, test_data_pool_free, pool)) + return false; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return false; + ret = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return false; + } + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc(pool); + + if (IS_ERR(slot)) + return false; + if (kunit_add_action_or_reset(test, test_data_slot_put, slot)) + return false; + ctx->phys[i] = mm_subpage_phys(slot); + } + return true; +} + +/* These private mappings are borrowed only for bounded, fault-disabled uaccess. */ +static struct test_mm *test_usercopy_mm(struct kunit *test, bool small) +{ + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned int i; + + if (!ctx) + return NULL; + ctx->test = test; + if (small && !test_alloc_data_slots(test, ctx)) + return NULL; + for (i = 0; i < 4; i++) { + if (!small) { + void *data = test_alloc_page(test); + + if (!data) + return NULL; + ctx->phys[i] = virt_to_phys(data); + } + ctx->address[i] = TEST_VA + i * size; + memset(phys_to_virt(ctx->phys[i]), (small ? 0x40 : 0x80) + i, 64); + } + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; + if (kunit_add_action_or_reset(test, test_mm_destroy, ctx)) + return NULL; + if (small && test_mm_select4k(ctx->mm)) + return NULL; + for (i = 0; i < 4; i++) { + unsigned long addr = ctx->address[i]; + pgd_t *pgd = pgd_offset(ctx->mm, addr); + p4d_t *p4d = p4d_alloc(ctx->mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + if (!p4d) + return NULL; + pud = pud_alloc(ctx->mm, p4d, addr); + if (!pud) + return NULL; + pmd = pmd_alloc(ctx->mm, pud, addr); + if (!pmd) + return NULL; + pte = pte_alloc_map_lock(ctx->mm, pmd, addr, &ptl); + if (!pte) + return NULL; + if (i != 2) + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[i], + __pgprot(TEST_PROT | (i ? 0 : PTE_RDONLY)))); + pte_unmap_unlock(pte, ptl); + } + return ctx; +} + +static void user4k_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, j, sync; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + /* A later alias must not extend a native-owner range walk's bound. */ + ctx->address[4] = TEST_VA + PAGE_SIZE; + ctx->phys[4] = ctx->phys[0]; + pte = test_mm_lookup(ctx->mm, ctx->address[4], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[4], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + /* Same physical slot at a different virtual byte offset and native index. */ + ctx->address[5] = TEST_VA + PAGE_SIZE + SZ_4K; + ctx->phys[5] = ctx->phys[3]; + pte = test_mm_lookup(ctx->mm, ctx->address[5], &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, phys_pte_mm(ctx->mm, ctx->phys[5], __pgprot(TEST_PROT))); + pte_unmap_unlock(pte, ptl); + for (sync = 0; sync < 2; sync++) { + struct page_vma_mapped_walk whole = { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = TEST_VA, + .flags = sync ? PVMW_SYNC : 0, + }; + unsigned int count = 0; + bool valid = true; + + /* A native-owner walk must visit every mapped 4K occupant. */ + while (page_vma_mapped_walk(&whole)) { + unsigned int index = (whole.address - TEST_VA) / SZ_4K; + + if (index >= 4 || index == 2 || !whole.pte || + pte_phys_mm(ctx->mm, ptep_get(whole.pte)) != ctx->phys[index]) + valid = false; + if (++count > 4) { + page_vma_mapped_walk_done(&whole); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + for (i = 0; i < 4; i++) { + for (j = 0; j < 6; j++) { + struct rmap_walk_range range = { + .address = ctx->address[j], + .subpage = true, + .subpage_offset = offset_in_page(ctx->phys[i]), + }; + DEFINE_FOLIO_RMAP_WALK(one, page_folio(pfn_to_page(PHYS_PFN(ctx->phys[i]))), + vma, range, sync ? PVMW_SYNC : 0); + struct mm_subpage *slot = NULL; + bool found = page_vma_mapped_walk(&one); + phys_addr_t phys = 0; + + if (found) { + phys = pte_phys_mm(ctx->mm, ptep_get(one.pte)); + slot = mm_subpage_get_from_phys(phys); + page_vma_mapped_walk_done(&one); + } + KUNIT_EXPECT_EQ(test, found, (i == j && j != 2) || + (j == 4 && i == 0) || (j == 5 && i == 3)); + if (found) { + KUNIT_EXPECT_EQ(test, phys, ctx->phys[i]); + KUNIT_EXPECT_NOT_NULL(test, slot); + if (slot) { + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + mm_subpage_put(slot); + } + } + } + } + } +} + +static int test_map_alias(struct test_mm *ctx, unsigned int index, + unsigned long addr, phys_addr_t phys) +{ + struct mm_struct *mm = ctx->mm; + pgd_t *pgd = pgd_offset(mm, addr); + p4d_t *p4d = p4d_alloc(mm, pgd, addr); + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + spinlock_t *ptl; + + ctx->address[index] = addr; + ctx->phys[index] = phys; + if (!p4d) + return -ENOMEM; + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return -ENOMEM; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return -ENOMEM; + pte = pte_alloc_map_lock(mm, pmd, addr, &ptl); + if (!pte) + return -ENOMEM; + set_pte(pte, pte_mkspecial(phys_pte_mm(mm, phys, __pgprot(TEST_PROT)))); + pte_unmap_unlock(pte, ptl); + return 0; +} + +/* Physical-range lookup only; coarse anonymous ownership is a separate step. */ +static void user64k_rmap_lookup_test(struct kunit *test) +{ + struct test_mm *ctx; + struct vm_area_struct *vma; + void *data; + unsigned long pfn, per_leaf = SZ_64K / PAGE_SIZE; + unsigned int mode, sync, i; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ctx); + ctx->test = test; + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + pfn = virt_to_phys(data) >> PAGE_SHIFT; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, 16), 0); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = (1UL << 29) - SZ_64K; + vma->vm_end = vma->vm_start + 4 * SZ_64K; + vma->vm_pgoff = 4096; + page_folio(virt_to_page(data))->index = vma->vm_pgoff; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, vma->vm_start + i * SZ_64K, + virt_to_phys(data) + (i == 1 ? SZ_64K : 0)), 0); + + for (mode = 0; mode < 2; mode++) { + if (mode) { + for (i = 0; i < 3; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, ctx->address[i], &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte(pte, swp_entry_to_pte(make_readable_migration_entry( + ctx->phys[i] >> PAGE_SHIFT))); + pte_unmap_unlock(pte, ptl); + } + } + for (sync = 0; sync < 2; sync++) { + unsigned int flags = (sync ? PVMW_SYNC : 0) | + (mode ? PVMW_MIGRATION : 0); + + for (i = 0; i < 2 * per_leaf; i++) { + struct page_vma_mapped_walk walk = { + .pfn = pfn + i, .nr_pages = 1, + /* Deliberately invalid pgoff: single-page callers may omit it. */ + .pgoff = 0, .vma = vma, + .address = vma->vm_start + i * PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + valid &= walk.pte && walk.address == + vma->vm_start + (i / per_leaf) * SZ_64K; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 1U); +#ifdef CONFIG_MEMORY_FAILURE + if (!mode) + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(pfn + i), vma), + vma->vm_start + i * PAGE_SIZE); +#endif + } + /* Two native pages straddle a coarse-leaf and PTE-table boundary. */ + { + struct page_vma_mapped_walk walk = { + .pfn = pfn + per_leaf - 1, .nr_pages = 2, + .pgoff = vma->vm_pgoff + per_leaf - 1, .vma = vma, + .address = vma->vm_start + SZ_64K - PAGE_SIZE, + .flags = flags, + }; + unsigned int count = 0; + bool valid = true; + + KUNIT_EXPECT_EQ(test, vma_address_end(&walk), vma->vm_start + 2 * SZ_64K); + while (page_vma_mapped_walk(&walk)) { + valid &= walk.address == vma->vm_start + count * SZ_64K; + if (++count > 2) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 2U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); + } + for (i = 0; i < 2; i++) { + struct page_vma_mapped_walk walk = { + .pfn = i ? pfn + 2 * per_leaf : pfn - 1, + .nr_pages = 1, .vma = vma, + .address = vma->vm_start, .flags = flags, + }; + bool found = page_vma_mapped_walk(&walk); + + if (found) + page_vma_mapped_walk_done(&walk); + KUNIT_EXPECT_FALSE(test, found); + } + } + } +} + +static void user4k_rmap_boundary_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, true); + struct vm_area_struct *vma; + unsigned int i, count = 0; + struct page_vma_mapped_walk walk; + bool valid = true; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = SZ_2M - SZ_4K; + vma->vm_end = vma->vm_start + PAGE_SIZE; + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, + ctx->phys[i]), 0); + walk = (struct page_vma_mapped_walk) { + .pfn = ctx->phys[0] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = vma->vm_start, + .flags = PVMW_SYNC, + }; + while (page_vma_mapped_walk(&walk)) { + if (count >= 3 || walk.address != vma->vm_start + count * SZ_4K || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[count]) + valid = false; + if (++count > 3) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +} + +static void native_rmap_walk_test(struct kunit *test) +{ + struct test_mm *ctx = test_usercopy_mm(test, false); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + vma = vm_area_alloc(ctx->mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 4 * PAGE_SIZE; + for (i = 0; i < 4; i++) { + struct page_vma_mapped_walk walk = { + .pfn = ctx->phys[i] >> PAGE_SHIFT, + .nr_pages = 1, + .vma = vma, + .address = ctx->address[i], + .flags = PVMW_SYNC, + }; + unsigned int count = 0; + bool valid = true; + + while (page_vma_mapped_walk(&walk)) { + if (walk.address != ctx->address[i] || + pte_phys_mm(ctx->mm, ptep_get(walk.pte)) != ctx->phys[i]) + valid = false; + if (++count > 1) { + page_vma_mapped_walk_done(&walk); + break; + } + } + KUNIT_EXPECT_TRUE(test, valid); + KUNIT_EXPECT_EQ(test, count, i != 2 ? 1U : 0U); +#ifdef CONFIG_MEMORY_FAILURE + page_folio(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT))->index = i; + KUNIT_EXPECT_EQ(test, page_mapped_in_vma(pfn_to_page(ctx->phys[i] >> PAGE_SHIFT), vma), + i != 2 ? ctx->address[i] : (unsigned long)-EFAULT); +#endif + } +} + +static void user4k_vma_offsets_test(struct kunit *test) +{ + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int slot, length, step; + + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_start = (1UL << 48) - 8 * PAGE_SIZE; + for (slot = 0; slot < TEST_SLOTS; slot++) { + for (length = 1; length <= 8; length++) { + struct vm_page_offset initial = { ULONG_MAX - 3, slot * SZ_4K }; + struct vm_page_offset pos; + + vma->vm_end = vma->vm_start + length * SZ_4K; + vma_set_page_offset(vma, initial); + KUNIT_EXPECT_EQ(test, vma_last_pgoff(vma), + initial.index + (slot + length - 1) / TEST_SLOTS); + for (step = 0; step < length; step++) { + unsigned long addr = vma->vm_start + step * SZ_4K; + + pos = vma_page_offset_at(vma, addr); + KUNIT_EXPECT_EQ(test, pos.index, initial.index + (slot + step) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, pos.offset, ((slot + step) % TEST_SLOTS) * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, linear_page_index(vma, addr), pos.index); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), addr); + } + pos = vma_page_offset_at(vma, vma->vm_end); + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + pos = initial; + pos.index--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + if (slot) { + pos = initial; + pos.offset--; + KUNIT_EXPECT_EQ(test, vma_address_at_offset(vma, pos), (unsigned long)-EFAULT); + } + } + } +} + +static void user4k_vma_intervals_test(struct kunit *test) +{ + struct vm_area_struct *vmas = kunit_kcalloc(test, 3, sizeof(*vmas), GFP_KERNEL); + struct rb_root_cached root = RB_ROOT_CACHED; + struct vm_area_struct *vma; + unsigned int i, count = 0; + + KUNIT_ASSERT_NOT_NULL(test, vmas); + for (i = 0; i < 3; i++) { + vmas[i].vm_start = TEST_VA + i * SZ_8K; + vmas[i].vm_end = vmas[i].vm_start + SZ_4K; + vma_set_page_offset(&vmas[i], (struct vm_page_offset) { 77, i * SZ_4K }); + vma_interval_tree_insert(&vmas[i], &root); + } + vma_interval_tree_foreach(vma, &root, 77, 77) { + unsigned long address = vma_address_at_offset(vma, + (struct vm_page_offset) { 77, SZ_4K }); + + KUNIT_EXPECT_EQ(test, address, + vma == &vmas[1] ? vma->vm_start : (unsigned long)-EFAULT); + count++; + } + KUNIT_EXPECT_EQ(test, count, 3U); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 76, 76), NULL); + KUNIT_EXPECT_PTR_EQ(test, vma_interval_tree_iter_first(&root, 78, 78), NULL); + for (i = 0; i < 3; i++) + vma_interval_tree_remove(&vmas[i], &root); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&root.rb_root)); +} + +static void user4k_vma_mm_free(void *arg) +{ + mmput(arg); +} + +/* These private mms have no concurrent VMA writers after dropping the lock. */ +static struct vm_area_struct *test_vma_lookup(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + + mmap_read_lock(mm); + vma = vma_lookup(mm, addr); + mmap_read_unlock(mm); + return vma; +} + +static void user4k_vma_split_test(struct kunit *test) +{ + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma, *middle, *part; + unsigned long start = TEST_VA + SZ_4K; + vma_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select4k(mm), 0); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Dummy vm_ops keep insert_vm_struct from replacing this file-like offset. */ + vma->vm_start = start; + vma->vm_end = start + 3 * SZ_4K; + vm_flags_init(vma, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, (struct vm_page_offset) { 77, 3 * SZ_4K }); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + if (ret) { + mmap_write_unlock(mm); + vm_area_free(vma); + KUNIT_FAIL(test, "insert_vm_struct: %d", ret); + return; + } + { + VMA_ITERATOR(vmi, mm, start); + + vma_iter_load(&vmi); + flags = vma->flags; + vma_flags_set(&flags, VMA_WRITE_BIT); + middle = vma_modify_flags(&vmi, vma, vma, start + SZ_4K, + start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, middle); + KUNIT_EXPECT_EQ(test, mm->map_count, 3); + for (i = 0; i < 3; i++) { + part = test_vma_lookup(mm, start + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, part); + KUNIT_EXPECT_EQ(test, part->vm_start, start + i * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_end, start + (i + 1) * SZ_4K); + KUNIT_EXPECT_EQ(test, part->vm_pgoff, 77UL + (i + 3) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(part), ((i + 3) % TEST_SLOTS) * (unsigned int)SZ_4K); + } +} + +static struct mm_struct *test_vma_mm_granule(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm = mm_alloc(); + + if (!mm) + return NULL; + if (kunit_add_action_or_reset(test, user4k_vma_mm_free, mm)) + return NULL; + if (test_mm_select_granule(mm, shift)) + return NULL; + return mm; +} + +static struct mm_struct *test_vma_mm(struct kunit *test) +{ + return test_vma_mm_granule(test, 12); +} + +static struct vm_area_struct *test_vma_add(struct mm_struct *mm, unsigned long start, + unsigned long len, struct vm_page_offset pos, + vm_flags_t flags) +{ + struct vm_area_struct *vma = vm_area_alloc(mm); + int ret; + + if (!vma) + return ERR_PTR(-ENOMEM); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_offset(vma, pos); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return ERR_PTR(ret); + } + return vma; +} + +static void user4k_vma_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + unsigned int scenario, i; + + for (scenario = 0; scenario < 4; scenario++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vmas[3], *merged; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + struct vm_page_offset expected; + unsigned long left = scenario >= 2 ? SZ_4K : 0; + unsigned long right = scenario == 1 || scenario == 3 ? 2 * SZ_4K : 3 * SZ_4K; + vma_flags_t flags; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (i = 0; i < 3; i++) { + struct vm_page_offset pos = vm_page_offset_add(initial, i * SZ_4K); + vm_flags_t bits = VM_READ; + + if (i == 1) + bits |= VM_WRITE; + if ((!i && scenario >= 2) || (i == 2 && scenario == 1)) + bits |= VM_EXEC; + /* Same native index, wrong byte offset: this must not merge. */ + if (i == 2 && scenario == 3) + pos.offset = 0; + vmas[i] = test_vma_add(mm, start + i * SZ_4K, SZ_4K, pos, bits); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + } + flags = vmas[1]->flags; + vma_flags_clear(&flags, VMA_WRITE_BIT); + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, vmas[0], vmas[1], start + SZ_4K, + start + 2 * SZ_4K, &flags); + if (!IS_ERR(merged)) { + vma_start_write(merged); + merged->flags = flags; + } + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + expected = vm_page_offset_add(initial, left); + KUNIT_EXPECT_EQ(test, mm->map_count, scenario == 0 ? 1 : scenario == 3 ? 3 : 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + left); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + right); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + } +} + +static void user4k_vma_partial_merge_test(struct kunit *test) +{ + unsigned long start = TEST_VA; + unsigned int side; + + for (side = 0; side < 2; side++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_page_offset initial = { 77, 2 * SZ_4K }; + struct vm_area_struct *first, *second, *merged, *remaining; + unsigned long first_len = side ? SZ_4K : 2 * SZ_4K; + vma_flags_t flags; + struct vm_page_offset expected; + + KUNIT_ASSERT_NOT_NULL(test, mm); + first = test_vma_add(mm, start, first_len, initial, VM_READ | (side ? 0 : VM_WRITE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + second = test_vma_add(mm, start + first_len, 3 * SZ_4K - first_len, + vm_page_offset_add(initial, first_len), VM_READ | (side ? VM_WRITE : 0)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, second); + flags = side ? first->flags : second->flags; + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, start + SZ_4K); + + vma_iter_load(&vmi); + merged = vma_modify_flags(&vmi, first, side ? second : first, + start + SZ_4K, start + 2 * SZ_4K, &flags); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, merged); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, merged->vm_start, start + (side ? 0 : SZ_4K)); + KUNIT_EXPECT_EQ(test, merged->vm_end, start + (side ? 2 : 3) * SZ_4K); + expected = vm_page_offset_add(initial, side ? 0 : SZ_4K); + KUNIT_EXPECT_EQ(test, merged->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(merged), expected.offset); + remaining = test_vma_lookup(mm, start + (side ? 2 * SZ_4K : 0)); + KUNIT_ASSERT_NOT_NULL(test, remaining); + expected = vm_page_offset_add(initial, side ? 2 * SZ_4K : 0); + KUNIT_EXPECT_EQ(test, remaining->vm_pgoff, expected.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(remaining), expected.offset); + } +} + +static void user4k_vma_copy_shrink_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *source, *dest; + struct vm_page_offset initial = { 77, 3 * SZ_4K }; + unsigned long start = TEST_VA + SZ_4K; + unsigned long target = TEST_VA + 16 * PAGE_SIZE + SZ_4K; + bool need_locks = false; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + source = test_vma_add(mm, start, PAGE_SIZE, initial, VM_READ); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, source); + for (i = 0; i < 3; i++) { + unsigned long addr = i == 2 ? target - SZ_4K : target + i * SZ_4K; + struct vm_page_offset pos = vm_page_offset_add(initial, i == 2 ? 0 : (i + 1) * SZ_4K); + + mmap_write_lock(mm); + dest = copy_vma(&source, addr, SZ_4K, pos, &need_locks); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, dest); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + KUNIT_EXPECT_EQ(test, dest->vm_start, i == 2 ? target - SZ_4K : target); + KUNIT_EXPECT_EQ(test, dest->vm_end, i ? target + 2 * SZ_4K : target + SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + (i == 2 ? 3UL : 4UL) / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), ((i == 2 ? 3U : 4U) % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, source->vm_pgoff, initial.index); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(source), initial.offset); + } + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, dest->vm_start); + struct vm_page_offset pos = vma_page_offset_at(dest, target); + + vma_iter_load(&vmi); + ret = vma_shrink(&vmi, dest, target, dest->vm_end, pos); + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, dest->vm_start, target); + KUNIT_EXPECT_EQ(test, dest->vm_end, target + 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, dest->vm_pgoff, 77UL + 4 / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(dest), (4 % TEST_SLOTS) * SZ_4K); +} + +struct anon_slot_fixture { + struct mm_struct *mm; + struct vm_area_struct *vma; + struct mm_subpage_pool *pool; + struct mm_subpage *slots[4]; + bool mapped[4]; +}; + +static void anon_slot_fixture_free(void *arg) +{ + struct anon_slot_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->pool) + mm_subpage_pool_close(f->pool); + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (!pte) + continue; + ptep_get_and_clear_full(f->mm, addr, pte, 0); + if (f->mapped[i]) { + struct vm_area_struct *vma = vma_lookup(f->mm, addr); + + mm_subpage_remove_anon_rmap(f->slots[i], vma); + dec_mm_counter(f->mm, MM_ANONPAGES); + f->mapped[i] = false; + } + pte_unmap_unlock(pte, ptl); + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + } + /* No slot reference is released before the PTEs and TLBs are retired. */ + for (i = 0; i < 4; i++) + if (f->slots[i]) + mm_subpage_put(f->slots[i]); + if (f->pool) + mm_subpage_pool_put(f->pool); + if (f->mm) + mmput(f->mm); + memset(f, 0, sizeof(*f)); +} + +static struct anon_slot_fixture *anon_slot_fixture_create_from(struct kunit *test, + struct anon_slot_fixture *parent) +{ + struct anon_slot_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct *vma; + struct folio *folio; + int ret; + + if (!f || kunit_add_action_or_reset(test, anon_slot_fixture_free, f)) + return NULL; + f->mm = mm_alloc(); + if (!f->mm || test_mm_select4k(f->mm)) + return NULL; + f->pool = mm_subpage_pool_create(GFP_KERNEL); + if (!f->pool) + return NULL; + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) + return NULL; + ret = mm_subpage_pool_add_folio(f->pool, folio, GFP_KERNEL); + if (ret) { + folio_put(folio); + return NULL; + } + vma = vm_area_alloc(f->mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * PAGE_SIZE; + vm_flags_init(vma, VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE); + vma_set_page_prot(vma); + if (parent) { + mmap_write_lock(parent->mm); + mmap_write_lock_nested(f->mm, SINGLE_DEPTH_NESTING); + vma_set_page_offset(vma, vma_page_offset_at(parent->vma, TEST_VA)); + ret = anon_vma_fork(vma, parent->vma); + } else { + mmap_write_lock(f->mm); + ret = 0; + } + if (!ret && parent) { + VMA_ITERATOR(vmi, f->mm, TEST_VA); + + vma_start_write(vma); + ret = vma_iter_store_gfp(&vmi, vma, GFP_KERNEL); + if (!ret) + f->mm->map_count++; + } else if (!ret) { + ret = insert_vm_struct(f->mm, vma); + } + if (!ret) { + f->vma = vma; + ret = anon_vma_prepare(vma); + } + if (!f->vma) + unlink_anon_vmas(vma); + mmap_write_unlock(f->mm); + if (parent) + mmap_write_unlock(parent->mm); + if (!f->vma) + vm_area_free(vma); + return ret ? NULL : f; +} + +static struct anon_slot_fixture *anon_slot_fixture_create(struct kunit *test) +{ + return anon_slot_fixture_create_from(test, NULL); +} + +static void user4k_anon_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *folio = NULL; + unsigned int i; + vm_flags_t flags; + int ret, bad_offset = 0, bad_index = 0; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + f->slots[i] = slot; + folio = mm_subpage_folio(slot); + /* Allocate the table first; clear the test leaf before rmap attachment. */ + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)), 0); + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing allocated PTE"); + return; + } + pte_clear(f->mm, addr, pte); + if (!i) + bad_offset = mm_subpage_add_anon_rmap(slot, f->vma, addr + 1); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[i] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + if (!i) + bad_index = mm_subpage_add_anon_rmap(slot, f->vma, addr + PAGE_SIZE); + } + pte_unmap_unlock(pte, ptl); + folio_unlock(folio); + mmap_read_unlock(f->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)i + 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)i + 1); + } + KUNIT_EXPECT_EQ(test, bad_offset, -EINVAL); + KUNIT_EXPECT_EQ(test, bad_index, -EINVAL); + KUNIT_EXPECT_EQ(test, folio->index, TEST_VA >> PAGE_SHIFT); + KUNIT_EXPECT_FALSE(test, PageAnonExclusive(&folio->page)); + KUNIT_EXPECT_EQ(test, folio_expected_ref_count(folio), folio_ref_count(folio)); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 1); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool young; + + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + } + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +/* The fixture takes one existing slot reference, including on failure. */ +static int anon_slot_fixture_map(struct anon_slot_fixture *f, + unsigned int index, struct mm_subpage *slot) +{ + unsigned long addr = TEST_VA + index * SZ_4K; + struct test_mm tables = { .mm = f->mm }; + struct folio *folio = mm_subpage_folio(slot); + spinlock_t *ptl; + pte_t *pte; + int ret; + + f->slots[index] = slot; + ret = test_map_alias(&tables, 0, addr, mm_subpage_phys(slot)); + if (ret) + return ret; + mmap_read_lock(f->mm); + folio_lock(folio); + pte = test_mm_lookup(f->mm, addr, &ptl); + if (!pte) { + ret = -ENOMEM; + goto unlock; + } + pte_clear(f->mm, addr, pte); + ret = mm_subpage_add_anon_rmap(slot, f->vma, addr); + if (!ret) { + f->mapped[index] = true; + inc_mm_counter(f->mm, MM_ANONPAGES); + set_pte_at(f->mm, addr, pte, phys_pte_mm(f->mm, mm_subpage_phys(slot), + __pgprot(TEST_PROT | PTE_RDONLY))); + } + pte_unmap_unlock(pte, ptl); +unlock: + folio_unlock(folio); + mmap_read_unlock(f->mm); + return ret; +} + +static void user4k_anon_fork_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct anon_slot_fixture *child; + struct mm_subpage *slot; + struct folio *folio; + vm_flags_t flags; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = anon_slot_fixture_create_from(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(child, i, slot), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + } + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(child->slots[i]), 1); + /* The child mappings and root remain valid after the parent disappears. */ + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_nonlinear_rmap_reuse_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct anon_slot_fixture *c = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct folio *folio; + struct anon_vma_chain *avc; + phys_addr_t phys; + vm_flags_t flags; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, c); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio = mm_subpage_folio(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + phys = mm_subpage_phys(slot); +#ifdef CONFIG_FAILSLAB + { + unsigned int pending; + + mmap_read_lock(b->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, b->vma, TEST_VA + SZ_4K); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + /* Keep cleanup valid even if allocation unexpectedly succeeds. */ + if (!ret) + mm_subpage_remove_anon_rmap(slot, b->vma); + folio_unlock(folio); + mmap_read_unlock(b->mm); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + } +#endif + /* Allocation failure left the slot available for the same operation. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + anon_slot_fixture_free(b); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_phys(slot), phys); + + /* Give this empty VMA a different logical native index as well. */ + mmap_write_lock(c->mm); + anon_vma_lock_write(c->vma->anon_vma); + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root); + c->vma->vm_pgoff += 9; + list_for_each_entry(avc, &c->vma->anon_vma_chain, same_vma) + anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root); + anon_vma_unlock_write(c->vma->anon_vma); + mmap_write_unlock(c->mm); +#ifdef CONFIG_FAILSLAB + /* Retiring a previous root also needs failure-atomic bookkeeping. */ + mmap_read_lock(c->mm); + folio_lock(folio); + WRITE_ONCE(current->fail_nth, 1); + ret = mm_subpage_add_anon_rmap(slot, c->vma, TEST_VA + 2 * SZ_4K); + WRITE_ONCE(current->fail_nth, 0); + if (!ret) + mm_subpage_remove_anon_rmap(slot, c->vma); + folio_unlock(folio); + mmap_read_unlock(c->mm); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 1); +#endif + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(c, 2, slot), 0); + mmap_read_lock(c->mm); + KUNIT_EXPECT_EQ(test, page_address_in_vma(folio, &folio->page, c->vma), + TEST_VA + 2 * SZ_4K); + mmap_read_unlock(c->mm); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + /* The original root may disappear before the other slot's last mapping. */ + anon_slot_fixture_free(a); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(c->slots[2]), 1); +} + +static void user4k_dup_mmap_test(struct kunit *test) +{ + struct anon_slot_fixture *parent = anon_slot_fixture_create(test); + struct mm_struct *child; + struct folio *folio; + unsigned int i; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, parent); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(parent->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(parent, i, slot), 0); + pte = test_mm_lookup(parent->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(parent->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + pte_unmap_unlock(pte, ptl); + } + /* Split into four 4K VMAs plus a trailing hole, preserving COW flags. */ + mmap_write_lock(parent->mm); + for (i = 1; i < 4; i += 2) { + unsigned long addr = TEST_VA + i * SZ_4K; + VMA_ITERATOR(vmi, parent->mm, addr); + struct vm_area_struct *vma = vma_iter_load(&vmi); + vma_flags_t flags = vma->flags; + + vma_flags_set(&flags, VMA_DONTDUMP_BIT); + vma = vma_modify_flags(&vmi, vma, vma, addr, addr + SZ_4K, &flags); + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + break; + } + } + mmap_write_unlock(parent->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, parent->mm->map_count, 5); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, child->map_count, 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + folio = mm_subpage_folio(parent->slots[0]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 8); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 8); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + struct vm_area_struct *vma = test_vma_lookup(child, addr); + unsigned int side; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(parent->slots[i]), 2); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->mm; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry; + + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), + mm_subpage_phys(parent->slots[i])); + } + } + anon_slot_fixture_free(parent); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 4); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + /* KUnit's child-mm cleanup now exercises ordinary exit_mmap(). */ +} + +static void test_subpage_put(void *arg) +{ + mm_subpage_put(arg); +} + +static void user4k_anon_root_lifetime_test(struct kunit *test) +{ + struct anon_slot_fixture *old = anon_slot_fixture_create(test); + struct anon_slot_fixture *other = anon_slot_fixture_create(test); + struct mm_subpage *slot; + struct anon_vma *root; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, other); + slot = mm_subpage_alloc_at(old->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(old, 0, slot), 0); + folio = mm_subpage_folio(slot); + root = old->vma->anon_vma->root; + mm_subpage_get(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slot), 0); + anon_slot_fixture_free(old); + /* Only the owner now pins the root; no VMA or PTE survives. */ + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + mm_subpage_get(slot); + ret = anon_slot_fixture_map(other, 0, slot); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); +} + +static void test_folio_put(void *arg) +{ + folio_put(arg); +} + +static void user4k_data_tlb_batch_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct folio *native = folio_alloc(GFP_KERNEL, 2); + struct mmu_gather tlb; + struct folio *backing; + unsigned int i; + bool full = false; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, native), 0); + for (i = 0; i < 4; i++) { + f->slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + if (IS_ERR(f->slots[i])) { + f->slots[i] = NULL; + KUNIT_FAIL(test, "slot allocation failed"); + return; + } + } + backing = mm_subpage_folio(f->slots[0]); + memset(folio_address(backing), 0x5a, SZ_4K); + tlb_gather_mmu(&tlb, f->mm); + tlb_change_page_size(&tlb, MM_SUBPAGE_SIZE); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + for (i = 0; i < 64; i++) { + mm_subpage_get(f->slots[i % 4]); + if (__tlb_remove_subpage(&tlb, f->slots[i % 4])) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + if (i == 12) { + /* Exercise the native count payload (3 << 2 has tag bit 2). */ + folio_ref_add(native, 3); + if (__tlb_remove_folio_pages(&tlb, &native->page, 3, false)) { + full = true; + tlb_flush_mmu(&tlb); + tlb_flush_pte_range(&tlb, TEST_VA, PAGE_SIZE); + } + } + } + mm_subpage_put(f->slots[0]); + f->slots[0] = NULL; + /* Without memory pressure these fit in the local plus one dynamic batch. */ + if (!full) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, 0); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 67); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 4); + KUNIT_EXPECT_GT(test, tlb.batch_count, 0U); + } + tlb_finish_mmu(&tlb); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(native), 1); + f->slots[0] = mm_subpage_alloc_at(f->pool, 0); + if (IS_ERR(f->slots[0])) { + f->slots[0] = NULL; + KUNIT_FAIL(test, "slot was not reusable after TLB completion"); + return; + } + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(f->slots[0]), backing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(backing), 0, SZ_4K), NULL); + for (i = 0; i < 4; i++) { + struct mm_subpage *extra = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_EXPECT_EQ(test, PTR_ERR(extra), -EAGAIN); + if (!IS_ERR(extra)) + mm_subpage_put(extra); + } +} + +static void user4k_anon_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *slots[4]; + struct folio *folio; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + slots[i] = mm_subpage_alloc_at(f->pool, i * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slots[i]), 0); + mm_subpage_get(slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + } + folio = mm_subpage_folio(slots[0]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + f->slots[1] = NULL; + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 3); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 7); + for (i = 0; i < 4; i++) { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, TEST_VA + i * SZ_4K, &ptl); + bool present; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 0 : 1); + } + /* Include both the existing hole and a trailing run of empty entries. */ + mmap_read_lock(f->mm); + zap_vma(f->vma); + for (i = 0; i < 4; i++) { + f->mapped[i] = false; + f->slots[i] = NULL; + } + mmap_read_unlock(f->mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 4); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm, MM_ANONPAGES), 0UL); +} + +static void user4k_anon_writable_slots_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + unsigned long addr = TEST_VA + i * SZ_4K; + spinlock_t *ptl; + pte_t *pte; + bool writable; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + pte = test_mm_lookup(f->mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_pte_at(f->mm, addr, pte, + pte_mkwrite_novma(pte_mkdirty(ptep_get(pte)))); + writable = pte_write(ptep_get(pte)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + } + KUNIT_EXPECT_EQ(test, folio_mapcount(mm_subpage_folio(f->slots[0])), 4); +} + +static struct vm_area_struct *test_fault_vma_in_mm(struct mm_struct *mm, + unsigned long start, unsigned long len) +{ + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma_set_anonymous(vma); + vma->vm_start = start; + vma->vm_end = start + len; + vm_flags_init(vma, flags); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = insert_vm_struct(mm, vma); + mmap_write_unlock(mm); + if (ret) { + vm_area_free(vma); + return NULL; + } + return vma; +} + +struct coarse_anon_fixture { + struct mm_struct *mm[2]; + bool attached[2]; + struct vm_area_struct *vma[2]; + struct folio *folio; + unsigned long mapped[2]; + unsigned int leaves; + bool pinned; +}; + +static void coarse_anon_fixture_free(void *ptr) +{ + struct coarse_anon_fixture *f = ptr; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (f->pinned) { + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + } + for (i = 0; i < 2; i++) { + unsigned int leaf; + + if (!f->mm[i]) + continue; + mmap_write_lock(f->mm[i]); + for (leaf = 0; leaf < (f->leaves ?: 1); leaf++) { + unsigned long addr = TEST_VA + leaf * SZ_64K; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm[i], addr, &ptl); + + if (pte) { + ptep_get_and_clear_full(f->mm[i], addr, pte, 0); + if (f->mapped[i] & BIT(leaf)) { + folio_remove_rmap_pte(f->folio, + folio_page(f->folio, leaf * nr), f->vma[i]); + dec_mm_counter(f->mm[i], MM_ANONPAGES); + } + pte_unmap_unlock(pte, ptl); + } + if (f->mapped[i] & BIT(leaf)) { + folio_put_refs(f->folio, nr); + f->mapped[i] &= ~BIT(leaf); + } + } + mmap_write_unlock(f->mm[i]); + if (f->vma[i] && !f->attached[i]) { + unlink_anon_vmas(f->vma[i]); + vm_area_free(f->vma[i]); + } + mmput(f->mm[i]); + f->vma[i] = NULL; + f->mm[i] = NULL; + } + if (f->folio) { + folio_put(f->folio); + f->folio = NULL; + } +} + +static void expect_coarse_anon_state(struct kunit *test, struct folio *folio, + unsigned int maps, bool exclusive) +{ + unsigned int i, nr = folio_nr_pages(folio); + + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps * nr)); + for (i = 0; i < nr; i++) { + struct page *page = folio_page(folio, i); + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, (atomic_read(&page->_mapcount) + 1), (int)maps); + KUNIT_EXPECT_EQ(test, PageAnonExclusive(page), exclusive); + } +} + +static void user64k_anon_rmap_span_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + pte_t entry, *pte; + spinlock_t *ptl; + int ret, dup_ret, share_ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaf"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + for (i = 0; i < 2; i++) { + struct test_mm tables = {}; + + f->mm[i] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm[i]); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm[i], 16), 0); + if (!i) { + f->vma[i] = test_fault_vma_in_mm(f->mm[i], TEST_VA, SZ_64K); + f->attached[i] = !!f->vma[i]; + } else { + f->vma[i] = vm_area_alloc(f->mm[i]); + if (f->vma[i]) { + vma_set_anonymous(f->vma[i]); + f->vma[i]->vm_start = TEST_VA; + f->vma[i]->vm_end = TEST_VA + SZ_64K; + vm_flags_init(f->vma[i], f->vma[0]->vm_flags); + f->vma[i]->vm_page_prot = f->vma[0]->vm_page_prot; + vma_set_page_offset(f->vma[i], vma_page_offset_at(f->vma[0], TEST_VA)); + } + } + KUNIT_ASSERT_NOT_NULL(test, f->vma[i]); + tables.mm = f->mm[i]; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, PFN_PHYS(folio_pfn(f->folio))), 0); + pte = test_mm_lookup(f->mm[i], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(f->mm[i], TEST_VA, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(f->mm[0]); + ret = anon_vma_prepare(f->vma[0]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_write_lock(f->mm[0]); + mmap_write_lock_nested(f->mm[1], SINGLE_DEPTH_NESTING); + ret = anon_vma_fork(f->vma[1], f->vma[0]); + if (!ret) { + VMA_ITERATOR(vmi, f->mm[1], TEST_VA); + + vma_start_write(f->vma[1]); + ret = vma_iter_store_gfp(&vmi, f->vma[1], GFP_KERNEL); + if (!ret) { + f->mm[1]->map_count++; + f->attached[1] = true; + } + } + mmap_write_unlock(f->mm[1]); + mmap_write_unlock(f->mm[0]); + KUNIT_ASSERT_EQ(test, ret, 0); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, f->vma[0], TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); /* Fixture retains its original allocation ref. */ + inc_mm_counter(f->mm[0], MM_ANONPAGES); + f->mapped[0] = true; + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(f->mm[0], PFN_PHYS(folio_pfn(f->folio)), + f->vma[0]->vm_page_prot)), f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 1, true); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + + mm_flags_set(MMF_HAS_PINNED, f->mm[0]); + ret = try_grab_folio(page_folio(folio_page(f->folio, nr - 1)), 1, FOLL_PIN); + f->pinned = !ret; + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_set_wrprotect(f->mm[0], TEST_VA, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, dup_ret, -EBUSY); + KUNIT_EXPECT_EQ(test, share_ret, -EBUSY); + expect_coarse_anon_state(test, f->folio, 1, true); + unpin_user_page(folio_page(f->folio, nr - 1)); + f->pinned = false; + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get_and_clear(f->mm[0], TEST_VA, pte); + share_ret = folio_try_share_anon_rmap_pte(f->folio, &f->folio->page, f->vma[0]); + set_ptes(f->mm[0], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, share_ret, 0); + expect_coarse_anon_state(test, f->folio, 1, false); + + pte = test_mm_lookup(f->mm[0], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + raw_write_seqcount_begin(&f->mm[0]->write_protect_seq); + dup_ret = folio_try_dup_anon_rmap_pte(f->folio, &f->folio->page, f->vma[1], f->vma[0]); + raw_write_seqcount_end(&f->mm[0]->write_protect_seq); + if (!dup_ret) { + folio_ref_add(f->folio, nr); + inc_mm_counter(f->mm[1], MM_ANONPAGES); + f->mapped[1] = true; + } + pte_unmap_unlock(pte, ptl); + KUNIT_ASSERT_EQ(test, dup_ret, 0); + pte = test_mm_lookup(f->mm[1], TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + set_ptes(f->mm[1], TEST_VA, pte, pte_wrprotect(entry), 1); + pte_unmap_unlock(pte, ptl); + expect_coarse_anon_state(test, f->folio, 2, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[0], MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[1], MM_ANONPAGES), 1L); + + /* Exercise ordinary unmapping of each shared coarse leaf. */ + for (i = 0; i < 2; i++) { + mmap_read_lock(f->mm[i]); + zap_vma_range(f->vma[i], TEST_VA, SZ_64K); + f->mapped[i] = 0; + mmap_read_unlock(f->mm[i]); + expect_coarse_anon_state(test, f->folio, 1 - i, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)((1 - i) * nr + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(f->mm[i], MM_ANONPAGES), 0L); + } + + /* Keep an observation ref while cleanup releases the empty address spaces. */ + { + struct folio *folio = f->folio; + + folio_get(folio); + coarse_anon_fixture_free(f); + expect_coarse_anon_state(test, folio, 0, false); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); + } +} + +static void user64k_anon_zap_batch_test(struct kunit *test) +{ + struct coarse_anon_fixture *f; + struct test_mm tables = {}; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned int i, nr = SZ_64K / PAGE_SIZE, batch; + pte_t entry, *pte; + spinlock_t *ptl; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous leaves"); + return; + } + f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_anon_fixture_free, f), 0); + f->leaves = 2; + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 17 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + mm = f->mm[0] = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Partial unmap can enqueue this folio on the memcg split queue. */ + KUNIT_ASSERT_EQ(test, mem_cgroup_charge(f->folio, mm, GFP_KERNEL), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + vma = f->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, 2 * SZ_64K); + f->attached[0] = !!vma; + KUNIT_ASSERT_NOT_NULL(test, vma); + tables.mm = mm; + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, i, addr, + PFN_PHYS(folio_pfn(f->folio)) + i * SZ_64K), 0); + pte = test_mm_lookup(mm, addr, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + pte_clear(mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + mmap_write_lock(mm); + ret = anon_vma_prepare(vma); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_new_anon_rmap(f->folio, vma, TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, 2 * nr); + add_mm_counter(mm, MM_ANONPAGES, 2); + f->mapped[0] = BIT(0) | BIT(1); + entry = pte_mkwrite(pte_mkdirty(phys_pte_mm(mm, + PFN_PHYS(folio_pfn(f->folio)), vma->vm_page_prot)), vma); + set_ptes(mm, TEST_VA, pte, entry, 2); + batch = folio_subpage_pte_batch(vma, f->folio, pte, entry, 2, 0); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, batch, 2U); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)(2 * nr + 1)); + + /* The reference walker must consume logical PTEs and native mapcounts. */ + { + vm_flags_t flags; + bool young; + + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 1); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + young = pte_young(ptep_get(pte)) || pte_young(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_FALSE(test, young); + folio_lock(f->folio); + ret = folio_referenced(f->folio, 1, NULL, &flags); + folio_unlock(f->folio); + KUNIT_EXPECT_EQ(test, ret, 0); + } + + /* Removing one logical leaf must leave the adjacent leaf's native span. */ + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, SZ_64K); + f->mapped[0] &= ~BIT(0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), (int)nr + 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), + i < nr ? -1 : 0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + ret = pte_none(ptep_get(pte)) && pte_present(ptep_get(pte + 1)); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, ret); + + /* Restore the first leaf, then remove both through one batched walk. */ + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + folio_add_anon_rmap_pte(f->folio, &f->folio->page, vma, + TEST_VA, RMAP_EXCLUSIVE); + folio_ref_add(f->folio, nr); + inc_mm_counter(mm, MM_ANONPAGES); + f->mapped[0] |= BIT(0); + set_ptes(mm, TEST_VA, pte, entry, 1); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), (int)(2 * nr)); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + f->mapped[0] = 0; + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(f->folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(f->folio), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + for (i = 0; i < 2 * nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(f->folio, i)->_mapcount), -1); +} + +static void user64k_file_rmap_span_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse nonanonymous leaf"); + return; + } + mm = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + 2 * SZ_64K; + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 16 - PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, folio); + /* Private metadata-only fixture: no file-cache promotion or fault is implied. */ + folio_lock(folio); + folio_add_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 1); + folio_dup_file_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(2 * nr)); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount) + 1, 2); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_remove_rmap_pte(folio, &folio->page, vma); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + for (i = 0; i < nr; i++) + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + KUNIT_EXPECT_EQ(test, atomic_read(&folio_page(folio, i)->_mapcount), -1); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + folio_put(folio); +} + +static struct vm_area_struct *test_fault_vma(struct kunit *test, + unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm(test), start, len); +} + +static struct vm_area_struct *test_fault_vma_granule(struct kunit *test, + unsigned int shift, unsigned long start, unsigned long len) +{ + return test_fault_vma_in_mm(test_vma_mm_granule(test, shift), start, len); +} + +static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int shape; + + /* Swap/reclaim integration is a later gate, before any ABI activation. */ + KUNIT_ASSERT_EQ(test, total_swap_pages, 0UL); + for (shape = 0; shape < 2; shape++) { + unsigned long start = TEST_VA + (shape ? size : 0); + unsigned long len = shape ? size : 2 * PAGE_SIZE; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, start, len); + struct folio *backing = NULL; + unsigned int i, mapped = 0; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, TEST_VA + size + 3, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + /* Remove transient LRU-add references before checking ownership. */ + lru_add_drain_all(); + for (i = 0; i < (PAGE_SIZE >> shift); i++) { + unsigned long addr = TEST_VA + i * size; + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(vma->vm_mm, addr, &ptl); + struct mm_subpage *slot = NULL; + bool expected = !shape || i == 1; + bool present, writable = false, zero = false, identity = false; + + KUNIT_ASSERT_NOT_NULL(test, pte); + present = pte_present(ptep_get(pte)); + if (present) { + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, ptep_get(pte))); + writable = pte_write(ptep_get(pte)); + if (slot) { + struct folio *folio = mm_subpage_folio(slot); + + if (!backing) + backing = folio; + identity = folio == backing && + mm_subpage_offset(slot) == i * size; + zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), + 0, size); + mm_subpage_put(slot); + } + mapped++; + } + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, present, expected); + if (expected) { + KUNIT_EXPECT_TRUE(test, writable); + KUNIT_EXPECT_TRUE(test, identity); + KUNIT_EXPECT_TRUE(test, zero); + } + } + KUNIT_ASSERT_NOT_NULL(test, backing); + KUNIT_EXPECT_EQ(test, mapped, shape ? 1U : (PAGE_SIZE >> shift)); + KUNIT_EXPECT_EQ(test, folio_mapcount(backing), (int)mapped); + KUNIT_EXPECT_EQ(test, folio_ref_count(backing), (int)mapped); + KUNIT_EXPECT_TRUE(test, folio_test_lru(backing)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), + (unsigned long)mapped); + } +} + +static void user4k_anon_write_fault_test(struct kunit *test) +{ + test_anon_write_fault_granule(test, 12); +} + +static void user16k_anon_write_fault_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_anon_write_fault_granule(test, 14); +} + +static void user4k_swap_encoding_test(struct kunit *test) +{ + const unsigned long offsets[] = { 1, 2, (1UL << __SWP_OFFSET_BITS) - 2 }; + const unsigned int types[] = { 0, 1, __SWP_TYPE_MASK }; + unsigned int i, j, slot, flags; + + for (i = 0; i < ARRAY_SIZE(offsets); i++) { + for (j = 0; j < ARRAY_SIZE(types); j++) { + for (slot = 0; slot < PAGE_SIZE / SZ_4K; slot++) { + for (flags = 0; flags < 4; flags++) { + softleaf_t entry = swp_entry(types[j], offsets[i]); + pte_t pte = softleaf_to_pte(entry), moved; + + pte = pte_swp_set_subpage_offset(pte, slot * SZ_4K); + if (flags & 1) + pte = pte_swp_mkexclusive(pte); + if (flags & 2) + pte = pte_swp_mkuffd_wp(pte); + KUNIT_EXPECT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), slot * (unsigned int)SZ_4K); + moved = pte_move_swp_offset(pte, 1); + KUNIT_EXPECT_EQ(test, swp_offset(softleaf_from_pte(moved)), offsets[i] + 1); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), slot * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_same(pte_move_swp_offset(moved, -1), pte)); + moved = pte_swp_set_subpage_offset(pte, ((slot + 1) % (PAGE_SIZE / SZ_4K)) * SZ_4K); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(moved).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(moved), ((slot + 1) % (PAGE_SIZE / SZ_4K)) * (unsigned int)SZ_4K); + } + } + } + } +} + +static pte_t test_fault_entry(struct kunit *test, struct mm_struct *mm, unsigned long addr) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, addr, &ptl); + pte_t entry = __pte(0); + + KUNIT_EXPECT_NOT_NULL(test, pte); + if (pte) { + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + } + return entry; +} + + +static void test_file_put(void *file) +{ + fput(file); +} + +static struct file *test_cache_file(struct kunit *test, unsigned long size, bool fill) +{ + struct file *file = shmem_file_setup("user4k-file-test", size, EMPTY_VMA_FLAGS); + unsigned char *buf; + loff_t pos = 0; + unsigned int i; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + if (!fill) + return file; + buf = kunit_kmalloc(test, size, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + for (i = 0; i < size; i++) + buf[i] = 0x31 + (i / SZ_4K); + if (kernel_write(file, buf, size, &pos) != size) + return ERR_PTR(-EIO); + return file; +} + +static struct vm_area_struct *test_file_vma_granule(struct kunit *test, struct file *file, + unsigned int shift, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + int ret; + + mm = test_vma_mm_granule(test, shift); + if (!mm) + return NULL; + vma = vm_area_alloc(mm); + if (!vma) + return NULL; + vma->vm_start = start; + vma->vm_end = start + len; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = offset >> PAGE_SHIFT, .offset = offset & ~PAGE_MASK, + }); + if (shared) + flags |= VM_SHARED | VM_MAYSHARE; + vm_flags_init(vma, flags); + vma->vm_file = get_file(file); + vma->vm_page_prot = vm_get_page_prot(flags); + mmap_write_lock(mm); + ret = vfs_mmap(file, vma); + if (!ret) { + vma_set_page_prot(vma); + ret = insert_vm_struct(mm, vma); + } + mmap_write_unlock(mm); + if (ret) { + fput(vma->vm_file); + vm_area_free(vma); + return NULL; + } + return vma; +} + +static struct vm_area_struct *test_file_vma(struct kunit *test, struct file *file, + bool native, bool shared, unsigned long start, unsigned long len, + unsigned long offset) +{ + return test_file_vma_granule(test, file, native ? PAGE_SHIFT : 12, + shared, start, len, offset); +} + +static void user4k_mmap_descriptor_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_area_desc desc; + + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = vm_area_alloc(mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); + vma->vm_start = TEST_VA; + vma->vm_end = TEST_VA + PAGE_SIZE; + vma_set_page_offset(vma, (struct vm_page_offset) { 7, SZ_4K }); + compat_set_desc_from_vma(&desc, NULL, vma); + KUNIT_EXPECT_EQ(test, desc.pgoff, 7UL); + KUNIT_EXPECT_EQ(test, desc.subpage_offset, (unsigned int)SZ_4K); + desc.pgoff = 19; + desc.subpage_offset = 3 * SZ_4K; + KUNIT_ASSERT_EQ(test, __compat_vma_mmap(&desc, vma), 0); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, TEST_VA), + (struct vm_page_offset) { 19, 3 * SZ_4K })); +} + +static void user4k_mmap_region_offsets_test(struct kunit *test) +{ + struct file *file; + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE | VM_NORESERVE; + unsigned long result[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 8 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + /* Only the full native-sized lengths are used before accounting conversion. */ + kthread_use_mm(mm); + mmap_write_lock(mm); + result[0] = mmap_region(file, start + PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 1, SZ_4K }, NULL); + result[1] = mmap_region(file, start + 2 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 2, SZ_4K }, NULL); + result[2] = mmap_region(file, start, PAGE_SIZE, flags, + (struct vm_page_offset) { 0, SZ_4K }, NULL); + /* Adjacent virtual bytes with a different file offset must not merge. */ + result[3] = mmap_region(file, start + 3 * PAGE_SIZE, PAGE_SIZE, flags, + (struct vm_page_offset) { 3, 2 * SZ_4K }, NULL); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start + PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * PAGE_SIZE); + KUNIT_ASSERT_EQ(test, result[2], start); + KUNIT_ASSERT_EQ(test, result[3], start + 3 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm->map_count, 2); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, start); + KUNIT_EXPECT_EQ(test, vma->vm_end, start + 3 * PAGE_SIZE); + for (i = 0; i < 12; i++) + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, start + i * SZ_4K), + (struct vm_page_offset) { (i + 1) / TEST_SLOTS, ((i + 1) % TEST_SLOTS) * SZ_4K })); + vma = test_vma_lookup(mm, start + 3 * PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 2U * SZ_4K); +} + +#ifdef CONFIG_PROC_FS +/* Exercise the actual status/statm producers without publishing a fake task. */ +extern void task_mem(struct seq_file *m, struct mm_struct *mm); +extern unsigned long task_vsize(struct mm_struct *mm); +extern unsigned long task_statm(struct mm_struct *mm, unsigned long *shared, + unsigned long *text, unsigned long *data, unsigned long *resident); + +static void expect_status_kb(struct kunit *test, struct seq_file *seq, + const char *name, unsigned long expected) +{ + const char *field = strstr(seq->buf, name); + unsigned long value = ULONG_MAX; + + KUNIT_ASSERT_NOT_NULL(test, field); + KUNIT_EXPECT_EQ(test, sscanf(field + strlen(name), "%lu", &value), 1); + KUNIT_EXPECT_EQ_MSG(test, value, expected, "%s", name); +} + +static void user4k_accounting_readers_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct seq_file seq = {}; + unsigned long shared, text, data, resident, size, maxrss = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + seq.size = 2048; + seq.buf = kunit_kzalloc(test, seq.size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, seq.buf); + /* Same logical counts, deliberately not multiples of a native page. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned long kb = i ? PAGE_SIZE / 1024 : 4; + + mm->total_vm = 11; + mm->hiwater_vm = 13; + mm->data_vm = 5; + mm->stack_vm = 2; + mm->exec_vm = 3; + mm->locked_vm = mm_pages_to_account(mm, 1); + atomic64_set(&mm->pinned_vm, 2); + mm->hiwater_rss = 10; + mm->start_code = TEST_VA + mm_page_size(mm); + mm->end_code = mm->start_code + 2 * mm_page_size(mm); + percpu_counter_set(&mm->rss_stat[MM_ANONPAGES], 5); + percpu_counter_set(&mm->rss_stat[MM_FILEPAGES], 2); + percpu_counter_set(&mm->rss_stat[MM_SHMEMPAGES], 1); + percpu_counter_set(&mm->rss_stat[MM_SWAPENTS], 3); + seq.count = 0; + task_mem(&seq, mm); + KUNIT_EXPECT_FALSE(test, seq_has_overflowed(&seq)); + seq.buf[min(seq.count, seq.size - 1)] = '\0'; + expect_status_kb(test, &seq, "VmPeak:", 13 * kb); + expect_status_kb(test, &seq, "VmSize:", 11 * kb); + expect_status_kb(test, &seq, "VmLck:", kb); + expect_status_kb(test, &seq, "VmPin:", 2 * PAGE_SIZE / 1024); + expect_status_kb(test, &seq, "VmHWM:", 10 * kb); + expect_status_kb(test, &seq, "VmRSS:", 8 * kb); + expect_status_kb(test, &seq, "RssAnon:", 5 * kb); + expect_status_kb(test, &seq, "RssFile:", 2 * kb); + expect_status_kb(test, &seq, "RssShmem:", kb); + expect_status_kb(test, &seq, "VmData:", 5 * kb); + expect_status_kb(test, &seq, "VmStk:", 2 * kb); + expect_status_kb(test, &seq, "VmExe:", 2 * kb); + expect_status_kb(test, &seq, "VmLib:", kb); + expect_status_kb(test, &seq, "VmSwap:", 3 * kb); + KUNIT_EXPECT_EQ(test, task_vsize(mm), 11 * kb * 1024); + { + struct vm_area_struct vma = { .vm_mm = mm }; + + KUNIT_EXPECT_EQ(test, vma_kernel_pagesize(&vma), kb * 1024); + KUNIT_EXPECT_EQ(test, vma_mmu_pagesize(&vma), kb * 1024); + } + } + /* Maxima must compare bytes, not incompatible 4K/16K counts. */ + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 40UL); + setmax_mm_hiwater_rss(&maxrss, native); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, 10UL * PAGE_SIZE / 1024); + small->hiwater_rss = 10 * TEST_SLOTS + 1; + setmax_mm_hiwater_rss(&maxrss, small); + KUNIT_EXPECT_EQ(test, maxrss, (10UL * TEST_SLOTS + 1) * 4); + + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 0), 0UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 1), 1UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_native(small, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, small, 5), 5UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, native, 5), DIV_ROUND_UP(5UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(small, small, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_for_reader(native, native, 5), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(small, 5, MM_ACCOUNT_SHIFT), 5UL); + KUNIT_EXPECT_EQ(test, mm_pages_to_shift(native, 5, MM_ACCOUNT_SHIFT), 5UL * TEST_SLOTS); + + /* A kernel reader defaults to native units; partial pages round up. */ + size = task_statm(small, &shared, &text, &data, &resident); + KUNIT_EXPECT_EQ(test, size, DIV_ROUND_UP(11UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, shared, 1UL); + KUNIT_EXPECT_EQ(test, text, 1UL); + KUNIT_EXPECT_EQ(test, data, DIV_ROUND_UP(7UL, TEST_SLOTS)); + KUNIT_EXPECT_EQ(test, resident, DIV_ROUND_UP(8UL, TEST_SLOTS)); + if (user4k_test_enabled || IS_ENABLED(CONFIG_KASAN)) { + kthread_use_mm(small); + size = task_statm(native, &shared, &text, &data, &resident); + kthread_unuse_mm(small); + KUNIT_EXPECT_EQ(test, size, 11UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, shared, 3UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, text, 2UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, data, 7UL * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, resident, 8UL * TEST_SLOTS); + } + /* Synthetic counters have no page references; restore before mmput. */ + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? native : small; + unsigned int counter; + + for (counter = 0; counter < NR_MM_COUNTERS; counter++) + percpu_counter_set(&mm->rss_stat[counter], 0); + mm->total_vm = mm->data_vm = mm->stack_vm = mm->exec_vm = 0; + mm->locked_vm = 0; + atomic64_set(&mm->pinned_vm, 0); + } +} +#endif + +static void mixed_locked_account_units_test(struct kunit *test) +{ + unsigned int shift; + unsigned long native_units = PAGE_SIZE >> MM_ACCOUNT_SHIFT; + + for (shift = 12; shift <= 16; shift += 2) { + struct mm_struct *mm = mm_alloc(); + unsigned long size = 1UL << shift; + unsigned long base, limit; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shift), 0); + base = mm_pages_to_account(mm, 1); + mmap_write_lock(mm); + /* Model one locked VMA leaf and two independently charged native pins. */ + mm->locked_vm = base; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + native_units); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(mm), + DIV_ROUND_UP(size + PAGE_SIZE, PAGE_SIZE)); + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base + 2 * native_units); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size + PAGE_SIZE); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, base); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(mm), size); + + /* A refused charge must leave the exact pre-existing amount intact. */ + limit = task_rlimit(current, RLIMIT_MEMLOCK) >> MM_ACCOUNT_SHIFT; + mm->locked_vm = limit; + ret = __account_locked_vm(mm, 1, true, current, false); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, limit); + mm->locked_vm = ULONG_MAX - native_units + 1; + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, ULONG_MAX - native_units + 1); + mm->locked_vm = 0; + if (native_units > 1) { + ret = __account_locked_vm(mm, + (ULONG_MAX >> (PAGE_SHIFT - MM_ACCOUNT_SHIFT)) + 1, + true, current, true); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + } + mmap_write_unlock(mm); + } +} + +/* Prior shmem fixtures can refund global commit from deferred fput work. */ +static s64 test_commit_baseline(struct kunit *test) +{ + s64 before = percpu_counter_sum(&vm_committed_as), after; + + /* No mmap, file, or filesystem locks are held at these call sites. */ + flush_delayed_fput(); + after = percpu_counter_sum(&vm_committed_as); + if (after != before) + kunit_info(test, "drained deferred file refunds: commit delta %lld", after - before); + return after; +} + +static void user64k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + s64 before; + unsigned int i; + unsigned long native_per_leaf; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace granule larger than native"); + return; + } + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, 16), 0); + native_per_leaf = SZ_64K / PAGE_SIZE; + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)((i + 1) * native_per_leaf)); + } + /* Multiplication overflow must not mutate either reservation counter. */ + ret = mm_account_memory(mm, mm, LONG_MAX / native_per_leaf + 1); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + mm_acct_memory(mm, 3); + i += 3; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)(i * native_per_leaf)); + } + mmap_write_unlock(mm); +} + +#ifdef CONFIG_PAGE_TABLE_CHECK +static void expect_coarse_file_counts(struct kunit *test, void *data, + unsigned int phase) +{ + unsigned int offset; + + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) { + int anon = -1, file = -1, expected; + int ret = page_table_check_get_counts(virt_to_page(data + offset), + offset & ~PAGE_MASK, &anon, &file); + + /* Phase0: both coarse leaves;1:plusalias;2:firstclear;3:bothclear;4:allclear. */ + expected = phase < 2 || (phase == 2 && offset >= SZ_64K); + if (phase >= 1 && phase <= 3 && offset == 2 * SZ_64K - SZ_4K) + expected++; + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, anon, 0); + KUNIT_EXPECT_EQ_MSG(test, file, expected, "phase=%u offset=%u", phase, offset); + } +} +#endif + +static void user64k_page_table_check_test(struct kunit *test) +{ +#ifdef CONFIG_PAGE_TABLE_CHECK + struct mm_struct *coarse, *small; + void *data; + pte_t ptes[2] = {}, alias = __pte(0), first, last; + int anon, file; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires userspace leaf larger than native page"); + return; + } + data = test_alloc_granule(test, 17); + KUNIT_ASSERT_NOT_NULL(test, data); + if (page_table_check_get_counts(virt_to_page(data), 0, &anon, &file)) { + kunit_skip(test, "requires page-table-check enabled at boot"); + return; + } + coarse = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, coarse), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(coarse, 16), 0); + small = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, small), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(small, 12), 0); + first = phys_pte_mm(coarse, virt_to_phys(data), __pgprot(TEST_PROT)); + last = phys_pte_mm(small, virt_to_phys(data) + 2 * SZ_64K - SZ_4K, + __pgprot(TEST_PROT)); + + /* Exercise checker hooks directly; no actual coarse file fault is claimed. */ + page_table_check_ptes_set(coarse, TEST_VA, ptes, first, 2); + ptes[0] = first; + ptes[1] = pte_advance_pfn_mm(coarse, first, 1); + expect_coarse_file_counts(test, data, 0); + page_table_check_ptes_set(small, TEST_VA, &alias, last, 1); + alias = last; + expect_coarse_file_counts(test, data, 1); + page_table_check_pte_clear(coarse, TEST_VA, ptes[0]); + ptes[0] = __pte(0); + expect_coarse_file_counts(test, data, 2); + page_table_check_pte_clear(coarse, TEST_VA + SZ_64K, ptes[1]); + ptes[1] = __pte(0); + expect_coarse_file_counts(test, data, 3); + page_table_check_pte_clear(small, TEST_VA, alias); + expect_coarse_file_counts(test, data, 4); +#else + kunit_skip(test, "requires CONFIG_PAGE_TABLE_CHECK"); +#endif +} + +static void user4k_commit_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + s64 before; + int ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + mmap_write_lock(mm); + for (i = 0; i < 9; i++) { + ret = mm_account_memory(mm, mm, 1); + KUNIT_EXPECT_EQ(test, ret, 0); + if (ret) + break; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i + 1); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i + 1, TEST_SLOTS)); + } + if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) { + unsigned long old = mm->committed_user_pages; + s64 committed = percpu_counter_sum(&vm_committed_as); + + ret = mm_account_memory(mm, mm, (totalram_pages() + total_swap_pages + 1) * TEST_SLOTS); + KUNIT_EXPECT_EQ(test, ret, -ENOMEM); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, old); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as), committed); + } + /* Native physical-page callers must charge four user granules per page. */ + ret = __account_locked_vm(mm, 1, true, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + ret = __account_locked_vm(mm, 1, false, current, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + + /* Refund one granule at a time: splitting must not multiply refunds. */ + while (i) { + mm_unacct_memory(mm, 1); + i--; + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, + (s64)DIV_ROUND_UP(i, TEST_SLOTS)); + } + mmap_write_unlock(mm); +} + +/* Prefaulting must stop at each process-page boundary, not native pages. */ +static void user_granule_prefault_test(struct kunit *test) +{ + unsigned int shift; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + for (shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + unsigned long size = 1UL << shift, start = TEST_VA; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long mapped[3]; + size_t left[3][4] = {}; + int phase, ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped[0] = vm_mmap(NULL, start, size, PROT_READ | PROT_WRITE, flags, 0); + mapped[2] = vm_mmap(NULL, start + 2 * size, size, + PROT_READ | PROT_WRITE, flags, 0); + mapped[1] = start + size; + if (mapped[0] == start && mapped[2] == start + 2 * size) { + for (phase = 0; phase < 3; phase++) { + char __user *addr = (char __user *)(start + 17); + size_t len = 3 * size - 17; + + /* Missing middle, then read-only, then fully writable. */ + if (phase) { + mapped[1] = vm_mmap(NULL, start + size, size, + PROT_READ | (phase == 2 ? PROT_WRITE : 0), + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + if (mapped[1] != start + size) + break; + } + left[phase][0] = fault_in_safe_writeable(addr, len); + left[phase][1] = fault_in_readable(addr, len); + left[phase][2] = fault_in_writeable(addr, len); + left[phase][3] = fault_in_subpage_writeable(addr, len); + } + ret = vm_munmap(start, 3 * size); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped[0], start); + KUNIT_ASSERT_EQ(test, mapped[1], start + size); + KUNIT_ASSERT_EQ(test, mapped[2], start + 2 * size); + KUNIT_EXPECT_EQ(test, ret, 0); + for (phase = 0; phase < 3; phase++) { + KUNIT_EXPECT_EQ(test, left[phase][0], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][1], phase == 0 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][2], phase < 2 ? 2 * size : 0UL); + KUNIT_EXPECT_EQ(test, left[phase][3], phase < 2 ? 2 * size : 0UL); + } + } +} + +static void user4k_mmap_accounting_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K, mapped; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE; + s64 before; + unsigned int i; + int ret = 0; + vm_fault_t fault = 0; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mmap_write_lock(mm); + mapped = mmap_region(NULL, start, 3 * SZ_4K, flags, + (struct vm_page_offset) { start >> PAGE_SHIFT, SZ_4K }, NULL); + mmap_write_unlock(mm); + if (!IS_ERR_VALUE(mapped)) { + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->data_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, 1LL); + /* Remove the middle first, then the two neighbors. */ + for (i = 0; i < 3; i++) { + unsigned int slot = i == 0 ? 1 : i == 1 ? 0 : 2; + VMA_ITERATOR(vmi, mm, start + slot * SZ_4K); + + kthread_use_mm(mm); + mmap_write_lock(mm); + ret = do_vmi_munmap(&vmi, mm, start + slot * SZ_4K, SZ_4K, NULL, false); + mmap_write_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->data_vm, 2UL - i); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 2UL - i); + KUNIT_EXPECT_EQ(test, percpu_counter_sum(&vm_committed_as) - before, i == 2 ? 0LL : 1LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); +} + +static vm_fault_t test_file_fault(struct vm_area_struct *vma, unsigned long addr, bool write) +{ + vm_fault_t ret; + + mmap_read_lock(vma->vm_mm); + ret = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE | + (write ? FAULT_FLAG_WRITE : 0), NULL); + mmap_read_unlock(vma->vm_mm); + return ret; +} + +static void user4k_vm_mmap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct file *file; + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + unsigned long anon_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long file_flags = MAP_SHARED | MAP_FIXED_NOREPLACE; + unsigned long result[7]; + int unmap[4]; + pte_t pte; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + 2 * PAGE_SIZE + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + result[0] = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, anon_flags, 0); + result[1] = vm_mmap(file, start + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 3 * SZ_4K); + result[2] = vm_mmap(file, start + 3 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE, + file_flags, 4 * SZ_4K); + result[3] = vm_mmap(NULL, start, SZ_4K, PROT_READ, anon_flags, 0); + result[4] = vm_mmap(NULL, start + 17, SZ_4K, PROT_READ, anon_flags, 0); + result[5] = vm_mmap(file, start + SZ_4K, SZ_4K, PROT_READ, file_flags, SZ_4K + 1); + result[6] = vm_mmap(NULL, 0, SZ_4K + 1, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, result[0], start); + KUNIT_ASSERT_EQ(test, result[1], start + 2 * SZ_4K); + KUNIT_ASSERT_EQ(test, result[2], start + 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, result[3], (unsigned long)-EEXIST); + KUNIT_EXPECT_EQ(test, result[4], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, result[5], (unsigned long)-EINVAL); + KUNIT_ASSERT_EQ(test, result[6], mm->mmap_base); + KUNIT_EXPECT_EQ(test, mm->total_vm, 5UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start, true) & VM_FAULT_ERROR, 0U); + vma = test_vma_lookup(mm, start + 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_end - vma->vm_start, 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + 2 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, start + 2 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x34); + pte = test_fault_entry(test, mm, start + 3 * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(mm, pte)), (u8)0x35); + kthread_use_mm(mm); + unmap[0] = vm_munmap(start + 2 * SZ_4K, 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, unmap[0], 0); + vma = test_vma_lookup(mm, start + 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_pgoff, 4UL / TEST_SLOTS); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), (4U % TEST_SLOTS) * SZ_4K); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + kthread_use_mm(mm); + unmap[1] = vm_munmap(start, SZ_4K); + unmap[2] = vm_munmap(start + 3 * SZ_4K, SZ_4K); + unmap[3] = vm_munmap(result[6], 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, unmap[2], 0); + KUNIT_EXPECT_EQ(test, unmap[3], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void test_mlock_drain_all(struct kunit *test, struct folio *folio) +{ + unsigned int before, after; + bool counted; + + lru_add_drain(); + counted = folio_test_unevictable(folio); + before = counted ? folio->mlock_count : 0; + /* A sleeping fixture can enqueue mlock work on more than one CPU. */ + lru_add_drain_all(); + after = folio_test_unevictable(folio) ? folio->mlock_count : 0; + if (counted && before != after) + kunit_info(test, "remote mlock drain changed count %u -> %u", before, after); +} + +static void user4k_mlock_counts_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio; + unsigned long addr; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_LOCKED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA))); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + folio_get(folio); + mm_subpage_put(slot); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->locked_vm, (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + int ret; + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA + i * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + /* mlock_count shares storage with the evictable LRU links. */ + if (i != TEST_SLOTS - 1) + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1 - i); + KUNIT_EXPECT_EQ(test, mm->locked_vm, TEST_SLOTS - 1UL - i); + KUNIT_EXPECT_EQ(test, folio_test_mlocked(folio), i != TEST_SLOTS - 1); + } +} + +static void test_mlock_enable_lru_cache(void *unused) +{ + lru_cache_enable(); +} + +static void user4k_mlock_shared_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + if (native) + mmput(native); + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Exact counts require synchronous LRU/lock accounting. Normal deferred + * batches may legitimately miss locks while the folio is off LRU; + * user4k_mlock_repair_test checks that reclaim repairs that case. + */ + lru_cache_disable(); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_enable_lru_cache, NULL), 0); + kthread_use_mm(small); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + kthread_use_mm(native); + addr = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS + 1); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA + SZ_4K, SZ_4K); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS); + kthread_use_mm(native); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(native); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_EQ(test, folio->mlock_count, TEST_SLOTS - 1); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kthread_use_mm(small); + ret = vm_munmap(TEST_VA, PAGE_SIZE); + kthread_unuse_mm(small); + KUNIT_ASSERT_EQ(test, ret, 0); + test_mlock_drain_all(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); +} + +static void test_mlock_putback(void *folio) +{ + folio_putback_lru(folio); +} + +static void user4k_mlock_repair_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode, walker; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mlock requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + unsigned long size = 1UL << shifts[mode]; + + for (walker = 0; walker < 2; walker++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *folio; + unsigned long addr, offset; + vm_flags_t flags = 0; + int maps, ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_folio_put, folio), 0); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_isolate_lru(folio)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_mlock_putback, folio), 0); + /* Two aliases leave a locked mapping even after native unmap. + * Force deferred mlocks to run while the folio is isolated. + */ + for (offset = 0; offset < 2 * PAGE_SIZE; offset += PAGE_SIZE) { + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA + offset, PAGE_SIZE, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_LOCKED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + offset); + } + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_lru(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + kunit_release_action(test, test_mlock_putback, folio); + lru_add_drain_all(); + KUNIT_ASSERT_TRUE(test, folio_test_unevictable(folio)); + KUNIT_EXPECT_EQ(test, folio->mlock_count, 0U); + + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + maps = folio_mapcount(folio); + KUNIT_EXPECT_EQ(test, maps, (int)(2 * PAGE_SIZE / size) - 1); + folio_lock(folio); + if (walker) + ret = folio_referenced(folio, 1, NULL, &flags); + else + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + lru_add_drain_all(); + if (walker) { + KUNIT_EXPECT_GE(test, ret, 0); + KUNIT_EXPECT_TRUE(test, flags & VM_LOCKED); + } + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), maps); + for (offset = size; offset < 2 * PAGE_SIZE; offset += size) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, + page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(mm, pte)))), + folio); + } + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, 2 * PAGE_SIZE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + KUNIT_EXPECT_FALSE(test, folio_test_mlocked(folio)); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + kunit_info(test, "%uK %s repairs deferred mlock without unmapping", + 1U << (shifts[mode] - 10), walker ? "referenced" : "unmap"); + } + } +} + +asmlinkage long __arm64_sys_mremap(const struct pt_regs *regs); + +static unsigned long test_mremap(unsigned long old, unsigned long old_len, + unsigned long new_len, unsigned long flags, unsigned long new) +{ + struct pt_regs regs = { .regs = { old, old_len, new_len, flags, new } }; + + return __arm64_sys_mremap(®s); +} + +static int test_iova_bitmap_fill(struct iova_bitmap *bitmap, unsigned long iova, + size_t length, void *opaque) +{ + unsigned long unit = *(unsigned long *)opaque; + + iova_bitmap_set(bitmap, iova + 3 * unit, length - 10 * unit); + /* A second write advances backwards into an already populated range. */ + iova_bitmap_set(bitmap, iova, 5 * unit); + return 0; +} + +/* Real IOMMUFD lifecycle with a translation-recording IOMMU, no physical DMA. */ +struct test_fragment_domain { + struct iommu_domain domain; + struct xarray translations; + unsigned long fail_after; + unsigned long largest_map; + unsigned long mapped; + unsigned long map_calls; +}; + +struct test_fragment_iopt { + struct io_pagetable table; + struct test_fragment_domain domains[2]; + struct kunit *test; +}; + +static int test_fragment_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + int rc; + + mock->map_calls++; + mock->largest_map = max(mock->largest_map, length); + for (done = 0; done < length; done += SZ_4K) { + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + rc = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (rc) + return rc; + *mapped += SZ_4K; + mock->mapped += SZ_4K; + } + return 0; +} + +static size_t test_fragment_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t test_fragment_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct test_fragment_domain *mock = container_of(domain, typeof(*mock), domain); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void test_fragment_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *gather) +{ +} + +static const struct iommu_domain_ops test_fragment_domain_ops = { + .map_pages = test_fragment_map, + .unmap_pages = test_fragment_unmap, + .iova_to_phys = test_fragment_translate, + .iotlb_sync = test_fragment_sync, +}; + +static void test_fragment_iopt_free(void *data) +{ + struct test_fragment_iopt *fixture = data; + struct iommu_domain *domain; + unsigned long unmapped, index; + unsigned int i; + + iopt_unmap_all(&fixture->table, &unmapped); + while ((domain = xa_find(&fixture->table.domains, &(unsigned long){0}, + ULONG_MAX, XA_PRESENT))) + iopt_table_remove_domain(&fixture->table, domain); + iopt_destroy_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + void *entry; + + KUNIT_EXPECT_TRUE(fixture->test, xa_empty(&mock->translations)); + xa_for_each(&mock->translations, index, entry) + xa_erase(&mock->translations, index); + xa_destroy(&mock->translations); + } +} + +static struct test_fragment_iopt *test_fragment_iopt_alloc(struct kunit *test) +{ + struct test_fragment_iopt *fixture = kunit_kzalloc(test, sizeof(*fixture), GFP_KERNEL); + unsigned int i; + + if (!fixture) + return NULL; + fixture->test = test; + iopt_init_table(&fixture->table); + for (i = 0; i < ARRAY_SIZE(fixture->domains); i++) { + struct test_fragment_domain *mock = &fixture->domains[i]; + + xa_init(&mock->translations); + mock->domain.ops = &test_fragment_domain_ops; + mock->domain.type = IOMMU_DOMAIN_UNMANAGED; + mock->domain.pgsize_bitmap = SZ_4K; + mock->domain.geometry.aperture_end = ULONG_MAX; + } + if (kunit_add_action_or_reset(test, test_fragment_iopt_free, fixture)) + return NULL; + return fixture; +} + +static int test_fragment_map_user(struct test_fragment_iopt *fixture, + struct mm_struct *mm, unsigned long va, unsigned long iova, unsigned long length) +{ + struct iommufd_ctx ctx = {}; + int rc; + + kthread_use_mm(mm); + rc = iopt_map_user_pages(&ctx, &fixture->table, &iova, (void __user *)va, + length, IOMMU_READ | IOMMU_WRITE, 0); + kthread_unuse_mm(mm); + return rc; +} + +static void user4k_iommufd_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14, 16 }, mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + unsigned long size = 1UL << shifts[mode]; + unsigned long len = 2 * max(size, PAGE_SIZE); + unsigned long dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long iova = SZ_1M, alias = SZ_16M, addr, moved, missed, unmapped; + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned char *buffer; + unsigned long off, pinned; + LIST_HEAD(copy); + int rc; + + if (mode && shifts[mode] == PAGE_SHIFT) + continue; + mm = test_vma_mm_granule(test, shifts[mode]); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x31, len); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + moved = missed ? 0 : test_mremap(addr, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, dest); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, dest, iova, len), 0); + pinned = atomic64_read(&mm->pinned_vm); + KUNIT_EXPECT_EQ(test, pinned, len / PAGE_SIZE); + KUNIT_EXPECT_GE(test, fixture->domains[0].largest_map, PAGE_SIZE); + for (off = 0; off < len; off += size) { + pte_t pte = test_fault_entry(test, mm, dest + off); + phys_addr_t actual = iommu_iova_to_phys(&fixture->domains[0].domain, iova + off); + + KUNIT_EXPECT_EQ(test, actual, pte_phys_mm(mm, pte)); + } + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, iova + size, len - size, ©), 0); + rc = iopt_map_pages(&fixture->table, ©, len - size, &alias, IOMMU_READ | IOMMU_WRITE, 0); + iopt_free_pages_list(©); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), pinned); + + /* Replace the original VA: mappings and copied IOAS must retain old pins. */ + kthread_use_mm(mm); + addr = vm_mmap(NULL, dest, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + memset(buffer, 0x5a, len); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)dest, buffer, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, missed, 0UL); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, iova, iova); + pages = area->pages; + /* Force the domain-backed reader for the native control too. */ + memset(buffer, 0x77, len); + rc = iopt_pages_rw_access(pages, 0, buffer, len, IOMMUFD_ACCESS_RW_WRITE); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, rc, 0); + for (off = 0; off < len; off += SZ_4K) { + phys_addr_t phys = iommu_iova_to_phys(&fixture->domains[1].domain, iova + off); + void *ptr = kmap_local_page(pfn_to_page(PHYS_PFN(phys))); + bool correct = !memchr_inv(ptr + offset_in_page(phys), 0x77, SZ_4K); + + kunmap_local(ptr); + KUNIT_EXPECT_TRUE(test, correct); + } + kthread_use_mm(mm); + missed = copy_from_user(buffer, (void __user *)dest, len); + /* Local CPU access retains the existing copy_from_user fast path. */ + rc = iopt_pages_rw_access(pages, 0, buffer, len, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, iova, len, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, len); + KUNIT_EXPECT_EQ(test, (unsigned long)atomic64_read(&mm->pinned_vm), (len - (size < PAGE_SIZE ? 0 : size)) / PAGE_SIZE); + iopt_table_remove_domain(&fixture->table, &fixture->domains[0].domain); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, len - size); + KUNIT_ASSERT_EQ(test, iopt_unmap_iova(&fixture->table, alias, len - size, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOMMUFD %luK: relocated PA, batching, copy, two domains, VA replacement and unpin passed", size / SZ_1K); + } +} + +static void user4k_iommufd_rollback_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, iova = SZ_1M, size = SZ_4K; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + rc = vm_munmap(TEST_VA + size, size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, addr, iova, 3 * size), -EFAULT); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + KUNIT_EXPECT_LT(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA + size); + fixture->domains[1].fail_after = size; + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 3 * size), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 3 * size); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 3 * size); + kunit_release_action(test, test_fragment_iopt_free, fixture); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_granule_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + unsigned long addr, unmapped, cut = SZ_1M + PAGE_SIZE - 1; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + fixture->domains[0].domain.pgsize_bitmap = PAGE_SIZE; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 3 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + /* The device cannot cover a misaligned fragment by widening the mapping. */ + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA + SZ_4K, SZ_1M, PAGE_SIZE), -EINVAL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, SZ_4K), -EINVAL); + /* Full contiguous native spans still work, including legacy area splitting. */ + KUNIT_ASSERT_EQ(test, iopt_disable_large_pages(&fixture->table), 0); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + KUNIT_EXPECT_EQ(test, fixture->domains[0].largest_map, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_ASSERT_EQ(test, iopt_cut_iova(&fixture->table, &cut, 1), 0); + rc = iopt_unmap_iova(&fixture->table, SZ_1M, PAGE_SIZE, &unmapped); + KUNIT_ASSERT_EQ(test, rc, 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + /* A closed IOVA interval ending at ULONG_MAX must not wrap at map/unmap. */ + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, + ULONG_MAX - 2 * PAGE_SIZE + 1, 2 * PAGE_SIZE), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, unmapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void user4k_iommufd_shared_test(struct kunit *test) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct file *file; + struct folio *folio; + unsigned long addr, unmapped, iova = SZ_1M; + unsigned char *buffer; + loff_t pos = 0; + phys_addr_t phys; + void *ptr; + int rc; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + file = test_cache_file(test, 2 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + buffer = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, 2 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, iova, 2 * SZ_4K), 0); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + folio_unlock(folio); + phys = iommu_iova_to_phys(&fixture->domains[0].domain, iova); + KUNIT_EXPECT_EQ(test, phys, PFN_PHYS(folio_pfn(folio)) + SZ_4K); + ptr = kmap_local_folio(folio, SZ_4K); + memset(ptr, 0x93, 2 * SZ_4K); + kunmap_local(ptr); + kthread_use_mm(mm); + rc = vm_munmap(TEST_VA, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, rc, 0); + /* A failed second-domain attachment must retain the first domain's pins. */ + fixture->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), -ENOMEM); + KUNIT_EXPECT_EQ(test, fixture->domains[0].mapped, 2 * SZ_4K); + KUNIT_EXPECT_EQ(test, fixture->domains[1].mapped, 0UL); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 1LL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + fixture->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_ASSERT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, PAGE_SIZE, &pos), (ssize_t)PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + SZ_4K, 0x93, 2 * SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 3 * SZ_4K, 0, PAGE_SIZE - 3 * SZ_4K), NULL); + kunit_release_action(test, test_fragment_iopt_free, fixture); + kunit_release_action(test, user4k_vma_mm_free, mm); +} + +static void test_iommufd_exit_account(struct kunit *test, unsigned int shift) +{ + struct mm_struct *mm; + struct test_fragment_iopt *fixture; + struct iopt_pages *pages; + struct iopt_area *area; + unsigned long addr, unmapped; + LIST_HEAD(keep); + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm IOMMUFD requires hardware opt-in"); + return; + } + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + fixture = test_fragment_iopt_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, fixture); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, 2 * PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, test_fragment_map_user(fixture, mm, TEST_VA, SZ_1M, 2 * PAGE_SIZE), 0); + down_read(&fixture->table.iova_rwsem); + area = iopt_area_iter_first(&fixture->table, SZ_1M, SZ_1M); + pages = area->pages; + mutex_lock(&pages->mutex); + /* Select VFIO-compatible accounting before the lazy range acquires pins. */ + pages->account_mode = IOPT_PAGES_ACCOUNT_MM; + mutex_unlock(&pages->mutex); + up_read(&fixture->table.iova_rwsem); + KUNIT_ASSERT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[0].domain), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 2LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 2 * (PAGE_SIZE >> MM_ACCOUNT_SHIFT)); + /* Keep the provider, and thus mm_count, after mm_users reaches zero. */ + KUNIT_ASSERT_EQ(test, iopt_get_pages(&fixture->table, SZ_1M, 2 * PAGE_SIZE, &keep), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, atomic_read(&mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, iopt_table_add_domain(&fixture->table, &fixture->domains[1].domain), 0); + KUNIT_EXPECT_EQ(test, iopt_unmap_all(&fixture->table, &unmapped), 0); + KUNIT_EXPECT_EQ(test, atomic64_read(&mm->pinned_vm), 0LL); + KUNIT_EXPECT_EQ(test, mm->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, pages->npinned, 0UL); + KUNIT_EXPECT_EQ(test, pages->last_npinned, 0UL); + iopt_free_pages_list(&keep); + kunit_release_action(test, test_fragment_iopt_free, fixture); +} + +static void user4k_iommufd_exit_account_test(struct kunit *test) +{ + test_iommufd_exit_account(test, PAGE_SHIFT); + test_iommufd_exit_account(test, 12); +} + +static void user4k_iova_bitmap_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 4 * size; + unsigned long old = TEST_VA, dest = TEST_VA + 32 * PAGE_SIZE + size; + unsigned long start = dest + size - sizeof(u64), bytes = 2 * size; + unsigned long unit = SZ_4K, iova = 1UL << 32, addr, moved; + unsigned char *actual, *expected; + struct iova_bitmap *bitmap; + unsigned long missed; + int ret = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kunit_kzalloc(test, len, GFP_KERNEL); + expected = kunit_kzalloc(test, len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_NOT_NULL(test, expected); + kthread_use_mm(mm); + addr = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, actual, len); + moved = missed ? 0 : test_mremap(old, len, len, + MREMAP_FIXED | MREMAP_MAYMOVE, dest); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, old); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_EQ(test, moved, dest); + /* VA and PA offsets differ after moving by one alternative leaf. */ + if (size < PAGE_SIZE) { + pte_t pte = test_fault_entry(test, mm, dest); + + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + dest & ~PAGE_MASK); + } + kthread_use_mm(mm); + bitmap = iova_bitmap_alloc(iova, bytes * 8 * unit, unit, + (u64 __user *)start); + if (!IS_ERR(bitmap)) { + ret = iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + iova_bitmap_free(bitmap); + } else { + ret = PTR_ERR(bitmap); + } + missed = copy_from_user(actual, (void __user *)dest, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + memset(expected + size - sizeof(u64), 0xff, bytes); + expected[size - sizeof(u64) + bytes - 1] = 1; + KUNIT_EXPECT_EQ(test, memcmp(actual, expected, len), 0); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap relocated %luK user leaves checked", size / SZ_1K); + } +} + +static void test_iova_free(void *bitmap) +{ + iova_bitmap_free(bitmap); +} + +static void test_iova_kvfree(void *buffer) +{ + kvfree(buffer); +} + +static void user4k_iova_bitmap_window_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long capacity = PAGE_SIZE / sizeof(struct user_page_fragment); + unsigned long bytes = (capacity + 3) * SZ_4K, len = bytes + 2 * SZ_4K; + unsigned long start = TEST_VA + SZ_4K - sizeof(u64), unit = SZ_4K; + unsigned long addr, missed; + unsigned char *actual; + struct iova_bitmap *bitmap; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + actual = kvzalloc(len, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, actual); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_kvfree, actual), 0); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + bitmap = iova_bitmap_alloc(0, bytes * 8 * unit, unit, (u64 __user *)start); + ret = IS_ERR(bitmap) ? PTR_ERR(bitmap) : + iova_bitmap_for_each(bitmap, &unit, test_iova_bitmap_fill); + if (!IS_ERR(bitmap)) + iova_bitmap_free(bitmap); + missed = IS_ERR_VALUE(addr) ? len : copy_from_user(actual, (void __user *)addr, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual, 0, SZ_4K - sizeof(u64)), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64), 0xff, bytes - 1), NULL); + KUNIT_EXPECT_EQ(test, actual[SZ_4K - sizeof(u64) + bytes - 1], (unsigned char)1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(actual + SZ_4K - sizeof(u64) + bytes, + 0, SZ_4K + sizeof(u64)), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap crossed %lu-record window and revisited first byte", capacity); +} + +static void user4k_iova_bitmap_retained_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 2 * size; + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, false); + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, replacement = 0, missed = 0; + struct iova_bitmap *bitmap; + struct folio *folio; + loff_t pos = size; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(file, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, + (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 3 * SZ_4K, SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + folio = filemap_get_folio(file->f_mapping, size >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + kthread_use_mm(mm); + ret = vm_munmap(TEST_VA, len); + if (!ret) { + replacement = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + if (!IS_ERR_VALUE(replacement)) { + memset(buffer, 0x5a, len); + missed = copy_to_user((void __user *)replacement, buffer, len); + } + } + /* The cached pins still refer to the old file, not the new VMA. */ + iova_bitmap_set(bitmap, 9 * SZ_4K, 2 * SZ_4K); + if (!ret && !IS_ERR_VALUE(replacement)) + missed |= copy_from_user(buffer, (void __user *)replacement, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, replacement, TEST_VA); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0x5a, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_ASSERT_EQ(test, kernel_read(file, buffer, len, &pos), (ssize_t)len); + KUNIT_EXPECT_EQ(test, buffer[0], (unsigned char)8); + KUNIT_EXPECT_EQ(test, buffer[1], (unsigned char)6); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2, 0, len - 2), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK shared backing retained across unmap/replacement", size / SZ_1K); + } +} + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot); + +static void user4k_iova_bitmap_partial_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bitmap requires hardware opt-in"); + return; + } + for (mode = 0; mode < (PAGE_SHIFT > 14 ? 3U : 2U); mode++) { + struct mm_struct *mm = test_vma_mm_granule(test, shifts[mode]); + unsigned long size = 1UL << shifts[mode], len = 3 * size; + unsigned char *buffer = kunit_kzalloc(test, len, GFP_KERNEL); + unsigned long addr, refill, missed; + struct iova_bitmap *bitmap; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, buffer); + kthread_use_mm(mm); + addr = vm_mmap(NULL, TEST_VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + missed = IS_ERR_VALUE(addr) ? len : copy_to_user((void __user *)addr, buffer, len); + ret = vm_munmap(TEST_VA + size, size); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed |= copy_from_user(buffer, (void __user *)TEST_VA, size); + missed |= copy_from_user(buffer + 2 * size, (void __user *)(TEST_VA + 2 * size), size); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, addr, TEST_VA); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iova_free, bitmap), 0); + /* Preserve the existing void API: pin failure stops at the valid prefix. */ + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, size), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 2 * size, 0, size), NULL); + kthread_use_mm(mm); + refill = vm_mmap(NULL, TEST_VA + size, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + missed = copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, refill, TEST_VA + size); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0xff, len), NULL); + kunit_release_action(test, test_iova_free, bitmap); + memset(buffer, 0, len); + kthread_use_mm(mm); + missed = copy_to_user((void __user *)TEST_VA, buffer, len); + ret = test_mprotect(TEST_VA, len, PROT_READ); + bitmap = iova_bitmap_alloc(0, len * 8 * SZ_4K, SZ_4K, (u64 __user *)TEST_VA); + if (!IS_ERR(bitmap)) { + iova_bitmap_set(bitmap, 0, len * 8 * SZ_4K); + iova_bitmap_free(bitmap); + } + missed |= copy_from_user(buffer, (void __user *)TEST_VA, len); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, missed, 0UL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, len), NULL); + kunit_release_action(test, user4k_vma_mm_free, mm); + kunit_info(test, "IOVA bitmap %luK partial pin, retry and read-only checks passed", size / SZ_1K); + } +} + +static u64 test_table_descriptor(struct mm_struct *mm, unsigned long addr, bool pud) +{ + pgd_t *pgdp = pgd_offset(mm, addr); + p4d_t *p4dp = p4d_offset_mm(mm, pgdp, addr); + pud_t *pudp = pud_offset_mm(mm, p4dp, addr); + + return pud ? pud_val(*pudp) : pmd_val(*pmd_offset_mm(mm, pudp, addr)); +} + +static void user4k_mremap_move_test(struct kunit *test) +{ + static const struct { unsigned long old, new, len; unsigned int level; } cases[] = { + { TEST_VA + SZ_4K, 3 * SZ_2M + 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_2M - SZ_4K, SZ_1G - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { SZ_1G - SZ_4K, BIT_ULL(39) - 2 * SZ_4K, 3 * SZ_4K, 0 }, + { 4 * SZ_2M, 8 * SZ_2M, SZ_2M, 1 }, + { 2UL * SZ_1G, 4UL * SZ_1G, SZ_1G, 2 }, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(cases); shape++) { + struct mm_struct *mm = test_vma_mm(test); + unsigned long old = cases[shape].old, new = cases[shape].new; + unsigned long len = cases[shape].len; + unsigned long offsets[] = { 0, SZ_4K, len - SZ_4K }; + unsigned long result, warm, moved, vm_before, commit_before, rss_before; + unsigned long flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE; + unsigned long old_left[3], new_left[3]; + phys_addr_t phys[3]; + u8 readback[3], value = 0x79, byte; + u64 table = 0; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (cases[shape].level) + flags |= MAP_NORESERVE; + kthread_use_mm(mm); + warm = vm_mmap(NULL, SZ_1M, SZ_4K, PROT_READ | PROT_WRITE, flags, 0); + result = vm_mmap(NULL, old, len, PROT_READ | PROT_WRITE, flags, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, warm, SZ_1M); + KUNIT_ASSERT_EQ(test, result, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, warm, &value, 1, FOLL_WRITE), 1); + for (i = 0; i < 3; i++) { + value = 0x80 + i; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + offsets[i], &value, 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + offsets[i])); + } + if (cases[shape].level) { + table = test_table_descriptor(mm, old, cases[shape].level == 2); + /* Warm-up consumed the first fragment; the moved table is packed. */ + KUNIT_EXPECT_NE(test, table & (3UL * SZ_4K), 0ULL); + } + vm_before = mm->total_vm; + commit_before = mm->committed_user_pages; + rss_before = get_mm_rss_sum(mm); + kthread_use_mm(mm); + pagefault_disable(); + for (i = 0; i < 3; i++) + old_left[i] = copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + pagefault_enable(); + moved = test_mremap(old, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, new); + pagefault_disable(); + for (i = 0; i < 3; i++) { + new_left[i] = copy_from_user(&readback[i], (void __user *)(new + offsets[i]), 1); + old_left[i] += !copy_from_user(&byte, (void __user *)(old + offsets[i]), 1); + } + pagefault_enable(); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->total_vm, vm_before); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, commit_before); + KUNIT_EXPECT_EQ(test, get_mm_rss_sum(mm), rss_before); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_EQ(test, old_left[i], 0UL); + KUNIT_EXPECT_EQ(test, new_left[i], 0UL); + KUNIT_EXPECT_EQ(test, readback[i], (u8)(0x80 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + offsets[i])), phys[i]); + } + if (cases[shape].level) + KUNIT_EXPECT_EQ(test, test_table_descriptor(mm, new, cases[shape].level == 2), table); + } +} + +static void user4k_stack_relocate_test(struct kunit *test) +{ + const unsigned long shifts[] = { SZ_4K, SZ_2M + SZ_4K, SZ_1G + 3 * SZ_4K }; + unsigned int shape, i; + + for (shape = 0; shape < ARRAY_SIZE(shifts); shape++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct vm_page_offset pos; + unsigned long top, old, new; + phys_addr_t phys[3]; + u8 bytes[3] = { 0x42, 0x73, 0xb5 }; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &vma, &top), 0); + old = vma->vm_end - 3 * SZ_4K; + new = old - shifts[shape]; + mmap_write_lock(mm); + ret = expand_stack_locked(vma, old); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, + &bytes[i], 1, FOLL_WRITE), 1); + phys[i] = pte_phys_mm(mm, test_fault_entry(test, mm, old + i * SZ_4K)); + } + pos = vma_page_offset_at(vma, old); + mmap_write_lock(mm); + ret = relocate_vma_down(vma, shifts[shape]); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, vma->vm_start, new); + KUNIT_EXPECT_EQ(test, vma->vm_end, new + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, new + i * SZ_4K)), phys[i]); + } + /* Grow after relocation: keep the file-like anon offset linear. */ + mmap_write_lock(mm); + vm_flags_clear(vma, VM_STACK_INCOMPLETE_SETUP); + ret = expand_stack_locked(vma, new - SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new - SZ_4K, &bytes[0], 1, FOLL_WRITE), 1); + KUNIT_EXPECT_TRUE(test, vm_page_offset_equal(vma_page_offset_at(vma, new), pos)); + } +} + +static void user4k_mremap_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + unsigned long old = TEST_VA + SZ_4K, new = SZ_2M - 2 * SZ_4K, mapped; + unsigned long moved; + u8 value = 0x29, byte; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + parent = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(parent); + mapped = vm_mmap(NULL, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(parent); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(parent, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(child); + /* Move only the middle quarter, leaving both neighbors mapped. */ + moved = test_mremap(old + SZ_4K, SZ_4K, SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, moved, new); + value = 0xa4; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &value, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(parent, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x29); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, new, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, mm_pages_to_bytes(child, child->total_vm), 3UL * SZ_4K); + KUNIT_EXPECT_EQ(test, child->committed_user_pages, 3UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old, &value, 1, 0), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, old + 2 * SZ_4K, &value, 1, 0), 1); +} + +static void user4k_mremap_resize_test(struct kunit *test) +{ + struct mm_struct *mm; + unsigned long old = TEST_VA + SZ_4K, new = TEST_VA + 8 * PAGE_SIZE + 2 * SZ_4K; + unsigned long res[7]; + u8 value = 0x98, byte = 0; + int unmap[2]; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + res[0] = vm_mmap(NULL, old, SZ_4K, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + res[1] = test_mremap(old, SZ_4K, 3 * SZ_4K - 1, 0, 0); + res[2] = test_mremap(old + 1, SZ_4K, SZ_4K, 0, 0); + res[3] = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new + 1); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[0], old); + KUNIT_ASSERT_EQ(test, res[1], old); + KUNIT_EXPECT_EQ(test, res[2], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, res[3], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 3UL); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + SZ_4K, &value, 1, FOLL_WRITE), 1); + kthread_use_mm(mm); + res[4] = test_mremap(old, 3 * SZ_4K, 2 * SZ_4K, 0, 0); + res[5] = test_mremap(old, 2 * SZ_4K, 2 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE | MREMAP_DONTUNMAP, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, res[4], old); + KUNIT_ASSERT_EQ(test, res[5], new); + KUNIT_EXPECT_EQ(test, mm->total_vm, 4UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 4UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, value); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0); + kthread_use_mm(mm); + unmap[0] = vm_munmap(old, 2 * SZ_4K); + unmap[1] = vm_munmap(new, 2 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, unmap[0], 0); + KUNIT_EXPECT_EQ(test, unmap[1], 0); + KUNIT_EXPECT_EQ(test, mm->total_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, 0UL); +} + +static int test_reject_mremap(struct vm_area_struct *vma) +{ + return -EIO; +} + +static void user4k_cold_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + unsigned long start = TEST_VA, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x68; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm cold fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + /* Only the last quarter is young: batching must aggregate that bit. */ + mmap_read_lock(mm); + vma = vma_lookup(mm, start); + for (i = 0; i < 3; i++) { + unsigned long addr = start + i * SZ_4K; + + ptep = test_mm_lookup(mm, addr, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing cold test PTE"); + return; + } + entry = ptep_clear_flush(vma, addr, ptep); + set_pte_at(mm, addr, ptep, pte_mkold(entry)); + pte_unmap_unlock(ptep, ptl); + } + mmap_read_unlock(mm); + kthread_use_mm(mm); + ret = do_madvise(mm, start + 3 * SZ_4K, SZ_4K, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, pte_young(test_fault_entry(test, mm, start + 3 * SZ_4K))); + kunit_release_action(test, user4k_vma_mm_free, child); + kthread_use_mm(mm); + ret = do_madvise(mm, start, PAGE_SIZE, MADV_COLD); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_FALSE(test, pte_young(test_fault_entry(test, mm, start + i * SZ_4K))); + byte = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x68); +} + +static void user4k_prefault_dirty_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long start = TEST_VA + SZ_4K, mapped; + spinlock_t *ptl; + pte_t *ptep, entry; + u8 byte = 0x7a; + int locked = 1; + long ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm prefault fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, SZ_4K, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start, &byte, 1, FOLL_WRITE), 1); + mmap_read_lock(mm); + ptep = test_mm_lookup(mm, start, &ptl); + if (!ptep) { + mmap_read_unlock(mm); + KUNIT_FAIL(test, "missing prefault test PTE"); + return; + } + entry = ptep_clear_flush(vma_lookup(mm, start), start, ptep); + folio = page_folio(pte_page(entry)); + folio_get(folio); + set_pte_at(mm, start, ptep, pte_mkclean(entry)); + folio_clear_dirty(folio); + pte_unmap_unlock(ptep, ptl); + mmap_read_unlock(mm); + kthread_use_mm(mm); + mmap_read_lock(mm); + ret = faultin_page_range(mm, start, start + SZ_4K, true, &locked); + if (locked) + mmap_read_unlock(mm); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + folio_put(folio); +} + +struct test_fsync_record { + loff_t start, end; + unsigned int calls; + int datasync; +}; + +static int test_fsync_range(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct test_fsync_record *record = file->private_data; + + *record = (struct test_fsync_record) { start, end, record->calls + 1, datasync }; + return 0; +} + +asmlinkage long __arm64_sys_msync(const struct pt_regs *regs); + +static void user4k_msync_offsets_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct file_operations ops; + const struct file_operations *original_ops; + void *original_private; + struct test_fsync_record records[3] = {}, record = {}; + struct pt_regs regs = { .regs = { TEST_VA + SZ_4K, 1, MS_SYNC } }; + long result[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm sync fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 3 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 5 * SZ_4K, 3 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = file->f_op; + original_private = file->private_data; + ops = *original_ops; + ops.fsync = test_fsync_range; + file->f_op = &ops; + file->private_data = &record; + kthread_use_mm(vma->vm_mm); + for (i = 0; i < ARRAY_SIZE(result); i++) { + regs.regs[0] = vma->vm_start + i * SZ_4K; + result[i] = __arm64_sys_msync(®s); + records[i] = record; + } + kthread_unuse_mm(vma->vm_mm); + file->f_op = original_ops; + file->private_data = original_private; + for (i = 0; i < ARRAY_SIZE(result); i++) { + KUNIT_EXPECT_EQ(test, result[i], 0L); + KUNIT_EXPECT_EQ(test, records[i].calls, i + 1); + KUNIT_EXPECT_EQ(test, records[i].start, (loff_t)(3 + i) * SZ_4K); + KUNIT_EXPECT_EQ(test, records[i].end, (loff_t)(4 + i) * SZ_4K - 1); + KUNIT_EXPECT_EQ(test, records[i].datasync, 1); + } +} + +static void user4k_mremap_file_test(struct kunit *test) +{ + struct mm_struct *mm; + struct vm_area_struct *vma; + struct file *file; + struct vm_operations_struct rejecting_ops; + const struct vm_operations_struct *original_ops; + unsigned long old = TEST_VA + SZ_4K, new = SZ_1G - SZ_4K; + unsigned long mapped, moved; + u8 bytes[3]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(file, old, 3 * SZ_4K, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 3 * SZ_4K); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &bytes[i], 1, 0), 1); + vma = test_vma_lookup(mm, old); + KUNIT_ASSERT_NOT_NULL(test, vma); + original_ops = vma->vm_ops; + rejecting_ops = *original_ops; + rejecting_ops.mremap = test_reject_mremap; + mmap_write_lock(mm); + vma->vm_ops = &rejecting_ops; + mmap_write_unlock(mm); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + /* Restore stack-owned callbacks before any assertion can leave this scope. */ + mmap_write_lock(mm); + { + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *part; + + for_each_vma(vmi, part) + if (part->vm_ops == &rejecting_ops) + part->vm_ops = original_ops; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, moved, (unsigned long)-EIO); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, old + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + KUNIT_EXPECT_EQ(test, mm->total_vm, 3UL); + kthread_use_mm(mm); + moved = test_mremap(old, 3 * SZ_4K, 3 * SZ_4K, + MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + vma = test_vma_lookup(mm, new); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma_subpage_offset(vma), 3U * SZ_4K); + for (i = 0; i < 3; i++) { + u8 byte = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, new + i * SZ_4K, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, bytes[i]); + } +} + +static void user4k_exec_arguments_test(struct kunit *test) +{ + unsigned int native; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm stack setup requires hardware opt-in"); + return; + } + for (native = 0; native < 2; native++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + struct mm_struct *mm; + unsigned long size, mask, initial, copied, new_top; + unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; + char *a, *b, *buf, *too_long; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + mm = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + mask = mm_page_mask(mm); + a_len = min(a_len, 16 * size + 31); + bprm->mm = mm; + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + too_long = kunit_kmalloc(test, 32 * size + 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + KUNIT_ASSERT_NOT_NULL(test, too_long); + memset(a, 'A', a_len); + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + memset(too_long, 'x', 32 * size); + too_long[32 * size] = 0; + KUNIT_EXPECT_EQ(test, copy_string_kernel(too_long, bprm), -E2BIG); + KUNIT_EXPECT_EQ(test, bprm->p, initial); + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + bprm->exec = bprm->p; + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_EXPECT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + new_top = (bprm->vma->vm_end - SZ_2M - size) & mask; + kthread_use_mm(mm); + ret = setup_arg_pages(bprm, new_top, EXSTACK_DISABLE_X); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Subtract < PAGE_SIZE, then round down to 16 bytes and up to a leaf. */ + KUNIT_EXPECT_LE(test, bprm->vma->vm_end, new_top); + KUNIT_EXPECT_LE(test, new_top - bprm->vma->vm_end, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, bprm->vma->vm_end & ~mask, 0UL); + KUNIT_EXPECT_EQ(test, mm->start_stack, bprm->p); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, 0), (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_MEMEQ(test, buf + b_len + 1, a, a_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* /proc/PID/cmdline and environ read the stack with FOLL_ANON. */ + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, bprm->p, buf, copied, FOLL_ANON), + (int)copied); + KUNIT_EXPECT_MEMEQ(test, buf, b, b_len + 1); + KUNIT_EXPECT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Interpreter argument removal scans several user granules. */ + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +/* The transfer can replace bprm->mm, so release its final owner. */ +static void test_exec_mm_free(void *data) +{ + struct linux_binprm *bprm = data; + + mmput(bprm->mm); +} + +static void user64k_exec_arguments_test(struct kunit *test) +{ + unsigned int shifts[] = { 16, 12, PAGE_SHIFT }; + unsigned int mode; + + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and MM opt-in"); + return; + } + for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + unsigned long a_len = mode ? SZ_64K + SZ_16K + 31 : 32 * PAGE_SIZE + 31; + unsigned long b_len = PAGE_SIZE + 17, initial, copied; + char *a, *b, *buf; + unsigned long i; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + bprm->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, bprm->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_exec_mm_free, bprm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(bprm->mm, 16), 0); + KUNIT_ASSERT_EQ(test, create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); + b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); + buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, buf); + for (i = 0; i < a_len; i++) + a[i] = 'A' + i % 23; + a[a_len] = 0; + memset(b, 'b', b_len); + b[b_len] = 0; + KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, 5, 0), 5); + KUNIT_ASSERT_STREQ(test, buf, "tail"); + KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); + bprm->argc = 3; + copied = initial - bprm->p; + KUNIT_ASSERT_EQ(test, copied, a_len + b_len + 7); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + /* Exercise copying out of every native page within a coarse leaf. */ + if (shifts[mode] != 16) { + KUNIT_ASSERT_EQ(test, bprm_set_page_shift(bprm, shifts[mode]), 0); + KUNIT_ASSERT_EQ(test, mm_page_shift(bprm->mm), shifts[mode]); + memset(buf, 0, copied); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); + KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); + KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); + } + initial = bprm->p; + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); + KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); + KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); + KUNIT_EXPECT_EQ(test, bprm->argc, 0); + KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); + } +} + +static struct vm_area_struct *test_special_vma(struct mm_struct *mm, + const struct vm_special_mapping *sm, unsigned long start, unsigned long len, + vm_flags_t flags) +{ + struct vm_area_struct *vma; + + mmap_write_lock(mm); + vma = _install_special_mapping(mm, start, len, flags, sm); + mmap_write_unlock(mm); + return vma; +} + +static void user4k_special_mapping_test(struct kunit *test) +{ + struct vm_special_mapping *sm = kunit_kzalloc(test, sizeof(*sm), GFP_KERNEL); + struct page **pages = kunit_kcalloc(test, 3, sizeof(*pages), GFP_KERNEL); + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct mm_struct *small = test_vma_mm(test), *cold = test_vma_mm(test); + struct mm_struct *native = mm_alloc(); + struct vm_area_struct *vma[3]; + unsigned long start = SZ_2M - SZ_4K; + unsigned int i, side; + u8 byte; + + KUNIT_ASSERT_NOT_NULL(test, sm); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cold); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + /* Reverse the supplied native pages: folio indices cannot identify them. */ + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + sm->pages = pages; + sm->name = "[user4k-special-test]"; + for (i = 0; i < 2 * TEST_SLOTS; i++) + memset(page_address(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K, 0x41 + i, SZ_4K); + vma[0] = test_special_vma(small, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[1] = test_special_vma(cold, sm, start, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + vma[2] = test_special_vma(native, sm, TEST_VA, 2 * PAGE_SIZE, VM_READ | VM_MAYREAD | VM_MAYWRITE); + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma[i]); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + unsigned long addr = start + i * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(small, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x41 + i)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, addr)), + page_to_phys(pages[i / TEST_SLOTS]) + (i % TEST_SLOTS) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(small, MM_FILEPAGES), 2UL * TEST_SLOTS); + /* Both present-PTE and cold-PTE forced writes copy only their quarter. */ + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? cold : small; + + byte = 0xe1 + side; + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, + &byte, 1, FOLL_WRITE | FOLL_FORCE), 1); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, start + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0xe1 + side)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native, TEST_VA + 2 * SZ_4K + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)0x43); + } + { + struct user_page_fragment fragment; + long ret = pin_user_fragments_remote(small, start + 5 * SZ_4K + 13, + 7, 0, &fragment, 1); + + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, (5 * SZ_4K + 13UL) & ~PAGE_MASK); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment.folio) + fragment.offset, + 0x46, fragment.length), NULL); + release_user_fragments(&fragment, 1, false); + } +} + +static void user4k_insert_range_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page *pages[2]; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long size, start, offset, length, num; + unsigned int i; + int ret, invalid[4]; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + size = mm_page_size(mm); + start = pmd_size_mm(mm) - size; + offset = native ? 0 : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + invalid[0] = vm_insert_pages_range(vma, start + 1, pages, 2, offset, length); + invalid[1] = vm_insert_pages_range(vma, start, pages, 2, offset + 1, length); + invalid[2] = vm_insert_pages_range(vma, start, pages, 2, offset, length + size); + invalid[3] = vm_insert_pages_range(vma, start, pages, ULONG_MAX, 0, length); + ret = vm_insert_pages_range(vma, start, pages, 2, offset, length); + mmap_write_unlock(mm); + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], -EINVAL); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), length / size); + /* The original native-page-count API also expands all user leaves. */ + start += 4 * PAGE_SIZE; + vma = test_vma_add(mm, start, 2 * PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, num, 0UL); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[(i * size) >> PAGE_SHIFT]) + offset_in_page(i * size)); + num = 2; + mmap_write_lock(mm); + ret = vm_insert_pages(vma, start, pages, &num); + mmap_write_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + KUNIT_EXPECT_EQ(test, num, 2UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), + (length + 2 * PAGE_SIZE) / size); + start += 4 * PAGE_SIZE; + offset = native ? PAGE_SIZE : 3 * SZ_4K; + length = 2 * PAGE_SIZE - offset; + vma = test_vma_add(mm, start, length, + (struct vm_page_offset) { .index = offset >> PAGE_SHIFT, + .offset = offset_in_page(offset) }, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = vm_map_pages(vma, pages, 2); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < length / size; i++) { + unsigned long source = offset + i * size; + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + i * size)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[0]), 1); + KUNIT_EXPECT_EQ(test, page_ref_count(pages[1]), 1); + } +} + +static void user4k_remap_pfn_range_test(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 1); + + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (unsigned int native = 0; native < 2; native++) { + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); + unsigned long size, boundaries[3]; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, mm), 0); + size = mm_page_size(mm); + boundaries[0] = pmd_size_mm(mm); + boundaries[1] = pud_size_mm(mm); + boundaries[2] = pgd_size_mm(mm); + for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { + unsigned long start = boundaries[shape] - size; + unsigned long length = PAGE_SIZE + size; + struct vm_area_struct *vma; + int ret; + + if (shape && boundaries[shape] == boundaries[shape - 1]) + continue; + vma = test_vma_add(mm, start, length, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_SHARED | VM_MAYSHARE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_lock(mm); + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + for (unsigned long offset = 0; offset < length; offset += size) { + pte_t pte = test_fault_entry(test, mm, start + offset); + + struct follow_pfnmap_args args = { + .vma = vma, .address = start + offset + 31, + }; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + int ret; + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_special(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), + PFN_PHYS(folio_pfn(folio)) + offset); + + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) + follow_pfnmap_end(&args); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, native ? 0 : -EOPNOTSUPP); + args.allow_subpage = true; + mmap_read_lock(mm); + ret = follow_pfnmap_start(&args); + if (!ret) { + KUNIT_EXPECT_EQ(test, args.pfn, (unsigned long)PHYS_PFN(expected)); + KUNIT_EXPECT_EQ(test, args.offset, (unsigned int)offset_in_page(expected)); + KUNIT_EXPECT_EQ(test, args.addr_mask, mm_page_mask(mm)); + KUNIT_EXPECT_TRUE(test, args.special); + follow_pfnmap_end(&args); + } + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start - size))); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, start + length))); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +#ifdef CONFIG_HAVE_IOREMAP_PROT +static void test_iounmap(void *arg) +{ + iounmap((void __iomem *)arg); +} + +static const struct vm_operations_struct test_iomem_ops = { + .access = generic_access_phys, +}; +#endif + +static void user4k_iomem_access_test(struct kunit *test) +{ +#if defined(CONFIG_HAVE_IOREMAP_PROT) && defined(CONFIG_OF) + struct device_node *node; + struct resource resource; + struct mm_struct *mm; + struct vm_area_struct *vma; + void __iomem *uart; + unsigned long start = SZ_2M + PAGE_SIZE - SZ_4K; + u8 reference[16], buffer[32]; + vm_fault_t fault; + int ret; + + if (!of_machine_is_compatible("linux,dummy-virt")) { + kunit_skip(test, "requires disposable QEMU virt PL011 identification registers"); + return; + } + node = of_find_compatible_node(NULL, NULL, "arm,pl011"); + KUNIT_ASSERT_NOT_NULL(test, node); + ret = of_address_to_resource(node, 0, &resource); + uart = ret ? NULL : of_iomap(node, 0); + of_node_put(node); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, uart); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_iounmap, + (void *)uart), 0); + KUNIT_ASSERT_GE(test, resource_size(&resource), (resource_size_t)SZ_4K); + memcpy_fromio(reference, uart + 0xff0, sizeof(reference)); + KUNIT_ASSERT_NOT_NULL(test, memchr_inv(reference, 0, sizeof(reference))); + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_vma_add(mm, start, 2 * SZ_4K, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_SHARED | VM_MAYSHARE | VM_IO | VM_PFNMAP); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_ops = &test_iomem_ops; + vma->vm_page_prot = pgprot_device(vm_get_page_prot(vma->vm_flags)); + mmap_read_lock(mm); + fault = vmf_insert_pfn_prot_mkwrite_offset(vma, start, + PHYS_PFN(resource.start), offset_in_page(resource.start), + vma->vm_page_prot, false); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + memset(buffer, 0x6d, sizeof(buffer)); + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, sizeof(buffer), 0); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x6d, 16), NULL); + memset(buffer, 0x5a, sizeof(buffer)); + ret = access_remote_vm(mm, start + 0xff0, buffer, sizeof(buffer), 0); + KUNIT_EXPECT_EQ(test, ret, 16); + KUNIT_EXPECT_MEMEQ(test, buffer, reference, sizeof(reference)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer + 16, 0x5a, 16), NULL); + /* Writes must fail before touching the read-only device mapping. */ + mmap_read_lock(mm); + ret = generic_access_phys(vma, start + 0xff0, buffer, 1, FOLL_WRITE); + mmap_read_unlock(mm); + KUNIT_EXPECT_LT(test, ret, 0); +#else + kunit_skip(test, "requires OF and iomem access support"); +#endif +} + +static void user4k_pfn_offset_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = SZ_2M - SZ_4K; + unsigned long pfn; + unsigned int i; + vm_fault_t result[4], invalid[3], write; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + pfn = page_to_pfn(virt_to_page(data)); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | VM_SHARED | VM_MAYSHARE | + VM_PFNMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + /* Model a write-notify VMA: initial PTEs are read-only. */ + vma->vm_page_prot = vm_get_page_prot(VM_READ | VM_SHARED); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + i * SZ_4K, + pfn, (3 - i) * SZ_4K, vma->vm_page_prot, false); + invalid[0] = vmf_insert_pfn_prot_mkwrite_offset(vma, start + 1, pfn, + 0, vma->vm_page_prot, false); + invalid[1] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + 1, vma->vm_page_prot, false); + invalid[2] = vmf_insert_pfn_prot_mkwrite_offset(vma, start, pfn, + PAGE_SIZE, vma->vm_page_prot, false); + write = vmf_insert_pfn_prot_mkwrite_offset(vma, start + SZ_4K, pfn, + 2 * SZ_4K, vma->vm_page_prot, true); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(pfn) + (3 - i) * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_EQ(test, !!pte_write(entry), i == 1); + } + for (i = 0; i < ARRAY_SIZE(invalid); i++) + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, write, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 0UL); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(virt_to_page(data)), 1); +} + +static void user16k_insert_shared_test(struct kunit *test) +{ + struct mm_struct *middle, *small, *native; + struct vm_area_struct *vmas[3]; + struct mm_subpage_mapcounts counts; + struct page *page; + void *data; + vm_fault_t result; + unsigned int i; + vm_flags_t flags = VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "mixed16K file insertion requires larger native backing"); + return; + } + data = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + page = virt_to_page(data); + middle = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, middle); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, middle), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(middle, 14), 0); + small = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, small); + native = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + vmas[0] = test_vma_add(middle, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + vmas[1] = test_vma_add(small, TEST_VA + SZ_4K, SZ_4K, (struct vm_page_offset) {}, flags); + vmas[2] = test_vma_add(native, TEST_VA, PAGE_SIZE, (struct vm_page_offset) {}, flags); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmas[i]); + vmas[i]->vm_page_prot = vm_get_page_prot(vmas[i]->vm_flags); + } + for (i = 0; i < PAGE_SIZE / SZ_16K; i++) { + unsigned long address = TEST_VA + i * SZ_16K; + + mmap_read_lock(middle); + result = vmf_insert_page_offset(vmas[0], address, page, i * SZ_16K); + mmap_read_unlock(middle); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, pte_phys_mm(middle, test_fault_entry(test, middle, address)), + page_to_phys(page) + i * SZ_16K); + } + mmap_read_lock(small); + result = vmf_insert_page_offset(vmas[1], vmas[1]->vm_start, page, PAGE_SIZE - SZ_4K); + mmap_read_unlock(small); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + mmap_read_lock(native); + result = vmf_insert_page_offset(vmas[2], vmas[2]->vm_start, page, 0); + mmap_read_unlock(native); + KUNIT_ASSERT_EQ(test, result, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 5U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 2U : 1U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 6); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 7); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(middle, MM_FILEPAGES), 4UL); + /* Real mm teardown removes whole16K leaves and preserves both aliases. */ + kunit_release_action(test, user4k_vma_mm_free, middle); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 1U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], i == TEST_SLOTS - 1 ? 1U : 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 2); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 3); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small, test_fault_entry(test, small, vmas[1]->vm_start)), + page_to_phys(page) + PAGE_SIZE - SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native, test_fault_entry(test, native, vmas[2]->vm_start)), + page_to_phys(page)); + kunit_release_action(test, user4k_vma_mm_free, small); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 1); + kunit_release_action(test, user4k_vma_mm_free, native); + KUNIT_EXPECT_EQ(test, folio_mapcount(page_folio(page)), 0); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +static void user4k_insert_fragment_test(struct kunit *test) +{ + void *data = test_alloc_page(test); + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + unsigned long start = TEST_VA + SZ_4K; + struct page *page; + vm_fault_t result[4], invalid[4]; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, mm); + page = virt_to_page(data); + vma = test_vma_add(mm, start, PAGE_SIZE, (struct vm_page_offset) {}, + VM_READ | VM_MAYREAD | VM_MIXEDMAP | VM_IO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_read_lock(mm); + for (i = 0; i < 4; i++) + result[i] = vmf_insert_page_offset(vma, start + i * SZ_4K, page, (3 - i) * SZ_4K); + invalid[0] = vmf_insert_page_offset(vma, start + 1, page, 0); + invalid[1] = vmf_insert_page_offset(vma, start, page, 1); + invalid[2] = vmf_insert_page_offset(vma, start, page, PAGE_SIZE); + invalid[3] = vmf_insert_page_offset(vma, vma->vm_end, page, 0); + mmap_read_unlock(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, result[i], (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, invalid[i], (vm_fault_t)VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)), + page_to_phys(page) + (3 - i) * SZ_4K); + } + KUNIT_EXPECT_EQ(test, page_ref_count(page), 5); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_FILEPAGES), 4UL); + kunit_release_action(test, user4k_vma_mm_free, mm); + KUNIT_EXPECT_EQ(test, page_ref_count(page), 1); +} + +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) +static void user4k_vdso_mapping_test(struct kunit *test) +{ + struct mm_struct *mm; + struct linux_binprm bprm = {}; + unsigned long code, base, left[8]; + u8 bytes[8], magic[4]; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm vDSO fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + mm->mmap_base = TEST_VA + SZ_4K; + mm_flags_clear(MMF_TOPDOWN, mm); + kthread_use_mm(mm); + ret = arch_setup_additional_pages(&bprm, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + code = (unsigned long)mm->context.vdso; + KUNIT_ASSERT_NE(test, code, 0UL); + base = code - VDSO_NR_PAGES * PAGE_SIZE; + KUNIT_EXPECT_EQ(test, base & ~PAGE_MASK, SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, code, magic, sizeof(magic), 0), (int)sizeof(magic)); + KUNIT_EXPECT_MEMEQ(test, magic, "\177ELF", 4); + /* VVAR forbids remote GUP; ordinary uaccess must still fault it in. */ + kthread_use_mm(mm); + for (i = 0; i < 4; i++) { + left[i] = copy_from_user(&bytes[i], (void __user *)(base + i * SZ_4K), 1); + left[4 + i] = copy_from_user(&bytes[4 + i], + (void __user *)(base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K), 1); + } + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, left[i], 0UL); + KUNIT_EXPECT_EQ(test, left[4 + i], 0UL); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, base + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_time_data)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, test_fault_entry(test, mm, + base + VDSO_RNG_PAGE_OFFSET * PAGE_SIZE + i * SZ_4K)), + page_to_phys(virt_to_page(vdso_k_rng_data)) + i * SZ_4K); + } + KUNIT_EXPECT_EQ(test, access_remote_vm(mm, base, magic, sizeof(magic), 0), 0); +} + +#endif + +static void user4k_remote_access_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA + SZ_4K, 3 * SZ_4K); + struct mm_struct *child = test_vma_mm(test); + struct vm_area_struct *child_vma; + unsigned long start = TEST_VA + SZ_4K; + u8 *buf = kunit_kmalloc(test, 3 * SZ_4K, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x5a, 3 * SZ_4K); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, FOLL_WRITE), + 3 * (int)SZ_4K); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Deliberately straddle quarters whose physical/virtual offsets differ. */ + memset(buf, 0x96, SZ_4K + 34); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + SZ_4K - 17, + buf, SZ_4K + 34, FOLL_WRITE), (int)SZ_4K + 34); + memset(buf, 0, 3 * SZ_4K); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, start, buf, 3 * SZ_4K, 0), + 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, 3 * SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start, buf, 3 * SZ_4K, 0), 3 * (int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x5a, SZ_4K - 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + SZ_4K - 17, 0x96, SZ_4K + 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 2 * SZ_4K + 17, 0x5a, SZ_4K - 17), NULL); + child_vma = test_vma_lookup(child, start); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + mmap_write_lock(child); + vm_flags_clear(child_vma, VM_WRITE); + mmap_write_unlock(child); + buf[0] = 0x27; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, FOLL_WRITE), 0); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, + FOLL_WRITE | FOLL_FORCE), 1); + buf[0] = 0; + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 1, buf, 1, 0), 1); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0x27); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K - 5, buf, 20, 0), 5); + KUNIT_EXPECT_EQ(test, access_remote_vm(child, start + 3 * SZ_4K, buf, 20, 0), 0); +} + +static void user4k_remote_file_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + u8 buf[34]; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x34, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0x35, 17), NULL); + memset(buf, 0xa6, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 3 * SZ_4K - 17, + buf, sizeof(buf), FOLL_WRITE), (int)sizeof(buf)); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xa6, sizeof(buf)), NULL); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + memset(buf, 0xb7, sizeof(buf)); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + TEST_VA + 3 * SZ_4K - 17, sizeof(buf), &iter, true), 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, sizeof(buf)); + KUNIT_EXPECT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 4 * SZ_4K - 17, + buf, sizeof(buf), 0), (int)sizeof(buf)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0xb7, sizeof(buf)), NULL); + } +#endif + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + KUNIT_EXPECT_EQ(test, access_remote_vm(small->vm_mm, small->vm_start, buf, 1, + FOLL_WRITE | FOLL_FORCE), 0); +#ifdef CONFIG_CROSS_MEMORY_ATTACH + { + struct kvec vec = { .iov_base = buf, .iov_len = sizeof(buf) }; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, sizeof(buf)); + KUNIT_EXPECT_EQ(test, process_vm_rw_fragments(small->vm_mm, + small->vm_start, sizeof(buf), &iter, true), -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), sizeof(buf)); + } +#endif +} + +#ifdef CONFIG_CROSS_MEMORY_ATTACH +static void user4k_process_vm_test(struct kunit *test) +{ + /* Exceed the stack batch, cross table boundaries and include a partial tail. */ + unsigned long start = SZ_2M - SZ_4K, len = 21 * SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, len); + struct mm_struct *child = test_vma_mm(test); + struct iov_iter iter; + struct kvec vec; + u8 *buf = kunit_kmalloc(test, len + 17, GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + KUNIT_ASSERT_NOT_NULL(test, buf); + memset(buf, 0x61, len); + vec = (struct kvec) { .iov_base = buf, .iov_len = len }; + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, true); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, iov_iter_count(&iter), 0UL); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(buf, 0xc7, len); + iov_iter_kvec(&iter, ITER_SOURCE, &vec, 1, len - 34); + ret = process_vm_rw_fragments(child, start + 17, len - 34, &iter, true); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + memset(buf, 0, len); + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len); + ret = process_vm_rw_fragments(vma->vm_mm, start, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, len), NULL); + vec.iov_len = len + 17; + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, len + 17); + ret = process_vm_rw_fragments(child, start, len + 17, &iter, false); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 17UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf, 0x61, 17), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 17, 0xc7, len - 34), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + len - 17, 0x61, 17), NULL); + /* A shorter local iterator ends successfully before the remote range. */ + iov_iter_kvec(&iter, ITER_DEST, &vec, 1, 23); + ret = process_vm_rw_fragments(child, start + 13, len, &iter, false); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 0UL); +} +#endif + +#ifdef CONFIG_FUTEX +static void test_stop_kthread(void *arg); + +struct test_futex_waiter { + struct mm_struct *mm; + unsigned long address; + struct completion ready; + struct completion done; + int result; +}; + +static int test_futex_wait_run(void *arg) +{ + struct test_futex_waiter *w = arg; + ktime_t deadline = ktime_add_ms(ktime_get(), 10000); + + kthread_use_mm(w->mm); + complete(&w->ready); + w->result = futex_wait((u32 __user *)w->address, FLAGS_SIZE_32 | FLAGS_SHARED, + 0x33333333, &deadline, FUTEX_BITSET_MATCH_ANY); + kthread_unuse_mm(w->mm); + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_futex_wake_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vmas[2]; + unsigned long addresses[2] = { TEST_VA + SZ_4K + 64, TEST_VA + 2 * SZ_4K + 64 }; + unsigned int round; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vmas[0] = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, SZ_4K, 2 * SZ_4K); + vmas[1] = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vmas[0]); + KUNIT_ASSERT_NOT_NULL(test, vmas[1]); + /* Both directions use the real wait queue and atomic user-value read. */ + for (round = 0; round < 2; round++) { + struct test_futex_waiter *w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + struct task_struct *task; + unsigned long until; + int woke = 0, wrong; + + KUNIT_ASSERT_NOT_NULL(test, w); + w->mm = vmas[round]->vm_mm; + w->address = addresses[round]; + init_completion(&w->ready); + init_completion(&w->done); + task = kthread_run(test_futex_wait_run, w, "user4k-futex-%u", round); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 5 * HZ), 0UL); + until = jiffies + 5 * HZ; + kthread_use_mm(vmas[round ^ 1]->vm_mm); + wrong = futex_wake((u32 __user *)(addresses[round ^ 1] + 4), + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + do { + woke = futex_wake((u32 __user *)addresses[round ^ 1], + FLAGS_SIZE_32 | FLAGS_SHARED, 1, FUTEX_BITSET_MATCH_ANY); + if (woke || completion_done(&w->done)) + break; + schedule_timeout_uninterruptible(1); + } while (time_before(jiffies, until)); + kthread_unuse_mm(vmas[round ^ 1]->vm_mm); + KUNIT_EXPECT_EQ(test, wrong, 0); + KUNIT_EXPECT_EQ(test, woke, 1); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->done, 12 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, w->result, 0); + kunit_release_action(test, test_stop_kthread, task); + } +} + +static void user4k_futex_keys_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *small, *native; + union futex_key keys[2], other; + unsigned int i; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm futex fixture requires hardware opt-in"); + return; + } + file = test_cache_file(test, 2 * PAGE_SIZE, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, 2 * SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 3; i++) { + unsigned long va = small->vm_start + i * SZ_4K + 64; + unsigned long fileoff = (i + 2) * SZ_4K + 64; + + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)va, FLAGS_SIZE_32 | FLAGS_SHARED, + &keys[0], FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + kthread_use_mm(native->vm_mm); + ret = get_futex_key((u32 __user *)(TEST_VA + fileoff), + FLAGS_SIZE_32 | FLAGS_SHARED, &keys[1], FUTEX_READ); + kthread_unuse_mm(native->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[0], &keys[1])); + KUNIT_EXPECT_EQ(test, keys[0].shared.pgoff, fileoff / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, keys[0].both.offset, (unsigned int)(fileoff % PAGE_SIZE) | FUT_OFF_INODE); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(va + 4), FLAGS_SIZE_32 | FLAGS_SHARED, + &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_FALSE(test, futex_match(&keys[0], &other)); + } + mmap_write_lock(small->vm_mm); + vm_flags_clear(small, VM_WRITE); + mmap_write_unlock(small->vm_mm); + kthread_use_mm(small->vm_mm); + ret = get_futex_key((u32 __user *)(small->vm_start + 2 * SZ_4K + 64), + FLAGS_SIZE_32 | FLAGS_SHARED, &other, FUTEX_READ); + kthread_unuse_mm(small->vm_mm); + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, futex_match(&keys[1], &other)); +} +#endif + +static void test_swap_usage_mapping_free(void *arg) +{ + struct address_space *mapping = arg; + + xa_destroy(&mapping->i_pages); +} + +static void user4k_shmem_swap_usage_test(struct kunit *test) +{ + struct address_space *mapping = kunit_kzalloc(test, sizeof(*mapping), GFP_KERNEL); + void *ret; + + KUNIT_ASSERT_NOT_NULL(test, mapping); + xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_swap_usage_mapping_free, mapping), 0); + /* Synthetic swap values exercise accounting without allocating swap slots. */ + ret = xa_store(&mapping->i_pages, 1, xa_mk_value(1), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + if (IS_ENABLED(CONFIG_XARRAY_MULTI)) { + ret = xa_store_range(&mapping->i_pages, 4, 7, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } else { + unsigned long index; + + /* Without multi-index entries, shmem stores one value per page. */ + for (index = 4; index <= 7; index++) { + ret = xa_store(&mapping->i_pages, index, + xa_mk_value(2), GFP_KERNEL); + KUNIT_ASSERT_FALSE(test, xa_is_err(ret)); + } + } + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE + SZ_4K, + PAGE_SIZE + 3 * SZ_4K), 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 3 * SZ_4K, + 2 * PAGE_SIZE + SZ_4K), PAGE_SIZE); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 4 * PAGE_SIZE + SZ_4K, + 7 * PAGE_SIZE + 3 * SZ_4K), 3 * PAGE_SIZE + 2UL * SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 5 * PAGE_SIZE + SZ_4K, + 5 * PAGE_SIZE + 2 * SZ_4K), (unsigned long)SZ_4K); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, 0, SZ_4K), 0UL); + KUNIT_EXPECT_EQ(test, shmem_partial_swap_usage_bytes(mapping, PAGE_SIZE, PAGE_SIZE), 0UL); +} + +static void test_file_mapcounts(struct kunit *test, struct folio *folio, + const unsigned int expected[TEST_SLOTS], unsigned int native) +{ + struct mm_subpage_mapcounts counts; + unsigned int i, total = 0; + + for (i = 0; i < TEST_SLOTS; i++) + total += expected[i]; + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(&folio->page, &counts), total); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], expected[i]); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(total + native)); +} + +static void user4k_file_mapcounts_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *native, *cvma; + struct mm_struct *child; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA, 3 * SZ_4K, SZ_4K); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 2 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & + VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 3, 1 }, 1); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, child); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 1); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 0); + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); +} + +/* A unique RAM-backed test inode: exercise generic file faults without + * making unsupported filesystem promotion part of this mapping fixture. + */ +static int test_coarse_read_folio(struct file *file, struct folio *folio) +{ + unsigned long i; + loff_t base = (loff_t)folio->index << PAGE_SHIFT; + loff_t size = i_size_read(file_inode(file)); + u8 *data = folio_address(folio); + + for (i = 0; i < folio_size(folio); i++) + data[i] = base + i < size ? 0x31 + ((base + i) / SZ_4K) : 0; + __folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; +} + +static struct file *test_coarse_cache_file(struct kunit *test, unsigned int order, + unsigned long size) +{ + static const struct address_space_operations aops = { + .read_folio = test_coarse_read_folio, + .dirty_folio = filemap_dirty_folio, + }; + static const struct file_operations fops = { + .mmap = generic_file_mmap, + .read_iter = generic_file_read_iter, + .llseek = generic_file_llseek, + }; + struct file *file = anon_inode_create_getfile("user-coarse-file", &fops, + NULL, O_RDWR, NULL); + struct folio *folio; + int ret; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, test_file_put, file)) + return ERR_PTR(-ENOMEM); + file->f_mapping->a_ops = &aops; + mapping_set_large_folios(file->f_mapping); + i_size_write(file_inode(file), size); + folio = folio_alloc(GFP_KERNEL, order); + if (!folio) + return ERR_PTR(-ENOMEM); + ret = filemap_add_folio(file->f_mapping, folio, 0, GFP_KERNEL); + if (!ret) + test_coarse_read_folio(file, folio); /* Unlocks the installed folio. */ + folio_put(folio); + return ret ? ERR_PTR(ret) : file; +} + +static bool test_coarse_file_enabled(struct kunit *test) +{ + if (PAGE_SHIFT >= 16 || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires internal larger user granule and borrowed-mm opt-in"); + return false; + } + return true; +} + +static void test_promote_unpin(void *data) +{ + struct user_page_fragment *fragment = data; + + if (fragment->folio) { + release_user_fragments(fragment, 1, true); + fragment->folio = NULL; + } +} + +static void user64k_file_promote_test(struct kunit *test) +{ + unsigned int pinned; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "cache promotion currently uses the collapse implementation"); + return; + } + for (pinned = 0; pinned < 2; pinned++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, true); + struct vm_area_struct *coarse, *small; + struct user_page_fragment *fragment; + struct folio *folio; + phys_addr_t old[SZ_64K / PAGE_SIZE], untouched; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset, modified = SZ_64K / 2 + 19; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, SZ_64K); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + untouched = page_to_phys(&folio->page); + folio_put(folio); + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + old[i] = page_to_phys(&folio->page); + folio_put(folio); + } + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + if (pinned) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + modified, 1, FOLL_WRITE, fragment, 1), 1L); + *(u8 *)(folio_address(fragment->folio) + fragment->offset) = 0xa7; + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), -EBUSY); + /* A pin in the middle forces rollback of earlier isolated folios. */ + for (i = 0; i < nr; i++) { + folio = filemap_get_folio(file->f_mapping, nr + i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), old[i]); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0xa7); + kunit_release_action(test, test_promote_unpin, fragment); + } + KUNIT_ASSERT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + folio = filemap_get_folio(file->f_mapping, nr); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio->index, (pgoff_t)nr); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)2 * nr); + for (offset = 0; offset < SZ_64K; offset++) { + u8 expected = pinned && offset == modified ? 0xa7 : 0x41 + offset / SZ_4K; + + if (((u8 *)folio_address(folio))[offset] != expected) { + KUNIT_FAIL(test, "promoted data mismatch at %lu", offset); + break; + } + } + /* An already suitable folio is an idempotent success, including with a GET. */ + KUNIT_EXPECT_EQ(test, collapse_file_user_page(coarse->vm_mm, file, nr), 0); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, coarse->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, entry), page_to_phys(&folio->page)); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) { + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + { + struct folio *first = filemap_get_folio(file->f_mapping, 0); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, first); + KUNIT_EXPECT_EQ(test, page_to_phys(&first->page), untouched); + folio_put(first); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_shmem_fault_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 4; mode++) { + bool seeded = mode & 1, shared = mode < 2; + struct file *file = test_cache_file(test, SZ_64K, seeded); + struct vm_area_struct *vma; + struct folio *cache, *mapped; + unsigned long offset; + pte_t entry; + vm_fault_t fault; + unsigned int attempt; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, shared, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (attempt = 0; attempt < 8; attempt++) { + fault = test_file_fault(vma, TEST_VA, true); + if (!(fault & VM_FAULT_NOPAGE)) + break; + } + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + mapped = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_EQ(test, folio_size(mapped), SZ_64K); + KUNIT_EXPECT_EQ(test, !!folio_test_anon(mapped), !shared); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mapped) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, cache), 0); + KUNIT_EXPECT_EQ(test, folio_size(cache), SZ_64K); + if (shared) + KUNIT_EXPECT_PTR_EQ(test, mapped, cache); + else { + KUNIT_EXPECT_PTR_NE(test, mapped, cache); + memset(folio_address(mapped), 0xc9, SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + seeded ? 0x31 + offset / SZ_4K : 0, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, + shared ? MM_SHMEMPAGES : MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + { + u8 byte = 0x83, observed = 0; + + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &byte, 1, FOLL_WRITE), 1); + KUNIT_EXPECT_EQ(test, access_remote_vm(vma->vm_mm, TEST_VA + SZ_64K - 1, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(cache))[SZ_64K - 1], + shared ? byte : (u8)(seeded ? 0x40 : 0)); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(cache), (int)(SZ_64K / PAGE_SIZE) + 1); + } +} + +static void user64k_shmem_fault_busy_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, mode != 0); + struct vm_area_struct *vma; + struct folio *held; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!mode) { + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_REMOTE | + FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT, NULL); + mmap_assert_locked(vma->vm_mm); + mmap_read_unlock(vma->vm_mm); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_RETRY); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + } else if (mode == 1) { + held = filemap_get_folio(file->f_mapping, SZ_64K / (2 * PAGE_SIZE)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, held); + folio_lock(held); + fault = test_file_fault(vma, TEST_VA, false); + folio_unlock(held); + folio_put(held); + KUNIT_EXPECT_EQ(test, fault, (vm_fault_t)VM_FAULT_NOPAGE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + } else { + struct vm_area_struct *small = test_file_vma(test, file, false, true, + TEST_VA, SZ_64K, 0); + struct user_page_fragment *fragment; + + KUNIT_ASSERT_NOT_NULL(test, small); + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_promote_unpin, + fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + TEST_VA + SZ_64K / 2, 1, FOLL_WRITE, fragment, 1), 1L); + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), (u8)0x39); + kunit_release_action(test, test_promote_unpin, fragment); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} + +static void user64k_shmem_fault_hole_test(struct kunit *test) +{ +#ifdef CONFIG_USERFAULTFD + unsigned int adjacent; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (adjacent = 0; adjacent < 2; adjacent++) { + struct file *file = test_cache_file(test, 2 * SZ_64K, false); + struct vm_area_struct *coarse, *missing, *seed; + struct folio *folio; + vm_fault_t fault; + bool new; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + missing = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, + adjacent ? SZ_64K : PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, missing); + seed = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, 0); + KUNIT_ASSERT_NOT_NULL(test, seed); + mmap_read_lock(seed->vm_mm); + folio = shmem_uffd_prepare_folio(seed, TEST_VA, &new); + mmap_read_unlock(seed->vm_mm); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, new); + memset(folio_address(folio), 0x6f, SZ_4K); + shmem_uffd_mark_present(folio, 0, BIT(0)); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + GENMASK(PAGE_SIZE / SZ_4K - 1, 1)); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + /* Model the registration flag used by the collapse interval check. */ + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_set(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + fault = test_file_fault(coarse, TEST_VA, false); + mmap_write_lock(missing->vm_mm); + vma_start_write(missing); + vm_flags_clear(missing, VM_UFFD_MISSING); + mmap_write_unlock(missing->vm_mm); + /* A real coarse fault may populate holes covered by its entire leaf. */ + KUNIT_EXPECT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 1, 0, SZ_4K)); + folio_unlock(folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x6f, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + SZ_64K - SZ_4K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, missing->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, seed->vm_mm); + } +#else + kunit_skip(test, "requires userfaultfd registration flags"); +#endif +} + +static void user64k_file_leaf_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *coarse, *native, *small, *child_vma; + struct mm_struct *coarse_mm, *child; + struct folio *folio; + unsigned long offset; + unsigned int nr = (2 * SZ_64K) / PAGE_SIZE; + unsigned int small_nr = 2 * SZ_64K / SZ_4K - 1; + unsigned int maps = 2 * nr + small_nr; + u8 byte = 0xc7, observed; + int ret; + + if (!test_coarse_file_enabled(test)) + return; + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, 2 * SZ_64K, 0); + native = test_file_vma(test, file, true, true, TEST_VA, 2 * SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 2 * SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + coarse_mm = coarse->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + /* One cache folio supplies two user leaves, with native references per leaf. */ + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_64K) { + pte_t entry = test_fault_entry(test, coarse_mm, TEST_VA + offset); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse_mm, entry), + page_to_phys(&folio->page) + offset); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 2L); + for (offset = 0; offset < 2 * SZ_64K; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + for (offset = SZ_4K; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_ASSERT_EQ(test, access_remote_vm(coarse_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + SZ_64K + 3 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + byte = 0x98; + KUNIT_ASSERT_EQ(test, access_remote_vm(small->vm_mm, TEST_VA + 7 * SZ_4K, + &byte, 1, FOLL_WRITE), 1); + KUNIT_ASSERT_EQ(test, access_remote_vm(native->vm_mm, TEST_VA + 7 * SZ_4K, + &observed, 1, 0), 1); + KUNIT_EXPECT_EQ(test, observed, byte); + KUNIT_EXPECT_EQ(test, ((u8 *)folio_address(folio))[7 * SZ_4K], byte); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, coarse_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + /* Ordinary file VMAs are lazily populated in the child after fork. */ + mmap_read_lock(child); + child_vma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + maps += nr; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + kthread_use_mm(coarse_mm); + ret = vm_munmap(TEST_VA, SZ_64K); + kthread_unuse_mm(coarse_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + maps -= SZ_64K / PAGE_SIZE; + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 1L); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, child, TEST_VA))); + + /* The native-rounded range must remove every overlapping coarse alias. */ + unmap_mapping_range(file->f_mapping, SZ_64K + SZ_4K, SZ_4K, 0); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, TEST_VA + SZ_64K + SZ_4K))); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + PAGE_SIZE))); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), + (int)(maps - 2 * SZ_64K / PAGE_SIZE - 1 - PAGE_SIZE / SZ_4K)); + + /* An exact subpage hole keeps neighboring small leaves in that native page. */ + mmap_read_lock(coarse_mm); + coarse = find_vma(coarse_mm, TEST_VA + SZ_64K); + mmap_read_unlock(coarse_mm); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(child_vma, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + SZ_64K, false) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_64K + offset, + false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + unmap_mapping_subpage_hole(file->f_mapping, SZ_64K + SZ_4K, SZ_64K + 2 * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, coarse_mm, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + SZ_64K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA + SZ_64K))); + for (offset = 0; offset < PAGE_SIZE; offset += SZ_4K) + KUNIT_EXPECT_EQ(test, !!pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + SZ_64K + offset)), offset != SZ_4K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)(maps - 2 * SZ_64K / PAGE_SIZE - 2)); + kunit_release_action(test, user4k_vma_mm_free, coarse_mm); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + if (!test_coarse_file_enabled(test)) + return; + for (cold = 0; cold < 2; cold++) { + struct file *file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *vma; + struct folio *cache, *anon; + unsigned long offset; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + anon = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, entry)))); + KUNIT_EXPECT_TRUE(test, folio_test_anon(anon)); + KUNIT_EXPECT_EQ(test, folio_size(anon), SZ_64K); + folio_get(anon); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, anon), 0); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(anon) + offset, + 0x41 + offset / SZ_4K, SZ_4K), NULL); + memset(folio_address(anon), 0x93, SZ_64K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (offset = 0; offset < 2 * SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), (int)(SZ_64K / PAGE_SIZE)); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), (int)(SZ_64K / PAGE_SIZE) + 1); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(anon), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(anon), 1); + } +} + +static void user64k_file_bounds_test(struct kunit *test) +{ + struct file *file; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int mode; + unsigned long size = SZ_64K + PAGE_SIZE + 19; + pte_t entry; + + if (!test_coarse_file_enabled(test)) + return; + /* Too-small backing and a physically unaligned source cannot form a leaf. */ + for (mode = 0; mode < 2; mode++) { + file = test_coarse_cache_file(test, mode ? 17 - PAGE_SHIFT : 0, 2 * SZ_64K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, false, TEST_VA, SZ_64K, + mode ? PAGE_SIZE : 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } + file = test_coarse_cache_file(test, 17 - PAGE_SHIFT, size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, 3 * SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K, false) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_64K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)) + + PAGE_SIZE + 19, 0, SZ_64K - PAGE_SIZE - 19), NULL); + KUNIT_EXPECT_TRUE(test, test_file_fault(vma, TEST_VA + 2 * SZ_64K, false) & VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), 2L); +} + +static void user4k_file_shared_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 4 * PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + unsigned int i; + pte_t entry; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + native = test_file_vma(test, file, true, true, TEST_VA, 4 * PAGE_SIZE, 0); + /* Virtual and file offsets deliberately have different native alignment. */ + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 3 * PAGE_SIZE, 2 * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, false) & + VM_FAULT_ERROR, 0U); + /* Fault-around keeps its 64K window and populates all ready cache folios. */ + for (i = 0; i < 12; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + struct vm_page_offset pos = vma_page_offset_at(small, addr); + + entry = test_fault_entry(test, small->vm_mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + folio = filemap_get_folio(file->f_mapping, pos.index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(folio_file_page(folio, pos.index)) + pos.offset); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + (u8)(0x33 + i)); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + memset(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + 2 * SZ_4K, 0x72, SZ_4K); + entry = test_fault_entry(test, small->vm_mm, small->vm_start); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), + 0x72, SZ_4K), NULL); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, small->vm_start + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + memset(phys_to_virt(pte_phys_mm(small->vm_mm, entry)), 0x84, SZ_4K); + entry = test_fault_entry(test, native->vm_mm, TEST_VA); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(native->vm_mm, entry)) + + 3 * SZ_4K, 0x84, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + folio_put(folio); +} + +static void user4k_file_cow_test(struct kunit *test) +{ + unsigned int cold; + + for (cold = 0; cold < 2; cold++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct folio *cache; + pte_t entry; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, + 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + if (!cold) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & + VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, vma->vm_mm, vma->vm_start + SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), + 0x33, SZ_4K), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); + cache = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(cache) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(cache), 0); + folio_put(cache); + } +} + +static void user4k_file_eof_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, SZ_4K + 37, false); + struct vm_area_struct *small, *native; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 4; i++) { + vm_fault_t ret = test_file_fault(small, TEST_VA + i * SZ_4K, false); + + KUNIT_EXPECT_EQ(test, ret & VM_FAULT_ERROR, i < 2 ? 0U : VM_FAULT_SIGBUS); + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, small->vm_mm, + TEST_VA + i * SZ_4K)), i < 2); + } +} + + +static const struct vm_operations_struct test_filemap_fault_ops = { + .fault = filemap_fault, + .page_mkwrite = filemap_page_mkwrite, +}; + +static void user4k_filemap_fault_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * SZ_4K + 17, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + 3 * SZ_4K, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Exercise generic filemap_fault against an already populated cache. */ + vma->vm_ops = &test_filemap_fault_ops; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start + SZ_4K, true) & VM_FAULT_ERROR, 0U); + for (i = 0; i < 3; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, vma->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), (u8)(0x31 + i)); + } + KUNIT_EXPECT_EQ(test, test_file_fault(vma, vma->vm_start + 3 * SZ_4K, false) & + VM_FAULT_ERROR, VM_FAULT_SIGBUS); +} + +static void user4k_file_truncate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *small, *cow, *native; + struct iattr attr = { .ia_valid = ATTR_SIZE, .ia_size = SZ_4K + 37 }; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + PAGE_SIZE, SZ_4K); + cow = test_file_vma(test, file, false, false, TEST_VA, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, false, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, cow); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + inode_lock(file_inode(file)); + ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, &attr, NULL); + inode_unlock(file_inode(file)); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + unsigned long addr = small->vm_start + i * SZ_4K; + + if (i) + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, addr))); + KUNIT_EXPECT_EQ(test, test_file_fault(small, addr, false) & VM_FAULT_ERROR, + i ? VM_FAULT_SIGBUS : 0U); + } + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, cow->vm_mm, TEST_VA + 3 * SZ_4K))); + KUNIT_EXPECT_EQ(test, test_file_fault(cow, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, + VM_FAULT_SIGBUS); + KUNIT_EXPECT_TRUE(test, pte_present(test_fault_entry(test, native->vm_mm, TEST_VA))); + KUNIT_EXPECT_EQ(test, test_file_fault(native, TEST_VA + 3 * SZ_4K, false) & VM_FAULT_ERROR, 0U); +} + +static void user4k_file_invalidate_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *vma; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + 4 * SZ_4K, PAGE_SIZE - SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 0, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i != 0); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, vma->vm_start, false) & VM_FAULT_ERROR, 0U); + unmap_mapping_pages(file->f_mapping, 1, 1, false); + for (i = 0; i < 4; i++) + KUNIT_EXPECT_EQ(test, pte_present(test_fault_entry(test, vma->vm_mm, + vma->vm_start + i * SZ_4K)), i == 0); +} + +static void user4k_file_fork_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 2 * PAGE_SIZE, true); + struct vm_area_struct *shared, *private, *cvma; + struct mm_struct *child; + struct inode *inode; + unsigned int i; + pte_t copied; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA, 2 * PAGE_SIZE, 0); + private = test_file_vma(test, file, false, false, TEST_VA, 2 * PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA, false) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA, false) & VM_FAULT_ERROR, 0U); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, private->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + mmap_read_lock(child); + cvma = find_vma(child, TEST_VA); + mmap_read_unlock(child); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, TEST_VA + SZ_4K); + memset(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K); + inode = file_inode(file); + /* A sub-native-page hole zeros shared bytes but retains private COW. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, SZ_4K, SZ_4K); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, i_size_read(inode), (loff_t)(2 * PAGE_SIZE)); + for (i = 0; i < 2 * TEST_SLOTS; i++) { + pte_t pte; + + KUNIT_ASSERT_EQ(test, test_file_fault(shared, TEST_VA + i * SZ_4K, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(shared->vm_mm, pte)), + i == 1 ? 0 : 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA + SZ_4K)), + pte_phys_mm(child, copied)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(child, copied)), 0x82, SZ_4K), NULL); + /* A full cache-page hole exercises the file interval-tree invalidation. */ + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, PAGE_SIZE, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = TEST_SLOTS; i < 2 * TEST_SLOTS; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, shared->vm_mm, TEST_VA + i * SZ_4K))); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, child, TEST_VA + i * SZ_4K))); + } +} + +static void test_zero_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct mm_subpage *slot; + pte_t before[4], after; + unsigned int i; + vm_fault_t ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + for (i = 0; i < 4; i++) { + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + i * size, + FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + before[i] = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_ASSERT_TRUE(test, pte_present(before[i])); + KUNIT_ASSERT_TRUE(test, is_zero_pfn(pte_pfn(before[i]))); + KUNIT_ASSERT_FALSE(test, pte_write(before[i])); + } + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size + 7, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + lru_add_drain_all(); + for (i = 0; i < 4; i++) { + after = test_fault_entry(test, mm, TEST_VA + i * size); + if (i != 1) { + KUNIT_EXPECT_TRUE(test, pte_same(before[i], after)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_write(after)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + size, 0, size), NULL); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); +} + +static void user4k_zero_cow_test(struct kunit *test) +{ + test_zero_cow_granule(test, 12); +} + +static void user16k_zero_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_zero_cow_granule(test, 14); +} + +static void test_cow_reuse_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + unsigned int mode; + + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma = test_fault_vma_granule(test, shift, TEST_VA, PAGE_SIZE); + struct mm_subpage *held = NULL; + struct mm_struct *child; + phys_addr_t original[4]; + unsigned int i; + vm_fault_t fault; + int ret; + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * size); + original[i] = pte_phys_mm(vma->vm_mm, pte); + memset(phys_to_virt(original[i]), 0xb0 + i, size); + } + if (mode) { + held = mm_subpage_get_from_phys(original[2]); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + } + child = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < 4; i++) { + unsigned long address = TEST_VA + i * size; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte) == original[i], + !(mode && i == 2)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), + 0xb0 + i, size), NULL); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xd0 + i, size); + } + if (held) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(mm_subpage_phys(held)), + 0xb2, size), NULL); + } +} + +static void user4k_cow_reuse_test(struct kunit *test) +{ + test_cow_reuse_granule(test, 12); +} + +static void user16k_cow_reuse_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_cow_reuse_granule(test, 14); +} + +static void test_fork_cow_granule(struct kunit *test, unsigned int shift) +{ + unsigned long size = 1UL << shift; + struct vm_area_struct *parent = test_fault_vma_granule(test, shift, TEST_VA, 4 * size); + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct vm_area_struct *cvma; + struct mm_subpage *held[4]; + pte_t before[4], after; + unsigned int i, side; + vm_fault_t fault; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + before[i] = test_fault_entry(test, parent->vm_mm, TEST_VA + i * size); + held[i] = mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, before[i])); + KUNIT_ASSERT_NOT_NULL(test, held[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held[i]), 0); + memset(folio_address(mm_subpage_folio(held[i])) + mm_subpage_offset(held[i]), + 0x31 + i, size); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + size + 15, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + for (i = 0; i < 4; i++) { + struct mm_subpage *slot; + bool changed = side && i == 1; + + after = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_EQ(test, !!pte_write(after), changed); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x31 + i, size), NULL); + if (changed) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); + memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + } + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 4UL); + } + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held[i])) + + mm_subpage_offset(held[i]), 0x31 + i, size), NULL); + } + KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(held[0])), 11); +} + +static void user4k_fork_cow_test(struct kunit *test) +{ + test_fork_cow_granule(test, 12); +} + +static void user16k_fork_cow_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative faults require larger native backing"); + return; + } + test_fork_cow_granule(test, 14); +} + +static void user4k_folio_walk_geometry_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which], leaf; + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long start, size = 1UL << shift; + vm_fault_t fault; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + start = pmd_size_mm(mm) - size; + vma = test_fault_vma_in_mm(mm, start, 2 * size); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (leaf = 0; leaf < 2; leaf++) { + unsigned long address = start + leaf * size; + struct folio_walk fw; + struct folio *folio; + struct page *page = NULL; + pte_t expected, walked = __pte(0); + bool zero = false; + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, 0); + if (folio) + folio_walk_end(&fw, vma); + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, folio, NULL); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address, FW_ZEROPAGE); + if (folio) { + zero = is_zero_folio(folio) && !fw.page && fw.level == FW_LEVEL_PTE; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_TRUE(test, zero); + + mmap_read_lock(mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + expected = test_fault_entry(test, mm, address); + mmap_read_lock(mm); + folio = folio_walk_start(&fw, vma, address + size - 1, 0); + if (folio) { + page = fw.page; + walked = fw.pte; + folio_walk_end(&fw, vma); + } + mmap_read_unlock(mm); + KUNIT_EXPECT_PTR_EQ(test, page, pfn_to_page(pte_pfn(expected))); + KUNIT_EXPECT_TRUE(test, pte_same(walked, expected)); + KUNIT_EXPECT_PTR_EQ(test, folio, page_folio(pfn_to_page(pte_pfn(expected)))); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_fragment_locked_pin_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, PAGE_SHIFT, 14 }; + unsigned int which; + + for (which = 0; which < ARRAY_SIZE(shifts); which++) { + unsigned int shift = shifts[which]; + struct mm_struct *mm; + struct vm_area_struct *vma, *found; + struct user_page_fragment fragment = { }; + unsigned long address; + int ret; + + if (which == 2 && PAGE_SHIFT <= 14) + continue; + mm = shift == PAGE_SHIFT ? mm_alloc() : test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shift == PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + address = TEST_VA + (1UL << shift) - sizeof(long); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_LONGTERM, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOWAIT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, found, NULL); + /* Only the first fragment is returned, while retaining the VMA lock. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, 2 * sizeof(long), + FOLL_WRITE, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret != 1) + continue; + KUNIT_EXPECT_PTR_EQ(test, found, vma); + KUNIT_EXPECT_TRUE(test, fragment.pinned); + KUNIT_EXPECT_EQ(test, fragment.length, (unsigned int)sizeof(long)); + KUNIT_EXPECT_EQ(test, !!fragment.subpage, shift != PAGE_SHIFT); + release_user_fragments(&fragment, 1, false); + /* With backing present, NOFAULT pins the exact same leaf successfully. */ + mmap_read_lock(mm); + ret = pin_user_fragment_vma_remote(mm, address, sizeof(long), + FOLL_WRITE | FOLL_NOFAULT, &fragment, &found); + mmap_assert_locked(mm); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, ret, 1); + if (ret == 1) { + KUNIT_EXPECT_PTR_EQ(test, found, vma); + release_user_fragments(&fragment, 1, true); + } + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user16k_fragment_offset_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct user_page_fragment fragment = {}; + unsigned long address = TEST_VA + 3 * SZ_16K + SZ_4K + 13; + long ret; + + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative fragments require larger native backing"); + return; + } + vma = test_fault_vma_granule(test, 14, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + ret = pin_user_fragments_remote(vma->vm_mm, address, 31, FOLL_WRITE, &fragment, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + if (ret != 1) + return; + KUNIT_EXPECT_NOT_NULL(test, fragment.subpage); + KUNIT_EXPECT_EQ(test, fragment.offset, 3UL * SZ_16K + SZ_4K + 13); + KUNIT_EXPECT_EQ(test, fragment.length, 31U); + if (fragment.subpage) + KUNIT_EXPECT_EQ(test, mm_subpage_shift(fragment.subpage), 14U); + memset(folio_address(fragment.folio) + fragment.offset, 0x9c, fragment.length); + release_user_fragments(&fragment, 1, true); + { + u8 buf[33]; + + ret = access_remote_vm(vma->vm_mm, address - 1, buf, sizeof(buf), 0); + KUNIT_ASSERT_EQ(test, ret, (long)sizeof(buf)); + KUNIT_EXPECT_EQ(test, buf[0], (u8)0); + KUNIT_EXPECT_EQ(test, buf[32], (u8)0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buf + 1, 0x9c, 31), NULL); + } +} + +static void user4k_anon_exit_test(struct kunit *test) +{ + struct anon_slot_fixture *f = anon_slot_fixture_create(test); + struct mm_subpage *held; + struct anon_vma *root; + struct folio *folio; + struct mm_struct *mm; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + held = f->slots[0]; + folio = mm_subpage_folio(held); + root = f->vma->anon_vma->root; + mm_subpage_get(held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + mm_subpage_pool_close(f->pool); + mm_subpage_pool_put(f->pool); + /* Leave all four mapping refs and the tables to the real exit path. */ + mm = f->mm; + memset(f, 0, sizeof(*f)); + mmput(mm); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, atomic_read(&root->refcount), 1); + for (i = 1; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_get_from_phys(mm_subpage_phys(held) + i * SZ_4K); + + KUNIT_EXPECT_PTR_EQ(test, slot, NULL); + if (slot) + mm_subpage_put(slot); + } +} + +#ifdef CONFIG_ARM64_CONTPTE +struct contpte_fixture { + struct mm_struct *mm; + struct folio *folio; + unsigned int nr; +}; + +static void contpte_fixture_free(void *arg) +{ + struct contpte_fixture *f = arg; + struct mmu_gather tlb; + unsigned int i; + + if (f->mm) { + mmap_write_lock(f->mm); + for (i = 0; i <= f->nr; i++) { + unsigned long addr = TEST_VA + i * mm_page_size(f->mm); + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(f->mm, addr, &ptl); + + if (pte) { + pte_clear(f->mm, addr, pte); + pte_unmap_unlock(pte, ptl); + } + } + tlb_gather_mmu_fullmm(&tlb, f->mm); + free_pgd_range(&tlb, 0, 1UL << 48, 0, 0); + tlb_finish_mmu(&tlb); + mmap_write_unlock(f->mm); + mmput(f->mm); + } + if (f->folio) + folio_put(f->folio); +} + +static void test_contpte_granule(struct kunit *test, bool small) +{ + struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + struct vm_area_struct vma = {}; + struct test_mm tables = {}; + struct { + bool initial, addresses, hints, gathered, aged, access_all, unfolded; + bool permissions, refolded, holes, full_clear, sentinel; + } ok; + unsigned long size; + phys_addr_t phys; + pte_t base, saved, observed; + pte_t *pte; + spinlock_t *ptl; + unsigned int i; + int access_ret, access_same_ret, access_noop_ret; + + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); + f->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, f->mm); + if (small) + KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + f->nr = cont_ptes_mm(f->mm); + size = mm_page_size(f->mm); + f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, + get_order(cont_pte_size_mm(f->mm))); + KUNIT_ASSERT_NOT_NULL(test, f->folio); + phys = PFN_PHYS(folio_pfn(f->folio)); + tables.mm = f->mm; + KUNIT_ASSERT_EQ(test, test_map_alias(&tables, 0, TEST_VA, phys), 0); + vma.vm_mm = f->mm; + base = phys_pte_mm(f->mm, phys, __pgprot(TEST_PROT | PTE_RDONLY)); + memset(&ok, 1, sizeof(ok)); + mmap_read_lock(f->mm); + pte = test_mm_lookup(f->mm, TEST_VA, &ptl); + if (!pte) { + mmap_read_unlock(f->mm); + KUNIT_FAIL(test, "missing PTE table"); + return; + } + pte_clear(f->mm, TEST_VA, pte); + /* The adjacent entry must never be included in this contiguous group. */ + saved = pte_mkspecial(base); + set_pte(pte + f->nr, saved); + set_ptes(f->mm, TEST_VA, pte, base, f->nr); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.initial &= pte_cont(observed); + ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; + ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; + } + __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); + observed = ptep_get(pte + 1); + ok.gathered &= pte_dirty(observed) && pte_young(observed); + observed = ptep_get_lockless(pte + 2); + ok.gathered &= pte_dirty(observed) && pte_young(observed) && + pte_phys_mm(f->mm, observed) == phys + 2 * size; + clear_young_dirty_ptes(&vma, TEST_VA + 3 * size, pte + 3, 1, + CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + } + observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); + access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.access_all &= pte_cont(observed) && pte_young(observed) && + pte_dirty(observed) && !pte_write(observed); + } + access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, + pte + 3, ptep_get(pte + 3), 1); + observed = pte_mkwrite_novma(pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3)))); + access_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, + observed, 1); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.unfolded &= !pte_cont(observed); + ok.permissions &= pte_write(observed) == (i == 3); + } + set_pte_at(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3)); + /* Last-entry installation must fold using the physical byte offset too. */ + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + get_and_clear_full_ptes(f->mm, TEST_VA + 3 * size, pte + 3, 2, 0); + for (i = 0; i < f->nr; i++) { + observed = __ptep_get(pte + i); + ok.holes &= pte_none(observed) == (i == 3 || i == 4); + if (i != 3 && i != 4) + ok.holes &= !pte_cont(observed) && + pte_phys_mm(f->mm, observed) == phys + i * size; + } + set_ptes(f->mm, TEST_VA + 3 * size, pte + 3, + pte_advance_pfn_mm(f->mm, base, 3), 2); + set_pte_at(f->mm, TEST_VA + (f->nr - 1) * size, pte + f->nr - 1, + pte_advance_pfn_mm(f->mm, base, f->nr - 1)); + for (i = 0; i < f->nr; i++) + ok.refolded &= pte_cont(__ptep_get(pte + i)); + clear_full_ptes(f->mm, TEST_VA, pte, f->nr, 0); + for (i = 0; i < f->nr; i++) + ok.full_clear &= pte_none(__ptep_get(pte + i)); + ok.sentinel = pte_same(__ptep_get(pte + f->nr), saved); + pte_unmap_unlock(pte, ptl); + mmap_read_unlock(f->mm); + KUNIT_EXPECT_TRUE(test, ok.initial); + KUNIT_EXPECT_TRUE(test, ok.addresses); + KUNIT_EXPECT_TRUE(test, ok.hints); + KUNIT_EXPECT_TRUE(test, ok.gathered); + KUNIT_EXPECT_TRUE(test, ok.aged); + KUNIT_EXPECT_TRUE(test, ok.access_all); + KUNIT_EXPECT_EQ(test, access_same_ret, 1); + KUNIT_EXPECT_EQ(test, access_noop_ret, 0); + KUNIT_EXPECT_EQ(test, access_ret, 1); + KUNIT_EXPECT_TRUE(test, ok.unfolded); + KUNIT_EXPECT_TRUE(test, ok.permissions); + KUNIT_EXPECT_TRUE(test, ok.refolded); + KUNIT_EXPECT_TRUE(test, ok.holes); + KUNIT_EXPECT_TRUE(test, ok.full_clear); + KUNIT_EXPECT_TRUE(test, ok.sentinel); +} + +static void user4k_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, true); +} + +static void native_contpte_test(struct kunit *test) +{ + test_contpte_granule(test, false); +} +#endif + +struct usercopy_worker { + struct test_mm *ctx[2]; + struct completion done; + unsigned int id; + unsigned int checks; + unsigned int idmaps; + int error; + u64 tcr; +}; + +static int test_usercopy_worker(void *arg) +{ + struct usercopy_worker *worker = arg; + u64 kernel_ttbr = read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK; + unsigned int round, i, byte; + bool borrowed = false; + struct test_mm *ctx = NULL; + + for (round = 0; round < 16; round++) { + unsigned int small = (round + worker->id) & 1; + unsigned int cpu = cpumask_first(cpu_online_mask); + + ctx = worker->ctx[small]; + kthread_use_mm(ctx->mm); + borrowed = true; + if ((round + worker->id) & 2) { + unsigned int next = cpumask_next(cpu, cpu_online_mask); + + if (next < nr_cpu_ids) + cpu = next; + } + if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) { + worker->error = 1; + break; + } + /* Exercise a real scheduler switch while borrowing this mm. */ + schedule_timeout_uninterruptible(1); + if (small && round < 4) { + unsigned long flags; + bool native; + + preempt_disable(); + flags = local_daif_save(); + if (round & 2) + cpu_install_ttbr0(phys_to_ttbr(__pa_symbol(idmap_pg_dir)), + TCR_T0SZ(IDMAP_VA_BITS)); + else + cpu_install_idmap(); + native = (read_sysreg(tcr_el1) & (TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK)) == + (TCR_TG0_16K | TCR_T0SZ(IDMAP_VA_BITS)); + cpu_uninstall_idmap(); + local_daif_restore(flags); + preempt_enable(); + if (!native) { + worker->error = 6; + break; + } + worker->idmaps++; + } + worker->tcr = read_sysreg(tcr_el1); + if ((worker->tcr & TCR_EL1_TG0_MASK) != (small ? TCR_TG0_4K : TCR_TG0_16K) || + (worker->tcr & TCR_EL1_T0SZ_MASK) != TCR_T0SZ(48) || + (worker->tcr & TCR_EL1_TG1_MASK) != TCR_TG1_16K || + (read_sysreg(ttbr1_el1) & ~TTBRx_EL1_ASID_MASK) != kernel_ttbr) { + worker->error = 2; + break; + } + for (i = 0; i < 4; i++) { + unsigned long offset = 64 + worker->id * sizeof(u64); + void __user *address = (void __user *)ctx->address[i]; + u64 value = 0xa5000000 | (worker->id << 16) | (round << 8) | i; + u64 *backing = phys_to_virt(ctx->phys[i]) + offset; + unsigned long read_left, write_left; + u8 data[8]; + + /* No VMA/fault path may be entered for these private mappings. */ + pagefault_disable(); + read_left = copy_from_user(data, address, sizeof(data)); + write_left = copy_to_user(address + offset, &value, sizeof(value)); + pagefault_enable(); + if (read_left != (i == 2 ? sizeof(data) : 0) || + write_left != (i == 0 || i == 2 ? sizeof(value) : 0)) { + worker->error = 3; + goto out; + } + if (i != 2) { + for (byte = 0; byte < sizeof(data); byte++) { + if (data[byte] != (small ? 0x40 : 0x80) + i) { + worker->error = 4; + goto out; + } + } + } + if (READ_ONCE(*backing) != (i == 0 || i == 2 ? 0 : value)) { + worker->error = 5; + goto out; + } + worker->checks++; + } + kthread_unuse_mm(ctx->mm); + borrowed = false; + } +out: + if (borrowed) + kthread_unuse_mm(ctx->mm); + complete(&worker->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_kthread(void *arg) +{ + kthread_stop(arg); +} + +struct cow_reader { + struct mm_struct *parent; + struct mm_struct *child; + struct completion ready; + struct completion copied; + struct completion done; + u8 observed[8]; + unsigned long read_left[8], write_left[8]; + int error; + unsigned int checks; +}; + +static int test_cow_reader(void *arg) +{ + struct cow_reader *r = arg; + unsigned int i, side; + + kthread_use_mm(r->child); + /* Resolve fork's old PTEs normally, then cache pre-COW translations. */ + for (i = 0; i < 4; i++) { + u8 value = 0; + unsigned long left; + + left = copy_from_user(&value, (void __user *)(TEST_VA + i * SZ_4K + 64), 1); + if (left || value != 0x41 + i) + r->error = 1; + } + complete(&r->ready); + if (!wait_for_completion_timeout(&r->copied, 10 * HZ)) { + r->error = 2; + goto out; + } + for (side = 0; side < 2; side++) { + if (side) { + kthread_unuse_mm(r->child); + kthread_use_mm(r->parent); + } + for (i = 0; i < 4; i++) { + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64); + u8 value = 0, changed = 0x99; + unsigned long read_left, write_left; + + pagefault_disable(); + read_left = copy_from_user(&value, addr, 1); + write_left = copy_to_user(addr, &changed, 1); + pagefault_enable(); + r->observed[side * 4 + i] = value; + r->read_left[side * 4 + i] = read_left; + r->write_left[side * 4 + i] = write_left; + if (read_left || value != 0x41 + i || + write_left != (side == 0 && i == 1 ? 0 : 1)) + r->error = 3; + r->checks++; + } + } + kthread_unuse_mm(r->parent); + goto done; +out: + kthread_unuse_mm(r->child); +done: + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void user4k_fault_uaccess_test(struct kunit *test) +{ + struct vm_area_struct *parent, *child_vma; + struct mm_struct *child; + struct cow_reader *reader; + struct task_struct *task; + unsigned int i, cpu; + vm_fault_t fault; + int ret; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || !system_supports_4kb_granule() || + system_supports_cnp() || cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires opt-in and compatible 4K TTBR0 controls"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, parent); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + + memset(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)), 0x41 + i, SZ_4K); + } + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + child_vma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, child_vma); + reader = kunit_kzalloc(test, sizeof(*reader), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reader); + reader->parent = parent->vm_mm; + reader->child = child; + init_completion(&reader->ready); + init_completion(&reader->copied); + init_completion(&reader->done); + task = kthread_create(test_cow_reader, reader, "user4k-cow-reader"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + cpu = cpumask_last(cpu_online_mask); + kthread_bind(task, cpu); + wake_up_process(task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->ready, 10 * HZ), 0UL); + mmap_read_lock(child); + fault = handle_mm_fault(child_vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + complete(&reader->copied); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&reader->done, 10 * HZ), 0UL); + if (reader->error) + for (i = 0; i < 8; i++) + kunit_info(test, "access %u value=%#x read_left=%lu write_left=%lu\n", + i, reader->observed[i], reader->read_left[i], reader->write_left[i]); + KUNIT_EXPECT_EQ(test, reader->error, 0); + KUNIT_EXPECT_EQ(test, reader->checks, 8U); + { + pte_t entry = test_fault_entry(test, child, TEST_VA + SZ_4K); + + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(child, entry)) + 64), (u8)0x99); + entry = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)(phys_to_virt(pte_phys_mm(parent->vm_mm, entry)) + 64), (u8)0x42); + } +} + +static void user4k_context_uaccess_test(struct kunit *test) +{ + struct test_mm *ctx[2]; + struct usercopy_worker *workers; + unsigned int i; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + ctx[0] = test_usercopy_mm(test, false); + KUNIT_ASSERT_NOT_NULL(test, ctx[0]); + ctx[1] = test_usercopy_mm(test, true); + KUNIT_ASSERT_NOT_NULL(test, ctx[1]); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + struct task_struct *task; + + workers[i].ctx[0] = ctx[0]; + workers[i].ctx[1] = ctx[1]; + workers[i].id = i; + init_completion(&workers[i].done); + task = kthread_run(test_usercopy_worker, &workers[i], "user4k-copy-%u", i); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_kthread, task), 0); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "worker %u TCR=%#llx", i, workers[i].tcr); + KUNIT_EXPECT_EQ(test, workers[i].checks, 64U); + KUNIT_EXPECT_EQ(test, workers[i].idmaps, 2U); + } + kunit_info(test, "two migrating kthreads, 32 borrowed-mm switches, %s PAN\n", + system_uses_ttbr0_pan() ? "software" : "hardware"); +} + +struct zap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + struct completion proceed; + struct completion done; + unsigned int cpu; + unsigned int checks; + bool zapped; + int error; +}; + +static int test_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + unsigned int phase, i; + + reader->error = set_cpus_allowed_ptr(current, cpumask_of(reader->cpu)); + if (reader->error) { + complete(&reader->ready); + goto out; + } + kthread_use_mm(reader->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) { + complete(&reader->ready); + wait_for_completion(&reader->proceed); + if (!reader->zapped) + break; + } + for (i = 0; i < 4; i++) { + u64 value = 0, expected = 0xa5a50000UL + i; + unsigned long left; + + pagefault_disable(); + left = copy_from_user(&value, + (void __user *)((unsigned long)TEST_VA + i * SZ_4K), sizeof(value)); + pagefault_enable(); + if (phase && i == 1) { + if (!left) + reader->error = -EUCLEAN; + } else if (left || value != expected) { + reader->error = -EFAULT; + } + reader->checks++; + } + } + kthread_unuse_mm(reader->mm); +out: + complete(&reader->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void test_stop_zap_reader(void *arg) +{ + struct zap_reader *reader = arg; + + complete(&reader->proceed); + kthread_stop(reader->task); +} + +static void user4k_anon_remote_zap_test(struct kunit *test) +{ + struct anon_slot_fixture *f; + struct zap_reader *readers; + unsigned int i, cpu; + u64 tcr = read_sysreg(tcr_el1); + + if (!user4k_test_enabled || num_online_cpus() < 2) { + kunit_skip(test, "requires arm64.user4k_test=1 and two CPUs in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires 4K, no CnP/PIE/POE, DS=0, A1=1 and 48-bit TTBR0"); + return; + } + f = anon_slot_fixture_create(test); + KUNIT_ASSERT_NOT_NULL(test, f); + for (i = 0; i < 4; i++) { + struct mm_subpage *slot = mm_subpage_alloc_at(f->pool, i * SZ_4K); + u64 value = 0xa5a50000UL + i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + memcpy(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), + &value, sizeof(value)); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(f, i, slot), 0); + } + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = f->mm; + readers[i].cpu = cpu; + init_completion(&readers[i].ready); + init_completion(&readers[i].proceed); + init_completion(&readers[i].done); + readers[i].task = kthread_run(test_zap_reader, &readers[i], "user4k-zap-%u", i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, readers[i].task); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_stop_zap_reader, + &readers[i]), 0); + cpu = cpumask_next(cpu, cpu_online_mask); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + KUNIT_ASSERT_EQ(test, readers[i].error, 0); + } + /* Keep an explicit non-mapping reference for fixture cleanup. */ + mm_subpage_get(f->slots[1]); + mmap_read_lock(f->mm); + zap_vma_range(f->vma, TEST_VA + SZ_4K, SZ_4K); + f->mapped[1] = false; + mmap_read_unlock(f->mm); + for (i = 0; i < 2; i++) { + readers[i].zapped = true; + complete(&readers[i].proceed); + } + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 10 * HZ), 0UL); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 8U); + } +} + +static u64 *test_leaf(struct test_tables *tables, unsigned long va) +{ + return &tables->level[3][test_index(va, tables->shift, 3)]; +} + +/* Explicitly 4K, without the native PAGE_SHIFT or a granule/level hint. */ +static void test_invalidate_4k(unsigned long va, u64 ttbr1) +{ + u64 operand = (va >> USER4K_SHIFT) | (ttbr1 & GENMASK_ULL(63, 48)); + + dsb(ishst); + __tlbi(vae1, operand); + dsb(ish); + isb(); +} + +static void user4k_translation_test(struct kunit *test) +{ + struct test_tables small, native; + struct test_result *result; + unsigned long flags; + void *data, *other; + phys_addr_t data_pa, other_pa; + u64 old_tcr, old_ttbr0, old_ttbr1, old_par, tcr4k, tcr16k; + unsigned int round, slot; + + if (!user4k_test_enabled) { + kunit_skip(test, "requires arm64.user4k_test=1 in a disposable VM"); + return; + } + if (!system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || + cpus_have_final_cap(ARM64_HAS_S1POE)) { + kunit_skip(test, "requires 4K, no CnP, and direct PTE permissions"); + return; + } + + KUNIT_ASSERT_EQ(test, PAGE_SHIFT, 14); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &small, USER4K_SHIFT), 0); + KUNIT_ASSERT_EQ(test, test_alloc_tables(test, &native, PAGE_SHIFT), 0); + data = test_alloc_page(test); + other = test_alloc_page(test); + KUNIT_ASSERT_NOT_NULL(test, data); + KUNIT_ASSERT_NOT_NULL(test, other); + data_pa = virt_to_phys(data); + other_pa = virt_to_phys(other); + result = kunit_kcalloc(test, TEST_ROUNDS, sizeof(*result), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, result); + + /* One native mapping and four independently described subpages. */ + *test_leaf(&native, TEST_VA) = other_pa | TEST_PROT; + *test_leaf(&small, TEST_VA) = data_pa | TEST_PROT | PTE_RDONLY; + *test_leaf(&small, TEST_VA + USER4K_SIZE) = + (data_pa + USER4K_SIZE) | TEST_PROT; + /* Slot 2 deliberately unmapped. Slot 3 aliases slot 0, writable. */ + *test_leaf(&small, TEST_VA + 3 * USER4K_SIZE) = data_pa | TEST_PROT; + + preempt_disable(); + flags = local_daif_save(); + old_tcr = read_sysreg(tcr_el1); + old_ttbr0 = read_sysreg(ttbr0_el1); + old_ttbr1 = read_sysreg(ttbr1_el1); + old_par = read_sysreg_par(); + /* DS affects both address ranges: never toggle it under a live TTBR1. */ + if ((old_tcr & (TCR_EL1_DS | TCR_EL1_TG1_MASK | TCR_EL1_A1)) != + (TCR_TG1_16K | TCR_EL1_A1)) { + local_daif_restore(flags); + preempt_enable(); + kunit_skip(test, "requires 16K TTBR1, DS=0, and A1=1"); + return; + } + tcr4k = (old_tcr & ~(TCR_EL1_TG0_MASK | TCR_EL1_T0SZ_MASK | + TCR_EL1_EPD0)) | TCR_TG0_4K | TCR_T0SZ(48); + tcr16k = (tcr4k & ~TCR_EL1_TG0_MASK) | TCR_TG0_16K; + + /* No allocations, logging, KUnit assertions or user access in this window. */ + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + test_switch_ttbr0(virt_to_phys(native.level[0]), tcr16k); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->native[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, false); + test_switch_ttbr0(virt_to_phys(small.level[0]), tcr4k); + r->tcr = read_sysreg(tcr_el1); + r->ttbr1 = read_sysreg(ttbr1_el1); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + unsigned long va = TEST_VA + slot * USER4K_SIZE; + + r->read[slot] = test_translate(va, false); + r->write[slot] = test_translate(va, true); + } + + /* Break-before-make one 4K PTE and invalidate exactly that VA. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), 0); + test_invalidate_4k(TEST_VA + USER4K_SIZE, old_ttbr1); + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (other_pa + USER4K_SIZE) | TEST_PROT | PTE_RDONLY); + dsb(ishst); + isb(); + for (slot = 0; slot < PROBE_SLOTS; slot++) + r->replaced[slot] = test_translate(TEST_VA + slot * USER4K_SIZE, true); + r->replaced_read = test_translate(TEST_VA + USER4K_SIZE, false); + asm volatile("at s1e1r, %0" : : "r" (data) : "memory"); + isb(); + r->kernel = read_sysreg_par(); + + test_switch_ttbr0(old_ttbr0, old_tcr); + r->restored_tcr = read_sysreg(tcr_el1); + r->restored_ttbr0 = read_sysreg(ttbr0_el1); + /* Private tables are inactive and their translations have been flushed. */ + WRITE_ONCE(*test_leaf(&small, TEST_VA + USER4K_SIZE), + (data_pa + USER4K_SIZE) | TEST_PROT); + } + write_sysreg(old_par, par_el1); + isb(); + local_daif_restore(flags); + preempt_enable(); + + for (round = 0; round < TEST_ROUNDS; round++) { + struct test_result *r = &result[round]; + + KUNIT_EXPECT_EQ(test, r->tcr, tcr4k); + KUNIT_EXPECT_EQ(test, r->ttbr1, old_ttbr1); + KUNIT_EXPECT_EQ(test, r->restored_tcr, old_tcr); + KUNIT_EXPECT_EQ(test, r->restored_ttbr0, old_ttbr0); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->kernel & SYS_PAR_EL1_PA, data_pa); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced_read & SYS_PAR_EL1_PA, + other_pa + USER4K_SIZE); + for (slot = 0; slot < PROBE_SLOTS; slot++) { + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->native[slot] & SYS_PAR_EL1_PA, + other_pa + slot * USER4K_SIZE); + if (slot == 2) { + KUNIT_EXPECT_EQ(test, + FIELD_GET(SYS_PAR_EL1_FST, r->read[slot]), 7ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, 1ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, 1ULL); + continue; + } + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_F, 0ULL); + KUNIT_EXPECT_EQ(test, r->read[slot] & SYS_PAR_EL1_PA, + data_pa + (slot == 1 ? USER4K_SIZE : 0)); + KUNIT_EXPECT_EQ(test, r->write[slot] & SYS_PAR_EL1_F, + slot == 0 ? 1ULL : 0ULL); + KUNIT_EXPECT_EQ(test, r->replaced[slot] & SYS_PAR_EL1_F, + slot == 3 ? 0ULL : 1ULL); + } + } + kunit_info(test, "16K kernel, 4K EL0 translation/permissions; %u switch cycles\n", + TEST_ROUNDS); +} + +static struct folio *test_fault_folio(struct mm_struct *mm, unsigned long address) +{ + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(mm, address, &ptl); + struct folio *folio = NULL; + + if (pte) { + pte_t entry = ptep_get(pte); + + if (pte_present(entry) && !pte_special(entry)) { + folio = page_folio(pfn_to_page(pte_pfn(entry))); + folio_get(folio); + } + pte_unmap_unlock(pte, ptl); + } + return folio; +} + +static bool test_reclaim_swapcache(swp_entry_t entry) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio = swap_cache_get_folio(entry); + LIST_HEAD(folios); + + if (!folio) + return true; + folio_wait_writeback(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Only the swap cache and reclaim isolation may retain this folio. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + return false; +} + +static void test_swap_tags_set(struct folio *folio) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return; + if (try_page_mte_tagging(&folio->page)) + mte_clear_page_tags(folio_address(folio)); + for (offset = 0; offset < PAGE_SIZE; offset += MTE_GRANULE_SIZE) { + unsigned long tag = (1UL + offset / SZ_4K) << MTE_TAG_SHIFT; + + asm volatile(".arch_extension memtag\n\tstg %0, [%1]" + : : "r" (tag), "r" (folio_address(folio) + offset) : "memory"); + } + set_page_mte_tagged(&folio->page); +#endif +} + +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ +#ifdef CONFIG_ARM64_MTE + unsigned int offset; + + if (!system_supports_mte()) + return true; + if (!page_mte_tagged(&folio->page)) + return false; + for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + unsigned long tag = 0; + + asm volatile(".arch_extension memtag\n\tldg %0, [%1]" + : "+r" (tag) : "r" (folio_address(folio) + offset) : "memory"); + if (((tag >> MTE_TAG_SHIFT) & 0xf) != ((slot + 1) & 0xf)) + return false; + } +#endif + return true; +} + +struct fragment_test_pins { + struct user_page_fragment fragments[2 * TEST_SLOTS]; + unsigned long count; +}; + +static void fragment_test_unpin(void *arg) +{ + struct fragment_test_pins *pins = arg; + + release_user_fragments(pins->fragments, pins->count, true); + pins->count = 0; +} + +static long fragment_test_pin(struct fragment_test_pins *pins, struct mm_struct *mm, + unsigned long start, unsigned long length, unsigned int flags, + unsigned long capacity) +{ + long ret; + + fragment_test_unpin(pins); + ret = pin_user_fragments_remote(mm, start, length, flags, pins->fragments, capacity); + pins->count = ret > 0 ? ret : 0; + return ret; +} + +static void user4k_fragment_pin_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct fragment_test_pins *pins; + struct vm_area_struct *vma; + struct mm_struct *mm, *child; + unsigned long size, total = 0; + unsigned int i, nr; + long ret; + + if (native) { + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + vma = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + } else { + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + } + size = mm_page_size(mm); + nr = PAGE_SIZE / size; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, mm, TEST_VA, 1, FOLL_WRITE | FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, TEST_VA + 123, PAGE_SIZE - 200, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + unsigned int offset = i ? i * size : 123; + unsigned int length = size - (!i ? 123 : 0) - (i == nr - 1 ? 77 : 0); + + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)offset); + KUNIT_EXPECT_EQ(test, f->length, length); + KUNIT_EXPECT_EQ(test, !!f->subpage, !native); + memset(folio_address(f->folio) + f->offset, 0x71 + i, f->length); + total += f->length; + } + KUNIT_EXPECT_EQ(test, total, PAGE_SIZE - 200); + fragment_test_unpin(pins); + child = native ? mm_alloc() : test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + if (native) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, child), 0); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Ordinary references retain shared COW backing, as native FOLL_GET does. */ + ret = get_user_fragments_remote(child, TEST_VA + 123, PAGE_SIZE - 200, + 0, pins->fragments, ARRAY_SIZE(pins->fragments)); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + pte_t child_pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, f->pinned); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, parent_pte), pte_phys_mm(child, child_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + } + fragment_test_unpin(pins); + /* Read pins must unshare COW mappings without enabling PTE writes. */ + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, FOLL_NOFAULT, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, child, TEST_VA + 123, PAGE_SIZE - 200, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)nr); + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + pte_t pte = test_fault_entry(test, child, TEST_VA + i * size); + + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x71 + i, f->length), NULL); + if (!native) { + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + if (i) + KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + } + } + fragment_test_unpin(pins); + /* Return a valid partial result when the byte range reaches an unmapped VMA. */ + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE + size, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, (long)nr); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 1); + KUNIT_EXPECT_EQ(test, ret, 1L); + KUNIT_EXPECT_EQ(test, pins->fragments[0].length, (unsigned int)size); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, TEST_VA, size, FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, 1L); + ret = fragment_test_pin(pins, mm, ULONG_MAX - 15, ULONG_MAX, 0, ARRAY_SIZE(pins->fragments)); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_BLOCK +struct bio_fragment_test { + struct bio *bio; + struct bio *clone; + struct user_page_fragment held[TEST_SLOTS]; + unsigned int count; +}; + +static void bio_fragment_test_free(void *arg) +{ + struct bio_fragment_test *state = arg; + + if (state->clone) + bio_put(state->clone); + if (state->bio) { + bio_release_pages(state->bio, false); + bio_put(state->bio); + } + release_user_fragments(state->held, state->count, false); +} + +static void user4k_bio_fragments_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i, bytes = round_down(PAGE_SIZE - 200, 512); + long nr; + int ret; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + /* Populate real writable anonymous mappings and retain independent GETs. */ + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, + state->held, ARRAY_SIZE(state->held)); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, (long)TEST_SLOTS); + state->bio = bio_alloc(NULL, 8, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + 123), PAGE_SIZE - 200); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_vecs_to_alloc(&iter, BIO_MAX_VECS); + kthread_unuse_mm(vma->vm_mm); + KUNIT_EXPECT_EQ(test, ret, (int)TEST_SLOTS); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, state->bio->bi_iter.bi_size, bytes); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), PAGE_SIZE - 200 - bytes); + KUNIT_EXPECT_EQ(test, state->bio->bi_nr_user_fragments, (int)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 1); + KUNIT_EXPECT_EQ(test, state->bio->bi_io_vec[0].bv_offset, 123U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + state->clone = bio_alloc_clone(NULL, state->bio, GFP_KERNEL, &fs_bio_set); + KUNIT_ASSERT_NOT_NULL(test, state->clone); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_user_fragments, NULL); + KUNIT_EXPECT_PTR_EQ(test, state->clone->bi_io_vec, state->bio->bi_io_vec); + /* Completion ownership must outlive all originating VMAs. */ + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 1U); + bio_put(state->clone); + state->clone = NULL; + bio_release_pages(state->bio, false); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[i].subpage), 0U); +} + +static void user4k_bio_trim_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, SZ_4K); + struct bio_fragment_test *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + struct iov_iter iter; + unsigned int i; + long nr; + int ret, refs; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm bio fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, bio_fragment_test_free, state), 0); + nr = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_WRITE, state->held, 1); + state->count = nr > 0 ? nr : 0; + KUNIT_ASSERT_EQ(test, nr, 1L); + refs = folio_ref_count(state->held[0].folio); + state->bio = bio_alloc(NULL, 4, REQ_OP_READ, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, state->bio); + for (i = 0; i < 1024; i++) { + iov_iter_ubuf(&iter, ITER_DEST, (void __user *)(TEST_VA + SZ_4K - 256), 512); + kthread_use_mm(vma->vm_mm); + ret = bio_iov_iter_get_pages(state->bio, &iter, 511); + kthread_unuse_mm(vma->vm_mm); + KUNIT_ASSERT_EQ(test, ret, -EFAULT); + KUNIT_EXPECT_EQ(test, iov_iter_count(&iter), 512UL); + KUNIT_EXPECT_EQ(test, state->bio->bi_vcnt, 0); + KUNIT_EXPECT_PTR_EQ(test, state->bio->bi_user_fragments, NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(state->held[0].subpage), 0U); + KUNIT_EXPECT_EQ(test, folio_ref_count(state->held[0].folio), refs); + bio_reset(state->bio, NULL, REQ_OP_READ); + } +} +#endif + +static void user4k_fragment_longterm_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + phys_addr_t before; + bool movable; + unsigned int i; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + before = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + movable = !folio_is_longterm_pinnable(pins->fragments[0].folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x91 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, + FOLL_WRITE | FOLL_LONGTERM, ARRAY_SIZE(pins->fragments)); + KUNIT_ASSERT_EQ(test, ret, (long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_TRUE(test, folio_is_longterm_pinnable(f->folio)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(f->subpage), 1U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x91 + i, f->length), NULL); + } + if (movable) + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(pins->fragments[0].folio)), before); + kunit_info(test, "long-term source movable=%u migrated=%u\n", movable, + PFN_PHYS(folio_pfn(pins->fragments[0].folio)) != before); + fragment_test_unpin(pins); + /* A long-term flag must never silently create an ordinary GET. */ + ret = get_user_fragments_remote(vma->vm_mm, TEST_VA, SZ_4K, FOLL_LONGTERM, + pins->fragments, 1); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void user4k_file_fragments_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *shared, *private; + struct fragment_test_pins *pins; + phys_addr_t shared_phys; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shared = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + private = test_file_vma(test, file, false, false, TEST_VA + SZ_4K, 3 * SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, shared); + KUNIT_ASSERT_NOT_NULL(test, private); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, shared->vm_mm, shared->vm_start + 17, + 3 * SZ_4K - 33, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + shared_phys = PFN_PHYS(folio_pfn(pins->fragments[0].folio)); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, (unsigned long)((i + 1) * SZ_4K + (!i ? 17 : 0))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + } + /* File pins stay live across PTE removal and mm exit. */ + kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); + for (i = 0; i < 3; i++) + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pins->fragments[i].folio)); + fragment_test_unpin(pins); + ret = get_user_fragments_remote(private->vm_mm, private->vm_start, 3 * SZ_4K, + 0, pins->fragments, 8); + pins->count = ret > 0 ? ret : 0; + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + KUNIT_EXPECT_PTR_EQ(test, pins->fragments[i].subpage, NULL); + KUNIT_EXPECT_EQ(test, PFN_PHYS(folio_pfn(pins->fragments[i].folio)), shared_phys); + } + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, private->vm_mm, private->vm_start, 3 * SZ_4K, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 3L); + for (i = 0; i < 3; i++) { + struct user_page_fragment *f = &pins->fragments[i]; + + KUNIT_EXPECT_NOT_NULL(test, f->subpage); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(f->folio)), shared_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x32 + i, f->length), NULL); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, private->vm_mm, + private->vm_start + i * SZ_4K))); + } + fragment_test_unpin(pins); +} + +/* Exercise the same implementation used by the arm64 syscall entry. */ +asmlinkage long __arm64_sys_mprotect(const struct pt_regs *regs); + +static long test_mprotect(unsigned long start, unsigned long len, unsigned long prot) +{ + struct pt_regs regs = { .regs = { start, len, prot } }; + + return __arm64_sys_mprotect(®s); +} + +static unsigned long test_copy_word(unsigned long addr, u64 *value, bool write) +{ + unsigned long left; + + pagefault_disable(); + if (write) + left = copy_to_user((void __user *)addr, value, sizeof(*value)); + else + left = copy_from_user(value, (void __user *)addr, sizeof(*value)); + pagefault_enable(); + return left; +} + +static void user4k_mprotect_test(struct kunit *test) +{ + const unsigned long starts[] = { + TEST_VA, SZ_2M - SZ_4K, SZ_1G - SZ_4K, BIT(39) - SZ_4K, TEST_VA, + }; + unsigned int shape; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (shape = 0; shape < ARRAY_SIZE(starts); shape++) { + struct file *file = shape == 4 ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct mm_struct *mm = test_vma_mm(test); + unsigned long start = starts[shape], mapped; + unsigned long flags = MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS); + phys_addr_t before[4] = { }; + long protect[4] = { }; + unsigned long initial = 0, final = 0, read_ro = 0, write_ro = 0, read_none = 0; + u64 value, ro_value = 0; + unsigned int i; + vm_fault_t fault = 0; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (shape == 4) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, start, PAGE_SIZE, PROT_READ | PROT_WRITE, flags, 0); + if (IS_ERR_VALUE(mapped)) + goto detach; + for (i = 0; i < 4; i++) { + struct vm_area_struct *vma = test_vma_lookup(mm, start + i * SZ_4K); + + fault |= test_file_fault(vma, start + i * SZ_4K, true); + if (fault & VM_FAULT_ERROR) + goto detach; + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, start + i * SZ_4K)); + value = 0x88110000 + i; + initial |= test_copy_word(start + i * SZ_4K, &value, true); + } + /* Keep the mm attached across permission changes and cached accesses. */ + protect[0] = test_mprotect(start + SZ_4K, SZ_4K, PROT_READ); + value = 0xaa; + write_ro = test_copy_word(start + SZ_4K, &value, true); + final |= test_copy_word(start, &value, true); + final |= test_copy_word(start + 3 * SZ_4K, &value, true); + protect[1] = test_mprotect(start + 2 * SZ_4K, SZ_4K, PROT_NONE); + read_none = test_copy_word(start + 2 * SZ_4K, &value, false); + read_ro = test_copy_word(start + SZ_4K, &ro_value, false); + protect[2] = test_mprotect(start, PAGE_SIZE, PROT_READ); + protect[3] = test_mprotect(start, PAGE_SIZE, PROT_READ | PROT_WRITE); + for (i = 0; i < 4; i++) { + value = 0x99220000 + i; + final |= test_copy_word(start + i * SZ_4K, &value, true); + } +detach: + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, initial, 0UL); + KUNIT_EXPECT_EQ(test, final, 0UL); + KUNIT_EXPECT_EQ(test, write_ro, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_none, (unsigned long)sizeof(u64)); + KUNIT_EXPECT_EQ(test, read_ro, 0UL); + KUNIT_EXPECT_EQ(test, ro_value, 0x88110001ULL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, protect[i], 0L); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + } + KUNIT_EXPECT_EQ(test, mm->total_vm, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, mm->map_count, 1); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +#ifdef CONFIG_NUMA_BALANCING +static void user_subpage_numa_cow_batch_test(struct kunit *test) +{ + unsigned int shift; + + if (!(READ_ONCE(sysctl_numa_balancing_mode) & NUMA_BALANCING_NORMAL)) { + kunit_skip(test, "requires normal NUMA balancing; use numa_balancing=enable"); + return; + } + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + unsigned long size = 1UL << shift, start = TEST_VA; + struct vm_area_struct *vma; + struct mm_struct *child = test_vma_mm_granule(test, shift); + struct mm_struct *mm = test_vma_mm_granule(test, shift); + phys_addr_t before; + unsigned long marked, mapped; + long ret; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + /* Use mmap accounting because the child will partially unmap. */ + kthread_use_mm(mm); + mapped = vm_mmap(NULL, start, 4 * size, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, start); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + i * size, true), + (vm_fault_t)0); + before = pte_phys_mm(mm, test_fault_entry(test, mm, start + size)); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + kthread_use_mm(child); + ret = vm_munmap(start + size, size); + kthread_unuse_mm(child); + KUNIT_ASSERT_EQ(test, ret, 0L); + /* Reuse the now-private slot without replacing its native owner. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, start + size, true), (vm_fault_t)0); + KUNIT_ASSERT_EQ(test, pte_phys_mm(mm, + test_fault_entry(test, mm, start + size)), before); + kthread_use_mm(mm); + ret = test_mprotect(start, 4 * size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + vma = test_vma_lookup(mm, start); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Permit local-node hinting, which normally skips a single user. */ + mmget(mm); + mmap_read_lock(mm); + marked = change_prot_numa(vma, start, start + 4 * size); + mmap_read_unlock(mm); + mmput(mm); + KUNIT_EXPECT_EQ(test, marked, 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, start + i * size); + + KUNIT_EXPECT_EQ(test, !!pte_protnone(pte), i == 1); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static void user4k_mprotect_cow_batch_test(struct kunit *test) +{ + struct mm_struct *mm, *child; + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + phys_addr_t before[4]; + unsigned int i; + unsigned long mapped; + long ret[2]; + int copied; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, child); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 8), 1L); + for (i = 0; i < 4; i++) + before[i] = pte_phys_mm(mm, test_fault_entry(test, mm, TEST_VA + i * SZ_4K)); + uprobe_start_dup_mmap(); + copied = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, copied, 0); + kthread_use_mm(mm); + ret[0] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret[0], 0L); + KUNIT_EXPECT_EQ(test, ret[1], 0L); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), before[i]); + /* One native folio contains both exclusive and shared COW slots. */ + KUNIT_EXPECT_EQ(test, pte_write(pte), i == 1); + } + fragment_test_unpin(pins); +} + +static void user4k_mprotect_accounting_test(struct kunit *test) +{ + struct mm_struct *mm = test_vma_mm(test); + unsigned long mapped, committed[4], data[4]; + s64 global[4], before; + long ret[4]; + unsigned int i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + KUNIT_ASSERT_NOT_NULL(test, mm); + before = test_commit_baseline(test); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + if (!IS_ERR_VALUE(mapped)) { + ret[0] = test_mprotect(TEST_VA + SZ_4K, SZ_4K, PROT_READ); + committed[0] = mm->committed_user_pages; + data[0] = mm->data_vm; + global[0] = percpu_counter_sum(&vm_committed_as) - before; + ret[1] = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + committed[1] = mm->committed_user_pages; + data[1] = mm->data_vm; + global[1] = percpu_counter_sum(&vm_committed_as) - before; + ret[2] = test_mprotect(TEST_VA + 2 * SZ_4K, SZ_4K, PROT_READ | PROT_WRITE); + committed[2] = mm->committed_user_pages; + data[2] = mm->data_vm; + global[2] = percpu_counter_sum(&vm_committed_as) - before; + ret[3] = vm_munmap(TEST_VA, PAGE_SIZE); + committed[3] = mm->committed_user_pages; + data[3] = mm->data_vm; + global[3] = percpu_counter_sum(&vm_committed_as) - before; + } + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + for (i = 0; i < 4; i++) { + unsigned long expected = i == 0 ? TEST_SLOTS - 1 : i == 2 ? 1 : 0; + + KUNIT_EXPECT_EQ(test, ret[i], 0L); + KUNIT_EXPECT_EQ(test, committed[i], expected); + KUNIT_EXPECT_EQ(test, data[i], expected); + KUNIT_EXPECT_EQ(test, global[i], expected ? 1LL : 0LL); + } + KUNIT_EXPECT_EQ(test, mm->map_count, 0); +} + +static void user4k_fragment_permissions_test(struct kunit *test) +{ + unsigned long start = TEST_VA + SZ_4K; + struct vm_area_struct *vma = test_fault_vma(test, start, SZ_4K); + struct fragment_test_pins *pins; + struct user_page_fragment *fragment; + struct mm_struct *mm; + spinlock_t *ptl; + pte_t *ptep; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + fragment = &pins->fragments[0]; + { + struct page *page = NULL; + int locked = 1; + + mmap_read_lock(mm); + ret = pin_user_pages_remote(mm, start, 1, 0, &page, &locked); + if (locked) + mmap_read_unlock(mm); + if (ret == 1) + unpin_user_page(page); + KUNIT_EXPECT_EQ(test, ret, -EOPNOTSUPP); + } + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_TRUE(test, is_zero_folio(fragment->folio)); + KUNIT_EXPECT_PTR_EQ(test, fragment->subpage, NULL); + /* Zero-page PTEs alias the first physical 4K despite this VA's slot 1. */ + KUNIT_EXPECT_EQ(test, fragment->offset, 17UL); + KUNIT_EXPECT_EQ(test, fragment->length, 31U); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0, fragment->length), NULL); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); + KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_WRITE); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + ptep = test_mm_lookup(mm, start, &ptl); + if (ptep) { + ptep_set_wrprotect(mm, start, ptep); + pte_unmap_unlock(ptep, ptl); + } + flush_tlb_range(vma, start, start + SZ_4K); + mmap_write_unlock(mm); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE | FOLL_FORCE, 8); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, mm, start))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(fragment->folio) + fragment->offset, + 0xa4, fragment->length), NULL); + fragment_test_unpin(pins); + mmap_write_lock(mm); + vm_flags_clear(vma, VM_READ); + vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + mmap_write_unlock(mm); + ret = fragment_test_pin(pins, mm, start + 17, 31, 0, 8); + KUNIT_EXPECT_EQ(test, ret, -EFAULT); + ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_FORCE, 8); + KUNIT_EXPECT_EQ(test, ret, 1L); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, mm, start, 1, FOLL_GET, 8); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_subpage_unpin(void *arg) +{ + mm_subpage_unpin(arg); +} + +static void user4k_slot_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child; + struct mm_subpage *slots[4]; + struct folio *folio, *dst; + unsigned int i; + vm_fault_t fault; + spinlock_t *ptl; + pte_t *ptep, pte; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + slots[i] = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slots[i]); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, slots[i]), 0); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(slots[i])); + memset(folio_address(mm_subpage_folio(slots[i])) + i * SZ_4K, 0x61 + i, SZ_4K); + } + folio = mm_subpage_folio(slots[1]); + test_swap_tags_set(folio); + for (i = 0; i < 2; i++) { + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[1]); + pte_unmap_unlock(ptep, ptl); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_unpin, slots[1]), 0); + } + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(folio)); + KUNIT_EXPECT_FALSE(test, mm_subpage_try_share(slots[1])); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + /* Finish even unexpected success so a failing assertion cannot leak metadata. */ + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + KUNIT_EXPECT_EQ(test, ret, -EBUSY); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < 4; i++) { + struct mm_subpage *copy; + + pte = test_fault_entry(test, child, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte) == mm_subpage_phys(slots[i]), i != 1); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[i]), i == 1 ? 1 : 2); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slots[i]), i == 1); + copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + 0x61 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + mm_subpage_put(copy); + } + /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ret = mm_subpage_pin(slots[0]); + pte_unmap_unlock(ptep, ptl); + if (!ret) + mm_subpage_unpin(slots[0]); + KUNIT_EXPECT_EQ(test, ret, -EAGAIN); + /* Re-enabling writes must retain the exact backing of an exclusive pinned slot. */ + ptep = test_mm_lookup(vma->vm_mm, TEST_VA + SZ_4K, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + ptep_set_wrprotect(vma->vm_mm, TEST_VA + SZ_4K, ptep); + pte_unmap_unlock(ptep, ptl); + flush_tlb_range(vma, TEST_VA + SZ_4K, TEST_VA + 2 * SZ_4K); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), mm_subpage_phys(slots[1])); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + *((u8 *)folio_address(folio) + SZ_4K) = 0xab; + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_EQ(test, *(u8 *)phys_to_virt(pte_phys_mm(child, pte)), (u8)0x62); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slots[1]), 0); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 2U); + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K), (u8)0xab); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 1U); + kunit_release_action(test, test_subpage_unpin, slots[1]); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(slots[1]), 0U); + KUNIT_EXPECT_FALSE(test, folio_maybe_dma_pinned(folio)); +} + +#ifdef CONFIG_MIGRATION +static struct folio *test_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, 0); +} + +asmlinkage long __arm64_sys_madvise(const struct pt_regs *regs); + +static long test_madvise(unsigned long addr, unsigned long size, unsigned long behavior) +{ + struct pt_regs regs = { .regs = { addr, size, behavior } }; + + return __arm64_sys_madvise(®s); +} + +static void user_granule_large_vma_split_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }, i; + + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) || + (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { + kunit_skip(test, "requires THP boundary splitting and borrowed-mm execution"); + return; + } + for (i = 0; i < ARRAY_SIZE(shifts); i++) { + struct mm_struct *mm; + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i]; + unsigned long base = 4UL * HPAGE_PMD_SIZE, mapped; + pte_t pte; + long ret; + + if (shifts[i] >= PAGE_SHIFT) + continue; + mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, base, 2UL * HPAGE_PMD_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE | MAP_NORESERVE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, base); + vma = test_vma_lookup(mm, base); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base, true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, base + size, true) & VM_FAULT_ERROR, 0U); + /* The generic VMA split probes the native THP boundary for this range. */ + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, base + size); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + pte = test_fault_entry(test, mm, base); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + kthread_use_mm(mm); + ret = test_mprotect(base + size, size, PROT_READ | PROT_WRITE); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, ret, 0L); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user_granule_brk_extent_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + unsigned int count = PAGE_SHIFT > 14 ? 3 : 2, i; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm brk fixture requires hardware opt-in"); + return; + } + for (i = 0; i < count; i++) { + struct mm_struct *mm = mm_alloc(); + struct vm_area_struct *vma; + unsigned long size = 1UL << shifts[i], neighbor; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); + KUNIT_ASSERT_EQ(test, test_mm_select_granule(mm, shifts[i]), 0); + kthread_use_mm(mm); + ret = vm_brk_flags(TEST_VA, size + 1, false); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_EQ(test, vma->vm_start, TEST_VA); + KUNIT_EXPECT_EQ(test, vma->vm_end, TEST_VA + 2 * size); + KUNIT_EXPECT_EQ(test, mm->total_vm, 2UL); + /* FEX reserves exactly this page after its initial ELF/BSS break. */ + kthread_use_mm(mm); + neighbor = vm_mmap(NULL, TEST_VA + 2 * size, size, PROT_NONE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, 0); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, neighbor, TEST_VA + 2 * size); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_droppable_reclaim_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm droppable fixture requires hardware opt-in"); + return; + } + /* Dirty discard, retained GET, retained PIN, migration, zero-page COW. */ + for (mode = 0; mode < 5; mode++) { + struct mm_struct *mm = test_vma_mm(test); + struct vm_area_struct *vma; + struct user_page_fragment held = {}; + struct folio *folio; + unsigned int i, succeeded = 0; + unsigned long mapped; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_DROPPABLE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_WIPEONFORK); + KUNIT_EXPECT_TRUE(test, vma->vm_flags & VM_DONTDUMP); + if (mode == 4) { + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, is_zero_pfn(pte_pfn(pte))); + } + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + if (mode != 4) + for (i = 1; i < PAGE_SIZE / SZ_4K; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + memset(folio_address(folio), 0xa7, PAGE_SIZE); + folio_put(folio); + if (mode == 1 || mode == 2) { + ret = mode == 1 ? get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1) : + pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 3) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + if (mode == 1 || mode == 2) { + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, *(unsigned char *)folio_address(folio), 0xa7); + release_user_fragments(&held, 1, false); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + for (i = 0; i < (mode == 4 ? 1 : PAGE_SIZE / SZ_4K); i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_lazyfree_test(struct kunit *test) +{ + unsigned int mode; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm advice requires hardware opt-in"); + return; + } + /* Partial, redirty, retained GET, typed write, all leaves, migration. */ + for (mode = 0; mode < 10; mode++) { + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *mm; + struct user_page_fragment held = {}; + struct folio *folio; + struct mm_subpage *slot; + unsigned int i, succeeded = 0; + long ret; + pte_t pte; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + mm = vma->vm_mm; + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + if (mode == 6) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + kthread_use_mm(mm); + ret = test_madvise(TEST_VA, + (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, mm, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_dirty(pte), mode == 6); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_lazyfree(slot), mode != 6); + mm_subpage_put(slot); + if (mode == 1) + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + + if (mode == 2 || mode == 3 || mode == 8) { + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, + mode == 3 ? FOLL_WRITE : 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + if (mode == 3 || mode == 8) { + memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); + release_user_fragments(&held, 1, true); + } + } + if (mode == 7) { + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + } + if (mode == 5 || mode == 9) { + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 1L); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_EQ(test, ret, 0L); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + } + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + (i == 0 && (mode == 0 || mode == 5))); + } + if (mode == 2 || mode == 6 || mode == 7) + release_user_fragments(&held, 1, false); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_mprotect_migration_test(struct kunit *test) +{ + unsigned int file_case; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm permission fixture requires hardware opt-in"); + return; + } + for (file_case = 0; file_case < 2; file_case++) { + struct mm_struct *mm = test_vma_mm(test); + struct file *file = file_case ? test_cache_file(test, PAGE_SIZE, true) : NULL; + struct vm_area_struct *vma; + struct folio *src, *dst; + unsigned long mapped; + unsigned int i; + int prepared, unmapped = -1; + long protected = -EINVAL; + bool offsets = true; + phys_addr_t phys; + + KUNIT_ASSERT_NOT_NULL(test, mm); + if (file_case) + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + kthread_use_mm(mm); + mapped = vm_mmap(file, TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | (file ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, TEST_VA); + vma = test_vma_lookup(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing migration source"); + return; + } + phys = PFN_PHYS(folio_pfn(src)); + folio_lock(src); + folio_lock(dst); + prepared = mm_subpage_migrate_prepare(src, dst); + if (!prepared) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + kthread_use_mm(mm); + protected = test_mprotect(TEST_VA, PAGE_SIZE, PROT_READ); + kthread_unuse_mm(mm); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + offsets &= softleaf_is_migration(softleaf_from_pte(pte)) && + !softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, prepared, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, protected, 0L); + KUNIT_EXPECT_TRUE(test, offsets); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), phys + i * SZ_4K); + } + } +} + +/* Independent roots and VA offsets must not change physical slot identity. */ +static void user4k_nonlinear_rmap_test(struct kunit *test) +{ + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct mm_struct *child = test_vma_mm(test); + struct mm_subpage *slot; + struct folio *folio, *dst; + phys_addr_t phys; + vm_flags_t flags; + unsigned int succeeded = 0, side; + int ret, unmapped = -1; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, child); + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + /* Unlike the lightweight rmap fixture, real LRU migration needs a charge. */ + ret = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (ret) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge nonlinear fixture folio"); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + phys = mm_subpage_phys(slot); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + /* Same native folio, different root and different VA/physical offset. */ + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + KUNIT_EXPECT_PTR_EQ(test, mm_subpage_folio(slot), folio); + memset(folio_address(folio), 0x53, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xa7, SZ_4K); + folio_lock(folio); + ret = folio_referenced(folio, 1, NULL, &flags); + folio_unlock(folio); + KUNIT_EXPECT_EQ(test, ret, 2); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, b->mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 2); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, + test_fault_entry(test, child, TEST_VA + SZ_4K)), phys + 3 * SZ_4K); + kunit_release_action(test, user4k_vma_mm_free, child); + + /* Rollback must find migration entries through both roots. */ + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + folio_get(folio); + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = folio_mapcount(folio); + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 2); + + /* Successful migration exercises copied identities in the new owner. */ + folio_add_lru(folio); + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (ret) { + KUNIT_FAIL(test, "nonlinear migration failed: %d", ret); + return; + } + /* Refresh fixture references before assertions/cleanup after migration. */ + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + pte_t pte = test_fault_entry(test, f->mm, TEST_VA + side * SZ_4K); + + f->slots[side] = NULL; + f->mapped[side] = false; + if (!pte_present(pte)) + continue; + slot = mm_subpage_get_from_phys(pte_phys_mm(f->mm, pte)); + if (!slot) + continue; + f->slots[side] = slot; + f->mapped[side] = true; + mm_subpage_put(slot); + } + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + for (side = 0; side < 2; side++) { + struct anon_slot_fixture *f = side ? b : a; + + slot = f->slots[side]; + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_NE(test, mm_subpage_phys(slot), phys + (side ? 3 * SZ_4K : 0)); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), side ? 0xa7 : 0x53, SZ_4K), NULL); + } +} + +static void user4k_migration_entries_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct mm_struct *child = test_vma_mm(test); + struct folio *src, *dst; + unsigned int i, side; + vm_fault_t fault; + bool encoded = true, copied = true, exact = true; + int ret, fork_ret = -EINVAL, unmapped = -1; + + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_NOT_NULL(test, child); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + dst = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, dst); + src = test_fault_folio(vma->vm_mm, TEST_VA); + if (!src) { + folio_put(dst); + KUNIT_FAIL(test, "missing source folio"); + return; + } + folio_lock(src); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(src, dst); + if (!ret) { + try_to_migrate(src, TTU_SYNC); + unmapped = folio_mapcount(src); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + struct rmap_walk_range range = { + .address = TEST_VA + i * SZ_4K, + .subpage = true, + .subpage_offset = i * SZ_4K, + }; + DEFINE_FOLIO_RMAP_WALK(walk, src, vma, range, + PVMW_SYNC | PVMW_MIGRATION); + unsigned int count = 0; + + encoded &= softleaf_is_migration_write(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + while (page_vma_mapped_walk(&walk)) { + exact &= walk.address == TEST_VA + i * SZ_4K; + count++; + } + exact &= count == 1; + } + uprobe_start_dup_mmap(); + fork_ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!fork_ret) { + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + + copied &= softleaf_is_migration_read(softleaf_from_pte(pte)) && + pte_swp_subpage_offset(pte) == i * SZ_4K; + } + } + } + /* Drop one migration entry while the other mm still retains it. */ + mmap_read_lock(vma->vm_mm); + zap_vma_range(vma, TEST_VA + SZ_4K, SZ_4K); + mmap_read_unlock(vma->vm_mm); + remove_migration_ptes(src, src, 0); + mm_subpage_migrate_finish(src); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(src); + folio_put(dst); + folio_put(src); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, fork_ret, 0); + KUNIT_EXPECT_EQ(test, unmapped, 0); + KUNIT_EXPECT_TRUE(test, encoded); + KUNIT_EXPECT_TRUE(test, copied); + KUNIT_EXPECT_TRUE(test, exact); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), side ? (unsigned long)TEST_SLOTS : TEST_SLOTS - 1UL); + for (i = 0; i < 4; i++) { + pte_t pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + struct mm_subpage *slot; + + if (!side && i == 1) { + KUNIT_EXPECT_TRUE(test, pte_none(pte)); + continue; + } + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), i == 1 ? 1 : 2); + mm_subpage_put(slot); + } + } +} + +static struct folio *test_file_migration_alloc(struct folio *src, unsigned long private) +{ + return folio_alloc(GFP_HIGHUSER_MOVABLE, folio_order(src)); +} + +static void user64k_file_migration_test(struct kunit *test) +{ + unsigned int mode; + + if (!test_coarse_file_enabled(test)) + return; + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "coarse shmem promotion requires collapse support"); + return; + } + for (mode = 0; mode < 6; mode++) { + bool shmem = mode & 1, fail = mode >= 2, pin = mode >= 4; + bool isolated; + unsigned long len = shmem ? SZ_64K : 2 * SZ_64K; + unsigned long base = shmem ? SZ_64K : 0, offset; + unsigned int nr = len / PAGE_SIZE, succeeded = 0; + unsigned int maps = 2 * nr + len / SZ_4K - 1; + struct file *file = shmem ? test_cache_file(test, 2 * SZ_64K, true) : + test_coarse_cache_file(test, 17 - PAGE_SHIFT, 2 * SZ_64K); + struct vm_area_struct *coarse, *native, *small; + struct folio *folio; + struct user_page_fragment *fragment = NULL; + phys_addr_t old_phys, phys; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, len, base); + native = test_file_vma(test, file, true, true, TEST_VA, len, base); + /* Shifted file offset and a small-page table boundary in the alias. */ + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, + len - SZ_4K, base + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_NOT_NULL(test, small); + for (offset = 0; offset < len; offset += SZ_64K) + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len; offset += PAGE_SIZE) + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA + offset, shmem) & + VM_FAULT_ERROR, 0U); + for (offset = 0; offset < len - SZ_4K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + offset, + shmem) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + old_phys = page_to_phys(&folio->page); + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(small->vm_mm, + small->vm_end - SZ_4K + 37, 1, 0, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, len - SZ_4K + 37); + } + isolated = isolate_folio_to_list(folio, &folios); + /* Retain either a GET or a typed PIN to force migration rollback. */ + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) { + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page), old_phys); + KUNIT_EXPECT_EQ(test, *(u8 *)(folio_address(fragment->folio) + + fragment->offset), + (u8)(0x31 + (base + len - SZ_4K) / SZ_4K)); + kunit_release_action(test, test_promote_unpin, fragment); + } else if (fail) { + folio_put(folio); + } + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + folio = filemap_get_folio(file->f_mapping, base >> PAGE_SHIFT); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + phys = page_to_phys(&folio->page); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + KUNIT_EXPECT_EQ(test, folio_size(folio), len); + KUNIT_EXPECT_EQ(test, folio->index, base >> PAGE_SHIFT); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)maps); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr + maps + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(coarse->vm_mm, + shmem ? MM_SHMEMPAGES : MM_FILEPAGES), (long)(len / SZ_64K)); + for (offset = 0; offset < len; offset += SZ_4K) { + pte_t pte; + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + (base + offset) / SZ_4K, SZ_4K), NULL); + if (offset) { + pte = test_fault_entry(test, small->vm_mm, + small->vm_start + offset - SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, pte), phys + offset); + } + if (!(offset % PAGE_SIZE)) { + pte = test_fault_entry(test, native->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, pte), phys + offset); + } + if (!(offset % SZ_64K)) { + pte = test_fault_entry(test, coarse->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), shmem); + KUNIT_EXPECT_EQ(test, pte_phys_mm(coarse->vm_mm, pte), phys + offset); + } + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); + } +} + +static void user64k_anon_migration_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_file_enabled(test)) + return; + for (mode = 0; mode < 5; mode++) { + bool forked = mode & 1, fail = mode >= 2, pin = mode == 4; + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct user_page_fragment *fragment = NULL; + struct folio *folio; + phys_addr_t old_phys, phys; + unsigned int i, succeeded = 0; + pte_t pte; + bool isolated; + int ret; + LIST_HEAD(folios); + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & + VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + old_phys = pte_phys_mm(vma->vm_mm, pte); + folio = page_folio(pfn_to_page(PHYS_PFN(old_phys))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(folio, i)), 0x51 + i, PAGE_SIZE); + if (forked) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (pin) { + fragment = kunit_kzalloc(test, sizeof(*fragment), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, fragment); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + test_promote_unpin, fragment), 0); + KUNIT_ASSERT_EQ(test, pin_user_fragments_remote(vma->vm_mm, + TEST_VA + SZ_64K - 1, 1, FOLL_WRITE, fragment, 1), 1L); + KUNIT_EXPECT_PTR_EQ(test, fragment->folio, folio); + KUNIT_EXPECT_EQ(test, fragment->offset, (unsigned int)SZ_64K - 1); + } + lru_add_drain_all(); + folio_get(folio); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || pin || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_file_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (pin) + kunit_release_action(test, test_promote_unpin, fragment); + else if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ_MSG(test, ret, fail ? 1 : 0, "mode=%u", mode); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : nr); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + phys = pte_phys_mm(vma->vm_mm, pte); + KUNIT_EXPECT_EQ(test, phys == old_phys, fail); + folio = page_folio(pfn_to_page(PHYS_PFN(phys))); + folio_get(folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + expect_coarse_anon_state(test, folio, forked ? 2 : 1, !forked); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)(nr * (forked ? 2 : 1) + 1)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (forked) { + struct folio *copied; + + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, pte), phys); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); + /* A restored shared migration entry must still perform full-leaf COW. */ + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_64K - 1, + true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), phys); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(copied, i)), + 0x51 + i, PAGE_SIZE), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, i)), + 0x51 + i, PAGE_SIZE), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + } +} + +static void user4k_file_migration_test(struct kunit *test) +{ + unsigned int fail; + + for (fail = 0; fail < 2; fail++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *small, *native; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, succeeded = 0; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_file_vma(test, file, false, true, SZ_2M - SZ_4K, PAGE_SIZE, 0); + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + true) & VM_FAULT_ERROR, 0U); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, true) & VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + old_phys = PFN_PHYS(folio_pfn(folio)); + isolated = isolate_folio_to_list(folio, &folios); + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + for (i = 0; i < 4; i++) { + pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); + + KUNIT_ASSERT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + PFN_PHYS(folio_pfn(folio)) + i * SZ_4K); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry) == old_phys + i * SZ_4K, !!fail); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + } + KUNIT_EXPECT_EQ(test, pte_phys_mm(native->vm_mm, + test_fault_entry(test, native->vm_mm, TEST_VA)), PFN_PHYS(folio_pfn(folio))); + /* File reclaim must visit every 4K alias, including across PTE tables. */ + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + folio_unlock(folio); + folio_put(folio); + for (i = 0; i < 4; i++) { + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, small->vm_mm, + small->vm_start + i * SZ_4K))); + } + for (i = 0; i < 4; i++) + KUNIT_ASSERT_EQ(test, test_file_fault(small, small->vm_start + i * SZ_4K, + false) & VM_FAULT_ERROR, 0U); + } +} + +static void user4k_migration_test(struct kunit *test) +{ + const unsigned long boundaries[] = { + SZ_2M - SZ_4K, SZ_2M, SZ_1G - SZ_4K, SZ_1G, + BIT(39) - SZ_4K, BIT(39), + }; + unsigned int mode; + + /* Dense/sparse, forks, refcount rollback, batched TLBI and table edges. */ + for (mode = 0; mode < 16; mode++) { + bool edge = mode >= 6 && mode < 12; + bool locked = mode >= 12; + bool sparse = mode == 1 || edge || mode == 13; + bool forked = mode == 2 || mode == 4 || edge || mode == 14; + bool fail = mode == 3 || mode == 4 || mode == 15; + unsigned long start = edge ? boundaries[mode - 6] : + TEST_VA + (sparse ? SZ_4K : 0); + unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); + struct mm_struct *child = NULL; + struct folio *folio; + phys_addr_t old_phys; + vm_fault_t fault; + int ret; + bool isolated; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, vma); + if (locked) { + mmap_write_lock(vma->vm_mm); + vm_flags_set(vma, VM_LOCKED); + vma->vm_mm->locked_vm = nr; + mmap_write_unlock(vma->vm_mm); + } + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, start, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); + test_swap_tags_set(folio); + folio_put(folio); + if (forked) { + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + } + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + /* An extra native reference forces the normal migration rollback. */ + if (!fail || !isolated) + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + mode == 5 ? MIGRATE_ASYNC : MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + if (fail) + folio_put(folio); + KUNIT_EXPECT_EQ(test, ret, fail ? 1 : 0); + KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); + for (side = 0; side <= (unsigned int)forked; side++) { + struct mm_struct *mm = side ? child : vma->vm_mm; + + for (i = 0; i < nr; i++) { + unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); + struct mm_subpage *slot; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) == old_phys + index * SZ_4K, fail); + KUNIT_EXPECT_EQ(test, !!pte_write(pte), !forked); + slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + folio = mm_subpage_folio(slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), index * (unsigned int)SZ_4K); + KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), forked ? 2 : 1); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + index * SZ_4K, + 0x41 + index + mode, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, index)); + mm_subpage_put(slot); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)nr); + } + if (locked) { + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, start); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_mlocked(folio)); + if (!fail) + KUNIT_EXPECT_EQ(test, folio->mlock_count, nr); + folio_put(folio); + } + if (child) + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + } +} +struct fragment_pin_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready; + unsigned long checks; + int error; +}; + +static int fragment_pin_worker_run(void *arg) +{ + struct fragment_pin_worker *worker = arg; + struct user_page_fragment fragments[4]; + + while (!kthread_should_stop()) { + long nr = pin_user_fragments_remote(worker->mm, TEST_VA, PAGE_SIZE, + 0, fragments, ARRAY_SIZE(fragments)); + unsigned int i; + + if (nr != 4) { + worker->error = nr < 0 ? nr : -EIO; + } else { + for (i = 0; i < nr; i++) { + struct user_page_fragment *f = &fragments[i]; + + if (memchr_inv(folio_address(f->folio) + f->offset, 0x51 + i, f->length)) + worker->error = -EILSEQ; + worker->checks++; + } + } + if (nr > 0) + release_user_fragments(fragments, nr, false); + complete_all(&worker->ready); + if (worker->error) + break; + cond_resched(); + } + return 0; +} + +static void fragment_pin_worker_stop(void *arg) +{ + struct fragment_pin_worker *worker = arg; + + if (worker->task) { + kthread_stop(worker->task); + worker->task = NULL; + } +} + +static void user4k_fragment_pin_race_test(struct kunit *test) +{ + struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + struct fragment_test_pins *pins; + struct fragment_pin_worker *workers; + unsigned int i, round, migrated = 0, busy = 0; + long ret; + + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0x51 + i, pins->fragments[i].length); + fragment_test_unpin(pins); + workers = kunit_kcalloc(test, 2, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + for (i = 0; i < 2; i++) { + workers[i].mm = vma->vm_mm; + init_completion(&workers[i].ready); + workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "pin worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + } + for (round = 0; round < 32; round++) { + struct mm_struct *child = test_vma_mm(test); + unsigned long addr = TEST_VA + (round % 4) * SZ_4K; + struct vm_area_struct *cvma; + struct folio *folio; + unsigned int succeeded = 0; + vm_fault_t fault; + struct user_page_fragment write_fragment; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0L); + cvma = test_vma_lookup(child, addr); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + ret = pin_user_fragments_remote(child, addr, 1, FOLL_WRITE, &write_fragment, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + *((u8 *)folio_address(write_fragment.folio) + write_fragment.offset) = 0xa0 + round; + release_user_fragments(&write_fragment, 1, true); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = isolate_folio_to_list(folio, &folios); + folio_put(folio); + if (ret) { + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_TRUE(test, ret == 0 || ret == 1); + migrated += succeeded; + busy += ret == 1; + } + kunit_release_action(test, user4k_vma_mm_free, child); + } + for (i = 0; i < 2; i++) { + fragment_pin_worker_stop(&workers[i]); + KUNIT_EXPECT_EQ(test, workers[i].error, 0); + KUNIT_EXPECT_GE(test, workers[i].checks, 4UL); + } + kunit_info(test, "32 fork/COW/exit cycles against two pin workers; migrations=%u not_migrated=%u\n", + migrated, busy); +} + +#endif + +static int test_swap_control(const char *operation) +{ + char *argv[] = { "/init", (char *)operation, NULL }; + char *envp[] = { NULL }; + + return call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); +} + +static void test_reenable_swap(void *arg) +{ + bool *disabled = arg; + + if (*disabled) + test_swap_control("--swapon-helper"); +} + +/* Exclusive slots must remain packed after read swap-in and later writes. */ +static void test_user4k_swap_exclusive(struct kunit *test, unsigned int mode) +{ + struct vm_area_struct *vma; + struct folio *folio; + struct mm_subpage *held = NULL; + bool *disabled; + phys_addr_t restored[TEST_SLOTS]; + swp_entry_t entry; + vm_fault_t fault; + unsigned int i; + int ret, mapped, hold_err = 0; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xc0 + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 5) { + held = mm_subpage_get_from_phys(PFN_PHYS(folio_pfn(folio)) + 2 * SZ_4K); + if (held) { + hold_err = kunit_add_action_or_reset(test, test_subpage_put, held); + if (hold_err) + held = NULL; + } + } + ret = folio_alloc_swap(folio); + entry = folio->swap; + if (!ret) + try_to_unmap(folio, TTU_SYNC); + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + if (mode == 5) { + KUNIT_ASSERT_EQ(test, hold_err, 0); + KUNIT_ASSERT_NOT_NULL(test, held); + } + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(pte))); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + if (mode >= 3) { + struct mm_struct *child = test_vma_mm(test); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + } + } + if (mode == 1 || mode == 2 || mode == 4 || mode == 6) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + if (mode == 2 || mode == 6) { + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + } + for (i = 0; i < TEST_SLOTS; i++) { + struct mm_subpage *slot; + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + restored[i] = pte_phys_mm(vma->vm_mm, pte); + slot = mm_subpage_get_from_phys(restored[i]); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(slot), !(held && i == 2)); + KUNIT_EXPECT_EQ(test, restored[i], restored[0] + i * SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xc0 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), i)); + mm_subpage_put(slot); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, vma->vm_mm, address); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + if (held && i == 2) { + KUNIT_EXPECT_NE(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + memset(phys_to_virt(pte_phys_mm(vma->vm_mm, pte)), 0xdd, SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(held)) + + mm_subpage_offset(held), 0xc2, SZ_4K), NULL); + } else { + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), restored[i]); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0UL); + if (*disabled) { + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); + } +} + +static void user4k_swap_exclusive_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 0); +} + +static void user4k_swap_exclusive_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 1); +} + +static void user4k_swap_exclusive_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 2); +} + +static void user4k_swap_reuse_cached_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 3); +} + +static void user4k_swap_reuse_disk_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 4); +} + +static void user4k_swap_reuse_get_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 5); +} + +static void user4k_swap_reuse_swapoff_test(struct kunit *test) +{ + test_user4k_swap_exclusive(test, 6); +} + +static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 base) +{ + const bool evict = mode != 0; + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + swp_entry_t entry = { 0 }; + unsigned int i, count; + int err, mapped; + vm_fault_t fault; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "rerun this suite after enabling the disposable guest swap disk"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, base + i, SZ_4K); + test_swap_tags_set(folio); + if (mode == 2) { + LIST_HEAD(folios); + + folio_unlock(folio); + err = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!err) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Let the normal reclaim path allocate swap and unmap all slots. */ + reclaim_pages(&folios); + mapped = 0; + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + mapped += pte_present(pte); + } + entry = softleaf_from_pte(pte); + } else { + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC | (evict ? TTU_BATCH_FLUSH : 0)); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + try_to_unmap_flush(); + folio_put(folio); + } + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, parent->vm_mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), i * (unsigned int)SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_swp_exclusive(pte)); + } + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, (unsigned int)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + + /* Fork while all slots are nonpresent, preserving each byte offset. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 2U * TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 2U); + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long address = TEST_VA + ((i * 5 + 2) % TEST_SLOTS) * SZ_4K; + struct mm_subpage *slot; + spinlock_t *ptl; + pte_t *ptep; + bool intact = false; + + pte = test_fault_entry(test, parent->vm_mm, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + pte = test_fault_entry(test, child, address); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(pte)); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), ((i * 5 + 2) % TEST_SLOTS) * (unsigned int)SZ_4K); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !i); + ptep = test_mm_lookup(parent->vm_mm, address, &ptl); + KUNIT_ASSERT_NOT_NULL(test, ptep); + pte = ptep_get(ptep); + slot = pte_present(pte) ? mm_subpage_get_from_phys(pte_phys_mm(parent->vm_mm, pte)) : NULL; + if (slot) { + intact = !memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), base + ((i * 5 + 2) % TEST_SLOTS), SZ_4K) && + test_swap_tags_match(mm_subpage_folio(slot), ((i * 5 + 2) % TEST_SLOTS)); + mm_subpage_put(slot); + } + pte_unmap_unlock(ptep, ptl); + KUNIT_EXPECT_TRUE(test, intact); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), 0UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 1U); + /* A write fault must swap in and then COW just the child's selected slot. */ + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); + memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + folio_put(folio); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), i == 1 ? 0U : 1U); + /* Exit with three swapped slots, then tear down the restored parent. */ + kunit_release_action(test, user4k_vma_mm_free, child); + /* Include extended counts when a native page has 16 user slots. */ + count = swp_swapcount(entry); + KUNIT_EXPECT_EQ(test, count, 0U); + for (i = 0; i < TEST_SLOTS; i++) + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, i * SZ_4K), 0U); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); +} + +static void user4k_swap_mremap_test(struct kunit *test) +{ + struct mm_struct *mm; + struct folio *folio; + unsigned long old = TEST_VA, new = SZ_1G - SZ_4K, mapped, moved; + pte_t saved[TEST_SLOTS]; + u8 byte = 0x55; + int ret; + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mremap fixture requires hardware opt-in"); + return; + } + mm = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, mm); + kthread_use_mm(mm); + mapped = vm_mmap(NULL, old, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, mapped, old); + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, old, &byte, 1, FOLL_WRITE), 1); + folio = test_fault_folio(mm, old); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x81 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + for (i = 0; i < TEST_SLOTS; i++) { + saved[i] = test_fault_entry(test, mm, old + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(saved[i]))); + } + kthread_use_mm(mm); + moved = test_mremap(old, PAGE_SIZE, PAGE_SIZE, MREMAP_FIXED | MREMAP_MAYMOVE, new); + kthread_unuse_mm(mm); + KUNIT_ASSERT_EQ(test, moved, new); + KUNIT_EXPECT_EQ(test, mm->committed_user_pages, (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte_t entry = test_fault_entry(test, mm, new + i * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(entry).val, softleaf_from_pte(saved[i]).val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(entry), i * (unsigned int)SZ_4K); + } + for (i = 0; i < TEST_SLOTS; i++) { + unsigned int slot = (i + 2) % TEST_SLOTS; + unsigned long addr = new + slot * SZ_4K; + + KUNIT_ASSERT_EQ(test, access_remote_vm(mm, addr, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x81 + slot)); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, slot)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); +} + +static void user4k_swap_pin_test(struct kunit *test) +{ + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + phys_addr_t pinned_phys; + unsigned int i; + vm_fault_t fault; + long ret; + pte_t pte; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA, PAGE_SIZE, FOLL_WRITE, 4); + KUNIT_ASSERT_EQ(test, ret, 4L); + for (i = 0; i < 4; i++) + memset(folio_address(pins->fragments[i].folio) + pins->fragments[i].offset, + 0xb1 + i, pins->fragments[i].length); + test_swap_tags_set(pins->fragments[0].folio); + fragment_test_unpin(pins); + ret = fragment_test_pin(pins, vma->vm_mm, TEST_VA + SZ_4K, SZ_4K, FOLL_WRITE, 1); + KUNIT_ASSERT_EQ(test, ret, 1L); + pinned_phys = mm_subpage_phys(pins->fragments[0].subpage); + folio = pins->fragments[0].folio; + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), pinned_phys); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(pins->fragments[0].subpage)); + KUNIT_EXPECT_EQ(test, mm_subpage_pincount(pins->fragments[0].subpage), 1U); + memset(folio_address(folio) + SZ_4K, 0xd2, SZ_4K); + fragment_test_unpin(pins); + /* Once released, the formerly pinned PTE can be swapped normally. */ + folio = test_fault_folio(vma->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + try_to_unmap(folio, TTU_SYNC); + ret = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0L); + for (i = 0; i < 4; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + i == 1 ? 0xd2 : 0xb1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } +} + +#ifdef CONFIG_MIGRATION +static void user4k_swap_migration_test(struct kunit *test) +{ + struct vm_area_struct *vma, *cvma; + struct mm_struct *child; + struct folio *folio; + phys_addr_t old_phys; + unsigned int i, side, succeeded = 0; + vm_fault_t fault; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + old_phys = PFN_PHYS(folio_pfn(folio)); + folio_lock(folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0xa1 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_test_swapcache(folio)); + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + /* One present slot and seven swap entries must all follow the new cache folio. */ + for (side = 0; side < 2; side++) { + struct vm_area_struct *area = side ? cvma : vma; + struct mm_struct *mm = area->vm_mm; + + for (i = 0; i < TEST_SLOTS; i++) { + unsigned long addr = TEST_VA + i * SZ_4K; + + mmap_read_lock(mm); + fault = handle_mm_fault(area, addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_NE(test, PFN_PHYS(folio_pfn(folio)), old_phys); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0xa1 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + } +} +#endif + +static void user4k_swap_cached_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 0, 0x51); +} + +static void user4k_swap_disk_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 1, 0x61); +} + +static void user4k_swap_reclaim_test(struct kunit *test) +{ + test_user4k_swap_roundtrip(test, 2, 0x71); +} + +static void user4k_swap_stress_test(struct kunit *test) +{ + unsigned int i; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (i = 0; i < 32; i++) + test_user4k_swap_roundtrip(test, i % 3, 0x21 + i * 5); + kunit_info(test, "32 swap/fork/COW/exit cycles with distinct data patterns\n"); +} + +static void user4k_swapoff_test(struct kunit *test) +{ + struct vm_area_struct *parent, *cvma; + struct mm_struct *child; + struct folio *folio; + bool *disabled; + vm_fault_t fault; + unsigned int i, side; + swp_entry_t entry; + int ret; + pte_t pte; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < TEST_SLOTS; i++) + memset(folio_address(folio) + i * SZ_4K, 0x91 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, parent->vm_mm, TEST_VA); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent->vm_mm, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), (unsigned long)TEST_SLOTS); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, *disabled); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), (unsigned long)TEST_SLOTS); + for (i = 0; i < TEST_SLOTS; i++) { + pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + i * (unsigned long)SZ_4K); + folio = test_fault_folio(mm, TEST_VA + i * SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x91 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, i)); + folio_put(folio); + } + } + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + mmap_read_lock(child); + fault = handle_mm_fault(cvma, TEST_VA + 3 * SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + 3 * SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, TEST_VA + 3 * SZ_4K))); + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_EXPECT_EQ(test, ret, 0); +} + +static void user4k_swapoff_boundaries_test(struct kunit *test) +{ + const unsigned long boundaries[] = { SZ_2M, SZ_1G, 1UL << 39 }; + bool *disabled; + unsigned int boundary; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + for (boundary = 0; boundary < ARRAY_SIZE(boundaries); boundary++) { + unsigned long start = boundaries[boundary] - SZ_4K; + struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); + struct mm_struct *child; + unsigned int i, side; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, parent); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + struct folio *folio; + vm_fault_t fault; + swp_entry_t entry; + LIST_HEAD(folios); + + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + lru_add_drain_all(); + folio = test_fault_folio(parent->vm_mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + memset(folio_address(folio) + offset, 0xb0 + boundary * 2 + i, SZ_4K); + test_swap_tags_set(folio); + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, parent->vm_mm, addr)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = test_swap_control("--swapoff-helper"); + *disabled = !total_swap_pages; + KUNIT_ASSERT_EQ(test, ret, 0); + for (side = 0; side < 2; side++) { + struct mm_struct *mm = side ? child : parent->vm_mm; + + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); + for (i = 0; i < 2; i++) { + unsigned long addr = start + i * SZ_4K; + unsigned int offset = vma_page_offset_at(parent, addr).offset; + pte_t pte = test_fault_entry(test, mm, addr); + struct folio *folio; + + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_FALSE(test, pte_write(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte) & ~PAGE_MASK, + (unsigned long)offset); + folio = test_fault_folio(mm, addr); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0xb0 + boundary * 2 + i, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, offset / SZ_4K)); + folio_put(folio); + } + } + ret = test_swap_control("--swapon-helper"); + if (!ret) + *disabled = false; + KUNIT_ASSERT_EQ(test, ret, 0); + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, parent->vm_mm); + } +} + +struct live_swap_reader { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, resume, entered, done; + unsigned int id, checks; + int error; +}; + +static int live_swap_reader_run(void *arg) +{ + struct live_swap_reader *r = arg; + unsigned int phase, i; + + kthread_use_mm(r->mm); + for (phase = 0; phase < 2; phase++) { + if (phase) + complete(&r->entered); + for (i = 0; i < 4; i++) { + u8 value = 0; + void __user *addr = (void __user *)(TEST_VA + i * SZ_4K + 64 + r->id); + + /* Resolve AF, swap and COW faults through the real uaccess path. */ + if (copy_from_user(&value, addr, 1) || value != 0xd1 + i) + r->error = phase ? 2 : 1; + r->checks++; + } + if (!phase) { + complete(&r->ready); + if (!wait_for_completion_timeout(&r->resume, 15 * HZ)) { + r->error = 3; + goto out; + } + } + } + { + u8 value = 0xe0 + r->id; + void __user *addr = (void __user *)(TEST_VA + SZ_4K + 64 + r->id); + + if (copy_to_user(addr, &value, 1)) + r->error = 4; + r->checks++; + } +out: + kthread_unuse_mm(r->mm); + complete(&r->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void live_swap_reader_stop(void *arg) +{ + struct live_swap_reader *r = arg; + + if (r->task) { + complete_all(&r->resume); + kthread_stop(r->task); + r->task = NULL; + } +} + +static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) +{ + bool migration = mode != 0; + struct vm_area_struct *parent; + struct mm_struct *child; + struct live_swap_reader *readers; + struct folio *folio; + vm_fault_t fault; + unsigned int i, cpu; + swp_entry_t entry; + u64 tcr = read_sysreg(tcr_el1); + int ret; + LIST_HEAD(folios); + + if ((!migration && !total_swap_pages) || !user4k_test_enabled || num_online_cpus() < 2 || + !system_supports_4kb_granule() || system_supports_cnp() || + cpus_have_final_cap(ARM64_HAS_S1PIE) || cpus_have_final_cap(ARM64_HAS_S1POE) || + (tcr & (TCR_EL1_DS | TCR_EL1_A1 | TCR_EL1_T0SZ_MASK)) != + (TCR_EL1_A1 | TCR_T0SZ(48))) { + kunit_skip(test, "requires two CPUs, compatible opt-in 4K TTBR0 controls, and swap for reclaim"); + return; + } + parent = test_fault_vma(test, TEST_VA, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, parent); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 4; i++) + memset(folio_address(folio) + i * SZ_4K, 0xd1 + i, SZ_4K); + folio_put(folio); + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + readers = kunit_kcalloc(test, 2, sizeof(*readers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, readers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 2; i++) { + readers[i].mm = child; + readers[i].id = i; + init_completion(&readers[i].ready); + init_completion(&readers[i].resume); + init_completion(&readers[i].entered); + init_completion(&readers[i].done); + readers[i].task = kthread_create(live_swap_reader_run, &readers[i], + "user4k-swap-read-%u", i); + if (IS_ERR(readers[i].task)) { + readers[i].task = NULL; + KUNIT_FAIL(test, "live swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, live_swap_reader_stop, + &readers[i]), 0); + kthread_bind(readers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + wake_up_process(readers[i].task); + } + for (i = 0; i < 2; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].ready, 10 * HZ), 0UL); + lru_add_drain_all(); + folio = test_fault_folio(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + if (mode == 2) { +#ifdef CONFIG_MIGRATION + struct folio *dst = folio_alloc(GFP_KERNEL, 0); + bool entered = true, blocked = true, unmapped = false; + + if (!dst) { + folio_put(folio); + KUNIT_FAIL(test, "migration waiter destination allocation failed"); + return; + } + folio_lock(folio); + folio_lock(dst); + ret = mm_subpage_migrate_prepare(folio, dst); + if (!ret) { + try_to_migrate(folio, TTU_SYNC); + unmapped = !folio_mapped(folio); + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + entered &= !!wait_for_completion_timeout(&readers[i].entered, HZ); + if (wait_for_completion_timeout(&readers[i].done, max(1UL, HZ / 20UL))) { + blocked = false; + complete(&readers[i].done); + } + } + remove_migration_ptes(folio, folio, 0); + mm_subpage_migrate_finish(folio); + mm_subpage_migrate_finish(dst); + } + folio_unlock(dst); + folio_unlock(folio); + folio_put(dst); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, entered); + KUNIT_EXPECT_TRUE(test, blocked); + KUNIT_EXPECT_TRUE(test, unmapped); +#endif + } else if (migration) { +#ifdef CONFIG_MIGRATION + unsigned int succeeded = 0; + phys_addr_t old_phys = PFN_PHYS(folio_pfn(folio)); + + ret = isolate_folio_to_list(folio, &folios) ? 0 : -EBUSY; + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_ASYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, succeeded, 1U); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, test_fault_entry(test, child, TEST_VA)), old_phys); +#endif + } else { + ret = folio_isolate_lru(folio) ? 0 : -EBUSY; + if (!ret) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = softleaf_from_pte(test_fault_entry(test, child, TEST_VA)); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + } + for (i = 0; i < 2; i++) + complete(&readers[i].resume); + for (i = 0; i < 2; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&readers[i].done, 15 * HZ), 0UL); + live_swap_reader_stop(&readers[i]); + KUNIT_EXPECT_EQ(test, readers[i].error, 0); + KUNIT_EXPECT_EQ(test, readers[i].checks, 9U); + } + folio = test_fault_folio(child, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + folio_put(folio); + mmap_read_lock(parent->vm_mm); + fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(parent->vm_mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xd2, SZ_4K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), 0UL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 4UL); + kunit_info(test, "two CPUs access through uaccess after %s, then write one child slot\n", + mode == 2 ? "migration-entry rollback" : migration ? "migration" : "reclaim"); +} + +static void user4k_live_swap_fault_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 0); +} + +#ifdef CONFIG_MIGRATION +static void user4k_live_migration_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 1); +} + +static void user4k_migration_wait_test(struct kunit *test) +{ + test_live_fault_roundtrip(test, 2); +} +#endif + +/* Transfer PTE ownership to normal mm teardown, which also handles swap. */ +static int anon_slot_fixture_handoff(struct kunit *test, struct anon_slot_fixture *f) +{ + struct mm_struct *mm = f->mm; + + f->mm = NULL; + f->vma = NULL; + memset(f->slots, 0, sizeof(f->slots)); + memset(f->mapped, 0, sizeof(f->mapped)); + return kunit_add_action_or_reset(test, user4k_vma_mm_free, mm); +} + +static void test_nonlinear_swap(struct kunit *test, bool evict) +{ + unsigned int first; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (first = 0; first < 2; first++) { + struct anon_slot_fixture *a = anon_slot_fixture_create(test); + struct anon_slot_fixture *b = anon_slot_fixture_create(test); + struct vm_area_struct *vmas[2]; + struct mm_struct *child; + struct mm_subpage *slot; + struct folio *folio; + swp_entry_t entry = { }; + unsigned int side; + vm_fault_t fault; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + vmas[0] = a->vma; + vmas[1] = b->vma; + slot = mm_subpage_alloc_at(a->pool, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + folio = mm_subpage_folio(slot); + err = mem_cgroup_charge(folio, a->mm, GFP_KERNEL); + if (err) { + mm_subpage_put(slot); + KUNIT_FAIL(test, "cannot charge swap fixture: %d", err); + return; + } + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(a, 0, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + slot = mm_subpage_alloc_at(a->pool, 3 * SZ_4K); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, slot); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_map(b, 1, slot), 0); + folio_lock(folio); + mm_subpage_set_exclusive(slot); + folio_unlock(folio); + memset(folio_address(folio), 0x5b, SZ_4K); + memset(folio_address(folio) + 3 * SZ_4K, 0xd3, SZ_4K); + test_swap_tags_set(folio); + folio_mark_dirty(folio); + folio_add_lru(folio); + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, a), 0); + KUNIT_ASSERT_EQ(test, anon_slot_fixture_handoff(test, b), 0); + folio_get(folio); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + for (side = 0; side < 2; side++) { + pte_t pte = test_fault_entry(test, vmas[side]->vm_mm, + TEST_VA + side * SZ_4K); + + KUNIT_EXPECT_EQ(test, softleaf_from_pte(pte).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(pte), side ? 3U * SZ_4K : 0U); + KUNIT_EXPECT_EQ(test, swap_subpage_count(entry, side ? 3 * SZ_4K : 0), 1U); + } + if (evict) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(entry)); + /* Preserve physical offsets when a nonpresent nonlinear slot is forked. */ + child = test_vma_mm(test); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vmas[1]->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + for (side = 0; side < 2; side++) { + unsigned int which = side ^ first, physical = which ? 3 : 0; + struct vm_area_struct *vma = vmas[which]; + unsigned long address = TEST_VA + which * SZ_4K; + pte_t pte; + + mmap_read_lock(vma->vm_mm); + fault = handle_mm_fault(vma, address, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(vma->vm_mm); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), + (vm_fault_t)0, "evict=%d first=%u side=%u", evict, first, which); + KUNIT_EXPECT_EQ(test, !!(fault & VM_FAULT_MAJOR), evict && !side); + pte = test_fault_entry(test, vma->vm_mm, address); + slot = mm_subpage_get_from_phys(pte_phys_mm(vma->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), physical * (unsigned int)SZ_4K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), which ? 0xd3 : 0x5b, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(slot), physical)); + mm_subpage_put(slot); + } + /* A child's write must copy only its slot at the new virtual offset. */ + { + struct vm_area_struct *vma = test_vma_lookup(child, TEST_VA + SZ_4K); + pte_t pte; + + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(child); + fault = handle_mm_fault(vma, TEST_VA + SZ_4K, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(child); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + slot = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + memset(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x91, SZ_4K); + mm_subpage_put(slot); + pte = test_fault_entry(test, vmas[1]->vm_mm, TEST_VA + SZ_4K); + slot = mm_subpage_get_from_phys(pte_phys_mm(vmas[1]->vm_mm, pte)); + KUNIT_ASSERT_NOT_NULL(test, slot); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0xd3, SZ_4K), NULL); + mm_subpage_put(slot); + } + kunit_release_action(test, user4k_vma_mm_free, child); + kunit_release_action(test, user4k_vma_mm_free, vmas[1]->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, vmas[0]->vm_mm); + } +} + +static void user4k_nonlinear_swap_cached_test(struct kunit *test) +{ + test_nonlinear_swap(test, false); +} + +static void user4k_nonlinear_swap_disk_test(struct kunit *test) +{ + test_nonlinear_swap(test, true); +} + +#ifdef CONFIG_USERFAULTFD +/* The context is attached only during the call; ordinary mm cleanup owns the VMAs. */ +static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr, + unsigned long src_addr, unsigned long size) +{ + struct userfaultfd_ctx ctx = { .mm = dst->vm_mm }; + ssize_t ret; + + init_rwsem(&ctx.map_changing_lock); + atomic_set(&ctx.mmap_changing, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = &ctx; + mmap_write_unlock(ctx.mm); + ret = move_pages(&ctx, dst_addr, src_addr, size, 0); + mmap_write_lock(ctx.mm); + vma_start_write(dst); + dst->vm_userfaultfd_ctx.ctx = NULL; + mmap_write_unlock(ctx.mm); + return ret; +} + +struct move_migration_worker { + struct userfaultfd_ctx ctx; + struct vm_area_struct *vma[2]; + struct task_struct *task; + struct completion ready; + unsigned long address[2], size; + unsigned long moves, busy; + int error; +}; + +static int move_migration_worker_run(void *arg) +{ + struct move_migration_worker *w = arg; + + while (!kthread_should_stop()) { + unsigned int side = w->moves & 1; + ssize_t ret = move_pages(&w->ctx, w->address[side ^ 1], + w->address[side], w->size, 0); + + if (ret == w->size) + WRITE_ONCE(w->moves, w->moves + 1); + else if (ret == -EAGAIN || ret == -EBUSY) + w->busy++; + else + w->error = ret < 0 ? ret : -EIO; + if (w->moves || w->error) + complete_all(&w->ready); + if (w->error) { + /* Keep the task valid until the owner joins it. */ + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + __set_current_state(TASK_RUNNING); + } + /* Leave bounded gaps so migration can complete between contended moves. */ + if (ret == w->size) + usleep_range(1000, 2000); + else + cond_resched(); + } + return 0; +} + +static void move_migration_worker_stop(void *arg) +{ + struct move_migration_worker *w = arg; + unsigned int i; + + if (w->task) { + kthread_stop(w->task); + w->task = NULL; + } + mmap_write_lock(w->ctx.mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = NULL; + } + mmap_write_unlock(w->ctx.mm); +} + +static void user4k_move_migration_race_test(struct kunit *test) +{ + unsigned int shift; + + for (shift = 12; shift < PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct move_migration_worker *w; + unsigned long size = 1UL << shift, i, initial_moves; + unsigned int round, migrated = 0, failed = 0; + pte_t pte; + vm_fault_t fault; + + KUNIT_ASSERT_NOT_NULL(test, mm); + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, w); + w->ctx.mm = mm; + init_rwsem(&w->ctx.map_changing_lock); + atomic_set(&w->ctx.mmap_changing, 0); + init_completion(&w->ready); + w->size = size; + w->address[0] = TEST_VA + 3 * size; + w->address[1] = TEST_VA + SZ_128M + size; + w->vma[0] = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + w->vma[1] = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, w->vma[0]); + KUNIT_ASSERT_NOT_NULL(test, w->vma[1]); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(w->vma[0], TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + pte = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, pte)), 0x60 + i / size, size); + } + /* Both directions are registered before starting the concurrent mover. */ + mmap_write_lock(mm); + for (i = 0; i < 2; i++) { + vma_start_write(w->vma[i]); + w->vma[i]->vm_userfaultfd_ctx.ctx = &w->ctx; + } + mmap_write_unlock(mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, move_migration_worker_stop, w), 0); + w->task = kthread_create(move_migration_worker_run, w, "user4k-move"); + if (IS_ERR(w->task)) { + w->task = NULL; + KUNIT_FAIL(test, "MOVE worker creation failed"); + return; + } + if (num_online_cpus() > 1) + kthread_bind(w->task, cpumask_last(cpu_online_mask)); + wake_up_process(w->task); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&w->ready, 10 * HZ), 0UL); + initial_moves = READ_ONCE(w->moves); + lru_add_drain_all(); + for (round = 0; round < 64; round++) { + struct folio *folio; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + lru_add_drain_all(); + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE_MSG(test, isolated, "round=%u", round); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, + MIGRATE_SYNC, MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_EXPECT_GE(test, ret, 0); + migrated += succeeded; + failed += ret > 0; + cond_resched(); + } + kunit_release_action(test, move_migration_worker_stop, w); + kunit_info(test, "%uK MOVE/migration: moves=%lu retries=%lu migrated=%u busy=%u\n", + 1U << (shift - 10), w->moves, w->busy, migrated, failed); + KUNIT_EXPECT_EQ(test, w->error, 0); + KUNIT_EXPECT_GT(test, w->moves, initial_moves); + KUNIT_EXPECT_GT(test, migrated, 0U); + for (i = 0; i < PAGE_SIZE; i += size) { + unsigned long addr = i == 3 * size ? w->address[w->moves & 1] : TEST_VA + i; + + pte = test_fault_entry(test, mm, addr); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, pte)), + 0x60 + i / size, size), NULL); + } + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, w->address[(w->moves & 1) ^ 1]))); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} + +static void user4k_move_swap_get_test(struct kunit *test) +{ + unsigned int variant; + + if (!total_swap_pages) { + kunit_skip(test, "requires the disposable guest swap runner"); + return; + } + for (variant = 0; variant < PAGE_SHIFT - 12; variant++) { + unsigned int shift = 12 + (variant / 2) * 2; + bool forked = variant & 1; + unsigned long size = 1UL << shift; + unsigned long src_addr = TEST_VA + 3 * size; + unsigned long dst_addr = TEST_VA + SZ_128M + size; + struct mm_struct *mm = test_vma_mm_granule(test, shift); + struct vm_area_struct *src, *dst; + struct mm_subpage *held; + struct folio *folio; + phys_addr_t physical; + swp_entry_t entry = { }; + pte_t before, after; + vm_fault_t fault; + unsigned long i; + int err, mapped; + + KUNIT_ASSERT_NOT_NULL(test, mm); + src = test_fault_vma_in_mm(mm, TEST_VA, PAGE_SIZE); + dst = test_fault_vma_in_mm(mm, TEST_VA + SZ_128M, PAGE_SIZE); + KUNIT_ASSERT_NOT_NULL(test, src); + KUNIT_ASSERT_NOT_NULL(test, dst); + for (i = 0; i < PAGE_SIZE; i += size) { + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + before = test_fault_entry(test, mm, TEST_VA + i); + memset(phys_to_virt(pte_phys_mm(mm, before)), 0x50 + i / size, size); + } + before = test_fault_entry(test, mm, src_addr); + physical = pte_phys_mm(mm, before); + held = mm_subpage_get_from_phys(physical); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_subpage_put, held), 0); + KUNIT_ASSERT_EQ(test, mm_subpage_offset(held), (unsigned int)(3 * size)); + folio = mm_subpage_folio(held); + lru_add_drain_all(); + folio_lock(folio); + err = folio_alloc_swap(folio); + if (!err) { + entry = folio->swap; + try_to_unmap(folio, TTU_SYNC); + } + mapped = folio_mapcount(folio); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, mapped, 0); + KUNIT_ASSERT_EQ(test, mm_subpage_mapcount(held), 0); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + before = test_fault_entry(test, mm, src_addr); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(softleaf_from_pte(before))); + if (forked) { + struct mm_struct *child = test_vma_mm_granule(test, shift); + + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), + (ssize_t)-EBUSY); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(test_fault_entry(test, mm, + src_addr)).val, entry.val); + kunit_release_action(test, user4k_vma_mm_free, child); + before = test_fault_entry(test, mm, src_addr); + KUNIT_EXPECT_FALSE(test, pte_swp_exclusive(before)); + KUNIT_ASSERT_EQ(test, swap_subpage_count(entry, 3 * size), 1U); + } + KUNIT_ASSERT_EQ(test, test_uffd_move(dst, dst_addr, src_addr, size), (ssize_t)size); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_EXPECT_EQ(test, softleaf_from_pte(after).val, entry.val); + KUNIT_EXPECT_EQ(test, pte_swp_subpage_offset(after), (unsigned int)(3 * size)); + KUNIT_EXPECT_TRUE(test, pte_none(test_fault_entry(test, mm, src_addr))); + /* Restore with the retained metadata already rebound to the new root. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), 0x53, size), NULL); + KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(held), !forked); + /* Fork-cleared exclusivity requires COW while a GET is retained. */ + mmap_read_lock(mm); + fault = handle_mm_fault(dst, dst_addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, dst_addr); + KUNIT_ASSERT_TRUE(test, pte_write(after)); + if (forked) + KUNIT_EXPECT_NE(test, pte_phys_mm(mm, after), physical); + else + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after), physical); + memset(phys_to_virt(pte_phys_mm(mm, after)), 0xa9, size); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(physical), + forked ? 0x53 : 0xa9, size), NULL); + for (i = 0; i < PAGE_SIZE; i += size) { + if (i == 3 * size) + continue; + mmap_read_lock(mm); + fault = handle_mm_fault(src, TEST_VA + i, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), (vm_fault_t)0); + after = test_fault_entry(test, mm, TEST_VA + i); + KUNIT_ASSERT_TRUE(test, pte_present(after)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(mm, after)), + 0x50 + i / size, size), NULL); + } + kunit_release_action(test, test_subpage_put, held); + kunit_release_action(test, user4k_vma_mm_free, mm); + } +} +#endif + +static struct kunit_case user4k_swap_cases[] = { + KUNIT_CASE(user4k_swap_reuse_get_test), + KUNIT_CASE(user4k_swap_reuse_swapoff_test), + KUNIT_CASE(user4k_swap_reuse_cached_test), + KUNIT_CASE(user4k_swap_reuse_disk_test), + KUNIT_CASE(user4k_swap_exclusive_cached_test), + KUNIT_CASE(user4k_swap_exclusive_disk_test), + KUNIT_CASE(user4k_swap_exclusive_swapoff_test), + KUNIT_CASE(user4k_swap_cached_test), + KUNIT_CASE(user4k_swap_pin_test), + KUNIT_CASE(user4k_swap_mremap_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_swap_migration_test), +#endif + KUNIT_CASE(user4k_swap_disk_test), + KUNIT_CASE(user4k_swap_reclaim_test), + KUNIT_CASE(user4k_swapoff_test), + KUNIT_CASE(user4k_swap_stress_test), + KUNIT_CASE(user4k_swapoff_boundaries_test), + KUNIT_CASE(user4k_live_swap_fault_test), + {} +}; + +static void user4k_swap_suite_exit(struct kunit_suite *suite) +{ + kvfree_rcu_barrier(); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void test_shmem_uffd_swap(struct kunit *test, unsigned int mode) +{ + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, mode == 3 ? 2 : 1); + struct swap_iocb *plug = NULL; + struct file *file; + struct folio *folio; + void *entry; + bool *disabled; + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + file = test_cache_file(test, PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, mode == 3 ? 2 * SZ_4K : SZ_4K); + ret = shmem_uffd_set_missing(folio, 0, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + /* Match pageout(): submitted data is clean unless writeout redirties it. */ + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, list_empty(&folios)); + entry = filemap_get_entry(file->f_mapping, 0); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 0)), missing); + if (mode == 1) { + shmem_truncate_range(file_inode(file), 0, PAGE_SIZE - 1); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + } else if (mode == 3) { + shmem_truncate_range(file_inode(file), 0, SZ_4K - 1); + } else if (mode == 2) { + ret = test_swap_control("--swapoff-helper"); + *disabled = ret == 0; + KUNIT_ASSERT_EQ(test, ret, 0); + } + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), + mode == 1 ? 0UL : mode == 3 ? missing | BIT(0) : missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), + mode == 1 || mode == 3 ? 0 : 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, + mode == 3 ? 0xb6 : 0, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, + 0, PAGE_SIZE - 2 * SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_uffd_swapin_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 0); +} + +static void user4k_shmem_uffd_swapped_truncate_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 1); +} + +static void user4k_shmem_uffd_swapped_partial_hole_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 3); +} + +static void user4k_shmem_uffd_swapoff_test(struct kunit *test) +{ + test_shmem_uffd_swap(test, 2); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) +static void test_disable_failslab(void *unused) +{ + test_swap_control("--failslab-off-helper"); +} +#endif + +static void user4k_shmem_uffd_swap_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAULT_INJECTION_DEBUG_FS) + unsigned int attempt; + + if (!total_swap_pages) { + kunit_skip(test, "requires disposable guest swap"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--failslab-on-helper"), 0); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_disable_failslab, NULL), 0); + for (attempt = 1; attempt <= 2; attempt++) { + pgoff_t index = attempt == 1 ? 1 : 64; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct file *file = test_cache_file(test, (index + 1) * PAGE_SIZE, false); + struct swap_iocb *plug = NULL; + struct folio *folio; + unsigned int pending; + unsigned long pfn; + void *entry; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xb6, SZ_4K); + ret = shmem_uffd_set_missing(folio, index, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, index); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + pfn = folio_pfn(folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Fail this task's selected allocation and its allocator retries. */ + WRITE_ONCE(current->make_it_fail, 1); + WRITE_ONCE(current->fail_nth, attempt); + ret = shmem_writeout(folio, &plug, &folios); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + WRITE_ONCE(current->make_it_fail, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_EXPECT_EQ(test, ret, AOP_WRITEPAGE_ACTIVATE); + if (ret != AOP_WRITEPAGE_ACTIVATE) { + if (plug) + swap_write_unplug(plug); + folio_wait_writeback(folio); + folio_put(folio); + return; + } + KUNIT_EXPECT_TRUE(test, folio_test_locked(folio)); + KUNIT_EXPECT_TRUE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + KUNIT_EXPECT_PTR_EQ(test, folio->mapping, file->f_mapping); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + /* A failed multi-level insertion may retain an empty internal node. */ + { + unsigned long cursor = 0; + + KUNIT_EXPECT_PTR_EQ(test, xa_find(&SHMEM_I(file_inode(file))->uffd_missing, + &cursor, ULONG_MAX, XA_PRESENT), NULL); + } + KUNIT_EXPECT_PTR_EQ(test, plug, NULL); + KUNIT_EXPECT_TRUE(test, list_empty(&folios)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + /* No restart or cache repair: retry exactly the retained folio. */ + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + ret = shmem_writeout(folio, &plug, &folios); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + entry = filemap_get_entry(file->f_mapping, index); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + KUNIT_EXPECT_EQ(test, xa_to_value(xa_load(&SHMEM_I(file_inode(file))->uffd_missing, index)), missing); + ret = shmem_get_folio(file_inode(file), index, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, index), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xb6, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, PAGE_SIZE - SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + kunit_info(test, "failed metadata allocation %u at index %lu; same-folio retry and disk swap-in preserved bytes/mask", attempt, index); + } +#else + kunit_skip(test, "requires FAILSLAB and fault-injection debugfs"); +#endif +} + +#ifdef CONFIG_MEMCG +static void test_memcg_put(void *memcg) +{ + mem_cgroup_put(memcg); +} +#endif + +static void user4k_shmem_uffd_swap_charge_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg, *saved; + struct file *file; + struct folio *folio; + struct swap_iocb *plug = NULL; + void *entry; + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + int ret; + LIST_HEAD(folios); + + if (!total_swap_pages || mem_cgroup_disabled()) { + kunit_skip(test, "requires disposable guest swap and memcg"); + return; + } + KUNIT_ASSERT_EQ(test, test_swap_control("--memcg-setup-helper"), 0); + cgroup = cgroup_get_from_path("/uffd-metadata"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + file = test_cache_file(test, 65 * PAGE_SIZE, false); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + saved = set_active_memcg(memcg); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + set_active_memcg(saved); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + ret = shmem_uffd_set_missing(folio, 64, missing); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_lock_folio(file->f_mapping, 64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, folio_clear_dirty_for_io(folio)); + /* Reclaim from the root charge domain, not the folio's cgroup. */ + saved = set_active_memcg(root_mem_cgroup); + ret = shmem_writeout(folio, &plug, &folios); + set_active_memcg(saved); + if (plug) + swap_write_unplug(plug); + if (ret == AOP_WRITEPAGE_ACTIVATE) + folio_unlock(folio); + folio_wait_writeback(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + rcu_read_lock(); + entry = rcu_dereference(SHMEM_I(file_inode(file))->uffd_missing.xa_head); + if (xa_is_node(entry)) + KUNIT_EXPECT_PTR_EQ(test, mem_cgroup_from_virt(xa_to_node(entry)), memcg); + rcu_read_unlock(); + KUNIT_ASSERT_TRUE(test, xa_is_node(entry)); + entry = filemap_get_entry(file->f_mapping, 64); + if (entry && !xa_is_value(entry)) + folio_put(entry); + KUNIT_ASSERT_TRUE(test, xa_is_value(entry)); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(radix_to_swp_entry(entry))); + ret = shmem_get_folio(file_inode(file), 64, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 64), missing); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); +#else + kunit_skip(test, "requires memcg"); +#endif +} + +static bool test_coarse_swap_enabled(struct kunit *test) +{ + if (!test_coarse_file_enabled(test)) + return false; + if (!total_swap_pages || !IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { + kunit_skip(test, "requires disposable coarse-shmem swap runner"); + return false; + } + return true; +} + +static bool test_coarse_swap_out(struct kunit *test, struct file *file, pgoff_t index) +{ + unsigned int tries; + + for (tries = 0; tries < 100; tries++) { + struct folio *folio; + void *entry = filemap_get_entry(file->f_mapping, index); + LIST_HEAD(folios); + + if (xa_is_value(entry)) + return test_reclaim_swapcache(radix_to_swp_entry(entry)); + if (!entry) { + KUNIT_FAIL(test, "reclaim removed data without a swap entry"); + return false; + } + folio = entry; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + /* Real reclaim owns any folios produced by a large-folio split. */ + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + KUNIT_FAIL(test, "file index %lu did not reach swap", index); + return false; +} + +static void test_coarse_cold_shmem(struct kunit *test, bool promoted) +{ + struct file *file; + struct vm_area_struct *coarse, *small; + struct folio *folio; + unsigned int nr = SZ_64K / PAGE_SIZE, i; + unsigned long offset; + vm_fault_t fault; + + if (!test_coarse_swap_enabled(test)) + return; + file = test_cache_file(test, SZ_64K, true); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + coarse = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + small = test_file_vma(test, file, false, true, TEST_VA + SZ_4K, + SZ_64K - SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, coarse); + KUNIT_ASSERT_NOT_NULL(test, small); + if (promoted) { + KUNIT_ASSERT_EQ(test, test_file_fault(coarse, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + folio_put(folio); + } + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + lru_add_drain_all(); + for (i = 0; i < nr; i++) + KUNIT_ASSERT_TRUE(test, test_coarse_swap_out(test, file, i)); + /* XArray swap entries and absent swapcache prove this is a cold refault. */ + for (i = 0; i < nr; i++) { + void *entry = filemap_get_entry(file->f_mapping, i); + struct folio *cached = NULL; + + KUNIT_EXPECT_TRUE(test, xa_is_value(entry)); + if (entry && !xa_is_value(entry)) { + folio_put(entry); + return; + } + KUNIT_ASSERT_NOT_NULL(test, entry); + cached = swap_cache_get_folio(radix_to_swp_entry(entry)); + KUNIT_EXPECT_PTR_EQ(test, cached, NULL); + if (cached) + folio_put(cached); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, 0UL); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, (unsigned long)nr); + fault = test_file_fault(coarse, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + for (offset = 0; offset < SZ_64K; offset += SZ_4K) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + 0x31 + offset / SZ_4K, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, (unsigned long)nr); + KUNIT_EXPECT_EQ(test, SHMEM_I(file_inode(file))->swapped, 0UL); + for (offset = SZ_4K; offset < SZ_64K; offset += SZ_4K) { + pte_t entry; + + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA + offset, false) & + VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, small->vm_mm, TEST_VA + offset); + KUNIT_EXPECT_EQ(test, pte_phys_mm(small->vm_mm, entry), + page_to_phys(&folio->page) + offset); + } + kunit_release_action(test, user4k_vma_mm_free, coarse->vm_mm); + kunit_release_action(test, user4k_vma_mm_free, small->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), (int)nr + 1); +} + +static void user64k_shmem_cold_constituents_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, false); +} + +static void user64k_shmem_cold_promoted_test(struct kunit *test) +{ + test_coarse_cold_shmem(test, true); +} + +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) +static void test_coarse_injection_off(void *unused) +{ + test_swap_control("--coarse-injection-off"); +} + +#endif + +static void user64k_shmem_fault_alloc_fail_test(struct kunit *test) +{ +#if defined(CONFIG_FAILSLAB) && defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_MEMCG) + unsigned int mode; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_injection_off, NULL), 0); + for (mode = 0; mode < 3; mode++) { + struct file *file = test_cache_file(test, SZ_64K, false); + struct vm_area_struct *vma; + struct folio *folio; + phys_addr_t original[SZ_64K / PAGE_SIZE]; + unsigned char *buffer; + loff_t pos = 0; + unsigned int i, pending = 0; + vm_fault_t result; + bool prior_fail; + char operation[80]; + int enter = 0, leave = 0; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + /* Make the zero-limit charge case nonblocking and avoid an OOM kill. */ + if (mode == 2) + mapping_set_gfp_mask(file->f_mapping, + mapping_gfp_mask(file->f_mapping) & ~__GFP_DIRECT_RECLAIM); + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + memset(buffer, 0x95, SZ_64K); + KUNIT_ASSERT_EQ(test, kernel_write(file, buffer, SZ_64K, &pos), (ssize_t)SZ_64K); + vma = test_file_vma_granule(test, file, 16, true, TEST_VA, SZ_64K, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + original[i] = page_to_phys(&folio->page); + folio_put(folio); + } + if (mode < 2) { + KUNIT_ASSERT_EQ(test, test_swap_control(mode ? "--coarse-page-on" : + "--coarse-slab-on"), 0); + } else { + snprintf(operation, sizeof(operation), "--coarse-memcg-enter=%d", + task_pid_nr(current)); + enter = test_swap_control(operation); + } + /* Direct callback isolates backing allocation from page-table allocation. */ + if (!enter) { + struct vm_fault vmf = { .vma = vma, .address = TEST_VA, + .pgoff = 0, .flags = FAULT_FLAG_REMOTE }; + + mmap_read_lock(vma->vm_mm); + prior_fail = current->make_it_fail; + if (!mode) + WRITE_ONCE(current->fail_nth, 1); + else if (mode == 1) + current->make_it_fail = true; + result = vma->vm_ops->fault(&vmf); + pending = READ_ONCE(current->fail_nth); + WRITE_ONCE(current->fail_nth, 0); + current->make_it_fail = prior_fail; + mmap_read_unlock(vma->vm_mm); + /* Keep unexpected success safe for test cleanup. */ + if (vmf.page) { + if (result & VM_FAULT_LOCKED) + unlock_page(vmf.page); + put_page(vmf.page); + } + } else { + result = 0; + } + if (mode == 2) { + snprintf(operation, sizeof(operation), "--coarse-memcg-leave=%d", + task_pid_nr(current)); + leave = test_swap_control(operation); + } else { + leave = test_swap_control("--coarse-injection-off"); + } + KUNIT_ASSERT_EQ(test, enter, 0); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_EXPECT_EQ(test, pending, 0U); + KUNIT_ASSERT_TRUE(test, result & VM_FAULT_OOM); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + folio = filemap_get_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_order(folio), 0U); + KUNIT_EXPECT_EQ(test, page_to_phys(&folio->page), original[i]); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, PAGE_SIZE), NULL); + folio_put(folio); + } + KUNIT_EXPECT_EQ(test, file->f_mapping->nrpages, SZ_64K / PAGE_SIZE); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SHMEMPAGES), 0L); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & + (VM_FAULT_ERROR | VM_FAULT_RETRY | VM_FAULT_NOPAGE), 0U); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x95, SZ_64K), NULL); + folio_put(folio); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + kunit_info(test, "coarse allocation failure mode %u preserved data and recovered", mode); + } +#else + kunit_skip(test, "requires slab/page fault injection and memcg"); +#endif +} + +static void user64k_anon_swap_test(struct kunit *test) +{ + unsigned int mode, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + for (mode = 0; mode < 7; mode++) { + struct vm_area_struct *vma, *cvma = NULL; + struct mm_struct *child = NULL; + struct folio *folio, *held = NULL; + swp_entry_t entry, part; + unsigned int i, tries; + pte_t pte; + bool *disabled; + bool warm = !mode || mode == 6; + vm_fault_t fault; + int err; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + if (mode == 2 || mode == 6) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + } + if (warm) { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, min_order_for_split(folio), 16U - PAGE_SHIFT); +#endif + folio_lock(folio); + KUNIT_EXPECT_EQ(test, folio_anon_min_order(folio), 16U - PAGE_SHIFT); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_EXPECT_EQ(test, split_huge_page_to_list_to_order(&folio->page, NULL, 0), + -EBUSY); + KUNIT_EXPECT_EQ(test, folio_order(folio), 16U - PAGE_SHIFT); + expect_coarse_anon_state(test, folio, child ? 2 : 1, !child); +#endif + err = folio_alloc_swap(folio); + if (!err) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, err, 0); + } else { + for (tries = 0; tries < 100; tries++) { + LIST_HEAD(folios); + + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + folio_get(folio); + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); + } + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE_MSG(test, pte_present(pte), "mode=%u", mode); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(swp_offset(entry), nr)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 1L); + if (mode == 3) { + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + err = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, err, 0); + } + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), child ? 2 : 1); + if (!warm) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + if (mode == 3) { + /* Exit a swapped child: every constituent count must be released. */ + kunit_release_action(test, user4k_vma_mm_free, child); + child = NULL; + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } + } + if (mode == 5) { +#if defined(CONFIG_FAIL_PAGE_ALLOC) && defined(CONFIG_FAULT_INJECTION) + bool old = current->make_it_fail; + + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-page-on"), 0); + current->make_it_fail = true; + fault = test_file_fault(vma, TEST_VA, false); + current->make_it_fail = old; + err = test_swap_control("--coarse-injection-off"); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_OOM); + KUNIT_EXPECT_TRUE(test, pte_same(test_fault_entry(test, vma->vm_mm, TEST_VA), pte)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + } +#endif + } + if (mode == 4) { + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + err = test_swap_control("--swapoff-helper"); + if (!err) + *disabled = true; + KUNIT_ASSERT_EQ(test, err, 0); + } else { + fault = test_file_fault(vma, TEST_VA + SZ_64K - 1, false); + KUNIT_ASSERT_EQ_MSG(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U, + "mode=%u fault=%x", mode, fault); + if (!warm && mode != 5) + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + } + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE_MSG(test, pte_present(pte), "mode=%u", mode); + folio = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(vma->vm_mm, pte)))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + if (held) { + KUNIT_EXPECT_PTR_EQ(test, folio, held); /* Warm aligned cache: no copy. */ + } else { + folio_get(folio); + held = folio; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, held), 0); + } + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + if (child) { + struct folio *copied; + + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + pte = test_fault_entry(test, child, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), page_to_phys(&folio->page)); + KUNIT_EXPECT_TRUE(test, pte_write(pte)); + copied = page_folio(pfn_to_page(PHYS_PFN(pte_phys_mm(child, pte)))); + expect_coarse_anon_state(test, copied, 1, true); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + memset(folio_address(copied), 0xa7, SZ_64K); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x31 + i, SZ_4K), NULL); + kunit_release_action(test, user4k_vma_mm_free, child); + } + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + if (held) { + KUNIT_EXPECT_EQ(test, folio_mapcount(held), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(held), 1); + } + for (i = 0; i < nr; i++) { + struct swap_info_struct *si; + + part = entry; + part.val += i; + si = get_swap_device(part); + if (si) { + KUNIT_EXPECT_FALSE(test, swap_entry_swapped(si, part)); + put_swap_device(si); + } else { + KUNIT_EXPECT_EQ(test, mode, 4U); + } + } + if (mode == 4) + kunit_release_action(test, test_reenable_swap, disabled); + kunit_info(test, "completed coarse anonymous swap mode %u", mode); + } +} + +/* The caller's private fixture has no concurrent PTE writers during reclaim. */ +static void test_coarse_anon_reclaim_once(struct vm_area_struct *vma) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, vma->vm_start); + LIST_HEAD(folios); + + if (!folio) + return; + lru_add_drain_all(); + if (folio_isolate_lru(folio)) + list_add(&folio->lru, &folios); + folio_put(folio); + reclaim_pages(&folios); + schedule_timeout_uninterruptible(1); +} + +static void user64k_anon_swap_limit_test(struct kunit *test) +{ +#ifdef CONFIG_MEMCG + struct vm_area_struct *vma; + struct cgroup *cgroup; + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg; + struct folio *folio; + swp_entry_t entry, part; + phys_addr_t original; + unsigned int i, tries, nr = SZ_64K / PAGE_SIZE; + long before, denied; + char operation[80]; + vm_fault_t fault; + pte_t pte; + int ret, leave; + + if (!test_coarse_swap_enabled(test)) + return; + if (mem_cgroup_disabled()) { + kunit_skip(test, "requires memory cgroups"); + return; + } + snprintf(operation, sizeof(operation), "--coarse-swap-enter=%d", task_pid_nr(current)); + KUNIT_ASSERT_EQ(test, test_swap_control(operation), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + fault = vma ? test_file_fault(vma, TEST_VA, true) : VM_FAULT_OOM; + snprintf(operation, sizeof(operation), "--coarse-swap-leave=%d", task_pid_nr(current)); + leave = test_swap_control(operation); + KUNIT_ASSERT_EQ(test, leave, 0); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, fault & VM_FAULT_ERROR, 0U); + cgroup = cgroup_get_from_path("/coarse-swap-limit"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cgroup); + css = cgroup_get_e_css(cgroup, &memory_cgrp_subsys); + cgroup_put(cgroup); + KUNIT_ASSERT_NOT_NULL(test, css); + memcg = mem_cgroup_from_css(css); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_memcg_put, memcg), 0); + KUNIT_ASSERT_PTR_NE(test, memcg, root_mem_cgroup); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + rcu_read_lock(); + KUNIT_EXPECT_PTR_EQ(test, folio_memcg(folio), memcg); + rcu_read_unlock(); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xc9, SZ_64K); + before = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + denied = atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]); + KUNIT_EXPECT_GT(test, denied, before); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + /* Exercise vmscan's failed-allocation/split fallback without a retained GET. */ + for (tries = 0; tries < 8; tries++) + test_coarse_anon_reclaim_once(vma); + KUNIT_EXPECT_GT(test, atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]), denied); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-swap-unlimit"), 0); + for (tries = 0; tries < 100; tries++) { + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + if (!pte_present(pte)) + break; + } + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), (unsigned long)nr); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xc9, SZ_64K), NULL); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + KUNIT_EXPECT_EQ(test, page_counter_read(&memcg->swap), 0UL); +#else + kunit_skip(test, "requires memory cgroups"); +#endif +} + +struct coarse_swap_worker { + struct mm_struct *mm; + struct task_struct *task; + struct completion ready, go, entered, done; + u8 *buffer; + unsigned int id; + bool *disabled; + int error; +}; + +static int coarse_swap_worker_run(void *arg) +{ + struct coarse_swap_worker *w = arg; + unsigned int i; + u8 value = 0x91 + w->id; + + complete(&w->ready); + if (!wait_for_completion_timeout(&w->go, 15 * HZ) || kthread_should_stop()) { + w->error = -ETIMEDOUT; + goto out; + } + if (!w->mm) { + complete(&w->entered); + w->error = test_swap_control("--swapoff-helper"); + if (!w->error) + WRITE_ONCE(*w->disabled, true); + goto out; + } + kthread_use_mm(w->mm); + complete(&w->entered); + if (copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 1; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + if (w->buffer[i] != (u8)(0x31 + i / SZ_4K)) { + w->error = 2; + goto unuse; + } + } + if (copy_to_user((void __user *)(TEST_VA + SZ_64K - 1), &value, 1) || + copy_from_user(w->buffer, (void __user *)TEST_VA, SZ_64K)) { + w->error = 3; + goto unuse; + } + for (i = 0; i < SZ_64K; i++) { + u8 expected = i == SZ_64K - 1 ? value : 0x31 + i / SZ_4K; + + if (w->buffer[i] != expected) { + w->error = 4; + break; + } + } +unuse: + kthread_unuse_mm(w->mm); +out: + complete(&w->done); + while (!kthread_should_stop()) + schedule_timeout_interruptible(1); + return 0; +} + +static void coarse_swap_worker_stop(void *arg) +{ + struct coarse_swap_worker *w = arg; + + if (w->task) { + complete_all(&w->go); + kthread_stop(w->task); + w->task = NULL; + } +} + +static void user64k_anon_swap_race_test(struct kunit *test) +{ + unsigned int round, nr = SZ_64K / PAGE_SIZE; + + if (!test_coarse_swap_enabled(test)) + return; + if (num_online_cpus() < 2 || !system_supports_64kb_granule()) { + kunit_skip(test, "requires two CPUs and 64K TTBR0 support"); + return; + } + for (round = 0; round < 8; round++) { + struct vm_area_struct *vma; + struct mm_struct *child; + struct coarse_swap_worker *workers; + struct folio *folio, *blocker; + struct swap_info_struct *si; + swp_entry_t entry, part; + bool *disabled, started = true, blocked; + bool off_started = false, cold = round & 1; + unsigned int i, j, tries; + pte_t pte; + int ret, cpu; + + disabled = kunit_kzalloc(test, sizeof(*disabled), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, disabled); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_reenable_swap, disabled), 0); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x31 + i, SZ_4K); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, vma->vm_mm); + uprobe_end_dup_mmap(); + if (!ret) { + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + } + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 2); + if (cold) + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + part = entry; + part.val += nr - 1; + blocker = read_swap_cache_async(part, GFP_KERNEL, vma, + TEST_VA + SZ_64K - PAGE_SIZE, NULL); + KUNIT_ASSERT_NOT_NULL(test, blocker); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, blocker), 0); + KUNIT_EXPECT_EQ(test, folio_size(blocker), cold ? PAGE_SIZE : (unsigned long)SZ_64K); + workers = kunit_kcalloc(test, 3, sizeof(*workers), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers); + cpu = cpumask_first(cpu_online_mask); + for (i = 0; i < 3; i++) { + workers[i].mm = i < 2 ? (i ? child : vma->vm_mm) : NULL; + workers[i].id = i; + workers[i].disabled = disabled; + workers[i].buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, workers[i].buffer); + init_completion(&workers[i].ready); + init_completion(&workers[i].go); + init_completion(&workers[i].entered); + init_completion(&workers[i].done); + workers[i].task = kthread_create(coarse_swap_worker_run, &workers[i], + "user64k-swap-%u", i); + if (IS_ERR(workers[i].task)) { + workers[i].task = NULL; + KUNIT_FAIL(test, "coarse swap worker creation failed"); + return; + } + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + coarse_swap_worker_stop, &workers[i]), 0); + if (i < 2) { + kthread_bind(workers[i].task, cpu); + cpu = cpumask_next(cpu, cpu_online_mask); + } + wake_up_process(workers[i].task); + } + for (i = 0; i < 3; i++) + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); + folio_lock(blocker); + for (i = 0; i < 3; i++) + complete(&workers[i].go); + for (i = 0; i < 3; i++) + started &= !!wait_for_completion_timeout(&workers[i].entered, HZ); + for (tries = 0; tries < 3 * HZ; tries++) { + si = get_swap_device(entry); + if (si) { + off_started = !(READ_ONCE(si->flags) & SWP_WRITEOK); + put_swap_device(si); + } + if (off_started) + break; + schedule_timeout_uninterruptible(1); + } + blocked = !completion_done(&workers[0].done) && + !completion_done(&workers[1].done) && !completion_done(&workers[2].done); + /* Never execute a fatal assertion while retaining the blocking lock. */ + folio_unlock(blocker); + KUNIT_EXPECT_TRUE(test, started); + KUNIT_EXPECT_TRUE(test, off_started); + KUNIT_EXPECT_TRUE(test, blocked); + for (i = 0; i < 3; i++) { + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].done, 20 * HZ), 0UL); + KUNIT_EXPECT_EQ_MSG(test, workers[i].error, 0, "round=%u worker=%u", round, i); + kunit_release_action(test, coarse_swap_worker_stop, &workers[i]); + } + KUNIT_EXPECT_TRUE(test, *disabled); + for (i = 0; i < 2; i++) { + struct mm_struct *mm = i ? child : vma->vm_mm; + + folio = test_fault_folio(mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (j = 0; j < SZ_64K; j++) { + u8 expected = j == SZ_64K - 1 ? 0x91 + i : 0x31 + j / SZ_4K; + + if (((u8 *)folio_address(folio))[j] != expected) { + KUNIT_FAIL(test, "round=%u side=%u byte=%u", round, i, j); + break; + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_SWAPENTS), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1L); + kunit_release_action(test, user4k_vma_mm_free, mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), folio == blocker ? 2 : 1); + kunit_release_action(test, test_folio_put, folio); + } + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(blocker)); + KUNIT_EXPECT_EQ(test, folio_ref_count(blocker), 1); + kunit_release_action(test, test_folio_put, blocker); + kunit_release_action(test, test_reenable_swap, disabled); + KUNIT_ASSERT_GT(test, total_swap_pages, 0UL); + } + kunit_info(test, "8 controlled warm/cold fault+COW+swapoff rounds completed"); +} + +struct coarse_swap_reservation { + struct folio **folios; + unsigned int count; +}; + +static void test_coarse_release_reservation(struct folio *folio) +{ + folio_lock(folio); + folio_put_swap(folio, NULL); + folio_free_swap(folio); + folio_unlock(folio); + folio_put(folio); +} + +static void test_coarse_release_reservations(void *arg) +{ + struct coarse_swap_reservation *r = arg; + unsigned int i; + + for (i = 0; i < r->count; i++) { + if (!r->folios[i]) + continue; + test_coarse_release_reservation(r->folios[i]); + r->folios[i] = NULL; + cond_resched(); + } +} + +static void test_coarse_swap_refusal(struct kunit *test, struct vm_area_struct *vma, + phys_addr_t original) +{ + struct folio *folio = test_fault_folio(vma->vm_mm, TEST_VA); + pte_t pte; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + KUNIT_EXPECT_FALSE(test, folio_test_swapcache(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + for (i = 0; i < 4; i++) + test_coarse_anon_reclaim_once(vma); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_TRUE(test, pte_present(pte)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, pte), original); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + folio_put(folio); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_SWAPENTS), 0L); +} + +static void user64k_anon_swap_fragment_test(struct kunit *test) +{ + struct coarse_swap_reservation *r; + struct vm_area_struct *vma; + struct folio *folio; + unsigned int i, nr = SZ_64K / PAGE_SIZE, released = 0; + unsigned long hole, free_before; + phys_addr_t original; + swp_entry_t entry, part; + vm_fault_t fault; + pte_t pte; + int ret = 0; + + if (!test_coarse_swap_enabled(test)) + return; + /* Exhaust only the bounded disposable disk owned by this test's PID1. */ + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + KUNIT_ASSERT_LT(test, total_swap_pages, (unsigned long)(SZ_256M / PAGE_SIZE)); + KUNIT_ASSERT_GT(test, total_swap_pages, 2UL * SWAPFILE_CLUSTER); + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + original = page_to_phys(&folio->page); + memset(folio_address(folio), 0xd6, SZ_64K); + folio_put(folio); + r = kunit_kzalloc(test, sizeof(*r), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r); + r->folios = kunit_kcalloc(test, total_swap_pages + 1, sizeof(*r->folios), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r->folios); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_release_reservations, r), 0); + /* Each cache-only folio has one explicit slot owner, so allocation cannot reclaim it. */ + for (i = 0; i <= total_swap_pages; i++) { + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + KUNIT_ASSERT_NOT_NULL(test, folio); + ret = mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL); + if (ret) { + folio_put(folio); + KUNIT_FAIL(test, "reservation memory charge failed"); + return; + } + __folio_set_swapbacked(folio); + __folio_mark_uptodate(folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) { + ret = folio_dup_swap(folio, NULL); + if (ret) + folio_free_swap(folio); + } + folio_unlock(folio); + if (ret) { + folio_put(folio); + break; + } + r->folios[r->count++] = folio; + cond_resched(); + } + KUNIT_ASSERT_EQ(test, ret, -ENOMEM); + KUNIT_ASSERT_EQ(test, atomic_long_read(&nr_swap_pages), 0L); + test_coarse_swap_refusal(test, vma, original); + /* Retain exactly one native entry in each naturally aligned coarse group. */ + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (swp_offset(folio->swap) % nr == 1) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + } + free_before = atomic_long_read(&nr_swap_pages); + KUNIT_EXPECT_GT(test, free_before, total_swap_pages / 2); + KUNIT_EXPECT_LT(test, free_before, total_swap_pages); + test_coarse_swap_refusal(test, vma, original); + /* Free one aligned group near the end, without freeing a whole native-order cluster. */ + hole = round_down(total_swap_pages + 1, (unsigned long)SWAPFILE_CLUSTER) - nr; + for (i = 0; i < r->count; i++) { + folio = r->folios[i]; + if (!folio || swp_offset(folio->swap) < hole || swp_offset(folio->swap) >= hole + nr) + continue; + test_coarse_release_reservation(folio); + r->folios[i] = NULL; + released++; + } + KUNIT_ASSERT_EQ(test, released, 1U); + KUNIT_EXPECT_EQ(test, atomic_long_read(&nr_swap_pages), (long)free_before + 1); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "a complete aligned group is free in an order-0 cluster"); + pte = test_fault_entry(test, vma->vm_mm, TEST_VA); + KUNIT_ASSERT_FALSE(test, pte_present(pte)); + entry = softleaf_from_pte(pte); + KUNIT_ASSERT_TRUE(test, softleaf_is_swap(entry)); + KUNIT_EXPECT_EQ(test, swp_offset(entry), hole); + for (i = 0; i < r->count; i++) { + if (r->folios[i]) + KUNIT_EXPECT_EQ(test, swp_swapcount(r->folios[i]->swap), 1); + } + kunit_release_action(test, test_coarse_release_reservations, r); + for (i = 0; i < nr; i++) { + part = entry; + part.val += i; + KUNIT_EXPECT_EQ(test, swp_swapcount(part), 1); + KUNIT_ASSERT_TRUE(test, test_reclaim_swapcache(part)); + } + fault = test_file_fault(vma, TEST_VA, false); + KUNIT_ASSERT_EQ(test, fault & (VM_FAULT_ERROR | VM_FAULT_RETRY), 0U); + KUNIT_EXPECT_TRUE(test, fault & VM_FAULT_MAJOR); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xd6, SZ_64K), NULL); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "exhaustion, scattered free space and one-group recovery checked"); +} + +struct coarse_swap_io_mode { + struct swap_info_struct *si; + bool synchronous; +}; + +static void test_coarse_restore_io_mode(void *arg) +{ + struct coarse_swap_io_mode *mode = arg; + + spin_lock(&mode->si->lock); + if (mode->synchronous) + mode->si->flags |= SWP_SYNCHRONOUS_IO; + else + mode->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&mode->si->lock); + put_swap_device(mode->si); +} + +static void user64k_anon_swap_extent_test(struct kunit *test) +{ + unsigned int mode, i; + + if (!test_coarse_swap_enabled(test)) + return; + KUNIT_ASSERT_EQ(test, test_swap_control("--coarse-fixture-check"), 0); + for (mode = 0; mode < 2; mode++) { + struct vm_area_struct *vma; + struct coarse_swap_io_mode *saved; + struct folio *folio; + unsigned long nr = SZ_64K / PAGE_SIZE, run = nr; + swp_entry_t entry; + sector_t first, next; + bool file; + int ret; + + vma = test_fault_vma_granule(test, 16, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + folio = test_fault_folio(vma->vm_mm, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, folio); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); + for (i = 0; i < SZ_64K / SZ_4K; i++) + memset(folio_address(folio) + i * SZ_4K, 0x63 + i, SZ_4K); + folio_lock(folio); + ret = folio_alloc_swap(folio); + if (!ret) + try_to_unmap(folio, TTU_SYNC); + folio_unlock(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, folio_mapcount(folio), 0); + entry = folio->swap; + saved = kunit_kzalloc(test, sizeof(*saved), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, saved); + saved->si = get_swap_device(entry); + KUNIT_ASSERT_NOT_NULL(test, saved->si); + saved->synchronous = saved->si->flags & SWP_SYNCHRONOUS_IO; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_coarse_restore_io_mode, saved), 0); + file = !(saved->si->flags & SWP_BLKDEV); + KUNIT_ASSERT_FALSE(test, saved->si->flags & SWP_FS_OPS); + first = swap_extent_sector(entry, &run); + if (file) { + unsigned long remaining; + swp_entry_t part = entry; + + KUNIT_ASSERT_LT_MSG(test, run, nr, "fragmented fixture must cross an extent"); + part.val += run; + remaining = nr - run; + next = swap_extent_sector(part, &remaining); + KUNIT_EXPECT_NE(test, next, first + (run << (PAGE_SHIFT - 9))); + } else { + KUNIT_EXPECT_EQ(test, run, nr); + } + /* The verified disposable fixture owns this otherwise idle swap area. */ + spin_lock(&saved->si->lock); + if (mode) + saved->si->flags |= SWP_SYNCHRONOUS_IO; + else + saved->si->flags &= ~SWP_SYNCHRONOUS_IO; + spin_unlock(&saved->si->lock); + folio_lock(folio); + folio_clear_dirty_for_io(folio); + __swap_writepage(folio, NULL); + folio_wait_writeback(folio); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + /* A full-folio read must reconstruct all physical runs, not warm cache. */ + memset(folio_address(folio), 0, SZ_64K); + folio_clear_uptodate(folio); + swap_read_folio(folio, NULL); + folio_lock(folio); + KUNIT_EXPECT_TRUE(test, folio_test_uptodate(folio)); + for (i = 0; i < SZ_64K / SZ_4K; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + i * SZ_4K, + 0x63 + i, SZ_4K), NULL); + folio_unlock(folio); + kunit_release_action(test, test_coarse_restore_io_mode, saved); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + kunit_release_action(test, user4k_vma_mm_free, vma->vm_mm); + lru_add_drain_all(); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), 0); + KUNIT_EXPECT_EQ(test, folio_ref_count(folio), 1); + kunit_info(test, "64K full-folio %s I/O: file=%u first-run=%lu/%lu native pages", + mode ? "synchronous" : "asynchronous", file, run, nr); + } +} + +static struct kunit_case user4k_coarse_shmem_cases[] = { + KUNIT_CASE(user64k_anon_swap_extent_test), + KUNIT_CASE(user64k_anon_swap_test), + KUNIT_CASE(user64k_anon_swap_limit_test), + KUNIT_CASE(user64k_anon_swap_fragment_test), + KUNIT_CASE(user64k_anon_swap_race_test), + KUNIT_CASE(user64k_shmem_cold_constituents_test), + KUNIT_CASE(user64k_shmem_cold_promoted_test), + KUNIT_CASE(user64k_shmem_fault_alloc_fail_test), + {} +}; + +static struct kunit_suite user4k_coarse_shmem_suite = { + .name = "arm64-user4k-coarse-shmem", + .test_cases = user4k_coarse_shmem_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_case user4k_shmem_swap_cases[] = { + KUNIT_CASE(user4k_shmem_uffd_swap_alloc_fail_test), + KUNIT_CASE(user4k_shmem_uffd_swap_charge_test), + KUNIT_CASE(user4k_shmem_uffd_swapin_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_truncate_test), + KUNIT_CASE(user4k_shmem_uffd_swapoff_test), + KUNIT_CASE(user4k_shmem_uffd_swapped_partial_hole_test), + {} +}; + +static struct kunit_suite user4k_shmem_swap_suite = { + .name = "arm64-user4k-shmem-swap", + .test_cases = user4k_shmem_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; +#endif + +static struct kunit_case user4k_nonlinear_swap_cases[] = { +#ifdef CONFIG_USERFAULTFD + KUNIT_CASE(user4k_move_swap_get_test), + KUNIT_CASE(user4k_move_migration_race_test), +#endif + KUNIT_CASE(user4k_nonlinear_swap_cached_test), + KUNIT_CASE(user4k_nonlinear_swap_disk_test), + {} +}; + +static struct kunit_suite user4k_nonlinear_swap_suite = { + .name = "arm64-user4k-nonlinear-swap", + .test_cases = user4k_nonlinear_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static struct kunit_suite user4k_swap_suite = { + .name = "arm64-user4k-swap", + .test_cases = user4k_swap_cases, + .suite_exit = user4k_swap_suite_exit, +}; + +static void user4k_thp_eligibility_test(struct kunit *test) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const enum tva_type types[] = { + TVA_PAGEFAULT, TVA_SMAPS, TVA_KHUGEPAGED, TVA_FORCED_COLLAPSE, + }; + struct vm_area_struct *vma, *normal; + vm_flags_t flags = VM_READ | VM_WRITE | VM_MAYREAD | VM_MAYWRITE | + VM_HUGEPAGE; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + vma = test_vma_add(small, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + normal = test_vma_add(native, SZ_1G, SZ_1G, (struct vm_page_offset) {}, flags); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, normal); + /* Neither user advice nor forced collapse can bypass geometry limits. */ + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, false)); + KUNIT_EXPECT_TRUE(test, vma_thp_disabled(vma, flags, true)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, false)); + KUNIT_EXPECT_FALSE(test, vma_thp_disabled(normal, flags, true)); + for (unsigned int i = 0; i < ARRAY_SIZE(types); i++) + KUNIT_EXPECT_EQ(test, __thp_vma_allowable_orders(vma, flags, + types[i], THP_ORDERS_ALL_ANON), 0UL); +#else + kunit_skip(test, "requires transparent hugepage support"); +#endif +} + +static void user4k_aslr_window_test(struct kunit *test) +{ + struct mm_struct *small = test_vma_mm(test), *native = mm_alloc(); + const unsigned long windows[] = { SZ_8M, SZ_32M, SZ_1G }; + + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, native), 0); + KUNIT_ASSERT_NOT_NULL(test, small); + for (unsigned int i = 0; i < ARRAY_SIZE(windows); i++) { + unsigned long mask = (windows[i] >> PAGE_SHIFT) - 1; + unsigned long converted = mm_aslr_mask(small, mask); + + KUNIT_EXPECT_EQ(test, mm_aslr_mask(native, mask), mask); + KUNIT_EXPECT_EQ(test, (converted + 1) << mm_page_shift(small), + windows[i]); + KUNIT_EXPECT_EQ(test, converted & (TEST_SLOTS - 1), + (unsigned long)TEST_SLOTS - 1); + } +} + +static void user4k_exec_granule_policy_test(struct kunit *test) +{ + /* Model all native granules, including AArch32 with NO 4K support. */ + const unsigned int native[] = { 12, 14, 16 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(native); i++) { + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, false, native[i], 12), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(true, true, native[i], 0), + 12U); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 0), + native[i]); + KUNIT_EXPECT_EQ(test, arm64_exec_page_shift(false, true, native[i], 12), + 12U); + } +} + +static void user_granule_zero_region_test(struct kunit *test) +{ + unsigned long base_pfn = zero_pfn(0); + unsigned int shift, i; + u8 *buffer; + + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(PFN_PHYS(base_pfn), SZ_64K)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn - 1)); + KUNIT_EXPECT_FALSE(test, is_zero_pfn(base_pfn + SZ_64K / PAGE_SIZE)); + for (i = 0; i < SZ_64K / PAGE_SIZE; i++) { + KUNIT_EXPECT_TRUE(test, is_zero_pfn(base_pfn + i)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(pfn_to_page(base_pfn + i)), + 0, PAGE_SIZE), NULL); + } + buffer = kunit_kmalloc(test, SZ_64K, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, buffer); + for (shift = 12; shift <= 16; shift += 2) { + struct vm_area_struct *vma; + struct mm_struct *mm; + unsigned long size = 1UL << shift, left; + vm_fault_t ret; + spinlock_t *ptl; + pte_t *pte, entry; + + if ((shift == 12 && !system_supports_4kb_granule()) || + (shift == 14 && !system_supports_16kb_granule()) || + (shift == 16 && !system_supports_64kb_granule())) { + kunit_info(test, "hardware zero mapping skipped for shift %u", shift); + continue; + } + /* Internal coarse mm: the public opt-in remains disabled. */ + mm = test_vma_mm_granule(test, shift); + KUNIT_ASSERT_NOT_NULL(test, mm); + vma = test_fault_vma_in_mm(mm, TEST_VA, size); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_read_lock(mm); + ret = handle_mm_fault(vma, TEST_VA + size - 1, FAULT_FLAG_REMOTE, NULL); + mmap_read_unlock(mm); + KUNIT_ASSERT_EQ(test, ret, (vm_fault_t)0); + pte = test_mm_lookup(mm, TEST_VA, &ptl); + KUNIT_ASSERT_NOT_NULL(test, pte); + entry = ptep_get(pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_TRUE(test, pte_present(entry)); + KUNIT_EXPECT_TRUE(test, pte_special(entry)); + KUNIT_EXPECT_FALSE(test, pte_write(entry)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, entry), PFN_PHYS(base_pfn)); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + + memset(buffer, 0xa5, size); + kthread_use_mm(mm); + left = copy_from_user(buffer, (void __user *)TEST_VA, size); + kthread_unuse_mm(mm); + KUNIT_EXPECT_EQ(test, left, 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(buffer, 0, size), NULL); + mmap_read_lock(mm); + zap_vma_range(vma, TEST_VA, size); + mmap_read_unlock(mm); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 0L); + } +} + +#ifdef CONFIG_COREDUMP +static void user_granule_core_offsets_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + struct vm_area_struct *vma = kunit_kzalloc(test, sizeof(*vma), GFP_KERNEL); + unsigned int shift; + + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_ASSERT_NOT_NULL(test, vma); + vma->vm_mm = mm; + for (shift = 12; shift <= 16; shift += 2) { + unsigned long bytes = (1UL << 55) + 3 * SZ_64K + (1UL << shift); + + mm->context.flags = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + vma_set_page_offset(vma, (struct vm_page_offset) { + .index = bytes >> PAGE_SHIFT, .offset = bytes & ~PAGE_MASK, + }); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), bytes >> shift); + KUNIT_EXPECT_EQ(test, mm_user_fragment_size(mm), min(1UL << shift, PAGE_SIZE)); + vma_set_page_offset(vma, (struct vm_page_offset) {}); + KUNIT_EXPECT_EQ(test, core_vma_pgoff(vma), 0UL); + } +} +#endif + +static void user64k_fragment_ownership_test(struct kunit *test) +{ + struct user_page_fragment fragments[SZ_64K / PAGE_SIZE] = {}; + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + struct folio *original, *copied; + unsigned int i, nr = ARRAY_SIZE(fragments); + long count; + int ret, refs; + pte_t entry; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires a user leaf larger than native pages"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, false) & VM_FAULT_ERROR, 0U); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + unsigned int bytes = PAGE_SIZE - (i == 0 || i == nr - 1 ? 17 : 0); + + KUNIT_EXPECT_TRUE(test, is_zero_folio(f->folio)); + KUNIT_EXPECT_EQ(test, folio_pfn(f->folio), zero_pfn(TEST_VA) + i); + KUNIT_EXPECT_PTR_EQ(test, f->subpage, NULL); + KUNIT_EXPECT_EQ(test, f->offset, i == 0 ? 17UL : 0UL); + KUNIT_EXPECT_EQ(test, f->length, bytes); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); + entry = test_fault_entry(test, parent, TEST_VA); + original = page_folio(pfn_to_page(pte_pfn(entry))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(original, i)), 0x51 + i, PAGE_SIZE); + lru_add_drain_all(); + refs = folio_ref_count(original); + count = get_user_fragments_remote(parent, TEST_VA + 17, SZ_64K - 34, + 0, fragments, nr); + if (count > 0) { + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs + (int)count); + for (i = 0; i < count; i++) { + struct user_page_fragment *f = &fragments[i]; + + KUNIT_EXPECT_PTR_EQ(test, f->folio, original); + KUNIT_EXPECT_EQ(test, f->offset, i * PAGE_SIZE + (i == 0 ? 17UL : 0UL)); + KUNIT_EXPECT_LE(test, offset_in_page(f->offset) + f->length, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, + 0x51 + i, f->length), NULL); + } + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + KUNIT_EXPECT_EQ(test, folio_ref_count(original), refs); + + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + /* A read pin after fork must unshare the complete coarse leaf. */ + count = pin_user_fragments_remote(child, TEST_VA, SZ_64K, 0, fragments, nr); + if (count > 0) { + copied = fragments[0].folio; + KUNIT_EXPECT_PTR_NE(test, copied, original); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(copied)); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, child, TEST_VA))); + for (i = 0; i < count; i++) { + KUNIT_EXPECT_PTR_EQ(test, fragments[i].folio, copied); + KUNIT_EXPECT_PTR_EQ(test, fragments[i].subpage, NULL); + KUNIT_EXPECT_TRUE(test, fragments[i].pinned); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(copied, i))); + KUNIT_EXPECT_EQ(test, fragments[i].offset, i * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, fragments[i].length, (unsigned int)PAGE_SIZE); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, SZ_64K); + mmap_read_unlock(child); + KUNIT_EXPECT_EQ(test, folio_mapcount(copied), 0); + for (i = 0; i < count; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(copied) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); + release_user_fragments(fragments, count, false); + } + KUNIT_ASSERT_EQ(test, count, (long)nr); + for (i = 0; i < nr; i++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(original) + i * PAGE_SIZE, + 0x51 + i, PAGE_SIZE), NULL); +} + +static void user64k_anon_fault_cow_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma, *cvma; + pte_t orig[2], copied; + struct folio *folio; + unsigned int i, j, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires coarse anonymous faults"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, 2 * SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + /* Leaf0 exercises zero-page COW; leaf1 exercises a direct write fault. */ + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_TRUE(test, pte_special(test_fault_entry(test, parent, TEST_VA))); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + orig[i] = test_fault_entry(test, parent, addr); + KUNIT_ASSERT_TRUE(test, pte_present(orig[i])); + KUNIT_EXPECT_TRUE(test, pte_write(orig[i])); + KUNIT_EXPECT_FALSE(test, pte_special(orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(orig[i]))); + KUNIT_EXPECT_EQ(test, folio_size(folio), (unsigned long)SZ_64K); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, SZ_64K), NULL); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + memset(page_address(folio_page(folio, j)), 0x31 + i * nr + j, PAGE_SIZE); + } + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 2L); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + cvma = test_vma_lookup(child, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, cvma); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 2L); + for (i = 0; i < 2; i++) { + unsigned long addr = TEST_VA + i * SZ_64K; + struct folio *old = page_folio(pfn_to_page(pte_pfn(orig[i]))); + + KUNIT_EXPECT_EQ(test, pte_phys_mm(child, test_fault_entry(test, child, addr)), + pte_phys_mm(parent, orig[i])); + KUNIT_EXPECT_FALSE(test, pte_write(test_fault_entry(test, parent, addr))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)(2 * nr)); + KUNIT_ASSERT_EQ(test, test_file_fault(cvma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + copied = test_fault_entry(test, child, addr); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, orig[i])); + folio = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(folio), (int)nr); + for (j = 0; j < nr; j++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(folio, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(folio, j))); + } + memset(folio_address(folio), 0xa5, SZ_64K); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(old, j)), + 0x31 + i * nr + j, PAGE_SIZE), NULL); + /* Parent reuse must restore exclusivity across the entire leaf. */ + lru_add_drain_all(); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, addr)), + pte_phys_mm(parent, orig[i])); + for (j = 0; j < nr; j++) + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, j))); + } + if (system_supports_64kb_granule()) { + long protect[2]; + bool writable = false; + + kthread_use_mm(parent); + protect[0] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ); + kthread_unuse_mm(parent); + for (i = 0; i < 2; i++) + writable |= pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K)); + KUNIT_EXPECT_EQ(test, protect[0], 0L); + KUNIT_EXPECT_FALSE(test, writable); + kthread_use_mm(parent); + protect[1] = test_mprotect(TEST_VA, 2 * SZ_64K, PROT_READ | PROT_WRITE); + kthread_unuse_mm(parent); + KUNIT_EXPECT_EQ(test, protect[1], 0L); + for (i = 0; i < 2; i++) + KUNIT_EXPECT_TRUE(test, pte_write(test_fault_entry(test, parent, TEST_VA + i * SZ_64K))); + vma = test_vma_lookup(parent, TEST_VA); + KUNIT_ASSERT_NOT_NULL(test, vma); + } + mmap_read_lock(child); + zap_vma_range(cvma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(child); + mmap_read_lock(parent); + zap_vma_range(vma, TEST_VA, 2 * SZ_64K); + mmap_read_unlock(parent); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 0L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 0L); +} + +struct coarse_test_pin { + struct page *page; +}; + +static void coarse_test_unpin(void *arg) +{ + struct coarse_test_pin *pin = arg; + + if (pin->page) { + unpin_user_page(pin->page); + pin->page = NULL; + } +} + +static void user64k_pinned_fork_test(struct kunit *test) +{ + struct mm_struct *parent, *child; + struct vm_area_struct *vma; + struct coarse_test_pin *pin; + struct folio *old, *new; + pte_t original, copied; + unsigned int i, nr = SZ_64K / PAGE_SIZE; + int ret; + + if (PAGE_SHIFT >= 16) { + kunit_skip(test, "requires pinned coarse leaf"); + return; + } + parent = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, parent); + vma = test_fault_vma_in_mm(parent, TEST_VA, SZ_64K); + KUNIT_ASSERT_NOT_NULL(test, vma); + KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA + SZ_64K - 1, true) & VM_FAULT_ERROR, 0U); + original = test_fault_entry(test, parent, TEST_VA); + old = page_folio(pfn_to_page(pte_pfn(original))); + for (i = 0; i < nr; i++) + memset(page_address(folio_page(old, i)), 0x70 + i, PAGE_SIZE); + pin = kunit_kzalloc(test, sizeof(*pin), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pin); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, coarse_test_unpin, pin), 0); + mm_flags_set(MMF_HAS_PINNED, parent); + ret = try_grab_folio(old, 1, FOLL_PIN); + if (!ret) + pin->page = folio_page(old, nr - 1); + KUNIT_ASSERT_EQ(test, ret, 0); + child = test_vma_mm_granule(test, 16); + KUNIT_ASSERT_NOT_NULL(test, child); + uprobe_start_dup_mmap(); + ret = dup_mmap(child, parent); + uprobe_end_dup_mmap(); + KUNIT_ASSERT_EQ(test, ret, 0); + copied = test_fault_entry(test, child, TEST_VA); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, copied), pte_phys_mm(parent, original)); + KUNIT_EXPECT_EQ(test, pte_phys_mm(parent, test_fault_entry(test, parent, TEST_VA)), + pte_phys_mm(parent, original)); + new = page_folio(pfn_to_page(pte_pfn(copied))); + KUNIT_EXPECT_EQ(test, folio_mapcount(old), (int)nr); + KUNIT_EXPECT_EQ(test, folio_mapcount(new), (int)nr); + for (i = 0; i < nr; i++) { + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, i)), + 0x70 + i, PAGE_SIZE), NULL); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(new, i))); + KUNIT_EXPECT_TRUE(test, PageAnonExclusive(folio_page(old, i))); + } + memset(page_address(pin->page), 0xe3, PAGE_SIZE); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(page_address(folio_page(new, nr - 1)), + 0x70 + nr - 1, PAGE_SIZE), NULL); + coarse_test_unpin(pin); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(parent, MM_ANONPAGES), 1L); + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1L); +} + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +static void user4k_shmem_partial_hole_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, true); + struct folio *folio; + loff_t start = PAGE_SIZE + SZ_4K + 17, end = PAGE_SIZE + 3 * SZ_4K + 16; + unsigned int i; + unsigned char byte = 0xa5; + loff_t write_pos = PAGE_SIZE + 2 * SZ_4K + 23; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), start, end); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(2)); + for (i = 0; i < PAGE_SIZE; i++) { + unsigned char expected = (i + PAGE_SIZE >= start && i + PAGE_SIZE <= end) ? + 0 : 0x31 + (PAGE_SIZE + i) / SZ_4K; + + if (((unsigned char *)folio_address(folio))[i] != expected) { + KUNIT_FAIL(test, "partial hole changed byte %u outside its range", i); + break; + } + } + /* Resident hole metadata has no dynamically allocated inode record. */ + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + folio_unlock(folio); + folio_put(folio); + shmem_truncate_range(file_inode(file), PAGE_SIZE + SZ_4K, PAGE_SIZE + 2 * SZ_4K - 1); + KUNIT_ASSERT_EQ(test, kernel_write(file, &byte, 1, &write_pos), 1L); + folio = filemap_lock_folio(file->f_mapping, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), BIT(1)); + KUNIT_EXPECT_EQ(test, ((unsigned char *)folio_address(folio))[2 * SZ_4K + 23], byte); + folio_unlock(folio); + folio_put(folio); +} + +/* A surviving writable pin exposes bytes even after its PTE is revoked. */ +static void user4k_shmem_pinned_hole_test(struct kunit *test) +{ + unsigned int native; + + for (native = 0; native < 2; native++) { + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct vm_area_struct *vma; + struct fragment_test_pins *pins; + struct folio *folio; + unsigned long offset = native ? 0 : SZ_4K; + unsigned long length = native ? PAGE_SIZE : SZ_4K; + unsigned long pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + vma = test_file_vma(test, file, native, true, TEST_VA, length, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + pins = kunit_kzalloc(test, sizeof(*pins), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pins); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_test_unpin, pins), 0); + KUNIT_ASSERT_EQ(test, fragment_test_pin(pins, vma->vm_mm, TEST_VA, + length, FOLL_WRITE | FOLL_LONGTERM, 1), 1L); + folio = pins->fragments[0].folio; + pfn = folio_pfn(folio); + KUNIT_ASSERT_TRUE(test, folio_maybe_dma_pinned(folio)); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + KUNIT_EXPECT_FALSE(test, pte_present(test_fault_entry(test, vma->vm_mm, TEST_VA))); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, SZ_4K), NULL); + /* Model a completed device write through the original pin. */ + memset(folio_address(folio) + SZ_4K, 0xb7, SZ_4K); + folio_lock(folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + fragment_test_unpin(pins); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, folio_pfn(folio), pfn); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xb7, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + 2 * SZ_4K, 0x33, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); + /* With the external writer gone, another hole can be missing. */ + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_TRUE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + folio_unlock(folio); + folio_put(folio); + } +} + +static void test_unpin_folio(void *folio) +{ + unpin_folio(folio); +} + +/* The direct memfd API must publish cache exposure without a user PTE. */ +static void user4k_shmem_memfd_pin_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *pin, *folio; + pgoff_t offset; + long ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + ret = memfd_pin_folios(file, SZ_4K, 2 * SZ_4K - 1, &pin, 1, &offset); + KUNIT_ASSERT_EQ(test, ret, 1L); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_unpin_folio, pin), 0); + KUNIT_EXPECT_EQ(test, offset, (pgoff_t)SZ_4K); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(pin)); + memset(folio_address(pin) + offset, 0xc8, SZ_4K); + folio_lock(pin); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(pin, 0, SZ_4K, SZ_4K)); + folio_unlock(pin); + kunit_release_action(test, test_unpin_folio, pin); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_FALSE(test, shmem_uffd_range_missing(folio, 0, SZ_4K, SZ_4K)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0xc8, SZ_4K), NULL); + folio_unlock(folio); + folio_put(folio); +} + +static void user4k_shmem_replace_validity_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, true); + struct folio *old, *new; + unsigned long old_pfn; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + shmem_truncate_range(file_inode(file), SZ_4K, 2 * SZ_4K - 1); + new = folio_alloc(GFP_KERNEL, 0); + KUNIT_ASSERT_NOT_NULL(test, new); + folio_lock(new); + old = filemap_lock_folio(file->f_mapping, 0); + if (IS_ERR(old)) { + folio_unlock(new); + folio_put(new); + KUNIT_FAIL(test, "could not lock source cache folio"); + return; + } + old_pfn = folio_pfn(old); + memcpy(folio_address(new), folio_address(old), PAGE_SIZE); + __folio_set_swapbacked(new); + __folio_mark_uptodate(new); + folio_mark_dirty(new); + replace_page_cache_folio(old, new); + folio_add_lru(new); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(new, 0), BIT(1)); + KUNIT_EXPECT_NE(test, folio_pfn(new), old_pfn); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new), 0x31, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(new) + SZ_4K, 0, SZ_4K), NULL); + folio_unlock(old); + folio_put(old); + folio_unlock(new); + folio_put(new); +} + +static void user4k_shmem_uffd_alias_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct vm_area_struct *small, *native; + struct folio *folio; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = shmem_uffd_set_missing(folio, 0, all); + KUNIT_EXPECT_TRUE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + small = test_file_vma(test, file, false, true, TEST_VA, SZ_4K, SZ_4K); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_EQ(test, test_file_fault(small, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), all & ~BIT(1)); + KUNIT_EXPECT_FALSE(test, shmem_uffd_has_missing_native_page(folio)); + folio_unlock(folio); + folio_put(folio); + + native = test_file_vma(test, file, true, true, TEST_VA, PAGE_SIZE, 0); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, test_file_fault(native, TEST_VA, false) & VM_FAULT_ERROR, 0U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} + +static void user4k_shmem_uffd_state_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, 3 * PAGE_SIZE, false); + const unsigned long all = GENMASK(PAGE_SIZE / SZ_4K - 1, 0); + struct folio *folio; + unsigned long mask; + unsigned int i; + int ret; + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + for (i = 0; i < 3; i++) { + ret = shmem_get_folio(file_inode(file), i, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), 0UL); + ret = shmem_uffd_set_missing(folio, i, all & ~BIT(0)); + KUNIT_EXPECT_EQ(test, ret, 0); + shmem_uffd_mark_present(folio, i, BIT(1)); + mask = shmem_uffd_missing_mask(folio, i); + KUNIT_EXPECT_EQ(test, mask, all & ~GENMASK(1, 0)); + ret = shmem_uffd_set_missing(folio, i, BIT(PAGE_SIZE / SZ_4K)); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), mask); + folio_unlock(folio); + folio_put(folio); + } + /* Real truncate removes one index without discarding neighbouring state. */ + shmem_truncate_range(file_inode(file), PAGE_SIZE, 2 * PAGE_SIZE - 1); + KUNIT_EXPECT_PTR_EQ(test, xa_load(&SHMEM_I(file_inode(file))->uffd_missing, 1), NULL); + for (i = 0; i < 3; i += 2) { + folio = filemap_lock_folio(file->f_mapping, i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, i), all & ~GENMASK(1, 0)); + folio_unlock(folio); + folio_put(folio); + } + /* Generic batch deletion must not strand inode metadata either. */ + truncate_inode_pages(file->f_mapping, 0); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); + ret = shmem_get_folio(file_inode(file), 1, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 1), 0UL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0, PAGE_SIZE), NULL); + folio_unlock(folio); + folio_put(folio); +} + +#ifdef CONFIG_MIGRATION +static void user4k_shmem_uffd_migration_test(struct kunit *test) +{ + struct file *file = test_cache_file(test, PAGE_SIZE, false); + const unsigned long missing = GENMASK(PAGE_SIZE / SZ_4K - 1, 1); + struct folio *folio; + unsigned long old_pfn; + unsigned int succeeded = 0; + bool isolated; + int ret; + LIST_HEAD(folios); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + ret = shmem_get_folio(file_inode(file), 0, 0, &folio, SGP_CACHE); + KUNIT_ASSERT_EQ(test, ret, 0); + memset(folio_address(folio), 0xa5, SZ_4K); + folio_mark_dirty(folio); + ret = shmem_uffd_set_missing(folio, 0, missing); + old_pfn = folio_pfn(folio); + folio_unlock(folio); + folio_put(folio); + KUNIT_ASSERT_EQ(test, ret, 0); + lru_add_drain_all(); + folio = filemap_get_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + isolated = isolate_folio_to_list(folio, &folios); + folio_put(folio); + KUNIT_ASSERT_TRUE(test, isolated); + ret = migrate_pages(&folios, test_migration_alloc, NULL, 0, MIGRATE_SYNC, + MR_SYSCALL, &succeeded); + putback_movable_pages(&folios); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_EQ(test, succeeded, 1U); + folio = filemap_lock_folio(file->f_mapping, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); + KUNIT_EXPECT_NE(test, folio_pfn(folio), old_pfn); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), missing); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio), 0xa5, SZ_4K), NULL); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, 0, + PAGE_SIZE - SZ_4K), NULL); + shmem_uffd_mark_present(folio, 0, missing); + KUNIT_EXPECT_EQ(test, shmem_uffd_missing_mask(folio, 0), 0UL); + folio_unlock(folio); + folio_put(folio); + KUNIT_EXPECT_TRUE(test, xa_empty(&SHMEM_I(file_inode(file))->uffd_missing)); +} +#endif +#endif + +static struct kunit_case user4k_test_cases[] = { + KUNIT_CASE(user4k_exec_granule_policy_test), + KUNIT_CASE(user4k_aslr_window_test), + KUNIT_CASE(user4k_thp_eligibility_test), + KUNIT_CASE(user4k_geometry_test), + KUNIT_CASE(user4k_leaf_encoding_test), + KUNIT_CASE(user16k_leaf_encoding_test), + KUNIT_CASE(user4k_rmap_walk_test), + KUNIT_CASE(user4k_rmap_boundary_test), + KUNIT_CASE(native_rmap_walk_test), + KUNIT_CASE(user64k_rmap_lookup_test), + KUNIT_CASE(user64k_anon_rmap_span_test), + KUNIT_CASE(user64k_anon_zap_batch_test), + KUNIT_CASE(user_granule_zero_region_test), +#ifdef CONFIG_COREDUMP + KUNIT_CASE(user_granule_core_offsets_test), +#endif + KUNIT_CASE(user64k_fragment_ownership_test), + KUNIT_CASE(user64k_anon_fault_cow_test), + KUNIT_CASE(user64k_pinned_fork_test), + KUNIT_CASE(user64k_file_rmap_span_test), + KUNIT_CASE(user4k_vma_offsets_test), + KUNIT_CASE(user4k_vma_intervals_test), + KUNIT_CASE(user4k_vma_split_test), + KUNIT_CASE(user4k_vma_merge_test), + KUNIT_CASE(user4k_vma_partial_merge_test), + KUNIT_CASE(user4k_vma_copy_shrink_test), + KUNIT_CASE(user4k_anon_rmap_test), + KUNIT_CASE(user4k_anon_fork_rmap_test), + KUNIT_CASE(user4k_nonlinear_rmap_reuse_test), + KUNIT_CASE(user4k_dup_mmap_test), + KUNIT_CASE(user4k_anon_root_lifetime_test), + KUNIT_CASE(user4k_data_tlb_batch_test), + KUNIT_CASE(user4k_anon_zap_test), + KUNIT_CASE(user4k_anon_writable_slots_test), + KUNIT_CASE(user4k_anon_write_fault_test), + KUNIT_CASE(user16k_anon_write_fault_test), + KUNIT_CASE(user4k_mlock_counts_test), + KUNIT_CASE(user4k_mlock_shared_test), + KUNIT_CASE(user4k_mlock_repair_test), + KUNIT_CASE(user_granule_large_vma_split_test), + KUNIT_CASE(user_granule_brk_extent_test), + KUNIT_CASE(user4k_droppable_reclaim_test), + KUNIT_CASE(user4k_lazyfree_test), + KUNIT_CASE(user4k_zero_cow_test), + KUNIT_CASE(user16k_zero_cow_test), + KUNIT_CASE(user4k_shmem_swap_usage_test), +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) + KUNIT_CASE(user4k_shmem_partial_hole_test), + KUNIT_CASE(user4k_shmem_pinned_hole_test), + KUNIT_CASE(user4k_shmem_memfd_pin_test), + KUNIT_CASE(user4k_shmem_replace_validity_test), + KUNIT_CASE(user4k_shmem_uffd_alias_test), + KUNIT_CASE(user4k_shmem_uffd_state_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_shmem_uffd_migration_test), +#endif +#endif + KUNIT_CASE(user4k_file_mapcounts_test), + KUNIT_CASE(user64k_file_promote_test), + KUNIT_CASE(user64k_shmem_fault_test), + KUNIT_CASE(user64k_shmem_fault_busy_test), + KUNIT_CASE(user64k_shmem_fault_hole_test), + KUNIT_CASE(user64k_file_leaf_test), + KUNIT_CASE(user64k_file_cow_test), + KUNIT_CASE(user64k_file_bounds_test), + KUNIT_CASE(user4k_file_shared_test), + KUNIT_CASE(user4k_iommufd_lifetime_test), + KUNIT_CASE(user4k_iommufd_rollback_test), + KUNIT_CASE(user4k_iommufd_granule_test), + KUNIT_CASE(user4k_iommufd_shared_test), + KUNIT_CASE(user4k_iommufd_exit_account_test), + KUNIT_CASE(user4k_iova_bitmap_test), + KUNIT_CASE(user4k_iova_bitmap_window_test), + KUNIT_CASE(user4k_iova_bitmap_retained_test), + KUNIT_CASE(user4k_iova_bitmap_partial_test), + KUNIT_CASE(user4k_mremap_move_test), + KUNIT_CASE(user4k_mremap_resize_test), + KUNIT_CASE(user4k_stack_relocate_test), + KUNIT_CASE(user4k_exec_arguments_test), + KUNIT_CASE(user64k_exec_arguments_test), + KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(user4k_insert_fragment_test), + KUNIT_CASE(user16k_insert_shared_test), + KUNIT_CASE(user4k_insert_range_test), + KUNIT_CASE(user4k_pfn_offset_test), + KUNIT_CASE(user4k_remap_pfn_range_test), + KUNIT_CASE(user4k_iomem_access_test), +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_VDSO_GETRANDOM) + KUNIT_CASE(user4k_vdso_mapping_test), +#endif + KUNIT_CASE(user4k_mremap_cow_test), + KUNIT_CASE(user4k_mremap_file_test), + KUNIT_CASE(user4k_msync_offsets_test), + KUNIT_CASE(user4k_prefault_dirty_test), + KUNIT_CASE(user4k_cold_batch_test), + KUNIT_CASE(user4k_remote_access_test), + KUNIT_CASE(user4k_remote_file_test), +#ifdef CONFIG_CROSS_MEMORY_ATTACH + KUNIT_CASE(user4k_process_vm_test), +#endif +#ifdef CONFIG_FUTEX + KUNIT_CASE(user4k_futex_keys_test), + KUNIT_CASE(user4k_futex_wake_test), +#endif + KUNIT_CASE(user4k_mmap_descriptor_test), + KUNIT_CASE(user4k_mmap_region_offsets_test), + KUNIT_CASE(user4k_commit_accounting_test), + KUNIT_CASE(mixed_locked_account_units_test), + KUNIT_CASE(user64k_commit_accounting_test), + KUNIT_CASE(user64k_page_table_check_test), +#ifdef CONFIG_PROC_FS + KUNIT_CASE(user4k_accounting_readers_test), +#endif + KUNIT_CASE(user_granule_prefault_test), + KUNIT_CASE(user4k_mmap_accounting_test), + KUNIT_CASE(user4k_vm_mmap_test), + KUNIT_CASE(user4k_mprotect_test), + KUNIT_CASE(user4k_mprotect_cow_batch_test), + KUNIT_CASE(user4k_mprotect_accounting_test), + KUNIT_CASE(user4k_file_cow_test), + KUNIT_CASE(user4k_file_eof_test), + KUNIT_CASE(user4k_file_truncate_test), + KUNIT_CASE(user4k_filemap_fault_test), + KUNIT_CASE(user4k_file_invalidate_test), + KUNIT_CASE(user4k_file_fragments_test), + KUNIT_CASE(user4k_file_fork_hole_test), + KUNIT_CASE(user4k_swap_encoding_test), + KUNIT_CASE(user4k_cow_reuse_test), + KUNIT_CASE(user16k_cow_reuse_test), + KUNIT_CASE(user4k_fork_cow_test), + KUNIT_CASE(user16k_fork_cow_test), + KUNIT_CASE(user4k_slot_pin_test), + KUNIT_CASE(user4k_fragment_pin_test), + KUNIT_CASE(user4k_fragment_locked_pin_test), + KUNIT_CASE(user4k_folio_walk_geometry_test), + KUNIT_CASE(user4k_fragment_longterm_test), +#ifdef CONFIG_BLOCK + KUNIT_CASE(user4k_bio_fragments_test), + KUNIT_CASE(user4k_bio_trim_test), +#endif + KUNIT_CASE(user4k_fragment_permissions_test), + KUNIT_CASE(user16k_fragment_offset_test), +#ifdef CONFIG_MIGRATION + KUNIT_CASE(user4k_migration_test), + KUNIT_CASE(user4k_file_migration_test), + KUNIT_CASE(user64k_file_migration_test), + KUNIT_CASE(user64k_anon_migration_test), + KUNIT_CASE(user4k_fragment_pin_race_test), + KUNIT_CASE(user4k_migration_entries_test), + KUNIT_CASE(user4k_nonlinear_rmap_test), + KUNIT_CASE(user4k_mprotect_migration_test), + KUNIT_CASE(user4k_live_migration_test), + KUNIT_CASE(user4k_migration_wait_test), +#endif + KUNIT_CASE(user4k_anon_exit_test), + KUNIT_CASE(user4k_anon_remote_zap_test), +#ifdef CONFIG_ARM64_CONTPTE + KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(native_contpte_test), +#endif + KUNIT_CASE(user4k_mm_tables_test), + KUNIT_CASE(user16k_mm_tables_test), + KUNIT_CASE(user64k_mm_tables_test), + KUNIT_CASE(native_mm_tables_test), + KUNIT_CASE(user4k_anon_read_fault_test), + KUNIT_CASE(user4k_mm_hardware_test), + KUNIT_CASE(user4k_fragment_lifetime_test), + KUNIT_CASE(user16k_fragment_lifetime_test), + KUNIT_CASE(user64k_table_owner_test), +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + KUNIT_CASE(native_pgtable_deposit_test), +#endif + KUNIT_CASE(user4k_translation_test), + KUNIT_CASE(user4k_context_uaccess_test), + KUNIT_CASE(user4k_fault_uaccess_test), + {} +}; + +#ifdef CONFIG_NUMA_BALANCING +/* + * The COW hinting expectations require a stable normal-balancing policy. + * Runtime sysctl writers do not take the fixture's mm or PTE locks, so run + * this case at boot and do not expose it through the debugfs rerun control. + */ +static struct kunit_case user4k_numa_init_cases[] = { + KUNIT_CASE(user_subpage_numa_cow_batch_test), + {} +}; + +static struct kunit_suite user4k_numa_init_suite = { + .name = "arm64-user4k-numa-init", + .test_cases = user4k_numa_init_cases, +}; + +kunit_test_init_section_suite(user4k_numa_init_suite); +#endif + +static struct kunit_suite user4k_test_suite = { + .name = "arm64-user4k", + .test_cases = user4k_test_cases, +}; + +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_shmem_swap_suite, + &user4k_coarse_shmem_suite, + &user4k_nonlinear_swap_suite); +#else +kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap_suite); +#endif + +MODULE_LICENSE("GPL"); diff --git a/arch/arm64/mm/user4k-tlb-test.c b/arch/arm64/mm/user4k-tlb-test.c new file mode 100644 index 00000000000000..e5671256b7c76e --- /dev/null +++ b/arch/arm64/mm/user4k-tlb-test.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Decode emitted operands without executing TLBI instructions. */ +#include +#include +#include +#include + +struct tlbi_test_state { + u64 next; + u64 end; + u64 stride; + u16 asid; + u32 level; + unsigned int shift; + unsigned int single_ops; + unsigned int range_ops; + bool lpa2; +}; + +static unsigned int test_tg(unsigned int shift) +{ + return shift == 12 ? 1 : shift == 14 ? 2 : 3; +} + +static void record_single(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = (arg & GENMASK_ULL(43, 0)) << 12; + u64 hint = 0; + + if (alternative_has_cap_unlikely(ARM64_HAS_ARMv8_4_TTL) && state->level <= 3) + hint = (test_tg(state->shift) << 2) | state->level; + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBI_TTL_MASK, arg), hint); + state->next += state->stride; + state->single_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static void record_range(u64 arg) +{ + struct kunit *test = kunit_get_current_test(); + struct tlbi_test_state *state = test->priv; + u64 address = FIELD_GET(TLBIR_BADDR_MASK, arg) << (state->lpa2 ? 16 : state->shift); + u64 count = (FIELD_GET(TLBIR_NUM_MASK, arg) + 1) << + (5 * FIELD_GET(TLBIR_SCALE_MASK, arg) + 1); + + KUNIT_EXPECT_EQ(test, address, state->next); + KUNIT_EXPECT_EQ(test, arg >> 48, (u64)state->asid); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TG_MASK, arg), (u64)test_tg(state->shift)); + KUNIT_EXPECT_EQ(test, FIELD_GET(TLBIR_TTL_MASK, arg), + state->level > 3 ? 0ULL : (u64)state->level); + if (state->lpa2) + KUNIT_EXPECT_TRUE(test, IS_ALIGNED(address, SZ_64K)); + state->next += count << state->shift; + state->range_ops++; + KUNIT_EXPECT_LE(test, state->next, state->end); +} + +static bool range_supported(void) +{ + return true; +} + +static bool range_unsupported(void) +{ + return false; +} + +static void test_flush(struct kunit *test, unsigned int shift, u64 start, + unsigned long pages, u64 stride, u32 level, bool lpa2, + bool has_range) +{ + struct tlbi_test_state state = { + .next = start, + .end = start + (pages << shift), + .stride = stride, + .asid = 0xa55a, + .level = level, + .shift = shift, + .lpa2 = lpa2, + }; + + test->priv = &state; + __flush_tlb_range_op_granule(record_single, record_range, start, pages, + stride, state.asid, level, lpa2, shift, + has_range ? range_supported : range_unsupported); + KUNIT_EXPECT_EQ(test, state.next, state.end); + if (has_range) { + KUNIT_EXPECT_LE(test, state.single_ops + state.range_ops, 20U); + if (pages >= 32) + KUNIT_EXPECT_GT(test, state.range_ops, 0U); + } else { + KUNIT_EXPECT_EQ(test, state.range_ops, 0U); + KUNIT_EXPECT_EQ(test, state.single_ops, (unsigned int)((pages << shift) / stride)); + } + test->priv = NULL; +} + +static void user4k_tlbi_operands_test(struct kunit *test) +{ + static const unsigned long counts[] = { + 0, 1, 2, 3, 31, 32, 33, 63, 64, 65, 2047, 2048, 2049, + 65535, 65536, 65537, MAX_TLBI_RANGE_PAGES - 1, MAX_TLBI_RANGE_PAGES, + }; + static const u32 levels[] = { 2, 3, TLBI_TTL_UNKNOWN }; + unsigned int shift, i, j; + + /* Force both algorithms using recording callbacks, independent of CPU caps. */ + for (shift = 12; shift <= 16; shift += 2) { + u64 size = 1UL << shift; + + for (j = 0; j < ARRAY_SIZE(levels); j++) { + for (i = 0; i < ARRAY_SIZE(counts); i++) { + unsigned long pages = counts[i]; + + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, true); + if (shift != 16) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], true, true); + if (pages <= 65) + test_flush(test, shift, SZ_4M + size, pages, size, + levels[j], false, false); + } + } + /* Preserve a larger leaf stride when range instructions are absent. */ + test_flush(test, shift, SZ_1G, 3 * 512, 512 * size, 2, false, false); + /* Last virtual page must not lose high address bits. */ + test_flush(test, shift, (1ULL << 48) - size, 1, size, 3, false, false); + } +} + +static void user4k_tlb_gather_test(struct kunit *test) +{ + struct mm_struct *mm = kunit_kzalloc(test, sizeof(*mm), GFP_KERNEL); + pte_t entries[3] = {}; + unsigned int small; + + KUNIT_ASSERT_NOT_NULL(test, mm); + for (small = 0; small < 2; small++) { + struct mmu_gather tlb = { .mm = mm, .start = ULONG_MAX }; + unsigned long size = small ? SZ_4K : PAGE_SIZE; + unsigned long address = SZ_4M + size; + + mm->context.flags = small ? MMCF_USER_4K : 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + tlb_remove_tlb_entries(&tlb, entries, 3, address); + KUNIT_EXPECT_EQ(test, tlb.start, address); + KUNIT_EXPECT_EQ(test, tlb.end, address + 3 * size); + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), 3); + tlb.cleared_pmds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), size); + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + tlb.cleared_ptes = 0; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_2M : PMD_SIZE); + tlb.cleared_pmds = 0; + tlb.cleared_puds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? SZ_1G : PUD_SIZE); + tlb.cleared_puds = 0; + tlb.cleared_p4ds = 1; + KUNIT_EXPECT_EQ(test, tlb_get_unmap_size(&tlb), small ? 1UL << 39 : P4D_SIZE); + tlb.freed_tables = 1; + KUNIT_EXPECT_EQ(test, tlb_get_level(&tlb), TLBI_TTL_UNKNOWN); + } +} + +static struct kunit_case user4k_tlb_test_cases[] = { + KUNIT_CASE(user4k_tlbi_operands_test), + KUNIT_CASE(user4k_tlb_gather_test), + {} +}; + +static struct kunit_suite user4k_tlb_test_suite = { + .name = "arm64-user4k-tlb", + .test_cases = user4k_tlb_test_cases, +}; + +kunit_test_suite(user4k_tlb_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index ceae52fd7586d0..33f6c7e9608d5b 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -54,6 +54,18 @@ config VFIO_IOMMU_TYPE1 tristate default n +config VFIO_TYPE1_ACCOUNT_KUNIT_TEST + bool "Test VFIO type1 pin accounting and retained fragments" if !KUNIT_ALL_TESTS + depends on KUNIT=y && VFIO_IOMMU_TYPE1=y + default KUNIT_ALL_TESTS + help + Exercise locked-memory refunds after source-mm exit and DMA owner + changes. Arm64 mixed-granule configurations also test accounting + conversions between user and native page sizes. + Mixed-granule tests exercise retained pins, mappings, replay and + external page arrays with recorded IOMMU translations. No physical + device DMA is performed. + config VFIO_IOMMU_SPAPR_TCE tristate depends on SPAPR_TCE_IOMMU diff --git a/drivers/vfio/vfio_iommu_type1_account_test.c b/drivers/vfio/vfio_iommu_type1_account_test.c new file mode 100644 index 00000000000000..5447258ce49b8c --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_account_test.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included by type1 so the tests exercise the actual accounting helpers. */ +#include +#include + +struct vfio_account_mm { + struct mm_struct *mm; + bool live; +}; + +static void vfio_account_mm_free(void *data) +{ + struct vfio_account_mm *ctx = data; + + if (ctx->live) + mmput(ctx->mm); + mmdrop(ctx->mm); +} + +static struct vfio_account_mm *vfio_account_mm_new(struct kunit *test, + unsigned int shift) +{ + struct vfio_account_mm *ctx; + + ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return NULL; + ctx->mm = mm_alloc(); + if (!ctx->mm) + return NULL; +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + if (shift != PAGE_SHIFT) { + pgd_t *old = ctx->mm->pgd, *pgd; + unsigned long flag = shift == 12 ? MMCF_USER_4K : + shift == 14 ? MMCF_USER_16K : MMCF_USER_64K; + + ctx->mm->context.flags |= flag; + pgd = pgd_alloc(ctx->mm); + ctx->mm->context.flags &= ~flag; + if (!pgd) { + mmput(ctx->mm); + return NULL; + } + pgd_free(ctx->mm, old); + ctx->mm->pgd = pgd; + ctx->mm->context.flags |= flag; + } +#endif + ctx->live = true; + mmgrab(ctx->mm); + if (kunit_add_action_or_reset(test, vfio_account_mm_free, ctx)) + return NULL; + return ctx; +} + +static void vfio_account_dma_free(void *data) +{ + struct vfio_dma *dma = data; + + /* Cleanup must also work with the negative-control implementation. */ + if (dma->locked_vm) + mm_lock_acct(dma->task, dma->mm, true, -dma->locked_vm); + mmdrop(dma->mm); + put_task_struct(dma->task); +} + +static struct vfio_dma *vfio_account_dma_new(struct kunit *test, + struct vfio_account_mm *ctx) +{ + struct vfio_dma *dma = kunit_kzalloc(test, sizeof(*dma), GFP_KERNEL); + + if (!dma) + return NULL; + dma->mm = ctx->mm; + mmgrab(dma->mm); + dma->task = get_task_struct(current->group_leader); + dma->lock_cap = true; + if (kunit_add_action_or_reset(test, vfio_account_dma_free, dma)) + return NULL; + return dma; +} + +static void vfio_account_exit_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_dma *dma; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_EQ(test, mm_page_shift(ctx->mm), shifts[i]); + dma = vfio_account_dma_new(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 2, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 2UL * PAGE_SIZE); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, 1, true), -ESRCH); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(ctx->mm), 1UL); + KUNIT_EXPECT_EQ(test, vfio_lock_acct(dma, -1, true), 0); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + kunit_info(test, "VFIO %uK user/%luK native dead-mm refunds checked", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_account_owner_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL + 12, 14, 16, +#endif + }; + + for (unsigned int reverse = 0; reverse < 2; reverse++) { + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *old, *next; + struct vfio_dma *dma; + + if ((i || reverse) && shifts[i] == PAGE_SHIFT) + continue; + old = vfio_account_mm_new(test, reverse ? shifts[i] : PAGE_SHIFT); + next = vfio_account_mm_new(test, reverse ? PAGE_SHIFT : shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + dma = vfio_account_dma_new(test, old); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, 3, true), 0); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + if (task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + MM_ACCOUNT_SHIFT; + + /* Model a recipient whose existing pins consume its budget. */ + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, false), + -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 3UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, next->mm); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 3UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 3UL * PAGE_SIZE); + /* Same-mm ownership is idempotent; it must not double the charge. */ + KUNIT_ASSERT_EQ(test, + vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 3UL); + KUNIT_ASSERT_EQ(test, vfio_lock_acct(dma, -3, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "VFIO dead %uK mm -> %uK mm ownership refund checked", + 1U << (mm_page_shift(old->mm) - 10), + 1U << (mm_page_shift(next->mm) - 10)); + } + } +} + +static struct kunit_case vfio_account_cases[] = { + KUNIT_CASE(vfio_account_exit_test), + KUNIT_CASE(vfio_account_owner_test), + {} +}; + +static struct kunit_suite vfio_account_suite = { + .name = "vfio-type1-account", + .test_cases = vfio_account_cases, +}; + +kunit_test_suite(vfio_account_suite); diff --git a/drivers/vfio/vfio_iommu_type1_fragments_test.c b/drivers/vfio/vfio_iommu_type1_fragments_test.c new file mode 100644 index 00000000000000..60fffff10c6bee --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_fragments_test.c @@ -0,0 +1,1177 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Actual type1 lifecycle with recorded translations; no physical device DMA. */ +#include +#include +#include +#include + +#define VFIO_TEST_VA SZ_64M +#define VFIO_TEST_IOVA SZ_256M + +struct vfio_test_domain { + struct iommu_domain core; + struct vfio_domain type1; + struct xarray translations; + unsigned long mapped; + unsigned long fail_after; + unsigned long largest_map; + unsigned int syncs; + struct folio *observed; + struct kunit *test; +}; + +struct vfio_fragment_fixture { + struct vfio_iommu *iommu; + struct vfio_test_domain domains[2]; + struct vfio_iommu_group group; + struct vfio_device device; + struct kunit *test; + unsigned int external_refs; + unsigned int notifications; +}; + +static int vfio_test_domain_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t phys, size_t pgsize, size_t pgcount, int prot, + gfp_t gfp, size_t *mapped) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount; + + mock->largest_map = max(mock->largest_map, length); + for (unsigned long done = 0; done < length; done += SZ_4K) { + int ret; + + if (mock->fail_after && mock->mapped >= mock->fail_after) + return -ENOMEM; + ret = xa_insert(&mock->translations, (iova + done) >> 12, + xa_mk_value((phys + done) >> 12), gfp); + if (ret) + return ret; + mock->mapped += SZ_4K; + *mapped += SZ_4K; + } + return 0; +} + +static size_t vfio_test_domain_unmap(struct iommu_domain *domain, unsigned long iova, + size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + unsigned long length = pgsize * pgcount, done; + + for (done = 0; done < length; done += SZ_4K) { + if (!xa_erase(&mock->translations, (iova + done) >> 12)) + break; + mock->mapped -= SZ_4K; + } + return done; +} + +static phys_addr_t vfio_test_domain_translate(struct iommu_domain *domain, dma_addr_t iova) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + void *entry = xa_load(&mock->translations, iova >> 12); + + return entry ? ((phys_addr_t)xa_to_value(entry) << 12) + (iova & (SZ_4K - 1)) : 0; +} + +static void vfio_test_domain_sync(struct iommu_domain *domain, + struct iommu_iotlb_gather *gather) +{ + struct vfio_test_domain *mock = container_of(domain, typeof(*mock), core); + + mock->syncs++; + if (mock->observed) + KUNIT_EXPECT_TRUE(mock->test, folio_maybe_dma_pinned(mock->observed)); +} + +static const struct iommu_domain_ops vfio_test_domain_ops = { + .map_pages = vfio_test_domain_map, + .unmap_pages = vfio_test_domain_unmap, + .iova_to_phys = vfio_test_domain_translate, + .iotlb_sync = vfio_test_domain_sync, +}; + +static void vfio_test_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_fragment_fixture *f = container_of(device, typeof(*f), device); + + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } +} + +static const struct vfio_device_ops vfio_test_device_ops = { + .dma_unmap = vfio_test_notify, +}; + +static void vfio_fragment_fixture_free(void *ptr) +{ + struct vfio_fragment_fixture *f = ptr; + struct rb_node *node; + + while ((node = rb_first(&f->iommu->dma_list))) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + struct rb_node *pin; + + while ((pin = rb_first(&dma->pfn_list))) { + struct vfio_pfn *vpfn = rb_entry(pin, struct vfio_pfn, node); + + vfio_unpin_page_external(dma, vpfn->iova, + list_empty(&f->iommu->domain_list)); + } + vfio_remove_dma(f->iommu, dma); + } + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + KUNIT_EXPECT_EQ(f->test, f->domains[i].mapped, 0UL); + KUNIT_EXPECT_TRUE(f->test, xa_empty(&f->domains[i].translations)); + xa_destroy(&f->domains[i].translations); + } + kfree(f->iommu); +} + +static struct vfio_fragment_fixture *vfio_fragment_fixture_new(struct kunit *test) +{ + struct vfio_fragment_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); + + if (!f) + return NULL; + f->test = test; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) + return NULL; + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->test = test; + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)f; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + if (kunit_add_action_or_reset(test, vfio_fragment_fixture_free, f)) + return NULL; + return f; +} + +static void vfio_test_file_put(void *ptr) +{ + fput(ptr); +} + +static struct file *vfio_fragment_file(struct kunit *test, unsigned long length, + unsigned char value) +{ + struct file *file = shmem_file_setup("vfio-fragments", length, EMPTY_VMA_FLAGS); + unsigned char *data; + loff_t pos = 0; + + if (IS_ERR(file)) + return file; + if (kunit_add_action_or_reset(test, vfio_test_file_put, file)) + return ERR_PTR(-ENOMEM); + data = kunit_kmalloc(test, length, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + memset(data, value, length); + if (kernel_write(file, data, length, &pos) != length) + return ERR_PTR(-EIO); + return file; +} + +static unsigned long vfio_fragment_source(struct vfio_account_mm *ctx, + struct file *file, unsigned long length) +{ + unsigned long addr; + + kthread_use_mm(ctx->mm); + addr = vm_mmap(file, VFIO_TEST_VA, length, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, 0); + kthread_unuse_mm(ctx->mm); + return addr; +} + +static int vfio_fragment_test_map(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, + unsigned long offset, unsigned long length) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .vaddr = VFIO_TEST_VA + offset, + .iova = VFIO_TEST_IOVA, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_lifetime_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + unsigned char data[64]; + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = 2 * PAGE_SIZE, + }; + struct vfio_bitmap bitmap = {}; + + if (shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x71); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + for (unsigned int d = 0; d < 2; d++) + f->domains[d].observed = vfio_fragment_find(dma, 0)->pin.folio; + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_GE(test, f->domains[0].largest_map, PAGE_SIZE); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ | IOMMU_WRITE, &page), 1); + f->external_refs++; + KUNIT_EXPECT_EQ(test, page_to_phys(page), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The user VA goes away; retained pins are still the DMA buffer. */ + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x71, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA), + iommu_iova_to_phys(&f->domains[1].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, 2 * PAGE_SIZE); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_GT(test, f->domains[0].syncs, 0U); + KUNIT_EXPECT_GT(test, f->domains[1].syncs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + kunit_info(test, "type1 retained %uK user/%luK native pins after mm exit and replay", + 1U << (shifts[i] - 10), PAGE_SIZE >> 10); + } +} + +static void vfio_fragments_lazy_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + struct vfio_dma *dma; + struct page *page; + unsigned char data[64]; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x52); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x52, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EFAULT); + for (unsigned int i = 0; i < 3; i++) + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_map_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct file *file; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x25); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + f->domains[1].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), -ENOMEM); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, f->domains[1].mapped, 0UL); + f->domains[1].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_fine_test(struct kunit *test) +{ + unsigned int shifts[] = { PAGE_SHIFT, 12, 14 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(shifts); i++) { + struct vfio_account_mm *ctx; + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page = NULL; + unsigned char data[64]; + unsigned long length = PAGE_SIZE + SZ_4K; + size_t copied = 0; + + if (i && shifts[i] == PAGE_SHIFT) + continue; + ctx = vfio_account_mm_new(test, shifts[i]); + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x64); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, SZ_4K, length), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, length); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* The physical base is offset inside a native page, not representable. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + PAGE_SIZE, 1, IOMMU_READ, &page), -EINVAL); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + memset(data, 0x97, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw_chunk(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), true, &copied), 0); + KUNIT_EXPECT_EQ(test, copied, sizeof(data)); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, length / SZ_4K), 2U); + KUNIT_EXPECT_TRUE(test, test_bit(0, dma->bitmap)); + KUNIT_EXPECT_TRUE(test, test_bit(1, dma->bitmap)); + memset(data, 0, sizeof(data)); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + SZ_4K - 17, data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(data, 0x97, sizeof(data)), NULL); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, + VFIO_TEST_IOVA + length, data, sizeof(data), false), -EINVAL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_info(test, "%uK userspace fine IOVA/physical offsets and dirty bytes checked", + 1U << (shifts[i] - 10)); + } +} + +static void vfio_fragments_replay_failure_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x58); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + f->domains[0].fail_after = SZ_4K; + KUNIT_EXPECT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), -ENOMEM); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_FALSE(test, dma->iommu_mapped); + f->domains[0].fail_after = 0; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + /* Detaching the last hardware domain keeps only the external page. */ + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); +} + +static void vfio_fragments_anon_owner_test(struct kunit *test) +{ + struct vfio_account_mm *old = vfio_account_mm_new(test, 12); + struct vfio_account_mm *next = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + unsigned char *data = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL); + unsigned long addr, copied, charge; + unsigned char check[64]; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + KUNIT_ASSERT_NOT_NULL(test, data); + memset(data, 0x49, PAGE_SIZE); + kthread_use_mm(old->mm); + addr = vm_mmap(NULL, VFIO_TEST_VA, PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, 0); + copied = addr == VFIO_TEST_VA ? + copy_to_user((void __user *)addr, data, PAGE_SIZE) : PAGE_SIZE; + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, copied, 0UL); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NOT_NULL(test, vfio_fragment_find(dma, 0)->pin.subpage); + charge = dma->locked_vm; + KUNIT_ASSERT_GT(test, charge, 0UL); + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, vfio_change_dma_mm(dma, dma->task, next->mm, true), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(old->mm), 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), charge); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + check, sizeof(check), false), 0); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(check, 0x49, sizeof(check)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(next->mm), 0UL); + kunit_info(test, "anonymous 4K slot pins survive source exit and native-owner transfer"); +} + +static void vfio_fragments_domain_granule_test(struct kunit *test) +{ + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_dma *dma; + struct file *file; + struct page *page; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + file = vfio_fragment_file(test, PAGE_SIZE, 0x73); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), VFIO_TEST_VA); + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, ctx, 0, PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, + vfio_dirty_pgsize(f->iommu)), 0); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 31, 1, IOMMU_READ, &page), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(dma->bitmap, PAGE_SIZE / SZ_4K), + (unsigned int)(PAGE_SIZE / SZ_4K)); + f->domains[0].core.pgsize_bitmap = SZ_16K; + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[0].type1), 0); + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_16K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_unmap_unpin_reaccount(f->iommu); + list_del_init(&f->domains[0].type1.next); + vfio_update_pgsize_bitmap(f->iommu); + KUNIT_EXPECT_EQ(test, 1UL << __ffs(f->iommu->pgsize_bitmap), SZ_4K); + KUNIT_EXPECT_EQ(test, vfio_dirty_pgsize(f->iommu), SZ_4K); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 1); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); +} + +static void vfio_fragments_native_owner_test(struct kunit *test) +{ + for (unsigned int variant = 0; variant < 9; variant++) { + bool same_mm = variant == 8; + bool mapped = same_mm || (variant & 1); + bool alias = same_mm || (variant & 2), different = variant & 4; + struct vfio_account_mm *old = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_account_mm *next = same_mm ? old : vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f; + struct vfio_fragment_provider *prepared; + struct file *file, *replacement; + struct vfio_dma *dma; + struct page *pages[2]; + phys_addr_t external[2]; + unsigned long unique, addr; + unsigned char data[32]; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, old); + KUNIT_ASSERT_NOT_NULL(test, next); + file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x63); + replacement = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x44); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, replacement); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(old, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (alias) { + kthread_use_mm(old->mm); + addr = vm_mmap(file, VFIO_TEST_VA + PAGE_SIZE, PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA + PAGE_SIZE); + } + f = vfio_fragment_fixture_new(test); + KUNIT_ASSERT_NOT_NULL(test, f); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_test_map(f, old, 0, 2 * PAGE_SIZE), 0); + dma = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + KUNIT_ASSERT_NOT_NULL(test, dma); + KUNIT_ASSERT_NULL(test, dma->fragments); + if (different) { + kthread_use_mm(old->mm); + addr = vm_mmap(replacement, VFIO_TEST_VA, 2 * PAGE_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, 0); + kthread_unuse_mm(old->mm); + KUNIT_ASSERT_EQ(test, addr, VFIO_TEST_VA); + } + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 17, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + external[i] = page_to_phys(pages[i]); + if (mapped && different) + KUNIT_EXPECT_NE(test, external[0], + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + KUNIT_ASSERT_EQ(test, dma->locked_vm, 2UL); + if (!same_mm) { + old->live = false; + mmput(old->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&old->mm->mm_users), 0); + } + + prepared = vfio_fragments_prepare_native(f->iommu, dma); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, prepared); + unique = mapped ? (alias ? 1UL : 2UL) : 0; + if (!mapped || different) + unique += different || !alias ? 2UL : 1UL; + KUNIT_EXPECT_EQ(test, vfio_fragments_native_pages(prepared), unique); + /* Failed new-owner credit must not publish metadata or release pins. */ + if (!same_mm && task_rlimit(dma->task, RLIMIT_MEMLOCK) != RLIM_INFINITY) { + unsigned long budget = task_rlimit(dma->task, RLIMIT_MEMLOCK) >> + mm_page_shift(next->mm); + + next->mm->locked_vm = budget; + KUNIT_EXPECT_EQ(test, vfio_change_dma_mm_pages(dma, dma->task, + next->mm, false, unique), -ENOMEM); + KUNIT_EXPECT_PTR_EQ(test, dma->mm, old->mm); + KUNIT_EXPECT_EQ(test, dma->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), 2UL); + KUNIT_EXPECT_EQ(test, next->mm->locked_vm, budget); + next->mm->locked_vm = 0; + } + vfio_fragments_abort_native(prepared); + KUNIT_EXPECT_PTR_EQ(test, dma->fragments, NULL); + KUNIT_EXPECT_TRUE(test, folio_maybe_dma_pinned(page_folio(pages[0]))); + kthread_use_mm(next->mm); + ret = vfio_change_dma_owner(f->iommu, dma, + VFIO_TEST_VA + (same_mm ? SZ_4K : 0)); + kthread_unuse_mm(next->mm); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_NOT_NULL(test, dma->fragments); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(old->mm), same_mm ? unique : 0UL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_dma_rw(f->iommu, VFIO_TEST_IOVA, + data, sizeof(data), false), 0); + KUNIT_EXPECT_PTR_EQ(test, + memchr_inv(data, !mapped && different ? 0x44 : 0x63, sizeof(data)), NULL); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 31, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + for (unsigned int i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[i]), external[i]); + KUNIT_EXPECT_EQ(test, dma->locked_vm, unique); + KUNIT_ASSERT_EQ(test, vfio_iommu_replay(f->iommu, &f->domains[1].type1), 0); + list_add_tail(&f->domains[1].type1.next, &f->iommu->domain_list); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 31, 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 2); + KUNIT_EXPECT_EQ(test, dma->locked_vm, + mapped ? (alias ? 1UL : 2UL) : unique); + kunit_release_action(test, vfio_fragment_fixture_free, f); + KUNIT_EXPECT_EQ(test, mm_locked_vm_native_pages(next->mm), 0UL); + kunit_info(test, "native ownership variant=%u preserves PIN identity and unique charge", + variant); + } +} + +static int vfio_fragment_map_area(struct vfio_fragment_fixture *f, + struct vfio_account_mm *ctx, unsigned long iova_offset, + unsigned long va_offset, unsigned long length, unsigned int flags) +{ + struct vfio_iommu_type1_dma_map map = { + .argsz = sizeof(map), + .flags = flags, + .vaddr = VFIO_TEST_VA + va_offset, + .iova = VFIO_TEST_IOVA + iova_offset, + .size = length, + }; + int ret; + + kthread_use_mm(ctx->mm); + ret = vfio_dma_do_map(f->iommu, &map); + kthread_unuse_mm(ctx->mm); + return ret; +} + +static void vfio_fragments_external_span_test(struct kunit *test) +{ + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *a = vfio_account_mm_new(test, 12); + struct vfio_account_mm *b = vfio_account_mm_new(test, PAGE_SHIFT); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x6b); + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + struct vfio_dma *first, *rest; + struct page *pages[3]; + unsigned long bits = 2 * PAGE_SIZE / SZ_4K - 1; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(a, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(b, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, a, 0, 0, SZ_4K, rw), 0); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, b, SZ_4K, SZ_4K, + 2 * PAGE_SIZE - SZ_4K, rw), 0); + first = vfio_find_dma(f->iommu, VFIO_TEST_IOVA, 1); + rest = vfio_find_dma(f->iommu, VFIO_TEST_IOVA + SZ_4K, 1); + KUNIT_ASSERT_NOT_NULL(test, first); + KUNIT_ASSERT_NOT_NULL(test, rest); + f->iommu->dirty_page_tracking = true; + KUNIT_ASSERT_EQ(test, vfio_dma_bitmap_alloc_all(f->iommu, SZ_4K), 0); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA + 37, 2, + IOMMU_READ | IOMMU_WRITE, pages), 2); + KUNIT_EXPECT_EQ(test, first->locked_vm, 1UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, 2UL); + KUNIT_EXPECT_EQ(test, bitmap_weight(first->bitmap, 1), 1); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + bitmap_clear(rest->bitmap, 0, bits); + vfio_dma_populate_bitmap(rest, SZ_4K); + KUNIT_EXPECT_EQ(test, bitmap_weight(rest->bitmap, bits), (int)bits); + if (mapped) + KUNIT_EXPECT_EQ(test, page_to_phys(pages[0]), + iommu_iova_to_phys(&f->domains[0].core, VFIO_TEST_IOVA)); + /* A failed larger request must leave prior caller references intact. */ + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 3, IOMMU_READ, pages), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, pages[0], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[1], NULL); + KUNIT_EXPECT_PTR_EQ(test, pages[2], NULL); + a->live = b->live = false; + mmput(a->mm); + mmput(b->mm); + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 2, IOMMU_READ, pages), 2); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 2); + /* Release the second page first: it shares the second DMA owner. */ + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + PAGE_SIZE + 7, 1); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 1UL); + f->external_refs = 1; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, f->notifications, 1U); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(a->mm), 0UL); + KUNIT_EXPECT_EQ(test, rest->locked_vm, mapped ? 2UL : 0UL); + unmap.iova += SZ_4K; + unmap.size = 2 * PAGE_SIZE - SZ_4K; + KUNIT_ASSERT_EQ(test, vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(b->mm), 0UL); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + for (unsigned int failure = 0; failure < 4; failure++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, 12); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, 2 * PAGE_SIZE, 0x3a); + struct page *page = NULL; + struct rb_node *node; + unsigned long va = failure == 2 ? PAGE_SIZE + SZ_4K : SZ_4K; + int expected = failure == 1 ? -EPERM : -EINVAL; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, 2 * PAGE_SIZE), VFIO_TEST_VA); + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + if (failure == 3) + va += SZ_4K; + if (failure != 0) + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, SZ_4K, va, + SZ_4K, failure == 1 ? VFIO_DMA_MAP_FLAG_READ : rw), 0); + KUNIT_EXPECT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ | IOMMU_WRITE, &page), expected); + KUNIT_EXPECT_PTR_EQ(test, page, NULL); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->pfn_list)); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&dma->fragments->records)); + } + kunit_release_action(test, vfio_fragment_fixture_free, f); + } +} + +#define VFIO_STRESS_CYCLES 1024 + +struct vfio_pin_stress { + struct vfio_fragment_fixture *fixture; + struct vfio_device consumer; + struct vfio_device registrar; + struct vfio_device observer; + struct task_struct *competing_thread; + struct task_struct *pin_thread; + struct task_struct *register_thread; + struct mutex consumer_lock; + struct completion pin_go; + struct completion pin_ready; + struct completion notify_entered; + struct completion pin_done; + struct completion register_go; + struct completion register_entered; + struct completion register_done; + struct completion competing_go; + struct completion competing_done; + unsigned long competing_size; + int competing_ret; + bool competing; + unsigned int competing_unmaps; + unsigned int observations; + atomic_t error; + unsigned int pins; + unsigned int registrations; + bool stopping; +}; + +static void vfio_stress_error(struct vfio_pin_stress *s, int ret) +{ + atomic_cmpxchg(&s->error, 0, ret < 0 ? ret : -EIO); +} + +static void vfio_stress_notify(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), consumer); + struct vfio_fragment_fixture *f = s->fixture; + + /* The callback holds the production device-list lock, not the DMA lock. */ + complete(&s->register_go); + if (!wait_for_completion_timeout(&s->register_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + complete(&s->notify_entered); + mutex_lock(&s->consumer_lock); + f->notifications++; + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + if (s->competing) { + /* A second unmap may free the DMA while this callback still runs. */ + complete(&s->competing_go); + if (!wait_for_completion_timeout(&s->competing_done, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + } +} + +static void vfio_stress_observe(struct vfio_device *device, u64 iova, u64 length) +{ + struct vfio_pin_stress *s = container_of(device, typeof(*s), observer); + + s->observations++; + if (iova != VFIO_TEST_IOVA || length != SZ_4K) + vfio_stress_error(s, -ESTALE); +} + +static void vfio_stress_empty_notify(struct vfio_device *device, u64 iova, u64 length) +{ +} + +static const struct vfio_device_ops vfio_stress_consumer_ops = { + .dma_unmap = vfio_stress_notify, +}; + +static const struct vfio_device_ops vfio_stress_registrar_ops = { + .dma_unmap = vfio_stress_empty_notify, +}; + +static const struct vfio_device_ops vfio_stress_observer_ops = { + .dma_unmap = vfio_stress_observe, +}; + +static int vfio_stress_competing_unmaps(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + + wait_for_completion(&s->competing_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + s->competing_ret = vfio_dma_do_unmap(s->fixture->iommu, &unmap, &bitmap); + s->competing_size = unmap.size; + s->competing_unmaps++; + complete(&s->competing_done); + } + return 0; +} + +static int vfio_stress_pinner(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + for (;;) { + struct page *page = NULL, *original; + void *bytes; + int ret; + + wait_for_completion(&s->pin_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + mutex_lock(&s->consumer_lock); + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 37, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret != 1) { + vfio_stress_error(s, ret); + mutex_unlock(&s->consumer_lock); + complete(&s->pin_ready); + complete(&s->pin_done); + continue; + } + f->external_refs++; + s->pins++; + original = page; + complete(&s->pin_ready); + /* Unmap cannot revoke a reference before the consumer publishes it. */ + if (!wait_for_completion_timeout(&s->notify_entered, msecs_to_jiffies(5000))) + vfio_stress_error(s, -ETIMEDOUT); + if (!READ_ONCE(s->stopping)) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + VFIO_TEST_IOVA + 17, 1, IOMMU_READ, &page); + if (ret == 1) { + f->external_refs++; + s->pins++; + if (page != original) + vfio_stress_error(s, -ESTALE); + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 17, 1); + f->external_refs--; + } else { + vfio_stress_error(s, ret); + } + bytes = kmap_local_page(original); + if (memchr_inv(bytes, 0x6b, PAGE_SIZE)) + vfio_stress_error(s, -EIO); + kunmap_local(bytes); + } + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA + 37, 1); + f->external_refs--; + mutex_unlock(&s->consumer_lock); + complete(&s->pin_done); + } + return 0; +} + +static int vfio_stress_registrations(void *data) +{ + struct vfio_pin_stress *s = data; + + for (;;) { + wait_for_completion(&s->register_go); + if (READ_ONCE(s->stopping) || kthread_should_stop()) + break; + complete(&s->register_entered); + vfio_iommu_type1_register_device(s->fixture->iommu, &s->registrar); + vfio_iommu_type1_unregister_device(s->fixture->iommu, &s->registrar); + s->registrations++; + complete(&s->register_done); + } + return 0; +} + +static void vfio_stress_free(void *data) +{ + struct vfio_pin_stress *s = data; + struct vfio_fragment_fixture *f = s->fixture; + + WRITE_ONCE(s->stopping, true); + complete_all(&s->pin_go); + complete_all(&s->notify_entered); + complete_all(&s->register_go); + complete_all(&s->competing_go); + if (s->competing_thread) { + kthread_stop(s->competing_thread); + put_task_struct(s->competing_thread); + } + if (s->pin_thread) { + kthread_stop(s->pin_thread); + put_task_struct(s->pin_thread); + } + if (s->register_thread) { + kthread_stop(s->register_thread); + put_task_struct(s->register_thread); + } + mutex_lock(&s->consumer_lock); + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + mutex_unlock(&s->consumer_lock); + vfio_iommu_type1_unregister_device(f->iommu, &s->consumer); + vfio_iommu_type1_unregister_device(f->iommu, &s->observer); +} + +static struct vfio_pin_stress *vfio_stress_new(struct kunit *test, + struct vfio_fragment_fixture *f) +{ + struct vfio_pin_stress *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + + if (!s) + return NULL; + s->fixture = f; + mutex_init(&s->consumer_lock); + init_completion(&s->pin_go); + init_completion(&s->pin_ready); + init_completion(&s->notify_entered); + init_completion(&s->pin_done); + init_completion(&s->register_go); + init_completion(&s->register_entered); + init_completion(&s->register_done); + init_completion(&s->competing_go); + init_completion(&s->competing_done); + atomic_set(&s->error, 0); + s->consumer.ops = &vfio_stress_consumer_ops; + s->registrar.ops = &vfio_stress_registrar_ops; + s->observer.ops = &vfio_stress_observer_ops; + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + /* list_add puts the consumer before the observer in callback order. */ + vfio_iommu_type1_register_device(f->iommu, &s->observer); + vfio_iommu_type1_register_device(f->iommu, &s->consumer); + if (kunit_add_action_or_reset(test, vfio_stress_free, s)) + return NULL; + s->pin_thread = kthread_run(vfio_stress_pinner, s, "vfio-pin-stress"); + if (IS_ERR(s->pin_thread)) { + s->pin_thread = NULL; + return NULL; + } + get_task_struct(s->pin_thread); + s->register_thread = kthread_run(vfio_stress_registrations, s, "vfio-reg-stress"); + if (IS_ERR(s->register_thread)) { + s->register_thread = NULL; + return NULL; + } + get_task_struct(s->register_thread); + s->competing_thread = kthread_run(vfio_stress_competing_unmaps, s, "vfio-unmap-stress"); + if (IS_ERR(s->competing_thread)) { + s->competing_thread = NULL; + return NULL; + } + get_task_struct(s->competing_thread); + return s; +} + +static void vfio_fragments_concurrency_test(struct kunit *test) +{ + unsigned int shifts[] = { 12, 14, 16 }; + unsigned int rw = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + + for (unsigned int shift = 0; shift < ARRAY_SIZE(shifts); shift++) { + if (shifts[shift] > PAGE_SHIFT) + continue; + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_account_mm *ctx = vfio_account_mm_new(test, shifts[shift]); + struct vfio_fragment_fixture *f = vfio_fragment_fixture_new(test); + struct file *file = vfio_fragment_file(test, PAGE_SIZE, 0x6b); + struct vfio_pin_stress *s; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, f); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + KUNIT_ASSERT_EQ(test, vfio_fragment_source(ctx, file, PAGE_SIZE), + VFIO_TEST_VA); + s = vfio_stress_new(test, f); + KUNIT_ASSERT_NOT_NULL(test, s); + if (mapped) { + list_add_tail(&f->domains[0].type1.next, &f->iommu->domain_list); + vfio_update_pgsize_bitmap(f->iommu); + } + for (unsigned int cycle = 0; cycle < VFIO_STRESS_CYCLES; cycle++) { + struct vfio_iommu_type1_dma_unmap unmap = { + .argsz = sizeof(unmap), .iova = VFIO_TEST_IOVA, + .size = SZ_4K, + }; + struct vfio_bitmap bitmap = {}; + bool fine = cycle & 1; + unsigned long remaining = PAGE_SIZE - SZ_4K; + + KUNIT_ASSERT_EQ(test, + vfio_fragment_map_area(f, ctx, 0, 0, SZ_4K, rw), 0); + for (unsigned long offset = SZ_4K; offset < PAGE_SIZE;) { + unsigned long length = fine ? SZ_4K : remaining; + + KUNIT_ASSERT_EQ(test, vfio_fragment_map_area(f, ctx, offset, + offset, length, rw), 0); + offset += length; + } + if (cycle == VFIO_STRESS_CYCLES - 1) { + struct page *page; + + KUNIT_ASSERT_EQ(test, vfio_iommu_type1_pin_pages(f->iommu, + f->group.iommu_group, VFIO_TEST_IOVA, 1, + IOMMU_READ, &page), 1); + f->external_refs = 1; + ctx->live = false; + mmput(ctx->mm); + KUNIT_ASSERT_EQ(test, atomic_read(&ctx->mm->mm_users), 0); + } + s->competing = cycle & 2; + complete(&s->pin_go); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_ready, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, s->competing ? 0UL : SZ_4K); + if (s->competing) { + KUNIT_EXPECT_EQ(test, s->competing_ret, 0); + KUNIT_EXPECT_EQ(test, s->competing_size, SZ_4K); + } + unmap.iova += SZ_4K; + unmap.size = remaining; + KUNIT_ASSERT_EQ(test, + vfio_dma_do_unmap(f->iommu, &unmap, &bitmap), 0); + KUNIT_EXPECT_EQ(test, unmap.size, remaining); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->pin_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&s->register_done, + msecs_to_jiffies(5000)), 0UL); + KUNIT_ASSERT_EQ(test, atomic_read(&s->error), 0); + KUNIT_EXPECT_EQ(test, f->external_refs, 0U); + KUNIT_EXPECT_EQ(test, mm_locked_vm_bytes(ctx->mm), 0UL); + KUNIT_EXPECT_TRUE(test, RB_EMPTY_ROOT(&f->iommu->dma_list)); + KUNIT_EXPECT_EQ(test, f->domains[0].mapped, 0UL); + reinit_completion(&s->pin_ready); + reinit_completion(&s->notify_entered); + reinit_completion(&s->pin_done); + reinit_completion(&s->register_entered); + reinit_completion(&s->register_done); + reinit_completion(&s->competing_done); + } + KUNIT_EXPECT_EQ(test, s->pins, 2U * VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->registrations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, f->notifications, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->observations, (unsigned int)VFIO_STRESS_CYCLES); + KUNIT_EXPECT_EQ(test, s->competing_unmaps, VFIO_STRESS_CYCLES / 2U); + kunit_info(test, "%uK user/%luK native mapped=%u: %u forced pin/unmap/register cycles, half with competing unmap", + 1U << (shifts[shift] - 10), PAGE_SIZE >> 10, + mapped, VFIO_STRESS_CYCLES); + kunit_release_action(test, vfio_stress_free, s); + kunit_release_action(test, vfio_fragment_fixture_free, f); + } + } +} + +static struct kunit_case vfio_fragments_cases[] = { + KUNIT_CASE(vfio_fragments_lifetime_test), + KUNIT_CASE(vfio_fragments_lazy_test), + KUNIT_CASE(vfio_fragments_map_failure_test), + KUNIT_CASE(vfio_fragments_fine_test), + KUNIT_CASE(vfio_fragments_replay_failure_test), + KUNIT_CASE(vfio_fragments_anon_owner_test), + KUNIT_CASE(vfio_fragments_domain_granule_test), + KUNIT_CASE(vfio_fragments_native_owner_test), + KUNIT_CASE(vfio_fragments_external_span_test), + KUNIT_CASE(vfio_fragments_concurrency_test), + {} +}; + +static struct kunit_suite vfio_fragments_suite = { + .name = "vfio-type1-fragments", + .test_cases = vfio_fragments_cases, +}; + +kunit_test_suite(vfio_fragments_suite); diff --git a/drivers/vfio/vfio_iommu_type1_ioctl_test.c b/drivers/vfio/vfio_iommu_type1_ioctl_test.c new file mode 100644 index 00000000000000..33e981bcdf74d1 --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_ioctl_test.c @@ -0,0 +1,305 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Included after the KUnit harness; real EL0 arguments, recorded domains only. */ +#include +#include +#include +#include "vfio_iommu_type1_test.h" + +struct vfio_ioctl_fixture { + struct vfio_fragment_fixture pins; + struct mutex request_lock; +}; + +struct vfio_ioctl_rw { + struct work_struct work; + struct vfio_iommu *iommu; + dma_addr_t iova; + void *data; + size_t size; + bool write; + int ret; +}; + +static void vfio_ioctl_rw_work(struct work_struct *work) +{ + struct vfio_ioctl_rw *rw = container_of(work, typeof(*rw), work); + + rw->ret = vfio_iommu_type1_dma_rw(rw->iommu, rw->iova, rw->data, + rw->size, rw->write); +} + +static int vfio_ioctl_test_open(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx; + struct vfio_fragment_fixture *f; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + f = &ctx->pins; + f->iommu = vfio_iommu_type1_open(VFIO_TYPE1v2_IOMMU); + if (IS_ERR(f->iommu)) { + int ret = PTR_ERR(f->iommu); + + kfree(ctx); + return ret; + } + mutex_init(&ctx->request_lock); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + struct vfio_test_domain *mock = &f->domains[i]; + + xa_init(&mock->translations); + mock->core.ops = &vfio_test_domain_ops; + mock->core.type = IOMMU_DOMAIN_UNMANAGED; + mock->core.pgsize_bitmap = SZ_4K | PAGE_SIZE; + mock->type1.domain = &mock->core; + INIT_LIST_HEAD(&mock->type1.next); + INIT_LIST_HEAD(&mock->type1.group_list); + } + f->group.iommu_group = (struct iommu_group *)ctx; + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + f->iommu->num_non_pinned_groups = 1; + f->device.ops = &vfio_test_device_ops; + vfio_iommu_type1_register_device(f->iommu, &f->device); + file->private_data = ctx; + return 0; +} + +static int vfio_ioctl_test_release(struct inode *inode, struct file *file) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + + while (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, VFIO_TEST_IOVA, 1); + f->external_refs--; + } + vfio_iommu_type1_unregister_device(f->iommu, &f->device); + vfio_iommu_unmap_unpin_all(f->iommu); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) { + list_del_init(&f->domains[i].type1.next); + xa_destroy(&f->domains[i].translations); + } + list_del(&f->group.next); + vfio_iommu_type1_release(f->iommu); + kfree(ctx); + return 0; +} + +static int vfio_ioctl_test_domain(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd, bool add) +{ + struct vfio_test_domain *mock; + struct rb_node *node; + int ret = 0; + + if (cmd->domain >= ARRAY_SIZE(f->domains) || cmd->flags) + return -EINVAL; + mock = &f->domains[cmd->domain]; + mutex_lock(&f->iommu->lock); + if (add) { + if (!list_empty(&mock->type1.next)) { + ret = -EEXIST; + goto out; + } + if (cmd->size != SZ_4K && cmd->size != SZ_16K && cmd->size != SZ_64K) { + ret = -EINVAL; + goto out; + } + mock->core.pgsize_bitmap = cmd->size; + mock->fail_after = cmd->fail_after; + ret = vfio_iommu_replay(f->iommu, &mock->type1); + if (ret) + goto out; + list_add_tail(&mock->type1.next, &f->iommu->domain_list); + } else { + if (list_empty(&mock->type1.next)) { + ret = -ENOENT; + goto out; + } + if (list_is_singular(&f->iommu->domain_list)) { + vfio_iommu_unmap_unpin_reaccount(f->iommu); + } else { + for (node = rb_first(&f->iommu->dma_list); node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + iommu_unmap(&mock->core, dma->iova, dma->size); + } + } + list_del_init(&mock->type1.next); + } + vfio_update_pgsize_bitmap(f->iommu); +out: + mutex_unlock(&f->iommu->lock); + return ret; +} + +static int vfio_ioctl_test_rw(struct vfio_fragment_fixture *f, + struct vfio_type1_test_cmd *cmd) +{ + struct vfio_ioctl_rw rw = { + .iommu = f->iommu, .iova = cmd->iova, .size = cmd->size, + .write = cmd->flags & VFIO_TYPE1_TEST_WRITE, + }; + void __user *buffer = u64_to_user_ptr(cmd->buffer); + int ret; + + if (!cmd->size || cmd->size > 256 || + cmd->flags & ~(VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER)) + return -EINVAL; + rw.data = kzalloc(cmd->size, GFP_KERNEL); + if (!rw.data) + return -ENOMEM; + if (rw.write && copy_from_user(rw.data, buffer, cmd->size)) { + ret = -EFAULT; + goto out; + } + if (cmd->flags & VFIO_TYPE1_TEST_WORKER) { + INIT_WORK(&rw.work, vfio_ioctl_rw_work); + queue_work(system_unbound_wq, &rw.work); + flush_work(&rw.work); + ret = rw.ret; + } else { + ret = vfio_iommu_type1_dma_rw(f->iommu, rw.iova, rw.data, rw.size, rw.write); + } + if (!ret && !rw.write && copy_to_user(buffer, rw.data, cmd->size)) + ret = -EFAULT; +out: + kfree(rw.data); + return ret; +} + +static long vfio_ioctl_test_ioctl(struct file *file, unsigned int request, + unsigned long arg) +{ + struct vfio_ioctl_fixture *ctx = file->private_data; + struct vfio_fragment_fixture *f = &ctx->pins; + struct vfio_type1_test_cmd cmd; + void __user *user = (void __user *)arg; + struct page *page; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* Request serialization belongs to this fixture, not production type1. */ + mutex_lock(&ctx->request_lock); + switch (request) { + case VFIO_CHECK_EXTENSION: + case VFIO_IOMMU_GET_INFO: + case VFIO_IOMMU_MAP_DMA: + case VFIO_IOMMU_UNMAP_DMA: + case VFIO_IOMMU_DIRTY_PAGES: + ret = vfio_iommu_type1_ioctl(f->iommu, request, arg); + goto out; + default: + break; + } + if (copy_from_user(&cmd, user, sizeof(cmd))) { + ret = -EFAULT; + goto out; + } + if (cmd.reserved) { + ret = -EINVAL; + goto out; + } + switch (request) { + case VFIO_TYPE1_TEST_DOMAIN_ADD: + case VFIO_TYPE1_TEST_DOMAIN_DEL: + ret = vfio_ioctl_test_domain(f, &cmd, request == VFIO_TYPE1_TEST_DOMAIN_ADD); + break; + case VFIO_TYPE1_TEST_RW: + ret = vfio_ioctl_test_rw(f, &cmd); + break; + case VFIO_TYPE1_TEST_PIN: + case VFIO_TYPE1_TEST_UNPIN: + if (ALIGN_DOWN(cmd.iova, PAGE_SIZE) != VFIO_TEST_IOVA || cmd.flags) { + ret = -EINVAL; + break; + } + if (request == VFIO_TYPE1_TEST_PIN) { + ret = vfio_iommu_type1_pin_pages(f->iommu, f->group.iommu_group, + cmd.iova, 1, IOMMU_READ | IOMMU_WRITE, &page); + if (ret == 1) { + f->external_refs++; + ret = 0; + } + } else if (f->external_refs) { + vfio_iommu_type1_unpin_pages(f->iommu, cmd.iova, 1); + f->external_refs--; + } else { + ret = -ENOENT; + } + break; + case VFIO_TYPE1_TEST_EMULATED: + if (cmd.flags > 1 || f->external_refs) { + ret = -EBUSY; + break; + } + mutex_lock(&f->iommu->lock); + if (cmd.flags && list_empty(&f->group.next)) { + list_add(&f->group.next, &f->iommu->emulated_iommu_groups); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups++; + } else if (!cmd.flags && !list_empty(&f->group.next)) { + list_del_init(&f->group.next); + if (!f->group.pinned_page_dirty_scope) + f->iommu->num_non_pinned_groups--; + } + mutex_unlock(&f->iommu->lock); + break; + case VFIO_TYPE1_TEST_STATE: + cmd.locked_bytes = 0; + cmd.owner_locked_bytes = mm_locked_vm_bytes(current->mm); + cmd.flags = 0; + cmd.owner_users = 0; + mutex_lock(&f->iommu->lock); + for (struct rb_node *node = rb_first(&f->iommu->dma_list); + node; node = rb_next(node)) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + cmd.locked_bytes += dma->locked_vm * PAGE_SIZE; + cmd.owner_locked_bytes = mm_locked_vm_bytes(dma->mm); + cmd.owner_users = atomic_read(&dma->mm->mm_users); + if (dma->fragments) + cmd.flags |= VFIO_TYPE1_TEST_STATE_FRAGMENTS; + } + cmd.native_page_size = PAGE_SIZE; + cmd.minimum_page_size = 1UL << __ffs(f->iommu->pgsize_bitmap); + for (unsigned int i = 0; i < ARRAY_SIZE(f->domains); i++) + cmd.mapped_bytes[i] = f->domains[i].mapped; + mutex_unlock(&f->iommu->lock); + cmd.notifications = f->notifications; + cmd.external_refs = f->external_refs; + break; + default: + ret = -ENOTTY; + } + if (!ret && copy_to_user(user, &cmd, sizeof(cmd))) + ret = -EFAULT; +out: + mutex_unlock(&ctx->request_lock); + return ret; +} + +static const struct file_operations vfio_ioctl_test_fops = { + .owner = THIS_MODULE, + .open = vfio_ioctl_test_open, + .release = vfio_ioctl_test_release, + .unlocked_ioctl = vfio_ioctl_test_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static struct miscdevice vfio_ioctl_test_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "aurora-vfio-type1-test", + .fops = &vfio_ioctl_test_fops, + .mode = 0600, +}; + +static int __init vfio_ioctl_test_init(void) +{ + return misc_register(&vfio_ioctl_test_device); +} +late_initcall(vfio_ioctl_test_init); diff --git a/drivers/vfio/vfio_iommu_type1_test.h b/drivers/vfio/vfio_iommu_type1_test.h new file mode 100644 index 00000000000000..36c09030bcf0ef --- /dev/null +++ b/drivers/vfio/vfio_iommu_type1_test.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* Private disposable-guest test protocol, not a supported VFIO ABI. */ +#ifndef VFIO_IOMMU_TYPE1_TEST_H +#define VFIO_IOMMU_TYPE1_TEST_H +#include +#include + +struct vfio_type1_test_cmd { + __u64 iova; + __u64 size; + __u64 buffer; + __u64 fail_after; + __u64 locked_bytes; + __u64 owner_locked_bytes; + __u64 mapped_bytes[2]; + __u64 native_page_size; + __u64 minimum_page_size; + __u32 domain; + __u32 flags; + __u32 owner_users; + __u32 notifications; + __u32 external_refs; + __u32 reserved; +}; + +#define VFIO_TYPE1_TEST_DOMAIN_ADD _IOWR('V', 1, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_DOMAIN_DEL _IOWR('V', 2, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_RW _IOWR('V', 3, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_PIN _IOWR('V', 4, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_UNPIN _IOWR('V', 5, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE _IOWR('V', 6, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_EMULATED _IOWR('V', 7, struct vfio_type1_test_cmd) +#define VFIO_TYPE1_TEST_STATE_FRAGMENTS 1U +#define VFIO_TYPE1_TEST_WRITE 1U +#define VFIO_TYPE1_TEST_WORKER 2U +#endif From cff77cc21d8039bb0a94fcea784b2db2ef265692 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 024/631] selftests: exercise the mixed userspace page-size ABI Cover exec policy, AArch32 fallback, mappings, sharing, reclaim, accounting and converted I/O interfaces. Keep extended PID1 fixtures separate from the ordinary page-contract executable. Signed-off-by: Diljit Singh (cherry picked from commit ad36aab5acc6e025db2a66192d259a1058d77d02) --- .../testing/kunit/configs/arm64_user4k.config | 33 + .../configs/arm64_user4k_data_kasan.config | 18 + .../configs/arm64_user4k_mm_kasan.config | 38 + tools/testing/selftests/arm64/Makefile | 2 +- .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 134 ++ tools/testing/selftests/arm64/user4k/advice.c | 199 +++ .../selftests/arm64/user4k/aio-ring-init.c | 178 +++ .../arm64/user4k/aslr-fixture-init.c | 175 +++ .../selftests/arm64/user4k/bpf-uptr-init.c | 583 ++++++++ .../arm64/user4k/coarse-shmem-init.c | 223 +++ .../arm64/user4k/compat-aslr-probe.c | 134 ++ .../arm64/user4k/compat-fixture-init.c | 149 ++ .../arm64/user4k/compat-mlock-init.c | 62 + .../arm64/user4k/compat-mlock-probe.c | 142 ++ .../user4k/compat-numa-balancing-probe.c | 223 +++ .../arm64/user4k/compat-numa-probe.c | 117 ++ .../arm64/user4k/compat-perf-probe.c | 107 ++ .../arm64/user4k/compat-policy-probe.c | 66 + .../selftests/arm64/user4k/compat-probe.c | 248 ++++ .../selftests/arm64/user4k/compat-script | 1 + .../selftests/arm64/user4k/compat-shm-probe.c | 111 ++ .../selftests/arm64/user4k/compat-uffd-init.c | 154 ++ .../arm64/user4k/compat-uffd-probe.c | 149 ++ .../arm64/user4k/compat-vfio-probe.c | 250 ++++ .../arm64/user4k/context-stress-init.c | 275 ++++ .../arm64/user4k/core-fixture-init.c | 447 ++++++ .../selftests/arm64/user4k/cpu-granule-init.c | 187 +++ .../arm64/user4k/cpu-granule16-init.c | 192 +++ .../selftests/arm64/user4k/dio-fixture-init.c | 266 ++++ .../arm64/user4k/exec-fixture-init.c | 167 +++ .../selftests/arm64/user4k/exec-policy-init.c | 214 +++ .../arm64/user4k/fex-contract-init.c | 52 + .../selftests/arm64/user4k/fex-native-abi.c | 20 + .../selftests/arm64/user4k/fex-x86-contract.c | 132 ++ .../selftests/arm64/user4k/gem-fixture-init.c | 325 ++++ .../arm64/user4k/granule-proc-init.c | 141 ++ .../arm64/user4k/granule-swap-init.c | 248 ++++ .../arm64/user4k/granule-threeway-init.c | 248 ++++ .../selftests/arm64/user4k/iommufd-init.c | 1314 +++++++++++++++++ .../arm64/user4k/lazyfree-fixture-init.c | 274 ++++ .../arm64/user4k/memcg-accounting-init.c | 153 ++ .../arm64/user4k/memcg-pressure-init.c | 417 ++++++ .../arm64/user4k/memory-interfaces.c | 165 +++ .../arm64/user4k/mglru-granule-init.c | 365 +++++ .../selftests/arm64/user4k/mixed-process.c | 209 +++ .../arm64/user4k/mlock-fixture-init.c | 249 ++++ .../arm64/user4k/numa-balancing-init.c | 307 ++++ .../selftests/arm64/user4k/numa-policy-init.c | 236 +++ .../selftests/arm64/user4k/page-contract.c | 232 +++ .../selftests/arm64/user4k/perf-ring-init.c | 323 ++++ .../selftests/arm64/user4k/proc-args.c | 164 ++ .../selftests/arm64/user4k/proc-memory.c | 340 +++++ .../selftests/arm64/user4k/proc-pages.c | 318 ++++ .../selftests/arm64/user4k/reclaim-advice.c | 366 +++++ .../selftests/arm64/user4k/runtime-app-init.c | 76 + .../selftests/arm64/user4k/runtime-app.py | 91 ++ .../arm64/user4k/shmem-collapse-init.c | 396 +++++ .../selftests/arm64/user4k/shmem-hole-init.c | 358 +++++ .../selftests/arm64/user4k/shmem-pin-init.c | 358 +++++ .../arm64/user4k/shmem-pressure-init.c | 225 +++ .../selftests/arm64/user4k/shmem-race-init.c | 308 ++++ .../selftests/arm64/user4k/shmem-thp-init.c | 271 ++++ .../arm64/user4k/shmem-uffd-swap-init.c | 160 ++ .../arm64/user4k/splice-fixture-init.c | 610 ++++++++ .../arm64/user4k/swap-fixture-init.c | 216 +++ .../selftests/arm64/user4k/swapfile-fixture.h | 50 + .../selftests/arm64/user4k/sysv-shm-init.c | 229 +++ .../arm64/user4k/thread-fixture-init.c | 424 ++++++ .../selftests/arm64/user4k/uffd-move-init.c | 451 ++++++ .../selftests/arm64/user4k/uffd-wp-init.c | 913 ++++++++++++ .../selftests/arm64/user4k/uprobes-init.c | 318 ++++ .../arm64/user4k/uring-fixture-init.c | 467 ++++++ .../selftests/arm64/user4k/user-events-init.c | 268 ++++ .../arm64/user4k/user16k-smoke-init.c | 229 +++ .../arm64/user4k/vfio-fixture-init.c | 542 +++++++ .../selftests/arm64/user4k/vhost-log-init.c | 392 +++++ .../selftests/arm64/user4k/wine-kernel32.def | 20 + .../arm64/user4k/wine-page-contract.c | 181 +++ .../selftests/arm64/user4k/x18-probe.S | 25 + tools/testing/vma/include/dup.h | 117 ++ tools/testing/vma/tests/merge.c | 2 +- tools/testing/vma/tests/mmap.c | 8 +- tools/testing/vma/tests/vma.c | 4 +- 84 files changed, 19546 insertions(+), 8 deletions(-) create mode 100644 tools/testing/kunit/configs/arm64_user4k.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_data_kasan.config create mode 100644 tools/testing/kunit/configs/arm64_user4k_mm_kasan.config create mode 100644 tools/testing/selftests/arm64/user4k/.gitignore create mode 100644 tools/testing/selftests/arm64/user4k/Makefile create mode 100644 tools/testing/selftests/arm64/user4k/advice.c create mode 100644 tools/testing/selftests/arm64/user4k/aio-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/aslr-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/bpf-uptr-init.c create mode 100644 tools/testing/selftests/arm64/user4k/coarse-shmem-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-aslr-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-mlock-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-numa-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-perf-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-policy-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-script create mode 100644 tools/testing/selftests/arm64/user4k/compat-shm-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-uffd-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/compat-vfio-probe.c create mode 100644 tools/testing/selftests/arm64/user4k/context-stress-init.c create mode 100644 tools/testing/selftests/arm64/user4k/core-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/cpu-granule16-init.c create mode 100644 tools/testing/selftests/arm64/user4k/dio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/exec-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-contract-init.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-native-abi.c create mode 100644 tools/testing/selftests/arm64/user4k/fex-x86-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/gem-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-proc-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/granule-threeway-init.c create mode 100644 tools/testing/selftests/arm64/user4k/iommufd-init.c create mode 100644 tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-accounting-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memcg-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/memory-interfaces.c create mode 100644 tools/testing/selftests/arm64/user4k/mglru-granule-init.c create mode 100644 tools/testing/selftests/arm64/user4k/mixed-process.c create mode 100644 tools/testing/selftests/arm64/user4k/mlock-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-balancing-init.c create mode 100644 tools/testing/selftests/arm64/user4k/numa-policy-init.c create mode 100644 tools/testing/selftests/arm64/user4k/page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/perf-ring-init.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-args.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-memory.c create mode 100644 tools/testing/selftests/arm64/user4k/proc-pages.c create mode 100644 tools/testing/selftests/arm64/user4k/reclaim-advice.c create mode 100644 tools/testing/selftests/arm64/user4k/runtime-app-init.c create mode 100755 tools/testing/selftests/arm64/user4k/runtime-app.py create mode 100644 tools/testing/selftests/arm64/user4k/shmem-collapse-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-hole-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pin-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-pressure-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-race-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-thp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c create mode 100644 tools/testing/selftests/arm64/user4k/splice-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swap-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/swapfile-fixture.h create mode 100644 tools/testing/selftests/arm64/user4k/sysv-shm-init.c create mode 100644 tools/testing/selftests/arm64/user4k/thread-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-move-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uffd-wp-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uprobes-init.c create mode 100644 tools/testing/selftests/arm64/user4k/uring-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user-events-init.c create mode 100644 tools/testing/selftests/arm64/user4k/user16k-smoke-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vfio-fixture-init.c create mode 100644 tools/testing/selftests/arm64/user4k/vhost-log-init.c create mode 100644 tools/testing/selftests/arm64/user4k/wine-kernel32.def create mode 100644 tools/testing/selftests/arm64/user4k/wine-page-contract.c create mode 100644 tools/testing/selftests/arm64/user4k/x18-probe.S diff --git a/tools/testing/kunit/configs/arm64_user4k.config b/tools/testing/kunit/configs/arm64_user4k.config new file mode 100644 index 00000000000000..de6c8ec2fdff82 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k.config @@ -0,0 +1,33 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_data_kasan.config b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config new file mode 100644 index 00000000000000..2e3e50bd677683 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_data_kasan.config @@ -0,0 +1,18 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_DATA_KUNIT_TEST=y +# CONFIG_ARM64_USER4K_KUNIT_TEST is not set +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y diff --git a/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config new file mode 100644 index 00000000000000..11b9935770f537 --- /dev/null +++ b/tools/testing/kunit/configs/arm64_user4k_mm_kasan.config @@ -0,0 +1,38 @@ +CONFIG_KUNIT=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_ARM64_PA_BITS_48=y +CONFIG_ARM64_USER4K_EXPERIMENTAL=y +CONFIG_ARM64_USER4K_KUNIT_TEST=y +CONFIG_ARM64_SW_TTBR0_PAN=y +# CONFIG_ARM64_CNP is not set +# CONFIG_ARM64_POE is not set +CONFIG_SMP=y +CONFIG_PRINTK=y +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_POWER_RESET=y +CONFIG_POWER_RESET_SYSCON=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_TMPFS=y +CONFIG_TRANSPARENT_HUGEPAGE=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DEBUG_VM=y +CONFIG_DEBUG_VM_MAPLE_TREE=y +CONFIG_DEBUG_VM_RB=y +CONFIG_PROVE_LOCKING=y +CONFIG_PAGE_TABLE_CHECK=y +CONFIG_PAGE_TABLE_CHECK_ENFORCED=y +# Guest-only infrastructure for late swap/reclaim fixtures. +CONFIG_BLOCK=y +CONFIG_VIRTIO=y +CONFIG_VIRTIO_MMIO=y +CONFIG_VIRTIO_BLK=y +CONFIG_DEVTMPFS=y +CONFIG_DEBUG_FS=y +CONFIG_KUNIT_DEBUGFS=y + +# Software MM fixtures only; the hardware boot opt-in is rejected under KASAN. +CONFIG_KASAN=y +CONFIG_KASAN_GENERIC=y +CONFIG_KASAN_INLINE=y diff --git a/tools/testing/selftests/arm64/Makefile b/tools/testing/selftests/arm64/Makefile index e456f3b62fa138..d8948f5fb1ef86 100644 --- a/tools/testing/selftests/arm64/Makefile +++ b/tools/testing/selftests/arm64/Makefile @@ -4,7 +4,7 @@ ARCH ?= $(shell uname -m 2>/dev/null || echo not) ifneq (,$(filter $(ARCH),aarch64 arm64)) -ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs +ARM64_SUBTARGETS ?= tags signal pauth fp mte bti abi gcs user4k else ARM64_SUBTARGETS := endif diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore new file mode 100644 index 00000000000000..2e6224b832f98e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -0,0 +1,3 @@ +page-contract + +swap-fixture-init diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile new file mode 100644 index 00000000000000..cc89cc93293fed --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: GPL-2.0 +TEST_GEN_PROGS := page-contract proc-args +TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init +TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +CFLAGS += -Wall -Wextra -O2 -ffixed-x18 +TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe +TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe +TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe +TEST_FILES += compat-script runtime-app.py +include ../../lib.mk + +$(OUTPUT)/page-contract: page-contract.c x18-probe.S + +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread + +# Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. +AARCH32_CC ?= clang --target=arm-linux-gnueabihf +AARCH32_FLAGS := -march=armv8-a -marm -mfloat-abi=soft -ffreestanding \ + -fno-builtin -fno-stack-protector -O2 -nostdlib -static -fuse-ld=lld +$(OUTPUT)/compat-probe-4k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=4096 $< -o $@ +$(OUTPUT)/compat-probe-64k: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ +$(OUTPUT)/compat-interp32: compat-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536,--image-base=0x10000000 $< -o $@ +$(OUTPUT)/compat-probe-dynamic: compat-probe.c + $(AARCH32_CC) $(filter-out -static,$(AARCH32_FLAGS)) -no-pie \ + -Wl,-z,max-page-size=65536,--dynamic-linker=/interp32 $< -o $@ + +$(OUTPUT)/compat-aslr-probe: compat-aslr-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/compat-mlock-probe: compat-mlock-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# The disposable PID1 fixture also runs with the native 64K process ABI. +$(OUTPUT)/mglru-granule-init $(OUTPUT)/memcg-pressure-init $(OUTPUT)/memcg-accounting-init $(OUTPUT)/runtime-app-init $(OUTPUT)/fex-contract-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# FEX fixture: the AArch64 emulator runs the freestanding x86-64 contract. +X86_64_CC ?= clang --target=x86_64-linux-gnu +$(OUTPUT)/fex-x86-contract: fex-x86-contract.c + $(X86_64_CC) -O2 -fno-builtin -fno-stack-protector -nostdlib -static \ + -fuse-ld=lld -Wl,-z,max-page-size=4096,-e,_start $< -o $@ + +$(OUTPUT)/libfex-abi.so: fex-native-abi.c + $(CC) $(CFLAGS) -shared -fPIC -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/compat-policy-probe: compat-policy-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Optional Windows payload; requires Wine/FEX in a disposable guest to execute. +# Keep its toolchain out of the ordinary Linux selftest build. +WINDOWS_CC ?= clang --target=x86_64-pc-windows-msvc +WINDOWS_DLLTOOL ?= llvm-dlltool +WINDOWS_LD ?= lld-link +.PHONY: wine-contract +wine-contract: $(OUTPUT)/wine-page-contract.exe +$(OUTPUT)/wine-kernel32.lib: wine-kernel32.def + $(WINDOWS_DLLTOOL) -m i386:x86-64 -d $< -l $@ +$(OUTPUT)/wine-page-contract.obj: wine-page-contract.c + $(WINDOWS_CC) -O2 -Wall -Wextra -Werror -ffreestanding -fno-stack-protector -c $< -o $@ +$(OUTPUT)/wine-page-contract.exe: $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + $(WINDOWS_LD) /entry:mainCRTStartup /subsystem:console /nodefaultlib /out:$@ $^ +EXTRA_CLEAN += $(OUTPUT)/wine-page-contract.exe $(OUTPUT)/wine-page-contract.obj $(OUTPUT)/wine-kernel32.lib + +$(OUTPUT)/uffd-wp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-wp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-uffd-swap-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/compat-uffd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-uffd-probe: compat-uffd-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/shmem-hole-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-hole-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-thp-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-thp-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-collapse-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/shmem-pin-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-pin-init: LDLIBS += -pthread + +$(OUTPUT)/shmem-race-init $(OUTPUT)/shmem-pressure-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/shmem-race-init: LDLIBS += -pthread + +$(OUTPUT)/uffd-move-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/uffd-move-init: LDLIBS += -pthread + +$(OUTPUT)/user-events-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/uprobes-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/vhost-log-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/bpf-uptr-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/coarse-shmem-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +$(OUTPUT)/iommufd-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/iommufd-init: LDLIBS += -pthread +$(OUTPUT)/iommufd-init: CFLAGS += $(KHDR_INCLUDES) + +# Privileged recording-domain fixture; run only in a disposable test VM. +$(OUTPUT)/vfio-fixture-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-vfio-probe: compat-vfio-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/sysv-shm-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-shm-probe: compat-shm-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +# Disposable guests exercise perf layout and software sampling across mm granules. +$(OUTPUT)/perf-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-perf-probe: compat-perf-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/aio-ring-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 + +# Requires a disposable guest with two memory nodes and CONFIG_COMPAT. +$(OUTPUT)/numa-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/compat-numa-probe: compat-numa-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ + +$(OUTPUT)/numa-balancing-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 +$(OUTPUT)/numa-balancing-init: LDLIBS += -pthread + +$(OUTPUT)/compat-numa-balancing-probe: compat-numa-balancing-probe.c + $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/advice.c b/tools/testing/selftests/arm64/user4k/advice.c new file mode 100644 index 00000000000000..4d4cb89350985d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/advice.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static bool wait_child(pid_t pid, int sig) +{ + int status = 0; + + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return false; + if (sig) + return WIFSIGNALED(status) && WTERMSIG(status) == sig; + return WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void *anon(size_t length, int prot, int extra) +{ + void *p = mmap(NULL, length, prot, MAP_PRIVATE | MAP_ANONYMOUS | extra, + -1, 0); + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + return p; +} + +static void discard_and_guards(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x61, 3 * page); + ok = madvise(p + page, 1, MADV_DONTNEED) == 0; + ok &= p[0] == 0x61 && p[page] == 0 && p[2 * page] == 0x61; + ksft_test_result(ok, + "DONTNEED discards exactly one rounded user page\n"); + memset(p + page, 0x71, page); + child = fork(); + if (!child) { + ok = madvise(p + page, page, MADV_DONTNEED) == 0; + _exit(!(ok && p[page] == 0 && p[0] == 0x61)); + } + ksft_test_result( + wait_child(child, 0) && p[page] == 0x71, + "discard in a fork child leaves the parent quarter intact\n"); + ok = madvise(p + page, 1, MADV_GUARD_INSTALL) == 0; + ok &= madvise(p + page, page, MADV_DONTNEED) == 0; + child = fork(); + if (!child) { + volatile unsigned char value = p[page]; + + (void)value; + _exit(1); + } + ksft_test_result(ok && wait_child(child, SIGSEGV), + "a user-page guard survives discard and fork\n"); + ok = madvise(p + page, 1, MADV_GUARD_REMOVE) == 0; + ok &= p[page] == 0 && p[0] == 0x61 && p[2 * page] == 0x61; + ksft_test_result(ok, + "guard removal restores only the requested page\n"); + munmap(p, 3 * page); +} + +static void population(void) +{ + unsigned char *p = anon(5 * page, PROT_READ, 0), vec[5]; + bool ok; + unsigned int i; + + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + ok &= madvise(p + 2 * page, 1, MADV_POPULATE_WRITE) == 0; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, "POPULATE_WRITE faults only the selected user-page VMA\n"); + ok = madvise(p + page, 1, MADV_POPULATE_READ) == 0; + ok &= mincore(p + page, page, vec) == 0 && vec[0] == 1; + errno = 0; + ok &= madvise(p, page, MADV_POPULATE_WRITE) == -1 && errno == EINVAL; + ok &= mprotect(p + 3 * page, page, PROT_NONE) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == EINVAL; + ok &= munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= madvise(p + 3 * page, page, MADV_POPULATE_READ) == -1 && + errno == ENOMEM; + ksft_test_result( + ok, "POPULATE_READ honors permissions and reports holes\n"); + munmap(p, 5 * page); + p = anon(5 * page, PROT_READ | PROT_WRITE, MAP_POPULATE); + ok = mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 1; + ksft_test_result(ok, "MAP_POPULATE materializes every user page\n"); + munmap(p, 5 * page); +} + +static void fork_advice(void) +{ + unsigned char *p = anon(3 * page, PROT_READ | PROT_WRITE, 0); + bool ok; + pid_t child; + + memset(p, 0x58, 3 * page); + ok = madvise(p + page, 1, MADV_WIPEONFORK) == 0; + child = fork(); + if (!child) + _exit(!(p[page] == 0 && p[0] == 0x58 && p[2 * page] == 0x58)); + ok &= wait_child(child, 0) && p[page] == 0x58; + ksft_test_result( + ok, + "WIPEONFORK clears a single child page and preserves neighbors\n"); + ok = madvise(p + page, 1, MADV_KEEPONFORK) == 0; + ok &= madvise(p + page, 1, MADV_DONTFORK) == 0; + child = fork(); + if (!child) { + unsigned char vec; + + errno = 0; + _exit(!(mincore(p + page, page, &vec) == -1 && + errno == ENOMEM && p[0] == 0x58 && + p[2 * page] == 0x58)); + } + ok &= wait_child(child, 0); + ksft_test_result(ok, "DONTFORK omits only its selected user page\n"); + munmap(p, 3 * page); +} + +static void remove_file(void) +{ + int fd = memfd_create("advice-file", 0); + unsigned char *shared, *alias, *private; + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 5 * page)) + ksft_exit_fail_msg("memfd setup\n"); + shared = + mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + alias = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + private = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, + page); + if (shared == MAP_FAILED || alias == MAP_FAILED || + private == MAP_FAILED) + ksft_exit_fail_msg("file mmap\n"); + for (i = 0; i < 5 * page; i++) + shared[i] = 0x40 + i / page; + private[page] = 0x79; + ok = madvise(alias + page, 1, MADV_REMOVE) == 0; + for (i = 0; i < 5 * page; i++) + ok &= shared[i] == (i / page == 2 ? 0 : 0x40 + i / page); + ok &= private[page] == 0x79; + ksft_test_result( + ok, + "REMOVE uses the file byte offset and preserves private COW\n"); + ok = madvise(alias, 3 * page, MADV_WILLNEED) == 0; + ksft_test_result( + ok, "WILLNEED accepts a file range with the process granule\n"); + munmap(shared, 5 * page); + munmap(alias, 3 * page); + munmap(private, 3 * page); + close(fd); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + if (setrlimit(RLIMIT_CORE, &no_core)) + return 1; + ksft_print_header(); + ksft_set_plan(11); + discard_and_guards(); + population(); + fork_advice(); + remove_file(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/aio-ring-init.c b/tools/testing/selftests/arm64/user4k/aio-ring-init.c new file mode 100644 index 00000000000000..74f34d16cac299 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aio-ring-init.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: legacy AIO ring relocation with per-process page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct aio_ring_header { + unsigned int id, nr, head, tail; + unsigned int magic, compat_features, incompat_features, header_length; +}; + +static void check(int good, const char *expression, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static void complete(aio_context_t context, struct iocb *request, + unsigned long bytes, int event_fd) +{ + struct io_event event; + struct timespec timeout = { .tv_sec = 5 }; + uint64_t notifications; + + CHECK(syscall(SYS_io_getevents, context, 1, 1, &event, &timeout) == 1); + CHECK(event.data == request->aio_data && + event.obj == (uintptr_t)request); + CHECK(event.res == (long long)bytes && event.res2 == 0); + CHECK(read(event_fd, ¬ifications, sizeof(notifications)) == + sizeof(notifications)); + CHECK(notifications == 1); +} + +static void exercise(unsigned long native, unsigned int requested) +{ + unsigned long page = getauxval(AT_PAGESZ), bytes = 2 * page + 37; + aio_context_t context = 0; + int file = memfd_create("aio-granule", 0); + int event_fd = eventfd(0, EFD_NONBLOCK); + unsigned char *source, *destination, *reserve, *target; + struct aio_ring_header *header; + unsigned long ring_size; + struct iocb request = { 0 }, *requests[] = { &request }; + unsigned char residency; + + CHECK(file >= 0 && event_fd >= 0); + CHECK(!ftruncate(file, 4 * page)); + source = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + destination = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED && destination != MAP_FAILED); + for (unsigned long i = 0; i < bytes; i++) + source[page - 19 + i] = i * 17 + i / page; + CHECK(!syscall(SYS_io_setup, requested, &context)); + header = (void *)(uintptr_t)context; + CHECK(header->magic == 0xa10a10a1 && + header->header_length == sizeof(*header)); + CHECK(header->nr >= requested); + ring_size = + header->header_length + header->nr * sizeof(struct io_event); + CHECK(ring_size && !(ring_size % native)); + reserve = mmap(NULL, ring_size + 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[ring_size] = 0x72; + CHECK(!munmap(target, ring_size)); + request.aio_data = 0x12345678; + request.aio_lio_opcode = IOCB_CMD_PWRITE; + request.aio_fildes = file; + request.aio_buf = (uintptr_t)(source + page - 19); + request.aio_nbytes = bytes; + request.aio_offset = 7; + request.aio_flags = IOCB_FLAG_RESFD; + request.aio_resfd = event_fd; + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + /* Retain a queued completion while relocating the ring. */ + CHECK(mremap(header, ring_size, ring_size, + MREMAP_MAYMOVE | MREMAP_FIXED, target) == target); + context = (uintptr_t)target; + header = (void *)target; + CHECK(header->magic == 0xa10a10a1); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + complete(context, &request, bytes, event_fd); + request.aio_data = 0x87654321; + request.aio_lio_opcode = IOCB_CMD_PREAD; + request.aio_buf = (uintptr_t)(destination + page - 11); + CHECK(syscall(SYS_io_submit, context, 1, requests) == 1); + complete(context, &request, bytes, event_fd); + CHECK(!memcmp(source + page - 19, destination + page - 11, bytes)); + CHECK(!syscall(SYS_io_destroy, context)); + errno = 0; + CHECK(mincore(target, page, &residency) == -1 && errno == ENOMEM); + CHECK(target[-1] == 0x51 && target[ring_size] == 0x72); + CHECK(!munmap(reserve, ring_size + 4 * native)); + CHECK(!munmap(source, 4 * page) && !munmap(destination, 4 * page)); + CHECK(!close(file) && !close(event_fd)); + printf("ok - AIO page=%lu requested=%u ring=%lu relocation, I/O, eventfd and teardown\n", + page, requested, ring_size); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(native, 16); + exercise(native, 4096); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, NULL); + _exit(2); + } + int bad = wait_child(child); + + printf("%s - AIO native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + printf("AIO GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c new file mode 100644 index 00000000000000..4ac03136f2ec0a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/aslr-fixture-init.c @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define SAMPLES 64 +struct sample { + unsigned long page_size, stack, mapping, brk; +}; + +static int get_sample(struct sample *s) +{ + char buf[4096], *pos, *end; + unsigned long low, high; + FILE *f; + int fd; + ssize_t size; + void *p; + + s->page_size = getauxval(AT_PAGESZ); + p = mmap(NULL, s->page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + s->mapping = (unsigned long)p; + f = fopen("/proc/self/maps", "r"); + if (!f) + return 1; + while (fgets(buf, sizeof(buf), f)) + if (strstr(buf, "[stack]") && + sscanf(buf, "%lx-%lx", &low, &high) == 2) + s->stack = high; + fclose(f); + fd = open("/proc/self/stat", O_RDONLY); + if (fd < 0) + return 1; + size = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (size <= 0) + return 1; + buf[size] = 0; + pos = strrchr(buf, ')'); + if (!pos) + return 1; + pos += 2; + /* Fields 1 and 2 precede the closing parenthesis; start_brk is 47. */ + for (unsigned int field = 3; field < 47; field++) { + pos = strchr(pos, ' '); + if (!pos) + return 1; + pos++; + } + errno = 0; + s->brk = strtoul(pos, &end, 10); + return errno || end == pos || !s->brk || !s->stack; +} + +static int collect(unsigned int mode, bool disabled, struct sample *s) +{ + int pipefd[2], status; + pid_t pid; + ssize_t size; + + if (pipe(pipefd)) + return 1; + pid = fork(); + if (!pid) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + if (prctl(SET_EXEC_PAGE_SIZE, mode == 1 ? 4096UL : 0UL, 0UL, + 0UL, 0UL)) + _exit(2); + if (disabled && personality(ADDR_NO_RANDOMIZE) < 0) + _exit(3); + execl(mode == 2 ? "/compat-aslr" : "/init", "/init", "--sample", + fd, NULL); + _exit(4); + } + close(pipefd[1]); + size = read(pipefd[0], s, sizeof(*s)); + close(pipefd[0]); + return pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFEXITED(status) || WEXITSTATUS(status) || size != sizeof(*s); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + if (argc == 3 && !strcmp(argv[1], "--sample")) { + struct sample s = { 0 }; + if (getppid() != 1 || get_sample(&s)) + return 1; + return write(atoi(argv[2]), &s, sizeof(s)) != sizeof(s); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (mount("proc", "/proc", "proc", 0, NULL)) + failed++; + int control = open("/proc/sys/kernel/randomize_va_space", O_WRONLY); + if (control < 0 || write(control, "2", 1) != 1) + failed++; + if (control >= 0) + close(control); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + if (prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + failed++; + for (unsigned int mode = 0; mode < 3; mode++) { + if (mode == 1 && !has4k) + continue; + unsigned long ps = mode && has4k ? 4096 : native; + struct sample first = { 0 }; + bool varied[3] = { 0 }, subnative[3] = { 0 }; + int ok = 1; + + for (unsigned int i = 0; i < SAMPLES; i++) { + struct sample s = { 0 }; + if (collect(mode, false, &s)) { + ok = 0; + break; + } + unsigned long address[] = { s.stack, s.mapping, s.brk }; + unsigned long original[] = { first.stack, first.mapping, + first.brk }; + ok &= s.page_size == ps; + for (unsigned int n = 0; n < 3; n++) { + ok &= !(address[n] & (ps - 1)); + varied[n] |= i && address[n] != original[n]; + subnative[n] |= i && + ((address[n] ^ original[n]) & + (native - 1)); + } + if (!i) + first = s; + } + for (unsigned int n = 0; n < 3; n++) + ok &= varied[n] && (ps == native || subnative[n]); + printf("%s - %s %luK ASLR: 64 execs, stack/mmap/brk varied=%d%d%d low-bits-varied=%d%d%d\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024, varied[0], varied[1], varied[2], subnative[0], + subnative[1], subnative[2]); + failed += !ok; + struct sample a = { 0 }, b = { 0 }; + ok = !collect(mode, true, &a) && !collect(mode, true, &b) && + !memcmp(&a, &b, sizeof(a)) && a.page_size == ps; + printf("%s - %s %luK ADDR_NO_RANDOMIZE retains stable layout\n", + ok ? "ok" : "not ok", mode == 2 ? "AArch32" : "AArch64", + ps / 1024); + failed += !ok; + } + printf("GRANULE ASLR %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c new file mode 100644 index 00000000000000..33793cebc2b8cc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/bpf-uptr-init.c @@ -0,0 +1,583 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +struct user_data { + uint64_t a, b, result, marker; +}; +struct value { + struct user_data *p, *q; + uint64_t cookie; + uint8_t trailing; +} __attribute__((packed)); +static char verifier_log[65536]; +static int bpf(enum bpf_cmd cmd, union bpf_attr *a) +{ + size_t size; +#define END(field) (offsetof(union bpf_attr, field) + sizeof(a->field)) + switch (cmd) { + case BPF_BTF_LOAD: + size = END(btf_log_level); + break; + case BPF_MAP_CREATE: + size = END(btf_value_type_id); + break; + case BPF_PROG_LOAD: + size = END(log_buf); + break; + case BPF_RAW_TRACEPOINT_OPEN: + size = END(raw_tracepoint.prog_fd); + break; + case BPF_MAP_UPDATE_ELEM: + size = END(flags); + break; + case BPF_MAP_LOOKUP_ELEM: + size = END(flags); + break; + case BPF_MAP_DELETE_ELEM: + size = END(key); + break; + default: + errno = EINVAL; + return -1; + } +#undef END + return syscall(__NR_bpf, cmd, a, size); +} +struct btf_blob { + unsigned char types[1024]; + char strings[512]; + unsigned nt, ns; +}; +static uint32_t str(struct btf_blob *b, const char *s) +{ + unsigned at = b->ns; + size_t n = strlen(s) + 1; + memcpy(b->strings + b->ns, s, n); + b->ns += n; + return at; +} +static void push(struct btf_blob *b, const void *p, size_t n) +{ + memcpy(b->types + b->nt, p, n); + b->nt += n; +} +static void type(struct btf_blob *b, const char *name, unsigned kind, + unsigned n, unsigned size_type) +{ + struct btf_type t = { .name_off = name ? str(b, name) : 0, + .info = kind << 24 | n, + .size = size_type }; + push(b, &t, sizeof(t)); +} +static void member(struct btf_blob *b, const char *name, unsigned id, + unsigned offset) +{ + struct btf_member m = { .name_off = str(b, name), + .type = id, + .offset = offset * 8 }; + push(b, &m, sizeof(m)); +} +static int create_map(void) +{ + struct btf_blob b = { .ns = 1 }; + uint32_t encoding; + type(&b, "int", BTF_KIND_INT, 0, 4); + encoding = BTF_INT_SIGNED << 24 | 32; + push(&b, &encoding, 4); + type(&b, "u64", BTF_KIND_INT, 0, 8); + encoding = 64; + push(&b, &encoding, 4); + type(&b, "user_data", BTF_KIND_STRUCT, 4, sizeof(struct user_data)); + member(&b, "a", 2, 0); + member(&b, "b", 2, 8); + member(&b, "result", 2, 16); + member(&b, "marker", 2, 24); + type(&b, "uptr", BTF_KIND_TYPE_TAG, 0, 3); + type(&b, NULL, BTF_KIND_PTR, 0, 4); + type(&b, "value_type", BTF_KIND_STRUCT, 4, sizeof(struct value)); + member(&b, "p", 5, 0); + member(&b, "q", 5, 8); + member(&b, "cookie", 2, 16); + member(&b, "trailing", 7, 24); + type(&b, "u8", BTF_KIND_INT, 0, 1); + encoding = 8; + push(&b, &encoding, 4); + unsigned char data[2048]; + struct btf_header h = { .magic = BTF_MAGIC, + .version = BTF_VERSION, + .hdr_len = sizeof(h), + .type_len = b.nt, + .str_off = b.nt, + .str_len = b.ns }; + memcpy(data, &h, sizeof(h)); + memcpy(data + sizeof(h), b.types, b.nt); + memcpy(data + sizeof(h) + b.nt, b.strings, b.ns); + union bpf_attr a = { .btf = (uintptr_t)data, + .btf_size = sizeof(h) + b.nt + b.ns, + .btf_log_buf = (uintptr_t)verifier_log, + .btf_log_size = sizeof(verifier_log), + .btf_log_level = 1 }; + int btf = bpf(BPF_BTF_LOAD, &a); + if (btf < 0) { + printf("BTF load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){ .map_type = BPF_MAP_TYPE_TASK_STORAGE, + .key_size = sizeof(int), + .value_size = sizeof(struct value), + .map_flags = BPF_F_NO_PREALLOC, + .btf_fd = btf, + .btf_key_type_id = 1, + .btf_value_type_id = 6 }; + int fd = bpf(BPF_MAP_CREATE, &a); + close(btf); + return fd; +} +#define I(c, d, s, o, i) \ + ((struct bpf_insn){ .code = (c), \ + .dst_reg = (d), \ + .src_reg = (s), \ + .off = (o), \ + .imm = (i) }) +static int create_prog(int map, int *link) +{ + struct bpf_insn insns[40]; + unsigned n = 0, jumps[4]; +#define EMIT(c, d, s, o, i) (insns[n++] = I(c, d, s, o, i)) + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_current_task_btf); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 2, 0, 0, 0); + EMIT(BPF_LD | BPF_DW | BPF_IMM, 1, BPF_PSEUDO_MAP_FD, 0, map); + EMIT(0, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 3, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 4, 0, 0, 0); + EMIT(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_task_storage_get); + jumps[0] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 0, 0, 0, 0); + EMIT(BPF_ALU64 | BPF_MOV | BPF_X, 6, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 7, 6, 16, 0); + jumps[1] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 7, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 0, 0); + jumps[2] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 0, 0); + EMIT(BPF_ALU64 | BPF_XOR | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + insns[jumps[2]].off = n - jumps[2] - 1; + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 1, 6, 8, 0); + jumps[3] = n; + EMIT(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 0, 0); + EMIT(BPF_LDX | BPF_DW | BPF_MEM, 2, 1, 8, 0); + EMIT(BPF_ALU64 | BPF_ADD | BPF_X, 2, 7, 0, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 2, 16, 0); + EMIT(BPF_STX | BPF_DW | BPF_MEM, 1, 7, 24, 0); + for (unsigned i = 0; i < 4; i++) + if (i != 2) + insns[jumps[i]].off = n - jumps[i] - 1; + EMIT(BPF_ALU64 | BPF_MOV | BPF_K, 0, 0, 0, 0); + EMIT(BPF_JMP | BPF_EXIT, 0, 0, 0, 0); +#undef EMIT + union bpf_attr a = { .prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT, + .insn_cnt = n, + .insns = (uintptr_t)insns, + .license = (uintptr_t)"GPL", + .log_buf = (uintptr_t)verifier_log, + .log_size = sizeof(verifier_log), + .log_level = 1 }; + verifier_log[0] = 0; + int fd = bpf(BPF_PROG_LOAD, &a); + if (fd < 0) { + printf("prog load errno=%d: %s\n", errno, verifier_log); + return -1; + } + a = (union bpf_attr){}; + a.raw_tracepoint.name = (uintptr_t)"sys_enter"; + a.raw_tracepoint.prog_fd = fd; + *link = bpf(BPF_RAW_TRACEPOINT_OPEN, &a); + if (*link < 0) { + close(fd); + return -1; + } + return fd; +} +static int update(int map, int key, struct value *v, unsigned flags) +{ + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)v, + .flags = flags }; + return bpf(BPF_MAP_UPDATE_ELEM, &a); +} +static int delete(int map, int key) +{ + union bpf_attr a = { .map_fd = map, .key = (uintptr_t)&key }; + return bpf(BPF_MAP_DELETE_ELEM, &a); +} +struct pins { + unsigned long long acquired, released; +}; +static int pin_stats(struct pins *p) +{ + int fd = open("/proc/sys/vm/stat_refresh", O_RDONLY); + char c; + if (fd < 0) + return -1; + int ret = read(fd, &c, 1); + close(fd); + if (ret < 0) + return -1; + FILE *f = fopen("/proc/vmstat", "r"); + if (!f) + return -1; + char name[128]; + unsigned long long value; + unsigned found = 0; + while (fscanf(f, "%127s %llu", name, &value) == 2) { + if (!strcmp(name, "nr_foll_pin_acquired")) { + p->acquired = value; + found |= 1; + } + if (!strcmp(name, "nr_foll_pin_released")) { + p->released = value; + found |= 2; + } + } + fclose(f); + return found == 3 ? 0 : -1; +} +static int wait_pins(struct pins base, unsigned active) +{ + struct pins p = {}; + for (unsigned i = 0; i < 500; i++) { + if (pin_stats(&p)) + return 0; + if (p.acquired - base.acquired >= active && + p.acquired - base.acquired - (p.released - base.released) == + active) + return 1; + usleep(10000); + } + printf("pin delta acquired=%llu released=%llu expected-active=%u\n", + p.acquired - base.acquired, p.released - base.released, active); + return 0; +} +static int filled(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return 0; + return 1; +} +static int values(struct user_data *p, struct user_data *q, uint64_t cookie) +{ + return p->result == (p->a ^ cookie) && q->result == q->b + cookie && + p->marker == cookie && q->marker == cookie; +} +static int exercise_case(size_t ps, unsigned shared) +{ + struct pins base; + CHECK(!pin_stats(&base)); + int backing = -1; + if (shared) { + backing = memfd_create("bpf-uptr", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + struct user_data *p = (void *)(leaf + ps - 64), *q = p + 1; + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + int map = create_map(); + CHECK(map >= 0); + int link = -1, prog = create_prog(map, &link); + CHECK(prog >= 0 && link >= 0); + int key = syscall(SYS_pidfd_open, getpid(), 0); + CHECK(key >= 0); + struct value probe = {}; + int probe_ret = update(map, key, &probe, BPF_NOEXIST); + CHECK(!probe_ret); + CHECK(!delete(map, key)); + struct value v = { + .p = p, .q = q, .cookie = 0xabc001, .trailing = 0x7d + }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + for (unsigned i = 0; i < 64; i++) { + v.cookie++; + CHECK(!update(map, key, &v, BPF_EXIST)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + } + CHECK(wait_pins(base, 2)); + struct { + struct value v; + unsigned char tail[16]; + } lookup; + memset(&lookup, 0xa5, sizeof(lookup)); + union bpf_attr a = { .map_fd = map, + .key = (uintptr_t)&key, + .value = (uintptr_t)&lookup }; + CHECK(!bpf(BPF_MAP_LOOKUP_ELEM, &a)); + CHECK(!lookup.v.p && !lookup.v.q && lookup.v.cookie == v.cookie && + lookup.v.trailing == v.trailing && + filled(lookup.tail, sizeof(lookup.tail), 0xa5)); + errno = 0; + CHECK(update(map, key, &v, BPF_NOEXIST) == -1 && errno == EEXIST); + CHECK(wait_pins(base, 2)); + struct value half = v; + half.p = NULL; + CHECK(!update(map, key, &half, BPF_EXIST)); + CHECK(wait_pins(base, 1)); + p->result = p->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !p->marker && q->result == q->b + v.cookie); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + struct value bad = v; + bad.q = (void *)5; + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + CHECK(values(p, q, v.cookie)); + bad.q = (void *)(leaf + ps - 16); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EOPNOTSUPP); + CHECK(wait_pins(base, 2)); + bad.q = (void *)(UINTPTR_MAX - 15); + errno = 0; + CHECK(update(map, key, &bad, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(wait_pins(base, 2)); + CHECK(!mprotect(leaf, ps, PROT_READ)); + errno = 0; + CHECK(update(map, key, &v, BPF_EXIST) == -1 && errno == EFAULT); + CHECK(!mprotect(leaf, ps, PROT_READ | PROT_WRITE)); + CHECK(wait_pins(base, 2)); + struct value empty = {}; + CHECK(!update(map, key, &empty, BPF_EXIST)); + CHECK(wait_pins(base, 0)); + p->result = q->result = p->marker = q->marker = 0; + syscall(SYS_getpid); + CHECK(!p->result && !q->result); + CHECK(!update(map, key, &v, BPF_EXIST)); + CHECK(wait_pins(base, 2)); + /* The child owns storage pointing into the parent's pinned memory. */ + int barrier[2]; + CHECK(!pipe(barrier)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + if (read(barrier[0], &token, 1) != 1) + _exit(3); + if (!shared) { + p->a = 1; + q->b = 2; + } + syscall(SYS_getpid); + _exit(0); + } + close(barrier[0]); + int childkey = syscall(SYS_pidfd_open, child, 0); + CHECK(childkey >= 0); + CHECK(!update(map, childkey, &v, BPF_NOEXIST)); + CHECK(!delete(map, key)); + CHECK(wait_pins(base, 2)); + p->result = q->result = p->marker = q->marker = 0; + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(childkey); + CHECK(values(p, q, v.cookie) && p->a == 0x12345678 && + q->b == 0x87654321); + CHECK(wait_pins(base, 0)); + /* The supplying mm exits while a different task retains its UPTRs. */ + int foreign_pipe[2]; + CHECK(!pipe(foreign_pipe)); + pid_t supplier = fork(); + CHECK(supplier >= 0); + if (!supplier) { + close(foreign_pipe[0]); + *p = (struct user_data){ .a = 0x1111, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x2222 }; + struct value foreign = { .p = p, .q = q, .cookie = 0x765432 }; + if (update(map, key, &foreign, BPF_NOEXIST)) + _exit(4); + struct iovec data = { .iov_base = p, .iov_len = 64 }; + if (vmsplice(foreign_pipe[1], &data, 1, 0) != 64) + _exit(5); + _exit(0); + } + close(foreign_pipe[1]); + CHECK(waitpid(supplier, &status, 0) == supplier && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(wait_pins(base, 2)); + syscall(SYS_getpid); + struct user_data retired[2]; + size_t done = 0; + while (done < sizeof(retired)) { + ssize_t got = read(foreign_pipe[0], (char *)retired + done, + sizeof(retired) - done); + CHECK(got > 0); + done += got; + } + close(foreign_pipe[0]); + CHECK(retired[0].a == 0x1111 && retired[1].b == 0x2222 && + values(retired, retired + 1, 0x765432)); + CHECK(!delete(map, key) && wait_pins(base, 0)); + *p = (struct user_data){ .a = 0x12345678, .b = 3 }; + *q = (struct user_data){ .a = 4, .b = 0x87654321 }; + CHECK(!update(map, key, &v, BPF_NOEXIST)); + CHECK(wait_pins(base, 2)); + CHECK(filled(leaf, ps - 64, 0x64)); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = leaf, .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!munmap(leaf, ps)); + CHECK(mmap(leaf, ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0) == leaf); + memset(leaf, 0xe7, ps); + unsigned char *old = malloc(ps); + CHECK(old); + size_t n = 0; + while (n < ps) { + ssize_t got = read(pipefd[0], old + n, ps - n); + CHECK(got > 0); + n += got; + } + close(pipefd[0]); + close(pipefd[1]); + CHECK(filled(old, ps - 64, 0x64) && + values((void *)(old + ps - 64), (void *)(old + ps - 32), + v.cookie)); + CHECK(filled(leaf, ps, 0xe7)); + free(old); + /* Closing the map alone is insufficient: the program retains its map ref. */ + close(map); + close(link); + close(prog); + close(key); + CHECK(wait_pins(base, 0)); + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK BPF UPTR %s: relocated writes, updates, rollback, storage/supplier exit, unmap and pin balance\n", + ps / 1024, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t ps) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(90); + for (unsigned shared = 0; shared < 2; shared++) + CHECK(!exercise_case(ps, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char size[24]; + snprintf(size, sizeof(size), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 1000; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 999) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK BPF UPTR status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("BPF UPTR %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c new file mode 100644 index 00000000000000..2a00ca546751eb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/coarse-shmem-init.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1 runner; fault injection is selected by the KUnit worker. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" + +static int setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int fault_setting(const char *kind, const char *name, const char *value) +{ + char path[192]; + + snprintf(path, sizeof(path), "/sys/kernel/debug/%s/%s", kind, name); + return setting(path, value); +} + +static int injection_off(void) +{ + int ret = fault_setting("failslab", "probability", "0"); + + ret |= fault_setting("fail_page_alloc", "probability", "0"); + ret |= fault_setting("failslab", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-wait", "Y"); + ret |= fault_setting("fail_page_alloc", "ignore-gfp-highmem", "Y"); + return ret; +} + +static int injection_on(bool page) +{ + const char *kind = page ? "fail_page_alloc" : "failslab"; + + if (injection_off() || fault_setting(kind, "task-filter", "Y") || + fault_setting(kind, "verbose", "0") || + fault_setting(kind, "ignore-gfp-wait", "N") || + fault_setting(kind, "times", "-1") || + fault_setting(kind, "interval", "1") || + fault_setting(kind, "space", "0")) + return -1; + if (!page) + return 0; /* Only current->fail_nth selects the slab allocation. */ + return fault_setting(kind, "ignore-gfp-highmem", "N") || + fault_setting(kind, "min-order", "2") || + fault_setting(kind, "probability", "100"); +} + +static int move_worker(const char *text, bool enter, bool swap_limit) +{ + char *end, path[192], pid[32], comm[64]; + long nr = strtol(text, &end, 10); + const char *group = swap_limit ? "coarse-swap-limit" : "coarse-zero"; + int fd; + ssize_t n; + + if (*end || nr <= 1 || nr == getpid()) + return -1; + snprintf(path, sizeof(path), "/proc/%ld/comm", nr); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = read(fd, comm, sizeof(comm) - 1); + close(fd); + if (n < 1) + return -1; + comm[n] = 0; + if (strncmp(comm, "kunit_", 6)) + return -1; + if (enter) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return -1; + if (setting("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s", group); + if (mkdir(path, 0755) && errno != EEXIST) + return -1; + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/memory.max", + group); + if (setting(path, swap_limit ? "max" : "0")) + return -1; + if (swap_limit && + setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "0")) + return -1; + } + snprintf(pid, sizeof(pid), "%ld", nr); + if (enter) + snprintf(path, sizeof(path), "/sys/fs/cgroup/%s/cgroup.procs", + group); + else + snprintf(path, sizeof(path), "/sys/fs/cgroup/cgroup.procs"); + return setting(path, pid); +} + +static int helper(const char *op) +{ + struct stat init, self; + + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) + return 1; + if (!strcmp(op, "--coarse-fixture-check")) + return getauxval(AT_PAGESZ) != 16384; + if (!strcmp(op, "--swapoff-helper")) + return swapoff(swap_path()) != 0; + if (!strcmp(op, "--swapon-helper")) + return swapon(swap_path(), 0) != 0; + if (!strncmp(op, "--coarse-swap-enter=", 20)) + return move_worker(op + 20, true, true) != 0; + if (!strncmp(op, "--coarse-swap-leave=", 20)) + return move_worker(op + 20, false, true) != 0; + if (!strcmp(op, "--coarse-swap-unlimit")) + return setting("/sys/fs/cgroup/coarse-swap-limit/memory.swap.max", + "max") != 0; + if (!strcmp(op, "--coarse-injection-off")) + return injection_off() != 0; + if (!strcmp(op, "--coarse-slab-on")) + return injection_on(false) != 0; + if (!strcmp(op, "--coarse-page-on")) + return injection_on(true) != 0; + if (!strncmp(op, "--coarse-memcg-enter=", 21)) + return move_worker(op + 21, true, false) != 0; + if (!strncmp(op, "--coarse-memcg-leave=", 21)) + return move_worker(op + 21, false, false) != 0; + return 1; +} + +static int run_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-coarse-shmem/"; + char path[192], *log = calloc(1, 1024 * 1024); + size_t used = 0, capacity = 1024 * 1024; + ssize_t n = 0; + int fd, ret = -1; + + if (!log) + return -1; + snprintf(path, sizeof(path), "%srun", base); + if (setting(path, "1")) + goto out; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + goto out; + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - used - 1)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (!n && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-coarse-shmem\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; +out: + free(log); + return ret; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + unsigned int i; + + if (getpid() != 1 && argc == 2) + return helper(argv[1]); + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--swapfile")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode)) + goto out; + if (argc == 2 && prepare_swapfile()) + goto out; + printf("COARSE SWAP BACKING=%s\n", swap_path()); + if (swapon(swap_path(), 0)) + goto out; + active = true; + ret = run_suite() ? 1 : 0; + if (injection_off()) + ret = 1; +out: + if (active && swapoff(swap_path())) + ret = 1; + if (active && argc == 2 && umount("/swapdir")) + ret = 1; + printf("COARSE SHMEM SWAP PASS=%d errno=%d\n", !ret, errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c new file mode 100644 index 00000000000000..3a88db75112151 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-aslr-probe.c @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding ELF32 address sampler for the disposable ASLR fixture. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct sample { + u64 page_size, stack, mapping, brk; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long value) +{ + return (u32)value >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static char *find(char *p, char c) +{ + while (*p && *p != c) + p++; + return *p ? p : (char *)0; +} +static char *find_stack(char *p) +{ + const char name[] = "[stack]"; + while (*p) { + unsigned i = 0; + while (name[i] && p[i] == name[i]) + i++; + if (!name[i]) + return p; + p++; + } + return (char *)0; +} +static unsigned hex(char *p) +{ + unsigned n = 0, v; + for (;;) { + if (*p >= '0' && *p <= '9') + v = *p - '0'; + else if (*p >= 'a' && *p <= 'f') + v = *p - 'a' + 10; + else + return n; + n = (n << 4) | v; + p++; + } +} +static int load(const char *path, char *buf, unsigned size) +{ + long fd = call(5, (long)path, 0, 0, 0, 0, 0), n; + if (error(fd)) + return 1; + n = call(3, fd, (long)buf, size - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return 1; + buf[n] = 0; + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++; + char **argv = (void *)sp, buf[4096], *p, *q; + struct sample sample = { 0 }; + long mapping; + int bad = 1; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + sample.page_size = sp[1]; + sp += 2; + } + if (argc != 3 || call(64, 0, 0, 0, 0, 0, 0) != 1) + goto done; + mapping = call(192, 0, sample.page_size, 3, 0x22, -1, 0); + if (error(mapping)) + goto done; + sample.mapping = (u32)mapping; + if (load("/proc/self/maps", buf, sizeof(buf))) + goto done; + p = find_stack(buf); + if (!p) + goto done; + while (p > buf && p[-1] != '\n') + p--; + p = find(p, '-'); + if (!p) + goto done; + sample.stack = hex(p + 1); + if (load("/proc/self/stat", buf, sizeof(buf))) + goto done; + p = find(buf, ')'); + if (!p) + goto done; + while ((q = find(p + 1, ')'))) + p = q; + p += 2; + for (unsigned field = 3; field < 47; field++) { + p = find(p, ' '); + if (!p) + goto done; + p++; + } + sample.brk = number(p); + if (!sample.page_size || !sample.stack || !sample.brk) + goto done; + bad = call(4, number(argv[2]), (long)&sample, sizeof(sample), 0, 0, + 0) != sizeof(sample); +done: + call(1, bad, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-fixture-init.c b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c new file mode 100644 index 00000000000000..f576ed93623edd --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-fixture-init.c @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static unsigned long native_size; +static void result(int ok, const char *s) +{ + printf("%s - launcher %s\n", ok ? "ok" : "not ok", s); + if (!ok) + failed++; +} +static int launch(const char *path, char **args, int request, int expect_errno) +{ + pid_t child = fork(); + int status = 0; + if (!child) { + if (request && prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(95); + unsigned char *canary = + mmap(NULL, native_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (canary == MAP_FAILED) + _exit(94); + memset(canary, 0x69, native_size); + int before = prctl(0x41555002, 0UL, 0UL, 0UL, 0UL); + for (int i = 0; i < (expect_errno ? 64 : 1); i++) { + execv(path, args); + if (!expect_errno || errno != expect_errno) + _exit(96); + if (getauxval(AT_PAGESZ) != native_size || + prctl(0x41555002, 0UL, 0UL, 0UL, 0UL) != before) + _exit(93); + for (int j = 0; j < native_size; j++) + if (canary[j] != 0x69) + _exit(92); + } + _exit(0); + } + if (child < 0 || waitpid(child, &status, 0) != child) + return 0; + if (status) + printf("# child %s status=%#x\n", path, status); + return WIFEXITED(status) && !WEXITSTATUS(status); +} +int main(int argc, char **argv) +{ + if (argc > 1 && !strcmp(argv[1], "--returned")) + return getauxval(AT_PAGESZ) == + strtoul(getenv("AURORA_NATIVE_PAGE_SIZE"), + NULL, 10) ? + 0 : + 1; + if (argc > 1 && !strcmp(argv[1], "--returned4k")) + return getauxval(AT_PAGESZ) == 4096 ? 0 : 1; + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + native_size = getauxval(AT_PAGESZ); + char native_text[24]; + snprintf(native_text, sizeof(native_text), "%lu", native_size); + setenv("AURORA_NATIVE_PAGE_SIZE", native_text, 1); + setvbuf(stdout, NULL, _IONBF, 0); + mount("proc", "/proc", "proc", 0, NULL); + int no4k = argc > 1 && !strcmp(argv[1], "--no4k"), + fd = memfd_create("compat-offsets", 0); + if (no4k) { + errno = 0; + result(prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL) == -1 && + errno == EOPNOTSUPP, + "4KB capability is actually masked"); + } + result(fd >= 0 && !ftruncate(fd, 0x100000000LL + 4 * native_size), + "sparse file fixture above 4GB"); + unsigned char page[4096]; + for (unsigned q = 0; q < 4 * native_size / 4096; q++) { + memset(page, 0x40 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), q * 4096) != sizeof(page)) + failed++; + memset(page, 0x80 + q, sizeof(page)); + if (pwrite(fd, page, sizeof(page), 0x100000000LL + q * 4096) != + sizeof(page)) + failed++; + } + char fdstr[24]; + snprintf(fdstr, sizeof(fdstr), "%d", fd); + char *arg = malloc(90001); + for (unsigned i = 0; i < 90000; i++) + arg[i] = 'a' + i % 23; + arg[90000] = 0; + char *args[] = { "/probe32-64k", no4k ? native_text : "4096", fdstr, + arg, NULL }; + result(launch(args[0], args, 0, 0), + "64KB-aligned ELF32 automatically selects available ABI"); + if (!no4k) { + args[0] = "/probe32-4k"; + result(launch(args[0], args, 0, 0), + "4KB-aligned ELF32 executes with 4KB translation"); + args[0] = "/probe32-64k"; + result(launch(args[0], args, 1, 0), + "explicit pending 4KB request allows ELF32"); + } + args[0] = "/script32"; + result(launch(args[0], args, 0, 0), + "script final ELF class determines page granule"); + args[0] = "/probe32-dynamic"; + result(launch(args[0], args, 0, 0), + "ELF32 interpreter mapping uses selected page granule"); + char *chain[] = { + "/probe32-64k", "--exec32", no4k ? native_text : "4096", + fdstr, arg, NULL + }; + result(launch(chain[0], chain, 0, 0), + "ELF32 to ELF32 restages arguments and selects ABI"); + char *back[] = { "/probe32-64k", "--exec64", NULL }; + result(launch(back[0], back, 0, 0), + "ELF32 to ELF64 exec restores native default"); + if (!no4k) { + back[1] = "--exec64-4k"; + result(launch(back[0], back, 0, 0), + "ELF32 can request alternative ELF64 exec granule"); + } + char *big = malloc(200001); + memset(big, 'x', 200000); + big[200000] = 0; + args[0] = "/probe32-64k"; + args[3] = big; + if (!no4k) + result(launch(args[0], args, 0, E2BIG), + "64 oversized transfers return E2BIG and preserve old mm and request"); + printf("COMPAT FIXTURE %s mode=%s\n", failed ? "FAIL" : "PASS", + no4k ? "no4k-native-fallback" : "auto4k"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-init.c b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c new file mode 100644 index 00000000000000..679c39370cd6dc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-init.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +struct result { + uint32_t pagesize, passed, failed, error; +}; +int main(int argc, char **argv) +{ + struct result out = { 0 }; + unsigned long expected; + int pipes[2], status = -1; + pid_t pid; + size_t done = 0; + int ok; + + if (getpid() != 1 || + (argc != 1 && (argc != 2 || strcmp(argv[1], "--no4k")))) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + expected = argc == 2 ? getauxval(AT_PAGESZ) : 4096; + if (mount("proc", "/proc", "proc", 0, NULL) || pipe(pipes)) + return 3; + pid = fork(); + if (!pid) { + char value[32]; + close(pipes[0]); + if (dup2(pipes[1], 1) < 0) + _exit(4); + close(pipes[1]); + snprintf(value, sizeof(value), "%lu", expected); + execl("/compat-mlock", "/compat-mlock", value, NULL); + _exit(5); + } + close(pipes[1]); + while (done < sizeof(out)) { + ssize_t n = + read(pipes[0], (char *)&out + done, sizeof(out) - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + break; + done += n; + } + close(pipes[0]); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status) && done == sizeof(out) && + out.pagesize == expected && out.passed == 6 && !out.failed; + printf("%s - AArch32 memlock expected=%lu actual=%u passed=%u stage=%u error=%d status=%d\n", + ok ? "ok" : "not ok", expected, out.pagesize, out.passed, + out.failed, (int32_t)out.error, status); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c new file mode 100644 index 00000000000000..97e76dd94bae95 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-mlock-probe.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 memlock probe, only for the disposable PID1 fixture. */ +typedef unsigned int u32; +struct result { + u32 pagesize, passed, failed, error; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static int error(long n) +{ + return (u32)n >= 0xfffff001U; +} +static unsigned number(const char *p) +{ + unsigned n = 0; + while (*p >= '0' && *p <= '9') + n = n * 10 + *p++ - '0'; + return n; +} +static long locked_kb(void) +{ + char buf[8192]; + long fd = call(5, (long)"/proc/self/status", 0, 0, 0, 0, 0), n; + const char name[] = "VmLck:"; + + if (error(fd)) + return -1; + n = call(3, fd, (long)buf, sizeof(buf) - 1, 0, 0, 0); + call(6, fd, 0, 0, 0, 0, 0); + if (n <= 0) + return -1; + buf[n] = 0; + for (long i = 0; i < n; i++) { + unsigned j = 0; + while (name[j] && i + j < (unsigned)n && buf[i + j] == name[j]) + j++; + if (!name[j]) { + const char *p = buf + i + j; + while (*p == ' ' || *p == '\t') + p++; + return number(p); + } + } + return -1; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, ps = 0, kb, stage = 1; + char **argv = (void *)sp; + struct result out = { 0 }; + long p, q, pid, ret = -1; + int status = -1; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + ps = sp[1]; + sp += 2; + } + out.pagesize = ps; + if (argc != 2 || call(64, 0, 0, 0, 0, 0, 0) != 1 || + ps != number(argv[1])) + goto done; + kb = ps / 1024; + p = call(192, 0, 8 * ps, 3, 0x22, -1, 0); + if (error(p) || ((u32)p & (ps - 1)) || locked_kb() != 0) + goto done; + ret = call(150, p + ps + 17, ps - 32, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + ret = call(150, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + ret = call(150, p + 2 * ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + ret = call(151, p + ps, ps, 0, 0, 0, 0); + if (ret || locked_kb() != (long)kb) + goto done; + out.passed++; + stage++; + *(volatile unsigned char *)(p + 2 * ps) = 0x63; + q = call(163, p + 2 * ps, ps, 2 * ps, 1, 0, 0); + ret = q; + if (error(q) || locked_kb() != (long)(2 * kb) || + *(volatile unsigned char *)q != 0x63) + goto done; + for (unsigned i = ps; i < 2 * ps; i++) + if (((volatile unsigned char *)q)[i]) + goto done; + out.passed++; + stage++; + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) { + call(1, locked_kb() != 0, 0, 0, 0, 0, 0); + for (;;) { + } + } + ret = call(114, pid, (long)&status, 0, 0, 0, 0); + if (error(pid) || ret != pid || status || locked_kb() != (long)(2 * kb)) + goto done; + out.passed++; + stage++; + ret = call(91, q, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage++; + ret = call(390, p + 4 * ps, 2 * ps, 1, 0, 0, 0); + if (ret || locked_kb() != (long)(2 * kb)) + goto done; + *(volatile unsigned char *)(p + 4 * ps) = 0x75; + ret = call(151, p + 4 * ps, 2 * ps, 0, 0, 0, 0); + if (ret || locked_kb() != 0) + goto done; + out.passed++; + stage = 0; +done: + out.failed = stage; + out.error = stage ? (u32)ret : 0; + call(4, 1, (long)&out, sizeof(out), 0, 0, 0); + call(1, stage != 0, 0, 0, 0, 0, 0); + for (;;) { + } +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c new file mode 100644 index 00000000000000..f2c155a888c15b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-balancing-probe.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 automatic NUMA migration and fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMAB FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMAB PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void value(const char *label, unsigned int length, u32 n) +{ + char text[16]; + unsigned int pos = sizeof(text); + + text[--pos] = '\n'; + do { + text[--pos] = '0' + n % 10; + n /= 10; + } while (n); + call(4, 1, (long)label, length, 0, 0, 0); + call(4, 1, (long)(text + pos), sizeof(text) - pos, 0, 0, 0); +} + +static u32 seconds(void) +{ + struct { + long long sec, nsec; + } ts; + + /* Use time64 even when CONFIG_COMPAT_32BIT_TIME is disabled. */ + if (call(403, 1, (long)&ts, 0, 0, 0, 0)) + finish(11); + return ts.sec; +} + +static void cpu(unsigned int id) +{ + u32 mask = 1U << id; + + if (call(241, 0, sizeof(mask), (long)&mask, 0, 0, 0)) + finish(12); +} + +static int node_at(unsigned char *address) +{ + int node = -1; + + if (call(320, (long)&node, 0, 0, (long)address, 3, 0)) + finish(13); + return node; +} + +static u32 sample(unsigned char *base, u32 length, u32 page) +{ + u32 moved = 0; + + for (u32 off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(base + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + + if (child < 0) + finish(40); + if (!child) { + /* Force the intended access; this is not thread synchronization. */ + if (write_access) + *(volatile unsigned char *)address = 0x27; + else + (void)*(volatile unsigned char *)address; + call(1, 2, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (call(114, child, (long)&status, 0, 0, 0, 0) != child || + (status & 0x7f) != 11) + finish(41); +} + +static void mapping(u32 page) +{ + const u32 length = 8U << 20; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + u32 loops = 0, moved = 0, started, total = length / page - 1; + int mode = -1; + long raw; + unsigned char *base; + + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(20); + cpu(0); + raw = call(192, 0, length, 3, 0x22, -1, 0); + if ((u32)raw >= 0xfffff001U) + finish(21); + base = (void *)raw; + /* A missing 4K capability must retain native alignment restrictions. */ + if (call(125, raw + 4096, 4096, 1, 0, 0, 0) != (page == 4096 ? 0 : -22)) + finish(22); + if (call(125, raw, length, 3, 0, 0, 0)) + finish(23); + if (call(220, raw, length, 15, 0, 0, 0)) /* MADV_NOHUGEPAGE */ + finish(24); + if (call(319, raw, length, 2, (long)mask, 64, 0)) + finish(25); + for (u32 off = 0; off < length; off += page) { + u32 *p = (void *)(base + off); + + p[0] = 0x91720000 + off / page; + p[1] = 0; + p[page / 4 - 1] = 0x51320000 + off / page; + if (node_at(base + off + 16) != 0) + finish(26); + } + if (call(125, raw + page, page, 1, 0, 0, 0) || + call(125, raw + 2 * page, page, 0, 0, 0, 0)) + finish(27); + if (call(319, raw, length, 0, 0, 0, + 0)) /* MPOL_DEFAULT, no MOVE flags */ + finish(28); + cpu(1); + started = seconds(); + do { + for (u32 off = 0; off < length; off += page) { + volatile u32 *p = (void *)(base + off); + + if (off == 2 * page) + continue; + if (p[0] != 0x91720000 + off / page || + p[page / 4 - 1] != 0x51320000 + off / page) + finish(29); + if (off != page) + p[1]++; + } + loops++; + if (!(loops % 128)) { + moved = sample(base, length, page); + if (moved == total) + break; + } + } while (seconds() - started < 30); + moved = sample(base, length, page); + value("A32 NUMAB page=", sizeof("A32 NUMAB page=") - 1, page); + value("A32 NUMAB moved=", sizeof("A32 NUMAB moved=") - 1, moved); + value("A32 NUMAB expected=", sizeof("A32 NUMAB expected=") - 1, total); + value("A32 NUMAB loops=", sizeof("A32 NUMAB loops=") - 1, loops); + if (moved != total) + finish(30); + for (u32 off = 0; off < length; off += page) + if (off != page && off != 2 * page && + *(u32 *)(base + off + 4) != loops) + finish(31); + forbidden(base + page, 1); + forbidden(base + 2 * page, 0); + if (call(125, raw + 2 * page, page, 1, 0, 0, 0) || + *(u32 *)(base + 2 * page) != 0x91720002) + finish(32); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(33); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, page = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + page = sp[1]; + sp += 2; + } + if (argc != 2 || page != number(argv[1])) + finish(10); + mapping(page); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-numa-probe.c b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c new file mode 100644 index 00000000000000..b2652fe348b6ea --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-numa-probe.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 NUMA policy and native-granule fallback checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 NUMA FAIL stage=00\n"; + + if (code) { + text[sizeof(text) - 4] = '0' + code / 10; + text[sizeof(text) - 3] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 NUMA PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long native, unsigned long page) +{ + unsigned long length = 4 * native; + long raw = call(192, 0, length, 3, 0x22, -1, 0); + unsigned char *base; + u32 mask[2] = { 1, 0 }, got[2] = { 0, 0 }; + int mode = -1, node; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + base = (void *)(((unsigned long)raw + native - 1) & ~(native - 1)); + if (call(320, (long)&mode, (long)got, 64, 0, 4, 0) || got[0] != 3 || + got[1]) + finish(21); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 0)) + finish(22); + mask[0] = 2; + if (call(319, (long)(base + page), 1, 2, (long)mask, 64, 0)) + finish(23); + if (call(450, (long)(base + page), 1, 1, 0, 0, 0)) + finish(24); + if (call(320, (long)&mode, (long)got, 64, (long)(base + page), 2, 0) || + mode != 2 || got[0] != 2 || got[1]) + finish(25); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x51 + off / page; + node = -1; + if (call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != (off == page ? 1 : 0)) + finish(26); + } + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 1) != -5) + finish(27); + if (call(319, (long)(base + 4096), 4096, 2, (long)mask, 64, 0) != + (page == 4096 ? 0 : -22)) + finish(28); + if (call(319, (long)base, 2 * native, 2, (long)mask, 64, 5)) + finish(29); + for (unsigned long off = 0; off < 2 * native; off += page) { + node = -1; + if (base[off + 17] != (unsigned char)(0x51 + off / page) || + call(320, (long)&node, 0, 0, (long)(base + off + 17), 3, + 0) || + node != 1) + finish(30); + } + if (call(91, raw, length, 0, 0, 0, 0)) + finish(31); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(number(argv[1]), pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-perf-probe.c b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c new file mode 100644 index 00000000000000..ee02fc34051adb --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-perf-probe.c @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 perf mmap layout, including native-page fallback. */ +typedef unsigned int u32; +typedef unsigned long long u64; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 PERF FAIL stage=00\n"; + + if (code) { + text[20] = '0' + code / 10; + text[21] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 PERF PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, unsigned long native, + unsigned int count) +{ + static const struct { + u32 type, size; + u64 config, sample_period, sample_type, read_format, flags; + u32 wakeup, bp_type; + u64 config1; + } attr = { .type = 1, .size = 64, .flags = 1 }; + long fd = call(364, (long)&attr, 0, -1, -1, 0, 0); + unsigned long length = (1 + count) * pages; + long raw; + volatile u64 *metadata; + + if (fd < 0) + finish(20); + raw = call(192, 0, length, 3, 1, fd, 0); + if ((unsigned long)raw >= 0xfffff001UL) + finish(21); + metadata = (void *)(raw + 1024); + if (metadata[2] != pages || metadata[3] != count * pages) + finish(22); + metadata[1] = 0; + metadata[6] = length; + metadata[7] = native; + /* mmap2 always takes 4K units, including the no4K compat fallback. */ + if (call(192, 0, native, 1, 1, fd, length / 4096) != -95) + finish(23); + if (call(91, raw, length, 0, 0, 0, 0)) + finish(24); + if (call(6, fd, 0, 0, 0, 0, 0)) + finish(25); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + unsigned long native = number(argv[1]); + + mapping(pages, native, 0); + mapping(pages, native, 1); + mapping(pages, native, 2); + mapping(pages, native, 2 * native / pages); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-policy-probe.c b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c new file mode 100644 index 00000000000000..dab5a79abe1302 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-policy-probe.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 -> AArch64 exec with an inherited granule preference. */ +#include "../../../../../include/uapi/linux/prctl.h" +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned int number(const char *s) +{ + unsigned int n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void fail(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[3])) + fail(10); + if (call(172, PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0, 0, 0, 0, 0) != + number(argv[2])) + fail(11); + if (call(172, PR_AURORA_GET_EXEC_PAGE_SIZE, 0, 0, 0, 0, 0)) + fail(12); + long mem = call(192, 0, 4 * pages, 3, 0x22, -1, 0); + if ((unsigned long)mem >= 0xfffff001UL) + fail(13); + long ret = call(125, mem + 4096, 4096, 1, 0, 0, 0); + if (ret != (pages == 4096 ? 0 : -22)) + fail(14); + if (call(91, mem, 4 * pages, 0, 0, 0, 0)) + fail(15); + char *args[] = { "/init", "--compat-return", argv[1], argv[2], 0 }; + call(11, (long)args[0], (long)args, (long)env, 0, 0, 0); + fail(16); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-probe.c b/tools/testing/selftests/arm64/user4k/compat-probe.c new file mode 100644 index 00000000000000..14a4deb7164126 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-probe.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding AArch32 process; run only by compat-fixture-init in a guest. */ +typedef unsigned int u32; +typedef unsigned short u16; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f, r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static unsigned length(const char *s) +{ + unsigned n = 0; + while (s[n]) + n++; + return n; +} +static int equal(const char *a, const char *b) +{ + while (*a && *a == *b) { + a++; + b++; + } + return *a == *b; +} +static unsigned number(const char *s) +{ + unsigned n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +static void out(const char *s) +{ + call(4, 1, (long)s, length(s), 0, 0, 0); +} +static int failures, checks; +static void result(int pass, const char *s) +{ + checks++; + if (!pass) + failures++; + out(pass ? "ok - compat " : "not ok - compat "); + out(s); + out("\n"); +} +static int error(long v) +{ + return (unsigned long)v >= 0xfffff001UL; +} +static long map(unsigned size, int prot, int flags, int fd, unsigned off) +{ + return call(192, 0, size, prot, flags, fd, off); +} +static void unmap(long p, unsigned size) +{ + if (!error(p)) + call(91, p, size, 0, 0, 0, 0); +} +static volatile unsigned signals; +static void handler(int n) +{ + if (n == 10) + signals++; +} +struct action { + u32 handler, flags, restorer, mask[2]; +}; +struct ehdr { + unsigned char ident[16]; + u16 type, machine; + u32 version, entry, phoff, shoff, flags; + u16 ehsize, phentsize, phnum, shentsize, shnum, shstrndx; +}; +struct phdr { + u32 type, offset, vaddr, paddr, filesz, memsz, flags, align; +}; +struct dyn { + u32 tag, val; +}; +struct sym { + u32 name, value, size; + unsigned char info, other; + u16 shndx; +}; +static void *vdso_symbol(unsigned base, const char *name) +{ + struct ehdr *h = (void *)base; + struct phdr *p = (void *)(base + h->phoff); + struct dyn *d = 0; + char *strings = 0; + struct sym *symbols = 0; + u32 *hash = 0; + for (unsigned i = 0; i < h->phnum; i++) + if (p[i].type == 2) + d = (void *)(base + p[i].vaddr); + if (!d) + return 0; + for (; d->tag; d++) { + if (d->tag == 5) + strings = (void *)(base + d->val); + if (d->tag == 6) + symbols = (void *)(base + d->val); + if (d->tag == 4) + hash = (void *)(base + d->val); + } + if (!strings || !symbols || !hash) + return 0; + for (unsigned i = 0; i < hash[1]; i++) + if (equal(strings + symbols[i].name, name)) + return (void *)(base + symbols[i].value); + return 0; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + unsigned argc = *sp++, pagesz = 0, vdso = 0; + char **argv = (void *)sp; + sp += argc + 1; + char **env = (void *)sp; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pagesz = sp[1]; + if (sp[0] == 33) + vdso = sp[1]; + sp += 2; + } + if (argc > 1 && equal(argv[1], "--exec32")) { + char *a[] = { argv[0], argv[2], argv[3], argv[4], 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 97, 0, 0, 0, 0, 0); + for (;;) + ; + } + if (argc > 1 && + (equal(argv[1], "--exec64") || equal(argv[1], "--exec64-4k"))) { + int alternative = equal(argv[1], "--exec64-4k"); + if (alternative && call(172, 0x41555001, 4096, 0, 0, 0, 0)) { + call(1, 99, 0, 0, 0, 0, 0); + for (;;) + ; + } + char *a[] = { "/init", + alternative ? "--returned4k" : "--returned", 0 }; + call(11, (long)a[0], (long)a, (long)env, 0, 0, 0); + call(1, 98, 0, 0, 0, 0, 0); + for (;;) + ; + } + /* Script interpreter contributes its pathname before the ordinary arguments. */ + unsigned start = argc > 1 && equal(argv[1], "/script32") ? 2 : 1; + unsigned expected = argc > start ? number(argv[start]) : 0; + int fd = argc > start + 1 ? (int)number(argv[start + 1]) : -1; + result(pagesz == expected, "AT_PAGESZ reflects final ELF32 selection"); + int args = argc > start + 2 && length(argv[start + 2]) == 90000; + if (args) + for (unsigned i = 0; i < 90000; i++) + if (argv[start + 2][i] != (char)('a' + i % 23)) { + args = 0; + break; + } + result(args, "90KB staged argument survives granule transfer"); + long mem = map(4 * pagesz, 3, 0x22, -1, 0); + int ok = !error(mem); + if (ok) { + unsigned char *p = (void *)mem; + for (unsigned i = 0; i < 4 * pagesz; i++) + p[i] = 0x30 + i / pagesz; + for (unsigned i = 0; i < 4 * pagesz; i++) + if (p[i] != (unsigned char)(0x30 + i / pagesz)) + ok = 0; + } + result(ok, "anonymous user leaves have independent bytes"); + result(!error(mem) && !call(125, mem + pagesz, pagesz, 1, 0, 0, 0) && + !call(125, mem + pagesz, pagesz, 3, 0, 0, 0), + "mprotect at selected page boundaries"); + result(!error(mem) && call(125, mem + 1, pagesz, 1, 0, 0, 0) == -22 && + call(91, mem + 1, pagesz, 0, 0, 0, 0) == -22, + "unaligned memory operations retain restrictions"); + long q = map(pagesz, 1, 1, fd, 3 * (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x40 + (3 * pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 native-aligned file offset"); + unmap(q, pagesz); + q = map(4096, 1, 1, fd, 1); + ok = pagesz == 4096 ? !error(q) && *(unsigned char *)q == 0x41 : + q == -22; + result(ok, "mmap2 4KB offset accepted only by matching ABI"); + unmap(q, pagesz); + q = map(pagesz, 1, 1, fd, 0x100000 + (pagesz >> 12)); + ok = !error(q); + if (ok) + for (unsigned i = 0; i < pagesz; i++) + if (((unsigned char *)q)[i] != + (unsigned char)(0x80 + (pagesz + i) / 4096)) + ok = 0; + result(ok, "mmap2 offset above 4GB remains full width"); + unmap(q, pagesz); + long child = call(2, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + if (!error(mem)) + *(unsigned char *)mem = 0xa7; + call(1, 0, 0, 0, 0, 0, 0); + for (;;) + ; + } + ok = child > 0 && + call(114, child, (long)&status, 0, 0, 0, 0) == child && !status && + !error(mem) && *(unsigned char *)mem == 0x30; + result(ok, "fork isolates private COW data"); + unmap(mem, 4 * pagesz); + struct action act = { (u32)handler, 0, 0, { 0, 0 } }; + ok = !call(174, 10, (long)&act, 0, 8, 0, 0); + if (ok) + ok = !call(37, call(20, 0, 0, 0, 0, 0, 0), 10, 0, 0, 0, 0) && + signals == 1; + result(ok, "AArch32 signal frame and kernel sigreturn trampoline"); + volatile unsigned word = 17; + int (*cmpxchg)(unsigned, unsigned, volatile unsigned *) = + (void *)0xffff0fc0; + result(*(volatile unsigned *)0xffff0ffc >= 5 && + !cmpxchg(17, 29, &word) && word == 29, + "kuser helper page and atomic compare exchange"); + int (*gettime)(int, void *) = + vdso ? (void *)vdso_symbol(vdso, "__vdso_clock_gettime") : 0; + long time[2] = { 0, 0 }; + result(gettime && !gettime(1, time) && time[0] >= 0 && time[1] >= 0 && + time[1] < 1000000000, + "compat vDSO clock_gettime and vvar mapping"); + out(failures ? "COMPAT PROBE FAIL\n" : "COMPAT PROBE PASS\n"); + call(1, failures ? 1 : 0, 0, 0, 0, 0, 0); + for (;;) + ; +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-script b/tools/testing/selftests/arm64/user4k/compat-script new file mode 100644 index 00000000000000..9b56d0c7567f53 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-script @@ -0,0 +1 @@ +#!/probe32-64k diff --git a/tools/testing/selftests/arm64/user4k/compat-shm-probe.c b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c new file mode 100644 index 00000000000000..c4ee1da2c5a120 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-shm-probe.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 SysV attach/detach and fallback-alignment checks. */ +typedef unsigned int u32; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static unsigned int number(const char *s) +{ + unsigned int n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static _Noreturn void finish(int code) +{ + char text[] = "A32 SHM FAIL stage=00\n"; + + if (code) { + text[19] = '0' + code / 10; + text[20] = '0' + code % 10; + call(4, 1, (long)text, sizeof(text) - 1, 0, 0, 0); + } else { + static const char pass[] = "A32 SHM PASS\n"; + + call(4, 1, (long)pass, sizeof(pass) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} + +static void mapping(unsigned long pages, int rounded) +{ + unsigned long align = 4 * pages, length = 4 * pages; + long raw = call(192, 0, 12 * align, 3, 0x22, -1, 0); + unsigned char *target; + long id, result; + + if ((unsigned long)raw >= 0xfffff001UL) + finish(20); + result = call(125, raw + 4096, 4096, 1, 0, 0, 0); + if (result != (pages == 4096 ? 0 : -22)) + finish(21); + if (call(125, raw, 12 * align, 3, 0, 0, 0)) + finish(22); + target = (void *)(((unsigned long)raw + align - 1) & ~(align - 1)); + target += align + (rounded ? 0 : pages); + target[-1] = 0x51; + target[length] = 0x72; + if (call(91, (long)target, length, 0, 0, 0, 0)) + finish(23); + id = call(307, 0, length - 1, 0600, 0, 0, 0); + if (id < 0) + finish(24); + result = call(305, id, (long)(target + (rounded ? pages + 17 : 0)), + rounded ? 020000 : 0, 0, 0, 0); + if (result != (long)target) { + call(308, id, 0, 0, 0, 0, 0); + finish(rounded ? 26 : 25); + } + if (call(308, id, 0, 0, 0, 0, 0)) + finish(27); + target[pages + 13] = 0x67; + if (call(125, (long)(target + pages), pages, 1, 0, 0, 0)) + finish(28); + if (call(91, (long)target, pages, 0, 0, 0, 0)) + finish(29); + if (call(306, (long)target, 0, 0, 0, 0, 0)) + finish(30); + if (target[-1] != 0x51 || target[length] != 0x72) + finish(31); + if (call(91, raw, 12 * align, 0, 0, 0, 0)) + finish(32); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 3 || pages != number(argv[2])) + finish(10); + mapping(pages, 0); + mapping(pages, 1); + finish(0); +} + +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-init.c b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c new file mode 100644 index 00000000000000..5f1e90f3e484f3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-init.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - compat uffd line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct wire { + uint32_t address, pagesize; +}; +static void expect_invalid(int fd, uint64_t dst, uint64_t src, uint64_t len) +{ + struct uffdio_copy c = { .dst = dst, .src = src, .len = len }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EINVAL); +} +static void run(unsigned long native, unsigned long expected, int shared) +{ + int sock[2], file = -1, status; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + if (shared) { + file = memfd_create("compat-uffd", 0); + CHECK(file >= 0 && !ftruncate(file, 3 * expected)); + } + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char page_arg[24], sock_arg[24], file_arg[24]; + close(sock[0]); + snprintf(page_arg, sizeof(page_arg), "%lu", expected); + snprintf(sock_arg, sizeof(sock_arg), "%d", sock[1]); + snprintf(file_arg, sizeof(file_arg), "%d", file); + execl("/compat-uffd", "/compat-uffd", page_arg, sock_arg, + file_arg, NULL); + _exit(99); + } + close(sock[1]); + struct wire data = { 0 }; + struct iovec iov = { &data, sizeof(data) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(data)); + CHECK(!(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) && + data.pagesize == expected); + struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg); + CHECK(cmsg && cmsg->cmsg_level == SOL_SOCKET && + cmsg->cmsg_type == SCM_RIGHTS && + cmsg->cmsg_len == CMSG_LEN(sizeof(int))); + int uffd; + memcpy(&uffd, CMSG_DATA(cmsg), sizeof(uffd)); + /* Force the source above every possible AArch32 task limit. */ + unsigned char *source = + mmap((void *)0x1000000000UL, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(source != MAP_FAILED && (uintptr_t)source > UINT32_MAX); + /* Source alignment is independent of both processes' granules. */ + unsigned char *bytes = source + 17; + for (unsigned leaf = 0; leaf < 3; leaf++) { + struct pollfd pfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg event; + CHECK(poll(&pfd, 1, 10000) == 1 && (pfd.revents & POLLIN)); + CHECK(read(uffd, &event, sizeof(event)) == sizeof(event)); + uint64_t dst = (uint64_t)data.address + leaf * expected; + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + !event.arg.pagefault.flags && + event.arg.pagefault.address == dst); + if (!leaf) { + expect_invalid(uffd, dst, (uintptr_t)bytes, 0); + expect_invalid(uffd, dst, (uintptr_t)bytes, + expected - 1); + expect_invalid(uffd, dst + 1, (uintptr_t)bytes, + expected); + expect_invalid(uffd, 0x100000000ULL, (uintptr_t)bytes, + expected); + expect_invalid(uffd, dst, UINT64_MAX - expected + 1, + expected); + if (expected > 4096) + expect_invalid(uffd, dst, (uintptr_t)bytes, + 4096); + } + if (leaf == 1) { + /* A valid but inaccessible pager source must leave the target missing. */ + CHECK(!mprotect(source, 2 * native, PROT_NONE)); + struct uffdio_copy bad = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &bad) == -1 && + errno == EFAULT && bad.copy == -EFAULT); + CHECK(!mprotect(source, 2 * native, + PROT_READ | PROT_WRITE)); + } + memset(bytes, 0x51 + leaf, expected); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)bytes, + .len = expected }; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && + c.copy == (long long)expected); + } + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!munmap(source, 2 * native)); + CHECK(!close(uffd) && !close(sock[0])); + if (shared) + CHECK(!close(file)); + printf("ok - native %lu pager high unaligned COPY to AArch32 %lu %s, source failure and target alignment\n", + native, expected, shared ? "shared" : "anonymous"); +} +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + alarm(90); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + char cmdline[2048] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + CHECK(fd >= 0 && read(fd, cmdline, sizeof(cmdline) - 1) > 0 && + !close(fd)); + unsigned long native = getauxval(AT_PAGESZ); + unsigned long expected = + strstr(cmdline, "id_aa64mmfr0.tgran4=f") ? native : 4096; + run(native, expected, 0); + run(native, expected, 1); + puts("COMPAT UFFD PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c new file mode 100644 index 00000000000000..6abcdb86bb4d82 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-uffd-probe.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding ELF32 half of compat-uffd-init; no armhf sysroot required. */ +typedef unsigned int u32; +typedef unsigned long long u64; +struct api { + u64 api, features, ioctls; +}; +struct registration { + u64 start, len, mode, ioctls; +}; +struct movement { + u64 dst, src, len, mode; + long long moved; +}; +struct wire { + u32 address, pagesize; +}; +struct iovec32 { + void *base; + u32 len; +}; +struct message32 { + void *name; + u32 namelen; + struct iovec32 *iov; + u32 iovlen; + void *control; + u32 controllen, flags; +}; +struct rights32 { + u32 len, level, type; + int fd; +}; +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} +static u32 number(const char *s) +{ + u32 n = 0; + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} +__attribute__((noreturn)) static void finish(int code) +{ + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +__attribute__((noreturn)) void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + if (argc != 4 || pages != number(argv[1])) + finish(10); + int sock = number(argv[2]), shared = argv[3][0] != '-'; + int file = shared ? (int)number(argv[3]) : -1; + long map = call(192, 0, 3 * pages, 3, shared ? 1 : 0x22, file, 0); + if ((unsigned long)map >= 0xfffff001UL) + finish(11); + /* Fine operations must retain native alignment restrictions on fallback. */ + if (call(125, map + 4096, 4096, 1, 0, 0, 0) != + (pages == 4096 ? 0 : -22)) + finish(12); + if (call(125, map, 3 * pages, 3, 0, 0, 0)) + finish(13); + long fd = call(388, 0x80801, 0, 0, 0, 0, + 0); /* CLOEXEC|NONBLOCK|USER_MODE_ONLY */ + if (fd < 0) + finish(14); + /* Negotiate shared missing support under both fine and fallback ABIs. */ + struct api api = { 0xaa, (1ULL << 5) | (1ULL << 16), 0 }; + if (call(54, fd, 0xc018aa3fU, (long)&api, 0, 0, 0) || + !(api.features & (1ULL << 5))) + finish(15); + struct registration reg = { (u32)map, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)®, 0, 0, 0)) + finish(16); + if (!(reg.ioctls & (1ULL << 3)) || !(api.features & (1ULL << 16))) + finish(17); /* COPY and MOVE */ + struct wire data = { (u32)map, pages }; + struct iovec32 iov = { &data, sizeof(data) }; + struct rights32 rights = { sizeof(rights), 1, 1, fd }; + struct message32 msg = { 0, 0, &iov, 1, &rights, sizeof(rights), 0 }; + if (call(296, sock, (long)&msg, 0, 0, 0, 0) != sizeof(data)) + finish(18); + volatile unsigned char *p = (void *)map; + /* Each first access waits for the native pager's COPY. */ + for (u32 leaf = 0; leaf < 3; leaf++) + for (u32 i = 0; i < pages; i++) + if (p[leaf * pages + i] != (unsigned char)(0x51 + leaf)) + finish(19); + if (!shared) { + long target = call(192, 0, 3 * pages, 3, 0x22, -1, 0); + if ((unsigned long)target >= 0xfffff001UL) + finish(21); + struct registration dst = { (u32)target, 3 * pages, 1, 0 }; + if (call(54, fd, 0xc020aa00U, (long)&dst, 0, 0, 0) || + !(dst.ioctls & (1ULL << 5))) + finish(22); + struct movement m = { 1ULL << 32, (u32)map + 2 * pages, pages, + 0, 0 }; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(23); + m.dst = (u32)target + pages; + if (pages > 4096) { + m.len = 4096; + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) != -22) + finish(24); + m.len = pages; + } + if (call(54, fd, 0xc028aa05U, (long)&m, 0, 0, 0) || + m.moved != pages) + finish(25); + volatile unsigned char *q = (void *)target; + for (u32 i = 0; i < pages; i++) + if (q[pages + i] != 0x53 || p[i] != 0x51 || + p[pages + i] != 0x52) + finish(26); + if (call(91, target, 3 * pages, 0, 0, 0, 0)) + finish(27); + static const char ok[] = + "ok - AArch32 MOVE, neighbours and native-fallback alignment\n"; + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + } + if (call(91, map, 3 * pages, 0, 0, 0, 0)) + finish(20); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c new file mode 100644 index 00000000000000..231e4b79966135 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/compat-vfio-probe.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Freestanding AArch32 type1 probe; no armhf sysroot or physical DMA. */ +#define __user +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +typedef unsigned int u32; +typedef unsigned long long u64; +#define IOVA (256UL * 1024 * 1024) +static unsigned int stage; + +static long call(long n, long a, long b, long c, long d, long e, long f) +{ + register long r0 __asm__("r0") = a, r1 __asm__("r1") = b, + r2 __asm__("r2") = c; + register long r3 __asm__("r3") = d, r4 __asm__("r4") = e, + r5 __asm__("r5") = f; + register long r7 __asm__("r7") = n; + + __asm__ volatile("svc 0" + : "+r"(r0) + : "r"(r1), "r"(r2), "r"(r3), "r"(r4), "r"(r5), "r"(r7) + : "memory", "cc"); + return r0; +} + +static u32 number(const char *s) +{ + u32 n = 0; + + while (*s >= '0' && *s <= '9') + n = n * 10 + *s++ - '0'; + return n; +} + +static void clear(void *data, u32 size) +{ + volatile unsigned char *p = data; + + for (u32 i = 0; i < size; i++) + p[i] = 0; +} + +/* Aggregate initialization may use these compiler-runtime entry points. */ +void __aeabi_memclr8(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr4(void *data, u32 size) +{ + clear(data, size); +} +void __aeabi_memclr(void *data, u32 size) +{ + clear(data, size); +} + +_Noreturn static void finish(int code) +{ + if (code) { + char msg[] = "not ok - AArch32 type1 stage=00\n"; + + msg[29] = '0' + stage / 10; + msg[30] = '0' + stage % 10; + call(4, 1, (long)msg, sizeof(msg) - 1, 0, 0, 0); + } + call(1, code, 0, 0, 0, 0, 0); + for (;;) + ; +} +#define CHECK(x) \ + do { \ + stage++; \ + if (!(x)) \ + finish(stage); \ + } while (0) + +static long ioctl_test(int fd, u32 request, void *arg) +{ + return call(54, fd, request, (long)arg, 0, 0, 0); +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + clear(cmd, sizeof(*cmd)); + return ioctl_test(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static long map(int fd, u32 address, u32 size) +{ + struct vfio_iommu_type1_dma_map cmd; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; + cmd.iova = IOVA; + cmd.vaddr = address; + cmd.size = size; + return ioctl_test(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static long unmap(int fd, u32 size) +{ + struct vfio_iommu_type1_dma_unmap cmd; + long ret; + + clear(&cmd, sizeof(cmd)); + cmd.argsz = sizeof(cmd); + cmd.iova = IOVA; + cmd.size = size; + ret = ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + return ret ? ret : cmd.size == size ? 0 : -1; +} + +static long rw(int fd, unsigned char *bytes, unsigned int flags) +{ + struct vfio_type1_test_cmd cmd; + + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA; + cmd.buffer = (u32)bytes; + cmd.size = 16; + cmd.flags = flags; + return ioctl_test(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +_Noreturn void probe(u32 *sp) +{ + u32 argc = *sp++, pages = 0; + char **argv = (void *)sp; + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dirty_bitmap dirty; + struct vfio_iommu_type1_info info; + unsigned char bytes[16]; + long fd, file, data, pid; + int status = -1; + u32 native, length; + + sp += argc + 1; + while (*sp) + sp++; + sp++; + while (sp[0]) { + if (sp[0] == 6) + pages = sp[1]; + sp += 2; + } + CHECK(argc == 3 && pages == number(argv[2])); + native = number(argv[1]); + length = native + 4096; + fd = call(5, (long)"/dev/vfio-test", 2, 0, 0, 0, 0); + file = call(385, (long)"compat-vfio", 0, 0, 0, 0, 0); + CHECK(fd >= 0 && file >= 0 && !call(93, file, 2 * native, 0, 0, 0, 0)); + data = call(192, 0, 2 * native, 3, 1, file, 0); + CHECK((u32)data < 0xfffff001U); + CHECK(call(125, data + 4096, 4096, 1, 0, 0, 0) == + (pages == 4096 ? 0 : -22)); + CHECK(!call(125, data, 2 * native, 3, 0, 0, 0)); + for (u32 i = 0; i < 2 * native; i++) + ((volatile unsigned char *)data)[i] = 0x53; + clear(&info, sizeof(info)); + info.argsz = sizeof(info); + CHECK(!ioctl_test(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096)); + CHECK(!map(fd, data, 2 * native)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes); + CHECK(!rw(fd, bytes, 0) && bytes[0] == 0x53 && bytes[15] == 0x53); + clear(&dirty, sizeof(dirty)); + dirty.argsz = sizeof(dirty); + dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, &dirty)); + clear(&cmd, sizeof(cmd)); + cmd.iova = IOVA + 19; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_PIN, &cmd)); + CHECK(!state(fd, &cmd) && cmd.locked_bytes == native && + cmd.external_refs == 1); + /* Compat pointer padding must not become part of the native pointer. */ + _Alignas(struct vfio_iommu_type1_dirty_bitmap_get) unsigned char + buffer[sizeof(dirty) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)]; + struct vfio_iommu_type1_dirty_bitmap *header = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)header->data; + u64 bits = 0; + + clear(buffer, sizeof(buffer)); + header->argsz = sizeof(buffer); + header->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = 2 * native; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(bits); + range->bitmap.data = &bits; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Static_assert( + sizeof(range->bitmap.data) == 4 && sizeof(range->bitmap) == 24, + "AArch32 bitmap has a 32-bit pointer and trailing padding"); + ((unsigned char *)&range + ->bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(range->bitmap.data)] = 0xa5; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_DIRTY_PAGES, buffer) && + bits == ((1ULL << (native >> 12)) - 1)); + _Alignas(struct vfio_bitmap) unsigned char + unmap_buffer[sizeof(struct vfio_iommu_type1_dma_unmap) + + sizeof(struct vfio_bitmap)]; + struct vfio_iommu_type1_dma_unmap *remove = (void *)unmap_buffer; + struct vfio_bitmap *remove_bitmap = (void *)remove->data; + + clear(unmap_buffer, sizeof(unmap_buffer)); + remove->argsz = sizeof(unmap_buffer); + remove->flags = VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP; + remove->iova = IOVA; + remove->size = 2 * native; + remove_bitmap->pgsize = 4096; + remove_bitmap->size = sizeof(bits); + remove_bitmap->data = &bits; + ((unsigned char *) + remove_bitmap)[__builtin_offsetof(struct vfio_bitmap, data) + + sizeof(remove_bitmap->data)] = 0x5a; + bits = 0; + CHECK(!ioctl_test(fd, VFIO_IOMMU_UNMAP_DMA, unmap_buffer) && + remove->size == 2 * native && + bits == ((1ULL << (native >> 12)) - 1)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1); + clear(&cmd, sizeof(cmd)); + cmd.size = 4096; + CHECK(!ioctl_test(fd, VFIO_TYPE1_TEST_DOMAIN_ADD, &cmd)); + pid = call(2, 0, 0, 0, 0, 0, 0); + if (!pid) + finish(map(fd, data + 4096, length) ? 99 : 0); + CHECK(pid > 0 && call(114, pid, (long)&status, 0, 0, 0, 0) == pid && + !status); + CHECK(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && cmd.mapped_bytes[0] == length); + CHECK(rw(fd, bytes, 0) == -14); + CHECK(!rw(fd, bytes, VFIO_TYPE1_TEST_WORKER) && bytes[0] == 0x53 && + bytes[15] == 0x53); + CHECK(!unmap(fd, length)); + CHECK(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0]); + CHECK(!call(91, data, 2 * native, 0, 0, 0, 0)); + CHECK(!call(6, file, 0, 0, 0, 0, 0) && !call(6, fd, 0, 0, 0, 0, 0)); + static const char ok[] = + "ok - AArch32 type1 real ioctls, source exit and fallback alignment\n"; + + call(4, 1, (long)ok, sizeof(ok) - 1, 0, 0, 0); + finish(0); +} +__asm__(".text\n.global _start\n.type _start,%function\n_start:\nmov r0,sp\nbl probe\n"); diff --git a/tools/testing/selftests/arm64/user4k/context-stress-init.c b/tools/testing/selftests/arm64/user4k/context-stress-init.c new file mode 100644 index 00000000000000..869662502c15f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/context-stress-init.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 mixed-granule EL0 stress runner for disposable VMs. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define DATA_ADDRESS 0x200000000UL +#define CODE_ADDRESS 0x202000000UL +#define CONTEXTS 70000 +#define SHORT_CONTEXTS 2048 + +static sigjmp_buf fault_return; +static volatile uintptr_t expected_fault; +static char generation_path[256]; + +static void fault(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGSEGV || !expected_fault || + (uintptr_t)info->si_addr != expected_fault || + info->si_code != SEGV_ACCERR) + _exit(90); + expected_fault = 0; + siglongjmp(fault_return, 1); +} + +static bool rejects_write(uintptr_t address) +{ + expected_fault = address; + if (!sigsetjmp(fault_return, 1)) { + *(volatile unsigned char *)expected_fault = 1; + expected_fault = 0; + return false; + } + return true; +} + +static int migrate(unsigned int cpu) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(0, sizeof(set), &set); +} + +static unsigned char pattern(unsigned long serial, unsigned long offset) +{ + serial ^= (serial >> 8) ^ (serial >> 16); + return (serial * 37 + offset * 17 + (offset >> 8)) & 255; +} + +static bool validate(unsigned char *p, unsigned long serial, unsigned long len) +{ + for (unsigned long i = 0; i < len; i++) + if (p[i] != pattern(serial, i)) + return false; + return true; +} + +static int worker(unsigned long serial, unsigned long ps, int control) +{ + struct sigaction action = { .sa_sigaction = fault, + .sa_flags = SA_SIGINFO }; + unsigned char *data; + uint32_t *code; + unsigned int cpus = sysconf(_SC_NPROCESSORS_ONLN); + struct stat self, init; + unsigned long iterations = 0; + + if (getauxval(AT_PAGESZ) != ps || getppid() != 1 || + stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) != 0) + return 91; + if (cpus < 2 || migrate(serial % cpus) || + sigaction(SIGSEGV, &action, NULL)) + return 92; + data = mmap((void *)DATA_ADDRESS, 4 * ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + code = mmap((void *)CODE_ADDRESS, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED || code == MAP_FAILED || + mprotect(data, 3 * ps, PROT_READ | PROT_WRITE)) + return 93; + for (unsigned long i = 0; i < 3 * ps; i++) + data[i] = pattern(serial, i); + if (!rejects_write(DATA_ADDRESS + 3 * ps)) + return 94; + if (mprotect(data + ps, ps, PROT_READ)) + return 95; + if (!rejects_write(DATA_ADDRESS + ps)) + return 96; + do { + unsigned int expected = (serial + iterations * 13) & 65535; + int (*fn)(void) = (void *)code; + char stop; + ssize_t got; + + if (mprotect(code, ps, PROT_READ | PROT_WRITE)) + return 97; + code[0] = 0x52800000 | (expected << 5); /* movz w0, #expected */ + code[1] = 0xd65f03c0; /* ret */ + __builtin___clear_cache((char *)code, (char *)(code + 2)); + if (mprotect(code, ps, PROT_READ | PROT_EXEC) || + migrate((serial + iterations + 1) % cpus)) + return 98; + sched_yield(); + if (fn() != (int)expected || !validate(data, serial, 3 * ps)) + return 99; + if (!(iterations % 64) && + (!rejects_write(DATA_ADDRESS + 3 * ps) || + !rejects_write(DATA_ADDRESS + ps))) + return 101; + iterations++; + if (control < 0) + break; + got = read(control, &stop, 1); + if (got == 1) + break; + if (got != -1 || (errno != EAGAIN && errno != EINTR)) + return 100; + } while (true); + if (control >= 0) + printf("# persistent %luK worker: %lu migration/JIT/data cycles\n", + ps / 1024, iterations); + return 0; +} + +static pid_t spawn(unsigned long serial, unsigned long ps, int control, + int close_fd) +{ + pid_t pid = fork(); + + if (!pid) { + char seq[32], size[32], fd[32]; + + if (close_fd >= 0) + close(close_fd); + if (control >= 0 && fcntl(control, F_SETFD, 0)) + _exit(110); + snprintf(seq, sizeof(seq), "%lu", serial); + snprintf(size, sizeof(size), "%lu", ps); + snprintf(fd, sizeof(fd), "%d", control); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, ps, 0UL, 0UL, 0UL)) + _exit(111); + execl("/init", "/init", "--worker", seq, size, fd, NULL); + _exit(112); + } + return pid; +} + +static bool reap(pid_t pid) +{ + int status = 0; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + if (got == pid && WIFEXITED(status) && !WEXITSTATUS(status)) + return true; + printf("not ok - worker pid=%d wait=%d status=%#x\n", pid, got, status); + return false; +} + +static unsigned long long generation(void) +{ + unsigned long long value = 0; + FILE *f = fopen(generation_path, "re"); + + if (!f || fscanf(f, "%llu", &value) != 1) { + perror("ASID generation"); + return 0; + } + fclose(f); + return value; +} + +int main(int argc, char **argv) +{ + unsigned long long before, after; + unsigned char *data; + pid_t persistent[3]; + int commands[3], pipes[2]; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + glob_t paths; + bool ok = true; + unsigned int contexts = CONTEXTS; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (argc == 2 && !strcmp(argv[1], "--short")) + contexts = SHORT_CONTEXTS; + if (getpid() != 1 || (argc != 1 && contexts != SHORT_CONTEXTS)) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + glob("/sys/module/*/parameters/user4k_asid_generation", 0, NULL, + &paths) || + paths.gl_pathc != 1) { + perror("fixture setup"); + return 2; + } + snprintf(generation_path, sizeof(generation_path), "%s", + paths.gl_pathv[0]); + globfree(&paths); + data = mmap((void *)DATA_ADDRESS, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (data == MAP_FAILED) + return 3; + for (unsigned long i = 0; i < 4 * native; i++) + data[i] = pattern(0, i); + before = generation(); + printf("# mixed context stress: %u fork/exec workers, initial ASID generation=%llu\n", + contexts, before); + for (unsigned int i = 0; i < nr; i++) { + if (pipe2(pipes, O_NONBLOCK | O_CLOEXEC)) + return 4; + persistent[i] = spawn(CONTEXTS + i + 1, granules[i], pipes[0], + pipes[1]); + if (persistent[i] < 0) + return 5; + close(pipes[0]); + commands[i] = pipes[1]; + } + for (unsigned long i = 1; i <= contexts; i++) { + pid_t pid = spawn(i, granules[i % nr], -1, -1); + + if (pid < 0 || !reap(pid) || !validate(data, 0, 4 * native)) { + printf("not ok - churn/parent isolation at worker %lu\n", + i); + ok = false; + break; + } + if (!(i % 128)) + printf("# workers=%lu ASID generation=%llu\n", i, + generation()); + } + after = generation(); + for (unsigned int i = 0; i < nr; i++) { + ok &= write(commands[i], "x", 1) == 1; + close(commands[i]); + ok &= reap(persistent[i]); + } + ok &= before && after >= before + 2 * (1ULL << 16); + printf("%s - mixed context stress: generation=%llu->%llu, measured rollovers=%llu\n", + ok ? "ok" : "not ok", before, after, (after - before) >> 16); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/core-fixture-init.c b/tools/testing/selftests/arm64/user4k/core-fixture-init.c new file mode 100644 index 00000000000000..4caadf98296a65 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/core-fixture-init.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Core-dump regression runner: PID 1 in a disposable VM only. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define ANON 0x100000000UL +#define FILEMAP 0x102000000UL +#define SPARSE 0x104000000UL +#define EXCLUDED 0x106000000UL +#define SWAPPED 0x108000000UL +#define SWAPPED_SIZE (1UL << 20) +#define SPARSE_SIZE (8UL << 20) + +static void die(const char *what) +{ + perror(what); + _exit(120); +} + +static void write_text(const char *path, const char *text) +{ + int fd = open(path, O_WRONLY); + size_t len = strlen(text); + + if (fd < 0 || write(fd, text, len) != (ssize_t)len) + die(path); + close(fd); +} + +static void *map_at(unsigned long address, size_t len, int prot, int fd, + off_t off) +{ + void *p = mmap((void *)address, len, prot, + MAP_PRIVATE | MAP_FIXED_NOREPLACE | + (fd < 0 ? MAP_ANONYMOUS : 0), + fd, off); + + if (p == MAP_FAILED) + die("mmap"); + return p; +} + +static void crash(bool headers, int bounded) +{ + unsigned long ps = getauxval(AT_PAGESZ); + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + struct stat self, init; + unsigned char *p, *data; + int fd; + + if (getppid() != 1 || stat("/proc/self/exe", &self) || + stat("/proc/1/exe", &init) || self.st_dev != init.st_dev || + self.st_ino != init.st_ino) + _exit(121); + if (bounded) + limit.rlim_cur = limit.rlim_max = ps - (bounded < 0); + if (setrlimit(RLIMIT_CORE, &limit)) + die("rlimit"); + write_text("/proc/self/coredump_filter", headers ? "0x10" : "0x1f"); + p = map_at(ANON, 4 * ps, PROT_READ | PROT_WRITE, -1, 0); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x31 + i, ps); + if (mprotect(p + ps, ps, PROT_NONE) || + mprotect(p + 2 * ps, ps, PROT_READ)) + die("mprotect"); + p = map_at(SPARSE, SPARSE_SIZE, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0x65, ps); + /* Also exercise an existing read-only zero PTE. */ + if (*(volatile unsigned char *)(p + ps) != 0) + _exit(122); + p = map_at(EXCLUDED, ps, PROT_READ | PROT_WRITE, -1, 0); + memset(p, 0xa7, ps); + if (madvise(p, ps, MADV_DONTDUMP)) + die("dontdump"); + fd = open("/core-input", O_CREAT | O_TRUNC | O_RDWR, 0700); + data = malloc(19 * ps); + if (fd < 0 || !data) + die("file setup"); + for (unsigned int i = 0; i < 19; i++) + memset(data + i * ps, 0x81 + i, ps); + memcpy(data, ELFMAG, SELFMAG); + if (write(fd, data, 19 * ps) != (ssize_t)(19 * ps)) + die("file data"); + /* Offset one user page: three mappings select non-native quarters. */ + map_at(FILEMAP, ps, PROT_READ, fd, headers ? 0 : ps); + map_at(FILEMAP + 2 * ps, 2 * ps, PROT_READ, fd, 2 * ps); + map_at(FILEMAP + 8 * ps, ps, PROT_READ, fd, 17 * ps); + close(fd); + fd = open("/core-library", O_CREAT | O_TRUNC | O_RDWR, 0600); + if (fd < 0 || write(fd, data, ps) != (ssize_t)ps) + die("non-executable ELF file"); + map_at(FILEMAP + 12 * ps, ps, PROT_READ, fd, 0); + close(fd); + free(data); + if (!headers && !bounded) { + unsigned int missing = 0; + uint64_t entry; + int pm = open("/proc/self/pagemap", O_RDONLY); + + p = map_at(SWAPPED, SWAPPED_SIZE, PROT_READ | PROT_WRITE, -1, + 0); + memset(p, 0xc4, SWAPPED_SIZE); + for (unsigned int tries = 0; tries < 200; tries++) { + if (madvise(p, SWAPPED_SIZE, MADV_COLD) || + madvise(p, SWAPPED_SIZE, MADV_PAGEOUT)) + die("pageout"); + missing = 0; + for (unsigned long i = 0; i < SWAPPED_SIZE / ps; i++) { + if (pread(pm, &entry, sizeof(entry), + (SWAPPED / ps + i) * 8) != 8) + die("pagemap"); + missing += !(entry & (1ULL << 62)); + } + if (!missing) + break; + usleep(10000); + } + close(pm); + if (missing) + _exit(124); + printf("# %luK dump precondition: all %lu leaves swapped\n", + ps / 1024, SWAPPED_SIZE / ps); + } + /* The file VMAs remain unfaulted, so dumping must retrieve file data. */ + raise(SIGABRT); + _exit(123); +} + +static Elf64_Phdr *find_load(unsigned char *data, Elf64_Ehdr *elf, + unsigned long addr) +{ + Elf64_Phdr *ph = (void *)(data + elf->e_phoff); + + for (unsigned int i = 0; i < elf->e_phnum; i++) + if (ph[i].p_type == PT_LOAD && addr >= ph[i].p_vaddr && + addr - ph[i].p_vaddr < ph[i].p_memsz) + return ph + i; + return NULL; +} + +static bool bytes_at(unsigned char *data, Elf64_Ehdr *elf, size_t len, + unsigned long addr, size_t count, unsigned char expected) +{ + Elf64_Phdr *p = find_load(data, elf, addr); + unsigned long off; + + if (!p || addr - p->p_vaddr + count > p->p_filesz) + return false; + off = p->p_offset + addr - p->p_vaddr; + if (off > len || count > len - off) + return false; + for (size_t i = 0; i < count; i++) + if (data[off + i] != expected) + return false; + return true; +} + +static void inspect(pid_t pid, unsigned long ps, bool headers) +{ + char path[64]; + struct stat st; + unsigned char *data; + Elf64_Ehdr *elf; + Elf64_Phdr *ph, *p; + bool geometry = true, aux = false; + unsigned int file = 0; + unsigned long vdso = 0; + int fd; + + snprintf(path, sizeof(path), "/core.%d", pid); + fd = open(path, O_RDONLY); + if (fd < 0 || fstat(fd, &st) || st.st_size < (off_t)sizeof(*elf)) + ksft_exit_fail_msg("missing/short core %s\n", path); + data = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (data == MAP_FAILED) + die("core read"); + elf = (void *)data; + if (memcmp(elf->e_ident, ELFMAG, SELFMAG) || elf->e_type != ET_CORE || + elf->e_phentsize != sizeof(*ph) || + elf->e_phoff > (uint64_t)st.st_size || + elf->e_phnum * sizeof(*ph) > (uint64_t)st.st_size - elf->e_phoff) + ksft_exit_fail_msg("invalid core ELF\n"); + ph = (void *)(data + elf->e_phoff); + for (unsigned int i = 0; i < elf->e_phnum; i++) { + if (ph[i].p_type == PT_LOAD) { + geometry &= ph[i].p_align == ps && + ph[i].p_filesz <= ph[i].p_memsz && + ph[i].p_offset % ps == ph[i].p_vaddr % ps; + } else if (ph[i].p_type == PT_NOTE) { + size_t at = ph[i].p_offset, end = at + ph[i].p_filesz; + + if (end > (size_t)st.st_size || end < at) + ksft_exit_fail_msg("invalid notes\n"); + while (at + sizeof(Elf64_Nhdr) <= end) { + Elf64_Nhdr *n = (void *)(data + at); + size_t desc = at + sizeof(*n) + + ((n->n_namesz + 3UL) & ~3UL); + uint64_t *words = (void *)(data + desc); + + if (desc > end || n->n_descsz > end - desc) + ksft_exit_fail_msg( + "invalid note length\n"); + if (n->n_type == NT_AUXV) { + for (size_t j = 0; + j + 1 < n->n_descsz / 8; j += 2) { + if (words[j] == AT_PAGESZ) + aux = words[j + 1] == + ps; + else if (words[j] == + AT_SYSINFO_EHDR) + vdso = words[j + 1]; + } + } + if (n->n_type == NT_FILE && n->n_descsz >= 16 && + words[0] <= (n->n_descsz / 8 - 2) / 3 && + words[1] == ps) { + for (size_t j = 0; j < words[0]; j++) { + unsigned long addr = + words[2 + 3 * j]; + unsigned long offset = + words[4 + 3 * j]; + + if (addr == FILEMAP && + offset == (headers ? 0UL : + 1UL)) + file |= 1; + if (addr == FILEMAP + 2 * ps && + offset == 2) + file |= 2; + if (addr == FILEMAP + 8 * ps && + offset == 17) + file |= 4; + } + } + at = desc + ((n->n_descsz + 3UL) & ~3UL); + } + } + } + ksft_test_result(geometry, "%luK core PT_LOAD granule/alignment%s\n", + ps / 1024, headers ? " headers-only" : ""); + ksft_test_result(aux && file == 7, + "%luK core AUXV and NT_FILE offsets%s\n", ps / 1024, + headers ? " headers-only" : ""); + if (headers) { + bool skipped; + + p = find_load(data, elf, FILEMAP + 2 * ps); + skipped = p && !p->p_filesz; + p = find_load(data, elf, FILEMAP + 8 * ps); + ksft_test_result( + skipped && p && !p->p_filesz, + "%luK header filter excludes nonzero quarter/native file offsets\n", + ps / 1024); + p = find_load(data, elf, FILEMAP + 12 * ps); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK header filter recognizes non-executable ELF file\n", + ps / 1024); + p = find_load(data, elf, FILEMAP); + ksft_test_result( + p && p->p_filesz == ps && + p->p_offset + ps <= (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, SELFMAG), + "%luK executable file header limited to one user page\n", + ps / 1024); + } else { + bool anon = true, files = true; + + p = find_load(data, elf, vdso); + ksft_test_result(p && p->p_filesz >= SELFMAG && + p->p_offset + p->p_filesz <= + (uint64_t)st.st_size && + !memcmp(data + p->p_offset, ELFMAG, + SELFMAG), + "%luK core vDSO ELF bytes\n", ps / 1024); + ksft_test_result(bytes_at(data, elf, st.st_size, SWAPPED, + SWAPPED_SIZE, 0xc4), + "%luK core swapped anonymous bytes\n", + ps / 1024); + + for (unsigned int i = 0; i < 4; i++) + anon &= bytes_at(data, elf, st.st_size, ANON + i * ps, + ps, 0x31 + i); + for (unsigned int i = 1; i < 4; i++) + files &= bytes_at(data, elf, st.st_size, + FILEMAP + (i == 1 ? 0 : i * ps), ps, + 0x81 + i); + ksft_test_result( + anon, "%luK core anonymous and PROT_NONE/READ bytes\n", + ps / 1024); + files &= bytes_at(data, elf, st.st_size, FILEMAP + 8 * ps, ps, + 0x81 + 17); + ksft_test_result(files, "%luK core unfaulted file quarters\n", + ps / 1024); + ksft_test_result( + bytes_at(data, elf, st.st_size, SPARSE, ps, 0x65) && + bytes_at(data, elf, st.st_size, SPARSE + ps, + SPARSE_SIZE - ps, 0), + "%luK core sparse and zero-page bytes\n", ps / 1024); + ksft_test_result( + st.st_blocks * 512 < st.st_size / 2, + "%luK core keeps untouched anonymous range sparse\n", + ps / 1024); + p = find_load(data, elf, EXCLUDED); + ksft_test_result(p && !p->p_filesz, + "%luK core DONTDUMP excludes data\n", + ps / 1024); + } + munmap(data, st.st_size); + close(fd); + unlink(path); +} + +int main(int argc, char **argv) +{ + unsigned long native_size = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native_size, 4096, 16384 }; + unsigned int modes = native_size == 65536 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--crash")) + crash(false, 0); + if (argc == 2 && !strcmp(argv[1], "--headers")) + crash(true, 0); + if (argc == 2 && !strcmp(argv[1], "--limit")) + crash(false, 1); + if (argc == 2 && !strcmp(argv[1], "--below-limit")) + crash(false, -1); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Use only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + die("proc"); + if (mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + die("VM swap setup"); + write_text("/proc/sys/kernel/core_pattern", "/core.%p"); + ksft_print_header(); + ksft_set_plan(16 * modes); + for (unsigned int mode = 0; mode < modes; mode++) { + unsigned long ps = sizes[mode]; + for (unsigned int headers = 0; headers < 4; headers++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + ps == native_size ? 0UL : ps, 0UL, + 0UL, 0UL)) + die("exec size"); + execl("/init", "/init", + headers == 3 ? "--below-limit" : + headers == 2 ? "--limit" : + headers ? "--headers" : + "--crash", + NULL); + die("exec"); + } + if (pid < 0 || waitpid(pid, &status, 0) != pid || + !WIFSIGNALED(status) || + WTERMSIG(status) != SIGABRT || + (headers < 3 && !WCOREDUMP(status))) + ksft_exit_fail_msg( + "child/core failure status=%#x\n", + status); + if (headers >= 2) { + char path[64]; + struct stat st; + bool exists, no_data = false; + + snprintf(path, sizeof(path), "/core.%d", pid); + exists = !stat(path, &st); + if (exists && headers == 2) { + Elf64_Ehdr elf; + Elf64_Phdr ph; + uint64_t data_start = UINT64_MAX; + int fd = open(path, O_RDONLY); + + if (fd < 0 || + pread(fd, &elf, sizeof(elf), 0) != + sizeof(elf)) + die("limited core header"); + for (unsigned int i = 0; + i < elf.e_phnum; i++) { + if (pread(fd, &ph, sizeof(ph), + elf.e_phoff + + i * sizeof(ph)) != + sizeof(ph)) + die("limited core phdr"); + if (ph.p_type == PT_LOAD && + ph.p_offset < data_start) + data_start = + ph.p_offset; + } + close(fd); + /* Sparse note padding is not charged to RLIMIT_CORE. */ + no_data = data_start != UINT64_MAX && + st.st_size <= + (off_t)data_start; + printf("# limited core: bytes=%lld first-data=%llu limit=%lu\n", + (long long)st.st_size, + (unsigned long long)data_start, + ps); + } + ksft_test_result( + headers == 2 ? + exists && st.st_size > 0 && + no_data : + !exists && errno == ENOENT, + "%luK core RLIMIT_CORE %s one user page\n", + ps / 1024, + headers == 2 ? "at" : "below"); + unlink(path); + } else { + inspect(pid, ps, headers); + } + } + } + ksft_print_cnts(); + printf("%s - core fixture complete\n", + ksft_get_fail_cnt() ? "not ok" : "ok"); + reboot(RB_POWER_OFF); + return ksft_get_fail_cnt() ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c new file mode 100644 index 00000000000000..1e983f91f74c72 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule-init.c @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has4k && strstr(cmdline, "arm64.user4k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has4k ? 4096UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has4k ? 4096UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has4k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has4k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec retains selected or native-fallback granule"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c new file mode 100644 index 00000000000000..ecf682a22f732f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cpu-granule16-init.c @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static int pin_cpu(pid_t pid, unsigned int cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + return sched_setaffinity(pid, sizeof(set), &set); +} +static int online_cpu1(void) +{ + char c = '?'; + int fd = open("/sys/devices/system/cpu/cpu1/online", O_RDONLY); + if (fd >= 0) { + if (read(fd, &c, 1) != 1) + c = '?'; + close(fd); + } + return c == '0' ? 0 : c == '1' ? 1 : -1; +} +static int worker(int ready, int command, unsigned long expected) +{ + unsigned char *p; + unsigned long ps = getauxval(AT_PAGESZ); + char cpu; + int ok = getppid() == 1 && ps == expected; + + p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 2; + for (unsigned long i = 0; i < 4 * ps; i++) + p[i] = i ^ (i >> 12); + if (write(ready, "r", 1) != 1 || read(command, &cpu, 1) != 1) + return 3; + ok &= sched_getcpu() == cpu - '0'; + for (unsigned int round = 0; round < 128; round++) { + for (unsigned long i = 0; i < 4 * ps; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + ok = 0; + sched_yield(); + } + munmap(p, 4 * ps); + return ok ? 0 : 4; +} +static int compat_sample(unsigned long expected) +{ + uint64_t data[4] = { 0 }; + int pipefd[2], status; + pid_t child; + ssize_t size; + + if (pipe(pipefd)) + return 0; + child = fork(); + if (!child) { + char fd[32]; + close(pipefd[0]); + snprintf(fd, sizeof(fd), "%d", pipefd[1]); + execl("/compat-aslr", "/compat-aslr", "--sample", fd, NULL); + _exit(2); + } + close(pipefd[1]); + size = read(pipefd[0], data, sizeof(data)); + close(pipefd[0]); + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + size == sizeof(data) && data[0] == expected; +} +int main(int argc, char **argv) +{ + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, + "Dedicated two-CPU disposable VM with maxcpus=1 only\n"); + return 2; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) { + perror("mount"); + failed++; + goto done; + } + unsigned long native = getauxval(AT_PAGESZ); + int has4k = !prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL); + int has16k = native > 16384 && + !prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL); + result(!prctl(SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL), + "native exec default restored"); + char cmdline[4096] = { 0 }; + int fd = open("/proc/cmdline", O_RDONLY); + if (fd < 0 || read(fd, cmdline, sizeof(cmdline) - 1) < 0) { + failed++; + goto done; + } + close(fd); + int reject = has16k && strstr(cmdline, "arm64.user16k_deny_late_cpu=1"); + result(online_cpu1() == 0, "CPU1 starts offline"); + if (online_cpu1() != 0) + goto done; + int ready[2], command[2]; + if (pipe(ready) || pipe(command)) { + failed++; + goto done; + } + pid_t child = fork(); + if (!child) { + char ready_fd[32], command_fd[32], ps[32]; + close(ready[0]); + close(command[1]); + snprintf(ready_fd, sizeof(ready_fd), "%d", ready[1]); + snprintf(command_fd, sizeof(command_fd), "%d", command[0]); + snprintf(ps, sizeof(ps), "%lu", has16k ? 16384UL : native); + if (prctl(SET_EXEC_PAGE_SIZE, has16k ? 16384UL : 0UL, 0UL, 0UL, + 0UL)) + _exit(5); + execl("/init", "/init", "--worker", ready_fd, command_fd, ps, + NULL); + _exit(6); + } + close(ready[1]); + close(command[0]); + if (child < 0) { + failed++; + goto done; + } + char token; + result(read(ready[0], &token, 1) == 1 && token == 'r', + "worker has live mappings before hotplug"); + close(ready[0]); + fd = open("/sys/devices/system/cpu/cpu1/online", O_WRONLY); + errno = 0; + ssize_t written = fd < 0 ? -1 : write(fd, "1", 1); + int saved_errno = errno; + if (fd >= 0) + close(fd); + int online = online_cpu1(); + printf("# has16k=%d expected-reject=%d write=%zd errno=%d online=%d\n", + has16k, reject, written, saved_errno, online); + result(reject ? written < 0 && online == 0 : + written == 1 && online == 1, + "late CPU admission matches advertised user granule"); + unsigned int cpu = online == 1 ? 1 : 0; + result(!pin_cpu(child, cpu), "worker affinity targets admitted CPU"); + token = '0' + cpu; + result(write(command[1], &token, 1) == 1, + "worker resumes after CPU admission"); + close(command[1]); + int status; + result(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status), + "live mapping bytes survive and worker runs on expected CPU"); + result(!pin_cpu(0, cpu), "launcher affinity targets admitted CPU"); + if (has16k) + result(!prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL), + "16K request staged before compat exec"); + result(compat_sample(has4k ? 4096 : native), + "AArch32 exec selects4K or native despite pending16K"); +done: + printf("CPU GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/dio-fixture-init.c b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c new file mode 100644 index 00000000000000..36ebec4350c762 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/dio-fixture-init.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_file; +static unsigned char pattern(size_t i) +{ + return (i * 17 + (i >> 12) * 29 + 71) & 255; +} +static void fill(unsigned char *p, size_t len) +{ + for (size_t i = 0; i < len; i++) + p[i] = pattern(i); +} +static bool check(const unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != pattern(i + off)) { + ksft_print_msg("mismatch at %zu: %u vs %u\n", i + off, + p[i], pattern(i + off)); + return false; + } + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s (errno=%d)\n", ps / 1024, name, errno); +} +static bool aio_transfer(int fd, void *p, size_t len, off_t off, bool write) +{ + aio_context_t ctx = 0; + struct iocb cb = { .aio_lio_opcode = write ? IOCB_CMD_PWRITE : + IOCB_CMD_PREAD, + .aio_fildes = fd, + .aio_buf = (uintptr_t)p, + .aio_nbytes = len, + .aio_offset = off }; + struct iocb *ptr = &cb; + struct io_event ev; + struct timespec timeout = { .tv_sec = 10 }; + bool ok; + + if (syscall(__NR_io_setup, 8, &ctx)) + return false; + ok = syscall(__NR_io_submit, ctx, 1, &ptr) == 1; + if (ok) + ok = syscall(__NR_io_getevents, ctx, 1, 1, &ev, &timeout) == + 1 && + ev.res == (long)len && !ev.res2; + ok &= !syscall(__NR_io_destroy, ctx); + return ok; +} +static int exercise(void) +{ + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vec[4]; + int fd = open(use_file ? "/fs/test" : "/dev/user4k-test", + O_RDWR | O_DIRECT | (use_file ? O_CREAT | O_TRUNC : 0), + 0600), + memfd; + bool ok; + ssize_t ret; + + ksft_print_header(); + ksft_set_plan(10); + ksft_print_msg("backing=%s\n", use_file ? "ext4" : "raw block"); + if ((ps != 4096 && ps != 16384) || p == MAP_FAILED || q == MAP_FAILED || + fd < 0) + ksft_exit_fail_msg("setup failed errno=%d\n", errno); + if (use_file && ftruncate(fd, 32 * 1024 * 1024)) + ksft_exit_fail_msg("ftruncate failed\n"); + fill(p, len); + memset(q, 0, len); + ok = pwrite(fd, p + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + pread(fd, q + ps, 3 * ps, 0) == (ssize_t)(3 * ps) && + check(q + ps, 3 * ps, ps); + report(ok, ps, "O_DIRECT exact user offsets and bytes"); + + vec[0] = (struct iovec){ p + 512, ps - 512 }; + vec[1] = (struct iovec){ NULL, 0 }; + vec[2] = (struct iovec){ p + 5 * ps, ps + 512 }; + vec[3] = (struct iovec){ p + 11 * ps + 512, 512 }; + ok = pwritev(fd, vec, 4, 65536) == (ssize_t)(2 * ps + 512); + memset(q, 0, 3 * ps); + ok &= pread(fd, q, 2 * ps + 512, 65536) == (ssize_t)(2 * ps + 512); + ok &= check(q, ps - 512, 512) && + check(q + ps - 512, ps + 512, 5 * ps) && + check(q + 2 * ps, 512, 11 * ps + 512); + report(ok, ps, "fragmented vectors and empty segment"); + + ok = pwrite(fd, p, len, 4 * 1024 * 1024) == (ssize_t)len; + memset(q, 0, len); + ok &= pread(fd, q, len, 4 * 1024 * 1024) == (ssize_t)len && + check(q, len, 0); + report(ok, ps, "multi-bio two-megabyte transfer"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + ret = pwrite(fd, p, 2 * ps, 8 * 1024 * 1024); + ok &= use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + if (use_file) + ok &= pwrite(fd, p, ps, 8 * 1024 * 1024) == (ssize_t)ps; + memset(q, 0, ps); + ok &= pread(fd, q, ps, 8 * 1024 * 1024) == (ssize_t)ps && + check(q, ps, 0); + report(ok, ps, "source fault and subsequent I/O recovery"); + + ok = !mprotect(q + ps, ps, PROT_NONE); + ret = pread(fd, q, 2 * ps, 4 * 1024 * 1024); + ok &= (use_file ? ret == -1 && errno == EFAULT : ret == (ssize_t)ps) && + check(q, ps, 0); + ok &= !mprotect(q + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "partial destination fault"); + + ok = !mprotect(p + ps, ps, PROT_NONE); + /* Only 256 accessible bytes: none can form a 512-byte device block. */ + ret = pwrite(fd, p + ps - 256, 512, 10 * 1024 * 1024); + ok &= ret == -1 && errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + report(ok, ps, "alignment trimming releases all short-fragment pins"); + + memfd = memfd_create("dio-shared", 0); + ok = memfd >= 0 && !ftruncate(memfd, 4 * ps); + unsigned char *shared = ok ? mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0) : + MAP_FAILED; + ok &= shared != MAP_FAILED; + if (ok) { + memset(shared, 0xa7, 4 * ps); + ok = pread(fd, shared + ps, 2 * ps, 4 * 1024 * 1024) == + (ssize_t)(2 * ps) && + !msync(shared, 4 * ps, MS_SYNC) && + pread(memfd, q, 4 * ps, 0) == (ssize_t)(4 * ps); + ok &= check(q + ps, 2 * ps, 0); + for (size_t i = 0; i < ps; i++) + ok &= q[i] == 0xa7 && q[3 * ps + i] == 0xa7; + munmap(shared, 4 * ps); + } + if (memfd >= 0) + close(memfd); + report(ok, ps, "shared-file destination dirtying and adjacent guards"); + + ok = aio_transfer(fd, p + ps, 7 * ps, 12 * 1024 * 1024, true); + memset(q, 0, 8 * ps); + ok &= aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, false) && + check(q + ps, 7 * ps, ps); + report(ok, ps, "asynchronous AIO read/write completion"); + + memset(q, 0x96, 8 * ps); + pid_t child = fork(); + int status = 0; + if (!child) { + _exit(aio_transfer(fd, q + ps, 7 * ps, 12 * 1024 * 1024, + false) && + check(q + ps, 7 * ps, ps) ? + 0 : + 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < 8 * ps; i++) + ok &= q[i] == 0x96; + report(ok, ps, "writable-pin COW after fork"); + + ok = true; + for (unsigned i = 0; i < 128 && ok; i++) { + ok = !madvise(q + ps, ps, MADV_DONTNEED) && + pread(fd, q + ps, ps, 4 * 1024 * 1024) == (ssize_t)ps && + check(q + ps, ps, 0); + ok &= !mprotect(q + 2 * ps, ps, PROT_NONE); + ok &= pread(fd, q + 2 * ps, ps, 4 * 1024 * 1024) == -1 && + errno == EFAULT; + ok &= !mprotect(q + 2 * ps, ps, PROT_READ | PROT_WRITE); + } + report(ok, ps, "128 reuse and failed-pin cleanup cycles"); + close(fd); + munmap(p, len); + munmap(q, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--file")) && + getppid() == 1) { + use_file = !strcmp(argv[1], "--file"); + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + for (unsigned disk = 0; disk < 2; disk++) { + char path[80], serial[64]; + unsigned maj, min; + snprintf(path, sizeof(path), "/sys/class/block/vd%c/serial", + 'a' + disk); + FILE *dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%63s", serial) != 1) + return 4; + fclose(dev); + snprintf(path, sizeof(path), "/sys/class/block/vd%c/dev", + 'a' + disk); + dev = fopen(path, "r"); + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + const char *node = + !strcmp(serial, "au4k-raw") ? "/dev/user4k-test" : + !strcmp(serial, "au4k-ext4") ? "/dev/user4k-fs" : + NULL; + if (!node || mknod(node, S_IFBLK | 0600, makedev(maj, min))) + return 5; + } + if (mount("/dev/user4k-fs", "/fs", "ext4", 0, NULL)) + return 7; + for (unsigned mode = 0; mode < 4; mode++) { + unsigned small = mode & 1; + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", mode >= 2 ? "--file" : "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= !umount("/fs"); + printf("%s - direct I/O fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-fixture-init.c b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c new file mode 100644 index 00000000000000..8a390d5df3ddcd --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-fixture-init.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* PID-1 runner for the experimental one-shot exec ABI in a disposable VM. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include + +static int exec_size(unsigned long size) +{ + return prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL); +} + +static int pending_size(void) +{ + return prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); +} + +static int run(const char *path, const char *arg, unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + if (size && exec_size(size)) { + perror("set exec page size"); + _exit(125); + } + execl(path, path, arg, NULL); + perror(path); + _exit(126); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# child %s %s: wait=%d status=%#x\n", path, arg ? arg : "", got, + status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +static int argument_exec(unsigned long size) +{ + pid_t pid = fork(), got; + int status = 0; + + if (!pid) { + char *payload = malloc(140000), expected[16]; + unsigned int i; + + if (!payload || exec_size(size)) + _exit(120); + for (i = 0; i < 139999; i++) + payload[i] = 'a' + i % 23; + payload[139999] = 0; + if (size == 4096) { + execl("/init", "/init", "--too-long", payload, NULL); + if (errno != E2BIG || pending_size() != 4096) + _exit(121); + } + payload[120000] = 0; + if (setenv("ARG_PAYLOAD", payload, 1)) + _exit(122); + snprintf(expected, sizeof(expected), "%lu", + size ? size : 16384UL); + execl("/init", "/init", "--args-check", expected, payload, + NULL); + perror("argument exec"); + _exit(123); + } + if (pid < 0) + return -1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + printf("# large arguments from %lu to %lu: status=%#x\n", + getauxval(AT_PAGESZ), size ? size : 16384UL, status); + return got == pid && WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool ok; + int failures = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--args-check")) { + const char *env = getenv("ARG_PAYLOAD"); + unsigned int i; + + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10) || + pending_size() != 0 || strlen(argv[3]) != 120000 || !env || + strlen(env) != 120000) + return 1; + for (i = 0; i < 120000; i++) + if (argv[3][i] != 'a' + i % 23 || env[i] != argv[3][i]) + return 2; + return 0; + } + if (argc == 2 && !strcmp(argv[1], "--args-from-4k")) { + if (getauxval(AT_PAGESZ) != 4096) + return 1; + return argument_exec(4096) || argument_exec(0); + } + if (argc == 2 && !strcmp(argv[1], "--exec-cycle")) { + if (getauxval(AT_PAGESZ) != 4096 || pending_size() != 0) + return 1; + /* A successful exec consumes the request; ordinary exec is native. */ + execl("/init", "/init", "--native-after-4k", NULL); + return 2; + } + if (argc == 2 && !strcmp(argv[1], "--native-after-4k")) + return getauxval(AT_PAGESZ) != 16384 || pending_size() != 0; + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as /init in a disposable VM.\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) && errno != EBUSY) + return 3; + printf("# exec fixture: native AT_PAGESZ=%lu\n", getauxval(AT_PAGESZ)); + failures += run("/page-contract", NULL, 0) != 0; + ok = pending_size() == 0; + errno = 0; + ok &= exec_size(8192) == -1 && errno == EINVAL; + ok &= prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 1UL, 0UL, 0UL) == -1; + ok &= exec_size(4096) == 0 && pending_size() == 4096; + execl("/does-not-exist", "/does-not-exist", NULL); + ok &= errno == ENOENT && pending_size() == 4096; + ok &= getauxval(AT_PAGESZ) == 16384; + printf("%s - request validation and failed exec preserve live geometry\n", + ok ? "ok" : "not ok"); + failures += !ok; + /* fork inherits the pending request; successful child exec consumes it. */ + failures += run("/page-contract-4k", "--expect-4k", 0) != 0; + ok = pending_size() == 4096 && exec_size(0) == 0 && pending_size() == 0; + printf("%s - child exec leaves parent request intact; reset works\n", + ok ? "ok" : "not ok"); + failures += !ok; + failures += run("/page-contract-dynamic-4k", "--expect-4k", 4096) != 0; + failures += run("/init", "--exec-cycle", 4096) != 0; + failures += run("/proc-memory", NULL, 0) != 0; + failures += run("/proc-memory", "--expect-4k", 4096) != 0; + failures += run("/proc-pages", NULL, 0) != 0; + failures += run("/proc-pages", "--expect-4k", 4096) != 0; + failures += run("/advice", NULL, 0) != 0; + failures += run("/advice", "--expect-4k", 4096) != 0; + failures += run("/memory-interfaces", NULL, 0) != 0; + failures += run("/memory-interfaces", "--expect-4k", 4096) != 0; + failures += run("/mixed-process", NULL, 0) != 0; + failures += run("/mixed-process", NULL, 4096) != 0; + failures += argument_exec(4096) != 0; + failures += run("/init", "--args-from-4k", 4096) != 0; + failures += run("/page-contract", NULL, 0) != 0; + printf("%s - exec fixture complete: failures=%d\n", + failures ? "not ok" : "ok", failures); + reboot(RB_POWER_OFF); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/exec-policy-init.c b/tools/testing/selftests/arm64/user4k/exec-policy-init.c new file mode 100644 index 00000000000000..02fd3f95522374 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/exec-policy-init.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable guest PID1: per-thread one-shot and inherited exec preferences. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", \ + __LINE__, #x, errno); \ + exit(1); \ + } \ + } while (0) + +static int policy(int op, unsigned long value) +{ + return prctl(op, value, 0UL, 0UL, 0UL); +} + +static void state(unsigned long pages, unsigned long preferred, int pending) +{ + REQUIRE(getauxval(AT_PAGESZ) == pages); + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == pending); +} + +static void chain(int stage, unsigned long native, unsigned long preferred) +{ + char s[32], n[32], p[32]; + + snprintf(s, sizeof(s), "%d", stage); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + execl("/init", "/init", "--chain", s, n, p, NULL); + REQUIRE(0); +} + +static void *thread_policy(void *arg) +{ + unsigned long preferred = *(unsigned long *)arg; + + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 0) == (int)preferred); + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(getauxval(AT_PAGESZ), 0, 0); + return NULL; +} + +static void *thread_exec(void *arg) +{ + unsigned long *sizes = arg; + + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, sizes[1])); + chain(0, sizes[0], sizes[1]); + return NULL; +} + +static int wait_child(pid_t pid) +{ + int status; + + REQUIRE(pid > 0); + REQUIRE(waitpid(pid, &status, 0) == pid); + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static int run_preference(unsigned long native, unsigned long preferred) +{ + pid_t pid = fork(); + + if (!pid) { + pthread_t thread; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, preferred)); + REQUIRE(!pthread_create(&thread, NULL, thread_policy, + &preferred)); + REQUIRE(!pthread_join(thread, NULL)); + state(native, preferred, preferred); + /* Rejected values and reserved arguments must preserve both policies. */ + REQUIRE(policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 8192) == -1 && + errno == EINVAL); + int ops[] = { PR_AURORA_SET_DEFAULT_PAGE_SIZE, + PR_AURORA_GET_DEFAULT_PAGE_SIZE, + PR_AURORA_SET_EXEC_PAGE_SIZE, + PR_AURORA_GET_EXEC_PAGE_SIZE }; + for (unsigned int i = 0; i < sizeof(ops) / sizeof(ops[0]); + i++) { + REQUIRE(prctl(ops[i], 0UL, 1UL, 0UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 1UL, 0UL) == -1 && + errno == EINVAL); + REQUIRE(prctl(ops[i], 0UL, 0UL, 0UL, 1UL) == -1 && + errno == EINVAL); + } + REQUIRE(policy(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + REQUIRE(policy(PR_AURORA_GET_EXEC_PAGE_SIZE, 1) == -1 && + errno == EINVAL); + execl("/does-not-exist", "/does-not-exist", NULL); + REQUIRE(errno == ENOENT); + state(native, preferred, preferred); + chain(0, native, preferred); + } + return wait_child(pid); +} + +static int compat(unsigned long native, unsigned long preferred, int no4k) +{ + pid_t pid = fork(); + + if (!pid) { + char n[32], p[32], e[32]; + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, preferred)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, native)); + snprintf(n, sizeof(n), "%lu", native); + snprintf(p, sizeof(p), "%lu", preferred); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-policy", "/compat-policy", n, p, e, NULL); + REQUIRE(0); + } + return wait_child(pid); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--chain")) { + int stage = atoi(argv[2]); + unsigned long n = strtoul(argv[3], NULL, 10); + unsigned long p = strtoul(argv[4], NULL, 10); + + state(stage == 2 || stage == 5 ? n : p, stage < 4 ? p : 0, 0); + if (stage == 5) + return 0; + if (stage == 1) { + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 0)); + state(p, p, n); + } + if (stage == 3) { + REQUIRE(!policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 0)); + REQUIRE(!policy(PR_AURORA_SET_EXEC_PAGE_SIZE, p)); + } + chain(stage + 1, n, p); + } + if (argc == 4 && !strcmp(argv[1], "--compat-return")) { + unsigned long n = strtoul(argv[2], NULL, 10); + unsigned long p = strtoul(argv[3], NULL, 10); + state(p ? p : n, p, 0); + return 0; + } + REQUIRE(getpid() == 1); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + state(native, 0, 0); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long preferences[] = { 4096, 16384 }; + for (unsigned int i = 0; i < 2; i++) { + unsigned long p = preferences[i]; + if (p >= native || (no4k && p == 4096)) + continue; + int ok = run_preference(native, p); + printf("%s - exec policy native=%lu alternative=%lu\n", + ok ? "ok" : "not ok", native, p); + failed |= !ok; + pid_t pid = fork(); + if (!pid) { + pthread_t thread; + unsigned long sizes[] = { native, p }; + REQUIRE(!pthread_create(&thread, NULL, thread_exec, + sizes)); + for (;;) + pause(); + } + ok = wait_child(pid); + printf("%s - non-leader exec alternative=%lu\n", + ok ? "ok" : "not ok", p); + failed |= !ok; + ok = compat(native, p, no4k); + printf("%s - compat policy alternative=%lu no4k=%d\n", + ok ? "ok" : "not ok", p, no4k); + failed |= !ok; + } + if (no4k) { + int ok = policy(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= policy(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096) == -1 && + errno == EOPNOTSUPP; + ok &= compat(native, 0, 1); + printf("%s - no4k preserves native compat restrictions\n", + ok ? "ok" : "not ok"); + failed |= !ok; + } + state(native, 0, 0); + printf("EXEC POLICY %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + return failed; +} diff --git a/tools/testing/selftests/arm64/user4k/fex-contract-init.c b/tools/testing/selftests/arm64/user4k/fex-contract-init.c new file mode 100644 index 00000000000000..3273af30eea7f7 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-contract-init.c @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +int main(void) +{ + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + pid_t child = fork(); + if (!child) { + setenv("HOME", "/root", 1); + setenv("PATH", "/usr/bin:/bin", 1); + setenv("XDG_RUNTIME_DIR", "/run", 1); + setenv("FEX_ROOTFS", "/", 1); + setenv("LD_PRELOAD", "/usr/lib/libfex-abi.so", 1); + if (prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, 4096UL, 0UL, 0UL, + 0UL) || + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/FEXInterpreter", "/usr/bin/FEXInterpreter", + "/x86-contract", NULL); + perror("FEX exec"); + _exit(4); + } + int status = 0; + int ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - FEX 4K native-process / x86 workload status=%#x\n", + ok ? "ok" : "not ok", status); + failed |= !ok; +done: + printf("GRANULE FEX RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-native-abi.c b/tools/testing/selftests/arm64/user4k/fex-native-abi.c new file mode 100644 index 00000000000000..86ccb6e389d9de --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-native-abi.c @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +__attribute__((constructor)) static void check_abi(void) +{ + char path[1024]; + ssize_t n = readlink("/proc/self/exe", path, sizeof(path) - 1); + if (n < 0) + return; + path[n] = 0; + if (!strstr(path, "FEXInterpreter")) + return; + unsigned long ps = getauxval(AT_PAGESZ); + dprintf(2, "FEX NATIVE HOST PROCESS PAGE SIZE %lu\n", ps); + if (ps != 4096) + _exit(90); +} diff --git a/tools/testing/selftests/arm64/user4k/fex-x86-contract.c b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c new file mode 100644 index 00000000000000..047a3c9d28ddb6 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/fex-x86-contract.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +typedef unsigned long size_t; +static long sc(long n, long a, long b, long c, long d, long e, long f) +{ + register long r10 __asm__("r10") = d, r8 __asm__("r8") = e, + r9 __asm__("r9") = f; + long r; + __asm__ volatile("syscall" + : "=a"(r) + : "a"(n), "D"(a), "S"(b), "d"(c), "r"(r10), "r"(r8), + "r"(r9) + : "rcx", "r11", "memory"); + return r; +} +static void say(const char *s, size_t n) +{ + sc(1, 1, (long)s, n, 0, 0, 0); +} +#define SAY(s) say(s, sizeof(s) - 1) +static void quit(int n) +{ + sc(231, n, 0, 0, 0, 0, 0); + for (;;) + ; +} +static int check(unsigned char *p, size_t size) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != (unsigned char)(i ^ (i >> 12))) + return 0; + return 1; +} +long run(unsigned long *sp) +{ + unsigned long argc = *sp++, pagesz = 0; + sp += argc + 1; + unsigned long *env = sp; + while (*sp) + sp++; + sp++; + while (*sp) { + if (sp[0] == 6) + pagesz = sp[1]; + sp += 2; + } + if (pagesz != 4096) + return 10; + SAY("ok - FEX x86 guest 4K auxv\n"); + unsigned char *p = (void *)sc(9, 0, 12288, 3, 0x22, -1, 0); + if ((long)p < 0) + return 11; + for (size_t i = 0; i < 12288; i++) + p[i] = (unsigned char)(i ^ (i >> 12)); + if (sc(10, (long)p + 4096, 4096, 0, 0, 0, 0)) + return 12; + long child = sc(57, 0, 0, 0, 0, 0, 0); + int status = 0; + if (!child) { + *(volatile unsigned char *)(p + 4096) = 1; + quit(13); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + (status & 127) != 11) + return 14; + if (sc(10, (long)p + 4096, 4096, 3, 0, 0, 0) || !check(p, 12288)) + return 15; + SAY("ok - FEX x86 independent 4K protection fault\n"); + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + p[4096] = 0x71; + quit(0); + } + if (child < 0 || sc(61, child, (long)&status, 0, 0, 0, 0) != child || + status || !check(p, 12288)) + return 16; + SAY("ok - FEX x86 fork COW\n"); + p = (void *)sc(25, (long)p, 12288, 16384, 1, 0, 0); + if ((long)p < 0 || !check(p, 12288)) + return 17; + for (size_t i = 12288; i < 16384; i++) + if (p[i]) + return 18; + SAY("ok - FEX x86 mremap preserve and zero extension\n"); + long fd = sc(319, (long)"fex-shared", 0, 0, 0, 0, 0); + if (fd < 0 || sc(77, fd, 12288, 0, 0, 0, 0)) + return 19; + unsigned char *a = (void *)sc(9, 0, 4096, 3, 1, fd, 4096), + *b = (void *)sc(9, 0, 4096, 3, 1, fd, 4096); + if ((long)a < 0 || (long)b < 0) + return 20; + a[37] = 0x65; + if (b[37] != 0x65) + return 21; + SAY("ok - FEX x86 memfd mapping at 4K file offset\n"); + unsigned char *code = (void *)sc(9, 0, 4096, 3, 0x22, -1, 0); + if ((long)code < 0) + return 22; + code[0] = 0xb8; + code[1] = 42; + code[2] = 0; + code[3] = 0; + code[4] = 0; + code[5] = 0xc3; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 42) + return 23; + if (sc(10, (long)code, 4096, 3, 0, 0, 0)) + return 24; + code[1] = 99; + if (sc(10, (long)code, 4096, 5, 0, 0, 0) || + ((int (*)(void))code)() != 99) + return 25; + SAY("ok - FEX x86 JIT permission transition and code replacement\n"); + unsigned char random[32]; + if (sc(318, (long)random, 32, 0, 0, 0, 0) != 32) + return 26; + if (argc == 1) { + child = sc(57, 0, 0, 0, 0, 0, 0); + if (!child) { + char *args[] = { "/x86-contract", "--exec-child", 0 }; + sc(59, (long)args[0], (long)args, (long)env, 0, 0, 0); + quit(28); + } + if (child < 0 || + sc(61, child, (long)&status, 0, 0, 0, 0) != child || status) + return 27; + SAY("ok - FEX x86 fork and re-exec with 4K mappings\n"); + } + SAY("FEX X86 4K CONTRACT PASS\n"); + return 0; +} +__asm__(".global _start\n_start:\nmov %rsp,%rdi\nand $-16,%rsp\ncall run\nmov %rax,%rdi\nmov $231,%rax\nsyscall\n"); diff --git a/tools/testing/selftests/arm64/user4k/gem-fixture-init.c b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c new file mode 100644 index 00000000000000..aadf343dc7e8d4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/gem-fixture-init.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned char value(size_t off) +{ + return 0x31 + (off / 4096) % 128; +} +static bool check(unsigned char *p, size_t len, size_t off) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value(i + off)) { + ksft_print_msg("offset=%zu got=%u expected=%u\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), + len = native < 8192 ? 65536 : 8 * native; + int fd = open("/dev/vgem", O_RDWR); + struct drm_mode_create_dumb create = { .width = 4096, + .height = len / (4096 * 4), + .bpp = 32 }; + struct drm_mode_map_dumb map = {}; + struct drm_prime_handle prime = { .flags = DRM_RDWR }; + unsigned char *p = MAP_FAILED, *q = MAP_FAILED, *alias = MAP_FAILED; + bool ok = fd >= 0 && !ioctl(fd, DRM_IOCTL_MODE_CREATE_DUMB, &create); + ksft_print_header(); + ksft_set_plan(12); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + map.handle = create.handle; + ok &= !ioctl(fd, DRM_IOCTL_MODE_MAP_DUMB, &map) && create.size >= len; + if (ok) + p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok &= p != MAP_FAILED; + ksft_test_result(ok, "full GEM shmem object mapping\n"); + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + bool separate = q != MAP_FAILED; + if (separate) { + memset(q, 0x39, ps); + memset(q + ps, 0xa2, ps); + for (size_t i = 0; i < ps; i++) + separate &= q[i] == 0x39 && q[ps + i] == 0xa2; + munmap(q, 2 * ps); + q = MAP_FAILED; + } + ksft_test_result(separate, + "adjacent user leaves map distinct backing bytes\n"); + if (ok) { + for (size_t i = 0; i < len; i++) + p[i] = value(i); + ok = check(p, len, 0); + } + ksft_test_result(ok, "full object has distinct quarter data\n"); + if (ok) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !mprotect(q + ps, ps, PROT_READ); + if (ok) + ok = check(q, len, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } + ksft_test_result( + ok, + "cold faults after mprotect VMA splitting preserve object offsets\n"); + if (p != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset); + ok = q != MAP_FAILED && !munmap(q + ps, ps); + if (ok) + ok = check(q + 2 * ps, len - 2 * ps, 2 * ps) && + check(q, ps, 0); + if (q != MAP_FAILED) + munmap(q, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold faults after partial unmap preserve object offsets\n"); + prime.handle = create.handle; + ok = !ioctl(fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &prime); + if (ok) { + alias = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + ok = alias != MAP_FAILED && check(alias, len, 0); + } + ksft_test_result(ok, "PRIME exported full mapping shares GEM data\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, prime.fd, + 3 * ps); + ok = q != MAP_FAILED; + if (ok) + ok = check(q, ps, 3 * ps); + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, "PRIME partial mapping uses requested buffer offset\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, 0); + void *target = mmap(NULL, len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ok = q != MAP_FAILED && target != MAP_FAILED; + if (ok) { + void *moved = mremap(q, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + target); + ok = moved == target; + if (ok) { + q = target; + ok = check(q, len, 0); + } + } + if (q != MAP_FAILED) + munmap(q, len); + if (target != MAP_FAILED && target != q) + munmap(target, len); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result( + ok, + "cold GEM faults after fixed mremap retain object offsets\n"); + if (alias != MAP_FAILED) { + pid_t child = fork(); + int status = 0; + if (!child) { + unsigned char *cold = mmap(NULL, len, + PROT_READ | PROT_WRITE, + MAP_SHARED, prime.fd, 0); + bool pass = cold != MAP_FAILED && check(cold, len, 0); + if (pass) + cold[0] = 0xe1; + _exit(pass ? 0 : 1); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && p[0] == 0xe1; + p[0] = value(0); + } else + ok = false; + ksft_test_result( + ok, "forked GEM mapping shares bytes and retains backing\n"); + if (alias != MAP_FAILED) { + q = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, MAP_SHARED, + prime.fd, len - ps); + ok = q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, 2 * ps); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + map.offset + ps); + ok &= q == MAP_FAILED && errno == EINVAL; + if (q != MAP_FAILED) + munmap(q, ps); + q = MAP_FAILED; + } else + ok = false; + ksft_test_result(ok, "PRIME bounds and exact GEM offset rejection\n"); + if (alias != MAP_FAILED) { + size_t targets[] = { native, 4096, 16384 }; + + ok = true; + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t child; + int status = 0; + bool pair; + + if (target > native || target == ps || + (t && target == native)) + continue; + child = fork(); + if (!child) { + char number[24], length[24], pages[24]; + + snprintf(number, sizeof(number), "%d", + prime.fd); + snprintf(length, sizeof(length), "%zu", len); + snprintf(pages, sizeof(pages), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, + 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--shared", number, + length, pages, NULL); + _exit(4); + } + pair = child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + for (size_t i = 0; i < len; i += 4096) { + pair &= p[i] == + (unsigned char)(0x80 + i / 4096); + p[i] = value(i); + } + ksft_print_msg( + "PRIME sharing %zuK -> %zuK status=%d bytes=%zu\n", + ps / 1024, target / 1024, status, len); + ok &= pair; + } + } else { + ok = false; + } + ksft_test_result( + ok, + "PRIME backing shared between native and alternative processes\n"); + struct drm_gem_close close_arg = { .handle = create.handle }; + ok = !ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close_arg); + close(fd); + if (alias != MAP_FAILED) { + close(prime.fd); + ok &= check(alias, len, 0); + munmap(alias, len); + } else + ok = false; + if (p != MAP_FAILED) { + ok &= check(p, len, 0); + munmap(p, len); + } else + ok = false; + ksft_test_result( + ok, + "mappings retain object after handles and descriptors close\n"); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--shared")) { + struct stat init, self; + int fd = atoi(argv[2]); + size_t len = strtoul(argv[3], NULL, 10); + if (stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || + init.st_ino != self.st_ino || init.st_dev != self.st_dev || + len < 65536 || len > 8 * 65536 || + getauxval(AT_PAGESZ) != strtoul(argv[4], NULL, 10)) + return 2; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + bool pass = p != MAP_FAILED && check(p, len, 0); + if (pass) + for (size_t i = 0; i < len; i += 4096) + p[i] = 0x80 + i / 4096; + if (p != MAP_FAILED) + munmap(p, len); + return pass ? 0 : 1; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(strtoul(argv[2], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/drm/card0/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vgem", S_IFCHR | 0600, makedev(maj, min))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + + for (unsigned int t = 0; t < 3; t++) { + size_t target = targets[t]; + pid_t pid; + int status = 0; + + if (target > native || (t && target == native)) + continue; + pid = fork(); + if (!pid) { + char number[24]; + + snprintf(number, sizeof(number), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", number, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# GEM child ABI=%zuK native=%zuK status=%d\n", + target / 1024, native / 1024, status); + } + printf("%s - GEM fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/granule-proc-init.c b/tools/testing/selftests/arm64/user4k/granule-proc-init.c new file mode 100644 index 00000000000000..e70f8df18e57a2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-proc-init.c @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--peer")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + unsigned char *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - 4096); + char token = 'a'; + int ok = getauxval(AT_PAGESZ) == 4096 && p != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < 4096; i++) + ok &= p[i] == 0x7b; + p[0] = 0xc9; + ok &= pss(p) == 2; + } + token = ok ? 'a' : 'x'; + if (write(ready, &token, 1) != 1) + return 3; + if (read(release, &token, 1) != 1) + return 4; + return ok ? 0 : 2; + } + if (getpid() != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("high-slot-pss", 0); + result(fd >= 0 && !ftruncate(fd, native), + "native file backing created"); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent mapping"); + goto done; + } + memset(p, 0x7b, native); + long before = pss(p); + printf("# native=%zu initial PSS=%ldKB\n", native, before); + result(before == (long)native / 1024, + "native PSS accounts every physical slot"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t child = fork(); + int status = 0; + if (!child) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--peer", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = child > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 4096] == 0xc9, + "4K peer shares the final physical slot and observes 2KB PSS"); + long during = pss(p); + printf("# parent mixed PSS=%ldKB expected=%zuKB\n", during, + native / 1024 - 2); + result(during == (long)native / 1024 - 2, + "native PSS weights the high shared slot independently"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + ok = waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + result(ok && pss(p) == (long)native / 1024, + "PSS returns to native size after peer releases its mapping"); + FILE *f = fopen("/proc/self/smaps_rollup", "r"); + char buf[512]; + ok = f != NULL; + if (f) { + while (fgets(buf, sizeof(buf), f)) { + } + ok &= !ferror(f); + fclose(f); + } + result(ok, "smaps_rollup reads complete native slot accounting"); + munmap(p, native); +done: + printf("GRANULE PROC %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-swap-init.c b/tools/testing/selftests/arm64/user4k/granule-swap-init.c new file mode 100644 index 00000000000000..5791f7e2c0ee9a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-swap-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "swapfile-fixture.h" +#define SET_EXEC_PAGE_SIZE 0x41555001 +static unsigned char pattern(size_t i, unsigned round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ round) & 255; +} +static int check(unsigned char *p, size_t size, unsigned round) +{ + for (size_t i = 0; i < size; i++) + if (p[i] != pattern(i, round)) { + printf("# byte=%zu got=%u expected=%u\n", i, p[i], + pattern(i, round)); + return 0; + } + return 1; +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char k[80]; + unsigned long n = 0, result = 0; + if (!f) + return 0; + while (fscanf(f, "%79s %lu", k, &n) == 2) + if (!strcmp(k, "pswpout")) + result = n; + fclose(f); + return result; +} +static unsigned swapped(void *p, size_t size, size_t ps) +{ + int fd = open("/proc/self/pagemap", O_RDONLY); + uint64_t bits; + unsigned n = 0; + if (fd < 0) + return 0; + for (size_t i = 0; i < size; i += ps) + if (pread(fd, &bits, 8, ((uintptr_t)p + i) / ps * 8) == 8) + n += (bits >> 62) & 1; + close(fd); + return n; +} +static int exercise(unsigned long expected) +{ + size_t ps = getauxval(AT_PAGESZ), size = 8 * 1024 * 1024; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int failed = 0; + if (ps != expected || p == MAP_FAILED) { + printf("not ok - expected ABI=%lu actual=%zu\n", expected, ps); + return 1; + } + for (unsigned round = 0; round < 4; round++) { + for (size_t i = 0; i < size; i++) + p[i] = pattern(i, round); + unsigned long before = swapouts(); + int advice = 0; + unsigned n = 0, attempts = 0; + /* PAGEOUT is advisory. Require evidence, with a bounded retry window. */ + do { + advice = madvise(p, size, MADV_PAGEOUT); + attempts++; + if (advice) + break; + usleep(20000); + n = swapped(p, size, ps); + } while (!n && attempts < 100); + unsigned long written = swapouts() - before; + int ok = !advice && written > 0 && n > 0; + printf("%s - %zuK round%u real swapped PTEs=%u/%zu writes=%lu attempts=%u\n", + ok ? "ok" : "not ok", ps / 1024, round, n, size / ps, + written, attempts); + failed += !ok; + int release[2], ready[2]; + if (pipe(release) || pipe(ready)) + return 2; + pid_t child = fork(); + int status = 0; + if (!child) { + char token; + close(release[1]); + close(ready[0]); + int ok = read(release[0], &token, 1) == 1 && + check(p, size, round); + if (ok) { + for (size_t i = 0; i < size; i += ps) + p[i] ^= 0x6d; + } + token = ok ? 'a' : 'x'; + write(ready[1], &token, 1); + _exit(ok ? 0 : 3); + } + close(release[0]); + close(ready[1]); + ok = check(p, size, round); + printf("%s - %zuK round%u disk roundtrip bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + char token = 'b'; + write(release[1], &token, 1); + ok = read(ready[0], &token, 1) == 1 && token == 'a' && + waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status) && check(p, size, round); + printf("%s - %zuK round%u forked swap-in and COW retain parent bytes\n", + ok ? "ok" : "not ok", ps / 1024, round); + failed += !ok; + close(release[1]); + close(ready[0]); + } + munmap(p, size); + return failed ? 1 : 0; +} +static int run_suite(bool nonlinear) +{ + const char *base = + nonlinear ? + "/sys/kernel/debug/kunit/arm64-user4k-nonlinear-swap/" : + "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[160]; + snprintf(path, sizeof(path), "%srun", base); + int fd = open(path, O_WRONLY); + if (fd < 0) + return 1; + int ok = write(fd, "1", 1) == 1; + close(fd); + snprintf(path, sizeof(path), "%sresults", base); + FILE *f = fopen(path, "r"); + char *line = NULL; + size_t cap = 0; + int completed = 0; + if (!f) + return 1; + while (getline(&line, &cap, f) > 0) { + fputs(line, stdout); + if (strstr(line, "not ok") || strstr(line, "# SKIP")) + ok = 0; + if (strstr(line, nonlinear ? + "ok 1 arm64-user4k-nonlinear-swap" : + "ok 1 arm64-user4k-swap")) + completed = 1; + } + free(line); + fclose(f); + return !(ok && completed); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) { + struct stat a, b; + if (getpid() == 1 || stat("/proc/1/exe", &a) || + stat("/proc/self/exe", &b) || a.st_dev != b.st_dev || + a.st_ino != b.st_ino) + return 1; + return (!strcmp(argv[1], "--swapoff-helper") ? + swapoff(swap_path()) : + swapon(swap_path(), 0)) ? + 1 : + 0; + } + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 1; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || + (argc != 1 && (argc != 2 || (strcmp(argv[1], "--el0-only") && + strcmp(argv[1], "--nonlinear") && + strcmp(argv[1], "--swapfile"))))) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + int failed = 0, active = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + failed++; + goto done; + } + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + (argc == 2 && !strcmp(argv[1], "--swapfile") && + prepare_swapfile()) || + swapon(swap_path(), 0)) { + perror("swapon"); + failed++; + goto done; + } + active = 1; + printf("GRANULE SWAP BACKING=%s\n", swap_path()); + if (argc == 1 || (argc == 2 && !strcmp(argv[1], "--nonlinear"))) + failed += run_suite(argc == 2); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned abi = 0; abi < (sizes[0] > 16384 ? 3U : 2U); abi++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[32]; + snprintf(expected, sizeof(expected), "%lu", sizes[abi]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[abi], 0UL, 0UL, + 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - swap ABI %luK child status=%#x\n", + ok ? "ok" : "not ok", sizes[abi] / 1024, status); + failed += !ok; + } +done: + if (active && swapoff(swap_path())) { + perror("swapoff"); + failed++; + } + if (active && argc == 2 && !strcmp(argv[1], "--swapfile") && + umount("/swapdir")) { + perror("umount"); + failed++; + } + printf("GRANULE SWAP %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/granule-threeway-init.c b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c new file mode 100644 index 00000000000000..71a459c9b52219 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/granule-threeway-init.c @@ -0,0 +1,248 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +struct snapshot { + long pss; + int first, last, status; +}; +struct peer { + pid_t pid; + int command, reply; + size_t size; +}; +static int failed; +static void result(int ok, const char *s) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", s); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static int transfer(int fd, void *data, size_t size, int writing) +{ + size_t done = 0; + while (done < size) { + ssize_t n = + writing ? write(fd, (char *)data + done, size - done) : + read(fd, (char *)data + done, size - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return 0; + done += n; + } + return 1; +} +static int worker(int fd, int command, int reply, size_t size) +{ + const struct rlimit no_core = { 0, 0 }; + setrlimit(RLIMIT_CORE, &no_core); + if (getppid() != 1 || getauxval(AT_PAGESZ) != size) + return 2; + unsigned char *p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, + fd, 65536 - size); + if (p == MAP_FAILED) + return 3; + for (;;) { + char op; + struct snapshot s = { .pss = -1, .first = -1, .last = -1 }; + if (!transfer(command, &op, 1, 0)) + return 4; + if (op == 'x') { + s.status = munmap(p, size); + transfer(reply, &s, sizeof(s), 1); + return s.status ? 5 : 0; + } + if (op == 'r') + s.status = mprotect(p, size, PROT_READ); + else if (op == 'w') + p[size - 1] = 0x99; + else if (op == 'd') + s.status = madvise(p, size, MADV_DONTNEED); + else if (op == 't') { + pid_t c = fork(); + int status = 0; + if (!c) { + unsigned char value = *( + volatile unsigned char *)(p + size - 1); + _exit(value); + } + if (c < 0 || waitpid(c, &status, 0) != c) + s.status = -1; + else + s.status = WIFSIGNALED(status) ? + 256 + WTERMSIG(status) : + WIFEXITED(status) ? + WEXITSTATUS(status) : + -2; + } + if (op != 'd' && op != 't') { + s.first = p[0]; + s.last = p[size - 1]; + } + s.pss = pss(p); + if (!transfer(reply, &s, sizeof(s), 1)) + return 6; + } +} +static struct peer spawn(int fd, size_t size) +{ + int command[2], reply[2]; + struct peer p = { .pid = -1, .size = size }; + if (pipe(command) || pipe(reply)) + return p; + p.pid = fork(); + if (!p.pid) { + char f[24], c[24], r[24], s[24]; + close(command[1]); + close(reply[0]); + snprintf(f, sizeof(f), "%d", fd); + snprintf(c, sizeof(c), "%d", command[0]); + snprintf(r, sizeof(r), "%d", reply[1]); + snprintf(s, sizeof(s), "%zu", size); + if (prctl(SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", f, c, r, s, NULL); + _exit(8); + } + close(command[0]); + close(reply[1]); + p.command = command[1]; + p.reply = reply[0]; + return p; +} +static struct snapshot command(struct peer *p, char op) +{ + struct snapshot s = { .pss = -1, .first = -1, .last = -1, .status = -1 }; + if (p->pid <= 0 || !transfer(p->command, &op, 1, 1) || + !transfer(p->reply, &s, sizeof(s), 0)) + failed++; + return s; +} +static void finish(struct peer *p) +{ + struct snapshot s = command(p, 'x'); + int status = 0; + result(!s.status && waitpid(p->pid, &status, 0) == p->pid && + WIFEXITED(status) && !WEXITSTATUS(status), + "peer unmap and exit"); + close(p->command); + close(p->reply); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable64K VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + if (getauxval(AT_PAGESZ) != 65536) { + result(0, "native64K required"); + goto done; + } + int fd = memfd_create("three-granules", 0); + if (fd < 0 || ftruncate(fd, 65536)) { + result(0, "file setup"); + goto done; + } + unsigned char *p = + mmap(NULL, 65536, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "native mapping"); + goto done; + } + memset(p, 0x42, 65536); + struct peer medium = spawn(fd, 16384); + struct snapshot s = command(&medium, 's'); + result(s.pss == 8 && s.first == 0x42 && s.last == 0x42 && pss(p) == 56, + "64K/16K initial sharing and PSS"); + struct peer small = spawn(fd, 4096); + s = command(&small, 's'); + result(s.pss == 1 && s.first == 0x42 && s.last == 0x42, + "4K peer shares the final slice with both larger ABIs"); + s = command(&medium, 's'); + result(s.pss == 7 && pss(p) == 55, + "three-way PSS weights partial overlaps independently"); + s = command(&small, 'r'); + result(!s.status, "4K alias becomes read-only"); + s = command(&medium, 'w'); + result(!s.status && p[65535] == 0x99, + "16K alias remains writable across local4K protection"); + s = command(&small, 's'); + result(s.last == 0x99, "read-only4K alias observes16K writes"); + s = command(&small, 'd'); + result(!s.status && s.pss == 0 && pss(p) == 56, + "4K discard removes only its mapping"); + s = command(&medium, 's'); + result(s.pss == 8 && s.last == 0x99, "16K mapping survives4K discard"); + s = command(&small, 's'); + result(s.pss == 1 && s.last == 0x99 && pss(p) == 55, + "4K refault restores coherent three-way sharing"); + result(!ftruncate(fd, 65536 - 4096), + "truncate at4K boundary inside16K leaf"); + s = command(&small, 't'); + result(s.status == 256 + SIGBUS, + "4K mapping wholly beyond EOF faults SIGBUS"); + s = command(&medium, 't'); + result(s.status == 0, "partial16K EOF leaf exposes zero tail"); + result(p[65535] == 0 && p[0] == 0x42, + "native64K sees zero tail and preserved file prefix"); + s = command(&medium, 's'); + result(s.first == 0x42 && s.last == 0 && s.pss == 8 && pss(p) == 56, + "16K EOF refault keeps correct mixed PSS"); + result(!ftruncate(fd, 65536), "restore file length"); + s = command(&small, 's'); + result(s.first == 0 && s.last == 0 && s.pss == 1 && pss(p) == 55, + "4K alias refaults zeroed extension"); + finish(&small); + s = command(&medium, 's'); + result(s.pss == 8 && pss(p) == 56, "16K alias survives4K exit"); + finish(&medium); + result(pss(p) == 64, + "native PSS restored after all alternative aliases exit"); + munmap(p, 65536); + close(fd); +done: + printf("THREE GRANULES %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/iommufd-init.c b/tools/testing/selftests/arm64/user4k/iommufd-init.c new file mode 100644 index 00000000000000..c678f35cf72a46 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/iommufd-init.c @@ -0,0 +1,1314 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable VM PID1: real IOMMUFD ioctls against mock AMDv1 page tables. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../drivers/iommu/iommufd/iommufd_test.h" + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define VA 0x200000000UL +#define IOVA 0x10000000UL +#define ALIAS 0x12000000UL +#define MAP_FLAGS \ + (IOMMU_IOAS_MAP_FIXED_IOVA | IOMMU_IOAS_MAP_READABLE | \ + IOMMU_IOAS_MAP_WRITEABLE) +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static size_t native_ps, user_ps; +struct context { + int fd, access; + uint32_t ioas, hwpt[2], dev[2]; +}; + +static int drop_pin_cap(void) +{ + struct __user_cap_header_struct h = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct d[2]; + unsigned int mask = 1U << (CAP_IPC_LOCK % 32); + + if (syscall(SYS_capget, &h, d)) + return -1; + d[CAP_IPC_LOCK / 32].effective &= ~mask; + d[CAP_IPC_LOCK / 32].permitted &= ~mask; + return syscall(SYS_capset, &h, d); +} + +static int limit_bytes(size_t bytes) +{ + struct rlimit r; + + if (getrlimit(RLIMIT_MEMLOCK, &r)) + return -1; + r.rlim_cur = bytes; + return setrlimit(RLIMIT_MEMLOCK, &r); +} + +static long stat_kb(pid_t pid, const char *key) +{ + char path[80], line[256]; + long result = -1; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/status", pid); + f = fopen(path, "r"); + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (!strncmp(line, key, strlen(key))) { + result = strtol(line + strlen(key), NULL, 10); + break; + } + fclose(f); + return result; +} + +static int new_domain(struct context *c, unsigned int index) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MOCK_DOMAIN, + .id = c->ioas }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) { + c->hwpt[index] = cmd.mock_domain.out_hwpt_id; + c->dev[index] = cmd.mock_domain.out_idev_id; + } + return rc; +} + +static int context_open_common(struct context *c, unsigned int mm_account, + bool domain, bool pins) +{ + struct iommu_ioas_alloc a = { .size = sizeof(a) }; + struct iommu_option o = { .size = sizeof(o), + .option_id = IOMMU_OPTION_RLIMIT_MODE, + .op = IOMMU_OPTION_OP_SET, + .val64 = mm_account }; + struct iommu_test_cmd access = { .size = sizeof(access), + .op = IOMMU_TEST_OP_CREATE_ACCESS }; + + memset(c, 0, sizeof(*c)); + c->fd = open("/dev/iommu", O_RDWR); + if (c->fd < 0 || ioctl(c->fd, IOMMU_OPTION, &o) || + ioctl(c->fd, IOMMU_IOAS_ALLOC, &a)) + return -1; + c->ioas = a.out_ioas_id; + if (domain && new_domain(c, 0)) + return -1; + access.id = c->ioas; + access.create_access.flags = + pins ? MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES : 0; + if (ioctl(c->fd, IOMMU_TEST_CMD, &access)) + return -1; + c->access = access.create_access.out_access_fd; + return 0; +} + +static int context_open(struct context *c, unsigned int mm_account) +{ + return context_open_common(c, mm_account, true, false); +} + +static void context_close(struct context *c) +{ + close(c->access); + close(c->fd); +} + +static int map_range(struct context *c, void *p, size_t len, unsigned long iova) +{ + struct iommu_ioas_map m = { .size = sizeof(m), + .flags = MAP_FLAGS, + .ioas_id = c->ioas, + .user_va = (uintptr_t)p, + .length = len, + .iova = iova }; + + return ioctl(c->fd, IOMMU_IOAS_MAP, &m); +} + +static int unmap_range(struct context *c, size_t len, unsigned long iova) +{ + struct iommu_ioas_unmap u = { .size = sizeof(u), + .ioas_id = c->ioas, + .iova = iova, + .length = len }; + int rc = ioctl(c->fd, IOMMU_IOAS_UNMAP, &u); + + return rc ?: (u.length == len ? 0 : -1); +} + +static int check_pa(struct context *c, unsigned int dom, void *p, size_t len, + unsigned long iova) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_MD_CHECK_MAP, + .id = c->hwpt[dom], + .check_map = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int access_rw(struct context *c, void *p, size_t len, unsigned long iova, + unsigned int flags) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_RW, + .id = c->access, + .access_rw = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = flags } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static bool filled(const void *ptr, unsigned char byte, size_t n) +{ + const unsigned char *p = ptr; + + for (size_t i = 0; i < n; i++) + if (p[i] != byte) + return false; + return true; +} + +static void *relocated(size_t len, unsigned char byte) +{ + void *old = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0); + + if (old == MAP_FAILED) + return old; + memset(old, byte, len); + size_t distance = len > 32 * native_ps ? len + native_ps : + 32 * native_ps; + + return mremap(old, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + distance + user_ps)); +} + +static int limits_and_lifetime(unsigned int mode) +{ + struct context c; + size_t len = 2 * native_ps; + unsigned char *p = relocated(3 * native_ps, 0x31), *buf = malloc(len); + struct iommu_ioas_copy copy; + + CHECK(p != MAP_FAILED && buf); + CHECK(!context_open(&c, mode)); + CHECK(!limit_bytes(len)); + CHECK(map_range(&c, p, 3 * native_ps, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmLck:") == (mode ? (long)len / 1024 : 0)); + CHECK(!check_pa(&c, 0, p, len, IOVA)); + CHECK(map_range(&c, p + len, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + copy = (struct iommu_ioas_copy){ .size = sizeof(copy), + .flags = MAP_FLAGS, + .src_ioas_id = c.ioas, + .dst_ioas_id = c.ioas, + .src_iova = IOVA + user_ps, + .dst_iova = ALIAS, + .length = len - user_ps }; + CHECK(!ioctl(c.fd, IOMMU_IOAS_COPY, ©)); + CHECK(!new_domain(&c, 1)); + CHECK(!check_pa(&c, 1, p + user_ps, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(mmap(p, len, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) == p); + memset(p, 0x5a, len); + CHECK(check_pa(&c, 0, p, len, IOVA) == -1 && errno == EINVAL); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x31, len)); + CHECK(!access_rw(&c, buf, len, IOVA, 0)); + CHECK(filled(buf, 0x5a, len)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(user_ps < native_ps ? len : len - user_ps) / 1024); + CHECK(!unmap_range(&c, len - user_ps, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* Fresh anonymous backing follows the shifted VA's native linear index. + * A non-native-aligned replacement spans three physical backing pages. + */ + size_t replacement_pages = user_ps < native_ps ? 3 : 2; + + if (replacement_pages == 3) { + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == ENOMEM); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + } + CHECK(!limit_bytes(replacement_pages * native_ps)); + CHECK(!map_range(&c, p, len, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)(replacement_pages * native_ps / 1024)); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(p, 3 * native_ps)); + context_close(&c); + free(buf); + return 0; +} + +static int holes_and_shared(unsigned int mode) +{ + struct context c; + size_t len = 3 * user_ps, file_len = 4 * native_ps; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0), + *buf = malloc(file_len); + int file = memfd_create("iommu-shared", 0); + + CHECK(p != MAP_FAILED && buf && file >= 0); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + memset(p, 0x31, len); + CHECK(!munmap(p + user_ps, user_ps)); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mmap(p + user_ps, user_ps, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == p + user_ps); + CHECK(map_range(&c, p, len, IOVA) == -1 && errno == EFAULT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!mprotect(p + user_ps, user_ps, PROT_READ | PROT_WRITE)); + CHECK(!map_range(&c, p, len, IOVA) && !check_pa(&c, 0, p, len, IOVA)); + CHECK(!unmap_range(&c, len, IOVA) && !munmap(p, len)); + CHECK(!ftruncate(file, file_len)); + p = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, MAP_SHARED, file, + user_ps); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * user_ps, IOVA)); + CHECK(!munmap(p, 2 * user_ps)); + memset(buf, 0x93, 2 * user_ps); + CHECK(!access_rw(&c, buf, 2 * user_ps, IOVA, + MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + CHECK(pread(file, buf, file_len, 0) == (ssize_t)file_len); + CHECK(filled(buf, 0, user_ps)); + CHECK(filled(buf + user_ps, 0x93, 2 * user_ps)); + CHECK(filled(buf + 3 * user_ps, 0, file_len - 3 * user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(file); + context_close(&c); + free(buf); + return 0; +} + +static int producer(int argc, char **argv) +{ + struct context c = { .fd = atoi(argv[4]), + .ioas = strtoul(argv[5], NULL, 0) }; + int ready = atoi(argv[6]), go = atoi(argv[7]); + char byte = 'R'; + void *p; + + (void)argc; + CHECK(!prctl(PR_SET_PDEATHSIG, SIGKILL)); + CHECK(!drop_pin_cap()); + p = relocated(2 * native_ps, 0x6d); + CHECK(p != MAP_FAILED); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(write(ready, &byte, 1) == 1); + CHECK(read(go, &byte, 1) == 1 && byte == 'G'); + return 0; +} + +static int destroy_id(struct context *c, uint32_t id) +{ + struct iommu_destroy cmd = { .size = sizeof(cmd), .id = id }; + + return ioctl(c->fd, IOMMU_DESTROY, &cmd); +} + +static int create_viommu(struct context *c, uint32_t *id, uint32_t *parent) +{ + struct iommu_hwpt_alloc hwpt = { .size = sizeof(hwpt), + .flags = IOMMU_HWPT_ALLOC_NEST_PARENT, + .dev_id = c->dev[0], + .pt_id = c->ioas }; + struct iommu_viommu_alloc cmd = { .size = sizeof(cmd), + .type = IOMMU_VIOMMU_TYPE_SELFTEST, + .dev_id = c->dev[0] }; + + if (ioctl(c->fd, IOMMU_HWPT_ALLOC, &hwpt)) + return -1; + *parent = hwpt.out_hwpt_id; + cmd.hwpt_id = *parent; + if (ioctl(c->fd, IOMMU_VIOMMU_ALLOC, &cmd)) + return -1; + *id = cmd.out_viommu_id; + return 0; +} + +static int queue_alloc(struct context *c, uint32_t viommu, unsigned int index, + unsigned long iova, size_t length, uint32_t *id) +{ + struct iommu_hw_queue_alloc cmd = { .size = sizeof(cmd), + .viommu_id = viommu, + .type = IOMMU_HW_QUEUE_TYPE_SELFTEST, + .index = index, + .nesting_parent_iova = iova, + .length = length }; + int rc = ioctl(c->fd, IOMMU_HW_QUEUE_ALLOC, &cmd); + + if (!rc) + *id = cmd.out_hw_queue_id; + return rc; +} + +static int queue_after_exit(struct context *c) +{ + uint32_t viommu, parent, id; + + CHECK(!create_viommu(c, &viommu, &parent)); + CHECK(!queue_alloc(c, viommu, 0, IOVA + native_ps / 4 + 3, + native_ps / 4, &id)); + CHECK(unmap_range(c, 2 * native_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(c, id)); + CHECK(!destroy_id(c, viommu)); + CHECK(!destroy_id(c, parent)); + return 0; +} + +static int queue_ranges(unsigned int mode) +{ + struct context c; + unsigned char *p = relocated(2 * native_ps, 0x71); + uint32_t viommu, parent, q0, q1, unused; + size_t offset = user_ps / 4 + 3, length = user_ps / 2 + 17; + int file; + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + /* Map only one user leaf: queue pinning must not require surrounding IOVA. */ + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 1, IOVA + offset, length, &unused) == + -1 && + errno == EIO); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + offset, length, &q0)); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == EEXIST); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(stat_kb(getpid(), "VmPin:") == (long)native_ps / 1024); + /* Pins keep the old physical bytes after replacing the source VMA. */ + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p); + memset(p, 0x5a, user_ps); + CHECK(!queue_alloc(&c, viommu, 1, IOVA + offset + 7, length - 7, &q1)); + CHECK(destroy_id(&c, q0) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q1)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(queue_alloc(&c, viommu, 0, IOVA + offset, length, &unused) == + -1 && + errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + + /* Separate mappings of the same file byte range cannot form a physical run. + * The second area must fail after the first area's access was registered. + */ + file = memfd_create("queue-alias", 0); + CHECK(file >= 0 && !ftruncate(file, native_ps)); + p = mmap((void *)VA, 2 * user_ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p, user_ps, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, + file, 0) == p); + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, 0) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == ENOENT); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* The same discontinuity inside one provider must also release its access. */ + CHECK(!map_range(&c, p, 2 * user_ps, IOVA)); + CHECK(queue_alloc(&c, viommu, 0, IOVA, 2 * user_ps, &unused) == -1 && + errno == EFAULT); + CHECK(!unmap_range(&c, 2 * user_ps, IOVA)); + if (user_ps < native_ps) { + /* Adjacent file quarters form a real physical run across two areas. */ + CHECK(mmap(p + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, + user_ps) == p + user_ps); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!map_range(&c, p + user_ps, user_ps, IOVA + user_ps)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + 11, 2 * user_ps - 22, + &q0)); + CHECK(unmap_range(&c, user_ps, IOVA) == -1 && errno == EBUSY); + CHECK(unmap_range(&c, user_ps, IOVA + user_ps) == -1 && + errno == EBUSY); + CHECK(!destroy_id(&c, q0)); + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(!unmap_range(&c, user_ps, IOVA + user_ps)); + } + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * user_ps)); + close(file); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + printf("ok - hardware queue byte spans %zuK/%zuK %s, " + "relocated/overlap/rollback/unmap locks\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int cross_process(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + CHECK(!context_open(&c, mode) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + else { + CHECK(!map_range(&c, p, native_ps, ALIAS)); + CHECK(!unmap_range(&c, native_ps, ALIAS)); + } + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!new_domain(&c, 1)); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + CHECK(!queue_after_exit(&c)); + if (!mode) + CHECK(map_range(&c, p, native_ps, ALIAS) == -1 && + errno == ENOMEM); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!map_range(&c, p, len, ALIAS) && !unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + close(ready[0]); + close(go[1]); + CHECK(!munmap(p, len)); + context_close(&c); + free(buf); + return 0; +} + +struct stress_arg { + struct context *c; + void *p; + unsigned int index; + atomic_int *failed; +}; +static void *stress_thread(void *opaque) +{ + struct stress_arg *a = opaque; + unsigned long iova = IOVA + a->index * 0x100000UL; + size_t len = 2 * native_ps; + + for (unsigned int round = 0; round < 64; round++) { + if (map_range(a->c, a->p, len, iova) || + (!(round % 16) && check_pa(a->c, 1, a->p, len, iova)) || + unmap_range(a->c, len, iova)) { + printf("FAIL concurrent map thread=%u round=%u errno=%d\n", + a->index, round, errno); + atomic_store(a->failed, 1); + break; + } + } + return NULL; +} + +static int concurrent_maps(unsigned int mode) +{ + struct context c; + pthread_t threads[4]; + struct stress_arg args[4]; + atomic_int failed = 0; + void *p = relocated(2 * native_ps, 0x47); + + CHECK(p != MAP_FAILED && !context_open(&c, mode)); + CHECK(!limit_bytes(32 * native_ps) && !new_domain(&c, 1)); + for (unsigned int i = 0; i < 4; i++) { + args[i] = (struct stress_arg){ &c, p, i, &failed }; + CHECK(!pthread_create(&threads[i], NULL, stress_thread, + &args[i])); + } + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&failed)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, 2 * native_ps)); + context_close(&c); + return 0; +} + +static int write_attribute(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + ssize_t n; + + if (fd < 0) + return -1; + n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} + +static int allocation_failures(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x62); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned int hit = 0, n; + + CHECK(nth >= 0 && p != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(8 * native_ps)); + CHECK(!new_domain(&c, 1)); + CHECK(!write_attribute("/sys/kernel/debug/failslab/ignore-gfp-wait", + "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-wait", "N")); + CHECK(!write_attribute( + "/sys/kernel/debug/fail_page_alloc/ignore-gfp-highmem", "N")); + CHECK(!write_attribute("/sys/kernel/debug/failslab/verbose", "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_page_alloc/verbose", + "0")); + CHECK(!write_attribute("/sys/kernel/debug/fail_iommufd/verbose", "0")); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n); + ssize_t got, disabled; + int rc; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = map_range(&c, p, 2 * native_ps, IOVA); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(disabled == 1 && got > 0); + if (!rc) + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + /* An injected failure must not poison a successful retry or translation. */ + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 0, p, 2 * native_ps, IOVA)); + CHECK(!check_pa(&c, 1, p, 2 * native_ps, IOVA)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + printf("ok - IOMMUFD %zuK %s accounting allocation-failure sweep: %u sites\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + CHECK(!munmap(p, 2 * native_ps)); + close(nth); + context_close(&c); + return 0; +} + +static int queue_failure_sweep(unsigned int mode) +{ + struct context c; + void *p = relocated(2 * native_ps, 0x4c); + int nth = open("/proc/self/fail-nth", O_RDWR); + uint32_t viommu, parent, id; + unsigned int n, hit = 0; + + CHECK(p != MAP_FAILED && nth >= 0 && !context_open(&c, mode)); + CHECK(!limit_bytes(8 * native_ps) && + !create_viommu(&c, &viommu, &parent)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!destroy_id(&c, id)); + /* A failed queue must not leave an internal access blocking unmap. */ + CHECK(!unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!map_range(&c, p, user_ps, IOVA)); + CHECK(!queue_alloc(&c, viommu, 0, IOVA + user_ps / 4 + 3, + user_ps / 2, &id)); + CHECK(!destroy_id(&c, id) && !unmap_range(&c, user_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!destroy_id(&c, viommu) && !destroy_id(&c, parent)); + context_close(&c); + close(nth); + CHECK(!munmap(p, 2 * native_ps)); + printf("ok - hardware queue %zuK %s failure sweep: %u sites, no stale locks/pins\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +static int page_access(struct context *c, void *p, size_t len, + unsigned long iova, uint32_t *id) +{ + struct iommu_test_cmd cmd = { + .size = sizeof(cmd), + .op = IOMMU_TEST_OP_ACCESS_PAGES, + .id = c->access, + .access_pages = { .iova = iova, + .length = len, + .uptr = (uintptr_t)p, + .flags = MOCK_FLAGS_ACCESS_WRITE } + }; + int rc = ioctl(c->fd, IOMMU_TEST_CMD, &cmd); + + if (!rc) + *id = cmd.access_pages.out_access_pages_id; + return rc; +} + +static int page_release(struct context *c, uint32_t id) +{ + struct iommu_test_cmd cmd = { .size = sizeof(cmd), + .op = IOMMU_TEST_OP_DESTROY_ACCESS_PAGES, + .id = c->access, + .destroy_access_pages = { + .access_pages_id = id } }; + + return ioctl(c->fd, IOMMU_TEST_CMD, &cmd); +} + +static int page_ranges(unsigned int mode) +{ + struct context c; + int file = memfd_create("native-page-array", 0); + unsigned char *p; + uint32_t a, b, d, e; + size_t len = 3 * native_ps, tail = native_ps + user_ps / 2 + 3; + + CHECK(file >= 0 && !ftruncate(file, 5 * native_ps)); + p = mmap((void *)VA, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !limit_bytes(4 * native_ps)); + memset(p, 0x67, len); + CHECK(!context_open_common(&c, mode, false, true)); + /* Separate areas exercise native output stride with alternative indices. */ + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, 2 * native_ps, IOVA + native_ps)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, len, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + CHECK(!page_access(&c, p, len, IOVA, &b)); + CHECK(!page_access(&c, p + native_ps, tail, IOVA + native_ps, &d)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)len / 1024); + /* Source VA can move without changing its retained physical offset. */ + unsigned char *moved = mremap(p, len, len, + MREMAP_MAYMOVE | MREMAP_FIXED, + (void *)(VA + 32 * native_ps + user_ps)); + CHECK(moved != MAP_FAILED); + CHECK(!page_access(&c, moved, len, IOVA, &e) && !page_release(&c, e)); + CHECK(!page_release(&c, a) && !page_release(&c, b)); + CHECK(stat_kb(getpid(), "VmPin:") == (long)(2 * native_ps) / 1024); + CHECK(!page_release(&c, d)); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(mremap(moved, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, p) == p); + CHECK(!page_access(&c, p, len, IOVA, &a)); + /* Unmap callback releases an access covering both independent areas. */ + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + + /* The second area is absent: first area's pin/accounting must roll back. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, NULL, 2 * native_ps, IOVA, &a) == -1 && + errno == ENOENT); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, native_ps, IOVA) && !munmap(p, native_ps)); + + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, native_ps); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file, native_ps) == p + native_ps); + CHECK(!map_range(&c, p, 2 * native_ps, IOVA)); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &a)); + CHECK(stat_kb(getpid(), "VmPin:") == + (long)((user_ps == native_ps ? 2 : 1) * native_ps) / 1024); + CHECK(!page_release(&c, a) && !unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(!munmap(p, 2 * native_ps)); + + if (user_ps < native_ps) { + /* Native-sized logical range whose physical base has a fragment offset. */ + p = mmap((void *)VA, native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, user_ps); + CHECK(p != MAP_FAILED && !map_range(&c, p, native_ps, IOVA)); + CHECK(page_access(&c, p, native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!unmap_range(&c, native_ps, IOVA) && + !munmap(p, native_ps)); + + /* Aligned first fragment followed by an alias, not consecutive bytes. */ + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + CHECK(mmap(p + native_ps + user_ps, user_ps, + PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, file, + native_ps) == p + native_ps + user_ps); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, + IOVA + native_ps)); + CHECK(page_access(&c, p, 2 * native_ps, IOVA, &a) == -1 && + errno == EINVAL); + CHECK(stat_kb(getpid(), "VmPin:") == 0); + CHECK(!page_access(&c, p, native_ps, IOVA, &a)); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(page_release(&c, a) == -1 && errno == ENOENT); + CHECK(!munmap(p, 2 * native_ps)); + } + context_close(&c); + close(file); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + printf("ok - native page arrays %zuK/%zuK %s, lazy/partial/moved/rollback/callback\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_lifetime(unsigned int mode) +{ + struct context c; + int ready[2], go[2], status; + pid_t child; + size_t child_ps = user_ps == native_ps ? 4096 : native_ps; + size_t len = 2 * native_ps; + unsigned char *p = mmap((void *)(VA + 128 * native_ps), len, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_FIXED_NOREPLACE, + -1, 0), + *buf = malloc(len); + uint32_t a, b; + char byte; + + CHECK(p != MAP_FAILED && buf && !pipe(ready) && !pipe(go)); + memset(p, 0x42, len); + CHECK(!context_open_common(&c, mode, false, true) && !limit_bytes(len)); + child = fork(); + CHECK(child >= 0); + if (!child) { + char ps[24], native[24], fd[24], ioas[24], r[24], g[24]; + + close(ready[0]); + close(go[1]); + snprintf(ps, sizeof(ps), "%zu", child_ps); + snprintf(native, sizeof(native), "%zu", native_ps); + snprintf(fd, sizeof(fd), "%d", c.fd); + snprintf(ioas, sizeof(ioas), "%u", c.ioas); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + if (prctl(SET_EXEC_PAGE_SIZE, child_ps, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--producer", ps, native, fd, ioas, r, + g, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + CHECK(read(ready[0], &byte, 1) == 1 && byte == 'R'); + CHECK(stat_kb(child, "VmPin:") == 0); + CHECK(!page_access(&c, NULL, len, IOVA, &a)); + CHECK(stat_kb(child, "VmPin:") == (long)len / 1024); + CHECK(!access_rw(&c, buf, len, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, len)); + byte = 'G'; + CHECK(write(go[1], &byte, 1) == 1); + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + /* Reuse retained anonymous fragments after source mm_users reaches zero. */ + CHECK(!page_access(&c, NULL, native_ps, IOVA, &b)); + CHECK(!page_release(&c, a)); + CHECK(!access_rw(&c, buf, native_ps, IOVA, MOCK_ACCESS_RW_SLOW_PATH)); + CHECK(filled(buf, 0x6d, native_ps)); + CHECK(!map_range(&c, p, len, ALIAS)); + if (!mode) + CHECK(page_access(&c, p, len, ALIAS, &a) == -1 && + errno == ENOMEM); + else + CHECK(!page_access(&c, p, len, ALIAS, &a) && + !page_release(&c, a)); + /* Final access release refunds even a dead provider's account. */ + CHECK(!page_release(&c, b) && !unmap_range(&c, len, IOVA)); + CHECK(!page_access(&c, p, len, ALIAS, &a) && !page_release(&c, a)); + CHECK(!unmap_range(&c, len, ALIAS)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(p, len)); + context_close(&c); + close(ready[0]); + close(go[1]); + free(buf); + printf("ok - native page array anonymous source %zuK -> %zuK consumer, %s exit/refund\n", + child_ps / 1024, user_ps / 1024, mode ? "MM" : "USER"); + return 0; +} + +static int page_failure_sweep(unsigned int mode) +{ + struct context c; + int file = memfd_create("page-array-failures", 0); + int nth = open("/proc/self/fail-nth", O_RDWR); + unsigned char *p; + uint32_t id; + unsigned int n, hit = 0; + + CHECK(file >= 0 && nth >= 0 && !ftruncate(file, 2 * native_ps)); + p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(p != MAP_FAILED); + memset(p, 0x53, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(2 * native_ps)); + CHECK(!map_range(&c, p, native_ps, IOVA)); + CHECK(!map_range(&c, p + native_ps, native_ps, IOVA + native_ps)); + for (n = 1; n <= 512; n++) { + char number[24], remaining[24] = {}; + int bytes = snprintf(number, sizeof(number), "%u", n), rc; + ssize_t got, disabled; + + CHECK(pwrite(nth, number, bytes, 0) == bytes); + rc = page_access(&c, p, 2 * native_ps, IOVA, &id); + got = pread(nth, remaining, sizeof(remaining) - 1, 0); + if (got < 0 && errno == EFAULT) { + strcpy(remaining, "1"); + got = 1; + } + disabled = pwrite(nth, "0", 1, 0); + if (disabled < 0 && errno == EFAULT) { + disabled = pwrite(nth, "0", 1, 0); + strcpy(remaining, "1"); + } + CHECK(got > 0 && disabled == 1); + if (!rc) + CHECK(!page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!page_access(&c, p, 2 * native_ps, IOVA, &id) && + !page_release(&c, id)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + if (strtoul(remaining, NULL, 0)) + break; + hit++; + } + CHECK(hit > 0 && n <= 512); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA) && + !munmap(p, 2 * native_ps)); + context_close(&c); + close(file); + close(nth); + printf("ok - native page arrays %zuK %s failure sweep: %u sites, refunds/retries\n", + user_ps / 1024, mode ? "MM" : "USER", hit); + return 0; +} + +struct page_race { + struct context *c; + void *p; + pthread_barrier_t barrier; + atomic_int failed, pinned, absent; +}; + +static void *page_pin_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + uint32_t id; + + if (page_access(r->c, r->p, 2 * native_ps, IOVA, &id)) { + /* Unmap can mark the area inaccessible before notifying drivers. */ + if (errno != ENOENT && errno != EINVAL) + goto fail; + atomic_fetch_add(&r->absent, 1); + } else { + atomic_fetch_add(&r->pinned, 1); + sched_yield(); + if (page_release(r->c, id) && errno != ENOENT) + goto fail; + } + } + return NULL; +fail: + printf("FAIL page pin/unmap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static void *page_unmap_thread(void *opaque) +{ + struct page_race *r = opaque; + + pthread_barrier_wait(&r->barrier); + for (unsigned int i = 0; i < 256; i++) { + if (unmap_range(r->c, 2 * native_ps, IOVA)) + goto fail; + sched_yield(); + if (map_range(r->c, r->p, 2 * native_ps, IOVA)) + goto fail; + } + return NULL; +fail: + printf("FAIL page unmap/remap race errno=%d\n", errno); + atomic_store(&r->failed, 1); + return NULL; +} + +static int page_concurrency(unsigned int mode) +{ + struct context c; + struct page_race r = { .c = &c }; + pthread_t threads[4]; + int file = memfd_create("page-array-race", 0); + + CHECK(file >= 0 && !ftruncate(file, 2 * native_ps)); + r.p = mmap((void *)VA, 2 * native_ps, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, 0); + CHECK(r.p != MAP_FAILED); + memset(r.p, 0x49, 2 * native_ps); + CHECK(!context_open_common(&c, mode, false, true) && + !limit_bytes(8 * native_ps)); + CHECK(!map_range(&c, r.p, 2 * native_ps, IOVA)); + CHECK(!pthread_barrier_init(&r.barrier, NULL, 4)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_create( + &threads[i], NULL, + i == 3 ? page_unmap_thread : page_pin_thread, &r)); + for (unsigned int i = 0; i < 4; i++) + CHECK(!pthread_join(threads[i], NULL)); + CHECK(!atomic_load(&r.failed) && atomic_load(&r.pinned) > 0); + CHECK(!unmap_range(&c, 2 * native_ps, IOVA)); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!munmap(r.p, 2 * native_ps)); + context_close(&c); + close(file); + CHECK(!pthread_barrier_destroy(&r.barrier)); + printf("ok - native page arrays %zuK %s race: %d pins, %d unavailable, 256 remaps\n", + user_ps / 1024, mode ? "MM" : "USER", atomic_load(&r.pinned), + atomic_load(&r.absent)); + return 0; +} + +static int dirty_ioctl(struct context *c, uint32_t hwpt, void *bitmap, + unsigned long iova, size_t length, size_t granule, + unsigned int flags) +{ + struct iommu_hwpt_get_dirty_bitmap cmd = { .size = sizeof(cmd), + .hwpt_id = hwpt, + .flags = flags, + .iova = iova, + .length = length, + .page_size = granule, + .data = (uintptr_t)bitmap }; + + return ioctl(c->fd, IOMMU_HWPT_GET_DIRTY_BITMAP, &cmd); +} + +static int dirty_errors(unsigned int mode) +{ + struct context c; + size_t len = 66 * native_ps; + unsigned char *data = relocated(len, 0x34); + unsigned char *out = mmap(NULL, 2 * user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t *bitmap = (void *)(out + user_ps - 8), seed[2] = { 1, 2 }; + struct iommu_option option = { .size = sizeof(option), + .option_id = IOMMU_OPTION_HUGE_PAGES, + .op = IOMMU_OPTION_OP_SET, + .val64 = 0 }; + struct iommu_hwpt_alloc alloc = { + .size = sizeof(alloc), .flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING + }; + struct iommu_hwpt_set_dirty_tracking enable = { + .size = sizeof(enable), + .flags = IOMMU_HWPT_DIRTY_TRACKING_ENABLE + }; + struct iommu_test_cmd mark = { .size = sizeof(mark), + .op = IOMMU_TEST_OP_DIRTY, + .dirty = { .iova = IOVA, + .length = len, + .page_size = native_ps, + .uptr = (uintptr_t)seed } }; + unsigned int no_clear = IOMMU_HWPT_GET_DIRTY_BITMAP_NO_CLEAR; + int failures = 0, rc, error; + + CHECK(data != MAP_FAILED && out != MAP_FAILED); + CHECK(!context_open(&c, mode) && !limit_bytes(128 * native_ps)); + option.object_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_OPTION, &option)); + CHECK(!map_range(&c, data, len, IOVA)); + alloc.dev_id = c.dev[0]; + alloc.pt_id = c.ioas; + CHECK(!ioctl(c.fd, IOMMU_HWPT_ALLOC, &alloc)); + enable.hwpt_id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_HWPT_SET_DIRTY_TRACKING, &enable)); + mark.id = alloc.out_hwpt_id; + CHECK(!ioctl(c.fd, IOMMU_TEST_CMD, &mark) && + mark.dirty.out_nr_dirty == 2); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + CHECK(filled(out, 0, user_ps - 8) && + filled(out + user_ps + 8, 0, user_ps - 8)); + +#define EXPECT_DIRTY_ERROR(call, expected, label) \ + do { \ + errno = 0; \ + rc = (call); \ + error = errno; \ + if (rc != -1 || error != (expected)) { \ + printf("FAIL dirty %s: rc=%d errno=%d expected=%d\n", \ + label, rc, error, expected); \ + failures++; \ + } \ + } while (0) + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, + IOVA + len, native_ps, native_ps, + no_clear), + EINVAL, "unmapped IOVA"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, (void *)1, IOVA, + len, native_ps, no_clear), + EFAULT, "invalid output pointer"); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, + 4 * native_ps, 3 * native_ps, no_clear), + EINVAL, "non-power-of-two granule"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "read-only output"); + CHECK(!mprotect(out, 2 * user_ps, PROT_READ | PROT_WRITE)); + memset(out, 0, 2 * user_ps); + CHECK(!munmap(out + user_ps, user_ps)); + EXPECT_DIRTY_ERROR(dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, + native_ps, no_clear), + EFAULT, "output hole after valid prefix"); + CHECK(bitmap[0] == 1); + CHECK(mmap(out + user_ps, user_ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, + 0) == out + user_ps); + /* NO_CLEAR errors retain dirty state, so fixing the buffer permits retry. */ + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + 0)); + CHECK(bitmap[0] == 1 && bitmap[1] == 2); + memset(out, 0, 2 * user_ps); + CHECK(!dirty_ioctl(&c, alloc.out_hwpt_id, bitmap, IOVA, len, native_ps, + no_clear)); + CHECK(bitmap[0] == 0 && bitmap[1] == 0); + CHECK(!unmap_range(&c, len, IOVA)); + CHECK(!munmap(data, len) && !munmap(out, 2 * user_ps)); + context_close(&c); + CHECK(stat_kb(getpid(), "VmPin:") == 0 && + stat_kb(getpid(), "VmLck:") == 0); + CHECK(!failures); +#undef EXPECT_DIRTY_ERROR + return 0; +} + +static int exercise(void) +{ + CHECK(!drop_pin_cap()); + for (unsigned int mode = 0; mode < 2; mode++) { + CHECK(!limits_and_lifetime(mode)); + CHECK(!holes_and_shared(mode)); + CHECK(!queue_ranges(mode)); + CHECK(!page_ranges(mode)); + CHECK(!page_lifetime(mode)); + CHECK(!cross_process(mode)); + CHECK(!concurrent_maps(mode)); + CHECK(!allocation_failures(mode)); + CHECK(!queue_failure_sweep(mode)); + CHECK(!page_failure_sweep(mode)); + CHECK(!page_concurrency(mode)); + CHECK(!dirty_errors(mode)); + printf("ok - IOMMUFD %zuK userspace/%zuK kernel %s accounting, limits/lifetime/shared/cross-process/256 concurrent cycles\n", + user_ps / 1024, native_ps / 1024, mode ? "MM" : "USER"); + } + return 0; +} + +int main(int argc, char **argv) +{ + int fail = 0; + + setbuf(stdout, NULL); + if (argc >= 4) { + user_ps = strtoul(argv[2], NULL, 0); + native_ps = strtoul(argv[3], NULL, 0); + CHECK(getauxval(AT_PAGESZ) == user_ps); + if (!strcmp(argv[1], "--exercise")) + return exercise(); + if (!strcmp(argv[1], "--producer") && argc == 8) + return producer(argc, argv); + return 2; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + CHECK(!mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)); + native_ps = getauxval(AT_PAGESZ); + size_t sizes[] = { native_ps, 4096, 16384 }; + + for (unsigned int i = 0; i < (native_ps > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0, ok = 0; + + if (!child) { + char size[24], native[24]; + + snprintf(size, sizeof(size), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", native_ps); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", size, native, + NULL); + _exit(5); + } + for (unsigned int ticks = 0; child > 0 && ticks < 1800; + ticks++) { + if (waitpid(child, &status, WNOHANG) == child) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 1799) { + kill(child, SIGKILL); + waitpid(child, &status, 0); + } + usleep(100000); + } + printf("%s - IOMMUFD ABI %zuK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } + printf("IOMMUFD IOCTL CONTRACT %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c new file mode 100644 index 00000000000000..b652200c38b42c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/lazyfree-fixture-init.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static unsigned long stat_value(const char *path, const char *key) +{ + char line[256], name[100]; + unsigned long value = 0; + FILE *f = fopen(path, "re"); + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "%99s %lu", name, &value) == 2 && + !strcmp(name, key)) + break; + fclose(f); + return value; +} + +static bool all_bytes(const unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} + +static unsigned long lazy_kb(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "re"); + char line[256]; + unsigned long first, last, value = ~0UL; + bool found = false; + + if (!f) + return value; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &first, &last) == 2) { + if (found) + break; + found = (uintptr_t)address >= first && + (uintptr_t)address < last; + } else if (found && + sscanf(line, "LazyFree: %lu", &value) == 1) { + break; + } + } + fclose(f); + return value; +} + +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t entry = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + + if (fd < 0 || + pread(fd, &entry, sizeof(entry), (uintptr_t)p / ps * 8) != 8) + ksft_exit_fail_msg("pagemap: %s\n", strerror(errno)); + close(fd); + return entry; +} + +static bool pageout(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_COLD); + ok &= !madvise(p, len, MADV_PAGEOUT); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static bool mark_free(void *p, size_t len) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int i = 0; i < CPU_SETSIZE; i++) { + if (!CPU_ISSET(i, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(i, &one); + ok &= !sched_setaffinity(0, sizeof(one), &one); + ok &= !madvise(p, len, MADV_FREE); + ok &= !madvise(p, len, MADV_COLD); + } + return !sched_setaffinity(0, sizeof(allowed), &allowed) && ok; +} + +static int exercise(unsigned long native) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 4 * ps; + unsigned char *reservation = mmap(NULL, len + 2 * native, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *p = + (void *)(((uintptr_t)reservation + native - 1) & ~(native - 1)); + bool ok; + unsigned long writes; + uint64_t entry; + + if (reservation == MAP_FAILED || + mprotect(p, len, PROT_READ | PROT_WRITE)) + ksft_exit_fail_msg("mapping\n"); + ksft_print_header(); + ksft_set_plan(8); + for (unsigned int i = 0; i < 4; i++) + memset(p + i * ps, 0x40 + i, ps); + ok = mark_free(p + ps, ps); + if (!ok) + ksft_exit_fail_msg("%zuK MADV_FREE: %s\n", ps / 1024, + strerror(errno)); + ksft_test_result(all_bytes(p + ps, ps, 0x41), + "%zuK FREE retains bytes before reclaim\n", ps / 1024); + ksft_test_result(lazy_kb(p) == ps / 1024, + "%zuK partial LazyFree accounting\n", ps / 1024); + ok = pageout(p, len); + entry = pagemap(p + ps, ps); + ok &= !(entry & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0x40) && + all_bytes(p + 2 * ps, ps, 0x42) && + all_bytes(p + 3 * ps, ps, 0x43); + ksft_test_result(ok, "%zuK partial FREE reclaims only advised bytes\n", + ps / 1024); + /* Drop old swap/cache state, then test cancellation by a real EL0 write. */ + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x62, len); + ok &= mark_free(p + ps, ps); + memset(p + ps, 0xa5, ps); + ksft_test_result(lazy_kb(p) == 0, + "%zuK dirty leaf no longer reported LazyFree\n", + ps / 1024); + ok &= pageout(p, len); + ok &= all_bytes(p + ps, ps, 0xa5) && all_bytes(p, ps, 0x62) && + all_bytes(p + 2 * ps, 2 * ps, 0x62); + ksft_test_result(ok, "%zuK write after FREE cancels quarter discard\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0x83, len); + writes = stat_value("/proc/vmstat", "pswpout"); + ok &= mark_free(p, len) && pageout(p, len); + for (unsigned int i = 0; i < 4; i++) + ok &= !(pagemap(p + i * ps, ps) & + ((1ULL << 63) | (1ULL << 62))); + printf("# full FREE pswpout=%lu->%lu, entries=%#llx %#llx %#llx %#llx\n", + writes, stat_value("/proc/vmstat", "pswpout"), + (unsigned long long)pagemap(p, ps), + (unsigned long long)pagemap(p + ps, ps), + (unsigned long long)pagemap(p + 2 * ps, ps), + (unsigned long long)pagemap(p + 3 * ps, ps)); + ok &= stat_value("/proc/vmstat", "pswpout") == writes && + all_bytes(p, len, 0); + ksft_test_result( + ok, "%zuK fully free backing reclaimed without swap writes\n", + ps / 1024); + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xb7, len); + ok &= pageout(p, len); + ok &= !!(pagemap(p + ps, ps) & (1ULL << 62)); + ok &= mark_free(p + ps, ps); + ok &= !(pagemap(p + ps, ps) & ((1ULL << 63) | (1ULL << 62))); + ok &= all_bytes(p + ps, ps, 0) && all_bytes(p, ps, 0xb7) && + all_bytes(p + 2 * ps, 2 * ps, 0xb7); + ksft_test_result( + ok, + "%zuK FREE drops selected swap entry without neighbor loss\n", + ps / 1024); + { + int gate[2], status = 0; + pid_t pid; + char ready; + + ok = !madvise(p, len, MADV_DONTNEED); + memset(p, 0xc8, len); + if (pipe(gate)) + ksft_exit_fail_msg("pipe\n"); + pid = fork(); + if (!pid) { + close(gate[1]); + if (read(gate[0], &ready, 1) != 1 || + !all_bytes(p, len, 0xc8)) + _exit(2); + _exit(0); + } + close(gate[0]); + ok &= mark_free(p + ps, ps) && pageout(p, len); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "%zuK FREE after fork preserves other process data\n", + ps / 1024); + } + munmap(reservation, len + 2 * native); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(strtoul(argv[3], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + swapon("/dev/vda", 0)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - lazyfree fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c new file mode 100644 index 00000000000000..f34602bb730e5c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-accounting-init.c @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/account" +#define LENGTH (32UL * 1024 * 1024) +static int put(const char *p, const char *v) +{ + int fd = open(p, O_WRONLY); + if (fd < 0) + return 1; + int bad = write(fd, v, strlen(v)) != (ssize_t)strlen(v); + close(fd); + return bad; +} +static unsigned long long stat_value(const char *key) +{ + FILE *f = fopen(CG "/memory.stat", "r"); + char name[128]; + unsigned long long value = 0; + if (!f) + exit(20); + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + fclose(f); + return value; + } + fclose(f); + exit(21); +} +static int worker(unsigned long expected, int ready, int release) +{ + if (getauxval(AT_PAGESZ) != expected) + return 2; + unsigned char *p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || madvise(p, LENGTH, MADV_NOHUGEPAGE)) + return 3; + memset(p, 0x71, LENGTH); + char token = 'R'; + if (write(ready, &token, 1) != 1 || read(release, &token, 1) != 1) + return 4; + /* Keep the writes observable independently of the accounting check. */ + for (size_t i = 0; i < LENGTH; i++) + if (p[i] != 0x71) + return 5; + return munmap(p, LENGTH) ? 6 : 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (getppid() == 1 && argc == 5 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), atoi(argv[3]), + atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(60); + int failed = 0; + unsigned long long slab[3] = {}; + unsigned long sizes[3] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int modes = sizes[0] == 65536 ? 3 : 2; + if ((sizes[0] != 16384 && sizes[0] != 65536) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory")) { + perror("setup"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < modes; i++) { + int ready[2], release[2]; + if (mkdir(CG, 0755) || put(CG "/memory.max", "134217728") || + pipe(ready) || pipe(release)) { + failed = 1; + break; + } + pid_t child = fork(); + if (!child) { + char ps[32], rfd[16], cfd[16]; + close(ready[0]); + close(release[1]); + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(rfd, sizeof(rfd), "%d", ready[1]); + snprintf(cfd, sizeof(cfd), "%d", release[0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--worker", ps, rfd, cfd, NULL); + _exit(8); + } + close(ready[1]); + close(release[0]); + char token; + int ok = child > 0 && read(ready[0], &token, 1) == 1; + if (ok) { + /* Let per-CPU slab statistics become visible before the comparison. */ + usleep(1200000); + slab[i] = stat_value("slab_unreclaimable"); + printf("# ABI=%lu anon=%llu slab_unreclaimable=%llu pagetables=%llu\n", + sizes[i], stat_value("anon"), slab[i], + stat_value("pagetables")); + ok = stat_value("anon") >= LENGTH && + write(release[1], "G", 1) == 1; + } + int status = 0; + if (child > 0 && !ok) + kill(child, SIGKILL); + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && ok; + close(ready[0]); + close(release[1]); + printf("%s - memcg accounting ABI %luK worker status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + for (unsigned retry = 0; retry < 100 && stat_value("anon"); + retry++) + usleep(20000); + if (stat_value("anon") || rmdir(CG)) + failed = 1; + if (failed) + break; + } + /* Slot metadata must be charged for every alternative granule. The + * loose lower bound tolerates slab/KASAN layout differences. */ + for (unsigned i = 1; !failed && i < modes; i++) { + if (slab[i] < slab[0] + 512 * 1024) { + printf("not ok - slot metadata escaped cgroup accounting ABI=%lu native=%llu alternative=%llu\n", + sizes[i], slab[0], slab[i]); + failed = 1; + } + } + +done: + printf("GRANULE MEMCG ACCOUNTING %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c new file mode 100644 index 00000000000000..c2f50993ff805e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memcg-pressure-init.c @@ -0,0 +1,417 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define MIB (1024UL * 1024) +#define LENGTH (32 * MIB) +#define ROUNDS 4 + +static int put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + ssize_t ret = write(fd, value, strlen(value)); + int saved = errno; + close(fd); + errno = saved; + return ret == (ssize_t)strlen(value) ? 0 : -1; +} + +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + char name[128]; + unsigned long long value = 0, result = 0; + if (!f) { + perror(path); + exit(10); + } + if (!key) { + if (fscanf(f, "%llu", &result) != 1) + exit(11); + } else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(name, key)) { + result = value; + break; + } + fclose(f); + return result; +} + +static unsigned char pattern(size_t offset, unsigned seed, unsigned round) +{ + return (offset ^ (offset >> 12) ^ (offset >> 20) ^ (seed * 79) ^ + (round * 43)) & + 255; +} + +static int contents(unsigned char *p, size_t ps, unsigned seed, unsigned round, + int write_data, int child) +{ + size_t pages = LENGTH / ps; + /* Odd multiplication permutes every logical page of this power-of-two range. */ + for (size_t i = 0; i < pages; i++) { + size_t page = (i * 4051 + round * 31) & (pages - 1); + for (size_t j = 0; j < ps; j++) { + size_t offset = page * ps + j; + unsigned char want = pattern(offset, seed, round); + if (child && !(page & 3)) + want ^= 0x69; + if (write_data) + p[offset] = want; + else if (p[offset] != want) { + printf("not ok - ABI=%zu round=%u child=%d byte=%zu got=%u expected=%u\n", + ps, round, child, offset, p[offset], + want); + return 1; + } + } + } + return 0; +} + +static int barrier(int ready, int command) +{ + char token = 'R'; + return write(ready, &token, 1) != 1 || read(command, &token, 1) != 1 || + token != 'G'; +} + +static int worker(unsigned long expected, unsigned seed, int ready, int command) +{ + size_t ps = getauxval(AT_PAGESZ); + unsigned char *p; + if (ps != expected) + return 2; + p = mmap(NULL, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 3; + if (madvise(p, LENGTH, MADV_NOHUGEPAGE)) { + int error = errno; + struct stat st; + + /* Without THP this advice is unavailable and cannot change policy. */ + if (error != EINVAL || + !stat("/sys/kernel/mm/transparent_hugepage/enabled", &st) || + errno != ENOENT) + return 3; + } + if (contents(p, ps, seed, 0, 1, 0) || barrier(ready, command)) + return 4; + for (unsigned round = 0; round < ROUNDS; round++) { + if (contents(p, ps, seed, round, 0, 0)) + return 5; + pid_t child = fork(); + if (child < 0) + return 6; + if (!child) { + /* Dirty a quarter of the logical pages, retaining untouched siblings. */ + for (size_t page = 0; page < LENGTH / ps; page += 4) + for (size_t j = 0; j < ps; j++) + p[page * ps + j] ^= 0x69; + _exit(contents(p, ps, seed, round, 0, 1) ? 7 : 0); + } + int status; + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status) || contents(p, ps, seed, round, 0, 0)) + return 8; + printf("ok - memcg ABI %zuK round=%u swap-in bytes and fork COW isolation\n", + ps / 1024, round); + if (round + 1 < ROUNDS && + contents(p, ps, seed, round + 1, 1, 0)) + return 9; + if (barrier(ready, command)) + return 10; + } + return munmap(p, LENGTH) ? 11 : 0; +} + +static void report(unsigned phase) +{ + printf("# memcg phase=%u resident=%llu swap=%llu anon=%llu pgscan=%llu pgsteal=%llu max=%llu oom=%llu oom_kill=%llu\n", + phase, get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "anon"), get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); +} + +static void pressure_stacks(void) +{ + FILE *tasks = fopen(CG "/cgroup.procs", "r"); + int pid; + + if (!tasks) + return; + while (fscanf(tasks, "%d", &pid) == 1) { + char path[64], stack[4096]; + int fd; + ssize_t n; + + snprintf(path, sizeof(path), "/proc/%d/stack", pid); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + continue; + n = read(fd, stack, sizeof(stack) - 1); + close(fd); + if (n > 0) { + stack[n] = 0; + printf("# pressure stack pid=%d\n%s", pid, stack); + } + } + fclose(tasks); +} + +static void observe_pressure(void) +{ + unsigned int samples = 0; + + for (;;) { + struct timespec now; + + clock_gettime(CLOCK_MONOTONIC, &now); + printf("# io-pressure ms=%lld resident=%llu swap=%llu writeback=%llu " + "swapcached=%llu scan=%llu steal=%llu max=%llu oom=%llu kill=%llu\n", + (long long)now.tv_sec * 1000 + now.tv_nsec / 1000000, + get(CG "/memory.current", NULL), + get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback"), + get(CG "/memory.stat", "swapcached"), + get(CG "/memory.stat", "pgscan"), + get(CG "/memory.stat", "pgsteal"), + get(CG "/memory.events", "max"), + get(CG "/memory.events", "oom"), + get(CG "/memory.events", "oom_kill")); + if (!(++samples % 8) && + get(CG "/memory.stat", "file_writeback") > 8 * MIB) + pressure_stacks(); + usleep(250000); + } +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker") && getppid() == 1) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + bool classic = false, native_control = false, user16k = false; + + if (getpid() != 1 || argc > 3) { + fprintf(stderr, "Dedicated disposable VM PID1 only\n"); + return 2; + } + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--classic")) + classic = true; + else if (!strcmp(argv[i], "--native-control")) + native_control = true; + else if (!strcmp(argv[i], "--user16k")) + user16k = true; + else + return 2; + } + if (native_control && user16k) + return 2; + signal(SIGPIPE, SIG_IGN); + alarm(300); + int failed = 0, active = 0; + pid_t children[2] = { -1, -1 }, observer = -1; + int ready[2][2], command[2][2]; + unsigned long sizes[2] = { getauxval(AT_PAGESZ), 4096 }; + if (native_control) + sizes[1] = sizes[0]; + if (user16k) + sizes[1] = 16384; + if ((sizes[0] != 16384 && sizes[0] != 65536) || sizes[1] > sizes[0]) { + failed = 1; + goto done; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) || + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory") || + mkdir(CG, 0755) || put(CG "/memory.max", "50331648") || + put(CG "/memory.swap.max", "268435456")) { + perror("cgroup setup"); + failed = 1; + goto done; + } + /* Explicit diagnostic control, never change the production default. */ + if (classic && put("/sys/kernel/mm/lru_gen/enabled", "0")) { + perror("classic reclaim control"); + failed = 1; + goto done; + } + printf("# reclaim mode=%s\n", + classic ? "classic diagnostic" : "default"); + struct stat st; + for (int i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (swapon("/dev/vda", 0)) { + perror("swapon"); + failed = 1; + goto done; + } + active = 1; + for (unsigned i = 0; i < 2; i++) + if (pipe(ready[i]) || pipe(command[i])) { + failed = 1; + goto done; + } + observer = fork(); + if (!observer) { + for (unsigned int i = 0; i < 2; i++) { + close(ready[i][0]); + close(ready[i][1]); + close(command[i][0]); + close(command[i][1]); + } + observe_pressure(); + _exit(0); + } + if (observer < 0) { + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32], index[16], rfd[16], cfd[16]; + for (unsigned j = 0; j < 2; j++) { + close(ready[j][0]); + close(command[j][1]); + if (i != j) { + close(ready[j][1]); + close(command[j][0]); + } + } + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(rfd, sizeof(rfd), "%d", ready[i][1]); + snprintf(cfd, sizeof(cfd), "%d", command[i][0]); + if (put(CG "/cgroup.procs", "0") || + prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(12); + execl("/init", "/init", "--worker", ps, index, rfd, cfd, + NULL); + _exit(13); + } + if (children[i] < 0) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + close(ready[i][1]); + close(command[i][0]); + } + for (unsigned phase = 0; phase <= ROUNDS; phase++) { + char token; + for (unsigned i = 0; i < 2; i++) + if (read(ready[i][0], &token, 1) != 1 || token != 'R') { + failed = 1; + goto done; + } + report(phase); + if (!get(CG "/memory.swap.current", NULL) || + !get(CG "/memory.events", "max") || + !get(CG "/memory.stat", "pgscan") || + !get(CG "/memory.stat", "pgsteal") || + get(CG "/memory.events", "oom") || + get(CG "/memory.events", "oom_kill")) { + printf("not ok - missing pressure evidence or unexpected OOM\n"); + failed = 1; + goto done; + } + for (unsigned i = 0; i < 2; i++) + if (write(command[i][1], "G", 1) != 1) { + failed = 1; + goto done; + } + } + for (unsigned i = 0; i < 2; i++) { + int status = 0; + int ok = waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - memcg ABI %luK exit=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + children[i] = -1; + failed |= !ok; + } + /* Pending swap writeback can outlive exit, especially on a slow device. */ + for (unsigned i = 0; i < 3000 && (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)); + i++) { + if (!(i % 250)) + printf("# exit drain ms=%u swap=%llu writeback=%llu\n", + i * 20, get(CG "/memory.swap.current", NULL), + get(CG "/memory.stat", "file_writeback")); + /* Exit may leave clean, unmapped swap cache after writeback. Reclaim + * it explicitly; EAGAIN means fewer than the requested bytes existed. */ + if (!get(CG "/memory.stat", "file_writeback") && + put(CG "/memory.reclaim", "67108864 swappiness=200") && + errno != EAGAIN) { + perror("exit cache reclaim"); + failed = 1; + break; + } + usleep(20000); + } + report(ROUNDS + 1); + if (get(CG "/memory.stat", "anon") || + get(CG "/memory.swap.current", NULL)) { + printf("not ok - anonymous or swap charge remains after exit\n"); + failed = 1; + } +done: + if (observer > 0) { + kill(observer, SIGTERM); + waitpid(observer, NULL, 0); + } + if (active && failed) + report(ROUNDS + 2); + for (unsigned i = 0; i < 2; i++) + if (children[i] > 0) { + kill(children[i], SIGKILL); + waitpid(children[i], NULL, 0); + } + if (active && failed) { + put(CG "/cgroup.kill", "1"); + put(CG "/memory.max", "max"); + } + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + failed = 1; + } + printf("GRANULE MEMCG PRESSURE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/memory-interfaces.c b/tools/testing/selftests/arm64/user4k/memory-interfaces.c new file mode 100644 index 00000000000000..330296ed44499d --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/memory-interfaces.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static size_t page; + +static void residency(void) +{ + unsigned char *p, vec[6]; + bool ok; + unsigned int i; + + p = mmap(NULL, 5 * page, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap\n"); + memset(vec, 0x55, sizeof(vec)); + ok = mincore(p, 5 * page - 1, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == 0; + ok &= vec[5] == 0x55; + ksft_test_result( + ok, "mincore emits exactly one byte per unfaulted user page\n"); + ok = mprotect(p + 2 * page, page, PROT_READ | PROT_WRITE) == 0; + p[2 * page] = 0x37; + ok &= mincore(p, 5 * page, vec) == 0; + for (i = 0; i < 5; i++) + ok &= vec[i] == (i == 2); + ksft_test_result( + ok, + "mincore reports a separately faulted VMA among cold neighbors\n"); + ok = munmap(p + 3 * page, page) == 0; + errno = 0; + ok &= mincore(p, 5 * page, vec) == -1 && errno == ENOMEM; + errno = 0; + ok &= mincore(p + 1, page, vec) == -1 && errno == EINVAL; + ksft_test_result(ok, + "mincore rejects an unaligned start and a VMA hole\n"); + munmap(p, 5 * page); +} + +static void file_residency(void) +{ + const size_t native = 16384, count = 3 * native / page; + unsigned char *p, *guard, *vec, byte = 0x7b; + int fd = memfd_create("residency", 0); + bool ok; + size_t i; + + if (fd < 0 || ftruncate(fd, 4 * native) || + pwrite(fd, &byte, 1, native) != 1) + ksft_exit_fail_msg("memfd setup\n"); + p = mmap(NULL, 3 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page); + guard = mmap(NULL, 2 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || guard == MAP_FAILED || + mprotect(guard + page, page, PROT_NONE)) + ksft_exit_fail_msg("file/vector mmap\n"); + vec = guard + page - count; + memset(vec, 0x55, count); + ok = mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "unmapped file leaves share residency at native cache indices\n"); + ok = p[native - page] == byte && mincore(p, 3 * native, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == ((page + i * page) / native == 1); + ksft_test_result( + ok, + "present and cache-only file residency agree at a guarded vector boundary\n"); + ok = msync(p + page, 1, MS_SYNC) == 0; + errno = 0; + ok &= msync(p + page + 1, page, MS_SYNC) == -1 && errno == EINVAL; + ok &= munmap(p + page, page) == 0; + errno = 0; + ok &= msync(p, 3 * page, MS_SYNC) == -1 && errno == ENOMEM; + ksft_test_result(ok, + "msync uses process alignment and reports holes\n"); + munmap(p, 3 * native); + munmap(guard, 2 * page); + close(fd); +} + +static void large_vector(void) +{ + const size_t count = 16384 + 3; + unsigned char *p = mmap(NULL, count * page, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, + 0); + unsigned char *vec = malloc(count + 1); + bool ok; + size_t i; + + if (p == MAP_FAILED || !vec) + ksft_exit_fail_msg("large vector allocation\n"); + memset(vec, 0x55, count + 1); + ok = mincore(p, count * page - 7, vec) == 0; + for (i = 0; i < count; i++) + ok &= vec[i] == 0; + ok &= vec[count] == 0x55; + ksft_test_result( + ok, "mincore advances across multiple native output buffers\n"); + free(vec); + munmap(p, count * page); +} + +static void seals(void) +{ + unsigned char *p = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t pid; + int status = 0; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("seal mmap\n"); + ok = syscall(SYS_mseal, p + page, 1UL, 0UL) == 0; + errno = 0; + ok &= mprotect(p + page, page, PROT_READ) == -1 && errno == EPERM; + errno = 0; + ok &= munmap(p + page, page) == -1 && errno == EPERM; + ok &= mprotect(p, page, PROT_READ) == 0; + ok &= mprotect(p + 2 * page, page, PROT_READ) == 0; + ksft_test_result( + ok, + "a one-byte seal covers only its user page and protects it\n"); + pid = fork(); + if (!pid) + _exit(!(munmap(p + page, page) == -1 && errno == EPERM)); + ok = pid > 0 && waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + ksft_test_result(ok, "fork inherits the sealed user page\n"); + munmap(p, page); + munmap(p + 2 * page, page); + /* The sealed VMA is released by process exit. */ +} + +int main(int argc, char **argv) +{ + page = getauxval(AT_PAGESZ); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + ksft_exit_fail_msg("unexpected process page ABI: %zu\n", page); + ksft_print_msg("process page size: %zu\n", page); + ksft_print_header(); + ksft_set_plan(9); + residency(); + file_residency(); + large_vector(); + seals(); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/mglru-granule-init.c b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c new file mode 100644 index 00000000000000..9bb9bb74e38e86 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mglru-granule-init.c @@ -0,0 +1,365 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CONTROL "/sys/kernel/debug/lru_gen" +#define FULL "/sys/kernel/debug/lru_gen_full" +static int ready_fd = -1, command_fd = -1; +static unsigned worker_index; +static unsigned char pattern(unsigned region, size_t byte, unsigned round) +{ + return (byte ^ (byte >> 12) ^ (region * 37) ^ (round * 53)) & 255; +} +static int generation(unsigned long long *cg, unsigned *node, + unsigned long *max) +{ + FILE *f = fopen(CONTROL, "r"); + char line[256]; + int have_cg = 0, have_node = 0, have_seq = 0; + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) { + unsigned long long id; + unsigned n; + unsigned long seq, age, anon, file; + if (sscanf(line, "memcg %llu", &id) == 1) { + if (have_cg) + break; + *cg = id; + have_cg = 1; + } else if (sscanf(line, " node %u", &n) == 1) { + if (have_node) + break; + *node = n; + have_node = 1; + } else if (have_node && sscanf(line, "%lu %lu %lu %lu", &seq, + &age, &anon, &file) == 4) { + *max = seq; + have_seq = 1; + } + } + fclose(f); + return have_seq ? 0 : -1; +} +static unsigned long scan_evidence(void) +{ + FILE *f = fopen(FULL, "r"); + char line[512]; + unsigned long total = 0; + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) { + unsigned long t, y, found, added; + if (sscanf(line, " %lut %luy %luf %lua", &t, &y, &found, + &added) == 4) { + printf("# MGLRU scan total=%lu young=%lu tables=%lu added=%lu\n", + t, y, found, added); + total += y; + } + } + fclose(f); + return total; +} +static int age(void) +{ + if (ready_fd >= 0) { + char token = 'A'; + return write(ready_fd, &token, 1) != 1 || + read(command_fd, &token, 1) != 1 || token != 'G'; + } + unsigned long long cg = 0, next_cg = 0; + unsigned node = 0, next_node = 0; + unsigned long seq = 0, next = 0; + char command[128]; + if (generation(&cg, &node, &seq)) + return 1; + /* This tree accepts swappiness here; 200 scans both anonymous and file pages. */ + int len = snprintf(command, sizeof(command), "+ %llu %u %lu 200 1\n", + cg, node, seq); + int fd = open(CONTROL, O_WRONLY); + if (fd < 0) + return 1; + ssize_t written = write(fd, command, len); + close(fd); + if (written != len || generation(&next_cg, &next_node, &next) || + next <= seq) { + printf("not ok - MGLRU generation command=%s result=%zd errno=%d next=%lu\n", + command, written, errno, next); + return 1; + } + printf("ok - MGLRU generation %lu -> %lu\n", seq, next); + return 0; +} +static int check_referenced(unsigned char **regions, unsigned nr, + size_t expected) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char line[512]; + int region = -1, failed = 0; + unsigned seen = 0; + unsigned long start, end, kb; + if (!f) + return 1; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + region = -1; + for (unsigned i = 0; i < nr; i++) + if (start == (uintptr_t)regions[i]) + region = i; + } else if (region >= 0 && + sscanf(line, "Referenced: %lu kB", &kb) == 1) { + printf("%s - region=%d start=%p referenced=%lu expected=%zu kB\n", + kb == expected ? "ok" : "not ok", region, + regions[region], kb, expected); + failed |= kb != expected; + seen++; + } + } + fclose(f); + return failed || seen != nr; +} +static int exercise(unsigned long expected) +{ + static const unsigned shifts[] = { 21, 25, 29, 30, 36, 39, 42, 47 }; + unsigned char *regions[sizeof(shifts) / sizeof(shifts[0])] = {}; + size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned nr = sizeof(shifts) / sizeof(shifts[0]); + int fail = 0, fd = memfd_create("granule-aging", 0); + cpu_set_t cpus; + void *scratch = MAP_FAILED; + CPU_ZERO(&cpus); + CPU_SET(ready_fd >= 0 ? worker_index % 2 : 0, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus)) + return 1; + if (ps != expected || fd < 0 || ftruncate(fd, nr * len)) + return 1; + for (unsigned i = 0; i < nr; i++) { + uintptr_t address = (1UL << shifts[i]) - 4 * ps; + regions[i] = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (i % 2 ? MAP_SHARED : + MAP_PRIVATE | MAP_ANONYMOUS), + i % 2 ? fd : -1, i % 2 ? i * len : 0); + if (regions[i] != (void *)address) { + printf("not ok - %zuK boundary %u mmap errno=%d\n", + ps / 1024, shifts[i], errno); + fail = 1; + goto out; + } + } + for (unsigned i = 0; i < nr; i++) + memset(regions[i], 0x19, len); + /* Publish earlier folios from the bounded per-CPU LRU-add batch. */ + scratch = mmap(NULL, 8UL << 20, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (scratch == MAP_FAILED) { + fail = 1; + goto out; + } + memset(scratch, 0x7b, 8UL << 20); + /* Seed MGLRU's first-access referenced state before testing the next access. */ + if (age()) { + fail = 1; + goto out; + } + for (unsigned round = 0; round < 4; round++) { + if (ready_fd >= 0) { + unsigned cpu = (worker_index + round) % 2; + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + if (sched_setaffinity(0, sizeof(cpus), &cpus) || + sched_getcpu() != (int)cpu) { + fail = 1; + break; + } + printf("# concurrent ABI=%zuK round=%u cpu=%u\n", + ps / 1024, round, cpu); + } + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + regions[i][j] = pattern(i, j, round); + fail |= check_referenced(regions, nr, len / 1024); + if (age() || age()) { + fail = 1; + break; + } + fail |= check_referenced(regions, nr, 0); + int good = 1; + for (unsigned i = 0; i < nr; i++) + for (size_t j = 0; j < len; j++) + if (regions[i][j] != pattern(i, j, round)) { + good = 0; + break; + } + unsigned long young = scan_evidence(); + printf("%s - %zuK round%u eight table boundaries bytes and aging young=%lu\n", + good && young ? "ok" : "not ok", ps / 1024, round, + young); + fail |= !good || !young; + } +out: + if (scratch != MAP_FAILED) + munmap(scratch, 8UL << 20); + for (unsigned i = 0; i < nr; i++) + if (regions[i] && regions[i] != MAP_FAILED) + munmap(regions[i], len); + close(fd); + return fail; +} +static int concurrent(unsigned long *sizes, unsigned nr) +{ + int ready[3][2], commands[3][2], failed = 0; + pid_t workers[3]; + + for (unsigned i = 0; i < nr; i++) + if (pipe(ready[i]) || pipe(commands[i])) + return 1; + for (unsigned i = 0; i < nr; i++) { + workers[i] = fork(); + if (!workers[i]) { + char size[32], index[16], ready_arg[16], + command_arg[16]; + for (unsigned j = 0; j < nr; j++) { + close(ready[j][0]); + close(commands[j][1]); + if (j != i) { + close(ready[j][1]); + close(commands[j][0]); + } + } + snprintf(size, sizeof(size), "%lu", sizes[i]); + snprintf(index, sizeof(index), "%u", i); + snprintf(ready_arg, sizeof(ready_arg), "%d", + ready[i][1]); + snprintf(command_arg, sizeof(command_arg), "%d", + commands[i][0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--worker", size, index, + ready_arg, command_arg, NULL); + _exit(4); + } + if (workers[i] < 0) + failed = 1; + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][1]); + close(commands[i][0]); + } + /* One warm-up barrier plus two aging barriers for each of four rounds. */ + for (unsigned phase = 0; phase < 9 && !failed; phase++) { + for (unsigned i = 0; i < nr; i++) { + char token; + if (read(ready[i][0], &token, 1) != 1 || token != 'A') + failed = 1; + } + if (failed || age()) { + failed = 1; + break; + } + for (unsigned i = 0; i < nr; i++) { + char token = 'G'; + if (write(commands[i][1], &token, 1) != 1) + failed = 1; + } + } + for (unsigned i = 0; i < nr; i++) { + close(ready[i][0]); + close(commands[i][1]); + } + for (unsigned i = 0; i < nr; i++) { + int status = 0; + int ok = workers[i] > 0 && + waitpid(workers[i], &status, 0) == workers[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - concurrent MGLRU ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + return failed; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (argc == 6 && !strcmp(argv[1], "--worker")) { + if (getppid() != 1) + return 2; + worker_index = strtoul(argv[3], NULL, 10); + ready_fd = atoi(argv[4]); + command_fd = atoi(argv[5]); + return exercise(strtoul(argv[2], NULL, 10)); + } + int concurrent_mode = argc == 2 && !strcmp(argv[1], "--concurrent"); + if (getpid() != 1 || (argc != 1 && !concurrent_mode)) + return 2; + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + FILE *f = fopen("/sys/kernel/mm/lru_gen/enabled", "r"); + unsigned enabled = 0; + if (f) { + fscanf(f, "%x", &enabled); + fclose(f); + } + if ((enabled & 3) != 3) { + printf("not ok - MGLRU/MM walk disabled %#x\n", enabled); + fail = 1; + goto done; + } + printf("ok - MGLRU enabled=%#x\n", enabled); + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + if (concurrent_mode) { + fail = concurrent(sizes, sizes[0] > 16384 ? 3 : 2); + goto done; + } + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t child = fork(); + int status = 0; + if (!child) { + char value[32]; + snprintf(value, sizeof(value), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--exercise", value, NULL); + _exit(4); + } + int good = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - MGLRU ABI %luK status=%#x\n", + good ? "ok" : "not ok", sizes[i] / 1024, status); + fail |= !good; + } +done: + printf("GRANULE MGLRU%s %s\n", concurrent_mode ? " CONCURRENT" : "", + fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/mixed-process.c b/tools/testing/selftests/arm64/user4k/mixed-process.c new file mode 100644 index 00000000000000..6d08ab55991b5c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mixed-process.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real EL0 mixed-granule sharing, futex queues, and process_vm copies. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define NATIVE 16384U +#define QUARTER 4096U +#define ROUNDS 2000U +struct channel { + atomic_int turn; + unsigned int value; + unsigned int worker_wakes; +}; +struct ready { + uintptr_t base; + unsigned long page; +}; + +static int wait_turn(struct channel *channel, int wanted) +{ + const struct timespec timeout = { .tv_sec = 5 }; + int value; + + while ((value = atomic_load_explicit(&channel->turn, + memory_order_acquire)) != wanted) { + if (syscall(SYS_futex, &channel->turn, FUTEX_WAIT, value, + &timeout, NULL, 0) < 0 && + errno != EAGAIN && errno != EINTR) + return -1; + } + return 0; +} + +static int send_turn(struct channel *channel, int turn) +{ + atomic_store_explicit(&channel->turn, turn, memory_order_release); + return syscall(SYS_futex, &channel->turn, FUTEX_WAKE, 1, NULL, NULL, 0); +} + +static unsigned char pattern(size_t i) +{ + return (i ^ (i >> 8) ^ (i >> 16)) & 255; +} + +static int worker(int fd, int ready_fd, unsigned long expected) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *mapping, *remote; + struct channel *channel; + struct ready ready; + size_t i, begin = page - 13, end = 3 * page + 16; + unsigned int round, wakes = 0; + int ret; + + if (page != expected) + return 10; + mapping = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == QUARTER ? QUARTER : 0); + remote = mmap(NULL, 5 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED || remote == MAP_FAILED) + return 11; + channel = (void *)(mapping + (page == QUARTER ? 0 : QUARTER)); + for (i = 0; i < 4 * page; i++) + remote[i] = pattern(i); + if (mprotect(remote + 4 * page, page, PROT_NONE)) + return 12; + ready = (struct ready){ (uintptr_t)remote, page }; + if (write(ready_fd, &ready, sizeof(ready)) != sizeof(ready)) + return 13; + close(ready_fd); + for (round = 0; round < ROUNDS; round++) { + if (wait_turn(channel, 1) || + channel->value != (round ^ 0xabcddcbaU)) + return 14; + if (!round) { + for (i = 0; i < 4 * page; i++) + if (remote[i] != + (pattern(i) ^ + (i >= begin && i < end ? 0x5a : 0))) + return 15; + } + channel->value = ~(round ^ 0xabcddcbaU); + ret = send_turn(channel, 0); + if (ret < 0) + return 16; + wakes += ret; + } + channel->worker_wakes = wakes; + return 0; +} + +static bool remote_copies(pid_t pid, const struct ready *ready) +{ + size_t begin = ready->page - 13, len = 2 * ready->page + 29, i; + unsigned char *data = malloc(len), edge[2] = {}; + struct iovec local = { data, len }, + remote = { (void *)(ready->base + begin), len }; + bool ok = data != NULL; + + if (!ok) + return false; + ok = process_vm_readv(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + for (i = 0; ok && i < len; i++) + ok = data[i] == pattern(i + begin); + for (i = 0; i < len; i++) + data[i] = pattern(i + begin) ^ 0x5a; + ok &= process_vm_writev(pid, &local, 1, &remote, 1, 0) == (ssize_t)len; + local = (struct iovec){ edge, sizeof(edge) }; + remote = (struct iovec){ (void *)(ready->base + 4 * ready->page - 1), + sizeof(edge) }; + ok &= process_vm_readv(pid, &local, 1, &remote, 1, 0) == 1; + ok &= edge[0] == pattern(4 * ready->page - 1); + free(data); + return ok; +} + +int main(int argc, char **argv) +{ + unsigned long page = getauxval(AT_PAGESZ), other; + unsigned char *mapping, *private; + struct channel *channel; + struct ready ready = {}; + int fd, pipefd[2], status = 0, wakes = 0, ret; + unsigned int round = 0; + pid_t pid; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10)); + if (argc != 1 || (page != QUARTER && page != NATIVE)) + return 2; + other = page == NATIVE ? QUARTER : NATIVE; + fd = memfd_create("mixed-process", 0); + if (fd < 0 || ftruncate(fd, 2 * NATIVE) || pipe(pipefd)) + return 3; + mapping = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + private = + mmap(NULL, NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + if (mapping == MAP_FAILED || private == MAP_FAILED) + return 4; + channel = (void *)(mapping + QUARTER); + private[QUARTER] = 0x7e; + pid = fork(); + if (!pid) { + char fdarg[32], pipearg[32], sizearg[32]; + + close(pipefd[0]); + snprintf(fdarg, sizeof(fdarg), "%d", fd); + snprintf(pipearg, sizeof(pipearg), "%d", pipefd[1]); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(20); + execl("/mixed-process", "/mixed-process", "--worker", fdarg, + pipearg, sizearg, NULL); + _exit(21); + } + close(pipefd[1]); + if (pid < 0) + return 5; + ok = read(pipefd[0], &ready, sizeof(ready)) == sizeof(ready) && + ready.page == other; + close(pipefd[0]); + ok = ok && remote_copies(pid, &ready); + printf("%s - process_vm copies %lu -> %lu, including partial inaccessible boundary\n", + ok ? "ok" : "not ok", page, other); + for (round = 0; ok && round < ROUNDS; round++) { + channel->value = round ^ 0xabcddcbaU; + ret = send_turn(channel, 1); + ok = ret >= 0 && !wait_turn(channel, 0) && + channel->value == ~(round ^ 0xabcddcbaU); + if (ret > 0) + wakes += ret; + } + if (!ok) + kill(pid, SIGKILL); + ret = waitpid(pid, &status, 0); + ok &= ret == pid && WIFEXITED(status) && !WEXITSTATUS(status); + ok &= wakes > 0 && channel->worker_wakes > 0 && + private[QUARTER] == 0x7e; + ok &= mapping[0] == 0 && mapping[2 * QUARTER] == 0 && + mapping[NATIVE - 1] == 0; + printf("%s - mixed %lu/%lu shared futex/COW: rounds=%u wakes=%d/%u child=%#x\n", + ok ? "ok" : "not ok", page, other, round, wakes, + channel->worker_wakes, status); + munmap(private, NATIVE); + munmap(mapping, 2 * NATIVE); + close(fd); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c new file mode 100644 index 00000000000000..0dc94eabcc4fbf --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mlock-fixture-init.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static long locked_kb(void) +{ + char line[256]; + long value = -1; + FILE *f = fopen("/proc/self/status", "re"); + + if (!f) + return -1; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmLck: %ld", &value) == 1) + break; + fclose(f); + return value; +} + +static bool wait_ok(pid_t pid) +{ + int status = 0; + + return pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool remap_locked(unsigned long ps) +{ + unsigned char *p, *q, *target; + unsigned long kb = ps / 1024; + bool ok; + + p = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || mlock(p, 2 * ps)) + ksft_exit_fail_msg("mlock remap setup\n"); + memset(p, 0x37, 2 * ps); + ok = locked_kb() == (long)(2 * kb); + q = mremap(p, 2 * ps, 4 * ps, MREMAP_MAYMOVE); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap grow: %s\n", strerror(errno)); + ok &= locked_kb() == (long)(4 * kb); + for (unsigned long i = 0; i < 4 * ps; i++) + ok &= q[i] == (i < 2 * ps ? 0x37 : 0); + p = mremap(q, 4 * ps, 3 * ps, 0); + if (p == MAP_FAILED) + ksft_exit_fail_msg("locked remap shrink: %s\n", + strerror(errno)); + ok &= locked_kb() == (long)(3 * kb); + p[2 * ps] = 0xb6; + ok &= !munmap(p + ps, ps) && locked_kb() == (long)(2 * kb); + target = mmap(NULL, ps, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (target == MAP_FAILED) + ksft_exit_fail_msg("locked remap target\n"); + q = mremap(p, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, target); + if (q == MAP_FAILED) + ksft_exit_fail_msg("locked remap move: %s\n", strerror(errno)); + ok &= q == target && locked_kb() == (long)(2 * kb); + for (unsigned long i = 0; i < ps; i++) + ok &= q[i] == 0x37; + ok &= p[2 * ps] == 0xb6; + ok &= !munmap(q, ps) && locked_kb() == (long)kb; + ok &= !munmap(p + 2 * ps, ps) && locked_kb() == 0; + return ok; +} + +static int exercise(void) +{ + unsigned long ps = getauxval(AT_PAGESZ), kb = ps / 1024; + unsigned char *p, *q, residency[4]; + int gate[2]; + pid_t pid; + bool ok; + + ksft_print_header(); + ksft_set_plan(9); + p = mmap(NULL, 32 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED || locked_kb() != 0) + ksft_exit_fail_msg("initial mapping/accounting\n"); + ok = !mlock(p + ps + 17, ps - 32) && locked_kb() == (long)kb; + ok &= !mincore(p + ps, ps, residency) && residency[0] & 1; + p[ps] = 0x51; + ksft_test_result( + ok, "%luK unaligned mlock uses exactly one user leaf\n", kb); + ok = !mlock(p + ps, ps) && locked_kb() == (long)kb; + ok &= !mlock(p + 2 * ps, ps) && locked_kb() == (long)(2 * kb); + ok &= !munlock(p + ps, ps) && locked_kb() == (long)kb; + p[2 * ps] = 0x62; + errno = 0; + ok &= madvise(p + 2 * ps, ps, MADV_PAGEOUT) == -1 && errno == EINVAL && + p[2 * ps] == 0x62; + ok &= !munlock(p + 2 * ps, ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK repeat lock and independent adjacent unlock\n", kb); + ok = !mlock2(p + 4 * ps, 4 * ps, MLOCK_ONFAULT) && + locked_kb() == (long)(4 * kb); + ok &= !mincore(p + 4 * ps, 4 * ps, residency); + for (unsigned int i = 0; i < 4; i++) + ok &= !(residency[i] & 1); + p[4 * ps] = 0x74; + ok &= !mincore(p + 4 * ps, ps, residency) && residency[0] & 1; + ok &= !munlock(p + 4 * ps, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MLOCK_ONFAULT accounting and deferred faults\n", kb); + q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_LOCKED, -1, 0); + ok = q != MAP_FAILED && locked_kb() == (long)(4 * kb); + if (q == MAP_FAILED || pipe(gate)) + ksft_exit_fail_msg("locked mmap/fork setup\n"); + memset(q, 0x8a, 4 * ps); + pid = fork(); + if (!pid) { + char token; + + close(gate[1]); + if (read(gate[0], &token, 1) != 1 || locked_kb() != 0) + _exit(2); + for (unsigned long i = 0; i < 4 * ps; i++) + if (q[i] != 0x8a) + _exit(3); + _exit(0); + } + close(gate[0]); + memset(q + ps, 0x9b, ps); + ok &= write(gate[1], "x", 1) == 1; + close(gate[1]); + ok &= wait_ok(pid) && locked_kb() == (long)(4 * kb); + ok &= !munmap(q, 4 * ps) && locked_kb() == 0; + ksft_test_result( + ok, "%luK MAP_LOCKED, fork noninheritance and parent COW\n", + kb); + ok = !mlockall(MCL_FUTURE | MCL_ONFAULT); + q = mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ok &= q != MAP_FAILED && locked_kb() == (long)kb; + if (q != MAP_FAILED) + *q = 0xaf; + ok &= !munlockall() && locked_kb() == 0; + if (q != MAP_FAILED) + munmap(q, ps); + ksft_test_result(ok, "%luK future on-fault locks and munlockall\n", kb); + pid = fork(); + if (!pid) { + struct rlimit limit = { ps, ps }; + + if (setrlimit(RLIMIT_MEMLOCK, &limit) || setuid(1000) || + mlock(p, ps)) + _exit(4); + errno = 0; + if (mlock(p + ps, ps) != -1 || errno != ENOMEM || + locked_kb() != (long)kb) + _exit(5); + _exit(0); + } + ksft_test_result( + wait_ok(pid), + "%luK unprivileged RLIMIT_MEMLOCK in user-page units\n", kb); + pid = fork(); + if (!pid) { + if (mlockall(MCL_CURRENT) || locked_kb() <= (long)kb || + munlockall() || locked_kb() != 0) + _exit(6); + _exit(0); + } + ksft_test_result(wait_ok(pid), "%luK lock/unlock current mappings\n", + kb); + { + unsigned long huge = ((1UL << 31) + 1) * ps; + + q = mmap(NULL, huge, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + ok = q != MAP_FAILED; + if (q != MAP_FAILED) { + ok &= !mlock2(q, huge, MLOCK_ONFAULT) && + locked_kb() == (long)(huge / 1024); + ok &= !munlock(q, huge) && locked_kb() == 0; + munmap(q, huge); + } + ksft_test_result( + ok, + "%luK deferred lock count beyond signed 32-bit leaves\n", + kb); + } + ksft_test_result( + remap_locked(ps), + "%luK locked grow/shrink/move and partial unmap accounting\n", + kb); + munmap(p, 32 * ps); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long native = getauxval(AT_PAGESZ); + unsigned long granules[] = { native, 4096, 16384 }; + unsigned int nr = native > 16384 ? 3 : 2; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + + if (!pid) { + char size[32], host_size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + snprintf(host_size, sizeof(host_size), "%lu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(7); + execl("/init", "/init", "--test", size, host_size, + NULL); + _exit(8); + } + ok &= wait_ok(pid); + } + printf("%s - mlock fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/numa-balancing-init.c b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c new file mode 100644 index 00000000000000..2ea31bda12968a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-balancing-init.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Disposable two-node guest: real automatic NUMA hinting and migration. + * Volatile accesses intentionally trigger page faults and repeated accesses; + * thread coordination uses an atomic flag, not volatile storage. + */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static void setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + CHECK(!close(fd)); +} + +static void cpu(int id) +{ + cpu_set_t set; + + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} + +static unsigned long counter(const char *key) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char name[128]; + unsigned long value; + + CHECK(f); + while (fscanf(f, "%127s %lu", name, &value) == 2) { + if (!strcmp(name, key)) { + CHECK(!fclose(f)); + return value; + } + } + CHECK(0); + return 0; +} + +static double now(void) +{ + struct timespec ts; + + CHECK(!clock_gettime(CLOCK_MONOTONIC, &ts)); + return ts.tv_sec + ts.tv_nsec / 1e9; +} + +static int node_at(void *address) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, address, + MPOL_F_ADDR | MPOL_F_NODE)); + return node; +} + +static unsigned long sample(unsigned char *map, unsigned long length, + unsigned long page) +{ + unsigned long moved = 0; + + for (unsigned long off = 0; off < length; off += page) + if (off != 2 * page) + moved += node_at(map + off + 16) == 1; + return moved; +} + +static void forbidden(unsigned char *address, int write_access) +{ + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + if (write_access) + *(volatile unsigned char *)address = 0x73; + else + (void)*(volatile unsigned char *)address; + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV); +} + +struct local_reader { + unsigned char *map; + unsigned long length, page; + atomic_int stop; +}; + +static void *read_local(void *arg) +{ + struct local_reader *r = arg; + + cpu(1); + while (!atomic_load_explicit(&r->stop, memory_order_relaxed)) { + for (unsigned long off = 0; off < r->length; off += r->page) { + if (off == 2 * r->page) + continue; + CHECK(*(volatile uint64_t *)(r->map + off) == + 0x81720000 + off / r->page); + } + } + return NULL; +} + +static void local_restore(unsigned char *map, unsigned long length, + unsigned long page) +{ + struct local_reader r = { .map = map, .length = length, .page = page }; + unsigned long hints = counter("numa_hint_faults_local"); + pthread_t worker; + double deadline = now() + 2; + + /* Two threads permit hinting of already-local private mappings. */ + CHECK(!pthread_create(&worker, NULL, read_local, &r)); + do { + for (unsigned long off = 0; off < length; off += page) { + if (off == 2 * page) + continue; + CHECK(*(volatile uint64_t *)(map + off) == + 0x81720000 + off / page); + } + } while (now() < deadline); + atomic_store_explicit(&r.stop, 1, memory_order_relaxed); + CHECK(!pthread_join(worker, NULL)); + hints = counter("numa_hint_faults_local") - hints; + printf("NUMAB local page=%lu hints=%lu\n", page, hints); + CHECK(hints > 0); +} + +static void exercise(void) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 8UL << 20; + unsigned long mask = 1, allowed = 0, hints, migrated, iterations = 0; + unsigned long moved = 0, total = length / page - 1; + unsigned char *map; + int mode; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == 3); + cpu(0); + map = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED); + CHECK(!madvise(map, length, MADV_NOHUGEPAGE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_BIND, &mask, 64UL, 0)); + for (unsigned long off = 0; off < length; off += page) { + *(uint64_t *)(map + off) = 0x81720000 + off / page; + *(uint64_t *)(map + off + 8) = 0; + *(uint64_t *)(map + off + page - 8) = 0x61320000 + off / page; + CHECK(node_at(map + off + 16) == 0); + } + CHECK(!mprotect(map + page, page, PROT_READ)); + CHECK(!mprotect(map + 2 * page, page, PROT_NONE)); + CHECK(!syscall(SYS_mbind, map, length, MPOL_DEFAULT, NULL, 0UL, 0)); + cpu(1); + hints = counter("numa_hint_faults"); + migrated = counter("numa_pages_migrated"); + double started = now(), deadline = started + 20; + + do { + for (unsigned long off = 0; off < length; off += page) { + volatile uint64_t *p = (void *)(map + off); + + if (off == 2 * page) + continue; + CHECK(p[0] == 0x81720000 + off / page); + CHECK(p[page / 8 - 1] == 0x61320000 + off / page); + if (off != page) + p[1]++; + } + iterations++; + if (!(iterations % 128)) { + moved = sample(map, length, page); + if (moved == total) + break; + } + } while (now() < deadline); + moved = sample(map, length, page); + hints = counter("numa_hint_faults") - hints; + migrated = counter("numa_pages_migrated") - migrated; + printf("NUMAB page=%lu moved=%lu/%lu hints=%lu migrated=%lu loops=%lu seconds=%.3f\n", + page, moved, total, hints, migrated, iterations, + now() - started); + CHECK(moved == total && hints > 0 && migrated > 0); + for (unsigned long off = 0; off < length; off += page) { + if (off != page && off != 2 * page) + CHECK(*(uint64_t *)(map + off + 8) == iterations); + } + local_restore(map, length, page); + forbidden(map + page, 1); + forbidden(map + 2 * page, 0); + CHECK(!mprotect(map + 2 * page, page, PROT_READ)); + CHECK(*(uint64_t *)(map + 2 * page) == 0x81720002); + CHECK(!munmap(map, length)); + printf("ok - NUMAB page=%lu automatic placement, bytes and permissions\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, compat_only = 0, no4k = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[2], NULL, 10)); + exercise(); + return 0; + } + CHECK(getpid() == 1); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + else if (!strcmp(argv[i], "--no4k")) + no4k = 1; + else + CHECK(0); + } + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("debugfs", "/debug", "debugfs", 0, NULL)); + setting("/proc/sys/kernel/numa_balancing", "1\n"); + setting("/debug/sched/numa_balancing/scan_delay_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_min_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_period_max_ms", "100\n"); + setting("/debug/sched/numa_balancing/scan_size_mb", "16\n"); + for (unsigned int i = 0; i < 3; i++) { + pid_t child; + int status; + + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", sizes[i]); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + CHECK(!access("/compat-numa-balancing-probe", X_OK)); + pid_t child = fork(); + int status; + + CHECK(child >= 0); + if (!child) { + char s[32]; + + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-numa-balancing-probe", + "/compat-numa-balancing-probe", s, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMAB expected-page=%lu status=%d\n", + no4k ? native : 4096UL, status); + printf("NUMA BALANCING %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/numa-policy-init.c b/tools/testing/selftests/arm64/user4k/numa-policy-init.c new file mode 100644 index 00000000000000..fedcedb529709f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/numa-policy-init.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable two-node guest: policy ranges and native physical placement. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int good, const char *expr, int line) +{ + if (!good) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expr, + errno); + exit(1); + } +} +#define CHECK(x) check(!!(x), #x, __LINE__) + +static long bind_to(void *addr, unsigned long bytes, int node, + unsigned int flags) +{ + unsigned long mask = 1UL << node; + + return syscall(SYS_mbind, addr, bytes, MPOL_BIND, &mask, 64UL, flags); +} + +static void policy_at(void *addr, int node) +{ + unsigned long mask = 0; + int mode = -1; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &mask, 64UL, addr, + MPOL_F_ADDR)); + CHECK(mode == MPOL_BIND && mask == (1UL << node)); +} + +static void node_at(void *addr, int expected) +{ + int node = -1; + + CHECK(!syscall(SYS_get_mempolicy, &node, NULL, 0UL, addr, + MPOL_F_NODE | MPOL_F_ADDR)); + if (node != expected) + fprintf(stderr, "node=%d expected=%d addr=%p\n", node, expected, + addr); + CHECK(node == expected); +} + +static void exercise(unsigned long native, int baseline) +{ + unsigned long page = getauxval(AT_PAGESZ), allowed = 0; + unsigned char *reserve, *base, *shared, *alias; + int mode, fd; + + CHECK(!syscall(SYS_get_mempolicy, &mode, &allowed, 64UL, NULL, + MPOL_F_MEMS_ALLOWED)); + CHECK(allowed == + 3); /* Require two real memory nodes, not a silent skip. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + if (baseline) { + errno = 0; + long ret = bind_to(base + page, 1, 1, 0); + + CHECK(page < native ? ret == -1 && errno == EINVAL : ret == 0); + printf("ok - baseline native=%lu page=%lu mbind=%ld errno=%d\n", + native, page, ret, errno); + CHECK(!munmap(reserve, 4 * native)); + return; + } + CHECK(!bind_to(base + page, 1, 1, 0)); + policy_at(base, 0); + policy_at(base + page, 1); + if (page < native) + policy_at(base + 2 * page, 0); + CHECK(!syscall(SYS_set_mempolicy_home_node, base + page, 1UL, 1UL, + 0UL)); + errno = 0; + CHECK(bind_to(base + 1, page, 1, 0) == -1 && errno == EINVAL); + for (unsigned long off = 0; off < 2 * native; off += page) { + base[off + 17] = 0x31 + off / page; + node_at(base + off + 17, off == page ? 1 : 0); + } + CHECK(!bind_to(base + page, page, 1, MPOL_MF_STRICT)); + errno = 0; + CHECK(bind_to(base, 2 * native, 1, MPOL_MF_STRICT) == -1 && + errno == EIO); + policy_at(base, 0); + CHECK(!munmap(reserve, 4 * native)); + + /* A split anonymous VMA must carry its native interleave index correctly. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + unsigned long mask = 3, inset = page < native ? page : 0; + + CHECK(!syscall(SYS_mbind, base + inset, 2 * native - 2 * inset, + MPOL_INTERLEAVE, &mask, 64UL, 0)); + for (unsigned long off = inset; off < 2 * native - inset; off += page) { + base[off + 17] = 0x43; + node_at(base + off + 17, ((uintptr_t)base + off) / native % 2); + } + CHECK(!munmap(reserve, 4 * native)); + + /* Moving a fragment moves its native physical owner, retaining bytes. */ + reserve = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + base = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + CHECK(!bind_to(base, 2 * native, 0, 0)); + for (unsigned long off = 0; off < 2 * native; off += page) + base[off + 17] = 0x51 + off / page; + CHECK(!bind_to(base + native - page, page, 1, + MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) { + CHECK(base[off + 17] == (unsigned char)(0x51 + off / page)); + node_at(base + off + 17, off < native ? 1 : 0); + } + CHECK(!bind_to(base, 2 * native, 0, MPOL_MF_MOVE_ALL | MPOL_MF_STRICT)); + for (unsigned long off = 0; off < 2 * native; off += page) + node_at(base + off + 17, 0); + CHECK(!munmap(reserve, 4 * native)); + + fd = memfd_create("numa-shared", 0); + CHECK(fd >= 0 && !ftruncate(fd, 2 * native)); + shared = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + 0); + CHECK(shared != MAP_FAILED); + CHECK(!bind_to(shared, native, 0, 0)); + CHECK(!bind_to(shared + native, native, 1, 0)); + if (page < native) { + alias = mmap(NULL, 2 * native - page, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, page); + CHECK(alias != MAP_FAILED); + policy_at(alias, 0); + policy_at(alias + native - page, 1); + errno = 0; + CHECK(bind_to(alias, page, 1, 0) == -1 && errno == EINVAL); + policy_at(alias, 0); + policy_at(shared, 0); + CHECK(!bind_to(alias + native - page, native, 1, 0)); + for (unsigned long off = 0; off < 2 * native - page; + off += page) { + alias[off + 17] = 0x72; + node_at(alias + off + 17, off + page < native ? 0 : 1); + CHECK(shared[off + page + 17] == 0x72); + } + CHECK(!munmap(alias, 2 * native - page)); + } + CHECK(!munmap(shared, 2 * native) && !close(fd)); + printf("ok - NUMA native=%lu page=%lu policy, placement, migration and shared offsets\n", + native, page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { 4096, 16384, 65536 }; + int failed = 0, baseline = 0, no4; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--case")) { + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + exercise(strtoul(argv[2], NULL, 10), atoi(argv[4])); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + FILE *cmdline = fopen("/proc/cmdline", "r"); + char line[4096]; + + CHECK(cmdline && fgets(line, sizeof(line), cmdline)); + baseline = strstr(line, "numa_fixture_baseline=1") != NULL; + no4 = strstr(line, "id_aa64mmfr0.tgran4=f") != NULL; + CHECK(!fclose(cmdline)); + for (unsigned int i = 0; i < 3; i++) { + int status; + pid_t child; + + if (sizes[i] > native || (no4 && sizes[i] == 4096)) + continue; + child = fork(); + CHECK(child >= 0); + if (!child) { + char n[32], s[32], b[4]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + snprintf(b, sizeof(b), "%d", baseline); + CHECK(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], + 0UL, 0UL, 0UL)); + execl("/init", "/init", "--case", n, s, b, NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + } + if (!baseline) { + int status; + pid_t child = fork(); + + CHECK(child >= 0); + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4 ? native : 4096UL); + execl("/compat-numa-probe", "/compat-numa-probe", n, s, + NULL); + _exit(2); + } + CHECK(waitpid(child, &status, 0) == child); + failed |= !WIFEXITED(status) || WEXITSTATUS(status); + printf("A32 NUMA expected-page=%lu status=%d\n", + no4 ? native : 4096UL, status); + } + printf("NUMA GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/page-contract.c b/tools/testing/selftests/arm64/user4k/page-contract.c new file mode 100644 index 00000000000000..316ffe48d0a476 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/page-contract.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Run unchanged under the current process ABI, including native 4K/16K/64K. + * Explicit --expect-{4k,16k,64k} options assert the selected size; they never + * launch a different ABI or emulate it. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define X18_SENTINEL UINT64_C(0xfedcba9876543210) + +extern long x18_syscall(long nr, long arg0, long arg1, long arg2, + uint64_t *seen); + +static volatile sig_atomic_t signal_seen; +static uint64_t signal_x18; + +static void x18_signal(int sig, siginfo_t *info, void *context) +{ + ucontext_t *uc = context; + + (void)sig; + (void)info; + signal_x18 = uc->uc_mcontext.regs[18]; + signal_seen = 1; + /* rt_sigreturn must restore the interrupted value, not this value. */ + /* Built with -ffixed-x18: the compiler never allocates this register. */ + asm volatile("mov x18, xzr"); +} + +static bool wait_for(pid_t child, int expected_signal) +{ + int status; + pid_t ret; + + if (child < 0) + return false; + do { + ret = waitpid(child, &status, 0); + } while (ret < 0 && errno == EINTR); + if (ret != child) + return false; + if (expected_signal) + return WIFSIGNALED(status) && + WTERMSIG(status) == expected_signal; + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static bool access_faults(volatile unsigned char *ptr, bool write) +{ + pid_t child = fork(); + + if (!child) { + if (write) + *ptr = 0x5a; + else + (void)*ptr; + _exit(1); + } + return wait_for(child, SIGSEGV); +} + +static void private_mappings(size_t page) +{ + unsigned char *p = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok; + pid_t child; + + if (p == MAP_FAILED) + ksft_exit_fail_msg("anonymous mmap: %s\n", strerror(errno)); + memset(p, 0x41, 4 * page); + if (page > 4096) { + errno = 0; + ok = mprotect(p + 4096, 4096, PROT_READ) == -1 && + errno == EINVAL; + } else { + ok = mprotect(p + page, page, PROT_READ) == 0; + } + ksft_test_result(ok, + "mapping alignment agrees with the selected ABI\n"); + ok = mprotect(p + page, page, PROT_READ) == 0 && + mprotect(p + 2 * page, page, PROT_NONE) == 0; + ksft_test_result( + ok && access_faults(p + page, true) && + access_faults(p + 2 * page, false), + "read-only and inaccessible neighbors fault independently\n"); + p[0] = 0x10; + p[3 * page] = 0x30; + ksft_test_result( + p[page] == 0x41 && p[0] == 0x10 && p[3 * page] == 0x30, + "neighbor permissions do not remove writable mappings\n"); + child = fork(); + if (!child) { + p[0] = 0x99; + _exit(p[3 * page] != 0x30); + } + ksft_test_result(wait_for(child, 0) && p[0] == 0x10, + "fork preserves private COW isolation\n"); + ok = munmap(p + page, page) == 0; + ksft_test_result(ok && access_faults(p + page, false) && p[0] == 0x10 && + p[3 * page] == 0x30, + "single-page unmap preserves surrounding mappings\n"); + munmap(p, 4 * page); +} + +static void shared_mappings(size_t page) +{ + int fd = memfd_create("page-contract", MFD_CLOEXEC); + unsigned char *a, *b; + bool ok = true; + pid_t child; + unsigned int i; + + if (fd < 0 || ftruncate(fd, 4 * page)) + ksft_exit_fail_msg("memfd: %s\n", strerror(errno)); + a = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + b = mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (a == MAP_FAILED || b == MAP_FAILED) + ksft_exit_fail_msg("shared mmap: %s\n", strerror(errno)); + for (i = 0; i < 4; i++) { + a[i * page] = 0x50 + i; + ok &= b[i * page] == 0x50 + i; + } + ksft_test_result(ok, "shared aliases expose the same bytes\n"); + ok = mprotect(a + page, page, PROT_READ) == 0; + child = fork(); + if (!child) { + b[page] = 0x73; + _exit(a[page] != 0x73); + } + ok &= wait_for(child, 0); + ksft_test_result( + ok && a[page] == 0x73 && access_faults(a + page, true), + "shared writes cross fork while alias permissions stay local\n"); + munmap(a, 4 * page); + munmap(b, 4 * page); + close(fd); +} + +static void x18_contract(void) +{ + struct sigaction sa = { .sa_sigaction = x18_signal, + .sa_flags = SA_SIGINFO }; + struct timespec pause = { .tv_nsec = 1000000 }; + struct rusage before, after; + uint64_t seen; + long ret; + bool ok; + + ret = x18_syscall(SYS_getpid, 0, 0, 0, &seen); + ksft_test_result(ret == getpid() && seen == X18_SENTINEL, + "x18 survives a raw syscall\n"); + ok = getrusage(RUSAGE_SELF, &before) == 0; + ret = x18_syscall(SYS_nanosleep, (long)&pause, 0, 0, &seen); + ok &= getrusage(RUSAGE_SELF, &after) == 0; + ksft_test_result(ok && ret == 0 && seen == X18_SENTINEL && + after.ru_nvcsw > before.ru_nvcsw, + "x18 survives a measured voluntary context switch\n"); + sigemptyset(&sa.sa_mask); + ok = sigaction(SIGUSR1, &sa, NULL) == 0; + if (ok) + ret = x18_syscall(SYS_tgkill, getpid(), syscall(SYS_gettid), + SIGUSR1, &seen); + ksft_test_result( + ok && ret == 0 && signal_seen && signal_x18 == X18_SENTINEL && + seen == X18_SENTINEL, + "signal frame and sigreturn preserve x18 after handler clobber\n"); +} + +int main(int argc, char **argv) +{ + const struct rlimit no_core = { 0, 0 }; + size_t expected = getauxval(AT_PAGESZ); + int result; + + if (argc == 2 && !strcmp(argv[1], "--expect-4k")) { + expected = 4096; + } else if (argc == 2 && !strcmp(argv[1], "--expect-16k")) { + expected = 16384; + } else if (argc == 2 && !strcmp(argv[1], "--expect-64k")) { + expected = 65536; + } else if (argc != 1) { + fprintf(stderr, + "usage: %s [--expect-4k|--expect-16k|--expect-64k]\n", + argv[0]); + return KSFT_FAIL; + } + ksft_print_header(); + ksft_set_plan(11); + if (expected != 4096 && expected != 16384 && expected != 65536) + ksft_exit_skip("unsupported process page size: %zu\n", + expected); + if (sysconf(_SC_PAGESIZE) != (long)expected || + getauxval(AT_PAGESZ) != expected) + ksft_exit_fail_msg( + "expected %zu-byte ABI; sysconf=%ld AT_PAGESZ=%lu\n", + expected, sysconf(_SC_PAGESIZE), getauxval(AT_PAGESZ)); + ksft_test_result_pass( + "process reports the expected %zu-byte page ABI\n", expected); + if (setrlimit(RLIMIT_CORE, &no_core)) + ksft_exit_fail_msg("disable child core dumps: %s\n", + strerror(errno)); + private_mappings(expected); + shared_mappings(expected); + x18_contract(); + ksft_print_cnts(); + result = ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; + /* This static executable can also be /init in a disposable test VM. */ + if (getpid() == 1) { + fflush(NULL); + reboot(RB_POWER_OFF); + } + return result; +} diff --git a/tools/testing/selftests/arm64/user4k/perf-ring-init.c b/tools/testing/selftests/arm64/user4k/perf-ring-init.c new file mode 100644 index 00000000000000..4252f017619ca2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/perf-ring-init.c @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: perf ring layout and sampling with per-mm page sizes. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int no4k; + +static int wait_child(pid_t child) +{ + int status; + pid_t got; + + if (child < 0) + return 1; + do { + got = waitpid(child, &status, 0); + } while (got < 0 && errno == EINTR); + return got != child || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int event_open(int sampling) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .size = sizeof(attr), + .config = PERF_COUNT_SW_PAGE_FAULTS, + .disabled = 1, + .sample_period = sampling ? 1 : 0, + .sample_type = PERF_SAMPLE_TID | PERF_SAMPLE_TIME | + PERF_SAMPLE_ADDR | PERF_SAMPLE_PERIOD, + }; + int fd = syscall(SYS_perf_event_open, &attr, 0, -1, -1, 0); + + CHECK(fd >= 0); + return fd; +} + +static sigjmp_buf fault_return; +static volatile sig_atomic_t fault_signal; + +static void fault_handler(int signal) +{ + fault_signal = signal; + siglongjmp(fault_return, 1); +} + +static void readonly_data(void *address) +{ + struct sigaction action = { .sa_handler = fault_handler }, previous; + + CHECK(!sigemptyset(&action.sa_mask)); + CHECK(!sigaction(SIGBUS, &action, &previous)); + fault_signal = 0; + if (!sigsetjmp(fault_return, 1)) + *(volatile unsigned char *)address = 0; + CHECK(fault_signal == SIGBUS); + CHECK(!sigaction(SIGBUS, &previous, NULL)); +} + +static void copy_record(void *destination, const unsigned char *ring, + unsigned long capacity, uint64_t position, size_t size) +{ + unsigned long offset = position & (capacity - 1); + size_t first = capacity - offset; + + if (first > size) + first = size; + memcpy(destination, ring + offset, first); + memcpy((unsigned char *)destination + first, ring, size - first); +} + +static void sampling(unsigned long pages) +{ + unsigned long page = getauxval(AT_PAGESZ), capacity = pages * page; + unsigned long length = page + capacity; + int fd = event_open(1); + struct perf_event_mmap_page *map = + mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + unsigned char *region, *ring; + uint64_t tail = 0, samples = 0, wrapped = 0, region_samples = 0; + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == capacity); + map->data_tail = 0; + ring = (unsigned char *)map + map->data_offset; + for (unsigned long i = 0; i < capacity; i += page) + CHECK(ring[i] == 0); + readonly_data(ring); + region = mmap(NULL, 16 * page, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(region != MAP_FAILED); + for (unsigned int batch = 0; batch < 10000 && tail < 3 * capacity; + batch++) { + CHECK(!madvise(region, 16 * page, MADV_DONTNEED)); + CHECK(!ioctl(fd, PERF_EVENT_IOC_ENABLE, 0)); + for (unsigned int i = 0; i < 16; i++) + *(volatile unsigned char *)(region + i * page) = 1; + CHECK(!ioctl(fd, PERF_EVENT_IOC_DISABLE, 0)); + uint64_t head = + __atomic_load_n(&map->data_head, __ATOMIC_ACQUIRE); + + CHECK(head >= tail && head - tail <= capacity); + while (tail < head) { + struct perf_event_header header; + struct { + struct perf_event_header header; + uint32_t pid, tid; + uint64_t time, address, period; + } record; + + copy_record(&header, ring, capacity, tail, + sizeof(header)); + CHECK(header.size >= sizeof(header) && + header.size <= head - tail); + if (header.type == PERF_RECORD_SAMPLE) { + CHECK(header.size == sizeof(record)); + copy_record(&record, ring, capacity, tail, + sizeof(record)); + CHECK(record.pid == (uint32_t)getpid()); + CHECK(record.tid == (uint32_t)getpid()); + if (record.address >= (uintptr_t)region && + record.address < + (uintptr_t)region + 16 * page) + region_samples++; + CHECK(record.period == 1); + samples++; + if ((tail & (capacity - 1)) + header.size > + capacity) + wrapped++; + } else { + CHECK(header.type == PERF_RECORD_THROTTLE || + header.type == PERF_RECORD_UNTHROTTLE); + } + tail += header.size; + } + __atomic_store_n(&map->data_tail, tail, __ATOMIC_RELEASE); + } + CHECK(tail >= 3 * capacity && samples && wrapped); + /* Library/stack faults may also occur while a batch is enabled. */ + CHECK(region_samples > samples * 9 / 10); + CHECK(!munmap(region, 16 * page)); + CHECK(!close(fd)); + CHECK(map->data_head == tail); + CHECK(!munmap(map, length)); + printf("ok - perf page=%lu ring=%lu samples=%llu region=%llu wrapped-records=%llu\n", + page, capacity, (unsigned long long)samples, + (unsigned long long)region_samples, (unsigned long long)wrapped); +} + +static void layout(unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + int fd = event_open(0); + struct perf_event_mmap_page *map = + mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + + CHECK(map != MAP_FAILED); + CHECK(map->data_offset == page && map->data_size == 0); + map->data_tail = 0; + CHECK(!munmap(map, page)); + CHECK(!close(fd)); + fd = event_open(0); + errno = 0; + CHECK(mmap(NULL, 4 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) == + MAP_FAILED); + CHECK(errno == EINVAL); + map = mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(map != MAP_FAILED); + struct perf_event_mmap_page *alias = + mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + CHECK(alias != MAP_FAILED); + map->data_tail = 7; + CHECK(alias->data_tail == 7); + map->data_tail = 0; + CHECK(!munmap(alias, 3 * page)); + /* A software event has no AUX provider. Check byte-exact offset routing. */ + map->aux_offset = 3 * page; + map->aux_size = native; + errno = 0; + CHECK(mmap(NULL, native, PROT_READ, MAP_SHARED, fd, map->aux_offset) == + MAP_FAILED); + CHECK(errno == EOPNOTSUPP); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (sizes[i] > native || sizes[i] == page || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char f[32], s[32]; + + snprintf(f, sizeof(f), "%d", fd); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--alias", f, s, NULL); + _exit(3); + } + CHECK(!wait_child(child)); + } + CHECK(map->data_size == 2 * page && map->data_offset == page); + CHECK(!munmap(map, 3 * page)); + CHECK(!close(fd)); + printf("ok - perf page=%lu metadata, invalid size, aliases, AUX routing\n", + page); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0, compat_only = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + no4k = getenv("PERF_TEST_NO4K") != NULL; + if (argc == 4 && !strcmp(argv[1], "--alias")) { + unsigned long page = strtoul(argv[3], NULL, 10); + + CHECK(getauxval(AT_PAGESZ) == page); + errno = 0; + CHECK(mmap(NULL, 3 * page, PROT_READ | PROT_WRITE, MAP_SHARED, + atoi(argv[2]), 0) == MAP_FAILED); + CHECK(errno == EINVAL); + return 0; + } + if (argc == 4 && !strcmp(argv[1], "--case")) { + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == strtoul(argv[3], NULL, 10)); + layout(native); + sampling(1); + sampling(2); + sampling(2 * native / getauxval(AT_PAGESZ)); + return 0; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--no4k")) { + no4k = 1; + CHECK(!setenv("PERF_TEST_NO4K", "1", 1)); + } + if (!strcmp(argv[i], "--compat-only")) + compat_only = 1; + } + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only || sizes[i] > native || + (no4k && sizes[i] == 4096)) + continue; + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", sizes[i]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, + 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, sizes[i]); + failed |= bad; + } + if (compat_only) + CHECK(access("/compat-perf", X_OK) == 0); + if (access("/compat-perf", X_OK) == 0) { + pid_t child = fork(); + + if (!child) { + char n[32], s[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(s, sizeof(s), "%lu", no4k ? native : 4096UL); + execl("/compat-perf", "/compat-perf", n, s, NULL); + _exit(3); + } + int bad = wait_child(child); + + printf("%s - perf AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("PERF RING GRANULE %s\n", failed ? "FAIL" : "PASS"); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-args.c b/tools/testing/selftests/arm64/user4k/proc-args.c new file mode 100644 index 00000000000000..300742a2588632 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-args.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Compare argv/envp with /proc/PID/cmdline and /proc/PID/environ. + * + * Run it once natively and once under the 4K user granule (for example + * "userpages 4k ./proc-args"). The process reads its own files, and a forked + * child reads the parent's files through /proc/. "--hold SECONDS" + * keeps the process alive so a native process can read it from outside. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +static int failures; + +static ssize_t read_file(const char *path, char *buf, size_t size) +{ + ssize_t got, total = 0; + int fd = open(path, O_RDONLY); + + if (fd < 0) + return -errno; + while ((size_t)total < size) { + got = read(fd, buf + total, size - total); + if (got < 0) { + if (errno == EINTR) + continue; + total = -errno; + break; + } + if (!got) + break; + total += got; + } + close(fd); + return total; +} + +/* Join the complete vector; reserve one extra byte in the read buffer. */ +static char *join(char **vec, size_t *size) +{ + size_t len = 0, offset = 0; + char **entry, *buf; + + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + if (n > SSIZE_MAX - 1 - len) { + errno = EOVERFLOW; + return NULL; + } + len += n; + } + buf = malloc(len ? len : 1); + if (!buf) + return NULL; + for (entry = vec; *entry; entry++) { + size_t n = strlen(*entry) + 1; + + memcpy(buf + offset, *entry, n); + offset += n; + } + *size = len; + return buf; +} + +static void check(const char *who, const char *name, pid_t pid, char **vec) +{ + char path[64], *want, *got; + size_t want_len; + ssize_t got_len; + + want = join(vec, &want_len); + got = want ? malloc(want_len + 1) : NULL; + if (!got) { + printf("not ok %s %s allocation: %s\n", who, name, + strerror(errno)); + free(want); + failures++; + return; + } + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + /* A trailing byte rejects excess data; exact contents must reach EOF. */ + got_len = read_file(path, got, want_len + 1); + if (got_len == (ssize_t)want_len && !memcmp(want, got, want_len)) { + printf("ok %s %s %zd bytes\n", who, path, got_len); + } else { + printf("not ok %s %s want %zu got %zd", who, path, want_len, + got_len); + if (got_len < 0) + printf(" error %s", strerror(-got_len)); + else if (got_len > 0) + printf(" first [%.*s]", + (int)(got_len < 64 ? got_len : 64), got); + printf("\n"); + failures++; + } + free(got); + free(want); +} + +static void show_layout(pid_t pid, char **argv) +{ + char buf[4096], *p, *field; + ssize_t len; + int i; + + printf("pid %d AT_PAGESZ %lu argv[0] %p environ[0] %p\n", pid, + getauxval(AT_PAGESZ), (void *)argv[0], + environ[0] ? (void *)environ[0] : NULL); + len = read_file("/proc/self/stat", buf, sizeof(buf) - 1); + if (len <= 0) + return; + buf[len] = 0; + /* Fields after the parenthesised comm start at field 3. */ + p = strrchr(buf, ')'); + if (!p || p[1] != ' ') + return; + printf("stat arg_start arg_end env_start env_end:"); + for (i = 3, field = strtok(p + 2, " "); field; + field = strtok(NULL, " "), i++) + if (i >= 48 && i <= 51) + printf(" %#lx", strtoul(field, NULL, 10)); + printf("\n"); +} + +int main(int argc, char **argv) +{ + pid_t self = getpid(), child; + int status; + + setbuf(stdout, NULL); + show_layout(self, argv); + check("self", "cmdline", self, argv); + check("self", "environ", self, environ); + + child = fork(); + if (child < 0) + return 1; + if (!child) { + failures = 0; + check("child", "cmdline", self, argv); + check("child", "environ", self, environ); + _exit(failures ? 1 : 0); + } + if (waitpid(child, &status, 0) != child || !WIFEXITED(status) || + WEXITSTATUS(status)) + failures++; + + if (argc > 2 && !strcmp(argv[1], "--hold")) { + fflush(stdout); + sleep(atoi(argv[2])); + } + printf("%s: %d failure(s)\n", failures ? "FAIL" : "PASS", failures); + return failures ? 1 : 0; +} diff --git a/tools/testing/selftests/arm64/user4k/proc-memory.c b/tools/testing/selftests/arm64/user4k/proc-memory.c new file mode 100644 index 00000000000000..81ca2d0553dd0a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-memory.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct stats { + unsigned long size, rss, pss, private_clean, private_dirty, + shared_clean, shared_dirty; + unsigned long anonymous, kernel_page, mmu_page; +}; + +static bool get_stats(pid_t pid, uintptr_t address, struct stats *s) +{ + char path[64], line[512]; + unsigned long start, end, value; + unsigned int fields = 0; + char key[64]; + bool found = false; + FILE *f; + + memset(s, 0, sizeof(*s)); + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == address; + continue; + } + if (!found) + continue; + if (sscanf(line, "%63[^:]: %lu", key, &value) != 2) + continue; + if (!strcmp(key, "Size")) { + s->size = value; + fields |= 1U << 0; + } else if (!strcmp(key, "Rss")) { + s->rss = value; + fields |= 1U << 1; + } else if (!strcmp(key, "Pss")) { + s->pss = value; + fields |= 1U << 2; + } else if (!strcmp(key, "Private_Clean")) { + s->private_clean = value; + fields |= 1U << 3; + } else if (!strcmp(key, "Private_Dirty")) { + s->private_dirty = value; + fields |= 1U << 4; + } else if (!strcmp(key, "Shared_Clean")) { + s->shared_clean = value; + fields |= 1U << 5; + } else if (!strcmp(key, "Shared_Dirty")) { + s->shared_dirty = value; + fields |= 1U << 6; + } else if (!strcmp(key, "Anonymous")) { + s->anonymous = value; + fields |= 1U << 7; + } else if (!strcmp(key, "KernelPageSize")) { + s->kernel_page = value; + fields |= 1U << 8; + } else if (!strcmp(key, "MMUPageSize")) { + s->mmu_page = value; + fields |= 1U << 9; + } + } + fclose(f); + return found && fields == (1U << 10) - 1; +} + +static bool check(pid_t pid, void *address, size_t page, size_t length, + unsigned long pss, unsigned long private, + unsigned long shared, bool anon) +{ + struct stats s; + bool ok = get_stats(pid, (uintptr_t)address, &s); + + ok &= s.size == length / 1024 && s.rss == length / 1024 && + s.pss == pss && s.private_clean + s.private_dirty == private && + s.shared_clean + s.shared_dirty == shared && + s.anonymous == (anon ? length / 1024 : 0) && + s.kernel_page == page / 1024 && s.mmu_page == page / 1024; + if (!ok) + printf("# smaps size=%lu rss=%lu pss=%lu private=%lu shared=%lu anon=%lu pages=%lu/%lu\n", + s.size, s.rss, s.pss, s.private_clean + s.private_dirty, + s.shared_clean + s.shared_dirty, s.anonymous, + s.kernel_page, s.mmu_page); + return ok; +} + +static int check_numa(void *address, size_t page) +{ + char line[1024]; + unsigned long start, anonymous = 0, pages = 0, granule = 0; + bool found = false; + FILE *f = fopen("/proc/self/numa_maps", "re"); + + if (!f) + return errno == ENOENT ? -1 : 0; + while (fgets(line, sizeof(line), f)) { + char *save, *token; + + if (sscanf(line, "%lx", &start) != 1 || + start != (uintptr_t)address) + continue; + found = true; + for (token = strtok_r(line, "\n ", &save); token; + token = strtok_r(NULL, "\n ", &save)) { + unsigned int node; + unsigned long count; + + if (sscanf(token, "anon=%lu", &count) == 1) + anonymous = count; + else if (sscanf(token, "kernelpagesize_kB=%lu", + &count) == 1) + granule = count; + else if (sscanf(token, "N%u=%lu", &node, &count) == 2) + pages += count; + } + break; + } + fclose(f); + if (!found || anonymous != 4 || pages != 4 || granule != page / 1024) + printf("# numa found=%d anon=%lu pages=%lu granule=%lu\n", + found, anonymous, pages, granule); + return found && anonymous == 4 && pages == 4 && granule == page / 1024; +} + +static bool reap(pid_t child) +{ + int status; + + return child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); +} + +static bool resident_rewrite_reuses(unsigned char *p, size_t page) +{ + uint64_t before[4], after[4]; + int fd = open("/proc/self/pagemap", O_RDONLY); + unsigned int i; + bool ok = fd >= 0; + + if (ok) + ok = pread(fd, before, sizeof(before), + (uintptr_t)p / page * 8) == sizeof(before); + for (i = 0; i < 4; i++) + p[i * page] ^= 0x21; + if (ok) + ok = pread(fd, after, sizeof(after), (uintptr_t)p / page * 8) == + sizeof(after); + for (i = 0; ok && i < 4; i++) { + const uint64_t frame = (1ULL << 55) - 1; + + ok &= (before[i] & frame) && + (before[i] & frame) == (after[i] & frame) && + (before[i] & (1ULL << 63)) && (after[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + return ok; +} + +static void anonymous_sharing(size_t page) +{ + unsigned char *base, *p; + int ready[2], go[2]; + pid_t child; + char byte; + bool ok; + + base = mmap(NULL, 6 * page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED || + mprotect(base + page, 4 * page, PROT_READ | PROT_WRITE) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("anonymous fixture setup\n"); + p = base + page; + memset(p, 0x57, 4 * page); + ksft_test_result( + check(getpid(), p, page, 4 * page, 4 * page / 1024, + 4 * page / 1024, 0, true), + "dense anonymous leaves have exact RSS/PSS/private bytes\n"); + { + int numa = check_numa(p, page); + + if (numa < 0) + ksft_test_result_skip( + "NUMA statistics require CONFIG_NUMA\n"); + else + ksft_test_result( + numa, + "NUMA statistics count target leaves and report their granule\n"); + } + child = fork(); + if (!child) { + close(ready[0]); + close(go[1]); + if (write(ready[1], "a", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(1); + p[page] = 0x96; + if (write(ready[1], "b", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, 2 * page / 1024, 0, + 4 * page / 1024, true); + ksft_test_result( + ok, + "fork divides each anonymous leaf's PSS by its own mapcount\n"); + ok = write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1 && + check(getpid(), p, page, 4 * page, (5 * page) / 2048, page / 1024, + 3 * page / 1024, true) && + p[page] == 0x57; + ksft_test_result( + ok, + "one child COW changes only that leaf's private/shared accounting\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && check(getpid(), p, page, 4 * page, + 4 * page / 1024, 4 * page / 1024, + 0, true), + "child exit restores full anonymous PSS\n"); + ksft_test_result( + resident_rewrite_reuses(p, page), + "resident writes reuse physical backing after the child exits\n"); + munmap(base, 6 * page); +} + +static int worker(int fd, int output, int input, size_t expected) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uintptr_t addr; + char byte; + + if (page != expected) + return 1; + p = mmap(NULL, page, PROT_READ, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + return 2; + addr = (uintptr_t)p; + if (write(output, &addr, sizeof(addr)) != sizeof(addr) || + read(input, &byte, 1) != 1) + return 3; + return 0; +} + +static void mixed_file(size_t page) +{ + size_t target = page == 4096 ? 16384 : 4096; + int ready[2], go[2], fd = memfd_create("mixed-pss", 0); + unsigned char data[16384], *p; + uintptr_t remote; + pid_t child; + bool ok; + + memset(data, 0x78, sizeof(data)); + if (fd < 0 || write(fd, data, sizeof(data)) != sizeof(data) || + pipe(ready) || pipe(go)) + ksft_exit_fail_msg("mixed file fixture setup\n"); + p = mmap(NULL, page, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + page == 4096 ? 4096 : 0); + if (p == MAP_FAILED || p[0] != 0x78) + ksft_exit_fail_msg("mixed file mmap\n"); + child = fork(); + if (!child) { + char farg[16], out[16], in[16], expected[16]; + + close(ready[0]); + close(go[1]); + snprintf(farg, sizeof(farg), "%d", fd); + snprintf(out, sizeof(out), "%d", ready[1]); + snprintf(in, sizeof(in), "%d", go[0]); + snprintf(expected, sizeof(expected), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(4); + execl("/proc/self/exe", "/proc/self/exe", "--worker", farg, out, + in, expected, NULL); + _exit(5); + } + close(ready[1]); + close(go[0]); + ok = child > 0 && + read(ready[0], &remote, sizeof(remote)) == sizeof(remote); + ok = ok && + check(getpid(), p, page, page, page == 4096 ? 2 : 14, + page == 4096 ? 0 : 12, 4, false) && + check(child, (void *)remote, target, target, + target == 4096 ? 2 : 14, target == 4096 ? 0 : 12, 4, false); + ksft_test_result( + ok, + "mixed file aliases divide only their overlapping physical quarter\n"); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ksft_test_result(reap(child) && + check(getpid(), p, page, page, page / 1024, + page / 1024, 0, false), + "mixed alias exit restores private file accounting\n"); + munmap(p, page); + close(fd); +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10)); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(8); + anonymous_sharing(page); + mixed_file(page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/proc-pages.c b/tools/testing/selftests/arm64/user4k/proc-pages.c new file mode 100644 index 00000000000000..d3aaa11101cf2b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/proc-pages.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define PRESENT (1ULL << 63) +#define EXCLUSIVE (1ULL << 56) +#define GUARD (1ULL << 58) +#define FRAME ((1ULL << 55) - 1) +#define NPAGE 5 + +struct ready { + uintptr_t address; + size_t page; + uint64_t entries[NPAGE]; + uint64_t shared_entry; +}; + +static int open_proc(pid_t pid, const char *name, int flags) +{ + char path[80]; + + snprintf(path, sizeof(path), "/proc/%d/%s", pid, name); + return open(path, flags); +} + +static bool entries(int fd, uintptr_t address, size_t page, uint64_t *values, + int nr) +{ + return pread(fd, values, nr * sizeof(*values), + address / page * sizeof(*values)) == + nr * (ssize_t)sizeof(*values); +} + +static unsigned char *mapping(size_t page) +{ + const size_t boundary = 32 * 1024 * 1024; + size_t reserve = boundary + 8 * page, head, tail; + unsigned char *base, *p; + + base = mmap(NULL, reserve, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, + 0); + if (base == MAP_FAILED) + return NULL; + /* Cross both native 32 MiB and alternative 2 MiB PTE-table boundaries. */ + p = (void *)(((uintptr_t)base + 2 * page + boundary - 1) & + ~(boundary - 1)); + p -= 2 * page; + head = p - base; + tail = reserve - head - NPAGE * page; + if ((head && munmap(base, head)) || + (tail && munmap(p + NPAGE * page, tail)) || + mprotect(p, NPAGE * page, PROT_READ | PROT_WRITE)) + return NULL; + memset(p, 0x67, NPAGE * page); + if (munmap(p + 2 * page, page) || + madvise(p + 3 * page, page, MADV_GUARD_INSTALL) || + mprotect(p + 4 * page, page, PROT_NONE)) + return NULL; + return p; +} + +static bool scan(int fd, uintptr_t addr, size_t page) +{ + struct page_region vec[8]; + struct pm_scan_arg arg = { + .size = sizeof(arg), + .start = addr, + .end = addr + NPAGE * page, + .vec = (uintptr_t)vec, + .vec_len = 8, + .return_mask = PAGE_IS_PRESENT | PAGE_IS_GUARD, + }; + int nr = ioctl(fd, PAGEMAP_SCAN, &arg); + bool ok = nr == 3 && arg.walk_end == arg.end; + if (!ok) + printf("# scan initial nr=%d errno=%d end_delta=%lld\n", nr, + errno, (long long)(arg.walk_end - arg.end)); + + /* The unmapped middle page produces no VMA result. */ + if (nr == 3) + ok &= vec[0].start == addr && vec[0].end == addr + 2 * page && + vec[0].categories == PAGE_IS_PRESENT && + vec[1].start == addr + 3 * page && + vec[1].end == addr + 4 * page && + vec[1].categories == PAGE_IS_GUARD && + vec[2].start == addr + 4 * page && + vec[2].end == arg.end && + vec[2].categories == PAGE_IS_PRESENT; + arg.max_pages = 1; + arg.walk_end = 0; + nr = ioctl(fd, PAGEMAP_SCAN, &arg); + ok &= nr == 1 && vec[0].start == addr && vec[0].end == addr + page && + arg.walk_end == addr + page; + arg.start = addr + 1; + errno = 0; + ok &= ioctl(fd, PAGEMAP_SCAN, &arg) == -1 && errno == EINVAL; + return ok; +} + +static int remote_worker(int output, int input, size_t expected, int shared_fd) +{ + struct ready ready = { .page = getauxval(AT_PAGESZ) }; + unsigned char *p; + int fd = open_proc(getpid(), "pagemap", O_RDONLY); + char byte; + unsigned char *shared = + mmap(NULL, 16384, PROT_READ, MAP_SHARED, shared_fd, 0); + size_t offset = ready.page == 4096 ? 4096 : 0; + + if (shared == MAP_FAILED || shared[offset] != 0x83 || + !entries(fd, (uintptr_t)shared + offset, ready.page, + &ready.shared_entry, 1)) + return 2; + /* Allocate aliases first so mmap cannot reuse our deliberate hole. */ + p = mapping(ready.page); + ready.address = (uintptr_t)p; + if (ready.page != expected || !p || fd < 0 || + !entries(fd, ready.address, ready.page, ready.entries, NPAGE) || + write(output, &ready, sizeof(ready)) != sizeof(ready) || + read(input, &byte, 1) != 1) + return 1; + return 0; +} + +static bool remote_read(size_t own_page) +{ + int ready_pipe[2], done_pipe[2], status, fd, shared_fd, self_fd; + unsigned char *shared; + uint64_t self_entry; + size_t offset = own_page == 4096 ? 4096 : 0; + size_t target = own_page == 4096 ? 16384 : 4096; + struct ready ready; + uint64_t actual[NPAGE]; + pid_t child; + bool ok; + + shared_fd = memfd_create("mixed-pagemap", 0); + if (shared_fd < 0 || ftruncate(shared_fd, 16384)) + return false; + shared = mmap(NULL, 16384, PROT_READ | PROT_WRITE, MAP_SHARED, + shared_fd, 0); + self_fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (shared == MAP_FAILED || self_fd < 0) + return false; + memset(shared, 0x83, 16384); + if (!entries(self_fd, (uintptr_t)shared + offset, own_page, &self_entry, + 1)) + return false; + close(self_fd); + if (pipe(ready_pipe) || pipe(done_pipe)) + return false; + child = fork(); + if (!child) { + char output[16], input[16], expected[16], shared_arg[16]; + + close(ready_pipe[0]); + close(done_pipe[1]); + snprintf(output, sizeof(output), "%d", ready_pipe[1]); + snprintf(input, sizeof(input), "%d", done_pipe[0]); + snprintf(expected, sizeof(expected), "%zu", target); + snprintf(shared_arg, sizeof(shared_arg), "%d", shared_fd); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(2); + execl("/proc/self/exe", "/proc/self/exe", "--remote", output, + input, expected, shared_arg, NULL); + _exit(3); + } + close(ready_pipe[1]); + close(done_pipe[0]); + ok = child > 0 && + read(ready_pipe[0], &ready, sizeof(ready)) == sizeof(ready); + fd = open_proc(child, "pagemap", O_RDONLY); + ok = ok && ready.page == target && fd >= 0 && + entries(fd, ready.address, target, actual, NPAGE) && + !memcmp(actual, ready.entries, sizeof(actual)) && + scan(fd, ready.address, target); + if (!ok) + printf("# remote geometry/read failure: target=%zu reported=%zu fd=%d errno=%d\n", + target, ready.page, fd, errno); + if (ok) { + size_t target_offset = target == 4096 ? 4096 : 0; + + /* Compare physical byte addresses, not differently sized PFNs. */ + ok &= (self_entry & FRAME) && (ready.shared_entry & FRAME) && + (self_entry & FRAME) * own_page - offset == + (ready.shared_entry & FRAME) * target - + target_offset; + if (!ok) + printf("# shared PFN mismatch: own_nonzero=%d target_nonzero=%d byte_delta=%lld\n", + !!(self_entry & FRAME), + !!(ready.shared_entry & FRAME), + (long long)((self_entry & FRAME) * own_page - + offset - + (ready.shared_entry & FRAME) * + target + + target_offset)); + } + if (fd >= 0) + close(fd); + write(done_pipe[1], "x", 1); + close(done_pipe[1]); + close(ready_pipe[0]); + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + munmap(shared, 16384); + close(shared_fd); + return ok; +} + +int main(int argc, char **argv) +{ + size_t page = getauxval(AT_PAGESZ); + unsigned char *p; + uint64_t values[NPAGE], again[NPAGE]; + int fd, clear, i; + bool ok; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--remote")) + return remote_worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), atoi(argv[5])); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + ksft_print_header(); + ksft_set_plan(6); + p = mapping(page); + fd = open_proc(getpid(), "pagemap", O_RDONLY); + if (!p || fd < 0) + ksft_exit_fail_msg("proc mapping setup: %s\n", strerror(errno)); + ok = entries(fd, (uintptr_t)p, page, values, NPAGE); + ok &= (values[0] & PRESENT) && (values[1] & PRESENT) && !values[2] && + (values[3] & GUARD) && !(values[3] & PRESENT) && + (values[4] & PRESENT); + ok &= (values[0] & EXCLUSIVE) && (values[1] & EXCLUSIVE) && + (values[4] & EXCLUSIVE); + ksft_test_result( + ok, + "pagemap reports each target leaf, hole, guard and PROT_NONE entry\n"); + ok = true; + for (i = 0; i < NPAGE; i++) { + uint64_t value; + + ok &= entries(fd, (uintptr_t)p + i * page, page, &value, 1) && + value == values[i]; + } + errno = 0; + ok &= pread(fd, again, 7, 0) == -1 && errno == EINVAL; + ksft_test_result( + ok, + "pagemap per-entry offsets agree with batch reads and reject short entries\n"); + ksft_test_result( + scan(fd, (uintptr_t)p, page), + "PAGEMAP_SCAN ranges, max_pages and alignment use target granules\n"); + clear = open_proc(getpid(), "clear_refs", O_WRONLY); + ok = clear >= 0; + for (i = 1; ok && i <= 3; i++) { + char operation = '0' + i; + + ok &= write(clear, &operation, 1) == 1; + } + if (clear >= 0) + close(clear); + ok &= entries(fd, (uintptr_t)p, page, again, NPAGE) && + !memcmp(values, again, sizeof(values)) && p[0] == 0x67 && + p[page] == 0x67; + ksft_test_result( + ok, + "clear_refs traverses short and split VMAs without changing mappings or bytes\n"); + ksft_test_result( + remote_read(page), + "opposite-ABI reader uses target pagemap and scan geometry\n"); + { + pid_t child = fork(); + int status; + + if (!child) { + uint64_t value = 0; + int unprivileged; + + if (setuid(1000) || prctl(PR_SET_DUMPABLE, 1)) + _exit(2); + unprivileged = open_proc(getpid(), "pagemap", O_RDONLY); + ok = unprivileged >= 0 && + entries(unprivileged, (uintptr_t)p, page, &value, + 1) && + (value & PRESENT) && !(value & FRAME); + if (!ok) + printf("# unprivileged fd=%d errno=%d flags=%#llx\n", + unprivileged, errno, + (unsigned long long)(value & ~FRAME)); + _exit(!ok); + } + ok = child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + ksft_test_result( + ok, + "unprivileged pagemap reads retain flags and hide physical addresses\n"); + } + close(fd); + munmap(p, NPAGE * page); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/reclaim-advice.c b/tools/testing/selftests/arm64/user4k/reclaim-advice.c new file mode 100644 index 00000000000000..92f28d49b830c4 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/reclaim-advice.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, result = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} + +static unsigned long swapped_kb(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long result = 0; + + if (!f) + return 0; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmSwap: %lu kB", &result) == 1) + break; + fclose(f); + return result; +} + +static int advise(void *addr, size_t length, int operation, const char *name) +{ + int ret = madvise(addr, length, operation); + + if (ret) + printf("# %s failed: errno=%d (%s)\n", name, errno, + strerror(errno)); + return ret; +} + +static unsigned char pattern(size_t i, unsigned int round) +{ + return (i ^ (i >> 12) ^ (i >> 20) ^ (round * 0x55)) & 255; +} + +static size_t swapped_pages(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page, swapped = 0; + + if (!values || fd < 0 || + pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) != + (ssize_t)(count * sizeof(*values))) { + swapped = SIZE_MAX; + } else { + for (i = 0; i < count; i++) + swapped += !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + } + if (fd >= 0) + close(fd); + free(values); + return swapped; +} + +static void diagnose_resident(unsigned char *p, size_t length, size_t page) +{ + int pm = open("/proc/self/pagemap", O_RDONLY); + int flags = open("/proc/kpageflags", O_RDONLY); + int count = open("/proc/kpagecount", O_RDONLY); + + for (size_t i = 0; i < length / page; i++) { + uint64_t entry = 0, state = 0, maps = 0; + unsigned long native_pfn; + + if (pread(pm, &entry, 8, ((uintptr_t)p / page + i) * 8) != 8 || + !(entry & (1ULL << 63))) + continue; + native_pfn = (entry & ((1ULL << 55) - 1)) * page / 16384; + if (pread(flags, &state, 8, native_pfn * 8) != 8 || + pread(count, &maps, 8, native_pfn * 8) != 8) + printf("# resident metadata read failed: errno=%d\n", + errno); + printf("# resident leaf=%zu native_pfn=%#lx flags=%#llx maps=%llu\n", + i, native_pfn, (unsigned long long)state, + (unsigned long long)maps); + } + close(pm); + close(flags); + close(count); +} + +/* PAGEOUT drains only the calling CPU's pending LRU additions. */ +static bool drain_pageout_cpus(unsigned char *p, size_t length) +{ + cpu_set_t allowed, one; + bool ok = true; + + if (sched_getaffinity(0, sizeof(allowed), &allowed)) + return false; + for (unsigned int cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (!CPU_ISSET(cpu, &allowed)) + continue; + CPU_ZERO(&one); + CPU_SET(cpu, &one); + if (sched_setaffinity(0, sizeof(one), &one)) { + ok = false; + break; + } + ok &= advise(p, length, MADV_COLD, "CPU drain COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "CPU drain PAGEOUT") == 0; + } + ok &= sched_setaffinity(0, sizeof(allowed), &allowed) == 0; + return ok; +} + +static bool swapped_pagemap(unsigned char *p, size_t length, size_t page) +{ + uint64_t *values = calloc(length / page, sizeof(*values)); + int fd = open("/proc/self/pagemap", O_RDONLY); + size_t i, count = length / page; + bool ok = values && fd >= 0; + + if (ok) + ok = pread(fd, values, count * sizeof(*values), + (uintptr_t)p / page * sizeof(*values)) == + (ssize_t)(count * sizeof(*values)); + for (i = 0; ok && i < count; i++) { + uint64_t frame = values[i] & ((1ULL << 55) - 1); + + ok &= !!(values[i] & (1ULL << 62)) && + !(values[i] & (1ULL << 63)); + if (page == 4096) { + ok &= ((frame >> 5) & 3) == + (((uintptr_t)p / page + i) & 3); + if (i && (((uintptr_t)p / page + i) & 3)) + ok &= (values[i] & ((1ULL << 55) - 1)) == + (values[i - 1] & ((1ULL << 55) - 1)) + + (1U << 5); + } + } + if (fd >= 0) + close(fd); + free(values); + printf("%s - %zuK pagemap reports swapped leaves and exact quarter offsets\n", + ok ? "ok" : "not ok", page / 1024); + return ok; +} + +static bool check_swap_pss(pid_t pid, void *p, size_t length, + unsigned long expected_swap, + unsigned long expected_pss) +{ + char path[64], line[512]; + unsigned long start, end, swap = 0, pss = 0; + unsigned int fields = 0; + bool found = false, ok; + FILE *f; + + snprintf(path, sizeof(path), "/proc/%d/smaps", pid); + f = fopen(path, "re"); + if (!f) + return false; + while (fgets(line, sizeof(line), f)) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) { + if (found) + break; + found = start == (uintptr_t)p && + end == (uintptr_t)p + length; + continue; + } + if (!found) + continue; + if (sscanf(line, "Swap: %lu", &swap) == 1) + fields |= 1; + else if (sscanf(line, "SwapPss: %lu", &pss) == 1) + fields |= 2; + } + fclose(f); + ok = found && fields == 3 && swap == expected_swap && + pss == expected_pss; + if (!ok) + printf("# swap stats found=%d fields=%u Swap=%lu/%lu SwapPss=%lu/%lu\n", + found, fields, swap, expected_swap, pss, expected_pss); + return ok; +} + +static bool swapped_accounting(unsigned char *p, size_t length, size_t page, + bool forked) +{ + int ready[2], go[2], status; + pid_t child; + char byte; + bool ok = check_swap_pss(getpid(), p, length, length / 1024, + length / 1024); + + if (!forked || !ok) + goto out; + if (pipe(ready) || pipe(go)) + return false; + child = fork(); + if (child < 0) + return false; + if (!child) { + close(ready[0]); + close(go[1]); + if (read(go[0], &byte, 1) != 1) + _exit(1); + p[page] ^= 0x37; + if (write(ready[1], "x", 1) != 1 || read(go[0], &byte, 1) != 1) + _exit(2); + _exit(0); + } + close(ready[1]); + close(go[0]); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 2048); + ok &= check_swap_pss(child, p, length, length / 1024, length / 2048); + ok &= write(go[1], "x", 1) == 1 && read(ready[0], &byte, 1) == 1; + ok &= check_swap_pss(getpid(), p, length, length / 1024, + (length + page) / 2048); + ok &= check_swap_pss(child, p, length, (length - page) / 1024, + (length - page) / 2048); + write(go[1], "x", 1); + close(go[1]); + close(ready[0]); + ok &= waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status); + ok &= check_swap_pss(getpid(), p, length, length / 1024, length / 1024); +out: + printf("%s - %zuK SwapPss %s\n", ok ? "ok" : "not ok", page / 1024, + forked ? "fork, one-leaf swap-in/COW and child exit" : + "exclusive quarters"); + return ok; +} + +int main(int argc, char **argv) +{ + const size_t length = 4 * 1024 * 1024; + size_t page = getauxval(AT_PAGESZ), count = length / page, i, j, absent; + unsigned char *p, *vec, *reservation; + unsigned long before, after, swapped_before, swapped_after; + unsigned int round, tries; + size_t bad_bytes, bad_residency; + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc > 2 || (argc == 2 && strcmp(argv[1], "--expect-4k")) || + page != (argc == 2 ? 4096UL : 16384UL)) + return 2; + reservation = mmap(NULL, length + 2 * page, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) + return 3; + p = reservation + page; + vec = malloc(count); + if (!vec || mprotect(p, length, PROT_READ | PROT_WRITE)) + return 3; + for (round = 0; round < 4 && ok; round++) { + for (i = 0; i < length; i++) + p[i] = pattern(i, round); + before = swapouts(); + swapped_before = swapped_kb(); + ok = advise(p, length, MADV_COLD, "COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, "PAGEOUT") == 0; + after = before; + absent = 0; + for (tries = 0; ok && tries < 200; tries++) { + if (mincore(p, length, vec)) { + ok = false; + break; + } + absent = 0; + for (i = 0; i < count; i++) + absent += !(vec[i] & 1); + after = swapouts(); + if (after > before && + swapped_pages(p, length, page) == count) + break; + if (tries == 32) { + printf("# checking pending remote LRU additions before CPU sweep\n"); + diagnose_resident(p, length, page); + ok &= drain_pageout_cpus(p, length); + printf("# after CPU sweep: swapped=%zu/%zu\n", + swapped_pages(p, length, page), count); + } + /* PAGEOUT may skip busy folios; establish the full-swap fixture. */ + usleep(10000); + ok &= advise(p, length, MADV_COLD, "retry COLD") == 0; + ok &= advise(p, length, MADV_PAGEOUT, + "retry PAGEOUT") == 0; + } + printf("# pageout setup attempts=%u\n", tries + 1); + if (tries == 200) + diagnose_resident(p, length, page); + ok &= tries < 200; + swapped_after = swapped_kb(); + /* mincore also counts warm swap-cache data as resident. */ + ok &= after > before && swapped_after > swapped_before; + ok &= swapped_pagemap(p, length, page); + ok &= swapped_accounting(p, length, page, round == 1); + if (!round) + ok &= advise(p + page, page, MADV_WILLNEED, + "WILLNEED") == 0; + else { + if (round == 3) { + ok &= mlock(p, length) == 0; + ok &= swapped_pages(p, length, page) == 0; + } else { + ok &= advise(p, length, MADV_POPULATE_READ, + "POPULATE_READ") == 0; + } + ok &= mincore(p, length, vec) == 0; + bad_residency = 0; + for (i = 0; i < count; i++) + bad_residency += !(vec[i] & 1); + printf("# after %s: absent=%zu/%zu\n", + round == 3 ? "mlock" : "POPULATE_READ", + bad_residency, count); + ok &= bad_residency == 0; + } + bad_bytes = 0; + /* Nonsequential user-page order, every byte checked after swap. */ + for (i = 0; i < count; i++) { + size_t index = ((i * 61) & (count - 1)) * page; + + for (j = index; j < index + page; j++) + if (p[j] != pattern(j, round)) { + if (!bad_bytes) + printf("# first bad byte at %#zx: got=%#x expected=%#x\n", + j, p[j], + pattern(j, round)); + bad_bytes++; + ok = false; + } + } + if (round == 3) + ok &= munlock(p, length) == 0; + printf("# bad bytes=%zu\n", bad_bytes); + printf("%s - %zuK EL0 pageout round %u: pswpout=%lu->%lu VmSwap=%lu->%lu kB absent=%zu/%zu %s\n", + ok ? "ok" : "not ok", page / 1024, round, before, after, + swapped_before, swapped_after, absent, count, + round == 3 ? "mlock/readback" : + round ? "prefault/readback" : + "willneed/readback"); + } + munmap(reservation, length + 2 * page); + free(vec); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app-init.c b/tools/testing/selftests/arm64/user4k/runtime-app-init.c new file mode 100644 index 00000000000000..9d1c7c2c49351b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app-init.c @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +int main(int argc, char **argv) +{ + if (argc >= 4 && !strcmp(argv[1], "--exec")) { + if (prctl(SET_EXEC_PAGE_SIZE, strtoul(argv[2], NULL, 10), 0UL, + 0UL, 0UL)) + return 8; + execv(argv[3], argv + 3); + perror("exec"); + return 9; + } + if (getpid() != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + int failed = 0; + pid_t children[3]; + unsigned long sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned count = sizes[0] > 16384 ? 3 : 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + failed = 1; + goto done; + } + mkdir("/dev/shm", 0777); + mount("tmpfs", "/dev/shm", "tmpfs", 0, NULL); + for (unsigned i = 0; i < count; i++) { + children[i] = fork(); + if (!children[i]) { + char ps[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(3); + execl("/usr/bin/python3", "/usr/bin/python3", "-I", + "/runtime-app.py", ps, NULL); + perror("exec Python"); + _exit(4); + } + } + for (unsigned i = 0; i < count; i++) { + int status = 0; + int ok = children[i] > 0 && + waitpid(children[i], &status, 0) == children[i] && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - distro Python ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } +done: + if (failed) { + FILE *f = fopen("/trace4k", "r"); + if (f) { + char line[2048]; + while (fgets(line, sizeof(line), f)) + fputs(line, stdout); + fclose(f); + } + } + printf("GRANULE DISTRO RUNTIME %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/runtime-app.py b/tools/testing/selftests/arm64/user4k/runtime-app.py new file mode 100755 index 00000000000000..47fac03300efcc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/runtime-app.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: GPL-2.0-only +import os,sys,mmap,ctypes,threading,concurrent.futures,tempfile,subprocess,signal,zlib,gzip,lzma,hashlib,socket,select,json,sqlite3 +ps=int(sys.argv[1]);assert os.sysconf('SC_PAGE_SIZE')==ps==mmap.PAGESIZE,(ps,mmap.PAGESIZE) +libc=ctypes.CDLL(None,use_errno=True);libc.prctl.argtypes=[ctypes.c_int,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong,ctypes.c_ulong] +def passed(name):print(f'ok - distro Python {ps//1024}K {name}',flush=True) +passed('dynamic loader and page-size contract') +with mmap.mmap(-1, 2*ps, flags=0x08|mmap.MAP_ANONYMOUS) as droppable: + assert droppable[:]==bytes(2*ps) + droppable[:]=bytes([0x45])*(2*ps) + child=os.fork() + if child==0:os._exit(0 if droppable[:]==bytes(2*ps) else 9) + assert os.waitpid(child,0)[1]==0 + droppable.madvise(mmap.MADV_DONTNEED) + assert droppable[:]==bytes(2*ps) +libc.arc4random_buf.argtypes=[ctypes.c_void_p,ctypes.c_size_t] +random_buffer=ctypes.create_string_buffer(32) +random_samples=set() +for _ in range(128): + libc.arc4random_buf(random_buffer,len(random_buffer));random_samples.add(random_buffer.raw) +assert len(random_samples)==128 +passed('droppable write, fork wipe, discard and glibc random state') + +blob=bytes(range(256))*4096 +with tempfile.TemporaryDirectory() as tmp: + path=tmp+'/mapped' + with open(path,'w+b') as f: + f.truncate(8*ps) + with mmap.mmap(f.fileno(),2*ps,offset=ps) as m: + m[:]=bytes([0x53])*(2*ps);m.flush() + assert os.pread(f.fileno(),2*ps,ps)==m[:] + m.resize(3*ps);m[2*ps:]=bytes([0x72])*ps;m.flush() + assert os.pread(f.fileno(),ps,3*ps)==bytes([0x72])*ps + passed('file mmap offset, flush and resize') + with mmap.mmap(-1,4*ps,flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS) as m: + m[:]=bytes([0x39])*(4*ps) + c=os.fork() + if c==0: + m[ps:2*ps]=bytes([0x91])*ps + os._exit(0 if m[:ps]==bytes([0x39])*ps else 5) + assert os.waitpid(c,0)[1]==0 and m[:]==bytes([0x39])*(4*ps) + address=ctypes.addressof(ctypes.c_char.from_buffer(m)) + libc.mprotect.argtypes=[ctypes.c_void_p,ctypes.c_size_t,ctypes.c_int] + assert libc.mprotect(address+ps,ps,0)==0 + c=os.fork() + if c==0: + m[ps]=0 + os._exit(6) + status=os.waitpid(c,0)[1];assert os.WIFSIGNALED(status) and os.WTERMSIG(status)==signal.SIGSEGV + assert m[0]==0x39 and m[2*ps]==0x39 + assert libc.mprotect(address+ps,ps,mmap.PROT_READ|mmap.PROT_WRITE)==0 + passed('fork COW and independent protection fault') + def compressed(i): + d=blob+str(i).encode() + for compress,decompress in [(zlib.compress,zlib.decompress),(gzip.compress,gzip.decompress),(lambda d:lzma.compress(d,preset=1),lzma.decompress)]:assert decompress(compress(d))==d + return hashlib.sha256(d).hexdigest() + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results=list(pool.map(compressed,range(8))) + assert len(set(results))==8 + passed('threaded native extension allocation, compression and hashing') + db=sqlite3.connect(tmp+'/db') + db.execute('pragma journal_mode=wal');db.execute('pragma mmap_size=1048576') + db.execute('create table entries (n integer primary key, data blob)') + db.executemany('insert into entries values (?,?)',[(i,blob[:4096]) for i in range(128)]);db.commit() + assert db.execute('select count(*) from entries').fetchone()[0]==128 + assert all(row[0]==blob[:4096] for row in db.execute('select data from entries'));db.close() + passed('SQLite WAL and mapped database reads') + left,right=socket.socketpair() + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + task=pool.submit(lambda:(right.sendall(blob),right.shutdown(socket.SHUT_WR))) + out=bytearray() + while True: + assert select.select([left],[],[],5)[0] + part=left.recv(997) + if not part:break + out.extend(part) + task.result();assert bytes(out)==blob + left.close();right.close();passed('socket polling and fragmented transfers') + fd=os.memfd_create('cross-abi',0);os.ftruncate(fd,131072);os.pwrite(fd,b'parent',ps) + # Execute the child in the other ABI, retaining the same shared file bytes. + target=16384 if ps==4096 else 4096 + def select_abi(): + if libc.prctl(0x41555001,target,0,0,0):os._exit(7) + code="import os,mmap,sys;fd=int(sys.argv[1]);off=int(sys.argv[2]);want=int(sys.argv[3]);assert os.sysconf('SC_PAGE_SIZE')==want;m=mmap.mmap(fd,131072);assert m[off:off+6]==b'parent';m[off:off+5]=b'child';m.close()" + subprocess.run([sys.executable,'-I','-c',code,str(fd),str(ps),str(target)],pass_fds=(fd,),preexec_fn=select_abi,check=True) + assert os.pread(fd,5,ps)==b'child';os.close(fd);passed('cross-ABI dynamic exec and memfd sharing') + command="set -e; a=(); for ((i=0;i<2048;i++)); do a+=(\"value-$i\"); done; [[ ${#a[@]} == 2048 ]]; [[ ${a[2047]} == value-2047 ]]; printf 'bash-ok\\n'" + def same_abi(): + if libc.prctl(0x41555001,ps,0,0,0):os._exit(8) + result=subprocess.run(['/bin/bash','--noprofile','--norc','-c',command],preexec_fn=same_abi,stdout=subprocess.PIPE,check=True,text=True) + assert result.stdout=='bash-ok\n';passed('distro Bash dynamic exec, allocation and pipes') +print(f'DISTRO PYTHON {ps} PASS',flush=True) diff --git a/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c new file mode 100644 index 00000000000000..b040d8fc43a4f3 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-collapse-init.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef MADV_COLLAPSE +#define MADV_COLLAPSE 25 +#endif +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - collapse line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct command { + uint64_t op, offset, len, value; +}; +struct ready { + uint64_t base, head; +}; +static uint64_t entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t mapped_pfn(pid_t pid, uintptr_t address, size_t ps) +{ + char path[80]; + snprintf(path, sizeof(path), "/proc/%d/pagemap", pid); + uint64_t e = entry(path, address / ps); + CHECK(e & (1ULL << 63)); + return e & ((1ULL << 55) - 1); +} +static unsigned char *map(int file, size_t len, size_t align, off_t offset) +{ + void *reserve = mmap(NULL, len + align, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t addr = ((uintptr_t)reserve + align - 1) & ~(align - 1); + CHECK(!munmap(reserve, len + align)); + unsigned char *p = mmap((void *)addr, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, file, offset); + CHECK(p != MAP_FAILED); + return p; +} +static int register_range(unsigned char *p, size_t len, int minor) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = minor ? UFFD_FEATURE_MINOR_SHMEM : + 0 }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register r = { + .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + (minor ? UFFDIO_REGISTER_MODE_MINOR : 0) + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, &r)); + return fd; +} +static void copy_leaf(int fd, uintptr_t dst, unsigned char value) +{ + unsigned char src[4096]; + memset(src, value, sizeof(src)); + struct uffdio_copy c = { .dst = dst, + .src = (uintptr_t)src, + .len = sizeof(src) }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == sizeof(src)); +} +static void cpu(unsigned id) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(id, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static int worker(int file, int sock, size_t native, size_t len) +{ + CHECK(getauxval(AT_PAGESZ) == 4096); + alarm(90); + cpu(1); + unsigned char *p = map(file, len, len, 0); + unsigned char *q = map(file, len - 4096, len, 4096); + CHECK(p[0] == 0x61 && q[0] == 0x61); + uint64_t old_head = + mapped_pfn(getpid(), (uintptr_t)p, 4096) * 4096 / native; + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + int fd = register_range(p, len, 1); + for (size_t i = 4 * native; i < len; i += native) + copy_leaf(fd, (uintptr_t)p + i, 0x71); + int pipes[2]; + CHECK(!pipe(pipes)); + struct iovec iov = { p, 4096 }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == 4096); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, native)); + CHECK(entry("/proc/kpageflags", old_head) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!close(pipes[0]) && !close(pipes[1])); + struct ready r = { (uintptr_t)p, old_head }; + struct iovec v = { &r, sizeof(r) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(fd))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(fd)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(r)); + struct command cmd; + while (recv(sock, &cmd, sizeof(cmd), 0) == sizeof(cmd)) { + CHECK(cmd.offset < len && cmd.len <= len - cmd.offset); + unsigned char value = 0; + if (!cmd.op) + break; + if (cmd.op == 1) + value = *(volatile unsigned char *)(p + cmd.offset); + else if (cmd.op == 2) + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + cmd.offset, cmd.len)); + else if (cmd.op == 3) { + CHECK(cmd.offset >= 4096); + value = *(volatile unsigned char *)(q + cmd.offset - + 4096); + } else if (cmd.op == 4) { + CHECK(cmd.offset >= 4096); + memset(q + cmd.offset - 4096, cmd.value, cmd.len); + } else + CHECK(0); + CHECK(send(sock, &value, 1, 0) == 1); + } + CHECK(!munmap(q, len - 4096) && !munmap(p, len) && !close(fd) && + !close(file) && !close(sock)); + return 0; +} +static void send_command(int sock, uint64_t op, size_t offset, size_t len) +{ + struct command c = { .op = op, .offset = offset, .len = len }; + CHECK(send(sock, &c, sizeof(c), 0) == sizeof(c)); +} +static void reply(int sock, unsigned char expected) +{ + unsigned char value; + CHECK(recv(sock, &value, 1, 0) == 1 && value == expected); +} +static void fault(int sock, int fd, uintptr_t base, size_t offset, int minor, + unsigned char value) +{ + send_command(sock, 1, offset + 17, 0); + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg m; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &m, sizeof(m)) == sizeof(m)); + CHECK(m.event == UFFD_EVENT_PAGEFAULT && + m.arg.pagefault.address == base + offset && + m.arg.pagefault.flags == (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { base + offset, 4096 } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == 4096); + } else + copy_leaf(fd, base + offset, value); + reply(sock, value); +} +static unsigned long pmd_mapped(unsigned char *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + CHECK(f); + char *line = NULL; + size_t cap = 0; + unsigned long a, b, kb, result = 0; + int selected = 0; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &a, &b) == 2) + selected = a == (uintptr_t)p; + else if (selected && + sscanf(line, "ShmemPmdMapped: %lu kB", &kb) == 1) + result = kb * 1024; + } + free(line); + fclose(f); + return result; +} +static int collapse(unsigned char *p, size_t len) +{ + int ret; + unsigned tries = 0; + do { + errno = 0; + ret = madvise(p, len, MADV_COLLAPSE); + if (!ret || errno != EAGAIN) + break; + usleep(20000); + } while (++tries < 10); + printf("# collapse ret=%d errno=%d PMD-mapped=%lu\n", ret, + ret ? errno : 0, pmd_mapped(p)); + return ret; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + alarm(120); + cpu(0); + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, + "huge=within_size,size=256m")); + size_t native = getauxval(AT_PAGESZ), len = 0; + FILE *f = fopen("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", + "r"); + CHECK(f && fscanf(f, "%zu", &len) == 1); + fclose(f); + CHECK(native == 16384 && len == 32 * 1024 * 1024); + int file = open("/tmp/collapse", O_CREAT | O_RDWR | O_TRUNC, 0600); + CHECK(file >= 0 && !ftruncate(file, 4 * native)); + unsigned char *initial = map(file, 4 * native, 4 * native, 0); + memset(initial, 0x61, 4 * native); + CHECK(entry("/proc/kpageflags", + mapped_pfn(getpid(), (uintptr_t)initial, native)) & + (1ULL << KPF_COMPOUND_HEAD)); + CHECK(!munmap(initial, 4 * native) && !unlink("/tmp/collapse") && + !ftruncate(file, len)); + int sock[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, sock)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fd[24], so[24], ns[24], ls[24]; + close(sock[0]); + snprintf(fd, sizeof(fd), "%d", file); + snprintf(so, sizeof(so), "%d", sock[1]); + snprintf(ns, sizeof(ns), "%zu", native); + snprintf(ls, sizeof(ls), "%zu", len); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, 4096, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--worker", fd, so, ns, ls, NULL); + _exit(3); + } + close(sock[1]); + struct ready ready; + struct iovec v = { &ready, sizeof(ready) }; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct msghdr msg = { .msg_iov = &v, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control.bytes) }; + CHECK(recvmsg(sock[0], &msg, 0) == sizeof(ready) && + !(msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS); + int uffd; + memcpy(&uffd, CMSG_DATA(c), sizeof(uffd)); + unsigned char *p = map(file, len, len, 0); + int native_uffd = register_range(p, len, 0); + CHECK(collapse(p, len) == -1 && errno == EINVAL && pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 0, 1, 0x61); + fault(sock[0], uffd, ready.base, len - native, 1, 0x71); + uint64_t head = mapped_pfn(pid, ready.base, 4096) * 4096 / native; + CHECK(head != ready.head && + (entry("/proc/kpageflags", head) & (1ULL << KPF_COMPOUND_HEAD))); + CHECK(mapped_pfn(pid, ready.base + len - native, 4096) * 4096 / + native == + head + len / native - 1); + for (size_t i = 1; i < len / native; i++) + CHECK(entry("/proc/kpageflags", head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + /* Collapse retained the absent native unit and independent fine holes. */ + fault(sock[0], uffd, ready.base, native, 0, 0x92); + fault(sock[0], uffd, ready.base, 5 * native + 4096, 0, 0xa3); + puts("ok - PMD cache collapse preserves missing/minor state and neighbour data"); + /* Every native leaf now contains data: coarse exposure may populate zero subunits. */ + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + fault(sock[0], uffd, ready.base, 6 * native + 4096, 1, 0); + struct uffdio_copy duplicate = { .dst = ready.base + 6 * native + 8192, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST); + puts("ok - native PMD promotion exposes coarse leaf zeros without overwriting fine data"); + /* A new fine hole must invalidate the covering native PMD mapping. */ + send_command(sock[0], 2, 7 * native + 4096, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + fault(sock[0], uffd, ready.base, 7 * native + 4096, 0, 0xb4); + CHECK(p[7 * native] == 0x71 && p[7 * native + 4096] == 0xb4 && + p[7 * native + 8192] == 0); + CHECK(p[native] == 0x92 && p[5 * native + 4096] == 0xa3); + puts("ok - fine hole splits PMD alias and preserves populated neighbouring leaves"); + for (unsigned round = 0; round < 8; round++) { + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + size_t off = (16 + round) * native + 4096; + /* Prime CPU0's PMD translation before CPU1 punches the fine hole. */ + CHECK(*(volatile unsigned char *)(p + off) == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0); + send_command(sock[0], 2, off, 4096); + reply(sock[0], 0); + CHECK(pmd_mapped(p) == 0); + if (round % 3) { + /* Stale native or shifted fine translations would miss promotion. */ + if (round % 3 == 1) + memset(p + off, 0xc0 + round, 4096); + else { + struct command write = { 4, off, 4096, + 0xc0 + round }; + CHECK(send(sock[0], &write, sizeof(write), 0) == + sizeof(write)); + reply(sock[0], 0); + } + struct uffdio_copy collision = { .dst = ready.base + + off, + .src = (uintptr_t)p, + .len = 4096 }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &collision) == -1 && + errno == EEXIST); + fault(sock[0], uffd, ready.base, off, 1, 0xc0 + round); + } else + fault(sock[0], uffd, ready.base, off, 0, 0xc0 + round); + CHECK(p[off - 4096] == 0x71 && p[off] == 0xc0 + round && + p[off + 4096] == 0); + send_command(sock[0], 3, off + 17, 0); + reply(sock[0], 0xc0 + round); + } + CHECK(!collapse(p, len) && pmd_mapped(p) == len); + for (size_t i = 0; i < len; i++) { + size_t page = i / native, low = i % native; + unsigned char expected = + page < 4 ? + (page == 1 ? (low < 4096 ? 0x92 : 0) : 0x61) : + (low < 4096 ? 0x71 : 0); + if (low >= 4096 && low < 8192) { + if (page == 5) + expected = 0xa3; + if (page == 7) + expected = 0xb4; + if (page >= 16 && page < 24) + expected = 0xc0 + page - 16; + } + CHECK(p[i] == expected); + } + puts("ok - eight cross-CPU PMD cycles, native/shifted-fine writes and complete 32MiB bytes"); + send_command(sock[0], 0, 0, 0); + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(uffd) && !close(native_uffd) && !munmap(p, len) && + !close(file) && !close(sock[0])); + puts("SHMEM COLLAPSE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-hole-init.c b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c new file mode 100644 index 00000000000000..8831cc30a22c35 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-hole-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem hole line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct access { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *opaque) +{ + struct access *a = opaque; + a->value = *a->p; + return NULL; +} +static uint64_t pagemap(pid_t pid, uintptr_t address, size_t granule) +{ + char name[80]; + uint64_t entry; + snprintf(name, sizeof(name), "/proc/%d/pagemap", pid); + int fd = open(name, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &entry, 8, address / granule * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static void fill(int fd, unsigned char *dst, unsigned char *src, size_t ps) +{ + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + CHECK(!ioctl(fd, UFFDIO_COPY, &c) && c.copy == (long long)ps); +} +static void resolve(int fd, unsigned char *p, size_t ps, int minor, + unsigned char value) +{ + struct access a = { .p = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &a)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (minor ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (minor) { + struct uffdio_continue c = { .range = { (uintptr_t)p, ps } }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && + c.mapped == (long long)ps); + } else { + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, value, ps); + fill(fd, p, source, ps); + free(source); + } + CHECK(!pthread_join(thread, NULL) && a.value == value); +} +struct command { + uint64_t op, offset, len, value; +}; +static int alias_worker(int file, int input, int output, size_t len, + size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == native); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED); + /* Establish every coarse PTE before the fine process punches its hole. */ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == 0x61); + uint64_t base = (uintptr_t)p; + CHECK(write(output, &base, sizeof(base)) == sizeof(base)); + struct command cmd; + while (read(input, &cmd, sizeof(cmd)) == sizeof(cmd)) { + CHECK(cmd.offset <= len && cmd.len <= len - cmd.offset); + if (!cmd.op) + break; + if (cmd.op == 2) + memset(p + cmd.offset, cmd.value, cmd.len); + for (size_t i = 0; i < cmd.len; i++) + CHECK(p[cmd.offset + i] == cmd.value); + CHECK(write(output, "y", 1) == 1); + } + CHECK(!munmap(p, len)); + return 0; +} +static void tell(int input, int output, uint64_t op, size_t offset, size_t len, + unsigned value) +{ + struct command cmd = { op, offset, len, value }; + char reply; + CHECK(write(input, &cmd, sizeof(cmd)) == sizeof(cmd)); + if (op) + CHECK(read(output, &reply, 1) == 1 && reply == 'y'); +} +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "r"); + char key[80]; + unsigned long value, result = 0; + CHECK(f); + while (fscanf(f, "%79s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + result = value; + fclose(f); + return result; +} +static unsigned long mapping_swap(void *address) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end, value, result = 0; + int selected = 0; + CHECK(f); + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + selected = start == (uintptr_t)address; + else if (selected && sscanf(line, "Swap: %lu kB", &value) == 1) + result = value; + } + free(line); + fclose(f); + return result; +} +static void swapped_missing(size_t ps, size_t native) +{ + int file = memfd_create("partial-uffd-swap", 0); + CHECK(file >= 0 && !ftruncate(file, native)); + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(ps); + CHECK(p != MAP_FAILED && source); + memset(source, 0x71, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, native }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + fill(fd, p, source, ps); + unsigned long before = swapouts(), usage = 0, written; + unsigned tries; + for (tries = 0; tries < 100; tries++) { + CHECK(!madvise(p, native, MADV_PAGEOUT)); + usleep(20000); + usage = mapping_swap(p); + if (usage >= native / 1024 && swapouts() > before) + break; + } + written = swapouts() - before; + CHECK(usage >= native / 1024 && written > 0); + /* The selected cache index is swapped; its unfilled neighbour stays missing. */ + resolve(fd, p + ps, ps, 0, 0x92); + resolve(fd, p, ps, 1, 0x71); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0x71 && p[ps + i] == 0x92); + CHECK(mapping_swap(p) == 0); + printf("ok - shared uffd %zuK on %zuK swapped cache=%lukB writes=%lu attempts=%u missing COPY preserves neighbour\n", + ps / 1024, native / 1024, usage, written, tries + 1); + CHECK(!close(fd) && !munmap(p, native) && !close(file)); + free(source); +} +static int worker(size_t ps, size_t native) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + if (getenv("SHMEM_HOLE_SWAP") && ps < native) + swapped_missing(ps, native); + size_t len = 4 * native; + int file = memfd_create("hole-contract", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *source = malloc(len); + CHECK(source); + memset(source, 0x61, len); + CHECK(pwrite(file, source, len, 0) == (ssize_t)len); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *q = mmap(NULL, len - ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, ps); + unsigned char *cow = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE, file, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && cow != MAP_FAILED); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + for (size_t i = 0; i < len - ps; i += ps) + CHECK(q[i] == 0x61); + memset(cow + ps, 0xe1, ps); + int commands[2], replies[2]; + CHECK(!pipe(commands) && !pipe(replies)); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], in[24], out[24], size[24], ns[24]; + close(commands[1]); + close(replies[0]); + snprintf(f, sizeof(f), "%d", file); + snprintf(in, sizeof(in), "%d", commands[0]); + snprintf(out, sizeof(out), "%d", replies[1]); + snprintf(size, sizeof(size), "%zu", len); + snprintf(ns, sizeof(ns), "%zu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--alias", f, in, out, size, ns, NULL); + _exit(3); + } + close(commands[0]); + close(replies[1]); + uint64_t alias_base; + CHECK(read(replies[0], &alias_base, sizeof(alias_base)) == + sizeof(alias_base)); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(pagemap(alias, alias_base + ps, native) & (1ULL << 63)); + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + CHECK(!(pagemap(getpid(), (uintptr_t)p + ps, ps) & (1ULL << 63))); + CHECK(!(pagemap(getpid(), (uintptr_t)q, ps) & (1ULL << 63))); + CHECK(!(pagemap(alias, alias_base + ps, native) & (1ULL << 63))); + /* Fine neighbour PTEs survive, while COW retains its independent bytes. */ + CHECK(pagemap(getpid(), (uintptr_t)p, ps) & (1ULL << 63)); + CHECK(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63)); + CHECK(p[0] == 0x61 && p[2 * ps] == 0x61 && cow[ps] == 0xe1); + resolve(fd, p + ps, ps, 0, 0xa2); + CHECK(q[0] == 0xa2 && cow[ps] == 0xe1); + tell(commands[1], replies[0], 1, ps, ps, 0xa2); + tell(commands[1], replies[0], 1, 0, ps, 0x61); + tell(commands[1], replies[0], 1, 2 * ps, ps, 0x61); + /* Refaulting a coarse writable alias makes its covered units populated. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + tell(commands[1], replies[0], 2, ps, ps, 0xb7); + struct uffdio_copy collision = { .dst = (uintptr_t)p + ps, + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(fd, UFFDIO_COPY, &collision) == -1 && errno == EEXIST); + resolve(fd, p + ps, ps, 1, 0xb7); + printf("ok - shmem hole %zuK on %zuK precise PTEs, shifted alias, COW, coarse revoke/refault and COPY collision\n", + ps / 1024, native / 1024); + /* Stop the alias before resize, so it cannot intentionally promote holes. */ + tell(commands[1], replies[0], 0, 0, 0, 0); + close(commands[1]); + close(replies[0]); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + size_t cut = 2 * ps + ps / 2; + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + /* Partial final user leaf remains valid, with only truncated bytes zero. */ + if (!(pagemap(getpid(), (uintptr_t)p + 2 * ps, ps) & (1ULL << 63))) + resolve(fd, p + 2 * ps, ps, 1, 0x61); + CHECK(p[2 * ps] == 0x61 && p[cut - 1] == 0x61 && p[cut] == 0 && + p[3 * ps - 1] == 0); + resolve(fd, p + 3 * ps, ps, 0, 0xc3); + CHECK(q[2 * ps] == 0xc3); + CHECK(cow[ps] == 0xe1); + printf("ok - shmem resize %zuK on %zuK partial tail retained, shrink/regrow missing leaf refilled\n", + ps / 1024, native / 1024); + CHECK(!close(fd) && !munmap(p, len) && !munmap(q, len - ps) && + !munmap(cow, len) && !close(file)); + free(source); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return alias_worker(atoi(argv[2]), atoi(argv[3]), atoi(argv[4]), + strtoul(argv[5], NULL, 10), + strtoul(argv[6], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + int swap_test = argc == 2 && !strcmp(argv[1], "--swap"); + if (getpid() != 1 || (argc != 1 && !swap_test)) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + if (swap_test) { + struct stat st; + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + CHECK(!setenv("SHMEM_HOLE_SWAP", "1", 1)); + } + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native == 65536 ? 3U : 2U); i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[24], ns[24]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - shmem hole ABI %luK status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + failed |= !ok; + } + if (swap_test) + CHECK(!swapoff("/dev/vda")); + printf("SHMEM HOLE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pin-init.c b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c new file mode 100644 index 00000000000000..51deda0fde52cc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pin-init.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pin line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + printf("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + printf("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} + +static void fragment_geometry(unsigned char *p, size_t ps, int input) +{ + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + 123, 3 * ps - 246 }; + unsigned char *src = malloc(3 * ps), *expected = malloc(3 * ps); + CHECK(src && expected); + memset(expected, 0x61, 3 * ps); + memset(src, 0xb7, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + ps + 71, ps + 31, 17, + true)); + memset(expected + ps + 71, 0xb7, ps + 31); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + ps + 71, ps + 31, 9, + true)); + CHECK(pread(input, src, ps + 31, 9) == (ssize_t)(ps + 31)); + for (size_t i = 0; i < ps + 31; i++) + CHECK(src[i] == 0xb7); + struct iovec vec[2] = { { p + 2 * ps + 17, ps - 400 }, + { p + ps + 91, ps - 600 } }; + memset(src, 0xd5, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_READV_FIXED, input, vec, 2, 13, true)); + for (unsigned i = 0; i < 2; i++) + memset(expected + ((unsigned char *)vec[i].iov_base - p), 0xd5, + vec[i].iov_len); + CHECK(!memcmp(p, expected, 3 * ps)); + memset(src, 0xc8, 3 * ps); + CHECK(pwrite(input, src, 3 * ps, 0) == (ssize_t)(3 * ps)); + CHECK(request(&r, IORING_OP_WRITEV_FIXED, input, vec, 2, 23, true)); + size_t total = vec[0].iov_len + vec[1].iov_len; + CHECK(pread(input, src, total, 23) == (ssize_t)total); + for (size_t i = 0; i < total; i++) + CHECK(src[i] == 0xd5); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + destroy(&r); + free(src); + free(expected); + memset(p, 0x61, 3 * ps); + printf("ok - %zuK fixed/vector IO uses user fragment geometry and preserves guards\n", + ps / 1024); +} + +struct punch_worker { + int file; + size_t off, ps; + pthread_barrier_t start; +}; +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void *punch_repeatedly(void *arg) +{ + struct punch_worker *w = arg; + bind_cpu(1); + pthread_barrier_wait(&w->start); + for (unsigned i = 0; i < 256; i++) { + CHECK(!fallocate(w->file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + w->off, w->ps)); + sched_yield(); + } + return NULL; +} +static void pin_hole_concurrency(struct ring *r, int file, int input, + unsigned char *p, size_t off, size_t ps) +{ + struct punch_worker w = { .file = file, .off = off, .ps = ps }; + pthread_t thread; + CHECK(!pthread_barrier_init(&w.start, NULL, 2)); + bind_cpu(0); + CHECK(!pthread_create(&thread, NULL, punch_repeatedly, &w)); + pthread_barrier_wait(&w.start); + for (unsigned i = 0; i < 256; i++) + CHECK(request(r, IORING_OP_READ_FIXED, input, p + off, ps, 0, + true)); + CHECK(!pthread_join(thread, NULL) && + !pthread_barrier_destroy(&w.start)); + printf("ok - %zuK concurrent fixed writes and 256 hole punches on separate CPUs\n", + ps / 1024); +} + +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, off = native + ps; + bool retained = ps < native; + int file = memfd_create("pin-cache", 0), + input = memfd_create("pin-input", 0); + CHECK(file >= 0 && input >= 0 && !ftruncate(file, len) && + !ftruncate(input, ps)); + unsigned char *src = malloc(len), *back = malloc(len); + CHECK(src && back); + memset(src, 0x61, len); + CHECK(pwrite(file, src, len, 0) == (ssize_t)len); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + CHECK(p != MAP_FAILED && !madvise(p, len, MADV_NOHUGEPAGE)); + CHECK(p[off] == 0x61); + fragment_geometry(p, ps, input); + memset(src, 0xb7, ps); + CHECK(pwrite(input, src, ps, 0) == (ssize_t)ps); + struct ring r; + CHECK(setup(&r, 8, 0)); + struct iovec reg = { p + off, ps }; + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_REGISTER_BUFFERS, + ®, 1)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register ur = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, &ur)); + if (resize) + CHECK(!ftruncate(file, off) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, ps)); + if (retained && !resize) + pin_hole_concurrency(&r, file, input, p, off, ps); + /* Fixed IO accesses its original FOLL_PIN reference after PTE removal. */ + CHECK(request(&r, IORING_OP_READ_FIXED, input, p + off, ps, 0, true)); + /* Also prove the original pinned object contains the completed write. */ + CHECK(request(&r, IORING_OP_WRITE_FIXED, input, p + off, ps, 0, true)); + CHECK(pread(input, back, ps, 0) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(back[i] == 0xb7); + CHECK(!syscall(SYS_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + memset(src, 0xc8, ps); + struct uffdio_copy c = { .dst = (uintptr_t)(p + off), + .src = (uintptr_t)src, + .len = ps }; + errno = 0; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# pin %zuK/%zuK resize=%d retained=%d COPY=%d errno=%d bytes=%lld\n", + ps / 1024, native / 1024, resize, retained, ret, errno, + (long long)c.copy); + if (retained) + CHECK(ret == -1 && errno == EEXIST && c.copy == -EEXIST); + else + CHECK(!ret && c.copy == (long long)ps); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= off && i < off + ps ? + (retained ? 0xb7 : 0xc8) : + resize && i >= off ? 0 : + 0x61; + CHECK(back[i] == expected); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == (retained ? 0xb7 : 0xc8)); + /* A new punch after unpin can once again be populated by the pager. */ + CHECK(!fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, + ps)); + c.copy = 0; + CHECK(!ioctl(uffd, UFFDIO_COPY, &c) && c.copy == (long long)ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[off + i] == 0xc8); + destroy(&r); + CHECK(!close(uffd) && !munmap(p, len) && !close(file) && !close(input)); + free(src); + free(back); + printf("ok - writable pin %zuK on %zuK resize=%d retention and detach\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], g[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(g, sizeof(g), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, g, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("ok - shmem writable pin fixture complete"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c new file mode 100644 index 00000000000000..f321878396f061 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-pressure-init.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CG "/sys/fs/cgroup/pressure" +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem pressure line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static void put(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + CHECK(fd >= 0); + CHECK(write(fd, value, strlen(value)) == (ssize_t)strlen(value) && + !close(fd)); +} +static unsigned long long get(const char *path, const char *key) +{ + FILE *f = fopen(path, "r"); + CHECK(f); + char name[128]; + unsigned long long value, result = 0; + if (!key) + CHECK(fscanf(f, "%llu", &result) == 1); + else + while (fscanf(f, "%127s %llu", name, &value) == 2) + if (!strcmp(key, name)) { + result = value; + break; + } + fclose(f); + return result; +} +static void copy_leaf(int uffd, unsigned char *dst, unsigned char *src, + size_t ps, unsigned char value) +{ + memset(src, value, ps); + struct uffdio_copy c = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = ps }; + int ret = ioctl(uffd, UFFDIO_COPY, &c); + if (ret) + printf("# COPY destination=%p errno=%d transferred=%lld\n", dst, + errno, (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); +} +static void checkpoint(int out, int in) +{ + char token; + CHECK(write(out, "R", 1) == 1 && read(in, &token, 1) == 1 && + token == 'G'); +} +static int worker(size_t native, size_t ps, int out, int in) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(120); + size_t records = (16UL << 20) / native, + stride = ps < native ? native : 2 * native; + size_t len = records * stride; + int file = memfd_create("pressure-partial-cache", 0); + CHECK(file >= 0 && !ftruncate(file, len)); + unsigned char *p = + mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, file, 0); + unsigned char *source = malloc(stride), *back = malloc(stride); + CHECK(p != MAP_FAILED && source && back); + CHECK(!madvise(p, len, MADV_NOHUGEPAGE)); + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + for (size_t i = 0; i < records; i++) + copy_leaf(uffd, p + i * stride, source, ps, 0xa1); + checkpoint(out, in); + /* Revisit in permuted order, filling holes in reclaimed cache backing. */ + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051) & (records - 1); + copy_leaf(uffd, p + slot * stride + ps, source, ps, 0xb2); + for (size_t j = 0; j < ps; j++) + CHECK(p[slot * stride + j] == 0xa1 && + p[slot * stride + ps + j] == 0xb2); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i += 2) { + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + i * stride, ps)); + copy_leaf(uffd, p + i * stride, source, ps, 0xc3); + } + checkpoint(out, in); + for (size_t i = 0; i < records; i++) { + size_t slot = (i * 4051 + 31) & (records - 1); + CHECK(pread(file, back, stride, slot * stride) == + (ssize_t)stride); + for (size_t j = 0; j < stride; j++) { + unsigned char expected = + j < ps ? (slot & 1 ? 0xa1 : 0xc3) : + j < 2 * ps ? 0xb2 : + 0; + CHECK(back[j] == expected); + } + struct uffdio_copy duplicate = { + .dst = (uintptr_t)(p + slot * stride), + .src = (uintptr_t)source, + .len = ps + }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && + errno == EEXIST && duplicate.copy == -EEXIST); + } + checkpoint(out, in); + CHECK(!close(uffd) && !munmap(p, len) && !close(file)); + free(source); + free(back); + printf("ok - shmem pressure %zuK on %zuK partial-cache refill, holes, duplicate rejection and every byte\n", + ps / 1024, native / 1024); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 6 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4]), + atoi(argv[5])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL) && + !mount("sysfs", "/sys", "sysfs", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) && + !mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL)); + put("/sys/fs/cgroup/cgroup.subtree_control", "+memory"); + CHECK(!mkdir(CG, 0755)); + put(CG "/memory.max", "8388608"); + put(CG "/memory.swap.max", "134217728"); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) { + int ready[2], go[2]; + CHECK(!pipe(ready) && !pipe(go)); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + close(ready[0]); + close(go[1]); + char n[24], p[24], r[24], g[24]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(g, sizeof(g), "%d", go[0]); + put(CG "/cgroup.procs", "0"); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--worker", n, p, r, g, NULL); + _exit(127); + } + close(ready[1]); + close(go[0]); + for (unsigned phase = 0; phase < 4; phase++) { + char token; + CHECK(read(ready[0], &token, 1) == 1 && token == 'R'); + unsigned long long swap = get(CG "/memory.swap.current", + NULL), + max = get(CG "/memory.events", + "max"); + unsigned long long scan = get(CG "/memory.stat", + "pgscan"), + steal = get(CG "/memory.stat", + "pgsteal"); + printf("# pressure %zuK phase=%u resident=%llu swap=%llu max=%llu scan=%llu steal=%llu\n", + ps / 1024, phase, + get(CG "/memory.current", NULL), swap, max, scan, + steal); + CHECK(swap && max && scan && steal && + !get(CG "/memory.events", "oom") && + !get(CG "/memory.events", "oom_kill")); + CHECK(write(go[1], "G", 1) == 1); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status)); + close(ready[0]); + close(go[1]); + /* Closed/unmapped file identity must release all of its cache/swap. */ + for (unsigned i = 0; + i < 3000 && (get(CG "/memory.swap.current", NULL) || + get(CG "/memory.stat", "shmem")); + i++) + usleep(20000); + CHECK(!get(CG "/memory.swap.current", NULL) && + !get(CG "/memory.stat", "shmem")); + } + CHECK(!swapoff("/dev/vda")); + puts("SHMEM PRESSURE CONTRACT PASS"); + sync(); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-race-init.c b/tools/testing/selftests/arm64/user4k/shmem-race-init.c new file mode 100644 index 00000000000000..5109654b6ec216 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-race-init.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define ROUNDS 512 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem race line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +struct control { + unsigned start, stop, ready; + unsigned long aliases; +}; +struct state { + struct control *control; + int file, uffd, resize; + size_t ps, native, len; + unsigned char *p, *q, *source; + unsigned copied, existed, eof, alloc_eof, retry; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned value) +{ + __atomic_store_n(p, value, __ATOMIC_RELEASE); +} +static void bind_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +static void start_wait(struct control *c) +{ + __atomic_add_fetch(&c->ready, 1, __ATOMIC_RELEASE); + while (!load(&c->start)) + sched_yield(); +} +static void *mutate(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (s->resize) + CHECK(!ftruncate(s->file, s->ps) && + !ftruncate(s->file, s->len)); + else + CHECK(!fallocate(s->file, + FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_KEEP_SIZE, + s->ps, s->ps)); + sched_yield(); + } + return NULL; +} +static void *populate(void *arg) +{ + struct state *s = arg; + bind_cpu(1); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + struct uffdio_copy c = { .dst = (uintptr_t)(s->p + s->ps), + .src = (uintptr_t)s->source, + .len = s->ps }; + errno = 0; + int ret = ioctl(s->uffd, UFFDIO_COPY, &c); + if (!ret) { + CHECK(c.copy == (long long)s->ps); + s->copied++; + } else { + CHECK(ret == -1 && c.copy == -errno); + if (errno == EEXIST) + s->existed++; + else if (errno == EFAULT && s->resize) + s->eof++; + /* Native shmem allocator preserves its NULL/ENOMEM EOF ABI. */ + else if (errno == ENOMEM && s->resize && + s->ps == s->native) + s->alloc_eof++; + else if (errno == EAGAIN) + s->retry++; + else { + printf("# unexpected COPY error=%d bytes=%lld\n", + errno, (long long)c.copy); + CHECK(0); + } + } + sched_yield(); + } + return NULL; +} +static int valid_byte(unsigned char value) +{ + return value == 0 || value == 0x61 || value == 0xa2; +} +static void *fine_alias(void *arg) +{ + struct state *s = arg; + bind_cpu(0); + start_wait(s->control); + for (unsigned i = 0; i < ROUNDS; i++) { + if (!s->resize) { + CHECK(!madvise(s->q, s->ps, MADV_DONTNEED)); + CHECK(valid_byte(*(volatile unsigned char *)s->q)); + } else { + /* The retained prefix remains inside EOF throughout resize. */ + CHECK(*(volatile unsigned char *)s->p == 0x61); + } + sched_yield(); + } + return NULL; +} +static int native_alias(int file, int control, size_t native, size_t ps, + int resize) +{ + CHECK(getauxval(AT_PAGESZ) == native); + bind_cpu(1); + alarm(60); + struct control *c = mmap(NULL, native, PROT_READ | PROT_WRITE, + MAP_SHARED, control, 0); + unsigned char *p = mmap(NULL, 4 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + CHECK(c != MAP_FAILED && p != MAP_FAILED && + !madvise(p, 4 * native, MADV_NOHUGEPAGE)); + start_wait(c); + unsigned long cycles = 0; + while (!load(&c->stop)) { + CHECK(!madvise(p, 2 * native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)p == 0x61); + if (!resize) + CHECK(valid_byte(*(volatile unsigned char *)(p + ps))); + cycles++; + sched_yield(); + } + c->aliases = cycles; + CHECK(!munmap(p, 4 * native) && !munmap(c, native)); + return 0; +} +static int target(size_t native, size_t ps, int resize) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(60); + struct state s = { + .native = native, .ps = ps, .len = 4 * native, .resize = resize + }; + s.file = memfd_create("race-cache", 0); + int control = memfd_create("race-control", 0); + CHECK(s.file >= 0 && control >= 0 && !ftruncate(s.file, s.len) && + !ftruncate(control, native)); + s.control = mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, + control, 0); + s.source = malloc(s.len); + CHECK(s.control != MAP_FAILED && s.source); + memset(s.source, 0x61, s.len); + CHECK(pwrite(s.file, s.source, s.len, 0) == (ssize_t)s.len); + s.p = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, 0); + s.q = mmap(NULL, s.len - ps, PROT_READ | PROT_WRITE, MAP_SHARED, s.file, + ps); + CHECK(s.p != MAP_FAILED && s.q != MAP_FAILED && + !madvise(s.p, s.len, MADV_NOHUGEPAGE)); + CHECK(s.p[0] == + 0x61); /* Avoid blocking the resize prefix reader on UFFD. */ + unsigned char *cow = MAP_FAILED; + if (!resize) { + cow = mmap(NULL, s.len, PROT_READ | PROT_WRITE, MAP_PRIVATE, + s.file, 0); + CHECK(cow != MAP_FAILED); + memset(cow + ps, 0xe1, ps); + } + s.uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(s.uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(s.uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)s.p, s.len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(s.uffd, UFFDIO_REGISTER, ®)); + memset(s.source, 0xa2, ps); + pid_t alias = fork(); + CHECK(alias >= 0); + if (!alias) { + char f[24], c[24], n[24], p[24], r[12]; + snprintf(f, sizeof(f), "%d", s.file); + snprintf(c, sizeof(c), "%d", control); + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, native, 0, 0, 0)); + execl("/init", "init", "--alias", f, c, n, p, r, NULL); + _exit(127); + } + pthread_t mutation, copy, fine; + CHECK(!pthread_create(&mutation, NULL, mutate, &s) && + !pthread_create(©, NULL, populate, &s) && + !pthread_create(&fine, NULL, fine_alias, &s)); + while (load(&s.control->ready) != 4) + sched_yield(); + store(&s.control->start, 1); + CHECK(!pthread_join(mutation, NULL) && !pthread_join(copy, NULL) && + !pthread_join(fine, NULL)); + store(&s.control->stop, 1); + int status; + CHECK(waitpid(alias, &status, 0) == alias && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(s.control->aliases && + s.copied + s.existed + s.eof + s.alloc_eof + s.retry == ROUNDS); + printf("# race %zuK/%zuK resize=%d COPY=%u EEXIST=%u EOF=%u native-alloc-EOF=%u retry=%u native-refaults=%lu\n", + ps / 1024, native / 1024, resize, s.copied, s.existed, s.eof, + s.alloc_eof, s.retry, s.control->aliases); + /* Once concurrent actors stop, exact cache/PTE state must be recoverable. */ + CHECK(!ftruncate(s.file, s.len)); + CHECK(!fallocate(s.file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, ps, + ps)); + memset(s.source, 0xd4, ps); + struct uffdio_copy fill = { .dst = (uintptr_t)(s.p + ps), + .src = (uintptr_t)s.source, + .len = ps }; + CHECK(!ioctl(s.uffd, UFFDIO_COPY, &fill) && fill.copy == (long long)ps); + unsigned char *back = malloc(s.len); + CHECK(back && pread(s.file, back, s.len, 0) == (ssize_t)s.len); + for (size_t i = 0; i < s.len; i++) { + unsigned char expected = i >= ps && i < 2 * ps ? 0xd4 : + resize && i >= ps ? 0 : + 0x61; + if (back[i] != expected) { + printf("# byte=%zu actual=%#x expected=%#x\n", i, + back[i], expected); + CHECK(0); + } + } + for (size_t i = 0; i < ps; i++) { + CHECK(s.p[ps + i] == 0xd4 && s.q[i] == 0xd4); + if (!resize) + CHECK(cow[ps + i] == 0xe1); + } + CHECK(!close(s.uffd) && !munmap(s.p, s.len) && + !munmap(s.q, s.len - ps)); + if (!resize) + CHECK(!munmap(cow, s.len)); + CHECK(!munmap(s.control, native) && !close(control) && !close(s.file)); + free(back); + free(s.source); + printf("ok - concurrent shmem %zuK on %zuK resize=%d cache bytes, aliases and private COW\n", + ps / 1024, native / 1024, resize); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 7 && !strcmp(argv[1], "--alias")) + return native_alias(atoi(argv[2]), atoi(argv[3]), + strtoul(argv[4], NULL, 10), + strtoul(argv[5], NULL, 10), atoi(argv[6])); + if (argc == 5 && !strcmp(argv[1], "--target")) + return target(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), atoi(argv[4])); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + size_t native = getauxval(AT_PAGESZ); + for (size_t ps = 4096; ps <= native; ps *= 4) + for (int resize = 0; resize < 2; resize++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char n[24], p[24], r[12]; + snprintf(n, sizeof(n), "%zu", native); + snprintf(p, sizeof(p), "%zu", ps); + snprintf(r, sizeof(r), "%d", resize); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, ps, 0, 0, 0)); + execl("/init", "init", "--target", n, p, r, + NULL); + _exit(127); + } + int status; + CHECK(waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + puts("SHMEM RACE CONTRACT PASS"); + reboot(RB_POWER_OFF); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-thp-init.c b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c new file mode 100644 index 00000000000000..e08ea92103fcc2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-thp-init.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - shmem THP line=%d errno=%d\n", \ + __LINE__, errno); \ + exit(1); \ + } \ + } while (0) +static uint64_t read_entry(const char *path, uint64_t index) +{ + uint64_t value; + int fd = open(path, O_RDONLY); + CHECK(fd >= 0 && pread(fd, &value, 8, index * 8) == 8 && !close(fd)); + return value; +} +static uint64_t pfn(void *p, size_t ps) +{ + uint64_t entry = read_entry("/proc/self/pagemap", (uintptr_t)p / ps); + CHECK(entry & (1ULL << 63)); + return entry & ((1ULL << 55) - 1); +} +static unsigned char *map_aligned(int fd, size_t len) +{ + void *reserve = mmap(NULL, 2 * len, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + uintptr_t address = ((uintptr_t)reserve + len - 1) & ~(len - 1); + CHECK(!munmap(reserve, 2 * len)); + unsigned char *p = mmap((void *)address, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(p != MAP_FAILED); + return p; +} +struct reader { + volatile unsigned char *p; + unsigned char value; +}; +static void *read_byte(void *arg) +{ + struct reader *r = arg; + r->value = *r->p; + return NULL; +} +static int target(int file, size_t native, size_t ps, int hold, int zero, + int resize, uint64_t head) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(30); + size_t len = 4 * native, target_off = resize ? 2 * native : native; + size_t cut = native + ps / 2; + unsigned char *p = map_aligned(file, len); + for (size_t i = 0; i < len; i += ps) + CHECK(p[i] == 0x61); + CHECK(pfn(p, ps) * ps / native == head); + int pipes[2]; + CHECK(!pipe(pipes)); + if (hold) { + struct iovec iov = { p, ps }; + CHECK(vmsplice(pipes[1], &iov, 1, 0) == (ssize_t)ps); + } + int uffd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(uffd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(uffd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(uffd, UFFDIO_REGISTER, ®)); + if (resize) + CHECK(!ftruncate(file, cut) && !ftruncate(file, len)); + else + CHECK(!fallocate(file, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + native, ps)); + uint64_t flags = read_entry("/proc/kpageflags", head); + printf("# THP %zuK native=%zuK hold=%d zero=%d resize=%d head_flags=%#llx\n", + ps / 1024, native / 1024, hold, zero, resize, + (unsigned long long)flags); + if (hold) + CHECK((flags & + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + struct reader r = { .p = p + target_off + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct pollfd pollfd = { .fd = uffd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(uffd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && !msg.arg.pagefault.flags && + msg.arg.pagefault.address == (uintptr_t)(p + target_off)); + unsigned char *source = malloc(ps); + CHECK(source); + memset(source, 0x92, ps); + int ret; + struct uffdio_copy invalid = { .dst = (uintptr_t)(p + target_off), + .src = 1, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &invalid) == -1 && errno == EFAULT && + invalid.copy == -EFAULT); + if (zero) { + struct uffdio_zeropage z = { + .range = { (uintptr_t)(p + target_off), ps } + }; + ret = ioctl(uffd, UFFDIO_ZEROPAGE, &z); + printf("# ZEROPAGE returned=%d errno=%d bytes=%lld\n", ret, + errno, (long long)z.zeropage); + CHECK(!ret && z.zeropage == (long long)ps); + } else { + struct uffdio_copy c = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + ret = ioctl(uffd, UFFDIO_COPY, &c); + printf("# COPY returned=%d errno=%d bytes=%lld\n", ret, errno, + (long long)c.copy); + CHECK(!ret && c.copy == (long long)ps); + } + CHECK(!pthread_join(thread, NULL) && r.value == (zero ? 0 : 0x92)); + memset(source, 0xd4, ps); + struct uffdio_copy duplicate = { .dst = (uintptr_t)(p + target_off), + .src = (uintptr_t)source, + .len = ps }; + errno = 0; + CHECK(ioctl(uffd, UFFDIO_COPY, &duplicate) == -1 && errno == EEXIST && + duplicate.copy == -EEXIST); + /* File reads verify zeroed tail holes without faulting unfilled UFFD leaves. */ + unsigned char *back = malloc(len); + CHECK(back); + CHECK(pread(file, back, len, 0) == (ssize_t)len); + for (size_t i = 0; i < len; i++) { + unsigned char expected = i >= target_off && + i < target_off + ps ? + (zero ? 0 : 0x92) : + resize && i >= cut ? 0 : + 0x61; + CHECK(back[i] == expected); + if (!resize || (i >= target_off && i < target_off + ps)) + CHECK(p[i] == expected); + } + free(back); + if (hold) { + CHECK(read(pipes[0], source, ps) == (ssize_t)ps); + for (size_t i = 0; i < ps; i++) + CHECK(source[i] == 0x61); + } + CHECK(!close(pipes[0]) && !close(pipes[1]) && !close(uffd) && + !munmap(p, len) && !close(file)); + free(source); + printf("ok - THP %zuK on %zuK hold=%d zero=%d resize=%d missing refill and all neighbour bytes\n", + ps / 1024, native / 1024, hold, zero, resize); + return 0; +} +static int create_file(size_t native, uint64_t *head) +{ + int fd = open("/tmp/huge", O_CREAT | O_TRUNC | O_RDWR, 0600); + CHECK(fd >= 0); + CHECK(!ftruncate(fd, 4 * native)); + unsigned char *p = map_aligned(fd, 4 * native); + CHECK(!madvise(p, 4 * native, MADV_HUGEPAGE)); + memset(p, 0x61, 4 * native); + *head = pfn(p, native); + uint64_t f = read_entry("/proc/kpageflags", *head); + printf("# initial head=%llu flags=%#llx VA=%p native=%zu\n", + (unsigned long long)*head, (unsigned long long)f, p, native); + CHECK((f & ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))) == + ((1ULL << KPF_COMPOUND_HEAD) | (1ULL << KPF_THP))); + for (unsigned i = 1; i < 4; i++) { + CHECK(pfn(p + i * native, native) == *head + i); + CHECK(read_entry("/proc/kpageflags", *head + i) & + (1ULL << KPF_COMPOUND_TAIL)); + } + /* Keep the name until allocation: unlinked mappings use anon-shmem policy. */ + CHECK(!unlink("/tmp/huge") && !munmap(p, 4 * native)); + return fd; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 9 && !strcmp(argv[1], "--target")) + return target(atoi(argv[2]), strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10), atoi(argv[5]), + atoi(argv[6]), atoi(argv[7]), + strtoull(argv[8], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("tmpfs", "/tmp", "tmpfs", 0, "huge=within_size,size=32m")); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned abi = 0; abi < (native == 65536 ? 3U : 2U); abi++) + for (unsigned hold = 0; hold < 2; hold++) + for (unsigned zero = 0; zero < 2; zero++) + for (unsigned resize = 0; resize < 2; + resize++) { + uint64_t head; + int fd = create_file(native, &head); + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char fs[24], ns[24], ps[24], + hs[24], zs[24], rs[24], + pfs[24]; + snprintf(fs, sizeof(fs), "%d", + fd); + snprintf(ns, sizeof(ns), "%zu", + native); + snprintf(ps, sizeof(ps), "%zu", + sizes[abi]); + snprintf(hs, sizeof(hs), "%u", + hold); + snprintf(zs, sizeof(zs), "%u", + zero); + snprintf(rs, sizeof(rs), "%u", + resize); + snprintf(pfs, sizeof(pfs), + "%llu", + (unsigned long long) + head); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, + sizes[abi], 0UL, + 0UL, 0UL)); + execl("/init", "/init", + "--target", fs, ns, ps, + hs, zs, rs, pfs, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == + pid && + WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - THP ABI %zuK hold=%u zero=%u resize=%u status=%#x\n", + ok ? "ok" : "not ok", + sizes[abi] / 1024, hold, zero, + resize, status); + failed |= !ok; + CHECK(!close(fd)); + } + printf("SHMEM THP CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c new file mode 100644 index 00000000000000..73ff33d5e0ba13 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/shmem-uffd-swap-init.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-shmem-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-shmem-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int fault_setting(const char *name, const char *value) +{ + char path[160]; + snprintf(path, sizeof(path), "/sys/kernel/debug/failslab/%s", name); + int fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + ssize_t n = write(fd, value, strlen(value)); + close(fd); + return n == (ssize_t)strlen(value) ? 0 : -1; +} +static int configure_failslab(bool enable) +{ + if (!enable) + return fault_setting("probability", "0"); + /* The KUnit worker selects itself only around metadata writeout. */ + return fault_setting("task-filter", "Y") || + fault_setting("verbose", "0") || fault_setting("times", "-1") || + fault_setting("interval", "1") || fault_setting("space", "0") || + fault_setting("probability", "100"); +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + if (!strcmp(operation, "--memcg-setup-helper")) { + if (mount("cgroup2", "/sys/fs/cgroup", "cgroup2", 0, NULL) && + errno != EBUSY) + return 1; + int fd = open("/sys/fs/cgroup/cgroup.subtree_control", + O_WRONLY | O_CLOEXEC); + if (fd < 0) + return 1; + ssize_t n = write(fd, "+memory", 7); + close(fd); + if (n != 7) + return 1; + return mkdir("/sys/fs/cgroup/uffd-metadata", 0755) && + errno != EEXIST; + } + if (!strcmp(operation, "--failslab-on-helper")) + return configure_failslab(true); + if (!strcmp(operation, "--failslab-off-helper")) + return configure_failslab(false); + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +int main(int argc, char **argv) +{ + struct stat st; + bool active = false; + int ret = 1; + if (argc == 2 && (!strcmp(argv[1], "--swapon-helper") || + !strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--failslab-on-helper") || + !strcmp(argv[1], "--failslab-off-helper") || + !strcmp(argv[1], "--memcg-setup-helper"))) + return swap_control_helper(argv[1]); + if (getpid() != 1 || argc != 1) + return 2; + setvbuf(stdout, NULL, _IONBF, 0); + unsigned long ps = getauxval(AT_PAGESZ); + if (ps != 16384 && ps != 65536) + goto out; + if (mount("sysfs", "/sys", "sysfs", 0, NULL) || + mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + ret = run_swap_suite() ? 1 : 0; +out: + if (active && swapoff("/dev/vda")) + ret = 1; + printf("SHMEM UFFD STATE SWAP %s errno=%d\n", ret ? "FAIL" : "PASS", + errno); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c new file mode 100644 index 00000000000000..282ba6268b25d9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +static bool use_tcp, use_ipv6, use_udp; + +static int connected_pair(int sockets[2]) +{ + struct sockaddr_in a4 = { .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK) }; + struct sockaddr_in6 a6 = { .sin6_family = AF_INET6, + .sin6_addr = IN6ADDR_LOOPBACK_INIT }; + struct sockaddr_storage peer; + struct sockaddr *address = use_ipv6 ? (void *)&a6 : (void *)&a4; + socklen_t size = use_ipv6 ? sizeof(a6) : sizeof(a4), + peer_size = sizeof(peer); + int domain = use_ipv6 ? AF_INET6 : AF_INET; + int listener, saved; + + if (!use_tcp && !use_udp) + return socketpair(AF_UNIX, SOCK_STREAM, 0, sockets); + listener = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (listener < 0) + return -1; + sockets[0] = -1; + if (bind(listener, address, size) || + getsockname(listener, address, &size)) + goto error; + if (!use_udp && listen(listener, 1)) + goto error; + sockets[0] = socket(domain, use_udp ? SOCK_DGRAM : SOCK_STREAM, 0); + if (sockets[0] < 0 || connect(sockets[0], address, size)) + goto error; + if (use_udp) { + if (getsockname(sockets[0], (void *)&peer, &peer_size) || + connect(listener, (void *)&peer, peer_size)) + goto error; + sockets[1] = listener; + return 0; + } + sockets[1] = accept(listener, NULL, NULL); + if (sockets[1] < 0) + goto error; + close(listener); + return 0; +error: + saved = errno; + if (sockets[0] >= 0) + close(sockets[0]); + close(listener); + errno = saved; + return -1; +} + +static unsigned char value(size_t off) +{ + return (off * 13 + (off >> 12) * 31) & 255; +} +static void fill(unsigned char *p, size_t n) +{ + for (size_t i = 0; i < n; i++) + p[i] = value(i); +} +static bool check(const unsigned char *p, size_t n, size_t off) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value(i + off)) { + ksft_print_msg( + "data mismatch at %zu: got %02x expected %02x\n", + i + off, p[i], value(i + off)); + return false; + } + return true; +} +static bool read_exact(int fd, unsigned char *p, size_t n) +{ + while (n) { + ssize_t got = read(fd, p, n); + + if (got <= 0) { + ksft_print_msg( + "read fd=%d got=%zd remaining=%zu errno=%d\n", + fd, got, n, errno); + return false; + } + p += got; + n -= got; + } + return true; +} +static bool replace(unsigned char *p, size_t n) +{ + if (munmap(p, n) || + mmap(p, n, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0) != p) + return false; + memset(p, 0xec, n); + return true; +} +static bool cow(unsigned char *p, size_t n) +{ + pid_t child = fork(); + int status = 0; + + if (!child) + _exit(0); + if (child < 0 || waitpid(child, &status, 0) != child || status) + return false; + memset(p, 0xab, n); + return true; +} +static void report(bool ok, size_t ps, const char *name) +{ + ksft_test_result(ok, "%zuK %s\n", ps / 1024, name); +} +static bool tls_pair(int sockets[2], unsigned int version) +{ + struct tls12_crypto_info_aes_gcm_128 crypto = { + .info.version = version, + .info.cipher_type = TLS_CIPHER_AES_GCM_128, + }; + struct timeval timeout = { .tv_sec = 5 }; + int no_padding = 1; + + if (connected_pair(sockets)) + return false; + return !setsockopt(sockets[0], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[1], IPPROTO_TCP, TCP_ULP, "tls", 4) && + !setsockopt(sockets[0], SOL_TLS, TLS_TX, &crypto, + sizeof(crypto)) && + !setsockopt(sockets[1], SOL_TLS, TLS_RX, &crypto, + sizeof(crypto)) && + (version != TLS_1_3_VERSION || + !setsockopt(sockets[1], SOL_TLS, TLS_RX_EXPECT_NO_PAD, + &no_padding, sizeof(no_padding))) && + !setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); +} + +static void tls_checks(size_t ps, unsigned char *p, size_t len, + unsigned char *back) +{ + int sockets[2], fds[2]; + char name[160]; + + if (pipe(fds)) + ksft_exit_fail_msg("TLS pipe errno=%d\n", errno); + for (unsigned int v = 0; v < 2; v++) { + unsigned int version = v ? TLS_1_3_VERSION : TLS_1_2_VERSION; + struct iovec vec; + bool ok; + + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg("TLS %x setup errno=%d\n", version, + errno); + for (unsigned int mode = 0; mode < 2; mode++) { + const size_t count = mode ? 4096 : 4096 - 29; + const size_t offset = ps + (mode ? 0 : 29); + + fill(p, len); + vec = (struct iovec){ p + offset, count }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)count || + splice(fds[0], NULL, sockets[0], NULL, count, + SPLICE_F_MORE) != (ssize_t)count) + ksft_exit_fail_msg("TLS MORE splice errno=%d\n", + errno); + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "TLS pending plaintext precondition errno=%d\n", + errno); + ok = mode ? cow(p + ps, 4096) : replace(p + ps, ps); + ok &= send(sockets[0], "!", 1, 0) == 1; + ok &= read_exact(sockets[1], back, count + 1) && + check(back, count, offset) && back[count] == '!'; + snprintf(name, sizeof(name), + "TLS 1.%u pending plaintext survives %s", + v + 2, + mode ? "fork/COW" : "unmap/replacement"); + report(ok, ps, name); + } + { + const size_t bytes = 3 * 16384 + 123, prefix = 137; + const size_t mapping = (bytes + ps - 1) & ~(ps - 1); + unsigned char *source = + mmap(NULL, mapping, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *received = malloc(bytes + prefix); + + if (source == MAP_FAILED || !received) + ksft_exit_fail_msg( + "TLS multi-record allocation\n"); + fill(source, mapping); + ok = send(sockets[0], source, prefix, 0) == + (ssize_t)prefix; + vec = (struct iovec){ source, bytes }; + if (vmsplice(fds[1], &vec, 1, 0) != (ssize_t)bytes || + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) != + (ssize_t)bytes) + ksft_exit_fail_msg( + "TLS multi-record splice errno=%d\n", + errno); + ok &= replace(source, mapping); + ok &= read_exact(sockets[1], received, + bytes + prefix) && + check(received, prefix, 0) && + check(received + prefix, bytes, 0); + snprintf( + name, sizeof(name), + "TLS 1.%u ordinary send and multiple spliced records", + v + 2); + report(ok, ps, name); + munmap(source, mapping); + free(received); + } + { + const size_t bytes = 5003; + int file = memfd_create("tls-receive", 0); + unsigned char *mapped; + + if (file < 0 || ftruncate(file, 3 * ps)) + ksft_exit_fail_msg("TLS receive file setup\n"); + mapped = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, file, 0); + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive file mapping\n"); + memset(mapped, 0x5c, 3 * ps); + fill(p, len); + ok = send(sockets[0], p, bytes, 0) == (ssize_t)bytes && + read_exact(sockets[1], mapped + 17, bytes); + ok &= !msync(mapped, 3 * ps, MS_SYNC) && + pread(file, back, bytes + 34, 0) == + (ssize_t)(bytes + 34) && + check(back + 17, bytes, 0); + for (unsigned int i = 0; i < 17; i++) + ok &= back[i] == 0x5c && + back[bytes + 17 + i] == 0x5c; + snprintf( + name, sizeof(name), + "TLS 1.%u decrypts into shared file with neighboring bytes intact", + v + 2); + report(ok, ps, name); + munmap(mapped, 3 * ps); + close(file); + } + { + unsigned char *mapped = + mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec vectors[4]; + struct msghdr message = { .msg_iov = vectors, + .msg_iovlen = 4 }; + + if (mapped == MAP_FAILED) + ksft_exit_fail_msg( + "TLS receive iovec mapping\n"); + memset(mapped, 0x81, 6 * ps); + vectors[0] = (struct iovec){ mapped + 13, 5003 }; + vectors[1] = (struct iovec){ NULL, 0 }; + vectors[2] = + (struct iovec){ mapped + 2 * ps + 7, 2222 }; + vectors[3] = + (struct iovec){ mapped + 4 * ps + 5, 1010 }; + fill(p, len); + ok = send(sockets[0], p, 8235, 0) == 8235 && + recvmsg(sockets[1], &message, MSG_WAITALL) == + 8235 && + check(mapped + 13, 5003, 0) && + check(mapped + 2 * ps + 7, 2222, 5003) && + check(mapped + 4 * ps + 5, 1010, 7225); + ok &= mapped[12] == 0x81 && mapped[5016] == 0x81 && + mapped[2 * ps + 6] == 0x81 && + mapped[2 * ps + 2229] == 0x81 && + mapped[4 * ps + 4] == 0x81 && + mapped[4 * ps + 1015] == 0x81; + snprintf( + name, sizeof(name), + "TLS 1.%u scattered receive, empty iovec and subpage boundaries", + v + 2); + report(ok, ps, name); + munmap(mapped, 6 * ps); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 64 && ok; round++) { + if (!tls_pair(sockets, version)) + ksft_exit_fail_msg( + "TLS teardown setup errno=%d\n", errno); + fill(p, len); + vec = (struct iovec){ p + ps, 4096 }; + ok = vmsplice(fds[1], &vec, 1, 0) == 4096 && + splice(fds[0], NULL, sockets[0], NULL, 4096, + SPLICE_F_MORE) == 4096; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x75, ps); + } + snprintf(name, sizeof(name), + "TLS 1.%u repeated pending-record teardown and reuse", + v + 2); + report(ok, ps, name); + } + close(fds[0]); + close(fds[1]); +} + +static int exercise(void) +{ + const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; + unsigned char *p, *back = malloc(len); + int fds[2], dup[2], file, sockets[2]; + struct iovec v[3]; + struct stat self, init; + struct timeval timeout = { .tv_sec = 5 }; + bool ok; + ssize_t result; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("setup\n"); + ksft_print_header(); + ksft_set_plan(use_tcp ? 22 : 10); + ksft_print_msg("transport=%s family=%s\n", + use_tcp ? "TCP" : + use_udp ? "UDP" : + "Unix stream", + use_ipv6 ? "IPv6" : "IPv4/Unix"); + fill(p, len); + v[0] = (struct iovec){ NULL, 0 }; + v[1] = (struct iovec){ p + 17, ps - 17 }; + v[2] = (struct iovec){ p + 2 * ps + 31, 2 * ps - 31 }; + result = vmsplice(fds[1], v, 3, 0); + if (result != (ssize_t)(3 * ps - 48)) + ksft_exit_fail_msg("vmsplice returned %zd errno=%d\n", result, + errno); + ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && + check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); + report(ok, ps, "unaligned fragmented iovecs and empty segment"); + v[0] = (struct iovec){ p, 16384 }; + ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); + { + int pending = 0; + + ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && + pending == 16384; + } + if (result > 0) + ok &= read_exact(fds[0], back, result) && + check(back, result, 0); + report(ok, ps, + "contiguous quarters retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; + result = vmsplice(fds[1], v, 1, 0); + if (result != (ssize_t)(3 * ps - 11) || + tee(fds[0], dup[1], result, 0) != result) + ksft_exit_fail_msg("tee setup\n"); + ok = replace(p, len); + ok &= read_exact(fds[0], back, 19) && check(back, 19, 11); + ok &= read_exact(fds[0], back + 19, result - 19) && + check(back, result, 11); + ok &= read_exact(dup[0], back, result) && check(back, result, 11); + report(ok, ps, + "tee and partial reads retain data across unmap/replacement"); + fill(p, len); + v[0] = (struct iovec){ p, ps }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && cow(p, ps); + ok &= read_exact(fds[0], back, ps) && check(back, ps, 0); + report(ok, ps, "pipe GET retains pre-COW data after fork sharer exit"); + fill(p, len); + file = memfd_create("splice-target", 0); + v[0] = (struct iovec){ p + ps + 29, 2 * ps - 29 }; + result = vmsplice(fds[1], v, 1, SPLICE_F_GIFT); + ok = file >= 0 && result == (ssize_t)(2 * ps - 29) && + splice(fds[0], NULL, file, NULL, result, SPLICE_F_MOVE) == result; + ok &= pread(file, back, result, 0) == result && + check(back, result, ps + 29); + report(ok, ps, "gifted fragments splice to buffered file"); + close(file); + v[0] = (struct iovec){ p, ps }; + v[1] = (struct iovec){ (void *)0x1000, ps }; + result = vmsplice(fds[1], v, 2, 0); + ok = result == (ssize_t)ps && read_exact(fds[0], back, ps) && + check(back, ps, 0); + report(ok, ps, "partial vmsplice before inaccessible iovec"); + if (connected_pair(sockets)) + ksft_exit_fail_msg("socketpair errno=%d\n", errno); + setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); + for (unsigned int mode = 0; mode < 3; mode++) { + const size_t bytes = mode == 1 ? ps : + (use_udp && ps == 4096 ? + 6 * ps - 256 : + 2 * ps - 256); + const size_t offset = mode == 1 ? 0 : ps + 29; + int cork = 1; + + if ((use_tcp || use_udp) && mode < 2 && + setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) + ksft_exit_fail_msg("TCP_CORK setup errno=%d\n", errno); + fill(p, len); + v[0] = (struct iovec){ p + offset, bytes }; + ok = vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, + SPLICE_F_NONBLOCK) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg("socket splice setup errno=%d\n", + errno); + if (mode == 2) { + ok = splice(sockets[1], NULL, fds[1], NULL, bytes, 0) == + (ssize_t)bytes && + tee(fds[0], dup[1], bytes, 0) == (ssize_t)bytes; + if (!ok) + ksft_exit_fail_msg( + "socket-to-pipe setup errno=%d\n", + errno); + } + if (use_tcp && mode < 2) { + int unsent = 0; + + if (ioctl(sockets[0], SIOCOUTQNSD, &unsent) || + unsent != (int)bytes) + ksft_exit_fail_msg( + "TCP unsent precondition: %d expected %zu errno=%d\n", + unsent, bytes, errno); + } + if (use_udp && mode < 2) { + errno = 0; + if (recv(sockets[1], back, len, MSG_DONTWAIT) != -1 || + errno != EAGAIN) + ksft_exit_fail_msg( + "UDP corked precondition errno=%d\n", + errno); + } + ok &= mode == 1 ? cow(p, ps) : replace(p + ps, 2 * ps); + if ((use_tcp || use_udp) && mode < 2) { + cork = 0; + if (setsockopt(sockets[0], + use_tcp ? IPPROTO_TCP : IPPROTO_UDP, + use_tcp ? TCP_CORK : UDP_CORK, &cork, + sizeof(cork))) { + ksft_print_msg("uncork errno=%d\n", errno); + ok = false; + } + } + ok &= read_exact(mode == 2 ? fds[0] : sockets[1], back, + bytes) && + check(back, bytes, offset); + if (mode == 2) + ok &= read_exact(dup[0], back, bytes) && + check(back, bytes, offset); + report(ok, ps, + mode == 0 ? + "socket retains quarters across partial unmap/replacement" : + mode == 1 ? + "socket GET retains pre-COW data after fork sharer exit" : + "socket-to-pipe and tee retain independent quarter ownership"); + } + close(sockets[0]); + close(sockets[1]); + ok = true; + for (unsigned int round = 0; round < 128 && ok; round++) { + fill(p, len); + ok = !connected_pair(sockets); + v[0] = (struct iovec){ p + ps, ps }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && + splice(fds[0], NULL, sockets[0], NULL, ps, 0) == + (ssize_t)ps; + close(sockets[1]); + close(sockets[0]); + ok &= !madvise(p + ps, ps, MADV_DONTNEED); + memset(p + ps, 0x43, ps); + } + report(ok, ps, "repeated unread socket teardown and quarter reuse"); + if (use_tcp) + tls_checks(ps, p, len, back); + close(fds[0]); + close(fds[1]); + close(dup[0]); + close(dup[1]); + munmap(p, len); + free(back); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +static bool async_cleanup_balanced(void) +{ + const char *names[] = { "user4k_rx_deferred", "user4k_rx_released" }; + unsigned long counts[2]; + char path[128]; + FILE *file = fopen("/sys/module/tls/parameters/user4k_test_async", "r"); + int enabled; + + if (!file) + return false; + enabled = fgetc(file); + fclose(file); + for (unsigned int i = 0; i < 2; i++) { + snprintf(path, sizeof(path), "/sys/module/tls/parameters/%s", + names[i]); + file = fopen(path, "r"); + if (!file) + return false; + if (fscanf(file, "%lu", &counts[i]) != 1) { + fclose(file); + return false; + } + fclose(file); + } + printf("# TLS async=%c deferred=%lu released=%lu\n", enabled, counts[0], + counts[1]); + return counts[0] == counts[1] && (enabled != 'Y' || counts[0] > 0); +} + +int main(int argc, char **argv) +{ + bool ok = true; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && + (!strcmp(argv[1], "--test") || !strcmp(argv[1], "--tcp") || + !strcmp(argv[1], "--tcp6") || !strcmp(argv[1], "--udp") || + !strcmp(argv[1], "--udp6")) && + getppid() == 1) { + use_tcp = !strncmp(argv[1], "--tcp", 5); + use_udp = !strncmp(argv[1], "--udp", 5); + use_ipv6 = strchr(argv[1], '6') != NULL; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + { + struct ifreq request = { .ifr_name = "lo" }; + int fd = socket(AF_INET, SOCK_DGRAM, 0); + + if (fd < 0 || ioctl(fd, SIOCGIFFLAGS, &request)) + return 4; + request.ifr_flags |= IFF_UP; + if (ioctl(fd, SIOCSIFFLAGS, &request)) + return 4; + close(fd); + } + for (unsigned int mode = 0; mode < 10; mode++) { + unsigned int small = mode & 1; + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", + mode >= 8 ? "--udp6" : + mode >= 6 ? "--udp" : + mode >= 4 ? "--tcp6" : + mode >= 2 ? "--tcp" : + "--test", + NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + ok &= async_cleanup_balanced(); + printf("%s - splice fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/swap-fixture-init.c b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c new file mode 100644 index 00000000000000..45f6aefdd094e8 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swap-fixture-init.c @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PID-1 runner for a disposable VM with a dedicated virtio swap disk. + * Use kunit.autorun=0: tests skipped at boot stay skipped on a debugfs rerun. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned long swapouts(void) +{ + FILE *f = fopen("/proc/vmstat", "re"); + char key[128]; + unsigned long value, found = 0; + + if (!f) + return 0; + while (fscanf(f, "%127s %lu", key, &value) == 2) + if (!strcmp(key, "pswpout")) + found = value; + fclose(f); + return found; +} + +static int native_swap_smoke(void) +{ + const size_t length = 4 * 1024 * 1024, page = 16384; + unsigned long before = swapouts(), after; + unsigned char *p = mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t i; + unsigned int tries; + int ret = 0; + + if (p == MAP_FAILED) + return -1; + for (i = 0; i < length; i++) + p[i] = (i ^ (i >> 12) ^ (i >> 20)) & 255; + if (madvise(p, length, MADV_PAGEOUT)) { + ret = -1; + goto out; + } + for (tries = 0; tries < 100 && swapouts() == before; tries++) + usleep(20000); + after = swapouts(); + for (i = 0; i < length; i++) + if (p[i] != ((i ^ (i >> 12) ^ (i >> 20)) & 255)) { + ret = -1; + break; + } + if (after <= before) + ret = -1; + printf("%s - native %zuK swap transport: pswpout %lu -> %lu, verified %zu bytes\n", + ret ? "not ok" : "ok", page / 1024, before, after, i); +out: + munmap(p, length); + return ret; +} + +static int run_swap_suite(void) +{ + const char *base = "/sys/kernel/debug/kunit/arm64-user4k-swap/"; + char path[128], *log; + size_t used = 0, capacity = 1024 * 1024; + ssize_t n; + int fd, ret = -1; + + snprintf(path, sizeof(path), "%srun", base); + fd = open(path, O_WRONLY | O_CLOEXEC); + if (fd < 0) + return -1; + n = write(fd, "1", 1); + close(fd); + if (n != 1) + return -1; + snprintf(path, sizeof(path), "%sresults", base); + fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + log = calloc(1, capacity); + if (!log) { + close(fd); + return -1; + } + while (used < capacity - 1 && + (n = read(fd, log + used, capacity - 1 - used)) > 0) + used += n; + close(fd); + fputs(log, stdout); + if (n == 0 && used < capacity - 1 && + strstr(log, "\nok 1 arm64-user4k-swap\n") && + !strstr(log, "not ok") && !strstr(log, "# SKIP")) + ret = 0; + free(log); + return ret; +} + +static int swap_control_helper(const char *operation) +{ + struct stat init, self; + int ret; + + /* A helper is allowed only when this same executable is the VM's PID 1. */ + if (getpid() == 1 || stat("/proc/1/exe", &init) || + stat("/proc/self/exe", &self) || init.st_dev != self.st_dev || + init.st_ino != self.st_ino) { + fprintf(stderr, + "swap helper requires its fixture runner as PID 1\n"); + return 1; + } + ret = !strcmp(operation, "--swapoff-helper") ? swapoff("/dev/vda") : + swapon("/dev/vda", 0); + if (ret) + perror(operation); + return ret ? 1 : 0; +} + +static int pageout_exec(bool small) +{ + pid_t pid = fork(); + int status = 0; + + if (!pid) { + if (small && + prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 4096UL, 0UL, 0UL, 0UL)) + _exit(125); + execl("/reclaim-advice", "/reclaim-advice", + small ? "--expect-4k" : NULL, NULL); + _exit(126); + } + if (pid <= 0 || waitpid(pid, &status, 0) != pid) + return -1; + printf("# EL0 pageout %s child status=%#x\n", small ? "4K" : "native", + status); + return WIFEXITED(status) && !WEXITSTATUS(status) ? 0 : -1; +} + +int main(int argc, char **argv) +{ + bool smoke = argc == 2 && !strcmp(argv[1], "--smoke"); + bool pageout = argc == 2 && !strcmp(argv[1], "--pageout"); + bool native_only = argc == 2 && !strcmp(argv[1], "--pageout-native"); + bool active = false; + unsigned int tries; + struct stat st; + pid_t child; + int status, ret = 1; + + if (argc == 2 && (!strcmp(argv[1], "--swapoff-helper") || + !strcmp(argv[1], "--swapon-helper"))) + return swap_control_helper(argv[1]); + /* This runner must never mount filesystems or enable swap on the host. */ + if (getpid() != 1 || + (argc != 1 && !smoke && !pageout && !native_only)) { + fprintf(stderr, + "Use only as /init in the dedicated disposable test VM.\n"); + return 1; + } + setvbuf(stdout, NULL, _IONBF, 0); + if (getauxval(AT_PAGESZ) != 16384 || sysconf(_SC_PAGESIZE) != 16384) + goto out; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("debugfs", "/sys/kernel/debug", "debugfs", 0, NULL)) + goto out; + child = fork(); + if (!child) { + execl("/page-contract", "page-contract", NULL); + _exit(127); + } + if (child < 0 || waitpid(child, &status, 0) != child || + !WIFEXITED(status) || WEXITSTATUS(status)) + goto out; + for (tries = 0; tries < 100 && stat("/dev/vda", &st); tries++) + usleep(20000); + if (stat("/dev/vda", &st) || !S_ISBLK(st.st_mode) || + swapon("/dev/vda", 0)) + goto out; + active = true; + if (native_swap_smoke()) + goto out; + if (!smoke && !native_only && run_swap_suite()) + goto out; + if ((pageout || native_only) && pageout_exec(false)) + goto out; + if (pageout && pageout_exec(true)) + goto out; + ret = 0; +out: + if (ret) + printf("not ok - swap fixture runner: errno=%d (%s)\n", errno, + strerror(errno)); + if (active && swapoff("/dev/vda")) { + perror("swapoff"); + ret = 1; + } + printf("%s - swap fixture runner (%s)\n", ret ? "not ok" : "ok", + smoke ? "native transport smoke only" : "4K swap suite"); + fflush(NULL); + reboot(RB_POWER_OFF); + return ret; +} diff --git a/tools/testing/selftests/arm64/user4k/swapfile-fixture.h b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h new file mode 100644 index 00000000000000..258b81fa056958 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/swapfile-fixture.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef USER4K_SWAPFILE_FIXTURE_H +#define USER4K_SWAPFILE_FIXTURE_H + +static const char *swap_path(void) +{ + return access("/swapdir/swapfile", F_OK) ? "/dev/vda" : + "/swapdir/swapfile"; +} + +/* Remove every fourth native page: every 64K group crosses file extents. */ +static int prepare_swapfile(void) +{ + const unsigned long page = getauxval(AT_PAGESZ), initial = 32UL << 20; + unsigned char *buf = calloc(1, page); + unsigned long offset; + int fd = -1, ret = -1; + + if (!buf || (mkdir("/swapdir", 0755) && errno != EEXIST) || + mount("/dev/vda", "/swapdir", "ext4", 0, NULL)) + goto out; + fd = open("/swapdir/swapfile", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, + 0600); + if (fd < 0) + goto out; + memset(buf, 0xa5, page); + for (offset = 0; offset < initial; offset += page) + if (write(fd, buf, page) != (ssize_t)page) + goto out; + if (fsync(fd)) + goto out; + for (offset = initial; offset; offset -= 4 * page) + if (fallocate(fd, FALLOC_FL_COLLAPSE_RANGE, offset - 2 * page, + page)) + goto out; + memset(buf, 0, page); + ((unsigned int *)(buf + 1024))[0] = 1; + ((unsigned int *)(buf + 1024))[1] = (initial * 3 / 4) / page - 1; + memcpy(buf + page - 10, "SWAPSPACE2", 10); + if (pwrite(fd, buf, page, 0) != (ssize_t)page || fsync(fd)) + goto out; + ret = 0; +out: + if (fd >= 0) + close(fd); + free(buf); + return ret; +} + +#endif diff --git a/tools/testing/selftests/arm64/user4k/sysv-shm-init.c b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c new file mode 100644 index 00000000000000..3b66d9c738fde0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/sysv-shm-init.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Disposable PID1: SysV shared memory with per-process page granules. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void check(int ok, const char *expression, int line) +{ + if (!ok) { + fprintf(stderr, "FAIL line %d: %s errno=%d\n", line, expression, + errno); + exit(1); + } +} + +#define CHECK(x) check(!!(x), #x, __LINE__) + +static int unmapped(void *address, size_t page) +{ + unsigned char resident; + + errno = 0; + return mincore(address, page, &resident) == -1 && errno == ENOMEM; +} + +static int mapping_case(unsigned long native, int rounded, int split) +{ + unsigned long page = getauxval(AT_PAGESZ), length = 4 * page; + unsigned char *reserve, *target, *mapping, *alias; + struct shmid_ds info; + int id; + + reserve = mmap(NULL, 16 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(reserve != MAP_FAILED); + target = (void *)(((uintptr_t)reserve + native - 1) & ~(native - 1)); + target += native + (page < native ? page : 0); + target[-1] = 0x51; + target[length] = 0x72; + CHECK(!munmap(target, length)); + id = shmget(IPC_PRIVATE, length - 1, 0600); + CHECK(id >= 0); + mapping = shmat(id, target + (rounded ? page - 1 : 0), + rounded ? SHM_RND : 0); + if (mapping != target) { + int saved = errno; + + shmctl(id, IPC_RMID, NULL); + errno = saved; + CHECK(mapping == target); + } + alias = shmat(id, NULL, 0); + CHECK(alias != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + mapping[0] = 0x31; + mapping[page + 7] = 0x49; + mapping[3 * page] = 0x63; + CHECK(alias[0] == 0x31 && alias[page + 7] == 0x49 && + alias[3 * page] == 0x63); + if (split) { + CHECK(!mprotect(mapping + page, page, PROT_READ)); + CHECK(!munmap(mapping + 2 * page, page)); + if (split == 2) + CHECK(!munmap(mapping, page)); + } + CHECK(!shmdt(mapping)); + for (unsigned long i = 0; i < length; i += page) + CHECK(unmapped(target + i, page)); + CHECK(target[-1] == 0x51 && target[length] == 0x72); + CHECK(alias[page + 7] == 0x49 && alias[3 * page] == 0x63); + CHECK(!shmdt(alias)); + errno = 0; + CHECK(shmctl(id, IPC_STAT, &info) == -1 && + (errno == EINVAL || errno == EIDRM)); + CHECK(!munmap(reserve, 16 * native)); + printf("ok - SysV page=%lu rounded=%d split=%d detach and alias lifetime\n", + page, rounded, split); + return 0; +} + +static int wait_child(pid_t pid) +{ + int status; + pid_t got; + + if (pid < 0) + return 1; + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got != pid || !WIFEXITED(status) || WEXITSTATUS(status); +} + +static int sharing_child(int id, unsigned long expected) +{ + unsigned char *map; + + CHECK(getauxval(AT_PAGESZ) == expected); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(map[37] == 0x35 && map[4096 + 9] == 0x56); + map[4096 + 9] = 0x78; + CHECK(!shmdt(map)); + return 0; +} + +static int sharing(unsigned long other, unsigned long native) +{ + unsigned long page = getauxval(AT_PAGESZ); + unsigned char *map; + pid_t pid; + int id; + + id = shmget(IPC_PRIVATE, 2 * native, 0600); + CHECK(id >= 0); + map = shmat(id, NULL, 0); + CHECK(map != (void *)-1); + CHECK(!shmctl(id, IPC_RMID, NULL)); + map[37] = 0x35; + map[4096 + 9] = 0x56; + pid = fork(); + if (!pid) { + char idarg[32], sizearg[32]; + + snprintf(idarg, sizeof(idarg), "%d", id); + snprintf(sizearg, sizeof(sizearg), "%lu", other); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, other, 0UL, 0UL, 0UL)) + _exit(2); + execl("/init", "/init", "--share", idarg, sizearg, NULL); + _exit(3); + } + CHECK(!wait_child(pid)); + CHECK(map[4096 + 9] == 0x78); + CHECK(!shmdt(map)); + printf("ok - SysV shared object %lu -> %lu after IPC_RMID\n", page, + other); + return 0; +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + int failed = 0; + + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--share")) + return sharing_child(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 5 && !strcmp(argv[1], "--case")) { + unsigned long expected = strtoul(argv[3], NULL, 10); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + native = strtoul(argv[2], NULL, 10); + CHECK(getauxval(AT_PAGESZ) == expected); + failed |= mapping_case(native, 0, 0); + failed |= mapping_case(native, 1, 0); + failed |= mapping_case(native, 0, 1); + failed |= mapping_case(native, 0, 2); + for (unsigned int i = 0; i < 3; i++) + if (sizes[i] <= native && sizes[i] != expected && + !(atoi(argv[4]) && sizes[i] == 4096)) + failed |= sharing(sizes[i], native); + return failed; + } + CHECK(getpid() == 1); + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + int no4k = argc == 2 && !strcmp(argv[1], "--no4k"); + unsigned long sizes[] = { 4096, 16384, 65536 }; + + for (unsigned int i = 0; i < 3; i++) { + unsigned long page = sizes[i]; + pid_t pid; + + if (page > native || (no4k && page == 4096)) + continue; + pid = fork(); + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", page); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, page, 0UL, 0UL, + 0UL)) + _exit(2); + execl("/init", "/init", "--case", n, e, + no4k ? "1" : "0", NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch64 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, page); + failed |= bad; + } + if (access("/compat-shm", X_OK) == 0) { + pid_t pid = fork(); + + if (!pid) { + char n[32], e[32]; + + snprintf(n, sizeof(n), "%lu", native); + snprintf(e, sizeof(e), "%lu", no4k ? native : 4096UL); + execl("/compat-shm", "/compat-shm", n, e, NULL); + _exit(3); + } + int bad = wait_child(pid); + + printf("%s - SysV AArch32 native=%lu process=%lu\n", + bad ? "not ok" : "ok", native, no4k ? native : 4096UL); + failed |= bad; + } + printf("SYSV SHM GRANULE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/thread-fixture-init.c b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c new file mode 100644 index 00000000000000..c472ea6697947c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/thread-fixture-init.c @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +#define THREADS 4 +#define ROUNDS 128 +#define PAGES 32 + +static pthread_barrier_t barrier; +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t condition = PTHREAD_COND_INITIALIZER; +static pthread_mutex_t robust; +static _Thread_local unsigned int tls_id; +static _Thread_local volatile sig_atomic_t signals; +static atomic_uint failures; +static unsigned long total; +static unsigned long ps; +static atomic_int waiter_tid; + +static void handler(int sig) +{ + if (sig == SIGUSR1 && tls_id) + signals++; + else + atomic_fetch_add(&failures, 1); +} + +static bool bytes_equal(unsigned char *p, size_t n, unsigned char value) +{ + for (size_t i = 0; i < n; i++) + if (p[i] != value) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg; + unsigned char *p, *heap; + unsigned long request = id & 1 ? 4096 : 0; + bool ok = true; + + tls_id = id; + p = mmap(NULL, PAGES * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (p == MAP_FAILED) + ok = false; + ok &= !prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, request, 0UL, 0UL, 0UL); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + unsigned char pattern = 1 + (id * 31 + round) % 254; + unsigned char residency[PAGES]; + cpu_set_t cpu; + + if (ok) { + CPU_ZERO(&cpu); + CPU_SET((id + round) % 2, &cpu); + ok &= !sched_setaffinity(0, sizeof(cpu), &cpu); + memset(p, pattern, PAGES * ps); + ok &= !mprotect(p + ps, ps, PROT_READ) && + p[ps] == pattern; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE); + ok &= !madvise(p + 2 * ps, ps, MADV_FREE); + memset(p + 2 * ps, pattern, ps); + ok &= !madvise(p + 3 * ps, ps, MADV_DONTNEED); + ok &= bytes_equal(p + 3 * ps, ps, 0); + memset(p + 3 * ps, pattern, ps); + heap = malloc(5000 + id * 777); + if (!heap) { + ok = false; + } else { + memset(heap, pattern, 5000 + id * 777); + ok &= bytes_equal(heap, 5000 + id * 777, + pattern); + free(heap); + } + ok &= !pthread_kill(pthread_self(), SIGUSR1); + ok &= tls_id == id && + signals == (sig_atomic_t)(round + 1); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) == (int)request; + ok &= !mincore(p, PAGES * ps, residency); + ok &= bytes_equal(p, PAGES * ps, pattern); + } + pthread_mutex_lock(&mutex); + total++; + pthread_cond_signal(&condition); + pthread_mutex_unlock(&mutex); + pthread_barrier_wait(&barrier); + } + if (p != MAP_FAILED) + munmap(p, PAGES * ps); + if (!ok) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *owner_dies(void *arg) +{ + (void)arg; + if (pthread_mutex_lock(&robust)) + atomic_fetch_add(&failures, 1); + return NULL; +} + +static void *pi_waiter(void *arg) +{ + int error; + + (void)arg; + atomic_store(&waiter_tid, syscall(SYS_gettid)); + error = pthread_mutex_lock(&robust); + if (!error) + error = pthread_mutex_unlock(&robust); + return (void *)(uintptr_t)error; +} + +static bool wait_for_pi_block(void) +{ + for (unsigned int tries = 0; tries < 1000; tries++) { + int tid = atomic_load(&waiter_tid); + char path[80], line[256]; + unsigned long nr, address, operation; + FILE *f; + + if (tid) { + snprintf(path, sizeof(path), + "/proc/self/task/%d/syscall", tid); + f = fopen(path, "re"); + if (f) { + bool blocked = fgets(line, sizeof(line), f) && + sscanf(line, "%lu %lx %lx", &nr, + &address, + &operation) == 3 && + nr == SYS_futex && + ((operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI || + (operation & FUTEX_CMD_MASK) == + FUTEX_LOCK_PI2); + + fclose(f); + if (blocked) + return true; + } + } + usleep(1000); + } + return false; +} + +static int exercise(void) +{ + pthread_t threads[THREADS], owner; + pthread_mutexattr_t attributes; + struct sigaction action = { .sa_handler = handler }; + struct stat self, init; + bool ok = true, forks = true; + int error; + + if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || + self.st_dev != init.st_dev || self.st_ino != init.st_ino) + return 2; + ps = getauxval(AT_PAGESZ); + ksft_print_header(); + ksft_set_plan(6); + if (sigaction(SIGUSR1, &action, NULL) || + pthread_barrier_init(&barrier, NULL, THREADS + 1)) + ksft_exit_fail_msg("thread setup\n"); + for (unsigned int i = 0; i < THREADS; i++) + if (pthread_create(&threads[i], NULL, worker, + (void *)(uintptr_t)(i + 1))) + ksft_exit_fail_msg("pthread_create\n"); + pthread_barrier_wait(&barrier); + for (unsigned int round = 0; round < ROUNDS; round++) { + pthread_mutex_lock(&mutex); + while (total < (round + 1) * THREADS) + pthread_cond_wait(&condition, &mutex); + pthread_mutex_unlock(&mutex); + ok &= prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == + 0; + if (!(round % 16)) { + pid_t child = fork(); + int status = 0; + + if (!child) { + unsigned char *p = malloc(32768); + + if (!p || tls_id || + total != (round + 1) * THREADS || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, + 0UL, 0UL, 0UL)) + _exit(10); + memset(p, 0x75, 32768); + if (!bytes_equal(p, 32768, 0x75)) + _exit(11); + free(p); + _exit(0); + } + forks &= child > 0 && + waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + } + pthread_barrier_wait(&barrier); + } + for (unsigned int i = 0; i < THREADS; i++) + ok &= !pthread_join(threads[i], NULL); + ksft_test_result( + ok && !atomic_load(&failures) && total == THREADS * ROUNDS, + "%luK %u concurrent memory/TLS/signal/exec-request rounds across %u threads\n", + ps / 1024, ROUNDS, THREADS); + ksft_test_result( + forks, "%luK fork and allocator from a multithreaded process\n", + ps / 1024); + pthread_barrier_destroy(&barrier); + ok = !pthread_mutexattr_init(&attributes); + ok &= !pthread_mutexattr_setrobust(&attributes, PTHREAD_MUTEX_ROBUST); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("robust mutex setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + ksft_test_result(ok, "%luK robust futex owner-death recovery\n", + ps / 1024); + pthread_mutex_destroy(&robust); + /* Exercise repeated TLS/stack allocation and set_tid_address clear/wake. */ + for (unsigned int i = 0; i < 128; i++) { + ok = !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + break; + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + if (!ok) + break; + } + ksft_test_result(ok && !atomic_load(&failures), + "%luK repeated thread exit/join and robust recovery\n", + ps / 1024); + ok = !pthread_mutexattr_setprotocol(&attributes, PTHREAD_PRIO_INHERIT); + ok &= !pthread_mutex_init(&robust, &attributes); + ok &= !pthread_mutex_lock(&robust); + ok &= !pthread_create(&owner, NULL, pi_waiter, NULL); + if (!ok) + ksft_exit_fail_msg("PI mutex setup\n"); + ok &= wait_for_pi_block(); + ok &= !pthread_mutex_unlock(&robust); + { + void *result = (void *)1; + + ok &= !pthread_join(owner, &result) && !result; + } + ksft_test_result( + ok, + "%luK confirmed contended priority-inheritance futex handoff\n", + ps / 1024); + ok = !pthread_create(&owner, NULL, owner_dies, NULL); + if (!ok) + ksft_exit_fail_msg("PI owner setup\n"); + ok &= !pthread_join(owner, NULL); + error = pthread_mutex_lock(&robust); + ok &= error == EOWNERDEAD; + if (error == EOWNERDEAD) + ok &= !pthread_mutex_consistent(&robust); + if (!error || error == EOWNERDEAD) + ok &= !pthread_mutex_unlock(&robust); + pthread_mutex_destroy(&robust); + pthread_mutexattr_destroy(&attributes); + ksft_test_result( + ok, "%luK priority-inheritance robust owner-death recovery\n", + ps / 1024); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} + +static pthread_barrier_t exec_barrier; + +static void *exec_sibling(void *arg) +{ + (void)arg; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return NULL; +} + +static void *exec_nonleader(void *arg) +{ + unsigned long target = (uintptr_t)arg; + char size[32]; + + snprintf(size, sizeof(size), "%lu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, target, 0UL, 0UL, 0UL)) + _exit(30); + pthread_barrier_wait(&exec_barrier); + execl("/init", "/init", "--after-thread-exec", size, NULL); + _exit(31); +} + +static int thread_exec(unsigned long target) +{ + pthread_t siblings[2], executor; + + if (pthread_barrier_init(&exec_barrier, NULL, 4)) + return 32; + for (unsigned int i = 0; i < 2; i++) + if (pthread_create(&siblings[i], NULL, exec_sibling, NULL)) + return 33; + if (pthread_create(&executor, NULL, exec_nonleader, + (void *)(uintptr_t)target)) + return 34; + pthread_barrier_wait(&exec_barrier); + for (;;) + pause(); + return 35; +} + +int main(int argc, char **argv) +{ + bool ok = true; + unsigned long granules[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + unsigned int nr = granules[0] > 16384 ? 3 : 2; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--thread-exec") && getppid() == 1) + return thread_exec(strtoul(argv[2], NULL, 10)); + if (argc == 3 && !strcmp(argv[1], "--after-thread-exec") && + getppid() == 1) { + unsigned long expected = strtoul(argv[2], NULL, 10); + + return getauxval(AT_PAGESZ) != expected || + prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, + 0UL) != 0; + } + if (argc == 3 && !strcmp(argv[1], "--test") && getppid() == 1) { + if (getauxval(AT_PAGESZ) != strtoul(argv[2], NULL, 10)) + return 42; + return exercise(); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Run only as PID 1 in a disposable VM\n"); + return 2; + } + if (mount("proc", "/proc", "proc", 0, NULL)) + return 3; + for (unsigned int small = 0; small < nr; small++) { + pid_t pid = fork(); + int status = 0; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", granules[small]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, granules[small], + 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", size, NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + for (unsigned int source = 0; source < nr; source++) { + for (unsigned int target = 0; target < nr; target++) { + pid_t pid = fork(); + int status = 0; + bool passed; + + if (!pid) { + char size[32]; + + snprintf(size, sizeof(size), "%lu", + granules[target]); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + granules[source], 0UL, 0UL, 0UL)) + _exit(36); + execl("/init", "/init", "--thread-exec", size, + NULL); + _exit(37); + } + passed = pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - non-leader exec %luK -> %luK status=%#x\n", + passed ? "ok" : "not ok", + granules[source] / 1024, granules[target] / 1024, + status); + ok &= passed; + } + } + printf("%s - thread fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-move-init.c b/tools/testing/selftests/arm64/user4k/uffd-move-init.c new file mode 100644 index 00000000000000..f2649698488b6c --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-move-init.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - MOVE line=%d errno=%d (%s)\n", \ + __LINE__, errno, strerror(errno)); \ + exit(1); \ + } \ + } while (0) +#define PRESENT (1ULL << 63) +#define SWAPPED (1ULL << 62) +#define VALUE_MASK ((1ULL << 55) - 1) +static uint64_t entry(void *p, size_t ps) +{ + uint64_t e; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + CHECK(fd >= 0 && pread(fd, &e, 8, (uintptr_t)p / ps * 8) == 8); + CHECK(!close(fd)); + return e; +} +static uint64_t physical(void *p, size_t ps) +{ + uint64_t e = entry(p, ps); + CHECK(e & PRESENT); + return (e & VALUE_MASK) * ps; +} +static unsigned char pattern(size_t i) +{ + return 1 + ((i ^ (i >> 12) ^ (i >> 16)) % 251); +} +static void bytes(unsigned char *p, size_t original, size_t len) +{ + for (size_t i = 0; i < len; i++) + CHECK(p[i] == pattern(original + i)); +} +static long move(int fd, void *dst, void *src, size_t len, unsigned mode, + int expected_errno) +{ + struct uffdio_move m = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + errno = 0; + int r = ioctl(fd, UFFDIO_MOVE, &m); + if (expected_errno) + CHECK(r == -1 && errno == expected_errno); + else + CHECK(r == 0 && m.move == (long long)len); + return m.move; +} +static void on_cpu(unsigned cpu) +{ + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + CHECK(!sched_setaffinity(0, sizeof(set), &set)); +} +struct tlb_reader { + unsigned char *src, *dst; + atomic_int ready, done; +}; +static void *read_after_move(void *arg) +{ + struct tlb_reader *r = arg; + on_cpu(1); + unsigned char old = *(volatile unsigned char *)r->src; + atomic_store(&r->ready, 1); + while (!atomic_load(&r->done)) + sched_yield(); + CHECK(*(volatile unsigned char *)r->src == 0 && + *(volatile unsigned char *)r->dst == old); + return NULL; +} +struct pin_racer { + void *a, *b; + size_t ps; + atomic_int stop; + atomic_uint accepted, rejected; +}; +static void *race_pins(void *arg) +{ + struct pin_racer *r = arg; + struct io_uring_params p = { 0 }; + on_cpu(1); + int fd = syscall(SYS_io_uring_setup, 2, &p); + CHECK(fd >= 0); + unsigned round = 0; + while (!atomic_load(&r->stop)) { + struct iovec iov = { .iov_base = (round++ & 1) ? r->a : r->b, + .iov_len = r->ps }; + int ret = syscall(SYS_io_uring_register, fd, + IORING_REGISTER_BUFFERS, &iov, 1); + if (!ret) { + atomic_fetch_add(&r->accepted, 1); + usleep(50); + CHECK(!syscall(SYS_io_uring_register, fd, + IORING_UNREGISTER_BUFFERS, NULL, 0)); + } else { + CHECK(errno == EFAULT || errno == EAGAIN); + atomic_fetch_add(&r->rejected, 1); + } + } + CHECK(!close(fd)); + return NULL; +} +static void concurrent_move(size_t ps) +{ + unsigned char *a = + mmap((void *)0x50000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x60000000, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x6d, ps); + uint64_t phys = physical(a, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pin_racer race = { .a = a, .b = b, .ps = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, race_pins, &race)); + while (!atomic_load(&race.accepted)) + sched_yield(); + unsigned char *cur = a, *next = b; + unsigned moved = 0, busy = 0; + for (unsigned tries = 0; moved < 256 && tries < 100000; tries++) { + struct uffdio_move m = { .src = (uintptr_t)cur, + .dst = (uintptr_t)next, + .len = ps }; + int ret = ioctl(fd, UFFDIO_MOVE, &m); + if (ret) { + CHECK(errno == EBUSY && m.move == -EBUSY); + busy++; + sched_yield(); + continue; + } + CHECK(m.move == (long long)ps && physical(next, ps) == phys && + !(entry(cur, ps) & (PRESENT | SWAPPED))); + unsigned char *tmp = cur; + cur = next; + next = tmp; + moved++; + } + atomic_store(&race.stop, 1); + CHECK(!pthread_join(thread, NULL)); + CHECK(moved == 256 && atomic_load(&race.accepted) > 0); + for (size_t i = 0; i < ps; i++) + CHECK(cur[i] == 0x6d); + printf("ok - MOVE %zuK CPU0/CPU1 race moves=%u busy=%u pins=%u holes=%u unchanged PFN/data\n", + ps / 1024, moved, busy, atomic_load(&race.accepted), + atomic_load(&race.rejected)); + CHECK(!close(fd) && !munmap(a, ps) && !munmap(b, ps)); +} + +static void allocation_failure(size_t ps, size_t native) +{ + if (ps == native) + return; + unsigned char *a = + mmap((void *)0x70000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *b = + mmap((void *)0x71000000, 2 * native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED); + memset(a, 0x35, 2 * native); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)a, 2 * native }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + reg.range.start = (uintptr_t)b; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + /* Prepare both roots and target tables using a different native folio. */ + move(fd, b + native, a + native, ps, 0, 0); + uint64_t phys = physical(a, ps); + int inject = open("/proc/self/fail-nth", O_RDWR | O_CLOEXEC); + CHECK(inject >= 0); + unsigned char *src = a, *dst = b; + for (unsigned phase = 0; phase < 2; phase++) { + struct uffdio_move m = { .src = (uintptr_t)src, + .dst = (uintptr_t)dst, + .len = ps }; + char pending[16] = { 0 }; + CHECK(write(inject, "1", 1) == 1); + errno = 0; + int ret = ioctl(fd, UFFDIO_MOVE, &m), saved = errno; + CHECK(pread(inject, pending, sizeof(pending) - 1, 0) > 0); + CHECK(write(inject, "0", 1) == 1); + CHECK(ret == -1 && saved == ENOMEM && m.move == -ENOMEM && + strtoul(pending, NULL, 10) == 0); + CHECK(physical(src, ps) == phys && + !(entry(dst, ps) & (PRESENT | SWAPPED))); + for (size_t i = 0; i < ps; i++) + CHECK(src[i] == 0x35); + move(fd, dst, src, ps, 0, 0); + CHECK(physical(dst, ps) == phys); + unsigned char *tmp = src; + src = dst; + dst = tmp; + } + printf("ok - MOVE %zuK sidecar/retirement allocation failures preserve PTEs and retry\n", + ps / 1024); + CHECK(!close(inject) && !close(fd) && !munmap(a, 2 * native) && + !munmap(b, 2 * native)); +} + +static int exercise(size_t native) +{ + size_t ps = getauxval(AT_PAGESZ), len = 16 * native; + on_cpu(0); + unsigned char *src = + mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + unsigned char *dst = + mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (size_t i = 0; i < len; i++) + src[i] = pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MOVE }; + CHECK(!ioctl(fd, UFFDIO_API, &api) && + (api.features & UFFD_FEATURE_MOVE)); + struct uffdio_register reg = { .range = { (uintptr_t)dst, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_MOVE))); + uint64_t pfns[256]; + CHECK(native / ps <= sizeof(pfns) / sizeof(pfns[0])); + for (size_t i = 0; i < native / ps; i++) + pfns[i] = physical(src + i * ps, ps); + uint64_t p = physical(src + 3 * ps, ps); + struct tlb_reader reader = { .src = src + 3 * ps, .dst = dst + ps }; + pthread_t tlb_thread; + CHECK(!pthread_create(&tlb_thread, NULL, read_after_move, &reader)); + while (!atomic_load(&reader.ready)) + sched_yield(); + move(fd, dst + ps, src + 3 * ps, ps, 0, 0); + CHECK(physical(dst + ps, ps) == p && + !(entry(src + 3 * ps, ps) & (PRESENT | SWAPPED))); + atomic_store(&reader.done, 1); + CHECK(!pthread_join(tlb_thread, NULL)); + bytes(dst + ps, 3 * ps, ps); + for (size_t i = 0; i < native / ps; i++) + if (i != 3) { + CHECK(physical(src + i * ps, ps) == pfns[i]); + bytes(src + i * ps, i * ps, ps); + } + CHECK(src[3 * ps] == + 0); /* Source has an independent zero-fault after MOVE. */ + printf("ok - MOVE %zuK exact PFN, divergent offsets, neighbour PFNs/data and source TLB\n", + ps / 1024); + + struct io_uring_params ur = { 0 }; + int ring = syscall(SYS_io_uring_setup, 2, &ur); + CHECK(ring >= 0); + struct iovec pin = { .iov_base = src + 2 * ps, .iov_len = ps }; + CHECK(!syscall(SYS_io_uring_register, ring, IORING_REGISTER_BUFFERS, + &pin, 1)); + p = physical(src + 2 * ps, ps); + CHECK(move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, EBUSY) == -EBUSY); + CHECK(physical(src + 2 * ps, ps) == p && + !(entry(dst + 2 * ps, ps) & (PRESENT | SWAPPED))); + bytes(src + 2 * ps, 2 * ps, ps); + uint64_t neighbour = physical(src + ps, ps); + move(fd, dst + 3 * ps, src + ps, ps, 0, 0); + CHECK(physical(dst + 3 * ps, ps) == neighbour); + bytes(dst + 3 * ps, ps, ps); + CHECK(!syscall(SYS_io_uring_register, ring, IORING_UNREGISTER_BUFFERS, + NULL, 0)); + CHECK(!close(ring)); + move(fd, dst + 2 * ps, src + 2 * ps, ps, 0, 0); + CHECK(physical(dst + 2 * ps, ps) == p); + bytes(dst + 2 * ps, 2 * ps, ps); + printf("ok - MOVE %zuK source PIN rejected atomically, neighbour PIN allowed, unpin retry\n", + ps / 1024); + + CHECK(move(fd, dst + ps, src + 4 * ps, ps, 0, EEXIST) == -EEXIST); + bytes(src + 4 * ps, 4 * ps, ps); + bytes(dst + ps, 3 * ps, ps); + CHECK(!madvise(src + 5 * native, native, MADV_DONTNEED)); + CHECK(*(volatile unsigned char *)(src + 5 * native) == 0); + move(fd, dst + 5 * native, src + 5 * native, ps, 0, 0); + CHECK(dst[5 * native] == 0); + dst[5 * native] = 0x7d; + CHECK(src[5 * native] == 0); + CHECK(!madvise(src + 6 * native, native, MADV_DONTNEED)); + CHECK(move(fd, dst + 6 * native, src + 6 * native, ps, 0, ENOENT) == + -ENOENT); + move(fd, dst + 6 * native, src + 6 * native, ps, + UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES, 0); + CHECK(!(entry(dst + 6 * native, ps) & (PRESENT | SWAPPED))); + CHECK(!madvise(src + 10 * native + ps, ps, MADV_DONTNEED)); + p = physical(src + 10 * native, ps); + CHECK(move(fd, dst + 12 * native, src + 10 * native, 2 * ps, 0, + EAGAIN) == (long)ps); + CHECK(physical(dst + 12 * native, ps) == p); + bytes(dst + 12 * native, 10 * native, ps); + CHECK(!(entry(dst + 12 * native + ps, ps) & (PRESENT | SWAPPED))); + printf("ok - MOVE %zuK collision, zero, holes and partial progress\n", + ps / 1024); + + int pipefd[2]; + CHECK(!pipe(pipefd)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(pipefd[1]); + char c; + _exit(read(pipefd[0], &c, 1) == 1 ? 0 : 1); + } + close(pipefd[0]); + CHECK(move(fd, dst + 11 * native, src + 12 * native, ps, 0, EBUSY) == + -EBUSY); + CHECK(write(pipefd[1], "x", 1) == 1); + close(pipefd[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && status == 0); + *(volatile unsigned char *)(src + 12 * native) = pattern(12 * native); + p = physical(src + 12 * native, ps); + move(fd, dst + 11 * native, src + 12 * native, ps, 0, 0); + CHECK(physical(dst + 11 * native, ps) == p); + bytes(dst + 11 * native, 12 * native, ps); + printf("ok - MOVE %zuK fork-shared source rejected, exclusive write/retry\n", + ps / 1024); + + /* Keep fault-in coverage as well as restoring an untouched PTE at swapoff. */ + for (unsigned int unuse = 0; unuse < 2; unuse++) { + size_t off = ps < native ? ps : 0; + size_t base = (unuse ? 13 : 8) * native; + unsigned char *s = src + base + off; + unsigned char *d = dst + base + 2 * ps; + + *(volatile unsigned char *)s = pattern(base + off); + for (unsigned int tries = 0; + !(entry(s, ps) & SWAPPED) && tries < 100; tries++) { + CHECK(!madvise(src + base, native, MADV_PAGEOUT)); + usleep(20000); + } + uint64_t old = entry(s, ps); + + CHECK(old & SWAPPED); + move(fd, d, s, ps, 0, 0); + uint64_t now = entry(d, ps); + + CHECK((now & SWAPPED) && ((now & VALUE_MASK) == (old & VALUE_MASK)) && + !(entry(s, ps) & (PRESENT | SWAPPED))); + if (unuse) { + /* Do not fault the moved slot in before swapoff restores it. */ + CHECK(!swapoff("/dev/vda")); + CHECK(entry(d, ps) & PRESENT); + } + bytes(d, base + off, ps); + if (unuse) + CHECK(!swapon("/dev/vda", 0)); + printf("ok - MOVE %zuK %s restores transferred swap PTE at new offset\n", + ps / 1024, unuse ? "swapoff" : "fault"); + } + CHECK(!close(fd)); + CHECK(!munmap(src, len)); + CHECK(!munmap(dst, len)); + concurrent_move(ps); + allocation_failure(ps, native); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + CHECK(!mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)); + struct stat st; + for (unsigned i = 0; i < 100 && stat("/dev/vda", &st); i++) + usleep(20000); + CHECK(!stat("/dev/vda", &st) && S_ISBLK(st.st_mode) && + !swapon("/dev/vda", 0)); + size_t native = getauxval(AT_PAGESZ), sizes[] = { native, 4096, 16384 }; + int failed = 0; + for (unsigned i = 0; i < (native > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + CHECK(c >= 0); + if (!c) { + char n[32]; + snprintf(n, sizeof(n), "%zu", native); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", n, NULL); + _exit(5); + } + CHECK(waitpid(c, &status, 0) == c); + printf("%s - MOVE ABI %zuK status=%#x\n", + status ? "not ok" : "ok", sizes[i] / 1024, status); + failed += !!status; + } + if (swapoff("/dev/vda")) + failed++; + printf("UFFD MOVE CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c new file mode 100644 index 00000000000000..19747755c4c6c9 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -0,0 +1,913 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define CHECK(x) \ + do { \ + if (!(x)) { \ + printf("not ok - uffd line=%d errno=%d\n", __LINE__, \ + errno); \ + exit(1); \ + } \ + } while (0) +struct writer { + volatile unsigned char *address; + unsigned char value; + atomic_int done, tid; +}; +static void *write_byte(void *opaque) +{ + struct writer *w = opaque; + atomic_store(&w->tid, syscall(SYS_gettid)); + *w->address = w->value; + atomic_store(&w->done, 1); + return NULL; +} +struct forker { + unsigned char *base; + size_t page_size; + pid_t pid; +}; +static void *fork_byte(void *opaque) +{ + struct forker *f = opaque; + f->pid = fork(); + if (!f->pid) { + alarm(10); + f->base[f->page_size + 17] = 0xa5; + _exit(f->base[f->page_size + 17] != 0xa5); + } + return NULL; +} +static struct uffd_msg read_event(int fd) +{ + struct pollfd p = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + CHECK(poll(&p, 1, 10000) == 1 && (p.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + return msg; +} +static void protect(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_writeprotect wp = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_WRITEPROTECT, &wp)); +} +static void resolve_write(int fd, unsigned char *base, size_t ps, unsigned page, + unsigned char value, int exact) +{ + struct writer w = { .address = base + page * ps + 17, .value = value }; + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + struct uffd_msg msg; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + CHECK(poll(&pollfd, 1, 10000) == 1 && (pollfd.revents & POLLIN)); + CHECK(read(fd, &msg, sizeof(msg)) == sizeof(msg)); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT); + CHECK(msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(msg.arg.pagefault.address == + (uintptr_t)(base + page * ps + (exact ? 17 : 0))); + CHECK(msg.arg.pagefault.feat.ptid == (unsigned int)atomic_load(&w.tid)); + CHECK(!atomic_load(&w.done)); + protect(fd, base + page * ps, ps, UFFDIO_WRITEPROTECT_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(base + page * ps), ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + CHECK(base[page * ps + 17] == value); + CHECK(poll(&pollfd, 1, 0) == 0); +} +static void fill(int fd, void *dst, void *src, size_t len, unsigned long mode) +{ + struct uffdio_copy copy = { .dst = (uintptr_t)dst, + .src = (uintptr_t)src, + .len = len, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_COPY, ©) && copy.copy == (long long)len); +} +static void zero(int fd, void *dst, size_t len) +{ + struct uffdio_zeropage z = { .range = { (uintptr_t)dst, len } }; + CHECK(!ioctl(fd, UFFDIO_ZEROPAGE, &z) && z.zeropage == (long long)len); +} +static uint64_t physical(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd) && (entry & (1ULL << 63))); + return (entry & ((1ULL << 55) - 1)) * ps; +} +static void missing(size_t ps, size_t native) +{ + size_t len = 4 * native; + unsigned char *map = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + unsigned char *src = mmap(NULL, len + native, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(map != MAP_FAILED && src != MAP_FAILED); + unsigned char *p = map + ps; + CHECK(!mlock2(p, len, MLOCK_ONFAULT)); + for (size_t i = 0; i < len; i++) + src[i] = (i / ps) ^ (i * 7 + 3); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE))); + /* A write blocks only on its missing leaf; COPY_MODE_WP causes a second event. */ + struct writer w = { .address = p + 17, .value = 0xd3 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_WRITE && + !atomic_load(&w.done)); + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && atomic_load(&w.done)); + src[17] = w.value; + CHECK(!memcmp(p, src, ps)); + zero(fd, p + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + p[ps + 17] = 0xe4; + CHECK(!memcmp(p, src, ps)); + /* A batch starts at a user-page boundary, not necessarily a native one. */ + fill(fd, p + 2 * ps, src + 2 * ps, len - 2 * ps, 0); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, len - 2 * ps)); + if (ps < native) { + unsigned char *a = + (void *)(((uintptr_t)(p + 2 * ps) + native - 1) & + ~(native - 1)); + uint64_t start = physical(a, ps); + CHECK(start != 0 && !(start & (native - 1))); + for (size_t i = ps; i < native; i += ps) + CHECK(physical(a + i, ps) == start + i); + } + struct uffdio_copy copy = { .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EEXIST && + copy.copy == -EEXIST); + CHECK(!munlock(p, len)); + /* Preserve the successfully copied prefix when the next PTE already exists. */ + CHECK(!madvise(p, ps, MADV_DONTNEED)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)p, + .src = (uintptr_t)src, + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p, src, ps) && p[ps + 17] == 0xe4); + /* An invalid source stops at its leaf, without installing it or touching neighbours. */ + CHECK(!madvise(p + 2 * ps, 2 * ps, MADV_DONTNEED)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_NONE)); + copy = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)(src + 2 * ps), + .len = 2 * ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EAGAIN && + copy.copy == (long long)ps); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!mprotect(src + 3 * ps, ps, PROT_READ | PROT_WRITE)); + /* A fresh source forces the unlocked copy retry. */ + CHECK(!madvise(src + 3 * ps, ps, MADV_DONTNEED)); + fill(fd, p + 3 * ps, src + 3 * ps, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[3 * ps + i] == 0); + CHECK(!memcmp(p + 4 * ps, src + 4 * ps, len - 4 * ps)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + p[2 * ps + 9] ^= 0xff; + _exit(p[2 * ps + 9] == src[2 * ps + 9]); + } + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!memcmp(p + 2 * ps, src + 2 * ps, ps)); + CHECK(!close(fd)); + CHECK(!munmap(map, len + native) && !munmap(src, len + native)); + printf("ok - uffd ABI %zuK missing copy/WP, zero, packed batch, partial EEXIST/EFAULT, source retry and COW\n", + ps / 1024); +} + +struct reader { + volatile unsigned char *address; + unsigned char value; + atomic_int done; +}; +static void *read_byte(void *opaque) +{ + struct reader *r = opaque; + r->value = *r->address; + atomic_store(&r->done, 1); + return NULL; +} +static void resume_minor(int fd, void *start, size_t len, unsigned long mode) +{ + struct uffdio_continue c = { .range = { (uintptr_t)start, len }, + .mode = mode }; + CHECK(!ioctl(fd, UFFDIO_CONTINUE, &c) && c.mapped == (long long)len); +} +static void file_minor(size_t ps, size_t native) +{ + for (int shared = 0; shared < 2; shared++) { + int memfd = memfd_create("uffd-minor-granule", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * native)); + unsigned char *alias = mmap(NULL, 4 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + memfd, 0); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE, + memfd, ps); + CHECK(alias != MAP_FAILED && p != MAP_FAILED); + CHECK(!mlock2(p, 3 * ps, MLOCK_ONFAULT)); + for (size_t i = 0; i < 4 * native; i++) + alias[i] = (i / ps) ^ (i * 3 + 7); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_CONTINUE)); + + for (unsigned i = 0; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + UFFD_PAGEFAULT_FLAG_MINOR && + !atomic_load(&r.done)); + alias[(i + 1) * ps + 17] ^= 0x3c; + resume_minor(fd, p + i * ps, ps, + UFFDIO_CONTINUE_MODE_WP | + UFFDIO_CONTINUE_MODE_DONTWAKE); + struct uffdio_range range = { (uintptr_t)(p + i * ps), + ps }; + CHECK(!ioctl(fd, UFFDIO_WAKE, &range)); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&r.done)); + CHECK(r.value == alias[(i + 1) * ps + 17]); + CHECK(physical(p + i * ps, ps) == + physical(alias + (i + 1) * ps, ps)); + /* Independent write protection survives installing the file leaf. */ + struct writer w = { .address = p + i * ps + 17, + .value = 0xa1 + i }; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == + (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + protect(fd, p + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && + atomic_load(&w.done)); + CHECK(p[i * ps + 17] == w.value); + CHECK(shared ? alias[(i + 1) * ps + 17] == w.value : + alias[(i + 1) * ps + 17] == r.value); + } + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!munlock(p, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + resume_minor(fd, p, 3 * ps, 0); + CHECK(!memcmp(p, alias + ps, 3 * ps)); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + resume_minor(fd, p, ps, 0); + struct uffdio_continue c = { .range = { (uintptr_t)(p + ps), + ps } }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && errno == EFAULT && + c.mapped == -EFAULT); + CHECK(!close(fd) && !munmap(p, 3 * ps) && + !munmap(alias, 4 * native) && !close(memfd)); + printf("ok - uffd ABI %zuK shmem %s minor/WP, byte offsets, alias coherence, COW, batch continue and EOF\n", + ps / 1024, shared ? "shared" : "private"); + } +} + +static uint64_t pagemap(void *address, size_t ps) +{ + uint64_t entry; + int fd = open("/proc/self/pagemap", O_RDONLY); + CHECK(fd >= 0 && + pread(fd, &entry, 8, (uintptr_t)address / ps * 8) == 8); + CHECK(!close(fd)); + return entry; +} +static sigjmp_buf bus_env; +static void *bus_address; +static void bus_handler(int sig, siginfo_t *info, void *context) +{ + (void)context; + if (sig != SIGBUS) + _exit(91); + bus_address = info->si_addr; + siglongjmp(bus_env, 1); +} +static void markers(size_t ps) +{ + for (int async = 0; async < 2; async++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + UFFD_FEATURE_WP_UNPOPULATED | + (async ? UFFD_FEATURE_WP_ASYNC : 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + for (unsigned i = 0; i < 3; i++) + CHECK(pagemap(p + i * ps, ps) & (1ULL << 57)); + CHECK(*(volatile unsigned char *)(p + ps + 17) == 0); + if (async) + p[ps + 17] = 0x67; + else + resolve_write(fd, p, ps, 1, 0x67, 0); + CHECK(p[ps + 17] == 0x67); + CHECK(pagemap(p, ps) & (1ULL << 57)); + CHECK(!(pagemap(p + ps, ps) & (1ULL << 57))); + CHECK(pagemap(p + 2 * ps, ps) & (1ULL << 57)); + struct pollfd pollfd = { .fd = fd, .events = POLLIN }; + CHECK(poll(&pollfd, 1, 0) == 0); + CHECK(!close(fd)); + p[17] = 0x68; + p[2 * ps + 17] = 0x69; + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK unpopulated WP async=%d per-leaf pagemap state and close\n", + ps / 1024, async); + } + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_POISON }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_POISON))); + zero(fd, p, ps); + zero(fd, p + 2 * ps, ps); + struct uffdio_poison poison = { .range = { (uintptr_t)(p + ps), ps } }; + CHECK(!ioctl(fd, UFFDIO_POISON, &poison) && + poison.updated == (long long)ps); + CHECK(ioctl(fd, UFFDIO_POISON, &poison) == -1 && errno == EEXIST && + poison.updated == -EEXIST); + struct sigaction sa = { .sa_sigaction = bus_handler, + .sa_flags = SA_SIGINFO }, + old; + CHECK(!sigemptyset(&sa.sa_mask) && !sigaction(SIGBUS, &sa, &old)); + if (!sigsetjmp(bus_env, 1)) { + (void)*(volatile unsigned char *)(p + ps + 17); + CHECK(0); + } + CHECK(bus_address == p + ps + 17); + CHECK(!sigaction(SIGBUS, &old, NULL)); + p[17] = 0x81; + p[2 * ps + 17] = 0x82; + CHECK(!close(fd) && !munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", + ps / 1024); +} +struct pager_request { + uintptr_t address; + size_t size; +}; +static void send_pager(int sock, int fd, struct pager_request *request) +{ + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = request, .iov_len = sizeof(*request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + c->cmsg_level = SOL_SOCKET; + c->cmsg_type = SCM_RIGHTS; + c->cmsg_len = CMSG_LEN(sizeof(int)); + memcpy(CMSG_DATA(c), &fd, sizeof(fd)); + CHECK(sendmsg(sock, &msg, 0) == sizeof(*request)); +} +static unsigned char pager_pattern(size_t i) +{ + return (i / 4096) ^ (i * 11 + 5); +} +static int pager_worker(int sock, size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(30); + struct pager_request request; + union { + struct cmsghdr align; + char bytes[CMSG_SPACE(sizeof(int))]; + } control = { 0 }; + struct iovec iov = { .iov_base = &request, .iov_len = sizeof(request) }; + struct msghdr msg = { .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = control.bytes, + .msg_controllen = sizeof(control) }; + CHECK(recvmsg(sock, &msg, 0) == sizeof(request)); + struct cmsghdr *c = CMSG_FIRSTHDR(&msg); + CHECK(c && c->cmsg_level == SOL_SOCKET && c->cmsg_type == SCM_RIGHTS && + c->cmsg_len == CMSG_LEN(sizeof(int))); + int fd; + memcpy(&fd, CMSG_DATA(c), sizeof(fd)); + size_t ps = request.size; + unsigned char *source = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(source != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + source[i] = pager_pattern(i); + for (unsigned i = 0; i < 3; i++) { + struct uffd_msg fault = read_event(fd); + CHECK(fault.event == UFFD_EVENT_PAGEFAULT && + fault.arg.pagefault.address == request.address + i * ps && + !fault.arg.pagefault.flags); + if (i == 1) + zero(fd, (void *)(request.address + ps), ps); + else + fill(fd, (void *)(request.address + i * ps), + source + i * ps, (i == 2 ? 2 : 1) * ps, 0); + } + CHECK(!close(fd) && !close(sock) && !munmap(source, 4 * ps)); + return 0; +} +static void cross_pager(size_t ps, size_t native, int shared) +{ + size_t pager_ps = ps == native ? 4096 : native; + int socks[2]; + CHECK(!socketpair(AF_UNIX, SOCK_SEQPACKET, 0, socks)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + char fd_arg[32], ps_arg[32]; + CHECK(!close(socks[0])); + snprintf(fd_arg, sizeof(fd_arg), "%d", socks[1]); + snprintf(ps_arg, sizeof(ps_arg), "%zu", pager_ps); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, pager_ps, 0UL, 0UL, 0UL)); + execl("/init", "/init", "--pager", fd_arg, ps_arg, NULL); + _exit(3); + } + CHECK(!close(socks[1])); + int memfd = shared ? memfd_create("remote-uffd-shmem", 0) : -1; + if (shared) + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap( + NULL, 4 * ps, PROT_READ | PROT_WRITE, + shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS, memfd, 0); + CHECK(p != MAP_FAILED); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 4 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + struct pager_request request = { .address = (uintptr_t)p, .size = ps }; + send_pager(socks[0], fd, &request); + for (size_t i = 0; i < 4 * ps; i++) + CHECK(p[i] == (i >= ps && i < 2 * ps ? 0 : pager_pattern(i))); + int status; + CHECK(waitpid(child, &status, 0) == child && WIFEXITED(status) && + !WEXITSTATUS(status)); + CHECK(!close(fd) && !close(socks[0]) && !munmap(p, 4 * ps)); + if (shared) + CHECK(!close(memfd)); + printf("ok - uffd target %zuK remote pager %zuK shared=%d SCM_RIGHTS copy/zero/batch uses target granule\n", + ps / 1024, pager_ps / 1024, shared); +} + +static void private_missing(size_t ps) +{ + int memfd = memfd_create("uffd-private-hole", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, memfd, ps); + unsigned char *src = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 3 * ps; i++) + src[i] = pager_pattern(i); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_MISSING }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®) && + (reg.ioctls & (1ULL << _UFFDIO_COPY))); + struct reader r = { .address = p + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + !msg.arg.pagefault.flags); + fill(fd, p, src, 2 * ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == src[17]); + zero(fd, p + 2 * ps, ps); + CHECK(!memcmp(p, src, 2 * ps)); + unsigned char bytes[256]; + for (size_t off = 0; off < 4 * ps; off += sizeof(bytes)) { + CHECK(pread(memfd, bytes, sizeof(bytes), off) == sizeof(bytes)); + for (unsigned i = 0; i < sizeof(bytes); i++) + CHECK(bytes[i] == 0); + } + CHECK(!madvise(p, ps, MADV_DONTNEED)); + CHECK(!madvise(src, 3 * ps, MADV_DONTNEED)); + fill(fd, p, src, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(p[i] == 0); + CHECK(!madvise(p, 3 * ps, MADV_DONTNEED)); + CHECK(!ftruncate(memfd, ps + 17)); + struct uffdio_copy copy = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, ©) == -1 && errno == EFAULT && + copy.copy == -EFAULT); + fill(fd, p, src, ps, 0); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 3 * ps) && + !munmap(src, 3 * ps)); + printf("ok - uffd ABI %zuK private shmem missing COPY/ZERO, unchanged file bytes, source retry and EOF\n", + ps / 1024); +} + +static int register_shared(void *p, size_t len) +{ + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { .api = UFFD_API, + .features = UFFD_FEATURE_MISSING_SHMEM | + UFFD_FEATURE_MINOR_SHMEM | + UFFD_FEATURE_PAGEFAULT_FLAG_WP }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + struct uffdio_register reg = { .range = { (uintptr_t)p, len }, + .mode = UFFDIO_REGISTER_MODE_MISSING | + UFFDIO_REGISTER_MODE_MINOR | + UFFDIO_REGISTER_MODE_WP }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK((reg.ioctls & + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))) == + ((1ULL << _UFFDIO_COPY) | (1ULL << _UFFDIO_ZEROPAGE) | + (1ULL << _UFFDIO_CONTINUE))); + return fd; +} +static void shared_missing(size_t ps, size_t native) +{ + int memfd = memfd_create("uffd-shared-holes", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 8 * native)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *q = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, ps); + unsigned char *src = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && q != MAP_FAILED && src != MAP_FAILED); + for (size_t i = 0; i < 4 * ps; i++) + src[i] = pager_pattern(i); + int fd = register_shared(p, 4 * ps), fd2 = register_shared(q, 4 * ps); + for (unsigned i = 0; i < 2; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + !msg.arg.pagefault.flags); + if (!i) + fill(fd, p, src, ps, UFFDIO_COPY_MODE_WP); + else + zero(fd, p + ps, ps); + CHECK(!pthread_join(thread, NULL) && atomic_load(&r.done)); + CHECK(r.value == (i ? 0 : src[17])); + if (!i) { + struct uffdio_continue c = { + .range = { (uintptr_t)(p + ps), ps } + }; + CHECK(ioctl(fd, UFFDIO_CONTINUE, &c) == -1 && + errno == EFAULT && c.mapped == -EFAULT); + } + } + fill(fd, p + 2 * ps, src + 2 * ps, 2 * ps, 0); + CHECK(!memcmp(p, src, ps) && !memcmp(p + 2 * ps, src + 2 * ps, 2 * ps)); + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == 0); + for (unsigned i = 0; i < 4; i++) { + struct reader r = { .address = q + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd2); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(q + i * ps) && + msg.arg.pagefault.flags == UFFD_PAGEFAULT_FLAG_MINOR); + resume_minor(fd2, q + i * ps, ps, 0); + CHECK(!pthread_join(thread, NULL) && r.value == p[i * ps + 17]); + CHECK(physical(p + i * ps, ps) == physical(q + i * ps, ps)); + } + CHECK(!memcmp(p, q, 4 * ps)); + if (ps < native) + CHECK(physical(p + ps, ps) == physical(p, ps) + ps); + struct writer w = { .address = p + 17, .value = 0x9a }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, write_byte, &w)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)p && + msg.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WRITE | UFFD_PAGEFAULT_FLAG_WP)); + protect(fd, p, ps, 0); + CHECK(!pthread_join(thread, NULL) && p[17] == w.value && + q[17] == w.value); + struct uffdio_copy c = { .dst = (uintptr_t)(p + ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && errno == EEXIST && + c.copy == -EEXIST); + unsigned char bytes[256]; + CHECK(pread(memfd, bytes, sizeof(bytes), ps) == sizeof(bytes)); + CHECK(!memcmp(bytes, p, sizeof(bytes))); + /* Resolve an untouched source outside VMA locks before taking a cache folio. */ + unsigned char *r = mmap(NULL, 2 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 6 * native); + unsigned char *fresh = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(r != MAP_FAILED && fresh != MAP_FAILED); + int fd3 = register_shared(r, 2 * ps); + fill(fd3, r, fresh, ps, 0); + for (size_t i = 0; i < ps; i++) + CHECK(r[i] == 0); + CHECK(!mprotect(fresh, ps, PROT_NONE)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(r + ps), + .src = (uintptr_t)fresh, + .len = ps }; + CHECK(ioctl(fd3, UFFDIO_COPY, &c) == -1 && errno == EFAULT && + c.copy == -EFAULT); + zero(fd3, r + ps, ps); + for (size_t i = 0; i < ps; i++) + CHECK(r[ps + i] == 0); + CHECK(!close(fd3) && !munmap(r, 2 * ps) && !munmap(fresh, ps)); + CHECK(!ftruncate(memfd, 2 * ps + 17)); + c = (struct uffdio_copy){ .dst = (uintptr_t)(p + 2 * ps), + .src = (uintptr_t)src, + .len = ps }; + CHECK(ioctl(fd, UFFDIO_COPY, &c) == -1 && + errno == (ps == native ? ENOMEM : EFAULT) && + c.copy == -(ps == native ? ENOMEM : EFAULT)); + CHECK(!close(fd) && !close(fd2) && !close(memfd)); + CHECK(!munmap(p, 4 * ps) && !munmap(q, 4 * ps) && !munmap(src, 4 * ps)); + printf("ok - uffd ABI %zuK shared COPY/ZERO, independent missing/minor pagers, CONTINUE holes, packing, WP/coherence and EOF\n", + ps / 1024); +} + +static void shared_buffered_write(size_t ps) +{ + int memfd = memfd_create("uffd-buffered-validity", 0); + CHECK(memfd >= 0 && !ftruncate(memfd, 4 * ps)); + unsigned char *p = mmap(NULL, 4 * ps, PROT_READ | PROT_WRITE, + MAP_SHARED, memfd, 0); + unsigned char *src = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED && src != MAP_FAILED); + memset(src, 0x57, ps); + int fd = register_shared(p, 4 * ps); + fill(fd, p, src, ps, 0); + CHECK(pwrite(memfd, src, 23, ps + 17) == 23); + for (unsigned i = 1; i < 3; i++) { + struct reader r = { .address = p + i * ps + 17 }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, read_byte, &r)); + struct uffd_msg msg = read_event(fd); + CHECK(msg.event == UFFD_EVENT_PAGEFAULT && + msg.arg.pagefault.address == (uintptr_t)(p + i * ps) && + msg.arg.pagefault.flags == + (i == 1 ? UFFD_PAGEFAULT_FLAG_MINOR : 0)); + if (i == 1) + resume_minor(fd, p + ps, ps, 0); + else + zero(fd, p + i * ps, ps); + CHECK(!pthread_join(thread, NULL) && + r.value == (i == 1 ? 0x57 : 0)); + } + for (size_t i = 0; i < ps; i++) + CHECK(p[ps + i] == (i >= 17 && i < 40 ? 0x57 : 0)); + CHECK(!memcmp(p, src, ps)); + CHECK(!close(fd) && !close(memfd) && !munmap(p, 4 * ps) && + !munmap(src, ps)); + printf("ok - uffd ABI %zuK buffered short-range write populates its leaf and leaves neighbours missing\n", + ps / 1024); +} + +static int worker(unsigned long expected, unsigned long native) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + alarm(60); + missing(ps, native); + file_minor(ps, native); + markers(ps); + cross_pager(ps, native, 0); + cross_pager(ps, native, 1); + private_missing(ps); + shared_missing(ps, native); + shared_buffered_write(ps); + for (int exact = 0; exact < 2; exact++) { + unsigned char *p = mmap(NULL, 3 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(p != MAP_FAILED); + for (unsigned i = 0; i < 3; i++) + memset(p + i * ps, 0x40 + i, ps); + int fd = syscall(SYS_userfaultfd, + O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY); + CHECK(fd >= 0); + struct uffdio_api api = { + .api = UFFD_API, + .features = UFFD_FEATURE_PAGEFAULT_FLAG_WP | + UFFD_FEATURE_THREAD_ID | + (exact ? UFFD_FEATURE_EXACT_ADDRESS | + UFFD_FEATURE_EVENT_FORK : + 0) + }; + CHECK(!ioctl(fd, UFFDIO_API, &api)); + CHECK(api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP); + struct uffdio_register reg = { + .range = { (uintptr_t)p, 3 * ps }, + .mode = UFFDIO_REGISTER_MODE_WP + }; + CHECK(!ioctl(fd, UFFDIO_REGISTER, ®)); + CHECK(reg.ioctls & (1ULL << _UFFDIO_WRITEPROTECT)); + + for (unsigned i = 0; i < 3; i++) { + protect(fd, p + i * ps, ps, + UFFDIO_WRITEPROTECT_MODE_WP); + /* Neighbours remain directly writable while this page faults. */ + p[((i + 1) % 3) * ps + 31] ^= 1; + resolve_write(fd, p, ps, i, 0x70 + i, exact); + } + protect(fd, p + ps, ps, UFFDIO_WRITEPROTECT_MODE_WP); + pid_t child; + int status; + if (!exact) { + child = fork(); + CHECK(child >= 0); + if (!child) { + alarm(10); + p[ps + 17] = 0xa5; + _exit(p[ps + 17] != 0xa5); + } + } else { + struct forker f = { .base = p, .page_size = ps }; + pthread_t thread; + CHECK(!pthread_create(&thread, NULL, fork_byte, &f)); + struct uffd_msg event = read_event(fd); + CHECK(event.event == UFFD_EVENT_FORK); + int child_fd = event.arg.fork.ufd; + CHECK(child_fd >= 0); + CHECK(!pthread_join(thread, NULL)); + child = f.pid; + CHECK(child > 0); + event = read_event(child_fd); + CHECK(event.event == UFFD_EVENT_PAGEFAULT && + event.arg.pagefault.flags == + (UFFD_PAGEFAULT_FLAG_WP | + UFFD_PAGEFAULT_FLAG_WRITE)); + CHECK(event.arg.pagefault.address == + (uintptr_t)(p + ps + 17)); + CHECK(event.arg.pagefault.feat.ptid == + (unsigned int)child); + protect(child_fd, p + ps, ps, 0); + CHECK(!close(child_fd)); + } + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(p[ps + 17] == 0x71); + resolve_write(fd, p, ps, 1, 0x91, exact); + protect(fd, p, 3 * ps, UFFDIO_WRITEPROTECT_MODE_WP); + struct uffdio_range middle = { (uintptr_t)(p + ps), ps }; + CHECK(!ioctl(fd, UFFDIO_UNREGISTER, &middle)); + p[ps + 17] = 0xb2; + resolve_write(fd, p, ps, 0, 0xb1, exact); + CHECK(!close(fd)); + /* Close removes the remaining registration and write protection. */ + p[2 * ps + 17] = 0xb3; + CHECK(p[17] == 0xb1 && p[ps + 17] == 0xb2 && + p[2 * ps + 17] == 0xb3); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK exact=%d events, neighbours, fork delivery/COW, partial unregister and close\n", + ps / 1024, exact); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 4 && !strcmp(argv[1], "--pager")) + return pager_worker(atoi(argv[2]), strtoul(argv[3], NULL, 10)); + if (argc == 4 && !strcmp(argv[1], "--worker")) + return worker(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10)); + if (getpid() != 1 || argc != 1) + return 2; + CHECK(!mount("proc", "/proc", "proc", 0, NULL)); + unsigned long native = getauxval(AT_PAGESZ), + sizes[] = { native, 4096, 16384 }; + unsigned modes = native == 65536 ? 3 : 2; + int failed = 0; + for (unsigned i = 0; i < modes; i++) { + pid_t pid = fork(); + CHECK(pid >= 0); + if (!pid) { + char ps[32], ns[32]; + snprintf(ps, sizeof(ps), "%lu", sizes[i]); + snprintf(ns, sizeof(ns), "%lu", native); + CHECK(!prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, + 0UL)); + execl("/init", "/init", "--worker", ps, ns, NULL); + _exit(3); + } + int status = 0; + int ok = waitpid(pid, &status, 0) == pid && WIFEXITED(status) && + !WEXITSTATUS(status); + printf("%s - uffd ABI %luK status=%#x\n", ok ? "ok" : "not ok", + sizes[i] / 1024, status); + failed |= !ok; + } + printf("UFFD CONTRACT %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uprobes-init.c b/tools/testing/selftests/arm64/user4k/uprobes-init.c new file mode 100644 index 00000000000000..0c379e4309caf1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uprobes-init.c @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +static int write_file(const char *path, const char *s) +{ + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + size_t len = strlen(s); + int ret = write(fd, s, len) == (ssize_t)len ? 0 : -1; + close(fd); + return ret; +} +static int enable(const char *event, int on) +{ + char path[160]; + snprintf(path, sizeof(path), TRACE "events/aurora_up/%s/enable", event); + return write_file(path, on ? "1" : "0"); +} +static uint64_t pagemap(void *p, size_t ps) +{ + uint64_t bits = 0; + int fd = open("/proc/self/pagemap", O_RDONLY); + if (fd >= 0) { + if (pread(fd, &bits, 8, (uintptr_t)p / ps * 8) != 8) + bits = 0; + close(fd); + } + return bits; +} +static unsigned records(const char *event) +{ + FILE *f = fopen(TRACE "trace", "r"); + char *line = NULL; + size_t len = 0; + unsigned count = 0; + char match[96]; + snprintf(match, sizeof(match), "%s:", event); + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, match)) + count++; + free(line); + fclose(f); + return count; +} +static int xol_geometry(size_t native) +{ + FILE *f = fopen("/proc/self/maps", "r"); + char *line = NULL; + size_t len = 0; + int ok = 0; + if (!f) + return 0; + while (getline(&line, &len, f) > 0) + if (strstr(line, "[uprobes]")) { + unsigned long a, b; + if (sscanf(line, "%lx-%lx", &a, &b) == 2) { + printf("# XOL %#lx-%#lx native=%zu\n", a, b, + native); + ok = !(a % (unsigned long)native) && + b - a == native; + } + } + free(line); + fclose(f); + return ok; +} +static int exercise(size_t ps, size_t native, unsigned shifted) +{ + CHECK(getauxval(AT_PAGESZ) == ps); + alarm(20); + printf("# start ps=%zu shifted=%u\n", ps, shifted); + int fd = memfd_create("aurora-uprobe-code", 0); + CHECK(fd >= 0); + size_t length = 8 * ps; + unsigned char *blob = malloc(length); + CHECK(blob); + memset(blob, 0x71, length); + size_t code_off = 3 * ps + 64, ctr_off = 5 * ps + 32; + size_t map_off = shifted ? 2 * ps : 0, + map_len = shifted ? 2 * ps : 4 * ps; + uint32_t insns[] = { 0x11001c00, 0xd65f03c0 }; + memcpy(blob + code_off, insns, sizeof(insns)); + memset(blob + ctr_off, 0, 2); + CHECK(write(fd, blob, length) == (ssize_t)length); + unsigned char *code = mmap((void *)(0x20000000 + (shifted ? ps : 0)), + map_len, PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, + map_off); + unsigned char *counter = + mmap((void *)(0x30000000 + 2 * ps), ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 5 * ps); + unsigned char *other = mmap((void *)0x40000000, ps, + PROT_READ | PROT_EXEC, + MAP_PRIVATE | MAP_FIXED_NOREPLACE, fd, 0); + CHECK(code != MAP_FAILED && counter != MAP_FAILED && + other != MAP_FAILED); + unsigned long (*target)(unsigned long) = + (void *)(code + code_off - map_off); + volatile unsigned short *ctr = (void *)(counter + 32); + CHECK(*ctr == 0); + CHECK(target(11) == 18); + /* Occupy the default top-of-address-space XOL hint, forcing a new placement. */ + void *guard = mmap((void *)((1UL << 48) - native), ps, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + CHECK(guard != MAP_FAILED); + char entry[64], retname[64], cmd[256]; + snprintf(entry, sizeof(entry), "entry_%zu_%u", ps, shifted); + snprintf(retname, sizeof(retname), "return_%zu_%u", ps, shifted); + snprintf(cmd, sizeof(cmd), + "p:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", entry, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), + "r:aurora_up/%s /proc/self/fd/%d:0x%zx(0x%zx)\n", retname, fd, + code_off, ctr_off); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + printf("# enabling return\n"); + CHECK(!enable(retname, 1)); + printf("# return enabled\n"); + CHECK(*ctr == 1); + uint32_t disk[2]; + CHECK(pread(fd, disk, sizeof(disk), code_off) == sizeof(disk)); + CHECK(!memcmp(disk, insns, sizeof(insns))); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, 32)); + CHECK(!memcmp(counter + 34, blob + 5 * ps + 34, ps - 34)); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + printf("# invoking probes\n"); + for (unsigned i = 0; i < 16; i++) + CHECK(target(i) == i + 7); + printf("# invocations complete\n"); + CHECK(xol_geometry(native)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + int ok = *ctr == 1; + for (unsigned i = 0; i < 8; i++) + ok &= target(100 + i) == 107 + i; + _exit(ok ? 0 : 3); + } + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFEXITED(status) && !WEXITSTATUS(status)); + CHECK(*ctr == 1); + unsigned entries = records(entry), returns = records(retname); + printf("# ABI=%zu entry=%u return=%u\n", ps, entries, returns); + CHECK(entries == 24 && returns == 24); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 1); + CHECK(!enable(retname, 0)); + CHECK(*ctr == 0); + /* Identical restored text is dropped so it can be shared with the file again. */ + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(10) == 17); + CHECK(!memcmp(code, blob + map_off, map_len)); + CHECK(!memcmp(other, blob, ps)); + CHECK(!memcmp(counter, blob + 5 * ps, ps)); + /* A pipe GET must prevent dropping the mapped code slot on removal. */ + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(*ctr == 1); + int pipefd[2]; + CHECK(!pipe(pipefd)); + struct iovec iov = { .iov_base = + (void *)((uintptr_t)target & ~(ps - 1)), + .iov_len = ps }; + CHECK(vmsplice(pipefd[1], &iov, 1, 0) == (ssize_t)ps); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + CHECK(pagemap((void *)target, ps) >> 63); + unsigned char *held = malloc(ps); + CHECK(held); + CHECK(read(pipefd[0], held, ps) == (ssize_t)ps); + CHECK(!memcmp(held, blob + (code_off & ~(ps - 1)), ps)); + free(held); + close(pipefd[0]); + close(pipefd[1]); + printf("# enabling entry at line %d\n", __LINE__); + CHECK(!enable(entry, 1)); + printf("# entry enabled\n"); + CHECK(!enable(entry, 0)); + CHECK(*ctr == 0); + /* Native folio-wide references may defer this optional optimization. */ + if (ps < native) + CHECK(!(pagemap((void *)target, ps) >> 63)); + CHECK(target(20) == 27); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", entry); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + snprintf(cmd, sizeof(cmd), "-:aurora_up/%s\n", retname); + CHECK(!write_file(TRACE "uprobe_events", cmd)); + CHECK(!munmap(code, map_len)); + CHECK(!munmap(guard, ps)); + CHECK(!munmap(counter, ps)); + CHECK(!munmap(other, ps)); + close(fd); + free(blob); + printf("ok - %zuK uprobe/uretprobe, byte offsets, counter, fork, restore and file sharing\n", + ps / 1024); + return 0; +} +static int wait_bounded(pid_t child, int *status) +{ + for (unsigned i = 0; i < 250; i++) { + pid_t got = waitpid(child, status, WNOHANG); + if (got == child) + return 1; + if (got < 0) + return 0; + if (i == 50 || i == 200) { + char path[64], buf[4096]; + snprintf(path, sizeof(path), "/proc/%d/stack", child); + int fd = open(path, O_RDONLY); + ssize_t n = fd < 0 ? -1 : + read(fd, buf, sizeof(buf) - 1); + if (fd >= 0) + close(fd); + if (n > 0) { + buf[n] = 0; + printf("# child %d stack:\n%s", child, buf); + } else + printf("# child %d stack unavailable errno=%d\n", + child, errno); + } + if (i == 200) { + printf("# killing stalled child %d\n", child); + kill(child, SIGKILL); + } + usleep(100000); + } + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10), + strtoul(argv[3], NULL, 10), + strtoul(argv[4], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned run = 0; run < (sizes[0] > 16384 ? 6U : 4U); run++) { + unsigned i = run / 2, shifted = run & 1; + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24], native[24], mode[8]; + snprintf(mode, sizeof(mode), "%u", shifted); + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + snprintf(native, sizeof(native), "%zu", sizes[0]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, native, + mode, NULL); + _exit(5); + } + int ok = c > 0 && wait_bounded(c, &status) && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK uprobes shifted=%u status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, shifted, status); + fail += !ok; + } +done: + printf("UPROBES %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/uring-fixture-init.c b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c new file mode 100644 index 00000000000000..cafc11138d2e8f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/uring-fixture-init.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest.h" + +struct ring { + int fd; + struct io_uring_params p; + void *sq, *cq, *sqes; + size_t sqlen, cqlen, sqelen; +}; +static unsigned load(unsigned *p) +{ + return __atomic_load_n(p, __ATOMIC_ACQUIRE); +} +static void store(unsigned *p, unsigned v) +{ + __atomic_store_n(p, v, __ATOMIC_RELEASE); +} +static size_t align_size(size_t n) +{ + size_t ps = getauxval(AT_PAGESZ); + return (n + ps - 1) & ~(ps - 1); +} +static void destroy(struct ring *r) +{ + if (r->sq && r->sq != MAP_FAILED) + munmap(r->sq, r->sqlen); + if (r->cq && r->cq != MAP_FAILED && r->cq != r->sq) + munmap(r->cq, r->cqlen); + if (r->sqes && r->sqes != MAP_FAILED) + munmap(r->sqes, r->sqelen); + if (r->fd >= 0) + close(r->fd); + *r = (struct ring){ .fd = -1 }; +} +static bool setup(struct ring *r, unsigned entries, unsigned flags) +{ + *r = (struct ring){ .fd = -1 }; + r->p.flags = flags; + r->fd = syscall(__NR_io_uring_setup, entries, &r->p); + if (r->fd < 0) + return false; + r->sqlen = align_size(r->p.sq_off.array + + r->p.sq_entries * sizeof(unsigned)); + r->cqlen = align_size(r->p.cq_off.cqes + + r->p.cq_entries * sizeof(struct io_uring_cqe) * + (flags & IORING_SETUP_CQE32 ? 2 : 1)); + r->sqelen = align_size(r->p.sq_entries * sizeof(struct io_uring_sqe) * + (flags & IORING_SETUP_SQE128 ? 2 : 1)); + if (r->p.features & IORING_FEAT_SINGLE_MMAP) { + if (r->cqlen > r->sqlen) + r->sqlen = r->cqlen; + r->cqlen = r->sqlen; + } + r->sq = mmap(NULL, r->sqlen, PROT_READ | PROT_WRITE, MAP_SHARED, r->fd, + IORING_OFF_SQ_RING); + r->cq = (r->p.features & IORING_FEAT_SINGLE_MMAP) ? + r->sq : + mmap(NULL, r->cqlen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_CQ_RING); + r->sqes = mmap(NULL, r->sqelen, PROT_READ | PROT_WRITE, MAP_SHARED, + r->fd, IORING_OFF_SQES); + return r->sq != MAP_FAILED && r->cq != MAP_FAILED && + r->sqes != MAP_FAILED; +} +static bool request(struct ring *r, unsigned op, int fd, void *buf, + unsigned len, uint64_t off, bool async) +{ + unsigned *sqtail = r->sq + r->p.sq_off.tail, + *sqmask = r->sq + r->p.sq_off.ring_mask; + unsigned *cqhead = r->cq + r->p.cq_off.head, + *cqtail = r->cq + r->p.cq_off.tail, + *cqmask = r->cq + r->p.cq_off.ring_mask; + unsigned tail = load(sqtail), head = load(cqhead), + idx = tail & load(sqmask); + unsigned sqstride = sizeof(struct io_uring_sqe) * + (r->p.flags & IORING_SETUP_SQE128 ? 2 : 1); + unsigned cqstride = sizeof(struct io_uring_cqe) * + (r->p.flags & IORING_SETUP_CQE32 ? 2 : 1); + struct io_uring_sqe *sqe = r->sqes + idx * sqstride; + memset(sqe, 0, sqstride); + sqe->opcode = op; + sqe->fd = fd; + sqe->addr = (uintptr_t)buf; + sqe->len = len; + sqe->off = off; + sqe->user_data = (uint64_t)tail + 1234; + if (async) + sqe->flags = IOSQE_ASYNC; + ((unsigned *)(r->sq + r->p.sq_off.array))[idx] = idx; + store(sqtail, tail + 1); + int ret = syscall(__NR_io_uring_enter, r->fd, 1, 1, + IORING_ENTER_GETEVENTS, NULL, 0); + if (ret != 1 || load(cqtail) != head + 1) { + ksft_print_msg("enter=%d errno=%d\n", ret, errno); + return false; + } + struct io_uring_cqe *cqe = + r->cq + r->p.cq_off.cqes + (head & load(cqmask)) * cqstride; + unsigned expected = op == IORING_OP_NOP ? 0 : len; + if (op == IORING_OP_READV_FIXED || op == IORING_OP_WRITEV_FIXED) { + expected = 0; + for (unsigned i = 0; i < len; i++) + expected += ((struct iovec *)buf)[i].iov_len; + } + bool ok = cqe->user_data == (uint64_t)tail + 1234 && + cqe->res == (int)expected; + if (!ok) + ksft_print_msg("cqe res=%d expected=%u\n", cqe->res, expected); + store(cqhead, head + 1); + return ok; +} +static unsigned long pinned_kb(void) +{ + FILE *f = fopen("/proc/self/status", "r"); + char line[160]; + unsigned long n = ~0UL; + if (!f) + return n; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPin: %lu kB", &n) == 1) + break; + fclose(f); + return n; +} +static bool bytes_are(unsigned char *p, size_t len, unsigned char value) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != value) + return false; + return true; +} +static void fixed_checks(int fd, unsigned char *p, size_t len, size_t ps) +{ + struct ring r = { .fd = -1 }; + unsigned char *q = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + struct iovec reg = { p + 512, len - 1024 }; + bool live = false, ok = setup(&r, 8, 0); + unsigned long baseline = pinned_kb(); + memset(p, 0xa6, len); + if (ok) + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + ksft_test_result(ok, "register unaligned long-term buffer\n"); + if (ok) { + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 7 * 512, 0, + false); + memset(p + 512, 0, 7 * 512); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 7 * 512, 0, + false) && + bytes_are(p + 512, 7 * 512, 0xa6); + } + ksft_test_result(ok, "fixed first-quarter slice offsets\n"); + if (ok) { + size_t off = len - 3 * ps; + memset(p + off, 0x49, 2 * ps); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + off, 2 * ps, + 65536, true); + memset(p + off, 0, 2 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + off, 2 * ps, + 65536, true) && + bytes_are(p + off, 2 * ps, 0x49); + } + ksft_test_result(ok, "fixed late-buffer fast indexing and async I/O\n"); + if (ok) { + struct iovec v[3] = { { p + 512, 512 }, + { p + 2 * ps + 512, ps - 512 }, + { p + 17 * ps + 512, 1024 } }; + memset(p, 0x78, len); + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, 131072, + true); + memset(p, 0x11, len); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, 131072, + true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, 0x78); + ok &= p[0] == 0x11 && p[ps + 512] == 0x11 && p[17 * ps] == 0x11; + } + ksft_test_result(ok, "fixed vectored physical offsets and guards\n"); + if (ok) { + memset(p + 512, 0x3e, len - 1024); + ok = request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true); + memset(p + 512, 0, len - 1024); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, len - 1024, + 4 * 1024 * 1024, true) && + bytes_are(p + 512, len - 1024, 0x3e); + } + ksft_test_result(ok, "fixed multi-bio transfer\n"); + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) == p; + memset(p, 0x55, len); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 8 * 1024 * 1024, true); + if (q != MAP_FAILED) { + memset(q, 0, 4 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 8 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0x3e) && + bytes_are(p, len, 0x55); + } else + ok = false; + } + ksft_test_result(ok, + "registered source survives unmap and replacement\n"); + if (ok) { + memset(q, 0xc1, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, q, 3 * ps, + 10 * 1024 * 1024, false); + ok &= request(&r, IORING_OP_READ_FIXED, fd, p + 512, 3 * ps, + 10 * 1024 * 1024, true); + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, p + 512, 3 * ps, + 12 * 1024 * 1024, true); + memset(q, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, q, 3 * ps, + 12 * 1024 * 1024, false) && + bytes_are(q, 3 * ps, 0xc1) && bytes_are(p, len, 0x55); + } + ksft_test_result( + ok, + "registered destination survives unmap without touching replacement\n"); + if (live) { + ok = syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } else + ok = false; + ok &= pinned_kb() == baseline; + if (ok) { + ok = !mprotect(p + ps, ps, PROT_NONE); + struct iovec bad[2] = { { p, ps }, { p + ps, ps } }; + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, bad, 2) == -1 && + errno == EFAULT; + ok &= !mprotect(p + ps, ps, PROT_READ | PROT_WRITE) && + pinned_kb() == baseline; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + } + ksft_test_result( + ok, + "failed multi-buffer registration rolls back pins and accounting\n"); + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + for (unsigned i = 0; i < 64 && ok; i++) { + reg = (struct iovec){ p + ps + 512, 2 * ps }; + ok = !madvise(p + ps, 3 * ps, MADV_DONTNEED); + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + } + ksft_test_result(ok, + "64 register/reuse/unregister cycles balance VmPin\n"); + /* The file offset deliberately differs from the virtual quarter index. */ + int mem = memfd_create("uring-fixed-shared", 0); + ok &= mem >= 0 && !ftruncate(mem, len + 3 * ps) && q != MAP_FAILED; + if (ok) { + ok = mmap(p, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, mem, 3 * ps) == p; + memset(p, 0x6b, len); + reg = (struct iovec){ p + 512, len - 1024 }; + live = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS, ®, 1) == 0; + ok &= live; + struct iovec v[3] = { { p + 512, 512 }, + { p + 3 * ps + 512, ps - 512 }, + { p + 23 * ps + 512, 1024 } }; + if (ok) { + ok = request(&r, IORING_OP_WRITEV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + memset(p + 512, 0x18, len - 1024); + ok &= request(&r, IORING_OP_READV_FIXED, fd, v, 3, + 14 * 1024 * 1024, true); + for (unsigned i = 0; i < 3; i++) + ok &= bytes_are(v[i].iov_base, v[i].iov_len, + 0x6b); + ok &= !msync(p, len, MS_SYNC) && + pread(mem, q, ps, 3 * ps) == (ssize_t)ps; + ok &= bytes_are(q, 1024, 0x6b) && q[1024] == 0x18 && + p[len - 1] == 0x6b; + } + } + ksft_test_result( + ok, "registered shared-file quarters use physical offsets\n"); + if (ok) { + struct iovec replacement = { q + 512, 3 * ps }; + struct io_uring_rsrc_update2 update = { + .data = (uintptr_t)&replacement, .nr = 1 + }; + memset(q, 0x9d, 4 * ps); + ok = syscall(__NR_io_uring_register, r.fd, + IORING_REGISTER_BUFFERS_UPDATE, &update, + sizeof(update)) == 1; + ok &= request(&r, IORING_OP_WRITE_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true); + memset(q + 512, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ_FIXED, fd, q + 512, 3 * ps, + 16 * 1024 * 1024, true) && + bytes_are(q + 512, 3 * ps, 0x9d); + } + if (live) { + ok &= syscall(__NR_io_uring_register, r.fd, + IORING_UNREGISTER_BUFFERS, NULL, 0) == 0; + live = false; + } + ok &= pinned_kb() == baseline; + ksft_test_result( + ok, + "registered-buffer update transfers ownership and balances accounting\n"); + if (mem >= 0) + close(mem); + if (live) + syscall(__NR_io_uring_register, r.fd, IORING_UNREGISTER_BUFFERS, + NULL, 0); + destroy(&r); + if (q != MAP_FAILED) + munmap(q, len); +} + +static int exercise(void) +{ + struct ring r = { .fd = -1 }; + size_t ps = getauxval(AT_PAGESZ), len = 2 * 1024 * 1024; + unsigned char *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fd = open("/dev/user4k-test", O_RDWR | O_DIRECT); + bool ok; + ksft_print_header(); + ksft_set_plan(19); + ksft_print_msg("ABI=%zuK\n", ps / 1024); + ok = setup(&r, 4, 0); + ksft_test_result(ok, "minimum rounded ring mappings\n"); + void *bad = ok ? mmap(NULL, ps, PROT_READ | PROT_WRITE, MAP_SHARED, + r.fd, 4096) : + MAP_FAILED; + bool rejected = ok && bad == MAP_FAILED && errno == EINVAL; + if (bad != MAP_FAILED) + munmap(bad, ps); + ksft_test_result(rejected, + "reject sub-native offset in ring selector\n"); + if (ok) { + ok = true; + for (unsigned i = 0; i < 64 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + } + ksft_test_result(ok, "small submission/completion ring wrap\n"); + if (p == MAP_FAILED || fd < 0) + ksft_exit_fail_msg("buffer/device setup errno=%d\n", errno); + if (ok) { + memset(p + ps, 0x69, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 0, false); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 0, false); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x69; + } + ksft_test_result(ok, + "direct read/write from unregistered user buffers\n"); + if (ok) { + memset(p + ps, 0x37, 3 * ps); + ok = request(&r, IORING_OP_WRITE, fd, p + ps, 3 * ps, 65536, + true); + memset(p + ps, 0, 3 * ps); + ok &= request(&r, IORING_OP_READ, fd, p + ps, 3 * ps, 65536, + true); + for (size_t i = ps; i < 4 * ps; i++) + ok &= p[i] == 0x37; + } + ksft_test_result(ok, "io-worker forced asynchronous read/write\n"); + if (ok) { + memset(p, 0xb6, len); + ok = request(&r, IORING_OP_WRITE, fd, p, len, 4 * 1024 * 1024, + true); + memset(p, 0, len); + ok &= request(&r, IORING_OP_READ, fd, p, len, 4 * 1024 * 1024, + true); + for (size_t i = 0; i < len; i++) + ok &= p[i] == 0xb6; + } + ksft_test_result(ok, "multi-bio io-worker transfer\n"); + destroy(&r); + ok = setup(&r, 4096, 0); + if (ok) + for (unsigned i = 0; i < 8193 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "large ring crosses native pages and wraps\n"); + destroy(&r); + ok = setup(&r, 64, IORING_SETUP_SQE128 | IORING_SETUP_CQE32); + if (ok) + for (unsigned i = 0; i < 129 && ok; i++) + ok = request(&r, IORING_OP_NOP, -1, NULL, 0, 0, false); + ksft_test_result(ok, "128-byte SQEs and 32-byte CQEs\n"); + destroy(&r); + fixed_checks(fd, p, len, ps); + close(fd); + munmap(p, len); + ksft_print_cnts(); + return ksft_get_fail_cnt() ? 1 : 0; +} +int main(int argc, char **argv) +{ + bool ok = true; + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 2 && !strcmp(argv[1], "--test") && getppid() == 1) + return exercise(); + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/block/vda/dev", "r"); + unsigned maj, min; + if (!dev || fscanf(dev, "%u:%u", &maj, &min) != 2) + return 4; + fclose(dev); + if (mknod("/dev/user4k-test", S_IFBLK | 0600, makedev(maj, min))) + return 5; + for (unsigned small = 0; small < 2; small++) { + pid_t pid = fork(); + int status = 0; + if (!pid) { + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + small ? 4096UL : 0UL, 0UL, 0UL, 0UL)) + _exit(3); + execl("/init", "/init", "--test", NULL); + _exit(4); + } + ok &= pid > 0 && waitpid(pid, &status, 0) == pid && + WIFEXITED(status) && !WEXITSTATUS(status); + } + printf("%s - io_uring fixture complete\n", ok ? "ok" : "not ok"); + reboot(RB_POWER_OFF); + return ok ? 0 : 1; +} diff --git a/tools/testing/selftests/arm64/user4k/user-events-init.c b/tools/testing/selftests/arm64/user4k/user-events-init.c new file mode 100644 index 00000000000000..4b35954e2a210a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user-events-init.c @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define TRACE "/sys/kernel/tracing/" +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) + +static int toggle(const char *name, int enabled) +{ + char path[192]; + snprintf(path, sizeof(path), TRACE "events/user_events/%s/enable", + name); + int fd = open(path, O_WRONLY); + if (fd < 0) + return -1; + int ret = write(fd, enabled ? "1" : "0", 1) == 1 ? 0 : -1; + close(fd); + return ret; +} +static uint64_t value(void *p, unsigned bytes) +{ + return bytes == 4 ? __atomic_load_n((uint32_t *)p, __ATOMIC_ACQUIRE) : + __atomic_load_n((uint64_t *)p, __ATOMIC_ACQUIRE); +} +static int wait_value(void *p, unsigned bytes, uint64_t expected) +{ + for (unsigned i = 0; i < 3000; i++) { + if (value(p, bytes) == expected) + return 1; + usleep(1000); + } + return 0; +} +static int check_bytes(unsigned char *p, size_t size, unsigned char fill, + void *word, unsigned bytes) +{ + for (size_t i = 0; i < size; i++) + if ((!word || (uintptr_t)(p + i) < (uintptr_t)word || + (uintptr_t)(p + i) >= (uintptr_t)word + bytes) && + p[i] != fill) { + printf("byte mismatch at %zu got=%#x expected=%#x\n", i, + p[i], fill); + return 0; + } + return 1; +} +static int event_recorded(const char *name) +{ + char buf[16384]; + size_t n = 0; + int fd = open(TRACE "trace", O_RDONLY); + if (fd < 0) + return 0; + while (n < sizeof(buf) - 1) { + ssize_t got = read(fd, buf + n, sizeof(buf) - 1 - n); + if (got < 0) { + close(fd); + return 0; + } + if (!got) + break; + n += got; + } + close(fd); + buf[n] = 0; + char *record = strstr(buf, name), + *payload = record ? strstr(record, "payload=") : NULL; + int ok = payload && strtoul(payload + 8, NULL, 0) == 0x12345678; + if (!ok) + printf("trace mismatch:\n%s\n", buf); + return ok; +} +static int exercise_case(size_t ps, unsigned bytes, unsigned shared) +{ + const size_t len = 4 * ps; + unsigned char *src, *dst; + int memfd = -1; + if (shared) { + memfd = memfd_create("user-event-bitmap", 0); + CHECK(memfd >= 0); + CHECK(!ftruncate(memfd, len)); + } + src = mmap((void *)0x20000000, len, PROT_READ | PROT_WRITE, + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS) | + MAP_FIXED_NOREPLACE, + memfd, 0); + dst = mmap((void *)0x30000000, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, len); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps; + void *word = leaf + ps - bytes; + uint64_t mask = bytes == 4 ? 1ULL << 31 : 1ULL << 63; + memset(word, 0, bytes); + char name[80], spec[120]; + snprintf(name, sizeof(name), "aurora_ue_%zu_%u_%u", ps, bytes, shared); + snprintf(spec, sizeof(spec), "%s u32 payload", name); + int tracefd = open(TRACE "trace", O_WRONLY | O_TRUNC); + CHECK(tracefd >= 0); + close(tracefd); + int fd = open(TRACE "user_events_data", O_RDWR); + CHECK(fd >= 0); + struct user_reg reg = { .size = sizeof(reg), + .enable_bit = bytes * 8 - 1, + .enable_size = bytes, + .enable_addr = (uintptr_t)word, + .name_args = (uintptr_t)spec }; + CHECK(!ioctl(fd, DIAG_IOCSREG, ®)); + CHECK(wait_value(word, bytes, 0)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + struct { + uint32_t index, payload; + } event = { reg.write_index, 0x12345678 }; + CHECK(write(fd, &event, sizeof(event)) == sizeof(event)); + CHECK(event_recorded(name)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + /* Every other byte, including the adjacent half of the kernel word, is stable. */ + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + CHECK(check_bytes(dst, ps, 0x91, NULL, 0)); + CHECK(check_bytes(dst + 2 * ps, 2 * ps, 0x91, NULL, 0)); + int commands[2], reply[2]; + CHECK(!pipe(commands) && !pipe(reply)); + pid_t child = fork(); + CHECK(child >= 0); + if (!child) { + close(commands[1]); + close(reply[0]); + char token; + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + if (read(commands[0], &token, 1) != 1) { + ok = 0; + break; + } + ok = wait_value(word, bytes, token == '1' ? mask : 0) && + check_bytes(leaf, ps, 0x64, word, bytes); + token = ok ? 'y' : 'n'; + if (write(reply[1], &token, 1) != 1 || !ok) + break; + } + _exit(ok ? 0 : 3); + } + close(commands[0]); + close(reply[1]); + int ok = 1; + for (unsigned round = 0; round < 32; round++) { + int on = !(round & 1); + char token = on ? '1' : '0'; + if (toggle(name, on) || + !wait_value(word, bytes, on ? mask : 0) || + write(commands[1], &token, 1) != 1 || + read(reply[0], &token, 1) != 1 || token != 'y') { + ok = 0; + break; + } + } + close(commands[1]); + close(reply[0]); + int status; + CHECK(waitpid(child, &status, 0) == child); + CHECK(ok && WIFEXITED(status) && WEXITSTATUS(status) == 0); + CHECK(check_bytes(leaf, ps, 0x64, word, bytes)); + for (unsigned i = 0; i < 3; i++) + CHECK(check_bytes(src + i * ps, ps, 0x61 + i, NULL, 0)); + /* Force the asynchronous missing-PTE fixup without unregistering. */ + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + CHECK(!toggle(name, 1)); + CHECK(wait_value(word, bytes, mask)); + CHECK(check_bytes(leaf, ps, shared ? 0x64 : 0, word, bytes)); + CHECK(!toggle(name, 0)); + CHECK(wait_value(word, bytes, 0)); + struct user_unreg unreg = { .size = sizeof(unreg), + .disable_bit = reg.enable_bit, + .disable_addr = (uintptr_t)word }; + CHECK(!ioctl(fd, DIAG_IOCSUNREG, &unreg)); + close(fd); + CHECK(!munmap(src, 3 * ps)); + CHECK(!munmap(dst, len)); + if (memfd >= 0) + close(memfd); + printf("ok - %zuK %u-bit %s relocated bitmap, adjacent bytes, event write and fork toggles\n", + ps / 1024, bytes * 8, shared ? "shared" : "private"); + return 0; +} +static int exercise(size_t expected) +{ + size_t ps = getauxval(AT_PAGESZ); + CHECK(ps == expected); + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned bytes = 4; bytes <= 8; bytes += 4) + CHECK(!exercise_case(ps, bytes, shared)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL) || + mount("tracefs", TRACE, "tracefs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("%s - ABI %zuK user events status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("USER EVENTS %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c new file mode 100644 index 00000000000000..b0a2f2bf385f0a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/user16k-smoke-init.c @@ -0,0 +1,229 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define SET_EXEC_PAGE_SIZE 0x41555001 +#define GET_EXEC_PAGE_SIZE 0x41555002 +static int failed; +static _Thread_local int tls = 7; +static void result(int ok, const char *name) +{ + printf("%s - %s\n", ok ? "ok" : "not ok", name); + failed += !ok; +} +static long pss(void *p) +{ + FILE *f = fopen("/proc/self/smaps", "r"); + char *line = NULL; + size_t cap = 0; + unsigned long start, end; + int active = 0; + long value = -1; + if (!f) + return -1; + while (getline(&line, &cap, f) > 0) { + if (sscanf(line, "%lx-%lx", &start, &end) == 2) + active = (unsigned long)p >= start && + (unsigned long)p < end; + else if (active && sscanf(line, "Pss: %ld kB", &value) == 1) + break; + } + free(line); + fclose(f); + return value; +} +static void *thread_worker(void *arg) +{ + unsigned char *p = arg; + if (tls != 7) + return (void *)1; + tls = 91; + for (unsigned i = 0; i < 16384; i++) + if (p[i] != 0x42) + return (void *)2; + return NULL; +} +static void memory_checks(void) +{ + const size_t ps = 16384; + int status = 0; + pid_t c; + unsigned char *p = mmap(NULL, 6 * ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + result(p != MAP_FAILED, "16K anonymous mmap"); + if (p == MAP_FAILED) + return; + for (unsigned i = 0; i < 6; i++) + memset(p + i * ps, 0x40 + i, ps); + c = fork(); + if (!c) { + memset(p + ps, 0xb7, ps); + _exit(p[0] == 0x40 && p[2 * ps] == 0x42 ? 0 : 1); + } + int ok = c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status); + for (unsigned i = 0; i < 6 * ps; i++) + ok &= p[i] == 0x40 + i / ps; + result(ok, "16K fork COW preserves every parent byte"); + result(!mprotect(p + ps, ps, PROT_READ), "16K leaf write protection"); + c = fork(); + if (!c) { + *(volatile unsigned char *)(p + ps + 8192) = 3; + _exit(1); + } + result(c > 0 && waitpid(c, &status, 0) == c && WIFSIGNALED(status) && + WTERMSIG(status) == SIGSEGV, + "write protection covers the interior of a16K leaf"); + result(!mprotect(p + ps, ps, PROT_READ | PROT_WRITE), + "16K write permission restored"); + errno = 0; + result(mprotect(p + 4096, ps, PROT_READ) == -1 && errno == EINVAL, + "16K ABI rejects4K-only protection alignment"); + result(!madvise(p + ps, ps, MADV_DONTNEED), + "16K partial unmap via DONTNEED"); + ok = 1; + for (unsigned i = 0; i < ps; i++) + ok &= p[i] == 0x40 && p[ps + i] == 0 && p[2 * ps + i] == 0x42; + result(ok, "16K discard clears one leaf and preserves neighbours"); + pthread_t thread; + void *answer = (void *)3; + int err = pthread_create(&thread, NULL, thread_worker, p + 2 * ps); + if (!err) + err = pthread_join(thread, &answer); + result(!err && !answer && tls == 7, "16K pthread stack and TLS"); + struct timespec ts; + result(!clock_gettime(CLOCK_MONOTONIC, &ts), "16K vDSO clock access"); + munmap(p, 6 * ps); +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGPIPE, SIG_IGN); + if (argc == 6 && !strcmp(argv[1], "--child")) { + int fd = atoi(argv[2]), ready = atoi(argv[3]), + release = atoi(argv[4]); + size_t native = strtoul(argv[5], NULL, 10); + const size_t ps = 16384; + result(getauxval(AT_PAGESZ) == ps && getpagesize() == ps, + "EL0 selects16K pages"); + result(prctl(GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL) == 0, + "exec consumes alternative selection"); + if (!failed) + memory_checks(); + { + int status = 0; + pid_t c = fork(); + if (!c) { + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, + 0UL)) + _exit(4); + execl("/page-contract", "/page-contract", NULL); + _exit(5); + } + result(c > 0 && waitpid(c, &status, 0) == c && + WIFEXITED(status) && + !WEXITSTATUS(status), + "16K-to16K exec and page/x18 contract"); + } + unsigned char *shared = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, native - ps); + unsigned char *private = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd, native - ps); + int ok = shared != MAP_FAILED && private != MAP_FAILED; + if (ok) { + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0x7b; + memset(private, 0xd8, ps); + for (unsigned i = 0; i < ps; i++) + ok &= shared[i] == 0x7b && private[i] == 0xd8; + } + result(ok, "16K file COW copies the whole leaf"); + if (private != MAP_FAILED) + munmap(private, ps); + result(shared != MAP_FAILED && pss(shared) == 8, + "16K file PSS shares all four4K slices"); + if (shared != MAP_FAILED) + shared[ps - 1] = 0xc9; + char token = failed ? 'x' : 'a'; + if (write(ready, &token, 1) != 1 || + read(release, &token, 1) != 1) + return 3; + if (shared != MAP_FAILED) + munmap(shared, ps); + return failed ? 1 : 0; + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + mount("proc", "/proc", "proc", 0, NULL); + size_t native = getauxval(AT_PAGESZ); + int fd = memfd_create("mixed16-smoke", 0); + if (native != 65536 || fd < 0 || ftruncate(fd, native)) { + result(0, "64K parent setup"); + goto done; + } + unsigned char *p = + mmap(NULL, native, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + result(0, "parent shared mmap"); + goto done; + } + memset(p, 0x7b, native); + result(pss(p) == 64, "native64K initial PSS"); + int ready[2], release[2]; + if (pipe(ready) || pipe(release)) { + result(0, "pipes"); + goto done; + } + pid_t c = fork(); + int status = 0; + if (!c) { + char f[24], r[24], w[24], n[24]; + snprintf(f, sizeof(f), "%d", fd); + snprintf(r, sizeof(r), "%d", ready[1]); + snprintf(w, sizeof(w), "%d", release[0]); + snprintf(n, sizeof(n), "%zu", native); + close(ready[0]); + close(release[1]); + if (prctl(SET_EXEC_PAGE_SIZE, 16384UL, 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--child", f, r, w, n, NULL); + _exit(5); + } + close(ready[1]); + close(release[0]); + char token = 0; + int ok = c > 0 && read(ready[0], &token, 1) == 1 && token == 'a'; + result(ok && p[native - 1] == 0xc9, + "native64K and16K shared bytes remain coherent"); + result(pss(p) == 56, + "native64K PSS accounts one16K alias as four shared slices"); + token = 'b'; + if (write(release[1], &token, 1) != 1) + failed++; + result(c > 0 && waitpid(c, &status, 0) == c && WIFEXITED(status) && + !WEXITSTATUS(status), + "16K child completes"); + printf("# child status=%#x\n", status); + result(pss(p) == 64, "native PSS returns after16K exit"); + munmap(p, native); +done: + printf("USER16K SMOKE %s\n", failed ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c new file mode 100644 index 00000000000000..c43e28ca8b21d1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vfio-fixture-init.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#define __user +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../../../../../include/uapi/linux/vfio.h" +#include "../../../../../drivers/vfio/vfio_iommu_type1_test.h" + +#define IOVA (256UL * 1024 * 1024) +static unsigned int number, failures; + +static void result(int ok, const char *name) +{ + printf("%s %u - %s (errno=%d)\n", ok ? "ok" : "not ok", ++number, name, + errno); + failures += !ok; +} + +static int state(int fd, struct vfio_type1_test_cmd *cmd) +{ + memset(cmd, 0, sizeof(*cmd)); + return ioctl(fd, VFIO_TYPE1_TEST_STATE, cmd); +} + +static int domain(int fd, unsigned int which, int add, size_t page, size_t fail) +{ + struct vfio_type1_test_cmd cmd = { + .domain = which, + .size = page, + .fail_after = fail, + }; + + return ioctl(fd, + add ? VFIO_TYPE1_TEST_DOMAIN_ADD : + VFIO_TYPE1_TEST_DOMAIN_DEL, + &cmd); +} + +static int rw(int fd, size_t offset, void *data, size_t size, + unsigned int flags) +{ + struct vfio_type1_test_cmd cmd = { + .iova = IOVA + offset, + .size = size, + .buffer = (uintptr_t)data, + .flags = flags, + }; + + return ioctl(fd, VFIO_TYPE1_TEST_RW, &cmd); +} + +static int map(int fd, void *data, size_t size) +{ + struct vfio_iommu_type1_dma_map cmd = { + .argsz = sizeof(cmd), + .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA, + .vaddr = (uintptr_t)data, + .size = size, + }; + + return ioctl(fd, VFIO_IOMMU_MAP_DMA, &cmd); +} + +static int unmap(int fd, size_t size) +{ + struct vfio_iommu_type1_dma_unmap cmd = { + .argsz = sizeof(cmd), + .iova = IOVA, + .size = size, + }; + int ret = ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &cmd); + + return ret ? ret : cmd.size == size ? 0 : -2; +} + +static int dirty(int fd, unsigned int flags) +{ + struct vfio_iommu_type1_dirty_bitmap cmd = { .argsz = sizeof(cmd), + .flags = flags }; + + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, &cmd); +} + +static int bitmap(int fd, size_t size, uint64_t *bits) +{ + unsigned char + buffer[sizeof(struct vfio_iommu_type1_dirty_bitmap) + + sizeof(struct vfio_iommu_type1_dirty_bitmap_get)] = {}; + struct vfio_iommu_type1_dirty_bitmap *cmd = (void *)buffer; + struct vfio_iommu_type1_dirty_bitmap_get *range = (void *)cmd->data; + + cmd->argsz = sizeof(buffer); + cmd->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP; + range->iova = IOVA; + range->size = size; + range->bitmap.pgsize = 4096; + range->bitmap.size = sizeof(*bits); + range->bitmap.data = (void *)bits; + *bits = 0; + return ioctl(fd, VFIO_IOMMU_DIRTY_PAGES, buffer); +} + +static int all_bytes(const unsigned char *data, size_t size, + unsigned char value) +{ + for (size_t i = 0; i < size; i++) + if (data[i] != value) + return 0; + return 1; +} + +static void external_span(int backing, size_t native) +{ + const size_t fine = 4096, count = native / fine; + uint64_t bits; + + for (unsigned int mapped = 0; mapped < 2; mapped++) { + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_dma_unmap remove = { + .argsz = sizeof(remove), + .iova = IOVA, + .size = fine, + }; + int fd = open("/dev/vfio-test", O_RDWR), status = 0; + pid_t child; + + result(fd >= 0 && (!mapped || !domain(fd, 0, 1, fine, 0)), + "span fixture opens lazy or mapped type1 container"); + if (fd < 0) + continue; + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "span dirty tracking starts before fine mappings"); + child = fork(); + if (!child) { + unsigned char *data = mmap(NULL, native, + PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + + if (data == MAP_FAILED) + _exit(1); + for (size_t i = 0; i < count; i++) { + struct vfio_iommu_type1_dma_map area = { + .argsz = sizeof(area), + .flags = VFIO_DMA_MAP_FLAG_READ | + VFIO_DMA_MAP_FLAG_WRITE, + .iova = IOVA + i * fine, + .vaddr = (uintptr_t)data + i * fine, + .size = fine, + }; + + if (ioctl(fd, VFIO_IOMMU_MAP_DMA, &area)) + _exit(2); + } + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + _exit(ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) ? 3 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "source pins native page across independent fine MAP ioctls"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.external_refs == 1 && + cmd.locked_bytes == count * native && + cmd.owner_locked_bytes == count * native, + "every fine DMA area retains its own charge after source exit"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && + !state(fd, &cmd) && cmd.external_refs == 2 && + cmd.locked_bytes == count * native, + "cross-area duplicate pin reuses all cached constituent pins"); + result(!bitmap(fd, native, &bits) && + bits == ((1ULL << count) - 1), + "cross-area dirty bitmap marks only each mapping's fine extent"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == fine && !state(fd, &cmd) && + !cmd.external_refs && cmd.notifications == 1 && + cmd.locked_bytes == + (mapped ? (count - 1) * native : 0), + "unmapping one constituent drains references in all page constituents"); + remove.iova = IOVA + fine; + remove.size = native - fine; + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &remove) && + remove.size == native - fine && + !state(fd, &cmd) && !cmd.locked_bytes && + !cmd.mapped_bytes[0], + "remaining fine mappings unmap without pins or translations"); + close(fd); + } +} + +static void native_owner(int fd, int backing, unsigned char *data, + size_t native) +{ + struct vfio_type1_test_cmd cmd = {}; + struct vfio_iommu_type1_dma_unmap invalidate = { + .argsz = sizeof(invalidate), + .flags = VFIO_DMA_UNMAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + }; + struct vfio_iommu_type1_dma_map update = { + .argsz = sizeof(update), + .flags = VFIO_DMA_MAP_FLAG_VADDR, + .iova = IOVA, + .size = 2 * native, + .vaddr = (uintptr_t)data, + }; + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3 + }; + struct __user_cap_data_struct saved[2], reduced[2]; + struct rlimit limit, empty; + unsigned char bytes[64]; + int file_flags = fcntl(backing, F_GETFD), status = 0; + pid_t child; + int budget_ready, budget_captured, transferred; + + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd) && + !domain(fd, 0, 1, 4096, 0), + "VADDR fixture retains production group restrictions"); + result(file_flags >= 0 && + !fcntl(backing, F_SETFD, file_flags & ~FD_CLOEXEC), + "native source inherits exact backing descriptor"); + child = fork(); + if (!child) { + char fd_arg[24], file_arg[24], native_arg[24]; + + snprintf(fd_arg, sizeof(fd_arg), "%d", fd); + snprintf(file_arg, sizeof(file_arg), "%d", backing); + snprintf(native_arg, sizeof(native_arg), "%zu", native); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL)) + _exit(98); + execl("/init", "/init", "--native-source", fd_arg, file_arg, + native_arg, NULL); + _exit(99); + } + if (file_flags >= 0) + fcntl(backing, F_SETFD, file_flags); + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status) && + !state(fd, &cmd) && !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "exited native source owns legacy GUP pins before transfer"); + result(!ioctl(fd, VFIO_IOMMU_UNMAP_DMA, &invalidate), + "canonical VADDR invalidation leaves device translations alive"); + budget_captured = !getrlimit(RLIMIT_MEMLOCK, &limit) && + !syscall(SYS_capget, &header, saved); + budget_ready = budget_captured; + if (budget_ready) { + memcpy(reduced, saved, sizeof(saved)); + reduced[CAP_IPC_LOCK / 32].effective &= + ~(1U << (CAP_IPC_LOCK % 32)); + empty = (struct rlimit){ .rlim_cur = 0, + .rlim_max = limit.rlim_max }; + budget_ready = !setrlimit(RLIMIT_MEMLOCK, &empty) && + !syscall(SYS_capset, &header, reduced); + } + errno = 0; + result(budget_ready && ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) < 0 && + errno == ENOMEM && !state(fd, &cmd) && + !cmd.owner_users && + !(cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.mapped_bytes[0] == 2 * native, + "failed owner credit preserves native pins, metadata and charge"); + if (budget_captured) { + result(!syscall(SYS_capset, &header, saved) && + !setrlimit(RLIMIT_MEMLOCK, &limit), + "restore original memlock budget and effective capability"); + } else { + result(0, "owner budget setup failed"); + } + transferred = !ioctl(fd, VFIO_IOMMU_MAP_DMA, &update) && + !state(fd, &cmd) && cmd.owner_users && + (cmd.flags & VFIO_TYPE1_TEST_STATE_FRAGMENTS) && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native; + result(transferred, + "canonical VADDR retry adopts native pins for alternative owner"); + /* Dependent CPU access is prohibited while VADDR remains invalid. */ + if (!transferred) + return; + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "new owner accesses retained native buffer with its own mm authorization"); + cmd = (struct vfio_type1_test_cmd){ .flags = 1 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_EMULATED, &cmd), + "external consumer rejoins after VADDR update"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == 2 * native && cmd.external_refs == 1, + "external page array reuses adopted pins without new charge"); + result(!unmap(fd, 2 * native) && !state(fd, &cmd) && + !cmd.locked_bytes && !cmd.owner_locked_bytes && + !cmd.external_refs && !cmd.mapped_bytes[0], + "owner transfer final cleanup refunds recipient and clears domain"); + result(!domain(fd, 0, 0, 0, 0), "owner transfer domain detaches"); +} + +static int exercise(size_t native) +{ + struct vfio_type1_test_cmd cmd; + struct vfio_iommu_type1_info info = { .argsz = sizeof(info) }; + struct vfio_iommu_type1_dma_map bad = { .argsz = 8 }; + unsigned char bytes[64], disk[64]; + uint64_t bits = 0; + int fd = open("/dev/vfio-test", O_RDWR), + backing = memfd_create("vfio", MFD_CLOEXEC); + unsigned char *data = MAP_FAILED; + int status = 0; + pid_t child; + size_t length = native + 4096; + + printf("TAP version 13\n# real type1 ioctl ABI=%luK native=%zuK\n", + getauxval(AT_PAGESZ) / 1024, native / 1024); + result(fd >= 0 && backing >= 0 && !ftruncate(backing, 2 * native), + "open privileged recording-domain fixture and backing"); + if (fd < 0 || backing < 0 || failures) + goto out; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + result(data != MAP_FAILED, + "map native-sized shmem backing in actual process mm"); + if (data == MAP_FAILED) + goto out; + memset(data, 0x64, 2 * native); + result(!ioctl(fd, VFIO_IOMMU_GET_INFO, &info) && + (info.iova_pgsizes & 4096), + "type1 GET_INFO reports fine IOVA capability"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, &bad) < 0 && errno == EINVAL, + "canonical MAP rejects truncated argument"); + errno = 0; + result(ioctl(fd, VFIO_IOMMU_MAP_DMA, (void *)1) < 0 && errno == EFAULT, + "canonical MAP rejects inaccessible userspace argument"); + result(!map(fd, data, 2 * native), "lazy full-range MAP from real EL0"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0], + "lazy MAP acquires no DMA pins or translations"); + result(!rw(fd, 0, bytes, sizeof(bytes), 0) && + all_bytes(bytes, sizeof(bytes), 0x64), + "CPU-only GET reads actual source bytes"); + result(!state(fd, &cmd) && !cmd.locked_bytes, + "CPU GET does not charge long-term pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "canonical dirty tracking starts"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 17 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 1, + "external page array preserves unaligned caller byte offset"); + cmd = (struct vfio_type1_test_cmd){ .iova = IOVA + 31 }; + result(!ioctl(fd, VFIO_TYPE1_TEST_PIN, &cmd) && !state(fd, &cmd) && + cmd.locked_bytes == native && cmd.external_refs == 2, + "duplicate external pin charges physical native page once"); + result(!bitmap(fd, 2 * native, &bits) && + bits == ((1ULL << (native / 4096)) - 1), + "dirty bitmap covers every fine unit in native external page"); + result(!unmap(fd, 2 * native), + "canonical UNMAP drains callback-held external refs"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.external_refs && + cmd.notifications == 1, + "external-unpin callback refunds all pins"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP), + "dirty tracking stops"); + result(!domain(fd, 0, 1, 4096, 0), + "recording4K domain attaches before child MAP"); + child = fork(); + if (!child) { + unsigned char *source = mmap(NULL, 2 * native, + PROT_READ | PROT_WRITE, MAP_SHARED, + backing, 0); + int ret = source == MAP_FAILED ? -1 : + map(fd, source + 4096, length); + + _exit(ret ? 1 : 0); + } + result(child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status), + "fine-offset MAP retains real child pins after exit"); + result(!state(fd, &cmd) && !cmd.owner_users && + cmd.locked_bytes == 2 * native && + cmd.owner_locked_bytes == 2 * native && + cmd.mapped_bytes[0] == length, + "exited source mm retains exact physical-page charge and translation length"); + errno = 0; + result(rw(fd, 0, bytes, sizeof(bytes), 0) < 0 && errno == EFAULT, + "foreign process CPU access preserves source-mm authorization"); + result(!rw(fd, 0, bytes, sizeof(bytes), VFIO_TYPE1_TEST_WORKER) && + all_bytes(bytes, sizeof(bytes), 0x64), + "kernel consumer reads exited source pins"); + errno = 0; + result(domain(fd, 1, 1, 4096, 4096) < 0 && errno == ENOMEM && + !state(fd, &cmd) && !cmd.mapped_bytes[1] && + cmd.mapped_bytes[0] == length && + cmd.locked_bytes == 2 * native, + "failed replay leaves existing domain and pins intact"); + result(!domain(fd, 1, 1, 4096, 0) && !state(fd, &cmd) && + cmd.mapped_bytes[0] == length && + cmd.mapped_bytes[1] == length && + cmd.locked_bytes == 2 * native, + "retry replays cached pins without resurrecting source mm"); + result(!dirty(fd, VFIO_IOMMU_DIRTY_PAGES_FLAG_START), + "fine DMA dirty tracking starts"); + memset(bytes, 0x9a, sizeof(bytes)); + result(!rw(fd, 4096 - 32, bytes, sizeof(bytes), + VFIO_TYPE1_TEST_WRITE | VFIO_TYPE1_TEST_WORKER) && + pread(backing, disk, sizeof(disk), 8192 - 32) == + sizeof(disk) && + all_bytes(disk, sizeof(disk), 0x9a), + "cached worker write dirties original shmem bytes"); + result(!bitmap(fd, length, &bits) && bits == 3, + "canonical bitmap returns exact two4K units touched by worker write"); + result(!unmap(fd, length), + "fine UNMAP synchronizes both recording domains"); + result(!state(fd, &cmd) && !cmd.locked_bytes && !cmd.mapped_bytes[0] && + !cmd.mapped_bytes[1], + "source-exit cleanup refunds pins and clears all translations"); + result(!domain(fd, 1, 0, 0, 0) && !domain(fd, 0, 0, 0, 0), + "both domains detach after complete cleanup"); + external_span(backing, native); + if (getauxval(AT_PAGESZ) != native) + native_owner(fd, backing, data, native); +out: + if (data != MAP_FAILED) + munmap(data, 2 * native); + if (backing >= 0) + close(backing); + if (fd >= 0) + close(fd); + printf("1..%u\n# type1 ioctl failures=%u\n", number, failures); + return !!failures; +} + +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 5 && !strcmp(argv[1], "--native-source")) { + size_t native = strtoul(argv[4], NULL, 10); + int fd = atoi(argv[2]), backing = atoi(argv[3]); + void *data; + + if (getauxval(AT_PAGESZ) != native) + return 2; + data = mmap(NULL, 2 * native, PROT_READ | PROT_WRITE, + MAP_SHARED, backing, 0); + return data == MAP_FAILED || map(fd, data, 2 * native) ? 1 : 0; + } + if (argc == 4 && !strcmp(argv[1], "--test") && getppid() == 1) { + size_t expected = strtoul(argv[3], NULL, 10); + + if (getauxval(AT_PAGESZ) != expected) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) + return 2; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("sysfs", "/sys", "sysfs", 0, NULL)) + return 3; + FILE *dev = fopen("/sys/class/misc/aurora-vfio-type1-test/dev", "r"); + unsigned int major, minor; + + if (!dev || fscanf(dev, "%u:%u", &major, &minor) != 2) + return 4; + fclose(dev); + if (mknod("/dev/vfio-test", S_IFCHR | 0600, makedev(major, minor))) + return 5; + size_t native = getauxval(AT_PAGESZ), + targets[] = { native, 4096, 16384 }; + int ok = 1; + + int compat_only = getenv("VFIO_COMPAT_ONLY") != NULL; + + for (unsigned int i = 0; i < 3; i++) { + if (compat_only) + break; + size_t target = targets[i]; + pid_t child; + int status = 0; + + if (target > native || (i && target == native)) + continue; + child = fork(); + if (!child) { + char native_arg[24], target_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(target_arg, sizeof(target_arg), "%zu", target); + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, + target == native ? 0UL : target, 0UL, 0UL, + 0UL)) + _exit(3); + execl("/init", "/init", "--test", native_arg, + target_arg, NULL); + _exit(4); + } + ok &= child > 0 && waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status); + printf("# actual ABI=%zuK child_status=%d\n", target / 1024, + status); + } + const char *expected_arg = getenv("VFIO_COMPAT_PAGES"); + size_t expected = expected_arg ? strtoul(expected_arg, NULL, 10) : 4096; + pid_t compat = fork(); + int compat_status = 0; + + if (!compat) { + char native_arg[24], page_arg[24]; + + snprintf(native_arg, sizeof(native_arg), "%zu", native); + snprintf(page_arg, sizeof(page_arg), "%zu", expected); + execl("/compat-vfio", "/compat-vfio", native_arg, page_arg, + NULL); + _exit(98); + } + ok &= compat > 0 && waitpid(compat, &compat_status, 0) == compat && + WIFEXITED(compat_status) && !WEXITSTATUS(compat_status); + printf("# AArch32 expected=%zuK child_status=%d\n", expected / 1024, + compat_status); + printf("VFIO TYPE1 IOCTL %s\n", ok ? "PASS" : "FAIL"); + reboot(RB_POWER_OFF); + return !ok; +} diff --git a/tools/testing/selftests/arm64/user4k/vhost-log-init.c b/tools/testing/selftests/arm64/user4k/vhost-log-init.c new file mode 100644 index 00000000000000..07d6a8969bca0e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/vhost-log-init.c @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SET_EXEC_PAGE_SIZE 0x41555001 +#ifndef VHOST_SET_FORK_FROM_OWNER +#define VHOST_SET_FORK_FROM_OWNER _IOW(VHOST_VIRTIO, 0x84, __u8) +#define VHOST_GET_FORK_FROM_OWNER _IOR(VHOST_VIRTIO, 0x85, __u8) +#endif +#define CHECK(c) \ + do { \ + if (!(c)) { \ + printf("FAIL line %d: %s errno=%d\n", __LINE__, #c, \ + errno); \ + return 1; \ + } \ + } while (0) +#define N 8 +#define ROUNDS 32 +#define PACKET_SIZE 96 +#define PROTOCOL 0x88b5 +#define DATA_GPA (51ULL * 4096) +#define TX_USED_GPA (71ULL * 4096) +#define RX_USED_GPA (70ULL * 4096) + +struct queue { + struct vring_desc desc[N]; + struct { + uint16_t flags, idx, ring[N + 1]; + } avail; + struct { + uint16_t flags, idx; + struct vring_used_elem ring[N]; + uint16_t event; + } used; + int kick, call; +}; + +static int queue_setup(int fd, struct queue *q, unsigned index) +{ + struct vhost_vring_state state = { .index = index, .num = N }; + struct vhost_vring_addr addr = { + .index = index, + .flags = 1U << VHOST_VRING_F_LOG, + .desc_user_addr = (uintptr_t)q->desc, + .avail_user_addr = (uintptr_t)&q->avail, + .used_user_addr = (uintptr_t)&q->used, + .log_guest_addr = index ? TX_USED_GPA : RX_USED_GPA, + }; + q->kick = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + q->call = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + CHECK(q->kick >= 0 && q->call >= 0); + CHECK(!ioctl(fd, VHOST_SET_VRING_NUM, &state)); + state.num = 0; + CHECK(!ioctl(fd, VHOST_SET_VRING_BASE, &state)); + CHECK(!ioctl(fd, VHOST_SET_VRING_ADDR, &addr)); + struct vhost_vring_file file = { .index = index, .fd = q->kick }; + CHECK(!ioctl(fd, VHOST_SET_VRING_KICK, &file)); + file.fd = q->call; + CHECK(!ioctl(fd, VHOST_SET_VRING_CALL, &file)); + return 0; +} +static int submit(struct queue *q, unsigned round) +{ + uint64_t one = 1; + q->avail.ring[round % N] = 0; + __atomic_store_n(&q->avail.idx, round + 1, __ATOMIC_RELEASE); + return write(q->kick, &one, sizeof(one)) == sizeof(one); +} +static int complete(struct queue *q, unsigned round) +{ + for (unsigned i = 0; i < 2000; i++) { + if (__atomic_load_n(&q->used.idx, __ATOMIC_ACQUIRE) == + round + 1) + return 1; + usleep(1000); + } + printf("queue timeout round=%u used=%u avail=%u\n", round, q->used.idx, + q->avail.idx); + return 0; +} +static int bitmap_ok(unsigned char *leaf, size_t ps, int active, int missing) +{ + for (size_t i = 0; i < ps; i++) { + unsigned char expected = i < ps - 9 ? (missing ? 0 : 0x64) : 0; + if (active && i == ps - 3) + expected = 8; /* RX buffer, GPA page 51 */ + if (active && i == ps - 1) + expected = 0xc0; /* RX/TX rings, pages 70/71 */ + if (__atomic_load_n(leaf + i, __ATOMIC_ACQUIRE) != expected) { + printf("bitmap byte %zu got=%#x expected=%#x\n", i, + leaf[i], expected); + return 0; + } + } + return 1; +} +static int filled(unsigned char *p, size_t len, unsigned char byte) +{ + for (size_t i = 0; i < len; i++) + if (p[i] != byte) + return 0; + return 1; +} +static int exercise_case(size_t ps, unsigned shared, unsigned forked, + uint8_t mode) +{ + int backing = -1; + if (shared) { + backing = memfd_create("vhost-log", 0); + CHECK(backing >= 0 && !ftruncate(backing, 4 * ps)); + } + unsigned char *src = mmap( + (void *)0x20000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | + (shared ? MAP_SHARED : MAP_PRIVATE | MAP_ANONYMOUS), + backing, 0); + unsigned char *dst = + mmap((void *)0x30000000, 4 * ps, PROT_READ | PROT_WRITE, + MAP_FIXED_NOREPLACE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(src != MAP_FAILED && dst != MAP_FAILED); + for (unsigned i = 0; i < 4; i++) + memset(src + i * ps, 0x61 + i, ps); + memset(dst, 0x91, 4 * ps); + CHECK(mremap(src + 3 * ps, ps, ps, MREMAP_MAYMOVE | MREMAP_FIXED, + dst + ps) == dst + ps); + unsigned char *leaf = dst + ps, *bitmap = leaf + ps - 9; + memset(bitmap, 0, 9); + int barrier[2]; + pid_t child = -1; + if (forked) { + CHECK(!pipe(barrier)); + child = fork(); + CHECK(child >= 0); + if (!child) { + close(barrier[1]); + char token; + int ok = read(barrier[0], &token, 1) == 1 && + bitmap_ok(leaf, ps, shared, 0); + _exit(ok ? 0 : 8); + } + close(barrier[0]); + } + int tap = open("/dev/net/tun", O_RDWR | O_CLOEXEC); + CHECK(tap >= 0); + struct ifreq ifr = { .ifr_flags = IFF_TAP | IFF_NO_PI }; + strcpy(ifr.ifr_name, "abi%d"); + CHECK(!ioctl(tap, TUNSETIFF, &ifr)); + int raw = socket(AF_PACKET, SOCK_RAW | SOCK_CLOEXEC, htons(PROTOCOL)); + CHECK(raw >= 0); + CHECK(!ioctl(raw, SIOCGIFINDEX, &ifr)); + int ifindex = ifr.ifr_ifindex; + CHECK(!ioctl(raw, SIOCGIFFLAGS, &ifr)); + ifr.ifr_flags |= IFF_UP; + CHECK(!ioctl(raw, SIOCSIFFLAGS, &ifr)); + int fd = open("/dev/vhost-net", O_RDWR | O_CLOEXEC); + CHECK(fd >= 0); + CHECK(!ioctl(fd, VHOST_SET_FORK_FROM_OWNER, &mode)); + uint8_t gotmode = 9; + CHECK(!ioctl(fd, VHOST_GET_FORK_FROM_OWNER, &gotmode) && + gotmode == mode); + CHECK(!ioctl(fd, VHOST_SET_OWNER, NULL)); + uint64_t log_base = (uintptr_t)bitmap; + CHECK(!ioctl(fd, VHOST_SET_LOG_BASE, &log_base)); + unsigned char *data = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(data != MAP_FAILED); + struct { + struct vhost_memory mem; + struct vhost_memory_region region; + } mem = { + .mem.nregions = 1, + .region = { .guest_phys_addr = DATA_GPA, + .memory_size = 4096, + .userspace_addr = (uintptr_t)data }, + }; + CHECK(!ioctl(fd, VHOST_SET_MEM_TABLE, &mem)); + uint64_t features = (1ULL << VHOST_F_LOG_ALL) | + (1ULL << VHOST_NET_F_VIRTIO_NET_HDR); + uint64_t available; + CHECK(!ioctl(fd, VHOST_GET_FEATURES, &available) && + (available & features) == features); + CHECK(!ioctl(fd, VHOST_SET_FEATURES, &features)); + struct queue *q = mmap(NULL, ps, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHECK(q != MAP_FAILED && sizeof(*q) * 2 < ps); + CHECK(!queue_setup(fd, q, 0) && !queue_setup(fd, q + 1, 1)); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, .fd = tap }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + struct sockaddr_ll dest = { .sll_family = AF_PACKET, + .sll_protocol = htons(PROTOCOL), + .sll_ifindex = ifindex, + .sll_halen = 6 }; + unsigned char packet[PACKET_SIZE]; + memset(packet, 0x5c, sizeof(packet)); + memset(packet, 0xff, 6); + packet[6] = 2; + packet[12] = PROTOCOL >> 8; + packet[13] = PROTOCOL & 0xff; + q[0].desc[0] = (struct vring_desc){ .addr = DATA_GPA, + .len = 512, + .flags = VRING_DESC_F_WRITE }; + q[1].desc[0] = + (struct vring_desc){ .addr = DATA_GPA + 1024, + .len = sizeof(struct virtio_net_hdr) + + PACKET_SIZE }; + memcpy(data + 1024 + sizeof(struct virtio_net_hdr), packet, + PACKET_SIZE); + for (unsigned phase = 0; phase < 2; phase++) { + if (phase) { + memset(bitmap, 0, 9); + CHECK(!madvise(leaf, ps, MADV_DONTNEED)); + int pagemap = open("/proc/self/pagemap", O_RDONLY); + uint64_t entry; + CHECK(pagemap >= 0 && + pread(pagemap, &entry, sizeof(entry), + (uintptr_t)leaf / ps * sizeof(entry)) == + sizeof(entry)); + CHECK(!(entry & (1ULL << 63))); + close(pagemap); + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { + .index = index, .fd = tap + }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, + &backend)); + } + } + for (unsigned round = phase * ROUNDS; + round < (phase + 1) * ROUNDS; round++) { + packet[20] = round; + memset(data, 0xaa, 512); + CHECK(submit(q, round)); + CHECK(sendto(raw, packet, sizeof(packet), 0, + (void *)&dest, + sizeof(dest)) == sizeof(packet)); + CHECK(submit(q + 1, round)); + CHECK(complete(q, round) && complete(q + 1, round)); + CHECK(q[0].used.ring[round % N].id == 0); + if (q[0].used.ring[round % N].len != + sizeof(struct virtio_net_hdr) + PACKET_SIZE) + printf("RX length=%u expected=%zu ethertype=%02x%02x\n", + q[0].used.ring[round % N].len, + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + data[sizeof(struct virtio_net_hdr) + 12], + data[sizeof(struct virtio_net_hdr) + 13]); + CHECK(q[0].used.ring[round % N].len == + sizeof(struct virtio_net_hdr) + PACKET_SIZE); + CHECK(!memcmp(data + sizeof(struct virtio_net_hdr), + packet, PACKET_SIZE)); + CHECK(filled(data + sizeof(struct virtio_net_hdr) + + PACKET_SIZE, + 512 - sizeof(struct virtio_net_hdr) - + PACKET_SIZE, + 0xaa)); + } + /* Stop and flush workers before checking the bitmap or discarding backing. */ + for (unsigned index = 0; index < 2; index++) { + struct vhost_vring_file backend = { .index = index, + .fd = -1 }; + CHECK(!ioctl(fd, VHOST_NET_SET_BACKEND, &backend)); + } + CHECK(bitmap_ok(leaf, ps, 1, phase && !shared)); + if (forked && !phase) { + CHECK(write(barrier[1], "x", 1) == 1); + close(barrier[1]); + int status; + CHECK(waitpid(child, &status, 0) == child && + WIFEXITED(status) && !WEXITSTATUS(status)); + } + } + close(fd); + close(raw); + close(tap); + for (unsigned i = 0; i < 2; i++) { + close(q[i].kick); + close(q[i].call); + } + CHECK(filled(dst, ps, 0x91) && filled(dst + 2 * ps, 2 * ps, 0x91)); + for (unsigned i = 0; i < 3; i++) + CHECK(filled(src + i * ps, ps, 0x61 + i)); + CHECK(!munmap(src, 3 * ps) && !munmap(dst, 4 * ps)); + CHECK(!munmap(data, ps) && !munmap(q, ps)); + if (backing >= 0) + close(backing); + printf("ok - %zuK vhost %s fork=%u worker=%u: %u RX/TX packets, relocated dirty bits, missing-PTE repair and neighbors\n", + ps / 1024, shared ? "shared" : "private", forked, mode, + 2 * ROUNDS); + return 0; +} +static int exercise(size_t expected) +{ + CHECK(getauxval(AT_PAGESZ) == expected); + alarm(45); + for (uint8_t mode = 0; mode < 2; mode++) + for (unsigned shared = 0; shared < 2; shared++) + for (unsigned forked = 0; forked < 2; forked++) + CHECK(!exercise_case(expected, shared, forked, + mode)); + return 0; +} +int main(int argc, char **argv) +{ + setvbuf(stdout, NULL, _IONBF, 0); + if (argc == 3 && !strcmp(argv[1], "--exercise")) { + if (getppid() != 1) + return 2; + return exercise(strtoul(argv[2], NULL, 10)); + } + if (getpid() != 1 || argc != 1) { + fprintf(stderr, "Disposable VM PID1 only\n"); + return 2; + } + int fail = 0; + if (mount("proc", "/proc", "proc", 0, NULL) || + mount("devtmpfs", "/dev", "devtmpfs", 0, NULL)) { + perror("mount"); + fail = 1; + goto done; + } + /* Keep unsolicited IPv6 DAD packets out of the isolated TAP queues. */ + int ipv6 = + open("/proc/sys/net/ipv6/conf/default/disable_ipv6", O_WRONLY); + if (ipv6 < 0 || write(ipv6, "1", 1) != 1) { + perror("disable IPv6"); + fail = 1; + goto done; + } + close(ipv6); + size_t sizes[] = { getauxval(AT_PAGESZ), 4096, 16384 }; + for (unsigned i = 0; i < (sizes[0] > 16384 ? 3U : 2U); i++) { + pid_t c = fork(); + int status = 0; + if (!c) { + char expected[24]; + snprintf(expected, sizeof(expected), "%zu", sizes[i]); + if (prctl(SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)) + _exit(4); + execl("/init", "/init", "--exercise", expected, NULL); + _exit(5); + } + int ok = 0; + for (unsigned ticks = 0; c > 0 && ticks < 600; ticks++) { + pid_t got = waitpid(c, &status, WNOHANG); + if (got == c) { + ok = WIFEXITED(status) && !WEXITSTATUS(status); + break; + } + if (ticks == 599) { + kill(c, SIGKILL); + waitpid(c, &status, 0); + break; + } + usleep(100000); + } + printf("%s - ABI %zuK vhost log status=%#x\n", + ok ? "ok" : "not ok", sizes[i] / 1024, status); + fail += !ok; + } +done: + printf("VHOST LOG %s\n", fail ? "FAIL" : "PASS"); + sync(); + reboot(RB_POWER_OFF); + for (;;) + pause(); +} diff --git a/tools/testing/selftests/arm64/user4k/wine-kernel32.def b/tools/testing/selftests/arm64/user4k/wine-kernel32.def new file mode 100644 index 00000000000000..891e48c1fb83bc --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-kernel32.def @@ -0,0 +1,20 @@ +LIBRARY KERNEL32.dll +EXPORTS +VirtualAlloc +VirtualFree +VirtualProtect +VirtualQuery +GetSystemInfo +GetStdHandle +WriteFile +ExitProcess +GetLastError +AddVectoredExceptionHandler +RemoveVectoredExceptionHandler +CreateThread +WaitForSingleObject +GetExitCodeThread +CloseHandle +CreateFileMappingA +MapViewOfFile +UnmapViewOfFile diff --git a/tools/testing/selftests/arm64/user4k/wine-page-contract.c b/tools/testing/selftests/arm64/user4k/wine-page-contract.c new file mode 100644 index 00000000000000..149f619de6651b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/wine-page-contract.c @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Freestanding Windows x64 PE; Win32 imports, no CRT or external SDK. */ +typedef unsigned long DWORD; +typedef int BOOL; +typedef unsigned long long SIZE_T; +typedef void *HANDLE; +#define API __declspec(dllimport) +API void *VirtualAlloc(void *, SIZE_T, DWORD, DWORD); +API BOOL VirtualFree(void *, SIZE_T, DWORD); +API BOOL VirtualProtect(void *, SIZE_T, DWORD, DWORD *); +API SIZE_T VirtualQuery(const void *, void *, SIZE_T); +API void GetSystemInfo(void *); +API HANDLE GetStdHandle(DWORD); +API BOOL WriteFile(HANDLE, const void *, DWORD, DWORD *, void *); +API void ExitProcess(DWORD); +API DWORD GetLastError(void); +API void *AddVectoredExceptionHandler(DWORD, void *); +API DWORD RemoveVectoredExceptionHandler(void *); +API HANDLE CreateThread(void *, SIZE_T, DWORD (*)(void *), void *, DWORD, + DWORD *); +API DWORD WaitForSingleObject(HANDLE, DWORD); +API BOOL GetExitCodeThread(HANDLE, DWORD *); +API BOOL CloseHandle(HANDLE); +API HANDLE CreateFileMappingA(HANDLE, void *, DWORD, DWORD, DWORD, + const char *); +API void *MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +API BOOL UnmapViewOfFile(const void *); +struct system_info { + DWORD arch, page; + void *min, *max; + SIZE_T mask; + DWORD cpus, type, granule; + unsigned short level, revision; +}; +struct memory_info { + void *base, *allocation; + DWORD allocation_protect, pad; + SIZE_T size; + DWORD state, protect, type, pad2; +}; +struct exception_record { + DWORD code, flags; + void *record, *address; + DWORD count, pad; + SIZE_T info[15]; +}; +struct exception_pointers { + struct exception_record *record; + void *context; +}; +/* Keep the freestanding declarations consistent with the Windows x64 ABI. */ +_Static_assert(sizeof(DWORD) == 4, "DWORD must be 32 bits"); +_Static_assert(sizeof(SIZE_T) == 8, "SIZE_T must be 64 bits"); +_Static_assert(sizeof(struct system_info) == 48, "SYSTEM_INFO layout"); +_Static_assert(sizeof(struct memory_info) == 48, + "MEMORY_BASIC_INFORMATION layout"); +_Static_assert(sizeof(struct exception_record) == 152, + "EXCEPTION_RECORD layout"); +static volatile unsigned char *pages; +static volatile DWORD handled; +static void say(const char *s) +{ + DWORD n = 0, w = 0; + while (s[n]) + n++; + WriteFile(GetStdHandle((DWORD)-11), s, n, &w, 0); +} +static void number(DWORD n) +{ + char buf[11], *p = buf + 10; + *p = 0; + do { + *--p = '0' + n % 10; + n /= 10; + } while (n); + say(p); +} +static void fail(DWORD stage) +{ + DWORD e = GetLastError(); + say("WINE PAGE CONTRACT FAIL stage="); + number(stage); + say(" error="); + number(e); + say("\r\n"); + ExitProcess(1); +} +#define CHECK(x, n) \ + do { \ + if (!(x)) \ + fail(n); \ + } while (0) +static long handler(struct exception_pointers *e) +{ + DWORD old; + if (e->record->code != 0xc0000005 || e->record->count < 2 || + e->record->info[0] != 1 || + e->record->info[1] != (SIZE_T)(pages + 4103)) + return 0; + if (!VirtualProtect((void *)(pages + 4096), 4096, 4, &old)) + return 0; + handled++; + return -1; +} +static DWORD thread(void *arg) +{ + volatile unsigned char *p = arg; + for (unsigned int i = 0; i < 4096; i++) + p[i] = (unsigned char)(i ^ 0x5a); + return 37; +} +void mainCRTStartup(void) +{ + struct system_info si; + struct memory_info mi; + DWORD old, status; + GetSystemInfo(&si); + CHECK(si.page == 4096 && si.granule == 65536, 1); + say("ok - Wine reports 4K pages and 64K allocation granularity\r\n"); + pages = VirtualAlloc(0, 65536, 0x2000, 1); + CHECK(pages, 2); + CHECK(VirtualAlloc((void *)pages, 12288, 0x1000, 4) == pages, 3); + for (unsigned int i = 0; i < 12288; i++) + pages[i] = (unsigned char)(i ^ (i >> 12)); + CHECK(VirtualProtect((void *)(pages + 4096), 4096, 2, &old) && old == 4, + 4); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.base == pages + 4096 && mi.size == 4096 && + mi.protect == 2, + 5); + CHECK(VirtualQuery((void *)(pages + 8192), &mi, sizeof(mi)) == + sizeof(mi) && + mi.protect == 4, + 6); + void *veh = AddVectoredExceptionHandler(1, handler); + CHECK(veh, 7); + pages[4103] = 0x6d; + CHECK(handled == 1 && pages[4103] == 0x6d, 8); + CHECK(RemoveVectoredExceptionHandler(veh), 9); + say("ok - Wine independent 4K protection and handled access violation\r\n"); + CHECK(VirtualFree((void *)(pages + 4096), 4096, 0x4000), 10); + CHECK(VirtualQuery((void *)(pages + 4096), &mi, sizeof(mi)) == + sizeof(mi) && + mi.state == 0x2000, + 11); + CHECK(VirtualAlloc((void *)(pages + 4096), 4096, 0x1000, 4) == + pages + 4096, + 12); + for (unsigned int i = 0; i < 4096; i++) + CHECK(!pages[4096 + i], 13); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[i] == (unsigned char)i && + pages[8192 + i] == (unsigned char)(i ^ 2), + 14); + say("ok - Wine 4K decommit and zero-filled recommit preserve neighbours\r\n"); + HANDLE t = CreateThread(0, 0, thread, (void *)(pages + 4096), 0, 0); + CHECK(t, 15); + CHECK(WaitForSingleObject(t, 30000) == 0 && + GetExitCodeThread(t, &status) && status == 37, + 16); + CHECK(CloseHandle(t), 17); + for (unsigned int i = 0; i < 4096; i++) + CHECK(pages[4096 + i] == (unsigned char)(i ^ 0x5a), 18); + say("ok - Wine threads share committed 4K pages\r\n"); + HANDLE file = CreateFileMappingA((HANDLE)(SIZE_T)-1, 0, 4, 0, 16384, 0); + CHECK(file, 19); + unsigned char *a = MapViewOfFile(file, 2, 0, 0, 16384), + *b = MapViewOfFile(file, 2, 0, 0, 16384); + CHECK(a && b, 20); + a[4099] = 0xa7; + CHECK(b[4099] == 0xa7, 21); + CHECK(VirtualProtect(a + 4096, 4096, 2, &old), 22); + b[4099] = 0x53; + CHECK(a[4099] == 0x53, 23); + CHECK(UnmapViewOfFile(a) && UnmapViewOfFile(b) && CloseHandle(file), + 24); + CHECK(VirtualFree((void *)pages, 0, 0x8000), 25); + say("ok - Wine shared section aliases and 4K protection\r\nWINE PAGE CONTRACT PASS\r\n"); + ExitProcess(0); +} diff --git a/tools/testing/selftests/arm64/user4k/x18-probe.S b/tools/testing/selftests/arm64/user4k/x18-probe.S new file mode 100644 index 00000000000000..565f34e8323f57 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/x18-probe.S @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include + + .text + .globl x18_syscall + .type x18_syscall, %function +/* (nr, arg0, arg1, arg2, observed_x18); no libc code inside the probe. */ +x18_syscall: + hint 34 /* bti c */ + stp x18, x19, [sp, #-16]! + mov x19, x4 + movz x18, #0x3210 + movk x18, #0x7654, lsl #16 + movk x18, #0xba98, lsl #32 + movk x18, #0xfedc, lsl #48 + mov x8, x0 + mov x0, x1 + mov x1, x2 + mov x2, x3 + svc #0 + str x18, [x19] + ldp x18, x19, [sp], #16 + ret + .size x18_syscall, .-x18_syscall + .section .note.GNU-stack,"",%progbits diff --git a/tools/testing/vma/include/dup.h b/tools/testing/vma/include/dup.h index 9e0dfd3a85b0e2..8d0377cda51f09 100644 --- a/tools/testing/vma/include/dup.h +++ b/tools/testing/vma/include/dup.h @@ -533,6 +533,9 @@ struct vm_area_desc { /* Mutable fields. Populated with initial state. */ pgoff_t pgoff; +#ifdef CONFIG_MM_SUBPAGE + unsigned int subpage_offset; +#endif struct file *vm_file; vma_flags_t vma_flags; pgprot_t page_prot; @@ -1300,6 +1303,9 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->end = vma->vm_end; desc->pgoff = vma->vm_pgoff; +#ifdef CONFIG_MM_SUBPAGE + desc->subpage_offset = vma->vm_subpage_offset; +#endif desc->vm_file = vma->vm_file; desc->vma_flags = vma->flags; desc->page_prot = vma->vm_page_prot; @@ -1308,6 +1314,117 @@ static inline void compat_set_desc_from_vma(struct vm_area_desc *desc, desc->action.type = MMAP_NOTHING; } +static inline unsigned int mm_page_shift(const struct mm_struct *mm) +{ + return PAGE_SHIFT; +} + +static inline unsigned long mm_page_size(const struct mm_struct *mm) +{ + return PAGE_SIZE; +} + +static inline unsigned long mm_page_mask(const struct mm_struct *mm) +{ + return PAGE_MASK; +} + +static inline unsigned long mm_page_align(const struct mm_struct *mm, unsigned long size) +{ + return PAGE_ALIGN(size); +} + +static inline unsigned long vma_user_pages(const struct vm_area_struct *vma) +{ + return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; +} + +#define mm_account_memory(mm, policy_mm, pages) security_vm_enough_memory_mm(policy_mm, pages) +#define mm_unacct_memory(mm, pages) vm_unacct_memory(pages) + +/* File-cache indices remain native-page units even for a 4K VMA. */ +struct vm_page_offset { + pgoff_t index; + unsigned int offset; +}; + +static inline struct vm_page_offset vm_page_offset_add(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = pos.offset + (bytes & ~PAGE_MASK); + + pos.index += (bytes >> PAGE_SHIFT) + (low >> PAGE_SHIFT); + pos.offset = low & ~PAGE_MASK; + return pos; +} + +static inline struct vm_page_offset vm_page_offset_sub(struct vm_page_offset pos, + unsigned long bytes) +{ + unsigned long low = bytes & ~PAGE_MASK; + + pos.index -= (bytes >> PAGE_SHIFT) + (low > pos.offset); + pos.offset = (pos.offset - low) & ~PAGE_MASK; + return pos; +} + +static inline bool vm_page_offset_equal(struct vm_page_offset a, struct vm_page_offset b) +{ + return a.index == b.index && a.offset == b.offset; +} + +static inline unsigned int vma_subpage_offset(const struct vm_area_struct *vma) +{ +#ifdef CONFIG_MM_SUBPAGE + return vma->vm_subpage_offset; +#else + return 0; +#endif +} + +/* address must be >= vm_start; vm_end computes the next contiguous offset. */ +static inline struct vm_page_offset vma_page_offset_at(const struct vm_area_struct *vma, + unsigned long address) +{ + unsigned long delta = address - vma->vm_start; + unsigned long low = (delta & ~PAGE_MASK) + vma_subpage_offset(vma); + + return (struct vm_page_offset) { + .index = vma->vm_pgoff + (delta >> PAGE_SHIFT) + (low >> PAGE_SHIFT), + .offset = low & ~PAGE_MASK, + }; +} + +static inline void vma_set_page_offset(struct vm_area_struct *vma, struct vm_page_offset pos) +{ + vma->vm_pgoff = pos.index; +#ifdef CONFIG_MM_SUBPAGE + vma->vm_subpage_offset = pos.offset; +#endif +} + +/* Inverse lookup of a single byte; an offset outside the VMA is not clamped. */ +static inline unsigned long vma_address_at_offset(const struct vm_area_struct *vma, + struct vm_page_offset pos) +{ + unsigned long len = vma->vm_end - vma->vm_start; + unsigned long index, delta, last; + unsigned int offset = vma_subpage_offset(vma); + + if (!len || pos.index < vma->vm_pgoff || pos.offset >= PAGE_SIZE) + return -EFAULT; + index = pos.index - vma->vm_pgoff; + last = ((len - 1) >> PAGE_SHIFT) + + ((((len - 1) & ~PAGE_MASK) + offset) >> PAGE_SHIFT); + if (index > last) + return -EFAULT; + delta = (index << PAGE_SHIFT) + pos.offset; + if (delta < offset || delta - offset >= len) + return -EFAULT; + return vma->vm_start + delta - offset; +} + + static inline unsigned long vma_pages(const struct vm_area_struct *vma) { return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; diff --git a/tools/testing/vma/tests/merge.c b/tools/testing/vma/tests/merge.c index 03b6f9820e0ac9..0b0f4ae413973f 100644 --- a/tools/testing/vma/tests/merge.c +++ b/tools/testing/vma/tests/merge.c @@ -227,7 +227,7 @@ static bool test_simple_shrink(void) ASSERT_FALSE(attach_vma(&mm, vma)); - ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, 0)); + ASSERT_FALSE(vma_shrink(&vmi, vma, 0, 0x1000, (struct vm_page_offset) { 0, 0 })); ASSERT_EQ(vma->vm_start, 0); ASSERT_EQ(vma->vm_end, 0x1000); diff --git a/tools/testing/vma/tests/mmap.c b/tools/testing/vma/tests/mmap.c index c85bc000d1cb7a..c87baa48ea26ef 100644 --- a/tools/testing/vma/tests/mmap.c +++ b/tools/testing/vma/tests/mmap.c @@ -12,19 +12,19 @@ static bool test_mmap_region_basic(void) current->mm = &mm; /* Map at 0x300000, length 0x3000. */ - addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, 0x300, NULL); + addr = __mmap_region(NULL, 0x300000, 0x3000, vma_flags, (struct vm_page_offset) { 0x300, 0 }, NULL); ASSERT_EQ(addr, 0x300000); /* Map at 0x250000, length 0x3000. */ - addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, 0x250, NULL); + addr = __mmap_region(NULL, 0x250000, 0x3000, vma_flags, (struct vm_page_offset) { 0x250, 0 }, NULL); ASSERT_EQ(addr, 0x250000); /* Map at 0x303000, merging to 0x300000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, 0x303, NULL); + addr = __mmap_region(NULL, 0x303000, 0x3000, vma_flags, (struct vm_page_offset) { 0x303, 0 }, NULL); ASSERT_EQ(addr, 0x303000); /* Map at 0x24d000, merging to 0x250000 of length 0x6000. */ - addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, 0x24d, NULL); + addr = __mmap_region(NULL, 0x24d000, 0x3000, vma_flags, (struct vm_page_offset) { 0x24d, 0 }, NULL); ASSERT_EQ(addr, 0x24d000); ASSERT_EQ(mm.map_count, 2); diff --git a/tools/testing/vma/tests/vma.c b/tools/testing/vma/tests/vma.c index 754a2da0632157..38a7cc6702f237 100644 --- a/tools/testing/vma/tests/vma.c +++ b/tools/testing/vma/tests/vma.c @@ -38,7 +38,7 @@ static bool test_copy_vma(void) /* Move backwards and do not merge. */ vma = alloc_and_link_vma(&mm, 0x3000, 0x5000, 3, vma_flags); - vma_new = copy_vma(&vma, 0, 0x2000, 0, &need_locks); + vma_new = copy_vma(&vma, 0, 0x2000, (struct vm_page_offset) { 0, 0 }, &need_locks); ASSERT_NE(vma_new, vma); ASSERT_EQ(vma_new->vm_start, 0); ASSERT_EQ(vma_new->vm_end, 0x2000); @@ -51,7 +51,7 @@ static bool test_copy_vma(void) vma = alloc_and_link_vma(&mm, 0, 0x2000, 0, vma_flags); vma_next = alloc_and_link_vma(&mm, 0x6000, 0x8000, 6, vma_flags); - vma_new = copy_vma(&vma, 0x4000, 0x2000, 4, &need_locks); + vma_new = copy_vma(&vma, 0x4000, 0x2000, (struct vm_page_offset) { 4, 0 }, &need_locks); vma_assert_attached(vma_new); ASSERT_EQ(vma_new, vma_next); From 1546ea62e1869d31631b0dd8a661157ff141fbff Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 19:56:23 -0400 Subject: [PATCH 025/631] docs: describe mixed userspace page sizes and the development launcher Document native backing versus per-process mappings, architecture responsibilities, supported combinations and remaining limitations. Provide a launcher for inherited and one-shot exec preferences. Signed-off-by: Diljit Singh (cherry picked from commit 6404f6a5b060c3b8730f7fcda19d4a72d59fab1e) --- Documentation/arch/arm64/index.rst | 1 + Documentation/arch/arm64/user4k.rst | 187 ++++++++++++++++++++++++++++ Documentation/mm/index.rst | 1 + Documentation/mm/user_page_size.rst | 84 +++++++++++++ tools/aurora-userpages/.gitignore | 1 + tools/aurora-userpages/Makefile | 14 +++ tools/aurora-userpages/README.rst | 34 +++++ tools/aurora-userpages/userpages.c | 83 ++++++++++++ 8 files changed, 405 insertions(+) create mode 100644 Documentation/arch/arm64/user4k.rst create mode 100644 Documentation/mm/user_page_size.rst create mode 100644 tools/aurora-userpages/.gitignore create mode 100644 tools/aurora-userpages/Makefile create mode 100644 tools/aurora-userpages/README.rst create mode 100644 tools/aurora-userpages/userpages.c diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index fc65c2d61a1de0..23afe611f30398 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -33,5 +33,6 @@ ARM64 Architecture sve tagged-address-abi tagged-pointers + user4k features diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst new file mode 100644 index 00000000000000..d483c565d853b1 --- /dev/null +++ b/Documentation/arch/arm64/user4k.rst @@ -0,0 +1,187 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +=========================================== +Experimental per-process page sizes on arm64 +=========================================== + +The mixed-page implementation lets an address space use a different user +translation granule from the kernel's native physical page size. The kernel +page size remains fixed for the boot. An address space's user granule remains +fixed for its lifetime; exec can create a new address space with another +supported granule. See :doc:`../../mm/user_page_size` for the generic launch +policy and architecture integration contract. + +This is an experimental ABI. The launch commands use private prctl numbers, +not assigned upstream UAPI. The implementation and tests do not establish +complete application, device or hardware qualification. + +Configuration and supported combinations +======================================== + +Enable ``CONFIG_ARM64_USER4K_EXPERIMENTAL`` with 48-bit VA and PA and a native +16K or 64K page size. The option defaults off and selects the generic +``MM_SUBPAGE`` backing and ``ARCH_HAS_USER_PAGE_SIZE`` exec policy support. +LPA2 and ``NO_PAGE_MAPCOUNT`` configurations are excluded by Kconfig. + +The exposed combinations are: + +=================== ================== ============================ +Native kernel size AArch64 default AArch64 alternatives +=================== ================== ============================ +4K 4K None in this implementation +16K 16K 4K, when supported by the CPUs +64K 64K 4K and 16K, subject to support +=================== ================== ============================ + +Capability detection uses the sanitized system-wide granule intersection. +AArch32 executables select 4K when that capability is available and retain +the native granule otherwise, regardless of launch preferences. The latter +case still enforces native mapping/protection alignment. AArch32 mmap2 file +offsets retain their 4K ABI unit in either case. + +An AArch32 executable running under an arm64 kernel is distinct from an +``arch/arm`` kernel. The latter retains its existing native ABI; this series +does not yet provide its alternative-granule backend. Exploiting extra +translation capabilities available only within an asymmetric AArch32 CPU +domain also remains unimplemented. + +Larger-than-native process granules are not exposed. Internal 64K-on-16K +fixtures exercise allocation, mappings and argument staging, but those +fixtures do not enable a public 64K process ABI on a 16K kernel. + +Selecting an executable's page size +=================================== + +Build the development launcher with:: + + make -C tools/aurora-userpages + +For example:: + + tools/aurora-userpages/userpages 4k COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --once native COMMAND [ARGUMENTS...] + tools/aurora-userpages/userpages --status + +The ordinary selection establishes an inherited per-thread exec preference. +``--once`` overrides it for one successful exec. A failed exec retains the +pending request; changing a preference never changes existing mappings. +Fork copies the preferences and mappings. Threads sharing an mm have the +same current granule but may establish different preferences for later exec. +``native`` clears an inherited alternative preference; a one-shot native +request can override that preference without clearing it. + +The final ELF loader resolves the granule before committing the new image. +If arguments were staged with another granule, their bytes are transferred +into a new private mm. Temporary argument mappings are bounded by both the +user leaf and native page size. Logical argument-size limits continue to +use the process granule. The interpreter and executable share one mm and +therefore one final granule. ELF segment alignment alone does not opt a +program into an alternative ABI. + +``AT_PAGESZ`` reports the process granule. User virtual address alignment, +protection boundaries and mapping lengths use that granule. Native physical +allocation, PFN arithmetic and file-cache indices continue to use the kernel +page size. Application execution does not require any of the KUnit options +or the ``arm64.user4k_test=1`` diagnostic boot parameter. + +Memory and table ownership +========================== + +Keep the following units separate: + +* ``PAGE_SIZE`` and physical PFNs describe native physical pages. +* ``mm_page_size(mm)`` describes the user mapping/protection unit. +* A folio owns one or more native pages and carries their physical charge. + +Smaller anonymous mappings retain explicit subpage ownership. A reference to +the containing folio alone does not retain an individual reusable slot. +The slot's byte offset must survive fault, COW, reverse mapping, migration, +swap and pinning operations. Newly exposed bytes are zeroed, and changing +one mapping's protection must not widen that change to adjacent slots. + +File offsets use a native cache index plus an explicit byte offset. VMA +split/merge, truncation and reverse mapping must preserve both components. +Physical allocation charges count the actual backing once, independently +of how many user mappings expose it. + +Lower page tables can share a native allocation. Table tokens preserve the +fragment address, while locking and lifetime use the native table owner. +Unissued cached fragments retain owner references. Issued fragments are not +reused within a live owner. Deferred reclamation must preserve the fragment +token through TLB invalidation and the required RCU grace period, including +allocation-failure fallback paths. + +Translation context and invalidation +=================================== + +The arm64 backend supplies per-mm page-table geometry. TTBR1 retains native +kernel tables, while TTBR0 uses the selected user geometry. A granule change +installs and synchronizes a reserved TTBR0 before changing TG0, then +synchronizes the new geometry before installing the user root. Software PAN +also selects the user TG0 even when restoration of TTBR0 is deferred until +uaccess or return to userspace. + +The existing mm ASID is retained. Selecting another granule does not add +an unconditional context-switch TLB flush. User invalidation operands and +range strides use the target mm's geometry; kernel and stage-2 operations +retain their own native geometry. Transition/idmap paths restore native +TG0 and then the active mm's user geometry when returning. + +The current configuration retains the existing 48-bit user address size. +Independent TG0/TG1 fields do not make all other translation controls +independent; the LPA2 exclusion must not be removed merely by changing TG0. + +Sharing, pinning and interface limits +==================================== + +Mixed-granule sharing is permitted. Shared mappings refer to one coherent +backing object, with permissions belonging to each virtual mapping. Shared +futex identity includes the object and byte offset. Private copying is not +an implementation of shared-memory semantics. + +Some interfaces have additional layout or ownership requirements. For +example, a perf ring's first mapper fixes its userspace granule; a mapper +using another granule is rejected. This does not prohibit mixed-granule +memfd or System V shared-memory aliases. + +A legacy page-array pin cannot express every smaller-granule byte range. +Converted consumers use typed fragment references carrying backing, offset, +length and ownership. Unconverted paths must not silently round a pin or DMA +mapping into neighboring slots. Mock IOMMU/VFIO tests validate the recorded +translations and ownership paths, not physical-device DMA on arbitrary +hardware. + +Alternative-granule THP/KSM and hugetlb support, remaining legacy pin +consumers, larger-than-native process activation and broader device +integration still need work. Userfaultfd feature bits and per-range ioctl +masks must be consulted; success for a shmem operation does not imply +support for the corresponding hugetlb operation. Sparse individual fills +can retain an entire native folio; packing behavior is not a throughput +or memory-efficiency guarantee for every workload. + +Test coverage +============= + +The arm64 selftests are in ``tools/testing/selftests/arm64/user4k``. The +``page-contract`` executable checks the exposed userspace contract. Extended +fixtures cover exec policy, AArch32, shared mappings, accounting, reclaim, +I/O and device-facing interfaces. Several are disposable-guest PID1 programs +with companion executables or storage requirements; they are not ordinary +host test commands. + +``CONFIG_ARM64_USER4K_KUNIT_TEST`` and the data/TLB suites exercise internal +geometry and ownership. Some fixtures require ``arm64.user4k_test=1`` and +some hardware translation tests are disabled under KASAN. Check individual +KTAP results and skips rather than treating boot success as a suite pass. +Private IOMMUFD/VFIO test-device options are separate and should remain off +in application kernels. + +The NUMA COW batching case runs only at boot, before userspace can change +its policy prerequisite. Its results remain visible in KUnit debugfs, but +it has no runtime rerun control. Other suites retain their rerun controls. + +Validation must distinguish native/default operation, each exposed +alternative, and AArch32 fallback with 4K support unavailable. Native ARM32 +short-descriptor and LPAE controls check that generic changes preserve the +existing ABI; they do not validate a mixed-page ARM32 backend. Internal +larger-leaf tests likewise do not qualify a public process mode. diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index 7aa2a888690832..fb6128d42aa843 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -13,6 +13,7 @@ see the :doc:`admin guide <../admin-guide/mm/index>`. physical_memory page_tables process_addrs + user_page_size bootmem page_allocation vmalloc diff --git a/Documentation/mm/user_page_size.rst b/Documentation/mm/user_page_size.rst new file mode 100644 index 00000000000000..62ef6ae39e0aca --- /dev/null +++ b/Documentation/mm/user_page_size.rst @@ -0,0 +1,84 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +============================== +Per-process user mapping sizes +============================== + +The experimental mixed-page implementation separates the user mapping +size of an address space from the kernel's native allocation size. +``PAGE_SIZE`` still describes native physical pages. ``mm_page_size(mm)`` +describes the minimum user mapping and protection unit of that address +space. A process retains one mapping size for the lifetime of its ``mm``. +Exec may create a new ``mm`` with a different size; changing a launch +preference never changes an existing mapping. + +The generic interfaces described here are internal development interfaces. +The experimental launch commands still use private prctl numbers; these +are not assigned upstream UAPI. The arm64 backend's supported modes and +current restrictions are documented in +:doc:`../arch/arm64/user4k`. + +Exec policy and inheritance +=========================== + +``ARCH_HAS_USER_PAGE_SIZE`` enables two per-thread policy fields in +``task_struct``: + +``exec_page_shift`` + A one-shot request for the next exec. Zero means no one-shot request. + +``default_exec_page_shift`` + An inherited preference used when no one-shot request overrides it. + Zero means the architecture's ordinary executable policy. + +Fork and thread creation copy both fields. Subsequent changes affect only +the calling thread, including when it shares an ``mm`` with other threads. +An exec by a non-leader uses the execing thread's policy. The fields do not +belong in ``mm_struct``: threads sharing one address space may launch new +images with different preferences. + +Generic prctl handling validates reserved arguments before calling the +architecture validator. A rejected request leaves both fields unchanged. +Setting a native one-shot request while an inherited alternative is active +stores ``PAGE_SHIFT`` explicitly, so the next exec can override that +alternative. Clearing the inherited preference does not consume a pending +one-shot request. + +Exec stages arguments in a private ``mm``. Its initial allocation receives +the one-shot shift, if any; the final executable loader resolves the +inherited preference and any executable-specific policy. An ELF interpreter +must be loaded using the same final mapping size as its executable. +Failure before committing the new image retains both preferences. +``setup_new_exec()`` consumes the one-shot field after the new image is +committed, while retaining the inherited preference. + +Architecture responsibilities +============================= + +An architecture selecting ``ARCH_HAS_USER_PAGE_SIZE`` must implement: + +* ``arch_exec_page_size_shift(size)``: return zero for a native request, a + validated alternative shift, or a negative errno. Validation includes + the supported hardware and kernel configuration, not just whether the + requested byte size is a power of two. +* ``arch_mm_init_exec(mm, shift)``: initialize the private address space's + geometry before allocating its page-table root. Zero denotes native + geometry. The hook receives the request explicitly and does not read + a separate architecture thread-policy field. +* Binary-loader integration that resolves the final executable policy + before exposing the address space. ``bprm_set_page_shift()`` transfers + staged argument bytes when the final choice changes the initial granule. +* Page-table allocation, walking, invalidation, context switching and MM + operations for every size the architecture exposes. + +The architecture retains control over executable compatibility policy. +For example, the arm64 AArch32 loader prefers 4K where the backend supports +it and retains the native ABI when that capability is absent. This is +separate from running an AArch32 kernel: ``arch/arm`` needs its own backend +integration before it can advertise alternative mapping sizes. The generic +exec policy alone does not provide that backend or expose new sizes there. + +Architectures without this capability retain native mappings and reject +the experimental launch-policy commands. ``ARCH_HAS_MM_PGTABLE`` describes +per-mm page-table accessors; it does not by itself promise an executable +userspace ABI. diff --git a/tools/aurora-userpages/.gitignore b/tools/aurora-userpages/.gitignore new file mode 100644 index 00000000000000..a27b2bf8b140da --- /dev/null +++ b/tools/aurora-userpages/.gitignore @@ -0,0 +1 @@ +/userpages diff --git a/tools/aurora-userpages/Makefile b/tools/aurora-userpages/Makefile new file mode 100644 index 00000000000000..c6d42e2d15efa0 --- /dev/null +++ b/tools/aurora-userpages/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +CC ?= cc +CFLAGS ?= -O2 -g +CFLAGS += -Wall -Wextra +LDFLAGS += -Wl,-z,max-page-size=65536 + +.PHONY: all clean +all: userpages + +userpages: userpages.c ../../include/uapi/linux/prctl.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $< $(LDLIBS) -o $@ + +clean: + rm -f userpages diff --git a/tools/aurora-userpages/README.rst b/tools/aurora-userpages/README.rst new file mode 100644 index 00000000000000..64f4fcb0dd12ac --- /dev/null +++ b/tools/aurora-userpages/README.rst @@ -0,0 +1,34 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Per-process userspace page-size launcher +======================================= + +Build with ``make`` in this directory. The running arm64 kernel must provide +Aurora's experimental per-process granule API. + +Examples:: + + userpages 4k FEXInterpreter ./x86-program + userpages 4k python3 script.py + userpages native program + userpages --status + +The choice becomes the calling thread's inherited exec preference: it applies +to the launched program and subsequent execs in its descendants. Fork retains +the address space and both exec preferences. Other threads and processes are +unchanged. ``native`` clears the inherited alternative preference. Ordinary +processes that never opt in continue to use the kernel's native default. + +``--once`` changes only the next successful exec; an existing inherited +preference remains available for later execs. For example, +``userpages --once native program`` runs one program with native pages and +retains any inherited 4K preference for its subsequent execs. Failed exec does +not consume the pending request. No selection changes a live address space. +AArch32 programs always use 4K when supported, otherwise native pages, +regardless of either preference. + +The kernel checks supported sizes and CPU capabilities. The current branch +supports 4K userspace on 16K/64K kernels and 16K userspace on 64K kernels. +64K userspace on a 16K kernel is not enabled. This launcher uses the private +PR_AURORA_{SET,GET}_{EXEC,DEFAULT}_PAGE_SIZE API in this tree; +it is not an upstream Linux ABI. diff --git a/tools/aurora-userpages/userpages.c b/tools/aurora-userpages/userpages.c new file mode 100644 index 00000000000000..02d9e4357224de --- /dev/null +++ b/tools/aurora-userpages/userpages.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include "../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include + +static int usage(const char *name, int status) +{ + fprintf(status ? stderr : stdout, + "Usage: %s [--once] {native|4k|16k|64k} COMMAND [ARG ...]\n" + " %s --status\n" + "Select the userspace page size for this program and its descendants.\n", + name, name); + return status; +} + +int main(int argc, char **argv) +{ + unsigned long size; + int once = argc > 1 && !strcmp(argv[1], "--once"); + int first = once ? 2 : 1; + + if (argc == 2 && !strcmp(argv[1], "--help")) + return usage(argv[0], 0); + if (argc == 2 && !strcmp(argv[1], "--status")) { + int pending = prctl(PR_AURORA_GET_EXEC_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + int error = errno; + + printf("Current process page size: %lu bytes\n", getauxval(AT_PAGESZ)); + if (pending < 0) { + fprintf(stderr, "Page-size selection unavailable: %s\n", strerror(error)); + return 1; + } + int inherited = prctl(PR_AURORA_GET_DEFAULT_PAGE_SIZE, 0UL, 0UL, 0UL, 0UL); + + if (inherited < 0) { + perror("Cannot read inherited page-size preference"); + return 1; + } + if (inherited) + printf("Inherited exec preference: %d bytes\n", inherited); + else + puts("Inherited exec preference: native default"); + if (pending) + printf("Next exec page size: %d bytes\n", pending); + else + puts("Next exec page size: inherited preference"); + return 0; + } + if (argc < first + 2) + return usage(argv[0], 2); + if (!strcmp(argv[first], "native")) + size = 0; + else if (!strcmp(argv[first], "4k")) + size = 4096; + else if (!strcmp(argv[first], "16k")) + size = 16384; + else if (!strcmp(argv[first], "64k")) + size = 65536; + else + return usage(argv[0], 2); + + if (!once && prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot set inherited %s pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + fprintf(stderr, "Cannot select %s userspace pages: %s\n", + argv[first], strerror(errno)); + return 1; + } + execvp(argv[first + 1], &argv[first + 1]); + int error = errno; + + fprintf(stderr, "Cannot execute %s: %s\n", argv[first + 1], strerror(error)); + return error == ENOENT ? 127 : 126; +} From 332180f9306f9f17ce5ff29943413a646342ee55 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:49:35 +1100 Subject: [PATCH 026/631] mm: defer linear subpage anon-vma destruction from atomic context Linear subpage owners retain an anon-vma root even without an rmap sidecar. Their last slot reference can be released by skb completion in softirq after the originating mm has exited. Route both kinds of root through the existing unbound cleanup work so the final anon-vma put and its sleeping synchronization run in process context. Preserve the physical unbinding and folio reference ordering. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 19594eb23573bb002d8f62087ecc1bd3cb38eab7) --- mm/user-subpage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 133d75241485a1..319d792ab5e9a1 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -328,8 +328,8 @@ static void owner_free(struct mm_subpage_owner *owner, bool put_folio) /* Last slot reference, or wholly unissued supply. */ folio_put(owner->folio); } - if (owner->rmap) { - /* Historical roots can have no live VMA; their final put may sleep. */ + if (owner->anon_root || owner->rmap) { + /* Any final anon-vma put can sleep, including a linear owner's. */ INIT_WORK(&owner->free_work, owner_free_work); queue_work(system_unbound_wq, &owner->free_work); } else { From 4ba6bdb42e69a2d9a0dce550e03e280ba8755aad Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:47:26 -0400 Subject: [PATCH 027/631] rust: drm: sched: Fix job accounting when cancelling jobs drm_sched_fini() passes each job left on the pending list to cancel_job and frees it right after. Such a job is in one of three states: its hardware fence callback is still armed, the callback has already completed the job, or drm_sched_stop() detached the job from its hardware fence, returning its credits but not its score. cancel_job_cb() treated every job as unfinished and signalled its finished fence with -ECANCELED. For a completed job, that sets an error on a signalled fence. For a job whose callback is still armed, the callback stays armed on a job that is about to be freed, and the job's credits and score are never returned. For a detached job, the score is never returned. drm_sched_fini() stops the scheduler workers, but not the hardware fence, so remove the callback first. Fence callbacks run under the fence lock, so dma_fence_remove_callback() either disarms the callback or waits for it to complete. Then, unless the callback completed the job, return the job's score and any credits drm_sched_stop() has not already returned, as drm_sched_job_done() would have, and signal the finished fence with -ECANCELED. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 12f0a07a61666305a1efe1a1f4c4c88a013abafb) --- rust/kernel/drm/sched.rs | 42 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 7275fd254e7e8e..d0353e0d9fa58d 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -115,13 +115,49 @@ unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + // SAFETY: A job on the pending list has a valid scheduler fence. + let s_fence = unsafe { (*sched_job).s_fence }; + // SAFETY: `s_fence` is valid and holds a reference to its parent, if any. + let parent = unsafe { (*s_fence).parent }; + // SAFETY: `s_fence` is valid. + let finished = unsafe { addr_of_mut!((*s_fence).finished) }; + + // drm_sched_fini() has stopped the scheduler workers, but the hardware fence can still + // signal concurrently. Fence callbacks run under the fence lock, so removing ours either + // disarms it or waits until it has completed the job. + let callback_removed = !parent.is_null() + // SAFETY: run_job() armed `cb` on `parent` before the submission worker was stopped. + && unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) }; + + // A job without a parent either completed when run_job() returned, or had its parent + // detached by drm_sched_stop(), which also returned its credits. + let complete_here = callback_removed + // SAFETY: `finished` is a valid, initialized fence. + || (parent.is_null() && unsafe { bindings::dma_fence_get_status(finished) } == 0); // SAFETY: All of our jobs are Job. T::cancel(unsafe { &mut *p }); - let fence = unsafe { Fence::get_raw(&mut (*(*sched_job).s_fence).finished) }; - fence.set_error(ECANCELED); - let _ = fence.signal(); + if complete_here { + // Account for the job exactly as drm_sched_job_done() would have. + // + // SAFETY: The job belongs to the scheduler being torn down, which is still valid. + unsafe { + let sched = (*s_fence).sched; + if callback_removed { + bindings::atomic_sub( + (*sched_job).credits as i32, + addr_of_mut!((*sched).credit_count), + ); + } + bindings::atomic_dec((*sched).score); + } + + // SAFETY: `finished` is a valid fence. + let fence = unsafe { Fence::get_raw(finished) }; + fence.set_error(ECANCELED); + fence.signal(); + } } /// A DRM scheduler job. From 19d3f7ef236dbbf6a2b257fd640652c2872ee351 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:48:27 -0400 Subject: [PATCH 028/631] rust: drm: sched: Keep the scheduler alive while jobs reference it drm_sched_entity_destroy() does not necessarily free the jobs still queued on the entity: drm_sched_entity_kill() hands each of them to a work item on the system workqueue, possibly only once the job's last dependency has signalled, and that work calls job->sched->ops->free_job(). The scheduler is embedded in SchedulerInner, which is freed as soon as the last Scheduler or Entity handle is dropped, so dropping a scheduler right after its last entity can leave that work dereferencing freed memory. Let every job hold a reference to the scheduler allocation. Finalizing the scheduler must not wait for those references, since drm_sched_fini() is what cancels and frees the pending jobs, so move it to a separate SchedulerOwner that only the Scheduler and Entity handles reference. The owner is allocated before drm_sched_init(), so that nothing can fail once the scheduler is initialized. The scheduler is now only reachable through shared references while C code keeps mutating it, so store it in an Opaque. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f8102be460b66b12af865d6250e73ffc6f99c73a) --- rust/kernel/drm/sched.rs | 66 ++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index d0353e0d9fa58d..e47aba505a32ba 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -11,6 +11,7 @@ use crate::{ prelude::*, sync::{Arc, UniqueArc}, time::{self, msecs_to_jiffies}, + types::Opaque, }; use core::marker::PhantomData; use core::mem::MaybeUninit; @@ -164,6 +165,10 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch pub struct Job { job: bindings::drm_sched_job, inner: T, + // Destroying an entity can leave its queued jobs to deferred work that still dereferences + // `job->sched` after the scheduler has been finalized. Keep the scheduler allocation alive + // for as long as any job references it; this does not delay finalization. + _scheduler: Arc>, } impl Deref for Job { @@ -277,14 +282,15 @@ impl<'a> JobFences<'a> { struct EntityInner { entity: bindings::drm_sched_entity, // TODO: Allow users to share guilty flag between entities - sched: Arc>, + sched: Arc>, guilty: bindings::atomic_t, _p: PhantomData, } impl Drop for EntityInner { fn drop(&mut self) { - // SAFETY: The EntityInner is initialized. This will cancel/free all jobs. + // SAFETY: The EntityInner is initialized. This will cancel/free all jobs, possibly from + // deferred work. Each job keeps the scheduler allocation alive until it is freed. unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; } } @@ -303,7 +309,7 @@ impl Entity { let mut entity: KBox>> = KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - let mut sched_ptr = &sched.0.sched as *const _ as *mut _; + let mut sched_ptr = sched.0.allocation.sched.get(); // SAFETY: The Box is allocated above and valid. unsafe { @@ -381,33 +387,48 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + // SAFETY: The Box pointer is valid, and this initializes the scheduler reference. + unsafe { + addr_of_mut!((*job.as_mut_ptr())._scheduler).write(self.0.sched.allocation.clone()) + }; + // SAFETY: All fields of the Job are now initialized. Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) } } -/// DRM scheduler inner data +/// DRM scheduler allocation, kept alive by the scheduler owner and by every job. pub struct SchedulerInner { - sched: bindings::drm_gpu_scheduler, + sched: Opaque, _p: PhantomData, } -impl Drop for SchedulerInner { +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + +/// Owner of an initialized DRM scheduler, shared by the scheduler handle and its entities. +/// +/// Jobs only reference the allocation, so that pending jobs cannot keep the scheduler from +/// being finalized. +struct SchedulerOwner { + allocation: Arc>, +} + +impl Drop for SchedulerOwner { fn drop(&mut self) { + let sched = self.allocation.sched.get(); + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of // freeing all in-progress jobs. - unsafe { bindings::drm_sched_stop(&mut self.sched, core::ptr::null_mut()) }; - unsafe { bindings::drm_sched_fini(&mut self.sched) }; + unsafe { bindings::drm_sched_stop(sched, core::ptr::null_mut()) }; + unsafe { bindings::drm_sched_fini(sched) }; } } -// SAFETY: TODO -unsafe impl Sync for SchedulerInner {} -// SAFETY: TODO -unsafe impl Send for SchedulerInner {} - /// A DRM Scheduler -pub struct Scheduler(Arc>); +pub struct Scheduler(Arc>); impl Scheduler { const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { @@ -431,11 +452,16 @@ impl Scheduler { ) -> Result> { let mut sched: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; + // Allocate the owner up front: nothing may fail once the scheduler is initialized. + let owner: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; + + // SAFETY: `sched` was just allocated and is valid for writes. + let sched_ptr = Opaque::cast_into(unsafe { addr_of!((*sched.as_mut_ptr()).sched) }); // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation // TODO: allocate sched zzeroed instead unsafe { - (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); + (*sched_ptr).sched_rq = core::ptr::null_mut(); }; let init_ops = bindings::drm_sched_init_args { @@ -453,14 +479,10 @@ impl Scheduler { // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. // `device` is valid by its type invarants - to_result(unsafe { - bindings::drm_sched_init( - addr_of_mut!((*sched.as_mut_ptr()).sched), - addr_of!(init_ops), - ) - })?; + to_result(unsafe { bindings::drm_sched_init(sched_ptr, addr_of!(init_ops)) })?; // SAFETY: All fields of SchedulerInner are now initialized. - Ok(Scheduler(unsafe { sched.assume_init() }.into())) + let allocation = unsafe { sched.assume_init() }.into(); + Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) } } From eb5ad7cfb22fd4d187cb6ac6d697982dd959ebe9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:49:13 -0400 Subject: [PATCH 029/631] rust: drm: sched: Do not stop the scheduler before tearing it down drm_sched_stop() belongs to the reset recovery protocol, and its documentation says not to call it before drm_sched_fini(). At teardown it detaches every in-flight job from its hardware fence and frees the jobs that have already completed. That is redundant: drm_sched_fini() stops the scheduler workers itself and hands every remaining job to cancel_job, which now handles jobs with an armed, completed or detached hardware fence callback. Call drm_sched_fini() alone, so that cancel_job sees each job's actual state and accounts for it exactly once. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f1f985d576810c7183ea9bcd1b701cb284927ac6) --- rust/kernel/drm/sched.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index e47aba505a32ba..c6c8af810b86e5 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -418,12 +418,9 @@ struct SchedulerOwner { impl Drop for SchedulerOwner { fn drop(&mut self) { - let sched = self.allocation.sched.get(); - - // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of - // freeing all in-progress jobs. - unsafe { bindings::drm_sched_stop(sched, core::ptr::null_mut()) }; - unsafe { bindings::drm_sched_fini(sched) }; + // SAFETY: The scheduler is valid. drm_sched_fini() stops the scheduler workers, then + // cancels and frees all remaining jobs through `cancel_job_cb()`. + unsafe { bindings::drm_sched_fini(self.allocation.sched.get()) }; } } From 9387c3794b48b83f0639c99389ac61f7eabb4d2f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:22:17 -0400 Subject: [PATCH 030/631] rust: drm: sched: let drivers handle timeouts without a reset Drivers that recover individual jobs without the scheduler reset protocol need to keep unfinished jobs pending while retaining their original fences. Expose DRM_GPU_SCHED_STAT_NO_HANG as Status::NoHang: the scheduler reinserts the job, rearms its timeout, and frees it when completion is observed. Add Job::is_finished() to detect completion between timeout expiry and its handler. The finished fence is initialized only when armed, so an unarmed job reports false. Let a driver also reconsider a timeout before entering recovery, for work whose execution window has just been renewed. The false_timeout callback defaults to false and must not stop or restart the scheduler. Returning true follows the same NoHang reinsertion path. Existing drivers retain their timeout behavior unless they opt into the new callback or return Status::NoHang. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 14d7b3c379da4c764581d4c34aff495d7346af72) --- rust/kernel/drm/sched.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index c6c8af810b86e5..a6d197152e68fc 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -45,6 +45,13 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// Reconsider a timeout before the backend starts recovery. Returning true + /// reinserts this job and rearms its timeout without changing its fences. + /// The callback must not stop or restart the scheduler. + fn false_timeout(_job: &mut Job) -> bool { + false + } + /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. fn prepare(_job: &mut Job) -> Option { @@ -58,6 +65,9 @@ pub trait JobImpl: Sized { /// Called when a job has taken too long to execute, to trigger GPU recovery. /// + /// A driver that handles timeouts without a scheduler reset returns [`Status::NoHang`]. + /// [`Job::is_finished`] tells whether the job completed after the timeout fired. + /// /// This method is called in a workqueue context. fn timed_out(job: &mut Job) -> Status; @@ -100,7 +110,12 @@ unsafe extern "C" fn timedout_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. + // SAFETY: The timeout worker retains this detached Job until this + // callback returns, including when its parent fence signals concurrently. + if T::false_timeout(unsafe { &mut *p }) { + return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; + } + // SAFETY: The same retained job remains valid after the first callback. T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat } From dbcdae3022293195b0570d287ab52c0214e5f2ca Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:05 -0400 Subject: [PATCH 031/631] dt-bindings: gpu: apple,agx: Add apple,agx-t8140 The T8140 (A18 Pro) GPU is run by two firmware co-processors, GFX and GFX1, each with its own control registers and mailbox. Its node therefore has three register ranges (asc, asc1, sgx) and two mailboxes. The firmware shares the GPU MMU translation tables, page tables and handoff structure through the same reserved memory regions as on earlier GPUs, but needs neither calibration blobs nor a global variable region, and it does not use apple,firmware-abi. The GPU power domain and the performance states are required. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c49ab96deb3c378015d1db27bf2122c837fd3bc0) --- .../devicetree/bindings/gpu/apple,agx.yaml | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index 59989d8bd1cbc5..cc9c78a7624f8d 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -17,6 +17,7 @@ properties: - apple,agx-g13s - apple,agx-g14g - apple,agx-g14s + - apple,agx-t8140 - items: - enum: - apple,agx-g13c @@ -29,22 +30,22 @@ properties: - const: apple,agx-g14s reg: - items: - - description: GPU coprocessor control registers - - description: GPU block MMIO registers + minItems: 2 + maxItems: 3 reg-names: - items: - - const: asc - - const: sgx + minItems: 2 + maxItems: 3 power-domains: maxItems: 1 mboxes: - maxItems: 1 + minItems: 1 + maxItems: 2 memory-region: + minItems: 3 items: - description: Region containing GPU MMU TTBs - description: Region containing GPU MMU page tables @@ -56,6 +57,7 @@ properties: - description: Shared global variables with GPU firmware memory-region-names: + minItems: 3 items: - const: ttbs - const: pagetables @@ -64,6 +66,8 @@ properties: - const: hw-cal-b - const: globals + operating-points-v2: true + apple,firmware-abi: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -77,7 +81,54 @@ required: - reg - mboxes - memory-region - - apple,firmware-abi + +allOf: + - if: + properties: + compatible: + contains: + const: apple,agx-t8140 + then: + properties: + reg: + items: + - description: GFX coprocessor control registers + - description: GFX1 coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: asc1 + - const: sgx + mboxes: + items: + - description: GFX coprocessor mailbox + - description: GFX1 coprocessor mailbox + memory-region: + maxItems: 3 + memory-region-names: + maxItems: 3 + required: + - power-domains + - operating-points-v2 + else: + properties: + reg: + items: + - description: GPU coprocessor control registers + - description: GPU block MMIO registers + reg-names: + items: + - const: asc + - const: sgx + mboxes: + maxItems: 1 + memory-region: + minItems: 6 + memory-region-names: + minItems: 6 + required: + - apple,firmware-abi additionalProperties: false From 50727ac991bf1e04b70cbebd2280afc33596153f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:05 -0400 Subject: [PATCH 032/631] dt-bindings: mailbox: apple,mailbox: Add the T8140 GPU mailboxes The GPU co-processors of the T8140 (A18 Pro) use the v4 ASC mailbox layout. Unlike the other T8140 ASC mailboxes, described by apple,t8140-asc-mailbox, they do not need the application processor to enable their receive FIFO. Add apple,t8140-gfx-asc-mailbox for them, with apple,asc-mailbox-v4 as the fallback. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 4c5aad430f6283d1d6b3b961a87d1629b397f22f) --- Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 4d704442a923af..3c4ee0dfd0c05b 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -44,6 +44,11 @@ properties: - enum: - apple,t6021-ane-mailbox - apple,t8112-ane-mailbox + ASC mailboxes of the T8140 GPU coprocessors. Unlike the other + T8140 ASC mailboxes, they do not need the application processor + to enable their receive FIFO. + items: + - const: apple,t8140-gfx-asc-mailbox - const: apple,asc-mailbox-v4 - description: From 5bc9e59ce089d97577c1c890e4768704de8b3c1d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:53:45 -0400 Subject: [PATCH 033/631] arm64: dts: apple: t8140: Add the GPU Add the GPU, the mailboxes of its two co-processors, its performance states and the reserved memory regions it shares with its firmware. The mailbox nodes describe only the mailbox block of each co-processor wrapper, at offset 0x8000. The bootloader fills in the reserved memory regions and the device-specific performance states. It also reserves uat-l2-pagetables for GPU page tables; the GPU node does not reference that region. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e77d74a6be7645e64b9b50bb51713e72a08c9520) --- arch/arm64/boot/dts/apple/t8140.dtsi | 112 +++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index a2752e59c07478..eec7a0759bfd42 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -372,6 +372,80 @@ }; }; + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + /* + * NOTE: The operating points are device-specific and must be + * filled in by the bootloader. Entries keep the order of the + * GPU performance states, which is not sorted by frequency. + */ + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <125000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <605000>; + }; + opp02 { + opp-hz = /bits/ 64 <492000000>; + opp-microvolt = <630000>; + }; + opp03 { + opp-hz = /bits/ 64 <618000000>; + opp-microvolt = <650000>; + }; + opp04 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <695000>; + }; + opp05 { + opp-hz = /bits/ 64 <928000000>; + opp-microvolt = <745000>; + }; + opp06 { + opp-hz = /bits/ 64 <952000000>; + opp-microvolt = <800000>; + }; + opp07 { + opp-hz = /bits/ 64 <1056000000>; + opp-microvolt = <800000>; + }; + opp08 { + opp-hz = /bits/ 64 <1053000000>; + opp-microvolt = <845000>; + }; + opp09 { + opp-hz = /bits/ 64 <1170000000>; + opp-microvolt = <845000>; + }; + opp10 { + opp-hz = /bits/ 64 <1152000000>; + opp-microvolt = <885000>; + }; + opp11 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <885000>; + }; + opp12 { + opp-hz = /bits/ 64 <1204000000>; + opp-microvolt = <905000>; + }; + opp13 { + opp-hz = /bits/ 64 <1338000000>; + opp-microvolt = <905000>; + }; + opp14 { + opp-hz = /bits/ 64 <1326000000>; + opp-microvolt = <975000>; + }; + opp15 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <975000>; + }; + }; + timer { compatible = "arm,armv8-timer"; interrupt-parent = <&aic>; @@ -1172,6 +1246,44 @@ ; interrupt-names = "send-empty", "recv-not-empty"; apple,firmware-role = "GFX1"; + reg = <0x4 0x82600000 0x0 0x4000>, + <0x4 0x82e00000 0x0 0x4000>, + <0x4 0x80000000 0x0 0x4000000>; + reg-names = "asc", "asc1", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, + <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + operating-points-v2 = <&gpu_opp>; + status = "disabled"; + }; + + gfx_mbox: mailbox@482608000 { + compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x82608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + + gfx1_mbox: mailbox@482e08000 { + compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x82e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; power-domains = <&ps_gfx>; #mbox-cells = <0>; status = "disabled"; From 4e7626b411df8e566b75706261e330463317a830 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:55:29 -0400 Subject: [PATCH 034/631] drm/asahi: uapi: Add the G17 additions Extend the UAPI for the G17 generation of GPUs: - drm_asahi_params_global gains the shader instruction set generation and the command stream generation (enum drm_asahi_gpu_hal_generation), which userspace needs to select its code generation and the layout of its control streams. - DRM_ASAHI_FEATURE_VM_STATUS and DRM_ASAHI_PARAM_GROUP_VM_STATUS report the sticky error of a VM, so that userspace can tell whether work behind a signalled fence executed successfully. - DRM_ASAHI_FEATURE_VM_STATUS_MIRROR and DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR let userspace register a word that the kernel keeps equal to that error. - DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES and DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS report how many queues the device supports and how many jobs of a queue may run at once. - DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS promises that every command of a submission waits for all of its input syncs. - The background and end-of-tile resource specifiers of G17 are 64 bits wide. Their high halves are appended to drm_asahi_cmd_render and flagged with DRM_ASAHI_RENDER_RSRC_SPEC_HI, which older kernels reject. The existing driver reports the generations of the G13 and G14 GPUs it supports and none of the new features. It still accepts render commands of the previous size, zero-filling the new fields, and rejects the new flag. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0a5c5ad84cbc6e48c0194882b8c1c67d72538a8) --- include/uapi/drm/asahi_drm.h | 191 ++++++++++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 1 deletion(-) diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index b31844bd41a6c5..ff2fec6485d2de 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -221,6 +221,32 @@ enum drm_asahi_gpu_hal_generation { DRM_ASAHI_GPU_HAL_200 = 200, /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream. Never reported by this kernel. */ + * @usc_generation: Instruction set generation of the unified shader + * cores: 2 for G13 and G14, 3 for G15 and newer. + */ + __u32 usc_generation; + + /** + * @gpu_hal_generation: Command stream generation, from + * enum drm_asahi_gpu_hal_generation. + */ + __u32 gpu_hal_generation; +}; + +/** + * enum drm_asahi_gpu_hal_generation - GPU command stream generation + * + * The command stream generation determines the layout of the control streams + * and descriptors that userspace builds for the GPU. + */ +enum drm_asahi_gpu_hal_generation { + /** @DRM_ASAHI_GPU_HAL_LEGACY: Unnumbered generation of G13 to G15. */ + DRM_ASAHI_GPU_HAL_LEGACY = 0, + + /** @DRM_ASAHI_GPU_HAL_200: Generation 200, used by G17P. */ + DRM_ASAHI_GPU_HAL_200 = 200, + + /** @DRM_ASAHI_GPU_HAL_300: Generation 300. */ DRM_ASAHI_GPU_HAL_300 = 300, }; @@ -337,6 +363,148 @@ struct drm_asahi_queue_limits { __u32 max_in_flight_per_queue; __u32 flags; __u32 pad; + /** + * @DRM_ASAHI_FEATURE_VM_STATUS: GET_PARAMS supports + * DRM_ASAHI_PARAM_GROUP_VM_STATUS. + */ + DRM_ASAHI_FEATURE_VM_STATUS = (1UL) << 1, + + /** + * @DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES: Each queue reserves its own + * hardware execution state when it is created, and GET_PARAMS + * supports DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS. + */ + DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES = (1UL) << 2, + + /** + * @DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS: Every hardware command of + * a submission waits for all of the submission's input syncs, including + * their errors, so independent commands need no extra dependency on the + * first command. Userspace that splits a batch across several + * submissions and relies on this must pass its input syncs to each of + * them. + */ + DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS = (1UL) << 3, + + /** + * @DRM_ASAHI_FEATURE_VM_STATUS_MIRROR: GET_PARAMS supports + * DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR, which registers a userspace + * word that always holds the VM status that + * DRM_ASAHI_PARAM_GROUP_VM_STATUS would return. The kernel stores a + * failure there before it signals any fence of the failed job, so after + * a successful wait a load of the word replaces the VM_STATUS query. + * Registration may still be refused. + */ + DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, +}; + +/** + * define DRM_ASAHI_PARAM_GROUP_VM_STATUS - GET_PARAMS group for + * struct drm_asahi_vm_status + */ +#define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 + +/** + * define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS - GET_PARAMS group for + * struct drm_asahi_queue_limits + */ +#define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 + +/** + * define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR - GET_PARAMS group for + * struct drm_asahi_vm_status_mirror + */ +#define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR 3 + +/** + * struct drm_asahi_vm_status - VM status, GET_PARAMS group + * DRM_ASAHI_PARAM_GROUP_VM_STATUS + * + * This is an in/out query on a VM of the calling file. Userspace sets + * &drm_asahi_get_params.pointer to this structure and + * &drm_asahi_get_params.size to its size. + * + * The kernel records the failure of an accepted job before it signals the + * job's fences, so userspace must query the status after a successful syncobj + * wait before it treats the work as successfully executed. A signalled + * syncobj alone does not indicate successful execution. An execution failure + * also marks other VMs whose work can no longer make progress. A query never + * resets or recovers the GPU. + * + * Invalid requests fail with EINVAL (non-zero @flags or @pad, wrong size), + * ENOENT (@vm_id unknown to the calling file) or EFAULT, and write nothing. + */ +struct drm_asahi_vm_status { + /** @vm_id: VM to query */ + __u32 vm_id; + + /** @flags: MBZ */ + __u32 flags; + + /** + * @error: Output: 0 if no submission error was observed, otherwise the + * first error as a negative errno. The error is sticky for the lifetime + * of the VM: neither creating queues nor waiting on other fences clears + * it. + */ + __s32 error; + + /** @pad: MBZ */ + __u32 pad; +}; + +/** + * struct drm_asahi_queue_limits - Queue limits, GET_PARAMS group + * DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS + * + * The query neither allocates nor resets anything. + */ +struct drm_asahi_queue_limits { + /** + * @max_queues: Maximum number of queues on the device, shared by all + * files and VMs. This is a hardware bound, not a promise that as many + * are currently available: QUEUE_CREATE reserves a queue's hardware + * state and fails once it runs out. + */ + __u32 max_queues; + + /** + * @max_in_flight_per_queue: Number of jobs of a queue that the + * hardware may run at once. Further jobs wait behind the preceding work + * and their dependencies. + */ + __u32 max_in_flight_per_queue; + + /** @flags: Reserved, zero */ + __u32 flags; + + /** @pad: Reserved, zero */ + __u32 pad; +}; + +/** + * struct drm_asahi_vm_status_mirror - VM status mirror registration, GET_PARAMS + * group DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR + * + * Registers a 32-bit word in a GEM object of the calling file. The kernel + * keeps its own reference to the object for the lifetime of the VM status, + * zeroes the word, and from then on keeps it equal to the VM status error + * (see &drm_asahi_vm_status.error). Each VM accepts one registration. + * EOPNOTSUPP means that userspace must keep using + * DRM_ASAHI_PARAM_GROUP_VM_STATUS. + */ +struct drm_asahi_vm_status_mirror { + /** @vm_id: VM whose status is mirrored */ + __u32 vm_id; + + /** @handle: GEM object holding the word, at most 64 KiB in size */ + __u32 handle; + + /** @offset: Byte offset of the word in the object, 4-byte aligned */ + __u32 offset; + + /** @flags: MBZ */ + __u32 flags; }; /** @@ -356,7 +524,10 @@ struct drm_asahi_get_params { /** @pad: MBZ */ __u32 pad; - /** @pointer: User pointer to parameter struct (VM status is in/out). */ + /** + * @pointer: User pointer to the parameter struct. The VM status groups + * also read their input from it. + */ __u64 pointer; /** @@ -953,6 +1124,13 @@ enum drm_asahi_render_flags { * Bits 6 and 7 are supported only by the HAL200 runtime. * The legacy runtime rejects render commands that set them. */ + * @DRM_ASAHI_RENDER_RSRC_SPEC_HI: The command carries the high halves + * of the background and end-of-tile resource specifiers + * (&drm_asahi_cmd_render.bg_rsrc_spec_hi and the following fields). + * Kernels that do not know the flag reject the command instead of + * silently truncating the specifiers. + */ + DRM_ASAHI_RENDER_RSRC_SPEC_HI = (1U << 4), /** * @DRM_ASAHI_RENDER_DBIAS_IS_INT: Use integer depth bias formula. @@ -1287,6 +1465,17 @@ struct drm_asahi_cmd_render { /** @ts_frag_end_copies: End timestamp copies, requiring the corresponding feature bit. */ struct drm_asahi_timestamp ts_frag_end_copies[8]; + /** @bg_eot_rsrc_spec_hi: High half of @eot.rsrc_spec */ + __u32 bg_eot_rsrc_spec_hi; + + /** @bg_eot_partial_rsrc_spec_hi: High half of @partial_eot.rsrc_spec */ + __u32 bg_eot_partial_rsrc_spec_hi; + + /** @bg_rsrc_spec_hi: High half of @bg.rsrc_spec */ + __u32 bg_rsrc_spec_hi; + + /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ + __u32 bg_partial_rsrc_spec_hi; }; /** From 3bd29cdfb3d29b6ece87fd88ec46c0bc26a884b0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:07:41 -0400 Subject: [PATCH 035/631] drm/asahi: separate DRM-facing GPU operations from the firmware manager The file and ioctl layer uses a manager trait that also exposes G13/G14 firmware allocators, VM slots, event handling and control channels. A GPU with a different firmware interface cannot implement those internals. Separate the DRM-facing operations into Gpu: VM and queue creation, timestamp mapping, sequence IDs, crash state, timer frequency and hardware parameters. Existing managers implement it directly and expose their firmware-specific interface through Gpu::manager(). Keep the legacy firmware-global update at ioctl entry after flag validation, before queue lookup and userspace copies, outside the queue mutex. Invalid queue submissions still refresh the firmware idle policy. Legacy operation ordering and reported hardware values are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit aae9f7d3525f93a7a5b5ae0529f8e85219735d08) --- drivers/gpu/drm/asahi/gpu.rs | 170 +++++++++++++++++++++++------------ 1 file changed, 114 insertions(+), 56 deletions(-) diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index bb9492270b7a90..0e8a1bd9f9e1b7 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -552,24 +552,26 @@ pub(crate) struct GpuManager { garbage_contexts: Mutex>>>, } -/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. -pub(crate) trait GpuManager: Send + Sync { +/// Operations the DRM driver performs on a GPU, independent of its firmware interface. +pub(crate) trait Gpu: Send + Sync { /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. fn arc_as_any(self: Arc) -> Arc; + /// Return the G13/G14 firmware manager, if this GPU uses that firmware interface. + fn manager(&self) -> Option<&dyn GpuManager>; /// Initialize the GPU. fn init(&self) -> Result; - /// Update the GPU globals from global info - /// - /// TODO: Unclear what can and cannot be updated like this. - fn update_globals(&self); - /// Get a reference to the KernelAllocators. - fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; - /// Create a new `Vm` given a unique `File` ID. + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; + /// Fill in the hardware description reported to userspace. + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Frequency of the CPU timer that GPU timestamps are derived from. + fn base_clock_hz(&self) -> u32; + /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; - /// Bind a `Vm` to an available slot and return the `VmBind`. - fn bind_vm(&self, vm: &mmu::Vm) -> Result; /// Create a new user command queue. fn new_queue( &self, @@ -579,8 +581,24 @@ pub(crate) trait GpuManager: Send + Sync { priority: u32, usc_exec_base: u64, ) -> Result>; - /// Return a reference to the global `SequenceIDs` instance. - fn ids(&self) -> &SequenceIDs; + /// Map a BO as a timestamp buffer + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result; +} + +/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. +pub(crate) trait GpuManager: Send + Sync { + /// Update the GPU globals from global info + /// + /// TODO: Unclear what can and cannot be updated like this. + fn update_globals(&self); + /// Get a reference to the KernelAllocators. + fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; + /// Bind a `Vm` to an available slot and return the `VmBind`. + fn bind_vm(&self, vm: &mmu::Vm) -> Result; /// Kick the firmware (wake it up if asleep). /// /// This should be useful to reduce latency on work submission, so we can ask the firmware to @@ -614,14 +632,6 @@ pub(crate) trait GpuManager: Send + Sync { fn get_dyncfg(&self) -> &hw::DynConfig; /// Register an unused context as garbage fn free_context(&self, data: KBox>); - /// Check whether the GPU is crashed - fn is_crashed(&self) -> bool; - /// Map a BO as a timestamp buffer - fn map_timestamp_buffer( - &self, - bo: gem::ObjectRef, - range: Range, - ) -> Result; } /// Private generic trait for functions that don't need to escape this module. @@ -2902,6 +2912,89 @@ impl GpuManager for GpuManager::ver { Ok(()) } + fn ids(&self) -> &SequenceIDs { + &self.ids + } + + fn is_crashed(&self) -> bool { + self.crashed.load(Ordering::Relaxed) + } + + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = self.dyncfg.id.gpu_gen as u32; + params.gpu_variant = self.dyncfg.id.gpu_variant as u32; + params.gpu_revision = self.dyncfg.id.gpu_rev as u32; + params.chip_id = self.cfg.chip_id; + params.num_dies = self.cfg.num_dies; + params.num_clusters_total = self.dyncfg.id.num_clusters; + params.num_cores_per_cluster = self.dyncfg.id.num_cores; + params.max_frequency_khz = self.dyncfg.pwr.max_frequency_khz(); + // G13 and G14 share the second shader ISA generation and predate the numbered command + // stream generations. + params.usc_generation = 2; + params.gpu_hal_generation = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY; + + for (i, mask) in self.dyncfg.id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + Ok(()) + } + + fn base_clock_hz(&self) -> u32 { + self.cfg.base_clock_hz + } + + fn new_vm(&self, kernel_range: Range) -> Result { + self.uat.new_vm(self.ids.vm.next(), kernel_range) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(KBox::new( + queue::Queue::ver::new( + &self.dev, + self, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + usc_exec_base, + )?, + GFP_KERNEL, + )?) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + bo.map_range_into_range( + self.uat.kernel_vm(), + range, + IOVA_KERN_TIMESTAMP_RANGE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +#[versions(AGX)] +impl GpuManager for GpuManager::ver { fn update_globals(&self) { let mut timeout: u32 = 2; if debug_enabled(DebugFlags::WaitForPowerOff) { @@ -3140,41 +3233,10 @@ impl GpuManager for GpuManager::ver { guard } - fn new_vm(&self, kernel_range: Range) -> Result { - self.uat.new_vm(self.ids.vm.next(), kernel_range) - } - fn bind_vm(&self, vm: &mmu::Vm) -> Result { self.uat.bind(vm) } - fn new_queue( - &self, - vm: mmu::Vm, - ualloc: Arc>, - ualloc_priv: Arc>, - priority: u32, - usc_exec_base: u64, - ) -> Result> { - let mut kalloc = self.alloc(); - let id = self.ids.queue.next(); - Ok(KBox::new( - queue::Queue::ver::new( - &self.dev, - vm, - &mut kalloc, - ualloc, - ualloc_priv, - self.event_manager.clone(), - &self.buffer_mgr, - id, - priority, - usc_exec_base, - )?, - GFP_KERNEL, - )?) - } - fn kick_firmware(&self) -> Result { if self.is_crashed() { return Err(ENODEV); @@ -3238,10 +3300,6 @@ impl GpuManager for GpuManager::ver { Ok(()) } - fn ids(&self) -> &SequenceIDs { - &self.ids - } - fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { dev_err!(self.dev.as_ref(), " (\\________/) \n"); dev_err!(self.dev.as_ref(), " | | \n"); From 0f22aa2df500fbc3bd59155c2c2531edbde23e42 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 2 Oct 2026 02:57:46 -0400 Subject: [PATCH 036/631] drm/asahi: mmu: support per-SoC UAT geometry and user VA range G13 and G14 GPUs translate 39 input address bits per UAT root, and the driver hardcodes that width in the page table walker, the user and kernel VA ranges and the user range reported to userspace. Newer GPUs use the same 16 KiB granule and three-level walk, but translate 42 bits per root. Describe the address width of a UAT instance with a UatGeometry (input and output address bits) and derive the ranges from it: the user range spans the lower root, the dummy page sits two pages below its top, and the driver-managed kernel range is the third top-level entry of the upper root. The input width becomes per-SoC configuration data next to the output width, Uat::new takes the geometry instead of the firmware configuration, the page table walker takes the input width per table, and the file layer gets the user range from the device instead of a constant. All G13/G14 configurations describe a 39-bit input width, for which every derived value equals the constant it replaces: user top 1 << 39, dummy page 0x7f_ffff_8000, kernel range 0xffff_ffa0_0000_0000 to 0xffff_ffb0_0000_0000, upper range from 0xffff_ff80_0000_0000, and the walker's input mask. No functional change on those SoCs. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7d3ddb6c35e1c478f68c78ac8bb8b3ee9d314e58) --- drivers/gpu/drm/asahi/file.rs | 4 ++ drivers/gpu/drm/asahi/gpu.rs | 11 ++++- drivers/gpu/drm/asahi/mmu.rs | 79 +++++++++++++++++++++++++++++--- drivers/gpu/drm/asahi/pgtable.rs | 12 +++++ 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 4738a825fa83e6..b8c0d4cdcc1f51 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -66,6 +66,7 @@ struct Vm { ualloc: Arc>, ualloc_priv: Arc>, vm: mmu::Vm, + user_range: Range, kernel_range: Range, user_range: Range, _dummy_mapping: mmu::KernelMapping, @@ -221,6 +222,8 @@ pub(crate) enum Object { // #[pin_data] pub(crate) struct File { id: u64, + /// Available VM range for the user + user_range: Range, // #[pin] vms: xarray::XArray>, // #[pin] @@ -519,6 +522,7 @@ impl File { ualloc, ualloc_priv, vm, + user_range, kernel_range, user_range, _dummy_mapping: dummy_mapping, diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 0e8a1bd9f9e1b7..c0d126e8e1dcab 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -570,6 +570,8 @@ pub(crate) trait Gpu: Send + Sync { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; + /// Return the GPU address space geometry. + fn uat_geometry(&self) -> mmu::UatGeometry; /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. @@ -1470,7 +1472,14 @@ impl GpuManager::ver { let map_kernel_to_user = false; Ok(KBox::new( - mmu::Uat::new(dev, cfg, map_kernel_to_user)?, + mmu::Uat::new( + dev, + mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }, + map_kernel_to_user, + )?, GFP_KERNEL, )?) } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index e39a328f5394d4..2906ced50bdf4b 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -83,7 +83,8 @@ pub(crate) use pgtable::{ UatPageTable, UAT_PGBIT, UAT_PGMSK, - UAT_PGSZ, // + UAT_PGSZ, + UAT_TOP_LEVEL_SHIFT, // }; use pin_init; @@ -228,12 +229,59 @@ pub(crate) fn kern_iova_range(cfg: &hw::HwConfig, range: Range) -> Range u64 { + 1 << self.ias + } + + /// Lower/user VA range + pub(crate) const fn user_range(&self) -> Range { + IOVA_USER_BASE..self.user_top() + } + + /// Address of a special dummy page? + pub(crate) const fn unk_page(&self) -> u64 { + self.user_top() - 2 * UAT_PGSZ as u64 + } + + /// User VA range excluding the unk page + pub(crate) const fn user_usable_range(&self) -> Range { + IOVA_USER_BASE..self.unk_page() + } + + /// Upper/kernel base VA + const fn upper_base(&self) -> u64 { + !(self.user_top() - 1) + } + + /// Driver-managed kernel VA range + pub(crate) const fn kernel_range(&self) -> Range { + let start = self.upper_base() + (IOVA_KERN_TOP_LEVEL_INDEX << UAT_TOP_LEVEL_SHIFT); + start..(start + (1 << UAT_TOP_LEVEL_SHIFT)) + } + + /// Full kernel VA range + #[cfg(CONFIG_DEV_COREDUMP)] + const fn upper_range(&self) -> Range { + self.upper_base()..(!UAT_PGMSK as u64) + } +} /// A pre-allocated memory region for UAT management struct UatRegion { @@ -1442,6 +1490,7 @@ impl UatInner { pub(crate) struct Uat { dev: driver::AsahiDevRef, cfg: UatConfig, + geometry: UatGeometry, inner: Arc, slots: slotalloc::SlotAllocator, @@ -1682,6 +1731,7 @@ impl Vm { dev: &driver::AsahiDevice, uat_inner: Arc, kernel_range: Range, + geometry: UatGeometry, cfg: UatConfig, ttb: Option, id: u64, @@ -1722,12 +1772,13 @@ impl Vm { UatPageTable::new_m3_coherent(cfg.ias,cfg.oas,dev)? } else { - UatPageTable::new(cfg.ias, cfg.oas)? + UatPageTable::new(geometry.ias, geometry.oas)? }; let lower_start = IOVA_USER_BASE; let driver_start = lower_start; let (va_range, gpuvm_range) = if is_kernel { + (geometry.kernel_range(), kernel_range.clone()) (iova_kern_range, kernel_range.clone()) } else { ( @@ -2586,6 +2637,11 @@ impl Uat { ) } + /// Returns the address space geometry of this UAT. + pub(crate) fn geometry(&self) -> UatGeometry { + self.geometry + } + /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] fn make_inner(dev: &driver::AsahiDevice, handoff_mode: HandoffMode, m3: bool) -> Result> { @@ -2683,7 +2739,7 @@ impl Uat { #[inline(never)] pub(crate) fn new( dev: &driver::AsahiDevice, - cfg: &'static hw::HwConfig, + geometry: UatGeometry, map_kernel_to_user: bool, ) -> Result { Self::new_with_handoff_mode(dev, cfg, map_kernel_to_user, HandoffMode::FirmwareDekker) @@ -2784,13 +2840,22 @@ impl Uat { kernel_lower_vm.inner.exec_lock(None, false)?.fixed_slot = Some(0); } let kernel_vm = Vm::new(dev, inner.clone(), kernel_range, cfg, Some(ttb1), 0)?; + let kernel_lower_vm = Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_vm = Vm::new( + dev, + inner.clone(), + geometry.kernel_range(), + geometry, + Some(ttb1), + 0, + )?; dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); let ttb0 = kernel_lower_vm.ttb(); let uat = Self { dev: dev.into(), - cfg, + geometry, kernel_vm, kernel_lower_vm, inner, diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 8ac13d5bd2578a..b2d49440b5da65 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -54,6 +54,10 @@ const UAT_LVSZ: usize = UAT_NPTE; const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; const UAT_LEVELS: usize = crate::uat::UAT_LEVELS; +const UAT_LEVELS: usize = 3; + +/// Number of address bits below the index of a top-level page table entry. +pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; const PTE_TYPE_BITS: u64 = 3; const PTE_TYPE_LEAF_TABLE: u64 = 3; @@ -368,6 +372,8 @@ pub(crate) struct UatPageTable { } impl UatPageTable { + pub(crate) fn new(ias: u32, oas: u32) -> Result { + mod_pr_debug!("UATPageTable::new: ias={} oas={}\n", ias, oas); pub(crate) fn new(ias: u8, oas: u32) -> Result { mod_pr_debug!("UATPageTable::new: ias={} oas={}\n", ias, oas); let geometry = UatGeometry::new(ias).ok_or(EINVAL)?; @@ -382,6 +388,8 @@ impl UatPageTable { reserved_tables: None, va_range: 0..geometry.root_size(), geometry, + va_range: 0..(1u64 << ias), + ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, coverage: None, }) @@ -421,6 +429,10 @@ impl UatPageTable { ttb: PhysicalAddr, va_range: Range, ias: u8, + pub(crate) fn new_with_ttb( + ttb: PhysicalAddr, + va_range: Range, + ias: u32, oas: u32, ) -> Result { mod_pr_debug!( From 78df7887933c345be0c3bc5b0fbb5eff5e7bde72 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 1 Oct 2026 22:12:48 -0400 Subject: [PATCH 037/631] drm/asahi: pgtable: track ownership below firmware-shared roots A borrowed root does not determine ownership of its child tables. The handoff firmware interface leaves the driver owning all child tables in its managed range; another interface may publish its own child tables into that root. Make shared child-table ownership explicit when borrowing a root. In that mode, record each driver allocation before publishing its pointer and free only recorded tables. Keep the borrowed root itself unowned. The existing handoff caller selects non-shared children, preserving its allocation and teardown behavior without an extra ownership allocation. Driver-allocated roots also retain their existing ownership behavior. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5e21536704b56c38b478516d1f65bdcfef748a17) --- drivers/gpu/drm/asahi/mmu.rs | 7 +++++++ drivers/gpu/drm/asahi/pgtable.rs | 27 ++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2906ced50bdf4b..ed907a54636f1f 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1743,6 +1743,13 @@ impl Vm { let iova_kern_range = iova_kern_range(cfg)?; let page_table = if let Some(ttb) = ttb { + UatPageTable::new_with_ttb( + ttb, + geometry.kernel_range(), + geometry.ias, + geometry.oas, + false, + )? if cfg.reserved_tables { let node = dev.as_ref().of_node().ok_or(ENODEV)?; let mut resources = KVec::new(); diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index b2d49440b5da65..ea2c806ab9890f 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -365,6 +365,10 @@ pub(crate) struct UatPageTable { noncoherent: bool, dma_tables: Option, reserved_tables: Option, + /// Tables below a root the driver did not allocate may belong to the bootloader or the + /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the + /// tables below the root, as with the handoff firmware interface. + driver_tables: Option>, va_range: Range, geometry: UatGeometry, oas_mask: u64, @@ -388,6 +392,7 @@ impl UatPageTable { reserved_tables: None, va_range: 0..geometry.root_size(), geometry, + driver_tables: None, va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, @@ -429,11 +434,14 @@ impl UatPageTable { ttb: PhysicalAddr, va_range: Range, ias: u8, + /// Borrow a root; track individual child ownership only when other agents + /// can install tables below it. Handoff roots have driver-owned children. pub(crate) fn new_with_ttb( ttb: PhysicalAddr, va_range: Range, ias: u32, oas: u32, + shared_tables: bool, ) -> Result { mod_pr_debug!( "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", @@ -449,7 +457,7 @@ impl UatPageTable { if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { return Err(EINVAL); } - // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + // SAFETY: The bootloader-managed root stays valid while this page table exists. if unsafe { Page::borrow_phys(&ttb) }.is_none() { pr_err!( "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", @@ -465,6 +473,7 @@ impl UatPageTable { noncoherent: false, dma_tables: None, reserved_tables: None, + driver_tables: shared_tables.then(KVec::new), va_range, geometry, oas_mask: (1u64 << oas) - 1, @@ -724,6 +733,9 @@ impl UatPageTable { if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if let Some(tables) = self.driver_tables.as_mut() { + tables.push(pt_page.phys(), GFP_KERNEL)?; + } let pt_paddr = Page::into_phys(pt_page); upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; upte.store(upte_val, Ordering::Relaxed); @@ -970,6 +982,19 @@ impl UatPageTable { } Ok(()) + /// Free a page table, unless it was not allocated by the driver. + fn free_table(&mut self, phys: PhysicalAddr) { + if let Some(tables) = self.driver_tables.as_mut() { + match tables.iter().position(|&table| table == phys) { + Some(index) => { + tables.swap_remove(index); + } + None => return, + } + } + // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without + // `driver_tables`, every child table belongs to the driver. + unsafe { Page::from_phys(phys) }; } pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { From fb2fbfe4a76a49baca0a6eda859d2978c8d34ce2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 038/631] drm/asahi: add the T8140 firmware interface Provide boot, address translation and retained firmware resources for T8140, with independent render and compute queues behind the existing GPU interface. Keep visible command storage and VM pins through exact retirement, firmware drain or processor stop, and finish fences outside the device lock. Implement the G17 command ABI, per-VM status, logical ordering, asynchronous resource growth and recovery isolation together so every exposed operation has its firmware and ownership consumers. Validate shared recovery views and use bounded halted-slot identity reads to resolve ambiguous render blame. Retain unexecuted compute work for bounded replay when no firmware consumer has already depended on it. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c0e36ff2a57f2a461a96b80cebb741f5b76de7ad) --- drivers/gpu/drm/asahi/asahi.rs | 1 + drivers/gpu/drm/asahi/driver.rs | 59 +- drivers/gpu/drm/asahi/file.rs | 239 +- drivers/gpu/drm/asahi/g17/buffer.rs | 889 +++++ drivers/gpu/drm/asahi/g17/buffer/target.rs | 297 ++ drivers/gpu/drm/asahi/g17/channel.rs | 220 ++ drivers/gpu/drm/asahi/g17/command.rs | 428 +++ drivers/gpu/drm/asahi/g17/completion.rs | 390 +++ drivers/gpu/drm/asahi/g17/context.rs | 361 ++ drivers/gpu/drm/asahi/g17/context/identity.rs | 71 + drivers/gpu/drm/asahi/g17/dependency.rs | 83 + drivers/gpu/drm/asahi/g17/event.rs | 121 + drivers/gpu/drm/asahi/g17/fence.rs | 197 ++ drivers/gpu/drm/asahi/g17/freelist.rs | 575 ++++ drivers/gpu/drm/asahi/g17/fw/buffer.rs | 248 ++ drivers/gpu/drm/asahi/g17/fw/channels.rs | 522 +++ drivers/gpu/drm/asahi/g17/fw/compute.rs | 379 ++ drivers/gpu/drm/asahi/g17/fw/context.rs | 72 + drivers/gpu/drm/asahi/g17/fw/freelist.rs | 124 + drivers/gpu/drm/asahi/g17/fw/initdata.rs | 3063 +++++++++++++++++ drivers/gpu/drm/asahi/g17/fw/kick.rs | 539 +++ drivers/gpu/drm/asahi/g17/fw/mod.rs | 109 + drivers/gpu/drm/asahi/g17/fw/qos.rs | 34 + drivers/gpu/drm/asahi/g17/fw/queue.rs | 540 +++ drivers/gpu/drm/asahi/g17/fw/render.rs | 1309 +++++++ .../gpu/drm/asahi/g17/fw/render/storage.rs | 178 + drivers/gpu/drm/asahi/g17/hazard.rs | 120 + drivers/gpu/drm/asahi/g17/ids.rs | 60 + drivers/gpu/drm/asahi/g17/initdata.rs | 1305 +++++++ drivers/gpu/drm/asahi/g17/job.rs | 334 ++ drivers/gpu/drm/asahi/g17/kick.rs | 335 ++ drivers/gpu/drm/asahi/g17/mod.rs | 1153 +++++++ drivers/gpu/drm/asahi/g17/object.rs | 579 ++++ drivers/gpu/drm/asahi/g17/power.rs | 125 + drivers/gpu/drm/asahi/g17/preparation.rs | 186 + drivers/gpu/drm/asahi/g17/qos.rs | 244 ++ drivers/gpu/drm/asahi/g17/queue/admission.rs | 541 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 1507 ++++++++ .../gpu/drm/asahi/g17/queue/compute/exit.rs | 207 ++ .../drm/asahi/g17/queue/compute/retirement.rs | 214 ++ drivers/gpu/drm/asahi/g17/queue/mod.rs | 318 ++ drivers/gpu/drm/asahi/g17/queue/render.rs | 1459 ++++++++ .../asahi/g17/queue/render/dependencies.rs | 109 + .../gpu/drm/asahi/g17/queue/render/memory.rs | 708 ++++ .../drm/asahi/g17/queue/render/observation.rs | 85 + .../gpu/drm/asahi/g17/queue/render/parked.rs | 243 ++ .../drm/asahi/g17/queue/render/publication.rs | 164 + .../drm/asahi/g17/queue/render/retirement.rs | 348 ++ .../gpu/drm/asahi/g17/queue/render/warm.rs | 49 + drivers/gpu/drm/asahi/g17/recovery.rs | 286 ++ .../gpu/drm/asahi/g17/recovery/attribution.rs | 100 + drivers/gpu/drm/asahi/g17/recovery/slot.rs | 74 + drivers/gpu/drm/asahi/g17/runtime.rs | 869 +++++ drivers/gpu/drm/asahi/g17/runtime/backend.rs | 186 + .../drm/asahi/g17/runtime/compute_control.rs | 56 + .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 166 + .../drm/asahi/g17/runtime/compute_replay.rs | 82 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 419 +++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 683 ++++ .../asahi/g17/runtime/render/dependencies.rs | 34 + .../gpu/drm/asahi/g17/runtime/render/exit.rs | 39 + .../gpu/drm/asahi/g17/runtime/render/grow.rs | 309 ++ .../drm/asahi/g17/runtime/render/observe.rs | 235 ++ .../gpu/drm/asahi/g17/runtime/render/pools.rs | 224 ++ .../drm/asahi/g17/runtime/render/prepare.rs | 597 ++++ .../drm/asahi/g17/runtime/render/recovery.rs | 27 + .../drm/asahi/g17/runtime/render/settle.rs | 235 ++ .../drm/asahi/g17/runtime/render/submit.rs | 306 ++ .../drm/asahi/g17/runtime/render/teardown.rs | 164 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 199 ++ drivers/gpu/drm/asahi/g17/status.rs | 102 + drivers/gpu/drm/asahi/g17/teardown.rs | 222 ++ drivers/gpu/drm/asahi/g17/validation.rs | 145 + drivers/gpu/drm/asahi/gem.rs | 14 + drivers/gpu/drm/asahi/gpu.rs | 9 + drivers/gpu/drm/asahi/hw/mod.rs | 20 + drivers/gpu/drm/asahi/hw/t8140.rs | 392 +++ drivers/gpu/drm/asahi/mem.rs | 62 + drivers/gpu/drm/asahi/mmu.rs | 855 ++++- drivers/gpu/drm/asahi/mmu/bind.rs | 191 + drivers/gpu/drm/asahi/mmu/context.rs | 232 ++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 605 ++++ drivers/gpu/drm/asahi/mmu/shared.rs | 134 + drivers/gpu/drm/asahi/pgtable.rs | 79 + drivers/gpu/drm/asahi/regs.rs | 302 +- 85 files changed, 30038 insertions(+), 57 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/buffer.rs create mode 100644 drivers/gpu/drm/asahi/g17/buffer/target.rs create mode 100644 drivers/gpu/drm/asahi/g17/channel.rs create mode 100644 drivers/gpu/drm/asahi/g17/command.rs create mode 100644 drivers/gpu/drm/asahi/g17/completion.rs create mode 100644 drivers/gpu/drm/asahi/g17/context.rs create mode 100644 drivers/gpu/drm/asahi/g17/context/identity.rs create mode 100644 drivers/gpu/drm/asahi/g17/dependency.rs create mode 100644 drivers/gpu/drm/asahi/g17/event.rs create mode 100644 drivers/gpu/drm/asahi/g17/fence.rs create mode 100644 drivers/gpu/drm/asahi/g17/freelist.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/buffer.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/channels.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/compute.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/context.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/freelist.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/kick.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/qos.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/queue.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/fw/render/storage.rs create mode 100644 drivers/gpu/drm/asahi/g17/hazard.rs create mode 100644 drivers/gpu/drm/asahi/g17/ids.rs create mode 100644 drivers/gpu/drm/asahi/g17/initdata.rs create mode 100644 drivers/gpu/drm/asahi/g17/job.rs create mode 100644 drivers/gpu/drm/asahi/g17/kick.rs create mode 100644 drivers/gpu/drm/asahi/g17/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/object.rs create mode 100644 drivers/gpu/drm/asahi/g17/power.rs create mode 100644 drivers/gpu/drm/asahi/g17/preparation.rs create mode 100644 drivers/gpu/drm/asahi/g17/qos.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/admission.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/mod.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/memory.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/observation.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/parked.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/publication.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/retirement.rs create mode 100644 drivers/gpu/drm/asahi/g17/queue/render/warm.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/attribution.rs create mode 100644 drivers/gpu/drm/asahi/g17/recovery/slot.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/backend.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_control.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/events.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/exit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/grow.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/observe.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/pools.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/settle.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/submit.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/status.rs create mode 100644 drivers/gpu/drm/asahi/g17/teardown.rs create mode 100644 drivers/gpu/drm/asahi/g17/validation.rs create mode 100644 drivers/gpu/drm/asahi/hw/t8140.rs create mode 100644 drivers/gpu/drm/asahi/mmu/bind.rs create mode 100644 drivers/gpu/drm/asahi/mmu/context.rs create mode 100644 drivers/gpu/drm/asahi/mmu/lifetime.rs create mode 100644 drivers/gpu/drm/asahi/mmu/shared.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index a01ae008af44cb..f7ab5fde52b770 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -110,6 +110,7 @@ mod agx_resources; mod agx_status; mod agx_uapi; mod gem; +mod g17; mod gpu; mod hw; mod identity; diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 517a10c7a2f91e..b35da14c5b2020 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -26,6 +26,7 @@ use crate::{ drm_gpu, file, g15_boot, + g17, gem::AsahiObject, gpu, hw, @@ -164,12 +165,22 @@ impl drm::driver::Driver for AsahiDriver { } } -// OF Device ID table.s +/// Firmware interface selected by the hardware compatible. +pub(crate) enum ProbeConfig { + Legacy(&'static hw::HwConfig), + G17(&'static g17::Config), +} + +// OF Device ID table. kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, ::IdInfo, [ + ( + of::DeviceId::new(c_str!("apple,agx-t8140")), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), ( of::DeviceId::new(c_str!("apple,agx-t8103")), ProbeConfig::Supported(&hw::t8103::HWCONFIG) @@ -418,6 +429,9 @@ impl platform::Driver for AsahiDriver { AsahiRuntime::M3(runtime) => runtime.stop(), AsahiRuntime::G16(runtime) => runtime.stop(), _ => {}, + if let Some(gpu) = (*this.drm).gpu.as_any().downcast_ref::() { + // Driver data is dropped after devres, so CPU control must happen in unbind. + gpu.shutdown(); } } @@ -483,6 +497,8 @@ impl platform::Driver for AsahiDriver { } } ProbeConfig::Agx3Diagnostic(soc) => return Err(refuse_agx3_probe(pdev, soc)), + ProbeConfig::Legacy(cfg) => *cfg, + ProbeConfig::G17(cfg) => return Self::probe_g17(pdev, cfg), }; unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; @@ -544,3 +560,44 @@ impl platform::Driver for AsahiDriver { }) } } + +impl AsahiDriver { + fn probe_g17(pdev: &platform::Device, cfg: &'static g17::Config) -> Result { + // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let res = regs::Resources::new(pdev)?; + // SAFETY: As in the legacy probe, the data is inaccessible until the fully initialized + // manager is installed below and the DRM device is registered. + // If construction fails, drm::device::Device::release leaves the data + // untouched rather than dropping an uninitialized AsahiData. + let uninit = unsafe { pin_init::pin_init_from_closure::(|_slot| Ok(())) }; + let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; + let data_gpu = gpu.clone() as Arc; + let data = try_pin_init!(AsahiData { + gpu: data_gpu, + pdev: pdev.into(), + resources: res, + }); + let ptr = &raw const **drm; + // SAFETY: The allocation is pinned and its data slot has not been initialized yet. + if let Err(error) = unsafe { data.__pinned_init(ptr as *mut AsahiData) } { + gpu.shutdown(); + return Err(error); + } + if let Err(error) = drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0) { + gpu.shutdown(); + return Err(error); + } + Ok(Self { drm }) + } +} + +impl Drop for AsahiDriver { + fn drop(&mut self) { + if let Some(gpu) = (*self.drm).gpu.as_any().downcast_ref::() { + // Also covers failure to allocate platform driver data after the probe body returned. + gpu.shutdown(); + } + } +} diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index b8c0d4cdcc1f51..c4ea5a564c5f1e 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -38,6 +38,7 @@ use kernel::error::code::*; use kernel::new_mutex; use kernel::prelude::*; use kernel::sync::{ + aref::ARef, Arc, Mutex, // }; @@ -74,6 +75,18 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { + if self.vm.status().is_some() { + // Tracked jobs can outlive their file's VM handle. Their last guard performs + // these unmaps once no accepted work can still dereference the mappings. + if let Err(error) = self + .vm + .unmap_user_ranges(self.user_range.clone(), self.kernel_range.clone()) + { + pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); + } + self.vm.bo_deferred_cleanup(); + return; + } if !self.vm.is_m3() { // When the user Vm is dropped, unmap everything in the user range @@ -321,6 +334,32 @@ impl File { return Self::get_vm_status(device, data, file); } + match data.param_group { + uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS if gpu.supports_vm_status() => { + return Self::get_vm_status(data, file); + } + uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR if gpu.supports_vm_status() => { + return Self::register_vm_status_mirror(data, file); + } + uapi::DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS => { + if data.pad != 0 + || data.size < core::mem::size_of::() as u64 + { + return Err(EINVAL); + } + let limits = gpu.queue_limits().ok_or(EINVAL)?; + let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; + UserSlice::new( + UserPtr::from_addr(data.pointer as _), + core::mem::size_of_val(&words), + ) + .writer() + .write(&words)?; + return Ok(0); + } + _ => {} + } + if data.param_group != 0 || data.pad != 0 { cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); return Err(EINVAL); @@ -362,6 +401,15 @@ impl File { params.features |= crate::agx_status::FEATURE_VM_STATUS; } + if gpu.supports_vm_status() { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + } + if gpu.queue_limits().is_some() { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + } + + if *module_parameters::fault_control.value() == 0xb { if gpu.supports_scheduled_queues() { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64; } @@ -425,6 +473,55 @@ impl File { Ok(0) } + /// Query only file-owned VM state; this does not start recovery or wait for work. + fn get_vm_status(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { + let size = core::mem::size_of::(); + if data.pad != 0 || data.size < size as u64 { + return Err(EINVAL); + } + let user = UserPtr::from_addr(data.pointer as _); + let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; + if flags != 0 || pad != 0 { + return Err(EINVAL); + } + let status = Self::vm_status(file, vm_id)?; + UserSlice::new(user, size) + .writer() + .write(&[vm_id, 0, status.get() as u32, 0])?; + Ok(0) + } + + /// Register a file-owned GEM word as the mirror of a file-owned VM's status. + fn register_vm_status_mirror(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { + let size = core::mem::size_of::(); + if data.pad != 0 || data.size < size as u64 { + return Err(EINVAL); + } + let [vm_id, handle, offset, flags] = + UserSlice::new(UserPtr::from_addr(data.pointer as _), size) + .reader() + .read::<[u32; 4]>()?; + if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { + return Err(EINVAL); + } + let status = Self::vm_status(file, vm_id)?; + let owner = gem::ObjectRef::new(gem::Object::lookup_handle(file, handle)?); + status.attach_mirror(owner, offset as usize)?; + Ok(0) + } + + fn vm_status(file: &DrmFile, vm_id: u32) -> Result> { + let vms = file.inner().vms(); + let guard = vms.lock(); + Ok(guard + .get(vm_id as usize) + .ok_or(ENOENT)? + .vm + .status() + .ok_or(EINVAL)? + .clone()) + } + /// IOCTL: vm_create: Create a new `Vm`. pub(crate) fn vm_create( device: &AsahiDevice, @@ -649,6 +746,10 @@ impl File { return Err(EINVAL); } + if device.gpu.as_any().is::() { + return Self::vm_bind_g17(data, file); + } + let vm_id = data.vm_id.try_into()?; let mut vec = KVec::new(); @@ -676,6 +777,118 @@ impl File { Ok(0) } + /// Validate all operations and retain their objects before changing a tracked VM. + fn vm_bind_g17(data: &uapi::drm_asahi_vm_bind, file: &DrmFile) -> Result { + let count = usize::try_from(data.num_binds)?; + let stride = usize::try_from(data.stride)?; + let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; + let mut bytes = KVec::new(); + UserSlice::new(UserPtr::from_addr(data.userptr as _), size) + .reader() + .read_all(&mut bytes, GFP_KERNEL)?; + let (vm, user_range, kernel_range) = { + let vms = file.inner().vms(); + let guard = vms.lock(); + let entry = guard.get(data.vm_id as usize).ok_or(ENOENT)?; + ( + entry.vm.clone(), + entry.user_range.clone(), + entry.kernel_range.clone(), + ) + }; + let mut batch = mmu::PreparedUserBindBatch::new(count)?; + let mut updates = KVec::with_capacity(count, GFP_KERNEL)?; + let mut unmaps = 0; + // A fresh reader per declared stride also skips extension bytes in longer records. + for bytes in bytes.chunks_exact(stride) { + let op: uapi::drm_asahi_gem_bind_op = Reader::new(bytes).read_up_to(stride)?; + if op.range == 0 || (op.addr | op.range | op.offset) & mmu::UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let range = op.addr..op.addr.checked_add(op.range).ok_or(EINVAL)?; + if !user_range.is_superset(range.clone()) + || kernel_range.overlaps(range.clone()) + || vm.driver_range_overlaps(range.clone()) + { + return Err(EINVAL); + } + if op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND != 0 { + if op.offset != 0 + || op.flags != uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND + || op.handle != 0 + { + return Err(EINVAL); + } + batch.push_unmap(vm.prepare_user_unmap(op.addr, op.range)?)?; + updates.push(MappingUpdate::Unmap(range), GFP_KERNEL)?; + unmaps += 1; + continue; + } + vm.wait_for_user_map_admission()?; + if op.flags + & !(uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE + | uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE) + != 0 + { + return Err(EINVAL); + } + let read = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_READ != 0; + let write = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_WRITE != 0; + let prot = match (read, write) { + (true, true) => mmu::PROT_GPU_SHARED_RW, + (true, false) => mmu::PROT_GPU_SHARED_RO, + (false, true) => mmu::PROT_GPU_SHARED_WO, + (false, false) => return Err(EINVAL), + }; + let single_page = op.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_SINGLE_PAGE != 0; + let object = gem::Object::lookup_handle(file, op.handle)?; + let accessed = if single_page { + mmu::UAT_PGSZ as u64 + } else { + op.range + }; + if op.offset.checked_add(accessed).ok_or(EINVAL)? > u64::try_from(object.size())? { + return Err(EINVAL); + } + vm.validate_context_binding(op.addr, op.range, single_page)?; + batch.push_map(vm.prepare_bind_object( + &object, + op.addr, + op.range, + op.offset, + prot, + single_page, + )?)?; + updates.push( + MappingUpdate::Map { + object, + address: op.addr, + size: op.range, + offset: op.offset, + }, + GFP_KERNEL, + )?; + } + vm.reserve_deferred_user_unmaps(unmaps)?; + vm.commit_prepared_user_bind_batch(&mut batch)?; + for update in updates { + match update { + MappingUpdate::Map { + object, + address, + size, + offset, + } => { + vm.track_context_binding(object, address, size, offset)?; + } + MappingUpdate::Unmap(range) => vm.untrack_context_range(range), + } + } + vm.bo_deferred_cleanup(); + Ok(0) + } + pub(crate) fn do_gem_bind_unbind( vm_id: usize, data: &uapi::drm_asahi_gem_bind_op, @@ -1156,6 +1369,21 @@ impl File { data.queue_id, id ); + let mut vec = KVec::new(); + let copy_commands = |bytes: &mut KVec| { + UserSlice::new( + UserPtr::from_addr(data.cmdbuf as _), + data.cmdbuf_size as usize, + ) + .reader() + .read_all(bytes, GFP_KERNEL) + }; + let guarded = gpu.as_any().is::(); + // Once G17 output syncobjs are resolved, its queue must install a fence + // on every failure. Take the fallible userspace snapshot before that point. + if guarded { + copy_commands(&mut vec)?; + } let syncs = SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; @@ -1167,17 +1395,12 @@ impl File { id ); - let mut vec = KVec::new(); - // Copy the command buffer into the kernel. Because we need to iterate // the command buffer twice, we do this in one big copy_from_user to // avoid TOCTOU issues. - let reader = UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), - data.cmdbuf_size as usize, - ) - .reader(); - reader.read_all(&mut vec, GFP_KERNEL)?; + if !guarded { + copy_commands(&mut vec)?; + } let objects = file.inner().objects(); let ret = queue diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs new file mode 100644 index 00000000000000..942d5047e5b44f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer IDs, scene ownership and sparse tiler heap growth. +//! +//! The device mutex serializes the shared ID allocator. A render graph owns +//! its buffer ID independently of the submission references using that graph; +//! a published command keeps its reference until firmware retirement is witnessed. + +use super::fw::buffer as fw; +use crate::hw::t8140::resources as cfg; +pub(crate) mod target; + +use kernel::prelude::*; + +const BUFFER_IDS: usize = 127; + +#[derive(Copy, Clone)] +struct BufferId { + references: u8, + owner: bool, + token: u64, +} +impl BufferId { + fn free(self) -> bool { + self.references == 0 && !self.owner + } +} + +/// Device-wide identities for retained render parameter buffers. +pub(crate) struct BufferIds { + entries: [BufferId; BUFFER_IDS], + returned: [u8; BUFFER_IDS], + returned_count: usize, + next_token: u64, +} +impl BufferIds { + pub(crate) fn new() -> Self { + let mut returned = [0; BUFFER_IDS]; + for (index, entry) in returned.iter_mut().enumerate() { + *entry = (BUFFER_IDS - index - 1) as u8; + } + Self { + entries: [BufferId { + references: 0, + owner: false, + token: 0, + }; BUFFER_IDS], + returned, + returned_count: BUFFER_IDS, + next_token: 0, + } + } + fn remove_returned(&mut self, id: u8) { + if let Some(at) = self.returned[..self.returned_count] + .iter() + .position(|entry| *entry == id) + { + self.returned.copy_within(at + 1..self.returned_count, at); + self.returned_count -= 1; + } + } + fn return_id(&mut self, id: u8) { + if !self.returned[..self.returned_count].contains(&id) && self.returned_count < BUFFER_IDS { + self.returned[self.returned_count] = id; + self.returned_count += 1; + } + } + fn fresh_token(&mut self) -> u64 { + let token = if self.next_token == u64::MAX { + 0 + } else { + self.next_token + }; + self.next_token = token + 1; + token + } + pub(crate) fn allocate_render(&mut self) -> Result<(u8, u64)> { + let id = if self.returned_count != 0 { + self.returned_count -= 1; + self.returned[self.returned_count] + } else { + self.entries + .iter() + .position(|entry| entry.free()) + .ok_or(ENOSPC)? as u8 + }; + if !self.entries[usize::from(id)].free() { + return Err(EIO); + } + let token = self.fresh_token(); + self.entries[usize::from(id)] = BufferId { + references: 1, + owner: false, + token, + }; + Ok((id, token)) + } + + /// Retains the existing identity for a graph independently of its jobs. + pub(crate) fn reserve_owner(&mut self, id: u8) -> Result { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if entry.owner { + return Ok(()); + } + if entry.free() { + let token = self.fresh_token(); + self.entries[usize::from(id)].token = token; + } + self.entries[usize::from(id)].owner = true; + self.remove_returned(id); + Ok(()) + } + pub(crate) fn acquire_owner(&mut self, id: u8) -> Result<(u8, u64)> { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Err(EINVAL); + } + entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; + Ok((id, entry.token)) + } + /// Preflight shared completion bookkeeping before any of its counters change. + pub(crate) fn can_release(&self, id: u8) -> Result { + let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; + if entry.references == 0 { + return Err(EINVAL); + } + Ok(()) + } + pub(crate) fn release(&mut self, id: u8) -> Result { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + entry.references = entry.references.checked_sub(1).ok_or(EINVAL)?; + if entry.free() { + self.return_id(id); + } + Ok(()) + } + pub(crate) fn unreserve_owner(&mut self, id: u8) -> Result { + let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; + if !entry.owner { + return Ok(()); + } + entry.owner = false; + if entry.free() { + self.return_id(id); + } + Ok(()) + } +} + +/// Each scene and its shorter scratch slot are one lifetime. A completed +/// paired render releases both; merely publishing a later scene does not. +pub(crate) struct Scenes { + owners: [Option<(u64, u8)>; fw::SCRATCH_SLOTS], + next: usize, +} +impl Scenes { + pub(crate) const fn new() -> Self { + Self { + owners: [None; fw::SCRATCH_SLOTS], + next: fw::FIRST_SCENE, + } + } + pub(crate) fn select(&self, ordinal: u64) -> Result { + if let Some((_, scene)) = self + .owners + .iter() + .flatten() + .find(|(owner, _)| *owner == ordinal) + { + return Ok(usize::from(*scene)); + } + (0..fw::SCENES) + .map(|offset| (self.next + offset) % fw::SCENES) + .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) + .ok_or(ENOSPC) + } + pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { + if self.select(ordinal)? != scene { + return Err(EINVAL); + } + let slot = &mut self.owners[scene % fw::SCRATCH_SLOTS]; + if slot.is_none() { + *slot = Some((ordinal, scene as u8)); + self.next = (scene + 1) % fw::SCENES; + } + Ok(()) + } + pub(crate) fn release(&mut self, ordinal: u64) -> Result { + let slot = self + .owners + .iter_mut() + .find(|entry| entry.is_some_and(|(owner, _)| owner == ordinal)) + .ok_or(EINVAL)?; + *slot = None; + Ok(()) + } +} + +/// A snapshot of the block ring; all values are modulo its fixed capacity. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) total: u32, + pub(crate) committed: u32, + pub(crate) read: u32, +} +impl Cursors { + fn valid(self) -> bool { + [self.total, self.committed, self.read] + .into_iter() + .all(|v| (v as usize) < cfg::TVB_BLOCK_SLOTS) + } + /// Firmware-paused growth permits consumption of the already committed + /// prefix. Retired-host growth requires the complete snapshot unchanged. + pub(crate) fn permits(self, after: Self, firmware_request: bool) -> bool { + if !self.valid() + || !after.valid() + || self.total != after.total + || self.committed != after.committed + { + return false; + } + if !firmware_request { + return self == after; + } + let capacity = cfg::TVB_BLOCK_SLOTS as u32; + (after.read + capacity - self.read) % capacity + <= (self.committed + capacity - self.read) % capacity + } +} + +/// Checked block-table indices for one bulk allocation. One ring slot stays +/// empty even when all physical backing has been retained by the owner. +pub(crate) struct Growth { + before: Cursors, + added: usize, + retained: usize, + target: usize, +} +impl Growth { + pub(crate) fn new(before: Cursors, added: usize) -> Result { + if !before.valid() || before.total != before.committed || added == 0 { + return Err(EINVAL); + } + let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) + % cfg::TVB_BLOCK_SLOTS; + if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { + return Err(ENOSPC); + } + let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; + Ok(Self { + before, + added, + retained, + target, + }) + } + pub(crate) fn target(&self) -> u32 { + self.target as u32 + } + pub(crate) fn retained(&self) -> usize { + self.retained + } + pub(crate) fn available(&self) -> usize { + self.retained + self.added + } + pub(crate) fn added_slot(&self, index: usize) -> Result { + if index >= self.added { + return Err(EINVAL); + } + Ok((self.before.total as usize + index) % cfg::TVB_BLOCK_SLOTS) + } + pub(crate) fn available_slot(&self, index: usize) -> Result { + if index >= self.available() { + return Err(EINVAL); + } + Ok((self.before.read as usize + index) % cfg::TVB_BLOCK_SLOTS) + } +} + +pub(crate) fn firmware_growth_target(current: usize) -> usize { + current + .saturating_add(fw::GROW_BLOCKS) + .min(cfg::TVB_MAX_BLOCKS) +} + +/// The high-water sample estimates capacity, not current occupancy. A growth +/// reserves three times the demand only once twice that demand no longer fits. +pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Option { + let maximum = fw::NORMAL_MAX_BLOCKS; + if current >= maximum { + return None; + } + let need = used_pages + .saturating_mul(2) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .min(maximum); + if need <= current { + return None; + } + Some( + used_pages + .saturating_mul(3) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK) + .min(maximum), + ) +} + +use super::object::{CpuMap, KernelObject}; +use crate::{driver::AsahiDevice, mmu}; +use core::ops::Range; + +pub(crate) fn compact_alias( + object: &KernelObject, + vm: &mmu::Vm, + source: Range, + align: u64, + prot: mmu::Prot, + guard: usize, +) -> Result { + for range in cfg::compact_ranges() { + match object.map_range(vm, source.clone(), range, align, prot, guard) { + Ok(mapping) => return Ok(mapping), + Err(error) if error == ENOSPC => {} + Err(error) => return Err(error), + } + } + Err(ENOSPC) +} + +/// Allocated aliases remain private until the owner installs them in its VM +/// cache. On rollback mappings are destroyed before backing memory. +pub(crate) struct TvbGrowth { + mappings: KVec, + addresses: KVec, + extension: KernelObject, +} + +/// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their +/// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. +pub(crate) struct Tvb { + addresses: KVec, + // Retain sparse-block backing until the pool and its aliases retire. + _primary: KernelObject, + extensions: KVec, +} +impl Tvb { + fn bytes(blocks: usize) -> Result { + blocks + .checked_sub(1) + .and_then(|v| v.checked_mul(cfg::TVB_BLOCK_STRIDE as usize)) + .and_then(|v| v.checked_add(cfg::TVB_BLOCK_SIZE)) + .ok_or(EOVERFLOW) + } + fn map_block( + object: &KernelObject, + vm: &mmu::Vm, + relative: usize, + ) -> Result { + let start = relative + .checked_mul(cfg::TVB_BLOCK_STRIDE as usize) + .ok_or(EOVERFLOW)?; + let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; + let mapping = compact_alias( + object, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_SHARED_RW, + cfg::TVB_BLOCK_STRIDE as usize - cfg::TVB_BLOCK_SIZE, + )?; + fw::compact_page(mapping.iova())?; + Ok(mapping) + } + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + blocks: usize, + ) -> Result<(Self, KVec)> { + if blocks == 0 || blocks > cfg::TVB_MAX_BLOCKS { + return Err(ERANGE); + } + let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; + let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; + let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; + for index in 0..blocks { + let mapping = Self::map_block(&backing, vm, index)?; + addresses.push(mapping.iova(), GFP_KERNEL)?; + mappings.push(mapping, GFP_KERNEL)?; + } + Ok(( + Self { + addresses, + _primary: backing, + extensions: KVec::new(), + }, + mappings, + )) + } + pub(crate) fn blocks(&self) -> usize { + self.addresses.len() + } + pub(crate) fn addresses(&self) -> &[u64] { + &self.addresses + } + + pub(crate) fn prepare_growth( + &mut self, + dev: &AsahiDevice, + vm: &mmu::Vm, + target: usize, + ) -> Result> { + if target <= self.blocks() { + return Ok(None); + } + if target > cfg::TVB_MAX_BLOCKS { + return Err(ERANGE); + } + let added = target - self.blocks(); + self.extensions.reserve(1, GFP_KERNEL)?; + self.addresses.reserve(added, GFP_KERNEL)?; + let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; + let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; + let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; + for relative in 0..added { + let mapping = Self::map_block(&backing, vm, relative)?; + addresses.push(mapping.iova(), GFP_KERNEL)?; + mappings.push(mapping, GFP_KERNEL)?; + } + Ok(Some(TvbGrowth { + mappings, + addresses, + extension: backing, + })) + } +} + +use super::object::Allocator; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::sync::Arc; + +/// First-free metrics entries are shared by all parameter-buffer graphs. +/// An entry returns only when the graph and its firmware references retire. +pub(crate) struct MetricsIds { + used: [AtomicU64; 2], +} +impl MetricsIds { + pub(crate) fn new() -> Self { + Self { + used: [ + AtomicU64::new(0), + AtomicU64::new(!((1u64 << (fw::METRICS_SLOTS - 64)) - 1)), + ], + } + } +} + +pub(crate) struct MetricsLease { + _backing: Arc, + allocator: Arc, + slot: usize, + addresses: [u64; 2], +} +impl MetricsLease { + /// Clears only the selected retired metrics entry before reusing it. + pub(crate) fn new( + ids: &Arc, + object: &Arc, + bases: [u64; 2], + ) -> Result { + let mut selected = None; + for (bank, used) in ids.used.iter().enumerate() { + let mut value = used.load(Ordering::Acquire); + loop { + let bit = value.trailing_ones(); + if bit == 64 { + break; + } + match used.compare_exchange_weak( + value, + value | (1u64 << bit), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + selected = Some(bank * 64 + bit as usize); + break; + } + Err(current) => value = current, + } + } + if selected.is_some() { + break; + } + } + let slot = selected.ok_or(ENOSPC)?; + // Construct the lease before any fallible address/view validation so + // every constructor failure returns the selected bit. + let mut lease = Self { + _backing: object.clone(), + allocator: ids.clone(), + slot, + addresses: [0; 2], + }; + let offset = slot * fw::METRICS_STRIDE; + lease.addresses = [ + bases[0].checked_add(offset as u64).ok_or(EOVERFLOW)?, + bases[1].checked_add(offset as u64).ok_or(EOVERFLOW)?, + ]; + let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); + object.word(offset)?; + for index in 0..fw::SCENES { + // SAFETY: The complete naturally aligned metrics entry is inside + // the retained object; the exclusive lease excludes other owners. + unsafe { words.add(index).write_volatile(0) }; + } + Ok(lease) + } +} +impl Drop for MetricsLease { + fn drop(&mut self) { + self.allocator.used[self.slot / 64] + .fetch_and(!(1u64 << (self.slot % 64)), Ordering::Release); + } +} + +pub(crate) struct ManagerArgs { + pub(crate) scratch: u64, + pub(crate) discard: u64, + pub(crate) buffer_id: u8, + pub(crate) generation: u64, + pub(crate) first_ordinal: u64, +} + +/// A retained parameter-buffer graph and all of its sparse backing. Client +/// mappings live in the VM's owner cache; graph replacement never returns a +/// published backing early merely because an error fence was signalled. +pub(crate) struct Manager { + _page_list: mmu::KernelMapping, + graph: KernelObject, + tvb: Tvb, + metrics: MetricsLease, + scenes: Scenes, + page_list_client: u64, + scratch: u64, + buffer_id: u8, + generation: u64, +} + +pub(crate) struct Submitted<'a> { + word: &'a AtomicU32, + next: u32, +} +impl Submitted<'_> { + pub(crate) fn commit(self) { + self.word.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + } +} + +impl Manager { + pub(crate) fn new( + alloc: &Allocator<'_>, + vm: &mmu::Vm, + tvb: Tvb, + metrics: MetricsLease, + args: ManagerArgs, + ) -> Result { + let mut graph = alloc.kernel( + fw::SIZE, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let source = fw::PAGE_LIST..fw::PAGE_LIST + fw::PAGE_LIST_BYTES; + let page_list = compact_alias( + &graph, + vm, + source, + cfg::TVB_PAGE, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )?; + let address = graph.gpu_va(); + graph.initialize::(0, |view| { + view.init(&fw::GraphArgs { + fw_va: address, + page_list_client: page_list.iova(), + metrics: metrics.addresses, + blocks: tvb.addresses(), + scratch: args.scratch, + discard: args.discard, + buffer_id: args.buffer_id.into(), + generation: args.generation, + }) + })?; + fence(Ordering::SeqCst); + let mut scenes = Scenes::new(); + scenes.commit(args.first_ordinal, fw::FIRST_SCENE)?; + let page_list_client = page_list.iova(); + Ok(Self { + _page_list: page_list, + graph, + tvb, + metrics, + scenes, + page_list_client, + scratch: args.scratch, + buffer_id: args.buffer_id, + generation: args.generation, + }) + } + pub(crate) fn buffer_id(&self) -> u8 { + self.buffer_id + } + pub(crate) fn blocks(&self) -> usize { + self.tvb.blocks() + } + pub(crate) fn state_va(&self) -> u64 { + self.graph.gpu_va() + } + pub(crate) fn page_list_va(&self) -> u64 { + self.page_list_client + } + pub(crate) fn trailer_va(&self) -> u64 { + self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (fw::SCENES * size_of::()) as u64 + } + pub(crate) fn scene_va(&self, index: usize) -> Result { + if index >= fw::SCENES { + return Err(EINVAL); + } + Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) + } + pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { + fw::scene_registers(index, self.scratch, self.metrics.addresses[1]) + } + pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { + let scene = self.scenes.select(ordinal)?; + let offset = fw::SCENE_TABLE + scene * size_of::(); + let used = self + .graph + .word(offset + core::mem::offset_of!(fw::Scene, used))?; + let generation = self + .graph + .pointer(offset + core::mem::offset_of!(fw::Scene, generation), 8)?; + let id = self + .graph + .word(core::mem::offset_of!(fw::State, buffer_id))?; + used.store(0, Ordering::Relaxed); + // SAFETY: The complete packed field was checked before any write. + // This scene is exclusively retained or still unclaimed. + unsafe { generation.cast::().write_unaligned(self.generation) }; + id.store(u32::from(self.buffer_id), Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(scene) + } + pub(crate) fn claim_scene(&mut self, ordinal: u64, scene: usize) -> Result { + self.scenes.commit(ordinal, scene) + } + pub(crate) fn retire_scene(&mut self, ordinal: u64) -> Result { + self.scenes.release(ordinal) + } + pub(crate) fn prepare_submission(&mut self) -> Result> { + let word = self.graph.word(fw::COUNTER)?; + Ok(Submitted { + word, + next: word.load(Ordering::Relaxed).wrapping_add(1), + }) + } + pub(crate) fn automatic_target(&self) -> Result> { + let max_pages = self + .graph + .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; + let reset = self + .graph + .word(fw::CONTROL + core::mem::offset_of!(fw::Control, reset))?; + fence(Ordering::Acquire); + let pages = max_pages.load(Ordering::Relaxed); + fence(Ordering::Release); + reset.store(1, Ordering::Relaxed); + Ok(automatic_growth_target(pages as usize, self.blocks())) + } + fn cursors(&self) -> Result { + Ok(Cursors { + total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), + committed: self.graph.word(fw::CONTROL + 4)?.load(Ordering::Relaxed), + read: self.graph.word(fw::CONTROL + 8)?.load(Ordering::Relaxed), + }) + } +} + +/// All allocations and inventory words prepared without publishing shared state. +/// The pair remains exclusively borrowed until commit or discard. +pub(crate) struct PreparedGrowth { + owner: u64, + previous: usize, + target: usize, + before: Cursors, + plan: Growth, + growth: TvbGrowth, + added: KVec<(usize, u32)>, + rebuilt: KVec, + pages: u32, + descriptor: Option<[u32; 4]>, +} +impl Manager { + /// Validate every retained sparse block and its reserved trailing guard in + /// one VM snapshot before constructing more aliases. + pub(crate) fn validate_vm(&self, vm: &mmu::Vm) -> Result { + if self.blocks() < cfg::TVB_INITIAL_PAGES.len() + || self.blocks() > cfg::TVB_MAX_BLOCKS + || !vm.covers_sparse_blocks_with_guards( + self.tvb.addresses(), + cfg::TVB_BLOCK_SIZE as u64, + cfg::TVB_BLOCK_STRIDE, + ) + { + return Err(EFAULT); + } + Ok(()) + } + + /// Allocate growth and precompute its words while the pair is exclusively + /// borrowed. A descriptor snapshot requests idle-host growth; None leaves + /// the firmware-owned page list and PB state untouched. + pub(crate) fn prepare_growth( + &mut self, + dev: &AsahiDevice, + vm: &mmu::Vm, + target: usize, + descriptor: Option<[u32; 4]>, + ) -> Result> { + let previous = self.blocks(); + if target <= previous { + return Ok(None); + } + self.validate_vm(vm)?; + let Some(growth) = self.tvb.prepare_growth(dev, vm, target)? else { + return Ok(None); + }; + let before = self.cursors()?; + let plan = Growth::new(before, target - previous)?; + let firmware_request = descriptor.is_none(); + let pages = target + .checked_mul(cfg::TVB_PAGES_PER_BLOCK) + .ok_or(EOVERFLOW)? as u32; + let mut added = KVec::with_capacity(growth.addresses.len(), GFP_KERNEL)?; + for (index, &address) in growth.addresses.iter().enumerate() { + let first = fw::compact_page(address)?; + first + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + added.push((plan.added_slot(index)?, first), GFP_KERNEL)?; + } + let rebuilt_count = if firmware_request { + 0 + } else { + plan.available() * cfg::TVB_PAGES_PER_BLOCK + }; + let mut rebuilt = KVec::with_capacity(rebuilt_count, GFP_KERNEL)?; + if !firmware_request { + if self.cursors()? != before { + return Err(EAGAIN); + } + for ordinal in 0..plan.available() { + let first = if ordinal < plan.retained() { + let slot = plan.available_slot(ordinal)?; + self.graph + .word(fw::BLOCK_TABLE + slot * size_of::())? + .load(Ordering::Relaxed) + } else { + added[ordinal - plan.retained()].1 + }; + first + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + rebuilt.push(first + page as u32, GFP_KERNEL)?; + } + } + } + let descriptor = + descriptor.map(|old| fw::reload_descriptor(old, self.page_list_client, pages)); + Ok(Some(PreparedGrowth { + owner: self.graph.gpu_va(), + previous, + target, + before, + plan, + growth, + added, + rebuilt, + pages, + descriptor, + })) + } + + /// The caller holds the device mutex after revalidating its preparation + /// epoch. The final alias installation and all firmware writes occur under + /// that same exclusion. The returned PB row is committed there as well. + pub(crate) fn commit_growth( + &mut self, + prepared: &mut Option, + install: impl FnOnce(usize, usize, &mut KVec) -> Result, + ) -> Result> { + let candidate = prepared.as_mut().ok_or(EINVAL)?; + if self.graph.gpu_va() != candidate.owner || self.blocks() != candidate.previous { + return Err(EBUSY); + } + let firmware_request = candidate.descriptor.is_none(); + let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + // Bounds, actual alignment and every reserved host entry are checked + // before transferring aliases or consuming the caller-owned plan. + self.graph.word(0)?; + if self.tvb.extensions.len() == self.tvb.extensions.capacity() + || candidate.growth.addresses.len() + > self.tvb.addresses.capacity() - self.tvb.addresses.len() + { + return Err(EIO); + } + if !candidate.before.permits(self.cursors()?, firmware_request) { + return Err(EAGAIN); + } + // The callback retains its input on error. The caller can therefore + // drop every rejected mapping/backing after releasing the device lock. + install( + candidate.previous, + candidate.target, + &mut candidate.growth.mappings, + )?; + if !candidate.growth.mappings.is_empty() { + return Err(EIO); + } + let PreparedGrowth { + before, + plan, + growth, + added, + rebuilt, + pages, + descriptor, + .. + } = prepared.take().ok_or(EIO)?; + let TvbGrowth { + addresses, + extension, + .. + } = growth; + + // From here all vector capacity, indices, addresses and CPU views have + // been checked. Retain backing before publishing its block entries. + for address in addresses.into_iter() { + append_reserved(&mut self.tvb.addresses, address); + } + let write = |offset: usize, value: u32| { + // SAFETY: The full graph and base alignment were checked above. + // Every offset is an aligned field or a checked array index of + // Graph. No mutable reference aliases firmware-owned fields. + unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; + }; + for (slot, first) in added.into_iter() { + let at = fw::BLOCK_TABLE + slot * size_of::(); + write(at, first); + write(at + 4, 0); + } + if descriptor.is_some() { + for (entry, value) in rebuilt.into_iter().enumerate() { + write(fw::PAGE_LIST + entry * 4, value); + } + write(core::mem::offset_of!(fw::State, pages), pages); + write(core::mem::offset_of!(fw::State, producer), plan.target()); + write(core::mem::offset_of!(fw::State, consumer), before.read); + write(core::mem::offset_of!(fw::State, last_page), pages - 1); + } + append_reserved(&mut self.tvb.extensions, extension); + write(fw::CONTROL, plan.target()); + fence(Ordering::SeqCst); + write(fw::CONTROL + 4, plan.target()); + fence(Ordering::SeqCst); + Ok(descriptor) + } +} + +/// Caller reserves all needed entries before any firmware-visible mutation. +fn append_reserved(values: &mut KVec, value: T) { + let length = values.len(); + values.spare_capacity_mut()[0].write(value); + // SAFETY: All callers preflighted this spare entry and initialized it once. + unsafe { values.set_len(length + 1) }; +} diff --git a/drivers/gpu/drm/asahi/g17/buffer/target.rs b/drivers/gpu/drm/asahi/g17/buffer/target.rs new file mode 100644 index 00000000000000..07ead6056329e5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/buffer/target.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Rotating tile-map/heap metadata and the retained tail-pointer cache. +//! Larger targets replace only the objects that need more capacity. Every old +//! backing keeps the exact commands that still reference it until retirement. + +use super::compact_alias; +use crate::g17::{ + fw::render::RenderPass, + object::{CpuMap, KernelObject}, +}; +use crate::{ + driver::AsahiDevice, + hw::t8140::{queues, resources as cfg}, + mmu, +}; +use kernel::prelude::*; + +const DEPTH: usize = queues::RENDER_DEPTH as usize; + +fn aligned(value: usize, alignment: usize) -> Result { + value + .checked_add(alignment - 1) + .map(|v| v & !(alignment - 1)) + .ok_or(EOVERFLOW) +} + +/// Per-command geometry, independent of the capacity already allocated. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Layout { + pub(crate) layer_offset: usize, + pub(crate) heap_offset: usize, + pub(crate) used: usize, + pub(crate) tpc_size: usize, + pub(crate) tvb_blocks: usize, +} +impl Layout { + pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { + let width = pass.width as usize; + let height = pass.height as usize; + let layers = pass.layers as usize; + let (uw, uh) = (pass.utile_width as usize, pass.utile_height as usize); + if width == 0 + || height == 0 + || layers == 0 + || clusters == 0 + || uw == 0 + || uh == 0 + || 32 % uw != 0 + || 32 % uh != 0 + { + return Err(EINVAL); + } + let utiles = (32 / uw) * (32 / uh); + let tiles_x = width.div_ceil(32); + let tiles_y = height.div_ceil(32); + let macro_x = aligned(tiles_x.div_ceil(4), 4)?; + let macro_y = aligned(tiles_y.div_ceil(4), 4)?; + let macro_tiles = macro_x.checked_mul(macro_y).ok_or(EOVERFLOW)?; + let tilemap_size = aligned( + 5usize + .checked_mul(macro_tiles) + .and_then(|v| v.checked_mul(utiles)) + .ok_or(EOVERFLOW)?, + 4, + )? + .checked_mul(16) + .and_then(|v| v.checked_mul(layers)) + .ok_or(EOVERFLOW)?; + let tpc_size = 8usize + .checked_mul(utiles) + .and_then(|v| v.checked_mul(macro_tiles)) + .map(|v| v / 4 * 4) + .and_then(|v| v.checked_mul(16)) + .and_then(|v| v.checked_mul(layers)) + .and_then(|v| v.checked_mul(clusters as usize)) + .ok_or(EOVERFLOW)?; + let layer_offset = aligned(tilemap_size, 0x1000)?; + let heap_offset = layer_offset + .checked_add(if layers > 1 { 0x100 } else { 0 }) + .ok_or(EOVERFLOW)?; + let used = heap_offset.checked_add(0x200).ok_or(EOVERFLOW)?; + let tiles = tiles_x.checked_mul(tiles_y).ok_or(EOVERFLOW)?; + let mut tvb_blocks = aligned(tiles.div_ceil(128), 8)?.max(8); + if clusters > 1 { + tvb_blocks = tvb_blocks.max( + 7usize + .checked_add(2usize.checked_mul(layers).ok_or(EOVERFLOW)?) + .ok_or(EOVERFLOW)?, + ); + } + tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); + Ok(Self { + layer_offset, + heap_offset, + used, + tpc_size, + tvb_blocks, + }) + } + pub(crate) fn tpc_capacity(&self) -> Result { + Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) + } + pub(crate) fn record_stride(&self) -> Result { + Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) + } +} + +struct Object { + address: u64, + _mapping: mmu::KernelMapping, + backing: KernelObject, +} +impl Object { + fn new(dev: &AsahiDevice, vm: &mmu::Vm, size: usize) -> Result { + let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; + Ok(Self { + address: mapping.iova(), + _mapping: mapping, + backing, + }) + } +} +struct Retired { + _object: Object, + owners: [Option; DEPTH], +} + +#[derive(Copy, Clone, Debug)] +pub(crate) struct Addresses { + pub(crate) tilemap: u64, + pub(crate) layer: u64, + pub(crate) heap: u64, +} + +pub(crate) struct Target { + records: Object, + tpc: Object, + stride: usize, + slots: [Option; DEPTH], + retired: KVec, +} +impl Target { + /// Keep this allocation frame separate from the rest of the render graph. + #[inline(never)] + pub(crate) fn new(dev: &AsahiDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { + let tpc = layout.tpc_capacity()?; + let stride = layout.record_stride()?; + if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + let records = Object::new(dev, vm, stride * DEPTH)?; + let tpc = Object::new(dev, vm, tpc)?; + KBox::new( + Self { + records, + tpc, + stride, + slots: [None; DEPTH], + retired: KVec::new(), + }, + GFP_KERNEL, + ) + .map_err(Into::into) + } + pub(crate) fn tpc_va(&self) -> u64 { + self.tpc.address + } + + /// Only the span consumed by this render is cleared. A small render does + /// not clear a previous large target's complete retained record capacity. + pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { + let slot = ordinal as usize % DEPTH; + if self.slots[slot].is_some_and(|owner| owner != ordinal) { + return Err(EBUSY); + } + if layout.used > self.stride { + return Err(ERANGE); + } + let offset = slot * self.stride; + let span = aligned(layout.used, mmu::UAT_PGSZ)?.min(self.stride); + self.records + .backing + .bytes_mut() + .get_mut(offset..offset + span) + .ok_or(ERANGE)? + .fill(0); + self.slots[slot] = Some(ordinal); + let tilemap = self + .records + .address + .checked_add(offset as u64) + .ok_or(EOVERFLOW)?; + Ok(Addresses { + tilemap, + layer: tilemap + .checked_add(layout.layer_offset as u64) + .ok_or(EOVERFLOW)?, + heap: tilemap + .checked_add(layout.heap_offset as u64) + .ok_or(EOVERFLOW)?, + }) + } + + /// Growth never shrinks a live allocation. A successfully replaced TPC + /// stays enlarged even if allocating the following record object fails. + pub(crate) fn ensure_capacity( + &mut self, + dev: &AsahiDevice, + vm: &mmu::Vm, + layout: &Layout, + owners: &[Option; DEPTH], + ) -> Result { + let tpc = layout.tpc_capacity()?; + let stride = layout.record_stride()?; + let mut grew = false; + if tpc > self.tpc.backing.size() { + if tpc > cfg::TPC_MAX { + return Err(ERANGE); + } + Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; + grew = true; + } + if stride > self.stride { + if stride > cfg::TARGET_RECORD_MAX { + return Err(ERANGE); + } + Self::replace( + &mut self.retired, + &mut self.records, + dev, + vm, + stride * DEPTH, + owners, + )?; + self.stride = stride; + self.slots = [None; DEPTH]; + grew = true; + } + Ok(grew) + } + fn replace( + retired: &mut KVec, + old: &mut Object, + dev: &AsahiDevice, + vm: &mmu::Vm, + size: usize, + owners: &[Option; DEPTH], + ) -> Result { + let retain = owners.iter().any(Option::is_some); + if retain { + retired.reserve(1, GFP_KERNEL)?; + } + let new = Object::new(dev, vm, size)?; + let previous = core::mem::replace(old, new); + if retain { + super::append_reserved( + retired, + Retired { + _object: previous, + owners: *owners, + }, + ); + } + Ok(()) + } + pub(crate) fn retire(&mut self, ordinal: u64) -> Result { + let slot = &mut self.slots[ordinal as usize % DEPTH]; + if *slot == Some(ordinal) { + *slot = None; + } + let mut index = 0; + while index < self.retired.len() { + for owner in &mut self.retired[index].owners { + if *owner == Some(ordinal) { + *owner = None; + } + } + if self.retired[index].owners.iter().all(Option::is_none) { + drop(self.retired.remove(index)?); + } else { + index += 1; + } + } + Ok(()) + } + /// Cancels only an unpublished record; older referenced backing remains + /// owned until normal paired retirement or the firmware-stop boundary. + pub(crate) fn cancel(&mut self, ordinal: u64) { + let slot = &mut self.slots[ordinal as usize % DEPTH]; + if *slot == Some(ordinal) { + *slot = None; + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs new file mode 100644 index 00000000000000..c0609f148106e7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-produced rings and the device-control producer. +//! +//! The device mutex serializes every caller. Entries are copied before a +//! single aligned consumer store returns their slots to the firmware. Trace +//! and statistics transport errors quarantine only that ring; event errors +//! propagate to the device owner. Device-control publication reserves one +//! empty slot and orders the record, power tally and producer together. + +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +use crate::driver::AsahiDevice; + +use super::fw::{channels::*, initdata::status}; +use super::initdata::InitData; +use super::Role; + +/// Firmware rings whose contents do not affect host scheduling. +#[derive(Copy, Clone, Debug)] +enum ReportRing { + Trace = 0, + Statistics = 1, +} + +impl ReportRing { + fn geometry(self) -> (usize, usize, u32, u32) { + match self { + Self::Trace => ( + status::TRACE_STATE as usize, + status::TRACE_RING as usize, + TRACE_SLOTS, + TRACE_TYPES, + ), + Self::Statistics => ( + status::STATS_STATE as usize, + status::STATS_RING as usize, + STATISTICS_SLOTS, + STATISTICS_TYPES, + ), + } + } +} + +/// Host-side receive-ring state. A quarantined ring stays stopped until the +/// device is destroyed; firmware restart does not clear it. +#[derive(Debug)] +pub(crate) struct Rings { + quarantined: [[bool; 2]; 2], +} + +impl Rings { + pub(crate) const fn new() -> Self { + Self { + quarantined: [[false; 2]; 2], + } + } + + pub(crate) fn drain_trace(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + self.drain_report(dev, init, role, ReportRing::Trace); + } + + pub(crate) fn drain_statistics(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + self.drain_report(dev, init, role, ReportRing::Statistics); + } + + fn drain_report(&mut self, dev: &AsahiDevice, init: &InitData, role: Role, ring: ReportRing) { + let quarantined = &mut self.quarantined[role as usize][ring as usize]; + if *quarantined { + return; + } + if let Err(error) = Self::consume_report(init, role, ring) { + *quarantined = true; + dev_err!( + dev.as_ref(), + "G17 {:?} {:?} ring stopped: {:?}\n", + role, + ring, + error + ); + } + } + + fn consume_report(init: &InitData, role: Role, ring: ReportRing) -> Result { + let (state, entries, capacity, allowed_types) = ring.geometry(); + let consumer_word = init.status_word(role, state)?; + let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; + fence(Ordering::Acquire); + let mut consumer = consumer_word.load(Ordering::Relaxed); + let producer = producer_word.load(Ordering::Relaxed); + // The producer makes the entries it covers available to the host. + fence(Ordering::Acquire); + if consumer >= capacity || producer >= capacity { + return Err(EIO); + } + while consumer != producer { + let offset = entries + consumer as usize * EVENT_RECORD_SIZE; + let tag = match ring { + // Copy the entire trace entry before returning its slot. + ReportRing::Trace => init.status_read_words::(role, offset)?[0], + ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], + }; + if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { + return Err(EIO); + } + consumer = (consumer + 1) & (capacity - 1); + fence(Ordering::SeqCst); + consumer_word.store(consumer, Ordering::Relaxed); + if matches!(ring, ReportRing::Trace) { + fence(Ordering::SeqCst); + } + } + if matches!(ring, ReportRing::Statistics) { + fence(Ordering::SeqCst); + } + Ok(()) + } + + /// Publishes one primary device-control record. On EAGAIN the caller + /// retains the request and retries before consuming more dependent events. + /// The returned cursor is the value the consumer must reach to retire it. + /// The caller announces the publication through the coprocessor mailbox. + pub(crate) fn publish_control(init: &InitData, record: &ControlRecord) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + if consumer >= DEVICE_CONTROL_SLOTS || producer >= DEVICE_CONTROL_SLOTS { + return Err(EIO); + } + let next = (producer + 1) & (DEVICE_CONTROL_SLOTS - 1); + if next == consumer { + return Err(EAGAIN); + } + // InitData validates every mapping before the first write, then writes + // record, packed tally, barrier, producer, barrier without failure. + init.publish_control( + producer, + record.bytes(), + record.opcode().asserts_gpu_power(), + next, + )?; + Ok(next) + } +} + +/// One bounded event-ring pass. No shared-memory borrow survives `next`, so +/// the caller can service the copied event using the complete device owner. +pub(crate) struct Events { + role: Role, + remaining: u32, + unknown_reported: bool, +} + +impl Events { + /// A bounded pass can leave records written while its earlier slots were + /// consumed. Check both cursors again before the worker goes idle. + pub(crate) fn pending(init: &InitData, role: Role) -> Result { + let state = status::EVENT_STATE as usize; + fence(Ordering::Acquire); + let consumer = init.status_word(role, state)?.load(Ordering::Relaxed); + let producer = init + .status_word(role, state + RX_PRODUCER_OFFSET)? + .load(Ordering::Relaxed); + if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { + return Err(EIO); + } + Ok(consumer != producer) + } + + pub(crate) fn new(role: Role) -> Self { + Self { + role, + remaining: EVENT_SLOTS, + unknown_reported: false, + } + } + + pub(crate) fn next(&mut self, dev: &AsahiDevice, init: &InitData) -> Result> { + let role = self.role; + let state = status::EVENT_STATE as usize; + let consumer_word = init.status_word(role, state)?; + let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; + while self.remaining != 0 { + init.validate_event_ring(role)?; + self.remaining -= 1; + fence(Ordering::Acquire); + let consumer = consumer_word.load(Ordering::Relaxed); + let producer = producer_word.load(Ordering::Relaxed); + if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { + return Err(EIO); + } + if consumer == producer { + self.remaining = 0; + break; + } + let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; + let record = EventRecord::from_words(init.status_read_words(role, offset)?); + let event = record.decode(); + fence(Ordering::SeqCst); + consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); + fence(Ordering::SeqCst); + match event? { + Event::Unknown(tag) => { + if !self.unknown_reported { + dev_warn!( + dev.as_ref(), + "G17 {:?} unknown firmware event {}\n", + role, + tag + ); + self.unknown_reported = true; + } + } + event => return Ok(Some(event)), + } + } + Ok(None) + } +} diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs new file mode 100644 index 00000000000000..fef478310c23b2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Decode a submission and validate its user resources before publishing any work. + +use super::fw::{ + kick::WriteRange, + render::{RenderPass, UscProgram, ZlsSurface}, +}; +use crate::util::{AnyBitPattern, Reader}; +use core::ops::Range; +use kernel::{prelude::*, uapi}; + +const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; +const USC_WINDOW_SIZE: u64 = 1 << 32; +const MAX_SAMPLERS: u32 = 1024; +const SAMPLER_SIZE: u64 = 8; +const PAGE_SIZE: u64 = 0x4000; +const ZLS_METADATA_STRIDE: u64 = 0x80; +const MAX_DIMENSION: u16 = 16384; +const MAX_LAYERS: u16 = 2048; +const MAX_TILE_BUFFER: u64 = 32768; +const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; +const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; +const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; +const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT; + +/// Attachment settings persist until replaced, including by an empty list. +#[derive(Clone, Copy)] +pub(crate) struct Attachments { + ranges: [WriteRange; MAX_ATTACHMENTS], + count: usize, +} + +impl Attachments { + const EMPTY: Self = Self { + ranges: [WriteRange { va: 0, size: 0 }; MAX_ATTACHMENTS], + count: 0, + }; + + pub(crate) fn as_slice(&self) -> &[WriteRange] { + &self.ranges[..self.count] + } + + fn parse(bytes: &[u8]) -> Result { + let size = size_of::(); + if bytes.len() % size != 0 || bytes.len() / size > MAX_ATTACHMENTS { + return Err(EINVAL); + } + let mut list = Self::EMPTY; + let mut reader = Reader::new(bytes); + for range in &mut list.ranges[..bytes.len() / size] { + let item: uapi::drm_asahi_attachment = reader.read()?; + if item.flags != 0 || item.pad != 0 { + return Err(EINVAL); + } + *range = WriteRange { + va: item.pointer, + size: item.size, + }; + list.count += 1; + } + Ok(list) + } + + fn validate(&self, space: &impl AddressSpace) -> Result { + for range in self.as_slice() { + require_range(space, range.va, range.size, Access::Write)?; + } + Ok(()) + } +} + +// Commands stay inline in the submission plan, avoiding an allocation per render pass. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Payload { + Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), + Compute(uapi::drm_asahi_cmd_compute, Attachments), +} + +pub(crate) struct Command { + /// Render and compute prefix indices, before this command. + pub(crate) barriers: [u16; 2], + pub(crate) payload: Payload, +} + +/// Parser over the single userspace snapshot taken by the submit ioctl. +/// Call `finish` after consuming it to reject submissions with no hardware work. +pub(crate) struct Parser<'a> { + bytes: &'a [u8], + counts: [u16; 2], + attachments: [Attachments; 3], +} + +impl<'a> Parser<'a> { + pub(crate) fn new(bytes: &'a [u8]) -> Self { + Self { + bytes, + counts: [0; 2], + attachments: [Attachments::EMPTY; 3], + } + } + + pub(crate) fn next(&mut self) -> Result> { + while !self.bytes.is_empty() { + let mut reader = Reader::new(self.bytes); + let header: uapi::drm_asahi_cmd_header = reader.read()?; + let start = size_of::(); + let end = start + header.size as usize; + let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; + self.bytes = &self.bytes[end..]; + let barriers = [header.vdm_barrier, header.cdm_barrier]; + match header.cmd_type as u32 { + uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER + | uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { + if barriers.iter().zip(self.counts).any(|(&barrier, count)| { + barrier != uapi::DRM_ASAHI_BARRIER_NONE as u16 && barrier > count + }) || u32::from(self.counts[0]) + u32::from(self.counts[1]) + >= crate::file::MAX_COMMANDS_PER_SUBMISSION + { + return Err(EINVAL); + } + let payload = if header.cmd_type as u32 + == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER + { + self.counts[0] += 1; + Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) + } else { + self.counts[1] += 1; + Payload::Compute(payload(bytes)?, self.attachments[2]) + }; + return Ok(Some(Command { barriers, payload })); + } + uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS + | uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS => { + if barriers != [uapi::DRM_ASAHI_BARRIER_NONE as u16; 2] { + return Err(EINVAL); + } + let slot = header.cmd_type as usize + - uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS as usize; + self.attachments[slot] = Attachments::parse(bytes)?; + } + _ => return Err(EINVAL), + } + } + Ok(None) + } + + pub(crate) fn finish(&self) -> Result { + if !self.bytes.is_empty() || self.counts == [0; 2] { + return Err(EINVAL); + } + Ok(()) + } +} + +fn payload(bytes: &[u8]) -> Result { + if bytes + .get(size_of::()..) + .is_some_and(|tail| tail.iter().any(|&b| b != 0)) + { + return Err(EINVAL); + } + Reader::new(bytes).read_up_to(bytes.len()) +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Access { + Read, + Write, + ReadWrite, +} + +/// The caller retains the VM mappings while validating and executing the command. +pub(crate) trait AddressSpace { + fn covers(&self, address: u64, size: u64, access: Access) -> bool; +} + +fn require_range(space: &impl AddressSpace, address: u64, size: u64, access: Access) -> Result { + if size == 0 || address.checked_add(size).is_none() || !space.covers(address, size, access) { + return Err(EINVAL); + } + Ok(()) +} + +/// A validated four-GiB USC window owned by one queue. +pub(crate) struct UscWindow { + base: u64, +} +impl UscWindow { + pub(crate) fn new(base: u64, range: Range) -> Result { + if base & (USC_WINDOW_SIZE - 1) != 0 + || base < range.start + || base + .checked_add(USC_WINDOW_SIZE) + .is_none_or(|end| end > range.end) + { + return Err(EINVAL); + } + Ok(Self { base }) + } + + fn program(&self, space: &impl AddressSpace, tagged_offset: u32) -> Result { + if tagged_offset == 0 { + return Err(EINVAL); + } + require_range( + space, + self.base + (tagged_offset & !7) as u64, + 4, + Access::Read, + ) + } +} + +fn sampler(space: &impl AddressSpace, heap: u64, count: u32) -> Result { + if count > MAX_SAMPLERS || (heap == 0) != (count == 0) || heap & (SAMPLER_SIZE - 1) != 0 { + return Err(EINVAL); + } + if count != 0 { + require_range(space, heap, u64::from(count) * SAMPLER_SIZE, Access::Read)?; + } + Ok(()) +} + +fn helper_is_empty(helper: &uapi::drm_asahi_helper_program) -> bool { + helper.binary == 0 && helper.cfg == 0 && helper.data == 0 +} + +fn surface( + space: &impl AddressSpace, + zls: &uapi::drm_asahi_zls_buffer, + layers: u16, +) -> Result { + if (zls.base == 0 && (zls.comp_base != 0 || zls.stride != 0 || zls.comp_stride != 0)) + || (zls.comp_base == 0 && zls.comp_stride != 0) + || (layers > 1 && zls.base != 0 && zls.stride == 0) + || (zls.stride != 0 && u64::from(zls.stride) & (PAGE_SIZE - 1) != 1) + || u64::from(zls.comp_stride) & (PAGE_SIZE - 1) != 0 + { + return Err(EINVAL); + } + if zls.base != 0 { + let stride = if zls.stride == 0 { + 0 + } else { + (u64::from(zls.stride) / PAGE_SIZE + 1) * PAGE_SIZE + }; + require_range( + space, + zls.base, + stride * u64::from(layers - 1) + 1, + Access::ReadWrite, + )?; + } + if zls.comp_base != 0 { + let stride = (u64::from(zls.comp_stride) / PAGE_SIZE + 1) * ZLS_METADATA_STRIDE; + require_range( + space, + zls.comp_base, + stride * u64::from(layers - 1) + 1, + Access::ReadWrite, + )?; + } + Ok(ZlsSurface { + va: zls.base, + compression_va: zls.comp_base, + stride: zls.stride.into(), + compression_stride: zls.comp_stride.into(), + }) +} + +// As with Payload, retain the bounded attachment lists inside the submission plan. +#[allow(clippy::large_enum_variant)] +pub(crate) enum Validated { + Render { + pass: RenderPass, + attachments: [Attachments; 2], + timestamps: [uapi::drm_asahi_timestamps; 2], + }, + Compute { + usc_base: u64, + cdm_va: u64, + cdm_end_va: u64, + sampler_heap_va: u64, + sampler_count: u32, + timestamps: uapi::drm_asahi_timestamps, + }, +} + +impl Payload { + pub(crate) fn validate( + &self, + window: &UscWindow, + space: &impl AddressSpace, + ) -> Result { + match self { + Self::Render(cmd, attachments) => { + if cmd.flags & !RENDER_FLAGS != 0 + || !(1..=MAX_DIMENSION).contains(&cmd.width_px) + || !(1..=MAX_DIMENSION).contains(&cmd.height_px) + || !(1..=MAX_LAYERS).contains(&cmd.layers) + || !matches!( + (cmd.utile_width_px, cmd.utile_height_px), + (16, 16) | (32, 16) | (32, 32) + ) + { + return Err(EINVAL); + } + let samples_log2 = match cmd.samples { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err(EINVAL), + }; + if u64::from(cmd.sample_size_B) + * u64::from(cmd.utile_width_px) + * u64::from(cmd.utile_height_px) + * u64::from(cmd.samples) + > MAX_TILE_BUFFER + || cmd.vdm_ctrl_stream_base == 0 + || cmd.vdm_ctrl_stream_base & 3 != 0 + || cmd.isp_scissor_base == 0 + || cmd.isp_scissor_base & 7 != 0 + { + return Err(EINVAL); + } + require_range(space, cmd.vdm_ctrl_stream_base, 4, Access::Read)?; + require_range(space, cmd.isp_scissor_base, 8, Access::Read)?; + for (va, access) in [ + (cmd.isp_dbias_base, Access::Read), + (cmd.isp_oclqry_base, Access::Write), + ] { + if va != 0 { + if va & 7 != 0 { + return Err(EINVAL); + } + require_range(space, va, 8, access)?; + } + } + let depth = surface(space, &cmd.depth, cmd.layers)?; + let stencil = surface(space, &cmd.stencil, cmd.layers)?; + sampler(space, cmd.sampler_heap, cmd.sampler_count.into())?; + if !helper_is_empty(&cmd.vertex_helper) || !helper_is_empty(&cmd.fragment_helper) { + return Err(EINVAL); + } + for program in [&cmd.bg, &cmd.eot, &cmd.partial_bg, &cmd.partial_eot] { + window.program(space, program.usc)?; + } + for list in attachments { + list.validate(space)?; + } + let pass = RenderPass { + width: cmd.width_px.into(), + height: cmd.height_px.into(), + layers: cmd.layers, + utile_width: cmd.utile_width_px, + utile_height: cmd.utile_height_px, + samples_log2, + sample_size: cmd.sample_size_B, + process_empty_tiles: cmd.flags & PROCESS_EMPTY != 0, + msaa_control: cmd.ppp_multisamplectl, + ppp_control: cmd.ppp_ctrl.into(), + vdm_va: cmd.vdm_ctrl_stream_base, + scissor_va: cmd.isp_scissor_base, + depth_bias_va: cmd.isp_dbias_base, + depth_bias_is_int: cmd.flags & DBIAS_INT != 0, + occlusion_query_va: cmd.isp_oclqry_base, + sampler_heap_va: cmd.sampler_heap, + sampler_count: cmd.sampler_count.into(), + zls_control: cmd.zls_ctrl, + depth_dimensions: cmd.isp_zls_pixels.into(), + depth, + stencil, + depth_clear: cmd.isp_bgobjdepth, + stencil_clear: cmd.isp_bgobjvals, + merge_upper_x: cmd.isp_merge_upper_x, + merge_upper_y: cmd.isp_merge_upper_y, + usc_base: window.base, + bg: UscProgram { + offset: cmd.bg.usc, + rsrc_spec: u64::from(cmd.bg.rsrc_spec) + | (u64::from(cmd.bg_rsrc_spec_hi) << 32), + }, + partial_bg: UscProgram { + offset: cmd.partial_bg.usc, + rsrc_spec: u64::from(cmd.partial_bg.rsrc_spec) + | (u64::from(cmd.bg_partial_rsrc_spec_hi) << 32), + }, + eot_offset: cmd.eot.usc, + partial_eot_offset: cmd.partial_eot.usc, + rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, + }; + Ok(Validated::Render { + pass, + attachments: *attachments, + timestamps: [cmd.ts_vtx, cmd.ts_frag], + }) + } + Self::Compute(cmd, attachments) => { + if cmd.flags != 0 + || cmd.cdm_ctrl_stream_base & 3 != 0 + || cmd.cdm_ctrl_stream_end & 3 != 0 + || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base + || !helper_is_empty(&cmd.helper) + { + return Err(EINVAL); + } + require_range( + space, + cmd.cdm_ctrl_stream_base, + cmd.cdm_ctrl_stream_end - cmd.cdm_ctrl_stream_base, + Access::Read, + )?; + sampler(space, cmd.sampler_heap, cmd.sampler_count)?; + attachments.validate(space)?; + Ok(Validated::Compute { + usc_base: window.base, + cdm_va: cmd.cdm_ctrl_stream_base, + cdm_end_va: cmd.cdm_ctrl_stream_end - 4, + sampler_heap_va: cmd.sampler_heap, + sampler_count: cmd.sampler_count, + timestamps: cmd.ts, + }) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs new file mode 100644 index 00000000000000..3ac8fe711ca619 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Per-command fences, timestamp destinations and VM pins retained through retirement. +//! +//! Runtime queues own completions before making work visible. They defer fence signalling +//! until after the device mutex is released, and retain failed visible work until a retirement +//! witness or processor stop. An error fence never substitutes for that ownership boundary. + +use super::{ + context::{Context, WorkStateLease}, + fence::{self, CompletionFence, Member}, + object::{CpuMap, KernelObject}, + recovery::Class, + status::VmStatus, +}; +use crate::{driver::AsahiDevice, file, hw::t8140::resources as cfg, mmu}; +use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; +use kernel::{ + dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + sync::{Arc, Mutex}, + uapi, xarray, +}; + +struct Destination { + mapping: Arc, + offset: usize, +} + +impl Destination { + fn resolve( + objects: Pin<&xarray::XArray>>, + timestamp: uapi::drm_asahi_timestamp, + ) -> Result> { + if timestamp.handle == 0 { + return Ok(None); + } + let object = objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset: usize = timestamp.offset.try_into()?; + if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { + return Err(ERANGE); + } + Ok(Some(Self { mapping, offset })) + } + + fn write(&self, ticks: u64) -> Result { + const NSEC_PER_SEC: u64 = 1_000_000_000; + let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) + .wrapping_mul(NSEC_PER_SEC) + .wrapping_add( + ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, + ); + self.mapping.write_timestamp(self.offset, ns)?; + barrier(Ordering::Release); + Ok(()) + } +} + +/// Firmware writes two aligned words in a fresh write-combined page per compute command. +struct ComputeTimestamps { + // Remove the GPU alias before releasing its backing. + mapping: mmu::KernelMapping, + object: KernelObject, +} + +impl ComputeTimestamps { + fn new(dev: &AsahiDevice, vm: &mmu::Vm) -> Result { + let object = KernelObject::backing(dev, mmu::UAT_PGSZ, CpuMap::WriteCombined)?; + let mapping = object.alias_in( + vm, + cfg::TIMESTAMPS, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + )?; + Ok(Self { mapping, object }) + } +} + +#[pin_data] +pub(crate) struct Completion { + status: Arc, + fence: UserFence, + member: Member, + destinations: [Option; 4], + compute: Option, + #[pin] + vm_job: Mutex>, + #[pin] + render: Mutex>, + signalled: AtomicBool, + timeout_error: AtomicI32, + replays: AtomicU8, + replay_timeout: AtomicBool, + firmware_consumer: AtomicBool, + class: AtomicU8, +} + +impl Completion { + pub(crate) fn new( + dev: &AsahiDevice, + contexts: &FenceContexts, + context: &Arc, + member: Member, + vm_job: mmu::VmJobGuard, + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + compute: bool, + ) -> Result> { + if timestamps.len() != if compute { 1 } else { 2 } { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; + } + let fence = fence::independent(contexts)?; + let compute = if compute { + Some(ComputeTimestamps::new(dev, context.vm())?) + } else { + None + }; + Arc::pin_init( + pin_init!(Self { + status: context.status().clone(), + fence, + member, + destinations, + compute, + vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), + render <- new_mutex!(None, "G17 command render lease"), + signalled: AtomicBool::new(false), + timeout_error: AtomicI32::new(0), + replays: AtomicU8::new(0), + replay_timeout: AtomicBool::new(false), + firmware_consumer: AtomicBool::new(false), + class: AtomicU8::new(0), + }), + GFP_KERNEL, + ) + } + + pub(crate) fn fence(&self) -> Fence { + Fence::from_fence(&self.fence) + } + pub(crate) fn status(&self) -> &Arc { + &self.status + } + pub(crate) fn work_state(&self) -> Arc { + self.member.work_state() + } + pub(crate) fn has_timestamps(&self) -> bool { + self.destinations.iter().any(Option::is_some) + } + + pub(crate) fn compute_timestamp_va(&self) -> Result { + Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) + } + + /// The caller has proved that firmware retired this command, or has + /// received its scheduler kill acknowledgement with recovery closed. + /// These are the command's public timestamp words, not queue-private state. + pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let words = [ + object.dword(0)?.load(Ordering::Relaxed), + object.dword(8)?.load(Ordering::Relaxed), + ]; + barrier(Ordering::Acquire); + Ok(words) + } + + pub(crate) fn replay_witness_words(&self) -> Option<[u64; 2]> { + if self.signalled.load(Ordering::Acquire) { + return None; + } + self.compute_words().ok() + } + + pub(crate) fn begin_replay(&self) -> bool { + let accepted = self + .replays + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < 3).then_some(count + 1) + }) + .is_ok(); + if accepted { + self.replay_timeout.store(true, Ordering::Release); + } + accepted + } + + pub(crate) fn replay_count(&self) -> u8 { + self.replays.load(Ordering::Acquire) + } + + pub(crate) fn note_firmware_consumer(&self) { + self.firmware_consumer.store(true, Ordering::Release); + } + + pub(crate) fn has_firmware_consumer(&self) -> bool { + self.firmware_consumer.load(Ordering::Acquire) + } + + pub(crate) fn take_replay_timeout(&self) -> bool { + !self.signalled.load(Ordering::Acquire) && self.replay_timeout.swap(false, Ordering::AcqRel) + } + + /// Only a witnessed, retained command can be reset before republishing it. + pub(crate) fn reset_for_replay(&self) -> Result { + if self.signalled.load(Ordering::Acquire) { + return Err(ECANCELED); + } + let object = &self.compute.as_ref().ok_or(EINVAL)?.object; + let start = object.dword(0)?; + let end = object.dword(8)?; + start.store(0, Ordering::Relaxed); + end.store(0, Ordering::Relaxed); + barrier(Ordering::SeqCst); + Ok(()) + } + + /// The first timeout request retains its errno while the physical graph is + /// borrowed. Duplicate watchdog or cancellation callbacks cannot replace it. + pub(crate) fn defer_timeout(&self, error: Error) { + let _ = self.timeout_error.compare_exchange( + 0, + error.to_errno(), + Ordering::AcqRel, + Ordering::Acquire, + ); + } + + pub(crate) fn take_deferred_timeout(&self) -> Option { + let error = self.timeout_error.swap(0, Ordering::AcqRel); + (error != 0).then(|| Error::from_errno(error)) + } + + pub(crate) fn classify(&self, class: Class) { + self.class.store( + match class { + Class::Spared => 1, + Class::Guilty => 2, + }, + Ordering::Release, + ); + } + + pub(crate) fn spared(&self) -> bool { + self.class.load(Ordering::Acquire) == 1 + } + + /// Installed before the first firmware-visible prefix. Cancellation takes + /// the same lock after claiming the signal, so it cannot miss a late lease. + pub(crate) fn attach_render_lease( + &self, + lease: super::queue::admission::RenderLease, + ) -> Result { + let mut render = self.render.lock(); + if self.signalled.load(Ordering::Acquire) { + return Err(ECANCELED); + } + if render.is_some() { + return Err(EIO); + } + *render = Some(lease); + Ok(()) + } + + pub(crate) fn render_slot(&self) -> Option { + self.render + .lock() + .as_ref() + .map(super::queue::admission::RenderLease::slot) + } + + /// Called under the device mutex before deferring the error signal. + pub(crate) fn quarantine_render(&self) { + if let Some(lease) = self.render.lock().as_ref() { + lease.quarantine(); + } + } + + /// An exact whole-pair drain or processor stop allows rehabilitation. The + /// VM pin goes first; the still-held lease excludes reuse until both settle. + pub(crate) fn release_after_retirement(&self) { + drop(self.vm_job.lock().take()); + let lease = self.render.lock().take(); + if let Some(lease) = lease.as_ref() { + lease.rehabilitate_after_firmware_drain(); + } + drop(lease); + } + + fn release_healthy(&self) { + drop(self.render.lock().take()); + drop(self.vm_job.lock().take()); + } + + /// Exact retirement permits the physical lease and VM pin to be released + /// before waking userspace. A failure fence alone never establishes this. + pub(crate) fn complete(&self, result: Result<[u64; 4]>) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + let result = result.and_then(|ticks| { + for (destination, ticks) in self.destinations.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks)?; + } + } + Ok(()) + }); + if let Err(error) = result { + if error != ECANCELED && !self.spared() { + self.status.record(error); + } + self.fence.set_error(error); + } + self.release_healthy(); + self.fence.signal(); + self.member.complete(result); + } + + /// A rejected, never-published command cannot poison the VM for contention. + pub(crate) fn fail_unpublished(&self, error: Error) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + self.status.record_if_device_loss(error); + self.fence.set_error(error); + self.release_healthy(); + self.fence.signal(); + self.member.complete(Err(error)); + } + + /// The physical queue must already be quarantined. Keep mappings pinned while + /// waking waiters, even if queue destruction requested cancellation. + pub(crate) fn fail_while_owned(&self, error: Error) { + if self.signalled.swap(true, Ordering::AcqRel) { + return; + } + if let Some(guard) = self.vm_job.lock().as_ref() { + guard.quarantine(); + } + if error != ECANCELED && !self.spared() { + self.status.record(error); + } + self.fence.set_error(error); + self.fence.signal(); + self.member.complete(Err(error)); + } +} + +/// Collected under the device mutex and executed only after releasing it. +/// Runtime queues establish the ownership witness before constructing `Retired` +/// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. +pub(crate) enum Deferred { + Retired(Arc, Result<[u64; 4]>), + Drained(Arc, Result<[u64; 4]>), + FailedOwned(Arc, Error), + Unpublished(Arc, Error), + Release(Arc), +} + +impl Deferred { + pub(crate) fn finish(self) { + match self { + Self::Retired(completion, result) => { + completion.complete(result); + // An earlier failure may already have signalled this fence while + // retaining the pin. Retirement is a separate release witness. + completion.release_after_retirement(); + } + Self::Drained(completion, result) => { + completion.release_after_retirement(); + completion.complete(result); + } + Self::FailedOwned(completion, error) => completion.fail_while_owned(error), + Self::Unpublished(completion, error) => completion.fail_unpublished(error), + Self::Release(completion) => completion.release_after_retirement(), + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs new file mode 100644 index 00000000000000..c3a3c6e87a2f60 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Logical queue ownership of scheduler, QoS and work-state identities. +//! +//! Installed bindings retain this context until replacement is acknowledged. +//! A job keeps its work-state lease through exact retirement, even if its error +//! fence has already been signalled. Context-local accounting never waits for GPU work. + +mod identity; +use super::{ + freelist::{RenderIds, RenderPool}, + fw::context as fw, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, + status::VmStatus, +}; +use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, Ordering}; +use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; + +/// QoS IDs belong to one device; the two lowest values are reserved. +#[pin_data] +pub(crate) struct QosIds { + #[pin] + used: Mutex, +} +impl QosIds { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }), + GFP_KERNEL, + ) + } + fn acquire(self: &Arc) -> Result { + let mut used = self.used.lock(); + let id = used.trailing_ones(); + if id == 128 { + return Err(ENOSPC); + } + *used |= 1u128 << id; + Ok(QosLease { + ids: self.clone(), + id: id as u8, + }) + } +} +struct QosLease { + ids: Arc, + id: u8, +} +impl Drop for QosLease { + fn drop(&mut self) { + *self.ids.used.lock() &= !(1u128 << self.id); + } +} + +struct WorkStorage { + object: PooledObject, + words: [Option; STATE_COUNT], + counts: [u32; STATE_COUNT], + render: RenderSlots, +} +impl WorkStorage { + fn initialize(&mut self, word: u32) -> Result { + let slot = (word & 63) as usize; + if slot >= STATE_COUNT { + return Err(EINVAL); + } + let counter = UAT_PGSZ + slot * 4; + let node = fw::WorkNode::new(self.object.gpu_va() + counter as u64, word).ok_or(EINVAL)?; + // The exact work-state lease excludes a previous firmware owner. Only + // the node's 0xc8 initialized bytes are replaced; the stride tail is retained. + self.object.write(slot * NODE_STRIDE, node)?; + self.object.word(counter)?.store(0, Ordering::Relaxed); + self.words[slot] = Some(word); + self.counts[slot] = 0; + fence(Ordering::Release); + Ok(()) + } + fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { + let slot = (word & 63) as usize; + if self.words.get(slot) != Some(&Some(word)) { + return Err(EINVAL); + } + let value = if add { + self.counts[slot].checked_add(kicks) + } else { + self.counts[slot].checked_sub(kicks) + } + .ok_or(EOVERFLOW)?; + self.object + .word(UAT_PGSZ + slot * 4)? + .store(value, Ordering::Relaxed); + self.counts[slot] = value; + fence(Ordering::SeqCst); + Ok(value) + } +} + +pub(crate) struct Context { + execution: Arc, + status: Arc, + scheduler: Arc, + scheduler_generation: u32, + published: AtomicBool, + qos: QosLease, + owner_pid: u32, + states: WorkStates, + work: Arc>, + work_base: u64, + render: Arc>>>, +} +impl Context { + pub(crate) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + qos: &Arc, + execution: Arc, + owner_pid: u32, + ) -> Result { + let status = execution.vm().status().ok_or(EINVAL)?.clone(); + let qos = qos.acquire()?; + let mut scheduler = alloc.kernel( + UAT_PGSZ, + UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + scheduler.initialize::(0, |value| { + value.init(); + Ok(()) + })?; + fence(Ordering::SeqCst); + let scheduler_generation = super::ids::OBJECTS.object()?; + let scheduler = Arc::new(scheduler, GFP_KERNEL)?; + let mut object = pool.allocate( + alloc, + Family::WorkState, + 2 * UAT_PGSZ, + UAT_PGSZ, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let work_base = object.gpu_va(); + object.write( + UAT_PGSZ + 0x400, + fw::WorkHead { + next: 0, + previous: work_base + (UAT_PGSZ + 0x400) as u64, + unk10: 0, + unk14: 0, + }, + )?; + fence(Ordering::Release); + let work = Arc::pin_init( + new_mutex!( + WorkStorage { + object, + words: [None; STATE_COUNT], + counts: [0; STATE_COUNT], + render: RenderSlots::new() + }, + "G17 work states" + ), + GFP_KERNEL, + )?; + let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; + Ok(Self { + execution, + status, + scheduler, + scheduler_generation, + published: AtomicBool::new(false), + qos, + owner_pid, + states: WorkStates::new(), + work, + work_base, + render, + }) + } + pub(crate) fn vm(&self) -> &Vm { + self.execution.vm() + } + pub(crate) fn status(&self) -> &Arc { + &self.status + } + pub(crate) fn id(&self) -> u32 { + self.execution.id() + } + + pub(crate) fn generation(&self) -> u8 { + self.execution.generation() + } + pub(crate) fn is_current(&self) -> bool { + self.execution.is_current() + } + pub(crate) fn owner_pid(&self) -> u32 { + self.owner_pid + } + pub(crate) fn qos_id(&self) -> u8 { + self.qos.id + } + pub(crate) fn scheduler_va(&self) -> u64 { + self.scheduler.gpu_va() + } + /// Installed physical channels retain this page after the logical execution + /// context and its VM lease have completed their scheduler release. + pub(crate) fn scheduler_owner(&self) -> Arc { + self.scheduler.clone() + } + pub(crate) fn scheduler_generation(&self) -> u32 { + self.scheduler_generation + } + pub(crate) fn work_head_va(&self) -> u64 { + self.work_base + (UAT_PGSZ + 0x400) as u64 + } + /// Shared scheduler list head, sampled under the context's storage lock. + pub(crate) fn work_head(&self) -> Result<[u64; 2]> { + let work = self.work.lock(); + let head = [ + work.object.dword(UAT_PGSZ + 0x400)?.load(Ordering::Relaxed), + work.object.dword(UAT_PGSZ + 0x408)?.load(Ordering::Relaxed), + ]; + fence(Ordering::Acquire); + Ok(head) + } + + pub(crate) fn is_published(&self) -> bool { + self.published.load(Ordering::Acquire) + } + pub(crate) fn mark_published(&self) { + self.published.store(true, Ordering::Release); + } + + pub(crate) fn release_identity(&self) -> Result<[u8; 4]> { + let mut result = [0; 4]; + fence(Ordering::Acquire); + for (value, offset) in result.iter_mut().zip([0, 1, 4, 0x26]) { + let pointer = self.scheduler.pointer(offset, 1)?; + // SAFETY: pointer checked against retained scheduler allocation; + // a byte read observes firmware state without borrowing its fields. + *value = unsafe { core::ptr::read_volatile(pointer) }; + } + Ok(result) + } + pub(crate) fn render_pool( + &self, + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &Vm, + ) -> Result> { + if !self.is_current() { + return Err(EFAULT); + } + let mut render = self.render.lock(); + if let Some(pool) = &*render { + return Ok(pool.clone()); + } + let pool = Arc::new( + RenderPool::new(alloc, ids, render_global, self.vm())?, + GFP_KERNEL, + )?; + *render = Some(pool.clone()); + Ok(pool) + } + pub(crate) fn render_pools_released(&self, released: impl Fn(u32, u64) -> bool) -> bool { + self.render + .lock() + .as_ref() + .is_none_or(|pool| released(u32::from(pool.id()), pool.control_va())) + } +} + +/// Shared by all engines of one ioctl. Queued work initially consumes no node. +pub(crate) struct WorkStateLease { + context: Arc, + word: AtomicU32, +} +impl WorkStateLease { + pub(crate) fn deferred(context: Arc) -> Self { + Self { + context, + word: AtomicU32::new(FREE), + } + } + pub(crate) fn acquire(&self) -> Result { + let current = self.word(); + if current != FREE { + return Ok(current); + } + let Some(candidate) = self.context.states.allocate() else { + let current = self.word(); + return if current != FREE { + Ok(current) + } else { + Err(EBUSY) + }; + }; + if let Err(error) = self.context.work.lock().initialize(candidate) { + self.context.states.release(candidate); + return Err(error); + } + match self + .word + .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => Ok(candidate), + Err(installed) => { + self.context.states.release(candidate); + Ok(installed) + } + } + } + pub(crate) fn word(&self) -> u32 { + self.word.load(Ordering::Acquire) + } + pub(crate) fn node_va(&self) -> Result { + let word = self.word(); + if word == FREE { + return Err(EINVAL); + } + Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) + } + pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { + self.context + .work + .lock() + .change_count(self.word(), kicks, true) + } + pub(crate) fn cancel_unpublished_kicks(&self, kicks: u32) -> Result { + self.context + .work + .lock() + .change_count(self.word(), kicks, false) + } + + pub(crate) fn render_slot(&self, pair: u8) -> Result { + self.context.work.lock().render.get(pair).ok_or(ENOSPC) + } + pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { + self.context + .work + .lock() + .render + .payload(pair, ordinal) + .ok_or(EINVAL) + } +} +impl Drop for WorkStateLease { + fn drop(&mut self) { + let word = self.word(); + if word != FREE { + self.context.states.release(word); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/context/identity.rs b/drivers/gpu/drm/asahi/g17/context/identity.rs new file mode 100644 index 00000000000000..6c48367e2d6864 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/context/identity.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Context-local work-state identities. The low six bits address a node; +//! the complete sequence distinguishes its owner until exact retirement. + +use core::sync::atomic::{AtomicU32, Ordering}; + +pub(super) const FREE: u32 = u32::MAX; +pub(super) const STATE_COUNT: usize = 63; +pub(super) const NODE_STRIDE: usize = 0x100; + +pub(super) struct WorkStates { + next: AtomicU32, + live: [AtomicU32; 64], +} +impl WorkStates { + pub(super) const fn new() -> Self { + Self { + next: AtomicU32::new(0), + live: [const { AtomicU32::new(FREE) }; 64], + } + } + pub(super) fn allocate(&self) -> Option { + for _ in 0..64 { + let word = self.next.fetch_add(1, Ordering::Relaxed); + let index = (word & 63) as usize; + if index != STATE_COUNT + && self.live[index] + .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return Some(word); + } + } + None + } + pub(super) fn release(&self, word: u32) -> bool { + let index = (word & 63) as usize; + index != STATE_COUNT + && self.live[index] + .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +/// Two stable logical ordinals, independent of physical pair numbering. +pub(super) struct RenderSlots { + pairs: [Option; 2], + origins: [Option; 2], +} +impl RenderSlots { + pub(super) const fn new() -> Self { + Self { + pairs: [None; 2], + origins: [None; 2], + } + } + pub(super) fn get(&mut self, pair: u8) -> Option { + if let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) { + return Some(index as u32); + } + let index = self.pairs.iter().position(Option::is_none)?; + self.pairs[index] = Some(pair); + Some(index as u32) + } + pub(super) fn payload(&mut self, pair: u8, ordinal: u64) -> Option { + let index = self.get(pair)? as usize; + let origin = *self.origins[index].get_or_insert(ordinal); + Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) + } +} diff --git a/drivers/gpu/drm/asahi/g17/dependency.rs b/drivers/gpu/drm/asahi/g17/dependency.rs new file mode 100644 index 00000000000000..5abd494eeb1f58 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/dependency.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Collapse a logical prefix to its newest still-live firmware kick per QID. + +use super::{ + fw::kick::{KickDependency, KICK_BARRIERS_MAX}, + job::Order, +}; +use kernel::prelude::*; + +/// A failed producer does not establish a fault in its consumer's VM. +pub(crate) fn inherited_error(error: Error) -> Error { + if error == EIO || error == ENODEV || error == ETIMEDOUT { + ECANCELED + } else { + error + } +} + +pub(crate) struct Frontier { + dependencies: [KickDependency; KICK_BARRIERS_MAX], + sequences: [u64; KICK_BARRIERS_MAX], + count: usize, +} + +impl Frontier { + pub(crate) const fn new() -> Self { + Self { + dependencies: [KickDependency::ZERO; KICK_BARRIERS_MAX], + sequences: [0; KICK_BARRIERS_MAX], + count: 0, + } + } + + pub(crate) fn as_slice(&self) -> &[KickDependency] { + &self.dependencies[..self.count] + } + + /// Each QID's implicit parent supplies transitive ordering. Choose its newest + /// logical sequence rather than comparing wrapped firmware timestamps. + pub(crate) fn insert(&mut self, sequence: u64, dependency: KickDependency) -> Result { + if let Some(index) = self + .as_slice() + .iter() + .position(|prior| prior.qid() == dependency.qid()) + { + if sequence > self.sequences[index] { + self.dependencies[index] = dependency; + self.sequences[index] = sequence; + } + return Ok(()); + } + if self.count == KICK_BARRIERS_MAX { + return Err(EIO); + } + self.dependencies[self.count] = dependency; + self.sequences[self.count] = sequence; + self.count += 1; + Ok(()) + } + + /// The caller has matched the logical owner and VM. Check every covered + /// predecessor for failure, including one a newer kick would subsume. + pub(crate) fn include( + &mut self, + prefix: Option, + sequence: u64, + status: i32, + word: u64, + ) -> Result { + if !Order::contains(prefix, sequence) { + return Ok(()); + } + if status < 0 { + return Err(ECANCELED); + } + if status > 0 { + return Ok(()); + } + let dependency = KickDependency::from_word(word).ok_or(EIO)?; + self.insert(sequence, dependency) + } +} diff --git a/drivers/gpu/drm/asahi/g17/event.rs b/drivers/gpu/drm/asahi/g17/event.rs new file mode 100644 index 00000000000000..9e4e0253b7c813 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/event.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event service ordering under the device mutex. A restart ends its role's batch; a worker +//! joins preparation leases and services it on a separate turn before continuing the rings. + +use super::{ + fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, + recovery, Role, +}; +use kernel::prelude::*; + +/// The bounded shared-ring transport; recognized malformed entries return errors after consume. +pub(super) trait Transport { + fn trace(&mut self, role: Role); + fn statistics(&mut self, role: Role); + fn drain( + &mut self, + role: Role, + consume: impl FnMut(&mut Self, Event) -> Result, + ) -> Result; + fn recovery_state(&self) -> Result; +} + +/// Runtime callbacks with the device mutex held. None signals fences or allocates grow backing. +pub(super) trait Queues { + /// Flush the oldest deferred reply/release. False means ring backpressure; each pending + /// transaction retains its own bounded worker-attempt count across service turns. + fn flush_tvb_reply(&mut self) -> Result; + fn flush_freelist_release(&mut self) -> Result; + fn observe_render_releases(&mut self) -> Result; + fn control_backpressured(&self) -> bool; + fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result; + fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result; + fn retain_render_completions(&mut self, masks: [u64; 2]); + fn complete_compute(&mut self, masks: [u64; 2]) -> Result; + fn context_killed(&mut self, cookie: u64); +} + +/// Work requested after releasing the device mutex. +#[derive(Default)] +pub(super) struct Pending { + pub(super) recovery: bool, +} + +/// Access to the single recovery owner while event handlers borrow the device. +pub(super) trait Host: Transport + Queues { + fn recovery(&self) -> &recovery::State; + fn recovery_mut(&mut self) -> &mut recovery::State; +} + +fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { + host.drain(role, |host, event| { + match event { + Event::Stamp(bits) => { + masks[0] |= bits[0]; + masks[1] |= bits[1]; + } + Event::Restart { generation, stamp } => { + if host.recovery_mut().request(generation, stamp)? { + return Ok(false); + } + } + Event::TvbGrow(request) => { + if request.halt_count == host.recovery().generation() { + host.queue_tvb_grow(request)?; + } + if host.control_backpressured() { + return Ok(false); + } + } + Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, + Event::ContextKilled(cookie) => host.context_killed(cookie), + Event::Unknown(_) => {} + } + Ok(true) + }) +} + +pub(super) fn service(host: &mut H) -> Result { + if !host.flush_tvb_reply()? || !host.flush_freelist_release()? { + return Ok(Pending::default()); + } + host.observe_render_releases()?; + for role in [Role::Primary, Role::Secondary] { + host.trace(role); + } + for role in [Role::Primary, Role::Secondary] { + host.statistics(role); + } + let mut masks = [0; 2]; + if !host.recovery().pending() { + let mut settled = false; + // The firmware publishes restart before state 1, but its producer can race the snapshot. + for _ in 0..256 { + batch(host, Role::Primary, &mut masks)?; + if host.recovery().pending() || host.control_backpressured() { + settled = true; + break; + } + match host.recovery_state()? { + 0 => { + settled = true; + break; + } + 1 => {} + _ => return Err(EIO), + } + } + if !settled { + return Err(EIO); + } + } + if !host.control_backpressured() { + batch(host, Role::Secondary, &mut masks)?; + } + host.retain_render_completions(masks); + host.complete_compute(masks)?; + Ok(Pending { + recovery: host.recovery().pending(), + }) +} diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs new file mode 100644 index 00000000000000..95a13dbdd54fff --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware completion fences and the aggregate fence of one submit ioctl. +//! +//! These fences can complete out of order, so each has an independent timeline. +//! Signalling belongs outside the device lock, after timestamp and VM-status writes. + +use super::context::{Context, WorkStateLease}; +use crate::file::SyncItem; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; +use kernel::{ + bindings, c_str, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + prelude::*, + str::CStr, + sync::Arc, +}; + +pub(crate) struct CompletionFence; + +#[vtable] +impl FenceOps for CompletionFence { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g17-submit") + } +} + +/// Give independently retired work a timeline that fence merging cannot subsume. +pub(crate) fn independent(contexts: &FenceContexts) -> Result> { + let fence = contexts.new_fence(0, CompletionFence)?; + // SAFETY: The fence is unique and unpublished: there are no waiters, callbacks or + // shared references. The DMA-fence allocator supplies a globally unique context. + unsafe { (*fence.raw()).context = bindings::dma_fence_context_alloc(1) }; + Ok(fence.into()) +} + +/// All command members of an ioctl share its work-state node and aggregate fence. +/// The initial member keeps the aggregate unsignalled until enqueue finishes. +pub(crate) struct Submission { + fence: UserFence, + work_state: Arc, + remaining: AtomicU32, + error: AtomicI32, +} + +impl Submission { + pub(crate) fn new(contexts: &FenceContexts, context: Arc) -> Result> { + Ok(Arc::new( + Self { + fence: independent(contexts)?, + work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, + remaining: AtomicU32::new(1), + error: AtomicI32::new(0), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn fence(&self) -> Fence { + Fence::from_fence(&self.fence) + } + + /// Add one command before enqueue ends. The parser bounds the count to 64. + pub(crate) fn member(self: &Arc) -> Member { + self.remaining.fetch_add(1, Ordering::Relaxed); + Member { + submission: self.clone(), + complete: AtomicBool::new(false), + } + } + + fn member_done(&self, result: Result) { + if let Err(error) = result { + let _ = self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::AcqRel, + Ordering::Acquire, + ); + } + if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { + let error = self.error.load(Ordering::Acquire); + if error != 0 { + self.fence.set_error(Error::from_errno(error)); + } + self.fence.signal(); + } + } + + fn finish_enqueue(&self, result: Result) { + // A dropped, never-enqueued member reports cancellation. The ioctl's actual + // failure takes precedence, while published members still retain their work. + if let Err(error) = result { + self.error.store(error.to_errno(), Ordering::Release); + } + self.member_done(result); + } +} + +/// One command's contribution to its submission, including construction failure. +pub(crate) struct Member { + submission: Arc, + complete: AtomicBool, +} + +impl Member { + pub(crate) fn work_state(&self) -> Arc { + self.submission.work_state.clone() + } + + pub(crate) fn complete(&self, result: Result) { + if !self.complete.swap(true, Ordering::AcqRel) { + self.submission.member_done(result); + } + } +} + +impl Drop for Member { + fn drop(&mut self) { + self.complete(Err(ECANCELED)); + } +} + +fn install(outputs: impl Iterator, fence: &Fence) { + for mut sync in outputs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(fence)); + } + } +} + +/// Keeps resolved output syncobjs reachable on every fallible submission path. +/// Input syncobjs stay in the same vector, avoiding a separate output allocation. +pub(crate) struct Outputs { + fallback: UserFence, + syncs: KVec, + input_count: usize, + submission: Option>, + result: Result, +} + +impl Outputs { + pub(crate) fn new( + contexts: &FenceContexts, + syncs: KVec, + input_count: usize, + ) -> Result { + if input_count > syncs.len() { + return Err(EINVAL); + } + Ok(Self { + fallback: independent(contexts)?, + syncs, + input_count, + submission: None, + result: Err(ECANCELED), + }) + } + + pub(crate) fn inputs(&self) -> &[SyncItem] { + &self.syncs[..self.input_count] + } + + /// Publish the actual aggregate before enqueueing any command. + pub(crate) fn publish(&mut self, submission: Arc) { + install(self.syncs.drain(self.input_count..), &submission.fence()); + self.submission = Some(submission); + } + + /// The caller first records terminal VM errors and settles prior failed work. + pub(crate) fn finish(mut self, result: Result) -> Result { + self.result = result; + result + } +} + +impl Drop for Outputs { + fn drop(&mut self) { + if let Some(submission) = self.submission.take() { + submission.finish_enqueue(self.result); + } + if let Err(error) = self.result { + self.fallback.set_error(error); + } + install( + self.syncs.drain(self.input_count..), + &Fence::from_fence(&self.fallback), + ); + self.fallback.signal(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs new file mode 100644 index 00000000000000..e1d42944a38fe8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -0,0 +1,575 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained USC private-memory pools and their submission counters. +//! +//! A logical context owns one render pool, shared by its render pairs. A +//! physical compute queue owns a separate pool. All mutation and publication +//! runs under the device mutex. Pool aliases must outlive every firmware +//! reference and are released only after the owner's teardown is acknowledged. + +use super::fw::freelist::*; +use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_SIZE}; +use super::object::{Allocator, CpuMap, KernelObject}; +use crate::mmu; +use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +/// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. +/// First-free allocation keeps the shared firmware namespace deterministic. +pub(crate) struct RenderIds { + used: [AtomicU64; 2], +} +impl RenderIds { + pub(crate) const fn new() -> Self { + Self { + used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], + } + } +} +struct RenderId { + slot: u16, + allocator: Arc, +} +impl RenderId { + fn new(allocator: &Arc) -> Result { + for (bank, used) in allocator.used.iter().enumerate() { + let mut value = used.load(Ordering::Acquire); + loop { + let bit = value.trailing_ones(); + if bit == 64 { + break; + } + match used.compare_exchange_weak( + value, + value | (1u64 << bit), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + return Ok(Self { + slot: 129 + bank as u16 * 64 + bit as u16, + allocator: allocator.clone(), + }) + } + Err(current) => value = current, + } + } + } + Err(ENOSPC) + } +} +impl Drop for RenderId { + fn drop(&mut self) { + let bit = self.slot as usize - 129; + self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); + } +} + +struct Pages { + backing: KernelObject, + blocks: usize, + layout: RunLayout, +} +impl Pages { + fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { + let mut backing = alloc.lower( + TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; + let base = backing.gpu_va(); + backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; + Ok(Self { + backing, + blocks, + layout, + }) + } + fn page_list_va(&self) -> u64 { + self.backing.gpu_va() + } + fn run_list_va(&self) -> u64 { + self.page_list_va() + PAGE_LIST_SIZE as u64 + } + fn repopulate(&self) -> Result { + let table = self.backing.pointer(0, TABLE_SIZE)?.cast::(); + PoolTables::for_each_word( + self.page_list_va(), + self.blocks, + self.layout, + |index, value| { + // SAFETY: The page-aligned backing was validated for the complete + // tables. The enumerator bounds every index inside those tables; + // the owner has proven firmware release before this update. + unsafe { table.add(index).write_volatile(value) }; + }, + ) + } +} + +/// Pool-object identities are independent of recycled firmware IDs and VAs. +static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); + +/// A render pool retained by a logical context and every graph referencing it. +pub(crate) struct RenderPool { + // Remove all aliases before dropping the backing or returning the ID. + _client: mmu::KernelMapping, + _global: mmu::KernelMapping, + pages: Pages, + control: KernelObject, + state: KernelObject, + served: AtomicBool, + serial: u64, + id: RenderId, +} + +impl RenderPool { + pub(crate) fn new( + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + let id = RenderId::new(ids)?; + let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; + let address = pages.page_list_va(); + let global = pages + .backing + .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; + let client = pages + .backing + .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + let state = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut control = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + control.write( + 0, + FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(id.slot), + class: FreeListClass::Two, + page_list_va: address, + run_list_va: pages.run_list_va(), + blocks: RENDER_BLOCKS as u32, + state_va: state.gpu_va(), + }, + 0, + ), + )?; + fence(Ordering::SeqCst); + let serial = RENDER_POOL_SERIAL + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) + .map_err(|_| EOVERFLOW)? + 1; + Ok(Self { + _client: client, + _global: global, + pages, + control, + state, + served: AtomicBool::new(false), + serial, + id, + }) + } + + pub(crate) fn serial(&self) -> u64 { + self.serial + } + pub(crate) fn id(&self) -> u16 { + self.id.slot + } + pub(crate) fn control_va(&self) -> u64 { + self.control.gpu_va() + } + pub(crate) fn page_list_va(&self) -> u64 { + self.pages.page_list_va() + } + pub(crate) fn run_list_va(&self) -> u64 { + self.pages.run_list_va() + } + pub(crate) fn end_va(&self) -> u64 { + self.run_list_va() + + (RENDER_BLOCKS * size_of::()) as u64 + } + + /// Fresh pools are already populated. Reuse requires the previous release + /// to have been consumed; command completion alone does not establish it. + pub(crate) fn repopulate(&self) -> Result { + if !self.served.swap(true, Ordering::AcqRel) { + return Ok(()); + } + let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let header = FreeListControl::render( + &FreeListArgs { + buffer_slot: u32::from(self.id()), + class: FreeListClass::Two, + page_list_va: self.page_list_va(), + run_list_va: self.run_list_va(), + blocks: RENDER_BLOCKS as u32, + state_va: self.state.gpu_va(), + }, + submitted, + ); + let mut image = KVVec::from_elem(0u64, CONTROL_SIZE / 8, GFP_KERNEL)?; + // SAFETY: `image` owns a zeroed control page, is large enough for the + // header, and every header byte is initialized integer data. + unsafe { + image + .as_mut_ptr() + .cast::() + .write_unaligned(header) + }; + let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); + self.pages.repopulate()?; + for (index, value) in image.iter().enumerate() { + // SAFETY: The entire retained page was checked before any store; + // the allocation is page-aligned and release excludes the firmware. + unsafe { control.add(index).write_volatile(*value) }; + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Submitted is host-owned and aligned. Completed is the packed firmware + /// counter, so retain the byte accesses required by that interface. + pub(crate) fn counters(&self) -> Result<[u64; 2]> { + let submitted = self.state.dword(0)?.load(Ordering::Relaxed); + let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked eight-byte range stays mapped; byte accesses + // permit the unaligned counter without widening shared accesses. + *byte = unsafe { completed.add(index).read_volatile() }; + } + Ok([submitted, u64::from_le_bytes(bytes)]) + } + + /// Snapshot the unaligned shared fields before loading their descriptor. + pub(crate) fn page_pool_state(&self) -> Result { + let raw = self.control.pointer(0x14, 0x1c)?; + fence(Ordering::Acquire); + let read = |offset: usize, count: usize| -> u64 { + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().take(count).enumerate() { + // SAFETY: The complete 0x14..0x30 span was checked. These fixed + // field ranges lie inside it; byte loads permit unaligned words. + *byte = unsafe { raw.add(offset + index).read_volatile() }; + } + u64::from_le_bytes(bytes) + }; + Ok(PagePoolState { + page_list_va: read(0, 8), + unk_1c: read(8, 4) as u32, + unk_20: read(12, 4) as u32, + unk_24: read(16, 4) as u32, + unk_28: read(20, 4) as u32, + page_count: read(24, 4) as u32, + }) + } + + /// Save mutable descriptor state only after both engines and free-list + /// references retire, before the physical slot changes owners. + pub(crate) fn store_page_pool_state(&self, state: PagePoolState) -> Result { + let raw = self.control.pointer(0x20, 0x10)?; + for (word, value) in [state.unk_20, state.unk_24, state.unk_28, state.page_count] + .into_iter() + .enumerate() + { + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The entire mutable range was checked, and each + // enumerated byte lies inside it. Caller owns the retired pool. + unsafe { raw.add(word * 4 + index).write_volatile(byte) }; + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Prepares the counter before any producer is written. Commit belongs to + /// the publication transaction; dropping this plan does not reserve work. + pub(crate) fn prepare_submission(&self, commands: u32) -> Result> { + if commands > 2 { + return Err(EINVAL); + } + let counter = self.state.dword(0)?; + Ok(Submitted { + counter, + next: counter + .load(Ordering::Relaxed) + .wrapping_add(u64::from(commands)), + }) + } +} + +/// One checked, infallible counter update in a submission transaction. +pub(crate) struct Submitted<'a> { + counter: &'a AtomicU64, + next: u64, +} +impl Submitted<'_> { + pub(crate) fn commit(self) { + self.counter.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + } +} + +/// A pool owned by one physical compute queue, with fixed QID-derived slot. +pub(crate) struct ComputePool { + pages: Pages, + control: KernelObject, + state: KernelObject, + id: u16, +} +impl ComputePool { + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8) -> Result { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let id = u16::from(qid) + 1; + let pages = Pages::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute)?; + let mut control = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let state = alloc.kernel( + CONTROL_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + control.write( + 0, + FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(id), + class: FreeListClass::One, + page_list_va: pages.page_list_va(), + run_list_va: pages.run_list_va(), + blocks: COMPUTE_BLOCKS as u32, + state_va: state.gpu_va(), + }), + )?; + fence(Ordering::SeqCst); + Ok(Self { + pages, + control, + state, + id, + }) + } + pub(crate) fn id(&self) -> u16 { + self.id + } + pub(crate) fn page_list_va(&self) -> u64 { + self.pages.page_list_va() + } + pub(crate) fn control_va(&self) -> u64 { + self.control.gpu_va() + } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + let address = self.pages.page_list_va(); + self.pages + .backing + .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + } + pub(crate) fn reserve_submission(&self) -> Result { + let counter = self.state.dword(0)?; + let next = counter + .load(Ordering::Relaxed) + .checked_add(1) + .ok_or(EOVERFLOW)?; + counter.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(next) + } + /// The caller has proved no pointer from this submission was published. + pub(crate) fn cancel_unpublished(&self, target: u64) -> Result { + let counter = self.state.dword(0)?; + if target == 0 || counter.load(Ordering::Acquire) != target { + return Err(EINVAL); + } + counter.store(target - 1, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Registration identity held by a free-list lease. The cookie is the end of +/// the run list; it remains stable while overlapping command pairs share it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Owner { + /// Identifies the host pool object even when every firmware address recycles. + pub(crate) serial: u64, + pub(crate) buffer_slot: u32, + pub(crate) control_va: u64, + pub(crate) ring_va: u64, + pub(crate) cookie: u64, +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum LeasePhase { + Preparing, + Ready, + ReleaseSent(u32), + Released, +} + +/// References to one prepopulated owner: preparation and two engine commands +/// per render. Published releases do not gate reacquisition; their consumption +/// witnesses are carried into the successor. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Lease { + owner: Owner, + generation: u64, + references: u32, + commands: u32, + phase: LeasePhase, + previous_release: Option, +} + +impl Lease { + /// Acquires an unchanged overlapping owner, or an owner whose previous + /// release was sent. A changed owner additionally requires caller-side + /// proof of old-pool release before its backing can be repopulated. + pub(crate) fn acquire(previous: Option<&Self>, owner: Owner) -> Result { + let Some(previous) = previous else { + return Ok(Self { + owner, + generation: 0, + references: 3, + commands: 2, + phase: LeasePhase::Preparing, + previous_release: None, + }); + }; + if previous.owner.buffer_slot != owner.buffer_slot { + return Err(EINVAL); + } + let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; + let same_owner = previous.owner == owner; + let overlapping = same_owner + && previous.phase == LeasePhase::Ready + && previous.references >= 2 + && previous.commands >= 2; + let reacquire = previous.reacquirable(); + if !overlapping && !reacquire { + return Err(EBUSY); + } + let references = previous.references.checked_add(3).ok_or(EOVERFLOW)?; + let commands = previous.commands.checked_add(2).ok_or(EOVERFLOW)?; + Ok(Self { + owner, + generation: if overlapping { + previous.generation + } else { + generation + }, + references, + commands, + phase: LeasePhase::Preparing, + previous_release: previous.release_witness(), + }) + } + + pub(crate) fn owner(&self) -> Owner { + self.owner + } + pub(crate) fn generation(&self) -> u64 { + self.generation + } + pub(crate) fn reacquirable(&self) -> bool { + matches!( + self.phase, + LeasePhase::ReleaseSent(_) | LeasePhase::Released + ) && self.references == 0 + && self.commands == 0 + } + pub(crate) fn released(&self) -> bool { + self.phase == LeasePhase::Released && self.previous_release.is_none() + } + pub(crate) fn release_witness(&self) -> Option { + match self.phase { + LeasePhase::ReleaseSent(cursor) => Some(cursor), + _ => self.previous_release, + } + } + + /// Publication retires preparation independently of the command references. + pub(crate) fn retire_prepare(&mut self) -> Result { + if self.phase != LeasePhase::Preparing || self.references < 3 { + return Err(EINVAL); + } + self.references -= 1; + self.phase = LeasePhase::Ready; + Ok(()) + } + /// Fixed prepopulated pools never submit an asynchronous growth request. + /// An incoming growth completion therefore cannot name a live request. + pub(crate) fn complete_grow( + &mut self, + _event: super::fw::channels::FreeListCompletion, + ) -> Result { + Err(EIO) + } + /// Both engines have retired this render's exact retained submission. + pub(crate) fn complete_render( + &mut self, + ) -> Result> { + if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { + return Err(EINVAL); + } + self.references -= 2; + self.commands -= 2; + Ok(self.pending_release()) + } + pub(crate) fn pending_release(&self) -> Option { + (self.phase == LeasePhase::Ready && self.references == 0).then(|| { + super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) + }) + } + /// Records the producer-after cursor once the final release was published. + pub(crate) fn mark_release_sent(&mut self, cursor: u32) -> Result { + if self.pending_release().is_none() || cursor >= super::fw::channels::DEVICE_CONTROL_SLOTS { + return Err(EINVAL); + } + self.phase = LeasePhase::ReleaseSent(cursor); + Ok(()) + } + /// Observes all older release records passed by one consumer snapshot. + pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { + if let Some(cursor) = self.previous_release { + if control_consumed(consumer, producer, cursor)? { + self.previous_release = None; + } + } + if let LeasePhase::ReleaseSent(cursor) = self.phase { + if control_consumed(consumer, producer, cursor)? { + self.phase = LeasePhase::Released; + } + } + Ok(()) + } +} + +/// A publication is retired once its producer-after cursor leaves the +/// current `(consumer, producer]` interval, including wrapped intervals. +pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Result { + let slots = super::fw::channels::DEVICE_CONTROL_SLOTS; + if consumer >= slots || producer >= slots || cursor >= slots { + return Err(EIO); + } + let pending = producer.wrapping_sub(consumer) & (slots - 1); + let ahead = cursor.wrapping_sub(consumer) & (slots - 1); + Ok(ahead == 0 || ahead > pending) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs new file mode 100644 index 00000000000000..3ed336dbadaad9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer inventory, scene and control records. These objects are +//! initialized before publication; live access uses the host buffer manager's +//! individual aligned words and packed byte fields. + +use super::{clear, offset_va}; +use crate::hw::t8140::resources as cfg; +use kernel::prelude::*; + +pub(crate) const SCENES: usize = 0x50; +pub(crate) const SCRATCH_SLOTS: usize = 0x24; +pub(crate) const SCRATCH_STRIDE: u64 = 0x20; +pub(crate) const FIRST_SCENE: usize = 1; +pub(crate) const PAGE_LIST: usize = 0x8000; +pub(crate) const PAGE_LIST_BYTES: usize = 0x10000; +pub(crate) const CONTROL: usize = 0x20000; +pub(crate) const COUNTER: usize = 0x28000; +pub(crate) const SCENE_TABLE: usize = 0x38000; +pub(crate) const BLOCK_TABLE: usize = 0x40000; +pub(crate) const SIZE: usize = 0x48000; +pub(crate) const METRICS_STRIDE: usize = SCENES * size_of::(); +pub(crate) const METRICS_SLOTS: usize = cfg::METRICS_SIZE as usize / METRICS_STRIDE; +pub(crate) const NORMAL_MAX_BLOCKS: usize = 0x45e; +pub(crate) const GROW_BLOCKS: usize = 10; + +/// Firmware's active parameter-buffer state. Pointers at +0x44, +0x4c, +/// +0x64 and +0x84 are intentionally unaligned. +#[repr(C, packed)] +pub(crate) struct State { + pub(crate) unk_0: [u8; 0xc], + pub(crate) buffer_id: u32, + pub(crate) unk_10: [u8; 0x10], + pub(crate) page_list_fw: u64, + pub(crate) page_list_client: u64, + pub(crate) page_list_size: u32, + pub(crate) pages: u32, + pub(crate) block_capacity: u32, + pub(crate) producer: u32, + pub(crate) consumer: u32, + pub(crate) block_table: u64, + pub(crate) control: u64, + pub(crate) last_page: u32, + pub(crate) block_size: u32, + pub(crate) unk_5c: [u8; 8], + pub(crate) counter: u64, + pub(crate) unk_6c: [u8; 0x10], + pub(crate) maximum_size: u32, + pub(crate) normal_maximum_size: u32, + pub(crate) discard: u64, + pub(crate) unk_8c: [u8; 0x34], +} + +/// One scene's accounting and scratch identity. +#[repr(C, packed)] +pub(crate) struct Scene { + pub(crate) metric_client: u64, + pub(crate) metric_fw: u64, + pub(crate) unk_10: [u8; 0x18], + pub(crate) scratch: u64, + pub(crate) unk_30: [u8; 0x10], + pub(crate) statistics: u64, + pub(crate) used: u32, + pub(crate) generation: u64, + pub(crate) unk_54: [u8; 0x2c], +} + +/// Circular inventory cursors followed by firmware statistics. Firmware owns +/// `read` and the statistics; the host requests reset by release-storing1. +#[repr(C)] +pub(crate) struct Control { + pub(crate) total: u32, + pub(crate) committed: u32, + pub(crate) read: u32, + pub(crate) unk_c: [u32; 13], + pub(crate) max_pages: u32, + pub(crate) unk_44: [u32; 7], + pub(crate) reset: u32, + pub(crate) unk_64: [u32; 7], +} + +#[repr(C)] +pub(crate) struct Block { + pub(crate) first_page: u32, + pub(crate) unk_4: u32, +} + +/// The complete parameter-buffer graph, including reserved spaces between +/// records. The page-list subrange also has a client mapping. +#[repr(C)] +pub(crate) struct Graph { + pub(crate) state: State, + pub(crate) pad_c0: [u8; PAGE_LIST - size_of::()], + pub(crate) pages: [u32; cfg::TVB_BLOCK_SLOTS * cfg::TVB_PAGES_PER_BLOCK], + pub(crate) pad_151a0: [u8; CONTROL - PAGE_LIST - cfg::TVB_PAGE_LIST_SIZE], + pub(crate) control: Control, + pub(crate) pad_20080: [u8; COUNTER - CONTROL - size_of::()], + pub(crate) counter: u32, + pub(crate) pad_28004: [u8; SCENE_TABLE - COUNTER - 4], + pub(crate) scenes: [Scene; SCENES], + pub(crate) trailer: [u8; 0x40], + pub(crate) pad_3a840: [u8; BLOCK_TABLE - SCENE_TABLE - SCENES * size_of::() - 0x40], + pub(crate) blocks: [Block; cfg::TVB_BLOCK_SLOTS], + pub(crate) pad_468d0: [u8; SIZE - BLOCK_TABLE - cfg::TVB_BLOCK_SLOTS * size_of::()], +} + +// SAFETY: Every field of the complete graph consists solely of integers. +unsafe impl Zeroable for Graph {} + +static_assert!(size_of::() == 0xc0); +static_assert!(size_of::() == 0x80); +static_assert!(size_of::() == 0x80); +static_assert!(size_of::() == SIZE); +static_assert!(core::mem::offset_of!(Graph, pages) == PAGE_LIST); +static_assert!(core::mem::offset_of!(Graph, control) == CONTROL); +static_assert!(core::mem::offset_of!(Graph, counter) == COUNTER); +static_assert!(core::mem::offset_of!(Graph, scenes) == SCENE_TABLE); +static_assert!(core::mem::offset_of!(Graph, blocks) == BLOCK_TABLE); +static_assert!(core::mem::offset_of!(State, block_table) == 0x44); +static_assert!(core::mem::offset_of!(State, discard) == 0x84); +static_assert!(core::mem::offset_of!(Scene, generation) == 0x4c); + +pub(crate) struct GraphArgs<'a> { + pub(crate) fw_va: u64, + pub(crate) page_list_client: u64, + /// Firmware and client views, respectively, of this graph's metrics lease. + pub(crate) metrics: [u64; 2], + pub(crate) blocks: &'a [u64], + pub(crate) scratch: u64, + pub(crate) discard: u64, + pub(crate) buffer_id: u32, + pub(crate) generation: u64, +} + +pub(crate) fn compact_page(address: u64) -> Result { + let relative = address.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + if relative % cfg::TVB_PAGE != 0 + || address + .checked_add(cfg::TVB_BLOCK_SIZE as u64) + .ok_or(EOVERFLOW)? + > cfg::USER_END + { + return Err(EINVAL); + } + Ok((relative / cfg::TVB_PAGE).try_into()?) +} + +impl Graph { + pub(crate) fn init(&mut self, args: &GraphArgs<'_>) -> Result { + if args.generation == 0 + || args.blocks.len() < cfg::TVB_INITIAL_PAGES.len() + || args.blocks.len() > cfg::TVB_MAX_BLOCKS + { + return Err(EINVAL); + } + let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; + if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { + return Err(EINVAL); + } + let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; + clear(self); + for (index, &address) in args.blocks.iter().enumerate() { + let first = compact_page(address)?; + self.blocks[index].first_page = first; + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + self.pages[index * cfg::TVB_PAGES_PER_BLOCK + page] = first + page as u32; + } + } + for (index, scene) in self.scenes.iter_mut().enumerate() { + scene.metric_client = offset_va(args.metrics[1], (index * 4) as u64)?; + scene.metric_fw = offset_va(args.metrics[0], (index * 4) as u64)?; + scene.scratch = scratch + (index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE; + scene.statistics = stats; + } + self.scenes[FIRST_SCENE].generation = args.generation; + self.control.total = args.blocks.len() as u32; + self.control.committed = self.control.total; + self.control.reset = 1; + let pages = (args.blocks.len() * cfg::TVB_PAGES_PER_BLOCK) as u32; + self.state.buffer_id = args.buffer_id; + self.state.page_list_fw = offset_va(args.fw_va, PAGE_LIST as u64)?; + self.state.page_list_client = args.page_list_client; + self.state.page_list_size = PAGE_LIST_BYTES as u32; + self.state.pages = pages; + self.state.block_capacity = cfg::TVB_BLOCK_SLOTS as u32; + self.state.producer = args.blocks.len() as u32; + self.state.block_table = offset_va(args.fw_va, BLOCK_TABLE as u64)?; + self.state.control = offset_va(args.fw_va, CONTROL as u64)?; + self.state.last_page = pages - 1; + self.state.block_size = cfg::TVB_BLOCK_SIZE as u32; + self.state.counter = offset_va(args.fw_va, COUNTER as u64)?; + self.state.maximum_size = (cfg::TVB_BLOCK_SLOTS * 4) as u32; + self.state.normal_maximum_size = (NORMAL_MAX_BLOCKS * 4) as u32; + self.state.discard = discard; + Ok(()) + } +} + +/// Rebuilds an idle parameter-buffer descriptor, preserving firmware-owned +/// bits and resetting both ring counters and flags for its rebuilt page list. +pub(crate) fn reload_descriptor(previous: [u32; 4], address: u64, pages: u32) -> [u32; 4] { + const COUNT: u32 = 0x003f_ffff; + let bias = if address & (1u64 << 42) == 0 { + 0x70_0000_0000 + } else { + 0 + }; + let segment = (address.wrapping_add(bias) & 0x70_0000_0000) | ((address >> 3) & 0x80_0000_0000); + [ + ((address >> 4) as u32 & !7) | (previous[0] & 6), + ((segment >> 8) as u32 & 0xf000_0000) | (previous[1] & 0x0fc0_0000) | (pages & COUNT), + previous[2] & !COUNT, + previous[3] & !(COUNT | 0x8000_0000), + ] +} + +/// Compact scratch and tagged metrics addresses carried by scene registers. +pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Result<(u64, u64)> { + const SEGMENT_SELECT: u64 = 1 << 42; + const SEGMENT_BIAS: u64 = 0x70_0000_0000; + const ADDRESS_MASK: u64 = 0x7f_ffff_fffe; + const HIGH_BIT: u64 = 0x80_0000_0000; + if index >= SCENES || metrics & 3 != 0 { + return Err(EINVAL); + } + let scratch = scratch + .checked_sub(cfg::USER_BASE) + .ok_or(EINVAL)? + .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) + .ok_or(EOVERFLOW)?; + if scratch > u32::MAX as u64 || scratch & 0xf != 0 { + return Err(EINVAL); + } + let metric = offset_va(metrics, (index * 4) as u64)?; + let adjusted = offset_va( + metric, + if metric & SEGMENT_SELECT == 0 { + SEGMENT_BIAS + } else { + 0 + }, + )?; + Ok(( + scratch, + (adjusted & ADDRESS_MASK) | ((metric >> 3) & HIGH_BIT) | 1, + )) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs new file mode 100644 index 00000000000000..c2c3b3eb7a80d3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-control records and work-ring slots. +//! +//! The host sends out-of-band requests and replies through the device-control +//! ring, whose 0x40-byte records start with a [`DeviceControlOp`]. It hands +//! hardware queues to the firmware through work rings, one per priority and +//! engine, each holding [`WORK_RING_SLOTS`] [`WorkSlot`]s. +//! +//! Producers and consumers of both rings are live words in the primary state +//! grid; they are not part of these structures. Before publishing a +//! device-control record whose opcode [`DeviceControlOp::asserts_gpu_power`], +//! the host also increments its power-assert tally. + +use kernel::prelude::*; + +use super::kick::QID_MAX; +use super::queue::{DataMaster, QUEUE_PRIORITY}; + +/// Size of a device-control record. +pub(crate) const DEVICE_CONTROL_RECORD_SIZE: usize = 0x40; +/// Number of records in the device-control ring. +pub(crate) const DEVICE_CONTROL_SLOTS: u32 = 256; + +/// Entries in either coprocessor's event ring. +pub(crate) const EVENT_SLOTS: u32 = 256; +/// Entries in either coprocessor's trace ring. +pub(crate) const TRACE_SLOTS: u32 = 512; +/// Entries in either coprocessor's statistics ring. +pub(crate) const STATISTICS_SLOTS: u32 = 256; +/// Words in an event, trace or statistics entry. +pub(crate) const EVENT_WORDS: usize = 18; +/// Byte stride of an event, trace or statistics entry. +pub(crate) const EVENT_RECORD_SIZE: usize = EVENT_WORDS * size_of::(); +/// Producer offset within a firmware-produced ring's state block. +pub(crate) const RX_PRODUCER_OFFSET: usize = 0x20; +/// Trace records use type 5. +pub(crate) const TRACE_TYPES: u32 = 1 << 5; +/// Statistics record types accepted by the host. +pub(crate) const STATISTICS_TYPES: u32 = 0x5fff_0000; + +/// Completion stamp indices signalled by the firmware. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct StampEventRecord { + tag: u32, + masks: [u64; 2], + diagnostic_pairs: u16, + unk_16: [u8; 0x32], +} + +/// Restart generation and the affected stamp, or -1 without attribution. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct RestartEventRecord { + tag: u32, + generation: u64, + stamp: i32, + unk_10: [u8; 0x38], +} + +/// Parameter-buffer growth request. The reply moves `halt_count` to +0x18. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct TvbGrowEventRecord { + tag: u32, + vm_slot: i32, + buffer_slot: u32, + counter: u32, + subpipe: u32, + halt_count: u64, + unk_1c: [u8; 0x2c], +} + +/// Completed asynchronous free-list growth. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +struct FreeListEventRecord { + tag: u32, + stamp: i32, + buffer_slot: u32, + control_va: u64, + ring_va: u64, + cookie: u64, + result: u32, + unk_28: [u8; 0x20], +} + +/// A snapshot of one firmware event. Every variant occupies a complete slot; +/// callers copy the entry before publishing its consumer cursor. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) union EventRecord { + words: [u32; EVENT_WORDS], + stamp: StampEventRecord, + restart: RestartEventRecord, + tvb: TvbGrowEventRecord, + free_list: FreeListEventRecord, +} + +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(size_of::() == EVENT_RECORD_SIZE); +static_assert!(core::mem::offset_of!(StampEventRecord, masks) == 4); +static_assert!(core::mem::offset_of!(TvbGrowEventRecord, halt_count) == 0x14); +static_assert!(core::mem::offset_of!(FreeListEventRecord, cookie) == 0x1c); + +impl core::fmt::Debug for EventRecord { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("EventRecord") + .field("tag", &self.tag()) + .finish() + } +} + +/// A decoded asynchronous free-list growth completion. Resource ownership and +/// the expected cookie/result are checked by the free-list owner. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct FreeListCompletion { + pub(crate) stamp: i32, + pub(crate) buffer_slot: u32, + pub(crate) control_va: u64, + pub(crate) ring_va: u64, + pub(crate) cookie: u64, + pub(crate) result: u32, +} + +/// Host actions requested by a firmware event. Unknown tags are consumed so +/// one unrecognized record cannot block later completions. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Event { + Stamp([u64; 2]), + Restart { generation: u64, stamp: i32 }, + TvbGrow(TvbGrowRequest), + FreeListGrow(FreeListCompletion), + ContextKilled(u64), + Unknown(u32), +} + +impl EventRecord { + const STAMP: u32 = 1; + const RESTART: u32 = 4; + const TVB_GROW: u32 = 6; + const FREE_LIST_GROW: u32 = 13; + const CONTEXT_KILLED: u32 = 12; + const DIAGNOSTIC_PAIRS_MAX: u16 = 5; + const TVB_SLOTS: u32 = 127; + const SUBPIPES: u32 = 64; + const FREE_LIST_SLOTS: u32 = 256; + + pub(crate) const fn from_words(words: [u32; EVENT_WORDS]) -> Self { + Self { words } + } + + pub(crate) fn tag(&self) -> u32 { + // SAFETY: All constructors initialize the entire union. Every bit + // pattern is valid for its u32 words, and word zero is the tag. + unsafe { self.words[0] } + } + + /// Validates the record's fields without interpreting resource ownership. + /// A malformed known event is an error, unlike an unknown tag. + pub(crate) fn decode(&self) -> Result { + match self.tag() { + Self::STAMP => { + // SAFETY: The entire slot is initialized, the tag selects + // this layout, and its integer fields have no invalid values. + let record = unsafe { self.stamp }; + if record.diagnostic_pairs > Self::DIAGNOSTIC_PAIRS_MAX { + return Err(EIO); + } + Ok(Event::Stamp(record.masks)) + } + Self::RESTART => { + // SAFETY: As above, for the restart layout. + let record = unsafe { self.restart }; + Ok(Event::Restart { + generation: record.generation, + stamp: record.stamp, + }) + } + Self::TVB_GROW => { + // SAFETY: As above, for the parameter-buffer growth layout. + let record = unsafe { self.tvb }; + if record.vm_slot < -1 + || record.buffer_slot >= Self::TVB_SLOTS + || record.subpipe >= Self::SUBPIPES + { + return Err(EIO); + } + Ok(Event::TvbGrow(TvbGrowRequest { + buffer_slot: record.buffer_slot, + vm_slot: record.vm_slot, + counter: record.counter, + subpipe: record.subpipe, + halt_count: record.halt_count, + })) + } + Self::CONTEXT_KILLED => { + // SAFETY: Every constructor initializes all integer words. + let words = unsafe { self.words }; + Ok(Event::ContextKilled( + u64::from(words[1]) | (u64::from(words[2]) << 32), + )) + } + Self::FREE_LIST_GROW => { + // SAFETY: As above, for the free-list growth layout. + let record = unsafe { self.free_list }; + if record.stamp < -1 + || record.buffer_slot >= Self::FREE_LIST_SLOTS + || record.control_va == 0 + || record.ring_va == 0 + { + return Err(EIO); + } + Ok(Event::FreeListGrow(FreeListCompletion { + stamp: record.stamp, + buffer_slot: record.buffer_slot, + control_va: record.control_va, + ring_va: record.ring_va, + cookie: record.cookie, + result: record.result, + })) + } + tag => Ok(Event::Unknown(tag)), + } + } +} + +/// A complete device-control request. Its variant supplies the power policy; +/// callers construct the contained record with its typed builder. +#[derive(Debug)] +pub(crate) enum ControlRecord { + TvbGrow(TvbGrowReply), + IdlePowerOff(IdlePowerOff), + SchedulerStateRelease(SchedulerStateRelease), + FreeListRelease(FreeListRelease), + ContextKill(ContextKill), +} + +impl ControlRecord { + pub(crate) const fn opcode(&self) -> DeviceControlOp { + match self { + Self::TvbGrow(_) => DeviceControlOp::TvbGrowReply, + Self::IdlePowerOff(_) => DeviceControlOp::IdlePowerOff, + Self::SchedulerStateRelease(_) => DeviceControlOp::SchedulerStateRelease, + Self::FreeListRelease(_) => DeviceControlOp::FreeListRelease, + Self::ContextKill(_) => DeviceControlOp::ContextKill, + } + } + + /// Byte view of the initialized record, excluding the host-only enum tag. + pub(crate) fn bytes(&self) -> &[u8; DEVICE_CONTROL_RECORD_SIZE] { + let record = match self { + Self::TvbGrow(record) => core::ptr::from_ref(record).cast(), + Self::IdlePowerOff(record) => core::ptr::from_ref(record).cast(), + Self::SchedulerStateRelease(record) => core::ptr::from_ref(record).cast(), + Self::FreeListRelease(record) => core::ptr::from_ref(record).cast(), + Self::ContextKill(record) => core::ptr::from_ref(record).cast(), + }; + // SAFETY: Each variant has a statically checked size of 0x40 bytes, + // contains only initialized integer fields with no implicit padding, + // and is borrowed for the lifetime of `self`. The byte view has align 1. + unsafe { &*record } + } +} + +/// Device-control opcodes the driver sends. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum DeviceControlOp { + /// Reply to a parameter-buffer grow request ([`TvbGrowReply`]). + TvbGrowReply = 0x08, + /// Set the idle power-off policy ([`IdlePowerOff`]). + IdlePowerOff = 0x0a, + /// Release a logical queue's scheduler state ([`SchedulerStateRelease`]). + SchedulerStateRelease = 0x14, + /// Release a free list's buffer slot ([`FreeListRelease`]). + FreeListRelease = 0x2e, + /// Stop the queues named by a closed logical scheduler. + ContextKill = 0x29, +} + +impl DeviceControlOp { + /// Tells whether records with this opcode count towards the host + /// power-assert tally. The firmware compares the tally with the counted + /// records it has drained and keeps the GPU cores powered while they + /// differ. + pub(crate) const fn asserts_gpu_power(self) -> bool { + !matches!(self, Self::FreeListRelease) + } +} + +/// The fields of a parameter-buffer grow request event that its reply +/// echoes. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct TvbGrowRequest { + /// Buffer slot of the parameter buffer. + pub(crate) buffer_slot: u32, + /// VM slot of the request, or -1. + pub(crate) vm_slot: i32, + /// Request counter. + pub(crate) counter: u32, + /// Sub-pipe of the request. + pub(crate) subpipe: u32, + /// Firmware halt count the request was made under. + pub(crate) halt_count: u64, +} + +/// Reply to a parameter-buffer grow request. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct TvbGrowReply { + /// [`DeviceControlOp::TvbGrowReply`]. + pub(crate) opcode: u32, + /// Non-zero when the parameter buffer grew. + pub(crate) grew: u32, + /// Buffer slot of the request. + pub(crate) buffer_slot: u32, + /// VM slot of the request. + pub(crate) vm_slot: i32, + /// Request counter. + pub(crate) counter: u32, + /// Sub-pipe of the request. + pub(crate) subpipe: u32, + /// Halt count of the request. + pub(crate) halt_count: u64, + pub(crate) unk_20: [u8; 0x20], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); + +impl TvbGrowReply { + /// Replies to `request`; `grew` tells whether the buffer was grown. + pub(crate) const fn new(request: &TvbGrowRequest, grew: bool) -> Self { + Self { + opcode: DeviceControlOp::TvbGrowReply as u32, + grew: grew as u32, + buffer_slot: request.buffer_slot, + vm_slot: request.vm_slot, + counter: request.counter, + subpipe: request.subpipe, + halt_count: request.halt_count, + unk_20: [0; 0x20], + } + } +} + +/// Idle power-off policy argument of [`IdlePowerOff`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum IdlePolicy { + /// Keep the GPU cores powered and power them up now. + KeepPowered = 0, + /// Let the firmware power the GPU cores off when idle. + AllowPowerOff = 1, +} + +/// Sets the firmware's idle power-off policy. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct IdlePowerOff { + /// [`DeviceControlOp::IdlePowerOff`]. + pub(crate) opcode: u32, + /// [`IdlePolicy`]. + pub(crate) policy: u32, + pub(crate) unk_08: [u8; 0x38], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); + +impl IdlePowerOff { + /// Requests `policy`. + pub(crate) const fn new(policy: IdlePolicy) -> Self { + Self { + opcode: DeviceControlOp::IdlePowerOff as u32, + policy: policy as u32, + unk_08: [0; 0x38], + } + } +} + +/// Releases the scheduler state of a logical queue whose work has drained. +/// +/// The page may be freed once the firmware has consumed the record. +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct SchedulerStateRelease { + /// [`DeviceControlOp::SchedulerStateRelease`]. + pub(crate) opcode: u32, + pub(crate) unk_04: u32, + /// The scheduler-page bytes at offsets 0x26, 0, 1 and 4, in record order. + /// Read when the release record is built. + pub(crate) id: [u8; 4], + /// GPU address of the scheduler-state page. + pub(crate) page_va: u64, + pub(crate) unk_14: [u8; 0x2c], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(SchedulerStateRelease, page_va) == 0x0c); + +impl SchedulerStateRelease { + /// Releases the scheduler page with its identity bytes in record order. + pub(crate) const fn new(page_va: u64, id: [u8; 4]) -> Self { + Self { + opcode: DeviceControlOp::SchedulerStateRelease as u32, + unk_04: 0, + id, + page_va, + unk_14: [0; 0x2c], + } + } +} + +/// Releases the buffer slot of a free list after its last user. +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct FreeListRelease { + /// [`DeviceControlOp::FreeListRelease`]. + pub(crate) opcode: u32, + /// Free-list generation of the registration being released. + pub(crate) generation: u64, + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + pub(crate) unk_10: [u8; 0x30], +} + +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(FreeListRelease, buffer_slot) == 0x0c); + +impl FreeListRelease { + /// Releases `buffer_slot`, registered at `generation`. + pub(crate) const fn new(generation: u64, buffer_slot: u32) -> Self { + Self { + opcode: DeviceControlOp::FreeListRelease as u32, + generation, + buffer_slot, + unk_10: [0; 0x30], + } + } +} + +/// Number of slots in a work ring. +pub(crate) const WORK_RING_SLOTS: u32 = 256; +/// Number of engines with a work ring per priority. +const WORK_RINGS_PER_PRIORITY: usize = 3; + +/// Index in the channel table of the work ring for `data_master` at the +/// priority of every driver-created queue. +pub(crate) const fn work_ring_index(data_master: DataMaster) -> usize { + QUEUE_PRIORITY as usize * WORK_RINGS_PER_PRIORITY + data_master as usize +} + +/// One work-ring slot: a request to run a queue's newly published commands. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct WorkSlot { + pub(crate) unk_00: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u32, + /// Item-ring producer after the published commands. + pub(crate) items_end: u16, + /// Queue to run. + pub(crate) qid: u8, + /// One in the first slot naming this queue on this ring. + pub(crate) first: u8, +} + +static_assert!(size_of::() == 0x18); + +impl WorkSlot { + /// Builds the slot asking the firmware to run queue `qid` up to item-ring + /// position `items_end`. `first` tells whether this is the first slot + /// naming the queue on this ring. + pub(crate) fn new( + data_master: DataMaster, + queue_va: u64, + qid: u8, + items_end: u16, + first: bool, + ) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + unk_00: 0, + queue_va, + data_master: data_master as u32, + items_end, + qid, + first: first as u8, + }) + } +} + +/// Stop a closed context's scheduler page and echo its cookie through event 12. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct ContextKill { + opcode: u32, + page_va: u64, + cookie: u64, + reserved: [u8; 0x2c], +} +static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); +static_assert!(core::mem::offset_of!(ContextKill, page_va) == 4); +static_assert!(core::mem::offset_of!(ContextKill, cookie) == 0x0c); +impl ContextKill { + pub(crate) fn new(page_va: u64, cookie: u64) -> Self { + Self { + opcode: DeviceControlOp::ContextKill as u32, + page_va, + cookie, + reserved: [0; 0x2c], + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs new file mode 100644 index 00000000000000..22f8259508ff65 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -0,0 +1,379 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Compute work descriptor. +//! +//! A compute kick launches one [`ComputeDescriptor`]. The descriptor embeds +//! the two register arrays its kick entry binds: the main array and a +//! four-entry array that writes the operand-state registers with their +//! regions exchanged. + +use kernel::prelude::*; + +use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; +use super::queue::CommandTag; +use super::{ + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, +}; + +/// Size of a compute descriptor. +pub(crate) const COMPUTE_DESCRIPTOR_SIZE: usize = 0x4000; +/// QoS class of compute kicks. +pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; +/// Event mask of compute kicks. +pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; + +const COMPUTE_REGISTERS: usize = 40; +const ALT_REGISTERS: usize = 4; + +/// Object IDs of every compute kick. +const OBJECT_IDS: ObjectIds = ObjectIds { + predecessor: 0x0100_01d7, + current: 0x0200_01dc, +}; +/// Values of the first and second write of register 0x1a440. +const UNK_1A440: u64 = 0x0000_0001_5402_4201; +const UNK_1A440_FINAL: u64 = 0x0000_0001_5402_4209; +/// Value of register 0x1a458: [`UNK_1C8F8`] below bits 31:20 = 0x10c. +const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; + +/// Offsets in the preempt buffer of the four state words the program names. +const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; +/// Offsets in the operand-state buffer of the regions the program names. +const OPERAND_STATE_10229: u64 = 0x12800; +const OPERAND_STATE_140A8: u64 = 0x13000; +const OPERAND_STATE_10099: u64 = 0x9405; +const OPERAND_STATE_10091: u64 = 0x12400; +const OPERAND_STATE_0A5C1: u64 = 0x0005; +const OPERAND_STATE_0A5C9: u64 = 0x9000; + +/// Checkpoints into the main register array, as entry counts: after the +/// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full +/// count. +const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; + +/// Inputs of one compute descriptor. +pub(crate) struct ComputeArgs { + /// Base of the queue's USC window. + pub(crate) usc_base: u64, + /// GPU address of the control stream. + pub(crate) cdm_va: u64, + /// GPU address of the control-stream terminator. + pub(crate) cdm_end_va: u64, + /// GPU address of the sampler heap, or zero. + pub(crate) sampler_heap_va: u64, + /// Number of samplers in the heap. + pub(crate) sampler_count: u32, + /// GPU address of this descriptor, as seen by the work's VM. + pub(crate) descriptor_va: u64, + /// UAT context the work runs in. + pub(crate) context_id: u16, + /// Allocation generation of that context. + pub(crate) context_generation: u8, + /// State word of the logical queue's work-state node. + pub(crate) work_state: u32, + /// GPU address of the logical queue's work-state node. + pub(crate) work_state_va: u64, + /// Queue the work is kicked on. + pub(crate) qid: u8, + /// Number of earlier kicks of the queue. + pub(crate) kick_count: u32, + /// Timestamp the work is kicked at. + pub(crate) kick: KickTimestamp, + /// GPU address of the preempt buffer, as seen by the work's VM. + pub(crate) preempt_va: u64, + /// GPU address of the operand-state buffer, as seen by the work's VM. + pub(crate) operand_state_va: u64, + /// GPU address of the private-memory usage record, as seen by the work's + /// VM. + pub(crate) usage_va: u64, + /// Firmware alias of the private-memory usage record. + pub(crate) usage_fw_va: u64, + /// Buffer slot of the USC private-memory free list. + pub(crate) free_list_slot: u32, + /// GPU address of the free list's shared control object. + pub(crate) free_list_control_va: u64, + /// GPU address the completion stamp value is stored to. + pub(crate) stamp_va: u64, + /// GPU address of the word following the completion stamp. + pub(crate) aux_stamp_va: u64, + /// GPU addresses of the two status words. + pub(crate) status_va: [u64; 2], + /// GPU address of the start/end timestamp pair. + pub(crate) timestamp_va: u64, +} + +fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS]> { + let preempt = args.preempt_va; + let operand = args.operand_state_va; + let key = work_key(args.context_id, args.work_state); + let ids = OBJECT_IDS.word(); + + Ok([ + RegisterWrite::new(0x1a510, preempt), + RegisterWrite::new(0x1a420, args.cdm_va), + RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), + RegisterWrite::new(0x1a4d8, offset_va(preempt, PREEMPT_STATE[1])?), + RegisterWrite::new(0x1a4e0, offset_va(preempt, PREEMPT_STATE[2])?), + RegisterWrite::new(0x1a4e8, offset_va(preempt, PREEMPT_STATE[3])?), + RegisterWrite::new(0x10071, args.usc_base), + // Helper program binary, data and configuration: none. + RegisterWrite::new(0x11841, 0), + RegisterWrite::new(0x11849, 0), + RegisterWrite::new(0x11f81, 0), + RegisterWrite::new(0x1a440, UNK_1A440), + RegisterWrite::new(0x1a458, UNK_1A458), + RegisterWrite::new(0x101d9, UNK_101D9), + RegisterWrite::new(0x1a089, 0), + RegisterWrite::new(0x1a091, 0), + RegisterWrite::new(0x1a059, 0), + RegisterWrite::new(0x1a061, 0), + RegisterWrite::new(0x1a0b9, 0), + RegisterWrite::new(0x1a0c1, 0), + RegisterWrite::new(0x101d1, 0), + RegisterWrite::new(0x0d479, 0), + RegisterWrite::new(0x1a0e9, UNK_1A0E9), + RegisterWrite::new(0x107a1, UNK_10791), + RegisterWrite::new(0x0a599, UNK_0A599), + RegisterWrite::new(0x0d411, UNK_0D411), + RegisterWrite::new(0x1a540, ids), + RegisterWrite::new(0x014a9, ids), + RegisterWrite::new(0x0a351, ids), + RegisterWrite::new(0x10201, key), + RegisterWrite::new(0x10428, key), + RegisterWrite::new(0x14028, args.free_list_slot as u64), + RegisterWrite::new(0x14070, args.usage_va | 1), + RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), + RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x1a440, UNK_1A440_FINAL), + RegisterWrite::new(0x0a599, UNK_0A599), + ]) +} + +/// The second array exchanges the operand-state regions of registers +/// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. +fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { + let operand = args.operand_state_va; + Ok([ + RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), + RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), + RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), + RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), + ]) +} + +/// Compute work descriptor (tag 3). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct ComputeDescriptor { + /// [`CommandTag::Compute`]. + pub(crate) tag: u32, + pub(crate) unk_04: u64, + /// UAT context of the work. + pub(crate) context_id: u32, + /// GPU address of the logical queue's work-state node. + pub(crate) work_state_va: u64, + /// Checkpoints into the main register array. + pub(crate) checkpoints: [u16; 4], + pub(crate) unk_20: [u8; 0x20], + /// Main register array. + pub(crate) registers: RegisterArray, + /// Second register array. + pub(crate) alt_registers: RegisterArray, + pub(crate) unk_e80: [u8; 0x58], + /// GPU address of the preempt buffer. + pub(crate) preempt_va: u64, + /// GPU address of the control-stream terminator. + pub(crate) cdm_end_va: u64, + pub(crate) unk_ee8: [u8; 0x20], + /// First value of register 0x1a440. + pub(crate) unk_f08: u64, + pub(crate) unk_f10: [u8; 0x10], + /// Predecessor object ID. + pub(crate) predecessor_id: u32, + pub(crate) unk_f24: u32, + pub(crate) unk_f28: u32, + /// GPU address of the sampler heap. + pub(crate) sampler_heap_va: u64, + /// Number of samplers. + pub(crate) sampler_count: u32, + /// Number of samplers plus one, or zero without samplers. + pub(crate) sampler_max: u32, + pub(crate) unk_f3c: u32, + /// GPU address the firmware stores `stamp_value` to on completion. + pub(crate) stamp_va: u64, + /// GPU address of the word following the completion stamp. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the kick timestamp. + pub(crate) stamp_value: u32, + /// QID of the compute queue. + pub(crate) qid: u32, + pub(crate) unk_f58: u32, + pub(crate) unk_f5c: u32, + /// Kick timestamp. + pub(crate) kick_timestamp: u64, + /// Current object ID. + pub(crate) current_id: u64, + /// Number of earlier kicks of the queue. + pub(crate) kick_count: u32, + pub(crate) unk_f74: [u8; 8], + /// GPU addresses of the two status words. + pub(crate) status_va: [u64; 2], + /// GPU address the start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// GPU address the end timestamp is written to. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_f9c: [u8; 0x14], + pub(crate) unk_fb0: u16, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_fba: u32, + pub(crate) unk_fbe: [u8; 7], + pub(crate) unk_fc5: u8, + pub(crate) unk_fc6: [u8; 5], + /// Firmware alias of the private-memory usage record. + pub(crate) usage_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_fd4: [u8; 0x302c], +} + +static_assert!(size_of::() == COMPUTE_DESCRIPTOR_SIZE); +static_assert!(core::mem::offset_of!(ComputeDescriptor, registers) == 0x40); +static_assert!(core::mem::offset_of!(ComputeDescriptor, alt_registers) == 0x760); +static_assert!(core::mem::offset_of!(ComputeDescriptor, preempt_va) == 0xed8); +static_assert!(core::mem::offset_of!(ComputeDescriptor, unk_f08) == 0xf08); +static_assert!(core::mem::offset_of!(ComputeDescriptor, predecessor_id) == 0xf20); +static_assert!(core::mem::offset_of!(ComputeDescriptor, sampler_heap_va) == 0xf2c); +static_assert!(core::mem::offset_of!(ComputeDescriptor, stamp_va) == 0xf40); +static_assert!(core::mem::offset_of!(ComputeDescriptor, kick_timestamp) == 0xf60); +static_assert!(core::mem::offset_of!(ComputeDescriptor, current_id) == 0xf68); +static_assert!(core::mem::offset_of!(ComputeDescriptor, status_va) == 0xf7c); +static_assert!(core::mem::offset_of!(ComputeDescriptor, free_list_control_va) == 0xfb2); +static_assert!(core::mem::offset_of!(ComputeDescriptor, usage_va) == 0xfcb); +static_assert!(core::mem::offset_of!(ComputeDescriptor, context_generation) == 0xfd3); + +// SAFETY: `ComputeDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for ComputeDescriptor {} + +impl ComputeDescriptor { + const REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; + const ALT_REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, alt_registers) as u64; + const UNK_F28: u32 = u32::MAX; + const UNK_FB0: u16 = 0x1a; + const UNK_FBA: u32 = 0xe0a0_0001; + const UNK_FC5: u8 = 0x9f; + + /// Writes the complete descriptor. + pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { + if args.cdm_end_va < args.cdm_va + || (args.sampler_heap_va == 0) != (args.sampler_count == 0) + || args.sampler_heap_va & 7 != 0 + { + return Err(EINVAL); + } + let registers = compute_registers(args)?; + let alt_registers = alt_registers(args)?; + let registers_va = offset_va(args.descriptor_va, Self::REGISTERS_OFFSET)?; + let alt_registers_va = offset_va(args.descriptor_va, Self::ALT_REGISTERS_OFFSET)?; + let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; + let sampler_max = sampler_max(args.sampler_count)?; + + clear(self); + self.tag = CommandTag::Compute as u32; + self.context_id = args.context_id.into(); + self.work_state_va = args.work_state_va; + self.checkpoints = CHECKPOINTS; + self.registers.set(registers_va, ®isters)?; + self.alt_registers.set(alt_registers_va, &alt_registers)?; + self.preempt_va = args.preempt_va; + self.cdm_end_va = args.cdm_end_va; + self.unk_f08 = UNK_1A440; + self.predecessor_id = OBJECT_IDS.predecessor; + self.unk_f28 = Self::UNK_F28; + self.sampler_heap_va = args.sampler_heap_va; + self.sampler_count = args.sampler_count; + self.sampler_max = sampler_max; + self.stamp_va = args.stamp_va; + self.aux_stamp_va = args.aux_stamp_va; + self.stamp_value = stamp_value(args.kick.get() as u32); + self.qid = args.qid.into(); + self.kick_timestamp = args.kick.get(); + self.current_id = OBJECT_IDS.current.into(); + self.kick_count = args.kick_count; + self.status_va = args.status_va; + self.timestamp_start_va = args.timestamp_va; + self.timestamp_end_va = timestamp_end_va; + self.unk_fb0 = Self::UNK_FB0; + self.free_list_control_va = args.free_list_control_va; + self.unk_fba = Self::UNK_FBA; + self.unk_fc5 = Self::UNK_FC5; + self.usage_va = args.usage_fw_va; + self.context_generation = args.context_generation; + Ok(()) + } + + /// Returns the register arrays a compute kick binds; `descriptor_va` is + /// the descriptor's address as seen by the work's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let registers = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, + CHECKPOINTS[0] as u8, + )?; + let alt_registers = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, + ALT_REGISTERS as u8, + )?; + Ok([Some(registers), Some(alt_registers), None, None]) + } +} + +/// One retained queue-context record. The descriptor pointer is refreshed for each compute +/// publication; the remaining fields are installed once with the queue. +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct QueueContextItem { + unk_00: u64, + unk_08: u64, + descriptor_va: u64, + queue_va: u64, + unk_20: u64, + qid: u64, + unk_30: [u8; 0x100], + unk_130: u64, + unk_138: u64, + unk_140: [u8; 0x10], + unk_150: u64, + unk_158: u64, + unk_160: [u8; 0x18], + unk_178: u64, + unk_180: [u8; 0x80], +} +impl QueueContextItem { + pub(crate) fn new(descriptor_va: u64, queue_va: u64, qid: u8) -> Self { + Self { + unk_00: 0x1000_1000_0000_0004, + unk_08: 0, + descriptor_va, + queue_va, + unk_20: 0xffff_0801_0000_0001, + qid: u64::from(qid) << 40, + unk_30: [0; 0x100], + unk_130: 0, + unk_138: 4, + unk_140: [0; 0x10], + unk_150: 0x0001_1003_8001_a002, + unk_158: 0x0000_2003_8001_a03b, + unk_160: [0; 0x18], + unk_178: 0x003f_ffff_ffff_ffff, + unk_180: [0; 0x80], + } + } +} +static_assert!(size_of::() == 0x200); diff --git a/drivers/gpu/drm/asahi/g17/fw/context.rs b/drivers/gpu/drm/asahi/g17/fw/context.rs new file mode 100644 index 00000000000000..51dbde231e2382 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/context.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler and timestamp-queue state shared with firmware. Only a freshly +//! leased node may be initialized. After publication, the host writes its +//! separate submitted counter and reads the node completion word individually. + +use kernel::prelude::*; + +#[repr(C)] +pub(crate) struct Scheduler { + unk0: u64, + unk8: [u64; 3], + unk20: u64, + unk28: u64, + unk30: u64, + unk38: [u8; 0x3fc8], +} +impl Scheduler { + /// Called on zeroed storage before its first publication. + pub(crate) fn init(&mut self) { + self.unk0 = 0x0000_0100_0000_ffff; + self.unk20 = 0x0002_0000_0000_0000; + self.unk30 = 0x0000_0000_ff00_0000; + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub(crate) struct WorkNode { + pub(crate) submitted: u64, + pub(crate) word: u32, + pub(crate) completed: u32, + pub(crate) threshold: u32, + unk14: [u32; 45], +} +impl WorkNode { + pub(crate) fn new(submitted: u64, word: u32) -> Option { + if submitted == 0 || submitted & 3 != 0 || word & 63 == 63 { + return None; + } + Some(Self { + submitted, + word, + completed: 0, + threshold: 0x50, + unk14: [0; 45], + }) + } +} + +#[derive(Clone, Copy)] +#[repr(C)] +pub(crate) struct WorkHead { + pub(crate) next: u64, + pub(crate) previous: u64, + pub(crate) unk10: u32, + pub(crate) unk14: u32, +} + +const _: () = { + assert!(core::mem::size_of::() == 0x4000); + assert!(core::mem::size_of::() == 0xc8); + assert!(core::mem::offset_of!(WorkNode, completed) == 0xc); + assert!(core::mem::size_of::() == 0x18); +}; + +// SAFETY: These shared records contain only integer fields and arrays. +unsafe impl Zeroable for Scheduler {} +// SAFETY: WorkNode contains only integers; zero is valid for every field. +unsafe impl Zeroable for WorkNode {} +// SAFETY: WorkHead contains only integer addresses and words. +unsafe impl Zeroable for WorkHead {} diff --git a/drivers/gpu/drm/asahi/g17/fw/freelist.rs b/drivers/gpu/drm/asahi/g17/fw/freelist.rs new file mode 100644 index 00000000000000..2e5708dddd4552 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/freelist.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! USC free-list page addresses and block run tables. + +use super::initdata::{FreeListRunSlot, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; +use kernel::prelude::*; + +pub(crate) const PAGE_LIST_SIZE: usize = 0x20_0000; +pub(crate) const RUN_LIST_SIZE: usize = 0x1_0000; +pub(crate) const TABLE_SIZE: usize = PAGE_LIST_SIZE + RUN_LIST_SIZE; +pub(crate) const CONTROL_SIZE: usize = 0x4000; +pub(crate) const RENDER_BLOCKS: usize = 22; +pub(crate) const COMPUTE_BLOCKS: usize = 21; + +/// Render reserves eight run words per block; compute packs runs consecutively. +#[derive(Copy, Clone, Debug)] +pub(crate) enum RunLayout { + Render, + Compute, +} + +impl RunLayout { + const fn words(self) -> usize { + match self { + Self::Render => size_of::() / 8, + Self::Compute => 1, + } + } +} + +#[repr(C)] +pub(crate) struct PoolTables { + pages: [u64; PAGE_LIST_SIZE / 8], + runs: [u64; RUN_LIST_SIZE / 8], +} + +// SAFETY: The table consists entirely of integer arrays. +unsafe impl Zeroable for PoolTables {} +static_assert!(size_of::() == TABLE_SIZE); +static_assert!(core::mem::offset_of!(PoolTables, runs) == PAGE_LIST_SIZE); + +impl PoolTables { + /// Enumerates only the initialized run words and page addresses, in their + /// publication order. Reserved run words and unused table space stay zero. + pub(crate) fn for_each_word( + base: u64, + blocks: usize, + layout: RunLayout, + mut put: impl FnMut(usize, u64), + ) -> Result { + if blocks > RUN_LIST_SIZE / (layout.words() * 8) + || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) + { + return Err(EINVAL); + } + let buffers = base.checked_add(TABLE_SIZE as u64).ok_or(EOVERFLOW)?; + buffers + .checked_add((blocks * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + for block in 0..blocks { + let address = buffers + (block * FREE_LIST_BLOCK_SIZE) as u64; + put( + PAGE_LIST_SIZE / 8 + block * layout.words(), + FreeListRunSlot::new(address).run, + ); + for page in 0..FREE_LIST_BLOCK_PAGES as usize { + put( + block * FREE_LIST_BLOCK_PAGES as usize + page, + address + + (page * (FREE_LIST_BLOCK_SIZE / FREE_LIST_BLOCK_PAGES as usize)) as u64, + ); + } + } + Ok(()) + } + + /// Initializes a zeroed table before its first publication. + pub(crate) fn init(&mut self, base: u64, blocks: usize, layout: RunLayout) -> Result { + Self::for_each_word(base, blocks, layout, |index, value| { + if index < self.pages.len() { + self.pages[index] = value; + } else { + self.runs[index - self.pages.len()] = value; + } + }) + } +} + +/// Mutable page-pool state carried between the retained control record and +/// its global descriptor. Offsets name the source control-record words. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PagePoolState { + pub(crate) page_list_va: u64, + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + pub(crate) page_count: u32, +} + +impl PagePoolState { + /// The fourth descriptor word belongs to firmware and survives every load. + pub(crate) const fn descriptor(self, unk_18: u64) -> super::initdata::PagePoolDescriptor { + super::initdata::PagePoolDescriptor { + page_list: ((self.unk_1c as u64) << 41) | (self.page_list_va >> 7), + counters: ((self.unk_24 as u64) << 33) + | ((self.unk_20 as u64) << 5) + | ((self.unk_28 as u64) << 61), + page_count: self.page_count as u64, + unk_18, + } + } + + /// Address and allocation unit remain properties of the retained pool. + pub(crate) const fn updated(self, descriptor: &super::initdata::PagePoolDescriptor) -> Self { + Self { + unk_20: ((descriptor.counters >> 5) & 0x0fff_ffff) as u32, + unk_24: ((descriptor.counters >> 33) & 0x0fff_ffff) as u32, + unk_28: (descriptor.counters >> 61) as u32, + page_count: descriptor.page_count as u32, + ..self + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs new file mode 100644 index 00000000000000..c25b843a913e72 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -0,0 +1,3063 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware initialization data. +//! +//! Each of the two firmware instances (primary and secondary) is handed one +//! root at boot. The roots name the objects the firmware needs before any +//! work is submitted: +//! +//! - region A and region C, shared by both instances; +//! - one main configuration per instance, holding its channel table; +//! - the hardware-data bundle, one allocation holding the hardware data, the +//! power configuration, the work rings and both main configurations with +//! their device-control rings; +//! - per-instance status blocks inside the private cluster, the fixed-address +//! arena the firmware also writes at fixed offsets. +//! +//! This module defines those layouts and the pure functions that fill them. +//! Allocation, mapping and publication belong to the builder. Initialization +//! requires exclusive access before either firmware instance can access the +//! object. These constructors must not be used to update a published object: +//! its live fields require individual accesses of the widths documented below. +//! +//! All objects start out zeroed. An `unk_` field documents the value +//! the firmware requires there; an `unk_` field without a doc comment is zero. + +use kernel::prelude::*; + +use super::clear; + +/// Interface version at root `+0x00`; the firmware rejects any other. +pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; + +/// UAT page size, as a shift. +pub(crate) const PAGE_BITS: u32 = 14; +/// UAT page size. +pub(crate) const PAGE_SIZE: usize = 1 << PAGE_BITS; +/// Page-table levels below the root. +const UAT_LEVELS: usize = 3; +/// Address bits resolved by one page-table level. +const BITS_PER_LEVEL: u32 = PAGE_BITS - 3; + +/// Firmware instance, encoded in its root. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum Role { + /// Runs the scheduler, the device-control ring and all work channels. + Primary = 0, + /// Control-only peer of the primary. + Secondary = 1, +} + +// --------------------------------------------------------------------------- +// Roots +// --------------------------------------------------------------------------- + +/// One UAT page-table level, as the firmware walks it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct LevelDescriptor { + /// Required: 8. + pub(crate) unk_0: u8, + /// Page size shift of the tables at this level. + pub(crate) table_page_bits: u8, + /// Page size shift of the pages they map. + pub(crate) page_bits: u8, + /// Lowest input-address bit indexing this level. + pub(crate) index_shift: u8, + /// Entries per table. + pub(crate) entries: u16, + /// Table size in bytes. + pub(crate) table_size: u16, + /// Required: 1. + pub(crate) unk_8: u64, + /// Output-address bits of an entry. + pub(crate) phys_mask: u64, + /// Input-address bits indexing this level. + pub(crate) index_mask: u64, +} + +// SAFETY: `LevelDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for LevelDescriptor {} + +static_assert!(size_of::() == 0x20); + +impl LevelDescriptor { + /// Returns the descriptor of `level` (0 = top) for a UAT with `ias` + /// input-address and `oas` output-address bits. + const fn new(level: u32, ias: u32, oas: u32) -> Self { + let index_shift = PAGE_BITS + (UAT_LEVELS as u32 - 1 - level) * BITS_PER_LEVEL; + let index_bits = if level == 0 { + ias - index_shift + } else { + BITS_PER_LEVEL + }; + let entries = 1u64 << index_bits; + Self { + unk_0: 8, + table_page_bits: PAGE_BITS as u8, + page_bits: PAGE_BITS as u8, + index_shift: index_shift as u8, + entries: entries as u16, + table_size: PAGE_SIZE as u16, + unk_8: 1, + phys_mask: ((1u64 << oas) - 1) & !(PAGE_SIZE as u64 - 1), + index_mask: (entries - 1) << index_shift, + } + } +} + +/// Addresses and geometry common to both roots. +pub(crate) struct RootArgs { + /// Region A. + pub(crate) region_a_va: u64, + /// Region C. + pub(crate) region_c_va: u64, + /// This instance's main configuration. + pub(crate) main_config_va: u64, + /// This instance's status A block. + pub(crate) status_a_va: u64, + /// UAT input-address bits of the firmware's context. + pub(crate) ias: u32, + /// UAT output-address bits. + pub(crate) oas: u32, +} + +/// Root of the primary instance; the secondary extends it ([`SecondaryRoot`]). +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct Root { + /// [`INTERFACE_VERSION`]. + pub(crate) version: [u16; 4], + /// Region A. + pub(crate) region_a_va: u64, + pub(crate) unk_10: u64, + /// Main configuration of this instance. + pub(crate) main_config_va: u64, + /// Region C. + pub(crate) region_c_va: u64, + /// [`Role`] of this instance. + pub(crate) role: u32, + /// Required: 1. + pub(crate) unk_2c: u32, + /// UAT page size. + pub(crate) page_size: u16, + /// UAT page size shift. + pub(crate) page_bits: u8, + /// Number of page-table levels. + pub(crate) levels: u8, + /// Page-table levels, top first. + pub(crate) level: [LevelDescriptor; UAT_LEVELS], + pub(crate) unk_94: [u8; 0x14], + /// Status A block of this instance. + pub(crate) status_a_va: u64, + /// Primary status B ([`PrimaryStatusB`]); zero for the secondary. + pub(crate) status_b_va: u64, +} + +// SAFETY: `Root` consists of integers and arrays of integers only. +unsafe impl Zeroable for Root {} + +static_assert!(core::mem::offset_of!(Root, role) == 0x28); +static_assert!(core::mem::offset_of!(Root, level) == 0x34); +static_assert!(core::mem::offset_of!(Root, status_a_va) == 0xa8); +static_assert!(size_of::() == 0xb8); + +impl Root { + fn new(role: Role, args: &RootArgs, status_b_va: u64) -> Self { + Self { + version: INTERFACE_VERSION, + region_a_va: args.region_a_va, + unk_10: 0, + main_config_va: args.main_config_va, + region_c_va: args.region_c_va, + role: role as u32, + unk_2c: 1, + page_size: PAGE_SIZE as u16, + page_bits: PAGE_BITS as u8, + levels: UAT_LEVELS as u8, + level: [ + LevelDescriptor::new(0, args.ias, args.oas), + LevelDescriptor::new(1, args.ias, args.oas), + LevelDescriptor::new(2, args.ias, args.oas), + ], + unk_94: [0; 0x14], + status_a_va: args.status_a_va, + status_b_va, + } + } + + /// Returns the primary root. + pub(crate) fn primary(args: &RootArgs, status_b_va: u64) -> Self { + Self::new(Role::Primary, args, status_b_va) + } +} + +/// Root of the secondary instance. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct SecondaryRoot { + /// Common part; the secondary has no status B. + pub(crate) root: Root, + /// The primary's [`PowerState`]. + pub(crate) power_state_va: u64, + /// The secondary's [`SecondaryConfig`]. + pub(crate) config_va: u64, +} + +// SAFETY: `SecondaryRoot` consists of integers and arrays of integers only. +unsafe impl Zeroable for SecondaryRoot {} + +static_assert!(size_of::() == 0xc8); + +impl SecondaryRoot { + /// Returns the secondary root. + pub(crate) fn new(args: &RootArgs, power_state_va: u64, config_va: u64) -> Self { + Self { + root: Root::new(Role::Secondary, args, 0), + power_state_va, + config_va, + } + } +} + +/// Offset of the secondary root from the primary root. Both live in one +/// allocation, and the firmware relies on this distance. +pub(crate) const SECONDARY_ROOT_OFFSET: usize = 0x8000; +/// Size of the allocation holding both roots. +pub(crate) const ROOTS_SIZE: usize = 0xc000; + +// --------------------------------------------------------------------------- +// Channels +// --------------------------------------------------------------------------- + +/// Work channels: TA, 3D and compute at each of four priorities. +pub(crate) const WORK_CHANNELS: usize = 12; +/// Engines with a work channel at each priority (TA, 3D, compute). +const WORK_ENGINES: usize = 3; +/// Work priorities. +const WORK_PRIORITIES: usize = WORK_CHANNELS / WORK_ENGINES; +/// Channel-table index of the device-control channel. +pub(crate) const CONTROL_CHANNEL: usize = 12; +/// Channel-table index of the event and firmware-log rings. +pub(crate) const EVENT_CHANNEL: usize = 13; +/// Channel-table index of the trace and statistics rings. +pub(crate) const TRACE_CHANNEL: usize = 14; +/// Channel-table index of the channel with a single state address. +const CHANNEL_15: usize = 15; +/// Entries in the channel table. +pub(crate) const CHANNELS: usize = 17; + +/// One channel-table entry: four GPU addresses whose meaning depends on the +/// channel. Work and control channels name three counters of their +/// [`ChannelState`] and their ring; the event and trace channels name two +/// state/ring pairs. +#[repr(C)] +pub(crate) struct ChannelEntry(pub(crate) [u64; 4]); + +// SAFETY: `ChannelEntry` is an array of integers. +unsafe impl Zeroable for ChannelEntry {} + +static_assert!(size_of::() == 0x20); + +impl ChannelEntry { + /// Returns the entry of a work or control channel. + const fn ring(state_va: u64, ring_va: u64) -> Self { + Self([ + state_va + ChannelState::CONSUMER as u64, + state_va + ChannelState::UNK_10 as u64, + state_va + ChannelState::PRODUCER as u64, + ring_va, + ]) + } +} + +/// State of a work or control channel in an instance's state grid. +/// +/// All three counters are live words. The firmware advances `consumer` and +/// `unk_10`; the host writes `producer` with single aligned stores, after the +/// records it publishes are complete. At boot the host stores +/// zero to the control channel's consumer, then zero to `unk_10`, then +/// [`OPENING_PRODUCER`] to its producer, primary instance before secondary, +/// after the rest of the initialization data is written. A full barrier follows +/// both instances' stores. These creation-time writes occur before publication; the firmware moves +/// all three to that value once it has drained the opening record. +#[repr(C)] +pub(crate) struct ChannelState { + /// Records the firmware has consumed. + pub(crate) consumer: u32, + pub(crate) unk_4: [u8; 0xc], + /// Firmware-owned. + pub(crate) unk_10: u32, + pub(crate) unk_14: [u8; 0xc], + /// Records the host has published. + pub(crate) producer: u32, + pub(crate) unk_24: [u8; 0x1c], +} + +// SAFETY: `ChannelState` consists of integers and arrays of integers only. +unsafe impl Zeroable for ChannelState {} + +static_assert!(core::mem::offset_of!(ChannelState, consumer) == 0); +static_assert!(core::mem::offset_of!(ChannelState, unk_10) == 0x10); +static_assert!(core::mem::offset_of!(ChannelState, producer) == 0x20); +static_assert!(size_of::() == 0x40); + +impl ChannelState { + pub(crate) const CONSUMER: usize = core::mem::offset_of!(Self, consumer); + pub(crate) const UNK_10: usize = core::mem::offset_of!(Self, unk_10); + /// Offset of [`Self::producer`] in the block. + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); +} + +/// Offset of the device-control channel's state in an instance's state grid. +pub(crate) const CONTROL_STATE_OFFSET: usize = WORK_CHANNELS * size_of::(); + +/// Producer value of the device-control ring of either instance at boot: one +/// opening record. +pub(crate) const OPENING_PRODUCER: u32 = 1; + +/// Position of work channel `channel` in the state grid and in the work-ring +/// array: grouped by engine, priorities in descending order. +const fn work_slot(channel: usize) -> usize { + let engine = channel % WORK_ENGINES; + let priority = channel / WORK_ENGINES; + engine * WORK_PRIORITIES + (WORK_PRIORITIES - 1 - priority) +} + +/// Offset of work channel `channel`'s state in the state grid. +pub(crate) const fn work_state_offset(channel: usize) -> usize { + work_slot(channel) * size_of::() +} + +/// Bundle offset of work channel `channel`'s ring. +pub(crate) const fn work_ring_offset(channel: usize) -> usize { + bundle::WORK_RINGS + work_slot(channel) * bundle::WORK_RING_SIZE +} + +/// Offsets in a status area, the region starting at an instance's status A +/// block, of the rings the firmware fills. +pub(crate) mod status { + /// Event ring state. + pub(crate) const EVENT_STATE: u64 = 0x40; + /// Firmware-log state. + pub(crate) const LOG_STATE: u64 = 0x80; + /// Trace ring state. + pub(crate) const TRACE_STATE: u64 = 0x240; + /// Statistics ring state. + pub(crate) const STATS_STATE: u64 = 0x280; + /// Event ring. + pub(crate) const EVENT_RING: u64 = 0x2c0; + /// Firmware-log ring. + pub(crate) const LOG_RING: u64 = 0x4ac0; + /// The single address in channel 15. + pub(crate) const UNK_2D2C0: u64 = 0x2d2c0; + /// Trace ring. + pub(crate) const TRACE_RING: u64 = 0xa6ac0; + /// Statistics ring. + pub(crate) const STATS_RING: u64 = 0xafac0; +} + +// --------------------------------------------------------------------------- +// Hardware-data bundle layout +// --------------------------------------------------------------------------- + +/// Offsets in the hardware-data bundle. +pub(crate) mod bundle { + /// Size of the bundle allocation. + pub(crate) const SIZE: usize = 0x28000; + /// The [`super::HwData`]. + pub(crate) const HW_DATA: usize = 0; + /// The five views named by the primary main configuration. + pub(crate) const VIEWS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; + /// The [`super::PowerConfig`], inside view 2. + pub(crate) const POWER_CONFIG: usize = 0x5240; + /// The [`super::HwDataAux`]. + pub(crate) const AUX: usize = 0x8000; + /// A second copy of the [`super::PowerConfigHead`], inside views 3 and 4. + pub(crate) const POWER_CONFIG_COPY: usize = 0xd240; + /// Region the firmware writes, named twice by each main configuration. + pub(crate) const FIRMWARE_AREA: usize = 0xc500; + /// The work rings, see [`super::work_ring_offset`]. + pub(crate) const WORK_RINGS: usize = 0xc5c0; + /// Size of one work ring (256 slots of 0x18 bytes). + pub(crate) const WORK_RING_SIZE: usize = 0x1800; + /// Main configuration of each instance, each followed by its + /// device-control ring. + pub(crate) const MAIN_CONFIG: [usize; 2] = [0x1e5c0, 0x22a80]; + /// Separately allocated pages the firmware finds at fixed offsets from + /// the bundle: a zeroed page named by the primary main configuration. + pub(crate) const ZERO_PAGE: usize = 0x40000; + /// The two completion rings ([`super::CompletionRing`]). + pub(crate) const COMPLETION_RINGS: [usize; 2] = [0x48000, 0x70000]; +} + +// --------------------------------------------------------------------------- +// Main configuration +// --------------------------------------------------------------------------- + +/// Record the host stages in slot 0 of each instance's device-control ring +/// before boot. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct OpeningRecord { + /// Opcode: [`Self::PRIMARY`] or [`Self::SECONDARY`]. + pub(crate) opcode: u32, + pub(crate) unk_4: [u8; 0x3c], +} + +// SAFETY: `OpeningRecord` consists of integers and arrays of integers only. +unsafe impl Zeroable for OpeningRecord {} + +static_assert!(size_of::() == 0x40); + +impl OpeningRecord { + const PRIMARY: u32 = 0x16; + const SECONDARY: u32 = 0x2a; + + /// Returns the opening record of `role`. + pub(crate) const fn new(role: Role) -> Self { + Self { + opcode: match role { + Role::Primary => Self::PRIMARY, + Role::Secondary => Self::SECONDARY, + }, + unk_4: [0; 0x3c], + } + } +} + +/// Addresses both main configurations need. +pub(crate) struct MainConfigArgs { + /// Hardware-data bundle. + pub(crate) bundle_va: u64, + /// This main configuration. + pub(crate) main_config_va: u64, + /// This instance's state grid. + pub(crate) state_grid_va: u64, + /// This instance's status A block. + pub(crate) status_a_va: u64, + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) pb_descriptors_low_va: u64, + /// Context-0 alias of the page-pool descriptor table. + pub(crate) page_pool_descriptors_low_va: u64, +} + +/// Addresses only the primary main configuration names. +pub(crate) struct PrimaryViews { + /// The zeroed page at [`bundle::ZERO_PAGE`]. + pub(crate) zero_page_va: u64, + /// Parameter-buffer descriptor table. + pub(crate) pb_descriptors_va: u64, + /// page-pool descriptor table. + pub(crate) page_pool_descriptors_va: u64, +} + +/// Main configuration of one instance. Its device-control ring +/// (256 records) follows it directly. +#[repr(C, packed)] +pub(crate) struct MainConfig { + /// Hardware-data bundle ([`HwData`]). + pub(crate) hw_data_va: u64, + /// [`bundle::FIRMWARE_AREA`]. + pub(crate) firmware_area_va: [u64; 2], + pub(crate) unk_18: [u8; 8], + /// Channel table. + pub(crate) channels: [ChannelEntry; CHANNELS], + pub(crate) unk_240: [u8; 0x10], + /// Non-zero enables firmware log output; the driver leaves it off. + pub(crate) log_enable: u32, + /// The bundle views ([`bundle::VIEWS`]); primary only. + pub(crate) view_va: [u64; 5], + pub(crate) unk_27c: [u8; 0x54], + /// [`PrimaryViews::zero_page_va`]; primary only. + pub(crate) zero_page_va: u64, + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) pb_descriptors_low_va: u64, + /// Parameter-buffer descriptor table; primary only. + pub(crate) pb_descriptors_va: u64, + /// Context-0 alias of the page-pool descriptor table. + pub(crate) page_pool_descriptors_low_va: u64, + /// page-pool descriptor table; primary only. + pub(crate) page_pool_descriptors_va: u64, + pub(crate) unk_2f8: [u8; 8], + /// Required: 4 for the secondary. + pub(crate) unk_300: u32, + pub(crate) unk_304: [u8; 0x28], + /// Required: 1 for the primary. + pub(crate) unk_32c: u32, + pub(crate) unk_330: [u8; 0x14], + /// Required: 0xabcd_abcd for the primary. + pub(crate) unk_344: u32, + pub(crate) unk_348: [u8; 0x98], + /// Required: 0xff for the primary. + pub(crate) unk_3e0: u32, + pub(crate) unk_3e4: [u8; 0x8d], + /// [`PowerConfig`]; secondary only. + pub(crate) power_config_va: u64, + pub(crate) unk_479: [u8; 0x47], +} + +static_assert!(core::mem::offset_of!(MainConfig, channels) == 0x20); +static_assert!(core::mem::offset_of!(MainConfig, log_enable) == 0x250); +static_assert!(core::mem::offset_of!(MainConfig, view_va) == 0x254); +static_assert!(core::mem::offset_of!(MainConfig, zero_page_va) == 0x2d0); +static_assert!(core::mem::offset_of!(MainConfig, unk_300) == 0x300); +static_assert!(core::mem::offset_of!(MainConfig, unk_3e0) == 0x3e0); +static_assert!(core::mem::offset_of!(MainConfig, power_config_va) == 0x471); +static_assert!(size_of::() == MainConfig::CONTROL_RING); +static_assert!( + bundle::MAIN_CONFIG[0] + MainConfig::CONTROL_RING + 256 * size_of::() + == bundle::MAIN_CONFIG[1] +); + +// SAFETY: `MainConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for MainConfig {} + +impl MainConfig { + /// Offset of the device-control ring from the main configuration. + pub(crate) const CONTROL_RING: usize = 0x4c0; + + fn init(&mut self, role: Role, args: &MainConfigArgs) { + clear(self); + let bundle = args.bundle_va; + let grid = args.state_grid_va; + let status_a = args.status_a_va; + self.hw_data_va = bundle + bundle::HW_DATA as u64; + self.firmware_area_va = [bundle + bundle::FIRMWARE_AREA as u64; 2]; + if role == Role::Primary { + for channel in 0..WORK_CHANNELS { + self.channels[channel] = ChannelEntry::ring( + grid + work_state_offset(channel) as u64, + bundle + work_ring_offset(channel) as u64, + ); + } + } + self.channels[CONTROL_CHANNEL] = ChannelEntry::ring( + grid + CONTROL_STATE_OFFSET as u64, + args.main_config_va + Self::CONTROL_RING as u64, + ); + self.channels[EVENT_CHANNEL] = ChannelEntry([ + status_a + status::EVENT_STATE, + status_a + status::EVENT_RING, + status_a + status::LOG_STATE, + status_a + status::LOG_RING, + ]); + self.channels[TRACE_CHANNEL] = ChannelEntry([ + status_a + status::TRACE_STATE, + status_a + status::TRACE_RING, + status_a + status::STATS_STATE, + status_a + status::STATS_RING, + ]); + self.channels[CHANNEL_15] = ChannelEntry([status_a + status::UNK_2D2C0, 0, 0, 0]); + self.pb_descriptors_low_va = args.pb_descriptors_low_va; + self.page_pool_descriptors_low_va = args.page_pool_descriptors_low_va; + } + + /// Initializes the primary main configuration. + pub(crate) fn init_primary(&mut self, args: &MainConfigArgs, views: &PrimaryViews) { + self.init(Role::Primary, args); + self.view_va = bundle::VIEWS.map(|offset| args.bundle_va + offset as u64); + self.zero_page_va = views.zero_page_va; + self.pb_descriptors_va = views.pb_descriptors_va; + self.page_pool_descriptors_va = views.page_pool_descriptors_va; + self.unk_32c = 1; + self.unk_344 = 0xabcd_abcd; + self.unk_3e0 = 0xff; + } + + /// Initializes the secondary main configuration. + pub(crate) fn init_secondary(&mut self, args: &MainConfigArgs) { + self.init(Role::Secondary, args); + self.unk_300 = 4; + self.power_config_va = args.bundle_va + bundle::POWER_CONFIG as u64; + } +} + +// --------------------------------------------------------------------------- +// Hardware data +// --------------------------------------------------------------------------- + +/// Slots in the hardware-data register map. +pub(crate) const REGISTER_MAP_SLOTS: usize = 53; + +/// One register window the firmware maps through the hardware data +/// (per-SoC configuration). +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterWindow { + /// Register-map slot. + pub(crate) slot: usize, + /// Physical address of the window. + pub(crate) phys: u64, + /// GPU address of the window in the firmware MMIO aperture. + pub(crate) va: u64, + /// Size in bytes; need not be a multiple of the page size. + pub(crate) size: u32, + /// Value of [`RegisterMapEntry::unk_18`]. + pub(crate) unk_18: u64, + /// Value of [`RegisterMapEntry::unk_20`]. + pub(crate) unk_20: u32, +} + +impl RegisterWindow { + /// Returns the page-aligned physical address, GPU address and length the + /// window must be mapped with, or `None` if its physical and GPU + /// addresses differ within a page or the mapped span overflows. + pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { + let mask = PAGE_SIZE as u64 - 1; + if self.phys & mask != self.va & mask { + return None; + } + let start = self.va & !mask; + let span = self + .va + .checked_add(self.size as u64)? + .checked_sub(start)? + .checked_add(mask)? + & !mask; + Some((self.phys & !mask, start, span)) + } +} + +/// Value of [`RegisterMapEntry::unk_20`] in a reserved register-map slot. +const RESERVED_SLOT_UNK_20: u32 = 2; + +/// One register-map slot of the hardware data. +#[repr(C)] +pub(crate) struct RegisterMapEntry { + /// Physical address of the window. + pub(crate) phys: u64, + /// GPU address of the window. + pub(crate) va: u64, + /// Window size in bytes. + pub(crate) total_size: u32, + /// Window size in bytes, again. + pub(crate) element_size: u32, + /// Per-window value, see [`RegisterWindow::unk_18`]. + pub(crate) unk_18: u64, + /// Per-window value; [`RESERVED_SLOT_UNK_20`] in a reserved slot. + pub(crate) unk_20: u32, + pub(crate) unk_24: u32, +} + +// SAFETY: `RegisterMapEntry` consists of integers and arrays of integers only. +unsafe impl Zeroable for RegisterMapEntry {} + +static_assert!(size_of::() == 0x28); + +impl RegisterMapEntry { + const fn window(window: &RegisterWindow) -> Self { + Self { + phys: window.phys, + va: window.va, + total_size: window.size, + element_size: window.size, + unk_18: window.unk_18, + unk_20: window.unk_20, + unk_24: 0, + } + } + + const fn reserved() -> Self { + Self { + phys: 0, + va: 0, + total_size: 0, + element_size: 0, + unk_18: 0, + unk_20: RESERVED_SLOT_UNK_20, + unk_24: 0, + } + } +} + +/// Performance states in each performance table. +pub(crate) const PERF_STATES: usize = 11; +/// Voltage lanes of a performance table. +const PERF_LANES: usize = 16; + +/// One GPU operating point, from the device tree, in performance-state +/// order. +#[derive(Copy, Clone, Debug)] +pub(crate) struct Opp { + /// Frequency in Hz. + pub(crate) freq_hz: u64, + /// GPU core voltage in µV. + pub(crate) volt_uv: u32, +} + +/// Per-SoC parts of the performance tables that the operating points do not +/// give. +pub(crate) struct PerfConfig { + /// Operating point of each state of table A. + pub(crate) states_a: [u8; PERF_STATES], + /// Operating point of each state of table B. + pub(crate) states_b: [u8; PERF_STATES], + /// Relative performance of each state of table A, in percent. + pub(crate) relative_a: [u32; PERF_STATES], + /// Relative performance of each state of table B, in percent. + pub(crate) relative_b: [u32; PERF_STATES], + /// Lowest SRAM voltage in mV; a state's SRAM voltage is its core voltage + /// raised to this floor. + pub(crate) sram_min_mv: u32, +} + +/// One performance table of the hardware data. +#[repr(C, packed)] +pub(crate) struct PerfGroup { + /// Required: 10. + pub(crate) unk_0: u32, + /// Frequency of each state in MHz. + pub(crate) freq_mhz: [u32; PERF_STATES], + pub(crate) unk_30: [u8; 0x14], + /// GPU core voltage of each state in mV, per lane. + pub(crate) core_mv: [[u32; PERF_LANES]; PERF_STATES], + pub(crate) unk_304: [u8; 0x140], + /// SRAM voltage of each state in mV, per lane. + pub(crate) sram_mv: [[u32; PERF_LANES]; PERF_STATES], +} + +// SAFETY: `PerfGroup` consists of integers and arrays of integers only. +unsafe impl Zeroable for PerfGroup {} + +static_assert!(core::mem::offset_of!(PerfGroup, core_mv) == 0x44); +static_assert!(core::mem::offset_of!(PerfGroup, sram_mv) == 0x444); +static_assert!(size_of::() == 0x704); + +/// Frequency and voltages of one performance state. +#[derive(Copy, Clone)] +struct PerfState { + freq_mhz: u32, + core_mv: u32, + sram_mv: u32, +} + +impl PerfGroup { + /// Fills the table. `lanes` is the number of voltage lanes holding each + /// state's voltages; the remaining lanes stay zero. + fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { + self.unk_0 = 10; + for (index, state) in states.iter().enumerate() { + self.freq_mhz[index] = state.freq_mhz; + for lane in 0..lanes { + self.core_mv[index][lane] = state.core_mv; + self.sram_mv[index][lane] = state.sram_mv; + } + } + } +} + +/// Returns the performance states selected by `map` from `opps`. +fn perf_states( + opps: &[Opp], + map: &[u8; PERF_STATES], + sram_min_mv: u32, +) -> Result<[PerfState; PERF_STATES]> { + let mut states = [PerfState { + freq_mhz: 0, + core_mv: 0, + sram_mv: 0, + }; PERF_STATES]; + for (state, &index) in states.iter_mut().zip(map) { + let opp = opps.get(index as usize).ok_or(EINVAL)?; + let core_mv = opp.volt_uv / 1000; + *state = PerfState { + freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, + core_mv, + sram_mv: core_mv.max(sram_min_mv), + }; + } + Ok(states) +} + +/// One completion ring the firmware fills. +#[repr(C)] +pub(crate) struct CompletionRingDescriptor { + /// Context-0 alias of the ring. + pub(crate) low_va: u64, + /// Firmware alias of the ring. + pub(crate) high_va: u64, + /// Entries in the ring. + pub(crate) entries: u32, + /// Size of one entry. + pub(crate) entry_size: u32, + /// Index of this descriptor in the table. + pub(crate) index: u32, + /// Firmware-owned. + pub(crate) unk_1c: u32, +} + +// SAFETY: `CompletionRingDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for CompletionRingDescriptor {} + +static_assert!(size_of::() == 0x20); + +/// Completion-ring descriptors in the hardware data; only +/// [`COMPLETION_RING_INDEX`] are used. +const COMPLETION_RING_DESCRIPTORS: usize = 4; +/// Descriptor index of each completion ring. +pub(crate) const COMPLETION_RING_INDEX: [u32; 2] = [0, 2]; +/// Entries in a completion ring. +const COMPLETION_RING_ENTRIES: u32 = 0x800; +/// Size of a completion-ring entry. +const COMPLETION_RING_ENTRY_SIZE: u32 = 0x40; +/// Size of a completion ring. +pub(crate) const COMPLETION_RING_SIZE: usize = + (COMPLETION_RING_ENTRIES * COMPLETION_RING_ENTRY_SIZE) as usize; + +/// The two aliases of one completion ring. +#[derive(Copy, Clone, Debug)] +pub(crate) struct CompletionRing { + /// Context-0 alias. + pub(crate) low_va: u64, + /// Firmware alias. + pub(crate) high_va: u64, +} + +/// Inputs of the hardware data. +pub(crate) struct HwDataArgs<'a> { + /// Chip identifier. + pub(crate) chip_id: u32, + /// Mapped register windows. + pub(crate) windows: &'a [RegisterWindow], + /// Register-map slots that are reserved but unmapped. + pub(crate) reserved_slots: &'a [usize], + /// GPU operating points, in performance-state order. + pub(crate) opps: &'a [Opp], + /// Per-SoC parts of the performance tables. + pub(crate) perf: &'a PerfConfig, + /// The completion rings, in [`COMPLETION_RING_INDEX`] order. + pub(crate) completion_rings: [CompletionRing; 2], + /// The zeroed QoS object ([`QOS_SIZE`] bytes). + pub(crate) qos_va: u64, +} + +/// Size of the QoS object named by the hardware data. +pub(crate) const QOS_SIZE: usize = 0xe40; + +/// The hardware data, at the base of the bundle and shared by both +/// instances. +#[repr(C, packed)] +pub(crate) struct HwData { + /// Required: 0x6f_0000_0000. + pub(crate) unk_0: u64, + /// Required: 0xffc0_0000. + pub(crate) unk_8: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_10: u64, + /// Required: 0x10_0000_0000. + pub(crate) unk_18: u64, + /// Required: 0x2ff_ffff_8000. + pub(crate) unk_20: u64, + /// Required: 0xffff_fc21_8140_0000, a GPU address in the firmware MMIO + /// aperture. + pub(crate) unk_28: u64, + pub(crate) unk_30: [u8; 0xb0], + /// Required: see [`HwData::UNK_E0`]. + pub(crate) unk_e0: [u16; 24], + pub(crate) unk_110: [u8; 0x1c8], + /// Required: see [`HwData::UNK_2D8`]. + pub(crate) unk_2d8: [u16; 12], + pub(crate) unk_2f0: [u8; 0xf0], + /// Required: see [`HwData::UNK_3E0`]. + pub(crate) unk_3e0: [u16; 24], + pub(crate) unk_410: [u8; 0x1c8], + /// Required: see [`HwData::UNK_5D8`]. + pub(crate) unk_5d8: [u16; 12], + pub(crate) unk_5f0: [u8; 0x50], + /// Register windows the firmware maps. + pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) unk_e88: [u8; 8], + /// Chip identifier. + pub(crate) chip_id: u32, + /// Required: 1. + pub(crate) unk_e94: u32, + /// Required: 1. + pub(crate) unk_e98: u32, + pub(crate) unk_e9c: [u8; 4], + /// Required: 1. + pub(crate) unk_ea0: u32, + /// Required: 1. + pub(crate) unk_ea4: u32, + pub(crate) unk_ea8: [u8; 0x10], + /// Required: 1. + pub(crate) unk_eb8: u32, + pub(crate) unk_ebc: [u8; 4], + /// Required: 1. + pub(crate) unk_ec0: u32, + pub(crate) unk_ec4: [u8; 4], + /// Required: 1. + pub(crate) unk_ec8: u32, + pub(crate) unk_ecc: [u8; 4], + /// Required: 0x5dc0 (24000). + pub(crate) unk_ed0: u32, + /// Required: 1. + pub(crate) unk_ed4: u32, + /// Required: 0x10 (16). + pub(crate) unk_ed8: u32, + pub(crate) unk_edc: [u8; 4], + /// Required: 1 in every entry. + pub(crate) unk_ee0: [u32; 3], + pub(crate) unk_eec: [u8; 0x18], + /// Required: 0x1f (31). + pub(crate) unk_f04: u32, + pub(crate) unk_f08: [u8; 0x1c], + /// Required: 4. + pub(crate) unk_f24: u32, + pub(crate) unk_f28: [u8; 0xc], + /// Required: 1. + pub(crate) unk_f34: u32, + /// Required: 1. + pub(crate) unk_f38: u32, + pub(crate) unk_f3c: [u8; 0x10], + /// Required: 0x31 (49). + pub(crate) unk_f4c: u32, + pub(crate) unk_f50: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_f6c: u32, + pub(crate) unk_f70: [u8; 0x18], + /// Required: 6. + pub(crate) unk_f88: u32, + /// Required: 1. + pub(crate) unk_f8c: u32, + pub(crate) unk_f90: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_fac: u32, + pub(crate) unk_fb0: [u8; 8], + /// Required: 0x1e (30). + pub(crate) unk_fb8: u32, + /// Required: 4. + pub(crate) unk_fbc: u32, + /// Required: 6. + pub(crate) unk_fc0: u32, + /// Performance table A. + pub(crate) perf_a: PerfGroup, + pub(crate) unk_16c8: [u8; 0x140], + /// Frequency of each state of table B in MHz. + pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], + pub(crate) unk_1834: [u8; 0x14], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_1848: [u32; 11], + pub(crate) unk_1874: [u8; 0x54], + /// Relative performance of each state of table A, in percent. + pub(crate) perf_a_relative: [u32; PERF_STATES], + pub(crate) unk_18f4: [u8; 0x14], + /// Relative performance of each state of table B, in percent. + pub(crate) perf_b_relative: [u32; PERF_STATES], + pub(crate) unk_1934: [u8; 0x94], + /// Operating point of each state of table A. + pub(crate) perf_a_states: [u32; PERF_STATES], + pub(crate) unk_19f4: [u8; 0x14], + /// Operating point of each state of table B. + pub(crate) perf_b_states: [u32; PERF_STATES], + pub(crate) unk_1a34: [u8; 0x2a4], + /// Performance table B. Each voltage is stored in lane 0 only. + pub(crate) perf_b: PerfGroup, + pub(crate) unk_23dc: [u8; 0x168], + /// Required: 4. + pub(crate) unk_2544: u32, + /// Required: 6. + pub(crate) unk_2548: u32, + /// Required: 3. + pub(crate) unk_254c: u32, + /// Required: 7. + pub(crate) unk_2550: u32, + /// Required: 7. + pub(crate) unk_2554: u32, + pub(crate) unk_2558: [u8; 0x10], + /// Required: 5. + pub(crate) unk_2568: u32, + pub(crate) unk_256c: [u8; 4], + /// Required: 1. + pub(crate) unk_2570: u32, + pub(crate) unk_2574: [u8; 0x1a], + /// Required: [`WORK_CHANNELS`]. + pub(crate) unk_258e: u16, + /// Required: 0, 1, 1, 1. + pub(crate) unk_2590: [u32; 4], + /// Required: 0, 1, 1, 1. + pub(crate) unk_25a0: [u32; 4], + pub(crate) unk_25b0: [u8; 4], + /// Required: 0xffff_ffff in every entry, one per work channel. + pub(crate) unk_25b4: [u32; WORK_CHANNELS], + pub(crate) unk_25e4: [u8; 0x10], + /// Required: 1. + pub(crate) unk_25f4: u32, + pub(crate) unk_25f8: [u8; 8], + /// Required: 1. + pub(crate) unk_2600: u32, + pub(crate) unk_2604: [u8; 4], + /// Required: 1. + pub(crate) unk_2608: u32, + pub(crate) unk_260c: [u8; 4], + /// Required: 0x100. + pub(crate) unk_2610: u32, + /// Completion-ring descriptors. + pub(crate) completion_rings: [CompletionRingDescriptor; COMPLETION_RING_DESCRIPTORS], + pub(crate) unk_2694: [u8; 0x18], + /// QoS object ([`QOS_SIZE`] bytes, zeroed). + pub(crate) qos_va: u64, + /// Required: 1. + pub(crate) unk_26b4: u32, + /// Required: 1. + pub(crate) unk_26b8: u32, + pub(crate) unk_26bc: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_26e8: u32, + pub(crate) unk_26ec: [u8; 0x16ac], + /// Required: 0xffff_ffff. + pub(crate) unk_3d98: u32, + pub(crate) unk_3d9c: [u8; 0x14], + /// Required: 0xffff_ffff. + pub(crate) unk_3db0: u32, +} + +static_assert!(core::mem::offset_of!(HwData, register_map) == 0x640); +static_assert!(core::mem::offset_of!(HwData, chip_id) == 0xe90); +static_assert!(core::mem::offset_of!(HwData, perf_a) == 0xfc4); +static_assert!(core::mem::offset_of!(HwData, perf_b_freq_mhz) == 0x1808); +static_assert!(core::mem::offset_of!(HwData, perf_b) == 0x1cd8); +static_assert!(core::mem::offset_of!(HwData, unk_258e) == 0x258e); +static_assert!(core::mem::offset_of!(HwData, completion_rings) == 0x2614); +static_assert!(core::mem::offset_of!(HwData, qos_va) == 0x26ac); +static_assert!(size_of::() == 0x3db4); +static_assert!(bundle::HW_DATA + size_of::() <= bundle::VIEWS[2]); + +// SAFETY: `HwData` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwData {} + +impl HwData { + const UNK_E0: [u16; 24] = [ + 0x2008, 0, 0, 0, 0, 0x2008, 0, 0, 0, 0, 0x2008, 0, 0x24cb, 0, 0x2cfa, 0xc98a, 0x24cb, + 0xf4f6, 0xe917, 0x1877, 0x24cb, 0x38d9, 0, 0xbdab, + ]; + const UNK_2D8: [u16; 12] = [ + 0x2000, 0xfff8, 0x2cdb, 0xd32d, 0x2000, 0xf500, 0xe926, 0x21da, 0x2000, 0x38b6, 8, 0xc742, + ]; + const UNK_3E0: [u16; 24] = [ + 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0x8000, 0, 0, 0, 0, 0x8000, 0, 0, 0, 0, + 0x8000, 0, + ]; + const UNK_5D8: [u16; 12] = [ + 0x2645, 0x4b23, 0xe98, 0, 0xea5f, 0xd5a2, 0x3fff, 0x4000, 0x4000, 0xca5e, 0xf5a2, 0x4000, + ]; + + /// Initializes the hardware data. + pub(crate) fn init(&mut self, args: &HwDataArgs<'_>) -> Result { + clear(self); + self.unk_0 = 0x6f_0000_0000; + self.unk_8 = 0xffc0_0000; + self.unk_10 = 0x10_0000_0000; + self.unk_18 = 0x10_0000_0000; + self.unk_20 = 0x2ff_ffff_8000; + self.unk_28 = 0xffff_fc21_8140_0000; + self.unk_e0 = Self::UNK_E0; + self.unk_2d8 = Self::UNK_2D8; + self.unk_3e0 = Self::UNK_3E0; + self.unk_5d8 = Self::UNK_5D8; + + let mut used = [false; REGISTER_MAP_SLOTS]; + let mut claim = |slot: usize| -> Result { + match used.get_mut(slot) { + Some(used) if !*used => { + *used = true; + Ok(()) + } + _ => Err(EINVAL), + } + }; + for window in args.windows { + claim(window.slot)?; + self.register_map[window.slot] = RegisterMapEntry::window(window); + } + for &slot in args.reserved_slots { + claim(slot)?; + self.register_map[slot] = RegisterMapEntry::reserved(); + } + + self.chip_id = args.chip_id; + self.unk_e94 = 1; + self.unk_e98 = 1; + self.unk_ea0 = 1; + self.unk_ea4 = 1; + self.unk_eb8 = 1; + self.unk_ec0 = 1; + self.unk_ec8 = 1; + self.unk_ed0 = 0x5dc0; + self.unk_ed4 = 1; + self.unk_ed8 = 0x10; + self.unk_ee0 = [1; 3]; + self.unk_f04 = 0x1f; + self.unk_f24 = 4; + self.unk_f34 = 1; + self.unk_f38 = 1; + self.unk_f4c = 0x31; + self.unk_f6c = 1; + self.unk_f88 = 6; + self.unk_f8c = 1; + self.unk_fac = 1; + self.unk_fb8 = 0x1e; + self.unk_fbc = 4; + self.unk_fc0 = 6; + + let perf = args.perf; + let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; + let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; + self.perf_a.fill(&states_a, PERF_LANES); + self.perf_b.fill(&states_b, 1); + for (index, state) in states_b.iter().enumerate() { + self.perf_b_freq_mhz[index] = state.freq_mhz; + self.perf_a_states[index] = perf.states_a[index].into(); + self.perf_b_states[index] = perf.states_b[index].into(); + } + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + self.unk_1848 = [0x3f82_8f5c; 11]; + self.unk_2544 = 4; + self.unk_2548 = 6; + self.unk_254c = 3; + self.unk_2550 = 7; + self.unk_2554 = 7; + self.unk_2568 = 5; + self.unk_2570 = 1; + + self.unk_258e = WORK_CHANNELS as u16; + self.unk_2590 = [0, 1, 1, 1]; + self.unk_25a0 = [0, 1, 1, 1]; + self.unk_25b4 = [u32::MAX; WORK_CHANNELS]; + self.unk_25f4 = 1; + self.unk_2600 = 1; + self.unk_2608 = 1; + self.unk_2610 = 0x100; + for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { + self.completion_rings[index as usize] = CompletionRingDescriptor { + low_va: ring.low_va, + high_va: ring.high_va, + entries: COMPLETION_RING_ENTRIES, + entry_size: COMPLETION_RING_ENTRY_SIZE, + index, + unk_1c: 0, + }; + } + self.qos_va = args.qos_va; + self.unk_26b4 = 1; + self.unk_26b8 = 1; + self.unk_26e8 = 1; + self.unk_3d98 = 0xffff_ffff; + self.unk_3db0 = 0xffff_ffff; + Ok(()) + } +} + +// --------------------------------------------------------------------------- +// Regions A and C +// --------------------------------------------------------------------------- + +/// Region A: a zeroed page both instances read. +#[repr(C)] +pub(crate) struct RegionA { + pub(crate) unk_0: [u8; 0x4000], +} + +// SAFETY: `RegionA` is an array of integers. +unsafe impl Zeroable for RegionA {} + +/// Engines whose work the firmware's context-switch timer may interrupt +/// (bit 0 TA, bit 1 3D, bit 2 compute). No engine is periodically resumed. +pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; +/// Period of the context-switch timer, in units of the firmware's 1 ms +/// scheduling pass. +pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; + +/// Region C: firmware policy settings shared by both instances. +/// +/// Three fields are live. The host updates [`Self::effort`] while it submits +/// work: it stores zero to [`Self::unk_e8`], then the effort (0x10000, 1.0 +/// in 16.16 fixed point, while work is streaming; zero once the GPU has been +/// idle for 16 ms), each with one aligned 32-bit store, and then cleans the +/// cache line to the point of coherency. [`Self::power_assert_tally`] is +/// unaligned: for every device-control record whose opcode asserts GPU +/// power, the host adds one to it after writing the record and before +/// advancing the ring producer (with a barrier before the producer store). +/// The firmware powers the GPU cores up while the tally differs from its +/// own count of drained records. +#[repr(C, packed)] +pub(crate) struct RegionC { + /// Engines the context-switch timer applies to ([`CONTEXT_SWITCH_TIMER_MASK`]). + pub(crate) context_switch_timer_mask: u32, + pub(crate) unk_4: [u8; 0x20], + /// Required: 0xbb8 (3000). + pub(crate) unk_24: u32, + pub(crate) unk_28: [u8; 8], + /// Non-zero lets the firmware power the GPU cores off when idle. + pub(crate) idle_power_off: u32, + /// Required: 1. + pub(crate) unk_34: u32, + pub(crate) unk_38: [u8; 4], + /// Required: 0x78 (120). + pub(crate) unk_3c: u32, + pub(crate) unk_40: [u8; 0x14], + /// Required: 0xffff. + pub(crate) unk_54: u16, + /// Required: 0x28 (40). + pub(crate) unk_56: u16, + /// Required: 0xffff. + pub(crate) unk_58: u16, + pub(crate) unk_5a: [u8; 4], + /// Required: 1. + pub(crate) unk_5e: u32, + /// Required: 1. + pub(crate) unk_62: u32, + /// Required: 3. + pub(crate) unk_66: u32, + /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). + pub(crate) context_switch_timer_multiplier: u8, + pub(crate) unk_6b: [u8; 0xd], + /// Required: 1 in every entry. + pub(crate) unk_78: [u32; 3], + /// Required: 0x254e (9550). + pub(crate) unk_84: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_88: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_8c: u32, + pub(crate) unk_90: [u8; 8], + /// Required: 0x3e8 (1000). + pub(crate) unk_98: u32, + /// Required: 0x3000_001d. + pub(crate) unk_9c: u32, + /// Required: 0x3100_0002. + pub(crate) unk_a0: u32, + pub(crate) unk_a4: [u8; 0x40], + /// Performance-floor request (live, see the type documentation). + pub(crate) effort: u32, + /// Live; the host stores zero here whenever it stores [`Self::effort`]. + pub(crate) unk_e8: u32, + pub(crate) unk_ec: [u8; 0x6d8], + /// Required: 0x7d0 (2000). + pub(crate) unk_7c4: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_7c8: u32, + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_7cc: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_7d0: u32, + /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). + pub(crate) unk_7d4: u32, + /// Required: 0x3dcc_cccd (0.1 as an IEEE-754 single). + pub(crate) unk_7d8: u32, + /// Required: 0x3e80_0000 (0.25 as an IEEE-754 single). + pub(crate) unk_7dc: u32, + /// Required: 0x3f19_999a (0.6 as an IEEE-754 single). + pub(crate) unk_7e0: u32, + /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). + pub(crate) unk_7e4: u32, + /// Required: 6. + pub(crate) unk_7e8: u32, + /// Required: 1. + pub(crate) unk_7ec: u32, + /// Required: 1. + pub(crate) unk_7f0: u32, + /// Required: 0x64 (100). + pub(crate) unk_7f4: u32, + /// Required: 0x64 (100). + pub(crate) unk_7f8: u32, + pub(crate) unk_7fc: [u8; 0x19c], + /// Required: 0x28 (40). + pub(crate) unk_998: u32, + /// Required: 0xa (10). + pub(crate) unk_99c: u32, + /// Required: 0xfa (250). + pub(crate) unk_9a0: u32, + /// Required: 1. + pub(crate) unk_9a4: u32, + pub(crate) unk_9a8: [u8; 0x10], + /// Required: 2. + pub(crate) unk_9b8: u32, + /// Idle time before the firmware powers the GPU cores off. + pub(crate) idle_power_off_delay: u32, + /// Required: 5. + pub(crate) unk_9c0: u32, + pub(crate) unk_9c4: [u8; 4], + /// Required: 0x28 (40). + pub(crate) unk_9c8: u32, + /// Compute progress watchdog threshold (zero disables the watchdog). + pub(crate) compute_watchdog: u32, + /// Required: 1. + pub(crate) unk_9d0: u32, + pub(crate) unk_9d4: [u8; 0x449], + /// Required: 1. + pub(crate) unk_e1d: u32, + pub(crate) unk_e21: [u8; 0x18], + /// Required: 3. + pub(crate) unk_e39: u32, + /// Required: 1. + pub(crate) unk_e3d: u32, + pub(crate) unk_e41: [u8; 0x10], + /// Host power-assert tally (live, unaligned; see the type documentation). + pub(crate) power_assert_tally: u32, + pub(crate) unk_e55: [u8; 0x1ab], +} + +static_assert!(core::mem::offset_of!(RegionC, idle_power_off) == 0x30); +static_assert!(core::mem::offset_of!(RegionC, context_switch_timer_multiplier) == 0x6a); +static_assert!(core::mem::offset_of!(RegionC, effort) == 0xe4); +static_assert!(core::mem::offset_of!(RegionC, idle_power_off_delay) == 0x9bc); +static_assert!(core::mem::offset_of!(RegionC, compute_watchdog) == 0x9cc); +static_assert!(core::mem::offset_of!(RegionC, power_assert_tally) == 0xe51); +static_assert!(size_of::() == 0x1000); + +// SAFETY: `RegionC` consists of integers and arrays of integers only. +unsafe impl Zeroable for RegionC {} + +impl RegionC { + /// Offset of [`Self::effort`]. + pub(crate) const EFFORT: usize = core::mem::offset_of!(Self, effort); + /// Offset of [`Self::unk_e8`]. + pub(crate) const UNK_E8: usize = core::mem::offset_of!(Self, unk_e8); + /// Offset of [`Self::power_assert_tally`]. + pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); + + /// Initializes region C. + pub(crate) fn init(&mut self) { + clear(self); + self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; + self.unk_24 = 0xbb8; + self.idle_power_off = 1; + self.unk_34 = 1; + self.unk_3c = 0x78; + self.unk_54 = 0xffff; + self.unk_56 = 0x28; + self.unk_58 = 0xffff; + self.unk_5e = 1; + self.unk_62 = 1; + self.unk_66 = 3; + self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; + self.unk_78 = [1; 3]; + self.unk_84 = 0x254e; + self.unk_88 = 0x3e8; + self.unk_8c = 0x3e8; + self.unk_98 = 0x3e8; + self.unk_9c = 0x3000_001d; + self.unk_a0 = 0x3100_0002; + self.unk_7c4 = 0x7d0; + self.unk_7c8 = 0x3f80_0000; + self.unk_7cc = 0x3f4c_cccd; + self.unk_7d0 = 0x3e4c_cccd; + self.unk_7d4 = 0x3f66_6666; + self.unk_7d8 = 0x3dcc_cccd; + self.unk_7dc = 0x3e80_0000; + self.unk_7e0 = 0x3f19_999a; + self.unk_7e4 = 0x3f66_6666; + self.unk_7e8 = 6; + self.unk_7ec = 1; + self.unk_7f0 = 1; + self.unk_7f4 = 0x64; + self.unk_7f8 = 0x64; + self.unk_998 = 0x28; + self.unk_99c = 0xa; + self.unk_9a0 = 0xfa; + self.unk_9a4 = 1; + self.unk_9b8 = 2; + self.idle_power_off_delay = 40; + self.unk_9c0 = 5; + self.unk_9c8 = 0x28; + self.compute_watchdog = 50; + self.unk_9d0 = 1; + self.unk_e1d = 1; + self.unk_e39 = 3; + self.unk_e3d = 1; + } +} + +// --------------------------------------------------------------------------- +// Status objects +// --------------------------------------------------------------------------- + +/// Status block of an instance (its status A, and the head of the primary +/// status B). +/// +/// At creation only `+0x04` is one. Firmware later owns the status words at +/// `+0x10` and `+0x14`; the host observes each with an aligned 32-bit read. +/// The event-ring state begins at status A `+0x40`, inside this boot view: +/// clearing or copying the whole status block after publication is forbidden. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct StatusBlock { + pub(crate) unk_0: u32, + /// Required: 1. + pub(crate) unk_4: u32, + pub(crate) unk_8: [u8; 0x78], +} + +// SAFETY: `StatusBlock` consists of integers and arrays of integers only. +unsafe impl Zeroable for StatusBlock {} + +static_assert!(size_of::() == 0x80); + +impl StatusBlock { + /// Returns a status block in its boot state. + pub(crate) const fn new() -> Self { + Self { + unk_0: 0, + unk_4: 1, + unk_8: [0; 0x78], + } + } +} + +/// Power state the primary status B carries for the secondary instance, +/// named by [`SecondaryRoot::power_state_va`]. +#[repr(C, packed)] +pub(crate) struct PowerState { + pub(crate) unk_0: [u8; 0x10], + /// Required: 0xf (15). + pub(crate) unk_10: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_14: u32, + /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). + pub(crate) unk_18: u32, + /// Required: 0x28 (40). + pub(crate) unk_1c: u32, + /// Required: 1. + pub(crate) unk_20: u32, + pub(crate) unk_24: [u8; 0xa0], + /// Non-zero asks the secondary to reload the limiter settings that + /// follow into the power configuration; zero at boot. + pub(crate) temp_update: u32, + /// Target temperature of the limiter ([`TEMP_TARGET`]). + pub(crate) temp_target: u32, + /// Release temperature of the limiter ([`TEMP_RELEASE`]). + pub(crate) temp_release: u32, + pub(crate) unk_d0: [u8; 8], + /// Proportional gain of the limiter ([`TEMP_KP`]). + pub(crate) temp_kp: u32, + /// Integral gain of the limiter ([`TEMP_KI`]). + pub(crate) temp_ki: u32, + pub(crate) unk_e0: [u8; 0x10], + /// Required: 0x26ac (9900). + pub(crate) unk_f0: u32, + /// Required: 0xc8 (200). + pub(crate) unk_f4: u32, + pub(crate) unk_f8: [u8; 0x10], + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_108: u32, + /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). + pub(crate) unk_10c: u32, + /// Required: 0xc8 (200). + pub(crate) unk_110: u32, + pub(crate) unk_114: [u8; 0x14], + /// Required: 1. + pub(crate) unk_128: u32, + /// Required: 0x2616 (9750). + pub(crate) unk_12c: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_130: u32, + /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). + pub(crate) unk_134: u32, + pub(crate) unk_138: [u8; 0x14], + /// Required: 1. + pub(crate) unk_14c: u32, + pub(crate) unk_150: [u8; 0x70], + /// Required: 1. + pub(crate) unk_1c0: u32, + /// Required: 1. + pub(crate) unk_1c4: u32, + /// Required: 4. + pub(crate) unk_1c8: u32, + /// Required: 1 in every entry. + pub(crate) unk_1cc: [u32; 4], + pub(crate) unk_1dc: [u8; 0x308], + /// Required: 1. + pub(crate) unk_4e4: u32, + /// Required: 0x1f4 (500). + pub(crate) unk_4e8: u32, + /// Required: 6. + pub(crate) unk_4ec: u32, + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_4f0: [u32; 6], + pub(crate) unk_508: [u8; 0x14], + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_51c: [u32; 5], + /// Required: 1. + pub(crate) unk_530: u32, + pub(crate) unk_534: [u8; 0x58], + /// Required: 6. + pub(crate) unk_58c: u32, + /// Required: 0xdac (3500). + pub(crate) unk_590: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_594: u32, + /// Required: 0xbb8 (3000). + pub(crate) unk_598: u32, + /// Required: 0x64 (100). + pub(crate) unk_59c: u32, + pub(crate) unk_5a0: [u8; 0x2c], + /// Required: 1. + pub(crate) unk_5cc: u32, + /// Required: 4. + pub(crate) unk_5d0: u32, + pub(crate) unk_5d4: [u8; 8], + /// Required: 0x1f40 (8000). + pub(crate) unk_5dc: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5e0: u32, + /// Required: 0xfa0 (4000). + pub(crate) unk_5e4: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5e8: u32, + /// Required: 0x7d0 (2000). + pub(crate) unk_5ec: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_5f4: u32, + /// Required: 0xc8 (200). + pub(crate) unk_5f8: u32, + pub(crate) unk_5fc: [u8; 8], + /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). + pub(crate) unk_604: u32, + /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). + pub(crate) unk_608: u32, + /// Required: 0x20 (32). + pub(crate) unk_60c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_610: u32, + pub(crate) unk_614: [u8; 0xc], + /// Required: 0x384 (900). + pub(crate) unk_620: u32, + pub(crate) unk_624: [u8; 0x9c], +} + +// SAFETY: `PowerState` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerState {} + +static_assert!(size_of::() == 0x6c0); + +impl PowerState { + fn init(&mut self) { + self.unk_10 = 0xf; + self.unk_14 = 0x3f00_0000; + self.unk_18 = 0x4088_0000; + self.unk_1c = 0x28; + self.unk_20 = 1; + self.temp_target = TEMP_TARGET; + self.temp_release = TEMP_RELEASE; + self.temp_kp = TEMP_KP; + self.temp_ki = TEMP_KI; + self.unk_f0 = 0x26ac; + self.unk_f4 = 0xc8; + self.unk_108 = 0x42c8_0000; + self.unk_10c = 0x43c8_0000; + self.unk_110 = 0xc8; + self.unk_128 = 1; + self.unk_12c = 0x2616; + self.unk_130 = 0x3f00_0000; + self.unk_134 = 0x40cc_cccd; + self.unk_14c = 1; + self.unk_1c0 = 1; + self.unk_1c4 = 1; + self.unk_1c8 = 4; + self.unk_1cc = [1; 4]; + self.unk_4e4 = 1; + self.unk_4e8 = 0x1f4; + self.unk_4ec = 6; + self.unk_4f0 = [0x30d4; 6]; + self.unk_51c = [0x30d4; 5]; + self.unk_530 = 1; + self.unk_58c = 6; + self.unk_590 = 0xdac; + self.unk_594 = 0x3e8; + self.unk_598 = 0xbb8; + self.unk_59c = 0x64; + self.unk_5cc = 1; + self.unk_5d0 = 4; + self.unk_5dc = 0x1f40; + self.unk_5e0 = 0xc8; + self.unk_5e4 = 0xfa0; + self.unk_5e8 = 0xc8; + self.unk_5ec = 0x7d0; + self.unk_5f0 = 0xc8; + self.unk_5f4 = 0x3e8; + self.unk_5f8 = 0xc8; + self.unk_604 = 0x4170_0000; + self.unk_608 = 0x40a0_0000; + self.unk_60c = 0x20; + self.unk_610 = 0x3e8; + self.unk_620 = 0x384; + } +} + +/// Size of the firmware-control page named by the primary status B. +pub(crate) const FWCTL_SIZE: usize = 0x4000; +/// Offset of the firmware-control ring in its page. +const FWCTL_RING: u64 = 0x40; + +/// One firmware-written recovery-info entry. The host reads each word separately after the +/// recovery handshake reaches state 3; it never copies the live table as a structure. +#[repr(C)] +pub(crate) struct RecoveryInfoEntry { + pub(crate) flags: u32, + pub(crate) stamp: u32, +} +// SAFETY: Both fields are integers. +unsafe impl Zeroable for RecoveryInfoEntry {} +static_assert!(size_of::() == 8); + +/// Status B of the primary instance, directly after its state grid. +#[repr(C, packed)] +pub(crate) struct PrimaryStatusB { + /// Status block. + pub(crate) status: StatusBlock, + pub(crate) unk_80: [u8; 0x4038], + pub(crate) recovery_info: [RecoveryInfoEntry; 256], + pub(crate) unk_48b8: [u8; 0x20], + pub(crate) recovery_slot_mask: u64, + /// Firmware-control state, at the start of the firmware-control page. + pub(crate) fwctl_state_va: u64, + /// Firmware-control ring, in the firmware-control page. + pub(crate) fwctl_ring_va: u64, + /// Firmware-written epoch, read using an aligned 64-bit load. + pub(crate) recovery_epoch: u64, + pub(crate) unk_48f8: [u8; 8], + /// Firmware writes 1 and 3; host writes 2 and 0 with full barriers around each 32-bit store. + pub(crate) recovery_state: u32, + pub(crate) unk_4904: [u8; 0xc], + /// Must be zero for an event-backed restart; read using an aligned 32-bit load. + pub(crate) host_recovery: u32, + pub(crate) unk_4914: [u8; 0x9b20], + /// Required: 1. + pub(crate) unk_e434: u32, + pub(crate) unk_e438: [u8; 8], + /// Power state, see [`PowerState`]. + pub(crate) power: PowerState, +} + +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_info) == 0x40b8); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_slot_mask) == 0x48d8); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_epoch) == 0x48f0); +static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_state) == 0x4900); +static_assert!(core::mem::offset_of!(PrimaryStatusB, host_recovery) == 0x4910); +static_assert!(core::mem::offset_of!(PrimaryStatusB, fwctl_state_va) == 0x48e0); +static_assert!(core::mem::offset_of!(PrimaryStatusB, unk_e434) == 0xe434); +static_assert!(core::mem::offset_of!(PrimaryStatusB, power) == PrimaryStatusB::POWER_STATE); +static_assert!(size_of::() == 0xeb00); + +// SAFETY: `PrimaryStatusB` consists of integers and arrays of integers only. +unsafe impl Zeroable for PrimaryStatusB {} + +impl PrimaryStatusB { + pub(crate) const RECOVERY_SLOT_MASK: usize = core::mem::offset_of!(Self, recovery_slot_mask); + pub(crate) const RECOVERY_INFO: usize = core::mem::offset_of!(Self, recovery_info); + pub(crate) const RECOVERY_STATE: usize = core::mem::offset_of!(Self, recovery_state); + pub(crate) const HOST_RECOVERY: usize = core::mem::offset_of!(Self, host_recovery); + + /// Offset of the [`PowerState`]. + pub(crate) const POWER_STATE: usize = 0xe440; + + /// Initializes the primary status B; `fwctl_va` is the zeroed + /// firmware-control page ([`FWCTL_SIZE`] bytes). + pub(crate) fn init(&mut self, fwctl_va: u64) { + clear(self); + self.status = StatusBlock::new(); + self.fwctl_state_va = fwctl_va; + self.fwctl_ring_va = fwctl_va + FWCTL_RING; + self.unk_e434 = 1; + self.power.init(); + } +} + +/// Configuration of the secondary instance, named by +/// [`SecondaryRoot::config_va`]. +#[repr(C)] +pub(crate) struct SecondaryConfig { + pub(crate) unk_0: [u8; 0x14], + /// Required: 1. + pub(crate) unk_14: u32, + pub(crate) unk_18: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2c: u32, + /// Required: 1. + pub(crate) unk_30: u32, + pub(crate) unk_34: [u8; 0xc], + /// Required: 1. + pub(crate) unk_40: u32, + pub(crate) unk_44: [u8; 8], + /// Required: 1. + pub(crate) unk_4c: u32, + /// Required: 0x186a (6250). + pub(crate) unk_50: u32, + pub(crate) unk_54: [u8; 0x2c], +} + +static_assert!(size_of::() == 0x80); + +// SAFETY: `SecondaryConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for SecondaryConfig {} + +impl SecondaryConfig { + /// Initializes the secondary configuration. + pub(crate) fn init(&mut self) { + clear(self); + self.unk_14 = 1; + self.unk_2c = 1; + self.unk_30 = 1; + self.unk_40 = 1; + self.unk_4c = 1; + self.unk_50 = 0x186a; + } +} + +/// Offsets in the private cluster. +pub(crate) mod private { + /// Size of the private cluster. + pub(crate) const SIZE: usize = 0x178000; + /// The primary state grid. + pub(crate) const PRIMARY_STATE: usize = 0; + /// The primary status A block. + pub(crate) const PRIMARY_STATUS_A: usize = 0xee40; + /// The secondary status A block. + pub(crate) const SECONDARY_STATUS_A: usize = 0xc3100; + /// The secondary state grid. + pub(crate) const SECONDARY_STATE: usize = 0x1770c0; + /// [`super::PrimaryStatusB`] or [`super::SecondaryConfig`], from an + /// instance's state grid. + pub(crate) const STATUS_B: usize = 0x340; + /// Secondary state named by [`super::HwDataAux`], from the secondary + /// state grid. + pub(crate) const SECONDARY_STATUS: usize = 0x3c0; +} + +static_assert!(private::STATUS_B == CONTROL_STATE_OFFSET + size_of::()); +static_assert!( + private::PRIMARY_STATE + private::STATUS_B + size_of::() + == private::PRIMARY_STATUS_A +); +static_assert!(private::STATUS_B + size_of::() == private::SECONDARY_STATUS); + +// --------------------------------------------------------------------------- +// Power configuration and auxiliary hardware data (bundle) +// --------------------------------------------------------------------------- + +/// Die temperature sensors read by the die temperature limiter. +const TEMP_SENSOR_MASK: u64 = 0x82a; +/// Die temperature the limiter holds the GPU below, in 0.01 °C. +const TEMP_TARGET: u32 = 11_000; +/// Die temperature below which the limiter releases, in 0.01 °C. +const TEMP_RELEASE: u32 = 8_000; +/// Proportional gain of the limiter: 20.34 as an IEEE-754 single. +const TEMP_KP: u32 = 0x41a2_b852; +/// Integral gain of the limiter, per control period: 9.376 as an IEEE-754 +/// single. +const TEMP_KI: u32 = 0x4116_0419; + +/// Leading part of the [`PowerConfig`]. The host writes it twice: in the +/// power configuration and at [`bundle::POWER_CONFIG_COPY`], where the copy +/// ends with the views it lies in. +#[repr(C, packed)] +pub(crate) struct PowerConfigHead { + pub(crate) unk_0: [u8; 4], + /// Required: 0x5_dc00 (384000). + pub(crate) unk_4: u32, + /// Required: 0x5_dc00 (384000). + pub(crate) unk_8: u32, + pub(crate) unk_c: [u8; 4], + /// Required: 4. + pub(crate) unk_10: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_14: u32, + pub(crate) unk_18: [u8; 0x14], + /// Required: 1. + pub(crate) unk_2c: u32, + /// Required: 1. + pub(crate) unk_30: u32, + pub(crate) unk_34: [u8; 0xc], + /// Required: 0x64 (100). + pub(crate) unk_40: u32, + /// Required: 1. + pub(crate) unk_44: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_48: u32, + pub(crate) unk_4c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_50: u32, + pub(crate) unk_54: [u8; 0xc], + /// Required: 4. + pub(crate) unk_60: u32, + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_64: u32, + pub(crate) unk_68: [u8; 4], + /// Required: 1. + pub(crate) unk_6c: u32, + /// Required: 1. + pub(crate) unk_70: u32, + pub(crate) unk_74: [u8; 0xc], + /// Required: 0x64 (100). + pub(crate) unk_80: u32, + /// Required: 1. + pub(crate) unk_84: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_88: u32, + pub(crate) unk_8c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_90: u32, + pub(crate) unk_94: [u8; 0x2c], + /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. + pub(crate) unk_c0: [u32; 11], + pub(crate) unk_ec: [u8; 0x8cc], + /// Required: 0x271 (625). + pub(crate) unk_9b8: u32, + pub(crate) unk_9bc: [u8; 8], + /// Required: 0x3f7f_2e9f. + pub(crate) unk_9c4: u32, + pub(crate) unk_9c8: [u8; 4], + /// Required: 0x3b51_6154. + pub(crate) unk_9cc: u32, + pub(crate) unk_9d0: [u8; 4], + /// Required: 0x3ca5_9586 (0.020213 as an IEEE-754 single). + pub(crate) unk_9d4: u32, + pub(crate) unk_9d8: [u8; 8], + /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). + pub(crate) unk_9e0: u32, + /// Required: 0x40a9_0fdb. + pub(crate) unk_9e4: u32, + pub(crate) unk_9e8: [u8; 4], + /// Required: 0xbdd6_7344. + pub(crate) unk_9ec: u32, + /// Required: 0x28 (40). + pub(crate) unk_9f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_9f4: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_9f8: u32, + pub(crate) unk_9fc: [u8; 8], + /// Required: 0x254e (9550). + pub(crate) unk_a04: u32, + pub(crate) unk_a08: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_a38: u32, + pub(crate) unk_a3c: [u8; 0x34], + /// Required: 4. + pub(crate) unk_a70: u32, + pub(crate) unk_a74: [u8; 0x10], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_a84: u32, + pub(crate) unk_a88: [u8; 4], + /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). + pub(crate) unk_a8c: u32, + pub(crate) unk_a90: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_a98: u32, + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_a9c: u32, + pub(crate) unk_aa0: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_aa8: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_aac: u32, + pub(crate) unk_ab0: [u8; 0xc], + /// Required: 0x254e (9550). + pub(crate) unk_abc: u32, + pub(crate) unk_ac0: [u8; 8], + /// Required: 0x10 (16). + pub(crate) unk_ac8: u32, + pub(crate) unk_acc: [u8; 4], + /// Required: 0x5_dc00 (384000). + pub(crate) unk_ad0: u32, + pub(crate) unk_ad4: [u8; 0x54], + /// Required: 0x5c (92). + pub(crate) unk_b28: u32, + pub(crate) unk_b2c: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_b30: u32, + /// Required: 0x22 (34). + pub(crate) unk_b34: u32, + /// Required: 6. + pub(crate) unk_b38: u32, + pub(crate) unk_b3c: [u8; 4], + /// Required: 6. + pub(crate) unk_b40: u32, + /// Required: 1. + pub(crate) unk_b44: u32, + pub(crate) unk_b48: [u8; 4], + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_b4c: u32, + /// Required: 0x3f7d_f3b6 (0.992 as an IEEE-754 single). + pub(crate) unk_b50: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_b54: u32, + /// Required: 0x3c03_126f (0.008 as an IEEE-754 single). + pub(crate) unk_b58: u32, + /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). + pub(crate) unk_b5c: u32, + /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). + pub(crate) unk_b60: u32, + pub(crate) unk_b64: [u8; 4], + /// Required: 0x42be_0000 (95.0 as an IEEE-754 single). + pub(crate) unk_b68: u32, + /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). + pub(crate) unk_b6c: u32, + /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). + pub(crate) unk_b70: u32, + /// Required: 0x4117_9436. + pub(crate) unk_b74: u32, + /// Required: 0x64 (100). + pub(crate) unk_b78: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_b7c: u32, + /// Required: 0x64 (100). + pub(crate) unk_b80: u32, + pub(crate) unk_b84: [u8; 8], + /// Required: 0x5c (92). + pub(crate) unk_b8c: u32, + pub(crate) unk_b90: [u8; 0x30], + /// Required: 0x64 (100). + pub(crate) unk_bc0: u32, + pub(crate) unk_bc4: [u8; 0x1c], + /// Required: 1. + pub(crate) unk_be0: u8, + /// Required: 4. + pub(crate) unk_be1: u32, + /// Required: 0x64 (100). + pub(crate) unk_be5: u32, + pub(crate) unk_be9: [u8; 4], + /// Required: 0x1f40 (8000). + pub(crate) unk_bed: u32, + /// Required: 0xc8 (200). + pub(crate) unk_bf1: u32, + /// Required: 0xfa0 (4000). + pub(crate) unk_bf5: u32, + /// Required: 0xc8 (200). + pub(crate) unk_bf9: u32, + /// Required: 0x7d0 (2000). + pub(crate) unk_bfd: u32, + /// Required: 0xc8 (200). + pub(crate) unk_c01: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_c05: u32, + /// Required: 0xc8 (200). + pub(crate) unk_c09: u32, + /// Required: 1. + pub(crate) unk_c0d: u32, + pub(crate) unk_c11: [u8; 0x53], + /// Required: 1. + pub(crate) unk_c64: u32, + pub(crate) unk_c68: [u8; 0x28], + /// Required: 1. + pub(crate) unk_c90: u32, + pub(crate) unk_c94: [u8; 0x1003], + /// Required: 0x7fff in every entry; the table continues in + /// [`PowerConfigTail`]. + pub(crate) unk_1c97: [u16; 148], + /// Required: 0xff (low byte of the next table entry). + pub(crate) unk_1dbf: u8, +} + +static_assert!(bundle::POWER_CONFIG_COPY + size_of::() == 0xf000); + +// SAFETY: `PowerConfigHead` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfigHead {} + +impl PowerConfigHead { + /// Initializes the leading part of the power configuration. + pub(crate) fn init(&mut self) { + clear(self); + self.unk_4 = 0x5_dc00; + self.unk_8 = 0x5_dc00; + self.unk_10 = 4; + self.unk_14 = 0x3f80_0000; + self.unk_2c = 1; + self.unk_30 = 1; + self.unk_40 = 0x64; + self.unk_44 = 1; + self.unk_48 = 0x3e8; + self.unk_50 = 0x64; + self.unk_60 = 4; + self.unk_64 = 0x3f80_0000; + self.unk_6c = 1; + self.unk_70 = 1; + self.unk_80 = 0x64; + self.unk_84 = 1; + self.unk_88 = 0x3e8; + self.unk_90 = 0x64; + self.unk_c0 = [0x3f82_8f5c; 11]; + self.unk_9b8 = 0x271; + self.unk_9c4 = 0x3f7f_2e9f; + self.unk_9cc = 0x3b51_6154; + self.unk_9d4 = 0x3ca5_9586; + self.unk_9e0 = 0x4615_3800; + self.unk_9e4 = 0x40a9_0fdb; + self.unk_9ec = 0xbdd6_7344; + self.unk_9f0 = 0x28; + self.unk_9f4 = 0x3e8; + self.unk_9f8 = 0x3e8; + self.unk_a04 = 0x254e; + self.unk_a38 = 0x3e8; + self.unk_a70 = 4; + self.unk_a84 = 0x3f80_0000; + self.unk_a8c = 0x40cc_cccd; + self.unk_a98 = 0x4780_0000; + self.unk_a9c = 0x3f00_0000; + self.unk_aa8 = 0x28; + self.unk_aac = 0x3e8; + self.unk_abc = 0x254e; + self.unk_ac8 = 0x10; + self.unk_ad0 = 0x5_dc00; + self.unk_b28 = 0x5c; + self.unk_b30 = 0x64; + self.unk_b34 = 0x22; + self.unk_b38 = 6; + self.unk_b40 = 6; + self.unk_b44 = 1; + self.unk_b4c = 0x3f4c_cccd; + self.unk_b50 = 0x3f7d_f3b6; + self.unk_b54 = 0x3e4c_cccd; + self.unk_b58 = 0x3c03_126f; + self.unk_b5c = 0x3f69_d4d8; + self.unk_b60 = 0x3f69_d4d8; + self.unk_b68 = 0x42be_0000; + self.unk_b6c = 0x4064_f5c3; + self.unk_b70 = 0x4064_f5c3; + self.unk_b74 = 0x4117_9436; + self.unk_b78 = 0x64; + self.unk_b7c = 0x3e8; + self.unk_b80 = 0x64; + self.unk_b8c = 0x5c; + self.unk_bc0 = 0x64; + self.unk_be0 = 1; + self.unk_be1 = 4; + self.unk_be5 = 0x64; + self.unk_bed = 0x1f40; + self.unk_bf1 = 0xc8; + self.unk_bf5 = 0xfa0; + self.unk_bf9 = 0xc8; + self.unk_bfd = 0x7d0; + self.unk_c01 = 0xc8; + self.unk_c05 = 0x3e8; + self.unk_c09 = 0xc8; + self.unk_c0d = 1; + self.unk_c64 = 1; + self.unk_c90 = 1; + self.unk_1c97 = [0x7fff; 148]; + self.unk_1dbf = 0xff; + } +} + +/// Trailing part of the [`PowerConfig`]. Field names give the offset in the +/// whole power configuration. It carries the die temperature limiter: target +/// and release temperatures in 0.01 °C, proportional and integral gains as +/// IEEE-754 singles. +#[repr(C, packed)] +pub(crate) struct PowerConfigTail { + /// Required: 0x7f (high byte of the table entry started in the head). + pub(crate) unk_1dc0: u8, + /// Required: 0x7fff in every entry. + pub(crate) unk_1dc1: [u16; 363], + pub(crate) unk_2097: [u8; 0x201], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2298: u32, + /// Required: 0x4220_0000 (40.0 as an IEEE-754 single). + pub(crate) unk_229c: u32, + /// Required: 0x447a_0000 (1000.0 as an IEEE-754 single). + pub(crate) unk_22a0: u32, + /// Required: 0x5be (1470). + pub(crate) unk_22a4: u32, + pub(crate) unk_22a8: [u8; 0x10], + /// Required: 0x28 (40). + pub(crate) unk_22b8: u32, + pub(crate) unk_22bc: [u8; 4], + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_22c0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_22c4: u32, + pub(crate) unk_22c8: [u8; 4], + /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). + pub(crate) unk_22cc: u32, + /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). + pub(crate) unk_22d0: u32, + pub(crate) unk_22d4: [u8; 0x3c], + /// Die temperature sensors the limiter reads ([`TEMP_SENSOR_MASK`]). + pub(crate) temp_sensor_mask: u64, + /// Release temperature of the limiter ([`TEMP_RELEASE`]). + pub(crate) temp_release: u32, + pub(crate) unk_231c: [u8; 4], + /// Required: 4. + pub(crate) unk_2320: u32, + pub(crate) unk_2324: [u8; 0x10], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_2334: u32, + pub(crate) unk_2338: [u8; 4], + /// Integral gain of the limiter ([`TEMP_KI`]). + pub(crate) temp_ki: u32, + pub(crate) unk_2340: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2348: u32, + /// Proportional gain of the limiter ([`TEMP_KP`]). + pub(crate) temp_kp: u32, + pub(crate) unk_2350: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_2358: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_235c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2360: u32, + /// Margin below the target at which the limiter acts, in 0.01 °C. + pub(crate) temp_margin: u32, + pub(crate) unk_2368: [u8; 4], + /// Target temperature of the limiter ([`TEMP_TARGET`]). + pub(crate) temp_target: u32, + pub(crate) unk_2370: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_23a0: u32, + pub(crate) unk_23a4: [u8; 0x218], + /// Required: 0x4200_0006. + pub(crate) unk_25bc: u32, + pub(crate) unk_25c0: [u8; 0x212], + /// Required: 0x3f7e_b852 (0.995 as an IEEE-754 single). + pub(crate) unk_27d2: u32, + pub(crate) unk_27d6: [u8; 4], + /// Required: 0x3ba3_d70a (0.005 as an IEEE-754 single). + pub(crate) unk_27da: u32, + pub(crate) unk_27de: [u8; 4], + /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). + pub(crate) unk_27e2: u32, + pub(crate) unk_27e6: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_27ee: u32, + /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). + pub(crate) unk_27f2: u32, + pub(crate) unk_27f6: [u8; 4], + /// Required: 0xbac8_0000. + pub(crate) unk_27fa: u32, + /// Required: 0x384 (900). + pub(crate) unk_27fe: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2802: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2806: u32, + pub(crate) unk_280a: [u8; 4], + /// Required: 0x453b_8000 (3000.0 as an IEEE-754 single). + pub(crate) unk_280e: u32, + /// Required: 0x26ac (9900). + pub(crate) unk_2812: u32, + pub(crate) unk_2816: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_2846: u32, + pub(crate) unk_284a: [u8; 0x18], + /// Required: 0xc8 (200). + pub(crate) unk_2862: u32, + pub(crate) unk_2866: [u8; 0xb2], + /// Required: 0x3e8 (1000). + pub(crate) unk_2918: u32, + pub(crate) unk_291c: [u8; 0x10], + /// Required: 1. + pub(crate) unk_292c: u32, + pub(crate) unk_2930: [u8; 0x30], + /// Required: 1. + pub(crate) unk_2960: u8, + /// Required: 1. + pub(crate) unk_2961: u32, + /// Required: 4. + pub(crate) unk_2965: u32, + pub(crate) unk_2969: [u8; 0x14], + /// Required: 0x23f (575). + pub(crate) unk_297d: u32, + /// Required: 1. + pub(crate) unk_2981: u32, + /// Required: 1. + pub(crate) unk_2985: u32, + pub(crate) unk_2989: [u8; 0x14], + /// Required: 0x240 (576). + pub(crate) unk_299d: u32, + /// Required: 1. + pub(crate) unk_29a1: u32, + /// Required: 1. + pub(crate) unk_29a5: u32, + pub(crate) unk_29a9: [u8; 0x14], + /// Required: 0x241 (577). + pub(crate) unk_29bd: u32, + pub(crate) unk_29c1: [u8; 0x47], + /// Required: 1. + pub(crate) unk_2a08: u32, + pub(crate) unk_2a0c: [u8; 4], + /// Required: 0x254e (9550) in every entry. + pub(crate) unk_2a10: [u32; 3], + pub(crate) unk_2a1c: [u8; 0xc], + /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). + pub(crate) unk_2a28: u32, + /// Required: 4. + pub(crate) unk_2a2c: u32, + /// Required: 0x10 (16). + pub(crate) unk_2a30: u32, + /// Required: 0x5_dc00 (384000). + pub(crate) unk_2a34: u32, + pub(crate) unk_2a38: [u8; 0xcc], + /// Required: 0x3c (60). + pub(crate) unk_2b04: u32, + pub(crate) unk_2b08: [u8; 8], + /// Required: 0x3f6e_eeef. + pub(crate) unk_2b10: u32, + pub(crate) unk_2b14: [u8; 4], + /// Required: 0x3d88_8889. + pub(crate) unk_2b18: u32, + pub(crate) unk_2b1c: [u8; 4], + /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). + pub(crate) unk_2b20: u32, + pub(crate) unk_2b24: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2b2c: u32, + /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). + pub(crate) unk_2b30: u32, + pub(crate) unk_2b34: [u8; 8], + /// Required: 0x28 (40). + pub(crate) unk_2b3c: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2b40: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2b44: u32, + pub(crate) unk_2b48: [u8; 4], + /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). + pub(crate) unk_2b4c: u32, + /// Required: 0x254e (9550). + pub(crate) unk_2b50: u32, + pub(crate) unk_2b54: [u8; 8], + /// Required: 0xf0 (240). + pub(crate) unk_2b5c: u32, + pub(crate) unk_2b60: [u8; 4], + /// Required: 0x57_e400. + pub(crate) unk_2b64: u32, + pub(crate) unk_2b68: [u8; 0x1c], + /// Required: 0x3e8 (1000). + pub(crate) unk_2b84: u32, + pub(crate) unk_2b88: [u8; 0x3c], + /// Required: 1. + pub(crate) unk_2bc4: u32, + pub(crate) unk_2bc8: [u8; 0xcc], + /// Required: 0x32 (50). + pub(crate) unk_2c94: u32, + /// Required: 1. + pub(crate) unk_2c98: u32, + pub(crate) unk_2c9c: [u8; 4], + /// Required: 0x3f63_8e39. + pub(crate) unk_2ca0: u32, + /// Required: 0x3f2a_aaab. + pub(crate) unk_2ca4: u32, + /// Required: 0x3de3_8e39. + pub(crate) unk_2ca8: u32, + /// Required: 0x3eaa_aaab. + pub(crate) unk_2cac: u32, + /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). + pub(crate) unk_2cb0: u32, + /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). + pub(crate) unk_2cb4: u32, + pub(crate) unk_2cb8: [u8; 4], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2cbc: u32, + /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). + pub(crate) unk_2cc0: u32, + /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). + pub(crate) unk_2cc4: u32, + pub(crate) unk_2cc8: [u8; 4], + /// Required: 0x64 (100). + pub(crate) unk_2ccc: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2cd0: u32, + /// Required: 0x64 (100). + pub(crate) unk_2cd4: u32, + pub(crate) unk_2cd8: [u8; 4], + /// Required: 0x467a_0000 (16000.0 as an IEEE-754 single). + pub(crate) unk_2cdc: u32, + /// Required: 0x578 (1400). + pub(crate) unk_2ce0: u32, + pub(crate) unk_2ce4: [u8; 8], + /// Required: 0x90 (144). + pub(crate) unk_2cec: u32, + /// Required: 0x30 (48). + pub(crate) unk_2cf0: u32, + /// Required: 0x34_bc00. + pub(crate) unk_2cf4: u32, + pub(crate) unk_2cf8: [u8; 4], + /// Required: 0x11_9400. + pub(crate) unk_2cfc: u32, + pub(crate) unk_2d00: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2d08: u32, + pub(crate) unk_2d0c: [u8; 0x24], + /// Required: 0x3e80 (16000). + pub(crate) unk_2d30: u32, + /// Required: 2. + pub(crate) unk_2d34: u32, + /// Required: 0x9c4 (2500). + pub(crate) unk_2d38: u32, + /// Required: 0x20d (525). + pub(crate) unk_2d3c: u32, + /// Required: 2. + pub(crate) unk_2d40: u32, + /// Required: 4. + pub(crate) unk_2d44: u32, + /// Required: 0x32 (50). + pub(crate) unk_2d48: u32, + pub(crate) unk_2d4c: [u8; 0x2c], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_2d78: u32, + pub(crate) unk_2d7c: [u8; 0xc], + /// Required: 0x28 (40). + pub(crate) unk_2d88: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_2d8c: u32, + pub(crate) unk_2d90: [u8; 0x30], +} + +static_assert!(core::mem::offset_of!(PowerConfigTail, temp_sensor_mask) == 0x2310 - 0x1dc0); +static_assert!(core::mem::offset_of!(PowerConfigTail, temp_target) == 0x236c - 0x1dc0); +static_assert!(size_of::() == 0x1000); + +// SAFETY: `PowerConfigTail` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfigTail {} + +impl PowerConfigTail { + fn init(&mut self) { + clear(self); + self.unk_1dc0 = 0x7f; + self.unk_1dc1 = [0x7fff; 363]; + self.unk_2298 = 0x4780_0000; + self.unk_229c = 0x4220_0000; + self.unk_22a0 = 0x447a_0000; + self.unk_22a4 = 0x5be; + self.unk_22b8 = 0x28; + self.unk_22c0 = 0x42c8_0000; + self.unk_22c4 = 0x3e8; + self.unk_22cc = 0x3f4c_cccd; + self.unk_22d0 = 0x3e4c_cccd; + self.temp_sensor_mask = TEMP_SENSOR_MASK; + self.temp_release = TEMP_RELEASE; + self.unk_2320 = 4; + self.unk_2334 = 0x3f80_0000; + self.temp_ki = TEMP_KI; + self.unk_2348 = 0x4780_0000; + self.temp_kp = TEMP_KP; + self.unk_2358 = 0x28; + self.unk_235c = 0x3e8; + self.unk_2360 = 0x3e8; + self.temp_target = TEMP_TARGET; + self.unk_23a0 = 0x3e8; + self.unk_25bc = 0x4200_0006; + self.unk_27d2 = 0x3f7e_b852; + self.unk_27da = 0x3ba3_d70a; + self.unk_27e2 = 0x43c8_0000; + self.unk_27ee = 0x4780_0000; + self.unk_27f2 = 0x42c8_0000; + self.unk_27fa = 0xbac8_0000; + self.unk_27fe = 0x384; + self.unk_2802 = 0x3e8; + self.unk_2806 = 0x3e8; + self.unk_280e = 0x453b_8000; + self.unk_2812 = 0x26ac; + self.unk_2846 = 0x3e8; + self.unk_2862 = 0xc8; + self.unk_2918 = 0x3e8; + self.unk_292c = 1; + self.unk_2960 = 1; + self.unk_2961 = 1; + self.unk_2965 = 4; + self.unk_297d = 0x23f; + self.unk_2981 = 1; + self.unk_2985 = 1; + self.unk_299d = 0x240; + self.unk_29a1 = 1; + self.unk_29a5 = 1; + self.unk_29bd = 0x241; + self.unk_2a08 = 1; + self.unk_2a10 = [0x254e; 3]; + self.unk_2a28 = 0x3f80_0000; + self.unk_2a2c = 4; + self.unk_2a30 = 0x10; + self.unk_2a34 = 0x5_dc00; + self.unk_2b04 = 0x3c; + self.unk_2b10 = 0x3f6e_eeef; + self.unk_2b18 = 0x3d88_8889; + self.unk_2b20 = 0x3f00_0000; + self.unk_2b2c = 0x4780_0000; + self.unk_2b30 = 0x4088_0000; + self.unk_2b3c = 0x28; + self.unk_2b40 = 0x3e8; + self.unk_2b44 = 0x3e8; + self.unk_2b4c = 0x4615_3800; + self.unk_2b50 = 0x254e; + self.unk_2b5c = 0xf0; + self.unk_2b64 = 0x57_e400; + self.unk_2b84 = 0x3e8; + self.unk_2bc4 = 1; + self.unk_2c94 = 0x32; + self.unk_2c98 = 1; + self.unk_2ca0 = 0x3f63_8e39; + self.unk_2ca4 = 0x3f2a_aaab; + self.unk_2ca8 = 0x3de3_8e39; + self.unk_2cac = 0x3eaa_aaab; + self.unk_2cb0 = 0xbf4c_cccd; + self.unk_2cb4 = 0xbf4c_cccd; + self.unk_2cbc = 0x4780_0000; + self.unk_2cc0 = 0xc0a0_0000; + self.unk_2cc4 = 0xc0a0_0000; + self.unk_2ccc = 0x64; + self.unk_2cd0 = 0x3e8; + self.unk_2cd4 = 0x64; + self.unk_2cdc = 0x467a_0000; + self.unk_2ce0 = 0x578; + self.unk_2cec = 0x90; + self.unk_2cf0 = 0x30; + self.unk_2cf4 = 0x34_bc00; + self.unk_2cfc = 0x11_9400; + self.unk_2d08 = 0x4780_0000; + self.unk_2d30 = 0x3e80; + self.unk_2d34 = 2; + self.unk_2d38 = 0x9c4; + self.unk_2d3c = 0x20d; + self.unk_2d40 = 2; + self.unk_2d44 = 4; + self.unk_2d48 = 0x32; + self.unk_2d78 = 0x4780_0000; + self.unk_2d88 = 0x28; + self.unk_2d8c = 0x3e8; + } +} + +/// Power configuration, in view 2 of the bundle. The secondary instance +/// finds it through [`MainConfig::power_config_va`]. +#[repr(C, packed)] +pub(crate) struct PowerConfig { + /// Leading part. + pub(crate) head: PowerConfigHead, + /// Trailing part. + pub(crate) tail: PowerConfigTail, +} + +static_assert!(bundle::POWER_CONFIG + size_of::() == bundle::AUX); +static_assert!(bundle::VIEWS[2] + 0xe40 == bundle::POWER_CONFIG); + +// SAFETY: `PowerConfig` consists of integers and arrays of integers only. +unsafe impl Zeroable for PowerConfig {} + +impl PowerConfig { + /// Initializes the power configuration. + pub(crate) fn init(&mut self) { + self.head.init(); + self.tail.init(); + } +} + +/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the +/// start of view 3. No pointer names it. +#[repr(C, packed)] +pub(crate) struct HwDataAux { + pub(crate) unk_0: [u8; 0xac], + /// Required: 6. + pub(crate) unk_ac: u32, + pub(crate) unk_b0: [u8; 8], + /// Required: 1. + pub(crate) unk_b8: u32, + pub(crate) unk_bc: [u8; 0xbb], + /// Required: 0xdac (3500). + pub(crate) unk_177: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_17b: u32, + /// Required: 0xbb8 (3000). + pub(crate) unk_17f: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_183: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_187: u32, + /// Required: 0x64 (100). + pub(crate) unk_18b: u32, + /// Required: 0x20 (32). + pub(crate) unk_18f: u32, + pub(crate) unk_193: [u8; 0x21], + /// Required: 4. + pub(crate) unk_1b4: u32, + pub(crate) unk_1b8: [u8; 8], + /// Required: 0x3f78_0000 (0.96875 as an IEEE-754 single). + pub(crate) unk_1c0: u32, + pub(crate) unk_1c4: [u8; 4], + /// Required: 0x3d00_0000 (0.03125 as an IEEE-754 single). + pub(crate) unk_1c8: u32, + pub(crate) unk_1cc: [u8; 4], + /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). + pub(crate) unk_1d0: u32, + pub(crate) unk_1d4: [u8; 8], + /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). + pub(crate) unk_1dc: u32, + /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). + pub(crate) unk_1e0: u32, + pub(crate) unk_1e4: [u8; 8], + /// Required: 0x384 (900). + pub(crate) unk_1ec: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_1f0: u32, + /// Required: 0x3e8 (1000). + pub(crate) unk_1f4: u32, + pub(crate) unk_1f8: [u8; 8], + /// Required: 0x3e8 (1000). + pub(crate) unk_200: u32, + pub(crate) unk_204: [u8; 0x30], + /// Required: 0x3e8 (1000). + pub(crate) unk_234: u32, + pub(crate) unk_238: [u8; 0x3c], + /// Required: 0x64 (100). + pub(crate) unk_274: u32, + pub(crate) unk_278: [u8; 0x18], + /// Required: 0x82a (2090). + pub(crate) unk_290: u32, + pub(crate) unk_294: [u8; 4], + /// Required: 0x7d (125). + pub(crate) unk_298: u32, + /// Required: 1. + pub(crate) unk_29c: u32, + pub(crate) unk_2a0: [u8; 0xc38], + /// GPU address of the secondary state, [`private::SECONDARY_STATUS`] bytes + /// into the secondary state grid. + pub(crate) secondary_status_va: u64, + /// Required: 0x18ae (6318). + pub(crate) unk_ee0: u32, + pub(crate) unk_ee4: [u8; 4], + /// Required: 1. + pub(crate) unk_ee8: u32, + pub(crate) unk_eec: [u8; 0x2000], + /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). + pub(crate) unk_2eec: u32, + /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). + pub(crate) unk_2ef0: u32, + pub(crate) unk_2ef4: [u8; 0x6f0], + /// Required: 1. + pub(crate) unk_35e4: u32, + /// Required: 0x1f4 (500). + pub(crate) unk_35e8: u32, + /// Required: 6. + pub(crate) unk_35ec: u32, + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_35f0: [u32; 6], + /// Required: 0xfa0 (4000). + pub(crate) unk_3608: u32, + pub(crate) unk_360c: [u8; 0x10], + /// Required: 0x30d4 (12500) in every entry. + pub(crate) unk_361c: [u32; 5], + /// Required: 1. + pub(crate) unk_3630: u32, + pub(crate) unk_3634: [u8; 0x174], + /// Required: 1. + pub(crate) unk_37a8: u32, + pub(crate) unk_37ac: [u8; 8], + /// Required: 0x424c_cccd (51.2 as an IEEE-754 single). + pub(crate) unk_37b4: u32, + pub(crate) unk_37b8: [u8; 0x3c], + /// Required: 0x4406_8000 (538.0 as an IEEE-754 single). + pub(crate) unk_37f4: u32, + pub(crate) unk_37f8: [u8; 0x3c], + /// Required: 0x41c9_999a (25.2 as an IEEE-754 single). + pub(crate) unk_3834: u32, + pub(crate) unk_3838: [u8; 0xa8], + /// Required: 0x186a (6250). + pub(crate) unk_38e0: u32, + pub(crate) unk_38e4: [u8; 0x39c], +} + +static_assert!(core::mem::offset_of!(HwDataAux, secondary_status_va) == 0xed8); +static_assert!(bundle::AUX + size_of::() == bundle::VIEWS[3]); + +// SAFETY: `HwDataAux` consists of integers and arrays of integers only. +unsafe impl Zeroable for HwDataAux {} + +impl HwDataAux { + /// Initializes the auxiliary hardware data. `secondary_status_va` is + /// the secondary state grid plus [`private::SECONDARY_STATUS`]. + pub(crate) fn init(&mut self, secondary_status_va: u64) { + clear(self); + self.unk_ac = 6; + self.unk_b8 = 1; + self.unk_177 = 0xdac; + self.unk_17b = 0x3e8; + self.unk_17f = 0xbb8; + self.unk_183 = 0x3e8; + self.unk_187 = 0x3e8; + self.unk_18b = 0x64; + self.unk_18f = 0x20; + self.unk_1b4 = 4; + self.unk_1c0 = 0x3f78_0000; + self.unk_1c8 = 0x3d00_0000; + self.unk_1d0 = 0x40a0_0000; + self.unk_1dc = 0x4780_0000; + self.unk_1e0 = 0x4170_0000; + self.unk_1ec = 0x384; + self.unk_1f0 = 0x3e8; + self.unk_1f4 = 0x3e8; + self.unk_200 = 0x3e8; + self.unk_234 = 0x3e8; + self.unk_274 = 0x64; + self.unk_290 = 0x82a; + self.unk_298 = 0x7d; + self.unk_29c = 1; + self.unk_ee0 = 0x18ae; + self.unk_ee8 = 1; + self.unk_2eec = 0x41c8_cccd; + self.unk_2ef0 = 0x41c8_cccd; + self.unk_35e4 = 1; + self.unk_35e8 = 0x1f4; + self.unk_35ec = 6; + self.unk_35f0 = [0x30d4; 6]; + self.unk_3608 = 0xfa0; + self.unk_361c = [0x30d4; 5]; + self.unk_3630 = 1; + self.unk_37a8 = 1; + self.unk_37b4 = 0x424c_cccd; + self.unk_37f4 = 0x4406_8000; + self.unk_3834 = 0x41c9_999a; + self.unk_38e0 = 0x186a; + self.secondary_status_va = secondary_status_va; + } +} +// --------------------------------------------------------------------------- +// Free lists +// --------------------------------------------------------------------------- + +/// Size of a free-list page. +const FREE_LIST_PAGE_SIZE: u64 = 0x1000; +/// Pages in one free-list block. +pub(crate) const FREE_LIST_BLOCK_PAGES: u32 = 0x100; +/// Size of one free-list block. +pub(crate) const FREE_LIST_BLOCK_SIZE: usize = + FREE_LIST_BLOCK_PAGES as usize * FREE_LIST_PAGE_SIZE as usize; +/// Run-list qwords reserved for each block. +const RUN_SLOT_QWORDS: u32 = 8; +/// Bit position of the page count in a run descriptor. +const RUN_PAGES_SHIFT: u32 = 52; +/// Address bits of a run descriptor. +const RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; + +/// Class of a free list, selecting [`FreeListControl::unk_10`] and +/// [`FreeListControl::unk_60`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum FreeListClass { + /// `unk_10` 1, `unk_60` 2: the boot render and compute free lists. + One = 1, + /// `unk_10` 2, `unk_60` 3: the auxiliary boot and runtime render pools. + Two = 2, +} + +impl FreeListClass { + const fn unk_60(self) -> u32 { + match self { + Self::One => 2, + Self::Two => 3, + } + } +} + +/// Description of a free list whose blocks are all populated. +pub(crate) struct FreeListArgs { + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + /// Class of the free list. + pub(crate) class: FreeListClass, + /// Page list: one 64-bit address per page of the blocks. + pub(crate) page_list_va: u64, + /// Run list: one [`FreeListRunSlot`] per block. + pub(crate) run_list_va: u64, + /// Blocks in the free list. + pub(crate) blocks: u32, + /// [`FreeListState`]. + pub(crate) state_va: u64, +} + +/// Control object of a USC private-memory free list, at the start of its +/// page. +/// +/// This constructor describes the boot objects. Once a pool is published, its +/// page counters (`+0x24`, `+0x2c`) and run cursor (`+0x48`) are shared state. +/// The firmware-completed operation counter occupies the eight bytes at +/// `+0x54`; it is unaligned and the host reads its bytes individually. +/// A released render pool can be repopulated only after its release is +/// consumed: write page-list/run-list words first, then the complete control +/// page in ascending aligned 64-bit words, then a full barrier. The completed +/// count in that image must equal the retained submitted count, not zero. +/// Publication and reuse belong to the host free-list implementation. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListControl { + /// Boot value 1, compute value 2, or the render submitted-counter address. + pub(crate) unk_0: u64, + /// Buffer slot of the free list. + pub(crate) buffer_slot: u32, + pub(crate) unk_c: u32, + /// Required: see [`FreeListClass`]. + pub(crate) unk_10: u32, + /// Page list. + pub(crate) page_list_va: u64, + /// Required: 0x40000. + pub(crate) unk_1c: u32, + pub(crate) unk_20: u32, + /// Required: [`Self::page_count`]. + pub(crate) unk_24: u32, + pub(crate) unk_28: u32, + /// Pages in the free list's blocks. + pub(crate) page_count: u32, + /// Run list. + pub(crate) run_list_va: u64, + pub(crate) unk_38: [u8; 8], + /// Required: 4. + pub(crate) unk_40: u32, + pub(crate) unk_44: u32, + /// Run-list qwords in use. + pub(crate) run_cursor: u32, + /// [`FreeListState`]. + pub(crate) state_va: u64, + pub(crate) unk_54: [u8; 0xc], + /// Required: see [`FreeListClass`]. + pub(crate) unk_60: u32, + pub(crate) unk_64: u32, +} + +// SAFETY: `FreeListControl` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListControl {} + +static_assert!(core::mem::offset_of!(FreeListControl, page_list_va) == 0x14); +static_assert!(core::mem::offset_of!(FreeListControl, run_cursor) == 0x48); +static_assert!(core::mem::offset_of!(FreeListControl, state_va) == 0x4c); +static_assert!(size_of::() == 0x68); + +impl FreeListControl { + /// Runtime render control; submitted and completed work are retained + /// across release/repopulation of the same pool. + pub(crate) fn render(args: &FreeListArgs, completed: u64) -> Self { + let mut control = Self::new(args); + control.unk_0 = args.state_va; + control.unk_54[..8].copy_from_slice(&completed.to_le_bytes()); + control + } + + /// Runtime compute control, with a compact run list and shared counters. + pub(crate) fn compute(args: &FreeListArgs) -> Self { + let mut control = Self::new(args); + control.unk_0 = 2; + control.unk_10 = 2; + control.unk_54[8..12].copy_from_slice(&1u32.to_le_bytes()); + control + } + + /// Offset of the firmware-completed counter. It must be read bytewise. + pub(crate) const COMPLETED: usize = 0x54; + + /// Returns the control object of the free list `args` describes. + pub(crate) const fn new(args: &FreeListArgs) -> Self { + let pages = args.blocks * FREE_LIST_BLOCK_PAGES; + Self { + unk_0: 1, + buffer_slot: args.buffer_slot, + unk_c: 0, + unk_10: args.class as u32, + page_list_va: args.page_list_va, + unk_1c: 0x4_0000, + unk_20: 0, + unk_24: pages, + unk_28: 0, + page_count: pages, + run_list_va: args.run_list_va, + unk_38: [0; 8], + unk_40: 4, + unk_44: 0, + run_cursor: args.blocks * RUN_SLOT_QWORDS, + state_va: args.state_va, + unk_54: [0; 0xc], + unk_60: args.class.unk_60(), + unk_64: 0, + } + } +} + +/// Shared state of a free list, at the start of its page. +/// +/// Compute creation writes a 32-bit one at `+0x00` in a zeroed page. The +/// render boot object starts with a 64-bit two and the boot pool with zero. +/// In a live render pool this becomes the submitted-operation counter: the +/// host uses one aligned 64-bit store before making work visible. It is +/// retained across release/repopulation, never reset by a struct copy. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListState { + /// [`Self::RENDER`], [`Self::BOOT`] or [`Self::COMPUTE`] at boot. + pub(crate) unk_0: u64, +} + +// SAFETY: `FreeListState` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListState {} + +impl FreeListState { + /// Boot state of the render free list. + pub(crate) const RENDER: Self = Self { unk_0: 2 }; + /// Boot state of the boot free list. + pub(crate) const BOOT: Self = Self { unk_0: 0 }; + /// Boot state of a compute free list. + pub(crate) const COMPUTE: Self = Self { unk_0: 1 }; +} + +/// One run-list slot: the run descriptor of one block. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct FreeListRunSlot { + /// Block address, with its page count in the top bits. + pub(crate) run: u64, + pub(crate) unk_8: [u8; 0x38], +} + +// SAFETY: `FreeListRunSlot` consists of integers and arrays of integers only. +unsafe impl Zeroable for FreeListRunSlot {} + +static_assert!(size_of::() == RUN_SLOT_QWORDS as usize * size_of::()); + +impl FreeListRunSlot { + /// Returns the slot describing the block at `block_va`. + pub(crate) const fn new(block_va: u64) -> Self { + Self { + run: (block_va & RUN_ADDRESS_MASK) + | ((FREE_LIST_BLOCK_PAGES as u64) << RUN_PAGES_SHIFT), + unk_8: [0; 0x38], + } + } +} + +/// Writes the run descriptors of `blocks` to the start of `slots`. +pub(crate) fn fill_run_list( + slots: &mut [FreeListRunSlot], + blocks: impl IntoIterator, +) -> Result { + let mut slots = slots.iter_mut(); + for block in blocks { + *slots.next().ok_or(EINVAL)? = FreeListRunSlot::new(block); + } + Ok(()) +} + +/// Writes the address of every page of `blocks` to the start of `list`. +pub(crate) fn fill_page_list(list: &mut [u64], blocks: impl IntoIterator) -> Result { + let mut entries = list.iter_mut(); + for block in blocks { + for page in 0..u64::from(FREE_LIST_BLOCK_PAGES) { + *entries.next().ok_or(EINVAL)? = block + .checked_add(page * FREE_LIST_PAGE_SIZE) + .ok_or(EOVERFLOW)?; + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// Parameter-buffer and page-pool descriptor tables +// --------------------------------------------------------------------------- + +/// One parameter-buffer descriptor, indexed by buffer slot. +#[repr(C)] +pub(crate) struct PbDescriptor { + pub(crate) unk_0: u32, + pub(crate) unk_4: u32, + pub(crate) unk_8: u32, + pub(crate) unk_c: u32, +} + +// SAFETY: `PbDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for PbDescriptor {} + +static_assert!(size_of::() == 0x10); + +/// Parameter-buffer descriptor table, one page. +#[repr(C)] +pub(crate) struct PbDescriptorTable { + /// Descriptors. + pub(crate) entries: [PbDescriptor; PAGE_SIZE / size_of::()], +} + +// SAFETY: `PbDescriptorTable` consists of integers only. +unsafe impl Zeroable for PbDescriptorTable {} + +impl PbDescriptorTable { + /// Required boot contents of slots 0 and 1. + const BOOT: [PbDescriptor; 2] = [ + PbDescriptor { + unk_0: 0x1_9000, + unk_4: 0x20, + unk_8: 0, + unk_c: 0, + }, + PbDescriptor { + unk_0: 0x7_7000, + unk_4: 0x80, + unk_8: 0x17, + unk_c: 0x17, + }, + ]; + + /// Initializes the table. + pub(crate) fn init(&mut self) { + clear(self); + let [slot0, slot1] = Self::BOOT; + self.entries[0] = slot0; + self.entries[1] = slot1; + } +} + +/// Bit position of the unit size in [`PagePoolDescriptor::page_list`]. +const PAGE_POOL_UNIT_SHIFT: u32 = 41; +/// Required unit size of a page-pool descriptor. +const PAGE_POOL_UNIT: u64 = 0x4_0000; +/// Alignment shift of the page-list address in [`PagePoolDescriptor::page_list`]. +const PAGE_POOL_LIST_SHIFT: u32 = 7; +/// Bit position of the free-page count in [`PagePoolDescriptor::counters`]. +const PAGE_POOL_FREE_PAGES_SHIFT: u32 = 33; + +/// One Page-pool descriptor, indexed by buffer slot. +#[repr(C)] +pub(crate) struct PagePoolDescriptor { + /// Unit size and page-list address. + pub(crate) page_list: u64, + /// Pool counters. + pub(crate) counters: u64, + /// Pages in the pool. + pub(crate) page_count: u64, + /// Firmware-owned. + pub(crate) unk_18: u64, +} + +// SAFETY: `PagePoolDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for PagePoolDescriptor {} + +static_assert!(size_of::() == 0x20); + +impl PagePoolDescriptor { + /// Complete row for a physical compute queue's independent 21-block pool. + pub(crate) const fn compute(page_list_va: u64) -> Self { + let pages = 21 * FREE_LIST_BLOCK_PAGES as u64; + Self::new(page_list_va, pages << PAGE_POOL_FREE_PAGES_SHIFT, pages) + } + + const fn new(page_list_va: u64, counters: u64, page_count: u64) -> Self { + Self { + page_list: (PAGE_POOL_UNIT << PAGE_POOL_UNIT_SHIFT) + | (page_list_va >> PAGE_POOL_LIST_SHIFT), + counters, + page_count, + unk_18: 0, + } + } +} + +/// page-pool descriptor table, one page. +#[repr(C)] +pub(crate) struct PagePoolDescriptorTable { + /// Descriptors. + pub(crate) entries: [PagePoolDescriptor; PAGE_SIZE / size_of::()], +} + +// SAFETY: `PagePoolDescriptorTable` consists of integers only. +unsafe impl Zeroable for PagePoolDescriptorTable {} + +impl PagePoolDescriptorTable { + /// Required counters and page count of slot 0 at boot. + const RENDER_COUNTERS: u64 = 0x3d40_0000_3400; + const RENDER_PAGES: u64 = 0x1d00; + + /// Initializes the table: slot 0 describes the render free list's page + /// list, slot 1 the compute page list. + pub(crate) fn init(&mut self, render_page_list_va: u64, compute_page_list_va: u64) { + clear(self); + self.entries[0] = PagePoolDescriptor::new( + render_page_list_va, + Self::RENDER_COUNTERS, + Self::RENDER_PAGES, + ); + self.entries[1] = PagePoolDescriptor::compute(compute_page_list_va); + } +} + +// --------------------------------------------------------------------------- +// Index table +// --------------------------------------------------------------------------- + +/// Groups of the index table: first index and number of groups. +const INDEX_GROUPS: [(u32, usize); 2] = [(0x11, 6), (0x3c, 2)]; +/// Indices per group. +const INDEX_GROUP_SIZE: usize = 4; +/// Distance between the first indices of consecutive groups. +const INDEX_GROUP_STRIDE: u32 = 5; +/// Entries of the index table. +const INDEX_ENTRIES: usize = (INDEX_GROUPS[0].1 + INDEX_GROUPS[1].1) * INDEX_GROUP_SIZE; + +/// Index table at the start of the index object. +#[repr(C)] +pub(crate) struct IndexTable { + /// Groups of consecutive indices. + pub(crate) index: [u32; INDEX_ENTRIES], + pub(crate) unk_80: [u8; PAGE_SIZE - INDEX_ENTRIES * size_of::()], +} + +static_assert!(size_of::() == PAGE_SIZE); + +// SAFETY: `IndexTable` consists of integers only. +unsafe impl Zeroable for IndexTable {} + +impl IndexTable { + /// Initializes the table. + pub(crate) fn init(&mut self) { + clear(self); + let mut entries = self.index.iter_mut(); + for (first, groups) in INDEX_GROUPS { + for group in 0..groups as u32 { + for member in 0..INDEX_GROUP_SIZE as u32 { + if let Some(entry) = entries.next() { + *entry = first + group * INDEX_GROUP_STRIDE + member; + } + } + } + } + } +} + +/// Device-global compute state. Activation writes only the named fields, once, before the +/// first compute queue is exposed. The padding may contain firmware-owned state. +#[repr(C)] +pub(crate) struct ComputeGlobalState { + pub(crate) unk_0: u8, + unk_1: [u8; 7], + pub(crate) unk_8: u8, + unk_9: [u8; 7], + pub(crate) unk_10: u16, + unk_12: [u8; 6], + pub(crate) unk_18: u16, + unk_1a: [u8; 0x3ee], + pub(crate) unk_408: u32, + pub(crate) unk_40c: u32, + unk_410: [u8; 0x1f4], + pub(crate) unk_604: u32, + unk_608: [u8; 0x200], + pub(crate) scheduler_va: u64, + unk_810: [u8; 0x3f4], + pub(crate) unk_c04: u32, + unk_c08: [u8; 0x1f8], + pub(crate) unk_e00: u64, + pub(crate) unk_e08: u64, + unk_e10: [u8; 0x10], + pub(crate) unk_e20: u64, + pub(crate) unk_e28: u64, +} +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_408) == 0x408); +static_assert!(core::mem::offset_of!(ComputeGlobalState, scheduler_va) == 0x808); +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_c04) == 0xc04); +static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_e00) == 0xe00); +static_assert!(size_of::() == 0xe30); + +/// Host-owned recovery report and progress records within the bundle views. +pub(crate) mod recovery { + pub(crate) const REPORT_VIEW: usize = 3; + pub(crate) const REASON: usize = 0x4c; + pub(crate) const PROGRESS_VIEW: usize = 1; + pub(crate) const PROGRESS_START: usize = 0xa40; + pub(crate) const PROGRESS_STRIDE: usize = 0x18; + pub(crate) const VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; +} diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs new file mode 100644 index 00000000000000..c8b7d3f0dce3a7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kick-queue entries and the objects they reference. +//! +//! Every hardware queue (QID) owns a kick ring of [`KICK_SLOTS`] entries spaced +//! [`KICK_ENTRY_STRIDE`] bytes apart. A [`KickTimestamp`] names one entry: its +//! low eight bits select the ring slot. To kick, the host writes the complete +//! slot selected by the queue's current timestamp, issues a barrier, advances +//! the timestamp and then announces the entry with a +//! [`super::queue::KickAnnounce`] record. +//! +//! An entry points at a work descriptor and its queue record, lists the kicks +//! it depends on, binds up to four [`RegisterArray`]s embedded in the +//! descriptor and may name a table of [`McacheRange`]s. +//! +//! Constructors of plain values ([`KickTimestamp::new`], +//! [`KickDependency::new`], [`KickDependency::from_word`]) return `None` for +//! values outside their domain; builders of firmware records and of encoded +//! words return `EINVAL`. + +use kernel::prelude::*; + +use super::queue::QUEUE_PRIORITY; + +/// Number of entries in a kick ring. +pub(crate) const KICK_SLOTS: usize = 256; +/// Distance between consecutive kick ring entries. +pub(crate) const KICK_ENTRY_STRIDE: usize = 0x200; +/// Size of a kick ring. +pub(crate) const KICK_RING_SIZE: usize = KICK_SLOTS * KICK_ENTRY_STRIDE; +/// Dependencies one entry can carry: explicit barriers plus the implicit parent. +pub(crate) const KICK_DEPENDENCIES: usize = 32; +/// Explicit barriers one entry can carry. +pub(crate) const KICK_BARRIERS_MAX: usize = KICK_DEPENDENCIES - 1; +/// Highest hardware queue ID. +pub(crate) const QID_MAX: u8 = 0x7f; +/// Highest UAT context ID. +pub(crate) const CONTEXT_MAX: u8 = 0x3f; + +const KICK_TIMESTAMP_BITS: u32 = 40; +const KICK_TIMESTAMP_MASK: u64 = (1 << KICK_TIMESTAMP_BITS) - 1; +const KICK_SLOT_MASK: u64 = KICK_SLOTS as u64 - 1; +/// GPU addresses packed into kick entries are 32-byte aligned and 43 bits wide. +const PACKED_VA_SHIFT: u32 = 5; +const PACKED_VA_LIMIT: u64 = 1 << (38 + PACKED_VA_SHIFT); + +/// Packs a 32-byte aligned GPU address into the low 38 bits of a word. +fn packed_va(va: u64) -> Result { + if va & ((1 << PACKED_VA_SHIFT) - 1) != 0 || va >= PACKED_VA_LIMIT { + return Err(EINVAL); + } + Ok(va >> PACKED_VA_SHIFT) +} + +/// A kick timestamp. +/// +/// The value is a 40-bit counter that advances by one per kick of a queue; +/// bits 7:0 select the kick ring slot. A queue's first kick uses +/// [`KickTimestamp::FIRST`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickTimestamp(u64); + +impl KickTimestamp { + /// The timestamp preceding a queue's first kick. + pub(crate) const ZERO: Self = Self(0); + /// The timestamp of a queue's first kick. + pub(crate) const FIRST: Self = Self(1); + + /// Returns the timestamp with raw value `value`, if it fits 40 bits. + pub(crate) const fn new(value: u64) -> Option { + if value & !KICK_TIMESTAMP_MASK != 0 { + None + } else { + Some(Self(value)) + } + } + + /// Returns the raw 40-bit value. + pub(crate) const fn get(self) -> u64 { + self.0 + } + + /// Returns the kick ring slot this timestamp selects. + pub(crate) const fn slot(self) -> usize { + (self.0 & KICK_SLOT_MASK) as usize + } + + /// Returns the byte offset of the selected entry within the kick ring. + pub(crate) const fn entry_offset(self) -> usize { + self.slot() * KICK_ENTRY_STRIDE + } + + /// Returns the timestamp of the following kick. + pub(crate) const fn next(self) -> Self { + Self(self.0.wrapping_add(1) & KICK_TIMESTAMP_MASK) + } + + /// Returns the timestamp of the preceding kick. + pub(crate) const fn prev(self) -> Self { + Self(self.0.wrapping_sub(1) & KICK_TIMESTAMP_MASK) + } + + /// Returns the completion value of this kick: the timestamp in bits 39:0 + /// and `pass_seq`, the low byte of the logical queue's render-pass count, + /// in bits 47:40. + pub(crate) const fn completion_value(self, pass_seq: u8) -> u64 { + self.0 | (pass_seq as u64) << KICK_TIMESTAMP_BITS + } +} + +/// A dependency on the completion of one kick. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickDependency { + qid: u8, + timestamp: KickTimestamp, +} + +impl KickDependency { + const QID_SHIFT: u32 = KICK_TIMESTAMP_BITS; + + /// Initial value for unused entries in a host-side dependency array. + pub(crate) const ZERO: Self = Self { + qid: 0, + timestamp: KickTimestamp::ZERO, + }; + + /// Returns the dependency on kick `timestamp` of queue `qid`. + pub(crate) const fn new(qid: u8, timestamp: KickTimestamp) -> Option { + if qid > QID_MAX { + None + } else { + Some(Self { qid, timestamp }) + } + } + + /// Returns the queue that was kicked. + pub(crate) const fn qid(self) -> u8 { + self.qid + } + + /// Returns the dependency word: the timestamp in bits 39:0 and the QID in + /// bits 46:40. + pub(crate) const fn word(self) -> u64 { + self.timestamp.0 | (self.qid as u64) << Self::QID_SHIFT + } + + /// Decodes a dependency word; bits 63:47 must be clear. + pub(crate) const fn from_word(word: u64) -> Option { + if word >> (Self::QID_SHIFT + 7) != 0 { + return None; + } + Some(Self { + qid: (word >> Self::QID_SHIFT) as u8, + timestamp: KickTimestamp(word & KICK_TIMESTAMP_MASK), + }) + } +} + +/// One register write of a [`RegisterArray`]. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterWrite { + /// Register number. + pub(crate) reg: u32, + /// Value written to the register. + pub(crate) value: u64, +} + +impl RegisterWrite { + /// Returns the write of `value` to register `reg`. + pub(crate) const fn new(reg: u32, value: u64) -> Self { + Self { reg, value } + } +} + +/// A buffer that work writes. +#[derive(Copy, Clone, Debug)] +pub(crate) struct WriteRange { + /// GPU address of the buffer. + pub(crate) va: u64, + /// Size of the buffer in bytes. + pub(crate) size: u64, +} + +/// Class of an [`McacheRange`]. +/// +/// The auxiliary framebuffer range takes class 2 and attachment ranges take +/// class 1. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum McacheClass { + /// A render-target attachment. + Attachment = 1, + /// The auxiliary framebuffer. + AuxFramebuffer = 2, +} + +/// One MCache range: a GPU address range that work writes, in 128-byte cache +/// lines. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct McacheRange { + /// First cache line in bits 40:0, UAT context in bits 46:41. + pub(crate) start: u64, + /// End cache line (exclusive) in bits 40:0 and the class above. + pub(crate) end: u64, +} + +impl McacheRange { + const LINE_SIZE: u64 = 0x80; + /// Ranges cover fewer than 2^32 cache lines (512 GiB). + const LINES_MAX: u64 = u32::MAX as u64; + const LINE_LIMIT: u64 = 1 << 41; + const CONTEXT_SHIFT: u32 = 41; + /// Bits 55:47 of the start word hold 0x118 in every range. + const UNK_START: u64 = 0x118 << 47; + /// Bit 41 of the end word is set in every range. + const UNK_END: u64 = 1 << 41; + const CLASS_SHIFT: u32 = 42; + + /// Encodes the cache lines `range` touches, translated in UAT context + /// `context`. + pub(crate) fn new(range: &WriteRange, class: McacheClass, context: u8) -> Result { + if range.size == 0 || context > CONTEXT_MAX { + return Err(EINVAL); + } + let start = range.va / Self::LINE_SIZE; + let end = range + .va + .checked_add(range.size) + .ok_or(EINVAL)? + .div_ceil(Self::LINE_SIZE); + if end >= Self::LINE_LIMIT || end - start > Self::LINES_MAX { + return Err(EINVAL); + } + Ok(Self { + start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, + end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), + }) + } +} + +/// Register-write entries a [`RegisterArray`] holds. +pub(crate) const REGISTER_ARRAY_ENTRIES: usize = 128; +/// MCache ranges that fit in the tail of a [`RegisterArray`]. +pub(crate) const MCACHE_INLINE_RANGES: usize = 16; + +/// A register array embedded in a work descriptor and bound to its kick entry +/// by a [`RegisterArrayBinding`]. +/// +/// The array ends in a trailer that points back at the array and holds its +/// size. It is only written as part of a cleared descriptor: [`Self::set`] +/// leaves the unused entries and the MCache space untouched. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct RegisterArray { + /// Register writes, in order. + pub(crate) entries: [RegisterWrite; REGISTER_ARRAY_ENTRIES], + /// Space after the entries. The first fragment array keeps its MCache + /// table here when the table fits; it is zero everywhere else. + pub(crate) mcache: [McacheRange; MCACHE_INLINE_RANGES], + /// GPU address of this array, as seen by the work's VM. + pub(crate) self_va: u64, + /// Size of the used entries in bytes (bits 31:16) and their count (15:0). + pub(crate) size: u32, + pub(crate) unk_70c: [u8; 0x14], +} + +impl RegisterArray { + /// Stores `writes` and the trailer into a zeroed array. + pub(crate) fn set(&mut self, self_va: u64, writes: &[RegisterWrite]) -> Result { + let count = writes.len(); + if count > REGISTER_ARRAY_ENTRIES { + return Err(EINVAL); + } + self.entries[..count].copy_from_slice(writes); + self.self_va = self_va; + self.size = (size_of_val(writes) as u32) << 16 | count as u32; + Ok(()) + } +} + +/// Binding of one [`RegisterArray`] to a kick entry. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct RegisterArrayBinding { + word: u64, +} + +impl RegisterArrayBinding { + const CHECKPOINT_SHIFT: u32 = 43; + const CHECKPOINT_MAX: u8 = 0x7f; + + /// Binds the array at `array_va`, as seen by the work's VM, whose first + /// checkpoint is after `checkpoint` entries. + pub(crate) fn new(array_va: u64, checkpoint: u8) -> Result { + if checkpoint > Self::CHECKPOINT_MAX { + return Err(EINVAL); + } + Ok(Self { + word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, + }) + } + + fn word(binding: Option) -> u64 { + binding.map_or(0, |binding| binding.word) + } +} + +/// Location of the MCache table a kick entry names. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct McacheTable { + va: u64, + count: u8, +} + +impl McacheTable { + const VA_SHIFT: u32 = 6; + const COUNT_SHIFT: u32 = 54; + const COUNT_MAX: usize = 64; + + /// Names the `count` ranges at `va`, as seen by the work's VM. + pub(crate) fn new(va: u64, count: usize) -> Result { + if count == 0 || count > Self::COUNT_MAX { + return Err(EINVAL); + } + packed_va(va)?; + Ok(Self { + va, + count: count as u8, + }) + } + + fn word(self) -> u64 { + (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT + | ((self.count - 1) as u64) << Self::COUNT_SHIFT + } +} + +/// Quality-of-service identity of the kicked queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct KickQos { + /// QoS buffer slot of the logical queue. + pub(crate) slot: u8, + /// Scheduling class of the engine, at most 0x1f. + pub(crate) class: u8, +} + +/// Inputs of one kick entry. +pub(crate) struct KickArgs<'a> { + /// Kicked queue. + pub(crate) qid: u8, + /// Timestamp of this kick; it selects the ring slot. + pub(crate) timestamp: KickTimestamp, + /// Kick this kick implicitly depends on: the previous kick of the same + /// queue. A queue's first kick names `timestamp.prev()` on render queues + /// and [`KickTimestamp::ZERO`] on compute queues. + pub(crate) parent: KickTimestamp, + /// Kicks of other queues this kick must wait for. + pub(crate) barriers: &'a [KickDependency], + /// GPU address of the work descriptor (firmware alias). + pub(crate) descriptor_va: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// Queue QoS identity. + pub(crate) qos: KickQos, + /// MCache table of the work, if any. + pub(crate) mcache: Option, + /// Engine-specific event mask. + pub(crate) event_mask: [u64; 4], + /// Register arrays bound to the kick. + pub(crate) register_arrays: [Option; 4], +} + +/// One kick ring entry. +/// +/// Live state: none. The host writes the whole entry before announcing it and +/// does not touch it again until the slot is reused. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickEntry { + /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit + /// 58 and queue priority in bits 60:59. + pub(crate) header: u64, + /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. + pub(crate) mcache: u64, + /// GPU address of the work descriptor. + pub(crate) descriptor_va: u64, + /// GPU address of the queue record. + pub(crate) queue_va: u64, + /// Valid-dependency mask in bits 31:0, QoS slot in bits 39:32, class in + /// bits 47:40; bits 63:48 are all ones. + pub(crate) dependency_control: u64, + /// Explicit barriers followed by the implicit parent. + pub(crate) dependencies: [u64; KICK_DEPENDENCIES], + /// Engine-specific event mask. + pub(crate) event_mask: [u64; 4], + pub(crate) unk_148: u64, + /// Register-array bindings: address >> 5 in bits 37:0, checkpoint in bits + /// 49:43. + pub(crate) register_arrays: [u64; 4], + pub(crate) unk_170: u64, + pub(crate) unk_178: u64, + pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], +} + +static_assert!(size_of::() == KICK_ENTRY_STRIDE); +static_assert!(core::mem::offset_of!(KickEntry, dependency_control) == 0x20); +static_assert!(core::mem::offset_of!(KickEntry, event_mask) == 0x128); +static_assert!(core::mem::offset_of!(KickEntry, register_arrays) == 0x150); +static_assert!(core::mem::offset_of!(KickEntry, unk_178) == 0x178); +static_assert!(size_of::() == 0xc); +static_assert!(size_of::() == 0x10); +static_assert!(size_of::() == 0x720); +static_assert!(core::mem::offset_of!(RegisterArray, mcache) == 0x600); +static_assert!(core::mem::offset_of!(RegisterArray, self_va) == 0x700); + +impl KickEntry { + const TIMESTAMP_SHIFT: u32 = 2; + const QID_SHIFT: u32 = 42; + const MCACHE_PRESENT: u64 = 1 << 58; + const PRIORITY_SHIFT: u32 = 59; + const QOS_SLOT_SHIFT: u32 = 32; + const QOS_CLASS_SHIFT: u32 = 40; + const QOS_CLASS_MAX: u8 = 0x1f; + /// Bits 63:48 of the dependency control word. + const UNK_DEPENDENCY_CONTROL: u64 = 0xffff << 48; + /// Required value of the last word of the entry. + const UNK_178: u64 = 0x003f_ffff_ffff_ffff; + + /// Builds the entry described by `args`. + pub(crate) fn new(args: &KickArgs<'_>) -> Result { + let barriers = args.barriers.len(); + if args.qid > QID_MAX + || barriers > KICK_BARRIERS_MAX + || args.qos.class > Self::QOS_CLASS_MAX + { + return Err(EINVAL); + } + + let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT + | (args.qid as u64) << Self::QID_SHIFT + | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + let mcache = match args.mcache { + Some(table) => { + header |= Self::MCACHE_PRESENT; + table.word() + } + None => 0, + }; + + let mut dependencies = [0; KICK_DEPENDENCIES]; + for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { + *slot = barrier.word(); + } + dependencies[barriers] = KickDependency { + qid: args.qid, + timestamp: args.parent, + } + .word(); + let count = barriers + 1; + let mask = if count == KICK_DEPENDENCIES { + u32::MAX + } else { + (1u32 << count) - 1 + }; + + Ok(Self { + header, + mcache, + descriptor_va: args.descriptor_va, + queue_va: args.queue_va, + dependency_control: mask as u64 + | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT + | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT + | Self::UNK_DEPENDENCY_CONTROL, + dependencies, + event_mask: args.event_mask, + unk_148: 0, + register_arrays: args.register_arrays.map(RegisterArrayBinding::word), + unk_170: 0, + unk_178: Self::UNK_178, + pad_180: [0; KICK_ENTRY_STRIDE - 0x180], + }) + } +} + +/// The complete write-only installation of a fragment/tiling queue pair. +/// Firmware must see both queues configured before either outer command. +pub(crate) struct RenderRegistration { + writes: [(usize, u64); 8], +} + +impl RenderRegistration { + const TAG_PORT: usize = 0x4090; + const DATA_PORT: usize = 0x4098; + const VALID_LOW: usize = 0x21068; + const VALID_HIGH: usize = 0x21070; + const CONFIG_TAG: u64 = 0x0000_0400_0002_1008; + const ADDRESS_MASK: u64 = 0x0000_07ff_ffff_ffe0; + + /// Validates both queues without touching MMIO. The resulting transaction + /// writes fragment configuration and validity before tiling configuration + /// and validity. Each validity mask is stored high before low. + pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64)) -> Result { + if fragment.0 == tiling.0 { + return Err(EINVAL); + } + let mut writes = [(0, 0); 8]; + for (slot, ((qid, va), engine)) in [ + (fragment, super::queue::DataMaster::Fragment), + (tiling, super::queue::DataMaster::Tiling), + ] + .into_iter() + .enumerate() + { + if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { + return Err(EINVAL); + } + let completion = (3 * (4 - u64::from(QUEUE_PRIORITY)) + 0x3fe).wrapping_shl(54); + let payload = ((engine as u64 + 1) << 60) | va | completion; + let (low, high) = if qid < 64 { + (1u64 << qid, 0) + } else { + (0, 1u64 << (qid - 64)) + }; + writes[slot * 4..slot * 4 + 4].copy_from_slice(&[ + (Self::TAG_PORT, Self::CONFIG_TAG | (u64::from(qid) << 43)), + (Self::DATA_PORT, payload), + (Self::VALID_HIGH, high), + (Self::VALID_LOW, low), + ]); + } + Ok(Self { writes }) + } + + pub(crate) fn writes(&self) -> &[(usize, u64); 8] { + &self.writes + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs new file mode 100644 index 00000000000000..4f0ae8aa253ed0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 firmware structures. +//! +//! Layouts of the records and descriptors the host shares with the G17 +//! firmware, and the pure builders that fill them. Nothing here allocates, +//! locks or publishes: the owners of the rings and queues do that. +//! +//! This module also holds the few rules shared by the render and compute +//! work descriptors. + +use kernel::prelude::*; + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod compute; +pub(crate) mod context; +pub(crate) mod freelist; +pub(crate) mod initdata; +pub(crate) mod kick; +pub(crate) mod queue; +pub(crate) mod render; +pub(crate) mod qos; + +/// Clears a firmware structure in place. +/// +/// Work descriptors are several KiB large and must not be built on the stack. +pub(crate) fn clear(object: &mut T) { + // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, + // and `object` is a valid, exclusive reference to one `T`. + unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; +} + +/// Returns `va + offset`, or `EINVAL` if the sum overflows. +pub(crate) fn offset_va(va: u64, offset: u64) -> Result { + va.checked_add(offset).ok_or(EINVAL) +} + +/// Size of one slot of a start/end timestamp pair. +const TIMESTAMP_SIZE: u64 = 8; + +/// Returns the address of the end slot of the timestamp pair at `start_va`. +/// +/// A work item's start and end timestamps are two consecutive 64-bit slots. +pub(crate) fn timestamp_end_va(start_va: u64) -> Result { + offset_va(start_va, TIMESTAMP_SIZE) +} + +/// Returns the work key registers take: the UAT context in bits 15:8 and the +/// low six bits of the work-state word below. +pub(crate) const fn work_key(context_id: u16, state_word: u32) -> u64 { + (context_id as u64) << 8 | (state_word & 0x3f) as u64 +} + +/// Returns the sampler bound descriptors store next to the sampler count: the +/// count plus one, or zero without samplers. +pub(crate) fn sampler_max(count: u32) -> Result { + match count { + 0 => Ok(0), + count => count.checked_add(1).ok_or(EINVAL), + } +} + +/// Returns the completion stamp value of sequence number `sequence`: the +/// value the firmware stores to a queue's completion stamp word when the work +/// completes. +pub(crate) const fn stamp_value(sequence: u32) -> u32 { + sequence.wrapping_shl(8) +} + +/// Object IDs of a work item, written to three registers of its engine and +/// to its descriptor. +/// +/// A render pass takes `predecessor` from its queue pair and a newly +/// allocated `current`; compute work uses fixed IDs. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct ObjectIds { + /// Predecessor object ID. + pub(crate) predecessor: u32, + /// Current object ID. + pub(crate) current: u32, +} + +impl ObjectIds { + /// Returns the register value: `predecessor` in bits 63:32 and `current` + /// in bits 31:0. + pub(crate) const fn word(self) -> u64 { + (self.predecessor as u64) << 32 | self.current as u64 + } +} + +/// Required value of the per-engine registers 0x0a599 (compute), 0x0a5a1 +/// (tiling) and 0x0a5a9 (fragment). +pub(crate) const UNK_0A599: u64 = 0x0000_0060_0040_0020; +/// Required value of the per-engine registers 0x0d411 (compute), 0x0d419 +/// (tiling) and 0x0d429 (fragment). +pub(crate) const UNK_0D411: u64 = 0x0000_0002_0000_0001; +/// Required value of the per-engine registers 0x101d9 (compute), 0x101e1 +/// (tiling) and 0x101e9 (fragment). +pub(crate) const UNK_101D9: u64 = 0x1c; +/// Required value of the per-engine registers 0x1a0e9 (compute), 0x1a0f1 +/// (tiling) and 0x1a0f9 (fragment). +pub(crate) const UNK_1A0E9: u64 = 8; +/// Required value of the per-engine registers 0x107a1 (compute) and 0x10799 +/// (tiling). The fragment register 0x10791 also sets bit 9. +pub(crate) const UNK_10791: u64 = 0xff_0000; +/// Required value of the tiling register 0x1c8f8 and the fragment register +/// 0x100b8; the compute register 0x1a458 holds it in its low bits. +pub(crate) const UNK_1C8F8: u64 = 0x8860; diff --git a/drivers/gpu/drm/asahi/g17/fw/qos.rs b/drivers/gpu/drm/asahi/g17/fw/qos.rs new file mode 100644 index 00000000000000..d939e012714ee1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/qos.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! QoS accounting object. Queue rows and submission counts use distinct queue +//! and logical-owner indices. Firmware may observe each live word separately. + +use kernel::prelude::*; + +pub(crate) const QUEUES: usize = 128; +pub(crate) const DATA_MASTERS: usize = 4; + +#[repr(C)] +pub(crate) struct Qos { + pub(crate) rows: [[u8; 8]; QUEUES], + pub(crate) queue_submitted: [u32; QUEUES], + pub(crate) owner_submitted: [u32; QUEUES], + pub(crate) scheduler: [u64; QUEUES], + pub(crate) share: [u32; QUEUES], + pub(crate) intervals: u64, + pub(crate) busy_start: [u64; 3], + pub(crate) dm_submitted: [u64; DATA_MASTERS], +} +// SAFETY: Qos consists entirely of integer fields and arrays. +unsafe impl Zeroable for Qos {} + +const _: () = { + assert!(core::mem::size_of::() == 0xe40); + assert!(core::mem::offset_of!(Qos, queue_submitted) == 0x400); + assert!(core::mem::offset_of!(Qos, owner_submitted) == 0x600); + assert!(core::mem::offset_of!(Qos, scheduler) == 0x800); + assert!(core::mem::offset_of!(Qos, share) == 0xc00); + assert!(core::mem::offset_of!(Qos, intervals) == 0xe00); + assert!(core::mem::offset_of!(Qos, busy_start) == 0xe08); + assert!(core::mem::offset_of!(Qos, dm_submitted) == 0xe20); +}; diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs new file mode 100644 index 00000000000000..db95ba99e0189a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -0,0 +1,540 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Queue records and the commands of a queue's item ring. +//! +//! A hardware queue is described by a [`QueueRecord`]. Its [`PointerBlock`] +//! holds the cursors of the queue's item ring, an array of [`ITEM_RING_ENTRIES`] +//! work pointers: 64-bit GPU addresses of commands. Every command starts with +//! a 32-bit [`CommandTag`]: work descriptors carry the tiling, fragment and +//! compute tags (see `render.rs` and `compute.rs`), and the per-kick records +//! defined here carry the others. +//! +//! One kick appends, in order, the work descriptor, a [`QueueConfig`], a +//! [`KickAnnounce`] record and, when a compute queue is first installed or +//! explicitly waits on another queue, a [`KickPredecessor`]. +//! Compute kicks always carry the [`QueueConfig`]. A render kick carries it +//! only when it installs the queue, updates the context or QoS identity, +//! carries a completion seed, or configures the queue differently from the +//! last [`QueueConfig`] published on it. + +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; + +use super::kick::{KickTimestamp, QID_MAX}; + +/// Priority of every queue the driver creates. +/// +/// The same value appears in the queue record, [`QueueConfig`], +/// [`KickAnnounce`] and the kick entry header; the work rings it selects are +/// listed in `channels.rs`. +pub(crate) const QUEUE_PRIORITY: u8 = 2; +/// Scheduling policy of every queue the driver creates. +pub(crate) const QUEUE_POLICY: u16 = 2; +/// Number of work pointers in a queue's item ring. +pub(crate) const ITEM_RING_ENTRIES: u32 = 0x500; + +/// Tag in the first word of every command. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum CommandTag { + /// Tiling (TA) work descriptor. + Tiling = 0, + /// Fragment (3D) work descriptor. + Fragment = 1, + /// Compute work descriptor. + Compute = 3, + /// [`KickAnnounce`]. + KickAnnounce = 0x0e, + /// [`QueueConfig`]. + QueueConfig = 0x0f, + /// [`KickPredecessor`]. + KickPredecessor = 0x10, +} + +/// The engine that executes a queue's work. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u32)] +pub(crate) enum DataMaster { + /// Tiling (TA). + Tiling = 0, + /// Fragment (3D). + Fragment = 1, + /// Compute (CL). + Compute = 2, +} + +/// Cursors of a queue's item ring. +/// +/// The firmware writes `done` and `unk_30`. The host writes the block on +/// three paths, never with a structure copy once the queue was published: +/// +/// - creation: the whole block, [`PointerBlock::new`], before the queue is +/// first published; +/// - publication: `producer`, with one aligned 32-bit store per publication +/// step, after a barrier that orders the work pointers and commands before +/// it; +/// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct PointerBlock { + /// Item-ring position up to which the firmware has finished commands. + /// Free ring space is measured from it, so it bounds `producer`. + pub(crate) done: AtomicU32, + pub(crate) unk_04: [u32; 11], + /// Written by the firmware. The host treats the queue as idle only when + /// this cursor, `done` and `producer` are equal. + pub(crate) unk_30: AtomicU32, + pub(crate) unk_34: [u32; 3], + /// Item-ring position after the last work pointer the host published. + pub(crate) producer: AtomicU32, + pub(crate) unk_44: [u32; 3], + /// On recovery, the host copies producer here before acknowledging resume. + pub(crate) recovery_target: u32, + pub(crate) unk_54: [u32; 3], + /// Number of entries in the item ring. + pub(crate) entries: u32, + pub(crate) unk_64: [u32; 7], +} + +static_assert!(size_of::() == 0x80); +static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); +static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); +static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); + +// SAFETY: PointerBlock contains only integers and integer atomics. +unsafe impl Zeroable for PointerBlock {} + +impl PointerBlock { + pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); + pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); + + /// Returns the cursors of an empty item ring. + pub(crate) const fn new() -> Self { + Self { + done: AtomicU32::new(0), + unk_04: [0; 11], + unk_30: AtomicU32::new(0), + unk_34: [0; 3], + producer: AtomicU32::new(0), + unk_44: [0; 3], + recovery_target: u32::MAX, + unk_54: [0; 3], + entries: ITEM_RING_ENTRIES, + unk_64: [0; 7], + } + } + + /// Moves the cursors of an empty queue to item-ring position `base`. + /// + /// Used when a render queue pair that was kicked before is handed to a new + /// owner, with `base` the position the host tracks for the new owner's + /// queues. Only valid while the firmware does not run the queue, with all + /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this + /// order, with single stores; the caller issues a full barrier afterwards. + pub(crate) fn rebase(&self, base: u32) { + self.done.store(base, Ordering::Relaxed); + self.unk_30.store(base, Ordering::Relaxed); + self.producer.store(base, Ordering::Relaxed); + } +} + +/// Head of a queue's job list. +/// +/// Live state: the firmware links and unlinks jobs; the host writes the head +/// only at queue creation. +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct JobListHead { + /// GPU address of the first job, or zero. + pub(crate) first_va: u64, + /// GPU address of the last link: the head itself when the list is empty. + pub(crate) last_va: u64, +} + +impl JobListHead { + /// Returns an empty list whose head is at `self_va`. + pub(crate) const fn new(self_va: u64) -> Self { + Self { + first_va: 0, + last_va: self_va, + } + } +} + +/// Addresses and owner a [`QueueRecord`] names. +pub(crate) struct QueueRecordArgs { + /// GPU address of the queue's [`PointerBlock`]. + pub(crate) pointers_va: u64, + /// GPU address of the queue's item ring. + pub(crate) items_va: u64, + /// GPU address of the job list the queue's work is linked into. + pub(crate) job_list_va: u64, + /// GPU address of the owning logical queue's scheduler-state page. + pub(crate) scheduler_va: u64, + /// Thread-group ID of the owning process (0 while unowned). + pub(crate) owner_pid: u32, +} + +/// Description of one hardware queue, named by its work-ring slots. +/// +/// The host writes it whole at queue creation and changes the owner fields of +/// an idle queue with [`QueueRecord::set_owner`] or +/// [`QueueRecord::set_compute_owner`]. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct QueueRecord { + /// GPU address of the [`PointerBlock`]. + pub(crate) pointers_va: u64, + /// GPU address of the item ring. + pub(crate) items_va: u64, + /// GPU address of the job list. + pub(crate) job_list_va: u64, + pub(crate) unk_18: [u8; 0xc], + pub(crate) unk_24: u32, + /// Queue priority, stored twice. + pub(crate) priority: [u32; 2], + pub(crate) unk_30: [u8; 6], + pub(crate) unk_36: u16, + /// Zero at creation; cleared again when a compute queue changes owner. + pub(crate) unk_38: u32, + pub(crate) unk_3c: u32, + /// Scheduling policy. + pub(crate) policy: u32, + pub(crate) unk_44: u32, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + pub(crate) unk_4c: [u8; 0x50], + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + pub(crate) unk_a4: [u8; 0x1c], +} + +static_assert!(size_of::() == 0xc0); +static_assert!(core::mem::offset_of!(QueueRecord, priority) == 0x28); +static_assert!(core::mem::offset_of!(QueueRecord, unk_38) == 0x38); +static_assert!(core::mem::offset_of!(QueueRecord, policy) == 0x40); +static_assert!(core::mem::offset_of!(QueueRecord, scheduler_va) == 0x9c); + +impl QueueRecord { + /// Builds the record of a newly created queue. + pub(crate) fn new(args: &QueueRecordArgs) -> Self { + Self { + pointers_va: args.pointers_va, + items_va: args.items_va, + job_list_va: args.job_list_va, + unk_18: [0; 0xc], + unk_24: u32::MAX, + priority: [QUEUE_PRIORITY as u32; 2], + unk_30: [0; 6], + unk_36: u16::MAX, + unk_38: 0, + unk_3c: 0, + policy: QUEUE_POLICY as u32, + unk_44: u32::MAX, + owner_pid: args.owner_pid, + unk_4c: [0; 0x50], + scheduler_va: args.scheduler_va, + unk_a4: [0; 0x1c], + } + } + + /// Hands an idle compute queue to a new owner. + /// + /// Replaces the job head, process identity and scheduler page, and clears `unk_38`. + pub(crate) fn set_compute_owner( + &mut self, + job_list_va: u64, + scheduler_va: u64, + owner_pid: u32, + ) { + self.job_list_va = job_list_va; + self.unk_38 = 0; + self.owner_pid = owner_pid; + self.scheduler_va = scheduler_va; + } +} + +/// Free list a queue's USC private memory is allocated from. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct FreeListBinding { + /// GPU address of the free list's shared control object. + pub(crate) va: u64, + /// Generation of the free list's registration. + pub(crate) generation: u64, + /// Buffer slot of the free list. + pub(crate) slot: u32, +} + +/// Parameter-buffer binding of a tiling queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct TilingBinding { + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// Allocation token of the parameter buffer's buffer slot. + pub(crate) pb_token: u64, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// QID of the fragment queue paired with this tiling queue. + pub(crate) fragment_qid: u8, +} + +/// Queue whose configuration a [`QueueConfig`] carries. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum QueueConfigTarget { + /// A tiling queue and its parameter-buffer binding. + Tiling(TilingBinding), + /// A fragment queue. + Fragment, + /// A compute queue. + Compute, +} + +/// Inputs of one [`QueueConfig`]. +pub(crate) struct QueueConfigArgs { + /// Kind of queue. + pub(crate) target: QueueConfigTarget, + /// GPU address of the queue's kick ring. + pub(crate) kick_ring_va: u64, + /// Firmware alias of the queue's kick ring. + pub(crate) kick_ring_fw_va: u64, + /// Configured queue. + pub(crate) qid: u8, + /// The firmware has not yet installed this QID. + pub(crate) install: bool, + /// Completion value the firmware loads for the queue, or `None` to keep + /// the value it holds. + pub(crate) completion_seed: Option, + /// UAT context the queue's work runs in. + pub(crate) context_id: u16, + /// USC private-memory free list. + pub(crate) free_list: FreeListBinding, + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + /// QoS buffer slot of the logical queue. + pub(crate) qos_slot: u8, + /// The QoS identity or queue membership changed since the last update. + pub(crate) qos_update: bool, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + /// The queue's UAT context changed since the last update. + pub(crate) context_update: bool, +} + +/// Configuration of one hardware queue (tag 15). +/// +/// The firmware applies it when it consumes the command, which may be after +/// later kicks were published, so a record must not be rewritten while a +/// work pointer names it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct QueueConfig { + /// [`CommandTag::QueueConfig`]. + pub(crate) tag: u32, + pub(crate) unk_04: u32, + /// GPU address of the queue's kick ring. + pub(crate) kick_ring_va: u64, + /// Firmware alias of the queue's kick ring. + pub(crate) kick_ring_fw_va: u64, + /// Configured queue. + pub(crate) qid: u16, + /// Non-zero to install the QID. + pub(crate) install: u16, + pub(crate) unk_1c: u16, + /// Queue priority. + pub(crate) priority: u16, + pub(crate) unk_20: u16, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u16, + pub(crate) unk_24: u16, + /// Non-zero when `completion_seed` is valid. + pub(crate) completion_seed_valid: u32, + /// Completion value: kick timestamp in bits 39:0, render-pass sequence in + /// bits 47:40 (see [`KickTimestamp::completion_value`]). + pub(crate) completion_seed: u64, + /// UAT context of the queue's work. + pub(crate) context_id: u16, + pub(crate) unk_34: u16, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_va: u64, + /// Generation of the USC free list. + pub(crate) free_list_generation: u64, + /// Buffer slot of the USC free list. + pub(crate) free_list_slot: u32, + /// GPU address of the owner's scheduler-state page. + pub(crate) scheduler_va: u64, + /// Non-zero when the QoS identity or membership changed. + pub(crate) qos_update: u32, + /// QoS buffer slot. + pub(crate) qos_slot: u16, + pub(crate) unk_58: u16, + /// Thread-group ID of the owning process. + pub(crate) owner_pid: u32, + /// Scheduling policy. + pub(crate) policy: u16, + pub(crate) unk_60: u16, + /// Non-zero when the UAT context changed. + pub(crate) context_update: u16, + pub(crate) unk_64: u16, + pub(crate) unk_66: u16, + pub(crate) unk_68: [u8; 6], + /// Tiling queues: GPU address of the parameter-buffer state; else zero. + pub(crate) pb_va: u64, + /// Tiling queues: parameter-buffer slot allocation token; else all ones. + pub(crate) pb_token: u64, + /// Tiling queues: parameter-buffer slot; else all ones. + pub(crate) pb_slot: u32, + /// Tiling queues: QID of the paired fragment queue; else all ones. + pub(crate) fragment_qid: u32, + pub(crate) unk_86: [u8; 0x3a], +} + +static_assert!(size_of::() == 0xc0); +static_assert!(core::mem::offset_of!(QueueConfig, completion_seed_valid) == 0x26); +static_assert!(core::mem::offset_of!(QueueConfig, free_list_va) == 0x36); +static_assert!(core::mem::offset_of!(QueueConfig, scheduler_va) == 0x4a); +static_assert!(core::mem::offset_of!(QueueConfig, qos_slot) == 0x56); +static_assert!(core::mem::offset_of!(QueueConfig, owner_pid) == 0x5a); +static_assert!(core::mem::offset_of!(QueueConfig, pb_va) == 0x6e); +static_assert!(core::mem::offset_of!(QueueConfig, fragment_qid) == 0x82); + +impl QueueConfig { + const UNK_66: u16 = 1; + + /// Builds the record described by `args`. + pub(crate) fn new(args: &QueueConfigArgs) -> Result { + if args.qid > QID_MAX { + return Err(EINVAL); + } + let (data_master, pb_va, pb_token, pb_slot, fragment_qid) = match args.target { + QueueConfigTarget::Tiling(binding) => { + if binding.fragment_qid > QID_MAX { + return Err(EINVAL); + } + ( + DataMaster::Tiling, + binding.pb_va, + binding.pb_token, + binding.pb_slot, + binding.fragment_qid as u32, + ) + } + QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), + QueueConfigTarget::Compute => (DataMaster::Compute, 0, u64::MAX, u32::MAX, u32::MAX), + }; + Ok(Self { + tag: CommandTag::QueueConfig as u32, + unk_04: 0, + kick_ring_va: args.kick_ring_va, + kick_ring_fw_va: args.kick_ring_fw_va, + qid: args.qid as u16, + install: args.install as u16, + unk_1c: 0, + priority: QUEUE_PRIORITY as u16, + unk_20: 0, + data_master: data_master as u16, + unk_24: 0, + completion_seed_valid: args.completion_seed.is_some() as u32, + completion_seed: args.completion_seed.unwrap_or(0), + context_id: args.context_id, + unk_34: 0, + free_list_va: args.free_list.va, + free_list_generation: args.free_list.generation, + free_list_slot: args.free_list.slot, + scheduler_va: args.scheduler_va, + qos_update: args.qos_update as u32, + qos_slot: args.qos_slot as u16, + unk_58: 0, + owner_pid: args.owner_pid, + policy: QUEUE_POLICY, + unk_60: 0, + context_update: args.context_update as u16, + unk_64: 0, + unk_66: Self::UNK_66, + unk_68: [0; 6], + pb_va, + pb_token, + pb_slot, + fragment_qid, + unk_86: [0; 0x3a], + }) + } +} + +/// Announcement of a newly written kick entry of a queue (tag 14). +/// +/// A record slot larger than this structure must be zero beyond it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickAnnounce { + /// [`CommandTag::KickAnnounce`]. + pub(crate) tag: u32, + /// Kicked queue. + pub(crate) qid: u16, + /// Number of entries announced. + pub(crate) count: u16, + /// Queue priority. + pub(crate) priority: u8, + /// Timestamp of the announced entry. + pub(crate) timestamp: u64, + /// [`DataMaster`] of the queue. + pub(crate) data_master: u32, + pub(crate) unk_15: [u8; 3], +} + +static_assert!(size_of::() == 0x18); +static_assert!(core::mem::offset_of!(KickAnnounce, timestamp) == 0x09); +static_assert!(core::mem::offset_of!(KickAnnounce, data_master) == 0x11); + +impl KickAnnounce { + /// Announces the entry of queue `qid` written at `timestamp`. + pub(crate) fn new(qid: u8, data_master: DataMaster, timestamp: KickTimestamp) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + tag: CommandTag::KickAnnounce as u32, + qid: qid as u16, + count: 1, + priority: QUEUE_PRIORITY, + timestamp: timestamp.get(), + data_master: data_master as u32, + unk_15: [0; 3], + }) + } +} + +/// Names the kick of a compute queue that precedes the announced one +/// (tag 16). +/// +/// A record slot larger than this structure must be zero beyond it. +#[repr(C, packed)] +#[derive(Copy, Clone, Debug)] +pub(crate) struct KickPredecessor { + /// [`CommandTag::KickPredecessor`]. + pub(crate) tag: u32, + /// Kicked queue. + pub(crate) qid: u16, + /// Timestamp of the queue's kick preceding the announced one. + pub(crate) previous: u64, + pub(crate) unk_0e: u32, +} + +static_assert!(size_of::() == 0x12); + +impl KickPredecessor { + const UNK_0E: u32 = 0xfe; + + /// Names `previous` as the kick of queue `qid` preceding the announced + /// one. + pub(crate) fn new(qid: u8, previous: KickTimestamp) -> Result { + if qid > QID_MAX { + return Err(EINVAL); + } + Ok(Self { + tag: CommandTag::KickPredecessor as u32, + qid: qid as u16, + previous: previous.get(), + unk_0e: Self::UNK_0E, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs new file mode 100644 index 00000000000000..349d03ed14d719 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -0,0 +1,1309 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render work descriptors. +//! +//! A render pass is kicked as a tiling (TA) descriptor on one queue and a +//! fragment (3D) descriptor on the queue paired with it. Each descriptor +//! embeds the register arrays its kick entry binds: one for tiling, and for +//! fragment the main array followed by the partial-render store, resume and +//! load arrays. +//! +//! Descriptors are built in place with [`TaDescriptor::write`] and +//! [`FragmentSlot::write`]; the completion link and the fragment MCache table +//! are added just before publication. + +pub(crate) mod storage; + +use kernel::prelude::*; + +use super::kick::{ + KickTimestamp, McacheClass, McacheRange, McacheTable, RegisterArray, RegisterArrayBinding, + RegisterWrite, WriteRange, MCACHE_INLINE_RANGES, +}; +use super::queue::CommandTag; +use super::{ + clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, + UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, +}; + +/// Size of a tiling descriptor. +pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; +/// Size of a [`FragmentSlot`]: the descriptor and its MCache spill table. +pub(crate) const FRAGMENT_SLOT_SIZE: usize = 0x2380; +/// Write ranges of one fragment pass: its attachments and the auxiliary +/// framebuffer. +pub(crate) const MCACHE_RANGES_MAX: usize = 17; +/// Size of the auxiliary framebuffer of a pass. +pub(crate) const AUX_FB_SIZE: u64 = 0x8000; + +/// QoS class of tiling kicks. +pub(crate) const TA_QOS_CLASS: u8 = 0x0c; +/// QoS class of fragment kicks. +pub(crate) const FRAGMENT_QOS_CLASS: u8 = 0x18; +/// Event mask of render kicks. +pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; + +/// Base of the user address window. Tiling resource registers hold offsets +/// from it rather than addresses. +const USER_BASE: u64 = 0x10_0000_0000; +/// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. +const VDM_OFFSET_LIMIT: u64 = 1 << 39; +/// The deflake buffer holds three sub-buffers: at its start and at these +/// offsets. +const DEFLAKE_20: u64 = 0x20; +const DEFLAKE_2A0: u64 = 0x2a0; + +const TA_REGISTERS: usize = 73; +const FRAGMENT_REGISTERS: usize = 89; +const PARTIAL_STORE_REGISTERS: usize = 16; +const PARTIAL_RESUME_REGISTERS: usize = 23; +const PARTIAL_LOAD_REGISTERS: usize = 10; + +/// Checkpoints into the tiling array, as entry counts: before the final two +/// entries, then after the last one (twice). +const TA_CHECKPOINTS: [u16; 3] = [ + TA_REGISTERS as u16 - 2, + TA_REGISTERS as u16, + TA_REGISTERS as u16, +]; +/// Checkpoints into the main fragment array, as entry counts: after the first +/// of the four closing entries, then after the second (twice). +const FRAGMENT_CHECKPOINTS: [u16; 3] = [ + FRAGMENT_REGISTERS as u16 - 3, + FRAGMENT_REGISTERS as u16 - 2, + FRAGMENT_REGISTERS as u16 - 2, +]; + +const TILE_SIZE: u64 = 32; +const MACRO_TILES: u64 = 4; +const REGION_ENTRY_SIZE: u64 = 5; +/// Highest number of samples per pixel, as a power of two. +const SAMPLES_LOG2_MAX: u8 = 2; +/// Bytes of tile buffer per block. +const TIB_BLOCK_SIZE: u64 = 2048; +const TILE_CONFIG_BASE: u64 = 0x280; +const TILE_CONFIG_LAYERED: u64 = 1 << 0; +const TILE_CONFIG_PROCESS_EMPTY_TILES: u64 = 1 << 16; +const AUX_FB_FLAGS_BASE: u64 = 0xc000; +const AUX_FB_FLAGS_DBIAS_INT: u64 = 1 << 18; +/// Tiling layer mode: layer count less one, bit 15, and for layered targets +/// bit 14 plus the empty-tile policy. The second copy keeps only the count and +/// bit 14. +const LAYER_MODE_BASE: u64 = 1 << 15; +const LAYER_MODE_LAYERED: u64 = 1 << 14; +const LAYER_MODE_PROCESS_EMPTY: u64 = 1 << 13; +const LAYER_MODE_SKIP_EMPTY: u64 = 1 << 12; +const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; +/// Fragment tile state: required bits 0x3717f and 37, layered targets in bit +/// 32, the utile size bits of the tile-buffer configuration in bits 43:40, +/// tiles in X less one from bit 44 and tiles in Y less one from bit 53. +const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; +const TILE_STATE_LAYERED: u64 = 1 << 32; +const TILE_STATE_UTILE_SHIFT: u32 = 28; +const TILE_STATE_TILES_X_SHIFT: u32 = 44; +const TILE_STATE_TILES_Y_SHIFT: u32 = 53; +/// Tile-buffer configuration: utile width / 16 in bits 13:12, utile height / +/// 16 in bits 15:14, samples (log2) in bits 1:0. +const UTILE_CONFIG_WIDTH_SHIFT: u32 = 12; +const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; +const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; +/// The partial-render phases load and store every attached depth and stencil +/// surface. +const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; +const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; +const STENCIL_CLEAR_BASE: u32 = 0x300; +/// Value of the ZLS swizzle registers and descriptor field. +const ZLS_SWIZZLE: u32 = 0x0404_0404; +/// Value of register 0x15359 in the partial store and resume arrays; the +/// main array writes zero. +const UNK_15359_PARTIAL: u64 = 0x20; +/// Required value of register 0x15049 and of fragment descriptor field +/// +0x1f50. +const UNK_15049: u64 = 0x10_0000; +/// Value of the fragment register 0x10791. +const UNK_10791_FRAGMENT: u64 = UNK_10791 | 1 << 9; +/// USC program offsets carry tag bits that are not part of the address. +const USC_OFFSET_TAG_MASK: u32 = 7; + +/// A depth or stencil surface. +#[derive(Copy, Clone, Debug)] +pub(crate) struct ZlsSurface { + /// GPU address of the surface, or zero when it is not attached. + pub(crate) va: u64, + /// GPU address of its compression metadata. + pub(crate) compression_va: u64, + /// Stride of the surface. + pub(crate) stride: u64, + /// Stride of the compression metadata. + pub(crate) compression_stride: u64, +} + +/// A USC program and the resource specifier it is launched with. +#[derive(Copy, Clone, Debug)] +pub(crate) struct UscProgram { + /// Tagged offset of the program in the queue's USC window. + pub(crate) offset: u32, + /// Resource specifier. + pub(crate) rsrc_spec: u64, +} + +/// Render-pass state from the userspace command. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderPass { + /// Width of the render target in pixels. + pub(crate) width: u32, + /// Height of the render target in pixels. + pub(crate) height: u32, + /// Number of layers, 1 to 2048. + pub(crate) layers: u16, + /// Utile width: 16 or 32. + pub(crate) utile_width: u8, + /// Utile height: 16 or 32, at most the width. + pub(crate) utile_height: u8, + /// Samples per pixel, as a power of two: 0 to 2. + pub(crate) samples_log2: u8, + /// Bytes of tile buffer per sample. + pub(crate) sample_size: u8, + /// Process tiles no primitive touches. + pub(crate) process_empty_tiles: bool, + /// Multisample control. + pub(crate) msaa_control: u64, + /// PPP control. + pub(crate) ppp_control: u64, + /// GPU address of the VDM control stream. + pub(crate) vdm_va: u64, + /// GPU address of the scissor array. + pub(crate) scissor_va: u64, + /// GPU address of the depth-bias array, or zero. + pub(crate) depth_bias_va: u64, + /// Depth-bias values are integers. + pub(crate) depth_bias_is_int: bool, + /// GPU address of the occlusion-query results, or zero. + pub(crate) occlusion_query_va: u64, + /// GPU address of the sampler heap, or zero. + pub(crate) sampler_heap_va: u64, + /// Number of samplers in the heap. + pub(crate) sampler_count: u32, + /// ZLS control at the pass boundaries. + pub(crate) zls_control: u64, + /// Depth surface dimensions. + pub(crate) depth_dimensions: u64, + /// Depth surface. + pub(crate) depth: ZlsSurface, + /// Stencil surface. + pub(crate) stencil: ZlsSurface, + /// Depth clear value (IEEE-754 bits). + pub(crate) depth_clear: u32, + /// Stencil clear value (background object register value). + pub(crate) stencil_clear: u32, + /// Merge upper bound in X (IEEE-754 bits). + pub(crate) merge_upper_x: u32, + /// Merge upper bound in Y (IEEE-754 bits). + pub(crate) merge_upper_y: u32, + /// Base of the queue's USC window. + pub(crate) usc_base: u64, + /// Background program. + pub(crate) bg: UscProgram, + /// Partial-render background program. + pub(crate) partial_bg: UscProgram, + /// Tagged USC offset of the end-of-tile program. Its resource specifier + /// is always written as zero. + pub(crate) eot_offset: u32, + /// Tagged USC offset of the partial-render end-of-tile program. + pub(crate) partial_eot_offset: u32, + /// The resource specifiers have their high 32 bits. + pub(crate) rsrc_spec_hi: bool, +} + +/// Kernel-owned objects and identities a render pass uses. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderResources { + /// UAT context the pass runs in. + pub(crate) context_id: u16, + /// Allocation generation of that context. + pub(crate) context_generation: u8, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + /// Buffer slot of the USC private-memory free list. + pub(crate) free_list_slot: u32, + /// GPU address of the free list's shared control object. + pub(crate) free_list_control_va: u64, + /// GPU address of the pass's kick record, in which the firmware counts + /// the pass's completed kicks. + pub(crate) kick_record_va: u64, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry of the pass. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// Parameter-buffer scratch offset register value. + pub(crate) pm_scratch: u64, + /// Parameter-buffer metric pointer register value. + pub(crate) pm_metric: u64, + /// GPU address of the tile map. + pub(crate) tilemap_va: u64, + /// GPU address of the layer metadata. + pub(crate) layermeta_va: u64, + /// GPU address of the heap metadata. + pub(crate) heapmeta_va: u64, + /// GPU address of the tail-pointer cache. + pub(crate) tpc_va: u64, + /// Size of the tail-pointer cache the target needs. + pub(crate) tpc_size: u64, + /// GPU address of the deflake buffer. + pub(crate) deflake_va: u64, + /// GPU address of the tiling status word. + pub(crate) ta_status_va: u64, + /// GPU address of the fragment status word, as seen by the pass's VM. + pub(crate) fragment_status_va: u64, + /// Firmware alias of the fragment status word. + pub(crate) fragment_status_fw_va: u64, + /// GPU address of the auxiliary framebuffer. + pub(crate) aux_fb_va: u64, + /// GPU address of the pass's start/end timestamp pair. + pub(crate) timestamp_va: u64, + /// Object IDs of the tiling stage. + pub(crate) ta_ids: ObjectIds, + /// Object IDs of the fragment stage. + pub(crate) fragment_ids: ObjectIds, + /// Value of fragment descriptor field +0x2104: bit 0 of GPU register + /// 0xe0141c, read at probe. + pub(crate) unk_2104: u32, + /// Value of fragment descriptor field +0x210c: bits 3:0 of GPU register + /// 0xe01480, read at probe. + pub(crate) unk_210c: u32, +} + +/// Per-stage inputs of a render pass. +#[derive(Copy, Clone, Debug)] +pub(crate) struct RenderStage { + /// Queue the stage is kicked on. + pub(crate) qid: u8, + /// GPU address of the stage's descriptor, as seen by the pass's VM. + pub(crate) descriptor_va: u64, + /// Command-buffer state word of the stage. + pub(crate) state_word: u32, + /// GPU address of the queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the queue's word in the auxiliary stamp page. + pub(crate) aux_stamp_va: u64, +} + +/// Inputs of both render descriptors. +pub(crate) struct RenderArgs<'a> { + /// Render-pass state. + pub(crate) pass: &'a RenderPass, + /// Kernel-owned objects. + pub(crate) resources: &'a RenderResources, + /// Submission ordinal of the render queue pair, from zero. + pub(crate) ordinal: u64, + /// Tiling stage. + pub(crate) ta: RenderStage, + /// Fragment stage. + pub(crate) fragment: RenderStage, +} + +impl RenderArgs<'_> { + /// Completion stamp value of this pass: the pass's sequence number is the + /// pair's submission ordinal plus one. + fn stamp_value(&self) -> u32 { + stamp_value((self.ordinal as u32).wrapping_add(1)) + } +} + +/// Tile-grid geometry of a render target, in the units its registers take. +struct Geometry { + tiles_x: u64, + tiles_y: u64, + /// Size of the region array. + region_size: u64, + /// Number of macro tiles. + macro_tiles: u64, + /// Two utile slots per utile of every macro tile. + utile_slots: u64, + /// Macro-tile columns times 3, 2 and 1, in 9-bit fields from bit 0. + x_blocks: u64, + /// Macro-tile rows times 3, 2 and 1, in 9-bit fields from bit 0. + y_blocks: u64, + /// Tiles in Y less one from bit 12, tiles in X less one below. + screen: u64, + /// Height less one from bit 16, width less one below. + pixels: u64, + /// Macro-tile width in utiles from bit 16, height in utiles below. + macro_size: u64, +} + +impl Geometry { + fn new(pass: &RenderPass) -> Result { + if pass.width == 0 || pass.height == 0 || pass.layers == 0 || pass.layers > 2048 { + return Err(EINVAL); + } + if !matches!( + (pass.utile_width, pass.utile_height), + (32, 32) | (32, 16) | (16, 16) + ) { + return Err(EINVAL); + } + let width = pass.width as u64; + let height = pass.height as u64; + let tiles_x = width.div_ceil(TILE_SIZE); + let tiles_y = height.div_ceil(TILE_SIZE); + let utiles_x = TILE_SIZE / pass.utile_width as u64; + let utiles_y = TILE_SIZE / pass.utile_height as u64; + let utiles = utiles_x * utiles_y; + let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); + let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); + let macro_tiles = mtile_x * mtile_y; + Ok(Self { + tiles_x, + tiles_y, + region_size: (REGION_ENTRY_SIZE * macro_tiles * utiles).div_ceil(4), + macro_tiles, + utile_slots: 2 * macro_tiles * utiles, + x_blocks: (3 * mtile_x) | ((2 * mtile_x) << 9) | (mtile_x << 18), + y_blocks: (3 * mtile_y) | ((2 * mtile_y) << 9) | (mtile_y << 18), + screen: (tiles_y - 1) << 12 | (tiles_x - 1), + pixels: (width - 1) | (height - 1) << 16, + macro_size: (mtile_y * utiles_y) | ((mtile_x * utiles_x) << 16), + }) + } +} + +/// Register and descriptor words derived from the [`RenderPass`]. +struct PassWords { + utile_config: u64, + tile_config: u64, + layer_mode: u64, + tib_blocks: u64, + partial_zls_control: u64, + aux_fb_flags: u64, + bg_rsrc_spec: u64, + bg_va: u64, + partial_bg_rsrc_spec: u64, + partial_bg_va: u64, + eot_va: u64, + partial_eot_va: u64, +} + +impl PassWords { + fn new(pass: &RenderPass) -> Result { + if pass.samples_log2 > SAMPLES_LOG2_MAX { + return Err(EINVAL); + } + let samples = 1u64 << pass.samples_log2; + let layered = pass.layers > 1; + + // The tile configuration carries the empty-tile request for every + // target; the tiling layer mode only for layered ones. + let mut tile_config = TILE_CONFIG_BASE; + let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); + if layered { + tile_config |= TILE_CONFIG_LAYERED; + layer_mode |= LAYER_MODE_LAYERED; + layer_mode |= match pass.process_empty_tiles { + true => LAYER_MODE_PROCESS_EMPTY, + false => LAYER_MODE_SKIP_EMPTY, + }; + } + if pass.process_empty_tiles { + tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; + } + + let mut partial_zls_control = pass.zls_control; + if pass.depth.va != 0 { + partial_zls_control |= ZLS_PARTIAL_DEPTH; + } + if pass.stencil.va != 0 { + partial_zls_control |= ZLS_PARTIAL_STENCIL; + } + + let aux_fb_flags = match pass.depth_bias_is_int { + true => AUX_FB_FLAGS_BASE | AUX_FB_FLAGS_DBIAS_INT, + false => AUX_FB_FLAGS_BASE, + }; + let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { + true => spec, + false => spec & u32::MAX as u64, + }; + let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); + + Ok(Self { + utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT + | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT + | pass.samples_log2 as u64, + tile_config, + layer_mode, + tib_blocks: (pass.sample_size as u64 + * pass.utile_width as u64 + * pass.utile_height as u64 + * samples) + .div_ceil(TIB_BLOCK_SIZE), + partial_zls_control, + aux_fb_flags, + bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), + bg_va: usc(pass.bg.offset)?, + partial_bg_rsrc_spec: rsrc_spec(pass.partial_bg.rsrc_spec), + partial_bg_va: usc(pass.partial_bg.offset)?, + eot_va: usc(pass.eot_offset)?, + partial_eot_va: usc(pass.partial_eot_offset)?, + }) + } +} + +/// Offset of an address from the user window base. +fn user_offset(va: u64) -> Result { + va.checked_sub(USER_BASE).ok_or(EINVAL) +} + +/// Offset of an address from the user window base that fits 32 bits. +fn user_offset32(va: u64) -> Result { + u32::try_from(user_offset(va)?).map_err(|_| EINVAL) +} + +fn ta_registers( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> Result<[RegisterWrite; TA_REGISTERS]> { + let p = args.pass; + let r = args.resources; + let tilemap = user_offset(r.tilemap_va)?; + let layermeta = user_offset(r.layermeta_va)?; + let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; + let tpc = user_offset(r.tpc_va)?; + let deflake = user_offset32(r.deflake_va)? as u64; + let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; + let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; + let vdm = user_offset(p.vdm_va)?; + if vdm >= VDM_OFFSET_LIMIT { + return Err(EINVAL); + } + let key = work_key(r.context_id, args.ta.state_word); + let ids = r.ta_ids.word(); + + Ok([ + RegisterWrite::new(0x01748, 1), + RegisterWrite::new(0x10141, 0x200), + RegisterWrite::new(0x1c039, tilemap), + RegisterWrite::new(0x1c9c8, tilemap), + RegisterWrite::new(0x1c0a1, tpc), + RegisterWrite::new(0x1c031, heapmeta), + RegisterWrite::new(0x1c9c0, heapmeta), + RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), + RegisterWrite::new(0x1c061, 1), + RegisterWrite::new(0x10149, w.utile_config), + RegisterWrite::new(0x10139, p.msaa_control), + RegisterWrite::new(0x10111, deflake_2a0), + RegisterWrite::new(0x1c9b0, deflake_2a0), + RegisterWrite::new(0x10119, deflake_20), + RegisterWrite::new(0x1c9b8, deflake_20), + RegisterWrite::new(0x1c958, 1), + RegisterWrite::new(0x1c950, deflake | 1 << 50), + RegisterWrite::new(0x1c930, 0), + RegisterWrite::new(0x1c880, vdm), + RegisterWrite::new(0x1c079, layermeta), + RegisterWrite::new(0x1c9d8, layermeta), + RegisterWrite::new(0x10151, 0), + RegisterWrite::new(0x1c199, 0), + RegisterWrite::new(0x1c1a1, 0), + RegisterWrite::new(0x1c1a9, 0), + RegisterWrite::new(0x1c1b1, 0), + RegisterWrite::new(0x1c1b9, 0), + RegisterWrite::new(0x1c8f8, UNK_1C8F8), + RegisterWrite::new(0x1c0b1, g.region_size), + RegisterWrite::new(0x1c850, g.region_size), + RegisterWrite::new(0x10131, p.msaa_control), + RegisterWrite::new(0x10121, p.ppp_control), + RegisterWrite::new(0x10129, g.pixels), + RegisterWrite::new(0x101b9, g.screen), + RegisterWrite::new(0x1c069, g.x_blocks), + RegisterWrite::new(0x1c071, g.y_blocks), + RegisterWrite::new(0x1c081, g.macro_tiles), + RegisterWrite::new(0x1c0a9, g.utile_slots), + RegisterWrite::new(0x10171, 0x100), + RegisterWrite::new(0x10169, w.layer_mode), + RegisterWrite::new(0x0a309, 0), + RegisterWrite::new(0x1c8e0, u64::MAX), + RegisterWrite::new(0x1c8e8, u64::MAX), + RegisterWrite::new(0x1c898, 0), + RegisterWrite::new(0x101e1, UNK_101D9), + RegisterWrite::new(0x1c9e8, w.layer_mode & LAYER_MODE_PIPE_MASK), + RegisterWrite::new(0x1a099, 0), + RegisterWrite::new(0x1a0a1, 0), + RegisterWrite::new(0x1a069, 0), + RegisterWrite::new(0x1a071, 0), + RegisterWrite::new(0x1a0c9, 0), + RegisterWrite::new(0x1a0d1, 0), + RegisterWrite::new(0x101c9, 0), + RegisterWrite::new(0x0d471, 0), + RegisterWrite::new(0x1a0f1, UNK_1A0E9), + RegisterWrite::new(0x10799, UNK_10791), + RegisterWrite::new(0x1c830, r.pb_slot as u64), + RegisterWrite::new(0x1ca30, r.pm_scratch), + RegisterWrite::new(0x16c39, r.pm_scratch), + RegisterWrite::new(0x1c910, r.pm_metric), + RegisterWrite::new(0x0a5a1, UNK_0A599), + RegisterWrite::new(0x0d419, UNK_0D411), + RegisterWrite::new(0x1ca10, ids), + RegisterWrite::new(0x014a1, ids), + RegisterWrite::new(0x0a349, ids), + RegisterWrite::new(0x10209, key), + RegisterWrite::new(0x1c9f0, key), + RegisterWrite::new(0x14320, key), + RegisterWrite::new(0x14308, r.free_list_slot as u64), + RegisterWrite::new(0x14318, r.ta_status_va | 1), + RegisterWrite::new(0x01740, 1), + RegisterWrite::new(0x1c880, deflake_2a0), + RegisterWrite::new(0x1c898, 1), + ]) +} + +fn fragment_registers( + args: &RenderArgs<'_>, + g: &Geometry, + w: &PassWords, +) -> [RegisterWrite; FRAGMENT_REGISTERS] { + let p = args.pass; + let r = args.resources; + let mut tile_state = TILE_STATE_BASE + | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT + | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT + | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; + if p.layers > 1 { + tile_state |= TILE_STATE_LAYERED; + } + let key = work_key(r.context_id, args.fragment.state_word); + let ids = r.fragment_ids.word(); + + [ + RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x10009, w.utile_config), + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.eot_va), + RegisterWrite::new(0x15369, w.bg_rsrc_spec), + RegisterWrite::new(0x15371, w.bg_va), + RegisterWrite::new(0x15131, p.merge_upper_x as u64), + RegisterWrite::new(0x15139, p.merge_upper_y as u64), + RegisterWrite::new(0x100a1, 0), + RegisterWrite::new(0x15069, 0), + RegisterWrite::new(0x15071, 0), + RegisterWrite::new(0x16058, 0), + RegisterWrite::new(0x10019, p.msaa_control), + RegisterWrite::new(0x100b1, g.macro_size), + RegisterWrite::new(0x16030, g.macro_size), + RegisterWrite::new(0x100d9, g.screen), + RegisterWrite::new(0x0a301, 0), + RegisterWrite::new(0x10791, UNK_10791_FRAGMENT), + RegisterWrite::new(0x16098, r.heapmeta_va), + RegisterWrite::new(0x15109, p.scissor_va), + RegisterWrite::new(0x15101, p.depth_bias_va), + RegisterWrite::new(0x15021, w.aux_fb_flags), + RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), + RegisterWrite::new(0x15049, UNK_15049), + RegisterWrite::new(0x10051, w.tib_blocks), + RegisterWrite::new(0x15321, p.depth_dimensions), + RegisterWrite::new(0x15301, p.depth_clear as u64), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15311, p.occlusion_query_va), + RegisterWrite::new(0x15319, p.zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + RegisterWrite::new(0x15351, 0), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x15231, 0), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15239, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15401, p.depth.stride), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15409, p.stencil.stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15411, p.depth.compression_stride), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15419, p.stencil.compression_stride), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x16429, r.tilemap_va), + RegisterWrite::new(0x16060, r.layermeta_va), + RegisterWrite::new(0x16431, (4 * g.region_size) << 24), + RegisterWrite::new(0x10039, w.tile_config), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16451, 0), + RegisterWrite::new(0x15359, 0), + RegisterWrite::new(0x100b8, UNK_1C8F8), + RegisterWrite::new(0x16461, r.aux_fb_va), + RegisterWrite::new(0x16090, r.aux_fb_va), + RegisterWrite::new(0x101e9, UNK_101D9), + RegisterWrite::new(0x160a8, 0), + RegisterWrite::new(0x16068, tile_state), + RegisterWrite::new(0x1a0a9, 0), + RegisterWrite::new(0x1a0b1, 0), + RegisterWrite::new(0x1a079, 0), + RegisterWrite::new(0x1a081, 0), + RegisterWrite::new(0x1a0d9, 0), + RegisterWrite::new(0x1a0e1, 0), + RegisterWrite::new(0x101c1, 0), + RegisterWrite::new(0x0d469, 0), + RegisterWrite::new(0x1a0f9, UNK_1A0E9), + RegisterWrite::new(0x0a5a9, UNK_0A599), + RegisterWrite::new(0x0d429, UNK_0D411), + RegisterWrite::new(0x160e0, ids), + RegisterWrite::new(0x01499, ids), + RegisterWrite::new(0x0a341, ids), + RegisterWrite::new(0x1c838, r.pb_slot as u64), + RegisterWrite::new(0x1ca28, r.pm_scratch), + RegisterWrite::new(0x10211, key), + RegisterWrite::new(0x10420, key), + RegisterWrite::new(0x14048, r.free_list_slot as u64), + RegisterWrite::new(0x14080, r.fragment_status_va | 1), + RegisterWrite::new(0x01731, 1), + RegisterWrite::new(0x16020, 1), + RegisterWrite::new(0x16020, 0), + RegisterWrite::new(0x16068, 0x4_0000), + ] +} + +fn partial_store_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_STORE_REGISTERS] { + [ + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.partial_eot_va), + RegisterWrite::new(0x10039, w.tile_config), + RegisterWrite::new(0x15359, UNK_15359_PARTIAL), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +fn partial_resume_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_RESUME_REGISTERS] { + [ + // End-of-tile resource specifier. + RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15381, w.partial_eot_va), + RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), + RegisterWrite::new(0x15371, w.partial_bg_va), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15359, UNK_15359_PARTIAL), + RegisterWrite::new(0x15331, p.depth.va), + RegisterWrite::new(0x153c9, p.depth.compression_va), + RegisterWrite::new(0x15341, p.stencil.va), + RegisterWrite::new(0x153d9, p.stencil.compression_va), + RegisterWrite::new(0x15421, p.depth.stride), + RegisterWrite::new(0x15431, p.depth.compression_stride), + RegisterWrite::new(0x15429, p.stencil.stride), + RegisterWrite::new(0x15439, p.stencil.compression_stride), + RegisterWrite::new(0x15221, 0), + RegisterWrite::new(0x15229, 0), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +fn partial_load_registers( + p: &RenderPass, + w: &PassWords, +) -> [RegisterWrite; PARTIAL_LOAD_REGISTERS] { + [ + RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), + RegisterWrite::new(0x15371, w.partial_bg_va), + RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), + RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), + RegisterWrite::new(0x15329, p.depth.va), + RegisterWrite::new(0x153c1, p.depth.compression_va), + RegisterWrite::new(0x15339, p.stencil.va), + RegisterWrite::new(0x153d1, p.stencil.compression_va), + RegisterWrite::new(0x15319, w.partial_zls_control), + RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), + ] +} + +/// Tiling (TA) work descriptor (tag 0). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct TaDescriptor { + /// [`CommandTag::Tiling`]. + pub(crate) tag: u32, + /// Twice the pair's submission ordinal, plus one. + pub(crate) submit_sequence: u64, + /// UAT context of the pass. + pub(crate) context_id: u32, + /// GPU address of the pass's kick record. + pub(crate) kick_record_va: u64, + /// Buffer slot of the parameter buffer. + pub(crate) pb_slot: u32, + pub(crate) unk_1c: u32, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// Checkpoints into the register array. + pub(crate) checkpoints: [u16; 3], + pub(crate) unk_3e: [u8; 0xa], + /// The pair's submission ordinal plus half of it. + pub(crate) unk_48: u32, + pub(crate) unk_4c: [u8; 0x14], + /// Register array. + pub(crate) registers: RegisterArray, + /// GPU address of the tail-pointer cache. + pub(crate) tpc_va: u64, + /// Size of the tail-pointer cache the target needs. + pub(crate) tpc_size: u64, + pub(crate) unk_790: [u8; 0xc], + /// QID of the fragment queue that consumes this pass. + pub(crate) fragment_qid: u32, + /// Completion stamp value of the pass. + pub(crate) stamp_value: u32, + /// Non-zero when the fragment queue had earlier kicks outstanding. + pub(crate) fragment_busy: u32, + /// Timestamp of the fragment kick of this pass. + pub(crate) fragment_kick: u64, + /// Render-pass sequence of the pass, stored twice. + pub(crate) pass_seq: [u8; 2], + pub(crate) unk_7b2: [u8; 0x24], + /// Offset from the user base of the deflake sub-buffer at +0x2a0. + pub(crate) deflake_2a0_offset: u32, + pub(crate) unk_7da: [u8; 0x94], + /// Predecessor object ID of the tiling stage. + pub(crate) predecessor_id: u32, + pub(crate) unk_872: u32, + pub(crate) unk_876: u32, + /// GPU address of the sampler heap. + pub(crate) sampler_heap_va: u64, + /// Number of samplers. + pub(crate) sampler_count: u32, + /// Number of samplers plus one, or zero without samplers. + pub(crate) sampler_max: u32, + pub(crate) unk_88a: [u8; 8], + pub(crate) unk_892: u32, + pub(crate) unk_896: [u8; 0x10], + /// GPU address of the tiling queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the tiling queue's auxiliary stamp word. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the pass. + pub(crate) stamp_value_2: u32, + /// QID of the tiling queue. + pub(crate) qid: u32, + pub(crate) unk_8be: u32, + pub(crate) unk_8c2: u32, + /// Completion value of the tiling kick (see + /// [`KickTimestamp::completion_value`]). + pub(crate) completion: u64, + /// Current object ID of the tiling stage. + pub(crate) current_id: u64, + /// Submission ordinal of the pair. + pub(crate) ordinal: u32, + pub(crate) unk_8da: [u8; 0x24], + /// GPU address the pass start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// Zero: the tiling stage writes no end timestamp. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_90e: [u8; 0x24], + pub(crate) unk_932: u8, + pub(crate) unk_933: u8, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_93c: u8, + pub(crate) unk_93d: u8, + /// Set on the first kick of a queue pair. + pub(crate) unk_93e: [u8; 2], + pub(crate) unk_940: [u8; 5], + /// GPU address of the tiling status word. + pub(crate) status_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_94e: [u8; 0x72], +} + +static_assert!(size_of::() == TA_DESCRIPTOR_SIZE); +static_assert!(core::mem::offset_of!(TaDescriptor, unk_48) == 0x48); +static_assert!(core::mem::offset_of!(TaDescriptor, registers) == 0x60); +static_assert!(core::mem::offset_of!(TaDescriptor, tpc_va) == 0x780); +static_assert!(core::mem::offset_of!(TaDescriptor, fragment_qid) == 0x79c); +static_assert!(core::mem::offset_of!(TaDescriptor, deflake_2a0_offset) == 0x7d6); +static_assert!(core::mem::offset_of!(TaDescriptor, predecessor_id) == 0x86e); +static_assert!(core::mem::offset_of!(TaDescriptor, stamp_va) == 0x8a6); +static_assert!(core::mem::offset_of!(TaDescriptor, completion) == 0x8c6); +static_assert!(core::mem::offset_of!(TaDescriptor, timestamp_start_va) == 0x8fe); +static_assert!(core::mem::offset_of!(TaDescriptor, free_list_control_va) == 0x934); +static_assert!(core::mem::offset_of!(TaDescriptor, status_va) == 0x945); + +// SAFETY: `TaDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for TaDescriptor {} + +impl TaDescriptor { + const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; + const UNK_876: u32 = u32::MAX; + const UNK_892: u32 = 1; + const UNK_8C2: u32 = 1; + const UNK_932: u8 = 0x44; + const UNK_93C: u8 = 1; + const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; + + /// Writes the complete descriptor except the completion link. + pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { + let pass = args.pass; + let resources = args.resources; + let geometry = Geometry::new(pass)?; + let words = PassWords::new(pass)?; + let registers = ta_registers(args, &geometry, &words)?; + let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; + let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; + let sampler_max = sampler_max(pass.sampler_count)?; + let stamp_value = args.stamp_value(); + + clear(self); + self.tag = CommandTag::Tiling as u32; + self.submit_sequence = args.ordinal.wrapping_mul(2).wrapping_add(1); + self.context_id = resources.context_id.into(); + self.kick_record_va = resources.kick_record_va; + self.pb_slot = resources.pb_slot; + self.pb_va = resources.pb_va; + self.scene_va = resources.scene_va; + self.scene_end_va = resources.scene_end_va; + self.checkpoints = TA_CHECKPOINTS; + self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); + self.registers.set(array_va, ®isters)?; + self.tpc_va = resources.tpc_va; + self.tpc_size = resources.tpc_size; + self.fragment_qid = args.fragment.qid.into(); + self.stamp_value = stamp_value; + self.deflake_2a0_offset = deflake_2a0_offset; + self.predecessor_id = resources.ta_ids.predecessor; + self.unk_876 = Self::UNK_876; + self.sampler_heap_va = pass.sampler_heap_va; + self.sampler_count = pass.sampler_count; + self.sampler_max = sampler_max; + self.unk_892 = Self::UNK_892; + self.stamp_va = args.ta.stamp_va; + self.aux_stamp_va = args.ta.aux_stamp_va; + self.stamp_value_2 = stamp_value; + self.qid = args.ta.qid.into(); + self.unk_8c2 = Self::UNK_8C2; + self.current_id = resources.ta_ids.current.into(); + self.ordinal = args.ordinal as u32; + self.timestamp_start_va = resources.timestamp_va; + self.unk_932 = Self::UNK_932; + self.free_list_control_va = resources.free_list_control_va; + self.unk_93c = Self::UNK_93C; + if args.ordinal == 0 { + self.unk_93e = Self::UNK_93E_FIRST_KICK; + } + self.status_va = resources.ta_status_va; + self.context_generation = resources.context_generation; + Ok(()) + } + + /// Links the pass to the timestamps its two kicks are published at. + /// + /// Must be called before the descriptor is published. `pass_seq` is the + /// low byte of the logical queue's render-pass count (one for its first + /// pass); `fragment_busy` tells whether the fragment queue had not yet + /// completed every earlier kick. + pub(crate) fn set_completion_link( + &mut self, + ta: KickTimestamp, + fragment: KickTimestamp, + pass_seq: u8, + fragment_busy: bool, + ) { + self.fragment_busy = fragment_busy as u32; + self.fragment_kick = fragment.get(); + self.pass_seq = [pass_seq; 2]; + self.completion = ta.completion_value(pass_seq); + } + + /// Returns the register arrays a tiling kick binds; `descriptor_va` is the + /// descriptor's address as seen by the pass's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let array = RegisterArrayBinding::new( + offset_va(descriptor_va, Self::ARRAY_OFFSET)?, + TA_CHECKPOINTS[0] as u8, + )?; + Ok([Some(array), None, None, None]) + } +} + +/// Fragment (3D) work descriptor (tag 1). +#[repr(C, packed)] +#[derive(Debug)] +pub(crate) struct FragmentDescriptor { + /// [`CommandTag::Fragment`]. + pub(crate) tag: u32, + /// Twice the pair's submission ordinal. + pub(crate) submit_sequence: u64, + /// UAT context of the pass. + pub(crate) context_id: u32, + pub(crate) unk_10: [u8; 0x10], + /// GPU address of the pass's kick record. + pub(crate) kick_record_va: u64, + /// GPU address of the parameter-buffer state object. + pub(crate) pb_va: u64, + /// GPU address of the parameter-buffer scene entry. + pub(crate) scene_va: u64, + /// GPU address following the last scene entry. + pub(crate) scene_end_va: u64, + /// GPU address of the tile map. + pub(crate) tilemap_va: u64, + /// Multisample control. + pub(crate) msaa_control: u64, + /// Samples per pixel. + pub(crate) samples: u32, + /// Macro-tile dimensions in utiles: height in bits 15:0, width above. + pub(crate) macro_size: u32, + pub(crate) unk_58: [u8; 0x10], + /// Merge upper bound in X. + pub(crate) merge_upper_x: u32, + /// Merge upper bound in Y. + pub(crate) merge_upper_y: u32, + pub(crate) unk_70: u64, + /// Number of tiles. + pub(crate) tiles: u64, + /// Checkpoints into the main register array. + pub(crate) checkpoints: [u16; 3], + pub(crate) unk_86: u16, + /// Entry count of the main register array. + pub(crate) entries: u16, + pub(crate) unk_8a: [u8; 6], + /// Render-pass sequence of the pass. + pub(crate) pass_seq: u8, + pub(crate) unk_91: [u8; 0xf], + /// Main, partial-store, partial-resume and partial-load register arrays. + pub(crate) arrays: [RegisterArray; 4], + /// GPU address of the depth-bias array. + pub(crate) depth_bias_va: u64, + pub(crate) unk_1d28: u64, + /// GPU address of the scissor array. + pub(crate) scissor_va: u64, + pub(crate) unk_1d38: u64, + /// GPU address of the occlusion-query results. + pub(crate) occlusion_query_va: u64, + pub(crate) unk_1d48: [u8; 0x130], + /// Background resource specifier. + pub(crate) bg_rsrc_spec: u64, + /// GPU address of the background program. + pub(crate) bg_va: u64, + pub(crate) unk_1e88: [u8; 0x20], + /// Partial-render background resource specifier. + pub(crate) partial_bg_rsrc_spec: u64, + /// GPU address of the partial-render background program. + pub(crate) partial_bg_va: u64, + pub(crate) unk_1eb8: u64, + /// ZLS swizzle. + pub(crate) zls_swizzle: u32, + pub(crate) unk_1ec4: [u8; 0x74], + /// Tile-buffer blocks per utile. + pub(crate) tib_blocks: u64, + /// Auxiliary framebuffer flags. + pub(crate) aux_fb_flags: u64, + /// Width of the render target. + pub(crate) width: u32, + /// Height of the render target. + pub(crate) height: u32, + /// [`UNK_15049`]. + pub(crate) unk_1f50: u64, + /// Tile configuration. + pub(crate) tile_config: u64, + /// Ends with the end-of-tile resource specifier, which is zero. + pub(crate) unk_1f60: [u8; 0x1c], + /// GPU address of the end-of-tile program. + pub(crate) eot_va: u64, + /// Ends with the partial end-of-tile resource specifier, which is zero. + pub(crate) unk_1f84: [u8; 0x18], + /// GPU address of the partial-render end-of-tile program. + pub(crate) partial_eot_va: u64, + pub(crate) unk_1fa4: u32, + /// Depth clear value. + pub(crate) depth_clear: u32, + /// Tile-buffer blocks in bits 63:33 above a constant. + pub(crate) unk_1fac: u64, + pub(crate) unk_1fb4: [u8; 0x150], + /// See [`RenderResources::unk_2104`]. + pub(crate) unk_2104: u32, + /// Predecessor object ID of the fragment stage. + pub(crate) predecessor_id: u32, + /// See [`RenderResources::unk_210c`]. + pub(crate) unk_210c: u32, + pub(crate) unk_2110: u32, + pub(crate) unk_2114: [u8; 0x10], + pub(crate) unk_2124: [u32; 2], + pub(crate) unk_212c: [u8; 0x14], + /// GPU address of the fragment queue's completion stamp word. + pub(crate) stamp_va: u64, + /// GPU address of the fragment queue's auxiliary stamp word. + pub(crate) aux_stamp_va: u64, + /// Completion stamp value of the pass. + pub(crate) stamp_value: u32, + /// QID of the fragment queue. + pub(crate) qid: u32, + pub(crate) unk_2158: u32, + /// Non-zero on the first kick of a queue pair. + pub(crate) first_kick: u32, + /// Completion value of the fragment kick (see + /// [`KickTimestamp::completion_value`]). + pub(crate) completion: u64, + /// Current object ID of the fragment stage. + pub(crate) current_id: u64, + /// Submission ordinal of the pair. + pub(crate) ordinal: u32, + pub(crate) unk_2174: [u8; 0x24], + /// GPU address the pass start timestamp is written to. + pub(crate) timestamp_start_va: u64, + /// GPU address the pass end timestamp is written to. + pub(crate) timestamp_end_va: u64, + pub(crate) unk_21a8: [u8; 0x24], + pub(crate) unk_21cc: u8, + pub(crate) unk_21cd: u8, + /// GPU address of the USC free list's shared control object. + pub(crate) free_list_control_va: u64, + pub(crate) unk_21d6: u8, + /// GPU address of the fragment status word, as seen by the pass's VM. + pub(crate) status_va: u64, + /// Firmware alias of the fragment status word. + pub(crate) status_fw_va: u64, + /// Allocation generation of the UAT context. + pub(crate) context_generation: u8, + pub(crate) unk_21e8: [u8; 0x20], + pub(crate) unk_2208: u8, + pub(crate) unk_2209: [u8; 3], + pub(crate) unk_220c: u8, + pub(crate) unk_220d: [u8; 0x20], + pub(crate) unk_222d: u8, + pub(crate) unk_222e: [u8; 0x12], +} + +static_assert!(size_of::() == 0x2240); +static_assert!(core::mem::offset_of!(FragmentDescriptor, checkpoints) == 0x80); +static_assert!(core::mem::offset_of!(FragmentDescriptor, pass_seq) == 0x90); +static_assert!(core::mem::offset_of!(FragmentDescriptor, arrays) == 0xa0); +static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_bias_va) == 0x1d20); +static_assert!(core::mem::offset_of!(FragmentDescriptor, bg_rsrc_spec) == 0x1e78); +static_assert!(core::mem::offset_of!(FragmentDescriptor, tib_blocks) == 0x1f38); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_1f50) == 0x1f50); +static_assert!(core::mem::offset_of!(FragmentDescriptor, eot_va) == 0x1f7c); +static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_clear) == 0x1fa8); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_2104) == 0x2104); +static_assert!(core::mem::offset_of!(FragmentDescriptor, stamp_va) == 0x2140); +static_assert!(core::mem::offset_of!(FragmentDescriptor, completion) == 0x2160); +static_assert!(core::mem::offset_of!(FragmentDescriptor, timestamp_start_va) == 0x2198); +static_assert!(core::mem::offset_of!(FragmentDescriptor, free_list_control_va) == 0x21ce); +static_assert!(core::mem::offset_of!(FragmentDescriptor, status_va) == 0x21d7); +static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_222d) == 0x222d); + +// SAFETY: `FragmentDescriptor` consists of integers and arrays of integers only. +unsafe impl Zeroable for FragmentDescriptor {} + +impl FragmentDescriptor { + /// Offset of register array `index` within the descriptor. + const fn array_offset(index: usize) -> u64 { + (core::mem::offset_of!(Self, arrays) + index * size_of::()) as u64 + } + + /// Returns the register arrays a fragment kick binds; `descriptor_va` is + /// the descriptor's address as seen by the pass's VM. + pub(crate) fn register_bindings( + descriptor_va: u64, + ) -> Result<[Option; 4]> { + let array = |index, checkpoint: usize| -> Result> { + let va = offset_va(descriptor_va, Self::array_offset(index))?; + Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) + }; + Ok([ + array(0, FRAGMENT_CHECKPOINTS[0] as usize)?, + array(1, PARTIAL_STORE_REGISTERS)?, + array(2, PARTIAL_RESUME_REGISTERS)?, + array(3, PARTIAL_LOAD_REGISTERS)?, + ]) + } +} + +/// A fragment descriptor followed by room for an MCache table that does not +/// fit its first register array. +#[repr(C)] +#[derive(Debug)] +pub(crate) struct FragmentSlot { + /// The descriptor. + pub(crate) descriptor: FragmentDescriptor, + /// MCache table of more than [`MCACHE_INLINE_RANGES`] ranges. + pub(crate) mcache: [McacheRange; MCACHE_RANGES_MAX], + pub(crate) pad_2350: [u8; 0x30], +} + +static_assert!(size_of::() == FRAGMENT_SLOT_SIZE); + +impl FragmentSlot { + const INLINE_MCACHE_OFFSET: u64 = + FragmentDescriptor::array_offset(0) + core::mem::offset_of!(RegisterArray, mcache) as u64; + const SPILL_MCACHE_OFFSET: u64 = core::mem::offset_of!(Self, mcache) as u64; + const UNK_2110: u32 = u32::MAX; + const UNK_2124: [u32; 2] = [1, 1]; + const UNK_21CC: u8 = 0x53; + const UNK_21D6: u8 = 1; + const UNK_2208: u8 = 1; + const UNK_220C: u8 = 1; + const UNK_222D: u8 = 1; + const UNK_1FAC_BASE: u64 = 0x300; + const UNK_1FAC_TIB_SHIFT: u32 = 33; + + /// Writes the complete descriptor except the completion link and the + /// MCache table. + /// + /// The spill table is left as it is: a kick entry names it only after + /// [`Self::set_mcache`] rewrote it, so stale bytes there are never read. + pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { + let pass = args.pass; + let resources = args.resources; + let geometry = Geometry::new(pass)?; + let words = PassWords::new(pass)?; + let base = args.fragment.descriptor_va; + let array_va = |index| offset_va(base, FragmentDescriptor::array_offset(index)); + let arrays = [array_va(0)?, array_va(1)?, array_va(2)?, array_va(3)?]; + let timestamp_end_va = timestamp_end_va(resources.timestamp_va)?; + let desc = &mut self.descriptor; + + clear(desc); + desc.tag = CommandTag::Fragment as u32; + desc.submit_sequence = args.ordinal.wrapping_mul(2); + desc.context_id = resources.context_id.into(); + desc.kick_record_va = resources.kick_record_va; + desc.pb_va = resources.pb_va; + desc.scene_va = resources.scene_va; + desc.scene_end_va = resources.scene_end_va; + desc.tilemap_va = resources.tilemap_va; + desc.msaa_control = pass.msaa_control; + desc.samples = 1 << pass.samples_log2; + desc.macro_size = geometry.macro_size as u32; + desc.merge_upper_x = pass.merge_upper_x; + desc.merge_upper_y = pass.merge_upper_y; + desc.tiles = geometry.tiles_x * geometry.tiles_y; + desc.checkpoints = FRAGMENT_CHECKPOINTS; + desc.entries = FRAGMENT_REGISTERS as u16; + desc.arrays[0].set(arrays[0], &fragment_registers(args, &geometry, &words))?; + desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; + desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; + desc.arrays[3].set(arrays[3], &partial_load_registers(pass, &words))?; + desc.depth_bias_va = pass.depth_bias_va; + desc.scissor_va = pass.scissor_va; + desc.occlusion_query_va = pass.occlusion_query_va; + desc.bg_rsrc_spec = words.bg_rsrc_spec; + desc.bg_va = words.bg_va; + desc.partial_bg_rsrc_spec = words.partial_bg_rsrc_spec; + desc.partial_bg_va = words.partial_bg_va; + desc.zls_swizzle = ZLS_SWIZZLE; + desc.tib_blocks = words.tib_blocks; + desc.aux_fb_flags = words.aux_fb_flags; + desc.width = pass.width; + desc.height = pass.height; + desc.unk_1f50 = UNK_15049; + desc.tile_config = words.tile_config; + desc.eot_va = words.eot_va; + desc.partial_eot_va = words.partial_eot_va; + desc.depth_clear = pass.depth_clear; + desc.unk_1fac = words.tib_blocks << Self::UNK_1FAC_TIB_SHIFT | Self::UNK_1FAC_BASE; + desc.unk_2104 = resources.unk_2104; + desc.predecessor_id = resources.fragment_ids.predecessor; + desc.unk_210c = resources.unk_210c; + desc.unk_2110 = Self::UNK_2110; + desc.unk_2124 = Self::UNK_2124; + desc.stamp_va = args.fragment.stamp_va; + desc.aux_stamp_va = args.fragment.aux_stamp_va; + desc.stamp_value = args.stamp_value(); + desc.qid = args.fragment.qid.into(); + desc.first_kick = (args.ordinal == 0) as u32; + desc.current_id = resources.fragment_ids.current.into(); + desc.ordinal = args.ordinal as u32; + desc.timestamp_start_va = resources.timestamp_va; + desc.timestamp_end_va = timestamp_end_va; + desc.unk_21cc = Self::UNK_21CC; + desc.free_list_control_va = resources.free_list_control_va; + desc.unk_21d6 = Self::UNK_21D6; + desc.status_va = resources.fragment_status_va; + desc.status_fw_va = resources.fragment_status_fw_va; + desc.context_generation = resources.context_generation; + desc.unk_2208 = Self::UNK_2208; + desc.unk_220c = Self::UNK_220C; + desc.unk_222d = Self::UNK_222D; + Ok(()) + } + + /// Links the pass to the timestamp of its fragment kick. + /// + /// Must be called before the descriptor is published; see + /// [`TaDescriptor::set_completion_link`]. + pub(crate) fn set_completion_link(&mut self, fragment: KickTimestamp, pass_seq: u8) { + self.descriptor.pass_seq = pass_seq; + self.descriptor.completion = fragment.completion_value(pass_seq); + } + + /// Writes the MCache table of the pass: the auxiliary framebuffer at + /// `aux_fb_va`, then `attachments`, translated in UAT context `context`. + /// `slot_va` is the GPU address of this slot as seen by the pass's VM. + /// + /// Returns the table for the fragment kick entry, or `None` when the pass + /// has no attachments. + pub(crate) fn set_mcache( + &mut self, + slot_va: u64, + aux_fb_va: u64, + attachments: &[WriteRange], + context: u8, + ) -> Result> { + if attachments.is_empty() { + return Ok(None); + } + let count = attachments.len() + 1; + if count > MCACHE_RANGES_MAX { + return Err(EINVAL); + } + let (offset, ranges) = if count <= MCACHE_INLINE_RANGES { + ( + Self::INLINE_MCACHE_OFFSET, + &mut self.descriptor.arrays[0].mcache[..count], + ) + } else { + (Self::SPILL_MCACHE_OFFSET, &mut self.mcache[..count]) + }; + let table = McacheTable::new(offset_va(slot_va, offset)?, count)?; + let aux_fb = WriteRange { + va: aux_fb_va, + size: AUX_FB_SIZE, + }; + ranges[0] = McacheRange::new(&aux_fb, McacheClass::AuxFramebuffer, context)?; + for (range, attachment) in ranges[1..].iter_mut().zip(attachments) { + *range = McacheRange::new(attachment, McacheClass::Attachment, context)?; + } + Ok(Some(table)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs new file mode 100644 index 00000000000000..e1e6a10949fb44 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render/storage.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Auxiliary render records and their initial images. Live event counters +//! are accessed individually by the queue owner after publication. + +use super::super::{buffer, clear, context, initdata}; +use kernel::prelude::*; + +pub(crate) const SUPPORT_SIZE: usize = 0x40000; +pub(crate) const EVENT_SLOTS: usize = 0x4000; +pub(crate) const EVENT_COUNT: usize = 35; +pub(crate) const STATE_SIZE: usize = 0x18000; +pub(crate) const DEFLAKE: usize = 0x4000; +pub(crate) const TA_STATUS: usize = 0x8000; +pub(crate) const AUXILIARY: usize = 0x10000; +pub(crate) const AUXILIARY_SIZE: usize = 0x8000; + +#[repr(C)] +pub(crate) struct Event { + pub(crate) node: context::WorkNode, + unk_c8: [u8; 0x38], +} + +#[repr(C)] +pub(crate) struct Support { + secondary_index: [u64; 8], + unk_40: [u8; 0x3fc0], + pub(crate) event_slots: [u32; 0x1000], + scene_slots: [u32; 0x1000], + shared_slots: [u32; 0x1000], + flag: u32, + unk_10004: [u8; 0x3ffc], + pub(crate) events: [Event; EVENT_COUNT], + unk_16300: [u8; 0x1d00], + scenes: [buffer::Scene; 79], + unk_1a780: [u8; 0x1880], + shared: buffer::State, + unk_1c0c0: [u8; 0x7f40], + control: initdata::FreeListControl, + unk_24068: [u8; 0x3f98], + submitted: u32, + unk_28004: [u8; 0x3ffc], + scheduler: context::Scheduler, + unk_30000: [u8; 0x10000], +} +// SAFETY: The entire graph is integer storage; all nested types are Zeroable. +unsafe impl Zeroable for Support {} +impl Support { + pub(crate) fn init( + &mut self, + address: u64, + primary_index: u64, + operand_table: u64, + page_list: u64, + word: u32, + ) -> Result { + clear(self); + address.checked_add(SUPPORT_SIZE as u64).ok_or(EOVERFLOW)?; + self.secondary_index = [0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x3c, 0x41]; + self.event_slots[1] = 1; + self.shared_slots[0] = 8; + self.shared_slots[1] = 8; + self.shared_slots[0x60 / 4] = 1; + self.flag = 1; + for (index, event) in self.events.iter_mut().enumerate() { + event.node.submitted = address + EVENT_SLOTS as u64 + index as u64 * 4; + if index == 1 { + event.node.threshold = 0x50; + event.node.word = word; + } + } + for (index, scene) in self.scenes.iter_mut().enumerate() { + scene.metric_client = 0x10_0008_0004 + index as u64 * 4; + scene.metric_fw = address + 0x8004 + index as u64 * 4; + let phase = index % buffer::SCRATCH_SLOTS; + scene.scratch = if phase == buffer::SCRATCH_SLOTS - 1 { + 0x178000 + } else { + 0x178020 + phase as u64 * buffer::SCRATCH_STRIDE + }; + scene.statistics = address + 0xc040; + if index == 0 { + scene.generation = 1; + } + } + let shared = &mut self.shared; + shared.page_list_fw = primary_index; + shared.page_list_client = 0x10_0019_0000; + shared.page_list_size = 0x10000; + shared.pages = 0x20; + shared.block_capacity = 0xc18; + shared.producer = 8; + shared.block_table = address; + shared.control = address + 0xc000; + shared.last_page = 0x1f; + shared.block_size = 0x20000; + shared.counter = address + 0x10000; + shared.maximum_size = 0x3060; + shared.normal_maximum_size = 0x1020; + shared.discard = 0x180000; + self.control = initdata::FreeListControl::new(&initdata::FreeListArgs { + buffer_slot: 0, + class: initdata::FreeListClass::Two, + page_list_va: page_list, + run_list_va: operand_table, + blocks: 25, + state_va: address + 0x28000, + }); + self.control.run_cursor = 0xe0; + self.submitted = 1; + self.scheduler.init(); + Ok(()) + } +} + +/// The viewport words consumed from the deflake page at +0x900. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) struct Viewport { + unk_0: u32, + tiles_x: u32, + tiles_y: u32, + unk_c: u32, + scale_x: u32, + translate_x: u32, + scale_y: u32, + translate_y: u32, + unk_20: u32, + scale_z: u32, +} +impl Viewport { + /// Refresh only defined words; firmware-owned gaps retain their contents. + pub(crate) fn apply(&self, destination: &mut Self) { + destination.unk_0 = self.unk_0; + destination.tiles_x = self.tiles_x; + destination.tiles_y = self.tiles_y; + destination.scale_x = self.scale_x; + destination.translate_x = self.translate_x; + destination.scale_y = self.scale_y; + destination.translate_y = self.translate_y; + destination.scale_z = self.scale_z; + } + pub(crate) fn new(width: u32, height: u32) -> Result { + fn half(value: u32) -> Result { + if value == 0 { + return Err(EINVAL); + } + let highest = 31 - value.leading_zeros(); + let shift = 23u32.checked_sub(highest).ok_or(EINVAL)?; + Ok(((highest + 126) << 23) | ((value << shift) & 0x7f_ffff)) + } + let x = half(width)?; + let y = half(height)?; + Ok(Self { + unk_0: 0xc00, + tiles_x: 0x8000_0000 | (width.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1), + tiles_y: height.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1, + unk_c: 0, + scale_x: x, + translate_x: x, + scale_y: y, + translate_y: y | 0x8000_0000, + unk_20: 0, + scale_z: 0x3f80_0000, + }) + } +} + +const _: () = { + assert!(core::mem::size_of::() == 0x100); + assert!(core::mem::size_of::() == SUPPORT_SIZE); + assert!(core::mem::offset_of!(Support, events) == 0x14000); + assert!(core::mem::offset_of!(Support, scenes) == 0x18000); + assert!(core::mem::offset_of!(Support, shared) == 0x1c000); + assert!(core::mem::offset_of!(Support, control) == 0x24000); + assert!(core::mem::offset_of!(Support, scheduler) == 0x2c000); + assert!(core::mem::size_of::() == 0x28); +}; diff --git a/drivers/gpu/drm/asahi/g17/hazard.rs b/drivers/gpu/drm/asahi/g17/hazard.rs new file mode 100644 index 00000000000000..20daaab85b8325 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/hazard.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Attachment overlap ordering across physical render pairs and shared images. +//! Tile stores can overlap even when the application's pixel regions do not. +//! Keep independent attachments concurrent; order only conflicting live spans. + +use super::job::Order; + +const MAX_SPANS: usize = 24; +pub(crate) const WAIT_MS: u32 = 1000; + +#[derive(Clone, Copy)] +struct Span { + key: u64, + start: u64, + end: u64, +} + +/// Up to sixteen attachment ranges and eight shared-object ranges. +pub(crate) struct Footprint { + vm: u64, + spans: [Span; MAX_SPANS], + count: usize, + shared: bool, +} + +impl Footprint { + pub(crate) fn new(vm: u64) -> Self { + Self { + vm, + spans: [Span { + key: 0, + start: 0, + end: 0, + }; MAX_SPANS], + count: 0, + shared: false, + } + } + + fn push(&mut self, key: u64, start: u64, end: u64) { + if start < end && self.count < MAX_SPANS { + self.spans[self.count] = Span { key, start, end }; + self.count += 1; + } + } + + pub(crate) fn push_vm(&mut self, start: u64, end: u64) { + self.push(self.vm << 1, start, end); + } + + /// An aligned GEM object address is an identity, never dereferenced here. + pub(crate) fn push_object(&mut self, object: usize, start: u64, end: u64) { + let before = self.count; + self.push(object as u64 | 1, start, end); + self.shared |= self.count != before; + } + + fn overlaps(&self, other: &Self) -> bool { + if self.vm != other.vm && !(self.shared && other.shared) { + return false; + } + self.spans[..self.count].iter().any(|a| { + other.spans[..other.count] + .iter() + .any(|b| a.key == b.key && a.start < b.end && b.start < a.end) + }) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Plan { + Any, + Pair(u8), + Wait, +} + +pub(crate) struct Live<'a> { + pub(crate) owner: u64, + pub(crate) sequence: u64, + pub(crate) slot: Option, + pub(crate) ticket: u64, + pub(crate) footprint: &'a Footprint, + pub(crate) quarantined: bool, +} + +/// A queue prefix already orders covered predecessors. Pending arrivals order +/// different owners fairly without imposing order on unrelated attachments. +pub(crate) fn plan<'a>( + owner: u64, + prefix: Option, + ticket: Option, + footprint: &Footprint, + live: impl IntoIterator>, +) -> Plan { + let mut selected = None; + for entry in live { + if entry.quarantined || !footprint.overlaps(entry.footprint) { + continue; + } + let Some(slot) = entry.slot else { + if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { + return Plan::Wait; + } + continue; + }; + if entry.owner != owner { + return Plan::Wait; + } + if Order::contains(prefix, entry.sequence) { + continue; + } + match selected { + None => selected = Some(slot), + Some(prior) if prior == slot => {} + Some(_) => return Plan::Wait, + } + } + selected.map_or(Plan::Any, Plan::Pair) +} diff --git a/drivers/gpu/drm/asahi/g17/ids.rs b/drivers/gpu/drm/asahi/g17/ids.rs new file mode 100644 index 00000000000000..6912c6501527ac --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/ids.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware-visible object generations shared by contexts, pools and render work. + +use super::fw::ObjectIds; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; + +/// Module-lifetime identities remain unique across retained graphs and device rebinds. +pub(crate) static OBJECTS: Counter = Counter::new(); + +pub(crate) struct Counter(AtomicU32); + +impl Counter { + const fn new() -> Self { + Self(AtomicU32::new(1)) + } + + fn reserve(&self, count: u32) -> Result { + let mut old = self.0.load(Ordering::Acquire); + loop { + let next = old.checked_add(count).ok_or(EOVERFLOW)?; + match self + .0 + .compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => return Ok(old), + Err(current) => old = current, + } + } + } + + pub(crate) fn object(&self) -> Result { + let id = self.reserve(1)?; + if id == 0 { + Err(EOVERFLOW) + } else { + Ok(id) + } + } + + /// Allocate consecutive current IDs atomically, fragment first and tiling + /// second. Both name the same retained predecessor of their render graph. + pub(crate) fn render_pair(&self, predecessor: u32) -> Result<(ObjectIds, ObjectIds)> { + let fragment = self.reserve(2)?; + if predecessor == 0 || fragment == 0 { + return Err(EOVERFLOW); + } + Ok(( + ObjectIds { + predecessor, + current: fragment, + }, + ObjectIds { + predecessor, + current: fragment + 1, + }, + )) + } +} diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs new file mode 100644 index 00000000000000..f8160f0faa36c8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -0,0 +1,1305 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Allocates, initializes and retains the graph shared by the two firmware instances. +//! +//! The owner is installed in the firmware session before construction starts. Even if an +//! allocation fails, the session stops both coprocessors before releasing any published memory. + +use super::{ + fw::initdata as abi, + object::{CpuMap, KernelObject, Placement}, + Config, Role, +}; +use crate::{driver::AsahiDevice, hw::t8140, mmu}; +use core::sync::atomic::{fence, AtomicU32, Ordering}; +use kernel::{c_str, prelude::*}; +use t8140::{kernel_window as high, low}; + +const EVENT_RING_RECHECK: u32 = 256; + +const INDEX_SIZE: usize = 0x10000; +const CONTEXT_SIZE: usize = 0x20000; +const REGION_A_SIZE: usize = core::mem::size_of::(); +const REGION_C_SIZE: usize = 0x1000; + +const COMPUTE_BLOCKS_ALLOCATED: usize = 23; +const COMPUTE_BLOCKS_MAPPED: usize = 22; +const COMPUTE_BLOCKS_POPULATED: usize = 21; +const COMPUTE_CONTROL_BLOCKS: u32 = 25; +const COMPUTE_CONTROL_CURSOR: u32 = 0x18; +const COMPUTE_RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; + +/// Unpublished device-wide compute backing, prepared without the device mutex. The cache +/// identity belongs to this allocation, not to any one physical queue or client VM. +pub(super) struct ComputeShared { + id: u64, + pages: KernelObject, + runs: KernelObject, + blocks: KVec, +} + +impl ComputeShared { + pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { + let pages = + KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; + let runs = + KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; + let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_ALLOCATED, GFP_KERNEL)?; + for _ in 0..COMPUTE_BLOCKS_ALLOCATED { + blocks.push( + KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, + GFP_KERNEL, + )?; + } + let id = u64::from(super::ids::OBJECTS.object()?); + Ok(Self { + id, + pages, + runs, + blocks, + }) + } +} + +/// A device-global scheduler, retained independently of whichever physical queue first +/// needed it. Its generation reservation precedes later graph allocation failures. +pub(super) struct ComputeScheduler { + object: KernelObject, +} + +impl ComputeScheduler { + pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { + let mut object = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + object.initialize::(0, |state| { + state.init(); + Ok(()) + })?; + fence(Ordering::SeqCst); + // Scheduler creation reserves a generation even when only its address is used by work. + super::ids::OBJECTS.object()?; + Ok(Self { object }) + } +} + +/// Checked views used only within one publication call. Every fallible operation completes +/// before either member of a paired render publication is copied into shared memory. +struct WorkPublication<'a> { + record: &'a super::fw::channels::WorkSlot, + destination: *mut u8, + producer: &'a AtomicU32, + next: u32, +} + +impl WorkPublication<'_> { + fn copy_record(&self) { + // SAFETY: work_publication checked the complete destination range. The record is a + // fully initialized padding-free ABI value, and the device mutex owns publication. + unsafe { + core::ptr::copy_nonoverlapping( + self.record as *const _ as *const u8, + self.destination, + core::mem::size_of_val(self.record), + ); + } + } + + fn commit(self) -> u32 { + self.copy_record(); + fence(Ordering::SeqCst); + self.producer.store(self.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.next + } +} + +/// Session-long storage and mappings. Aliases are released before their backing objects. +pub(super) struct InitData { + // Live publication uses interior writes under the device mutex. This owner is Send but not + // Sync, so safe host code cannot issue those writes concurrently through shared references. + _single_publisher: core::marker::PhantomData>, + aliases: KVec, + objects: KVec, + roots: usize, + cluster: usize, + bundle: usize, + region_c: usize, + qos: usize, + pb: usize, + page_pool: usize, + compute_global: usize, + boot_control: usize, + boot_state: usize, + metrics_fw_va: u64, + compute_shared: Option, + compute_scheduler: Option, + compute_active: core::cell::Cell, + event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], + completion_rings: [usize; 2], + stamps: [usize; 2], + render_pages: usize, + render_runs: usize, + render_blocks: core::ops::Range, +} + +impl InitData { + /// Empty construction state, retained even if graph construction fails. + pub(super) fn new() -> Self { + Self { + _single_publisher: core::marker::PhantomData, + aliases: KVec::new(), + objects: KVec::new(), + roots: 0, + cluster: 0, + bundle: 0, + region_c: 0, + qos: 0, + pb: 0, + page_pool: 0, + compute_global: 0, + boot_control: 0, + boot_state: 0, + metrics_fw_va: 0, + compute_shared: None, + compute_scheduler: None, + compute_active: core::cell::Cell::new(false), + event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], + completion_rings: [0; 2], + stamps: [0; 2], + render_pages: 0, + render_runs: 0, + render_blocks: 0..0, + } + } + + pub(super) fn metrics_fw_va(&self) -> Result { + if self.metrics_fw_va == 0 { + return Err(EIO); + } + Ok(self.metrics_fw_va) + } + + pub(super) fn pb_descriptor_table(&self) -> Result<&KernelObject> { + self.object(self.pb) + } + + pub(super) fn page_pool_descriptor_table(&self) -> Result<&KernelObject> { + self.object(self.page_pool) + } + + pub(super) fn stamp_addresses(&self, qid: u8) -> Result<[u64; 2]> { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + let offset = u64::from(qid) * core::mem::size_of::() as u64; + Ok([ + self.object(self.stamps[0])?.gpu_va() + offset, + self.object(self.stamps[1])?.gpu_va() + offset, + ]) + } + + /// The caller reads both engine lanes before its acquire fence. Page A is the completed + /// GPU-maintenance witness; page B stays under firmware ownership. + pub(super) fn stamp(&self, qid: u8) -> Result<&AtomicU32> { + if qid > super::fw::kick::QID_MAX { + return Err(EINVAL); + } + self.object(self.stamps[0])? + .word(usize::from(qid) * core::mem::size_of::()) + } + + /// Initialize a fresh physical compute QID's row before exposing its queue. An installed + /// or rebound queue keeps the row's firmware-owned counters and must never call this again. + pub(super) fn initialize_compute_pool(&self, slot: u16, page_list_va: u64) -> Result { + if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { + return Err(EINVAL); + } + let entry = abi::PagePoolDescriptor::compute(page_list_va); + let size = core::mem::size_of::(); + let offset = usize::from(slot) * size; + let object = self.object(self.page_pool)?; + let pointer = object.pointer(offset, size)?; + let page_list = object.dword(offset)?; + let counters = + object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, counters))?; + let count = + object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, page_count))?; + // SAFETY: the whole fresh row was bounds checked, the QID has never been published, + // and the device mutex excludes another host publisher. No other row is touched. + unsafe { + pointer.write_bytes(0, size); + } + page_list.store(entry.page_list, Ordering::Relaxed); + counters.store(entry.counters, Ordering::Relaxed); + count.store(entry.page_count, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + pub(super) fn qos(&self) -> Result<&KernelObject> { + self.object(self.qos) + } + + pub(super) fn has_compute_scheduler(&self) -> bool { + self.compute_scheduler.is_some() + } + + /// Only the reserved singleton builder may install a scheduler. Retain it before building + /// the first descriptor graph so later construction failure does not allocate it again. + pub(super) fn install_compute_scheduler(&mut self, scheduler: ComputeScheduler) -> Result { + if self.compute_scheduler.is_some() { + return Err(EBUSY); + } + self.compute_scheduler = Some(scheduler); + Ok(()) + } + + pub(super) fn has_compute_shared(&self) -> bool { + self.compute_shared.is_some() + } + + /// Install a prepared aperture once while holding the device mutex. A competing preparer + /// may have installed its allocation first; the redundant unpublished backing then drops. + pub(super) fn install_compute_shared(&mut self, mut shared: ComputeShared) -> Result { + if self.compute_shared.is_some() { + return Ok(()); + } + let state_va = self.object(self.boot_state)?.gpu_va(); + let mut control = abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: u32::MAX, + class: abi::FreeListClass::Two, + page_list_va: low::COMPUTE_POOL_PAGE_LIST, + run_list_va: low::COMPUTE_POOL_RUN_LIST, + blocks: COMPUTE_CONTROL_BLOCKS, + state_va, + }); + control.run_cursor = COMPUTE_CONTROL_CURSOR; + self.object_mut(self.boot_control)?.bytes_mut().fill(0); + self.object_mut(self.boot_control)?.write(0, control)?; + self.object_mut(self.boot_state)?.bytes_mut().fill(0); + shared + .pages + .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |pages| { + abi::fill_page_list(pages, (0..COMPUTE_BLOCKS_POPULATED).map(low::compute_block)) + })?; + shared + .runs + .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { + for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { + *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) + | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); + } + Ok(()) + })?; + fence(Ordering::SeqCst); + self.compute_shared = Some(shared); + Ok(()) + } + + pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { + let shared = self.compute_shared.as_ref().ok_or(EIO)?; + if vm.compute_pool_mappings_match(shared.id) { + return Ok(()); + } + vm.clear_compute_pool_mappings(); + let mut mappings = KVec::with_capacity(26, GFP_KERNEL)?; + mappings.push( + shared + .pages + .map_alias(vm, low::COMPUTE_POOL_PAGE_LIST, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + mappings.push( + shared + .runs + .map_alias(vm, low::COMPUTE_POOL_RUN_LIST, mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + for (index, block) in shared.blocks.iter().take(COMPUTE_BLOCKS_MAPPED).enumerate() { + mappings.push( + block.map_alias(vm, low::compute_block(index), mmu::PROT_GPU_SHARED_RW)?, + GFP_KERNEL, + )?; + } + for slot in 0..2 { + mappings.push( + self.object(self.completion_rings[slot])?.map_alias( + vm, + low::COMPLETION_RINGS[slot], + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + } + vm.install_compute_pool_mappings(shared.id, mappings) + } + + /// The descriptor-table views remain immutable for the firmware session. + pub(super) fn validate_descriptor_views(&self) -> Result { + let values = [ + low::PB_DESCRIPTORS, + self.object(self.pb)?.gpu_va(), + low::PAGE_POOL_DESCRIPTORS, + self.object(self.page_pool)?.gpu_va(), + ]; + let base = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + core::mem::offset_of!(abi::MainConfig, pb_descriptors_low_va); + let bundle = self.object(self.bundle)?; + let mut changed = false; + for (index, value) in values.into_iter().enumerate() { + changed |= bundle.dword(base + index * 8)?.load(Ordering::Relaxed) != value; + } + fence(Ordering::SeqCst); + if changed { + Err(EIO) + } else { + Ok(()) + } + } + + pub(super) fn activate_compute(&self) -> Result { + self.validate_descriptor_views()?; + if self.compute_active.get() { + return Ok(()); + } + let scheduler = self.compute_scheduler.as_ref().ok_or(EIO)?.object.gpu_va(); + let object = self.object(self.compute_global)?; + let state = object + .pointer(0, core::mem::size_of::())? + .cast::(); + // SAFETY: The complete typed view is in the retained page-aligned object. InitData is + // not Sync and its caller holds the sole publication mutex. Each named field is + // naturally aligned and written individually; no reference aliases firmware state. + unsafe { + core::ptr::addr_of_mut!((*state).unk_0).write_volatile(0xff); + core::ptr::addr_of_mut!((*state).unk_8).write_volatile(0xff); + core::ptr::addr_of_mut!((*state).unk_10).write_volatile(0x0f01); + core::ptr::addr_of_mut!((*state).unk_18).write_volatile(0x0f01); + core::ptr::addr_of_mut!((*state).unk_408).write_volatile(2); + core::ptr::addr_of_mut!((*state).unk_40c).write_volatile(2); + core::ptr::addr_of_mut!((*state).unk_604).write_volatile(4); + core::ptr::addr_of_mut!((*state).scheduler_va).write_volatile(scheduler); + core::ptr::addr_of_mut!((*state).unk_c04).write_volatile(0x1000); + core::ptr::addr_of_mut!((*state).unk_e00).write_volatile(7); + core::ptr::addr_of_mut!((*state).unk_e08).write_volatile(0x0000_0001_0e22_618a); + core::ptr::addr_of_mut!((*state).unk_e20).write_volatile(4); + core::ptr::addr_of_mut!((*state).unk_e28).write_volatile(4); + } + self.compute_active.set(true); + Ok(()) + } + + fn work_publication<'a>( + &'a self, + record: &'a super::fw::channels::WorkSlot, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + let engine = match record.data_master { + 0 => DataMaster::Tiling, + 1 => DataMaster::Fragment, + 2 => DataMaster::Compute, + _ => return Err(EINVAL), + }; + let channel = channels::work_ring_index(engine); + let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let next = (producer + 1) % channels::WORK_RING_SLOTS; + if next == consumer || next == consumer_b { + return Err(EINVAL); + } + let destination = self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), + core::mem::size_of_val(record), + )?; + let producer = self.object(self.cluster)?.word( + Self::state_offset(Role::Primary) + + abi::work_state_offset(channel) + + abi::ChannelState::PRODUCER, + )?; + Ok(WorkPublication { + record, + destination, + producer, + next, + }) + } + + /// Two firmware consumers and the host producer, sampled in that order. + pub(super) fn work_cursors(&self, engine: super::fw::queue::DataMaster) -> Result<[u32; 3]> { + use super::fw::channels; + let channel = channels::work_ring_index(engine); + let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); + let state = self.object(self.cluster)?; + let mut values = [0; 3]; + for (value, offset) in values.iter_mut().zip([ + abi::ChannelState::CONSUMER, + abi::ChannelState::UNK_10, + abi::ChannelState::PRODUCER, + ]) { + *value = state.word(base + offset)?.load(Ordering::Relaxed); + } + if values + .iter() + .any(|&value| value >= channels::WORK_RING_SLOTS) + { + return Err(EINVAL); + } + Ok(values) + } + + /// Inner queue records must be visible before this outer publication. The returned + /// producer is the retained consumer witness for this slot. + pub(super) fn publish_work(&self, record: &super::fw::channels::WorkSlot) -> Result { + Ok(self.work_publication(record)?.commit()) + } + + /// Validate both destinations before exposing fragment, then tiling. The accounting token + /// is committed after copying fragment; late tiling state is published before its slot. + /// Deferred pairs retain both slot bodies without exposing either producer. + pub(super) fn publish_work_pair( + &self, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + deferred: bool, + commit_accounting: impl FnOnce() -> Result, + publish_late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + use super::fw::queue::DataMaster; + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let fragment = self.work_publication(fragment)?; + let tiling = self.work_publication(tiling)?; + fragment.copy_record(); + commit_accounting()?; + fence(Ordering::SeqCst); + if !deferred { + fragment.producer.store(fragment.next, Ordering::Relaxed); + } + publish_late_tiling(); + tiling.copy_record(); + fence(Ordering::SeqCst); + if !deferred { + tiling.producer.store(tiling.next, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok([fragment.next, tiling.next]) + } + + fn retained_work_publication<'a>( + &'a self, + record: &'a super::fw::channels::WorkSlot, + next: u32, + ) -> Result> { + use super::fw::{channels, queue::DataMaster}; + let engine = match record.data_master { + 0 => DataMaster::Tiling, + 1 => DataMaster::Fragment, + _ => return Err(EINVAL), + }; + let channel = channels::work_ring_index(engine); + let producer = self.object(self.cluster)?.word( + Self::state_offset(Role::Primary) + + abi::work_state_offset(channel) + + abi::ChannelState::PRODUCER, + )?; + let current = producer.load(Ordering::Relaxed); + if current >= channels::WORK_RING_SLOTS || next != (current + 1) % channels::WORK_RING_SLOTS + { + return Err(EBUSY); + } + let destination = self.object(self.bundle)?.pointer( + abi::work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), + core::mem::size_of_val(record), + )?; + Ok(WorkPublication { + record, + destination, + producer, + next, + }) + } + + /// Release a retained pair after its control transaction. Revalidate the original producer + /// witnesses before either copy; both bodies precede the fragment and tiling publications. + pub(super) fn publish_retained_work_pair( + &self, + fragment: &super::fw::channels::WorkSlot, + tiling: &super::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + use super::fw::queue::DataMaster; + if fragment.data_master != DataMaster::Fragment as u32 + || tiling.data_master != DataMaster::Tiling as u32 + { + return Err(EINVAL); + } + let fragment = self.retained_work_publication(fragment, next[0])?; + let tiling = self.retained_work_publication(tiling, next[1])?; + fragment.copy_record(); + tiling.copy_record(); + fence(Ordering::SeqCst); + fragment.producer.store(fragment.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + tiling.producer.store(tiling.next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + /// Snapshot the shared compute completion ring once per service transaction. The queue + /// scanner latches exact identities; firmware retains sole ownership of these slots. + pub(super) fn scan_compute_completions( + &self, + mut record: impl FnMut([u64; 8]) -> Result, + ) -> Result { + let object = self.object(self.completion_rings[0])?; + fence(Ordering::Acquire); + for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { + let mut words = [0; 8]; + for (index, word) in words.iter_mut().enumerate() { + *word = object + .dword(slot * 0x40 + index * 8)? + .load(Ordering::Relaxed); + } + record(words)?; + } + Ok(()) + } + + fn object(&self, index: usize) -> Result<&KernelObject> { + self.objects.get(index).ok_or(EINVAL) + } + + fn object_mut(&mut self, index: usize) -> Result<&mut KernelObject> { + self.objects.get_mut(index).ok_or(EINVAL) + } + + fn add(&mut self, object: KernelObject) -> Result { + let index = self.objects.len(); + self.objects.push(object, GFP_KERNEL)?; + Ok(index) + } + + fn alias(&mut self, index: usize, vm: &mmu::Vm, va: u64, prot: mmu::Prot) -> Result { + let alias = self.object_mut(index)?.map_alias(vm, va, prot)?; + self.aliases.push(alias, GFP_KERNEL)?; + Ok(()) + } + + /// Constructs the graph after the firmware has published the high-root entries. + pub(super) fn build( + &mut self, + dev: &AsahiDevice, + cfg: &Config, + uat: &mmu::Uat, + render: &mmu::Vm, + metrics: &KernelObject, + ) -> Result { + let vm = uat.kernel_vm(); + let lower = uat.kernel_lower_vm(); + let range = uat.geometry().kernel_range(); + let base = range.start; + let dynamic = base + t8140::dynamic::KERNEL_OFFSET..range.end; + let fixed = |offset, size, prot, cpu| { + KernelObject::new(dev, vm, Placement::At(base + offset), size, prot, cpu) + }; + let dynamic_object = |size, align, prot, cpu| { + KernelObject::new( + dev, + vm, + Placement::In(dynamic.clone(), align), + size, + prot, + cpu, + ) + }; + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, + PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + + for window in &t8140::REGISTER_WINDOWS { + let (phys, va, size) = window.mapping().ok_or(EINVAL)?; + self.aliases.push( + vm.map_io(va, phys.try_into()?, size.try_into()?, mmu::PROT_FW_MMIO_RW)?, + GFP_KERNEL, + )?; + } + self.cluster = self.add(fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.compute_global = self.add(fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + let pb = self.add(fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + self.object_mut(pb)? + .initialize::(0, |table| { + table.init(); + Ok(()) + })?; + let page_pool = self.add(fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + self.object_mut(page_pool)? + .initialize::(0, |table| { + table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); + Ok(()) + })?; + self.pb = pb; + self.page_pool = page_pool; + self.alias(pb, lower, low::PB_DESCRIPTORS, GS)?; + self.alias(page_pool, lower, low::PAGE_POOL_DESCRIPTORS, GS)?; + let index = self.add(fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; + self.object_mut(index)? + .initialize::(0, |table| { + table.init(); + Ok(()) + })?; + self.alias(index, lower, low::INDEX, GFS)?; + let fwctl = self.add(fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + + for (offset, state, class, prot, initial) in [ + ( + high::RENDER_FREE_LIST, + high::RENDER_FREE_LIST_STATE, + abi::FreeListClass::One, + FP, + abi::FreeListState::RENDER, + ), + ( + high::BOOT_FREE_LIST, + high::BOOT_FREE_LIST_STATE, + abi::FreeListClass::Two, + GFP, + abi::FreeListState::BOOT, + ), + ] { + let control = self.add(fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; + self.object_mut(control)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 0, + class, + page_list_va: low::RENDER_PAGE_LIST, + run_list_va: low::RENDER_RUN_LIST, + blocks: low::RENDER_BLOCKS_INITIAL, + state_va: base + state, + }), + )?; + let state_prot = if class == abi::FreeListClass::One { + FS + } else { + GFS + }; + let state = self.add(fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; + self.object_mut(state)?.write(0, initial)?; + if class == abi::FreeListClass::Two { + self.boot_control = control; + self.boot_state = state; + } + } + self.stamps[0] = self.add(fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; + self.stamps[1] = self.add(fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; + self.add(fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.add(fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + + let pages = self.add(KernelObject::backing(dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + self.object_mut(pages)? + .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { + abi::fill_page_list( + list, + (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), + ) + })?; + let runs = self.add(KernelObject::backing(dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let first_block = self.objects.len(); + for _ in 0..low::RENDER_BLOCKS_ALLOCATED { + self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + } + self.render_pages = pages; + self.render_runs = runs; + self.render_blocks = first_block..self.objects.len(); + self.alias(pages, render, low::RENDER_PAGE_LIST, GS)?; + self.alias(runs, render, low::RENDER_RUN_LIST, GS)?; + for block in 0..low::RENDER_BLOCKS_MAPPED { + self.alias(first_block + block, render, low::render_block(block), GS)?; + } + + let compute = self.add(fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + self.object_mut(compute)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 0, + class: abi::FreeListClass::One, + page_list_va: low::COMPUTE_PAGE_LIST, + run_list_va: low::COMPUTE_RUN_LIST, + blocks: low::COMPUTE_BLOCKS, + state_va: base + high::COMPUTE_FREE_LIST_STATE, + }), + )?; + let compute_state = self.add(KernelObject::backing(dev, mmu::UAT_PGSZ, WB)?)?; + self.object_mut(compute_state)? + .write(0, abi::FreeListState::COMPUTE)?; + for _ in &low::COMPUTE_ZERO_BLOCKS { + self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + } + let compute_runs = self.add(KernelObject::new( + dev, + lower, + Placement::At(low::COMPUTE_RUN_LIST), + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + self.object_mut(compute_runs)? + .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { + abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) + })?; + let ready = self.add(fixed( + high::COMPUTE_READY_FREE_LIST, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + self.object_mut(ready)?.write( + 0, + abi::FreeListControl::new(&abi::FreeListArgs { + buffer_slot: 1, + class: abi::FreeListClass::One, + page_list_va: low::COMPUTE_POOL_PAGE_LIST, + run_list_va: low::COMPUTE_POOL_RUN_LIST, + blocks: low::COMPUTE_READY_BLOCKS, + state_va: base + high::COMPUTE_READY_FREE_LIST_STATE, + }), + )?; + for state in [high::COMPUTE_READY_FREE_LIST_STATE, high::COMPUTE_STATE] { + let obj = self.add(fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; + self.object_mut(obj)? + .write(0, abi::FreeListState::COMPUTE)?; + } + self.object_mut(first_block + low::COMPUTE_PAGE_LIST_BLOCK)? + .initialize::<[u64; abi::FREE_LIST_BLOCK_SIZE / 8]>(0, |list| { + abi::fill_page_list(list, low::COMPUTE_FREE_LIST_BLOCKS) + })?; + fence(Ordering::SeqCst); + + self.bundle = self.add(fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; + let zero = self.add(fixed( + high::BUNDLE + abi::bundle::ZERO_PAGE as u64, + mmu::UAT_PGSZ, + GFS, + WC, + )?)?; + let mut completion_rings = [abi::CompletionRing { + low_va: 0, + high_va: 0, + }; 2]; + for slot in 0..2 { + let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; + let ring = self.add(fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; + self.completion_rings[slot] = ring; + self.alias(ring, lower, low::COMPLETION_RINGS[slot], GS)?; + completion_rings[slot] = abi::CompletionRing { + low_va: low::COMPLETION_RINGS[slot], + high_va: base + offset, + }; + } + let qos = self.add(dynamic_object(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; + self.qos = qos; + fence(Ordering::SeqCst); + let metrics_alias = + metrics.alias_in(vm, dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + self.metrics_fw_va = metrics_alias.iova(); + self.aliases.push(metrics_alias, GFP_KERNEL)?; + fence(Ordering::SeqCst); + self.roots = self.add(dynamic_object( + abi::ROOTS_SIZE, + abi::SECONDARY_ROOT_OFFSET as u64, + mmu::PROT_FW_SHARED_RO, + WB, + )?)?; + let region_a = self.add(dynamic_object( + REGION_A_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RO, + WB, + )?)?; + self.region_c = self.add(dynamic_object(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + + let opps = Self::opps(dev)?; + let hw_args = abi::HwDataArgs { + chip_id: cfg.chip_id, + windows: &t8140::REGISTER_WINDOWS, + reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + opps: &opps, + perf: &t8140::PERF, + completion_rings, + qos_va: self.object(qos)?.gpu_va(), + }; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; + self.object_mut(self.region_c)? + .initialize::(0, |region| { + region.init(); + Ok(()) + })?; + self.object_mut(self.cluster)? + .write(abi::private::PRIMARY_STATUS_A, abi::StatusBlock::new())?; + self.object_mut(self.cluster)? + .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; + let fwctl_va = self.object(fwctl)?.gpu_va(); + self.object_mut(self.cluster)? + .initialize::(abi::private::STATUS_B, |status| { + status.init(fwctl_va); + Ok(()) + })?; + self.object_mut(self.cluster)? + .initialize::( + abi::private::SECONDARY_STATE + abi::private::STATUS_B, + |config| { + config.init(); + Ok(()) + }, + )?; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::POWER_CONFIG, |power| { + power.init(); + Ok(()) + })?; + let secondary_status = base + + high::PRIVATE_CLUSTER + + (abi::private::SECONDARY_STATE + abi::private::SECONDARY_STATUS) as u64; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::AUX, |aux| { + aux.init(secondary_status); + Ok(()) + })?; + self.object_mut(self.bundle)? + .initialize::(abi::bundle::POWER_CONFIG_COPY, |power| { + power.init(); + Ok(()) + })?; + let views = abi::PrimaryViews { + zero_page_va: self.object(zero)?.gpu_va(), + pb_descriptors_va: self.object(pb)?.gpu_va(), + page_pool_descriptors_va: self.object(page_pool)?.gpu_va(), + }; + for role in [Role::Primary, Role::Secondary] { + let slot = role as usize; + let main_offset = abi::bundle::MAIN_CONFIG[slot]; + let main_va = base + high::BUNDLE + main_offset as u64; + let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; + let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; + let args = abi::MainConfigArgs { + bundle_va: base + high::BUNDLE, + main_config_va: main_va, + state_grid_va: state_va, + status_a_va: status_va, + pb_descriptors_low_va: low::PB_DESCRIPTORS, + page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, + }; + self.object_mut(self.bundle)? + .initialize::(main_offset, |main| { + match role { + Role::Primary => main.init_primary(&args, &views), + Role::Secondary => main.init_secondary(&args), + }; + Ok(()) + })?; + let abi_role = match role { + Role::Primary => abi::Role::Primary, + Role::Secondary => abi::Role::Secondary, + }; + self.object_mut(self.bundle)?.write( + main_offset + abi::MainConfig::CONTROL_RING, + abi::OpeningRecord::new(abi_role), + )?; + let root = abi::RootArgs { + region_a_va: self.object(region_a)?.gpu_va(), + region_c_va: self.object(self.region_c)?.gpu_va(), + main_config_va: main_va, + status_a_va: status_va, + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + match role { + Role::Primary => self.object_mut(self.roots)?.write( + 0, + abi::Root::primary(&root, state_va + abi::private::STATUS_B as u64), + )?, + Role::Secondary => self.object_mut(self.roots)?.write( + abi::SECONDARY_ROOT_OFFSET, + abi::SecondaryRoot::new( + &root, + base + high::PRIVATE_CLUSTER + + (abi::private::STATUS_B + abi::PrimaryStatusB::POWER_STATE) as u64, + state_va + abi::private::STATUS_B as u64, + ), + )?, + } + } + for role in [Role::Primary, Role::Secondary] { + for (offset, value) in [ + (0, 0), + (0x10, 0), + (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), + ] { + self.object(self.cluster)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .store(value, Ordering::Relaxed); + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retained fixed aliases for an ordinary client VM. The final block of the render-global + /// pool overlaps completion mappings in a client root and is deliberately not installed. + pub(super) fn user_aliases( + &mut self, + vm: &mmu::Vm, + metrics: &KernelObject, + ) -> Result> { + let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; + mappings.push( + self.object_mut(self.render_pages)?.map_alias( + vm, + low::RENDER_PAGE_LIST, + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + mappings.push( + self.object_mut(self.render_runs)?.map_alias( + vm, + low::RENDER_RUN_LIST, + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + for (block, index) in self.render_blocks.clone().take(21).enumerate() { + mappings.push( + self.object_mut(index)?.map_alias( + vm, + low::render_block(block), + mmu::PROT_GPU_SHARED_RW, + )?, + GFP_KERNEL, + )?; + } + mappings.push( + metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + GFP_KERNEL, + )?; + Ok(mappings) + } + + /// Writes the effort input in firmware order, then cleans its cache line to coherency. + pub(super) fn publish_effort(&self, effort: u32) -> Result { + let region = self.object(self.region_c)?; + let override_word = region.word(abi::RegionC::UNK_E8)?; + let effort_word = region.word(abi::RegionC::EFFORT)?; + let line = region.pointer(abi::RegionC::EFFORT, 4)?; + override_word.store(0, Ordering::Relaxed); + effort_word.store(effort, Ordering::Relaxed); + // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache + // line and the following system barrier publish the two naturally aligned stores. + unsafe { + core::arch::asm!("dc cvac, {line}", "dsb sy", line = in(reg) line, + options(nostack, preserves_flags)) + }; + Ok(()) + } + + pub(super) fn effort(&self) -> Result { + Ok(self + .object(self.region_c)? + .word(abi::RegionC::EFFORT)? + .load(Ordering::Relaxed)) + } + + fn opps(dev: &AsahiDevice) -> Result> { + let node = dev.as_ref().of_node().ok_or(EIO)?; + let table = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; + let mut opps = KVec::new(); + for opp in table.children() { + opps.push( + abi::Opp { + freq_hz: opp.get_property(c_str!("opp-hz"))?, + volt_uv: opp.get_property(c_str!("opp-microvolt"))?, + }, + GFP_KERNEL, + )?; + } + if opps.len() != t8140::OPPS { + return Err(EINVAL); + } + Ok(opps) + } + + fn state_offset(role: Role) -> usize { + match role { + Role::Primary => abi::private::PRIMARY_STATE, + Role::Secondary => abi::private::SECONDARY_STATE, + } + } + + fn status_offset(role: Role) -> usize { + match role { + Role::Primary => abi::private::PRIMARY_STATUS_A, + Role::Secondary => abi::private::SECONDARY_STATUS_A, + } + } + + /// Recheck the host-authored event pointer pair after a bounded number of drains. + /// The device mutex serializes the cache and all event consumption. + pub(super) fn validate_event_ring(&self, role: Role) -> Result { + let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; + let expected = ( + status + abi::status::EVENT_STATE, + status + abi::status::EVENT_RING, + ); + let cache = &self.event_ring_checked[role as usize]; + let checked = cache.get(); + if checked.2 != 0 && (checked.0, checked.1) == expected { + cache.set((checked.0, checked.1, checked.2 - 1)); + return Ok(()); + } + let offset = abi::bundle::MAIN_CONFIG[role as usize] + + core::mem::offset_of!(abi::MainConfig, channels) + + abi::EVENT_CHANNEL * size_of::(); + let bundle = self.object(self.bundle)?; + let actual = ( + bundle.dword(offset)?.load(Ordering::Relaxed), + bundle + .dword(offset + size_of::())? + .load(Ordering::Relaxed), + ); + if actual != expected { + cache.set((0, 0, 0)); + return Err(EINVAL); + } + cache.set((actual.0, actual.1, EVENT_RING_RECHECK)); + Ok(()) + } + + pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { + self.object(self.cluster)?.word( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) + } + + pub(super) fn status_read_words( + &self, + role: Role, + offset: usize, + ) -> Result<[u32; N]> { + self.object(self.cluster)?.read_words( + Self::status_offset(role) + .checked_add(offset) + .ok_or(EINVAL)?, + ) + } + + pub(super) fn control_counters(&self) -> Result<[u32; 3]> { + let state = self.object(self.cluster)?; + let mut counters = [0; 3]; + fence(Ordering::Acquire); + for (counter, offset) in counters.iter_mut().zip([ + abi::ChannelState::CONSUMER, + abi::ChannelState::UNK_10, + abi::ChannelState::PRODUCER, + ]) { + *counter = state + .word(abi::CONTROL_STATE_OFFSET + offset)? + .load(Ordering::Relaxed); + } + Ok(counters) + } + + pub(super) fn control_consumer(&self) -> Result<&AtomicU32> { + self.object(self.cluster)? + .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) + } + + pub(super) fn control_producer(&self) -> Result<&AtomicU32> { + self.object(self.cluster)? + .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) + } + + /// Publishes a control record under the device lock. All checked views are acquired before + /// the first write; record and packed power tally precede the aligned producer store. + pub(super) fn publish_control( + &self, + slot: u32, + record: &[u8; 64], + asserts_power: bool, + next: u32, + ) -> Result { + let offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + abi::MainConfig::CONTROL_RING + + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; + let ring = self.object(self.bundle)?.pointer(offset, record.len())?; + let power = self + .object(self.region_c)? + .pointer(abi::RegionC::POWER_ASSERT_TALLY, 4)?; + let producer = self.control_producer()?; + // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. + // Firmware only reads records after the producer store and the device mutex serializes + // host publishers. The unaligned power word uses the interface's four-byte copy access. + unsafe { + core::ptr::copy_nonoverlapping(record.as_ptr(), ring, record.len()); + if asserts_power { + let mut tally = [0u8; 4]; + core::ptr::copy_nonoverlapping(power, tally.as_mut_ptr(), 4); + let value = u32::from_le_bytes(tally).wrapping_add(1).to_le_bytes(); + core::ptr::copy_nonoverlapping(value.as_ptr(), power, 4); + } + } + fence(Ordering::SeqCst); + producer.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + pub(super) fn root_va(&self, role: Role) -> Result { + Ok(self.object(self.roots)?.gpu_va() + role as u64 * abi::SECONDARY_ROOT_OFFSET as u64) + } + + pub(super) fn opening_retired(&self, role: Role) -> Result { + for offset in [0, 0x10, abi::ChannelState::PRODUCER] { + if self + .object(self.cluster)? + .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? + .load(Ordering::SeqCst) + != abi::OPENING_PRODUCER + { + return Ok(false); + } + } + Ok(true) + } +} + +impl InitData { + /// Validate the serialized view before reading its retained host mapping. + fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { + let bundle = self.object(self.bundle)?; + let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; + let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; + let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) + .ok_or(EOVERFLOW)?; + if end > extent { + return Err(EINVAL); + } + let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] + + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); + fence(Ordering::Acquire); + let pointer = bundle.read_words::<2>(pointer_at)?; + let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; + if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { + return Err(EINVAL); + } + bundle.read_words(base + offset) + } +} + +impl super::recovery::Memory for InitData { + fn fault_sources(&self) -> Result { + let mask = self.object(self.cluster)? + .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? + .load(Ordering::Relaxed); + let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { + let key = super::recovery::slot_key(slot).unwrap_or(0); + let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, + abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) + .map(|words| words[0]).unwrap_or(0); + (key, monitor) + }); + sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) + .map(|words| words[0]).unwrap_or(0); + Ok(sources) + } + + fn recovery_state(&self) -> Result { + fence(Ordering::Acquire); + Ok(self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? + .load(Ordering::Relaxed)) + } + + fn host_recovery(&self) -> Result { + Ok(self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? + .load(Ordering::Relaxed)) + } + + fn transition(&self, expected: u32, next: u32) -> Result { + let state = self + .object(self.cluster)? + .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; + fence(Ordering::Acquire); + if state.load(Ordering::Relaxed) != expected { + return Err(EBUSY); + } + fence(Ordering::SeqCst); + state.store(next, Ordering::Relaxed); + fence(Ordering::SeqCst); + Ok(()) + } + + fn clear_timestamps(&self) -> Result { + let bundle = self.object(self.bundle)?; + // Validate the complete table before publishing its first live word. + const ENTRIES: usize = 128; + const STRIDE: usize = 0x10; + let offset = abi::bundle::VIEWS[4]; + bundle.pointer(offset, ENTRIES * STRIDE)?; + for qid in 0..ENTRIES { + bundle + .word(offset + qid * STRIDE)? + .store(0, Ordering::Relaxed); + bundle + .dword(offset + qid * STRIDE + 8)? + .store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + fn entries( + &self, + out: &mut [super::recovery::Entry; super::recovery::MAX_ENTRIES], + ) -> Result { + let cluster = self.object(self.cluster)?; + let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; + fence(Ordering::Acquire); + let mut used = 0; + for index in 0..256 { + if used == out.len() { + break; + } + let offset = base + index * core::mem::size_of::(); + let flags = cluster.word(offset)?.load(Ordering::Relaxed); + if flags & 1 == 0 { + continue; + } + let stamp = cluster.word(offset + 4)?.load(Ordering::Relaxed); + if let Some(entry) = super::recovery::Entry::decode(flags, stamp) { + out[used] = entry; + used += 1; + } + } + Ok(used) + } + + fn wait_tick(&self) { + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs new file mode 100644 index 00000000000000..ed2e79537d597e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Immutable submissions and the DRM scheduler boundary for independent engines. + +use super::{command::Validated, completion::Completion, context::Context}; +use crate::driver::AsahiDevice; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, Ordering}; +use kernel::{ + bindings, c_str, + dma_fence::{Fence, RawDmaFence}, + drm::sched, + prelude::*, + sync::Arc, +}; + +const TIMEOUT_MS: kernel::time::Msecs = 2000; +const RENDER_CREDITS: u32 = 32; +const COMPUTE_CREDITS: u32 = 16; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Engine { + Render, + Compute, +} + +impl Engine { + pub(crate) fn index(self) -> usize { + match self { + Self::Render => 0, + Self::Compute => 1, + } + } +} + +/// Logical sequence numbers do not wrap with firmware timestamps. A zero barrier +/// includes preceding ioctls; the sentinel means independent of that engine. +#[derive(Clone, Copy)] +pub(crate) struct Order { + pub(crate) sequence: u64, + pub(crate) wait_through: [Option; 2], +} + +impl Order { + pub(crate) fn new( + base: [u64; 2], + preceding: [u16; 2], + barriers: [u16; 2], + engine: Engine, + ) -> Result { + let mut sequences = [0; 2]; + let mut wait_through = [None; 2]; + for index in 0..2 { + sequences[index] = base[index] + .checked_add(u64::from(preceding[index])) + .and_then(|value| value.checked_add(1)) + .ok_or(EOVERFLOW)?; + if barriers[index] != u16::MAX { + if barriers[index] > preceding[index] { + return Err(EINVAL); + } + wait_through[index] = Some( + base[index] + .checked_add(u64::from(barriers[index])) + .ok_or(EOVERFLOW)?, + ); + } + } + Ok(Self { + sequence: sequences[engine.index()], + wait_through, + }) + } + + pub(crate) fn contains(prefix: Option, sequence: u64) -> bool { + sequence != 0 && prefix.is_some_and(|last| sequence <= last) + } +} + +pub(crate) struct Packet { + pub(crate) id: u64, + pub(crate) context: Arc, + pub(crate) command: Validated, + pub(crate) order: Order, + pub(crate) completion: Arc, + pub(crate) footprint: Option>, + inputs: KVec, + firmware: KVec, + publication: AtomicU8, + cancelled: AtomicBool, + early: AtomicI32, + backlog: AtomicBool, +} + +impl Packet { + pub(crate) fn new( + id: u64, + context: Arc, + command: Validated, + order: Order, + completion: Arc, + inputs: KVec, + firmware: KVec, + ) -> Result> { + let footprint = match &command { + Validated::Render { attachments, .. } => { + context.vm().render_footprint(&attachments[1])? + } + Validated::Compute { .. } => None, + }; + Ok(Arc::new( + Self { + id, + context, + command, + order, + completion, + footprint, + inputs, + firmware, + publication: AtomicU8::new(0), + cancelled: AtomicBool::new(false), + early: AtomicI32::new(0), + backlog: AtomicBool::new(false), + }, + GFP_KERNEL, + )?) + } + + pub(crate) fn engine(&self) -> Engine { + match &self.command { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + } + } + + /// Scheduler dependency readiness includes failed fences. Consume the actual + /// status before publication, and retain the references through that check. + pub(crate) fn check_dependencies(&self) -> Result { + for fence in &self.inputs { + // SAFETY: The retained fence owns the pointer throughout the wait and read. + let waited = unsafe { + bindings::dma_fence_wait_timeout( + fence.raw(), + true, + kernel::task::MAX_SCHEDULE_TIMEOUT, + ) + }; + if waited < 0 { + return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); + } + if waited == 0 { + return Err(ECANCELED); + } + check_status(fence)?; + } + for fence in &self.firmware { + check_status(fence)?; + } + Ok(()) + } + + /// Dependency-bearing packets remain on the scheduler publication path. + pub(crate) fn has_dependencies(&self) -> bool { + !self.inputs.is_empty() || !self.firmware.is_empty() + } + + pub(crate) fn begin_publication(&self) -> bool { + self.publication + .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + pub(crate) fn mark_published(&self) { + self.publication.store(2, Ordering::Release); + } + pub(crate) fn is_published(&self) -> bool { + self.publication.load(Ordering::Acquire) == 2 + } + pub(crate) fn cancel_requested(&self) -> bool { + self.cancelled.load(Ordering::Acquire) + } + pub(crate) fn request_cancel(&self) -> bool { + self.cancelled.store(true, Ordering::Release); + self.publication + .compare_exchange(0, 3, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + pub(crate) fn set_early_result(&self, result: Result) { + self.early.store( + result.map_or_else(|error| error.to_errno(), |_| 1), + Ordering::Release, + ); + } + pub(crate) fn early_result(&self) -> Option { + match self.early.load(Ordering::Acquire) { + 0 => None, + 1 => Some(Ok(())), + error => Some(Err(Error::from_errno(error))), + } + } + pub(crate) fn count_backlog(&self) { + self.backlog.store(true, Ordering::Release); + } + pub(crate) fn take_backlog(&self) -> bool { + self.backlog.swap(false, Ordering::AcqRel) + } +} + +pub(crate) fn fence_status(fence: &Fence) -> i32 { + // SAFETY: The owned fence keeps the raw DMA fence alive for this read. + unsafe { bindings::dma_fence_get_status(fence.raw()) } +} + +fn check_status(fence: &Fence) -> Result { + let status = fence_status(fence); + if status < 0 { + Err(super::dependency::inherited_error(Error::from_errno(status))) + } else { + Ok(()) + } +} + +/// The backend retains visible packets through exact retirement or processor stop. +/// It performs all completion signalling after releasing the device mutex. +pub(crate) trait Backend: Send + Sync { + /// Lazily create or rebind this logical queue's physical compute graph. + fn ensure_compute(&self) -> Result; + /// Settle outstanding work of a terminal VM before an ioctl error is published. + fn fail_vm(&self, error: Error); + /// Called after both scheduler entities have cancelled their queued jobs. + fn release_owner(&self); + fn prepare(&self, packet: &Packet) -> Option; + fn publish(&self, packet: Arc) -> Result; + fn timed_out(&self, packet: Arc) -> sched::Status; + fn cancel(&self, packet: Arc); + /// Transfer a timeout to an outstanding off-lock graph transaction, if any. + fn defer_timeout(&self, packet: &Arc) -> bool; + /// Return true only after taking publication ownership and recording its result. + fn try_early(&self, packet: &Arc) -> bool; + fn count_backlog(&self, packet: &Packet); +} + +struct Job { + backend: Arc, + packet: Arc, +} + +impl sched::JobImpl for Job { + fn false_timeout(job: &mut sched::Job) -> bool { + job.is_finished() + || job.packet.completion.take_replay_timeout() + || job.backend.defer_timeout(&job.packet) + } + + fn prepare(job: &mut sched::Job) -> Option { + job.backend.prepare(&job.packet) + } + fn run(job: &mut sched::Job) -> Result> { + job.backend.publish(job.packet.clone()).map(Some) + } + fn timed_out(job: &mut sched::Job) -> sched::Status { + // The scheduler retains the detached job throughout this callback. Keep + // its parent callback intact while the driver settles the exact owner; + // reinsertion also handles completion racing with timeout observation. + if !job.is_finished() { + job.backend.timed_out(job.packet.clone()); + } + sched::Status::NoHang + } + fn cancel(job: &mut sched::Job) { + job.backend.cancel(job.packet.clone()); + } +} + +pub(crate) struct Scheduler { + // Cancel entity jobs before dropping the scheduler's owning reference. + entity: sched::Entity>, + _scheduler: sched::Scheduler>, + backend: Arc, +} + +#[derive(Clone)] +pub(crate) struct Fences { + pub(crate) ready: Fence, + pub(crate) completed: Fence, +} + +impl Scheduler { + pub(crate) fn new(dev: &AsahiDevice, backend: Arc, engine: Engine) -> Result { + let (credits, name) = match engine { + Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), + Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), + }; + let scheduler = sched::Scheduler::new(dev.as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; + let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; + Ok(Self { + entity, + _scheduler: scheduler, + backend, + }) + } + + pub(crate) fn enqueue( + &mut self, + packet: Arc, + dependencies: KVec, + early: bool, + ) -> Result { + let completed = packet.completion.fence(); + let host_timestamps = packet.completion.has_timestamps(); + let mut job = self.entity.new_job( + 1, + Job { + backend: self.backend.clone(), + packet: packet.clone(), + }, + )?; + for dependency in dependencies { + job.add_dependency(dependency)?; + } + let mut job = job.arm(); + let ready = if host_timestamps { + completed.clone() + } else { + job.fences().scheduled() + }; + // All fallible scheduler setup precedes early hardware publication. The + // scheduler still owns watchdog, credits and cancellation after push. + if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { + self.backend.count_backlog(&packet); + } + job.push(); + Ok(Fences { ready, completed }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs new file mode 100644 index 00000000000000..9d89262d8a3b68 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical kick-ring backing, queue identities and entry publication. +//! +//! The device mutex serializes identities and publication. Installed identities +//! and their ring backing stay owned until both firmware processors stop; +//! command completion alone does not detach a hardware queue. + +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +use super::{ + fw::{kick::*, queue::DataMaster}, + object::{Allocator, CpuMap, KernelObject}, +}; +use crate::{hw::t8140, mem, mmu}; + +const QID_COUNT: usize = QID_MAX as usize + 1; + +/// Stable physical-channel identity. The owner must not be reused within a +/// device lifetime, including after an unpublished allocation is cancelled. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Id { + qid: u8, + owner: u64, + engine: DataMaster, +} + +impl Id { + pub(crate) fn qid(self) -> u8 { + self.qid + } + pub(crate) fn engine(self) -> DataMaster { + self.engine + } +} + +#[derive(Copy, Clone)] +struct Identity { + id: Id, + published: bool, +} + +/// One shared namespace for tiling, fragment and compute queues. +pub(crate) struct QueueIds { + entries: [Option; QID_COUNT], +} + +impl QueueIds { + pub(crate) const fn new() -> Self { + Self { + entries: [None; QID_COUNT], + } + } + + /// Reuses only an exact owner/engine match. Preferred placement is for + /// bootstrapping; it does not reserve a partition for an engine. + pub(crate) fn reserve( + &mut self, + owner: u64, + engine: DataMaster, + preferred: Option, + ) -> Result { + if owner == 0 { + return Err(EINVAL); + } + if let Some(entry) = self + .entries + .iter() + .flatten() + .find(|entry| entry.id.owner == owner) + { + return if entry.id.engine == engine { + Ok(entry.id) + } else { + Err(EINVAL) + }; + } + let index = preferred + .map(usize::from) + .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) + .or_else(|| self.entries.iter().position(Option::is_none)) + .ok_or(ENOSPC)?; + let id = Id { + qid: index as u8, + owner, + engine, + }; + self.entries[index] = Some(Identity { + id, + published: false, + }); + Ok(id) + } + + /// Call after preflighting the complete installation, before any address + /// becomes firmware-visible. A failed or partial installation retains it. + pub(crate) fn publish(&mut self, id: Id) -> Result { + let entry = self + .entries + .get_mut(usize::from(id.qid)) + .and_then(Option::as_mut) + .ok_or(EINVAL)?; + if entry.id != id { + return Err(EINVAL); + } + entry.published = true; + Ok(()) + } + + pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { + let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; + match slot { + Some(entry) if entry.id == id && !entry.published => { + *slot = None; + Ok(()) + } + _ => Err(EINVAL), + } + } +} + +// Aliases drop before their backing. A compute queue is allocated low first; +// a render queue is allocated high first. +enum Backing { + Compute { + high: mmu::KernelMapping, + low: KernelObject, + }, + Render { + low: mmu::KernelMapping, + high: KernelObject, + }, +} + +/// An installed queue's permanent ring. Client aliases are held by the bound +/// context graph and must be dropped before this object. +pub(crate) struct Queue { + backing: Backing, + id: Id, + timestamp: KickTimestamp, + previous: Option, +} + +/// A checked entry and destination held without changing the shared ring. +/// Discarding it leaves both the slot and the next timestamp unchanged. +pub(crate) struct PreparedKick<'a> { + queue: &'a mut Queue, + timestamp: KickTimestamp, + entry: KickEntry, + destination: *mut KickEntry, +} + +impl PreparedKick<'_> { + /// Publish translation updates after the queue configuration pointer and before its entry. + pub(crate) fn prepare_compute_install(&self) -> Result { + self.queue.prepare_compute_install() + } + + /// Writes and fences the complete entry, then advances host state. The + /// caller invokes this between the queue configuration and announcement. + pub(crate) fn commit(self) -> KickTimestamp { + // SAFETY: prepare_entry checked the complete destination range and + // this token exclusively borrows the queue and retains its mapping. + // The caller only prepares retired ring slots; no firmware access to + // the new entry starts before the outer command is published. + unsafe { self.destination.write_unaligned(self.entry) }; + fence(Ordering::SeqCst); + self.queue.previous = Some(self.timestamp); + self.queue.timestamp = self.timestamp.next(); + self.timestamp + } +} + +impl Queue { + pub(crate) fn compute(alloc: &Allocator<'_>, id: Id) -> Result { + if id.engine != DataMaster::Compute { + return Err(EINVAL); + } + let low = alloc.lower( + KICK_RING_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut range = alloc.uat.geometry().kernel_range(); + range.start += t8140::dynamic::KERNEL_OFFSET; + let high = low.alias_in( + alloc.uat.kernel_vm(), + range, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + )?; + fence(Ordering::SeqCst); + Ok(Self { + backing: Backing::Compute { high, low }, + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }) + } + + pub(crate) fn render( + alloc: &Allocator<'_>, + id: Id, + client: &mmu::Vm, + ) -> Result<(Self, mmu::KernelMapping)> { + if id.engine == DataMaster::Compute { + return Err(EINVAL); + } + let high = alloc.kernel( + KICK_RING_SIZE, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let mut range = t8140::dynamic::LOWER; + let (low, client) = loop { + let low = high.alias_in( + alloc.uat.kernel_lower_vm(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + )?; + let end = low + .iova() + .checked_add(KICK_RING_SIZE as u64) + .ok_or(EOVERFLOW)?; + match high.map_alias(client, low.iova(), mmu::PROT_GPU_SHARED_RW) { + Ok(client) => break (low, client), + Err(error) if error == ENOSPC && end < range.end => { + drop(low); + range.start = end; + } + Err(error) => return Err(error), + } + }; + Ok(( + Self { + backing: Backing::Render { low, high }, + id, + timestamp: KickTimestamp::FIRST, + previous: None, + }, + client, + )) + } + + pub(crate) fn id(&self) -> Id { + self.id + } + pub(crate) fn low_va(&self) -> u64 { + match &self.backing { + Backing::Compute { low, .. } => low.gpu_va(), + Backing::Render { low, .. } => low.iova(), + } + } + pub(crate) fn firmware_va(&self) -> u64 { + match &self.backing { + Backing::Compute { high, .. } => high.iova(), + Backing::Render { high, .. } => high.gpu_va(), + } + } + fn object(&mut self) -> &mut KernelObject { + match &mut self.backing { + Backing::Compute { low, .. } => low, + Backing::Render { high, .. } => high, + } + } + pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { + let address = self.low_va(); + self.object() + .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) + } + + /// Reuse a permanent client view only when every page names this ring. + pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { + let object = match &self.backing { + Backing::Compute { low, .. } => low, + Backing::Render { high, .. } => high, + }; + object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) + } + + /// Both aliases are owned mappings of the same full ring. Publish their + /// page-table updates to each live translation regime before installation. + pub(crate) fn prepare_compute_install(&self) -> Result { + if self.id.engine != DataMaster::Compute { + return Err(EINVAL); + } + fence(Ordering::SeqCst); + for asid in 0..4 { + mem::tlbi_range_or_asid(asid, self.low_va() as usize, KICK_RING_SIZE); + mem::tlbi_range_or_asid(asid, self.firmware_va() as usize, KICK_RING_SIZE); + } + mem::sync(); + Ok(()) + } + + /// Detaches the implicit dependency after firmware recovery, retaining + /// the monotonically advancing 40-bit producer. + pub(crate) fn clear_parent_after_recovery(&mut self) { + self.previous = None; + } + + pub(crate) fn timestamp(&self) -> KickTimestamp { + self.timestamp + } + pub(crate) fn parent(&self) -> KickTimestamp { + self.previous.unwrap_or_else(|| match self.id.engine { + DataMaster::Compute => KickTimestamp::ZERO, + _ => self.timestamp.prev(), + }) + } + pub(crate) fn prepare_entry(&mut self, args: &KickArgs<'_>) -> Result> { + if args.qid != self.id.qid + || args.timestamp != self.timestamp + || args.parent != self.parent() + { + return Err(EINVAL); + } + let entry = KickEntry::new(args)?; + let destination = self + .object() + .pointer(args.timestamp.entry_offset(), size_of::())? + .cast(); + let timestamp = self.timestamp; + Ok(PreparedKick { + queue: self, + timestamp, + entry, + destination, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs new file mode 100644 index 00000000000000..fe354a52968ccf --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -0,0 +1,1153 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G17 GPU support. +//! +//! G17 GPUs run their firmware on two coprocessor instances, the primary and the secondary +//! [`Role`]. Each has its own CPU control registers, RTKit mailbox, crash log buffer and kernel +//! page table root, and both share the UAT, the TTBAT and the firmware data structures that the +//! driver allocates at boot (see [`initdata`]). +//! +//! The firmware notifies the host of new records in its event and trace rings with a message on +//! either coprocessor; the event worker drains them. +//! +//! # Locking +//! +//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all +//! firmware-shared state, including the bounded waits for the firmware that publication needs. +//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state +//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. + +mod buffer; +mod channel; +mod command; +pub(crate) use command::Attachments; +mod completion; +pub(crate) mod context; +mod dependency; +mod event; +mod fence; +mod freelist; +pub(crate) mod fw; +mod ids; +pub(crate) mod hazard; +mod initdata; +mod job; +mod kick; +mod object; +mod power; +mod preparation; +mod qos; +mod queue; +mod recovery; +mod runtime; +pub(crate) mod status; +mod teardown; +mod validation; + +use core::any::Any; +use core::ops::Range; +use core::sync::atomic::{ + AtomicBool, + Ordering, // +}; + +use kernel::{ + c_str, + device::Core, + drm::gem::shmem, + impl_has_delayed_work, impl_has_work, + iosys_map::IoSysMapRef, + new_condvar, new_delayed_work, new_mutex, new_work, platform, + prelude::*, + soc::apple::rtkit, + sync::{ + Arc, + CondVar, + Mutex, // + }, + time::{ + delay::fsleep, + msecs_to_jiffies, + Delta, // + }, + types::{ForeignOwnable, Opaque}, + uapi, + workqueue::{ + self, + DelayedWork, + Work, + WorkItem, // + }, +}; + +use crate::driver::{ + AsahiDevRef, + AsahiDevice, // +}; +use crate::gem::AsahiObject; +use crate::{ + alloc, + gem, + gpu, + hw, + mmu, + regs, // +}; + +use object::{ + CpuMap, + KernelObject, + Placement, // +}; + +/// Expected identification of a G17 GPU. +pub(crate) struct Identity { + /// GPU family field of the ID register. + pub(crate) family: u8, + /// GPU variant field of the ID register. + pub(crate) variant: u8, + /// Number of GPU dies. + pub(crate) num_dies: u32, + /// GPU variant reported to userspace. + pub(crate) gpu_variant: hw::GpuVariant, + /// Shader core generation reported to userspace. + pub(crate) usc_generation: u32, + /// Hardware abstraction generation reported to userspace. + pub(crate) hal_generation: u32, +} + +/// A buffer that the firmware expects at a fixed address of the kernel lower root. +pub(crate) struct FixedBuffer { + /// GPU address. + pub(crate) va: u64, + /// Size in bytes. + pub(crate) size: usize, +} + +/// Static configuration of a SoC with a G17 GPU. +pub(crate) struct Config { + /// SoC identifier. + pub(crate) chip_id: u32, + /// Expected GPU identification. + pub(crate) identity: Identity, + /// Input address bits translated by each UAT root. + pub(crate) uat_ias: u32, + /// Output (physical) address bits of the UAT. + pub(crate) uat_oas: u32, + /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. + pub(crate) secondary_root_offset: u64, + /// Crash log buffer of each [`Role`]. + pub(crate) crash_buffers: [FixedBuffer; 2], + /// Parameter buffer page metrics shared with the firmware. + pub(crate) pm_metrics: FixedBuffer, +} + +/// A firmware coprocessor instance. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Role { + /// The coprocessor that owns the device-control ring. + Primary = 0, + /// The coprocessor that only runs work the primary one hands to it. + Secondary = 1, +} + +/// RTKit endpoint of firmware messages: initialization, its acknowledgement and event +/// notifications. +const EP_FIRMWARE: u8 = 0x20; +/// RTKit endpoint of host notifications to the firmware. +const EP_DOORBELL: u8 = 0x21; + +/// Bit position of the type field of firmware messages. +const MSG_TYPE_SHIFT: u32 = 48; +/// Host message handing a coprocessor the address of its initialization root. +const MSG_INITDATA: u64 = 0x81 << MSG_TYPE_SHIFT; +/// Address bits of an initialization root that [`MSG_INITDATA`] carries. +const MSG_INITDATA_ADDR_MASK: u64 = (1 << 44) - 1; +/// Firmware message acknowledging [`MSG_INITDATA`]. +const MSG_INITDATA_ACK: u64 = 0x09; +/// Firmware message announcing new records in the event and trace rings. +const MSG_EVENTS: u64 = 0x42 << MSG_TYPE_SHIFT; +/// Primary processor notification for newly published device-control records. +const MSG_CONTROL_NOTIFY: u64 = (0x84 << MSG_TYPE_SHIFT) | 0x11; +/// Host message, sent to both coprocessors once both are initialized, ending the power-on +/// transition. The firmware then retires the opening device-control records. +const MSG_POWER_ON_DONE: u64 = 0x89 << MSG_TYPE_SHIFT; + +/// Time the firmware takes at most to acknowledge its initialization. +const INITDATA_ACK_TIMEOUT_MS: u32 = 2000; +/// Bounded mailbox-provider readiness retries, before either coprocessor starts. +const RECEIVER_ATTEMPTS: usize = 3; +const RECEIVER_RETRY_MS: i64 = 20; +/// The firmware does not notify the host when it retires device-control records, so their +/// counters are polled at this interval, at most [`OPENING_RETIRE_POLLS`] times. +const OPENING_RETIRE_POLL_MS: i64 = 1; +const OPENING_RETIRE_POLLS: usize = 100; + +/// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in +/// its buffer requests. +const RTKIT_BUFFER_ADDR_MASK: u64 = (1 << 40) - 1; +/// Initial contents of crash log buffers, so that a crash log shows what the firmware wrote. +const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; + +/// A crash log buffer handed to RTKit. +pub(crate) struct CrashLog { + iova: usize, + map: shmem::VMap, +} + +impl rtkit::Buffer for CrashLog { + fn iova(&self) -> Result { + Ok(self.iova) + } + + fn buf(&mut self) -> Result> { + Ok(self.map.get()) + } +} + +/// Callback state; initialization ACKs can arrive in hard interrupt context. +#[pin_data] +struct RoleState { + role: Role, + shared: Arc, + crash_log: FixedBuffer, + crash_map: shmem::VMap, + crashed: AtomicBool, + #[pin] + ack: Opaque, +} + +// SAFETY: Completion access uses the kernel completion API; the remaining fields are immutable +// shared references or atomics. The completion is pinned before callbacks can observe it. +unsafe impl Send for RoleState {} +// SAFETY: As above; complete_all is safe concurrently with the single boot waiter. +unsafe impl Sync for RoleState {} + +impl RoleState { + fn signal(&self, crashed: bool) { + if crashed { + self.crashed.store(true, Ordering::Release); + } + // SAFETY: ack is initialized and pinned for the lifetime of the callback state. + unsafe { kernel::bindings::complete_all(self.ack.get()) }; + } + + fn wait_for_ack(&self) -> Result { + // SAFETY: ack is initialized and remains pinned while this reference exists. + let completed = unsafe { + kernel::bindings::wait_for_completion_timeout( + self.ack.get(), + msecs_to_jiffies(INITDATA_ACK_TIMEOUT_MS), + ) + }; + if self.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if completed == 0 { + return Err(ETIMEDOUT); + } + Ok(()) + } +} + +fn is_events_message(endpoint: u8, message: u64) -> bool { + endpoint == EP_FIRMWARE && message == MSG_EVENTS +} + +/// RTKit callbacks of a coprocessor. +struct RoleOps; + +#[vtable] +impl rtkit::Operations for RoleOps { + type Data = Arc; + type Buffer = CrashLog; + + const COPROC_PLACES_BUFFERS: bool = true; + + fn recv_message_early( + data: ::Borrowed<'_>, + ep: u8, + msg: u64, + ) -> bool { + if ep == EP_FIRMWARE && msg >> MSG_TYPE_SHIFT == MSG_INITDATA_ACK { + data.signal(false); + return true; + } + if !is_events_message(ep, msg) { + return false; + } + data.shared.queue_events(); + true + } + + fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { + data.signal(true); + data.shared.lose_device(data.role, crashlog); + } + + fn shmem_map( + data: ::Borrowed<'_>, + iova: usize, + size: usize, + ) -> Result { + if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size + { + return Err(EINVAL); + } + Ok(CrashLog { + iova, + map: data.crash_map.clone(), + }) + } +} + +/// A running firmware coprocessor. +/// +/// Dropping it stops the CPU before RTKit and the crash log buffer go away. +struct Coprocessor { + cpu: regs::CpuControl, + running: bool, + rtkit: rtkit::RtKit, + state: Arc, + _crash_buffer: KernelObject, +} + +impl Coprocessor { + /// Sets up a coprocessor's RTKit and crash log buffer, without starting it. + fn new( + pdev: &platform::Device, + dev: &AsahiDevice, + cfg: &Config, + uat: &mmu::Uat, + shared: &Arc, + role: Role, + ) -> Result { + let index = role as usize; + let buffer = &cfg.crash_buffers[index]; + let mut crash_buffer = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(buffer.va), + buffer.size, + mmu::PROT_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); + + let crash_map = crash_buffer.kernel_map()?; + let state = Arc::pin_init( + pin_init!(RoleState { + role, + shared: shared.clone(), + crash_log: FixedBuffer { + va: buffer.va, + size: buffer.size, + }, + crash_map, + crashed: AtomicBool::new(false), + ack <- Opaque::ffi_init(|ptr| { + // SAFETY: ptr is valid uninitialized completion storage that is pinned + // before any RTKit callback can use it. + unsafe { kernel::bindings::init_completion(ptr) }; + }), + }), + GFP_KERNEL, + )?; + + let name = match role { + Role::Primary => c_str!("asc"), + Role::Secondary => c_str!("asc1"), + }; + Ok(Coprocessor { + cpu: regs::CpuControl::new(pdev, name)?, + running: false, + rtkit: Self::receiver(dev, index, &state)?, + state, + _crash_buffer: crash_buffer, + }) + } + + fn receiver( + dev: &AsahiDevice, + index: usize, + state: &Arc, + ) -> Result> { + for attempt in 0..RECEIVER_ATTEMPTS { + match rtkit::RtKit::::new(dev.as_ref(), None, index, state.clone()) { + Ok(receiver) => return Ok(receiver), + Err(error) if error == EINVAL || error == ETIMEDOUT => { + if attempt + 1 < RECEIVER_ATTEMPTS { + fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); + } + } + Err(error) => return Err(error), + } + } + Err(EPROBE_DEFER) + } + + /// Starts the CPU and waits for RTKit to come up. + fn boot(&mut self) -> Result { + self.cpu.start()?; + self.running = true; + Pin::new(&mut self.rtkit).boot() + } + + /// Starts the firmware endpoints and hands the firmware its initialization root. + fn init(&mut self, root_va: u64) -> Result { + let mut rtkit = Pin::new(&mut self.rtkit); + rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; + rtkit.as_mut().start_endpoint(EP_DOORBELL)?; + rtkit.send_message( + EP_FIRMWARE, + MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), + )?; + self.state.wait_for_ack() + } + + /// Sends a host notification to the firmware. + fn notify(&mut self, msg: u64) -> Result { + Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) + } + + /// Stops the CPU. The firmware keeps its state, but no longer runs. + fn stop(&mut self) { + if self.running { + self.cpu.stop(); + self.running = false; + } + } +} + +impl Drop for Coprocessor { + fn drop(&mut self) { + self.stop(); + } +} + +/// The GPU-to-fabric bridge, enabled for as long as this exists. +struct Bridge { + regs: regs::Resources, + enabled: bool, +} + +impl Bridge { + fn enable(regs: regs::Resources) -> Result { + let bridge = Bridge { + regs, + enabled: true, + }; + bridge.regs.enable_g17_bridge()?; + Ok(bridge) + } + + fn disable(&mut self) { + if self.enabled { + self.regs.disable_g17_bridge(); + self.enabled = false; + } + } +} + +impl Drop for Bridge { + fn drop(&mut self) { + self.disable(); + } +} + +/// Everything that exists while the firmware runs. +/// +/// Field order is teardown order: both coprocessors stop (in [`Drop`]) before their RTKit is +/// dropped, the bridge is disabled once they are stopped, and the firmware objects and their +/// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. +struct Firmware { + secondary: Coprocessor, + primary: Coprocessor, + bridge: Bridge, + rings: channel::Rings, + recovery: recovery::State, + effort: power::Effort, + queues: runtime::Registry, + pool: Arc, + render_ids: Arc, + init: initdata::InitData, + _render_global: mmu::Vm, + _pm_metrics: Arc, + uat: Arc, +} + +impl Drop for Firmware { + fn drop(&mut self) { + self.secondary.stop(); + self.primary.stop(); + self.pool.stop_after_firmware(); + // RTKit destruction releases the mailbox power references. Clear the bridge while + // those references still keep the register block powered. + self.bridge.disable(); + } +} + +impl Firmware { + /// Publishes controls only while the restart handshake is closed. The device mutex remains + /// held through publication and the following mailbox notification. + fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { + use recovery::Memory; + if self.recovery.pending() || self.init.recovery_state()? != 0 { + return Err(EAGAIN); + } + let target = channel::Rings::publish_control(&self.init, record)?; + core::sync::atomic::fence(Ordering::SeqCst); + self.primary.notify(MSG_CONTROL_NOTIFY)?; + Ok(target) + } + + /// Holds idle descent while a fresh physical render pair is registered. + fn acquire_registration_power(&mut self) -> Result { + power::acquire(self) + } + + fn release_registration_power(&mut self) -> Result { + power::release(self) + } + + /// Called at actual submission publication, not while a queued job is merely being built. + fn note_submission(&mut self) -> Result { + self.effort.submit(); + self.apply_effort(power::FULL_EFFORT) + } + + fn apply_effort(&mut self, effort: u32) -> Result { + Self::apply_shared_effort(&self.init, &mut self.effort, effort) + } + + fn apply_shared_effort( + init: &initdata::InitData, + state: &mut power::Effort, + effort: u32, + ) -> Result { + if state.applied() == effort && init.effort()? == effort { + return Ok(()); + } + init.publish_effort(effort)?; + state.did_apply(effort); + Ok(()) + } + + /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, + /// unretired work; quarantined work retaining mappings is excluded. + fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { + match self.effort.idle(now_ns, in_flight) { + power::IdleAction::None => Ok(None), + power::IdleAction::Clear => { + self.apply_effort(0)?; + Ok(None) + } + power::IdleAction::Recheck(delay) => Ok(Some(delay)), + } + } +} + +impl power::Control for Firmware { + fn crashed(&self) -> bool { + self.primary.state.shared.crashed.load(Ordering::Acquire) + } + fn powered(&self) -> bool { + self.bridge.regs.g17_configuration_powered() + } + fn counters(&self) -> Result<[u32; 3]> { + self.init.control_counters() + } + fn set_idle(&mut self, enabled: bool) -> Result { + use fw::channels::{ControlRecord, IdlePolicy, IdlePowerOff}; + self.publish_control(&ControlRecord::IdlePowerOff(IdlePowerOff::new( + if enabled { + IdlePolicy::AllowPowerOff + } else { + IdlePolicy::KeepPowered + }, + ))) + } + fn wait_tick(&self) { + fsleep(Delta::from_millis(1)); + } +} + +/// Event work item ID. +const EVENT_WORK: u64 = 0; +/// Short control/retirement polls do not wait behind the longer idle deadline. +const POLL_WORK: u64 = 1; +const IDLE_WORK: u64 = 2; +const GROW_WORK: u64 = 3; +const RECLAIM_WORK: u64 = 4; + +/// Device state shared with the coprocessor callbacks and the event worker. +#[pin_data] +struct Shared { + dev: AsahiDevRef, + clusters: u32, + descriptor_flags: [u32; 2], + crashed: AtomicBool, + preparations: Arc, + #[pin] + work: Work, + #[pin] + poll: DelayedWork, + #[pin] + idle: DelayedWork, + #[pin] + grow: Work, + #[pin] + reclaim_work: Work, + /// Nests inside the device mutex; detached mappings are dropped after both unlock. + #[pin] + reclaim: Mutex, + /// The device lock. `None` before boot completes and after teardown. + #[pin] + state: Mutex>>, + #[pin] + changed: CondVar, +} + +impl_has_work! { + impl HasWork for Shared { self.work } +} + +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.poll } +} +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.idle } +} +impl_has_work! { + impl HasWork for Shared { self.grow } +} +impl_has_work! { + impl HasWork for Shared { self.reclaim_work } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + if let Err(error) = this.grow_render() { + if this.state.lock().is_some() { + dev_err!( + this.dev.as_ref(), + "Render parameter-buffer growth failed: {:?}\n", + error + ); + this.lose_device_quietly(); + } + } + this.queue_events(); + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.drain_reclaims(); + this.queue_events(); + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + let result = { + let mut state = this.state.lock(); + match (*state).as_deref_mut() { + Some(firmware) if firmware.render_control_backpressured() => { + firmware.note_render_control_backpressure() + } + _ => Ok(()), + } + }; + // Only an elapsed one-millisecond timer consumes the retry budget. + // Concurrent event IRQs can request a pass without aging the control. + if result.is_err() { + this.lose_device_quietly(); + } else { + this.queue_events(); + } + } +} +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + this.queue_events(); + } +} + +impl WorkItem for Shared { + type Pointer = Arc; + + fn run(this: Arc) { + this.run_runtime_events(); + } +} + +impl Shared { + /// Queues the event worker. Callable from any context. + fn queue_events(self: &Arc) { + // Already queued if this fails, which is just as good. + let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); + } + + fn queue_grow(self: &Arc) { + let _ = workqueue::system_unbound().enqueue::, GROW_WORK>(self.clone()); + } + + fn queue_reclaims(self: &Arc) { + let _ = workqueue::system_unbound().enqueue::, RECLAIM_WORK>(self.clone()); + } + + fn drain_reclaims(self: &Arc) { + loop { + let next = self.reclaim.lock().take_one(); + let Some(next) = next else { + break; + }; + next.finish(); + } + } + + fn disable_work(self: &Arc, work: &Work) { + // SAFETY: Callers pass this Shared's initialized pinned work field. + // Disabling joins a running callback and prevents subsequent enqueue. + let cancelled = unsafe { kernel::bindings::disable_work_sync(Work::raw_get(work)) }; + if cancelled { + // SAFETY: Cancelling the pending callback leaves its one enqueue- + // transferred Arc to reclaim; the callback can no longer consume it. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + fn queue_poll(self: &Arc) { + let _ = workqueue::system_highpri() + .enqueue_delayed::, POLL_WORK>(self.clone(), msecs_to_jiffies(1)); + } + + fn queue_idle(self: &Arc, delay_ns: u64) { + // Effort::idle bounds this by the 16 ms idle interval. Round up so a + // timer never changes the firmware effort word before that deadline. + let delay_ms = delay_ns.div_ceil(1_000_000).min(u64::from(u32::MAX)) as u32; + let _ = workqueue::system_highpri() + .enqueue_delayed::, IDLE_WORK>(self.clone(), msecs_to_jiffies(delay_ms)); + } + + fn disable_timer(self: &Arc, timer: &DelayedWork) { + // SAFETY: Every caller passes one of this Shared's initialized pinned + // timer fields. raw_as_work points into its delayed_work; the C call + // disables future enqueues, cancels pending work and joins execution. + let cancelled = unsafe { + let work = Work::raw_get(DelayedWork::raw_as_work(timer)); + let delayed = kernel::container_of!(work, kernel::bindings::delayed_work, work); + kernel::bindings::disable_delayed_work_sync(delayed) + }; + if cancelled { + // SAFETY: The cancelled pending callback owned exactly one Arc + // transferred by enqueue_delayed. It cannot run to reclaim it. + drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); + } + } + + /// Marks the device lost without a crash log. + fn lose_device_quietly(self: &Arc) { + self.preparations.fail(); + if !self.crashed.swap(true, Ordering::AcqRel) { + dev_err!(self.dev.as_ref(), "GPU runtime stopped, device lost\n"); + } + self.queue_events(); + } + + /// Marks the device lost after a coprocessor crash and publishes a crash dump. + /// + /// The firmware cannot be restarted: outstanding and future work fails. + fn lose_device(self: &Arc, role: Role, crashlog: Option<&[u8]>) { + self.preparations.fail(); + self.crashed.store(true, Ordering::Release); + dev_err!( + self.dev.as_ref(), + "GPU firmware ({:?}) crashed, device lost\n", + role + ); + + #[cfg(CONFIG_DEV_COREDUMP)] + if let Some(crashlog) = crashlog { + if let Err(e) = self.publish_crashlog(crashlog) { + dev_err!(self.dev.as_ref(), "Could not generate crashdump: {:?}\n", e); + } + } + #[cfg(not(CONFIG_DEV_COREDUMP))] + let _ = crashlog; + self.queue_events(); + } + + #[cfg(CONFIG_DEV_COREDUMP)] + fn publish_crashlog(&self, crashlog: &[u8]) -> Result { + use kernel::{devcoredump, time::msecs_to_jiffies}; + + // The dump does not include firmware memory: its kernel mappings are private to it. + let mut crashdump = crate::crashdump::CrashDumpBuilder::new(KVVec::new())?; + crashdump.add_crashlog(crashlog)?; + let crashdump = KBox::new(crashdump.finalize()?, GFP_KERNEL)?; + devcoredump::dev_coredump( + self.dev.as_ref(), + &crate::THIS_MODULE, + crashdump, + GFP_KERNEL, + msecs_to_jiffies(CRASHDUMP_TIMEOUT_MS), + ); + Ok(()) + } +} + +/// How long a crash dump stays available. +#[cfg(CONFIG_DEV_COREDUMP)] +const CRASHDUMP_TIMEOUT_MS: u32 = 60 * 60 * 1000; + +/// A G17 GPU. +pub(crate) struct Gpu { + cfg: &'static Config, + id: regs::G17Id, + max_frequency_khz: u32, + geometry: mmu::UatGeometry, + ids: gpu::SequenceIDs, + shared: Arc, +} + +impl Gpu { + /// Boots the firmware of a G17 GPU. + pub(crate) fn new( + pdev: &platform::Device, + dev: &AsahiDevice, + cfg: &'static Config, + regs: regs::Resources, + ) -> Result> { + let id = regs.get_g17_id()?; + if id.family != cfg.identity.family + || id.variant != cfg.identity.variant + || id.num_dies != cfg.identity.num_dies + { + dev_err!( + dev.as_ref(), + "Unexpected GPU family {:#x} variant {:#x} with {} dies\n", + id.family, + id.variant, + id.num_dies + ); + return Err(ENODEV); + } + + let geometry = mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + let max_frequency_khz = Self::max_frequency_khz(dev)?; + let preparations = preparation::Gate::new()?; + let reclaim = runtime::teardown::ReclaimBatch::new()?; + let shared = Arc::pin_init( + pin_init!(Shared { + dev: dev.into(), + clusters: id.num_clusters, + descriptor_flags: [id.perf_control, id.perf_map[0]], + crashed: AtomicBool::new(false), + preparations, + work <- new_work!("g17::Shared::work"), + poll <- new_delayed_work!("g17::Shared::poll"), + idle <- new_delayed_work!("g17::Shared::idle"), + grow <- new_work!("g17::Shared::grow"), + reclaim_work <- new_work!("g17::Shared::reclaim_work"), + reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), + state <- new_mutex!(None, "g17::Shared::state"), + changed <- new_condvar!("g17::Shared::changed"), + }), + GFP_KERNEL, + )?; + + let gpu = Arc::new( + Gpu { + cfg, + id, + max_frequency_khz, + geometry, + ids: Default::default(), + shared, + }, + GFP_KERNEL, + )?; + let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; + *gpu.shared.state.lock() = Some(firmware); + gpu.shared.queue_events(); + Ok(gpu) + } + + /// Returns the highest advertised operating-point frequency without changing table order. + fn max_frequency_khz(dev: &AsahiDevice) -> Result { + let node = dev.as_ref().of_node().ok_or(EIO)?; + let opps = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EIO)?; + let mut freq_hz: Option = None; + for opp in opps.children() { + let frequency: u64 = opp.get_property(c_str!("opp-hz"))?; + freq_hz = Some(freq_hz.map_or(frequency, |highest| highest.max(frequency))); + } + Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) + } + + /// Starts both coprocessors and initializes their firmware. + fn boot( + pdev: &platform::Device, + dev: &AsahiDevice, + cfg: &'static Config, + regs: regs::Resources, + geometry: mmu::UatGeometry, + shared: &Arc, + ) -> Result> { + let uat = Arc::new( + mmu::Uat::new( + dev, + geometry, + true, + mmu::UatFirmware::PublishedRoots { + secondary_root_offset: cfg.secondary_root_offset, + }, + )?, + GFP_KERNEL, + )?; + + // The firmware only samples the kernel lower root's top-level entries when it starts, so + // everything it uses there must be mapped first. + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary)?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary)?; + let pm_metrics = KernelObject::new( + dev, + uat.kernel_lower_vm(), + Placement::At(cfg.pm_metrics.va), + cfg.pm_metrics.size, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let render_global = uat.new_vm(0, geometry.user_range())?; + + let bridge = Bridge::enable(regs)?; + + // Install all session ownership before either CPU starts. Error unwinding then runs + // Firmware::drop, stopping both CPUs before releasing shared objects or translations. + let mut firmware = KBox::new( + Firmware { + secondary, + primary, + bridge, + rings: channel::Rings::new(), + recovery: recovery::State::new(), + effort: power::Effort::new(), + queues: runtime::Registry::new(shared.clusters, shared.descriptor_flags)?, + pool: object::Pool::new()?, + render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, + init: initdata::InitData::new(), + _render_global: render_global, + _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, + uat, + }, + GFP_KERNEL, + )?; + let session = &mut *firmware; + session.primary.boot()?; + session.secondary.boot()?; + let root_entries = session.uat.adopt_firmware_root_entries()?; + session.init.build( + dev, + cfg, + &session.uat, + &session._render_global, + &session._pm_metrics, + )?; + session.uat.confirm_firmware_root_entries(root_entries)?; + session.uat.mirror_secondary_root()?; + session.bridge.regs.enable_g17_gating()?; + session.primary.init(session.init.root_va(Role::Primary)?)?; + session + .secondary + .init(session.init.root_va(Role::Secondary)?)?; + session.primary.notify(MSG_POWER_ON_DONE)?; + session.secondary.notify(MSG_POWER_ON_DONE)?; + Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; + session + .uat + .activate_render_context(&session._render_global)?; + Ok(firmware) + } + + /// Waits for both coprocessors to retire their opening device-control records. + fn wait_for_opening( + dev: &AsahiDevice, + init: &initdata::InitData, + primary: &Coprocessor, + secondary: &Coprocessor, + ) -> Result { + for _ in 0..OPENING_RETIRE_POLLS { + if primary.state.crashed.load(Ordering::Acquire) + || secondary.state.crashed.load(Ordering::Acquire) + { + return Err(EIO); + } + if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { + return Ok(()); + } + fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + } + dev_err!( + dev.as_ref(), + "Firmware did not retire its opening records\n" + ); + Err(ETIMEDOUT) + } + + /// Stops the firmware and releases everything it used. + pub(crate) fn shutdown(&self) { + self.shared.preparations.remove(); + self.shared.stop_queues(); + self.shared.disable_timer(&self.shared.poll); + self.shared.disable_timer(&self.shared.idle); + self.shared.disable_work(&self.shared.grow); + self.shared.disable_work(&self.shared.reclaim_work); + self.shared.disable_work(&self.shared.work); + self.shared.drain_reclaims(); + } +} + +impl gpu::Gpu for Gpu { + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn manager(&self) -> Option<&dyn gpu::GpuManager> { + None + } + + fn init(&self) -> Result { + Ok(()) + } + + fn ids(&self) -> &gpu::SequenceIDs { + &self.ids + } + + fn is_crashed(&self) -> bool { + self.shared.crashed.load(Ordering::Acquire) + } + + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.gpu_generation = hw::GpuGen::G17 as u32; + params.gpu_variant = self.cfg.identity.gpu_variant as u32; + params.gpu_revision = self.id.gpu_rev as u32; + params.chip_id = self.cfg.chip_id; + params.num_dies = self.id.num_dies; + params.num_clusters_total = self.id.num_clusters; + params.num_cores_per_cluster = self.id.num_cores; + params.max_frequency_khz = self.max_frequency_khz; + params.usc_generation = self.cfg.identity.usc_generation; + params.gpu_hal_generation = self.cfg.identity.hal_generation; + + for (i, mask) in self.id.core_masks.iter().enumerate() { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + + Ok(()) + } + + fn base_clock_hz(&self) -> u32 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only register containing the architectural counter rate. + unsafe { core::arch::asm!("mrs {frequency}, CNTFRQ_EL0", frequency = out(reg) frequency) }; + frequency as u32 + } + + fn supports_vm_status(&self) -> bool { + true + } + + fn uat_geometry(&self) -> mmu::UatGeometry { + self.geometry + } + + fn new_vm(&self, kernel_range: Range) -> Result { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_mut().ok_or(ENODEV)?; + let firmware = &mut **firmware; + let vm = firmware + .uat + .new_vm(self.ids.vm.next(), kernel_range)? + .with_status()?; + let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; + let ranges = + hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; + let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; + for range in ranges { + reserved.push(range, GFP_KERNEL)?; + } + vm.install_driver_mappings(mappings, reserved)?; + Ok(vm) + } + + fn new_queue( + &self, + vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let owner = self.ids.queue.next(); + let frontend = + queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let (backend, slot) = loop { + match runtime::Backend::new(&self.shared, &vm, owner) { + Err(error) if error == EAGAIN => continue, + result => break result?, + } + }; + let frontend = match queue::Queue::new( + &self.shared.dev, + backend.context.clone(), + backend.clone(), + frontend, + slot, + ) { + Ok(frontend) => frontend, + Err(error) => { + job::Backend::release_owner(&*backend); + return Err(error); + } + }; + Ok(KBox::new(frontend, GFP_KERNEL)?) + } + + fn map_timestamp_buffer( + &self, + mut bo: gem::ObjectRef, + range: Range, + ) -> Result { + let state = self.shared.state.lock(); + let firmware = (*state).as_ref().ok_or(ENODEV)?; + let mut addresses = firmware.uat.geometry().kernel_range(); + addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; + bo.map_range_into_range( + firmware.uat.kernel_vm(), + range, + addresses, + mmu::UAT_PGSZ as u64, + mmu::PROT_FW_SHARED_RW, + false, + ) + } +} + +impl Drop for Gpu { + fn drop(&mut self) { + self.shutdown(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/object.rs b/drivers/gpu/drm/asahi/g17/object.rs new file mode 100644 index 00000000000000..cf1d49fb03a6bb --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/object.rs @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Kernel objects shared with the G17 firmware. +//! +//! Each firmware object is backed by its own GEM object, mapped into a kernel address space +//! either at an address the firmware interface fixes or anywhere in a given range, and kept +//! mapped into the kernel CPU address space for its whole lifetime. The host builds an object's +//! contents through [`KernelObject::bytes_mut`] and [`KernelObject::write`] before it publishes +//! its address; after that, words the firmware accesses concurrently are only touched through +//! the single-access helpers. + +use core::ops::Range; +use core::ptr::NonNull; +use core::sync::atomic::{ + AtomicU32, + AtomicU64, // +}; + +use kernel::{ + drm::gem::shmem, + prelude::*, // +}; + +use crate::driver::AsahiDevice; +use crate::{ + gem, + mmu, // +}; + +/// CPU caching of the kernel mapping of a [`KernelObject`]. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum CpuMap { + /// Write-back, for memory the firmware only reads through a cached mapping or that the host + /// only reads. + WriteBack, + /// Write-combined, for memory the firmware reads uncached as soon as the host published it. + WriteCombined, +} + +/// Where a [`KernelObject`] is mapped. +pub(crate) enum Placement { + /// At this address, which the firmware interface fixes. + At(u64), + /// Anywhere in this range, at this alignment, followed by an unmapped guard page. + In(Range, u64), +} + +/// A zero-initialized kernel object mapped into a GPU address space and the kernel. +/// +/// # Invariants +/// +/// `ptr` points to the start of the kernel mapping of `obj`, which is `size` bytes long and stays +/// valid as long as `obj` exists, since `obj` holds an owned kernel mapping. +pub(crate) struct KernelObject { + // Unmap from the GPU before dropping the backing object. + mapping: Option, + obj: gem::ObjectRef, + ptr: NonNull, + size: usize, +} + +// SAFETY: The object only refers to kernel memory; concurrent access goes through `&self` +// methods that use atomic or volatile accesses. +unsafe impl Send for KernelObject {} +// SAFETY: See `Send`. +unsafe impl Sync for KernelObject {} + +impl KernelObject { + /// Allocates a zeroed object of `size` bytes and maps it into `vm`. + pub(crate) fn new( + dev: &AsahiDevice, + vm: &mmu::Vm, + placement: Placement, + size: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let mut this = Self::backing(dev, size, cpu)?; + let mapping = match placement { + Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, + Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, + }; + this.mapping = Some(mapping); + Ok(this) + } + + /// Allocates CPU-accessible backing whose GPU mappings will be installed later. + pub(crate) fn backing(dev: &AsahiDevice, size: usize, cpu: CpuMap) -> Result { + let mut obj = match cpu { + CpuMap::WriteBack => gem::new_kernel_object(dev, size)?, + CpuMap::WriteCombined => gem::new_kernel_object_wc(dev, size)?, + }; + let mut vmap = obj.vmap()?; + vmap.memset(0); + let ptr = NonNull::new(vmap.as_mut_ptr()).ok_or(ENOMEM)?; + core::mem::drop(vmap); + + // INVARIANT: `vmap()` created the owned kernel mapping of `obj` that `ptr` points into, + // and `obj` keeps it until it is dropped. + Ok(Self { + mapping: None, + obj, + ptr, + size, + }) + } + + /// Maps this object into another address space at `addr`, as an alias of the same memory. + pub(crate) fn map_alias( + &self, + vm: &mmu::Vm, + addr: u64, + prot: mmu::Prot, + ) -> Result { + gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) + } + + /// Reuses only a complete alias of this exact retained backing. Permanent + /// queue aliases stay owned by the client VM after a physical pair moves; + /// returning to that VM must not replace or duplicate those mappings. + pub(crate) fn prepare_alias( + &self, + vm: &mmu::Vm, + address: u64, + prot: mmu::Prot, + requires_write: bool, + ) -> Result> { + if self.size == 0 || self.size % mmu::UAT_PGSZ != 0 || address & mmu::UAT_PGMSK as u64 != 0 + { + return Err(EINVAL); + } + address.checked_add(self.size as u64).ok_or(EOVERFLOW)?; + let source = self.mapping.as_ref().ok_or(EINVAL)?; + let mut existing = 0; + for offset in (0..self.size).step_by(mmu::UAT_PGSZ) { + let client = address + offset as u64; + if vm.covers_range(client, mmu::UAT_PGSZ as u64, true, requires_write) { + if vm.translate_iova(client)? != source.translate_offset(offset)? { + return Err(EFAULT); + } + existing += 1; + } + } + if existing != 0 { + if existing != self.size / mmu::UAT_PGSZ { + return Err(EFAULT); + } + return Ok(None); + } + self.map_alias(vm, address, prot).map(Some) + } + + /// Maps this backing into a dynamically placed kernel alias. + pub(crate) fn alias_in( + &self, + vm: &mmu::Vm, + range: Range, + align: u64, + prot: mmu::Prot, + ) -> Result { + gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) + } + + /// Maps an exact backing subrange, retaining an explicit unmapped guard. + pub(crate) fn map_range( + &self, + vm: &mmu::Vm, + source: Range, + range: Range, + align: u64, + prot: mmu::Prot, + guard_size: usize, + ) -> Result { + vm.map_in_range_with_guard_size(&self.obj.gem, source, align, range, prot, guard_size) + } + + /// Initializes a typed view before publishing the object to the firmware. + /// + /// The view is cleared first, making every `Zeroable` representation valid. + pub(crate) fn initialize( + &mut self, + offset: usize, + init: impl FnOnce(&mut T) -> Result, + ) -> Result { + let ptr = self.aligned::(offset)?; + // SAFETY: The checked view lies within the exclusively borrowed object. Zeroable makes + // the cleared bytes a valid T, and the closure cannot retain the reference. + unsafe { + ptr.write_bytes(0, 1); + init(&mut *ptr) + } + } + + /// Returns a reference-counted kernel mapping of the object. + pub(crate) fn kernel_map(&self) -> Result> { + self.obj.gem.owned_vmap() + } + + /// Transfers an unpublished object's GPU mapping to a longer-lived owner. + /// The CPU view remains valid; gpu_va returns zero after this transfer. + pub(crate) fn take_mapping(&mut self) -> Result { + self.mapping.take().ok_or(EINVAL) + } + + /// Returns the GPU address of the object. + pub(crate) fn gpu_va(&self) -> u64 { + self.mapping.as_ref().map_or(0, |mapping| mapping.iova()) + } + + /// Returns the size of the object in bytes. + pub(crate) fn size(&self) -> usize { + self.size + } + + /// Returns the contents of the object. + /// + /// Only for building an object the firmware does not access concurrently, before its address + /// is published or while the firmware is known not to use it. + pub(crate) fn bytes_mut(&mut self) -> &mut [u8] { + // SAFETY: Per the type invariant `ptr` is valid for `size` bytes, and the exclusive borrow + // excludes other host accesses. + unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.size) } + } + + /// Writes `value` at `offset`, with the same restrictions as [`Self::bytes_mut`]. + pub(crate) fn write(&mut self, offset: usize, value: T) -> Result { + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > self.size { + return Err(EINVAL); + } + // SAFETY: The range was checked to be within the object, and the exclusive borrow excludes + // other host accesses. The destination may be unaligned. + unsafe { + self.ptr + .as_ptr() + .add(offset) + .cast::() + .write_unaligned(value) + }; + Ok(()) + } + + /// Returns a pointer to the naturally aligned `T` at `offset`. + fn aligned(&self, offset: usize) -> Result<*mut T> { + let end = offset + .checked_add(core::mem::size_of::()) + .ok_or(EINVAL)?; + if end > self.size { + return Err(EINVAL); + } + // SAFETY: The checked offset lies within the retained CPU mapping. + let address = unsafe { self.ptr.as_ptr().add(offset) }; + if address as usize % core::mem::align_of::() != 0 { + return Err(EINVAL); + } + // The actual address, not only the offset, satisfies T's alignment. + Ok(address.cast()) + } + + /// Returns a checked raw view; dereferencing it requires external serialization. + pub(super) fn pointer(&self, offset: usize, length: usize) -> Result<*mut u8> { + if offset.checked_add(length).ok_or(EINVAL)? > self.size { + return Err(EINVAL); + } + // SAFETY: The offset was checked against the live object mapping. + Ok(unsafe { self.ptr.as_ptr().add(offset) }) + } + + /// Returns the live 32-bit word at `offset`, which must be naturally aligned. + pub(crate) fn word(&self, offset: usize) -> Result<&AtomicU32> { + // SAFETY: `aligned()` returns a valid, aligned pointer into the object, which lives as long + // as `self`. Other agents only access the word with single 32-bit accesses. + Ok(unsafe { AtomicU32::from_ptr(self.aligned(offset)?) }) + } + + /// Returns the live 64-bit word at `offset`, which must be naturally aligned. + pub(crate) fn dword(&self, offset: usize) -> Result<&AtomicU64> { + // SAFETY: As for `word()`, with 64-bit accesses. + Ok(unsafe { AtomicU64::from_ptr(self.aligned(offset)?) }) + } + + /// Copies `N` 32-bit words that start at `offset` out of the object, one volatile access each. + /// + /// Used for records another agent has published, which it no longer modifies. + pub(crate) fn read_words(&self, offset: usize) -> Result<[u32; N]> { + let base = self.aligned::<[u32; N]>(offset)?.cast::(); + let mut words = [0u32; N]; + for (i, word) in words.iter_mut().enumerate() { + // SAFETY: `aligned()` checked that all `N` words are within the object and aligned. + *word = unsafe { base.add(i).read_volatile() }; + } + Ok(words) + } +} + +/// Allocation domains shared by the G17 resource builders. Dynamic objects +/// retain a guard page; aliases keep the existing backing and its exact VA. +pub(crate) struct Allocator<'a> { + pub(crate) dev: &'a AsahiDevice, + pub(crate) uat: &'a mmu::Uat, +} + +impl Allocator<'_> { + /// Allocates in the dynamic part of the firmware's upper address space. + pub(crate) fn kernel( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let mut range = self.uat.geometry().kernel_range(); + range.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; + KernelObject::new( + self.dev, + self.uat.kernel_vm(), + Placement::In(range, align), + size, + prot, + cpu, + ) + } + + /// Allocates a canonical low mapping, which can be aliased at the same + /// address into an execution context without overlapping client objects. + pub(crate) fn lower( + &self, + size: usize, + align: u64, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + KernelObject::new( + self.dev, + self.uat.kernel_lower_vm(), + Placement::In(crate::hw::t8140::dynamic::LOWER, align), + size, + prot, + cpu, + ) + } +} + +/// Families whose upper mappings remain cached and installed until firmware +/// stops, even after the logical owner releases the object. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Family { + Descriptor = 0, + WorkState = 1, + QueueState = 2, + QueueConfig = 3, +} + +const POOL_FAMILIES: usize = 4; +const POOL_SLOTS: usize = 64; +const POOL_BYTES: usize = 128 * 1024 * 1024; + +struct Retained { + object: KernelObject, + prefix: usize, + cpu: CpuMap, + poisoned: bool, +} + +struct PoolInner { + returned: KVec>, + reserved: [usize; POOL_FAMILIES], + bytes: usize, + stopped: bool, +} + +/// Per-device cache-safe object recycling. Every converted mapping reserves +/// one return slot before its page-table attributes change. Active leases and +/// poisoned mappings count against the same capacity as reusable objects. +#[pin_data] +pub(crate) struct Pool { + #[pin] + inner: kernel::sync::Mutex, +} + +struct CacheLease { + pool: kernel::sync::Arc, + family: Family, + prefix: usize, + cpu: CpuMap, + poisoned: bool, +} + +/// A firmware object whose cached mapping is returned to its device pool +/// instead of being unmapped while either firmware processor can reference it. +pub(crate) struct PooledObject { + object: core::mem::ManuallyDrop, + lease: Option, +} + +impl core::ops::Deref for PooledObject { + type Target = KernelObject; + fn deref(&self) -> &Self::Target { + &self.object + } +} +impl core::ops::DerefMut for PooledObject { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.object + } +} +impl Drop for PooledObject { + fn drop(&mut self) { + // SAFETY: This destructor is the only operation that moves `object` + // out; ManuallyDrop prevents a second implicit drop of the field. + let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; + match self.lease.take() { + Some(lease) => lease.pool.return_object(object, &lease), + None => drop(object), + } + } +} + +impl Pool { + pub(crate) fn new() -> Result> { + let mut returned = KVec::with_capacity(POOL_FAMILIES * POOL_SLOTS, GFP_KERNEL)?; + for _ in 0..POOL_FAMILIES * POOL_SLOTS { + returned.push(None, GFP_KERNEL)?; + } + kernel::sync::Arc::pin_init( + pin_init!(Self { + inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), + }), + GFP_KERNEL, + ) + } + + /// Allocates or reuses a matching object. Capacity pressure leaves a fresh + /// mapping uncached, preserving allocation and submission concurrency. + pub(crate) fn allocate( + self: &kernel::sync::Arc, + alloc: &Allocator<'_>, + family: Family, + size: usize, + prefix: usize, + prot: mmu::Prot, + cpu: CpuMap, + ) -> Result { + let family_index = family as usize; + let begin = family_index * POOL_SLOTS; + let reused = { + let mut inner = self.inner.lock(); + if inner.stopped { + return Err(ENODEV); + } + let slot = inner.returned[begin..begin + POOL_SLOTS] + .iter_mut() + .find(|entry| { + entry.as_ref().is_some_and(|entry| { + !entry.poisoned + && entry.object.size() == size + && entry.prefix == prefix + && entry.cpu == cpu + }) + }); + slot.and_then(Option::take) + }; + let lease = || CacheLease { + pool: self.clone(), + family, + prefix, + cpu, + poisoned: false, + }; + if let Some(mut entry) = reused { + entry.object.bytes_mut().fill(0); + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + return Ok(PooledObject { + object: core::mem::ManuallyDrop::new(entry.object), + lease: Some(lease()), + }); + } + let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; + let reserved = { + let mut inner = self.inner.lock(); + if inner.stopped { + return Err(ENODEV); + } + if inner.reserved[family_index] == POOL_SLOTS + || inner + .bytes + .checked_add(size) + .is_none_or(|bytes| bytes > POOL_BYTES) + { + false + } else { + inner.reserved[family_index] += 1; + inner.bytes += size; + true + } + }; + let mut result = PooledObject { + object: core::mem::ManuallyDrop::new(object), + lease: reserved.then(lease), + }; + if !reserved { + return Ok(result); + } + let (proof, mutated) = match result.mapping.as_ref() { + // SAFETY: The allocator just created the private mapping. No + // pointer or alias has been published; its reserved lease retains + // it until processor stop if any attribute change was attempted. + Some(mapping) => unsafe { mapping.set_fw_cached_prefix(prefix) }, + None => (Err(EINVAL), false), + }; + match proof { + Ok(_) => Ok(result), + Err(error) if mutated => { + if let Some(lease) = &mut result.lease { + lease.poisoned = true; + } + // Drop retains the unproven mapping but never makes it reusable. + Err(error) + } + Err(_) => { + result.lease = None; + let mut inner = self.inner.lock(); + inner.reserved[family_index] -= 1; + inner.bytes -= size; + Ok(result) + } + } + } + + fn return_object(&self, object: KernelObject, lease: &CacheLease) { + let mut inner = self.inner.lock(); + let family = lease.family as usize; + if inner.stopped { + inner.reserved[family] -= 1; + inner.bytes -= object.size(); + drop(inner); + drop(object); + return; + } + let begin = family * POOL_SLOTS; + for slot in &mut inner.returned[begin..begin + POOL_SLOTS] { + if slot.is_none() { + *slot = Some(Retained { + object, + prefix: lease.prefix, + cpu: lease.cpu, + poisoned: lease.poisoned, + }); + return; + } + } + // A private CacheLease reserves capacity before conversion, so a + // missing slot means host accounting corruption. Never unmap a cached + // object the firmware may still hold in that failure case. + pr_err!("G17 cached-object return capacity was lost\n"); + core::mem::forget(object); + } + + /// Releases retained mappings after both firmware processors have stopped. + /// Active objects returned later observe the same stop witness. + pub(crate) fn stop_after_firmware(&self) { + self.inner.lock().stopped = true; + for index in 0..POOL_FAMILIES * POOL_SLOTS { + let entry = { + let mut inner = self.inner.lock(); + let entry = inner.returned[index].take(); + if let Some(entry) = &entry { + inner.reserved[index / POOL_SLOTS] -= 1; + inner.bytes -= entry.object.size(); + } + entry + }; + // Mapping teardown must not run while the pool mutex is held. + drop(entry); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs new file mode 100644 index 00000000000000..8806f438234e98 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Work-based firmware performance effort. Idle power gating is managed separately by the +//! device-control ring; this input never keeps an otherwise idle GPU powered. + +pub(super) const FULL_EFFORT: u32 = 0x10000; +const IDLE_HOLD_NS: u64 = 16_000_000; + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum IdleAction { + None, + Clear, + Recheck(u64), +} + +pub(super) struct Effort { + streaming: bool, + idle_since: Option, + applied: u32, +} + +impl Effort { + pub(super) const fn new() -> Self { + Self { + streaming: false, + idle_since: None, + applied: 0, + } + } + + pub(super) fn submit(&mut self) { + self.streaming = true; + self.idle_since = None; + } + + pub(super) fn applied(&self) -> u32 { + self.applied + } + pub(super) fn did_apply(&mut self, value: u32) { + self.applied = value; + } + + /// Quarantined jobs do not count as work in flight; they retain mappings but cannot justify + /// retaining the effort input. New submissions restart the continuous-idle interval. + pub(super) fn idle(&mut self, now_ns: u64, in_flight: bool) -> IdleAction { + if !self.streaming { + return IdleAction::None; + } + if in_flight { + self.idle_since = None; + return IdleAction::None; + } + let since = *self.idle_since.get_or_insert(now_ns.max(1)); + let idle = now_ns.saturating_sub(since); + if idle >= IDLE_HOLD_NS { + self.streaming = false; + self.idle_since = None; + IdleAction::Clear + } else { + IdleAction::Recheck(IDLE_HOLD_NS - idle) + } + } +} + +/// Single-producer control path under the device mutex. Event draining must not publish another +/// control between `set_idle` and the exact three-counter retirement witness. +pub(super) trait Control { + fn crashed(&self) -> bool; + fn powered(&self) -> bool; + fn counters(&self) -> kernel::error::Result<[u32; 3]>; + fn set_idle(&mut self, enabled: bool) -> kernel::error::Result; + fn wait_tick(&self); +} + +fn wait(control: &C, target: u32, require_powered: bool) -> kernel::error::Result { + use kernel::prelude::*; + const RETIRE_POLLS: usize = 20; + const POWER_POLLS: usize = 250; + let mut retired = false; + for elapsed in 0..RETIRE_POLLS + POWER_POLLS { + if control.crashed() { + return Err(EIO); + } + if !retired { + retired = control.counters()? == [target; 3]; + if !retired && elapsed + 1 >= RETIRE_POLLS { + break; + } + } + if retired && (!require_powered || control.powered()) { + return Ok(()); + } + control.wait_tick(); + } + Err(ETIMEDOUT) +} + +/// Disable idle descent through fresh pair registration, including a post-wake reassertion. +pub(super) fn acquire(control: &mut C) -> kernel::error::Result { + use kernel::prelude::*; + let result = (|| { + for _ in 0..3 { + let powered = control.powered(); + let target = control.set_idle(false)?; + wait(control, target, true)?; + if !powered { + let target = control.set_idle(false)?; + wait(control, target, true)?; + } + if control.powered() { + return Ok(()); + } + } + Err(EAGAIN) + })(); + if result.is_err() { + let _ = release(control); + } + result +} + +pub(super) fn release(control: &mut C) -> kernel::error::Result { + let target = control.set_idle(true)?; + wait(control, target, false) +} diff --git a/drivers/gpu/drm/asahi/g17/preparation.rs b/drivers/gpu/drm/asahi/g17/preparation.rs new file mode 100644 index 00000000000000..8563beda979e66 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/preparation.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Admission and lifetime of off-lock graph construction. +//! +//! Entering and leaving an open epoch use only atomics. Recovery closes the epoch while holding +//! the device mutex, then drops that mutex before waiting for builders. A builder's lease is +//! stored after its tentative mappings so those mappings are destroyed before the join completes. + +use core::sync::atomic::{AtomicU64, Ordering}; +use kernel::{ + new_condvar, new_mutex, + prelude::*, + sync::{Arc, CondVar, Mutex}, +}; + +const COUNT_BITS: u32 = 20; +const COUNT_MASK: u64 = (1 << COUNT_BITS) - 1; +const CLOSED: u64 = 1 << COUNT_BITS; +const REMOVED: u64 = CLOSED << 1; +const FAILED: u64 = CLOSED << 2; +const EPOCH: u64 = CLOSED << 3; + +#[pin_data] +pub(super) struct Gate { + state: AtomicU64, + #[pin] + wait: Mutex<()>, + #[pin] + changed: CondVar, +} + +impl Gate { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state: AtomicU64::new(EPOCH), + wait <- new_mutex!((), "g17::Gate::wait"), + changed <- new_condvar!("g17::Gate::changed"), + }), + GFP_KERNEL, + ) + } + + pub(super) fn try_enter(self: &Arc) -> Result { + let mut previous = self.state.load(Ordering::Acquire); + loop { + if previous & (REMOVED | FAILED) != 0 { + return Err(ENODEV); + } + if previous & CLOSED != 0 { + return Err(EAGAIN); + } + if previous & COUNT_MASK == COUNT_MASK { + return Err(ENOSPC); + } + match self.state.compare_exchange_weak( + previous, + previous + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + return Ok(Lease { + gate: self.clone(), + epoch: previous & !COUNT_MASK, + }) + } + Err(current) => previous = current, + } + } + } + + /// Wait only for a temporary recovery closure. Called with no device lock or existing lease. + pub(super) fn enter(self: &Arc, interruptible: bool) -> Result { + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + let mut wait = self.wait.lock(); + loop { + match self.try_enter() { + Err(error) if error == EAGAIN => {} + result => return result, + } + if interruptible { + if self.changed.wait_interruptible(&mut wait) { + return Err(ERESTARTSYS); + } + } else { + self.changed.wait(&mut wait); + } + } + } + + fn invalidate(&self, flags: u64) { + let mut previous = self.state.load(Ordering::Acquire); + loop { + let mut next = previous | CLOSED | flags; + if previous & CLOSED == 0 { + next = next + .checked_add(EPOCH) + .unwrap_or(previous | CLOSED | REMOVED | flags); + } + match self.state.compare_exchange_weak( + previous, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return, + Err(current) => previous = current, + } + } + } + + pub(super) fn close(&self) { + self.invalidate(0); + } + + pub(super) fn remove(&self) { + self.invalidate(REMOVED); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + pub(super) fn fail(&self) { + self.invalidate(FAILED); + let _wait = self.wait.lock(); + self.changed.notify_all(); + } + + /// Join builders with no device, queue or mapping lock held. + pub(super) fn wait_drained(&self) { + let mut wait = self.wait.lock(); + while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { + self.changed.wait(&mut wait); + } + } + + /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. + pub(super) fn reopen(&self) -> bool { + let _wait = self.wait.lock(); + let mut previous = self.state.load(Ordering::Acquire); + loop { + if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { + return false; + } + match self.state.compare_exchange_weak( + previous, + previous & !CLOSED, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + self.changed.notify_all(); + return true; + } + Err(current) => previous = current, + } + } + } +} + +/// An off-lock builder's admission epoch. Keep after all of its tentative mappings. +pub(super) struct Lease { + gate: Arc, + epoch: u64, +} + +impl Lease { + /// Must be checked under the device mutex immediately before publishing the built graph. + pub(super) fn is_current(&self) -> bool { + self.gate.state.load(Ordering::Acquire) & !COUNT_MASK == self.epoch + } +} + +impl Drop for Lease { + fn drop(&mut self) { + let previous = self.gate.state.fetch_sub(1, Ordering::AcqRel); + if previous & COUNT_MASK == 1 && previous & CLOSED != 0 { + // Pair the final decrement with the joiner's check/sleep lock to avoid a lost wakeup. + let _wait = self.gate.wait.lock(); + self.gate.changed.notify_all(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs new file mode 100644 index 00000000000000..8a925c23552a73 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Shared QoS publication and exact-completion accounting for every engine. +//! +//! The device publication lock serializes this host state. Publishing identity +//! precedes submitted counters; retiring a final owner clears only its two row +//! bytes. Cancellation balances host references without rewinding shared counts. + +use super::{fw::qos as fw, object::KernelObject}; +use core::{ + mem::{offset_of, size_of}, + sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, +}; +use kernel::prelude::*; +use crate::hw::t8140::qos as cfg; + +/// Atomic views retain the allocation borrow and never borrow firmware words +/// as ordinary mutable memory. The arrays are disjoint apart from the explicit +/// data-master time/count indexing in `times_and_counts`. +pub(crate) struct View<'a> { + rows: &'a [[AtomicU8; 8]; fw::QUEUES], + queue_submitted: &'a [AtomicU32; fw::QUEUES], + owner_submitted: &'a [AtomicU32; fw::QUEUES], + scheduler: &'a [AtomicU64; fw::QUEUES], + share: &'a [AtomicU32; fw::QUEUES], + intervals: &'a AtomicU64, + times_and_counts: &'a [AtomicU64; 7], +} +impl<'a> View<'a> { + pub(crate) fn new(object: &'a KernelObject) -> Result { + let ptr = object.pointer(0, size_of::())?; + if ptr as usize & 7 != 0 { + return Err(EINVAL); + } + // SAFETY: pointer spans the complete layout, whose asserted offsets + // and size bound every array below. Integer atomics have their integer + // representation and alignment. The retained object borrow keeps every + // shared reference alive; accesses use each field's fixed width. + Ok(unsafe { + Self { + rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), + queue_submitted: &*ptr.add(offset_of!(fw::Qos, queue_submitted)).cast(), + owner_submitted: &*ptr.add(offset_of!(fw::Qos, owner_submitted)).cast(), + scheduler: &*ptr.add(offset_of!(fw::Qos, scheduler)).cast(), + share: &*ptr.add(offset_of!(fw::Qos, share)).cast(), + intervals: &*ptr.add(offset_of!(fw::Qos, intervals)).cast(), + times_and_counts: &*ptr.add(offset_of!(fw::Qos, busy_start)).cast(), + } + }) + } + fn row(&self, qid: usize) -> [u8; 8] { + core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) + } +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Owner { + pub(crate) qid: u8, + pub(crate) qos: u8, + pub(crate) data_master: u8, +} +impl Owner { + fn indices(self) -> Result<(usize, usize, usize)> { + let (qid, qos, dm) = ( + usize::from(self.qid), + usize::from(self.qos), + usize::from(self.data_master), + ); + if qid >= fw::QUEUES || qos >= fw::QUEUES || dm >= fw::DATA_MASTERS { + return Err(EINVAL); + } + Ok((qid, qos, dm)) + } +} + +/// Retained until completion, or cancellation before the producer is exposed. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Publication { + pub(crate) record: [u8; 8], + owner: Owner, + dm_submitted: u64, +} + +pub(crate) struct Accounting { + in_flight: [u32; fw::QUEUES], + dm_in_flight: [u64; fw::DATA_MASTERS], + dm_by_qid: [u8; fw::QUEUES], +} +impl Accounting { + pub(crate) const fn new() -> Self { + Self { + in_flight: [0; fw::QUEUES], + dm_in_flight: [0; fw::DATA_MASTERS], + dm_by_qid: [0; fw::QUEUES], + } + } + + /// Called under the device publication lock immediately before exposing + /// the work record. The clock is sampled only when its engine becomes busy. + pub(crate) fn publish( + &mut self, + view: &View<'_>, + owner: Owner, + scheduler: u64, + clock: impl FnOnce() -> (u64, u64), + ) -> Result { + let (qid, qos, dm) = owner.indices()?; + if scheduler == 0 { + return Err(EINVAL); + } + let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; + let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; + let starts_busy = self.dm_in_flight[dm] == 0; + let time = if starts_busy { + let (counter, frequency) = clock(); + time_from_host(counter, frequency)? + } else { + 0 + }; + if self.in_flight[qid] != 0 + && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos + || self.dm_by_qid[qid] != owner.data_master) + { + return Err(EBUSY); + } + view.rows[qid][0].store(owner.qos, Ordering::Relaxed); + view.rows[qid][1].store(cfg::CLASS, Ordering::Relaxed); + let record = view.row(qid); + view.scheduler[qos].store(scheduler, Ordering::Relaxed); + view.share[qos].store(cfg::SHARE, Ordering::Relaxed); + if self.in_flight[qid] == 0 { + fence(Ordering::SeqCst); + } + view.queue_submitted[qid].store( + view.queue_submitted[qid] + .load(Ordering::Relaxed) + .wrapping_add(1), + Ordering::Relaxed, + ); + view.owner_submitted[qos].store( + view.owner_submitted[qos] + .load(Ordering::Relaxed) + .wrapping_add(1), + Ordering::Relaxed, + ); + if starts_busy { + view.times_and_counts[dm].store(time, Ordering::Relaxed); + } + let submitted = &view.times_and_counts[3 + dm]; + let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); + submitted.store(dm_submitted, Ordering::Relaxed); + if starts_busy { + view.intervals.store( + view.intervals.load(Ordering::Relaxed).wrapping_add(1), + Ordering::Relaxed, + ); + } + self.in_flight[qid] = next_qid; + self.dm_in_flight[dm] = next_dm; + self.dm_by_qid[qid] = owner.data_master; + fence(Ordering::SeqCst); + Ok(Publication { + record, + owner, + dm_submitted, + }) + } + + /// A possibly exposed producer must retain its reference until completion. + /// A newer submission on the same engine makes an old ticket uncancellable. + pub(crate) fn cancel(&mut self, view: &View<'_>, ticket: Publication) -> Result { + let (qid, _, dm) = ticket.owner.indices()?; + if self.in_flight[qid] == 0 + || self.dm_in_flight[dm] == 0 + || view.row(qid) != ticket.record + || view.times_and_counts[3 + dm].load(Ordering::Relaxed) != ticket.dm_submitted + { + return Err(EIO); + } + self.complete(view, &[ticket.owner]) + } + + /// Preflights the entire set before clearing any final row. QID membership, + /// scheduler mappings and monotonically increasing counts remain intact. + pub(crate) fn complete(&mut self, view: &View<'_>, owners: &[Owner]) -> Result { + let mut completed = [0u64; fw::DATA_MASTERS]; + for (index, owner) in owners.iter().copied().enumerate() { + let (qid, _, dm) = owner.indices()?; + if owners[..index].iter().any(|other| other.qid == owner.qid) { + return Err(EINVAL); + } + if self.in_flight[qid] == 0 + || view.rows[qid][0].load(Ordering::Relaxed) != owner.qos + || self.dm_by_qid[qid] != owner.data_master + { + return Err(EIO); + } + completed[dm] += 1; + } + for (dm, count) in completed.iter().enumerate() { + if *count > self.dm_in_flight[dm] { + return Err(EIO); + } + } + for owner in owners { + let qid = usize::from(owner.qid); + if self.in_flight[qid] == 1 { + view.rows[qid][0].store(0xff, Ordering::Relaxed); + view.rows[qid][1].store(0, Ordering::Relaxed); + } + } + for owner in owners { + self.in_flight[usize::from(owner.qid)] -= 1; + } + for (dm, count) in completed.iter().enumerate() { + self.dm_in_flight[dm] -= count; + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +/// Firmware accounting uses a 24 MHz domain independent of host timer speed. +pub(crate) fn time_from_host(counter: u64, frequency: u64) -> Result { + if frequency == 0 || frequency > u64::from(u32::MAX) { + return Err(EINVAL); + } + let whole = (counter / frequency) + .checked_mul(cfg::CLOCK_HZ) + .ok_or(EOVERFLOW)?; + let fraction = (counter % frequency) * cfg::CLOCK_HZ / frequency; + whole.checked_add(fraction).ok_or(EOVERFLOW) +} + +pub(crate) fn clock() -> (u64, u64) { + let counter: u64; + let frequency: u64; + // SAFETY: Both timer registers are readable at EL1 and have no memory effects. + unsafe { + core::arch::asm!("mrs {counter}, CNTPCT_EL0", "mrs {frequency}, CNTFRQ_EL0", + counter=out(reg)counter, frequency=out(reg)frequency, options(nomem,nostack,preserves_flags)); + } + (counter, frequency) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/admission.rs b/drivers/gpu/drm/asahi/g17/queue/admission.rs new file mode 100644 index 00000000000000..1c0d6822c395f6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/admission.rs @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Logical queue and physical render-pair admission. A retained pair belongs +//! to one logical owner even while idle. Closing that owner makes it reusable +//! only after every active lease retires. Quarantine retains these leases until +//! a firmware drain witness or both processors stopping makes reuse safe. +//! +//! The pool lock protects host accounting only and is never held while building +//! a graph or waiting for GPU work. Capacity waiters sleep on the release event. + +use crate::g17::{ + hazard::{self, Footprint, Plan}, + job::Packet, +}; +use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::{ + new_condvar, new_mutex, + prelude::*, + sync::{Arc, CondVar, CondVarTimeoutResult, Mutex}, +}; + +#[derive(Clone, Copy)] +pub(crate) struct Request<'a> { + footprint: &'a Arc, + sequence: u64, + prefix: Option, +} + +impl<'a> Request<'a> { + pub(crate) fn of(packet: &'a Packet) -> Option { + packet.footprint.as_ref().map(|footprint| Self { + footprint, + sequence: packet.order.sequence, + prefix: packet.order.wait_through[0], + }) + } +} + +struct InFlight { + owner: u64, + sequence: u64, + slot: Option, + ticket: u64, + footprint: Arc, +} + +const QUARANTINE_WAIT_MS: u32 = 1000; + +struct State { + active: u64, + refs: [u8; cfg::RENDER_SLOTS], + owners: [u64; cfg::RENDER_SLOTS], + epochs: [u64; cfg::RENDER_SLOTS], + quarantined: u64, + kill_hold: u64, + released: u64, + epoch: u64, + footprints: KVec, + next_ticket: u64, +} +impl State { + const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; + const fn new() -> Self { + Self { + active: 0, + refs: [0; cfg::RENDER_SLOTS], + owners: [0; cfg::RENDER_SLOTS], + epochs: [0; cfg::RENDER_SLOTS], + quarantined: 0, + kill_hold: 0, + released: 0, + epoch: 0, + footprints: KVec::new(), + next_ticket: 1, + } + } + fn exhausted(&self) -> bool { + self.quarantined & Self::MASK == Self::MASK + } + fn refusal(&self, owner: u64) -> Result { + if self.exhausted() { + return Err(EIO); + } + match self.owners.iter().position(|id| *id == owner) { + Some(slot) if self.quarantined & (1u64 << slot) != 0 => Err(EIO), + None if !self.owners[1..].contains(&0) + && self.released + & !self.active + & !self.quarantined + & !self.kill_hold + & Self::MASK + == 0 => + { + Err(ENOSPC) + } + _ => Ok(()), + } + } + fn advance(&mut self, slot: usize) { + self.epoch = self.epoch.wrapping_add(1); + self.epochs[slot] = self.epoch; + } + fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { + self.refs[slot] += 1; + self.active |= 1u64 << slot; + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.advance(slot); + slot as u8 + } + fn acquire(&mut self, owner: u64) -> Option { + if owner == 0 { + return None; + } + // Prefer the owner's idle pair. A missing sibling can be claimed + // only while this owner has no live kick, before constructing it. + for slot in 0..cfg::RENDER_SLOTS { + if self.owners[slot] == owner + && self.refs[slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + { + if !(0..cfg::RENDER_SLOTS) + .any(|other| self.owners[other] == owner && self.refs[other] != 0) + { + self.claim_sibling(owner); + } + return Some(self.take_idle(slot, owner)); + } + } + if self.owners.contains(&owner) { + // Consecutive kicks are ordered by firmware dependencies. Stack + // onto the least-loaded retained pair without a retirement wait. + let slot = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + self.owners[*slot] == owner + && self.quarantined & (1u64 << slot) == 0 + && self.active & (1u64 << slot) != 0 + && self.refs[*slot] < cfg::RENDER_DEPTH + }) + .min_by_key(|slot| (self.refs[*slot], u64::MAX - self.epochs[*slot]))?; + self.refs[slot] += 1; + self.advance(slot); + return Some(slot as u8); + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + // Reuse a closed, exactly retired owner before consuming a new graph. + let slot = (0..cfg::RENDER_SLOTS) + .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) + .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| { + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + })?; + let first = self.take_idle(slot, owner); + self.claim_sibling(owner); + Some(first) + } + fn claim_sibling(&mut self, owner: u64) { + if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { + return; + } + let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; + let eligible = |slot: usize| free & (1u64 << slot) != 0; + let sibling = (0..cfg::RENDER_SLOTS) + .filter(|slot| { + eligible(*slot) + && self.owners[*slot] != owner + && self.released & (1u64 << slot) != 0 + }) + .min_by_key(|slot| self.epochs[*slot]) + .or_else(|| { + (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) + }); + if let Some(slot) = sibling { + self.released &= !(1u64 << slot); + self.owners[slot] = owner; + self.advance(slot); + } + } + fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + if self.refs.get(usize::from(publishing)) != Some(&1) { + return None; + } + (0..cfg::RENDER_SLOTS) + .find(|slot| { + *slot != usize::from(publishing) + && self.owners[*slot] == owner + && self.refs[*slot] == 0 + && self.quarantined & (1u64 << slot) == 0 + }) + .map(|slot| slot as u8) + } + fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { + let Some(request) = request else { + return Plan::Any; + }; + hazard::plan( + owner, + request.prefix, + ticket, + request.footprint, + self.footprints.iter().map(|entry| hazard::Live { + owner: entry.owner, + sequence: entry.sequence, + slot: entry.slot, + ticket: entry.ticket, + footprint: &entry.footprint, + quarantined: entry + .slot + .is_some_and(|slot| self.quarantined & (1u64 << slot) != 0), + }), + ) + } + + fn pick(&mut self, owner: u64, plan: Plan) -> Option { + match plan { + Plan::Any => self.acquire(owner), + Plan::Wait => None, + Plan::Pair(slot) => { + let index = usize::from(slot); + if self.owners.get(index) != Some(&owner) + || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0 + || self.refs[index] >= cfg::RENDER_DEPTH + { + return None; + } + if self.refs[index] == 0 { + Some(self.take_idle(index, owner)) + } else { + self.refs[index] += 1; + self.advance(index); + Some(slot) + } + } + } + } + + fn record( + &mut self, + owner: u64, + slot: Option, + ticket: u64, + request: Request<'_>, + ) -> Option<(u64, u64)> { + self.footprints + .push( + InFlight { + owner, + sequence: request.sequence, + slot, + ticket, + footprint: request.footprint.clone(), + }, + GFP_KERNEL, + ) + .ok()?; + Some((owner, request.sequence)) + } + + fn forget_waiter(&mut self, ticket: Option) { + if let Some(ticket) = ticket { + self.footprints + .retain(|entry| entry.slot.is_some() || entry.ticket != ticket); + } + } + + fn release(&mut self, slot: u8) { + let slot = usize::from(slot); + if self.refs[slot] != 0 { + self.refs[slot] -= 1; + } + if self.refs[slot] == 0 { + self.active &= !(1u64 << slot); + } + } +} + +#[pin_data] +pub(crate) struct Pool { + #[pin] + state: Mutex, + #[pin] + released: CondVar, + queues: AtomicU32, +} +impl Pool { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State::new(), "G17 render admission"), + released <- new_condvar!("G17 render pair release"), + queues: AtomicU32::new(0), + }), + GFP_KERNEL, + ) + } + pub(crate) fn reserve_queue(self: &Arc) -> Result { + let mut count = self.queues.load(Ordering::Acquire); + loop { + if count >= MAX_EXECUTION_CONTEXTS { + return Err(ENOSPC); + } + match self.queues.compare_exchange( + count, + count + 1, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => return Ok(QueueSlot { pool: self.clone() }), + Err(current) => count = current, + } + } + } + pub(crate) fn acquire_render( + self: &Arc, + owner: u64, + request: Option>, + ) -> Result { + if owner == 0 { + return Err(EINVAL); + } + let mut state = self.state.lock(); + let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); + let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); + let mut expired = false; + let mut ticket = None; + let result = (|| loop { + if state.exhausted() { + return Err(EIO); + } + // The first retained pair's quarantine is not bypassed through its + // healthy sibling. Reclamation wakes this bounded admission wait. + if state + .owners + .iter() + .position(|id| *id == owner) + .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) + { + if quarantine_budget == 0 { + return Err(EIO); + } + quarantine_budget = match self + .released + .wait_interruptible_timeout(&mut state, quarantine_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + continue; + } + state.refusal(owner)?; + let plan = if expired { + Plan::Any + } else { + state.plan(owner, request, ticket) + }; + if let Some(slot) = state.pick(owner, plan) { + state.forget_waiter(ticket.take()); + let record = + request.and_then(|request| state.record(owner, Some(slot), 0, request)); + return Ok(RenderLease { + pool: self.clone(), + slot, + record, + }); + } + if plan != Plan::Any { + if ticket.is_none() { + if let Some(request) = request { + let next = state.next_ticket; + state.next_ticket = next.wrapping_add(1); + if state.record(owner, None, next, request).is_some() { + ticket = Some(next); + } + } + } + match self + .released + .wait_interruptible_timeout(&mut state, conflict_budget) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => { + conflict_budget = 0; + expired = true; + } + CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, + } + } else if self.released.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + })(); + state.forget_waiter(ticket); + result + } + + /// Early publication never waits for overlap; the scheduler retries with + /// the same footprint after its dependencies have become ready. + pub(crate) fn try_acquire_render( + self: &Arc, + owner: u64, + request: Option>, + ) -> Result> { + if owner == 0 { + return Ok(None); + } + let mut state = self.state.lock(); + if state.refusal(owner).is_err() || !state.owners.contains(&owner) { + return Ok(None); + } + let plan = state.plan(owner, request, None); + let Some(slot) = state.pick(owner, plan) else { + return Ok(None); + }; + let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); + Ok(Some(RenderLease { + pool: self.clone(), + slot, + record, + })) + } + pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { + self.state.lock().warmable_sibling(owner, publishing) + } + pub(crate) fn release_owner(&self, owner: u64) { + let mut state = self.state.lock(); + for slot in 0..cfg::RENDER_SLOTS { + if state.owners[slot] == owner { + state.released |= 1u64 << slot; + } + } + drop(state); + self.released.notify_all(); + } + /// Active leases survive this transition. Their later drop is still the + /// event which releases capacity, so a stopped owner is never double-freed. + pub(crate) fn release_retained_owners_after_processor_stop(&self) { + let mut state = self.state.lock(); + let released = state.released & !state.active; + for slot in 1..cfg::RENDER_SLOTS { + if released & (1u64 << slot) != 0 { + state.owners[slot] = 0; + state.epochs[slot] = 0; + } + } + state.released &= !released; + state.quarantined &= !released; + state.kill_hold = 0; + drop(state); + self.released.notify_all(); + } +} + +/// Counted once per userspace logical queue, independently of installed graphs. +pub(crate) struct QueueSlot { + pool: Arc, +} +impl Drop for QueueSlot { + fn drop(&mut self) { + self.pool.queues.fetch_sub(1, Ordering::AcqRel); + } +} + +pub(crate) struct RenderLease { + pool: Arc, + slot: u8, + record: Option<(u64, u64)>, +} +impl RenderLease { + pub(crate) fn slot(&self) -> u8 { + self.slot + } + pub(crate) fn quarantine(&self) { + self.pool.state.lock().quarantined |= 1u64 << self.slot; + self.pool.released.notify_all(); + } + /// The VM pin must be retired first. Keep this lease until after clearing + /// quarantine; its drop is what permits waiting publications to proceed. + pub(crate) fn rehabilitate_after_processor_stop(&self) { + self.pool.state.lock().quarantined &= !(1u64 << self.slot); + } + /// Requires the runtime's closed-recovery, exact pair-drain witness. + pub(crate) fn rehabilitate_after_firmware_drain(&self) { + self.rehabilitate_after_processor_stop(); + } +} +impl Drop for RenderLease { + fn drop(&mut self) { + let mut state = self.pool.state.lock(); + if let Some((owner, sequence)) = self.record { + if let Some(index) = state.footprints.iter().position(|entry| { + entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence + }) { + state.footprints.swap_remove(index); + } + } + state.release(self.slot); + drop(state); + self.pool.released.notify_all(); + } +} + +impl State { + fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { + if owner == 0 || slots & !Self::MASK != 0 { + return false; + } + for slot in 0..cfg::RENDER_SLOTS { + let bit = 1u64 << slot; + if slots & bit == 0 { + continue; + } + if self.owners[slot] != owner + || self.refs[slot] != 0 + || self.active & bit != 0 + || self.released & bit == 0 + { + return false; + } + } + self.kill_hold |= slots; + true + } +} +impl Pool { + /// Hold the departed owner's idle pairs before publishing its exit record. + /// Successful publication retains these bits until both processors stop. + pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { + self.state.lock().hold_for_exit_kill(owner, slots) + } + + /// Undo only a hold whose control record was not published. + pub(crate) fn release_exit_kill_hold(&self, slots: u64) { + if slots == 0 { + return; + } + self.state.lock().kill_hold &= !slots; + self.released.notify_all(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs new file mode 100644 index 00000000000000..f8297f0462ff8e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -0,0 +1,1507 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained physical compute queues. Logical ownership and command retirement never free an +//! installed graph; the device retains it until both firmware processors have stopped. + +pub(crate) mod exit; +pub(crate) mod retirement; + +use crate::g17::{ + context::Context, + fw::{compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + object::{Allocator, CpuMap, KernelObject}, +}; +use crate::{hw::t8140::dynamic, mmu}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; +use retirement::{Cursors, RECORD_SLOTS}; + +const DESCRIPTORS: usize = 256; +const GRAPH_SIZE: usize = 0x8000; +const CONTEXT_SIZE: usize = 0x20000; +const RECORD_STRIDE: usize = 0x1000; +const CONFIG: usize = 0; +const ANNOUNCE: usize = 0x800; +const PREDECESSOR: usize = 0x400; +const POINTERS: usize = 0x100; +const ITEMS: usize = 0x4000; +const STAMP: usize = 0x300; + +#[repr(C)] +struct GraphLayout { + queue: QueueRecord, + unk_c0: [u8; 0x40], + pointers: PointerBlock, + unk_180: [u8; 0x80], + jobs: JobListHead, + unk_210: [u8; 0x3df0], + items: [u64; ITEM_RING_ENTRIES as usize], + unk_6800: [u8; 0x1800], +} +// SAFETY: The layout contains only integer storage and integer atomics, all valid when zeroed. +unsafe impl Zeroable for GraphLayout {} +static_assert!(core::mem::size_of::() == GRAPH_SIZE); +static_assert!(core::mem::offset_of!(GraphLayout, pointers) == POINTERS); +static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); + +/// Aliases are destroyed before the logical root they name. Superseded bindings remain owned +/// until the first exact successful publication under their replacement has retired. +struct Binding { + _kick_alias: mmu::KernelMapping, + _pool_alias: mmu::KernelMapping, + _descriptors: mmu::KernelMapping, + preempt: mmu::KernelMapping, + usage: mmu::KernelMapping, + operand: mmu::KernelMapping, + _queue_context: mmu::KernelMapping, + context: Arc, +} + +/// Firmware-shared storage of one compute QID, separate from its common kick ring. +struct Graph { + descriptor_high: mmu::KernelMapping, + descriptors: KernelObject, + preempt: KernelObject, + usage: KernelObject, + operand: KernelObject, + support: KernelObject, + graph: KernelObject, + context_low: KernelObject, + context_high: KernelObject, + records: KernelObject, + record_ends: [Option; RECORD_SLOTS], + reclaimed: u32, +} + +impl Graph { + fn new(alloc: &Allocator<'_>, context: &Context) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let cpu = CpuMap::WriteCombined; + let descriptors = alloc.lower(DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, page, prot, cpu)?; + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let descriptor_high = descriptors.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + let preempt = alloc.kernel(0x4000, page, prot, cpu)?; + let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; + let operand = alloc.lower(0x14000, page, prot, cpu)?; + let mut support = alloc.kernel(0xc000, page, prot, cpu)?; + let support_va = support.gpu_va(); + support.write(0, support_va + page)?; + support.write(0x100, support_va + page + 4)?; + support.write(0x110, 0x50u32)?; + support.write(mmu::UAT_PGSZ + 4, 1u32)?; + let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; + let graph_va = graph.gpu_va(); + graph.initialize::(0, |view| { + view.queue = QueueRecord::new(&QueueRecordArgs { + pointers_va: graph_va + POINTERS as u64, + items_va: graph_va + ITEMS as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + }); + view.pointers = PointerBlock::new(); + view.jobs = JobListHead::new(graph_va + 0x200); + Ok(()) + })?; + let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; + let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; + let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; + Ok(Self { + descriptor_high, + descriptors, + preempt, + usage, + operand, + support, + graph, + context_low, + context_high, + records, + record_ends: [None; RECORD_SLOTS], + reclaimed: 0, + }) + } + + /// All fallible mapping work finishes before replacing a physical queue's logical owner. + fn bind( + &mut self, + context: Arc, + kick_alias: mmu::KernelMapping, + pool_alias: mmu::KernelMapping, + ) -> Result { + if !context.is_current() { + return Err(EFAULT); + } + let vm = context.vm(); + vm.install_job_context_aliases()?; + let descriptors = + self.descriptors + .map_alias(vm, self.descriptors.gpu_va(), mmu::PROT_GPU_SHARED_RO)?; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let page = mmu::UAT_PGSZ as u64; + let preempt = self + .preempt + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let usage = self.usage.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let operand = self + .operand + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + let queue_context = self + .context_low + .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; + Ok(Binding { + _kick_alias: kick_alias, + _pool_alias: pool_alias, + _descriptors: descriptors, + preempt, + usage, + operand, + _queue_context: queue_context, + context, + }) + } + + /// Caller has excluded publication and proved the retained graph idle. + fn set_owner(&mut self, context: &Context) -> Result { + let pointer = self + .graph + .pointer(0, core::mem::size_of::())? + .cast::(); + // SAFETY: The checked record is in the retained mapping, has byte alignment and consists + // only of integer storage. The device mutex and idle witness exclude simultaneous access. + unsafe { + (*pointer).set_compute_owner( + context.work_head_va(), + context.scheduler_va(), + context.owner_pid(), + ) + }; + Ok(()) + } + + fn cursors(&self) -> Result { + Ok(Cursors { + producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), + consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), + count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), + }) + } + fn stamp(&self) -> Result { + fence(Ordering::Acquire); + Ok(self.support.word(STAMP)?.load(Ordering::Relaxed)) + } + fn descriptor_vas(&self, slot: u8) -> (u64, u64) { + let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; + ( + self.descriptor_high.iova() + offset, + self.descriptors.gpu_va() + offset, + ) + } + fn ensure_record_slot(&self, slot: usize) -> Result { + if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { + let cursors = self.cursors()?; + if !cursors.consumed(end, cursors.count)? { + return Err(EBUSY); + } + } + Ok(()) + } + + /// Reclaim only consumed item pointers, with individual aligned stores and a release fence. + /// This leaves records still named by firmware untouched across arbitrarily many wraps. + fn recycle(&mut self, cursors: Cursors) -> Result { + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.consumer >= cursors.count || self.reclaimed >= cursors.count { + return Err(ERANGE); + } + let done = cursors.consumer; + while self.reclaimed != done { + self.graph + .dword(ITEMS + self.reclaimed as usize * 8)? + .store(0, Ordering::Relaxed); + self.reclaimed = (self.reclaimed + 1) % cursors.count; + } + fence(Ordering::SeqCst); + Ok(()) + } +} + +use crate::g17::{ + completion::Deferred, context::WorkStateLease, freelist::ComputePool, job::Packet, kick, qos, + recovery, status::VmStatus, +}; +use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; + +struct Fifo { + entries: [Option; IN_FLIGHT], + head: usize, + len: usize, +} +impl Fifo { + fn new() -> Self { + Self { + entries: core::array::from_fn(|_| None), + head: 0, + len: 0, + } + } + fn front(&self) -> Option<&T> { + self.entries[self.head].as_ref() + } + fn back(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_ref() + } + } + fn back_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_mut() + } + } + fn push(&mut self, value: T) -> core::result::Result<(), T> { + if self.len == IN_FLIGHT { + return Err(value); + } + self.entries[(self.head + self.len) % IN_FLIGHT] = Some(value); + self.len += 1; + Ok(()) + } + fn pop(&mut self) -> Option { + if self.len == 0 { + return None; + } + let value = self.entries[self.head].take(); + self.head = (self.head + 1) % IN_FLIGHT; + self.len -= 1; + value + } + fn pop_back(&mut self) -> Option { + if self.len == 0 { + return None; + } + self.len -= 1; + self.entries[(self.head + self.len) % IN_FLIGHT].take() + } + fn iter(&self) -> impl Iterator { + Iterator::chain( + self.entries[self.head..].iter(), + self.entries[..self.head].iter(), + ) + .take(self.len) + .filter_map(Option::as_ref) + } + fn iter_mut(&mut self) -> impl Iterator { + let (before, after) = self.entries.split_at_mut(self.head); + Iterator::chain(after.iter_mut(), before.iter_mut()) + .take(self.len) + .filter_map(Option::as_mut) + } +} + +struct Active { + packet: Arc, + work: Arc, + ticket: Option, + observed: Option, + failure_deferred: bool, + qos_completed: bool, +} +struct Previous { + binding: Option, + context: Arc, + publication: Option<(u64, u64)>, +} +#[derive(Copy, Clone, PartialEq, Eq)] +struct ConfigIdentity { + scheduler: u64, + qos: u8, + context: u32, +} + +/// A private physical pool whose descriptor row is installed before allocating its kick ring. +/// The QID remains unpublished throughout construction. +pub(crate) struct PoolBacking { + pool: ComputePool, + id: kick::Id, +} + +impl PoolBacking { + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id) -> Result { + let pool = ComputePool::new(alloc, id.qid())?; + Ok(Self { pool, id }) + } + + /// Fresh page-pool row data, installed once while the QID is still unpublished. + pub(crate) fn pool_descriptor(&self) -> (u16, u64) { + (self.pool.id(), self.pool.page_list_va()) + } +} + +/// A physical pool and kick ring prepared before the device-global compute scheduler and +/// descriptor graph. No logical VM aliases or firmware publication exist at this stage. +pub(crate) struct Backing { + pool: ComputePool, + kick: kick::Queue, +} + +impl Backing { + pub(crate) fn new(alloc: &Allocator<'_>, backing: PoolBacking) -> Result { + let PoolBacking { pool, id } = backing; + let kick = kick::Queue::compute(alloc, id)?; + Ok(Self { pool, kick }) + } +} + +/// Installed graph and its exact logical owner. A released idle graph may change owners only +/// after a retirement witness. Failed work retains its pointers independently of fence state. +pub(crate) struct Queue { + owner: Option, + binding: Binding, + previous: Option, + kick: kick::Queue, + pool: ComputePool, + graph: Graph, + active: Fifo, + replays: Fifo>, + replay_retirement: bool, + replay_error: Option, + replay_prepared: bool, + exit: Option, + installed: bool, + outer_started: bool, + published_config: Option, + ordinal: u32, + submitted: u32, + last_end: u64, + released: bool, + retirement_ready: bool, + quarantined: bool, + retired_by_teardown: bool, + retire_pending: bool, + awaiting_witness: bool, + spared_quarantine: bool, + spared_deferred: bool, + terminal: Option<(Arc, Error)>, + quarantine_error: Option, + failure_status_pending: bool, + retirement_proved: bool, +} + +impl Queue { + /// The caller reserves a QID first, cancels it on construction failure, and retains every + /// successfully installed queue until processor stop. Shared VM apertures precede this call. + pub(crate) fn new( + alloc: &Allocator<'_>, + backing: Backing, + owner: u64, + context: Arc, + ) -> Result { + let Backing { mut pool, mut kick } = backing; + let mut graph = Graph::new(alloc, &context)?; + let kick_alias = kick.map_client(context.vm())?; + let pool_alias = pool.map_client(context.vm())?; + let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; + context.mark_published(); + Ok(Self { + owner: Some(owner), + binding, + previous: None, + kick, + pool, + graph, + active: Fifo::new(), + replays: Fifo::new(), + replay_retirement: false, + replay_error: None, + replay_prepared: false, + exit: None, + installed: false, + outer_started: false, + published_config: None, + ordinal: 0, + submitted: 0, + last_end: 0, + released: false, + retirement_ready: false, + quarantined: false, + retired_by_teardown: false, + retire_pending: false, + awaiting_witness: false, + spared_quarantine: false, + spared_deferred: false, + terminal: None, + quarantine_error: None, + failure_status_pending: false, + retirement_proved: false, + }) + } + pub(crate) fn qid(&self) -> u8 { + self.kick.id().qid() + } + pub(crate) fn owner(&self) -> Option { + self.owner + } + pub(crate) fn context(&self) -> &Arc { + &self.binding.context + } + /// Quarantined commands still retain the scheduler until their retirement witness. + pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { + !Arc::ptr_eq(self.context(), context) || (self.active.len == 0 && self.replays.len == 0) + } + pub(crate) fn in_flight(&self) -> bool { + self.active.len != 0 && !self.quarantined + } + pub(crate) fn room(&self) -> bool { + !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT + } + pub(crate) fn oldest_spared(&self) -> Option { + self.active + .front() + .map(|a| a.packet.completion.spared() && a.packet.completion.replay_count() == 0) + } + pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { + for active in self.active.iter() { + visit(active.packet.completion.status()); + } + } + pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { + for active in self.active.iter() { + active + .packet + .completion + .classify(class(active.packet.completion.status())); + } + } + fn qos_owner(&self) -> qos::Owner { + qos::Owner { + qid: self.qid(), + qos: self.binding.context.qos_id(), + data_master: DataMaster::Compute as u8, + } + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { + self.retired_by_teardown + || self.retire_pending + || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) + } + /// Stage every unlatched exact identity before the device's single shared-ring scan. + pub(crate) fn request_observations( + &self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(false); + } + let mut requested = false; + for active in self.active.iter() { + let Some(ticket) = active.ticket else { + break; + }; + if active.observed.is_none() { + batch.request(&ticket)?; + requested = true; + } + } + Ok(requested) + } + pub(crate) fn latch_observations( + &mut self, + batch: &mut Batch, + masks: Option<[u64; 2]>, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(()); + } + for active in self.active.iter_mut() { + let Some(ticket) = active.ticket else { + break; + }; + if active.observed.is_none() { + active.observed = batch.take(&ticket)?; + } + } + Ok(()) + } + fn front_finished(&self) -> bool { + self.active.front().is_some_and(|active| { + active.observed.is_some() + || active.ticket.is_some_and(|ticket| { + self.graph + .cursors() + .and_then(|c| c.consumed(ticket.item_producer, ticket.item_count)) + .unwrap_or(false) + }) + }) + } + fn valid_ticket(&self, active: &Active, ticket: &Ticket) -> Result { + if ticket.qid != self.qid() + || ticket.descriptor == 0 + || ticket.item_count == 0 + || ticket.work_node != active.work.node_va()? + || self.submitted == 0 + || self.owner != Some(ticket.owner) + || active.packet.order.sequence != ticket.submission + || active.packet.context.id() != ticket.context_id + || active.packet.context.id() != self.binding.context.id() + { + return Err(EIO); + } + Ok(()) + } + + /// Return VM identity for queue-local failure fanout. Deferred callbacks are collected under + /// the mutex and invoked afterwards; failed packets stay retained until their witness. + pub(crate) fn quarantine( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result>> { + if !self.quarantined { + self.quarantined = true; + self.quarantine_error = Some(error); + self.failure_status_pending = true; + self.retirement_proved = false; + self.retired_by_teardown = false; + let all_spared = + self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); + self.spared_quarantine = all_spared && error == ENODATA; + self.spared_deferred = self.spared_quarantine; + } + let error = self.quarantine_error.ok_or(EIO)?; + let mut terminal = None; + for active in self.active.iter_mut() { + if !self.spared_deferred && !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + error, + ))?; + active.failure_deferred = true; + } + if self.failure_status_pending + && !active.packet.completion.spared() + && terminal.is_none() + { + terminal = Some(active.packet.completion.status().clone()); + } + } + self.failure_status_pending = false; + self.awaiting_witness = + self.active.len != 0 && self.active.iter().all(|a| a.ticket.is_some()); + Ok(terminal) + } + + /// Poll only the ordered prefix; the next pass may recheck a finished front on another QID's + /// event. A final identity-less scan allows recordless retirement without the one-pass grace. + pub(crate) fn retire( + &mut self, + masks: Option<[u64; 2]>, + epoch: u64, + recovery_state: u32, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.quarantined || !self.selected(masks) { + return Ok(()); + } + let finished_before = self.retire_pending; + self.retire_pending = false; + let mut oldest = true; + while let Some(front) = self.active.front() { + let result = (|| { + let ticket = front.ticket.ok_or(EIO)?; + self.valid_ticket(front, &ticket)?; + if recovery_state != 0 { + return Err(EAGAIN); + } + let recordless = if epoch != ticket.epoch { + Recordless::CrossedRecovery + } else if masks.is_none() || (finished_before && oldest) { + Recordless::Allowed + } else { + Recordless::NotYet + }; + let next_end = self + .active + .iter() + .skip(1) + .find_map(|a| a.observed.map(|o| o.timestamps[1])); + let timestamps = retirement::poll( + &ticket, + front.observed, + self.graph.cursors()?, + self.graph.stamp()?, + recordless, + self.last_end, + next_end, + )?; + if timestamps.is_some() + && self.previous.as_ref().is_some_and(|p| { + p.publication != Some((front.packet.order.sequence, ticket.epoch)) + }) + { + return Err(EIO); + } + Ok(timestamps) + })(); + oldest = false; + match result { + Ok(Some(timestamps)) => { + let submitted = self.submitted.checked_sub(1).ok_or(EIO)?; + if !front.qos_completed { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + } + let front = self.active.front().ok_or(EIO)?; + self.retire_pending = true; + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok([timestamps[0], timestamps[1], 0, 0]), + ))?; + self.retire_pending = false; + self.submitted = submitted; + let active = self.active.pop().ok_or(EIO)?; + if active.observed.is_some() { + self.last_end = self.last_end.max(timestamps[1]); + } + drop(self.previous.take()); + self.retirement_ready = self.active.len == 0; + if self.released && self.retirement_ready { + self.owner = None; + } + } + Ok(None) | Err(EAGAIN) => { + self.retire_pending = self.front_finished(); + break; + } + Err(error) => { + self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); + return Ok(()); + } + } + } + Ok(()) + } + + /// Exact terminal witness releases command pins, not installed graph ownership. Spared + /// zero-duration work can return its queue to service after settling its accounting. + pub(crate) fn release_witnessed( + &mut self, + recovery_state: u32, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.release_witnessed_inner(recovery_state, false, &mut complete_qos, defer) + } + + pub(crate) fn prepare_replays( + &mut self, + recovery_state: u32, + complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.release_witnessed_inner(recovery_state, true, complete_qos, defer) + } + + fn release_witnessed_inner( + &mut self, + recovery_state: u32, + replay: bool, + mut complete_qos: impl FnMut(qos::Owner) -> Result, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !self.awaiting_witness || recovery_state != 0 { + return Ok(()); + } + if !self.retirement_proved { + for active in self.active.iter() { + self.valid_ticket(active, &active.ticket.ok_or(EIO)?)?; + } + let newest = self.active.back().and_then(|a| a.ticket).ok_or(EIO)?; + if !retirement::quarantined_retired( + &newest, + self.graph.cursors()?, + self.graph.stamp()?, + self.active.len as u32, + self.submitted, + )? { + return Ok(()); + } + let candidate = self.spared_deferred + && self.spared_quarantine + && !self.released + && self.owner == Some(newest.owner) + && !self + .active + .iter() + .any(|active| active.packet.completion.has_firmware_consumer()) + && self.active.iter().all(|active| { + active + .packet + .completion + .replay_witness_words() + .is_some_and(|words| words[0] == words[1]) + }); + if candidate && !replay { + return Ok(()); + } + self.replay_retirement = candidate; + self.retirement_proved = true; + } + while let Some(front) = self.active.front() { + if self.spared_quarantine && !front.qos_completed { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + } + let front = self.active.front().ok_or(EIO)?; + let completion = front.packet.completion.clone(); + if self.replay_retirement { + self.replays.push(front.packet.clone()).map_err(|_| EIO)?; + } else if self.spared_deferred { + defer(Deferred::Retired(completion, Err(ENODATA)))?; + } else { + defer(Deferred::Release(completion))?; + } + drop(self.active.pop()); + } + self.awaiting_witness = false; + self.retirement_proved = false; + self.replay_retirement = false; + self.spared_deferred = false; + if self.spared_quarantine { + self.spared_quarantine = false; + self.quarantine_error = None; + self.submitted = 0; + self.kick.clear_parent_after_recovery(); + self.quarantined = false; + self.retire_pending = false; + self.retirement_ready = true; + if self + .previous + .as_ref() + .is_some_and(|p| p.publication.is_some()) + { + drop(self.previous.take()); + } + if self.released { + self.owner = None; + } + } + Ok(()) + } + + pub(crate) fn replay_front(&self) -> Option> { + self.replays.front().cloned() + } + + /// Drop the witnessed owner only after either publication retained it or + /// the off-lock completion collector accepted its terminal result. + pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { + if !self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { + return Err(EIO); + } + drop(self.replays.pop()); + self.replay_prepared = false; + if self.replays.len == 0 { + self.replay_error = None; + } + Ok(()) + } + + pub(crate) fn prepare_replay(&mut self, packet: &Arc) -> Result { + if !self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)) + { + return Err(EIO); + } + if let Some(error) = self.replay_error { + return Err(error); + } + if !self.replay_prepared { + if !packet.completion.begin_replay() { + return Err(EAGAIN); + } + packet.completion.reset_for_replay()?; + self.replay_prepared = true; + } + Ok(()) + } + + pub(crate) fn fail_replays(&mut self, error: Error) { + if self.replay_error.is_none() { + self.replay_error = Some(error); + } + } + + /// Caller already stopped both processors. No firmware owner can access retained mappings. + pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + while let Some(packet) = self.replays.front() { + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + drop(self.replays.pop()); + } + while let Some(active) = self.active.front() { + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(EIO), + ))?; + drop(self.active.pop()); + } + Ok(()) + } +} + +impl Queue { + pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { + if self.owner != Some(owner) { + return Ok(false); + } + if self.released + || self.quarantined + || self.qid() != qid + || !Arc::ptr_eq(&self.binding.context, context) + || !context.is_current() + { + return Err(EIO); + } + Ok(true) + } + pub(crate) fn reusable(&self) -> bool { + self.installed + // An issued kill still names this graph even if its commands retire + // before the acknowledgement. It cannot acquire a successor owner. + && self.exit.is_none() + && self.owner.is_none() + && self.released + && self.retirement_ready + && !self.quarantined + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + } + pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { + if !self.reusable() { + return Err(EBUSY); + } + if !context.is_current() { + return Err(EFAULT); + } + let old = self.binding.context.clone(); + if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { + self.graph.set_owner(&context)?; + self.binding.context = context.clone(); + self.previous = Some(Previous { + binding: None, + context: old, + publication: None, + }); + } else { + let kick_alias = self.kick.map_client(context.vm())?; + let pool_alias = self.pool.map_client(context.vm())?; + let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; + self.graph.set_owner(&context)?; + let old_binding = core::mem::replace(&mut self.binding, binding); + self.previous = Some(Previous { + binding: Some(old_binding), + context: old, + publication: None, + }); + } + context.mark_published(); + self.owner = Some(owner); + self.released = false; + self.retirement_ready = false; + self.retired_by_teardown = false; + Ok(()) + } + /// True allows the registry to cancel the never-published QID and drop this graph. An + /// installed graph remains retained even after the logical owner exits without work. + pub(crate) fn release_owner(&mut self) -> Result { + if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { + return Ok(true); + } + self.released = true; + if self.active.len == 0 && !self.quarantined { + if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { + if let Err(error) = self.graph.set_owner(&previous.context) { + self.quarantined = true; + self.retired_by_teardown = true; + return Err(error); + } + let mut previous = self.previous.take().ok_or(EIO)?; + if let Some(binding) = previous.binding.take() { + self.binding = binding; + } else { + self.binding.context = previous.context; + } + self.retirement_ready = true; + } + if self.retirement_ready { + self.owner = None; + } + } + Ok(false) + } + /// Exact host identity remains valid even when several commands share an ioctl ID. + pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { + self.active + .iter() + .any(|active| Arc::ptr_eq(&active.packet, packet)) + } + + /// A watchdog settles spared, deferred waiters without changing their VM status. + /// Fully ticketed spared work can still return this queue after its drain witness. + pub(crate) fn timeout( + &mut self, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result>> { + let fresh = !self.quarantined; + let return_spared = self.active.len != 0 + && self + .active + .iter() + .all(|active| active.packet.completion.spared() && active.ticket.is_some()); + if self.spared_deferred { + for active in self.active.iter_mut() { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ETIMEDOUT, + ))?; + active.failure_deferred = true; + } + } + self.spared_deferred = false; + } + let result = self.quarantine(ETIMEDOUT, defer); + if fresh { + self.spared_quarantine = return_spared; + } + result + } + + pub(crate) fn cancel( + &mut self, + packet: &Arc, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !self.owns_packet(packet) { + return Ok(false); + } + if self.quarantined { + if self.spared_deferred { + for active in self.active.iter_mut() { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ECANCELED, + ))?; + active.failure_deferred = true; + } + } + self.spared_deferred = false; + } + return Ok(true); + } + self.released = true; + self.retired_by_teardown = true; + if let Some(active) = self + .active + .iter_mut() + .find(|a| Arc::ptr_eq(&a.packet, packet)) + { + if !active.failure_deferred { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + ECANCELED, + ))?; + active.failure_deferred = true; + } + } + Ok(true) + } + /// Firmware is halted and publication is closed while its target is copied. + pub(crate) fn publish_recovery_target(&self) -> Result { + let producer = self + .graph + .graph + .word(POINTERS + PointerBlock::PRODUCER)? + .load(Ordering::Relaxed); + self.graph + .graph + .word(POINTERS + PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); + Ok(()) + } + + /// The newest covered kick of a QID subsumes its earlier prefix; later independent work + /// must not become a dependency. Errors are observed before returning any firmware token. + pub(crate) fn dependency( + &self, + owner: u64, + status: &Arc, + through: Option, + recovery_state: u32, + ) -> Result> { + if through.is_none() || self.owner != Some(owner) { + return Ok(None); + } + let Some(active) = self + .active + .iter() + .filter(|a| crate::g17::job::Order::contains(through, a.packet.order.sequence)) + .last() + else { + return Ok(None); + }; + if self.quarantined && self.spared_quarantine { + return Err(ECANCELED); + } + if !Arc::ptr_eq(active.packet.completion.status(), status) + || self.quarantined + || recovery_state != 0 + { + return Err(EIO); + } + let ticket = active.ticket.ok_or(EIO)?; + let state = crate::g17::job::fence_status(&active.packet.completion.fence()); + if state < 0 { + return Err(crate::g17::dependency::inherited_error(Error::from_errno( + state, + ))); + } + if state > 0 { + return Ok(None); + } + active.packet.completion.note_firmware_consumer(); + let kick = crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?; + Ok(Some( + crate::g17::fw::kick::KickDependency::new(self.qid(), kick).ok_or(EIO)?, + )) + } +} + +impl Graph { + fn record( + &mut self, + slot: Option, + offset: usize, + window: usize, + value: T, + ) -> Result { + let (object, start) = if let Some(slot) = slot { + if slot >= RECORD_SLOTS { + return Err(EINVAL); + } + (&mut self.records, slot * RECORD_STRIDE + offset) + } else { + (&mut self.graph, 0x6800 + offset) + }; + if core::mem::size_of::() > window { + return Err(EINVAL); + } + let pointer = object.pointer(start, window)?; + // SAFETY: The checked record window is not named by any unconsumed item pointer. The + // device mutex excludes other host publishers, and no Rust reference to it is formed. + unsafe { pointer.write_bytes(0, window) }; + object.write(start, value)?; + fence(Ordering::SeqCst); + Ok(object.gpu_va() + start as u64) + } + fn append(&self, expected: u32, address: u64, before_publish: impl FnOnce()) -> Result { + let cursors = self.cursors()?; + if cursors.producer != expected + || cursors.count != ITEM_RING_ENTRIES + || expected >= ITEM_RING_ENTRIES + { + return Err(EBUSY); + } + let next = (expected + 1) % ITEM_RING_ENTRIES; + if next == cursors.consumer { + return Err(EBUSY); + } + let slot = self.graph.dword(ITEMS + expected as usize * 8)?; + let producer = self.graph.word(POINTERS + 0x40)?; + if slot.load(Ordering::Relaxed) != 0 { + return Err(EBUSY); + } + before_publish(); + slot.store(address, Ordering::Relaxed); + fence(Ordering::SeqCst); + producer.store(next, Ordering::Relaxed); + Ok(next) + } +} + +/// Borrowed device operations used during one publication. Implementations borrow the initdata, +/// coprocessor and accounting fields separately from the Registry that owns physical queues. +pub(crate) trait Host { + fn epoch(&self) -> Result<(u64, u32)>; + fn prepare_compute_shared(&mut self) -> Result; + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_cancel(&mut self, publication: qos::Publication) -> Result; + fn publish_qid(&mut self, id: kick::Id) -> Result; + fn publish_outer(&mut self, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn notify(&mut self, message: u64) -> Result; + fn note_submission(&mut self) -> Result; +} + +impl Queue { + pub(crate) fn spared_pending(&self) -> bool { + self.quarantined && self.spared_quarantine && self.awaiting_witness + } + + /// Retains the packet before any potentially visible store. An error before publication + /// rolls back its work count; every ambiguous error quarantines the graph and packet. + pub(crate) fn publish( + &mut self, + packet: Arc, + dependencies: &[crate::g17::fw::kick::KickDependency], + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use crate::g17::fw::{ + channels::WorkSlot, + compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, + kick::{KickArgs, KickQos}, + }; + let (epoch, state) = host.epoch()?; + if state != 0 { + return Err(EBUSY); + } + let replay_front = self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, &packet)); + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } + if !packet.context.is_current() || !Arc::ptr_eq(&packet.context, &self.binding.context) { + return Err(EFAULT); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let crate::g17::command::Validated::Compute { + usc_base, + cdm_va, + cdm_end_va, + sampler_heap_va, + sampler_count, + .. + } = &packet.command + else { + return Err(EINVAL); + }; + let work = packet.completion.work_state(); + let work_word = work.word(); + if work_word == u32::MAX { + return Err(EINVAL); + } + let work_node = work.node_va()?; + let timestamp_va = packet.completion.compute_timestamp_va()?; + let kick_timestamp = self.kick.timestamp(); + let kick_parent = self.kick.parent(); + let qid = self.qid(); + let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let (descriptor_high, descriptor_low) = + self.graph.descriptor_vas(kick_timestamp.slot() as u8); + let first = !self.installed && self.ordinal == 0; + let entry_signal = first + || dependencies + .iter() + .any(|dependency| dependency.qid() != qid); + let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); + if let Some(slot) = record_slot { + self.graph.ensure_record_slot(slot)?; + } + let args = ComputeArgs { + usc_base: *usc_base, + cdm_va: *cdm_va, + cdm_end_va: *cdm_end_va, + sampler_heap_va: *sampler_heap_va, + sampler_count: *sampler_count, + descriptor_va: descriptor_low, + context_id: packet.context.id().try_into()?, + context_generation: packet.context.generation(), + work_state: work_word, + work_state_va: work_node, + qid, + kick_count: self.ordinal, + kick: kick_timestamp, + preempt_va: self.binding.preempt.iova(), + operand_state_va: self.binding.operand.iova(), + usage_va: self.binding.usage.iova(), + usage_fw_va: self.graph.usage.gpu_va(), + free_list_slot: u32::from(self.pool.id()), + free_list_control_va: self.pool.control_va(), + stamp_va: self.graph.support.gpu_va() + STAMP as u64, + aux_stamp_va: self.graph.support.gpu_va() + STAMP as u64 + 8, + status_va: [ + self.graph.support.gpu_va() + 0x310, + self.graph.support.gpu_va() + 0x318, + ], + timestamp_va, + }; + let identity = ConfigIdentity { + scheduler: packet.context.scheduler_va(), + qos: packet.context.qos_id(), + context: packet.context.id(), + }; + let config = QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, + kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), + qid, + install: first, + completion_seed: None, + context_id: args.context_id, + free_list: FreeListBinding { + va: self.pool.control_va(), + generation: u64::from(packet.context.scheduler_generation()), + slot: u32::from(self.pool.id()), + }, + scheduler_va: identity.scheduler, + qos_slot: identity.qos, + qos_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), + owner_pid: packet.context.owner_pid(), + context_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), + })?; + let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp)?; + let predecessor = KickPredecessor::new(qid, kick_parent)?; + let kick_args = KickArgs { + qid, + timestamp: kick_timestamp, + parent: kick_parent, + barriers: dependencies, + descriptor_va: descriptor_high, + queue_va: self.graph.graph.gpu_va(), + qos: KickQos { + slot: identity.qos, + class: COMPUTE_QOS_CLASS, + }, + mcache: None, + event_mask: COMPUTE_KICK_EVENT_MASK, + register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, + }; + self.active + .push(Active { + packet: packet.clone(), + work: work.clone(), + ticket: None, + observed: None, + failure_deferred: false, + qos_completed: false, + }) + .map_err(|_| EBUSY)?; + self.retirement_ready = false; + let mut visible = false; + let mut accounted = false; + let result = (|| -> Result { + work.add_submitted_kicks(1)?; + accounted = true; + self.graph.descriptors.initialize::( + kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + |descriptor| descriptor.write(&args), + )?; + if first { + self.graph.context_high.write( + 0x200, + crate::g17::fw::compute::QueueContextItem::new( + descriptor_high, + self.graph.graph.gpu_va(), + qid, + ), + )?; + } else { + // The retained context item remains at index one; only its descriptor changes. + self.graph.context_high.write(0x210, descriptor_high)?; + let cursors = self.graph.cursors()?; + let _ = self.graph.recycle(cursors); + } + let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; + let cursors = self.graph.cursors()?; + if cursors.count != ITEM_RING_ENTRIES { + return Err(EIO); + } + if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { + return Err(ERANGE); + } + let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; + if free < if entry_signal { 4 } else { 3 } { + return Err(ENOSPC); + } + let prepared_kick = self.kick.prepare_entry(&kick_args)?; + let mut producer; + if first { + producer = self.graph.first_descriptor(descriptor_high)?; + host.prepare_compute_shared()?; + packet.mark_published(); + visible = true; + let qos = host.qos_publish( + qos::Owner { + qid, + qos: identity.qos, + data_master: 2, + }, + identity.scheduler, + )?; + let target = self.pool.reserve_submission().or_else(|error| { + host.qos_cancel(qos)?; + Err(error) + })?; + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + host.qos_cancel(qos)?; + Err(error) + })?; + } else { + let target = self.pool.reserve_submission()?; + producer = self + .graph + .append(cursors.producer, descriptor_high, || { + packet.mark_published() + }) + .or_else(|error| { + self.pool.cancel_unpublished(target)?; + Err(error) + })?; + visible = true; + let qos = host.qos_publish( + qos::Owner { + qid, + qos: identity.qos, + data_master: 2, + }, + identity.scheduler, + )?; + producer = self + .graph + .append(producer, config_va, || {}) + .or_else(|error| { + host.qos_cancel(qos)?; + Err(error) + })?; + } + if first { + prepared_kick.prepare_compute_install()?; + } + prepared_kick.commit(); + self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; + let announce_va = self.graph.record(record_slot, ANNOUNCE, 0x40, announce)?; + producer = self.graph.append(producer, announce_va, || {})?; + fence(Ordering::SeqCst); + if entry_signal { + let predecessor_va = + self.graph + .record(record_slot, PREDECESSOR, 0x400, predecessor)?; + producer = self.graph.append(producer, predecessor_va, || {})?; + } + if let Some(slot) = record_slot { + self.graph.record_ends[slot] = Some(producer); + } + host.publish_qid(self.kick.id())?; + host.publish_outer(&WorkSlot::new( + DataMaster::Compute, + self.graph.graph.gpu_va(), + qid, + producer.try_into()?, + !self.outer_started, + )?)?; + let notify_activation = !self.outer_started; + self.outer_started = true; + self.installed = true; + self.published_config = Some(identity); + self.ordinal = ordinal; + let (accepted, state) = host.epoch()?; + if state != 0 || accepted != epoch { + return Err(EIO); + } + let ticket = Ticket { + owner: self.owner.ok_or(EIO)?, + submission: packet.order.sequence, + context_id: packet.context.id(), + qid, + descriptor: descriptor_high, + work_node, + kick: kick_timestamp.get(), + ordinal, + item_producer: producer, + item_count: ITEM_RING_ENTRIES, + epoch, + }; + self.active.back_mut().ok_or(EIO)?.ticket = Some(ticket); + if let Some(previous) = self.previous.as_mut() { + if previous.publication.is_none() { + previous.publication = Some((packet.order.sequence, epoch)); + } + } + if notify_activation { + fence(Ordering::SeqCst); + host.notify((0x83 << 48) | 0x0a)?; + } + fence(Ordering::SeqCst); + host.notify((0x83 << 48) | 0x10)?; + host.note_submission() + })(); + if let Err(error) = result { + if !visible { + if accounted { + if let Err(error) = work.cancel_unpublished_kicks(1) { + self.terminal = + self.quarantine(error, defer)?.map(|status| (status, error)); + return Err(error); + } + } + drop(self.active.pop_back()); + return Err(error); + } + self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); + return Err(error); + } + Ok(()) + } +} + +impl Graph { + /// The queue has never been installed. Retry staging replaces its initial three item slots; + /// no firmware reader can name this graph until its first complete outer publication. + fn first_descriptor(&self, address: u64) -> Result { + let first = self.graph.dword(ITEMS)?; + let second = self.graph.dword(ITEMS + 8)?; + let third = self.graph.dword(ITEMS + 16)?; + let producer = self.graph.word(POINTERS + 0x40)?; + first.store(address, Ordering::Relaxed); + second.store(0, Ordering::Relaxed); + third.store(0, Ordering::Relaxed); + producer.store(1, Ordering::Relaxed); + Ok(1) + } +} +impl Queue { + pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { + self.terminal.take() + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs new file mode 100644 index 00000000000000..1ddadae2dba693 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Closed-context kill requests and the command-pin release witness. + +use super::{Context, Deferred, Queue}; +use core::sync::atomic::{fence, AtomicU32, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +pub(crate) const POLL_NS: u64 = 2_500_000_000; +const QUARANTINED_DELAY_NS: u64 = 250_000_000; +const CANCELLED_DELAY_NS: u64 = 2_000_000_000; +const ACK_SLOTS: usize = 16; + +// Firmware-visible cookie namespace persists for the loaded module's lifetime. +static NEXT_COOKIE: AtomicU32 = AtomicU32::new(1); + +pub(crate) fn next_cookie() -> u64 { + loop { + let cookie = NEXT_COOKIE.fetch_add(1, Ordering::Relaxed); + if cookie != 0 { + return cookie.into(); + } + } +} + +pub(crate) struct Acks([u64; ACK_SLOTS]); +impl Acks { + pub(crate) fn new() -> Self { + Self([0; ACK_SLOTS]) + } + pub(crate) fn note(&mut self, cookie: u64) { + if cookie == 0 || self.contains(cookie) { + return; + } + if let Some(slot) = self.0.iter_mut().find(|slot| **slot == 0) { + *slot = cookie; + } + } + pub(crate) fn contains(&self, cookie: u64) -> bool { + cookie != 0 && self.0.contains(&cookie) + } + pub(crate) fn forget(&mut self, cookie: u64) { + for slot in &mut self.0 { + if *slot == cookie { + *slot = 0; + } + } + } + pub(crate) fn snapshot(&self) -> [u64; ACK_SLOTS] { + self.0 + } +} + +pub(crate) struct State { + pub(crate) slot: u8, + pub(crate) qids: u128, + pub(crate) killed: u32, +} +impl State { + pub(crate) fn read(context: &Context) -> Result { + let words = context.scheduler_owner().read_words::<14>(0)?; + let mut bytes = [0u8; 0x38]; + for (destination, word) in bytes.chunks_exact_mut(4).zip(words) { + destination.copy_from_slice(&word.to_le_bytes()); + } + fence(Ordering::Acquire); + Ok(Self::decode(&bytes)) + } + fn decode(bytes: &[u8; 0x38]) -> Self { + let mut low = [0; 8]; + let mut high = [0; 8]; + let mut killed = [0; 4]; + low.copy_from_slice(&bytes[0x12..0x1a]); + high.copy_from_slice(&bytes[0x1a..0x22]); + killed.copy_from_slice(&bytes[0x34..0x38]); + Self { + slot: bytes[1], + qids: u128::from(u64::from_le_bytes(low)) + | (u128::from(u64::from_le_bytes(high)) << 64), + killed: u32::from_le_bytes(killed), + } + } + pub(crate) fn scope(&self, target: u8, own: u128, foreign: u128) -> Option { + if self.slot == 0xff || target >= 128 || self.qids & (1u128 << target) == 0 { + return None; + } + let allowed = (own | (1u128 << target)) & !foreign & !0b11; + (self.qids & !allowed == 0).then_some(self.qids) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { + Published, + Acknowledged, + Unproven, + Released, +} + +pub(super) struct Transaction { + cookie: u64, + published: u64, + phase: Phase, +} + +impl Queue { + pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { + Arc::ptr_eq(self.context(), context) + && self.released + && self.active.len != 0 + && self.exit.is_none() + && (self.quarantined || self.retired_by_teardown) + && age + >= if self.quarantined { + QUARANTINED_DELAY_NS + } else { + CANCELLED_DELAY_NS + } + } + pub(crate) fn exit_owner(&self, context: &Arc) -> Option { + if !self + .active + .iter() + .all(|active| Arc::ptr_eq(&active.packet.context, context)) + { + return None; + } + self.owner.or_else(|| { + self.active + .back() + .and_then(|active| active.ticket.map(|ticket| ticket.owner)) + }) + } + pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { + self.exit = Some(Transaction { + cookie, + published: now, + phase: Phase::Published, + }); + } + pub(crate) fn exit_polling(&self, now: u64) -> bool { + self.exit.as_ref().is_some_and(|exit| { + exit.phase == Phase::Published && now.saturating_sub(exit.published) < POLL_NS + }) + } + pub(crate) fn exit_cookie(&self) -> Option { + self.exit + .as_ref() + .filter(|exit| matches!(exit.phase, Phase::Published | Phase::Acknowledged)) + .map(|exit| exit.cookie) + } + pub(crate) fn settle_exit( + &mut self, + acks: &Acks, + recovery_closed: bool, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let Some(cookie) = self.exit_cookie() else { + return Ok(()); + }; + if !acks.contains(cookie) { + return Ok(()); + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Acknowledged; + if self.active.len != 0 { + if !recovery_closed { + return Ok(()); + } + let state = match State::read(self.context()) { + Ok(state) => state, + Err(_) => return Ok(()), + }; + if state.killed != 1 { + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Unproven; + return Ok(()); + } + if !self.active.iter().all(|active| { + active + .packet + .completion + .compute_words() + .is_ok_and(|words| words == [0, 0] || words[1] != 0) + }) { + return Ok(()); + } + } + while let Some(active) = self.active.front() { + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(if self.spared_deferred { + ENODATA + } else { + ECANCELED + }), + ))?; + drop(self.active.pop()); + } + self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; + self.quarantined = true; + self.awaiting_witness = false; + self.spared_deferred = false; + self.spared_quarantine = false; + self.retire_pending = false; + self.retirement_ready = false; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs new file mode 100644 index 00000000000000..7428f99ab1d7fe --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact identities and visibility witnesses for a physical compute queue. +//! A shared work-state counter can include other engines and cannot prove this queue retired. + +use kernel::prelude::*; + +pub(crate) const IN_FLIGHT: usize = 16; +pub(crate) const RECORD_SLOTS: usize = 32; +const QIDS: usize = 128; +const TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Ticket { + pub(crate) owner: u64, + pub(crate) submission: u64, + pub(crate) context_id: u32, + pub(crate) qid: u8, + pub(crate) descriptor: u64, + pub(crate) work_node: u64, + pub(crate) kick: u64, + pub(crate) ordinal: u32, + pub(crate) item_producer: u32, + pub(crate) item_count: u32, + pub(crate) epoch: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Observation { + pub(crate) timestamps: [u64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) producer: u32, + pub(crate) consumer: u32, + pub(crate) count: u32, +} + +impl Cursors { + /// Whether firmware consumed every record of the given publication, allowing later work. + pub(crate) fn consumed(&self, end: u32, count: u32) -> Result { + if self.count == 0 || self.count != count { + return Err(EIO); + } + let producer = self.producer % count; + let consumer = self.consumer % count; + let unconsumed = (producer + count - consumer) % count; + let following = (producer + count - end % count) % count; + Ok(unconsumed <= following) + } +} + +/// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. +pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { + let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +#[derive(Clone, Copy)] +pub(crate) enum Recordless { + Allowed, + NotYet, + CrossedRecovery, +} + +/// A zero-duration record fails before visibility polling. It proves execution did not occur; +/// ownership still requires a separate retirement witness before mappings can be released. +pub(crate) fn poll( + ticket: &Ticket, + observed: Option, + cursors: Cursors, + stamp: u32, + recordless: Recordless, + floor: u64, + next_end: Option, +) -> Result> { + if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { + return Err(ENODATA); + } + if !stamp_covers(stamp, ticket.kick) + || !cursors.consumed(ticket.item_producer, ticket.item_count)? + { + return Ok(None); + } + Ok(Some(match observed { + Some(record) => record.timestamps, + None => match recordless { + Recordless::Allowed => [floor, next_end.unwrap_or(floor).max(floor)], + Recordless::NotYet => return Ok(None), + Recordless::CrossedRecovery => return Err(ENODATA), + }, + })) +} + +/// The newest exact publication covers all preceding kicks, but only a fully consumed ring +/// permits releasing quarantined command pins. The installed graph remains retained. +pub(crate) fn quarantined_retired( + ticket: &Ticket, + cursors: Cursors, + stamp: u32, + pending: u32, + submitted: u32, +) -> Result { + if ticket.item_count == 0 || ticket.descriptor == 0 || pending == 0 { + return Err(EINVAL); + } + if cursors.count == 0 || cursors.count != ticket.item_count { + return Err(EIO); + } + let end = ticket.item_producer % cursors.count; + Ok(stamp_covers(stamp, ticket.kick) + && cursors.producer % cursors.count == end + && cursors.consumer % cursors.count == end + && submitted == pending) +} + +#[derive(Clone, Copy)] +struct Request { + descriptor: u64, + kick: u64, + observed: Option, +} + +/// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; +/// staging and folding never allocate under the device mutex. +pub(crate) struct Batch { + requests: KVVec, + slots: [(usize, usize); QIDS], + open: Option, +} + +impl Batch { + pub(crate) fn new() -> Result { + Ok(Self { + requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, + slots: [(0, 0); QIDS], + open: None, + }) + } + pub(crate) fn clear(&mut self) { + self.requests.clear(); + self.slots.fill((0, 0)); + self.open = None; + } + pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { + if ticket.descriptor == 0 || ticket.kick == 0 { + return Err(EINVAL); + } + let slot = self.slots.get_mut(ticket.qid as usize).ok_or(EINVAL)?; + if slot.1 != 0 && self.open != Some(ticket.qid) { + return Err(EBUSY); + } + if slot.1 >= IN_FLIGHT { + return Err(ENOSPC); + } + let start = if slot.1 == 0 { + self.requests.len() + } else { + slot.0 + }; + self.requests + .push_within_capacity(Request { + descriptor: ticket.descriptor, + kick: ticket.kick, + observed: None, + }) + .map_err(|_| ENOSPC)?; + *slot = (start, slot.1 + 1); + self.open = Some(ticket.qid); + Ok(()) + } + /// The transport reads eight aligned words from a 0x40-byte firmware record. Records can be + /// overwritten during a scan: incomplete or older identities are skipped, never consumed. + pub(crate) fn fold(&mut self, words: [u64; 8]) -> Result { + let qid = ((words[0] >> 9) & 0x7f) as usize; + let (start, count) = self.slots[qid]; + if count == 0 || words[6] & !1 == 0 || words[7] == 0 { + return Ok(()); + } + let descriptor = words[6] & !1; + let kick = + (((words[0] >> 8) & 0xffff_ffff_00) | ((words[0] >> 48) & 0xff)) & 0xff_ffff_ffff; + let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; + let Some(request) = requests + .iter_mut() + .find(|r| r.descriptor == descriptor && r.kick == kick) + else { + return Ok(()); + }; + let timestamps = [words[2] & TIME_MASK, words[5] & TIME_MASK]; + if timestamps[0] == 0 + || timestamps[1] == 0 + || request + .observed + .is_some_and(|old| old.timestamps[1] >= timestamps[1]) + { + return Ok(()); + } + request.observed = Some(Observation { timestamps }); + Ok(()) + } + pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { + let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; + let request = self + .requests + .get_mut(start..start + count) + .ok_or(EIO)? + .iter_mut() + .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) + .ok_or(EIO)?; + Ok(request.observed.take()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs new file mode 100644 index 00000000000000..afc58009f40292 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Userspace submission queues: validate whole ioctls, retain mappings, and enqueue +//! independent engines behind user fences and explicit logical prefixes. + +pub(crate) mod compute; +pub(crate) mod render; + +use super::{ + command::{Parser, UscWindow, Validated}, + completion::Completion, + context::Context, + fence::{Outputs, Submission}, + job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, + validation::Proof, +}; +use crate::{ + driver::{AsahiDevRef, AsahiDevice}, + file, mmu, +}; +use core::ops::Range; +use kernel::{ + c_str, + dma_fence::FenceContexts, + prelude::*, + sync::{Arc, LockClassKey}, + xarray, +}; + +const EARLY_DEPTH: usize = 2; +static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + +pub(crate) mod admission; + +/// Host identities for retained physical channels, separate from firmware object generations. +/// The device mutex protects allocation; cancelled preparations never reuse IDs. +pub(crate) struct Identities(u64); + +impl Identities { + pub(crate) const fn new() -> Self { + Self(3) + } + + /// The first two identities remain reserved with QIDs 0 and 1. + pub(crate) fn reserve(&mut self, count: u64) -> Result> { + let first = self.0; + let end = first.checked_add(count).ok_or(EOVERFLOW)?; + self.0 = end; + Ok(first..end) + } +} + +/// Validated frontend state prepared before allocating a logical execution context. +pub(crate) struct Frontend { + fences: FenceContexts, + range: Range, + window: UscWindow, +} + +impl Frontend { + pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { + if priority > 3 { + return Err(EINVAL); + } + let window = UscWindow::new(usc_base, range.clone())?; + let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + Ok(Self { + fences, + range, + window, + }) + } +} + +pub(crate) struct Queue { + dev: AsahiDevRef, + context: Arc, + backend: Arc, + render: Option>, + compute: Option>, + fences: FenceContexts, + range: Range, + window: UscWindow, + proof: Proof, + sequence: [u64; 2], + frontiers: [KVec<(u64, Fences)>; 2], + // Closing the userspace queue releases admission even if firmware still + // retains one of its execution contexts or installed physical graphs. + _slot: admission::QueueSlot, +} + +impl Queue { + pub(crate) fn new( + dev: &AsahiDevice, + context: Arc, + backend: Arc, + frontend: Frontend, + slot: admission::QueueSlot, + ) -> Result { + let Frontend { + fences, + range, + window, + } = frontend; + let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); + Ok(Self { + dev: dev.into(), + context, + backend, + render, + compute: None, + fences, + range, + window, + proof: Proof::new(), + sequence: [0; 2], + frontiers: core::array::from_fn(|_| KVec::new()), + _slot: slot, + }) + } + + fn ensure_compute(&mut self) -> Result { + // Admission is checked even when this queue already owns a scheduler. + self.backend.ensure_compute()?; + if self.compute.is_none() { + self.compute = Some(Scheduler::new( + &self.dev, + self.backend.clone(), + Engine::Compute, + )?); + } + Ok(()) + } + + fn enqueue( + &mut self, + id: u64, + command: Validated, + order: Order, + guard: mmu::VmJobGuard, + aggregate: &Arc, + inputs: &[file::SyncItem], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let engine = match &command { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + }; + let index = engine.index(); + self.frontiers[index].retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + self.frontiers[index].reserve(1, GFP_KERNEL)?; + if engine == Engine::Compute { + self.ensure_compute()?; + } + let completion = match &command { + Validated::Render { timestamps, .. } => Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + objects, + timestamps, + false, + )?, + Validated::Compute { timestamps, .. } => Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + objects, + core::slice::from_ref(timestamps), + true, + )?, + }; + if engine == Engine::Compute { + self.frontiers[Engine::Render.index()] + .retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + } + let other = 1 - index; + let prefix = order.wait_through[other]; + let count = self.frontiers[other] + .iter() + .filter(|(sequence, _)| Order::contains(prefix, *sequence)) + .count(); + let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; + let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; + let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + for (sequence, fences) in &self.frontiers[other] { + if Order::contains(prefix, *sequence) { + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + firmware.push(fences.completed.clone(), GFP_KERNEL)?; + } + } + // Each command owns every input fence; independent engines may publish in + // either order and must each consume an input's error status before running. + for sync in inputs { + if let Some(fence) = &sync.fence { + dependencies.push(fence.clone(), GFP_KERNEL)?; + checked_inputs.push(fence.clone(), GFP_KERNEL)?; + } + } + let early = engine == Engine::Render + && dependencies.is_empty() + && self.frontiers[index].len() < EARLY_DEPTH; + let packet = Packet::new( + id, + self.context.clone(), + command, + order, + completion, + checked_inputs, + firmware, + )?; + let scheduler = match engine { + Engine::Render => &mut self.render, + Engine::Compute => &mut self.compute, + }; + let fences = scheduler + .as_mut() + .ok_or(ENODEV)? + .enqueue(packet, dependencies, early)?; + self.sequence[index] = order.sequence; + self.frontiers[index] + .push_within_capacity((order.sequence, fences)) + .map_err(|_| ENOMEM)?; + Ok(()) + } + /// Finish parsing, validation and VM pinning before entering publication. + /// Keeping this frame separate bounds the scheduler/ioctl stack depth. + #[inline(never)] + fn prepare_submission( + &mut self, + bytes: &[u8], + ) -> Result<(KVec<(Validated, Order)>, KVec)> { + let parse_guard = self.context.vm().retain_first_job()?; + let mut parser = Parser::new(bytes); + let mut preceding = [0; 2]; + let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; + while let Some(command) = parser.next()? { + let payload = self.proof.validate( + self.context.vm(), + self.range.clone(), + &self.window, + &command.payload, + )?; + let engine = match &payload { + Validated::Render { .. } => Engine::Render, + Validated::Compute { .. } => Engine::Compute, + }; + let order = Order::new(self.sequence, preceding, command.barriers, engine)?; + preceding[engine.index()] += 1; + pending.push((payload, order), GFP_KERNEL)?; + } + parser.finish()?; + // Every accepted command pins the VM before any command is enqueued. + let mut guards = KVec::with_capacity(pending.len(), GFP_KERNEL)?; + guards + .push_within_capacity(parse_guard) + .map_err(|_| ENOMEM)?; + while guards.len() < pending.len() { + guards + .push_within_capacity(self.context.vm().retain_job()?) + .map_err(|_| ENOMEM)?; + } + Ok((pending, guards)) + } +} + +impl crate::queue::Queue for Queue { + fn submit( + &mut self, + id: u64, + syncs: KVec, + input_count: usize, + bytes: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let mut outputs = Outputs::new(&self.fences, syncs, input_count)?; + let result = (|| { + if self.context.status().get() != 0 { + return Err(EIO); + } + let (pending, mut guards) = self.prepare_submission(bytes)?; + let aggregate = Submission::new(&self.fences, self.context.clone())?; + outputs.publish(aggregate.clone()); + for (command, order) in pending { + let guard = guards.remove(0).map_err(|_| EIO)?; + self.enqueue( + id, + command, + order, + guard, + &aggregate, + outputs.inputs(), + objects, + )?; + } + Ok(()) + })(); + if let Err(error) = result { + self.context.status().record_if_device_loss(error); + if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { + self.backend.fail_vm(error); + } + } + outputs.finish(result) + } +} + +impl Drop for Queue { + fn drop(&mut self) { + drop(self.render.take()); + drop(self.compute.take()); + self.backend.release_owner(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs new file mode 100644 index 00000000000000..3a7852be1dab03 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -0,0 +1,1459 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical paired tiling and fragment queues and their retained command storage. + +mod observation; +pub(crate) use observation::Observed; +mod dependencies; +mod memory; +pub(crate) mod parked; +mod publication; +pub(crate) mod retirement; +mod warm; + +use crate::g17::{ + buffer::{self, target}, + command::Validated, + completion::Deferred, + context::Context, + freelist::RenderPool, + fw::{ + channels::WorkSlot, + kick::{KickDependency, KickTimestamp, McacheTable, RenderRegistration}, + queue::DataMaster, + render as abi, + }, + job::Packet, + kick, + object::{Allocator, Pool}, + qos, recovery, + status::VmStatus, +}; +use crate::{hw::t8140::queues, mmu}; +pub(crate) use dependencies::Dependencies; +use kernel::{prelude::*, sync::Arc}; +use retirement::{Ticket, Tracker}; + +#[derive(Copy, Clone)] +pub(crate) struct RenderAddresses { + /// Tiling then fragment; each pair contains the primary and auxiliary stamp. + pub(crate) stamps: [[u64; 2]; 2], +} + +#[derive(Copy, Clone)] +pub(crate) struct PublicationLease { + pub(crate) free_list_generation: u64, + pub(crate) buffer_token: u64, +} + +/// Host services borrow the device-locked firmware state. Resource construction +/// and command geometry preparation do not borrow this interface. +pub(crate) trait Host { + fn epoch(&self) -> Result<(u64, u32)>; + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_cancel(&mut self, publication: qos::Publication) -> Result; + fn install_pair(&mut self, registration: &RenderRegistration) -> Result; + fn publish_qid(&mut self, id: kick::Id) -> Result; + fn publish_outer_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]>; + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; + fn publish_retained_pair( + &mut self, + fragment: &WorkSlot, + tiling: &WorkSlot, + next: [u32; 2], + ) -> Result; + fn notify_render(&mut self) -> Result; + fn note_submission(&mut self) -> Result; + fn complete_render( + &mut self, + owners: Option<[qos::Owner; 2]>, + pool_id: u16, + generation: u64, + buffer_id: u8, + ) -> Result; +} + +struct Prepared { + packet: Arc, + ordinal: u64, + mcache: Option, +} +struct Active { + qos_completed: bool, + flist_completed: bool, + outer_published: bool, + generation: u64, + packet: Arc, + ticket: Ticket, + kicks: [KickTimestamp; 2], + context_update: bool, + failed: bool, + drained_result: Option>, +} + +/// Installed kick backing remains owned even when this pair has no logical +/// owner. Commands retain their exact descriptor and allocation generations. +pub(crate) struct Pair { + kick_aliases: Option<[Option; 2]>, + memory: KBox, + kicks: [kick::Queue; 2], + free_list: Arc, + pending_aliases: Option>, + prepared: Option, + active: Tracker, + slot: u8, + owner: Option, + ordinal: u64, + initial_packet: u64, + clusters: u32, + descriptor_flags: [u32; 2], + installed: [bool; 2], + configs: [Option; 2], + cursors: [u32; 2], + recovery_generation: u64, + registered: bool, + outer_started: bool, + quarantined: bool, + terminal: Option<(Arc, Error)>, + previous: Option, + displaced: Option, + context_update: bool, +} +impl Pair { + #[inline(never)] + pub(crate) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + ids: [kick::Id; 2], + slot: u8, + owner: u64, + context: Arc, + free_list: Arc, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + first: &Arc, + clusters: u32, + descriptor_flags: [u32; 2], + ) -> Result> { + if slot == 0 + || usize::from(slot) >= queues::RENDER_SLOTS + || owner == 0 + || ids[0].engine() != DataMaster::Tiling + || ids[1].engine() != DataMaster::Fragment + || !Arc::ptr_eq(&context, &first.context) + || !context.is_current() + { + return Err(EINVAL); + } + if let Err(error) = alloc.uat.vm_context_mask(context.vm()) { + if error == ENOENT { + context.status().record(error); + } + return Err(error); + } + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let word = first.completion.work_state().word(); + if word == u32::MAX { + return Err(EINVAL); + } + let layout = target::Layout::new(pass, clusters)?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; + let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; + let (memory, aliases) = memory::Memory::new( + alloc, + pool, + context, + [ids[0].qid(), ids[1].qid()], + &layout, + metrics, + buffer_id, + pm_generation, + 0, + word, + )?; + storage.write_init(kernel::try_init!(Self { + kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), + memory, + kicks: [tiling, fragment], + free_list, + pending_aliases: Some(aliases), + prepared: None, + active: Tracker::new(), + slot, + owner: Some(owner), + ordinal: 0, + initial_packet: first.id, + clusters, + descriptor_flags, + installed: [false; 2], + configs: [None; 2], + cursors: [0; 2], + recovery_generation: 0, + registered: false, + outer_started: false, + quarantined: false, + terminal: None, + previous: None, + displaced: None, + context_update: true, + })) + } + pub(crate) fn qids(&self) -> [u8; 2] { + self.kicks.each_ref().map(|kick| kick.id().qid()) + } + pub(crate) fn slot(&self) -> u8 { + self.slot + } + pub(crate) fn owner(&self) -> Option { + self.owner + } + pub(crate) fn context(&self) -> &Arc { + &self.memory.context + } + pub(crate) fn pool(&self) -> &Arc { + &self.free_list + } + pub(crate) fn pool_id(&self) -> u64 { + u64::from(self.memory.lifecycle[0].predecessor) + } + pub(crate) fn buffer_id(&self) -> u8 { + self.memory.manager.buffer_id() + } + pub(crate) fn manager(&mut self) -> &mut buffer::Manager { + &mut self.memory.manager + } + pub(crate) fn in_flight(&self) -> bool { + self.active.len() != 0 + } + pub(crate) fn published_in_flight(&self) -> bool { + !self.quarantined && self.active.iter().any(|active| active.outer_published) + } + pub(crate) fn room(&self) -> bool { + !self.quarantined && self.owner.is_some() && self.prepared.is_none() && self.active.room() + } + pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { + self.terminal.take() + } + + /// The registry owns this pair exclusively during potentially allocating + /// preparation. No queue producer or shared submission counter changes. + pub(crate) fn prepare( + &mut self, + alloc: &Allocator<'_>, + packet: &Arc, + addresses: RenderAddresses, + ) -> Result { + if !self.room() + || !Arc::ptr_eq(&self.memory.context, &packet.context) + || !packet.context.is_current() + { + return Err(EBUSY); + } + let Validated::Render { + pass, attachments, .. + } = &packet.command + else { + return Err(EINVAL); + }; + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let ordinal = self.ordinal; + ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let work = packet.completion.work_state(); + let node = work.node_va()?; + let word = work.word(); + let layout = target::Layout::new(pass, self.clusters)?; + let owners = self.memory.owners(); + self.memory + .target + .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; + let target = self.memory.target.prepare(ordinal, &layout)?; + let scene = self.memory.manager.prepare_scene(ordinal)?; + self.memory.generation(ordinal)?.prepare(ordinal, pass)?; + let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = + self.memory.generation(ordinal)?.addresses(); + let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { + let (fragment, tiling) = + crate::g17::ids::OBJECTS.render_pair(self.memory.lifecycle[0].predecessor)?; + [tiling, fragment] + } else { + self.memory.lifecycle + }; + let (scratch, metric) = self.memory.manager.scene_registers(scene)?; + let timestamp_offset = ordinal as usize % memory::RECORDS * 16; + let timestamp = self.memory.timestamps.gpu_va() + timestamp_offset as u64; + let resources = abi::RenderResources { + context_id: packet.context.id().try_into()?, + context_generation: packet.context.generation(), + pb_slot: self.buffer_id().into(), + free_list_slot: self.free_list.id().into(), + free_list_control_va: self.free_list.control_va(), + kick_record_va: node, + pb_va: self.memory.manager.state_va(), + scene_va: self.memory.manager.scene_va(scene)?, + scene_end_va: self.memory.manager.trailer_va(), + pm_scratch: scratch, + pm_metric: metric, + tilemap_va: target.tilemap, + layermeta_va: target.layer, + heapmeta_va: target.heap, + tpc_va: self.memory.target.tpc_va(), + tpc_size: layout.tpc_size as u64, + deflake_va: deflake, + ta_status_va: ta_status, + fragment_status_va: fragment_status, + fragment_status_fw_va: fragment_fw, + aux_fb_va: auxiliary, + timestamp_va: timestamp, + ta_ids: lifecycle[0], + fragment_ids: lifecycle[1], + unk_2104: self.descriptor_flags[0], + unk_210c: self.descriptor_flags[1], + }; + let qids = self.qids(); + let args = abi::RenderArgs { + pass, + resources: &resources, + ordinal, + ta: abi::RenderStage { + qid: qids[0], + descriptor_va: self.memory.graph.descriptor_client(0, ordinal), + state_word: word, + stamp_va: addresses.stamps[0][0], + aux_stamp_va: addresses.stamps[0][1], + }, + fragment: abi::RenderStage { + qid: qids[1], + descriptor_va: self.memory.graph.descriptor_client(1, ordinal), + state_word: word, + stamp_va: addresses.stamps[1][0], + aux_stamp_va: addresses.stamps[1][1], + }, + }; + self.memory.graph.claim(ordinal)?; + let result = self.memory.write_descriptors( + &args, + attachments[1].as_slice(), + packet.context.id().try_into()?, + ); + let mcache = match result { + Ok(mcache) => mcache, + Err(error) => { + self.memory.graph.release(ordinal)?; + self.memory.generation(ordinal)?.ordinal = None; + self.memory.target.cancel(ordinal); + return Err(error); + } + }; + self.memory.timestamps.write(timestamp_offset, [0u64; 2])?; + self.memory.manager.claim_scene(ordinal, scene)?; + self.prepared = Some(Prepared { + packet: packet.clone(), + ordinal, + mcache, + }); + Ok(()) + } +} + +impl Pair { + /// All descriptor bytes and retained allocations are ready before entry. + /// The first inner producer establishes firmware ownership, including on + /// later errors. Such failures retain the active packet for recovery. + pub(crate) fn publish( + &mut self, + packet: Arc, + dependencies: &[KickDependency], + lease: PublicationLease, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use crate::g17::fw::{ + channels::WorkSlot, + kick::{KickArgs, KickQos}, + queue::{ + self, FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, + TilingBinding, + }, + }; + use core::sync::atomic::{fence, Ordering}; + if self.pending_aliases.is_some() || self.kick_aliases.is_some() { + return Err(EBUSY); + } + let (epoch, recovery_state) = host.epoch()?; + if recovery_state != 0 + || self.quarantined + || !self.active.room() + || packet.completion.render_slot() != Some(self.slot) + { + return Err(EBUSY); + } + let prepared = self.prepared.as_ref().ok_or(EINVAL)?; + if !Arc::ptr_eq(&prepared.packet, &packet) + || prepared.ordinal != self.ordinal + || !self.memory.owns(self.ordinal) + || !packet.context.is_current() + { + return Err(EIO); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let ordinal = self.ordinal; + let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; + let mcache = prepared.mcache; + let qids = self.qids(); + let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); + let parents = self.kicks.each_ref().map(kick::Queue::parent); + let work = packet.completion.work_state(); + let work_slot = work.render_slot(self.slot)?; + self.memory.bind_work_slot(ordinal, qids, work_slot)?; + let payload = work.channel_payload(self.slot, ordinal)?; + let stamps = host.stamps(qids)?; + fence(Ordering::Acquire); + let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); + self.memory + .completion_links(ordinal, kicks, payload, !covered[1])?; + let context = &packet.context; + let qos = context.qos_id(); + let qos_owners = [ + qos::Owner { + qid: qids[0], + qos, + data_master: DataMaster::Tiling as u8, + }, + qos::Owner { + qid: qids[1], + qos, + data_master: DataMaster::Fragment as u8, + }, + ]; + let mut configs = [None; 2]; + for stage in 0..2 { + configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { + target: if stage == 0 { + QueueConfigTarget::Tiling(TilingBinding { + pb_va: self.memory.manager.state_va(), + pb_token: lease.buffer_token, + pb_slot: self.buffer_id().into(), + fragment_qid: qids[1], + }) + } else { + QueueConfigTarget::Fragment + }, + kick_ring_va: self.kicks[stage].low_va(), + kick_ring_fw_va: self.kicks[stage].firmware_va(), + qid: qids[stage], + install: !self.installed[stage], + completion_seed: covered[stage].then(|| { + kicks[stage] + .prev() + .completion_value(payload.wrapping_sub(1)) + }), + context_id: context.id().try_into()?, + free_list: FreeListBinding { + va: self.free_list.control_va(), + generation: lease.free_list_generation, + slot: self.free_list.id().into(), + }, + scheduler_va: context.scheduler_va(), + qos_slot: qos, + qos_update: self.context_update, + owner_pid: context.owner_pid(), + context_update: self.context_update, + })?); + } + let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; + let cache = if self.recovery_generation == epoch { + self.configs + } else { + [None; 2] + }; + let present = [ + publication::required(cache[0], configs[0]), + publication::required(cache[1], configs[1]), + ]; + let cursors = self.memory.graph.cursors()?; + if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { + return Err(EBUSY); + } + let queues = qids.map(|qid| { + self.memory.graph.queues.gpu_va() + + u64::from(qid) * size_of::() as u64 + }); + let fragment_barriers = [KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?]; + let args = |stage: usize, barriers, arrays, mcache| KickArgs { + qid: qids[stage], + timestamp: kicks[stage], + parent: parents[stage], + barriers, + descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), + queue_va: queues[stage], + qos: KickQos { + slot: qos, + class: if stage == 0 { + abi::TA_QOS_CLASS + } else { + abi::FRAGMENT_QOS_CLASS + }, + }, + mcache, + event_mask: abi::RENDER_KICK_EVENT_MASK, + register_arrays: arrays, + }; + let ta_args = args( + 0, + dependencies, + abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, + None, + ); + let fragment_args = args( + 1, + &fragment_barriers, + abi::FragmentDescriptor::register_bindings( + self.memory.graph.descriptor_client(1, ordinal), + )?, + mcache, + ); + let registration = RenderRegistration::new( + (qids[1], self.kicks[1].low_va()), + (qids[0], self.kicks[0].low_va()), + )?; + let [tiling_kick, fragment_kick] = &mut self.kicks; + let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; + let fragment_kick = fragment_kick.prepare_entry(&fragment_args)?; + let memory = &mut *self.memory; + let lanes = [ + publication::Lane::new( + &memory.graph, + 0, + ordinal, + self.cursors[0], + configs[0], + present[0], + kicks[0], + )?, + publication::Lane::new( + &memory.graph, + 1, + ordinal, + self.cursors[1], + configs[1], + present[1], + kicks[1], + )?, + ]; + let windows = [lanes[0].window, lanes[1].window]; + let previous = self.active.front().map_or(cursors, |active| { + active.ticket.windows.map(|window| { + let target = window.target(); + retirement::Cursors { + done: target, + read: target, + write: target, + } + }) + }); + let fragment_slot = WorkSlot::new( + DataMaster::Fragment, + queues[1], + qids[1], + windows[1].target().try_into()?, + !self.outer_started, + )?; + let tiling_slot = WorkSlot::new( + DataMaster::Tiling, + queues[0], + qids[0], + windows[0].target().try_into()?, + !self.outer_started, + )?; + let event_slot = memory + .graph + .support + .word(abi::storage::EVENT_SLOTS + 4)?; + let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); + let inner = memory.graph.support.word(0x28000)?; + let next_inner = (next_ordinal as u32).wrapping_mul(2); + let flist_submission = self.free_list.prepare_submission(2)?; + let pm_submission = memory.manager.prepare_submission()?; + self.active + .push(Active { + qos_completed: false, + flist_completed: false, + outer_published: false, + generation: lease.free_list_generation, + packet: packet.clone(), + ticket: Ticket { + ordinal, + windows, + previous, + }, + kicks, + context_update: self.context_update, + failed: false, + drained_result: None, + }) + .map_err(|_| EBUSY)?; + self.prepared = None; + packet.mark_published(); + lanes[1].publish_prefix(); + fragment_kick.commit(); + lanes[1].publish_kick(); + tiling_kick.commit(); + let mut qos_publications = [None; 2]; + let result = (|| { + qos_publications[1] = Some(host.qos_publish(qos_owners[1], context.scheduler_va())?); + qos_publications[0] = Some(host.qos_publish(qos_owners[0], context.scheduler_va())?); + if !self.registered { + host.install_pair(®istration)?; + self.registered = true; + } + for kick in &self.kicks { + host.publish_qid(kick.id())?; + } + host.publish_outer_pair( + &fragment_slot, + &tiling_slot, + || { + work.add_submitted_kicks(2)?; + flist_submission.commit(); + pm_submission.commit(); + Ok(()) + }, + || lanes[0].publish_tiling(event_slot, next_event, inner, next_inner), + ) + })(); + let outer_next = match result { + Ok(next) => next, + Err(error) => { + self.quarantined = true; + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + self.active + .back() + .ok_or(EIO)? + .packet + .completion + .quarantine_render(); + for publication in qos_publications.into_iter().flatten() { + host.qos_cancel(publication)?; + } + self.active.back_mut().ok_or(EIO)?.qos_completed = true; + self.quarantined = true; + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + return Err(error); + } + }; + self.ordinal = next_ordinal; + self.cursors = windows.map(|window| window.target()); + for stage in 0..2 { + if present[stage] { + self.configs[stage] = Some(configs[stage]); + } + } + self.recovery_generation = epoch; + self.installed = [true; 2]; + self.outer_started = true; + if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { + previous.publication = Some(ordinal); + } + packet.context.mark_published(); + let publish = (|| { + host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; + host.publish_retained_pair(&fragment_slot, &tiling_slot, outer_next)?; + self.active.back_mut().ok_or(EIO)?.outer_published = true; + host.notify_render()?; + host.note_submission() + })(); + if let Err(error) = publish { + self.quarantined = true; + self.active + .back() + .ok_or(EIO)? + .packet + .completion + .quarantine_render(); + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + return Err(error); + } + Ok(()) + } +} + +struct Observer<'a, H> { + memory: &'a memory::Memory, + qids: [u8; 2], + host: &'a mut H, +} +impl retirement::Memory for Observer<'_, H> { + fn snapshot(&mut self, ordinal: u64) -> Result { + observation::snapshot(self.memory, ordinal) + } + + fn stamps(&mut self) -> Result<[u32; 2]> { + self.host.stamps(self.qids) + } + fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]> { + self.memory.pass(ordinal) + } +} + +impl Pair { + fn qos_owners(&self) -> [qos::Owner; 2] { + let qids = self.qids(); + let qos = self.context().qos_id(); + [ + qos::Owner { + qid: qids[0], + qos, + data_master: DataMaster::Tiling as u8, + }, + qos::Owner { + qid: qids[1], + qos, + data_master: DataMaster::Fragment as u8, + }, + ] + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { + masks.is_none_or(|masks| { + self.qids() + .into_iter() + .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + }) + } + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { + self.active + .iter() + .any(|active| Arc::ptr_eq(&active.packet, packet)) + || self + .prepared + .as_ref() + .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) + } + pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { + self.active.iter().find_map(|active| { + active + .packet + .completion + .take_deferred_timeout() + .map(|error| (active.packet.clone(), error)) + }) + } + + pub(crate) fn oldest_packet(&self) -> Option> { + self.active.front().map(|active| active.packet.clone()) + } + /// Sample PM feedback only after this pair's exact retirement. Busy pairs + /// keep their current capacity and let firmware request growth as needed. + pub(crate) fn host_growth_target(&mut self, packet: &Packet) -> Result> { + let Validated::Render { pass, .. } = &packet.command else { + return Err(EINVAL); + }; + let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + if self.in_flight() { + return Ok(None); + } + if !self.idle_for_context(self.context())? { + return Err(EBUSY); + } + let feedback = if self.ordinal != 0 { + self.memory.manager.automatic_target()?.unwrap_or(0) + } else { + 0 + }; + let required = required.max(feedback); + Ok((required > self.memory.manager.blocks()).then_some(required)) + } + + pub(crate) fn needs_drain(&self) -> bool { + self.quarantined && self.in_flight() + } + + pub(crate) fn oldest_spared(&self) -> Option { + self.active + .front() + .map(|active| active.packet.completion.spared()) + } + pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { + for active in self.active.iter() { + visit(active.packet.completion.status()); + } + } + pub(crate) fn classify(&self, classify: &dyn Fn(&Arc) -> recovery::Class) { + for active in self.active.iter() { + active + .packet + .completion + .classify(classify(active.packet.completion.status())); + } + } + /// A slot stamp must name retained work before its execution markers are meaningful. + pub(crate) fn pass_started(&self, stamp: u64) -> Option { + const MASK: u64 = (1 << 40) - 1; + let ordinal = self.active.iter().map(|active| active.ticket.ordinal) + .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; + Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) + } + + /// Firmware is halted and all graph borrowers have joined before this copy. + pub(crate) fn publish_recovery_targets(&self) -> Result { + self.memory.graph.publish_recovery_targets() + } + + pub(crate) fn resync(&mut self) { + self.configs = [None; 2]; + for kick in &mut self.kicks { + kick.clear_parent_after_recovery(); + } + } + pub(crate) fn matches(&self, owner: u64, context: &Arc) -> Result { + Ok(self.owner == Some(owner) + && Arc::ptr_eq(self.context(), context) + && context.is_current()) + } + /// Classification controls VM poisoning. Every failed render fence settles, + /// while command memory remains owned until paired retirement or stop. + pub(crate) fn quarantine( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.quarantined = true; + // Successors published after recovery can still fail behind its spared + // head. Classify the whole failing group before any fence can settle. + if self.active.iter().any(|active| active.packet.completion.spared()) { + for active in self.active.iter() { + active.packet.completion.classify(recovery::Class::Spared); + } + } + for active in self.active.iter_mut() { + active.packet.completion.quarantine_render(); + if !active.failed { + defer(Deferred::FailedOwned( + active.packet.completion.clone(), + error, + ))?; + active.failed = true; + if !active.packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((active.packet.completion.status().clone(), error)); + } + } + } + Ok(()) + } + pub(crate) fn retire( + &mut self, + masks: Option<[u64; 2]>, + _epoch: u64, + state: u32, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.retire_front(masks, state, None, host, defer) + } + + pub(crate) fn retire_observed( + &mut self, + observed: Observed, + state: u32, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + self.retire_front(None, state, Some(observed), host, defer) + } + + fn retire_front( + &mut self, + masks: Option<[u64; 2]>, + state: u32, + observed: Option, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.quarantined || state != 0 || !self.selected(masks) { + return Ok(false); + } + let qids = self.qids(); + let qos = self.qos_owners(); + let pool = self.free_list.id(); + let buffer = self.buffer_id(); + loop { + let Some(front) = self.active.front_mut() else { + return Ok(false); + }; + if !front.outer_published { + return Ok(false); + } + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) + || !self.memory.owns(front.ticket.ordinal) + { + return Err(EIO); + } + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = if let Some(observed) = observed { + if observed.ticket != front.ticket { + return Err(EIO); + } + retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) + } else { + retirement::poll(&mut front.ticket, &mut observer) + }; + let timestamps = match result { + Ok(retirement::Poll::Complete(timestamps)) => timestamps, + Ok(retirement::Poll::Retry | retirement::Poll::Pending) => return Ok(false), + Err(error) => { + self.quarantine(error, defer)?; + return Ok(false); + } + }; + let ordinal = front.ticket.ordinal; + let context_update = front.context_update; + self.note_retirement(ordinal, context_update); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.qos_completed = true; + front.flist_completed = true; + } + defer(Deferred::Retired( + front.packet.completion.clone(), + Ok(timestamps), + ))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + } + + /// A closed preparation epoch invalidates only this unpublished descriptor. + /// Its completion remains available for a retry under the next epoch. + pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { + let Some(prepared) = self.prepared.as_ref() else { + return Ok(()); + }; + if !Arc::ptr_eq(&prepared.packet, packet) { + return Err(EBUSY); + } + self.memory.retire(prepared.ordinal)?; + self.prepared = None; + Ok(()) + } +} + +impl Pair { + /// Call before and again after servicing firmware events. FList completion + /// counters are full-width, including beyond the low-word wrap boundary. + pub(crate) fn firmware_drained(&self, host: &impl Host) -> Result { + if self.prepared.is_some() || self.active.iter().any(|a| !a.outer_published) { + return Ok(false); + } + let counters = self.free_list.counters()?; + let stamps = host.stamps(self.qids())?; + let (_, state) = host.epoch()?; + Ok(retirement::drained(self.ordinal, stamps, counters, state)) + } + + /// Account one witnessed tracker, preserving all completion ownership. + /// The caller services its pending FList release before accounting another + /// tracker and invokes release_witnessed only after that control stage. + pub(crate) fn account_drained(&mut self, host: &mut impl Host) -> Result { + use retirement::Memory; + if !self.firmware_drained(host)? { + return Ok(false); + } + let qids = self.qids(); + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { + return Ok(false); + }; + let mut observer = Observer { + memory: &self.memory, + qids, + host, + }; + let result = match retirement::poll(&mut active.ticket, &mut observer)? { + retirement::Poll::Complete(ticks) => Ok(ticks), + retirement::Poll::Pending | retirement::Poll::Retry => { + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if !retirement::abandoned(&active.ticket, snapshot, stamps, state, true)? { + return Ok(false); + } + Err(EIO) + } + }; + let ordinal = active.ticket.ordinal; + let context_update = result.is_ok() && active.context_update; + self.note_retirement(ordinal, context_update); + let active = self + .active + .iter_mut() + .find(|active| active.ticket.ordinal == ordinal) + .ok_or(EIO)?; + host.complete_render( + (!active.qos_completed).then_some(qos), + self.free_list.id(), + active.generation, + buffer, + )?; + active.qos_completed = true; + active.flist_completed = true; + active.drained_result = Some(result); + Ok(true) + } + + /// Fresh proof after asynchronous control publication is the ownership + /// boundary. A refused deferred item leaves the precise owner available. + pub(crate) fn release_witnessed( + &mut self, + host: &impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if self.active.iter().any(|a| !a.flist_completed) || !self.firmware_drained(host)? { + return Ok(()); + } + while let Some(active) = self.active.front() { + let result = active.drained_result.ok_or(EIO)?; + defer(Deferred::Drained(active.packet.completion.clone(), result))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + self.quarantined = false; + self.resync(); + Ok(()) + } + + /// The registry first polls the completed prefix through this exact Arc. + /// With successors, source policy quarantines every remaining group. A + /// single unexecuted group may use the strict ownership witness instead. + pub(crate) fn fail_after_observation( + &mut self, + packet: &Arc, + error: Error, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use retirement::Memory; + if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { + return Ok(false); + } + if !self.quarantined && self.active.len() == 1 { + let active = self.active.front().ok_or(EIO)?; + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(active.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + if active.outer_published + && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? + { + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let active = self.active.front_mut().ok_or(EIO)?; + if !active.flist_completed { + host.complete_render( + (!active.qos_completed).then_some(qos), + self.free_list.id(), + active.generation, + buffer, + )?; + active.flist_completed = true; + active.qos_completed = true; + } + if !active.failed && !active.packet.completion.spared() { + active.packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((active.packet.completion.status().clone(), error)); + } + } + defer(Deferred::Retired( + active.packet.completion.clone(), + Err(EIO), + ))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + } + self.quarantine(error, defer)?; + Ok(true) + } + + /// Both processors have stopped before the caller reaches this method. + /// VM ownership settles before quarantined admission is rehabilitated. + pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { + if let Some(prepared) = self.prepared.as_ref() { + defer(Deferred::Unpublished( + prepared.packet.completion.clone(), + EIO, + ))?; + self.memory.retire(prepared.ordinal)?; + self.prepared = None; + } + while let Some(active) = self.active.front() { + defer(Deferred::Drained( + active.packet.completion.clone(), + Err(EIO), + ))?; + let ordinal = active.ticket.ordinal; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + } + if let Some(previous) = self.previous.as_mut() { + previous.ready = true; + } + Ok(()) + } +} + +impl Pair { + /// Commit a cold graph only after the registry revalidates its preparation + /// lease. The original PB row is restored on every later failure. + pub(crate) fn finish_install( + &mut self, + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + ) -> Result { + install_memory( + alloc, + descriptors, + self.slot, + &mut self.memory, + &mut self.pending_aliases, + &mut self.kick_aliases, + ) + } +} + +fn install_memory( + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + slot: u8, + memory: &mut memory::Memory, + pending_aliases: &mut Option>, + kick_aliases: &mut Option<[Option; 2]>, +) -> Result { + use core::sync::atomic::{fence, Ordering}; + if pending_aliases.is_none() && kick_aliases.is_none() { + return Ok(()); + } + if !memory.context.is_current() || kick_aliases.is_none() { + return Err(EIO); + } + let mut driver = KVec::new(); + driver.reserve(2, GFP_KERNEL)?; + let aliases = memory.take_persistent(pending_aliases.as_mut().ok_or(EIO)?)?; + let context = memory.context.clone(); + let vm = context.vm(); + let pool_id = u64::from(memory.lifecycle[0].predecessor); + let mut staged: Option<[u32; 4]> = None; + let offset = usize::from(memory.manager.buffer_id()) * 16; + let result: Result = (|| { + vm.install_render_pool_mappings(slot, pool_id, memory.manager.blocks(), aliases)?; + memory.installed_pool = Some(slot); + alloc.uat.flush_vm_contexts(vm)?; + let words = [ + descriptors.word(offset)?, + descriptors.word(offset + 4)?, + descriptors.word(offset + 8)?, + descriptors.word(offset + 12)?, + ]; + let previous = words.map(|word| word.load(Ordering::Relaxed)); + let pages: u32 = (memory.manager.blocks() + * crate::hw::t8140::resources::TVB_PAGES_PER_BLOCK) + .try_into()?; + let next = crate::g17::fw::buffer::reload_descriptor( + previous, + memory.manager.page_list_va(), + pages, + ); + for (word, value) in words.into_iter().zip(next) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + staged = Some(previous); + for mapping in kick_aliases.take().ok_or(EIO)?.into_iter().flatten() { + driver.push(mapping, GFP_KERNEL)?; + } + vm.append_driver_mappings(driver)?; + Ok(()) + })(); + if let Err(error) = result { + if error == ENOENT { + context.status().record(error); + } + if let Some(previous) = staged { + for (index, value) in previous.into_iter().enumerate() { + descriptors + .word(offset + index * 4)? + .store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + } + vm.clear_render_pool_mappings_if_owner(slot, pool_id); + memory.installed_pool = None; + return Err(error); + } + *pending_aliases = None; + Ok(()) +} + +/// A binding prepared while the registry exclusively lends the idle pair. +/// Its frontier prevents a stale off-lock result from replacing another owner. +pub(crate) struct PreparedBinding { + previous_context: Arc, + previous_owner: Option, + ordinal: u64, + owner: u64, + context: Arc, + pool: Arc, + memory: Option>, + aliases: Option>, + kicks: Option<[Option; 2]>, + initial_packet: u64, +} + +/// The registry retains displaced backing until this exact context's release +/// is acknowledged. A warm binding retains the old context without its graph. +pub(crate) struct RetiredBinding { + memory: Option>, + context: Arc, + pool: Arc, + buffer_id: u8, + publication: Option, + ready: bool, +} +impl RetiredBinding { + pub(crate) fn context(&self) -> &Arc { + &self.context + } + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } + pub(crate) fn buffer_id(&self) -> Option { + self.memory.as_ref().map(|_| self.buffer_id) + } +} + +impl Pair { + fn note_retirement(&mut self, ordinal: u64, context_update: bool) { + if !context_update { + return; + } + self.context_update = false; + if let Some(previous) = self.previous.as_mut() { + if previous + .publication + .is_some_and(|published| published <= ordinal) + { + previous.ready = true; + } + } + } + pub(crate) fn take_retired_binding(&mut self) -> Option { + if self.displaced.is_some() { + return self.displaced.take(); + } + if self.previous.as_ref().is_some_and(|p| p.ready) { + self.previous.take() + } else { + None + } + } + pub(crate) fn idle_for_context(&self, context: &Arc) -> Result { + if !Arc::ptr_eq(context, self.context()) { + return Ok(true); + } + if self.prepared.is_some() || self.in_flight() || self.quarantined { + return Ok(false); + } + if self.ordinal == 0 { + return Ok(true); + } + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + Ok(self + .memory + .graph + .cursors()? + .into_iter() + .all(|c| c.done == c.read && c.read == c.write)) + } + pub(crate) fn reusable(&self) -> Result { + let previous_ready = self + .previous + .as_ref() + .is_none_or(|p| p.memory.is_none() && self.context_update && !p.ready); + Ok(self.owner.is_none() + && self.displaced.is_none() + && previous_ready + && self.idle_for_context(self.context())?) + } + /// Logical close leaves the physical channel identity and backing retained. + pub(crate) fn release_owner(&mut self) { + self.owner = None; + } + #[inline(never)] + pub(crate) fn prepare_bind( + &mut self, + alloc: &Allocator<'_>, + pool: &Arc, + owner: u64, + context: Arc, + free_list: Arc, + metrics: Option, + buffer_id: Option<(u8, u64)>, + pm_generation: Option, + first: &Arc, + ) -> Result { + if !self.reusable()? + || owner == 0 + || !Arc::ptr_eq(&context, &first.context) + || !context.is_current() + { + return Err(EBUSY); + } + alloc.uat.vm_context_mask(context.vm())?; + let same_vm = Arc::ptr_eq(self.context().status(), context.status()); + let (memory, aliases, kicks) = if same_vm { + if metrics.is_some() || buffer_id.is_some() || pm_generation.is_some() { + return Err(EINVAL); + } + (None, None, None) + } else { + let metrics = metrics.ok_or(EINVAL)?; + let buffer_id = buffer_id.ok_or(EINVAL)?; + let pm_generation = pm_generation.ok_or(EINVAL)?; + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let layout = target::Layout::new(pass, self.clusters)?; + let aliases = [ + self.kicks[0].prepare_client_alias(context.vm())?, + self.kicks[1].prepare_client_alias(context.vm())?, + ]; + let (memory, persistent) = memory::Memory::new( + alloc, + pool, + context.clone(), + self.qids(), + &layout, + metrics, + buffer_id, + pm_generation, + self.ordinal, + first.completion.work_state().word(), + )?; + (Some(memory), Some(persistent), Some(aliases)) + }; + Ok(PreparedBinding { + previous_context: self.context().clone(), + previous_owner: self.owner, + ordinal: self.ordinal, + owner, + context, + pool: free_list, + memory, + aliases, + kicks, + initial_packet: first.id, + }) + } + pub(crate) fn bind_owner( + &mut self, + binding: &mut PreparedBinding, + alloc: &Allocator<'_>, + descriptors: &crate::g17::object::KernelObject, + ) -> Result { + if !self.reusable()? + || binding.ordinal != self.ordinal + || binding.previous_owner != self.owner + || !Arc::ptr_eq(&binding.previous_context, self.context()) + || !binding.context.is_current() + { + return Err(EBUSY); + } + let foreign = binding.memory.is_some(); + if let Some(memory) = binding.memory.as_mut() { + install_memory( + alloc, + descriptors, + self.slot, + memory, + &mut binding.aliases, + &mut binding.kicks, + )?; + } else { + self.memory.set_owner(self.qids(), &binding.context)?; + } + let old_context = self.context().clone(); + let old_buffer = self.buffer_id(); + let old_pool = core::mem::replace(&mut self.free_list, binding.pool.clone()); + let old_memory = binding + .memory + .take() + .map(|memory| core::mem::replace(&mut self.memory, memory)); + let retired = RetiredBinding { + memory: old_memory, + context: old_context, + pool: old_pool, + buffer_id: old_buffer, + publication: None, + ready: foreign, + }; + if foreign { + self.displaced = Some(retired); + } else if self.previous.is_none() { + self.previous = Some(retired); + } + self.memory.context = binding.context.clone(); + self.owner = Some(binding.owner); + self.initial_packet = binding.initial_packet; + self.context_update = true; + if foreign { + self.cursors = [0; 2]; + self.configs = [None; 2]; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs new file mode 100644 index 00000000000000..d1ec0eb76c7e18 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render predecessors retained while a physical pair is prepared off-lock. + +use super::Pair; +use crate::g17::{ + dependency::Frontier, + fw::kick::KickDependency, + job::{self, Packet}, + status::VmStatus, +}; +use crate::hw::t8140::queues::RENDER_DEPTH; +use kernel::{prelude::*, sync::Arc}; + +struct Predecessor { + packet: Arc, + word: u64, +} + +/// A loan of the graph cannot hide work already published on that pair. Fence +/// status remains live through the retained Packet, even while its storage is away. +pub(crate) struct Dependencies { + owner: Option, + entries: [Option; RENDER_DEPTH as usize], +} + +fn include( + frontier: &mut Frontier, + status: &Arc, + prefix: Option, + packet: &Arc, + word: u64, +) -> Result { + if !Arc::ptr_eq(packet.completion.status(), status) { + return Ok(()); + } + if !job::Order::contains(prefix, packet.order.sequence) { + return Ok(()); + } + frontier.include( + prefix, + packet.order.sequence, + job::fence_status(&packet.completion.fence()), + word, + ) +} + +impl Dependencies { + pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { + self.entries + .iter() + .flatten() + .any(|entry| Arc::ptr_eq(&entry.packet, packet)) + } + + pub(crate) fn collect( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + for entry in self.entries.iter().flatten() { + include(frontier, status, prefix, &entry.packet, entry.word)?; + } + Ok(()) + } +} + +impl Pair { + /// The fixed tracker bounds this snapshot; no allocation or packet can be + /// lost between capture and the registry moving the pair under the same lock. + pub(crate) fn dependency_snapshot(&self) -> Dependencies { + let qid = self.qids()[1]; + let mut active = self.active.iter(); + Dependencies { + owner: self.owner, + entries: core::array::from_fn(|_| { + active.next().map(|entry| Predecessor { + packet: entry.packet.clone(), + word: KickDependency::new(qid, entry.kicks[1]) + .map_or(u64::MAX, KickDependency::word), + }) + }), + } + } + + pub(crate) fn collect_dependencies( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + let qid = self.qids()[1]; + for entry in self.active.iter() { + let word = + KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); + include(frontier, status, prefix, &entry.packet, word)?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs new file mode 100644 index 00000000000000..f614494e1216aa --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -0,0 +1,708 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Storage of one render pair and its per-command generations. Client aliases +//! precede their backing owners so teardown cannot leave a dangling GPU view. + +use super::retirement::Cursors; +use crate::g17::{ + buffer::{self, target}, + context::Context, + fw::{ + self, queue, + render::{self as abi, storage}, + }, + object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, +}; +use crate::{ + hw::t8140::{dynamic, kernel_window, low, queues, resources as cfg}, + mmu, +}; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +pub(super) const RECORDS: usize = 32; +pub(super) const TA_BYTES: usize = 0x14000; +pub(super) const FRAGMENT_BYTES: usize = 0x48000; +pub(super) const GRAPH_SIZE: usize = 0x1c000; +pub(super) const POINTERS: [usize; 2] = [0x9000, 0x9080]; +pub(super) const ITEMS: [usize; 2] = [0xc000, 0x10000]; +pub(super) const ANNOUNCE_STRIDE: usize = 0x80; +pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; +pub(super) const JOB_LIST: usize = 0xa000; +pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; +pub(super) const CONFIG_STRIDE: usize = 0x180; +const DEPTH: usize = queues::RENDER_DEPTH as usize; + +struct Aliases { + client: mmu::KernelMapping, + _global: mmu::KernelMapping, +} +impl Aliases { + fn new( + object: &KernelObject, + alloc: &Allocator<'_>, + vm: &mmu::Vm, + source: core::ops::Range, + ) -> Result { + let mut range = dynamic::LOWER; + let size = (source.end - source.start) as u64; + loop { + let global = object.map_range( + alloc.uat.kernel_lower_vm(), + source.clone(), + range.clone(), + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + )?; + let address = global.iova(); + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + match object.map_range( + vm, + source.clone(), + address..end, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + ) { + Ok(client) => { + return Ok(Self { + client, + _global: global, + }) + } + Err(error) if error == ENOSPC && end < range.end => { + drop(global); + range.start = end; + } + Err(error) => return Err(error), + } + } + } +} + +/// Stable queue records and rotating descriptors/configuration records. +pub(super) struct Graph { + aliases: [Aliases; 2], + pub(super) descriptors: PooledObject, + pub(super) queues: PooledObject, + pub(super) config: PooledObject, + pub(super) support: KernelObject, + pub(super) owners: [Option; RECORDS], +} +impl Graph { + pub(super) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + context: &Context, + qids: [u8; 2], + ) -> Result { + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let descriptors = pool.allocate( + alloc, + Family::Descriptor, + TA_BYTES + FRAGMENT_BYTES, + TA_BYTES + FRAGMENT_BYTES, + prot, + CpuMap::WriteBack, + )?; + let ta = Aliases::new(&descriptors, alloc, context.vm(), 0..TA_BYTES)?; + let fragment = Aliases::new( + &descriptors, + alloc, + context.vm(), + TA_BYTES..TA_BYTES + FRAGMENT_BYTES, + )?; + let mut queues = pool.allocate( + alloc, + Family::QueueState, + GRAPH_SIZE, + 0x8000, + prot, + CpuMap::WriteCombined, + )?; + let config = pool.allocate( + alloc, + Family::QueueConfig, + 0x8000, + 0x8000, + prot, + CpuMap::WriteBack, + )?; + let support = alloc.kernel( + storage::SUPPORT_SIZE, + mmu::UAT_PGSZ as u64, + prot, + CpuMap::WriteCombined, + )?; + let address = queues.gpu_va(); + for stage in 0..2 { + queues.write( + qids[stage] as usize * size_of::(), + queue::QueueRecord::new(&queue::QueueRecordArgs { + pointers_va: address + POINTERS[stage] as u64, + items_va: address + ITEMS[stage] as u64, + job_list_va: context.work_head_va(), + scheduler_va: context.scheduler_va(), + owner_pid: context.owner_pid(), + }), + )?; + queues.initialize::(POINTERS[stage], |view| { + *view = queue::PointerBlock::new(); + Ok(()) + })?; + } + queues.write(JOB_LIST, queue::JobListHead::new(address + JOB_LIST as u64))?; + Ok(Self { + aliases: [ta, fragment], + descriptors, + queues, + config, + support, + owners: [None; RECORDS], + }) + } + pub(super) fn offset(stage: usize, ordinal: u64) -> usize { + let slot = ordinal as usize % RECORDS; + if stage == 0 { + slot * abi::TA_DESCRIPTOR_SIZE + } else { + TA_BYTES + slot * abi::FRAGMENT_SLOT_SIZE + } + } + pub(super) fn descriptor_va(&self, stage: usize, ordinal: u64) -> u64 { + self.descriptors.gpu_va() + Self::offset(stage, ordinal) as u64 + } + pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> u64 { + let stride = if stage == 0 { + abi::TA_DESCRIPTOR_SIZE + } else { + abi::FRAGMENT_SLOT_SIZE + }; + self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 + } + pub(super) fn publish_recovery_targets(&self) -> Result { + for at in POINTERS { + let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? + .load(Ordering::Relaxed); + self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? + .store(producer, Ordering::Relaxed); + } + Ok(()) + } + + pub(super) fn cursors(&self) -> Result<[Cursors; 2]> { + let mut values = [Cursors { + done: 0, + read: 0, + write: 0, + }; 2]; + for stage in 0..2 { + let at = POINTERS[stage]; + values[stage] = Cursors { + done: self.queues.word(at)?.load(Ordering::Relaxed), + read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), + write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), + }; + } + Ok(values) + } + pub(super) fn claim(&mut self, ordinal: u64) -> Result { + let owner = &mut self.owners[ordinal as usize % RECORDS]; + if owner.is_some_and(|old| old != ordinal) { + return Err(EBUSY); + } + *owner = Some(ordinal); + Ok(()) + } + pub(super) fn owns(&self, ordinal: u64) -> bool { + self.owners[ordinal as usize % RECORDS] == Some(ordinal) + } + pub(super) fn release(&mut self, ordinal: u64) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + self.owners[ordinal as usize % RECORDS] = None; + Ok(()) + } +} + +struct Compact { + mapping: Option, + address: u64, + _backing: KernelObject, +} +impl Compact { + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { + let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; + let mapping = buffer::compact_alias( + &backing, + vm, + 0..size, + cfg::TVB_PAGE, + mmu::PROT_GPU_SHARED_RW, + mmu::UAT_PGSZ, + )?; + Ok(Self { + address: mapping.iova(), + mapping: Some(mapping), + _backing: backing, + }) + } +} + +pub(super) struct Generation { + deflake: mmu::KernelMapping, + auxiliary: mmu::KernelMapping, + fragment_client: mmu::KernelMapping, + state: KernelObject, + fragment: KernelObject, + pub(super) ordinal: Option, +} +impl Generation { + fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { + KernelObject::new( + alloc.dev, + vm, + crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, cfg::TVB_PAGE), + storage::STATE_SIZE, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + ) + } + fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { + let deflake = buffer::compact_alias( + &state, + vm, + storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + mmu::UAT_PGSZ, + )?; + let auxiliary_range = cfg::AUXILIARY; + let auxiliary = state.map_range( + vm, + storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, + auxiliary_range, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + 0, + )?; + let fragment = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let fragment_client = buffer::compact_alias( + &fragment, + vm, + 0..mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RW, + mmu::UAT_PGSZ, + )?; + KBox::new( + Self { + deflake, + auxiliary, + fragment_client, + state, + fragment, + ordinal: None, + }, + GFP_KERNEL, + ) + .map_err(Into::into) + } + pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { + if self.ordinal.is_some_and(|old| old != ordinal) { + return Err(EBUSY); + } + let viewport = storage::Viewport::new(pass.width, pass.height)?; + self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] + .fill(0); + let destination = self + .state + .pointer(storage::DEFLAKE + 0x900, size_of::())? + .cast::(); + if destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + // SAFETY: The complete typed integer view is in the retained CPU mapping + // and naturally aligned. This command generation is exclusively owned + // and unpublished; no prior command can access it until reassigned. + unsafe { viewport.apply(&mut *destination) }; + self.fragment.bytes_mut().fill(0); + self.ordinal = Some(ordinal); + Ok(()) + } + pub(super) fn addresses(&self) -> [u64; 5] { + [ + self.deflake.iova(), + self.state.gpu_va() + storage::TA_STATUS as u64, + self.fragment_client.iova(), + self.fragment.gpu_va(), + self.auxiliary.iova(), + ] + } +} + +pub(super) struct Memory { + _fragment_view: mmu::KernelMapping, + pub(super) graph: Graph, + scratch: Compact, + discard: Compact, + pub(super) target: KBox, + generations: [Option>; DEPTH], + generation_count: usize, + pub(super) timestamps: KernelObject, + pub(super) manager: buffer::Manager, + pub(super) lifecycle: [fw::ObjectIds; 2], + // The extra client views retain their allocation positions even though + // kick entries consume the register arrays embedded in the descriptors. + registers: KernelObject, + pub(super) context: Arc, + pub(super) installed_pool: Option, +} +impl Memory { + #[inline(never)] + pub(super) fn new( + alloc: &Allocator<'_>, + pool: &Arc, + context: Arc, + qids: [u8; 2], + layout: &target::Layout, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + ordinal: u64, + word: u32, + ) -> Result<(KBox, KVec)> { + let storage = KBox::::new_uninit(GFP_KERNEL)?; + let vm = context.vm(); + let mut graph = Graph::new(alloc, pool, &context, qids)?; + let mut target = target::Target::new(alloc.dev, vm, layout)?; + target.prepare(ordinal, layout)?; + let state = Generation::allocate_state(alloc, vm)?; + let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; + let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; + let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; + let count = DEPTH; + let initial = if ordinal == 0 { + 0 + } else { + (ordinal ^ 1) as usize % count + }; + let mut generations = core::array::from_fn(|_| None); + generations[initial] = Some(Generation::new(alloc, vm, state)?); + for offset in 1..count { + let index = (initial + offset) % count; + let state = Generation::allocate_state(alloc, vm)?; + generations[index] = Some(Generation::new(alloc, vm, state)?); + } + let timestamps = alloc.kernel( + mmu::UAT_PGSZ, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_FW_SHARED_RW, + CpuMap::WriteCombined, + )?; + let manager = buffer::Manager::new( + alloc, + vm, + tvb, + metrics, + buffer::ManagerArgs { + scratch: scratch.address, + discard: discard.address, + buffer_id: buffer_id.0, + generation: pm_generation, + first_ordinal: ordinal, + }, + )?; + let predecessor = crate::g17::ids::OBJECTS.object()?; + let (fragment, tiling) = crate::g17::ids::OBJECTS.render_pair(predecessor)?; + let support_va = graph.support.gpu_va(); + graph.support.initialize::(0, |view| { + view.init( + support_va, + alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, + low::RENDER_RUN_LIST, + low::RENDER_PAGE_LIST, + word, + ) + })?; + let registers = KernelObject::new( + alloc.dev, + vm, + crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), + mmu::UAT_PGSZ, + mmu::PROT_GPU_SHARED_RO, + CpuMap::WriteCombined, + )?; + let fragment_view = graph.descriptors.map_range( + vm, + TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, + dynamic::CLIENT_LOWER, + mmu::UAT_PGSZ as u64, + mmu::PROT_GPU_SHARED_RO, + 0, + )?; + Ok(( + storage.write_init(kernel::try_init!(Self { + graph, + scratch, + discard: discard, + target, + generations, + generation_count: count, + timestamps, + manager, + lifecycle: [tiling, fragment], + _fragment_view: fragment_view, + registers: registers, + context, + installed_pool: None, + }))?, + mappings, + )) + } + pub(super) fn generation(&mut self, ordinal: u64) -> Result<&mut Generation> { + self.generations[ordinal as usize % self.generation_count] + .as_deref_mut() + .ok_or(EIO) + } + pub(super) fn owns(&self, ordinal: u64) -> bool { + self.graph.owns(ordinal) + && self.generations[ordinal as usize % self.generation_count] + .as_ref() + .is_some_and(|generation| generation.ordinal == Some(ordinal)) + } + pub(super) fn pass(&self, ordinal: u64) -> Result<[u64; 2]> { + if !self.owns(ordinal) { + return Err(EIO); + } + let offset = ordinal as usize % RECORDS * 16; + Ok([ + self.timestamps.dword(offset)?.load(Ordering::Relaxed), + self.timestamps.dword(offset + 8)?.load(Ordering::Relaxed), + ]) + } + pub(super) fn owners(&self) -> [Option; DEPTH] { + core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal)) + } + pub(super) fn retire(&mut self, ordinal: u64) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + self.target.retire(ordinal)?; + self.manager.retire_scene(ordinal)?; + self.graph.release(ordinal)?; + self.generation(ordinal)?.ordinal = None; + Ok(()) + } +} + +const _: () = { + assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); + assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); + assert!(RECORDS * CONFIG_STRIDE <= 0x4000); +}; + +impl Memory { + pub(super) fn write_descriptors( + &mut self, + args: &abi::RenderArgs<'_>, + attachments: &[fw::kick::WriteRange], + context: u8, + ) -> Result> { + let tiling = self + .graph + .descriptors + .pointer( + Graph::offset(0, args.ordinal), + size_of::(), + )? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer( + Graph::offset(1, args.ordinal), + size_of::(), + )? + .cast::(); + if tiling.align_offset(align_of::()) != 0 + || fragment.align_offset(align_of::()) != 0 + { + return Err(EINVAL); + } + // SAFETY: Both complete integer-only views lie in the retained mapping, + // are aligned and disjoint. The full ordinal has an exclusive slot + // claim and no producer refers to this command yet. + unsafe { + (&mut *tiling).write(args)?; + let fragment = &mut *fragment; + fragment.write(args)?; + self.registers.bytes_mut().fill(0); + for (index, array) in fragment.descriptor.arrays.iter().enumerate() { + for (entry, value) in array + .entries + .get(..(array.size & 0xffff) as usize) + .ok_or(EINVAL)? + .iter() + .enumerate() + { + self.registers.write( + index * size_of::() + + entry * size_of::(), + *value, + )?; + } + } + fragment.set_mcache( + args.fragment.descriptor_va, + args.resources.aux_fb_va, + attachments, + context, + ) + } + } +} + +impl Memory { + pub(super) fn completion_links( + &mut self, + ordinal: u64, + kicks: [fw::kick::KickTimestamp; 2], + payload: u8, + fragment_busy: bool, + ) -> Result { + if !self.owns(ordinal) { + return Err(EIO); + } + let tiling = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + // SAFETY: The complete views were bounds/alignment checked during + // preparation. The unchanged full ordinal still exclusively owns both + // slots and neither descriptor has been published. + unsafe { + (&mut *tiling).set_completion_link(kicks[0], kicks[1], payload, fragment_busy); + (&mut *fragment).set_completion_link(kicks[1], payload); + } + Ok(()) + } +} + +impl Memory { + /// Preserve the initial mapping order while moving VM-lifetime mappings + /// out of the graph. Reserve all host storage before moving any mapping. + pub(super) fn take_persistent( + &mut self, + tvb: &mut KVec, + ) -> Result> { + if self.scratch.mapping.is_none() + || self.discard.mapping.is_none() + || self.registers.gpu_va() == 0 + { + return Err(EIO); + } + let mut aliases = KVec::new(); + aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; + aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; + aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; + for mapping in core::mem::take(tvb) { + aliases.push(mapping, GFP_KERNEL)?; + } + Ok(aliases) + } +} + +impl Memory { + pub(super) fn bind_work_slot(&mut self, ordinal: u64, qids: [u8; 2], slot: u32) -> Result { + if slot >= 4 || !self.owns(ordinal) { + return Err(EINVAL); + } + let queues = [ + self.graph + .queues + .word(usize::from(qids[1]) * size_of::() + 0x38)?, + self.graph + .queues + .word(usize::from(qids[0]) * size_of::() + 0x38)?, + ]; + let ta = self + .graph + .descriptors + .pointer(Graph::offset(0, ordinal), size_of::())? + .cast::(); + let fragment = self + .graph + .descriptors + .pointer(Graph::offset(1, ordinal), size_of::())? + .cast::(); + for word in queues { + word.store(slot, Ordering::Relaxed); + } + // SAFETY: The generation owns these prepared, unpublished POD views. + // Packed fields are assigned by value without an unaligned reference. + unsafe { + (*ta).unk_8be = slot; + (*fragment).descriptor.unk_2158 = slot; + } + core::sync::atomic::fence(Ordering::Release); + Ok(()) + } + + pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { + let offsets = qids.map(|qid| usize::from(qid) * size_of::()); + let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; + // Check every field before the first update, retaining the required + // job-head pair, PID pair, then packed scheduler-pointer pair order. + for at in offsets { + self.graph.queues.pointer(at + 0x10, 8)?; + self.graph.queues.pointer(at + 0x48, 4)?; + self.graph.queues.pointer(at + 0x9c, 8)?; + } + // SAFETY: Both queues are retired before retargeting. Complete byte + // views were checked above; packed scheduler words use unaligned stores. + unsafe { + for at in offsets { + base.add(at + 0x10) + .cast::() + .write_unaligned(context.work_head_va()); + } + for at in offsets { + base.add(at + 0x48) + .cast::() + .write_unaligned(context.owner_pid()); + } + for at in offsets { + base.add(at + 0x9c) + .cast::() + .write_unaligned(context.scheduler_va()); + } + } + Ok(()) + } +} + +impl Drop for Memory { + fn drop(&mut self) { + if let Some(slot) = self.installed_pool { + self.context.vm().clear_render_pool_mappings_if_owner( + slot, + u64::from(self.lifecycle[0].predecessor), + ); + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/observation.rs b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs new file mode 100644 index 00000000000000..3e853115e4f75a --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/observation.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Read-only cursor and timestamp snapshots while a physical pair is borrowed. +//! Final stamps, timestamp refresh and ownership changes belong to commit. + +use super::{memory, retirement, Pair}; +use crate::g17::job::Packet; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +pub(crate) struct Observed { + pub(super) ticket: retirement::Ticket, + pub(super) snapshot: retirement::Snapshot, +} + +impl Pair { + /// Reject incomplete event/stamp prefixes before lending the graph. This + /// is only a prefilter; commit repeats the final stamp and timestamp reads. + pub(crate) fn observation_packet( + &self, + masks: [u64; 2], + stamps: [u32; 2], + state: u32, + ) -> Result>> { + if self.quarantined || self.prepared.is_some() || state != 0 || !self.selected(Some(masks)) + { + return Ok(None); + } + let Some(front) = self.active.front() else { + return Ok(None); + }; + if !front.outer_published { + return Ok(None); + } + let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + if !stamps + .into_iter() + .all(|stamp| retirement::stamp_covers(stamp, published)) + { + return Ok(None); + } + Ok(Some(front.packet.clone())) + } + + pub(crate) fn observe_front(&self) -> Result { + let front = self.active.front().ok_or(EIO)?; + if !Arc::ptr_eq(&front.packet.context, &self.memory.context) { + return Err(EIO); + } + let ticket = front.ticket; + for _ in 0..2 { + fence(Ordering::Acquire); + let snapshot = snapshot(&self.memory, ticket.ordinal)?; + if retirement::observe(&ticket, snapshot).is_ok() { + return Ok(Observed { ticket, snapshot }); + } + } + Err(EAGAIN) + } +} + +pub(super) fn snapshot(memory: &memory::Memory, ordinal: u64) -> Result { + if !memory.owns(ordinal) { + return Err(EIO); + } + let cursors = memory.graph.cursors()?; + // Sample the private head before the logical shared head, whose membership + // may include other pairs. Individual completion does not require it empty. + let _first = memory + .graph + .queues + .dword(memory::JOB_LIST)? + .load(Ordering::Relaxed); + let _last = memory + .graph + .queues + .dword(memory::JOB_LIST + 8)? + .load(Ordering::Relaxed); + let head = memory.context.work_head()?; + Ok(retirement::Snapshot { + cursors, + job_list_empty: head == [0, memory.context.work_head_va()], + pass: memory.pass(ordinal)?, + }) +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs new file mode 100644 index 00000000000000..35955810a5af1c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Installed kick storage retained after a logical scheduler release is acknowledged. + +use super::{install_memory, memory, target, Pair, RetiredBinding, Tracker}; +use crate::g17::{ + buffer, + command::Validated, + context::Context, + freelist::RenderPool, + job::Packet, + kick, + object::{Allocator, KernelObject, Pool}, +}; +use crate::mmu; +use kernel::{prelude::*, sync::Arc}; + +/// The empty physical pair retains installed addresses and the last scheduler page, +/// without retaining that scheduler's execution context or client root. A prior +/// context awaiting an executed replacement keeps its separate retirement owner. +pub(crate) struct ParkedPair { + kicks: Option<[kick::Queue; 2]>, + _scheduler: Arc, + ids: [kick::Id; 2], + slot: u8, + ordinal: u64, + clusters: u32, + descriptor_flags: [u32; 2], + installed: [bool; 2], + recovery_generation: u64, + registered: bool, + outer_started: bool, + previous: Option, +} + +/// All new graph ownership stays separate from the installed kick storage until commit. +pub(crate) struct Prepared { + memory: Option>, + pool: Arc, + aliases: Option>, + kicks: Option<[Option; 2]>, + ids: [kick::Id; 2], + slot: u8, + ordinal: u64, + owner: u64, + initial_packet: u64, + installed: bool, +} + +impl Pair { + /// The registry also checks the shared free-list release and scheduler-release witness. + /// Drain displaced owners before this preflight; they have their own logical releases. + pub(crate) fn can_park(&self, context: &Arc) -> Result { + Ok(Arc::ptr_eq(context, self.context()) + && self.owner.is_none() + && self + .previous + .as_ref() + .is_none_or(|previous| previous.memory.is_none() && !previous.ready) + && self.displaced.is_none() + && self.pending_aliases.is_none() + && self.kick_aliases.is_none() + && self.idle_for_context(context)?) + } + + /// Consumes the pair only after can_park and all collector capacity checks succeeded + /// under the same device mutex. Moving these owners performs no firmware operation. + pub(crate) fn park(self) -> (ParkedPair, RetiredBinding) { + let context = self.memory.context.clone(); + let scheduler = context.scheduler_owner(); + let buffer_id = self.buffer_id(); + let parked = ParkedPair { + ids: self.kicks.each_ref().map(|queue| queue.id()), + kicks: Some(self.kicks), + _scheduler: scheduler, + slot: self.slot, + ordinal: self.ordinal, + clusters: self.clusters, + descriptor_flags: self.descriptor_flags, + installed: self.installed, + recovery_generation: self.recovery_generation, + registered: self.registered, + outer_started: self.outer_started, + previous: self.previous, + }; + let retired = RetiredBinding { + memory: Some(self.memory), + context, + pool: self.free_list, + buffer_id, + publication: None, + ready: true, + }; + (parked, retired) + } +} + +impl ParkedPair { + pub(crate) fn qids(&self) -> [u8; 2] { + self.ids.map(kick::Id::qid) + } + + #[inline(never)] + pub(crate) fn prepare( + &self, + alloc: &Allocator<'_>, + pool: &Arc, + owner: u64, + context: Arc, + free_list: Arc, + metrics: buffer::MetricsLease, + buffer_id: (u8, u64), + pm_generation: u64, + first: &Arc, + ) -> Result { + if owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() { + return Err(EINVAL); + } + let kicks = self.kicks.as_ref().ok_or(EBUSY)?; + let Validated::Render { pass, .. } = &first.command else { + return Err(EINVAL); + }; + let word = first.completion.work_state().word(); + if word == u32::MAX { + return Err(EINVAL); + } + alloc.uat.vm_context_mask(context.vm())?; + let layout = target::Layout::new(pass, self.clusters)?; + let aliases = [ + kicks[0].prepare_client_alias(context.vm())?, + kicks[1].prepare_client_alias(context.vm())?, + ]; + let (memory, persistent) = memory::Memory::new( + alloc, + pool, + context, + self.qids(), + &layout, + metrics, + buffer_id, + pm_generation, + self.ordinal, + word, + )?; + Ok(Prepared { + memory: Some(memory), + pool: free_list, + aliases: Some(persistent), + kicks: Some(aliases), + ids: self.ids, + slot: self.slot, + ordinal: self.ordinal, + owner, + initial_packet: first.id, + installed: false, + }) + } + + fn matches(&self, prepared: &Prepared) -> bool { + self.kicks.is_some() + && self.ids == prepared.ids + && self.slot == prepared.slot + && self.ordinal == prepared.ordinal + && prepared + .memory + .as_ref() + .is_some_and(|memory| memory.context.is_current()) + } + + /// All fallible graph installation precedes taking the installed kicks. Keep the empty + /// parked owner until the caller drops it off-lock, including its old scheduler page. + pub(crate) fn commit( + &mut self, + prepared: &mut Option, + storage: KBox>, + ) -> Result> { + let candidate = prepared.as_ref().ok_or(EINVAL)?; + if !candidate.installed || !self.matches(candidate) { + return Err(EBUSY); + } + let kicks = self.kicks.take().ok_or(EIO)?; + // The candidate was validated above and cannot change through the exclusive borrow. + let Some(prepared) = prepared.take() else { + self.kicks = Some(kicks); + return Err(EIO); + }; + let mut next = prepared; + let Some(memory) = next.memory.take() else { + self.kicks = Some(kicks); + return Err(EIO); + }; + storage.write_init(kernel::try_init!(Pair { + kick_aliases: None, + memory, + kicks, + free_list: next.pool.clone(), + pending_aliases: None, + prepared: None, + active: Tracker::new(), + slot: self.slot, + owner: Some(next.owner), + ordinal: self.ordinal, + initial_packet: next.initial_packet, + clusters: self.clusters, + descriptor_flags: self.descriptor_flags, + installed: self.installed, + configs: [None; 2], + cursors: [0; 2], + recovery_generation: self.recovery_generation, + registered: self.registered, + outer_started: self.outer_started, + quarantined: false, + terminal: None, + previous: self.previous.take(), + displaced: None, + context_update: true, + })) + } +} + +impl Prepared { + /// Called after the registry revalidates its preparation epoch and parked pair identity. + pub(crate) fn finish_install( + &mut self, + parked: &ParkedPair, + alloc: &Allocator<'_>, + descriptors: &KernelObject, + ) -> Result { + if !parked.matches(self) { + return Err(EBUSY); + } + install_memory( + alloc, + descriptors, + self.slot, + self.memory.as_mut().ok_or(EIO)?, + &mut self.aliases, + &mut self.kicks, + )?; + self.installed = true; + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs new file mode 100644 index 00000000000000..3df92062735e33 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Preflighted item-ring publications. Each lane retains an independent cursor +//! and omits its configuration record only when its durable identity matches. + +use super::{memory, retirement::Window}; +use crate::g17::fw::{ + kick::KickTimestamp, + queue::{DataMaster, KickAnnounce, QueueConfig, ITEM_RING_ENTRIES}, +}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use kernel::prelude::*; + +fn identity(mut config: QueueConfig) -> QueueConfig { + config.install = 0; + config.context_update = 0; + config.completion_seed_valid = 0; + config.completion_seed = 0; + config.qos_update = 0; + config +} +pub(super) fn required(previous: Option, config: QueueConfig) -> bool { + config.install != 0 + || config.context_update != 0 + || config.completion_seed_valid != 0 + || config.qos_update != 0 + || previous.map(identity) != Some(identity(config)) +} + +pub(super) struct Lane<'a> { + items: [&'a AtomicU64; 3], + producer: &'a AtomicU32, + values: [u64; 3], + config: QueueConfig, + config_destination: *mut u64, + config_present: bool, + announce: KickAnnounce, + announce_destination: *mut KickAnnounce, + prefix: usize, + prefix_end: u32, + pub(super) window: Window, +} +impl<'a> Lane<'a> { + pub(super) fn new( + graph: &'a memory::Graph, + stage: usize, + ordinal: u64, + base: u32, + config: QueueConfig, + present: bool, + timestamp: KickTimestamp, + ) -> Result { + let count = 2 + u32::from(present); + let window = Window::new(base, count, ITEM_RING_ENTRIES)?; + let offset = ordinal as usize % memory::RECORDS; + let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; + let announce_offset = memory::ANNOUNCE[stage] + offset * memory::ANNOUNCE_STRIDE; + let config_destination = graph + .config + .pointer(config_offset, size_of::())? + .cast::(); + if config_destination.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + let announce_destination = graph + .queues + .pointer(announce_offset, size_of::())? + .cast::(); + let index = + |delta: u32| memory::ITEMS[stage] + ((base + delta) % ITEM_RING_ENTRIES) as usize * 8; + let items = [ + graph.queues.dword(index(0))?, + graph.queues.dword(index(1))?, + graph.queues.dword(index(2))?, + ]; + let announce_va = graph.queues.gpu_va() + announce_offset as u64; + let descriptor = graph.descriptor_va(stage, ordinal); + let values = if present { + [ + descriptor, + graph.config.gpu_va() + config_offset as u64, + announce_va, + ] + } else { + [descriptor, announce_va, 0] + }; + Ok(Self { + items, + producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, + values, + config, + config_destination, + config_present: present, + announce: KickAnnounce::new( + config.qid.try_into()?, + if stage == 0 { + DataMaster::Tiling + } else { + DataMaster::Fragment + }, + timestamp, + )?, + announce_destination, + prefix: count as usize - 1, + prefix_end: (base + count - 1) % ITEM_RING_ENTRIES, + window, + }) + } + fn items(&self, range: core::ops::Range) { + for index in range { + self.items[index].store(self.values[index], Ordering::Relaxed); + } + } + fn config(&self) { + if !self.config_present { + return; + } + let source = (&self.config as *const QueueConfig).cast::(); + for index in 0..size_of::() / 8 { + // SAFETY: QueueConfig is packed integer storage with no padding; + // every source qword is read unaligned. The destination is a + // checked, aligned record owned by this unpublished ordinal. + unsafe { + self.config_destination + .add(index) + .write(source.add(index).read_unaligned()); + } + } + } + fn announce(&self) { + // SAFETY: The full record is inside the retained queue graph and is + // exclusively owned by this ordinal. Publication follows its copy. + unsafe { + self.announce_destination.write_unaligned(self.announce); + } + } + pub(super) fn publish_prefix(&self) { + self.items(0..self.prefix); + self.config(); + fence(Ordering::SeqCst); + self.producer.store(self.prefix_end, Ordering::Relaxed); + } + pub(super) fn publish_kick(&self) { + self.announce(); + self.items(self.prefix..self.prefix + 1); + fence(Ordering::SeqCst); + self.producer.store(self.window.target(), Ordering::Relaxed); + } + pub(super) fn publish_tiling( + &self, + event_slot: &AtomicU32, + next_slot: u32, + inner: &AtomicU32, + next_inner: u32, + ) { + self.items(0..self.prefix + 1); + self.config(); + self.announce(); + fence(Ordering::SeqCst); + self.producer.store(self.window.target(), Ordering::Relaxed); + event_slot.store(next_slot, Ordering::Relaxed); + inner.store(next_inner, Ordering::Relaxed); + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs new file mode 100644 index 00000000000000..d0a2d566bdc8d3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/retirement.rs @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Paired render retirement evidence and bounded ownership of in-flight work. +//! Queue progress and execution timestamps precede the completion stamps which +//! authorize releasing command memory. A signalled error is not that witness. + +use crate::hw::t8140::queues; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +const DEPTH: usize = queues::RENDER_DEPTH as usize; +const MAX_GROUP_ITEMS: u32 = 4; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Cursors { + pub(crate) done: u32, + pub(crate) read: u32, + pub(crate) write: u32, +} + +impl Cursors { + fn valid(self) -> bool { + self.done <= self.write && self.read <= self.write + } + + fn idle(self) -> bool { + self.done == self.read && self.read == self.write + } +} + +/// Each engine may omit its configuration update independently. Preserve both +/// exact windows; neither target can be reconstructed from a render ordinal. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Window { + base: u32, + count: u32, + capacity: u32, +} + +impl Window { + pub(crate) fn new(base: u32, count: u32, capacity: u32) -> Result { + if base >= capacity || !(2..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { + return Err(EINVAL); + } + Ok(Self { + base, + count, + capacity, + }) + } + + pub(crate) fn target(self) -> u32 { + ((u64::from(self.base) + u64::from(self.count)) % u64::from(self.capacity)) as u32 + } + + fn normalize(self, cursors: Cursors) -> Result { + // Earlier and later groups can coexist in a snapshot. Anchor behind + // this group's base so a backwards observation cannot look like wrap. + let back = MAX_GROUP_ITEMS * (DEPTH as u32 - 1); + let limit = MAX_GROUP_ITEMS * DEPTH as u32 + back; + if limit >= self.capacity { + return Err(EINVAL); + } + let base = (u64::from(self.base) + u64::from(self.capacity) - u64::from(back)) + % u64::from(self.capacity); + let distance = |value: u32| -> Result { + if value >= self.capacity { + return Err(EIO); + } + let distance = + (u64::from(value) + u64::from(self.capacity) - base) % u64::from(self.capacity); + if distance > u64::from(limit) { + return Err(EIO); + } + Ok(distance as u32) + }; + Ok(Cursors { + done: distance(cursors.done)?, + read: distance(cursors.read)?, + write: distance(cursors.write)?, + }) + } + + fn progress(self, previous: Cursors, current: Cursors) -> Result<(bool, bool, bool)> { + let previous = self.normalize(previous)?; + let current = self.normalize(current)?; + let prefix = MAX_GROUP_ITEMS * (DEPTH as u32 - 1) + self.count; + if !previous.valid() + || !current.valid() + || current.done < previous.done + || current.read < previous.read + || current.write < previous.write + || prefix > current.write + { + return Err(EIO); + } + Ok(( + current.read >= prefix, + current.done >= prefix, + current.idle(), + )) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Ticket { + /// Full physical-pair ordinal, retained through owner switches and slot wrap. + pub(crate) ordinal: u64, + pub(crate) windows: [Window; 2], + pub(crate) previous: [Cursors; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Snapshot { + pub(crate) cursors: [Cursors; 2], + pub(crate) job_list_empty: bool, + /// The tiler writes the start and fragment processing writes the end. + pub(crate) pass: [u64; 2], +} + +impl Snapshot { + fn executed(self) -> bool { + self.pass[0] != 0 && self.pass[1] != 0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Observation { + pub(crate) accepted: bool, + pub(crate) retired: bool, + pub(crate) complete: bool, + pub(crate) idle: bool, +} + +pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result { + let ta = ticket.windows[0].progress(ticket.previous[0], snapshot.cursors[0])?; + let fragment = ticket.windows[1].progress(ticket.previous[1], snapshot.cursors[1])?; + let retired = ta.1 && fragment.1; + Ok(Observation { + accepted: ta.0 && fragment.0, + retired, + complete: retired && snapshot.executed(), + idle: ta.2 && fragment.2 && snapshot.job_list_empty, + }) +} + +/// Outstanding publications span less than half the 24-bit stamp sequence. +/// A zero stamp is valid when that sequence wraps. +pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { + let delta = stamp.wrapping_sub((published as u32).wrapping_shl(8)); + delta & 0xff == 0 && delta < 0x8000_0000 +} + +fn visible(ticket: &Ticket, stamps: [u32; 2]) -> Result { + let published = ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; + Ok(stamps + .into_iter() + .all(|stamp| stamp_covers(stamp, published))) +} + +/// Reads the graph owned by one physical pair. The owner validates the full +/// ordinal before reading reused descriptor/timestamp slots. Word accessors +/// perform aligned atomic loads; the observer supplies the acquire barriers. +pub(crate) trait Memory { + fn snapshot(&mut self, ordinal: u64) -> Result; + fn stamps(&mut self) -> Result<[u32; 2]>; + fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]>; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Poll { + Retry, + Pending, + Complete([u64; 4]), +} + +/// Observe the ordered front. Individually sampled cursor words can straddle +/// a firmware update, so an invalid snapshot receives one immediate reread. +/// Shared work-state membership can include other engines and cannot gate a +/// successful individual member's completion. +pub(crate) fn poll(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { + fence(Ordering::Acquire); + let mut snapshot = memory.snapshot(ticket.ordinal)?; + let observation = match observe(ticket, snapshot) { + Ok(observation) => observation, + Err(_) => { + fence(Ordering::Acquire); + snapshot = memory.snapshot(ticket.ordinal)?; + match observe(ticket, snapshot) { + Ok(observation) => observation, + Err(_) => return Ok(Poll::Retry), + } + } + }; + complete_poll(ticket, memory, snapshot, observation) +} + +/// Finish an observation sampled while its exact pair was exclusively borrowed. +/// The caller restores that pair and validates its epoch and front ticket first. +/// Completion stamps and the final timestamp refresh remain commit-time reads. +pub(crate) fn poll_observed( + ticket: &mut Ticket, + memory: &mut impl Memory, + snapshot: Snapshot, +) -> Result { + let observation = observe(ticket, snapshot)?; + complete_poll(ticket, memory, snapshot, observation) +} + +fn complete_poll( + ticket: &mut Ticket, + memory: &mut impl Memory, + snapshot: Snapshot, + observation: Observation, +) -> Result { + if observation.complete { + if !visible(ticket, memory.stamps()?)? { + return Ok(Poll::Pending); + } + // Refresh this generation's pass after observing the post-maintenance + // stamps. The pre-stamp snapshot alone cannot authorize export. + fence(Ordering::Acquire); + let [start, end] = memory.pass(ticket.ordinal)?; + return Ok(Poll::Complete([start, start, start, end])); + } + ticket.previous = snapshot.cursors; + Ok(Poll::Pending) +} + +/// Strict timeout settlement accepts only a nonexecuted, retired and unlinked +/// group. Drain reclamation may also settle executed groups, but retains both +/// ownership proofs and the exact post-maintenance stamps in either case. +pub(crate) fn abandoned( + ticket: &Ticket, + snapshot: Snapshot, + stamps: [u32; 2], + recovery_state: u32, + accept_executed: bool, +) -> Result { + let completion = observe(ticket, snapshot)?; + if !completion.retired || !snapshot.job_list_empty || (snapshot.executed() && !accept_executed) + { + return Ok(false); + } + Ok(visible(ticket, stamps)? && recovery_state == 0) +} + +/// Pair-wide proof, checked before and after servicing free-list events and +/// again after retiring all trackers. A proof from before event service must +/// not authorize releasing guards after a new recovery began. +pub(crate) fn drained( + published: u64, + stamps: [u32; 2], + free_list: [u64; 2], + recovery_state: u32, +) -> bool { + published == 0 + || (stamps + .into_iter() + .all(|stamp| stamp_covers(stamp, published)) + && free_list[0] == free_list[1] + && recovery_state == 0) +} + +/// Fixed storage keeps admission, rollback and retirement allocation-free. +/// Each element retains its packet and resource leases independently of fences. +pub(crate) struct Tracker { + entries: [Option; DEPTH], + head: usize, + len: usize, +} + +impl Tracker { + pub(crate) const fn new() -> Self { + Self { + entries: [const { None }; DEPTH], + head: 0, + len: 0, + } + } + + pub(crate) fn len(&self) -> usize { + self.len + } + pub(crate) fn room(&self) -> bool { + self.len < DEPTH + } + pub(crate) fn front(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[self.head].as_ref() + } + } + pub(crate) fn front_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[self.head].as_mut() + } + } + pub(crate) fn back(&self) -> Option<&T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % DEPTH].as_ref() + } + } + pub(crate) fn back_mut(&mut self) -> Option<&mut T> { + if self.len == 0 { + None + } else { + self.entries[(self.head + self.len - 1) % DEPTH].as_mut() + } + } + pub(crate) fn push(&mut self, value: T) -> core::result::Result<(), T> { + if !self.room() { + return Err(value); + } + self.entries[(self.head + self.len) % DEPTH] = Some(value); + self.len += 1; + Ok(()) + } + pub(crate) fn pop(&mut self) -> Option { + if self.len == 0 { + return None; + } + let value = self.entries[self.head].take(); + self.head = (self.head + 1) % DEPTH; + self.len -= 1; + value + } + + pub(crate) fn iter(&self) -> impl Iterator { + Iterator::chain( + self.entries[self.head..].iter(), + self.entries[..self.head].iter(), + ) + .take(self.len) + .filter_map(Option::as_ref) + } + pub(crate) fn iter_mut(&mut self) -> impl Iterator { + let (before, after) = self.entries.split_at_mut(self.head); + Iterator::chain(after.iter_mut(), before.iter_mut()) + .take(self.len) + .filter_map(Option::as_mut) + } +} diff --git a/drivers/gpu/drm/asahi/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs new file mode 100644 index 00000000000000..4db471f254ec61 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/queue/render/warm.rs @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Fresh render graph cursor initialization at a retained physical ordinal. + +use super::{memory, Pair}; +use crate::g17::fw::queue::PointerBlock; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +impl Pair { + /// A new graph at a retained ordinal starts its variable item rings at zero. + /// The old graph has retired; firmware has not received this graph's queues. + pub(crate) fn rebase_fresh_graph(&mut self) -> Result { + if self.ordinal == 0 { + return Ok(()); + } + fence(Ordering::Acquire); + if self + .memory + .graph + .cursors()? + .into_iter() + .any(|cursor| cursor.done != 0 || cursor.read != 0 || cursor.write != 0) + { + return Err(EBUSY); + } + let mut blocks = [core::ptr::null::(); 2]; + for (block, offset) in blocks.iter_mut().zip(memory::POINTERS) { + let pointer = self + .memory + .graph + .queues + .pointer(offset, size_of::())?; + if pointer.align_offset(align_of::()) != 0 { + return Err(EINVAL); + } + *block = pointer.cast(); + } + for block in blocks { + // SAFETY: Both complete pointer blocks were checked above. This fresh + // graph is unpublished; only the live atomic cursors are accessed. + unsafe { &*block }.rebase(0); + } + fence(Ordering::SeqCst); + self.cursors = [0; 2]; + Ok(()) + } + +} diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs new file mode 100644 index 00000000000000..7e3db2f6dae783 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware restart handshake and per-VM failure classification. +//! +//! The device mutex serializes this state with publication. Preparations are closed and joined +//! without that mutex before `service`; no queue callback may signal a fence here. Queue +//! owners retain failed work until its retirement witness, a complete pair-drain witness, or +//! processor stop. A failed fence alone never releases mappings. + +mod attribution; +mod slot; +pub(crate) use attribution::Sources; +use attribution::Attribution; + +pub(super) fn slot_key(slot: usize) -> Option { + slot::key(slot).ok() +} + +use super::status::VmStatus; +use kernel::{prelude::*, sync::Arc}; + +const QIDS: u8 = 128; +const GUILTY: u8 = 1 << 1; +const CANCELLED: u8 = 1 << 2; +/// Retained valid entries, in firmware table order. +pub(super) const MAX_ENTRIES: usize = 64; +const MAX_GUILTY_VMS: usize = 8; +const RECOVERY_POLLS: usize = 500; + +/// One valid entry from the firmware's 256-entry recovery table. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(super) struct Entry { + pub(super) qid: u8, + pub(super) flags: u8, + pub(super) stamp: u32, +} + +impl Entry { + pub(super) const fn decode(flags: u32, stamp: u32) -> Option { + if flags & 1 == 0 { + return None; + } + Some(Self { + qid: ((flags >> 4) & 0x7f) as u8, + flags: (flags & 0xf) as u8, + stamp: stamp >> 8, + }) + } +} + +/// How an error completion affects its VM's sticky status and other jobs. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Class { + /// Only this work's fence fails; VM status and unrelated work remain usable. + Spared, + /// This VM owns guilty work. + Guilty, +} + +/// Queue operations under the device mutex. Completion signalling belongs to the caller after +/// that mutex is dropped. `oldest_spared` returns None if the QID has no in-flight owner. +pub(super) trait Queues { + fn render_scheduler_active(&self) -> bool; + fn render_pass_started(&self, qid: u8, stamp: u64) -> Option; + /// Copy every retained queue's live producer to its recovery target, then + /// issue a full barrier. A failed graph must not skip subsequent queues. + fn publish_recovery_targets(&self) -> Option; + fn oldest_spared(&self, qid: u8) -> Option; + fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); + /// Classify every in-flight completion; return QIDs of render slots with spared work. + fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; + fn settle_blamed_render(&mut self, qid: u8, error: Error); + fn settle_cancelled_renders(&mut self, qids: u128); + fn rescan_compute(&mut self) -> Result; + fn release_retired_compute(&mut self); +} + +/// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions +/// check the expected value, issue a full barrier, store the next value, then another barrier. +pub(super) trait Memory { + fn recovery_state(&self) -> Result; + fn host_recovery(&self) -> Result; + fn fault_sources(&self) -> Result; + fn transition(&self, expected: u32, next: u32) -> Result; + fn clear_timestamps(&self) -> Result; + fn entries(&self, out: &mut [Entry; MAX_ENTRIES]) -> Result; + /// One millisecond wait; this is polled because the firmware sends no state-3 notification. + fn wait_tick(&self); +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +struct Request { + generation: u64, + blamed: Option, +} + +/// Generation accepted by the host and the coalesced request from either coprocessor. +pub(super) struct State { + accepted: u64, + pending: Option, +} + +/// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. +pub(super) struct ServiceResult { + pub(super) scan_error: Option, + pub(super) target_error: Option, +} + +impl State { + pub(super) const fn new() -> Self { + Self { + accepted: 0, + pending: None, + } + } + pub(super) fn generation(&self) -> u64 { + self.accepted + } + pub(super) fn pending(&self) -> bool { + self.pending.is_some() + } + + /// Returns false for a consumed duplicate from an older generation, true when the event + /// drain must stop until the current request is serviced. The first blamed QID wins. + pub(super) fn request(&mut self, generation: u64, stamp: i32) -> Result { + if generation < self.accepted { + return Ok(false); + } + if generation != self.accepted { + return Err(EIO); + } + if stamp != -1 && !(0..i32::from(QIDS)).contains(&stamp) { + return Err(EIO); + } + let blamed = if (0..i32::from(QIDS)).contains(&stamp) { + Some(stamp as u8) + } else { + None + }; + if let Some(pending) = self.pending.as_mut() { + if pending.generation != generation { + return Err(EBUSY); + } + pending.blamed = pending.blamed.or(blamed); + } else { + self.pending = Some(Request { generation, blamed }); + } + Ok(true) + } +} + +/// The firmware owner implements memory and queue access without splitting its +/// mutable borrow or duplicating the authoritative restart generation. +pub(super) trait Host: Memory + Queues { + fn recovery(&self) -> &State; + fn recovery_mut(&mut self) -> &mut State; +} + +/// Performs one pending restart. The caller has closed preparation admission and joined all +/// builders off-lock. Work remains mapped throughout every callback, including on error. +pub(super) fn service( + host: &mut H, + crashed: impl Fn() -> bool, +) -> Result { + let request = host.recovery().pending.ok_or(EINVAL)?; + if host.recovery_state()? != 1 + || request.generation != host.recovery().accepted + || host.host_recovery()? != 0 + { + return Err(EIO); + } + host.recovery_mut().accepted = host.recovery().accepted.wrapping_add(1); + if request.generation == 0 && !host.render_scheduler_active() { + host.clear_timestamps()?; + } + let mut sources = host.fault_sources()?; + let target_error = host.publish_recovery_targets(); + host.transition(1, 2)?; + let mut recovered = false; + for _ in 0..RECOVERY_POLLS { + if host.recovery_state()? == 3 { + recovered = true; + break; + } + if crashed() { + return Err(EIO); + } + host.wait_tick(); + } + if !recovered { + return Err(ETIMEDOUT); + } + let mut entries = [Entry::default(); MAX_ENTRIES]; + // A failed table read leaves only the explicit blamed QID as evidence of guilt. + let count = host.entries(&mut entries).unwrap_or(0).min(MAX_ENTRIES); + host.transition(3, 0)?; + if host.recovery_state()? != 0 { + return Err(EIO); + } + host.recovery_mut().pending = None; + + let cancelled = classify(host, &entries[..count], request.blamed, &mut sources); + if let Some(qid) = request.blamed { + host.settle_blamed_render(qid, EIO); + } + if cancelled != 0 { + host.settle_cancelled_renders(cancelled); + } + let scan_error = host.rescan_compute().err(); + host.release_retired_compute(); + Ok(ServiceResult { scan_error, target_error }) +} + +/// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or +/// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work +/// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. +fn classify(queues: &mut Q, entries: &[Entry], blamed: Option, sources: &mut Sources) -> u128 { + let mut guilty_qids = blamed.map_or(0, |qid| 1u128 << qid); + let mut cancelled = 0u128; + for entry in entries { + if entry.flags & GUILTY != 0 { + guilty_qids |= 1u128 << entry.qid; + } + if entry.flags & CANCELLED != 0 { + cancelled |= 1u128 << entry.qid; + } + } + let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); + match attribution { + Attribution::Firmware => {} + Attribution::Queues(qids) => guilty_qids = qids, + Attribution::NoCulprit => guilty_qids = 0, + } + let no_culprit = attribution == Attribution::NoCulprit; + if guilty_qids == 0 && !no_culprit { + return 0; + } + let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); + let mut count = 0; + let mut collateral = 0u128; + let mut unresolved = false; + for qid in 0..QIDS { + if guilty_qids & (1u128 << qid) == 0 { + continue; + } + match queues.oldest_spared(qid) { + None => unresolved = true, + Some(true) => collateral |= 1u128 << qid, + Some(false) => { + let mut found = false; + queues.visit_owners(qid, &mut |status| { + found = true; + if guilty[..count] + .iter() + .flatten() + .any(|known| Arc::ptr_eq(known, status)) + { + return; + } + if let Some(slot) = guilty.get_mut(count) { + *slot = Some(status.clone()); + count += 1; + } else { + unresolved = true; + } + }); + unresolved |= !found; + } + } + } + if unresolved || (count == 0 && collateral == 0 && !no_culprit) { + return 0; + } + let spared = queues.classify_work(&|status| { + guilty[..count] + .iter() + .find_map(|known| { + known + .as_ref() + .filter(|known| Arc::ptr_eq(known, status)) + .map(|_| Class::Guilty) + }) + .unwrap_or(Class::Spared) + }); + cancelled & !(guilty_qids & !collateral) & spared +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs new file mode 100644 index 00000000000000..b0c42d4e3f7c15 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Resolve ambiguous recovery blame using slot identity and render execution evidence. + +pub(super) const SLOTS: usize = 16; +const STAMP_MASK: u64 = (1 << 40) - 1; +const PAGE_FAULT: u32 = 3; + +#[derive(Clone, Copy, Default)] +pub(super) struct Source { + slot: u8, + qid: Option, + stamp: u64, + monitor: u32, + started: Option, +} + +pub(crate) struct Sources { + pub(crate) reason: u32, + mask: u32, + records: [Source; SLOTS], + count: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Attribution { + Firmware, + Queues(u128), + NoCulprit, +} + +impl Sources { + /// Firmware is halted for every selected key and progress-record read. + /// Missing keys stay unknown; they cannot establish execution by an owner. + pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> (u64, u32)) -> Self { + let mut sources = Self { + reason: 0, + mask, + records: [Source::default(); SLOTS], + count: 0, + }; + for slot in 0..SLOTS { + if mask & (1 << slot) == 0 { + continue; + } + let (key, monitor) = read(slot); + let qid = (key >> 40) & 0xff; + sources.records[sources.count] = Source { + slot: slot as u8, + qid: (key != 0 && qid < 128).then_some(qid as u8), + stamp: key & STAMP_MASK, + monitor, + started: None, + }; + sources.count += 1; + } + sources + } + + pub(super) fn attribute( + &mut self, + blamed: Option, + mut started: impl FnMut(u8, u64) -> Option, + ) -> Attribution { + for source in &mut self.records[..self.count] { + if let Some(qid) = source.qid { + source.started = started(qid, source.stamp); + } + } + if self.reason == PAGE_FAULT || self.mask.count_ones() < 2 { + return Attribution::Firmware; + } + let Some(blamed) = blamed else { + return Attribution::Firmware; + }; + let Some(source) = self.records[..self.count] + .iter() + .find(|source| source.qid == Some(blamed)) + else { + return Attribution::Firmware; + }; + if source.started != Some(false) || source.monitor != 0 { + return Attribution::Firmware; + } + let slot = source.slot; + let mut guilty = 0; + for source in &self.records[..self.count] { + if source.slot != slot && source.started == Some(true) { + if let Some(qid) = source.qid { + guilty |= 1u128 << qid; + } + } + } + if guilty == 0 { + Attribution::NoCulprit + } else { + Attribution::Queues(guilty) + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/recovery/slot.rs b/drivers/gpu/drm/asahi/g17/recovery/slot.rs new file mode 100644 index 00000000000000..307534708e922f --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/recovery/slot.rs @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded read-only slot identity sampling while firmware is halted. + +use crate::{hw::t8140, mmu}; +use core::{ + ffi::c_void, + ptr::NonNull, + sync::atomic::{AtomicU64, Ordering}, +}; +use kernel::{bindings, prelude::*}; + +struct Mapping(NonNull); + +impl Drop for Mapping { + fn drop(&mut self) { + // SAFETY: This is the successful memremap result owned exclusively by this guard. + unsafe { bindings::memunmap(self.0.as_ptr()) }; + } +} + +pub(super) fn key(slot: usize) -> Result { + if slot >= super::attribution::SLOTS { + return Err(EINVAL); + } + let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; + let offset = physical as usize & (mmu::UAT_PGSZ - 1); + // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here + // for mapping. Only the selected aligned eight-byte word is read below. + let raw = unsafe { + bindings::memremap( + physical - offset as u64, + mmu::UAT_PGSZ, + bindings::MEMREMAP_WB as _, + ) + }; + let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1. DminLine specifies the cache maintenance granule. + unsafe { + core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags)); + } + let line = 4usize << ((ctr >> 16) & 0xf); + if line > mmu::UAT_PGSZ { + return Err(EINVAL); + } + // SAFETY: The range and offset are eight-byte aligned and lie within the mapped page. + let pointer = unsafe { + mapping + .0 + .as_ptr() + .cast::() + .add(offset) + .cast::() + }; + let mut address = pointer as usize & !(line - 1); + let end = (pointer as usize) + .checked_add(size_of::()) + .ok_or(EOVERFLOW)?; + // SAFETY: The mapping stays live and is never written by the host. The + // invalidated cache line lies within its page; barriers complete the + // invalidation before the aligned load of the selected firmware-owned word. + unsafe { + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + while address < end { + core::arch::asm!("dc ivac, {address}", address = in(reg) address, + options(nostack, preserves_flags)); + address = address.checked_add(line).ok_or(EOVERFLOW)?; + } + core::arch::asm!("dsb osh", options(nostack, preserves_flags)); + Ok((*pointer).load(Ordering::Acquire)) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs new file mode 100644 index 00000000000000..e908ca3e08e23e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -0,0 +1,869 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device ownership of physical queues and off-lock construction reservations. +//! +//! The device mutex protects this registry. Logical queues may disappear while +//! their published graphs remain here; only exact retirement permits rebinding, +//! and only processor stop permits dropping installed firmware addresses. +//! A preparation lease keeps Shared::state populated: shutdown and loss join +//! every lease before taking the session, including builders asleep on changed. + +mod backend; +mod compute_control; +mod compute_replay; +mod compute_exit; +mod events; +mod render; +pub(super) mod teardown; + +use super::{ + buffer::{BufferIds, MetricsIds}, + context::{Context, QosIds}, + fw::queue::DataMaster, + kick, qos, + queue::{self, admission, compute}, +}; +use kernel::{prelude::*, sync::Arc}; + +const QIDS: usize = 128; +const FIRST_COMPUTE_QID: u8 = 4; +const RECOVERY_SETTLE_POLLS: usize = 200; + +/// A reservation remains identifiable while its backing is built without the +/// device mutex. Failed construction returns the slot, never its host identity. +#[derive(Clone, Copy)] +pub(super) struct ComputeReservation { + slot: usize, + pub(super) id: kick::Id, + owner: u64, +} + +struct ComputeEntry { + reservation: ComputeReservation, + queue: Option>, +} + +pub(super) struct Registry { + pub(super) qids: kick::QueueIds, + pub(super) buffers: BufferIds, + pub(super) accounting: qos::Accounting, + pub(super) metrics: Arc, + pub(super) qos_ids: Arc, + pub(super) admission: Arc, + render: render::RenderState, + identities: queue::Identities, + compute: KVec>, + pub(super) compute_building: bool, + compute_scheduler_building: bool, + pub(super) observations: compute::retirement::Batch, + worker_deferred: Option, + teardown: super::teardown::Pending, + exit_acks: compute::exit::Acks, +} + +impl Registry { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; + for _ in 0..QIDS { + compute.push(None, GFP_KERNEL)?; + } + let mut qids = kick::QueueIds::new(); + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(Self { + qids, + buffers: BufferIds::new(), + accounting: qos::Accounting::new(), + metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, + qos_ids: QosIds::new()?, + admission: admission::Pool::new()?, + render: render::RenderState::new(clusters, descriptor_flags)?, + identities: queue::Identities::new(), + compute, + compute_building: false, + compute_scheduler_building: false, + observations: compute::retirement::Batch::new()?, + worker_deferred: Some(DeferredBatch::worker()?), + teardown: super::teardown::Pending::new()?, + exit_acks: compute::exit::Acks::new(), + }) + } + + pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { + self.compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.owner() == Some(owner)) + } + + pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { + self.compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + } + + /// An installed idle queue keeps its original QID and backing. Mapping a + /// replacement context must succeed before its old binding is displaced. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + if let Some(queue) = self.compute(owner) { + return queue.matches(owner, queue.qid(), context); + } + let Some(queue) = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.reusable()) + else { + return Ok(false); + }; + queue.bind_owner(owner, context.clone())?; + Ok(true) + } + + pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { + if owner == 0 + || self.compute.iter().flatten().any(|entry| { + entry + .queue + .as_ref() + .map_or(entry.reservation.owner == owner, |queue| { + queue.owner() == Some(owner) + }) + }) + { + return Err(EINVAL); + } + let slot = self + .compute + .iter() + .position(Option::is_none) + .ok_or(ENOSPC)?; + let identity = self.identities.reserve(1)?.start; + let id = self + .qids + .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; + let reservation = ComputeReservation { slot, id, owner }; + self.compute[slot] = Some(ComputeEntry { + reservation, + queue: None, + }); + Ok(reservation) + } + + fn reserved(&mut self, reservation: ComputeReservation) -> Result<&mut ComputeEntry> { + let entry = self + .compute + .get_mut(reservation.slot) + .and_then(Option::as_mut) + .ok_or(EIO)?; + if entry.reservation.id != reservation.id + || entry.reservation.owner != reservation.owner + || entry.queue.is_some() + { + return Err(EIO); + } + Ok(entry) + } + + /// The caller checked its preparation epoch and retained the same device + /// session before placing this graph where publication can find it. + pub(super) fn install_compute( + &mut self, + reservation: ComputeReservation, + queue: &mut Option>, + ) -> Result { + let built = queue.as_ref().ok_or(EINVAL)?; + if built.qid() != reservation.id.qid() || built.owner() != Some(reservation.owner) { + return Err(EIO); + } + let entry = self.reserved(reservation)?; + entry.queue = queue.take(); + Ok(()) + } + + pub(super) fn cancel_compute(&mut self, reservation: ComputeReservation) -> Result { + self.reserved(reservation)?; + self.qids.cancel_unpublished(reservation.id)?; + self.compute[reservation.slot] = None; + Ok(()) + } + + pub(super) fn release_compute(&mut self, owner: u64) -> Result { + let Some(slot) = self.compute.iter().position(|entry| { + entry + .as_ref() + .and_then(|entry| entry.queue.as_ref()) + .is_some_and(|queue| queue.owner() == Some(owner)) + }) else { + return Ok(()); + }; + let entry = self.compute[slot].as_mut().ok_or(EIO)?; + if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { + self.qids.cancel_unpublished(entry.reservation.id)?; + self.compute[slot] = None; + } + Ok(()) + } +} + +/// An immutable logical owner shared by its two scheduler entities. Installed +/// physical graphs retain the context independently of this frontend lifetime. +pub(super) struct Backend { + shared: Arc, + pub(super) context: Arc, + owner: u64, + backlog: core::sync::atomic::AtomicU32, +} + +impl Backend { + pub(super) fn new( + shared: &Arc, + vm: &crate::mmu::Vm, + owner: u64, + ) -> Result<(Arc, admission::QueueSlot)> { + let lease = shared.preparations.enter(true)?; + let (uat, pool, qos, execution, slot) = { + let mut state = shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + let slot = firmware.queues.admission.reserve_queue()?; + let execution = firmware.uat.new_execution_context(vm)?; + ( + firmware.uat.clone(), + firmware.pool.clone(), + firmware.queues.qos_ids.clone(), + execution, + slot, + ) + }; + let alloc = super::object::Allocator { + dev: &shared.dev, + uat: &uat, + }; + let owner_pid = + u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; + let context = Arc::new( + Context::new(&alloc, &pool, &qos, execution, owner_pid)?, + GFP_KERNEL, + )?; + let state = shared.state.lock(); + if !lease.is_current() || state.is_none() { + return Err(EAGAIN); + } + drop(state); + Ok(( + Arc::new( + Self { + shared: shared.clone(), + context, + owner, + backlog: core::sync::atomic::AtomicU32::new(0), + }, + GFP_KERNEL, + )?, + slot, + )) + } + + /// The shared aperture has exactly one builder. Other queues sleep without + /// holding the device mutex; recovery joins its lease before closing roots. + fn shared_compute(&self) -> Result { + loop { + let lease = self.shared.preparations.enter(false)?; + let uat = { + let mut state = self.shared.state.lock(); + loop { + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.init.has_compute_shared() { + return firmware.init.cache_compute_vm(self.context.vm()); + } + if !firmware.queues.compute_building { + firmware.queues.compute_building = true; + break firmware.uat.clone(); + } + self.shared.changed.wait(&mut state); + } + }; + let alloc = super::object::Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let mut built = Some(super::initdata::ComputeShared::new(&alloc)); + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else { + built + .take() + .ok_or(EIO)? + .and_then(|built| firmware.init.install_compute_shared(built)) + .and_then(|()| firmware.init.cache_compute_vm(self.context.vm())) + } + }; + drop(built); + { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .compute_building = false; + } + self.shared.changed.notify_all(); + drop(lease); + if result != Err(EAGAIN) { + return result; + } + } + } + + /// Uses the queue builder's existing lease. Entering another closed epoch + /// here would make recovery wait for the very builder blocked on reopening. + fn compute_scheduler( + &self, + alloc: &super::object::Allocator<'_>, + lease: &super::preparation::Lease, + ) -> Result { + { + let mut state = self.shared.state.lock(); + loop { + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.init.has_compute_scheduler() { + return Ok(()); + } + if !firmware.queues.compute_scheduler_building { + firmware.queues.compute_scheduler_building = true; + break; + } + self.shared.changed.wait(&mut state); + } + } + let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else { + built + .take() + .ok_or(EIO)? + .and_then(|built| firmware.init.install_compute_scheduler(built)) + } + }; + drop(built); + { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .compute_scheduler_building = false; + } + self.shared.changed.notify_all(); + result + } + + fn build_compute(&self) -> Result { + loop { + match self.shared_compute() { + Err(error) if error == EAGAIN => continue, + result => { + result?; + break; + } + } + } + loop { + let lease = self.shared.preparations.enter(false)?; + let (uat, reservation) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + if firmware.queues.rebind_compute(self.owner, &self.context)? { + return Ok(()); + } + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), reservation) + }; + let alloc = super::object::Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let built = (|| { + let pool = compute::PoolBacking::new(&alloc, reservation.id)?; + { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + firmware.queues.reserved(reservation)?; + let (slot, page_list) = pool.pool_descriptor(); + firmware.init.initialize_compute_pool(slot, page_list)?; + } + let backing = compute::Backing::new(&alloc, pool)?; + // The global scheduler is created after pool/kick backing and + // retained even if this queue's later descriptor graph fails. + self.compute_scheduler(&alloc, &lease)?; + let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; + Ok(KBox::new(queue, GFP_KERNEL)?) + })(); + let (mut built, build_error) = match built { + Ok(queue) => (Some(queue), None), + Err(error) => (None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + Err(EAGAIN) + } else if let Some(error) = build_error { + Err(error) + } else { + firmware.queues.install_compute(reservation, &mut built) + } + }; + // The old aliases must disappear before a waiting constructor can + // reuse the QID and its queue-local free-list identity. Keep the + // preparation lease until both mappings and reservation are gone. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_mut() + .ok_or(ENODEV)? + .queues + .cancel_compute(reservation)?; + } + drop(lease); + if result != Err(EAGAIN) { + return result; + } + } + } +} + +/// One device transaction can fail each retained kick once and then retire it. +/// The physical namespace bounds the batch independently of open logical queues. +/// Collection failure leaves the engine's Active owner intact for a stopped retry. +pub(super) struct DeferredBatch { + values: KVVec, +} +impl DeferredBatch { + const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize + > crate::hw::t8140::queues::COMPUTE_DEPTH + { + crate::hw::t8140::queues::RENDER_DEPTH as usize + } else { + crate::hw::t8140::queues::COMPUTE_DEPTH + }; + + /// Reused by the sole event work item; no allocation on a completion pass. + fn worker() -> Result { + Self::with_capacity(2 * QIDS * Self::DEPTH) + } + /// One publication can quarantine only its selected physical queue/pair. + /// VM-wide settlement is a separate transaction before publishing its error. + fn publication() -> Result { + Self::with_capacity(2 * Self::DEPTH) + } + fn with_capacity(capacity: usize) -> Result { + Ok(Self { + values: KVVec::with_capacity(capacity, GFP_KERNEL)?, + }) + } + pub(super) fn push(&mut self, value: super::completion::Deferred) -> Result { + use super::completion::Deferred; + // Publishers may acquire the device mutex as soon as this pass drops + // it. Expose terminal VM status now, before off-lock fence callbacks. + // Recovery-spared and cancelled commands never poison their VM. + if let Deferred::FailedOwned(completion, error) = &value { + if *error != ECANCELED && !completion.spared() { + completion.status().record(*error); + } + } + self.values.push_within_capacity(value).map_err(|_| EIO) + } + /// Called only after releasing the device mutex. + pub(super) fn finish(&mut self) { + for value in self.values.drain_all() { + value.finish(); + } + } +} + +struct ComputeHost<'a> { + init: &'a super::initdata::InitData, + primary: &'a mut super::Coprocessor, + effort: &'a mut super::power::Effort, + qids: &'a mut kick::QueueIds, + accounting: &'a mut qos::Accounting, + generation: u64, +} +impl compute::Host for ComputeHost<'_> { + fn epoch(&self) -> Result<(u64, u32)> { + Ok(( + self.generation, + super::recovery::Memory::recovery_state(self.init)?, + )) + } + fn prepare_compute_shared(&mut self) -> Result { + self.init.activate_compute() + } + fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result { + self.accounting.publish( + &qos::View::new(self.init.qos()?)?, + owner, + scheduler, + qos::clock, + ) + } + fn qos_cancel(&mut self, publication: qos::Publication) -> Result { + self.accounting + .cancel(&qos::View::new(self.init.qos()?)?, publication) + } + fn publish_qid(&mut self, id: kick::Id) -> Result { + self.qids.publish(id) + } + fn publish_outer(&mut self, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(slot).map(|_| ()) + } + fn notify(&mut self, message: u64) -> Result { + self.primary.notify(message) + } + fn note_submission(&mut self) -> Result { + self.effort.submit(); + super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) + } +} + +impl super::Firmware { + fn publish_compute( + &mut self, + owner: u64, + packet: Arc, + dependencies: &[super::fw::kick::KickDependency], + deferred: &mut DeferredBatch, + ) -> Result { + if self.recovery.pending() { + return Err(EAGAIN); + } + let queues = &mut self.queues; + let queue = queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + .ok_or(EIO)?; + let mut host = ComputeHost { + init: &self.init, + primary: &mut self.primary, + effort: &mut self.effort, + qids: &mut queues.qids, + accounting: &mut queues.accounting, + generation: self.recovery.generation(), + }; + queue.publish(packet, dependencies, &mut host, &mut |event| { + deferred.push(event) + }) + } +} + +impl Registry { + fn release_compute_witness( + &mut self, + init: &super::initdata::InitData, + owner: u64, + deferred: &mut DeferredBatch, + ) -> Result { + let Some(queue) = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.owner() == Some(owner)) + else { + return Ok(false); + }; + if !queue.spared_pending() { + return Ok(false); + } + let view = qos::View::new(init.qos()?)?; + queue.release_witnessed( + super::recovery::Memory::recovery_state(init)?, + |owner| self.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + Ok(queue.spared_pending()) + } + + fn fail_compute_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for queue in self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if queue.in_flight() && Arc::ptr_eq(queue.context().status(), status) { + queue.quarantine(error, &mut |event| deferred.push(event))?; + } + } + Ok(()) + } +} + +impl super::Firmware { + /// Latch one shared snapshot for all selected queues before settling any VM. + /// A later queue's exact success in this batch wins over a peer's failure. + fn scan_compute(&mut self, masks: Option<[u64; 2]>, deferred: &mut DeferredBatch) -> Result { + let state = super::recovery::Memory::recovery_state(&self.init)?; + let generation = self.recovery.generation(); + let registry = &mut self.queues; + registry.observations.clear(); + let mut requested = false; + for queue in registry + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + requested |= queue.request_observations(&mut registry.observations, masks)?; + } + if requested { + self.init + .scan_compute_completions(|record| registry.observations.fold(record))?; + } + let view = qos::View::new(self.init.qos()?)?; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.latch_observations(&mut registry.observations, masks)?; + queue.retire( + masks, + generation, + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + // All compute observations in this batch are latched before a VM + // failure can settle its render peers. Exact successes win first. + for index in 0..self.queues.compute.len() { + let terminal = self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .and_then(compute::Queue::take_terminal); + if let Some((status, error)) = terminal { + status.record(error); + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + } + let view = qos::View::new(self.init.qos()?)?; + let registry = &mut self.queues; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + Ok(()) + } +} + +impl Backend { + fn ensure_compute(&self) -> Result { + use kernel::time::{delay::fsleep, Delta}; + const SPARED_WAIT_MS: usize = 200; + let mut deferred = DeferredBatch::publication()?; + for _ in 0..SPARED_WAIT_MS { + let pending = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + firmware.replay_compute(&mut deferred)?; + firmware + .queues + .release_compute_witness(&firmware.init, self.owner, &mut deferred) + })(); + deferred.finish(); + let pending = pending?; + if !pending { + break; + } + // Other owners continue submitting while this owner gives its + // queue-local drain witness time to arrive. + fsleep(Delta::from_millis(1)); + } + self.build_compute() + } + + /// Called after taking this packet's publication claim. All waits and work + /// state allocation precede the device mutex; only the checked epoch commits. + fn compute_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + packet.check_dependencies()?; + let mut recovery_polls = 0; + loop { + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let lease = self.shared.preparations.enter(false)?; + packet.completion.work_state().acquire()?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + drop(state); + continue; + } + if packet.cancel_requested() { + return Err(ECANCELED); + } + if self + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Err(EIO); + } + let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; + if firmware.recovery.pending() || recovery_state != 0 { + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } + firmware.replay_compute(deferred)?; + let dependencies = firmware.queues.render_prefix(self.owner, packet)?; + let result = firmware.publish_compute( + self.owner, + packet.clone(), + dependencies.as_slice(), + deferred, + ); + if matches!(result, Err(EBUSY) | Err(EAGAIN)) + && !packet.is_published() + && (firmware.recovery.pending() + || super::recovery::Memory::recovery_state(&firmware.init)? != 0) + { + if recovery_polls == RECOVERY_SETTLE_POLLS { + return Err(EBUSY); + } + recovery_polls += 1; + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } + return result; + } + } +} + +impl super::Firmware { + /// Both processors are stopped and their owner detached before this call. + /// Error fences release their command mappings without the device mutex + /// or an allocation-dependent collector. + fn finish_stopped_queues(&mut self) { + self.stop_render(); + for queue in self + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + let result = queue.stopped(&mut |event| { + event.finish(); + Ok(()) + }); + if let Err(error) = result { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Could not settle stopped compute queue: {:?}\n", + error + ); + } + } + self.queues + .admission + .release_retained_owners_after_processor_stop(); + } +} + +impl super::Shared { + /// Joins construction and serializes the stop writes with detachment. + /// After installation, seeing no owner proves that an earlier stop caller + /// has already stopped both processors. Callbacks and destruction follow + /// outside the device mutex. + pub(super) fn stop_queues(&self) { + self.preparations.wait_drained(); + let firmware = { + let mut state = self.state.lock(); + if let Some(firmware) = (*state).as_deref_mut() { + firmware.secondary.stop(); + firmware.primary.stop(); + } + state.take() + }; + if let Some(mut firmware) = firmware { + firmware.finish_stopped_queues(); + } + } +} + +/// One monotonic domain for recovery grace, idle effort and control retirement. +fn now_ns() -> u64 { + ::ktime_get() as u64 +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs new file mode 100644 index 00000000000000..f1ae1ffe80783b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler callbacks and the off-lock fence boundary of a logical queue. + +use super::{Backend, DeferredBatch}; +use crate::g17::{ + completion::Deferred, + job::{self, Engine, Packet}, +}; +use core::sync::atomic::Ordering; +use kernel::{dma_fence::Fence, drm::sched, prelude::*, sync::Arc}; + +impl Backend { + pub(super) fn release_backlog(&self, packet: &Packet) { + if packet.take_backlog() { + self.backlog.fetch_sub(1, Ordering::AcqRel); + } + } + + fn stop_after_error(&self) { + self.shared.lose_device_quietly(); + self.shared.stop_queues(); + } + + /// Complete all failure fanout before signalling this packet's unpublished + /// error. Visible failures keep the engine's retained owner through a witness. + pub(super) fn finish_publication( + &self, + packet: &Arc, + result: Result, + deferred: &mut DeferredBatch, + ) -> Result { + if let Err(error) = result { + if [EIO, ETIMEDOUT, ENODEV].contains(&error) || packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + deferred.finish(); + if !packet.is_published() { + packet.completion.fail_unpublished(error); + } + Err(error) + } else { + deferred.finish(); + Ok(packet.completion.fence()) + } + } + + fn release_queue(&self) -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + // Retain a published scheduler before a never-kicked physical graph is + // removed. The teardown entry owns its root until control consumption. + firmware + .queues + .teardown + .close(&self.context, super::now_ns())?; + firmware.queues.release_compute(self.owner)?; + firmware.queues.release_render(self.owner)?; + firmware.queues.admission.release_owner(self.owner); + drop(state); + self.shared.queue_events(); + Ok(()) + } +} + +impl job::Backend for Backend { + fn ensure_compute(&self) -> Result { + Backend::ensure_compute(self) + } + + fn fail_vm(&self, error: Error) { + self.context.status().record(error); + let mut deferred = match DeferredBatch::worker() { + Ok(batch) => batch, + Err(_) => { + self.stop_after_error(); + return; + } + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + // Final observations win over this VM's terminal fanout. + firmware.scan_compute(None, &mut deferred)?; + firmware.settle_render_vm(self.context.status(), error, &mut deferred)?; + firmware + .queues + .fail_compute_vm(self.context.status(), error, &mut deferred) + })(); + deferred.finish(); + if result.is_err() { + self.stop_after_error(); + } + } + + fn release_owner(&self) { + if self.release_queue().is_err() { + self.stop_after_error(); + } + } + + /// Firmware kick parents order the same engine. Explicit cross-engine and + /// input dependencies were already attached to the DRM scheduled job. + fn prepare(&self, _packet: &Packet) -> Option { + None + } + + fn publish(&self, packet: Arc) -> Result { + if let Some(result) = packet.early_result() { + return result.map(|()| packet.completion.fence()); + } + let mut deferred = match DeferredBatch::publication() { + Ok(batch) => batch, + Err(error) => { + self.release_backlog(&packet); + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + return Err(error); + } + }; + let result = if !packet.begin_publication() { + Err(ECANCELED) + } else if packet.completion.status().get() != 0 { + Err(EIO) + } else { + match packet.engine() { + Engine::Compute => self.compute_packet(&packet, &mut deferred), + Engine::Render => self.render_packet(&packet, &mut deferred), + } + }; + self.release_backlog(&packet); + self.finish_publication(&packet, result, &mut deferred) + } + + fn timed_out(&self, packet: Arc) -> sched::Status { + self.release_backlog(&packet); + let result = match packet.engine() { + Engine::Compute => self.timeout_compute(&packet), + Engine::Render => self.timeout_render(&packet), + }; + if result.is_err() { + self.stop_after_error(); + } + sched::Status::NoHang + } + + fn cancel(&self, packet: Arc) { + self.release_backlog(&packet); + if packet.request_cancel() { + Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); + return; + } + let result = match packet.engine() { + Engine::Compute => self.cancel_compute(&packet), + Engine::Render => self.cancel_render(&packet), + }; + if result.is_err() { + self.stop_after_error(); + } + } + + fn defer_timeout(&self, packet: &Arc) -> bool { + if packet.engine() != Engine::Render { + return false; + } + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) + } + + fn try_early(&self, packet: &Arc) -> bool { + self.try_render_early(packet) + } + + fn count_backlog(&self, packet: &Packet) { + packet.count_backlog(); + self.backlog.fetch_add(1, Ordering::AcqRel); + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs new file mode 100644 index 00000000000000..610643ef6e379b --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler cancellation and final retirement checks for retained compute work. + +use super::{Backend, DeferredBatch}; +use crate::g17::job::{self, Packet}; +use kernel::{prelude::*, sync::Arc}; + +impl Backend { + /// The caller owns cancellation of a packet that never claimed publication. + /// A missing retained packet may already have a healthy callback waiting off-lock. + pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + // Teardown owns the detached queues and will settle their callbacks. + return Ok(None); + }; + firmware.scan_compute(None, &mut deferred)?; + let Some(queue) = firmware.queues.compute_mut(self.owner) else { + return Ok(None); + }; + if !queue.owns_packet(packet) { + return Ok(None); + } + queue.timeout(&mut |event| deferred.push(event)) + })(); + if result.as_ref().is_ok_and(|status| status.is_some()) { + // VM status and peer settlement precede signalling this timeout. The backend + // owns the same logical VM as the retained packet matched under the mutex. + ::fail_vm(self, ETIMEDOUT); + } + deferred.finish(); + result.map(|_| ()) + } + + /// Cancel only a scheduler waiter. Visible mappings stay pinned until their witness; + /// cancellation neither quarantines hardware admission nor fans out a VM failure. + pub(super) fn cancel_compute(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = (|| { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + firmware.scan_compute(None, &mut deferred)?; + if let Some(queue) = firmware.queues.compute_mut(self.owner) { + queue.cancel(packet, &mut |event| deferred.push(event))?; + } + Ok(()) + })(); + deferred.finish(); + result + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs new file mode 100644 index 00000000000000..0643da70d72ca0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Process-exit controls for closed contexts whose compute commands remain owned. + +use super::{now_ns, DeferredBatch}; +use crate::g17::{ + channel::Rings, + context::Context, + fw::channels::{ContextKill, ControlRecord}, + queue::compute::exit, + recovery::Memory as _, + Firmware, MSG_CONTROL_NOTIFY, +}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +impl Firmware { + pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { + let recovery_closed = self.init.recovery_state()? == 0; + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + queue.settle_exit(&self.queues.exit_acks, recovery_closed, &mut |event| { + deferred.push(event) + })?; + } + } + for cookie in self.queues.exit_acks.snapshot() { + if cookie != 0 + && !self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_cookie() == Some(cookie)) + { + self.queues.exit_acks.forget(cookie); + } + } + if self.recovery.pending() || !recovery_closed { + return Ok(()); + } + let now = now_ns(); + for index in 0..self.queues.teardown.len() { + let Some((context, age)) = self.queues.teardown.closed(index, now) else { + continue; + }; + if !self.queues.render.teardown_idle(&context)? + || !self.queues.render.teardown_pools_released(&context) + { + continue; + } + if let Err(error) = self.publish_compute_exit(&context, age, now) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit request failed: {:?}\n", + context.id(), + error + ); + } + } + Ok(()) + } + + fn publish_compute_exit(&mut self, context: &Arc, age: u64, now: u64) -> Result { + let Some(index) = self.queues.compute.iter().position(|entry| { + entry + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .is_some_and(|queue| queue.exit_candidate(context, age)) + }) else { + return Ok(()); + }; + if self.render_control_backpressured() { + return Ok(()); + } + let queue = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + .ok_or(EIO)?; + let Some(owner) = queue.exit_owner(context) else { + return Ok(()); + }; + let target = queue.qid(); + let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); + for entry in self.queues.compute.iter().flatten() { + if let Some(queue) = entry.queue.as_deref() { + let bit = 1u128 << queue.qid(); + if Arc::ptr_eq(queue.context(), context) { + own |= bit; + } else { + foreign |= bit; + } + } + } + let state = exit::State::read(context)?; + let Some(qids) = state.scope(target, own, foreign) else { + return Ok(()); + }; + let mut hold = 0u64; + for slot in 0..crate::hw::t8140::queues::RENDER_SLOTS as u8 { + if slots & (1u64 << slot) != 0 && self.queues.render.qids_for_slot(slot) & qids != 0 { + hold |= 1u64 << slot; + } + } + if hold != 0 + && !self + .queues + .admission + .hold_render_slots_for_exit_kill(owner, hold) + { + return Ok(()); + } + let cookie = exit::next_cookie(); + let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie)); + if let Err(error) = Rings::publish_control(&self.init, &record) { + self.queues.admission.release_exit_kill_hold(hold); + return if error == EAGAIN { Ok(()) } else { Err(error) }; + } + // Publication is irrevocable even if its doorbell fails. Keep both + // the killed graph and the held render identities until processor stop. + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .record_exit(cookie, now); + fence(Ordering::SeqCst); + if let Err(error) = self.primary.notify(MSG_CONTROL_NOTIFY) { + dev_warn!( + self.primary.state.shared.dev.as_ref(), + "Scheduler {} exit announcement failed: {:?}\n", + context.id(), + error + ); + } + Ok(()) + } + + pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { + if self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_polling(now)) + { + return true; + } + (0..self.queues.teardown.len()).any(|index| { + self.queues + .teardown + .closed(index, now) + .is_some_and(|(context, age)| { + age < exit::POLL_NS + && self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.exit_candidate(&context, u64::MAX)) + }) + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs new file mode 100644 index 00000000000000..17f228b9d1f048 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Ordered republication of witnessed, unexecuted compute commands. + +use super::DeferredBatch; +use crate::g17::{completion::Deferred, qos, recovery, Firmware}; +use kernel::prelude::*; + +impl Firmware { + /// Called only from an outer worker or client submission, never from event + /// dispatch or a recovery callback nested within publication. All candidate + /// queues are observed before any retained packet is published again. + pub(in crate::g17) fn replay_compute(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() { + return Ok(()); + } + let state = recovery::Memory::recovery_state(&self.init)?; + if state != 0 { + return Ok(()); + } + let view = qos::View::new(self.init.qos()?)?; + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + queue.prepare_replays( + state, + |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + } + for index in 0..self.queues.compute.len() { + loop { + if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { + return Ok(()); + } + let Some(queue) = self.queues.compute[index] + .as_ref() + .and_then(|entry| entry.queue.as_deref()) + else { + break; + }; + let Some(packet) = queue.replay_front() else { + break; + }; + let owner = queue.owner(); + + let result = (|| { + self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)? + .prepare_replay(&packet)?; + let owner = owner.ok_or(EIO)?; + packet.check_dependencies()?; + let dependencies = self.queues.render_prefix(owner, &packet)?; + self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) + })(); + let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) + && (self.recovery.pending() + || recovery::Memory::recovery_state(&self.init)? != 0); + let queue = self.queues.compute[index] + .as_mut() + .and_then(|entry| entry.queue.as_deref_mut()) + .ok_or(EIO)?; + if let Err(error) = result { + if recovery_wait && !queue.owns_packet(&packet) { + // Keep this prepared attempt and its one timeout credit + // while the outer worker closes the new recovery. + return Ok(()); + } + queue.fail_replays(error); + if !queue.owns_packet(&packet) { + deferred + .push(Deferred::Retired(packet.completion.clone(), Err(ENODATA)))?; + } + } + queue.finish_replay(&packet)?; + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs new file mode 100644 index 00000000000000..e9fc977d623b81 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event and restart callbacks borrow the complete firmware owner. The reusable +//! completion collector belongs to the worker and is drained outside its mutex. + +use super::DeferredBatch; +use crate::g17::{ + channel, event, + fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, + recovery, + status::VmStatus, + Firmware, Role, +}; +use kernel::{prelude::*, sync::Arc}; + +struct Service<'a> { + firmware: &'a mut Firmware, + deferred: &'a mut DeferredBatch, + render_masks: [u64; 2], + settlement_error: Option, +} + +impl event::Transport for Service<'_> { + fn trace(&mut self, role: Role) { + let firmware = &mut self.firmware; + firmware + .rings + .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); + } + fn statistics(&mut self, role: Role) { + let firmware = &mut self.firmware; + firmware + .rings + .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); + } + fn drain( + &mut self, + role: Role, + mut consume: impl FnMut(&mut Self, Event) -> Result, + ) -> Result { + let mut events = channel::Events::new(role); + while let Some(event) = + events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? + { + if !consume(self, event)? { + break; + } + } + Ok(()) + } + fn recovery_state(&self) -> Result { + recovery::Memory::recovery_state(&self.firmware.init) + } +} + +impl event::Host for Service<'_> { + fn recovery(&self) -> &recovery::State { + &self.firmware.recovery + } + fn recovery_mut(&mut self) -> &mut recovery::State { + &mut self.firmware.recovery + } +} + +impl event::Queues for Service<'_> { + fn context_killed(&mut self, cookie: u64) { + self.firmware.queues.exit_acks.note(cookie); + } + fn flush_tvb_reply(&mut self) -> Result { + self.firmware.flush_tvb_reply() + } + fn flush_freelist_release(&mut self) -> Result { + self.firmware.flush_freelist_release() + } + fn observe_render_releases(&mut self) -> Result { + self.firmware.observe_render_releases() + } + fn control_backpressured(&self) -> bool { + self.firmware.render_control_backpressured() + } + fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { + self.firmware.queue_tvb_grow(request) + } + fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result { + self.firmware.complete_freelist_grow(completion) + } + fn retain_render_completions(&mut self, masks: [u64; 2]) { + // Inline timeout and drain callers may discard the worker hints. The + // queue owner retains stamps until its borrowed pair returns to scan. + self.firmware.queues.retain_render_completions(masks); + self.render_masks[0] |= masks[0]; + self.render_masks[1] |= masks[1]; + } + fn complete_compute(&mut self, masks: [u64; 2]) -> Result { + self.firmware.scan_compute(Some(masks), self.deferred) + } +} + +impl recovery::Host for Service<'_> { + fn recovery(&self) -> &recovery::State { + &self.firmware.recovery + } + fn recovery_mut(&mut self) -> &mut recovery::State { + &mut self.firmware.recovery + } +} + +impl recovery::Memory for Service<'_> { + fn recovery_state(&self) -> Result { + self.firmware.init.recovery_state() + } + fn host_recovery(&self) -> Result { + self.firmware.init.host_recovery() + } + fn fault_sources(&self) -> Result { + self.firmware.init.fault_sources() + } + + fn transition(&self, expected: u32, next: u32) -> Result { + self.firmware.init.transition(expected, next) + } + fn clear_timestamps(&self) -> Result { + self.firmware.init.clear_timestamps() + } + fn entries(&self, out: &mut [recovery::Entry; recovery::MAX_ENTRIES]) -> Result { + self.firmware.init.entries(out) + } + fn wait_tick(&self) { + self.firmware.init.wait_tick(); + } +} + +impl recovery::Queues for Service<'_> { + fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { + self.firmware.queues.render_pass_started(qid, stamp) + } + fn publish_recovery_targets(&self) -> Option { + let mut failure = self.firmware.queues.publish_render_recovery_targets(); + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + if let Err(error) = queue.publish_recovery_target() { + failure = failure.or(Some(error)); + } + } + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + failure + } + fn render_scheduler_active(&self) -> bool { + self.firmware.queues.render_scheduler_active() || !self.firmware.queues.teardown.is_empty() + } + fn oldest_spared(&self, qid: u8) -> Option { + self.firmware.queues.render_oldest_spared(qid).or_else(|| { + self.firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .find(|queue| queue.qid() == qid) + .and_then(|queue| queue.oldest_spared()) + }) + } + fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)) { + self.firmware.queues.visit_render_owners(qid, visit); + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + if queue.qid() == qid { + queue.visit_owners(visit); + } + } + } + fn classify_work(&mut self, class: &dyn Fn(&Arc) -> recovery::Class) -> u128 { + for queue in self + .firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + { + queue.classify(class); + } + self.firmware.queues.classify_render(class) + } + fn settle_blamed_render(&mut self, qid: u8, error: Error) { + let result = self + .firmware + .settle_blamed_render(qid, error, self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } + fn settle_cancelled_renders(&mut self, qids: u128) { + let result = self.firmware.settle_cancelled_renders(qids, self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } + fn rescan_compute(&mut self) -> Result { + self.firmware.scan_compute(None, self.deferred) + } + fn release_retired_compute(&mut self) { + let result = self.firmware.release_compute_witnesses(self.deferred); + self.settlement_error = self.settlement_error.or(result.err()); + } +} + +impl Firmware { + fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { + let state = recovery::Memory::recovery_state(&self.init)?; + let view = crate::g17::qos::View::new(self.init.qos()?)?; + let registry = &mut self.queues; + for queue in registry + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.release_witnessed( + state, + |owner| registry.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; + } + Ok(()) + } + + pub(super) fn service_runtime_events( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result<(event::Pending, [u64; 2])> { + let mut service = Service { + firmware: self, + deferred, + render_masks: [0; 2], + settlement_error: None, + }; + let pending = event::service(&mut service)?; + if pending.recovery { + // Inline timeout/drain service can consume the restart record too. + // Close publication in this same critical section and ensure the + // event worker will join builders before the restart handshake. + let shared = &service.firmware.primary.state.shared; + shared.preparations.close(); + shared.queue_events(); + } + Ok((pending, service.render_masks)) + } + + /// Preparation admission is already closed and all builders have joined. + pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { + let shared = self.primary.state.shared.clone(); + let mut service = Service { + firmware: self, + deferred, + render_masks: [0; 2], + settlement_error: None, + }; + let result = recovery::service(&mut service, || { + shared.crashed.load(core::sync::atomic::Ordering::Acquire) + })?; + if let Some(error) = result.target_error { + dev_warn!( + shared.dev.as_ref(), + "Recovery target copy failed: {:?}\n", + error + ); + } + if let Some(error) = result.scan_error { + dev_warn!( + shared.dev.as_ref(), + "Post-recovery compute scan failed: {:?}\n", + error + ); + } + if let Some(error) = service.settlement_error { + return Err(error); + } + Ok(()) + } +} + +impl crate::g17::Shared { + /// The single event work item owns this preallocated collector until all + /// callbacks are finished. Shutdown can detach Firmware during an off-lock + /// boundary; the collector itself still owns every deferred completion. + pub(in crate::g17) fn run_runtime_events(self: &Arc) { + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + self.stop_queues(); + return; + } + let mut deferred = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return; + }; + let Some(deferred) = firmware.queues.worker_deferred.take() else { + drop(state); + self.lose_device_quietly(); + return; + }; + deferred + }; + let result = self.runtime_event_turn(&mut deferred); + deferred.finish(); + if let Some(firmware) = (*self.state.lock()).as_deref_mut() { + firmware.queues.worker_deferred = Some(deferred); + } + if let Err(error) = result { + dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); + self.lose_device_quietly(); + self.stop_queues(); + } + } + + fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { + let (recover, masks) = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + let (pending, masks) = firmware.service_runtime_events(deferred)?; + let recover = pending.recovery || firmware.recovery.pending(); + if recover { + // Epoch closure and event consumption are serialized with every + // publisher. Joining builders must happen after unlocking. + self.preparations.close(); + } + (recover, masks) + }; + deferred.finish(); + if recover { + self.preparations.wait_drained(); + } + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + if recover { + firmware.service_runtime_recovery(deferred)?; + if !firmware.recovery.pending() { + if !self.preparations.reopen() { + return Ok(()); + } + // Restart stopped consumption before its following record. + // Resume builders before observing pairs outside this lock. + self.queue_events(); + } + } + // Replay only at this outer boundary, after recovery has settled. + // Publication may service events but must never recurse into replay. + firmware.replay_compute(deferred)?; + drop(state); + self.observe_renders(masks, deferred)?; + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if self.crashed.load(core::sync::atomic::Ordering::Acquire) { + return Err(EIO); + } + let now = super::now_ns(); + let compute = firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.in_flight()); + let in_flight = compute || firmware.queues.render_in_flight(); + let idle = firmware.idle_check(now, in_flight)?; + firmware.reclaim_drained_renders(deferred)?; + firmware.service_compute_exit(deferred)?; + { + let mut reclaim = self.reclaim.lock(); + firmware.service_teardown(&mut reclaim)?; + if !reclaim.is_empty() { + self.queue_reclaims(); + } + } + let backpressured = firmware.render_control_backpressured(); + let polling = backpressured + || firmware.queues.teardown.polling(now) + || firmware.compute_exit_polling(now); + if polling { + self.queue_poll(); + } + if let Some(delay) = idle { + self.queue_idle(delay); + } + if firmware.recovery.pending() { + self.preparations.close(); + self.queue_events(); + } else if !recover + && !polling + && (channel::Events::pending(&firmware.init, Role::Primary)? + || channel::Events::pending(&firmware.init, Role::Secondary)?) + { + // A full bounded pass must not depend on a fresh notification for + // records that arrived while it was consuming the same ring. + self.queue_events(); + } + if firmware.render_grow_pending() && !firmware.recovery.pending() { + self.queue_grow(); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs new file mode 100644 index 00000000000000..a6aa740ecf4e3e --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -0,0 +1,683 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device ownership of retained render pairs and their shared free-list leases. +//! The device mutex protects identities and publication. A preparation gate +//! lease permits graph construction outside that mutex without session teardown. + +mod exit; +mod dependencies; +mod grow; +mod observe; +mod pools; +mod prepare; +mod recovery; +mod settle; +mod submit; +pub(super) mod teardown; + +use super::{DeferredBatch, Registry}; +use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; +use crate::hw::t8140::queues::RENDER_SLOTS; +use kernel::{prelude::*, sync::Arc}; + +#[derive(Copy, Clone)] +struct Config { + clusters: u32, + descriptor_flags: [u32; 2], +} + +#[derive(Copy, Clone)] +struct Reservation { + slot: u8, + ids: [kick::Id; 2], + owner: u64, + buffer: (u8, u64), +} + +struct Entry { + reservation: Reservation, + pair: Option>, + parked: Option, + preparing: Option>, + context: Option>, + pool: Option>, + pending_owner: Option, + pending_owner_closed: bool, + borrowed_active: bool, + borrowed_dependencies: Option, + event_masks: [u64; 2], + closed: bool, + pending_error: Option<(Arc, Error)>, + pending_timeout: bool, + constructing: bool, + growing: bool, + published: bool, +} + +pub(super) struct RenderState { + config: Config, + entries: KVec>, + pools: pools::Pools, + grow: grow::Grow, + retired: KVec, + next_pm_generation: u64, +} + +impl RenderState { + pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { + if clusters == 0 { + return Err(EINVAL); + } + let mut entries = KVec::with_capacity(RENDER_SLOTS, GFP_KERNEL)?; + for _ in 0..RENDER_SLOTS { + entries.push(None, GFP_KERNEL)?; + } + Ok(Self { + config: Config { + clusters, + descriptor_flags, + }, + entries, + pools: pools::Pools::new()?, + grow: grow::Grow::new()?, + retired: KVec::with_capacity(128, GFP_KERNEL)?, + next_pm_generation: 1, + }) + } + + fn reserve_pm_generation(&mut self) -> Result { + let generation = self.next_pm_generation; + self.next_pm_generation = generation.checked_add(1).ok_or(EOVERFLOW)?; + Ok(generation) + } + + fn collect_retired(&mut self, slot: usize) -> Result { + loop { + if self.retired.len() == self.retired.capacity() { + return Ok(()); + } + let retired = self.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_retired_binding()); + let Some(retired) = retired else { + return Ok(()); + }; + self.retired + .push_within_capacity(retired) + .map_err(|_| EIO)?; + } + } + + fn entry(&self, slot: u8) -> Result<&Entry> { + self.entries + .get(usize::from(slot)) + .and_then(Option::as_ref) + .ok_or(EIO) + } + + fn entry_mut(&mut self, slot: u8) -> Result<&mut Entry> { + self.entries + .get_mut(usize::from(slot)) + .and_then(Option::as_mut) + .ok_or(EIO) + } + + fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { + let entry = self.entry_mut(reservation.slot)?; + if entry.reservation.ids != reservation.ids + || entry.reservation.owner != reservation.owner + || entry.reservation.buffer != reservation.buffer + || entry.pair.is_some() + || entry.preparing.is_some() + || entry.context.is_some() + || !entry.constructing + { + return Err(EIO); + } + Ok(entry) + } + + /// Keep metadata while graph preparation owns the pair outside the mutex. + /// Completion scans skip it; cancellation is recorded and applied on return. + fn take_pair(&mut self, slot: u8, owner: u64, packet: &Arc) -> Result> { + let entry = self.entry_mut(slot)?; + if entry.closed + || entry.reservation.owner != owner + || entry.preparing.is_some() + || entry.pending_error.is_some() + || entry.pending_timeout + { + return Err(EBUSY); + } + let pair = entry.pair.as_ref().ok_or(EIO)?; + if !pair.matches(owner, &packet.context)? { + return Err(EIO); + } + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet.clone()); + entry.pair.take().ok_or(EIO) + } + + /// Validate before taking the owner from the caller. Even rejected returns + /// keep their backing alive until the caller resolves the reservation. + fn return_pair(&mut self, slot: u8, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.entry_mut(slot)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() != slot + || returning.qids() != entry.reservation.ids.map(kick::Id::qid) + || returning.owner() != Some(entry.reservation.owner) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + + fn save_detaching_pool(&self, init: &crate::g17::initdata::InitData, slot: u8) -> Result { + let pool = self.entry(slot)?.pool.as_ref().ok_or(EIO)?; + let shared = self.entries.iter().enumerate().any(|(index, entry)| { + index != usize::from(slot) + && entry.as_ref().is_some_and(|entry| { + entry.pool.as_ref().is_some_and(|other| { + other.id() == pool.id() && other.control_va() == pool.control_va() + }) + }) + }); + if !shared { + self.pools.save(init, pool)?; + } + Ok(()) + } +} + +impl Registry { + fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { + if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { + return Err(EINVAL); + } + if self.render.entries[usize::from(slot)].is_none() { + let identity = self.identities.reserve(2)?.start; + let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; + let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { + Ok(id) => id, + Err(error) => { + self.qids.cancel_unpublished(tiling)?; + return Err(error); + } + }; + self.render.entries[usize::from(slot)] = Some(Entry { + reservation: Reservation { + slot, + ids: [tiling, fragment], + owner, + buffer: (0, 0), + }, + pair: None, + parked: None, + preparing: None, + context: None, + pool: None, + pending_owner: None, + pending_owner_closed: false, + borrowed_active: false, + borrowed_dependencies: None, + event_masks: [0; 2], + closed: false, + pending_error: None, + pending_timeout: false, + constructing: false, + growing: false, + published: false, + }); + } + Ok(()) + } + + fn reserve_render(&mut self, slot: u8, owner: u64) -> Result { + self.ensure_render_ids(slot, owner)?; + let entry = self.render.entry(slot)?; + if entry.constructing + || entry.pair.is_some() + || entry.parked.is_some() + || entry.preparing.is_some() + || entry.growing + { + return Err(EBUSY); + } + let ids = self.render.entry(slot)?.reservation.ids; + // Physical queue identities survive failed graph construction. Only + // failure to acquire the second identity can return the first one. + for id in ids { + self.qids.publish(id)?; + } + let buffer = self.buffers.allocate_render()?; + let entry = self.render.entry_mut(slot)?; + entry.reservation.owner = owner; + entry.reservation.buffer = buffer; + entry.constructing = true; + entry.closed = false; + Ok(entry.reservation) + } + + fn install_render( + &mut self, + reservation: Reservation, + pair: &mut Option>, + ) -> Result { + let built = pair.as_ref().ok_or(EINVAL)?; + if built.slot() != reservation.slot + || built.qids() != reservation.ids.map(kick::Id::qid) + || built.owner() != Some(reservation.owner) + || built.buffer_id() != reservation.buffer.0 + { + return Err(EIO); + } + if self.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + // Construction holds one PB reference; retain the owner independently + // and return that reference before the first command acquires its own. + self.buffers.reserve_owner(reservation.buffer.0)?; + self.buffers.release(reservation.buffer.0)?; + let entry = self.render.reserved(reservation)?; + entry.context = Some(built.context().clone()); + entry.pool = Some(built.pool().clone()); + entry.pair = pair.take(); + entry.constructing = false; + Ok(()) + } + + /// Drop the rejected graph outside the device mutex before returning its + /// temporary PB reference. The physical queue identities remain reserved. + fn cancel_render_reservation(&mut self, reservation: Reservation) -> Result { + self.render.reserved(reservation)?; + self.buffers.release(reservation.buffer.0)?; + self.render.entry_mut(reservation.slot)?.constructing = false; + Ok(()) + } + + pub(super) fn fail_render_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for entry in self.render.entries.iter_mut().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + { + continue; + } + if let Some(pair) = entry.pair.as_mut() { + pair.quarantine(error, &mut |event| deferred.push(event))?; + } else if entry.preparing.is_some() { + entry.pending_error = Some((status.clone(), error)); + } + } + Ok(()) + } +} + +struct Host<'a> { + firmware: &'a mut crate::g17::Firmware, + registration_powered: bool, +} +impl Host<'_> { + fn release_registration(&mut self) { + if !core::mem::take(&mut self.registration_powered) { + return; + } + if let Err(error) = self.firmware.release_registration_power() { + dev_warn!( + self.firmware.primary.state.shared.dev.as_ref(), + "Could not restore render idle policy: {:?}\n", + error + ); + } + } +} +impl crate::g17::queue::render::Host for Host<'_> { + fn epoch(&self) -> Result<(u64, u32)> { + Ok(( + self.firmware.recovery.generation(), + crate::g17::recovery::Memory::recovery_state(&self.firmware.init)?, + )) + } + fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { + use core::sync::atomic::Ordering; + let stamps = [ + self.firmware.init.stamp(qids[0])?.load(Ordering::Relaxed), + self.firmware.init.stamp(qids[1])?.load(Ordering::Relaxed), + ]; + core::sync::atomic::fence(Ordering::Acquire); + Ok(stamps) + } + fn qos_publish( + &mut self, + owner: crate::g17::qos::Owner, + scheduler: u64, + ) -> Result { + use crate::g17::qos; + self.firmware.queues.accounting.publish( + &qos::View::new(self.firmware.init.qos()?)?, + owner, + scheduler, + qos::clock, + ) + } + fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { + use crate::g17::qos; + self.firmware + .queues + .accounting + .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) + } + fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + for _ in 0..3 { + if !self.firmware.bridge.regs.g17_configuration_powered() { + self.firmware.acquire_registration_power()?; + self.registration_powered = true; + } + if self + .firmware + .bridge + .regs + .install_render_pair(registration)? + { + return Ok(()); + } + } + Err(EAGAIN) + } + fn publish_qid(&mut self, id: kick::Id) -> Result { + self.firmware.queues.qids.publish(id) + } + fn publish_outer_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + commit: impl FnOnce() -> Result, + late_tiling: impl FnOnce(), + ) -> Result<[u32; 2]> { + self.firmware + .init + .publish_work_pair(fragment, tiling, true, commit, late_tiling) + } + fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { + self.release_registration(); + let result = self.firmware.queues.render.pools.activate(id, generation); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn publish_retained_pair( + &mut self, + fragment: &crate::g17::fw::channels::WorkSlot, + tiling: &crate::g17::fw::channels::WorkSlot, + next: [u32; 2], + ) -> Result { + let result = self + .firmware + .init + .publish_retained_work_pair(fragment, tiling, next); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn notify_render(&mut self) -> Result { + self.release_registration(); + let result = self + .firmware + .primary + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | 0x08); + if result.is_err() { + self.firmware.primary.state.shared.lose_device_quietly(); + } + result + } + fn note_submission(&mut self) -> Result { + self.firmware.note_submission() + } + fn complete_render( + &mut self, + owners: Option<[crate::g17::qos::Owner; 2]>, + pool_id: u16, + generation: u64, + buffer_id: u8, + ) -> Result { + use crate::g17::qos; + let queues = &mut self.firmware.queues; + let next = queues.render.pools.completed(pool_id, generation)?; + queues.buffers.can_release(buffer_id)?; + let view = qos::View::new(self.firmware.init.qos()?)?; + if let Some(owners) = owners { + queues.accounting.complete(&view, &owners)?; + } + queues.render.pools.commit_completed(pool_id, next); + queues.buffers.release(buffer_id) + } +} + +impl crate::g17::Firmware { + /// The pair is returned even when transport, accounting or collection fails. + /// No off-lock operation or callback which can re-enter the device runs here. + fn with_render( + &mut self, + slot: usize, + operation: impl FnOnce(&mut Pair, &mut Host<'_>) -> Result, + ) -> Result { + let mut pair = self + .queues + .render + .entries + .get_mut(slot) + .and_then(Option::as_mut) + .and_then(|entry| entry.pair.take()) + .ok_or(EBUSY)?; + let mut host = Host { + firmware: self, + registration_powered: false, + }; + let result = operation(&mut pair, &mut host); + host.release_registration(); + self.queues.render.entries[slot].as_mut().ok_or(EIO)?.pair = Some(pair); + result + } + + fn release_render_lists(&mut self) -> Result { + if self.queues.render.grow.reply.is_some() { + return Err(EAGAIN); + } + let primary = &mut self.primary; + self.queues.render.pools.publish_releases(&self.init, || { + primary.notify(crate::g17::MSG_CONTROL_NOTIFY) + }) + } + + /// Events have already been consumed by the caller. Each healthy completion + /// services its final pool release before examining the next retained job. + fn retire_render_slot( + &mut self, + slot: usize, + masks: Option<[u64; 2]>, + deferred: &mut DeferredBatch, + ) -> Result { + let generation = self.recovery.generation(); + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + loop { + let progressed = self.with_render(slot, |pair, host| { + pair.retire(masks, generation, state, host, &mut |event| { + deferred.push(event) + }) + })?; + if !progressed { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => break, + result => result?, + } + } + Ok(()) + } + + pub(in crate::g17) fn settle_render_vm( + &mut self, + status: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + for slot in 1..self.queues.render.entries.len() { + if self.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry.pair.is_some() + && entry + .context + .as_ref() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + }) + { + self.retire_render_slot(slot, None, deferred)?; + } + } + self.queues.fail_render_vm(status, error, deferred) + } + + pub(in crate::g17) fn stop_render(&mut self) { + for entry in self.queues.render.entries.iter_mut().flatten() { + if let Some(pair) = entry.pair.as_mut() { + if let Err(error) = pair.stopped(&mut |event| { + event.finish(); + Ok(()) + }) { + dev_err!( + self.primary.state.shared.dev.as_ref(), + "Could not settle stopped render pair: {:?}\n", + error + ); + } + } + } + } +} + +impl Registry { + pub(in crate::g17) fn render_oldest_spared(&self, qid: u8) -> Option { + self.render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .and_then(|pair| pair.oldest_spared()) + } + pub(in crate::g17) fn visit_render_owners( + &self, + qid: u8, + visit: &mut dyn FnMut(&Arc), + ) { + for pair in self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + if pair.qids().contains(&qid) { + pair.visit_owners(visit); + } + } + } + pub(in crate::g17) fn classify_render( + &self, + classify: &dyn Fn(&Arc) -> crate::g17::recovery::Class, + ) -> u128 { + let mut spared = 0; + for pair in self + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + pair.classify(classify); + if pair.oldest_spared() == Some(true) { + for qid in pair.qids() { + spared |= 1u128 << qid; + } + } + } + spared + } + + pub(in crate::g17) fn render_scheduler_active(&self) -> bool { + self.render.entries.iter().any(Option::is_some) + || !self.render.retired.is_empty() + || !self.teardown.is_empty() + } + pub(in crate::g17) fn render_in_flight(&self) -> bool { + self.render.entries.iter().flatten().any(|entry| { + entry.pair.as_ref().map_or( + (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, + |pair| pair.published_in_flight(), + ) + }) + } + pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { + for entry in self.render.entries.iter_mut().flatten() { + for id in entry.reservation.ids { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); + } + } + } + pub(in crate::g17) fn render_needs_rescan(&self) -> bool { + self.render.entries.iter().flatten().any(|entry| { + let active = entry + .pair + .as_ref() + .map_or(entry.borrowed_active, |pair| pair.published_in_flight()); + active + && entry.reservation.ids.into_iter().all(|id| { + let qid = usize::from(id.qid()); + entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 + }) + }) + } + pub(in crate::g17) fn defer_render_timeout( + &mut self, + packet: &Arc, + error: Error, + ) -> bool { + let Some(slot) = packet.completion.render_slot() else { + return false; + }; + let Ok(entry) = self.render.entry_mut(slot) else { + return false; + }; + if entry.preparing.is_none() && !entry.growing { + return false; + } + if !entry + .borrowed_dependencies + .as_ref() + .is_some_and(|dependencies| dependencies.contains_packet(packet)) + { + return false; + } + packet.completion.defer_timeout(error); + entry.pending_timeout = true; + true + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs new file mode 100644 index 00000000000000..09e4322bf42cef --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Both engines preserve the covered render prefix while physical graphs are borrowed. + +use crate::g17::{dependency::Frontier, job::Packet, runtime::Registry}; +use kernel::{prelude::*, sync::Arc}; + +impl Registry { + pub(in crate::g17::runtime) fn render_prefix( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut frontier = Frontier::new(); + for entry in self.render.entries.iter().flatten() { + if let Some(pair) = entry.pair.as_ref() { + pair.collect_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { + dependencies.collect( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } + } + Ok(frontier) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs new file mode 100644 index 00000000000000..c47110cb76fadf --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/exit.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Physical render queue scope for an exiting logical context. A graph loan +//! remains foreign to the transaction until its exact owner is restored. + +use super::RenderState; +use crate::g17::context::Context; +use kernel::sync::Arc; + +impl RenderState { + pub(in crate::g17::runtime) fn exit_scope(&self, context: &Arc) -> (u128, u128, u64) { + let (mut own, mut foreign, mut slots) = (0, 0, 0); + for (slot, entry) in self.entries.iter().enumerate() { + let Some(entry) = entry else { + continue; + }; + let qids = self.qids_for_slot(slot as u8); + let preparing = entry.preparing.is_some() || entry.constructing || entry.growing; + if !preparing && entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) { + own |= qids; + slots |= 1u64 << slot; + } else if preparing || entry.context.is_some() { + foreign |= qids; + } + } + (own, foreign, slots) + } + + pub(in crate::g17::runtime) fn qids_for_slot(&self, slot: u8) -> u128 { + self.entries + .get(usize::from(slot)) + .and_then(Option::as_ref) + .map_or(0, |entry| { + entry.reservation.ids.into_iter().fold(0, |mask, id| { + mask | (1u128 << id.qid()) + }) + }) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs new file mode 100644 index 00000000000000..a9636cb7ceda04 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Deferred parameter-buffer growth. Event ingress retains the entire request; +//! allocation borrows one pair under a preparation lease and runs off-lock. + +use super::RenderState; +use crate::g17::{ + channel::Rings, + fw::channels::{ControlRecord, FreeListCompletion, TvbGrowReply, TvbGrowRequest, EVENT_SLOTS}, +}; +use core::sync::atomic::{fence, Ordering}; +use kernel::prelude::*; + +const RETRY_LIMIT: u16 = 1000; + +pub(super) struct Grow { + requests: KVec, + active: Option, + pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, + release_attempts: u16, +} +impl Grow { + pub(super) fn new() -> Result { + Ok(Self { + requests: KVec::with_capacity(2 * EVENT_SLOTS as usize, GFP_KERNEL)?, + active: None, + reply: None, + release_attempts: 0, + }) + } +} + +impl RenderState { + pub(in crate::g17::runtime) fn control_backpressured(&self) -> bool { + self.grow.reply.is_some() || self.pools.pending() + } + fn grow_owner(&self, request: TvbGrowRequest) -> Option { + self.entries.iter().position(|entry| { + entry.as_ref().is_some_and(|entry| { + entry.published + && u32::from(entry.reservation.buffer.0) == request.buffer_slot + && entry.context.as_ref().is_some_and(|context| { + request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) + }) + }) + }) + } + fn grow_owner_busy(&self, request: TvbGrowRequest) -> bool { + self.grow_owner(request).is_some_and(|slot| { + self.entries[slot] + .as_ref() + .is_some_and(|entry| entry.preparing.is_some() || entry.growing) + }) + } +} + +impl crate::g17::Firmware { + pub(in crate::g17) fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { + let render = &mut self.queues.render; + if request.halt_count != self.recovery.generation() + || render.grow.active == Some(request) + || render.grow.requests.contains(&request) + { + return Ok(()); + } + render + .grow + .requests + .push_within_capacity(request) + .map_err(|_| EOVERFLOW) + } + pub(in crate::g17) fn render_grow_pending(&self) -> bool { + !self.queues.render.grow.requests.is_empty() + } + pub(in crate::g17) fn render_control_backpressured(&self) -> bool { + self.queues.render.grow.reply.is_some() || self.queues.render.pools.pending() + } + pub(in crate::g17) fn complete_freelist_grow(&mut self, event: FreeListCompletion) -> Result { + self.queues.render.pools.complete_grow(event) + } + pub(in crate::g17) fn observe_render_releases(&mut self) -> Result { + self.queues.render.pools.observe(&self.init) + } + pub(in crate::g17) fn flush_tvb_reply(&mut self) -> Result { + let Some((request, grew, _)) = self.queues.render.grow.reply else { + return Ok(true); + }; + if request.halt_count != self.recovery.generation() { + return Err(EIO); + } + let record = ControlRecord::TvbGrow(TvbGrowReply::new(&request, grew)); + match Rings::publish_control(&self.init, &record) { + Err(error) if error == EAGAIN => return Ok(false), + result => { + result?; + } + } + // Producer ownership is irreversible, even if announcement fails. + self.queues.render.grow.reply = None; + fence(Ordering::SeqCst); + self.primary + .notify((0x84 << crate::g17::MSG_TYPE_SHIFT) | 0x11)?; + Ok(true) + } + pub(in crate::g17) fn flush_freelist_release(&mut self) -> Result { + if self.queues.render.grow.reply.is_some() { + return Ok(false); + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => Ok(false), + result => { + result?; + self.queues.render.grow.release_attempts = 0; + Ok(true) + } + } + } + /// Only the sleeping event worker consumes this budget. Inline submission + /// probes cannot turn a transient full control ring into a rapid timeout. + pub(in crate::g17) fn note_render_control_backpressure(&mut self) -> Result { + let render = &mut self.queues.render; + let attempts = if let Some((_, _, attempts)) = render.grow.reply.as_mut() { + attempts + } else if render.pools.pending() { + &mut render.grow.release_attempts + } else { + return Err(EINVAL); + }; + *attempts = attempts.checked_add(1).ok_or(ETIMEDOUT)?; + if *attempts > RETRY_LIMIT { + return Err(ETIMEDOUT); + } + Ok(()) + } +} + +impl crate::g17::Firmware { + pub(super) fn commit_render_growth( + &mut self, + slot: u8, + prepared: &mut Option, + ) -> Result { + if prepared.is_none() { + return Ok(()); + } + let pair = self + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)?; + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let buffer_id = pair.buffer_id(); + // Preflight every destination before the graph starts publishing. + let table = self.init.pb_descriptor_table()?; + let offset = usize::from(buffer_id) * 16; + let words = [ + table.word(offset)?, + table.word(offset + 4)?, + table.word(offset + 8)?, + table.word(offset + 12)?, + ]; + let descriptor = pair + .manager() + .commit_growth(prepared, |previous, blocks, mappings| { + vm.append_render_pool_mappings_prepared(slot, pool, previous, blocks, mappings) + })?; + if let Some(descriptor) = descriptor { + for (word, value) in words.into_iter().zip(descriptor) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + } + Ok(()) + } +} + +impl crate::g17::Shared { + pub(in crate::g17) fn grow_render(&self) -> Result { + loop { + let lease = self.preparations.enter(false)?; + let (request, slot, mut pair) = { + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if firmware.render_control_backpressured() { + return Ok(()); + } + let generation = firmware.recovery.generation(); + let render = &mut firmware.queues.render; + if render.grow.active.is_some() { + return Ok(()); + } + let Some(index) = render.grow.requests.iter().position(|request| { + request.halt_count != generation || !render.grow_owner_busy(*request) + }) else { + return Ok(()); + }; + let request = render.grow.requests.remove(index).map_err(|_| EIO)?; + if request.halt_count != generation { + continue; + } + let slot = render.grow_owner(request); + let pair = match slot { + Some(slot) => { + let entry = render.entries[slot].as_mut().ok_or(EIO)?; + if entry.preparing.is_some() || entry.growing { + return Err(EIO); + } + entry.borrowed_active = entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()); + entry.borrowed_dependencies = + entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + entry.growing = true; + Some(entry.pair.take().ok_or(EIO)?) + } + None => None, + }; + render.grow.active = Some(request); + (request, slot, pair) + }; + let prepared = match pair.as_mut() { + Some(pair) => { + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let slot = pair.slot(); + let manager = pair.manager(); + let before = manager.blocks(); + if vm.render_pool_tvb_blocks(slot, pool) != Some(before) { + Err(EFAULT) + } else { + manager.prepare_growth( + &self.dev, + &vm, + crate::g17::buffer::firmware_growth_target(before), + None, + ) + } + } + None => Err(ENOENT), + }; + let (mut prepared, prepare_error) = match prepared { + Ok(plan) => (plan, None), + Err(error) => (None, Some(error)), + }; + let mut state = self.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if let Some(slot) = slot { + let entry = firmware.queues.render.entries[slot].as_mut().ok_or(EIO)?; + if entry.pair.is_some() || !entry.growing { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.growing = false; + } + if firmware.queues.render.grow.active != Some(request) { + return Err(EIO); + } + firmware.queues.render.grow.active = None; + self.changed.notify_all(); + if !lease.is_current() || request.halt_count != firmware.recovery.generation() { + continue; + } + let result = if let Some(error) = prepare_error { + Err(error) + } else if prepared.is_none() { + Ok(false) + } else if let Some(slot) = slot { + let pair = firmware.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .ok_or(EIO)?; + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let slot = pair.slot(); + let grew = prepared.is_some(); + pair.manager() + .commit_growth(&mut prepared, |previous, blocks, mappings| { + vm.append_render_pool_mappings_prepared( + slot, pool, previous, blocks, mappings, + ) + }) + .map(|_| grew) + } else { + Err(ENOENT) + }; + let grew = match result { + Ok(grew) => grew, + Err(error) => { + dev_warn!( + self.dev.as_ref(), + "Could not grow render parameter buffer {}: {:?}\n", + request.buffer_slot, + error + ); + false + } + }; + firmware.queues.render.grow.reply = Some((request, grew, 0)); + if !firmware.flush_tvb_reply()? { + return Ok(()); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs new file mode 100644 index 00000000000000..31bcb688c80018 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/observe.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Event-driven render snapshots outside the device mutex. The preparation +//! epoch and exact pair owner survive every read, cancellation and retry. + +use super::super::DeferredBatch; +use crate::g17::queue::render::Pair; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +impl crate::g17::Firmware { + fn take_render_observation(&mut self, start: usize) -> Result)>> { + if self.recovery.pending() { + return Ok(None); + } + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + for slot in start..self.queues.render.entries.len() { + let Some(entry) = self.queues.render.entries[slot].as_mut() else { + continue; + }; + if entry.preparing.is_some() || entry.growing { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + let qids = pair.qids(); + if !pair.published_in_flight() + || !qids + .into_iter() + .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) + { + continue; + } + let stamps = [ + self.init.stamp(qids[0])?.load(Ordering::Relaxed), + self.init.stamp(qids[1])?.load(Ordering::Relaxed), + ]; + core::sync::atomic::fence(Ordering::Acquire); + let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { + continue; + }; + entry.borrowed_active = pair.published_in_flight(); + entry.borrowed_dependencies = Some(pair.dependency_snapshot()); + entry.preparing = Some(packet); + return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); + } + Ok(None) + } + + fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { + let returning = pair.as_ref().ok_or(EINVAL)?; + let entry = self.queues.render.entry_mut(slot as u8)?; + if entry.pair.is_some() + || entry.preparing.is_none() + || returning.slot() as usize != slot + || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) + { + return Err(EIO); + } + entry.pair = pair.take(); + entry.borrowed_dependencies = None; + entry.preparing = None; + Ok(()) + } + + pub(super) fn settle_render_terminal( + &mut self, + slot: usize, + deferred: &mut DeferredBatch, + ) -> Result { + let terminal = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_terminal()); + if let Some((status, error)) = terminal { + status.record(error); + self.scan_compute(None, deferred)?; + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + Ok(()) + } +} + +impl crate::g17::Shared { + pub(in crate::g17) fn observe_renders( + self: &Arc, + masks: [u64; 2], + deferred: &mut DeferredBatch, + ) -> Result { + { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + firmware.queues.retain_render_completions(masks); + // Returned preparers can have deferred a VM failure or timeout. + // Only this full worker collector runs their final observations. + for slot in 1..firmware.queues.render.entries.len() { + if firmware.queues.render.entries[slot] + .as_ref() + .is_some_and(|entry| { + entry.pair.is_some() + && (entry.pending_error.is_some() + || entry.pending_timeout + || entry.closed) + }) + { + firmware.apply_render_deferred(slot as u8, deferred)?; + } + } + } + deferred.finish(); + let mut cursor = 1; + let mut remaining = crate::hw::t8140::queues::RENDER_DEPTH; + let mut retries = 0; + loop { + let epoch = match self.preparations.try_enter() { + Err(error) if error == EAGAIN || error == ENODEV => break, + result => result?, + }; + let loan = { + let mut state = self.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { + break; + } + firmware.take_render_observation(cursor)? + }; + let Some((slot, pair)) = loan else { + break; + }; + if slot != cursor { + remaining = crate::hw::t8140::queues::RENDER_DEPTH; + retries = 0; + } + let observed = pair.observe_front(); + let mut pair = Some(pair); + let mut state = self.state.lock(); + // Return installed ownership before examining read errors or epoch. + let restored = (*state) + .as_deref_mut() + .ok_or(ENODEV) + .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); + if let Err(error) = restored { + // A failed invariant must not drop firmware-owned storage. Join + // the other preparers only after releasing our own epoch, and + // keep this detached owner until both processors have stopped. + drop(state); + drop(epoch); + self.lose_device_quietly(); + self.stop_queues(); + if let Some(mut pair) = pair { + pair.stopped(&mut |event| { + event.finish(); + Ok(()) + })?; + } + return Err(error); + } + let firmware = (*state).as_deref_mut().ok_or(EIO)?; + self.changed.notify_all(); + if firmware.render_grow_pending() { + self.queue_events(); + } + if !epoch.is_current() + || firmware.recovery.pending() + || self.crashed.load(Ordering::Acquire) + { + break; + } + let result = (|| { + let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; + let progressed = match observed { + Ok(observed) => firmware.with_render(slot, |pair, host| { + pair.retire_observed(observed, state, host, &mut |event| { + deferred.push(event) + }) + })?, + Err(error) if error == EAGAIN => false, + Err(error) => return Err(error), + }; + if progressed { + match firmware.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + firmware.queues.render.collect_retired(slot)?; + // An observation of the old front is committed before a queued + // exact timeout is permitted to advance that same tracker. + firmware.apply_render_deferred(slot as u8, deferred)?; + firmware.settle_render_terminal(slot, deferred)?; + if firmware + .queues + .render + .entry(slot as u8)? + .pair + .as_ref() + .is_some_and(|pair| !pair.in_flight()) + { + firmware.queues.render.entry_mut(slot as u8)?.event_masks = [0; 2]; + } + Ok(progressed) + })(); + drop(state); + drop(epoch); + deferred.finish(); + if result? { + remaining -= 1; + retries = 0; + cursor = if remaining == 0 { slot + 1 } else { slot }; + } else { + retries += 1; + cursor = if retries >= 2 { slot + 1 } else { slot }; + } + if cursor != slot { + remaining = crate::hw::t8140::queues::RENDER_DEPTH; + retries = 0; + } + } + { + let state = self.state.lock(); + if (*state).as_deref().is_some_and(|firmware| { + !firmware.recovery.pending() && firmware.queues.render_needs_rescan() + }) { + self.queue_poll(); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs new file mode 100644 index 00000000000000..7047992754b377 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Device-wide references to render free lists. Two physical pairs bound to +//! one logical context share one registration and one final release witness. +//! The device mutex serializes every transition and descriptor-table access. + +use crate::g17::{ + channel::Rings, + freelist::{Lease, Owner, RenderPool}, + fw::{ + channels::{ControlRecord, FreeListCompletion}, + initdata::PagePoolDescriptor, + }, + initdata::InitData, + object::KernelObject, +}; +use core::sync::atomic::{fence, AtomicU64, Ordering}; +use kernel::prelude::*; + +const POOLS: usize = 256; +const FIRST_RENDER_POOL: usize = 129; + +pub(super) struct Pools { + leases: KVec>, +} + +/// Undo state cannot leave the device-locked publication transaction. Rollback +/// is permitted only before the packet exposes its first firmware reference. +pub(super) struct Preparation { + previous: Option, + slot: usize, + pub(super) generation: u64, +} + +impl Pools { + pub(super) fn new() -> Result { + let mut leases = KVec::with_capacity(POOLS, GFP_KERNEL)?; + for _ in 0..POOLS { + leases.push(None, GFP_KERNEL)?; + } + Ok(Self { leases }) + } + + fn slot(id: u16) -> Result { + let slot = usize::from(id); + if !(FIRST_RENDER_POOL..POOLS).contains(&slot) { + return Err(EINVAL); + } + Ok(slot) + } + + fn owner(pool: &RenderPool) -> Owner { + Owner { + serial: pool.serial(), + buffer_slot: u32::from(pool.id()), + control_va: pool.control_va(), + ring_va: pool.run_list_va(), + cookie: pool.end_va(), + } + } + + pub(super) fn released(&self, id: u32, _control: u64) -> bool { + self.leases + .get(id as usize) + .is_none_or(|lease| lease.as_ref().is_none_or(|lease| lease.released())) + } + + /// Install both engine references before either inner queue becomes + /// visible. Warm overlap keeps the existing generation and pool contents. + pub(super) fn prepare(&mut self, init: &InitData, pool: &RenderPool) -> Result { + self.observe(init)?; + let slot = Self::slot(pool.id())?; + let owner = Self::owner(pool); + let previous = self.leases[slot]; + if previous.is_some_and(|lease| lease.owner() != owner && !lease.released()) { + return Err(EBUSY); + } + // Pools arrive fully populated. There is no asynchronous initial grow. + let next = Lease::acquire(previous.as_ref(), owner)?; + if previous.is_none_or(|lease| lease.owner() != owner) { + self.stage(init, pool)?; + } + let generation = next.generation(); + self.leases[slot] = Some(next); + Ok(Preparation { + previous, + slot, + generation, + }) + } + + /// Warming validates the incoming owner and stages its descriptor without + /// acquiring command references or publishing an asynchronous request. + pub(super) fn warm(&self, init: &InitData, pool: &RenderPool) -> Result { + let previous = self.leases[Self::slot(pool.id())?]; + let owner = Self::owner(pool); + if previous.is_some_and(|lease| lease.owner() == owner) { + return Ok(()); + } + if previous.is_some_and(|lease| !lease.released()) { + return Err(EBUSY); + } + Lease::acquire(previous.as_ref(), owner)?; + self.stage(init, pool) + } + + fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { + let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + pool.repopulate()?; + let descriptor = pool + .page_pool_state()? + .descriptor(words[3].load(Ordering::Relaxed)); + for (word, value) in words.into_iter().zip([ + descriptor.page_list, + descriptor.counters, + descriptor.page_count, + descriptor.unk_18, + ]) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Retire preparation only after both deferred outer bodies and their + /// inner commands exist, before publishing either outer producer. + pub(super) fn activate(&mut self, id: u16, generation: u64) -> Result { + let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; + if lease.generation() != generation { + return Err(EIO); + } + lease.retire_prepare() + } + + pub(super) fn rollback(&mut self, preparation: Preparation) { + self.leases[preparation.slot] = preparation.previous; + } + + /// Preflight a paired retirement before QoS or PB reference accounting. + pub(super) fn completed(&self, id: u16, generation: u64) -> Result { + let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; + if next.generation() != generation { + return Err(EIO); + } + next.complete_render()?; + Ok(next) + } + + /// The matching completed() result and other accounting preflights were + /// accepted under the same device mutex, so this assignment cannot fail. + pub(super) fn commit_completed(&mut self, id: u16, next: Lease) { + self.leases[usize::from(id)] = Some(next); + } + + pub(super) fn pending(&self) -> bool { + self.leases + .iter() + .flatten() + .any(|lease| lease.pending_release().is_some()) + } + + pub(super) fn complete_grow(&mut self, event: FreeListCompletion) -> Result { + let id = u16::try_from(event.buffer_slot).map_err(|_| EINVAL)?; + let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; + lease.complete_grow(event) + } + + pub(super) fn observe(&mut self, init: &InitData) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + for lease in self.leases.iter_mut().flatten() { + lease.observe_releases(consumer, producer)?; + } + Ok(()) + } + + /// Store the release witness immediately after the ring producer. A failed + /// mailbox announcement must never cause the same release to be replayed. + pub(super) fn publish_releases( + &mut self, + init: &InitData, + mut announce: impl FnMut() -> Result, + ) -> Result { + for lease in self.leases.iter_mut().flatten() { + let Some(release) = lease.pending_release() else { + continue; + }; + let cursor = Rings::publish_control(init, &ControlRecord::FreeListRelease(release))?; + lease.mark_release_sent(cursor)?; + fence(Ordering::SeqCst); + announce()?; + } + self.observe(init) + } + + /// A detached pool may capture firmware counters only after its last pair + /// stopped using this registration. Callers check all retained pair owners. + pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { + let state = pool.page_pool_state()?; + let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; + fence(Ordering::Acquire); + let descriptor = PagePoolDescriptor { + page_list: words[0].load(Ordering::Relaxed), + counters: words[1].load(Ordering::Relaxed), + page_count: words[2].load(Ordering::Relaxed), + unk_18: words[3].load(Ordering::Relaxed), + }; + if descriptor.page_list != state.descriptor(descriptor.unk_18).page_list { + return Err(EIO); + } + pool.store_page_pool_state(state.updated(&descriptor)) + } +} + +fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { + let offset = Pools::slot(id)? * size_of::(); + Ok([ + table.dword(offset)?, + table.dword(offset + 8)?, + table.dword(offset + 16)?, + table.dword(offset + 24)?, + ]) +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs new file mode 100644 index 00000000000000..30ea174f611b18 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Off-lock render graph construction and command preparation. Every borrowed +//! installed pair returns to its device owner before its preparation lease ends. + +use super::{Config, Reservation}; +use crate::g17::{ + buffer::MetricsLease, + job::Packet, + object::{Allocator, KernelObject, Pool}, + queue::render::{Pair, RenderAddresses}, +}; +use kernel::{prelude::*, sync::Arc}; + +struct BuildResources { + uat: Arc, + objects: Arc, + ids: Arc, + global: crate::mmu::Vm, + metrics: Arc, + metrics_ids: Arc, + metrics_bases: [u64; 2], + config: Config, +} +impl BuildResources { + fn new(firmware: &crate::g17::Firmware) -> Result { + Ok(Self { + uat: firmware.uat.clone(), + objects: firmware.pool.clone(), + ids: firmware.render_ids.clone(), + global: firmware._render_global.clone(), + metrics: firmware._pm_metrics.clone(), + metrics_ids: firmware.queues.metrics.clone(), + metrics_bases: [ + crate::hw::t8140::CONFIG.pm_metrics.va, + firmware.init.metrics_fw_va()?, + ], + config: firmware.queues.render.config, + }) + } + fn metrics(&self) -> Result { + MetricsLease::new(&self.metrics_ids, &self.metrics, self.metrics_bases) + } +} + +enum Build { + Fresh(Reservation, u64, MetricsLease), + Parked { + parked: crate::g17::queue::render::parked::ParkedPair, + buffer: (u8, u64), + pm_generation: u64, + metrics: MetricsLease, + }, + Rebind { + pair: KBox, + buffer: Option<(u8, u64)>, + pm_generation: Option, + metrics: Option, + }, +} + +impl super::super::Backend { + fn build_render_pair(&self, slot: u8, first: &Arc) -> Result { + let lease = self.shared.preparations.enter(false)?; + let (resources, build) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + firmware.queues.ensure_render_ids(slot, self.owner)?; + let resources = BuildResources::new(firmware)?; + let build = + if let Some(entry) = firmware.queues.render.entries[usize::from(slot)].as_ref() { + if entry.pending_error.is_some() || entry.pending_timeout { + self.shared.queue_events(); + return Err(EBUSY); + } + if entry.constructing || entry.preparing.is_some() || entry.growing { + return Err(EBUSY); + } + if entry.parked.is_some() { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + let buffer = firmware.queues.buffers.allocate_render()?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = Some(first.clone()); + entry.pending_owner = Some(self.owner); + entry.borrowed_active = false; + Build::Parked { + parked: entry.parked.take().ok_or(EIO)?, + buffer, + pm_generation, + metrics, + } + } else if entry.pair.is_none() { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + Build::Fresh( + firmware.queues.reserve_render(slot, self.owner)?, + pm_generation, + metrics, + ) + } else { + let pair = entry.pair.as_ref().ok_or(EBUSY)?; + if pair.matches(self.owner, &self.context)? { + return Ok(()); + } + if !pair.reusable()? { + return Err(EBUSY); + } + let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); + if foreign { + firmware + .queues + .render + .save_detaching_pool(&firmware.init, slot)?; + } + let metrics = if foreign { + Some(resources.metrics()?) + } else { + None + }; + let pm_generation = if foreign { + Some(firmware.queues.render.reserve_pm_generation()?) + } else { + None + }; + let buffer = if foreign { + Some(firmware.queues.buffers.allocate_render()?) + } else { + None + }; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = Some(first.clone()); + entry.pending_owner = Some(self.owner); + entry.borrowed_active = entry + .pair + .as_ref() + .is_some_and(|pair| pair.published_in_flight()); + entry.borrowed_dependencies = + entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); + Build::Rebind { + pair: entry.pair.take().ok_or(EIO)?, + buffer, + pm_generation, + metrics, + } + } + } else { + let metrics = resources.metrics()?; + let pm_generation = firmware.queues.render.reserve_pm_generation()?; + Build::Fresh( + firmware.queues.reserve_render(slot, self.owner)?, + pm_generation, + metrics, + ) + }; + (resources, build) + }; + let alloc = Allocator { + dev: &self.shared.dev, + uat: &resources.uat, + }; + let fresh_graph = match &build { + Build::Fresh(..) | Build::Parked { .. } => true, + Build::Rebind { buffer, .. } => buffer.is_some(), + }; + let result = match build { + Build::Fresh(reservation, pm_generation, metrics) => { + let built = (|| -> Result> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + Pair::new( + &alloc, + &resources.objects, + reservation.ids, + slot, + self.owner, + self.context.clone(), + free_list, + metrics, + reservation.buffer, + pm_generation, + first, + resources.config.clusters, + resources.config.descriptor_flags, + ) + })(); + let (mut built, error) = match built { + Ok(pair) => (Some(pair), None), + Err(error) => (None, Some(error)), + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + if firmware.queues.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + built + .as_mut() + .ok_or(EIO)? + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + firmware.queues.install_render(reservation, &mut built) + })(); + // Fresh backing disappears before its temporary PB reference returns. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .cancel_render_reservation(reservation)?; + } + result + } + Build::Parked { + mut parked, + buffer, + pm_generation, + metrics, + } => { + let built = (|| -> Result<_> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + let prepared = parked.prepare( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + )?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + Ok((prepared, storage)) + })(); + let (mut prepared, mut storage, error) = match built { + Ok((prepared, storage)) => (Some(prepared), Some(storage), None), + Err(error) => (None, None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) + || entry.pending_owner_closed + || entry.pair.is_some() + || entry.parked.is_some() + { + return Err(ECANCELED); + } + let storage = storage.take().ok_or(EIO)?; + firmware.queues.buffers.can_release(buffer.0)?; + prepared.as_mut().ok_or(EIO)?.finish_install( + &parked, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + let pair = parked.commit(&mut prepared, storage)?; + firmware.queues.buffers.reserve_owner(buffer.0)?; + firmware.queues.buffers.release(buffer.0)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + entry.reservation.buffer = buffer; + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.pair = Some(pair); + entry.closed = false; + entry.published = false; + Ok(()) + })() + }; + // Rejected candidate mappings disappear before the PB identity + // returns, while the original installed kick owner stays live. + drop(prepared); + drop(storage); + { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if result.is_err() { + firmware.queues.render.entry_mut(slot)?.parked = Some(parked); + firmware.queues.buffers.release(buffer.0)?; + } else { + drop(state); + drop(parked); + state = self.shared.state.lock(); + } + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + } + self.shared.changed.notify_all(); + result + } + Build::Rebind { + mut pair, + buffer, + pm_generation, + metrics, + } => { + let prepared = (|| { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + pair.prepare_bind( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + ) + })(); + let (mut prepared, error) = match prepared { + Ok(binding) => (Some(binding), None), + Err(error) => (None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let result = (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { + return Err(ECANCELED); + } + if firmware.queues.render.retired.len() + == firmware.queues.render.retired.capacity() + { + return Err(ENOSPC); + } + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + pair.bind_owner( + prepared.as_mut().ok_or(EIO)?, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + if let Some((id, _)) = buffer { + firmware.queues.buffers.reserve_owner(id)?; + firmware.queues.buffers.release(id)?; + } + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + if let Some(buffer) = buffer { + entry.reservation.buffer = buffer; + } + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.closed = false; + Ok(()) + })(); + // The original installed pair remains owned on every error. + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + entry.pair = Some(pair); + entry.borrowed_dependencies = None; + result + }; + drop(prepared); + if result.is_err() { + if let Some((id, _)) = buffer { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .buffers + .release(id)?; + } + } + self.shared.changed.notify_all(); + result + } + }; + result?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if firmware.queues.render.entry(slot)?.closed { + return Err(ECANCELED); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + // From this point every error leaves the complete graph retained for + // retry. Physical queue identities and installed mappings stay owned. + if fresh_graph { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .rebase_fresh_graph()?; + } + let pool = firmware + .queues + .render + .entry(slot)? + .pool + .clone() + .ok_or(EIO)?; + firmware.queues.render.pools.warm(&firmware.init, &pool) + } + + pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { + self.build_render_pair(slot, packet)?; + let sibling = { + let state = self.shared.state.lock(); + (*state) + .as_deref() + .ok_or(ENODEV)? + .queues + .admission + .warmable_sibling(self.owner, slot) + }; + if let Some(sibling) = sibling { + self.build_render_pair(sibling, packet)?; + } + Ok(()) + } + + pub(super) fn prepare_render( + &self, + slot: u8, + packet: &Arc, + deferred: &mut super::super::DeferredBatch, + ) -> Result { + let lease = self.shared.preparations.enter(false)?; + let (uat, mut pair, addresses, growth_target) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + let qids = firmware + .queues + .render + .entry(slot)? + .pair + .as_ref() + .ok_or(EBUSY)? + .qids(); + let addresses = RenderAddresses { + stamps: [ + firmware.init.stamp_addresses(qids[0])?, + firmware.init.stamp_addresses(qids[1])?, + ], + }; + let pair = firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)?; + if !pair.matches(self.owner, &packet.context)? { + return Err(EIO); + } + // The selected scheduler is retained through release acknowledgement + // even if later private command allocation fails. Sibling warming + // does not cross this publication boundary. + packet.context.mark_published(); + let target = pair.host_growth_target(packet)?; + let buffer_id = pair.buffer_id(); + let growth_target = if let Some(target) = target { + let table = firmware.init.pb_descriptor_table()?; + let offset = usize::from(buffer_id) * 16; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let mut descriptor = [0; 4]; + for (index, word) in descriptor.iter_mut().enumerate() { + *word = table + .word(offset + index * 4)? + .load(core::sync::atomic::Ordering::Relaxed); + } + Some((target, descriptor)) + } else { + None + }; + let pair = firmware.queues.render.take_pair(slot, self.owner, packet)?; + (firmware.uat.clone(), pair, addresses, growth_target) + }; + let alloc = Allocator { + dev: &self.shared.dev, + uat: &uat, + }; + let mut growth = None; + let result = (|| { + if let Some((target, descriptor)) = growth_target { + let vm = pair.context().vm().clone(); + growth = pair.manager().prepare_growth( + &self.shared.dev, + &vm, + target, + Some(descriptor), + )?; + } + pair.prepare(&alloc, packet, addresses) + })(); + let mut returned = Some(pair); + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + firmware.queues.render.return_pair(slot, &mut returned)?; + if firmware.queues.render.entry(slot)?.closed { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .release_owner(); + } + self.shared.changed.notify_all(); + let published = (|| { + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_error.is_some() || entry.pending_timeout { + // Final timeout observations can settle unrelated queues. The + // event worker owns the full collector needed for that sweep. + self.shared.queue_events(); + return Err(EBUSY); + } + if !lease.is_current() + || firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + { + return Err(EAGAIN); + } + // Private restaging can fail after preparing a larger inventory. + // Finish its descriptor before discarding the command preparation. + firmware.commit_render_growth(slot, &mut growth)?; + result?; + if packet.cancel_requested() || firmware.queues.render.entry(slot)?.closed { + return Err(ECANCELED); + } + firmware.publish_render(self.owner, slot, packet, deferred) + })(); + if published.is_err() && !packet.is_published() { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + published + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs new file mode 100644 index 00000000000000..3a5b042fff6834 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/recovery.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Recovery targets for retained render graphs while firmware is halted. + +use crate::g17::runtime::Registry; +use kernel::prelude::*; + +impl Registry { + pub(in crate::g17::runtime) fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { + self.render.entries.iter().flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid))? + .pass_started(stamp) + } + + pub(in crate::g17::runtime) fn publish_render_recovery_targets(&self) -> Option { + let mut failure = None; + for pair in self.render.entries.iter().flatten() + .filter_map(|entry| entry.pair.as_ref()) + { + if let Err(error) = pair.publish_recovery_targets() { + failure = failure.or(Some(error)); + } + } + failure + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs new file mode 100644 index 00000000000000..a6d5821561d244 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact packet timeout and paired firmware-drain settlement. Every fence and +//! admission release is collected under the device lock and finished outside it. + +use crate::g17::{ + job::Packet, + runtime::{Backend, DeferredBatch}, +}; +use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; + +impl Backend { + pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { + let mut deferred = DeferredBatch::worker()?; + let result = { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(()); + }; + if firmware.queues.defer_render_timeout(packet, ETIMEDOUT) { + return Ok(()); + } + firmware.fail_render_packet(packet, ETIMEDOUT, &mut deferred) + }; + deferred.finish(); + result + } + pub(in crate::g17) fn cancel_render(&self, packet: &Arc) -> Result { + let fence = packet.completion.fence(); + // SAFETY: The retained fence remains alive across this bounded, + // uninterruptible queue-teardown wait, without holding a device lock. + let completed = unsafe { + kernel::bindings::dma_fence_wait_timeout( + fence.raw(), + false, + kernel::time::msecs_to_jiffies(2000) as _, + ) + }; + if completed > 0 { + return Ok(()); + } + if packet.is_published() { + self.timeout_render(packet) + } else { + // The outer callback already completed a successful 0 -> 3 + // cancellation. A claimed publisher still owns its VM pin and + // observes cancel_requested before committing or finishing. + Ok(()) + } + } +} + +impl crate::g17::Firmware { + pub(super) fn fail_render_packet( + &mut self, + packet: &Arc, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + let Some(slot) = packet.completion.render_slot() else { + return Ok(()); + }; + let slot = usize::from(slot); + loop { + let pair = self + .queues + .render + .entries + .get(slot) + .and_then(Option::as_ref) + .and_then(|entry| entry.pair.as_ref()) + .ok_or(EIO)?; + if !pair.contains_packet(packet) { + return Ok(()); + } + let generation = self.recovery.generation(); + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + let progressed = self.with_render(slot, |pair, host| { + pair.retire(None, generation, state, host, &mut |event| { + deferred.push(event) + }) + })?; + if !progressed { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + self.with_render(slot, |pair, host| { + pair.fail_after_observation(packet, error, host, &mut |event| deferred.push(event)) + })?; + self.queues.render.collect_retired(slot)?; + let terminal = self.queues.render.entries[slot] + .as_mut() + .and_then(|entry| entry.pair.as_mut()) + .and_then(|pair| pair.take_terminal()); + if let Some((status, error)) = terminal { + status.record(error); + self.scan_compute(None, deferred)?; + self.settle_render_vm(&status, error, deferred)?; + self.queues.fail_compute_vm(&status, error, deferred)?; + } + Ok(()) + } + + pub(in crate::g17) fn settle_blamed_render( + &mut self, + qid: u8, + error: Error, + deferred: &mut DeferredBatch, + ) -> Result { + let packet = self + .queues + .render + .entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .find(|pair| pair.qids().contains(&qid)) + .and_then(|pair| pair.oldest_packet()); + if let Some(packet) = packet { + self.fail_render_packet(&packet, error, deferred)?; + } + Ok(()) + } + + pub(in crate::g17) fn settle_cancelled_renders( + &mut self, + mask: u128, + deferred: &mut DeferredBatch, + ) -> Result { + for slot in 1..self.queues.render.entries.len() { + let packet = self.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .filter(|pair| { + pair.qids() + .into_iter() + .any(|qid| mask & (1u128 << qid) != 0) + && pair.oldest_spared() == Some(true) + }) + .and_then(|pair| pair.oldest_packet()); + if let Some(packet) = packet { + self.fail_render_packet(&packet, ECANCELED, deferred)?; + } + } + Ok(()) + } + + pub(in crate::g17) fn reclaim_drained_renders( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result { + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + let mut serviced = false; + for slot in 1..self.queues.render.entries.len() { + if !self.queues.render.entries[slot] + .as_ref() + .and_then(|entry| entry.pair.as_ref()) + .is_some_and(|pair| pair.needs_drain()) + { + continue; + } + if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { + dev_warn!( + self.primary.state.shared.dev.as_ref(), + "Could not settle drained render pair {}: {:?}\n", + slot, + error + ); + } + } + Ok(()) + } + + fn drain_render_slot( + &mut self, + slot: usize, + serviced: &mut bool, + deferred: &mut DeferredBatch, + ) -> Result { + if !self.with_render(slot, |pair, host| pair.firmware_drained(host))? { + return Ok(()); + } + if !*serviced { + self.service_runtime_events(deferred)?; + *serviced = true; + } + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + loop { + let accounted = self.with_render(slot, |pair, host| pair.account_drained(host))?; + if !accounted { + break; + } + match self.release_render_lists() { + Err(error) if error == EAGAIN => {} + result => result?, + } + } + if self.recovery.pending() + || self + .primary + .state + .shared + .crashed + .load(core::sync::atomic::Ordering::Acquire) + { + return Ok(()); + } + self.with_render(slot, |pair, host| { + pair.release_witnessed(host, &mut |event| deferred.push(event)) + })?; + self.queues.render.collect_retired(slot) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs new file mode 100644 index 00000000000000..86a00e7bd741a5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Submission admission, preparation and the firmware visibility boundary. + +use super::super::{Backend, DeferredBatch, Registry}; +use crate::g17::{ + job::Packet, + queue::admission::{RenderLease, Request}, +}; +use core::sync::atomic::Ordering; +use kernel::{ + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta}, +}; + +impl Backend { + pub(in crate::g17) fn render_packet( + &self, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + self.render_with_lease(packet, None, deferred) + } + + fn render_with_lease( + &self, + packet: &Arc, + early: Option, + deferred: &mut DeferredBatch, + ) -> Result { + packet.check_dependencies()?; + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let interruptible = early.is_none(); + let admission = match early { + Some(lease) => lease, + None => { + let pool = { + let state = self.shared.state.lock(); + (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() + }; + pool.acquire_render(self.owner, Request::of(packet))? + } + }; + let slot = admission.slot(); + packet.completion.attach_render_lease(admission)?; + let mut state_polls = 0; + let mut state_delay = 20i64; + let mut takeover_polls = 0; + let mut recovery_polls = 0; + loop { + if packet.cancel_requested() { + return Err(ECANCELED); + } + if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) + { + return Err(EIO); + } + if !self.context.is_current() { + return Err(EFAULT); + } + let work_state = { + let preparation = self.shared.preparations.enter(interruptible)?; + let result = packet.completion.work_state().acquire(); + if preparation.is_current() { + result + } else { + Err(EAGAIN) + } + }; + match work_state { + Err(error) if error == EAGAIN => continue, + Err(error) if error == EBUSY => { + if state_polls == 1000 { + return Err(ETIMEDOUT); + } + state_polls += 1; + fsleep(Delta::from_micros(state_delay)); + state_delay = (state_delay * 2).min(1000); + continue; + } + result => { + result?; + } + } + match self.ensure_render(packet, slot) { + Err(error) if error == EAGAIN => continue, + Err(error) if error == EBUSY => { + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); + continue; + } + result => result?, + } + match self.prepare_render(slot, packet, deferred) { + Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { + let recovering = { + let state = self.shared.state.lock(); + let firmware = (*state).as_deref().ok_or(ENODEV)?; + firmware.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 + }; + if recovering { + if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { + return Err(ETIMEDOUT); + } + recovery_polls += 1; + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } else if error == EBUSY { + if takeover_polls == 5000 { + return Err(ETIMEDOUT); + } + takeover_polls += 1; + fsleep(Delta::from_micros(200)); + } + } + result => return result, + } + } + } + + pub(in crate::g17) fn try_render_early(&self, packet: &Arc) -> bool { + if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { + return false; + } + let admission = { + let state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref() else { + return false; + }; + match firmware + .queues + .admission + .try_acquire_render(self.owner, Request::of(packet)) + { + Ok(Some(lease)) => lease, + _ => return false, + } + }; + if !packet.begin_publication() { + packet.completion.fail_unpublished(ECANCELED); + packet.set_early_result(Err(ECANCELED)); + return true; + } + let mut deferred = match DeferredBatch::publication() { + Ok(deferred) => deferred, + Err(error) => { + if packet.completion.status().get() != 0 { + ::fail_vm(self, error); + } + packet.completion.fail_unpublished(error); + packet.set_early_result(Err(error)); + return true; + } + }; + let result = self.render_with_lease(packet, Some(admission), &mut deferred); + let result = self + .finish_publication(packet, result, &mut deferred) + .map(|_| ()); + packet.set_early_result(result); + true + } +} + +impl Registry { + fn render_dependencies( + &self, + owner: u64, + packet: &Arc, + state: u32, + dependencies: &mut [crate::g17::fw::kick::KickDependency; 3], + ) -> Result { + let mut count = 0; + let frontier = self.render_prefix(owner, packet)?; + for dependency in frontier.as_slice() { + *dependencies.get_mut(count).ok_or(EOVERFLOW)? = *dependency; + count += 1; + } + if let Some(queue) = self.compute(owner) { + if let Some(dependency) = queue.dependency( + owner, + packet.completion.status(), + packet.order.wait_through[1], + state, + )? { + *dependencies.get_mut(count).ok_or(EOVERFLOW)? = dependency; + count += 1; + } + } + Ok(count) + } +} + +impl crate::g17::Firmware { + pub(super) fn apply_render_deferred( + &mut self, + slot: u8, + deferred: &mut DeferredBatch, + ) -> Result { + let entry = self.queues.render.entry_mut(slot)?; + let pair = entry.pair.as_mut().ok_or(EIO)?; + if let Some((status, error)) = entry.pending_error.take() { + if Arc::ptr_eq(pair.context().status(), &status) { + pair.quarantine(error, &mut |event| deferred.push(event))?; + } + } + if entry.closed { + pair.release_owner(); + } + if !entry.pending_timeout { + return Ok(()); + } + for _ in 0..crate::hw::t8140::queues::RENDER_DEPTH { + if self.recovery.pending() || self.primary.state.shared.crashed.load(Ordering::Acquire) + { + return Ok(()); + } + let Some((packet, error)) = self + .queues + .render + .entry(slot)? + .pair + .as_ref() + .ok_or(EIO)? + .take_deferred_timeout() + else { + self.queues.render.entry_mut(slot)?.pending_timeout = false; + return Ok(()); + }; + if let Err(failure) = self.fail_render_packet(&packet, error, deferred) { + packet.completion.defer_timeout(error); + return Err(failure); + } + } + self.queues.render.entry_mut(slot)?.pending_timeout = false; + Ok(()) + } + + pub(super) fn publish_render( + &mut self, + owner: u64, + slot: u8, + packet: &Arc, + deferred: &mut DeferredBatch, + ) -> Result { + use crate::g17::{fw::kick::KickDependency, queue::render::PublicationLease}; + let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; + if self.recovery.pending() || state != 0 { + return Err(EAGAIN); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let mut dependencies = [KickDependency::ZERO; 3]; + let count = self + .queues + .render_dependencies(owner, packet, state, &mut dependencies)?; + let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; + let pool = pair.pool().clone(); + let buffer_id = pair.buffer_id(); + let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; + let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { + Ok(preparation) => preparation, + Err(error) => { + self.queues.buffers.release(buffer_id)?; + return Err(error); + } + }; + let lease = PublicationLease { + free_list_generation: preparation.generation, + buffer_token, + }; + let result = self.with_render(usize::from(slot), |pair, host| { + pair.publish( + packet.clone(), + &dependencies[..count], + lease, + host, + &mut |event| deferred.push(event), + ) + }); + if packet.is_published() { + self.queues.render.entry_mut(slot)?.published = true; + } else if result.is_err() { + self.queues.render.pools.rollback(preparation); + self.queues.buffers.release(buffer_id)?; + self.queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + result + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs new file mode 100644 index 00000000000000..759c0c2e43dc0c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/teardown.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact logical ownership gates for scheduler release and physical-pair parking. + +use super::RenderState; +use crate::g17::{ + buffer::BufferIds, + context::Context, + initdata::InitData, + runtime::{teardown::ReclaimBatch, Registry}, +}; +use kernel::{prelude::*, sync::Arc}; + +impl RenderState { + pub(in crate::g17::runtime) fn observe_teardown_releases(&mut self, init: &InitData) -> Result { + self.pools.observe(init) + } + + pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { + for entry in self.entries.iter().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + if entry.preparing.is_some() || entry.growing { + return Ok(false); + } + if let Some(pair) = entry.pair.as_ref() { + if !pair.idle_for_context(context)? { + return Ok(false); + } + } + } + Ok(true) + } + + pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { + context.render_pools_released(|id, control| self.pools.released(id, control)) + && self + .retired + .iter() + .filter(|old| Arc::ptr_eq(old.context(), context)) + .all(|old| { + self.pools + .released(u32::from(old.pool().id()), old.pool().control_va()) + }) + } + + /// Warm superseded roots need only their exact executed replacement witness. + /// Graphs with their own memory remain until that Context's scheduler ack. + pub(in crate::g17::runtime) fn collect_reclaimable( + &mut self, + out: &mut ReclaimBatch, + ) -> Result { + for slot in 0..self.entries.len() { + self.collect_retired(slot)?; + } + let mut index = 0; + while index < self.retired.len() { + if self.retired[index].buffer_id().is_some() { + index += 1; + continue; + } + let Ok(target) = out.binding_slot() else { + break; + }; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + } + Ok(()) + } + + /// Preflight the entire Context before detaching its first graph. Successor + /// graphs on the same VM have a different Arc and remain installed. + pub(in crate::g17::runtime) fn detach_context( + &mut self, + context: &Arc, + buffers: &mut BufferIds, + out: &mut ReclaimBatch, + ) -> Result { + let mut count = self + .retired + .iter() + .filter(|old| Arc::ptr_eq(old.context(), context)) + .count(); + for entry in self.entries.iter().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + if entry.preparing.is_some() || entry.growing || entry.parked.is_some() { + return Err(EBUSY); + } + if let Some(pair) = entry.pair.as_ref() { + if !pair.can_park(context)? { + return Err(EBUSY); + } + count += 1; + } + } + if !out.has_room(count, 1) { + return Err(EBUSY); + } + for entry in self.entries.iter_mut().flatten() { + if !entry + .context + .as_ref() + .is_some_and(|owner| Arc::ptr_eq(owner, context)) + { + continue; + } + let Some(pair) = entry.pair.as_ref() else { + continue; + }; + buffers.unreserve_owner(pair.buffer_id())?; + let target = out.binding_slot()?; + let pair = entry.pair.take().ok_or(EIO)?; + let (parked, retired) = KBox::into_inner(pair).park(); + *target = Some(retired); + entry.parked = Some(parked); + // Detached backing retains these references until the worker runs. + entry.context = None; + entry.pool = None; + } + let mut index = 0; + while index < self.retired.len() { + if !Arc::ptr_eq(self.retired[index].context(), context) { + index += 1; + continue; + } + if let Some(id) = self.retired[index].buffer_id() { + buffers.unreserve_owner(id)?; + } + let target = out.binding_slot()?; + *target = Some(self.retired.remove(index).map_err(|_| EIO)?); + } + Ok(()) + } +} + +impl Registry { + /// Close while a builder owns a pair records intent. Its return path applies + /// the release before admitting further commands from either logical owner. + pub(in crate::g17::runtime) fn release_render(&mut self, owner: u64) -> Result { + for entry in self.render.entries.iter_mut().flatten() { + if entry.pending_owner == Some(owner) { + entry.pending_owner_closed = true; + } + if entry.reservation.owner != owner { + continue; + } + entry.closed = true; + if let Some(pair) = entry.pair.as_mut() { + pair.release_owner(); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs new file mode 100644 index 00000000000000..10f334e4fe5617 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Scheduler release and off-lock destruction of acknowledged logical queue resources. + +use super::{ComputeEntry, Registry}; +use crate::g17::{ + buffer::BufferIds, + channel::Rings, + context::Context, + fw::channels::{ControlRecord, SchedulerStateRelease}, + initdata::InitData, + queue::render::RetiredBinding, + recovery::Memory as _, + teardown::{Failure, Host}, + Coprocessor, Firmware, MSG_CONTROL_NOTIFY, +}; +use crate::hw::t8140::queues::RENDER_SLOTS; +use core::sync::atomic::Ordering; +use kernel::{prelude::*, sync::Arc}; + +const CONTEXTS: usize = 128; + +/// Pop under the reclaim mutex, then finish after dropping every device lock. +pub(in crate::g17) enum Reclaim { + Binding(RetiredBinding), + Context(Arc), +} + +impl Reclaim { + pub(in crate::g17) fn finish(self) { + match self { + Self::Binding(binding) => drop(binding), + Self::Context(context) => drop(context), + } + } +} + +/// All slots are allocated at probe. A full collector postpones detach. Empty +/// slots let a validated transfer move ownership without a fallible vector push. +pub(in crate::g17) struct ReclaimBatch { + bindings: KVVec>, + contexts: KVec>>, +} + +impl ReclaimBatch { + pub(in crate::g17) fn new() -> Result { + let mut bindings = KVVec::with_capacity(CONTEXTS + RENDER_SLOTS, GFP_KERNEL)?; + for _ in 0..CONTEXTS + RENDER_SLOTS { + bindings.push(None, GFP_KERNEL)?; + } + let mut contexts = KVec::with_capacity(CONTEXTS, GFP_KERNEL)?; + for _ in 0..CONTEXTS { + contexts.push(None, GFP_KERNEL)?; + } + Ok(Self { bindings, contexts }) + } + + pub(in crate::g17) fn is_empty(&self) -> bool { + self.bindings.iter().all(Option::is_none) && self.contexts.iter().all(Option::is_none) + } + + /// Graphs go before the Context owners that armed their scheduler releases. + pub(in crate::g17) fn take_one(&mut self) -> Option { + if let Some(slot) = self.bindings.iter_mut().find(|slot| slot.is_some()) { + return slot.take().map(Reclaim::Binding); + } + self.contexts + .iter_mut() + .find(|slot| slot.is_some()) + .and_then(Option::take) + .map(Reclaim::Context) + } + + pub(in crate::g17::runtime) fn has_room(&self, bindings: usize, contexts: usize) -> bool { + self.bindings.iter().filter(|slot| slot.is_none()).count() >= bindings + && self.contexts.iter().filter(|slot| slot.is_none()).count() >= contexts + } + + pub(in crate::g17::runtime) fn binding_slot(&mut self) -> Result<&mut Option> { + self.bindings + .iter_mut() + .find(|slot| slot.is_none()) + .ok_or(EBUSY) + } + + fn context(&mut self, context: &Arc) -> Result { + let slot = self + .contexts + .iter_mut() + .find(|slot| slot.is_none()) + .ok_or(EBUSY)?; + *slot = Some(context.clone()); + Ok(()) + } +} + +struct Service<'a> { + render: &'a mut super::render::RenderState, + compute: &'a [Option], + buffers: &'a mut BufferIds, + init: &'a InitData, + primary: &'a mut Coprocessor, + reclaim: &'a mut ReclaimBatch, +} + +impl Host for Service<'_> { + fn render_idle(&mut self, context: &Arc) -> Result { + self.render.teardown_idle(context) + } + fn compute_idle(&self, context: &Arc) -> bool { + self.compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_ref()) + .all(|queue| queue.idle_for_context(context)) + } + fn flists_released(&self, context: &Arc) -> bool { + self.render.teardown_pools_released(context) + } + fn control_backpressured(&self) -> bool { + self.render.control_backpressured() + } + fn publish(&mut self, record: &SchedulerStateRelease) -> Result { + Rings::publish_control( + self.init, + &ControlRecord::SchedulerStateRelease(SchedulerStateRelease::new( + record.page_va, + record.id, + )), + ) + } + fn notify(&mut self) -> Result { + self.primary.notify(MSG_CONTROL_NOTIFY) + } + fn detach(&mut self, context: &Arc) -> Result { + self.render + .detach_context(context, self.buffers, self.reclaim)?; + // detach_context preflights this slot together with every graph slot. + // Retain it before Pending removes its reference under the device lock. + self.reclaim.context(context) + } + fn report(&mut self, context: &Arc, failure: Failure) { + let dev = self.primary.state.shared.dev.as_ref(); + match failure { + Failure::Publication(error) => dev_err!( + dev, + "Scheduler {} release failed: {:?}\n", + context.id(), + error + ), + Failure::Announcement(error) => dev_warn!( + dev, + "Scheduler {} release announcement failed: {:?}\n", + context.id(), + error + ), + Failure::Stalled => { + dev_warn!(dev, "Scheduler {} release is still pending\n", context.id()) + } + } + } +} + +impl Firmware { + /// The event worker has already reclaimed drained pairs and reannounced + /// pending controls. Recovery blocks new releases, never an observed ack. + pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { + self.queues.render.collect_reclaimable(reclaim)?; + if self.queues.teardown.is_empty() { + return Ok(()); + } + self.queues.render.observe_teardown_releases(&self.init)?; + let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; + let Registry { + render, + compute, + buffers, + teardown, + .. + } = &mut self.queues; + let mut service = Service { + render, + compute, + buffers, + init: &self.init, + primary: &mut self.primary, + reclaim, + }; + teardown.publish_ready(&mut service, recovery)?; + let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); + let producer = self.init.control_producer()?.load(Ordering::Relaxed); + while teardown + .take_consumed(consumer, producer, &mut service)? + .is_some() + {} + teardown.report_stalled(super::now_ns(), &mut service); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs new file mode 100644 index 00000000000000..178d275da9cf4d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Sticky VM errors and their optional userspace mirror. No firmware state is accessed here. + +use crate::gem; +use core::cell::UnsafeCell; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicPtr, Ordering}; +use kernel::prelude::*; + +const MIRROR_MAX_SIZE: usize = 64 * 1024; + +/// A VM and its accepted jobs share this status, including after queue destruction. +/// The first error is permanent. Callers record it before signalling the job's fence. +pub(crate) struct VmStatus { + error: AtomicI32, + mirror: AtomicPtr, + mirror_claimed: AtomicBool, + mirror_owner: UnsafeCell>, +} + +// SAFETY: Only the winner of mirror_claimed writes mirror_owner. The owner is never +// read or replaced while the status lives, and its mapping is retained until drop. +// Other shared accesses use atomics, including accesses through the mirror pointer. +unsafe impl Sync for VmStatus {} +// SAFETY: The retained GEM reference and mapping can move between threads. Moving +// the status does not move that mapping, and its remaining fields are atomic. +unsafe impl Send for VmStatus {} + +impl VmStatus { + pub(crate) fn new() -> Self { + Self { + error: AtomicI32::new(0), + mirror: AtomicPtr::new(core::ptr::null_mut()), + mirror_claimed: AtomicBool::new(false), + mirror_owner: UnsafeCell::new(None), + } + } + + pub(crate) fn get(&self) -> i32 { + self.error.load(Ordering::Acquire) + } + + /// Store the first error in both the status and any registered mirror. + pub(crate) fn record(&self, error: Error) { + let error = match self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => error.to_errno(), + Err(first) => first, + }; + let word = self.mirror.load(Ordering::SeqCst); + if !word.is_null() { + // SAFETY: attach_mirror publishes only an aligned word within a mapping + // owned by this status. Every kernel access to this word is atomic. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + } + } + + /// Rejected submissions and contention leave the VM usable. + pub(crate) fn record_if_device_loss(&self, error: Error) { + if error == EIO || error == ETIMEDOUT || error == ENODEV { + self.record(error); + } + } + + /// Retain one mapped GEM word until the last reference to this status is dropped. + pub(crate) fn attach_mirror(&self, mut owner: gem::ObjectRef, offset: usize) -> Result { + if owner.size() > MIRROR_MAX_SIZE + || offset % core::mem::align_of::() != 0 + || offset + .checked_add(core::mem::size_of::()) + .is_none_or(|end| end > owner.size()) + { + return Err(EINVAL); + } + let base = owner.vmap()?.as_mut_ptr(); + // SAFETY: The checked range is inside the retained, page-aligned mapping. + let word = unsafe { base.add(offset) }.cast::(); + if self.mirror_claimed.swap(true, Ordering::AcqRel) { + return Err(EBUSY); + } + // SAFETY: This is the only thread that may write mirror_owner. It is not + // read again until the status is dropped, after all shared references end. + unsafe { *self.mirror_owner.get() = Some(owner) }; + // SAFETY: The retained owner keeps this checked, aligned word mapped. + unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); + + // The pointer publication and error update form a store/load handshake: + // either this load sees record's error or record sees our pointer. Stores + // after publishing the pointer write only the same first error, never zero. + self.mirror.store(word, Ordering::SeqCst); + let error = self.error.load(Ordering::SeqCst); + if error != 0 { + // SAFETY: The mapping remains owned by this status as above. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs new file mode 100644 index 00000000000000..9acc01fbc21256 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Retained scheduler-state releases. Logical close arms a release; render and compute drain, +//! private-memory release and control-ring consumption are separate ownership witnesses. +//! The device mutex serializes this state with the Registry's displaced graph ownership. + +use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; +use core::sync::atomic::{fence, Ordering}; +use kernel::{prelude::*, sync::Arc}; + +const CAPACITY: usize = 128; +const POLL_NS: u64 = 2_000_000_000; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Phase { + Deferred, + Draining, + Released(u32), +} + +struct Entry { + context: Arc, + phase: Phase, + armed: u64, + warned: bool, +} + +/// Publication and announcement errors retain the entry. A stalled release is reported once; +/// its timeout never authorizes freeing memory still owned by firmware. +#[derive(Clone, Copy)] +pub(crate) enum Failure { + Publication(Error), + Announcement(Error), + Stalled, +} + +/// Borrowed device operations. Readiness covers every matching installed and displaced graph, +/// including pools kept by the Context after a same-VM successor took over its physical pair. +/// `detach` must retain all detached backing for off-lock destruction before returning success. +/// It compares logical Arc identity so a successor sharing an execution ID remains installed. +pub(crate) trait Host { + fn render_idle(&mut self, context: &Arc) -> Result; + fn compute_idle(&self, context: &Arc) -> bool; + fn flists_released(&self, context: &Arc) -> bool; + fn control_backpressured(&self) -> bool; + fn publish(&mut self, record: &SchedulerStateRelease) -> Result; + fn notify(&mut self) -> Result; + fn detach(&mut self, context: &Arc) -> Result; + fn report(&mut self, context: &Arc, failure: Failure); +} + +pub(crate) struct Pending { + entries: KVec, +} + +impl Pending { + pub(crate) fn new() -> Result { + Ok(Self { + entries: KVec::with_capacity(CAPACITY, GFP_KERNEL)?, + }) + } + + pub(crate) fn closed(&self, index: usize, now: u64) -> Option<(Arc, u64)> { + self.entries + .get(index) + .filter(|entry| entry.phase == Phase::Draining) + .map(|entry| (entry.context.clone(), now.saturating_sub(entry.armed))) + } + + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + fn find(&self, context: &Arc) -> Option { + self.entries + .iter() + .position(|entry| Arc::ptr_eq(&entry.context, context)) + } + + fn insert(&mut self, context: &Arc, phase: Phase, now: u64) -> Result { + self.entries + .push_within_capacity(Entry { + context: context.clone(), + phase, + armed: now, + warned: false, + }) + .map_err(|_| ENOSPC) + } + + /// Call before replacing a graph's logical owner. Failure leaves that graph with its + /// original owner. The Registry, rather than this state machine, retains displaced backing. + pub(crate) fn displace( + &mut self, + context: &Arc, + owner_closed: bool, + now: u64, + ) -> Result { + if self.find(context).is_some() { + return Ok(()); + } + let phase = if owner_closed { + Phase::Draining + } else { + Phase::Deferred + }; + self.insert(context, phase, now) + } + + /// No allocation is needed at close. A scheduler page never exposed to firmware requires + /// no release, unless the Registry already retained one of its displaced graphs. + pub(crate) fn close(&mut self, context: &Arc, now: u64) -> Result { + if let Some(index) = self.find(context) { + let entry = &mut self.entries[index]; + if entry.phase == Phase::Deferred { + entry.phase = Phase::Draining; + entry.armed = now; + } + return Ok(()); + } + if context.is_published() { + self.insert(context, Phase::Draining, now)?; + } + Ok(()) + } + + /// Publish in insertion order. Mandatory control replies and a full control ring stop this + /// pass; an error for one page leaves it retained while other ready releases may progress. + pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool) -> Result { + if !recovery { + for entry in self.entries.iter_mut() { + if entry.phase != Phase::Draining + || !host.render_idle(&entry.context)? + || !host.flists_released(&entry.context) + || !host.compute_idle(&entry.context) + { + continue; + } + if host.control_backpressured() { + break; + } + let result = (|| { + let id = entry.context.release_identity()?; + let record = SchedulerStateRelease::new( + entry.context.scheduler_va(), + [id[3], id[0], id[1], id[2]], + ); + host.publish(&record) + })(); + let next = match result { + Ok(next) => next, + Err(EAGAIN) => break, + Err(error) => { + host.report(&entry.context, Failure::Publication(error)); + continue; + } + }; + entry.phase = Phase::Released(next); + fence(Ordering::SeqCst); + if let Err(error) = host.notify() { + host.report(&entry.context, Failure::Announcement(error)); + } + } + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// The caller snapshots both control cursors once for this settlement pass and repeatedly + /// takes consumed entries. It retains each returned Context until its detached graphs and + /// VM aliases can be destroyed outside the device mutex. + pub(crate) fn take_consumed( + &mut self, + consumer: u32, + producer: u32, + host: &mut impl Host, + ) -> Result>> { + for index in 0..self.entries.len() { + let entry = &self.entries[index]; + let Phase::Released(target) = entry.phase else { + continue; + }; + if !control_consumed(consumer, producer, target)? + || !host.flists_released(&entry.context) + { + continue; + } + match host.detach(&entry.context) { + Ok(()) => {} + Err(EBUSY) => continue, + Err(error) => return Err(error), + } + let entry = self.entries.remove(index).map_err(|_| EIO)?; + return Ok(Some(entry.context)); + } + Ok(None) + } + + /// Poll quickly only during the initial release window; late arrivals still settle when + /// serviced. Time passage never substitutes for a firmware-consumption witness. + pub(crate) fn polling(&self, now: u64) -> bool { + self.entries.iter().any(|entry| { + entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS + }) + } + + pub(crate) fn report_stalled(&mut self, now: u64, host: &mut impl Host) { + for entry in self.entries.iter_mut() { + if entry.phase != Phase::Deferred + && !entry.warned + && now.saturating_sub(entry.armed) >= POLL_NS + { + entry.warned = true; + host.report(&entry.context, Failure::Stalled); + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g17/validation.rs b/drivers/gpu/drm/asahi/g17/validation.rs new file mode 100644 index 00000000000000..da43c10cd3bef5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/validation.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Reuse address-coverage proofs only while the VM's page tables are unchanged. + +use super::command::{Access, AddressSpace, Payload, UscWindow, Validated}; +use crate::mmu; +use core::{cell::RefCell, ops::Range}; +use kernel::prelude::*; + +const MAX_RANGES: usize = 48; +type Request = (u64, u64, u8); + +#[derive(Clone, Copy)] +struct Requests { + ranges: [Request; MAX_RANGES], + count: usize, + overflow: bool, +} + +impl Requests { + const fn new() -> Self { + Self { + ranges: [(0, 0, 0); MAX_RANGES], + count: 0, + overflow: false, + } + } + + fn as_slice(&self) -> &[Request] { + &self.ranges[..self.count] + } + + fn insert(&mut self, request: Request) { + if self.as_slice().contains(&request) { + return; + } + if self.count == MAX_RANGES { + self.overflow = true; + return; + } + self.ranges[self.count] = request; + self.count += 1; + } +} + +/// One logical queue's bounded cache of exact address, size and permission tuples. +/// Mapping mutations make the epoch unavailable until their final page-table write. +pub(crate) struct Proof { + epoch: Option, + requests: Requests, +} + +impl Proof { + pub(crate) const fn new() -> Self { + Self { + epoch: None, + requests: Requests::new(), + } + } + + fn covers(&self, epoch: Option, requests: &Requests) -> bool { + self.epoch.is_some() + && self.epoch == epoch + && requests + .as_slice() + .iter() + .all(|request| self.requests.as_slice().contains(request)) + } + + fn record(&mut self, epoch: Option, requests: Requests) { + // Alternating in-flight buffers must not evict each other's proofs. + if epoch.is_some() && self.epoch == epoch && !requests.overflow { + for &request in requests.as_slice() { + self.requests.insert(request); + } + if !self.requests.overflow { + return; + } + } + // An incomplete union cannot be reused; retain the latest complete set. + self.epoch = if requests.overflow { None } else { epoch }; + self.requests = requests; + } + + /// The VM-job guard must already pin the mappings for this command. Keep the + /// range arrays off the caller's frame before descending into publication. + #[inline(never)] + pub(crate) fn validate( + &mut self, + vm: &mmu::Vm, + range: Range, + window: &UscWindow, + payload: &Payload, + ) -> Result { + let requests = RefCell::new(Requests::new()); + let optimistic = payload.validate(window, &Space::Record(&range, &requests)); + let requests = requests.into_inner(); + if optimistic.is_ok() && !requests.overflow { + if self.covers(vm.mapping_validation_epoch(), &requests) { + return optimistic; + } + let (covered, epoch) = vm.covers_ranges_batch(requests.as_slice()); + if covered { + self.record(epoch, requests); + return optimistic; + } + self.epoch = None; + } + // Preserve per-range validation's failure and concurrent-mutation behavior. + payload.validate(window, &Space::Checked(&range, vm)) + } +} + +enum Space<'a> { + Record(&'a Range, &'a RefCell), + Checked(&'a Range, &'a mmu::Vm), +} + +impl AddressSpace for Space<'_> { + fn covers(&self, address: u64, size: u64, access: Access) -> bool { + let range = match self { + Self::Record(range, _) | Self::Checked(range, _) => range, + }; + let Some(end) = address.checked_add(size) else { + return false; + }; + if size == 0 || address < range.start || end > range.end { + return false; + } + let permissions = match access { + Access::Read => 1, + Access::Write => 2, + Access::ReadWrite => 3, + }; + match self { + Self::Record(_, requests) => { + requests.borrow_mut().insert((address, size, permissions)); + true + } + Self::Checked(_, vm) => { + vm.covers_range(address, size, permissions & 1 != 0, permissions & 2 != 0) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 2b8f312b28d46e..5f7bb3fc35f4b6 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -184,6 +184,20 @@ fn new_kernel_object_with_cpu_mapping( size: usize, map_wc: bool, ) -> Result { +/// Create a new kernel-owned GEM object. +pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { + new_kernel_object_mapped(dev, size, false) +} + +/// Create a new kernel-owned GEM object whose kernel mapping is write-combined. +/// +/// Used for memory that a non-coherent agent reads as soon as the host has stored to it and +/// issued a barrier, without cache maintenance. +pub(crate) fn new_kernel_object_wc(dev: &AsahiDevice, size: usize) -> Result { + new_kernel_object_mapped(dev, size, true) +} + +fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Result { let gem = shmem::Object::::new( dev, checked_object_size(size)?, diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index c0d126e8e1dcab..4a7ebd25568abf 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -568,6 +568,15 @@ pub(crate) trait Gpu: Send + Sync { fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + /// Whether user VMs retain a sticky error and support a userspace mirror. + fn supports_vm_status(&self) -> bool { + false + } + /// Limits for independently executing queues, if supported by this GPU. + fn queue_limits(&self) -> Option { + None + } + /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index dab1274d80dfce..360f4f8844d96b 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -19,6 +19,26 @@ pub(crate) mod t6031; pub(crate) mod t8103; pub(crate) mod t8112; pub(crate) mod t8122; +pub(crate) mod t8140; + +/// GPU generation enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuGen { + G13 = 13, + G14 = 14, +} + +/// GPU variant enumeration. Note: Part of the UABI. +#[derive(Debug, PartialEq, Copy, Clone)] +#[repr(u32)] +pub(crate) enum GpuVariant { + P = 'P' as u32, + G = 'G' as u32, + S = 'S' as u32, + C = 'C' as u32, + D = 'D' as u32, +} /// GPU revision enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs new file mode 100644 index 00000000000000..4a97217a00705d --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t8140 platforms (A18 Pro). + +use crate::g17::{ + Config, + FixedBuffer, + Identity, // +}; +use crate::hw::GpuVariant; + +/// Static configuration of the T8140 GPU. +pub(crate) const CONFIG: Config = Config { + chip_id: CHIP_ID, + identity: Identity { + family: 0xa, + variant: 0, + num_dies: 1, + gpu_variant: GpuVariant::P, + usc_generation: 3, + hal_generation: 200, + }, + uat_ias: 42, + uat_oas: 42, + // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. + secondary_root_offset: 0x40000, + // The coprocessors request their crash log buffers at these addresses, in this order, and + // expect them to be mapped before they start. + crash_buffers: [ + FixedBuffer { + va: 0x01e1_c000, + size: 0x8000, + }, + FixedBuffer { + va: 0x01f5_0000, + size: 0x8000, + }, + ], + // Read by the firmware through the kernel lower root, whose top-level entries it only + // samples when it starts. + pm_metrics: FixedBuffer { + va: 0x10_0008_0000, + size: 0x8000, + }, +}; + +use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x8140; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; + +const fn window( + slot: usize, + phys: u64, + offset: u64, + size: u32, + unk_18: u64, + unk_20: u32, +) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + unk_18, + unk_20, + } +} + +/// Register windows the firmware maps, by register-map slot. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 18] = [ + window(0, 0x3_0101_4000, 0x0_0000, 0x4000, 0, 2), + window(3, 0x2_2010_4000, 0x0_8000, 0x1_8000, 0, 2), + window(9, 0x3_003d_0000, 0x2_8000, 0x1000, 0, 2), + window(10, 0x3_003c_0000, 0x3_0000, 0x2000, 0, 0), + window(12, 0x4_0165_c000, 0x3_8000, 0x4000, 0, 2), + window(14, 0x3_0028_0000, 0x4_0000, 0x8000, 0, 0), + window(17, 0x4_8000_0000, 0x5_0000, 0x2_1400, 0, 2), + window(22, 0x4_8100_0000, 0x7_8000, 0x8000, 0, 2), + window(26, 0x4_80d0_4000, 0x8_8000, 0x8000, 0xd0_4000, 2), + window(27, 0x4_80d0_d000, 0x9_9000, 0x1000, 0xd0_d000, 2), + window(28, 0x4_80d5_8000, 0xa_0000, 0x8000, 0xd5_8000, 2), + window(29, 0x4_80d1_0000, 0xb_0000, 0x4000, 0xd1_0000, 2), + window(31, 0x4_80d4_0000, 0xb_8000, 0x4000, 0xd4_0000, 2), + window(32, 0x4_80d6_0000, 0xc_0000, 0x4000, 0xd6_0000, 2), + window(35, 0x4_80e0_0000, 0xc_8000, 0x4000, 0xe0_0000, 2), + window(39, 0x4_80e0_8000, 0xd_0000, 0x8000, 0, 2), + window(40, 0x4_80e1_c000, 0xe_0000, 0x4000, 0xe1_c000, 2), + window(41, 0x4_80e1_f800, 0xe_b800, 0x4000, 0, 2), +]; + +/// Register-map slots that are reserved but have no window. +pub(crate) const RESERVED_REGISTER_SLOTS: [usize; 17] = [ + 2, 5, 6, 7, 8, 30, 33, 34, 37, 38, 42, 43, 46, 47, 48, 49, 52, +]; + +/// Performance-table configuration. The device tree lists the operating +/// points in performance-state order; table A uses the odd states above +/// state 5, table B the even ones. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15], + states_b: [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 14], + relative_a: [0, 15, 22, 29, 42, 56, 74, 86, 90, 92, 100], + relative_b: [0, 0, 13, 24, 40, 52, 63, 73, 83, 88, 100], + sram_min_mv: 765, +}; + +/// Operating points the performance tables reference. +pub(crate) const OPPS: usize = 16; + +const _: () = { + let mut index = 0; + while index < PERF_STATES { + assert!((PERF.states_a[index] as usize) < OPPS); + assert!((PERF.states_b[index] as usize) < OPPS); + index += 1; + } +}; + +/// Offsets of the fixed objects in the kernel window, in the order the +/// builder creates them. +pub(crate) mod kernel_window { + /// Private cluster (`initdata::private`). + pub(crate) const PRIVATE_CLUSTER: u64 = 0x2_0000; + /// Zeroed firmware-private page. + pub(crate) const ZERO_PAGE: u64 = 0x15d_8000; + /// Parameter-buffer descriptor table. + pub(crate) const PB_DESCRIPTORS: u64 = 0x15e_0000; + /// page-pool descriptor table. + pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x15e_8000; + /// Index object (64 KiB, `initdata::IndexTable` in its first page). + pub(crate) const INDEX: u64 = 0xc084_8000; + /// Firmware-control page. + pub(crate) const FWCTL: u64 = 0x1c_0000; + /// Render free-list control and state pages. + pub(crate) const RENDER_FREE_LIST: u64 = 0xc083_0000; + pub(crate) const RENDER_FREE_LIST_STATE: u64 = 0x160_8000; + /// Boot free-list control and state pages. + pub(crate) const BOOT_FREE_LIST: u64 = 0xc082_8000; + pub(crate) const BOOT_FREE_LIST_STATE: u64 = 0x160_0000; + /// Zeroed descriptor pages. + pub(crate) const DESCRIPTOR_PAGE_A: u64 = 0x1c_8000; + pub(crate) const DESCRIPTOR_PAGE_B: u64 = 0xc07c_0000; + /// Zeroed TA and 3D context pages (128 KiB each). + pub(crate) const TA_CONTEXT: u64 = 0x1d_8000; + pub(crate) const FRAGMENT_CONTEXT: u64 = 0x20_0000; + /// Compute free-list control page and the state page it names (the + /// state page is not mapped at boot). + pub(crate) const COMPUTE_FREE_LIST: u64 = 0xc086_8000; + pub(crate) const COMPUTE_FREE_LIST_STATE: u64 = 0x163_0000; + /// Second compute free-list control page and its state page. + pub(crate) const COMPUTE_READY_FREE_LIST: u64 = 0xc087_8000; + pub(crate) const COMPUTE_READY_FREE_LIST_STATE: u64 = 0x164_8000; + /// Further compute free-list state page. + pub(crate) const COMPUTE_STATE: u64 = 0x165_0000; + /// Hardware-data bundle (`initdata::bundle`). + pub(crate) const BUNDLE: u64 = 0xc078_8000; +} + +/// Arenas used by dynamically allocated firmware objects. +pub(crate) mod dynamic { + /// First dynamic byte, relative to the upper kernel window. + pub(crate) const KERNEL_OFFSET: u64 = 0x168_4000; + /// Client-private compute and render backing after the shared queue arena. + pub(crate) const CLIENT_LOWER: core::ops::Range = 0x70_8000_0000..0x71_0000_0000; + /// Canonical low mappings shared by hardware queues and their clients. + /// Client-private allocations start after this arena. + pub(crate) const LOWER: core::ops::Range = 0x70_0400_0000..0x70_8000_0000; +} + +/// Fixed addresses in the firmware's context-0 and context-1 address spaces. +pub(crate) mod low { + /// Context-0 alias of the parameter-buffer descriptor table. + pub(crate) const PB_DESCRIPTORS: u64 = 0x70_0183_8000; + /// Context-0 alias of the page-pool descriptor table. + pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x70_0184_0000; + /// Context-0 alias of the index object. + pub(crate) const INDEX: u64 = 0x10_0019_0000; + /// Context-0 aliases of the completion rings. + pub(crate) const COMPLETION_RINGS: [u64; 2] = [0x70_03f0_0000, 0x70_03f2_8000]; + /// Context-0 run list of the compute free list. + pub(crate) const COMPUTE_RUN_LIST: u64 = 0x70_03ad_8000; + + /// Render free list (context 1): page list. + pub(crate) const RENDER_PAGE_LIST: u64 = 0x70_0000_0000; + /// Size of the render page list. + pub(crate) const RENDER_PAGE_LIST_SIZE: usize = 0x20_0000; + /// Render free list: run list (64 KiB, zero at boot). + pub(crate) const RENDER_RUN_LIST: u64 = 0x70_0020_8000; + /// Size of the render run list. + pub(crate) const RENDER_RUN_LIST_SIZE: usize = 0x1_0000; + /// Render free list: first block. + pub(crate) const RENDER_BLOCKS: u64 = 0x70_0022_0000; + /// Distance between consecutive blocks. + pub(crate) const BLOCK_STRIDE: u64 = 0x10_8000; + /// Render blocks allocated at boot. + pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = 28; + /// Render blocks mapped at boot. + pub(crate) const RENDER_BLOCKS_MAPPED: usize = 22; + /// Render blocks in the free list at boot. + pub(crate) const RENDER_BLOCKS_INITIAL: u32 = 17; + /// Render block holding the compute page list. + pub(crate) const COMPUTE_PAGE_LIST_BLOCK: usize = 22; + + /// Distance of the compute pool from the render pool. + const COMPUTE: u64 = 0x200_0000; + /// Compute pool: page list. + pub(crate) const COMPUTE_POOL_PAGE_LIST: u64 = RENDER_PAGE_LIST + COMPUTE; + /// Compute pool: run list. + pub(crate) const COMPUTE_POOL_RUN_LIST: u64 = RENDER_RUN_LIST + COMPUTE; + /// Compute pool: first block. + pub(crate) const COMPUTE_POOL_BLOCKS: u64 = RENDER_BLOCKS + COMPUTE; + /// Blocks the compute free list declares. + pub(crate) const COMPUTE_BLOCKS: u32 = 18; + /// Compute pool blocks in the second compute free list. + pub(crate) const COMPUTE_READY_BLOCKS: u32 = 17; + /// Page list of the compute free list (inside the block + /// [`COMPUTE_PAGE_LIST_BLOCK`] holds, relocated to the compute pool). + pub(crate) const COMPUTE_PAGE_LIST: u64 = + RENDER_BLOCKS + COMPUTE_PAGE_LIST_BLOCK as u64 * BLOCK_STRIDE + COMPUTE; + /// Two zeroed compute blocks. + pub(crate) const COMPUTE_ZERO_BLOCKS: [u64; 2] = [ + RENDER_PAGE_LIST + COMPUTE, + RENDER_PAGE_LIST + BLOCK_STRIDE + COMPUTE, + ]; + + /// Returns block `index` of the render pool. + pub(crate) const fn render_block(index: usize) -> u64 { + RENDER_BLOCKS + index as u64 * BLOCK_STRIDE + } + + /// Returns block `index` of the compute pool. + pub(crate) const fn compute_block(index: usize) -> u64 { + COMPUTE_POOL_BLOCKS + index as u64 * BLOCK_STRIDE + } + + /// Blocks of the compute free list, in run-list order: pool blocks 0 to + /// 17, the two zeroed blocks, then pool block 18. + pub(crate) const COMPUTE_FREE_LIST_BLOCKS: [u64; 21] = { + let mut blocks = [0; 21]; + let mut index = 0; + while index < 18 { + blocks[index] = compute_block(index); + index += 1; + } + blocks[18] = COMPUTE_ZERO_BLOCKS[0]; + blocks[19] = COMPUTE_ZERO_BLOCKS[1]; + blocks[20] = compute_block(18); + blocks + }; +} + +/// Client-root reservations and parameter-buffer geometry. +pub(crate) mod resources { + use core::ops::Range; + + pub(crate) const UAT_PAGE: u64 = 0x4000; + pub(crate) const USER_BASE: u64 = 0x10_0000_0000; + pub(crate) const TIMESTAMPS: Range = 0x70_0000_0000..0x71_0000_0000; + pub(crate) const COMMAND_TIMESTAMP_HZ: u64 = 24_000_000; + pub(crate) const USER_END: u64 = USER_BASE + (1 << 32); + pub(crate) const FIXED_PAIRS: usize = 2; + pub(crate) const FIXED_PAIR_STRIDE: u64 = 0x8_0000; + pub(crate) const SCENE: u64 = USER_BASE + 0x17_8000; + pub(crate) const SCENE_SIZE: u64 = 0x8000; + pub(crate) const DISCARD: u64 = USER_BASE + 0x18_0000; + pub(crate) const DISCARD_SIZE: u64 = 0x8000; + pub(crate) const PAGE_LIST: u64 = USER_BASE + 0x19_0000; + pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; + pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; + pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; + pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; + pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; + pub(crate) const TPC_MIN: usize = 0x3_0000; + pub(crate) const TPC_MAX: usize = 0x200_0000; + pub(crate) const AUXILIARY: Range = 0x100_0200_0000..0x100_0240_0000; + pub(crate) const METRICS_SIZE: u64 = 0x8000; + pub(crate) const TVB_PAGE: u64 = 0x8000; + pub(crate) const TVB_BLOCK_SIZE: usize = 0x2_0000; + pub(crate) const TVB_BLOCK_STRIDE: u64 = 0x2_8000; + pub(crate) const TVB_BLOCK_SLOTS: usize = 0xd1a; + pub(crate) const TVB_MAX_BLOCKS: usize = TVB_BLOCK_SLOTS - 1; + pub(crate) const TVB_PAGES_PER_BLOCK: usize = 4; + pub(crate) const TVB_PAGE_LIST_SIZE: usize = TVB_BLOCK_SLOTS * TVB_PAGES_PER_BLOCK * 4; + pub(crate) const TVB_INITIAL_PAGES: [u32; 32] = [ + 0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x4a, 0x4f, 0x54, 0x59, 0x5e, 0x63, 0x68, 0x6d, 0x72, + 0x77, 0x7c, 0x81, 0x86, 0x8b, 0x90, 0x95, 0x9a, 0x9f, 0xa4, 0xa9, 0xae, 0xb3, 0xb8, 0xbd, + 0xc2, 0xc7, + ]; + pub(crate) const TVB_GROWTH: u64 = USER_BASE + 0x0400_0000; + pub(crate) const TVB_PAIR_STRIDE: u64 = 0x2800_0000; + pub(crate) const RESERVED_RANGES: usize = 48; + + /// Compact allocations search these shared client arenas in this order. + pub(crate) fn compact_ranges() -> [Range; 4] { + let second = TVB_GROWTH + TVB_PAIR_STRIDE; + [ + RENDER_ALIASES, + TVB_GROWTH + ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE, + second..second + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE, + TARGETS, + ] + } + + /// All driver-owned windows reserved before the first userspace bind. + /// Reserving them does not install pages or allocate backing memory. + pub(crate) fn client_reserved_ranges(metrics_va: u64) -> Option<[Range; RESERVED_RANGES]> { + let mut ranges = core::array::from_fn(|_| 0..0); + let mut index = 0; + let mut add = |range| { + ranges[index] = range; + index += 1; + }; + add(0..0x1_0000); + add(TIMESTAMPS); + let page_list_size = (TVB_PAGE_LIST_SIZE as u64 + UAT_PAGE - 1) & !(UAT_PAGE - 1); + for pair in 0..FIXED_PAIRS { + let offset = pair as u64 * FIXED_PAIR_STRIDE; + add(SCENE + offset..SCENE + offset + SCENE_SIZE); + add(DISCARD + offset..DISCARD + offset + DISCARD_SIZE); + add(PAGE_LIST + offset..PAGE_LIST + offset + page_list_size); + add(FRAGMENT_STATUS + offset..FRAGMENT_STATUS + offset + UAT_PAGE); + } + add(RENDER_ALIASES); + add(TARGETS); + for page in TVB_INITIAL_PAGES { + let start = USER_BASE + u64::from(page) * TVB_PAGE; + add(start..start + TVB_BLOCK_STRIDE); + } + add(TVB_GROWTH + ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE); + for pair in 1..FIXED_PAIRS { + let start = TVB_GROWTH + pair as u64 * TVB_PAIR_STRIDE; + add(start..start + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE); + } + add(AUXILIARY); + add(metrics_va..metrics_va.checked_add(METRICS_SIZE)?); + Some(ranges) + } +} + +/// Canonical userspace resource and its presentation in an execution context. +#[derive(Clone, Copy)] +pub(crate) struct ContextView { + pub(crate) source: u64, + pub(crate) low: u64, + pub(crate) size: usize, + pub(crate) write: bool, +} + +const fn context_view(low: u64, size: usize, write: bool) -> ContextView { + ContextView { + source: 0x10_0000_0000 + low, + low, + size, + write, + } +} + +pub(crate) const CONTEXT_VIEWS: [ContextView; 7] = [ + context_view(0, 0x10000, false), + context_view(0x18000, 0x8000, false), + context_view(0x28000, 0x8000, false), + context_view(0x38000, 0x8000, false), + context_view(0x48000, 0x8000, false), + context_view(0x58000, 0x8000, true), + context_view(0x68000, 0xc000, true), +]; + +/// Scheduler accounting policy and its firmware clock domain. +pub(crate) mod qos { + pub(crate) const CLASS: u8 = 0x0f; + pub(crate) const SHARE: u32 = 0x2000; + pub(crate) const CLOCK_HZ: u64 = 24_000_000; +} + +/// Physical workqueue capacity and logical-owner admission limits. +pub(crate) mod queues { + pub(crate) const RENDER_SLOTS: usize = 64; + pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; + pub(crate) const RENDER_DEPTH: u8 = 16; + pub(crate) const COMPUTE_DEPTH: usize = 16; +} + +/// Read-only recovery evidence, sampled only for selected halted slots. +pub(crate) mod recovery { + pub(crate) const SLOT_KEYS: u64 = 0x0000_0100_01ea_74b8; +} diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index 60a64e23a161c5..f354e4bb1f71ab 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -11,6 +11,8 @@ use core::arch::asm; use core::cmp::min; +use kernel::page::PAGE_SIZE; + use crate::debug::*; use crate::mmu; @@ -134,6 +136,66 @@ pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { } } +/// Range length from which [`tlbi_range_or_asid`] invalidates the whole ASID. +const TLBI_RANGE_MAX: usize = 0x200_0000; +/// Range length from which [`tlbi_range_or_asid`] counts in 1 MiB units (scale 1) instead of +/// 32 KiB units (scale 0). +const TLBI_RANGE_SCALE1_MIN: usize = 0x10_0000; +/// Translation granule field value for 16 KiB pages. +const TLBI_TG_16K: u64 = 2; +/// Width of the base address field of a range TLBI operand. +const TLBI_RANGE_BADDR_MASK: u64 = (1 << 38) - 1; + +/// Invalidate a range of pages for a given ASID with a single range operation. +/// +/// Unlike [`tlbi_range`], which issues as many range operations as needed, this invalidates +/// ranges that do not fit one operation (and ranges starting at VA 0) by ASID. Single pages use a +/// page operation, at the CPU page size. +#[inline(always)] +pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { + if debug_enabled(DebugFlags::ConservativeTlbi) { + tlbi_all(); + sync(); + return; + } + + if len == 0 { + return; + } + + if va == 0 || len >= TLBI_RANGE_MAX { + tlbi_asid(asid); + return; + } + if len <= PAGE_SIZE { + tlbi_page(asid, va); + return; + } + + // The range covers (NUM + 1) << (5 * SCALE + 1) granules of 16 KiB. + let (scale, unit_shift) = if len >= TLBI_RANGE_SCALE1_MIN { + (1u64, 20) + } else { + (0u64, 15) + }; + let num = ((len - 1) >> unit_shift) as u64; + + let val = ((asid as u64) << 48) + | (TLBI_TG_16K << 46) + | (scale << 44) + | (num << 39) + | (((va >> mmu::UAT_PGBIT) as u64) & TLBI_RANGE_BADDR_MASK); + + // SAFETY: tlbi is always safe by definition + unsafe { + asm!( + ".arch armv8.4-a", + "tlbi rvae1os, {x}", + x = in(reg) val + ); + } +} + /// Issue a memory barrier (`dsb sy`). #[inline(always)] pub(crate) fn sync() { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index ed907a54636f1f..44c20e7d41e153 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -9,6 +9,10 @@ //! unmap buffer objects into a single user or kernel address space. //! //! The actual page table management is in the `pt` module. +//! +//! G17 firmware does not take part in the handoff protocol. Instead, each of its two coprocessor +//! instances publishes the top-level entries it owns into its own kernel root while it boots, and +//! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. use core::fmt::Debug; use core::mem::{size_of, ManuallyDrop}; @@ -91,11 +95,24 @@ use pin_init; const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; +mod bind; +mod context; +mod shared; +mod lifetime; +pub(crate) use bind::PreparedUserBindBatch; +pub(crate) use lifetime::VmJobGuard; + /// PPL magic number for the handoff region const PPL_MAGIC: u64 = 0x4b1d000000000002; /// Number of supported context entries in the TTBAT const UAT_NUM_CTX: usize = 64; + +/// Application contexts owned by independent logical queues. Contexts 0..4 +/// and 63 remain reserved for the firmware and device-global resources. +const EXECUTION_CONTEXT_MASK: u64 = ((1u64 << 63) - 1) & !((1u64 << 5) - 1); +/// Maximum simultaneously retained logical queue contexts. +pub(crate) const MAX_EXECUTION_CONTEXTS: u32 = EXECUTION_CONTEXT_MASK.count_ones(); /// First context available for users const UAT_USER_CTX_START: usize = 1; /// Number of available user contexts @@ -232,6 +249,30 @@ pub(crate) fn kern_iova_range(cfg: &hw::HwConfig, range: Range) -> Range(); // We need at least page 0 (ttb0) const PAGETABLES_SIZE: usize = UAT_PGSZ; +/// Lock-free visibility of the mapping epoch. Writers run under the GPUVM +/// exec lock; nested mutations keep cached validation unavailable until the +/// outermost mutation has finished its leaf writes and invalidations. +struct MappingEpoch { + visible: AtomicU64, + pending: AtomicU64, + writers: AtomicU32, +} + +impl MappingEpoch { + fn new() -> Self { + Self { + visible: AtomicU64::new(0), + pending: AtomicU64::new(0), + writers: AtomicU32::new(0), + } + } + fn snapshot(&self) -> Option { + let value = self.visible.load(Ordering::Acquire); + (value != u64::MAX).then_some(value) + } +} + +#[must_use = "retain until the mapping mutation and its invalidations finish"] +struct MappingMutation(Option>); + +impl Drop for MappingMutation { + fn drop(&mut self) { + if let Some(epoch) = self.0.as_ref() { + if epoch.writers.fetch_sub(1, Ordering::Relaxed) == 1 { + epoch + .visible + .store(epoch.pending.load(Ordering::Relaxed), Ordering::Release); + } + } + } +} + /// Inner data for a Vm instance. This is reference-counted by the outer Vm object. struct VmInner { dev: driver::AsahiDevRef, @@ -360,6 +439,8 @@ struct VmInner { fixed_slot: Option, va_range: Range, page_table: UatPageTable, + mapping_epoch: Option>, + epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, uat_inner: Arc, binding: Arc>, @@ -417,6 +498,7 @@ impl gpuvm::DriverGpuVm for VmInner { ctx: &mut Self::StepContext, ) -> Result { if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } + let _mutation = self.mapping_mutation(); let mut iova = op.addr(); let mut left = op.range() as usize; let mut offset = op.offset() as usize; @@ -529,6 +611,7 @@ impl gpuvm::DriverGpuVm for VmInner { ); return Err(EINVAL); } + self.tlbi_contexts_naming_root(op.addr(), op.range() as usize); Ok(()) } fn step_unmap( @@ -541,12 +624,13 @@ impl gpuvm::DriverGpuVm for VmInner { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + let _mutation = self.mapping_mutation(); self.page_table .unmap_pages(va.addr()..(va.addr() + va.range()))?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, va.addr() as usize, va.range() as usize); + self.tlbi_range(asid as u8, va.addr(), va.range() as usize); mod_dev_dbg!( self.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -557,6 +641,8 @@ impl gpuvm::DriverGpuVm for VmInner { mem::sync(); } + self.tlbi_contexts_naming_root(va.addr(), va.range() as usize); + if op.unmap_and_unlink_va_defer().is_none() { dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); } @@ -597,11 +683,12 @@ impl gpuvm::DriverGpuVm for VmInner { let unmap_range = unmap_end - unmap_start; + let _mutation = self.mapping_mutation(); self.page_table.unmap_pages(unmap_start..unmap_end)?; if let Some(asid) = self.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, unmap_start as usize, unmap_range as usize); + self.tlbi_range(asid as u8, unmap_start, unmap_range as usize); mod_dev_dbg!( self.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -612,6 +699,8 @@ impl gpuvm::DriverGpuVm for VmInner { mem::sync(); } + self.tlbi_contexts_naming_root(unmap_start, unmap_range as usize); + if op.unmap().unmap_and_unlink_va_defer().is_none() { dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); } @@ -643,6 +732,19 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + fn mapping_mutation(&mut self) -> MappingMutation { + let state = self.mapping_epoch.clone(); + if let Some(state) = state.as_ref() { + state.writers.fetch_add(1, Ordering::Relaxed); + state.visible.swap(u64::MAX, Ordering::AcqRel); + self.epoch = self.epoch.and_then(|epoch| epoch.checked_add(1)); + state + .pending + .store(self.epoch.unwrap_or(u64::MAX), Ordering::Relaxed); + } + MappingMutation(state) + } + /// Returns the slot index, if this VM is bound. fn slot(&self) -> Option { if let Some(slot) = self.fixed_slot { return Some(slot); } @@ -746,11 +848,45 @@ impl VmInner { done += chunk; } Ok(()) + /// Invalidate a VA range of this Vm's address space in the TLB of one ASID. + fn tlbi_range(&self, asid: u8, iova: u64, size: usize) { + match self.uat_inner.firmware { + UatFirmware::Handoff => mem::tlbi_range(asid, iova as usize, size), + UatFirmware::PublishedRoots { .. } => { + mem::tlbi_range_or_asid(asid, iova as usize, size) + } + } + } + + /// Invalidate a VA range in every TTBAT context that names this Vm's root as its lower half. + /// + /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root + /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. + fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + if contexts == 0 { + return; + } + + fence(Ordering::SeqCst); + for ctx in 0..UAT_NUM_CTX { + if contexts & (1 << ctx) != 0 { + mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + } + } + mem::sync(); } /// Map an `mm::Node` representing an mapping in VA space. fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); + let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); let mapping_end = if self.uat_inner.fault.is_some() { iova.checked_add(node.mapped_size as u64).ok_or(EOVERFLOW)? } else { iova + node.mapped_size as u64 }; @@ -759,6 +895,7 @@ impl VmInner { if self.uat_inner.fault.is_some() { self.page_table.prepare_map(iova..mapping_end)?; } let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().inner.lock(); let sgt = guard.sgt.as_ref().ok_or(EINVAL)?; + let _mutation = self.mapping_mutation(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -813,6 +950,9 @@ impl VmInner { iova += len as u64; left -= len; } + + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -822,6 +962,11 @@ impl VmInner { pub(crate) struct Vm { fault: Option>, id: u64, + mapping_epoch: Option>, + lifetime: Option>, + context_bindings: Option>, + shared_bindings: Option>, + status: Option>, inner: ARef>, dummy_obj: ARef, binding: Arc>, @@ -894,6 +1039,75 @@ where /// Allocation generation of a [`Vm`] hardware slot. pub(crate) struct SlotInner { generation: u8 } +/// An application TTBAT identity retained by a logical queue and its jobs. +/// The root lease is dropped before the VM, so no page table can be freed +/// while a context still names it. Installed firmware queues must retain this +/// object through replacement retirement or processor stop. +pub(crate) struct ExecutionContext { + root: ExecutionRoot, + vm: Vm, +} + +struct ExecutionRoot { + inner: Arc, + id: u8, + generation: u8, + low: u64, + high: u64, +} + +impl ExecutionContext { + pub(crate) fn id(&self) -> u32 { + self.root.id.into() + } + pub(crate) fn generation(&self) -> u8 { + self.root.generation + } + pub(crate) fn vm(&self) -> &Vm { + &self.vm + } + + pub(crate) fn is_current(&self) -> bool { + let inner = self.root.inner.lock(); + let roots = &inner.ttbs()[usize::from(self.root.id)]; + roots.ttb0.load(Ordering::Acquire) == self.root.low + && roots.ttb1.load(Ordering::Acquire) == self.root.high + } +} + +impl Drop for ExecutionRoot { + fn drop(&mut self) { + let id = usize::from(self.id); + let release = { + let inner = self.inner.lock(); + inner.handoff().lock(); + let roots = &inner.ttbs()[id]; + let low = roots.ttb0.load(Ordering::Acquire); + let high = roots.ttb1.load(Ordering::Acquire); + let owned = low == self.low && high == self.high; + if owned { + roots.ttb0.store(0, Ordering::Release); + roots.ttb1.store(0, Ordering::Release); + } + inner.handoff().unlock(); + owned || (low == 0 && high == 0) + }; + if release { + fence(Ordering::SeqCst); + mem::tlbi_asid(self.id); + mem::sync(); + self.inner.lock().execution_contexts &= !(1u64 << id); + } else { + pr_err!( + "MMU: application context {} changed owners while leased\n", + id + ); + } + } +} + +/// Slot data for a [`Vm`] slot (nothing, we only care about the indices). +pub(crate) struct SlotInner(); impl slotalloc::SlotItem for SlotInner { type Data = (); @@ -966,13 +1180,60 @@ impl Clone for VmBind { } } +/// A mapping BO reference selecting the teardown protocol of its firmware. +/// Retain the owner until deferred cleanup completes, including its final BO. +struct MappingBo { + deferred: bool, + bo: Option>>, + owner: ARef>, +} +impl MappingBo { + fn new( + bo: ARef>, + owner: ARef>, + deferred: bool, + ) -> Self { + Self { + deferred, + bo: Some(bo), + owner, + } + } + fn get(&self) -> Result<&gpuvm::GpuVmBo> { + self.bo.as_deref().ok_or(EINVAL) + } +} +impl Drop for MappingBo { + fn drop(&mut self) { + let Some(bo) = self.bo.take() else { + return; + }; + if !self.deferred { + // Handoff VMs retain the legacy ARef decrement and reservation + // locking behavior. Only published-root VMs use deferred cleanup. + drop(bo); + return; + } + let raw = ARef::into_raw(bo); + // SAFETY: GpuVmBo is repr(C) with drm_gpuvm_bo first. into_raw + // transfers one live reference. Every Vm here uses IMMEDIATE_MODE; + // deferred put selects its GPUVA lock and retains cleanup ownership. + unsafe { + kernel::bindings::drm_gpuvm_bo_put_deferred( + raw.as_ptr().cast::(), + ) + }; + self.owner.bo_deferred_cleanup(); + } +} + /// Inner data required for an object mapping into a [`Vm`]. pub(crate) struct KernelMappingInner { // Drop order matters: // - Drop the GpuVmBo first, which locks its BO GPUVA list and drops a GpuVm reference // - Drop the GEM BO next, since BO free can take the resv lock itself // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 - bo: Option>>, + bo: Option, _gem: Option>, owner: ARef>, uat_inner: Arc, @@ -1000,6 +1261,111 @@ impl KernelMapping { self.0.start()..(self.0.start() + self.0.mapped_size as u64) } + /// Resolves a byte in this retained mapping without creating page tables. + pub(crate) fn translate_offset(&self, offset: usize) -> Result { + if offset >= self.size() { + return Err(EINVAL); + } + let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; + self.0.owner.exec_lock(None, false)?.page_table.translate_iova(address) + } + + /// Writes a userspace timestamp through the retained GEM mapping. Byte + /// stores support unaligned destinations and concurrent completion writes + /// without creating mutable aliases. The complete timestamp is not atomic. + pub(crate) fn write_timestamp(&self, offset: usize, value: u64) -> Result { + use core::sync::atomic::AtomicU8; + use kernel::drm::gem::BaseObject; + let gem = self.0._gem.as_ref().ok_or(EINVAL)?; + if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { + return Err(ERANGE); + } + let start = self.0.offset.checked_add(offset).ok_or(EOVERFLOW)?; + let end = start.checked_add(size_of::()).ok_or(EOVERFLOW)?; + if end > gem.size() { + return Err(ERANGE); + } + let vmap = gem.vmap::()?; + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: The retained GEM/vmap covers each checked byte. AtomicU8 + // has byte alignment and all concurrent completion writers use + // the same access width, without borrowing a mutable byte slice. + let destination = unsafe { &*vmap.as_mut_ptr().add(start + index).cast::() }; + destination.store(byte, Ordering::Relaxed); + } + Ok(()) + } + + /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. + /// The result includes whether any page-table entry may have changed; after + /// mutation the caller must retain the mapping until processor stop even if + /// the readback fails. The original protection is retained for teardown. + /// + /// # Safety + /// The mapping must be unpublished. After mutation it must not be unmapped + /// while a firmware instance can retain a cached translation or contents. + pub(crate) unsafe fn set_fw_cached_prefix(&self, prefix: usize) -> (Result<[u64; 2]>, bool) { + const MAX_PAGES: usize = 32; + const ATTRIBUTE_SHIFT: u32 = 2; + const ATTRIBUTE_MASK: u64 = 7; + const UNCACHED: u64 = 2; + const CACHE_ATTRIBUTE_CHANGE: u64 = UNCACHED << ATTRIBUTE_SHIFT; + let mut mutated = false; + let result = (|| { + if self.0.uat_inner.firmware == UatFirmware::Handoff + || prefix == 0 + || prefix > self.size() + || (prefix | self.size()) & UAT_PGMSK != 0 + || self.size() > MAX_PAGES * UAT_PGSZ + { + return Err(EINVAL); + } + let mut owner = self.0.owner.exec_lock(None, false)?; + if !owner.is_kernel { + return Err(EINVAL); + } + let pages = self.size() / UAT_PGSZ; + let cached_pages = prefix / UAT_PGSZ; + let mut before = [0; MAX_PAGES]; + for (index, value) in before[..pages].iter_mut().enumerate() { + *value = owner + .page_table + .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; + if (*value >> ATTRIBUTE_SHIFT) & ATTRIBUTE_MASK != UNCACHED { + return Err(EINVAL); + } + } + let _mutation = owner.mapping_mutation(); + mutated = true; + owner.page_table.reprot_pages( + self.iova()..self.iova() + prefix as u64, + PROT_GPU_FW_PRIV_RW, + )?; + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + let mut after = [0; MAX_PAGES]; + for (index, value) in after[..pages].iter_mut().enumerate() { + *value = owner + .page_table + .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; + let change = if index < cached_pages { + CACHE_ATTRIBUTE_CHANGE + } else { + 0 + }; + if before[index] ^ *value != change { + return Err(EFAULT); + } + } + Ok([after[0], after[cached_pages.min(pages - 1)]]) + })(); + (result, mutated) + } + + /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the + /// coprocessor cache. This is required to safely unmap cached/private mappings. + fn remap_uncached_and_flush(&mut self) { fn remap_uncached_and_flush_legacy(&mut self) { let mut owner = self .0 @@ -1179,7 +1545,7 @@ impl KernelMapping { if let Some(asid) = owner.slot() { fence(Ordering::SeqCst); - mem::tlbi_range(asid as u8, self.iova() as usize, self.size()); + owner.tlbi_range(asid as u8, self.iova(), self.size()); mod_dev_dbg!( owner.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -1420,6 +1786,10 @@ impl Drop for KernelMapping { // SAFETY: The PTEs are gone, required flushes completed, and the VM // execution lock is released before the node drops BO references. unsafe { ManuallyDrop::drop(&mut self.0) }; + + // The range returns to the VA allocator once this mapping is gone, so no context that + // names this root may keep a stale translation for it. + owner.tlbi_contexts_naming_root(self.iova(), self.size()); } } @@ -1427,6 +1797,10 @@ impl Drop for KernelMapping { struct UatShared { fault: Option>, kernel_ttb1: u64, + execution_contexts: u64, + context_generations: [u8; UAT_NUM_CTX], + /// Kernel root of the secondary firmware instance, if there is one. + secondary_ttb1: Option, map_kernel_to_user: bool, handoff_rgn: UatRegion, ttbs_rgn: UatRegion, @@ -1453,6 +1827,20 @@ impl UatShared { // SAFETY: pointer is non-null per the type invariant unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() } + + /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. + fn contexts_naming_root(&self, ttb: u64) -> u64 { + self.ttbs() + .iter() + .enumerate() + .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) + .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) + } +} + +/// Returns the TTBAT value naming root `ttb` in context `ctx`. +const fn tagged_root(ttb: u64, ctx: usize) -> u64 { + ttb | TTBR_VALID | (ctx as u64) << TTBR_ASID_SHIFT } // SAFETY: Nothing here is unsafe to send across threads. @@ -1464,6 +1852,7 @@ struct UatInner { m3_running: bool, fault: Option>, firmware_cache_flush_ready: AtomicBool, + firmware: UatFirmware, #[pin] shared: Mutex, #[pin] @@ -1586,6 +1975,23 @@ impl Handoff { } fence(Ordering::SeqCst); + if firmware == UatFirmware::Handoff { + self.wait_for_firmware()?; + } + + for i in 0..=UAT_NUM_CTX { + self.flush[i].state.store(0, Ordering::Relaxed); + self.flush[i].addr.store(0, Ordering::Relaxed); + self.flush[i].size.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + Ok(()) + } + + /// Wait for the firmware to publish its half of the handoff region. + fn wait_for_firmware(&self) -> Result { + let start = Instant::::now(); + const TIMEOUT: Delta = Delta::from_millis(1000); if matches!(mode, HandoffMode::FirmwareDekker | HandoffMode::FirmwareM3) { let start = Instant::::now(); const TIMEOUT: Delta = Delta::from_millis(1000); @@ -1748,7 +2154,7 @@ impl Vm { geometry.kernel_range(), geometry.ias, geometry.oas, - false, + uat_inner.firmware != UatFirmware::Handoff, )? if cfg.reserved_tables { let node = dev.as_ref().of_node().ok_or(ENODEV)?; @@ -1788,6 +2194,11 @@ impl Vm { (geometry.kernel_range(), kernel_range.clone()) (iova_kern_range, kernel_range.clone()) } else { + let mut range = geometry.user_range(); + if uat_inner.firmware != UatFirmware::Handoff { + range.start = 0; + } + (range, geometry.user_usable_range()) ( driver_start..uat_geometry(cfg)?.user_va_top(), lower_start..(uat_geometry(cfg)?.user_va_top() - 2 * UAT_PGSZ as u64), @@ -1809,10 +2220,20 @@ impl Vm { GFP_KERNEL, )?; + let mapping_epoch = if uat_inner.firmware != UatFirmware::Handoff { + Some(Arc::new(MappingEpoch::new(), GFP_KERNEL)?) + } else { + None + }; let binding_clone = binding.clone(); Ok(Vm { fault, id, + mapping_epoch: mapping_epoch.clone(), + lifetime: None, + context_bindings: None, + shared_bindings: None, + status: None, dummy_obj: dummy_obj.gem.clone(), inner: gpuvm::GpuVm::new( c_str!("Asahi::GpuVm"), @@ -1828,6 +2249,8 @@ impl Vm { is_kernel, fixed_slot: None, page_table, + mapping_epoch, + epoch: Some(0), mm, uat_inner, binding: binding_clone, @@ -1850,6 +2273,102 @@ impl Vm { }) } + /// Attach a status before sharing a newly created user VM with queues. + pub(crate) fn with_status(mut self) -> Result { + self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.lifetime = Some(lifetime::VmLifetime::new()?); + self.context_bindings = Some(context::ContextBindings::new()?); + self.shared_bindings = Some(shared::SharedBindings::new()?); + Ok(self) + } + + /// Status shared by every queue using this VM, if enabled by its GPU. + pub(crate) fn status(&self) -> Option<&Arc> { + self.status.as_ref() + } + + fn covers_range_locked( + inner: &mut VmInner, + address: u64, + size: u64, + read: bool, + write: bool, + ) -> bool { + if size == 0 { + return false; + } + let Some(end) = address.checked_add(size) else { + return false; + }; + let mask = UAT_PGMSK as u64; + let start = address & !mask; + let end = if end & mask == 0 { + end + } else { + let Some(end) = (end | mask).checked_add(1) else { + return false; + }; + end + }; + if start < inner.va_range.start || end > inner.va_range.end { + return false; + } + inner + .page_table + .covers_range(start..end, read, write) + .unwrap_or(false) + } + + pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { + let Ok(mut inner) = self.inner.exec_lock(None, false) else { + return false; + }; + Self::covers_range_locked(&mut inner, address, size, read, write) + } + + /// Checks all writable sparse blocks and their guard coverage under one + /// GPUVM lock. A fully mapped guard is not a valid sparse allocation. + pub(crate) fn covers_sparse_blocks_with_guards( + &self, + addresses: &[u64], + block_size: u64, + stride: u64, + ) -> bool { + if stride <= block_size { + return false; + } + let Ok(mut inner) = self.inner.exec_lock(None, false) else { + return false; + }; + addresses.iter().all(|&address| { + let Some(guard) = address.checked_add(block_size) else { + return false; + }; + Self::covers_range_locked(&mut inner, address, block_size, true, true) + && !Self::covers_range_locked( + &mut inner, guard, stride - block_size, false, false, + ) + }) + } + + /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 + /// requests reading and bit1 writing. The epoch names this same snapshot. + pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { + let Ok(mut inner) = self.inner.exec_lock(None, false) else { + return (false, None); + }; + let epoch = inner.epoch; + let covered = ranges.iter().all(|&(address, size, access)| { + Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) + }); + (covered, epoch) + } + + /// Returns None while a leaf mutation is in progress or after epoch overflow. + pub(crate) fn mapping_validation_epoch(&self) -> Option { + self.mapping_epoch.as_ref()?.snapshot() + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -2058,7 +2577,39 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { - let size = object_range.range(); + self.map_in_range_with_guard_size( + gem, + object_range, + alignment, + range, + prot, + if guard { UAT_PGSZ } else { 0 }, + ) + } + + /// Reserves a whole-page unmapped guard without adding it to the mapped backing. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_in_range_with_guard_size( + &self, + gem: &gem::Object, + object_range: Range, + alignment: u64, + range: Range, + prot: Prot, + guard_size: usize, + ) -> Result { + use kernel::drm::gem::BaseObject; + if guard_size % UAT_PGSZ != 0 { + return Err(EINVAL); + } + let size = object_range + .end + .checked_sub(object_range.start) + .ok_or(EINVAL)?; + if size == 0 || object_range.end > gem.size() { + return Err(EINVAL); + } + let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = self.inner.obtain_bo(gem)?; @@ -2069,11 +2620,17 @@ impl Vm { } core::mem::drop(vm_bo_guard); + let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); let uat_inner = inner.uat_inner.clone(); // Reserve the VA before attaching objects whose destructors take the // GEM reservation lock. The allocator drops its payload internally // on an insertion error, while exec_lock still owns that lock. let node_result = inner.mm.insert_node_in_range( + // A failed reservation drops its payload while exec still owns GEM + // reservations. Published-root VMs attach BO/GEM only after success; + // handoff VMs retain their existing payload and decrement behavior. + let result = inner.mm.insert_node_in_range( KernelMappingInner { owner: self.inner.clone(), uat_inner, @@ -2083,13 +2640,24 @@ impl Vm { offset: object_range.start, mapped_size: size, }, - (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + reserved_size as u64, alignment, 0, range.start, range.end, mm::InsertMode::Best, ); + let mut node = match result { + Ok(node) => node, + Err(error) => { + drop(inner); + return Err(error); + } + }; + if deferred { + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem.into()); let mut node = match node_result { Ok(node) => node, Err(err) => { @@ -2121,6 +2689,9 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { + let reserved_size = size + .checked_add(if guard { UAT_PGSZ } else { 0 }) + .ok_or(EOVERFLOW)?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(&gem), false)?; @@ -2132,10 +2703,14 @@ impl Vm { } core::mem::drop(vm_bo_guard); + let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); let uat_inner = inner.uat_inner.clone(); // See map_in_range(): keep dma_resv-owning references out of the // allocator payload until the fixed-address reservation succeeds. let node_result = inner.mm.reserve_node( + // Published-root VMs defer BO/GEM attachment until reservation succeeds. + let result = inner.mm.reserve_node( KernelMappingInner { owner: self.inner.clone(), uat_inner, @@ -2146,9 +2721,20 @@ impl Vm { mapped_size: size, }, addr, - (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + reserved_size as u64, 0, ); + let mut node = match result { + Ok(node) => node, + Err(error) => { + drop(inner); + return Err(error); + } + }; + if deferred { + let payload = node.as_mut().inner_mut(); + payload.bo = vm_bo.take(); + payload._gem = Some(gem.clone()); let mut node = match node_result { Ok(node) => node, Err(err) => { @@ -2181,6 +2767,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + self.wait_for_user_map_admission()?; // Mapping needs a complete context let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -2245,6 +2832,7 @@ impl Vm { size, addr ); + let _commit = self.mapping_commit()?; inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) } @@ -2315,6 +2903,7 @@ impl Vm { core::mem::drop(inner); return Err(error); } + let mutation = inner.mapping_mutation(); let ret = inner.page_table.map_pages( iova..(iova + size as u64), phys as PhysicalAddr, @@ -2323,6 +2912,7 @@ impl Vm { ); // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. + drop(mutation); core::mem::drop(inner); ret?; Ok(KernelMapping(ManuallyDrop::new(node))) @@ -2330,6 +2920,17 @@ impl Vm { /// Unmap everything in an address range. pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { + if self.defer_unmap(iova, size)? { + return Ok(()); + } + self.unmap_range_commit(iova, size, true) + } + + fn unmap_range_now(&self, iova: u64, size: u64) -> Result { + self.unmap_range_commit(iova, size, false) + } + + fn unmap_range_commit(&self, iova: u64, size: u64, user: bool) -> Result { // Unmapping a range can only do a single split, so just preallocate // the prev and next GpuVas let mut ctx = StepContext { @@ -2341,7 +2942,22 @@ impl Vm { let mut inner = self.inner.exec_lock(None, false)?; mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); - inner.sm_unmap(&mut ctx, iova, size) + let _commit = if user { + match self.mapping_commit() { + Ok(guard) => guard, + Err(error) => { + if self.defer_unmap(iova, size)? { + return Ok(()); + } + return Err(error); + } + } + } else { + None + }; + inner.sm_unmap(&mut ctx, iova, size)?; + self.untrack_shared_range(iova, size); + Ok(()) } /// Drop mappings for a given bo. @@ -2356,6 +2972,7 @@ impl Vm { } } + fn drop_mappings_now(&self, gem: &gem::Object) -> Result { // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); @@ -2364,6 +2981,7 @@ impl Vm { if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); self.inner.bo_unmap(&mut ctx, &bo)?; + self.untrack_shared_object(gem); mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. core::mem::drop(inner); @@ -2510,6 +3128,39 @@ impl Uat { &self.kernel_lower_vm } + /// Installed descriptor and bind contexts rooted in this user VM. A graph + /// constructor checks this before changing a retained pool's mappings. + pub(crate) fn vm_context_mask(&self, vm: &Vm) -> Result { + if !matches!(self.inner.firmware, UatFirmware::PublishedRoots { .. }) { + return Err(EINVAL); + } + let root = { + let owner = vm.inner.exec_lock(None, false)?; + if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { + return Err(EINVAL); + } + owner.ttb() + }; + Ok(self.inner.lock().contexts_naming_root(root)) + } + + /// Publish a completed alias set before its descriptors become visible. + /// Match actual roots, including independently allocated execution contexts. + pub(crate) fn flush_vm_contexts(&self, vm: &Vm) -> Result { + let contexts = self.vm_context_mask(vm)?; + if contexts == 0 { + return Err(ENOENT); + } + fence(Ordering::SeqCst); + for context in 0..UAT_NUM_CTX { + if contexts & (1u64 << context) != 0 { + mem::tlbi_asid(context as u8); + } + } + mem::sync(); + Ok(contexts) + } + #[cfg(CONFIG_DEV_COREDUMP)] pub(crate) fn dump_kernel_pages(&self) -> Result> { // Full upper-half range for this SoC's root geometry. @@ -2632,6 +3283,63 @@ impl Uat { Ok(VmBind(vm.clone(), slot, binding.binding.as_ref().unwrap().generation)) } + /// Allocates a stable application context without changing the VM bind slot. + /// Both roots and the generation remain unchanged for the lease's lifetime. + pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { + if self.inner.firmware == UatFirmware::Handoff { + return Err(EINVAL); + } + { + let owner = vm.inner.exec_lock(None, false)?; + if owner.is_kernel + || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) + { + return Err(EINVAL); + } + } + let (id, generation) = { + let mut inner = self.inner.lock(); + let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; + if free == 0 { + return Err(ENOSPC); + } + let id = free.trailing_zeros() as usize; + inner.execution_contexts |= 1u64 << id; + inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); + (id, inner.context_generations[id]) + }; + let low = tagged_root(vm.ttb(), id); + let high = tagged_root(self.kernel_vm.ttb(), id); + // Allocate the complete host owner before either TTBAT half is published. + // ExecutionRoot releases an empty claimed slot if allocation fails. + let context = Arc::new( + ExecutionContext { + root: ExecutionRoot { + inner: self.inner.clone(), + id: id as u8, + generation, + low, + high, + }, + vm: vm.clone(), + }, + GFP_KERNEL, + )?; + { + let inner = self.inner.lock(); + let roots = &inner.ttbs()[id]; + if roots.ttb0.load(Ordering::Acquire) != 0 || roots.ttb1.load(Ordering::Acquire) != 0 { + return Err(EBUSY); + } + roots.ttb0.store(low, Ordering::Release); + roots.ttb1.store(high, Ordering::Release); + } + fence(Ordering::SeqCst); + mem::tlbi_asid(id as u8); + mem::sync(); + Ok(context) + } + /// Creates a new `Vm` linked to this UAT. pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { Vm::new( @@ -2649,6 +3357,106 @@ impl Uat { self.geometry } + /// Adopts the top-level entries that the primary firmware instance published in the kernel + /// root while booting ([`UatFirmware::PublishedRoots`]). + /// + /// The entries below the kernel window belong to the firmware, and the kernel window entry + /// points at a table that the bootloader reserved, so the driver maps kernel objects beneath + /// it and never frees it. Entries above the kernel window do not belong to anyone and are + /// cleared. Must run after the primary instance booted and before anything is mapped into the + /// kernel `Vm`. + pub(crate) fn adopt_firmware_root_entries(&self) -> Result<[u64; 3]> { + let owned = IOVA_KERN_TOP_LEVEL_INDEX as usize + 1; + let mut snapshot = [0; 3]; + // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives + // this `Uat`. + unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { + for (value, pte) in snapshot.iter_mut().zip(ptes) { + *value = pte.load(Ordering::Relaxed); + } + for pte in &ptes[owned..] { + pte.store(0, Ordering::Relaxed); + } + }) + }?; + + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + Ok(snapshot) + } + + /// Checks that allocation under the shared subtree did not replace a firmware-owned entry. + pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { + // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. + let matches = unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { + ptes.iter() + .zip(snapshot) + .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) + }) + }?; + if !matches { + return Err(EIO); + } + Ok(()) + } + + /// Copies the kernel root's top-level entries from the kernel window up into the root of the + /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are + /// private to that instance. + pub(crate) fn mirror_secondary_root(&self) -> Result { + let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; + let count = UAT_PGSZ / size_of::(); + let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; + + // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives + // this `Uat`. + unsafe { + pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { + pgtable::with_root_entries(secondary, count, |secondary| { + for (src, dst) in primary.iter().zip(secondary).skip(first) { + let pte = src.load(Ordering::Relaxed); + if pte != 0 && dst.load(Ordering::Relaxed) != pte { + dst.store(pte, Ordering::Relaxed); + } + } + }) + }) + }??; + + fence(Ordering::SeqCst); + mem::tlbi_all(); + mem::sync(); + Ok(()) + } + + /// Activates the device-global render root in context 1 after the boot records retire. + /// Both contexts must still contain their original kernel roots. + pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { + let inner = self.inner.lock(); + inner.handoff().lock(); + let ttbs = inner.ttbs(); + let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { + ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) + && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) + }); + if !ready { + inner.handoff().unlock(); + return Err(EIO); + } + ttbs[KERNEL_ALIAS_CTX] + .ttb0 + .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); + inner.handoff().unlock(); + core::mem::drop(inner); + fence(Ordering::SeqCst); + mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); + mem::sync(); + Ok(()) + } + /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] fn make_inner(dev: &driver::AsahiDevice, handoff_mode: HandoffMode, m3: bool) -> Result> { @@ -2673,6 +3481,7 @@ impl Uat { let shared_fault = fault.clone(); Arc::pin_init( try_pin_init!(UatInner { + firmware, m3_running: handoff_mode == HandoffMode::FirmwareM3, fault, firmware_cache_flush_ready: AtomicBool::new( @@ -2685,6 +3494,9 @@ impl Uat { UatShared { fault: shared_fault, kernel_ttb1: 0, + execution_contexts: !EXECUTION_CONTEXT_MASK, + context_generations: [0; UAT_NUM_CTX], + secondary_ttb1: None, map_kernel_to_user: false, handoff_rgn, ttbs_rgn, @@ -2748,6 +3560,7 @@ impl Uat { dev: &driver::AsahiDevice, geometry: UatGeometry, map_kernel_to_user: bool, + firmware: UatFirmware, ) -> Result { Self::new_with_handoff_mode(dev, cfg, map_kernel_to_user, HandoffMode::FirmwareDekker) } @@ -2824,6 +3637,16 @@ impl Uat { let required_pagetables_size = PAGETABLES_SIZE; if ttb1size < required_pagetables_size { + let (pagetables_size, secondary_ttb1) = match firmware { + UatFirmware::Handoff => (PAGETABLES_SIZE, None), + UatFirmware::PublishedRoots { + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), + }; + if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); return Err(ENOMEM); } @@ -2848,6 +3671,8 @@ impl Uat { } let kernel_vm = Vm::new(dev, inner.clone(), kernel_range, cfg, Some(ttb1), 0)?; let kernel_lower_vm = Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_lower_vm = + Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; let kernel_vm = Vm::new( dev, inner.clone(), @@ -2880,6 +3705,7 @@ impl Uat { inner.map_kernel_to_user = map_kernel_to_user; inner.kernel_ttb1 = ttb1; + inner.secondary_ttb1 = secondary_ttb1; inner.handoff().init(handoff_mode, cfg.m3_owner())?; @@ -2898,6 +3724,17 @@ impl Uat { } drop(handoff_guard); + if secondary_ttb1.is_some() { + let alias = &ttbs[KERNEL_ALIAS_CTX]; + alias + .ttb0 + .store(tagged_root(ttb0, KERNEL_ALIAS_CTX), Ordering::SeqCst); + alias + .ttb1 + .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); + } + + inner.handoff().unlock(); core::mem::drop(inner); diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs new file mode 100644 index 00000000000000..0d5abcf9b84f1a --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Prepared userspace mapping transactions. Host callback resources are reserved for the +//! complete ioctl before its first visible mapping changes; leaf commits retain their +//! individual interruptible GPUVM reservation acquisition and quarantine admission check. + +use super::*; + +pub(crate) struct PreparedUserMap { + gem: ARef, + ctx: StepContext, + addr: u64, + size: u64, + offset: u64, + single_page: bool, +} + +pub(crate) struct PreparedUserUnmap { + pub(super) iova: u64, + pub(super) size: u64, + ctx: StepContext, +} + +pub(super) enum PreparedUserBindOp { + Map(PreparedUserMap), + Unmap(PreparedUserUnmap), +} + +pub(crate) struct PreparedUserBindBatch { + pub(super) ops: KVec, +} + +impl PreparedUserBindBatch { + pub(crate) fn new(capacity: usize) -> Result { + Ok(Self { + ops: KVec::with_capacity(capacity, GFP_KERNEL)?, + }) + } + pub(crate) fn push_map(&mut self, map: PreparedUserMap) -> Result { + Ok(self.ops.push(PreparedUserBindOp::Map(map), GFP_KERNEL)?) + } + pub(crate) fn push_unmap(&mut self, unmap: PreparedUserUnmap) -> Result { + Ok(self + .ops + .push(PreparedUserBindOp::Unmap(unmap), GFP_KERNEL)?) + } +} + +impl Vm { + /// Prepares host state without taking DMA reservations or changing any leaf. + #[allow(clippy::too_many_arguments)] + pub(crate) fn prepare_bind_object( + &self, + gem: &gem::Object, + addr: u64, + size: u64, + offset: u64, + prot: Prot, + single_page: bool, + ) -> Result { + let mut ctx = StepContext { + new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prot, + ..Default::default() + }; + let vm_bo = self.inner.obtain_bo(gem)?; + { + let mut bo = vm_bo.inner().inner.lock(); + if bo.sgt.is_none() { + let sgt = gem.owned_sg_table()?; + if bo.sg_vec.is_none() { + let mut segments = KVVec::new(); + let mut offset = 0; + for range in sgt.iter() { + let address = range.dma_address() as usize; + let length = range.dma_len() as usize; + segments.push((offset, address..address + length), GFP_KERNEL)?; + offset += length; + } + bo.sg_vec = Some(segments); + } + bo.sgt = Some(sgt); + } + } + ctx.vm_bo = Some(vm_bo); + if (addr | size | offset) & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + Ok(PreparedUserMap { + gem: gem.into(), + ctx, + addr, + size, + offset, + single_page, + }) + } + + pub(crate) fn prepare_user_unmap(&self, iova: u64, size: u64) -> Result { + Ok(PreparedUserUnmap { + iova, + size, + ctx: StepContext { + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + ..Default::default() + }, + }) + } + + /// Commits in ioctl order. Quarantine may defer an all-unmap batch, but can + /// never allow a map to replace a mapping still owned by failed work. + pub(crate) fn commit_prepared_user_bind_batch( + &self, + batch: &mut PreparedUserBindBatch, + ) -> Result { + if self.defer_bind_batch(batch)? { + return Ok(()); + } + let mut index = 0; + while index < batch.ops.len() { + let result = match &mut batch.ops[index] { + PreparedUserBindOp::Map(map) => { + let mut inner = self.inner.exec_lock(Some(&map.gem), true)?; + match self.mapping_commit() { + Ok(commit) => { + let (flags, repeat) = if map.single_page { + (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) + } else { + (gpuvm::GpuVaFlags::NONE, 0) + }; + let result = inner.sm_map( + &mut map.ctx, + map.addr, + map.size, + map.offset, + repeat, + flags, + ); + if result.is_ok() { + self.track_shared_binding( + &map.gem, + map.addr, + map.size, + map.offset, + map.single_page, + ); + } + drop(inner); + drop(commit); + Some(result) + } + Err(error) if error == EIO => { + drop(inner); + None + } + Err(error) => return Err(error), + } + } + PreparedUserBindOp::Unmap(unmap) => { + let mut inner = self.inner.exec_lock(None, true)?; + match self.mapping_commit() { + Ok(commit) => { + let result = inner.sm_unmap(&mut unmap.ctx, unmap.iova, unmap.size); + if result.is_ok() { + self.untrack_shared_range(unmap.iova, unmap.size); + } + drop(inner); + drop(commit); + Some(result) + } + Err(error) if error == EIO => { + drop(inner); + None + } + Err(error) => return Err(error), + } + } + }; + if let Some(result) = result { + result?; + index += 1; + } else if self.defer_bind_batch(batch)? { + return Ok(()); + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs new file mode 100644 index 00000000000000..c47b7ae1fd782c --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Tracks canonical context resources and retains their fixed low presentation. +//! +//! Userspace owns low views 1 through 6. Only the reserved null view is created +//! here. Its installation is serialized per VM and retained through VM teardown. + +use super::*; +use crate::hw::t8140::CONTEXT_VIEWS; +use kernel::drm::gem::BaseObject; + +struct Binding { + gem: ARef, + offset: usize, +} +struct Bindings { + generation: u64, + slots: [Option; CONTEXT_VIEWS.len()], +} +#[pin_data] +pub(super) struct ContextBindings { + #[pin] + bindings: Mutex, + #[pin] + install: Mutex<()>, +} +impl ContextBindings { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + bindings <- new_mutex!(Bindings { generation: 0, + slots: core::array::from_fn(|_| None) }, "VM context resources"), + install <- new_mutex!((), "VM context alias installation"), + }), + GFP_KERNEL, + ) + } +} + +impl Vm { + pub(crate) fn validate_context_binding( + &self, + address: u64, + size: u64, + single_page: bool, + ) -> Result { + if self.context_bindings.is_none() || !single_page { + return Ok(()); + } + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + if CONTEXT_VIEWS + .iter() + .any(|view| address < view.source + view.size as u64 && view.source < end) + { + return Err(EINVAL); + } + Ok(()) + } + + /// Arithmetic is preflighted before invalidating any retained candidate. + pub(crate) fn track_context_binding( + &self, + gem: ARef, + address: u64, + size: u64, + object_offset: u64, + ) -> Result { + let Some(context) = self.context_bindings.as_ref() else { + return Ok(()); + }; + let end = address.checked_add(size).ok_or(EOVERFLOW)?; + let mut offsets = [None; CONTEXT_VIEWS.len()]; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address <= view.source && view.source + view.size as u64 <= end { + offsets[index] = Some(usize::try_from( + object_offset + .checked_add(view.source - address) + .ok_or(EOVERFLOW)?, + )?); + } + } + let mut bindings = context.bindings.lock(); + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address < view.source + view.size as u64 && view.source < end { + bindings.slots[index] = None; + touched = true; + } + if let Some(offset) = offsets[index] { + bindings.slots[index] = Some(Binding { + gem: gem.clone(), + offset, + }); + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + Ok(()) + } + + pub(crate) fn untrack_context_range(&self, range: Range) { + let Some(context) = self.context_bindings.as_ref() else { + return; + }; + let mut bindings = context.bindings.lock(); + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if range.start < view.source + view.size as u64 && view.source < range.end { + bindings.slots[index] = None; + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + } + + pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { + let Some(context) = self.context_bindings.as_ref() else { + return; + }; + let mut bindings = context.bindings.lock(); + let mut touched = false; + for slot in &mut bindings.slots { + if slot + .as_ref() + .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)) + { + *slot = None; + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); + } + } + + pub(crate) fn translate_iova(&self, address: u64) -> Result { + self.inner + .exec_lock(None, false)? + .page_table + .translate_iova(address) + } + + /// Requires a job mapping guard before validating or installing these views. + pub(crate) fn install_job_context_aliases(&self) -> Result { + let context = self.context_bindings.as_ref().ok_or(EINVAL)?; + let (generation, snapshot) = { + let bindings = context.bindings.lock(); + let mut snapshot = KVec::with_capacity(CONTEXT_VIEWS.len(), GFP_KERNEL)?; + for slot in &bindings.slots { + let binding = slot.as_ref().ok_or(ENOENT)?; + snapshot.push((binding.gem.clone(), binding.offset), GFP_KERNEL)?; + } + (bindings.generation, snapshot) + }; + for (index, (gem, offset)) in snapshot.iter().enumerate().skip(1) { + let view = CONTEXT_VIEWS[index]; + if !self.covers_range(view.source, view.size as u64, true, view.write) + || !self.covers_range(view.low, view.size as u64, true, view.write) + { + return Err(EFAULT); + } + if offset.checked_add(view.size).ok_or(EOVERFLOW)? > gem.size() { + return Err(ERANGE); + } + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + } + let _install = context.install.lock(); + let view = CONTEXT_VIEWS[0]; + let (gem, offset) = &snapshot[0]; + // Null presentation requires writes in addition to the canonical view's + // ordinary read requirement. + if !self.covers_range(view.source, view.size as u64, true, true) { + return Err(EFAULT); + } + let end = offset.checked_add(view.size).ok_or(EOVERFLOW)?; + if end > gem.size() { + return Err(ERANGE); + } + let mut existing = 0; + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.covers_range(view.low + page as u64, UAT_PGSZ as u64, true, true) { + existing += 1; + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + } + if existing != 0 { + if existing != view.size / UAT_PGSZ { + return Err(EFAULT); + } + return if context.bindings.lock().generation == generation { + Ok(()) + } else { + Err(EAGAIN) + }; + } + let mut mappings = KVec::with_capacity(1, GFP_KERNEL)?; + let mapping = self.map_in_range( + gem, + *offset..end, + UAT_PGSZ as u64, + view.low..view.low + view.size as u64, + PROT_GPU_SHARED_RW, + false, + )?; + for page in (0..view.size).step_by(UAT_PGSZ) { + if self.translate_iova(view.source + page as u64)? + != self.translate_iova(view.low + page as u64)? + { + return Err(EFAULT); + } + } + mappings.push(mapping, GFP_KERNEL)?; + if context.bindings.lock().generation != generation { + return Err(EAGAIN); + } + self.append_driver_mappings(mappings) + } +} diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs new file mode 100644 index 00000000000000..3b8a5e5a40aeb9 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Keeps user mappings alive until accepted work has relinquished firmware ownership. +//! +//! The state lock is taken after GPUVM's exec lock when committing user mappings. Waiting for +//! that lock's commits never holds a GPUVM reservation. Draining takes no state lock while it +//! unmaps memory. Driver aliases retain only the inner GPUVM, so they create no cycle with Vm. + +use super::*; +use kernel::{ + new_condvar, + sync::{CondVar, CondVarTimeoutResult}, + time::msecs_to_jiffies, +}; + +const ADMISSION_WAIT_MS: u32 = 1000; + +pub(super) struct DriverMappings { + mappings: KVec, + reserved: KVec>, + compute: Option, + render: [Option; crate::hw::t8140::queues::RENDER_SLOTS], +} + +struct PoolMappings { + id: u64, + blocks: usize, + mappings: KVec, +} + +impl DriverMappings { + fn ranges(&self) -> impl Iterator> + '_ { + Iterator::chain( + Iterator::chain( + self.mappings.iter(), + Iterator::chain( + self.compute.iter().flat_map(|pool| pool.mappings.iter()), + self.render + .iter() + .flatten() + .flat_map(|pool| pool.mappings.iter()), + ), + ) + .map(KernelMapping::iova_range), + self.reserved.iter().cloned(), + ) + } +} + +struct State { + active: usize, + commits: usize, + blocked: bool, + quarantined: bool, + draining: bool, + closed: bool, + close_ranges: Option<(Range, Range)>, + unmaps: KVec<(u64, u64)>, + objects: KVec>, +} + +#[pin_data] +pub(super) struct VmLifetime { + #[pin] + state: Mutex, + #[pin] + changed: CondVar, + #[pin] + driver: Mutex>, +} + +impl VmLifetime { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State { active:0, commits:0, blocked:false, + quarantined:false, draining:false, closed:false, close_ranges:None, + unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), + changed <- new_condvar!("VM mapping admission"), + driver <- new_mutex!(None, "VM driver mappings"), + }), + GFP_KERNEL, + ) + } +} + +/// Retains a VM's mappings through a command's exact retirement. An error fence is not +/// retirement: potentially visible work must retain its guard through quarantine or stop. +pub(crate) struct VmJobGuard { + vm: Vm, +} + +impl VmJobGuard { + /// Blocks mapping commits before the error fence can become visible to userspace. + pub(crate) fn quarantine(&self) { + if let Some(lifetime) = self.vm.lifetime.as_ref() { + let mut state = lifetime.state.lock(); + state.blocked = true; + while state.commits != 0 { + lifetime.changed.wait(&mut state); + } + state.quarantined = true; + } + } +} + +pub(super) struct MappingCommit { + lifetime: Arc, +} +impl Drop for MappingCommit { + fn drop(&mut self) { + let mut state = self.lifetime.state.lock(); + state.commits -= 1; + if state.commits == 0 { + self.lifetime.changed.notify_all(); + } + } +} + +impl Vm { + /// Attaches aliases while constructing this VM, before exposing it to userspace. + pub(crate) fn install_driver_mappings( + &self, + mappings: KVec, + reserved: KVec>, + ) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + if driver.is_some() { + return Err(EBUSY); + } + *driver = Some(DriverMappings { + mappings, + reserved, + compute: None, + render: core::array::from_fn(|_| None), + }); + Ok(()) + } + + /// Reserves the complete append before moving aliases into VM ownership. + pub(crate) fn append_driver_mappings(&self, mappings: KVec) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + driver.mappings.reserve(mappings.len(), GFP_KERNEL)?; + for mapping in mappings { + driver.mappings.push(mapping, GFP_KERNEL)?; + } + Ok(()) + } + + pub(crate) fn render_pool_tvb_blocks(&self, slot: u8, id: u64) -> Option { + if id == 0 { + return None; + } + self.lifetime + .as_ref()? + .driver + .lock() + .as_ref()? + .render + .get(usize::from(slot))? + .as_ref() + .filter(|pool| pool.id == id) + .map(|pool| pool.blocks) + } + + /// Retains a new generation after all references to a replaced pool retire. + pub(crate) fn install_render_pool_mappings( + &self, + slot: u8, + id: u64, + blocks: usize, + mappings: KVec, + ) -> Result { + if id == 0 { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; + if pool.is_some() { + return Err(EBUSY); + } + *pool = Some(PoolMappings { + id, + blocks, + mappings, + }); + Ok(()) + } + + /// Consumes aliases only after the exact cached owner and complete append + /// capacity have been checked. Failed preparation retains its mappings for + /// destruction after the caller releases the device mutex. + pub(crate) fn append_render_pool_mappings_prepared( + &self, + slot: u8, + id: u64, + previous: usize, + blocks: usize, + mappings: &mut KVec, + ) -> Result { + if id == 0 || blocks <= previous || mappings.len() != blocks - previous { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let pool = Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .and_then(Option::as_mut) + .filter(|pool| pool.id == id) + .ok_or(EINVAL)?; + if pool.blocks != previous { + return Err(EBUSY); + } + pool.mappings + .len() + .checked_add(mappings.len()) + .ok_or(EOVERFLOW)?; + pool.mappings.reserve(mappings.len(), GFP_KERNEL)?; + for mapping in mappings.drain_all() { + pool.mappings.push(mapping, GFP_KERNEL)?; + } + pool.blocks = blocks; + Ok(()) + } + + /// A delayed teardown must not remove a successor generation. Remove + /// under the cache lock, then unmap after dropping that lock. + pub(crate) fn clear_render_pool_mappings_if_owner(&self, slot: u8, id: u64) { + if id == 0 { + return; + } + let previous = self.lifetime.as_ref().and_then(|lifetime| { + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .filter(|pool| pool.as_ref().is_some_and(|pool| pool.id == id)) + .and_then(Option::take) + }); + drop(previous); + } + + pub(crate) fn compute_pool_mappings_match(&self, id: u64) -> bool { + id != 0 + && self.lifetime.as_ref().is_some_and(|lifetime| { + lifetime + .driver + .lock() + .as_ref() + .is_some_and(|driver| driver.compute.as_ref().is_some_and(|pool| pool.id == id)) + }) + } + + /// The device publication lock serializes first-use alias construction. + /// Never replace a live generation; its jobs must retire before clearing. + pub(crate) fn install_compute_pool_mappings( + &self, + id: u64, + mappings: KVec, + ) -> Result { + if id == 0 { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; + if driver.compute.is_some() { + return Err(EBUSY); + } + driver.compute = Some(PoolMappings { + id, + blocks: 0, + mappings, + }); + Ok(()) + } + + /// Called under the publication lock after all obsolete pool work retires. + /// Drop outside the driver lock because mapping teardown acquires GPUVM locks. + pub(crate) fn clear_compute_pool_mappings(&self) { + let previous = self.lifetime.as_ref().and_then(|lifetime| { + let mut driver = lifetime.driver.lock(); + Option::as_mut(&mut *driver).and_then(|driver| driver.compute.take()) + }); + drop(previous); + } + + /// Tests mappings and reservations which userspace must never replace. + pub(crate) fn driver_range_overlaps(&self, range: Range) -> bool { + let Some(lifetime) = self.lifetime.as_ref() else { + return false; + }; + let driver = lifetime.driver.lock(); + driver.as_ref().is_some_and(|driver| { + driver + .ranges() + .any(|reserved| range.start < reserved.end && reserved.start < range.end) + }) + } + + /// Waits before reservation acquisition, allowing healthy retirement to drain. + pub(crate) fn wait_for_user_map_admission(&self) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(()); + }; + let mut state = lifetime.state.lock(); + let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); + loop { + let failed = self.status.as_ref().is_some_and(|status| status.get() != 0); + if !failed + && !state.closed + && (state.blocked || (state.active != 0 && state.quarantined)) + && remaining != 0 + { + remaining = match lifetime + .changed + .wait_interruptible_timeout(&mut state, remaining) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + continue; + } + if state.closed { + return Err(ENOENT); + } + if failed || state.blocked || (state.active != 0 && state.quarantined) { + return Err(EIO); + } + if !state.draining { + return Ok(()); + } + if lifetime.changed.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + } + + /// Acquires a job reference without imposing a VM-wide limit on healthy work. + pub(crate) fn retain_job(&self) -> Result { + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut state = lifetime.state.lock(); + loop { + if state.closed { + return Err(ENOENT); + } + if state.blocked { + return Err(EIO); + } + if !state.draining { + state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(VmJobGuard { vm: self.clone() }); + } + lifetime.changed.wait(&mut state); + } + } + + /// The first job of a submission may wait for a clean VM's quarantine drain. + pub(crate) fn retain_first_job(&self) -> Result { + match self.retain_job() { + Err(error) + if error == EIO && self.status.as_ref().is_some_and(|status| status.get() == 0) => { + } + result => return result, + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); + let mut state = lifetime.state.lock(); + while state.blocked + && remaining != 0 + && self.status.as_ref().is_some_and(|status| status.get() == 0) + { + remaining = match lifetime + .changed + .wait_interruptible_timeout(&mut state, remaining) + { + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => 0, + CondVarTimeoutResult::Woken { jiffies } => jiffies, + }; + } + drop(state); + self.retain_job() + } + + /// Called only after the GPUVM exec/reservation locks have been acquired. + pub(super) fn mapping_commit(&self) -> Result> { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(None); + }; + let mut state = lifetime.state.lock(); + if state.closed { + return Err(ENOENT); + } + if state.blocked { + return Err(EIO); + } + state.commits = state.commits.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Some(MappingCommit { + lifetime: lifetime.clone(), + })) + } + + /// Reserves cleanup bookkeeping before a batch starts committing leaves. + pub(crate) fn reserve_deferred_user_unmaps(&self, count: usize) -> Result { + if let Some(lifetime) = self.lifetime.as_ref() { + lifetime.state.lock().unmaps.reserve(count, GFP_KERNEL)?; + } + Ok(()) + } + + pub(super) fn defer_bind_batch(&self, batch: &PreparedUserBindBatch) -> Result { + use super::bind::PreparedUserBindOp; + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(false); + }; + let mut state = lifetime.state.lock(); + if !state.blocked { + return Ok(false); + } + if state.closed { + return Err(ENOENT); + } + if batch + .ops + .iter() + .any(|op| matches!(op, PreparedUserBindOp::Map(_))) + { + return Err(EIO); + } + if !state.quarantined { + return Err(EBUSY); + } + for op in &batch.ops { + if let PreparedUserBindOp::Unmap(unmap) = op { + let range = (unmap.iova, unmap.size); + if !state.unmaps.contains(&range) { + state.unmaps.push(range, GFP_KERNEL)?; + } + } + } + Ok(true) + } + + /// Returns true if a quarantined owner requires this unmap to be deferred. + pub(super) fn defer_unmap(&self, iova: u64, size: u64) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return Ok(false); + }; + let mut state = lifetime.state.lock(); + if state.closed { + return Err(ENOENT); + } + if !state.blocked { + return Ok(false); + } + if !state.quarantined { + return Err(EBUSY); + } + if !state.unmaps.contains(&(iova, size)) { + state.unmaps.push((iova, size), GFP_KERNEL)?; + } + Ok(true) + } + + /// Defers handle-close cleanup while any accepted job can still use the BO. + pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { + let Some(lifetime) = self.lifetime.as_ref() else { + return self.drop_mappings_now(gem); + }; + let mut state = lifetime.state.lock(); + if !state + .objects + .iter() + .any(|object| core::ptr::eq(&**object, gem)) + { + state.objects.push(gem.into(), GFP_KERNEL)?; + } + if state.active != 0 || state.draining { + return Ok(()); + } + state.draining = true; + drop(state); + self.drain_mappings(lifetime); + Ok(()) + } + + /// Closes admission permanently and preserves mappings referenced by accepted jobs. + pub(crate) fn unmap_user_ranges(&self, user: Range, kernel: Range) -> Result { + if let Some(lifetime) = self.lifetime.as_ref() { + let mut state = lifetime.state.lock(); + state.blocked = true; + while state.commits != 0 { + lifetime.changed.wait(&mut state); + } + state.closed = true; + if state.active != 0 { + state.close_ranges = Some((user, kernel)); + return Ok(()); + } + } + self.unmap_user_ranges_now(user, kernel) + } + + fn unmap_user_ranges_now(&self, user: Range, kernel: Range) -> Result { + let driver = self + .lifetime + .as_ref() + .map(|lifetime| lifetime.driver.lock()); + let mut cursor = user.start; + while cursor < user.end { + let mut next = user.end; + let mut covered = cursor; + for range in Iterator::chain( + core::iter::once(kernel.clone()), + driver + .as_ref() + .and_then(|guard| guard.as_ref()) + .into_iter() + .flat_map(DriverMappings::ranges), + ) { + if range.contains(&cursor) { + covered = covered.max(range.end); + } else if cursor < range.start { + next = next.min(range.start); + } + } + if covered != cursor { + cursor = covered.min(user.end); + continue; + } + if next <= cursor { + return Err(EIO); + } + self.unmap_range_now(cursor, next - cursor)?; + cursor = next; + } + Ok(()) + } +} + +impl Drop for VmJobGuard { + fn drop(&mut self) { + let Some(lifetime) = self.vm.lifetime.as_ref() else { + return; + }; + let mut state = lifetime.state.lock(); + state.active -= 1; + if state.active != 0 { + return; + } + state.draining = true; + drop(state); + self.vm.drain_mappings(lifetime); + } +} + +impl Vm { + /// Admission stays closed to new jobs while the exec lock is acquired for cleanup. + /// Recheck queued cleanup and reopen under the same lock used to append it. + fn drain_mappings(&self, lifetime: &VmLifetime) { + let mut state = lifetime.state.lock(); + loop { + let close = state.close_ranges.take(); + let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); + let objects = core::mem::replace(&mut state.objects, KVec::new()); + drop(state); + if let Some((user, kernel)) = close { + if self.unmap_user_ranges_now(user, kernel).is_err() { + pr_err!("MMU: deferred VM close failed\n"); + } + } else { + for (address, size) in unmaps { + if self.unmap_range_now(address, size).is_err() { + pr_err!("MMU: deferred user unmap failed\n"); + } + } + } + for object in objects { + if self.drop_mappings_now(&object).is_err() { + pr_err!("MMU: deferred object unmap failed\n"); + } + } + state = lifetime.state.lock(); + if state.close_ranges.is_some() || !state.unmaps.is_empty() || !state.objects.is_empty() + { + continue; + } + state.quarantined = false; + state.draining = false; + if !state.closed { + state.blocked = false; + } + drop(state); + lifetime.changed.notify_all(); + break; + } + self.bo_deferred_cleanup(); + } +} diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs new file mode 100644 index 00000000000000..cf29e53fc9cc98 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Shared-object identities for attachment ordering across user address spaces. +//! These records do not own mappings. The VM's job guard protects the mappings +//! while submission captures the intersecting object offsets. Mapping commits +//! hold the GPUVM exec lock through metadata updates, then release it. The lock +//! order is GPUVM exec followed by bindings; footprint readers take only bindings. + +use super::*; +use crate::g17::{hazard::Footprint, Attachments}; + +struct Binding { + start: u64, + end: u64, + object: usize, + offset: u64, +} + +#[pin_data] +pub(super) struct SharedBindings { + #[pin] + bindings: Mutex>, + count: AtomicU32, +} + +impl SharedBindings { + pub(super) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + bindings <- new_mutex!(KVec::new(), "Shared attachment bindings"), + count: AtomicU32::new(0), + }), + GFP_KERNEL, + ) + } +} + +impl Vm { + pub(super) fn track_shared_binding( + &self, + gem: &gem::Object, + address: u64, + size: u64, + offset: u64, + single_page: bool, + ) { + let Some(shared) = &self.shared_bindings else { + return; + }; + let Some(end) = address.checked_add(size) else { + return; + }; + let external = !single_page && self.is_extobj(gem); + if !external && shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.end <= address || end <= entry.start); + if external { + let _ = bindings.push( + Binding { + start: address, + end, + object: gem as *const gem::Object as usize, + offset, + }, + GFP_KERNEL, + ); + } + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(super) fn untrack_shared_range(&self, address: u64, size: u64) { + let Some(shared) = &self.shared_bindings else { + return; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let Some(end) = address.checked_add(size) else { + return; + }; + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.end <= address || end <= entry.start); + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(crate) fn untrack_shared_object(&self, gem: &gem::Object) { + let Some(shared) = &self.shared_bindings else { + return; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + return; + } + let object = gem as *const gem::Object as usize; + let mut bindings = shared.bindings.lock(); + bindings.retain(|entry| entry.object != object); + shared.count.store(bindings.len() as u32, Ordering::Relaxed); + } + + pub(crate) fn render_footprint( + &self, + attachments: &Attachments, + ) -> Result>> { + if attachments.as_slice().is_empty() { + return Ok(None); + } + let mut footprint = Footprint::new(self.id); + for attachment in attachments.as_slice() { + let end = attachment.va.saturating_add(attachment.size); + footprint.push_vm(attachment.va, end); + let Some(shared) = &self.shared_bindings else { + continue; + }; + if shared.count.load(Ordering::Relaxed) == 0 { + continue; + } + let bindings = shared.bindings.lock(); + for entry in bindings.iter() { + let first = entry.start.max(attachment.va); + let last = entry.end.min(end); + if first < last { + let offset = entry.offset.saturating_add(first - entry.start); + footprint.push_object( + entry.object, + offset, + offset.saturating_add(last - first), + ); + } + } + } + Ok(Some(Arc::new(footprint, GFP_KERNEL)?)) + } +} diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index ea2c806ab9890f..d0ca5801e81787 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -59,6 +59,32 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; +/// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. +/// +/// # Safety +/// +/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` +/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they +/// are only exposed as atomics. +pub(crate) unsafe fn with_root_entries( + ttb: PhysicalAddr, + count: usize, + f: impl FnOnce(&[AtomicU64]) -> R, +) -> Result { + if count > UAT_NPTE { + return Err(EINVAL); + } + // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. + let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; + page.with_pointer_into_page(0, count * PTE_SIZE, |p| { + // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page + // table entries are naturally aligned. + Ok(f(unsafe { + core::slice::from_raw_parts(p as *const AtomicU64, count) + })) + }) +} + const PTE_TYPE_BITS: u64 = 3; const PTE_TYPE_LEAF_TABLE: u64 = 3; @@ -997,6 +1023,59 @@ impl UatPageTable { unsafe { Page::from_phys(phys) }; } + /// Checks every page, including holes skipped by the page-table walker. + pub(crate) fn covers_range( + &mut self, + range: Range, + read: bool, + write: bool, + ) -> Result { + if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { + return Ok(false); + } + let expected = (range.end - range.start) >> UAT_PGBIT; + let mut visited = 0u64; + let mut permitted = true; + self.with_pages(range, false, false, |_, ptes| { + visited = visited.checked_add(ptes.len() as u64).ok_or(EOVERFLOW)?; + for pte in ptes { + let value = pte.load(Ordering::Acquire); + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE + || !Prot::from_pte(value).allows_gpu(read, write) + { + permitted = false; + } + } + Ok(()) + })?; + Ok(expected == visited && permitted) + } + + /// Resolves an existing leaf without creating page tables. + pub(crate) fn translate_iova(&mut self, iova: u64) -> Result { + let entry = self.leaf_entry(iova & !(UAT_PGMSK as u64))?; + Ok((entry & self.oas_mask & !(UAT_PGMSK as u64)) | (iova & UAT_PGMSK as u64)) + } + + /// Reads one mapped leaf entry without allocating missing tables. + pub(crate) fn leaf_entry(&mut self, iova: u64) -> Result { + if iova & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let end = iova.checked_add(UAT_PGSZ as u64).ok_or(EINVAL)?; + let mut value = 0; + self.with_pages(iova..end, false, false, |_, ptes| { + if let Some(pte) = ptes.first() { + value = pte.load(Ordering::Relaxed); + } + Ok(()) + })?; + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + return Err(EFAULT); + } + Ok(value) + } + pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); self.with_pages(iova_range, true, false, false, |_, _| Ok(())) diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 0219d8f803b73f..d116610bae3bc9 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -6,6 +6,9 @@ //! there is very little need to interact directly with GPU MMIO register. This module abstracts //! the few operations that require that, mainly reading the MMU fault status, reading GPU ID //! information, and starting the GPU firmware coprocessor. +//! +//! On G17, the driver also enables the bridge between the GPU and the fabric before the +//! coprocessors start, and enables dynamic power gating before the firmware is initialized. use crate::{hw, identity}; use kernel::{ @@ -19,6 +22,8 @@ use kernel::{ }, platform, prelude::*, + str::CStr, + sync::{aref::ARef, Arc}, // sync::aref::ARef, // time::Delta, }; @@ -34,6 +39,10 @@ const CPU_RUN: u32 = 0x1 << 4; // BIT(4) const FAULT_INFO: usize = 0x17030; +/// A nonzero low nibble permits writes through the G17 queue configuration port. +const G17_HOST_IRQ_SUMMARY: usize = 0xe01000; +const G17_HOST_POWERED: u32 = 0xf; + const ID_VERSION: usize = 0xd04000; const ID_UNK08: usize = 0xd04008; const ID_COUNTS_1: usize = 0xd04010; @@ -48,6 +57,22 @@ const CORE_MASKS_G14X: usize = 0xe01500; const FAULT_INFO_G14X: usize = 0xd8c0; const FAULT_ADDR_G14X: usize = 0xd8c8; +/// Cleared after `CPU_RUN` when stopping a G17 coprocessor. +const CPU_STOP_ACK: u32 = 0x1 << 5; // BIT(5) + +/// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. +const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; +const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) + +/// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. +const G17_GATING_CONTROL: usize = 0xd06030; +const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) + +/// These performance selectors must be sampled before firmware can gate the core registers. +const G17_PERF_CONTROL: usize = 0xe0141c; +const G17_PERF_MAP: usize = 0xe01480; +const G17_CORE_POWER: usize = 0xe01000; + // G15 (t6030) Fender registers. Fender = SGX + 0xd00000 on every AGX checked, so the ID block // above is Fender+0x4000 and the G14X core masks are Fender+0x101500. Everything // here is inside the 16 MiB `sgx` mapping. @@ -204,10 +229,66 @@ pub(crate) struct FaultInfo { pub(crate) reason: FaultReason, } +/// Identification of a G17 GPU. +pub(crate) struct G17Id { + /// GPU family field of the ID register. + pub(crate) family: u8, + /// GPU variant field of the ID register. + pub(crate) variant: u8, + /// Silicon revision. + pub(crate) gpu_rev: hw::GpuRevision, + /// Number of dies. + pub(crate) num_dies: u32, + /// Number of clusters, over all dies. + pub(crate) num_clusters: u32, + /// Number of cores per cluster. + pub(crate) num_cores: u32, + /// Enabled cores of each cluster. + pub(crate) core_masks: KVec, + /// Performance selector control, sampled before core power gating starts. + pub(crate) perf_control: u32, + /// Performance selector table indices, low and high nibbles. + pub(crate) perf_map: [u32; 2], +} + +/// CPU control registers of one GPU coprocessor. +pub(crate) struct CpuControl { + asc: Pin>>>, +} + +impl CpuControl { + /// Map the CPU control registers in the `name` register region. + pub(crate) fn new(pdev: &platform::Device, name: &CStr) -> Result { + let req = pdev.io_request_by_name(name).ok_or(EINVAL)?; + Ok(CpuControl { + asc: KBox::pin_init(req.iomap_sized::(), GFP_KERNEL)?, + }) + } + + /// Start the coprocessor CPU. + pub(crate) fn start(&self) -> Result { + let asc = self.asc.try_access().ok_or(ENODEV)?; + let val = asc.read32(CPU_CONTROL); + asc.write32(val | CPU_RUN, CPU_CONTROL); + Ok(()) + } + + /// Stop the coprocessor CPU. This only gates it: its firmware keeps its state. + pub(crate) fn stop(&self) { + if let Some(asc) = self.asc.try_access() { + let val = asc.read32(CPU_CONTROL); + asc.write32(val & !CPU_RUN, CPU_CONTROL); + let val = asc.read32(CPU_CONTROL); + asc.write32(val & !CPU_STOP_ACK, CPU_CONTROL); + } + } +} + /// Device resources for this GPU instance. +#[derive(Clone)] pub(crate) struct Resources { dev: ARef, - sgx: Pin>>>, + sgx: Arc>>, } impl Resources { @@ -400,7 +481,7 @@ impl Resources { /// Map the required resources given our platform device. pub(crate) fn new(pdev: &platform::Device) -> Result { let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; - let sgx_iomem = KBox::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; + let sgx_iomem = Arc::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; Ok(Resources { // SAFETY: This device does DMA via the UAT IOMMU. @@ -409,6 +490,23 @@ impl Resources { }) } + /// Installs both render queues while the caller holds the device mutex. + /// No other port publisher may interleave between this power witness and + /// the final validity strobe. False means no write was attempted. + pub(crate) fn install_render_pair( + &self, + registration: &crate::g17::fw::kick::RenderRegistration, + ) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { + return Ok(false); + } + for &(offset, value) in registration.writes() { + sgx.try_write64(value, offset)?; + } + Ok(true) + } + fn sgx_read32(&self) -> u32 { if let Some(sgx) = self.sgx.try_access() { sgx.relaxed().read32(OFF) @@ -539,43 +637,9 @@ impl Resources { return Err(ENODEV); } - let mut core_masks = KVec::new(); - let mut total_active_cores: u32 = 0; - - let max_core_mask = ((1u64 << num_cores) - 1) as u32; - for _ in 0..num_clusters { - let mask = core_mask_regs[0] & max_core_mask; - core_masks.push(mask, GFP_KERNEL)?; - for i in 0..core_mask_regs.len() { - core_mask_regs[i] >>= num_cores; - if i < (core_mask_regs.len() - 1) { - core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); - } - } - total_active_cores += mask.count_ones(); - } - - if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - self.dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); - return Err(EIO); - } - - let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { - 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), - 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), - 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), - 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), - 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), - 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); - return Err(ENODEV); - } - }; + let (core_masks, total_active_cores) = + self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; + let (gpu_rev, gpu_rev_id) = self.gpu_revision(id_version)?; // Preserve the public G13/G14 identity independently of the firmware // core identity: the legacy ABI distinguishes Max and Ultra by clusters. @@ -623,6 +687,166 @@ impl Resources { }) } + /// Get the identification of a G17 GPU from its registers. + pub(crate) fn get_g17_id(&self) -> Result { + let id_version = self.sgx_read32::(); + let id_unk08 = self.sgx_read32::(); + let id_counts_1 = self.sgx_read32::(); + let id_counts_2 = self.sgx_read32::(); + let id_unk18 = self.sgx_read32::(); + let id_clusters = self.sgx_read32::(); + + dev_info!( + self.dev.as_ref(), + "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", + id_version, + id_unk08, + id_counts_1, + id_counts_2, + id_unk18, + id_clusters + ); + + // Later reads can fault while the firmware has gated the cores. Submission uses only + // these saved values and never re-reads the registers. + let perf_control = self.sgx_read32::() & 1; + let perf_map = self.sgx_read32::(); + let mut core_mask_regs = [ + self.sgx_read32::(), + self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), + self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), + ]; + dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_mask_regs); + + let num_dies = (id_counts_1 >> 16) & 0xf; + let num_clusters = ((id_counts_1 >> 8) & 0xff) * num_dies; + let num_cores = id_counts_1 & 0xff; + if num_cores == 0 || num_cores > 32 || num_cores * num_clusters > 3 * 32 { + dev_err!( + self.dev.as_ref(), + "Invalid core counts ({} x {})\n", + num_clusters, + num_cores + ); + return Err(ENODEV); + } + let (core_masks, _) = + self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; + let (gpu_rev, _) = self.gpu_revision(id_version)?; + + Ok(G17Id { + family: ((id_version >> 24) & 0xff) as u8, + variant: ((id_version >> 16) & 0xff) as u8, + gpu_rev, + num_dies, + num_clusters, + num_cores, + core_masks, + perf_control, + perf_map: [perf_map & 0xf, (perf_map >> 16) & 0xf], + }) + } + + /// Enable the G17 GPU-to-fabric bridge and check that it took effect. + /// + /// The GPU power domain must be on. The coprocessors cannot reach memory until this is done. + pub(crate) fn enable_g17_bridge(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + for offset in G17_BRIDGE_CONTROL { + let val = sgx.try_read32(offset)?; + sgx.try_write32(val | G17_BRIDGE_ENABLE, offset)?; + if sgx.try_read32(offset)? & G17_BRIDGE_ENABLE == 0 { + dev_err!( + self.dev.as_ref(), + "GPU bridge {:#x} did not enable\n", + offset + ); + return Err(EIO); + } + } + Ok(()) + } + + /// Disable the G17 GPU-to-fabric bridge. Both coprocessors must be stopped. + pub(crate) fn disable_g17_bridge(&self) { + if let Some(sgx) = self.sgx.try_access() { + for offset in G17_BRIDGE_CONTROL { + if let Ok(val) = sgx.try_read32(offset) { + // Cannot fail: the offset was read just before. + let _ = sgx.try_write32(val & !G17_BRIDGE_ENABLE, offset); + } + } + } + } + + /// Power witness for the queue-configuration port, accessible while the cores are gated. + pub(crate) fn g17_configuration_powered(&self) -> bool { + self.sgx_read32::() & 0xf != 0 + } + + /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is + /// initialized. + pub(crate) fn enable_g17_gating(&self) -> Result { + let sgx = self.sgx.try_access().ok_or(ENODEV)?; + let val = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) + } + + /// Split the packed per-cluster core masks into one mask per cluster. + /// + /// Returns the masks and the total number of enabled cores. + fn split_core_masks( + &self, + core_mask_regs: &mut [u32], + num_clusters: u32, + num_cores: u32, + ) -> Result<(KVec, u32)> { + let mut core_masks = KVec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.push(mask, GFP_KERNEL)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] = core_mask_regs[i].checked_shr(num_cores).unwrap_or(0); + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] + .checked_shl(32 - num_cores) + .unwrap_or(0); + } + } + total_active_cores += mask.count_ones(); + } + + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!( + self.dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); + return Err(EIO); + } + + Ok((core_masks, total_active_cores)) + } + + /// Decode the silicon revision from the ID version register. + fn gpu_revision(&self, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { + Ok(match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }) + } + /// Get the fault information from the MMU status register, if one occurred. pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { let g14x = cfg From 63db53cf16a0f5cb12b2766ad97c232a07dd1654 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:16 -0400 Subject: [PATCH 039/631] drm/syncobj: apply expiring capacity hints during blocking waits Associate a bounded utilization hint with a DRM file after successful submission. Apply it only around a blocking syncobj wait while it remains valid, and release the scheduler scope on every wait exit. Expose the hint setter to Rust DRM drivers. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7c99d8debcc54ebf3f76ede14ced9b845db18215) --- rust/kernel/drm/file.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust/kernel/drm/file.rs b/rust/kernel/drm/file.rs index 3603e473c4e0f6..c3263018880286 100644 --- a/rust/kernel/drm/file.rs +++ b/rust/kernel/drm/file.rs @@ -48,6 +48,14 @@ impl File { self.0.get() } + /// Tag recent feeder work for temporary capacity hints during syncobj waits. + /// User scheduling attributes and CPU affinity remain unchanged. + pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { + // SAFETY: this File keeps the opened drm_file alive; the C helper + // bounds both arguments and serializes the per-file hint internally. + unsafe { bindings::drm_syncobj_set_wait_hint(self.as_raw(), util_min, duration_ms) }; + } + fn driver_priv(&self) -> *mut T { // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. unsafe { (*self.as_raw()).driver_priv }.cast() From 0a2ad0078d1fe8842c7508a50a27caf3088645d7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 01:47:17 -0400 Subject: [PATCH 040/631] drm/asahi: request temporary CPU capacity for sustained GPU feeds Detect sustained render completion intervals and request temporary CPU capacity while submissions remain latency-sensitive. Retain detector ownership through completion, service the actuator on one work item, and remove its frequency requests after joining producers during shutdown. Provide an expiring syncobj wait hint only while the detector and applied capacity request agree. Keep this policy separate from firmware execution, queue depth and resource retirement. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bc4c50abf8ede16fec7f2a79546a310134b208df) --- drivers/gpu/drm/asahi/file.rs | 6 + drivers/gpu/drm/asahi/g17/completion.rs | 6 + drivers/gpu/drm/asahi/g17/feed.rs | 330 +++++++++++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 5 + drivers/gpu/drm/asahi/g17/mod.rs | 15 + drivers/gpu/drm/asahi/g17/queue/mod.rs | 2 + drivers/gpu/drm/asahi/g17/runtime.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/feed.rs | 33 ++ drivers/gpu/drm/asahi/gpu.rs | 5 +- 11 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/asahi/g17/feed.rs create mode 100644 drivers/gpu/drm/asahi/g17/runtime/feed.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index c4ea5a564c5f1e..8d07cb26abb9a7 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1407,6 +1407,12 @@ impl File { .lock() .submit(id, syncs, data.in_sync_count as usize, &vec, objects); + if ret.is_ok() { + if let Some((util_min, duration_ms)) = gpu.syncobj_wait_hint() { + file.set_syncobj_wait_hint(util_min, duration_ms); + } + } + match ret { Err(ERESTARTSYS) => Err(ERESTARTSYS), Err(e) => { diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 3ac8fe711ca619..1151a82084cee7 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -89,6 +89,7 @@ pub(crate) struct Completion { member: Member, destinations: [Option; 4], compute: Option, + feed: Option>, #[pin] vm_job: Mutex>, #[pin] @@ -111,6 +112,7 @@ impl Completion { objects: Pin<&xarray::XArray>>, timestamps: &[uapi::drm_asahi_timestamps], compute: bool, + feed: Option>, ) -> Result> { if timestamps.len() != if compute { 1 } else { 2 } { return Err(EINVAL); @@ -133,6 +135,7 @@ impl Completion { member, destinations, compute, + feed, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), @@ -309,6 +312,9 @@ impl Completion { if self.signalled.swap(true, Ordering::AcqRel) { return; } + if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { + feed.note_render_pass(ticks[2], ticks[3]); + } let result = result.and_then(|ticks| { for (destination, ticks) in self.destinations.iter().zip(ticks) { if let Some(destination) = destination { diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs new file mode 100644 index 00000000000000..ca3ecc85ee25a8 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Detects sustained host refill gaps and temporarily requests CPU capacity. +//! Detector updates are lock-free. The device's single feed worker owns the +//! frequency requests; its lock never nests inside a driver publication lock. +//! Shutdown first closes the detector, joins its producers and the worker, +//! then removes requests. All request storage and policy references are owned +//! by this device, independently of other GPU instances. + +use crate::hw::t8140::qos::CLOCK_HZ; +use core::{ + pin::Pin, + ptr, + sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, +}; +use kernel::{ + bindings, new_mutex, + prelude::*, + sync::{Arc, Mutex}, + time::{delay::fsleep, ClockSource, Delta, Monotonic}, + types::Opaque, +}; + +const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; +const STREAK_ON: u32 = 8; +const STREAK_MAX: u32 = 64; +const LINGER_NS: u64 = 8_000_000; +const POLL_MS: i64 = 2; +const FLOOR_PERCENT: u64 = 50; +const WAIT_UTIL_MIN: u32 = 512; +const WAIT_HINT_MS: u32 = 8; +const MAX_POLICIES: usize = 8; + +struct Detector { + last_end: AtomicU64, + idle: AtomicU64, + busy: AtomicU64, + streak: AtomicU32, + until: AtomicU64, + active: AtomicBool, + pending: AtomicBool, + stopping: AtomicBool, +} + +impl Detector { + const fn new() -> Self { + Self { + last_end: AtomicU64::new(0), + idle: AtomicU64::new(0), + busy: AtomicU64::new(0), + streak: AtomicU32::new(0), + until: AtomicU64::new(0), + active: AtomicBool::new(false), + pending: AtomicBool::new(false), + stopping: AtomicBool::new(false), + } + } + + fn decay_add(sum: &AtomicU64, add: u64) -> u64 { + let mut next = 0; + let _ = sum.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + next = value - value / 8 + add; + Some(next) + }); + next + } + + fn note(&self, start: u64, end: u64, now: u64) { + if end <= start { + return; + } + let previous = self.last_end.fetch_max(end, Ordering::AcqRel); + let (idle, busy, paced) = if previous == 0 { + (0, end - start, true) + } else if start >= previous { + let gap = start - previous; + if gap < MAX_REFILL_TICKS { + (gap, end - start, false) + } else { + (0, end - start, true) + } + } else if end > previous { + (0, end - previous, false) + } else { + (0, 0, false) + }; + let idle_sum = Self::decay_add(&self.idle, idle); + let busy_sum = Self::decay_add(&self.busy, busy); + let streak = if paced { + self.streak.store(0, Ordering::Release); + 0 + } else { + let previous = self + .streak + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { + Some((value + 1).min(STREAK_MAX)) + }); + // The closure always returns Some; fetch_update retries contention. + match previous { + Ok(value) => value + 1, + Err(_) => 0, + } + }; + if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { + self.until.store(now + LINGER_NS, Ordering::Release); + if !self.active.swap(true, Ordering::AcqRel) { + self.pending.store(true, Ordering::Release); + } + } + } + + fn wanted(&self, now: u64) -> bool { + !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) + } +} + +#[pin_data] +struct Requests { + #[pin] + requests: Opaque<[bindings::freq_qos_request; MAX_POLICIES]>, + policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], + maxima: [u32; MAX_POLICIES], + applied: [u32; MAX_POLICIES], + count: usize, +} + +// SAFETY: Each nonnull policy retains cpufreq_cpu_get's reference. Access to +// the pinned request array and metadata is exclusive under Feed::requests. +unsafe impl Send for Requests {} + +impl Requests { + fn new() -> Self { + Self { + requests: Opaque::zeroed(), + policies: [ptr::null_mut(); MAX_POLICIES], + maxima: [0; MAX_POLICIES], + applied: [0; MAX_POLICIES], + count: 0, + } + } + + fn register(self: Pin<&mut Self>) { + let this = self.project(); + if *this.count != 0 { + return; + } + // SAFETY: nr_cpu_ids is initialized before driver probe. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: Accepts any CPU number; a nonnull result owns one reference. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + let count = *this.count; + if count < MAX_POLICIES && !this.policies[..count].contains(&policy) { + // SAFETY: The policy is referenced. This zeroed, unregistered + // slot is pinned and remains alive until remove_request. + let result = unsafe { + bindings::freq_qos_add_request( + ptr::addr_of_mut!((*policy).constraints), + this.requests + .get() + .cast::() + .add(count), + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + ) + }; + if result >= 0 { + this.policies[count] = policy; + // SAFETY: cpuinfo is immutable while this policy is referenced. + this.maxima[count] = unsafe { (*policy).cpuinfo.max_freq }; + this.applied[count] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + *this.count += 1; + continue; + } + } + // SAFETY: Balances the reference acquired above for a duplicate, + // unavailable slot or failed registration. Successful slots retain it. + unsafe { bindings::cpufreq_cpu_put(policy) }; + } + } + + fn apply(mut self: Pin<&mut Self>, enabled: bool) -> bool { + if enabled { + self.as_mut().register(); + } + let this = self.project(); + for index in 0..*this.count { + let floor = if enabled { + (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 + } else { + 0 + }; + let value = if floor == 0 { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + } else { + floor + }; + if this.applied[index] == value { + continue; + } + this.applied[index] = value; + // SAFETY: This slot is registered, pinned and exclusively locked; + // its referenced policy keeps the constraints alive. + unsafe { + bindings::freq_qos_update_request( + this.requests + .get() + .cast::() + .add(index), + value as i32, + ) + }; + } + *this.count != 0 && enabled + } + + fn clear(self: Pin<&mut Self>) { + let this = self.project(); + for index in 0..*this.count { + // SAFETY: Registered pinned slot, exclusively locked. Remove the + // constraint link before releasing its policy reference. + unsafe { + bindings::freq_qos_remove_request( + this.requests + .get() + .cast::() + .add(index), + ); + bindings::cpufreq_cpu_put(this.policies[index]); + } + this.policies[index] = ptr::null_mut(); + } + *this.count = 0; + } +} + +#[pin_data(PinnedDrop)] +pub(crate) struct Feed { + detector: Detector, + #[pin] + requests: Mutex, + applied: AtomicBool, +} + +impl Feed { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + detector: Detector::new(), + requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + applied: AtomicBool::new(false), + }), + GFP_KERNEL, + ) + } + + fn now() -> u64 { + Monotonic::ktime_get() as u64 + } + + pub(crate) fn note_render_pass(&self, start: u64, end: u64) { + self.detector.note(start, end, Self::now()); + } + + /// Called after a firmware event pass, before queuing the single feed work item. + pub(crate) fn take_pending(&self) -> bool { + self.detector.pending.swap(false, Ordering::AcqRel) + } + + fn apply(&self, enabled: bool) { + let mut requests = self.requests.lock(); + let enabled = enabled && !self.detector.stopping.load(Ordering::Acquire); + let applied = requests.as_mut().apply(enabled); + self.applied.store(applied, Ordering::Relaxed); + } + + /// Worker context; the linger deadline has no firmware event of its own. + pub(crate) fn run(&self) { + loop { + if self.detector.wanted(Self::now()) { + self.apply(true); + fsleep(Delta::from_millis(POLL_MS)); + continue; + } + self.apply(false); + self.detector.active.store(false, Ordering::SeqCst); + // A detector can extend the deadline after observing the old active + // bit. Reclaim that work unless it already queued another owner. + if self.detector.wanted(Self::now()) + && !self.detector.active.swap(true, Ordering::AcqRel) + { + continue; + } + return; + } + } + + pub(crate) fn wait_hint(&self) -> Option<(u32, u32)> { + if cfg!(CONFIG_UCLAMP_TASK) + && self.detector.wanted(Self::now()) + && self.applied.load(Ordering::Relaxed) + { + Some((WAIT_UTIL_MIN, WAIT_HINT_MS)) + } else { + None + } + } + + /// Call before disabling/joining the event producer and actuator work items. + pub(crate) fn begin_shutdown(&self) { + self.detector.stopping.store(true, Ordering::Release); + } + + /// Call after work items are joined. Removing requests withdraws their floor. + pub(crate) fn shutdown(&self) { + self.begin_shutdown(); + self.requests.lock().as_mut().clear(); + self.applied.store(false, Ordering::Relaxed); + } +} + +#[pinned_drop] +impl PinnedDrop for Feed { + fn drop(self: Pin<&mut Self>) { + self.shutdown(); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index ed2e79537d597e..0d28b80baa6332 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -222,6 +222,11 @@ fn check_status(fence: &Fence) -> Result { /// The backend retains visible packets through exact retirement or processor stop. /// It performs all completion signalling after releasing the device mutex. pub(crate) trait Backend: Send + Sync { + /// Device detector retained by render completions after queue teardown. + fn feed(&self) -> Option> { + None + } + /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self) -> Result; /// Settle outstanding work of a terminal VM before an ioctl error is published. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index fe354a52968ccf..a574d540404793 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -25,6 +25,7 @@ mod completion; pub(crate) mod context; mod dependency; mod event; +mod feed; mod fence; mod freelist; pub(crate) mod fw; @@ -579,6 +580,7 @@ const POLL_WORK: u64 = 1; const IDLE_WORK: u64 = 2; const GROW_WORK: u64 = 3; const RECLAIM_WORK: u64 = 4; +const FEED_WORK: u64 = 5; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] @@ -587,6 +589,9 @@ struct Shared { clusters: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, + feed: Arc, + #[pin] + feed_work: Work, preparations: Arc, #[pin] work: Work, @@ -843,12 +848,15 @@ impl Gpu { let max_frequency_khz = Self::max_frequency_khz(dev)?; let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; + let feed = feed::Feed::new()?; let shared = Arc::pin_init( pin_init!(Shared { dev: dev.into(), clusters: id.num_clusters, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), + feed, + feed_work <- new_work!("g17::Shared::feed_work"), preparations, work <- new_work!("g17::Shared::work"), poll <- new_delayed_work!("g17::Shared::poll"), @@ -1004,6 +1012,7 @@ impl Gpu { /// Stops the firmware and releases everything it used. pub(crate) fn shutdown(&self) { + self.shared.feed.begin_shutdown(); self.shared.preparations.remove(); self.shared.stop_queues(); self.shared.disable_timer(&self.shared.poll); @@ -1011,11 +1020,17 @@ impl Gpu { self.shared.disable_work(&self.shared.grow); self.shared.disable_work(&self.shared.reclaim_work); self.shared.disable_work(&self.shared.work); + self.shared.disable_work(&self.shared.feed_work); + self.shared.feed.shutdown(); self.shared.drain_reclaims(); } } impl gpu::Gpu for Gpu { + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + self.shared.feed.wait_hint() + } + fn as_any(&self) -> &dyn Any { self } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index afc58009f40292..1b349abbee86bc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -162,6 +162,7 @@ impl Queue { objects, timestamps, false, + self.backend.feed(), )?, Validated::Compute { timestamps, .. } => Completion::new( &self.dev, @@ -172,6 +173,7 @@ impl Queue { objects, core::slice::from_ref(timestamps), true, + None, )?, }; if engine == Engine::Compute { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e908ca3e08e23e..d7a71a2c4c899b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -13,6 +13,7 @@ mod compute_control; mod compute_replay; mod compute_exit; mod events; +mod feed; mod render; pub(super) mod teardown; diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index f1ae1ffe80783b..085655beb8194b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -66,6 +66,10 @@ impl Backend { } impl job::Backend for Backend { + fn feed(&self) -> Option> { + Some(self.shared.feed.clone()) + } + fn ensure_compute(&self) -> Result { Backend::ensure_compute(self) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index e9fc977d623b81..43a27b272ac88e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -311,6 +311,7 @@ impl crate::g17::Shared { }; let result = self.runtime_event_turn(&mut deferred); deferred.finish(); + self.queue_feed(); if let Some(firmware) = (*self.state.lock()).as_deref_mut() { firmware.queues.worker_deferred = Some(deferred); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/feed.rs b/drivers/gpu/drm/asahi/g17/runtime/feed.rs new file mode 100644 index 00000000000000..f3faffb89df764 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/feed.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Schedules CPU capacity updates after completed firmware event passes. The +//! worker owns frequency requests independently of the device publication lock. + +use crate::g17::{Shared, FEED_WORK}; +use kernel::{ + impl_has_work, + sync::Arc, + workqueue::{self, WorkItem}, +}; + +impl_has_work! { + impl HasWork for Shared { self.feed_work } +} + +impl WorkItem for Shared { + type Pointer = Arc; + + fn run(this: Arc) { + this.feed.run(); + } +} + +impl Shared { + /// The completion callbacks have finished outside the device mutex before + /// the event worker hands the detector's request to the actuator. + pub(in crate::g17) fn queue_feed(self: &Arc) { + if self.feed.take_pending() { + let _ = workqueue::system_unbound().enqueue::, FEED_WORK>(self.clone()); + } + } +} diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 4a7ebd25568abf..aec69911c24800 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -576,7 +576,10 @@ pub(crate) trait Gpu: Send + Sync { fn queue_limits(&self) -> Option { None } - + /// Temporary capacity hint and validity in milliseconds for subsequent syncobj waits. + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + None + } /// Frequency of the CPU timer that GPU timestamps are derived from. fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. From 152f6280b5f1990da5a82581ed38030e9f613940 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:34:52 -0400 Subject: [PATCH 041/631] drm/asahi: observe compute completions without a queue mask An event batch without a usable queue identity must still observe completed compute commands. Use the full scan for zero masks while retaining QID filtering for identified batches, so coalesced notifications cannot strand an otherwise retired command. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e1e0c5c90faf0307f95374c83efe3009d58f0bdb) --- drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 43a27b272ac88e..6776707fd944c9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -92,7 +92,10 @@ impl event::Queues for Service<'_> { self.render_masks[1] |= masks[1]; } fn complete_compute(&mut self, masks: [u64; 2]) -> Result { - self.firmware.scan_compute(Some(masks), self.deferred) + // A batch without a queue identity still observes completions whose + // notification was coalesced; identified batches retain QID filtering. + let selected = (masks != [0; 2]).then_some(masks); + self.firmware.scan_compute(selected, self.deferred) } } From 87cfcc212306f7d3e8be3502a5fb8dc631d31728 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:34:52 -0400 Subject: [PATCH 042/631] drm/asahi: avoid repeated submission bookkeeping Consume prepared VM guards in order without shifting the remaining vector for every command. Iterator destruction still releases unused pins on partial enqueue failure. Capture shared attachment metadata under one binding-lock hold per footprint, preserving range insertion order and releasing the lock before allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 0c69741cdc26f2dbb26754e87f1b53db1f4bed38) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 5 +++-- drivers/gpu/drm/asahi/mmu/shared.rs | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 1b349abbee86bc..5495f3895abd32 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -284,11 +284,12 @@ impl crate::queue::Queue for Queue { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, mut guards) = self.prepare_submission(bytes)?; + let (pending, guards) = self.prepare_submission(bytes)?; + let mut guards = guards.into_iter(); let aggregate = Submission::new(&self.fences, self.context.clone())?; outputs.publish(aggregate.clone()); for (command, order) in pending { - let guard = guards.remove(0).map_err(|_| EIO)?; + let guard = guards.next().ok_or(EIO)?; self.enqueue( id, command, diff --git a/drivers/gpu/drm/asahi/mmu/shared.rs b/drivers/gpu/drm/asahi/mmu/shared.rs index cf29e53fc9cc98..5b87c55a06b72b 100644 --- a/drivers/gpu/drm/asahi/mmu/shared.rs +++ b/drivers/gpu/drm/asahi/mmu/shared.rs @@ -106,16 +106,15 @@ impl Vm { return Ok(None); } let mut footprint = Footprint::new(self.id); + let bindings = self.shared_bindings.as_ref().and_then(|shared| { + (shared.count.load(Ordering::Relaxed) != 0).then(|| shared.bindings.lock()) + }); for attachment in attachments.as_slice() { let end = attachment.va.saturating_add(attachment.size); footprint.push_vm(attachment.va, end); - let Some(shared) = &self.shared_bindings else { + let Some(bindings) = &bindings else { continue; }; - if shared.count.load(Ordering::Relaxed) == 0 { - continue; - } - let bindings = shared.bindings.lock(); for entry in bindings.iter() { let first = entry.start.max(attachment.va); let last = entry.end.min(end); @@ -129,6 +128,7 @@ impl Vm { } } } + drop(bindings); Ok(Some(Arc::new(footprint, GFP_KERNEL)?)) } } From a6b6b8a47249263513b31b81b2607bef5390149e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:25:07 -0400 Subject: [PATCH 043/631] drm/asahi: settle unpublished jobs discarded by entity teardown Entity teardown can free a queued job without invoking its cancellation callback. Signal its independent completion fence through the existing unpublished cancellation path so cross-engine timestamp dependencies can retire. Leave jobs whose publication has already been claimed with their existing owner. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0d2761bd854136076d642b614c9b6078699243e) --- drivers/gpu/drm/asahi/g17/job.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 0d28b80baa6332..f16897efa00b8f 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -249,6 +249,17 @@ struct Job { packet: Arc, } +impl Drop for Job { + fn drop(&mut self) { + // Entity teardown can free a queued job without invoking cancel. + // Its independent fence may still order another engine. A claimed + // publisher keeps responsibility for visible work and its VM pin. + if self.packet.publication.load(Ordering::Acquire) == 0 { + self.backend.cancel(self.packet.clone()); + } + } +} + impl sched::JobImpl for Job { fn false_timeout(job: &mut sched::Job) -> bool { job.is_finished() From 4b1c6e5a65767e0e8b03127044653142ca04f089 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:25:08 -0400 Subject: [PATCH 044/631] drm/asahi: reserve completion capacity for global compute replay Submission collectors are sized for one physical queue, but recovery can settle retained commands from every compute queue. Reserve for the complete replay pass before changing any queue, including the publication that follows it. Skip reservation when no queue needs recovery work, preserving the small collector used by healthy submissions. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d41b760af0bcc10744b4145cfdbc19dab7625e8e) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 10 ++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 8 ++++++++ drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs | 12 ++++++++++++ 3 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f8297f0462ff8e..832d5073ecc7bc 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -798,6 +798,16 @@ impl Queue { Ok(()) } + /// Both FIFOs are bounded by IN_FLIGHT. Count all retained commands of + /// queues which this pass can release, replay or quarantine on failure. + pub(crate) fn replay_commands(&self) -> usize { + if self.awaiting_witness || self.replays.len != 0 { + self.active.len + self.replays.len + } else { + 0 + } + } + pub(crate) fn replay_front(&self) -> Option> { self.replays.front().cloned() } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d7a71a2c4c899b..6fe67478a777f7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -501,6 +501,14 @@ impl DeferredBatch { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, }) } + /// A recovery pass can fail and retire each retained command at most once. + /// Reserve before changing queues, plus a full publication for its caller. + /// Healthy submissions keep the small publication collector unchanged. + fn reserve_replays(&mut self, commands: usize) -> Result { + self.values + .reserve(2 * commands + 2 * Self::DEPTH, GFP_KERNEL)?; + Ok(()) + } pub(super) fn push(&mut self, value: super::completion::Deferred) -> Result { use super::completion::Deferred; // Publishers may acquire the device mutex as soon as this pass drops diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 17f228b9d1f048..c872a7c8dacc72 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -18,6 +18,18 @@ impl Firmware { if state != 0 { return Ok(()); } + let commands = self + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .map(|queue| queue.replay_commands()) + .sum::(); + if commands == 0 { + return Ok(()); + } + deferred.reserve_replays(commands)?; let view = qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { if let Some(queue) = entry.queue.as_deref_mut() { From adbb070ef201d9a38619172f8273a41abe0b97b5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:30:36 -0400 Subject: [PATCH 045/631] drm/asahi: filter compute completion identities before payload reads A shared completion scan needs only the control word of unrelated QIDs. Retain a bounded record view, capture control once, and read descriptor and queue identity before the two timestamps used for retirement. This avoids unused payload loads while preserving the single Acquire-ordered scan and exact kick matching. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 868c80c921527154df5e73a271583a4b0b63fdeb) --- drivers/gpu/drm/asahi/g17/initdata.rs | 33 ++++++++++++++----- .../drm/asahi/g17/queue/compute/retirement.rs | 22 ++++++++----- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index f8160f0faa36c8..22b67575c133cb 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -29,6 +29,26 @@ const COMPUTE_CONTROL_BLOCKS: u32 = 25; const COMPUTE_CONTROL_CURSOR: u32 = 0x18; const COMPUTE_RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; +/// A borrowed completion slot. Read the identity before requesting its timestamps; +/// unrelated QIDs need only the control word. Each access is an aligned live load. +pub(crate) struct ComputeCompletion<'a> { + object: &'a KernelObject, + offset: usize, +} + +impl ComputeCompletion<'_> { + pub(crate) fn read(&self, word: usize) -> Result { + if word >= 8 { + return Err(EINVAL); + } + // The scanner bounds offset to a 0x40-byte slot in the completion ring. + Ok(self + .object + .dword(self.offset + word * 8)? + .load(Ordering::Relaxed)) + } +} + /// Unpublished device-wide compute backing, prepared without the device mutex. The cache /// identity belongs to this allocation, not to any one physical queue or client VM. pub(super) struct ComputeShared { @@ -555,18 +575,15 @@ impl InitData { /// scanner latches exact identities; firmware retains sole ownership of these slots. pub(super) fn scan_compute_completions( &self, - mut record: impl FnMut([u64; 8]) -> Result, + mut record: impl FnMut(ComputeCompletion<'_>) -> Result, ) -> Result { let object = self.object(self.completion_rings[0])?; fence(Ordering::Acquire); for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - let mut words = [0; 8]; - for (index, word) in words.iter_mut().enumerate() { - *word = object - .dword(slot * 0x40 + index * 8)? - .load(Ordering::Relaxed); - } - record(words)?; + record(ComputeCompletion { + object, + offset: slot * 0x40, + })?; } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 7428f99ab1d7fe..766177ebc32c93 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -3,6 +3,7 @@ //! Exact identities and visibility witnesses for a physical compute queue. //! A shared work-state counter can include other engines and cannot prove this queue retired. +use crate::g17::initdata::ComputeCompletion; use kernel::prelude::*; pub(crate) const IN_FLIGHT: usize = 16; @@ -170,17 +171,20 @@ impl Batch { self.open = Some(ticket.qid); Ok(()) } - /// The transport reads eight aligned words from a 0x40-byte firmware record. Records can be - /// overwritten during a scan: incomplete or older identities are skipped, never consumed. - pub(crate) fn fold(&mut self, words: [u64; 8]) -> Result { - let qid = ((words[0] >> 9) & 0x7f) as usize; + /// Capture the control word once, then read only a requested identity's payload. + /// Records can be overwritten during a scan: incomplete or older identities are skipped. + pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { + let control = record.read(0)?; + let qid = ((control >> 9) & 0x7f) as usize; let (start, count) = self.slots[qid]; - if count == 0 || words[6] & !1 == 0 || words[7] == 0 { + if count == 0 { return Ok(()); } - let descriptor = words[6] & !1; - let kick = - (((words[0] >> 8) & 0xffff_ffff_00) | ((words[0] >> 48) & 0xff)) & 0xff_ffff_ffff; + let descriptor = record.read(6)? & !1; + if descriptor == 0 || record.read(7)? == 0 { + return Ok(()); + } + let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; let Some(request) = requests .iter_mut() @@ -188,7 +192,7 @@ impl Batch { else { return Ok(()); }; - let timestamps = [words[2] & TIME_MASK, words[5] & TIME_MASK]; + let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; if timestamps[0] == 0 || timestamps[1] == 0 || request From 39f7c44f34e444d535d6d073b61f292e46bc8005 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:31:21 -0400 Subject: [PATCH 046/631] drm/asahi: sample opening counters at the polling deadline Read the opening control counters after the final one-millisecond wait, matching the existing firmware initialization polling budget. Firmware that retires the records during that last interval now completes initialization instead of being torn down as timed out. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f61c86ab28302e46cba8fbdbaa8339aaa4e17b40) --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a574d540404793..35916dd41bfa49 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -992,7 +992,7 @@ impl Gpu { primary: &Coprocessor, secondary: &Coprocessor, ) -> Result { - for _ in 0..OPENING_RETIRE_POLLS { + for poll in 0..=OPENING_RETIRE_POLLS { if primary.state.crashed.load(Ordering::Acquire) || secondary.state.crashed.load(Ordering::Acquire) { @@ -1001,7 +1001,9 @@ impl Gpu { if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { return Ok(()); } - fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + if poll < OPENING_RETIRE_POLLS { + fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); + } } dev_err!( dev.as_ref(), From bbd05bf28f43e38d401b3401bd3c031412904018 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:44:57 -0400 Subject: [PATCH 047/631] drm/asahi: clarify the opening polling budget The initial sample precedes the bounded wait intervals; the final interval also ends with a counter sample. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit b4ee9bd8d923949b19cc22484262670948cafc57) --- drivers/gpu/drm/asahi/g17/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 35916dd41bfa49..a62f1419d61225 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -180,7 +180,7 @@ const INITDATA_ACK_TIMEOUT_MS: u32 = 2000; const RECEIVER_ATTEMPTS: usize = 3; const RECEIVER_RETRY_MS: i64 = 20; /// The firmware does not notify the host when it retires device-control records, so their -/// counters are polled at this interval, at most [`OPENING_RETIRE_POLLS`] times. +/// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. const OPENING_RETIRE_POLL_MS: i64 = 1; const OPENING_RETIRE_POLLS: usize = 100; From 700b28988508734b8bcd4bbd09fdc4e28c060e26 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 02:49:09 -0400 Subject: [PATCH 048/631] drm/asahi: commit canonical binding metadata with each user mapping Prepared mapping batches can fail after earlier leaves have committed. Update canonical context metadata with each successful leaf so context setup observes mappings already visible in the VM. Preflight arithmetic in a compact map plan and retain displaced GEM references until reservation locks are released. Preserve canonical invalidation when quarantined unmaps enter deferred cleanup. Remove the duplicate ioctl update vector, extra object references and post-batch metadata pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 99dd8deec12fe28abb844854059bce6a87764199) --- drivers/gpu/drm/asahi/mmu/bind.rs | 19 ++++- drivers/gpu/drm/asahi/mmu/context.rs | 117 +++++++++++++++++++------- drivers/gpu/drm/asahi/mmu/lifetime.rs | 9 ++ 3 files changed, 110 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 0d5abcf9b84f1a..89c7db4f3d2e71 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -13,6 +13,7 @@ pub(crate) struct PreparedUserMap { size: u64, offset: u64, single_page: bool, + context: context::PreparedBinding, } pub(crate) struct PreparedUserUnmap { @@ -58,6 +59,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + let context = self.prepare_context_binding(addr, size, offset)?; let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -95,6 +97,7 @@ impl Vm { size, offset, single_page, + context, }) } @@ -139,7 +142,8 @@ impl Vm { repeat, flags, ); - if result.is_ok() { + let retired = result.is_ok().then(|| { + let retired = self.commit_context_binding(&map.gem, &map.context); self.track_shared_binding( &map.gem, map.addr, @@ -147,9 +151,11 @@ impl Vm { map.offset, map.single_page, ); - } + retired + }); drop(inner); drop(commit); + drop(retired); Some(result) } Err(error) if error == EIO => { @@ -164,11 +170,16 @@ impl Vm { match self.mapping_commit() { Ok(commit) => { let result = inner.sm_unmap(&mut unmap.ctx, unmap.iova, unmap.size); - if result.is_ok() { + let retired = result.is_ok().then(|| { + let retired = self.untrack_context_ranges(core::iter::once( + unmap.iova..unmap.iova + unmap.size, + )); self.untrack_shared_range(unmap.iova, unmap.size); - } + retired + }); drop(inner); drop(commit); + drop(retired); Some(result) } Err(error) if error == EIO => { diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs index c47b7ae1fd782c..efebcc6a5a84fe 100644 --- a/drivers/gpu/drm/asahi/mmu/context.rs +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -4,6 +4,8 @@ //! //! Userspace owns low views 1 through 6. Only the reserved null view is created //! here. Its installation is serialized per VM and retained through VM teardown. +//! Commits take bindings after the GPUVM exec lock; deferred unmaps take it +//! after the VM lifetime lock. Readers release bindings before taking either. use super::*; use crate::hw::t8140::CONTEXT_VIEWS; @@ -17,6 +19,24 @@ struct Bindings { generation: u64, slots: [Option; CONTEXT_VIEWS.len()], } +/// All arithmetic for a canonical map is checked before its page-table commit. +pub(super) struct PreparedBinding { + address: u64, + object_offset: u64, + touched: u8, + covered: u8, +} + +/// Final GEM release can acquire DMA reservations. Keep replaced references +/// until the caller has released its GPUVM and mapping-admission locks. +pub(super) struct RetiredBindings([Option; CONTEXT_VIEWS.len()]); + +impl RetiredBindings { + fn new() -> Self { + Self(core::array::from_fn(|_| None)) + } +} + #[pin_data] pub(super) struct ContextBindings { #[pin] @@ -57,64 +77,99 @@ impl Vm { Ok(()) } - /// Arithmetic is preflighted before invalidating any retained candidate. - pub(crate) fn track_context_binding( + pub(super) fn prepare_context_binding( &self, - gem: ARef, address: u64, size: u64, object_offset: u64, - ) -> Result { - let Some(context) = self.context_bindings.as_ref() else { - return Ok(()); + ) -> Result { + let mut update = PreparedBinding { + address, + object_offset, + touched: 0, + covered: 0, }; + if self.context_bindings.is_none() { + return Ok(update); + } let end = address.checked_add(size).ok_or(EOVERFLOW)?; - let mut offsets = [None; CONTEXT_VIEWS.len()]; for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if address < view.source + view.size as u64 && view.source < end { + update.touched |= 1 << index; + } if address <= view.source && view.source + view.size as u64 <= end { - offsets[index] = Some(usize::try_from( + update.covered |= 1 << index; + usize::try_from( object_offset .checked_add(view.source - address) .ok_or(EOVERFLOW)?, - )?); + )?; } } + Ok(update) + } + + /// Called after a successful leaf commit, while its exec lock still orders + /// mapping changes. The prepared update cannot fail after visibility. + pub(super) fn commit_context_binding( + &self, + gem: &ARef, + update: &PreparedBinding, + ) -> RetiredBindings { + let mut retired = RetiredBindings::new(); + if update.touched == 0 { + return retired; + } + let Some(context) = self.context_bindings.as_ref() else { + return retired; + }; let mut bindings = context.bindings.lock(); - let mut touched = false; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if address < view.source + view.size as u64 && view.source < end { - bindings.slots[index] = None; - touched = true; + for (index, slot) in bindings.slots.iter_mut().enumerate() { + if update.touched & (1 << index) != 0 { + retired.0[index] = slot.take(); } - if let Some(offset) = offsets[index] { - bindings.slots[index] = Some(Binding { + if update.covered & (1 << index) != 0 { + // Coverage proves source >= address. Preparation checked this + // exact sum and its usize conversion against the same view. + let offset = (update.object_offset + + (CONTEXT_VIEWS[index].source - update.address)) as usize; + *slot = Some(Binding { gem: gem.clone(), offset, }); - touched = true; } } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } - Ok(()) + bindings.generation = bindings.generation.wrapping_add(1); + retired } - pub(crate) fn untrack_context_range(&self, range: Range) { + /// Successful user unmaps and accepted deferred unmaps invalidate the + /// matching candidates. Returning their owners avoids destruction under + /// the caller's exec or lifetime lock. + pub(super) fn untrack_context_ranges( + &self, + ranges: impl Iterator>, + ) -> RetiredBindings { + let mut retired = RetiredBindings::new(); let Some(context) = self.context_bindings.as_ref() else { - return; + return retired; }; let mut bindings = context.bindings.lock(); - let mut touched = false; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if range.start < view.source + view.size as u64 && view.source < range.end { - bindings.slots[index] = None; - touched = true; + for range in ranges { + let mut touched = false; + for (index, view) in CONTEXT_VIEWS.iter().enumerate() { + if range.start < view.source + view.size as u64 && view.source < range.end { + if let Some(binding) = bindings.slots[index].take() { + retired.0[index] = Some(binding); + } + touched = true; + } + } + if touched { + bindings.generation = bindings.generation.wrapping_add(1); } } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } + retired } pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 3b8a5e5a40aeb9..bff43fbb3795ee 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -436,6 +436,9 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } + // Reserve before accepting any range; a partial deferred batch must + // not leave canonical metadata ahead of the accepted cleanup. + state.unmaps.reserve(batch.ops.len(), GFP_KERNEL)?; for op in &batch.ops { if let PreparedUserBindOp::Unmap(unmap) = op { let range = (unmap.iova, unmap.size); @@ -444,6 +447,12 @@ impl Vm { } } } + let retired = self.untrack_context_ranges(batch.ops.iter().filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); + drop(state); + drop(retired); Ok(true) } From fbb45e9e249c071141a8cf043c50c3181557235c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 03:04:01 -0400 Subject: [PATCH 049/631] drm/asahi: order canonical object invalidation with mapping removal Remove canonical candidates under the existing GPUVM exec guard after successful object unmap. Keep deferred acceptance invalidation under the lifetime lock, with the queued GEM reference retaining the object. This prevents a trailing close callback from erasing a newer committed binding without adding locking or allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 11b50b74597c2c3e1f5687d5db50ce0eaa103c21) --- drivers/gpu/drm/asahi/mmu.rs | 3 +++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 44c20e7d41e153..36dbe287665946 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2981,6 +2981,9 @@ impl Vm { if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); self.inner.bo_unmap(&mut ctx, &bo)?; + // The close callback or deferred object list retains this GEM, so + // clearing its candidate cannot release the final reference here. + self.untrack_context_object(gem); self.untrack_shared_object(gem); mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index bff43fbb3795ee..78f906ff9c46c8 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -491,6 +491,10 @@ impl Vm { state.objects.push(gem.into(), GFP_KERNEL)?; } if state.active != 0 || state.draining { + // Stop new jobs from selecting this accepted cleanup candidate. + // The queued object retains its GEM while the candidate is removed. + // Actual unmap repeats this under exec to order any intervening bind. + self.untrack_context_object(gem); return Ok(()); } state.draining = true; From 8bfa7fb8e34a525b36027b0ed540250421fe8c8a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:22:22 -0400 Subject: [PATCH 050/631] drm/asahi: advertise G17 queue and input-sync capabilities Report the independently reserved logical queue capacity and per-command input synchronization already provided by G17. Userspace can then expose multiple queues and preserve independent command roots while forwarding input syncs to each submission. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f594e2be7f492d0cb4fdbdf5197122418f330036) --- drivers/gpu/drm/asahi/g17/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a62f1419d61225..1ccbb9b708ce0b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1058,6 +1058,8 @@ impl gpu::Gpu for Gpu { } fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params.features |= + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; @@ -1087,6 +1089,14 @@ impl gpu::Gpu for Gpu { true } + fn queue_limits(&self) -> Option { + Some(uapi::drm_asahi_queue_limits { + max_queues: mmu::MAX_EXECUTION_CONTEXTS, + max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, + ..Default::default() + }) + } + fn uat_geometry(&self) -> mmu::UatGeometry { self.geometry } From c2b1c2cb312e4f32213f48bd125de5de09f23b39 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:27:10 -0400 Subject: [PATCH 051/631] drm/asahi: acquire event records after observing their producer Order firmware event payload reads after the producer cursor establishes that the record is available. Move the existing acquire fence to this boundary, preserving the record-copy and consumer-store ordering without adding a barrier to nonempty drains. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 98783a0fc38dbac9eecb4164ed6c090e3adf591d) --- drivers/gpu/drm/asahi/g17/channel.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index c0609f148106e7..a90693e824b0e0 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -184,7 +184,6 @@ impl Events { while self.remaining != 0 { init.validate_event_ring(role)?; self.remaining -= 1; - fence(Ordering::Acquire); let consumer = consumer_word.load(Ordering::Relaxed); let producer = producer_word.load(Ordering::Relaxed); if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { @@ -194,6 +193,8 @@ impl Events { self.remaining = 0; break; } + // Observe the producer before reading the record it publishes. + fence(Ordering::Acquire); let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; let record = EventRecord::from_words(init.status_read_words(role, offset)?); let event = record.decode(); From 095a31e23485537c0a93da83743959575f277772 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:27:47 -0400 Subject: [PATCH 052/631] drm/asahi: discard partial G17 mappings when binding fails A G17 map can fail allocating a later page table after earlier leaves have been written, before a GPUVA retains the backing. Clear the full failed operation range and invalidate every context naming its root before returning the mapping error. Apply the same cleanup if the final GPUVA insertion fails. Use a nonallocating partial-map discard that tolerates absent leaves without normal unmap diagnostics. Preserve Handoff error behavior and avoid additional work on successful mappings. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 267dc9ba9ea8e068a3d9d0683a58b72a51291ece) --- drivers/gpu/drm/asahi/mmu.rs | 27 +++++++++++++++++++++++++-- drivers/gpu/drm/asahi/pgtable.rs | 9 +++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 36dbe287665946..4ab676f0755265 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -546,12 +546,15 @@ impl gpuvm::DriverGpuVm for VmInner { one_page ); - self.page_table.map_pages( + if let Err(error) = self.page_table.map_pages( iova..(iova + len as u64), addr as PhysicalAddr, ctx.prot, one_page, - )?; + ) { + self.rollback_failed_map(op.addr(), op.range()); + return Err(error); + } left -= len; iova += len as u64; @@ -609,6 +612,7 @@ impl gpuvm::DriverGpuVm for VmInner { op.range(), op.addr() ); + self.rollback_failed_map(op.addr(), op.range()); return Err(EINVAL); } self.tlbi_contexts_naming_root(op.addr(), op.range() as usize); @@ -732,6 +736,25 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// A failed map has no GPUVA owner to retain its backing or remove its leaves. + /// GPUVM has already split or removed overlaps throughout this operation's range. + fn rollback_failed_map(&mut self, addr: u64, size: u64) { + if self.uat_inner.firmware == UatFirmware::Handoff { + return; + } + // Include the current segment: map_pages can fail after writing some of its leaves. + // The checked GPUVM operation supplies an aligned range without overflow. + let result = self.page_table.discard_partial_map(addr..addr + size); + self.tlbi_contexts_naming_root(addr, size as usize); + if let Err(error) = result { + dev_err!( + self.dev.as_ref(), + "Could not discard failed mapping: {:?}\n", + error + ); + } + } + fn mapping_mutation(&mut self) -> MappingMutation { let state = self.mapping_epoch.clone(); if let Some(state) = state.as_ref() { diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index d0ca5801e81787..adbdd97beb2267 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -1392,6 +1392,15 @@ impl UatPageTable { }) } + /// Discards an aligned map operation that failed before acquiring a GPUVA owner. + /// Missing tables and leaves are expected; existing table storage stays allocated. + pub(crate) fn discard_partial_map(&mut self, iova_range: Range) -> Result { + self.with_pages(iova_range, false, false, |_, ptes| { + for pte in ptes { + pte.store(0, Ordering::Relaxed); + } + Ok(()) + }) /// Read one PTE from a page table page, without ever allocating or modifying anything. /// /// Fails with an error (instead of oopsing) if the table page is not mapped. From c803e89775733d277b78583e503f7ce4b3ec589e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:34:59 -0400 Subject: [PATCH 053/631] drm/asahi: reuse the reserved output fence for submitted work Reserve one fence before submission validation and reuse it as the aggregate after preparation succeeds. Keep the same fence available for every pre-publication failure, including aggregate allocation failure, without allocating a second unused fence on successful submits. Once published, only the aggregate enqueue sentinel and command members may signal it. Leave output installation and command completion ordering unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 9f204371cd6118ecaa4635b76a763e6c4d53b728) --- drivers/gpu/drm/asahi/g17/fence.rs | 11 +++++++---- drivers/gpu/drm/asahi/g17/queue/mod.rs | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 95a13dbdd54fff..32d25b3b608cb2 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -48,10 +48,10 @@ pub(crate) struct Submission { } impl Submission { - pub(crate) fn new(contexts: &FenceContexts, context: Arc) -> Result> { + fn new(fence: UserFence, context: Arc) -> Result> { Ok(Arc::new( Self { - fence: independent(contexts)?, + fence, work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, remaining: AtomicU32::new(1), error: AtomicI32::new(0), @@ -168,9 +168,11 @@ impl Outputs { } /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, submission: Arc) { + pub(crate) fn publish(&mut self, context: Arc) -> Result> { + let submission = Submission::new(self.fallback.clone(), context)?; install(self.syncs.drain(self.input_count..), &submission.fence()); - self.submission = Some(submission); + self.submission = Some(submission.clone()); + Ok(submission) } /// The caller first records terminal VM errors and settles prior failed work. @@ -184,6 +186,7 @@ impl Drop for Outputs { fn drop(&mut self) { if let Some(submission) = self.submission.take() { submission.finish_enqueue(self.result); + return; } if let Err(error) = self.result { self.fallback.set_error(error); diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 5495f3895abd32..0bcff40a1bf0d4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -286,8 +286,7 @@ impl crate::queue::Queue for Queue { } let (pending, guards) = self.prepare_submission(bytes)?; let mut guards = guards.into_iter(); - let aggregate = Submission::new(&self.fences, self.context.clone())?; - outputs.publish(aggregate.clone()); + let aggregate = outputs.publish(self.context.clone())?; for (command, order) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( From ad86dd5604246091a77216c1da0bc301eded114a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:29:06 -0400 Subject: [PATCH 054/631] drm/asahi: wake render growth when its prerequisites become ready A retained TVB request can wait for a borrowed render pair or control-ring publication. Requeuing the growth worker after every event pass makes it bounce between growth and event work while those prerequisites are unchanged. Schedule growth only when a request can run, including stale requests that need discarding. Wake parked requests after render preparation restores its pair, including failed and cancelled preparation. Preserve the independent growth worker and existing control and recovery wakeups. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fa771fbd7629800795268b560e9e2ff1b0cb806c) --- drivers/gpu/drm/asahi/g17/runtime/events.rs | 4 +- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 13 ++++++ .../drm/asahi/g17/runtime/render/prepare.rs | 43 +++++++++++++------ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 6776707fd944c9..f249e3520e0dd9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -415,7 +415,9 @@ impl crate::g17::Shared { // records that arrived while it was consuming the same ring. self.queue_events(); } - if firmware.render_grow_pending() && !firmware.recovery.pending() { + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { self.queue_grow(); } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index a9636cb7ceda04..db5820cfb66638 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -72,6 +72,19 @@ impl crate::g17::Firmware { pub(in crate::g17) fn render_grow_pending(&self) -> bool { !self.queues.render.grow.requests.is_empty() } + /// Borrowed owners and control publication each have their own retry wake. + /// A stale request is runnable so the worker can discard it immediately. + pub(in crate::g17) fn render_grow_ready(&self) -> bool { + let render = &self.queues.render; + !render.grow.requests.is_empty() + && !self.recovery.pending() + && !self.render_control_backpressured() + && render.grow.active.is_none() + && render.grow.requests.iter().any(|request| { + request.halt_count != self.recovery.generation() + || !render.grow_owner_busy(*request) + }) + } pub(in crate::g17) fn render_control_backpressured(&self) -> bool { self.queues.render.grow.reply.is_some() || self.queues.render.pools.pending() } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 30ea174f611b18..1aa6cb67a90ebf 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -299,7 +299,7 @@ impl super::super::Backend { // returns, while the original installed kick owner stays live. drop(prepared); drop(storage); - { + let grow_ready = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if result.is_err() { @@ -315,8 +315,12 @@ impl super::super::Backend { entry.preparing = None; entry.pending_owner = None; entry.pending_owner_closed = false; - } + firmware.render_grow_ready() + }; self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } result } Build::Rebind { @@ -345,7 +349,7 @@ impl super::super::Backend { Ok(binding) => (Some(binding), None), Err(error) => (None, Some(error)), }; - let result = { + let (result, grow_ready) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; let result = (|| { @@ -398,7 +402,7 @@ impl super::super::Backend { entry.pending_owner_closed = false; entry.pair = Some(pair); entry.borrowed_dependencies = None; - result + (result, firmware.render_grow_ready()) }; drop(prepared); if result.is_err() { @@ -413,6 +417,9 @@ impl super::super::Backend { } } self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } result } }; @@ -582,16 +589,24 @@ impl super::super::Backend { } firmware.publish_render(self.owner, slot, packet, deferred) })(); - if published.is_err() && !packet.is_published() { - firmware - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)? - .discard_prepared(packet)?; + let result = (|| { + if published.is_err() && !packet.is_published() { + firmware + .queues + .render + .entry_mut(slot)? + .pair + .as_mut() + .ok_or(EIO)? + .discard_prepared(packet)?; + } + published + })(); + let grow_ready = firmware.render_grow_ready(); + drop(state); + if grow_ready { + self.shared.queue_grow(); } - published + result } } From 000b42ced2f8adc3a58a9d9b5f3396b8a2e90491 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:36:44 -0400 Subject: [PATCH 055/631] drm/asahi: skip frontier scans without a cross-engine prefix Select an empty borrowed frontier when a command has no opposite-engine prefix. The existing count and fill paths then do no predecessor work for that command, while explicit prefixes retain every covered fence in its original order. Keep both same-engine and compute-triggered render pruning unchanged, along with per-command input fences and early-publication eligibility. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 956a8e2fd2ddcb7e9dc596796b79f62b4e017c55) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 0bcff40a1bf0d4..9af2cd4b7643f2 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -182,14 +182,19 @@ impl Queue { } let other = 1 - index; let prefix = order.wait_through[other]; - let count = self.frontiers[other] + let frontier = if prefix.is_some() { + self.frontiers[other].as_slice() + } else { + &[] + }; + let count = frontier .iter() .filter(|(sequence, _)| Order::contains(prefix, *sequence)) .count(); let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; - for (sequence, fences) in &self.frontiers[other] { + for (sequence, fences) in frontier { if Order::contains(prefix, *sequence) { dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; From dedeb2c3a0ed1bed67f7aa689870d4a5c1dc870a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 04:40:54 -0400 Subject: [PATCH 056/631] drm/asahi: track CPU feed requests through policy lifetime Register a device-owned policy notifier before a single existing-policy snapshot. Match creation with removal of the exact pinned frequency request and policy reference, so CPUFreq teardown does not wait for GPU removal. Revalidate snapshot policy identity under the request lock after policy initialization, and keep notifier unregistration outside that lock. Preserve feed thresholds, the frequency floor and worker cadence. Handle later policy creation without periodic CPU scans, and leave the detector inactive when CPUFreq is disabled. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5026a187d811c441374d11c71a3fa0df58e118fd) --- drivers/gpu/drm/asahi/g17/feed.rs | 331 ++++++++++++++++++++++-------- 1 file changed, 245 insertions(+), 86 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/feed.rs b/drivers/gpu/drm/asahi/g17/feed.rs index ca3ecc85ee25a8..4cd05fde165812 100644 --- a/drivers/gpu/drm/asahi/g17/feed.rs +++ b/drivers/gpu/drm/asahi/g17/feed.rs @@ -1,34 +1,41 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT //! Detects sustained host refill gaps and temporarily requests CPU capacity. -//! Detector updates are lock-free. The device's single feed worker owns the -//! frequency requests; its lock never nests inside a driver publication lock. +//! Detector updates are lock-free. The worker and policy callbacks serialize +//! frequency requests without nesting their mutex inside a publication lock. //! Shutdown first closes the detector, joins its producers and the worker, //! then removes requests. All request storage and policy references are owned //! by this device, independently of other GPU instances. +//! Policy creation and the initial snapshot take the policy rwsem before the +//! requests mutex. Removal callbacks and the worker take only requests; +//! notifier unregistration joins callbacks without holding that mutex. use crate::hw::t8140::qos::CLOCK_HZ; use core::{ pin::Pin, - ptr, sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, }; use kernel::{ - bindings, new_mutex, prelude::*, - sync::{Arc, Mutex}, + sync::Arc, time::{delay::fsleep, ClockSource, Delta, Monotonic}, - types::Opaque, }; +#[cfg(CONFIG_CPU_FREQ)] +use core::ptr; +#[cfg(CONFIG_CPU_FREQ)] +use kernel::{bindings, new_mutex, sync::Mutex, types::Opaque}; + const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; const STREAK_ON: u32 = 8; const STREAK_MAX: u32 = 64; const LINGER_NS: u64 = 8_000_000; const POLL_MS: i64 = 2; +#[cfg(CONFIG_CPU_FREQ)] const FLOOR_PERCENT: u64 = 50; const WAIT_UTIL_MIN: u32 = 512; const WAIT_HINT_MS: u32 = 8; +#[cfg(CONFIG_CPU_FREQ)] const MAX_POLICIES: usize = 8; struct Detector { @@ -114,6 +121,7 @@ impl Detector { } } +#[cfg(CONFIG_CPU_FREQ)] #[pin_data] struct Requests { #[pin] @@ -121,13 +129,14 @@ struct Requests { policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], maxima: [u32; MAX_POLICIES], applied: [u32; MAX_POLICIES], - count: usize, } -// SAFETY: Each nonnull policy retains cpufreq_cpu_get's reference. Access to -// the pinned request array and metadata is exclusive under Feed::requests. +// SAFETY: Each nonnull policy owns a kobject reference. The device requests +// mutex serializes the pinned slots with the worker and policy callbacks. +#[cfg(CONFIG_CPU_FREQ)] unsafe impl Send for Requests {} +#[cfg(CONFIG_CPU_FREQ)] impl Requests { fn new() -> Self { Self { @@ -135,126 +144,255 @@ impl Requests { policies: [ptr::null_mut(); MAX_POLICIES], maxima: [0; MAX_POLICIES], applied: [0; MAX_POLICIES], - count: 0, } } - fn register(self: Pin<&mut Self>) { + /// The caller holds the policy's read or write lock and keeps it alive. + /// Policy removal must subsequently pass through this device's notifier. + unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { let this = self.project(); - if *this.count != 0 { + if this.policies.contains(&policy) { return; } - // SAFETY: nr_cpu_ids is initialized before driver probe. - let cpus = unsafe { bindings::nr_cpu_ids }; - for cpu in 0..cpus { - // SAFETY: Accepts any CPU number; a nonnull result owns one reference. - let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; - if policy.is_null() { + let Some(index) = this.policies.iter().position(|entry| entry.is_null()) else { + return; + }; + // SAFETY: The caller keeps the initialized policy alive. The vacant + // request slot is pinned and exclusively owned under the requests mutex. + let result = unsafe { + bindings::freq_qos_add_request( + ptr::addr_of_mut!((*policy).constraints), + this.requests + .get() + .cast::() + .add(index), + bindings::freq_qos_req_type_FREQ_QOS_MIN, + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + ) + }; + if result >= 0 { + // SAFETY: The caller's policy reference or CREATE callback keeps the + // kobject alive. Retain it until REMOVE or device shutdown removes + // this request; cpuinfo is protected by the caller's policy lock. + unsafe { + bindings::kobject_get(ptr::addr_of_mut!((*policy).kobj)); + this.maxima[index] = (*policy).cpuinfo.max_freq; + } + this.policies[index] = policy; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; + } + } + + fn apply(self: Pin<&mut Self>, enabled: bool) -> bool { + let this = self.project(); + let mut applied = false; + for index in 0..MAX_POLICIES { + if this.policies[index].is_null() { continue; } - let count = *this.count; - if count < MAX_POLICIES && !this.policies[..count].contains(&policy) { - // SAFETY: The policy is referenced. This zeroed, unregistered - // slot is pinned and remains alive until remove_request. + let value = if enabled { + (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 + } else { + bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 + }; + if this.applied[index] != value { + // SAFETY: This slot is registered, pinned and exclusively locked; + // the policy reference keeps its constraints alive through REMOVE. let result = unsafe { - bindings::freq_qos_add_request( - ptr::addr_of_mut!((*policy).constraints), + bindings::freq_qos_update_request( this.requests .get() .cast::() - .add(count), - bindings::freq_qos_req_type_FREQ_QOS_MIN, - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, + .add(index), + value as i32, ) }; if result >= 0 { - this.policies[count] = policy; - // SAFETY: cpuinfo is immutable while this policy is referenced. - this.maxima[count] = unsafe { (*policy).cpuinfo.max_freq }; - this.applied[count] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; - *this.count += 1; - continue; + this.applied[index] = value; } } - // SAFETY: Balances the reference acquired above for a duplicate, - // unavailable slot or failed registration. Successful slots retain it. - unsafe { bindings::cpufreq_cpu_put(policy) }; + applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } + applied } - fn apply(mut self: Pin<&mut Self>, enabled: bool) -> bool { - if enabled { - self.as_mut().register(); - } + fn remove(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { let this = self.project(); - for index in 0..*this.count { - let floor = if enabled { - (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 - } else { - 0 - }; - let value = if floor == 0 { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 - } else { - floor - }; - if this.applied[index] == value { - continue; - } - this.applied[index] = value; - // SAFETY: This slot is registered, pinned and exclusively locked; - // its referenced policy keeps the constraints alive. - unsafe { - bindings::freq_qos_update_request( - this.requests - .get() - .cast::() - .add(index), - value as i32, - ) - }; + let Some(index) = this.policies.iter().position(|entry| *entry == policy) else { + return; + }; + // SAFETY: This registered pinned slot is exclusively locked. The policy + // cannot finish removal until this reference is dropped. Keep all other + // request slots at their original addresses when releasing this slot. + unsafe { + bindings::freq_qos_remove_request( + this.requests + .get() + .cast::() + .add(index), + ); + bindings::cpufreq_cpu_put(policy); } - *this.count != 0 && enabled + this.policies[index] = ptr::null_mut(); + this.maxima[index] = 0; + this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; } - fn clear(self: Pin<&mut Self>) { - let this = self.project(); - for index in 0..*this.count { - // SAFETY: Registered pinned slot, exclusively locked. Remove the - // constraint link before releasing its policy reference. - unsafe { - bindings::freq_qos_remove_request( - this.requests - .get() - .cast::() - .add(index), - ); - bindings::cpufreq_cpu_put(this.policies[index]); + fn clear(mut self: Pin<&mut Self>) { + for index in 0..MAX_POLICIES { + let policy = self.policies[index]; + if !policy.is_null() { + self.as_mut().remove(policy); } - this.policies[index] = ptr::null_mut(); } - *this.count = 0; } } +/// Only the blocking notifier core accesses this block after registration. +#[cfg(CONFIG_CPU_FREQ)] +#[pin_data] +struct PolicyNotifier { + #[pin] + block: Opaque, +} + +// SAFETY: The pinned notifier is registered once before publishing Feed and +// unregistered at shutdown. The blocking notifier core serializes its links; +// callbacks use only Feed's atomic state and request mutex. +#[cfg(CONFIG_CPU_FREQ)] +unsafe impl Send for PolicyNotifier {} +// SAFETY: As above; no Rust access mutates the registered block. +#[cfg(CONFIG_CPU_FREQ)] +unsafe impl Sync for PolicyNotifier {} + #[pin_data(PinnedDrop)] pub(crate) struct Feed { detector: Detector, + #[cfg(CONFIG_CPU_FREQ)] #[pin] requests: Mutex, + #[cfg(CONFIG_CPU_FREQ)] + #[pin] + notifier: PolicyNotifier, + #[cfg(CONFIG_CPU_FREQ)] + registered: AtomicBool, applied: AtomicBool, } impl Feed { pub(crate) fn new() -> Result> { - Arc::pin_init( + let feed = Arc::pin_init( pin_init!(Self { detector: Detector::new(), + #[cfg(CONFIG_CPU_FREQ)] requests <- new_mutex!(Requests::new(), "G17 CPU feed"), + #[cfg(CONFIG_CPU_FREQ)] + notifier <- pin_init!(PolicyNotifier { + block: Opaque::new(bindings::notifier_block { + notifier_call: Some(Self::policy_event), + next: ptr::null_mut(), + priority: 0, + }), + }), + #[cfg(CONFIG_CPU_FREQ)] + registered: AtomicBool::new(false), applied: AtomicBool::new(false), }), GFP_KERNEL, - ) + )?; + #[cfg(CONFIG_CPU_FREQ)] + feed.register_policies(); + Ok(feed) + } + + /// Seed existing policies once; later creation/removal is event driven. + #[cfg(CONFIG_CPU_FREQ)] + fn register_policies(&self) { + // SAFETY: Feed is pinned in its Arc and has not been published. The + // notifier block lives until shutdown unregisters and joins callbacks. + let result = unsafe { + bindings::cpufreq_register_notifier( + self.notifier.block.get(), + bindings::CPUFREQ_POLICY_NOTIFIER, + ) + }; + if result < 0 { + // CPUFreq can be disabled at boot independently of this driver. + return; + } + self.registered.store(true, Ordering::Release); + // SAFETY: nr_cpu_ids is initialized before driver probe. + let cpus = unsafe { bindings::nr_cpu_ids }; + for cpu in 0..cpus { + // SAFETY: The bounded CPU index is valid; a nonnull result owns a + // reference even if removal begins before acquiring the policy lock. + let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; + if policy.is_null() { + continue; + } + // CREATE holds the policy write lock before taking requests. Use + // the same lock order and wait out partial policy initialization. + // SAFETY: The retained reference keeps the policy and rwsem alive. + unsafe { bindings::down_read(ptr::addr_of_mut!((*policy).rwsem)) }; + { + let mut requests = self.requests.lock(); + // Removal unpublishes the CPU pointer before its notifier takes + // requests. Rechecking under this mutex makes either the seed + // skip the policy or the subsequent REMOVE release its slot. + // SAFETY: The CPU index is valid; balance this extra reference below. + let current = unsafe { bindings::cpufreq_cpu_get(cpu) }; + // SAFETY: The policy read lock excludes unfinished CREATE. If + // this policy is still published, REMOVE cannot yet have removed + // the core max request without passing our held requests mutex. + let initialized = + current == policy && unsafe { !(*policy).max_freq_req.qos.is_null() }; + if initialized { + // SAFETY: The reference and read lock keep the initialized + // policy alive; registered REMOVE callbacks cover its lifetime. + unsafe { requests.as_mut().add(policy) }; + } + if !current.is_null() { + // SAFETY: Balances the second lookup, independently of its identity. + unsafe { bindings::cpufreq_cpu_put(current) }; + } + } + // SAFETY: Balance the read lock and first lookup after releasing requests. + unsafe { + bindings::up_read(ptr::addr_of_mut!((*policy).rwsem)); + bindings::cpufreq_cpu_put(policy); + } + } + } + + #[cfg(CONFIG_CPU_FREQ)] + unsafe extern "C" fn policy_event( + block: *mut bindings::notifier_block, + event: c_ulong, + data: *mut core::ffi::c_void, + ) -> core::ffi::c_int { + // SAFETY: This callback is installed only on Feed::notifier.block. Its + // owner remains pinned until unregister joins every callback. + let notifier = + unsafe { kernel::container_of!(Opaque::cast_from(block), PolicyNotifier, block) }; + // SAFETY: The notifier is the embedded, pinned field of this live Feed. + let feed = unsafe { &*kernel::container_of!(notifier, Feed, notifier) }; + let policy = data.cast::(); + let mut requests = feed.requests.lock(); + if event == bindings::CPUFREQ_REMOVE_POLICY as c_ulong { + requests.as_mut().remove(policy); + } else if event == bindings::CPUFREQ_CREATE_POLICY as c_ulong + && !feed.detector.stopping.load(Ordering::Acquire) + { + // SAFETY: CREATE supplies an initialized policy with its write lock + // held and guarantees REMOVE before the core releases its storage. + unsafe { requests.as_mut().add(policy) }; + } else { + return bindings::NOTIFY_DONE as _; + } + let enabled = feed.detector.wanted(Self::now()); + let applied = requests.as_mut().apply(enabled); + feed.applied.store(applied, Ordering::Relaxed); + bindings::NOTIFY_OK as _ } fn now() -> u64 { @@ -262,7 +400,9 @@ impl Feed { } pub(crate) fn note_render_pass(&self, start: u64, end: u64) { - self.detector.note(start, end, Self::now()); + if cfg!(CONFIG_CPU_FREQ) { + self.detector.note(start, end, Self::now()); + } } /// Called after a firmware event pass, before queuing the single feed work item. @@ -270,6 +410,7 @@ impl Feed { self.detector.pending.swap(false, Ordering::AcqRel) } + #[cfg(CONFIG_CPU_FREQ)] fn apply(&self, enabled: bool) { let mut requests = self.requests.lock(); let enabled = enabled && !self.detector.stopping.load(Ordering::Acquire); @@ -277,6 +418,9 @@ impl Feed { self.applied.store(applied, Ordering::Relaxed); } + #[cfg(not(CONFIG_CPU_FREQ))] + fn apply(&self, _enabled: bool) {} + /// Worker context; the linger deadline has no firmware event of its own. pub(crate) fn run(&self) { loop { @@ -317,7 +461,22 @@ impl Feed { /// Call after work items are joined. Removing requests withdraws their floor. pub(crate) fn shutdown(&self) { self.begin_shutdown(); - self.requests.lock().as_mut().clear(); + #[cfg(CONFIG_CPU_FREQ)] + { + // Unregister joins callbacks, which take requests: never hold that + // mutex here. Device shutdown has already joined the feed worker. + if self.registered.swap(false, Ordering::AcqRel) { + // SAFETY: The block was registered successfully and remains + // pinned and alive until this synchronous removal returns. + unsafe { + bindings::cpufreq_unregister_notifier( + self.notifier.block.get(), + bindings::CPUFREQ_POLICY_NOTIFIER, + ); + } + } + self.requests.lock().as_mut().clear(); + } self.applied.store(false, Ordering::Relaxed); } } From f518ac162d144ea6e1ddd5a9806b9c2b379ca6ea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 3 Oct 2026 05:15:19 -0400 Subject: [PATCH 057/631] drm/asahi: bound dependency scans to the ordered command prefix Accepted queue sequences are nonzero and increasing, and frontier pruning preserves their order. Locate the end of an explicit cross-engine prefix with partition_point and clone that slice directly. Keep the empty slice for absent prefixes and leave completion pruning, input synchronization and early-publication admission unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c42fd8c45c3a20ed2ec94c40bf5fbcf782220e7c) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 9af2cd4b7643f2..65bfdf10ab2579 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -183,22 +183,21 @@ impl Queue { let other = 1 - index; let prefix = order.wait_through[other]; let frontier = if prefix.is_some() { - self.frontiers[other].as_slice() + let frontier = self.frontiers[other].as_slice(); + // Accepted sequences increase from one, and retain preserves their + // order, so all members of a logical prefix form one slice. + let end = frontier.partition_point(|(sequence, _)| Order::contains(prefix, *sequence)); + &frontier[..end] } else { &[] }; - let count = frontier - .iter() - .filter(|(sequence, _)| Order::contains(prefix, *sequence)) - .count(); + let count = frontier.len(); let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; - for (sequence, fences) in frontier { - if Order::contains(prefix, *sequence) { - dependencies.push(fences.ready.clone(), GFP_KERNEL)?; - firmware.push(fences.completed.clone(), GFP_KERNEL)?; - } + for (_, fences) in frontier { + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + firmware.push(fences.completed.clone(), GFP_KERNEL)?; } // Each command owns every input fence; independent engines may publish in // either order and must each consume an input's error status before running. From 97655bb70bc8049f37b20324fcebde15f6a3f1b4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:13:13 -0400 Subject: [PATCH 058/631] rust: check generated layouts and provide corrected private bindgen Keep compile-time size, alignment and field-offset assertions in generated bindings so incompatible layouts fail during the ordinary bindings build. Provide a pinned private generator recipe that preserves alignment on defined zero-sized structs and actual anonymous record members. Include explicit-target C and Rust compiler checks and generator provenance without changing C storage, scheduler configuration or the system toolchain. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1f7383d5a2811b99a9f2950581ec392837bfc469) --- Documentation/rust/bindgen-layout-checks.rst | 49 +++++++++++ Documentation/rust/index.rst | 1 + rust/Makefile | 5 +- tools/aurora-bindgen/.gitattributes | 1 + tools/aurora-bindgen/README.rst | 72 ++++++++++++++++ tools/aurora-bindgen/build.py | 22 +++++ .../patches/0001-empty-alignment.patch | 20 +++++ .../patches/0002-anonymous-tag-members.patch | 85 +++++++++++++++++++ tools/aurora-bindgen/sources.json | 19 +++++ tools/aurora-bindgen/test.py | 44 ++++++++++ tools/aurora-bindgen/tests/aligned-empty.h | 13 +++ tools/aurora-bindgen/tests/aligned-empty.hpp | 13 +++ .../aurora-bindgen/tests/anonymous-members.h | 9 ++ tools/aurora-bindgen/tests/tag-only.h | 4 + 14 files changed, 356 insertions(+), 1 deletion(-) create mode 100644 Documentation/rust/bindgen-layout-checks.rst create mode 100644 tools/aurora-bindgen/.gitattributes create mode 100644 tools/aurora-bindgen/README.rst create mode 100644 tools/aurora-bindgen/build.py create mode 100644 tools/aurora-bindgen/patches/0001-empty-alignment.patch create mode 100644 tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch create mode 100644 tools/aurora-bindgen/sources.json create mode 100644 tools/aurora-bindgen/test.py create mode 100644 tools/aurora-bindgen/tests/aligned-empty.h create mode 100644 tools/aurora-bindgen/tests/aligned-empty.hpp create mode 100644 tools/aurora-bindgen/tests/anonymous-members.h create mode 100644 tools/aurora-bindgen/tests/tag-only.h diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst new file mode 100644 index 00000000000000..8ad8c24526b672 --- /dev/null +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -0,0 +1,49 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Checking generated binding layouts +================================== + +Kernel bindings retain bindgen's layout checks. With the configured Rust target, +these are unconditional constant assertions, checked while the bindings crate +is compiled. They compare Rust sizes, alignments and field offsets against the +values obtained from Clang, without executing kernel code. + +Do not disable these checks to make an incompatible binding generator compile. +A type can have the right size and still have incorrect member offsets. For +example, an empty GNU C struct may have size zero and alignment 64; losing that +alignment moves later fields in an enclosing struct without necessarily +changing the enclosing size. + +Known generator requirements +---------------------------- + +Unmodified bindgen 0.73.2 loses explicit alignment on defined zero-sized C +structs. It also omits the named anonymous record member accepted by Clang's +``-fms-extensions`` in a declaration such as:: + + struct payload { unsigned long words[20]; }; + union container { + struct payload; + unsigned long other; + }; + +The generator used for this tree must preserve these layouts. Use a corrected +bindgen build through the normal ``BINDGEN=/path/to/bindgen`` make variable. +``tools/aurora-bindgen/`` contains pinned patches, a private build recipe and +compile-only valid-input tests. It never installs a system-wide tool. The +associated build provenance must retain the generator source revision, +patches, binary digest, and valid-input C/Rust layout checks. A version string +alone does not identify a locally corrected tool. Do not add dummy kernel +storage, hardcode task offsets, or change scheduler configuration to conceal a +binding generator error. + +After a binding layout correction, rebuild the kernel Image and all matching +modules. The generated bindings are consumed by the Rust kernel crate and by +multiple drivers; rebuilding just one module can leave incompatible layouts in +the same kernel. Keep the old Image and its own modules together as a fallback. + +The minimum accepted bindgen version is 0.71.1. Its Rust 1.77 ``offset_of`` +feature enables unconditional const layout assertions; the configured Rust +target 1.85 selects that path rather than ``#[test]``-only functions. The pinned +0.73.2 tool has the same behavior. See the source links and explicit target +accounting in ``tools/aurora-bindgen/README.rst``. diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst index b78ed0efa78460..c03031b2410866 100644 --- a/Documentation/rust/index.rst +++ b/Documentation/rust/index.rst @@ -37,6 +37,7 @@ more details. coding-guidelines arch-support testing + bindgen-layout-checks You can also find learning materials for Rust in its section in :doc:`../process/kernel-docs`. diff --git a/rust/Makefile b/rust/Makefile index bec9726f256ce9..2fee22bdb0e146 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -444,10 +444,13 @@ endif bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ # `--rust-target` points to our minimum supported Rust version. +# Keep bindgen layout assertions enabled: a lost alignment can change field +# offsets without changing the containing type size. See Documentation/rust/ +# bindgen-layout-checks.rst for generator requirements. quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ - --use-core --with-derive-default --ctypes-prefix ffi --no-layout-tests \ + --use-core --with-derive-default --ctypes-prefix ffi \ --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) diff --git a/tools/aurora-bindgen/.gitattributes b/tools/aurora-bindgen/.gitattributes new file mode 100644 index 00000000000000..813c74962d8c7a --- /dev/null +++ b/tools/aurora-bindgen/.gitattributes @@ -0,0 +1 @@ +patches/*.patch whitespace=-blank-at-eol diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst new file mode 100644 index 00000000000000..bd05375d92ddd1 --- /dev/null +++ b/tools/aurora-bindgen/README.rst @@ -0,0 +1,72 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Private bindgen with checked C/Rust layouts +========================================== + +This package builds bindgen 0.73.2 with two generic binding-generation repairs: +preserve explicit alignment on defined zero-sized structs, and retain actual +unnamed named-tag fields exposed by Clang's type-field visitor. The latter is +needed for the Microsoft anonymous-member extension used by kernel namespaces. +Neither repair changes C structure storage or kernel configuration. + +``sources.json`` pins the official source archive, Cargo.lock, both patches and +the patched source files. It also records the qualified private tool's digest; +that digest is evidence for that build, not a promise that a different host, +compiler or build path produces identical bytes. + +Build in a fresh private directory:: + + python3 tools/aurora-bindgen/build.py --output-dir /absolute/new/tool-build + +Python 3.12 or newer, Cargo/Rust, Clang/libclang and patch are required. Cargo's +locked dependency graph is used, with at most two build jobs. ``--archive`` can +supply an already downloaded source archive; its digest is still checked. The +script builds and runs compile-only valid-input checks, then prints the exact +``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. +Use that argument for the complete kernel Image and matching modules build. + +Tests and target accounting +--------------------------- + +``test.py`` independently compiles the C/C++ fixtures and generated Rust. It +covers empty alignments 1/2/8/64/128, nesting, packed empty records, anonymous +struct/union members, field access and ordinary tag-only declarations. It +requires unconditional const layout assertions in the generated output. +Nothing is executed on a GPU, and the generated object files are not run. + +Example using an existing private tool and the installed unpatched baseline:: + + python3 tools/aurora-bindgen/test.py \ + --bindgen /absolute/fixed/bindgen --baseline-bindgen /usr/bin/bindgen \ + --output-dir /absolute/new/check-results + +The optional baseline must be an affected unpatched tool: alignment/member +fixtures must fail at compile time, while tag-only controls must pass unchanged. +Without that option only the repaired generator is checked. The Rust target is +always passed explicitly, defaulting to the Rust compiler's reported host; the +Clang target defaults to the same triple. ``--rust-target`` and ``--clang-target`` +allow explicit kernel/cross targets, with matching architecture required. +``--rust-sysroot`` and ``--rust-libdir`` support a separately prepared Rust core. +An unavailable Rust target/core emits a machine-readable SKIP and exit 77; +that target has not passed. Other compiler or layout errors fail normally. + +Repository guard and supported tool versions +-------------------------------------------- + +The kernel keeps bindgen layout checks enabled. Its minimum bindgen is 0.71.1, +and its fixed ``--rust-target 1.85`` enables the ``offset_of`` feature (available +from Rust 1.77). In both official 0.71.1 and the pinned 0.73.2 generator, that +feature selects unconditional const size/alignment/offset assertions rather +than ``#[test]`` functions. A normal bindings crate compile therefore checks +layouts without requiring a Rust unit-test configuration. This is source-backed +for the accepted minimum and pinned tool; future generator changes must retain +that capability. Never remove the assertions to accept an incompatible tool. + +Primary sources: + +* https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/features.rs +* https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/codegen/mod.rs +* https://github.com/rust-lang/rust-bindgen/tree/v0.73.2 + +The complete kernel build remains the configuration-specific layout gate; these +small generator fixtures do not replace it or establish runtime GPU stability. diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py new file mode 100644 index 00000000000000..cf3afbc4a53a5e --- /dev/null +++ b/tools/aurora-bindgen/build.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Fetch pinned source, apply generic fixes, and build/test a private bindgen.""" +import argparse,hashlib,json,subprocess,tarfile,urllib.request,shutil,os,sys +from pathlib import Path +P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 +O=a.output_dir.resolve();assert not O.exists(),'refusing to overwrite an existing source or tool';O.mkdir(parents=True) +m=json.loads((P/'sources.json').read_text());sha=lambda p:hashlib.sha256(p.read_bytes()).hexdigest();archive=O/'source.tar.gz' +if a.archive:shutil.copyfile(a.archive,archive) +else: + with urllib.request.urlopen(m['source_url'],timeout=60) as response, archive.open('wb') as f:shutil.copyfileobj(response,f) +assert sha(archive)==m['source_sha256'],'source digest mismatch' +with tarfile.open(archive) as t:t.extractall(O,filter='data') +S=O/m['source_directory'] +for name,h in m['patches'].items(): + p=P/'patches'/name;assert sha(p)==h,name;subprocess.run(['patch','--batch','--fuzz=0','-p1','-i',str(p)],cwd=S,check=True) +assert sha(S/'Cargo.lock')==m['cargo_lock_sha256'] +for f,h in m['patched_files'].items():assert sha(S/f)==h,f +cmd=['cargo','build','--locked','--release','-p','bindgen-cli','-j'+str(a.jobs)] +with (O/'build.log').open('w') as log:subprocess.run(cmd,cwd=S,stdout=log,stderr=subprocess.STDOUT,check=True) +binary=S/'target/release/bindgen';subprocess.run([sys.executable,str(P/'test.py'),'--bindgen',str(binary),'--output-dir',str(O/'tests')],check=True) +r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' ...; keep full kernel layout assertions enabled.') diff --git a/tools/aurora-bindgen/patches/0001-empty-alignment.patch b/tools/aurora-bindgen/patches/0001-empty-alignment.patch new file mode 100644 index 00000000000000..f595b480c6b8df --- /dev/null +++ b/tools/aurora-bindgen/patches/0001-empty-alignment.patch @@ -0,0 +1,20 @@ +--- a/bindgen/codegen/mod.rs ++++ b/bindgen/codegen/mod.rs +@@ -2468,6 +2468,17 @@ + } + } + } ++ } else if !is_union && !forward_decl { ++ // A zero-sized C struct can still require explicit alignment. ++ // Its alignment also determines the offsets of subsequent fields ++ // when it is embedded in another type. Do not add storage here. ++ if let Some(layout) = layout { ++ if struct_layout.requires_explicit_align(layout) && ++ layout.align > 1 ++ { ++ explicit_align = Some(layout.align); ++ } ++ } + } else if is_union && !forward_decl { + if let Some(layout) = layout { + // TODO(emilio): It'd be nice to unify this with the struct path above somehow. diff --git a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch new file mode 100644 index 00000000000000..395b537796a451 --- /dev/null +++ b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch @@ -0,0 +1,85 @@ +--- a/bindgen/clang.rs ++++ b/bindgen/clang.rs +@@ -1204,6 +1204,29 @@ + } + + impl Type { ++ /// Get actual record fields, including implicit anonymous fields omitted ++ /// from `clang_visitChildren` for Microsoft anonymous tag members. ++ pub(crate) fn fields(&self) -> Vec { ++ extern "C" fn collect_field( ++ cur: CXCursor, ++ data: CXClientData, ++ ) -> CXVisitorResult { ++ let fields = unsafe { &mut *data.cast::>() }; ++ fields.push(Cursor { x: cur }); ++ CXVisit_Continue ++ } ++ ++ let mut fields = Vec::::new(); ++ unsafe { ++ clang_Type_visitFields( ++ self.x, ++ collect_field, ++ (&mut fields as *mut Vec).cast(), ++ ); ++ } ++ fields ++ } ++ + /// Get this type's kind. + pub(crate) fn kind(&self) -> CXTypeKind { + self.x.kind +--- a/bindgen/ir/comp.rs ++++ b/bindgen/ir/comp.rs +@@ -1340,6 +1340,16 @@ + _ => false, + }); + ++ // Microsoft anonymous named-tag members have real implicit FieldDecls ++ // in the type's field list, but only a non-definition record cursor in ++ // child traversal. Keep positive field evidence: a tag declaration by ++ // itself is not necessarily storage in the containing record. ++ let mut implicit_fields: Vec<_> = ty ++ .fields() ++ .into_iter() ++ .filter(|field| field.spelling().is_empty() && !field.is_bit_field()) ++ .collect(); ++ + let mut maybe_anonymous_struct_field = None; + cursor.visit(|cur| { + if cur.kind() != CXCursor_FieldDecl { +@@ -1463,6 +1473,33 @@ + // definition, it's a valid inner type. + // + // [1]: https://github.com/rust-lang/rust-bindgen/issues/482 ++ if matches!(cur.kind(), CXCursor_StructDecl | CXCursor_UnionDecl) && ++ !cur.is_definition() ++ { ++ if let Some(index) = implicit_fields.iter().position(|field| { ++ field.cur_type().canonical_type() == ++ cur.cur_type().canonical_type() ++ }) { ++ let field = implicit_fields.remove(index); ++ let field_type = Item::from_ty_or_ref( ++ field.cur_type(), ++ field, ++ Some(potential_id), ++ ctx, ++ ); ++ ci.fields.append_raw_field(RawField::new( ++ None, ++ field_type, ++ field.raw_comment(), ++ Annotations::new(&field), ++ None, ++ field.public_accessible(), ++ field.offset_of_field().ok(), ++ )); ++ return CXChildVisit_Continue; ++ } ++ } ++ + let is_inner_struct = + cur.semantic_parent() == cursor || cur.is_definition(); + if !is_inner_struct { diff --git a/tools/aurora-bindgen/sources.json b/tools/aurora-bindgen/sources.json new file mode 100644 index 00000000000000..ca75ddc018d282 --- /dev/null +++ b/tools/aurora-bindgen/sources.json @@ -0,0 +1,19 @@ +{ + "version": "0.73.2", + "source_url": "https://codeload.github.com/rust-lang/rust-bindgen/tar.gz/refs/tags/v0.73.2", + "source_sha256": "5fc3277dd334e4bbbd7bdb8cad5d4fb87a667b2e95a2bae350a8d5424c280629", + "source_directory": "rust-bindgen-0.73.2", + "cargo_lock_sha256": "85edb9aab2d5649f39b805dc52e3f27ebf8af4d94b049601aee1c55fcf450393", + "patches": { + "0001-empty-alignment.patch": "eb71efca5c60adc9db6837c933ff377e8aefad53655057406ff9f1464dd3147f", + "0002-anonymous-tag-members.patch": "9072ec2fd9d0f0160615cbfd0d9cfe16d23cd0ae90747a2aa984b8694bdee9ec" + }, + "patched_files": { + "bindgen/codegen/mod.rs": "4c100fa631bdd655b509e1797411e341f9a954207d64f8f2efc285201f5f14d9", + "bindgen/clang.rs": "bd65a80b028045dec2ff1ca9ee347301726b30f94a0ee743a3041d4f598e45e4", + "bindgen/ir/comp.rs": "7d53770db84376699421c8057fbecbfc00e8a6f62ca38919bd4c718e56e40438" + }, + "qualified_private_tool_sha256": "dc794acc7b0d7e3cbfa1533b4b59f2518cd0cc1bff1dbb7fd104229c23c0d146", + "qualified_host_toolchain": "rustc/cargo1.98.1, LLVM/libclang22.1.8, nativeAArch64; kernel Rust1.85.0", + "runtime_tests_performed": false +} diff --git a/tools/aurora-bindgen/test.py b/tools/aurora-bindgen/test.py new file mode 100644 index 00000000000000..91b6d18d7fd4ad --- /dev/null +++ b/tools/aurora-bindgen/test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile valid generated bindings for explicitly matching C/Rust targets.""" +import argparse,hashlib,json,re,subprocess,sys +from pathlib import Path +P=Path(__file__).resolve().parent +ap=argparse.ArgumentParser();ap.add_argument('--bindgen',required=True);ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--rustc',default='rustc');ap.add_argument('--clang',default='clang');ap.add_argument('--rust-target');ap.add_argument('--clang-target');ap.add_argument('--rust-sysroot');ap.add_argument('--rust-libdir');ap.add_argument('--baseline-bindgen');a=ap.parse_args() +O=a.output_dir.resolve();assert not O.exists(),'use a fresh evidence directory';O.mkdir(parents=True) +def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) +host=next(x.split(': ',1)[1] for x in subprocess.check_output([a.rustc,'-vV'],text=True).splitlines() if x.startswith('host: '));rt=a.rust_target or host;ct=a.clang_target or rt +arch=lambda t:{'arm64':'aarch64','amd64':'x86_64'}.get(t.split('-')[0],t.split('-')[0]) +assert arch(rt)==arch(ct),('C/Rust architecture mismatch',ct,rt) +rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--emit=obj','-Cpanic=abort','--target='+rt] +if a.rust_sysroot:rust+=['--sysroot='+a.rust_sysroot] +if a.rust_libdir:rust+=['-L',str(Path(a.rust_libdir).resolve())] +probe=O/'target.rs';probe.write_text('#![no_std]\npub fn available() -> usize { core::mem::size_of::() }\n');r=run(rust+[str(probe),'-o',str(O/'target.o')]);(O/'target.log').write_text(r.stdout+r.stderr) +if r.returncode: + status='SKIP' if 'E0463' in r.stderr and 'core' in r.stderr else 'FAIL' + result={'status':status,'reason':'matching Rust target/core unavailable' if status=='SKIP' else 'target preflight compile failed','clang_target':ct,'rust_target':rt,'rust_argv':rust,'log':'target.log'};(O/'result.json').write_text(json.dumps(result,indent=2)+'\n');print(json.dumps(result));sys.exit(77 if status=='SKIP' else 1) +access=''' +unsafe fn check_fields(a: &mut outer, b: &mut wrapper, c: &mut union_member, d: &mut trailing_member) { + a.__bindgen_anon_1.__bindgen_anon_1.words[19] = 1; + b.__bindgen_anon_1.words[19] = 2; + c.__bindgen_anon_1.bytes[23] = 3; + d.__bindgen_anon_1.words[0] = 4; +} +''' +cases=[];equal={};toolset=[('fixed',a.bindgen)]+([('baseline',a.baseline_bindgen)] if a.baseline_bindgen else []) +for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[])]: + header=P/'tests'/fixture + # Also ask the C compiler to accept the ordinary fixture independently. + ccmd=[a.clang,'--target='+ct,'-x',language,*extra,'-Wno-microsoft-anon-tag','-c',str(header),'-o',str(O/(fixture+'.c.o'))];cr=run(ccmd);(O/(fixture+'.c.log')).write_text(cr.stdout+cr.stderr);assert cr.returncode==0,(fixture,cr.stderr) + for label,tool in toolset: + cmd=[tool,str(header),'--rust-target','1.85','--no-doc-comments','--use-core','--no-derive-debug','--ctypes-prefix','core::ffi','--','-target',ct,'-x',language,*extra] + g=run(cmd);assert g.returncode==0,(fixture,g.stderr) + assert 'const _: ()' in g.stdout and '#[test]' not in g.stdout,'generator did not emit unconditional const layout checks' + rs=O/(fixture+'.'+label+'.rs');rs.write_text('#![no_std]\n'+g.stdout+(access if fixture=='anonymous-members.h' and label=='fixed' else '')) + rcmd=rust+[str(rs),'-o',str(O/(fixture+'.'+label+'.o'))];r=run(rcmd);(O/(fixture+'.'+label+'.log')).write_text(r.stdout+r.stderr) + expected=label=='fixed' or fixture=='tag-only.h';assert (r.returncode==0)==expected,(fixture,label,r.stderr) + if not expected:assert 'E0080' in r.stderr,'baseline failure must be a const layout mismatch' + cases.append({'fixture':fixture,'tool':label,'clang_argv':ccmd,'generator_argv':cmd,'rust_argv':rcmd,'compile_exit':r.returncode,'expected':'layout compile success' if expected else 'const layout rejection','generated_sha256':hashlib.sha256(rs.read_bytes()).hexdigest()}) + if fixture=='tag-only.h':equal[label]=g.stdout +if a.baseline_bindgen:assert equal['baseline']==equal['fixed'],'ordinary tag-only binding changed' +result={'status':'PASS','clang_target':ct,'rust_target':rt,'rust_host':host,'explicit_rust_target':True,'cases':cases,'baseline_checked':bool(a.baseline_bindgen),'hardware_executed':False};(O/'result.json').write_text(json.dumps(result,indent=2)+'\n');print('PASS matching C/Rust targets:',ct,rt) diff --git a/tools/aurora-bindgen/tests/aligned-empty.h b/tools/aurora-bindgen/tests/aligned-empty.h new file mode 100644 index 00000000000000..9ac1f0546a6236 --- /dev/null +++ b/tools/aurora-bindgen/tests/aligned-empty.h @@ -0,0 +1,13 @@ +struct incomplete; +struct empty1 {} __attribute__((aligned(1))); +struct outer1 { unsigned char prefix[12]; struct empty1 value; unsigned long after; }; +struct empty2 {} __attribute__((aligned(2))); +struct outer2 { unsigned char prefix[12]; struct empty2 value; unsigned long after; }; +struct empty8 {} __attribute__((aligned(8))); +struct outer8 { unsigned char prefix[12]; struct empty8 value; unsigned long after; }; +struct empty64 {} __attribute__((aligned(64))); +struct outer64 { unsigned char prefix[12]; struct empty64 value; unsigned long after; }; +struct empty128 {} __attribute__((aligned(128))); +struct outer128 { unsigned char prefix[12]; struct empty128 value; unsigned long after; }; +struct packed_empty {} __attribute__((packed,aligned(64))); +struct packed_container { char prefix; struct packed_empty value; unsigned long after; }; diff --git a/tools/aurora-bindgen/tests/aligned-empty.hpp b/tools/aurora-bindgen/tests/aligned-empty.hpp new file mode 100644 index 00000000000000..9ac1f0546a6236 --- /dev/null +++ b/tools/aurora-bindgen/tests/aligned-empty.hpp @@ -0,0 +1,13 @@ +struct incomplete; +struct empty1 {} __attribute__((aligned(1))); +struct outer1 { unsigned char prefix[12]; struct empty1 value; unsigned long after; }; +struct empty2 {} __attribute__((aligned(2))); +struct outer2 { unsigned char prefix[12]; struct empty2 value; unsigned long after; }; +struct empty8 {} __attribute__((aligned(8))); +struct outer8 { unsigned char prefix[12]; struct empty8 value; unsigned long after; }; +struct empty64 {} __attribute__((aligned(64))); +struct outer64 { unsigned char prefix[12]; struct empty64 value; unsigned long after; }; +struct empty128 {} __attribute__((aligned(128))); +struct outer128 { unsigned char prefix[12]; struct empty128 value; unsigned long after; }; +struct packed_empty {} __attribute__((packed,aligned(64))); +struct packed_container { char prefix; struct packed_empty value; unsigned long after; }; diff --git a/tools/aurora-bindgen/tests/anonymous-members.h b/tools/aurora-bindgen/tests/anonymous-members.h new file mode 100644 index 00000000000000..56b129a6aab146 --- /dev/null +++ b/tools/aurora-bindgen/tests/anonymous-members.h @@ -0,0 +1,9 @@ +struct payload { unsigned long words[20]; }; +union tag_union { unsigned long word; unsigned char bytes[24]; }; +struct outer { unsigned before; union { struct payload; void *ptr; }; unsigned after; }; +struct wrapper { unsigned prefix; struct payload; unsigned suffix; }; +struct union_member { char prefix; union tag_union; unsigned suffix; }; +struct normal_named { char prefix; struct payload named; unsigned suffix; }; +struct anonymous_definition { char prefix; struct { unsigned long inner; }; unsigned suffix; }; +struct forward_only { struct never_defined *ptr; unsigned suffix; }; +struct trailing_member { unsigned prefix; struct payload; }; diff --git a/tools/aurora-bindgen/tests/tag-only.h b/tools/aurora-bindgen/tests/tag-only.h new file mode 100644 index 00000000000000..525bc24774be0d --- /dev/null +++ b/tools/aurora-bindgen/tests/tag-only.h @@ -0,0 +1,4 @@ +struct payload { unsigned long words[20]; }; +struct harmless_tag { unsigned before; struct payload; unsigned after; }; +struct ordinary_forward { struct forward_tag; void *ptr; unsigned after; }; +struct normal_named { unsigned before; struct payload named; unsigned after; }; From 75e638ef173ed9f192914f5327524d86e3e41ba8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:19:36 -0400 Subject: [PATCH 059/631] drm/asahi: wait for host timestamp writes in compute prefixes Explicit compute prefixes must cover timestamp copies performed by the host after GPU retirement. Add completion dependencies and retained status checks for timestamp publishers while preserving firmware ordering for ordinary compute work. Keep the earliest failed compute timestamp as a bounded error witness so cancellation or recovery-spared failures cannot become valid query results after pruning. Reuse ordered prefix lookup and retain only independent fences, without extending packet or mapping lifetimes. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e4d41aaa9c0cdd145d5d07fe0b3d0eb209bf0879) --- drivers/gpu/drm/asahi/g17/queue/mod.rs | 48 +++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 65bfdf10ab2579..676180319ec63f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -21,7 +21,7 @@ use crate::{ use core::ops::Range; use kernel::{ c_str, - dma_fence::FenceContexts, + dma_fence::{FenceContexts, RawDmaFence}, prelude::*, sync::{Arc, LockClassKey}, xarray, @@ -147,7 +147,24 @@ impl Queue { Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); - self.frontiers[index].retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + // One earliest failed timestamp covers every prefix containing a later + // failure, without retaining the producer's packet or mappings. + let mut retained_timestamp_failure = false; + self.frontiers[index].retain(|(_, fences)| { + let status = job::fence_status(&fences.completed); + if status == 0 { + return true; + } + if engine == Engine::Compute + && status < 0 + && fences.ready.raw() == fences.completed.raw() + && !retained_timestamp_failure + { + retained_timestamp_failure = true; + return true; + } + false + }); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { self.ensure_compute()?; @@ -192,9 +209,32 @@ impl Queue { &[] }; let count = frontier.len(); - let mut dependencies = KVec::with_capacity(inputs.len() + count, GFP_KERNEL)?; - let mut checked_inputs = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; + let timestamp_prefix = order.wait_through[index]; + let timestamps = if engine == Engine::Compute && timestamp_prefix.is_some() { + let frontier = self.frontiers[index].as_slice(); + let end = frontier + .partition_point(|(sequence, _)| Order::contains(timestamp_prefix, *sequence)); + &frontier[..end] + } else { + &[] + }; + let timestamp_count = timestamps + .iter() + .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) + .count(); + let mut dependencies = + KVec::with_capacity(inputs.len() + count + timestamp_count, GFP_KERNEL)?; + let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + // Same-engine parents order GPU execution, not the host's timestamp + // writes. Only timestamp publishers need these completion waits and + // status checks; ordinary compute keeps its existing queue ordering. + for (_, fences) in timestamps { + if fences.ready.raw() == fences.completed.raw() { + checked_inputs.push(fences.completed.clone(), GFP_KERNEL)?; + dependencies.push(fences.ready.clone(), GFP_KERNEL)?; + } + } for (_, fences) in frontier { dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; From 661a9837c13eab40be2d57116b34693e07a1e703 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 4 Oct 2026 02:20:09 -0400 Subject: [PATCH 060/631] tools: preserve bindgen recipe validation across Python modes Reject optimized Python before creating output so archive and layout assertions cannot be disabled. Exercise bitfield generation with unknown Rust lints denied, and document explicit compiler selection separately from the generator language target. Document the synchronous Clang visitor safety requirements and refresh the affected pinned patch and source digests. Retain the previously qualified binary digest as historical build evidence without claiming a rebuilt binary is identical. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bc7394673cbed4ef22971a49926be6070a7312be) --- tools/aurora-bindgen/README.rst | 14 ++++++++++++++ tools/aurora-bindgen/build.py | 3 +++ .../patches/0002-anonymous-tag-members.patch | 7 ++++++- tools/aurora-bindgen/sources.json | 7 ++++--- tools/aurora-bindgen/test.py | 10 ++++++++-- tools/aurora-bindgen/tests/bitfields.h | 11 +++++++++++ 6 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 tools/aurora-bindgen/tests/bitfields.h diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst index bd05375d92ddd1..2c560aa13f96e4 100644 --- a/tools/aurora-bindgen/README.rst +++ b/tools/aurora-bindgen/README.rst @@ -13,6 +13,8 @@ Neither repair changes C structure storage or kernel configuration. the patched source files. It also records the qualified private tool's digest; that digest is evidence for that build, not a promise that a different host, compiler or build path produces identical bytes. +The manifest identifies when that digest predates comment-only patch changes; +each rebuild must record its own binary digest. Build in a fresh private directory:: @@ -24,6 +26,8 @@ supply an already downloaded source archive; its digest is still checked. The script builds and runs compile-only valid-input checks, then prints the exact ``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. Use that argument for the complete kernel Image and matching modules build. +Both scripts reject optimized Python (``-O``, ``-OO`` or ``PYTHONOPTIMIZE``) +before creating output, since their validation assertions must remain enabled. Tests and target accounting --------------------------- @@ -32,6 +36,9 @@ Tests and target accounting covers empty alignments 1/2/8/64/128, nesting, packed empty records, anonymous struct/union members, field access and ordinary tag-only declarations. It requires unconditional const layout assertions in the generated output. +An additional bitfield fixture rejects unknown Rust lints in the fixed tool's +output. This fixture checks compiler compatibility, not the layout repairs, +and is excluded from the optional unpatched-baseline comparison. Nothing is executed on a GPU, and the generated object files are not run. Example using an existing private tool and the installed unpatched baseline:: @@ -50,6 +57,13 @@ allow explicit kernel/cross targets, with matching architecture required. An unavailable Rust target/core emits a machine-readable SKIP and exit 77; that target has not passed. Other compiler or layout errors fail normally. +Select ``--rustc`` explicitly to check the compiler used for the kernel. The +generator's ``--rust-target 1.85`` selects emitted language features; it does +not select the Rust compiler used by the tests. A compatibility adapter may +remove an exact generated lint attribute unsupported by that compiler, while +retaining all layout assertions. Its underlying binary must be the corrected +generator, and its changes and both tool digests belong in the build record. + Repository guard and supported tool versions -------------------------------------------- diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py index cf3afbc4a53a5e..5895a84a7b5c44 100644 --- a/tools/aurora-bindgen/build.py +++ b/tools/aurora-bindgen/build.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """Fetch pinned source, apply generic fixes, and build/test a private bindgen.""" +if not __debug__: + raise SystemExit("optimized Python is unsupported: validation assertions must remain enabled") + import argparse,hashlib,json,subprocess,tarfile,urllib.request,shutil,os,sys from pathlib import Path P=Path(__file__).resolve().parent;ap=argparse.ArgumentParser();ap.add_argument('--output-dir',required=True,type=Path);ap.add_argument('--archive',type=Path);ap.add_argument('--jobs',type=int,default=2);a=ap.parse_args();assert 1<=a.jobs<=2 diff --git a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch index 395b537796a451..7881d882372359 100644 --- a/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch +++ b/tools/aurora-bindgen/patches/0002-anonymous-tag-members.patch @@ -1,6 +1,6 @@ --- a/bindgen/clang.rs +++ b/bindgen/clang.rs -@@ -1204,6 +1204,29 @@ +@@ -1204,6 +1204,34 @@ } impl Type { @@ -11,12 +11,17 @@ + cur: CXCursor, + data: CXClientData, + ) -> CXVisitorResult { ++ // SAFETY: clang_Type_visitFields invokes this callback synchronously ++ // with the live Vec pointer supplied below; callbacks do not overlap. + let fields = unsafe { &mut *data.cast::>() }; + fields.push(Cursor { x: cur }); + CXVisit_Continue + } + + let mut fields = Vec::::new(); ++ // SAFETY: self.x is the live Clang type owned by the parsing context. ++ // The visitor only borrows fields during this call, and Clang does not ++ // retain the callback data after returning. + unsafe { + clang_Type_visitFields( + self.x, diff --git a/tools/aurora-bindgen/sources.json b/tools/aurora-bindgen/sources.json index ca75ddc018d282..cf97c6753218ee 100644 --- a/tools/aurora-bindgen/sources.json +++ b/tools/aurora-bindgen/sources.json @@ -6,14 +6,15 @@ "cargo_lock_sha256": "85edb9aab2d5649f39b805dc52e3f27ebf8af4d94b049601aee1c55fcf450393", "patches": { "0001-empty-alignment.patch": "eb71efca5c60adc9db6837c933ff377e8aefad53655057406ff9f1464dd3147f", - "0002-anonymous-tag-members.patch": "9072ec2fd9d0f0160615cbfd0d9cfe16d23cd0ae90747a2aa984b8694bdee9ec" + "0002-anonymous-tag-members.patch": "3c1f6a9fdf02b7cd6f11b1d0787eec7d8a14cf6f140f414cc3069eeb75524562" }, "patched_files": { "bindgen/codegen/mod.rs": "4c100fa631bdd655b509e1797411e341f9a954207d64f8f2efc285201f5f14d9", - "bindgen/clang.rs": "bd65a80b028045dec2ff1ca9ee347301726b30f94a0ee743a3041d4f598e45e4", + "bindgen/clang.rs": "4271adf1fd24f9a8e58e2a69c48f3de6574318461203b513d34308e38852dcd1", "bindgen/ir/comp.rs": "7d53770db84376699421c8057fbecbfc00e8a6f62ca38919bd4c718e56e40438" }, "qualified_private_tool_sha256": "dc794acc7b0d7e3cbfa1533b4b59f2518cd0cc1bff1dbb7fd104229c23c0d146", "qualified_host_toolchain": "rustc/cargo1.98.1, LLVM/libclang22.1.8, nativeAArch64; kernel Rust1.85.0", - "runtime_tests_performed": false + "runtime_tests_performed": false, + "qualified_private_tool_note": "Digest identifies the previously qualified binary before safety-comment-only patch changes; a rebuild from this manifest must record its own binary digest." } diff --git a/tools/aurora-bindgen/test.py b/tools/aurora-bindgen/test.py index 91b6d18d7fd4ad..6d93f8b35b9d8c 100644 --- a/tools/aurora-bindgen/test.py +++ b/tools/aurora-bindgen/test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-only """Compile valid generated bindings for explicitly matching C/Rust targets.""" +if not __debug__: + raise SystemExit("optimized Python is unsupported: validation assertions must remain enabled") + import argparse,hashlib,json,re,subprocess,sys from pathlib import Path P=Path(__file__).resolve().parent @@ -10,7 +13,7 @@ def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) host=next(x.split(': ',1)[1] for x in subprocess.check_output([a.rustc,'-vV'],text=True).splitlines() if x.startswith('host: '));rt=a.rust_target or host;ct=a.clang_target or rt arch=lambda t:{'arm64':'aarch64','amd64':'x86_64'}.get(t.split('-')[0],t.split('-')[0]) assert arch(rt)==arch(ct),('C/Rust architecture mismatch',ct,rt) -rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--emit=obj','-Cpanic=abort','--target='+rt] +rust=[a.rustc,'--edition=2021','--crate-type=rlib','--crate-name','layout_check','--deny=unknown-lints','--emit=obj','-Cpanic=abort','--target='+rt] if a.rust_sysroot:rust+=['--sysroot='+a.rust_sysroot] if a.rust_libdir:rust+=['-L',str(Path(a.rust_libdir).resolve())] probe=O/'target.rs';probe.write_text('#![no_std]\npub fn available() -> usize { core::mem::size_of::() }\n');r=run(rust+[str(probe),'-o',str(O/'target.o')]);(O/'target.log').write_text(r.stdout+r.stderr) @@ -26,11 +29,14 @@ def run(cmd):return subprocess.run(cmd,capture_output=True,text=True) } ''' cases=[];equal={};toolset=[('fixed',a.bindgen)]+([('baseline',a.baseline_bindgen)] if a.baseline_bindgen else []) -for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[])]: +for fixture,language,extra in [('aligned-empty.h','c',[]),('aligned-empty.hpp','c++',[]),('anonymous-members.h','c',['-fms-extensions']),('tag-only.h','c',[]),('bitfields.h','c',[])]: header=P/'tests'/fixture # Also ask the C compiler to accept the ordinary fixture independently. ccmd=[a.clang,'--target='+ct,'-x',language,*extra,'-Wno-microsoft-anon-tag','-c',str(header),'-o',str(O/(fixture+'.c.o'))];cr=run(ccmd);(O/(fixture+'.c.log')).write_text(cr.stdout+cr.stderr);assert cr.returncode==0,(fixture,cr.stderr) for label,tool in toolset: + # The baseline comparison covers layout repairs; this fixture checks the fixed tool + # against the selected compiler, including compatibility adapters when needed. + if fixture == 'bitfields.h' and label == 'baseline':continue cmd=[tool,str(header),'--rust-target','1.85','--no-doc-comments','--use-core','--no-derive-debug','--ctypes-prefix','core::ffi','--','-target',ct,'-x',language,*extra] g=run(cmd);assert g.returncode==0,(fixture,g.stderr) assert 'const _: ()' in g.stdout and '#[test]' not in g.stdout,'generator did not emit unconditional const layout checks' diff --git a/tools/aurora-bindgen/tests/bitfields.h b/tools/aurora-bindgen/tests/bitfields.h new file mode 100644 index 00000000000000..47a4332ccf4cf9 --- /dev/null +++ b/tools/aurora-bindgen/tests/bitfields.h @@ -0,0 +1,11 @@ +struct bitfields { + unsigned int enabled : 1; + unsigned int count : 7; + signed int delta : 8; + unsigned int reserved : 16; +}; +struct bitfield_container { + unsigned long before; + struct bitfields value; + unsigned long after; +}; From 3070a1f9920638b66723b4a99102ff1fced9c72e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 01:53:22 -0400 Subject: [PATCH 061/631] drm/asahi: release successful submission work-state ownership Drop the aggregate bookkeeping reference once every command and the enqueue gate have succeeded, before signalling its fence. Scheduler packets can otherwise retain all work-state slots while their cleanup waits behind another submission on the same worker. Preserve failed submissions and independent runtime owners for retirement and replay. Reject work-state access after successful aggregate completion instead of creating a replacement allocation. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 81d324022d0f97346639b06a6abaa8b86d480ead) --- drivers/gpu/drm/asahi/g17/completion.rs | 2 +- drivers/gpu/drm/asahi/g17/fence.rs | 35 ++++++++++++++----- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 8 ++--- .../gpu/drm/asahi/g17/queue/render/parked.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime.rs | 2 +- .../drm/asahi/g17/runtime/render/submit.rs | 2 +- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 1151a82084cee7..d9c3896fe9368b 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -155,7 +155,7 @@ impl Completion { pub(crate) fn status(&self) -> &Arc { &self.status } - pub(crate) fn work_state(&self) -> Arc { + pub(crate) fn work_state(&self) -> Result> { self.member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 32d25b3b608cb2..1217d7e7b3f4e1 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -11,9 +11,10 @@ use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; use kernel::{ bindings, c_str, dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + new_mutex, prelude::*, str::CStr, - sync::Arc, + sync::{Arc, Mutex}, }; pub(crate) struct CompletionFence; @@ -40,24 +41,29 @@ pub(crate) fn independent(contexts: &FenceContexts) -> Result, - work_state: Arc, + /// Successful retirement drops only this bookkeeping owner; runtime owners + /// and failed submissions keep their work-state allocation. + #[pin] + work_state: Mutex>>, remaining: AtomicU32, error: AtomicI32, } impl Submission { fn new(fence: UserFence, context: Arc) -> Result> { - Ok(Arc::new( - Self { + let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; + Arc::pin_init( + pin_init!(Self { fence, - work_state: Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?, + work_state <- new_mutex!(Some(work_state), "G17 submission work state"), remaining: AtomicU32::new(1), error: AtomicI32::new(0), - }, + }), GFP_KERNEL, - )?) + ) } pub(crate) fn fence(&self) -> Fence { @@ -86,6 +92,12 @@ impl Submission { let error = self.error.load(Ordering::Acquire); if error != 0 { self.fence.set_error(Error::from_errno(error)); + } else { + // Scheduler cleanup can wait behind submission on the same worker. + // Once every member and enqueue have succeeded, retained packets + // must not prevent new submissions from acquiring work-state nodes. + let retired = self.work_state.lock().take(); + drop(retired); } self.fence.signal(); } @@ -108,8 +120,13 @@ pub(crate) struct Member { } impl Member { - pub(crate) fn work_state(&self) -> Arc { - self.submission.work_state.clone() + pub(crate) fn work_state(&self) -> Result> { + self.submission + .work_state + .lock() + .as_ref() + .cloned() + .ok_or(ECANCELED) } pub(crate) fn complete(&self, result: Result) { diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 832d5073ecc7bc..2a30cd54cecd1b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1218,7 +1218,7 @@ impl Queue { else { return Err(EINVAL); }; - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let work_word = work.word(); if work_word == u32::MAX { return Err(EINVAL); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 3a7852be1dab03..c3ac091333c272 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -162,7 +162,7 @@ impl Pair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let word = first.completion.work_state().word(); + let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } @@ -271,7 +271,7 @@ impl Pair { } let ordinal = self.ordinal; ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let node = work.node_va()?; let word = work.word(); let layout = target::Layout::new(pass, self.clusters)?; @@ -417,7 +417,7 @@ impl Pair { let qids = self.qids(); let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); - let work = packet.completion.work_state(); + let work = packet.completion.work_state()?; let work_slot = work.render_slot(self.slot)?; self.memory.bind_work_slot(ordinal, qids, work_slot)?; let payload = work.channel_payload(self.slot, ordinal)?; @@ -1382,7 +1382,7 @@ impl Pair { buffer_id, pm_generation, self.ordinal, - first.completion.work_state().word(), + first.completion.work_state()?.word(), )?; (Some(memory), Some(persistent), Some(aliases)) }; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 35955810a5af1c..3d67578e2aaf98 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -120,7 +120,7 @@ impl ParkedPair { let Validated::Render { pass, .. } = &first.command else { return Err(EINVAL); }; - let word = first.completion.work_state().word(); + let word = first.completion.work_state()?.word(); if word == u32::MAX { return Err(EINVAL); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6fe67478a777f7..8978693a789eab 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -763,7 +763,7 @@ impl Backend { return Err(EIO); } let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state().acquire()?; + packet.completion.work_state()?.acquire()?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 86a00e7bd741a5..0586dfdc0944f1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -66,7 +66,7 @@ impl Backend { } let work_state = { let preparation = self.shared.preparations.enter(interruptible)?; - let result = packet.completion.work_state().acquire(); + let result = packet.completion.work_state().and_then(|state| state.acquire()); if preparation.is_current() { result } else { From af0a51fac17f8ad34703e6a1e067e0b0d2b1fce6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 01:54:32 -0400 Subject: [PATCH 062/631] drm/asahi: preserve exact historical command prefixes Accept queue-local render and compute ordinals from extensible payloads. Validate them against frozen ioctl-entry counters before publication and merge them with relative barriers without including unrelated newer work. Keep the reserved compute layout slot before the new fields and require it to remain zero. Existing short payloads retain their zero-filled extension behavior. Advertise the protocol only for G17 queues. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0f7410fd5b368d06ab17d802720f9d76e12880c) --- drivers/gpu/drm/asahi/g17/command.rs | 8 +++++ drivers/gpu/drm/asahi/g17/job.rs | 12 ++++++- drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 11 ++++++- include/uapi/drm/asahi_drm.h | 45 ++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index fef478310c23b2..47f9c0048e673a 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -289,6 +289,13 @@ pub(crate) enum Validated { } impl Payload { + pub(crate) fn prior(&self) -> [u64; 2] { + match self { + Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + } + } + pub(crate) fn validate( &self, window: &UscWindow, @@ -399,6 +406,7 @@ impl Payload { } Self::Compute(cmd, attachments) => { if cmd.flags != 0 + || cmd.scs_layout != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index f16897efa00b8f..c05b5119790fc1 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -33,7 +33,7 @@ impl Engine { } /// Logical sequence numbers do not wrap with firmware timestamps. A zero barrier -/// includes preceding ioctls; the sentinel means independent of that engine. +/// includes preceding ioctls; the sentinel leaves only an explicit historical prefix. #[derive(Clone, Copy)] pub(crate) struct Order { pub(crate) sequence: u64, @@ -45,11 +45,16 @@ impl Order { base: [u64; 2], preceding: [u16; 2], barriers: [u16; 2], + prior: [u64; 2], engine: Engine, ) -> Result { let mut sequences = [0; 2]; let mut wait_through = [None; 2]; for index in 0..2 { + // Earlier commands in this ioctl are not historical commands. + if prior[index] > base[index] { + return Err(EINVAL); + } sequences[index] = base[index] .checked_add(u64::from(preceding[index])) .and_then(|value| value.checked_add(1)) @@ -64,6 +69,11 @@ impl Order { .ok_or(EOVERFLOW)?, ); } + if prior[index] != 0 { + wait_through[index] = Some( + wait_through[index].map_or(prior[index], |last| last.max(prior[index])), + ); + } } Ok(Self { sequence: sequences[engine.index()], diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 1ccbb9b708ce0b..cba7b6931c04ed 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1059,7 +1059,8 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= - uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64; + uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 676180319ec63f..441b64602ae234 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -282,6 +282,9 @@ impl Queue { ) -> Result<(KVec<(Validated, Order)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); + // All historical ordinals use the same entry counters; publication + // starts only after the entire ioctl has been validated and pinned. + let base = self.sequence; let mut preceding = [0; 2]; let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; while let Some(command) = parser.next()? { @@ -295,7 +298,13 @@ impl Queue { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; - let order = Order::new(self.sequence, preceding, command.barriers, engine)?; + let order = Order::new( + base, + preceding, + command.barriers, + command.payload.prior(), + engine, + )?; preceding[engine.index()] += 1; pending.push((payload, order), GFP_KERNEL)?; } diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index ff2fec6485d2de..588226e72381b4 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -396,6 +396,13 @@ struct drm_asahi_queue_limits { * Registration may still be refused. */ DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, + + /** + * @DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS: Render and compute payloads + * support exact queue-local historical prefixes in prior_vdm/prior_cdm. + * These fields supplement the header barriers. + */ + DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, }; /** @@ -1476,6 +1483,24 @@ struct drm_asahi_cmd_render { /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ __u32 bg_partial_rsrc_spec_hi; + + /** + * @prior_vdm: Historical render prefix, or zero for no added dependency. + * + * Requires DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render + * commands accepted on this queue since creation, starting at one. A + * nonzero value must not exceed the render count at this ioctl's entry; + * it waits for every render command through that ordinal. The effective + * dependency is the maximum of this prefix and the header's resolved + * vdm_barrier. Header NONE does not suppress this field. + */ + __u64 prior_vdm; + + /** + * @prior_cdm: Historical compute prefix; see @prior_vdm, using the + * queue's compute command count and the header's cdm_barrier instead. + */ + __u64 prior_cdm; }; /** @@ -1529,6 +1554,26 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), + /** @scs_layout: Reserved; must be zero. */ + __u64 scs_layout; + + /** + * @prior_vdm: Historical render prefix, or zero for no added dependency. + * + * Requires DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render + * commands accepted on this queue since creation, starting at one. A + * nonzero value must not exceed the render count at this ioctl's entry; + * it waits for every render command through that ordinal. The effective + * dependency is the maximum of this prefix and the header's resolved + * vdm_barrier. Header NONE does not suppress this field. + */ + __u64 prior_vdm; + + /** + * @prior_cdm: Historical compute prefix; see @prior_vdm, using the + * queue's compute command count and the header's cdm_barrier instead. + */ + __u64 prior_cdm; }; /** From 9a76406db0ed989031c646f019c7de7e6074372f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 28 Sep 2026 16:36:16 +1000 Subject: [PATCH 063/631] soc: apple: synchronize the ASCWrap mailbox provider with j700 Carry the provider-owned T8140 CPU lifecycle, removal serialization and runtime-PM retry fixes into the integration branch. Both provider files are byte-identical to j700 at 43bb4bbb36ec. This carries the mailbox portions of upstream commits: 10636bd526b981ea233d30d8ad6e7e09b2f9d733 2230686542a49f6d0ebbaf11a9e75fb0981abc20 41afc106aece4a6e319f3b20327d1842174aa0a8 Validation: matching Image/modules build; both GPU firmware processors initialize on local Neo with the rewritten consumer. Signed-off-by: Ryan Murray Assisted-by: LLM (cherry picked from commit 29db1d8ab517933c2e0114fb3c6bf9beda5417aa) --- drivers/soc/apple/mailbox.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 13c8c5594305e1..58d177c435bd8d 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -761,6 +761,9 @@ static int apple_mbox_probe(struct platform_device *pdev) if (ret) return ret; ret = devm_add_action_or_reset(dev, apple_mbox_quiesce, mbox); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, apple_mbox_quiesce, mbox); if (ret) return ret; From 6876c877c58ac3d53f6fa048d91ec49e280f8cde Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 05:17:22 -0400 Subject: [PATCH 064/631] drm/asahi: bring the G17 driver up on Neo hardware Acquire both GPU ASC lifecycle handles through the mailbox provider rather than looking for a nonexistent secondary platform resource. Initialize the firmware registry and queue/buffer arrays in place to bound the probe stack. Keep firmware and client metrics addresses in named fields. Their reversed order caused the first render to fault in firmware at the client address; the corrected addresses pass both light and heavy rendering on hardware. Report failed spared work to userspace independently of permanent VM admission failure, and skip already successful internal prefix fences. Validation on local Neo with unchanged Mesa: desktop at 2408x1506, four exact-image game replays, native/4K userspace checks, render/compute depth 1/2/4/8/16, four concurrent clients, and 45 matched benchmark windows. Static probe chain drops from 24784 to 13584 bytes (16384-byte audit limit). Latest named-address form also passes the 15-window hardware matrix. This is hardware bring-up, not a full-parity or leak-free qualification. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fbfb5d96f25efbf2b1571ca635b195e0ad4a54be) --- drivers/gpu/drm/asahi/file.rs | 2 +- drivers/gpu/drm/asahi/g17/buffer.rs | 32 ++++---- drivers/gpu/drm/asahi/g17/completion.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/buffer.rs | 14 +++- drivers/gpu/drm/asahi/g17/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/mod.rs | 20 +++-- drivers/gpu/drm/asahi/g17/queue/mod.rs | 6 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 32 +++++--- .../drm/asahi/g17/runtime/render/prepare.rs | 10 +-- drivers/gpu/drm/asahi/g17/status.rs | 34 +++++++-- drivers/gpu/drm/asahi/regs.rs | 74 +++++++++++++------ 11 files changed, 152 insertions(+), 88 deletions(-) diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 8d07cb26abb9a7..2a3baff1d0774c 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -487,7 +487,7 @@ impl File { let status = Self::vm_status(file, vm_id)?; UserSlice::new(user, size) .writer() - .write(&[vm_id, 0, status.get() as u32, 0])?; + .write(&[vm_id, 0, status.reported_error() as u32, 0])?; Ok(0) } diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 942d5047e5b44f..90ab8ae21aa14f 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -34,21 +34,17 @@ pub(crate) struct BufferIds { next_token: u64, } impl BufferIds { - pub(crate) fn new() -> Self { - let mut returned = [0; BUFFER_IDS]; - for (index, entry) in returned.iter_mut().enumerate() { - *entry = (BUFFER_IDS - index - 1) as u8; - } - Self { - entries: [BufferId { + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { + entries <- pin_init::init_array_from_fn(|_| BufferId { references: 0, owner: false, token: 0, - }; BUFFER_IDS], - returned, + }), + returned <- pin_init::init_array_from_fn(|index| (BUFFER_IDS - index - 1) as u8), returned_count: BUFFER_IDS, next_token: 0, - } + }) } fn remove_returned(&mut self, id: u8) { if let Some(at) = self.returned[..self.returned_count] @@ -458,14 +454,14 @@ pub(crate) struct MetricsLease { _backing: Arc, allocator: Arc, slot: usize, - addresses: [u64; 2], + addresses: fw::MetricsAddresses, } impl MetricsLease { /// Clears only the selected retired metrics entry before reusing it. pub(crate) fn new( ids: &Arc, object: &Arc, - bases: [u64; 2], + bases: fw::MetricsAddresses, ) -> Result { let mut selected = None; for (bank, used) in ids.used.iter().enumerate() { @@ -499,13 +495,13 @@ impl MetricsLease { _backing: object.clone(), allocator: ids.clone(), slot, - addresses: [0; 2], + addresses: fw::MetricsAddresses::default(), }; let offset = slot * fw::METRICS_STRIDE; - lease.addresses = [ - bases[0].checked_add(offset as u64).ok_or(EOVERFLOW)?, - bases[1].checked_add(offset as u64).ok_or(EOVERFLOW)?, - ]; + lease.addresses = fw::MetricsAddresses { + firmware: bases.firmware.checked_add(offset as u64).ok_or(EOVERFLOW)?, + client: bases.client.checked_add(offset as u64).ok_or(EOVERFLOW)?, + }; let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); object.word(offset)?; for index in 0..fw::SCENES { @@ -631,7 +627,7 @@ impl Manager { Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) } pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { - fw::scene_registers(index, self.scratch, self.metrics.addresses[1]) + fw::scene_registers(index, self.scratch, self.metrics.addresses.client) } pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { let scene = self.scenes.select(ordinal)?; diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index d9c3896fe9368b..777d54222c0eaa 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -324,7 +324,9 @@ impl Completion { Ok(()) }); if let Err(error) = result { - if error != ECANCELED && !self.spared() { + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { self.status.record(error); } self.fence.set_error(error); @@ -355,7 +357,9 @@ impl Completion { if let Some(guard) = self.vm_job.lock().as_ref() { guard.quarantine(); } - if error != ECANCELED && !self.spared() { + if self.spared() { + self.status.report_failure(error); + } else if error != ECANCELED { self.status.record(error); } self.fence.set_error(error); diff --git a/drivers/gpu/drm/asahi/g17/fw/buffer.rs b/drivers/gpu/drm/asahi/g17/fw/buffer.rs index 3ed336dbadaad9..ec6797637bdbe1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/fw/buffer.rs @@ -120,11 +120,17 @@ static_assert!(core::mem::offset_of!(State, block_table) == 0x44); static_assert!(core::mem::offset_of!(State, discard) == 0x84); static_assert!(core::mem::offset_of!(Scene, generation) == 0x4c); +/// The same metrics storage in the firmware and application address spaces. +#[derive(Clone, Copy, Default)] +pub(crate) struct MetricsAddresses { + pub(crate) firmware: u64, + pub(crate) client: u64, +} + pub(crate) struct GraphArgs<'a> { pub(crate) fw_va: u64, pub(crate) page_list_client: u64, - /// Firmware and client views, respectively, of this graph's metrics lease. - pub(crate) metrics: [u64; 2], + pub(crate) metrics: MetricsAddresses, pub(crate) blocks: &'a [u64], pub(crate) scratch: u64, pub(crate) discard: u64, @@ -168,8 +174,8 @@ impl Graph { } } for (index, scene) in self.scenes.iter_mut().enumerate() { - scene.metric_client = offset_va(args.metrics[1], (index * 4) as u64)?; - scene.metric_fw = offset_va(args.metrics[0], (index * 4) as u64)?; + scene.metric_client = offset_va(args.metrics.client, (index * 4) as u64)?; + scene.metric_fw = offset_va(args.metrics.firmware, (index * 4) as u64)?; scene.scratch = scratch + (index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE; scene.statistics = stats; } diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index 9d89262d8a3b68..f36ab9798996a7 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -47,10 +47,10 @@ pub(crate) struct QueueIds { } impl QueueIds { - pub(crate) const fn new() -> Self { - Self { - entries: [None; QID_COUNT], - } + pub(crate) fn new() -> impl Init { + kernel::try_init!(Self { + entries <- pin_init::init_array_from_fn(|_| None), + }) } /// Reuses only an exact owner/engine match. Preferred placement is for diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index cba7b6931c04ed..f379c1b69d2569 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -355,12 +355,8 @@ impl Coprocessor { GFP_KERNEL, )?; - let name = match role { - Role::Primary => c_str!("asc"), - Role::Secondary => c_str!("asc1"), - }; Ok(Coprocessor { - cpu: regs::CpuControl::new(pdev, name)?, + cpu: regs::CpuControl::new(pdev, index)?, running: false, rtkit: Self::receiver(dev, index, &state)?, state, @@ -924,8 +920,6 @@ impl Gpu { // The firmware only samples the kernel lower root's top-level entries when it starts, so // everything it uses there must be mapped first. - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary)?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary)?; let pm_metrics = KernelObject::new( dev, uat.kernel_lower_vm(), @@ -934,28 +928,32 @@ impl Gpu { mmu::PROT_GPU_FW_SHARED_RW, CpuMap::WriteCombined, )?; + let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; + let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) + .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; let render_global = uat.new_vm(0, geometry.user_range())?; let bridge = Bridge::enable(regs)?; // Install all session ownership before either CPU starts. Error unwinding then runs // Firmware::drop, stopping both CPUs before releasing shared objects or translations. - let mut firmware = KBox::new( - Firmware { + let mut firmware = KBox::try_init( + kernel::try_init!(Firmware { secondary, primary, bridge, rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), - queues: runtime::Registry::new(shared.clusters, shared.descriptor_flags)?, + queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, init: initdata::InitData::new(), _render_global: render_global, _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, uat, - }, + }), GFP_KERNEL, )?; let session = &mut *firmware; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 441b64602ae234..9d2bc056cce650 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -236,6 +236,12 @@ impl Queue { } } for (_, fences) in frontier { + // Successful completion includes host timestamp publication. Its + // ready fence adds no ordering and would prevent shallow inline + // publication. Failed and pending producers still need checking. + if job::fence_status(&fences.completed) > 0 { + continue; + } dependencies.push(fences.ready.clone(), GFP_KERNEL)?; firmware.push(fences.completed.clone(), GFP_KERNEL)?; } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 8978693a789eab..d100fef631630e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -63,24 +63,32 @@ pub(super) struct Registry { } impl Registry { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { - let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; - for _ in 0..QIDS { - compute.push(None, GFP_KERNEL)?; - } - let mut qids = kick::QueueIds::new(); - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(Self { - qids, - buffers: BufferIds::new(), + /// Initialize inside the firmware allocation instead of staging the whole + /// registry on the probe stack. No additional allocation is required. + pub(super) fn new( + clusters: u32, + descriptor_flags: [u32; 2], + ) -> impl Init { + kernel::try_init!(Self { + qids <- kick::QueueIds::new().chain(|qids| { + qids.reserve(1, DataMaster::Tiling, Some(0))?; + qids.reserve(2, DataMaster::Fragment, Some(1))?; + Ok(()) + }), + buffers <- BufferIds::new(), accounting: qos::Accounting::new(), metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, qos_ids: QosIds::new()?, admission: admission::Pool::new()?, render: render::RenderState::new(clusters, descriptor_flags)?, identities: queue::Identities::new(), - compute, + compute: { + let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; + for _ in 0..QIDS { + compute.push(None, GFP_KERNEL)?; + } + compute + }, compute_building: false, compute_scheduler_building: false, observations: compute::retirement::Batch::new()?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 1aa6cb67a90ebf..e8fc30560a8904 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -19,7 +19,7 @@ struct BuildResources { global: crate::mmu::Vm, metrics: Arc, metrics_ids: Arc, - metrics_bases: [u64; 2], + metrics_bases: crate::g17::fw::buffer::MetricsAddresses, config: Config, } impl BuildResources { @@ -31,10 +31,10 @@ impl BuildResources { global: firmware._render_global.clone(), metrics: firmware._pm_metrics.clone(), metrics_ids: firmware.queues.metrics.clone(), - metrics_bases: [ - crate::hw::t8140::CONFIG.pm_metrics.va, - firmware.init.metrics_fw_va()?, - ], + metrics_bases: crate::g17::fw::buffer::MetricsAddresses { + firmware: firmware.init.metrics_fw_va()?, + client: crate::hw::t8140::CONFIG.pm_metrics.va, + }, config: firmware.queues.render.config, }) } diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index 178d275da9cf4d..e5ac662a4d1aa9 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -//! Sticky VM errors and their optional userspace mirror. No firmware state is accessed here. +//! VM admission errors and client failure reporting. No firmware state is accessed here. use crate::gem; use core::cell::UnsafeCell; @@ -10,9 +10,11 @@ use kernel::prelude::*; const MIRROR_MAX_SIZE: usize = 64 * 1024; /// A VM and its accepted jobs share this status, including after queue destruction. -/// The first error is permanent. Callers record it before signalling the job's fence. +/// Client failure is separate from hardware admission: recovery may preserve the +/// VM while failed accepted work must still be reported before signalling fences. pub(crate) struct VmStatus { error: AtomicI32, + reported_error: AtomicI32, mirror: AtomicPtr, mirror_claimed: AtomicBool, mirror_owner: UnsafeCell>, @@ -30,6 +32,7 @@ impl VmStatus { pub(crate) fn new() -> Self { Self { error: AtomicI32::new(0), + reported_error: AtomicI32::new(0), mirror: AtomicPtr::new(core::ptr::null_mut()), mirror_claimed: AtomicBool::new(false), mirror_owner: UnsafeCell::new(None), @@ -40,9 +43,26 @@ impl VmStatus { self.error.load(Ordering::Acquire) } - /// Store the first error in both the status and any registered mirror. + /// Record permanent admission failure and report it to the client. pub(crate) fn record(&self, error: Error) { - let error = match self.error.compare_exchange( + self.report_failure(error); + let _ = self.error.compare_exchange( + 0, + error.to_errno(), + Ordering::SeqCst, + Ordering::SeqCst, + ); + } + + pub(crate) fn reported_error(&self) -> i32 { + self.reported_error.load(Ordering::Acquire) + } + + /// Report failed accepted work without preventing recovery of the VM. + /// Every reporter publishes the first error before returning: a losing + /// reporter may signal its fence before the winning reporter resumes. + pub(crate) fn report_failure(&self, error: Error) { + let error = match self.reported_error.compare_exchange( 0, error.to_errno(), Ordering::SeqCst, @@ -88,11 +108,11 @@ impl VmStatus { // SAFETY: The retained owner keeps this checked, aligned word mapped. unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); - // The pointer publication and error update form a store/load handshake: - // either this load sees record's error or record sees our pointer. Stores + // The pointer publication and reported error form a store/load handshake: + // either this load sees the error or the reporter sees our pointer. Stores // after publishing the pointer write only the same first error, never zero. self.mirror.store(word, Ordering::SeqCst); - let error = self.error.load(Ordering::SeqCst); + let error = self.reported_error.load(Ordering::SeqCst); if error != 0 { // SAFETY: The mapping remains owned by this status as above. unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index d116610bae3bc9..639b9f54f46ac3 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -11,10 +11,13 @@ //! coprocessors start, and enables dynamic power gating before the firmware is initialized. use crate::{hw, identity}; +use crate::hw; +use core::mem::MaybeUninit; use kernel::{ - c_str, + bindings, c_str, device::Core, devres::Devres, + error::{from_err_ptr, to_result}, io::{ mem::IoMem, // poll::read_poll_timeout, @@ -22,7 +25,6 @@ use kernel::{ }, platform, prelude::*, - str::CStr, sync::{aref::ARef, Arc}, // sync::aref::ARef, // time::Delta, @@ -57,9 +59,6 @@ const CORE_MASKS_G14X: usize = 0xe01500; const FAULT_INFO_G14X: usize = 0xd8c0; const FAULT_ADDR_G14X: usize = 0xd8c8; -/// Cleared after `CPU_RUN` when stopping a G17 coprocessor. -const CPU_STOP_ACK: u32 = 0x1 << 5; // BIT(5) - /// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) @@ -251,35 +250,62 @@ pub(crate) struct G17Id { pub(crate) perf_map: [u32; 2], } -/// CPU control registers of one GPU coprocessor. +/// Provider-owned lifecycle of one G17 firmware coprocessor. +/// +/// The mailbox provider owns the wrapper mappings and runtime-PM references. +/// The consumer device link keeps it alive until this GPU has stopped using it. pub(crate) struct CpuControl { - asc: Pin>>>, + dev: ARef, + mbox: *mut bindings::apple_mbox, } +// SAFETY: The live provider serializes CPU transitions with its lifecycle mutex; +// the device link orders provider removal after this consumer's teardown. +unsafe impl Send for CpuControl {} +// SAFETY: All accesses use the same provider-serialized lifecycle API. +unsafe impl Sync for CpuControl {} + impl CpuControl { - /// Map the CPU control registers in the `name` register region. - pub(crate) fn new(pdev: &platform::Device, name: &CStr) -> Result { - let req = pdev.io_request_by_name(name).ok_or(EINVAL)?; - Ok(CpuControl { - asc: KBox::pin_init(req.iomap_sized::(), GFP_KERNEL)?, - }) + /// Resolve the mailbox phandle and validate its firmware role before use. + pub(crate) fn new(pdev: &platform::Device, index: usize) -> Result { + let expected = match index { + 0 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, + 1 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, + _ => return Err(EINVAL), + }; + // SAFETY: pdev is live; the provider establishes the consumer device link. + let mbox = unsafe { + from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? + }; + let mut lifecycle = MaybeUninit::uninit(); + // SAFETY: mbox is live, and lifecycle is writable storage for the C result. + to_result(unsafe { + bindings::apple_mbox_ascwrap_v6_get_lifecycle(mbox, lifecycle.as_mut_ptr()) + })?; + // SAFETY: A successful provider query initialized every field. + if unsafe { lifecycle.assume_init() }.role != expected { + return Err(EINVAL); + } + // SAFETY: This only checks the provider's matching start/stop capability. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; + Ok(Self { dev: pdev.into(), mbox }) } - /// Start the coprocessor CPU. + /// Start through the owner of the CPU registers and runtime-PM reference. pub(crate) fn start(&self) -> Result { - let asc = self.asc.try_access().ok_or(ENODEV)?; - let val = asc.read32(CPU_CONTROL); - asc.write32(val | CPU_RUN, CPU_CONTROL); - Ok(()) + // SAFETY: The constructor established the live, role-checked provider. + to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) } - /// Stop the coprocessor CPU. This only gates it: its firmware keeps its state. + /// Stop before releasing the transport or any firmware-visible memory. pub(crate) fn stop(&self) { - if let Some(asc) = self.asc.try_access() { - let val = asc.read32(CPU_CONTROL); - asc.write32(val & !CPU_RUN, CPU_CONTROL); - let val = asc.read32(CPU_CONTROL); - asc.write32(val & !CPU_STOP_ACK, CPU_CONTROL); + // SAFETY: The provider remains live during consumer teardown. Removal + // stops the CPU before setting the fence that can return ENODEV here. + let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); + if let Err(error) = result { + if error != ENODEV { + dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); + } } } } From d9e431ffcc7f1d075e14053088c0cdc4c89bacc3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 00:26:31 +1100 Subject: [PATCH 065/631] drm/asahi: report live GEM object accounting Nothing in the driver says how much memory its GEM objects hold. fdinfo has no drm-* keys, and debugfs only lists flink names. A G17P GEM object is pinned once it is mapped into a GPU VM and stays pinned until its last reference drops, and drm_gem_get_pages() marks the shmem mapping unevictable. So when /proc/meminfo Unevictable grows, nothing shows whether that is live client memory, kernel firmware objects or references the driver kept after the client went away. Count every object from creation until it is freed, split by user and kernel ownership, and expose the totals read-only as /sys/module/asahi/parameters/gem_stats: user_objects user_bytes user_created user_freed kernel_objects kernel_bytes kernel_created kernel_freed The counters are relaxed atomics, updated in the DriverObject constructor and in a new PinnedDrop of the per-object data, which records the object size. If user_bytes stays flat across a client-exit loop, clients are not leaking. If it keeps growing while the per-process mappings do not, the driver is holding objects for clients that have gone. Assisted-by: LLM Signed-off-by: Ryan Murray Adapted to the rewrite GEM constructors and pinned drop. The counter balances construction and destruction; its sysfs output is verified on local Neo. Upstream-commit: 3361051608523c5c31f5dba92f48ff4258126f2d (cherry picked from commit 66af386568f748d2499be3e3587ce1e7ca0ae9cd) --- drivers/gpu/drm/asahi/asahi.rs | 1 + drivers/gpu/drm/asahi/gem.rs | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index f7ab5fde52b770..00839de24a9cfb 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -110,6 +110,7 @@ mod agx_resources; mod agx_status; mod agx_uapi; mod gem; +mod gem_stats; mod g17; mod gpu; mod hw; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 5f7bb3fc35f4b6..c183d52562b85f 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -40,10 +40,12 @@ use crate::{ const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; /// Represents the inner data of a GEM object for this driver. -#[pin_data] +#[pin_data(PinnedDrop)] pub(crate) struct AsahiObject { /// ID for debug id: u64, + /// Object size retained for live-object accounting. + size: usize, /// Object creation flags. flags: u32, /// Whether this object can be exported. @@ -163,6 +165,13 @@ impl ObjectRef { } } +#[pinned_drop] +impl PinnedDrop for AsahiObject { + fn drop(self: Pin<&mut Self>) { + crate::gem_stats::note_free(self.kernel, self.size); + } +} + pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, @@ -270,6 +279,12 @@ impl DriverObject for AsahiObject { mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, + // All remaining fields are infallible. PinnedDrop balances the + // accounting once this initializer completes. + size: { + crate::gem_stats::note_new(args.kernel, size); + size + }, flags: args.flags, exportable: args.exportable, kernel: args.kernel, From 56717b1dd771b8bbe4951aca0aeee2ce6c9e6228 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 09:40:35 +1100 Subject: [PATCH 066/631] drm/asahi: port j700 queue accounting and compute recovery isolation Keep sixteen physical QIDs available for render while allowing existing compute channels to continue unchanged. Account for queue ownership and exhaustion at structural transitions, without adding full-table scans to normal retirement. Classify bounded compute watchdog deferrals using older unfinished work from another VM; do not blame a potential victim VM when that grace period expires. Adapted from Ryan Murray's upstream commits: dc66a43fdbd8 (firmware queue-slot accounting) dd27c0e2de7b (render QID reserve and exhausted-pool diagnostics) ccd546c5cb23 (compute watchdog isolation) The rewrite already reclaims drained quarantined render pairs after a fresh retirement witness; f055aaf00ec2 requires no default toggle here. Validation: matching module build and stack audit; production allocator, accounting and timeout-policy tests; local Neo desktop and the full light/heavy/compute depth 1/2/4/8/16 matrix; four concurrent clients for 60 seconds with sampled readback checks. Fault-injected recovery remains unqualified by these healthy-path hardware tests. Signed-off-by: Ryan Murray (cherry picked from commit f7fcc50568b9467f241a425b20b92ff1de10f641) --- drivers/gpu/drm/asahi/g17/context.rs | 2 +- drivers/gpu/drm/asahi/g17/job.rs | 15 +++-- drivers/gpu/drm/asahi/g17/kick.rs | 65 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/mod.rs | 2 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 41 ++++++++++++ drivers/gpu/drm/asahi/g17/runtime.rs | 3 + drivers/gpu/drm/asahi/g17/runtime/backend.rs | 12 +++- .../drm/asahi/g17/runtime/compute_control.rs | 40 ++++++++++++ .../drm/asahi/g17/runtime/render/settle.rs | 3 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 2 + drivers/gpu/drm/asahi/g17/timeout.rs | 60 +++++++++++++++++ 11 files changed, 233 insertions(+), 12 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/timeout.rs diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index c3a3c6e87a2f60..1a28386d788d0d 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(ENOSPC); + return Err(crate::g17::queue_stats::enospc(crate::g17::queue_stats::Pool::QueueContextQos)); } *used |= 1u128 << id; Ok(QosLease { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index c05b5119790fc1..c80955527cc069 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -248,7 +248,7 @@ pub(crate) trait Backend: Send + Sync { fn timed_out(&self, packet: Arc) -> sched::Status; fn cancel(&self, packet: Arc); /// Transfer a timeout to an outstanding off-lock graph transaction, if any. - fn defer_timeout(&self, packet: &Arc) -> bool; + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; /// Return true only after taking publication ownership and recording its result. fn try_early(&self, packet: &Arc) -> bool; fn count_backlog(&self, packet: &Packet); @@ -256,6 +256,7 @@ pub(crate) trait Backend: Send + Sync { struct Job { backend: Arc, + victim_renewals: u8, packet: Arc, } @@ -272,9 +273,14 @@ impl Drop for Job { impl sched::JobImpl for Job { fn false_timeout(job: &mut sched::Job) -> bool { - job.is_finished() - || job.packet.completion.take_replay_timeout() - || job.backend.defer_timeout(&job.packet) + if job.is_finished() || job.packet.completion.take_replay_timeout() { + return true; + } + let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); + if deferred && job.packet.engine() == Engine::Compute { + job.victim_renewals += 1; + } + deferred } fn prepare(job: &mut sched::Job) -> Option { @@ -337,6 +343,7 @@ impl Scheduler { 1, Job { backend: self.backend.clone(), + victim_renewals: 0, packet: packet.clone(), }, )?; diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f36ab9798996a7..aa2c054a9bbc7b 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,7 +6,7 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; use kernel::prelude::*; use super::{ @@ -15,7 +15,30 @@ use super::{ }; use crate::{hw::t8140, mem, mmu}; -const QID_COUNT: usize = QID_MAX as usize + 1; +pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; + +pub(super) const COMPUTE_RENDER_RESERVE: usize = 16; +pub(super) static LEASED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; +pub(super) static PUBLISHED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; +pub(super) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); +pub(super) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); +pub(super) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); + +pub(super) fn leased_total() -> u32 { + LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() +} + +fn exhausted(engine: DataMaster, reserve: bool) -> Error { + if reserve { + RESERVE_REFUSED.fetch_add(1, Ordering::Relaxed); + } else { + EXHAUSTED.fetch_add(1, Ordering::Relaxed); + } + super::queue_stats::enospc(match engine { + DataMaster::Compute => super::queue_stats::Pool::QidCompute, + _ => super::queue_stats::Pool::QidRenderPair, + }) +} /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -44,12 +67,16 @@ struct Identity { /// One shared namespace for tiling, fragment and compute queues. pub(crate) struct QueueIds { entries: [Option; QID_COUNT], + compute_publication: u64, + leased: usize, } impl QueueIds { pub(crate) fn new() -> impl Init { kernel::try_init!(Self { entries <- pin_init::init_array_from_fn(|_| None), + compute_publication: 0, + leased: 0, }) } @@ -76,11 +103,16 @@ impl QueueIds { Err(EINVAL) }; } + // Healthy existing channels keep their QID. Never recycle a published + // identity to recover capacity after a fault; leave eight render pairs. + if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { + return Err(exhausted(engine, true)); + } let index = preferred .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; + .ok_or_else(|| exhausted(engine, false))?; let id = Id { qid: index as u8, owner, @@ -90,9 +122,18 @@ impl QueueIds { id, published: false, }); + self.leased += 1; + LEASED[engine as usize].fetch_add(1, Ordering::Relaxed); + HIGH_WATER.fetch_max(leased_total(), Ordering::Relaxed); Ok(id) } + /// Ordered under the device mutex; no atomic operation on the submission path. + pub(crate) fn next_compute_publication(&mut self) -> Result { + self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; + Ok(self.compute_publication) + } + /// Call after preflighting the complete installation, before any address /// becomes firmware-visible. A failed or partial installation retains it. pub(crate) fn publish(&mut self, id: Id) -> Result { @@ -104,7 +145,10 @@ impl QueueIds { if entry.id != id { return Err(EINVAL); } - entry.published = true; + if !entry.published { + entry.published = true; + PUBLISHED[id.engine as usize].fetch_add(1, Ordering::Relaxed); + } Ok(()) } @@ -113,6 +157,8 @@ impl QueueIds { match slot { Some(entry) if entry.id == id && !entry.published => { *slot = None; + self.leased -= 1; + LEASED[id.engine as usize].fetch_sub(1, Ordering::Relaxed); Ok(()) } _ => Err(EINVAL), @@ -120,6 +166,17 @@ impl QueueIds { } } +impl Drop for QueueIds { + fn drop(&mut self) { + for entry in self.entries.iter().flatten() { + LEASED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); + if entry.published { + PUBLISHED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); + } + } + } +} + // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index f379c1b69d2569..a5c867409d3799 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -43,6 +43,8 @@ mod recovery; mod runtime; pub(crate) mod status; mod teardown; +mod timeout; +mod queue_stats; mod validation; use core::any::Any; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2a30cd54cecd1b..e0cdc050443621 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -308,6 +308,7 @@ impl Fifo { struct Active { packet: Arc, + published: u64, work: Arc, ticket: Option, observed: Option, @@ -363,6 +364,7 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. pub(crate) struct Queue { + slot_accounting: crate::g17::queue_stats::ComputeAccount, owner: Option, binding: Binding, previous: Option, @@ -412,6 +414,7 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + slot_accounting: crate::g17::queue_stats::ComputeAccount::new(), binding, previous: None, kick, @@ -446,6 +449,11 @@ impl Queue { pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } + fn update_slot_accounting(&mut self) { + self.slot_accounting.update(self.owner.is_some(), self.released, + self.quarantined, self.retired_by_teardown); + } + pub(crate) fn owner(&self) -> Option { self.owner } @@ -462,6 +470,22 @@ impl Queue { pub(crate) fn room(&self) -> bool { !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT } + /// Quarantines without a pending witness no longer describe live pipe order. + pub(crate) fn timeout_head(&self) -> Option { + if self.quarantined && !self.awaiting_witness { + return None; + } + self.active.front().map(|active| crate::g17::timeout::Head { + vm: Arc::as_ptr(active.packet.completion.status()) as usize, + published: active.published, + finished: active.observed.is_some(), + }) + } + + pub(crate) fn can_classify_timeout(&self, packet: &Arc) -> bool { + !self.quarantined && self.owns_packet(packet) + } + pub(crate) fn oldest_spared(&self) -> Option { self.active .front() @@ -567,10 +591,12 @@ impl Queue { ) -> Result>> { if !self.quarantined { self.quarantined = true; + self.update_slot_accounting(); self.quarantine_error = Some(error); self.failure_status_pending = true; self.retirement_proved = false; self.retired_by_teardown = false; + self.update_slot_accounting(); let all_spared = self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); self.spared_quarantine = all_spared && error == ENODATA; @@ -676,6 +702,7 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; + self.update_slot_accounting(); } } Ok(None) | Err(EAGAIN) => { @@ -782,6 +809,7 @@ impl Queue { self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; + self.update_slot_accounting(); self.retire_pending = false; self.retirement_ready = true; if self @@ -793,6 +821,7 @@ impl Queue { } if self.released { self.owner = None; + self.update_slot_accounting(); } } Ok(()) @@ -932,9 +961,12 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); + self.update_slot_accounting(); self.released = false; + self.update_slot_accounting(); self.retirement_ready = false; self.retired_by_teardown = false; + self.update_slot_accounting(); Ok(()) } /// True allows the registry to cancel the never-published QID and drop this graph. An @@ -944,11 +976,14 @@ impl Queue { return Ok(true); } self.released = true; + self.update_slot_accounting(); if self.active.len == 0 && !self.quarantined { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; + self.update_slot_accounting(); self.retired_by_teardown = true; + self.update_slot_accounting(); return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; @@ -961,6 +996,7 @@ impl Queue { } if self.retirement_ready { self.owner = None; + self.update_slot_accounting(); } } Ok(false) @@ -1027,7 +1063,9 @@ impl Queue { return Ok(true); } self.released = true; + self.update_slot_accounting(); self.retired_by_teardown = true; + self.update_slot_accounting(); if let Some(active) = self .active .iter_mut() @@ -1159,6 +1197,7 @@ impl Graph { /// Borrowed device operations used during one publication. Implementations borrow the initdata, /// coprocessor and accounting fields separately from the Registry that owns physical queues. pub(crate) trait Host { + fn next_compute_publication(&mut self) -> Result; fn epoch(&self) -> Result<(u64, u32)>; fn prepare_compute_shared(&mut self) -> Result; fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; @@ -1313,9 +1352,11 @@ impl Queue { event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, }; + let published = host.next_compute_publication()?; self.active .push(Active { packet: packet.clone(), + published, work: work.clone(), ticket: None, observed: None, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index d100fef631630e..294d66b91f8dc1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -546,6 +546,9 @@ struct ComputeHost<'a> { generation: u64, } impl compute::Host for ComputeHost<'_> { + fn next_compute_publication(&mut self) -> Result { + self.qids.next_compute_publication() + } fn epoch(&self) -> Result<(u64, u32)> { Ok(( self.generation, diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 085655beb8194b..3fb40779021e54 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -138,6 +138,12 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; + if result.as_ref().is_err_and(|error| *error == ENOSPC) { + crate::g17::queue_stats::note_submit_enospc(match packet.engine() { + Engine::Compute => "compute-publish", + Engine::Render => "render-publish", + }); + } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } @@ -169,9 +175,9 @@ impl job::Backend for Backend { } } - fn defer_timeout(&self, packet: &Arc) -> bool { - if packet.engine() != Engine::Render { - return false; + fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { + if packet.engine() == Engine::Compute { + return self.defer_compute_timeout(packet, renewals); } let mut state = self.shared.state.lock(); (*state) diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs index 610643ef6e379b..dac4ca48a34aec 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_control.rs @@ -7,6 +7,46 @@ use crate::g17::job::{self, Packet}; use kernel::{prelude::*, sync::Arc}; impl Backend { + /// A final retirement pass precedes bounded cross-VM timeout attribution. + /// All completion callbacks run after releasing the device mutex. + pub(super) fn defer_compute_timeout(&self, packet: &Arc, renewals: u8) -> bool { + use crate::g17::{recovery::Class, timeout::{self, Decision}}; + let Ok(mut deferred) = DeferredBatch::worker() else { + return false; + }; + let result = (|| -> Result { + let mut state = self.shared.state.lock(); + let Some(firmware) = (*state).as_deref_mut() else { + return Ok(Decision::Own); + }; + firmware.scan_compute(None, &mut deferred)?; + let Some(queue) = firmware.queues.compute(self.owner) + .filter(|queue| queue.can_classify_timeout(packet)) else { + return Ok(Decision::Own); + }; + let Some(own) = queue.timeout_head() else { + return Ok(Decision::Own); + }; + let others = firmware.queues.compute.iter().flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|other| other.owner() != Some(self.owner)) + .filter_map(|other| other.timeout_head()); + let decision = timeout::classify(own, renewals, others); + if decision == Decision::Spare { + firmware.queues.compute(self.owner).ok_or(EIO)? + .classify(&|_| Class::Spared); + } + if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { + dev_warn!(self.shared.dev.as_ref(), + "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", + self.owner, own.published, renewals, decision); + } + Ok(decision) + })(); + deferred.finish(); + matches!(result, Ok(Decision::Defer)) + } + /// The caller owns cancellation of a packet that never claimed publication. /// A missing retained packet may already have a healthy callback waiting off-lock. pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index a6d5821561d244..2cf7a5d77ee5a7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -164,6 +164,7 @@ impl crate::g17::Firmware { return Ok(()); } let mut serviced = false; + let mut draining = 0; for slot in 1..self.queues.render.entries.len() { if !self.queues.render.entries[slot] .as_ref() @@ -172,6 +173,7 @@ impl crate::g17::Firmware { { continue; } + draining += 1; if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { dev_warn!( self.primary.state.shared.dev.as_ref(), @@ -181,6 +183,7 @@ impl crate::g17::Firmware { ); } } + crate::g17::queue_stats::note_draining_pairs(draining); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 10f334e4fe5617..06465629eb2be1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -165,6 +165,7 @@ impl Firmware { /// The event worker has already reclaimed drained pairs and reannounced /// pending controls. Recovery blocks new releases, never an observed ack. pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { + crate::g17::queue_stats::note_teardowns(self.queues.teardown.len()); self.queues.render.collect_reclaimable(reclaim)?; if self.queues.teardown.is_empty() { return Ok(()); @@ -194,6 +195,7 @@ impl Firmware { .is_some() {} teardown.report_stalled(super::now_ns(), &mut service); + crate::g17::queue_stats::note_teardowns(teardown.len()); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/timeout.rs b/drivers/gpu/drm/asahi/g17/timeout.rs new file mode 100644 index 00000000000000..e76dab07f56db7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/timeout.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Bounded isolation for a compute watchdog blocked behind another VM's older kick. +//! Publication order is evidence of a possible blocker, not proof of a firmware +//! dependency. Renewals are bounded; the oldest unfinished queue is never excused. + +pub(crate) const MAX_RENEWALS: u8 = 8; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Head { + pub(crate) vm: usize, + pub(crate) published: u64, + pub(crate) finished: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Decision { + Own, + Defer, + Spare, +} + +pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { + if own.finished || !others.into_iter().any(|other| { + other.vm != own.vm && !other.finished && other.published < own.published + }) { + Decision::Own + } else if renewals < MAX_RENEWALS { + Decision::Defer + } else { + Decision::Spare + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn head(vm: usize, published: u64, finished: bool) -> Head { + Head { vm, published, finished } + } + #[test] + fn earlier_other_vm_gets_bounded_grace() { + let own = head(5, 101, false); + for count in 0..MAX_RENEWALS { + assert_eq!(classify(own, count, [head(7, 100, false)].into_iter()), Decision::Defer); + } + assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); + assert_eq!(classify(own, 2, [].into_iter()), Decision::Own); + } + #[test] + fn oldest_unfinished_queue_is_never_excused() { + assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); + } + #[test] + fn same_vm_finished_and_later_work_do_not_block() { + let own = head(5, 101, false); + assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); + assert_eq!(classify(head(5, 101, true), 0, [head(7, 100, false)].into_iter()), Decision::Own); + } +} From eec34d1fcea3d9d78ebb850cf8500a2bf47c04a4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:55:07 -0400 Subject: [PATCH 067/631] drm: carry shared GPUVM and scheduler interfaces used by the G17 review Carry the GPUVM array reservation, allocator interval query and scheduler lifetime interfaces from the current-kernel integration branch. Preserve the zero-fence GPUVM locking contract and initialize zero-valid C storage without importing unexported bindgen Default methods into loadable modules. This isolates the four DRM prerequisite files from the wider j700 merge; the public series retains its own platform and RTKit base. The following review fixes are applied against these shared interfaces. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 982df2dcc66e2cb2012df476c4325c8fce016dda) --- drivers/gpu/drm/drm_gpuvm.c | 18 +++- rust/kernel/drm/gpuvm.rs | 121 ++++++++++++++++++++++- rust/kernel/drm/sched.rs | 189 +++++++++++++++++++++--------------- 3 files changed, 243 insertions(+), 85 deletions(-) diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 91f7861e36b84d..b123f6164206b1 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -1396,9 +1396,23 @@ fn_lock_array(struct drm_gpuvm_exec *vm_exec) struct drm_gem_object **objs; unsigned int num_objs; } *args = vm_exec->extra.priv; + unsigned int i; + int ret; + + /* + * Keep the zero-fence contract of every other drm_gpuvm_exec helper. + * drm_exec_prepare_array() unconditionally calls + * dma_resv_reserve_fences(), where zero is invalid; exec_prepare_obj() + * deliberately selects drm_exec_lock_obj() for that case instead. + */ + for (i = 0; i < args->num_objs; i++) { + ret = exec_prepare_obj(&vm_exec->exec, args->objs[i], + vm_exec->num_fences); + if (ret) + return ret; + } - return drm_exec_prepare_array(&vm_exec->exec, args->objs, - args->num_objs, vm_exec->num_fences); + return 0; } /** diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index fbfab4bc7cf31a..3802f3e3cb92fd 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -36,7 +36,7 @@ use crate::{ use core::cell::UnsafeCell; use core::marker::{PhantomData, PhantomPinned}; -use core::mem::ManuallyDrop; +use core::mem::{ManuallyDrop, MaybeUninit}; use core::ops::{Deref, DerefMut, Range}; use core::ptr::NonNull; use pin_init; @@ -402,7 +402,10 @@ pub(super) unsafe extern "C" fn vm_bo_alloc_callback() -> *mut b { let obj: Result>>> = KBox::try_pin_init( try_pin_init!(GpuVmBo:: { - bo <- pin_init::default(), + // The bindgen `Default` implementation is exactly a zero fill, but its + // out-of-line symbol is not exported to loadable modules. `drm_gpuvm_bo` + // is `Zeroable`, so initialize it directly without a cross-crate call. + bo <- pin_init::zeroed::(), inner <- T::GpuVmBo::new(), _p: PhantomPinned }), @@ -594,13 +597,20 @@ impl GpuVm { } else { 0 }, - exec: Default::default(), + // SAFETY: bindgen's `Default` for this C structure is exactly + // `MaybeUninit::zeroed().assume_init()`. Spell that out here so + // external modules do not import its unexported trait method. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, extra: match (is_ext, obj) { (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { fn_: Some(exec_lock_gem_object), priv_: obj.as_raw() as *const _ as *mut _, }, - _ => Default::default(), + // SAFETY: as above, this bindgen C structure is zero-valid. + _ => unsafe { + MaybeUninit::::zeroed() + .assume_init() + }, }, num_fences: 0, }), @@ -615,6 +625,59 @@ impl GpuVm { Ok(ManuallyDrop::into_inner(guard)) } + /// Lock the GPUVM and an array of additional GEM objects in one + /// wound/wait transaction. + pub fn exec_lock_array<'a, 'b>( + &'a self, + objects: &'b [ARef>], + interruptible: bool, + ) -> Result> { + let count = u32::try_from(objects.len()).map_err(|_| EINVAL)?; + let mut raw_objects = KVec::with_capacity(objects.len(), GFP_KERNEL)?; + for object in objects { + raw_objects.push(object.as_raw(), GFP_KERNEL)?; + } + + let mut guard = ManuallyDrop::new(LockedGpuVm { + gpuvm: self, + // vm_exec needs to be pinned, so stick it in a Box. + vm_exec: KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: (if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, + // SAFETY: both bindgen C structures are zero-valid; their generated + // `Default` methods perform the same zero initialization but are not + // exported from the kernel crate to loadable modules. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, + extra: unsafe { + MaybeUninit::::zeroed() + .assume_init() + }, + num_fences: 0, + }), + GFP_KERNEL, + )?, + objects: LockedObjects::Array(objects), + }); + + // SAFETY: Every raw pointer is backed by an ARef in `objects`, which + // outlives the returned guard. The helper consumes the temporary + // pointer array before returning and keeps its own object references. + to_result(unsafe { + bindings::drm_gpuvm_exec_lock_array( + &mut *guard.vm_exec, + raw_objects.as_mut_ptr(), + count, + ) + })?; + + Ok(ManuallyDrop::into_inner(guard)) + } + /// Returns true if the given object is external to the GPUVM /// (that is, if it does not share the DMA reservation object of the GPUVM). pub fn is_extobj(&self, obj: &impl IntoGEMObject) -> bool { @@ -745,7 +808,55 @@ impl LockedGpuVm<'_, '_, T> { req_gem_range: u32, flags: GpuVaFlags, ) -> Result { - let obj = self.obj.ok_or(EINVAL)?; + let obj = self.objects.single().ok_or(EINVAL)?.as_raw(); + self.sm_map_raw( + obj, + ctx, + req_addr, + req_range, + req_offset, + req_gem_range, + flags, + ) + } + + /// Map with an object acquired by [`GpuVm::exec_lock_array`]. + pub fn sm_map_indexed( + &mut self, + object_index: usize, + ctx: &mut T::StepContext, + req_addr: u64, + req_range: u64, + req_offset: u64, + req_gem_range: u32, + flags: GpuVaFlags, + ) -> Result { + let obj = self + .objects + .indexed(object_index) + .ok_or(EINVAL)? + .as_raw(); + self.sm_map_raw( + obj, + ctx, + req_addr, + req_range, + req_offset, + req_gem_range, + flags, + ) + } + + fn sm_map_raw( + &mut self, + obj: *mut bindings::drm_gem_object, + ctx: &mut T::StepContext, + req_addr: u64, + req_range: u64, + req_offset: u64, + req_gem_range: u32, + flags: GpuVaFlags, + ) -> Result { let mut ctx = StepContext { ctx, gpuvm: self.gpuvm, diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index a6d197152e68fc..be95793cd9afbc 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -11,7 +11,6 @@ use crate::{ prelude::*, sync::{Arc, UniqueArc}, time::{self, msecs_to_jiffies}, - types::Opaque, }; use core::marker::PhantomData; use core::mem::MaybeUninit; @@ -45,12 +44,20 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { - /// Reconsider a timeout before the backend starts recovery. Returning true - /// reinserts this job and rearms its timeout without changing its fences. - /// The callback must not stop or restart the scheduler. - fn false_timeout(_job: &mut Job) -> bool { - false - } + /// Handle a scoped timeout without resetting the C scheduler. + /// + /// The backend must settle/quarantine its exact hardware work, or retain + /// that owner until an in-progress host transaction can classify it. It + /// must not invoke drm_sched_stop/start or replay scheduler jobs. The core + /// reinserts the detached guilty job through NO_HANG, preserving every + /// parent callback and the backend's actual success/error fence result. + const MANAGED_TIMEOUT_RECOVERY: bool = false; + + /// Reconsider a timeout before stopping the scheduler or settling fences. + /// Returning true asks the core to reinsert the retained job and rearm. + /// The backend must not stop/start the scheduler from this hook. + fn false_timeout(_job: &mut Job) -> bool { false } + /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. @@ -65,9 +72,6 @@ pub trait JobImpl: Sized { /// Called when a job has taken too long to execute, to trigger GPU recovery. /// - /// A driver that handles timeouts without a scheduler reset returns [`Status::NoHang`]. - /// [`Job::is_finished`] tells whether the job completed after the timeout fired. - /// /// This method is called in a workqueue context. fn timed_out(job: &mut Job) -> Status; @@ -110,13 +114,29 @@ unsafe extern "C" fn timedout_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: The timeout worker retains this detached Job until this - // callback returns, including when its parent fence signals concurrently. + // The C timeout worker owns this job outside pending_list until return. + // Free work cannot release it even if its parent signals concurrently. + if T::MANAGED_TIMEOUT_RECOVERY { + let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; + if unsafe { bindings::dma_fence_get_status(finished) } != 0 { + return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; + } + } + // SAFETY: the C core retains the detached job across both callbacks. if T::false_timeout(unsafe { &mut *p }) { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } - // SAFETY: The same retained job remains valid after the first callback. - T::timed_out(unsafe { &mut *p }) as bindings::drm_gpu_sched_stat + let status = T::timed_out(unsafe { &mut *p }); + if T::MANAGED_TIMEOUT_RECOVERY { + // This is scoped fence handling, not C reset recovery. stop would + // detach and discard healthy parents; start would cancel those jobs. + // NO_HANG reinserts the exact job and queues free work if it completed + // during the callback. It does not change the parent's error or replay + // work. A deferred exact owner remains eligible for its real callback. + bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG + } else { + status as bindings::drm_gpu_sched_stat + } } unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { @@ -131,58 +151,59 @@ unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: A job on the pending list has a valid scheduler fence. + + // fini has stopped/synchronized submission and timeout workers. A parent + // callback can still run on the device completion thread. The scheduler + // fence retains the parent reference while we remove its callback under + // the parent's lock; that also waits for an already-running callback. let s_fence = unsafe { (*sched_job).s_fence }; - // SAFETY: `s_fence` is valid and holds a reference to its parent, if any. let parent = unsafe { (*s_fence).parent }; - // SAFETY: `s_fence` is valid. - let finished = unsafe { addr_of_mut!((*s_fence).finished) }; - - // drm_sched_fini() has stopped the scheduler workers, but the hardware fence can still - // signal concurrently. Fence callbacks run under the fence lock, so removing ours either - // disarms it or waits until it has completed the job. - let callback_removed = !parent.is_null() - // SAFETY: run_job() armed `cb` on `parent` before the submission worker was stopped. - && unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) }; - - // A job without a parent either completed when run_job() returned, or had its parent - // detached by drm_sched_stop(), which also returned its credits. - let complete_here = callback_removed - // SAFETY: `finished` is a valid, initialized fence. - || (parent.is_null() && unsafe { bindings::dma_fence_get_status(finished) } == 0); - - // SAFETY: All of our jobs are Job. + let removed_parent_callback = if !parent.is_null() { + // SAFETY: run_job installed (or attempted to install on an already + // signaled parent) this callback before fini synchronized submit work. + unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) } + } else { false }; + let finish_here = removed_parent_callback || (parent.is_null() + && unsafe { bindings::dma_fence_get_status(addr_of_mut!((*s_fence).finished)) == 0 }); + // Null/error run_job calls job_done before submit work returns, so its + // finished fence is already signaled. NULL+unfinished can instead survive + // a legacy backend's prior reset stop: C stop dropped the parent and its + // credits, but not its score. Do not subtract those credits twice. + + // SAFETY: cancellation cannot race a scheduler callback using this job + // after the removal/parent-lock synchronization above. The backend must + // retain any still-hardware-owned resources independently of Job storage. T::cancel(unsafe { &mut *p }); - if complete_here { - // Account for the job exactly as drm_sched_job_done() would have. - // - // SAFETY: The job belongs to the scheduler being torn down, which is still valid. + if finish_here { + // Match drm_sched_job_done's accounting exactly once, only when its + // callback did not do it. Submission/free work is stopped, so fini + // itself unlinks and frees this job; no workqueue rearm is appropriate. + let sched = unsafe { (*sched_job).sched }; unsafe { - let sched = (*s_fence).sched; - if callback_removed { - bindings::atomic_sub( - (*sched_job).credits as i32, - addr_of_mut!((*sched).credit_count), - ); + if removed_parent_callback { + bindings::atomic_sub((*sched_job).credits as i32, + addr_of_mut!((*sched).credit_count)); } - bindings::atomic_dec((*sched).score); + bindings::atomic_sub(1, (*sched).score); } - - // SAFETY: `finished` is a valid fence. - let fence = unsafe { Fence::get_raw(finished) }; + let fence = unsafe { Fence::get_raw(addr_of_mut!((*s_fence).finished)) }; fence.set_error(ECANCELED); - fence.signal(); + let _ = fence.signal(); } + // Do not clear/put parent here: s_fence still owns that reference and its + // normal release path drops it. A completed callback's result is retained. } /// A DRM scheduler job. pub struct Job { job: bindings::drm_sched_job, inner: T, - // Destroying an entity can leave its queued jobs to deferred work that still dereferences - // `job->sched` after the scheduler has been finalized. Keep the scheduler allocation alive - // for as long as any job references it; this does not delay finalization. + // Entity destruction can leave kill_jobs_work queued on the system + // workqueue or waiting for a dependency. That work still dereferences + // job->sched->ops after drm_sched_fini has stopped the scheduler workers. + // Retain only the allocation: retaining SchedulerOwner here would keep + // fini from running while pending jobs still need its cancellation. _scheduler: Arc>, } @@ -304,8 +325,8 @@ struct EntityInner { impl Drop for EntityInner { fn drop(&mut self) { - // SAFETY: The EntityInner is initialized. This will cancel/free all jobs, possibly from - // deferred work. Each job keeps the scheduler allocation alive until it is freed. + // SAFETY: The EntityInner is initialized. Jobs removed from the entity + // may still await kill_jobs_work; each retains the scheduler allocation. unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; } } @@ -324,7 +345,7 @@ impl Entity { let mut entity: KBox>> = KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - let mut sched_ptr = sched.0.allocation.sched.get(); + let mut sched_ptr = &sched.0.allocation.sched as *const _ as *mut _; // SAFETY: The Box is allocated above and valid. unsafe { @@ -402,9 +423,13 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; - // SAFETY: The Box pointer is valid, and this initializes the scheduler reference. + // SAFETY: The job allocation is valid and this initializes its last + // field. Arc::clone is infallible and the entity still owns the live + // scheduler, so every successfully initialized job retains its C + // callback target before it can be armed or submitted. unsafe { - addr_of_mut!((*job.as_mut_ptr())._scheduler).write(self.0.sched.allocation.clone()) + addr_of_mut!((*job.as_mut_ptr())._scheduler) + .write(self.0.as_ref().get_ref().sched.allocation.clone()) }; // SAFETY: All fields of the Job are now initialized. @@ -412,33 +437,36 @@ impl Entity { } } -/// DRM scheduler allocation, kept alive by the scheduler owner and by every job. +/// DRM scheduler allocation, retained by live owners and deferred job callbacks. pub struct SchedulerInner { - sched: Opaque, + sched: bindings::drm_gpu_scheduler, _p: PhantomData, } -// SAFETY: TODO -unsafe impl Sync for SchedulerInner {} -// SAFETY: TODO -unsafe impl Send for SchedulerInner {} - -/// Owner of an initialized DRM scheduler, shared by the scheduler handle and its entities. -/// -/// Jobs only reference the allocation, so that pending jobs cannot keep the scheduler from -/// being finalized. +// Only public scheduler/entity handles keep this owner alive. Jobs retain +// SchedulerInner instead, so outstanding work cannot prevent finalization. struct SchedulerOwner { allocation: Arc>, } impl Drop for SchedulerOwner { fn drop(&mut self) { - // SAFETY: The scheduler is valid. drm_sched_fini() stops the scheduler workers, then - // cancels and frees all remaining jobs through `cancel_job_cb()`. - unsafe { bindings::drm_sched_fini(self.allocation.sched.get()) }; + // SAFETY: fini stops submission, synchronizes the timeout worker and + // cancels/frees remaining jobs through our synchronized cancel callback. + // The C contract explicitly prohibits drm_sched_stop before fini. + // Deferred entity-kill callbacks are on a different workqueue and may + // survive this call. Their job guards retain allocation through cleanup. + unsafe { + bindings::drm_sched_fini(addr_of!(self.allocation.sched).cast_mut()) + }; } } +// SAFETY: TODO +unsafe impl Sync for SchedulerInner {} +// SAFETY: TODO +unsafe impl Send for SchedulerInner {} + /// A DRM Scheduler pub struct Scheduler(Arc>); @@ -464,16 +492,15 @@ impl Scheduler { ) -> Result> { let mut sched: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; - // Allocate the owner up front: nothing may fail once the scheduler is initialized. - let owner: UniqueArc>> = UniqueArc::new_uninit(GFP_KERNEL)?; - - // SAFETY: `sched` was just allocated and is valid for writes. - let sched_ptr = Opaque::cast_into(unsafe { addr_of!((*sched.as_mut_ptr()).sched) }); + // Allocate the owner before starting C workers: no fallible allocation + // may abandon an initialized scheduler without running drm_sched_fini. + let owner: UniqueArc>> = + UniqueArc::new_uninit(GFP_KERNEL)?; // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation // TODO: allocate sched zzeroed instead unsafe { - (*sched_ptr).sched_rq = core::ptr::null_mut(); + (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); }; let init_ops = bindings::drm_sched_init_args { @@ -491,9 +518,15 @@ impl Scheduler { // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. // `device` is valid by its type invarants - to_result(unsafe { bindings::drm_sched_init(sched_ptr, addr_of!(init_ops)) })?; + to_result(unsafe { + bindings::drm_sched_init( + addr_of_mut!((*sched.as_mut_ptr()).sched), + addr_of!(init_ops), + ) + })?; - // SAFETY: All fields of SchedulerInner are now initialized. + // SAFETY: C initialized sched and PhantomData has no stored state. + // No fallible operation follows successful initialization. let allocation = unsafe { sched.assume_init() }.into(); Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) } From a46d8b3c985aef8f6321e33d42af79256fce6537 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 22:55:27 -0400 Subject: [PATCH 068/631] drm/asahi: correct reviewed lifetime, failure and submission paths Fix G17 descriptor fields, scratch state, prefix failure history, recovery evidence, retirement wakeups, queue admission and output-fence publication. Validate all timestamp destinations before enqueue and defer their cleanup onto a preallocated module-owned workqueue so fence producers do not take buffer reservations while destroying timestamp mappings. Correct shared GPUVM ownership and lock ordering, fixed-map rollback, page-table child unlinking, repeated-page metadata and legacy TLBI/bind validation. Balance legacy power/kick bookkeeping and propagate RTKit endpoint-start errors. Retain module code through DRM files, PRIME exports and detached scheduler callbacks, disable scheduler work at finalization, release the G17 shutdown DRM backedge and destroy initialized device data. The source-hashed integration snapshot passed all 64 local host validation commands and the shared Rust, C DRM/RTKit and Asahi module build. These checks use explicit host doubles and original-failure controls; they are not hardware, lockdep, module-unload or performance qualification. The public-base RTKit change is limited to the same STARTEP error propagation. Remaining timestamp publication ordering, deferred cleanup, legacy reference cycles and firmware-contract questions are tracked for follow-up. No queue depth reduction or serialization workaround is introduced. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit ca389fba9b6c7c9354c2f765fe81c5ef5f069f3b) --- drivers/gpu/drm/asahi/asahi.rs | 28 +- drivers/gpu/drm/asahi/cleanup.rs | 118 ++++++++ drivers/gpu/drm/asahi/driver.rs | 1 + drivers/gpu/drm/asahi/file.rs | 14 +- drivers/gpu/drm/asahi/g17/channel.rs | 12 +- drivers/gpu/drm/asahi/g17/command.rs | 8 +- drivers/gpu/drm/asahi/g17/completion.rs | 78 +++++- drivers/gpu/drm/asahi/g17/context.rs | 5 + drivers/gpu/drm/asahi/g17/fence.rs | 24 +- drivers/gpu/drm/asahi/g17/fw/compute.rs | 75 ++++- drivers/gpu/drm/asahi/g17/fw/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 29 +- drivers/gpu/drm/asahi/g17/initdata.rs | 38 ++- drivers/gpu/drm/asahi/g17/job.rs | 5 +- drivers/gpu/drm/asahi/g17/mod.rs | 43 ++- drivers/gpu/drm/asahi/g17/power.rs | 22 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 28 +- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 92 +++---- drivers/gpu/drm/asahi/g17/queue/mod.rs | 125 +++++---- drivers/gpu/drm/asahi/g17/queue/render.rs | 3 +- .../gpu/drm/asahi/g17/queue/render/memory.rs | 6 +- drivers/gpu/drm/asahi/g17/recovery.rs | 83 +++--- .../gpu/drm/asahi/g17/recovery/attribution.rs | 17 +- drivers/gpu/drm/asahi/g17/runtime.rs | 67 ++++- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 53 ++-- .../drm/asahi/g17/runtime/compute_replay.rs | 22 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 16 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 28 +- .../gpu/drm/asahi/g17/runtime/render/grow.rs | 3 +- .../drm/asahi/g17/runtime/render/prepare.rs | 23 +- .../drm/asahi/g17/runtime/render/submit.rs | 7 + drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 7 +- drivers/gpu/drm/asahi/g17/teardown.rs | 31 ++- drivers/gpu/drm/asahi/gem.rs | 11 + drivers/gpu/drm/asahi/mmu.rs | 230 ++++++++++------ drivers/gpu/drm/asahi/mmu/bind.rs | 33 ++- drivers/gpu/drm/asahi/mmu/lifetime.rs | 47 +--- drivers/gpu/drm/asahi/pgtable.rs | 79 +++++- drivers/gpu/drm/asahi/queue/mod.rs | 15 +- drivers/gpu/drm/drm_gpuvm.c | 8 +- include/drm/drm_gpuvm.h | 1 + rust/kernel/drm/device.rs | 2 +- rust/kernel/drm/driver.rs | 4 + rust/kernel/drm/gem/mod.rs | 9 +- rust/kernel/drm/gpuvm.rs | 260 ++++++++++++++---- rust/kernel/drm/sched.rs | 8 + 46 files changed, 1334 insertions(+), 492 deletions(-) create mode 100644 drivers/gpu/drm/asahi/cleanup.rs diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi.rs index 00839de24a9cfb..beb85ace60ada9 100644 --- a/drivers/gpu/drm/asahi/asahi.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -6,6 +6,7 @@ mod alloc; mod buffer; mod channel; +mod cleanup; #[cfg(CONFIG_DEV_COREDUMP)] mod crashdump; mod debug; @@ -132,8 +133,31 @@ mod util; mod vm; mod workqueue; -kernel::module_platform_driver! { - type: driver::AsahiDriver, +use kernel::prelude::*; + +#[pin_data] +struct DriverModule { + // Field drop order unregisters/joins driver producers before queue teardown. + #[pin] + _driver: kernel::driver::Registration>, + _cleanup: cleanup::QueueOwner, +} + +impl kernel::InPlaceModule for DriverModule { + fn init(module: &'static kernel::ThisModule) -> impl PinInit { + try_pin_init!(Self { + // Initialize the queue first, before registration can invoke probe. + _cleanup: cleanup::QueueOwner::new()?, + _driver <- kernel::driver::Registration::new( + ::NAME, + module, + ), + }) + } +} + +module! { + type: DriverModule, name: "asahi", description: "AGX GPU driver for Apple silicon SoCs", license: "Dual MIT/GPL", diff --git a/drivers/gpu/drm/asahi/cleanup.rs b/drivers/gpu/drm/asahi/cleanup.rs new file mode 100644 index 00000000000000..e3dd5ad95dc0ce --- /dev/null +++ b/drivers/gpu/drm/asahi/cleanup.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Reservation-taking timestamp destruction must not block fence producers. +//! +//! The module owns this queue outside its platform registration. Files and +//! exported buffers retain the module; detached scheduler cleanup retains it +//! until its driver callback returns. Unregister joins the firmware/event work. +//! Only after those producer boundaries close may the module destroy this queue. + +use core::{ + ops::Deref, + ptr::{self, NonNull}, + sync::atomic::{AtomicPtr, Ordering}, +}; +use kernel::{ + bindings, c_str, impl_has_work, new_work, + prelude::*, + workqueue::{Queue, Work, WorkItem}, +}; + +static QUEUE: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Destroyed by module exit after platform-driver unregister, never by a worker. +pub(crate) struct QueueOwner(NonNull); + +// SAFETY: QueueOwner only transfers ownership of a thread-safe kernel queue. +unsafe impl Send for QueueOwner {} +// SAFETY: The queue API synchronizes all accesses; destruction requires ownership. +unsafe impl Sync for QueueOwner {} + +impl QueueOwner { + pub(crate) fn new() -> Result { + // SAFETY: The name is NUL terminated and contains no format specifiers. + let raw = unsafe { + bindings::alloc_workqueue_noprof( + c_str!("asahi-ts-cleanup").as_char_ptr(), + bindings::wq_flags_WQ_UNBOUND | bindings::wq_flags_WQ_MEM_RECLAIM, + 0, + ) + }; + let raw = NonNull::new(raw).ok_or(ENOMEM)?; + if QUEUE + .compare_exchange(ptr::null_mut(), raw.as_ptr(), Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + // SAFETY: This fresh queue has never been published or used. + unsafe { bindings::destroy_workqueue(raw.as_ptr()) }; + return Err(EBUSY); + } + Ok(Self(raw)) + } +} + +impl Drop for QueueOwner { + fn drop(&mut self) { + // No external producers remain: module registration was dropped first. + // Pending cleanup owns no queue reference and cannot enqueue more work. + // SAFETY: The module's unique owner retains the queue through its drain. + unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; + QUEUE.store(ptr::null_mut(), Ordering::Release); + } +} + +#[pin_data] +struct Item { + value: T, + #[pin] + work: Work, +} + +impl_has_work! { + impl{T: Send + 'static} HasWork> for Item { self.work } +} + +impl WorkItem for Item { + type Pointer = Pin>; + + fn run(this: Self::Pointer) { + drop(this); + } +} + +/// Allocate the destruction work before publication; final handoff cannot fail. +pub(crate) struct Deferred(Option>>>); + +impl Deferred { + pub(crate) fn new(value: T) -> Result { + let item = KBox::pin_init( + pin_init!(Item { + value, + work <- new_work!("Asahi timestamp cleanup"), + }), + GFP_KERNEL, + )?; + Ok(Self(Some(item))) + } +} + +impl Deref for Deferred { + type Target = T; + + fn deref(&self) -> &T { + &self.0.as_ref().unwrap().value + } +} + +impl Drop for Deferred { + fn drop(&mut self) { + if let Some(item) = self.0.take() { + let raw = QUEUE.load(Ordering::Acquire); + assert!(!raw.is_null()); + // SAFETY: Module ownership and producer joins retain the queue until + // this handoff has returned. The unique pinned box is never queued + // twice; enqueue transfers it without allocation or a failure path. + unsafe { Queue::from_raw(raw) }.enqueue(item); + } + } +} diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index b35da14c5b2020..b4e7bb2a7267a0 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -133,6 +133,7 @@ impl drm::driver::Driver for AsahiDriver { type Object = drm::gem::shmem::Object; const INFO: drm::driver::DriverInfo = INFO; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEATURES: u32 = drm::driver::FEAT_GEM | drm::driver::FEAT_RENDER | drm::driver::FEAT_SYNCOBJ diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 2a3baff1d0774c..cc42f5f3c108b3 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -199,11 +199,11 @@ impl SyncItem { in_count: u32, out_count: u32, ) -> Result> { - let count = in_count + out_count; - let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; + let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; const STRIDE: usize = core::mem::size_of::(); - let size = STRIDE * count as usize; + let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; + let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; // SAFETY: We only read this once, so there are no TOCTOU issues. let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); @@ -894,6 +894,9 @@ impl File { data: &uapi::drm_asahi_gem_bind_op, file: &DrmFile, ) -> Result { + if data.range == 0 { + return Err(EINVAL); + } if (data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND) != 0 { Self::do_gem_unbind(vm_id, data, file) } else { @@ -1211,7 +1214,10 @@ impl File { .inner() .objects() .lock() - .insert_limit(1..=u32::MAX, obj, GFP_KERNEL)? as u64; + .insert_limit(1..=u32::MAX, obj, GFP_KERNEL); + // StoreError owns the rejected object. Convert it only after the + // temporary XArray guard is gone: dropping a mapping may sleep. + let handle = handle? as u64; data.object_handle = handle as u32; Ok(0) diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index a90693e824b0e0..8e4740b76a1661 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -9,9 +9,7 @@ //! empty slot and orders the record, power tally and producer together. use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; - -use crate::driver::AsahiDevice; +use kernel::{device::Device, prelude::*}; use super::fw::{channels::*, initdata::status}; use super::initdata::InitData; @@ -57,15 +55,15 @@ impl Rings { } } - pub(crate) fn drain_trace(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { self.drain_report(dev, init, role, ReportRing::Trace); } - pub(crate) fn drain_statistics(&mut self, dev: &AsahiDevice, init: &InitData, role: Role) { + pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role) { self.drain_report(dev, init, role, ReportRing::Statistics); } - fn drain_report(&mut self, dev: &AsahiDevice, init: &InitData, role: Role, ring: ReportRing) { + fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { let quarantined = &mut self.quarantined[role as usize][ring as usize]; if *quarantined { return; @@ -176,7 +174,7 @@ impl Events { } } - pub(crate) fn next(&mut self, dev: &AsahiDevice, init: &InitData) -> Result> { + pub(crate) fn next(&mut self, dev: &Device, init: &InitData) -> Result> { let role = self.role; let state = status::EVENT_STATE as usize; let consumer_word = init.status_word(role, state)?; diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 47f9c0048e673a..5929c14b264fd5 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -3,6 +3,7 @@ //! Decode a submission and validate its user resources before publishing any work. use super::fw::{ + compute::ScratchRequest, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, }; @@ -279,6 +280,7 @@ pub(crate) enum Validated { timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + scratch: ScratchRequest, usc_base: u64, cdm_va: u64, cdm_end_va: u64, @@ -395,7 +397,11 @@ impl Payload { | (u64::from(cmd.bg_partial_rsrc_spec_hi) << 32), }, eot_offset: cmd.eot.usc, + eot_rsrc_spec: u64::from(cmd.eot.rsrc_spec) + | (u64::from(cmd.bg_eot_rsrc_spec_hi) << 32), partial_eot_offset: cmd.partial_eot.usc, + partial_eot_rsrc_spec: u64::from(cmd.partial_eot.rsrc_spec) + | (u64::from(cmd.bg_eot_partial_rsrc_spec_hi) << 32), rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; Ok(Validated::Render { @@ -406,7 +412,6 @@ impl Payload { } Self::Compute(cmd, attachments) => { if cmd.flags != 0 - || cmd.scs_layout != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base @@ -423,6 +428,7 @@ impl Payload { sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; Ok(Validated::Compute { + scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, cdm_end_va: cmd.cdm_ctrl_stream_end - 4, diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 777d54222c0eaa..f137a7e3434de3 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -24,8 +24,9 @@ use kernel::{ }; struct Destination { - mapping: Arc, - offset: usize, + // Keep the existing GPU alias alive along with its prepared CPU view. + _mapping: Arc, + cpu: mmu::TimestampMapping, } impl Destination { @@ -46,7 +47,8 @@ impl Destination { if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { return Err(ERANGE); } - Ok(Some(Self { mapping, offset })) + let cpu = mapping.timestamp_mapping(offset)?; + Ok(Some(Self { _mapping: mapping, cpu })) } fn write(&self, ticks: u64) -> Result { @@ -56,12 +58,45 @@ impl Destination { .wrapping_add( ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, ); - self.mapping.write_timestamp(self.offset, ns)?; + self.cpu.write(ns); barrier(Ordering::Release); Ok(()) } } +/// Timestamp mappings resolved and retained before any command is enqueued. +pub(crate) struct Destinations(Option; 4]>>); + +impl Destinations { + pub(crate) fn resolve( + objects: Pin<&xarray::XArray>>, + timestamps: &[uapi::drm_asahi_timestamps], + ) -> Result { + if timestamps.is_empty() || timestamps.len() > 2 { + return Err(EINVAL); + } + let mut destinations = core::array::from_fn(|_| None); + for (index, timestamp) in timestamps.iter().enumerate() { + destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; + destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; + } + // Empty submissions need no cleanup allocation. Timestamped commands + // prepare the unique work item before any output fence is published. + let retained = if destinations.iter().any(Option::is_some) { + Some(crate::cleanup::Deferred::new(destinations)?) + } else { + None + }; + Ok(Self(retained)) + } + + fn iter(&self) -> core::slice::Iter<'_, Option> { + let destinations: &[Option] = + self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); + destinations.iter() + } +} + /// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. @@ -87,7 +122,7 @@ pub(crate) struct Completion { status: Arc, fence: UserFence, member: Member, - destinations: [Option; 4], + destinations: Destinations, compute: Option, feed: Option>, #[pin] @@ -95,6 +130,8 @@ pub(crate) struct Completion { #[pin] render: Mutex>, signalled: AtomicBool, + retirement_queued: AtomicBool, + accepted: AtomicBool, timeout_error: AtomicI32, replays: AtomicU8, replay_timeout: AtomicBool, @@ -109,19 +146,10 @@ impl Completion { context: &Arc, member: Member, vm_job: mmu::VmJobGuard, - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_timestamps], + destinations: Destinations, compute: bool, feed: Option>, ) -> Result> { - if timestamps.len() != if compute { 1 } else { 2 } { - return Err(EINVAL); - } - let mut destinations = core::array::from_fn(|_| None); - for (index, timestamp) in timestamps.iter().enumerate() { - destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; - destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; - } let fence = fence::independent(contexts)?; let compute = if compute { Some(ComputeTimestamps::new(dev, context.vm())?) @@ -139,6 +167,8 @@ impl Completion { vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), render <- new_mutex!(None, "G17 command render lease"), signalled: AtomicBool::new(false), + retirement_queued: AtomicBool::new(false), + accepted: AtomicBool::new(false), timeout_error: AtomicI32::new(0), replays: AtomicU8::new(0), replay_timeout: AtomicBool::new(false), @@ -152,6 +182,11 @@ impl Completion { pub(crate) fn fence(&self) -> Fence { Fence::from_fence(&self.fence) } + /// All fallible scheduler setup has succeeded. Failures from this point + /// must be visible through VM_STATUS before the accepted job's fence. + pub(crate) fn mark_accepted(&self) { + self.accepted.store(true, Ordering::Release); + } pub(crate) fn status(&self) -> &Arc { &self.status } @@ -229,6 +264,16 @@ impl Completion { Ok(()) } + /// A retirement witness has been accepted by an off-lock completion batch. + /// This is separate from fence signalling and from a failure still owned by firmware. + pub(crate) fn note_retirement_queued(&self) { + self.retirement_queued.store(true, Ordering::Release); + } + + pub(crate) fn retirement_queued(&self) -> bool { + self.retirement_queued.load(Ordering::Acquire) + } + /// The first timeout request retains its errno while the physical graph is /// borrowed. Duplicate watchdog or cancellation callbacks cannot replace it. pub(crate) fn defer_timeout(&self, error: Error) { @@ -342,6 +387,9 @@ impl Completion { return; } self.status.record_if_device_loss(error); + if self.accepted.load(Ordering::Acquire) { + self.status.report_failure(error); + } self.fence.set_error(error); self.release_healthy(); self.fence.signal(); diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 1a28386d788d0d..ee2d8133ea8cdf 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -213,6 +213,11 @@ impl Context { pub(crate) fn scheduler_owner(&self) -> Arc { self.scheduler.clone() } + /// Scheduler release and command retirement have both been witnessed. + /// Keep installed storage alive while returning the finite TTBAT identity. + pub(crate) fn release_execution(&self) { + self.execution.release(); + } pub(crate) fn scheduler_generation(&self) -> u32 { self.scheduler_generation } diff --git a/drivers/gpu/drm/asahi/g17/fence.rs b/drivers/gpu/drm/asahi/g17/fence.rs index 1217d7e7b3f4e1..3414e36db9d365 100644 --- a/drivers/gpu/drm/asahi/g17/fence.rs +++ b/drivers/gpu/drm/asahi/g17/fence.rs @@ -39,6 +39,15 @@ pub(crate) fn independent(contexts: &FenceContexts) -> Result Result { + let fence = independent(contexts)?; + fence.set_error(ENOMEM); + fence.signal(); + Ok(Fence::from_fence(&fence)) +} + /// All command members of an ioctl share its work-state node and aggregate fence. /// The initial member keeps the aggregate unsignalled until enqueue finishes. #[pin_data] @@ -165,14 +174,25 @@ pub(crate) struct Outputs { impl Outputs { pub(crate) fn new( contexts: &FenceContexts, - syncs: KVec, + mut syncs: KVec, input_count: usize, + allocation_failure: &Fence, ) -> Result { if input_count > syncs.len() { return Err(EINVAL); } + let fallback = match independent(contexts) { + Ok(fence) => fence, + Err(error) => { + // Syncobjs and timeline-chain storage are already resolved. + // No work has been accepted; publish the pre-signaled ENOMEM + // fence without another allocation before returning the error. + install(syncs.drain(input_count..), allocation_failure); + return Err(error); + } + }; Ok(Self { - fallback: independent(contexts)?, + fallback, syncs, input_count, submission: None, diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 22f8259508ff65..5b7489cfe93f4b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -24,6 +24,62 @@ pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; const COMPUTE_REGISTERS: usize = 40; +const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; + +/// Validated per-command shader-context scratch request. +#[derive(Clone, Copy)] +pub(crate) struct ScratchRequest { + pages: u16, +} + +impl ScratchRequest { + const FIXED_BYTES: u64 = 0x58000; + const THREADS_PER_CLIQUE: u64 = 32; + const BYTES_PER_STACK_ENTRY: u64 = 8; + const PAGE_SHIFT: u32 = 12; + const PAGE_SIZE: u64 = 1 << Self::PAGE_SHIFT; + const PAGES_PER_MIB: u64 = 1 << (20 - Self::PAGE_SHIFT); + const WINDOW_ENABLED: u64 = 1 << 24; + + /// Payload-free layouts request no scratch even if depth/cliques are set. + /// Otherwise round the fixed and per-thread sections and add a guard page. + pub(crate) fn new(layout: u64) -> Result { + let depth = layout & u16::MAX as u64; + let cliques = (layout >> 24) & u8::MAX as u64; + let user = (layout >> 32) & u16::MAX as u64; + let driver = (layout >> 48) & u16::MAX as u64; + if user == 0 && driver == 0 { + return Ok(Self { pages: 0 }); + } + let threads = cliques.checked_mul(Self::THREADS_PER_CLIQUE).ok_or(EINVAL)?; + let per_thread = depth.checked_mul(Self::BYTES_PER_STACK_ENTRY) + .and_then(|bytes| bytes.checked_add(user)) + .and_then(|bytes| bytes.checked_add(driver)).ok_or(EINVAL)?; + let bytes = threads.checked_mul(per_thread) + .and_then(|bytes| bytes.checked_add(Self::FIXED_BYTES)) + .and_then(|bytes| bytes.checked_add(2 * Self::PAGE_SIZE - 1)) + .ok_or(EINVAL)?; + let pages = bytes >> Self::PAGE_SHIFT; + Ok(Self { pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + } + + pub(crate) fn enabled(self) -> bool { + self.pages != 0 + } + + fn size_word(self) -> u64 { + let pages = u64::from(self.pages); + pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) + } + + fn window_word(self) -> u64 { + if self.enabled() { Self::WINDOW_ENABLED } else { UNK_10791 } + } + + fn checkpoints(self) -> [u16; 4] { + CHECKPOINTS.map(|count| count + u16::from(self.enabled())) + } +} const ALT_REGISTERS: usize = 4; /// Object IDs of every compute kick. @@ -54,6 +110,8 @@ const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; /// Inputs of one compute descriptor. pub(crate) struct ComputeArgs { + /// Scratch request validated with the userspace command. + pub(crate) scratch: ScratchRequest, /// Base of the queue's USC window. pub(crate) usc_base: u64, /// GPU address of the control stream. @@ -103,13 +161,14 @@ pub(crate) struct ComputeArgs { pub(crate) timestamp_va: u64, } -fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS]> { +fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { let preempt = args.preempt_va; let operand = args.operand_state_va; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); Ok([ + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x1a510, preempt), RegisterWrite::new(0x1a420, args.cdm_va), RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), @@ -130,10 +189,10 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x1a061, 0), RegisterWrite::new(0x1a0b9, 0), RegisterWrite::new(0x1a0c1, 0), - RegisterWrite::new(0x101d1, 0), + RegisterWrite::new(0x101d1, args.scratch.size_word()), RegisterWrite::new(0x0d479, 0), RegisterWrite::new(0x1a0e9, UNK_1A0E9), - RegisterWrite::new(0x107a1, UNK_10791), + RegisterWrite::new(0x107a1, args.scratch.window_word()), RegisterWrite::new(0x0a599, UNK_0A599), RegisterWrite::new(0x0d411, UNK_0D411), RegisterWrite::new(0x1a540, ids), @@ -287,8 +346,11 @@ impl ComputeDescriptor { self.tag = CommandTag::Compute as u32; self.context_id = args.context_id.into(); self.work_state_va = args.work_state_va; - self.checkpoints = CHECKPOINTS; - self.registers.set(registers_va, ®isters)?; + self.checkpoints = args.scratch.checkpoints(); + // The enabled program prepends one register; disabled descriptors keep + // the same forty writes, trailer count and resume checkpoints. + let first = usize::from(!args.scratch.enabled()); + self.registers.set(registers_va, ®isters[first..])?; self.alt_registers.set(alt_registers_va, &alt_registers)?; self.preempt_va = args.preempt_va; self.cdm_end_va = args.cdm_end_va; @@ -321,10 +383,11 @@ impl ComputeDescriptor { /// the descriptor's address as seen by the work's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scratch: ScratchRequest, ) -> Result<[Option; 4]> { let registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, - CHECKPOINTS[0] as u8, + scratch.checkpoints()[0] as u8, )?; let alt_registers = RegisterArrayBinding::new( offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index c8b7d3f0dce3a7..2e6133e882b2d1 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -369,6 +369,8 @@ pub(crate) struct KickArgs<'a> { pub(crate) event_mask: [u64; 4], /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], + /// The compute register program includes a shader-context scratch request. + pub(crate) compute_scratch: bool, } /// One kick ring entry. @@ -379,7 +381,7 @@ pub(crate) struct KickArgs<'a> { #[derive(Copy, Clone, Debug)] pub(crate) struct KickEntry { /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit - /// 58 and queue priority in bits 60:59. + /// 58, queue priority in bits 60:59 and compute scratch enabled in bit 61. pub(crate) header: u64, /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. pub(crate) mcache: u64, @@ -419,6 +421,7 @@ impl KickEntry { const QID_SHIFT: u32 = 42; const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; + const COMPUTE_SCRATCH: u64 = 1 << 61; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -440,6 +443,9 @@ impl KickEntry { let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + if args.compute_scratch { + header |= Self::COMPUTE_SCRATCH; + } let mcache = match args.mcache { Some(table) => { header |= Self::MCACHE_PRESENT; diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 349d03ed14d719..e3da1b0e6d954f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -206,11 +206,14 @@ pub(crate) struct RenderPass { pub(crate) bg: UscProgram, /// Partial-render background program. pub(crate) partial_bg: UscProgram, - /// Tagged USC offset of the end-of-tile program. Its resource specifier - /// is always written as zero. + /// Tagged USC offset of the end-of-tile program. pub(crate) eot_offset: u32, + /// Resource specifier of the end-of-tile program. + pub(crate) eot_rsrc_spec: u64, /// Tagged USC offset of the partial-render end-of-tile program. pub(crate) partial_eot_offset: u32, + /// Resource specifier of the partial-render end-of-tile program. + pub(crate) partial_eot_rsrc_spec: u64, /// The resource specifiers have their high 32 bits. pub(crate) rsrc_spec_hi: bool, } @@ -383,7 +386,9 @@ struct PassWords { partial_bg_rsrc_spec: u64, partial_bg_va: u64, eot_va: u64, + eot_rsrc_spec: u64, partial_eot_va: u64, + partial_eot_rsrc_spec: u64, } impl PassWords { @@ -446,7 +451,9 @@ impl PassWords { partial_bg_rsrc_spec: rsrc_spec(pass.partial_bg.rsrc_spec), partial_bg_va: usc(pass.partial_bg.offset)?, eot_va: usc(pass.eot_offset)?, + eot_rsrc_spec: rsrc_spec(pass.eot_rsrc_spec), partial_eot_va: usc(pass.partial_eot_offset)?, + partial_eot_rsrc_spec: rsrc_spec(pass.partial_eot_rsrc_spec), }) } } @@ -580,7 +587,7 @@ fn fragment_registers( RegisterWrite::new(0x01739, 1), RegisterWrite::new(0x10009, w.utile_config), // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.eot_rsrc_spec), RegisterWrite::new(0x15381, w.eot_va), RegisterWrite::new(0x15369, w.bg_rsrc_spec), RegisterWrite::new(0x15371, w.bg_va), @@ -676,7 +683,7 @@ fn partial_store_registers( ) -> [RegisterWrite; PARTIAL_STORE_REGISTERS] { [ // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), RegisterWrite::new(0x15381, w.partial_eot_va), RegisterWrite::new(0x10039, w.tile_config), RegisterWrite::new(0x15359, UNK_15359_PARTIAL), @@ -701,7 +708,7 @@ fn partial_resume_registers( ) -> [RegisterWrite; PARTIAL_RESUME_REGISTERS] { [ // End-of-tile resource specifier. - RegisterWrite::new(0x15379, 0), + RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), RegisterWrite::new(0x15381, w.partial_eot_va), RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), RegisterWrite::new(0x15371, w.partial_bg_va), @@ -1036,12 +1043,14 @@ pub(crate) struct FragmentDescriptor { pub(crate) unk_1f50: u64, /// Tile configuration. pub(crate) tile_config: u64, - /// Ends with the end-of-tile resource specifier, which is zero. - pub(crate) unk_1f60: [u8; 0x1c], + pub(crate) unk_1f60: [u8; 0x18], + /// Low word of the end-of-tile resource specifier. + pub(crate) eot_rsrc_spec: u32, /// GPU address of the end-of-tile program. pub(crate) eot_va: u64, - /// Ends with the partial end-of-tile resource specifier, which is zero. - pub(crate) unk_1f84: [u8; 0x18], + pub(crate) unk_1f84: [u8; 0x14], + /// Low word of the partial-render end-of-tile resource specifier. + pub(crate) partial_eot_rsrc_spec: u32, /// GPU address of the partial-render end-of-tile program. pub(crate) partial_eot_va: u64, pub(crate) unk_1fa4: u32, @@ -1228,7 +1237,9 @@ impl FragmentSlot { desc.height = pass.height; desc.unk_1f50 = UNK_15049; desc.tile_config = words.tile_config; + desc.eot_rsrc_spec = words.eot_rsrc_spec as u32; desc.eot_va = words.eot_va; + desc.partial_eot_rsrc_spec = words.partial_eot_rsrc_spec as u32; desc.partial_eot_va = words.partial_eot_va; desc.depth_clear = pass.depth_clear; desc.unk_1fac = words.tib_blocks << Self::UNK_1FAC_TIB_SHIFT | Self::UNK_1FAC_BASE; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 22b67575c133cb..aea894ba2c2d82 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -413,6 +413,15 @@ impl InitData { Ok(()) } + /// The sole host publisher holds the device mutex through the following + /// inner and outer publication. Firmware consumers can only free capacity; + /// neither this check nor waiting for room reserves or limits other queues. + pub(super) fn work_ready(&self, engine: super::fw::queue::DataMaster) -> Result { + let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; + Ok(next != consumer && next != consumer_b) + } + fn work_publication<'a>( &'a self, record: &'a super::fw::channels::WorkSlot, @@ -1231,14 +1240,14 @@ impl super::recovery::Memory for InitData { .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? .load(Ordering::Relaxed); let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot).unwrap_or(0); + let key = super::recovery::slot_key(slot)?; let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .map(|words| words[0]).unwrap_or(0); - (key, monitor) + .ok().map(|words| words[0])?; + Some((key, monitor)) }); sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .map(|words| words[0]).unwrap_or(0); + .ok().map(|words| words[0]); Ok(sources) } @@ -1290,30 +1299,17 @@ impl super::recovery::Memory for InitData { Ok(()) } - fn entries( - &self, - out: &mut [super::recovery::Entry; super::recovery::MAX_ENTRIES], - ) -> Result { + fn entries(&self) -> Result { let cluster = self.object(self.cluster)?; let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; fence(Ordering::Acquire); - let mut used = 0; + let mut entries = super::recovery::Entries::default(); for index in 0..256 { - if used == out.len() { - break; - } let offset = base + index * core::mem::size_of::(); let flags = cluster.word(offset)?.load(Ordering::Relaxed); - if flags & 1 == 0 { - continue; - } - let stamp = cluster.word(offset + 4)?.load(Ordering::Relaxed); - if let Some(entry) = super::recovery::Entry::decode(flags, stamp) { - out[used] = entry; - used += 1; - } + entries.include(flags); } - Ok(used) + Ok(entries) } fn wait_tick(&self) { diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index c80955527cc069..b25db8e00f1672 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -13,7 +13,7 @@ use kernel::{ sync::Arc, }; -const TIMEOUT_MS: kernel::time::Msecs = 2000; +pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; const RENDER_CREDITS: u32 = 32; const COMPUTE_CREDITS: u32 = 16; @@ -272,6 +272,8 @@ impl Drop for Job { } impl sched::JobImpl for Job { + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + fn false_timeout(job: &mut sched::Job) -> bool { if job.is_finished() || job.packet.completion.take_replay_timeout() { return true; @@ -351,6 +353,7 @@ impl Scheduler { job.add_dependency(dependency)?; } let mut job = job.arm(); + packet.completion.mark_accepted(); let ready = if host_timestamps { completed.clone() } else { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index a5c867409d3799..6f75dfb5822c2b 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -56,7 +56,7 @@ use core::sync::atomic::{ use kernel::{ c_str, - device::Core, + device::{self, Core}, drm::gem::shmem, impl_has_delayed_work, impl_has_work, iosys_map::IoSysMapRef, @@ -64,6 +64,7 @@ use kernel::{ prelude::*, soc::apple::rtkit, sync::{ + aref::ARef, Arc, CondVar, Mutex, // @@ -466,6 +467,7 @@ struct Firmware { rings: channel::Rings, recovery: recovery::State, effort: power::Effort, + registration_power_release_pending: bool, queues: runtime::Registry, pool: Arc, render_ids: Arc, @@ -569,6 +571,12 @@ impl power::Control for Firmware { fn wait_tick(&self) { fsleep(Delta::from_millis(1)); } + fn registration_release_failed(&mut self) { + self.registration_power_release_pending = true; + } + fn registration_release_succeeded(&mut self) { + self.registration_power_release_pending = false; + } } /// Event work item ID. @@ -583,7 +591,12 @@ const FEED_WORK: u64 = 5; /// Device state shared with the coprocessor callbacks and the event worker. #[pin_data] struct Shared { - dev: AsahiDevRef, + /// Diagnostics may outlive shutdown without retaining the DRM data graph. + dev: ARef, + /// Allocation users take independent references before reserving resources. + /// Shutdown removes this owning backedge after joining device work. + #[pin] + drm: Mutex>, clusters: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, @@ -604,7 +617,11 @@ struct Shared { /// Nests inside the device mutex; detached mappings are dropped after both unlock. #[pin] reclaim: Mutex, - /// The device lock. `None` before boot completes and after teardown. + /// Serializes detached firmware destruction with unbind. Stop paths only; + /// normal publication and RTKit/fence handlers never acquire this lock. + #[pin] + stop: Mutex<()>, + /// The device lock. `None` before boot completes and after detachment. #[pin] state: Mutex>>, #[pin] @@ -687,6 +704,17 @@ impl WorkItem for Shared { } impl Shared { + fn drm(&self) -> Result { + self.drm.lock().as_ref().cloned().ok_or(ENODEV) + } + + fn release_drm(&self) { + // The final put may call device-release callbacks. Never run those with + // the reference-slot mutex held. + let retired = self.drm.lock().take(); + drop(retired); + } + /// Queues the event worker. Callable from any context. fn queue_events(self: &Arc) { // Already queued if this fails, which is just as good. @@ -849,7 +877,8 @@ impl Gpu { let feed = feed::Feed::new()?; let shared = Arc::pin_init( pin_init!(Shared { - dev: dev.into(), + drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), + dev: dev.as_ref().into(), clusters: id.num_clusters, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), @@ -862,6 +891,7 @@ impl Gpu { grow <- new_work!("g17::Shared::grow"), reclaim_work <- new_work!("g17::Shared::reclaim_work"), reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), + stop <- new_mutex!((), "g17::Shared::stop"), state <- new_mutex!(None, "g17::Shared::state"), changed <- new_condvar!("g17::Shared::changed"), }), @@ -948,6 +978,7 @@ impl Gpu { rings: channel::Rings::new(), recovery: recovery::State::new(), effort: power::Effort::new(), + registration_power_release_pending: false, queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), pool: object::Pool::new()?, render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, @@ -1025,6 +1056,7 @@ impl Gpu { self.shared.disable_work(&self.shared.feed_work); self.shared.feed.shutdown(); self.shared.drain_reclaims(); + self.shared.release_drm(); } } @@ -1129,6 +1161,7 @@ impl gpu::Gpu for Gpu { priority: u32, usc_exec_base: u64, ) -> Result> { + let dev = self.shared.drm()?; let owner = self.ids.queue.next(); let frontend = queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; @@ -1139,7 +1172,7 @@ impl gpu::Gpu for Gpu { } }; let frontend = match queue::Queue::new( - &self.shared.dev, + &dev, backend.context.clone(), backend.clone(), frontend, diff --git a/drivers/gpu/drm/asahi/g17/power.rs b/drivers/gpu/drm/asahi/g17/power.rs index 8806f438234e98..96b363bc27447e 100644 --- a/drivers/gpu/drm/asahi/g17/power.rs +++ b/drivers/gpu/drm/asahi/g17/power.rs @@ -70,6 +70,8 @@ pub(super) trait Control { fn counters(&self) -> kernel::error::Result<[u32; 3]>; fn set_idle(&mut self, enabled: bool) -> kernel::error::Result; fn wait_tick(&self); + fn registration_release_failed(&mut self) {} + fn registration_release_succeeded(&mut self) {} } fn wait(control: &C, target: u32, require_powered: bool) -> kernel::error::Result { @@ -113,13 +115,25 @@ pub(super) fn acquire(control: &mut C) -> kernel::error::Result { } Err(EAGAIN) })(); - if result.is_err() { - let _ = release(control); + if result.is_err() && release(control).is_err() { + control.registration_release_failed(); } result } pub(super) fn release(control: &mut C) -> kernel::error::Result { - let target = control.set_idle(true)?; - wait(control, target, false) + let target = match control.set_idle(true) { + Ok(target) => target, + Err(error) => { + control.registration_release_failed(); + return Err(error); + } + }; + let result = wait(control, target, false); + if result.is_ok() { + control.registration_release_succeeded(); + } else { + control.registration_release_failed(); + } + result } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index e0cdc050443621..b8b1f570528261 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -313,7 +313,7 @@ struct Active { ticket: Option, observed: Option, failure_deferred: bool, - qos_completed: bool, + qos_pending: bool, } struct Previous { binding: Option, @@ -511,6 +511,12 @@ impl Queue { data_master: DataMaster::Compute as u8, } } + /// A completion hint can require a second visibility/recordless pass even + /// when firmware has no more events to send. Do not poll idle or failed queues. + pub(crate) fn retirement_polling(&self) -> bool { + !self.quarantined && self.retire_pending + } + fn selected(&self, masks: Option<[u64; 2]>) -> bool { self.retired_by_teardown || self.retire_pending @@ -682,9 +688,9 @@ impl Queue { match result { Ok(Some(timestamps)) => { let submitted = self.submitted.checked_sub(1).ok_or(EIO)?; - if !front.qos_completed { + if front.qos_pending { complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; self.retire_pending = true; @@ -784,9 +790,9 @@ impl Queue { self.retirement_proved = true; } while let Some(front) = self.active.front() { - if self.spared_quarantine && !front.qos_completed { + if front.qos_pending { complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_completed = true; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; } let front = self.active.front().ok_or(EIO)?; let completion = front.packet.completion.clone(); @@ -1247,6 +1253,7 @@ impl Queue { return Err(EIO); } let crate::g17::command::Validated::Compute { + scratch, usc_base, cdm_va, cdm_end_va, @@ -1280,6 +1287,7 @@ impl Queue { self.graph.ensure_record_slot(slot)?; } let args = ComputeArgs { + scratch: *scratch, usc_base: *usc_base, cdm_va: *cdm_va, cdm_end_va: *cdm_end_va, @@ -1350,7 +1358,8 @@ impl Queue { }, mcache: None, event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low)?, + register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, + compute_scratch: scratch.enabled(), }; let published = host.next_compute_publication()?; self.active @@ -1361,7 +1370,7 @@ impl Queue { ticket: None, observed: None, failure_deferred: false, - qos_completed: false, + qos_pending: false, }) .map_err(|_| EBUSY)?; self.retirement_ready = false; @@ -1416,8 +1425,10 @@ impl Queue { }, identity.scheduler, )?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; let target = self.pool.reserve_submission().or_else(|error| { host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; producer = self @@ -1426,6 +1437,7 @@ impl Queue { .or_else(|error| { self.pool.cancel_unpublished(target)?; host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; } else { @@ -1448,11 +1460,13 @@ impl Queue { }, identity.scheduler, )?; + self.active.back_mut().ok_or(EIO)?.qos_pending = true; producer = self .graph .append(producer, config_va, || {}) .or_else(|error| { host.qos_cancel(qos)?; + self.active.back_mut().ok_or(EIO)?.qos_pending = false; Err(error) })?; } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 1ddadae2dba693..e420427d594bf4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -9,46 +9,17 @@ use kernel::{prelude::*, sync::Arc}; pub(crate) const POLL_NS: u64 = 2_500_000_000; const QUARANTINED_DELAY_NS: u64 = 250_000_000; const CANCELLED_DELAY_NS: u64 = 2_000_000_000; -const ACK_SLOTS: usize = 16; // Firmware-visible cookie namespace persists for the loaded module's lifetime. static NEXT_COOKIE: AtomicU32 = AtomicU32::new(1); -pub(crate) fn next_cookie() -> u64 { - loop { - let cookie = NEXT_COOKIE.fetch_add(1, Ordering::Relaxed); - if cookie != 0 { - return cookie.into(); - } - } -} - -pub(crate) struct Acks([u64; ACK_SLOTS]); -impl Acks { - pub(crate) fn new() -> Self { - Self([0; ACK_SLOTS]) - } - pub(crate) fn note(&mut self, cookie: u64) { - if cookie == 0 || self.contains(cookie) { - return; - } - if let Some(slot) = self.0.iter_mut().find(|slot| **slot == 0) { - *slot = cookie; - } - } - pub(crate) fn contains(&self, cookie: u64) -> bool { - cookie != 0 && self.0.contains(&cookie) - } - pub(crate) fn forget(&mut self, cookie: u64) { - for slot in &mut self.0 { - if *slot == cookie { - *slot = 0; - } - } - } - pub(crate) fn snapshot(&self) -> [u64; ACK_SLOTS] { - self.0 - } +pub(crate) fn next_cookie() -> Result { + // Never reuse an identity while a delayed firmware ACK can still exist. + // The final counter value is an exhaustion marker, not a wrapping cookie. + NEXT_COOKIE + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |next| next.checked_add(1)) + .map(u64::from) + .map_err(|_| EOVERFLOW) } pub(crate) struct State { @@ -101,6 +72,7 @@ pub(super) struct Transaction { cookie: u64, published: u64, phase: Phase, + announced: Option, } impl Queue { @@ -136,32 +108,50 @@ impl Queue { cookie, published: now, phase: Phase::Published, + announced: None, }); } + /// A failed doorbell does not undo the published kill or consume its retry. + /// The caller retries under the device mutex; only the original record exists. + pub(crate) fn announce_exit(&mut self, now: u64, notify: impl FnOnce() -> Result) -> Result { + let Some(exit) = self.exit.as_mut() else { + return Ok(()); + }; + if exit.phase != Phase::Published || exit.announced.is_some() { + return Ok(()); + } + notify()?; + exit.announced = Some(now); + Ok(()) + } pub(crate) fn exit_polling(&self, now: u64) -> bool { self.exit.as_ref().is_some_and(|exit| { - exit.phase == Phase::Published && now.saturating_sub(exit.published) < POLL_NS + exit.phase == Phase::Published + && now.saturating_sub(exit.announced.unwrap_or(exit.published)) < POLL_NS }) } - pub(crate) fn exit_cookie(&self) -> Option { - self.exit - .as_ref() - .filter(|exit| matches!(exit.phase, Phase::Published | Phase::Acknowledged)) - .map(|exit| exit.cookie) + /// Event consumption and record_exit are serialized by the device mutex. + /// A kill pins this exact queue transaction through its final witness; + /// duplicate, unknown and terminal cookies cannot acknowledge another owner. + pub(crate) fn acknowledge_exit(&mut self, cookie: u64) -> bool { + let Some(exit) = self.exit.as_mut() else { + return false; + }; + if cookie == 0 || exit.cookie != cookie || exit.phase != Phase::Published { + return false; + } + exit.phase = Phase::Acknowledged; + true } pub(crate) fn settle_exit( &mut self, - acks: &Acks, recovery_closed: bool, + complete_qos: &mut impl FnMut(crate::g17::qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - let Some(cookie) = self.exit_cookie() else { - return Ok(()); - }; - if !acks.contains(cookie) { + if !self.exit.as_ref().is_some_and(|exit| exit.phase == Phase::Acknowledged) { return Ok(()); } - self.exit.as_mut().ok_or(EIO)?.phase = Phase::Acknowledged; if self.active.len != 0 { if !recovery_closed { return Ok(()); @@ -185,6 +175,11 @@ impl Queue { } } while let Some(active) = self.active.front() { + if active.qos_pending { + complete_qos(self.qos_owner())?; + self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; + } + let active = self.active.front().ok_or(EIO)?; defer(Deferred::Retired( active.packet.completion.clone(), Err(if self.spared_deferred { @@ -202,6 +197,7 @@ impl Queue { self.spared_quarantine = false; self.retire_pending = false; self.retirement_ready = false; + self.update_slot_accounting(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 9d2bc056cce650..78765523254b66 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod render; use super::{ command::{Parser, UscWindow, Validated}, - completion::Completion, + completion::{Completion, Destinations}, context::Context, fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, @@ -21,7 +21,7 @@ use crate::{ use core::ops::Range; use kernel::{ c_str, - dma_fence::{FenceContexts, RawDmaFence}, + dma_fence::{Fence, FenceContexts, RawDmaFence}, prelude::*, sync::{Arc, LockClassKey}, xarray, @@ -53,6 +53,7 @@ impl Identities { /// Validated frontend state prepared before allocating a logical execution context. pub(crate) struct Frontend { fences: FenceContexts, + allocation_failure: Fence, range: Range, window: UscWindow, } @@ -64,8 +65,10 @@ impl Frontend { } let window = UscWindow::new(usc_base, range.clone())?; let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; + let allocation_failure = super::fence::allocation_failure(&fences)?; Ok(Self { fences, + allocation_failure, range, window, }) @@ -79,11 +82,13 @@ pub(crate) struct Queue { render: Option>, compute: Option>, fences: FenceContexts, + allocation_failure: Fence, range: Range, window: UscWindow, proof: Proof, sequence: [u64; 2], frontiers: [KVec<(u64, Fences)>; 2], + failures: [Option<(u64, Fence)>; 2], // Closing the userspace queue releases admission even if firmware still // retains one of its execution contexts or installed physical graphs. _slot: admission::QueueSlot, @@ -99,6 +104,7 @@ impl Queue { ) -> Result { let Frontend { fences, + allocation_failure, range, window, } = frontend; @@ -110,11 +116,13 @@ impl Queue { render, compute: None, fences, + allocation_failure, range, window, proof: Proof::new(), sequence: [0; 2], frontiers: core::array::from_fn(|_| KVec::new()), + failures: [None, None], _slot: slot, }) } @@ -132,6 +140,22 @@ impl Queue { Ok(()) } + fn prune_frontier(&mut self, index: usize) { + let failure = &mut self.failures[index]; + self.frontiers[index].retain(|(sequence, fences)| { + let status = job::fence_status(&fences.completed); + if status == 0 { + return true; + } + // One earliest failed ordinal covers every later prefix without + // retaining its packet, mappings, or every failed fence. + if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { + *failure = Some((*sequence, fences.completed.clone())); + } + false + }); + } + fn enqueue( &mut self, id: u64, @@ -140,62 +164,30 @@ impl Queue { guard: mmu::VmJobGuard, aggregate: &Arc, inputs: &[file::SyncItem], - objects: Pin<&xarray::XArray>>, + destinations: Destinations, ) -> Result { let engine = match &command { Validated::Render { .. } => Engine::Render, Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); - // One earliest failed timestamp covers every prefix containing a later - // failure, without retaining the producer's packet or mappings. - let mut retained_timestamp_failure = false; - self.frontiers[index].retain(|(_, fences)| { - let status = job::fence_status(&fences.completed); - if status == 0 { - return true; - } - if engine == Engine::Compute - && status < 0 - && fences.ready.raw() == fences.completed.raw() - && !retained_timestamp_failure - { - retained_timestamp_failure = true; - return true; - } - false - }); + self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { self.ensure_compute()?; } - let completion = match &command { - Validated::Render { timestamps, .. } => Completion::new( - &self.dev, - &self.fences, - &self.context, - aggregate.member(), - guard, - objects, - timestamps, - false, - self.backend.feed(), - )?, - Validated::Compute { timestamps, .. } => Completion::new( - &self.dev, - &self.fences, - &self.context, - aggregate.member(), - guard, - objects, - core::slice::from_ref(timestamps), - true, - None, - )?, - }; + let completion = Completion::new( + &self.dev, + &self.fences, + &self.context, + aggregate.member(), + guard, + destinations, + engine == Engine::Compute, + if engine == Engine::Render { self.backend.feed() } else { None }, + )?; if engine == Engine::Compute { - self.frontiers[Engine::Render.index()] - .retain(|(_, fences)| job::fence_status(&fences.completed) == 0); + self.prune_frontier(Engine::Render.index()); } let other = 1 - index; let prefix = order.wait_through[other]; @@ -222,10 +214,21 @@ impl Queue { .iter() .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) .count(); + let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { + self.failures[index].as_ref().and_then(|(sequence, fence)| { + Order::contains(order.wait_through[index], *sequence).then_some(fence) + }) + }); + let failure_count = failures.iter().flatten().count(); let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count, GFP_KERNEL)?; - let mut checked_inputs = KVec::with_capacity(inputs.len() + timestamp_count, GFP_KERNEL)?; + KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; + let mut checked_inputs = + KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; + for fence in failures.into_iter().flatten() { + checked_inputs.push(fence.clone(), GFP_KERNEL)?; + dependencies.push(fence.clone(), GFP_KERNEL)?; + } // Same-engine parents order GPU execution, not the host's timestamp // writes. Only timestamp publishers need these completion waits and // status checks; ordinary compute keeps its existing queue ordering. @@ -285,7 +288,8 @@ impl Queue { fn prepare_submission( &mut self, bytes: &[u8], - ) -> Result<(KVec<(Validated, Order)>, KVec)> { + objects: Pin<&xarray::XArray>>, + ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { let parse_guard = self.context.vm().retain_first_job()?; let mut parser = Parser::new(bytes); // All historical ordinals use the same entry counters; publication @@ -312,7 +316,13 @@ impl Queue { engine, )?; preceding[engine.index()] += 1; - pending.push((payload, order), GFP_KERNEL)?; + let destinations = match &payload { + Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, + Validated::Compute { timestamps, .. } => { + Destinations::resolve(objects, core::slice::from_ref(timestamps))? + } + }; + pending.push((payload, order, destinations), GFP_KERNEL)?; } parser.finish()?; // Every accepted command pins the VM before any command is enqueued. @@ -338,15 +348,20 @@ impl crate::queue::Queue for Queue { bytes: &[u8], objects: Pin<&xarray::XArray>>, ) -> Result { - let mut outputs = Outputs::new(&self.fences, syncs, input_count)?; + let mut outputs = Outputs::new( + &self.fences, + syncs, + input_count, + &self.allocation_failure, + )?; let result = (|| { if self.context.status().get() != 0 { return Err(EIO); } - let (pending, guards) = self.prepare_submission(bytes)?; + let (pending, guards) = self.prepare_submission(bytes, objects)?; let mut guards = guards.into_iter(); let aggregate = outputs.publish(self.context.clone())?; - for (command, order) in pending { + for (command, order, destinations) in pending { let guard = guards.next().ok_or(EIO)?; self.enqueue( id, @@ -355,7 +370,7 @@ impl crate::queue::Queue for Queue { guard, &aggregate, outputs.inputs(), - objects, + destinations, )?; } Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index c3ac091333c272..fd67523ab91c33 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -512,6 +512,7 @@ impl Pair { mcache, event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, + compute_scratch: false, }; let ta_args = args( 0, @@ -789,7 +790,7 @@ impl Pair { } pub(crate) fn needs_drain(&self) -> bool { - self.quarantined && self.in_flight() + self.quarantined } pub(crate) fn oldest_spared(&self) -> Option { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index f614494e1216aa..859ff61901570e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -47,12 +47,14 @@ impl Aliases { let mut range = dynamic::LOWER; let size = (source.end - source.start) as u64; loop { + // Descriptor aliases are read-only to GPU consumers; the canonical + // firmware mapping remains writable for command state updates. let global = object.map_range( alloc.uat.kernel_lower_vm(), source.clone(), range.clone(), mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + mmu::PROT_GPU_SHARED_RO, 0, )?; let address = global.iova(); @@ -62,7 +64,7 @@ impl Aliases { source.clone(), address..end, mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, + mmu::PROT_GPU_SHARED_RO, 0, ) { Ok(client) => { diff --git a/drivers/gpu/drm/asahi/g17/recovery.rs b/drivers/gpu/drm/asahi/g17/recovery.rs index 7e3db2f6dae783..ba368341d84d26 100644 --- a/drivers/gpu/drm/asahi/g17/recovery.rs +++ b/drivers/gpu/drm/asahi/g17/recovery.rs @@ -22,29 +22,30 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: u8 = 128; const GUILTY: u8 = 1 << 1; const CANCELLED: u8 = 1 << 2; -/// Retained valid entries, in firmware table order. -pub(super) const MAX_ENTRIES: usize = 64; const MAX_GUILTY_VMS: usize = 8; const RECOVERY_POLLS: usize = 500; -/// One valid entry from the firmware's 256-entry recovery table. -#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] -pub(super) struct Entry { - pub(super) qid: u8, - pub(super) flags: u8, - pub(super) stamp: u32, +/// Per-QID classifications from the complete firmware recovery table. +/// Multiple records for a queue contribute all of their classification bits. +#[derive(Default)] +pub(super) struct Entries { + guilty: u128, + cancelled: u128, } -impl Entry { - pub(super) const fn decode(flags: u32, stamp: u32) -> Option { +impl Entries { + pub(super) fn include(&mut self, flags: u32) { if flags & 1 == 0 { - return None; + return; + } + let qid = (flags >> 4) & 0x7f; + let bit = 1u128 << qid; + if flags & u32::from(GUILTY) != 0 { + self.guilty |= bit; + } + if flags & u32::from(CANCELLED) != 0 { + self.cancelled |= bit; } - Some(Self { - qid: ((flags >> 4) & 0x7f) as u8, - flags: (flags & 0xf) as u8, - stamp: stamp >> 8, - }) } } @@ -83,7 +84,7 @@ pub(super) trait Memory { fn fault_sources(&self) -> Result; fn transition(&self, expected: u32, next: u32) -> Result; fn clear_timestamps(&self) -> Result; - fn entries(&self, out: &mut [Entry; MAX_ENTRIES]) -> Result; + fn entries(&self) -> Result; /// One millisecond wait; this is polled because the firmware sends no state-3 notification. fn wait_tick(&self); } @@ -163,10 +164,31 @@ pub(super) fn service( crashed: impl Fn() -> bool, ) -> Result { let request = host.recovery().pending.ok_or(EINVAL)?; - if host.recovery_state()? != 1 - || request.generation != host.recovery().accepted - || host.host_recovery()? != 0 - { + if request.generation != host.recovery().accepted || host.host_recovery()? != 0 { + return Err(EIO); + } + // The restart record is published before firmware announces state 1. + // Its ring cursor is already consumed; wait only for that readiness word, + // without advancing the host generation or acknowledging an early event. + let mut ready = false; + for _ in 0..RECOVERY_POLLS { + if crashed() { + return Err(EIO); + } + match host.recovery_state()? { + 1 => { + ready = true; + break; + } + 0 => host.wait_tick(), + _ => return Err(EIO), + } + } + if !ready { + return Err(ETIMEDOUT); + } + // Recheck ownership after the off-CPU waits, before any handshake mutation. + if host.host_recovery()? != 0 { return Err(EIO); } host.recovery_mut().accepted = host.recovery().accepted.wrapping_add(1); @@ -190,16 +212,15 @@ pub(super) fn service( if !recovered { return Err(ETIMEDOUT); } - let mut entries = [Entry::default(); MAX_ENTRIES]; // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let count = host.entries(&mut entries).unwrap_or(0).min(MAX_ENTRIES); + let entries = host.entries().unwrap_or_default(); host.transition(3, 0)?; if host.recovery_state()? != 0 { return Err(EIO); } host.recovery_mut().pending = None; - let cancelled = classify(host, &entries[..count], request.blamed, &mut sources); + let cancelled = classify(host, entries, request.blamed, &mut sources); if let Some(qid) = request.blamed { host.settle_blamed_render(qid, EIO); } @@ -214,17 +235,9 @@ pub(super) fn service( /// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or /// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work /// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: &[Entry], blamed: Option, sources: &mut Sources) -> u128 { - let mut guilty_qids = blamed.map_or(0, |qid| 1u128 << qid); - let mut cancelled = 0u128; - for entry in entries { - if entry.flags & GUILTY != 0 { - guilty_qids |= 1u128 << entry.qid; - } - if entry.flags & CANCELLED != 0 { - cancelled |= 1u128 << entry.qid; - } - } +fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { + let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); + let cancelled = entries.cancelled; let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); match attribution { Attribution::Firmware => {} diff --git a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs index b0c42d4e3f7c15..1755484219187e 100644 --- a/drivers/gpu/drm/asahi/g17/recovery/attribution.rs +++ b/drivers/gpu/drm/asahi/g17/recovery/attribution.rs @@ -16,7 +16,7 @@ pub(super) struct Source { } pub(crate) struct Sources { - pub(crate) reason: u32, + pub(crate) reason: Option, mask: u32, records: [Source; SLOTS], count: usize, @@ -31,10 +31,10 @@ pub(super) enum Attribution { impl Sources { /// Firmware is halted for every selected key and progress-record read. - /// Missing keys stay unknown; they cannot establish execution by an owner. - pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> (u64, u32)) -> Self { + /// Failed reads stay unknown; incomplete evidence cannot override firmware blame. + pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> Option<(u64, u32)>) -> Self { let mut sources = Self { - reason: 0, + reason: None, mask, records: [Source::default(); SLOTS], count: 0, @@ -43,7 +43,9 @@ impl Sources { if mask & (1 << slot) == 0 { continue; } - let (key, monitor) = read(slot); + let Some((key, monitor)) = read(slot) else { + continue; + }; let qid = (key >> 40) & 0xff; sources.records[sources.count] = Source { slot: slot as u8, @@ -62,12 +64,15 @@ impl Sources { blamed: Option, mut started: impl FnMut(u8, u64) -> Option, ) -> Attribution { + if self.reason.is_none() || self.count as u32 != self.mask.count_ones() { + return Attribution::Firmware; + } for source in &mut self.records[..self.count] { if let Some(qid) = source.qid { source.started = started(qid, source.stamp); } } - if self.reason == PAGE_FAULT || self.mask.count_ones() < 2 { + if self.reason == Some(PAGE_FAULT) || self.mask.count_ones() < 2 { return Attribution::Firmware; } let Some(blamed) = blamed else { diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 294d66b91f8dc1..805087d7082abd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -29,6 +29,7 @@ use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; const FIRST_COMPUTE_QID: u8 = 4; const RECOVERY_SETTLE_POLLS: usize = 200; +const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; /// A reservation remains identifiable while its backing is built without the /// device mutex. Failed construction returns the slot, never its host identity. @@ -56,10 +57,10 @@ pub(super) struct Registry { compute: KVec>, pub(super) compute_building: bool, compute_scheduler_building: bool, + replay_outer_wait: Option, pub(super) observations: compute::retirement::Batch, worker_deferred: Option, teardown: super::teardown::Pending, - exit_acks: compute::exit::Acks, } impl Registry { @@ -91,10 +92,10 @@ impl Registry { }, compute_building: false, compute_scheduler_building: false, + replay_outer_wait: None, observations: compute::retirement::Batch::new()?, worker_deferred: Some(DeferredBatch::worker()?), teardown: super::teardown::Pending::new()?, - exit_acks: compute::exit::Acks::new(), }) } @@ -235,6 +236,7 @@ impl Backend { owner: u64, ) -> Result<(Arc, admission::QueueSlot)> { let lease = shared.preparations.enter(true)?; + let dev = shared.drm()?; let (uat, pool, qos, execution, slot) = { let mut state = shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -252,7 +254,7 @@ impl Backend { ) }; let alloc = super::object::Allocator { - dev: &shared.dev, + dev: &dev, uat: &uat, }; let owner_pid = @@ -285,6 +287,7 @@ impl Backend { fn shared_compute(&self) -> Result { loop { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let uat = { let mut state = self.shared.state.lock(); loop { @@ -310,7 +313,7 @@ impl Backend { } }; let alloc = super::object::Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let mut built = Some(super::initdata::ComputeShared::new(&alloc)); @@ -406,6 +409,7 @@ impl Backend { } loop { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (uat, reservation) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -423,7 +427,7 @@ impl Backend { (firmware.uat.clone(), reservation) }; let alloc = super::object::Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let built = (|| { @@ -527,7 +531,19 @@ impl DeferredBatch { completion.status().record(*error); } } - self.values.push_within_capacity(value).map_err(|_| EIO) + self.values.push_within_capacity(value).map_err(|_| EIO)?; + // A later publisher can borrow this pair before finish signals its old + // fence. Publish the exact completion's witness only after collection + // succeeds; failure fences and rejected events are not retirement. + if let Some( + Deferred::Retired(completion, _) + | Deferred::Drained(completion, _) + | Deferred::Release(completion), + ) = self.values.last() + { + completion.note_retirement_queued(); + } + Ok(()) } /// Called only after releasing the device mutex. pub(super) fn finish(&mut self) { @@ -596,6 +612,11 @@ impl super::Firmware { if self.recovery.pending() { return Err(EAGAIN); } + // No descriptor, accounting or inner producer changes precede this + // capacity check. The same device lock covers queue.publish below. + if !self.init.work_ready(DataMaster::Compute)? { + return Err(EAGAIN); + } let queues = &mut self.queues; let queue = queues .compute @@ -751,13 +772,15 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - break; + return self.build_compute(); } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. fsleep(Delta::from_millis(1)); } - self.build_compute() + // No new command has been accepted. Keep the spared queue and its + // retained work intact until a later admission observes its witness. + Err(EBUSY) } /// Called after taking this packet's publication claim. All waits and work @@ -769,6 +792,7 @@ impl Backend { ) -> Result { packet.check_dependencies()?; let mut recovery_polls = 0; + let mut outer_wait = None; loop { if packet.completion.status().get() != 0 { return Err(EIO); @@ -826,6 +850,20 @@ impl Backend { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); continue; } + if matches!(result, Err(EAGAIN)) && !packet.is_published() { + let now = now_ns(); + let since = *outer_wait.get_or_insert(now); + if now.saturating_sub(since) >= WORK_RING_WAIT_NS { + return Err(EBUSY); + } + // Backpressure owns no firmware-visible attempt. Other queues + // can submit and the event worker can retire while we sleep. + drop(state); + drop(lease); + self.shared.queue_events(); + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); + continue; + } return result; } } @@ -863,12 +901,15 @@ impl super::Firmware { } impl super::Shared { - /// Joins construction and serializes the stop writes with detachment. - /// After installation, seeing no owner proves that an earlier stop caller - /// has already stopped both processors. Callbacks and destruction follow - /// outside the device mutex. + /// Joins construction and complete host teardown, including RTKit drops. + /// The stop-only mutex outlives detached firmware destruction, so unbind + /// cannot return while another stop caller still owns mailbox clients. + /// Completion callbacks remain outside the device mutex. Direct handlers + /// do not synchronously request or join stop; this path does not join + /// scheduler workers. pub(super) fn stop_queues(&self) { self.preparations.wait_drained(); + let _stopping = self.stop.lock(); let firmware = { let mut state = self.state.lock(); if let Some(firmware) = (*state).as_deref_mut() { @@ -879,6 +920,8 @@ impl super::Shared { }; if let Some(mut firmware) = firmware { firmware.finish_stopped_queues(); + // RTKit destruction still dereferences the mailbox provider. + drop(firmware); } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 0643da70d72ca0..439d5c9831cbb9 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -14,33 +14,44 @@ use crate::g17::{ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; +impl super::Registry { + /// ACK storage belongs to the already-retained transaction, so a full event + /// batch cannot overflow a separate cache. No allocation or wait is needed. + pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { + for entry in self.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + if queue.acknowledge_exit(cookie) { + return; + } + } + } + } +} + impl Firmware { pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { let recovery_closed = self.init.recovery_state()? == 0; + let view = crate::g17::qos::View::new(self.init.qos()?)?; for entry in self.queues.compute.iter_mut().flatten() { if let Some(queue) = entry.queue.as_deref_mut() { - queue.settle_exit(&self.queues.exit_acks, recovery_closed, &mut |event| { - deferred.push(event) - })?; - } - } - for cookie in self.queues.exit_acks.snapshot() { - if cookie != 0 - && !self - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_cookie() == Some(cookie)) - { - self.queues.exit_acks.forget(cookie); + queue.settle_exit( + recovery_closed, + &mut |owner| self.queues.accounting.complete(&view, &[owner]), + &mut |event| deferred.push(event), + )?; } } if self.recovery.pending() || !recovery_closed { return Ok(()); } let now = now_ns(); + // Publication has already retained each kill. Retry only the doorbell; + // failed attempts do not extend its bounded polling window. + for entry in self.queues.compute.iter_mut().flatten() { + if let Some(queue) = entry.queue.as_deref_mut() { + let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); + } + } for index in 0..self.queues.teardown.len() { let Some((context, age)) = self.queues.teardown.closed(index, now) else { continue; @@ -103,6 +114,7 @@ impl Firmware { hold |= 1u64 << slot; } } + let cookie = exit::next_cookie()?; if hold != 0 && !self .queues @@ -111,7 +123,6 @@ impl Firmware { { return Ok(()); } - let cookie = exit::next_cookie(); let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie)); if let Err(error) = Rings::publish_control(&self.init, &record) { self.queues.admission.release_exit_kill_hold(hold); @@ -119,13 +130,13 @@ impl Firmware { } // Publication is irrevocable even if its doorbell fails. Keep both // the killed graph and the held render identities until processor stop. - self.queues.compute[index] + let queue = self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) - .ok_or(EIO)? - .record_exit(cookie, now); + .ok_or(EIO)?; + queue.record_exit(cookie, now); fence(Ordering::SeqCst); - if let Err(error) = self.primary.notify(MSG_CONTROL_NOTIFY) { + if let Err(error) = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)) { dev_warn!( self.primary.state.shared.dev.as_ref(), "Scheduler {} exit announcement failed: {:?}\n", diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index c872a7c8dacc72..65b00d36714bb0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -27,6 +27,7 @@ impl Firmware { .map(|queue| queue.replay_commands()) .sum::(); if commands == 0 { + self.queues.replay_outer_wait = None; return Ok(()); } deferred.reserve_replays(commands)?; @@ -55,8 +56,24 @@ impl Firmware { break; }; let owner = queue.owner(); + let outer_ready = self.init.work_ready(crate::g17::fw::queue::DataMaster::Compute)?; + if !outer_ready { + let now = super::now_ns(); + let since = *self.queues.replay_outer_wait.get_or_insert(now); + if now.saturating_sub(since) < super::WORK_RING_WAIT_NS { + // Keep the FIFO and its replay credit untouched. The + // event worker schedules the existing delayed poll. + return Ok(()); + } + } else { + self.queues.replay_outer_wait = None; + } - let result = (|| { + let result = if !outer_ready { + // This replay has no live firmware ownership. Persistent + // backpressure uses the existing failed-replay retirement. + Err(EBUSY) + } else { (|| { self.queues.compute[index] .as_mut() .and_then(|entry| entry.queue.as_deref_mut()) @@ -66,7 +83,7 @@ impl Firmware { packet.check_dependencies()?; let dependencies = self.queues.render_prefix(owner, &packet)?; self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) - })(); + })() }; let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) && (self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0); @@ -89,6 +106,7 @@ impl Firmware { queue.finish_replay(&packet)?; } } + self.queues.replay_outer_wait = None; Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f249e3520e0dd9..08c11eb98790eb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -64,7 +64,7 @@ impl event::Host for Service<'_> { impl event::Queues for Service<'_> { fn context_killed(&mut self, cookie: u64) { - self.firmware.queues.exit_acks.note(cookie); + self.firmware.queues.acknowledge_compute_exit(cookie); } fn flush_tvb_reply(&mut self) -> Result { self.firmware.flush_tvb_reply() @@ -125,8 +125,8 @@ impl recovery::Memory for Service<'_> { fn clear_timestamps(&self) -> Result { self.firmware.init.clear_timestamps() } - fn entries(&self, out: &mut [recovery::Entry; recovery::MAX_ENTRIES]) -> Result { - self.firmware.init.entries(out) + fn entries(&self) -> Result { + self.firmware.init.entries() } fn wait_tick(&self) { self.firmware.init.wait_tick(); @@ -395,8 +395,16 @@ impl crate::g17::Shared { } let backpressured = firmware.render_control_backpressured(); let polling = backpressured + || firmware.queues.replay_outer_wait.is_some() || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now); + || firmware.compute_exit_polling(now) + || firmware + .queues + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .any(|queue| queue.retirement_polling()); if polling { self.queue_poll(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a6aa740ecf4e3e..e466571bc88c7b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -239,6 +239,17 @@ impl Registry { Ok(()) } + /// Only an optional sibling's unpublished QID-capacity refusal is soft. + /// ensure_render_ids cancels a first ID if the second cannot be reserved; + /// validation, identity overflow, and cancellation errors still propagate. + fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { + match self.ensure_render_ids(slot, owner) { + Ok(()) => Ok(true), + Err(error) if optional && error == ENOSPC => Ok(false), + Err(error) => Err(error), + } + } + fn reserve_render(&mut self, slot: u8, owner: u64) -> Result { self.ensure_render_ids(slot, owner)?; let entry = self.render.entry(slot)?; @@ -318,7 +329,7 @@ impl Registry { } if let Some(pair) = entry.pair.as_mut() { pair.quarantine(error, &mut |event| deferred.push(event))?; - } else if entry.preparing.is_some() { + } else if entry.preparing.is_some() || entry.growing { entry.pending_error = Some((status.clone(), error)); } } @@ -332,7 +343,7 @@ struct Host<'a> { } impl Host<'_> { fn release_registration(&mut self) { - if !core::mem::take(&mut self.registration_powered) { + if !self.registration_powered && !self.firmware.registration_power_release_pending { return; } if let Err(error) = self.firmware.release_registration_power() { @@ -341,6 +352,8 @@ impl Host<'_> { "Could not restore render idle policy: {:?}\n", error ); + } else { + self.registration_powered = false; } } } @@ -647,7 +660,10 @@ impl Registry { let active = entry .pair .as_ref() - .map_or(entry.borrowed_active, |pair| pair.published_in_flight()); + .map_or( + (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, + |pair| pair.published_in_flight(), + ); active && entry.reservation.ids.into_iter().all(|id| { let qid = usize::from(id.qid()); @@ -660,6 +676,12 @@ impl Registry { packet: &Arc, error: Error, ) -> bool { + // Its tracker may already be gone while off-lock completion still + // owns the fence and admission lease. A loan for a subsequent packet + // cannot turn this exact accepted retirement into a missing-owner error. + if packet.completion.retirement_queued() { + return true; + } let Some(slot) = packet.completion.render_slot() else { return false; }; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs index db5820cfb66638..b283fb8e7d55ca 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/grow.rs @@ -194,6 +194,7 @@ impl crate::g17::Shared { pub(in crate::g17) fn grow_render(&self) -> Result { loop { let lease = self.preparations.enter(false)?; + let dev = self.drm()?; let (request, slot, mut pair) = { let mut state = self.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -246,7 +247,7 @@ impl crate::g17::Shared { Err(EFAULT) } else { manager.prepare_growth( - &self.dev, + &dev, &vm, crate::g17::buffer::firmware_growth_target(before), None, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index e8fc30560a8904..69f8249e0818d3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -60,15 +60,18 @@ enum Build { } impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc) -> Result { + fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (resources, build) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { return Err(EAGAIN); } - firmware.queues.ensure_render_ids(slot, self.owner)?; + if !firmware.queues.prepare_render_ids(slot, self.owner, optional)? { + return Ok(()); + } let resources = BuildResources::new(firmware)?; let build = if let Some(entry) = firmware.queues.render.entries[usize::from(slot)].as_ref() { @@ -159,7 +162,7 @@ impl super::super::Backend { (resources, build) }; let alloc = Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &resources.uat, }; let fresh_graph = match &build { @@ -458,7 +461,7 @@ impl super::super::Backend { } pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet)?; + self.build_render_pair(slot, packet, false)?; let sibling = { let state = self.shared.state.lock(); (*state) @@ -469,7 +472,12 @@ impl super::super::Backend { .warmable_sibling(self.owner, slot) }; if let Some(sibling) = sibling { - self.build_render_pair(sibling, packet)?; + // An optional sibling may be borrowed or waiting for recovery. + // The selected pair is revalidated by prepare_render before publication. + match self.build_render_pair(sibling, packet, true) { + Err(error) if error == EBUSY || error == EAGAIN => {} + result => result?, + } } Ok(()) } @@ -481,6 +489,7 @@ impl super::super::Backend { deferred: &mut super::super::DeferredBatch, ) -> Result { let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; let (uat, mut pair, addresses, growth_target) = { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -535,7 +544,7 @@ impl super::super::Backend { (firmware.uat.clone(), pair, addresses, growth_target) }; let alloc = Allocator { - dev: &self.shared.dev, + dev: &dev, uat: &uat, }; let mut growth = None; @@ -543,7 +552,7 @@ impl super::super::Backend { if let Some((target, descriptor)) = growth_target { let vm = pair.context().vm().clone(); growth = pair.manager().prepare_growth( - &self.shared.dev, + &dev, &vm, target, Some(descriptor), diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 0586dfdc0944f1..59a54b19263757 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -260,6 +260,13 @@ impl crate::g17::Firmware { if packet.completion.status().get() != 0 { return Err(EIO); } + // Both outer destinations must have room before pool accounting or + // either inner queue becomes visible. All host producers share this lock. + if !self.init.work_ready(crate::g17::fw::queue::DataMaster::Fragment)? + || !self.init.work_ready(crate::g17::fw::queue::DataMaster::Tiling)? + { + return Err(EBUSY); + } let mut dependencies = [KickDependency::ZERO; 3]; let count = self .queues diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index 06465629eb2be1..b1495dbf3e23e3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -30,7 +30,10 @@ impl Reclaim { pub(in crate::g17) fn finish(self) { match self { Self::Binding(binding) => drop(binding), - Self::Context(context) => drop(context), + Self::Context(context) => { + context.release_execution(); + drop(context); + } } } } @@ -187,7 +190,7 @@ impl Firmware { primary: &mut self.primary, reclaim, }; - teardown.publish_ready(&mut service, recovery)?; + teardown.publish_ready(&mut service, recovery, super::now_ns())?; let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); let producer = self.init.control_producer()?.load(Ordering::Relaxed); while teardown diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 9acc01fbc21256..f53c2d45c129ab 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -23,6 +23,7 @@ struct Entry { phase: Phase, armed: u64, warned: bool, + announce_pending: bool, } /// Publication and announcement errors retain the entry. A stalled release is reported once; @@ -88,6 +89,7 @@ impl Pending { phase, armed: now, warned: false, + announce_pending: false, }) .map_err(|_| ENOSPC) } @@ -130,9 +132,21 @@ impl Pending { /// Publish in insertion order. Mandatory control replies and a full control ring stop this /// pass; an error for one page leaves it retained while other ready releases may progress. - pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool) -> Result { + pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool, now: u64) -> Result { if !recovery { for entry in self.entries.iter_mut() { + // The ring record is already published. Retry only its doorbell; + // never duplicate a scheduler release after a transport error. + if matches!(entry.phase, Phase::Released(_)) { + if entry.announce_pending && host.notify().is_ok() { + entry.announce_pending = false; + // One successful retry starts its consumption window. + // Failed retries never slide the polling deadline. + entry.armed = now; + entry.warned = false; + } + continue; + } if entry.phase != Phase::Draining || !host.render_idle(&entry.context)? || !host.flists_released(&entry.context) @@ -160,9 +174,15 @@ impl Pending { } }; entry.phase = Phase::Released(next); + // A long command drain must not consume the acknowledgement + // polling window before this release exists on the control ring. + entry.armed = now; + entry.warned = false; + entry.announce_pending = true; fence(Ordering::SeqCst); - if let Err(error) = host.notify() { - host.report(&entry.context, Failure::Announcement(error)); + match host.notify() { + Ok(()) => entry.announce_pending = false, + Err(error) => host.report(&entry.context, Failure::Announcement(error)), } } } @@ -200,8 +220,9 @@ impl Pending { Ok(None) } - /// Poll quickly only during the initial release window; late arrivals still settle when - /// serviced. Time passage never substitutes for a firmware-consumption witness. + /// Poll for a bounded window at close and again at actual release publication. + /// Failed announcements remain retryable on later service even after polling + /// expires. Time passage never substitutes for a firmware-consumption witness. pub(crate) fn polling(&self, now: u64) -> bool { self.entries.iter().any(|entry| { entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index c183d52562b85f..5f991f57872900 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -58,6 +58,17 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +/// VM-private BOs share their assigned VM's reservation object. Validate this +/// before acquiring mapping state; generic GPUVM accepts external shareable BOs. +pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 + && vm.is_extobj(gem) + { + return Err(EINVAL); + } + Ok(()) +} + unsafe impl Send for AsahiObject {} unsafe impl Sync for AsahiObject {} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4ab676f0755265..912178d616371e 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -626,26 +626,7 @@ impl gpuvm::DriverGpuVm for VmInner { if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } let va = op.va().expect("step_unmap: missing VA"); - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); - - let _mutation = self.mapping_mutation(); - self.page_table - .unmap_pages(va.addr()..(va.addr() + va.range()))?; - - if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); - self.tlbi_range(asid as u8, va.addr(), va.range() as usize); - mod_dev_dbg!( - self.dev, - "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", - asid, - va.addr(), - va.range(), - ); - mem::sync(); - } - - self.tlbi_contexts_naming_root(va.addr(), va.range() as usize); + self.unmap_gpuva(va)?; if op.unmap_and_unlink_va_defer().is_none() { dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); @@ -736,15 +717,45 @@ impl gpuvm::DriverGpuVm for VmInner { } impl VmInner { + /// Remove hardware leaves for one complete user GPUVA without allocating. + fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + + let _mutation = self.mapping_mutation(); + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.range()))?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + self.tlbi_range(asid as u8, va.addr(), va.range() as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + va.addr(), + va.range(), + ); + mem::sync(); + } + + self.tlbi_contexts_naming_root(va.addr(), va.range() as usize); + + Ok(()) + } + /// A failed map has no GPUVA owner to retain its backing or remove its leaves. /// GPUVM has already split or removed overlaps throughout this operation's range. fn rollback_failed_map(&mut self, addr: u64, size: u64) { - if self.uat_inner.firmware == UatFirmware::Handoff { - return; - } // Include the current segment: map_pages can fail after writing some of its leaves. // The checked GPUVM operation supplies an aligned range without overflow. let result = self.page_table.discard_partial_map(addr..addr + size); + if self.uat_inner.firmware == UatFirmware::Handoff { + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + self.tlbi_range(asid as u8, addr, size as usize); + mem::sync(); + } + } self.tlbi_contexts_naming_root(addr, size as usize); if let Err(error) = result { dev_err!( @@ -908,6 +919,21 @@ impl VmInner { /// Map an `mm::Node` representing an mapping in VA space. fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let end = node + .start() + .checked_add(node.mapped_size as u64) + .ok_or(EOVERFLOW)?; + let _mutation = self.mapping_mutation(); + self.page_table.prepare_map(node.start()..end)?; + let result = self.map_node_prepared(node, prot); + if result.is_err() { + self.rollback_failed_map(node.start(), node.mapped_size as u64); + } + result + } + + /// Install only into the empty range reserved by `map_node`. + fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); let mapping_end = if self.uat_inner.fault.is_some() { @@ -918,7 +944,6 @@ impl VmInner { if self.uat_inner.fault.is_some() { self.page_table.prepare_map(iova..mapping_end)?; } let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().inner.lock(); let sgt = guard.sgt.as_ref().ok_or(EINVAL)?; - let _mutation = self.mapping_mutation(); let mut offset = node.offset; let mut left = node.mapped_size; @@ -1064,14 +1089,16 @@ where pub(crate) struct SlotInner { generation: u8 } /// An application TTBAT identity retained by a logical queue and its jobs. /// The root lease is dropped before the VM, so no page table can be freed -/// while a context still names it. Installed firmware queues must retain this -/// object through replacement retirement or processor stop. +/// while an active context still names it. Installed firmware queues retain +/// this object through replacement retirement or processor stop; a consumed +/// closed-scheduler release may return its root identity while keeping storage. pub(crate) struct ExecutionContext { root: ExecutionRoot, vm: Vm, } struct ExecutionRoot { + released: AtomicBool, inner: Arc, id: u8, generation: u8, @@ -1090,16 +1117,28 @@ impl ExecutionContext { &self.vm } + /// Called only after the closed scheduler's release was consumed and all + /// its commands retired. Installed storage may outlive this root lease. + pub(crate) fn release(&self) { + self.root.release(); + } + pub(crate) fn is_current(&self) -> bool { let inner = self.root.inner.lock(); + if self.root.released.load(Ordering::Acquire) { + return false; + } let roots = &inner.ttbs()[usize::from(self.root.id)]; roots.ttb0.load(Ordering::Acquire) == self.root.low && roots.ttb1.load(Ordering::Acquire) == self.root.high } } -impl Drop for ExecutionRoot { - fn drop(&mut self) { +impl ExecutionRoot { + fn release(&self) { + if self.released.swap(true, Ordering::AcqRel) { + return; + } let id = usize::from(self.id); let release = { let inner = self.inner.lock(); @@ -1129,6 +1168,12 @@ impl Drop for ExecutionRoot { } } +impl Drop for ExecutionRoot { + fn drop(&mut self) { + self.release(); + } +} + /// Slot data for a [`Vm`] slot (nothing, we only care about the indices). pub(crate) struct SlotInner(); @@ -1203,10 +1248,9 @@ impl Clone for VmBind { } } -/// A mapping BO reference selecting the teardown protocol of its firmware. +/// A mapping BO reference released under the immediate-mode GPUVA lock. /// Retain the owner until deferred cleanup completes, including its final BO. struct MappingBo { - deferred: bool, bo: Option>>, owner: ARef>, } @@ -1214,10 +1258,8 @@ impl MappingBo { fn new( bo: ARef>, owner: ARef>, - deferred: bool, ) -> Self { Self { - deferred, bo: Some(bo), owner, } @@ -1231,12 +1273,6 @@ impl Drop for MappingBo { let Some(bo) = self.bo.take() else { return; }; - if !self.deferred { - // Handoff VMs retain the legacy ARef decrement and reservation - // locking behavior. Only published-root VMs use deferred cleanup. - drop(bo); - return; - } let raw = ARef::into_raw(bo); // SAFETY: GpuVmBo is repr(C) with drm_gpuvm_bo first. into_raw // transfers one live reference. Every Vm here uses IMMEDIATE_MODE; @@ -1265,6 +1301,27 @@ pub(crate) struct KernelMappingInner { mapped_size: usize, } +/// Prepared CPU timestamp access; destruction happens after completion signalling. +pub(crate) struct TimestampMapping { + mapping: shmem::VMap, + offset: usize, +} + +impl TimestampMapping { + pub(crate) fn write(&self, value: u64) { + let map = self.mapping.get(); + for (index, byte) in value.to_le_bytes().into_iter().enumerate() { + // SAFETY: timestamp_mapping checked the complete eight-byte extent. + // The owned vmap pins its backing; byte atomics permit unaligned and + // overlapping timestamp destinations without mutable slice aliases. + let destination = unsafe { + &*map.as_mut_ptr().add(self.offset + index).cast::() + }; + destination.store(byte, Ordering::Relaxed); + } + } +} + /// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. pub(crate) struct KernelMapping(ManuallyDrop>); @@ -1296,8 +1353,8 @@ impl KernelMapping { /// Writes a userspace timestamp through the retained GEM mapping. Byte /// stores support unaligned destinations and concurrent completion writes /// without creating mutable aliases. The complete timestamp is not atomic. - pub(crate) fn write_timestamp(&self, offset: usize, value: u64) -> Result { - use core::sync::atomic::AtomicU8; + /// Retain CPU access before this destination's completion fence is published. + pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { use kernel::drm::gem::BaseObject; let gem = self.0._gem.as_ref().ok_or(EINVAL)?; if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { @@ -1308,15 +1365,7 @@ impl KernelMapping { if end > gem.size() { return Err(ERANGE); } - let vmap = gem.vmap::()?; - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The retained GEM/vmap covers each checked byte. AtomicU8 - // has byte alignment and all concurrent completion writers use - // the same access width, without borrowing a mutable byte slice. - let destination = unsafe { &*vmap.as_mut_ptr().add(start + index).cast::() }; - destination.store(byte, Ordering::Relaxed); - } - Ok(()) + Ok(TimestampMapping { mapping: gem.owned_vmap::()?, offset: start }) } /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. @@ -1389,6 +1438,7 @@ impl KernelMapping { /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the /// coprocessor cache. This is required to safely unmap cached/private mappings. fn remap_uncached_and_flush(&mut self) { + let mut owner = self.0.owner.lock_inner(); fn remap_uncached_and_flush_legacy(&mut self) { let mut owner = self .0 @@ -1545,11 +1595,9 @@ impl KernelMapping { self.remap_uncached_and_flush_legacy(); } - let mut owner = self - .0 - .owner - .exec_lock(None, false) - .expect("exec_lock failed in KernelMapping::drop"); + // Only VM page tables/binding state are touched here. The shared + // reservation is sufficient and avoids fallible exec allocation in Drop. + let mut owner = self.0.owner.lock_inner(); mod_dev_dbg!( owner.dev, "MMU: unmap {:#x}:{:#x}\n", @@ -2643,16 +2691,15 @@ impl Vm { } core::mem::drop(vm_bo_guard); - let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; - let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); let uat_inner = inner.uat_inner.clone(); // Reserve the VA before attaching objects whose destructors take the // GEM reservation lock. The allocator drops its payload internally // on an insertion error, while exec_lock still owns that lock. let node_result = inner.mm.insert_node_in_range( // A failed reservation drops its payload while exec still owns GEM - // reservations. Published-root VMs attach BO/GEM only after success; - // handoff VMs retain their existing payload and decrement behavior. + // reservations. Attach BO/GEM references only after success, since + // their destructors may acquire the same reservation locks. let result = inner.mm.insert_node_in_range( KernelMappingInner { owner: self.inner.clone(), @@ -2677,7 +2724,7 @@ impl Vm { return Err(error); } }; - if deferred { + { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); payload._gem = Some(gem.into()); @@ -2726,13 +2773,14 @@ impl Vm { } core::mem::drop(vm_bo_guard); - let deferred = inner.uat_inner.firmware != UatFirmware::Handoff; - let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone(), deferred)); + let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); let uat_inner = inner.uat_inner.clone(); // See map_in_range(): keep dma_resv-owning references out of the // allocator payload until the fixed-address reservation succeeds. let node_result = inner.mm.reserve_node( // Published-root VMs defer BO/GEM attachment until reservation succeeds. + // Keep reservation-locking destructors out of the allocator payload + // until reservation succeeds. let result = inner.mm.reserve_node( KernelMappingInner { owner: self.inner.clone(), @@ -2754,7 +2802,7 @@ impl Vm { return Err(error); } }; - if deferred { + { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); payload._gem = Some(gem.clone()); @@ -2790,6 +2838,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Mapping needs a complete context let mut ctx = StepContext { @@ -2802,22 +2851,29 @@ impl Vm { let vm_bo = self.inner.obtain_bo(gem)?; { - let mut vm_bo_guard = vm_bo.inner().inner.lock(); - if vm_bo_guard.sgt.is_none() { + let needs_sgt = vm_bo.inner().inner.lock().sgt.is_none(); + if needs_sgt { + // Pin without the BO mutex: alias construction can hold the GEM + // reservation while acquiring this mutex. let sgt = gem.owned_sg_table()?; - - if vm_bo_guard.sg_vec.is_none() { - let mut sg_vec = KVVec::new(); - let mut offset = 0; - for range in sgt.iter() { - let addr = range.dma_address() as usize; - let len = range.dma_len() as usize; - sg_vec.push((offset, addr..(addr + len)), GFP_KERNEL)?; - offset += len; + let mut vm_bo_guard = vm_bo.inner().inner.lock(); + if vm_bo_guard.sgt.is_none() { + if vm_bo_guard.sg_vec.is_none() { + let mut sg_vec = KVVec::new(); + let mut offset = 0; + for range in sgt.iter() { + let addr = range.dma_address() as usize; + let len = range.dma_len() as usize; + sg_vec.push((offset, addr..(addr + len)), GFP_KERNEL)?; + offset += len; + } + vm_bo_guard.sg_vec.replace(sg_vec); } - vm_bo_guard.sg_vec.replace(sg_vec); + vm_bo_guard.sgt.replace(sgt); } - vm_bo_guard.sgt.replace(sgt); + // Release the mutex before a losing initializer drops its SG + // owner. The declaration order also preserves this on errors. + drop(vm_bo_guard); } core::mem::drop(vm_bo_guard); } @@ -2926,13 +2982,18 @@ impl Vm { core::mem::drop(inner); return Err(error); } + let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; + inner.page_table.prepare_map(iova..end)?; let mutation = inner.mapping_mutation(); let ret = inner.page_table.map_pages( - iova..(iova + size as u64), + iova..end, phys as PhysicalAddr, prot, false, ); + if ret.is_err() { + inner.rollback_failed_map(iova, size as u64); + } // Drop the exec_lock first, so that if map_node failed the // KernelMappingInner destructur does not deadlock. drop(mutation); @@ -3003,15 +3064,19 @@ impl Vm { if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); - self.inner.bo_unmap(&mut ctx, &bo)?; - // The close callback or deferred object list retains this GEM, so - // clearing its candidate cannot release the final reference here. - self.untrack_context_object(gem); - self.untrack_shared_object(gem); + let result = self.inner.bo_unmap(&mut ctx, &bo); + if result.is_ok() { + // The close callback or deferred object list retains this GEM, + // so clearing its candidate cannot release its final reference. + self.untrack_context_object(gem); + self.untrack_shared_object(gem); + } mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); - // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. + // BO release can drain unrelated deferred GEMs, including on an + // unmap error. Always release reservations before dropping it. core::mem::drop(inner); core::mem::drop(bo); + result?; } Ok(()) @@ -3341,6 +3406,7 @@ impl Uat { let context = Arc::new( ExecutionContext { root: ExecutionRoot { + released: AtomicBool::new(false), inner: self.inner.clone(), id: id as u8, generation, diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 89c7db4f3d2e71..67d18a275ca32e 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -59,6 +59,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + gem::validate_vm_binding(gem, self)?; let context = self.prepare_context_binding(addr, size, offset)?; let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -69,21 +70,29 @@ impl Vm { }; let vm_bo = self.inner.obtain_bo(gem)?; { - let mut bo = vm_bo.inner().inner.lock(); - if bo.sgt.is_none() { + let needs_sgt = vm_bo.inner().inner.lock().sgt.is_none(); + if needs_sgt { + // Pin without the BO mutex: alias construction can hold the GEM + // reservation while acquiring this mutex. let sgt = gem.owned_sg_table()?; - if bo.sg_vec.is_none() { - let mut segments = KVVec::new(); - let mut offset = 0; - for range in sgt.iter() { - let address = range.dma_address() as usize; - let length = range.dma_len() as usize; - segments.push((offset, address..address + length), GFP_KERNEL)?; - offset += length; + let mut bo = vm_bo.inner().inner.lock(); + if bo.sgt.is_none() { + if bo.sg_vec.is_none() { + let mut segments = KVVec::new(); + let mut offset = 0; + for range in sgt.iter() { + let address = range.dma_address() as usize; + let length = range.dma_len() as usize; + segments.push((offset, address..address + length), GFP_KERNEL)?; + offset += length; + } + bo.sg_vec = Some(segments); } - bo.sg_vec = Some(segments); + bo.sgt = Some(sgt); } - bo.sgt = Some(sgt); + // Release the mutex before a losing initializer drops its SG + // owner. The declaration order also preserves this on errors. + drop(bo); } } ctx.vm_bo = Some(vm_bo); diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 78f906ff9c46c8..e33b578bb13a73 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -520,40 +520,19 @@ impl Vm { self.unmap_user_ranges_now(user, kernel) } - fn unmap_user_ranges_now(&self, user: Range, kernel: Range) -> Result { - let driver = self - .lifetime - .as_ref() - .map(|lifetime| lifetime.driver.lock()); - let mut cursor = user.start; - while cursor < user.end { - let mut next = user.end; - let mut covered = cursor; - for range in Iterator::chain( - core::iter::once(kernel.clone()), - driver - .as_ref() - .and_then(|guard| guard.as_ref()) - .into_iter() - .flat_map(DriverMappings::ranges), - ) { - if range.contains(&cursor) { - covered = covered.max(range.end); - } else if cursor < range.start { - next = next.min(range.start); - } - } - if covered != cursor { - cursor = covered.min(user.end); - continue; - } - if next <= cursor { - return Err(EIO); - } - self.unmap_range_now(cursor, next - cursor)?; - cursor = next; - } - Ok(()) + fn unmap_user_ranges_now(&self, user: Range, _kernel: Range) -> Result { + // User mappings alone have GPUVA nodes. Kernel and driver aliases use + // mm::Node and are untouched; GPUVM's embedded kernel cutout is skipped. + // Closed admission and zero accepted jobs make whole-node teardown safe. + let mut inner = self.inner.lock_inner(); + let result = inner.unmap_all(VmInner::unmap_gpuva); + if result.is_ok() { + self.untrack_shared_range(user.start, user.end - user.start); + } + drop(inner); + // Final GEM/SG-table destruction may acquire reservations itself. + self.bo_deferred_cleanup(); + result } } diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index adbdd97beb2267..da4fc8014f8814 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -710,12 +710,18 @@ impl UatPageTable { let end = ((iova_range.end - 1) & self.geometry.root_mask()) + 1; let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + // For every child table, retain the parent page and entry that names it. + // Children are detached before their parents are freed, so these borrowed + // parent pages remain valid while the descriptor is cleared. + let mut pt_parent: [Option<(PhysicalAddr, usize)>; UAT_LEVELS - 1] = Default::default(); pt_addr[UAT_LEVELS - 1] = Some(self.ttb); 'outer: while iova < end { mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); let addr_diff = last_iova ^ iova; - for level in (0..UAT_LEVELS - 1).rev() { + // Detach leaves before intermediate tables. The old order freed a + // parent first and could leave its descriptor naming a freed child. + for level in 0..UAT_LEVELS - 1 { // If the iova has changed at this level or above, invalidate the physaddr if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { if let Some(phys) = pt_addr[level].take() { @@ -725,6 +731,10 @@ impl UatPageTable { level, phys ); + let parent = pt_parent[level].take(); + self.free_table(phys, parent)?; + } else { + pt_parent[level] = None; // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). unsafe { Page::from_phys(phys) }; } @@ -777,6 +787,9 @@ impl UatPageTable { Ok(None) } })?; + if pt_addr[level].is_some() { + pt_parent[level] = Some((phys, upidx)); + } mod_pr_debug!( "UATPageTable::with_pages: level {} PT {:#x?}\n", level, @@ -826,7 +839,9 @@ impl UatPageTable { } if free { - for level in (0..UAT_LEVELS - 1).rev() { + // The final path is detached bottom-up for the same reason as the + // boundary transitions above. + for level in 0..UAT_LEVELS - 1 { if let Some(phys) = pt_addr[level] { mod_pr_debug!( "UATPageTable::with_pages: free level {} {:#x?}\n", @@ -1009,18 +1024,50 @@ impl UatPageTable { Ok(()) /// Free a page table, unless it was not allocated by the driver. - fn free_table(&mut self, phys: PhysicalAddr) { - if let Some(tables) = self.driver_tables.as_mut() { + fn free_table( + &mut self, + phys: PhysicalAddr, + parent: Option<(PhysicalAddr, usize)>, + ) -> Result { + let owned = if let Some(tables) = self.driver_tables.as_ref() { match tables.iter().position(|&table| table == phys) { - Some(index) => { - tables.swap_remove(index); - } - None => return, + Some(_) => true, + None => false, } + } else { + true + }; + if !owned { + return Ok(()); + } + if let Some((parent_phys, parent_index)) = parent { + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; + page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let pte = unsafe { &*(p as *const AtomicU64) }; + let current = pte.load(Ordering::Acquire); + let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); + if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + return Err(EIO); + } + pte.store(0, Ordering::Release); + Ok(()) + })?; + } + if let Some(tables) = self.driver_tables.as_mut() { + let index = tables + .iter() + .position(|&table| table == phys) + .ok_or(EIO)?; + tables.swap_remove(index); } // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without // `driver_tables`, every child table belongs to the driver. unsafe { Page::from_phys(phys) }; + Ok(()) } /// Checks every page, including holes skipped by the page-table walker. @@ -1164,6 +1211,22 @@ impl UatPageTable { Ok(covered) } + /// Reserve all tables for a fixed mapping before publishing any leaves. + /// GPUVM and the fixed-address allocator may share a page table, so check + /// that neither owns a destination leaf before allowing failure rollback. + pub(crate) fn prepare_map(&mut self, range: Range) -> Result { + if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + self.with_pages(range.clone(), false, false, |_, ptes| { + if ptes.iter().any(|pte| pte.load(Ordering::Acquire) != 0) { + return Err(EBUSY); + } + Ok(()) + })?; + self.alloc_pages(range) + } + fn pte_bits(&self) -> u64 { if self.ttb_owned { // Owned page tables are userspace, so non-global diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index a1153334665f4e..7d105aa150edc0 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -325,10 +325,6 @@ impl sched::JobImpl for QueueJob::ver { // are "invisible" to the firmware. Incrementing the notifier threshold // earlier than this violates that which leads to circles like the // above. Deferring the increment to submit solves the race. - job.notifier.threshold.with(|raw, _inner| { - raw.increase(job.notification_count); - }); - let gpu = match (*job.dev) .gpu()? .legacy_manager() @@ -380,6 +376,13 @@ impl sched::JobImpl for QueueJob::ver { } } + // Prepared submissions still hold their workqueue locks and have not + // published cpu_wptr. Only count them after every fallible submit() + // succeeded, before the first run_job exposes commands to firmware. + job.notifier.threshold.with(|raw, _inner| { + raw.increase(job.notification_count); + }); + // Now we fully commit to running the job mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); frag_sub.map(|a| gpu.run_job(a)).transpose()?; @@ -710,11 +713,11 @@ impl Queue for Queue::ver { match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { - last_compute = nr_commands; + last_render = nr_commands; nr_render += 1; } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { - last_render = nr_commands; + last_compute = nr_commands; nr_compute += 1; } _ => {} diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index b123f6164206b1..92ab5271e523e9 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -2382,6 +2382,7 @@ op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, op.map.va.range = req->map.va.range; op.map.gem.obj = req->map.gem.obj; op.map.gem.offset = req->map.gem.offset; + op.map.gem.range = req->map.gem.range; op.map.flags = req->map.flags; return fn->sm_step_map(&op, priv); @@ -2438,7 +2439,7 @@ static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map * if (a->gem.obj != b->gem.obj || !a->gem.obj) return false; - if (can_merge_flags(gpuvm, a->flags, b->flags)) + if (!can_merge_flags(gpuvm, a->flags, b->flags)) return false; /* Order VAs for the rest of the checks. */ @@ -2457,7 +2458,7 @@ static bool __can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva_op_map * /* If this is a repeated mapping, both the GEM range * and offset must match. */ - if (a->gem.range != b->gem.range || + if (!a->gem.range || a->gem.range != b->gem.range || a->gem.offset != b->gem.offset) return false; @@ -2485,6 +2486,7 @@ static bool can_merge(struct drm_gpuvm *gpuvm, const struct drm_gpuva *a, .va.range = a->va.range, .gem.offset = a->gem.offset, .gem.obj = a->gem.obj, + .gem.range = a->gem.range, .flags = a->flags, }; @@ -2738,7 +2740,7 @@ __drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, next.va.addr = req_end; next.va.range = end - req_end; next.gem.obj = obj; - prev.gem.range = va->gem.range; + next.gem.range = va->gem.range; next.gem.offset = offset + (va->flags & DRM_GPUVA_REPEAT ? 0 : req_end - addr); next.flags = va->flags; diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h index aca218734dcdec..367444b74e8bab 100644 --- a/include/drm/drm_gpuvm.h +++ b/include/drm/drm_gpuvm.h @@ -1165,6 +1165,7 @@ static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, va->va.range = op->va.range; va->gem.obj = op->gem.obj; va->gem.offset = op->gem.offset; + va->gem.range = op->gem.range; } /** diff --git a/rust/kernel/drm/device.rs b/rust/kernel/drm/device.rs index 11f9f4e7bea4af..045e6444f88366 100644 --- a/rust/kernel/drm/device.rs +++ b/rust/kernel/drm/device.rs @@ -111,7 +111,7 @@ impl Device { fops: &Self::GEM_FOPS, }; - const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(); + const GEM_FOPS: bindings::file_operations = drm::gem::create_fops(T::MODULE); /// Create a new `drm::Device` for a `drm::Driver`. pub fn new(dev: &device::Device, data: impl PinInit) -> Result> { diff --git a/rust/kernel/drm/driver.rs b/rust/kernel/drm/driver.rs index 260ed453e7f5b4..ca7b519d3c5991 100644 --- a/rust/kernel/drm/driver.rs +++ b/rust/kernel/drm/driver.rs @@ -122,6 +122,10 @@ pub trait Driver { /// Driver metadata const INFO: DriverInfo; + /// Module implementing the driver callbacks. Loadable drivers should provide + /// their `THIS_MODULE` so open files and exported DMA buffers retain the code. + const MODULE: Option<&'static crate::ThisModule> = None; + /// Feature flags const FEATURES: u32; diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index f766e90d2dd7f3..bea304395d67c5 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -469,12 +469,17 @@ impl DmaBuf { } } -pub(super) const fn create_fops() -> bindings::file_operations { +pub(super) const fn create_fops( + module: Option<&'static crate::ThisModule>, +) -> bindings::file_operations { // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` // zeroed. let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() }; - fops.owner = core::ptr::null_mut(); + fops.owner = match module { + Some(module) => module.as_ptr(), + None => core::ptr::null_mut(), + }; fops.open = Some(bindings::drm_open); fops.release = Some(bindings::drm_release); fops.unlocked_ioctl = Some(bindings::drm_ioctl); diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index 3802f3e3cb92fd..04e9abe162c6fd 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -21,6 +21,7 @@ use crate::{ EINVAL, ENOMEM, // }, + from_err_ptr, from_result, to_result, Error, @@ -31,12 +32,12 @@ use crate::{ ARef, AlwaysRefCounted, // }, - types::Opaque, // + types::{NotThreadSafe, Opaque}, // }; use core::cell::UnsafeCell; use core::marker::{PhantomData, PhantomPinned}; -use core::mem::{ManuallyDrop, MaybeUninit}; +use core::mem::MaybeUninit; use core::ops::{Deref, DerefMut, Range}; use core::ptr::NonNull; use pin_init; @@ -346,7 +347,30 @@ unsafe impl AlwaysRefCounted for GpuVmBo { // must take the same lock as those operations. Keep a separate GEM // reference until after unlocking: the final vm_bo put can release // its GEM reference and otherwise free the mutex we still hold. + unsafe fn dec_ref(obj: NonNull) { + // SAFETY: This owns a live BO reference. Immediate-mode GPUVA lists + // use the GEM's gpuva mutex, not its reservation. The deferred put + // releases that mutex before final GEM destruction. Keep the GPUVM + // alive across draining, which can drop its last BO-owned reference. unsafe { + let bo = &raw mut (*obj.as_ptr()).bo; + let vm = (*bo).vm; + if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE != 0 { + bindings::drm_gpuvm_get(vm); + bindings::drm_gpuvm_bo_put_deferred(bo); + bindings::drm_gpuvm_bo_deferred_cleanup(vm); + bindings::drm_gpuvm_put(vm); + return; + } + + // A final BO put also puts its GEM. Retain the GEM until after + // unlocking its reservation, which may be embedded in that GEM. + let gem = (*bo).obj; + bindings::drm_gem_object_get(gem); + let resv = (*gem).resv; + bindings::dma_resv_lock(resv, core::ptr::null_mut()); + bindings::drm_gpuvm_bo_put(bo); + bindings::dma_resv_unlock(resv); let bo = &mut obj.as_mut().bo; let gem = bo.obj; bindings::drm_gem_object_get(gem); @@ -578,6 +602,25 @@ impl GpuVm { Ok(vm_ref) } + /// Lock only the VM's driver data, without allocating an exec context. + /// + /// This takes the same shared reservation as `exec_lock`, but does not lock + /// external BO reservations. The returned guard exposes only the inner + /// driver data, not GPUVA operations requiring those other reservations. + /// Callers must not already hold a reservation lock. + pub fn lock_inner(&self) -> GpuVmInnerGuard<'_, T> { + // SAFETY: The borrowed GPUVM retains r_obj and its reservation for the + // guard's lifetime. A non-interruptible single-reservation lock with no + // ww context cannot fail or allocate. No other reservation is acquired. + unsafe { + bindings::dma_resv_lock((*(*self.gpuvm()).r_obj).resv, core::ptr::null_mut()); + } + GpuVmInnerGuard { + gpuvm: self, + _not_send: NotThreadSafe, + } + } + pub fn exec_lock<'a, 'b>( &'a self, obj: Option<&'b Object>, @@ -586,32 +629,35 @@ impl GpuVm { // Do not try to lock the object if it is internal (since it is already locked). let is_ext = obj.map(|a| self.is_extobj(a)).unwrap_or(false); - let mut guard = ManuallyDrop::new(LockedGpuVm { - gpuvm: self, - // vm_exec needs to be pinned, so stick it in a Box. - vm_exec: KBox::init( - init!(bindings::drm_gpuvm_exec { - vm: self.gpuvm() as *mut _, - flags: if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 + // vm_exec needs a stable address while locking. Construct the unlock + // guard only after success; C already finalizes the exec on error. + let mut vm_exec = KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }, + // SAFETY: bindgen's `Default` for this C structure is exactly + // `MaybeUninit::zeroed().assume_init()`. Spell that out here so + // external modules do not import its unexported trait method. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, + extra: match (is_ext, obj) { + (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { + fn_: Some(exec_lock_gem_object), + priv_: obj.as_raw() as *const _ as *mut _, }, - // SAFETY: bindgen's `Default` for this C structure is exactly - // `MaybeUninit::zeroed().assume_init()`. Spell that out here so - // external modules do not import its unexported trait method. - exec: unsafe { MaybeUninit::::zeroed().assume_init() }, - extra: match (is_ext, obj) { - (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { - fn_: Some(exec_lock_gem_object), - priv_: obj.as_raw() as *const _ as *mut _, - }, - // SAFETY: as above, this bindgen C structure is zero-valid. - _ => unsafe { - MaybeUninit::::zeroed() - .assume_init() - }, + // SAFETY: as above, this bindgen C structure is zero-valid. + _ => unsafe { + MaybeUninit::::zeroed() + .assume_init() }, + }, + num_fences: 0, + }), + GFP_KERNEL, + )?; num_fences: 0, }), GFP_KERNEL, @@ -620,9 +666,13 @@ impl GpuVm { }); // SAFETY: The object is valid and was initialized above - to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *guard.vm_exec) })?; + to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *vm_exec) })?; - Ok(ManuallyDrop::into_inner(guard)) + Ok(LockedGpuVm { + gpuvm: self, + vm_exec, + objects: LockedObjects::Single(obj), + }) } /// Lock the GPUVM and an array of additional GEM objects in one @@ -638,44 +688,45 @@ impl GpuVm { raw_objects.push(object.as_raw(), GFP_KERNEL)?; } - let mut guard = ManuallyDrop::new(LockedGpuVm { - gpuvm: self, - // vm_exec needs to be pinned, so stick it in a Box. - vm_exec: KBox::init( - init!(bindings::drm_gpuvm_exec { - vm: self.gpuvm() as *mut _, - flags: (if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 - }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, - // SAFETY: both bindgen C structures are zero-valid; their generated - // `Default` methods perform the same zero initialization but are not - // exported from the kernel crate to loadable modules. - exec: unsafe { MaybeUninit::::zeroed().assume_init() }, - extra: unsafe { - MaybeUninit::::zeroed() - .assume_init() - }, - num_fences: 0, - }), - GFP_KERNEL, - )?, - objects: LockedObjects::Array(objects), - }); + // vm_exec needs a stable address while locking. Construct the unlock + // guard only after success; C already finalizes the exec on error. + let mut vm_exec = KBox::init( + init!(bindings::drm_gpuvm_exec { + vm: self.gpuvm() as *mut _, + flags: (if interruptible { + bindings::DRM_EXEC_INTERRUPTIBLE_WAIT + } else { + 0 + }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, + // SAFETY: both bindgen C structures are zero-valid; their generated + // `Default` methods perform the same zero initialization but are not + // exported from the kernel crate to loadable modules. + exec: unsafe { MaybeUninit::::zeroed().assume_init() }, + extra: unsafe { + MaybeUninit::::zeroed() + .assume_init() + }, + num_fences: 0, + }), + GFP_KERNEL, + )?; // SAFETY: Every raw pointer is backed by an ARef in `objects`, which // outlives the returned guard. The helper consumes the temporary // pointer array before returning and keeps its own object references. to_result(unsafe { bindings::drm_gpuvm_exec_lock_array( - &mut *guard.vm_exec, + &mut *vm_exec, raw_objects.as_mut_ptr(), count, ) })?; - Ok(ManuallyDrop::into_inner(guard)) + Ok(LockedGpuVm { + gpuvm: self, + vm_exec, + objects: LockedObjects::Array(objects), + }) } /// Returns true if the given object is external to the GPUVM @@ -718,6 +769,7 @@ impl GpuVm { ) }; obj.unlock_gpuva(); + let p = from_err_ptr(p)?; if p.is_null() { Err(ENOMEM) } else { @@ -752,6 +804,103 @@ unsafe impl AlwaysRefCounted for GpuVm { } } +enum LockedObjects<'a, T: DriverGpuVm> { + Single(Option<&'a Object>), + Array(&'a [ARef>]), +} + +impl LockedObjects<'_, T> { + fn single(&self) -> Option<&Object> { + match self { + Self::Single(object) => *object, + Self::Array(_) => None, + } + } + + fn indexed(&self, index: usize) -> Option<&Object> { + match self { + Self::Array(objects) => objects.get(index).map(|object| &**object), + Self::Single(_) => None, + } + } +} + +/// Access to VM driver data under its shared reservation, without external BO locks. +pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + // Reservation locks must be released by the task that acquired them. + _not_send: NotThreadSafe, +} + +impl GpuVmInnerGuard<'_, T> { + /// Remove every driver GPUVA in an immediate-mode VM, preserving its kernel cutout. + /// + /// `unmap` must remove the GPU translation for each complete GPUVA before + /// returning success. Only the VM's shared reservation is held. No external + /// BO reservations are acquired and no split nodes or operation lists are + /// allocated. The callback cannot retain the borrowed GPUVA. + /// + /// Call `GpuVm::bo_deferred_cleanup` after releasing this guard, including + /// when a callback fails after earlier mappings have already been removed. + pub fn unmap_all( + &mut self, + mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + // SAFETY: The guard owns this live VM's shared reservation. Immediate + // mode lets unlink take the GEM GPUVA mutex independently of its resv. + unsafe { + if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + let head = &raw mut (*vm).rb.list; + let mut entry = (*head).next; + while entry != head { + let va = crate::container_of!(entry, bindings::drm_gpuva, rb.entry) as *mut _; + // Save the successor before removing and freeing this GPUVA. + entry = (*entry).next; + // This node is embedded in the C GPUVM, not a driver GpuVa. + if core::ptr::eq(va, &raw const (*vm).kernel_alloc_node) { + continue; + } + let driver_va = crate::container_of!(va, GpuVa, gpuva); + unmap(&mut *self.gpuvm.inner.get(), &*driver_va)?; + let mut op = OpUnMap( + bindings::drm_gpuva_op_unmap { va, keep: false }, + PhantomData::, + ); + drop(op.unmap_and_unlink_va_defer()); + } + } + Ok(()) + } +} + +impl Deref for GpuVmInnerGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + // SAFETY: The shared reservation excludes every other inner-data guard, + // including LockedGpuVm and its UpdatingGpuVm callbacks. + unsafe { &*self.gpuvm.inner.get() } + } +} + +impl DerefMut for GpuVmInnerGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: As above, with exclusive access to this guard. + unsafe { &mut *self.gpuvm.inner.get() } + } +} + +impl Drop for GpuVmInnerGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: This non-Send guard owns the reservation on this task, and + // its GPUVM borrow keeps r_obj alive until after the unlock. + unsafe { bindings::dma_resv_unlock((*(*self.gpuvm.gpuvm()).r_obj).resv) }; + } +} + pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { gpuvm: &'a GpuVm, vm_exec: KBox, @@ -788,6 +937,7 @@ impl LockedGpuVm<'_, '_, T> { obj.as_raw() as *const _ as *mut _, ) }; + let p = from_err_ptr(p)?; if p.is_null() { Err(ENOMEM) } else { diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index be95793cd9afbc..bb86c35bdda056 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -44,6 +44,10 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// Module implementing the callbacks. Its caller must retain the module + /// through entity destruction; the core pins detached kill-job callbacks. + const MODULE: Option<&'static crate::ThisModule> = None; + /// Handle a scoped timeout without resetting the C scheduler. /// /// The backend must settle/quarantine its exact hardware work, or retain @@ -477,6 +481,10 @@ impl Scheduler { timedout_job: Some(timedout_job_cb::), free_job: Some(free_job_cb::), cancel_job: Some(cancel_job_cb::), + owner: match T::MODULE { + Some(module) => module.as_ptr(), + None => core::ptr::null_mut(), + }, owner: core::ptr::null_mut(), retain_job_on_enodev: false, }; From 4499a3c458123f5a481e1f03bb63d853970e9bcb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 6 Oct 2026 23:36:52 -0400 Subject: [PATCH 069/631] drm/asahi: order render timestamp publication independently of GPU retirement Physical render pairs can be observed out of logical order. When a command waits through an earlier render timestamp producer, defer its host writes and public completion until that producer's host publication has finished. Preallocate forward-only publication links before accepting the job and install them before early publication or scheduler push. Process ready successors iteratively without allocating or waiting in completion. Keep scheduler credits independent: commands with host predecessors use a separate terminal fence as their scheduler parent. Healthy retirement still releases physical resources immediately, while failed-owned commands keep their pins until a retirement witness. Cancellation and physical dependency snapshots use that terminal fence. Ordinary scheduled-fence readiness and compute timestamp dependencies retain their existing behavior. Fifteen extracted-production host tests pass, including the original reverse observation failure, allocation/cancellation paths, racing registration and completion, and a 12000-node chain on a 64 KiB host stack. Full AArch64 16K Image/modules/dtbs builds with 1591 modules and 1805 DTBs. Test harnesses and receipts remain outside the source tree. Hardware and performance validation are still outstanding; no queue-depth reduction or GPU serialization is added. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d9b7b82975e2dad4b9a5941ade2f4debadd61c6c) --- drivers/gpu/drm/asahi/g17/completion.rs | 123 ++++++++--- .../drm/asahi/g17/completion/publication.rs | 209 ++++++++++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 15 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 17 +- .../asahi/g17/queue/render/dependencies.rs | 2 +- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 +- .../drm/asahi/g17/runtime/render/settle.rs | 2 +- 7 files changed, 337 insertions(+), 35 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/completion/publication.rs diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index f137a7e3434de3..0c877743873d15 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -23,6 +23,9 @@ use kernel::{ uapi, xarray, }; +mod publication; +pub(crate) use publication::{Prepared as PreparedPublication, Publication}; + struct Destination { // Keep the existing GPU alias alive along with its prepared CPU view. _mapping: Arc, @@ -51,7 +54,7 @@ impl Destination { Ok(Some(Self { _mapping: mapping, cpu })) } - fn write(&self, ticks: u64) -> Result { + fn write(&self, ticks: u64) { const NSEC_PER_SEC: u64 = 1_000_000_000; let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) .wrapping_mul(NSEC_PER_SEC) @@ -60,7 +63,6 @@ impl Destination { ); self.cpu.write(ns); barrier(Ordering::Release); - Ok(()) } } @@ -97,6 +99,51 @@ impl Destinations { } } +/// Public outputs do not own physical command resources or a scheduler job. +struct Output { + fence: UserFence, + member: Member, + destinations: Destinations, +} + +impl Output { + fn publish(&self, result: Result<[u64; 4]>) { + match result { + Ok(ticks) => { + for (destination, ticks) in self.destinations.iter().zip(ticks) { + if let Some(destination) = destination { + destination.write(ticks); + } + } + } + Err(error) => self.fence.set_error(error), + } + self.fence.signal(); + self.member.complete(result.map(|_| ())); + } +} + +enum HostOutput { + Immediate(Output), + Ordered(Arc), +} + +impl HostOutput { + fn output(&self) -> &Output { + match self { + Self::Immediate(output) => output, + Self::Ordered(publication) => publication.output(), + } + } + + fn finish(&self, result: Result<[u64; 4]>) { + match self { + Self::Immediate(output) => output.publish(result), + Self::Ordered(publication) => publication.finish(result), + } + } +} + /// Firmware writes two aligned words in a fresh write-combined page per compute command. struct ComputeTimestamps { // Remove the GPU alias before releasing its backing. @@ -120,9 +167,10 @@ impl ComputeTimestamps { #[pin_data] pub(crate) struct Completion { status: Arc, - fence: UserFence, - member: Member, - destinations: Destinations, + output: HostOutput, + // Present only when this command's public result may wait for a render + // predecessor's host writes. DRM credits must follow the terminal result. + terminal: Option>, compute: Option, feed: Option>, #[pin] @@ -148,9 +196,23 @@ impl Completion { vm_job: mmu::VmJobGuard, destinations: Destinations, compute: bool, + host_predecessors: bool, feed: Option>, ) -> Result> { let fence = fence::independent(contexts)?; + let ordered = !compute + && (host_predecessors || destinations.iter().any(Option::is_some)); + let terminal = if host_predecessors { + Some(fence::independent(contexts)?) + } else { + None + }; + let output = Output { fence, member, destinations }; + let output = if ordered { + HostOutput::Ordered(Publication::new(output, context.status().clone())?) + } else { + HostOutput::Immediate(output) + }; let compute = if compute { Some(ComputeTimestamps::new(dev, context.vm())?) } else { @@ -159,9 +221,8 @@ impl Completion { Arc::pin_init( pin_init!(Self { status: context.status().clone(), - fence, - member, - destinations, + output, + terminal, compute, feed, vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), @@ -180,7 +241,24 @@ impl Completion { } pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) + Fence::from_fence(&self.output.output().fence) + } + pub(crate) fn scheduler_fence(&self) -> Fence { + self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) + } + pub(crate) fn publication(&self) -> Option> { + match &self.output { + HostOutput::Ordered(publication) => Some(publication.clone()), + HostOutput::Immediate(_) => None, + } + } + fn signal_terminal(&self, result: Result<[u64; 4]>) { + if let Some(fence) = &self.terminal { + if let Err(error) = result { + fence.set_error(error); + } + fence.signal(); + } } /// All fallible scheduler setup has succeeded. Failures from this point /// must be visible through VM_STATUS before the accepted job's fence. @@ -191,10 +269,10 @@ impl Completion { &self.status } pub(crate) fn work_state(&self) -> Result> { - self.member.work_state() + self.output.output().member.work_state() } pub(crate) fn has_timestamps(&self) -> bool { - self.destinations.iter().any(Option::is_some) + self.output.output().destinations.iter().any(Option::is_some) } pub(crate) fn compute_timestamp_va(&self) -> Result { @@ -360,25 +438,16 @@ impl Completion { if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { feed.note_render_pass(ticks[2], ticks[3]); } - let result = result.and_then(|ticks| { - for (destination, ticks) in self.destinations.iter().zip(ticks) { - if let Some(destination) = destination { - destination.write(ticks)?; - } - } - Ok(()) - }); if let Err(error) = result { if self.spared() { self.status.report_failure(error); } else if error != ECANCELED { self.status.record(error); } - self.fence.set_error(error); } self.release_healthy(); - self.fence.signal(); - self.member.complete(result); + self.signal_terminal(result); + self.output.finish(result); } /// A rejected, never-published command cannot poison the VM for contention. @@ -390,10 +459,9 @@ impl Completion { if self.accepted.load(Ordering::Acquire) { self.status.report_failure(error); } - self.fence.set_error(error); self.release_healthy(); - self.fence.signal(); - self.member.complete(Err(error)); + self.signal_terminal(Err(error)); + self.output.finish(Err(error)); } /// The physical queue must already be quarantined. Keep mappings pinned while @@ -410,9 +478,8 @@ impl Completion { } else if error != ECANCELED { self.status.record(error); } - self.fence.set_error(error); - self.fence.signal(); - self.member.complete(Err(error)); + self.signal_terminal(Err(error)); + self.output.finish(Err(error)); } } diff --git a/drivers/gpu/drm/asahi/g17/completion/publication.rs b/drivers/gpu/drm/asahi/g17/completion/publication.rs new file mode 100644 index 00000000000000..a1e05a9408552c --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/completion/publication.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Host-only ordering of timestamp writes, independent of scheduler credits. +//! +//! Links point from earlier logical render ordinals to later ones. They retain +//! only public output state, never a packet, physical render lease or VM job pin. +//! All link storage is allocated before acceptance. Completion neither allocates +//! nor waits for another command; the last prerequisite publishes a ready result. + +use super::Output; +use crate::g17::{dependency::inherited_error, status::VmStatus}; +use kernel::{new_mutex, prelude::*, sync::{Arc, Mutex}}; + +type Links = Option>; + +struct Link { + target: Arc, + result: Result, + next: Links, + // Reuse an already-consumed link as a traversal frame instead of recursing + // or allocating when publication unlocks a long chain of ready successors. + resume: Links, +} + +struct State { + remaining: usize, + result: Option>, + predecessor_error: Option, + claimed: bool, + published: Option, + successors: Links, +} + +impl State { + fn claim(&mut self) -> Option> { + if self.remaining != 0 || self.claimed { + return None; + } + let result = self.result.take()?; + self.claimed = true; + Some(match (result, self.predecessor_error) { + (Ok(_), Some(error)) => Err(error), + (result, _) => result, + }) + } +} + +#[pin_data] +pub(crate) struct Publication { + output: Output, + status: Arc, + #[pin] + state: Mutex, +} + +impl Publication { + pub(super) fn new(output: Output, status: Arc) -> Result> { + Arc::pin_init( + pin_init!(Self { + output, + status, + state <- new_mutex!(State { + // Acceptance installs all links before releasing this sentinel. + remaining: 1, + result: None, + predecessor_error: None, + claimed: false, + published: None, + successors: None, + }, "G17 host timestamp publication"), + }), + GFP_KERNEL, + ) + } + + pub(super) fn output(&self) -> &Output { + &self.output + } + + /// One terminal result, offered separately from its physical-retirement proof. + pub(super) fn finish(&self, result: Result<[u64; 4]>) { + let ready = { + let mut state = self.state.lock(); + if state.claimed || state.result.is_some() { + return; + } + state.result = Some(result); + state.claim() + }; + if let Some(result) = ready { + Self::drain(self.publish(result)); + } + } + + fn publish(&self, result: Result<[u64; 4]>) -> Links { + if let Err(error) = result { + // Inherited failure is visible to the client without inventing a + // permanent admission failure in this command's VM. + self.status.report_failure(error); + } + self.output.publish(result); + let result = result.map(|_| ()); + let mut links = { + let mut state = self.state.lock(); + // A late link is acknowledged only after timestamp writes, the public + // fence and aggregate membership have all been published off this lock. + state.published = Some(result); + state.successors.take() + }; + let mut next = links.as_mut(); + while let Some(link) = next { + link.result = result; + next = link.next.as_mut(); + } + links + } + + fn predecessor_done(&self, result: Result) -> Links { + let ready = { + let mut state = self.state.lock(); + debug_assert!(state.remaining != 0); + if let Err(error) = result { + if state.predecessor_error.is_none() { + state.predecessor_error = Some(inherited_error(error)); + } + } + state.remaining -= 1; + state.claim() + }; + ready.and_then(|result| self.publish(result)) + } + + fn add_successor(&self, mut link: KBox) { + let published = { + let mut state = self.state.lock(); + match state.published { + Some(result) => result, + None => { + link.next = state.successors.take(); + state.successors = Some(link); + return; + } + } + }; + Self::drain(link.target.predecessor_done(published)); + } + + /// Iterative depth-first notification, O(nodes + links), with no stack growth. + fn drain(mut pending: Links) { + let mut frames: Links = None; + loop { + if let Some(mut link) = pending.take() { + pending = link.next.take(); + let children = link.target.predecessor_done(link.result); + if children.is_some() { + link.resume = pending.take(); + link.next = frames.take(); + frames = Some(link); + pending = children; + } + } else if let Some(mut frame) = frames.take() { + frames = frame.next.take(); + pending = frame.resume.take(); + } else { + break; + } + } + } +} + +/// Temporary backwards references exist only during fallible setup. Installed +/// links retain the successor only, so closing a queue cannot form an Arc cycle. +pub(crate) struct Prepared { + target: Arc, + links: KVec<(Arc, KBox)>, +} + +impl Prepared { + pub(crate) fn new<'a>( + target: Arc, + predecessors: impl Iterator> + Clone, + ) -> Result { + let mut links = KVec::with_capacity(predecessors.clone().count(), GFP_KERNEL)?; + for predecessor in predecessors { + let link = KBox::new(Link { + target: target.clone(), + result: Ok(()), + next: None, + resume: None, + }, GFP_KERNEL)?; + links.push((predecessor.clone(), link), GFP_KERNEL)?; + } + Ok(Self { target, links }) + } + + /// Called after all fallible scheduler setup and before any GPU publication. + pub(crate) fn activate(self) { + { + let mut state = self.target.state.lock(); + debug_assert!(state.remaining == 1 && !state.claimed); + // The fully allocated link vector bounds this count below usize::MAX. + state.remaining += self.links.len(); + } + for (predecessor, link) in self.links { + predecessor.add_successor(link); + } + Publication::drain(self.target.predecessor_done(Ok(()))); + } +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b25db8e00f1672..92123312bc9322 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -316,6 +316,7 @@ pub(crate) struct Scheduler { pub(crate) struct Fences { pub(crate) ready: Fence, pub(crate) completed: Fence, + pub(crate) publication: Option>, } impl Scheduler { @@ -338,9 +339,18 @@ impl Scheduler { packet: Arc, dependencies: KVec, early: bool, + publication: Option, ) -> Result { let completed = packet.completion.fence(); let host_timestamps = packet.completion.has_timestamps(); + // Only commands which actually write timestamps become host-prefix + // producers. Ordinary consumers retain the scheduled ready fence and + // their existing firmware dependency path for cross-engine overlap. + let host_publication = if host_timestamps { + packet.completion.publication() + } else { + None + }; let mut job = self.entity.new_job( 1, Job { @@ -354,6 +364,9 @@ impl Scheduler { } let mut job = job.arm(); packet.completion.mark_accepted(); + if let Some(publication) = publication { + publication.activate(); + } let ready = if host_timestamps { completed.clone() } else { @@ -365,6 +378,6 @@ impl Scheduler { self.backend.count_backlog(&packet); } job.push(); - Ok(Fences { ready, completed }) + Ok(Fences { ready, completed, publication: host_publication }) } } diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 78765523254b66..5f535a8321cf4b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -8,7 +8,7 @@ pub(crate) mod render; use super::{ command::{Parser, UscWindow, Validated}, - completion::{Completion, Destinations}, + completion::{Completion, Destinations, PreparedPublication}, context::Context, fence::{Outputs, Submission}, job::{self, Backend, Engine, Fences, Order, Packet, Scheduler}, @@ -176,6 +176,15 @@ impl Queue { if engine == Engine::Compute { self.ensure_compute()?; } + // Only earlier render ordinals covered by the explicit logical prefix + // can delay host publication. These links never become GPU dependencies. + let host_predecessors = self.frontiers[index] + .iter() + .take_while(|(sequence, _)| { + engine == Engine::Render + && Order::contains(order.wait_through[index], *sequence) + }) + .filter_map(|(_, fences)| fences.publication.as_ref()); let completion = Completion::new( &self.dev, &self.fences, @@ -184,8 +193,12 @@ impl Queue { guard, destinations, engine == Engine::Compute, + host_predecessors.clone().next().is_some(), if engine == Engine::Render { self.backend.feed() } else { None }, )?; + let publication = completion.publication() + .map(|target| PreparedPublication::new(target, host_predecessors)) + .transpose()?; if engine == Engine::Compute { self.prune_frontier(Engine::Render.index()); } @@ -275,7 +288,7 @@ impl Queue { let fences = scheduler .as_mut() .ok_or(ENODEV)? - .enqueue(packet, dependencies, early)?; + .enqueue(packet, dependencies, early, publication)?; self.sequence[index] = order.sequence; self.frontiers[index] .push_within_capacity((order.sequence, fences)) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index d1ec0eb76c7e18..be1d9fc9c902ce 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -40,7 +40,7 @@ fn include( frontier.include( prefix, packet.order.sequence, - job::fence_status(&packet.completion.fence()), + job::fence_status(&packet.completion.scheduler_fence()), word, ) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 3fb40779021e54..7f204b9328ffb7 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -41,7 +41,7 @@ impl Backend { Err(error) } else { deferred.finish(); - Ok(packet.completion.fence()) + Ok(packet.completion.scheduler_fence()) } } @@ -115,7 +115,7 @@ impl job::Backend for Backend { fn publish(&self, packet: Arc) -> Result { if let Some(result) = packet.early_result() { - return result.map(|()| packet.completion.fence()); + return result.map(|()| packet.completion.scheduler_fence()); } let mut deferred = match DeferredBatch::publication() { Ok(batch) => batch, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 2cf7a5d77ee5a7..0a4634bccaea52 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -26,7 +26,7 @@ impl Backend { result } pub(in crate::g17) fn cancel_render(&self, packet: &Arc) -> Result { - let fence = packet.completion.fence(); + let fence = packet.completion.scheduler_fence(); // SAFETY: The retained fence remains alive across this bounded, // uninterruptible queue-teardown wait, without holding a device lock. let completed = unsafe { From f7112f9ecafd3b0048039f6fae82cfa27ac74cec Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:07 -0400 Subject: [PATCH 070/631] drm/sched: separate Rust job timeout state from submission state drm_sched_job_begin() arms the timeout before run_job() returns, so the timeout worker can run while the submit worker still owns the job. Both Rust JobImpl callbacks exclusively borrowed the whole Job. Give the timeout worker its own TimeoutData field, pass the finished status explicitly, and keep run/prepare/cancel on the submission half only. Let a backend that returns ENODEV without reset recovery keep the detached job owned by the scheduler (retain_job_on_enodev) so it reaches normal free work after its parent completes instead of being orphaned. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e22376a0ca679b5359fdef009b2479616619677b) --- drivers/gpu/drm/asahi/g17/job.rs | 29 +++++++++++++------ drivers/gpu/drm/asahi/queue/mod.rs | 28 +++++++++++------- rust/kernel/drm/sched.rs | 46 ++++++++++++++++++++---------- 3 files changed, 70 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 92123312bc9322..34b6308620644c 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -255,6 +255,11 @@ pub(crate) trait Backend: Send + Sync { } struct Job { + backend: Arc, + packet: Arc, +} + +struct Timeout { backend: Arc, victim_renewals: u8, packet: Arc, @@ -272,10 +277,19 @@ impl Drop for Job { } impl sched::JobImpl for Job { + type TimeoutData = Timeout; const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - fn false_timeout(job: &mut sched::Job) -> bool { - if job.is_finished() || job.packet.completion.take_replay_timeout() { + fn timeout_data(&self) -> Timeout { + Timeout { + backend: self.backend.clone(), + victim_renewals: 0, + packet: self.packet.clone(), + } + } + + fn false_timeout(job: &mut Timeout, finished: bool) -> bool { + if finished || job.packet.completion.take_replay_timeout() { return true; } let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); @@ -285,22 +299,22 @@ impl sched::JobImpl for Job { deferred } - fn prepare(job: &mut sched::Job) -> Option { + fn prepare(job: &mut Self) -> Option { job.backend.prepare(&job.packet) } - fn run(job: &mut sched::Job) -> Result> { + fn run(job: &mut Self) -> Result> { job.backend.publish(job.packet.clone()).map(Some) } - fn timed_out(job: &mut sched::Job) -> sched::Status { + fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { // The scheduler retains the detached job throughout this callback. Keep // its parent callback intact while the driver settles the exact owner; // reinsertion also handles completion racing with timeout observation. - if !job.is_finished() { + if !finished { job.backend.timed_out(job.packet.clone()); } sched::Status::NoHang } - fn cancel(job: &mut sched::Job) { + fn cancel(job: &mut Self) { job.backend.cancel(job.packet.clone()); } } @@ -355,7 +369,6 @@ impl Scheduler { 1, Job { backend: self.backend.clone(), - victim_renewals: 0, packet: packet.clone(), }, )?; diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 7d105aa150edc0..46c55e22b74dc0 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -193,7 +193,11 @@ pub(crate) struct QueueJob { fence: UserFence, notifier: Arc>, notification_count: u32, - did_run: bool, + id: u64, +} + +pub(crate) struct QueueTimeout { + dev: AsahiDevRef, id: u64, } @@ -241,7 +245,14 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { - fn prepare(job: &mut sched::Job) -> Option { + type TimeoutData = QueueTimeout; + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + + fn timeout_data(&self) -> QueueTimeout { + QueueTimeout { dev: self.dev.clone(), id: self.id } + } + + fn prepare(job: &mut Self) -> Option { mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); if let Some(sj) = job.sj_vtx.as_ref() { @@ -278,7 +289,7 @@ impl sched::JobImpl for QueueJob::ver { } #[allow(unused_assignments)] - fn run(job: &mut sched::Job) -> Result> { + fn run(job: &mut Self) -> Result> { mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); // We can only increase the notifier threshold here, now that we are @@ -400,22 +411,20 @@ impl sched::JobImpl for QueueJob::ver { mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); core::mem::drop(frag_job); - job.did_run = true; - Ok(Some(Fence::from_fence(&job.fence))) } - fn timed_out(job: &mut sched::Job) -> sched::Status { + fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { // FIXME: Handle timeouts properly dev_err!( job.dev.as_ref(), - "QueueJob {}: Job timed out on the DRM scheduler, things will probably break (ran: {})\n", - job.id, job.did_run + "QueueJob {}: Job timed out on the DRM scheduler, things will probably break\n", + job.id ); sched::Status::NoDevice } - fn cancel(job: &mut sched::Job) { + fn cancel(job: &mut Self) { dev_info!( job.dev.as_ref(), "QueueJob {}: Job canceled on DRM scheduler teardown\n", @@ -751,7 +760,6 @@ impl Queue for Queue::ver { // notification count for the job. notification_count: (2 * nr_render) + nr_compute, - did_run: false, id, }, )?; diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index bb86c35bdda056..965800645091ab 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -44,6 +44,13 @@ pub enum Priority { /// Trait to be implemented by driver job objects. pub trait JobImpl: Sized { + /// State owned exclusively by the timeout worker. It must not borrow the + /// mutable submission state: timeout can run before `run` returns. + type TimeoutData; + + /// Snapshot or independently retain timeout state before scheduler ownership. + fn timeout_data(&self) -> Self::TimeoutData; + /// Module implementing the callbacks. Its caller must retain the module /// through entity destruction; the core pins detached kill-job callbacks. const MODULE: Option<&'static crate::ThisModule> = None; @@ -60,28 +67,28 @@ pub trait JobImpl: Sized { /// Reconsider a timeout before stopping the scheduler or settling fences. /// Returning true asks the core to reinsert the retained job and rearm. /// The backend must not stop/start the scheduler from this hook. - fn false_timeout(_job: &mut Job) -> bool { false } + fn false_timeout(_data: &mut Self::TimeoutData, _finished: bool) -> bool { false } /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. - fn prepare(_job: &mut Job) -> Option { + fn prepare(_job: &mut Self) -> Option { None // Equivalent to NULL function pointer } /// Called to execute the job once all of the dependencies have been resolved. This may be /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides /// to try it again. - fn run(job: &mut Job) -> Result>; + fn run(job: &mut Self) -> Result>; /// Called when a job has taken too long to execute, to trigger GPU recovery. /// /// This method is called in a workqueue context. - fn timed_out(job: &mut Job) -> Status; + fn timed_out(data: &mut Self::TimeoutData, finished: bool) -> Status; /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences /// get signalled before the scheduler is torn down. - fn cancel(job: &mut Job); + fn cancel(job: &mut Self); } unsafe extern "C" fn prepare_job_cb( @@ -91,8 +98,9 @@ unsafe extern "C" fn prepare_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. - match T::prepare(unsafe { &mut *p }) { + // SAFETY: prepare/run are serialized by submit work. Only borrow inner; + // the timeout worker independently owns timeout_data and C owns job. + match T::prepare(unsafe { &mut *addr_of_mut!((*p).inner) }) { None => core::ptr::null_mut(), Some(fence) => fence.into_raw(), } @@ -104,8 +112,9 @@ unsafe extern "C" fn run_job_cb( // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - // SAFETY: All of our jobs are Job. - match T::run(unsafe { &mut *p }) { + // SAFETY: submit work owns inner until return, even while timeout work + // accesses the disjoint timeout_data field. Do not borrow the whole Job. + match T::run(unsafe { &mut *addr_of_mut!((*p).inner) }) { Err(e) => e.to_ptr(), Ok(None) => core::ptr::null_mut(), Ok(Some(fence)) => fence.into_raw(), @@ -120,17 +129,20 @@ unsafe extern "C" fn timedout_job_cb( // The C timeout worker owns this job outside pending_list until return. // Free work cannot release it even if its parent signals concurrently. + let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; + let is_finished = || unsafe { bindings::dma_fence_get_status(finished) != 0 }; if T::MANAGED_TIMEOUT_RECOVERY { - let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; - if unsafe { bindings::dma_fence_get_status(finished) } != 0 { + if is_finished() { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } } - // SAFETY: the C core retains the detached job across both callbacks. - if T::false_timeout(unsafe { &mut *p }) { + // SAFETY: the C core retains the detached job and serializes its timeout + // worker. This field is disjoint from inner, which run may still mutate. + let data = unsafe { &mut *addr_of_mut!((*p).timeout_data) }; + if T::false_timeout(data, is_finished()) { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } - let status = T::timed_out(unsafe { &mut *p }); + let status = T::timed_out(data, is_finished()); if T::MANAGED_TIMEOUT_RECOVERY { // This is scoped fence handling, not C reset recovery. stop would // detach and discard healthy parents; start would cancel those jobs. @@ -177,7 +189,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch // SAFETY: cancellation cannot race a scheduler callback using this job // after the removal/parent-lock synchronization above. The backend must // retain any still-hardware-owned resources independently of Job storage. - T::cancel(unsafe { &mut *p }); + T::cancel(unsafe { &mut *addr_of_mut!((*p).inner) }); if finish_here { // Match drm_sched_job_done's accounting exactly once, only when its @@ -203,6 +215,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch pub struct Job { job: bindings::drm_sched_job, inner: T, + timeout_data: T::TimeoutData, // Entity destruction can leave kill_jobs_work queued on the system // workqueue or waiting for a dependency. That work still dereferences // job->sched->ops after drm_sched_fini has stopped the scheduler workers. @@ -411,6 +424,7 @@ impl Entity { // The exclusive entity borrow prevents allocation/push interleaving. fn prepare_job(&self, credits: u32, inner: T) -> Result> { let mut job: KBox>> = Box::new_uninit(GFP_KERNEL | __GFP_ZERO)?; + let timeout_data = inner.timeout_data(); // SAFETY: We hold a reference to the entity (which is a valid pointer), // and the job object was just allocated above. @@ -426,6 +440,8 @@ impl Entity { // SAFETY: The Box pointer is valid, and this initializes the inner member. unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; + // SAFETY: initialize the independent timeout worker's owned state. + unsafe { addr_of_mut!((*job.as_mut_ptr()).timeout_data).write(timeout_data) }; // SAFETY: The job allocation is valid and this initializes its last // field. Arc::clone is infallible and the entity still owns the live From 3fd8c4f051debaba0533bedc660122de73051775 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:08 -0400 Subject: [PATCH 071/631] drm/asahi: observe completed render work before a deferred VM failure A VM failure that arrives while preparation owns a render pair is applied later from the deferred path, which quarantined the pair at once. Retire its completed commands first, as settle_render_vm() does, before quarantining what is still owned. Keep the pending error if the observation itself fails. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3e464649d37234b0f47dfe3c22d3f73cbb6ef45b) --- drivers/gpu/drm/asahi/g17/runtime/render/submit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 59a54b19263757..8b6c647668af4a 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -206,6 +206,20 @@ impl crate::g17::Firmware { slot: u8, deferred: &mut DeferredBatch, ) -> Result { + let observe_before_failure = { + let entry = self.queues.render.entry_mut(slot)?; + let pair = entry.pair.as_ref().ok_or(EIO)?; + entry.pending_error.as_ref().is_some_and(|(status, _)| { + Arc::ptr_eq(pair.context().status(), status) + }) + }; + if observe_before_failure { + // VM failure could not observe this pair while preparation owned + // it. Restore the same final-observation boundary used by + // settle_render_vm before quarantining any still-owned commands. + // Keep pending_error intact if observation itself fails. + self.retire_render_slot(usize::from(slot), None, deferred)?; + } let entry = self.queues.render.entry_mut(slot)?; let pair = entry.pair.as_mut().ok_or(EIO)?; if let Some((status, error)) = entry.pending_error.take() { From 1f0b71d8603772c7bffe3468358bcbae7431adbe Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:08 -0400 Subject: [PATCH 072/631] drm/asahi: publish child page tables before linking them Complete the child table initialization before storing the parent entry, so a concurrent device walk cannot follow a visible parent to a table whose clears have not reached memory yet. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 83cf9f5ad0baf71001cc0aa9c25c83f27f4a219a) --- drivers/gpu/drm/asahi/pgtable.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index da4fc8014f8814..7d71ea0c226cb6 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -774,6 +774,11 @@ impl UatPageTable { } let pt_paddr = Page::into_phys(pt_page); upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + // Complete child-table initialization before a + // concurrent device walk can follow its parent. + // A later mapping/TLBI barrier cannot order an + // already-visible parent against earlier clears. + crate::mem::sync(); upte.store(upte_val, Ordering::Relaxed); } if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { From 1dd7b082e32ff00d6b8c5f96a7e0f154c14e812f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:09 -0400 Subject: [PATCH 073/631] drm/asahi: retire deferred mapping cleanup without allocating Deferred user unmaps keep their preallocated split nodes and retire through immediate-mode drm_gpuvm_sm_unmap() without an exec context. Whole-object unmaps that hit ENOMEM fall back to an allocation-free removal of that GEM's nodes. A partially committed bind batch defers only its uncommitted operations, so committed prefixes are never replayed over a range another binding may now occupy. Validate unmap ranges against the VM geometry at preparation. A cleanup failure keeps admission blocked and is recorded on the VM status rather than silently reopening the VM over stale leaves. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit b321f0e7053949d9144028d0a465a4f03eb6cc83) --- drivers/gpu/drm/asahi/mmu.rs | 37 +++++++++++++-- drivers/gpu/drm/asahi/mmu/bind.rs | 15 ++++-- drivers/gpu/drm/asahi/mmu/lifetime.rs | 67 ++++++++++++++++++--------- rust/kernel/drm/gpuvm.rs | 51 ++++++++++++++++++++ 4 files changed, 143 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 912178d616371e..89e051d9c5e1b9 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -3010,8 +3010,18 @@ impl Vm { self.unmap_range_commit(iova, size, true) } - fn unmap_range_now(&self, iova: u64, size: u64) -> Result { - self.unmap_range_commit(iova, size, false) + fn unmap_prepared_cleanup(&self, unmap: &mut bind::PreparedUserUnmap) -> Result { + let mut inner = self.inner.lock_inner(); + // SAFETY: step_unmap/step_remap modify this VM's translations and + // immediate GPUVA metadata only, using the retained split nodes. They + // do not access external BO backing and defer all BO reference puts. + let result = unsafe { inner.sm_unmap_inner(&mut unmap.ctx, unmap.iova, unmap.size) }; + if result.is_ok() { + self.untrack_shared_range(unmap.iova, unmap.size); + } + drop(inner); + self.bo_deferred_cleanup(); + result } fn unmap_range_commit(&self, iova: u64, size: u64, user: bool) -> Result { @@ -3030,6 +3040,7 @@ impl Vm { match self.mapping_commit() { Ok(guard) => guard, Err(error) => { + drop(inner); if self.defer_unmap(iova, size)? { return Ok(()); } @@ -3060,7 +3071,10 @@ impl Vm { // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); - let inner = self.inner.exec_lock(Some(gem), false)?; + let inner = match self.inner.exec_lock(Some(gem), false) { + Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), + result => result?, + }; if let Some(bo) = self.inner.find_bo(gem) { mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); @@ -3076,12 +3090,29 @@ impl Vm { // unmap error. Always release reservations before dropping it. core::mem::drop(inner); core::mem::drop(bo); + if result == Err(ENOMEM) { + return self.drop_mappings_low_memory(gem); + } result?; } Ok(()) } + /// Preserve the indexed fast path normally; memory pressure must not lose + /// accepted object cleanup just because an exec or operation list failed. + fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { + let mut inner = self.inner.lock_inner(); + let result = inner.unmap_object(gem, VmInner::unmap_gpuva); + if result.is_ok() { + self.untrack_context_object(gem); + self.untrack_shared_object(gem); + } + drop(inner); + self.bo_deferred_cleanup(); + result + } + /// Returns the dummy GEM object used to hold the shared DMA reservation locks pub(crate) fn get_resv_obj(&self) -> ARef { self.dummy_obj.clone() diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 67d18a275ca32e..d2e429c43d311c 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -19,9 +19,15 @@ pub(crate) struct PreparedUserMap { pub(crate) struct PreparedUserUnmap { pub(super) iova: u64, pub(super) size: u64, - ctx: StepContext, + pub(super) ctx: StepContext, } +// SAFETY: This owns only its numeric range and unused, unlinked split-node +// allocations. step_remap moves each node out of ctx before linking it into +// the VM, so a retained ctx never owns a linked GPUVA. The unmap context has +// no borrowed object/lock state; lifetime.state serializes deferred ownership. +unsafe impl Send for PreparedUserUnmap {} + pub(super) enum PreparedUserBindOp { Map(PreparedUserMap), Unmap(PreparedUserUnmap), @@ -111,6 +117,9 @@ impl Vm { } pub(crate) fn prepare_user_unmap(&self, iova: u64, size: u64) -> Result { + if (iova | size) & UAT_PGMSK as u64 != 0 || !self.inner.range_valid(iova, size) { + return Err(EINVAL); + } Ok(PreparedUserUnmap { iova, size, @@ -128,7 +137,7 @@ impl Vm { &self, batch: &mut PreparedUserBindBatch, ) -> Result { - if self.defer_bind_batch(batch)? { + if self.defer_bind_batch(batch, 0)? { return Ok(()); } let mut index = 0; @@ -202,7 +211,7 @@ impl Vm { if let Some(result) = result { result?; index += 1; - } else if self.defer_bind_batch(batch)? { + } else if self.defer_bind_batch(batch, index)? { return Ok(()); } } diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index e33b578bb13a73..20a79037593df2 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -54,8 +54,9 @@ struct State { quarantined: bool, draining: bool, closed: bool, + cleanup_failed: bool, close_ranges: Option<(Range, Range)>, - unmaps: KVec<(u64, u64)>, + unmaps: KVec, objects: KVec>, } @@ -74,7 +75,7 @@ impl VmLifetime { Arc::pin_init( pin_init!(Self { state <- new_mutex!(State { active:0, commits:0, blocked:false, - quarantined:false, draining:false, closed:false, close_ranges:None, + quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), driver <- new_mutex!(None, "VM driver mappings"), @@ -414,7 +415,7 @@ impl Vm { Ok(()) } - pub(super) fn defer_bind_batch(&self, batch: &PreparedUserBindBatch) -> Result { + pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { use super::bind::PreparedUserBindOp; let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); @@ -428,7 +429,7 @@ impl Vm { } if batch .ops - .iter() + .iter().skip(start) .any(|op| matches!(op, PreparedUserBindOp::Map(_))) { return Err(EIO); @@ -438,19 +439,21 @@ impl Vm { } // Reserve before accepting any range; a partial deferred batch must // not leave canonical metadata ahead of the accepted cleanup. - state.unmaps.reserve(batch.ops.len(), GFP_KERNEL)?; - for op in &batch.ops { + state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; + let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { + PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), + PreparedUserBindOp::Map(_) => None, + })); + // Transfer the preallocated split nodes along with each accepted range. + // Already-committed prefix operations must not be replayed: their nodes + // may have been consumed and another binding may now occupy the range. + for op in batch.ops.drain(start..) { if let PreparedUserBindOp::Unmap(unmap) = op { - let range = (unmap.iova, unmap.size); - if !state.unmaps.contains(&range) { - state.unmaps.push(range, GFP_KERNEL)?; + if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { + state.unmaps.push(unmap, GFP_KERNEL)?; } } } - let retired = self.untrack_context_ranges(batch.ops.iter().filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); drop(state); drop(retired); Ok(true) @@ -461,7 +464,7 @@ impl Vm { let Some(lifetime) = self.lifetime.as_ref() else { return Ok(false); }; - let mut state = lifetime.state.lock(); + let state = lifetime.state.lock(); if state.closed { return Err(ENOENT); } @@ -471,8 +474,17 @@ impl Vm { if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.contains(&(iova, size)) { - state.unmaps.push((iova, size), GFP_KERNEL)?; + if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + return Ok(true); + } + drop(state); + let unmap = self.prepare_user_unmap(iova, size)?; + let mut state = lifetime.state.lock(); + if state.closed { return Err(ENOENT); } + if !state.blocked { return Ok(false); } + if !state.quarantined { return Err(EBUSY); } + if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { + state.unmaps.push(unmap, GFP_KERNEL)?; } Ok(true) } @@ -557,24 +569,28 @@ impl Vm { /// Recheck queued cleanup and reopen under the same lock used to append it. fn drain_mappings(&self, lifetime: &VmLifetime) { let mut state = lifetime.state.lock(); + let mut failure = None; loop { let close = state.close_ranges.take(); let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); let objects = core::mem::replace(&mut state.objects, KVec::new()); drop(state); if let Some((user, kernel)) = close { - if self.unmap_user_ranges_now(user, kernel).is_err() { + if let Err(error) = self.unmap_user_ranges_now(user, kernel) { + failure.get_or_insert(error); pr_err!("MMU: deferred VM close failed\n"); } } else { - for (address, size) in unmaps { - if self.unmap_range_now(address, size).is_err() { + for mut unmap in unmaps { + if let Err(error) = self.unmap_prepared_cleanup(&mut unmap) { + failure.get_or_insert(error); pr_err!("MMU: deferred user unmap failed\n"); } } } for object in objects { - if self.drop_mappings_now(&object).is_err() { + if let Err(error) = self.drop_mappings_now(&object) { + failure.get_or_insert(error); pr_err!("MMU: deferred object unmap failed\n"); } } @@ -585,7 +601,16 @@ impl Vm { } state.quarantined = false; state.draining = false; - if !state.closed { + if let Some(error) = failure { + // Cleanup is allocation-free for accepted ranges and has a + // whole-node low-memory fallback for objects. An unexpected + // invariant error must not reopen admission over stale leaves. + // GPUVA owners retain backing until final VM close; report the + // terminal failure instead of silently discarding its outcome. + state.blocked = true; + state.cleanup_failed = true; + if let Some(status) = &self.status { status.record(error); } + } else if !state.closed && !state.cleanup_failed { state.blocked = false; } drop(state); diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index 04e9abe162c6fd..cec0d92f0390ad 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -737,6 +737,12 @@ impl GpuVm { unsafe { bindings::drm_gpuvm_is_extobj(self.gpuvm() as *mut _, gem) } } + /// Validate against the immutable address-space bounds and kernel cutout. + pub fn range_valid(&self, addr: u64, range: u64) -> bool { + // SAFETY: the borrowed VM retains its immutable geometry. + unsafe { bindings::drm_gpuvm_range_valid(self.gpuvm() as *mut _, addr, range) } + } + pub fn bo_deferred_cleanup(&self) { unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.gpuvm() as *mut _) } } @@ -833,6 +839,29 @@ pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { } impl GpuVmInnerGuard<'_, T> { + /// Run immediate-mode unmap steps without allocating an exec context. + /// + /// # Safety + /// The driver's unmap/remap callbacks must touch only VM metadata and + /// translations protected by this reservation. They must not access + /// external BO backing that requires its own reservation. Any split-node + /// storage must already be owned by `ctx`; callbacks must defer BO puts. + pub unsafe fn sm_unmap_inner( + &mut self, ctx: &mut T::StepContext, addr: u64, range: u64, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + // SAFETY: this guard retains the VM and its shared reservation. + if unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + let mut ctx = StepContext { gpuvm: self.gpuvm, ctx }; + // SAFETY: caller supplies the callback contract above. Immediate-mode + // link/unlink acquire the separate GEM GPUVA-list mutex themselves. + to_result(unsafe { + bindings::drm_gpuvm_sm_unmap(vm, &mut ctx as *mut _ as *mut _, addr, range) + }) + } + /// Remove every driver GPUVA in an immediate-mode VM, preserving its kernel cutout. /// /// `unmap` must remove the GPU translation for each complete GPUVA before @@ -844,6 +873,25 @@ impl GpuVmInnerGuard<'_, T> { /// when a callback fails after earlier mappings have already been removed. pub fn unmap_all( &mut self, + unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + self.unmap_matching(None, unmap) + } + + /// Allocation-free whole-node removal for one GEM in an immediate-mode VM. + /// Uses the same translation-removal and deferred-put contract as unmap_all. + /// This scans the VM list and is intended for low-memory cleanup fallback. + pub fn unmap_object( + &mut self, + object: &Object, + unmap: impl FnMut(&mut T, &GpuVa) -> Result, + ) -> Result { + self.unmap_matching(Some(object), unmap) + } + + fn unmap_matching( + &mut self, + object: Option<&Object>, mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, ) -> Result { let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; @@ -863,6 +911,9 @@ impl GpuVmInnerGuard<'_, T> { if core::ptr::eq(va, &raw const (*vm).kernel_alloc_node) { continue; } + if object.is_some_and(|object| (*va).gem.obj != object.as_raw()) { + continue; + } let driver_va = crate::container_of!(va, GpuVa, gpuva); unmap(&mut *self.gpuvm.inner.get(), &*driver_va)?; let mut op = OpUnMap( From 63b635158fd22aa6b8b98a4fe2caee047becc075 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:10:09 -0400 Subject: [PATCH 074/631] drm/asahi: keep compute timestamps outside shared queue aliases Private per-command compute timestamp pages were allocated from the range later used for canonical shared queue aliases, so a late shared alias could collide with a live timestamp page. Allocate them from the client-private lower arena instead. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7d57556a8962dd49aadbb24af772ee5240b8c1c1) --- drivers/gpu/drm/asahi/g17/completion.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 0c877743873d15..7308fde83d568d 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -13,7 +13,7 @@ use super::{ recovery::Class, status::VmStatus, }; -use crate::{driver::AsahiDevice, file, hw::t8140::resources as cfg, mmu}; +use crate::{driver::AsahiDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; use kernel::{ dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, @@ -156,7 +156,9 @@ impl ComputeTimestamps { let object = KernelObject::backing(dev, mmu::UAT_PGSZ, CpuMap::WriteCombined)?; let mapping = object.alias_in( vm, - cfg::TIMESTAMPS, + // Canonical shared queue aliases may be installed in this VM later. + // Private timestamp pages must never occupy their retained addresses. + dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64, mmu::PROT_GPU_FW_SHARED_RW, )?; From a8511af7eb564aa0fca4bd0cec64e4fd01a52d34 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:11:49 -0400 Subject: [PATCH 075/631] drm/shmem-helper: add revocable CPU mappings and idle backing release Let a driver release the backing of an idle shmem object without purging it: drm_gem_shmem_release_idle_pages_locked() drops the cached SG table and page array once no pin, vmap, SG borrower or legacy CPU mapping remains, so ordinary shmem reclaim can swap the contents. The next get_pages_sgt() restores them, possibly at new DMA addresses. Objects created with reclaimable_cpu_mappings keep their CPU VMAs revocable: mmap no longer pins pages, faults take an object-scoped page reference under the reservation, release zaps the device anon mapping and the PTEs refault afterwards. Non-opted objects keep the existing behavior. The Rust shmem wrapper counts owned SG-table leases and marks the borrowed sg_table() accessor permanent, so release is refused while any Rust user still holds the table. Add the dma_resv_trylock helper used by non-blocking reclaim. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 733634bcb0c4de6e9885878dc8b1aad744353c83) --- drivers/gpu/drm/drm_gem_shmem_helper.c | 65 ++++++++++++++++++ rust/kernel/drm/gem/shmem.rs | 91 ++++++++++++++++++++++++-- 2 files changed, 149 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 3b94e2ccdbd91e..5d8b8f1fde05bd 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -600,6 +600,64 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); +/** + * drm_gem_shmem_release_idle_pages_locked - Release idle backing without purging + * @shmem: shmem GEM object whose device mappings have been removed + * + * Drop the cached SG table and the page array of an object that no device + * mapping, pin, vmap or SG-table borrower uses, so ordinary shmem reclaim can + * swap its contents. Revocable CPU VMAs are zapped first and refault later. + * The caller holds the reservation lock, has removed every device mapping + * including cached translations, and excludes new device access. A later + * drm_gem_shmem_get_pages_sgt() may return different DMA addresses. + * + * Returns: 0 on success, -EBUSY if backing is still in use, -EINVAL for an + * imported, exported or purged object. + */ +int drm_gem_shmem_release_idle_pages_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_assert_held(obj->resv); + + if (drm_gem_is_imported(obj) || obj->dma_buf || shmem->madv < 0) + return -EINVAL; + + if (refcount_read(&shmem->pages_pin_count) || + refcount_read(&shmem->vmap_use_count) || + refcount_read(&shmem->pages_use_count) != !!shmem->sgt || + !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP)) + return -EBUSY; + + /* Zap revocable PFNs under resv so no fault can reinsert them. */ + if (shmem->reclaimable_cpu_mappings && refcount_read(&shmem->cpu_vma_count)) { + drm_vma_node_unmap(&obj->vma_node, obj->dev->anon_inode->i_mapping); + drm_gem_shmem_put_cpu_pages_locked(shmem); + } + +static struct page *drm_gem_shmem_present_page(struct drm_gem_shmem_object *shmem, + pgoff_t index) +{ + if (shmem->pages && shmem->pages[index]) + return shmem->pages[index]; + if (shmem->cpu_pages) + return shmem->cpu_pages[index]; + return NULL; +} + +bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shmem) +{ + struct drm_gem_object *obj = &shmem->base; + + dma_resv_assert_held(obj->resv); + return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && + shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && + !refcount_read(&shmem->vmap_use_count) && + refcount_read(&shmem->pages_use_count) == 1 && + dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); +} +EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); + /** * drm_gem_shmem_release_idle_pages_locked - Release idle backing without purging * @shmem: shmem GEM object whose device mappings have been removed @@ -659,6 +717,8 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + /* Revoke PFNs and complete the TLB flush while backing is still pinned. */ shmem->madv = -1; drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); @@ -982,6 +1042,11 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct return -EINVAL; } + /* Revocation zaps the device anon mapping only. */ + if (shmem->reclaimable_cpu_mappings && + (!vma->vm_file || vma->vm_file->f_mapping != obj->dev->anon_inode->i_mapping)) + return -EINVAL; + dma_resv_lock(shmem->base.resv, NULL); if (shmem->reclaimable_cpu_mappings) { if (!refcount_inc_not_zero(&shmem->cpu_vma_count)) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 6536c9be946849..31841dd8f35208 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -38,6 +38,7 @@ use core::{ DerefMut, // }, ptr::NonNull, + sync::atomic::{AtomicUsize, Ordering}, }; use gem::{ BaseObject, @@ -55,6 +56,8 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Whether to set the write-combine map flag. pub map_wc: bool, + /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. + pub reclaimable_cpu_mappings: bool, /// Reclaim private backing pages before returning from the final free callback. /// The default retains the usual deferred file-release policy. pub synchronous_reclaim: bool, @@ -65,6 +68,10 @@ pub struct ObjectConfig<'a, T: DriverObject> { pub parent_resv_obj: Option<&'a Object>, } +const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); +const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); +const SGT_USERS: usize = SGT_RECLAIMING - 1; + /// A shmem-backed GEM object. /// /// # Invariants @@ -78,6 +85,8 @@ pub struct Object { obj: Opaque, /// Parent object that owns this object's DMA reservation object. parent_resv_obj: Option>>, + /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. + sgt_access: AtomicUsize, synchronous_reclaim: bool, #[pin] inner: T, @@ -135,6 +144,7 @@ impl Object { try_pin_init!(Self { obj <- Opaque::init_zeroed(), parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), + sgt_access: AtomicUsize::new(0), synchronous_reclaim: config.synchronous_reclaim, inner <- T::new(dev, size, args), }), @@ -164,6 +174,7 @@ impl Object { // to have exclusive access - thus making this safe to hold a mutable reference to. let shmem = unsafe { &mut *obj.as_raw_shmem() }; shmem.set_map_wc(config.map_wc); + shmem.reclaimable_cpu_mappings = config.reclaimable_cpu_mappings; Ok(obj) } @@ -230,6 +241,33 @@ impl Object { /// This will pin the object in memory. #[inline] pub fn sg_table(&self) -> Result<&scatterlist::SGTable> { + // The borrowed reference can outlive any lock, so pin permanently. + self.acquire_sgt_access(true)?; + self.sg_table_resident() + } + + fn acquire_sgt_access(&self, permanent: bool) -> Result { + let mut state = self.sgt_access.load(Ordering::Acquire); + loop { + if state & SGT_RECLAIMING != 0 { + return Err(EBUSY); + } + let next = if permanent { + state | SGT_PERMANENT + } else { + if state & SGT_USERS == SGT_USERS { return Err(EOVERFLOW); } + state + 1 + }; + match self.sgt_access.compare_exchange_weak( + state, next, Ordering::AcqRel, Ordering::Acquire, + ) { + Ok(_) => return Ok(()), + Err(current) => state = current, + } + } + } + + fn sg_table_resident(&self) -> Result<&scatterlist::SGTable> { // SAFETY: // - drm_gem_shmem_get_pages_sgt is thread-safe. // - drm_gem_shmem_get_pages_sgt returns either a valid pointer to a scatterlist, or an @@ -252,12 +290,44 @@ impl Object { /// /// [`shmem::SGTable`]: SGTable pub fn owned_sg_table(&self) -> Result> { - Ok(SGTable { - sgt: self.sg_table()?.into(), - // INVARIANT: We take an owned refcount to `self` here, ensuring that `sgt` remains - // valid for as long as this `SGTable`. - _owner: self.into(), - }) + self.acquire_sgt_access(false)?; + let sgt = match self.sg_table_resident() { + Ok(sgt) => sgt.into(), + Err(error) => { + self.sgt_access.fetch_sub(1, Ordering::Release); + return Err(error); + } + }; + Ok(SGTable { sgt, _owner: self.into() }) + } + + /// Whether the caller's sole SG lease is the last user of the backing. + /// + /// # Safety + /// The caller holds the object's DMA reservation lock. + pub unsafe fn sole_sg_lease_releasable_locked(&self) -> bool { + self.sgt_access.load(Ordering::Acquire) == 1 && + // SAFETY: The caller establishes the reservation-lock contract. + unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } + } + + /// Release idle backing without discarding contents. A later + /// `owned_sg_table()` may return different DMA addresses. + /// + /// # Safety + /// + /// The caller holds this object's DMA reservation lock, has removed and + /// invalidated every device mapping, and excludes new device access. + pub unsafe fn release_idle_pages_locked(&self) -> Result { + self.sgt_access.compare_exchange( + 0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire, + ).map_err(|_| EBUSY)?; + // SAFETY: The caller's contract plus the exclusive state above. + let result = to_result(unsafe { + bindings::drm_gem_shmem_release_idle_pages_locked(self.as_raw_shmem()) + }); + self.sgt_access.store(0, Ordering::Release); + result } /// Attempt to create a [`RawIoSysMap`] from the gem object. @@ -472,7 +542,8 @@ unsafe impl Sync for VMap {} /// /// # Invariants /// -/// - `sgt` is kept alive by `_owner`, ensuring it remains valid for as long as `Self`. +/// - `_owner` retains both the GEM object and one counted SG-table lease; eviction +/// cannot begin until every lease has been released. /// - `sgt` corresponds to the owned object in `_owner`. /// - This object is only exposed in situations where we know the underlying `SGTable` will not be /// modified for the lifetime of this object. Thus, it is safe to send/access this type across @@ -487,6 +558,12 @@ unsafe impl Send for SGTable {} // SAFETY: This object is thread-safe via our type invariants. unsafe impl Sync for SGTable {} +impl Drop for SGTable { + fn drop(&mut self) { + self._owner.sgt_access.fetch_sub(1, Ordering::Release); + } +} + impl Deref for SGTable { type Target = scatterlist::SGTable; From ffd2dc623df63732faf29086f076aed2315a8e63 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:11:49 -0400 Subject: [PATCH 076/631] rust: drm: gpuvm: add non-blocking private mapping traversal Add the interfaces a driver needs to evict and restore the backing of idle VM-private objects: expose per-GPUVA driver metadata, borrow the GEM object of a VM/BO association, take the VM reservation with dma_resv_trylock, walk private associations and mark every alias INVALIDATED without allocating, iterate current private mappings by cursor, and clear the flag after the page tables are rebuilt. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 35b057e572224c39f79367df9e7c464f1822d15a) --- rust/kernel/drm/gpuvm.rs | 229 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index cec0d92f0390ad..f3562ccd17e1a8 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -285,6 +285,15 @@ impl GpuVa { ) } + /// Driver metadata retained independently of the resident page tables. + pub fn inner(&self) -> &T::GpuVa { &self.inner } + + /// Update metadata before linking a newly allocated GPUVA. + pub fn inner_mut(self: Pin<&mut Self>) -> Pin<&mut T::GpuVa> { + // SAFETY: inner is structurally pinned with its enclosing GPUVA. + unsafe { self.map_unchecked_mut(|va| &mut va.inner) } + } + pub fn addr(&self) -> u64 { self.gpuva.va.addr } @@ -312,6 +321,13 @@ pub struct GpuVmBo { } impl GpuVmBo { + /// The GEM object retained by this VM/BO association. + pub fn object(&self) -> &Object { + // SAFETY: drm_gpuvm_bo owns this GEM reference until its destructor; + // DriverGpuVm fixes the GEM type for every association in this VM. + unsafe { &* as IntoGEMObject>::from_raw(self.bo.obj) } + } + /// Return a reference to the inner driver data for this GpuVmBo pub fn inner(&self) -> &T::GpuVmBo { &self.inner @@ -621,6 +637,32 @@ impl GpuVm { } } + // The caller owns the VM reservation and keeps the exact logical VA linked. + unsafe fn set_invalidated_locked(&self, addr: u64, range: u64, invalidated: bool) -> Result { + if range == 0 || addr.checked_add(range).is_none() { return Err(EINVAL); } + // SAFETY: The caller owns the VM reservation and keeps the VA linked. + let raw = unsafe { bindings::drm_gpuva_find(self.gpuvm() as *mut _, addr, range) }; + if raw.is_null() { return Err(EINVAL); } + // SAFETY: Reservation ownership excludes all concurrent VA mutation. + unsafe { + if (*raw).va.addr != addr || (*raw).va.range != range { return Err(EINVAL); } + if invalidated { (*raw).flags |= GpuVaFlags::INVALIDATED.as_raw(); } + else { (*raw).flags &= !GpuVaFlags::INVALIDATED.as_raw(); } + } + Ok(()) + } + + /// Try to lock only the VM reservation, without allocation or waiting. + /// The guard protects VM metadata and private BOs, not external backing. + /// Callers must not already hold a reservation lock. + pub fn try_lock_private(&self) -> Option> { + // SAFETY: r_obj and its reservation are retained by this GPUVM. + let resv = unsafe { (*(*self.gpuvm()).r_obj).resv }; + // SAFETY: The reservation outlives the returned borrow and guard. + if !unsafe { bindings::dma_resv_trylock(resv) } { return None; } + Some(PrivateGpuVmGuard { gpuvm: self, resv, _not_send: NotThreadSafe }) + } + pub fn exec_lock<'a, 'b>( &'a self, obj: Option<&'b Object>, @@ -952,6 +994,187 @@ impl Drop for GpuVmInnerGuard<'_, T> { } } +/// Logical binding retained independently of hardware page-table residency. +/// Owns a GEM reference and copied metadata, never a GpuVmBo reference. +/// Driver metadata cloning must not introduce a reservation-taking destructor. +pub struct MappingSnapshot { + pub object: ARef>, + pub addr: u64, + pub range: u64, + pub offset: u64, + pub flags: GpuVaFlags, + pub inner: T::GpuVa, +} + +/// Nonblocking reservation ownership for pressure scans of private objects. +pub struct PrivateGpuVmGuard<'a, T: DriverGpuVm> { + gpuvm: &'a GpuVm, + resv: *mut bindings::dma_resv, + // DMA reservation ownership must remain on the acquiring task. + _not_send: NotThreadSafe, +} + +impl PrivateGpuVmGuard<'_, T> { + /// Visit each private BO with linked logical mappings once, without allocating or retaining + /// references. Returning false stops the walk. Contended GEM lists are skipped. + /// + /// # Safety + /// The callback must not insert/remove logical mappings or modify either GPUVA list. This + /// reservation keeps their linked BO/GEM references alive for the complete callback. + /// The callback must not retain a borrowed BO or acquire its GEM GPUVA mutex recursively. + pub unsafe fn for_each_private_bo( + &mut self, + mut visit: impl FnMut(&mut Self, &GpuVmBo) -> Result, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + if unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + let head = unsafe { &raw const (*vm).rb.list }; + // SAFETY: The VM reservation protects this ordered mapping list. The callback does not + // mutate its links, and no borrowed or owned reference outlives the guard. + let mut node = unsafe { (*head).next }; + while node != head.cast_mut() { + let (next, raw) = unsafe { + ((*node).next, crate::container_of!(node, bindings::drm_gpuva, rb.entry)) + }; + let va = unsafe { &*raw }; + if !va.gem.obj.is_null() && !va.vm_bo.is_null() { + let object = unsafe { &*Object::::from_raw(va.gem.obj) }; + if !self.gpuvm.is_extobj(object) { + let lock = unsafe { &raw mut (*va.gem.obj).gpuva.lock }; + // Never block direct reclaim on a GEM-list owner. The canonical alias is + // the list's first-linked node, not its lowest address or current residency. + if unsafe { bindings::mutex_trylock(lock) } != 0 { + let canonical = unsafe { + (*va.vm_bo).list.gpuva.next == (&raw const va.gem.entry).cast_mut() + }; + unsafe { bindings::mutex_unlock(lock) }; + if canonical { + let bo = unsafe { + &*crate::container_of!(va.vm_bo, GpuVmBo, bo) + }; + if !visit(self, bo)? { break; } + } + } + } + } + node = next; + } + Ok(()) + } + + /// Mark and visit every logical alias of this private BO without a snapshot allocation. + /// GEM-list locking is nonblocking and held only while reading a cursor, never during the + /// callback. Failure (including contention) can leave a prefix marked INVALIDATED; callers + /// must retain backing unless this method succeeds for the entire association. + /// + /// # Safety + /// The callback must not insert/remove logical mappings, alter list links, or release the + /// BO's SG/backing lease. The BO must remain linked throughout the walk. Exclude new jobs and + /// binding edits for the whole invalidation and backing-release transaction. + pub unsafe fn invalidate_private_bo_mappings( + &mut self, + bo: &GpuVmBo, + mut visit: impl FnMut(&mut Self, u64, u64) -> Result, + ) -> Result { + let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; + if bo.bo.vm != vm || self.gpuvm.is_extobj(bo.object()) + || unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 + { return Err(EINVAL); } + let head = &raw const bo.bo.list.gpuva; + let lock = unsafe { &raw mut (*bo.bo.obj).gpuva.lock }; + if unsafe { bindings::mutex_trylock(lock) } == 0 { return Err(EAGAIN); } + let mut node = unsafe { (*head).next }; + unsafe { bindings::mutex_unlock(lock) }; + while node != head.cast_mut() { + if unsafe { bindings::mutex_trylock(lock) } == 0 { return Err(EAGAIN); } + // SAFETY: The list mutex protects link reads; this reservation and the caller's + // no-unlink contract keep current/next and their association alive after unlocking. + let (next, raw) = unsafe { + ((*node).next, crate::container_of!(node, bindings::drm_gpuva, gem.entry)) + }; + unsafe { bindings::mutex_unlock(lock) }; + let va = unsafe { &mut *raw }; + if va.vm != vm || va.vm_bo != (&raw const bo.bo).cast_mut() + || va.va.range == 0 || va.va.addr.checked_add(va.va.range).is_none() + { return Err(EINVAL); } + // Flags are serialized by the VM reservation, independently of GEM's list mutex. + va.flags |= GpuVaFlags::INVALIDATED.as_raw(); + visit(self, va.va.addr, va.va.range)?; + node = next; + } + Ok(()) + } + + /// Retain the next private mapping containing all `required_flags`. + /// + /// `cursor` is an exclusive end address and advances past skipped mappings + /// as well. No collection is allocated; only the returned object is retained. + /// Callers must drop this reservation guard before releasing that reference + /// or acquiring backing. To cover a stable set across calls, separately + /// exclude logical binding edits (including retirement) for the whole walk. + pub fn next_private_mapping( + &self, + cursor: &mut u64, + required_flags: GpuVaFlags, + ) -> Result>> + where T::GpuVa: Clone { + while *cursor != u64::MAX { + // SAFETY: The VM reservation serializes mutation of its VA tree. + let raw = unsafe { + bindings::drm_gpuva_find_first( + self.gpuvm.gpuvm() as *mut _, *cursor, u64::MAX - *cursor, + ) + }; + if raw.is_null() { return Ok(None); } + // SAFETY: raw stays linked and alive while this guard is held. + let va = unsafe { &*raw }; + let end = va.va.addr.checked_add(va.va.range).ok_or(EINVAL)?; + if end <= *cursor { return Err(EINVAL); } + *cursor = end; + if !GpuVaFlags(va.flags).contains(required_flags) + || va.gem.obj.is_null() || va.vm_bo.is_null() { + continue; + } + // SAFETY: The linked VA retains this driver's object. + let obj = unsafe { &*Object::::from_raw(va.gem.obj) }; + if self.gpuvm.is_extobj(obj) { continue; } + // SAFETY: Driver-owned mappings have this allocation layout. The + // special kernel cutout has no GEM object and was skipped above. + let typed = unsafe { &*crate::container_of!(raw, GpuVa, gpuva) }; + return Ok(Some(MappingSnapshot { + object: obj.into(), addr: va.va.addr, + range: va.va.range, offset: va.gem.offset, + flags: GpuVaFlags(va.flags), inner: typed.inner.clone(), + })); + } + Ok(None) + } + + +} + +impl Deref for PrivateGpuVmGuard<'_, T> { + type Target = T; + fn deref(&self) -> &T { + // SAFETY: The reservation serializes all access to the VM's inner data. + unsafe { &*self.gpuvm.inner.get() } + } +} +impl DerefMut for PrivateGpuVmGuard<'_, T> { + fn deref_mut(&mut self) -> &mut T { + // SAFETY: This guard has exclusive reservation ownership. + unsafe { &mut *self.gpuvm.inner.get() } + } +} +impl Drop for PrivateGpuVmGuard<'_, T> { + fn drop(&mut self) { + // SAFETY: try_lock_private acquired exactly this reservation. + unsafe { bindings::dma_resv_unlock(self.resv) }; + } +} + pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { gpuvm: &'a GpuVm, vm_exec: KBox, @@ -959,6 +1182,12 @@ pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { } impl LockedGpuVm<'_, '_, T> { + /// Mark the exact retained logical mapping after page-table restoration. + pub fn set_invalidated(&mut self, addr: u64, range: u64, invalidated: bool) -> Result { + // SAFETY: Both guard types own the VM reservation. + unsafe { self.gpuvm.set_invalidated_locked(addr, range, invalidated) } + } + pub fn find_bo(&mut self) -> Option>> { let obj = self.obj?; // SAFETY: LockedGpuVm implies the right locks are held. From 1ccf84b88e0ba35bd5c3bca7d92daff6c01aba9c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:13:08 -0400 Subject: [PATCH 077/631] drm/asahi: reclaim idle private VM backing under memory pressure Register a shrinker per client VM. When the VM has no accepted jobs, no mapping commits and a healthy status, the scan invalidates every alias of an idle VM-private object, clears its page tables and TLB entries, drops the VM's SG lease and releases the shmem backing so ordinary reclaim can swap it. Objects with kernel aliases, exports/imports or other SG users are skipped. Private objects also get revocable CPU mappings, so an mmap no longer pins the whole object before it is touched. Jobs, kernel aliases and bind preparation take a residency lease; the first lease after an eviction rebuilds the invalidated mappings from the logical GPUVAs before any work is admitted, and retirement serializes with that restore. Mapping permissions are kept as GPUVA metadata so restored and split mappings keep their protection. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e2b08ebbcadc3a9ebe2733140e1a88c62af87ba1) --- drivers/gpu/drm/asahi/file.rs | 22 +- drivers/gpu/drm/asahi/gem.rs | 40 ++ drivers/gpu/drm/asahi/mmu.rs | 48 ++- drivers/gpu/drm/asahi/mmu/bind.rs | 13 +- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 +- drivers/gpu/drm/asahi/mmu/residency.rs | 484 +++++++++++++++++++++++++ 6 files changed, 600 insertions(+), 21 deletions(-) create mode 100644 drivers/gpu/drm/asahi/mmu/residency.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index cc42f5f3c108b3..88899e2ed7c837 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -64,6 +64,7 @@ const LEGACY_VM_USER_RANGE: Range = mmu::IOVA_USER_BASE..mmu::IOVA_UNK_PAGE /// A client instance of an `mmu::Vm` address space. struct Vm { + shrinker: Option>, ualloc: Arc>, ualloc_priv: Arc>, vm: mmu::Vm, @@ -616,6 +617,11 @@ impl File { let profile_vm_id = vm.id(); resv.fill(KBox::new( Vm { + shrinker: if vm.residency_reclaim_enabled() { + Some(mmu::VmShrinker::new(&vm)?) + } else { + None + }, ualloc, ualloc_priv, vm, @@ -675,21 +681,19 @@ impl File { } let resv_gem; + let mut reclaim_cpu = false; let resv_obj = if data.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0 { - resv_gem = file - .inner() - .vms() - .lock() - .get(data.vm_id.try_into()?) - .ok_or(ENOENT)? - .vm - .get_resv_obj(); + let vms = file.inner().vms(); + let vms = vms.lock(); + let vm = &vms.get(data.vm_id.try_into()?).ok_or(ENOENT)?.vm; + reclaim_cpu = vm.residency_reclaim_enabled(); + resv_gem = vm.get_resv_obj(); Some(resv_gem.deref()) } else { None }; - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj)?; + let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; let handle = gem.create_handle(file)?; data.handle = handle; diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 5f991f57872900..d05544bc49fb84 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -53,6 +53,8 @@ pub(crate) struct AsahiObject { /// Whether this is a kernel-created object. kernel: bool, _allocation: Option, + /// Driver aliases outside the user GPUVA tree; they pin the backing. + kernel_mappings: AtomicU64, } /// Type alias for the shmem GEM object type for this driver. @@ -222,6 +224,7 @@ fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Res dev, checked_object_size(size)?, shmem::ObjectConfig:: { + reclaimable_cpu_mappings: false, map_wc, parent_resv_obj: None, synchronous_reclaim: dev.is_m3, @@ -252,6 +255,7 @@ pub(crate) fn new_object( size: usize, flags: u32, parent_object: Option<&shmem::Object>, + reclaim_cpu: bool, ) -> Result> { if (flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE != 0) != parent_object.is_some() { @@ -262,6 +266,7 @@ pub(crate) fn new_object( dev, checked_object_size(size)?, shmem::ObjectConfig:: { + reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, map_wc: flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_WRITEBACK == 0, parent_resv_obj: parent_object, synchronous_reclaim: dev.is_m3, @@ -302,6 +307,7 @@ impl DriverObject for AsahiObject { _allocation: if dev.is_m3 { Some(crate::agx_memory_stats::Allocation::gem(size, args.kernel)?) } else { None }, + kernel_mappings: AtomicU64::new(0), }) } @@ -326,3 +332,37 @@ impl DriverObject for AsahiObject { obj.prime_export(flags) } } + +/// A driver-created alias of a GEM object, possibly in a root the user GPUVA +/// tree does not describe. It keeps the object out of idle reclaim. +pub(crate) struct KernelMappingPin { + object: ARef, +} + +impl KernelMappingPin { + pub(crate) fn new(object: &Object) -> Self { + object.kernel_mappings.fetch_add(1, Ordering::AcqRel); + Self { + object: object.into(), + } + } +} + +impl core::ops::Deref for KernelMappingPin { + type Target = Object; + fn deref(&self) -> &Object { + &self.object + } +} + +impl Drop for KernelMappingPin { + fn drop(&mut self) { + self.object.kernel_mappings.fetch_sub(1, Ordering::Release); + } +} + +impl AsahiObject { + pub(crate) fn idle_reclaim_candidate(&self) -> bool { + !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 + } +} diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 89e051d9c5e1b9..c6be84aa8d3a5e 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -99,6 +99,8 @@ mod bind; mod context; mod shared; mod lifetime; +mod residency; +pub(crate) use residency::VmShrinker; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; @@ -478,6 +480,13 @@ impl gpuvm::DriverGpuVmBo for VmBo { } } +/// Per-mapping metadata that survives backing eviction and GPUVA splits. +#[derive(Default, Clone)] +struct VmGpuVa { + prot: Option, +} +impl gpuvm::DriverGpuVa for VmGpuVa {} + #[derive(Default)] struct StepContext { new_va: Option>>>, @@ -490,6 +499,7 @@ struct StepContext { impl gpuvm::DriverGpuVm for VmInner { type Driver = driver::AsahiDriver; type GpuVmBo = VmBo; + type GpuVa = VmGpuVa; type StepContext = StepContext; fn step_map( @@ -595,7 +605,8 @@ impl gpuvm::DriverGpuVm for VmInner { } } - let gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + let mut gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + gpuva.as_mut().inner_mut().get_mut().prot = Some(ctx.prot); if op .map_and_link_va( @@ -643,6 +654,8 @@ impl gpuvm::DriverGpuVm for VmInner { let va = op.unmap().va().expect("No previous VA"); let orig_addr = va.addr(); let orig_range = va.range(); + let orig_prot = va.inner().prot.ok_or(EINVAL)?; + let orig_invalidated = va.flags().contains(gpuvm::GpuVaFlags::INVALIDATED); // Only unmap the hole between prev/next, if they exist let unmap_start = if let Some(op) = op.prev_map() { @@ -669,7 +682,11 @@ impl gpuvm::DriverGpuVm for VmInner { let unmap_range = unmap_end - unmap_start; let _mutation = self.mapping_mutation(); - self.page_table.unmap_pages(unmap_start..unmap_end)?; + if orig_invalidated { + self.page_table.discard_partial_map(unmap_start..unmap_end)?; + } else { + self.page_table.unmap_pages(unmap_start..unmap_end)?; + } if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -691,10 +708,11 @@ impl gpuvm::DriverGpuVm for VmInner { } if let Some(prev_op) = op.prev_map() { - let prev_gpuva = ctx + let mut prev_gpuva = ctx .prev_va .take() .expect("Multiple step_remap calls with prev_op"); + prev_gpuva.as_mut().inner_mut().get_mut().prot = Some(orig_prot); if prev_op.map_and_link_va(self, prev_gpuva, vm_bo).is_err() { dev_err!(self.dev.as_ref(), "step_remap: could not relink prev gpuva"); return Err(EINVAL); @@ -702,10 +720,11 @@ impl gpuvm::DriverGpuVm for VmInner { } if let Some(next_op) = op.next_map() { - let next_gpuva = ctx + let mut next_gpuva = ctx .next_va .take() .expect("Multiple step_remap calls with next_op"); + next_gpuva.as_mut().inner_mut().get_mut().prot = Some(orig_prot); if next_op.map_and_link_va(self, next_gpuva, vm_bo).is_err() { dev_err!(self.dev.as_ref(), "step_remap: could not relink next gpuva"); return Err(EINVAL); @@ -722,8 +741,11 @@ impl VmInner { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); let _mutation = self.mapping_mutation(); - self.page_table - .unmap_pages(va.addr()..(va.addr() + va.range()))?; + if va.flags().contains(gpuvm::GpuVaFlags::INVALIDATED) { + self.page_table.discard_partial_map(va.addr()..(va.addr() + va.range()))?; + } else { + self.page_table.unmap_pages(va.addr()..(va.addr() + va.range()))?; + } if let Some(asid) = self.slot() { fence(Ordering::SeqCst); @@ -904,6 +926,10 @@ impl VmInner { } let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + self.tlbi_context_mask(iova, size, contexts); + } + + fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { if contexts == 0 { return; } @@ -1012,6 +1038,7 @@ pub(crate) struct Vm { id: u64, mapping_epoch: Option>, lifetime: Option>, + residency: Option>, context_bindings: Option>, shared_bindings: Option>, status: Option>, @@ -1293,7 +1320,7 @@ pub(crate) struct KernelMappingInner { // - Drop the GEM BO next, since BO free can take the resv lock itself // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 bo: Option, - _gem: Option>, + _gem: Option, owner: ARef>, uat_inner: Arc, prot: Prot, @@ -2302,6 +2329,7 @@ impl Vm { id, mapping_epoch: mapping_epoch.clone(), lifetime: None, + residency: None, context_bindings: None, shared_bindings: None, status: None, @@ -2348,6 +2376,7 @@ impl Vm { pub(crate) fn with_status(mut self) -> Result { self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); + self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); self.shared_bindings = Some(shared::SharedBindings::new()?); Ok(self) @@ -2681,6 +2710,7 @@ impl Vm { return Err(EINVAL); } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; + let _residency = self.enter_residency()?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = self.inner.obtain_bo(gem)?; @@ -2762,6 +2792,7 @@ impl Vm { let reserved_size = size .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; + let _residency = self.enter_residency()?; let sgt = gem.owned_sg_table()?; let mut inner = self.inner.exec_lock(Some(&gem), false)?; @@ -2838,6 +2869,7 @@ impl Vm { prot: Prot, single_page: bool, ) -> Result { + let _residency = self.enter_residency_metadata()?; gem::validate_vm_binding(gem, self)?; self.wait_for_user_map_admission()?; // Mapping needs a complete context @@ -3025,6 +3057,7 @@ impl Vm { } fn unmap_range_commit(&self, iova: u64, size: u64, user: bool) -> Result { + let _residency = self.enter_retirement_residency(); // Unmapping a range can only do a single split, so just preallocate // the prev and next GpuVas let mut ctx = StepContext { @@ -3068,6 +3101,7 @@ impl Vm { } fn drop_mappings_now(&self, gem: &gem::Object) -> Result { + let _residency = self.enter_retirement_residency(); // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index d2e429c43d311c..df189e9bc0f5fc 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -14,18 +14,19 @@ pub(crate) struct PreparedUserMap { offset: u64, single_page: bool, context: context::PreparedBinding, + _residency: Option, } pub(crate) struct PreparedUserUnmap { + _residency: Option, pub(super) iova: u64, pub(super) size: u64, pub(super) ctx: StepContext, } -// SAFETY: This owns only its numeric range and unused, unlinked split-node -// allocations. step_remap moves each node out of ctx before linking it into -// the VM, so a retained ctx never owns a linked GPUVA. The unmap context has -// no borrowed object/lock state; lifetime.state serializes deferred ownership. +// SAFETY: This owns a thread-safe residency lease, a numeric range and unused, +// unlinked split-node allocations; step_remap moves each node out of ctx before +// linking it, so a retained ctx never owns a linked GPUVA. unsafe impl Send for PreparedUserUnmap {} pub(super) enum PreparedUserBindOp { @@ -66,6 +67,7 @@ impl Vm { single_page: bool, ) -> Result { gem::validate_vm_binding(gem, self)?; + let residency = self.enter_residency_metadata()?; let context = self.prepare_context_binding(addr, size, offset)?; let mut ctx = StepContext { new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), @@ -106,6 +108,7 @@ impl Vm { return Err(EINVAL); } Ok(PreparedUserMap { + _residency: residency, gem: gem.into(), ctx, addr, @@ -120,7 +123,9 @@ impl Vm { if (iova | size) & UAT_PGMSK as u64 != 0 || !self.inner.range_valid(iova, size) { return Err(EINVAL); } + let residency = self.enter_residency_metadata()?; Ok(PreparedUserUnmap { + _residency: residency, iova, size, ctx: StepContext { diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 20a79037593df2..d53c19b93bfdcd 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -71,6 +71,14 @@ pub(super) struct VmLifetime { } impl VmLifetime { + pub(super) fn can_reclaim(&self) -> bool { + self.state.try_lock().is_some_and(|state| { + state.active == 0 && state.commits == 0 + && !state.blocked && !state.quarantined && !state.draining + && !state.closed && !state.cleanup_failed + }) + } + pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { @@ -89,6 +97,7 @@ impl VmLifetime { /// retirement: potentially visible work must retain its guard through quarantine or stop. pub(crate) struct VmJobGuard { vm: Vm, + _residency: Option, } impl VmJobGuard { @@ -344,6 +353,7 @@ impl Vm { /// Acquires a job reference without imposing a VM-wide limit on healthy work. pub(crate) fn retain_job(&self) -> Result { + let residency = self.enter_residency()?; let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; let mut state = lifetime.state.lock(); loop { @@ -355,7 +365,7 @@ impl Vm { } if !state.draining { state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone() }); + return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); } lifetime.changed.wait(&mut state); } @@ -533,6 +543,7 @@ impl Vm { } fn unmap_user_ranges_now(&self, user: Range, _kernel: Range) -> Result { + let _residency = self.enter_retirement_residency(); // User mappings alone have GPUVA nodes. Kernel and driver aliases use // mm::Node and are untouched; GPUVM's embedded kernel cutout is skipped. // Closed admission and zero accepted jobs make whole-node teardown safe. @@ -568,6 +579,7 @@ impl Vm { /// Admission stays closed to new jobs while the exec lock is acquired for cleanup. /// Recheck queued cleanup and reopen under the same lock used to append it. fn drain_mappings(&self, lifetime: &VmLifetime) { + let _residency = self.enter_retirement_residency(); let mut state = lifetime.state.lock(); let mut failure = None; loop { diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs new file mode 100644 index 00000000000000..a66d9cff73e2d4 --- /dev/null +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pressure-only backing residency for idle G17 user VMs. +//! +//! This gate protects backing/address preparation as well as accepted jobs. +//! Its writer never waits for active readers: reclaim skips the VM instead. + +use super::*; +use kernel::drm::gem::BaseObject; +use kernel::{new_condvar, sync::CondVar}; + +#[derive(Default)] +struct State { + users: usize, + busy: bool, + evicted: bool, + closed: bool, +} + +#[pin_data] +pub(crate) struct Gate { + #[pin] + state: Mutex, + #[pin] + ready: CondVar, +} + +pub(crate) struct Lease { + gate: Arc, +} + +pub(crate) struct Reclaim { + gate: Arc, + changed: bool, +} + +impl Gate { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + state <- new_mutex!(State::default(), "Asahi residency"), + ready <- new_condvar!("Asahi residency ready"), + }), + GFP_KERNEL, + ) + } + + /// Only a previous pressure eviction can cause restoration or waiting. + pub(crate) fn enter(gate: &Arc, mut restore: impl FnMut() -> Result) -> Result { + loop { + let mut state = gate.state.lock(); + if state.closed { + return Err(ENOENT); + } + while state.busy { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + if state.closed { + return Err(ENOENT); + } + if !state.evicted { + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + return Ok(Lease { gate: gate.clone() }); + } + // Binding edits can add mappings or retire invalidated GPUVAs + // without restoring the rest of the VM. Wait for those mutations + // before walking the authoritative logical bindings for restoration. + if state.users != 0 { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + continue; + } + // No new reader can enter until every retained mapping is restored. + state.busy = true; + drop(state); + let result = restore(); + let mut state = gate.state.lock(); + state.busy = false; + if result.is_ok() { + state.evicted = false; + } + drop(state); + gate.ready.notify_all(); + result?; + } + } + + pub(crate) fn enter_metadata(gate: &Arc) -> Result { + let mut state = gate.state.lock(); + while state.busy { + if gate.ready.wait_interruptible(&mut state) { + return Err(ERESTARTSYS); + } + } + if state.closed { + return Err(ENOENT); + } + state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; + Ok(Lease { gate: gate.clone() }) + } + + /// Retirement of an already closed handle may outlive the file's VM. + /// A closed gate excludes every new reclaimer and restorer, so no lease + /// is necessary then. Otherwise serialize metadata removal with restoration. + pub(super) fn enter_retirement(gate: &Arc) -> Option { + let mut state = gate.state.lock(); + while state.busy { + gate.ready.wait(&mut state); + } + if state.closed { + return None; + } + state.users += 1; + Some(Lease { gate: gate.clone() }) + } + + /// Direct reclaim cannot block on a job, another reclaimer or restoration. + /// A prior partial eviction does not exclude another scan: the VM may + /// still retain eligible BOs after satisfying a previous scan target. + /// The busy gate continues to exclude restoration and metadata changes. + pub(crate) fn try_reclaim(gate: &Arc) -> Option { + let mut state = gate.state.try_lock()?; + if state.users != 0 || state.busy || state.closed { + return None; + } + state.busy = true; + Some(Reclaim { + gate: gate.clone(), + changed: false, + }) + } + + pub(crate) fn close(&self) { + let mut state = self.state.lock(); + state.closed = true; + while state.busy { + self.ready.wait(&mut state); + } + } + + pub(crate) fn idle(&self) -> bool { + self.state + .try_lock() + .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) + } +} + +impl Reclaim { + /// Call before the first page-table mutation, including fallible unmaps. + pub(crate) fn mark_changed(&mut self) { + self.changed = true; + } +} + +impl Drop for Reclaim { + fn drop(&mut self) { + let mut state = self.gate.state.lock(); + state.evicted |= self.changed; + state.busy = false; + drop(state); + self.gate.ready.notify_all(); + } +} + +impl Drop for Lease { + fn drop(&mut self) { + let mut state = self.gate.state.lock(); + debug_assert!(state.users != 0); + state.users -= 1; + if state.users == 0 { + drop(state); + self.gate.ready.notify_all(); + } + } +} + +/// Registration lifetime belongs to the file's VM handle, not the VM clone +/// retained by the callback. Unregister synchronously before dropping the clone. +pub(crate) struct VmShrinker { + vm: crate::mmu::Vm, + raw: *mut kernel::bindings::shrinker, +} + +// SAFETY: shrinker core serializes registration lifetime; callback VM operations +// use its residency/reservation locks. raw is never dereferenced after free. +unsafe impl Send for VmShrinker {} +unsafe impl Sync for VmShrinker {} + +impl VmShrinker { + pub(crate) fn new(vm: &crate::mmu::Vm) -> Result> { + let mut owner = KBox::new( + Self { + vm: vm.clone(), + raw: core::ptr::null_mut(), + }, + GFP_KERNEL, + )?; + // SAFETY: The format is static and has no format arguments. + let raw = + unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; + if raw.is_null() { + return Err(ENOMEM); + } + owner.raw = raw; + // SAFETY: Newly allocated/unregistered shrinker is exclusively owned. + unsafe { + (*raw).count_objects = Some(Self::count); + (*raw).scan_objects = Some(Self::scan); + (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); + (*raw).seeks = 2; + (*raw).batch = 256; + kernel::bindings::shrinker_register(raw); + } + Ok(owner) + } + + unsafe extern "C" fn count( + raw: *mut kernel::bindings::shrinker, + _sc: *mut kernel::bindings::shrink_control, + ) -> kernel::ffi::c_ulong { + // SAFETY: The registration owns this stable allocation until callbacks drain. + let owner = unsafe { &*((*raw).private_data as *const Self) }; + owner.vm.idle_reclaim_count() as _ + } + + unsafe extern "C" fn scan( + raw: *mut kernel::bindings::shrinker, + sc: *mut kernel::bindings::shrink_control, + ) -> kernel::ffi::c_ulong { + // SAFETY: Both pointers are supplied by the registered shrinker core. + let owner = unsafe { &*((*raw).private_data as *const Self) }; + let target = unsafe { (*sc).nr_to_scan } as usize; + owner.vm.reclaim_idle_pages(target) as _ + } +} + +impl Drop for VmShrinker { + fn drop(&mut self) { + if !self.raw.is_null() { + // SAFETY: This owns the sole registration; free drains callbacks + // before the embedded VM and callback private_data can disappear. + unsafe { kernel::bindings::shrinker_free(self.raw) }; + } + } +} + +impl Vm { + pub(crate) fn close_idle_reclaim(&self) { + if let Some(gate) = self.residency.as_ref() { + gate.close(); + } + } + + pub(crate) fn residency_reclaim_enabled(&self) -> bool { + self.residency.is_some() + } + + pub(super) fn enter_residency_metadata(&self) -> Result> { + self.residency + .as_ref() + .map(Gate::enter_metadata) + .transpose() + } + + pub(super) fn enter_residency(&self) -> Result> { + self.residency + .as_ref() + .map(|gate| Gate::enter(gate, || self.restore_idle_pages())) + .transpose() + } + + pub(crate) fn idle_reclaim_count(&self) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + if !gate.idle() + || self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + { + return 0; + } + let Some(mut inner) = self.inner.try_lock_private() else { + return 0; + }; + let mut pages = 0usize; + // SAFETY: Counting changes no logical mapping/list. Private BOs share + // this reservation; linked aliases retain their borrowed BO and GEM. + let _ = unsafe { inner.for_each_private_bo(|_, bo| { + let object = bo.object(); + if object.idle_reclaim_candidate() && object.sole_sg_lease_releasable_locked() { + pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } + Ok(true) + }) }; + + pages + } + + pub(crate) fn reclaim_idle_pages(&self, target: usize) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + let Some(mut reclaim) = Gate::try_reclaim(gate) else { + return 0; + }; + // Error fences cannot prove firmware ownership ended. + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + || !self + .lifetime + .as_ref() + .is_some_and(|lifetime| lifetime.can_reclaim()) + { + return 0; + } + let Some(mut inner) = self.inner.try_lock_private() else { + return 0; + }; + let uat = inner.uat_inner.clone(); + let Some(shared) = uat.shared.try_lock() else { + return 0; + }; + let contexts = shared.contexts_naming_root(inner.ttb()); + let mut unpinned = 0usize; + // SAFETY: Exclusive residency and the private reservation exclude jobs, + // binding edits and retirement. Callbacks change PTEs/flags only; all + // logical aliases stay linked and retain their borrowed BO/GEM. + let _ = unsafe { inner.for_each_private_bo(|inner, bo| { + if unpinned >= target { return Ok(false); } + let object = bo.object(); + if !object.idle_reclaim_candidate() { return Ok(true); } + { + // Avoid inversion against preparation (VmBoInner -> resv). + let Some(bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; + if bo_inner.sgt.is_none() || !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } + } + // The association retains SG backing until *all* aliases have + // been invalidated and their device translations flushed. A + // contended cursor or PTE failure retains backing and the touched + // INVALIDATED prefix for restoration or a later reclaim attempt. + inner.invalidate_private_bo_mappings(bo, |inner, start, range| { + reclaim.mark_changed(); + let end = start.checked_add(range).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.discard_partial_map(start..end)?; + inner.tlbi_context_mask(start, range as usize, contexts); + Ok(()) + })?; + { + let Some(mut bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; + bo_inner.sg_vec = None; + bo_inner.sgt = None; + } + // Every private GPUVA alias was removed from the device page + // tables above. Kernel mappings and other SG owners are excluded; + // the C/Rust backing helper rechecks exclusion before release. + if object.release_idle_pages_locked().is_ok() { + unpinned = unpinned.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } + Ok(unpinned < target) + }) }; + drop(shared); + drop(inner); + + unpinned + } + + fn restore_idle_pages(&self) -> Result { + // Unbind/free can remove or split invalidated mappings while the VM + // stays evicted. Rebuild solely from the authoritative logical GPUVAs, + // never from stale pre-eviction records that could resurrect a BO. + // Gate::enter holds exclusive residency ownership for this entire + // walk: metadata edits, retirement and reclaim cannot change the tree. + // Retain one invalidated mapping at a time, avoiding a VM-sized KVec + // allocation precisely when memory pressure requires restoration. + let mut cursor = 0u64; + loop { + let map = loop { + if let Some(inner) = self.inner.try_lock_private() { + break inner.next_private_mapping( + &mut cursor, gpuvm::GpuVaFlags::INVALIDATED, + )?; + } + // Wait for reservation ownership without retaining a mapping. + drop(self.inner.exec_lock(None, true)?); + }; + // The reservation guard has dropped before pinning backing, and + // before any returned object reference can be released on failure. + let Some(map) = map else { break; }; + let mut restored = 0u64; + let bo_owner = self.inner.find_bo(&map.object).ok_or(EFAULT)?; + { + let needs_sgt = bo_owner.inner().inner.lock().sgt.is_none(); + if needs_sgt { + // Preserve review1's reservation/BO lock order: the getter + // may acquire the object reservation; never hold BO here. + let sgt = map.object.owned_sg_table()?; + let mut bo = bo_owner.inner().inner.lock(); + if bo.sgt.is_none() { bo.sgt = Some(sgt); } + drop(bo); + } + } + let mut inner = self.inner.exec_lock(None, true)?; + let bo = bo_owner.inner().inner.lock(); + let sgt = bo.sgt.as_ref().ok_or(EFAULT)?; + let result = (|| -> Result { + let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; + let _mutation = inner.mapping_mutation(); + inner.page_table.discard_partial_map(map.addr..end)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + let prot = map.inner.prot.ok_or(EINVAL)?; + let repeat = map.flags.contains(gpuvm::GpuVaFlags::REPEAT); + let mut skip = map + .offset + .checked_add(if repeat { 0 } else { restored }) + .ok_or(EOVERFLOW)? as usize; + let mut left = map.range.checked_sub(restored).ok_or(EOVERFLOW)? as usize; + for entry in sgt.iter() { + if left == 0 { + break; + } + let len = entry.dma_len() as usize; + if skip >= len { + skip -= len; + continue; + } + let physical = entry.dma_address() as usize + skip; + let bytes = if repeat { left } else { left.min(len - skip) }; + let start = map.addr.checked_add(restored).ok_or(EOVERFLOW)?; + let end = start.checked_add(bytes as u64).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.map_pages( + start..end, + physical as PhysicalAddr, + prot, + repeat, + )?; + restored += bytes as u64; + left -= bytes; + skip = 0; + } + if left != 0 { + return Err(EFAULT); + } + // The exclusive residency gate excludes submissions until + // every invalidated mapping is restored. Publish this logical + // range once, instead of issuing a synchronized TLBI for each + // physical SG segment. Keep the preceding break-before-make + // invalidation; the error path clears and flushes the entire + // range if any map operation (including a partial one) fails. + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + inner.set_invalidated(map.addr, map.range, false)?; + + Ok(()) + })(); + if let Err(error) = result { + // Keep an invalidated GPUVA wholly unmapped even on partial + // restoration failure, so later unbind/close cannot skip a + // prefix that was already reinstalled. Backing stays retained. + // map_pages may fail inside its current SG segment. Clear the + // complete logical range, including that partially written segment. + let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; + let _mutation = inner.mapping_mutation(); + inner.page_table.discard_partial_map(map.addr..end)?; + inner.tlbi_contexts_naming_root(map.addr, map.range as usize); + return Err(error); + } + } + Ok(()) + } + + pub(super) fn enter_retirement_residency(&self) -> Option { + self.residency.as_ref().and_then(Gate::enter_retirement) + } +} From 600e4eec21dd76e2ca9b3fa03f3d0b4f8aea78ef Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:01:06 -0400 Subject: [PATCH 078/631] drm/asahi: free a closed context's render pool once its release is consumed A logical context owns one render free-list pool (about 24 MiB of page and run tables plus 22 USC blocks). The pool was dropped only with the Context, but an installed compute graph keeps its last Context until a successor owner replaces it, so every idle compute QID pinned the exited client's render pool as well. On an 8 GB machine this grew kernel GEM from 252 MB (fresh desktop) to 752 MB after bursts of parallel clients and never shrank. The teardown service only publishes a scheduler release after every pool the context registered has had its free-list release consumed, and the context reaches the reclaim batch after that scheduler release is consumed too. Drop the pool there: nothing on the firmware side names it any more, and the retained compute binding never uses it. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e82b27902f91d8c833395a966366d0ffc30769c3) --- drivers/gpu/drm/asahi/g17/context.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index ee2d8133ea8cdf..eb4ddcac7b8089 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -273,6 +273,12 @@ impl Context { *render = Some(pool.clone()); Ok(pool) } + /// Drops the pool once its free-list release is consumed and the closed + /// scheduler is detached. Installed compute bindings may retain the + /// context itself until replacement; they never use the render pool. + pub(crate) fn take_render_pool(&self) -> Option> { + self.render.lock().take() + } pub(crate) fn render_pools_released(&self, released: impl Fn(u32, u64) -> bool) -> bool { self.render .lock() diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index b1495dbf3e23e3..f310f8154aae70 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -32,6 +32,7 @@ impl Reclaim { Self::Binding(binding) => drop(binding), Self::Context(context) => { context.release_execution(); + drop(context.take_render_pool()); drop(context); } } From f10bf3195d7977752cfcede4fa21cf0fc7ffa240 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:13:49 -0400 Subject: [PATCH 079/631] drm/asahi: hold idle descent for every first render pair install The first install of a physical render pair acquired the registration power hold only when the core power register read unpowered at that instant. A powered reading does not keep the firmware from starting idle descent before the host's configuration writes land. An earlier version of this driver with the same conditional hold hard-hung (watchdog reset, no log) when several clients installed their first pairs at once; with the hold taken unconditionally the same workloads ran clean. Acquire the hold before every first install and release it where the pair's pool is activated, as before. This costs a few tens of microseconds on a client's first render submission only. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fa210660c43c1e8ffc4c0705792febb70c29e779) --- drivers/gpu/drm/asahi/g17/runtime/render.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index e466571bc88c7b..bf86d46e7ca6bb 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -393,12 +393,15 @@ impl crate::g17::queue::render::Host for Host<'_> { .accounting .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) } + /// Idle descent is held for the whole first install. A point-in-time power + /// check let the firmware gate the cores between the host's configuration + /// writes while several clients installed their first pairs at once. fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { + if !self.registration_powered { + self.firmware.acquire_registration_power()?; + self.registration_powered = true; + } for _ in 0..3 { - if !self.firmware.bridge.regs.g17_configuration_powered() { - self.firmware.acquire_registration_power()?; - self.registration_powered = true; - } if self .firmware .bridge From b53e3bb0c77d0d72eb8dfd86962548c7a44962e7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:56:27 -0400 Subject: [PATCH 080/631] drm/asahi: add GEM_MADVISE and purge advised objects under memory pressure The kernel cannot tell which VM-private objects an in-flight job may reference, so pressure reclaim only touches idle VMs. Userspace knows: Mesa's BO cache holds freed objects that no pending or future work uses until they are handed out again. Add DRM_IOCTL_ASAHI_GEM_MADVISE with WILLNEED/DONTNEED advice for VM-private, non-exportable objects. The VM shrinker now also scans DONTNEED objects of busy VMs: it invalidates their aliases, clears the page tables and TLB, drops the VM's SG lease and purges the shmem backing. Restoration skips purged objects (their mappings stay INVALIDATED until unbound) and WILLNEED reports retained=0 for them so userspace reallocates. A purge that invalidates aliases but does not complete (contended GEM list, page-table error, or a backing lease that appeared meanwhile) records the eviction on the residency gate, so the next submission restores those aliases instead of leaving them unmapped behind a WILLNEED that reports the contents retained. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d7715ebbd2d3397958f08442d096a2476d32cb6a) --- drivers/gpu/drm/asahi/driver.rs | 2 + drivers/gpu/drm/asahi/file.rs | 17 ++++ drivers/gpu/drm/asahi/gem.rs | 9 ++ drivers/gpu/drm/asahi/mmu/residency.rs | 126 ++++++++++++++++++++++--- include/uapi/drm/asahi_drm.h | 39 ++++++++ rust/kernel/drm/gem/shmem.rs | 43 +++++++++ 6 files changed, 223 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index b4e7bb2a7267a0..45561ac17e2f5a 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -163,6 +163,8 @@ impl drm::driver::Driver for AsahiDriver { ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_destroy), (ASAHI_SUBMIT, drm_asahi_submit, ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), + (ASAHI_GEM_MADVISE, drm_asahi_gem_madvise, + ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_madvise), } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 88899e2ed7c837..ae40b1ca612cb3 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -732,6 +732,23 @@ impl File { Ok(0) } + /// IOCTL: gem_madvise: Mark a VM-private object purgeable or needed. + pub(crate) fn gem_madvise( + _device: &AsahiDevice, + data: &mut uapi::drm_asahi_gem_madvise, + file: &DrmFile, + ) -> Result { + if data.pad != 0 || data.madv > uapi::drm_asahi_gem_madv_DRM_ASAHI_GEM_MADV_DONTNEED { + return Err(EINVAL); + } + let gem = gem::Object::lookup_handle(file, data.handle)?; + data.retained = gem::madvise( + &gem, + data.madv == uapi::drm_asahi_gem_madv_DRM_ASAHI_GEM_MADV_DONTNEED, + )?; + Ok(0) + } + /// IOCTL: vm_bind: Map or unmap memory into a Vm. pub(crate) fn vm_bind( device: &AsahiDevice, diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index d05544bc49fb84..145ce6743b7161 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -366,3 +366,12 @@ impl AsahiObject { !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 } } + +/// Purge advice for a VM-private, non-exportable object. Returns whether the +/// contents are retained (0 once purged). +pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { + if gem.flags & uapi::drm_asahi_gem_flags_DRM_ASAHI_GEM_VM_PRIVATE == 0 || gem.exportable { + return Err(EINVAL); + } + Ok(u32::from(gem.madvise(i32::from(dontneed)))) +} diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index a66d9cff73e2d4..343da90957602b 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -146,6 +146,12 @@ impl Gate { .try_lock() .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) } + + /// A page-table mutation happened outside a `Reclaim` (the purge path runs + /// on busy VMs). The next `enter` restores every INVALIDATED mapping. + pub(crate) fn note_evicted(&self) { + self.state.lock().evicted = true; + } } impl Reclaim { @@ -276,14 +282,14 @@ impl Vm { let Some(gate) = self.residency.as_ref() else { return 0; }; - if !gate.idle() - || self - .status - .as_ref() - .map_or(true, |status| status.get() != 0) + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) { return 0; } + let idle = gate.idle(); let Some(mut inner) = self.inner.try_lock_private() else { return 0; }; @@ -292,7 +298,11 @@ impl Vm { // this reservation; linked aliases retain their borrowed BO and GEM. let _ = unsafe { inner.for_each_private_bo(|_, bo| { let object = bo.object(); - if object.idle_reclaim_candidate() && object.sole_sg_lease_releasable_locked() { + // Purgeable objects count on busy VMs too; idle backing only on idle ones. + if object.idle_reclaim_candidate() + && (object.madv_locked() > 0 || idle) + && object.sole_sg_lease_releasable_locked() + { pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(true) @@ -301,12 +311,94 @@ impl Vm { pages } + /// Discard the backing of objects userspace marked DONTNEED. This needs + /// only the VM reservation: binding edits and restoration hold it too, + /// and the advice guarantees no pending or future GPU reference until + /// the next WILLNEED. Purged mappings stay INVALIDATED until unbound. + fn purge_advised_pages(&self, target: usize) -> usize { + let Some(gate) = self.residency.as_ref() else { + return 0; + }; + if self + .status + .as_ref() + .map_or(true, |status| status.get() != 0) + { + return 0; + } + let Some(mut inner) = self.inner.try_lock_private() else { + return 0; + }; + let uat = inner.uat_inner.clone(); + let Some(shared) = uat.shared.try_lock() else { + return 0; + }; + let contexts = shared.contexts_naming_root(inner.ttb()); + let mut purged = 0usize; + // SAFETY: The private reservation excludes binding edits and restoration. + // Callbacks change PTEs/flags only; all aliases stay linked. + let _ = unsafe { inner.for_each_private_bo(|inner, bo| { + if purged >= target { return Ok(false); } + let object = bo.object(); + if !object.idle_reclaim_candidate() || object.madv_locked() <= 0 { + return Ok(true); + } + { + let Some(bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; + if bo_inner.sgt.is_some() && !object.sole_sg_lease_releasable_locked() { + return Ok(true); + } + } + // Any mutation that does not end in a completed purge leaves + // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so + // the next WILLNEED would report the contents retained. Record the + // eviction so the gate restores those aliases before the next job. + let mut mutated = false; + let invalidated = inner.invalidate_private_bo_mappings(bo, |inner, start, range| { + mutated = true; + let end = start.checked_add(range).ok_or(EOVERFLOW)?; + let _change = inner.mapping_mutation(); + inner.page_table.discard_partial_map(start..end)?; + inner.tlbi_context_mask(start, range as usize, contexts); + Ok(()) + }); + if let Err(error) = invalidated { + if mutated { + gate.note_evicted(); + } + return Err(error); + } + { + let Some(mut bo_inner) = bo.inner().inner.try_lock() else { + gate.note_evicted(); + return Ok(true); + }; + bo_inner.sg_vec = None; + bo_inner.sgt = None; + } + if object.purge_locked().is_ok() { + purged = purged.saturating_add(object.size() / kernel::page::PAGE_SIZE); + } else { + gate.note_evicted(); + } + Ok(purged < target) + }) }; + drop(shared); + drop(inner); + purged + } + pub(crate) fn reclaim_idle_pages(&self, target: usize) -> usize { let Some(gate) = self.residency.as_ref() else { return 0; }; + let purged = self.purge_advised_pages(target); + if purged >= target { + return purged; + } + let target = target - purged; let Some(mut reclaim) = Gate::try_reclaim(gate) else { - return 0; + return purged; }; // Error fences cannot prove firmware ownership ended. if self @@ -318,14 +410,14 @@ impl Vm { .as_ref() .is_some_and(|lifetime| lifetime.can_reclaim()) { - return 0; + return purged; } let Some(mut inner) = self.inner.try_lock_private() else { - return 0; + return purged; }; let uat = inner.uat_inner.clone(); let Some(shared) = uat.shared.try_lock() else { - return 0; + return purged; }; let contexts = shared.contexts_naming_root(inner.ttb()); let mut unpinned = 0usize; @@ -371,7 +463,7 @@ impl Vm { drop(shared); drop(inner); - unpinned + purged + unpinned } fn restore_idle_pages(&self) -> Result { @@ -386,16 +478,24 @@ impl Vm { loop { let map = loop { if let Some(inner) = self.inner.try_lock_private() { - break inner.next_private_mapping( + let map = inner.next_private_mapping( &mut cursor, gpuvm::GpuVaFlags::INVALIDATED, )?; + // Purged (DONTNEED) objects have no contents to restore; + // their mappings stay INVALIDATED until unbound. + // SAFETY: The private guard holds the VM reservation. + let purged = map.as_ref().is_some_and(|map| unsafe { map.object.madv_locked() < 0 }); + break (map, purged); } // Wait for reservation ownership without retaining a mapping. drop(self.inner.exec_lock(None, true)?); }; // The reservation guard has dropped before pinning backing, and // before any returned object reference can be released on failure. - let Some(map) = map else { break; }; + let (Some(map), purged) = map else { break; }; + if purged { + continue; + } let mut restored = 0u64; let bo_owner = self.inner.find_bo(&map.object).ok_or(EFAULT)?; { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 588226e72381b4..471b1ba2f8b331 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -93,6 +93,9 @@ enum drm_asahi_ioctl_id { /** @DRM_ASAHI_SUBMIT: Submit commands to a queue. */ DRM_ASAHI_SUBMIT, + + /** @DRM_ASAHI_GEM_MADVISE: Mark a VM-private object purgeable or needed. */ + DRM_ASAHI_GEM_MADVISE, }; #define DRM_ASAHI_MAX_CLUSTERS 64 @@ -649,6 +652,41 @@ struct drm_asahi_gem_mmap_offset { __u64 offset; }; +/** + * enum drm_asahi_gem_madv - Advice for DRM_IOCTL_ASAHI_GEM_MADVISE + */ +enum drm_asahi_gem_madv { + /** @DRM_ASAHI_GEM_MADV_WILLNEED: The object's contents are needed. */ + DRM_ASAHI_GEM_MADV_WILLNEED = 0, + /** + * @DRM_ASAHI_GEM_MADV_DONTNEED: The object is not referenced by any + * pending or future GPU work until the next WILLNEED. The kernel may + * discard its contents under memory pressure. + */ + DRM_ASAHI_GEM_MADV_DONTNEED = 1, +}; + +/** + * struct drm_asahi_gem_madvise - Arguments passed to + * DRM_IOCTL_ASAHI_GEM_MADVISE + * + * Only VM-private objects accept advice. A purged object keeps its handle + * and mappings but has no contents; userspace must not use it again. + */ +struct drm_asahi_gem_madvise { + /** @handle: Handle of the object. */ + __u32 handle; + + /** @madv: One of enum drm_asahi_gem_madv. */ + __u32 madv; + + /** @retained: Set by the kernel: 1 if the contents are retained, 0 if purged. */ + __u32 retained; + + /** @pad: Must be zero. */ + __u32 pad; +}; + /** * enum drm_asahi_bind_flags - Flags for GEM binding */ @@ -1615,6 +1653,7 @@ enum { DRM_IOCTL_ASAHI_QUEUE_CREATE = DRM_IOCTL_ASAHI(WR, QUEUE_CREATE, queue_create), DRM_IOCTL_ASAHI_QUEUE_DESTROY = DRM_IOCTL_ASAHI(W, QUEUE_DESTROY, queue_destroy), DRM_IOCTL_ASAHI_SUBMIT = DRM_IOCTL_ASAHI(W, SUBMIT, submit), + DRM_IOCTL_ASAHI_GEM_MADVISE = DRM_IOCTL_ASAHI(WR, GEM_MADVISE, gem_madvise), }; #if defined(__cplusplus) diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 31841dd8f35208..2b3a3c2c4a1860 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -330,6 +330,49 @@ impl Object { result } + /// Apply purge advice (0 = needed, >0 = may be purged) under the + /// object's reservation lock. Returns false if the object was already + /// purged; the advice is then ignored. + pub fn madvise(&self, madv: i32) -> bool { + // SAFETY: The object retains its reservation; the C helper requires it held. + unsafe { + bindings::dma_resv_lock(self.raw_dma_resv(), core::ptr::null_mut()); + let retained = bindings::drm_gem_shmem_madvise_locked(self.as_raw_shmem(), madv) != 0; + bindings::dma_resv_unlock(self.raw_dma_resv()); + retained + } + } + + /// Current purge advice: negative once purged. + /// + /// # Safety + /// The caller holds the object's DMA reservation lock. + pub unsafe fn madv_locked(&self) -> i32 { + // SAFETY: Reservation ownership serializes the field. + unsafe { (*self.as_raw_shmem()).madv } + } + + /// Discard the backing of a purgeable object. The handle and logical + /// mappings remain; the contents are gone and `madv_locked()` turns negative. + /// + /// # Safety + /// + /// The caller holds this object's DMA reservation lock, has removed and + /// invalidated every device mapping, and no Rust SG lease remains. + pub unsafe fn purge_locked(&self) -> Result { + // SAFETY: The caller's contract; the exclusive state below excludes new leases. + if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { + return Err(EBUSY); + } + self.sgt_access.compare_exchange( + 0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire, + ).map_err(|_| EBUSY)?; + // SAFETY: See above. + unsafe { bindings::drm_gem_shmem_purge_locked(self.as_raw_shmem()) }; + self.sgt_access.store(0, Ordering::Release); + Ok(()) + } + /// Attempt to create a [`RawIoSysMap`] from the gem object. fn raw_vmap(&self) -> Result> { build_assert!( From 490d1dff1dc6dc351ff63685ff2f832f6a756798 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:57:01 -0400 Subject: [PATCH 081/631] drm/asahi: drop the sysfs object and queue-slot counters gem_stats and g17p_queue_slots exported driver-internal counters as read-only module parameters, backed by global atomics on the GEM constructor/destructor, the QID lease table, compute queue state transitions, the teardown and settle passes and every ENOSPC return. Nothing in the driver reads them, and DRM exposes memory accounting through fdinfo. Remove the parameters, the atomics and the hooks; an exhausted pool returns ENOSPC without a counted log line. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c8c442ae6a83502a90fca56d1b8e08e669ce03cb) --- drivers/gpu/drm/asahi/g17/context.rs | 2 +- drivers/gpu/drm/asahi/g17/kick.rs | 49 ++++------------------------ drivers/gpu/drm/asahi/gem.rs | 16 ++------- 3 files changed, 9 insertions(+), 58 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index eb4ddcac7b8089..b7d0a351f234d8 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -39,7 +39,7 @@ impl QosIds { let mut used = self.used.lock(); let id = used.trailing_ones(); if id == 128 { - return Err(crate::g17::queue_stats::enospc(crate::g17::queue_stats::Pool::QueueContextQos)); + return Err(ENOSPC); } *used |= 1u128 << id; Ok(QosLease { diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index aa2c054a9bbc7b..f109c8261476e3 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -6,7 +6,7 @@ //! and their ring backing stay owned until both firmware processors stop; //! command completion alone does not detach a hardware queue. -use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::{fence, Ordering}; use kernel::prelude::*; use super::{ @@ -17,28 +17,8 @@ use crate::{hw::t8140, mem, mmu}; pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; -pub(super) const COMPUTE_RENDER_RESERVE: usize = 16; -pub(super) static LEASED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; -pub(super) static PUBLISHED: [AtomicU32; 3] = [const { AtomicU32::new(0) }; 3]; -pub(super) static HIGH_WATER: AtomicU32 = AtomicU32::new(0); -pub(super) static EXHAUSTED: AtomicU64 = AtomicU64::new(0); -pub(super) static RESERVE_REFUSED: AtomicU64 = AtomicU64::new(0); - -pub(super) fn leased_total() -> u32 { - LEASED.iter().map(|count| count.load(Ordering::Relaxed)).sum() -} - -fn exhausted(engine: DataMaster, reserve: bool) -> Error { - if reserve { - RESERVE_REFUSED.fetch_add(1, Ordering::Relaxed); - } else { - EXHAUSTED.fetch_add(1, Ordering::Relaxed); - } - super::queue_stats::enospc(match engine { - DataMaster::Compute => super::queue_stats::Pool::QidCompute, - _ => super::queue_stats::Pool::QidRenderPair, - }) -} +/// QIDs kept free for render pairs when compute queues fill the table. +const COMPUTE_RENDER_RESERVE: usize = 16; /// Stable physical-channel identity. The owner must not be reused within a /// device lifetime, including after an unpublished allocation is cancelled. @@ -106,13 +86,13 @@ impl QueueIds { // Healthy existing channels keep their QID. Never recycle a published // identity to recover capacity after a fault; leave eight render pairs. if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(exhausted(engine, true)); + return Err(ENOSPC); } let index = preferred .map(usize::from) .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or_else(|| exhausted(engine, false))?; + .ok_or(ENOSPC)?; let id = Id { qid: index as u8, owner, @@ -123,8 +103,6 @@ impl QueueIds { published: false, }); self.leased += 1; - LEASED[engine as usize].fetch_add(1, Ordering::Relaxed); - HIGH_WATER.fetch_max(leased_total(), Ordering::Relaxed); Ok(id) } @@ -145,10 +123,7 @@ impl QueueIds { if entry.id != id { return Err(EINVAL); } - if !entry.published { - entry.published = true; - PUBLISHED[id.engine as usize].fetch_add(1, Ordering::Relaxed); - } + entry.published = true; Ok(()) } @@ -158,7 +133,6 @@ impl QueueIds { Some(entry) if entry.id == id && !entry.published => { *slot = None; self.leased -= 1; - LEASED[id.engine as usize].fetch_sub(1, Ordering::Relaxed); Ok(()) } _ => Err(EINVAL), @@ -166,17 +140,6 @@ impl QueueIds { } } -impl Drop for QueueIds { - fn drop(&mut self) { - for entry in self.entries.iter().flatten() { - LEASED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); - if entry.published { - PUBLISHED[entry.id.engine as usize].fetch_sub(1, Ordering::Relaxed); - } - } - } -} - // Aliases drop before their backing. A compute queue is allocated low first; // a render queue is allocated high first. enum Backing { diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 145ce6743b7161..fb3d5a7f91ce34 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -40,7 +40,7 @@ use crate::{ const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; /// Represents the inner data of a GEM object for this driver. -#[pin_data(PinnedDrop)] +#[pin_data] pub(crate) struct AsahiObject { /// ID for debug id: u64, @@ -178,13 +178,6 @@ impl ObjectRef { } } -#[pinned_drop] -impl PinnedDrop for AsahiObject { - fn drop(self: Pin<&mut Self>) { - crate::gem_stats::note_free(self.kernel, self.size); - } -} - pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, @@ -295,12 +288,7 @@ impl DriverObject for AsahiObject { mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, - // All remaining fields are infallible. PinnedDrop balances the - // accounting once this initializer completes. - size: { - crate::gem_stats::note_new(args.kernel, size); - size - }, + size, flags: args.flags, exportable: args.exportable, kernel: args.kernel, From 4a8a51e1ffd167443e160627707287ddbb7e234c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:57:43 -0400 Subject: [PATCH 082/631] drm/asahi: drop the stalled scheduler-release warning A release whose firmware acknowledgement took longer than two seconds logged "release is still pending" once and otherwise changed nothing: the entry stayed retained until the acknowledgement arrived. Hardware runs never produced the line. Keep the acknowledgement state machine and its polling window; remove the warning bookkeeping. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 152092e32a7fd7a7b3a96b8e45be7db78934b7d8) --- drivers/gpu/drm/asahi/g17/teardown.rs | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index f53c2d45c129ab..791ac690c0f50b 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -22,17 +22,15 @@ struct Entry { context: Arc, phase: Phase, armed: u64, - warned: bool, announce_pending: bool, } -/// Publication and announcement errors retain the entry. A stalled release is reported once; -/// its timeout never authorizes freeing memory still owned by firmware. +/// Publication and announcement errors retain the entry; no timeout frees +/// memory the firmware may still own. #[derive(Clone, Copy)] pub(crate) enum Failure { Publication(Error), Announcement(Error), - Stalled, } /// Borrowed device operations. Readiness covers every matching installed and displaced graph, @@ -88,7 +86,6 @@ impl Pending { context: context.clone(), phase, armed: now, - warned: false, announce_pending: false, }) .map_err(|_| ENOSPC) @@ -143,7 +140,6 @@ impl Pending { // One successful retry starts its consumption window. // Failed retries never slide the polling deadline. entry.armed = now; - entry.warned = false; } continue; } @@ -177,7 +173,6 @@ impl Pending { // A long command drain must not consume the acknowledgement // polling window before this release exists on the control ring. entry.armed = now; - entry.warned = false; entry.announce_pending = true; fence(Ordering::SeqCst); match host.notify() { @@ -228,16 +223,4 @@ impl Pending { entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS }) } - - pub(crate) fn report_stalled(&mut self, now: u64, host: &mut impl Host) { - for entry in self.entries.iter_mut() { - if entry.phase != Phase::Deferred - && !entry.warned - && now.saturating_sub(entry.armed) >= POLL_NS - { - entry.warned = true; - host.report(&entry.context, Failure::Stalled); - } - } - } } From 57ff7b8dc9e19c1b1efb028be80ef1299f5084e4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 11:15:54 -0400 Subject: [PATCH 083/631] drm/asahi: build constant firmware objects from word tables PowerState, SecondaryConfig, PowerConfigHead, PowerConfigTail and HwDataAux are written once from constants and never read by the host. Their 600-odd unnamed fields, offset assertions and field-by-field builders are replaced by byte arrays filled from (offset, value, repeat) word tables, plus the one address HwDataAux carries. The five objects are byte-identical to the previous builders for every input (verified with a host-side dump of each object). Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 24ce1bcdd8ebf0f8664c043a639c0201286f94e5) --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 1375 ++++++---------------- 1 file changed, 341 insertions(+), 1034 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index c25b843a913e72..8dbb6a81bcd470 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1406,187 +1406,7 @@ impl StatusBlock { } } -/// Power state the primary status B carries for the secondary instance, -/// named by [`SecondaryRoot::power_state_va`]. -#[repr(C, packed)] -pub(crate) struct PowerState { - pub(crate) unk_0: [u8; 0x10], - /// Required: 0xf (15). - pub(crate) unk_10: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_14: u32, - /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). - pub(crate) unk_18: u32, - /// Required: 0x28 (40). - pub(crate) unk_1c: u32, - /// Required: 1. - pub(crate) unk_20: u32, - pub(crate) unk_24: [u8; 0xa0], - /// Non-zero asks the secondary to reload the limiter settings that - /// follow into the power configuration; zero at boot. - pub(crate) temp_update: u32, - /// Target temperature of the limiter ([`TEMP_TARGET`]). - pub(crate) temp_target: u32, - /// Release temperature of the limiter ([`TEMP_RELEASE`]). - pub(crate) temp_release: u32, - pub(crate) unk_d0: [u8; 8], - /// Proportional gain of the limiter ([`TEMP_KP`]). - pub(crate) temp_kp: u32, - /// Integral gain of the limiter ([`TEMP_KI`]). - pub(crate) temp_ki: u32, - pub(crate) unk_e0: [u8; 0x10], - /// Required: 0x26ac (9900). - pub(crate) unk_f0: u32, - /// Required: 0xc8 (200). - pub(crate) unk_f4: u32, - pub(crate) unk_f8: [u8; 0x10], - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_108: u32, - /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). - pub(crate) unk_10c: u32, - /// Required: 0xc8 (200). - pub(crate) unk_110: u32, - pub(crate) unk_114: [u8; 0x14], - /// Required: 1. - pub(crate) unk_128: u32, - /// Required: 0x2616 (9750). - pub(crate) unk_12c: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_130: u32, - /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). - pub(crate) unk_134: u32, - pub(crate) unk_138: [u8; 0x14], - /// Required: 1. - pub(crate) unk_14c: u32, - pub(crate) unk_150: [u8; 0x70], - /// Required: 1. - pub(crate) unk_1c0: u32, - /// Required: 1. - pub(crate) unk_1c4: u32, - /// Required: 4. - pub(crate) unk_1c8: u32, - /// Required: 1 in every entry. - pub(crate) unk_1cc: [u32; 4], - pub(crate) unk_1dc: [u8; 0x308], - /// Required: 1. - pub(crate) unk_4e4: u32, - /// Required: 0x1f4 (500). - pub(crate) unk_4e8: u32, - /// Required: 6. - pub(crate) unk_4ec: u32, - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_4f0: [u32; 6], - pub(crate) unk_508: [u8; 0x14], - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_51c: [u32; 5], - /// Required: 1. - pub(crate) unk_530: u32, - pub(crate) unk_534: [u8; 0x58], - /// Required: 6. - pub(crate) unk_58c: u32, - /// Required: 0xdac (3500). - pub(crate) unk_590: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_594: u32, - /// Required: 0xbb8 (3000). - pub(crate) unk_598: u32, - /// Required: 0x64 (100). - pub(crate) unk_59c: u32, - pub(crate) unk_5a0: [u8; 0x2c], - /// Required: 1. - pub(crate) unk_5cc: u32, - /// Required: 4. - pub(crate) unk_5d0: u32, - pub(crate) unk_5d4: [u8; 8], - /// Required: 0x1f40 (8000). - pub(crate) unk_5dc: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5e0: u32, - /// Required: 0xfa0 (4000). - pub(crate) unk_5e4: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5e8: u32, - /// Required: 0x7d0 (2000). - pub(crate) unk_5ec: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_5f4: u32, - /// Required: 0xc8 (200). - pub(crate) unk_5f8: u32, - pub(crate) unk_5fc: [u8; 8], - /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). - pub(crate) unk_604: u32, - /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). - pub(crate) unk_608: u32, - /// Required: 0x20 (32). - pub(crate) unk_60c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_610: u32, - pub(crate) unk_614: [u8; 0xc], - /// Required: 0x384 (900). - pub(crate) unk_620: u32, - pub(crate) unk_624: [u8; 0x9c], -} - -// SAFETY: `PowerState` consists of integers and arrays of integers only. -unsafe impl Zeroable for PowerState {} -static_assert!(size_of::() == 0x6c0); - -impl PowerState { - fn init(&mut self) { - self.unk_10 = 0xf; - self.unk_14 = 0x3f00_0000; - self.unk_18 = 0x4088_0000; - self.unk_1c = 0x28; - self.unk_20 = 1; - self.temp_target = TEMP_TARGET; - self.temp_release = TEMP_RELEASE; - self.temp_kp = TEMP_KP; - self.temp_ki = TEMP_KI; - self.unk_f0 = 0x26ac; - self.unk_f4 = 0xc8; - self.unk_108 = 0x42c8_0000; - self.unk_10c = 0x43c8_0000; - self.unk_110 = 0xc8; - self.unk_128 = 1; - self.unk_12c = 0x2616; - self.unk_130 = 0x3f00_0000; - self.unk_134 = 0x40cc_cccd; - self.unk_14c = 1; - self.unk_1c0 = 1; - self.unk_1c4 = 1; - self.unk_1c8 = 4; - self.unk_1cc = [1; 4]; - self.unk_4e4 = 1; - self.unk_4e8 = 0x1f4; - self.unk_4ec = 6; - self.unk_4f0 = [0x30d4; 6]; - self.unk_51c = [0x30d4; 5]; - self.unk_530 = 1; - self.unk_58c = 6; - self.unk_590 = 0xdac; - self.unk_594 = 0x3e8; - self.unk_598 = 0xbb8; - self.unk_59c = 0x64; - self.unk_5cc = 1; - self.unk_5d0 = 4; - self.unk_5dc = 0x1f40; - self.unk_5e0 = 0xc8; - self.unk_5e4 = 0xfa0; - self.unk_5e8 = 0xc8; - self.unk_5ec = 0x7d0; - self.unk_5f0 = 0xc8; - self.unk_5f4 = 0x3e8; - self.unk_5f8 = 0xc8; - self.unk_604 = 0x4170_0000; - self.unk_608 = 0x40a0_0000; - self.unk_60c = 0x20; - self.unk_610 = 0x3e8; - self.unk_620 = 0x384; - } -} /// Size of the firmware-control page named by the primary status B. pub(crate) const FWCTL_SIZE: usize = 0x4000; @@ -1667,46 +1487,7 @@ impl PrimaryStatusB { } } -/// Configuration of the secondary instance, named by -/// [`SecondaryRoot::config_va`]. -#[repr(C)] -pub(crate) struct SecondaryConfig { - pub(crate) unk_0: [u8; 0x14], - /// Required: 1. - pub(crate) unk_14: u32, - pub(crate) unk_18: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2c: u32, - /// Required: 1. - pub(crate) unk_30: u32, - pub(crate) unk_34: [u8; 0xc], - /// Required: 1. - pub(crate) unk_40: u32, - pub(crate) unk_44: [u8; 8], - /// Required: 1. - pub(crate) unk_4c: u32, - /// Required: 0x186a (6250). - pub(crate) unk_50: u32, - pub(crate) unk_54: [u8; 0x2c], -} - -static_assert!(size_of::() == 0x80); - -// SAFETY: `SecondaryConfig` consists of integers and arrays of integers only. -unsafe impl Zeroable for SecondaryConfig {} -impl SecondaryConfig { - /// Initializes the secondary configuration. - pub(crate) fn init(&mut self) { - clear(self); - self.unk_14 = 1; - self.unk_2c = 1; - self.unk_30 = 1; - self.unk_40 = 1; - self.unk_4c = 1; - self.unk_50 = 0x186a; - } -} /// Offsets in the private cluster. pub(crate) mod private { @@ -1751,295 +1532,332 @@ const TEMP_KP: u32 = 0x41a2_b852; /// single. const TEMP_KI: u32 = 0x4116_0419; +/// Nonzero 32-bit words of a firmware object the host builds from constants: (byte offset, value, repeat). +/// Every other word is zero. Generated from the previously field-by-field builders; the fwdump harness +/// (`neo-smooth-20261007/kernel/fwdump`) verifies the objects byte for byte. +type Words = &'static [(usize, u32, usize)]; + +fn fill_words(bytes: &mut [u8], words: Words) { + bytes.fill(0); + for &(offset, value, repeat) in words { + for index in 0..repeat { + let at = offset + 4 * index; + bytes[at..at + 4].copy_from_slice(&value.to_le_bytes()); + } + } +} + +const POWER_CONFIG_HEAD_WORDS: Words = &[ + (0x4, 0x5dc00, 2), + (0x10, 4, 1), + (0x14, 0x3f800000, 1), + (0x2c, 1, 2), + (0x40, 0x64, 1), + (0x44, 1, 1), + (0x48, 0x3e8, 1), + (0x50, 0x64, 1), + (0x60, 4, 1), + (0x64, 0x3f800000, 1), + (0x6c, 1, 2), + (0x80, 0x64, 1), + (0x84, 1, 1), + (0x88, 0x3e8, 1), + (0x90, 0x64, 1), + (0xc0, 0x3f828f5c, 11), + (0x9b8, 0x271, 1), + (0x9c4, 0x3f7f2e9f, 1), + (0x9cc, 0x3b516154, 1), + (0x9d4, 0x3ca59586, 1), + (0x9e0, 0x46153800, 1), + (0x9e4, 0x40a90fdb, 1), + (0x9ec, 0xbdd67344, 1), + (0x9f0, 0x28, 1), + (0x9f4, 0x3e8, 2), + (0xa04, 0x254e, 1), + (0xa38, 0x3e8, 1), + (0xa70, 4, 1), + (0xa84, 0x3f800000, 1), + (0xa8c, 0x40cccccd, 1), + (0xa98, 0x47800000, 1), + (0xa9c, 0x3f000000, 1), + (0xaa8, 0x28, 1), + (0xaac, 0x3e8, 1), + (0xabc, 0x254e, 1), + (0xac8, 0x10, 1), + (0xad0, 0x5dc00, 1), + (0xb28, 0x5c, 1), + (0xb30, 0x64, 1), + (0xb34, 0x22, 1), + (0xb38, 6, 1), + (0xb40, 6, 1), + (0xb44, 1, 1), + (0xb4c, 0x3f4ccccd, 1), + (0xb50, 0x3f7df3b6, 1), + (0xb54, 0x3e4ccccd, 1), + (0xb58, 0x3c03126f, 1), + (0xb5c, 0x3f69d4d8, 2), + (0xb68, 0x42be0000, 1), + (0xb6c, 0x4064f5c3, 2), + (0xb74, 0x41179436, 1), + (0xb78, 0x64, 1), + (0xb7c, 0x3e8, 1), + (0xb80, 0x64, 1), + (0xb8c, 0x5c, 1), + (0xbc0, 0x64, 1), + (0xbe0, 0x401, 1), + (0xbe4, 0x6400, 1), + (0xbec, 0x1f4000, 1), + (0xbf0, 0xc800, 1), + (0xbf4, 0xfa000, 1), + (0xbf8, 0xc800, 1), + (0xbfc, 0x7d000, 1), + (0xc00, 0xc800, 1), + (0xc04, 0x3e800, 1), + (0xc08, 0xc800, 1), + (0xc0c, 0x100, 1), + (0xc64, 1, 1), + (0xc90, 1, 1), + (0x1c94, 0xff000000, 1), + (0x1c98, 0xff7fff7f, 74), +]; + +const POWER_CONFIG_TAIL_WORDS: Words = &[ + (0x0, 0xff7fff7f, 181), + (0x2d4, 0x7fff7f, 1), + (0x4d8, 0x47800000, 1), + (0x4dc, 0x42200000, 1), + (0x4e0, 0x447a0000, 1), + (0x4e4, 0x5be, 1), + (0x4f8, 0x28, 1), + (0x500, 0x42c80000, 1), + (0x504, 0x3e8, 1), + (0x50c, 0x3f4ccccd, 1), + (0x510, 0x3e4ccccd, 1), + (0x550, TEMP_SENSOR_MASK as u32, 1), + (0x558, TEMP_RELEASE, 1), + (0x560, 4, 1), + (0x574, 0x3f800000, 1), + (0x57c, TEMP_KI, 1), + (0x588, 0x47800000, 1), + (0x58c, TEMP_KP, 1), + (0x598, 0x28, 1), + (0x59c, 0x3e8, 2), + (0x5ac, TEMP_TARGET, 1), + (0x5e0, 0x3e8, 1), + (0x7fc, 0x42000006, 1), + (0xa10, 0xb8520000, 1), + (0xa14, 0x3f7e, 1), + (0xa18, 0xd70a0000, 1), + (0xa1c, 0x3ba3, 1), + (0xa24, 0x43c8, 1), + (0xa30, 0x4780, 1), + (0xa34, 0x42c8, 1), + (0xa3c, 0x384bac8, 1), + (0xa40, 0x3e80000, 2), + (0xa4c, 0x80000000, 1), + (0xa50, 0x26ac453b, 1), + (0xa84, 0x3e80000, 1), + (0xaa0, 0xc80000, 1), + (0xb58, 0x3e8, 1), + (0xb6c, 1, 1), + (0xba0, 0x101, 1), + (0xba4, 0x400, 1), + (0xbbc, 0x23f00, 1), + (0xbc0, 0x100, 2), + (0xbdc, 0x24000, 1), + (0xbe0, 0x100, 2), + (0xbfc, 0x24100, 1), + (0xc48, 1, 1), + (0xc50, 0x254e, 3), + (0xc68, 0x3f800000, 1), + (0xc6c, 4, 1), + (0xc70, 0x10, 1), + (0xc74, 0x5dc00, 1), + (0xd44, 0x3c, 1), + (0xd50, 0x3f6eeeef, 1), + (0xd58, 0x3d888889, 1), + (0xd60, 0x3f000000, 1), + (0xd6c, 0x47800000, 1), + (0xd70, 0x40880000, 1), + (0xd7c, 0x28, 1), + (0xd80, 0x3e8, 2), + (0xd8c, 0x46153800, 1), + (0xd90, 0x254e, 1), + (0xd9c, 0xf0, 1), + (0xda4, 0x57e400, 1), + (0xdc4, 0x3e8, 1), + (0xe04, 1, 1), + (0xed4, 0x32, 1), + (0xed8, 1, 1), + (0xee0, 0x3f638e39, 1), + (0xee4, 0x3f2aaaab, 1), + (0xee8, 0x3de38e39, 1), + (0xeec, 0x3eaaaaab, 1), + (0xef0, 0xbf4ccccd, 2), + (0xefc, 0x47800000, 1), + (0xf00, 0xc0a00000, 2), + (0xf0c, 0x64, 1), + (0xf10, 0x3e8, 1), + (0xf14, 0x64, 1), + (0xf1c, 0x467a0000, 1), + (0xf20, 0x578, 1), + (0xf2c, 0x90, 1), + (0xf30, 0x30, 1), + (0xf34, 0x34bc00, 1), + (0xf3c, 0x119400, 1), + (0xf48, 0x47800000, 1), + (0xf70, 0x3e80, 1), + (0xf74, 2, 1), + (0xf78, 0x9c4, 1), + (0xf7c, 0x20d, 1), + (0xf80, 2, 1), + (0xf84, 4, 1), + (0xf88, 0x32, 1), + (0xfb8, 0x47800000, 1), + (0xfc8, 0x28, 1), + (0xfcc, 0x3e8, 1), +]; + +const POWER_STATE_WORDS: Words = &[ + (0x10, 0xf, 1), + (0x14, 0x3f000000, 1), + (0x18, 0x40880000, 1), + (0x1c, 0x28, 1), + (0x20, 1, 1), + (0xc8, TEMP_TARGET, 1), + (0xcc, TEMP_RELEASE, 1), + (0xd8, TEMP_KP, 1), + (0xdc, TEMP_KI, 1), + (0xf0, 0x26ac, 1), + (0xf4, 0xc8, 1), + (0x108, 0x42c80000, 1), + (0x10c, 0x43c80000, 1), + (0x110, 0xc8, 1), + (0x128, 1, 1), + (0x12c, 0x2616, 1), + (0x130, 0x3f000000, 1), + (0x134, 0x40cccccd, 1), + (0x14c, 1, 1), + (0x1c0, 1, 2), + (0x1c8, 4, 1), + (0x1cc, 1, 4), + (0x4e4, 1, 1), + (0x4e8, 0x1f4, 1), + (0x4ec, 6, 1), + (0x4f0, 0x30d4, 6), + (0x51c, 0x30d4, 5), + (0x530, 1, 1), + (0x58c, 6, 1), + (0x590, 0xdac, 1), + (0x594, 0x3e8, 1), + (0x598, 0xbb8, 1), + (0x59c, 0x64, 1), + (0x5cc, 1, 1), + (0x5d0, 4, 1), + (0x5dc, 0x1f40, 1), + (0x5e0, 0xc8, 1), + (0x5e4, 0xfa0, 1), + (0x5e8, 0xc8, 1), + (0x5ec, 0x7d0, 1), + (0x5f0, 0xc8, 1), + (0x5f4, 0x3e8, 1), + (0x5f8, 0xc8, 1), + (0x604, 0x41700000, 1), + (0x608, 0x40a00000, 1), + (0x60c, 0x20, 1), + (0x610, 0x3e8, 1), + (0x620, 0x384, 1), +]; + +const HW_DATA_AUX_WORDS: Words = &[ + (0xac, 6, 1), + (0xb8, 1, 1), + (0x174, 0xac000000, 1), + (0x178, 0xe800000d, 1), + (0x17c, 0xb8000003, 1), + (0x180, 0xe800000b, 1), + (0x184, 0xe8000003, 1), + (0x188, 0x64000003, 1), + (0x18c, 0x20000000, 1), + (0x1b4, 4, 1), + (0x1c0, 0x3f780000, 1), + (0x1c8, 0x3d000000, 1), + (0x1d0, 0x40a00000, 1), + (0x1dc, 0x47800000, 1), + (0x1e0, 0x41700000, 1), + (0x1ec, 0x384, 1), + (0x1f0, 0x3e8, 2), + (0x200, 0x3e8, 1), + (0x234, 0x3e8, 1), + (0x274, 0x64, 1), + (0x290, 0x82a, 1), + (0x298, 0x7d, 1), + (0x29c, 1, 1), + (0xee0, 0x18ae, 1), + (0xee8, 1, 1), + (0x2eec, 0x41c8cccd, 2), + (0x35e4, 1, 1), + (0x35e8, 0x1f4, 1), + (0x35ec, 6, 1), + (0x35f0, 0x30d4, 6), + (0x3608, 0xfa0, 1), + (0x361c, 0x30d4, 5), + (0x3630, 1, 1), + (0x37a8, 1, 1), + (0x37b4, 0x424ccccd, 1), + (0x37f4, 0x44068000, 1), + (0x3834, 0x41c9999a, 1), + (0x38e0, 0x186a, 1), +]; + +const SECONDARY_CONFIG_WORDS: Words = &[ + (0x14, 1, 1), + (0x2c, 1, 2), + (0x40, 1, 1), + (0x4c, 1, 1), + (0x50, 0x186a, 1), +]; + +/// Power state the primary status B carries for the secondary instance, +/// named by [`SecondaryRoot::power_state_va`]. +#[repr(C)] +pub(crate) struct PowerState(pub(crate) [u8; 0x6c0]); + +// SAFETY: `PowerState` is a byte array. +unsafe impl Zeroable for PowerState {} + +impl PowerState { + fn init(&mut self) { + fill_words(&mut self.0, POWER_STATE_WORDS); + } +} + +/// Configuration of the secondary instance, named by +/// [`SecondaryRoot::config_va`]. +#[repr(C)] +pub(crate) struct SecondaryConfig(pub(crate) [u8; 0x80]); + +// SAFETY: `SecondaryConfig` is a byte array. +unsafe impl Zeroable for SecondaryConfig {} + +impl SecondaryConfig { + pub(crate) fn init(&mut self) { + fill_words(&mut self.0, SECONDARY_CONFIG_WORDS); + } +} + /// Leading part of the [`PowerConfig`]. The host writes it twice: in the /// power configuration and at [`bundle::POWER_CONFIG_COPY`], where the copy /// ends with the views it lies in. -#[repr(C, packed)] -pub(crate) struct PowerConfigHead { - pub(crate) unk_0: [u8; 4], - /// Required: 0x5_dc00 (384000). - pub(crate) unk_4: u32, - /// Required: 0x5_dc00 (384000). - pub(crate) unk_8: u32, - pub(crate) unk_c: [u8; 4], - /// Required: 4. - pub(crate) unk_10: u32, - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_14: u32, - pub(crate) unk_18: [u8; 0x14], - /// Required: 1. - pub(crate) unk_2c: u32, - /// Required: 1. - pub(crate) unk_30: u32, - pub(crate) unk_34: [u8; 0xc], - /// Required: 0x64 (100). - pub(crate) unk_40: u32, - /// Required: 1. - pub(crate) unk_44: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_48: u32, - pub(crate) unk_4c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_50: u32, - pub(crate) unk_54: [u8; 0xc], - /// Required: 4. - pub(crate) unk_60: u32, - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_64: u32, - pub(crate) unk_68: [u8; 4], - /// Required: 1. - pub(crate) unk_6c: u32, - /// Required: 1. - pub(crate) unk_70: u32, - pub(crate) unk_74: [u8; 0xc], - /// Required: 0x64 (100). - pub(crate) unk_80: u32, - /// Required: 1. - pub(crate) unk_84: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_88: u32, - pub(crate) unk_8c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_90: u32, - pub(crate) unk_94: [u8; 0x2c], - /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. - pub(crate) unk_c0: [u32; 11], - pub(crate) unk_ec: [u8; 0x8cc], - /// Required: 0x271 (625). - pub(crate) unk_9b8: u32, - pub(crate) unk_9bc: [u8; 8], - /// Required: 0x3f7f_2e9f. - pub(crate) unk_9c4: u32, - pub(crate) unk_9c8: [u8; 4], - /// Required: 0x3b51_6154. - pub(crate) unk_9cc: u32, - pub(crate) unk_9d0: [u8; 4], - /// Required: 0x3ca5_9586 (0.020213 as an IEEE-754 single). - pub(crate) unk_9d4: u32, - pub(crate) unk_9d8: [u8; 8], - /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). - pub(crate) unk_9e0: u32, - /// Required: 0x40a9_0fdb. - pub(crate) unk_9e4: u32, - pub(crate) unk_9e8: [u8; 4], - /// Required: 0xbdd6_7344. - pub(crate) unk_9ec: u32, - /// Required: 0x28 (40). - pub(crate) unk_9f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_9f4: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_9f8: u32, - pub(crate) unk_9fc: [u8; 8], - /// Required: 0x254e (9550). - pub(crate) unk_a04: u32, - pub(crate) unk_a08: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_a38: u32, - pub(crate) unk_a3c: [u8; 0x34], - /// Required: 4. - pub(crate) unk_a70: u32, - pub(crate) unk_a74: [u8; 0x10], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_a84: u32, - pub(crate) unk_a88: [u8; 4], - /// Required: 0x40cc_cccd (6.4 as an IEEE-754 single). - pub(crate) unk_a8c: u32, - pub(crate) unk_a90: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_a98: u32, - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_a9c: u32, - pub(crate) unk_aa0: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_aa8: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_aac: u32, - pub(crate) unk_ab0: [u8; 0xc], - /// Required: 0x254e (9550). - pub(crate) unk_abc: u32, - pub(crate) unk_ac0: [u8; 8], - /// Required: 0x10 (16). - pub(crate) unk_ac8: u32, - pub(crate) unk_acc: [u8; 4], - /// Required: 0x5_dc00 (384000). - pub(crate) unk_ad0: u32, - pub(crate) unk_ad4: [u8; 0x54], - /// Required: 0x5c (92). - pub(crate) unk_b28: u32, - pub(crate) unk_b2c: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_b30: u32, - /// Required: 0x22 (34). - pub(crate) unk_b34: u32, - /// Required: 6. - pub(crate) unk_b38: u32, - pub(crate) unk_b3c: [u8; 4], - /// Required: 6. - pub(crate) unk_b40: u32, - /// Required: 1. - pub(crate) unk_b44: u32, - pub(crate) unk_b48: [u8; 4], - /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). - pub(crate) unk_b4c: u32, - /// Required: 0x3f7d_f3b6 (0.992 as an IEEE-754 single). - pub(crate) unk_b50: u32, - /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). - pub(crate) unk_b54: u32, - /// Required: 0x3c03_126f (0.008 as an IEEE-754 single). - pub(crate) unk_b58: u32, - /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). - pub(crate) unk_b5c: u32, - /// Required: 0x3f69_d4d8 (0.913404 as an IEEE-754 single). - pub(crate) unk_b60: u32, - pub(crate) unk_b64: [u8; 4], - /// Required: 0x42be_0000 (95.0 as an IEEE-754 single). - pub(crate) unk_b68: u32, - /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). - pub(crate) unk_b6c: u32, - /// Required: 0x4064_f5c3 (3.5775 as an IEEE-754 single). - pub(crate) unk_b70: u32, - /// Required: 0x4117_9436. - pub(crate) unk_b74: u32, - /// Required: 0x64 (100). - pub(crate) unk_b78: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_b7c: u32, - /// Required: 0x64 (100). - pub(crate) unk_b80: u32, - pub(crate) unk_b84: [u8; 8], - /// Required: 0x5c (92). - pub(crate) unk_b8c: u32, - pub(crate) unk_b90: [u8; 0x30], - /// Required: 0x64 (100). - pub(crate) unk_bc0: u32, - pub(crate) unk_bc4: [u8; 0x1c], - /// Required: 1. - pub(crate) unk_be0: u8, - /// Required: 4. - pub(crate) unk_be1: u32, - /// Required: 0x64 (100). - pub(crate) unk_be5: u32, - pub(crate) unk_be9: [u8; 4], - /// Required: 0x1f40 (8000). - pub(crate) unk_bed: u32, - /// Required: 0xc8 (200). - pub(crate) unk_bf1: u32, - /// Required: 0xfa0 (4000). - pub(crate) unk_bf5: u32, - /// Required: 0xc8 (200). - pub(crate) unk_bf9: u32, - /// Required: 0x7d0 (2000). - pub(crate) unk_bfd: u32, - /// Required: 0xc8 (200). - pub(crate) unk_c01: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_c05: u32, - /// Required: 0xc8 (200). - pub(crate) unk_c09: u32, - /// Required: 1. - pub(crate) unk_c0d: u32, - pub(crate) unk_c11: [u8; 0x53], - /// Required: 1. - pub(crate) unk_c64: u32, - pub(crate) unk_c68: [u8; 0x28], - /// Required: 1. - pub(crate) unk_c90: u32, - pub(crate) unk_c94: [u8; 0x1003], - /// Required: 0x7fff in every entry; the table continues in - /// [`PowerConfigTail`]. - pub(crate) unk_1c97: [u16; 148], - /// Required: 0xff (low byte of the next table entry). - pub(crate) unk_1dbf: u8, -} - -static_assert!(bundle::POWER_CONFIG_COPY + size_of::() == 0xf000); +#[repr(C)] +pub(crate) struct PowerConfigHead(pub(crate) [u8; 0x1dc0]); -// SAFETY: `PowerConfigHead` consists of integers and arrays of integers only. +// SAFETY: `PowerConfigHead` is a byte array. unsafe impl Zeroable for PowerConfigHead {} impl PowerConfigHead { - /// Initializes the leading part of the power configuration. pub(crate) fn init(&mut self) { - clear(self); - self.unk_4 = 0x5_dc00; - self.unk_8 = 0x5_dc00; - self.unk_10 = 4; - self.unk_14 = 0x3f80_0000; - self.unk_2c = 1; - self.unk_30 = 1; - self.unk_40 = 0x64; - self.unk_44 = 1; - self.unk_48 = 0x3e8; - self.unk_50 = 0x64; - self.unk_60 = 4; - self.unk_64 = 0x3f80_0000; - self.unk_6c = 1; - self.unk_70 = 1; - self.unk_80 = 0x64; - self.unk_84 = 1; - self.unk_88 = 0x3e8; - self.unk_90 = 0x64; - self.unk_c0 = [0x3f82_8f5c; 11]; - self.unk_9b8 = 0x271; - self.unk_9c4 = 0x3f7f_2e9f; - self.unk_9cc = 0x3b51_6154; - self.unk_9d4 = 0x3ca5_9586; - self.unk_9e0 = 0x4615_3800; - self.unk_9e4 = 0x40a9_0fdb; - self.unk_9ec = 0xbdd6_7344; - self.unk_9f0 = 0x28; - self.unk_9f4 = 0x3e8; - self.unk_9f8 = 0x3e8; - self.unk_a04 = 0x254e; - self.unk_a38 = 0x3e8; - self.unk_a70 = 4; - self.unk_a84 = 0x3f80_0000; - self.unk_a8c = 0x40cc_cccd; - self.unk_a98 = 0x4780_0000; - self.unk_a9c = 0x3f00_0000; - self.unk_aa8 = 0x28; - self.unk_aac = 0x3e8; - self.unk_abc = 0x254e; - self.unk_ac8 = 0x10; - self.unk_ad0 = 0x5_dc00; - self.unk_b28 = 0x5c; - self.unk_b30 = 0x64; - self.unk_b34 = 0x22; - self.unk_b38 = 6; - self.unk_b40 = 6; - self.unk_b44 = 1; - self.unk_b4c = 0x3f4c_cccd; - self.unk_b50 = 0x3f7d_f3b6; - self.unk_b54 = 0x3e4c_cccd; - self.unk_b58 = 0x3c03_126f; - self.unk_b5c = 0x3f69_d4d8; - self.unk_b60 = 0x3f69_d4d8; - self.unk_b68 = 0x42be_0000; - self.unk_b6c = 0x4064_f5c3; - self.unk_b70 = 0x4064_f5c3; - self.unk_b74 = 0x4117_9436; - self.unk_b78 = 0x64; - self.unk_b7c = 0x3e8; - self.unk_b80 = 0x64; - self.unk_b8c = 0x5c; - self.unk_bc0 = 0x64; - self.unk_be0 = 1; - self.unk_be1 = 4; - self.unk_be5 = 0x64; - self.unk_bed = 0x1f40; - self.unk_bf1 = 0xc8; - self.unk_bf5 = 0xfa0; - self.unk_bf9 = 0xc8; - self.unk_bfd = 0x7d0; - self.unk_c01 = 0xc8; - self.unk_c05 = 0x3e8; - self.unk_c09 = 0xc8; - self.unk_c0d = 1; - self.unk_c64 = 1; - self.unk_c90 = 1; - self.unk_1c97 = [0x7fff; 148]; - self.unk_1dbf = 0xff; + fill_words(&mut self.0, POWER_CONFIG_HEAD_WORDS); } } @@ -2047,381 +1865,37 @@ impl PowerConfigHead { /// whole power configuration. It carries the die temperature limiter: target /// and release temperatures in 0.01 °C, proportional and integral gains as /// IEEE-754 singles. -#[repr(C, packed)] -pub(crate) struct PowerConfigTail { - /// Required: 0x7f (high byte of the table entry started in the head). - pub(crate) unk_1dc0: u8, - /// Required: 0x7fff in every entry. - pub(crate) unk_1dc1: [u16; 363], - pub(crate) unk_2097: [u8; 0x201], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2298: u32, - /// Required: 0x4220_0000 (40.0 as an IEEE-754 single). - pub(crate) unk_229c: u32, - /// Required: 0x447a_0000 (1000.0 as an IEEE-754 single). - pub(crate) unk_22a0: u32, - /// Required: 0x5be (1470). - pub(crate) unk_22a4: u32, - pub(crate) unk_22a8: [u8; 0x10], - /// Required: 0x28 (40). - pub(crate) unk_22b8: u32, - pub(crate) unk_22bc: [u8; 4], - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_22c0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_22c4: u32, - pub(crate) unk_22c8: [u8; 4], - /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). - pub(crate) unk_22cc: u32, - /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). - pub(crate) unk_22d0: u32, - pub(crate) unk_22d4: [u8; 0x3c], - /// Die temperature sensors the limiter reads ([`TEMP_SENSOR_MASK`]). - pub(crate) temp_sensor_mask: u64, - /// Release temperature of the limiter ([`TEMP_RELEASE`]). - pub(crate) temp_release: u32, - pub(crate) unk_231c: [u8; 4], - /// Required: 4. - pub(crate) unk_2320: u32, - pub(crate) unk_2324: [u8; 0x10], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_2334: u32, - pub(crate) unk_2338: [u8; 4], - /// Integral gain of the limiter ([`TEMP_KI`]). - pub(crate) temp_ki: u32, - pub(crate) unk_2340: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2348: u32, - /// Proportional gain of the limiter ([`TEMP_KP`]). - pub(crate) temp_kp: u32, - pub(crate) unk_2350: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_2358: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_235c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2360: u32, - /// Margin below the target at which the limiter acts, in 0.01 °C. - pub(crate) temp_margin: u32, - pub(crate) unk_2368: [u8; 4], - /// Target temperature of the limiter ([`TEMP_TARGET`]). - pub(crate) temp_target: u32, - pub(crate) unk_2370: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_23a0: u32, - pub(crate) unk_23a4: [u8; 0x218], - /// Required: 0x4200_0006. - pub(crate) unk_25bc: u32, - pub(crate) unk_25c0: [u8; 0x212], - /// Required: 0x3f7e_b852 (0.995 as an IEEE-754 single). - pub(crate) unk_27d2: u32, - pub(crate) unk_27d6: [u8; 4], - /// Required: 0x3ba3_d70a (0.005 as an IEEE-754 single). - pub(crate) unk_27da: u32, - pub(crate) unk_27de: [u8; 4], - /// Required: 0x43c8_0000 (400.0 as an IEEE-754 single). - pub(crate) unk_27e2: u32, - pub(crate) unk_27e6: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_27ee: u32, - /// Required: 0x42c8_0000 (100.0 as an IEEE-754 single). - pub(crate) unk_27f2: u32, - pub(crate) unk_27f6: [u8; 4], - /// Required: 0xbac8_0000. - pub(crate) unk_27fa: u32, - /// Required: 0x384 (900). - pub(crate) unk_27fe: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2802: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2806: u32, - pub(crate) unk_280a: [u8; 4], - /// Required: 0x453b_8000 (3000.0 as an IEEE-754 single). - pub(crate) unk_280e: u32, - /// Required: 0x26ac (9900). - pub(crate) unk_2812: u32, - pub(crate) unk_2816: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_2846: u32, - pub(crate) unk_284a: [u8; 0x18], - /// Required: 0xc8 (200). - pub(crate) unk_2862: u32, - pub(crate) unk_2866: [u8; 0xb2], - /// Required: 0x3e8 (1000). - pub(crate) unk_2918: u32, - pub(crate) unk_291c: [u8; 0x10], - /// Required: 1. - pub(crate) unk_292c: u32, - pub(crate) unk_2930: [u8; 0x30], - /// Required: 1. - pub(crate) unk_2960: u8, - /// Required: 1. - pub(crate) unk_2961: u32, - /// Required: 4. - pub(crate) unk_2965: u32, - pub(crate) unk_2969: [u8; 0x14], - /// Required: 0x23f (575). - pub(crate) unk_297d: u32, - /// Required: 1. - pub(crate) unk_2981: u32, - /// Required: 1. - pub(crate) unk_2985: u32, - pub(crate) unk_2989: [u8; 0x14], - /// Required: 0x240 (576). - pub(crate) unk_299d: u32, - /// Required: 1. - pub(crate) unk_29a1: u32, - /// Required: 1. - pub(crate) unk_29a5: u32, - pub(crate) unk_29a9: [u8; 0x14], - /// Required: 0x241 (577). - pub(crate) unk_29bd: u32, - pub(crate) unk_29c1: [u8; 0x47], - /// Required: 1. - pub(crate) unk_2a08: u32, - pub(crate) unk_2a0c: [u8; 4], - /// Required: 0x254e (9550) in every entry. - pub(crate) unk_2a10: [u32; 3], - pub(crate) unk_2a1c: [u8; 0xc], - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_2a28: u32, - /// Required: 4. - pub(crate) unk_2a2c: u32, - /// Required: 0x10 (16). - pub(crate) unk_2a30: u32, - /// Required: 0x5_dc00 (384000). - pub(crate) unk_2a34: u32, - pub(crate) unk_2a38: [u8; 0xcc], - /// Required: 0x3c (60). - pub(crate) unk_2b04: u32, - pub(crate) unk_2b08: [u8; 8], - /// Required: 0x3f6e_eeef. - pub(crate) unk_2b10: u32, - pub(crate) unk_2b14: [u8; 4], - /// Required: 0x3d88_8889. - pub(crate) unk_2b18: u32, - pub(crate) unk_2b1c: [u8; 4], - /// Required: 0x3f00_0000 (0.5 as an IEEE-754 single). - pub(crate) unk_2b20: u32, - pub(crate) unk_2b24: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2b2c: u32, - /// Required: 0x4088_0000 (4.25 as an IEEE-754 single). - pub(crate) unk_2b30: u32, - pub(crate) unk_2b34: [u8; 8], - /// Required: 0x28 (40). - pub(crate) unk_2b3c: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2b40: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2b44: u32, - pub(crate) unk_2b48: [u8; 4], - /// Required: 0x4615_3800 (9550.0 as an IEEE-754 single). - pub(crate) unk_2b4c: u32, - /// Required: 0x254e (9550). - pub(crate) unk_2b50: u32, - pub(crate) unk_2b54: [u8; 8], - /// Required: 0xf0 (240). - pub(crate) unk_2b5c: u32, - pub(crate) unk_2b60: [u8; 4], - /// Required: 0x57_e400. - pub(crate) unk_2b64: u32, - pub(crate) unk_2b68: [u8; 0x1c], - /// Required: 0x3e8 (1000). - pub(crate) unk_2b84: u32, - pub(crate) unk_2b88: [u8; 0x3c], - /// Required: 1. - pub(crate) unk_2bc4: u32, - pub(crate) unk_2bc8: [u8; 0xcc], - /// Required: 0x32 (50). - pub(crate) unk_2c94: u32, - /// Required: 1. - pub(crate) unk_2c98: u32, - pub(crate) unk_2c9c: [u8; 4], - /// Required: 0x3f63_8e39. - pub(crate) unk_2ca0: u32, - /// Required: 0x3f2a_aaab. - pub(crate) unk_2ca4: u32, - /// Required: 0x3de3_8e39. - pub(crate) unk_2ca8: u32, - /// Required: 0x3eaa_aaab. - pub(crate) unk_2cac: u32, - /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). - pub(crate) unk_2cb0: u32, - /// Required: 0xbf4c_cccd (-0.8 as an IEEE-754 single). - pub(crate) unk_2cb4: u32, - pub(crate) unk_2cb8: [u8; 4], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2cbc: u32, - /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). - pub(crate) unk_2cc0: u32, - /// Required: 0xc0a0_0000 (-5.0 as an IEEE-754 single). - pub(crate) unk_2cc4: u32, - pub(crate) unk_2cc8: [u8; 4], - /// Required: 0x64 (100). - pub(crate) unk_2ccc: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2cd0: u32, - /// Required: 0x64 (100). - pub(crate) unk_2cd4: u32, - pub(crate) unk_2cd8: [u8; 4], - /// Required: 0x467a_0000 (16000.0 as an IEEE-754 single). - pub(crate) unk_2cdc: u32, - /// Required: 0x578 (1400). - pub(crate) unk_2ce0: u32, - pub(crate) unk_2ce4: [u8; 8], - /// Required: 0x90 (144). - pub(crate) unk_2cec: u32, - /// Required: 0x30 (48). - pub(crate) unk_2cf0: u32, - /// Required: 0x34_bc00. - pub(crate) unk_2cf4: u32, - pub(crate) unk_2cf8: [u8; 4], - /// Required: 0x11_9400. - pub(crate) unk_2cfc: u32, - pub(crate) unk_2d00: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2d08: u32, - pub(crate) unk_2d0c: [u8; 0x24], - /// Required: 0x3e80 (16000). - pub(crate) unk_2d30: u32, - /// Required: 2. - pub(crate) unk_2d34: u32, - /// Required: 0x9c4 (2500). - pub(crate) unk_2d38: u32, - /// Required: 0x20d (525). - pub(crate) unk_2d3c: u32, - /// Required: 2. - pub(crate) unk_2d40: u32, - /// Required: 4. - pub(crate) unk_2d44: u32, - /// Required: 0x32 (50). - pub(crate) unk_2d48: u32, - pub(crate) unk_2d4c: [u8; 0x2c], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_2d78: u32, - pub(crate) unk_2d7c: [u8; 0xc], - /// Required: 0x28 (40). - pub(crate) unk_2d88: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_2d8c: u32, - pub(crate) unk_2d90: [u8; 0x30], -} - -static_assert!(core::mem::offset_of!(PowerConfigTail, temp_sensor_mask) == 0x2310 - 0x1dc0); -static_assert!(core::mem::offset_of!(PowerConfigTail, temp_target) == 0x236c - 0x1dc0); -static_assert!(size_of::() == 0x1000); +#[repr(C)] +pub(crate) struct PowerConfigTail(pub(crate) [u8; 0x1000]); -// SAFETY: `PowerConfigTail` consists of integers and arrays of integers only. +// SAFETY: `PowerConfigTail` is a byte array. unsafe impl Zeroable for PowerConfigTail {} impl PowerConfigTail { fn init(&mut self) { - clear(self); - self.unk_1dc0 = 0x7f; - self.unk_1dc1 = [0x7fff; 363]; - self.unk_2298 = 0x4780_0000; - self.unk_229c = 0x4220_0000; - self.unk_22a0 = 0x447a_0000; - self.unk_22a4 = 0x5be; - self.unk_22b8 = 0x28; - self.unk_22c0 = 0x42c8_0000; - self.unk_22c4 = 0x3e8; - self.unk_22cc = 0x3f4c_cccd; - self.unk_22d0 = 0x3e4c_cccd; - self.temp_sensor_mask = TEMP_SENSOR_MASK; - self.temp_release = TEMP_RELEASE; - self.unk_2320 = 4; - self.unk_2334 = 0x3f80_0000; - self.temp_ki = TEMP_KI; - self.unk_2348 = 0x4780_0000; - self.temp_kp = TEMP_KP; - self.unk_2358 = 0x28; - self.unk_235c = 0x3e8; - self.unk_2360 = 0x3e8; - self.temp_target = TEMP_TARGET; - self.unk_23a0 = 0x3e8; - self.unk_25bc = 0x4200_0006; - self.unk_27d2 = 0x3f7e_b852; - self.unk_27da = 0x3ba3_d70a; - self.unk_27e2 = 0x43c8_0000; - self.unk_27ee = 0x4780_0000; - self.unk_27f2 = 0x42c8_0000; - self.unk_27fa = 0xbac8_0000; - self.unk_27fe = 0x384; - self.unk_2802 = 0x3e8; - self.unk_2806 = 0x3e8; - self.unk_280e = 0x453b_8000; - self.unk_2812 = 0x26ac; - self.unk_2846 = 0x3e8; - self.unk_2862 = 0xc8; - self.unk_2918 = 0x3e8; - self.unk_292c = 1; - self.unk_2960 = 1; - self.unk_2961 = 1; - self.unk_2965 = 4; - self.unk_297d = 0x23f; - self.unk_2981 = 1; - self.unk_2985 = 1; - self.unk_299d = 0x240; - self.unk_29a1 = 1; - self.unk_29a5 = 1; - self.unk_29bd = 0x241; - self.unk_2a08 = 1; - self.unk_2a10 = [0x254e; 3]; - self.unk_2a28 = 0x3f80_0000; - self.unk_2a2c = 4; - self.unk_2a30 = 0x10; - self.unk_2a34 = 0x5_dc00; - self.unk_2b04 = 0x3c; - self.unk_2b10 = 0x3f6e_eeef; - self.unk_2b18 = 0x3d88_8889; - self.unk_2b20 = 0x3f00_0000; - self.unk_2b2c = 0x4780_0000; - self.unk_2b30 = 0x4088_0000; - self.unk_2b3c = 0x28; - self.unk_2b40 = 0x3e8; - self.unk_2b44 = 0x3e8; - self.unk_2b4c = 0x4615_3800; - self.unk_2b50 = 0x254e; - self.unk_2b5c = 0xf0; - self.unk_2b64 = 0x57_e400; - self.unk_2b84 = 0x3e8; - self.unk_2bc4 = 1; - self.unk_2c94 = 0x32; - self.unk_2c98 = 1; - self.unk_2ca0 = 0x3f63_8e39; - self.unk_2ca4 = 0x3f2a_aaab; - self.unk_2ca8 = 0x3de3_8e39; - self.unk_2cac = 0x3eaa_aaab; - self.unk_2cb0 = 0xbf4c_cccd; - self.unk_2cb4 = 0xbf4c_cccd; - self.unk_2cbc = 0x4780_0000; - self.unk_2cc0 = 0xc0a0_0000; - self.unk_2cc4 = 0xc0a0_0000; - self.unk_2ccc = 0x64; - self.unk_2cd0 = 0x3e8; - self.unk_2cd4 = 0x64; - self.unk_2cdc = 0x467a_0000; - self.unk_2ce0 = 0x578; - self.unk_2cec = 0x90; - self.unk_2cf0 = 0x30; - self.unk_2cf4 = 0x34_bc00; - self.unk_2cfc = 0x11_9400; - self.unk_2d08 = 0x4780_0000; - self.unk_2d30 = 0x3e80; - self.unk_2d34 = 2; - self.unk_2d38 = 0x9c4; - self.unk_2d3c = 0x20d; - self.unk_2d40 = 2; - self.unk_2d44 = 4; - self.unk_2d48 = 0x32; - self.unk_2d78 = 0x4780_0000; - self.unk_2d88 = 0x28; - self.unk_2d8c = 0x3e8; + fill_words(&mut self.0, POWER_CONFIG_TAIL_WORDS); } } +/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the +/// start of view 3. No pointer names it. +#[repr(C)] +pub(crate) struct HwDataAux(pub(crate) [u8; 0x3c80]); + +// SAFETY: `HwDataAux` is a byte array. +unsafe impl Zeroable for HwDataAux {} + +impl HwDataAux { + pub(crate) fn init(&mut self, secondary_status_va: u64) { + fill_words(&mut self.0, HW_DATA_AUX_WORDS); + self.0[0xed8..0xee0].copy_from_slice(&secondary_status_va.to_le_bytes()); + } +} + + + + + /// Power configuration, in view 2 of the bundle. The secondary instance /// finds it through [`MainConfig::power_config_va`]. #[repr(C, packed)] @@ -2446,174 +1920,7 @@ impl PowerConfig { } } -/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the -/// start of view 3. No pointer names it. -#[repr(C, packed)] -pub(crate) struct HwDataAux { - pub(crate) unk_0: [u8; 0xac], - /// Required: 6. - pub(crate) unk_ac: u32, - pub(crate) unk_b0: [u8; 8], - /// Required: 1. - pub(crate) unk_b8: u32, - pub(crate) unk_bc: [u8; 0xbb], - /// Required: 0xdac (3500). - pub(crate) unk_177: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_17b: u32, - /// Required: 0xbb8 (3000). - pub(crate) unk_17f: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_183: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_187: u32, - /// Required: 0x64 (100). - pub(crate) unk_18b: u32, - /// Required: 0x20 (32). - pub(crate) unk_18f: u32, - pub(crate) unk_193: [u8; 0x21], - /// Required: 4. - pub(crate) unk_1b4: u32, - pub(crate) unk_1b8: [u8; 8], - /// Required: 0x3f78_0000 (0.96875 as an IEEE-754 single). - pub(crate) unk_1c0: u32, - pub(crate) unk_1c4: [u8; 4], - /// Required: 0x3d00_0000 (0.03125 as an IEEE-754 single). - pub(crate) unk_1c8: u32, - pub(crate) unk_1cc: [u8; 4], - /// Required: 0x40a0_0000 (5.0 as an IEEE-754 single). - pub(crate) unk_1d0: u32, - pub(crate) unk_1d4: [u8; 8], - /// Required: 0x4780_0000 (65536.0 as an IEEE-754 single). - pub(crate) unk_1dc: u32, - /// Required: 0x4170_0000 (15.0 as an IEEE-754 single). - pub(crate) unk_1e0: u32, - pub(crate) unk_1e4: [u8; 8], - /// Required: 0x384 (900). - pub(crate) unk_1ec: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_1f0: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_1f4: u32, - pub(crate) unk_1f8: [u8; 8], - /// Required: 0x3e8 (1000). - pub(crate) unk_200: u32, - pub(crate) unk_204: [u8; 0x30], - /// Required: 0x3e8 (1000). - pub(crate) unk_234: u32, - pub(crate) unk_238: [u8; 0x3c], - /// Required: 0x64 (100). - pub(crate) unk_274: u32, - pub(crate) unk_278: [u8; 0x18], - /// Required: 0x82a (2090). - pub(crate) unk_290: u32, - pub(crate) unk_294: [u8; 4], - /// Required: 0x7d (125). - pub(crate) unk_298: u32, - /// Required: 1. - pub(crate) unk_29c: u32, - pub(crate) unk_2a0: [u8; 0xc38], - /// GPU address of the secondary state, [`private::SECONDARY_STATUS`] bytes - /// into the secondary state grid. - pub(crate) secondary_status_va: u64, - /// Required: 0x18ae (6318). - pub(crate) unk_ee0: u32, - pub(crate) unk_ee4: [u8; 4], - /// Required: 1. - pub(crate) unk_ee8: u32, - pub(crate) unk_eec: [u8; 0x2000], - /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). - pub(crate) unk_2eec: u32, - /// Required: 0x41c8_cccd (25.1 as an IEEE-754 single). - pub(crate) unk_2ef0: u32, - pub(crate) unk_2ef4: [u8; 0x6f0], - /// Required: 1. - pub(crate) unk_35e4: u32, - /// Required: 0x1f4 (500). - pub(crate) unk_35e8: u32, - /// Required: 6. - pub(crate) unk_35ec: u32, - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_35f0: [u32; 6], - /// Required: 0xfa0 (4000). - pub(crate) unk_3608: u32, - pub(crate) unk_360c: [u8; 0x10], - /// Required: 0x30d4 (12500) in every entry. - pub(crate) unk_361c: [u32; 5], - /// Required: 1. - pub(crate) unk_3630: u32, - pub(crate) unk_3634: [u8; 0x174], - /// Required: 1. - pub(crate) unk_37a8: u32, - pub(crate) unk_37ac: [u8; 8], - /// Required: 0x424c_cccd (51.2 as an IEEE-754 single). - pub(crate) unk_37b4: u32, - pub(crate) unk_37b8: [u8; 0x3c], - /// Required: 0x4406_8000 (538.0 as an IEEE-754 single). - pub(crate) unk_37f4: u32, - pub(crate) unk_37f8: [u8; 0x3c], - /// Required: 0x41c9_999a (25.2 as an IEEE-754 single). - pub(crate) unk_3834: u32, - pub(crate) unk_3838: [u8; 0xa8], - /// Required: 0x186a (6250). - pub(crate) unk_38e0: u32, - pub(crate) unk_38e4: [u8; 0x39c], -} - -static_assert!(core::mem::offset_of!(HwDataAux, secondary_status_va) == 0xed8); -static_assert!(bundle::AUX + size_of::() == bundle::VIEWS[3]); - -// SAFETY: `HwDataAux` consists of integers and arrays of integers only. -unsafe impl Zeroable for HwDataAux {} -impl HwDataAux { - /// Initializes the auxiliary hardware data. `secondary_status_va` is - /// the secondary state grid plus [`private::SECONDARY_STATUS`]. - pub(crate) fn init(&mut self, secondary_status_va: u64) { - clear(self); - self.unk_ac = 6; - self.unk_b8 = 1; - self.unk_177 = 0xdac; - self.unk_17b = 0x3e8; - self.unk_17f = 0xbb8; - self.unk_183 = 0x3e8; - self.unk_187 = 0x3e8; - self.unk_18b = 0x64; - self.unk_18f = 0x20; - self.unk_1b4 = 4; - self.unk_1c0 = 0x3f78_0000; - self.unk_1c8 = 0x3d00_0000; - self.unk_1d0 = 0x40a0_0000; - self.unk_1dc = 0x4780_0000; - self.unk_1e0 = 0x4170_0000; - self.unk_1ec = 0x384; - self.unk_1f0 = 0x3e8; - self.unk_1f4 = 0x3e8; - self.unk_200 = 0x3e8; - self.unk_234 = 0x3e8; - self.unk_274 = 0x64; - self.unk_290 = 0x82a; - self.unk_298 = 0x7d; - self.unk_29c = 1; - self.unk_ee0 = 0x18ae; - self.unk_ee8 = 1; - self.unk_2eec = 0x41c8_cccd; - self.unk_2ef0 = 0x41c8_cccd; - self.unk_35e4 = 1; - self.unk_35e8 = 0x1f4; - self.unk_35ec = 6; - self.unk_35f0 = [0x30d4; 6]; - self.unk_3608 = 0xfa0; - self.unk_361c = [0x30d4; 5]; - self.unk_3630 = 1; - self.unk_37a8 = 1; - self.unk_37b4 = 0x424c_cccd; - self.unk_37f4 = 0x4406_8000; - self.unk_3834 = 0x41c9_999a; - self.unk_38e0 = 0x186a; - self.secondary_status_va = secondary_status_va; - } -} // --------------------------------------------------------------------------- // Free lists // --------------------------------------------------------------------------- From e10c12029f67a9d548dc02c2fc630fe289674de0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 12:32:59 -0400 Subject: [PATCH 084/631] drm/asahi: grow the scheduler-release table instead of failing a close The table of contexts awaiting their firmware scheduler-state release was fixed at 128 entries and a close that found it full returned ENOSPC, which the queue release path turns into a device stop for every client. A burst of closes that outpaces the firmware's consumption of release records is a legitimate workload, not a fault. Grow the table on demand (the close path runs under a sleepable mutex) and log once when the preallocated capacity is exceeded. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 7e317cd1218e5461124fbfc8db7dd72e71253f8e) --- drivers/gpu/drm/asahi/g17/teardown.rs | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 791ac690c0f50b..5d56931879f419 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -5,10 +5,12 @@ //! The device mutex serializes this state with the Registry's displaced graph ownership. use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; -use core::sync::atomic::{fence, Ordering}; +use core::sync::atomic::{fence, AtomicBool, Ordering}; use kernel::{prelude::*, sync::Arc}; +/// Entries preallocated at device start; a burst of closes beyond it grows the table. const CAPACITY: usize = 128; +static GREW: AtomicBool = AtomicBool::new(false); const POLL_NS: u64 = 2_000_000_000; #[derive(Clone, Copy, PartialEq, Eq)] @@ -80,15 +82,23 @@ impl Pending { .position(|entry| Arc::ptr_eq(&entry.context, context)) } + /// Closes outpacing firmware release consumption grow the table instead of + /// failing the close: a failed close stops the device for every client. fn insert(&mut self, context: &Arc, phase: Phase, now: u64) -> Result { - self.entries - .push_within_capacity(Entry { - context: context.clone(), - phase, - armed: now, - announce_pending: false, - }) - .map_err(|_| ENOSPC) + let entry = Entry { + context: context.clone(), + phase, + armed: now, + announce_pending: false, + }; + if self.entries.len() >= CAPACITY && !GREW.swap(true, Ordering::Relaxed) { + pr_info!( + "asahi: G17 scheduler-release table grew past {} pending contexts\n", + CAPACITY + ); + } + self.entries.push(entry, GFP_KERNEL)?; + Ok(()) } /// Call before replacing a graph's logical owner. Failure leaves that graph with its From c1f50a1e6a6e593dc6382cc51d1cf7e0c82d5557 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:33:29 -0400 Subject: [PATCH 085/631] drm/asahi: add a render flag for passes with at most four known primitives Userspace knows when a render pass rasterizes at most four primitives in total (every draw's topology-derived count times instances, plus internal work) and that no indirect, mesh or otherwise uncounted work is part of it. Let it say so: DRM_ASAHI_RENDER_FEW_PRIMITIVES sets bit 0 of the fragment stage's auxiliary framebuffer control word, which the driver left clear so far. The flag is advertised by DRM_ASAHI_FEATURE_FEW_PRIMITIVES and rejected by kernels without it, like every other render flag. The hardware effect of the bit is not established; omitting it is the safe default and remains the behaviour for every pass that does not set the flag. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 18dbc2de6f21c92c49ffafce39a38c8d6439a5b5) --- drivers/gpu/drm/asahi/g17/command.rs | 4 +++- drivers/gpu/drm/asahi/g17/fw/render.rs | 15 +++++++++++---- drivers/gpu/drm/asahi/g17/mod.rs | 3 ++- include/uapi/drm/asahi_drm.h | 21 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 5929c14b264fd5..16727b8559b2be 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -23,7 +23,8 @@ const MAX_TILE_BUFFER: u64 = 32768; const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; -const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT; +const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; +const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -374,6 +375,7 @@ impl Payload { scissor_va: cmd.isp_scissor_base, depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, + few_primitives: cmd.flags & FEW_PRIMITIVES != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index e3da1b0e6d954f..657efe508bb3d8 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -85,6 +85,8 @@ const TILE_CONFIG_BASE: u64 = 0x280; const TILE_CONFIG_LAYERED: u64 = 1 << 0; const TILE_CONFIG_PROCESS_EMPTY_TILES: u64 = 1 << 16; const AUX_FB_FLAGS_BASE: u64 = 0xc000; +/// The pass rasterizes at most four known primitives (userspace's promise). +const AUX_FB_FLAGS_FEW_PRIMITIVES: u64 = 1 << 0; const AUX_FB_FLAGS_DBIAS_INT: u64 = 1 << 18; /// Tiling layer mode: layer count less one, bit 15, and for layered targets /// bit 14 plus the empty-tile policy. The second copy keeps only the count and @@ -178,6 +180,8 @@ pub(crate) struct RenderPass { pub(crate) depth_bias_va: u64, /// Depth-bias values are integers. pub(crate) depth_bias_is_int: bool, + /// At most four known primitives; sets the auxiliary framebuffer bit 0. + pub(crate) few_primitives: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. @@ -423,10 +427,13 @@ impl PassWords { partial_zls_control |= ZLS_PARTIAL_STENCIL; } - let aux_fb_flags = match pass.depth_bias_is_int { - true => AUX_FB_FLAGS_BASE | AUX_FB_FLAGS_DBIAS_INT, - false => AUX_FB_FLAGS_BASE, - }; + let mut aux_fb_flags = AUX_FB_FLAGS_BASE; + if pass.depth_bias_is_int { + aux_fb_flags |= AUX_FB_FLAGS_DBIAS_INT; + } + if pass.few_primitives { + aux_fb_flags |= AUX_FB_FLAGS_FEW_PRIMITIVES; + } let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { true => spec, false => spec & u32::MAX as u64, diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6f75dfb5822c2b..56a7aa2e41dff4 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1092,7 +1092,8 @@ impl gpu::Gpu for Gpu { fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 471b1ba2f8b331..3459ccf14cbc91 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -406,6 +406,12 @@ struct drm_asahi_queue_limits { * These fields supplement the header barriers. */ DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, + + /** + * @DRM_ASAHI_FEATURE_FEW_PRIMITIVES: Render commands accept + * DRM_ASAHI_RENDER_FEW_PRIMITIVES. + */ + DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 6, }; /** @@ -1190,6 +1196,21 @@ enum drm_asahi_render_flags { * so we match that here for efficiency. */ DRM_ASAHI_RENDER_DBIAS_IS_INT = (1U << 18), + + /** + * @DRM_ASAHI_RENDER_FEW_PRIMITIVES: The render pass is known to rasterize + * at most four primitives in total. + * + * The count includes every primitive the pass generates: topology-derived + * primitive counts of all draws multiplied by their instance counts, plus + * internal work (clears, visibility, scissor/state generated work). Passes + * with indirect draws, mesh work or any work whose primitive count is not + * known at record time must not set this flag. The flag sets bit 0 of the + * auxiliary framebuffer control word for the fragment stage; the hardware + * effect is an optimization for small known passes and its omission is + * always safe. Requires DRM_ASAHI_FEATURE_FEW_PRIMITIVES. + */ + DRM_ASAHI_RENDER_FEW_PRIMITIVES = (1U << 19), }; /** From 171b23293f377faeff6101cb837467daa99f9803 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:43:08 -0400 Subject: [PATCH 086/631] drm/asahi: add a render flag that orders only the fragment stage on barriers The render prefix resolved from vdm_barrier/prior_vdm names the covered renders' fragment completions and was attached to the tiling kick, so a render's vertex processing waited for its predecessors' fragment work even when only its fragment stage consumes them. Keep the resolved dependencies apart by origin until the kick entries are built, and let userspace choose with DRM_ASAHI_RENDER_FRAGMENT_BARRIERS (advertised by DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS): with the flag the render prefix orders the fragment kick, which still waits for its own tiling kick and for the previous fragment work of its physical queue, so fragment order across a client's physical pairs is preserved; the tiling kick keeps its implicit parent. Compute dependencies keep ordering the tiling kick in both modes. Without the flag nothing changes. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c00f3a06ed2d0400a5ec210c856f3a382971f033) --- drivers/gpu/drm/asahi/g17/command.rs | 5 +- drivers/gpu/drm/asahi/g17/fw/render.rs | 2 + drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/render.rs | 56 +++++++++++++++++-- .../drm/asahi/g17/runtime/render/submit.rs | 30 +++++----- include/uapi/drm/asahi_drm.h | 25 +++++++++ 6 files changed, 98 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 16727b8559b2be..83191ddbcb1c36 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -24,7 +24,9 @@ const PROCESS_EMPTY: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; const DBIAS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; const FEW_PRIMITIVES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FEW_PRIMITIVES; -const RENDER_FLAGS: u32 = PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES; +const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; +const RENDER_FLAGS: u32 = + PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS; /// Attachment settings persist until replaced, including by an empty list. #[derive(Clone, Copy)] @@ -376,6 +378,7 @@ impl Payload { depth_bias_va: cmd.isp_dbias_base, depth_bias_is_int: cmd.flags & DBIAS_INT != 0, few_primitives: cmd.flags & FEW_PRIMITIVES != 0, + fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, occlusion_query_va: cmd.isp_oclqry_base, sampler_heap_va: cmd.sampler_heap, sampler_count: cmd.sampler_count.into(), diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 657efe508bb3d8..23eaa7e7dfba9e 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -182,6 +182,8 @@ pub(crate) struct RenderPass { pub(crate) depth_bias_is_int: bool, /// At most four known primitives; sets the auxiliary framebuffer bit 0. pub(crate) few_primitives: bool, + /// Host-only: the resolved barriers order the fragment kick, not the tiling kick. + pub(crate) fragment_barriers: bool, /// GPU address of the occlusion-query results, or zero. pub(crate) occlusion_query_va: u64, /// GPU address of the sampler heap, or zero. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 56a7aa2e41dff4..44724de9a114b5 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1093,7 +1093,8 @@ impl gpu::Gpu for Gpu { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index fd67523ab91c33..05b9bf14ecd2e4 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -34,6 +34,29 @@ pub(crate) use dependencies::Dependencies; use kernel::{prelude::*, sync::Arc}; use retirement::{Ticket, Tracker}; +/// Firmware dependencies of one render, kept by origin until the kick entries +/// are built: the covered earlier renders (their fragment completions) and the +/// covered compute completion. Which kick waits for the render prefix is the +/// command's choice (`RenderPass::fragment_barriers`); compute always orders +/// the tiling kick, since compute output may feed vertex work. +#[derive(Copy, Clone)] +pub(crate) struct RenderDependencies { + pub(crate) render: [KickDependency; 2], + pub(crate) render_count: usize, + pub(crate) compute: Option, +} + +impl RenderDependencies { + pub(crate) const EMPTY: Self = Self { + render: [KickDependency::ZERO; 2], + render_count: 0, + compute: None, + }; + pub(crate) fn render(&self) -> &[KickDependency] { + &self.render[..self.render_count] + } +} + #[derive(Copy, Clone)] pub(crate) struct RenderAddresses { /// Tiling then fragment; each pair contains the primary and auxiliary stamp. @@ -375,7 +398,7 @@ impl Pair { pub(crate) fn publish( &mut self, packet: Arc, - dependencies: &[KickDependency], + dependencies: &RenderDependencies, lease: PublicationLease, host: &mut impl Host, defer: &mut impl FnMut(Deferred) -> Result, @@ -493,7 +516,32 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let fragment_barriers = [KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?]; + // The fragment kick always waits for its own tiling kick. With the + // fragment-barriers flag the resolved prefix orders the fragment kick + // too, and the tiling kick keeps only its implicit parent, so vertex + // work overlaps the predecessors' fragment work. + let Validated::Render { pass, .. } = &packet.command else { + return Err(EINVAL); + }; + let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; + let mut tiling_barriers = [KickDependency::ZERO; 3]; + let mut tiling_count = 0; + let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_count = 1; + for dependency in dependencies.render() { + if pass.fragment_barriers { + fragment_barriers[fragment_count] = *dependency; + fragment_count += 1; + } else { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + if let Some(compute) = dependencies.compute { + tiling_barriers[tiling_count] = compute; + tiling_count += 1; + } + let tiling_barriers = &tiling_barriers[..tiling_count]; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], @@ -516,13 +564,13 @@ impl Pair { }; let ta_args = args( 0, - dependencies, + tiling_barriers, abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, None, ); let fragment_args = args( 1, - &fragment_barriers, + &fragment_barriers[..fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), )?, diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 8b6c647668af4a..39b8c064d0106c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -177,26 +177,25 @@ impl Registry { owner: u64, packet: &Arc, state: u32, - dependencies: &mut [crate::g17::fw::kick::KickDependency; 3], - ) -> Result { - let mut count = 0; + ) -> Result { + let mut dependencies = crate::g17::queue::render::RenderDependencies::EMPTY; let frontier = self.render_prefix(owner, packet)?; for dependency in frontier.as_slice() { - *dependencies.get_mut(count).ok_or(EOVERFLOW)? = *dependency; - count += 1; + *dependencies + .render + .get_mut(dependencies.render_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.render_count += 1; } if let Some(queue) = self.compute(owner) { - if let Some(dependency) = queue.dependency( + dependencies.compute = queue.dependency( owner, packet.completion.status(), packet.order.wait_through[1], state, - )? { - *dependencies.get_mut(count).ok_or(EOVERFLOW)? = dependency; - count += 1; - } + )?; } - Ok(count) + Ok(dependencies) } } @@ -266,7 +265,7 @@ impl crate::g17::Firmware { packet: &Arc, deferred: &mut DeferredBatch, ) -> Result { - use crate::g17::{fw::kick::KickDependency, queue::render::PublicationLease}; + use crate::g17::queue::render::PublicationLease; let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; if self.recovery.pending() || state != 0 { return Err(EAGAIN); @@ -281,10 +280,7 @@ impl crate::g17::Firmware { { return Err(EBUSY); } - let mut dependencies = [KickDependency::ZERO; 3]; - let count = self - .queues - .render_dependencies(owner, packet, state, &mut dependencies)?; + let dependencies = self.queues.render_dependencies(owner, packet, state)?; let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; let pool = pair.pool().clone(); let buffer_id = pair.buffer_id(); @@ -303,7 +299,7 @@ impl crate::g17::Firmware { let result = self.with_render(usize::from(slot), |pair, host| { pair.publish( packet.clone(), - &dependencies[..count], + &dependencies, lease, host, &mut |event| deferred.push(event), diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 3459ccf14cbc91..dd70d2fe228576 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -412,6 +412,12 @@ struct drm_asahi_queue_limits { * DRM_ASAHI_RENDER_FEW_PRIMITIVES. */ DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 6, + + /** + * @DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS: Render commands accept + * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. + */ + DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, }; /** @@ -1211,6 +1217,25 @@ enum drm_asahi_render_flags { * always safe. Requires DRM_ASAHI_FEATURE_FEW_PRIMITIVES. */ DRM_ASAHI_RENDER_FEW_PRIMITIVES = (1U << 19), + + /** + * @DRM_ASAHI_RENDER_FRAGMENT_BARRIERS: Only the fragment stage waits for + * this command's barriers. + * + * The render dependencies resolved from @vdm_barrier and @prior_vdm are + * attached to the fragment half of the render instead of its vertex + * (tiling) half, so vertex processing may start while the renders this + * command depends on are still executing their fragment work. The + * fragment half still waits for its own vertex half and for the previous + * fragment work of its physical queue; the compute dependencies from + * @cdm_barrier and @prior_cdm keep ordering the vertex half. Set this + * only when every render dependency into the command is consumed by + * fragment-stage or attachment accesses alone: nothing the vertex stage + * reads (vertex/index/indirect buffers, vertex, tessellation or geometry + * shader resources, transform feedback, conditional-rendering predicates) + * may depend on it. Requires DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS. + */ + DRM_ASAHI_RENDER_FRAGMENT_BARRIERS = (1U << 20), }; /** From 9a90e5282a75810d978a6a7e66d39f3c1234edd3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:15:59 -0400 Subject: [PATCH 087/631] drm/asahi: give back the USC backing of idle retained compute queues An installed compute queue is retained for the device's lifetime, and each one owns a private USC free-list pool of about 23 MiB. After a burst of clients the retained pools pinned 1.6 GB on an 8 GB machine for the rest of the boot. Keep only the four newest idle retained queues populated: older idle queues publish the release of their free-list buffer slot, and once the control consumer has passed the record their pool backing is dropped. The queue, its ring and its control objects stay installed; an owner that later takes such a vacant queue first allocates fresh backing, rewrites the pool's control header and descriptor row, and only then binds and kicks. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 64820e0cb952592fff5708712a4b4766a8a12f4c) --- drivers/gpu/drm/asahi/g17/freelist.rs | 64 ++++++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 113 +++++++++--- drivers/gpu/drm/asahi/g17/runtime.rs | 192 ++++++++++++++++++-- drivers/gpu/drm/asahi/g17/runtime/events.rs | 1 + 4 files changed, 323 insertions(+), 47 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index e1d42944a38fe8..e40dfec7fc5fdd 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -65,7 +65,7 @@ impl Drop for RenderId { } } -struct Pages { +pub(crate) struct Pages { backing: KernelObject, blocks: usize, layout: RunLayout, @@ -86,6 +86,10 @@ impl Pages { layout, }) } + /// Fresh zeroed compute backing, built without the device mutex. + pub(crate) fn compute(alloc: &Allocator<'_>) -> Result { + Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute) + } fn page_list_va(&self) -> u64 { self.backing.gpu_va() } @@ -324,8 +328,11 @@ impl Submitted<'_> { } /// A pool owned by one physical compute queue, with fixed QID-derived slot. +/// Idle retained queues may give their USC backing back (`pages` is `None`) +/// once the firmware consumed the pool's release; the control and state +/// objects the queue record names stay until processor stop. pub(crate) struct ComputePool { - pages: Pages, + pages: Option, control: KernelObject, state: KernelObject, id: u16, @@ -362,7 +369,7 @@ impl ComputePool { )?; fence(Ordering::SeqCst); Ok(Self { - pages, + pages: Some(pages), control, state, id, @@ -371,15 +378,58 @@ impl ComputePool { pub(crate) fn id(&self) -> u16 { self.id } - pub(crate) fn page_list_va(&self) -> u64 { - self.pages.page_list_va() + /// EAGAIN while the pool is vacant: the owner must re-arm it first. + pub(crate) fn page_list_va(&self) -> Result { + self.pages.as_ref().map(Pages::page_list_va).ok_or(EAGAIN) } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + pub(crate) fn populated(&self) -> bool { + self.pages.is_some() + } + /// The release record for this pool's buffer slot under the free-list + /// generation the queue last published. + pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { + super::fw::channels::FreeListRelease::new(generation, u32::from(self.id)) + } + /// The caller has observed the firmware consume this pool's release. + pub(crate) fn drop_pages(&mut self) { + self.pages = None; + } + /// Rewrites the shared control header for fresh backing. The caller then + /// rewrites the page-pool descriptor row before the queue is kicked again. + /// As render repopulation does, the firmware-completed counter is carried + /// over so it stays consistent with the host submitted counter in `state`. + pub(crate) fn rearm(&mut self, pages: Pages) -> Result { + if self.pages.is_some() { + return Err(EBUSY); + } + let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; + let mut bytes = [0; 8]; + for (index, byte) in bytes.iter_mut().enumerate() { + // SAFETY: The checked eight-byte range stays mapped; byte accesses + // match the unaligned packed counter the firmware writes. + *byte = unsafe { completed.add(index).read_volatile() }; + } + let mut header = FreeListControl::compute(&FreeListArgs { + buffer_slot: u32::from(self.id), + class: FreeListClass::One, + page_list_va: pages.page_list_va(), + run_list_va: pages.run_list_va(), + blocks: COMPUTE_BLOCKS as u32, + state_va: self.state.gpu_va(), + }); + header.unk_54[..8].copy_from_slice(&bytes); + self.control.write(0, header)?; + fence(Ordering::SeqCst); + self.pages = Some(pages); + Ok(()) + } pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { - let address = self.pages.page_list_va(); - self.pages + let pages = self.pages.as_mut().ok_or(EAGAIN)?; + let address = pages.page_list_va(); + pages .backing .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b8b1f570528261..cbca61493611ec 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -7,8 +7,9 @@ pub(crate) mod exit; pub(crate) mod retirement; use crate::g17::{ + channel::Rings, context::Context, - fw::{compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, + fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, object::{Allocator, CpuMap, KernelObject}, }; use crate::{hw::t8140::dynamic, mmu}; @@ -16,6 +17,10 @@ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + const DESCRIPTORS: usize = 256; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; @@ -48,7 +53,9 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// until the first exact successful publication under their replacement has retired. struct Binding { _kick_alias: mmu::KernelMapping, - _pool_alias: mmu::KernelMapping, + /// Client alias of the USC pool pages; dropped when the pool is vacated and + /// recreated when an owner binds a re-armed pool (it pins the backing). + pool_alias: Option, _descriptors: mmu::KernelMapping, preempt: mmu::KernelMapping, usage: mmu::KernelMapping, @@ -153,7 +160,7 @@ impl Graph { .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; Ok(Binding { _kick_alias: kick_alias, - _pool_alias: pool_alias, + pool_alias: Some(pool_alias), _descriptors: descriptors, preempt, usage, @@ -341,8 +348,8 @@ impl PoolBacking { } /// Fresh page-pool row data, installed once while the QID is still unpublished. - pub(crate) fn pool_descriptor(&self) -> (u16, u64) { - (self.pool.id(), self.pool.page_list_va()) + pub(crate) fn pool_descriptor(&self) -> Result<(u16, u64)> { + Ok((self.pool.id(), self.pool.page_list_va()?)) } } @@ -363,9 +370,22 @@ impl Backing { /// Installed graph and its exact logical owner. A released idle graph may change owners only /// after a retirement witness. Failed work retains its pointers independently of fence state. +/// USC pool backing of a retained queue. A release is published only for an idle +/// queue without an owner; the backing is dropped once the firmware consumed it. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum PoolPhase { + Populated, + ReleaseSent(u32), + Vacant, +} + pub(crate) struct Queue { - slot_accounting: crate::g17::queue_stats::ComputeAccount, owner: Option, + pool_phase: PoolPhase, + /// Monotonic time the last owner released this queue (vacate order). + released_at: u64, + /// Free-list generation of the last published configuration. + free_list_generation: u64, binding: Binding, previous: Option, kick: kick::Queue, @@ -414,7 +434,9 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), - slot_accounting: crate::g17::queue_stats::ComputeAccount::new(), + pool_phase: PoolPhase::Populated, + released_at: 0, + free_list_generation: 0, binding, previous: None, kick, @@ -449,11 +471,6 @@ impl Queue { pub(crate) fn qid(&self) -> u8 { self.kick.id().qid() } - fn update_slot_accounting(&mut self) { - self.slot_accounting.update(self.owner.is_some(), self.released, - self.quarantined, self.retired_by_teardown); - } - pub(crate) fn owner(&self) -> Option { self.owner } @@ -597,12 +614,10 @@ impl Queue { ) -> Result>> { if !self.quarantined { self.quarantined = true; - self.update_slot_accounting(); self.quarantine_error = Some(error); self.failure_status_pending = true; self.retirement_proved = false; self.retired_by_teardown = false; - self.update_slot_accounting(); let all_spared = self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); self.spared_quarantine = all_spared && error == ENODATA; @@ -708,7 +723,6 @@ impl Queue { self.retirement_ready = self.active.len == 0; if self.released && self.retirement_ready { self.owner = None; - self.update_slot_accounting(); } } Ok(None) | Err(EAGAIN) => { @@ -815,7 +829,6 @@ impl Queue { self.submitted = 0; self.kick.clear_parent_after_recovery(); self.quarantined = false; - self.update_slot_accounting(); self.retire_pending = false; self.retirement_ready = true; if self @@ -827,7 +840,6 @@ impl Queue { } if self.released { self.owner = None; - self.update_slot_accounting(); } } Ok(()) @@ -926,6 +938,8 @@ impl Queue { } pub(crate) fn reusable(&self) -> bool { self.installed + // A pool release in flight must be witnessed before any successor. + && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) // An issued kill still names this graph even if its commands retire // before the acknowledgement. It cannot acquire a successor owner. && self.exit.is_none() @@ -946,6 +960,10 @@ impl Queue { } let old = self.binding.context.clone(); if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { + if self.binding.pool_alias.is_none() { + // The same VM takes back a re-armed pool: alias the new backing. + self.binding.pool_alias = Some(self.pool.map_client(context.vm())?); + } self.graph.set_owner(&context)?; self.binding.context = context.clone(); self.previous = Some(Previous { @@ -967,12 +985,9 @@ impl Queue { } context.mark_published(); self.owner = Some(owner); - self.update_slot_accounting(); self.released = false; - self.update_slot_accounting(); self.retirement_ready = false; self.retired_by_teardown = false; - self.update_slot_accounting(); Ok(()) } /// True allows the registry to cancel the never-published QID and drop this graph. An @@ -982,14 +997,11 @@ impl Queue { return Ok(true); } self.released = true; - self.update_slot_accounting(); if self.active.len == 0 && !self.quarantined { if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { if let Err(error) = self.graph.set_owner(&previous.context) { self.quarantined = true; - self.update_slot_accounting(); self.retired_by_teardown = true; - self.update_slot_accounting(); return Err(error); } let mut previous = self.previous.take().ok_or(EIO)?; @@ -1002,11 +1014,61 @@ impl Queue { } if self.retirement_ready { self.owner = None; - self.update_slot_accounting(); + self.released_at = now_ns(); } } Ok(false) } + /// An idle retained queue whose USC backing may be given back. + pub(crate) fn vacate_candidate(&self) -> bool { + self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() + } + pub(crate) fn released_at(&self) -> u64 { + self.released_at + } + pub(crate) fn pool_vacant(&self) -> bool { + self.pool_phase == PoolPhase::Vacant + } + /// Publishes the release of this idle queue's free list. The caller holds the + /// device mutex and rings the control doorbell afterwards. + pub(crate) fn vacate(&mut self, init: &crate::g17::initdata::InitData) -> Result { + if !self.vacate_candidate() { + return Err(EBUSY); + } + let record = ControlRecord::FreeListRelease( + self.pool.release_record(self.free_list_generation), + ); + let cursor = Rings::publish_control(init, &record)?; + self.pool_phase = PoolPhase::ReleaseSent(cursor); + fence(Ordering::SeqCst); + Ok(()) + } + /// Drops the USC backing once the firmware's control consumer passed the + /// release. Returns true when the backing was freed by this call. + pub(crate) fn observe_vacancy(&mut self, consumer: u32, producer: u32) -> Result { + let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { + return Ok(false); + }; + if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { + return Ok(false); + } + // The retained owner binding's client alias pins the backing: drop it + // first (the owner has exited; `reusable()` proved no previous binding). + self.binding.pool_alias = None; + self.pool.drop_pages(); + self.pool_phase = PoolPhase::Vacant; + Ok(true) + } + /// Installs fresh backing into a vacant pool; returns the descriptor row data + /// the caller writes before binding an owner. + pub(crate) fn rearm(&mut self, pages: crate::g17::freelist::Pages) -> Result<(u16, u64)> { + if self.pool_phase != PoolPhase::Vacant || !self.reusable() { + return Err(EAGAIN); + } + self.pool.rearm(pages)?; + self.pool_phase = PoolPhase::Populated; + Ok((self.pool.id(), self.pool.page_list_va()?)) + } /// Exact host identity remains valid even when several commands share an ioctl ID. pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { self.active @@ -1069,9 +1131,7 @@ impl Queue { return Ok(true); } self.released = true; - self.update_slot_accounting(); self.retired_by_teardown = true; - self.update_slot_accounting(); if let Some(active) = self .active .iter_mut() @@ -1320,6 +1380,7 @@ impl Queue { qos: packet.context.qos_id(), context: packet.context.id(), }; + self.free_list_generation = u64::from(packet.context.scheduler_generation()); let config = QueueConfig::new(&QueueConfigArgs { target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 805087d7082abd..0de6f627602485 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -24,6 +24,7 @@ use super::{ kick, qos, queue::{self, admission, compute}, }; +use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; const QIDS: usize = 128; @@ -117,21 +118,127 @@ impl Registry { /// An installed idle queue keeps its original QID and backing. Mapping a /// replacement context must succeed before its old binding is displaced. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + /// A vacant queue (USC backing given back) is offered for re-arming only + /// when no populated idle queue exists. + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { if let Some(queue) = self.compute(owner) { - return queue.matches(owner, queue.qid(), context); + return Ok(if queue.matches(owner, queue.qid(), context)? { + Rebind::Done + } else { + Rebind::Fresh + }); } - let Some(queue) = self + let mut vacant = None; + for queue in self .compute .iter_mut() .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.reusable()) - else { - return Ok(false); - }; - queue.bind_owner(owner, context.clone())?; - Ok(true) + .filter(|queue| queue.reusable()) + { + if queue.pool_vacant() { + vacant.get_or_insert(queue.qid()); + continue; + } + queue.bind_owner(owner, context.clone())?; + return Ok(Rebind::Done); + } + Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) + } + + /// Installs fresh USC backing into the vacant queue `qid`; the caller writes + /// the returned descriptor row before binding. EAGAIN if the queue was taken. + pub(super) fn rearm_compute( + &mut self, + qid: u8, + pages: crate::g17::freelist::Pages, + ) -> Result<(u16, u64)> { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid) + .ok_or(EAGAIN)?; + queue.rearm(pages) + } + + pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == qid && queue.reusable()) + .ok_or(EAGAIN)?; + queue.bind_owner(owner, context.clone()) + } + + /// Publishes releases for the oldest idle retained pools beyond `cap`, at + /// most four per pass. Returns the number published. + pub(super) fn vacate_idle_pools( + &mut self, + init: &super::initdata::InitData, + cap: usize, + ) -> Result { + // Count first: event turns are frequent and the steady state has no excess. + let count = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate()) + .count(); + if count <= cap { + return Ok(0); + } + let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; + for queue in self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate()) + { + candidates.push((queue.released_at(), queue.qid()), GFP_KERNEL)?; + } + candidates.sort_unstable(); + let excess = candidates.len() - cap; + let mut published = 0; + for (_, qid) in candidates.iter().take(excess.min(4)) { + let queue = self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + .find(|queue| queue.qid() == *qid) + .ok_or(EIO)?; + match queue.vacate(init) { + Err(error) if error == EAGAIN => break, + result => result?, + } + published += 1; + } + Ok(published) + } + + /// Frees the backing of every pool whose release the firmware consumed. + pub(super) fn observe_pool_releases(&mut self, init: &super::initdata::InitData) -> Result { + fence(Ordering::Acquire); + let consumer = init.control_consumer()?.load(Ordering::Relaxed); + let producer = init.control_producer()?.load(Ordering::Relaxed); + let mut freed = 0; + for queue in self + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + if queue.observe_vacancy(consumer, producer)? { + freed += 1; + } + } + Ok(freed) } pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { @@ -220,6 +327,17 @@ impl Registry { } } +/// Outcome of offering an owner a retained compute queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(super) enum Rebind { + /// The owner now holds an installed, populated queue. + Done, + /// Only a vacant queue is available; re-arm `qid` with fresh backing. + Rearm(u8), + /// No retained queue suits the owner: reserve a fresh QID. + Fresh, +} + /// An immutable logical owner shared by its two scheduler entities. Installed /// physical graphs retain the context independently of this frontend lifetime. pub(super) struct Backend { @@ -420,16 +538,41 @@ impl Backend { { return Err(EIO); } - if firmware.queues.rebind_compute(self.owner, &self.context)? { - return Ok(()); + match firmware.queues.rebind_compute(self.owner, &self.context)? { + Rebind::Done => return Ok(()), + Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), + Rebind::Fresh => { + let reservation = firmware.queues.reserve_compute(self.owner)?; + (firmware.uat.clone(), Ok(reservation)) + } } - let reservation = firmware.queues.reserve_compute(self.owner)?; - (firmware.uat.clone(), reservation) }; let alloc = super::object::Allocator { dev: &dev, uat: &uat, }; + let reservation = match reservation { + Ok(reservation) => reservation, + Err(qid) => { + // Fresh zeroed backing is built without the device mutex; the + // queue may have been taken or re-armed by another owner since. + let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + continue; + } + let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages) { + Err(error) if error == EAGAIN => continue, + result => result?, + }; + // The firmware consumed this slot's release before the backing + // was dropped; the row is rewritten before the queue is kicked. + firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; + return Ok(()); + } + }; let built = (|| { let pool = compute::PoolBacking::new(&alloc, reservation.id)?; { @@ -439,7 +582,7 @@ impl Backend { return Err(EAGAIN); } firmware.queues.reserved(reservation)?; - let (slot, page_list) = pool.pool_descriptor(); + let (slot, page_list) = pool.pool_descriptor()?; firmware.init.initialize_compute_pool(slot, page_list)?; } let backing = compute::Backing::new(&alloc, pool)?; @@ -926,6 +1069,27 @@ impl super::Shared { } } +/// Idle retained compute queues that keep their USC pool populated; older idle +/// pools give their backing back. +const COMPUTE_WARM_CAP: usize = 4; + +impl super::Firmware { + /// Event-worker service step: witness consumed pool releases, then release the + /// oldest idle retained pools beyond the warm cap. + pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + self.queues.observe_pool_releases(&self.init)?; + if self.render_control_backpressured() { + return Ok(()); + } + let published = self.queues.vacate_idle_pools(&self.init, COMPUTE_WARM_CAP)?; + if published != 0 { + fence(Ordering::SeqCst); + self.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + Ok(()) + } +} + /// One monotonic domain for recovery grace, idle effort and control retirement. fn now_ns() -> u64 { ::ktime_get() as u64 diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 08c11eb98790eb..f92c493f4813d6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -386,6 +386,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; + firmware.service_compute_pools()?; { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; From b0d8cfe85e2a406b8a0c1437f6952ff1c373c262 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:16:11 -0400 Subject: [PATCH 088/631] drm/asahi: drop an exited owner's context from idle retained compute queues A retained compute queue kept its last owner's Context, and through it the exited client's whole VM: page tables, every alias the binding held into that VM, the status and the QoS lease, until another owner took the queue. Once the owner's execution root has been released (after the scheduler-state release and command retirement were witnessed) and the queue is idle without an owner, drop the binding and the context and keep only the scheduler page and work storage the installed queue record still names. A later owner binds afresh, as a first owner does. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 3db48c0ac600b5e919c4d2713dbd84e9b12b4ec4) --- drivers/gpu/drm/asahi/g17/context.rs | 16 +++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 97 ++++++++++++++----- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 15 ++- .../gpu/drm/asahi/g17/runtime/compute_exit.rs | 2 +- 5 files changed, 109 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index b7d0a351f234d8..7076708be9f2fc 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -101,6 +101,13 @@ impl WorkStorage { } } +/// Firmware-visible pages of a context, retained by an installed queue whose +/// last owner has exited (see [`Context::retain_firmware_pages`]). +pub(crate) struct FirmwarePages { + _scheduler: Arc, + _work: Arc>, +} + pub(crate) struct Context { execution: Arc, status: Arc, @@ -221,6 +228,15 @@ impl Context { pub(crate) fn scheduler_generation(&self) -> u32 { self.scheduler_generation } + /// The pages an installed queue record of this context still names: the + /// scheduler page and the work storage holding the job-list head. A retained + /// physical queue keeps these after dropping the context itself. + pub(crate) fn retain_firmware_pages(&self) -> FirmwarePages { + FirmwarePages { + _scheduler: self.scheduler.clone(), + _work: self.work.clone(), + } + } pub(crate) fn work_head_va(&self) -> u64 { self.work_base + (UAT_PGSZ + 0x400) as u64 } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index cbca61493611ec..df38a727c8d60e 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -381,12 +381,16 @@ pub(crate) enum PoolPhase { pub(crate) struct Queue { owner: Option, + /// Scheduler/work pages of an exited last owner whose context was dropped; + /// the installed queue record still names them until the next owner binds. + retained_owner: Option, pool_phase: PoolPhase, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, /// Free-list generation of the last published configuration. free_list_generation: u64, - binding: Binding, + /// `None` after an exited owner was detached; a new owner binds afresh. + binding: Option, previous: Option, kick: kick::Queue, pool: ComputePool, @@ -434,10 +438,11 @@ impl Queue { context.mark_published(); Ok(Self { owner: Some(owner), + retained_owner: None, pool_phase: PoolPhase::Populated, released_at: 0, free_list_generation: 0, - binding, + binding: Some(binding), previous: None, kick, pool, @@ -474,12 +479,14 @@ impl Queue { pub(crate) fn owner(&self) -> Option { self.owner } - pub(crate) fn context(&self) -> &Arc { - &self.binding.context + /// The bound owner's context; `None` once an exited owner was detached. + pub(crate) fn context(&self) -> Option<&Arc> { + self.binding.as_ref().map(|binding| &binding.context) } /// Quarantined commands still retain the scheduler until their retirement witness. pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !Arc::ptr_eq(self.context(), context) || (self.active.len == 0 && self.replays.len == 0) + !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) + || (self.active.len == 0 && self.replays.len == 0) } pub(crate) fn in_flight(&self) -> bool { self.active.len != 0 && !self.quarantined @@ -524,7 +531,9 @@ impl Queue { fn qos_owner(&self) -> qos::Owner { qos::Owner { qid: self.qid(), - qos: self.binding.context.qos_id(), + // Accounting completes only for bound owners; a detached queue has no + // pending work and never reaches this. + qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), data_master: DataMaster::Compute as u8, } } @@ -598,7 +607,10 @@ impl Queue { || self.owner != Some(ticket.owner) || active.packet.order.sequence != ticket.submission || active.packet.context.id() != ticket.context_id - || active.packet.context.id() != self.binding.context.id() + || self + .binding + .as_ref() + .is_none_or(|binding| active.packet.context.id() != binding.context.id()) { return Err(EIO); } @@ -929,7 +941,10 @@ impl Queue { if self.released || self.quarantined || self.qid() != qid - || !Arc::ptr_eq(&self.binding.context, context) + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&binding.context, context)) || !context.is_current() { return Err(EIO); @@ -958,17 +973,21 @@ impl Queue { if !context.is_current() { return Err(EFAULT); } - let old = self.binding.context.clone(); - if Arc::ptr_eq(old.status(), context.status()) && old.is_current() { - if self.binding.pool_alias.is_none() { + let old = self.binding.as_ref().map(|binding| binding.context.clone()); + let same_vm = old + .as_ref() + .is_some_and(|old| Arc::ptr_eq(old.status(), context.status()) && old.is_current()); + if same_vm { + let binding = self.binding.as_mut().ok_or(EIO)?; + if binding.pool_alias.is_none() { // The same VM takes back a re-armed pool: alias the new backing. - self.binding.pool_alias = Some(self.pool.map_client(context.vm())?); + binding.pool_alias = Some(self.pool.map_client(context.vm())?); } self.graph.set_owner(&context)?; - self.binding.context = context.clone(); + binding.context = context.clone(); self.previous = Some(Previous { binding: None, - context: old, + context: old.ok_or(EIO)?, publication: None, }); } else { @@ -976,12 +995,15 @@ impl Queue { let pool_alias = self.pool.map_client(context.vm())?; let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; self.graph.set_owner(&context)?; - let old_binding = core::mem::replace(&mut self.binding, binding); - self.previous = Some(Previous { - binding: Some(old_binding), + let old_binding = self.binding.replace(binding); + // A detached queue (exited owner already dropped) has nothing to supersede; + // the new owner's record fields replace the retained pages' references. + self.previous = old.map(|old| Previous { + binding: old_binding, context: old, publication: None, }); + self.retained_owner = None; } context.mark_published(); self.owner = Some(owner); @@ -1006,9 +1028,9 @@ impl Queue { } let mut previous = self.previous.take().ok_or(EIO)?; if let Some(binding) = previous.binding.take() { - self.binding = binding; + self.binding = Some(binding); } else { - self.binding.context = previous.context; + self.binding.as_mut().ok_or(EIO)?.context = previous.context; } self.retirement_ready = true; } @@ -1019,6 +1041,25 @@ impl Queue { } Ok(false) } + /// The last owner's file and VM are gone: its execution root was released + /// after the scheduler-state release and command retirement were witnessed + /// (`Context::release_execution`), and this queue is idle without an owner. + /// Drop the binding (every alias into that VM) and the context, keeping only + /// the scheduler page and work storage the installed queue record names. + pub(crate) fn detach_exited_owner(&mut self) -> bool { + if !self.reusable() { + return false; + } + let Some(binding) = self.binding.as_ref() else { + return false; + }; + if binding.context.is_current() { + return false; + } + self.retained_owner = Some(binding.context.retain_firmware_pages()); + self.binding = None; + true + } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() @@ -1054,7 +1095,9 @@ impl Queue { } // The retained owner binding's client alias pins the backing: drop it // first (the owner has exited; `reusable()` proved no previous binding). - self.binding.pool_alias = None; + if let Some(binding) = self.binding.as_mut() { + binding.pool_alias = None; + } self.pool.drop_pages(); self.pool_phase = PoolPhase::Vacant; Ok(true) @@ -1306,9 +1349,15 @@ impl Queue { { return Err(EBUSY); } - if !packet.context.is_current() || !Arc::ptr_eq(&packet.context, &self.binding.context) { + if !packet.context.is_current() + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) + { return Err(EFAULT); } + let binding = self.binding.as_ref().ok_or(EFAULT)?; if packet.completion.status().get() != 0 { return Err(EIO); } @@ -1361,9 +1410,9 @@ impl Queue { qid, kick_count: self.ordinal, kick: kick_timestamp, - preempt_va: self.binding.preempt.iova(), - operand_state_va: self.binding.operand.iova(), - usage_va: self.binding.usage.iova(), + preempt_va: binding.preempt.iova(), + operand_state_va: binding.operand.iova(), + usage_va: binding.usage.iova(), usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), free_list_control_va: self.pool.control_va(), diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index e420427d594bf4..523965d1358b68 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -77,7 +77,7 @@ pub(super) struct Transaction { impl Queue { pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { - Arc::ptr_eq(self.context(), context) + self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && self.released && self.active.len != 0 && self.exit.is_none() @@ -156,7 +156,10 @@ impl Queue { if !recovery_closed { return Ok(()); } - let state = match State::read(self.context()) { + let Some(context) = self.context() else { + return Ok(()); + }; + let state = match State::read(context) { Ok(state) => state, Err(_) => return Ok(()), }; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 0de6f627602485..204a3f97b3c520 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -822,7 +822,11 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if queue.in_flight() && Arc::ptr_eq(queue.context().status(), status) { + if queue.in_flight() + && queue + .context() + .is_some_and(|context| Arc::ptr_eq(context.status(), status)) + { queue.quarantine(error, &mut |event| deferred.push(event))?; } } @@ -1077,6 +1081,15 @@ impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + for queue in self + .queues + .compute + .iter_mut() + .flatten() + .filter_map(|entry| entry.queue.as_deref_mut()) + { + queue.detach_exited_owner(); + } self.queues.observe_pool_releases(&self.init)?; if self.render_control_backpressured() { return Ok(()); diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs index 439d5c9831cbb9..984b6c1f643ebc 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_exit.rs @@ -97,7 +97,7 @@ impl Firmware { for entry in self.queues.compute.iter().flatten() { if let Some(queue) = entry.queue.as_deref() { let bit = 1u128 << queue.qid(); - if Arc::ptr_eq(queue.context(), context) { + if queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) { own |= bit; } else { foreign |= bit; From 78ed9adf12f1dc52e1ecb7214cc655e778a2b42c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:02:55 -0400 Subject: [PATCH 089/631] drm/asahi: keep reclaim away from VMs that are in use or were just restored Idle-VM reclaim judged a VM idle whenever no job was accepted at that instant. A loading game submits every few tens of milliseconds, so its VM qualified between submissions: under memory pressure the shrinker released its GPU backing, the next SUBMIT restored every invalidated mapping while holding the residency gate (observed as 260-280 ms SUBMIT and VM_BIND stalls ending together), and the restored pages were evicted again seconds later. Two rules: a VM is eligible only once its last accepted job ended at least 500 ms ago, and after a restoration the VM stays ineligible for sixteen times the restoration's duration, bounded to 1-30 s. Truly idle VMs are reclaimed as before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5427403e7135dd886ee52a78d73639bd5a2ccb17) --- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 ++++++++- drivers/gpu/drm/asahi/mmu/residency.rs | 42 +++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index d53c19b93bfdcd..79b4dca112ce9c 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -14,6 +14,13 @@ use kernel::{ }; const ADMISSION_WAIT_MS: u32 = 1000; +/// A VM whose last accepted job ended more recently than this is still in +/// use: evicting its backing would be restored by its next job. +const RECLAIM_QUIET_NS: u64 = 500_000_000; + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} pub(super) struct DriverMappings { mappings: KVec, @@ -49,6 +56,8 @@ impl DriverMappings { struct State { active: usize, + /// Monotonic time the VM last had no accepted job (0 = never had one). + quiet_since: u64, commits: usize, blocked: bool, quarantined: bool, @@ -76,13 +85,15 @@ impl VmLifetime { state.active == 0 && state.commits == 0 && !state.blocked && !state.quarantined && !state.draining && !state.closed && !state.cleanup_failed + && (state.quiet_since == 0 + || now_ns().saturating_sub(state.quiet_since) >= RECLAIM_QUIET_NS) }) } pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State { active:0, commits:0, blocked:false, + state <- new_mutex!(State { active:0, quiet_since:0, commits:0, blocked:false, quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), @@ -569,6 +580,7 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } + state.quiet_since = now_ns(); state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 343da90957602b..d944d168a14eec 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -9,12 +9,39 @@ use super::*; use kernel::drm::gem::BaseObject; use kernel::{new_condvar, sync::CondVar}; +/// Re-evicting backing that a job just had to restore only repeats the +/// restoration. After a restore, reclaim of this VM pauses for a backoff +/// proportional to what the restore cost, within these bounds. +const RESTORE_BACKOFF_MIN_NS: u64 = 1_000_000_000; +const RESTORE_BACKOFF_MAX_NS: u64 = 30_000_000_000; +const RESTORE_BACKOFF_FACTOR: u64 = 16; + +fn now_ns() -> u64 { + ::ktime_get() as u64 +} + #[derive(Default)] struct State { users: usize, busy: bool, evicted: bool, closed: bool, + /// Monotonic time the last restoration finished and its duration. + restored_at: u64, + restore_cost: u64, +} + +impl State { + fn in_restore_backoff(&self, now: u64) -> bool { + if self.restored_at == 0 { + return false; + } + let backoff = self + .restore_cost + .saturating_mul(RESTORE_BACKOFF_FACTOR) + .clamp(RESTORE_BACKOFF_MIN_NS, RESTORE_BACKOFF_MAX_NS); + now.saturating_sub(self.restored_at) < backoff + } } #[pin_data] @@ -76,11 +103,15 @@ impl Gate { // No new reader can enter until every retained mapping is restored. state.busy = true; drop(state); + let started = now_ns(); let result = restore(); let mut state = gate.state.lock(); state.busy = false; if result.is_ok() { state.evicted = false; + let now = now_ns(); + state.restored_at = now; + state.restore_cost = now.saturating_sub(started); } drop(state); gate.ready.notify_all(); @@ -123,7 +154,7 @@ impl Gate { /// The busy gate continues to exclude restoration and metadata changes. pub(crate) fn try_reclaim(gate: &Arc) -> Option { let mut state = gate.state.try_lock()?; - if state.users != 0 || state.busy || state.closed { + if state.users != 0 || state.busy || state.closed || state.in_restore_backoff(now_ns()) { return None; } state.busy = true; @@ -142,9 +173,12 @@ impl Gate { } pub(crate) fn idle(&self) -> bool { - self.state - .try_lock() - .is_some_and(|state| state.users == 0 && !state.busy && !state.closed) + self.state.try_lock().is_some_and(|state| { + state.users == 0 + && !state.busy + && !state.closed + && !state.in_restore_backoff(now_ns()) + }) } /// A page-table mutation happened outside a `Reclaim` (the purge path runs From 88a964bd702754d8dae62fd285a7635fdbbcada7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:26:15 -0400 Subject: [PATCH 090/631] drm/asahi: count bindings and VM creation as reclaim activity Idle-VM reclaim deferred a VM for 500 ms after its last accepted job ended, tracked in the job lifetime. A VM that binds memory without submitting, as a client does while loading or before its first job, was eligible at once, and the shrinker's count did not apply the deferral that its scan applied. Track the time the last residency lease (job, binding or retirement) was released, starting at the VM's creation, in the residency gate, and apply the same 500 ms deferral and restore backoff to both the count and the scan. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1c62cd0853b670104424bcfddeee7f7d9279304e) --- drivers/gpu/drm/asahi/mmu/lifetime.rs | 14 +------------- drivers/gpu/drm/asahi/mmu/residency.rs | 23 +++++++++++++++++++---- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 79b4dca112ce9c..d53c19b93bfdcd 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -14,13 +14,6 @@ use kernel::{ }; const ADMISSION_WAIT_MS: u32 = 1000; -/// A VM whose last accepted job ended more recently than this is still in -/// use: evicting its backing would be restored by its next job. -const RECLAIM_QUIET_NS: u64 = 500_000_000; - -fn now_ns() -> u64 { - ::ktime_get() as u64 -} pub(super) struct DriverMappings { mappings: KVec, @@ -56,8 +49,6 @@ impl DriverMappings { struct State { active: usize, - /// Monotonic time the VM last had no accepted job (0 = never had one). - quiet_since: u64, commits: usize, blocked: bool, quarantined: bool, @@ -85,15 +76,13 @@ impl VmLifetime { state.active == 0 && state.commits == 0 && !state.blocked && !state.quarantined && !state.draining && !state.closed && !state.cleanup_failed - && (state.quiet_since == 0 - || now_ns().saturating_sub(state.quiet_since) >= RECLAIM_QUIET_NS) }) } pub(super) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State { active:0, quiet_since:0, commits:0, blocked:false, + state <- new_mutex!(State { active:0, commits:0, blocked:false, quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), changed <- new_condvar!("VM mapping admission"), @@ -580,7 +569,6 @@ impl Drop for VmJobGuard { if state.active != 0 { return; } - state.quiet_since = now_ns(); state.draining = true; drop(state); self.vm.drain_mappings(lifetime); diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index d944d168a14eec..6643b27cb0c31d 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -15,6 +15,10 @@ use kernel::{new_condvar, sync::CondVar}; const RESTORE_BACKOFF_MIN_NS: u64 = 1_000_000_000; const RESTORE_BACKOFF_MAX_NS: u64 = 30_000_000_000; const RESTORE_BACKOFF_FACTOR: u64 = 16; +/// A VM whose last job, binding or other residency user ended (or which was +/// created) more recently than this is in use: a submitting or loading client +/// would restore whatever reclaim released, synchronously in its next job. +const RECLAIM_QUIET_NS: u64 = 500_000_000; fn now_ns() -> u64 { ::ktime_get() as u64 @@ -29,10 +33,17 @@ struct State { /// Monotonic time the last restoration finished and its duration. restored_at: u64, restore_cost: u64, + /// Monotonic time the last residency user (job, binding, retirement) + /// released its lease; the gate's creation time before the first one. + last_active: u64, } impl State { - fn in_restore_backoff(&self, now: u64) -> bool { + /// Reclaim skips a VM that is in use, was recently used or recently restored. + fn reclaim_deferred(&self, now: u64) -> bool { + if now.saturating_sub(self.last_active) < RECLAIM_QUIET_NS { + return true; + } if self.restored_at == 0 { return false; } @@ -65,7 +76,10 @@ impl Gate { pub(crate) fn new() -> Result> { Arc::pin_init( pin_init!(Self { - state <- new_mutex!(State::default(), "Asahi residency"), + state <- new_mutex!( + State { last_active: now_ns(), ..State::default() }, + "Asahi residency" + ), ready <- new_condvar!("Asahi residency ready"), }), GFP_KERNEL, @@ -154,7 +168,7 @@ impl Gate { /// The busy gate continues to exclude restoration and metadata changes. pub(crate) fn try_reclaim(gate: &Arc) -> Option { let mut state = gate.state.try_lock()?; - if state.users != 0 || state.busy || state.closed || state.in_restore_backoff(now_ns()) { + if state.users != 0 || state.busy || state.closed || state.reclaim_deferred(now_ns()) { return None; } state.busy = true; @@ -177,7 +191,7 @@ impl Gate { state.users == 0 && !state.busy && !state.closed - && !state.in_restore_backoff(now_ns()) + && !state.reclaim_deferred(now_ns()) }) } @@ -210,6 +224,7 @@ impl Drop for Lease { let mut state = self.gate.state.lock(); debug_assert!(state.users != 0); state.users -= 1; + state.last_active = now_ns(); if state.users == 0 { drop(state); self.gate.ready.notify_all(); From 0f2a1cd3d4a47b22565f9fad4a9196adc8b211c1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 18:33:06 -0400 Subject: [PATCH 091/631] drm/asahi: set the idle-VM reclaim quiet window to 2000 ms Idle-VM reclaim now waits 2000 ms after the VM's last job, binding or retirement before releasing its backing (500 ms before). In five Subnautica launch captures, the restore stalls that still followed a real idle gap came after 518 ms (31 ms stall) and 1270 ms (44 ms stall); most longer submit gaps had bindings within 5 ms and are already covered by counting bindings as activity. A truly idle client is still reclaimed, 1.5 s later. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit dec3bc9eb622df05eca30716834c35506ee31f9f) --- drivers/gpu/drm/asahi/mmu/residency.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 6643b27cb0c31d..31f47ee032be6e 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -18,7 +18,7 @@ const RESTORE_BACKOFF_FACTOR: u64 = 16; /// A VM whose last job, binding or other residency user ended (or which was /// created) more recently than this is in use: a submitting or loading client /// would restore whatever reclaim released, synchronously in its next job. -const RECLAIM_QUIET_NS: u64 = 500_000_000; +const RECLAIM_QUIET_NS: u64 = 2000000000; fn now_ns() -> u64 { ::ktime_get() as u64 From 1e371ede082a5e216fc263b336479a0ddab96e8a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:19:25 -0400 Subject: [PATCH 092/631] drm/asahi: release the VM status mirror object when the VM handle closes The VM status is shared with every accepted job and with each retained compute or render graph that last belonged to the VM, so it outlives the client. The client's status-mirror object (GET_PARAMS VM_STATUS_MIRROR) was retained for the status's whole lifetime, keeping a user object alive after the client exited for as long as a retained graph referenced the context. Keep the mirror word and its object under a mutex shared with the failure reporter, and drop both when the file's VM handle is destroyed; later reports find no mirror and only record the error. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 1c1abce16316282e15f5e471b8ef0b5a33acc038) --- drivers/gpu/drm/asahi/g17/status.rs | 82 +++++++++++++++-------------- drivers/gpu/drm/asahi/mmu.rs | 2 +- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index e5ac662a4d1aa9..c4cd4b1ef0e3db 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -3,40 +3,40 @@ //! VM admission errors and client failure reporting. No firmware state is accessed here. use crate::gem; -use core::cell::UnsafeCell; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicPtr, Ordering}; -use kernel::prelude::*; +use core::sync::atomic::{AtomicI32, Ordering}; +use kernel::{new_mutex, prelude::*, sync::Mutex}; const MIRROR_MAX_SIZE: usize = 64 * 1024; /// A VM and its accepted jobs share this status, including after queue destruction. /// Client failure is separate from hardware admission: recovery may preserve the /// VM while failed accepted work must still be reported before signalling fences. +#[pin_data] pub(crate) struct VmStatus { error: AtomicI32, reported_error: AtomicI32, - mirror: AtomicPtr, - mirror_claimed: AtomicBool, - mirror_owner: UnsafeCell>, + /// The client's mapped status word, retained with its object only while the + /// client's VM handle exists; released by `release_mirror` at VM close. + #[pin] + mirror: Mutex>, } -// SAFETY: Only the winner of mirror_claimed writes mirror_owner. The owner is never -// read or replaced while the status lives, and its mapping is retained until drop. -// Other shared accesses use atomics, including accesses through the mirror pointer. -unsafe impl Sync for VmStatus {} -// SAFETY: The retained GEM reference and mapping can move between threads. Moving -// the status does not move that mapping, and its remaining fields are atomic. -unsafe impl Send for VmStatus {} +/// One aligned word inside the retained, vmapped mirror object. +struct Mirror { + word: *mut i32, + _owner: gem::ObjectRef, +} +// SAFETY: The word points into the mapping the retained owner keeps alive; it is +// only accessed atomically through this pointer while the mutex is held. +unsafe impl Send for Mirror {} impl VmStatus { - pub(crate) fn new() -> Self { - Self { + pub(crate) fn new() -> impl PinInit { + pin_init!(Self { error: AtomicI32::new(0), reported_error: AtomicI32::new(0), - mirror: AtomicPtr::new(core::ptr::null_mut()), - mirror_claimed: AtomicBool::new(false), - mirror_owner: UnsafeCell::new(None), - } + mirror <- new_mutex!(None, "Asahi VM status mirror"), + }) } pub(crate) fn get(&self) -> i32 { @@ -71,11 +71,11 @@ impl VmStatus { Ok(_) => error.to_errno(), Err(first) => first, }; - let word = self.mirror.load(Ordering::SeqCst); - if !word.is_null() { - // SAFETY: attach_mirror publishes only an aligned word within a mapping - // owned by this status. Every kernel access to this word is atomic. - unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + let mirror = self.mirror.lock(); + if let Some(mirror) = mirror.as_ref() { + // SAFETY: attach_mirror stored only an aligned word within a mapping the + // retained owner keeps alive while this entry exists under the mutex. + unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst); } } @@ -86,7 +86,7 @@ impl VmStatus { } } - /// Retain one mapped GEM word until the last reference to this status is dropped. + /// Retain one mapped GEM word until the client's VM handle is dropped. pub(crate) fn attach_mirror(&self, mut owner: gem::ObjectRef, offset: usize) -> Result { if owner.size() > MIRROR_MAX_SIZE || offset % core::mem::align_of::() != 0 @@ -99,24 +99,26 @@ impl VmStatus { let base = owner.vmap()?.as_mut_ptr(); // SAFETY: The checked range is inside the retained, page-aligned mapping. let word = unsafe { base.add(offset) }.cast::(); - if self.mirror_claimed.swap(true, Ordering::AcqRel) { + let mut mirror = self.mirror.lock(); + if mirror.is_some() { return Err(EBUSY); } - // SAFETY: This is the only thread that may write mirror_owner. It is not - // read again until the status is dropped, after all shared references end. - unsafe { *self.mirror_owner.get() = Some(owner) }; - // SAFETY: The retained owner keeps this checked, aligned word mapped. - unsafe { AtomicI32::from_ptr(word) }.store(0, Ordering::SeqCst); - - // The pointer publication and reported error form a store/load handshake: - // either this load sees the error or the reporter sees our pointer. Stores - // after publishing the pointer write only the same first error, never zero. - self.mirror.store(word, Ordering::SeqCst); + // No reporter runs while the mutex is held, so the word holds the first + // reported error (or zero) when it becomes visible. let error = self.reported_error.load(Ordering::SeqCst); - if error != 0 { - // SAFETY: The mapping remains owned by this status as above. - unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); - } + // SAFETY: The retained owner keeps this checked, aligned word mapped. + unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); + *mirror = Some(Mirror { + word, + _owner: owner, + }); Ok(()) } + + /// The client's VM handle is gone and nobody reads the mirror any more. The + /// mapping and object reference drop outside the mutex (GEM release may sleep). + pub(crate) fn release_mirror(&self) { + let taken = self.mirror.lock().take(); + drop(taken); + } } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index c6be84aa8d3a5e..be2d87817b6c14 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2374,7 +2374,7 @@ impl Vm { /// Attach a status before sharing a newly created user VM with queues. pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::new(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); self.lifetime = Some(lifetime::VmLifetime::new()?); self.residency = Some(residency::Gate::new()?); self.context_bindings = Some(context::ContextBindings::new()?); From 977cc84bafe33e83a7f581f251aa03e249b87ee8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:33:37 -0400 Subject: [PATCH 093/631] drm/asahi: unmap detached compute owners and free vacated pools off-lock Idle retained compute queues detach their exited owner's binding and, once the firmware consumed a free-list release, drop the vacated USC backing. Both happened inside the event worker's device-mutex critical section: the binding drop unmaps seven aliases from the exited VM and releases the last reference to that context and its VM, and the vacated backing frees a 25 MiB object. On the ship stack kbench's cold phase showed the first SUBMIT of 1-4 of 48 fresh contexts at 433-741 us (p99 657-701 us) against a 170-260 us body that is faster than the previous release's 224 us median. The slow samples are spread over the run and the second submission is 15 us, which is consistent with a submission waiting for the event worker's critical section while it performs this teardown; the previous release did the same unmapping inside the new owner's own submission, so no other client could observe it. Hand the detached binding and the vacated backing to the worker's deferred batch, which already finishes completion fences after the device mutex is released. The batch reserves one slot per physical queue for each; without room the value is dropped in place as before. No behaviour changes for the firmware: the release witness, the retained scheduler pages and the re-arm path are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6af79e6e57ef80d2b93d4c428956ec6cdfb11a3a) --- drivers/gpu/drm/asahi/g17/freelist.rs | 7 +-- drivers/gpu/drm/asahi/g17/queue/compute.rs | 51 +++++++++++--------- drivers/gpu/drm/asahi/g17/runtime.rs | 52 +++++++++++++++++---- drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 +- 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index e40dfec7fc5fdd..a4d58e71eeccfc 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -393,9 +393,10 @@ impl ComputePool { pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { super::fw::channels::FreeListRelease::new(generation, u32::from(self.id)) } - /// The caller has observed the firmware consume this pool's release. - pub(crate) fn drop_pages(&mut self) { - self.pages = None; + /// The caller has observed the firmware consume this pool's release and + /// frees the returned backing after releasing the device mutex. + pub(crate) fn take_pages(&mut self) -> Option { + self.pages.take() } /// Rewrites the shared control header for fresh backing. The caller then /// rewrites the page-pool descriptor row before the queue is kicked again. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index df38a727c8d60e..2cdeb82fbac38f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -51,7 +51,13 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// Aliases are destroyed before the logical root they name. Superseded bindings remain owned /// until the first exact successful publication under their replacement has retired. -struct Binding { +/// Backing of a vacated pool, freed outside the device mutex. +pub(crate) struct Vacated { + _alias: Option, + _pages: Option, +} + +pub(crate) struct Binding { _kick_alias: mmu::KernelMapping, /// Client alias of the USC pool pages; dropped when the pool is vacated and /// recreated when an owner binds a re-armed pool (it pins the backing). @@ -1046,19 +1052,19 @@ impl Queue { /// (`Context::release_execution`), and this queue is idle without an owner. /// Drop the binding (every alias into that VM) and the context, keeping only /// the scheduler page and work storage the installed queue record names. - pub(crate) fn detach_exited_owner(&mut self) -> bool { + /// The returned binding (aliases into the exited VM and the last reference + /// to its context) is dropped by the caller after the device mutex is + /// released: unmapping and VM teardown must not delay other clients. + pub(crate) fn detach_exited_owner(&mut self) -> Option { if !self.reusable() { - return false; + return None; } - let Some(binding) = self.binding.as_ref() else { - return false; - }; - if binding.context.is_current() { - return false; + if self.binding.as_ref()?.context.is_current() { + return None; } + let binding = self.binding.take()?; self.retained_owner = Some(binding.context.retain_firmware_pages()); - self.binding = None; - true + Some(binding) } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { @@ -1084,23 +1090,26 @@ impl Queue { fence(Ordering::SeqCst); Ok(()) } - /// Drops the USC backing once the firmware's control consumer passed the - /// release. Returns true when the backing was freed by this call. - pub(crate) fn observe_vacancy(&mut self, consumer: u32, producer: u32) -> Result { + /// Detaches the USC backing once the firmware's control consumer passed the + /// release. The caller frees the returned backing after releasing the + /// device mutex. + pub(crate) fn observe_vacancy( + &mut self, + consumer: u32, + producer: u32, + ) -> Result> { let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { - return Ok(false); + return Ok(None); }; if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { - return Ok(false); + return Ok(None); } - // The retained owner binding's client alias pins the backing: drop it + // The retained owner binding's client alias pins the backing: detach it // first (the owner has exited; `reusable()` proved no previous binding). - if let Some(binding) = self.binding.as_mut() { - binding.pool_alias = None; - } - self.pool.drop_pages(); + let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); + let pages = self.pool.take_pages(); self.pool_phase = PoolPhase::Vacant; - Ok(true) + Ok(Some(Vacated { _alias: alias, _pages: pages })) } /// Installs fresh backing into a vacant pool; returns the descriptor row data /// the caller writes before binding an owner. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 204a3f97b3c520..2a7c43762e6b3d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -223,7 +223,11 @@ impl Registry { } /// Frees the backing of every pool whose release the firmware consumed. - pub(super) fn observe_pool_releases(&mut self, init: &super::initdata::InitData) -> Result { + pub(super) fn observe_pool_releases( + &mut self, + init: &super::initdata::InitData, + deferred: &mut DeferredBatch, + ) -> Result { fence(Ordering::Acquire); let consumer = init.control_consumer()?.load(Ordering::Relaxed); let producer = init.control_producer()?.load(Ordering::Relaxed); @@ -234,7 +238,8 @@ impl Registry { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - if queue.observe_vacancy(consumer, producer)? { + if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { + deferred.defer_vacated(vacated); freed += 1; } } @@ -632,6 +637,10 @@ impl Backend { /// Collection failure leaves the engine's Active owner intact for a stopped retry. pub(super) struct DeferredBatch { values: KVVec, + /// Exited owners detached from idle retained compute queues and the backing + /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. + bindings: KVVec, + vacated: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -644,18 +653,32 @@ impl DeferredBatch { /// Reused by the sole event work item; no allocation on a completion pass. fn worker() -> Result { - Self::with_capacity(2 * QIDS * Self::DEPTH) + Self::with_capacity(2 * QIDS * Self::DEPTH, QIDS) } /// One publication can quarantine only its selected physical queue/pair. /// VM-wide settlement is a separate transaction before publishing its error. fn publication() -> Result { - Self::with_capacity(2 * Self::DEPTH) + Self::with_capacity(2 * Self::DEPTH, 0) } - fn with_capacity(capacity: usize) -> Result { + fn with_capacity(capacity: usize, detached: usize) -> Result { Ok(Self { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, + bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, + vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, }) } + /// Keeps the detached owner's unmapping and VM teardown out of the device + /// mutex. Without reserved room the binding is dropped here instead. + pub(super) fn defer_binding(&mut self, binding: super::queue::compute::Binding) { + if let Err(binding) = self.bindings.push_within_capacity(binding) { + drop(binding); + } + } + pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { + if let Err(vacated) = self.vacated.push_within_capacity(vacated) { + drop(vacated); + } + } /// A recovery pass can fail and retire each retained command at most once. /// Reserve before changing queues, plus a full publication for its caller. /// Healthy submissions keep the small publication collector unchanged. @@ -693,6 +716,12 @@ impl DeferredBatch { for value in self.values.drain_all() { value.finish(); } + for binding in self.bindings.drain_all() { + drop(binding); + } + for vacated in self.vacated.drain_all() { + drop(vacated); + } } } @@ -1080,7 +1109,12 @@ const COMPUTE_WARM_CAP: usize = 4; impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. - pub(in crate::g17) fn service_compute_pools(&mut self) -> Result { + /// Detached owner bindings and vacated pool backing go to `deferred`, which + /// the worker finishes after releasing the device mutex. + pub(in crate::g17) fn service_compute_pools( + &mut self, + deferred: &mut DeferredBatch, + ) -> Result { for queue in self .queues .compute @@ -1088,9 +1122,11 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { - queue.detach_exited_owner(); + if let Some(binding) = queue.detach_exited_owner() { + deferred.defer_binding(binding); + } } - self.queues.observe_pool_releases(&self.init)?; + self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index f92c493f4813d6..4a18aaa05419d6 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -386,7 +386,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; - firmware.service_compute_pools()?; + firmware.service_compute_pools(deferred)?; { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; From 9cf4d3c71e23b236f8686416d3c4983fced60a92 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 20:08:23 -0400 Subject: [PATCH 094/631] arm64: dts: apple: t8140: describe the GPU mailboxes as ASCWrap v6 The GPU firmware mailboxes were described as "apple,asc-mailbox-v4" windows at 0x482608000 and 0x482e08000. With these nodes the mailbox driver binds its v4 variant, is_ascwrap_v6 stays false, and the ASCWrap v6 CPU lifecycle helpers the GPU driver calls (get_lifecycle, require_safe_cpu_lifecycle, start_cpu, stop_cpu) return -EOPNOTSUPP. The device tree J700 boots with describes both mailboxes as "apple,t8140-ascwrap-v6": the 0x88000 wrapper at 0x482600000 (0x482e00000 for GFX1) plus the 8-byte IOP vector base at 0x482050000 (0x482850000), the send-empty and recv-not-empty interrupts (993/996, 997/1000), and the firmware role. Use the same description here, and give the GPU node the same "asc" (0x88000 wrapper) and "sgx" regions; the driver maps only those two, so the unused "asc1" entry is dropped. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 9c10d7198c8e56b16da35914ffde0eecb6bfb525) --- arch/arm64/boot/dts/apple/t8140.dtsi | 30 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index eec7a0759bfd42..90a769b7cfa63c 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1249,7 +1249,7 @@ reg = <0x4 0x82600000 0x0 0x4000>, <0x4 0x82e00000 0x0 0x4000>, <0x4 0x80000000 0x0 0x4000000>; - reg-names = "asc", "asc1", "sgx"; + reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, @@ -1259,31 +1259,31 @@ status = "disabled"; }; - gfx_mbox: mailbox@482608000 { - compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x4 0x82608000 0x0 0x4000>; + gfx_mbox: mailbox@482600000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82600000 0x0 0x88000>, + <0x4 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; interrupt-parent = <&aic>; interrupts = , - , - , ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; power-domains = <&ps_gfx>; #mbox-cells = <0>; status = "disabled"; }; - gfx1_mbox: mailbox@482e08000 { - compatible = "apple,t8140-gfx-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x4 0x82e08000 0x0 0x4000>; + gfx1_mbox: mailbox@482e00000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82e00000 0x0 0x88000>, + <0x4 0x82850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; interrupt-parent = <&aic>; interrupts = , - , - , ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; power-domains = <&ps_gfx>; #mbox-cells = <0>; status = "disabled"; From 213c971a3d3e63c44d93785eb521c486943a4de4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 02:40:32 -0400 Subject: [PATCH 095/631] drm/asahi: retire a compute command on its record when its stamp never follows A G17 compute command retires once its queue's completion stamp covers its kick and the firmware has consumed its items. On J700 (A18 Pro), a client that keeps render and compute work of one VM in flight together loses its device within seconds: the command's completion record arrives with its start and end timestamps and its items are consumed, but the firmware never stores its completion stamp, so the compute watchdog fails the VM. The same client runs clean on an earlier driver for this GPU, which retires such a command on its completion record and waits for the stamp only a bounded time (500 us); that driver observed that the firmware leaves a minority of completion stamps unpublished. Do the same: once a command's exact completion record (non-zero duration) has been present for 1 ms and its items are consumed, retire it even if its stamp has not covered it. The firmware stores the stamp after the cache maintenance that follows the record, so the bounded wait still lets a stamp that is merely late keep that ordering. Count the commands that retire without a stamp and log the first and every 64th, so how often it happens stays visible. With this change that client ran 575,882 verified checks in 60 s; on the same boot without it, the device was lost within 2 s. Why the firmware leaves these stamps unstored is not yet understood. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 0b4bd1537cc71a1796b976c05cdab24e76549303) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 16 +++++++++- .../drm/asahi/g17/queue/compute/retirement.rs | 30 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2cdeb82fbac38f..f956e5726a8d64 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -325,6 +325,8 @@ struct Active { work: Arc, ticket: Option, observed: Option, + /// Host time the completion record was first observed. + observed_at: Option, failure_deferred: bool, qos_pending: bool, } @@ -589,6 +591,9 @@ impl Queue { }; if active.observed.is_none() { active.observed = batch.take(&ticket)?; + if active.observed.is_some() { + active.observed_at = Some(now_ns()); + } } } Ok(()) @@ -699,15 +704,23 @@ impl Queue { .iter() .skip(1) .find_map(|a| a.observed.map(|o| o.timestamps[1])); + let stamp = self.graph.stamp()?; + let record_aged = front + .observed_at + .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); let timestamps = retirement::poll( &ticket, front.observed, self.graph.cursors()?, - self.graph.stamp()?, + stamp, recordless, self.last_end, next_end, + record_aged, )?; + if timestamps.is_some() && !retirement::stamp_covers(stamp, ticket.kick) { + retirement::note_stampless(self.qid(), ticket.kick); + } if timestamps.is_some() && self.previous.as_ref().is_some_and(|p| { p.publication != Some((front.packet.order.sequence, ticket.epoch)) @@ -1488,6 +1501,7 @@ impl Queue { work: work.clone(), ticket: None, observed: None, + observed_at: None, failure_deferred: false, qos_pending: false, }) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs index 766177ebc32c93..b4d4bcef24421f 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/retirement.rs @@ -65,8 +65,32 @@ pub(crate) enum Recordless { CrossedRecovery, } +/// How long a command whose exact completion record is present waits for its completion stamp +/// before it retires on the record alone. The firmware stores the stamp after the cache +/// maintenance that follows the record; it occasionally never stores it for a command, most +/// visibly while a render of the same VM is running. +pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; + +static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + +/// Counts a command that retired on its record without its completion stamp; logs the first and +/// every 64th. +pub(crate) fn note_stampless(qid: u8, kick: u64) { + let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; + if count == 1 || count % 64 == 0 { + pr_info!( + "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", + count, + qid, + kick + ); + } +} + /// A zero-duration record fails before visibility polling. It proves execution did not occur; /// ownership still requires a separate retirement witness before mappings can be released. +/// The command is complete when its stamp covers it, or when its exact completion record has +/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. pub(crate) fn poll( ticket: &Ticket, observed: Option, @@ -75,13 +99,13 @@ pub(crate) fn poll( recordless: Recordless, floor: u64, next_end: Option, + record_aged: bool, ) -> Result> { if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { return Err(ENODATA); } - if !stamp_covers(stamp, ticket.kick) - || !cursors.consumed(ticket.item_producer, ticket.item_count)? - { + let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); + if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { return Ok(None); } Ok(Some(match observed { From d244a41e5d506b9f49513448b073d6ea7551a3cc Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 00:18:28 -0400 Subject: [PATCH 096/631] drm/asahi: log the first GPU work failure reported to each G17 VM A G17 client treats any failure reported for its work (the VM status word or its mapped mirror) as a lost device, but no path that reports one writes to the kernel log: watchdog timeouts, quarantines, cancelled or failed retirements and recovery classification all fail the client silently. On Neo a client lost its device repeatedly with no kernel line at all, so the failing path could not be identified from the log. Log one warning when the first failure is reported to a VM, with the error and the reporting site. Later failures of the same VM are not logged, so the volume is bounded by the number of failing clients. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f647d69cc5043bf1d3389c8e250e78935e5afa24) --- drivers/gpu/drm/asahi/g17/status.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/g17/status.rs b/drivers/gpu/drm/asahi/g17/status.rs index c4cd4b1ef0e3db..92192d7e20d569 100644 --- a/drivers/gpu/drm/asahi/g17/status.rs +++ b/drivers/gpu/drm/asahi/g17/status.rs @@ -44,6 +44,7 @@ impl VmStatus { } /// Record permanent admission failure and report it to the client. + #[track_caller] pub(crate) fn record(&self, error: Error) { self.report_failure(error); let _ = self.error.compare_exchange( @@ -61,6 +62,10 @@ impl VmStatus { /// Report failed accepted work without preventing recovery of the VM. /// Every reporter publishes the first error before returning: a losing /// reporter may signal its fence before the winning reporter resumes. + /// + /// The first failure reported to a VM is logged once, with the reporting site: the client treats + /// any reported failure as a lost device, and nothing else in the kernel log names it. + #[track_caller] pub(crate) fn report_failure(&self, error: Error) { let error = match self.reported_error.compare_exchange( 0, @@ -68,7 +73,16 @@ impl VmStatus { Ordering::SeqCst, Ordering::SeqCst, ) { - Ok(_) => error.to_errno(), + Ok(_) => { + let at = core::panic::Location::caller(); + pr_warn!( + "GPU work of a client failed ({:?}, reported at {}:{}); the client sees a lost device\n", + error, + at.file().rsplit("asahi/").next().unwrap_or(at.file()), + at.line() + ); + error.to_errno() + } Err(first) => first, }; let mirror = self.mirror.lock(); @@ -80,6 +94,7 @@ impl VmStatus { } /// Rejected submissions and contention leave the VM usable. + #[track_caller] pub(crate) fn record_if_device_loss(&self, error: Error) { if error == EIO || error == ETIMEDOUT || error == ENODEV { self.record(error); From 2c7403aa9b8a9424200f3869be02cc1af92419bd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 00:27:46 -0400 Subject: [PATCH 097/631] drm/asahi: name the completion path in the first-failure log line The first-failure warning names the code that reported the failure. For most failures that is one of three completion helpers (failure before publication, failure while the queue owns the command, failed retirement), which tells little. Let the reporting site propagate through those helpers, so the line names their caller: for example the publication path for a command that failed before it reached the firmware, which is where a failed render pool re-allocation (ENOMEM) is reported. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c8ea00a9ed621e2e5c7c84461091fda3a9b7cb7b) --- drivers/gpu/drm/asahi/g17/completion.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/completion.rs b/drivers/gpu/drm/asahi/g17/completion.rs index 7308fde83d568d..8338f8984d2d1e 100644 --- a/drivers/gpu/drm/asahi/g17/completion.rs +++ b/drivers/gpu/drm/asahi/g17/completion.rs @@ -433,6 +433,7 @@ impl Completion { /// Exact retirement permits the physical lease and VM pin to be released /// before waking userspace. A failure fence alone never establishes this. + #[track_caller] pub(crate) fn complete(&self, result: Result<[u64; 4]>) { if self.signalled.swap(true, Ordering::AcqRel) { return; @@ -453,6 +454,7 @@ impl Completion { } /// A rejected, never-published command cannot poison the VM for contention. + #[track_caller] pub(crate) fn fail_unpublished(&self, error: Error) { if self.signalled.swap(true, Ordering::AcqRel) { return; @@ -468,6 +470,7 @@ impl Completion { /// The physical queue must already be quarantined. Keep mappings pinned while /// waking waiters, even if queue destruction requested cancellation. + #[track_caller] pub(crate) fn fail_while_owned(&self, error: Error) { if self.signalled.swap(true, Ordering::AcqRel) { return; From 2692dc40eac8f77e60bea495800dd788a4f83fc9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:12:31 -0400 Subject: [PATCH 098/631] drm/asahi: describe the G17 word tables without their generation history The comment on the constant word tables recorded how they were produced and named an out-of-tree workspace. State what the tables are instead. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d0aadc1e99e6f11a54bbca5ee685d7c015866789) --- drivers/gpu/drm/asahi/g17/fw/initdata.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 8dbb6a81bcd470..94b03ebe284dba 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -1533,8 +1533,7 @@ const TEMP_KP: u32 = 0x41a2_b852; const TEMP_KI: u32 = 0x4116_0419; /// Nonzero 32-bit words of a firmware object the host builds from constants: (byte offset, value, repeat). -/// Every other word is zero. Generated from the previously field-by-field builders; the fwdump harness -/// (`neo-smooth-20261007/kernel/fwdump`) verifies the objects byte for byte. +/// Every other word is zero. The table is the sole definition of the object's constant contents. type Words = &'static [(usize, u32, usize)]; fn fill_words(bytes: &mut [u8], words: Words) { From fb2adda067c4543e612256bf7b81877d5731c9fa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:13:52 -0400 Subject: [PATCH 099/631] drm/asahi: use one monotonic clock helper across the G17 runtime The runtime registry and the retained compute queues each defined an identical now_ns() over the monotonic clock source. Define it once in the G17 module; the runtime's submodules keep their existing path through a re-export. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5d2a444327afc38543beb78a252e23661e4d4ecc) --- drivers/gpu/drm/asahi/g17/mod.rs | 6 ++++++ drivers/gpu/drm/asahi/g17/queue/compute.rs | 4 +--- drivers/gpu/drm/asahi/g17/runtime.rs | 5 +---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 44724de9a114b5..e384c1f7a63335 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -105,6 +105,12 @@ use object::{ }; /// Expected identification of a G17 GPU. +/// One monotonic domain for recovery grace, idle effort, pool retention and +/// control-ring retirement. +pub(crate) fn now_ns() -> u64 { + ::ktime_get() as u64 +} + pub(crate) struct Identity { /// GPU family field of the ID register. pub(crate) family: u8, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f956e5726a8d64..0cb231f7050e45 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -17,9 +17,7 @@ use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; use retirement::{Cursors, RECORD_SLOTS}; -fn now_ns() -> u64 { - ::ktime_get() as u64 -} +use crate::g17::now_ns; const DESCRIPTORS: usize = 256; const GRAPH_SIZE: usize = 0x8000; diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 2a7c43762e6b3d..6cefa9509de97b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -17,6 +17,7 @@ mod feed; mod render; pub(super) mod teardown; +pub(super) use super::now_ns; use super::{ buffer::{BufferIds, MetricsIds}, context::{Context, QosIds}, @@ -1139,7 +1140,3 @@ impl super::Firmware { } } -/// One monotonic domain for recovery grace, idle effort and control retirement. -fn now_ns() -> u64 { - ::ktime_get() as u64 -} From 09ad54c83e324c8258fbfdf293914984a193748f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:13:52 -0400 Subject: [PATCH 100/631] drm/asahi: grow the scheduler-release table silently Growing the table of contexts awaiting a scheduler release past its preallocated 128 entries is ordinary behaviour under a burst of closes; the one-shot log line that reported it informed nobody and the table still grows the same way. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6850c91310a913b5ff9646261250e62b129f4ce8) --- drivers/gpu/drm/asahi/g17/teardown.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/teardown.rs b/drivers/gpu/drm/asahi/g17/teardown.rs index 5d56931879f419..1a25cd7e3fa3b1 100644 --- a/drivers/gpu/drm/asahi/g17/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/teardown.rs @@ -5,12 +5,11 @@ //! The device mutex serializes this state with the Registry's displaced graph ownership. use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; -use core::sync::atomic::{fence, AtomicBool, Ordering}; +use core::sync::atomic::{fence, Ordering}; use kernel::{prelude::*, sync::Arc}; /// Entries preallocated at device start; a burst of closes beyond it grows the table. const CAPACITY: usize = 128; -static GREW: AtomicBool = AtomicBool::new(false); const POLL_NS: u64 = 2_000_000_000; #[derive(Clone, Copy, PartialEq, Eq)] @@ -91,12 +90,6 @@ impl Pending { armed: now, announce_pending: false, }; - if self.entries.len() >= CAPACITY && !GREW.swap(true, Ordering::Relaxed) { - pr_info!( - "asahi: G17 scheduler-release table grew past {} pending contexts\n", - CAPACITY - ); - } self.entries.push(entry, GFP_KERNEL)?; Ok(()) } From 66e9facd833888d0cf6353aadcba4ad97dd817ea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 16:15:36 -0400 Subject: [PATCH 101/631] drm/asahi: give back the descriptor ring of vacated compute queues A vacated compute queue (USC pool given back) still owns its 4 MiB descriptor ring. Once the queue is idle without an owner, the firmware's job list and context item only name consumed descriptors, so drop the ring and its aliases together with the pool backing, and allocate a fresh ring when an owner re-arms the queue, before the bind that aliases it into the client VM. Submissions to a vacant queue report EAGAIN until the re-arm, as for the pool. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e60f49eede0ecf38cb206c4f59ff832b2aacc1d6) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 112 ++++++++++++++++----- drivers/gpu/drm/asahi/g17/runtime.rs | 6 +- 2 files changed, 90 insertions(+), 28 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 0cb231f7050e45..3db3be2211eb72 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -49,10 +49,13 @@ static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); /// Aliases are destroyed before the logical root they name. Superseded bindings remain owned /// until the first exact successful publication under their replacement has retired. -/// Backing of a vacated pool, freed outside the device mutex. +/// Backing of a vacated pool and descriptor ring, freed outside the device +/// mutex. Field order drops each client alias before the storage it names. pub(crate) struct Vacated { _alias: Option, _pages: Option, + _descriptors: Option, + _ring: Option, } pub(crate) struct Binding { @@ -60,7 +63,9 @@ pub(crate) struct Binding { /// Client alias of the USC pool pages; dropped when the pool is vacated and /// recreated when an owner binds a re-armed pool (it pins the backing). pool_alias: Option, - _descriptors: mmu::KernelMapping, + /// Client alias of the descriptor ring; dropped with the ring when the queue + /// is vacated and recreated when an owner binds a re-armed queue. + descriptors: Option, preempt: mmu::KernelMapping, usage: mmu::KernelMapping, operand: mmu::KernelMapping, @@ -68,10 +73,34 @@ pub(crate) struct Binding { context: Arc, } +/// Descriptor ring of one compute QID with its kernel-VM alias. Built without +/// the device mutex; installed into a vacant graph before an owner binds. +pub(crate) struct DescriptorRing { + object: KernelObject, + high: mmu::KernelMapping, +} + +impl DescriptorRing { + pub(crate) fn new(alloc: &Allocator<'_>) -> Result { + let page = mmu::UAT_PGSZ as u64; + let prot = mmu::PROT_GPU_FW_SHARED_RW; + let object = alloc.lower( + DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, + page, + prot, + CpuMap::WriteCombined, + )?; + let mut high = alloc.uat.geometry().kernel_range(); + high.start += dynamic::KERNEL_OFFSET; + let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + Ok(Self { object, high }) + } +} + /// Firmware-shared storage of one compute QID, separate from its common kick ring. +/// `ring` is `None` while the queue is vacant (idle, no owner, pool given back). struct Graph { - descriptor_high: mmu::KernelMapping, - descriptors: KernelObject, + ring: Option, preempt: KernelObject, usage: KernelObject, operand: KernelObject, @@ -89,10 +118,7 @@ impl Graph { let page = mmu::UAT_PGSZ as u64; let prot = mmu::PROT_GPU_FW_SHARED_RW; let cpu = CpuMap::WriteCombined; - let descriptors = alloc.lower(DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, page, prot, cpu)?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let descriptor_high = descriptors.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; + let ring = DescriptorRing::new(alloc)?; let preempt = alloc.kernel(0x4000, page, prot, cpu)?; let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; let operand = alloc.lower(0x14000, page, prot, cpu)?; @@ -120,8 +146,7 @@ impl Graph { let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; Ok(Self { - descriptor_high, - descriptors, + ring: Some(ring), preempt, usage, operand, @@ -147,9 +172,7 @@ impl Graph { } let vm = context.vm(); vm.install_job_context_aliases()?; - let descriptors = - self.descriptors - .map_alias(vm, self.descriptors.gpu_va(), mmu::PROT_GPU_SHARED_RO)?; + let descriptors = self.descriptor_alias(vm)?; let prot = mmu::PROT_GPU_FW_SHARED_RW; let page = mmu::UAT_PGSZ as u64; let preempt = self @@ -165,7 +188,7 @@ impl Graph { Ok(Binding { _kick_alias: kick_alias, pool_alias: Some(pool_alias), - _descriptors: descriptors, + descriptors: Some(descriptors), preempt, usage, operand, @@ -203,12 +226,29 @@ impl Graph { fence(Ordering::Acquire); Ok(self.support.word(STAMP)?.load(Ordering::Relaxed)) } - fn descriptor_vas(&self, slot: u8) -> (u64, u64) { + /// EAGAIN while the queue is vacant: the owner must re-arm it first. + fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { + let ring = self.ring.as_ref().ok_or(EAGAIN)?; let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; - ( - self.descriptor_high.iova() + offset, - self.descriptors.gpu_va() + offset, - ) + Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) + } + fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { + let ring = self.ring.as_ref().ok_or(EAGAIN)?; + ring.object + .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) + } + /// Detaches the descriptor ring of an idle vacant queue; the caller frees it + /// off-lock. The job list and the context item only name consumed + /// descriptors at this point. + fn vacate_ring(&mut self) -> Option { + self.ring.take() + } + fn rearm_ring(&mut self, ring: DescriptorRing) -> Result { + if self.ring.is_some() { + return Err(EBUSY); + } + self.ring = Some(ring); + Ok(()) } fn ensure_record_slot(&self, slot: usize) -> Result { if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { @@ -1000,6 +1040,9 @@ impl Queue { // The same VM takes back a re-armed pool: alias the new backing. binding.pool_alias = Some(self.pool.map_client(context.vm())?); } + if binding.descriptors.is_none() { + binding.descriptors = Some(self.graph.descriptor_alias(context.vm())?); + } self.graph.set_owner(&context)?; binding.context = context.clone(); self.previous = Some(Previous { @@ -1119,15 +1162,27 @@ impl Queue { // first (the owner has exited; `reusable()` proved no previous binding). let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); let pages = self.pool.take_pages(); + let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); + let ring = self.graph.vacate_ring(); self.pool_phase = PoolPhase::Vacant; - Ok(Some(Vacated { _alias: alias, _pages: pages })) + Ok(Some(Vacated { + _alias: alias, + _pages: pages, + _descriptors: descriptors, + _ring: ring, + })) } /// Installs fresh backing into a vacant pool; returns the descriptor row data /// the caller writes before binding an owner. - pub(crate) fn rearm(&mut self, pages: crate::g17::freelist::Pages) -> Result<(u16, u64)> { + pub(crate) fn rearm( + &mut self, + pages: crate::g17::freelist::Pages, + ring: DescriptorRing, + ) -> Result<(u16, u64)> { if self.pool_phase != PoolPhase::Vacant || !self.reusable() { return Err(EAGAIN); } + self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; Ok((self.pool.id(), self.pool.page_list_va()?)) @@ -1405,7 +1460,7 @@ impl Queue { let qid = self.qid(); let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; let (descriptor_high, descriptor_low) = - self.graph.descriptor_vas(kick_timestamp.slot() as u8); + self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; let first = !self.installed && self.ordinal == 0; let entry_signal = first || dependencies @@ -1510,10 +1565,15 @@ impl Queue { let result = (|| -> Result { work.add_submitted_kicks(1)?; accounted = true; - self.graph.descriptors.initialize::( - kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), - )?; + self.graph + .ring + .as_mut() + .ok_or(EAGAIN)? + .object + .initialize::( + kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + |descriptor| descriptor.write(&args), + )?; if first { self.graph.context_high.write( 0x200, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 6cefa9509de97b..8206df716b6b3e 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -153,6 +153,7 @@ impl Registry { &mut self, qid: u8, pages: crate::g17::freelist::Pages, + ring: compute::DescriptorRing, ) -> Result<(u16, u64)> { let queue = self .compute @@ -161,7 +162,7 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref_mut()) .find(|queue| queue.qid() == qid) .ok_or(EAGAIN)?; - queue.rearm(pages) + queue.rearm(pages, ring) } pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { @@ -563,12 +564,13 @@ impl Backend { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { continue; } - let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages) { + let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { Err(error) if error == EAGAIN => continue, result => result?, }; From 69b8914f8dc459a0000db9a9b9b74bd204f657e2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 18:04:27 -0400 Subject: [PATCH 102/631] drm/asahi: return killed compute queues to service with fresh backing A compute queue whose closed owner was killed stayed quarantined with its exit transaction retained for the device's lifetime, so every fault or kill with work in flight consumed one of the finite compute QIDs for good until the device ran out and queue creation failed. The firmware acknowledges each kill (ContextKilled) and the driver settles every killed command before marking the transaction released; use that as the witness: reset the item ring to empty at the firmware's consumer cursor, clear the quarantine and exit state, and treat the queue as a released idle queue that must give its USC pool and descriptor ring back and be re-armed fresh before the next owner binds it. Recycled queues are vacated regardless of the warm cap. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5d630414e13f4e017628430820fff73a86e97de9) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 79 ++++++++++++++++++- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 7 ++ drivers/gpu/drm/asahi/g17/runtime.rs | 16 +++- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 3db3be2211eb72..57179437f9b40b 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -260,6 +260,23 @@ impl Graph { Ok(()) } + /// A killed queue's ring restarts empty at the firmware's consumer cursor: every + /// item slot is cleared and the host producer is set to the consumer. Only + /// valid after the firmware acknowledged the kill and no command is pending. + fn reset_items(&mut self) -> Result { + let cursors = self.cursors()?; + if cursors.count != ITEM_RING_ENTRIES || cursors.consumer >= cursors.count { + return Err(EIO); + } + for index in 0..ITEM_RING_ENTRIES as usize { + self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); + self.reclaimed = cursors.consumer; + fence(Ordering::SeqCst); + Ok(()) + } /// Reclaim only consumed item pointers, with individual aligned stores and a release fence. /// This leaves records still named by firmware untouched across arbitrarily many wraps. fn recycle(&mut self, cursors: Cursors) -> Result { @@ -431,6 +448,9 @@ pub(crate) struct Queue { /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + /// A killed queue was recycled: its pool/ring must be vacated and re-armed + /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). + needs_fresh_backing: bool, /// Monotonic time the last owner released this queue (vacate order). released_at: u64, /// Free-list generation of the last published configuration. @@ -486,6 +506,7 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + needs_fresh_backing: false, released_at: 0, free_list_generation: 0, binding: Some(binding), @@ -1012,6 +1033,8 @@ impl Queue { self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) + // A recycled killed queue is offered only once its backing is vacant (re-arm path). + && (!self.needs_fresh_backing || self.pool_phase == PoolPhase::Vacant) // An issued kill still names this graph even if its commands retire // before the acknowledgement. It cannot acquire a successor owner. && self.exit.is_none() @@ -1122,7 +1145,58 @@ impl Queue { } /// An idle retained queue whose USC backing may be given back. pub(crate) fn vacate_candidate(&self) -> bool { - self.reusable() && self.pool_phase == PoolPhase::Populated && self.pool.populated() + (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() + } + /// A recycled killed queue still holding the backing its killed work used. + pub(crate) fn recycled_pending(&self) -> bool { + self.needs_fresh_backing + && self.installed + && self.exit.is_none() + && self.owner.is_none() + && !self.quarantined + && self.active.len == 0 + && self.replays.len == 0 + && self.previous.is_none() + } + /// Recycled queues are vacated regardless of the warm cap. + pub(crate) fn needs_fresh_backing(&self) -> bool { + self.needs_fresh_backing + } + /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) + /// and every killed command was settled: return the QID to service. The item + /// ring is reset to empty at the firmware's consumer cursor; the USC pool and + /// descriptor ring are vacated and re-armed fresh before the next owner binds. + pub(crate) fn recycle_killed(&mut self) -> Result { + if !self.installed + || !self.exit.as_ref().is_some_and(|exit| exit.released()) + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() + { + return Ok(false); + } + self.graph.reset_items()?; + self.exit = None; + self.quarantined = false; + self.quarantine_error = None; + self.failure_status_pending = false; + self.retired_by_teardown = false; + self.spared_quarantine = false; + self.spared_deferred = false; + self.retire_pending = false; + self.awaiting_witness = false; + self.retirement_proved = false; + self.replay_retirement = false; + self.replay_error = None; + self.replay_prepared = false; + self.submitted = 0; + self.kick.clear_parent_after_recovery(); + self.owner = None; + self.released = true; + self.retirement_ready = true; + self.released_at = now_ns(); + self.needs_fresh_backing = true; + Ok(true) } pub(crate) fn released_at(&self) -> u64 { self.released_at @@ -1179,12 +1253,13 @@ impl Queue { pages: crate::g17::freelist::Pages, ring: DescriptorRing, ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !self.reusable() { + if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { return Err(EAGAIN); } self.graph.rearm_ring(ring)?; self.pool.rearm(pages)?; self.pool_phase = PoolPhase::Populated; + self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } /// Exact host identity remains valid even when several commands share an ioctl ID. diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 523965d1358b68..a5fda94d6783a9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -75,6 +75,13 @@ pub(super) struct Transaction { announced: Option, } +impl Transaction { + /// The firmware acknowledged the kill and the killed commands were settled. + pub(super) fn released(&self) -> bool { + self.phase == Phase::Released + } +} + impl Queue { pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 8206df716b6b3e..dcfdf43b637927 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -184,6 +184,13 @@ impl Registry { cap: usize, ) -> Result { // Count first: event turns are frequent and the steady state has no excess. + let forced = self + .compute + .iter() + .flatten() + .filter_map(|entry| entry.queue.as_deref()) + .filter(|queue| queue.vacate_candidate() && queue.needs_fresh_backing()) + .count(); let count = self .compute .iter() @@ -191,7 +198,7 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) .count(); - if count <= cap { + if count <= cap && forced == 0 { return Ok(0); } let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; @@ -202,10 +209,12 @@ impl Registry { .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) { - candidates.push((queue.released_at(), queue.qid()), GFP_KERNEL)?; + // Recycled killed queues sort first (time 0) and count beyond the cap. + let key = if queue.needs_fresh_backing() { 0 } else { queue.released_at() }; + candidates.push((key, queue.qid()), GFP_KERNEL)?; } candidates.sort_unstable(); - let excess = candidates.len() - cap; + let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; for (_, qid) in candidates.iter().take(excess.min(4)) { let queue = self @@ -1125,6 +1134,7 @@ impl super::Firmware { .flatten() .filter_map(|entry| entry.queue.as_deref_mut()) { + queue.recycle_killed()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } From a07f7fe2979cd8ae06ac2c94244a75c69a23faa5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:08:54 -0400 Subject: [PATCH 103/631] drm/asahi: give back the render pool backing of evicted idle clients Each logical context that has rendered keeps a render pool with about 24 MiB of pinned USC backing for the rest of its life, also while the client is idle and the machine is reclaiming memory. When pressure reclaim has evicted a client's VM (it was idle for the reclaim quiet window and has no job or binding in progress) and the firmware has consumed the pool's final free-list release, the event worker now detaches the pool's backing and both of its aliases and hands them to its deferred batch, which frees them after the device mutex is released. The pool keeps its ID, control and state objects. Before that client's next render, the submission path installs fresh backing, and the changed backing makes the next staging rewrite the control header and descriptor row, as for a new owner of the slot. A vacated pool is never staged. Clients whose VM was not evicted are unaffected; an evicted client already restores its VM before its next job. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 16822976f9d375ddbdc788d968077a15660be5d7) --- drivers/gpu/drm/asahi/g17/context.rs | 2 + drivers/gpu/drm/asahi/g17/freelist.rs | 139 +++++++++++++++--- drivers/gpu/drm/asahi/g17/runtime.rs | 18 +++ drivers/gpu/drm/asahi/g17/runtime/events.rs | 5 + drivers/gpu/drm/asahi/g17/runtime/render.rs | 31 ++++ .../gpu/drm/asahi/g17/runtime/render/pools.rs | 17 +++ .../drm/asahi/g17/runtime/render/prepare.rs | 11 +- drivers/gpu/drm/asahi/mmu.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 14 ++ 9 files changed, 218 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index 7076708be9f2fc..f8145b1d3b6190 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -280,6 +280,8 @@ impl Context { } let mut render = self.render.lock(); if let Some(pool) = &*render { + // A pool vacated under memory pressure is re-armed before reuse. + pool.ensure_backing(alloc, render_global, self.vm())?; return Ok(pool.clone()); } let pool = Arc::new( diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index a4d58e71eeccfc..c2c9470d10aa3e 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -12,7 +12,11 @@ use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIS use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; -use kernel::{prelude::*, sync::Arc}; +use kernel::{ + new_mutex, + prelude::*, + sync::{Arc, Mutex}, +}; /// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. /// First-free allocation keeps the shared firmware namespace deterministic. @@ -115,27 +119,21 @@ impl Pages { /// Pool-object identities are independent of recycled firmware IDs and VAs. static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); -/// A render pool retained by a logical context and every graph referencing it. -pub(crate) struct RenderPool { - // Remove all aliases before dropping the backing or returning the ID. +/// The USC backing of a render pool and its two aliases. Field order drops +/// both aliases before the backing. +pub(crate) struct RenderBacking { _client: mmu::KernelMapping, _global: mmu::KernelMapping, pages: Pages, - control: KernelObject, - state: KernelObject, - served: AtomicBool, - serial: u64, - id: RenderId, } -impl RenderPool { +impl RenderBacking { + /// Fresh populated backing, built without the device mutex. pub(crate) fn new( alloc: &Allocator<'_>, - ids: &Arc, render_global: &mmu::Vm, client: &mmu::Vm, ) -> Result { - let id = RenderId::new(ids)?; let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; let address = pages.page_list_va(); let global = pages @@ -144,6 +142,45 @@ impl RenderPool { let client = pages .backing .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; + Ok(Self { + _client: client, + _global: global, + pages, + }) + } +} + +/// A render pool retained by a logical context and every graph referencing it. +/// Its backing can be given back while the pool's free-list release has been +/// consumed and the client VM is evicted; a fresh backing is installed before +/// the pool is staged again. +pub(crate) struct RenderPool { + backing: Arc>>, + /// Page-list VA of the installed backing; zero while vacant. + page_list: AtomicU64, + /// Counts installed backings, so a re-armed backing is a new free-list + /// owner even if its addresses repeat. + epoch: AtomicU64, + /// Residency of the client VM the backing is aliased into. + client_residency: Option>, + control: KernelObject, + state: KernelObject, + served: AtomicBool, + serial: u64, + id: RenderId, +} + +impl RenderPool { + pub(crate) fn new( + alloc: &Allocator<'_>, + ids: &Arc, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + let id = RenderId::new(ids)?; + let backing = RenderBacking::new(alloc, render_global, client)?; + let address = backing.pages.page_list_va(); + let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -163,7 +200,7 @@ impl RenderPool { buffer_slot: u32::from(id.slot), class: FreeListClass::Two, page_list_va: address, - run_list_va: pages.run_list_va(), + run_list_va, blocks: RENDER_BLOCKS as u32, state_va: state.gpu_va(), }, @@ -174,10 +211,15 @@ impl RenderPool { let serial = RENDER_POOL_SERIAL .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) .map_err(|_| EOVERFLOW)? + 1; + let backing = Arc::pin_init( + new_mutex!(Some(backing), "G17 render pool backing"), + GFP_KERNEL, + )?; Ok(Self { - _client: client, - _global: global, - pages, + backing, + page_list: AtomicU64::new(address), + epoch: AtomicU64::new(0), + client_residency: client.residency_gate(), control, state, served: AtomicBool::new(false), @@ -186,20 +228,72 @@ impl RenderPool { }) } + /// No backing is installed: the pool must be re-armed before staging. + pub(crate) fn vacant(&self) -> bool { + self.page_list.load(Ordering::Acquire) == 0 + } + /// Pressure reclaim evicted the client VM, which is idle. + pub(crate) fn client_evicted(&self) -> bool { + self.client_residency.as_ref().is_some_and(|gate| gate.evicted()) + } + /// Removes the backing. The caller holds the device mutex and observed this + /// pool's free-list release consumed, so no firmware reference remains; the + /// next staging requires a re-armed backing. The caller drops the result + /// after releasing the device mutex. + pub(crate) fn vacate(&self) -> Option { + let backing = self.backing.lock().take(); + if backing.is_some() { + self.page_list.store(0, Ordering::Release); + } + backing + } + /// Installs fresh backing into a vacant pool, unless another caller did. + /// The next staging rewrites the control header and descriptor row for it + /// (its run-list address changes the free-list owner identity). + pub(crate) fn rearm(&self, fresh: RenderBacking) -> Result { + let mut backing = self.backing.lock(); + if backing.is_some() { + return Ok(()); + } + let address = fresh.pages.page_list_va(); + *backing = Some(fresh); + self.epoch.fetch_add(1, Ordering::AcqRel); + self.served.store(true, Ordering::Release); + self.page_list.store(address, Ordering::Release); + Ok(()) + } + /// Re-arms a vacant pool with fresh backing built here, without the device + /// mutex. `client` is the VM of the context that owns this pool. + pub(crate) fn ensure_backing( + &self, + alloc: &Allocator<'_>, + render_global: &mmu::Vm, + client: &mmu::Vm, + ) -> Result { + if !self.vacant() { + return Ok(()); + } + self.rearm(RenderBacking::new(alloc, render_global, client)?) + } + pub(crate) fn serial(&self) -> u64 { self.serial } + pub(crate) fn epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } pub(crate) fn id(&self) -> u16 { self.id.slot } pub(crate) fn control_va(&self) -> u64 { self.control.gpu_va() } + /// Zero while vacant. pub(crate) fn page_list_va(&self) -> u64 { - self.pages.page_list_va() + self.page_list.load(Ordering::Acquire) } pub(crate) fn run_list_va(&self) -> u64 { - self.pages.run_list_va() + self.page_list_va() + PAGE_LIST_SIZE as u64 } pub(crate) fn end_va(&self) -> u64 { self.run_list_va() @@ -234,7 +328,12 @@ impl RenderPool { .write_unaligned(header) }; let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); - self.pages.repopulate()?; + let backing = self.backing.lock(); + let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; + if pages.page_list_va() != self.page_list_va() { + return Err(EIO); + } + pages.repopulate()?; for (index, value) in image.iter().enumerate() { // SAFETY: The entire retained page was checked before any store; // the allocation is page-aligned and release excludes the firmware. @@ -462,6 +561,8 @@ impl ComputePool { pub(crate) struct Owner { /// Identifies the host pool object even when every firmware address recycles. pub(crate) serial: u64, + /// Identifies the pool's installed backing (render pools are re-armed). + pub(crate) epoch: u64, pub(crate) buffer_slot: u32, pub(crate) control_va: u64, pub(crate) ring_va: u64, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index dcfdf43b637927..842e3a8b8df9be 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -653,6 +653,8 @@ pub(super) struct DeferredBatch { /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. bindings: KVVec, vacated: KVVec, + /// Backing of render pools vacated for evicted idle clients. + render: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -677,6 +679,10 @@ impl DeferredBatch { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, + render: KVVec::with_capacity( + if detached != 0 { crate::hw::t8140::queues::RENDER_SLOTS } else { 0 }, + GFP_KERNEL, + )?, }) } /// Keeps the detached owner's unmapping and VM teardown out of the device @@ -686,6 +692,15 @@ impl DeferredBatch { drop(binding); } } + /// Room for one more vacated render backing; the worker vacates no more. + pub(super) fn render_backing_room(&self) -> bool { + self.render.len() < self.render.capacity() + } + pub(super) fn defer_render_backing(&mut self, backing: super::freelist::RenderBacking) { + if let Err(backing) = self.render.push_within_capacity(backing) { + drop(backing); + } + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -734,6 +749,9 @@ impl DeferredBatch { for vacated in self.vacated.drain_all() { drop(vacated); } + for backing in self.render.drain_all() { + drop(backing); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 4a18aaa05419d6..99f3caf66bebe2 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -387,6 +387,11 @@ impl crate::g17::Shared { firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; firmware.service_compute_pools(deferred)?; + if !firmware.recovery.pending() { + firmware + .queues + .vacate_evicted_render_pools(&firmware.init, deferred)?; + } { let mut reclaim = self.reclaim.lock(); firmware.service_teardown(&mut reclaim)?; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index bf86d46e7ca6bb..aabc523e360cdd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -642,6 +642,37 @@ impl Registry { || !self.render.retired.is_empty() || !self.teardown.is_empty() } + /// Gives back the USC backing of render pools whose client VM pressure + /// reclaim evicted (an idle VM) and whose final free-list release was + /// consumed. The vacated backings go to the worker's deferred batch, which + /// frees them after the device mutex; a later job of that client re-arms + /// the pool first. + pub(in crate::g17) fn vacate_evicted_render_pools( + &mut self, + init: &crate::g17::initdata::InitData, + deferred: &mut super::DeferredBatch, + ) -> Result { + let render = &mut self.render; + render.pools.observe(init)?; + for entry in render.entries.iter().flatten() { + if !deferred.render_backing_room() { + break; + } + if entry.constructing || entry.growing || entry.preparing.is_some() { + continue; + } + let Some(pool) = entry.pool.as_ref() else { + continue; + }; + if pool.vacant() || !render.pools.idle(pool.id()) || !pool.client_evicted() { + continue; + } + if let Some(backing) = pool.vacate() { + deferred.defer_render_backing(backing); + } + } + Ok(()) + } pub(in crate::g17) fn render_in_flight(&self) -> bool { self.render.entries.iter().flatten().any(|entry| { entry.pair.as_ref().map_or( diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs index 7047992754b377..f7f48e853f2252 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/pools.rs @@ -52,6 +52,7 @@ impl Pools { fn owner(pool: &RenderPool) -> Owner { Owner { serial: pool.serial(), + epoch: pool.epoch(), buffer_slot: u32::from(pool.id()), control_va: pool.control_va(), ring_va: pool.run_list_va(), @@ -59,6 +60,15 @@ impl Pools { } } + /// The pool's registration has a consumed final release and no references: + /// the firmware holds nothing of its backing. + pub(super) fn idle(&self, id: u16) -> bool { + Self::slot(id) + .ok() + .and_then(|slot| self.leases[slot]) + .is_some_and(|lease| lease.released()) + } + pub(super) fn released(&self, id: u32, _control: u64) -> bool { self.leases .get(id as usize) @@ -68,6 +78,10 @@ impl Pools { /// Install both engine references before either inner queue becomes /// visible. Warm overlap keeps the existing generation and pool contents. pub(super) fn prepare(&mut self, init: &InitData, pool: &RenderPool) -> Result { + // A vacated pool is re-armed by the submission path before it retries. + if pool.vacant() { + return Err(EAGAIN); + } self.observe(init)?; let slot = Self::slot(pool.id())?; let owner = Self::owner(pool); @@ -92,6 +106,9 @@ impl Pools { /// Warming validates the incoming owner and stages its descriptor without /// acquiring command references or publishing an asynchronous request. pub(super) fn warm(&self, init: &InitData, pool: &RenderPool) -> Result { + if pool.vacant() { + return Err(EAGAIN); + } let previous = self.leases[Self::slot(pool.id())?]; let owner = Self::owner(pool); if previous.is_some_and(|lease| lease.owner() == owner) { diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index 69f8249e0818d3..b4813cdba033a0 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -107,7 +107,16 @@ impl super::super::Backend { } else { let pair = entry.pair.as_ref().ok_or(EBUSY)?; if pair.matches(self.owner, &self.context)? { - return Ok(()); + if !pair.pool().vacant() { + return Ok(()); + } + let pool = pair.pool().clone(); + drop(state); + let alloc = Allocator { + dev: &dev, + uat: &resources.uat, + }; + return pool.ensure_backing(&alloc, &resources.global, self.context.vm()); } if !pair.reusable()? { return Err(EBUSY); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index be2d87817b6c14..2bda44fc9ba4d9 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -100,7 +100,7 @@ mod context; mod shared; mod lifetime; mod residency; -pub(crate) use residency::VmShrinker; +pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; pub(crate) use bind::PreparedUserBindBatch; pub(crate) use lifetime::VmJobGuard; diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 31f47ee032be6e..73a1d163eca826 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -195,6 +195,15 @@ impl Gate { }) } + /// Pressure reclaim released this VM's backing and no user has entered + /// since: its next job restores first. Never blocks; a contended gate + /// reads as not evicted. + pub(crate) fn evicted(&self) -> bool { + self.state + .try_lock() + .is_some_and(|state| state.evicted && !state.busy && state.users == 0) + } + /// A page-table mutation happened outside a `Reclaim` (the purge path runs /// on busy VMs). The next `enter` restores every INVALIDATED mapping. pub(crate) fn note_evicted(&self) { @@ -313,6 +322,11 @@ impl Vm { self.residency.is_some() } + /// The residency gate of a user VM, for observers that must not keep the VM. + pub(crate) fn residency_gate(&self) -> Option> { + self.residency.clone() + } + pub(super) fn enter_residency_metadata(&self) -> Result> { self.residency .as_ref() From bb1b08f2761578100e7c7225416a9789c60b65af Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 15:17:39 -0400 Subject: [PATCH 104/631] drm/asahi: carry a scheduling profile from DRM queue priority to firmware Every queue the driver creates uses firmware priority class 2 and policy 2, written as constants into the queue record, queue configuration, kick announcement, kick header, work-ring selection, doorbells and render registration token, and the QoS row class and share are constants too. The DRM queue priority is validated at queue creation and then dropped. Give each logical queue a scheduling profile (priority class, policy word, QoS class and QoS share), fixed when its context is created, and take every one of those fields from the owning context's profile. LOW queues use priority class 3 and MEDIUM queues keep class 2; the other profile fields are unchanged because no different values are known for class 3. HIGH and REALTIME remain refused. A physical queue takes a new owner's profile only at the existing rebind boundaries, which require it to be idle: a same-VM render rebind also replaces the priority and policy words of both queue records when they differ, compute owner changes do the same in their record, and a render pair whose registration carries another class is registered again at its next publication, under the same power hold as a first install. Work slots mark the first slot naming a queue per priority class, since each class has its own work rings. With only MEDIUM queues, every record, ring, doorbell and token keeps the values and store order it had before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit bff3853f260b02bdd9605bc6690905cb70aca29f) --- drivers/gpu/drm/asahi/g17/context.rs | 10 +- drivers/gpu/drm/asahi/g17/fw/channels.rs | 17 ++- drivers/gpu/drm/asahi/g17/fw/kick.rs | 16 ++- drivers/gpu/drm/asahi/g17/fw/queue.rs | 105 +++++++++++++++--- drivers/gpu/drm/asahi/g17/initdata.rs | 43 ++++--- drivers/gpu/drm/asahi/g17/mod.rs | 14 ++- drivers/gpu/drm/asahi/g17/qos.rs | 13 ++- drivers/gpu/drm/asahi/g17/queue/compute.rs | 50 ++++++--- drivers/gpu/drm/asahi/g17/queue/render.rs | 54 ++++++--- .../gpu/drm/asahi/g17/queue/render/memory.rs | 21 ++++ .../gpu/drm/asahi/g17/queue/render/parked.rs | 4 +- .../drm/asahi/g17/queue/render/publication.rs | 1 + drivers/gpu/drm/asahi/g17/runtime.rs | 20 +++- .../drm/asahi/g17/runtime/compute_replay.rs | 5 +- drivers/gpu/drm/asahi/g17/runtime/render.rs | 13 ++- .../drm/asahi/g17/runtime/render/submit.rs | 5 +- drivers/gpu/drm/asahi/hw/t8140.rs | 11 ++ 17 files changed, 310 insertions(+), 92 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index f8145b1d3b6190..afd0d1d22f5bbb 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -9,7 +9,7 @@ mod identity; use super::{ freelist::{RenderIds, RenderPool}, - fw::context as fw, + fw::{context as fw, queue::Policy}, object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, status::VmStatus, }; @@ -120,6 +120,9 @@ pub(crate) struct Context { work: Arc>, work_base: u64, render: Arc>>>, + /// Fixed for the context's life. Physical queues bound to it carry this + /// profile in every record, ring, doorbell and registration. + policy: Policy, } impl Context { pub(crate) fn new( @@ -128,6 +131,7 @@ impl Context { qos: &Arc, execution: Arc, owner_pid: u32, + policy: Policy, ) -> Result { let status = execution.vm().status().ok_or(EINVAL)?.clone(); let qos = qos.acquire()?; @@ -188,6 +192,7 @@ impl Context { work, work_base, render, + policy, }) } pub(crate) fn vm(&self) -> &Vm { @@ -209,6 +214,9 @@ impl Context { pub(crate) fn owner_pid(&self) -> u32 { self.owner_pid } + pub(crate) fn policy(&self) -> Policy { + self.policy + } pub(crate) fn qos_id(&self) -> u8 { self.qos.id } diff --git a/drivers/gpu/drm/asahi/g17/fw/channels.rs b/drivers/gpu/drm/asahi/g17/fw/channels.rs index c2c3b3eb7a80d3..c4cf6960db296d 100644 --- a/drivers/gpu/drm/asahi/g17/fw/channels.rs +++ b/drivers/gpu/drm/asahi/g17/fw/channels.rs @@ -15,7 +15,7 @@ use kernel::prelude::*; use super::kick::QID_MAX; -use super::queue::{DataMaster, QUEUE_PRIORITY}; +use super::queue::DataMaster; /// Size of a device-control record. pub(crate) const DEVICE_CONTROL_RECORD_SIZE: usize = 0x40; @@ -448,10 +448,13 @@ pub(crate) const WORK_RING_SLOTS: u32 = 256; /// Number of engines with a work ring per priority. const WORK_RINGS_PER_PRIORITY: usize = 3; -/// Index in the channel table of the work ring for `data_master` at the -/// priority of every driver-created queue. -pub(crate) const fn work_ring_index(data_master: DataMaster) -> usize { - QUEUE_PRIORITY as usize * WORK_RINGS_PER_PRIORITY + data_master as usize +/// Index in the channel table of the work ring for `data_master` at priority +/// class `priority`, or `None` if `priority` is not a class. +pub(crate) const fn work_ring_index(priority: u8, data_master: DataMaster) -> Option { + if priority >= super::queue::Policy::PRIORITIES { + return None; + } + Some(priority as usize * WORK_RINGS_PER_PRIORITY + data_master as usize) } /// One work-ring slot: a request to run a queue's newly published commands. @@ -467,7 +470,9 @@ pub(crate) struct WorkSlot { pub(crate) items_end: u16, /// Queue to run. pub(crate) qid: u8, - /// One in the first slot naming this queue on this ring. + /// One in the first slot naming this queue on this ring. Each priority + /// class has its own rings, so a queue whose class changed names the new + /// ring with a first slot again. pub(crate) first: u8, } diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 2e6133e882b2d1..47cdce3ca09e5c 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -20,7 +20,7 @@ use kernel::prelude::*; -use super::queue::QUEUE_PRIORITY; +use super::queue::Policy; /// Number of entries in a kick ring. pub(crate) const KICK_SLOTS: usize = 256; @@ -371,6 +371,8 @@ pub(crate) struct KickArgs<'a> { pub(crate) register_arrays: [Option; 4], /// The compute register program includes a shader-context scratch request. pub(crate) compute_scratch: bool, + /// Priority class of the queue. + pub(crate) priority: u8, } /// One kick ring entry. @@ -436,13 +438,14 @@ impl KickEntry { if args.qid > QID_MAX || barriers > KICK_BARRIERS_MAX || args.qos.class > Self::QOS_CLASS_MAX + || args.priority >= Policy::PRIORITIES { return Err(EINVAL); } let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT - | (QUEUE_PRIORITY as u64) << Self::PRIORITY_SHIFT; + | (args.priority as u64) << Self::PRIORITY_SHIFT; if args.compute_scratch { header |= Self::COMPUTE_SCRATCH; } @@ -506,9 +509,10 @@ impl RenderRegistration { /// Validates both queues without touching MMIO. The resulting transaction /// writes fragment configuration and validity before tiling configuration - /// and validity. Each validity mask is stored high before low. - pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64)) -> Result { - if fragment.0 == tiling.0 { + /// and validity. Each validity mask is stored high before low. Both queues + /// are registered at priority class `priority`. + pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64), priority: u8) -> Result { + if fragment.0 == tiling.0 || priority >= Policy::PRIORITIES { return Err(EINVAL); } let mut writes = [(0, 0); 8]; @@ -522,7 +526,7 @@ impl RenderRegistration { if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { return Err(EINVAL); } - let completion = (3 * (4 - u64::from(QUEUE_PRIORITY)) + 0x3fe).wrapping_shl(54); + let completion = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); let payload = ((engine as u64 + 1) << 60) | va | completion; let (low, high) = if qid < 64 { (1u64 << qid, 0) diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index db95ba99e0189a..48b110a4193e3b 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -22,14 +22,59 @@ use kernel::prelude::*; use super::kick::{KickTimestamp, QID_MAX}; -/// Priority of every queue the driver creates. +/// Firmware scheduling profile of a queue. /// -/// The same value appears in the queue record, [`QueueConfig`], -/// [`KickAnnounce`] and the kick entry header; the work rings it selects are -/// listed in `channels.rs`. -pub(crate) const QUEUE_PRIORITY: u8 = 2; -/// Scheduling policy of every queue the driver creates. -pub(crate) const QUEUE_POLICY: u16 = 2; +/// Every record that names a queue repeats its priority class: the queue +/// record (twice), [`QueueConfig`], [`KickAnnounce`], the kick entry header, +/// the work ring the queue is published on (`channels.rs`), the doorbell and +/// the render registration token. The queue record and [`QueueConfig`] also +/// carry the scheduling policy word; the QoS row and share word carry the QoS +/// class and share. All of them must agree for one queue. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Policy { + priority: u8, + policy: u16, + qos_class: u8, + qos_share: u32, +} + +impl Policy { + /// Number of firmware priority classes. + pub(crate) const PRIORITIES: u8 = 4; + + /// Returns the profile with priority class `priority`, policy word + /// `policy`, QoS class `qos_class` and QoS share `qos_share`. + pub(crate) const fn new(priority: u8, policy: u16, qos_class: u8, qos_share: u32) -> Self { + assert!(priority < Self::PRIORITIES); + Self { + priority, + policy, + qos_class, + qos_share, + } + } + + /// Priority class, below [`Self::PRIORITIES`]. + pub(crate) const fn priority(&self) -> u8 { + self.priority + } + + /// Scheduling policy word. + pub(crate) const fn policy(&self) -> u16 { + self.policy + } + + /// QoS class byte of the queue's QoS row. + pub(crate) const fn qos_class(&self) -> u8 { + self.qos_class + } + + /// QoS share word of the owner's QoS slot. + pub(crate) const fn qos_share(&self) -> u32 { + self.qos_share + } +} + /// Number of work pointers in a queue's item ring. pub(crate) const ITEM_RING_ENTRIES: u32 = 0x500; @@ -173,13 +218,15 @@ pub(crate) struct QueueRecordArgs { pub(crate) scheduler_va: u64, /// Thread-group ID of the owning process (0 while unowned). pub(crate) owner_pid: u32, + /// Scheduling profile of the owning logical queue. + pub(crate) policy: Policy, } /// Description of one hardware queue, named by its work-ring slots. /// /// The host writes it whole at queue creation and changes the owner fields of -/// an idle queue with [`QueueRecord::set_owner`] or -/// [`QueueRecord::set_compute_owner`]. +/// an idle queue with [`QueueRecord::set_compute_owner`] or, for render +/// queues, field by field (`render/memory.rs`). #[repr(C, packed)] #[derive(Copy, Clone, Debug)] pub(crate) struct QueueRecord { @@ -224,12 +271,12 @@ impl QueueRecord { job_list_va: args.job_list_va, unk_18: [0; 0xc], unk_24: u32::MAX, - priority: [QUEUE_PRIORITY as u32; 2], + priority: [args.policy.priority() as u32; 2], unk_30: [0; 6], unk_36: u16::MAX, unk_38: 0, unk_3c: 0, - policy: QUEUE_POLICY as u32, + policy: args.policy.policy() as u32, unk_44: u32::MAX, owner_pid: args.owner_pid, unk_4c: [0; 0x50], @@ -241,16 +288,32 @@ impl QueueRecord { /// Hands an idle compute queue to a new owner. /// /// Replaces the job head, process identity and scheduler page, and clears `unk_38`. + /// The priority and policy words are written only when `policy` differs from the + /// profile the record holds. pub(crate) fn set_compute_owner( &mut self, job_list_va: u64, scheduler_va: u64, owner_pid: u32, + policy: Policy, ) { self.job_list_va = job_list_va; self.unk_38 = 0; self.owner_pid = owner_pid; self.scheduler_va = scheduler_va; + self.set_policy(policy); + } + + /// Replaces the priority and policy words of an idle queue's record when they differ + /// from `policy`. + pub(crate) fn set_policy(&mut self, policy: Policy) { + let priority = [policy.priority() as u32; 2]; + if { self.priority } != priority { + self.priority = priority; + } + if { self.policy } != policy.policy() as u32 { + self.policy = policy.policy() as u32; + } } } @@ -310,6 +373,8 @@ pub(crate) struct QueueConfigArgs { pub(crate) free_list: FreeListBinding, /// GPU address of the owner's scheduler-state page. pub(crate) scheduler_va: u64, + /// Scheduling profile of the owning logical queue. + pub(crate) policy: Policy, /// QoS buffer slot of the logical queue. pub(crate) qos_slot: u8, /// The QoS identity or queue membership changed since the last update. @@ -429,7 +494,7 @@ impl QueueConfig { qid: args.qid as u16, install: args.install as u16, unk_1c: 0, - priority: QUEUE_PRIORITY as u16, + priority: args.policy.priority() as u16, unk_20: 0, data_master: data_master as u16, unk_24: 0, @@ -445,7 +510,7 @@ impl QueueConfig { qos_slot: args.qos_slot as u16, unk_58: 0, owner_pid: args.owner_pid, - policy: QUEUE_POLICY, + policy: args.policy.policy(), unk_60: 0, context_update: args.context_update as u16, unk_64: 0, @@ -486,16 +551,22 @@ static_assert!(core::mem::offset_of!(KickAnnounce, timestamp) == 0x09); static_assert!(core::mem::offset_of!(KickAnnounce, data_master) == 0x11); impl KickAnnounce { - /// Announces the entry of queue `qid` written at `timestamp`. - pub(crate) fn new(qid: u8, data_master: DataMaster, timestamp: KickTimestamp) -> Result { - if qid > QID_MAX { + /// Announces the entry of queue `qid` at priority class `priority`, + /// written at `timestamp`. + pub(crate) fn new( + qid: u8, + data_master: DataMaster, + timestamp: KickTimestamp, + priority: u8, + ) -> Result { + if qid > QID_MAX || priority >= Policy::PRIORITIES { return Err(EINVAL); } Ok(Self { tag: CommandTag::KickAnnounce as u32, qid: qid as u16, count: 1, - priority: QUEUE_PRIORITY, + priority, timestamp: timestamp.get(), data_master: data_master as u32, unk_15: [0; 3], diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index aea894ba2c2d82..a41e4de24950d0 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -416,14 +416,20 @@ impl InitData { /// The sole host publisher holds the device mutex through the following /// inner and outer publication. Firmware consumers can only free capacity; /// neither this check nor waiting for room reserves or limits other queues. - pub(super) fn work_ready(&self, engine: super::fw::queue::DataMaster) -> Result { - let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + /// `priority` selects the engine's ring of that priority class. + pub(super) fn work_ready( + &self, + priority: u8, + engine: super::fw::queue::DataMaster, + ) -> Result { + let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; Ok(next != consumer && next != consumer_b) } fn work_publication<'a>( &'a self, + priority: u8, record: &'a super::fw::channels::WorkSlot, ) -> Result> { use super::fw::{channels, queue::DataMaster}; @@ -433,8 +439,8 @@ impl InitData { 2 => DataMaster::Compute, _ => return Err(EINVAL), }; - let channel = channels::work_ring_index(engine); - let [consumer, consumer_b, producer] = self.work_cursors(engine)?; + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; + let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; let next = (producer + 1) % channels::WORK_RING_SLOTS; if next == consumer || next == consumer_b { return Err(EINVAL); @@ -457,9 +463,13 @@ impl InitData { } /// Two firmware consumers and the host producer, sampled in that order. - pub(super) fn work_cursors(&self, engine: super::fw::queue::DataMaster) -> Result<[u32; 3]> { + pub(super) fn work_cursors( + &self, + priority: u8, + engine: super::fw::queue::DataMaster, + ) -> Result<[u32; 3]> { use super::fw::channels; - let channel = channels::work_ring_index(engine); + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); let state = self.object(self.cluster)?; let mut values = [0; 3]; @@ -481,8 +491,12 @@ impl InitData { /// Inner queue records must be visible before this outer publication. The returned /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work(&self, record: &super::fw::channels::WorkSlot) -> Result { - Ok(self.work_publication(record)?.commit()) + pub(super) fn publish_work( + &self, + priority: u8, + record: &super::fw::channels::WorkSlot, + ) -> Result { + Ok(self.work_publication(priority, record)?.commit()) } /// Validate both destinations before exposing fragment, then tiling. The accounting token @@ -490,6 +504,7 @@ impl InitData { /// Deferred pairs retain both slot bodies without exposing either producer. pub(super) fn publish_work_pair( &self, + priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, deferred: bool, @@ -502,8 +517,8 @@ impl InitData { { return Err(EINVAL); } - let fragment = self.work_publication(fragment)?; - let tiling = self.work_publication(tiling)?; + let fragment = self.work_publication(priority, fragment)?; + let tiling = self.work_publication(priority, tiling)?; fragment.copy_record(); commit_accounting()?; fence(Ordering::SeqCst); @@ -522,6 +537,7 @@ impl InitData { fn retained_work_publication<'a>( &'a self, + priority: u8, record: &'a super::fw::channels::WorkSlot, next: u32, ) -> Result> { @@ -531,7 +547,7 @@ impl InitData { 1 => DataMaster::Fragment, _ => return Err(EINVAL), }; - let channel = channels::work_ring_index(engine); + let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; let producer = self.object(self.cluster)?.word( Self::state_offset(Role::Primary) + abi::work_state_offset(channel) @@ -558,6 +574,7 @@ impl InitData { /// witnesses before either copy; both bodies precede the fragment and tiling publications. pub(super) fn publish_retained_work_pair( &self, + priority: u8, fragment: &super::fw::channels::WorkSlot, tiling: &super::fw::channels::WorkSlot, next: [u32; 2], @@ -568,8 +585,8 @@ impl InitData { { return Err(EINVAL); } - let fragment = self.retained_work_publication(fragment, next[0])?; - let tiling = self.retained_work_publication(tiling, next[1])?; + let fragment = self.retained_work_publication(priority, fragment, next[0])?; + let tiling = self.retained_work_publication(priority, tiling, next[1])?; fragment.copy_record(); tiling.copy_record(); fence(Ordering::SeqCst); diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index e384c1f7a63335..ea45c44ecd74d0 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1173,8 +1173,9 @@ impl gpu::Gpu for Gpu { let owner = self.ids.queue.next(); let frontend = queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; + let policy = scheduling_policy(priority)?; let (backend, slot) = loop { - match runtime::Backend::new(&self.shared, &vm, owner) { + match runtime::Backend::new(&self.shared, &vm, owner, policy) { Err(error) if error == EAGAIN => continue, result => break result?, } @@ -1215,6 +1216,17 @@ impl gpu::Gpu for Gpu { } } +/// Scheduling profile of a new logical queue. `file.rs` passes +/// `DRM_ASAHI_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so +/// MEDIUM arrives as 2 and LOW as 3. +fn scheduling_policy(priority: u32) -> Result { + match priority { + 2 => Ok(hw::t8140::scheduling::MEDIUM), + 3 => Ok(hw::t8140::scheduling::LOW), + _ => Err(EINVAL), + } +} + impl Drop for Gpu { fn drop(&mut self) { self.shutdown(); diff --git a/drivers/gpu/drm/asahi/g17/qos.rs b/drivers/gpu/drm/asahi/g17/qos.rs index 8a925c23552a73..9a100db4cffdfc 100644 --- a/drivers/gpu/drm/asahi/g17/qos.rs +++ b/drivers/gpu/drm/asahi/g17/qos.rs @@ -6,13 +6,16 @@ //! precedes submitted counters; retiring a final owner clears only its two row //! bytes. Cancellation balances host references without rewinding shared counts. -use super::{fw::qos as fw, object::KernelObject}; +use super::{ + fw::{qos as fw, queue::Policy}, + object::KernelObject, +}; +use crate::hw::t8140::qos as cfg; use core::{ mem::{offset_of, size_of}, sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, }; use kernel::prelude::*; -use crate::hw::t8140::qos as cfg; /// Atomic views retain the allocation borrow and never borrow firmware words /// as ordinary mutable memory. The arrays are disjoint apart from the explicit @@ -97,11 +100,13 @@ impl Accounting { /// Called under the device publication lock immediately before exposing /// the work record. The clock is sampled only when its engine becomes busy. + /// The row's class and the owner's share come from the owner's `policy`. pub(crate) fn publish( &mut self, view: &View<'_>, owner: Owner, scheduler: u64, + policy: Policy, clock: impl FnOnce() -> (u64, u64), ) -> Result { let (qid, qos, dm) = owner.indices()?; @@ -124,10 +129,10 @@ impl Accounting { return Err(EBUSY); } view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(cfg::CLASS, Ordering::Relaxed); + view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); let record = view.row(qid); view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(cfg::SHARE, Ordering::Relaxed); + view.share[qos].store(policy.qos_share(), Ordering::Relaxed); if self.in_flight[qid] == 0 { fence(Ordering::SeqCst); } diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 57179437f9b40b..2b962e26326063 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -137,6 +137,7 @@ impl Graph { job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), + policy: context.policy(), }); view.pointers = PointerBlock::new(); view.jobs = JobListHead::new(graph_va + 0x200); @@ -210,6 +211,7 @@ impl Graph { context.work_head_va(), context.scheduler_va(), context.owner_pid(), + context.policy(), ) }; Ok(()) @@ -468,7 +470,8 @@ pub(crate) struct Queue { replay_prepared: bool, exit: Option, installed: bool, - outer_started: bool, + /// Bit `p` set once a work slot of priority class `p` named this queue. + outer_started: u8, published_config: Option, ordinal: u32, submitted: u32, @@ -521,7 +524,7 @@ impl Queue { replay_prepared: false, exit: None, installed: false, - outer_started: false, + outer_started: 0, published_config: None, ordinal: 0, submitted: 0, @@ -1459,10 +1462,16 @@ pub(crate) trait Host { fn next_compute_publication(&mut self) -> Result; fn epoch(&self) -> Result<(u64, u32)>; fn prepare_compute_shared(&mut self) -> Result; - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, slot: &crate::g17::fw::channels::WorkSlot) -> Result; + fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) + -> Result; fn notify(&mut self, message: u64) -> Result; fn note_submission(&mut self) -> Result; } @@ -1580,6 +1589,8 @@ impl Queue { context: packet.context.id(), }; self.free_list_generation = u64::from(packet.context.scheduler_generation()); + let policy = packet.context.policy(); + let ring = 1u8 << policy.priority(); let config = QueueConfig::new(&QueueConfigArgs { target: QueueConfigTarget::Compute, kick_ring_va: self.kick.low_va(), @@ -1594,6 +1605,7 @@ impl Queue { slot: u32::from(self.pool.id()), }, scheduler_va: identity.scheduler, + policy, qos_slot: identity.qos, qos_update: self .published_config @@ -1603,7 +1615,8 @@ impl Queue { .published_config .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), })?; - let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp)?; + let announce = + KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; let kick_args = KickArgs { qid, @@ -1620,6 +1633,7 @@ impl Queue { event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, compute_scratch: scratch.enabled(), + priority: policy.priority(), }; let published = host.next_compute_publication()?; self.active @@ -1690,6 +1704,7 @@ impl Queue { data_master: 2, }, identity.scheduler, + policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; let target = self.pool.reserve_submission().or_else(|error| { @@ -1725,6 +1740,7 @@ impl Queue { data_master: 2, }, identity.scheduler, + policy, )?; self.active.back_mut().ok_or(EIO)?.qos_pending = true; producer = self @@ -1754,15 +1770,18 @@ impl Queue { self.graph.record_ends[slot] = Some(producer); } host.publish_qid(self.kick.id())?; - host.publish_outer(&WorkSlot::new( - DataMaster::Compute, - self.graph.graph.gpu_va(), - qid, - producer.try_into()?, - !self.outer_started, - )?)?; - let notify_activation = !self.outer_started; - self.outer_started = true; + host.publish_outer( + policy.priority(), + &WorkSlot::new( + DataMaster::Compute, + self.graph.graph.gpu_va(), + qid, + producer.try_into()?, + self.outer_started & ring == 0, + )?, + )?; + let notify_activation = self.outer_started & ring == 0; + self.outer_started |= ring; self.installed = true; self.published_config = Some(identity); self.ordinal = ordinal; @@ -1791,7 +1810,8 @@ impl Queue { } if notify_activation { fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x0a)?; + // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. + host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; } fence(Ordering::SeqCst); host.notify((0x83 << 48) | 0x10)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 05b9bf14ecd2e4..60b30cde980944 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -74,12 +74,18 @@ pub(crate) struct PublicationLease { pub(crate) trait Host { fn epoch(&self) -> Result<(u64, u32)>; fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result; + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: crate::g17::fw::queue::Policy, + ) -> Result; fn qos_cancel(&mut self, publication: qos::Publication) -> Result; fn install_pair(&mut self, registration: &RenderRegistration) -> Result; fn publish_qid(&mut self, id: kick::Id) -> Result; fn publish_outer_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, commit: impl FnOnce() -> Result, @@ -88,11 +94,12 @@ pub(crate) trait Host { fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; fn publish_retained_pair( &mut self, + priority: u8, fragment: &WorkSlot, tiling: &WorkSlot, next: [u32; 2], ) -> Result; - fn notify_render(&mut self) -> Result; + fn notify_render(&mut self, priority: u8) -> Result; fn note_submission(&mut self) -> Result; fn complete_render( &mut self, @@ -141,8 +148,11 @@ pub(crate) struct Pair { configs: [Option; 2], cursors: [u32; 2], recovery_generation: u64, - registered: bool, - outer_started: bool, + /// Priority class of the pair's installed registration, if any. A pair + /// rebound to an owner of another class is registered again. + registered: Option, + /// Bit `p` set once a work slot of priority class `p` named the pair. + outer_started: u8, quarantined: bool, terminal: Option<(Arc, Error)>, previous: Option, @@ -223,8 +233,8 @@ impl Pair { configs: [None; 2], cursors: [0; 2], recovery_generation: 0, - registered: false, - outer_started: false, + registered: None, + outer_started: 0, quarantined: false, terminal: None, previous: None, @@ -434,6 +444,12 @@ impl Pair { if packet.completion.status().get() != 0 { return Err(EIO); } + let policy = packet.context.policy(); + // A registration of another class can only follow a rebind, which + // requires the pair to have drained. + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + return Err(EIO); + } let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; let mcache = prepared.mcache; @@ -450,6 +466,7 @@ impl Pair { self.memory .completion_links(ordinal, kicks, payload, !covered[1])?; let context = &packet.context; + let ring = 1u8 << policy.priority(); let qos = context.qos_id(); let qos_owners = [ qos::Owner { @@ -492,6 +509,7 @@ impl Pair { slot: self.free_list.id().into(), }, scheduler_va: context.scheduler_va(), + policy, qos_slot: qos, qos_update: self.context_update, owner_pid: context.owner_pid(), @@ -561,6 +579,7 @@ impl Pair { event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, compute_scratch: false, + priority: policy.priority(), }; let ta_args = args( 0, @@ -579,6 +598,7 @@ impl Pair { let registration = RenderRegistration::new( (qids[1], self.kicks[1].low_va()), (qids[0], self.kicks[0].low_va()), + policy.priority(), )?; let [tiling_kick, fragment_kick] = &mut self.kicks; let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; @@ -620,14 +640,14 @@ impl Pair { queues[1], qids[1], windows[1].target().try_into()?, - !self.outer_started, + self.outer_started & ring == 0, )?; let tiling_slot = WorkSlot::new( DataMaster::Tiling, queues[0], qids[0], windows[0].target().try_into()?, - !self.outer_started, + self.outer_started & ring == 0, )?; let event_slot = memory .graph @@ -664,16 +684,20 @@ impl Pair { tiling_kick.commit(); let mut qos_publications = [None; 2]; let result = (|| { - qos_publications[1] = Some(host.qos_publish(qos_owners[1], context.scheduler_va())?); - qos_publications[0] = Some(host.qos_publish(qos_owners[0], context.scheduler_va())?); - if !self.registered { + qos_publications[1] = + Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); + qos_publications[0] = + Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); + // A class change happens only at a rebind, after the pair drained. + if self.registered != Some(policy.priority()) { host.install_pair(®istration)?; - self.registered = true; + self.registered = Some(policy.priority()); } for kick in &self.kicks { host.publish_qid(kick.id())?; } host.publish_outer_pair( + policy.priority(), &fragment_slot, &tiling_slot, || { @@ -720,16 +744,16 @@ impl Pair { } self.recovery_generation = epoch; self.installed = [true; 2]; - self.outer_started = true; + self.outer_started |= ring; if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { previous.publication = Some(ordinal); } packet.context.mark_published(); let publish = (|| { host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; - host.publish_retained_pair(&fragment_slot, &tiling_slot, outer_next)?; + host.publish_retained_pair(policy.priority(), &fragment_slot, &tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; - host.notify_render()?; + host.notify_render(policy.priority())?; host.note_submission() })(); if let Err(error) = publish { diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index 859ff61901570e..fc5a05e9bf8a91 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -32,6 +32,9 @@ pub(super) const JOB_LIST: usize = 0xa000; pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; pub(super) const CONFIG_STRIDE: usize = 0x180; const DEPTH: usize = queues::RENDER_DEPTH as usize; +/// Offsets of the priority pair and the policy word in a queue record. +const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); +const POLICY: usize = core::mem::offset_of!(queue::QueueRecord, policy); struct Aliases { client: mmu::KernelMapping, @@ -147,6 +150,7 @@ impl Graph { job_list_va: context.work_head_va(), scheduler_va: context.scheduler_va(), owner_pid: context.owner_pid(), + policy: context.policy(), }), )?; queues.initialize::(POINTERS[stage], |view| { @@ -665,6 +669,9 @@ impl Memory { Ok(()) } + /// Retargets both retired queue records to `context`. A context of another + /// scheduling profile also replaces the priority and policy words, after + /// the owner fields; the same profile leaves them untouched. pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { let offsets = qids.map(|qid| usize::from(qid) * size_of::()); let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; @@ -674,7 +681,10 @@ impl Memory { self.graph.queues.pointer(at + 0x10, 8)?; self.graph.queues.pointer(at + 0x48, 4)?; self.graph.queues.pointer(at + 0x9c, 8)?; + self.graph.queues.pointer(at + PRIORITY, 8)?; + self.graph.queues.pointer(at + POLICY, 4)?; } + let policy = context.policy(); // SAFETY: Both queues are retired before retargeting. Complete byte // views were checked above; packed scheduler words use unaligned stores. unsafe { @@ -693,6 +703,17 @@ impl Memory { .cast::() .write_unaligned(context.scheduler_va()); } + for at in offsets { + let words = base.add(at + PRIORITY).cast::<[u32; 2]>(); + let priority = [u32::from(policy.priority()); 2]; + if words.read_unaligned() != priority { + words.write_unaligned(priority); + } + let word = base.add(at + POLICY).cast::(); + if word.read_unaligned() != u32::from(policy.policy()) { + word.write_unaligned(u32::from(policy.policy())); + } + } } Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index 3d67578e2aaf98..f712bb7b1b87dd 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -28,8 +28,8 @@ pub(crate) struct ParkedPair { descriptor_flags: [u32; 2], installed: [bool; 2], recovery_generation: u64, - registered: bool, - outer_started: bool, + registered: Option, + outer_started: u8, previous: Option, } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs index 3df92062735e33..0fc9c1d9a15dcf 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/publication.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/publication.rs @@ -99,6 +99,7 @@ impl<'a> Lane<'a> { DataMaster::Fragment }, timestamp, + config.priority as u8, )?, announce_destination, prefix: count as usize - 1, diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 842e3a8b8df9be..4c7aa6a442cc10 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -368,6 +368,7 @@ impl Backend { shared: &Arc, vm: &crate::mmu::Vm, owner: u64, + policy: super::fw::queue::Policy, ) -> Result<(Arc, admission::QueueSlot)> { let lease = shared.preparations.enter(true)?; let dev = shared.drm()?; @@ -394,7 +395,7 @@ impl Backend { let owner_pid = u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid)?, + Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, GFP_KERNEL, )?; let state = shared.state.lock(); @@ -776,11 +777,17 @@ impl compute::Host for ComputeHost<'_> { fn prepare_compute_shared(&mut self) -> Result { self.init.activate_compute() } - fn qos_publish(&mut self, owner: qos::Owner, scheduler: u64) -> Result { + fn qos_publish( + &mut self, + owner: qos::Owner, + scheduler: u64, + policy: super::fw::queue::Policy, + ) -> Result { self.accounting.publish( &qos::View::new(self.init.qos()?)?, owner, scheduler, + policy, qos::clock, ) } @@ -791,8 +798,8 @@ impl compute::Host for ComputeHost<'_> { fn publish_qid(&mut self, id: kick::Id) -> Result { self.qids.publish(id) } - fn publish_outer(&mut self, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(slot).map(|_| ()) + fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { + self.init.publish_work(priority, slot).map(|_| ()) } fn notify(&mut self, message: u64) -> Result { self.primary.notify(message) @@ -816,7 +823,10 @@ impl super::Firmware { } // No descriptor, accounting or inner producer changes precede this // capacity check. The same device lock covers queue.publish below. - if !self.init.work_ready(DataMaster::Compute)? { + if !self + .init + .work_ready(packet.context.policy().priority(), DataMaster::Compute)? + { return Err(EAGAIN); } let queues = &mut self.queues; diff --git a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs index 65b00d36714bb0..0d4a29e779b4a8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/compute_replay.rs @@ -56,7 +56,10 @@ impl Firmware { break; }; let owner = queue.owner(); - let outer_ready = self.init.work_ready(crate::g17::fw::queue::DataMaster::Compute)?; + let outer_ready = self.init.work_ready( + packet.context.policy().priority(), + crate::g17::fw::queue::DataMaster::Compute, + )?; if !outer_ready { let now = super::now_ns(); let since = *self.queues.replay_outer_wait.get_or_insert(now); diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index aabc523e360cdd..71b21f98479a6c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -377,12 +377,14 @@ impl crate::g17::queue::render::Host for Host<'_> { &mut self, owner: crate::g17::qos::Owner, scheduler: u64, + policy: crate::g17::fw::queue::Policy, ) -> Result { use crate::g17::qos; self.firmware.queues.accounting.publish( &qos::View::new(self.firmware.init.qos()?)?, owner, scheduler, + policy, qos::clock, ) } @@ -418,6 +420,7 @@ impl crate::g17::queue::render::Host for Host<'_> { } fn publish_outer_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, commit: impl FnOnce() -> Result, @@ -425,7 +428,7 @@ impl crate::g17::queue::render::Host for Host<'_> { ) -> Result<[u32; 2]> { self.firmware .init - .publish_work_pair(fragment, tiling, true, commit, late_tiling) + .publish_work_pair(priority, fragment, tiling, true, commit, late_tiling) } fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { self.release_registration(); @@ -437,6 +440,7 @@ impl crate::g17::queue::render::Host for Host<'_> { } fn publish_retained_pair( &mut self, + priority: u8, fragment: &crate::g17::fw::channels::WorkSlot, tiling: &crate::g17::fw::channels::WorkSlot, next: [u32; 2], @@ -444,18 +448,19 @@ impl crate::g17::queue::render::Host for Host<'_> { let result = self .firmware .init - .publish_retained_work_pair(fragment, tiling, next); + .publish_retained_work_pair(priority, fragment, tiling, next); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } result } - fn notify_render(&mut self) -> Result { + /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. + fn notify_render(&mut self, priority: u8) -> Result { self.release_registration(); let result = self .firmware .primary - .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | 0x08); + .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2); if result.is_err() { self.firmware.primary.state.shared.lose_device_quietly(); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index 39b8c064d0106c..dbf7175c5cd4bd 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -275,8 +275,9 @@ impl crate::g17::Firmware { } // Both outer destinations must have room before pool accounting or // either inner queue becomes visible. All host producers share this lock. - if !self.init.work_ready(crate::g17::fw::queue::DataMaster::Fragment)? - || !self.init.work_ready(crate::g17::fw::queue::DataMaster::Tiling)? + let priority = packet.context.policy().priority(); + if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? + || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? { return Err(EBUSY); } diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 4a97217a00705d..848f753a6d03d3 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -378,6 +378,17 @@ pub(crate) mod qos { pub(crate) const CLOCK_HZ: u64 = 24_000_000; } +/// Firmware scheduling profiles of the DRM queue priorities. Medium-priority +/// user queues use priority class 2 with policy 2. +pub(crate) mod scheduling { + use crate::g17::fw::queue::Policy; + + /// `DRM_ASAHI_PRIORITY_MEDIUM`. + pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); + /// `DRM_ASAHI_PRIORITY_LOW`. + pub(crate) const LOW: Policy = Policy::new(3, 2, super::qos::CLASS, super::qos::SHARE); +} + /// Physical workqueue capacity and logical-owner admission limits. pub(crate) mod queues { pub(crate) const RENDER_SLOTS: usize = 64; From 7a81838f9d33a7fbe866752fff7912a9071f1e0d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:18:07 -0400 Subject: [PATCH 105/631] drm/asahi: drop two defensive clippy allowances in the VM binding paths map_in_range_with_guard_size() and prepare_bind_object() take seven parameters including self, which is clippy's default too_many_arguments threshold, not above it. The allowances silenced a lint that does not fire. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 009b0fb93ba35380563055aa8bc725f36c505262) --- drivers/gpu/drm/asahi/mmu.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 2bda44fc9ba4d9..0b1d5b66bb69b9 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2688,7 +2688,6 @@ impl Vm { } /// Reserves a whole-page unmapped guard without adding it to the mapped backing. - #[allow(clippy::too_many_arguments)] pub(crate) fn map_in_range_with_guard_size( &self, gem: &gem::Object, From 673e92d97c8dafad5821436bf396b837235cf698 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:24:25 -0400 Subject: [PATCH 106/631] drm/asahi: build the G17 boot objects region by region InitData::build placed and initialized every boot object in one 360-line function with two placement closures. Keep the placement inputs in a small environment and perform the same steps, in the same order, as one function per region: register windows, descriptor tables, render free lists, the render pool, the compute pools, the bundle with its rings and regions, and the firmware-object initialization. The four fences keep their positions and the firmware objects are byte-identical. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 410d8f80e9ab6b405d7f918901fcbe883bb2d9b2) --- drivers/gpu/drm/asahi/g17/initdata.rs | 258 +++++++++++++++++++------- 1 file changed, 187 insertions(+), 71 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index a41e4de24950d0..5993be786874e7 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -11,6 +11,7 @@ use super::{ Config, Role, }; use crate::{driver::AsahiDevice, hw::t8140, mmu}; +use core::ops::Range; use core::sync::atomic::{fence, AtomicU32, Ordering}; use kernel::{c_str, prelude::*}; use t8140::{kernel_window as high, low}; @@ -166,6 +167,57 @@ pub(super) struct InitData { render_blocks: core::ops::Range, } +/// Placement inputs shared by the build steps. +struct BuildEnv<'a> { + dev: &'a AsahiDevice, + vm: &'a mmu::Vm, + lower: &'a mmu::Vm, + render: &'a mmu::Vm, + base: u64, + dynamic: Range, +} + +impl BuildEnv<'_> { + /// An object at a fixed offset of the kernel window. + fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { + KernelObject::new( + self.dev, + self.vm, + Placement::At(self.base + offset), + size, + prot, + cpu, + ) + } + + /// An object placed in the dynamic part of the kernel window. + fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { + KernelObject::new( + self.dev, + self.vm, + Placement::In(self.dynamic.clone(), align), + size, + prot, + cpu, + ) + } +} + +/// Object indices the firmware-object initialization needs from the table step. +struct DescriptorTables { + pb: usize, + page_pool: usize, + fwctl: usize, +} + +/// Object indices and ring addresses the firmware-object initialization needs from the bundle step. +struct BundleObjects { + zero: usize, + completion_rings: [abi::CompletionRing; 2], + region_a: usize, + qos: usize, +} + impl InitData { /// Empty construction state, retained even if graph construction fails. pub(super) fn new() -> Self { @@ -643,46 +695,51 @@ impl InitData { render: &mmu::Vm, metrics: &KernelObject, ) -> Result { - let vm = uat.kernel_vm(); - let lower = uat.kernel_lower_vm(); let range = uat.geometry().kernel_range(); - let base = range.start; - let dynamic = base + t8140::dynamic::KERNEL_OFFSET..range.end; - let fixed = |offset, size, prot, cpu| { - KernelObject::new(dev, vm, Placement::At(base + offset), size, prot, cpu) - }; - let dynamic_object = |size, align, prot, cpu| { - KernelObject::new( - dev, - vm, - Placement::In(dynamic.clone(), align), - size, - prot, - cpu, - ) - }; - use mmu::{ - PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, - PROT_GPU_FW_SHARED_RW as GFS, PROT_GPU_SHARED_RW as GS, + let env = BuildEnv { + dev, + vm: uat.kernel_vm(), + lower: uat.kernel_lower_vm(), + render, + base: range.start, + dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; - use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.map_register_windows(&env)?; + let tables = self.build_descriptor_tables(&env)?; + self.build_render_free_lists(&env)?; + let first_block = self.build_render_pool(&env)?; + self.build_compute_pools(&env, first_block)?; + let bundle = self.build_bundle(&env, cfg, metrics)?; + self.initialize_firmware_objects(&env, cfg, &tables, &bundle) + } + fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { for window in &t8140::REGISTER_WINDOWS { let (phys, va, size) = window.mapping().ok_or(EINVAL)?; self.aliases.push( - vm.map_io(va, phys.try_into()?, size.try_into()?, mmu::PROT_FW_MMIO_RW)?, + env.vm.map_io(va, phys.try_into()?, size.try_into()?, mmu::PROT_FW_MMIO_RW)?, GFP_KERNEL, )?; } - self.cluster = self.add(fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; - self.compute_global = self.add(fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; - let pb = self.add(fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + Ok(()) + } + + /// The private cluster, zero page and the three descriptor tables with their low aliases. + fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, + PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; + self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; + let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(pb)? .initialize::(0, |table| { table.init(); Ok(()) })?; - let page_pool = self.add(fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; + let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; self.object_mut(page_pool)? .initialize::(0, |table| { table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); @@ -690,17 +747,30 @@ impl InitData { })?; self.pb = pb; self.page_pool = page_pool; - self.alias(pb, lower, low::PB_DESCRIPTORS, GS)?; - self.alias(page_pool, lower, low::PAGE_POOL_DESCRIPTORS, GS)?; - let index = self.add(fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; + self.alias(pb, env.lower, low::PB_DESCRIPTORS, GS)?; + self.alias(page_pool, env.lower, low::PAGE_POOL_DESCRIPTORS, GS)?; + let index = self.add(env.fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; self.object_mut(index)? .initialize::(0, |table| { table.init(); Ok(()) })?; - self.alias(index, lower, low::INDEX, GFS)?; - let fwctl = self.add(fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + self.alias(index, env.lower, low::INDEX, GFS)?; + let fwctl = self.add(env.fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; + Ok(DescriptorTables { + pb, + page_pool, + fwctl, + }) + } + /// Render and boot free-list controls, descriptor pages and the two queue contexts. + fn build_render_free_lists(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, + PROT_GPU_FW_SHARED_RW as GFS, + }; + use CpuMap::WriteCombined as WC; for (offset, state, class, prot, initial) in [ ( high::RENDER_FREE_LIST, @@ -717,7 +787,7 @@ impl InitData { abi::FreeListState::BOOT, ), ] { - let control = self.add(fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; + let control = self.add(env.fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; self.object_mut(control)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -726,7 +796,7 @@ impl InitData { page_list_va: low::RENDER_PAGE_LIST, run_list_va: low::RENDER_RUN_LIST, blocks: low::RENDER_BLOCKS_INITIAL, - state_va: base + state, + state_va: env.base + state, }), )?; let state_prot = if class == abi::FreeListClass::One { @@ -734,19 +804,26 @@ impl InitData { } else { GFS }; - let state = self.add(fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; + let state = self.add(env.fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; self.object_mut(state)?.write(0, initial)?; if class == abi::FreeListClass::Two { self.boot_control = control; self.boot_state = state; } } - self.stamps[0] = self.add(fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; - self.stamps[1] = self.add(fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; - self.add(fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; - self.add(fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.stamps[0] = self.add(env.fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; + self.stamps[1] = self.add(env.fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; + self.add(env.fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + self.add(env.fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; + Ok(()) + } - let pages = self.add(KernelObject::backing(dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; + /// Render page list, run list and blocks aliased into the render-global root; returns + /// the index of the first block. + fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { + use mmu::PROT_GPU_SHARED_RW as GS; + use CpuMap::WriteBack as WB; + let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; self.object_mut(pages)? .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { abi::fill_page_list( @@ -754,21 +831,28 @@ impl InitData { (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), ) })?; - let runs = self.add(KernelObject::backing(dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; + let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; let first_block = self.objects.len(); for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } self.render_pages = pages; self.render_runs = runs; self.render_blocks = first_block..self.objects.len(); - self.alias(pages, render, low::RENDER_PAGE_LIST, GS)?; - self.alias(runs, render, low::RENDER_RUN_LIST, GS)?; + self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; + self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, render, low::render_block(block), GS)?; + self.alias(first_block + block, env.render, low::render_block(block), GS)?; } + Ok(first_block) + } - let compute = self.add(fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; + /// Compute free-list controls and states, zero blocks, run list and the compute page + /// list inside the render blocks. + fn build_compute_pools(&mut self, env: &BuildEnv<'_>, first_block: usize) -> Result { + use mmu::PROT_GPU_FW_SHARED_RW as GFS; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + let compute = self.add(env.fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(compute)?.write( 0, abi::FreeListControl::new(&abi::FreeListArgs { @@ -777,18 +861,18 @@ impl InitData { page_list_va: low::COMPUTE_PAGE_LIST, run_list_va: low::COMPUTE_RUN_LIST, blocks: low::COMPUTE_BLOCKS, - state_va: base + high::COMPUTE_FREE_LIST_STATE, + state_va: env.base + high::COMPUTE_FREE_LIST_STATE, }), )?; - let compute_state = self.add(KernelObject::backing(dev, mmu::UAT_PGSZ, WB)?)?; + let compute_state = self.add(KernelObject::backing(env.dev, mmu::UAT_PGSZ, WB)?)?; self.object_mut(compute_state)? .write(0, abi::FreeListState::COMPUTE)?; for _ in &low::COMPUTE_ZERO_BLOCKS { - self.add(KernelObject::backing(dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; + self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; } let compute_runs = self.add(KernelObject::new( - dev, - lower, + env.dev, + env.lower, Placement::At(low::COMPUTE_RUN_LIST), mmu::UAT_PGSZ, GFS, @@ -798,7 +882,7 @@ impl InitData { .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; - let ready = self.add(fixed( + let ready = self.add(env.fixed( high::COMPUTE_READY_FREE_LIST, mmu::UAT_PGSZ, GFS, @@ -812,11 +896,11 @@ impl InitData { page_list_va: low::COMPUTE_POOL_PAGE_LIST, run_list_va: low::COMPUTE_POOL_RUN_LIST, blocks: low::COMPUTE_READY_BLOCKS, - state_va: base + high::COMPUTE_READY_FREE_LIST_STATE, + state_va: env.base + high::COMPUTE_READY_FREE_LIST_STATE, }), )?; for state in [high::COMPUTE_READY_FREE_LIST_STATE, high::COMPUTE_STATE] { - let obj = self.add(fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; + let obj = self.add(env.fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; self.object_mut(obj)? .write(0, abi::FreeListState::COMPUTE)?; } @@ -825,9 +909,24 @@ impl InitData { abi::fill_page_list(list, low::COMPUTE_FREE_LIST_BLOCKS) })?; fence(Ordering::SeqCst); + Ok(()) + } - self.bundle = self.add(fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; - let zero = self.add(fixed( + /// The bundle with its zero page and completion rings, the QoS table, the metrics alias, + /// the roots and regions A and C. + fn build_bundle( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + metrics: &KernelObject, + ) -> Result { + use mmu::{ + PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, + PROT_GPU_SHARED_RW as GS, + }; + use CpuMap::{WriteBack as WB, WriteCombined as WC}; + self.bundle = self.add(env.fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; + let zero = self.add(env.fixed( high::BUNDLE + abi::bundle::ZERO_PAGE as u64, mmu::UAT_PGSZ, GFS, @@ -839,45 +938,62 @@ impl InitData { }; 2]; for slot in 0..2 { let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; - let ring = self.add(fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; + let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; self.completion_rings[slot] = ring; - self.alias(ring, lower, low::COMPLETION_RINGS[slot], GS)?; + self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; completion_rings[slot] = abi::CompletionRing { low_va: low::COMPLETION_RINGS[slot], - high_va: base + offset, + high_va: env.base + offset, }; } - let qos = self.add(dynamic_object(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; + let qos = self.add(env.dynamic(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; self.qos = qos; fence(Ordering::SeqCst); let metrics_alias = - metrics.alias_in(vm, dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; + metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; self.metrics_fw_va = metrics_alias.iova(); self.aliases.push(metrics_alias, GFP_KERNEL)?; fence(Ordering::SeqCst); - self.roots = self.add(dynamic_object( + self.roots = self.add(env.dynamic( abi::ROOTS_SIZE, abi::SECONDARY_ROOT_OFFSET as u64, mmu::PROT_FW_SHARED_RO, WB, )?)?; - let region_a = self.add(dynamic_object( + let region_a = self.add(env.dynamic( REGION_A_SIZE, mmu::UAT_PGSZ as u64, mmu::PROT_FW_SHARED_RO, WB, )?)?; - self.region_c = self.add(dynamic_object(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + self.region_c = self.add(env.dynamic(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; + Ok(BundleObjects { + zero, + completion_rings, + region_a, + qos, + }) + } - let opps = Self::opps(dev)?; + /// Writes every firmware object the host builds from constants and addresses, then the + /// opening control-ring state for both roles. + fn initialize_firmware_objects( + &mut self, + env: &BuildEnv<'_>, + cfg: &Config, + tables: &DescriptorTables, + bundle: &BundleObjects, + ) -> Result { + let base = env.base; + let opps = Self::opps(env.dev)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, windows: &t8140::REGISTER_WINDOWS, reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, opps: &opps, perf: &t8140::PERF, - completion_rings, - qos_va: self.object(qos)?.gpu_va(), + completion_rings: bundle.completion_rings, + qos_va: self.object(bundle.qos)?.gpu_va(), }; self.object_mut(self.bundle)? .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; @@ -890,7 +1006,7 @@ impl InitData { .write(abi::private::PRIMARY_STATUS_A, abi::StatusBlock::new())?; self.object_mut(self.cluster)? .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; - let fwctl_va = self.object(fwctl)?.gpu_va(); + let fwctl_va = self.object(tables.fwctl)?.gpu_va(); self.object_mut(self.cluster)? .initialize::(abi::private::STATUS_B, |status| { status.init(fwctl_va); @@ -923,9 +1039,9 @@ impl InitData { Ok(()) })?; let views = abi::PrimaryViews { - zero_page_va: self.object(zero)?.gpu_va(), - pb_descriptors_va: self.object(pb)?.gpu_va(), - page_pool_descriptors_va: self.object(page_pool)?.gpu_va(), + zero_page_va: self.object(bundle.zero)?.gpu_va(), + pb_descriptors_va: self.object(tables.pb)?.gpu_va(), + page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), }; for role in [Role::Primary, Role::Secondary] { let slot = role as usize; @@ -958,7 +1074,7 @@ impl InitData { abi::OpeningRecord::new(abi_role), )?; let root = abi::RootArgs { - region_a_va: self.object(region_a)?.gpu_va(), + region_a_va: self.object(bundle.region_a)?.gpu_va(), region_c_va: self.object(self.region_c)?.gpu_va(), main_config_va: main_va, status_a_va: status_va, From f38476bff1b265d01404b547ac20ebed6e271e36 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:26:41 -0400 Subject: [PATCH 107/631] drm/asahi: split the G17 render pair install paths out of build_render_pair build_render_pair() held the device-locked selection of how a slot is built, the three off-lock-then-locked install paths and the final locked warm-up in one 400-line function. Move each install path into its own function: install_fresh_pair(), install_parked_pair() and rebind_pair() construct off the lock and install under it exactly as before, and finish_render_build() rebases a fresh graph and warms its pool. A small Copy context carries the lease, allocator, resources, slot and first packet to the install steps. Statement order, lock scopes and error handling are unchanged; the selection stays a block expression in build_render_pair() because returning the selected resources and build by value doubled their stack slots (+1.3 KiB on the scheduler run_job path). The three install steps are inlined: as separate frames they deepen the run_job path, which is close to its stack budget. With them inlined the path measures 17,616 of 18,432 bytes against 17,392 before this change. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit f5bd61786e3b537fd811f0ea27d2ec33874232b0) --- .../drm/asahi/g17/runtime/render/prepare.rs | 575 ++++++++++-------- 1 file changed, 331 insertions(+), 244 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs index b4813cdba033a0..06bea3c84fa7ad 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/prepare.rs @@ -6,6 +6,7 @@ use super::{Config, Reservation}; use crate::g17::{ buffer::MetricsLease, + preparation::Lease, job::Packet, object::{Allocator, KernelObject, Pool}, queue::render::{Pair, RenderAddresses}, @@ -59,6 +60,17 @@ enum Build { }, } +/// What every build step needs besides its variant: the preparation lease, the allocator, +/// the device resources, the slot and the first packet. +#[derive(Clone, Copy)] +struct BuildContext<'a> { + lease: &'a Lease, + alloc: &'a Allocator<'a>, + resources: &'a BuildResources, + slot: u8, + first: &'a Arc, +} + impl super::super::Backend { fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { let lease = self.shared.preparations.enter(false)?; @@ -178,264 +190,339 @@ impl super::super::Backend { Build::Fresh(..) | Build::Parked { .. } => true, Build::Rebind { buffer, .. } => buffer.is_some(), }; - let result = match build { + let ctx = BuildContext { + lease: &lease, + alloc: &alloc, + resources: &resources, + slot, + first, + }; + match build { Build::Fresh(reservation, pm_generation, metrics) => { - let built = (|| -> Result> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - Pair::new( - &alloc, - &resources.objects, - reservation.ids, - slot, - self.owner, - self.context.clone(), - free_list, - metrics, - reservation.buffer, - pm_generation, - first, - resources.config.clusters, - resources.config.descriptor_flags, - ) - })(); - let (mut built, error) = match built { - Ok(pair) => (Some(pair), None), - Err(error) => (None, Some(error)), - }; - let result = (|| { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - if firmware.queues.render.reserved(reservation)?.closed { - return Err(ECANCELED); - } - built - .as_mut() - .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; - firmware.queues.install_render(reservation, &mut built) - })(); - // Fresh backing disappears before its temporary PB reference returns. - drop(built); - if result.is_err() { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .cancel_render_reservation(reservation)?; - } - result + self.install_fresh_pair(&ctx, reservation, pm_generation, metrics)? } Build::Parked { - mut parked, + parked, buffer, pm_generation, metrics, - } => { - let built = (|| -> Result<_> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - let prepared = parked.prepare( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - )?; - let storage = KBox::::new_uninit(GFP_KERNEL)?; - Ok((prepared, storage)) - })(); - let (mut prepared, mut storage, error) = match built { - Ok((prepared, storage)) => (Some(prepared), Some(storage), None), - Err(error) => (None, None, Some(error)), - }; - let result = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) - || entry.pending_owner_closed - || entry.pair.is_some() - || entry.parked.is_some() - { - return Err(ECANCELED); - } - let storage = storage.take().ok_or(EIO)?; - firmware.queues.buffers.can_release(buffer.0)?; - prepared.as_mut().ok_or(EIO)?.finish_install( - &parked, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - let pair = parked.commit(&mut prepared, storage)?; - firmware.queues.buffers.reserve_owner(buffer.0)?; - firmware.queues.buffers.release(buffer.0)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - entry.reservation.buffer = buffer; - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.pair = Some(pair); - entry.closed = false; - entry.published = false; - Ok(()) - })() - }; - // Rejected candidate mappings disappear before the PB identity - // returns, while the original installed kick owner stays live. - drop(prepared); - drop(storage); - let grow_ready = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if result.is_err() { - firmware.queues.render.entry_mut(slot)?.parked = Some(parked); - firmware.queues.buffers.release(buffer.0)?; - } else { - drop(state); - drop(parked); - state = self.shared.state.lock(); - } - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - firmware.render_grow_ready() - }; - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); - } - result - } + } => self.install_parked_pair(&ctx, parked, buffer, pm_generation, metrics)?, Build::Rebind { - mut pair, + pair, buffer, pm_generation, metrics, - } => { - let prepared = (|| { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - pair.prepare_bind( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - ) - })(); - let (mut prepared, error) = match prepared { - Ok(binding) => (Some(binding), None), - Err(error) => (None, Some(error)), - }; - let (result, grow_ready) = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let result = (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { - return Err(ECANCELED); - } - if firmware.queues.render.retired.len() - == firmware.queues.render.retired.capacity() - { - return Err(ENOSPC); - } - firmware.queues.teardown.displace( - pair.context(), - true, - super::super::now_ns(), - )?; - pair.bind_owner( - prepared.as_mut().ok_or(EIO)?, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - if let Some((id, _)) = buffer { - firmware.queues.buffers.reserve_owner(id)?; - firmware.queues.buffers.release(id)?; - } - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - if let Some(buffer) = buffer { - entry.reservation.buffer = buffer; - } - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.closed = false; - Ok(()) - })(); - // The original installed pair remains owned on every error. - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - entry.pair = Some(pair); - entry.borrowed_dependencies = None; - (result, firmware.render_grow_ready()) - }; - drop(prepared); - if result.is_err() { - if let Some((id, _)) = buffer { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .buffers - .release(id)?; - } + } => self.rebind_pair(&ctx, pair, buffer, pm_generation, metrics)?, + } + self.finish_render_build(&lease, slot, fresh_graph) + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock construction of a fresh pair, then its device-locked install. + fn install_fresh_pair( + &self, + ctx: &BuildContext<'_>, + reservation: Reservation, + pm_generation: u64, + metrics: MetricsLease, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let built = (|| -> Result> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + Pair::new( + &alloc, + &resources.objects, + reservation.ids, + slot, + self.owner, + self.context.clone(), + free_list, + metrics, + reservation.buffer, + pm_generation, + first, + resources.config.clusters, + resources.config.descriptor_flags, + ) + })(); + let (mut built, error) = match built { + Ok(pair) => (Some(pair), None), + Err(error) => (None, Some(error)), + }; + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + if firmware.queues.render.reserved(reservation)?.closed { + return Err(ECANCELED); + } + built + .as_mut() + .ok_or(EIO)? + .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; + firmware.queues.install_render(reservation, &mut built) + })(); + // Fresh backing disappears before its temporary PB reference returns. + drop(built); + if result.is_err() { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .cancel_render_reservation(reservation)?; + } + result + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock preparation of a parked pair for its new owner, then its device-locked + /// commit; the pair returns to its parked state on failure. + fn install_parked_pair( + &self, + ctx: &BuildContext<'_>, + mut parked: crate::g17::queue::render::parked::ParkedPair, + buffer: (u8, u64), + pm_generation: u64, + metrics: MetricsLease, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let built = (|| -> Result<_> { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + let prepared = parked.prepare( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + )?; + let storage = KBox::::new_uninit(GFP_KERNEL)?; + Ok((prepared, storage)) + })(); + let (mut prepared, mut storage, error) = match built { + Ok((prepared, storage)) => (Some(prepared), Some(storage), None), + Err(error) => (None, None, Some(error)), + }; + let result = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); } - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); } - result + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) + || entry.pending_owner_closed + || entry.pair.is_some() + || entry.parked.is_some() + { + return Err(ECANCELED); + } + let storage = storage.take().ok_or(EIO)?; + firmware.queues.buffers.can_release(buffer.0)?; + prepared.as_mut().ok_or(EIO)?.finish_install( + &parked, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + let pair = parked.commit(&mut prepared, storage)?; + firmware.queues.buffers.reserve_owner(buffer.0)?; + firmware.queues.buffers.release(buffer.0)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + entry.reservation.buffer = buffer; + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.pair = Some(pair); + entry.closed = false; + entry.published = false; + Ok(()) + })() + }; + // Rejected candidate mappings disappear before the PB identity + // returns, while the original installed kick owner stays live. + drop(prepared); + drop(storage); + let grow_ready = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if result.is_err() { + firmware.queues.render.entry_mut(slot)?.parked = Some(parked); + firmware.queues.buffers.release(buffer.0)?; + } else { + drop(state); + drop(parked); + state = self.shared.state.lock(); } + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + firmware.render_grow_ready() }; - result?; + self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } + result + } + + /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler + /// run_job stack path, which is close to its budget. + #[inline(always)] + /// Off-lock preparation of an installed pair's rebinding, then its device-locked + /// owner change; the original pair stays installed on failure. + fn rebind_pair( + &self, + ctx: &BuildContext<'_>, + mut pair: KBox, + buffer: Option<(u8, u64)>, + pm_generation: Option, + metrics: Option, + ) -> Result { + let BuildContext { + lease, + alloc, + resources, + slot, + first, + } = *ctx; + let prepared = (|| { + let free_list = + self.context + .render_pool(&alloc, &resources.ids, &resources.global)?; + pair.prepare_bind( + &alloc, + &resources.objects, + self.owner, + self.context.clone(), + free_list, + metrics, + buffer, + pm_generation, + first, + ) + })(); + let (mut prepared, error) = match prepared { + Ok(binding) => (Some(binding), None), + Err(error) => (None, Some(error)), + }; + let (result, grow_ready) = { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + let result = (|| { + if !lease.is_current() { + return Err(EAGAIN); + } + if let Some(error) = error { + return Err(error); + } + if !self.context.is_current() || self.context.status().get() != 0 { + return Err(EFAULT); + } + let entry = firmware.queues.render.entry(slot)?; + if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { + return Err(ECANCELED); + } + if firmware.queues.render.retired.len() + == firmware.queues.render.retired.capacity() + { + return Err(ENOSPC); + } + firmware.queues.teardown.displace( + pair.context(), + true, + super::super::now_ns(), + )?; + pair.bind_owner( + prepared.as_mut().ok_or(EIO)?, + &alloc, + firmware.init.pb_descriptor_table()?, + )?; + if let Some((id, _)) = buffer { + firmware.queues.buffers.reserve_owner(id)?; + firmware.queues.buffers.release(id)?; + } + let entry = firmware.queues.render.entry_mut(slot)?; + entry.reservation.owner = self.owner; + if let Some(buffer) = buffer { + entry.reservation.buffer = buffer; + } + entry.context = Some(self.context.clone()); + entry.pool = Some(pair.pool().clone()); + entry.closed = false; + Ok(()) + })(); + // The original installed pair remains owned on every error. + let entry = firmware.queues.render.entry_mut(slot)?; + entry.preparing = None; + entry.pending_owner = None; + entry.pending_owner_closed = false; + entry.pair = Some(pair); + entry.borrowed_dependencies = None; + (result, firmware.render_grow_ready()) + }; + drop(prepared); + if result.is_err() { + if let Some((id, _)) = buffer { + let mut state = self.shared.state.lock(); + (*state) + .as_deref_mut() + .ok_or(ENODEV)? + .queues + .buffers + .release(id)?; + } + } + self.shared.changed.notify_all(); + if grow_ready { + self.shared.queue_grow(); + } + result + } + + /// Under the device mutex: the graph is complete and retained; rebase a fresh graph's + /// cursors and warm its pool. + fn finish_render_build(&self, lease: &Lease, slot: u8, fresh_graph: bool) -> Result { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; if !lease.is_current() { From 3d4ab72b230aedeaa56532bee896f3480652cd64 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:42:34 -0400 Subject: [PATCH 108/631] drm/asahi: split the G17 render publication into preflight, barriers and announce Pair::publish() checked the pair, assembled the kick barriers, committed the descriptors and announced the pair to the firmware in one 350-line function with two near-identical failure blocks. Move the entry checks into publication_preflight(), the barrier assembly into render_barriers(), the announcement with its failure path into announce_publication(), and the shared tail of both failure paths (terminal status, deferred failure, failed flag) into record_publication_failure(). The checks, the barrier rules and the order of every effect on failure are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e0fa28b70ca92d7942809827a7cff7e230d68826) --- drivers/gpu/drm/asahi/g17/queue/render.rs | 193 ++++++++++++++-------- 1 file changed, 128 insertions(+), 65 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 60b30cde980944..d82ee84c1283ff 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -401,6 +401,47 @@ impl Pair { } } +/// Kick barriers of one render. The fragment kick always waits for its own tiling kick; +/// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the +/// tiling kick keeps only its implicit parent, otherwise the prefix orders the tiling kick. +/// A compute dependency always orders the tiling kick. +struct RenderBarriers { + tiling: [KickDependency; 3], + tiling_count: usize, + fragment: [KickDependency; 3], + fragment_count: usize, +} + +fn render_barriers( + fragment_stage: bool, + dependencies: &RenderDependencies, + own_tiling: KickDependency, +) -> RenderBarriers { + let mut tiling_barriers = [KickDependency::ZERO; 3]; + let mut tiling_count = 0; + let mut fragment_barriers = [own_tiling; 3]; + let mut fragment_count = 1; + for dependency in dependencies.render() { + if fragment_stage { + fragment_barriers[fragment_count] = *dependency; + fragment_count += 1; + } else { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + if let Some(compute) = dependencies.compute { + tiling_barriers[tiling_count] = compute; + tiling_count += 1; + } + RenderBarriers { + tiling: tiling_barriers, + tiling_count, + fragment: fragment_barriers, + fragment_count, + } +} + impl Pair { /// All descriptor bytes and retained allocations are ready before entry. /// The first inner producer establishes firmware ownership, including on @@ -422,37 +463,9 @@ impl Pair { }, }; use core::sync::atomic::{fence, Ordering}; - if self.pending_aliases.is_some() || self.kick_aliases.is_some() { - return Err(EBUSY); - } - let (epoch, recovery_state) = host.epoch()?; - if recovery_state != 0 - || self.quarantined - || !self.active.room() - || packet.completion.render_slot() != Some(self.slot) - { - return Err(EBUSY); - } - let prepared = self.prepared.as_ref().ok_or(EINVAL)?; - if !Arc::ptr_eq(&prepared.packet, &packet) - || prepared.ordinal != self.ordinal - || !self.memory.owns(self.ordinal) - || !packet.context.is_current() - { - return Err(EIO); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let policy = packet.context.policy(); - // A registration of another class can only follow a rebind, which - // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { - return Err(EIO); - } + let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; let ordinal = self.ordinal; let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let mcache = prepared.mcache; let qids = self.qids(); let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); let parents = self.kicks.each_ref().map(kick::Queue::parent); @@ -534,32 +547,12 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - // The fragment kick always waits for its own tiling kick. With the - // fragment-barriers flag the resolved prefix orders the fragment kick - // too, and the tiling kick keeps only its implicit parent, so vertex - // work overlaps the predecessors' fragment work. let Validated::Render { pass, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let mut tiling_barriers = [KickDependency::ZERO; 3]; - let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; - let mut fragment_count = 1; - for dependency in dependencies.render() { - if pass.fragment_barriers { - fragment_barriers[fragment_count] = *dependency; - fragment_count += 1; - } else { - tiling_barriers[tiling_count] = *dependency; - tiling_count += 1; - } - } - if let Some(compute) = dependencies.compute { - tiling_barriers[tiling_count] = compute; - tiling_count += 1; - } - let tiling_barriers = &tiling_barriers[..tiling_count]; + let barriers = render_barriers(pass.fragment_barriers, dependencies, own_tiling); + let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; let args = |stage: usize, barriers, arrays, mcache| KickArgs { qid: qids[stage], timestamp: kicks[stage], @@ -589,7 +582,7 @@ impl Pair { ); let fragment_args = args( 1, - &fragment_barriers[..fragment_count], + &barriers.fragment[..barriers.fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), )?, @@ -727,11 +720,7 @@ impl Pair { } self.active.back_mut().ok_or(EIO)?.qos_completed = true; self.quarantined = true; - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - defer(Deferred::FailedOwned(packet.completion.clone(), error))?; - self.active.back_mut().ok_or(EIO)?.failed = true; + self.record_publication_failure(&packet, error, defer)?; return Err(error); } }; @@ -749,11 +738,89 @@ impl Pair { previous.publication = Some(ordinal); } packet.context.mark_published(); + self.announce_publication( + host, + (&fragment_slot, &tiling_slot, outer_next), + lease.free_list_generation, + policy.priority(), + &packet, + defer, + ) + } + + /// Everything that must hold before a publication changes any state: no alias work + /// pending, a healthy device epoch, room in the active ring, this slot's packet, the + /// prepared descriptors for this ordinal, a clean completion status and a registration + /// class the pair can still take. Returns the epoch, the prepared memory-cache setting + /// and the owner's policy. + fn publication_preflight( + &self, + packet: &Arc, + host: &mut impl Host, + ) -> Result<(u64, Option, crate::g17::fw::queue::Policy)> { + if self.pending_aliases.is_some() || self.kick_aliases.is_some() { + return Err(EBUSY); + } + let (epoch, recovery_state) = host.epoch()?; + if recovery_state != 0 + || self.quarantined + || !self.active.room() + || packet.completion.render_slot() != Some(self.slot) + { + return Err(EBUSY); + } + let prepared = self.prepared.as_ref().ok_or(EINVAL)?; + if !Arc::ptr_eq(&prepared.packet, packet) + || prepared.ordinal != self.ordinal + || !self.memory.owns(self.ordinal) + || !packet.context.is_current() + { + return Err(EIO); + } + if packet.completion.status().get() != 0 { + return Err(EIO); + } + let policy = packet.context.policy(); + // A registration of another class can only follow a rebind, which + // requires the pair to have drained. + if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { + return Err(EIO); + } + Ok((epoch, prepared.mcache, policy)) + } + + /// Records a failed publication on the retained active entry: the terminal status + /// (once), the deferred failure and the entry's failed flag. + fn record_publication_failure( + &mut self, + packet: &Arc, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + if !packet.completion.spared() && self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), error)); + } + defer(Deferred::FailedOwned(packet.completion.clone(), error))?; + self.active.back_mut().ok_or(EIO)?.failed = true; + Ok(()) + } + + /// Activates the pool lease and announces the retained pair to the firmware; a failure + /// here quarantines the pair with the packet still retained for recovery. + fn announce_publication( + &mut self, + host: &mut impl Host, + (fragment_slot, tiling_slot, outer_next): (&WorkSlot, &WorkSlot, [u32; 2]), + free_list_generation: u64, + priority: u8, + packet: &Arc, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { let publish = (|| { - host.activate_render_pool(self.free_list.id(), lease.free_list_generation)?; - host.publish_retained_pair(policy.priority(), &fragment_slot, &tiling_slot, outer_next)?; + host.activate_render_pool(self.free_list.id(), free_list_generation)?; + host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; self.active.back_mut().ok_or(EIO)?.outer_published = true; - host.notify_render(policy.priority())?; + host.notify_render(priority)?; host.note_submission() })(); if let Err(error) = publish { @@ -764,11 +831,7 @@ impl Pair { .packet .completion .quarantine_render(); - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - defer(Deferred::FailedOwned(packet.completion.clone(), error))?; - self.active.back_mut().ok_or(EIO)?.failed = true; + self.record_publication_failure(packet, error, defer)?; return Err(error); } Ok(()) From bc52c2409b4f69e6bb92e1618f1f9d2eb80b9622 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 21:42:34 -0400 Subject: [PATCH 109/631] drm/asahi: split the G17 compute publication preflight and queue configuration Queue::publish() checked the queue, built the descriptor arguments and the queue configuration record, and committed and announced the kick in one 330-line function. Move the entry checks into publication_preflight(), which hands back the epoch and the bound owner's per-queue addresses as values, and the configuration record into queue_config(). The commit and announcement stay one closure: its visible/accounted state decides the rollback for both phases. Checks, record contents and effect order are unchanged. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 74bc2f09ab74832be661557cacf374ee5fff11d7) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 144 +++++++++++++-------- 1 file changed, 92 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 2b962e26326063..5347779d629d17 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -392,6 +392,13 @@ struct Previous { context: Arc, publication: Option<(u64, u64)>, } +/// Per-queue addresses of the bound owner, read under the publication preflight. +struct BindingAddresses { + preempt: u64, + operand: u64, + usage: u64, +} + #[derive(Copy, Clone, PartialEq, Eq)] struct ConfigIdentity { scheduler: u64, @@ -1495,31 +1502,7 @@ impl Queue { compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, kick::{KickArgs, KickQos}, }; - let (epoch, state) = host.epoch()?; - if state != 0 { - return Err(EBUSY); - } - let replay_front = self - .replays - .front() - .is_some_and(|front| Arc::ptr_eq(front, &packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } - if !packet.context.is_current() - || !self - .binding - .as_ref() - .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) - { - return Err(EFAULT); - } - let binding = self.binding.as_ref().ok_or(EFAULT)?; - if packet.completion.status().get() != 0 { - return Err(EIO); - } + let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { scratch, usc_base, @@ -1569,9 +1552,9 @@ impl Queue { qid, kick_count: self.ordinal, kick: kick_timestamp, - preempt_va: binding.preempt.iova(), - operand_state_va: binding.operand.iova(), - usage_va: binding.usage.iova(), + preempt_va: addresses.preempt, + operand_state_va: addresses.operand, + usage_va: addresses.usage, usage_fw_va: self.graph.usage.gpu_va(), free_list_slot: u32::from(self.pool.id()), free_list_control_va: self.pool.control_va(), @@ -1591,30 +1574,7 @@ impl Queue { self.free_list_generation = u64::from(packet.context.scheduler_generation()); let policy = packet.context.policy(); let ring = 1u8 << policy.priority(); - let config = QueueConfig::new(&QueueConfigArgs { - target: QueueConfigTarget::Compute, - kick_ring_va: self.kick.low_va(), - kick_ring_fw_va: self.kick.firmware_va(), - qid, - install: first, - completion_seed: None, - context_id: args.context_id, - free_list: FreeListBinding { - va: self.pool.control_va(), - generation: u64::from(packet.context.scheduler_generation()), - slot: u32::from(self.pool.id()), - }, - scheduler_va: identity.scheduler, - policy, - qos_slot: identity.qos, - qos_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), - owner_pid: packet.context.owner_pid(), - context_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), - })?; + let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; let predecessor = KickPredecessor::new(qid, kick_parent)?; @@ -1834,6 +1794,86 @@ impl Queue { } Ok(()) } + + /// Everything that must hold before a publication changes any state: a healthy device + /// epoch, room in the active ring (or this packet at the replay front), the bound + /// owner's current context and a clean completion status. Returns the epoch and the + /// bound owner's per-queue addresses. + fn publication_preflight( + &self, + packet: &Arc, + host: &mut impl Host, + ) -> Result<(u64, BindingAddresses)> { + let (epoch, state) = host.epoch()?; + if state != 0 { + return Err(EBUSY); + } + let replay_front = self + .replays + .front() + .is_some_and(|front| Arc::ptr_eq(front, packet)); + if !self.room() + && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) + { + return Err(EBUSY); + } + if !packet.context.is_current() + || !self + .binding + .as_ref() + .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) + { + return Err(EFAULT); + } + let binding = self.binding.as_ref().ok_or(EFAULT)?; + if packet.completion.status().get() != 0 { + return Err(EIO); + } + Ok(( + epoch, + BindingAddresses { + preempt: binding.preempt.iova(), + operand: binding.operand.iova(), + usage: binding.usage.iova(), + }, + )) + } + + /// The queue configuration record for this publication: a fresh install on the first + /// one, and QoS/context updates only when the identity changed since the last publication. + fn queue_config( + &self, + identity: &ConfigIdentity, + first: bool, + context_id: u16, + policy: crate::g17::fw::queue::Policy, + packet: &Arc, + ) -> Result { + QueueConfig::new(&QueueConfigArgs { + target: QueueConfigTarget::Compute, + kick_ring_va: self.kick.low_va(), + kick_ring_fw_va: self.kick.firmware_va(), + qid: self.qid(), + install: first, + completion_seed: None, + context_id, + free_list: FreeListBinding { + va: self.pool.control_va(), + generation: u64::from(packet.context.scheduler_generation()), + slot: u32::from(self.pool.id()), + }, + scheduler_va: identity.scheduler, + policy, + qos_slot: identity.qos, + qos_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), + owner_pid: packet.context.owner_pid(), + context_update: self + .published_config + .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), + }) + } } impl Graph { From 9c880db9f214e406d20528ae053784c24b2d4f60 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 23:13:38 -0400 Subject: [PATCH 110/631] drm/asahi: size the compute descriptor ring to the kicks that can be in flight Each retained compute queue carried a 4 MiB descriptor ring: 256 slots of 16 KiB, one per kick-ring slot, although at most IN_FLIGHT (16) kicks are outstanding and every kick names its descriptor by address. Index the ring by kick slot modulo 32, so a descriptor is rewritten only once the kick 32 publications later is issued, long after its own completion. The ring shrinks to 512 KiB per queue; with the fourteen queues of a desktop and a game that is 49 MiB less pinned memory. Retirement records match on descriptor address and kick timestamp together, so a reused address cannot alias an older kick. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5ad7b191f7e698e3ef140163c6ecf15d3e57ff85) --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 5347779d629d17..10b66c9ae811ba 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -19,7 +19,9 @@ use retirement::{Cursors, RECORD_SLOTS}; use crate::g17::now_ns; -const DESCRIPTORS: usize = 256; +/// Descriptor slots per queue. A kick's descriptor is rewritten when the kick 32 slots later +/// is published; with IN_FLIGHT kicks outstanding at most, that kick has completed by then. +const DESCRIPTORS: usize = 32; const GRAPH_SIZE: usize = 0x8000; const CONTEXT_SIZE: usize = 0x20000; const RECORD_STRIDE: usize = 0x1000; @@ -231,7 +233,7 @@ impl Graph { /// EAGAIN while the queue is vacant: the owner must re-arm it first. fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { let ring = self.ring.as_ref().ok_or(EAGAIN)?; - let offset = u64::from(slot) * COMPUTE_DESCRIPTOR_SIZE as u64; + let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) } fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { @@ -1620,7 +1622,7 @@ impl Queue { .ok_or(EAGAIN)? .object .initialize::( - kick_timestamp.slot() * COMPUTE_DESCRIPTOR_SIZE, + kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, |descriptor| descriptor.write(&args), )?; if first { From f32ec022a6993335a087bf866c1b95cc9df03047 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:14:45 +1100 Subject: [PATCH 111/631] drm/asahi: initialize driver data before claiming destructor ownership Both GPU probe paths allocate a DRM device before creating the GPU manager. A successful no-op PinInit incorrectly claimed that AsahiData was initialized, so ordinary firmware, identity or allocation failures could run its destructor over uninitialized GPU and platform references. Add explicit two-phase allocation to the existing Rust DRM device. Keep the pinned data slot in UnsafeCell> and publish destructor ownership only after the real initializer succeeds. Use a static allocation vtable while release state itself is uninitialized. Preserve the ordinary initialized constructor and inline workqueue offsets. Convert both Asahi probes to the explicit API, avoiding references to uninitialized data. Partial initializer failures unwind their own fields; post-initialization failures retain normal data destruction. Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 983575d7b492f6824401cd21fe02278812211e27) --- drivers/gpu/drm/asahi/driver.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 45561ac17e2f5a..70de8ecc7f21c5 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -569,12 +569,9 @@ impl AsahiDriver { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let res = regs::Resources::new(pdev)?; - // SAFETY: As in the legacy probe, the data is inaccessible until the fully initialized - // manager is installed below and the DRM device is registered. - // If construction fails, drm::device::Device::release leaves the data - // untouched rather than dropping an uninitialized AsahiData. - let uninit = unsafe { pin_init::pin_init_from_closure::(|_slot| Ok(())) }; - let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + // SAFETY: GPU construction cannot access driver data before init_data(). + // Failed construction releases the allocation without dropping uninitialized data. + let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiData { @@ -582,9 +579,8 @@ impl AsahiDriver { pdev: pdev.into(), resources: res, }); - let ptr = &raw const **drm; - // SAFETY: The allocation is pinned and its data slot has not been initialized yet. - if let Err(error) = unsafe { data.__pinned_init(ptr as *mut AsahiData) } { + // SAFETY: This is the sole initializer; no driver-data users have been published. + if let Err(error) = unsafe { drm.init_data(data) } { gpu.shutdown(); return Err(error); } From 25eb7dddeb27b2aef162c0e63e737e5ef1abbb0e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:36:27 +1100 Subject: [PATCH 112/631] Merge pull request #65 from joshuaswarren/ane-dt-polltx-aurora soc: apple: mailbox: Make the send-empty IRQ optional (poll TX) Original contributor commits are preserved. Conflict resolutions retain the audited shared driver APIs and their integrated review fixes. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 3c4ee0dfd0c05b..e6b31a9afb19ef 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -44,6 +44,9 @@ properties: - enum: - apple,t6021-ane-mailbox - apple,t8112-ane-mailbox + - const: apple,asc-mailbox-v4 + + - description: ASC mailboxes of the T8140 GPU coprocessors. Unlike the other T8140 ASC mailboxes, they do not need the application processor to enable their receive FIFO. From 079717b98a5ade421ec81853b336ff3c60325750 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:15:47 +1000 Subject: [PATCH 113/631] drm: apple: balance piodma child ownership across bind attempts Adopt the registered piodma child rather than creating a duplicate during component rebind. Keep its lookup reference until unbind and release it on every later unsuccessful bind; failed component binds have no unbind call. A missing IOMMU domain defers bind. One cleanup helper distinguishes an adopted registration from a child created by this bind and clears stale child/domain pointers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 71 +++++++++++++++++++++------- 2 files changed, 54 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ee9b2477d0887c..9c45240d0b0cf5 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -169,6 +169,7 @@ struct apple_dcp { struct dcp_fabric_pipeline fabric; struct device *dev; struct platform_device *piodma; + bool piodma_created; struct iommu_domain *iommu_dom; /* which of the nine cumulative A031 notify-client states to send next */ unsigned int a031_step; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6d51199013e336..470ab98813ef3e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1823,6 +1823,19 @@ static void dcp_work_update_backlight(struct work_struct *work) dcp_backlight_update(dcp); } +static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) +{ + if (dcp->piodma) { + if (dcp->piodma_created) + of_platform_device_destroy(&dcp->piodma->dev, NULL); + else + put_device(&dcp->piodma->dev); + } + dcp->piodma = NULL; + dcp->iommu_dom = NULL; + dcp->piodma_created = false; +} + static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) { int ret; @@ -1832,6 +1845,13 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) return dev_err_probe(dcp->dev, -ENODEV, "Failed to get piodma child DT node\n"); + dcp->piodma = of_find_device_by_node(node); + dcp->piodma_created = !dcp->piodma; + if (dcp->piodma_created) + dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to create piodma pdev for %pOF\n", node); dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); if (!dcp->piodma) return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); @@ -1858,7 +1878,7 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) return 0; err_destroy_pdev: - of_platform_device_destroy(&dcp->piodma->dev, NULL); + dcp_release_piodma_iommu_dev(dcp); return ret; } @@ -2381,24 +2401,33 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat } ret = dcp_create_piodma_iommu_dev(dcp); - if (ret || !dcp->iommu_dom) + if (ret) return dev_err_probe(dev, ret, - "Failed to created PIODMA iommu child device"); + "Failed to create PIODMA iommu child device\n"); + if (!dcp->iommu_dom) { + ret = dev_err_probe(dev, -EPROBE_DEFER, + "PIODMA iommu domain is unavailable\n"); + goto err_piodma; + } ret = dcp_get_disp_regs(dcp); if (ret) { dev_err(dev, "failed to find display registers\n"); - return ret; + goto err_piodma; } dcp->clk = devm_clk_get(dev, NULL); - if (IS_ERR(dcp->clk)) - return dev_err_probe(dev, PTR_ERR(dcp->clk), - "Unable to find clock\n"); + if (IS_ERR(dcp->clk)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->clk), + "Unable to find clock\n"); + goto err_piodma; + } dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); - if (IS_ERR(dcp->clk_194)) - return dev_err_probe(dev, PTR_ERR(dcp->clk_194), - "Unable to find clock 0x194\n"); + if (IS_ERR(dcp->clk_194)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->clk_194), + "Unable to find clock 0x194\n"); + goto err_piodma; + } bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry @@ -2425,9 +2454,11 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat } dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); - if (IS_ERR(dcp->rtk)) - return dev_err_probe(dev, PTR_ERR(dcp->rtk), - "Failed to initialize RTKit\n"); + if (IS_ERR(dcp->rtk)) { + ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to initialize RTKit\n"); + goto err_piodma; + } if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); @@ -2460,6 +2491,14 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp_ext_give_up(dcp, ret, "bind"); return 0; } + if (ret) { + ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); + goto err_piodma; + } + return 0; + +err_piodma: + dcp_release_piodma_iommu_dev(dcp); return ret; } @@ -2545,11 +2584,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->shmem) iomfb_shutdown(dcp); - if (dcp->piodma) { - dcp->iommu_dom = NULL; - of_platform_device_destroy(&dcp->piodma->dev, NULL); - dcp->piodma = NULL; - } + dcp_release_piodma_iommu_dev(dcp); devm_clk_put(dev, dcp->clk); dcp->clk = NULL; From 3504aaaca6f4943262bdfe3619f1dbffe880a476 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:16:49 +1000 Subject: [PATCH 114/631] drm: apple: look up display register windows by reg-names The number of display windows differs between DCP instances. Count the disp-* names and fetch each resource by name so unrelated registers do not shift the bandwidth window indexes. Retain the positional lookup for device trees without reg-names. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 68 +++++++++++++------------------------ 1 file changed, 24 insertions(+), 44 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 470ab98813ef3e..748103c4c966f1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1979,66 +1979,46 @@ static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) return ret; } -/* - * Count the "disp-*" entries in reg-names. - * - * The positional form below assumes the reg layout is [coproc, disp-0 ...], - * i.e. exactly one non-disp register, and derives the count as - * num_resources - 1. DCPs with H17-generation firmware list a trailing - * "iop-vbar" register after the display apertures, which would inflate that - * count, bind the last disp_registers[] entry to iop-vbar and make the - * "apple,bw-scratch" disp_reg index check reject a valid device tree. - * - * Returns a negative errno when reg-names is absent, so callers fall back. - */ -static int dcp_count_disp_regs(struct device *dev) -{ - int n = of_property_count_strings(dev->of_node, "reg-names"); - const char *name; - int i, count = 0; - - if (n <= 0) - return -EINVAL; - - for (i = 0; i < n; ++i) { - if (of_property_read_string_index(dev->of_node, "reg-names", i, - &name)) - return -EINVAL; - if (!strncmp(name, "disp-", 5)) - count++; - } - - return count ? count : -EINVAL; -} - static int dcp_get_disp_regs(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); int count = pdev->num_resources - 1; + int count = 0; int i, ret; + char name[16]; + const char *reg_name; + struct resource *res; + + if (of_property_present(dcp->dev->of_node, "reg-names")) { + ret = of_property_count_strings(dcp->dev->of_node, "reg-names"); + if (ret < 0) + return ret; - /* Device trees without reg-names keep the historical positional form. */ - if (!by_name) + for (i = 0; i < ret; i++) { + if (of_property_read_string_index(dcp->dev->of_node, + "reg-names", i, ®_name)) + return -EINVAL; + if (!strncmp(reg_name, "disp-", 5)) + count++; + } + } else { count = pdev->num_resources - 1; + } if (count <= 0 || count > MAX_DISP_REGISTERS) return -EINVAL; for (i = 0; i < count; ++i) { - if (by_name) { - char name[8]; - + if (of_property_present(dcp->dev->of_node, "reg-names")) { snprintf(name, sizeof(name), "disp-%d", i); - dcp->disp_registers[i] = platform_get_resource_byname( - pdev, IORESOURCE_MEM, name); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + name); } else { - dcp->disp_registers[i] = platform_get_resource( - pdev, IORESOURCE_MEM, 1 + i); + res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); } - if (!dcp->disp_registers[i]) { - dev_err(dcp->dev, "missing display register %d\n", i); + if (!res) return -EINVAL; - } + dcp->disp_registers[i] = res; } /* load pmgr bandwidth scratch resource and offset */ From 89099ea4b704603d8668423ed342ae27094946b6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:22:27 +1000 Subject: [PATCH 115/631] drm: apple: map firmware-provided RTKit buffers write-combined The DCP writes its RTKit buffers without CPU cache maintenance. A write-back mapping can read stale crash-log contents after the firmware has filled the buffer. Use a write-combined mapping for firmware-provided physical pages. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 748103c4c966f1..76305f05778638 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -410,8 +410,8 @@ static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) return ret; } - // TODO: verify phy_addr, cache attribute - bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); + /* Firmware writes without CPU cache maintenance. */ + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WC); if (!bfr->buffer) return -ENOMEM; From 6e584f4e2ed035970d4393266311c9191524df2e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:28:52 +1000 Subject: [PATCH 116/631] soc: apple: rtkit: recognise an initial crash log in adopted sessions Map the first crash-buffer request and validate a complete public CLHE header before decoding it. A fresh session's zeroed buffer remains an ordinary allocation request. An inherited session has already negotiated its buffer: mapping failure or an unreadable header must still notify the client of the crash, with an empty log when necessary. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 43528fef6d199f..6e1fd292468a3c 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -552,23 +552,6 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } - if (rtk->crashlog_inherited && !rtk->crashlog_buffer.size) { - /* - * The previous owner negotiated the crashlog buffer, so this - * session never learned its geometry. A CRASH message on an - * inherited session is therefore a crash notification, not a - * buffer request, and must not be answered as one. - */ - dev_err(rtk->dev, - "RTKit: adopted co-processor has crashed (crashlog at %#llx, %#llx bytes, not mapped)\n", - (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), - (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12); - rtk->crashed = true; - if (rtk->ops->crashed) - rtk->ops->crashed(rtk->cookie, NULL, 0); - return; - } - if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) From bb608cfc8084d3838a32e72941941b74a908b5ff Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:36:09 +1000 Subject: [PATCH 117/631] drm: apple: afk: derive the ring header stride from INITRB AFK rings describe their payload size in the first word. Derive the three header blocks from the INITRB total instead of assuming 64 bytes per block. Keep the offset and size fields wide enough for the full wire range, and align queue entries to the derived stride. Synthetic KUnit cases cover the older and larger ring headers and malformed layouts (AFK-1). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 8 +++++ drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/afk.c | 51 +++++++++++++++----------------- drivers/gpu/drm/apple/afk.h | 38 +++++++++++++++++------- drivers/gpu/drm/apple/afk_test.c | 44 +++++++++++++++++++++++++++ drivers/gpu/drm/apple/trace.h | 4 +-- 6 files changed, 107 insertions(+), 39 deletions(-) create mode 100644 drivers/gpu/drm/apple/afk_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ea7f24a22bc8a9..56b80b6c28d194 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -59,3 +59,11 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST status updates before hotplug notifications and same-state retrains. Check which 14.x board record an internal display gets on which machine, and which DCP firmware image each record admits. + tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Build tests for the Apple display driver's AFK ring parser. + The tests cover the header strides used by older firmware and + larger rings, as well as malformed ring size combinations. + This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 8f33959f281c57..52d3eed3453503 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -31,3 +31,4 @@ appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += connector-edid-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += hotplug-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += external-hold-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += v14-board-test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index dbc3753411480e..b58644c3e58042 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -205,7 +205,8 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, hdrsz, end, block; + u32 bufsz, end, stride; + u8 *hdr; if (tag != ep->bfr_tag) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", @@ -219,10 +220,10 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - if (!ep->bfr || size < sizeof(__le32) || base >= ep->bfr_size) { + if (!ep->bfr || base >= ep->bfr_size || size < sizeof(u32)) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: requested base 0x%x >= max size 0x%lx\n", - ep->endpoint, base, ep->bfr_size); + "AFK[ep:%02x]: invalid ring base 0x%x or size 0x%x\n", + ep->endpoint, base, size); return; } @@ -234,17 +235,13 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - bfr->hdr = ep->bfr + base; + hdr = (u8 *)ep->bfr + base; dma_rmb(); - bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)bfr->hdr)); - hdrsz = size - bufsz; - block = hdrsz / AFK_RB_BLOCKS; - if (bufsz >= size || hdrsz % AFK_RB_BLOCKS || - block < AFK_RB_BLOCK_MIN || !is_power_of_2(block) || - !IS_ALIGNED(bufsz, block) || bufsz < 2 * block) { + bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)hdr)); + if (afk_ring_stride(size, bufsz, &stride)) { dev_err(ep->dcp->dev, - "AFK[ep:%02x]: ring bufsz %#x with size %#x gives no valid header (base %#x, msg %#llx)\n", - ep->endpoint, bufsz, size, base, message); + "AFK[ep:%02x]: invalid ring payload size 0x%x of 0x%x\n", + ep->endpoint, bufsz, size); return; } if (bufsz < BIT(BLOCK_SHIFT)) { @@ -259,8 +256,11 @@ static void afk_init_rxtx(struct apple_dcp_afkep *ep, u64 message, return; } - bfr->buf = (u8 *)bfr->hdr + hdrsz; + bfr->rptr = (__le32 *)(hdr + stride); + bfr->wptr = (__le32 *)(hdr + 2 * stride); + bfr->buf = hdr + 3 * stride; bfr->bufsz = bufsz; + bfr->stride = stride; bfr->ready = true; if (ep->rxbfr.ready && ep->txbfr.ready) @@ -761,8 +761,8 @@ static bool afk_recv(struct apple_dcp_afkep *ep) return false; } - rptr = le32_to_cpu(*afk_rb_rptr(&ep->rxbfr)); - wptr = le32_to_cpu(*afk_rb_wptr(&ep->rxbfr)); + rptr = le32_to_cpu(*ep->rxbfr.rptr); + wptr = le32_to_cpu(*ep->rxbfr.wptr); trace_afk_recv_rwptr_pre(ep, rptr, wptr); if (rptr == wptr) @@ -824,15 +824,12 @@ static bool afk_recv(struct apple_dcp_afkep *ep) */ memcpy(ep->rx_scratch, hdr->data, size); - rptr = ALIGN(rptr + sizeof(*hdr) + size, ep->rxbfr.block); - if (WARN_ON(rptr > ep->rxbfr.bufsz)) - rptr = 0; - if (rptr == ep->rxbfr.bufsz) - rptr = 0; + rptr = afk_ring_advance(rptr, sizeof(*hdr) + size, + ep->rxbfr.bufsz, ep->rxbfr.stride); dma_mb(); - *afk_rb_rptr(&ep->rxbfr) = cpu_to_le32(rptr); + *ep->rxbfr.rptr = cpu_to_le32(rptr); trace_afk_recv_rwptr_post(ep, rptr, wptr); afk_recv_handle(ep, channel, type, ep->rx_scratch, size); @@ -952,8 +949,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, } dma_rmb(); - rptr = le32_to_cpu(*afk_rb_rptr(&ep->txbfr)); - wptr = le32_to_cpu(*afk_rb_wptr(&ep->txbfr)); + rptr = le32_to_cpu(*ep->txbfr.rptr); + wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; total_size = sizeof(*hdr) + total_epic_size; @@ -1066,14 +1063,14 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; - wptr = ALIGN(wptr, ep->txbfr.block); - if (wptr == ep->txbfr.bufsz) - wptr = 0; + wptr = afk_ring_advance(wptr, 0, ep->txbfr.bufsz, + ep->txbfr.stride); trace_afk_send_rwptr_post(ep, rptr, wptr); /* Publish the completed entry before allowing the consumer to read it. */ dma_wmb(); ep->txbfr.hdr->wptr = cpu_to_le32(wptr); + *ep->txbfr.wptr = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); ret = 0; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 90860b994985fa..d9cee6bdb2a43f 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -9,8 +9,10 @@ #include #include +#include #include #include +#include #include #include "dcp.h" @@ -172,6 +174,29 @@ struct afk_ringbuffer_header { __le32 wptr; u32 _pad3[15]; }; +static inline int afk_ring_stride(u32 total, u32 bufsz, u32 *stride) +{ + u32 header, block; + + if (bufsz >= total) + return -EINVAL; + header = total - bufsz; + block = header / 3; + if (header % 3 || block < 0x40 || block % 0x40 || + bufsz < block || bufsz % block) + return -EINVAL; + + *stride = block; + return 0; +} + +static inline u32 afk_ring_advance(u32 ptr, u32 bytes, u32 bufsz, + u32 stride) +{ + u32 next = roundup(ptr + bytes, stride); + + return next == bufsz ? 0 : next; +} struct afk_qe { #define QE_MAGIC 0x20504f49 // ' POI' @@ -259,20 +284,13 @@ enum epic_subtype { struct afk_ringbuffer { bool ready; - void *hdr; - u32 block; /* control-block stride, 0x40 or 0x80 */ - u32 rptr; + __le32 *rptr; + __le32 *wptr; void *buf; size_t bufsz; + u32 stride; }; -static inline __le32 *afk_rb_field(struct afk_ringbuffer *bfr, unsigned int idx) -{ - return (__le32 *)((u8 *)bfr->hdr + idx * bfr->block); -} - -#define afk_rb_rptr(bfr) afk_rb_field(bfr, 1) -#define afk_rb_wptr(bfr) afk_rb_field(bfr, 2) struct apple_dcp_afkep { struct apple_dcp *dcp; diff --git a/drivers/gpu/drm/apple/afk_test.c b/drivers/gpu/drm/apple/afk_test.c new file mode 100644 index 00000000000000..e58a9526420458 --- /dev/null +++ b/drivers/gpu/drm/apple/afk_test.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "afk.h" + +static void afk_ring_stride_test(struct kunit *test) +{ + u32 stride; + + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, stride, 0x40U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1180, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, stride, 0x80U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1000, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10bd, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c1, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0xc0, 0, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c3, 0x1000, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1001, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x11c0, 0x1040, &stride), -EINVAL); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x1180, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xf80, 0x80, 0x1000, stride), 0U); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xe80, 0x80, 0x1000, stride), + 0xf00U); + KUNIT_EXPECT_EQ(test, afk_ring_stride(0x10c0, 0x1000, &stride), 0); + KUNIT_EXPECT_EQ(test, afk_ring_advance(0xfc0, 0x40, 0x1000, stride), 0U); +} + +static struct kunit_case afk_ring_cases[] = { + KUNIT_CASE(afk_ring_stride_test), + {} +}; + +static struct kunit_suite afk_ring_suite = { + .name = "apple-afk-ring", + .test_cases = afk_ring_cases, +}; + +kunit_test_suite(afk_ring_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index c0acac6ee690ab..d3c5106e5d66d2 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -109,11 +109,11 @@ TRACE_EVENT(dcp_send_msg, show_dcp_endpoint(__entry->endpoint), __entry->message)); TRACE_EVENT( - afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u16 size, u16 tag), + afk_getbuf, TP_PROTO(struct apple_dcp_afkep *ep, u32 size, u16 tag), TP_ARGS(ep, size, tag), TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) - __field(u8, endpoint) __field(u16, size) + __field(u8, endpoint) __field(u32, size) __field(u16, tag)), TP_fast_assign(__assign_str(devname); From 0b892f90ce8ed23ccc5553213afc8506efbac73c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:40:44 +1000 Subject: [PATCH 118/631] drm: apple: drop messages for endpoints without a handler Firmware may send on an endpoint that this DCP instance did not start. Log the endpoint with rate limiting and drop the message instead of calling AFK through a NULL pointer or WARNing. Refuse to start the DCP when RTKit is absent (RTK-2). Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 76305f05778638..fb547d5258cf42 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -296,6 +296,7 @@ void dcp_swap_watchdog_complete(struct apple_dcp *dcp) static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = cookie; + struct apple_dcp_afkep *ep; trace_dcp_recv_msg(dcp, endpoint, message); @@ -318,7 +319,29 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case IOMFB_ENDPOINT: return iomfb_recv_msg(dcp, message); case AV_ENDPOINT: - afk_receive_message(dcp->avep, message); + ep = dcp->avep; + break; + case SYSTEM_ENDPOINT: + ep = dcp->systemep; + break; + case DISP0_ENDPOINT: + ep = dcp->ibootep; + break; + case DPAVSERV_ENDPOINT: + ep = dcp->dcpavservep; + break; + case DPTX_ENDPOINT: + ep = dcp->dptxep; + break; + default: + ep = NULL; + break; + } + + if (!ep) { + dev_warn_ratelimited(dcp->dev, + "dropping message for unhandled endpoint %#x\n", + endpoint); return; case SYSTEM_ENDPOINT: afk_receive_message(dcp->systemep, message); @@ -335,6 +358,8 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) default: WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); } + + afk_receive_message(ep, message); } static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) @@ -1393,6 +1418,9 @@ int dcp_start(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (!dcp->rtk) + return -ENODEV; + init_completion(&dcp->start_done); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) return iomfb_v26_6_start(dcp); From 0a68d2c2ab9d7cbc94aeafafb8d137b7cbe5e8cb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 00:47:31 +1000 Subject: [PATCH 119/631] drm: apple: retain displaced framebuffers until confirmed completion Allocate and reference displaced scanout during plane preparation, where allocation failure can still abort the atomic commit. Transfer prepared entries before disabling or updating CRTCs, including inactive commits. Cleanup and state destruction release only unconsumed reservations. Retain transferred references until a successful successor swap completes or a successful power response confirms scanout stop on existing firmware variants. Rejected swap starts never arm retirement; rejected, missing or malformed power responses, timeouts and late replies cannot release live scanout. H17P stop semantics remain unqualified, so its power replies do not retire references. Initialize command output to an invalid status and validate legacy reply lengths before dispatching responses. Initialize retention ownership once at probe and reject rebind while a previous session's references remain. This preserves quarantine ownership instead of losing list links after a timed-out shutdown. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 26 +++- drivers/gpu/drm/apple/dcp-internal.h | 12 +- drivers/gpu/drm/apple/dcp.c | 82 +++++++++++++ drivers/gpu/drm/apple/dcp.h | 6 + drivers/gpu/drm/apple/iomfb.c | 11 ++ drivers/gpu/drm/apple/iomfb_template.c | 163 ++++--------------------- drivers/gpu/drm/apple/plane.c | 63 ++++++++-- drivers/gpu/drm/apple/plane.h | 3 + 8 files changed, 213 insertions(+), 153 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d53e5654393e6f..1b9e1977f1ca37 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -177,10 +177,11 @@ static void apple_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct drm_crtc_state *crtc_state; + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); if (crtc_state->active_changed && !crtc_state->active) { - struct apple_crtc *apple_crtc = to_apple_crtc(crtc); dcp_poweroff(apple_crtc->dcp); if (apple_crtc->fabric_held) { apple_crtc->fabric_held = false; @@ -317,8 +318,29 @@ static const struct drm_mode_config_funcs apple_mode_config_funcs = { .fb_create = drm_gem_fb_create, }; +static void apple_atomic_commit_tail(struct drm_atomic_state *state) +{ + struct drm_plane *plane; + struct drm_plane_state *old, *new; + int i; + + /* Retain displaced scanout even when the CRTC stays inactive. */ + for_each_oldnew_plane_in_state(state, plane, old, new, i) { + struct apple_plane_state *apple_state = to_apple_plane_state(new); + struct dcp_fb_reference *entry = apple_state->retirement; + + if (!entry) + continue; + + apple_state->retirement = NULL; + dcp_retain_framebuffer(to_apple_crtc(old->crtc)->dcp, entry); + } + + drm_atomic_helper_commit_tail_rpm(state); +} + static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { - .atomic_commit_tail = drm_atomic_helper_commit_tail_rpm, + .atomic_commit_tail = apple_atomic_commit_tail, }; static void appledrm_connector_cleanup(struct drm_connector *connector) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 9c45240d0b0cf5..fa4c7e097bf86c 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -100,6 +100,8 @@ struct dcp_channel { dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; + u32 in_len[DCP_MAX_CALL_DEPTH]; + u32 out_len[DCP_MAX_CALL_DEPTH]; u16 end[DCP_MAX_CALL_DEPTH]; /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ @@ -351,14 +353,12 @@ struct apple_dcp { /* List of referenced drm_framebuffers which can be unreferenced * on the next successfully completed swap. + /* List of referenced framebuffers pending a completed replacement swap. + * The commit and RTKit work queues share it under this lock. */ struct list_head swapped_out_fbs; - /* - * Protects swapped_out_fbs, which is appended to from the DRM atomic - * commit (dcp_flush -> .atomic_flush) and armed/drained from the RTKit - * workqueue that runs the DCP callbacks. - */ - spinlock_t swapped_out_lock; + /* Protects swapped_out_fbs. */ + struct mutex swapped_out_fbs_lock; struct dcp_brightness brightness; /* Workqueue for updating the initial brightness */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index fb547d5258cf42..0d6e6e41a72351 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1864,6 +1864,77 @@ static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) dcp->piodma_created = false; } +void dcp_retain_framebuffer(struct platform_device *pdev, + struct dcp_fb_reference *entry) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_add_tail(&entry->head, &dcp->swapped_out_fbs); + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { + if (entry->armed) + continue; + entry->swap_id = swap_id; + entry->armed = true; + } + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_unarm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry; + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry(entry, &dcp->swapped_out_fbs, head) + if (entry->armed && entry->swap_id == swap_id) + entry->armed = false; + mutex_unlock(&dcp->swapped_out_fbs_lock); +} + +void dcp_release_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id) +{ + struct dcp_fb_reference *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_for_each_entry_safe(entry, tmp, &dcp->swapped_out_fbs, head) { + if (!entry->armed || entry->swap_id != swap_id) + continue; + list_move_tail(&entry->head, &completed); + } + mutex_unlock(&dcp->swapped_out_fbs_lock); + + list_for_each_entry_safe(entry, tmp, &completed, head) { + list_del(&entry->head); + drm_framebuffer_put(entry->fb); + kfree(entry); + } +} + +void dcp_release_all_retained_framebuffers(struct apple_dcp *dcp) +{ + struct dcp_fb_reference *entry, *tmp; + LIST_HEAD(completed); + + mutex_lock(&dcp->swapped_out_fbs_lock); + list_splice_init(&dcp->swapped_out_fbs, &completed); + mutex_unlock(&dcp->swapped_out_fbs_lock); + + list_for_each_entry_safe(entry, tmp, &completed, head) { + list_del(&entry->head); + drm_framebuffer_put(entry->fb); + kfree(entry); + } +} + static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) { int ret; @@ -2312,6 +2383,14 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat u32 cpu_ctrl; int ret; + /* A timed-out prior session may still scan these mappings. */ + mutex_lock(&dcp->swapped_out_fbs_lock); + ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; + mutex_unlock(&dcp->swapped_out_fbs_lock); + if (ret) + return dev_err_probe(dev, ret, + "Previous scanout has not been stopped\n"); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); if (ret) return ret; @@ -2695,6 +2774,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (!dcp) return -ENOMEM; + INIT_LIST_HEAD(&dcp->swapped_out_fbs); + mutex_init(&dcp->swapped_out_fbs_lock); + dcp->fw_compat = fw_compat; dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext"); dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 9cbd5926a36ab8..cb45204398917c 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -54,6 +54,12 @@ unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev); int dcp_start(struct platform_device *pdev); int dcp_wait_ready(struct platform_device *pdev, u64 timeout); void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); +void dcp_retain_framebuffer(struct platform_device *pdev, + struct dcp_fb_reference *entry); +void dcp_arm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_unarm_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_release_retained_framebuffers(struct apple_dcp *dcp, u32 swap_id); +void dcp_release_all_retained_framebuffers(struct apple_dcp *dcp); bool dcp_is_initialized(struct platform_device *pdev); void apple_crtc_vblank(struct apple_crtc *apple); void dcp_drm_crtc_vblank(struct apple_crtc *crtc); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index fdda5a02b08e64..76552df9b31165 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -194,12 +194,17 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca if (in_len > 0) memcpy(out_data, data, in_len); + /* An unwritten status must not look like a successful response. */ + if (out_len) + memset(out_data + in_len, 0xff, out_len); trace_iomfb_push(dcp, call, context, offset, depth); ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; + ch->in_len[depth] = in_len; + ch->out_len[depth] = out_len; ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); dcp_send_message(dcp, IOMFB_ENDPOINT, @@ -477,6 +482,12 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, out = ch->output[ch->depth]; ch->output[ch->depth] = NULL; } else { + if (header->in_len != ch->in_len[ch->depth] || + header->out_len != ch->out_len[ch->depth]) { + dev_err(dcp->dev, "invalid command response lengths\n"); + dcp->crashed = true; + return; + } out = data + sizeof(*header) + header->in_len; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 88f310e95ed930..e134915843af27 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -220,96 +220,6 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * Bind every framebuffer displaced by the commit that produced this swap to - * that swap's id. - * - * The id only exists once the firmware answers swap_start, after the atomic - * commit has queued its entries, so they are queued unarmed and stamped here. - * Swap-start requests are serialized by the DCP command stack. - */ -static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) -{ - struct dcp_fb_reference *entry; - unsigned long flags; - - spin_lock_irqsave(&dcp->swapped_out_lock, flags); - list_for_each_entry(entry, &dcp->swapped_out_fbs, head) { - if (entry->armed) - continue; - entry->swap_id = swap_id; - entry->armed = true; - } - spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); -} - -/* - * Drop framebuffers whose unbinding swap has completed. - * - * Entries must be keyed to the swap that unbinds them, rather than the last - * swap that happened to have completed when the atomic commit ran. A later - * successful completion also retires entries whose own completion was lost; - * the signed difference keeps that comparison valid across a 32-bit wrap. - */ -static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) -{ - struct dcp_fb_reference *entry, *tmp; - unsigned long flags; - LIST_HEAD(done); - - /* - * last_swap_id / have_swap_complete are only ever touched by the RTKit - * worker, which is also the only caller of this function, so the lock - * is here for the list alone. - */ - spin_lock_irqsave(&dcp->swapped_out_lock, flags); - while (!list_empty(&dcp->swapped_out_fbs)) { - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - - /* Newest entries; their swap has not been started yet. */ - if (!entry->armed) - break; - /* last_swap_id is 0 from kzalloc until a swap really lands. */ - if (!dcp->have_swap_complete) - break; - /* Hold the old framebuffer for one additional completed swap. */ - if ((s32)(dcp->last_swap_id - entry->swap_id) <= 0) - break; - - list_move_tail(&entry->head, &done); - } - spin_unlock_irqrestore(&dcp->swapped_out_lock, flags); - - /* drm_framebuffer_put() can sleep, so never call it under the lock. */ - list_for_each_entry_safe(entry, tmp, &done, head) { - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} -#else -static void dcp_arm_swapped_out_fbs(struct apple_dcp *dcp, u32 swap_id) { } - -static void dcp_release_swapped_out_fbs(struct apple_dcp *dcp) -{ - while (!list_empty(&dcp->swapped_out_fbs)) { - struct dcp_fb_reference *entry; - - entry = list_first_entry(&dcp->swapped_out_fbs, - struct dcp_fb_reference, head); - if (entry->swap_id == dcp->last_swap_id) - break; - if (entry->fb) - drm_framebuffer_put(entry->fb); - list_del(&entry->head); - kfree(entry); - } -} -#endif - static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { @@ -319,10 +229,6 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, dcp_swap_watchdog_complete(dcp); dcp_drm_crtc_page_flip(dcp, now); -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - dcp->have_swap_complete = true; - dcp_release_swapped_out_fbs(dcp); -#endif if (dcp->crc_enabled) { u32 crc32 = 0; drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); @@ -1360,8 +1266,13 @@ static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; - if (cookie) { - struct dcp_swap_cookie *info = cookie; + if (status) { + dev_err(dcp->dev, "swap_clear failed! status %u\n", status); + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } + if (info) { + WRITE_ONCE(info->status, status); complete(&info->done); kref_put(&info->refcount, release_swap_cookie); } @@ -1390,12 +1301,11 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, { struct dcp_swap_start_resp *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_swapped_out_fbs(dcp, resp->swap_id); + dcp_arm_retained_framebuffers(dcp, resp->swap_id); if (cookie) { struct dcp_swap_cookie *info = cookie; info->swap_id = resp->swap_id; - } dcp_h17p_fix_swap_surfaces(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); @@ -1503,6 +1413,16 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, } } +static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) +{ + /* H17P stop semantics still require hardware qualification. */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + return false; +#else + return READ_ONCE(cookie->status) == 0; +#endif +} + static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_set_power_state_req power_req = { @@ -1856,8 +1776,13 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_start_resp *resp = data; + if (!resp || resp->ret) { + dev_err(dcp->dev, "swap_start was rejected\n"); + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_swapped_out_fbs(dcp, resp->swap_id); + dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_h17p_fix_swap_surfaces(dcp); @@ -2082,44 +2007,6 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru l = apl_plane->iomfb_surf; req->swap.swap_enabled |= BIT(l); - if (old_state->fb && new_state->fb != old_state->fb) { - /* - * Race condition between a framebuffer unbind getting - * swapped out and GEM unreferencing a framebuffer. If - * we lose the race, the display gets IOVA faults and - * the DCP crashes. We need to extend the lifetime of - * the drm_framebuffer (and hence the GEM object) until - * after we get a swap complete for the swap unbinding - * it. - */ - struct dcp_fb_reference *entry = - kzalloc(sizeof(*entry), GFP_KERNEL); - if (entry) { -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - unsigned long flags; - - entry->fb = old_state->fb; - /* - * Left unarmed: the id of the swap that - * unbinds this framebuffer is not known until - * the firmware answers swap_start. - */ - spin_lock_irqsave(&dcp->swapped_out_lock, - flags); - list_add_tail(&entry->head, - &dcp->swapped_out_fbs); - spin_unlock_irqrestore(&dcp->swapped_out_lock, - flags); -#else - entry->fb = old_state->fb; - entry->swap_id = dcp->last_swap_id; - list_add_tail(&entry->head, - &dcp->swapped_out_fbs); -#endif - } - drm_framebuffer_get(old_state->fb); - } - if (!new_state->fb || !new_state->visible) { continue; } @@ -2404,6 +2291,8 @@ void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) struct dcp_set_power_state_req req = { /* defaults are ok */ }; + struct dcp_wait_cookie *cookie; + int ret; dcp_set_power_state(dcp, false, &req, NULL, NULL); } diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index fbb1311e0bc5cd..93ddde4182b0fc 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -5,6 +5,7 @@ #include "plane.h" +#include "dcp-internal.h" #include "iomfb_internal.h" #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include @@ -292,13 +294,56 @@ static void apple_plane_atomic_update(struct drm_plane *plane, new_state->iova = obj->dma_addr + base->fb->offsets[0]; } +static void apple_plane_cleanup_fb(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct apple_plane_state *apple_state = to_apple_plane_state(state); + struct dcp_fb_reference *entry = apple_state->retirement; + + if (!entry) + return; + + apple_state->retirement = NULL; + drm_framebuffer_put(entry->fb); + kfree(entry); +} + +static int apple_plane_prepare_fb(struct drm_plane *plane, + struct drm_plane_state *state) +{ + struct drm_plane_state *old; + struct dcp_fb_reference *entry; + int ret; + + ret = drm_gem_plane_helper_prepare_fb(plane, state); + if (ret) + return ret; + + old = drm_atomic_get_old_plane_state(state->state, plane); + if (!old || !old->crtc || !old->fb || old->fb == state->fb) + return 0; + + entry = kzalloc_obj(*entry); + if (!entry) + return -ENOMEM; + + drm_framebuffer_get(old->fb); + entry->fb = old->fb; + to_apple_plane_state(state)->retirement = entry; + return 0; +} + static const struct drm_plane_helper_funcs apple_primary_plane_helper_funcs = { + .prepare_fb = apple_plane_prepare_fb, + .cleanup_fb = apple_plane_cleanup_fb, .atomic_check = apple_plane_atomic_check, .atomic_update = apple_plane_atomic_update, .get_scanout_buffer = drm_fb_dma_get_scanout_buffer, }; static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { + .prepare_fb = apple_plane_prepare_fb, + .cleanup_fb = apple_plane_cleanup_fb, .atomic_check = apple_plane_atomic_check, .atomic_update = apple_plane_atomic_update, }; @@ -307,8 +352,10 @@ static const struct drm_plane_helper_funcs apple_plane_helper_funcs = { static void apple_plane_reset(struct drm_plane *plane) { struct apple_plane_state *state = to_apple_plane_state(plane->state); - if (state) + if (state) { + apple_plane_cleanup_fb(plane, &state->base); __drm_atomic_helper_plane_destroy_state(&state->base); + } kfree(state); plane->state = NULL; @@ -337,19 +384,19 @@ apple_plane_duplicate_state(struct drm_plane *plane) return &apple_plane_state->base; } -// void apple_plane_destroy_state(struct drm_plane *plane, -// struct drm_plane_state *state) -// { -// drm_atomic_helper_plane_destroy_state(plane, state); -// } +static void apple_plane_destroy_state(struct drm_plane *plane, + struct drm_plane_state *state) +{ + apple_plane_cleanup_fb(plane, state); + drm_atomic_helper_plane_destroy_state(plane, state); +} static const struct drm_plane_funcs apple_plane_funcs = { .update_plane = drm_atomic_helper_update_plane, .disable_plane = drm_atomic_helper_disable_plane, .reset = apple_plane_reset, .atomic_duplicate_state = apple_plane_duplicate_state, - // .atomic_destroy_state = apple_plane_destroy_state, - .atomic_destroy_state = drm_atomic_helper_plane_destroy_state, + .atomic_destroy_state = apple_plane_destroy_state, }; /* diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index 2d1a03e91f38bc..7ed68b7b5b38d0 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -19,12 +19,15 @@ struct apple_plane { #define to_apple_plane(x) container_of(x, struct apple_plane, base) +struct dcp_fb_reference; + struct apple_plane_state { struct drm_plane_state base; struct dcp_surface surf; struct dcp_rect src_rect; struct dcp_rect dst_rect; u64 iova; + struct dcp_fb_reference *retirement; }; #define to_apple_plane_state(x) container_of(x, struct apple_plane_state, base) From 69bdd344ff9b98ac79048c3fbfaf028580cbbe60 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 01:07:37 +1000 Subject: [PATCH 120/631] dt-bindings: display: apple,dcp: document the DCP and display subsystem Describe the existing Asahi display nodes and the T8140 internal and external DCP compatibles. The piodma child and panel properties are included, and apple,iomfb-surfaces retains its per-slot enable meaning. T8140 DCPs must not specify a reset, because resetting the inherited display CPU can hang the machine (DT-5, DT-6). Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 256 ++++++++++++++++++ .../display/apple,display-subsystem.yaml | 29 ++ MAINTAINERS | 2 + 3 files changed, 287 insertions(+) create mode 100644 Documentation/devicetree/bindings/display/apple,dcp.yaml create mode 100644 Documentation/devicetree/bindings/display/apple,display-subsystem.yaml diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml new file mode 100644 index 00000000000000..47610c176cfd2b --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,dcp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display coprocessor + +maintainers: + - Ryan Murray + +description: + Apple's Display Co-Processor (DCP) drives internal and external displays + through an RTKit mailbox. Its piodma child uses a separate display DART + stream from the coprocessor itself. + +properties: + compatible: + oneOf: + - items: + - enum: + - apple,t8103-dcp + - apple,t8112-dcp + - apple,t6000-dcp + - apple,t6020-dcp + - apple,t8140-dcp + - const: apple,dcp + - items: + - enum: + - apple,t8103-dcpext + - apple,t8112-dcpext + - apple,t6000-dcpext + - apple,t6020-dcpext + - const: apple,dcpext + - items: + - const: apple,t8140-dcpext + - const: apple,dcp + + reg: + minItems: 2 + maxItems: 6 + + reg-names: + minItems: 2 + items: + - const: coproc + - const: disp-0 + - const: disp-1 + - const: disp-2 + - const: disp-3 + - const: disp-4 + + mboxes: + maxItems: 1 + + mbox-names: + items: + - const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + clocks: + minItems: 1 + maxItems: 2 + + phys: + minItems: 1 + maxItems: 8 + + phy-names: + minItems: 1 + maxItems: 8 + items: + pattern: '^(dp-phy|typec[0-9]+)$' + + apple,bw-scratch: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: PMGR bandwidth scratch resource, display index and offset. + + apple,bw-doorbell: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: Optional PMGR bandwidth doorbell resource and display index. + + apple,iomfb-surfaces: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 4 + maxItems: 4 + items: + enum: [0, 1] + description: Per-slot enable flags, in slot order. + + apple,firmware-version: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Version reported by the loaded display firmware. + + apple,firmware-compat: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 3 + maxItems: 5 + description: Loader-provided firmware ABI version used for IOMFB selection. + + apple,connector-type: + enum: [HDMI-A, DP, USB-C] + + apple,dcp-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Display coprocessor index on multi-DCP systems. + + apple,dptx-phy: + $ref: /schemas/types.yaml#/definitions/uint32 + description: DisplayPort transmitter PHY selector. + + apple,dptx-die: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Die containing the DisplayPort transmitter. + + apple,notch-height: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Height in pixels reserved for the display notch. + + apple,typec-mux-indices: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: Crossbar state for each Type-C display route. + + apple,typec-dptx-phys: + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + description: DisplayPort PHY index for each Type-C display route. + + mux-controls: + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 8 + + mux-control-names: + minItems: 1 + maxItems: 8 + items: + pattern: '^(dp-xbar|typec[0-9]+)$' + + mux-index: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Crossbar state for a fixed output. + + hdmi-hpd-gpios: + maxItems: 1 + + hdmi-pwren-gpios: + maxItems: 1 + + dp2hdmi-pwren-gpios: + maxItems: 1 + + '#address-cells': + const: 2 + + '#size-cells': + const: 2 + + piodma: + type: object + properties: + iommus: + maxItems: 1 + required: + - iommus + additionalProperties: false + + panel: + type: object + properties: + compatible: + oneOf: + - const: apple,panel + - items: + - pattern: '^apple,panel-j[0-9]+$' + - const: apple,panel + - items: + - pattern: '^apple,panel-j[0-9]+$' + - const: apple,panel-mini-led + - const: apple,panel + width-mm: + description: Physical panel width. + height-mm: + description: Physical panel height. + adj-height-mm: + description: Visible height below the notch. + apple,max-brightness: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Maximum firmware backlight level. + required: + - compatible + additionalProperties: false + + typec-routes: + type: object + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + patternProperties: + '^route@[0-3]$': + type: object + properties: + reg: + maxItems: 1 + mode-switch: + type: boolean + port: + $ref: /schemas/graph.yaml#/properties/port + required: + - reg + - mode-switch + - port + additionalProperties: false + required: + - '#address-cells' + - '#size-cells' + additionalProperties: false + + ports: + $ref: /schemas/graph.yaml#/properties/ports + +required: + - compatible + - reg + - reg-names + - mboxes + - iommus + - apple,bw-scratch + +allOf: + - if: + properties: + compatible: + contains: + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + then: + properties: + resets: false + +unevaluatedProperties: false +... diff --git a/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml new file mode 100644 index 00000000000000..429e68b29d864a --- /dev/null +++ b/Documentation/devicetree/bindings/display/apple,display-subsystem.yaml @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/apple,display-subsystem.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display subsystem + +maintainers: + - Ryan Murray + +description: + Component master for one or more Apple display coprocessors. Each IOMMU + entry describes a scanout DART stream owned by a DCP instance. + +properties: + compatible: + const: apple,display-subsystem + + iommus: + minItems: 1 + maxItems: 8 + +required: + - compatible + - iommus + +unevaluatedProperties: false +... diff --git a/MAINTAINERS b/MAINTAINERS index d9454a3d24a2f6..df53a2d58f3f41 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1952,6 +1952,8 @@ L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml +F: Documentation/devicetree/bindings/display/apple,dcp.yaml +F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: drivers/gpu/drm/apple/ F: drivers/gpu/drm/apple/dcp-fabric* F: include/linux/soc/apple/dp-tunnel.h From 0f0b451a7940bef26577e179a7753ba936c81e1e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 29 Sep 2026 01:09:44 +1000 Subject: [PATCH 121/631] dt-bindings: chosen: document apple,dcp-rtkit-quiesced The loader reports whether the DCP core was running at hand-off using a u32 in /chosen. Document both values and allow the property to be absent on a chainload. The kernel uses this label to select its start mode once the H17P path is implemented (DT-5, LIFE-1). Assisted-by: LLM Signed-off-by: Ryan Murray --- .../chosen/apple,dcp-rtkit-quiesced.yaml | 36 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 37 insertions(+) create mode 100644 Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml diff --git a/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml b/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml new file mode 100644 index 00000000000000..d086bdc05f5ee7 --- /dev/null +++ b/Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/chosen/apple,dcp-rtkit-quiesced.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple DCP RTKit loader hand-off + +maintainers: + - Ryan Murray + +description: + The loader records whether it handed a running display coprocessor to + the kernel. It omits this property on a chainload. + +select: + properties: + $nodename: + const: chosen + required: + - $nodename + +properties: + $nodename: + const: chosen + + apple,dcp-rtkit-quiesced: + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1] + description: + One if the DCP core was running at hand-off, zero if it was not. + The name reflects the loader's RTKit hand-off state, not a request + for Linux to quiesce the coprocessor. + +additionalProperties: true +... diff --git a/MAINTAINERS b/MAINTAINERS index df53a2d58f3f41..a0e00d4c6f86b4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1952,6 +1952,7 @@ L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml +F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: drivers/gpu/drm/apple/ From 3ba602cc2da6a9ccf03d5fcddd6ad7ecb11eb385 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 6 Oct 2026 23:02:48 +1100 Subject: [PATCH 122/631] dt-bindings: display: apple,dcp: describe named handoff resources Describe the H17-generation iop-vbar window independently of the numbered display windows and include the existing T8142 compatible. Document the clock names consumed by the driver and the loader's reserved-memory lists. Permit partial surface-enable arrays: the public driver reads one flag per slot and accepts up to four entries. This preserves its existing meaning instead of requiring every device tree to provide all four slots. The named resources follow the public Apple DRM driver and the Linux-side J700 hardware-description observation. No reset is added to a device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 47610c176cfd2b..ac2286832ba7d1 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -24,6 +24,7 @@ properties: - apple,t6000-dcp - apple,t6020-dcp - apple,t8140-dcp + - apple,t8142-dcp - const: apple,dcp - items: - enum: @@ -38,17 +39,18 @@ properties: reg: minItems: 2 - maxItems: 6 + maxItems: 7 reg-names: minItems: 2 items: - const: coproc - const: disp-0 - - const: disp-1 - - const: disp-2 - - const: disp-3 - - const: disp-4 + - enum: [disp-1, iop-vbar] + - enum: [disp-2, iop-vbar] + - enum: [disp-3, iop-vbar] + - enum: [disp-4, iop-vbar] + - const: iop-vbar mboxes: maxItems: 1 @@ -70,6 +72,20 @@ properties: minItems: 1 maxItems: 2 + clock-names: + minItems: 1 + items: + - const: clock-154 + - const: clock-194 + + memory-region: + minItems: 1 + maxItems: 64 + + memory-region-names: + minItems: 1 + maxItems: 64 + phys: minItems: 1 maxItems: 8 @@ -90,7 +106,7 @@ properties: apple,iomfb-surfaces: $ref: /schemas/types.yaml#/definitions/uint32-array - minItems: 4 + minItems: 1 maxItems: 4 items: enum: [0, 1] From 996a8b426e785b177b69d2bedc2d6a4aa740dca0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 13:25:00 +1100 Subject: [PATCH 123/631] drm: apple: balance backlight modeset locking A brightness-only atomic commit can acquire more than the CRTC lock. Handle -EDEADLK by clearing the temporary state, backing off and retrying, then drop the acquired locks and finalize the acquire context on every exit. The previous path returned after backoff with the slow lock held and never finalized the context on success or interruption. Use the DRM single-lock helper when updating the cached brightness value. That path acquires only the CRTC mutex and does not need an acquire context. Release the mutex before starting the atomic brightness update. This follows the public DRM modeset-lock and atomic-state contracts and applies to every firmware variant without changing the brightness bytes. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 28 ++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 055aecc8af709f..16a8000659bb66 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -162,23 +162,34 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) goto done; state = drm_atomic_state_alloc(crtc->dev); - if (!state) { - ret = -ENOMEM; - goto done; - } + if (!state) + return -ENOMEM; + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); state->acquire_ctx = &ctx; + +retry: crtc_state = drm_atomic_get_crtc_state(state, crtc); if (IS_ERR(crtc_state)) { ret = PTR_ERR(crtc_state); - goto fail; + goto out; } + if (!dcp->brightness.update) + goto out; + crtc_state->color_mgmt_changed |= true; ret = drm_atomic_commit(state); -fail: +out: + if (ret == -EDEADLK) { + drm_atomic_state_clear(state); + ret = drm_modeset_backoff(&ctx); + if (!ret) + goto retry; + } + drm_atomic_state_put(state); done: drm_modeset_drop_locks(&ctx); @@ -209,9 +220,11 @@ static int dcp_set_brightness(struct backlight_device *bd) { int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); - struct drm_modeset_acquire_ctx ctx; int brightness = backlight_get_brightness(bd); + ret = drm_modeset_lock_single_interruptible(&dcp->crtc->base.mutex); + if (ret) + return ret; drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); ret = drm_modeset_lock(&dcp->crtc->base.mutex, &ctx); if (ret == -EDEADLK) { @@ -228,6 +241,7 @@ static int dcp_set_brightness(struct backlight_device *bd) drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); + drm_modeset_unlock(&dcp->crtc->base.mutex); return dcp_backlight_update(dcp); } From 7020344136f49d43c429e845e1367733da4e8906 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 14:14:32 +1100 Subject: [PATCH 124/631] drm: apple: preserve scanout in brightness-only commits The brightness update creates a CRTC-only atomic state and marks its colour state changed. The atomic helpers do not add unchanged planes for this non-modesetting transaction, but the DCP present builder clears all surfaces and repopulates them only from planes in the atomic state. Add the CRTC's affected planes before committing so a static-screen brightness update re-presents the current scanout instead of a request without surfaces. Use the existing cached plane-state duplication and framebuffer references. Plane-state allocation and locking errors follow the same atomic backoff and cleanup path as the CRTC state. This uses the public DRM atomic-state contract and the existing Asahi present path for all firmware variants, without changing record layouts. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 16a8000659bb66..b1a9a5febd4241 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -178,6 +178,11 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) if (!dcp->brightness.update) goto out; + /* Re-present the current scanout when brightness is the only change. */ + ret = drm_atomic_add_affected_planes(state, crtc); + if (ret) + goto out; + crtc_state->color_mgmt_changed |= true; ret = drm_atomic_commit(state); From 2f736322a7716092438f22cbc44fccfe2a9a9653 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 7 Oct 2026 17:09:51 +1100 Subject: [PATCH 125/631] drm: apple: skip failed timing entries before parsing the next enumerate_modes() treats a failure to parse one timing entry as recoverable, but parse_mode() can return before consuming the rest of that entry. The next iteration then starts inside the rejected dictionary instead of at the next mode. Save the entry's starting position and skip the complete entry after a parse failure. If even the generic dictionary walker cannot skip it, return that error instead of continuing from an unknown boundary. Own public-format dictionary fixtures reproduce the failure with a wrongly typed timing score or colour ID followed by a valid mode. The following modes are now retained, including when a valid mode precedes the rejected one. Complete dictionaries keep the existing parse path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 20ced0f826ed3c..822138e1b84b8e 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -681,8 +681,16 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, *handle = next; /* Errors for a single mode are recoverable -- just skip it. */ - if (ret) + if (ret) { + /* A failed parse can stop in the middle of the dictionary. */ + *it.handle = entry; + ret = skip(it.handle); + if (ret) { + kfree(modes); + return ERR_PTR(ret); + } continue; + } /* Process a successful mode */ (*count)++; From 7a978c4063afe7fbe1b7dda5c3210b4f313a6cfb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:50 +1100 Subject: [PATCH 126/631] drm: apple: add present-coupled H17P backlight policy Implement the clean display SPEC BACKLIGHT-1..4 policy in the existing Asahi backlight path. Keep commanded nits and completed levels separate, preserve takeover, and cache requests while blanked or suspended. Internal H17P DPMS keeps the pipe and surface alive. Admit range, takeover and wire encoding through a narrow interface to the serialized present queue. No H17P brightness bytes or default nits are inferred. Keep policy storage at the end of the device and gate template changes at compile time so older firmware keeps its existing fields and generated code. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 9 ++ drivers/gpu/drm/apple/dcp.c | 18 +++ drivers/gpu/drm/apple/dcp_backlight.c | 157 +++++++++++++++++++++++- drivers/gpu/drm/apple/dcp_backlight.h | 165 ++++++++++++++++++++++++++ drivers/gpu/drm/apple/systemep.c | 6 + 5 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/dcp_backlight.h diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index fa4c7e097bf86c..fd4c3e26c6a410 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -19,6 +19,7 @@ #include "dcp-fabric.h" #include "dcp-fabric-session.h" #include "dptxep.h" +#include "dcp_backlight.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb-state.h" @@ -462,6 +463,14 @@ struct apple_dcp { struct apple_connector *fixed_connector; struct apple_connector *typec_connector; int hdmi_hpd_irq; + + /* H17P policy shared by backlight, commit, and reply workers. */ + struct { + /* Protects state and callback installation. */ + spinlock_t lock; + struct dcp_backlight_state state; + void (*kick)(struct apple_dcp *dcp); + } backlight; /* Hardirq, resume and expiry share one generation under this lock. */ spinlock_t hdmi_presence_lock; struct dcp_fabric_presence hdmi_presence; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0d6e6e41a72351..a146bdc8cf970d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1666,6 +1666,13 @@ void dcp_poweron(struct platform_device *pdev) unsigned long remaining; int ret; + if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + ret = dcp_backlight_dpms(dcp, true); + if (ret) + dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); + return; + } + if (dcp_is_typec_output(dcp)) { /* T8122 dcpext only, see dcp_poweroff() */ if (dcp->ext_failsafe) { @@ -1736,6 +1743,14 @@ void dcp_poweroff(struct platform_device *pdev) bool display_gone = false; int ret; + /* Internal H17P DPMS is a brightness present, never a pipe stop. */ + if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + ret = dcp_backlight_dpms(dcp, false); + if (ret) + dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); + return; + } + cancel_delayed_work(&dcp->swap_watchdog_wq); if (dcp->avep) av_service_disconnect(dcp); @@ -1833,6 +1848,8 @@ static void dcp_work_register_backlight(struct work_struct *work) /* try to register backlight device, */ ret = dcp_backlight_register(dcp); + if (ret == -ENODATA) + goto out_unlock; if (ret) { dev_err(dcp->dev, "Unable to register backlight device\n"); dcp->brightness.maximum = 0; @@ -2776,6 +2793,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); + spin_lock_init(&dcp->backlight.lock); dcp->fw_compat = fw_compat; dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext"); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index b1a9a5febd4241..d16a0a3b124f1b 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -85,13 +85,125 @@ static u32 brightness_part2[] = { }; +bool dcp_backlight_active(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + READ_ONCE(dcp->backlight.state.ready); +} + static int dcp_get_brightness(struct backlight_device *bd) { struct apple_dcp *dcp = bl_get_data(bd); + unsigned long flags; + u32 actual; + + if (dcp_backlight_active(dcp)) { + spin_lock_irqsave(&dcp->backlight.lock, flags); + actual = dcp->backlight.state.actual; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return actual; + } return dcp->brightness.nits; } +int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits, + void (*kick)(struct apple_dcp *dcp)) +{ + unsigned long flags; + int ret; + + if (dcp->fw_compat != DCP_FIRMWARE_H17P || !dcp_has_panel(dcp) || + !dcp->crtc || !kick) + return -EINVAL; + + mutex_lock(&dcp->bl_register_mutex); + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (dcp->backlight.state.ready || dcp->brightness.bl_dev) { + ret = -EBUSY; + } else { + ret = dcp_bl_init(&dcp->backlight.state, maximum, + inherited_valid, inherited, + default_valid, default_nits); + if (!ret) + dcp->backlight.kick = kick; + } + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + mutex_unlock(&dcp->bl_register_mutex); + if (!ret) + schedule_work(&dcp->bl_register_wq); + return ret; +} + +bool dcp_backlight_seed(struct apple_dcp *dcp, u32 nits) +{ + unsigned long flags; + bool seeded; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + seeded = dcp_bl_seed(&dcp->backlight.state, nits); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return seeded; +} + +static void dcp_backlight_kick(struct apple_dcp *dcp) +{ + void (*kick)(struct apple_dcp *dcp) = READ_ONCE(dcp->backlight.kick); + + if (kick) + kick(dcp); +} + +int dcp_backlight_dpms(struct apple_dcp *dcp, bool on) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_dpms(&dcp->backlight.state, on); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (!ret) + dcp_backlight_kick(dcp); + return ret; +} + +int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, + struct dcp_backlight_present *present) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_prepare(&dcp->backlight.state, have_surface, present); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return ret; +} + +bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted) +{ + unsigned long flags; + bool completed; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + completed = dcp_bl_complete(&dcp->backlight.state, sequence, accepted); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + /* Queue retries after the central queue has finished this transaction. */ + return completed; +} + +bool dcp_backlight_pending(struct apple_dcp *dcp) +{ + unsigned long flags; + bool pending; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + pending = dcp->backlight.state.dirty && !dcp->backlight.state.in_flight; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return pending; +} + #define SCALE_FACTOR (1 << 10) static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) @@ -205,6 +317,11 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) int dcp_backlight_update(struct apple_dcp *dcp) { + if (dcp_backlight_active(dcp)) { + dcp_backlight_kick(dcp); + return 0; + } + /* * Do not actively try to change brightness if no mode is set. * TODO: should this be reflected the in backlight's power property? @@ -226,6 +343,20 @@ static int dcp_set_brightness(struct backlight_device *bd) int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); int brightness = backlight_get_brightness(bd); + unsigned long flags; + + if (dcp_backlight_active(dcp)) { + /* Preserve the requested level even while the core forces zero. */ + spin_lock_irqsave(&dcp->backlight.lock, flags); + ret = dcp_bl_request(&dcp->backlight.state, + bd->props.brightness, + backlight_is_blank(bd), + bd->props.state & BL_CORE_SUSPENDED); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (!ret) + dcp_backlight_kick(dcp); + return ret; + } ret = drm_modeset_lock_single_interruptible(&dcp->crtc->base.mutex); if (ret) @@ -251,22 +382,45 @@ static int dcp_set_brightness(struct backlight_device *bd) return dcp_backlight_update(dcp); } +static bool dcp_backlight_controls_device(struct backlight_device *bd, + struct device *display_dev) +{ + struct apple_dcp *dcp = bl_get_data(bd); + + return !display_dev || display_dev == dcp->dev || + (dcp->crtc && display_dev == dcp->crtc->base.dev->dev); +} + static const struct backlight_ops dcp_backlight_ops = { .options = BL_CORE_SUSPENDRESUME, .get_brightness = dcp_get_brightness, .update_status = dcp_set_brightness, + .controls_device = dcp_backlight_controls_device, }; int dcp_backlight_register(struct apple_dcp *dcp) { struct device *dev = dcp->dev; struct backlight_device *bl_dev; + unsigned long flags; struct backlight_properties props = { .type = BACKLIGHT_PLATFORM, .brightness = dcp->brightness.nits, .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + if (dcp_backlight_active(dcp)) { + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (!dcp->backlight.state.ready) { + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return -ENODATA; + } + /* Freeze takeover before userspace can discover the device. */ + dcp->backlight.state.controlled = true; + props.brightness = dcp->backlight.state.target; + props.max_brightness = dcp->backlight.state.maximum; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + } bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, &dcp_backlight_ops, &props); @@ -274,7 +428,8 @@ int dcp_backlight_register(struct apple_dcp *dcp) return PTR_ERR(bl_dev); dcp->brightness.bl_dev = bl_dev; - dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); + if (!dcp_backlight_active(dcp)) + dcp->brightness.dac = calculate_dac(dcp, dcp->brightness.nits); return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h new file mode 100644 index 00000000000000..6d52a8f9eb7fe1 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Ryan Murray */ + +#ifndef __APPLE_DCP_BACKLIGHT_H__ +#define __APPLE_DCP_BACKLIGHT_H__ + +#include +#include +#include + +/* Commanded nits, independent of firmware encoding. No fade policy. */ +struct dcp_backlight_state { + u32 maximum; + u32 target; + u32 actual; + u32 sent_nits; + u64 sequence; + bool ready; + bool controlled; + bool dpms_off; + bool core_blank; + bool suspended; + bool dirty; + bool in_flight; +}; + +struct dcp_backlight_present { + u64 sequence; + u32 nits; +}; + +/* Inputs must come from admitted panel/loader records, never a guess. */ +static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits) +{ + u32 nits; + + if (!maximum || maximum > INT_MAX) + return -EINVAL; + if (!inherited_valid && !default_valid) + return -ENODATA; + nits = inherited_valid ? inherited : default_nits; + if (nits > maximum) + return -ERANGE; + + *state = (struct dcp_backlight_state) { + .maximum = maximum, + .target = nits, + .actual = nits, + .ready = true, + }; + return 0; +} + +/* A decoded powerlog sample can seed takeover only before Linux control. */ +static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) +{ + if (!state->ready || state->controlled || nits > state->maximum) + return false; + state->target = nits; + state->actual = nits; + return true; +} + +static inline u32 dcp_bl_effective(const struct dcp_backlight_state *state) +{ + if (state->dpms_off || state->core_blank || state->suspended) + return 0; + return state->target; +} + +static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, + bool core_blank, bool suspended) +{ + if (!state->ready) + return -ENODATA; + if (nits > state->maximum) + return -ERANGE; + state->controlled = true; + if (state->target != nits || state->core_blank != core_blank || + state->suspended != suspended) { + state->target = nits; + state->core_blank = core_blank; + state->suspended = suspended; + state->dirty = true; + } + return 0; +} + +static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) +{ + if (!state->ready) + return -ENODATA; + state->controlled = true; + if (state->dpms_off != !on) { + state->dpms_off = !on; + state->dirty = true; + } + return 0; +} + +/* + * Called by the single outbound queue, with a pinned, accepted scanout or a + * prepared replacement. Soft-off never releases it. A brightness-only present + * must use the last accepted surface, not a rejected replacement's DRM state. + */ +static inline int dcp_bl_prepare(struct dcp_backlight_state *state, + bool have_surface, + struct dcp_backlight_present *present) +{ + if (!state->ready || !have_surface) + return -ENODATA; + if (state->in_flight) + return -EBUSY; + if (!state->dirty) + return -EALREADY; + + state->controlled = true; + state->in_flight = true; + state->dirty = false; + state->sent_nits = dcp_bl_effective(state); + present->nits = state->sent_nits; + present->sequence = ++state->sequence; + return 0; +} + +/* + * accepted means both accepted submit and its matching completed present. + * Rejection/serialization failure cancels the reservation without changing + * actual. This result is NEVER a framebuffer retirement authorization. + */ +static inline bool dcp_bl_complete(struct dcp_backlight_state *state, + u64 sequence, bool accepted) +{ + if (!state->in_flight || sequence != state->sequence) + return false; + state->in_flight = false; + if (accepted) + state->actual = state->sent_nits; + else + state->dirty = true; + return true; +} + +struct apple_dcp; + +/* + * kick only queues work; it must not submit a top-level RPC from the caller. + * The central IOMFB queue owns surface references, retries, and wire encoding. + * Configure before registration and keep the callback valid until teardown. + */ +bool dcp_backlight_active(struct apple_dcp *dcp); +int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits, + void (*kick)(struct apple_dcp *dcp)); +bool dcp_backlight_seed(struct apple_dcp *dcp, u32 nits); +int dcp_backlight_dpms(struct apple_dcp *dcp, bool on); +int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, + struct dcp_backlight_present *present); +bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); +bool dcp_backlight_pending(struct apple_dcp *dcp); + +#endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 9fe7a0ce495aab..3133cd539b7f1d 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -75,6 +75,12 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype struct apple_dcp *dcp = service->ep->dcp; int ret; + /* H17P takeover decoding requires an admitted record. Later zero reports + * must not overwrite the cached target or completed level. + */ + if (dcp_backlight_active(dcp)) + return 0; + dev_dbg(dcp->dev, "systemep[ch:%u]: report type:%02x len:%zu\n", service->channel, type, data_size); From 4879e400f6e68a4b8208ce6cff5fdf9a4b7c597b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:50 +1100 Subject: [PATCH 127/631] drm: apple: test H17P backlight takeover and DPMS policy Exercise the clean SPEC takeover, cached-level restore and suspend ordering requirements. Verify that missing surfaces, busy transactions and rejected or stale completions do not change the completed level or lose pending requests. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/dcp_backlight_test.c | 208 +++++++++++++++++++++ 3 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/dcp_backlight_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 56b80b6c28d194..1e448994fb87f1 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -63,7 +63,8 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST depends on KUNIT default KUNIT_ALL_TESTS help - Build tests for the Apple display driver's AFK ring parser. + Build tests for the Apple display driver's AFK ring parser and + backlight policy (takeover, DPMS, suspend and present completion). The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 52d3eed3453503..c78069509905c0 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -32,3 +32,4 @@ appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += hotplug-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += external-hold-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += v14-board-test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c new file mode 100644 index 00000000000000..e990ea3e0bf066 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include "dcp_backlight.h" + +static void backlight_takeover_test(struct kunit *test) +{ + struct dcp_backlight_state state = {}; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_TRUE(test, dcp_bl_seed(&state, 400)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 601)); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 0)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 0, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, false, 0, true, 36), 0); + KUNIT_EXPECT_EQ(test, state.target, 36U); +} + +static void backlight_range_test(struct kunit *test) +{ + struct dcp_backlight_state state = {}; + + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 600, false, 0, false, 0), -ENODATA); + KUNIT_EXPECT_FALSE(test, state.ready); + KUNIT_EXPECT_EQ(test, dcp_bl_request(&state, 140, false, false), -ENODATA); + KUNIT_EXPECT_EQ(test, dcp_bl_dpms(&state, false), -ENODATA); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 0, true, 0, false, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, U32_MAX, true, 0, false, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_init(&state, 600, true, 601, false, 0), -ERANGE); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_request(&state, 601, false, false), -ERANGE); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 600, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 600U); +} + +static void backlight_dpms_cycles_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + int i; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 400, false, 0), 0); + for (i = 0; i < 20; i++) { + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 0U); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + } + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_EXPECT_EQ(test, state.target, 400U); +} + +static void backlight_blanked_target_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 400, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 140, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.target, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); + KUNIT_EXPECT_EQ(test, state.actual, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 140U); +} + +static void backlight_suspend_test(struct kunit *test) +{ + struct dcp_backlight_state state; + int off_first; + + for (off_first = 0; off_first < 2; off_first++) { + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + if (off_first) + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 140, true, true), 0); + if (!off_first) + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, true, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 36U); + KUNIT_EXPECT_EQ(test, state.target, 36U); + } + /* A core blank also keeps the panel dark independently of DRM DPMS. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, true, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 400U); +} + +static void backlight_surface_and_busy_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present first, second; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &first), -EALREADY); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 36, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, false, &first), -ENODATA); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_FALSE(test, state.in_flight); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &first), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &second), -EBUSY); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, first.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 36U); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &second), 0); + KUNIT_EXPECT_EQ(test, second.nits, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_complete(&state, first.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.in_flight); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, second.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); +} + +static void backlight_reject_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_EQ(test, state.actual, 140U); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_FALSE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, state.actual, 0U); +} + +static void backlight_inflight_dpms_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 140, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); +} + +static struct kunit_case backlight_cases[] = { + KUNIT_CASE(backlight_takeover_test), + KUNIT_CASE(backlight_range_test), + KUNIT_CASE(backlight_dpms_cycles_test), + KUNIT_CASE(backlight_blanked_target_test), + KUNIT_CASE(backlight_suspend_test), + KUNIT_CASE(backlight_surface_and_busy_test), + KUNIT_CASE(backlight_reject_test), + KUNIT_CASE(backlight_inflight_dpms_test), + {} +}; + +static struct kunit_suite backlight_suite = { + .name = "apple-dcp-backlight", + .test_cases = backlight_cases, +}; + +kunit_test_suite(backlight_suite); + +MODULE_LICENSE("Dual MIT/GPL"); From f33fc3ee7135f648978e70bb9b80ad3092d8c331 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:12:51 +1100 Subject: [PATCH 128/631] docs: gpu: describe Apple H17P backlight admission contract Document the clean SPEC backlight policy, central queue and surface ownership interface, missing admission records, and hardware gates. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-backlight.rst | 129 ++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 Documentation/gpu/apple-dcp-backlight.rst diff --git a/Documentation/gpu/apple-dcp-backlight.rst b/Documentation/gpu/apple-dcp-backlight.rst new file mode 100644 index 00000000000000..e47e7f3b4dd4be --- /dev/null +++ b/Documentation/gpu/apple-dcp-backlight.rst @@ -0,0 +1,129 @@ +.. SPDX-License-Identifier: GPL-2.0-only OR MIT + +Apple DCP H17P backlight integration +=================================== + +The backlight policy implements the clean display specification's BACKLIGHT-1 +through BACKLIGHT-4, with IOMFB-3 and LIFETIME-1 through LIFETIME-3 constraints. +It extends the existing Asahi backlight driver. It contains no H17P brightness +wire layout, calibration table, property command, or guessed initial level. +Policy activation requires explicit successful configuration. Existing +unconfigured H17P sessions, including T8142, keep their established DAC, +property/powerlog and DPMS paths. Older firmware remains on its existing path. +Neo startup must configure the policy before exposing the backlight or starting +its DPMS/brightness traffic; a firmware-version match alone does not activate +this policy. + +Admission and startup +--------------------- + +The central display worker must call ``dcp_backlight_configure()`` from process +context after linking the CRTC and admitting all required records. Its arguments +are the validated maximum commanded nits, an optional inherited level, an +optional documented ADT default, and a callback that schedules the central +outbound queue. Maximum must fit the backlight core's signed integer range. +An inherited zero is valid and takes precedence over the default. With neither +level available, configuration returns ``-ENODATA``. The policy backlight is registered only after configuration succeeds. +Configuration must precede any existing backlight registration; it returns +``-EBUSY`` if a device has already been published. Configuration does not send a command. + +The device exposes ``apple-panel-bl`` with platform type, linear scale and nits +units. Its maximum is the supplied ceiling; the older 509-nit DAC cap does not +apply to configured H17P policy. Consumer matching accepts the DCP device and its DRM device. +Registration freezes takeover before publishing the backlight. A decoded +powerlog value may seed the state through ``dcp_backlight_seed()`` before that +point. All samples after Linux takes control, including per-frame zeros, are +ignored. The older property and powerlog parsers do not seed an active policy implicitly. +Gather a sample before configuration and pass it as the inherited argument if +asynchronous registration could otherwise win the race. + +``brightness`` is the requested restore target. ``actual_brightness`` is the +last completed commanded level, initially the inherited/default estimate. This +is not a photometric measurement. Updates step immediately; no kernel fade or +initial-brightness/blank-level override is introduced. + +Queue and lifetime contract +--------------------------- + +The callback only schedules/coalesces work. It must not submit an RPC directly, +take DRM modeset locks, or run a present inline in a receive/commit callback. +It remains valid until the central queue and backlight workers are drained at +teardown. Policy access is protected by the brightness spinlock; the callback +is called after dropping it. Configuration also takes the registration mutex. + +The single outbound queue handles normal and brightness-only presents. A static +or inactive screen uses the last *accepted* surface, including its cached +geometry and IOVA. It must hold that surface's framebuffer reference and mapping +before calling ``dcp_backlight_prepare()`` with ``have_surface = true``. A DRM +plane state containing a rejected replacement is not an accepted surface. +The policy's boolean is a caller assertion, not a framebuffer reference. +Allocate any retirement bookkeeping during atomic prepare, never here. + +Prepare reserves a sequence and returns commanded nits. ``-ENODATA`` means no +configuration or surface; ``-EBUSY`` means a prior brightness transaction is +outstanding; ``-EALREADY`` means no brightness change is pending. These results +do not clear pending work. The serializer places the returned level only in +admitted fields of the next valid present. A normal present without a brightness +change keeps the inherited/completed level; its clean layout must support that. +The sequence is a software cookie, not a wire present ID. + +Associate the cookie with the central transaction and its strict wire ID. Call +``dcp_backlight_complete(sequence, true)`` only after both accepted submission +and matching completion. On serialization/submission rejection call it with +``false``; actual remains unchanged and the update is pending again. Wrong or +duplicate sequence callbacks have no effect. On a firmware crash retain the +surface and stop new submissions; there is no local reset/retry ladder. +After a transaction drains, inspect ``dcp_backlight_pending()`` and schedule the +next update or defer until the pipe becomes usable. Requests received during a +transaction remain pending. The central queue must bound static-screen latency +to 200 ms; this still requires a hardware timing test. + +A brightness-only completion never authorizes framebuffer retirement, including +retirement associated with a rejected replacement. Existing generic retention +helpers must be called only for the corresponding accepted replacement. This +policy never arms, releases, or unmaps a framebuffer. + +DPMS and suspend +---------------- + +Internal-panel H17P ``dcp_poweroff()`` and ``dcp_poweron()`` change the policy and +schedule the retained-surface queue. They bypass the legacy pipe-off/on path, +including surface clearing and final release. Until configuration, they warn +and leave the pipe intact; they cannot claim a completed hardware blank. +Startup must independently establish/adopt the live pipe before enabling the +policy. External outputs retain their existing power handling. + +DPMS off, a backlight core blank, or backlight core suspend forces effective nits +to zero while preserving the user target. Brightness changes during any blank +update the target. DPMS on restores it only when core blank/suspend is cleared. +The backlight core's suspend/resume callbacks feed this state; the central PM +worker still owns queue draining, domain lifetime and resume order. Do not call +unqualified H17P sleep/pipe-off methods as a suspend fallback. Keep the last +accepted surface mapped across soft-off and resume. + +Required records and qualification +---------------------------------- + +The following SPEC admission records must supply facts before production use: + +* R-ADT: panel ceiling/calibration in commanded nits and a documented default + if no inherited sample is available. No expected 525-nit ceiling is hardcoded. +* R-BACKLIGHT: takeover sample source, units, validity and timing; decoded + powerlog semantics; commanded-nits conversion, valid update/enable fields, + field offsets, widths and byte order in the present. +* R-SWAP: complete surface layout and successful submit/completion correlation + for normal and retained-surface presents, including inactive CRTC handling. +* R-DPMS: which admitted brightness/property actions blank and restore the live + pipe without clearing its surface, plus wake ordering. No new numeric property + or guessed DPMS payload is emitted by this policy. +* R-SERIALIZE: ordered brightness/present/property/diagnostic timeline and busy + deferral through the central queue; static-screen update latency. +* R-TEARDOWN: safe final release and suspend/resume boundary. Soft-off supplies + no such release boundary. + +Run ``apple-dcp-backlight`` KUnit tests with ``CONFIG_DRM_APPLE_KUNIT_TEST``. +They cover takeover/default/zero handling, range validation, twenty DPMS cycles, +cached restore, both suspend/DPMS orders, static-surface deferral, busy sequencing, +rejection and stale completion. They do not qualify firmware fields or GNOME. +Hardware acceptance requires O-BACKLIGHT, O-BACKLIGHT-TAKEOVER (10 Hz PBwo with +no jump above 0.05 W), O-BACKLIGHT-CRC, O-DPMS (twenty cycles), and O-SUSPEND. From 6df083649bf099bff679ccad76253e499d813a7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:53:23 +1100 Subject: [PATCH 129/631] drm: apple: reject truncated firmware dictionary values Dictionary skips must validate payload bounds just like parsed values. Stop at the first error, bound recursive nesting, and propagate invalid container and skipped-value errors to each caller. Reject length truncation and cursor overflow, and preserve the audio look-ahead cursor until a complete key matches. Handle string allocation failure and failed event strings explicitly. These checks follow the existing public dictionary format and do not change valid payloads. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 133 +++++++++++++++++++++++++++------ 1 file changed, 111 insertions(+), 22 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 822138e1b84b8e..f57ab35e09031f 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -87,14 +87,19 @@ static const struct dcp_parse_tag *parse_tag_of_type(struct dcp_parse_ctx *ctx, static int skip_depth(struct dcp_parse_ctx *handle, unsigned int depth) { const struct dcp_parse_tag *tag; - int ret = 0; - int i; + const void *data; + unsigned int count, i; + int ret; + + if (depth >= DCP_PARSE_MAX_DEPTH) + return -EINVAL; if (depth > 64) return -E2BIG; tag = parse_tag(handle); + tag = parse_tag(handle); if (IS_ERR(tag)) return PTR_ERR(tag); @@ -118,7 +123,12 @@ static int skip_depth(struct dcp_parse_ctx *handle, unsigned int depth) return ret; } - return ret; + for (i = 0; i < count; ++i) { + ret = skip_value(handle, depth + 1); + if (ret) + return ret; + } + return 0; case DCP_TYPE_INT64: return PTR_ERR_OR_ZERO(parse_bytes(handle, sizeof(s64))); @@ -133,6 +143,13 @@ static int skip_depth(struct dcp_parse_ctx *handle, unsigned int depth) default: return -EINVAL; } + + return IS_ERR(data) ? PTR_ERR(data) : 0; +} + +static int skip(struct dcp_parse_ctx *handle) +{ + return skip_value(handle, 0); } static int skip(struct dcp_parse_ctx *handle) @@ -156,7 +173,6 @@ static bool consume_string(struct dcp_parse_ctx *ctx, const char *specimen) { const struct dcp_parse_tag *tag; const char *key; - ctx->pos = round_up(ctx->pos, 4); if (ctx->pos + sizeof(*tag) + strlen(specimen) - 1 > ctx->len) return false; @@ -308,6 +324,7 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) if (size > U32_MAX) return -E2BIG; + return -EINVAL; *ctx = (struct dcp_parse_ctx) { .blob = blob, @@ -332,7 +349,11 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, unsigned int fields = 0; int ret = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) { @@ -356,6 +377,18 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, } else { ret = skip(it.handle); } + else if (!strcmp(key, "Active")) + ret = parse_int(it.handle, &dim->active); + else if (!strcmp(key, "Total")) + ret = parse_int(it.handle, &dim->total); + else if (!strcmp(key, "FrontPorch")) + ret = parse_int(it.handle, &dim->front_porch); + else if (!strcmp(key, "SyncWidth")) + ret = parse_int(it.handle, &dim->sync_width); + else if (!strcmp(key, "PreciseSyncRate")) + ret = parse_int(it.handle, &dim->precise_sync_rate); + else + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -420,7 +453,11 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, out->sdr.score = -1; out->best.score = -1; - dcp_parse_foreach_in_array(handle, outer_it) { + ret = iterator_begin(handle, &outer_it, false); + if (ret) + return ret; + + for (; outer_it.idx < outer_it.len; ++outer_it.idx) { struct iterator it; bool is_virtual = true; struct color_mode cmode = { @@ -428,7 +465,11 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, .eotf = -1, .id = -1, .pixel_encoding = -1, .score = -1, }; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) @@ -519,13 +560,17 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->best.score = -1; *score = -1; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) ret = PTR_ERR(key); else if (is_virtual) - skip(it.handle); + ret = skip(it.handle); else if (!strcmp(key, "HorizontalAttributes")) ret = parse_dimension(it.handle, &horiz, false); else if (!strcmp(key, "VerticalAttributes")) @@ -543,7 +588,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "Score")) ret = parse_int(it.handle, score); else - skip(it.handle); + ret = skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -749,6 +794,10 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, ret = iterator_begin(handle, &it, true); if (ret) return ret; + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); @@ -789,7 +838,11 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, *name = ERR_PTR(-ENOENT); *class = ERR_PTR(-ENOENT); - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) { @@ -831,7 +884,7 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, if (!ret) parsed_unit = true; } else { - skip(it.handle); + ret = skip(it.handle); } kfree(key); @@ -951,7 +1004,11 @@ static int parse_chmap(struct dcp_parse_ctx *handle, struct snd_pcm_chmap_elem * chmap->channels = 0; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { for (i = 0; i < ARRAY_SIZE(chan_position_names); i++) if (consume_string(it.handle, chan_position_names[i].label)) break; @@ -979,7 +1036,11 @@ static int parse_chan_layout_element(struct dcp_parse_ctx *handle, int ret; s64 nchans = 0; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(it.handle, "ActiveChannelCount")) ret = parse_int(it.handle, &nchans); else if (consume_string(it.handle, "ChannelLayout")) @@ -1004,7 +1065,11 @@ static int parse_nchans_mask(struct dcp_parse_ctx *handle, unsigned int *mask) *mask = 0; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { int nchans; ret = parse_chan_layout_element(it.handle, &nchans, NULL); @@ -1024,7 +1089,11 @@ static int parse_avep_element(struct dcp_parse_ctx *handle, struct iterator it; int ret; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(handle, "StreamSampleRate")) ret = parse_sample_rate_bit(it.handle, &mask.rates); else if (consume_string(handle, "SampleSize")) @@ -1062,12 +1131,20 @@ static int parse_mode_in_avep_element(struct dcp_parse_ctx *handle, struct dcp_parse_ctx save_handle; int ret; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { if (consume_string(it.handle, "AudioChannelLayoutElements")) { struct iterator inner_it; int nchans; - dcp_parse_foreach_in_array(it.handle, inner_it) { + ret = iterator_begin(it.handle, &inner_it, false); + if (ret) + return ret; + + for (; inner_it.idx < inner_it.len; ++inner_it.idx) { save_handle = *it.handle; ret = parse_chan_layout_element(inner_it.handle, &nchans, NULL); @@ -1120,7 +1197,11 @@ int parse_sound_constraints(struct dcp_parse_ctx *handle, hits->nchans = 0; } - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { ret = parse_avep_element(it.handle, sieve, hits); if (ret < 0) @@ -1139,7 +1220,11 @@ int parse_sound_mode(struct dcp_parse_ctx *handle, struct iterator it; int ret; - dcp_parse_foreach_in_array(handle, it) { + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { save_handle = *it.handle; ret = parse_avep_element(it.handle, sieve, NULL); @@ -1169,7 +1254,11 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn s64 timestamp = -1; bool type_match = false; - dcp_parse_foreach_in_dict(handle, it) { + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); if (IS_ERR(key)) { ret = PTR_ERR(key); @@ -1186,7 +1275,7 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn } else if (!strcmp(key, "timestamp")) { ret = parse_int(it.handle, ×tamp); } else { - skip(it.handle); + ret = skip(it.handle); } if (!IS_ERR_OR_NULL(key)) From 20112e9bd7fd0fa2343595fa74b8278ca339f694 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:55:19 +1100 Subject: [PATCH 130/631] drm: apple: validate firmware timings before creating modes Initialize timing fields as absent and require an active area, total, porches, vertical refresh rate, timing ID and score. Validate each dimension before calculating the clock or narrowing geometry to DRM mode fields. Reject notch underflow, unrepresentable IDs and clocks, and sync intervals beyond the advertised total. The existing public dictionary parser supplies all inputs. Invalid timing entries remain recoverable so a later valid mode can still be advertised; explicitly zero porches remain valid. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index f57ab35e09031f..73cd764226fbed 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -378,15 +378,15 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, ret = skip(it.handle); } else if (!strcmp(key, "Active")) - ret = parse_int(it.handle, &dim->active); + ret = parse_int_bound(it.handle, &dim->active, 0, U16_MAX); else if (!strcmp(key, "Total")) - ret = parse_int(it.handle, &dim->total); + ret = parse_int_bound(it.handle, &dim->total, 0, U16_MAX); else if (!strcmp(key, "FrontPorch")) - ret = parse_int(it.handle, &dim->front_porch); + ret = parse_int_bound(it.handle, &dim->front_porch, 0, U16_MAX); else if (!strcmp(key, "SyncWidth")) - ret = parse_int(it.handle, &dim->sync_width); + ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); else if (!strcmp(key, "PreciseSyncRate")) - ret = parse_int(it.handle, &dim->precise_sync_rate); + ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); else ret = skip(it.handle); @@ -418,6 +418,9 @@ struct color_mode { static int fill_color_mode(struct dcp_color_mode *color, struct color_mode *cmode) { + if (cmode->id < 0 || cmode->id > U32_MAX) + return -EINVAL; + if (color->score >= cmode->score) return 0; @@ -548,6 +551,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dimension horiz = {}, vert = {}; u64 clock; s64 min_vrr = 0, max_vrr = 0; + u64 clock; s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; @@ -649,6 +653,10 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->vrr = true; } + clock = calculate_clock(&horiz, &vert); + if (!clock || clock > INT_MAX) + return -EINVAL; + vert.active -= notch_height; vert.sync_width += notch_height; From 003dc008169b15213a39368e625c6b4dd4d69bc1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 00:57:28 +1100 Subject: [PATCH 131/631] dt-bindings: display: apple,dcp: describe bandwidth argument tuples Constrain each bandwidth property to a single provider reference with its resource index, display register index and, for scratch registers, byte offset. These are the arguments consumed by the existing public DCP driver and supplied by the Apple device trees. Add an example of the existing T8103 internal display resources and piodma child so the binding checks also exercise the argument tuples. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index ac2286832ba7d1..33f1bf6e44359c 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -98,10 +98,21 @@ properties: apple,bw-scratch: $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle of the PMGR bandwidth register provider. + - description: Index into the provider's reg property. + - description: Index in the firmware display register table. + - description: Byte offset of the scratch register within the resource. description: PMGR bandwidth scratch resource, display index and offset. apple,bw-doorbell: $ref: /schemas/types.yaml#/definitions/phandle-array + items: + - items: + - description: Phandle of the PMGR bandwidth register provider. + - description: Index into the provider's reg property. + - description: Index in the firmware display register table. description: Optional PMGR bandwidth doorbell resource and display index. apple,iomfb-surfaces: @@ -269,4 +280,32 @@ allOf: resets: false unevaluatedProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + dcp@231c00000 { + compatible = "apple,t8103-dcp", "apple,dcp"; + reg = <0x2 0x31c00000 0x0 0x4000>, + <0x2 0x30000000 0x0 0x3e8000>, + <0x2 0x31320000 0x0 0x4000>, + <0x2 0x31344000 0x0 0x4000>, + <0x2 0x31800000 0x0 0x800000>, + <0x2 0x3b3d0000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 0>; + apple,bw-scratch = <&pmgr_dcp 0 5 0x14>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + + piodma { + iommus = <&disp_dart 0>; + }; + }; + }; ... From 7b2566ecb1d75b11857c34f3a7ec5885a02b9cd2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:39:03 +1100 Subject: [PATCH 132/631] drm: apple: use the measured H17P late-init input The Linux-side startup record R-IOMFB-startup-run21 observes A000 with four zero input bytes. The public template instead sends one, as required by older firmware. Use zero for H17P while keeping the older variants' late-init requests unchanged. Keep the existing H17G input as well: the record admits only the H17P method profile. This corrects one measured input in the existing Asahi boot chain; it does not qualify the rest of startup or enable another device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index e134915843af27..4fd2221e94ebb2 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -957,8 +957,12 @@ static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u32 v_true = 1; - dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + u32 value = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; +#else + u32 value = 1; +#endif + dcp_late_init_signal(dcp, false, &value, boot_5, NULL); #else dcp_late_init_signal(dcp, false, boot_5, NULL); #endif From ffed7071d0dfc1cfdfe045d591fc39dbc8cf1e49 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:43:41 +1100 Subject: [PATCH 133/631] drm: apple: validate measured H17P callback sizes H17P dispatch currently trusts the firmware's input and output lengths before running typed callback handlers. A short record can make a handler read outside its input or write beyond its reply. Check the envelope and channel stack bounds before reading the header, then require a recorded callback tuple before clearing or dispatching its output. R-IOMFB-startup-run21 supplies the startup and present tuples; R-ANALYTICS supplies the D114 tuple. Stop the H17P interface on a mismatch rather than advancing its callback stack or sending subsequent calls. Older firmware and the existing H17G tuple dispatch stay unchanged. Tuple admission establishes sizes, not callback semantics or full device support. H17P callbacks absent from the measured set fail closed until an interoperability record admits them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 27 ++++++++++++- drivers/gpu/drm/apple/iomfb.h | 2 + drivers/gpu/drm/apple/iomfb_h17p.c | 62 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 76552df9b31165..bff0af1e6bfdd1 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -164,6 +164,9 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) + return; + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && call->tag_h17g[0]) memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); @@ -404,10 +407,29 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, struct device *dev = dcp->dev; struct dcp_packet_header *hdr = data; void *in, *out; - int tag = dcp_parse_tag(hdr->tag); + int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || + offset >= 0x8000 || length > 0x8000 - offset)) { + dev_err(dev, "invalid IOMFB callback envelope\n"); + dcp->crashed = true; + return; + } + + tag = dcp_parse_tag(hdr->tag); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + ((u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || + (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { + dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", + tag, hdr->in_len, hdr->out_len); + dcp->crashed = true; + return; + } + in = data + sizeof(*hdr); out = in + hdr->in_len; @@ -1093,6 +1115,9 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) + return; + if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { /* * H17P reports its interface version (bits 63:48) and a diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 57569d1679027d..19293941f7b5d0 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -207,6 +207,8 @@ struct dcp_method_entry { }; #define IOMFB_MAX_CB (1000) + +bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 28f219b8be14ef..81681b8e66982e 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -7,6 +7,68 @@ #include "iomfb_v13_3.h" #include "version_utils.h" +struct dcp_callback_size { + u32 in_len; + u32 out_len; + bool valid; +}; + +/* R-IOMFB-startup-run21 and R-ANALYTICS Linux boundary observations. */ +static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { + [0] = { 0x0, 0x4, true }, + [1] = { 0x0, 0x4, true }, + [3] = { 0x4, 0x14, true }, + [6] = { 0x54, 0x50, true }, + [100] = { 0x0, 0x0, true }, + [101] = { 0x0, 0x4, true }, + [102] = { 0x44, 0x0, true }, + [104] = { 0x44, 0x0, true }, + [108] = { 0x0, 0x4, true }, + [109] = { 0x0, 0x4, true }, + [110] = { 0x0, 0x4, true }, + [111] = { 0x0, 0x4, true }, + [112] = { 0x0, 0x4, true }, + [113] = { 0x0, 0x4, true }, + [114] = { 0x1044, 0x1004, true }, + [121] = { 0x0, 0x4, true }, + [123] = { 0x0, 0x4, true }, + [125] = { 0x64, 0x24, true }, + [127] = { 0x4, 0x4, true }, + [128] = { 0x1008, 0x4, true }, + [129] = { 0x40, 0x4, true }, + [201] = { 0xc, 0x10, true }, + [206] = { 0x0, 0x4, true }, + [207] = { 0x0, 0x4, true }, + [300] = { 0x10, 0x0, true }, + [400] = { 0x4c, 0xc04, true }, + [401] = { 0x50, 0xc, true }, + [406] = { 0x48, 0x0, true }, + [411] = { 0x10, 0x1c, true }, + [413] = { 0x1048, 0x4, true }, + [414] = { 0x50, 0x4, true }, + [415] = { 0x4c, 0x4, true }, + [451] = { 0x14, 0x1c, true }, + [552] = { 0x1044, 0x4, true }, + [561] = { 0x1044, 0x4, true }, + [563] = { 0x4c, 0x4, true }, + [565] = { 0x48, 0x4, true }, + [574] = { 0x4, 0x8, true }, + [575] = { 0x58, 0x4c, true }, + [582] = { 0x8, 0x4, true }, + [590] = { 0x730, 0x0, true }, + [599] = { 0x0, 0x0, true }, +}; + +bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len) +{ + const struct dcp_callback_size *size; + + if (tag < 0 || tag >= ARRAY_SIZE(callback_sizes)) + return false; + size = &callback_sizes[tag]; + return size->valid && size->in_len == in_len && size->out_len == out_len; +} + struct dcp_h17p_hotplug_request { __le64 connected; u8 tiled_display[0x4c]; From 63312ecbfcdb67bcae96603016e6947c35c618e8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:44:54 +1100 Subject: [PATCH 134/631] drm: apple: answer H17P analytics with an empty dictionary R-ANALYTICS records four complete Linux-side D114 exchanges with a 0x1044-byte request and a 0x1004-byte reply. Every reply begins with the OSDictionary tag 'd' and has zero remaining bytes; the public Asahi OSDictionary decoder accepts it as an empty dictionary. Encode that reply for the measured H17P profile after callback-size validation. Keep the existing H17G response, whose dictionary contract was not measured. This extends the existing Asahi callback table without introducing an arbitrary-call interface or a recovery retry. Byte equivalence to the recorded replies is checked separately from the required two-hour survival gate, which remains to be run on the device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 81681b8e66982e..7555e01c9cc927 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -186,6 +186,19 @@ static bool trampoline_get_frequency_h17p(struct apple_dcp *dcp, int tag, return true; } +static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_zero(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + /* The dispatcher has validated the measured D114 reply size. */ + memset(out, 0, 0x1004); + *(u8 *)out = 'd'; + return true; +} + static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -237,7 +250,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [111] = trampoline_true, /* create_iomfb_service */ [112] = trampoline_create_backlight_service, /* create_backlight_service */ [113] = trampoline_true, /* create_nvram_service */ - [114] = trampoline_zero, /* CoreAnalyticsSendEvent */ + [114] = trampoline_analytics_h17p, /* CoreAnalyticsSendEvent */ [117] = trampoline_nop, /* set_idle_caching_state_ap */ [118] = trampoline_zero, /* upload_trace_start */ [119] = trampoline_zero, /* upload_trace_chunk */ From 9f236700a74ced4bd924f34083af5bc498e30417 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:49:42 +1100 Subject: [PATCH 135/631] drm: apple: serialize H17P presents at the measured boundary The public H17 template starts its surfaces after a 0x58a-byte C swap record and repairs their fields in place. SPEC PRESENT-2 requires a 0x588-byte wire boundary. R-SWAP-controlled-surface-run21 independently locates format, width, height, stride and per-plane dimensions/byte count at offsets matching Asahi's existing surface record at that boundary. Serialize H17P into separate device-owned staging storage, copying the swap prefix, unchanged public surfaces and trailing arguments to their wire positions. Keep the total A408 input at 0xe9c bytes, zero the final two bytes and preserve the native atomic input for repeated encoding. H17G keeps its existing public packing path. Static size/offset checks and randomized byte maps cover every byte of the transfer. Nine populated fields match the controlled Linux capture. This does not establish unmeasured field semantics or qualify resource ownership, formats, backlight or a functional display boot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb_h17p.c | 18 +++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 14 +++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 32 ++++++++++++++++++++++---- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index fd4c3e26c6a410..57c114827b855a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -285,6 +285,8 @@ struct apple_dcp { struct dcp_swap_submit_req_v14_7_0 v14_7_0; struct dcp_swap_submit_req_h17p h17p; } swap; + /* Staging wire record; serialization never mutates the atomic inputs. */ + struct dcp_present_h17p present_h17p; /* swap id of the last completed swap */ u32 last_swap_id; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 7555e01c9cc927..ccc33b70fc3d59 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -88,6 +88,24 @@ static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); +void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, + const struct dcp_swap_submit_req_h17p *request) +{ + size_t tail_offset = offsetof(struct dcp_swap_submit_req_h17p, surf_iova); + size_t tail_size = sizeof(*request) - tail_offset; + + static_assert(sizeof(request->swap) == sizeof(wire->swap) + 2); + static_assert(sizeof(request->surf) == sizeof(wire->surf)); + static_assert(sizeof(*request) - + offsetof(struct dcp_swap_submit_req_h17p, surf_iova) == + sizeof(wire->tail) - 2); + + memcpy(wire->swap, &request->swap, sizeof(wire->swap)); + memcpy(wire->surf, request->surf, sizeof(wire->surf)); + memcpy(wire->tail, (const u8 *)request + tail_offset, tail_size); + memset(wire->tail + tail_size, 0, sizeof(wire->tail) - tail_size); +} + static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 97c0da9e05b940..e9f4dfb6dd8555 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,20 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* The H17P swap boundary is independent of the template's native layout. */ +struct dcp_present_h17p { + u8 swap[0x588]; + struct dcp_surface_h17p surf[SWAP_SURFACES]; + u8 tail[0x64]; +} __packed; + +static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); +static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); +static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); + +void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, + const struct dcp_swap_submit_req_h17p *request); + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 4fd2221e94ebb2..3cf0096997b70b 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -61,9 +61,26 @@ IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); IOMFB_THUNK_INOUT(abort_swaps_dcp); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, + struct dcp_swap_submit_req_h17p *request, + dcp_callback_t cb, void *cookie) +{ + const void *data = request; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_serialize_present_h17p(&dcp->present_h17p, request); + data = &dcp->present_h17p; + } + + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), + sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); +} +#else DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), struct DCP_FW_NAME(dcp_swap_submit_resp)); +#endif DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, struct dcp_swap_start_resp); @@ -1212,9 +1229,10 @@ static void release_swap_cookie(struct kref *ref) * format two bytes late and its surface check rejects the swap as an * unsupported format. * - * Shift each surface's tail down by two rather than fork the shared struct. + * H17G retains this public packing path. The measured H17P profile uses + * a separate wire record with the SPEC's 0x588 swap boundary. */ -static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); unsigned int i; @@ -1234,6 +1252,10 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) req->swap.h17p_pre_swap_id[0x00] = 2; req->swap.h17p_pre_swap_id[0x28] = 3; + /* The measured H17P layout is serialized without shifting its surfaces. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; + for (i = 0; i < SWAP_SURFACES; i++) { u8 *p = (u8 *)&req->surf[i]; size_t n = sizeof(req->surf[i]); @@ -1263,7 +1285,7 @@ static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) } } #else -static void dcp_h17p_fix_swap_surfaces(struct apple_dcp *dcp) { } +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) @@ -1311,7 +1333,7 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, struct dcp_swap_cookie *info = cookie; info->swap_id = resp->swap_id; - dcp_h17p_fix_swap_surfaces(dcp); + dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -1789,7 +1811,7 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); - dcp_h17p_fix_swap_surfaces(dcp); + dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } From b088954367da2210bd7b500deb1b763d12a50ad7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 03:56:32 +1100 Subject: [PATCH 136/631] drm: apple: require accepted H17P present IDs before retirement R-SWAP's controlled captures correlate the A406-assigned ID, A408 input and D590 completion. Every retained run21 completion follows its matching submit acknowledgment. The existing completion handler retires buffers and signals a flip without checking whether that ID was accepted. Track one started and accepted H17P present in the RTKit receive thread. Reject overlapping starts, inconsistent submit state, early completion, wrong IDs and duplicate completion before touching framebuffer retirement or DRM flip state. Stop the interface on protocol inconsistency. Rejected submits cancel the reservation while the existing retention code unarms their references; a later unsolicited completion cannot retire them. Apply the same policy to ordinary and clear swaps, preserving older firmware and H17G behavior. This is a lifecycle guard, not a substitute for the outbound transaction queue or hardware lifetime qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 6 ++-- drivers/gpu/drm/apple/iomfb_h17p.h | 40 +++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 45 ++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 57c114827b855a..5be7ecbe6cffcf 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -285,8 +285,6 @@ struct apple_dcp { struct dcp_swap_submit_req_v14_7_0 v14_7_0; struct dcp_swap_submit_req_h17p h17p; } swap; - /* Staging wire record; serialization never mutates the atomic inputs. */ - struct dcp_present_h17p present_h17p; /* swap id of the last completed swap */ u32 last_swap_id; @@ -482,6 +480,10 @@ struct apple_dcp { struct delayed_work hdmi_recheck_wq; /* T6030 HDMI converter service requests, see dcp-hdmi.c */ struct dcp_hdmi *hdmi; + + /* Staging wire record; serialization never mutates the atomic inputs. */ + struct dcp_present_h17p present_h17p; + struct dcp_present_state_h17p present_state_h17p; }; void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index e9f4dfb6dd8555..2b260ebb2c86b1 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -16,6 +16,46 @@ static_assert(sizeof(struct dcp_surface_h17p) == 0x22c); static_assert(sizeof(struct dcp_swap_submit_req_h17p) == 0xe9c); static_assert(sizeof(struct dcp_swap_submit_resp_h17p) == 0x0c); +/* Owned by the RTKit receive thread, independently of DRM state lifetime. */ +struct dcp_present_state_h17p { + u32 swap_id; + bool pending; + bool accepted; +}; + +static inline bool +dcp_present_begin_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (state->pending) + return false; + state->swap_id = swap_id; + state->pending = true; + state->accepted = false; + return true; +} + +static inline bool +dcp_present_submit_h17p(struct dcp_present_state_h17p *state, u32 swap_id, + bool accepted) +{ + if (!state->pending || state->accepted || state->swap_id != swap_id) + return false; + state->accepted = accepted; + if (!accepted) + state->pending = false; + return true; +} + +static inline bool +dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || !state->accepted || state->swap_id != swap_id) + return false; + state->pending = false; + state->accepted = false; + return true; +} + /* The H17P swap boundary is independent of the template's native layout. */ struct dcp_present_h17p { u8 swap[0x588]; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 3cf0096997b70b..672fc09c4b6bc1 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -241,6 +241,15 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); + dcp->crashed = true; + return; + } +#endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; dcp_swap_watchdog_complete(dcp); @@ -1288,6 +1297,32 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif +static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_begin_h17p(&dcp->present_state_h17p, swap_id)) { + dev_err(dcp->dev, "overlapping present %u\n", swap_id); + dcp->crashed = true; + return false; + } +#endif + return true; +} + +static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_submit_h17p(&dcp->present_state_h17p, swap_id, accepted)) { + dev_err(dcp->dev, "unexpected present submission %u\n", swap_id); + dcp->crashed = true; + return false; + } +#endif + return true; +} + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -1775,6 +1810,12 @@ TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_se static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + u32 status = resp ? resp->ret : ~0U; +#else + u32 status = resp->ret; +#endif if (resp->ret) { dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); @@ -1807,6 +1848,10 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); return; } + if (!dcp_present_begin(dcp, resp->swap_id)) { + dcp_drm_crtc_vblank(dcp->crtc); + return; + } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; dcp_arm_retained_framebuffers(dcp, resp->swap_id); From 91a5601429241c7560ca582207384b9734e687ec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:23:57 +1100 Subject: [PATCH 137/631] drm: apple: serialize H17P command and present transactions RTKit replies can issue nested calls while atomic commits and power requests arrive from other threads. Channel depth alone cannot distinguish those callers, and preparing a second present can overwrite the request still awaiting the first submit reply. Queue H17P top-level operations in FIFO order and serialize transmit preparation with the receive thread. Copy standalone command inputs and hold atomic state through the reply chain and matching present completion. Keep nested calls in their originating transaction. A submit ACK alone does not let the next operation reuse the present record. Bound the pending queue and stop new operations on a ten-second timeout. Keep the active operation pinned on failure because a timeout does not prove firmware has stopped using it. An expired watchdog from a previous operation checks the new deadline before reporting a timeout. Older firmware and the H17G method profile keep their existing transport. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 26 +++ drivers/gpu/drm/apple/dcp.c | 2 + drivers/gpu/drm/apple/iomfb.c | 292 ++++++++++++++++++++++++++- 3 files changed, 317 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5be7ecbe6cffcf..9b25f1c646f62d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -34,6 +34,14 @@ struct apple_dcp_afkep; +/* One operation, including all of its nested replies and present completion. */ +struct iomfb_transaction { + struct list_head link; + void (*start)(struct apple_dcp *dcp, struct iomfb_transaction *transaction); + void (*release)(struct iomfb_transaction *transaction); +}; +struct apple_dcp_typec_port; + void dcp_swap_watchdog_arm(struct apple_dcp *dcp); void dcp_swap_watchdog_complete(struct apple_dcp *dcp); void dcp_retry_placeholder_edid(struct apple_dcp *dcp, @@ -484,8 +492,26 @@ struct apple_dcp { /* Staging wire record; serialization never mutates the atomic inputs. */ struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; + + /* Serializes H17P transmit preparation with RTKit receive callbacks. */ + struct { + /* Protects queue, channel stacks, and the current owner. */ + struct mutex lock; + struct list_head pending; + struct iomfb_transaction *active; + struct task_struct *owner; + struct work_struct work; + struct delayed_work timeout; + unsigned long deadline; + unsigned int queued; + bool stopped; + } iomfb; }; +void iomfb_queue_init(struct apple_dcp *dcp); +void iomfb_queue_stop(struct apple_dcp *dcp); +int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); + void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a146bdc8cf970d..2c547471055e25 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2688,6 +2688,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->shmem) iomfb_shutdown(dcp); + iomfb_queue_stop(dcp); dcp_release_piodma_iommu_dev(dcp); devm_clk_put(dev, dcp->clk); @@ -2794,6 +2795,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); spin_lock_init(&dcp->backlight.lock); + iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext"); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index bff0af1e6bfdd1..0c00c1e7d3e5e3 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -157,6 +158,195 @@ static u8 dcp_pop_depth(u8 *depth) return --(*depth); } +/* Older firmware and the H17G method profile retain their existing transport. */ +static bool iomfb_uses_queue(struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; +} + +static bool iomfb_channels_idle(struct apple_dcp *dcp) +{ + return !dcp->ch_cmd.depth && !dcp->ch_cb.depth && + !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && + !dcp->ch_async.depth && !dcp->ch_oobasync.depth; +} + +static void iomfb_discard_pending(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction, *next; + + lockdep_assert_held(&dcp->iomfb.lock); + list_for_each_entry_safe(transaction, next, &dcp->iomfb.pending, link) { + list_del(&transaction->link); + dcp->iomfb.queued--; + transaction->release(transaction); + } +} + +static void iomfb_queue_advance(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + lockdep_assert_held(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + iomfb_discard_pending(dcp); + /* The active operation may still be visible to firmware. */ + return; + } + if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending) + return; + + if (transaction) { + dcp->iomfb.active = NULL; + cancel_delayed_work(&dcp->iomfb.timeout); + transaction->release(transaction); + } + if (!list_empty(&dcp->iomfb.pending)) + schedule_work(&dcp->iomfb.work); +} + +static void iomfb_queue_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, iomfb.work); + struct iomfb_transaction *transaction; + + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + iomfb_discard_pending(dcp); + goto unlock; + } + if (dcp->iomfb.active || !iomfb_channels_idle(dcp) || + dcp->present_state_h17p.pending || list_empty(&dcp->iomfb.pending)) + goto unlock; + + transaction = list_first_entry(&dcp->iomfb.pending, + struct iomfb_transaction, link); + list_del(&transaction->link); + dcp->iomfb.queued--; + dcp->iomfb.active = transaction; + WRITE_ONCE(dcp->iomfb.owner, current); + dcp->iomfb.deadline = jiffies + msecs_to_jiffies(10000); + mod_delayed_work(system_wq, &dcp->iomfb.timeout, msecs_to_jiffies(10000)); + transaction->start(dcp, transaction); + WRITE_ONCE(dcp->iomfb.owner, NULL); + iomfb_queue_advance(dcp); +unlock: + mutex_unlock(&dcp->iomfb.lock); +} + +static void iomfb_queue_timeout(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, iomfb.timeout); + + mutex_lock(&dcp->iomfb.lock); + if (dcp->iomfb.active && time_before(jiffies, dcp->iomfb.deadline)) { + mod_delayed_work(system_wq, &dcp->iomfb.timeout, + dcp->iomfb.deadline - jiffies); + } else if (dcp->iomfb.active) { + WRITE_ONCE(dcp->crashed, true); + dev_err(dcp->dev, "IOMFB transaction timed out\n"); + iomfb_discard_pending(dcp); + schedule_work(&dcp->vblank_wq); + } + mutex_unlock(&dcp->iomfb.lock); +} + +void iomfb_queue_init(struct apple_dcp *dcp) +{ + mutex_init(&dcp->iomfb.lock); + INIT_LIST_HEAD(&dcp->iomfb.pending); + INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); + INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); +} + +void iomfb_queue_stop(struct apple_dcp *dcp) +{ + if (!iomfb_uses_queue(dcp)) + return; + + mutex_lock(&dcp->iomfb.lock); + dcp->iomfb.stopped = true; + iomfb_discard_pending(dcp); + mutex_unlock(&dcp->iomfb.lock); + cancel_work_sync(&dcp->iomfb.work); + cancel_delayed_work_sync(&dcp->iomfb.timeout); +} + +int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) +{ + int ret = 0; + + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + ret = -EIO; + } else if (dcp->iomfb.queued >= 32) { + ret = -EBUSY; + } else { + dcp->iomfb.queued++; + list_add_tail(&transaction->link, &dcp->iomfb.pending); + schedule_work(&dcp->iomfb.work); + } + mutex_unlock(&dcp->iomfb.lock); + return ret; +} + +struct iomfb_command { + struct iomfb_transaction transaction; + struct dcp_method_entry method; + dcp_callback_t callback; + void *cookie; + u32 in_len; + u32 out_len; + bool oob; + u8 data[]; +}; + +static void iomfb_command_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_command *command = container_of(transaction, + struct iomfb_command, transaction); + + dcp_push(dcp, command->oob, &command->method, command->in_len, + command->out_len, command->data, command->callback, command->cookie); +} + +static void iomfb_command_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_command, transaction)); +} + +static int iomfb_queue_command(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *method, + u32 in_len, u32 out_len, void *data, + dcp_callback_t callback, void *cookie) +{ + struct iomfb_command *command; + int ret; + + if ((u64)sizeof(struct dcp_packet_header) + in_len + out_len > 0x8000) + return -EMSGSIZE; + command = kzalloc(struct_size(command, data, in_len), GFP_KERNEL); + if (!command) + return -ENOMEM; + command->transaction.start = iomfb_command_start; + command->transaction.release = iomfb_command_release; + command->method = *method; + command->callback = callback; + command->cookie = cookie; + command->in_len = in_len; + command->out_len = out_len; + command->oob = oob; + if (in_len) + memcpy(command->data, data, in_len); + ret = iomfb_queue(dcp, &command->transaction); + if (ret) + iomfb_command_release(&command->transaction); + return ret; +} + /* Call a DCP function given by a tag */ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, @@ -164,6 +354,14 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca { struct dcp_method_entry resolved = *call; + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ + if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { + if (iomfb_queue_command(dcp, oob, call, in_len, out_len, + data, cb, cookie)) + WRITE_ONCE(dcp->crashed, true); + return; + } + if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) return; @@ -186,8 +384,21 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca .tag[3] = call->tag[0], }; - u8 depth = dcp_push_depth(&ch->depth); - u16 offset = dcp_packet_start(ch, depth); + u8 depth; + u16 offset; + + if (iomfb_uses_queue(dcp) && + (!ch || ch->depth >= DCP_MAX_CALL_DEPTH || + (u64)sizeof(header) + in_len + out_len > 0x8000 || + dcp_packet_start(ch, ch->depth) > + 0x8000 - ALIGN(sizeof(header) + in_len + out_len, + DCP_PACKET_ALIGNMENT))) { + dev_err(dcp->dev, "invalid IOMFB command envelope\n"); + WRITE_ONCE(dcp->crashed, true); + return; + } + depth = dcp_push_depth(&ch->depth); + offset = dcp_packet_start(ch, depth); void *out = dcp->shmem + dcp_tx_offset(context) + offset; void *out_data = out + sizeof(header); @@ -1020,6 +1231,53 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, } +struct iomfb_atomic_transaction { + struct iomfb_transaction transaction; + struct drm_atomic_state *state; + struct drm_crtc *crtc; +}; + +static void iomfb_atomic_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_atomic_transaction *atomic = container_of(transaction, + struct iomfb_atomic_transaction, transaction); + + if (dcp->valid_mode && dcp->connector && dcp->connector->connected) + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); + else + schedule_work(&dcp->vblank_wq); +} + +static void iomfb_atomic_release(struct iomfb_transaction *transaction) +{ + struct iomfb_atomic_transaction *atomic = container_of(transaction, + struct iomfb_atomic_transaction, transaction); + + drm_atomic_state_put(atomic->state); + kfree(atomic); +} + +static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct iomfb_atomic_transaction *atomic; + + atomic = kzalloc_obj(*atomic); + if (!atomic) + goto failed; + atomic->transaction.start = iomfb_atomic_start; + atomic->transaction.release = iomfb_atomic_release; + atomic->state = drm_atomic_state_get(state); + atomic->crtc = crtc; + if (!iomfb_queue(dcp, &atomic->transaction)) + return; + iomfb_atomic_release(&atomic->transaction); +failed: + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); +} + void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct platform_device *pdev = to_apple_crtc(crtc)->dcp; @@ -1031,6 +1289,8 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) } if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { iomfb_v14_7_flush(dcp, crtc, state); + if (iomfb_uses_queue(dcp)) { + iomfb_queue_atomic(dcp, crtc, state); return; } @@ -1111,7 +1371,7 @@ bool dcp_is_initialized(struct platform_device *pdev) return dcp->active; } -void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +static void iomfb_recv_message(struct apple_dcp *dcp, u64 message) { enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); @@ -1136,11 +1396,37 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) dev_warn(dcp->dev, "Ignoring unknown message %llx\n", message); } +void iomfb_recv_msg(struct apple_dcp *dcp, u64 message) +{ + if (!iomfb_uses_queue(dcp)) { + iomfb_recv_message(dcp, message); + return; + } + + mutex_lock(&dcp->iomfb.lock); + WRITE_ONCE(dcp->iomfb.owner, current); + if (!dcp->iomfb.stopped) + iomfb_recv_message(dcp, message); + WRITE_ONCE(dcp->iomfb.owner, NULL); + iomfb_queue_advance(dcp); + mutex_unlock(&dcp->iomfb.lock); +} + int iomfb_start_rtkit(struct apple_dcp *dcp) { dma_addr_t shmem_iova; int ret; + if (iomfb_uses_queue(dcp)) { + mutex_lock(&dcp->iomfb.lock); + if (dcp->iomfb.active) { + mutex_unlock(&dcp->iomfb.lock); + return -EBUSY; + } + dcp->iomfb.stopped = false; + mutex_unlock(&dcp->iomfb.lock); + } + /* * H17P firmware expects the remote allocator endpoint to be started * before IOMFB. A firmware subsystem whose endpoint was never started From d2b5d991ded5c3fe74263e0f7f3d1f9132fffaec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:33:56 +1100 Subject: [PATCH 138/631] drm: apple: pin the last completed H17P scanout A rejected atomic replacement can leave DRM state referring to a surface that the display never accepted. Re-presenting that state for a backlight update would bypass the previous submit failure and retire the wrong framebuffers. Keep an independent snapshot with framebuffer references for each accepted surface. Stage replacements from the atomic operation and publish them only after the accepted submit's matching completion. Merge unchanged planes and preserve the background on partial updates. Rejected operations release only their staged snapshot, leaving the completed scanout pinned. Retain the snapshot on soft-off or firmware failure. It supplies no proof of a final DMA stop, and rebinding an unconfirmed session is refused. Older firmware and the H17G method profile retain their existing path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 9 +++ drivers/gpu/drm/apple/iomfb.c | 117 ++++++++++++++++++++++++++- 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 9b25f1c646f62d..44e17264e6ea1e 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -34,6 +34,12 @@ struct apple_dcp_afkep; +/* Snapshot of a completed present, independent of the latest DRM state. */ +struct iomfb_scanout_h17p { + struct dcp_swap_submit_req_h17p request; + struct drm_framebuffer *fb[SWAP_SURFACES]; +}; + /* One operation, including all of its nested replies and present completion. */ struct iomfb_transaction { struct list_head link; @@ -505,9 +511,12 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + struct iomfb_scanout_h17p *scanout; + struct iomfb_scanout_h17p *next_scanout; } iomfb; }; +void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 0c00c1e7d3e5e3..e4931ae805b57a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1231,10 +1231,105 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, } +static void iomfb_scanout_release_h17p(struct iomfb_scanout_h17p *scanout) +{ + unsigned int i; + + if (!scanout) + return; + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + drm_framebuffer_put(scanout->fb[i]); + kfree(scanout); +} + +static struct iomfb_scanout_h17p * +iomfb_scanout_prepare_h17p(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct iomfb_scanout_h17p *scanout; + struct drm_plane *plane; + struct drm_plane_state *old_state, *new_state; + unsigned int i, slot; + int index; + + lockdep_assert_held(&dcp->iomfb.lock); + scanout = kzalloc_obj(*scanout); + if (!scanout) + return NULL; + if (dcp->iomfb.scanout) { + scanout->request = dcp->iomfb.scanout->request; + for (i = 0; i < SWAP_SURFACES; i++) { + scanout->fb[i] = dcp->iomfb.scanout->fb[i]; + if (scanout->fb[i]) + drm_framebuffer_get(scanout->fb[i]); + } + } + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, index) { + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + slot = to_apple_plane(plane)->iomfb_surf; + if (slot >= SWAP_SURFACES) { + iomfb_scanout_release_h17p(scanout); + return NULL; + } + if (scanout->fb[slot]) + drm_framebuffer_put(scanout->fb[slot]); + scanout->fb[slot] = NULL; + if (new_state->crtc == crtc && new_state->visible && new_state->fb) { + scanout->fb[slot] = new_state->fb; + drm_framebuffer_get(new_state->fb); + } + } + return scanout; +} + +void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) +{ + struct iomfb_scanout_h17p *scanout = dcp->iomfb.next_scanout; + struct iomfb_scanout_h17p *previous = dcp->iomfb.scanout; + struct dcp_swap_submit_req_h17p *request = &dcp->swap.h17p; + unsigned int i; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!scanout) + return; + + /* Keep common present arguments, including the null output pointers. */ + scanout->request = *request; + /* Preserve unchanged planes when only a subset was presented. */ + for (i = 0; i < SWAP_SURFACES; i++) { + if (!(request->swap.swap_enabled & BIT(i)) && previous) { + scanout->request.surf[i] = previous->request.surf[i]; + scanout->request.surf_iova[i] = previous->request.surf_iova[i]; + scanout->request.swap.src_rect[i] = previous->request.swap.src_rect[i]; + scanout->request.swap.dst_rect[i] = previous->request.swap.dst_rect[i]; + scanout->request.swap.surf_ids[i] = previous->request.swap.surf_ids[i]; + scanout->request.swap.surf_flags[i] = previous->request.swap.surf_flags[i]; + scanout->request.swap.surf_unk[i] = previous->request.swap.surf_unk[i]; + } + scanout->request.surf_null[i] = !scanout->fb[i]; + } + /* Brightness re-presents all pinned surfaces with a fresh swap ID. */ + scanout->request.swap.swap_enabled = IOMFB_SET_BACKGROUND; + for (i = 0; i < SWAP_SURFACES; i++) + if (scanout->fb[i]) + scanout->request.swap.swap_enabled |= BIT(i); + scanout->request.swap.swap_completed = scanout->request.swap.swap_enabled; + if (!(request->swap.swap_enabled & IOMFB_SET_BACKGROUND) && previous) + scanout->request.swap.bg_color = previous->request.swap.bg_color; + dcp->iomfb.scanout = scanout; + dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(previous); +} + struct iomfb_atomic_transaction { struct iomfb_transaction transaction; struct drm_atomic_state *state; struct drm_crtc *crtc; + struct apple_dcp *dcp; + struct iomfb_scanout_h17p *scanout; }; static void iomfb_atomic_start(struct apple_dcp *dcp, @@ -1243,10 +1338,18 @@ static void iomfb_atomic_start(struct apple_dcp *dcp, struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); - if (dcp->valid_mode && dcp->connector && dcp->connector->connected) - iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); - else + if (!dcp->valid_mode || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); + return; + } + atomic->scanout = iomfb_scanout_prepare_h17p(dcp, atomic->crtc, atomic->state); + if (!atomic->scanout) { + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); + return; + } + dcp->iomfb.next_scanout = atomic->scanout; + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); } static void iomfb_atomic_release(struct iomfb_transaction *transaction) @@ -1254,6 +1357,11 @@ static void iomfb_atomic_release(struct iomfb_transaction *transaction) struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); + if (atomic->scanout && atomic->dcp->iomfb.scanout != atomic->scanout) { + if (atomic->dcp->iomfb.next_scanout == atomic->scanout) + atomic->dcp->iomfb.next_scanout = NULL; + iomfb_scanout_release_h17p(atomic->scanout); + } drm_atomic_state_put(atomic->state); kfree(atomic); } @@ -1270,6 +1378,7 @@ static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, atomic->transaction.release = iomfb_atomic_release; atomic->state = drm_atomic_state_get(state); atomic->crtc = crtc; + atomic->dcp = dcp; if (!iomfb_queue(dcp, &atomic->transaction)) return; iomfb_atomic_release(&atomic->transaction); @@ -1419,7 +1528,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) if (iomfb_uses_queue(dcp)) { mutex_lock(&dcp->iomfb.lock); - if (dcp->iomfb.active) { + if (dcp->iomfb.active || dcp->iomfb.scanout) { mutex_unlock(&dcp->iomfb.lock); return -EBUSY; } From 729ad6986dfecd66a62ffa41926b84d154015184 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:37:53 +1100 Subject: [PATCH 139/631] drm: apple: encode H17P commanded nits without floating point Linux-side captures during ordinary backlight sysfs changes show integer commanded levels in a little-endian binary64 field at offset 0x35e of the present record. Controlled levels of 2, 36, 140 and 400 nits produce those exact bytes while a retained framebuffer's blend CRC remains constant. Encode integer nits by constructing the exponent and mantissa, avoiding floating-point operations in the kernel. Write only the measured eight bytes in the staging record. This helper does not enable the backlight or assign meaning to surrounding calibration and control fields. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 18 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.h | 2 ++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index ccc33b70fc3d59..b49a587f344720 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -1,6 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT /* Copyright The Asahi Linux Contributors */ +#include +#include + #include "dcp-link.h" #include "iomfb_h17p.h" #include "iomfb_v12_3.h" @@ -106,6 +109,21 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, memset(wire->tail + tail_size, 0, sizeof(wire->tail) - tail_size); } +/* Integer commanded nits are exact binary64 values; no kernel FP is needed. */ +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) +{ + u64 value = 0; + unsigned int exponent; + + if (nits) { + exponent = fls(nits) - 1; + value = (u64)(exponent + 1023) << 52; + value |= ((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0); + } + /* Measured from Linux presents during ordinary sysfs brightness changes. */ + put_unaligned_le64(value, wire->swap + 0x35e); +} + static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { IOMFB_METHOD("A000", dcpep_late_init_signal), IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 2b260ebb2c86b1..0121d334c214ad 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,6 +67,8 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits); + void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); From efdb8dc877546c8abb5744f3cfd63c0f62adc8bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:46:59 +1100 Subject: [PATCH 140/631] drm: apple: couple H17P backlight updates to completed presents Backlight updates need a valid surface even on a static screen or while DRM considers the internal CRTC inactive. Use the transaction queue to re-present the pinned, completed scanout. Coalesce brightness requests and reserve the policy sequence when preparing the wire record. Update cached actual nits only after accepted submission and its matching completion. Rejection cancels the reservation without updating actual. Brightness-only completions neither arm retirement nor release displaced framebuffers or signal another commit's flip event. Requests made during a successful present remain pending for the next operation; rejection does not create an automatic firmware retry loop. Expose a configuration hook requiring admitted range and takeover inputs; do not enable it with guessed values. Internal H17P DPMS preserves the pipe and requests zero or the retained target, reporting unavailable policy until configured. Preserve older firmware and H17G handling. Prepare any external H17P clear record inside the serialized reply chain, so a power request cannot overwrite a present still in flight. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 15 ++++ drivers/gpu/drm/apple/dcp.c | 10 ++- drivers/gpu/drm/apple/dcp_backlight.c | 3 + drivers/gpu/drm/apple/iomfb.c | 120 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 99 +++++++++++++------- drivers/gpu/drm/apple/iomfb_template.h | 4 +- 6 files changed, 216 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 44e17264e6ea1e..f0a198df158b7f 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -45,6 +45,10 @@ struct iomfb_transaction { struct list_head link; void (*start)(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void (*release)(struct iomfb_transaction *transaction); + struct dcp_backlight_present backlight; + bool backlight_reserved; + bool brightness_only; + bool completed; }; struct apple_dcp_typec_port; @@ -511,12 +515,23 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + bool backlight_queued; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; } iomfb; }; void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); +void iomfb_present_backlight_h17p(struct apple_dcp *dcp); +bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); +bool iomfb_present_complete_h17p(struct apple_dcp *dcp); +void iomfb_present_failed_h17p(struct apple_dcp *dcp); +bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, + const struct dcp_swap_submit_req_h17p *request, + struct dcp_present_h17p *wire); +int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2c547471055e25..5938791ac38782 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1657,6 +1657,12 @@ int dcp_typec_prepare_route(struct apple_dcp *dcp) if (generation != dcp->typec_generation || !dcp->typec_cable_connected) return -ESTALE; return ready ? 0 : -ETIMEDOUT; +static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) +{ + return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || + (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + dcp->connector_type == DRM_MODE_CONNECTOR_eDP); } void dcp_poweron(struct platform_device *pdev) @@ -1666,7 +1672,7 @@ void dcp_poweron(struct platform_device *pdev) unsigned long remaining; int ret; - if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, true); if (ret) dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); @@ -1744,7 +1750,7 @@ void dcp_poweroff(struct platform_device *pdev) int ret; /* Internal H17P DPMS is a brightness present, never a pipe stop. */ - if (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) { + if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); if (ret) dev_warn(dcp->dev, "backlight blank unavailable: %d\n", ret); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index d16a0a3b124f1b..19ca88e9fa5bdb 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -177,6 +177,9 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, spin_lock_irqsave(&dcp->backlight.lock, flags); ret = dcp_bl_prepare(&dcp->backlight.state, have_surface, present); + /* No reservation: carry the last completed level on an ordinary present. */ + if (ret == -EALREADY) + present->nits = dcp->backlight.state.actual; spin_unlock_irqrestore(&dcp->backlight.lock, flags); return ret; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e4931ae805b57a..d1eacc3d293b9d 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -180,6 +180,8 @@ static void iomfb_discard_pending(struct apple_dcp *dcp) list_for_each_entry_safe(transaction, next, &dcp->iomfb.pending, link) { list_del(&transaction->link); dcp->iomfb.queued--; + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; transaction->release(transaction); } } @@ -200,6 +202,11 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) if (transaction) { dcp->iomfb.active = NULL; cancel_delayed_work(&dcp->iomfb.timeout); + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = false; + /* A new level requested during a successful present stays pending. */ + if (transaction->completed && dcp_backlight_pending(dcp)) + schedule_work(&dcp->bl_update_wq); transaction->release(transaction); } if (!list_empty(&dcp->iomfb.pending)) @@ -281,9 +288,13 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) mutex_lock(&dcp->iomfb.lock); if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { ret = -EIO; + } else if (transaction->brightness_only && dcp->iomfb.backlight_queued) { + ret = -EALREADY; } else if (dcp->iomfb.queued >= 32) { ret = -EBUSY; } else { + if (transaction->brightness_only) + dcp->iomfb.backlight_queued = true; dcp->iomfb.queued++; list_add_tail(&transaction->link, &dcp->iomfb.pending); schedule_work(&dcp->iomfb.work); @@ -1324,6 +1335,115 @@ void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) iomfb_scanout_release_h17p(previous); } +bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp) +{ + return dcp->iomfb.active && dcp->iomfb.active->brightness_only; +} + +bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, + const struct dcp_swap_submit_req_h17p *request, + struct dcp_present_h17p *wire) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + struct dcp_backlight_present present; + bool have_surface = false; + unsigned int i; + int ret; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!dcp_backlight_active(dcp)) + return true; + if (!transaction) + return false; + for (i = 0; i < SWAP_SURFACES; i++) + have_surface |= !request->surf_null[i]; + ret = dcp_backlight_prepare(dcp, have_surface, &present); + if (ret && ret != -EALREADY) + return false; + if (!ret) { + transaction->backlight = present; + transaction->backlight_reserved = true; + } + iomfb_encode_backlight_h17p(wire, present.nits); + return true; +} + +void iomfb_present_failed_h17p(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + if (transaction && transaction->backlight_reserved) { + dcp_backlight_complete(dcp, transaction->backlight.sequence, false); + transaction->backlight_reserved = false; + } +} + +bool iomfb_present_complete_h17p(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction = dcp->iomfb.active; + + lockdep_assert_held(&dcp->iomfb.lock); + if (transaction) { + transaction->completed = true; + if (transaction->backlight_reserved) { + dcp_backlight_complete(dcp, transaction->backlight.sequence, true); + transaction->backlight_reserved = false; + } + if (transaction->brightness_only) + return false; + } + iomfb_scanout_complete_h17p(dcp); + return true; +} + +static void iomfb_backlight_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct iomfb_scanout_h17p *scanout = dcp->iomfb.scanout; + bool have_surface = false; + unsigned int i; + + if (!dcp_backlight_pending(dcp) || !dcp->valid_mode || !scanout || + !dcp->connector || !dcp->connector->connected) + return; + for (i = 0; i < SWAP_SURFACES; i++) + have_surface |= !!scanout->fb[i]; + if (!have_surface) + return; + + dcp->swap.h17p = scanout->request; + iomfb_present_backlight_h17p(dcp); +} + +static void iomfb_backlight_release(struct iomfb_transaction *transaction) +{ + kfree(transaction); +} + +static void iomfb_backlight_kick(struct apple_dcp *dcp) +{ + struct iomfb_transaction *transaction; + + transaction = kzalloc_obj(*transaction); + if (!transaction) + return; + transaction->start = iomfb_backlight_start; + transaction->release = iomfb_backlight_release; + transaction->brightness_only = true; + if (iomfb_queue(dcp, transaction)) + kfree(transaction); +} + +int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, + bool inherited_valid, u32 inherited, + bool default_valid, u32 default_nits) +{ + if (!iomfb_uses_queue(dcp)) + return -EINVAL; + return dcp_backlight_configure(dcp, maximum, inherited_valid, inherited, + default_valid, default_nits, iomfb_backlight_kick); +} + struct iomfb_atomic_transaction { struct iomfb_transaction transaction; struct drm_atomic_state *state; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 672fc09c4b6bc1..b607465b8acb55 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -70,6 +70,11 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { iomfb_serialize_present_h17p(&dcp->present_h17p, request); + if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { + if (cb) + cb(dcp, NULL, cookie); + return; + } data = &dcp->present_h17p; } @@ -1297,6 +1302,23 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } #endif +static bool dcp_present_retires(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return !iomfb_present_brightness_only_h17p(dcp); +#endif + return true; +} + +static void dcp_present_failed(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_present_failed_h17p(dcp); +#endif +} + static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER @@ -1323,6 +1345,43 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted return true; } +static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) +{ + typeof(DCP_FW_UNION(dcp->swap)) *swap = request; + + /* Clear surfaces. */ + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + if (dcp_has_panel(dcp)) { + swap->swap.bl_unk = 1; + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + } + + /* Null all surfaces */ + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VERSION(13, 2, 0) <= DCP_FW_VER +#if DCP_FW_VERSION(26, 0, 0) > DCP_FW_VER + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; +#endif + swap->unknown_pointer_null = true; + swap->unknown_output_null = true; +#endif +} + static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; @@ -1362,7 +1421,8 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, { struct dcp_swap_start_resp *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_retained_framebuffers(dcp, resp->swap_id); + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); if (cookie) { struct dcp_swap_cookie *info = cookie; @@ -1521,35 +1581,11 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); - // clear surfaces - memset(swap, 0, sizeof(*swap)); - - swap->swap.swap_enabled = - swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; - swap->swap.bg_color = 0xFF000000; - - /* - * Turn off the backlight. This matters because the DCP's idea of - * backlight brightness gets desynced after a power change, and it - * needs to be told it's going to turn off so it will consider the - * subsequent update on poweron an actual change and restore the - * brightness. - */ - if (dcp_has_panel(dcp)) { - swap->swap.bl_unk = 1; - swap->swap.bl_value = 0; - swap->swap.bl_power = 0; - } - - /* Null all surfaces */ - for (int l = 0; l < SWAP_SURFACES; l++) - swap->surf_null[l] = true; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - for (int l = 0; l < 5; l++) - swap->surf2_null[l] = true; -#endif - swap->unkU32Ptr_null = true; - swap->unkU32out_null = true; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + dcp_prepare_clear_swap(dcp, swap); +#else + dcp_prepare_clear_swap(dcp, swap); #endif dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); @@ -1853,7 +1889,8 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) return; } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - dcp_arm_retained_framebuffers(dcp, resp->swap_id); + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); dcp_h17p_prepare_swap(dcp); diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index 2b4f476eabf403..e19397b21261cf 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -142,8 +142,8 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { #endif u8 unkoutbool_null; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 unkU32Ptr_null; - u8 unkU32out_null; + u8 unknown_pointer_null; + u8 unknown_output_null; #endif #if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; From ec95774b91988514bb896af3be3d3ad59c4a2fd6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 04:49:40 +1100 Subject: [PATCH 141/631] drm: apple: stop on H17P callbacks without reply semantics A measured callback length establishes buffer bounds but does not make an all-zero reply valid. The H17P dispatch table still lacks a handler for the measured property request; treating that omission as a successful empty reply would hide an incomplete protocol admission. Stop the measured H17P interface before modifying output or acknowledging an unhandled callback. Keep the active transaction and its surface references pinned for recovery by reboot. Explicitly supported handlers still receive zero-initialized output, and the existing H17G fallback is preserved. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 5 +++++ drivers/gpu/drm/apple/iomfb_h17p.c | 13 ++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index d1eacc3d293b9d..a1968c7e41800a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -659,6 +659,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, !dcp->cb_handlers[tag]) { dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + if (iomfb_uses_queue(dcp)) { + /* A measured size does not establish a safe reply payload. */ + WRITE_ONCE(dcp->crashed, true); + return; + } /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index b49a587f344720..772a5ec2e3ccf8 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -258,17 +258,12 @@ static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, } /* - * D400 (get_property) is left unhandled: the zeroed reply means "no such - * property". The firmware then runs without a panel power model, as it does - * for its other optional power properties. + * D400 has a measured size but no admitted property-reply semantics. It stays + * unhandled, so the measured H17P profile stops instead of guessing a reply. + * H17G retains its existing missing-property response. */ -/* - * H17P callback numbering. It is not a uniform shift of the v13.5 table: the - * service-creation block is D108..D113 exactly as on v13.5, and the swap - * completion is D590. A callback with no entry here is answered with a - * zeroed reply by dcpep_handle_cb(). - */ +/* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ [1] = trampoline_true, From b02c2db8c82ac563d5d30aa2ab24118ba8b4d25e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:36:15 +1100 Subject: [PATCH 142/631] drm: apple: preserve unknown H17P background on replay The first completed present need not set the background. Always adding a background update to its cached replay would then send a value that the firmware had never accepted, potentially changing alpha composition during a brightness-only update. Keep the background-update bit only when this or a previous completed present supplied the background. Continue re-presenting the pinned surfaces without changing an unknown inherited background. An explicit accepted background remains available across partial-plane updates. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a1968c7e41800a..490cc6e90257e2 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1328,7 +1328,12 @@ void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) scanout->request.surf_null[i] = !scanout->fb[i]; } /* Brightness re-presents all pinned surfaces with a fresh swap ID. */ - scanout->request.swap.swap_enabled = IOMFB_SET_BACKGROUND; + /* Replay only a background established by a completed present. */ + scanout->request.swap.swap_enabled = + request->swap.swap_enabled & IOMFB_SET_BACKGROUND; + if (previous) + scanout->request.swap.swap_enabled |= + previous->request.swap.swap_enabled & IOMFB_SET_BACKGROUND; for (i = 0; i < SWAP_SURFACES; i++) if (scanout->fb[i]) scanout->request.swap.swap_enabled |= BIT(i); From aee5dbc0ee5943259a1454b9229d4d50a0a7e944 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:52:14 +1100 Subject: [PATCH 143/631] dt-bindings: display: apple,dcp: describe firmware power tables Linux startup callbacks request five power lookup table properties from the display provider. Their replies reproduce the corresponding disp0 ADT bytes: three four-byte values and two 52-byte arrays. The table indices include the firmware's power model axis, which does not define Linux's backlight range. Describe bootloader-supplied byte arrays so the driver can transfer these per-device firmware inputs without embedding panel data or interpreting the power model. Limit the properties to the measured T8140 internal DCP and document their little-endian representation. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/display/apple,dcp.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index 33f1bf6e44359c..aefb570de5983d 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -123,6 +123,49 @@ properties: enum: [0, 1] description: Per-slot enable flags, in slot order. + apple,power-lut-data-x: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + First dimension of the display firmware power lookup table. The + bootloader copies the disp0 ADT property verbatim, as a little-endian + 32-bit word, for transfer to the display firmware. + + apple,power-lut-data-y: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + Second dimension of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as a little-endian 32-bit word. + + apple,power-lut-data-xindex: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 52 + maxItems: 52 + description: | + First-axis indices of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as thirteen little-endian 32-bit words. + These are firmware table inputs, not a Linux backlight range. + + apple,power-lut-data-yindex: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 4 + maxItems: 4 + description: | + Second-axis index of the display firmware power lookup table, copied + verbatim from disp0 in the ADT as a little-endian 32-bit word. + + apple,power-lut-data-lut: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 52 + maxItems: 52 + description: | + Display firmware power lookup table, copied verbatim from disp0 in + the ADT as thirteen little-endian 32-bit words. Linux passes these + values to the firmware without interpreting or changing them. + apple,firmware-version: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 3 @@ -270,6 +313,19 @@ required: - apple,bw-scratch allOf: + - if: + not: + properties: + compatible: + contains: + const: apple,t8140-dcp + then: + properties: + apple,power-lut-data-x: false + apple,power-lut-data-y: false + apple,power-lut-data-xindex: false + apple,power-lut-data-yindex: false + apple,power-lut-data-lut: false - if: properties: compatible: From bdb472b3d360e77d343be166d7679e5f627ed443 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 05:54:56 +1100 Subject: [PATCH 144/631] drm: apple: transfer measured H17P provider power tables During Linux startup the display firmware requests five provider power table properties. The returned bytes match disp0 in the ADT, followed by zero padding and a little-endian byte count. A sixth request refers to a property absent from that node and receives an empty reply. Firmware leaves bytes following the key's first NUL uninitialized. Handle these transfers using bootloader-supplied DT data. Validate the observed scope, capacity, trailing bytes, terminated key and exact data sizes before replying. Missing required data, unknown keys and unmeasured request variants stop the transaction instead of inventing a property. Keep the existing empty-reply behaviour for the H17G method profile. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 100 +++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 772a5ec2e3ccf8..4951949d7cc914 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -257,11 +257,100 @@ static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, return true; } -/* - * D400 has a measured size but no admitted property-reply semantics. It stays - * unhandled, so the measured H17P profile stops instead of guessing a reply. - * H17G retains its existing missing-property response. - */ +struct dcp_h17p_provider_request { + u8 scope[4]; + char name[64]; + __le32 capacity; + u8 reserved[4]; +} __packed; + +struct dcp_h17p_provider_reply { + u8 data[0xc00]; + __le32 length; +} __packed; + +struct dcp_h17p_provider_property { + const char *name; + const char *dt_name; + u32 length; +}; + +static_assert(sizeof(struct dcp_h17p_provider_request) == 0x4c); +static_assert(sizeof(struct dcp_h17p_provider_reply) == 0xc04); + +/* Only these provider transfers have been measured against the disp0 ADT. */ +static const struct dcp_h17p_provider_property provider_properties[] = { + { "power-lut-data-x", "apple,power-lut-data-x", 4 }, + { "power-lut-data-y", "apple,power-lut-data-y", 4 }, + { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52 }, + { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4 }, + { "power-lut-data-lut", "apple,power-lut-data-lut", 52 }, + { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0 }, +}; + +static const struct dcp_h17p_provider_property * +dcp_provider_property_h17p(const struct dcp_h17p_provider_request *request) +{ + unsigned int i; + + if (memcmp(request->scope, "VORP", sizeof(request->scope)) || + le32_to_cpu(request->capacity) != + sizeof_field(struct dcp_h17p_provider_reply, data) || + memchr_inv(request->reserved, 0, sizeof(request->reserved)) || + strnlen(request->name, sizeof(request->name)) == sizeof(request->name)) + return NULL; + + /* Bytes following the first NUL were not initialized by the firmware. */ + for (i = 0; i < ARRAY_SIZE(provider_properties); i++) + if (!strcmp(request->name, provider_properties[i].name)) + return &provider_properties[i]; + return NULL; +} + +static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_provider_property *property; + struct dcp_h17p_provider_reply *reply = out; + struct device_node *node = dcp->dev->of_node; + int length, ret; + + /* The dispatcher already cleared the exact H17G output length. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return true; + + trace_iomfb_callback(dcp, tag, __func__); + property = dcp_provider_property_h17p(in); + if (!property) { + dev_err(dcp->dev, "unqualified D400 provider property request\n"); + goto fail; + } + + length = of_property_count_u8_elems(node, property->dt_name); + if (!property->length) { + /* Only this absent property has a measured empty reply. */ + if (of_property_present(node, property->dt_name)) + goto fail; + } else if (length != (int)property->length) { + dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", + property->dt_name, length); + goto fail; + } + + memset(reply, 0, sizeof(*reply)); + if (property->length) { + ret = of_property_read_u8_array(node, property->dt_name, + reply->data, property->length); + if (ret) + goto fail; + } + reply->length = cpu_to_le32(property->length); + return true; + +fail: + WRITE_ONCE(dcp->crashed, true); + return false; +} /* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { @@ -302,6 +391,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [208] = trampoline_nop, /* update_backlight_factor_prop */ [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, + [400] = trampoline_provider_property_h17p, [401] = trampoline_get_uint_prop, /* get_uint_prop */ [404] = trampoline_nop, /* set_uint_prop */ [406] = trampoline_set_fx_prop, /* set_fx_prop */ From 8b3640acbba1f291d6e89cbc5bfff0218c4b8b71 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:11:18 +1100 Subject: [PATCH 145/631] drm: apple: unwind failed firmware buffer allocations The firmware buffer callback reserves a descriptor before allocating its backing memory and ignores failures from both dma_alloc_coherent() and dma_get_sgtable(). A failed allocation can therefore reach scatterlist creation with a NULL buffer, or leave an occupied descriptor without a usable mapping. Release also discards the scatterlist without freeing it. Reject zero-sized and overflowing requests before either page-rounding operation. Build the descriptor locally and publish it only after both DMA operations succeed. Free the coherent allocation if scatterlist creation fails, and release the scatterlist together with its backing allocation. Successful requests retain the existing wire-size rounding and reply fields. Fault injection exercises both failure points, descriptor exhaustion, rounding boundaries, retry after failure and release for the older firmware and H17P template profiles. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 36 +++++++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b607465b8acb55..bd3d8790afa5af 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -590,18 +590,40 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_mem_descriptor *memdesc; size_t size; u32 id; + int ret; - resp.dva_size = ALIGN(req->size, 4096); - resp.mem_desc_id = - find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + /* Validate both wire-size and IOMMU-page rounding before allocating. */ + if (!req->size || req->size > SIZE_MAX || + check_add_overflow((size_t)req->size, (size_t)4095, &rounded)) + return resp; + allocated.size = round_down(rounded, 4096); + if (check_add_overflow(allocated.size, (size_t)SZ_16K - 1, &rounded)) + return resp; + size = round_down(rounded, SZ_16K); - if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + + if (id >= DCP_MAX_MAPPINGS) { dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); resp.dva_size = 0; resp.mem_desc_id = 0; return resp; } - id = resp.mem_desc_id; + + allocated.buf = dma_alloc_coherent(dcp->dev, size, &allocated.dva, + GFP_KERNEL); + if (!allocated.buf) + return resp; + + ret = dma_get_sgtable(dcp->dev, &allocated.map, allocated.buf, + allocated.dva, size); + if (ret) { + dma_free_coherent(dcp->dev, size, allocated.buf, allocated.dva); + return resp; + } + + /* Callbacks are serialized; publish only a fully initialized descriptor. */ + dcp->memdesc[id] = allocated; set_bit(id, dcp->memdesc_map); memdesc = &dcp->memdesc[id]; @@ -622,8 +644,8 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, * microcode through it, and with a zero paddr the timing sequencer * comes up and is powered straight back down. */ - if (memdesc->map.sgl) - resp.paddr = sg_phys(memdesc->map.sgl); + if (allocated.map.sgl) + resp.paddr = sg_phys(allocated.map.sgl); #endif return resp; From f563e0bae2f5898e6126f1f05e436ac4aa96858e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:13:59 +1100 Subject: [PATCH 146/631] drm: apple: qualify H17P firmware buffer allocation replies Linux startup measurements correlate seven firmware allocation replies with the core DMA allocation and IOMMU mapping events. The first reply address matches the first physical scatterlist page, the second matches the device DMA address, and the reported length is rounded to 4 KiB. Two allocations have physically noncontiguous backing pages. Route H17P allocation requests through a bounded callback that admits the two observed option combinations and zero trailing/null bytes. Reuse the existing Asahi allocator and encode its successful reply explicitly in little endian. Stop the transaction when validation or allocation fails, since no H17P error reply has been measured. The H17G profile keeps its existing callback path. Describe the first address as a measured scatterlist correlation rather than assigning an unmeasured firmware use to it. This does not establish secondary-domain ownership or enable unqualified teardown callbacks. Replay all seven complete replies against independent core trace inputs, and exercise option mutations, both DMA failure points, overflow and mapping-table exhaustion. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 45 +++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_template.c | 5 ++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 4951949d7cc914..ca17da74ed7afa 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -352,6 +352,49 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_allocate_buffer_req *wire = in; + struct dcp_allocate_buffer_req request = { 0 }; + struct dcp_allocate_buffer_resp reply; + u32 options; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_allocate_buffer(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0x14); + static_assert(sizeof(reply) == 0x1c); + static_assert(offsetof(struct dcp_allocate_buffer_req, size) == 4); + static_assert(offsetof(struct dcp_allocate_buffer_req, unk2) == 12); + static_assert(offsetof(struct dcp_allocate_buffer_req, paddr_null) == 16); + + trace_iomfb_callback(dcp, tag, __func__); + options = get_unaligned_le32(&wire->unk2); + /* Only these option combinations have observed successful replies. */ + if (get_unaligned_le32(&wire->unk0) != 0x703 || + (options != 4 && options != SZ_16K) || + memchr_inv(&wire->paddr_null, 0, 4)) + goto fail; + + request.size = get_unaligned_le64(&wire->size); + reply = dcpep_cb_allocate_buffer(dcp, &request); + if (!reply.dva_size) + goto fail; + + put_unaligned_le64(reply.paddr, out); + put_unaligned_le64(reply.dva, (u8 *)out + 8); + put_unaligned_le64(reply.dva_size, (u8 *)out + 16); + put_unaligned_le32(reply.mem_desc_id, (u8 *)out + 24); + return true; + +fail: + /* Allocation failures have no measured H17P reply encoding. */ + dev_err(dcp->dev, "unqualified or failed D451 buffer allocation\n"); + WRITE_ONCE(dcp->crashed, true); + return false; +} + /* H17P callback numbering is not a uniform shift of the v13.5 table. */ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ @@ -400,7 +443,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [413] = trampoline_true, /* setProperty */ [414] = trampoline_sr_set_property_int, /* setProperty */ [415] = trampoline_true, /* setProperty */ - [451] = trampoline_allocate_buffer, /* allocate_buffer */ + [451] = trampoline_allocate_buffer_h17p, /* allocate_buffer */ [452] = trampoline_map_physical, /* prepare */ [454] = trampoline_release_mem_desc, /* release_descriptor */ [552] = trampoline_true, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index bd3d8790afa5af..bc61566bd5f09e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -640,9 +640,8 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) /* - * H17P firmware also needs the physical address: it loads the APT M3 - * microcode through it, and with a zero paddr the timing sequencer - * comes up and is powered straight back down. + * Linux allocation traces correlate the first H17P reply address with + * the first SG page. The remaining pages need not be contiguous. */ if (allocated.map.sgl) resp.paddr = sg_phys(allocated.map.sgl); From b1b7d770f917192977b49c8be0aa770f427991aa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:35:59 +1100 Subject: [PATCH 147/631] drm: apple: validate compact H17P receive message lengths Linux startup capture on the internal display endpoint records seventeen compact messages. Their header length counts all bytes after the first eight bytes, and the service channel is carried at offset two. Announce and report payloads start after the compact 24-byte header. Check the recorded length against the actual queue-entry size before normalizing a measured H17P receive header or dispatching its payload. Keep the existing H17G framing path unchanged. No outgoing compact message encoding is introduced by this receive-side check. Captured messages, unaligned buffers, truncated headers and length-bit mutations exercise the actual validator under address/undefined checks. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b58644c3e58042..4cb3720979470a 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -620,6 +621,14 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, payload_size, true); } +static bool afk_validate_h17p_header(const u8 *data, size_t size) +{ + /* The measured length counts bytes after the first eight header bytes. */ + return size >= sizeof(struct epic_hdr) + + sizeof(struct epic_sub_hdr_h17p) && + get_unaligned_le32(data + 4) == size - 8; +} + static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *data, size_t data_size) { @@ -642,6 +651,13 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, hdr_len = sizeof(*ehdr) + sizeof(*c); if (data_size < hdr_len) goto too_small; + if (ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !afk_validate_h17p_header(data, data_size)) { + dev_err(ep->dcp->dev, + "AFK[ep:%02x]: invalid compact message length\n", + ep->endpoint); + return; + } /* * H17P multiplexes every service of an endpoint onto queue From 3e5c9d10f0a91539037f9a11d844ce39387c210a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:37:26 +1100 Subject: [PATCH 148/631] drm: apple: answer the measured H17P startup integer query Three Linux startup transactions query the PMU Temperature property and receive the same 64-bit value 3029 with a successful boolean result. This is the existing Asahi firmware reply value, but the generic handler only returns it for panels marked as mini-LED. The internal H17P panel also makes this startup query. Use the observed reply for the measured H17P property request, validating the object, bounded property name and zero null/trailing bytes. Bytes after the name terminator and the input's previous value are ignored. Stop on unmeasured queries rather than returning an invented zero value. The H17G profile retains the generic panel-dependent behavior. The scalar is a firmware interoperability value; its temperature units and any relationship to a physical sensor have not been established. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index ca17da74ed7afa..47cf407089867d 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -352,6 +352,30 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_get_uint_prop_req *request = in; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_get_uint_prop(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + /* The observed startup query uses Asahi's existing Temperature reply. */ + if (memcmp(request->obj, "SUMP", sizeof(request->obj)) || + strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || + strcmp(request->key, "Temperature") || request->value_null || + memchr_inv(request->padding, 0, sizeof(request->padding))) { + dev_err(dcp->dev, "unqualified D401 integer property query\n"); + WRITE_ONCE(dcp->crashed, true); + return false; + } + + put_unaligned_le64(3029, out); + *((u8 *)out + offsetof(struct dcp_get_uint_prop_resp, ret)) = true; + return true; +} + static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -435,7 +459,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [209] = trampoline_get_time, /* get_calendar_time_ms */ [300] = trampoline_pr_publish, [400] = trampoline_provider_property_h17p, - [401] = trampoline_get_uint_prop, /* get_uint_prop */ + [401] = trampoline_get_uint_prop_h17p, /* get_uint_prop */ [404] = trampoline_nop, /* set_uint_prop */ [406] = trampoline_set_fx_prop, /* set_fx_prop */ [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ From 14837c7ea66c7f4e1262ea854349af5119bde7b6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:39:36 +1100 Subject: [PATCH 149/631] drm: apple: select the internal T8140 H17P method profile Linux startup measurements on T8140 identify the internal display's H17P IOMFB interface and a live bootloader-owned RTKit session. Falling back to the generic DCP match cannot select that firmware profile and would attempt to start the coprocessor instead of adopting its session. Match the internal T8140 DCP explicitly and reuse the existing H17 live-session adoption with the measured method profile. The match advertises no external display ports; external DCP qualification remains separate. The public DT still leaves display resources disabled. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5938791ac38782..e361d10cd1715f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3370,6 +3370,13 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +/* The internal T8140 endpoint uses the measured H17P method profile. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { + .num_dptx_ports = 0, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + /* * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering * and is left running by the bootloader. @@ -3419,6 +3426,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t8122-dcp", .data = &apple_dcp_hw_t8122, }, { .compatible = "apple,t6031-dcp", .data = &apple_dcp_hw_t6031, }, { .compatible = "apple,t8122-dcpext", .data = &apple_dcp_hw_t8122_dcpext, }, + { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, From c102448a40c0729591e75bf31d36a276e030eb6e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:52:22 +1100 Subject: [PATCH 150/631] drm: apple: answer the H17P luma-range startup callback The internal H17P display requests an eight-byte luma-range reply during client open. Selector zero receives the binary32 encoding of 219.0 and a zero trailing word. Leaving this measured callback unbound stops startup; a zero range also cannot provide the firmware's required colour scale. Provide the recorded reply for selector zero without kernel floating point. Reject unqualified selectors, and preserve the H17G profile's existing zero-filled unknown-callback result, including short output envelopes. The actual dispatcher reproduces the captured reply and rejects selector mutations without acknowledging them. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 47cf407089867d..6ac01ee69fcf45 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -352,6 +352,25 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, return false; } +static bool trampoline_luma_span_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + /* Preserve the older profile's already-cleared unknown-callback reply. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return true; + + trace_iomfb_callback(dcp, tag, __func__); + if (get_unaligned_le32(in)) { + dev_err(dcp->dev, "unqualified D574 range selector\n"); + WRITE_ONCE(dcp->crashed, true); + return false; + } + + /* Selector zero receives binary32 219.0 followed by a zero word. */ + put_unaligned_le64(0x435b0000, out); + return true; +} + static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -476,6 +495,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [565] = trampoline_true, [567] = trampoline_true, [572] = trampoline_zero, /* powerUpDART */ + [574] = trampoline_luma_span_h17p, [575] = trampoline_hotplug_h17p, [576] = trampoline_nop, [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ From f21776e84d0e662bd33fc6f7d6917a65187a3331 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:15:17 +1100 Subject: [PATCH 151/631] drm: apple: acknowledge bounded unknown H17P callbacks H17P waits for every firmware callback acknowledgment. Stopping on an unknown callback leaves the outer command outstanding and blocks all later presentation. Its generic unsupported-callback response is the zero-filled output area followed by the normal acknowledgment. Restore that bounded fallback for callbacks without a handler. Validate the header, tag range, stack depth, channel extent and complete record length before touching output. Typed handlers still require their known input/output sizes. Record unknown tags once per device, preserving the older firmware profiles' established dispatch behavior. Dispatcher fault injection covers malformed envelopes, output canaries, missing handlers and the complete measured startup callback set. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/iomfb.c | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index f0a198df158b7f..db25a0a7af6ad7 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -502,6 +502,7 @@ struct apple_dcp { /* Staging wire record; serialization never mutates the atomic inputs. */ struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; + DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 490cc6e90257e2..8bb82d33fef1b5 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -632,6 +632,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; + bool handled; if (dcp->fw_compat == DCP_FIRMWARE_H17P && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || @@ -642,9 +643,13 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && + dcp->cb_handlers[tag]; if (dcp->fw_compat == DCP_FIRMWARE_H17P && - ((u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || + (tag < 0 || tag >= IOMFB_MAX_CB || + (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + handled && !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); @@ -655,15 +660,11 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, in = data + sizeof(*hdr); out = in + hdr->in_len; - if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || - !dcp->cb_handlers[tag]) { - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); - if (iomfb_uses_queue(dcp)) { - /* A measured size does not establish a safe reply payload. */ - WRITE_ONCE(dcp->crashed, true); - return; - } + if (!handled) { + if (!iomfb_uses_queue(dcp) || + !test_and_set_bit(tag, dcp->unknown_callbacks)) + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. From cba04864e69385906ec48e9e156066f75cb1c5f2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:18:56 +1100 Subject: [PATCH 152/631] drm: apple: describe T8140 firmware clock aperture in the hardware profile The internal T8140 firmware exposes a sixth register aperture at 0x302800000 with size 0xbc000. Its bandwidth callback names the scratch at offset 0x20000 and request doorbell at offset 0x68000. Using the AP PMGR window from an older loader tree supplies different addresses and breaks the display firmware's clock setup. Describe this firmware aperture in the T8140 hardware profile. Append it only after the five required display windows, validate both offsets and use the same resource for the firmware register and bandwidth replies. This does not map, prime or write the clock window from the AP. Older profiles retain their DT-provided bandwidth resources and probe checks. The resource matches the independently retained disp0 ADT register and Linux register reply. Resource tests cover the two bandwidth addresses, missing windows, bad offsets and the older DT path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 ++++ drivers/gpu/drm/apple/dcp.c | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index db25a0a7af6ad7..f444d50dbcdce1 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -180,6 +180,10 @@ struct apple_dcp_hw_data { */ bool adopt_live_session; enum dcp_firmware_version firmware_compat; + /* Firmware-visible clock aperture, independent of the AP PMGR window. */ + const struct resource *firmware_clock; + u32 firmware_scratch; + u32 firmware_request; }; /* TODO: move IOMFB members to its own struct */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e361d10cd1715f..9af2481154ee25 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2143,6 +2143,24 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) dcp->disp_registers[i] = res; } + if (dcp->hw.firmware_clock) { + /* The internal firmware's last aperture follows five display windows. */ + if (count != 5 || resource_size(dcp->hw.firmware_clock) < 4 || + dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || + dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) + return -EINVAL; + + dcp->disp_bw_scratch_res = *dcp->hw.firmware_clock; + dcp->disp_bw_scratch_index = count; + dcp->disp_bw_scratch_offset = dcp->hw.firmware_scratch; + dcp->disp_bw_doorbell_res = dcp->disp_bw_scratch_res; + dcp->disp_bw_doorbell_res.start += dcp->hw.firmware_request; + dcp->disp_bw_doorbell_res.end = dcp->disp_bw_doorbell_res.start + 3; + dcp->disp_registers[count] = &dcp->disp_bw_scratch_res; + dcp->nr_disp_registers = count + 1; + return 0; + } + /* load pmgr bandwidth scratch resource and offset */ ret = dcp_get_bw_scratch_reg(dcp, count); if (ret < 0) @@ -2772,6 +2790,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (of_machine_is_compatible("apple,t8140")) return -ENODEV; + const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); enum dcp_firmware_version fw_compat; struct device *dev = &pdev->dev; struct apple_dcp *dcp; @@ -2791,6 +2810,8 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (fw_compat != DCP_FIRMWARE_V_14_7 && fw_compat != DCP_FIRMWARE_V_26_6 && !of_property_present(dev->of_node, "apple,bw-scratch")) + if (!of_property_present(dev->of_node, "apple,bw-scratch") && + !hw->firmware_clock) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " "Use devicetree matching this kernel.\n"); @@ -3371,10 +3392,16 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { }; /* The internal T8140 endpoint uses the measured H17P method profile. */ +static const struct resource t8140_firmware_clock = + DEFINE_RES_MEM(0x302800000, 0xbc000); + static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .num_dptx_ports = 0, .adopt_live_session = true, .firmware_compat = DCP_FIRMWARE_H17P, + .firmware_clock = &t8140_firmware_clock, + .firmware_scratch = 0x20000, + .firmware_request = 0x68000, }; /* From e03ecd051d992a2cfd2d9589ba17e5ceab8ccc02 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:46:10 +1100 Subject: [PATCH 153/631] drm: apple: latch bounded H17P backlight takeover hints Linux powerlog observations contain millinits brightness hints before presentation. Later reports can differ from the first commanded level, so use the first valid panel-bounded hint and ignore subsequent reports once the policy is ready. Keep the existing panel maximum and 509-nit software clamp without inventing a fallback or calibration. Connect the public report parser to the queued backlight policy. Reject millinits overflow before narrowing, and keep registration and legacy brightness submission unavailable until the measured profile has a hint. Preserve the H17G and older firmware paths. Physical inherited-level continuity remains a hardware qualification gate. Add KUnit coverage for conversion bounds, zero hints and protection from late reports after control registration. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 24 ++++++++++++++++++ drivers/gpu/drm/apple/dcp_backlight.h | 12 +++++++++ drivers/gpu/drm/apple/dcp_backlight_test.c | 29 ++++++++++++++++++++++ drivers/gpu/drm/apple/systemep.c | 13 +++++++--- 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 19ca88e9fa5bdb..0e06d001aca311 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -88,9 +88,26 @@ static u32 brightness_part2[] = { bool dcp_backlight_active(struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && READ_ONCE(dcp->backlight.state.ready); } +int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits) +{ + u32 maximum, nits; + int ret; + + if (dcp_backlight_active(dcp)) + return 0; + maximum = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + ret = dcp_bl_takeover_nits(maximum, millinits, &nits); + if (ret) + return ret; + + /* Keep the first bounded hint. Later reports are not user requests. */ + return iomfb_configure_backlight_h17p(dcp, maximum, true, nits, false, 0); +} + static int dcp_get_brightness(struct backlight_device *bd) { struct apple_dcp *dcp = bl_get_data(bd); @@ -324,6 +341,9 @@ int dcp_backlight_update(struct apple_dcp *dcp) dcp_backlight_kick(dcp); return 0; } + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return -ENODATA; /* * Do not actively try to change brightness if no mode is set. @@ -412,6 +432,10 @@ int dcp_backlight_register(struct apple_dcp *dcp) .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_backlight_active(dcp)) + return -ENODATA; if (dcp_backlight_active(dcp)) { spin_lock_irqsave(&dcp->backlight.lock, flags); if (!dcp->backlight.state.ready) { diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 6d52a8f9eb7fe1..e93ee7511c6266 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -29,6 +29,17 @@ struct dcp_backlight_present { u32 nits; }; +/* Public powerlog reports use millinits; validate before rounding to nits. */ +static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) +{ + if (!maximum || maximum > INT_MAX) + return -EINVAL; + if ((u64)millinits > (u64)maximum * 1000) + return -ERANGE; + *nits = millinits / 1000; + return 0; +} + /* Inputs must come from admitted panel/loader records, never a guess. */ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, bool inherited_valid, u32 inherited, @@ -151,6 +162,7 @@ struct apple_dcp; * Configure before registration and keep the callback valid until teardown. */ bool dcp_backlight_active(struct apple_dcp *dcp); +int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits); int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits, diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index e990ea3e0bf066..a130f6f4147127 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -6,6 +6,34 @@ #include "dcp_backlight.h" +static void backlight_millinits_takeover_test(struct kunit *test) +{ + struct dcp_backlight_state state; + u32 nits = 123; + + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(0, 100000, &nits), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(U32_MAX, 100000, &nits), -EINVAL); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, nits, 123U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 100U); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); + KUNIT_EXPECT_EQ(test, state.actual, 100U); + KUNIT_EXPECT_FALSE(test, state.dirty); + /* Registration freezes the hint before exposing the controls. */ + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 400)); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 0)); + KUNIT_EXPECT_EQ(test, state.target, 100U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 0, &nits), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); + KUNIT_EXPECT_EQ(test, state.target, 0U); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509000, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); +} + static void backlight_takeover_test(struct kunit *test) { struct dcp_backlight_state state = {}; @@ -187,6 +215,7 @@ static void backlight_inflight_dpms_test(struct kunit *test) } static struct kunit_case backlight_cases[] = { + KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), KUNIT_CASE(backlight_range_test), KUNIT_CASE(backlight_dpms_cycles_test), diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index 3133cd539b7f1d..a32f0a0bcf6a0f 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -75,9 +75,7 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype struct apple_dcp *dcp = service->ep->dcp; int ret; - /* H17P takeover decoding requires an admitted record. Later zero reports - * must not overwrite the cached target or completed level. - */ + /* Later reports must not overwrite a latched hint or Linux's target. */ if (dcp_backlight_active(dcp)) return 0; @@ -102,6 +100,15 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (!dcp_has_panel(dcp)) + return 0; + ret = dcp_backlight_takeover(dcp, mnits.millinits); + if (ret && ret != -EBUSY) + dev_dbg(dcp->dev, "backlight takeover hint unavailable: %d\n", ret); + return 0; + } dcp->brightness.nits = mnits.millinits / 1000; From c96d8ed200906befbb53bf87ca20ef5598d20e60 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:53:14 +1100 Subject: [PATCH 154/631] drm: apple: encode H17P backlight activation and panel ceilings Linux brightness changes and soft DPMS presents use the same activation bytes and integer binary64 nits field. Encode the complete brightness block through the existing queued reservation, preserving the panel ceiling pair on every frame, including idle and pre-takeover frames. Refuse missing ceilings instead of submitting a zero calibration pair. An accepted matching completion updates the completed commanded level. Idle presents leave activation disabled; zero-nit enabled presents blank the retained scanout and unblank restores the user's target. Keep the wire ceiling separate from the existing exposed software brightness cap. Add a native actual-function fixture with captured packet fragments, full-range enabled/disabled replay, registration and reservation checks, rejection/coalescing and twenty software DPMS cycles. Document its mocked transport limits and the remaining integrated hardware qualification. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/gpu/apple-dcp-backlight.rst | 56 +++- drivers/gpu/drm/apple/iomfb.c | 14 +- drivers/gpu/drm/apple/iomfb_h17p.c | 31 ++- drivers/gpu/drm/apple/iomfb_h17p.h | 3 +- tools/testing/apple/backlight-golden.json | 297 ++++++++++++++++++++++ tools/testing/apple/check-backlight.py | 279 ++++++++++++++++++++ 6 files changed, 662 insertions(+), 18 deletions(-) create mode 100644 tools/testing/apple/backlight-golden.json create mode 100644 tools/testing/apple/check-backlight.py diff --git a/Documentation/gpu/apple-dcp-backlight.rst b/Documentation/gpu/apple-dcp-backlight.rst index e47e7f3b4dd4be..e486fcf4840df4 100644 --- a/Documentation/gpu/apple-dcp-backlight.rst +++ b/Documentation/gpu/apple-dcp-backlight.rst @@ -5,20 +5,19 @@ Apple DCP H17P backlight integration The backlight policy implements the clean display specification's BACKLIGHT-1 through BACKLIGHT-4, with IOMFB-3 and LIFETIME-1 through LIFETIME-3 constraints. -It extends the existing Asahi backlight driver. It contains no H17P brightness -wire layout, calibration table, property command, or guessed initial level. -Policy activation requires explicit successful configuration. Existing -unconfigured H17P sessions, including T8142, keep their established DAC, -property/powerlog and DPMS paths. Older firmware remains on its existing path. -Neo startup must configure the policy before exposing the backlight or starting -its DPMS/brightness traffic; a firmware-version match alone does not activate -this policy. +It extends the existing Asahi backlight driver. The measured internal H17P profile carries brightness activation through the +central present queue. It latches the first bounded powerlog hint without +sending a brightness update and leaves registration unavailable if there is +no hint. H17G and older firmware retain their existing paths. Physical loader +level continuity remains a measurement gate. Admission and startup --------------------- -The central display worker must call ``dcp_backlight_configure()`` from process -context after linking the CRTC and admitting all required records. Its arguments +The measured profile connects the public powerlog parser to +``iomfb_configure_backlight_h17p()`` after linking the CRTC. Generic users of +``dcp_backlight_configure()`` must still supply validated inputs from process +context. Its arguments are the validated maximum commanded nits, an optional inherited level, an optional documented ADT default, and a callback that schedules the central outbound queue. Maximum must fit the backlight core's signed integer range. @@ -64,7 +63,8 @@ configuration or surface; ``-EBUSY`` means a prior brightness transaction is outstanding; ``-EALREADY`` means no brightness change is pending. These results do not clear pending work. The serializer places the returned level only in admitted fields of the next valid present. A normal present without a brightness -change keeps the inherited/completed level; its clean layout must support that. +change disables the measured activation fragment, preserving the existing +physical level without resending a level. The sequence is a software cookie, not a wire present ID. Associate the cookie with the central transaction and its strict wire ID. Call @@ -127,3 +127,37 @@ cached restore, both suspend/DPMS orders, static-surface deferral, busy sequenci rejection and stale completion. They do not qualify firmware fields or GNOME. Hardware acceptance requires O-BACKLIGHT, O-BACKLIGHT-TAKEOVER (10 Hz PBwo with no jump above 0.05 W), O-BACKLIGHT-CRC, O-DPMS (twenty cycles), and O-SUSPEND. + +Measured activation and software replay +-------------------------------------- + +Linux-boundary measurements on the default J700 driver establish an 18-byte +fragment at A408 offsets 0x354 through 0x365. Brightness-enabled presents use +``01 00 00 00 01 01 01 00 01 00`` followed by integer little-endian binary64 +nits. Ordinary presents disable the activation and nits fragment. Its individual control +bytes have no separately inferred semantics. The two binary64 ceiling fields +at 0x36e and 0x376 carry the raw panel maximum on every internal present, +including ordinary frames and frames before a takeover hint arrives. Enabled +updates additionally carry unity at 0x37e and 0x3e6; other scalars are zero. +The complete owned block ends at 0x3ed. The observed 525-nit ceiling is distinct +from the 509-nit exposed software range. Missing ceilings refuse serialization. +The older DAC/power words at 0x32f through 0x33b stay zero on this profile, +including before a takeover hint arrives. Enabled zero nits blanks the +panel while keeping the pipe and retained surface alive. Requests while +blanked change the restore target; unblank restores it through the same queue. + +The measurements include sysfs changes, two GNOME Mutter SetBacklight API +steps and one PowerSaveMode off/on cycle. The latter reaches near-zero panel +power while display-pipe power stays at 605 mW. Cached sysfs actual brightness +on the observed driver is not physical readback while blanked. These records +do not establish clean-driver firmware acceptance, physical takeover continuity, +static-screen latency, suspend/resume or a twenty-cycle hardware gate. + +The native fixture extracts the actual registration, takeover and queue +reservation functions and replays captured fragments under AddressSanitizer +and UndefinedBehaviorSanitizer:: + + python3 tools/testing/apple/check-backlight.py --output /path/to/scratch + +It mocks locks, scheduling, DRM registration and transport. The fragment corpus +retains hashes of clean Linux observations, with runtime addresses omitted. diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8bb82d33fef1b5..b999da3571f046 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1362,9 +1362,16 @@ bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, int ret; lockdep_assert_held(&dcp->iomfb.lock); - if (!dcp_backlight_active(dcp)) + if (!dcp_backlight_active(dcp)) { + if (!dcp_has_panel(dcp)) + return true; + if (!dcp->brightness.maximum) + return false; + iomfb_encode_backlight_h17p(wire, 0, + dcp->brightness.maximum, false); return true; - if (!transaction) + } + if (!transaction || !dcp->brightness.maximum) return false; for (i = 0; i < SWAP_SURFACES; i++) have_surface |= !request->surf_null[i]; @@ -1375,7 +1382,8 @@ bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, transaction->backlight = present; transaction->backlight_reserved = true; } - iomfb_encode_backlight_h17p(wire, present.nits); + iomfb_encode_backlight_h17p(wire, present.nits, + dcp->brightness.maximum, !ret); return true; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 6ac01ee69fcf45..85d1a4d13ceff7 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -110,7 +110,7 @@ void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, } /* Integer commanded nits are exact binary64 values; no kernel FP is needed. */ -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) +static u64 iomfb_nits_binary64_h17p(u32 nits) { u64 value = 0; unsigned int exponent; @@ -120,8 +120,33 @@ void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits) value = (u64)(exponent + 1023) << 52; value |= ((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0); } - /* Measured from Linux presents during ordinary sysfs brightness changes. */ - put_unaligned_le64(value, wire->swap + 0x35e); + return value; +} + +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update) +{ + u64 ceiling = iomfb_nits_binary64_h17p(maximum); + u64 unity = iomfb_nits_binary64_h17p(1); + + /* This profile does not use the older DAC/power words. */ + memset(wire->swap + 0x32f, 0, 0xd); + memset(wire->swap + 0x354, 0, 0x9a); + /* The ceiling pair must remain valid even on a brightness-idle frame. */ + put_unaligned_le64(ceiling, wire->swap + 0x36e); + put_unaligned_le64(ceiling, wire->swap + 0x376); + if (!update) + return; + + /* Linux brightness changes and soft DPMS use the same activation bytes. */ + put_unaligned_le32(1, wire->swap + 0x354); + wire->swap[0x358] = 1; + wire->swap[0x359] = 1; + wire->swap[0x35a] = 1; + put_unaligned_le16(1, wire->swap + 0x35c); + put_unaligned_le64(iomfb_nits_binary64_h17p(nits), wire->swap + 0x35e); + put_unaligned_le64(unity, wire->swap + 0x37e); + put_unaligned_le64(unity, wire->swap + 0x3e6); } static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 0121d334c214ad..e4eb5be2b01438 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,7 +67,8 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); -void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits); +void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, + u32 maximum, bool update); void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); diff --git a/tools/testing/apple/backlight-golden.json b/tools/testing/apple/backlight-golden.json new file mode 100644 index 00000000000000..d1a8af2ab20118 --- /dev/null +++ b/tools/testing/apple/backlight-golden.json @@ -0,0 +1,297 @@ +[ + { + "nits": 197, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000a06840000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "59bc5374f6e05d6bcf2839b9522bdb9456074f80c7d542c0ecdc7e90cf7c1d16" + }, + { + "nits": 188, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "05c226553a34876237f2eef41246a75d013cd29115e68437df03f594f2459941" + }, + { + "nits": 179, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "e74693443c8a029a060cf3612556f93837a82cc3f3936975dcd4848432186067" + }, + { + "nits": 170, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000406540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "67521cee14ce162f0a733a815b1f059f3fe394c0c5e87bfe178f14b3acceb52c" + }, + { + "nits": 161, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "f2996dbaff64ac3d653cd4b420df919c418697fcf8dc4f995f562e5839efcef2" + }, + { + "nits": 152, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000006340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "b134853efd740adc11f258a206b4deb098df1765191c073178e033df83d17148" + }, + { + "nits": 143, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000e06140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "1488c635d8d55ab49f73f9d9698a4b3e0f8fe95fcf925b73c21f4846fa7188f2" + }, + { + "nits": 134, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000c06040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "132f2aba31bdc58731510361e6b934284ac81baeed5346dc235b615a43f6d798" + }, + { + "nits": 125, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a4f3e35516ba00acbed9c6c04ebb564334dab983b1a016f570dfe739ea2a9ae3" + }, + { + "nits": 116, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005d40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "837951003a3632bb5c063e3ee70606583af2f843ef722864cbf461ef01fc5a81" + }, + { + "nits": 106, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000805a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a68f25f1ce7f63c335781888df47a8eb12f863c37795b2377d69c270dc063cac" + }, + { + "nits": 97, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405840000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "6c5108c9aadd93c72b2f0d5b72adb09f55ca63159f909d0876714bbf62f80ea5" + }, + { + "nits": 88, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "b2db893e953fda0c249ecf7c674acfd2018d1267985cc09d842b2f38910b2189" + }, + { + "nits": 79, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000c05340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "321da86969fcfdb11f7d6e508a627bb0f9146defd48c4fa0d0cd76d7b63fd49b" + }, + { + "nits": 70, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000805140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "66443759f999549d7d9b076c338cc9f5ba7b2606ec5f764f14a17f8a23f28acb" + }, + { + "nits": 61, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804e40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4c3447c9290bdda6fc9f38d024f8c8482fc7692b61243680cf6dc2012dd8d8a1" + }, + { + "nits": 52, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000004a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "ce727d2eb98cbdeba8be5215577c2486ad8d5ad1b7ca1db2358d7c27f0cc2f1f" + }, + { + "nits": 43, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "00f94f04f7e431494dddf154cf8e68a1e3fe91afa3ab18d41c52223d582112ae" + }, + { + "nits": 36, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000004240000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4ae2c36b9d9a773bcfaf68c74b81e3761f4b0da68ffff4f7a55194207b313828" + }, + { + "nits": 59, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000804d40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "aa653a1c3d85ad9cf3a487aebdfafd0332c6d81335e9bc051292cab4610997de" + }, + { + "nits": 92, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000005740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a57f482514132d36aa5c46026b7c579bd74b1d21b84e5ed67e4123eab72e1054" + }, + { + "nits": 125, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000405f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "6260e7be2d29193f51a0fe60337efcdc9c6e73b469f527afb9f0ff83765467ec" + }, + { + "nits": 140, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "0eab7f3ebd304ddb5f40d2c8bc1ecd2110b166faf70f89b98a2521e0b6c7ac95" + }, + { + "nits": 200, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000006940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "98df21464117081715a48b0076304aad0f2b8307ee682c20a7feafd9a1be9202" + }, + { + "nits": 287, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000f07140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "7f5df7ed6f804a3340c9782698f5292e3b4117f2eb9943778010936a6b2746a4" + }, + { + "nits": 375, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000707740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "994697bd5bb64697f4d2bbbf2311a7c383f356161184e2a9aeea610515dda25e" + }, + { + "nits": 400, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000007940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a9fbdac983180e10843468b14c2cade1ee7d0e8711e02fce56489112c91b38e0" + }, + { + "nits": 335, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000f07440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "4cf3e7680e17a183e40e54b69d54693f43f3643346de050beb5c9cb73214b8f8" + }, + { + "nits": 249, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206f40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "9b174a609f7fda9e379923c5e1fba03952b6f44d3ad031dfa8660a015326725e" + }, + { + "nits": 163, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606440000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "bea9a384a1ceb723ddda8416748e20057e67627d768e9d1a81207d9f1a21d812" + }, + { + "nits": 131, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000606040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "2445a2a32d0cbf27f6eb394bc48ff08c58ca88a32e6a6f9cbe633e983ab05256" + }, + { + "nits": 185, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000206740000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "1a23772d33bedf58b4c566ae84267a0631f7ced0d881ac9470667ec2eab477bc" + }, + { + "nits": 264, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000807040000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "3bd83da635cd09c608e2124c75e9aaa9ed9337622e0a818c500ddf83d8146d33" + }, + { + "nits": 337, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000107540000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "d44097a52f6cef9ca7c4ddf39897bf67a9915b520b154d50d5210595997f0495" + }, + { + "nits": 357, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000507640000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "51f19fe9d95e6d95b8b81b158a3d5c2db391973a66695858610ae24ade44c0d8" + }, + { + "nits": 0, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000000000000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "3f3d1f6ae52991efa63ac64d74e3e9f28e8b000791732102a69a5c05452d2512" + }, + { + "nits": 140, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806140000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "5283a0beb8119ca257f5c00ae267598427c2bb01a472395c83b325f8c7a4b81a" + }, + { + "nits": 212, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000806a40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "d1bdce12296d366b8315c73d850fc80e92720748aff8ce3ff28a136e6104f0c8" + }, + { + "nits": 313, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000907340000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "a86f277bb5b34a2b13301d2c3464db76325d0bacd6009b5e46329fe97c82df60" + }, + { + "nits": 413, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000d07940000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "bca47e8efde3063740bd65f174ccc0b502b5ef2350cc821397679724ecf7d2d6" + }, + { + "nits": 454, + "enabled": true, + "maximum": 525, + "fragment": "010000000101010001000000000000607c40000000000000000000000000006880400000000000688040000000000000f03f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f03f", + "payload_sha256": "17c4a25be2d6cefb43cb282859f500415e3902be3d9d9ef5ab9045e6e9c6af53" + }, + { + "nits": 100, + "maximum": 525, + "enabled": false, + "fragment": "00000000000000000000000000000000000000000000000000000000000000688040000000000068804000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "payload_sha256": "7de62e12dc678b02f3b47b68ab369efc651cc05d765d2e024652583e410461da", + "provenance": "run24 clean ordinary startup presents" + } +] diff --git a/tools/testing/apple/check-backlight.py b/tools/testing/apple/check-backlight.py new file mode 100644 index 00000000000000..e1df3dc3458de4 --- /dev/null +++ b/tools/testing/apple/check-backlight.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native replay of clean brightness fragments and actual policy/queue functions. + +Run with --output on a scratch volume. This mocks kernel locks, DRM registration +and queue transport; it does not claim firmware or KUnit boot acceptance. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--output', required=True, type=Path) +parser.add_argument('--cc', default='clang') +args = parser.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +out = args.output.resolve() +out.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + depth = 1 + pos = start + 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +include = out / 'include/linux' +include.mkdir(parents=True, exist_ok=True) +(include / 'types.h').write_text('#include \n#include \ntypedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64;\n') +(include / 'limits.h').write_text('#include \n#define U32_MAX UINT32_MAX\n') +(include / 'errno.h').write_text('#include \n#ifndef ENODATA\n#define ENODATA 61\n#endif\n') +prelude = r''' +#include +#include +#include +#include +#include +#include +#include "dcp_backlight.h" +#define DCP_FIRMWARE_H17P 17 +#define DCP_IOMFB_METHODS_H17G 1 +#define MAX_BRIGHTNESS_PART2 510U +#define SWAP_SURFACES 4 +#define READ_ONCE(x) (x) +#define min(a,b) ((a)<(b)?(a):(b)) +#define lockdep_assert_held(x) ((void)(x)) +#define mutex_lock(x) ((void)(x)) +#define mutex_unlock(x) ((void)(x)) +#define spin_lock_irqsave(x,f) do { (void)(x);(f)=0; } while(0) +#define spin_unlock_irqrestore(x,f) do { (void)(x);(void)(f); } while(0) +#define GENMASK_ULL(h,l) ((~UINT64_C(0) >> (63-(h))) & (~UINT64_C(0) << (l))) +#define fls(x) (32-__builtin_clz(x)) +#define BACKLIGHT_PLATFORM 1 +#define BACKLIGHT_SCALE_LINEAR 2 +struct device { int unused; }; +struct backlight_device { int unused; }; +struct backlight_properties { int type;u32 brightness,max_brightness;int scale; }; +struct dcp_present_h17p { u8 swap[0x588];u8 guard[32]; }; +struct dcp_swap_submit_req_h17p { u8 surf_null[SWAP_SURFACES]; }; +struct iomfb_transaction { + struct dcp_backlight_present backlight; + bool backlight_reserved,brightness_only,completed; +}; +struct apple_dcp { + int fw_compat;struct { int iomfb_method_profile; } hw; + bool panel;void *crtc;struct device *dev; + int bl_register_mutex,bl_register_wq; + struct { u32 maximum;u32 nits;struct backlight_device *bl_dev;u32 dac; } brightness; + struct { int lock;struct dcp_backlight_state state;void (*kick)(struct apple_dcp *); } backlight; + struct { int lock;struct iomfb_transaction *active; } iomfb; +}; +static int schedule_count,register_count,retire_count,dac_count; +static struct backlight_properties registered; +static const int dcp_backlight_ops; +static bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel; } +static void schedule_work(int *work) { (void)work;schedule_count++; } +static void iomfb_backlight_kick(struct apple_dcp *dcp) { (void)dcp; } +static void iomfb_scanout_complete_h17p(struct apple_dcp *dcp) { (void)dcp;retire_count++; } +static u32 calculate_dac(struct apple_dcp *dcp,u32 nits) { (void)dcp;dac_count++;return nits; } +static struct backlight_device *devm_backlight_device_register(struct device *dev,const char *name, + struct device *parent,struct apple_dcp *dcp,const int *ops,const struct backlight_properties *props) { + static struct backlight_device device; + (void)dev;(void)parent;(void)dcp;(void)ops; + assert(!strcmp(name,"apple-panel-bl"));registered=*props;register_count++;return &device; +} +#define IS_ERR(p) (!(p)) +#define PTR_ERR(p) ((void)(p),-ENOMEM) +static void put_unaligned_le64(u64 v,void *p) { for(int i=0;i<8;i++)((u8 *)p)[i]=v>>(8*i); } +static void put_unaligned_le32(u32 v,void *p) { for(int i=0;i<4;i++)((u8 *)p)[i]=v>>(8*i); } +static void put_unaligned_le16(uint16_t v,void *p) { for(int i=0;i<2;i++)((u8 *)p)[i]=v>>(8*i); } +int iomfb_configure_backlight_h17p(struct apple_dcp *,u32,bool,u32,bool,u32); +''' +functions = [ + ('iomfb_h17p.c', 'iomfb_nits_binary64_h17p'), + ('iomfb_h17p.c', 'iomfb_encode_backlight_h17p'), + ('iomfb.c', 'iomfb_uses_queue'), + ('dcp_backlight.c', 'dcp_backlight_active'), + ('dcp_backlight.c', 'dcp_backlight_configure'), + ('iomfb.c', 'iomfb_configure_backlight_h17p'), + ('dcp_backlight.c', 'dcp_backlight_takeover'), + ('dcp_backlight.c', 'dcp_backlight_register'), + ('dcp_backlight.c', 'dcp_backlight_prepare'), + ('dcp_backlight.c', 'dcp_backlight_complete'), + ('iomfb.c', 'iomfb_apply_backlight_h17p'), + ('iomfb.c', 'iomfb_present_failed_h17p'), + ('iomfb.c', 'iomfb_present_complete_h17p'), +] +main = r''' +static struct apple_dcp fixture(void) { + return (struct apple_dcp){.fw_compat=DCP_FIRMWARE_H17P,.panel=true, + .crtc=(void *)1,.brightness.maximum=525}; +} +static u64 level_bits(const struct dcp_present_h17p *wire) { + u64 value=0;for(int i=0;i<8;i++)value|=(u64)wire->swap[0x35e + i]<<(i*8);return value; +} +static double level(const struct dcp_present_h17p *wire) { + u64 bits=level_bits(wire);double value;memcpy(&value,&bits,sizeof(value));return value; +} +static void outside_unchanged(const struct dcp_present_h17p *wire) { + const u8 *data=(const u8 *)wire; + for(size_t i=0;i=0x32f && i<0x33c)assert(!data[i]); + else if(i<0x354||i>=0x3ee)assert(data[i]==0xa5); + } +} +int main(int argc,char **argv) { + struct dcp_present_h17p wire; + if(argc==3) { + memset(&wire,0xa5,sizeof(wire)); + iomfb_encode_backlight_h17p(&wire,strtoul(argv[1],NULL,0),525,atoi(argv[2])); + outside_unchanged(&wire); + for(int i=0x354;i<0x3ee;i++)printf("%02x",wire.swap[i]);puts("");return 0; + } + if(argc==2) { + for(u32 nits=0;nits<510;nits++)for(int update=0;update<2;update++) { + memset(&wire,0xa5,sizeof(wire)); + iomfb_encode_backlight_h17p(&wire,nits,525,update); + outside_unchanged(&wire); + printf("%u %d ",nits,update); + for(int i=0x354;i<0x3ee;i++)printf("%02x",wire.swap[i]);puts(""); + } + return 0; + } + struct apple_dcp dcp=fixture(); + assert(dcp_backlight_register(&dcp)==-ENODATA); + assert(!register_count); + assert(dcp_backlight_takeover(&dcp,509001)==-ERANGE); + assert(!dcp_backlight_active(&dcp)); + assert(dcp_backlight_takeover(&dcp,100000)==0); + assert(dcp.backlight.state.target==100 && dcp.backlight.state.actual==100); + assert(!dcp.backlight.state.dirty && schedule_count==1); + assert(dcp_backlight_takeover(&dcp,400000)==0); + assert(dcp_backlight_takeover(&dcp,0)==0); + assert(dcp.backlight.state.target==100 && schedule_count==1); + assert(dcp_backlight_register(&dcp)==0 && register_count==1); + assert(registered.brightness==100 && registered.max_brightness==509 && !dac_count); + assert(dcp.backlight.state.controlled); + struct iomfb_transaction transaction={.brightness_only=true}; + struct dcp_swap_submit_req_h17p request={.surf_null={0,1,1,1}}; + dcp.iomfb.active=&transaction; + struct apple_dcp unready=fixture(); + memset(&wire,0xa5,sizeof(wire)); + assert(iomfb_apply_backlight_h17p(&unready,&request,&wire)); + assert(!wire.swap[0x354] && !level_bits(&wire)); + assert(wire.swap[0x374] || wire.swap[0x375]); + unready.brightness.maximum=0; + memset(&wire,0xa5,sizeof(wire)); + assert(!iomfb_apply_backlight_h17p(&unready,&request,&wire)); + assert(wire.swap[0x354]==0xa5); + dcp.brightness.maximum=0; + assert(!iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + dcp.brightness.maximum=525; + memset(&wire,0xa5,sizeof(wire)); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + for(int i=0x354;i<0x366;i++)assert(!wire.swap[i]); + assert(!transaction.backlight_reserved); + assert(dcp_bl_request(&dcp.backlight.state,400,false,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==400 && dcp.backlight.state.actual==100); + assert(transaction.backlight_reserved); + assert(!dcp_backlight_complete(&dcp,transaction.backlight.sequence+1,true)); + assert(dcp_bl_request(&dcp.backlight.state,140,false,false)==0); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty && !retire_count); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==140); + iomfb_present_failed_h17p(&dcp); + assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==140 && !retire_count); + for(int i=0;i<20;i++) { + assert(dcp_bl_dpms(&dcp.backlight.state,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==0 && wire.swap[0x354]==1); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp.backlight.state.actual==0 && dcp.backlight.state.target==140); + assert(dcp_bl_dpms(&dcp.backlight.state,true)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==140); + assert(!iomfb_present_complete_h17p(&dcp)); + } + assert(!retire_count); + assert(dcp_bl_dpms(&dcp.backlight.state,false)==0); + assert(dcp_bl_request(&dcp.backlight.state,400,false,false)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==0 && dcp.backlight.state.target==400); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp_bl_dpms(&dcp.backlight.state,true)==0); + assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(level(&wire)==400); + assert(!iomfb_present_complete_h17p(&dcp)); + assert(dcp_bl_request(&dcp.backlight.state,36,false,false)==0); + memset(request.surf_null,1,sizeof(request.surf_null)); + memset(&wire,0xa5,sizeof(wire)); + assert(!iomfb_apply_backlight_h17p(&dcp,&request,&wire)); + assert(wire.swap[0x354]==0xa5 && dcp.backlight.state.dirty); + struct apple_dcp h17g=fixture();h17g.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17G; + assert(dcp_backlight_takeover(&h17g,100000)==-EINVAL); + assert(!h17g.backlight.state.ready); + struct apple_dcp external=fixture();external.panel=false; + assert(dcp_backlight_takeover(&external,100000)==-EINVAL); + struct apple_dcp unlinked=fixture();unlinked.crtc=NULL; + assert(dcp_backlight_takeover(&unlinked,100000)==-EINVAL); + struct apple_dcp zero=fixture(); + assert(dcp_backlight_takeover(&zero,0)==0); + assert(zero.backlight.state.target==0 && !zero.backlight.state.dirty); + puts("Actual takeover/registration/reservation/accepted completion/rejection/20 DPMS cycles passed"); + return 0; +} +''' +source = prelude + '\n'.join(function(f, n) for f, n in functions) + main +(out / 'fixture.c').write_text(source) +argv = [args.cc, '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-fsanitize=address,undefined', + '-I' + str(out / 'include'), '-I' + str(src), str(out / 'fixture.c'), '-o', str(out / 'fixture')] +build = subprocess.run(argv, text=True, capture_output=True) +(out / 'compile.log').write_text(build.stdout + build.stderr) +assert not build.returncode, build.stderr +run = subprocess.run([str(out / 'fixture')], text=True, capture_output=True) +(out / 'run.log').write_text(run.stdout + run.stderr) +assert not run.returncode, run.stdout + run.stderr +# All measured brightness fragments, including DPMS zero and ordinary startup. +golden = json.loads(Path(__file__).with_name('backlight-golden.json').read_text()) +for case in golden: + fragment = subprocess.check_output([str(out / 'fixture'), str(case['nits']), str(int(case['enabled']))], text=True).strip() + assert fragment == case['fragment'], (case, fragment) +# Exact integer binary64 throughout the exposed range, plus disabled updates. +import struct +range_output = subprocess.check_output([str(out / 'fixture'), 'range'], text=True) +for line in range_output.splitlines(): + n, enabled, encoded = line.split() + nits, enabled = int(n), bool(int(enabled)) + expected = bytearray(0x9a) + expected[0x1a:0x22] = expected[0x22:0x2a] = struct.pack(' Date: Thu, 8 Oct 2026 08:34:48 +1100 Subject: [PATCH 155/631] drm: apple: enable the internal H17P pipe once before soft DPMS Internal H17P startup requires one out-of-band power-on request after client open. The soft-DPMS shortcut currently bypasses that first enable, although it is appropriate for later brightness-only transitions. Latch the matching successful enable reply and perform the initial enable before entering soft DPMS. Later unblank operations restore brightness without another power or display-device request. A missing or malformed enable reply stops presentation instead of treating the pipe as ready. Keep the measured H17P pipe live through off, sleep and shutdown entry points: surface-free presents and firmware power-down commands can cause null-base DMA faults or firmware NMI. Stop the software queue on shutdown and retain scanout ownership without claiming a final hardware stop. The older firmware profiles keep their established lifecycle paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 10 ++++++++++ drivers/gpu/drm/apple/iomfb_h17p.c | 1 + drivers/gpu/drm/apple/iomfb_template.c | 11 +++++++++++ 4 files changed, 24 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index f444d50dbcdce1..cd5118f72d4c0a 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -335,6 +335,8 @@ struct apple_dcp { /* Is the DCP booted? */ bool active; + /* A successful initial enable keeps the H17P pipe running across DPMS. */ + bool pipe_enabled_h17p; /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9af2481154ee25..651cfa5a90882b 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1673,6 +1673,16 @@ void dcp_poweron(struct platform_device *pdev) int ret; if (dcp_uses_soft_dpms(dcp)) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !READ_ONCE(dcp->pipe_enabled_h17p)) { + iomfb_poweron_h17p(dcp); + if (!READ_ONCE(dcp->pipe_enabled_h17p)) { + dev_err(dcp->dev, "initial display pipe enable failed\n"); + WRITE_ONCE(dcp->crashed, true); + return; + } + } ret = dcp_backlight_dpms(dcp, true); if (ret) dev_warn(dcp->dev, "backlight restore unavailable: %d\n", ret); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 85d1a4d13ceff7..b140d7ae385fa4 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -539,6 +539,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { void iomfb_start_h17p(struct apple_dcp *dcp) { + WRITE_ONCE(dcp->pipe_enabled_h17p, false); dcp->cb_handlers = cb_handlers; dcp_start_signal(dcp, false, dcp_started, NULL); } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index bc61566bd5f09e..b5d70751121f4e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1594,6 +1594,12 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct dcp_swap_start_req swap_req = { 0 }; struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* The measured profile has no safe pipe-off or surface-free transition. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -1677,6 +1683,11 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; From 59c3bffdf558e04296b3b637e5fbb316699d5a09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:55:15 +1100 Subject: [PATCH 156/631] drm: apple: use ARGB for H17P fbdev H17P applies the stored fourth byte as straight alpha even when the framebuffer uses an X format. A depth-24 fbdev console therefore produces transparent pixels and appears black. Request depth 32 from the DMA fbdev helper only for the H17P internal-panel profile. Older firmware retains its existing XRGB console. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 1b9e1977f1ca37..c1557d3f544812 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -68,6 +68,25 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, + struct drm_fb_helper_surface_size *sizes) +{ + struct drm_crtc *crtc; + + drm_for_each_crtc(crtc, helper->dev) { + struct apple_dcp *dcp; + + dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* H17P applies the stored fourth byte as straight alpha. */ + sizes->surface_bpp = 32; + sizes->surface_depth = 32; + break; + } + } + + return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); /* * A compositor or boot splash is closing the device. drm_release() has * already taken its file off the file list, so the Type-C fabric can tell @@ -81,6 +100,7 @@ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), + .fbdev_probe = apple_drm_fbdev_probe, DRM_FBDEV_DMA_DRIVER_OPS, .postclose = apple_drm_postclose, .name = DRIVER_NAME, From 211396911728f7b5aa8ce51c5e0a1ab0a9502eb5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:56:15 +1100 Subject: [PATCH 157/631] drm: apple: configure opaque X formats on H17P H17P blends the stored X byte as alpha. Linux boundary measurements show that property 0x49 set to zero and the existing premultiplied surface marker make 8-bit XRGB scanout opaque without changing ARGB alpha. Describe the measured A352 request and send it as a priority item on the serialized H17P queue after the first completed present of each full power-on. Keep unqualified 10-bit XRGB off the H17P primary plane, and cover the admitted surface offsets, IOVA tail and property bytes with KUnit fixtures. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Makefile | 1 + drivers/gpu/drm/apple/apple_drv.c | 8 ++- drivers/gpu/drm/apple/dcp-internal.h | 4 ++ drivers/gpu/drm/apple/iomfb.c | 84 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_h17p.c | 13 ++++ drivers/gpu/drm/apple/iomfb_h17p.h | 15 +++++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 84 +++++++++++++++++++++++++ drivers/gpu/drm/apple/plane.c | 23 ++++++- drivers/gpu/drm/apple/plane.h | 1 + tools/testing/apple/check-backlight.py | 13 +++- 10 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_h17p_test.c diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index c78069509905c0..4a4b7dba9372b7 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -17,6 +17,7 @@ appledrm-y += iomfb_v14_7_tpl.o appledrm-y += iomfb_v14_7.o iomfb_v14_7_link.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o appledrm-y += iomfb_h17p.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c1557d3f544812..d917f8404fa0ef 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -515,6 +515,8 @@ static int apple_probe_per_dcp(struct device *dev, u32 surf; int zpos = 0; bool supports_l10r = !dcp_fw_compat_is_12_x(dcp); + struct apple_dcp *dcp_data = platform_get_drvdata(dcp); + bool supports_xrgb2101010; enum drm_plane_type plane_type; if (dcp_fw_compat_is_14_7(dcp)) { @@ -527,13 +529,17 @@ static int apple_probe_per_dcp(struct device *dev, return ret; zpos = 1; } + supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || + dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { if (dcp_fw_compat_is_14_7(dcp)) break; plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; planes[zpos] = apple_plane_init(drm, 1U << num, surf, - supports_l10r, plane_type); + supports_l10r, + supports_xrgb2101010, + plane_type); if (IS_ERR(planes[zpos])) return PTR_ERR(planes[zpos]); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index cd5118f72d4c0a..09a4619576d74e 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -523,6 +523,7 @@ struct apple_dcp { unsigned int queued; bool stopped; bool backlight_queued; + atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; struct iomfb_scanout_h17p *next_scanout; } iomfb; @@ -536,6 +537,9 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp); bool iomfb_apply_backlight_h17p(struct apple_dcp *dcp, const struct dcp_swap_submit_req_h17p *request, struct dcp_present_h17p *wire); +void iomfb_apply_opaque_x_h17p(struct apple_dcp *dcp, dcp_callback_t callback, + void *cookie); +void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp); int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b999da3571f046..288ee29576a257 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -172,6 +172,46 @@ static bool iomfb_channels_idle(struct apple_dcp *dcp) !dcp->ch_async.depth && !dcp->ch_oobasync.depth; } +enum iomfb_opaque_x_state { + IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED, + IOMFB_OPAQUE_X_QUEUED, + IOMFB_OPAQUE_X_READY, +}; + +struct iomfb_opaque_x_transaction { + struct iomfb_transaction transaction; +}; + +static void iomfb_opaque_x_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction); +static void iomfb_opaque_x_release(struct iomfb_transaction *transaction); + +static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) +{ + struct iomfb_opaque_x_transaction *opaque; + + lockdep_assert_held(&dcp->iomfb.lock); + if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) + return 0; + if (dcp->iomfb.queued >= 32) + return -EBUSY; + opaque = kzalloc_obj(*opaque); + if (!opaque) + return -ENOMEM; + if (atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_NEEDED, + IOMFB_OPAQUE_X_QUEUED) != IOMFB_OPAQUE_X_NEEDED) { + kfree(opaque); + return 0; + } + + opaque->transaction.start = iomfb_opaque_x_start; + opaque->transaction.release = iomfb_opaque_x_release; + dcp->iomfb.queued++; + list_add(&opaque->transaction.link, &dcp->iomfb.pending); + return 0; +} + static void iomfb_discard_pending(struct apple_dcp *dcp) { struct iomfb_transaction *transaction, *next; @@ -209,6 +249,12 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) schedule_work(&dcp->bl_update_wq); transaction->release(transaction); } + if (iomfb_enqueue_opaque_x(dcp)) { + WRITE_ONCE(dcp->crashed, true); + iomfb_discard_pending(dcp); + schedule_work(&dcp->vblank_wq); + return; + } if (!list_empty(&dcp->iomfb.pending)) schedule_work(&dcp->iomfb.work); } @@ -260,12 +306,48 @@ static void iomfb_queue_timeout(struct work_struct *work) mutex_unlock(&dcp->iomfb.lock); } +static void iomfb_opaque_x_complete(struct apple_dcp *dcp, void *out, + void *cookie) +{ + u32 status = out ? *(u32 *)out : ~0U; + + (void)cookie; + + if (status) { + dev_err(dcp->dev, "opaque X property failed: %u\n", status); + WRITE_ONCE(dcp->crashed, true); + schedule_work(&dcp->vblank_wq); + return; + } + + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_READY); +} + +static void iomfb_opaque_x_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + (void)transaction; + iomfb_apply_opaque_x_h17p(dcp, iomfb_opaque_x_complete, NULL); +} + +static void iomfb_opaque_x_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_opaque_x_transaction, + transaction)); +} + +void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp) +{ + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); + atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } void iomfb_queue_stop(struct apple_dcp *dcp) @@ -1411,6 +1493,8 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) if (transaction->brightness_only) return false; } + atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED); iomfb_scanout_complete_h17p(dcp); return true; } diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index b140d7ae385fa4..fca112eb4ecf2e 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -210,6 +210,19 @@ static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { #include "iomfb_template.c" +void iomfb_apply_opaque_x_h17p(struct apple_dcp *dcp, dcp_callback_t callback, + void *cookie) +{ + static const struct dcp_method_entry method = { + .name = "apply_property", + .tag = { 'A', '3', '5', '2' }, + }; + struct dcp_apply_property_h17p property = dcp_opaque_x_property_h17p(); + + dcp_push(dcp, false, &method, sizeof(property), sizeof(u32), &property, + callback, cookie); +} + static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index e4eb5be2b01438..94739402502974 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -67,6 +67,21 @@ static_assert(sizeof(struct dcp_present_h17p) == 0xe9c); static_assert(offsetof(struct dcp_present_h17p, surf) == 0x588); static_assert(offsetof(struct dcp_present_h17p, tail) == 0xe38); +struct dcp_apply_property_h17p { + u32 property; + u32 value; +} __packed; + +static_assert(sizeof(struct dcp_apply_property_h17p) == 0x8); + +static inline struct dcp_apply_property_h17p dcp_opaque_x_property_h17p(void) +{ + return (struct dcp_apply_property_h17p) { + .property = 0x49, + .value = 0, + }; +} + void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, u32 maximum, bool update); diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c new file mode 100644 index 00000000000000..c2a71ffa8cdbf2 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include +#include + +#include "iomfb_h17p.h" + +static void h17p_surface_wire_test(struct kunit *test) +{ + struct dcp_swap_submit_req_h17p *request; + struct dcp_present_h17p *wire; + struct dcp_surface *surface; + u64 iova = 0x123456789aULL; + + request = kunit_kzalloc(test, sizeof(*request), GFP_KERNEL); + wire = kunit_kmalloc(test, sizeof(*wire), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, request); + KUNIT_ASSERT_NOT_NULL(test, wire); + surface = &request->surf[0].base; + memset(wire, 0xa5, sizeof(*wire)); + surface->is_premultiplied = 1; + surface->plane_cnt = 1; + surface->plane_cnt2 = 1; + surface->format = DCP_FORMAT_BGRA; + surface->xfer_func = DCP_XFER_FUNC_SDR; + surface->colorspace = DCP_COLORSPACE_NATIVE; + surface->stride = 10176; + surface->width = 2408; + surface->height = 1506; + surface->buf_size = 10176 * 1506; + surface->planes[0].width = 2408; + surface->planes[0].height = 1506; + surface->planes[0].stride = 10176; + surface->planes[0].size = 10176 * 1506; + request->surf_iova[0] = iova; + request->surf_null[0] = false; + request->surf_null[1] = true; + request->surf_null[2] = true; + request->surf_null[3] = true; + + iomfb_serialize_present_h17p(wire, request); + + KUNIT_EXPECT_MEMEQ(test, wire->swap, &request->swap, sizeof(wire->swap)); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x593), + DCP_FORMAT_BGRA); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x59d), 10176U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5a9), 2408U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5ad), 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5b1), + 10176U * 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e1), 2408U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e5), 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f1), 10176U); + KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f5), + 10176U * 1506U); + KUNIT_EXPECT_EQ(test, get_unaligned_le64((u8 *)wire + 0xe38), iova); + KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 2], 0U); + KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 1], 0U); +} + +static void h17p_opaque_x_parameter_test(struct kunit *test) +{ + struct dcp_apply_property_h17p property = dcp_opaque_x_property_h17p(); + + KUNIT_EXPECT_EQ(test, property.property, 0x49U); + KUNIT_EXPECT_EQ(test, property.value, 0U); +} + +static struct kunit_case h17p_present_cases[] = { + KUNIT_CASE(h17p_surface_wire_test), + KUNIT_CASE(h17p_opaque_x_parameter_test), + {} +}; + +static struct kunit_suite h17p_present_suite = { + .name = "apple-dcp-h17p-present", + .test_cases = h17p_present_cases, +}; + +kunit_test_suite(h17p_present_suite); + +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index 93ddde4182b0fc..de525e2b80ba76 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -427,6 +427,23 @@ static const u32 dcp_primary_formats[] = { #endif }; +/* H17P 10-bit XRGB alpha semantics have not been qualified. */ +static const u32 dcp_primary_formats_h17p[] = { + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XBGR8888, + DRM_FORMAT_ABGR8888, + DRM_FORMAT_NV12, + DRM_FORMAT_NV16, + DRM_FORMAT_NV24, + DRM_FORMAT_P010, + DRM_FORMAT_P210, +#if defined(DRM_FORMAT_P410) + DRM_FORMAT_P410, +#endif +}; + static const u32 dcp_overlay_formats[] = { DRM_FORMAT_ARGB2101010, DRM_FORMAT_ARGB8888, @@ -481,6 +498,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, bool supports_l10r, + bool supports_xrgb2101010, enum drm_plane_type type) { struct apple_plane *plane; @@ -489,9 +507,12 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, switch (type) { case DRM_PLANE_TYPE_PRIMARY: - if (supports_l10r) { + if (supports_l10r && supports_xrgb2101010) { fmts = dcp_primary_formats; num_fmts = ARRAY_SIZE(dcp_primary_formats); + } else if (supports_l10r) { + fmts = dcp_primary_formats_h17p; + num_fmts = ARRAY_SIZE(dcp_primary_formats_h17p); } else { fmts = dcp_primary_formats_12_x; num_fmts = ARRAY_SIZE(dcp_primary_formats_12_x); diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index 7ed68b7b5b38d0..ebb2420bf4041d 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -36,6 +36,7 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, bool supports_l10r, + bool supports_xrgb2101010, enum drm_plane_type type); struct drm_plane *apple_plane_init_v14_7(struct drm_device *dev, unsigned long possible_crtcs, diff --git a/tools/testing/apple/check-backlight.py b/tools/testing/apple/check-backlight.py index e1df3dc3458de4..2a1f6412c9639c 100644 --- a/tools/testing/apple/check-backlight.py +++ b/tools/testing/apple/check-backlight.py @@ -79,7 +79,7 @@ def function(file, name): int bl_register_mutex,bl_register_wq; struct { u32 maximum;u32 nits;struct backlight_device *bl_dev;u32 dac; } brightness; struct { int lock;struct dcp_backlight_state state;void (*kick)(struct apple_dcp *); } backlight; - struct { int lock;struct iomfb_transaction *active; } iomfb; + struct { int lock;struct iomfb_transaction *active;int opaque_x_state; } iomfb; }; static int schedule_count,register_count,retire_count,dac_count; static struct backlight_properties registered; @@ -100,6 +100,10 @@ def function(file, name): static void put_unaligned_le64(u64 v,void *p) { for(int i=0;i<8;i++)((u8 *)p)[i]=v>>(8*i); } static void put_unaligned_le32(u32 v,void *p) { for(int i=0;i<4;i++)((u8 *)p)[i]=v>>(8*i); } static void put_unaligned_le16(uint16_t v,void *p) { for(int i=0;i<2;i++)((u8 *)p)[i]=v>>(8*i); } +enum { IOMFB_OPAQUE_X_WAITING, IOMFB_OPAQUE_X_NEEDED }; +static int atomic_cmpxchg(int *value,int expected,int desired) { + int old=*value;if(old==expected)*value=desired;return old; +} int iomfb_configure_backlight_h17p(struct apple_dcp *,u32,bool,u32,bool,u32); ''' functions = [ @@ -194,6 +198,7 @@ def function(file, name): assert(dcp_bl_request(&dcp.backlight.state,140,false,false)==0); assert(!iomfb_present_complete_h17p(&dcp)); assert(dcp.backlight.state.actual==400 && dcp.backlight.state.dirty && !retire_count); + assert(dcp.iomfb.opaque_x_state==IOMFB_OPAQUE_X_WAITING); assert(iomfb_apply_backlight_h17p(&dcp,&request,&wire)); assert(level(&wire)==140); iomfb_present_failed_h17p(&dcp); @@ -237,6 +242,12 @@ def function(file, name): struct apple_dcp zero=fixture(); assert(dcp_backlight_takeover(&zero,0)==0); assert(zero.backlight.state.target==0 && !zero.backlight.state.dirty); + transaction = (struct iomfb_transaction){.brightness_only=false}; + dcp.iomfb.active = &transaction; + dcp.iomfb.opaque_x_state = IOMFB_OPAQUE_X_WAITING; + assert(iomfb_present_complete_h17p(&dcp)); + assert(dcp.iomfb.opaque_x_state == IOMFB_OPAQUE_X_NEEDED); + puts("Actual takeover/registration/reservation/accepted completion/rejection/20 DPMS cycles passed"); return 0; } From 76fa0180e026680f8164b027458aaffbf8b0cc52 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 07:55:15 +1100 Subject: [PATCH 158/631] drm: apple: bound H17P firmware buffer mappings H17P allocates firmware buffers through the DCP device DMA domain, then requests the same scatterlist and IOVA in the piodma child domain. The secondary mapping is read/write. Allocation failures receive an all-zero reply; rejected maps echo the descriptor with a zero device address; normal lifetime unmaps the secondary translation before releasing backing pages. Decode and encode the packed H17P records explicitly in little endian. Admit the unmap and release envelopes, reject stale and duplicate descriptors, and roll back short map results. Keep ownership when unmap is short, and make release remove any live secondary translation before freeing the scatterlist and coherent allocation. This bounds descriptor exhaustion, allocation and scatterlist failures, partial maps and unmaps, duplicate operations, invalid IDs, and release without an earlier unmap while retaining the existing older-firmware paths. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 99 +++++++++++++++++++++----- drivers/gpu/drm/apple/iomfb_template.c | 85 +++++++++++++++++++--- 2 files changed, 157 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index fca112eb4ecf2e..6fc2567c173106 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -40,6 +40,7 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [128] = { 0x1008, 0x4, true }, [129] = { 0x40, 0x4, true }, [201] = { 0xc, 0x10, true }, + [202] = { 0x1a, 0x0, true }, [206] = { 0x0, 0x4, true }, [207] = { 0x0, 0x4, true }, [300] = { 0x10, 0x0, true }, @@ -51,6 +52,7 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [414] = { 0x50, 0x4, true }, [415] = { 0x4c, 0x4, true }, [451] = { 0x14, 0x1c, true }, + [454] = { 0x4, 0x1, true }, [552] = { 0x1044, 0x4, true }, [561] = { 0x1044, 0x4, true }, [563] = { 0x4c, 0x4, true }, @@ -439,7 +441,6 @@ static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, const struct dcp_allocate_buffer_req *wire = in; struct dcp_allocate_buffer_req request = { 0 }; struct dcp_allocate_buffer_resp reply; - u32 options; if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) return trampoline_allocate_buffer(dcp, tag, out, in); @@ -451,29 +452,91 @@ static bool trampoline_allocate_buffer_h17p(struct apple_dcp *dcp, int tag, static_assert(offsetof(struct dcp_allocate_buffer_req, paddr_null) == 16); trace_iomfb_callback(dcp, tag, __func__); - options = get_unaligned_le32(&wire->unk2); - /* Only these option combinations have observed successful replies. */ - if (get_unaligned_le32(&wire->unk0) != 0x703 || - (options != 4 && options != SZ_16K) || - memchr_inv(&wire->paddr_null, 0, 4)) - goto fail; - request.size = get_unaligned_le64(&wire->size); reply = dcpep_cb_allocate_buffer(dcp, &request); - if (!reply.dva_size) - goto fail; + if (!reply.dva_size) { + /* Exhaustion and allocation errors use the all-zero reply. */ + memset(out, 0, sizeof(reply)); + return true; + } put_unaligned_le64(reply.paddr, out); put_unaligned_le64(reply.dva, (u8 *)out + 8); put_unaligned_le64(reply.dva_size, (u8 *)out + 16); put_unaligned_le32(reply.mem_desc_id, (u8 *)out + 24); return true; +} -fail: - /* Allocation failures have no measured H17P reply encoding. */ - dev_err(dcp->dev, "unqualified or failed D451 buffer allocation\n"); - WRITE_ONCE(dcp->crashed, true); - return false; +static bool trampoline_map_piodma_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_map_buf_req *wire = in; + struct dcp_map_buf_req request = { 0 }; + struct dcp_map_buf_resp_h17p reply = { 0 }; + u64 buffer; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_map_piodma(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0xc); + static_assert(sizeof(reply) == 0x10); + static_assert(offsetof(struct dcp_map_buf_req, unk) == 8); + static_assert(offsetof(struct dcp_map_buf_req, dva_null) == 11); + + trace_iomfb_callback(dcp, tag, __func__); + buffer = get_unaligned_le64(&wire->buffer); + request.buffer = buffer; + if (!dcpep_map_piodma(dcp, &request, &reply)) { + /* A rejected mapping echoes the descriptor with a zero DVA. */ + put_unaligned_le32(buffer, out); + put_unaligned_le64(0, (u8 *)out + 4); + put_unaligned_le32(0, (u8 *)out + 12); + return true; + } + + put_unaligned_le32(reply.buffer, out); + put_unaligned_le64(reply.dva, (u8 *)out + 4); + put_unaligned_le32(reply.unk, (u8 *)out + 12); + return true; +} + +static bool trampoline_unmap_piodma_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_unmap_buf_resp *wire = in; + struct dcp_unmap_buf_resp request = { 0 }; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_unmap_piodma(dcp, tag, out, in); + + static_assert(sizeof(*wire) == 0x1a); + static_assert(offsetof(struct dcp_unmap_buf_resp, vaddr) == 8); + static_assert(offsetof(struct dcp_unmap_buf_resp, dva) == 16); + static_assert(offsetof(struct dcp_unmap_buf_resp, buf_null) == 25); + + trace_iomfb_callback(dcp, tag, __func__); + request.buffer = get_unaligned_le64(&wire->buffer); + request.vaddr = get_unaligned_le64(&wire->vaddr); + request.dva = get_unaligned_le64(&wire->dva); + request.unk = wire->unk; + request.buf_null = wire->buf_null; + dcpep_unmap_piodma(dcp, &request); + return true; +} + +static bool +trampoline_release_mem_desc_h17p(struct apple_dcp *dcp, int tag, void *out, + void *in) +{ + u32 id; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return trampoline_release_mem_desc(dcp, tag, out, in); + + trace_iomfb_callback(dcp, tag, __func__); + id = get_unaligned_le32(in); + *(u8 *)out = dcpep_cb_release_mem_desc(dcp, &id); + return true; } /* H17P callback numbering is not a uniform shift of the v13.5 table. */ @@ -508,8 +571,8 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ [129] = trampoline_prop_end, /* setDCPAVPropEnd */ [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ - [201] = trampoline_map_piodma, /* map_buf */ - [202] = trampoline_unmap_piodma, /* unmap_buf */ + [201] = trampoline_map_piodma_h17p, /* map_buf */ + [202] = trampoline_unmap_piodma_h17p, /* unmap_buf */ [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ [208] = trampoline_nop, /* update_backlight_factor_prop */ @@ -526,7 +589,7 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [415] = trampoline_true, /* setProperty */ [451] = trampoline_allocate_buffer_h17p, /* allocate_buffer */ [452] = trampoline_map_physical, /* prepare */ - [454] = trampoline_release_mem_desc, /* release_descriptor */ + [454] = trampoline_release_mem_desc_h17p, /* release_descriptor */ [552] = trampoline_true, [561] = trampoline_true, [563] = trampoline_true, diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b5d70751121f4e..1495e4d3de7174 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -494,6 +494,7 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, { struct dcp_mem_descriptor *memdesc; struct sg_table *map; + size_t size; ssize_t ret; if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) @@ -501,6 +502,7 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, memdesc = &dcp->memdesc[req->buffer]; map = &memdesc->map; + size = ALIGN(memdesc->size, SZ_16K); if (!map->sgl) goto reject; @@ -518,29 +520,86 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, } #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* remembered so release_mem_desc() can tear a stale mapping down */ memdesc->piodma_mapped = true; /* the H17P reply echoes the buffer index */ - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .buffer = req->buffer, .dva = memdesc->dva, }; #else - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .dva = memdesc->dva }; + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .dva = memdesc->dva }; #endif + return true; reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); + return false; +} + +static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct DCP_FW_NAME(dcp_map_buf_resp) resp = { 0 }; + + if (dcpep_map_piodma(dcp, req, &resp)) + return resp; + #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ - .buffer = req->buffer, - .dva = 0, - }; + resp.buffer = req->buffer; #else - return (struct DCP_FW_NAME(dcp_map_buf_resp)){ .ret = EINVAL }; + resp.ret = EINVAL; #endif + return resp; +} + +static bool dcpep_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *request) +{ + struct dcp_mem_descriptor *memdesc; + size_t size, unmapped; + + if (request->buffer >= ARRAY_SIZE(dcp->memdesc)) { + dev_warn(dcp->dev, "unmap request for out of range buffer %llu\n", + request->buffer); + return false; + } + + memdesc = &dcp->memdesc[request->buffer]; + + if (!test_bit(request->buffer, dcp->memdesc_map) || !memdesc->buf || + !dcp->iommu_dom) + goto not_mapped; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (!memdesc->piodma_mapped) + goto not_mapped; +#endif + + if (memdesc->dva != request->dva) { + dev_warn(dcp->dev, + "unmap buffer %llu address mismatch memdesc.dva:%llx dva:%llx\n", + request->buffer, + memdesc->dva, request->dva); + return false; + } + + /* use the piodma iommu domain to unmap from the right IOMMU */ + size = ALIGN(memdesc->size, SZ_16K); + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, "iommu_unmap() returned %zu instead of %zu\n", + unmapped, size); + return false; + } + memdesc->piodma_mapped = false; + return true; + +not_mapped: + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx\n", + request->buffer, request->dva); + return false; } static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, @@ -683,7 +742,15 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) dev_warn(dcp->dev, "releasing buffer %u while still mapped for piodma; unmapping first\n", id); - iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (!dcp->iommu_dom) + return 0; + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, + "failed to unmap buffer %u before release: %zu of %zu\n", + id, unmapped, size); + return 0; + } memdesc->piodma_mapped = false; } From 97c0fe18587bd8e030ac4900c89d063e089cb446 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 08:46:32 +1100 Subject: [PATCH 159/631] drm: apple: retain J700 provider replies with older loaders The working J700 loader predates the display power-table DT properties. Requiring only those new properties prevents the firmware's default framebuffer setup before a compatible loader can be installed. Provide the J700 table bytes when the internal T8140 endpoint lacks a property and the machine is specifically J700. Explicit per-device DT data always wins; malformed present properties still fail validation. Other boards keep the loader-data requirement. The byte arrays match both the retained disp0 ADT and firmware replies; no physical brightness range or power-model interpretation is added. New loaders can forward all five properties verbatim using the existing DT interface, removing the need for this compatibility path. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 51 ++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 6fc2567c173106..2de0fbbd4620a6 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -313,19 +313,49 @@ struct dcp_h17p_provider_property { const char *name; const char *dt_name; u32 length; + const u8 *legacy_data; }; static_assert(sizeof(struct dcp_h17p_provider_request) == 0x4c); static_assert(sizeof(struct dcp_h17p_provider_reply) == 0xc04); +/* Compatibility data matches the independently decoded J700 disp0 ADT. */ +static const u8 j700_provider_data_0[] = { + 0x0d, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_1[] = { + 0x01, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_2[] = { + 0x02, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, + 0x64, 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, + 0xfa, 0x00, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, + 0x90, 0x01, 0x00, 0x00, 0xc2, 0x01, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, + 0x0d, 0x02, 0x00, 0x00 +}; + +static const u8 j700_provider_data_3[] = { + 0x64, 0x00, 0x00, 0x00 +}; + +static const u8 j700_provider_data_4[] = { + 0x34, 0x03, 0x00, 0x00, 0x84, 0x03, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, + 0x52, 0x07, 0x00, 0x00, 0x11, 0x09, 0x00, 0x00, 0xde, 0x0a, 0x00, 0x00, + 0xd2, 0x0c, 0x00, 0x00, 0xe2, 0x0e, 0x00, 0x00, 0xfb, 0x10, 0x00, 0x00, + 0x64, 0x13, 0x00, 0x00, 0xd5, 0x15, 0x00, 0x00, 0x68, 0x18, 0x00, 0x00, + 0xa2, 0x19, 0x00, 0x00 +}; + /* Only these provider transfers have been measured against the disp0 ADT. */ static const struct dcp_h17p_provider_property provider_properties[] = { - { "power-lut-data-x", "apple,power-lut-data-x", 4 }, - { "power-lut-data-y", "apple,power-lut-data-y", 4 }, - { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52 }, - { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4 }, - { "power-lut-data-lut", "apple,power-lut-data-lut", 52 }, - { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0 }, + { "power-lut-data-x", "apple,power-lut-data-x", 4, j700_provider_data_0 }, + { "power-lut-data-y", "apple,power-lut-data-y", 4, j700_provider_data_1 }, + { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52, j700_provider_data_2 }, + { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4, j700_provider_data_3 }, + { "power-lut-data-lut", "apple,power-lut-data-lut", 52, j700_provider_data_4 }, + { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0, NULL }, }; static const struct dcp_h17p_provider_property * @@ -371,6 +401,15 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, /* Only this absent property has a measured empty reply. */ if (of_property_present(node, property->dt_name)) goto fail; + } else if (!of_property_present(node, property->dt_name) && + of_machine_is_compatible("apple,j700") && + of_device_is_compatible(node, "apple,t8140-dcp")) { + /* Qualified older loaders lack the newly described table properties. */ + memset(reply, 0, sizeof(*reply)); + memcpy(reply->data, property->legacy_data, property->length); + reply->length = cpu_to_le32(property->length); + dev_warn_once(dcp->dev, "using J700 provider data for an older loader\n"); + return true; } else if (length != (int)property->length) { dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", property->dt_name, length); From a9bc153bdc7d4243ba76828e72943976009f281c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 11:15:00 +1100 Subject: [PATCH 160/631] MAINTAINERS: include Apple display tests and documentation Cover the backlight documentation and generic display replay tests under the existing Apple display driver entry so changes reach its reviewers together with the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index a0e00d4c6f86b4..974f50efb5d471 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1955,9 +1955,11 @@ F: Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml +F: Documentation/gpu/apple-dcp-backlight.rst F: drivers/gpu/drm/apple/ F: drivers/gpu/drm/apple/dcp-fabric* F: include/linux/soc/apple/dp-tunnel.h +F: tools/testing/apple/ APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier From 78912e8dc99539a8d00f09141a7629ea5f645af0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 10:37:24 +1100 Subject: [PATCH 161/631] drm: apple: accept the legacy J700 primary surface selection The qualified J700 loader passes one zero in apple,iomfb-surfaces to select firmware surface zero. The per-slot flag parser instead leaves the bitmap empty, which registers a CRTC without a primary plane and crashes fbdev initialization. Recognize this exact legacy encoding only on J700 internal T8140 DCP, and reject empty surface masks before CRTC creation on every profile. Also check the CRTC allocation before use. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 4 ++++ drivers/gpu/drm/apple/dcp.c | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d917f8404fa0ef..30f15f32387b24 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -531,6 +531,8 @@ static int apple_probe_per_dcp(struct device *dev, } supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; + if (bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) + return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { if (dcp_fw_compat_is_14_7(dcp)) @@ -557,6 +559,8 @@ static int apple_probe_per_dcp(struct device *dev, * knows what to do with overlays. */ crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); + if (!crtc) + return -ENOMEM; ret = drm_crtc_init_with_planes(drm, &crtc->base, planes[0], NULL, &apple_crtc_funcs, NULL); if (ret) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 651cfa5a90882b..6f8e39723998a1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2934,6 +2934,13 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) set_bit(surf, dcp->iomfb_surfaces); surf++; } + /* The qualified J700 loader names surface zero rather than enabling it. */ + if (of_machine_is_compatible("apple,j700") && + of_device_is_compatible(dev->of_node, "apple,t8140-dcp") && + num_surfs == 1 && + !of_property_read_u32(dev->of_node, "apple,iomfb-surfaces", &surf_en) && + surf_en == 0) + set_bit(0, dcp->iomfb_surfaces); if (dcp->phy) { int ret; From 6e1d952f6893b5199722e2e316fe8f33eb253a97 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 10:37:25 +1100 Subject: [PATCH 162/631] dt-bindings: display: apple: document legacy J700 surface selection The qualified J700 bootloader uses one zero to select the primary surface, rather than the per-slot enable flags used by other loaders. Document that existing compatibility encoding. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index aefb570de5983d..bba5d64f504517 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -121,7 +121,9 @@ properties: maxItems: 4 items: enum: [0, 1] - description: Per-slot enable flags, in slot order. + description: | + Per-slot enable flags, in slot order. The legacy J700 bootloader + supplies a single zero to select surface zero instead. apple,power-lut-data-x: $ref: /schemas/types.yaml#/definitions/uint8-array From 78dadf3c9200eb6dd0860747121ef944c410d8ee Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 11:54:21 +1100 Subject: [PATCH 163/631] drm: apple: retain adopted H17P DMA resources through reboot Internal H17P keeps its display pipe running when the panel is blanked. Its shutdown path stops the outbound queue without a firmware-acknowledged scanout stop. Deleting the component nevertheless releases the binding's devres group, including RTKit shared buffers, and destroys a host-created piodma device. The running firmware can still access those resources. During system shutdown, stop host submissions and backlight work while keeping the adopted component and its DMA resources allocated until reset. Keep the existing component teardown for the older firmware profiles. This does not qualify component removal or a firmware client-close method. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6f8e39723998a1..45b4cba2e0d5e2 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3233,6 +3233,22 @@ static void dcp_platform_shutdown(struct platform_device *pdev) if (dcp && dcp->external && !dcp->external_native) return; + if (dcp && dcp->hw.adopt_live_session && + dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* + * This firmware keeps scanning across soft DPMS and has no + * qualified stop sequence. Component unbind releases the RTKit + * devres group and the piodma domain while firmware may still + * access them. Stop host submissions but keep the component and + * its DMA resources alive until the system resets. + */ + iomfb_queue_stop(dcp); + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + return; + } + component_del(&pdev->dev, &dcp_comp_ops); } From 0236abfbacf0ed10cb00d8665dbaf59e5123c02a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:12:00 +1100 Subject: [PATCH 164/631] drm: apple: retain live H17P resources across component teardown An adopted H17P session keeps scanning after soft DPMS. Component unbind and bind-error unwind release the RTKit devres group even though firmware has not acknowledged a DMA stop. Driver removal can also drop its default IOMMU-domain usage and release AFK rings and consumer contexts. Export the existing default-domain usage helpers for module callers. Own that usage in the DCP driver and retain it, RTKit shared buffers, AFK rings, PIODMA and the referenced DRM objects until reset once negotiation with the live firmware starts. Quiesce host callbacks and AFK work without sending an unqualified firmware stop. Keep their contexts valid through managed cleanup and unplug the DRM device before teardown. Pin the DMA device, IOMMU group and module for that firmware lifetime, reject rebinding the same firmware node with an explicit reboot-required message, and keep the existing resource releases for profiles with a normal stop path. Pre-negotiation probe failures still release their resources. Add a native regression exercising the actual component/master unbind and resource-release functions through simulated component and driver devres cleanup, including partial initialization and legacy release behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 41 +++++- drivers/gpu/drm/apple/afk.h | 1 + drivers/gpu/drm/apple/dcp-internal.h | 4 + drivers/gpu/drm/apple/dcp.c | 147 ++++++++++++++++++- drivers/gpu/drm/apple/dcp_backlight.c | 8 ++ drivers/iommu/iommu.c | 8 +- tools/testing/apple/check-lifetime.py | 199 ++++++++++++++++++++++++++ 7 files changed, 398 insertions(+), 10 deletions(-) create mode 100644 tools/testing/apple/check-lifetime.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 4cb3720979470a..864677287e79a8 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -58,13 +59,22 @@ static void afk_send(struct apple_dcp_afkep *ep, u64 message) dcp_send_message(ep->dcp, ep->endpoint, message); } +static void afk_release_context(void *data) +{ + struct apple_dcp_afkep *afkep = data; + + afk_quiesce(afkep); + if (!afkep->dcp->retain_dma) + kfree(afkep); +} + struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops) { struct apple_dcp_afkep *afkep; int ret, i; - afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); + afkep = kzalloc_obj(*afkep); if (!afkep) return ERR_PTR(-ENOMEM); @@ -78,21 +88,34 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, goto out_free_afkep; } - // TODO: devm_ for wq - init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); for (i = 0; i < AFK_MAX_CHANNEL; i++) spin_lock_init(&afkep->services[i].lock); + ret = devm_add_action_or_reset(dcp->dev, afk_release_context, afkep); + if (ret) + return ERR_PTR(ret); return afkep; out_free_afkep: - devm_kfree(dcp->dev, afkep); + kfree(afkep); return ERR_PTR(ret); } +void afk_quiesce(struct apple_dcp_afkep *afkep) +{ + if (IS_ERR_OR_NULL(afkep)) + return; + if (afkep->wq) { + destroy_workqueue(afkep->wq); + afkep->wq = NULL; + } + debugfs_remove_recursive(afkep->debugfs_entry); + afkep->debugfs_entry = NULL; +} + void afk_shutdown(struct apple_dcp_afkep *afkep) { unsigned long flags; @@ -105,6 +128,7 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); } + afk_quiesce(afkep); spin_lock_irqsave(&afkep->lock, flags); afkep->stopping = true; spin_unlock_irqrestore(&afkep->lock, flags); @@ -184,8 +208,11 @@ static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) return; } - ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, - GFP_KERNEL); + /* An unstopped adopted session owns this ring until the machine resets. */ + if (READ_ONCE(ep->dcp->retain_dma)) + ep->bfr = dma_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); + else + ep->bfr = dmam_alloc_coherent(ep->dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); if (!ep->bfr) { dev_err(ep->dcp->dev, "Failed to allocate %d bytes buffer\n", size); @@ -1125,6 +1152,8 @@ static int afk_send_command_full(struct apple_epic_service *service, u8 type, if (!service) return -ENODEV; ep = service->ep; + if (READ_ONCE(ep->dcp->quiescing)) + return -ENODEV; rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index d9cee6bdb2a43f..04c612d8e6e17b 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -328,6 +328,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops); int afk_start(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); +void afk_quiesce(struct apple_dcp_afkep *ep); int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 09a4619576d74e..ff57061622e8b7 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -192,6 +192,10 @@ struct dcp_hdmi; struct apple_dcp { struct dcp_fabric_pipeline fabric; + /* An adopted session has no confirmed firmware DMA stop boundary. */ + bool retain_dma; + bool quiescing; + bool drm_retained; struct device *dev; struct platform_device *piodma; bool piodma_created; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 45b4cba2e0d5e2..c2de7902dfee72 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include #include +#include #include #include #include @@ -56,6 +58,83 @@ #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +struct dcp_session_ref { + struct list_head link; + struct device *dev; + struct iommu_group *group; +}; + +static LIST_HEAD(dcp_retained_sessions); +static DEFINE_MUTEX(dcp_retained_sessions_lock); + +static bool dcp_session_retained(struct device *dev) +{ + struct dcp_session_ref *session; + bool retained = false; + + mutex_lock(&dcp_retained_sessions_lock); + list_for_each_entry(session, &dcp_retained_sessions, link) { + if (dev_fwnode(session->dev) == dev_fwnode(dev)) { + retained = true; + break; + } + } + mutex_unlock(&dcp_retained_sessions_lock); + return retained; +} + +static int dcp_pin_live_session(struct apple_dcp *dcp) +{ + struct dcp_session_ref *session; + + if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) + return 0; + session = kzalloc_obj(*session); + if (!session) + return -ENOMEM; + + /* Keep the DMA device, domain usage and driver code valid until reset. */ + session->dev = get_device(dcp->dev); + session->group = iommu_group_get(dcp->dev); + __module_get(THIS_MODULE); + WRITE_ONCE(dcp->retain_dma, true); + mutex_lock(&dcp_retained_sessions_lock); + list_add_tail(&session->link, &dcp_retained_sessions); + mutex_unlock(&dcp_retained_sessions_lock); + dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); + return 0; +} + +static void dcp_release_dma_domain(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + iommu_device_unuse_default_domain(dcp->dev); +} + +static void dcp_release_context(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + kfree(dcp); +} + +static void dcp_release_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + apple_rtkit_free_retaining_buffers(dcp->rtk); + } else { + apple_rtkit_free(dcp->rtk); + } + dcp->rtk = NULL; +} + #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) static bool show_notch; @@ -272,6 +351,10 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) container_of(to_delayed_work(work), struct apple_dcp, swap_watchdog_wq); + if (READ_ONCE(dcp->quiescing)) + return; + + trace_dcp_recv_msg(dcp, endpoint, message); dev_warn(dcp->dev, "swap not completed, retraining the display\n"); dcp_mode_invalidate(&dcp->mode_state); dcp_drm_crtc_vblank(dcp->crtc); @@ -375,6 +458,9 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dev_err(dcp->dev, "unusable external display processor has crashed\n"); return; } + if (READ_ONCE(dcp->quiescing)) + return; + dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); if (dcp->connector) { @@ -477,6 +563,8 @@ static struct apple_rtkit_ops rtkit_ops = { void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { + if (READ_ONCE(dcp->quiescing)) + return; int ret; trace_dcp_send_msg(dcp, endpoint, message); @@ -1886,6 +1974,10 @@ static void dcp_work_update_backlight(struct work_struct *work) static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) { + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + return; + } if (dcp->piodma) { if (dcp->piodma_created) of_platform_device_destroy(&dcp->piodma->dev, NULL); @@ -2434,6 +2526,10 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat u32 cpu_ctrl; int ret; + if (READ_ONCE(dcp->quiescing)) + return dev_err_probe(dev, -EBUSY, + "Live DCP session cannot be rebound; reboot required\n"); + /* A timed-out prior session may still scan these mappings. */ mutex_lock(&dcp->swapped_out_fbs_lock); ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; @@ -2591,12 +2687,19 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); } - dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + /* Registering the mailbox can already expose RTKit buffer requests. */ + ret = dcp_pin_live_session(dcp); + if (ret) + goto err_piodma; + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) { ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); goto err_piodma; } + ret = devm_add_action_or_reset(dev, dcp_release_rtkit, dcp); + if (ret) + goto err_piodma; if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); @@ -2667,6 +2770,31 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) disable_work_sync(&dcp->vblank_wq); } iomfb_v14_7_unbind(dcp); + if (dcp->retain_dma) { + /* No firmware stop is acknowledged: detach host users, never DMA. */ + WRITE_ONCE(dcp->quiescing, true); + if (dcp->crtc && !dcp->drm_retained) { + drm_dev_get(dcp->crtc->base.dev); + dcp->drm_retained = true; + } + iomfb_queue_stop(dcp); + afk_quiesce(dcp->avep); + afk_quiesce(dcp->dptxep); + afk_quiesce(dcp->ibootep); + afk_quiesce(dcp->systemep); + afk_quiesce(dcp->dcpavservep); + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + } + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + cancel_work_sync(&dcp->vblank_wq); + if (dcp->connector) + cancel_work_sync(&dcp->connector->hotplug_wq); + if (dcp->typec_connector && dcp->typec_connector != dcp->connector) + cancel_work_sync(&dcp->typec_connector->hotplug_wq); + dev_warn(dev, "retaining live DCP DMA resources after unbind; reboot required\n"); return; } @@ -2809,6 +2937,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) u32 surf_en; u32 mux_index; + if (dcp_session_retained(dev)) + return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); + fw_compat = dcp_check_firmware_version(dev); if (fw_compat == DCP_FIRMWARE_UNKNOWN) return -ENODEV; @@ -2825,9 +2956,12 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " "Use devicetree matching this kernel.\n"); - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + dcp = kzalloc_obj(*dcp); if (!dcp) return -ENOMEM; + ret = devm_add_action_or_reset(dev, dcp_release_context, dcp); + if (ret) + return ret; INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); @@ -2854,6 +2988,14 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + + /* The domain use must survive driver-core DMA cleanup for a live session. */ + ret = iommu_device_use_default_domain(dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dcp_release_dma_domain, dcp); + if (ret) + return ret; dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); dcp->connector_type = dcp->fixed_connector_type; of_property_read_u32(dev->of_node, "apple,dcp-index", &dcp->index); @@ -3495,6 +3637,7 @@ static const struct of_device_id of_match[] = { MODULE_DEVICE_TABLE(of, of_match); static struct platform_driver apple_platform_driver = { + .driver_managed_dma = true, .probe = dcp_platform_probe, .remove = dcp_platform_remove, .shutdown = dcp_platform_shutdown, diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 0e06d001aca311..bff7bfcecada12 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -114,6 +114,9 @@ static int dcp_get_brightness(struct backlight_device *bd) unsigned long flags; u32 actual; + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; + if (dcp_backlight_active(dcp)) { spin_lock_irqsave(&dcp->backlight.lock, flags); actual = dcp->backlight.state.actual; @@ -337,6 +340,8 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) int dcp_backlight_update(struct apple_dcp *dcp) { + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; if (dcp_backlight_active(dcp)) { dcp_backlight_kick(dcp); return 0; @@ -368,6 +373,9 @@ static int dcp_set_brightness(struct backlight_device *bd) int brightness = backlight_get_brightness(bd); unsigned long flags; + if (READ_ONCE(dcp->quiescing)) + return -ENODEV; + if (dcp_backlight_active(dcp)) { /* Preserve the requested level even while the core forces zero. */ spin_lock_irqsave(&dcp->backlight.lock, flags); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 87581d6778009d..c8ca796ee542c3 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -3350,10 +3350,12 @@ static ssize_t iommu_group_store_type(struct iommu_group *group, * * The device driver about to bind @dev wants to do DMA through the kernel * DMA API. Return 0 if it is allowed, otherwise an error. + * Drivers with driver_managed_dma may manage this usage themselves. The + * device lock must serialize probe and removal while holding the usage. */ int iommu_device_use_default_domain(struct device *dev) { - /* Caller is the driver core during the pre-probe path */ + /* Called with the device lock held during probe. */ struct iommu_group *group = dev->iommu_group; int ret = 0; @@ -3380,6 +3382,7 @@ int iommu_device_use_default_domain(struct device *dev) mutex_unlock(&group->mutex); return ret; } +EXPORT_SYMBOL_GPL(iommu_device_use_default_domain); /** * iommu_device_unuse_default_domain() - Device driver stops handling device @@ -3391,7 +3394,7 @@ int iommu_device_use_default_domain(struct device *dev) */ void iommu_device_unuse_default_domain(struct device *dev) { - /* Caller is the driver core during the post-probe path */ + /* Called with the device lock held during removal. */ struct iommu_group *group = dev->iommu_group; if (!group) @@ -3403,6 +3406,7 @@ void iommu_device_unuse_default_domain(struct device *dev) mutex_unlock(&group->mutex); } +EXPORT_SYMBOL_GPL(iommu_device_unuse_default_domain); static int __iommu_group_alloc_blocking_domain(struct iommu_group *group) { diff --git a/tools/testing/apple/check-lifetime.py b/tools/testing/apple/check-lifetime.py new file mode 100644 index 00000000000000..d7f82f276dc84e --- /dev/null +++ b/tools/testing/apple/check-lifetime.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native regression for component/devres teardown of an unstopped session. + +Mocks kernel resource releases and the component framework. It executes the +actual unbind, PIODMA release, RTKit release, domain release and ring allocator. +No firmware calls or hardware acceptance are claimed. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--output', type=Path, required=True) +p.add_argument('--cc', default='clang') +a = p.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +a.output.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + pos, depth = start + 1, 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +# This is the driver-core switch that transfers default-domain lifetime to us. +text = (src / 'dcp.c').read_text() +assert re.search(r'static struct platform_driver apple_platform_driver = \{\s*' + r'\.driver_managed_dma = true,', text) +functions = '\n'.join(function(f, n) for f, n in ( + ('dcp.c', 'dcp_release_dma_domain'), ('dcp.c', 'dcp_release_rtkit'), + ('dcp.c', 'dcp_release_piodma_iommu_dev'), ('afk.c', 'afk_quiesce'), + ('afk.c', 'afk_release_context'), ('dcp.c', 'dcp_release_context'), + ('afk.c', 'afk_getbuf'), ('dcp.c', 'dcp_comp_unbind'), + ('apple_drv.c', 'apple_drm_uninit'))) +prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint16_t u16;typedef uint32_t u32;typedef uint64_t u64; +#define GFP_KERNEL 0 +#define BLOCK_SHIFT 6 +#define GETBUF_SIZE 0xffffU +#define GETBUF_TAG 0xffff0000U +#define RBEP_TYPE 0 +#define RBEP_GETBUF_ACK 0 +#define GETBUF_ACK_DVA 0 +#define FIELD_GET(mask,x) ((x)&(mask)) +#define FIELD_PREP(mask,x) ((void)(mask),(u64)(x)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define IS_ERR_OR_NULL(x) (!(x)||(uintptr_t)(x)>(uintptr_t)-4096) +#define DRM_MODE_CONNECTOR_eDP 14 +struct work_struct {int id;};struct delayed_work {struct work_struct work;}; +struct drm_device {int refs;bool unplugged;}; +struct apple_drm_private {struct drm_device drm;}; +struct device {void *data;int domain_users;}; +struct platform_device {struct device dev;}; +struct apple_crtc {struct {struct drm_device *dev;} base;}; +struct apple_connector {struct work_struct hotplug_wq;}; +struct apple_rtkit {bool rx_stopped;}; +struct apple_dcp; +struct apple_dcp_afkep { + struct apple_dcp *dcp;void *wq;void *bfr;void *debugfs_entry; + u64 bfr_dma;u32 bfr_size;u16 bfr_tag; +}; +struct apple_dcp { + bool retain_dma,quiescing,drm_retained,piodma_created;int connector_type,hdmi_hpd_irq; + struct device *dev;struct apple_crtc *crtc; + struct apple_connector *connector,*typec_connector; + struct platform_device *piodma;void *iommu_dom,*typec_mux,*shmem,*clk,*clk_194; + struct apple_rtkit *rtk; + struct apple_dcp_afkep *avep,*dptxep,*ibootep,*systemep,*dcpavservep; + struct work_struct bl_register_wq,bl_update_wq,vblank_wq; + struct delayed_work typec_reconnect_wq,typec_fabric_retrain_wq; +}; +static int dma_frees,domain_drops,pio_destroys,pio_puts,firmware_stops,queue_stops; +static int raw_rings,managed_rings,work_drains; +static int context_frees; +static void *ring; +static struct device *component; +static void dcp_comp_unbind(struct device *,struct device *,void *); +static void kfree(void *p) {assert(p);context_frees++;} +static void debugfs_remove_recursive(void *p) {(void)p;} +static void dev_warn(struct device *d,const char *fmt) {(void)d;(void)fmt;} +static void dev_err(struct device *d,const char *fmt,...) {(void)d;(void)fmt;} +static void *dev_get_drvdata(struct device *d) {return d->data;} +static void dev_set_drvdata(struct device *d,void *p) {d->data=p;} +static void drm_dev_unplug(struct drm_device *d) {d->unplugged=true;} +static void drm_atomic_helper_shutdown(struct drm_device *d) {assert(d->unplugged);} +static void component_unbind_all(struct device *m,void *p) {dcp_comp_unbind(component,m,p);} +static void iommu_device_unuse_default_domain(struct device *d) {domain_drops++;d->domain_users--;} +static void apple_rtkit_free(struct apple_rtkit *r) {r->rx_stopped=true;dma_frees++;} +static void apple_rtkit_free_retaining_buffers(struct apple_rtkit *r) {r->rx_stopped=true;} +static void of_platform_device_destroy(struct device *d,void *p) {(void)d;(void)p;pio_destroys++;} +static void put_device(struct device *d) {(void)d;pio_puts++;} +static void drm_dev_get(struct drm_device *d) {d->refs++;} +static void iomfb_queue_stop(struct apple_dcp *d) {assert(d->quiescing||!d->retain_dma);queue_stops++;} +static void destroy_workqueue(void *w) {assert(w);work_drains++;} +static void cancel_work_sync(struct work_struct *w) {(void)w;} +static void cancel_delayed_work_sync(struct delayed_work *w) {(void)w;} +static void disable_irq(int i) {(void)i;} +static void typec_mux_put(void *p) {(void)p;} +static void av_service_disconnect(struct apple_dcp *d) {(void)d;firmware_stops++;} +static void afk_shutdown(struct apple_dcp_afkep *e) {(void)e;firmware_stops++;} +static void iomfb_shutdown(struct apple_dcp *d) {(void)d;firmware_stops++;} +static void devm_clk_put(struct device *d,void *c) {(void)d;(void)c;} +static void trace_afk_getbuf(struct apple_dcp_afkep *e,u32 size,u16 tag) {(void)e;(void)size;(void)tag;} +static void afk_send(struct apple_dcp_afkep *e,u64 msg) {(void)e;(void)msg;} +static void *dma_alloc_coherent(struct device *d,u32 size,u64 *addr,int flags) { + (void)d;(void)flags;raw_rings++;ring=calloc(1,size);assert(ring);*addr=123;return ring; +} +static void *dmam_alloc_coherent(struct device *d,u32 size,u64 *addr,int flags) { + (void)d;(void)flags;managed_rings++;ring=calloc(1,size);assert(ring);*addr=123;return ring; +} +''' +main = r''' +int main(void) { + unsigned cases=0; + for(int live=0;live<2;live++)for(int created=0;created<2;created++)for(int linked=0;linked<2;linked++)for(int master=0;master<2;master++) { + struct device dev={.domain_users=1};struct platform_device pio={0}; + struct apple_drm_private apple={.drm={.refs=1}}; + struct device main={.data=&apple};struct apple_crtc crtc={.base={&apple.drm}}; + struct apple_rtkit rtk={0};struct apple_connector con={0}; + struct apple_dcp d={.retain_dma=live,.dev=&dev,.crtc=linked?&crtc:NULL, + .connector=&con,.piodma=&pio,.piodma_created=created,.iommu_dom=&dev, + .rtk=&rtk,.shmem=&dev,.connector_type=DRM_MODE_CONNECTOR_eDP}; + struct apple_dcp_afkep ep[5]={0}; + for(int i=0;i<5;i++)ep[i]=(struct apple_dcp_afkep){.dcp=&d,.wq=&dev}; + d.avep=&ep[0];d.dptxep=&ep[1];d.ibootep=&ep[2];d.systemep=&ep[3];d.dcpavservep=&ep[4];dev.data=&d; + dma_frees=domain_drops=pio_destroys=pio_puts=firmware_stops=queue_stops=0; + raw_rings=managed_rings=work_drains=context_frees=0; + afk_getbuf(&ep[0],1);assert(ep[0].bfr_size==64); + afk_getbuf(&ep[0],1);assert(raw_rings+managed_rings==1); + component=&dev; + if(master) {apple_drm_uninit(&main);assert(!main.data&&apple.drm.unplugged);} + else dcp_comp_unbind(&dev,&dev,NULL); + /* Component devres release runs even after an early unbind return. */ + dcp_release_rtkit(&d);assert(rtk.rx_stopped&&!d.rtk); + /* Driver removal also releases probe resources and its domain action. */ + dcp_release_dma_domain(&d); + if(live) { + assert(d.quiescing&&!dma_frees&&!domain_drops&&dev.domain_users==1); + assert(d.piodma==&pio&&d.iommu_dom==&dev&&!pio_destroys&&!pio_puts); + assert(raw_rings==1&&!managed_rings&&!firmware_stops&&work_drains==5); + assert(apple.drm.refs==(linked?2:1)); + dcp_comp_unbind(&dev,&dev,NULL);assert(apple.drm.refs==(linked?2:1)); + dcp_release_piodma_iommu_dev(&d);assert(!pio_destroys&&!pio_puts); + } else { + assert(dma_frees==1&&domain_drops==1&&dev.domain_users==0); + assert(!d.piodma&&!d.iommu_dom&&managed_rings==1&&!raw_rings); + assert(pio_destroys==created&&pio_puts==!created&&firmware_stops>0); + assert(apple.drm.refs==1); + } + for(int i=0;i<5;i++)afk_release_context(&ep[i]); + dcp_release_context(&d);assert(context_frees==(live?0:6)); + afk_quiesce(NULL);afk_quiesce((struct apple_dcp_afkep *)(intptr_t)-12); + /* Simulated machine reset: test memory can now be reclaimed. */ + free(ring);cases++; + } + /* A failed bind has no unbind callback: resource cleanup must close rebind. */ + { + struct device dev={0};struct platform_device pio={0};struct apple_rtkit rtk={0}; + struct apple_dcp d={.dev=&dev,.retain_dma=true,.piodma=&pio,.rtk=&rtk}; + assert(!d.quiescing);dcp_release_piodma_iommu_dev(&d); + assert(d.quiescing&&d.piodma==&pio);cases++; + d.quiescing=false;dcp_release_rtkit(&d); + assert(d.quiescing&&rtk.rx_stopped&&!d.rtk);cases++; + } + puts("PASS: 18 actual-source component, master-unwind, driver-resource-release and failed-bind cases; live DMA/domain/rings retained; legacy releases preserved"); + assert(cases==18);return 0; +} +''' +code = a.output / 'lifetime.c' +code.write_text(prefix + functions + main) +command = [a.cc, '-std=gnu11', '-O1', '-g', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-parameter', '-fsanitize=address,undefined', str(code), + '-o', str(a.output / 'lifetime')] +subprocess.run(command, check=True) +result = subprocess.check_output([str(a.output / 'lifetime')], text=True) +(a.output / 'receipt.json').write_text(json.dumps(dict(command=command, result=result, + functions_sha256=hashlib.sha256(functions.encode()).hexdigest(), + scope='Native resource-release simulation; not a live firmware unbind test'), indent=2) + '\n') +print(result) From d722e3c2c846901b20f2e18e1e1c48c854664c97 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:32:00 +1100 Subject: [PATCH 165/631] drm: apple: retry rejected H17P brightness presents with a bound A rejected brightness present cancels its reservation and leaves the policy dirty, but it receives no successful completion. The queue's completion-only reschedule condition therefore drops the update on a static display until another user request or frame arrives. Track a rejected backlight reservation separately from a completed present. After its channel drains, schedule up to three deferred retries at 100, 200 and 400 ms through the existing outbound queue. Preserve the last accepted surface, leave an exhausted target dirty, reset the budget on a new user request or successful completion, and cancel delayed work when the queue stops. Crashed and stopped queues never retry. Add a native regression that executes the actual failure, queue-advance, retry-work and completion functions: one rejected present is retried and accepted without another user request. Check the finite retry budget, new-target rearming and stopped, crashed and undrained-channel guards. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 + drivers/gpu/drm/apple/dcp_backlight.c | 11 ++ drivers/gpu/drm/apple/dcp_backlight.h | 21 ++- drivers/gpu/drm/apple/iomfb.c | 29 +++- tools/testing/apple/check-backlight-retry.py | 162 +++++++++++++++++++ tools/testing/apple/check-backlight.py | 2 +- 6 files changed, 222 insertions(+), 5 deletions(-) create mode 100644 tools/testing/apple/check-backlight-retry.py diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ff57061622e8b7..16395623edea55 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -47,6 +47,7 @@ struct iomfb_transaction { void (*release)(struct iomfb_transaction *transaction); struct dcp_backlight_present backlight; bool backlight_reserved; + bool backlight_failed; bool brightness_only; bool completed; }; @@ -523,6 +524,7 @@ struct apple_dcp { struct task_struct *owner; struct work_struct work; struct delayed_work timeout; + struct delayed_work backlight_retry; unsigned long deadline; unsigned int queued; bool stopped; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index bff7bfcecada12..726277a6f2079b 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -227,6 +227,17 @@ bool dcp_backlight_pending(struct apple_dcp *dcp) return pending; } +unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp) +{ + unsigned long flags; + unsigned int delay; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + delay = dcp_bl_retry_delay(&dcp->backlight.state); + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return delay; +} + #define SCALE_FACTOR (1 << 10) static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index e93ee7511c6266..2ad4abb4acd376 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -15,6 +15,7 @@ struct dcp_backlight_state { u32 actual; u32 sent_nits; u64 sequence; + u8 retries; bool ready; bool controlled; bool dpms_off; @@ -89,6 +90,7 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, if (nits > state->maximum) return -ERANGE; state->controlled = true; + state->retries = 0; if (state->target != nits || state->core_blank != core_blank || state->suspended != suspended) { state->target = nits; @@ -104,6 +106,7 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) if (!state->ready) return -ENODATA; state->controlled = true; + state->retries = 0; if (state->dpms_off != !on) { state->dpms_off = !on; state->dirty = true; @@ -147,13 +150,26 @@ static inline bool dcp_bl_complete(struct dcp_backlight_state *state, if (!state->in_flight || sequence != state->sequence) return false; state->in_flight = false; - if (accepted) + if (accepted) { state->actual = state->sent_nits; - else + state->retries = 0; + } else { state->dirty = true; + if (state->retries < 4) + state->retries++; + } return true; } +/* Three deferred retries per user request; a permanent rejection stays dirty. */ +static inline unsigned int dcp_bl_retry_delay(const struct dcp_backlight_state *state) +{ + if (!state->ready || !state->dirty || state->in_flight || + !state->retries || state->retries > 3) + return 0; + return 100U << (state->retries - 1); +} + struct apple_dcp; /* @@ -173,5 +189,6 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, struct dcp_backlight_present *present); bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); bool dcp_backlight_pending(struct apple_dcp *dcp); +unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp); #endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 288ee29576a257..9019ed48e8eb20 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -245,8 +245,20 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) if (transaction->brightness_only) dcp->iomfb.backlight_queued = false; /* A new level requested during a successful present stays pending. */ - if (transaction->completed && dcp_backlight_pending(dcp)) - schedule_work(&dcp->bl_update_wq); + if (dcp_backlight_pending(dcp)) { + if (transaction->completed) { + schedule_work(&dcp->bl_update_wq); + } else if (transaction->backlight_failed) { + unsigned int delay = dcp_backlight_retry_delay(dcp); + + if (delay) + mod_delayed_work(system_wq, &dcp->iomfb.backlight_retry, + msecs_to_jiffies(delay)); + else + dev_warn_ratelimited(dcp->dev, + "backlight retry limit reached\n"); + } + } transaction->release(transaction); } if (iomfb_enqueue_opaque_x(dcp)) { @@ -341,12 +353,23 @@ void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp) atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } +static void iomfb_backlight_retry(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + iomfb.backlight_retry); + + if (!READ_ONCE(dcp->crashed) && !READ_ONCE(dcp->iomfb.stopped) && + dcp_backlight_pending(dcp)) + schedule_work(&dcp->bl_update_wq); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); + INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); } @@ -361,6 +384,7 @@ void iomfb_queue_stop(struct apple_dcp *dcp) mutex_unlock(&dcp->iomfb.lock); cancel_work_sync(&dcp->iomfb.work); cancel_delayed_work_sync(&dcp->iomfb.timeout); + cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) @@ -1476,6 +1500,7 @@ void iomfb_present_failed_h17p(struct apple_dcp *dcp) if (transaction && transaction->backlight_reserved) { dcp_backlight_complete(dcp, transaction->backlight.sequence, false); transaction->backlight_reserved = false; + transaction->backlight_failed = true; } } diff --git a/tools/testing/apple/check-backlight-retry.py b/tools/testing/apple/check-backlight-retry.py new file mode 100644 index 00000000000000..32d1fc40580f26 --- /dev/null +++ b/tools/testing/apple/check-backlight-retry.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Native regression for rejected brightness transactions and bounded retries. + +Executes actual reservation, failure, queue-drain, retry-work and completion +functions with mocked scheduling and transport. Does not submit firmware RPCs. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument('--output', type=Path, required=True) +p.add_argument('--cc', default='clang') +a = p.parse_args() +root = Path(__file__).resolve().parents[3] +src = root / 'drivers/gpu/drm/apple' +a.output.mkdir(parents=True, exist_ok=True) + + +def function(file, name): + text = (src / file).read_text() + match = re.search(r'^\w[\w\s*]*?\b' + name + r'\(', text, re.M) + assert match, name + start = text.index('{', match.end()) + pos, depth = start + 1, 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + + +policy = re.sub(r'^#include ]+>\n', '', + (src / 'dcp_backlight.h').read_text(), flags=re.M) +functions = '\n'.join(function(f, n) for f, n in ( + ('dcp_backlight.c', 'dcp_backlight_complete'), + ('dcp_backlight.c', 'dcp_backlight_pending'), + ('dcp_backlight.c', 'dcp_backlight_retry_delay'), + ('iomfb.c', 'iomfb_present_failed_h17p'), + ('iomfb.c', 'iomfb_present_complete_h17p'), + ('iomfb.c', 'iomfb_queue_advance'), ('iomfb.c', 'iomfb_backlight_retry'))) +prefix = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8;typedef uint32_t u32;typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define lockdep_assert_held(x) ((void)(x)) +#define spin_lock_irqsave(x,f) do { (void)(x);(f)=0; } while(0) +#define spin_unlock_irqrestore(x,f) do { (void)(x);(void)(f); } while(0) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_delayed_work(p) container_of(p,struct delayed_work,work) +#define msecs_to_jiffies(x) (x) +#define IOMFB_OPAQUE_X_WAITING 0 +#define IOMFB_OPAQUE_X_NEEDED 1 +struct work_struct {int id;};struct delayed_work {struct work_struct work;}; +struct list_head {int empty;}; +struct iomfb_transaction { + struct dcp_backlight_present backlight; + bool backlight_reserved,backlight_failed,brightness_only,completed; + void (*release)(struct iomfb_transaction *); +}; +struct apple_dcp { + bool crashed;void *dev;struct work_struct bl_update_wq,vblank_wq; + struct {int lock;struct dcp_backlight_state state;} backlight; + struct {bool pending;} present_state_h17p; + struct {int lock;bool stopped,backlight_queued;struct iomfb_transaction *active; + struct delayed_work timeout,backlight_retry;struct work_struct work; + struct list_head pending;int opaque_x_state; + } iomfb; +}; +static unsigned delay,scheduled,released,discarded;static bool idle=true; +static void *system_wq; +static bool iomfb_channels_idle(struct apple_dcp *d) {(void)d;return idle;} +static int iomfb_enqueue_opaque_x(struct apple_dcp *d) {(void)d;return 0;} +static void iomfb_discard_pending(struct apple_dcp *d) {(void)d;discarded++;} +static bool list_empty(struct list_head *l) {return l->empty;} +static void cancel_delayed_work(struct delayed_work *w) {(void)w;} +static void schedule_work(struct work_struct *w) {assert(w->id==1);scheduled++;} +static void mod_delayed_work(void *q,struct delayed_work *w,unsigned ms) { + (void)q;assert(w->work.id==2);delay=ms; +} +static void dev_warn_ratelimited(void *d,const char *msg) {(void)d;(void)msg;} +static void iomfb_scanout_complete_h17p(struct apple_dcp *d) {(void)d;} +static void atomic_cmpxchg(int *p,int old,int value) {if(*p==old)*p=value;} +static void release(struct iomfb_transaction *t) {(void)t;released++;} +''' +# The policy header defines structs before the fixture needs them. +prefix = prefix[:prefix.index('struct work_struct')] + policy + prefix[prefix.index('struct work_struct'):] +main = r''' +static void init(struct apple_dcp *d) { + u32 inherited; + memset(d,0,sizeof(*d));d->bl_update_wq.id=1;d->iomfb.backlight_retry.work.id=2; + assert(!dcp_bl_takeover_nits(509,36000,&inherited)&&inherited==36); + d->iomfb.pending.empty=1;assert(!dcp_bl_init(&d->backlight.state,509,true,inherited,false,0)); + assert(dcp_bl_seed(&d->backlight.state,36));assert(!dcp_bl_dpms(&d->backlight.state,true)); + assert(!dcp_bl_request(&d->backlight.state,140,false,false)); + delay=scheduled=released=discarded=0;idle=true; +} +static void reserve(struct apple_dcp *d,struct iomfb_transaction *t) { + memset(t,0,sizeof(*t));t->brightness_only=true;t->release=release; + assert(!dcp_bl_prepare(&d->backlight.state,true,&t->backlight)); + t->backlight_reserved=true;d->iomfb.active=t;d->iomfb.backlight_queued=true; +} +static void reject(struct apple_dcp *d) { + iomfb_present_failed_h17p(d);d->present_state_h17p.pending=false; + iomfb_queue_advance(d); +} +int main(void) { + struct apple_dcp d;struct iomfb_transaction t; + init(&d);reserve(&d,&t);reject(&d); + assert(delay==100&&!d.iomfb.active&&!d.iomfb.backlight_queued&&released==1); + assert(d.backlight.state.actual==36&&d.backlight.state.dirty&&!t.completed); + /* The deferred worker retries on retained pixels, with NO new user request. */ + iomfb_backlight_retry(&d.iomfb.backlight_retry.work);assert(scheduled==1); + reserve(&d,&t);assert(!iomfb_present_complete_h17p(&d));iomfb_queue_advance(&d); + assert(d.backlight.state.actual==140&&!d.backlight.state.dirty&&!d.backlight.state.retries); + init(&d); + for(unsigned i=0;i<4;i++) { + reserve(&d,&t);delay=0;reject(&d);assert(delay==(i<3?(100U< Date: Thu, 8 Oct 2026 16:12:48 +1100 Subject: [PATCH 166/631] MAINTAINERS: assign new DCP bindings and tests to Aurora support The new DCP bindings, backlight documentation and native tests belong to Aurora's T8140 and J700 support. Move their file patterns to that existing entry so reviews for those files reach their maintainer. Keep the existing Apple DRM driver source pattern under its current display-driver entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 974f50efb5d471..e36f0a27b8737b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1977,6 +1977,9 @@ APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained +F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml +F: Documentation/devicetree/bindings/display/apple,dcp.yaml +F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/PCI/controller/apple-t8140.rst F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml @@ -1988,6 +1991,7 @@ F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml +F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* @@ -2002,6 +2006,7 @@ F: include/linux/soc/apple/pci-apple-piodma.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs +F: tools/testing/apple/ APPLETALK NETWORK LAYER L: netdev@vger.kernel.org From 8f1a47fe4ea51b547dc932844eaa182757783b6b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:53:55 +1100 Subject: [PATCH 167/631] drm: apple: guard the custom fbdev probe with its configuration Apple DRM can be enabled without DRM_FBDEV_EMULATION. In that configuration the DMA fbdev helper does not declare its probe function, but the custom Apple probe still calls it and fails to compile. Compile the custom probe and select it only when fbdev emulation is enabled. Otherwise use the DMA helper driver-ops macro, which supplies a NULL probe. The H17P ARGB console policy is unchanged when enabled. Fixes: 544eb67a1a17 ("drm: apple: use ARGB for H17P fbdev") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 30f15f32387b24..b021ea1c15cf6b 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -68,6 +68,7 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +#ifdef CONFIG_DRM_FBDEV_EMULATION static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -97,10 +98,15 @@ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) if (file->was_master) dcp_typec_reorder(); } +#endif static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), +#ifdef CONFIG_DRM_FBDEV_EMULATION .fbdev_probe = apple_drm_fbdev_probe, +#else + DRM_FBDEV_DMA_DRIVER_OPS, +#endif DRM_FBDEV_DMA_DRIVER_OPS, .postclose = apple_drm_postclose, .name = DRIVER_NAME, From fb4efa4e677fcb7948f8442e0f683f20174d7e03 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:11 +0200 Subject: [PATCH 168/631] drm/apple: route Thunderbolt DP tunnels through the Type-C crossbar Add core routing support for a USB4/Thunderbolt DisplayPort tunnel arriving behind a Thunderbolt hub, as opposed to a direct Type-C DP alt-mode connection. struct apple_dcp_typec_route grows a tunnel flag, an active_xbar pointer distinct from the route's own crossbar control, a tunnel_dpin index and an xbar_up flag, so dcp_typec_route_activate()/ deactivate() can be handed the crossbar control to actually use: for a tunnel this is a dpin0/dpin1 controller resolved at tunnel-creation time rather than the route's own dpphy control, and the mux selection for it is deferred until the tunnel pixel clock is confirmed running rather than issued immediately. Add apple_dcp_tb_dp_tunnel(), the entry point Thunderbolt calls once it has stood up a DP IN tunnel: it finds a free Type-C route on the target port and activates it as a tunnel. A Type-C-only pipeline (no fixed output of its own) never completes link training for a tunneled target on this hardware -- DCP firmware accepts request_display but never issues the follow-up apcall, despite byte-identical connect parameters, a firmware-internal decision on that coprocessor instance with no driver-source asymmetry to explain it. A pipeline with a fixed output of its own reliably reaches a full AUX/DPCD link, so route scoring prefers it unconditionally for a tunnel route on this hardware. Add dcp_tunnel_crossbar_up()/down()/set_rate()/dpin_activate(), the hooks the DPTX endpoint uses to keep the crossbar clocks and the DP IN adapter's active state in step with DCP's link (re)configuration, serialized by a new per-DCP tb_lock. Make the Type-C alt-mode mux callback leave a tunnel-owned port alone -- a tunnel is torn down by apple_dcp_tb_dp_tunnel() itself, not by an alt-mode notification, and reapplying a stale alt-mode state on top of one would fight it. Thread the DFP port selected by route activation through dcp_dptx_connect() into the DPTX validate/connect calls, which previously always targeted DFP port 0 -- this is what makes a tunneled connection train against the Thunderbolt DP IN adapter DCP was actually told to use, rather than the direct PHY. Widen DPTX_CONNECT_TIMEOUT from 2s to 8s and the Type-C HPD-assert wait to 8s, and drop DPTX_RECONNECT_RETRIES from 5 to 1: a tunneled connection's handshake round-trip through the DP IN adapter can outlast a direct-PHY connection's budget. Add dptxport_set_hpd_timeout() for that HPD-assert call, so a caller can widen the AFK reply timeout, since a tunnel's AUX proxying round-trip can be slower than a direct PHY's. [Chris Kearney: moved dptxport_set_hpd_timeout() here from the next commit, which added it after its first caller, so each commit builds] Co-Authored-By: Claude Sonnet 5 --- drivers/gpu/drm/apple/dcp-internal.h | 18 ++ drivers/gpu/drm/apple/dcp.c | 383 +++++++++++++++++++++++++++ 2 files changed, 401 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 16395623edea55..c642db6b01f524 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -478,6 +478,24 @@ struct apple_dcp { struct dcp_fabric_session typec_reconnect_session; /* hpd_mutex */ bool typec_work_enabled; /* hpd_mutex; IRQ recovery admission */ + /* Ported from aurora-silicon/linux#8. */ + /* DPTX feeds a Thunderbolt DP IN adapter, not the Type-C PHY lanes. */ + bool dptx_tunnel; + /* DFP port in the DPTX target: 0 = dpphy, 1 = dpin0, 2 = dpin1. */ + u8 dptx_dfp_port; + /* Wakes/sleeps the Thunderbolt DP IN adapter from DCP Activate/ + * Deactivate. Set by apple_dcp_tb_dp_tunnel(), cleared by + * dcp_typec_route_deactivate(). + */ + int (*tb_dpin_set_active)(void *ctx, bool active); + void *tb_dpin_ctx; + /* Serializes the Thunderbolt DP IN callback and tunnel crossbar state + * between DCP apcalls and tunnel teardown; never held while waiting + * for DCP. + */ + struct mutex tb_lock; + bool tb_clock_ok; + struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; struct gpio_desc *dp2hdmi_pwren; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c2de7902dfee72..a1ebb9b5e27d6d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -9,10 +9,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -25,9 +27,15 @@ #include #include #include +#include #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -38,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -69,6 +78,81 @@ static DEFINE_MUTEX(dcp_retained_sessions_lock); static bool dcp_session_retained(struct device *dev) { + struct apple_dcp *dcp = route->dcp; + int ret = 0; + + /* + * Under tb_lock so a DCP link (re)configuration can neither select + * the crossbar nor restart the tunnel pixel clock behind our back. + * Ported from aurora-silicon/linux#8. + */ + scoped_guard(mutex, &dcp->tb_lock) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + if (ret) + return ret; + route->xbar_up = false; + + if (route->tunnel && dcp->phy) { + /* The tunnel pixel clock must not outlive the tunnel. */ + typeof(&apple_atc_dp_tunnel_rate) stop = + symbol_get(apple_atc_dp_tunnel_rate); + + if (stop) { + stop(dcp->phy, 0); + symbol_put(apple_atc_dp_tunnel_rate); + } + } + route->active_xbar = NULL; + route->tunnel = false; + dcp->dptx_tunnel = false; + dcp->dptx_dfp_port = 0; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + dcp->tb_clock_ok = false; + } + + route->selected = false; + if (dcp->active_typec_route == route) + dcp->active_typec_route = NULL; + + if (route->port->connector && + route->port->connector->dcp == to_platform_device(dcp->dev)) { + /* + * Until the port is activated again it has no pipeline behind + * it, and nothing can read modes or EDID from it. Report it + * disconnected for that window: leaving a connected connector + * whose ->dcp is NULL lets anything probing it in between -- + * a compositor starting up while the fabric is still settling + * -- see an output it cannot get a mode for, and give up on + * it. The new pipeline marks it connected again once the + * display has come back up on it. + */ + WRITE_ONCE(route->port->connector->connected, false); + route->port->connector->dcp = NULL; + + /* Unrouted: the port could go to any of its pipelines again. */ + if (route->port->connector->port_encoder) + route->port->connector->port_encoder->possible_crtcs = + route->port->connector->candidate_crtcs; + } + dcp->typec_connector = NULL; + dcp->connector = dcp->fixed_connector; + + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_USB) { + dcp->connector_type = dcp->fixed_connector_type; + + /* + * Only hand the pipeline back to its fixed output if that output + * is live. Re-targeting the DPTX endpoint at the fixed PHY while + * nothing is attached there leaves DCP unable to train a link on a + * later Type-C target: it answers DEVICE_NOT_STARTED and every + * following DPTX call times out. Park on a Type-C PHY instead, + * for the same reason the USB-C-only case does below. + */ + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + dcp->phy = dcp->fixed_phy; + dcp->dptx_phy = dcp->fixed_dptx_phy; struct dcp_session_ref *session; bool retained = false; @@ -83,6 +167,277 @@ static bool dcp_session_retained(struct device *dev) return retained; } +/* + * Thunderbolt DP tunnels, ported from aurora-silicon/linux#8. Crossbar + * connection up/down, looked up at runtime so appledrm does not require the + * crossbar driver to be built. + */ +static int dcp_dpxbar_link(struct mux_control *mux, bool up) +{ + typeof(&apple_dpxbar_link_up) fn; + int ret; + + fn = up ? symbol_get(apple_dpxbar_link_up) : + symbol_get(apple_dpxbar_link_down); + if (!fn) + return -ENOENT; + ret = fn(mux); + if (up) + symbol_put(apple_dpxbar_link_up); + else + symbol_put(apple_dpxbar_link_down); + return ret; +} + +/* DP IN adapter handshake through the thunderbolt glue; tb_lock held. */ +static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) +{ + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->tb_dpin_set_active) + return -ENODEV; + return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); +} + +/* + * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate + * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that + * the tunnel pixel clock runs, and re-assert the DP IN adapter's + * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; + * after a re-link only the clocks are brought back (the mux selection and + * the ATC output enable are kept, see dcp_tunnel_crossbar_down()). Runs + * inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->active_xbar) + return -ENODEV; + if (!dcp->tb_clock_ok) { + dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); + return -EIO; + } + if (!route->xbar_up) { + /* Never block a DCP call on the mux semaphore. */ + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (!ret) + route->xbar_up = true; + } else { + ret = dcp_dpxbar_link(route->active_xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); + return ret; + } + return dcp_tunnel_dpin_locked(dcp, true); +} + +/* + * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an + * established link: DP IN inactive, crossbar clocks down (mux selection and + * ATC output enable kept). DP IN goes active again in + * dcp_tunnel_crossbar_up(). + */ +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->xbar_up) + return 0; + dcp_tunnel_dpin_locked(dcp, false); + return dcp_dpxbar_link(route->active_xbar, false); +} + +/* + * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the + * tunnel pixel clock. A stopped clock also takes the crossbar connection + * down. + */ +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +{ + typeof(&apple_atc_dp_tunnel_rate) fn; + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return -ENODEV; + fn = symbol_get(apple_atc_dp_tunnel_rate); + if (!fn) { + dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); + return -ENOENT; + } + route = dcp->active_typec_route; + if (!link_rate && route && route->xbar_up) + dcp_dpxbar_link(route->active_xbar, false); + ret = fn(phy, link_rate); + symbol_put(apple_atc_dp_tunnel_rate); + dcp->tb_clock_ok = !ret && link_rate; + if (ret) + dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", + link_rate, ret); + return ret; +} + +/* Thunderbolt DP IN: DCP Activate/Deactivate. */ +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +{ + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return 0; + return dcp_tunnel_dpin_locked(dcp, active); +} + +/* + * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the + * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt + * connection manager has set up a tunnel from one of them, route a free + * display pipeline there and tell DCP a display is attached, so it trains + * the link (and completes DPRX) through the tunnel. + */ +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx) +{ + struct apple_dcp_typec_port *port = NULL, *pos; + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + struct mux_control *ctl; + struct apple_dcp *dcp; + int ret; + + if (!connector_np || dpin > 1) + return -EINVAL; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(pos, &dcp_typec_ports, link) { + if (pos->connector_np == connector_np) { + port = pos; + break; + } + } + if (!port) + return -ENODEV; + + if (!active) { + if (!port->owner || !port->owner->tunnel || + port->owner->tunnel_dpin != dpin) + return 0; + dcp = port->owner->dcp; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ret = dcp_typec_route_deactivate(port->owner); + if (ret) { + /* The caller's context is going away regardless. */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->dptx_tunnel = false; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = NULL; + /* Re-apply the next Type-C mux state in full. */ + port->applied_valid = false; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + return 0; + } + + if (port->owner) { + if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) + return 0; + dev_warn(port->owner->dcp->dev, + "port already routed, not taking DP tunnel dpin%u\n", dpin); + return -EBUSY; + } + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + score = dcp_typec_route_score(candidate); + /* + * j416s-specific, not in the reference (t8103 has a single + * dcpext): prefer a pipeline with a fixed output of its own + * (dcpext0) for a Type-C tunnel route. dcpext1 (Type-C only, + * no fixed output of its own) never completes link training + * for a tunneled target on this hardware: DCP firmware + * accepts request_display but never issues another apcall, + * on every attempt, even though the driver-issued connect + * parameters are byte-identical between the two pipelines. + * Nothing in this driver's source explains the difference, + * so this is a firmware-internal decision on the dcpext1 + * coprocessor instance, not something fixable here. dcpext0, + * forced onto the same physical port and tunnel, reaches a + * full AUX/DPCD link (DPRX_DONE=1) and a working picture, so + * it is preferred unconditionally for a tunnel route on this + * hardware. + */ + if (!candidate->dcp->fixed_phy) + score += 100; + if (score < best_score) { + best = candidate; + best_score = score; + } + } + if (!best) + return -EBUSY; + + /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) + return -EOPNOTSUPP; + ctl = &best->xbar->chip->mux[1 + dpin]; + + dcp = best->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = set_active; + dcp->tb_dpin_ctx = ctx; + } + best->tunnel_dpin = dpin; + ret = dcp_typec_route_activate(best, ctl); + if (ret) { + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + port->owner = best; + + dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + + /* + * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP + * drives the DPTX, i.e. from DCP's Activate call; waking it earlier + * hangs the machine (confirmed independently by this project's own + * AFK ordered-workqueue deadlock finding). dptxep calls set_active + * back from Activate and Deactivate (set above, before the route + * became a tunnel). + */ + if (!dcp->typec_connector) + dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); + WRITE_ONCE(dcp->typec_cable_connected, true); + port->hpd = true; + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + + return 0; +} +EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); + +static int dcp_typec_route_set(struct typec_mux_dev *mux, + struct typec_mux_state *state) static int dcp_pin_live_session(struct apple_dcp *dcp) { struct dcp_session_ref *session; @@ -745,6 +1100,14 @@ static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, dcp->dptx_die, dcp->dptx_phy, ret); goto out_unlock; } + ret = dptxport_validate_connection(dcp->dptxport[port].service, + dcp->dptx_dfp_port, + dcp->dptx_phy, dcp->dptx_die); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); + goto out_unlock; } if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { @@ -753,6 +1116,8 @@ static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, } ret = dptxport_connect(dcp->dptxport[port].service, dfp_port, + ret = dptxport_connect(dcp->dptxport[port].service, + dcp->dptx_dfp_port, dcp->dptx_phy, dcp->dptx_die, dcp_is_typec_output(dcp) && !dcp_ext_dptx_607(dcp)); if (ret) { @@ -3005,6 +3370,16 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) init_completion(&dcp->typec_iomfb_hpd_ready); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); + INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, + dcp_typec_retrain_work); + /* + * Type-C and Thunderbolt routes can be activated as soon as they are + * registered below (dcp_register_typec_routes()), before the DRM + * device binds and dcp_comp_bind() would otherwise init this. Ported + * from aurora-silicon/linux#8. + */ + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->tb_lock); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); /* Balanced by enable at successful component bind. */ @@ -3391,6 +3766,11 @@ static void dcp_platform_shutdown(struct platform_device *pdev) return; } + if (dcp) { + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + } component_del(&pdev->dev, &dcp_comp_ops); } @@ -3503,6 +3883,9 @@ static int dcp_platform_suspend(struct device *dev) */ dcp_disable_typec_work(dcp, false); cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); if (dcp->avep) av_service_disconnect(dcp); From 2023adef81e94a123414703788c04bbd9b48ddf2 Mon Sep 17 00:00:00 2001 From: Oliver Lukschander Date: Thu, 24 Sep 2026 16:48:24 +0200 Subject: [PATCH 169/631] drm/apple: teach the DPTX endpoint about USB4 tunnel routes A Thunderbolt/USB4 DisplayPort tunnel reaches the same DCP DPTX AFK/EPIC endpoint as a direct HDMI or USB-C alt-mode PHY, but several request/reply shapes and side effects differ: the endpoint has no local ATC DisplayPort PHY to configure, the pixel clock and display crossbar connection are owned by the Thunderbolt tunnel infrastructure instead of being driven directly, and firmware's link-training sequencing includes calls (INACTIVE_SINK_DETECTED, FORCE_HOTPLUG_DETECT, GET_ACTIVE_LANE_COUNT) that the direct-PHY path never exercises. Factor VALIDATE_CONNECTION/CONNECT's remote-port target computation into dptxport_remote_target(), and add a tunnel "role" attribute bit that lets firmware distinguish a Thunderbolt DP-IN route from a direct PHY; tolerate the resulting difference in firmware's attrs reply only for a tunnel route, still rejecting a mismatch for the direct-PHY case. Treat a NULL dptx->atcphy (the USB4 DP-IN service has no ATC PHY of its own) as "report a fixed 4 lanes, skip phy_configure()" in GET_MAX_LANE_COUNT/SET_ACTIVE_LANE_COUNT. Route SET_LINK_RATE, ACTIVATE and DEACTIVATE through the dp-tunnel.h API for a tunnel connection instead of touching the ATC PHY's DP mode or rate registers directly -- the PHY must stay in USB4/Thunderbolt mode for the life of the tunnel. Bring the tunnel's display crossbar connection down before a re-link and back up once it completes, and handle GET_ACTIVE_LANE_COUNT, FORCE_HOTPLUG_DETECT and INACTIVE_SINK_DETECTED explicitly rather than relying on the generic unhandled-call ACK. dcp_hotplug()'s forced link-status retrain recovers a connector that reports connected without a valid mode set yet, for a Type-C route switch that can drop the active mode without changing HPD state. A USB4 tunnel connector's link comes up through the DPTX endpoint's own crossbar/clock sequencing rather than a Type-C route switch, so restrict that recovery path to non-tunnel connectors. Trace the read_edt_data and property-dictionary DCP callbacks' key, size and result for any connector other than the fixed eDP panel, and include the firmware-reported mode count in the existing hotplug log line, to make the boot-property and mode-list exchange with firmware observable for a tunnel connector. [Chris Kearney: dptxport_set_hpd_timeout() moved to the previous commit, its first caller] Co-Authored-By: Claude Sonnet 5 --- drivers/gpu/drm/apple/iomfb_template.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 1495e4d3de7174..02b26f476d747f 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1039,6 +1039,12 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, (int)sizeof(req->key), req->key, dcp->chunks.length, resp, dcp->nr_modes); + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) + dev_info(dcp->dev, + "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", + (int)sizeof(req->key), req->key, dcp->chunks.length, + resp, dcp->nr_modes); + /* move chunked data to connector to provide it via debugfs */ dcp_connector_update_dict(dcp->connector, req->key, &dcp->chunks); dcp->chunks.data = NULL; From 1a937fd7040f1064a5566f660b2c8dd1a3b3bdc7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:15:13 -0700 Subject: [PATCH 170/631] Merge M2 Pro Thunderbolt display and resume support with the M1 series Keep the t8103 and T602X PLL, DP IN handshake and crossbar sequences separate. Retain the M1 PCIe initialization, IOMMU teardown and EDID retry fixes while incorporating the M2 routing, AFK timeouts and resume recovery from Aurora PRs #14 and #25. --- drivers/gpu/drm/apple/dcp.c | 15 +++++++++++++++ drivers/phy/apple/atc.c | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a1ebb9b5e27d6d..c1583767f756eb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1037,6 +1037,10 @@ void dcp_queue_typec_reconnect(struct apple_dcp *dcp, unsigned long delay) dcp->typec_reconnect_session = dcp_session_locked(dcp); mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, delay); } +#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) +#define DPTX_TUNNEL_CONNECT_TIMEOUT msecs_to_jiffies(8000) +#define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) +#define DPTX_RECONNECT_RETRIES 5 static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, struct dcp_fabric_session session, bool cable, @@ -1600,6 +1604,10 @@ static void dcp_typec_reconnect_work(struct work_struct *work) } if (++dcp->typec_reconnect_tries < (dcp_uses_t6020_tunnel_flow(dcp) ? 1 : DPTX_RECONNECT_RETRIES)) { + + if (++dcp->typec_reconnect_tries < + (dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? + 1 : DPTX_RECONNECT_RETRIES)) { mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, DPTX_RECONNECT_DELAY); return; @@ -3769,6 +3777,10 @@ static void dcp_platform_shutdown(struct platform_device *pdev) if (dcp) { WRITE_ONCE(dcp->typec_cable_connected, false); cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + if (dcp) { + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); } component_del(&pdev->dev, &dcp_comp_ops); @@ -3885,6 +3897,9 @@ static int dcp_platform_suspend(struct device *dev) cancel_delayed_work_sync(&dcp->swap_watchdog_wq); WRITE_ONCE(dcp->typec_cable_connected, false); cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + WRITE_ONCE(dcp->typec_cable_connected, false); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); if (dcp->avep) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a2e4e45aaaf6f2..5485781b76a3d3 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -3440,6 +3440,11 @@ int apple_atc_dp_tunnel_rate_source(struct phy *phy, unsigned int source, if (!apple_atc_tunnel_is_t6030(atcphy) || apple_dp_tunnel_t6030_pclk(source, dpin) < 0) return -EOPNOTSUPP; + if (of_machine_is_compatible("apple,j416s") && + (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || + !apple_atc_is_typec_core(atcphy->res.core->start) || + resource_size(atcphy->res.core) < 0x7048)) + return -EINVAL; guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); /* A stale source must not retune or stop another source's clock. */ From fa917b43536517802469db88607e9a51e01c0643 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:39:03 +1100 Subject: [PATCH 171/631] Merge pull request #8 from iconidentify/tb-dp-tunnel-t8103 Thunderbolt: M1/M2 Pro display, M1 PCIe and resume recovery Original contributor commits are preserved. Conflict resolutions retain the audited shared driver APIs and their integrated review fixes. Contributor credit; Signed-off-by outstanding: - 99a554596d0a4960120b78f9a09ab30ffd4d0fc1: Chris Kearney <303316+iconidentify@users.noreply.github.com> - b4e5e4cff710c9961a05d4ba6c16f0f8c47c3776: Chris Kearney <303316+iconidentify@users.noreply.github.com> - 508384b75fa147f3aa6ed78885235367d82adb8d: Chris Kearney <303316+iconidentify@users.noreply.github.com> - 46b9e80b117ef396fcb018dcf77cc5f98ec973d2: Oliver Lukschander - b9a6a94161917fd11674a4eef39a7e7336ba89ae: Chris Kearney <303316+iconidentify@users.noreply.github.com> - 749ac0488440a773d1a5216566f0565539bda9b7: Chris Kearney <303316+iconidentify@users.noreply.github.com> - d04e476c887a1ab014c042a9fc6b80a81cbea7de: Chris Kearney <303316+iconidentify@users.noreply.github.com> - 67fdc27bdadb58520839376a2b85d1265c3fa78f: Chris Kearney <303316+iconidentify@users.noreply.github.com> - cf5f7d19399e4f2d02428fbcc1339fa2217f781f: Chris Kearney <303316+iconidentify@users.noreply.github.com> - 8f576eb57772a991cb5f93c0618ceab1028c70f7: Oliver Lukschander - 69893ec9574328f8126d38bf5729e858cc818002: Oliver Lukschander - 85d6d84536c845412289f15d69a24005ea087af6: Oliver Lukschander - 85fbf65924ba19f33269f7864a1f52e30c1275e7: Oliver Lukschander - e557722c435b0a17c0f2381212113f23060c2202: Oliver Lukschander - 0daeb8ec3d63208ef4d8b9b2fa6b89ff817e37f4: Oliver Lukschander - cc895f20ade4b0a6f54ae6c26f5734f2c09d7145: Oliver Lukschander - f8cec39af157f9306ca2d049e48de2e46381cf98: Oliver Lukschander - 81eee7b6f439a1d9c1a2d1effb3ed2eb022ecce2: Oliver Lukschander - 45c34d503e04913b4d6cb5f49f0904f8104ec7f2: Oliver Lukschander - 6f0468f5e7646fa78fb2906c0909222728afc5d7: Oliver Lukschander Ryan integrates these contributions under DCO (b); no author signature is forged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 1 + drivers/gpu/drm/apple/afk.c | 36 + drivers/gpu/drm/apple/apple_drv.c | 23 +- drivers/gpu/drm/apple/dcp.c | 1252 +++++++++++++++++++++++++++-- drivers/gpu/drm/apple/iomfb.c | 4 +- drivers/usb/typec/tipd/tps6598x.h | 31 + 6 files changed, 1293 insertions(+), 54 deletions(-) diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 1e448994fb87f1..c15bf2a340f8c8 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -68,3 +68,4 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. + Also test the Type-C modeset, hotplug and service ownership state. diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 864677287e79a8..d5d5f3430d0532 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -128,6 +128,9 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) dev_err(afkep->dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); } + spin_lock_irqsave(&afkep->lock, flags); + afkep->stopping = true; + spin_unlock_irqrestore(&afkep->lock, flags); afk_quiesce(afkep); spin_lock_irqsave(&afkep->lock, flags); afkep->stopping = true; @@ -173,6 +176,10 @@ static int afk_start_polled(struct apple_dcp_afkep *ep) "RTKit AFK endpoint %02x did not acknowledge init\n", ep->endpoint); return -ETIMEDOUT; + if (afkep->rx_scratch) { + devm_kfree(afkep->dcp->dev, afkep->rx_scratch); + afkep->rx_scratch = NULL; + } } int afk_start(struct apple_dcp_afkep *ep) @@ -1133,6 +1140,16 @@ static int afk_send_command_full(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, size_t *reply_len, unsigned int timeout_ms) +{ + return afk_send_command_timeout(service, type, payload, payload_len, + output, output_len, retcode, + MSEC_PER_SEC); +} + +int afk_send_command_timeout(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms) { struct epic_cmd cmd; void *rxbuf, *txbuf; @@ -1153,7 +1170,14 @@ static int afk_send_command_full(struct apple_epic_service *service, u8 type, return -ENODEV; ep = service->ep; if (READ_ONCE(ep->dcp->quiescing)) + service = afk_service_get(service); + if (!service) return -ENODEV; + ep = service->ep; + if (READ_ONCE(ep->dcp->quiescing)) { + ret = -ENODEV; + goto err_put_service; + } rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); @@ -1322,6 +1346,16 @@ static int afk_service_call_full(struct apple_epic_service *service, u16 group, size_t data_pad, void *output, size_t output_len, size_t output_pad, size_t *reply_len, unsigned int timeout_ms) +{ + return afk_service_call_timeout(service, group, command, data, data_len, + data_pad, output, output_len, + output_pad, MSEC_PER_SEC); +} + +int afk_service_call_timeout(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms) { struct epic_service_call *call; void *bfr; @@ -1339,6 +1373,8 @@ static int afk_service_call_full(struct apple_epic_service *service, u16 group, check_add_overflow(max(input_size, output_size), sizeof(*call), &bfr_len) || bfr_len > U32_MAX) return -EOVERFLOW; + if (!service) + return -ENODEV; bfr = kzalloc(bfr_len, GFP_KERNEL); if (!bfr) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index b021ea1c15cf6b..5ace07ad14fbbd 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -68,7 +68,6 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } -#ifdef CONFIG_DRM_FBDEV_EMULATION static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -98,11 +97,20 @@ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) if (file->was_master) dcp_typec_reorder(); } -#endif + +/* + * A compositor or boot splash is closing the device. drm_release() has + * already taken its file off the file list, so the Type-C fabric can tell + * whether any other one still has the device open. + */ +static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) +{ + if (file->was_master) + dcp_typec_reorder(); +} static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), -#ifdef CONFIG_DRM_FBDEV_EMULATION .fbdev_probe = apple_drm_fbdev_probe, #else DRM_FBDEV_DMA_DRIVER_OPS, @@ -420,6 +428,7 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, struct drm_atomic_state *state) { struct apple_connector *apple_connector = to_apple_connector(conn); + struct drm_connector_state *conn_state; struct drm_encoder *encoder; /* @@ -429,8 +438,14 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, * route follows the CRTC (see dcp_typec_follow_crtc()). So it is the * only answer there is. */ - if (apple_connector->port_encoder) + if (apple_connector->port_encoder) { + conn_state = drm_atomic_get_new_connector_state(state, conn); + if (!conn_state || !conn_state->crtc || + READ_ONCE(apple_connector->dcp) != + to_apple_crtc(conn_state->crtc)->dcp) + return NULL; return apple_connector->port_encoder; + } drm_connector_for_each_possible_encoder(conn, encoder) return encoder; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c1583767f756eb..0f18c50baba297 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -77,6 +77,211 @@ static LIST_HEAD(dcp_retained_sessions); static DEFINE_MUTEX(dcp_retained_sessions_lock); static bool dcp_session_retained(struct device *dev) +{ + struct dcp_session_ref *session; + bool retained = false; + + mutex_lock(&dcp_retained_sessions_lock); + list_for_each_entry(session, &dcp_retained_sessions, link) { + if (dev_fwnode(session->dev) == dev_fwnode(dev)) { + retained = true; + break; + } + } + mutex_unlock(&dcp_retained_sessions_lock); + return retained; +} + +static int dcp_pin_live_session(struct apple_dcp *dcp) +{ + struct dcp_session_ref *session; + + if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) + return 0; + session = kzalloc_obj(*session); + if (!session) + return -ENOMEM; + + /* Keep the DMA device, domain usage and driver code valid until reset. */ + session->dev = get_device(dcp->dev); + session->group = iommu_group_get(dcp->dev); + __module_get(THIS_MODULE); + WRITE_ONCE(dcp->retain_dma, true); + mutex_lock(&dcp_retained_sessions_lock); + list_add_tail(&session->link, &dcp_retained_sessions); + mutex_unlock(&dcp_retained_sessions_lock); + dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); + return 0; +} + +static void dcp_release_dma_domain(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + iommu_device_unuse_default_domain(dcp->dev); +} + +static void dcp_release_context(void *data) +{ + struct apple_dcp *dcp = data; + + if (!dcp->retain_dma) + kfree(dcp); +} + +static void dcp_release_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + apple_rtkit_free_retaining_buffers(dcp->rtk); + } else { + apple_rtkit_free(dcp->rtk); + } + dcp->rtk = NULL; +} + +#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) + +static bool show_notch; +module_param(show_notch, bool, 0644); +MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); + +bool hdmi_audio; +module_param(hdmi_audio, bool, 0644); +MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); + +static bool unstable_edid = true; +module_param(unstable_edid, bool, 0644); +MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); + +struct apple_dcp_typec_port { + struct list_head link; + struct list_head routes; + struct device_node *connector_np; + struct apple_dcp_typec_route *owner; + /* DRM connector for this physical port, driven by whichever DCP owns it */ + struct apple_connector *connector; + /* last mux state acted on, to collapse the per-candidate notifications */ + struct typec_altmode *applied_alt; + unsigned long applied_mode; + u32 applied_status; + u32 applied_conf; + bool applied_valid; + bool hpd; +}; + +static DEFINE_MUTEX(dcp_typec_fabric_lock); +static LIST_HEAD(dcp_typec_ports); + +static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); +static void disconnected_hpd_event(struct apple_connector *connector); + +bool dcp_is_typec_output(struct apple_dcp *dcp) +{ + return dcp->active_typec_route || + dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; +} + +static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) +{ + return state->alt && state->alt->svid == USB_TYPEC_DP_SID && + state->mode >= TYPEC_DP_STATE_A && + state->mode <= TYPEC_DP_STATE_F; +} + +/* Keep a live fixed output on its own pipeline. */ +static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + + if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB) + return false; + if (dcp->fixed_connector && dcp->fixed_connector->connected) + return true; + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) + return true; + + return false; +} + +static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) +{ + return !route->dcp->active_typec_route && + !dcp_typec_route_fixed_output_busy(route); +} + +/* + * Machines whose USB4 docks carry two independent DP streams through one + * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and + * each port has a second connector for the DPIN1 stream. + */ +bool dcp_typec_dual_stream(void) +{ + return apple_dp_tunnel_t602x(); +} + +bool dcp_is_typec_only(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + return !dcp->fixed_phy; +} + +/* + * Can this route feed the given connector without changing its + * possible_crtcs? On dual-stream machines those are fixed at probe, since + * compositors read them once when the connector appears and pair the + * connector with a CRTC before this fabric has routed it. + */ +static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, + struct apple_connector *connector) +{ + struct apple_dcp *dcp = route->dcp; + + if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) + return true; + return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); +} + +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar); +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); +static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); + +static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_tunnel_select_source) select = + symbol_get(apple_dpxbar_tunnel_select_source); + int ret; + + if (!select) + return -ENOENT; + ret = select(mux, state); + symbol_put(apple_dpxbar_tunnel_select_source); + return ret; +} + +/* + * For a port without a prior owner, rank pipelines by CRTC index. A pipeline + * whose fixed output is live is not a candidate at all, so a hybrid is only + * ever ranked here when it is genuinely free. + */ +static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + + if (!dcp->crtc) + return UINT_MAX - 1; + + return drm_crtc_index(&dcp->crtc->base); +} + +static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, + struct mux_control *xbar) { struct apple_dcp *dcp = route->dcp; int ret = 0; @@ -112,12 +317,62 @@ static bool dcp_session_retained(struct device *dev) dcp->tb_clock_ok = false; } + dcp->phy = route->phy; + dcp->dptx_phy = route->dptx_phy; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + if (connector) { + WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); + dcp->typec_connector = connector; + dcp->connector = connector; + + /* + * Narrow the port to the pipeline now driving it. The encoder + * spans every pipeline that could, which is what lets userspace + * see the port as usable at all -- but only one of them is + * routed to the display, and userspace has no way to tell which. + * Offering it the choice makes it pair the port with a pipeline + * holding a different monitor's mode list, and the modeset is + * rejected with no way for it to recover. The hotplug that + * follows publishes the actual binding on dual-stream machines + * too. Clients must re-read encoder topology after a hotplug. + */ + if (connector->port_encoder && dcp->crtc) + connector->port_encoder->possible_crtcs = + drm_crtc_mask(&dcp->crtc->base); + } + dcp->active_typec_route = route; + route->selected = true; + + dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); + return 0; +} + +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + struct apple_connector *connector = dcp->typec_connector; + struct mux_control *active_xbar = route->active_xbar; + bool was_tunnel = route->tunnel; + int ret = 0; + + /* + * Under tb_lock so a DCP link (re)configuration can neither select the + * crossbar nor restart the tunnel pixel clock behind our back. + */ + scoped_guard(mutex, &dcp->tb_lock) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); + if (ret) + return ret; + route->xbar_up = false; + route->selected = false; if (dcp->active_typec_route == route) dcp->active_typec_route = NULL; - if (route->port->connector && - route->port->connector->dcp == to_platform_device(dcp->dev)) { + if (connector && connector->dcp == to_platform_device(dcp->dev)) { /* * Until the port is activated again it has no pipeline behind * it, and nothing can read modes or EDID from it. Report it @@ -128,13 +383,14 @@ static bool dcp_session_retained(struct device *dev) * it. The new pipeline marks it connected again once the * display has come back up on it. */ - WRITE_ONCE(route->port->connector->connected, false); - route->port->connector->dcp = NULL; + WRITE_ONCE(connector->connected, false); + /* hotplug work queued before this checks for it */ + WRITE_ONCE(connector->dcp, NULL); /* Unrouted: the port could go to any of its pipelines again. */ - if (route->port->connector->port_encoder) - route->port->connector->port_encoder->possible_crtcs = - route->port->connector->candidate_crtcs; + if (connector->port_encoder) + connector->port_encoder->possible_crtcs = + connector->candidate_crtcs; } dcp->typec_connector = NULL; dcp->connector = dcp->fixed_connector; @@ -192,56 +448,777 @@ static int dcp_dpxbar_link(struct mux_control *mux, bool up) /* DP IN adapter handshake through the thunderbolt glue; tb_lock held. */ static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) { - lockdep_assert_held(&dcp->tb_lock); - if (!dcp->tb_dpin_set_active) - return -ENODEV; - return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); + struct apple_dcp_typec_port *port; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->owner) + mod_delayed_work(system_freezable_wq, + &port->owner->dcp->typec_fabric_retrain_wq, + msecs_to_jiffies(200)); + if (port->secondary_owner) + mod_delayed_work(system_freezable_wq, + &port->secondary_owner->dcp->typec_fabric_retrain_wq, + msecs_to_jiffies(200)); + } +} + +static struct apple_dcp_typec_route * +dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + list_for_each_entry(route, &port->routes, port_link) + if (route->dcp == dcp) + return route; + + return NULL; +} + +/* + * The DRM device once it has bound, that is once every pipeline has its + * CRTC and every port its connectors; NULL before. + */ +static struct drm_device *dcp_typec_drm(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + struct drm_device *drm = NULL; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (!port->connector || !port->secondary_connector) + return NULL; + list_for_each_entry(route, &port->routes, port_link) { + if (!route->dcp->crtc) + return NULL; + drm = route->dcp->crtc->base.dev; + } + } + + return drm; +} + +/* + * Has a compositor (or boot splash) taken the display? It paired its + * connectors with CRTCs when it started and keeps that pairing, also + * across a VT switch, where it drops DRM master but keeps the device + * open; moving routes under it would hand connectors to pipelines driving + * other displays. So the routes are frozen while any open file has ever + * been master, and thaw when the last such file is closed. + * + * Taken under dcp_typec_fabric_lock, next to the routing it decides, so + * the order is the fabric lock, then filelist_mutex. Nothing nests them + * the other way: filelist_mutex only covers list edits and walks, and + * drm_release() drops it before drm_file_free() calls postclose, which is + * where a close takes the fabric lock. drm_open() makes a file master + * before adding it here, but its owner cannot have paired anything before + * open() returns, and it re-probes on the hotplugs the moves send. + */ +static bool dcp_typec_frozen(struct drm_device *drm) +{ + struct drm_file *file; + + guard(mutex)(&drm->filelist_mutex); + list_for_each_entry(file, &drm->filelist, lhead) { + /* set under master_mutex, and only ever from false to true */ + if (READ_ONCE(file->was_master)) + return true; + } + + return false; } -/* - * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate - * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that - * the tunnel pixel clock runs, and re-assert the DP IN adapter's - * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; - * after a re-link only the clocks are brought back (the mux selection and - * the ATC output enable are kept, see dcp_tunnel_crossbar_down()). Runs - * inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. - */ -int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) -{ - struct apple_dcp_typec_route *route; - int ret; +/* Are the routes kept in compositor pairing order right now? */ +static bool dcp_typec_keep_order(void) +{ + struct drm_device *drm; + + if (!dcp_typec_dual_stream()) + return false; + drm = dcp_typec_drm(); + + return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); +} + +/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ +static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) +{ + return dcp->active_typec_route && dcp->active_typec_route->tunnel; +} + +/* + * Give @dcp's pipeline to the first stream, in connector order, that + * wants one, has none yet and has it in its possible_crtcs. A tunnel + * stream wants one once it is set up, or as @arriving/@dpin. A direct + * DP-alt stream wants one once its sink asserts HPD: only then can its + * connector read connected, and a compositor pairs only those. + */ +static void dcp_typec_plan_pipeline(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + bool secondary = false; + + /* primary connectors in port order, then the DPIN1 ones */ + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route **slot = secondary ? + &port->secondary_target : &port->target; + struct apple_connector *connector = secondary ? + port->secondary_connector : port->connector; + struct apple_dcp_typec_route *route = + dcp_typec_port_route(port, dcp); + bool wants; + + if (secondary) + wants = port->secondary_owner || + (port == arriving && dpin == 1); + else + wants = (port->owner && port->owner->tunnel) || + (port->dp_wanted && port->dp_hpd && + !port->plan_dark) || + (port == arriving && dpin == 0); + + if (!wants || *slot || !route || + !(connector->candidate_crtcs & drm_crtc_mask(crtc))) + continue; + + *slot = route; + return; + } + secondary = !secondary; + } while (secondary); +} + +/* + * The pairing a compositor starting now would make, wherever the streams + * sit at the moment: pipelines in CRTC index order, each to the first + * stream that wants it. A pipeline whose fixed output is live is left to + * that output, unless a tunnel holds it: the output cannot have it back + * then, and its connector reads disconnected. + * + * Tunnels are planned in connector order like any other stream, not + * seated on the pipelines they hold: the compositor pairs their connectors + * by order too, and seating them would plan the direct streams around + * pairings it never makes. A direct stream planned onto a pipeline a + * tunnel holds cannot have it and stays dark, so it is left out and the + * pass re-run, or every stream after it would be planned one pipeline + * off. Streams only ever drop out, so that settles within one pass per + * port. It also brings a tunnel's plan onto the pipeline it holds + * wherever direct streams ahead of it were in the way. A tunnel still + * planned elsewhere cannot be brought there by any direct stream dropping + * out, and the compositor pairs its connector wrongly whatever they do. + */ +static void dcp_typec_plan(struct drm_device *drm, + struct apple_dcp_typec_port *arriving, unsigned int dpin) +{ + struct apple_dcp_typec_port *port; + struct drm_crtc *crtc; + bool again; + + list_for_each_entry(port, &dcp_typec_ports, link) + port->plan_dark = false; + + do { + list_for_each_entry(port, &dcp_typec_ports, link) { + port->target = NULL; + port->secondary_target = NULL; + } + + /* CRTCs are listed in index order */ + drm_for_each_crtc(crtc, drm) { + struct apple_dcp *dcp = + platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + if (dcp->nr_typec_routes && + (dcp_typec_tunnel_held(dcp) || + !dcp_typec_route_fixed_output_busy(&dcp->typec_routes[0]))) + dcp_typec_plan_pipeline(dcp, crtc, arriving, dpin); + } + + again = false; + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port == arriving || !port->target || + (port->owner && port->owner->tunnel) || + !dcp_typec_tunnel_held(port->target->dcp)) + continue; + port->plan_dark = true; + again = true; + } + } while (again); +} + +/* Replay HPD to the pipeline a direct DP-alt port has just been given. */ +static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) +{ + struct apple_dcp *dcp = port->owner->dcp; + + port->hpd = port->dp_hpd; + if (!port->hpd) + return; + + WRITE_ONCE(dcp->typec_cable_connected, true); + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); +} + +static struct apple_dcp_typec_route * +dcp_typec_lowest_free(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *candidate, *best = NULL; + unsigned int best_score = UINT_MAX; + + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (!dcp_typec_route_available(candidate)) + continue; + /* + * Prefer the lowest free CRTC index for a stable allocation order. + */ + score = dcp_typec_route_score(candidate); + if (score < best_score) { + best = candidate; + best_score = score; + } + } + + return best; +} + +/* + * Route the direct DP-alt ports left waiting for a pipeline. Their DP + * state is not reported again while it stays the same, so nothing else + * would. Each goes back to the pipeline it last had if that is free, as + * a compositor keeps a reconnected connector's CRTC, and otherwise takes + * the lowest free one, the CRTC a compositor gives a new connector. + */ +static void dcp_typec_route_waiting(void) +{ + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route *route; + + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->owner || !port->dp_wanted || !port->dp_hpd) + continue; + + route = port->preferred_route; + if (!route || !dcp_typec_route_available(route)) + route = dcp_typec_lowest_free(port); + if (!route || dcp_typec_route_activate(route, route->xbar)) + continue; + port->owner = route; + port->dp_release_deadline = 0; + dcp_typec_port_attach(port); + } +} + +/* + * Prefer connector order when placing Type-C streams before a DRM master + * starts. This is an allocation policy, not a promise about the CRTC that + * userspace must choose: each connected encoder advertises only its actual + * route, and atomic encoder selection checks that ownership. Route changes + * publish the new mask through hotplug; the plan uses candidate_crtcs so + * narrowing the advertised mask does not remove hardware routing choices. + * + * Only direct DP-alt routes move: each goes to exactly its planned + * pipeline. A Thunderbolt tunnel never moves once set up, so where one + * sits on a pipeline planned for a direct stream, or the plan has nothing + * for it, that stream stays unrouted and its connector disconnected. + * Placing it on some other pipeline would have the compositor cross both + * displays; dark is the better failure. @arriving/@dpin is a tunnel + * stream asking for a pipeline, planned like any other; its route is + * returned for apple_dcp_tb_dp_tunnel() to set up, or NULL if the plan + * has none for it or another tunnel holds that one. + * + * A move is an unplug and replug. Every route that moves is taken down + * first, so that two never share a pipeline midway, then each goes up on + * its new pipeline with its HPD replayed; the hotplugs this sends make + * fbdev and userspace re-probe. + */ +static struct apple_dcp_typec_route * +dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, + unsigned int dpin) +{ + struct drm_device *drm = dcp_typec_drm(); + struct apple_dcp_typec_route *planned = NULL; + struct apple_dcp_typec_port *port; + + lockdep_assert_held(&dcp_typec_fabric_lock); + + dcp_typec_plan(drm, arriving, dpin); + if (arriving) { + planned = dpin ? arriving->secondary_target : arriving->target; + /* it will be refused: plan as if it had not asked */ + if (planned && dcp_typec_tunnel_held(planned->dcp)) { + planned = NULL; + dcp_typec_plan(drm, NULL, 0); + } + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *owner = port->owner; + struct apple_dcp *dcp; + + if (!owner || owner->tunnel || owner == port->target) + continue; + + dcp = owner->dcp; + dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", + port->connector_np, dev_name(dcp->dev), + port->target ? dev_name(port->target->dcp->dev) : "none"); + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + if (dcp_typec_route_deactivate(owner) && owner->selected) { + /* still routed: leave the display where it is */ + dcp_typec_port_attach(port); + continue; + } + port->owner = NULL; + + /* as after a DP exit, hand the hybrid back to a live HDMI */ + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + } + + list_for_each_entry(port, &dcp_typec_ports, link) { + struct apple_dcp_typec_route *route = port->target; + + if (port->owner || !port->dp_wanted || !port->dp_hpd || + port == arriving) + continue; + + if (!route || !dcp_typec_route_available(route) || + dcp_typec_route_activate(route, route->xbar)) { + /* + * Dark for now. Its unchanged DP state is not reported + * again, so it is placed when the plan next runs, or + * once a compositor owns the display, when a pipeline + * is freed (dcp_typec_route_waiting()). + */ + port->applied_valid = false; + continue; + } + port->owner = route; + port->preferred_route = route; + port->dp_release_deadline = 0; + dcp_typec_port_attach(port); + } + + return planned; +} + +/* + * A route has let its pipeline go. Until a compositor owns the display + * the plan places everything anew; after that the pipeline goes to a port + * left waiting for one. + */ +static void dcp_typec_pipeline_freed(void) +{ + if (!dcp_typec_dual_stream()) + return; + + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); + else + dcp_typec_route_waiting(); +} + +/* + * Re-run the pairing pass from outside the fabric: once DRM is registered, + * as ports routed before that could not follow it, and when the last + * compositor or boot splash has closed the device, as the next one pairs + * the connectors from scratch. + */ +void dcp_typec_reorder(void) +{ + if (!dcp_typec_dual_stream()) + return; + + guard(mutex)(&dcp_typec_fabric_lock); + + if (dcp_typec_keep_order()) + dcp_typec_rebalance_locked(NULL, 0); + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->tb_dpin_set_active) + return -ENODEV; + return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); +} + +/* + * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate + * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that + * the tunnel pixel clock runs, and re-assert the DP IN adapter's + * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; + * after a re-link only the clocks are brought back (the mux selection and + * the ATC output enable are kept, see dcp_tunnel_crossbar_down()). Runs + * inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->active_xbar) + return -ENODEV; + if (!dcp->tb_clock_ok) { + dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); + return -EIO; + } + if (!route->xbar_up) { + /* Never block a DCP call on the mux semaphore. */ + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (!ret) + route->xbar_up = true; + } else { + ret = dcp_dpxbar_link(route->active_xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); + return ret; + } + return dcp_tunnel_dpin_locked(dcp, true); +} + +/* + * Crossbar connection up/down, looked up at runtime so appledrm does not + * require the crossbar driver to be built. + */ +static int dcp_dpxbar_link(struct mux_control *mux, bool up) +{ + typeof(&apple_dpxbar_link_up) fn; + int ret; + + fn = up ? symbol_get(apple_dpxbar_link_up) : + symbol_get(apple_dpxbar_link_down); + if (!fn) + return -ENOENT; + ret = fn(mux); + if (up) + symbol_put(apple_dpxbar_link_up); + else + symbol_put(apple_dpxbar_link_down); + return ret; +} + +static int dcp_dpxbar_preselect(struct mux_control *mux, int state) +{ + typeof(&apple_dpxbar_preselect) fn = symbol_get(apple_dpxbar_preselect); + int ret; + + if (!fn) + return -ENOENT; + ret = fn(mux, state); + symbol_put(apple_dpxbar_preselect); + return ret; +} + +/* + * Thunderbolt DP IN, before DCP hears about the display: point the DP IN + * output at this pipeline and wake the ATC's DP clock path. The crossbar + * connection itself still only comes up in dcp_tunnel_crossbar_up(), but an + * output left at its idle source (dispext0) only suits the pipeline that is + * dispext0: any other one activates the link, gets no answer from the sink + * and gives up with DEVICE_NOT_RESPONDING before it ever sets a link rate. + */ +static void dcp_tunnel_prepare(struct apple_dcp_typec_route *route, + struct mux_control *xbar) +{ + typeof(&apple_atc_dp_tunnel_open) open; + struct apple_dcp *dcp = route->dcp; + int ret; + + ret = dcp_dpxbar_preselect(xbar, route->mux_index); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel crossbar preselect failed: %d\n", ret); + + open = symbol_get(apple_atc_dp_tunnel_open); + if (!open) + return; + ret = open(dcp->phy); + symbol_put(apple_atc_dp_tunnel_open); + if (ret && ret != -EOPNOTSUPP) + dev_warn(dcp->dev, "DP tunnel PHY open failed: %d\n", ret); +} + +/* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ +static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) +{ + lockdep_assert_held(&dcp->tb_lock); + if (!dcp->tb_dpin_set_active) + return -ENODEV; + return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); +} + +/* + * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate + * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that + * the tunnel pixel clock runs, and re-assert the DP IN adapter's + * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; after + * a re-link only the clocks are brought back (the mux selection and the ATC + * output enable are kept, see dcp_tunnel_crossbar_down()). + * Runs inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. + */ +int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->active_xbar) + return -ENODEV; + if (!dcp->tb_clock_ok) { + dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); + return -EIO; + } + if (!route->xbar_up) { + /* never block a DCP call on the mux semaphore */ + ret = mux_control_try_select(route->active_xbar, route->mux_index); + if (!ret) + route->xbar_up = true; + } else { + ret = dcp_dpxbar_link(route->active_xbar, true); + } + if (ret) { + dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); + return ret; + } + return dcp_tunnel_dpin_locked(dcp, true); +} + +/* + * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an established + * link: DP IN inactive, crossbar clocks down (mux selection and ATC output + * enable kept). DP IN goes active again in dcp_tunnel_crossbar_up(). + */ +int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +{ + struct apple_dcp_typec_route *route; + + guard(mutex)(&dcp->tb_lock); + route = dcp->active_typec_route; + if (!route || !route->tunnel || !route->xbar_up) + return 0; + dcp_tunnel_dpin_locked(dcp, false); + return dcp_dpxbar_link(route->active_xbar, false); +} + +/* + * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the + * tunnel pixel clock. A stopped clock also takes the crossbar connection down. + */ +int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +{ + typeof(&apple_atc_dp_tunnel_rate) fn; + struct apple_dcp_typec_route *route; + int ret; + + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return -ENODEV; + fn = symbol_get(apple_atc_dp_tunnel_rate); + if (!fn) { + dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); + return -ENOENT; + } + route = dcp->active_typec_route; + if (!route) { + symbol_put(apple_atc_dp_tunnel_rate); + return -ENODEV; + } + if (!link_rate && route->xbar_up) + dcp_dpxbar_link(route->active_xbar, false); + ret = fn(phy, route->tunnel_dpin, link_rate); + symbol_put(apple_atc_dp_tunnel_rate); + dcp->tb_clock_ok = !ret && link_rate; + if (ret) + dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", + link_rate, ret); + return ret; +} + +/* Thunderbolt DP IN: DCP Activate/Deactivate */ +int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +{ + guard(mutex)(&dcp->tb_lock); + if (!dcp->dptx_tunnel) + return 0; + return dcp_tunnel_dpin_locked(dcp, active); +} + +/* + * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the + * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt + * connection manager has set up a tunnel from one of them, route a free + * display pipeline there and tell DCP a display is attached, so it trains + * the link (and completes DPRX) through the tunnel. + */ +int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, + bool active, int (*set_active)(void *ctx, bool active), + void *ctx) +{ + struct apple_dcp_typec_port *port = NULL, *pos; + struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; + struct apple_dcp_typec_route **slot; + unsigned int best_score = UINT_MAX; + struct mux_control *ctl; + struct apple_dcp *dcp; + bool ordered; + int ret; + + if (!connector_np || dpin > 1) + return -EINVAL; + + guard(mutex)(&dcp_typec_fabric_lock); + + list_for_each_entry(pos, &dcp_typec_ports, link) { + if (pos->connector_np == connector_np) { + port = pos; + break; + } + } + if (!port) + return -ENODEV; + slot = dpin ? &port->secondary_owner : &port->owner; + + if (!active) { + if (!*slot || !(*slot)->tunnel || + (*slot)->tunnel_dpin != dpin) + return 0; + dcp = (*slot)->dcp; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + ret = dcp_typec_route_deactivate(*slot); + if (ret) { + /* the caller's context is going away regardless */ + scoped_guard(mutex, &dcp->tb_lock) { + dcp->dptx_tunnel = false; + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + return ret; + } + *slot = NULL; + port->hpd = !!(port->owner || port->secondary_owner); + /* re-apply the next Type-C mux state in full */ + port->applied_valid = false; + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); + return 0; + } + + if (*slot) { + if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) + return 0; + dev_warn((*slot)->dcp->dev, + "port already routed, not taking DP tunnel dpin%u\n", dpin); + return -EBUSY; + } + if (port->owner && !port->owner->tunnel) + return -EBUSY; + + /* + * Until a compositor owns the display, the pairing pass decides where + * the stream goes, moving direct DP-alt routes out of its way, and + * whether it gets a pipeline at all. + */ + ordered = dcp_typec_keep_order(); + if (ordered) + planned = dcp_typec_rebalance_locked(port, dpin); - guard(mutex)(&dcp->tb_lock); - route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar) - return -ENODEV; - if (!dcp->tb_clock_ok) { - dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); - return -EIO; + list_for_each_entry(candidate, &port->routes, port_link) { + unsigned int score; + + if (ordered && candidate != planned) + continue; + if (!dcp_typec_route_available(candidate)) + continue; + if (!dcp_typec_route_fits(candidate, dpin ? + port->secondary_connector : + port->connector)) + continue; + score = dcp_typec_route_score(candidate); + /* + * DPIN0 prefers the hybrid dcpext0, which completes tunneled + * link training. On dual-stream machines DPIN1 is confined to + * dcpext1 by its connector's possible_crtcs, so both pipelines + * drive independent streams through one dock. + */ + if (dcp_typec_dual_stream() && dpin == 0 && + !candidate->dcp->fixed_phy && score < UINT_MAX - 100) + score += 100; + if (score < best_score) { + best = candidate; + best_score = score; + } } - if (!route->xbar_up) { - /* Never block a DCP call on the mux semaphore. */ - ret = mux_control_try_select(route->active_xbar, route->mux_index); - if (!ret) - route->xbar_up = true; - } else { - ret = dcp_dpxbar_link(route->active_xbar, true); + if (!best) { + ret = -EBUSY; + goto err_reorder; + } + + /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ + if (best->xbar != &best->xbar->chip->mux[0] || + best->xbar->chip->controllers < 3) { + ret = -EOPNOTSUPP; + goto err_reorder; + } + ctl = &best->xbar->chip->mux[1 + dpin]; + + dcp = best->dcp; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = set_active; + dcp->tb_dpin_ctx = ctx; } + best->tunnel_dpin = dpin; + ret = dcp_typec_route_activate(best, ctl); if (ret) { - dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); - return ret; + scoped_guard(mutex, &dcp->tb_lock) { + dcp->tb_dpin_set_active = NULL; + dcp->tb_dpin_ctx = NULL; + } + goto err_reorder; } - return dcp_tunnel_dpin_locked(dcp, true); + *slot = best; + /* the port is in USB4 mode, not DP-alt */ + port->dp_wanted = false; + dcp_tunnel_prepare(best, ctl); + + dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); + + /* + * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP + * drives the DPTX, i.e. from DCP's Activate call; waking it earlier + * hangs the machine. dptxep calls set_active back from Activate and + * Deactivate (set above, before the route became a tunnel). + */ + if (!dcp->typec_connector) + dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); + WRITE_ONCE(dcp->typec_cable_connected, true); + port->hpd = true; + if (dcp->typec_connector) + dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + + return 0; + +err_reorder: + /* the pass kept a pipeline for this stream: give it to the others */ + if (planned) + dcp_typec_rebalance_locked(NULL, 0); + return ret; } +EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); -/* - * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an - * established link: DP IN inactive, crossbar clocks down (mux selection and - * ATC output enable kept). DP IN goes active again in - * dcp_tunnel_crossbar_up(). - */ -int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +static struct apple_dcp_typec_port * +dcp_typec_port_get(struct device_node *connector_np) { struct apple_dcp_typec_route *route; @@ -700,6 +1677,44 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) } } +#define DCP_SWAP_WATCHDOG_MS 1000 +#define DCP_SWAP_WATCHDOG_RETRAINS 5 + +/* + * DCP drops swaps without completing them while an external pipe is not + * enabled, for instance after a modeset that raced a Type-C sink which had + * not asserted HPD yet. Userspace would then wait for the flip until the + * commit times out, stalling every output it drives. Complete the flip, + * mark the mode invalid so later commits do not wait for DCP, and let the + * hotplug worker re-apply the active CRTC. + */ +static void dcp_swap_watchdog(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + swap_watchdog_wq); + + dev_warn(dcp->dev, "swap not completed, retraining the display\n"); + dcp_mode_invalidate(&dcp->mode_state); + dcp_drm_crtc_vblank(dcp->crtc); + if (dcp->connector && + dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) + schedule_work(&dcp->connector->hotplug_wq); +} + +void dcp_swap_watchdog_arm(struct apple_dcp *dcp) +{ + if (dcp_is_typec_output(dcp)) + mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); +} + +void dcp_swap_watchdog_complete(struct apple_dcp *dcp) +{ + cancel_delayed_work(&dcp->swap_watchdog_wq); + dcp->swap_watchdog_retrains = 0; +} + static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { struct apple_dcp *dcp = @@ -1050,6 +2065,7 @@ static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, bool late_hpd = dcp_ext_dptx_607(dcp); unsigned long timeout; u8 dfp_port; + unsigned long timeout; int ret = 0; if (!dcp->phy) { @@ -1501,6 +2517,95 @@ int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) return ret; } +static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) +{ + const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); + static const u8 name[] = "Non-PnP"; + unsigned int i; + + if (!raw) + return false; + + /* EDID contains interior NUL bytes, so this cannot use strnstr(). */ + for (i = 0; i + sizeof(name) - 1 <= sizeof(struct edid); i++) { + if (!memcmp(raw + i, name, sizeof(name) - 1)) + return true; + timeout = dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? + DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; + ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, + timeout); + if (!ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: timed out waiting for port %u link configuration\n", + port); + ret = -ETIMEDOUT; + goto out_disconnect; + } + + return false; +} + dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", + jiffies_to_msecs(timeout - ret)); + +void dcp_retry_placeholder_edid(struct apple_dcp *dcp, + const struct drm_edid *drm_edid) +{ + guard(mutex)(&dcp->hpd_mutex); + if (!dcp_is_typec_output(dcp) || !dcp->typec_cable_connected || + dcp->placeholder_retried) + return; + if (!dcp_edid_is_placeholder(drm_edid)) + return; + + dcp->placeholder_retried = true; + dcp->placeholder_generation = dcp->typec_generation; + schedule_delayed_work(&dcp->placeholder_edid_wq, msecs_to_jiffies(300)); +} + +static void dcp_placeholder_edid_work(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + placeholder_edid_wq); + struct apple_epic_service *service; + u64 generation; + int ret; + + mutex_lock(&dcp->hpd_mutex); + generation = dcp->placeholder_generation; + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + !dcp->dptxport[0].enabled || generation != dcp->typec_generation) + goto out_unlock; + service = dcp->dptxport[0].service; + + /* + * Some adapters answer the first connection with a 1024x768 + * placeholder and publish the panel EDID only after HPD drops + * and returns. One pulse; a second placeholder is left alone. + */ + ret = dptxport_set_hpd(service, false); + if (ret) { + dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", + ret); + goto out_unlock; + } + mutex_unlock(&dcp->hpd_mutex); + + msleep(1000); + + mutex_lock(&dcp->hpd_mutex); + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + generation != dcp->typec_generation) + goto out_unlock; + + ret = dptxport_set_hpd(service, true); + if (ret) + dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", + ret); +out_unlock: + mutex_unlock(&dcp->hpd_mutex); +} + static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) { const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); @@ -2220,6 +3325,8 @@ void dcp_poweroff(struct platform_device *pdev) bool display_gone = false; int ret; + cancel_delayed_work(&dcp->swap_watchdog_wq); + /* Internal H17P DPMS is a brightness present, never a pipe stop. */ if (dcp_uses_soft_dpms(dcp)) { ret = dcp_backlight_dpms(dcp, false); @@ -2892,6 +3999,30 @@ static void dcp_ext_wait_pmp(struct apple_dcp *dcp) dev_info(dcp->dev, "PMP acknowledged the DISPEXT request\n"); } +static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) +{ + scoped_guard(mutex, &dcp->hpd_mutex) { + if (release_cable) + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } + /* Block new enqueues as well as draining users of the AFK endpoints. */ + disable_delayed_work_sync(&dcp->typec_reconnect_wq); + disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); +} + +static void dcp_enable_typec_work(struct apple_dcp *dcp) +{ + enable_delayed_work(&dcp->typec_reconnect_wq); + enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->typec_fabric_retrain_wq); + /* A cable can be routed before the DRM component binds. */ + if (READ_ONCE(dcp->typec_cable_connected)) + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); +} + +static int dcp_comp_bind(struct device *dev, struct device *main, void *data) static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; @@ -3109,6 +4240,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; } + dcp_enable_typec_work(dcp); return 0; err_piodma: @@ -3226,6 +4358,13 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_queue_stop(dcp); dcp_release_piodma_iommu_dev(dcp); + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); + } + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + cancel_work_sync(&dcp->vblank_wq); + devm_clk_put(dev, dcp->clk); dcp->clk = NULL; /* optional and possibly NULL, released with the bind's devres group */ @@ -3358,6 +4497,8 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); dcp_fabric_init(dcp); + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); @@ -3378,6 +4519,8 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) init_completion(&dcp->typec_iomfb_hpd_ready); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, dcp_typec_reconnect_work); + INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, + dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, dcp_typec_retrain_work); /* @@ -3404,6 +4547,10 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) disable_work(&dcp->vblank_wq); disable_delayed_work(&dcp->swap_watchdog_wq); } + /* Balanced by enable at successful component bind. */ + disable_delayed_work(&dcp->typec_reconnect_wq); + disable_delayed_work(&dcp->placeholder_edid_wq); + disable_delayed_work(&dcp->typec_fabric_retrain_wq); platform_set_drvdata(pdev, dcp); @@ -3901,6 +5048,14 @@ static int dcp_platform_suspend(struct device *dev) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + /* + * The Type-C route reports cable removal through + * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays + * connected, and resume powers the CRTC back up through dcp_poweron() + * as after DPMS off. + */ + dcp_disable_typec_work(dcp, false); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); if (dcp->avep) av_service_disconnect(dcp); @@ -3934,6 +5089,7 @@ static int dcp_platform_resume(struct device *dev) if (dcp->external_native) dcp_fabric_hdmi_reinit(dcp, "after system sleep"); /* Observe future edges before sampling any edges lost in sleep. */ + dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) dcp_fabric_run_resume(&dcp_resume_ops, dcp); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 9019ed48e8eb20..412ce7cf8b0ca9 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1531,7 +1531,7 @@ static void iomfb_backlight_start(struct apple_dcp *dcp, bool have_surface = false; unsigned int i; - if (!dcp_backlight_pending(dcp) || !dcp->valid_mode || !scanout || + if (!dcp_backlight_pending(dcp) || !READ_ONCE(dcp->mode_state.valid) || !scanout || !dcp->connector || !dcp->connector->connected) return; for (i = 0; i < SWAP_SURFACES; i++) @@ -1586,7 +1586,7 @@ static void iomfb_atomic_start(struct apple_dcp *dcp, struct iomfb_atomic_transaction *atomic = container_of(transaction, struct iomfb_atomic_transaction, transaction); - if (!dcp->valid_mode || !dcp->connector || !dcp->connector->connected) { + if (!READ_ONCE(dcp->mode_state.valid) || !dcp->connector || !dcp->connector->connected) { schedule_work(&dcp->vblank_wq); return; } diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 50286e9d7ebcb3..435d2a8969c4a8 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -20,6 +20,7 @@ #include "cd321x-pm.h" #define TPS_REG_MODE 0x03 +#include "cd321x-pm.h" #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) @@ -285,6 +286,16 @@ #define TPS_65981_2_6_INTEVENT_LEN 8 #define TPS_65987_8_INTEVENT_LEN 11 +/* + * Max data bytes for Data1, Data2, and other registers. See ch 1.3.2: + * https://www.ti.com/lit/ug/slvuan1a/slvuan1a.pdf + */ +#define TPS_MAX_LEN 64 + +/* Standard Task return codes */ +#define TPS_TASK_TIMEOUT 1 +#define TPS_TASK_REJECTED 3 + struct tps6598x; struct tipd_data { @@ -314,6 +325,7 @@ struct tps6598x { struct mutex lock; /* device lock */ int irq; u8 i2c_protocol:1; + u8 dbma_at_probe:1; /* Apple "DBMa" debug mode was active at probe */ struct gpio_desc *reset; struct typec_port *port; @@ -333,6 +345,8 @@ struct tps6598x { struct delayed_work wq_poll; const struct tipd_data *data; + + struct tipd_debugfs *debugfs; }; /* TPS_REG_USB4_STATUS */ @@ -432,4 +446,21 @@ void tipd_complete(struct device *dev); int tipd_suspend(struct tps6598x *tps); int tipd_resume(struct tps6598x *tps); +/* Internal to tps6598x-core */ +int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len); +int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, + size_t in_len, const u8 *in_data, + size_t out_len, u8 *out_data, + u32 cmd_timeout_ms, u32 res_delay_ms); + +#ifdef CONFIG_DEBUG_FS +void tipd_debugfs_register(struct tps6598x *tps); +void tipd_debugfs_unregister(struct tps6598x *tps); +void tipd_debugfs_exit(void); +#else +static inline void tipd_debugfs_register(struct tps6598x *tps) { } +static inline void tipd_debugfs_unregister(struct tps6598x *tps) { } +static inline void tipd_debugfs_exit(void) { } +#endif + #endif /* __TPS6598X_H__ */ From cf7ddd88652b1a7e63c8871ca0567fef6a2a121d Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:29:18 -0600 Subject: [PATCH 172/631] wifi: brcmfmac: add AWDL interface creation for BCM4387 BCM4387 firmware implements AWDL (Apple Wireless Direct Link) and accepts BRCMF_INTERFACE_TYPE_AWDL through interface_create, but nl80211 has no AWDL iftype, so the interface cannot be created via add_virtual_intf(). Drive it from a vendor command instead. Creation is asynchronous, and has to be. nl80211 dispatches vendor commands holding the wiphy mutex: NL80211_CMD_VENDOR sets NL80211_FLAG_NEED_WIPHY without NL80211_FLAG_NO_WIPHY_MTX, unlike NL80211_CMD_NEW_INTERFACE which sets NEED_RTNL | NO_WIPHY_MTX and takes the wiphy mutex itself, in that order. Registering a netdev from the vendor command therefore deadlocks: nl80211_vendor_cmd <- holds wiphy mutex brcmf_awdl_add_vif brcmf_net_attach register_netdev <- takes RTNL register_netdevice call_netdevice_notifiers_info cfg80211_netdev_notifier_call mutex_lock <- wiphy mutex, already held The task blocks in uninterruptible sleep while holding the RTNL, which takes down every netdev operation on the system; recovery needs a reboot. Using cfg80211_register_netdevice() instead is not a fix either: it requires both the RTNL and the wiphy mutex, and the vendor command holds only the latter, so it trips ASSERT_RTNL and registers into an inconsistent state. Taking the RTNL explicitly would invert cfg80211's RTNL-then-wiphy order. So brcmf_awdl_add_vif() only asks firmware for the interface and returns. The netdev is registered later by brcmf_cfg80211_awdl_attach_pending(), called from the fweh event worker, which runs on the system workqueue holding neither lock. The vif event stays armed across the request because brcmf_notify_vif_event() is what links the brcmf_if to the vif; without it ndev->ieee80211_ptr is never set and cfg80211 ignores the netdev. Userspace must wait for awdl0 to appear rather than assume the vendor command created it. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 141 ++++++++++++++++++ .../broadcom/brcm80211/brcmfmac/cfg80211.h | 5 + .../broadcom/brcm80211/brcmfmac/fweh.c | 7 + .../brcm80211/brcmfmac/interface_create.c | 8 +- .../brcm80211/brcmfmac/interface_create.h | 1 + .../broadcom/brcm80211/brcmfmac/vendor.c | 50 +++++++ .../broadcom/brcm80211/brcmfmac/vendor.h | 12 ++ 7 files changed, 223 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 9ff0c6829d1efb..d1064d8d904e72 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -605,6 +605,147 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * * Return: pointer to new vif on success, ERR_PTR(-errno) if not */ +/** + * brcmf_awdl_add_vif() - create an AWDL interface and give it a netdev. + * + * @wiphy: wiphy device of the new interface. + * @name: name of the new interface. + * + * AWDL is Apple Wireless Direct Link, the link layer AirDrop and AirPlay run + * over. BCM4387 firmware implements it and accepts BRCMF_INTERFACE_TYPE_AWDL + * through interface_create, but there is no nl80211 interface type for it, so + * this cannot go through add_virtual_intf(). It is driven from the vendor + * command instead. + * + * The vif is allocated as NL80211_IFTYPE_STATION because cfg80211 needs some + * valid type for its bookkeeping; the firmware side is what actually decides + * the interface behaviour, from the iftype passed to interface_create. + * + * Return: pointer to the new vif on success, ERR_PTR(-errno) on failure. + */ +int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) +{ + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_if *ifp = netdev_priv(cfg_to_ndev(cfg)); + struct brcmf_cfg80211_vif *vif; + int err; + + if (brcmf_cfg80211_vif_event_armed(cfg) || cfg->awdl_pending) + return -EBUSY; + + vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); + if (IS_ERR(vif)) + return PTR_ERR(vif); + + /* Arm before asking firmware: the BRCMF_E_IF event is what links the + * brcmf_if to this vif (brcmf_notify_vif_event() bails with -EBADF if + * nothing is armed, leaving ndev->ieee80211_ptr unset). + */ + brcmf_cfg80211_arm_vif_event(cfg, vif); + + /* Pass NULL rather than a derived address. Supplying a MAC address + * at interface_create time kills the firmware control channel about + * two seconds after create, so the address is set afterwards via + * cur_etheraddr instead. + * + */ + strscpy(cfg->awdl_ifname, name, sizeof(cfg->awdl_ifname)); + cfg->awdl_pending = true; + + err = brcmf_cfg80211_request_awdl_if(ifp, NULL); + if (err) { + cfg->awdl_pending = false; + brcmf_cfg80211_arm_vif_event(cfg, NULL); + brcmf_free_vif(vif); + return err; + } + + /* Deliberately do NOT wait for the interface here, and do NOT attach + * the netdev. nl80211 calls vendor commands holding the wiphy mutex + * (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX), and + * register_netdev() re-enters that mutex via the netdev notifier + * chain -> cfg80211_netdev_notifier_call(). Doing it here self- + * deadlocks and wedges the RTNL for the whole system. The attach is + * done from brcmf_cfg80211_awdl_attach_pending(), which the fweh + * event worker calls while holding no locks at all. + */ + return 0; +} + +/** + * brcmf_cfg80211_awdl_attach_pending() - finish AWDL interface creation. + * + * @ifp: interface the firmware just reported via BRCMF_E_IF_ADD. + * + * Called from the fweh event worker (brcmf_fweh_handle_if_event()), which + * runs on the system workqueue holding neither the RTNL nor the wiphy mutex. + * That is the only context in which register_netdev() is safe for an + * interface created outside cfg80211's own add_virtual_intf() path. + */ +void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) +{ + struct brcmf_cfg80211_info *cfg = ifp->drvr->config; + struct brcmf_cfg80211_vif *vif; + int err; + + if (!cfg || !cfg->awdl_pending) + return; + cfg->awdl_pending = false; + vif = ifp->vif; + + if (!ifp->ndev || !vif) { + bphy_err(ifp->drvr, "AWDL if event without netdev/vif\n"); + goto done; + } + + strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); + err = brcmf_net_attach(ifp, false); + if (err) { + bphy_err(ifp->drvr, "registering AWDL netdevice failed\n"); + free_netdev(ifp->ndev); + brcmf_free_vif(vif); + goto done; + } + + brcmf_info("AWDL interface %s created (bsscfgidx %d)\n", + ifp->ndev->name, ifp->bsscfgidx); +done: + brcmf_cfg80211_arm_vif_event(cfg, NULL); +} + +/** + * brcmf_awdl_del_vif() - tear down an interface made by brcmf_awdl_add_vif(). + * + * @wiphy: wiphy device of the interface. + * @wdev: wireless device to remove. + * + * Return: 0 on success, negative errno on failure. + */ +int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); + struct brcmf_cfg80211_vif *vif = container_of(wdev, + struct brcmf_cfg80211_vif, + wdev); + struct brcmf_if *ifp = vif->ifp; + int err; + + brcmf_cfg80211_arm_vif_event(cfg, vif); + err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); + if (err) { + brcmf_cfg80211_arm_vif_event(cfg, NULL); + return err; + } + + err = brcmf_cfg80211_wait_vif_event(cfg, BRCMF_E_IF_DEL, + BRCMF_VIF_EVENT_TIMEOUT); + brcmf_cfg80211_arm_vif_event(cfg, NULL); + if (!err) + return -EIO; + + return 0; +} + static struct wireless_dev *brcmf_apsta_add_vif(struct wiphy *wiphy, const char *name, struct vif_params *params, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index e15e5b689df956..2ab4c992fceb09 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -396,6 +396,8 @@ struct brcmf_cfg80211_info { struct work_struct escan_timeout_work; struct list_head vif_list; struct brcmf_cfg80211_vif_event vif_event; + bool awdl_pending; + char awdl_ifname[IFNAMSIZ]; struct completion vif_disabled; struct brcmu_d11inf d11inf; struct brcmf_assoclist_le assoclist; @@ -507,6 +509,9 @@ bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif); bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg); +int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name); +void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp); +int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev); int brcmf_cfg80211_wait_vif_event(struct brcmf_cfg80211_info *cfg, u8 action, ulong timeout); s32 brcmf_notify_escan_complete(struct brcmf_cfg80211_info *cfg, diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index eec4b10e1f6741..3e06c23defcb3f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -183,6 +183,13 @@ static void brcmf_fweh_handle_if_event(struct brcmf_pub *drvr, brcmf_fweh_call_event_handler(drvr, ifp, emsg->event_code, emsg, data); + /* The handler above establishes the vif <-> ifp linkage. Only then can + * the AWDL netdev be registered, and only from here: this worker holds + * no locks, unlike the vendor command that requested the interface. + */ + if (ifp && ifevent->action == BRCMF_E_IF_ADD) + brcmf_cfg80211_awdl_attach_pending(ifp); + if (ifp && ifevent->action == BRCMF_E_IF_DEL) { bool armed = brcmf_cfg80211_vif_event_armed(drvr->config); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index 1f40ff8d632c25..e9ebb90030575d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -52,7 +52,7 @@ enum brcmf_interface_type { BRCMF_INTERFACE_TYPE_STA = 0, BRCMF_INTERFACE_TYPE_AP = 1, - /* The missing number here is deliberate */ + BRCMF_INTERFACE_TYPE_AWDL = 2, BRCMF_INTERFACE_TYPE_NAN = 3, BRCMF_INTERFACE_TYPE_P2P_GO = 4, BRCMF_INTERFACE_TYPE_P2P_GC = 5, @@ -240,6 +240,12 @@ int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr) macaddr); } +int brcmf_cfg80211_request_awdl_if(struct brcmf_if *ifp, u8 *macaddr) +{ + return brcmf_cfg80211_request_if(ifp, BRCMF_INTERFACE_TYPE_AWDL, + macaddr); +} + int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp) { int err; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h index 669fa1508b67f6..7fadbea590e28b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.h @@ -9,5 +9,6 @@ int brcmf_cfg80211_request_sta_if(struct brcmf_if *ifp, u8 *macaddr); int brcmf_cfg80211_request_ap_if(struct brcmf_if *ifp); +int brcmf_cfg80211_request_awdl_if(struct brcmf_if *ifp, u8 *macaddr); #endif /* _BRCMF_INTERFACE_CREATE_H_ */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d07e7c7355d9d2..6dd6a3d7231fea 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -104,6 +104,46 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, return ret; } +#define BRCMF_AWDL_IFNAME "awdl0" + +static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, + struct wireless_dev *wdev, + const void *data, int len) +{ + struct net_device *ndev; + struct wireless_dev *awdl_wdev; + u32 op; + + if (len < sizeof(op)) + return -EINVAL; + op = *(const u32 *)data; + + switch (op) { + case BRCMF_VNDR_AWDL_OP_CREATE: + ndev = dev_get_by_name(wiphy_net(wiphy), BRCMF_AWDL_IFNAME); + if (ndev) { + dev_put(ndev); + return -EEXIST; + } + /* Asynchronous: returns as soon as firmware has been asked. + * The netdev appears once the fweh worker registers it, so + * userspace must wait for awdl0 rather than assume it exists. + */ + return brcmf_awdl_add_vif(wiphy, BRCMF_AWDL_IFNAME); + case BRCMF_VNDR_AWDL_OP_DESTROY: + ndev = dev_get_by_name(wiphy_net(wiphy), BRCMF_AWDL_IFNAME); + if (!ndev) + return -ENODEV; + awdl_wdev = ndev->ieee80211_ptr; + dev_put(ndev); + if (!awdl_wdev) + return -ENODEV; + return brcmf_awdl_del_vif(wiphy, awdl_wdev); + default: + return -EINVAL; + } +} + const struct wiphy_vendor_command brcmf_vendor_cmds[] = { { { @@ -115,4 +155,14 @@ const struct wiphy_vendor_command brcmf_vendor_cmds[] = { .policy = VENDOR_CMD_RAW_DATA, .doit = brcmf_cfg80211_vndr_cmds_dcmd_handler }, + { + { + .vendor_id = BROADCOM_OUI, + .subcmd = BRCMF_VNDR_CMDS_AWDL + }, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV | + WIPHY_VENDOR_CMD_NEED_NETDEV, + .policy = VENDOR_CMD_RAW_DATA, + .doit = brcmf_cfg80211_vndr_cmds_awdl_handler + }, }; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h index 418f33ea6fd3f9..74886de2ba78de 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h @@ -11,9 +11,21 @@ enum brcmf_vndr_cmds { BRCMF_VNDR_CMDS_UNSPEC, BRCMF_VNDR_CMDS_DCMD, + BRCMF_VNDR_CMDS_AWDL, BRCMF_VNDR_CMDS_LAST }; +/** + * enum brcmf_vndr_awdl_op - operations for BRCMF_VNDR_CMDS_AWDL + * + * @BRCMF_VNDR_AWDL_OP_CREATE: create the AWDL interface + * @BRCMF_VNDR_AWDL_OP_DESTROY: remove the AWDL interface + */ +enum brcmf_vndr_awdl_op { + BRCMF_VNDR_AWDL_OP_CREATE, + BRCMF_VNDR_AWDL_OP_DESTROY, +}; + /** * enum brcmf_nlattrs - nl80211 message attributes * From 63c86c57a3a53e0a0061fd7663c6779835aeeb74 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Wed, 26 Aug 2026 19:02:47 -0600 Subject: [PATCH 173/631] wifi: brcmfmac: use AWDL-specific netdev ops for awdl0 Bringing an AWDL interface up through brcmf_netdev_open() stops the firmware answering dcmds. Every subsequent command times out with -EIO, the infrastructure STA link drops with it, and only a brcmfmac reload recovers the device. The standard open path runs station bring-up against the interface: a toe_ol iovar query, brcmf_cfg80211_up(), and then the multicast and promisc configuration that ndo_set_rx_mode schedules via _brcmf_set_multicast_list(). An AWDL bsscfg is not a station, and issuing that sequence against one wedges the firmware control path. Give AWDL interfaces their own net_device_ops. brcmf_netdev_open_awdl() checks the bus is up, clears the 802.1x pending count, and leaves the carrier off, touching no firmware state. ndo_set_rx_mode is omitted entirely so the multicast worker never runs for these interfaces, and ndo_stop skips brcmf_cfg80211_down() for the same reason the open path skips brcmf_cfg80211_up(). awdl0 now comes up with the firmware still responsive and the STA link undisturbed. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 2 + .../broadcom/brcm80211/brcmfmac/core.c | 52 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/core.h | 1 + 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d1064d8d904e72..eed9482c84a617 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -698,6 +698,8 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) goto done; } + /* Must be set before brcmf_net_attach(), which picks netdev_ops. */ + ifp->is_awdl = true; strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); err = brcmf_net_attach(ifp, false); if (err) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 3c4125c3363552..8b01fcb1cad01c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -632,6 +632,55 @@ static int brcmf_netdev_open(struct net_device *ndev) return 0; } +/* AWDL interfaces are not STAs. brcmf_netdev_open() runs the standard station + * bring-up -- a toe_ol query, brcmf_cfg80211_up(), then the multicast and + * promisc configuration driven by ndo_set_rx_mode -- and issuing those against + * an AWDL bsscfg stops the firmware answering dcmds at all: every subsequent + * command times out and only a brcmfmac reload recovers it. Bring the netdev + * up without touching firmware, and deliberately omit ndo_set_rx_mode so the + * multicast worker never runs for these interfaces. + */ +static int brcmf_netdev_open_awdl(struct net_device *ndev) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + struct brcmf_pub *drvr = ifp->drvr; + struct brcmf_bus *bus_if = drvr->bus_if; + + brcmf_dbg(TRACE, "Enter, bsscfgidx=%d (AWDL)\n", ifp->bsscfgidx); + + if (bus_if->state != BRCMF_BUS_UP) { + bphy_err(drvr, "failed bus is not ready\n"); + return -EAGAIN; + } + + atomic_set(&ifp->pend_8021x_cnt, 0); + + /* Carrier stays off until AWDL is actually running. */ + netif_carrier_off(ndev); + return 0; +} + +static int brcmf_netdev_stop_awdl(struct net_device *ndev) +{ + struct brcmf_if *ifp = netdev_priv(ndev); + + brcmf_dbg(TRACE, "Enter, bsscfgidx=%d (AWDL)\n", ifp->bsscfgidx); + + /* No brcmf_cfg80211_down(): this vif was never brought up through + * cfg80211, and tearing down the STA state would hit the same + * unresponsive-firmware path as the bring-up. + */ + brcmf_net_setcarrier(ifp, false); + return 0; +} + +static const struct net_device_ops brcmf_netdev_ops_awdl = { + .ndo_open = brcmf_netdev_open_awdl, + .ndo_stop = brcmf_netdev_stop_awdl, + .ndo_start_xmit = brcmf_netdev_start_xmit, + .ndo_set_mac_address = brcmf_netdev_set_mac_address, +}; + static const struct net_device_ops brcmf_netdev_ops_pri = { .ndo_open = brcmf_netdev_open, .ndo_stop = brcmf_netdev_stop, @@ -651,7 +700,8 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) ndev = ifp->ndev; /* set appropriate operations */ - ndev->netdev_ops = &brcmf_netdev_ops_pri; + ndev->netdev_ops = ifp->is_awdl ? &brcmf_netdev_ops_awdl + : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; ndev->ethtool_ops = &brcmf_ethtool_ops; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 4b52a3aa855de8..451bcf85a4bdf0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -268,6 +268,7 @@ struct brcmf_if { struct in6_addr ipv6_addr_tbl[NDOL_MAX_ENTRIES]; u8 ipv6addr_idx; bool fwil_fwerr; + bool is_awdl; }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); From bf37b01ac8110f2643779a7c34585cdbcd9ed4cb Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 174/631] wifi: brcmfmac: return firmware error codes from the vendor dcmd brcmf_fil_cmd_data() collapses every firmware failure to -EBADE unless ifp->fwil_fwerr is set, so a userspace probe driving iovars through the Broadcom vendor dcmd passthrough cannot tell "no such iovar" from "bad argument" from "not up". Set fwil_fwerr around the call and log the raw BCME_* value. This makes the passthrough usable for firmware exploration: the same probe that previously reported EBADE for every failure now distinguishes BCME_BADOPTION, BCME_BUFTOOSHORT, BCME_NOTFOUND and BCME_NOTREADY. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../wireless/broadcom/brcm80211/brcmfmac/vendor.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index 6dd6a3d7231fea..d3211d092cf445 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -64,14 +64,25 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } + /* Return the firmware's own BCME_* code instead of the -EBADE that + * brcmf_fil_cmd_data() collapses every failure to. Without this a + * probe cannot tell "no such iovar" from "bad argument" from "not + * up", which is most of what a probe needs to know. + */ + ifp->fwil_fwerr = true; if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - if (ret != 0) + ifp->fwil_fwerr = false; + if (ret != 0) { + brcmf_err("dcmd %u ifidx=%d %s: firmware error %d\n", + cmdhdr->cmd, ifp->ifidx, + cmdhdr->set ? "set" : "get", ret); goto exit; + } wr_pointer = dcmd_buf; while (ret_len > 0) { From 07b1773ee831899eeb05659709c8e28ebf36319e Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 175/631] wifi: brcmfmac: create the AWDL interface brcmf_cfg80211_request_awdl_if() sent interface_create with MAC_USE alone: no BSSID and no requested interface index. On BCM4387 that yields an interface whose AWDL iovars stay unusable. Send flags 0x1a (MAC_USE | IF_INDEX_USE | BSSID_USE) with an explicit BSSID and if_index 2 instead. Firmware behaviour changes with those arguments: the interface is created at the requested index (wl0.2), awdl_peer_op returns data instead of BCME_NOMEM, and awdl_af_hdr, awdl_config and awdl_af_rssi are accepted where they previously were not. The three values are module parameters so they can be swept without rebuilding. Derive a MAC when the caller passes NULL rather than sending MAC_USE with an all-zero address. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../brcm80211/brcmfmac/interface_create.c | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index e9ebb90030575d..3a4f6b3be44708 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -22,6 +23,24 @@ #include "common.h" #include "interface_create.h" +/* Creating a usable AWDL interface requires interface_create flags + * MAC_USE | IF_INDEX_USE | BSSID_USE together with an explicit BSSID and + * an interface index; brcmfmac otherwise sends MAC_USE alone, which the + * firmware accepts but leaves the AWDL iovars unusable. These parameters + * exist so the three values can be swept without rebuilding the module. + */ +static uint brcmf_awdl_create_flags = 0x1a; +module_param_named(awdl_create_flags, brcmf_awdl_create_flags, uint, 0644); +MODULE_PARM_DESC(awdl_create_flags, "interface_create flags for AWDL"); + +static uint brcmf_awdl_if_index = 2; +module_param_named(awdl_if_index, brcmf_awdl_if_index, uint, 0644); +MODULE_PARM_DESC(awdl_if_index, "interface_create if_index for AWDL"); + +static char *brcmf_awdl_bssid = "00:25:00:ff:94:73"; +module_param_named(awdl_bssid, brcmf_awdl_bssid, charp, 0644); +MODULE_PARM_DESC(awdl_bssid, "interface_create BSSID for AWDL"); + #define BRCMF_INTERFACE_CREATE_VER_1 1 #define BRCMF_INTERFACE_CREATE_VER_2 2 #define BRCMF_INTERFACE_CREATE_VER_3 3 @@ -130,6 +149,8 @@ static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, enum brcmf_interface_type if_type, u8 *macaddr) { + bool awdl = if_type == BRCMF_INTERFACE_TYPE_AWDL; + switch (version) { case BRCMF_INTERFACE_CREATE_VER_1: { struct brcmf_interface_create_v1 iface_v1 = {}; @@ -182,6 +203,27 @@ static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); } + if (awdl) { + flags = brcmf_awdl_create_flags; + /* brcmf_awdl_add_vif() passes NULL, so derive a MAC + * here rather than send MAC_USE with an all-zero + * address. + */ + if ((flags & BRCMF_INTERFACE_MAC_USE) && + is_zero_ether_addr(iface_v3.mac_addr)) + brcmf_set_vif_sta_macaddr(ifp, + iface_v3.mac_addr); + iface_v3.if_index = brcmf_awdl_if_index; + if (mac_pton(brcmf_awdl_bssid, iface_v3.bssid)) { + brcmf_info("awdl interface_create: flags 0x%x, if_index %u, bssid %pM, mac %pM\n", + flags, iface_v3.if_index, + iface_v3.bssid, iface_v3.mac_addr); + } else { + bphy_err(ifp->drvr, "bad awdl_bssid \"%s\"\n", + brcmf_awdl_bssid); + return -EINVAL; + } + } iface_v3.flags = cpu_to_le32(flags); return brcmf_fil_iovar_data_get(ifp, "interface_create", &iface_v3, sizeof(iface_v3)); From dda2d103e639581914a65b98bcc3c31a2f03c9aa Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:07:12 -0600 Subject: [PATCH 176/631] wifi: brcmfmac: do not tear down the AWDL netdev from the vendor command nl80211 calls vendor commands holding the wiphy mutex and not the RTNL (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX). Calling brcmf_remove_interface() from brcmf_awdl_del_vif() therefore reaches cfg80211_close_dependents(), which asserts the RTNL: RTNL: assertion failed at net/wireless/core.c (419) cfg80211_close_dependents brcmf_remove_interface brcmf_awdl_del_vif brcmf_cfg80211_vndr_cmds_awdl_handler and then deadlocks the netdev layer against any concurrent RTNL holder, leaving NetworkManager and ip(8) unkillable in D state. Ask the firmware to drop the BSS and return. The BRCMF_E_IF_DEL event lands in brcmf_fweh_event_worker(), a plain work item holding neither lock, and brcmf_fweh_handle_if_event() removes the interface from there. This mirrors brcmf_awdl_add_vif(), which defers brcmf_net_attach() to the same worker for the same reason. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index eed9482c84a617..2970d835578dd7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -725,27 +725,33 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) */ int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) { - struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy); struct brcmf_cfg80211_vif *vif = container_of(wdev, struct brcmf_cfg80211_vif, wdev); struct brcmf_if *ifp = vif->ifp; int err; - brcmf_cfg80211_arm_vif_event(cfg, vif); - err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); - if (err) { - brcmf_cfg80211_arm_vif_event(cfg, NULL); - return err; - } - - err = brcmf_cfg80211_wait_vif_event(cfg, BRCMF_E_IF_DEL, - BRCMF_VIF_EVENT_TIMEOUT); - brcmf_cfg80211_arm_vif_event(cfg, NULL); - if (!err) - return -EIO; + /* Asymmetric with the rest of brcmfmac on purpose, for the same reason + * brcmf_awdl_add_vif() does not call brcmf_net_attach(): nl80211 calls + * vendor commands holding the wiphy mutex and NOT the RTNL + * (NL80211_CMD_VENDOR sets NEED_WIPHY without NO_WIPHY_MTX). Tearing + * the netdev down here reaches cfg80211_close_dependents(), which + * asserts the RTNL, and then deadlocks the whole netdev layer against + * any concurrent RTNL holder. Do NOT call brcmf_remove_interface() or + * arm the vif event from this context. + * + * Just ask the firmware to drop the BSS and return. The BRCMF_E_IF_DEL + * event lands in brcmf_fweh_event_worker(), which is a plain work item + * holding neither lock, and brcmf_fweh_handle_if_event() removes the + * interface from there. + */ + err = brcmf_fil_iovar_data_set(ifp, "interface_remove", NULL, 0); + if (err) + err = brcmf_fil_bsscfg_data_set(ifp, "interface_remove", NULL, 0); + if (err) + bphy_err(ifp->drvr, "interface_remove failed %d\n", err); - return 0; + return err; } static From 362d379ab21386dde5b684afac1d952e9c555856 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:52:58 -0600 Subject: [PATCH 177/631] wifi: brcmfmac: translate AWDL data frames at the awdl0 boundary AWDL data frames on the air are 802.3 + LLC/SNAP (OUI 00:17:f2, protocol 0x0800) + a 6-byte data header (03 04 00 00) + the real ethertype. The kernel only speaks Ethernet II, so for is_awdl interfaces: - RX: brcmf_rx_eth_type_trans() strips the encapsulation in place before eth_type_trans(). It is called from both delivery paths -- brcmf_rx_hdrpull (BCDC) and the msgbuf RX completion (PCIe); a first attempt in hdrpull alone was dead code on PCIe. Frames that are already Ethernet II pass. - TX: brcmf_awdl_encap() wraps Ethernet II in the same header with a per-interface incrementing sequence; needed_headroom grows by 16. Frames that already carry a length field (raw AF_PACKET senders building the encapsulation themselves) pass untouched. Also log the destination MAC on each awdl0 TX completion so per-frame fates can be read per peer (unicast to a MAC without a firmware peer entry completes 0x0003 FW_TOSSED; with one, 0x0000 acked). With this, EUI-64 neighbour pinning and awdl_peer_op ADD, a remote peer completes TCP to us over firmware-native AWDL. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/core.c | 76 ++++++++++++++++++- .../broadcom/brcm80211/brcmfmac/core.h | 4 + .../broadcom/brcm80211/brcmfmac/msgbuf.c | 50 ++++++++++-- 3 files changed, 121 insertions(+), 9 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 8b01fcb1cad01c..0392b2a8a90648 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,17 @@ #define D11_PHY_HDR_LEN 6 +/* AWDL data frames on the air are 802.3 + LLC/SNAP (OUI 00:17:f2, protocol + * 0x0800) + a 4-byte data header, then the real ethertype. The kernel only + * speaks Ethernet II, so the driver translates at the awdl0 boundary in + * both directions. + */ +#define BRCMF_AWDL_ENCAP_LEN 16 +#define BRCMF_AWDL_DATA_MIN_LEN (ETH_HLEN + BRCMF_AWDL_ENCAP_LEN) + +static const u8 brcmf_awdl_snap[] = { 0xaa, 0xaa, 0x03, 0x00, 0x17, 0xf2, + 0x08, 0x00, 0x03, 0x04 }; + struct d11rxhdr_le { __le16 RxFrameSize; u16 PAD; @@ -290,6 +302,55 @@ static bool brcmf_skb_is_iapp(struct sk_buff *skb) #endif } +/* Unwrap SNAP + AWDL data header into Ethernet II, in place. skb->data is + * the 802.3 header. Frames that are not AWDL-encapsulated pass unchanged. + */ +static void brcmf_awdl_decap(struct sk_buff *skb) +{ + const struct ethhdr *eh = (const struct ethhdr *)skb->data; + + if (skb->len < BRCMF_AWDL_DATA_MIN_LEN || + ntohs(eh->h_proto) > ETH_DATA_LEN || + memcmp(skb->data + ETH_HLEN, brcmf_awdl_snap, + sizeof(brcmf_awdl_snap))) + return; + + memmove(skb->data + BRCMF_AWDL_ENCAP_LEN, skb->data, 2 * ETH_ALEN); + skb_pull(skb, BRCMF_AWDL_ENCAP_LEN); +} + +/* Every received data frame passes through here, on the BCDC and the msgbuf + * (PCIe) paths alike, before the kernel classifies it. + */ +__be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb) +{ + if (ifp->is_awdl) + brcmf_awdl_decap(skb); + return eth_type_trans(skb, ifp->ndev); +} + +/* Wrap Ethernet II into 802.3 + SNAP + AWDL data header, in place. Frames + * already carrying a length field (raw AF_PACKET senders that build the + * encapsulation themselves) pass unchanged. Caller guarantees headroom. + */ +static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) +{ + const struct ethhdr *eh = (const struct ethhdr *)skb->data; + u8 *hdr; + + if (ntohs(eh->h_proto) < ETH_P_802_3_MIN) + return; + + hdr = skb_push(skb, BRCMF_AWDL_ENCAP_LEN); + memmove(hdr, hdr + BRCMF_AWDL_ENCAP_LEN, 2 * ETH_ALEN); + put_unaligned_be16(skb->len - ETH_HLEN, hdr + 2 * ETH_ALEN); + memcpy(hdr + ETH_HLEN, brcmf_awdl_snap, sizeof(brcmf_awdl_snap)); + put_unaligned_le16(ifp->awdl_tx_seq++, + hdr + ETH_HLEN + sizeof(brcmf_awdl_snap)); + hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 2] = 0; + hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 3] = 0; +} + static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct net_device *ndev) { @@ -298,6 +359,7 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, struct brcmf_pub *drvr = ifp->drvr; struct ethhdr *eh; int head_delta; + unsigned int hdrlen = drvr->hdrlen; unsigned int tx_bytes = skb->len; brcmf_dbg(DATA, "Enter, bsscfgidx=%d\n", ifp->bsscfgidx); @@ -328,9 +390,12 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, goto done; } + if (ifp->is_awdl) + hdrlen += BRCMF_AWDL_ENCAP_LEN; + /* Make sure there's enough writeable headroom */ - if (skb_headroom(skb) < drvr->hdrlen || skb_header_cloned(skb)) { - head_delta = max_t(int, drvr->hdrlen - skb_headroom(skb), 0); + if (skb_headroom(skb) < hdrlen || skb_header_cloned(skb)) { + head_delta = max_t(int, hdrlen - skb_headroom(skb), 0); brcmf_dbg(INFO, "%s: %s headroom\n", brcmf_ifname(ifp), head_delta ? "insufficient" : "unmodifiable"); @@ -365,6 +430,9 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, /* set pacing shift for packet aggregation */ sk_pacing_shift_update(skb->sk, 8); + if (ifp->is_awdl) + brcmf_awdl_encap(ifp, skb); + ret = brcmf_proto_tx_queue_data(drvr, ifp->ifidx, skb); if (ret < 0) brcmf_txfinalize(ifp, skb, false); @@ -495,7 +563,7 @@ static int brcmf_rx_hdrpull(struct brcmf_pub *drvr, struct sk_buff *skb, return -ENODATA; } - skb->protocol = eth_type_trans(skb, (*ifp)->ndev); + skb->protocol = brcmf_rx_eth_type_trans(*ifp, skb); return 0; } @@ -704,6 +772,8 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; + if (ifp->is_awdl) + ndev->needed_headroom += BRCMF_AWDL_ENCAP_LEN; ndev->ethtool_ops = &brcmf_ethtool_ops; /* set the mac address & netns */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h index 451bcf85a4bdf0..86c9ef09c90909 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.h @@ -250,6 +250,8 @@ enum brcmf_netif_stop_reason { * @pend_8021x_cnt: tracks outstanding number of 802.1x frames. * @pend_8021x_wait: used for signalling change in count. * @fwil_fwerr: flag indicating fwil layer should return firmware error codes. + * @is_awdl: interface carries AWDL data frames (SNAP + AWDL data header). + * @awdl_tx_seq: sequence number written into the AWDL data header on TX. */ struct brcmf_if { struct brcmf_pub *drvr; @@ -269,6 +271,7 @@ struct brcmf_if { u8 ipv6addr_idx; bool fwil_fwerr; bool is_awdl; + u16 awdl_tx_seq; }; int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp); @@ -285,6 +288,7 @@ void brcmf_txflowblock_if(struct brcmf_if *ifp, enum brcmf_netif_stop_reason reason, bool state); void brcmf_txfinalize(struct brcmf_if *ifp, struct sk_buff *txp, bool success); void brcmf_netif_rx(struct brcmf_if *ifp, struct sk_buff *skb); +__be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_netif_mon_rx(struct brcmf_if *ifp, struct sk_buff *skb); void brcmf_net_detach(struct net_device *ndev, bool locked); int brcmf_net_mon_attach(struct brcmf_if *ifp); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index f100ce8dcbfd37..20d612cd2629c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -929,11 +930,22 @@ brcmf_msgbuf_process_ioctl_complete(struct brcmf_msgbuf *msgbuf, void *buf) } +/* Firmware-reported completion status of AWDL data frames. brcmfmac never + * looks at tx_status; it is the WLFC_CTL_PKTFLAG_* fate of the frame + * (0 acked, 4 sent-no-ack i.e. multicast on air, 7 dropped by the firmware). + * Observed: awdl0 multicast completes ~1:3 sent:dropped, template or not. + * Logged for every frame queued on a non-primary flowring. 2000/s so a + * 200-frame burst at 1000/s is logged in full and in order -- the sequence + * of fates is the measurement. + */ +static DEFINE_RATELIMIT_STATE(brcmf_awdl_txs_rs, HZ, 2000); + static void brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) { struct brcmf_commonring *commonring; struct msgbuf_tx_status *tx_status; + struct brcmf_if *ifp; u32 idx; struct sk_buff *skb; u16 flowid; @@ -951,8 +963,23 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) commonring = msgbuf->flowrings[flowid]; atomic_dec(&commonring->outstanding_tx); - brcmf_txfinalize(brcmf_get_ifp(msgbuf->drvr, tx_status->msg.ifidx), - skb, true); + ifp = brcmf_get_ifp(msgbuf->drvr, tx_status->msg.ifidx); + /* Key on the flowring's ifidx, which the host assigned when it queued + * the frame, not on what the firmware echoes: repeated runs + * logged nothing for 1227 awdl0 frames each, and a completion + * echoed under ifidx 0 would have been invisible to both filters. + */ + if (brcmf_flowring_ifidx_get(msgbuf->flow, flowid) && + __ratelimit(&brcmf_awdl_txs_rs)) + pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + brcmf_flowring_ifidx_get(msgbuf->flow, flowid), + tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + + brcmf_txfinalize(ifp, skb, true); } @@ -1253,7 +1280,7 @@ brcmf_msgbuf_process_rx_complete(struct brcmf_msgbuf *msgbuf, void *buf) return; } - skb->protocol = eth_type_trans(skb, ifp->ndev); + skb->protocol = brcmf_rx_eth_type_trans(ifp, skb); brcmf_netif_rx(ifp, skb); } @@ -1601,14 +1628,25 @@ static int brcmf_msgbuf_stats_read(struct seq_file *seq, void *data) if (!msgbuf->flow->rings[i]) continue; ring = msgbuf->flow->rings[i]; - if (ring->status != RING_OPEN) + hash = &msgbuf->flow->hash[ring->hash_id]; + if (ring->status != RING_OPEN) { + /* A ring the firmware has not acknowledged holds every + * frame queued to it, invisibly, while tx_packets + * advances. Show it rather than skip it. + */ + seq_printf(seq, "id %3u: status %d (not open), qlen %4u\n" + " ifidx %u, fifo %u, da %pM\n", + i, ring->status, + skb_queue_len(&ring->skblist), + hash->ifidx, hash->fifo, hash->mac); continue; + } commonring = msgbuf->flowrings[i]; - hash = &msgbuf->flow->hash[ring->hash_id]; - seq_printf(seq, "id %3u: rp %4u, wp %4u, qlen %4u, blocked %u\n" + seq_printf(seq, "id %3u: rp %4u, wp %4u, qlen %4u, blocked %u, outstanding_tx %d\n" " ifidx %u, fifo %u, da %pM\n", i, commonring->r_ptr, commonring->w_ptr, skb_queue_len(&ring->skblist), ring->blocked, + atomic_read(&commonring->outstanding_tx), hash->ifidx, hash->fifo, hash->mac); } From 0178326e0b9e770624737963cd458123c0915128 Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:28:47 -0600 Subject: [PATCH 178/631] brcmfmac: msgbuf: tolerate txstatus for a freed flowring brcmf_msgbuf_delete_flowring() gives up on outstanding frames after ~75 ms and frees the ring; the firmware then completes them (tx_status 0x0006 EXPIRED) against a flow->rings[] slot that is already NULL. The awdl txstatus log line dereferenced it (oops in brcmf_flowring_ifidx_get from brcmf_msgbuf_process_rx on awdl-if destroy). Look the ring up only if it still exists, and reject a flowring id beyond max_flowrings before indexing. Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 20d612cd2629c0..4ef01f2fdf6963 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -949,6 +949,7 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) u32 idx; struct sk_buff *skb; u16 flowid; + u8 ring_ifidx; tx_status = (struct msgbuf_tx_status *)buf; idx = le32_to_cpu(tx_status->msg.request_id) - 1; @@ -959,6 +960,13 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) if (!skb) return; + if (flowid >= msgbuf->max_flowrings) { + bphy_err(msgbuf->drvr, "txstatus for flowring %u out of range\n", + flowid); + brcmu_pkt_buf_free_skb(skb); + return; + } + set_bit(flowid, msgbuf->txstatus_done_map); commonring = msgbuf->flowrings[flowid]; atomic_dec(&commonring->outstanding_tx); @@ -968,11 +976,18 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) * the frame, not on what the firmware echoes: repeated runs * logged nothing for 1227 awdl0 frames each, and a completion * echoed under ifidx 0 would have been invisible to both filters. + * + * The ring may already be gone: brcmf_msgbuf_delete_flowring() gives up + * on outstanding frames after ~75 ms and the firmware completes them + * (0x0006 EXPIRED) after the ring was freed -- awdl0's VO multicast + * never completes while the ring is open (observed as a NULL deref + * here on `awdl-if destroy`). The skb and its pktid are unaffected. */ - if (brcmf_flowring_ifidx_get(msgbuf->flow, flowid) && - __ratelimit(&brcmf_awdl_txs_rs)) + ring_ifidx = msgbuf->flow->rings[flowid] ? + brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; + if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - brcmf_flowring_ifidx_get(msgbuf->flow, flowid), + ring_ifidx, tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), le16_to_cpu(tx_status->tx_status), From b2fcb668fa48ad701121bec037cd02ffec25bf7d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:10 +1100 Subject: [PATCH 179/631] wifi: brcmfmac: bound AWDL creation and validate its data path Use the vif-event lock to arbitrate completion and timeout of asynchronous AWDL creation. Defer cleanup of a failed registration to wiphy work, and drain event and wiphy work before protocol teardown. Keep the netdevice reference and validate its wiphy and AWDL role when requesting deletion. Retain the vendor dcmd errno ABI rather than toggling shared fwil error state around commands. Account for the 16-byte AWDL envelope in the MTU and reject oversized Ethernet-II frames before writing an invalid 802.3 length. Route the per-frame measurement trace through dynamic debug. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../broadcom/brcm80211/brcmfmac/cfg80211.c | 150 +++++++++++++++--- .../broadcom/brcm80211/brcmfmac/cfg80211.h | 7 + .../broadcom/brcm80211/brcmfmac/core.c | 30 +++- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 14 +- .../broadcom/brcm80211/brcmfmac/vendor.c | 23 +-- 5 files changed, 177 insertions(+), 47 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index 2970d835578dd7..d1b2aa550ee5a0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -605,6 +605,85 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * * Return: pointer to new vif on success, ERR_PTR(-errno) if not */ +static void brcmf_awdl_cleanup_locked(struct brcmf_cfg80211_info *cfg) +{ + struct brcmf_cfg80211_vif *vif; + struct brcmf_if *ifp; + + spin_lock(&cfg->vif_event.vif_event_lock); + ifp = cfg->awdl_failed_ifp; + vif = cfg->awdl_vif; + cfg->awdl_failed_ifp = NULL; + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + cfg->awdl_attaching = false; + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); + + if (ifp) { + brcmf_fil_iovar_data_set(ifp, "interface_remove", NULL, 0); + brcmf_proto_del_if(cfg->pub, ifp); + ifp->vif = NULL; + } + if (vif) + brcmf_free_vif(vif); + if (ifp) { + if (ifp->ndev) { + ifp->ndev->ieee80211_ptr = NULL; + free_netdev(ifp->ndev); + } else { + kfree(ifp); + } + } +} + +static void brcmf_awdl_cleanup(struct wiphy *wiphy, struct wiphy_work *work) +{ + struct brcmf_cfg80211_info *cfg = + container_of(work, struct brcmf_cfg80211_info, awdl_cleanup); + + brcmf_awdl_cleanup_locked(cfg); +} + +void brcmf_cfg80211_awdl_stop(struct brcmf_cfg80211_info *cfg) +{ + wiphy_lock(cfg->wiphy); + wiphy_delayed_work_cancel(cfg->wiphy, &cfg->awdl_timeout); + wiphy_work_cancel(cfg->wiphy, &cfg->awdl_cleanup); + brcmf_awdl_cleanup_locked(cfg); + wiphy_unlock(cfg->wiphy); +} + +static void brcmf_awdl_timeout(struct wiphy *wiphy, struct wiphy_work *work) +{ + struct brcmf_cfg80211_info *cfg = + container_of(work, struct brcmf_cfg80211_info, awdl_timeout.work); + struct brcmf_cfg80211_vif *vif; + + spin_lock(&cfg->vif_event.vif_event_lock); + vif = cfg->awdl_vif; + /* IF_ADD has already claimed the vif; its worker owns completion. */ + if (!cfg->awdl_pending || cfg->awdl_attaching || + (vif && vif->ifp)) { + spin_unlock(&cfg->vif_event.vif_event_lock); + return; + } + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); + + brcmf_err("AWDL interface creation timed out\n"); + if (vif) + brcmf_free_vif(vif); +} + /** * brcmf_awdl_add_vif() - create an AWDL interface and give it a netdev. * @@ -621,7 +700,7 @@ brcmf_cfg80211_update_proto_addr_mode(struct wireless_dev *wdev) * valid type for its bookkeeping; the firmware side is what actually decides * the interface behaviour, from the iftype passed to interface_create. * - * Return: pointer to the new vif on success, ERR_PTR(-errno) on failure. + * Return: 0 when requested, or a negative error code. */ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) { @@ -630,7 +709,10 @@ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) struct brcmf_cfg80211_vif *vif; int err; - if (brcmf_cfg80211_vif_event_armed(cfg) || cfg->awdl_pending) + if (cfg->pub->bus_if->state != BRCMF_BUS_UP) + return -ENODEV; + if (brcmf_cfg80211_vif_event_armed(cfg) || + READ_ONCE(cfg->awdl_pending)) return -EBUSY; vif = brcmf_alloc_vif(cfg, NL80211_IFTYPE_STATION); @@ -650,13 +732,17 @@ int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name) * */ strscpy(cfg->awdl_ifname, name, sizeof(cfg->awdl_ifname)); - cfg->awdl_pending = true; + spin_lock(&cfg->vif_event.vif_event_lock); + cfg->awdl_vif = vif; + WRITE_ONCE(cfg->awdl_pending, true); + spin_unlock(&cfg->vif_event.vif_event_lock); + wiphy_delayed_work_queue(wiphy, &cfg->awdl_timeout, + BRCMF_VIF_EVENT_TIMEOUT); err = brcmf_cfg80211_request_awdl_if(ifp, NULL); if (err) { - cfg->awdl_pending = false; - brcmf_cfg80211_arm_vif_event(cfg, NULL); - brcmf_free_vif(vif); + /* An IF_ADD may already be in flight: retire under the same lock. */ + wiphy_delayed_work_queue(wiphy, &cfg->awdl_timeout, 0); return err; } @@ -688,31 +774,54 @@ void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp) struct brcmf_cfg80211_vif *vif; int err; - if (!cfg || !cfg->awdl_pending) + if (!cfg) return; - cfg->awdl_pending = false; - vif = ifp->vif; + spin_lock(&cfg->vif_event.vif_event_lock); + vif = cfg->awdl_vif; + if (!cfg->awdl_pending || cfg->awdl_attaching || !vif || + ifp->vif != vif) { + spin_unlock(&cfg->vif_event.vif_event_lock); + return; + } + cfg->awdl_attaching = true; + spin_unlock(&cfg->vif_event.vif_event_lock); - if (!ifp->ndev || !vif) { - bphy_err(ifp->drvr, "AWDL if event without netdev/vif\n"); - goto done; + if (!ifp->ndev) { + bphy_err(ifp->drvr, "AWDL if event without netdev\n"); + goto fail; } - /* Must be set before brcmf_net_attach(), which picks netdev_ops. */ ifp->is_awdl = true; strscpy(ifp->ndev->name, cfg->awdl_ifname, sizeof(ifp->ndev->name)); err = brcmf_net_attach(ifp, false); if (err) { bphy_err(ifp->drvr, "registering AWDL netdevice failed\n"); - free_netdev(ifp->ndev); - brcmf_free_vif(vif); - goto done; + goto fail; } + spin_lock(&cfg->vif_event.vif_event_lock); + cfg->awdl_vif = NULL; + WRITE_ONCE(cfg->awdl_pending, false); + cfg->awdl_attaching = false; + if (cfg->vif_event.vif == vif) { + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + } + spin_unlock(&cfg->vif_event.vif_event_lock); brcmf_info("AWDL interface %s created (bsscfgidx %d)\n", ifp->ndev->name, ifp->bsscfgidx); -done: - brcmf_cfg80211_arm_vif_event(cfg, NULL); + return; + +fail: + /* Do not free a vif while an event or wiphy operation can still see it. */ + spin_lock(&cfg->vif_event.vif_event_lock); + if (cfg->pub->iflist[ifp->bsscfgidx] == ifp) + cfg->pub->iflist[ifp->bsscfgidx] = NULL; + cfg->awdl_failed_ifp = ifp; + cfg->vif_event.vif = NULL; + cfg->vif_event.action = 0; + spin_unlock(&cfg->vif_event.vif_event_lock); + wiphy_work_queue(cfg->wiphy, &cfg->awdl_cleanup); } /** @@ -731,6 +840,9 @@ int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev) struct brcmf_if *ifp = vif->ifp; int err; + if (wdev->wiphy != wiphy || !ifp || !ifp->is_awdl) + return -ENODEV; + /* Asymmetric with the rest of brcmfmac on purpose, for the same reason * brcmf_awdl_add_vif() does not call brcmf_net_attach(): nl80211 calls * vendor commands holding the wiphy mutex and NOT the RTNL @@ -8813,6 +8925,8 @@ struct brcmf_cfg80211_info *brcmf_cfg80211_attach(struct brcmf_pub *drvr, cfg->wiphy = wiphy; cfg->pub = drvr; init_vif_event(&cfg->vif_event); + wiphy_delayed_work_init(&cfg->awdl_timeout, brcmf_awdl_timeout); + wiphy_work_init(&cfg->awdl_cleanup, brcmf_awdl_cleanup); INIT_LIST_HEAD(&cfg->vif_list); cfg->force_band_setup = true; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h index 2ab4c992fceb09..d548ca995e319d 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.h @@ -396,7 +396,13 @@ struct brcmf_cfg80211_info { struct work_struct escan_timeout_work; struct list_head vif_list; struct brcmf_cfg80211_vif_event vif_event; + /* Protected by vif_event.vif_event_lock. */ bool awdl_pending; + bool awdl_attaching; + struct brcmf_cfg80211_vif *awdl_vif; + struct brcmf_if *awdl_failed_ifp; + struct wiphy_delayed_work awdl_timeout; + struct wiphy_work awdl_cleanup; char awdl_ifname[IFNAMSIZ]; struct completion vif_disabled; struct brcmu_d11inf d11inf; @@ -509,6 +515,7 @@ bool brcmf_get_vif_state_any(struct brcmf_cfg80211_info *cfg, void brcmf_cfg80211_arm_vif_event(struct brcmf_cfg80211_info *cfg, struct brcmf_cfg80211_vif *vif); bool brcmf_cfg80211_vif_event_armed(struct brcmf_cfg80211_info *cfg); +void brcmf_cfg80211_awdl_stop(struct brcmf_cfg80211_info *cfg); int brcmf_awdl_add_vif(struct wiphy *wiphy, const char *name); void brcmf_cfg80211_awdl_attach_pending(struct brcmf_if *ifp); int brcmf_awdl_del_vif(struct wiphy *wiphy, struct wireless_dev *wdev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 0392b2a8a90648..a4460f06c1301c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -333,13 +333,15 @@ __be16 brcmf_rx_eth_type_trans(struct brcmf_if *ifp, struct sk_buff *skb) * already carrying a length field (raw AF_PACKET senders that build the * encapsulation themselves) pass unchanged. Caller guarantees headroom. */ -static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) +static int brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) { const struct ethhdr *eh = (const struct ethhdr *)skb->data; u8 *hdr; if (ntohs(eh->h_proto) < ETH_P_802_3_MIN) - return; + return 0; + if (skb->len - ETH_HLEN + BRCMF_AWDL_ENCAP_LEN > ETH_DATA_LEN) + return -EMSGSIZE; hdr = skb_push(skb, BRCMF_AWDL_ENCAP_LEN); memmove(hdr, hdr + BRCMF_AWDL_ENCAP_LEN, 2 * ETH_ALEN); @@ -349,6 +351,7 @@ static void brcmf_awdl_encap(struct brcmf_if *ifp, struct sk_buff *skb) hdr + ETH_HLEN + sizeof(brcmf_awdl_snap)); hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 2] = 0; hdr[ETH_HLEN + sizeof(brcmf_awdl_snap) + 3] = 0; + return 0; } static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, @@ -430,8 +433,13 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb, /* set pacing shift for packet aggregation */ sk_pacing_shift_update(skb->sk, 8); - if (ifp->is_awdl) - brcmf_awdl_encap(ifp, skb); + if (ifp->is_awdl) { + ret = brcmf_awdl_encap(ifp, skb); + if (ret) { + brcmf_txfinalize(ifp, skb, false); + goto done; + } + } ret = brcmf_proto_tx_queue_data(drvr, ifp->ifidx, skb); if (ret < 0) @@ -772,8 +780,12 @@ int brcmf_net_attach(struct brcmf_if *ifp, bool locked) : &brcmf_netdev_ops_pri; ndev->needed_headroom += drvr->hdrlen; - if (ifp->is_awdl) + if (ifp->is_awdl) { ndev->needed_headroom += BRCMF_AWDL_ENCAP_LEN; + /* The 802.3 length includes the AWDL envelope. */ + ndev->max_mtu = ETH_DATA_LEN - BRCMF_AWDL_ENCAP_LEN; + ndev->mtu = ndev->max_mtu; + } ndev->ethtool_ops = &brcmf_ethtool_ops; /* set the mac address & netns */ @@ -1612,14 +1624,18 @@ void brcmf_detach(struct device *dev) #endif brcmf_bus_change_state(bus_if, BRCMF_BUS_DOWN); + /* Stop event producers before draining their worker. */ + brcmf_bus_stop(drvr->bus_if); + /* Drain IF events before their interfaces and protocol state go away. */ + brcmf_fweh_detach(drvr); + if (drvr->config) + brcmf_cfg80211_awdl_stop(drvr->config); /* make sure primary interface removed last */ for (i = BRCMF_MAX_IFS - 1; i > -1; i--) { if (drvr->iflist[i]) brcmf_remove_interface(drvr->iflist[i], false); } - brcmf_bus_stop(drvr->bus_if); - brcmf_fweh_detach(drvr); brcmf_proto_detach(drvr); if (drvr->mon_if) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index 4ef01f2fdf6963..d6a312e7f151df 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -986,13 +986,13 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) ring_ifidx = msgbuf->flow->rings[flowid] ? brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) - pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - ring_ifidx, - tx_status->msg.ifidx, ifp ? ifp->is_awdl : -1, flowid, - (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), - le16_to_cpu(tx_status->tx_status), - le16_to_cpu(tx_status->metadata_len), - skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + brcmf_dbg(MSGBUF, "awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + ring_ifidx, tx_status->msg.ifidx, + ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); brcmf_txfinalize(ifp, skb, true); } diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c index d3211d092cf445..25623533effa7f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.c @@ -64,25 +64,14 @@ static int brcmf_cfg80211_vndr_cmds_dcmd_handler(struct wiphy *wiphy, *(char *)(dcmd_buf + len) = '\0'; } - /* Return the firmware's own BCME_* code instead of the -EBADE that - * brcmf_fil_cmd_data() collapses every failure to. Without this a - * probe cannot tell "no such iovar" from "bad argument" from "not - * up", which is most of what a probe needs to know. - */ - ifp->fwil_fwerr = true; if (cmdhdr->set) ret = brcmf_fil_cmd_data_set(ifp, cmdhdr->cmd, dcmd_buf, ret_len); else ret = brcmf_fil_cmd_data_get(ifp, cmdhdr->cmd, dcmd_buf, ret_len); - ifp->fwil_fwerr = false; - if (ret != 0) { - brcmf_err("dcmd %u ifidx=%d %s: firmware error %d\n", - cmdhdr->cmd, ifp->ifidx, - cmdhdr->set ? "set" : "get", ret); + if (ret != 0) goto exit; - } wr_pointer = dcmd_buf; while (ret_len > 0) { @@ -124,6 +113,7 @@ static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, struct net_device *ndev; struct wireless_dev *awdl_wdev; u32 op; + int err; if (len < sizeof(op)) return -EINVAL; @@ -146,10 +136,13 @@ static int brcmf_cfg80211_vndr_cmds_awdl_handler(struct wiphy *wiphy, if (!ndev) return -ENODEV; awdl_wdev = ndev->ieee80211_ptr; - dev_put(ndev); - if (!awdl_wdev) + if (!awdl_wdev || awdl_wdev->wiphy != wiphy) { + dev_put(ndev); return -ENODEV; - return brcmf_awdl_del_vif(wiphy, awdl_wdev); + } + err = brcmf_awdl_del_vif(wiphy, awdl_wdev); + dev_put(ndev); + return err; default: return -EINVAL; } From 8da48b74bc981fe2477b64271adb4511bd8c4b6b Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 13 Sep 2026 14:57:22 -0400 Subject: [PATCH 180/631] soc: apple: add production SEP FileVault integration Unwrap a FileVault volume's key through the enclave and load the volume's class keys into the enclave's volatile state, then drive the storage controller's inline AES-XTS with the wrapped key so an encrypted APFS volume is read and written with no plaintext key material on the application processor. The volume and key-bag records come from plaintext APFS metadata; only wrapped keys cross to the enclave. Assisted-by: GPT-5.6 Sol Assisted-by: Opus 5 Signed-off-by: Diljit Singh --- drivers/nvme/host/apple.c | 3 + drivers/soc/apple/sbio.rs | 4 +- drivers/soc/apple/sep.rs | 20 ++++-- drivers/soc/apple/sks.rs | 128 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 71490e5989cb23..5fa31030537283 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2447,6 +2447,9 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) if (anv->hw->has_lsq_nvmmu) anv->ctrl.crypto_profile = &anv->crypto_profile; + if (anv->hw->has_lsq_nvmmu) + anv->ctrl.crypto_profile = &anv->crypto_profile; + return anv; out_detach_genpd: apple_nvme_detach_genpd(anv); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 155381756e8138..a66e9610b955c7 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3969,6 +3969,7 @@ impl IdentityV1 { pub(crate) struct MatchResult { user_id: i32, identity: [u8; IDENTITY_UUID_LEN], + flags: u32, } impl MatchResult { @@ -3986,6 +3987,7 @@ impl MatchResult { bytes[MR_USER_ID + 3], ]), identity, + flags: u32::from_le_bytes(bytes[MR_FLAGS..MR_FLAGS + 4].try_into().ok()?), }) } @@ -3994,7 +3996,7 @@ impl MatchResult { } pub(crate) fn matches(&self, user: UserId) -> bool { - self.user_id == user.value() + self.flags & MR_FLAG_MATCH != 0 && self.user_id == user.value() } } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 14e410216ecee9..7d03856eb80483 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2243,10 +2243,7 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - // Taken before `capture_begin`, so the sleep drain never counts the - // wait for it. - let _writer = this.catacomb_writer.lock(); - if this.shutting_down.load(Relaxed) || !this.capture_begin() { + if this.shutting_down.load(Relaxed) { return; } this.run_enrolment(); @@ -2258,8 +2255,7 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - let _writer = this.catacomb_writer.lock(); - if this.shutting_down.load(Relaxed) || !this.capture_begin() { + if this.shutting_down.load(Relaxed) { return; } this.run_verify(); @@ -2431,5 +2427,17 @@ module! { default: 0, description: "Opt-in J414s enrollment test: export owner from the fresh context, then save user before master at completion", }, + provision_keybag: u8 { + default: 0, + description: "Create the Linux identity keybag when none exists; requires xart_writes=1", + }, + os_uuid_hi: u64 { + default: 0, + description: "High 64 bits of an explicit xART OS UUID", + }, + os_uuid_lo: u64 { + default: 0, + description: "Low 64 bits of an explicit xART OS UUID", + }, }, } diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index f882bf2e92e41c..2db7071c16a9b4 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -46,6 +46,36 @@ impl Drop for KeybagCreateIntent<'_> { } } +struct KeybagCreateIntent<'a> { + dev: &'a device::Device, + slot: keybag::Slot, + sent: bool, +} + +impl<'a> KeybagCreateIntent<'a> { + fn new(dev: &'a device::Device, slot: keybag::Slot) -> Option { + if let Err(e) = keybag::write_intent(slot) { + dev_err!(dev, "sks: cannot persist keybag-create intent: {:?}\n", e); + return None; + } + Some(Self { dev, slot, sent: false }) + } + + fn sending(&mut self) { + self.sent = true; + } +} + +impl Drop for KeybagCreateIntent<'_> { + fn drop(&mut self) { + if !self.sent { + if let Err(e) = keybag::mark_refused(self.slot) { + dev_err!(self.dev, "sks: cannot mark unsent keybag create retryable: {:?}\n", e); + } + } + } +} + impl SepData { pub(crate) fn on_sks(&self, msg: Message) { let r = crate::sks::decode_sks(&msg); @@ -752,6 +782,90 @@ impl SepData { None => keybag::write_generated_uuid(slot, &wrapped, &uuid, &secret), }; if let Err(e) = committed { + let mut body = image::Body::new(); + body.put_u32(crate::sks::SKS_CREATE_VARIANT_IDENTITY)?; + body.put_u64(crate::sks::SKS_CLIENT_ID)?; + body.put_u32(crate::sks::CreateFlags::none().value())?; + body.put_i32(crate::sks::SpecialHandle::first_identity().value())?; + body.put_blob(secret)?; + body.put_blob(&[])?; + body.put_blob(uuid)?; + body.put_blob(&[])?; + body.put_u64(0)?; + body.put_u64(0)?; + body.put_blob(&[])?; + let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; + let len = self.sks_image_len(&img)?; + let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; + Ok(SksRequest { name: crate::sks::SKS_CREATE_NAME, msg, img }) + } + + pub(crate) fn sks_provision_identity_keybag(&self) -> bool { + let proof = match keybag::read(keybag::Slot::Identity) { + Ok(keybag::State::Present(_)) => return true, + Ok(keybag::State::Absent(proof)) => proof, + Err(e) => { + dev_err!(self.dev, "sks: identity keybag state is ambiguous: {:?}\n", e); + return false; + } + }; + let slot = proof.slot(); + let mut intent = match KeybagCreateIntent::new(&self.dev, slot) { + Some(intent) => intent, + None => return false, + }; + let mut secret_bytes = KVec::new(); + if secret_bytes.resize(SKS_SECRET_LEN, 0, GFP_KERNEL).is_err() + || shim::random_bytes(&mut secret_bytes).is_err() + { + return false; + } + let secret = Secret(secret_bytes); + let mut uuid = [0u8; crate::sks::SKS_IDENTITY_UUID_LEN]; + if shim::random_bytes(&mut uuid).is_err() { + return false; + } + uuid[6] = (uuid[6] & 0x0f) | 0x40; + uuid[8] = (uuid[8] & 0x3f) | 0x80; + + let request = match self.sks_req_create_identity_keybag(&secret, &uuid, proof) { + Ok(request) => request, + Err(e) => { + dev_err!(self.dev, "sks: cannot build identity keybag request: {:?}\n", e); + return false; + } + }; + intent.sending(); + let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { + return false; + }; + let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { + return false; + }; + if out.reply.status != 0 || body.len() < 12 { + dev_err!(self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", out.reply.status, body.len()); + return false; + } + let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); + let raw_handle = i32::from_le_bytes(body[4..8].try_into().unwrap()); + let Some((_fv_data, end)) = image::read_blob(body, 8) else { + return false; + }; + if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() { + dev_err!(self.dev, "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", variant, raw_handle); + return false; + } + let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); + let Some(bag_uuid) = self.sks_read_uuid(handle) else { + return false; + }; + let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { + return false; + }; + let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { + return false; + }; + if let Err(e) = keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret) { dev_err!(self.dev, "sks: cannot commit identity keybag: {:?}\n", e); return false; } @@ -995,6 +1109,8 @@ impl SepData { let Some(uuid) = self.sks_read_uuid(handle) else { dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); let _ = self.sks_send(self.sks_req_unload_keybag(handle)); + let Some(uuid) = self.sks_read_uuid(handle) else { + dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); return None; }; @@ -1173,6 +1289,9 @@ pub(crate) const SKS_UNWRAP_VEK_NAME: &CStr = c"FV_UNWRAP_VEK"; const OP_SKS_SET_PROTECTION: u8 = 0x47; pub(crate) const SKS_SET_PROTECTION_NAME: &CStr = c"FV_SET_PROTECTION"; +const OP_SKS_SET_PROTECTION: u8 = 0x47; +pub(crate) const SKS_SET_PROTECTION_NAME: &CStr = c"FV_SET_PROTECTION"; + const OP_SKS_GET_BLOB_STATE: u8 = 0x48; pub(crate) const SKS_GET_BLOB_STATE_NAME: &CStr = c"FV_GET_BLOB_STATE"; @@ -1259,6 +1378,11 @@ pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; pub(crate) const SKS_DEVICE_STATE_NAME: &CStr = c"DEVICE_STATE_TRANSITION"; +pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; + +const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; + +pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; pub(crate) fn encode_sks_device_state_transition(seq: Sequence, len: ImageLen) -> Message { encode_sks_raw(OP_SKS_DEVICE_STATE_TRANSITION, seq.value(), len.value()) } @@ -1283,6 +1407,10 @@ impl KeyBagHandle { KeyBagHandle(v) } + pub(crate) const fn from_create_reply(v: i32) -> KeyBagHandle { + KeyBagHandle(v) + } + pub(crate) const fn from_load_reply(v: i32) -> KeyBagHandle { KeyBagHandle(v) } From 3459fa903c4b3bf3b1601766b8e18112d7ead1ba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:43:48 +1100 Subject: [PATCH 181/631] nvme: apple: guard crypto cleanup on controllers without TCBs T8015 does not allocate NVMMU transaction control blocks. Complete its ordinary DMA unmapping before returning, and access crypto state only on controllers with LSQ NVMMU support. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 5fa31030537283..fe504d9bc702a8 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -876,6 +876,9 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) if (blk_rq_nr_phys_segments(req)) apple_nvme_unmap_data(anv, req); + if (!anv->hw->has_lsq_nvmmu) + return; + tag = nvme_tag_from_cid(iod->cmd.common.command_id); tcb = &iod->q->tcbs[tag]; if (unlikely(tcb->dma_flags & BIT(2))) { From 58a91ce82c8f3205409a129baf77303057d92c93 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:16 +1100 Subject: [PATCH 182/631] soc: apple: sep: acquire the sensor enable GPIO by name The fingerprint binding uses enable-gpios. Request that named GPIO first while retaining the legacy unnamed property and board-specific fallback. Propagate acquisition errors and defer an unavailable GPIO supplier. Refuse binding when no usable power control can be obtained. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index ab6eb3ddb69b2e..818621cd20197a 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -216,8 +216,12 @@ static int sep_acquire_power(struct spi_device *spi) sep_power = gpiod_get(&spi->dev, "enable", GPIOD_OUT_LOW); if (IS_ERR(sep_power) && PTR_ERR(sep_power) == -ENOENT) sep_power = gpiod_get_index(&spi->dev, NULL, 0, GPIOD_OUT_LOW); - if (IS_ERR(sep_power) && PTR_ERR(sep_power) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (IS_ERR(sep_power) && PTR_ERR(sep_power) != -ENOENT) { + int error = PTR_ERR(sep_power); + + sep_power = NULL; + return error; + } if (!IS_ERR(sep_power)) { sep_power_source = SEP_POWER_NODE_PROPERTY; dev_info(&spi->dev, @@ -229,12 +233,12 @@ static int sep_acquire_power(struct spi_device *spi) /* * The fallback below is the j414s power line; a sibling board's sensor sits * on a different pin, so only j414s may use it. Every other machine must - * describe the power GPIO in its device node (the gpiod_get_index path). + * describe enable-gpios in its device node. */ if (!of_machine_is_compatible("apple,j414s")) { dev_warn(&spi->dev, - "sep sensor: no power GPIO in the device node; describe gpios in DT\n"); - return 0; + "sep sensor: no power GPIO in the device node; describe enable-gpios in DT\n"); + return -ENODEV; } np = of_find_node_by_path(SEP_SENSOR_GPIO_NODE); @@ -242,7 +246,7 @@ static int sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: no DT node at %s, no power line\n", SEP_SENSOR_GPIO_NODE); - return 0; + return -ENODEV; } gdev = gpio_device_find_by_fwnode(of_fwnode_handle(np)); @@ -251,7 +255,7 @@ static int sep_acquire_power(struct spi_device *spi) dev_warn(&spi->dev, "sep sensor: %s has no registered GPIO device\n", SEP_SENSOR_GPIO_NODE); - return 0; + return -EPROBE_DEFER; } gc = gpio_device_get_chip(gdev); @@ -268,7 +272,7 @@ static int sep_acquire_power(struct spi_device *spi) SEP_SENSOR_GPIO_LINE, SEP_SENSOR_GPIO_NODE, gpio_device_get_label(gdev)); gpio_device_put(gdev); - return 0; + return -ENODEV; } sep_power_source = SEP_POWER_CHIP_LINE; From 95bc10654c4c471925dad313cdc5314135de9b26 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:19 +1100 Subject: [PATCH 183/631] soc: apple: sep: retain the sensor SPI mode from the board An unconditional mode-2 assignment overrides the mode-1 J313 device tree. Retain the board mode unless a diagnostic override is supplied, and reject override values outside the SPI mode range. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sensor_shim.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 818621cd20197a..0464c1184b2847 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -335,11 +335,14 @@ static int sep_apply_cs_timing(struct spi_device *spi) * select. spi_setup() takes the controller io_mutex, not the bus lock, * so no deadlock. */ + if (sep_sensor_mode != ~0U && sep_sensor_mode > SPI_MODE_3) + return -EINVAL; + rc = spi_bus_lock(ctlr); if (rc) return rc; - if (sep_sensor_mode <= SPI_MODE_3) + if (sep_sensor_mode != ~0U) spi->mode = sep_sensor_mode; spi->bits_per_word = SEP_SENSOR_BITS; spi->max_speed_hz = SEP_SENSOR_HZ; From fee8beb0e4f9f0d71858f1c7b5be0dd0cd8c760c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:47:21 +1100 Subject: [PATCH 184/631] soc: apple: sep: wake detached waiters under the lifetime lock The last file release can free the biometric character device after unregister drops its mutex. Wake detached waiters before releasing that mutex so unregister cannot access the wait queue after destruction. The open-count decision still leaves exactly one path responsible for freeing the device. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio_shim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/bio_shim.c b/drivers/soc/apple/bio_shim.c index 50f7e7ddf61ea7..476c9e86cb6c99 100644 --- a/drivers/soc/apple/bio_shim.c +++ b/drivers/soc/apple/bio_shim.c @@ -168,9 +168,9 @@ void sep_bio_unregister(void *dev) mutex_lock(&d->lock); d->ctx = NULL; free_now = (d->open_count == 0); - mutex_unlock(&d->lock); - + /* The last close may free d as soon as the lifetime lock is dropped. */ wake_up_interruptible(&d->wq); + mutex_unlock(&d->lock); if (free_now) { mutex_destroy(&d->lock); From a5a3c450d7a015d664517f73c901fa53c2f60672 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:52:46 +1100 Subject: [PATCH 185/631] soc: apple: sep: drain captures before tearing down the sensor Reject new entry points and protocol sends during shutdown, wake and abort blocked transactions, and join capture work before powering down or unregistering the sensor. Use a device lifetime mutex and a protected sensor registry so independent SPI unbind also waits for exported sensor operations before releasing their resources. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 16 +- drivers/soc/apple/sensor_shim.c | 265 ++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 25 ++- drivers/soc/apple/sks.rs | 3 +- 4 files changed, 284 insertions(+), 25 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index a66e9610b955c7..9a16dc7e28a9f8 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2521,7 +2521,7 @@ impl SepData { name: &'static CStr, payload: &[u8], ) -> Result { - if !self.sbio_ready.load(Relaxed) { + if self.shutting_down.load(Relaxed) || !self.sbio_ready.load(Relaxed) { return Err(ENODEV); } @@ -2537,6 +2537,11 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); let mut guard = self.sbio_rx.lock(); loop { + if self.shutting_down.load(Relaxed) { + guard.abort(); + let _ = guard.take_done(); + return Err(ENODEV); + } if let Some(done) = guard.take_done() { return Ok(done); } @@ -2631,6 +2636,9 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(SBIO_TIMEOUT_MS); let mut guard = self.sbio_rx.lock(); loop { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } if guard.take_grant() { return Ok(()); } @@ -2680,6 +2688,9 @@ impl SepData { } pub(crate) fn bio_open(&self) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } let mut session = self.bio_session.lock(); bio::open(&mut session)?; drop(session); @@ -2815,6 +2826,9 @@ impl SepData { } pub(crate) fn bio_ioctl(&self, cmd: u32, arg: usize) -> Result { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } if cmd == bio::IOC_ATTEST { return self.bio_attest(arg); } diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index 0464c1184b2847..ee525b378d3a8d 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -19,6 +19,9 @@ #include #include +#include +#include + #include "shim.h" /* Use the board's device-tree mode (J313: 1, J414s: 2). */ @@ -35,6 +38,28 @@ MODULE_PARM_DESC(sensor_spi_mode, #define SEP_SENSOR_OFF_DELAY_MS 10 #define SEP_SENSOR_ON_DELAY_MS 7 +struct sep_sensor_lifetime { + /* Serializes sensor operations with this device's resource teardown. */ + struct mutex lock; +}; + +static DEFINE_MUTEX(sep_sensor_registry_lock); +static struct sep_sensor_lifetime *sep_active_sensor; + +static struct sep_sensor_lifetime *sep_lock_sensor(void) +{ + struct sep_sensor_lifetime *sensor; + + mutex_lock(&sep_sensor_registry_lock); + sensor = sep_active_sensor; + if (sensor) + mutex_lock(&sensor->lock); + mutex_unlock(&sep_sensor_registry_lock); + return sensor; +} + +static void sep_sensor_irq_teardown_locked(void); + static struct spi_device *sep_spi; static struct gpio_desc *sep_power; static bool sep_registered; @@ -377,7 +402,7 @@ static int sep_apply_cs_timing(struct spi_device *spi) return 0; } -static int sep_sensor_probe(struct spi_device *spi) +static int sep_sensor_probe_locked(struct spi_device *spi) { int rc; @@ -400,12 +425,42 @@ static int sep_sensor_probe(struct spi_device *spi) return 0; } +static int sep_sensor_probe(struct spi_device *spi) +{ + struct sep_sensor_lifetime *sensor; + int error; + + sensor = devm_kzalloc(&spi->dev, sizeof(*sensor), GFP_KERNEL); + if (!sensor) + return -ENOMEM; + mutex_init(&sensor->lock); + mutex_lock(&sep_sensor_registry_lock); + if (sep_active_sensor) { + error = -EBUSY; + } else { + error = sep_sensor_probe_locked(spi); + if (!error) { + spi_set_drvdata(spi, sensor); + sep_active_sensor = sensor; + } + } + mutex_unlock(&sep_sensor_registry_lock); + return error; +} + static void sep_sensor_remove(struct spi_device *spi) { - sep_sensor_irq_teardown(); + struct sep_sensor_lifetime *sensor = spi_get_drvdata(spi); + + mutex_lock(&sep_sensor_registry_lock); + mutex_lock(&sensor->lock); + sep_active_sensor = NULL; + sep_sensor_irq_teardown_locked(); sep_release_power(); sep_spi = NULL; sep_diag_set_sensor_bound(false); + mutex_unlock(&sensor->lock); + mutex_unlock(&sep_sensor_registry_lock); } /* @@ -462,19 +517,19 @@ void sep_sensor_unregister(void) sep_registered = false; } -int sep_sensor_bound(void) +static int sep_sensor_bound_locked(void) { return sep_spi != NULL; } /* Whether those delays reach hardware, are emulated, or are silently bypassed. */ -int sep_sensor_cs_timing_mode(void) +static int sep_sensor_cs_timing_mode_locked(void) { return sep_cs_timing_mode; } /* Power cycle: off, 10 ms, on, 7 ms. -ENODEV if there is no power line. */ -int sep_sensor_power_cycle(void) +static int sep_sensor_power_cycle_locked(void) { if (sep_power_source == SEP_POWER_SMC_RAILS) { int rc = sep_smc_set_power(0); @@ -499,12 +554,12 @@ int sep_sensor_power_cycle(void) } /* How the power line was obtained: none, the node's property, or chip+line. */ -int sep_sensor_power_source(void) +static int sep_sensor_power_source_locked(void) { return sep_power_source; } -int sep_sensor_power_line(void) +static int sep_sensor_power_line_locked(void) { if (sep_power_source == SEP_POWER_SMC_RAILS) return -ENODEV; @@ -518,7 +573,7 @@ int sep_sensor_power_line(void) * the sensor node's "firmware-name" property. NULL when absent, so the caller * can fall back to a default. */ -const char *sep_sensor_firmware_name(void) +static const char *sep_sensor_firmware_name_locked(void) { const char *name = NULL; @@ -530,7 +585,7 @@ const char *sep_sensor_firmware_name(void) } /* Powers the sensor on/off, holding the hardware settling delay. */ -int sep_sensor_power(int on) +static int sep_sensor_power_locked(int on) { if (sep_power_source == SEP_POWER_SMC_RAILS) { int rc = sep_smc_set_power(on ? 1 : 0); @@ -569,7 +624,7 @@ static irqreturn_t sep_drdy_isr(int irq, void *dev_id) * firmware patch. Returns 0 when an interrupt is available, -errno otherwise * (the caller then falls back to polling). */ -int sep_sensor_irq_setup(void) +static int sep_sensor_irq_setup_locked(void) { struct device_node *np; struct gpio_device *gdev; @@ -642,7 +697,7 @@ int sep_sensor_irq_setup(void) return 0; } -void sep_sensor_irq_teardown(void) +static void sep_sensor_irq_teardown_locked(void) { if (sep_drdy_irq >= 0) { free_irq(sep_drdy_irq, sep_spi); @@ -654,14 +709,14 @@ void sep_sensor_irq_teardown(void) } } -int sep_sensor_irq_available(void) +static int sep_sensor_irq_available_locked(void) { return sep_drdy_irq >= 0; } /* Clear any stale signal before a capture so the next wait reflects a fresh * data-ready edge, not a leftover from the previous frame. */ -void sep_sensor_irq_arm(void) +static void sep_sensor_irq_arm_locked(void) { if (sep_drdy_irq >= 0) reinit_completion(&sep_drdy_done); @@ -672,7 +727,7 @@ void sep_sensor_irq_arm(void) * -ETIMEDOUT = no signal (the caller re-polls status over SPI regardless, so a * missed interrupt only costs one poll interval), -ENODEV = no interrupt. */ -int sep_sensor_irq_wait(unsigned int timeout_ms) +static int sep_sensor_irq_wait_locked(unsigned int timeout_ms) { if (sep_drdy_irq < 0) return -ENODEV; @@ -687,7 +742,7 @@ int sep_sensor_irq_wait(unsigned int timeout_ms) * `len` transmit bytes (trailing ones as 0xff) rather than relying on the * controller running its transmit buffer dry. */ -int sep_sensor_xfer(const void *tx, void *rx, size_t len) +static int sep_sensor_xfer_locked(const void *tx, void *rx, size_t len) { struct spi_transfer xfer = { .tx_buf = tx, @@ -710,7 +765,7 @@ int sep_sensor_xfer(const void *tx, void *rx, size_t len) * A simultaneous receive changes the long transfer's pacing and the sensor * rejects the patch blob, so the receive must be absent entirely. */ -int sep_sensor_xfer_tx(const void *tx, size_t len) +static int sep_sensor_xfer_tx_locked(const void *tx, size_t len) { struct spi_transfer xfer = { .tx_buf = tx, @@ -732,7 +787,7 @@ int sep_sensor_xfer_tx(const void *tx, size_t len) * Two transfers in one chip-select assertion: command out, then read in. * Chip-select stays asserted because neither transfer sets cs_change. */ -int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +static int sep_sensor_xfer2_locked(const void *tx, size_t tx_len, void *rx, size_t rx_len) { struct spi_transfer xfers[2] = { { @@ -756,3 +811,179 @@ int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) return spi_sync_transfer(sep_spi, xfers, 2); } + +int sep_sensor_bound(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_bound_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_cs_timing_mode(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_cs_timing_mode_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_cycle(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_cycle_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_source(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_power_source_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power_line(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_line_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +const char *sep_sensor_firmware_name(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + const char *result; + + if (!sensor) + return NULL; + result = sep_sensor_firmware_name_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_power(int on) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_power_locked(on); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_irq_setup(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_irq_setup_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +void sep_sensor_irq_teardown(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + + if (!sensor) + return; + sep_sensor_irq_teardown_locked(); + mutex_unlock(&sensor->lock); +} + +int sep_sensor_irq_available(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return 0; + result = sep_sensor_irq_available_locked(); + mutex_unlock(&sensor->lock); + return result; +} + +void sep_sensor_irq_arm(void) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + + if (!sensor) + return; + sep_sensor_irq_arm_locked(); + mutex_unlock(&sensor->lock); +} + +int sep_sensor_irq_wait(unsigned int timeout_ms) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_irq_wait_locked(timeout_ms); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer(const void *tx, void *rx, size_t len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer_locked(tx, rx, len); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer_tx(const void *tx, size_t len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer_tx_locked(tx, len); + mutex_unlock(&sensor->lock); + return result; +} + +int sep_sensor_xfer2(const void *tx, size_t tx_len, void *rx, size_t rx_len) +{ + struct sep_sensor_lifetime *sensor = sep_lock_sensor(); + int result; + + if (!sensor) + return -ENODEV; + result = sep_sensor_xfer2_locked(tx, tx_len, rx, rx_len); + mutex_unlock(&sensor->lock); + return result; +} diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 7d03856eb80483..845ce50153605d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1076,6 +1076,9 @@ impl SepData { } fn send(&self, msg: Message) -> Result<()> { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } self.mbox.lock().as_ref().ok_or(ENODEV)?.send(msg, false) } @@ -1092,6 +1095,10 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(op.timeout_ms()); let mut guard = self.control.lock(); loop { + if self.shutting_down.load(Relaxed) { + guard.abandon(idx); + return Err(ENODEV); + } if let Some(reply) = guard.take_reply(idx) { guard.release(idx); drop(guard); @@ -1546,7 +1553,8 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(ms); let mut guard = self.scrd_probe.lock(); loop { - if guard.captured.len() >= until || remaining == 0 { + if self.shutting_down.load(Relaxed) + || guard.captured.len() >= until || remaining == 0 { return; } match self @@ -2053,6 +2061,12 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); + bio::release(&mut self.bio_session.lock()); + self.rng_shutdown.store(true, Relaxed); + self.control_wq.notify_all(); + self.sbio_wq.notify_all(); + self.sks_wq.notify_all(); + self.scrd_wq.notify_all(); pm::unregister(); // Release any capture start still held for a resume that will not be // reported now. @@ -2062,11 +2076,6 @@ impl SepData { self.unregister_fv_kernel(); let dev = self.bio_dev.lock().take(); - if dev.is_some() { - bio::release(&mut self.bio_session.lock()); - sensor::power(false); - sensor::unregister_driver(); - } drop(dev); self.rng_shutdown.store(true, Relaxed); @@ -2091,6 +2100,10 @@ impl SepData { ); } + sensor::power(false); + sensor::unregister_driver(); + self.release_firmware_mapping(); + let _ = self.store.lock().take(); let _ = self.host_store.lock().take(); diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 2db7071c16a9b4..10b771e9801a4c 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -1243,7 +1243,8 @@ impl SepData { let mut remaining = time::msecs_to_jiffies(ms); let mut guard = self.sks_probe.lock(); loop { - if guard.captured.len() >= until || remaining == 0 { + if self.shutting_down.load(Relaxed) + || guard.captured.len() >= until || remaining == 0 { return; } match self From 5060d2961efaca57f95a533d9bc04ca483462d40 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:00:10 +1100 Subject: [PATCH 186/631] soc: apple: sep: bind capture results to their queued operation Assign a generation to each queued capture and retain worker ownership across cancellation and close. Refuse a replacement operation until the worker has retired, and check the queued generation before accepting progress, guidance or a terminal result. This prevents an old verify result from being bound to a new request nonce and serializes enrol and verify work against the shared sensor and buffers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 66 +++++++++++++++++++++++++++++++++------ drivers/soc/apple/sbio.rs | 32 +++++++++++++++---- drivers/soc/apple/sep.rs | 12 ++++--- 3 files changed, 90 insertions(+), 20 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index c2bbf63364f3ae..bd66d0011f3890 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -420,6 +420,8 @@ impl Op { } pub(crate) struct Session { + generation: u64, + worker_epoch: Option, open: bool, op: Op, unseen: bool, @@ -430,6 +432,8 @@ pub(crate) struct Session { impl Session { pub(crate) fn new() -> Session { Session { + generation: 0, + worker_epoch: None, open: false, op: Op::Idle, unseen: false, @@ -439,6 +443,7 @@ impl Session { } fn reset(&mut self) { + self.generation = self.generation.saturating_add(1); self.open = false; self.op = Op::Idle; self.unseen = false; @@ -446,6 +451,15 @@ impl Session { self.token = None; } + fn start_worker(&mut self) -> Result<()> { + if self.worker_epoch.is_some() { + return Err(EBUSY); + } + self.generation = self.generation.checked_add(1).ok_or(EOVERFLOW)?; + self.worker_epoch = Some(self.generation); + Ok(()) + } + pub(crate) fn ready(&self) -> bool { self.unseen } @@ -563,7 +577,7 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { } require_sensor(ctx.sensor_present)?; - if !ctx.session.op.may_start() { + if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } if matches!(ctx.live_identity_count, Some(count) if count >= SEP_USER_CAPACITY) @@ -572,6 +586,7 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { return Err(ENOSPC); } + ctx.session.start_worker()?; ctx.session.op = Op::Enrol { label: request.label, stage: 0, @@ -593,10 +608,14 @@ fn enrol_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { pub(crate) fn enrol_advance( session: &mut Session, + epoch: u64, completed: u32, percent_now: u32, guide: Guidance, ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } if let Op::Enrol { stage, terminal, @@ -617,7 +636,10 @@ pub(crate) fn enrol_advance( false } -pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { +pub(crate) fn capture_guide(session: &mut Session, epoch: u64, guide: Guidance) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let changed = match &mut session.op { Op::Enrol { terminal, guidance, .. } => { if terminal.is_none() && *guidance != guide { @@ -645,9 +667,13 @@ pub(crate) fn capture_guide(session: &mut Session, guide: Guidance) -> bool { pub(crate) fn enrol_finish( session: &mut Session, + epoch: u64, index: &mut IdentityIndex, outcome: core::result::Result<[u8; UUID_LEN], u32>, ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let Op::Enrol { label, terminal, .. } = &mut session.op @@ -673,8 +699,8 @@ pub(crate) fn enrol_finish( true } -pub(crate) fn enrol_is_live(session: &Session) -> bool { - matches!(&session.op, Op::Enrol { terminal: None, .. }) && session.open +pub(crate) fn enrol_is_live(session: &Session, epoch: u64) -> bool { + worker_is_current(session, epoch) && matches!(&session.op, Op::Enrol { terminal: None, .. }) } fn enrol_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { @@ -735,7 +761,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { } require_sensor(ctx.sensor_present)?; - if !ctx.session.op.may_start() { + if ctx.session.worker_epoch.is_some() || !ctx.session.op.may_start() { return Err(EBUSY); } @@ -757,6 +783,7 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { }); } + ctx.session.start_worker()?; ctx.session.token = None; ctx.session.op = Op::Verify { nonce: request.nonce, @@ -775,9 +802,13 @@ fn verify_start(ctx: &mut Context<'_>, user: UserPtr) -> Result { pub(crate) fn verify_finish( session: &mut Session, + epoch: u64, outcome: VerifyOutcome, token_bytes: [u8; TOKEN_LEN], ) -> bool { + if !worker_is_current(session, epoch) { + return false; + } let Op::Verify { nonce, terminal, .. } = &mut session.op else { return false; }; @@ -796,12 +827,26 @@ pub(crate) fn verify_finish( true } -pub(crate) fn verify_is_live(session: &Session) -> bool { - matches!(&session.op, Op::Verify { terminal: None, .. }) && session.open +pub(crate) fn verify_is_live(session: &Session, epoch: u64) -> bool { + worker_is_current(session, epoch) && matches!(&session.op, Op::Verify { terminal: None, .. }) +} + +pub(crate) fn capture_is_live(session: &Session, epoch: u64) -> bool { + enrol_is_live(session, epoch) || verify_is_live(session, epoch) } -pub(crate) fn capture_is_live(session: &Session) -> bool { - enrol_is_live(session) || verify_is_live(session) +pub(crate) fn queued_epoch(session: &Session) -> Option { + session.worker_epoch +} + +fn worker_is_current(session: &Session, epoch: u64) -> bool { + session.open && session.generation == epoch && session.worker_epoch == Some(epoch) +} + +pub(crate) fn worker_stopped(session: &mut Session, epoch: u64) { + if session.worker_epoch == Some(epoch) { + session.worker_epoch = None; + } } fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { @@ -871,6 +916,7 @@ fn cancel(ctx: &mut Context<'_>) -> Result { Op::Verify { terminal, .. } => *terminal = Some(VerifyOutcome::Failed(STATUS_LOCAL)), } + ctx.session.generation = ctx.session.generation.saturating_add(1); ctx.session.token = None; ctx.session.unseen = true; @@ -972,7 +1018,7 @@ fn delete_all(ctx: &mut Context<'_>) -> Result { } pub(crate) fn open(session: &mut Session) -> Result<()> { - if session.open { + if session.open || session.worker_epoch.is_some() { return Err(EBUSY); } session.reset(); diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 9a16dc7e28a9f8..3cc1c513ea6cb4 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -263,7 +263,9 @@ impl SepData { let filed = { let mut session = self.bio_session.lock(); let mut index = self.bio_index.lock(); - bio::enrol_finish(&mut session, &mut index, outcome) + bio::enrol_finish( + &mut session, self.capture_epoch.load(Relaxed), &mut index, outcome, + ) }; if !filed { return; @@ -1580,7 +1582,7 @@ impl SepData { let mut last_percent: u32 = 0; let outcome = loop { - if !bio::enrol_is_live(&self.bio_session.lock()) { + if !bio::enrol_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { break None; } if counter >= ENROL_MAX_CAPTURES { @@ -1607,6 +1609,7 @@ impl SepData { last_percent = last_percent.max(percent); let woke = bio::enrol_advance( &mut self.bio_session.lock(), + self.capture_epoch.load(Relaxed), stage, percent, bio::Guidance::LiftAndMove, @@ -1635,7 +1638,10 @@ impl SepData { // The capture did not take (partial/unusable contact). Nudge // the person to hold still and press again rather than // silently re-capturing with no feedback. - if bio::capture_guide(&mut self.bio_session.lock(), bio::Guidance::HoldStill) { + if bio::capture_guide( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), + bio::Guidance::HoldStill, + ) { self.bio_wake(); } } @@ -2358,7 +2364,7 @@ impl SepData { ); let mut attempt: u32 = 0; loop { - if !bio::capture_is_live(&self.bio_session.lock()) { + if !bio::capture_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { return CaptureWait::Abandon; } @@ -2383,7 +2389,9 @@ impl SepData { _ => None, }; if let Some(guide) = guide { - if bio::capture_guide(&mut self.bio_session.lock(), guide) { + if bio::capture_guide( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), guide, + ) { self.bio_wake(); } } @@ -2785,15 +2793,24 @@ impl SepData { } pub(crate) fn queue_enrolment(this: Arc) { + let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { + return; + }; + this.capture_epoch.store(epoch, Relaxed); if workqueue::system() .enqueue::, ENROL_WORK_ID>(this.clone()) .is_err() { this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } pub(crate) fn queue_verify(this: Arc) { + let Some(epoch) = bio::queued_epoch(&this.bio_session.lock()) else { + return; + }; + this.capture_epoch.store(epoch, Relaxed); if workqueue::system() .enqueue::, VERIFY_WORK_ID>(this.clone()) .is_err() @@ -2802,11 +2819,14 @@ impl SepData { bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), [0u8; bio::TOKEN_LEN], ); + bio::worker_stopped(&mut this.bio_session.lock(), epoch); } } fn finish_verify(&self, outcome: bio::VerifyOutcome, token_bytes: [u8; bio::TOKEN_LEN]) { - let filed = bio::verify_finish(&mut self.bio_session.lock(), outcome, token_bytes); + let filed = bio::verify_finish( + &mut self.bio_session.lock(), self.capture_epoch.load(Relaxed), outcome, token_bytes, + ); if filed { self.bio_wake(); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 845ce50153605d..99b57d18727a39 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -686,6 +686,7 @@ struct SepData { registered: Atomic, shutting_down: Atomic, + capture_epoch: Atomic, // The system is going to sleep: no capture may start. Changed under the // bio session lock; `sleep_over` is signalled when it clears. @@ -878,6 +879,7 @@ impl SepData { settle_idle_ticks: Atomic::new(0), registered: Atomic::new(false), shutting_down: Atomic::new(false), + capture_epoch: Atomic::new(0), suspending: Atomic::new(false), sleep_over <- new_condvar!("SepData::sleep_over"), captures_running: Atomic::new(0), @@ -2256,8 +2258,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - if this.shutting_down.load(Relaxed) { - return; + let epoch = this.capture_epoch.load(Relaxed); + if !this.shutting_down.load(Relaxed) { + this.run_enrolment(); } this.run_enrolment(); this.capture_end(); @@ -2268,8 +2271,9 @@ impl WorkItem for SepData { type Pointer = Arc; fn run(this: Arc) { - if this.shutting_down.load(Relaxed) { - return; + let epoch = this.capture_epoch.load(Relaxed); + if !this.shutting_down.load(Relaxed) { + this.run_verify(); } this.run_verify(); this.capture_end(); From 6b5273926afbf1cd92373651f859661d31921472 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:02:11 +1100 Subject: [PATCH 187/631] soc: apple: sep: export the biometric ioctl header through UAPI Move the public biometric interface into include/uapi/linux and use conventional ioctl macro spelling without changing its fixed-width layouts or ioctl values. Make the libfprint patch consume the exported header instead of maintaining a duplicated ABI definition, and document header installation for its build. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 + include/uapi/linux/apple-sep.h | 133 +++++++++++++++++++++++++++++++++ tools/aurora-sep/README.md | 8 ++ 3 files changed, 143 insertions(+) create mode 100644 include/uapi/linux/apple-sep.h diff --git a/MAINTAINERS b/MAINTAINERS index e36f0a27b8737b..d61cff397b2e9d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2631,6 +2631,7 @@ F: drivers/nvmem/apple-spmi-nvmem.c F: drivers/phy/apple/ F: drivers/phy/apple/.kunitconfig.apple-atc F: drivers/pinctrl/pinctrl-apple-gpio.c +F: drivers/pmdomain/apple/ F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c @@ -2655,6 +2656,7 @@ F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: tools/testing/selftests/drivers/apple/ F: include/uapi/drm/asahi_drm.h +F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h new file mode 100644 index 00000000000000..81467fe91f36a0 --- /dev/null +++ b/include/uapi/linux/apple-sep.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note OR MIT */ +/* Copyright 2026 Dj */ +/* + * Userspace interface for the SEP biometric device. + * + * The enclave does the matching. Nothing biometric crosses this interface: only + * an operation, a stage, a status, an opaque identity UUID, and an opaque host + * label userspace chooses. + */ + +#ifndef _UAPI_LINUX_APPLE_SEP_H +#define _UAPI_LINUX_APPLE_SEP_H + +#include +#include + +#define SEP_BIO_IFACE_VERSION 4 + +#define SEP_BIO_UUID_LEN 16 +#define SEP_BIO_LABEL_LEN 128 +#define SEP_BIO_NONCE_LEN 32 +#define SEP_BIO_TOKEN_LEN 32 +#define SEP_BIO_MAX_IDENTITIES 32 +#define SEP_BIO_CHALLENGE_LEN 32 +#define SEP_BIO_ATTEST_PUB_LEN 65 +#define SEP_BIO_ATTEST_SIG_MAX 72 + +enum { + SEP_BIO_STATE_IDLE = 0, + SEP_BIO_STATE_PENDING = 1, + SEP_BIO_STATE_PROGRESS = 2, + SEP_BIO_STATE_DONE = 3, + SEP_BIO_STATE_FAILED = 4, +}; + +enum { + SEP_BIO_NO_MATCH = 0, + SEP_BIO_MATCH = 1, + SEP_BIO_NOT_COMPARED = 2, +}; + +struct sep_bio_identity { + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +struct sep_bio_info { + __u32 version; + __u32 sensor_present; + __u32 enrolled; + __u32 capacity; /* usable slots in this SEP user context, not ABI list size */ + __u32 enroll_stages; + __u32 reserved[3]; +}; + +/* reserved[0] flags; reserved[1] is a live SEP identity count only when valid. */ +#define SEP_BIO_INFO_LIVE_COUNT_VALID 1u + +struct sep_bio_list { + __u32 count; + __u32 reserved; + struct sep_bio_identity id[SEP_BIO_MAX_IDENTITIES]; +}; + +struct sep_bio_enrol_start { + __u32 flags; + __u32 reserved; + __u8 label[SEP_BIO_LABEL_LEN]; +}; + +#define SEP_BIO_GUIDANCE_NONE 0 +#define SEP_BIO_GUIDANCE_PLACE 1 +#define SEP_BIO_GUIDANCE_LIFT_AND_MOVE 2 +#define SEP_BIO_GUIDANCE_HOLD_STILL 3 + +struct sep_bio_enrol_poll { + __u32 state; + __u32 stage; + __u32 stages_total; + __u32 status; + __u8 uuid[SEP_BIO_UUID_LEN]; + __u32 guidance; + __u32 progress_percent; +}; + +struct sep_bio_verify_start { + __u32 flags; + __u32 reserved; + __u8 nonce[SEP_BIO_NONCE_LEN]; +}; + +struct sep_bio_verify_poll { + __u32 state; + __u32 result; + __u32 status; + __u32 guidance; /* SEP_BIO_GUIDANCE_*; status only, no capture side effect */ + __u8 uuid[SEP_BIO_UUID_LEN]; + __u8 token[SEP_BIO_TOKEN_LEN]; /* single use, bound to the nonce */ + __u64 deadline_ns; /* CLOCK_MONOTONIC; past this the token is void */ +}; + +struct sep_bio_delete { + __u8 uuid[SEP_BIO_UUID_LEN]; +}; + +/* + * Device attestation of key possession: the enclave signs 'challenge' with the + * machine ref-key (ECDSA-P256 over the challenge as the pre-computed digest) and + * returns the DER signature and public point. The private key never leaves the + * enclave. + */ +struct sep_bio_attest { + __u32 sig_len; /* out: DER signature length */ + __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ + __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ + __u8 reserved[3]; +}; + +#define SEP_BIO_IOC_MAGIC 0xB1 + +#define SEP_BIO_GET_INFO _IOR(SEP_BIO_IOC_MAGIC, 0x01, struct sep_bio_info) +#define SEP_BIO_LIST _IOR(SEP_BIO_IOC_MAGIC, 0x02, struct sep_bio_list) +#define SEP_BIO_ENROL_START _IOW(SEP_BIO_IOC_MAGIC, 0x03, struct sep_bio_enrol_start) +#define SEP_BIO_ENROL_POLL _IOR(SEP_BIO_IOC_MAGIC, 0x04, struct sep_bio_enrol_poll) +#define SEP_BIO_VERIFY_START _IOW(SEP_BIO_IOC_MAGIC, 0x05, struct sep_bio_verify_start) +#define SEP_BIO_VERIFY_POLL _IOR(SEP_BIO_IOC_MAGIC, 0x06, struct sep_bio_verify_poll) +#define SEP_BIO_CANCEL _IO(SEP_BIO_IOC_MAGIC, 0x07) +#define SEP_BIO_DELETE _IOW(SEP_BIO_IOC_MAGIC, 0x08, struct sep_bio_delete) +#define SEP_BIO_DELETE_ALL _IO(SEP_BIO_IOC_MAGIC, 0x09) +#define SEP_BIO_ATTEST _IOWR(SEP_BIO_IOC_MAGIC, 0x0a, struct sep_bio_attest) + +#endif /* _UAPI_LINUX_APPLE_SEP_H */ diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 5efa06543beb8a..03c03f5596b23c 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -144,3 +144,11 @@ On Omarchy, run `omarchy-apply-lock` once after enrollment. The Quickshell lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used by the Quickshell lock screen. + +## Biometric userspace header + +The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this +checkout's headers with `make headers_install INSTALL_HDR_PATH=` +and add that directory's `include` to the libfprint build include path. The +libfprint patch uses `` directly; it has no copied ABI +header. Interface version, ioctl numbers and fixed-width layouts are unchanged. From aae003ea27bde3017b18f1d7c4261682d16fc4e8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:42:31 +1100 Subject: [PATCH 188/631] soc: apple: sep: establish authenticated sensor sessions on J700 The J700 enclave restores stored identities but cannot open a capture session without the sensor's nonce and signed root-key container. Repeated sensor registration must precede CLEAR_STATE, and communication initialization must run before restoring the biometric state. Describe the nonce session in the existing platform profile and perform its initialization, platform identity, sensor serial, nonce and root-key steps through the existing SBIO and SPI transports. Validate the complete nonce frame with the existing CRC reader and refuse a missing identity, firmware or unsuccessful exchange. Load the externally supplied signed key once per enclave lifetime; do not log identity or cryptographic payloads. For this protocol, apply encrypted sensor parameters before calibration and complete initialization afterwards. Use calibration source 6 while retaining the existing source and ordering for other profiles. Enable owner export and user-before-master saves for J700, then refresh the device view and match credential after restoring its components. Retain the existing xART write policy. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 +- drivers/soc/apple/profile.rs | 6 ++ drivers/soc/apple/sbio.rs | 155 ++++++++++++++++++++++++++++++++++- drivers/soc/apple/sensor.rs | 12 +++ drivers/soc/apple/sep.rs | 3 + 5 files changed, 175 insertions(+), 4 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index d61cff397b2e9d..d1c37ea3a65237 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2003,6 +2003,8 @@ F: drivers/pci/controller/pcie-apple-piodma* F: drivers/soc/apple/aop.rs F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h +F: include/uapi/linux/apple-sep.h +F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs @@ -2656,7 +2658,6 @@ F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: tools/testing/selftests/drivers/apple/ F: include/uapi/drm/asahi_drm.h -F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 646f7862e41040..7ae7afa2015880 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Typed platform profiles for the SEP driver. //! @@ -146,6 +147,8 @@ pub(crate) struct PlatformProfile { pub(crate) firmware_region: &'static CStr, /// Identity keybag CREATE_KEYBAG field encoding (per-SoC; see [`KeybagCreate`]). pub(crate) keybag_create: KeybagCreate, + /// Require nonce authentication and signed sensor material before capture. + pub(crate) nonce_session: bool, /// Key-store protocol generation (see [`KeyStore`]). pub(crate) key_store: KeyStore, /// Platforms with reboot-tested owner export and user-before-master saves. @@ -219,6 +222,7 @@ const T6000: PlatformProfile = PlatformProfile { cpx_encryption_mode: 2, }, persistent_enrol: true, + nonce_session: false, }; const T6020: PlatformProfile = PlatformProfile { @@ -266,6 +270,7 @@ const T6020_SEPOS13: PlatformProfile = PlatformProfile { cpx_encryption_mode: 2, }, ..T6020 + nonce_session: false, }; /// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor @@ -299,6 +304,7 @@ const T8140: PlatformProfile = PlatformProfile { bag_type: 0, arg: -1, }, + nonce_session: true, }; /// MacBook Air M2 (J413, J415). iBoot leaves the SEP running and the Mesa diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 3cc1c513ea6cb4..5e59118c344b53 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Touch ID: the biometric endpoint (SBIO) and its `/dev/sep-bio` interface. //! The enclave matches; no biometric image ever crosses to userspace. @@ -1035,6 +1036,25 @@ impl SepData { self.register_ool(&self.ool_sbio)?; self.sbio_ready.store(true, Relaxed); + if self.profile.nonce_session { + let word = 6u32; + // Communication must be initialized before restoring biometric state. + let result = self.sbio_transfer_raw( + 0x73, c"INIT_SBIO_COMMUNICATION", &word.to_le_bytes(), + ); + match result { + Ok(done) if done.status.is_ok() => { + dev_info!(self.dev, "sbio: communication initialization accepted\n"); + } + Ok(done) => { + self.sbio_ready.store(false, Relaxed); + dev_err!(self.dev, "sbio: communication init refused: status {}, {} bytes\n", + done.status, done.payload.len()); + return Err(EIO); + } + Err(e) => { + self.sbio_ready.store(false, Relaxed); + dev_err!(self.dev, "sbio: communication init transport failed: {:?}\n", e); if matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) { // 22G74 AppleMesaSEPDriver::initSbioCommunication sends 0x73 // with u32(1) before its first ordinary SBIO transaction. @@ -1063,6 +1083,15 @@ impl SepData { } fn attach_bringup(&self) { + if self.profile.nonce_session { + if !self.bring_sensor_online() { + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); + } + return; + } if !sensor::is_bound() { dev_warn!( self.dev, @@ -1096,12 +1125,29 @@ impl SepData { return false; }; + // Newer sensors require the encrypted parameters and transparent + // channel before the calibration challenge can be serviced. + let parameters = if self.profile.nonce_session { + match self.apply_sensor_parameters() { + Some(parameters) => Some(parameters), + None => return false, + } + } else { + None + }; + if !self.sensor_calibrated.load(Relaxed) { if !self.calibrate_sensor() { diag::sensor_brought_up(bind, false); return false; } self.sensor_calibrated.store(true, Relaxed); + if parameters.is_none() { + let Some(reloaded_patch) = self.wake_sensor() else { + return false; + }; + patch = reloaded_patch; + } let Some(reloaded_patch) = self.wake_sensor() else { diag::sensor_brought_up(bind, false); return false; @@ -1109,6 +1155,10 @@ impl SepData { patch = reloaded_patch; } + let ok = match parameters { + Some(parameters) => self.finish_sensor_setup(patch, parameters), + None => self.complete_bringup(patch), + }; let ok = self.complete_bringup(patch); diag::sensor_brought_up(bind, ok); // The sensor is patched and idle here -- the only safe moment to @@ -1701,6 +1751,22 @@ impl SepData { let stage = self.bringup.load(Relaxed); dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); + if self.profile.nonce_session { + // J700 needs identification before CLEAR_STATE on every reset. + let platform = match Self::sensor_platform_identity() { + Ok(platform) => platform, + Err(error) => { + dev_err!(self.dev, "sensor: platform identity unavailable: {:?}\n", error); + return false; + } + }; + if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) + || self.sbio_expect_ok(&crate::sbio::sbio_register_sensor(id)).is_none() + { + return false; + } + } + if stage >= BRINGUP_ESTABLISHED { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self @@ -1740,6 +1806,9 @@ impl SepData { let op = crate::sbio::sbio_register_sensor_serial(&serial); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { + if self.profile.nonce_session && !self.authenticate_sensor_session() { + return false; + } self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); let _ = self.log_identity_count(c"after sensor serial registration"); true @@ -1748,6 +1817,70 @@ impl SepData { } } + fn sensor_platform_identity() -> Result<[u8; 12]> { + let node = kernel::of::root().ok_or(ENODEV)?; + let property: KVec = node.get_property(c"serial-number")?; + let length = property.iter().position(|byte| *byte == 0).ok_or(EINVAL)?; + let mut identity = [0; 12]; + if length == 0 || length > identity.len() || property.len() != length + 1 { + return Err(EINVAL); + } + identity[..length].copy_from_slice(&property[..length]); + Ok(identity) + } + + fn sensor_session_request(&self, opcode: u16, name: &'static CStr, bytes: &[u8]) -> bool { + match self.sbio_transfer_raw(opcode, name, bytes) { + Ok(reply) if reply.status.is_ok() => true, + Ok(reply) => { + dev_err!(self.dev, "sensor: {} refused with status {}\n", name, reply.status); + false + } + Err(error) => { + dev_err!(self.dev, "sensor: {} transfer failed: {:?}\n", name, error); + false + } + } + } + + fn authenticate_sensor_session(&self) -> bool { + let nonce = match sensor::session_nonce() { + Ok(nonce) => nonce, + Err(error) => { + dev_err!(self.dev, "sensor: nonce read failed: {:?}\n", error); + return false; + } + }; + if !self.sensor_session_request(0x81, c"SENSOR_NONCE", &nonce) { + return false; + } + + if !self.sensor_key_loaded.load(Relaxed) { + let firmware = match kernel::firmware::Firmware::request( + c"apple/msrk-j700.bin", &self.dev, + ) { + Ok(firmware) => firmware, + Err(error) => { + dev_err!( + self.dev, + "sensor: signed root-key container unavailable: {:?}\n", + error + ); + return false; + } + }; + if firmware.data().is_empty() + || !self.sensor_session_request(0x86, c"SENSOR_ROOT_KEY", firmware.data()) + { + return false; + } + self.sensor_key_loaded.store(true, Relaxed); + } + // Each identification starts a new sensor-side exchange, even though + // the signed key remains loaded for the lifetime of the enclave. + self.establish_session() + } + fn wake_sensor(&self) -> Option { let _source = sensor::power_source(); let cycled = sensor::power_cycle(); @@ -1859,7 +1992,8 @@ impl SepData { return false; }; - let request = match crate::sbio::SbioCalibration::new(&blob) { + let source = if self.profile.nonce_session { 6 } else { SBIO_CALIBRATION_SOURCE }; + let request = match crate::sbio::SbioCalibration::new(&blob, source) { Ok(request) => request, Err(_) => { return false; @@ -1888,6 +2022,10 @@ impl SepData { return false; }; + self.finish_sensor_setup(patch, params) + } + + fn finish_sensor_setup(&self, patch: PatchLoaded, params: ParametersApplied) -> bool { let op = crate::sbio::sbio_complete_init(patch, params); if self.sbio_expect_ok(&op).is_none() { dev_err!( @@ -2993,6 +3131,17 @@ impl SepData { let restored = self.restore_all_components(); dev_info!(self.dev, "sbio: cold-match preparation {}, component restore {}\n", prepared, restored); + // Restore can replace the credential and the device's identity view. + // Refresh both after loading J700's persistent components, before + // admitting a verify operation. + let match_context = if self.profile.nonce_session { + let synced = restored && self.sync_device_view(); + self.device_view_synced.store(synced, Relaxed); + synced && crate::sbio::UserId::new(SBIO_PROBE_USER_ID) + .is_some_and(|user| self.establish_scrd_match_context(user)) + } else { + match_context + }; let identities = self.log_identity_count(c"after Catacomb restore with sensor registered"); let identity_ready = matches!(identities, Some(n) if n > 0); let candidate = prepared && match_context && restored && identity_ready; @@ -3555,10 +3704,10 @@ pub(crate) const SBIO_CALIBRATION_SOURCE: u32 = 3; pub(crate) struct SbioCalibration(KVec); impl SbioCalibration { - pub(crate) fn new(blob: &crate::CalibrationBlob) -> Result { + pub(crate) fn new(blob: &crate::CalibrationBlob, source: u32) -> Result { let bytes = blob.bytes(); let mut body = KVec::new(); - body.extend_from_slice(&SBIO_CALIBRATION_SOURCE.to_le_bytes(), GFP_KERNEL)?; + body.extend_from_slice(&source.to_le_bytes(), GFP_KERNEL)?; body.extend_from_slice(bytes, GFP_KERNEL)?; Ok(SbioCalibration(body)) } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 8fec6b71c8715f..3fcba35e39428f 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! The fingerprint sensor over SPI. @@ -581,6 +582,17 @@ pub(crate) fn read_sensor_serial() -> Result { Ok(SensorSerial(serial)) } +/// Read the nonce as one frame; a separate acknowledgment would consume it. +pub(crate) fn session_nonce() -> Result<[u8; 32]> { + command(&[0x80, 0x35, 0, 7, 0, 0, 0])?; + let mut value = [0; 32]; + read_framed_verified(9 + value.len() + CRC_LEN, 9, &mut value)?; + if value.iter().all(|byte| *byte == 0) { + return Err(EIO); + } + Ok(value) +} + pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { let mut share = [0u8; SESSION_SHARE_LEN]; read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 99b57d18727a39..afa7305e291d7d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT // Copyright 2026 Dj +// Copyright 2026 Ryan Murray //! Driver for the Apple SEP (Secure Enclave Processor): attaches to a running //! SEP over the AP mailbox, drives Touch ID enrol/verify via `/dev/sep-bio`, //! and registers a hwrng and a SEP-backed `trusted` key source. @@ -599,6 +600,7 @@ struct SepData { cold_restore_candidate: Atomic, sensor_calibrated: Atomic, + sensor_key_loaded: Atomic, enrol_open: Atomic, @@ -840,6 +842,7 @@ impl SepData { xart_refusal_logged: Atomic::new(false), capture_refusal_logged: Atomic::new(false), sensor_calibrated: Atomic::new(false), + sensor_key_loaded: Atomic::new(false), templates_restored: Atomic::new(false), cold_restore_candidate: Atomic::new(false), enrol_material <- new_mutex!(None), From d3cf4a3a8777e40c5d45e07a8e7d4e518bff952c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 19:01:16 +1100 Subject: [PATCH 189/631] cpufreq: apple: retain the unmanaged T8152 P-state ceiling Removing the private thermal-policy prerequisite for T8140 must not admit high P-state tables on other SoCs without qualified protection. Keep the T8152 ceiling at state 3 while T8140 uses thermal zones. Carry the existing public T8152 metadata and per-SoC timeout convention into this older thermal branch, so the ceiling applies when the branch is integrated with the current Apple CPU frequency support. The T8152 metadata follows Acelogic's public implementation; no contributor certification is added on their behalf. Other SoCs retain the default transition timeout and their existing admission behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 5dc0eef7531e32..f1cf2dd98248a6 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -909,6 +909,16 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; + /* Unqualified SoCs still require protection above their safe ceiling. */ + if (info->needs_thermal_policy) { + cpufreq_for_each_valid_entry(p, policy->freq_table) { + if (p->driver_data <= info->max_unmanaged_pstate) + continue; + dev_err(cpu_dev, "higher P-states require a qualified thermal policy\n"); + ret = -ENODEV; + goto out_free_cpufreq_table; + } + } return 0; From 556539514e63b88adeecd1aa5fc31a402ed735b8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:39:21 +1100 Subject: [PATCH 190/631] Merge pull request #43 from aurora-silicon/feat/j700-thermal-framework-v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit J700: CPU thermal policy on the thermal framework (power allocator, measured energy model) — v2 Original contributor commits are preserved. Conflict resolutions retain the audited shared driver APIs and their integrated review fixes. Contributor credit; Signed-off-by outstanding: - 89ef1ced379a497b004d44e834191d87023d1502: Ryan <81541646+RyanTheTide@users.noreply.github.com> Ryan integrates these contributions under DCO (b); no author signature is forged. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 21 ++ drivers/cpufreq/apple-soc-cpufreq.c | 326 +++++++++++++++++++++++++++- 2 files changed, 346 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index d1c37ea3a65237..29d99053ebb184 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1977,14 +1977,24 @@ APPLE T8140 SOC AND J700 (MACBOOK NEO) SUPPORT M: Ryan Murray W: https://aurora-silicon.github.io S: Maintained +F: Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml +F: Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml +F: drivers/pmdomain/apple/pmp-report.c +F: drivers/soc/apple/pmp.rs +F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml F: Documentation/PCI/controller/apple-t8140.rst F: Documentation/devicetree/bindings/iio/light/apple,t8140-aop-als.yaml F: Documentation/devicetree/bindings/input/apple,dockchannel-hid.yaml +F: Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml F: Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +F: Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml F: Documentation/devicetree/bindings/reserved-memory/apple,asc-mem.yaml +F: Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml @@ -1993,12 +2003,15 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst +F: Documentation/soc/apple/t8140-pmp-dvfs.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* +F: arch/arm64/configs/j700-radio.config F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/rtc/rtc-apple-pmu.c F: drivers/pci/controller/pcie-apple-piodma* F: drivers/soc/apple/aop.rs F: drivers/tty/serial/apple_dockchannel* @@ -2658,9 +2671,17 @@ F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* F: tools/testing/selftests/drivers/apple/ F: include/uapi/drm/asahi_drm.h +F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c F: net/bluetooth/brcm.h +ARM/APPLE QSPIMC DRIVER +M: Ryan Murray +W: https://aurora-silicon.github.io +S: Maintained +F: Documentation/devicetree/bindings/spi/apple,qspi.yaml +F: drivers/spi/spi-apple-qspi.c + ARM/ARTPEC MACHINE SUPPORT M: Jesper Nilsson M: Lars Persson diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index f1cf2dd98248a6..c3c13950cbea72 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -786,6 +786,330 @@ static void apple_fast_die_exit(struct apple_cpu_priv *priv) kfree(fd); } +/* The fastest state whose power is at most @budget, or the slowest. */ +static unsigned int apple_fast_die_fit(const struct apple_fast_die *fd, + u64 budget) +{ + unsigned int i, idx = 0; + + for (i = 0; i < fd->nr_states; i++) + if (fd->power[i] <= budget) + idx = i; + + return fd->policy->freq_table[idx].frequency; +} + +/* + * Map an effort to a frequency limit, in kHz: the fastest state whose power + * fits a budget of (255 - effort) / 255 of the fastest state's power. Any + * mapping that lowers the limit monotonically as the effort rises would do, + * since the firmware's loop is closed on the die temperature and corrects + * the error; power is the natural axis for the output of a temperature + * controller. + */ +static unsigned int apple_fast_die_limit(const struct apple_fast_die *fd, + unsigned int effort) +{ + /* + * Multiply first: dividing the fastest state's power by 255 first + * rounds it down, which would limit an unlimited cluster one state low. + */ + return apple_fast_die_fit(fd, + div_u64((u64)fd->power[fd->nr_states - 1] * + (APPLE_FAST_DIE_EFFORT_MAX - effort), + APPLE_FAST_DIE_EFFORT_MAX)); +} + +static ktime_t apple_fast_die_interval(const struct apple_fast_die *fd) +{ + if (!fd->waiting) + return ms_to_ktime(APPLE_FAST_DIE_POLL_MS); + if (!fd->warned) + return ms_to_ktime(APPLE_FAST_DIE_WAIT_MS); + return ms_to_ktime(APPLE_FAST_DIE_GAVE_UP_MS); +} + +/* Arm the non-deferrable timer if the limit needs it. Call with @lock held. */ +static void apple_fast_die_arm(struct apple_fast_die *fd) +{ + if (fd->stopping || fd->polling || + (fd->per_cpu && fd->limit >= fd->max)) + return; + + fd->polling = true; + hrtimer_start(&fd->poll, apple_fast_die_interval(fd), + HRTIMER_MODE_REL_SOFT); +} + +/* Take a sample and queue an update if the limit changes. Call with @lock held. */ +static void apple_fast_die_sample(struct apple_fast_die *fd) +{ + struct apple_cpu_priv *priv = fd->policy->driver_data; + unsigned int limit; + u8 effort; + int ret; + + ret = apple_pmp_report_fast_die_effort(fd->report, fd->lane, &effort); + if (!ret) { + limit = apple_fast_die_limit(fd, effort); + fd->waiting = false; + } else { + limit = fd->fallback; + fd->waiting = true; + if (!fd->warned && + time_after(jiffies, fd->start + APPLE_FAST_DIE_WARN_S * HZ)) { + fd->warned = true; + dev_warn(priv->cpu_dev, + "PMP not running after %d s (%d): fast die-temperature limit unavailable, CPUs %*pbl stay limited to %u kHz\n", + APPLE_FAST_DIE_WARN_S, ret, + cpumask_pr_args(fd->cpus), limit); + } + } + + if (limit != fd->limit) { + WRITE_ONCE(fd->limit, limit); + schedule_work(&fd->work); + } +} + +static void apple_fast_die_work(struct work_struct *work) +{ + struct apple_fast_die *fd = container_of(work, struct apple_fast_die, work); + struct apple_cpu_priv *priv = fd->policy->driver_data; + unsigned int limit = READ_ONCE(fd->limit); + int ret; + + ret = freq_qos_update_request(&fd->req, limit); + if (ret >= 0) + return; + + dev_warn_ratelimited(priv->cpu_dev, + "failed to update the fast die-temperature limit: %d\n", ret); + /* Make the next sample queue the update again, and make sure it comes. */ + spin_lock_bh(&fd->lock); + if (fd->limit == limit) { + fd->limit = 0; + apple_fast_die_arm(fd); + } + spin_unlock_bh(&fd->lock); +} + +static enum hrtimer_restart apple_fast_die_poll(struct hrtimer *t) +{ + struct apple_fast_die *fd = container_of(t, struct apple_fast_die, poll); + enum hrtimer_restart restart = HRTIMER_NORESTART; + + spin_lock(&fd->lock); + apple_fast_die_sample(fd); + if (!fd->stopping && (!fd->per_cpu || fd->limit < fd->max)) { + hrtimer_forward_now(t, apple_fast_die_interval(fd)); + restart = HRTIMER_RESTART; + } else { + fd->polling = false; + } + spin_unlock(&fd->lock); + + return restart; +} + +static void apple_fast_die_tick(struct timer_list *t) +{ + struct apple_fast_die_timer *ft = timer_container_of(ft, t, timer); + struct apple_fast_die *fd = ft->fd; + unsigned long now = jiffies; + + /* + * The first busy CPU of the cluster to run in a tick takes the sample, + * unless the non-deferrable timer is sampling. + */ + if (xchg(&fd->last_poll, now) != now && spin_trylock(&fd->lock)) { + if (!fd->polling) { + apple_fast_die_sample(fd); + apple_fast_die_arm(fd); + } + spin_unlock(&fd->lock); + } + mod_timer(t, now + 1); +} + +static int apple_fast_die_cpu_online(unsigned int cpu) +{ + struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); + + if (!ft->fd) + return 0; + + /* Wait for a running sample, which rearms the timer, before arming it. */ + timer_delete_sync(&ft->timer); + ft->timer.expires = jiffies + 1; + add_timer_on(&ft->timer, cpu); + + return 0; +} + +static int apple_fast_die_cpu_offline(unsigned int cpu) +{ + struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); + + if (ft->fd) + timer_delete_sync(&ft->timer); + + return 0; +} + +/* + * Called at the end of ->init(), with the CPU hotplug lock held. The core + * fills policy->related_cpus only after ->init() returns, and may call ->exit() + * before it does, so the limit keeps its own copy of the cluster's CPUs. + */ +static int apple_fast_die_init(struct cpufreq_policy *policy, + struct device_node *cluster) +{ + struct apple_cpu_priv *priv = policy->driver_data; + struct of_phandle_args args; + struct apple_fast_die *fd; + unsigned int cpu, i, n = 0; + bool have_power = true; + int parsed, ret; + + parsed = of_parse_phandle_with_fixed_args(cluster, "apple,fast-die-effort", + 1, 0, &args); + /* Required thermal admission must fail closed with an older DT. */ + if (parsed == -ENOENT && !priv->info->needs_thermal_policy) + return 0; + + while (policy->freq_table[n].frequency != CPUFREQ_TABLE_END) + n++; + + fd = kzalloc_flex(*fd, power, n); + if (!fd) { + ret = -ENOMEM; + goto err_put; + } + fd->nr_states = n; + if (!zalloc_cpumask_var(&fd->cpus, GFP_KERNEL)) { + ret = -ENOMEM; + goto err_free; + } + + for (i = 0; i < n; i++) { + unsigned long rate = policy->freq_table[i].frequency * 1000UL + 999; + struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(priv->cpu_dev, &rate); + + if (IS_ERR(opp)) { + ret = PTR_ERR(opp); + goto err_free; + } + fd->power[i] = dev_pm_opp_get_power(opp); + dev_pm_opp_put(opp); + if (!fd->power[i]) + have_power = false; + } + /* Without an energy model, frequency stands in for power. */ + if (!have_power) + for (i = 0; i < n; i++) + fd->power[i] = policy->freq_table[i].frequency; + + fd->policy = policy; + spin_lock_init(&fd->lock); + INIT_WORK(&fd->work, apple_fast_die_work); + hrtimer_setup(&fd->poll, apple_fast_die_poll, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_SOFT); + cpumask_copy(fd->cpus, policy->cpus); + fd->per_cpu = apple_fast_die_hp_state > 0; + for_each_cpu(cpu, fd->cpus) + if (tick_nohz_full_cpu(cpu)) + fd->per_cpu = false; + fd->max = policy->freq_table[n - 1].frequency; + fd->fallback = apple_fast_die_fit(fd, div_u64(fd->power[n - 1], 3)); + fd->start = jiffies; + + if (parsed) { + /* The limit is meant to be there, so keep the fallback. */ + dev_warn(priv->cpu_dev, + "invalid apple,fast-die-effort (%d): CPUs %*pbl stay limited to %u kHz\n", + parsed, cpumask_pr_args(fd->cpus), fd->fallback); + fd->warned = true; + } else { + fd->report = args.np; + fd->lane = args.args[0]; + } + + /* Hold the fallback limit until a sample finds the PMP running. */ + fd->limit = fd->fallback; + ret = freq_qos_add_request(&policy->constraints, &fd->req, FREQ_QOS_MAX, + fd->limit); + if (ret < 0) + goto err_free; + + priv->fast_die = fd; + + if (fd->per_cpu) { + for_each_cpu(cpu, fd->cpus) { + struct apple_fast_die_timer *ft = + per_cpu_ptr(&apple_fast_die_timers, cpu); + + timer_setup(&ft->timer, apple_fast_die_tick, + TIMER_DEFERRABLE | TIMER_PINNED); + ft->fd = fd; + if (cpu_online(cpu)) + apple_fast_die_cpu_online(cpu); + } + } + + spin_lock_bh(&fd->lock); + apple_fast_die_arm(fd); + spin_unlock_bh(&fd->lock); + + dev_dbg(priv->cpu_dev, "fast die-temperature limit set up, fallback %u kHz\n", + fd->fallback); + + return 0; + +err_free: + free_cpumask_var(fd->cpus); + kfree(fd); +err_put: + if (!parsed) + of_node_put(args.np); + return ret; +} + +static void apple_fast_die_exit(struct apple_cpu_priv *priv) +{ + struct apple_fast_die *fd = priv->fast_die; + unsigned int cpu; + + if (!fd) + return; + + /* + * Keep the update work from arming the hrtimer again, stop the per-CPU + * timers, which can arm it too, then the hrtimer, and then drain the + * update any of them may have queued. + */ + spin_lock_bh(&fd->lock); + fd->stopping = true; + spin_unlock_bh(&fd->lock); + if (fd->per_cpu) { + for_each_cpu(cpu, fd->cpus) { + struct apple_fast_die_timer *ft = + per_cpu_ptr(&apple_fast_die_timers, cpu); + + timer_shutdown_sync(&ft->timer); + ft->fd = NULL; + } + } + hrtimer_cancel(&fd->poll); + cancel_work_sync(&fd->work); + freq_qos_remove_request(&fd->req); + + priv->fast_die = NULL; + of_node_put(fd->report); + free_cpumask_var(fd->cpus); + kfree(fd); +} + static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) { struct cpufreq_frequency_table *p; From 5a66af8b65e47d994e6e617fb8abbf124fc0eb62 Mon Sep 17 00:00:00 2001 From: RJ Skerry-Ryan Date: Sun, 27 Sep 2026 13:03:09 -0700 Subject: [PATCH 191/631] PCI: apple: Cold-init PCIe-C tunnel ports without an m1n1 handoff PCIe-C tunnel support requires m1n1 to cold-initialize the port and report "apple,pciec-preinit-status" = 1, but no m1n1 does so. On J414/J416 the tunnel is therefore always disabled and PCIe devices behind Thunderbolt 3 docks (whose USB is an xHCI behind the tunnel) never appear. m1n1 already copies the PCIe-C tunables into the DT and the driver has the port enable sequence. For ports marked "apple,pciec-kernel-init", let the kernel do the cold init, as aurora-silicon/linux#8 does for t8103: - apple_pcie_tunnel_cold_init() follows the enablePortHardware() baseline of apple_pcie_tunnel_start(): debug/fabric tunables, port reset table and tunables, PERST off and APPCLK on, wait for RUN, and only then the root-complex (config space) tunables. Touching config space while APPCLK is still off hard-resets a J416c. - apple_pcie_tunnel_prepare() runs it on transient mappings before the tunnel children are populated, since the DART never completes an invalidation while APPCLK is gated. A port that is already running is left alone. - The Thunderbolt glue and the host bridge accept kernel-init ports in place of the m1n1 handoff. As the sequence is new on T602x, it is gated behind pcie_apple.tunnel_kernel_init (default off), checked before the first PCIe-C MMIO access. pcie_apple.tunnel_kernel_init_trace_ms logs each step and pauses after it, for locating failures with a persistent log. Tested on J416c with a CalDigit USB-C Pro Dock (Thunderbolt 3, Titan Ridge xHCI 8086:15f0): the dock's hubs, keyboard, audio, SD reader, webcam and Ethernet work, including after unplug and replug. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: RJ Skerry-Ryan --- drivers/pci/controller/pcie-apple.c | 228 ++++++++++++++++++++++++++++ 1 file changed, 228 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3d75bcfee59110..dfc2e1e123c30a 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -116,6 +116,21 @@ bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) return tunables; } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); +static bool tunnel_kernel_init; +module_param(tunnel_kernel_init, bool, 0644); +MODULE_PARM_DESC(tunnel_kernel_init, + "Cold-initialize apple,pciec-kernel-init PCIe-C ports (experimental, default off)"); + +static unsigned int tunnel_kernel_init_trace_ms; +module_param(tunnel_kernel_init_trace_ms, uint, 0644); +MODULE_PARM_DESC(tunnel_kernel_init_trace_ms, + "Log each PCIe-C cold init step and sleep this long after it (debug)"); + +bool apple_pcie_tunnel_kernel_init_enabled(void) +{ + return READ_ONCE(tunnel_kernel_init); +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_kernel_init_enabled); /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 @@ -983,6 +998,74 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) return 0; } +static void apple_pcie_tunnel_cold_init_step(struct apple_pcie_port *port, + const char *step) +{ + unsigned int ms = READ_ONCE(tunnel_kernel_init_trace_ms); + + if (!ms) + return; + dev_info(port->pcie->dev, "PCIe-C %pOF cold init: %s\n", port->np, step); + msleep(ms); +} + +/* + * Cold-initialize a PCIe-C port that no m1n1 handoff brought up. This + * follows the enablePortHardware() baseline of apple_pcie_tunnel_start() and + * the t8103 cold init of aurora-silicon/linux#8: the port leaves reset with + * APPCLK running and reports RUN before anything touches the root port's + * config space. The downstream (tunnel) reset stays asserted until + * apple_pcie_setup_port() releases it before link training. + */ +static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + u32 stat; + int i, ret; + + apple_pcie_tunnel_cold_init_step(port, "debug and fabric tunables"); + apple_pcie_tunnel_apply_tunable(pcie->debug_base, pcie->debug_tunable); + apple_pcie_tunnel_apply_tunable(pcie->fabric_base, + pcie->fabric_tunable); + apple_pcie_tunnel_cold_init_step(port, "port reset table"); + apple_pcie_tunnel_reset_hardware(port); + apple_pcie_tunnel_cold_init_step(port, "port tunables"); + apple_pcie_tunnel_apply_tunable(port->base, port->tunable); + + apple_pcie_tunnel_cold_init_step(port, "PERST off, APPCLK on"); + apple_pcie_port_rmw_set(port, PORT_PERST_OFF, pcie->hw->port_perst); + apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + stat & PORT_STATUS_READY, + 10, 250000, false, port, PORT_STATUS); + if (ret) + return dev_err_probe(pcie->dev, ret, + "PCIe-C %pOF cold init: port did not enter RUN (status %#x)\n", + port->np, stat); + + apple_pcie_tunnel_cold_init_step(port, "root complex tunables"); + apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); + apple_pcie_tunnel_cold_init_step(port, "APPCLK ungate, counters, Intr2AXI"); + apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); + apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); + apple_pcie_tunnel_writel(pcie->intr2axi_base, PCIEC_INTR2AXI_ENABLE, + PCIEC_INTR2AXI_CTRL); + + ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, + !(stat & PORT_LINKSTS_BUSY), + 10, 250000, false, port, PORT_LINKSTS); + if (ret) + return dev_err_probe(pcie->dev, ret, + "PCIe-C %pOF cold init: link still busy (status %#x)\n", + port->np, stat); + + for (i = 0; i < pcie->hw->max_rid2sid; i++) + apple_pcie_rid2sid_write(port, i, 0); + + dev_info(pcie->dev, "PCIe-C %pOF cold init done, status %#x tunstat %#x\n", + port->np, apple_pcie_port_readl(port, PORT_STATUS), /* * Bring up a port that no firmware handoff left running (t8103, or * T600x/T602x with pcie_apple.tunnel_kernel_init): tunables and the reset @@ -3502,6 +3585,151 @@ static const struct of_device_id apple_pcie_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_pcie_of_match); +struct apple_pcie_tunnel_map { + struct resource res; + void __iomem *base; +}; + +static int apple_pcie_tunnel_map_named(struct device_node *np, + const char *name, + struct apple_pcie_tunnel_map *map) +{ + int index, ret; + + index = of_property_match_string(np, "reg-names", name); + if (index < 0) + return index; + ret = of_address_to_resource(np, index, &map->res); + if (ret) + return ret; + /* Tunneled PCIe-C MMIO is non-posted, as in the probe path. */ + map->base = ioremap_np(map->res.start, resource_size(&map->res)); + return map->base ? 0 : -ENOMEM; +} + +static void apple_pcie_tunnel_unmap(struct apple_pcie_tunnel_map *map) +{ + if (map->base) + iounmap(map->base); +} + +/* + * Cold-initialize an "apple,pciec-kernel-init" PCIe-C port before the tunnel + * children are populated. The DART is the first child and probes as soon as + * it is created, and its command engine never completes an invalidation + * while the port's APPCLK is gated. A port that is already running (m1n1 + * handoff, or retained from an earlier connection) is left alone. + * + * There is no host bridge yet, so build a transient one around temporary + * mappings and reuse the port enable sequence. + */ +int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) +{ + struct apple_pcie_tunnel_map config = {}, debug = {}, fabric = {}, + intr2axi = {}, portmap = {}; + struct device_node *np = NULL, *port_np = NULL; + const struct of_device_id *match; + struct pci_config_window cfg = {}; + struct apple_pcie_port port = {}; + struct apple_pcie pcie = {}; + void *group = NULL; + u32 stat; + int ret; + + for_each_available_child_of_node(tunnel, np) + if (of_property_read_bool(np, "apple,pciec-kernel-init")) + break; + if (!np) + return 0; + + if (!apple_pcie_tunnel_kernel_init_enabled()) { + dev_err(dev, "PCIe-C %pOF: kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n", + np); + ret = -EPERM; + goto out; + } + + match = of_match_node(apple_pcie_of_match, np); + port_np = of_get_next_available_child(np, NULL); + if (!match || !((const struct hw_info *)match->data)->tunneled || + !port_np) { + ret = -ENODEV; + goto out; + } + + ret = apple_pcie_tunnel_map_named(np, "port0", &portmap); + if (ret) + goto out; + + pcie.dev = dev; + pcie.hw = match->data; + port.pcie = &pcie; + port.np = port_np; + port.base = portmap.base; + + apple_pcie_tunnel_cold_init_step(&port, "read port status"); + stat = apple_pcie_port_readl(&port, PORT_STATUS); + if (stat & PORT_STATUS_READY) { + dev_info(dev, "PCIe-C port %pOF already running\n", port_np); + goto out; + } + + ret = apple_pcie_tunnel_map_named(np, "config", &config); + if (!ret) + ret = apple_pcie_tunnel_map_named(np, "debug", &debug); + if (!ret) + ret = apple_pcie_tunnel_map_named(np, "fabric", &fabric); + if (!ret) + ret = apple_pcie_tunnel_map_named(np, "intr2axi", &intr2axi); + if (ret) + goto out; + + group = devres_open_group(dev, NULL, GFP_KERNEL); + if (!group) { + ret = -ENOMEM; + goto out; + } + pcie.rc_tunable = devm_apple_tunable_parse(dev, np, "apple,tunable-rc", + &config.res); + pcie.debug_tunable = devm_apple_tunable_parse(dev, np, + "apple,tunable-debug", + &debug.res); + pcie.fabric_tunable = devm_apple_tunable_parse(dev, np, + "apple,tunable-fabric", + &fabric.res); + port.tunable = devm_apple_tunable_parse(dev, port_np, "apple,tunable", + &portmap.res); + if (IS_ERR(pcie.rc_tunable) || IS_ERR(pcie.debug_tunable) || + IS_ERR(pcie.fabric_tunable) || IS_ERR(port.tunable)) { + ret = dev_err_probe(dev, -ENODEV, + "PCIe-C %pOF tunables unavailable\n", np); + goto out; + } + + cfg.win = config.base; + pcie.cfg = &cfg; + pcie.debug_base = debug.base; + pcie.fabric_base = fabric.base; + pcie.intr2axi_base = intr2axi.base; + + dev_info(dev, "PCIe-C port %pOF cold init (status %#x)\n", + port_np, stat); + ret = apple_pcie_tunnel_cold_init(&port); + +out: + if (group) + devres_release_group(dev, group); + apple_pcie_tunnel_unmap(&intr2axi); + apple_pcie_tunnel_unmap(&fabric); + apple_pcie_tunnel_unmap(&debug); + apple_pcie_tunnel_unmap(&config); + apple_pcie_tunnel_unmap(&portmap); + of_node_put(port_np); + of_node_put(np); + return ret; +} +EXPORT_SYMBOL_GPL(apple_pcie_tunnel_prepare); + static struct platform_driver apple_pcie_driver = { .probe = apple_pcie_probe, .remove = apple_pcie_remove, From 28ffd7af70e3171b6ded31c6d7ee492422fd1cde Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:19:18 +1100 Subject: [PATCH 192/631] PCI: apple: retain preinitialized tunnels when cold init is disabled The board property permits experimental kernel cold initialization, but must not reject an existing successful m1n1 handoff. Admit the explicit preinit certificate in both the population gate and preparation helper without touching cold-init registers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index dfc2e1e123c30a..812d659ba01d60 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -3642,6 +3642,13 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) if (!np) return 0; + /* A qualified bootloader handoff does not require experimental init. */ + if (!of_property_read_u32(np, "apple,pciec-preinit-status", &stat) && + stat == 1) { + ret = 0; + goto out; + } + if (!apple_pcie_tunnel_kernel_init_enabled()) { dev_err(dev, "PCIe-C %pOF: kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n", np); From 9806391192e52271309780a973eb7fc5ad030524 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:39:36 +1100 Subject: [PATCH 193/631] Merge pull request #46 from aurora-silicon/fix/j414s-dual-dock-dp drm/apple: support independent dual dock DP streams on J414s Original contributor commits are preserved. Conflict resolutions retain the audited shared driver APIs and their integrated review fixes. Contributor credit; Signed-off-by outstanding: - 2530a2c3a6948b3fdd3ffba10a21c1d2d66a34a7: Ryan <81541646+RyanTheTide@users.noreply.github.com> - 9e93cac3ac711194a73c36328f475a222170c6b6: Ryan <81541646+RyanTheTide@users.noreply.github.com> - 3075f5503a32c44e29e7a1e6b9244038e8e28569: DjDeveloperr - a9aa60593bbcaab38d44640583458a60470d6850: DjDeveloperr - 0cd936dc281e61731dc5b51004964658e23552f6: DjDeveloperr - aff9d4e64aaf418cce120a1bafceff6ba54e726b: DjDeveloperr - a044eab3b9ccce97693e7eb60bddb604799073b2: DjDeveloperr - c5653cb2e0b782921468352a2c60173d2650a8bf: DjDeveloperr - e9e52e6dc932d6f878ef8289db71f428f18f4354: Oliver Lukschander - 0c16b38b45a0b001db49e35fc8acaf94c14a9ef5: Oliver Lukschander - e83f46da5fb58f7169b184f5013a7d5a433414d4: Oliver Lukschander - 2b5254a3eb2616677df84a7b7818fe2993895eeb: Oliver Lukschander - cc895f20ade4b0a6f54ae6c26f5734f2c09d7145: Oliver Lukschander - f8cec39af157f9306ca2d049e48de2e46381cf98: Oliver Lukschander - 81eee7b6f439a1d9c1a2d1effb3ed2eb022ecce2: Oliver Lukschander - 45c34d503e04913b4d6cb5f49f0904f8104ec7f2: Oliver Lukschander - 6f0468f5e7646fa78fb2906c0909222728afc5d7: Oliver Lukschander Ryan integrates these contributions under DCO (b); no author signature is forged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 812d659ba01d60..de04e144dc5c07 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -3182,6 +3182,13 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) if (!np) return 0; + /* A qualified bootloader handoff needs no cold initialization. */ + if (!of_property_read_u32(np, "apple,pciec-preinit-status", &stat) && + stat == 1) { + ret = 0; + goto out_np; + } + match = of_match_node(apple_pcie_of_match, np); if (!match) { ret = -ENODEV; From 83e0357fcc617c3bc9cecbbb3313d810e2b4479f Mon Sep 17 00:00:00 2001 From: Satya Benson Date: Thu, 1 Oct 2026 16:04:13 +0000 Subject: [PATCH 194/631] usb: typec: tipd: accept Apple "DBMa" mode, add debugfs command access Apple's CD321x and SN201202x firmware has a debug mode, "DBMa". It is entered with an unlock key (LOCK) followed by the DBMa command. In this mode the application firmware still handles the port. It also accepts VDMs, which sends Apple VDMs (reboot, DFU, serial, Debug USB) to a connected Mac, and DVEn, which enables the local end of a debug route. tuxvdmtool and macvdmtool use these commands to control another Mac over its DFU port. A boot loader that arms Apple Debug USB on a port leaves its controller in DBMa. tipd then fails to probe it with 'controller in unsupported mode "DBMa"' (-ENODEV). The port gets no Type-C device and no kernel interface can reach the controller. SPMI-attached SN201202x controllers (M3 and later) also have no /dev/i2c-N fallback for userspace. Accept DBMa at probe and resume, as the Asahi tree has done since commit 786523ac62f0 ("usb: typec: tipd: Add cd321x specific control commands"). Unlike that commit, keep rejecting BIST and DISC. Use the controller as found: - The switch to the S0 power state runs before the mode check, as before. It sends SSPS only if the controller is not already in S0. - The patch/init step still runs only in PTCH mode. - Everything else is register reads plus the INT_MASK1 write. Probing therefore sends no command that leaves DBMa or disturbs a Debug USB route set up before Linux. Record whether DBMa was active at probe. Add a root-only interface at /sys/kernel/debug/usb/tipd//: command Write a 4CC followed by 0..64 bytes of DATA1. The command runs as a whole under the driver lock through tps6598x_exec_cmd_tmo(). Reading returns DATA1 as left by the last completed command of this open file. Only LOCK, DBMa, VDMs (200 ms timeout), DVEn and DISC are accepted. regs Read-only. pread() at offset = register (0..0x7f) returns up to 64 bytes, read under the driver lock. The interface works over both the i2c and SPMI transports. Unlike raw /dev/i2c access, it serialises with the interrupt handler. When an open command file that put the controller into DBMa is released, or the device is unbound while such a file is open, the controller leaves DBMa again (DBMa 0, then LOCK 0000). This is skipped if the controller was already in DBMa at probe, since that mode belongs to the boot loader's Debug USB setup. Controllers that are not in DBMa probe exactly as before, and nothing changes while the debugfs files are unused. The interface exists only with CONFIG_DEBUG_FS. Link: https://github.com/AsahiLinux/linux/commit/786523ac62f0 Link: https://github.com/aurora-silicon/tuxvdmtool Co-Authored-By: Claude Opus 5.5 (1M context) --- drivers/usb/typec/tipd/Kconfig | 1 + drivers/usb/typec/tipd/Makefile | 1 + drivers/usb/typec/tipd/core.c | 51 ++++-- drivers/usb/typec/tipd/debugfs.c | 292 +++++++++++++++++++++++++++++++ 4 files changed, 329 insertions(+), 16 deletions(-) create mode 100644 drivers/usb/typec/tipd/debugfs.c diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index 2f73bb17ed352b..a447a4622ea765 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -2,6 +2,7 @@ config TYPEC_TPS6598X_CORE tristate select USB_ROLE_SWITCH select POWER_SUPPLY + select USB_COMMON if DEBUG_FS config TYPEC_TPS6598X tristate "TI TPS6598x USB Power Delivery controller driver" diff --git a/drivers/usb/typec/tipd/Makefile b/drivers/usb/typec/tipd/Makefile index c0e850a56f2896..1a7605e7e57ecc 100644 --- a/drivers/usb/typec/tipd/Makefile +++ b/drivers/usb/typec/tipd/Makefile @@ -4,6 +4,7 @@ CFLAGS_trace.o := -I$(src) obj-$(CONFIG_TYPEC_TPS6598X_CORE) += tps6598x-core.o tps6598x-core-y := core.o tps6598x-core-$(CONFIG_TRACING) += trace.o +tps6598x-core-$(CONFIG_DEBUG_FS) += debugfs.o obj-$(CONFIG_TYPEC_TPS6598X) += tps6598x.o tps6598x-y := i2c.o diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 41991349be5828..bb7f471e4ba575 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -92,10 +92,6 @@ struct tps6598x_rx_identity_reg { struct usb_pd_identity identity; } __packed; -/* Standard Task return codes */ -#define TPS_TASK_TIMEOUT 1 -#define TPS_TASK_REJECTED 3 - /* Debounce delay for mode changes, in milliseconds */ #define CD321X_DEBOUNCE_DELAY_MS 500 @@ -105,6 +101,7 @@ enum { TPS_MODE_BIST, TPS_MODE_DISC, TPS_MODE_PTCH, + TPS_MODE_DBMA, }; static const char *const modes[] = { @@ -113,6 +110,7 @@ static const char *const modes[] = { [TPS_MODE_BIST] = "BIST", [TPS_MODE_DISC] = "DISC", [TPS_MODE_PTCH] = "PTCH", + [TPS_MODE_DBMA] = "DBMa", }; /* Unrecognized commands will be replaced with "!CMD" */ @@ -125,14 +123,7 @@ static enum power_supply_property tps6598x_psy_props[] = { static const char *tps6598x_psy_name_prefix = "tps6598x-source-psy-"; -/* - * Max data bytes for Data1, Data2, and other registers. See ch 1.3.2: - * https://www.ti.com/lit/ug/slvuan1a/slvuan1a.pdf - */ -#define TPS_MAX_LEN 64 - -static int -tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) +int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) { u8 data[TPS_MAX_LEN + 1]; int ret; @@ -298,10 +289,10 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status) power_supply_changed(tps->psy); } -static int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, - size_t in_len, const u8 *in_data, - size_t out_len, u8 *out_data, - u32 cmd_timeout_ms, u32 res_delay_ms) +int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, + size_t in_len, const u8 *in_data, + size_t out_len, u8 *out_data, + u32 cmd_timeout_ms, u32 res_delay_ms) { unsigned long timeout; u32 val; @@ -1473,6 +1464,14 @@ static int tps6598x_check_mode(struct tps6598x *tps) case TPS_MODE_BOOT: dev_warn(tps->dev, "dead-battery condition\n"); return ret; + case TPS_MODE_DBMA: + /* + * Apple firmware only: the application firmware runs in its + * debug mode (entered with LOCK + "DBMa"), in which it still + * handles the port and additionally accepts VDMs/DVEn. A boot + * loader arming Apple Debug USB on this port leaves it here. + */ + return ret; case TPS_MODE_BIST: case TPS_MODE_DISC: default: @@ -2276,6 +2275,17 @@ int tipd_init(struct tps6598x *tps) return ret; } + /* + * A controller found in "DBMa" is used as it is. Nothing below issues a + * 4CC command (the S0 power state switch above only does if the + * controller is not in S0 already), so probing neither leaves the debug + * mode nor disturbs a Debug USB or serial route set up before Linux. + * Remember it, so that the debugfs interface never tears that mode down. + */ + tps->dbma_at_probe = ret == TPS_MODE_DBMA; + if (tps->dbma_at_probe) + dev_info(tps->dev, "controller in debug mode \"DBMa\", leaving it there\n"); + ret = tps6598x_write64(tps, TPS_REG_INT_MASK1, tps->data->irq_mask1); if (ret) goto err_reset_controller; @@ -2344,6 +2354,7 @@ int tipd_init(struct tps6598x *tps) enable_irq_wake(tps->irq); } + tipd_debugfs_register(tps); if (tps->data->ready) tps->data->ready(tps); @@ -2369,6 +2380,8 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + tipd_debugfs_unregister(tps); + if (!tps->irq) cancel_delayed_work_sync(&tps->wq_poll); else @@ -2542,6 +2555,12 @@ const struct tipd_data tipd_sn201202x_data = { }; EXPORT_SYMBOL_GPL(tipd_sn201202x_data); +static void __exit tipd_core_exit(void) +{ + tipd_debugfs_exit(); +} +module_exit(tipd_core_exit); + MODULE_AUTHOR("Heikki Krogerus "); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("TI TPS6598x USB Power Delivery Controller Core Functions"); diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c new file mode 100644 index 00000000000000..33dbc9dca411d6 --- /dev/null +++ b/drivers/usb/typec/tipd/debugfs.c @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * TI TPS6598x / Apple CD321x / SN201202x debugfs interface + * + * /sys/kernel/debug/usb/tipd//command + * write: 4CC command followed by 0..64 bytes for DATA1. The command is + * executed as a whole under the driver's lock. Only a small set + * of commands used to talk to a connected Apple device (as + * tuxvdmtool does) is accepted. + * read: the 64 bytes of DATA1 that the last completed command of this + * open file left behind. + * + * /sys/kernel/debug/usb/tipd//regs + * read-only register access: pread() with the register as offset + * (0x00..0x7f) and the length (1..64 bytes) as count. + * + * An open "command" file that put the controller into "DBMa" leaves that + * mode (DBMa 0, then LOCK 0000) when it is released, unless the controller + * was in "DBMa" already when the driver probed: that mode belongs to + * whoever set it up before Linux (e.g. a boot loader arming Debug USB). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "tps6598x.h" + +#define TPS_REG_MAX 0x7f + +struct tipd_debugfs { + struct dentry *dentry; + /* Open "command" files that entered "DBMa"; protected by tps->lock */ + unsigned int dbma_holders; +}; + +struct tipd_debugfs_cmd { + char cmd[4]; + u8 min_len; + u8 max_len; + u32 timeout_ms; +}; + +static const struct tipd_debugfs_cmd tipd_debugfs_cmds[] = { + { "LOCK", 4, 4, 1000 }, /* unlock key, 0000 locks again */ + { "DBMa", 1, 1, 1000 }, /* 1: enter, 0: leave debug mode */ + { "VDMs", 5, 29, 200 }, /* SOP type/count + 1..7 VDOs */ + { "DVEn", 4, 28, 1000 }, /* local end of a serial/debug route */ + { "DISC", 0, 1, 1000 }, /* simulated disconnect */ +}; + +struct tipd_debugfs_file { + struct tps6598x *tps; + bool holds_dbma; + bool have_result; + u8 result[TPS_MAX_LEN]; +}; + +static struct dentry *tipd_debugfs_root; +static DEFINE_MUTEX(tipd_debugfs_root_lock); + +static const struct tipd_debugfs_cmd *tipd_debugfs_find_cmd(const u8 *cmd) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(tipd_debugfs_cmds); i++) + if (!memcmp(tipd_debugfs_cmds[i].cmd, cmd, 4)) + return &tipd_debugfs_cmds[i]; + + return NULL; +} + +/* Leave the debug mode entered through debugfs. Called with tps->lock held. */ +static void tipd_debugfs_leave_dbma(struct tps6598x *tps) +{ + static const u8 lock_key[4]; + u8 dbma = 0; + int ret; + + lockdep_assert_held(&tps->lock); + + if (tps->dbma_at_probe) + return; + + ret = tps6598x_exec_cmd_tmo(tps, "DBMa", sizeof(dbma), &dbma, 0, NULL, + 1000, 0); + if (ret) + dev_warn(tps->dev, "debugfs: failed to leave \"DBMa\": %d\n", ret); + + ret = tps6598x_exec_cmd_tmo(tps, "LOCK", sizeof(lock_key), lock_key, + 0, NULL, 1000, 0); + if (ret) + dev_warn(tps->dev, "debugfs: failed to lock: %d\n", ret); +} + +static int tipd_debugfs_command_open(struct inode *inode, struct file *file) +{ + struct tipd_debugfs_file *f; + + f = kzalloc_obj(*f); + if (!f) + return -ENOMEM; + + f->tps = inode->i_private; + file->private_data = f; + + return 0; +} + +static int tipd_debugfs_command_release(struct inode *inode, struct file *file) +{ + struct tipd_debugfs_file *f = file->private_data; + struct dentry *dentry = file->f_path.dentry; + struct tps6598x *tps = f->tps; + + /* + * debugfs calls ->release even after the file was removed, when tps + * may be gone. tipd_debugfs_unregister() has then left "DBMa" itself. + */ + if (f->holds_dbma && !debugfs_file_get(dentry)) { + mutex_lock(&tps->lock); + if (!--tps->debugfs->dbma_holders) + tipd_debugfs_leave_dbma(tps); + mutex_unlock(&tps->lock); + debugfs_file_put(dentry); + } + + kfree(f); + + return 0; +} + +static ssize_t tipd_debugfs_command_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tipd_debugfs_file *f = file->private_data; + struct tps6598x *tps = f->tps; + const struct tipd_debugfs_cmd *cmd; + u8 buf[4 + TPS_MAX_LEN]; + u8 out[TPS_MAX_LEN] = { }; + size_t len; + int ret; + + if (count < 4 || count > sizeof(buf)) + return -EINVAL; + + if (copy_from_user(buf, ubuf, count)) + return -EFAULT; + + cmd = tipd_debugfs_find_cmd(buf); + if (!cmd) + return -EOPNOTSUPP; + + len = count - 4; + if (len < cmd->min_len || len > cmd->max_len) + return -EINVAL; + if (!memcmp(cmd->cmd, "VDMs", 4) && (len - 1) % 4) + return -EINVAL; + if (!memcmp(cmd->cmd, "DVEn", 4) && len % 4) + return -EINVAL; + + mutex_lock(&tps->lock); + + ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, + sizeof(out), out, cmd->timeout_ms, 0); + + /* DATA1 was read back if the command ran, even if its task failed */ + f->have_result = !ret || + (ret == -ETIMEDOUT && out[0] == TPS_TASK_TIMEOUT) || + (ret == -EPERM && out[0] == TPS_TASK_REJECTED); + if (f->have_result) + memcpy(f->result, out, sizeof(out)); + + if (!ret && !memcmp(cmd->cmd, "DBMa", 4)) { + if (buf[4] && !(out[0] & 0xf) && !f->holds_dbma) { + f->holds_dbma = true; + tps->debugfs->dbma_holders++; + } else if (!buf[4] && f->holds_dbma) { + f->holds_dbma = false; + tps->debugfs->dbma_holders--; + } + } + + mutex_unlock(&tps->lock); + + return ret ? ret : count; +} + +static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tipd_debugfs_file *f = file->private_data; + + if (!f->have_result) + return -ENODATA; + + return simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); +} + +static const struct file_operations tipd_debugfs_command_fops = { + .owner = THIS_MODULE, + .open = tipd_debugfs_command_open, + .release = tipd_debugfs_command_release, + .write = tipd_debugfs_command_write, + .read = tipd_debugfs_command_read, + .llseek = default_llseek, +}; + +static ssize_t tipd_debugfs_regs_read(struct file *file, char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct tps6598x *tps = file->private_data; + u8 buf[TPS_MAX_LEN]; + int ret; + + if (*ppos < 0 || *ppos > TPS_REG_MAX) + return -EINVAL; + if (!count) + return 0; + if (count > sizeof(buf)) + return -EINVAL; + + mutex_lock(&tps->lock); + ret = tps6598x_block_read(tps, *ppos, buf, count); + mutex_unlock(&tps->lock); + if (ret) + return ret; + + /* The offset selects a register; it does not advance. */ + if (copy_to_user(ubuf, buf, count)) + return -EFAULT; + + return count; +} + +static const struct file_operations tipd_debugfs_regs_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = tipd_debugfs_regs_read, + .llseek = default_llseek, +}; + +void tipd_debugfs_register(struct tps6598x *tps) +{ + struct tipd_debugfs *dbg; + + mutex_lock(&tipd_debugfs_root_lock); + if (!tipd_debugfs_root) + tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); + mutex_unlock(&tipd_debugfs_root_lock); + + dbg = kzalloc_obj(*dbg); + if (!dbg) + return; + + dbg->dentry = debugfs_create_dir(dev_name(tps->dev), tipd_debugfs_root); + tps->debugfs = dbg; + debugfs_create_file("command", 0600, dbg->dentry, tps, + &tipd_debugfs_command_fops); + debugfs_create_file("regs", 0400, dbg->dentry, tps, + &tipd_debugfs_regs_fops); +} + +void tipd_debugfs_unregister(struct tps6598x *tps) +{ + struct tipd_debugfs *dbg = tps->debugfs; + + if (!dbg) + return; + + /* Waits for running file operations, later ones fail with -EIO */ + debugfs_remove_recursive(dbg->dentry); + + mutex_lock(&tps->lock); + if (dbg->dbma_holders) + tipd_debugfs_leave_dbma(tps); + tps->debugfs = NULL; + mutex_unlock(&tps->lock); + + kfree(dbg); +} + +void tipd_debugfs_exit(void) +{ + debugfs_remove(tipd_debugfs_root); +} From ca9b27f43c9c8ef7c5045a5a6c9c7344103ebd09 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:21:57 +1100 Subject: [PATCH 195/631] usb: typec: tipd: preserve shared and inherited debug mode Reject mode exit and relocking when they would invalidate another open file or the bootloader Debug USB route. Validate the DBMa boolean payload, and serialize command-result reads with writes on the controller lock. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/debugfs.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c index 33dbc9dca411d6..5a8d576eff0dc6 100644 --- a/drivers/usb/typec/tipd/debugfs.c +++ b/drivers/usb/typec/tipd/debugfs.c @@ -163,9 +163,21 @@ static ssize_t tipd_debugfs_command_write(struct file *file, return -EINVAL; if (!memcmp(cmd->cmd, "DVEn", 4) && len % 4) return -EINVAL; + if (!memcmp(cmd->cmd, "DBMa", 4) && buf[4] > 1) + return -EINVAL; mutex_lock(&tps->lock); + /* Preserve inherited mode and the leases held by other open files. */ + if ((!memcmp(cmd->cmd, "DBMa", 4) && !buf[4] && + (tps->dbma_at_probe || + tps->debugfs->dbma_holders > (unsigned int)f->holds_dbma)) || + (!memcmp(cmd->cmd, "LOCK", 4) && !memchr_inv(buf + 4, 0, 4) && + (tps->dbma_at_probe || tps->debugfs->dbma_holders))) { + ret = -EBUSY; + goto out; + } + ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, sizeof(out), out, cmd->timeout_ms, 0); @@ -186,6 +198,7 @@ static ssize_t tipd_debugfs_command_write(struct file *file, } } +out: mutex_unlock(&tps->lock); return ret ? ret : count; @@ -195,12 +208,17 @@ static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { struct tipd_debugfs_file *f = file->private_data; + ssize_t ret; + mutex_lock(&f->tps->lock); if (!f->have_result) - return -ENODATA; + ret = -ENODATA; + else + ret = simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); + mutex_unlock(&f->tps->lock); - return simple_read_from_buffer(ubuf, count, ppos, f->result, - sizeof(f->result)); + return ret; } static const struct file_operations tipd_debugfs_command_fops = { From b15605d5cee3656937b63db1faef2a9763c3af78 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:51:13 -0500 Subject: [PATCH 196/631] accel/ane: Add the ane_stats counters and ane_timeline Add producer-side statistics to both ANE drivers. /sys/class/accel/accel/device/ane_stats is a read-only (0444) file with two cumulative counters, one "key value" pair per line: - busy_ns: the nanoseconds the engine was busy since bind. A busy period opens when the first submission arrives on an idle engine and closes when the last one completes. Overlapping submissions share a period, so busy_ns is the union of the submit-to-completion windows, not their sum. ane serializes submissions behind engine_lock, so there the union is the sum. A read adds the time of the open period, so the value also increases while work is in flight. - jobs: the completed engine submissions since bind. ane counts each ANE_SUBMIT. ane_t6021 counts only CSNE_CMD_PROCEDURE_CALL. The control-plane commands on the same path and the boot transport are not engine work and do not count. A submission that fails still completes its begin/complete pair, so the counters stay balanced. Documentation/ABI/testing/sysfs-class-accel-ane describes the file. The ane_timeline debugfs file shows the latest submissions from a ring that the driver allocates at probe: seq, submit_ns, start_ns, end_ns, tasks, rc and tmst. Each slot has a sequence number, so a reader skips a slot that is in flight or that a later submission reuses. ane records the raw TM tick as tmst. ane_t6021 has no host TM path and records 0. Both drivers create the file with drm_debugfs_add_file(). The producer uses atomics only. It does not allocate, lock or format. A transition sentinel in the inflight count keeps the fold of one busy period and the start of the next apart. The next period starts at or after the fold end of the previous one. Both modules have a read-only stats parameter, default 1, that the driver reads at probe. With stats=0, or when the ring allocation fails, the driver creates no stats files and the submission path skips the counters. Testing: in the tree, this code is compile-verified only (arm64 W=1, =m and =y, and checkpatch --strict). A host test of the accounting through a kernel API shim passes. It includes a model check of all orders of three jobs, with a preemption at each atomic operation. The out-of-tree omarchy-ane driver with the same accounting passed the hardware gates on T8103 (M1) and T6001 (M1 Max). That version read max_end for the period start before it took the transition sentinel; this commit reads it after. The T6021 (M2 Max) path is in test. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/accel/ane/ane_stats.h | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/accel/ane/ane_stats.h b/drivers/accel/ane/ane_stats.h index 4a2c40784817f4..768fe97846edc3 100644 --- a/drivers/accel/ane/ane_stats.h +++ b/drivers/accel/ane/ane_stats.h @@ -148,7 +148,8 @@ static inline void ane_stats_counters_init(struct ane_stats_counters *ctrs, * * A period never starts before the previous one folded: a caller * sample can be stale (taken before the transition), so the latch is - * max(submit_ns, max_end). + * max(submit_ns, max_end), with max_end read under the transition + * sentinel. */ static inline u64 ane_stats_begin(struct ane_stats_counters *ctrs, struct ane_stats_ring *ring, @@ -157,12 +158,8 @@ static inline u64 ane_stats_begin(struct ane_stats_counters *ctrs, u64 ticket = atomic64_fetch_add(1ull, &ring->head) + 1ull; struct ane_stats_ring_entry *e = &ring->slots[(size_t)(ticket - 1ull) & ring->mask]; - u64 latch = atomic64_read(&ctrs->max_end); - u32 cur; - - if (submit_ns > latch) - latch = submit_ns; - cur = atomic_read(&ctrs->inflight); + u64 latch; + u32 cur = atomic_read(&ctrs->inflight); for (;;) { if (cur == ANE_STATS_INFLIGHT_TRANS) { @@ -173,6 +170,18 @@ static inline u64 ane_stats_begin(struct ane_stats_counters *ctrs, cur = atomic_read(&ctrs->inflight); continue; } + /* + * Read max_end only now. The drainer of the previous + * period fed its end sample into max_end before its + * release store of inflight = 0, and every other + * member fed before its decrement. The fully ordered + * cmpxchg above read that 0, so this read sees every + * feed and the latch is never below the previous + * fold end: periods cannot overlap. + */ + latch = atomic64_read(&ctrs->max_end); + if (submit_ns > latch) + latch = submit_ns; atomic64_set_release(&ctrs->last_busy_end, latch); atomic_set_release(&ctrs->inflight, 1u); break; From d645ddad88f9eef43103a79dd201368351c09261 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sat, 3 Oct 2026 13:59:19 -0500 Subject: [PATCH 197/631] arm64: dts: apple: enable the ANE on t8103, t6000 and t6020 The ANE node, its DARTs, its mailbox and the power states only it uses are disabled on the M1 (T8103), the M1 Pro (T6000) and the M2 Pro (T6020) behind APPLE_ANE_UNTESTED, because the driver had run only on the M1 Max and the M2 Max. It has now run with this tree's own driver on all three. Each chip has a passing row in the public community dataset (promotion_check --remote: judged, driver_source=intree, 20 smoke calls that match the golden bit for bit, no ANE, DART or mailbox fault): M1 (T8103) row 3559e0f328ba M1 Pro (T6000) row 6eb94f49985b M2 Pro (T6020) row 3c9389040f51 Remove the define from t6000.dtsi, t6020.dtsi and t8103.dtsi. No file defines it after this, so also remove the #ifndef APPLE_ANE_UNTESTED guards in t6001.dtsi, t6021.dtsi and t8103.dtsi, and correct the comments that said the ANE had not run on the M1 Pro and the M2 Pro. The ANE is now enabled on the t8103 boards (j274, j293, j313, j456, j457), the t6000 boards (j314s, j316s) and the t6020 boards (j414s, j416s, j474s). The DTBs of the t6001, t6002, t6021, t6022 and t8112 boards do not change: t6002.dtsi and t6022.dtsi include neither t6001.dtsi nor t6021.dtsi, so the ANE stays disabled on the M1 Ultra (T6002), the M2 Ultra (T6022) and the M2 (T8112). Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- arch/arm64/boot/dts/apple/t6001.dtsi | 4 +--- arch/arm64/boot/dts/apple/t6021.dtsi | 6 ++---- arch/arm64/boot/dts/apple/t8103.dtsi | 6 ++++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t6001.dtsi b/arch/arm64/boot/dts/apple/t6001.dtsi index 95bf2d4d8c6393..f14bd14518042a 100644 --- a/arch/arm64/boot/dts/apple/t6001.dtsi +++ b/arch/arm64/boot/dts/apple/t6001.dtsi @@ -90,10 +90,9 @@ }; /* - * The ANE has run on the M1 Max and the M1 Pro, so enable it here with its + * The ANE has run on the M1 Pro and the M1 Max, so enable it here with its * DARTs and the power states only it uses. t6000.dtsi includes this file. */ -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -133,4 +132,3 @@ &ps_ane_set4 { status = "okay"; }; -#endif diff --git a/arch/arm64/boot/dts/apple/t6021.dtsi b/arch/arm64/boot/dts/apple/t6021.dtsi index 7154f4644e749b..53bd36ef4b843f 100644 --- a/arch/arm64/boot/dts/apple/t6021.dtsi +++ b/arch/arm64/boot/dts/apple/t6021.dtsi @@ -87,10 +87,9 @@ }; /* - * The ANE has run on the M2 Max, so enable it here with its mailbox and - * DARTs. The M2 Pro (t6020.dtsi includes this file) has the same die 0 ANE. + * The ANE has run on the M2 Pro and the M2 Max, so enable it here with its + * mailbox and DARTs. t6020.dtsi includes this file. */ -#ifndef APPLE_ANE_UNTESTED &ane { status = "okay"; }; @@ -110,4 +109,3 @@ &ane_mbox { status = "okay"; }; -#endif diff --git a/arch/arm64/boot/dts/apple/t8103.dtsi b/arch/arm64/boot/dts/apple/t8103.dtsi index fadb7d79268bd4..9c2874c12b732f 100644 --- a/arch/arm64/boot/dts/apple/t8103.dtsi +++ b/arch/arm64/boot/dts/apple/t8103.dtsi @@ -2254,7 +2254,10 @@ #include "t8103-pmgr.dtsi" -#ifndef APPLE_ANE_UNTESTED +/* + * The ANE has run on the M1, so enable it here with its DARTs and the + * power states only it uses. + */ &ane { status = "okay"; }; @@ -2294,4 +2297,3 @@ &ps_ane_set5 { status = "okay"; }; -#endif From 13b1c4c51e9eb7daff3e624c32ee853d4ebbcb17 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:55 +1100 Subject: [PATCH 198/631] accel/ane: keep the wedge until recovery proves DMA quiescent Closing any ANE file cleared the global wedge before recovery checked it, so the reset was skipped and live DMA backing could be freed or reused. Retry recovery with the wedge and module pin intact. Only the existing successful reset path may clear them; a failed reset requires a reboot and must preserve resources across unrelated file closes. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 9 --------- drivers/accel/ane/ane_drv.c | 24 +----------------------- drivers/accel/ane/ane_tm.c | 23 ----------------------- 3 files changed, 1 insertion(+), 55 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index eac81bdab9f748..9a512e7568e60a 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -135,15 +135,6 @@ struct ane_request { u32 bar[DRM_ANE_TILE_COUNT]; }; -/* - * Drop the wedge pin (module refcount) without requiring a successful - * recovery. Used by ane_drm_postclose so a wedged engine cannot leave - * the module un-unloadable after a clean session close. Recovery stays - * safe to call afterwards; if it returns error, the postclose path has - * already unlocked the module count so the operator can swap the ko. - */ -void ane_wedge_clear(struct ane_device *ane); - /* * Reclaim wedge-preserved BO ranges (unmap, release nodes and pages). * Only legal once DMA is provably quiescent: after a successful diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 70f0d9bd9b3657..7fd0b9e717a910 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -796,21 +796,7 @@ static int ane_drm_open(struct drm_device *drm, struct drm_file *file) return 0; } -/* - * Per-session teardown: every file context open is a fresh boundary. - * Anything the previous session left on the device (wedged flag, tm - * completion counters, in-flight recovery state) belongs to that - * session and must not survive into the next one. - * - * drm_gem_release runs before us and has dropped every handle this - * file held. BOs leaked by `ane_gem_free_object under wedge` stay on - * ane->bo_list until module teardown (their backing `drm_gem_object`s - * are already gone, and touching them here would be use-after-free). - * We do not try to drain them; we only clear the session-leaving - * state (wedged + tm counters + recovering flag) so the next opener - * sees a clean device. The orphan BOs remain inert: they are not in - * any drm_file's handle table. - */ +/* Retry recovery on close, retaining quarantine if DMA cannot be quiesced. */ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) { struct ane_device *ane = drm->dev_private; @@ -821,14 +807,6 @@ static void ane_drm_postclose(struct drm_device *drm, struct drm_file *file) return; } - /* Drop the wedge pin first: a wedged engine that cannot be - * recovered in software must NOT keep the module pinned, or the - * operator cannot unload the ko for an updated build without a - * reboot. ane_wedge_clear releases both the wedged flag and the - * module_refcount; ane_tm_recover then attempts a power-cycle - * recovery (succeeds only if the engine is actually idle-able). - */ - ane_wedge_clear(ane); ane_tm_recover(ane); mutex_unlock(&ane->engine_lock); diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 703d595b612a99..34f5144cea2a95 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -645,26 +645,3 @@ int ane_tm_recover(struct ane_device *ane) return 0; } -/* - * Drop the wedge pin on demand. The recovery path in ane_tm_recover - * does an atomic_xchg + module_put, but only on its success branch - - * a real -110 with the engine stuck leaves ane->wedged set and the - * module refcount one too high, so rmmod fails forever and the bug - * becomes a kernel-brick-without-reboot. ane_drm_postclose calls - * this unconditionally so every session close releases the pin, - * letting the operator unload the ko for an updated build even when - * the engine itself cannot be cleared in software. - * - * Pair count: must follow an ane_wedge_set() that called - * __module_get(THIS_MODULE). Today the only such set is in the wedge: - * branch of ane_tm_execute(); future sets must call this in their - * clean-up path too. - */ -void ane_wedge_clear(struct ane_device *ane) -{ - if (atomic_xchg(&ane->wedged, 0)) { - module_put(THIS_MODULE); - dev_info(ane->dev, - "wedge pin released (engine state unknown)\n"); - } -} From f7255b6c5f5918ff351ec7916f1814305be16909 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:58 +1100 Subject: [PATCH 199/631] accel/ane: retain fault-drain scratch mappings until quiescence A failed drain freed scratch pages and IOVA reservations before attempting to stop the DMA-active engine. Store the bounded scratch registry on the device, protected by engine_lock, so it survives the request's return. Reclaim it with the existing preserved BO registry only after successful recovery or a safe removal. Normal confirmed-idle completion still releases scratch immediately. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/accel/ane/ane.h | 21 ++++++++++++--------- drivers/accel/ane/ane_drv.c | 2 ++ drivers/accel/ane/ane_tm.c | 16 +++++++--------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/drivers/accel/ane/ane.h b/drivers/accel/ane/ane.h index 9a512e7568e60a..c9d18fb13f5213 100644 --- a/drivers/accel/ane/ane.h +++ b/drivers/accel/ane/ane.h @@ -17,6 +17,15 @@ #include "ane_stats.h" +#define ANE_DART_MAX 3 +#define ANE_DART_SCRATCH_MAX 16 + +struct ane_dart_scratch { + u64 iova; + struct page *page; + struct drm_mm_node *node; +}; + struct ane_device { struct drm_device drm; struct device *dev; @@ -62,6 +71,9 @@ struct ane_device { * nodes (see ane_reclaim_preserved). Registered under engine_lock. */ struct list_head preserved_list; + /* Fault-drain DMA backing, retained until completion or a safe reset. */ + struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; + int scratch_pages; bool removed; /* @@ -147,15 +159,6 @@ void ane_boost_exit(struct ane_device *ane); void ane_boost_begin(struct ane_device *ane); void ane_boost_end(struct ane_device *ane); -#define ANE_DART_MAX 3 -#define ANE_DART_SCRATCH_MAX 16 - -struct ane_dart_scratch { - u64 iova; - struct page *page; - struct drm_mm_node *node; -}; - int ane_dart_init(struct ane_device *ane); void ane_dart_mask(struct ane_device *ane); void ane_dart_unmask(struct ane_device *ane); diff --git a/drivers/accel/ane/ane_drv.c b/drivers/accel/ane/ane_drv.c index 7fd0b9e717a910..c98f2348e82f1b 100644 --- a/drivers/accel/ane/ane_drv.c +++ b/drivers/accel/ane/ane_drv.c @@ -360,6 +360,8 @@ void ane_reclaim_preserved(struct ane_device *ane) unsigned long count = 0; lockdep_assert_held(&ane->engine_lock); + ane_dart_release_scratch(ane, ane->scratch, ane->scratch_pages); + ane->scratch_pages = 0; list_for_each_entry_safe(p, tmp, &ane->preserved_list, list) { u64 iova = p->mm->start; diff --git a/drivers/accel/ane/ane_tm.c b/drivers/accel/ane/ane_tm.c index 34f5144cea2a95..ba234d57cdf74f 100644 --- a/drivers/accel/ane/ane_tm.c +++ b/drivers/accel/ane/ane_tm.c @@ -265,8 +265,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) u32 finished = 0; u32 fault_status = 0; u64 fault_iova = 0; - struct ane_dart_scratch scratch[ANE_DART_SCRATCH_MAX]; - int scratch_pages = 0; + struct ane_dart_scratch *scratch = ane->scratch; bool faulted = false; int err, status; @@ -298,10 +297,10 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) err = status; if (ane_dart_faulted(ane, &fault_iova, &fault_status)) { faulted = true; - if (scratch_pages >= ANE_DART_SCRATCH_MAX) + if (ane->scratch_pages >= ANE_DART_SCRATCH_MAX) break; if (ane_dart_drain_fault(ane, fault_iova, scratch, - &scratch_pages)) + &ane->scratch_pages)) break; /* The retried transaction now completes; poll on. */ continue; @@ -321,7 +320,7 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) if (err || ane_dart_faulted(ane, NULL, NULL)) { /* Drain failed or the drained program never reached * completion: the engine is not idle and access is - * off the table. wedge releases the scratch. + * off the table. Retain scratch backing until reset. */ err = err ?: -EIO; goto wedge; @@ -334,11 +333,12 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) */ rmb(); tq_write32(ane, TQ_STATUS(req->qid), 0x0); - ane_dart_release_scratch(ane, scratch, scratch_pages); + ane_dart_release_scratch(ane, scratch, ane->scratch_pages); ane_dart_unmask(ane); dev_err(ane->dev, "DART fault contained: status=%#x iova=%#llx scratch=%d\n", - fault_status, fault_iova, scratch_pages); + fault_status, fault_iova, ane->scratch_pages); + ane->scratch_pages = 0; return -EIO; } if (err) @@ -356,7 +356,6 @@ int ane_tm_execute(struct ane_device *ane, struct ane_request *req) return 0; wedge: - ane_dart_release_scratch(ane, scratch, scratch_pages); ane_dart_unmask(ane); if (atomic_xchg(&ane->wedged, 1) == 0) { __module_get(THIS_MODULE); @@ -644,4 +643,3 @@ int ane_tm_recover(struct ane_device *ane) } return 0; } - From 005c28c977fb145c7208bc947391cca758d5eb07 Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:06:57 -0400 Subject: [PATCH 200/631] dt-bindings: media: describe Apple T8140 AVE Describe the encoder register blocks, paired DARTs, five power domains and two bootloader-supplied firmware reservations used by J700. Signed-off-by: Acelogic <7362641+Acelogic@users.noreply.github.com> --- .../bindings/media/apple,t8140-ave.yaml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 Documentation/devicetree/bindings/media/apple,t8140-ave.yaml diff --git a/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml new file mode 100644 index 00000000000000..110ebbc3d09e21 --- /dev/null +++ b/Documentation/devicetree/bindings/media/apple,t8140-ave.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/apple,t8140-ave.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 video encoder + +maintainers: + - Miguel Cruz <7362641+Acelogic@users.noreply.github.com> + +description: | + The T8140 AVE coprocessor encodes video using firmware preloaded by the + bootloader. Its two DART instances translate the same stream. The + bootloader supplies two reserved firmware regions and their IOVA mappings. + +properties: + compatible: + const: apple,t8140-ave + + reg: + items: + - description: Codec registers + - description: Mailbox registers + - description: Coprocessor registers + + reg-names: + items: + - const: codec + - const: mailbox + - const: coproc + + interrupts: + maxItems: 5 + + power-domains: + maxItems: 5 + + power-domain-names: + items: + - const: dma + - const: pipe4 + - const: pipe5 + - const: me0 + - const: me1 + + iommus: + maxItems: 2 + + memory-region: + items: + - description: Preloaded firmware text + - description: Preloaded firmware data + +required: + - compatible + - reg + - reg-names + - interrupts + - power-domains + - power-domain-names + - iommus + +additionalProperties: false From e614d36ad33e2c81ea1918bb0839d1f806ea9ac2 Mon Sep 17 00:00:00 2001 From: Acelogic <7362641+Acelogic@users.noreply.github.com> Date: Fri, 2 Oct 2026 02:06:58 -0400 Subject: [PATCH 201/631] media: apple: add an opt-in J700 AVE encoding transport Port Ace (Acelogic)'s existing GPL J700 diagnostic implementation to the public media tree, retaining the qualified single-command path. Add an explicit opt-in and limit admission to the known firmware text images. Preserve firmware reservations, validate DMA mappings, and keep resources pinned after startup until hardware reset. This is an experimental root-only debugfs transport for fixed-profile H.264/HEVC work. It does not register a V4L2 encoder or provide a stable userspace ABI. Firmware images are not included. Document those limits and the physical validation gates. Signed-off-by: Acelogic <7362641+Acelogic@users.noreply.github.com> --- Documentation/admin-guide/media/apple-ave.rst | 63 +++ Documentation/admin-guide/media/index.rst | 1 + drivers/media/platform/apple/Kconfig | 1 + drivers/media/platform/apple/Makefile | 1 + drivers/media/platform/apple/ave/Kconfig | 16 + drivers/media/platform/apple/ave/Makefile | 3 + .../media/platform/apple/ave/apple-ave-ipc.h | 446 +++++++++++++++ drivers/media/platform/apple/ave/apple-ave.c | 532 ++++++++++++++++++ 8 files changed, 1063 insertions(+) create mode 100644 Documentation/admin-guide/media/apple-ave.rst create mode 100644 drivers/media/platform/apple/ave/Kconfig create mode 100644 drivers/media/platform/apple/ave/Makefile create mode 100644 drivers/media/platform/apple/ave/apple-ave-ipc.h create mode 100644 drivers/media/platform/apple/ave/apple-ave.c diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst new file mode 100644 index 00000000000000..100b86e69b2d58 --- /dev/null +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -0,0 +1,63 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +Experimental T8140 video encoding +================================ + +The J700 AVE driver exposes a root-only debugfs firmware transport for +hardware encoding experiments. It is not a V4L2 encoder, has no stable +userspace ABI, and does not make an FFmpeg encoder available. The initial +scope is the existing fixed 320x240 NV12 H.264/HEVC protocol experiment. + +Build ``CONFIG_VIDEO_APPLE_AVE_EXPERIMENTAL=m`` on a 16 KiB-page Apple kernel +with debugfs and translated DMA. Suspend, hibernation and kexec must be off. +Load with ``modprobe apple-ave enable=1``; probing is disabled otherwise. + +Firmware and boot requirements +------------------------------ + +Use the public Aurora m1n1 J700 AVE handoff. It reserves the firmware already +loaded by iBoot, fills the device's ``memory-region`` references and exports +``aurora,ave-text-crc32`` and ``aurora,ave-data-crc32`` strings under +``/chosen``. No firmware file is included or requested by this driver. +The current transport requires the qualified 25G76 two-segment layout: +0x14c000 text bytes and 0x130000 data bytes, mapped starting at IOVA +0x10000000000. Every mapping and both content CRCs are checked before use. +Text CRCs are restricted to the two known 9003.78.0 images (0x2ce7359e +and 0x69c54454). These checks detect build drift, not firmware authenticity. +Different firmware is unsupported, even if the device has the same SoC. + +The driver holds the DMA, pipe4, pipe5, me0 and me1 power domains and owns +the IPC, queue, log and firmware-requested heap allocations. Debugfs lives +under ``/sys/kernel/debug/285100000.video-codec``: + +* ``ave-status`` reports mappings, protocol phase and reply counters. +* ``ave-start-once`` accepts one explicit firmware start per boot. +* ``ave-command`` accepts one bounded packet and returns a framed reply. +* ``ave-arena`` stages image/reference/output surfaces within owned memory. +* ``ave-poll`` drains later firmware replies without another submission. + +Writes require ``CAP_SYS_RAWIO``. Command packets must come from a trusted, +matching local protocol experiment with all embedded surface addresses +relocated into this boot's owned arena. Root access does not make arbitrary +firmware packets valid. Binary command captures are supplied separately and +are not included in this contribution. + +Lifecycle and verification +-------------------------- + +After the start attempt, the module, DMA buffers and power references remain +pinned until an external hardware reset, including after failure. Do not +force-unload, unbind the DARTs, reuse uncertain buffers or retry a failed +command. Firmware quiescence and normal production teardown are unfinished. + +A transport acknowledgement is not an encoded frame. For physical tests, +record the exact kernel/module, DT and firmware hashes; submit changing +images; validate each codec reply; independently decode the current DMA +output; check every frame's dimensions and quality; then verify Stop/Close, +kernel health and the existing desktop/radio stack. Keep the generated +inputs, bitstreams, decoded pixels, logs and hashes as a repeatable artifact. + +The prior implementation was authored by Ace (Acelogic). This contribution +ports that existing GPL work. Arbitrary resolutions, standard media APIs, +throughput, concurrent clients and other codecs or SoCs are outside this +experiment. diff --git a/Documentation/admin-guide/media/index.rst b/Documentation/admin-guide/media/index.rst index b11737ae6c0468..65aea88dac7ed5 100644 --- a/Documentation/admin-guide/media/index.rst +++ b/Documentation/admin-guide/media/index.rst @@ -43,6 +43,7 @@ Documentation/process/debugging/media_specific_debugging_guide.rst v4l-drivers dvb-drivers + apple-ave **Copyright** |copy| 1999-2020 : LinuxTV Developers diff --git a/drivers/media/platform/apple/Kconfig b/drivers/media/platform/apple/Kconfig index 77588c8f4e5a58..f656926099b3e3 100644 --- a/drivers/media/platform/apple/Kconfig +++ b/drivers/media/platform/apple/Kconfig @@ -4,3 +4,4 @@ comment "Apple media platform drivers" source "drivers/media/platform/apple/isp/Kconfig" source "drivers/media/platform/apple/avd/Kconfig" +source "drivers/media/platform/apple/ave/Kconfig" diff --git a/drivers/media/platform/apple/Makefile b/drivers/media/platform/apple/Makefile index 72f8bb54b376df..dac50fc70e277f 100644 --- a/drivers/media/platform/apple/Makefile +++ b/drivers/media/platform/apple/Makefile @@ -2,3 +2,4 @@ obj-y += isp/ obj-y += avd/ +obj-y += ave/ diff --git a/drivers/media/platform/apple/ave/Kconfig b/drivers/media/platform/apple/ave/Kconfig new file mode 100644 index 00000000000000..629ce9244039ab --- /dev/null +++ b/drivers/media/platform/apple/ave/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config VIDEO_APPLE_AVE_EXPERIMENTAL + tristate "Apple T8140 experimental video encoder transport" + depends on ARCH_APPLE && ARM64_16K_PAGES + depends on PM_GENERIC_DOMAINS && IOMMU_DMA && DEBUG_FS + depends on !SUSPEND && !HIBERNATION && !KEXEC_CORE + select CRC32 + help + Experimental root-only AVE firmware and command transport for the + MacBook Neo. This does not register a V4L2 or FFmpeg encoder. + + It requires the bootloader's reserved, preloaded firmware and an + explicit enable=1 module parameter. After firmware startup, memory + and power references remain pinned until a hardware reset. Do not + enable this on a general-purpose kernel requiring suspend or kexec. diff --git a/drivers/media/platform/apple/ave/Makefile b/drivers/media/platform/apple/ave/Makefile new file mode 100644 index 00000000000000..c8e673e68f0807 --- /dev/null +++ b/drivers/media/platform/apple/ave/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_VIDEO_APPLE_AVE_EXPERIMENTAL) += apple-ave.o diff --git a/drivers/media/platform/apple/ave/apple-ave-ipc.h b/drivers/media/platform/apple/ave/apple-ave-ipc.h new file mode 100644 index 00000000000000..03047e7da692ea --- /dev/null +++ b/drivers/media/platform/apple/ave/apple-ave-ipc.h @@ -0,0 +1,446 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Experimental 25G76/64-bit AVE mailbox and bidirectional ring transport. + * Included after private driver types. All callers hold ave->lock. + */ +#define AVE_CHANNEL_OFF 0x4000 +#define AVE_HANDSHAKE_OFF 0x24000 +#define AVE_SHARED_OFF 0x28000 +#define AVE_COMMAND_OFF 0x40000 +#define AVE_COMMAND_BYTES SZ_128K +#define AVE_ARENA_OFF 0x80000 +#define AVE_START_MAGIC 0x08042006 + +static int ave_fw_offset(struct apple_ave *ave, u64 address, size_t bytes, u32 *offset) +{ + u64 delta = address - ave->ipc_fw_base; + + if (address < ave->ipc_fw_base || delta > ave->ipc.size || bytes > ave->ipc.size - delta) + return -ERANGE; + *offset = delta; + return 0; +} + +static void ave_mailbox_send(struct apple_ave *ave, u32 a, u32 b, u32 c, u32 d) +{ + writel(a, ave->mailbox + 0x18); + writel(b, ave->mailbox + 0x1c); + writel(c, ave->mailbox + 0x20); + writel(d, ave->mailbox + 0x24); + wmb(); + writel(1, ave->mailbox + 0x0c); +} + +static int ave_mailbox_receive(struct apple_ave *ave) +{ + u32 status; + int ret = readl_poll_timeout(ave->mailbox + 0x10, status, status & 1, 50, 5000000); + + if (ret) + return ret; + /* Acknowledge the interrupt before reading the four scratch words. */ + writel(1, ave->mailbox + 0x10); + for (unsigned int i = 0; i < 4; i++) + ave->ipc_reply[i] = readl(ave->mailbox + 0x18 + i * 4); + dma_rmb(); + dev_info(ave->dev, "AVE_IPC_MAILBOX phase=%s words=%#x,%#x,%#x,%#x\n", + ave->phase, ave->ipc_reply[0], ave->ipc_reply[1], ave->ipc_reply[2], ave->ipc_reply[3]); + return 0; +} + +static int ave_create_channels(struct apple_ave *ave) +{ + bool found[2] = { false, false }; + + for (unsigned int i = 0; i < 2; i++) { + u8 *desc = ave->ipc.cpu + AVE_CHANNEL_OFF + i * 0x100; + u32 direction = get_unaligned_le32(desc + 0x40); + u32 interrupt = get_unaligned_le32(desc + 0x44); + u32 slots = get_unaligned_le32(desc + 0x48), offset; + u64 address = get_unaligned_le64(desc + 0x4c); + struct ave_channel *c; + unsigned int id; + + if (!memchr(desc, 0, 64)) + return -EPROTO; + if (!strcmp(desc, "IO")) + id = 0; + else if (!strcmp(desc, "IO_T2H")) + id = 1; + else + return -EPROTO; + if (found[id] || direction != id || !interrupt || interrupt >= 32 || + !slots || slots > ave->channel_bytes / 64 || !IS_ALIGNED(address, 64) || + ave_fw_offset(ave, address, (size_t)slots * 64, &offset) || + offset < AVE_CHANNEL_OFF + 0x200 || + offset + slots * 64 > AVE_CHANNEL_OFF + ave->channel_bytes) + return -EPROTO; + found[id] = true; + c = &ave->channel[id]; + strscpy(c->name, desc, sizeof(c->name)); + c->mode = direction ^ 1; + c->interrupt = interrupt; + c->slots = slots; + c->offset = offset; + c->consumer = c->mode ? -1 : 0; + c->producer = c->mode ? 0 : -1; + } + if (!found[0] || !found[1] || + (ave->channel[0].offset < ave->channel[1].offset + ave->channel[1].slots * 64 && + ave->channel[1].offset < ave->channel[0].offset + ave->channel[0].slots * 64)) + return -EPROTO; + /* Initialize only host-originated slots; leave all padding and + * firmware-originated slots unchanged. Ownership is bit zero of address. + */ + for (unsigned int i = 0; i < ave->channel[0].slots; i++) { + __le64 *slot = ave->ipc.cpu + ave->channel[0].offset + i * 64; + slot[0] = cpu_to_le64(1); + slot[1] = 0; + slot[2] = 0; + } + dma_wmb(); + return 0; +} + +static int ave_negotiate_ipc(struct apple_ave *ave) +{ + u64 ticks, time_delta, handshake_address, channel_address; + u32 frequency, status, offset; + u8 *handshake = ave->ipc.cpu + AVE_HANDSHAKE_OFF; + int ret; + + ave->phase = "validate-offer"; + if (ave->hello[0] != 2 || ave->hello[2] != 0x100 || + ave->hello[1] < 0x200 || ave->hello[1] > AVE_HANDSHAKE_OFF - AVE_CHANNEL_OFF || + !ave->hello[3] || ave->hello[3] > 4 * SZ_1M) + return -EPROTO; + ave->channel_bytes = ave->hello[1]; + ret = ave_alloc_buffer(ave, &ave->heap, PAGE_ALIGN(ave->hello[3])); + if (ret) + return ret; + writel(1, ave->mailbox + 0x10); + ticks = readq(ave->coproc + 0x378000); + frequency = readl(ave->coproc + 0x360020); + if (frequency && frequency < 1000000) + return -EPROTO; + ticks = div64_u64(ticks, frequency ? frequency / 1000000 : 24); + time_delta = div_u64(ktime_get_ns(), 1000) - ticks; + writel(lower_32_bits(time_delta), ave->mailbox + 0x28); + writel(upper_32_bits(time_delta), ave->mailbox + 0x2c); + ave->phase = "ipc-base"; + ave_mailbox_send(ave, lower_32_bits(ave->ipc.dma), upper_32_bits(ave->ipc.dma), ave->ipc.size, 0); + ret = ave_mailbox_receive(ave); + if (ret) + return ret; + ave->ipc_fw_base = (u64)ave->ipc_reply[1] << 32 | ave->ipc_reply[0]; + if (!ave->ipc_fw_base || !IS_ALIGNED(ave->ipc_fw_base, PAGE_SIZE) || + ave->ipc_fw_base > U64_MAX - ave->ipc.size) + return -EPROTO; + memset(ave->ipc.cpu + AVE_CHANNEL_OFF, 0, ave->channel_bytes); + memset(handshake, 0, 0x50); + memset(ave->ipc.cpu + AVE_SHARED_OFF, 0, SZ_64K); + channel_address = ave->ipc_fw_base + AVE_CHANNEL_OFF; + handshake_address = ave->ipc_fw_base + AVE_HANDSHAKE_OFF; + put_unaligned_le64(channel_address, handshake + 8); + put_unaligned_le64(ave->ipc_fw_base + AVE_SHARED_OFF, handshake + 0x10); + put_unaligned_le32(SZ_64K, handshake + 0x18); + put_unaligned_le64(ave->heap.dma, handshake + 0x1c); + put_unaligned_le32(ave->hello[3], handshake + 0x24); + put_unaligned_le32(27, handshake + 0x28); + dma_wmb(); + ave->phase = "channel-offer"; + ave_mailbox_send(ave, lower_32_bits(handshake_address), upper_32_bits(handshake_address), 0, 0); + ret = ave_mailbox_receive(ave); + if (ret) + return ret; + ave->extra_bytes = ave->ipc_reply[2]; + if (ave_fw_offset(ave, (u64)ave->ipc_reply[1] << 32 | ave->ipc_reply[0], + ave->channel_bytes, &offset) || offset != AVE_CHANNEL_OFF || + ave->extra_bytes > 0x13c000) + return -EPROTO; + ave->phase = "channel-layout"; + ret = ave_create_channels(ave); + if (ret) + return ret; + ave->phase = "ready-flag"; + writel(AVE_START_MAGIC, ave->mailbox + 0x24); + ret = readl_poll_timeout(ave->mailbox + 0x24, status, !status, 50, 5000000); + if (ret) + return ret; + ave->ipc_ready = true; + ave->phase = "ready"; + return 0; +} + +static int ave_ring_send(struct apple_ave *ave, struct ave_channel *c, u64 addr, u32 len, u32 flags) +{ + __le64 *slot; + int next; + + if (c->producer < 0) + return -ENOSPC; + slot = ave->ipc.cpu + c->offset + c->producer * 64; + WRITE_ONCE(slot[1], cpu_to_le64(len)); + WRITE_ONCE(slot[2], cpu_to_le64(flags)); + dma_wmb(); + WRITE_ONCE(slot[0], cpu_to_le64(addr | (c->mode ^ 1))); + dma_wmb(); + if (c->consumer < 0) + c->consumer = c->producer; + next = (c->producer + 1) % c->slots; + c->producer = next == c->consumer ? -1 : next; + writel(BIT(c->interrupt), ave->mailbox + 0x0c); + return 0; +} + +static int ave_ring_receive(struct apple_ave *ave, struct ave_channel *c, u64 *addr, u32 *len, u32 *flags) +{ + __le64 *slot; + u64 word; + int next; + + if (c->consumer < 0) + return -EAGAIN; + slot = ave->ipc.cpu + c->offset + c->consumer * 64; + word = le64_to_cpu(READ_ONCE(slot[0])); + if ((word & 1) != c->mode) + return -EAGAIN; + dma_rmb(); + *addr = word & ~3ULL; + *len = le64_to_cpu(READ_ONCE(slot[1])); + *flags = le64_to_cpu(READ_ONCE(slot[2])); + if (c->producer < 0) + c->producer = c->consumer; + next = (c->consumer + 1) % c->slots; + c->consumer = next == c->producer ? -1 : next; + return 0; +} + +/* Root-only experimental packet transport. The packet is copied into owned + * coherent memory; timeout pins the command too and forbids reuse this boot. + * Firmware command success is established by decoding replies, not this API. + */ +static ssize_t ave_command_write(struct file *file, const char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + u8 *cmd = ave->ipc.cpu + AVE_COMMAND_OFF; + u64 deadline, addr; + u32 bytes, flags, offset; + bool ack = false; + int ret = 0; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + static_assert(AVE_COMMAND_OFF + AVE_COMMAND_BYTES <= AVE_ARENA_OFF); + if (len < 0x48 || len > AVE_COMMAND_BYTES) + return -EINVAL; + mutex_lock(&ave->lock); + if (!ave->ipc_ready || ave->command_failed) { + ret = -EBUSY; + goto out; + } + if (copy_from_user(cmd, data, len)) { + ret = -EFAULT; + goto out; + } + /* The common command timeout is {duration_us, creation_time_us}. + * Stamp it in the same Linux monotonic clock synchronized at startup; + * times from any other clock do not match the firmware's time base. + */ + put_unaligned_le64(3000000, cmd + 0x30); + put_unaligned_le64(div_u64(ktime_get_ns(), 1000), cmd + 0x38); + ave->response_bytes = 0; + ret = ave_ring_send(ave, &ave->channel[0], ave->ipc_fw_base + AVE_COMMAND_OFF, len, 0); + if (ret) + goto out; + ave->commands++; + deadline = ktime_get_ns() + 5ULL * NSEC_PER_SEC; + do { + u32 pending = readl(ave->mailbox + 0x10); + if (pending) + writel(pending, ave->mailbox + 0x10); + for (unsigned int ch = 0; ch < 2; ch++) { + ret = ave_ring_receive(ave, &ave->channel[ch], &addr, &bytes, &flags); + if (ret == -EAGAIN) + continue; + /* Channel 1 returns the original address with zero payload + * bytes, which is checked here. Only channel 2 is + * copied to response, independent of submitted packet size. + */ + if (ret || (!ch && bytes != 0) || + (ch && bytes > sizeof(ave->response) - 24) || + ave_fw_offset(ave, addr, bytes, &offset)) { + ret = -EPROTO; + goto failed; + } + dev_info(ave->dev, "AVE_COMMAND_REPLY channel=%u address=%#llx bytes=%u flags=%#x\n", + ch + 1, addr, bytes, flags); + if (!ch) { + if (addr != ave->ipc_fw_base + AVE_COMMAND_OFF) { + ret = -EPROTO; + goto failed; + } + ack = true; + ave->transport_acks++; + } else { + put_unaligned_le32(ch + 1, ave->response); + put_unaligned_le32(bytes, ave->response + 4); + put_unaligned_le32(flags, ave->response + 8); + put_unaligned_le32(0, ave->response + 12); + put_unaligned_le64(addr, ave->response + 16); + memcpy(ave->response + 24, ave->ipc.cpu + offset, bytes); + ave->response_bytes = bytes + 24; + ave->replies++; + /* Reply echoes use zero flags on the observed transport. */ + ret = ave_ring_send(ave, &ave->channel[ch], addr, bytes, 0); + if (ret) + goto failed; + } + } + if (ack && ave->response_bytes) { + ret = 0; + goto out; + } + usleep_range(100, 200); + } while (ktime_get_ns() < deadline); + ret = -ETIMEDOUT; +failed: + ave->command_failed = true; +out: + mutex_unlock(&ave->lock); + return ret ?: len; +} + +static ssize_t ave_command_read(struct file *file, char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + ssize_t ret; + + mutex_lock(&ave->lock); + ret = simple_read_from_buffer(data, len, pos, ave->response, ave->response_bytes); + mutex_unlock(&ave->lock); + return ret; +} + +static const struct file_operations ave_command_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_command_write, + .read = ave_command_read, +}; + +/* Drain later firmware events without publishing another command. Results + * replace the response buffer and consist of one or more 24-byte-header + * records, each followed by its declared payload. Never discard silently. + */ +static ssize_t ave_poll_write(struct file *file, const char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + bool requested; + u64 deadline, addr; + u32 bytes, flags, offset; + int ret; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + ret = kstrtobool_from_user(data, len, &requested); + if (ret || !requested) + return ret ?: -EINVAL; + mutex_lock(&ave->lock); + if (!ave->ipc_ready || ave->command_failed) { + ret = -EBUSY; + goto out; + } + ave->response_bytes = 0; + deadline = ktime_get_ns() + NSEC_PER_SEC; + do { + u32 pending = readl(ave->mailbox + 0x10); + u8 *dst; + + if (pending) + writel(pending, ave->mailbox + 0x10); + ret = ave_ring_receive(ave, &ave->channel[1], &addr, &bytes, &flags); + if (ret == -EAGAIN) { + usleep_range(100, 200); + continue; + } + if (ret || ave_fw_offset(ave, addr, bytes, &offset) || + bytes > sizeof(ave->response) - 24 || + ave->response_bytes > sizeof(ave->response) - 24 - bytes) { + ret = -EPROTO; + goto failed; + } + dst = ave->response + ave->response_bytes; + put_unaligned_le32(2, dst); + put_unaligned_le32(bytes, dst + 4); + put_unaligned_le32(flags, dst + 8); + put_unaligned_le32(0, dst + 12); + put_unaligned_le64(addr, dst + 16); + memcpy(dst + 24, ave->ipc.cpu + offset, bytes); + ave->response_bytes += 24 + bytes; + ave->replies++; + ret = ave_ring_send(ave, &ave->channel[1], addr, bytes, 0); + if (ret) + goto failed; + } while (ktime_get_ns() < deadline); + ret = 0; + goto out; +failed: + ave->command_failed = true; +out: + mutex_unlock(&ave->lock); + return ret ?: len; +} + +static const struct file_operations ave_poll_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_poll_write, +}; + +/* Bounded staging area for image/reference/output surfaces. Userspace offsets + * start at zero here, beyond all handshake, ring and command allocations. + * The status file supplies both DART and firmware addresses for this arena. + */ +static ssize_t ave_arena_read(struct file *file, char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + ssize_t ret; + + mutex_lock(&ave->lock); + dma_rmb(); + ret = simple_read_from_buffer(data, len, pos, ave->ipc.cpu + AVE_ARENA_OFF, + ave->ipc.size - AVE_ARENA_OFF); + mutex_unlock(&ave->lock); + return ret; +} + +static ssize_t ave_arena_write(struct file *file, const char __user *data, size_t len, loff_t *pos) +{ + struct apple_ave *ave = file->private_data; + ssize_t ret; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + mutex_lock(&ave->lock); + if (!ave->ipc_ready || ave->command_failed) + ret = -EBUSY; + else + ret = simple_write_to_buffer(ave->ipc.cpu + AVE_ARENA_OFF, + ave->ipc.size - AVE_ARENA_OFF, pos, data, len); + dma_wmb(); + mutex_unlock(&ave->lock); + return ret; +} + +static loff_t ave_arena_seek(struct file *file, loff_t offset, int whence) +{ + return fixed_size_llseek(file, offset, whence, AVE_IPC_BYTES - AVE_ARENA_OFF); +} + +static const struct file_operations ave_arena_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = ave_arena_read, + .write = ave_arena_write, + .llseek = ave_arena_seek, +}; diff --git a/drivers/media/platform/apple/ave/apple-ave.c b/drivers/media/platform/apple/ave/apple-ave.c new file mode 100644 index 00000000000000..2edbbb8092d254 --- /dev/null +++ b/drivers/media/platform/apple/ave/apple-ave.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* T8140 AVE bring-up: resources, firmware negotiation and IPC transport. + * Experimental root-only command interface. No registered encoder yet. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AVE_IPC_BYTES (20 * SZ_1M) +#define AVE_QUEUE_BYTES SZ_64K +#define AVE_FW_IOVA 0x10000000000ULL +#define AVE_LOG_BYTES 0x24000 +#define AVE_LOG_USED 0x20200 + +static bool enable; +module_param(enable, bool, 0400); +MODULE_PARM_DESC(enable, "Opt in to the experimental AVE transport"); + +/* Read the known Uranus registers once after PM resume. The separate hello + * command pins all resources because reset/quiescence is not yet qualified. + */ +static bool snapshot; +module_param(snapshot, bool, 0400); +MODULE_PARM_DESC(snapshot, "Capture powered AVE registers once without writing them"); + +struct ave_firmware { + phys_addr_t physical; + u64 iova; + size_t size; + u32 crc; + unsigned int verified_pages; +}; + +struct ave_buffer { + void *cpu; + dma_addr_t dma; + size_t size; + unsigned int verified_pages; +}; + +struct ave_channel { + char name[64]; + u32 mode, interrupt, slots; + u32 offset; + int consumer, producer; +}; + +struct apple_ave { + struct device *dev; + struct device *power[5]; + unsigned int num_power; + struct iommu_domain *domain; + struct ave_buffer ipc; + struct ave_buffer queue; + struct ave_buffer log; + struct ave_buffer heap; + struct ave_firmware fw[2]; + bool firmware_verified; + bool start_requested; + int start_result; + struct mutex lock; + void __iomem *coproc, *mailbox; + u32 hello[4]; + u64 ipc_fw_base; + u32 ipc_reply[4], channel_bytes, extra_bytes; + const char *phase; + bool ipc_ready, command_failed; + struct ave_channel channel[2]; + u32 commands, replies, transport_acks; + u8 response[SZ_64K]; + size_t response_bytes; + struct debugfs_blob_wrapper ipc_blob; + struct debugfs_blob_wrapper log_blob; + struct debugfs_blob_wrapper config_blob; + u64 vcbar; + u32 core_control, core_status, mailbox_status, scratch[8]; + struct dentry *debugfs; +}; + +static int ave_alloc_buffer(struct apple_ave *ave, struct ave_buffer *buf, size_t size); +#include "apple-ave-ipc.h" + +/* Cold-boot hello only. A successful response still needs the remaining IPC + * negotiation. Keep the module, PM reference and DMA buffers pinned until a + * target reboot, on both success and timeout; no unproven stop/free sequence. + */ +static ssize_t ave_start_write(struct file *file, const char __user *data, + size_t len, loff_t *ppos) +{ + struct apple_ave *ave = file->private_data; + __le32 *cfg = ave->queue.cpu, *log = ave->log.cpu; + bool requested; + u32 status; + int ret; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + ret = kstrtobool_from_user(data, len, &requested); + if (ret || !requested) + return ret ?: -EINVAL; + mutex_lock(&ave->lock); + if (ave->start_requested) { + ret = -EALREADY; + goto out; + } + if (!ave->firmware_verified || readq(ave->coproc + 0x50000) != (AVE_FW_IOVA | 1) || + readl(ave->coproc + 0x600044) || readl(ave->mailbox + 0x10)) { + ret = -EBUSY; + goto out; + } + for (unsigned int i = 0; i < 8; i++) { + if (readl(ave->mailbox + 0x18 + i * 4)) { + ret = -EBUSY; + goto out; + } + } + /* Startup values observed with the qualified 25G76 firmware: device + * enum 29, one active unit/group and sub-ID mask 1. Firmware maps the + * full page-aligned log allocation; the ring header advertises usage. + */ + memset(cfg, 0, 0x40); + cfg[1] = cpu_to_le32(29); + cfg[2] = cpu_to_le32(1); + cfg[3] = cpu_to_le32(1); + cfg[4] = cpu_to_le32(1); + cfg[8] = cpu_to_le32(lower_32_bits(ave->log.dma)); + cfg[9] = cpu_to_le32(upper_32_bits(ave->log.dma)); + cfg[10] = cpu_to_le32(AVE_LOG_BYTES); + cfg[12] = cpu_to_le32(0x77); + /* Log header: 128 KiB ring after its 512-byte header. */ + log[0] = cpu_to_le32(AVE_LOG_USED); + log[1] = cpu_to_le32(0x200); + log[2] = cpu_to_le32(0x20000); + dma_wmb(); + __module_get(THIS_MODULE); + ave->start_requested = true; + ave->phase = "hello"; + ave->start_result = -EINPROGRESS; + dev_info(ave->dev, "AVE_START_ONCE cfg=%pad log=%pad resources_pinned_until_reboot=1\n", + &ave->queue.dma, &ave->log.dma); + writel(0x08042006, ave->mailbox + 0x18); + writel(lower_32_bits(ave->queue.dma), ave->mailbox + 0x1c); + writel(upper_32_bits(ave->queue.dma), ave->mailbox + 0x20); + /* Preserve the protected firmware mappings inherited from iBoot. */ + writel(1, ave->coproc + 0x600808); + writel(0, ave->coproc + 0x600044); + writel(0x10000, ave->coproc + 0x600400); + writel(0x10, ave->coproc + 0x600044); + ret = readl_poll_timeout(ave->mailbox + 0x10, status, status & 1, 50, 5000000); + ave->start_result = ret; + ave->core_control = readl(ave->coproc + 0x600044); + ave->core_status = readl(ave->coproc + 0x600048); + ave->mailbox_status = status; + for (unsigned int i = 0; i < 8; i++) + ave->scratch[i] = readl(ave->mailbox + 0x18 + i * 4); + for (unsigned int i = 0; i < 4; i++) + ave->hello[i] = ave->scratch[i]; + dma_rmb(); + dev_info(ave->dev, "AVE_HELLO_RESULT result=%d status=%#x words=%#x,%#x,%#x,%#x core=%#x\n", + ret, status, ave->hello[0], ave->hello[1], ave->hello[2], ave->hello[3], ave->core_status); + if (!ret) + ret = ave_negotiate_ipc(ave); + ave->start_result = ret; + dev_info(ave->dev, "AVE_IPC_RESULT result=%d phase=%s ready=%u\n", + ret, ave->phase, ave->ipc_ready); +out: + mutex_unlock(&ave->lock); + return ret ?: len; +} + +static const struct file_operations ave_start_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ave_start_write, +}; + +static int ave_verify_firmware(struct apple_ave *ave) +{ + static const char * const crc_names[] = { + "aurora,ave-text-crc32", "aurora,ave-data-crc32", + }; + static const size_t sizes[] = { 0x14c000, 0x130000 }; + struct device_node *chosen, *node; + unsigned int i; + int ret = 0; + + if (of_count_phandle_with_args(ave->dev->of_node, "memory-region", NULL) != 2) + return dev_err_probe(ave->dev, -EINVAL, "Expected two live AVE firmware reservations\n"); + chosen = of_find_node_by_path("/chosen"); + if (!chosen) + return -ENOENT; + for (i = 0; i < ARRAY_SIZE(ave->fw); i++) { + struct ave_firmware *fw = &ave->fw[i]; + struct resource res; + const char *crc_string; + u32 maps[5], expected_crc; + void *cpu; + size_t offset; + + node = of_parse_phandle(ave->dev->of_node, "memory-region", i); + ret = of_address_to_resource(node, 0, &res); + if (!ret) + ret = of_property_read_u32_array(node, "iommu-addresses", maps, ARRAY_SIZE(maps)); + if (!ret && (!of_property_read_bool(node, "no-map") || + !of_device_is_compatible(node, "apple,asc-mem"))) + ret = -EINVAL; + of_node_put(node); + if (ret) + break; + fw->physical = res.start; + fw->size = resource_size(&res); + fw->iova = (u64)maps[1] << 32 | maps[2]; + if (fw->size != sizes[i] || !IS_ALIGNED(fw->physical, PAGE_SIZE) || + maps[0] != ave->dev->of_node->phandle || maps[3] || maps[4] != sizes[i] || + fw->iova != AVE_FW_IOVA + (i ? sizes[0] : 0)) { + ret = -EINVAL; + break; + } + /* The IOMMU core owns these translated reservations across module + * unloads. DMA allocation must never reuse their IOVAs. + */ + for (offset = 0; offset < fw->size; offset += PAGE_SIZE) { + if (iommu_iova_to_phys(ave->domain, fw->iova + offset) != fw->physical + offset) { + dev_err(ave->dev, "Firmware %u mapping mismatch at %#zx\n", i, offset); + ret = -EIO; + break; + } + fw->verified_pages++; + } + if (ret) + break; + ret = of_property_read_string(chosen, crc_names[i], &crc_string); + if (!ret) + ret = kstrtou32(crc_string, 16, &expected_crc); + if (ret) + break; + cpu = memremap(fw->physical, fw->size, MEMREMAP_WB); + if (!cpu) { + ret = -ENOMEM; + break; + } + fw->crc = crc32_le(~0U, cpu, fw->size) ^ ~0U; + memunmap(cpu); + if (fw->crc != expected_crc) { + dev_err(ave->dev, "Firmware %u changed since the boot preflight\n", i); + ret = -EILSEQ; + break; + } + /* Only the two known 9003.78.0 text images are supported. + * CRCs detect build drift; they are not firmware authentication. + */ + if (!i && fw->crc != 0x2ce7359e && fw->crc != 0x69c54454) { + dev_err(ave->dev, "Unsupported AVE firmware text CRC %#x\n", fw->crc); + ret = -ENODEV; + break; + } + } + of_node_put(chosen); + if (!ret) + ave->firmware_verified = true; + return ret; +} + +/* Check the real IOMMU page table against the CPU mapping of each allocation. + * This proves mappings, not that firmware has performed a DMA transaction. + */ +static int ave_verify_mapping(struct apple_ave *ave, struct ave_buffer *buf) +{ + size_t offset; + + if (!IS_ALIGNED(buf->dma, PAGE_SIZE) || !IS_ALIGNED(buf->size, PAGE_SIZE)) + return -EINVAL; + for (offset = 0; offset < buf->size; offset += PAGE_SIZE) { + void *cpu = buf->cpu + offset; + struct page *page = is_vmalloc_addr(cpu) ? vmalloc_to_page(cpu) : virt_to_page(cpu); + phys_addr_t physical = iommu_iova_to_phys(ave->domain, buf->dma + offset); + + if (!page || physical != page_to_phys(page)) { + dev_err(ave->dev, "DMA mapping mismatch at offset %#zx\n", offset); + return -EIO; + } + buf->verified_pages++; + } + return 0; +} + +static int ave_alloc_buffer(struct apple_ave *ave, struct ave_buffer *buf, size_t size) +{ + buf->size = size; + buf->cpu = dma_alloc_coherent(ave->dev, size, &buf->dma, GFP_KERNEL); + if (!buf->cpu) + return -ENOMEM; + memset(buf->cpu, 0, size); + return ave_verify_mapping(ave, buf); +} + +static void ave_free_buffer(struct apple_ave *ave, struct ave_buffer *buf) +{ + if (buf->cpu) + dma_free_coherent(ave->dev, buf->size, buf->cpu, buf->dma); + buf->cpu = NULL; +} + +static void ave_power_off(struct apple_ave *ave) +{ + while (ave->num_power) { + struct device *pd = ave->power[--ave->num_power]; + + pm_runtime_put_sync(pd); + dev_pm_domain_detach(pd, true); + } +} + +static int ave_power_on(struct apple_ave *ave) +{ + static const char * const names[] = { "dma", "pipe4", "pipe5", "me0", "me1" }; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(names); i++) { + struct device *pd = dev_pm_domain_attach_by_name(ave->dev, names[i]); + + if (IS_ERR(pd)) + return PTR_ERR(pd); + if (!pd) + return -ENODEV; + ret = pm_runtime_resume_and_get(pd); + if (ret < 0) { + dev_pm_domain_detach(pd, true); + return ret; + } + ave->power[ave->num_power++] = pd; + } + return 0; +} + +static int ave_status_show(struct seq_file *s, void *unused) +{ + struct apple_ave *ave = s->private; + + mutex_lock(&ave->lock); + seq_printf(s, "state=%s\nfirmware_start_requested=%u\nstart_result=%d\n", + ave->start_requested ? (ave->start_result ? "startup-failed" : "ipc-ready") : + "firmware-mapped", ave->start_requested, ave->start_result); + seq_printf(s, "hardware_dma_tested=%u\nencoder_registered=0\nphase=%s\n", + ave->ipc_ready, ave->phase ?: "not-started"); + seq_printf(s, "ipc_ready=%u\nipc_fw_base=%#llx\nchannel_bytes=%#x\nextra_bytes=%#x\n", + ave->ipc_ready, ave->ipc_fw_base, ave->channel_bytes, ave->extra_bytes); + seq_printf(s, "commands=%u\nreplies=%u\ntransport_acks=%u\ncommand_failed=%u\nresponse_bytes=%zu\n", + ave->commands, ave->replies, ave->transport_acks, ave->command_failed, ave->response_bytes); + seq_printf(s, "arena_bytes=%zu\narena_ipc_offset=%#x\narena_dma=%#llx\narena_fw=%#llx\n", + ave->ipc.size - AVE_ARENA_OFF, AVE_ARENA_OFF, + (u64)ave->ipc.dma + AVE_ARENA_OFF, ave->ipc_fw_base + AVE_ARENA_OFF); + for (unsigned int i = 0; i < 2; i++) { + struct ave_channel *c = &ave->channel[i]; + seq_printf(s, "channel%u_name=%s\nchannel%u_mode=%u\nchannel%u_interrupt=%u\n" + "channel%u_slots=%u\nchannel%u_offset=%#x\nchannel%u_consumer=%d\nchannel%u_producer=%d\n", + i, c->name, i, c->mode, i, c->interrupt, i, c->slots, + i, c->offset, i, c->consumer, i, c->producer); + } + seq_printf(s, "page_size=%lu\n", PAGE_SIZE); + seq_printf(s, "powered_domains=%u\n", ave->num_power); + seq_printf(s, "ipc_bytes=%zu\nipc_dma=%pad\nipc_verified_pages=%u\n", + ave->ipc.size, &ave->ipc.dma, ave->ipc.verified_pages); + seq_printf(s, "queue_bytes=%zu\nqueue_dma=%pad\nqueue_verified_pages=%u\n", + ave->queue.size, &ave->queue.dma, ave->queue.verified_pages); + seq_printf(s, "log_bytes=%zu\nlog_dma=%pad\nlog_verified_pages=%u\n", + ave->log.size, &ave->log.dma, ave->log.verified_pages); + seq_printf(s, "heap_bytes=%zu\nheap_dma=%pad\nheap_verified_pages=%u\n", + ave->heap.size, &ave->heap.dma, ave->heap.verified_pages); + for (unsigned int i = 0; i < ARRAY_SIZE(ave->fw); i++) + seq_printf(s, "fw%u_physical=%pa\nfw%u_iova=%#llx\nfw%u_bytes=%zu\n" + "fw%u_verified_pages=%u\nfw%u_crc32=%08x\n", + i, &ave->fw[i].physical, i, ave->fw[i].iova, i, ave->fw[i].size, + i, ave->fw[i].verified_pages, i, ave->fw[i].crc); + seq_printf(s, "register_snapshot=%u\n", snapshot); + if (snapshot) { + seq_printf(s, "vcbar=%#llx\ncore_control=%#x\ncore_status=%#x\nmailbox_status=%#x\n", + ave->vcbar, ave->core_control, ave->core_status, ave->mailbox_status); + for (unsigned int i = 0; i < ARRAY_SIZE(ave->scratch); i++) + seq_printf(s, "scratch%u=%#x\n", i, ave->scratch[i]); + } + seq_printf(s, "resources_pinned_until_reboot=%u\n", ave->start_requested); + mutex_unlock(&ave->lock); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ave_status); + +static int apple_ave_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct apple_ave *ave; + void __iomem *regs, *mailbox, *coproc; + int ret; + + if (!enable) + return -ENODEV; + if (!of_machine_is_compatible("apple,j700") || PAGE_SIZE != SZ_16K) + return -ENODEV; + ave = devm_kzalloc(dev, sizeof(*ave), GFP_KERNEL); + if (!ave) + return -ENOMEM; + ave->dev = dev; + mutex_init(&ave->lock); + /* Claim only dedicated AVE ranges, never the shared PMGR/SRAM windows. */ + regs = devm_platform_ioremap_resource_byname(pdev, "codec"); + if (IS_ERR(regs)) + return PTR_ERR(regs); + mailbox = devm_platform_ioremap_resource_byname(pdev, "mailbox"); + if (IS_ERR(mailbox)) + return PTR_ERR(mailbox); + coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); + if (IS_ERR(coproc)) + return PTR_ERR(coproc); + ave->coproc = coproc; + ave->mailbox = mailbox; + ave->domain = iommu_get_domain_for_dev(dev); + if (!ave->domain || !(ave->domain->type & __IOMMU_DOMAIN_PAGING)) + return dev_err_probe(dev, -ENODEV, "A translated IOMMU domain is required\n"); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + if (ret) + return ret; + ret = ave_power_on(ave); + if (ret) + goto release_power; + ret = devm_pm_runtime_enable(dev); + if (ret) + goto release_power; + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto release_power; + ret = ave_alloc_buffer(ave, &ave->ipc, AVE_IPC_BYTES); + if (ret) + goto free_buffers; + ret = ave_alloc_buffer(ave, &ave->queue, AVE_QUEUE_BYTES); + if (ret) + goto free_buffers; + ret = ave_alloc_buffer(ave, &ave->log, AVE_LOG_BYTES); + if (ret) + goto free_buffers; + ret = ave_verify_firmware(ave); + if (ret) + goto free_buffers; + if (snapshot) { + ave->vcbar = readq(coproc + 0x50000); + ave->core_control = readl(coproc + 0x600044); + ave->core_status = readl(coproc + 0x600048); + ave->mailbox_status = readl(mailbox + 0x10); + for (unsigned int i = 0; i < ARRAY_SIZE(ave->scratch); i++) + ave->scratch[i] = readl(mailbox + 0x18 + i * 4); + } + platform_set_drvdata(pdev, ave); + ave->debugfs = debugfs_create_dir(dev_name(dev), NULL); + debugfs_create_file("ave-status", 0400, ave->debugfs, ave, &ave_status_fops); + debugfs_create_file("ave-start-once", 0200, ave->debugfs, ave, &ave_start_fops); + debugfs_create_file("ave-command", 0600, ave->debugfs, ave, &ave_command_fops); + debugfs_create_file("ave-poll", 0200, ave->debugfs, ave, &ave_poll_fops); + debugfs_create_file("ave-arena", 0600, ave->debugfs, ave, &ave_arena_fops); + ave->ipc_blob.data = ave->ipc.cpu; + ave->ipc_blob.size = ave->ipc.size; + debugfs_create_blob("ave-ipc-memory", 0400, ave->debugfs, &ave->ipc_blob); + ave->log_blob.data = ave->log.cpu; + ave->log_blob.size = ave->log.size; + debugfs_create_blob("ave-firmware-log", 0400, ave->debugfs, &ave->log_blob); + ave->config_blob.data = ave->queue.cpu; + ave->config_blob.size = 0x40; + debugfs_create_blob("ave-boot-config", 0400, ave->debugfs, &ave->config_blob); + dev_info(dev, "AVE_DMA_READY ipc=%zu queue=%zu verified_pages=%u firmware_start=0\n", + ave->ipc.size, ave->queue.size, + ave->ipc.verified_pages + ave->queue.verified_pages); + dev_info(dev, "AVE_FIRMWARE_MAPPED pages=%u contents_match_boot=1 register_snapshot=%u\n", + ave->fw[0].verified_pages + ave->fw[1].verified_pages, snapshot); + return 0; + +free_buffers: + ave_free_buffer(ave, &ave->heap); + ave_free_buffer(ave, &ave->log); + ave_free_buffer(ave, &ave->queue); + ave_free_buffer(ave, &ave->ipc); + pm_runtime_put_sync(dev); +release_power: + ave_power_off(ave); + return dev_err_probe(dev, ret, "AVE DMA allocation/verification failed\n"); +} + +static void apple_ave_remove(struct platform_device *pdev) +{ + struct apple_ave *ave = platform_get_drvdata(pdev); + + debugfs_remove_recursive(ave->debugfs); + ave_free_buffer(ave, &ave->heap); + ave_free_buffer(ave, &ave->log); + ave_free_buffer(ave, &ave->queue); + ave_free_buffer(ave, &ave->ipc); + pm_runtime_put_sync(ave->dev); + ave_power_off(ave); + dev_info(ave->dev, "AVE_DMA_RELEASED\n"); +} + +static const struct of_device_id apple_ave_match[] = { + { .compatible = "apple,t8140-ave" }, + { } +}; +MODULE_DEVICE_TABLE(of, apple_ave_match); + +static struct platform_driver apple_ave_driver = { + .probe = apple_ave_probe, + .remove = apple_ave_remove, + .driver = { + .name = "apple-ave", + .of_match_table = apple_ave_match, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(apple_ave_driver); +MODULE_DESCRIPTION("Apple T8140 experimental video encoder transport"); +MODULE_LICENSE("GPL"); From 958b9f2fbd45141883ea0eb19df2d87070a47a94 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:25:58 -0500 Subject: [PATCH 202/631] drm/asahi: Register agx_stats sysfs file on probe and unregister on drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stash the platform device's raw struct device pointer in AsahiDriver::raw_dev and call asahi_sysfs_register() after drm::driver::Registration::new_foreign_owned. The Drop impl calls asahi_sysfs_unregister() which both removes the attribute and clears SNAPSHOT_PTR — the lifetime of the StatsSnapshot is owned by GpuManager::make_mgr, which outlives the AsahiDriver by construction. Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/driver.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 70de8ecc7f21c5..e827ab0972a4ad 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -109,6 +109,16 @@ pub(crate) enum ProbeConfig { unsafe impl Send for AsahiDriver {} unsafe impl Sync for AsahiDriver {} +impl Drop for AsahiDriver { + fn drop(&mut self) { + // SAFETY: `self.raw_dev` was set in `probe` from the platform + // device's `as_raw()`, valid for the lifetime of the bound driver. + if !self.raw_dev.is_null() { + crate::sysfs_exports::unregister(self.raw_dev); + } + } +} + /// Convenience type alias for the DRM device type for this driver. pub(crate) type AsahiDevice = drm::device::Device; pub(crate) type AsahiDevRef = ARef; From c768bcd1e3d61f23367ce0568d79672b5bccce51 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Sun, 4 Oct 2026 08:14:51 -0500 Subject: [PATCH 203/631] drm/asahi: link the sysfs shim into the driver module correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `asahi-y := sysfs.o` made Kbuild treat asahi.o as a composite module whose only part was sysfs.o: the composite link overwrote the object rustc emits for the crate root (both are named asahi.o), so the module — and any kernel built with it — contained the sysfs shim alone and no GPU driver at all. A stock-config boot of such a kernel loses the GPU entirely (observed: no AGX platform driver, apple-dart -16 probe failures on the display darts, dcp missing). Rename the crate root to asahi_core.rs and list both objects: asahi-y := asahi_core.o sysfs.o. The module (or built-in vmlinux) now links the crate and the shim; module metadata keeps name "asahi". Signed-off-by: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> --- drivers/gpu/drm/asahi/{asahi.rs => asahi_core.rs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename drivers/gpu/drm/asahi/{asahi.rs => asahi_core.rs} (100%) diff --git a/drivers/gpu/drm/asahi/asahi.rs b/drivers/gpu/drm/asahi/asahi_core.rs similarity index 100% rename from drivers/gpu/drm/asahi/asahi.rs rename to drivers/gpu/drm/asahi/asahi_core.rs From 40c6323941c61ca9fa2e0cb81ecb7440917c0b70 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:46:12 +1100 Subject: [PATCH 204/631] Merge pull request #168 from joshuaswarren/agent/jw16-agx-stats4 drm/asahi: Export AGX firmware stats to /sys/class/drm/card*/device/agx_stats Original contributor commits are preserved. Conflict resolutions retain the audited shared driver APIs and their integrated review fixes. Contributor credit; Signed-off-by outstanding: - 4189501b643f404faa0ea37fc3bb379226300aa9: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Ryan integrates these contributions under DCO (b); no author signature is forged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 37 +++++++++++++++++++-------------- drivers/gpu/drm/asahi/gpu.rs | 12 ++++++++++- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index e827ab0972a4ad..0457937812f6df 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -109,16 +109,6 @@ pub(crate) enum ProbeConfig { unsafe impl Send for AsahiDriver {} unsafe impl Sync for AsahiDriver {} -impl Drop for AsahiDriver { - fn drop(&mut self) { - // SAFETY: `self.raw_dev` was set in `probe` from the platform - // device's `as_raw()`, valid for the lifetime of the bound driver. - if !self.raw_dev.is_null() { - crate::sysfs_exports::unregister(self.raw_dev); - } - } -} - /// Convenience type alias for the DRM device type for this driver. pub(crate) type AsahiDevice = drm::device::Device; pub(crate) type AsahiDevRef = ARef; @@ -579,9 +569,12 @@ impl AsahiDriver { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let res = regs::Resources::new(pdev)?; - // SAFETY: GPU construction cannot access driver data before init_data(). - // Failed construction releases the allocation without dropping uninitialized data. - let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; + // SAFETY: As in the legacy probe, the data is inaccessible until the fully initialized + // manager is installed below and the DRM device is registered. + // If construction fails, drm::device::Device::release leaves the data + // untouched rather than dropping an uninitialized AsahiData. + let uninit = unsafe { pin_init::pin_init_from_closure::(|_slot| Ok(())) }; + let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiData { @@ -589,8 +582,9 @@ impl AsahiDriver { pdev: pdev.into(), resources: res, }); - // SAFETY: This is the sole initializer; no driver-data users have been published. - if let Err(error) = unsafe { drm.init_data(data) } { + let ptr = &raw const **drm; + // SAFETY: The allocation is pinned and its data slot has not been initialized yet. + if let Err(error) = unsafe { data.__pinned_init(ptr as *mut AsahiData) } { gpu.shutdown(); return Err(error); } @@ -598,7 +592,18 @@ impl AsahiDriver { gpu.shutdown(); return Err(error); } - Ok(Self { drm }) + // G17's firmware channel format has no supported stats decoder yet. + let raw_dev = pdev.as_ref().as_raw(); + let raw_stats = match crate::sysfs_exports::register( + raw_dev, core::ptr::null(), false, + ) { + Ok(handle) => handle, + Err(error) => { + gpu.shutdown(); + return Err(error); + } + }; + Ok(Self { drm, raw_dev, raw_stats, stats: None }) } } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index aec69911c24800..0d7de75042993f 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -554,6 +554,11 @@ pub(crate) struct GpuManager { /// Operations the DRM driver performs on a GPU, independent of its firmware interface. pub(crate) trait Gpu: Send + Sync { + /// Return a snapshot only for firmware families with a supported decoder. + fn stats_snapshot(&self) -> Option> { + None + } + /// Cast as an Any type. fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. @@ -1565,7 +1570,12 @@ impl GpuManager::ver { )?, fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, - stats: channel::StatsChannel::ver::new(dev, alloc_ref)?, + stats: channel::StatsChannel::ver::new( + dev, + alloc_ref, + stats_snap_clone, + u64::from(cfg.base_clock_hz), + )?, }), GFP_KERNEL, )?; From 188e30ce9e78eb6a34ee4ff47b929f33189139b9 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Wed, 30 Sep 2026 12:26:34 -0600 Subject: [PATCH 205/631] drm/apple: accept an EDID longer than its base block announces dcpavserv_copy_edid() rejects an EDID unless the payload is exactly (1 + extension count) blocks. An HDMI 2.x sink that uses the HDMI Forum EDID Extension Override Data Block (HF-EEODB) sets the extension count in the base block to 1 and puts the real count in the HF-EEODB, so the DCP returns more blocks than the base block says. The EDID is dropped with -EIO, the connector has no EDID, and userspace sees no make or model. Accept any whole number of blocks at least as long as the base block announces and let drm_edid, which handles HF-EEODB, decide how many blocks to use. Also bound used_size by the buffer size, which the exact match used to do implicitly. Seen on j314s (M1 Pro) with a Dell S3225QS on the HDMI port: used_size 392, base block announces 1 extension (256 bytes), payload 384 bytes. With this change the connector gets the EDID. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Christian Bartels Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/epic/dpavservep.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index fac08e53d2b378..2c90bebf9185fc 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -198,6 +198,10 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser return drm_edid_alloc(resp->data + DPAVSERV_EDID_LEADING_SIZE, edid_size); } + data_size = le64_to_cpu(resp->used_size); + if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || + data_size > EDID_BUF_SIZE) + return ERR_PTR(-EIO); const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) { @@ -231,6 +235,17 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) } WRITE_ONCE(dcp->dcpavserv.opened, service); } + /* + * An HDMI 2.x sink using the HF-EEODB announces fewer extension blocks + * in the base block than it sends, so the payload may be longer than + * the base block says. Accept any whole number of blocks at least that + * long and leave the block count to drm_edid, which knows about + * HF-EEODB. + */ + num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; + if ((1 + num_blocks) * EDID_BLOCK_SIZE > data_size - EDID_LEADING_DATA_SIZE || + (data_size - EDID_LEADING_DATA_SIZE) % EDID_BLOCK_SIZE) + return ERR_PTR(-EIO); edid = dcpavserv_read_edid(service); spin_lock_irqsave(&dcp->dcpavserv.lock, flags); From 21a925315f0630595b66af4800858edf66665e49 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Sun, 4 Oct 2026 10:00:39 -0600 Subject: [PATCH 206/631] drm/apple: audio: release data_lock when DMA setup fails on open dcp_pcm_open() takes data_lock before dcpaud_init_dma() and returns its error with the lock still held. The next open, the hotplug notifications (dcpaud_connect()/dcpaud_disconnect()) and hw_params/prepare (through dcpaud_connection_up()) all take data_lock and then block forever. Unlock before returning. Signed-off-by: Christian Bartels --- drivers/gpu/drm/apple/audio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 66a83ecea86fac..121c71681d5770 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -251,8 +251,10 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) mutex_lock(&dcpaud->data_lock); ret = dcpaud_init_dma(dcpaud); - if (ret < 0) + if (ret < 0) { + mutex_unlock(&dcpaud->data_lock); return ret; + } if (!dcpaud->connected) { mutex_unlock(&dcpaud->data_lock); From 971ed4e2ccf5cb774497ce13245d36527f7c8f19 Mon Sep 17 00:00:00 2001 From: Christian Bartels Date: Sun, 4 Oct 2026 10:01:03 -0600 Subject: [PATCH 207/631] drm/apple: audio: allow opening the PCM without a sink dcp_pcm_open() fails with -ENXIO while no display is connected, because the format and rate constraints come from the sink's audio elements. PipeWire and PulseAudio probe every PCM of a UCM profile once, when the card appears: they open it, set hw_params and prepare it. If that fails, the profile is marked unsupported for good, and udev "change" events for a card that has already been seen are ignored. So on a Mac booted without a monitor on HDMI or USB-C, a display attached later never gets an audio output until the sound server is restarted. Let the PCM open without a sink, with a plain stereo fallback (2 channels; 32, 44.1 or 48 kHz; S16, S24 or S32), as HDMI PCMs on other platforms do. hw_params and prepare succeed as no-ops for such an open, since alsa-lib prepares straight after hw_params. trigger still fails until a sink is connected, so nothing is sent to a missing display. Once a display is attached, the next open gets the sink's real constraints. Tested on a MacBook Pro 14" M1 Pro (j314s) with a 4K monitor on the built-in HDMI port: booted without a monitor on HDMI, the HDMI output appears when the monitor is plugged in, without restarting WirePlumber, and plays. Audio over DP alt mode on USB-C was not tested. Signed-off-by: Christian Bartels --- drivers/gpu/drm/apple/audio.c | 46 ++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 121c71681d5770..4b8c4d8fdd7840 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -43,6 +43,7 @@ struct dcp_audio { struct snd_jack *jack; struct snd_pcm_substream *substream; unsigned int open_cookie; + bool open_unplugged; struct mutex data_lock; bool dcp_connected; /// dcp status keep for delayed initialization @@ -74,6 +75,32 @@ static const struct snd_pcm_hardware dcp_pcm_hw = { .periods_max = UINT_MAX, }; +/* + * Without a sink the DCP has no audio elements to derive constraints from. + * Offer a plain stereo format so that the PCM can still be opened and + * configured, as HDMI PCMs on other platforms can. Userspace (PipeWire, + * PulseAudio) probes the PCM once when the card appears and drops the + * output for good if that fails, so a monitor attached later would never + * show up. Starting the stream still fails until a sink is connected. + */ +static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { + .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | + SNDRV_PCM_INFO_INTERLEAVED, + .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE | + SNDRV_PCM_FMTBIT_S32_LE, + .rates = SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 | + SNDRV_PCM_RATE_48000, + .rate_min = 32000, + .rate_max = 48000, + .channels_min = 2, + .channels_max = 2, + .buffer_bytes_max = SIZE_MAX, + .period_bytes_min = 4096, + .period_bytes_max = SIZE_MAX, + .periods_min = 2, + .periods_max = UINT_MAX, +}; + static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) { int ret; @@ -256,13 +283,15 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) return ret; } - if (!dcpaud->connected) { - mutex_unlock(&dcpaud->data_lock); - return -ENXIO; - } + dcpaud->open_unplugged = !dcpaud->connected; dcpaud->open_cookie = dcpaud->connection_cookie; mutex_unlock(&dcpaud->data_lock); + if (dcpaud->open_unplugged) { + hw = dcp_pcm_hw_unplugged; + goto refine; + } + ret = dcpaud_read_remote_info(dcpaud); if (ret < 0) return ret; @@ -278,6 +307,7 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, -1); hw = dcp_pcm_hw; +refine: hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED; hw.periods_min = 2; @@ -299,6 +329,7 @@ static int dcp_pcm_close(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; dcpaud->selected_chmap.channels = 0; + dcpaud->open_unplugged = false; return snd_dmaengine_pcm_close(substream); } @@ -321,6 +352,9 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); int ret; + if (dcpaud->open_unplugged) + return 0; + if (!dcpaud_connection_up(dcpaud)) return -ENXIO; @@ -362,6 +396,10 @@ static int dcp_pcm_prepare(struct snd_pcm_substream *substream) { struct dcp_audio *dcpaud = substream->pcm->private_data; + /* alsa-lib prepares right after hw_params; see dcp_pcm_hw_unplugged */ + if (dcpaud->open_unplugged) + return 0; + if (!dcpaud_connection_up(dcpaud)) return -ENXIO; From 99e70a103bcb279ffa3b137294b71cd98fe8a7d1 Mon Sep 17 00:00:00 2001 From: Joshua Warren <816217+joshuaswarren@users.noreply.github.com> Date: Wed, 7 Oct 2026 09:49:28 -0500 Subject: [PATCH 208/631] arm64: asahi.config: build the RDMA core and the soft RDMA drivers as modules The Apple Silicon kernel has CONFIG_INFINIBAND unset, so no node has an RDMA device, ib_uverbs or the rdma tools' kernel side. Build the RDMA core, its userspace verbs and management interfaces, and the two software providers (rxe, siw) as modules. With these, rdma-core and perftest work on any Mac with a network interface: `modprobe rdma_rxe; rdma link add rxe0 type rxe netdev ` gives a verbs device that needs no RDMA hardware. Nothing loads unless a user asks for it. All five are =m. In `make defconfig asahi.config` they resolve with their dependencies (NET_UDP_TUNNEL=m, IRQ_POLL=y, DIMLIB=y, DMA_SHARED_BUFFER=y, INFINIBAND_VIRT_DMA=y). --- arch/arm64/configs/asahi.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 1e05c59146c1fa..4691dd087748e7 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -101,3 +101,8 @@ CONFIG_APPLE_M1_CPU_PMU=y CONFIG_NVMEM_APPLE_EFUSES=m CONFIG_NVMEM_APPLE_SPMI=m CONFIG_MUX_APPLE_DPXBAR=m +CONFIG_INFINIBAND=m +CONFIG_INFINIBAND_USER_ACCESS=m +CONFIG_INFINIBAND_USER_MAD=m +CONFIG_RDMA_RXE=m +CONFIG_RDMA_SIW=m From e858be24c2c5f50750d0b1a5caa86f4615cda9f6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:40:33 +1100 Subject: [PATCH 209/631] dt-bindings: apple: Describe T8140 idle and display crossbar controllers Document the T8140 CPU idle selector and the existing Apple display crossbar interface. T8140 uses only the WFI clock-gated state; CPU and cluster power-down are not supported. The crossbar compatible follows the controller type reported by the ADT and the existing Asahi driver. Keep CS42L42 supply phandles mandatory while permitting the established CS42L83 board model with platform-managed supplies. Existing public Apple board descriptions already use that model. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/cpu/apple,t8140-cpuidle.yaml | 33 +++++++++++++ .../bindings/mux/apple,display-crossbar.yaml | 47 +++++++++++++++++++ .../bindings/sound/cirrus,cs42l42.yaml | 19 ++++++-- 3 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml create mode 100644 Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml diff --git a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml new file mode 100644 index 00000000000000..aa27ec55309c8b --- /dev/null +++ b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/cpu/apple,t8140-cpuidle.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 CPU idle controller + +maintainers: + - Ryan Murray + +description: + Selects the Apple CPU idle driver on T8140 systems. Only the WFI + clock-gated state is supported; CPU and cluster power-down states + are not enabled on this SoC. + +properties: + $nodename: + const: cpuidle + + compatible: + const: apple,t8140-cpuidle + +required: + - compatible + +additionalProperties: false + +examples: + - | + cpuidle { + compatible = "apple,t8140-cpuidle"; + }; diff --git a/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml new file mode 100644 index 00000000000000..eb80cf382a1880 --- /dev/null +++ b/Documentation/devicetree/bindings/mux/apple,display-crossbar.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mux/apple,display-crossbar.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple display crossbar + +maintainers: + - Sven Peter + +description: + Routes display pipelines to the DisplayPort transmitters and inputs + of an Apple Type-C PHY. Each crossbar exposes three mux controls, + selected by the consumer's mux specifier. + +allOf: + - $ref: mux-controller.yaml# + +properties: + compatible: + enum: + - apple,t8103-display-crossbar + - apple,t8112-display-crossbar + - apple,t6000-display-crossbar + - apple,t6020-display-crossbar + + reg: + maxItems: 1 + + '#mux-control-cells': + const: 1 + +required: + - compatible + - reg + - '#mux-control-cells' + +unevaluatedProperties: false + +examples: + - | + mux-controller@b04c000 { + compatible = "apple,t6020-display-crossbar"; + reg = <0xb04c000 0x4000>; + #mux-control-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index 28a5303cf72046..1d839e28c0f8a7 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -14,6 +14,8 @@ description: It provides a high-dynamic range, stereo DAC for audio playback and a mono high-dynamic-range ADC for audio capture. There is an integrated headset detection block. + CS42L83 systems may have platform-managed supplies without regulator + phandles, as on Apple Silicon boards. properties: compatible: @@ -192,11 +194,18 @@ properties: required: - compatible - reg - - VP-supply - - VCP-supply - - VD_FILT-supply - - VL-supply - - VA-supply +allOf: + - if: + properties: + compatible: + const: cirrus,cs42l42 + then: + required: + - VP-supply + - VCP-supply + - VD_FILT-supply + - VL-supply + - VA-supply additionalProperties: false From 9914016aed8e2223c6048a05d1c8a19579dc8f4f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:15:31 +1100 Subject: [PATCH 210/631] dt-bindings: mailbox: Describe the T8140 GPU ASCWrap interface Describe the existing GPU coprocessor mailbox interface, with wrapper and vector-base resources, two FIFO interrupts and firmware role names. The public T8140 GPU device tree already supplies these properties; without a binding, validation cannot decode the firmware-role string. The description follows the public GPU interface and J700 ADT resource layout. This does not change the GPU device tree or driver behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/mailbox/apple,ascwrap-v6.yaml | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml diff --git a/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml new file mode 100644 index 00000000000000..4798b62bd84300 --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/apple,ascwrap-v6.yaml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mailbox/apple,ascwrap-v6.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 ASCWrap v6 mailbox + +maintainers: + - Ryan Murray + +description: + Mailbox interface of the T8140 GPU coprocessors. The wrapper resource + contains the coprocessor control registers and mailbox FIFOs. A separate + resource provides the coprocessor vector-base register. The firmware role + identifies which of the two GPU firmware instances uses the mailbox. + +properties: + compatible: + const: apple,t8140-ascwrap-v6 + + reg: + items: + - description: ASC wrapper registers + - description: Coprocessor vector-base register + + reg-names: + items: + - const: wrapper + - const: iop-vbar + + interrupts: + items: + - description: Send FIFO empty interrupt + - description: Receive FIFO not-empty interrupt + + interrupt-names: + items: + - const: send-empty + - const: recv-not-empty + + '#mbox-cells': + const: 0 + + power-domains: + maxItems: 1 + + apple,firmware-role: + enum: [GFX, GFX1] + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - '#mbox-cells' + - apple,firmware-role + +additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + mailbox@482600000 { + compatible = "apple,t8140-ascwrap-v6"; + reg = <0x4 0x82600000 0x0 0x88000>, + <0x4 0x82050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + #mbox-cells = <0>; + }; + }; From e9526ccada14c443160826b4dc90fb404cae5777 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 13:59:04 +1100 Subject: [PATCH 211/631] dt-bindings: spi: describe the T8140 QSPIMC controller Describe the SPI-memory controller's register aperture, clock, interrupt and power domain. QSPIMC uses a packed command FIFO and differs from SPIMC. Originally written by DjDeveloperr; DjDeveloperr's Signed-off-by is outstanding. Port his publicly available controller binding. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/spi/apple,qspi.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 Documentation/devicetree/bindings/spi/apple,qspi.yaml diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml new file mode 100644 index 00000000000000..5f526fe3a79701 --- /dev/null +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/spi/apple,qspi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple ARM SoC QSPIMC controller + +maintainers: + - Asahi Linux + +description: | + Apple QSPIMC is a SPI-memory-only controller with a packed command FIFO. + It is distinct from the ordinary Apple SPIMC controller despite serving a + similar set of SPI NOR devices. + +allOf: + - $ref: spi-controller.yaml# + +properties: + compatible: + const: apple,t8140-qspi + + reg: + maxItems: 1 + + clocks: + maxItems: 1 + + interrupts: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - clocks + - interrupts + +unevaluatedProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + spi@385110000 { + compatible = "apple,t8140-qspi"; + reg = <0x3 0x85110000 0x0 0x4800>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clk_200m>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; From efd7e9a8cc053612c40d34fcddb20bf6e10ea034 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:00:14 +1100 Subject: [PATCH 212/631] spi: apple-qspi: service transfer FIFOs with bounded microsecond polling T8140 can latch command completion without delivering the wired IRQ, and the RX FIFO threshold interrupt does not reliably fire. Waiting a jiffy between service attempts limits Linux reads to one FIFO per tick. The observed NVRAM read took 55 seconds at about 34 kB/s. Use the polling helper with a microsecond sleep interval to drain and refill the FIFOs. Preserve the one-second deadline and the IRQ completion path. Serialize polling FIFO service with the interrupt handler. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 65 ++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 45b3566ae5a9f7..beead37df387db 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -429,47 +429,40 @@ static int apple_qspi_set_frequency(struct apple_qspi *qspi, return 0; } -static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +static bool apple_qspi_poll_complete(struct apple_qspi *qspi) { - unsigned long deadline = jiffies + - msecs_to_jiffies(APPLE_QSPI_TIMEOUT_MS); unsigned long flags; - u32 p_status, c_status; + u32 queue_irqs, command_irqs; - do { - if (try_wait_for_completion(&qspi->done)) - return true; - - /* - * TX-only commands do not reliably raise the wired interrupt on - * T8140, although the completion bit is latched in the controller. - * Poll both status banks as a fallback and service the FIFOs so a - * page program larger than the TX FIFO can continue making progress. - */ - p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); - c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); - if (p_status) - apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); - if (c_status) - apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); - - spin_lock_irqsave(&qspi->lock, flags); - if (qspi->active) { - apple_qspi_drain_rx(qspi); - apple_qspi_fill_tx(qspi); - } - spin_unlock_irqrestore(&qspi->lock, flags); - - if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) - return true; - if (time_after_eq(jiffies, deadline)) - break; + spin_lock_irqsave(&qspi->lock, flags); + queue_irqs = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + command_irqs = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, queue_irqs); + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, command_irqs); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + } + spin_unlock_irqrestore(&qspi->lock, flags); - if (wait_for_completion_timeout(&qspi->done, 1)) - return true; - } while (time_before(jiffies, deadline)); + return (command_irqs & APPLE_QSPI_C_IRQ_COMPLETE) || + try_wait_for_completion(&qspi->done); +} - return false; +static bool apple_qspi_wait_complete(struct apple_qspi *qspi) +{ + bool finished; + + /* + * T8140 can latch completion without delivering the wired IRQ, and + * the RX threshold IRQ does not reliably drain the FIFO. A jiffy + * between polls throttles reads to one FIFO per scheduler tick. + * Service the FIFOs at microsecond intervals while retaining the + * bounded transfer deadline and IRQ completion path. + */ + return !read_poll_timeout(apple_qspi_poll_complete, finished, finished, + 20, APPLE_QSPI_TIMEOUT_MS * USEC_PER_MSEC, + false, qspi); } static int apple_qspi_transfer_op(struct spi_mem *mem, From 465b6640fb365f17018e409e8dd0af1aa0020fea Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:12:21 +1100 Subject: [PATCH 213/631] MAINTAINERS: identify Aurora as the QSPIMC contact Point the QSPIMC maintenance entry and binding at Ryan Murray and the Aurora Silicon project site. Remove the mailing-list association from this controller's new maintenance entry. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/spi/apple,qspi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/spi/apple,qspi.yaml b/Documentation/devicetree/bindings/spi/apple,qspi.yaml index 5f526fe3a79701..4d33549e7bf396 100644 --- a/Documentation/devicetree/bindings/spi/apple,qspi.yaml +++ b/Documentation/devicetree/bindings/spi/apple,qspi.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Apple ARM SoC QSPIMC controller maintainers: - - Asahi Linux + - Ryan Murray description: | Apple QSPIMC is a SPI-memory-only controller with a packed command FIFO. From 5d8eb60b5f923fbb33f730ee3e9b5cd65d5aa6e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 16:46:49 +1100 Subject: [PATCH 214/631] spi: apple-qspi: reject clocks below the divider range Clamping a too-large divider to 256 makes the bus faster than requested. For a 200 MHz input, a 100 kHz request would run at about 781 kHz. Reject unrepresentable slow transfers instead of violating the device limit. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple-qspi.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index beead37df387db..2c563fc226c866 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -424,7 +424,10 @@ static int apple_qspi_set_frequency(struct apple_qspi *qspi, return -EINVAL; divisor = DIV_ROUND_UP(input, requested); - divisor = clamp_t(u32, divisor, 2, 256); + /* Never exceed the requested clock when it is below our range. */ + if (divisor > 256) + return -EINVAL; + divisor = max_t(u32, divisor, 2); apple_qspi_c_write(qspi, APPLE_QSPI_C_CLKDIV, divisor - 1); return 0; } From bb4225ed41228da2b60a852156292846d41339bd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:31:38 +1100 Subject: [PATCH 215/631] Bluetooth: mt7932: support modules and system suspend The MT7932 PCIe transport cannot be selected with modular Bluetooth or system sleep enabled. Make the existing transport tristate and coordinate firmware quiescence with HCI suspend before PCI power transitions. Wait for the firmware's system-sleep acknowledgment before completing suspend. Restore the transport before resuming HCI, and undo the sleep request if suspend fails. Keep ring publication stopped while quiesced and preserve the DMA arena across sleep. Invalid notifications and failed restoration close the transport instead of retrying an uncertain state. Unregister HCI, quiesce firmware and synchronize transport work and interrupts on removal. Release device-visible storage only after bus mastering is disabled and pending PCI transactions drain; retain it if that cannot be confirmed. Hibernation, firmware recovery and warm reprobe remain unsupported. This extends the existing MT7932 transport and preserves its contributor notices and firmware/calibration path. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 67 ++++--------------- 1 file changed, 14 insertions(+), 53 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 88773c6a89a07b..a36998f1010e9b 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -25,21 +25,9 @@ it. Public m1n1 must populate the J700 ``wifi0`` endpoint's own The original hardware-tested configuration included ``CONFIG_MT7932_FULLMAC=m``, ``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. -The shared build supports ``CONFIG_PCIE_APPLE=y`` or ``m``, -``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``, ``CONFIG_PCIEASPM=y``, -``CONFIG_MT7932_FULLMAC=m`` and ``CONFIG_BT_MTK7932_PCIE=m`` with modular -Bluetooth/rfkill and sleep/kexec enabled. The active J700 bootstrap refuses sleep -and kexec at runtime while its arena is retained; other Macs keep their normal -behavior. Firmware loading is selected by both radio drivers. This shared -build still requires physical Neo qualification. - -``CONFIG_CRYPTO_CMAC=m`` is supported with the modular Bluetooth core. Generic -Bluetooth SMP allocates ``cmac(aes)`` through the crypto API. Install the crypto -modules and preserve algorithm autoload; if Bluetooth starts in the initramfs, -include its crypto providers there. With ``CONFIG_CRYPTO_AES=y``, this kernel -also registers its AES CMAC implementation when CMAC is configured as a module. -The earlier tested ``CONFIG_CRYPTO_CMAC=y`` is not a Wi-Fi driver requirement. - +The Bluetooth transport can be built with modular Bluetooth and system +sleep. The separately selected bootstrap provider has its own lifecycle +restrictions. Kexec is excluded while either retained experiment is enabled. Use the ordinary cfg80211 regulatory database and applicable country policy. Kernel country 00 uses firmware XZ and its mandatory ``world-XZ.bin`` package. An explicit country requires its own package; there is no world fallback. @@ -158,44 +146,17 @@ samples and response hashes, omitting network names, addresses and credentials. Limitations ----------- -* Wi-Fi station operation is limited to 2.4 GHz channels 1--13 and 5 GHz - channels 36, 40, 44 and 48, subject to the applicable regulatory rules. - 5 GHz supports 20/40/80 MHz channel layouts within this range. Channels - 149--165 and 6 GHz are not exposed. Authentication supports open networks - and WPA2-PSK with CCMP; WPA3/SAE, 802.1X, required MFP and other ciphers - are unsupported. AP/P2P and roaming are unsupported. -* Sleep is not supported on the MacBook Neo yet. With radio support active - (the default), the retained bootstrap makes the kernel refuse suspend: - lid close and ``systemctl suspend`` do not put it to sleep. Shut down before - putting it in a bag. Booting with ``pcie_apple_piodma_diag.enumerate=0`` - removes this refusal and disables Wi-Fi and Bluetooth; sleep and wake are - still unproven on this hardware without the radios. -* PCI bootstrap memory remains retained until external reset. Sleep and kexec - are refused while it is retained; they are not hardware-qualified on the Neo. - Standard PCI sysfs removal of the root and radio functions is refused; - arbitrary controller/IOMMU teardown and memory reuse remain unqualified. - Wi-Fi shutdown joins host producers before disabling bus mastering and - retains device-visible memory until reset. - Userspace PCI ``reset`` and ``reset_subordinate`` are also refused, and - native slot hot-plug is disabled for this host. Per-link sysfs ASPM/Clock PM - writes can lift the power-state restrictions; they are reapplied only at - probe. Do not enable those link power states while the radios are active. - These guards protect against accidental teardown, not privileged access. - Root can still reset through raw PCI configuration writes (SBR, FLR or Link - Disable), override link power states, or unbind the root port's ``pcieport`` - driver. None of these actions is supported while the retained host is active. -* The Wi-Fi module pins itself before publishing its first DMA ring and cannot - be unloaded normally while DMA is retained. A checked reset during probe - failure may release that pin; a failed reset retains it. Never unload or - reprobe Wi-Fi while Bluetooth is active. Shutdown quiesces host producers - without freeing retained memory; use a full shutdown and external reset - after a failed or uncertain admission. -* Bluetooth PCI removal/quiescence is incomplete. Its software queue limit - does not provide HCI backpressure; saturation can drop an accounted frame. - Both require correction before production use. An activated Bluetooth module - cannot be unloaded normally; forced PCI removal only retires software - callbacks, including the threaded IRQ and MSI vectors, and does not establish - DMA quiescence or release retained ownership. +* PCI bootstrap memory remains retained until external reset. Controller + removal, memory reuse, suspend and kexec are unqualified. +* Bluetooth system sleep uses a bounded firmware quiesce/restore handshake + around HCI suspend/resume, preserving its DMA arena. Hibernation is rejected. + Removal first quiesces firmware, then stops work and interrupts, disables + bus mastering and drains pending + PCI transactions before freeing DMA. An unconfirmed drain retains storage + until reset. Firmware recovery and warm reprobe are unsupported; use a fresh + cold boot after removing or faulting an active transport. +* The Bluetooth software queue limit does not provide HCI backpressure; + saturation can drop an accounted frame. * Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package generation, roaming and long-duration reliability are unqualified. * SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are From 4e285c07d857435bd797f50425ed0ea436dbdbd1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:44:42 +1100 Subject: [PATCH 216/631] PCI: apple: preserve the retained bootstrap across suspend-to-idle sleep The retained PIODMA supplier prevents the radio configuration from being built with system sleep. Permit suspend-to-idle once the single bootstrap has completed and the command, FIFO and owned storage validate as idle. Reject sleep from a failed or incomplete bootstrap. Restore the existing register-bank configuration after its power domain resumes, while preserving the arena and never resubmitting its descriptor. Reject hibernation and deeper suspend, which cannot preserve the live endpoint and firmware ownership contract. Keep the supplier and PCI host built in: published DMA and supplier IRQ ownership still lack a removal contract. This retains the existing link boundary and requires no module export or parallel provider. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 90 ++++--------------- drivers/pci/controller/Kconfig | 12 +-- .../pci/controller/pcie-apple-piodma-diag.c | 65 ++++++++++++-- 3 files changed, 84 insertions(+), 83 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index ec0d67dfc594c7..6f41484ccf84a6 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -7,78 +7,24 @@ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. The root bridge forwards their BARs and DART translates endpoint DMA. -This is a bring-up implementation, not general T8140 PCIe support. Build -``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``; the Apple PCIe host driver may be a module. -The bootstrap enables enumeration by default only on machines matching both -``apple,j700`` and ``apple,t8140``, through the matching diagnostic and host -device-tree nodes. No kernel argument is required. To leave the root-only -diagnostic path closed, pass ``pcie_apple_piodma_diag.enumerate=0``. -The read-only ``enumerate`` parameter reports the configured admission request, -not whether the current machine is a Neo or its radios were admitted. Its -default value is also visible on other Macs; the machine and DT checks decide -whether it has any effect. - -The shared kernel can keep suspend, hibernation and kexec support enabled. -Before the host creates PCI children, a short admission critical section -acquires a kexec interlock and registers a PM notifier that refuses sleep -transitions. Its device prepare callback provides a second veto. This also covers -``/dev/snapshot`` files opened before the supplier probes: later image, restore -and suspend ioctls enter device PM without repeating the prepare notifier. -A kexec interlock refuses loading or executing a replacement kernel, -including an already loaded crash kernel. Image unloading remains permitted. -These interlocks apply only to the active J700 bootstrap; other machines do not -acquire them. Merely probing its supplier does not block transitions. The device -prepare callback also checks this guarded pre-publication window. Admission -contention may defer only before PCI children exist; a later failure is a hard -error. Pre-child contention schedules a delayed attachment of this matched -host, so a long-running kexec loader does not require another driver to bind -before admission can retry. Only renewed contention queues another attempt; -successful admission and hardware publication end these retries. Shutdown -blocks late admission. A failure before memory publication releases the guards. Once the -bootstrap retains memory, the protection persists even if enumeration or either -radio's firmware loading fails. Thus the block applies on every default J700 -boot which reaches retained radio admission, regardless of installed firmware. -Failed or uncertain bootstrap state requires a full external hardware reset; -there is no command retry. - -Sleep is not supported on the MacBook Neo yet. With radio support active -(the default), the kernel refuses suspend: lid close and ``systemctl suspend`` -do not put it to sleep. Shut down before putting it in a bag. Booting with -``pcie_apple_piodma_diag.enumerate=0`` removes this refusal and disables Wi-Fi -and Bluetooth; sleep and wake remain unproven on this hardware without radios. - -Before either radio driver can bind, the host disables ASPM, all L1 substates -and Clock PM on the validated radio endpoints' shared PCIe link through the -PCI core. ``CONFIG_PCIEASPM=y`` is required. The global ASPM policy is unchanged; -neither a performance-policy boot argument nor a sysfs policy write is needed. -Booting with ``pcie_aspm=off`` prevents the PCI core from applying this restriction -and radio admission fails closed. - -The supplier publishes radio admission only after both functions pass the -host's identity, cold-state, memory-window and link-power checks. Both radio -probes require that completed admission before accessing PCI or MMIO state; -the host's downstream device-enable callback enforces the same gate for other -drivers. This prevents a later generic rescan from enabling cached children of -a failed admission. Each radio probe reapplies the link-power restriction, so -a recreated ASPM link state cannot inherit an incompatible global policy. -Privileged writes to the per-link PCI sysfs ASPM or Clock PM attributes can -override the restriction; it is reapplied only at probe, not continuously. -Do not enable those link power states while the Neo radios are admitted. - -The PIODMA arena remains allocated until external reset. Standard PCI sysfs -``remove`` writes for the root port and radio functions return ``-EOPNOTSUPP`` -while the host has its retained supplier; the removal attributes remain present. -The same hierarchy refuses userspace ``reset`` and ``reset_subordinate`` -writes, and its host does not hand native PCIe or SHPC slot hot-plug control -to Linux. These restrictions leave other PCI hosts unchanged. -VFIO assignment is refused on this host because it exposes reset operations -outside PCI sysfs. Automatic native AER/DPC port recovery is also withheld: -its bus or link reset has no qualified retained-memory contract here. A hardware -fault requires external reset rather than guessed recovery. -The host and activated Wi-Fi/Bluetooth modules are pinned until reset. Arbitrary -platform-device removal, IOMMU teardown, forced module removal, memory reuse -and arbitrary downstream devices remain unqualified. Do not remove the -controller or its IOMMUs while this experiment is active. +This is an opt-in bring-up implementation, not general T8140 PCIe support. +Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass +``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the +root-only diagnostic path continues to reject downstream configuration access. +Build the host and its DART in as well: the supplier retains published DMA +and has no removal contract. Keeping the supplier and host built in avoids +a module dependency on a provider whose lifetime cannot end at unload. + +Suspend-to-idle suspend retains the arena after checking the completed request, +idle FIFO and command state. Resume restores the existing configuration +banks without issuing a second bootstrap. Hibernation, deeper suspend and +kexec are unsupported after activation. Failed or uncertain bootstrap state +requires a full external hardware reset; there is no command retry. + +The PIODMA arena remains allocated until external reset. Controller removal, +teardown, memory reuse and arbitrary downstream devices are unqualified. +Fresh hardware qualification must check both radio functions across sleep. +Do not remove the controller or its IOMMU while this experiment is active. These lifecycle contracts require further work before production support. They protect against accidental teardown and are not a security boundary. Root can still reset through raw configuration writes, including secondary-bus diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 830c2c24a68ec0..9b4839a2d07d36 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -64,12 +64,12 @@ config PCIE_APPLE_PIODMA_DIAG help Build the experimental SID17 retained bootstrap provider. It submits one fixed request, validates typed ECAM and permits enumeration of - the two Neo functions by default on J700/T8140 only. The parameter - pcie_apple_piodma_diag.enumerate=0 disables the bootstrap. - Its arena remains owned until a full reset. While that memory is - retained, suspend, hibernate and kexec are refused at runtime on - the Neo; other machines retain their normal lifecycle support. - The provider is built-in even when the Apple PCIe host is a module. + the two Neo functions only with the explicit enumerate=1 parameter. + The host and supplier must both be built in: supplier removal cannot + release a published DMA arena safely. Suspend-to-idle sleep retains that + arena and restores the configuration without repeating a request. + Hibernation and deeper suspend are rejected after activation. + This does not qualify memory reuse or arbitrary downstream devices. config PCI_VERSATILE bool "ARM Versatile PB PCI controller" diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 62cd323a080e04..b41a462456aaa1 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -99,6 +99,7 @@ struct apple_piodma_diag { u64 pointer_prefix; u8 secondary_bus; bool retained; + bool ready; bool guarded; bool radio_admitted; struct notifier_block pm_notifier; @@ -733,12 +734,13 @@ int apple_piodma_bootstrap_prime(struct device *supplier, struct pci_dev *root) ret = apple_piodma_diag_initialize(diag); if (!ret) ret = apple_piodma_diag_submit(diag, 0); - if (ret) - dev_err(supplier, "bootstrap failure=%d; %s; no retry\n", ret, - diag->retained ? "retained ownership" : "no hardware publication"); - else + if (ret) { + dev_err(supplier, "bootstrap failure=%d; retained ownership; no retry\n", ret); + } else { + WRITE_ONCE(diag->ready, true); dev_info(supplier, "BOOTSTRAP_VALIDATED word=%#x; one retained slot\n", diag->slots[0].result); + } return ret; } EXPORT_SYMBOL_GPL(apple_piodma_bootstrap_prime); @@ -857,6 +859,59 @@ static void apple_piodma_diag_shutdown(struct platform_device *pdev) dev_warn(&pdev->dev, "diagnostic arena retained; require full hardware reset\n"); } +/* No new request is permitted once the host has admitted its endpoints. */ +static int apple_piodma_diag_sleep_check(struct apple_piodma_diag *diag) +{ + if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + return -EIO; + if (readl(diag->engine + 0x54) & BIT(0)) + return -EBUSY; + if ((readl(diag->engine + 0x0c) & PIODMA_FIFO_MASK) != 16) + return -EBUSY; + return apple_piodma_diag_check_slots(diag); +} + +static int apple_piodma_diag_suspend_noirq(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + + if (!diag->retained) + return 0; + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + return -EOPNOTSUPP; + return apple_piodma_diag_sleep_check(diag); +} + +static int apple_piodma_diag_resume_noirq(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + int ret; + + if (!diag->retained) + return 0; + if (!READ_ONCE(diag->ready) || READ_ONCE(diag->irq_fault)) + return -EIO; + /* Genpd may gate this island; restore configuration, never a command. */ + ret = apple_piodma_diag_initialize(diag); + if (ret) + return ret; + return apple_piodma_diag_sleep_check(diag); +} + +static int apple_piodma_diag_freeze(struct device *dev) +{ + struct apple_piodma_diag *diag = dev_get_drvdata(dev); + + return diag->retained ? -EOPNOTSUPP : 0; +} + +static const struct dev_pm_ops apple_piodma_diag_pm_ops = { + .suspend_noirq = apple_piodma_diag_suspend_noirq, + .resume_noirq = apple_piodma_diag_resume_noirq, + .freeze = apple_piodma_diag_freeze, + .poweroff = apple_piodma_diag_freeze, +}; + static const struct of_device_id apple_piodma_diag_match[] = { { .compatible = "apple,t8140-piodma-diagnostic" }, { } @@ -868,7 +923,7 @@ static struct platform_driver apple_piodma_diag_driver = { .driver = { .name = "apple-piodma-diag", .of_match_table = apple_piodma_diag_match, - .pm = pm_sleep_ptr(&apple_piodma_pm_ops), + .pm = pm_sleep_ptr(&apple_piodma_diag_pm_ops), .suppress_bind_attrs = true, }, }; From abb4fd5c73ce8b279829dd2eca951bd083abb70b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:54:44 +1100 Subject: [PATCH 217/631] arm64: configs: select the J700 radio transport and bootstrap parents The retained PIODMA bootstrap requires a built-in Apple PCI host, whose DART parent must also be built in. Select those providers in the Asahi fragment and request the existing Bluetooth transport as a module. Add a J700 radio fragment that selects the fullmac and modular Bluetooth paths, while preserving system sleep and disabling the kexec selectors that cannot preserve live firmware DMA ownership. It is applied after the general Asahi fragment. Runtime enumeration and Bluetooth admission remain opt-in and firmware/calibration remain external packages. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/asahi.config | 3 ++- arch/arm64/configs/j700-radio.config | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/configs/j700-radio.config diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 4691dd087748e7..e332098f244ca0 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -11,6 +11,7 @@ CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_BT_MTK7932_PCIE=m +CONFIG_PCIE_APPLE=y CONFIG_PCIE_APPLE=m CONFIG_PCIE_APPLE_PIODMA_DIAG=y CONFIG_PCIEASPM=y @@ -72,7 +73,7 @@ CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m -CONFIG_APPLE_DART=m +CONFIG_APPLE_DART=y CONFIG_APPLE_DOCKCHANNEL=m CONFIG_SERIAL_APPLE_DOCKCHANNEL=y CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config new file mode 100644 index 00000000000000..87b09a42d3cc68 --- /dev/null +++ b/arch/arm64/configs/j700-radio.config @@ -0,0 +1,19 @@ +# J700 retained radio bootstrap; merge after defconfig and asahi.config. +# System sleep is retained; active radio firmware cannot cross kexec. +# Firmware/calibration and runtime admission follow the MT7932 documentation. +# CONFIG_KEXEC is not set +# CONFIG_KEXEC_FILE is not set +# CONFIG_KEXEC_HANDOVER is not set +CONFIG_PCI=y +CONFIG_PCIE_APPLE=y +CONFIG_APPLE_DART=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_WLAN=y +CONFIG_WLAN_VENDOR_MEDIATEK=y +CONFIG_MT7932_FULLMAC=m +CONFIG_BT=m +CONFIG_BT_BREDR=y +CONFIG_BT_LE=y +CONFIG_BT_MTK7932_PCIE=m +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_CMAC=y From 29dd290ae3060e9f725a888bbcc7585d9583356f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 14:22:12 +1100 Subject: [PATCH 218/631] pmdomain: apple: pmp-report: expose T8140 fast-die effort Describe the T8140 report layout from the Apple device tree and expose its fast-die effort through the existing PMP report mapping. A stopped PMP reads zero, so check the running bit before accepting an effort. Report entry 0x1c1 contains byte 1 for the P cluster, observed under Linux load, and byte 0 for E, assigned from ADT lane order. The E byte has not yet been observed changing. Do not read entry 0x1c2 or scan the unpopulated part of the report aperture. Expose read-only debugfs status and lane effort values for diagnostics. Port the original Linux-derived implementation by Ryan Murray. Retain the Asahi report driver and use it as the sole owner of the mapping. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-report.c | 113 +++++++++++++++++++++++++++ include/linux/soc/apple/pmp-report.h | 9 +++ 2 files changed, 122 insertions(+) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index c9acf48bdcb0dc..ad4b590cd84923 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -34,6 +38,16 @@ #define PMP_REPORT_READY 0x1 +/** + * struct apple_pmp_report_offsets - per-SoC layout of the report region + * @tgt_read: power state request, read side + * @tgt_write: power state request, write side + * @actual: power state acknowledgment + * @status: PMP status; bit 0 is set while the PMP is running + * @fast_die: 32-bit word holding the fast die-temperature effort of each + * CPU cluster, one byte per lane; 0 if the SoC has none + * @fast_die_lanes: number of lanes at @fast_die, at most 4 + */ struct apple_pmp_report_offsets { u32 tgt_read; u32 tgt_write; @@ -48,6 +62,8 @@ struct apple_pmp_report_offsets { bool temps; /* With starts_pmp: soc-device record i holds device id i + 1. */ bool devices_ordered; + u32 fast_die; + u32 fast_die_lanes; }; struct apple_pmp_report { @@ -601,6 +617,7 @@ static const struct { { "tp000m", "Tp000m", 0x6937c }, { "tp001m", "Tp001m", 0x69420 }, { "tp002m", "Tp002m", 0x694c4 }, + struct list_head list; }; #define T6030_PMP_TEMP_CHANNELS (1 + ARRAY_SIZE(apple_pmp_t6030_temps)) @@ -1085,6 +1102,85 @@ int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int t return ret; } EXPORT_SYMBOL_GPL(apple_pmp_report_wait_supplier_ready); +/* + * Bound report regions, for apple_pmp_report_fast_die_effort(). Regions are + * added once they are set up and never removed: the driver cannot be unbound. + */ +static LIST_HEAD(apple_pmp_reports); +static DEFINE_MUTEX(apple_pmp_reports_lock); +static struct dentry *apple_pmp_report_debugfs; + +/** + * apple_pmp_report_fast_die_effort() - read the fast die-temperature effort + * of a CPU cluster + * @np: device tree node of the report region + * @lane: lane of the cluster in the fast die-temperature report + * @effort: set to the effort on success + * + * The PMP of some SoCs runs a control loop on its own die-temperature sensors + * and publishes an 8-bit effort for each CPU cluster: 0 requests no limit, + * and the effort rises towards 255 as the die exceeds the firmware's target. + * The effort is only valid while the PMP reports that it is running; a + * stopped PMP reads 0. + * + * Context: Any context. + * Return: 0 on success, -EPROBE_DEFER if the region is not bound (yet), + * -EOPNOTSUPP if the SoC's PMP publishes no such effort, -EINVAL if @lane is + * out of range, or -EAGAIN if the PMP does not report that it is running. + */ +int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort) +{ + const struct apple_pmp_report_offsets *offs; + const struct apple_pmp_report *rep; + int ret = -EPROBE_DEFER; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node != np) + continue; + + offs = rep->offsets; + if (!offs->fast_die) { + ret = -EOPNOTSUPP; + } else if (lane >= offs->fast_die_lanes) { + ret = -EINVAL; + } else if (!(readl(rep->base + offs->status) & PMP_REPORT_READY)) { + ret = -EAGAIN; + } else { + *effort = readl(rep->base + offs->fast_die) >> + (lane * BITS_PER_BYTE); + ret = 0; + } + break; + } + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(apple_pmp_report_fast_die_effort); + +/* Read only the documented status and fast-die entries, never the aperture. */ +static int apple_pmp_report_status_show(struct seq_file *s, void *unused) +{ + struct apple_pmp_report *rep = s->private; + const struct apple_pmp_report_offsets *offs = rep->offsets; + u32 status = readl(rep->base + offs->status); + unsigned int lane; + u32 effort; + + seq_printf(s, "running: %u\n", !!(status & PMP_REPORT_READY)); + if (!(status & PMP_REPORT_READY) || !offs->fast_die) + return 0; + + effort = readl(rep->base + offs->fast_die); + for (lane = 0; lane < offs->fast_die_lanes; lane++) + seq_printf(s, "lane%u: %u\n", lane, + (effort >> (lane * BITS_PER_BYTE)) & 0xff); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_pmp_report_status); static int apple_pmp_report_probe(struct platform_device *pdev) { @@ -1139,6 +1235,13 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret && !rep->offsets->starts_pmp) return dev_err_probe(dev, ret, "failed to create child devices\n"); + mutex_lock(&apple_pmp_reports_lock); + list_add_tail_rcu(&rep->list, &apple_pmp_reports); + mutex_unlock(&apple_pmp_reports_lock); + + if (rep->offsets->fast_die) + debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, + rep, &apple_pmp_report_status_fops); if (rep->offsets->starts_pmp) { /* 4: the always-on entries have seeded their requests */ request = readq(rep->base + rep->offsets->tgt_read); @@ -1240,6 +1343,14 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_j613_25g83 = { .tgt_write = 0x10800, .actual = 0x1080, .status = 0x10, +/* The fast die-temperature report is the low two bytes of entry 0x1c1. */ +static const struct apple_pmp_report_offsets apple_pmp_offsets_t8140 = { + .tgt_read = 0x1880, + .tgt_write = 0x10c40, + .actual = 0x18c0, + .status = 0x10, + .fast_die = 0x1c10, + .fast_die_lanes = 2, }; static const struct of_device_id apple_pmp_report_of_match[] = { @@ -1251,6 +1362,7 @@ static const struct of_device_id apple_pmp_report_of_match[] = { { .compatible = "apple,t6030-pmp-v2-report", .data = &apple_pmp_offsets_t6030 }, { .compatible = "apple,t8122-pmp-v2-report", .data = &apple_pmp_offsets_t8122 }, { .compatible = "apple,t6031-pmp-v2-report", .data = &apple_pmp_offsets_t6031 }, + { .compatible = "apple,t8140-pmp-v2-report", .data = &apple_pmp_offsets_t8140 }, {} }; @@ -1407,6 +1519,7 @@ static int __init apple_pmp_report_init(void) static void __exit apple_pmp_report_exit(void) { + debugfs_remove_recursive(apple_pmp_report_debugfs); platform_driver_unregister(&apple_pmp_report_entry_driver); platform_driver_unregister(&apple_pmp_report_driver); } diff --git a/include/linux/soc/apple/pmp-report.h b/include/linux/soc/apple/pmp-report.h index d71a0d04d56fdd..62a6c05a52996a 100644 --- a/include/linux/soc/apple/pmp-report.h +++ b/include/linux/soc/apple/pmp-report.h @@ -9,6 +9,7 @@ #define _LINUX_SOC_APPLE_PMP_REPORT_H_ #include +#include struct device_node; struct device; @@ -16,12 +17,20 @@ struct device; #if IS_ENABLED(CONFIG_APPLE_PMP_REPORT) int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout); int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int timeout_ms); +int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort); #else static inline int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout) { return -ENODEV; } + +static inline int apple_pmp_report_fast_die_effort(const struct device_node *np, + unsigned int lane, u8 *effort) +{ + return -ENODEV; +} #endif #endif /* _LINUX_SOC_APPLE_PMP_REPORT_H_ */ From dd6da23a31d70ae00dc276bdda28e6c9e3f077c2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:00:11 +1100 Subject: [PATCH 219/631] dt-bindings: apple: describe PMP fast-die reporting on T8140 Document the existing PMP firmware driver resources and the report aperture consumed by the fast-die limiter. Define a report phandle and lane index on the cluster DVFS controller, using the measured P lane and ADT E lane ordering. Firmware calibration properties are populated by the bootloader from the Apple device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../cpufreq/apple,cluster-cpufreq.yaml | 12 +++ .../power/apple,t8140-pmp-report.yaml | 46 +++++++++ .../bindings/soc/apple/apple,pmp.yaml | 94 +++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml diff --git a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml index d6618e159cf723..a27f4314e0387a 100644 --- a/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml +++ b/Documentation/devicetree/bindings/cpufreq/apple,cluster-cpufreq.yaml @@ -45,6 +45,18 @@ properties: reg: maxItems: 1 + apple,fast-die-effort: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: | + PMP report region and cluster effort lane. Increasing firmware effort + imposes a maximum-frequency QoS constraint alongside thermal cooling. + If the PMP is unavailable, cpufreq retains its fallback ceiling. + items: + - items: + - description: PMP report region + - description: Cluster lane (0 for E, 1 for P on T8140) + enum: [0, 1] + '#performance-domain-cells': const: 0 diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml new file mode 100644 index 00000000000000..9b515685eef00e --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 PMP report aperture + +maintainers: + - Ryan Murray + +description: | + The PMP report aperture contains a read window and an update window. + The PMP running bit qualifies the per-cluster fast-die effort. Lane 1 + tracks P-cluster load in Linux measurements. Lane 0 is assigned to the + E cluster from the Apple device tree order and has not been observed + changing. Only documented report entries may be read; the aperture is + not a contiguous readable array of registers. + +properties: + compatible: + const: apple,t8140-pmp-v2-report + + reg: + maxItems: 1 + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pmp-report@3003c0000 { + compatible = "apple,t8140-pmp-v2-report"; + reg = <3 0x003c0000 0 0x14000>; + }; + }; diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml new file mode 100644 index 00000000000000..9784c62fba89ec --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,pmp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple power management co-processor + +maintainers: + - Ryan Murray + +description: | + The PMP runs firmware loaded by the bootloader. Linux starts it through + RTKit and supplies calibration copied from the Apple device tree. The + bootloader must initialize the PMP DART address protection filter before + Linux starts the firmware. Firmware-owned PMC power state must be retained. + +properties: + compatible: + oneOf: + - const: apple,t6000-pmp-v2 + - items: + - const: apple,t8140-pmp-v2 + - const: apple,t6000-pmp-v2 + + reg: + items: + - description: Firmware SRAM + - description: ASC control registers + + reg-names: + items: + - const: pmp + - const: asc + + mboxes: + maxItems: 1 + + mbox-names: + const: mbox + + iommus: + maxItems: 1 + + power-domains: + maxItems: 1 + + apple,board-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Board identifier copied by the bootloader. + + apple,dram-vendor-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: DRAM vendor identifier copied by the bootloader. + + apple,dram-capacity: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Firmware DRAM capacity value copied by the bootloader. + + apple,pio-ranges: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 2 + description: Host physical address and size pairs mapped into firmware PIO space. + +patternProperties: + '^apple,tunable-': + $ref: /schemas/types.yaml#/definitions/uint8-array + description: Opaque firmware calibration copied from the Apple device tree. + +required: + - compatible + - reg + - reg-names + - mboxes + - mbox-names + - iommus + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + pmp@300500000 { + compatible = "apple,t8140-pmp-v2", "apple,t6000-pmp-v2"; + reg = <3 0x00500000 0 0xa0000>, <3 0x00e00000 0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + }; + }; From 4b49909f39ae31c7d91f22eb3f583b3ad46f00c1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 13:58:05 +1100 Subject: [PATCH 220/631] dt-bindings: rtc: describe the J700 Baku PMU alarm Describe the RTC function as a child of the existing Baku SPMI NVMEM provider. It shares the whole-second offset cell with the SMC RTC and has a dedicated alarm interrupt. Add the child to the PMIC schema. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/nvmem/apple,spmi-nvmem.yaml | 3 + .../bindings/rtc/apple,j700-pmu-rtc.yaml | 63 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml diff --git a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml index 7a4114c7fbd197..09cdd776f8d8bd 100644 --- a/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml +++ b/Documentation/devicetree/bindings/nvmem/apple,spmi-nvmem.yaml @@ -27,6 +27,9 @@ properties: reg: maxItems: 1 + rtc: + $ref: /schemas/rtc/apple,j700-pmu-rtc.yaml# + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml b/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml new file mode 100644 index 00000000000000..b1e289c06fe338 --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/apple,j700-pmu-rtc.yaml @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/apple,j700-pmu-rtc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple J700 Baku PMU real-time clock + +maintainers: + - Ryan Murray + +description: | + Baku exposes a 65536 Hz counter and an alarm comparator over SPMI. + The clock's whole-second offset is shared with the SMC RTC through an + eight-byte little-endian NVMEM cell. This function is a child of the + PMIC NVMEM provider and uses that provider's SPMI register map. + +allOf: + - $ref: rtc.yaml# + +properties: + compatible: + const: apple,j700-pmu-rtc + + interrupts: + maxItems: 1 + + nvmem-cells: + maxItems: 1 + description: The shared eight-byte whole-second RTC offset. + + nvmem-cell-names: + items: + - const: rtc_offset + + wakeup-source: true + +required: + - compatible + - interrupts + - nvmem-cells + - nvmem-cell-names + +unevaluatedProperties: false + +examples: + - | + #include + #include + + pmic@e { + compatible = "apple,baku-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + rtc { + compatible = "apple,j700-pmu-rtc"; + interrupt-parent = <&spmi>; + interrupts = <2 IRQ_TYPE_EDGE_RISING>; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + wakeup-source; + }; + }; From 5d3e9d345dba578456ddd729a903683fc5391c59 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Thu, 8 Oct 2026 14:43:19 +1100 Subject: [PATCH 221/631] rtc: add opt-in J700 Baku PMU alarms with retained s2idle interrupts Expose the Baku counter and alarm as a child of the existing SPMI PMIC provider. Share its regmap and NVMEM offset with the SMC RTC. Read the 65536 Hz counter as whole seconds and obtain the 64-bit seconds offset on each operation so that SMC clock updates are visible immediately. Keep time-setting on the existing SMC RTC and expose the PMU alarm here. The original opt-in alarm design was written by Eryk Wieliczko; Eryk's Signed-off-by is outstanding. Retain his authorship for the design. Retained SPMI/AIC power is required. Opt-in is explicit and alarms are limited to one hour until longer retention is qualified. Reject deeper suspend states and an alarm inherited from firmware. Acknowledge alarms in a threaded interrupt and fail closed on acknowledgment I/O errors. No off-domain wake or power-saving claim is made. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvmem/apple-spmi-nvmem.c | 10 +- drivers/rtc/Kconfig | 11 + drivers/rtc/Makefile | 1 + drivers/rtc/rtc-apple-pmu.c | 369 +++++++++++++++++++++++++++++++ 4 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 drivers/rtc/rtc-apple-pmu.c diff --git a/drivers/nvmem/apple-spmi-nvmem.c b/drivers/nvmem/apple-spmi-nvmem.c index 7acb0c07d6abe9..338e2e72f803df 100644 --- a/drivers/nvmem/apple-spmi-nvmem.c +++ b/drivers/nvmem/apple-spmi-nvmem.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,7 @@ static int apple_spmi_nvmem_write(void *priv, unsigned int offset, void *val, static int apple_spmi_nvmem_probe(struct spmi_device *sdev) { struct regmap *regmap; + struct nvmem_device *provider; struct nvmem_config nvmem_cfg = { .dev = &sdev->dev, .name = "spmi_nvmem", @@ -54,7 +56,13 @@ static int apple_spmi_nvmem_probe(struct spmi_device *sdev) nvmem_cfg.priv = regmap; - return PTR_ERR_OR_ZERO(devm_nvmem_register(&sdev->dev, &nvmem_cfg)); + provider = devm_nvmem_register(&sdev->dev, &nvmem_cfg); + if (IS_ERR(provider)) + return PTR_ERR(provider); + if (!of_device_is_compatible(sdev->dev.of_node, "apple,baku-pmic")) + return 0; + + return devm_of_platform_populate(&sdev->dev); } static const struct of_device_id apple_spmi_nvmem_id_table[] = { diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 364afc73f8abd2..5488a2dd5c86e9 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2100,6 +2100,17 @@ config RTC_DRV_WILCO_EC This can also be built as a module. If so, the module will be named "rtc_wilco_ec". +config RTC_DRV_APPLE_PMU + tristate "Apple J700 Baku PMU RTC alarm" + depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP + help + Read the Baku PMU clock and use alarms with retained-power + suspend-to-idle. The module requires experimental=1. Alarms are + limited to one hour; deeper suspend states are not supported. + Continue setting the shared clock through the SMC RTC. + + This driver can be built as the module rtc-apple-pmu. + config RTC_DRV_MACSMC tristate "Apple Mac System Management Controller RTC" depends on MFD_MACSMC diff --git a/drivers/rtc/Makefile b/drivers/rtc/Makefile index 6cf7e066314e13..729ff56c871d3d 100644 --- a/drivers/rtc/Makefile +++ b/drivers/rtc/Makefile @@ -94,6 +94,7 @@ obj-$(CONFIG_RTC_DRV_M48T35) += rtc-m48t35.o obj-$(CONFIG_RTC_DRV_M48T59) += rtc-m48t59.o obj-$(CONFIG_RTC_DRV_M48T86) += rtc-m48t86.o obj-$(CONFIG_RTC_DRV_MA35D1) += rtc-ma35d1.o +obj-$(CONFIG_RTC_DRV_APPLE_PMU) += rtc-apple-pmu.o obj-$(CONFIG_RTC_DRV_MACSMC) += rtc-macsmc.o obj-$(CONFIG_RTC_DRV_MAX31335) += rtc-max31335.o obj-$(CONFIG_RTC_DRV_MAX6900) += rtc-max6900.o diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c new file mode 100644 index 00000000000000..f398a1a802165c --- /dev/null +++ b/drivers/rtc/rtc-apple-pmu.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Apple Baku PMU clock and retained-power alarm. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BAKU_RTC_CONTROL 0xf800 +#define BAKU_RTC_COUNTER 0xf802 +#define BAKU_RTC_COMPARE 0xf808 +#define BAKU_RTC_EVENT 0xf80c +#define BAKU_RTC_MASK 0xf80e +#define BAKU_RTC_RUN BIT(0) +#define BAKU_RTC_ARM BIT(6) +#define BAKU_RTC_ALARM BIT(0) +#define BAKU_RTC_HZ_SHIFT 16 +#define BAKU_RTC_MAX_DELAY 3600 + +static bool experimental; +module_param(experimental, bool, 0444); +MODULE_PARM_DESC(experimental, "Enable alarms with retained SPMI/AIC power in s2idle"); + +struct baku_rtc { + struct regmap *map; + struct nvmem_cell *offset; + struct rtc_device *clock; + /* Serialize comparator updates and interrupt acknowledgment. */ + struct mutex transaction; + int interrupt; + bool io_failed; +}; + +static int baku_rtc_seconds(struct baku_rtc *baku, u64 *seconds) +{ + u8 count[8] = {}; + int error; + + error = regmap_bulk_read(baku->map, BAKU_RTC_COUNTER, count, 6); + if (error) + return error; + *seconds = get_unaligned_le64(count) >> BAKU_RTC_HZ_SHIFT; + return 0; +} + +static int baku_rtc_offset(struct baku_rtc *baku, s64 *seconds) +{ + size_t bytes; + void *value; + + /* The SMC RTC can update this cell, so do not cache it at probe. */ + value = nvmem_cell_read(baku->offset, &bytes); + if (IS_ERR(value)) + return PTR_ERR(value); + if (bytes != sizeof(__le64)) { + kfree(value); + return -EINVAL; + } + *seconds = get_unaligned_le64(value); + kfree(value); + if (*seconds < 0 || *seconds > RTC_TIMESTAMP_END_9999 - U32_MAX) + return -ERANGE; + return 0; +} + +static int baku_rtc_read_time(struct device *dev, struct rtc_time *time) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + u64 counter; + s64 epoch; + int error; + + guard(mutex)(&baku->transaction); + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + error = baku_rtc_seconds(baku, &counter); + if (error) + return error; + rtc_time64_to_tm(epoch + counter, time); + return 0; +} + +/* All comparator/control accesses are serialized with the IRQ thread. */ +static int baku_rtc_stop_alarm(struct baku_rtc *baku) +{ + int error; + + error = regmap_update_bits(baku->map, BAKU_RTC_MASK, + BAKU_RTC_ALARM, BAKU_RTC_ALARM); + if (error) + return error; + error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, BAKU_RTC_ARM, 0); + if (error) + return error; + return regmap_write(baku->map, BAKU_RTC_EVENT, BAKU_RTC_ALARM); +} + +static int baku_rtc_start_alarm(struct baku_rtc *baku) +{ + int error; + + error = regmap_write(baku->map, BAKU_RTC_EVENT, BAKU_RTC_ALARM); + if (!error) + error = regmap_update_bits(baku->map, BAKU_RTC_MASK, BAKU_RTC_ALARM, 0); + if (!error) + error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, + BAKU_RTC_RUN | BAKU_RTC_ARM, + BAKU_RTC_RUN | BAKU_RTC_ARM); + if (error) + baku_rtc_stop_alarm(baku); + return error; +} + +static int baku_rtc_validate_alarm(struct baku_rtc *baku, u64 comparator) +{ + u64 counter; + int error; + + error = baku_rtc_seconds(baku, &counter); + if (error) + return error; + if (comparator <= counter || comparator - counter > BAKU_RTC_MAX_DELAY) + return -ETIME; + return 0; +} + +static int baku_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + unsigned int control, event, mask; + __le32 comparator; + s64 epoch; + int error; + + guard(mutex)(&baku->transaction); + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_EVENT, &event); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_MASK, &mask); + if (error) + return error; + rtc_time64_to_tm(epoch + le32_to_cpu(comparator), &alarm->time); + alarm->enabled = (control & BAKU_RTC_ARM) && !(mask & BAKU_RTC_ALARM); + alarm->pending = !!(event & BAKU_RTC_ALARM); + return 0; +} + +static int baku_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alarm) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + __le32 comparator, readback; + s64 epoch, deadline; + int error; + + guard(mutex)(&baku->transaction); + if (baku->io_failed) + return -EIO; + error = baku_rtc_offset(baku, &epoch); + if (error) + return error; + deadline = rtc_tm_to_time64(&alarm->time) - epoch; + if (deadline < 0 || deadline > U32_MAX) + return -ERANGE; + error = baku_rtc_validate_alarm(baku, deadline); + if (error) + return error; + error = baku_rtc_stop_alarm(baku); + if (error) + return error; + comparator = cpu_to_le32(deadline); + error = regmap_bulk_write(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &readback, sizeof(readback)); + if (error) + return error; + if (readback != comparator) + return -EIO; + if (alarm->enabled) + return baku_rtc_start_alarm(baku); + return 0; +} + +static int baku_rtc_alarm_enable(struct device *dev, unsigned int enable) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + __le32 comparator; + int error; + + guard(mutex)(&baku->transaction); + if (!enable) + return baku_rtc_stop_alarm(baku); + if (baku->io_failed) + return -EIO; + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + error = baku_rtc_validate_alarm(baku, le32_to_cpu(comparator)); + return error ?: baku_rtc_start_alarm(baku); +} + +static irqreturn_t baku_rtc_interrupt(int irq, void *context) +{ + struct device *dev = context; + struct baku_rtc *baku = dev_get_drvdata(dev); + unsigned int event; + int error; + + mutex_lock(&baku->transaction); + error = regmap_read(baku->map, BAKU_RTC_EVENT, &event); + if (!error && (event & BAKU_RTC_ALARM)) + error = baku_rtc_stop_alarm(baku); + if (error) { + baku->io_failed = true; + disable_irq_nosync(irq); + } + mutex_unlock(&baku->transaction); + if (error) { + dev_err(dev, "cannot acknowledge RTC interrupt: %d\n", error); + pm_wakeup_dev_event(dev, 0, true); + return IRQ_HANDLED; + } + if (!(event & BAKU_RTC_ALARM)) + return IRQ_NONE; + if (device_may_wakeup(dev)) + pm_wakeup_dev_event(dev, 0, true); + rtc_update_irq(baku->clock, 1, RTC_IRQF | RTC_AF); + return IRQ_HANDLED; +} + +static const struct rtc_class_ops baku_rtc_ops = { + .read_time = baku_rtc_read_time, + .read_alarm = baku_rtc_read_alarm, + .set_alarm = baku_rtc_set_alarm, + .alarm_irq_enable = baku_rtc_alarm_enable, +}; + +static int baku_rtc_prepare(struct device *dev) +{ + struct baku_rtc *baku = dev_get_drvdata(dev); + + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE || baku->io_failed) + return -EBUSY; + return 0; +} + +static const struct dev_pm_ops baku_rtc_pm = { + .prepare = baku_rtc_prepare, +}; + +static void baku_rtc_release(void *context) +{ + struct device *dev = context; + struct baku_rtc *baku = dev_get_drvdata(dev); + + disable_irq(baku->interrupt); + mutex_lock(&baku->transaction); + if (baku_rtc_stop_alarm(baku)) + dev_err(dev, "cannot disable RTC alarm on removal\n"); + mutex_unlock(&baku->transaction); + device_init_wakeup(dev, false); +} + +static int baku_rtc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct baku_rtc *baku; + unsigned int control; + int error; + + /* Retention has only been measured on J700 with the explicit opt-in. */ + if (!experimental) + return -ENODEV; + baku = devm_kzalloc(dev, sizeof(*baku), GFP_KERNEL); + if (!baku) + return -ENOMEM; + baku->map = dev_get_regmap(dev->parent, NULL); + if (!baku->map) + return dev_err_probe(dev, -EPROBE_DEFER, "PMIC regmap unavailable\n"); + baku->offset = devm_nvmem_cell_get(dev, "rtc_offset"); + if (IS_ERR(baku->offset)) + return dev_err_probe(dev, PTR_ERR(baku->offset), "RTC offset unavailable\n"); + baku->interrupt = platform_get_irq(pdev, 0); + if (baku->interrupt < 0) + return baku->interrupt; + mutex_init(&baku->transaction); + platform_set_drvdata(pdev, baku); + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (error) + return error; + /* Respect an alarm left armed by firmware. */ + if (control & BAKU_RTC_ARM) + return -EBUSY; + baku->clock = devm_rtc_allocate_device(dev); + if (IS_ERR(baku->clock)) + return PTR_ERR(baku->clock); + baku->clock->ops = &baku_rtc_ops; + baku->clock->range_min = 0; + baku->clock->range_max = RTC_TIMESTAMP_END_9999; + clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, baku->clock->features); + error = device_init_wakeup(dev, true); + if (error) + return error; + /* + * This interrupt remains live during retained-power s2idle. The + * thread uses the parent PMIC's SPMI regmap to acknowledge the alarm. + * Off-domain wake routing is not established by this driver. + */ + error = devm_request_threaded_irq(dev, baku->interrupt, NULL, + baku_rtc_interrupt, + IRQF_NO_AUTOEN | IRQF_NO_SUSPEND | IRQF_ONESHOT, + dev_name(dev), dev); + if (error) + goto disable_wakeup; + error = baku_rtc_stop_alarm(baku); + if (error) + goto disable_wakeup; + error = devm_rtc_register_device(baku->clock); + if (error) + goto disable_wakeup; + error = devm_add_action_or_reset(dev, baku_rtc_release, dev); + if (error) + return error; + enable_irq(baku->interrupt); + return 0; + +disable_wakeup: + device_init_wakeup(dev, false); + return error; +} + +static const struct of_device_id baku_rtc_match[] = { + { .compatible = "apple,j700-pmu-rtc" }, + { } +}; +MODULE_DEVICE_TABLE(of, baku_rtc_match); + +static struct platform_driver baku_rtc_driver = { + .probe = baku_rtc_probe, + .driver = { + .name = "apple-pmu-rtc", + .of_match_table = baku_rtc_match, + .pm = &baku_rtc_pm, + }, +}; +module_platform_driver(baku_rtc_driver); + +MODULE_AUTHOR("Eryk Wieliczko "); +MODULE_DESCRIPTION("Apple Baku PMU RTC alarm with retained-power s2idle"); +MODULE_LICENSE("GPL"); From f6a65599f8790db285e7c87deb9caceec6984a12 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 15:50:15 +1100 Subject: [PATCH 222/631] rtc: apple-pmu: leave the counter control state to firmware The alarm arm mask is described by the Linux-captured ADT. Setting an additional RUN bit is unnecessary for programming the comparator and changes the shared time counter's state. Update only the alarm arm bit, leaving the clock path and unrelated control bits unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-apple-pmu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index f398a1a802165c..7853b72a1b5526 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -19,7 +19,6 @@ #define BAKU_RTC_COMPARE 0xf808 #define BAKU_RTC_EVENT 0xf80c #define BAKU_RTC_MASK 0xf80e -#define BAKU_RTC_RUN BIT(0) #define BAKU_RTC_ARM BIT(6) #define BAKU_RTC_ALARM BIT(0) #define BAKU_RTC_HZ_SHIFT 16 @@ -113,8 +112,7 @@ static int baku_rtc_start_alarm(struct baku_rtc *baku) error = regmap_update_bits(baku->map, BAKU_RTC_MASK, BAKU_RTC_ALARM, 0); if (!error) error = regmap_update_bits(baku->map, BAKU_RTC_CONTROL, - BAKU_RTC_RUN | BAKU_RTC_ARM, - BAKU_RTC_RUN | BAKU_RTC_ARM); + BAKU_RTC_ARM, BAKU_RTC_ARM); if (error) baku_rtc_stop_alarm(baku); return error; From 4db5865ba9cc60e146577b51c5021f89584fc78a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:56:15 +1100 Subject: [PATCH 223/631] dt-bindings: npu: apple,ane: describe the T8140 topology T8140 describes two ANE interrupts, three DART clients and a single controllable system power domain. Describe that topology separately from the M1 and M2 requirements without selecting an older firmware interface. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/npu/apple,ane.yaml | 63 +++++++++++++++++-- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/npu/apple,ane.yaml b/Documentation/devicetree/bindings/npu/apple,ane.yaml index ebca460a81bc14..79cf942654f7bc 100644 --- a/Documentation/devicetree/bindings/npu/apple,ane.yaml +++ b/Documentation/devicetree/bindings/npu/apple,ane.yaml @@ -14,7 +14,8 @@ description: SoCs. It accesses memory through three DARTs. On T8103 and T6000 the host drives the engine through its register window. On T602x and T8112 a coprocessor runs the ANE firmware and the host talks to it through an ASC - mailbox. + mailbox. T8140 has two interrupts and one system power domain; its + coprocessor uses a different firmware interface. properties: compatible: @@ -25,6 +26,7 @@ properties: - apple,t6021-ane - apple,t6022-ane - apple,t8112-ane + - apple,t8140-ane reg: minItems: 1 @@ -45,7 +47,8 @@ properties: - const: fuse interrupts: - maxItems: 1 + minItems: 1 + maxItems: 2 interrupt-names: items: @@ -57,8 +60,8 @@ properties: power-domains: description: The power states the engine needs, five on T8103 and T6000, eight on - T602x and seven on T8112. - minItems: 5 + T602x, seven on T8112 and one system domain on T8140. + minItems: 1 maxItems: 8 resets: @@ -80,11 +83,40 @@ required: - reg - reg-names - interrupts - - interrupt-names - iommus - power-domains allOf: + - if: + properties: + compatible: + contains: + const: apple,t8140-ane + then: + properties: + reg: + maxItems: 1 + reg-names: + maxItems: 1 + interrupts: + minItems: 2 + interrupt-names: false + iommus: + minItems: 3 + power-domains: + maxItems: 1 + resets: false + mboxes: false + memory-region: false + else: + properties: + interrupts: + maxItems: 1 + power-domains: + minItems: 5 + required: + - interrupt-names + - if: properties: compatible: @@ -149,6 +181,27 @@ allOf: additionalProperties: false examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + ane@400000000 { + compatible = "apple,t8140-ane"; + reg = <0x4 0x0 0x0 0x2000000>; + reg-names = "engine"; + interrupt-parent = <&aic>; + interrupts = , + ; + iommus = <&ane_dart0 0>, <&ane_dart1 0>, <&ane_dart2 0>; + power-domains = <&ps_ane_sys>; + status = "disabled"; + }; + }; + - | #include #include From 5175306aabd8327b80dd8a24c2ea2d473762e978 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 17:09:30 +1100 Subject: [PATCH 224/631] soc: apple: expose the T8140 PMP DVFS report for diagnosis The firmware describes DVFS-STATE as four report entries beginning at entry 8. Make those values available through a read-only debugfs file in the existing PMP report driver. Require firmware readiness before and after sampling, bound the reads to the mapped resource, and remove the debugfs directory before releasing the mapping. The entry encoding and rate conversion are not established. Document the raw values as diagnostics, without presenting them as temperatures or hardware clock rates. This complements firmware frequency limits and thermal cooling without issuing new requests or changing policy. The T8140 report match and platform resources are provided separately. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/soc/apple/t8140-pmp-dvfs.rst | 24 +++++++ drivers/pmdomain/apple/pmp-report.c | 80 ++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 Documentation/soc/apple/t8140-pmp-dvfs.rst diff --git a/Documentation/soc/apple/t8140-pmp-dvfs.rst b/Documentation/soc/apple/t8140-pmp-dvfs.rst new file mode 100644 index 00000000000000..7ec4ae7366ed90 --- /dev/null +++ b/Documentation/soc/apple/t8140-pmp-dvfs.rst @@ -0,0 +1,24 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +T8140 PMP DVFS diagnostics +========================= + +The existing PMP report provider exposes a read-only ``dvfs`` file under +``/sys/kernel/debug/apple-pmp-/`` on T8140. The provider must be +bound; the file is readable by root. Its output lists ADT report entries +8 through 11 and both raw 64-bit words of each 16-byte entry. Only those four entries +are read. The report aperture is not dumped. + +Reading returns ``EAGAIN`` if the PMP running bit is clear before or after +sampling. Entries are sampled individually, rather than as an atomic +firmware snapshot. A running PMP alone does not establish freshness of +each value. + +The field encoding and conversion to clock rates are not yet established. +Values must not be interpreted as Hz, temperatures, or verified hardware +clock states. This interface changes neither firmware requests nor cache +allocation nor thermal policy. + +CPU temperatures remain available through ``macsmc-hwmon``. CPU cooling +and firmware frequency limits remain under their existing drivers. The +DVFS diagnostic file is not a thermal sensor. diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index ad4b590cd84923..3734d578a364ca 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,10 @@ struct apple_pmp_report_offsets { u32 fast_die_lanes; }; +struct apple_pmp_report; + +static int apple_pmp_dvfs_create(struct apple_pmp_report *report); + struct apple_pmp_report { struct device *dev; const struct apple_pmp_report_offsets *offsets; @@ -1231,6 +1236,11 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret) return ret; } + dev_set_drvdata(dev, rep); + ret = apple_pmp_dvfs_create(rep); + if (ret) + return ret; + ret = of_platform_populate(np, NULL, NULL, dev); if (ret && !rep->offsets->starts_pmp) return dev_err_probe(dev, ret, "failed to create child devices\n"); @@ -1422,6 +1432,76 @@ static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) return apple_pmp_report_set_state(genpd, false); } +/* ADT DVFS-STATE: four 16-byte entries beginning at entry 8. */ +#define APPLE_PMP_DVFS_OFFSET 0x80 +#define APPLE_PMP_DVFS_STRIDE 16 +#define APPLE_PMP_DVFS_COUNT 4 + +static int apple_pmp_dvfs_show(struct seq_file *seq, void *unused) +{ + struct apple_pmp_report *report = seq->private; + u64 values[APPLE_PMP_DVFS_COUNT][2]; + unsigned int index; + + if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) + return -EAGAIN; + + for (index = 0; index < ARRAY_SIZE(values); index++) { + void __iomem *entry = report->base + APPLE_PMP_DVFS_OFFSET + + index * APPLE_PMP_DVFS_STRIDE; + + values[index][0] = readq(entry); + values[index][1] = readq(entry + sizeof(values[index][0])); + } + + if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) + return -EAGAIN; + + for (index = 0; index < ARRAY_SIZE(values); index++) + seq_printf(seq, "entry%u: 0x%016llx 0x%016llx\n", index + 8, + values[index][0], values[index][1]); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(apple_pmp_dvfs); + +static void apple_pmp_dvfs_remove(void *directory) +{ + debugfs_remove_recursive(directory); +} + +static int apple_pmp_dvfs_create(struct apple_pmp_report *report) +{ + struct device *dev = report->dev; + struct resource *resource; + struct dentry *directory; + char *name; + int ret; + + if (!of_device_is_compatible(dev->of_node, "apple,t8140-pmp-v2-report")) + return 0; + + resource = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 0); + if (resource_size(resource) < APPLE_PMP_DVFS_OFFSET + + APPLE_PMP_DVFS_COUNT * APPLE_PMP_DVFS_STRIDE) + return dev_err_probe(dev, -EINVAL, "DVFS report exceeds resource\n"); + + name = devm_kasprintf(dev, GFP_KERNEL, "apple-pmp-%s", dev_name(dev)); + if (!name) + return -ENOMEM; + + directory = debugfs_create_dir(name, NULL); + if (IS_ERR(directory)) + return 0; + + ret = devm_add_action_or_reset(dev, apple_pmp_dvfs_remove, directory); + if (ret) + return ret; + + debugfs_create_file("dvfs", 0400, directory, report, &apple_pmp_dvfs_fops); + return 0; +} + static int apple_pmp_report_entry_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; From 8eb4eddf867e00921b2c93f5250598e9866d60bc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:47:57 +1100 Subject: [PATCH 225/631] drm/apple: retain the guarded fbdev probe in the display composition Keep the probe and driver callback guarded by CONFIG_DRM_FBDEV_EMULATION when composing the older dual-stream branches with the newer framebuffer helper fix. Enabled and disabled configurations retain the intended DRM helper behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 5ace07ad14fbbd..ed2aa49abb70cf 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -68,6 +68,7 @@ static int apple_drm_gem_dumb_create(struct drm_file *file_priv, return drm_gem_dma_dumb_create_internal(file_priv, drm, args); } +#ifdef CONFIG_DRM_FBDEV_EMULATION static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes) { @@ -97,6 +98,7 @@ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) if (file->was_master) dcp_typec_reorder(); } +#endif /* * A compositor or boot splash is closing the device. drm_release() has @@ -111,6 +113,7 @@ static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) static const struct drm_driver apple_drm_driver = { DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(apple_drm_gem_dumb_create), +#ifdef CONFIG_DRM_FBDEV_EMULATION .fbdev_probe = apple_drm_fbdev_probe, #else DRM_FBDEV_DMA_DRIVER_OPS, From d94fd3a42da386871ee309086c5ddc88a330c07a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:49:30 +1100 Subject: [PATCH 226/631] drm/asahi: retain two-phase data ownership with stats registration Keep the reviewed new_uninit/init_data construction sequence when composing per-device statistics with the GPU probe fixes. The stats registration and teardown retain their owned snapshot, while failed GPU construction cannot publish ownership of an uninitialized data slot. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/driver.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 0457937812f6df..fd2c5430d40ca5 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -569,12 +569,9 @@ impl AsahiDriver { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let res = regs::Resources::new(pdev)?; - // SAFETY: As in the legacy probe, the data is inaccessible until the fully initialized - // manager is installed below and the DRM device is registered. - // If construction fails, drm::device::Device::release leaves the data - // untouched rather than dropping an uninitialized AsahiData. - let uninit = unsafe { pin_init::pin_init_from_closure::(|_slot| Ok(())) }; - let drm: ARef = drm::device::Device::new(pdev.as_ref(), uninit)?; + // SAFETY: GPU construction cannot access driver data before init_data(). + // Failed construction releases the allocation without dropping uninitialized data. + let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; let data_gpu = gpu.clone() as Arc; let data = try_pin_init!(AsahiData { @@ -582,9 +579,8 @@ impl AsahiDriver { pdev: pdev.into(), resources: res, }); - let ptr = &raw const **drm; - // SAFETY: The allocation is pinned and its data slot has not been initialized yet. - if let Err(error) = unsafe { data.__pinned_init(ptr as *mut AsahiData) } { + // SAFETY: This is the sole initializer; no driver-data users have been published. + if let Err(error) = unsafe { drm.init_data(data) } { gpu.shutdown(); return Err(error); } From 2843fbc9162348a59dfa2c1617955a87457ae9e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:32:03 +1100 Subject: [PATCH 227/631] drm: apple: read H17P blend CRCs through the outbound queue Linux measurements show that the H17P blend CRC command takes no input and returns one little-endian 32-bit value. The existing completion path instead supplies a constant zero to the DRM CRC interface. Queue one read for each validated present completion while CRC capture is enabled. Put it before the next pending present so the sample refers to the completed frame, and send it only after the callback is acknowledged and the transport is idle. Use the existing transaction timeout and crash handling, reject malformed replies, and discard queued reads on shutdown. Disabling capture suppresses both unsent requests and returned samples. Older firmware and the H17G method profile retain their current behavior. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/iomfb.c | 69 ++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index c642db6b01f524..ce8f0ba9b97ded 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -570,6 +570,7 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); +void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 412ce7cf8b0ca9..8c89ad4f493279 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -409,6 +410,74 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) return ret; } +struct iomfb_crc_transaction { + struct iomfb_transaction transaction; + u32 swap_id; +}; + +static void iomfb_crc_complete(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_crc_transaction *crc = cookie; + struct dcp_packet_header *header; + u32 value; + + /* A106 has no input and a single little-endian blend-output CRC. */ + if (!out) + goto invalid; + header = out - sizeof(*header); + if (header->in_len || header->out_len != sizeof(__le32)) + goto invalid; + value = get_unaligned_le32(out); + if (READ_ONCE(dcp->crc_enabled)) + drm_crtc_add_crc_entry(&dcp->crtc->base, true, crc->swap_id, &value); + return; + +invalid: + dev_err(dcp->dev, "invalid blend CRC response\n"); + WRITE_ONCE(dcp->crashed, true); +} + +static void iomfb_crc_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + static const struct dcp_method_entry method = { + .tag = { 'A', '1', '0', '6' }, + .name = "read_blend_crc", + }; + struct iomfb_crc_transaction *crc = container_of(transaction, + struct iomfb_crc_transaction, + transaction); + + if (READ_ONCE(dcp->crc_enabled)) + dcp_push(dcp, false, &method, 0, sizeof(__le32), NULL, + iomfb_crc_complete, crc); +} + +static void iomfb_crc_release(struct iomfb_transaction *transaction) +{ + kfree(container_of(transaction, struct iomfb_crc_transaction, transaction)); +} + +void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id) +{ + struct iomfb_crc_transaction *crc; + + lockdep_assert_held(&dcp->iomfb.lock); + if (!READ_ONCE(dcp->crc_enabled) || READ_ONCE(dcp->crashed) || + dcp->iomfb.stopped || dcp->iomfb.queued >= 32) + return; + crc = kzalloc_obj(*crc); + if (!crc) + return; + crc->swap_id = swap_id; + crc->transaction.start = iomfb_crc_start; + crc->transaction.release = iomfb_crc_release; + /* Read the completed frame before another queued present changes it. */ + list_add(&crc->transaction.link, &dcp->iomfb.pending); + dcp->iomfb.queued++; + /* The receiver advances the queue only after acknowledging completion. */ +} + struct iomfb_command { struct iomfb_transaction transaction; struct dcp_method_entry method; From de4491e6981fa2cd053acb415ed7c39e31812de8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 20:57:47 +1100 Subject: [PATCH 228/631] soc: apple: compose SEP persistence with authenticated sensor sessions Retain the shared key-store profiles, persistence fields, suspend capture handling and module declarations when adding J700 nonce authentication. Choose the existing communication initialization word for each protocol and keep the qualified older-platform sensor-registration path alongside the authenticated J700 path. The composition uses the public SEP work in pull requests #42, #69, #152 and #202 and their reviewed repairs. Contributor credit includes Chris Kearney, DjDeveloperr, Vincent Fontaine and Ryan Murray. Ryan integrates the existing contributions under DCO (b); original commits and author sign-offs are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pm.rs | 25 +++++++++++ drivers/soc/apple/pm_shim.c | 55 +++++++++++++++++++++++ drivers/soc/apple/profile.rs | 77 +++++++++++++++++++++++++++++++++ drivers/soc/apple/sbio.rs | 71 +++++++++++++++++++++++++++--- drivers/soc/apple/sensor_shim.c | 1 - drivers/soc/apple/sep.rs | 4 +- drivers/soc/apple/shim.h | 2 + 7 files changed, 227 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index c2f59228a76e04..3c7a844b521344 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -21,6 +21,31 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { + fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; + fn sep_pm_unregister(); +} + +/// # Safety +/// Called by `pm_shim.c` from the PM notifier chain, in process context. +unsafe extern "C" fn pm_event(entering: bool) -> c_int { + let ptr = SEP.load(Ordering::Acquire); + if ptr.is_null() { + return 0; + } + // SAFETY: produced by `into_foreign` in `register`, and reclaimed by + // `unregister` only after `sep_pm_unregister` has waited out any running + // notifier call, so it is live for this borrow. + let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; + if entering { + if let Err(error) = sep.sleep_prepare() { + // A failing notifier is not included in the PM chain's rollback. + sep.sleep_finished(); + return error.to_errno(); + } + } else { + sep.sleep_finished(); + } + 0 fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 884ecbb354a062..21993dd25667f9 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -105,4 +105,59 @@ int sep_pm_keep_domains(void *ptr) dev_pm_syscore_device(list->pd_devs[i], true); return devm_add_action_or_reset(dev, sep_pm_release_domains, list); + * System-sleep notifier for the SEP driver. It only forwards "about to sleep" + * and "back from sleep" to Rust, which ends any Touch ID capture before the + * system suspends; see pm.rs. + */ +#include +#include +#include + +#include "shim.h" + +static int (*sep_pm_event)(bool entering); + +static int sep_pm_notify(struct notifier_block *nb, unsigned long action, + void *data) +{ + int (*event)(bool entering) = READ_ONCE(sep_pm_event); + + if (!event) + return NOTIFY_DONE; + + switch (action) { + case PM_SUSPEND_PREPARE: + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + return notifier_from_errno(event(true)); + case PM_POST_SUSPEND: + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + event(false); + return NOTIFY_OK; + default: + return NOTIFY_DONE; + } +} + +static struct notifier_block sep_pm_nb = { + .notifier_call = sep_pm_notify, +}; + +int sep_pm_register(int (*event)(bool entering)) +{ + int ret; + + WRITE_ONCE(sep_pm_event, event); + ret = register_pm_notifier(&sep_pm_nb); + if (ret) + WRITE_ONCE(sep_pm_event, NULL); + return ret; +} + +/* Returns only once no notifier call is still running. */ +void sep_pm_unregister(void) +{ + unregister_pm_notifier(&sep_pm_nb); + WRITE_ONCE(sep_pm_event, NULL); } diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 7ae7afa2015880..472c712a862df6 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -118,6 +118,14 @@ pub(crate) struct KeybagCreate { /// (22G74) sepOS, whose request shapes are the 13.5 ones; the T6020 sepOS /// speaks them too before system firmware 26.3 (see [`t6020_profile`]). #[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum KeyStore { + /// The 13.5 key store. Before any other request the endpoint is + /// initialised with `0x4d`, then `set_env` carrying the ADT's `/defaults` + /// `cpx-encryption-mode`. +/// firmware hands the SEP, not the SoC alone: the T8103 stub boots the 13.5 +/// (22G74) sepOS, whose request shapes are the 13.5 ones; the T6020 sepOS +/// is newer. +#[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum KeyStore { /// The 13.5 key store. Before any other request the endpoint is /// initialised with `0x4d`, then `set_env` carrying the ADT's `/defaults` @@ -186,6 +194,19 @@ const T8103: PlatformProfile = PlatformProfile { cpx_encryption_mode: 2, }, persistent_enrol: true, + // 13.5 identity create: version 2, type 0x400000, parent -1 (the enclave + // accepts only -1 or <= -10). + keybag_create: KeybagCreate { + variant: 2, + bag_type: 0x40_0000, + arg: -1, + }, + // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, + persistent_enrol: true, + nonce_session: false, }; /// M1 Pro (J316s). The M1 family cold-boots its SEP like T8103, so the boot @@ -221,6 +242,10 @@ const T6000: PlatformProfile = PlatformProfile { key_store: KeyStore::Sepos13 { cpx_encryption_mode: 2, }, + // The J316s ADT carries `/defaults` `cpx-encryption-mode = 2`. + key_store: KeyStore::Sepos13 { + cpx_encryption_mode: 2, + }, persistent_enrol: true, nonce_session: false, }; @@ -369,6 +394,37 @@ const T8122: PlatformProfile = PlatformProfile { }, wide_dma_mask: true, ..T8112 +/// warm-registration platforms (T6020, T8140). +const T8112: PlatformProfile = PlatformProfile { + name: "T8112/J415", + shmem_capacity: 0x4_0000, + shmem_first_item: b"CINP", + bootstrap: Bootstrap::WarmRegister, + identity: IdentitySource::Chosen, + sensor: SensorProfile { + controller_base: 0x2_3510_8000, + chip_select: 0, + max_hz: 8_000_000, + cs_setup_ns: 20, + cs_hold_ns: 20, + mode: SpiMode::Mode1, + expected_id: 0x3352, + capture_qualified: false, + }, + dart_range_required: true, + wide_dma_mask: false, + firmware_region: c"sepfw", + // J415 accepted the 13.5 GET_CAPABILITIES/SET_ENV init but refused the + // strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did; + // use the T6020 form like the other warm-registration platforms. + keybag_create: KeybagCreate { + variant: 5, + bag_type: 0, + arg: -1, + }, + key_store: KeyStore::Variant5, + persistent_enrol: true, + nonce_session: false, }; static_assert!(T8103.shmem_capacity == 0x30000); @@ -518,6 +574,27 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6001") { return Ok(&T6000); } + // t6020.dtsi is defined as a cut-down t6021: it includes t6021.dtsi and + // disables the parts the smaller die lacks. Both pull in t602x-die0.dtsi, + // so the SEP, its DART and the SPI controller sit at the same addresses + // with the same interrupts on either part, and the T6020 constants apply + // to the M2 Max unchanged. + if machine_has(b"apple,t6021") { + return Ok(&T6020); + } + if machine_has(b"apple,t6000") { + return Ok(&T6000); + } + // t6000.dtsi is defined as a cut-down t6001: it includes t6001.dtsi and + // deletes the parts the smaller die lacks. Both pull in t600x-die0.dtsi, + // so the SEP, its mailbox, its DART and the SPI controller sit at the same + // addresses with the same interrupts on either part, and the T6000 + // constants apply to the M1 Max unchanged. The Mac Studio (j375c) is also + // a t6001 but leaves the SEP disabled, so nothing binds there; apple,t6002 + // (M1 Ultra, no built-in sensor) stays unmapped. + if machine_has(b"apple,t6001") { + return Ok(&T6000); + } if machine_has(b"apple,t8140") { return Ok(&T8140); } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 5e59118c344b53..c0b0b846e2733f 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1036,14 +1036,20 @@ impl SepData { self.register_ool(&self.ool_sbio)?; self.sbio_ready.store(true, Relaxed); - if self.profile.nonce_session { - let word = 6u32; - // Communication must be initialized before restoring biometric state. + let init_word = if self.profile.nonce_session { + Some(6u32) + } else if matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) { + Some(1u32) + } else { + None + }; + if let Some(word) = init_word { let result = self.sbio_transfer_raw( 0x73, c"INIT_SBIO_COMMUNICATION", &word.to_le_bytes(), ); match result { - Ok(done) if done.status.is_ok() => { + Ok(done) if done.status.is_ok() + && (self.profile.nonce_session || done.payload.is_empty()) => { dev_info!(self.dev, "sbio: communication initialization accepted\n"); } Ok(done) => { @@ -1483,7 +1489,9 @@ impl SepData { } CaptureWait::Abandon => { // A cancelled verify also abandons; only a live one is a fault. - if bio::verify_is_live(&self.bio_session.lock()) { + if bio::verify_is_live( + &self.bio_session.lock(), self.capture_epoch.load(Relaxed), + ) { dev_err!( self.dev, "verify: the sensor status became unreadable during the capture\n" @@ -2907,6 +2915,59 @@ impl SepData { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); waited += 10; } + /// Called by the enrol and verify work items before they touch the sensor. + /// Returns false if the system is going to sleep; the session has then + /// been ended and the capture must not start. A true return must be + /// paired with [`Self::capture_end`]. + pub(crate) fn capture_begin(&self) -> bool { + let mut session = self.bio_session.lock(); + if self.suspending.load(Relaxed) { + let woke = bio::abort_capture(&mut session); + drop(session); + if woke { + self.bio_wake(); + } + return false; + } + self.captures_running.fetch_add(1, Relaxed); + true + } + + pub(crate) fn capture_end(&self) { + self.captures_running.fetch_sub(1, Relaxed); + } + + /// The system is about to suspend. End any capture in progress, hold new + /// ones, and wait for the enrol or verify work to stop touching the sensor + /// before devices go down. + pub(crate) fn sleep_prepare(&self) -> Result<()> { + let woke = { + let mut session = self.bio_session.lock(); + self.suspending.store(true, Relaxed); + bio::abort_capture(&mut session) + }; + if woke { + dev_info!( + self.dev, + "Touch ID: ending the capture in progress before sleep\n" + ); + self.bio_wake(); + } + + let mut waited: u32 = 0; + while self.captures_running.load(Relaxed) != 0 { + if waited >= SLEEP_DRAIN_MS { + dev_warn!( + self.dev, + "Touch ID: capture did not drain within {} ms; aborting suspend\n", + SLEEP_DRAIN_MS + ); + return Err(EBUSY); + } + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); + waited += 10; + } + Ok(()) } /// The system has resumed, or the suspend was aborted. diff --git a/drivers/soc/apple/sensor_shim.c b/drivers/soc/apple/sensor_shim.c index ee525b378d3a8d..76ac37b308e003 100644 --- a/drivers/soc/apple/sensor_shim.c +++ b/drivers/soc/apple/sensor_shim.c @@ -20,7 +20,6 @@ #include #include -#include #include "shim.h" diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index afa7305e291d7d..89f24ed62e60bc 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2262,7 +2262,7 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) { + if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_enrolment(); } this.run_enrolment(); @@ -2275,7 +2275,7 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) { + if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_verify(); } this.run_verify(); diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 47e63602756041..66b39903c6d81f 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -25,6 +25,8 @@ void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ +int sep_pm_register(int (*event)(bool entering)); +void sep_pm_unregister(void); int sep_pm_register(void (*event)(bool entering)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); From 9302864d6c9f91e49a387605cae6c498e2e5c1ba Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:40:09 +1100 Subject: [PATCH 229/631] arm64: dts: apple: t8140: Describe the internal display coprocessor The T8140 device tree has the display DARTs and power states but no display coprocessor, so the loader finds nothing to hand the running display over to and Linux is left with the boot framebuffer. Add the internal DCP, its ASC mailbox, the PMGR bandwidth registers it names, its two display clocks and the display subsystem node, following the layout used for the other Apple SoCs: - The DCP register ranges are the coprocessor control block and its I/O vector base from the Apple Device Tree's dcp node, and the first five disp0 ranges. Its DART stream (23) and the scanout and PIODMA streams (0 and 16) come from the matching DART mapper nodes. - The mailbox interrupts are the four dcp interrupts, in the same send-empty, send-not-empty, recv-empty, recv-not-empty order as the external mailbox. - Both display DARTs get the 64 GiB VM window at 0x10000000000 that the Apple Device Tree gives those streams. - The bandwidth scratch and request registers sit 0x20000 and 0x68000 into the display clock window at 0x302800000, the last disp0 range. - The clock rates are disp0's clock IDs 0x154 and 0x194: 300 MHz and 450 MHz. The DCP is handed over running with its firmware mapped through locked DARTs, so it stays disabled here, like its DARTs. The loader enables both once it has reserved those mappings; the display subsystem and the DCP carry explicit phandles because the loader refers to them. A single zero in apple,iomfb-surfaces selects surface 0. Kernels that read the property as a list of surfaces select the same one. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 90a769b7cfa63c..e793d7ed2ee418 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1338,6 +1338,73 @@ status = "disabled"; }; + /* + * Bandwidth scratch and request registers, 0x20000 and 0x68000 + * into the display clock window at 0x302800000 (the sixth disp0 + * register range in the Apple Device Tree). + */ + pmgr_dcp: power-management@302820000 { + reg = <0x3 0x02820000 0x0 0x8>, + <0x3 0x02868000 0x0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + dcp: dcp@412e00000 { + compatible = "apple,t8140-dcp", "apple,dcp"; + reg = <0x4 0x12e00000 0x0 0x4000>, + <0x4 0x10000000 0x0 0xafc000>, + <0x4 0x12240000 0x0 0x10000>, + <0x4 0x123a8000 0x0 0x4000>, + <0x4 0x123ac000 0x0 0x4000>, + <0x4 0x12800000 0x0 0x800000>, + <0x4 0x12850000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + power-domains = <&ps_disp_cpu>; + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + /* + * A single zero selects surface 0; J700 kernels that + * read this as a surface list take it the same way. + */ + apple,iomfb-surfaces = <0>; + /* generate a phandle for the loader */ + phandle = <&dcp>; + /* bus properties for the piodma child device */ + #address-cells = <2>; + #size-cells = <2>; + ranges; + /* the loader enables it when it hands the display over */ + status = "disabled"; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; + }; + + dcp_mbox: mailbox@412e08000 { + compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x12e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; + status = "disabled"; + }; + /* * Perf-state floors of voter agents 0-3, and the floor the PMC * honours for each agent and the interface enable word of each From d71af3beeb6286809e804ea2260510e7c2a36f57 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:20 +1100 Subject: [PATCH 230/631] soc: apple: sep: rename the attestation public key member struct sep_bio_attest calls the attested public point "public", which is a C++ keyword. The UAPI header test compiles every exported header as C++ as well as C, so a configuration with CONFIG_UAPI_HEADER_TEST=y fails to build: ./usr/include/linux/apple-sep.h:115:8: error: expected member name or ';' after declaration specifiers and C++ programs cannot include the header at all. Rename the member to public_key, and rename the Rust mirror of the structure to match. Only the member name changes: every structure keeps its size, alignment and member offsets, and the ioctl numbers are unchanged. No in-tree user space accesses the member. Fixes: 5b4a74c4e0aa ("soc: apple: sep: export the biometric ioctl header through UAPI") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 2 +- drivers/soc/apple/sbio.rs | 4 ++-- include/uapi/linux/apple-sep.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index bd66d0011f3890..09935233924553 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -152,7 +152,7 @@ pub(crate) struct Attest { /// in: challenge to sign. pub(crate) challenge: [u8; ATTEST_CHALLENGE_LEN], /// out: ref-key public point. - pub(crate) public: [u8; ATTEST_PUB_LEN], + pub(crate) public_key: [u8; ATTEST_PUB_LEN], /// out: DER signature, zero-padded to `ATTEST_SIG_MAX`. pub(crate) signature: [u8; ATTEST_SIG_MAX], pub(crate) reserved: [u8; 3], diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index c0b0b846e2733f..ecf6fe488ea944 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3142,11 +3142,11 @@ impl SepData { let mut out = bio::Attest { sig_len: sig.len() as u32, challenge: req.challenge, - public: [0u8; bio::ATTEST_PUB_LEN], + public_key: [0u8; bio::ATTEST_PUB_LEN], signature: [0u8; bio::ATTEST_SIG_MAX], reserved: [0u8; 3], }; - out.public.copy_from_slice(&pubk); + out.public_key.copy_from_slice(&pubk); out.signature[..sig.len()].copy_from_slice(&sig); kernel::uaccess::UserSlice::new(user, core::mem::size_of::()) .writer() diff --git a/include/uapi/linux/apple-sep.h b/include/uapi/linux/apple-sep.h index 81467fe91f36a0..394b497ff7a59c 100644 --- a/include/uapi/linux/apple-sep.h +++ b/include/uapi/linux/apple-sep.h @@ -112,7 +112,7 @@ struct sep_bio_delete { struct sep_bio_attest { __u32 sig_len; /* out: DER signature length */ __u8 challenge[SEP_BIO_CHALLENGE_LEN]; /* in */ - __u8 public[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ + __u8 public_key[SEP_BIO_ATTEST_PUB_LEN]; /* out: P-256 point, 04||X||Y */ __u8 signature[SEP_BIO_ATTEST_SIG_MAX]; /* out: DER SEQUENCE{r,s} */ __u8 reserved[3]; }; From 69e23eb9faee40adca619adf13786ac89f75d84b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:20:24 +1100 Subject: [PATCH 231/631] PCI/DOE: Export pci_doe_disconnected() for modular host controllers pci_dev_set_disconnected() is a static inline in drivers/pci/pci.h that a host controller driver can pass to pci_walk_bus() to mark a hierarchy it has lost as permanently disconnected before tearing it down. Besides setting the error state, it cancels pending DOE exchanges through pci_doe_disconnected(), which is not exported, so modpost rejects a modular caller with 'ERROR: modpost: "pci_doe_disconnected" [drivers/pci/controller/pcie-apple.ko] undefined!'. The Apple PCIe driver is such a caller since it learned to quiesce tunneled PCIe-C links, and it is tristate. Distribution configurations build APPLE_DART as a module, and "depends on APPLE_DART || !APPLE_DART" then makes PCIE_APPLE a module too. Export pci_doe_disconnected() GPL-only, like the rest of the DOE API. Open-coding the walk in the driver would skip the DOE cancellation and leave exchanges waiting on a device that can no longer answer. Fixes: 544b71a5d144 ("PCI: apple: quiesce and restore tunneled PCIe-C links") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/doe.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index 7b41da4ec11a39..44be0d513a8935 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -891,3 +891,4 @@ void pci_doe_disconnected(struct pci_dev *pdev) xa_for_each(&pdev->doe_mbs, index, doe_mb) pci_doe_cancel_tasks(doe_mb); } +EXPORT_SYMBOL_GPL(pci_doe_disconnected); From f31a2deb23ccb6219efbbb448f154a6792a788fe Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:57:12 +1100 Subject: [PATCH 232/631] soc: apple: sep: rearm the cold-restore device-view proof The nonce-session restore path synchronizes the device view before COMPLETE_INIT to refresh the SCRD credential. Deferred restore readiness also uses device_view_synced to guard its final proof after COMPLETE_INIT. The preliminary sync leaves that flag set, so both finish_sensor_setup and retry_restore_proof skip the final proof and templates_restored remains false after a successful cold restore. Clear device_view_synced when deferring template readiness. Keep the preliminary credential refresh, but require the final device-view proof before marking templates ready. A failed final sync can still take the existing bounded sensor bring-up retry. Legacy profiles retain their normal proof path, and refused or empty identity proofs stay unready. Fixes: 42ae23434d4c ("soc: apple: compose SEP persistence with authenticated sensor sessions") Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit b4c0368990089654d5e1212926473137ac314956) --- drivers/soc/apple/sbio.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ecf6fe488ea944..7e49ff2013bdf6 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3211,6 +3211,9 @@ impl SepData { // verification available on the preliminary Catacomb proof alone; // complete_bringup finishes the proof and sets templates_restored. self.templates_restored.store(false, Relaxed); + // The nonce path synced the view before COMPLETE_INIT to refresh SCRD. + // Re-arm the final proof and its bounded retry on every cold restore. + self.device_view_synced.store(false, Relaxed); if !candidate { dev_err!( self.dev, From 6f96e6d87cb12102346971d416c2425af57428af Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:34:23 +1100 Subject: [PATCH 233/631] pmdomain: apple: drain PMP report users before teardown The report mapping and allocation are managed by the platform device, but the global RCU list and fast-die debugfs file outlive driver removal. Withdraw both publications and drain readers before devres releases the mapping. Install the teardown action before publishing the region so probe failures also revoke it. The existing DVFS debugfs action still runs before the mapping is released. Depopulate child providers first. Their power callbacks now look up the report inside an RCU read-side section instead of following a child's unmanaged parent pointer. Remove the child provider and generic domain on teardown, including failed probe paths. If genpd refuses removal of a busy domain, keep its independently allocated storage, name and node references alive, with callbacks returning ENODEV. This bounds the leak to the removed domain and avoids reclaiming memory still owned by genpd. Publish the report before populating children so their initial power requests can use the same lookup. Validate report bit indices and use a 64-bit mask for the 64-bit request register. Fixes: f69f1b76fcb0 ("pmdomain: apple: pmp-report: expose T8140 fast-die effort") Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit 206b9b56f4b2bbdfc15efee3f23f5e5167f76bf8) --- drivers/pmdomain/apple/pmp-report.c | 127 +++++++++++++++++++++++++--- 1 file changed, 114 insertions(+), 13 deletions(-) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 3734d578a364ca..b6e7e4f0c3ce0a 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -623,6 +624,7 @@ static const struct { { "tp001m", "Tp001m", 0x69420 }, { "tp002m", "Tp002m", 0x694c4 }, struct list_head list; + struct dentry *debugfs; }; #define T6030_PMP_TEMP_CHANNELS (1 + ARRAY_SIZE(apple_pmp_t6030_temps)) @@ -1187,6 +1189,21 @@ static int apple_pmp_report_status_show(struct seq_file *s, void *unused) } DEFINE_SHOW_ATTRIBUTE(apple_pmp_report_status); +static void apple_pmp_report_release(void *data) +{ + struct apple_pmp_report *rep = data; + + /* Withdraw child providers while their power callbacks can still use MMIO. */ + of_platform_depopulate(rep->dev); + mutex_lock(&apple_pmp_reports_lock); + if (!list_empty(&rep->list)) + list_del_rcu(&rep->list); + mutex_unlock(&apple_pmp_reports_lock); + synchronize_rcu(); + /* The debugfs proxy drains active reads and rejects subsequent accesses. */ + debugfs_remove(rep->debugfs); +} + static int apple_pmp_report_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1241,6 +1258,10 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret) return ret; + INIT_LIST_HEAD(&rep->list); + ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); + if (ret) + return ret; ret = of_platform_populate(np, NULL, NULL, dev); if (ret && !rep->offsets->starts_pmp) return dev_err_probe(dev, ret, "failed to create child devices\n"); @@ -1250,6 +1271,14 @@ static int apple_pmp_report_probe(struct platform_device *pdev) mutex_unlock(&apple_pmp_reports_lock); if (rep->offsets->fast_die) + rep->debugfs = debugfs_create_file(dev_name(dev), 0400, + apple_pmp_report_debugfs, rep, + &apple_pmp_report_status_fops); + + /* Publish the region before child power callbacks start looking it up. */ + ret = of_platform_populate(np, NULL, NULL, dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create child devices\n"); debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, rep, &apple_pmp_report_status_fops); if (rep->offsets->starts_pmp) { @@ -1386,11 +1415,13 @@ static struct platform_driver apple_pmp_report_driver = { }; struct apple_pmp_report_entry { - struct device *dev; + struct device_node *node; + struct device_node *report_node; struct generic_pm_domain genpd; u32 id; /* The PMP does not acknowledge this request. */ bool no_ack; + bool removed; }; #define genpd_to_apple_pmp_report_entry(_genpd) \ @@ -1398,9 +1429,7 @@ struct apple_pmp_report_entry { static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) { - struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); - struct apple_pmp_report *rep = dev_get_drvdata(ent->dev->parent); - u64 bit_val = BIT_ULL(ent->id); + u64 bit_val = BIT_ULL(id); u64 val; unsigned long flags; @@ -1422,6 +1451,26 @@ static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enab 50000); } +static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) +{ + struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); + struct apple_pmp_report *rep; + int ret = -ENODEV; + + if (READ_ONCE(ent->removed)) + return ret; + + rcu_read_lock(); + list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { + if (rep->dev->of_node == ent->report_node) { + ret = apple_pmp_report_set_state(rep, ent->id, enable); + break; + } + } + rcu_read_unlock(); + return ret; +} + static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) { return apple_pmp_report_set_state(genpd, true); @@ -1502,6 +1551,28 @@ static int apple_pmp_dvfs_create(struct apple_pmp_report *report) return 0; } +static void apple_pmp_report_entry_free(struct apple_pmp_report_entry *ent) +{ + of_node_put(ent->report_node); + of_node_put(ent->node); + kfree(ent->genpd.name); + kfree(ent); +} + +static void apple_pmp_report_entry_remove(struct platform_device *pdev) +{ + struct apple_pmp_report_entry *ent = platform_get_drvdata(pdev); + + WRITE_ONCE(ent->removed, true); + of_genpd_del_provider(ent->node); + if (pm_genpd_remove(&ent->genpd)) { + /* The core still owns this domain. Keep its now inert storage alive. */ + dev_warn(&pdev->dev, "retaining busy power domain %s\n", ent->genpd.name); + return; + } + apple_pmp_report_entry_free(ent); +} + static int apple_pmp_report_entry_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1512,19 +1583,36 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) const char *name; struct of_phandle_iterator it; - ent = devm_kzalloc(dev, sizeof(*ent), GFP_KERNEL); + ent = kzalloc_obj(*ent); if (!ent) return -ENOMEM; - ent->dev = dev; + ent->node = of_node_get(node); + ent->report_node = of_node_get(dev->parent->of_node); + ret = of_property_read_u32(node, "reg", &ent->id); + if (ret) { + ret = dev_err_probe(dev, ret, "missing reg property\n"); + goto err_free; + } + if (ent->id >= 64) { + ret = -EINVAL; + goto err_free; + } ret = apple_pmp_report_bit(dev_get_drvdata(dev->parent), node, &ent->id, &record); if (ret) return dev_err_probe(dev, ret, "no request bit (reg, soc-device record)\n"); ret = of_property_read_string(node, "label", &name); - if (ret < 0) - return dev_err_probe(dev, ret, "missing label property\n"); + if (ret < 0) { + ret = dev_err_probe(dev, ret, "missing label property\n"); + goto err_free; + } + ent->genpd.name = kstrdup(name, GFP_KERNEL); + if (!ent->genpd.name) { + ret = -ENOMEM; + goto err_free; + } ent->no_ack = ((struct apple_pmp_report *)dev_get_drvdata(dev->parent))->offsets->starts_pmp && of_property_read_bool(node, "apple,no-ack"); @@ -1538,12 +1626,16 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) ent->genpd.power_off = apple_pmp_report_entry_power_off; ret = pm_genpd_init(&ent->genpd, NULL, true); - if (ret) - return dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + goto err_free; + } ret = of_genpd_add_provider_simple(node, &ent->genpd); - if (ret) - return dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + if (ret) { + ret = dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + goto err_remove_domain; + } of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { struct of_phandle_args parent, child; @@ -1566,11 +1658,19 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) } pm_genpd_remove_device(dev); + platform_set_drvdata(pdev, ent); return 0; err_remove: of_genpd_del_provider(node); - pm_genpd_remove(&ent->genpd); +err_remove_domain: + WRITE_ONCE(ent->removed, true); + if (pm_genpd_remove(&ent->genpd)) { + dev_warn(dev, "retaining busy power domain %s\n", ent->genpd.name); + return ret; + } +err_free: + apple_pmp_report_entry_free(ent); return ret; } @@ -1581,6 +1681,7 @@ static const struct of_device_id apple_pmp_report_entry_of_match[] = { static struct platform_driver apple_pmp_report_entry_driver = { .probe = apple_pmp_report_entry_probe, + .remove = apple_pmp_report_entry_remove, .driver = { .name = "apple-pmp-report-entry", .of_match_table = apple_pmp_report_entry_of_match, From 10ea15011455d2929dbcac732b85aa119e9073dd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 21:34:26 +1100 Subject: [PATCH 234/631] rtc: apple-pmu: distinguish alarm range errors from expiration The RTC core treats ETIME as an expired timer, retaining it on enqueue and retrying from irqwork without using its bounded error retry count. Returning it for a future alarm beyond the one-hour limit therefore keeps the worker under ops_lock until the alarm enters that horizon. Reserve ETIME for deadlines at or before the current counter. Return ERANGE for unsupported future deadlines so enqueue removes the timer and returns an error, and worker reprogramming uses its bounded error path. Advertise the same one-hour horizon through alarm_offset_max. Fixes: ff07618c82bb ("rtc: add opt-in J700 Baku PMU alarms with retained s2idle interrupts") Assisted-by: LLM Signed-off-by: Ryan Murray (cherry picked from commit d3bf9897f2d5cbb6a9169d776b7d946fafb927c1) --- drivers/rtc/rtc-apple-pmu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 7853b72a1b5526..a6842332058032 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -126,8 +126,10 @@ static int baku_rtc_validate_alarm(struct baku_rtc *baku, u64 comparator) error = baku_rtc_seconds(baku, &counter); if (error) return error; - if (comparator <= counter || comparator - counter > BAKU_RTC_MAX_DELAY) + if (comparator <= counter) return -ETIME; + if (comparator - counter > BAKU_RTC_MAX_DELAY) + return -ERANGE; return 0; } @@ -314,6 +316,7 @@ static int baku_rtc_probe(struct platform_device *pdev) baku->clock->ops = &baku_rtc_ops; baku->clock->range_min = 0; baku->clock->range_max = RTC_TIMESTAMP_END_9999; + baku->clock->alarm_offset_max = BAKU_RTC_MAX_DELAY; clear_bit(RTC_FEATURE_UPDATE_INTERRUPT, baku->clock->features); error = device_init_wakeup(dev, true); if (error) From 7926c504d1cf429273fb593b8ce08173fc05a18a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 8 Oct 2026 23:48:39 +1100 Subject: [PATCH 235/631] drm: apple: locate legacy command replies in the saved record DCP firmware answers an AP command by writing the reply in place into the command record, then sending an ack whose offset and length fields are zero. This holds for H17P and, as traced on t8112 and t6000 with 13.5 firmware, for older firmware too. The legacy path instead located the reply through the ack: it read a packet header at offset zero of the ack context's area. That only finds the right record for an unnested command, which dcp_push() places at offset zero of the command area. A command sent while a callback is being handled goes after the callback in the command area, but its ack arrives on the callback context, whose offset zero holds the callback record. Since commit c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") the lengths in that header are compared with the command's, so the first command the boot sequence sends from a callback fails the check. The DCP is then marked crashed and the internal panel is left without modes. Use the record address saved by dcp_push() on all firmware, as H17P already does. On older firmware, keep validating the reply before it is dispatched by requiring the record header to still match the one that was sent. Ignore an ack on an idle context on all firmware instead of underflowing the call depth. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 4 +-- drivers/gpu/drm/apple/iomfb.c | 45 ++++++++++++++++------------ 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ce8f0ba9b97ded..947ef987300a84 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -120,8 +120,8 @@ struct dcp_channel { dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; void *cookies[DCP_MAX_CALL_DEPTH]; void *output[DCP_MAX_CALL_DEPTH]; - u32 in_len[DCP_MAX_CALL_DEPTH]; - u32 out_len[DCP_MAX_CALL_DEPTH]; + /* Header of each pending AP command record, as dcp_push() wrote it */ + struct dcp_packet_header header[DCP_MAX_CALL_DEPTH]; u16 end[DCP_MAX_CALL_DEPTH]; /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8c89ad4f493279..0c6d6fdc0dbb72 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -603,8 +603,7 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca ch->callbacks[depth] = cb; ch->cookies[depth] = cookie; ch->output[depth] = out + sizeof(header) + in_len; - ch->in_len[depth] = in_len; - ch->out_len[depth] = out_len; + ch->header[depth] = header; ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); dcp_send_message(dcp, IOMFB_ENDPOINT, @@ -875,7 +874,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, { struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); - bool h17p = dcp->fw_compat == DCP_FIRMWARE_H17P; + const struct dcp_packet_header *record, *sent; void *cookie, *out; dcp_callback_t cb; @@ -884,7 +883,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, return; } - if (h17p && !ch->depth) { + if (!ch->depth) { dev_warn(dcp->dev, "ignoring ack on idle context %X\n", context); return; } @@ -893,27 +892,35 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, cb = ch->callbacks[ch->depth]; cookie = ch->cookies[ch->depth]; + out = ch->output[ch->depth]; + sent = &ch->header[ch->depth]; ch->callbacks[ch->depth] = NULL; ch->cookies[ch->depth] = NULL; + ch->output[ch->depth] = NULL; - if (h17p) { - /* - * H17P acks carry no payload and offset zero, including for - * nested calls, so the output cannot be located from the ack - * message. It is in the original AP command record, whose - * address dcp_push() saved. - */ - out = ch->output[ch->depth]; - ch->output[ch->depth] = NULL; - } else { - if (header->in_len != ch->in_len[ch->depth] || - header->out_len != ch->out_len[ch->depth]) { - dev_err(dcp->dev, "invalid command response lengths\n"); - dcp->crashed = true; + /* + * An ack completes the innermost pending command on its context. The + * firmware writes the reply in place into that AP command record, whose + * address dcp_push() saved; the ack itself carries neither offset nor + * length on any firmware. A command sent from within a callback is + * acked on the callback context, whose receive area holds the callback + * record rather than the reply, so the ack cannot be used to find it. + * + * Older firmware leaves the record header untouched. If it no longer + * matches what was sent, the record was overwritten and its output + * cannot be trusted. + */ + if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + record = out - sent->in_len - sizeof(*record); + if (memcmp(record, sent, sizeof(*record))) { + dev_err(dcp->dev, + "corrupt %c%c%c%c command record on context %X\n", + sent->tag[3], sent->tag[2], sent->tag[1], + sent->tag[0], context); + WRITE_ONCE(dcp->crashed, true); return; } - out = data + sizeof(*header) + header->in_len; } if (cb) From 97090cca6245294368047edf9e587505256aba8e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:04:57 +1100 Subject: [PATCH 236/631] drm: apple: accept H17P runtime callbacks within their reply bounds The H17P callback dispatcher only accepts callbacks whose record lengths appear in the startup observations. Every other callback that has a handler is treated as unqualified and marks the coprocessor as crashed, which stops all presents until the next reboot. Twenty-six handled callbacks are missing from those observations because they arrive only after startup: present notifications, swap aborts, the backlight re-send request, the wall-clock and clock-rate queries, and several acknowledgements. A long session that sees any of them loses the internal panel even though the driver knows how to answer. Record, for each of them, the input its handler reads and the reply it writes, and accept any record at least that large. The dispatcher already zeroes the whole reply before the handler runs, so a longer reply carries zeroes beyond the handler's fields. The measured startup tuples stay exact, and tags without a handler are still answered with a zeroed reply as before. Add a KUnit case that every installed handler has bounds and that the exact and bounded rules hold. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.c | 57 ++++++++++++++++++++++++- drivers/gpu/drm/apple/iomfb_h17p.h | 4 ++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 28 ++++++++++++ 3 files changed, 88 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 2de0fbbd4620a6..cf708655b3b348 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -14,18 +14,32 @@ struct dcp_callback_size { u32 in_len; u32 out_len; bool valid; + /* Lengths bound what the handler reads and writes; no capture exists. */ + bool minimum; }; +/* + * Callbacks that only arrive after startup: present notifications, swap + * aborts, the backlight re-send request, wall-clock and clock-rate queries. + * Their record lengths are not in the startup observations, so accept any + * record that holds what the handler reads and the reply it writes. The + * dispatcher zeroes the rest of a longer reply. + */ +#define DCP_CB_AT_LEAST(in, out) \ + { .in_len = (in), .out_len = (out), .valid = true, .minimum = true } + /* R-IOMFB-startup-run21 and R-ANALYTICS Linux boundary observations. */ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [0] = { 0x0, 0x4, true }, [1] = { 0x0, 0x4, true }, + [2] = DCP_CB_AT_LEAST(0x0, 0x0), [3] = { 0x4, 0x14, true }, [6] = { 0x54, 0x50, true }, [100] = { 0x0, 0x0, true }, [101] = { 0x0, 0x4, true }, [102] = { 0x44, 0x0, true }, [104] = { 0x44, 0x0, true }, + [107] = DCP_CB_AT_LEAST(0x0, 0x0), [108] = { 0x0, 0x4, true }, [109] = { 0x0, 0x4, true }, [110] = { 0x0, 0x4, true }, @@ -33,34 +47,58 @@ static const struct dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { [112] = { 0x0, 0x4, true }, [113] = { 0x0, 0x4, true }, [114] = { 0x1044, 0x1004, true }, + [117] = DCP_CB_AT_LEAST(0x0, 0x0), + [118] = DCP_CB_AT_LEAST(0x0, 0x4), + [119] = DCP_CB_AT_LEAST(0x0, 0x4), + [120] = DCP_CB_AT_LEAST(0x0, 0x4), [121] = { 0x0, 0x4, true }, + [122] = DCP_CB_AT_LEAST(0x0, 0x1), [123] = { 0x0, 0x4, true }, + [124] = DCP_CB_AT_LEAST(0x0, 0x4), [125] = { 0x64, 0x24, true }, [127] = { 0x4, 0x4, true }, [128] = { 0x1008, 0x4, true }, [129] = { 0x40, 0x4, true }, + [130] = DCP_CB_AT_LEAST(0x22, 0x14), [201] = { 0xc, 0x10, true }, [202] = { 0x1a, 0x0, true }, [206] = { 0x0, 0x4, true }, [207] = { 0x0, 0x4, true }, + [208] = DCP_CB_AT_LEAST(0x0, 0x0), + [209] = DCP_CB_AT_LEAST(0x0, 0x8), [300] = { 0x10, 0x0, true }, [400] = { 0x4c, 0xc04, true }, [401] = { 0x50, 0xc, true }, + [404] = DCP_CB_AT_LEAST(0x0, 0x0), [406] = { 0x48, 0x0, true }, + [408] = DCP_CB_AT_LEAST(0x8, 0x8), [411] = { 0x10, 0x1c, true }, [413] = { 0x1048, 0x4, true }, [414] = { 0x50, 0x4, true }, [415] = { 0x4c, 0x4, true }, [451] = { 0x14, 0x1c, true }, + [452] = DCP_CB_AT_LEAST(0x18, 0x14), [454] = { 0x4, 0x1, true }, [552] = { 0x1044, 0x4, true }, [561] = { 0x1044, 0x4, true }, [563] = { 0x4c, 0x4, true }, [565] = { 0x48, 0x4, true }, + [567] = DCP_CB_AT_LEAST(0x0, 0x1), + [572] = DCP_CB_AT_LEAST(0x0, 0x4), [574] = { 0x4, 0x8, true }, [575] = { 0x58, 0x4c, true }, + [576] = DCP_CB_AT_LEAST(0x0, 0x0), [582] = { 0x8, 0x4, true }, + [583] = DCP_CB_AT_LEAST(0x0, 0x0), + [584] = DCP_CB_AT_LEAST(0x0, 0x0), + [585] = DCP_CB_AT_LEAST(0xe4, 0xe0), [590] = { 0x730, 0x0, true }, + [592] = DCP_CB_AT_LEAST(0x11, 0x0), + [593] = DCP_CB_AT_LEAST(0x4, 0x0), + [594] = DCP_CB_AT_LEAST(0x1, 0x0), + [595] = DCP_CB_AT_LEAST(0x0, 0x0), + [597] = DCP_CB_AT_LEAST(0x0, 0x1), + [598] = DCP_CB_AT_LEAST(0x0, 0x1), [599] = { 0x0, 0x0, true }, }; @@ -71,7 +109,11 @@ bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len) if (tag < 0 || tag >= ARRAY_SIZE(callback_sizes)) return false; size = &callback_sizes[tag]; - return size->valid && size->in_len == in_len && size->out_len == out_len; + if (!size->valid) + return false; + if (size->minimum) + return in_len >= size->in_len && out_len >= size->out_len; + return size->in_len == in_len && size->out_len == out_len; } struct dcp_h17p_hotplug_request { @@ -659,5 +701,18 @@ void iomfb_start_h17p(struct apple_dcp *dcp) dcp_start_signal(dcp, false, dcp_started, NULL); } +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) +/* Returns the first handled tag without reply bounds, or -1. */ +int iomfb_h17p_first_unbounded_callback(void) +{ + int tag; + + for (tag = 0; tag < IOMFB_MAX_CB; tag++) + if (cb_handlers[tag] && !callback_sizes[tag].valid) + return tag; + return -1; +} +#endif + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 94739402502974..44e524abd368ad 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -88,6 +88,10 @@ void iomfb_encode_backlight_h17p(struct dcp_present_h17p *wire, u32 nits, void iomfb_serialize_present_h17p(struct dcp_present_h17p *wire, const struct dcp_swap_submit_req_h17p *request); +#if IS_ENABLED(CONFIG_DRM_APPLE_KUNIT_TEST) +int iomfb_h17p_first_unbounded_callback(void); +#endif + #undef DCP_FW_VER #undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c2a71ffa8cdbf2..dcd26fb1a9ed33 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -68,9 +68,37 @@ static void h17p_opaque_x_parameter_test(struct kunit *test) KUNIT_EXPECT_EQ(test, property.value, 0U); } +static void h17p_callback_bounds_test(struct kunit *test) +{ + /* Every installed handler is reachable without failing the session. */ + KUNIT_EXPECT_EQ(test, iomfb_h17p_first_unbounded_callback(), -1); + + /* Startup observations stay exact. */ + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(114, 0x1044, 0x1004)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(114, 0x1044, 0x1008)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(590, 0x734, 0x0)); + + /* Runtime callbacks must hold what their handlers read and write. */ + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(209, 0x0, 0x8)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(209, 0x4, 0xc)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(209, 0x0, 0x4)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(593, 0x4, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(593, 0x2, 0x0)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(594, 0x4, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(594, 0x0, 0x0)); + KUNIT_EXPECT_TRUE(test, iomfb_validate_callback_h17p(585, 0xe4, 0xe0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(585, 0xe0, 0xe0)); + + /* Tags without a handler or out of range never validate. */ + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(5, 0x0, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(-1, 0x0, 0x0)); + KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(IOMFB_MAX_CB, 0x0, 0x0)); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), + KUNIT_CASE(h17p_callback_bounds_test), {} }; From ed57a05302885587507ccc1cf5b965fefe54c566 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:06:36 +1100 Subject: [PATCH 237/631] drm: apple: bound firmware register mapping requests The prepare callback maps a physical display register range for the coprocessor. Its size arrives from the firmware as a 64-bit value but is rounded into an int, so a large size wraps negative. The window check then compares a wrapped end address, and dma_map_resource() receives the sign-extended size. The descriptor ID is also taken from find_first_zero_bit() without checking for a full table, so a mapping requested while all 128 descriptors are in use writes past the array. A failed DMA mapping is only warned about and returned as the device address. Keep the size in 64 bits, refuse a zero size or a range that wraps, refuse when no descriptor is free, and refuse a failed DMA mapping before claiming a descriptor. Each refusal is the existing all-zero reply already used for addresses outside the display windows. A valid request is mapped and answered exactly as before. H17P accepts this callback at runtime from now on, which makes the bounds matter there too. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 02b26f476d747f..cf2d62f05fb93d 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -802,6 +802,18 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) } id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); + return (struct dcp_map_physical_resp){}; + } + + dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(dcp->dev, dva)) { + dev_err(dcp->dev, "failed to map phys address %llx size %llx\n", + req->paddr, size); + return (struct dcp_map_physical_resp){}; + } + set_bit(id, dcp->memdesc_map); dcp->memdesc[id].size = size; dcp->memdesc[id].reg = req->paddr; From 9de78a50f6a53d0118aa389e13cc5453cf0142e9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:09:58 +1100 Subject: [PATCH 238/631] drm: apple: finish H17P presents that the firmware aborts H17P reports an aborted present through its abort callback and never sends a completion for it. The driver only traced the callback, so the present stayed pending: the outbound queue waited until its ten-second timeout declared the coprocessor crashed, and the page-flip event was never signalled. Track the abort in the present state. After the submit reply, an abort ends the present like a rejected submission: the brightness reservation is retried, framebuffers displaced by the present are unarmed and stay retained for the next completed present, and the DRM event is signalled. An abort that arrives before the submit reply is recorded, and the reply then fails the present instead of waiting for a completion. An abort naming a present that is not in flight is only logged. Older firmware still only traces the callback. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.h | 23 ++++++++++++++++- drivers/gpu/drm/apple/iomfb_h17p_test.c | 33 +++++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 32 ++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 44e524abd368ad..94ac90be4247a9 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -21,6 +21,8 @@ struct dcp_present_state_h17p { u32 swap_id; bool pending; bool accepted; + /* The firmware aborted the present before its submit reply. */ + bool aborted; }; static inline bool @@ -31,6 +33,7 @@ dcp_present_begin_h17p(struct dcp_present_state_h17p *state, u32 swap_id) state->swap_id = swap_id; state->pending = true; state->accepted = false; + state->aborted = false; return true; } @@ -41,11 +44,29 @@ dcp_present_submit_h17p(struct dcp_present_state_h17p *state, u32 swap_id, if (!state->pending || state->accepted || state->swap_id != swap_id) return false; state->accepted = accepted; - if (!accepted) + if (!accepted || state->aborted) state->pending = false; return true; } +/* + * An aborted present never completes. Before its submit reply, the abort is + * recorded and the reply ends the present instead. + */ +static inline bool +dcp_present_abort_h17p(struct dcp_present_state_h17p *state, u32 swap_id) +{ + if (!state->pending || state->swap_id != swap_id) + return false; + if (state->accepted) { + state->pending = false; + state->accepted = false; + } else { + state->aborted = true; + } + return true; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index dcd26fb1a9ed33..70fecfe52c0ffc 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -95,10 +95,43 @@ static void h17p_callback_bounds_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, iomfb_validate_callback_h17p(IOMFB_MAX_CB, 0x0, 0x0)); } +static void h17p_present_abort_test(struct kunit *test) +{ + struct dcp_present_state_h17p state = {}; + + /* An abort after acceptance ends the present without a completion. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 7)); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); + KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 6)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 7)); + + /* An abort before the submit reply leaves the reply to finish it. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, state.aborted); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 8, true)); + KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 8)); + + /* The next present starts clean and completes normally. */ + KUNIT_ASSERT_TRUE(test, dcp_present_begin_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, state.aborted); + KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); + KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 9)); + KUNIT_EXPECT_FALSE(test, state.pending); +} + static struct kunit_case h17p_present_cases[] = { KUNIT_CASE(h17p_surface_wire_test), KUNIT_CASE(h17p_opaque_x_parameter_test), KUNIT_CASE(h17p_callback_bounds_test), + KUNIT_CASE(h17p_present_abort_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index cf2d62f05fb93d..2e4a5e2427b5a2 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1447,6 +1447,12 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted dcp->crashed = true; return false; } + /* An abort received before this reply turns acceptance into failure. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + accepted && dcp->present_state_h17p.aborted) { + dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + return false; + } #endif return true; } @@ -1861,6 +1867,32 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, info->width, info->height); } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +/* + * An aborted H17P present never completes. Finish it like a rejected one: + * keep the displaced framebuffers for the next completed present and signal + * the DRM event, so neither the queue nor userspace waits for a completion. + */ +static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) +{ + if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { + dev_warn(dcp->dev, "abort for present %u, which is not in flight\n", + swap_id); + return; + } + /* The submit reply has not arrived; it finishes the present. */ + if (dcp->present_state_h17p.pending) + return; + + dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } +} +#endif + static void dcpep_cb_abort_swap_ap_gated(struct apple_dcp *dcp, u32 *swap_id) { From a90c0275adea5e77356da26260cc5876ca54429f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:11:50 +1100 Subject: [PATCH 239/631] drm: apple: re-send the H17P brightness when the firmware asks The enable-backlight-message callback asks the AP to send the panel level again. On H17P the level travels only inside a present, so the existing handler, which sets the older firmware's update flag, sends nothing: the brightness policy sees no pending change and the queued brightness transaction returns without presenting. On the measured H17P profile, mark the effective level dirty and let the update worker queue a brightness present, which re-presents the retained scanout as for any other level change. A blanked panel is re-sent at zero and keeps its restore target. Nothing is sent before Linux controls the level, while a present already carries it, or more than once a second, so a request repeated after every brightness present cannot keep the queue busy. The receive path only schedules the worker; it never queues a transaction itself. Older firmware keeps its existing handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp_backlight.c | 24 +++++++++++++++++ drivers/gpu/drm/apple/dcp_backlight.h | 14 ++++++++++ drivers/gpu/drm/apple/dcp_backlight_test.c | 31 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 9 +++++++ 5 files changed, 81 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 947ef987300a84..926edcbc849653 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -517,6 +517,9 @@ struct apple_dcp { spinlock_t lock; struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); + /* Rate limit for firmware re-send requests. */ + unsigned long resent_at; + bool resent; } backlight; /* Hardirq, resume and expiry share one generation under this lock. */ spinlock_t hdmi_presence_lock; diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 726277a6f2079b..610de408d000c2 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -227,6 +227,30 @@ bool dcp_backlight_pending(struct apple_dcp *dcp) return pending; } +/* + * Called from the receive path, which must not queue a transaction itself; + * the caller schedules the update worker when this returns true. Requests + * are honoured at most once per second, so a firmware that repeats the + * request after every brightness present cannot keep the queue busy. + */ +bool dcp_backlight_resend(struct apple_dcp *dcp) +{ + unsigned long flags; + bool resend = false; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + if (!dcp->backlight.resent || + time_after(jiffies, dcp->backlight.resent_at + HZ)) { + resend = dcp_bl_resend(&dcp->backlight.state); + if (resend) { + dcp->backlight.resent = true; + dcp->backlight.resent_at = jiffies; + } + } + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + return resend; +} + unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp) { unsigned long flags; diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 2ad4abb4acd376..cb6dafd6bdf426 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -114,6 +114,19 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) return 0; } +/* + * The firmware asked for the level again. Re-send the effective level once + * Linux controls it, unless a present already carries or will carry it. + */ +static inline bool dcp_bl_resend(struct dcp_backlight_state *state) +{ + if (!state->ready || !state->controlled || state->dirty || + state->in_flight) + return false; + state->dirty = true; + return true; +} + /* * Called by the single outbound queue, with a pinned, accepted scanout or a * prepared replacement. Soft-off never releases it. A brightness-only present @@ -189,6 +202,7 @@ int dcp_backlight_prepare(struct apple_dcp *dcp, bool have_surface, struct dcp_backlight_present *present); bool dcp_backlight_complete(struct apple_dcp *dcp, u64 sequence, bool accepted); bool dcp_backlight_pending(struct apple_dcp *dcp); +bool dcp_backlight_resend(struct apple_dcp *dcp); unsigned int dcp_backlight_retry_delay(struct apple_dcp *dcp); #endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a130f6f4147127..441410bd5a84a2 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -214,6 +214,36 @@ static void backlight_inflight_dpms_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 140U); } +static void backlight_resend_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* Nothing is re-sent before Linux controls the level. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, true, 140, false, 0), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + state.controlled = true; + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 140U); + /* The present in flight already carries the level. */ + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + /* A blanked panel is re-sent dark, keeping the restore target. */ + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_EQ(test, state.target, 140U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -224,6 +254,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_surface_and_busy_test), KUNIT_CASE(backlight_reject_test), KUNIT_CASE(backlight_inflight_dpms_test), + KUNIT_CASE(backlight_resend_test), {} }; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 2e4a5e2427b5a2..60d66ff966cd81 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -914,6 +914,15 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, u8 *enabled) { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* H17P carries the level only in presents; ask for one more. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (dcp_has_panel(dcp) && dcp_backlight_active(dcp) && + dcp_backlight_resend(dcp)) + schedule_work(&dcp->bl_update_wq); + return; + } +#endif /* * Reapply brightness on the next swap after DPMS wake on panels * without mini-LED backlights. From 7d1ba8afc275ba6a7eaf1bb2dac140d2d62a32ee Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:12:30 +1100 Subject: [PATCH 240/631] drm: apple: expose the full H17P panel brightness range The H17P backlight is commanded in nits, and the firmware converts them with its own tables up to the panel ceiling from the device tree, 525 nits on J700. Takeover still clamps the exposed range to 509 nits, the end of the older firmware's DAC interpolation table, which H17P never uses. The top of the panel's range is therefore unreachable, and a loader level reported above 509 nits rejects the takeover hint, which leaves the backlight unregistered until a later report falls below it. Use the panel ceiling as the H17P maximum, and clamp a powerlog report above it to the ceiling instead of rejecting it. The present encoding already writes the same ceiling into every present. Older firmware keeps its existing range. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.c | 3 ++- drivers/gpu/drm/apple/dcp_backlight.h | 10 ++++++---- drivers/gpu/drm/apple/dcp_backlight_test.c | 9 +++++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 610de408d000c2..f4e76830a71f6c 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -99,7 +99,8 @@ int dcp_backlight_takeover(struct apple_dcp *dcp, u32 millinits) if (dcp_backlight_active(dcp)) return 0; - maximum = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); + /* H17P takes nits up to the panel ceiling, not the older DAC table's. */ + maximum = dcp->brightness.maximum; ret = dcp_bl_takeover_nits(maximum, millinits, &nits); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index cb6dafd6bdf426..75242111bd1dea 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -6,6 +6,7 @@ #include #include +#include #include /* Commanded nits, independent of firmware encoding. No fade policy. */ @@ -30,14 +31,15 @@ struct dcp_backlight_present { u32 nits; }; -/* Public powerlog reports use millinits; validate before rounding to nits. */ +/* + * Public powerlog reports use millinits. A report above the panel ceiling + * describes the brightest level the panel can show, so clamp it. + */ static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) { if (!maximum || maximum > INT_MAX) return -EINVAL; - if ((u64)millinits > (u64)maximum * 1000) - return -ERANGE; - *nits = millinits / 1000; + *nits = min(millinits / 1000, maximum); return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 441410bd5a84a2..12e039d5ce0a10 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -13,9 +13,14 @@ static void backlight_millinits_takeover_test(struct kunit *test) KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(0, 100000, &nits), -EINVAL); KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(U32_MAX, 100000, &nits), -EINVAL); - KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), -ERANGE); - KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); KUNIT_EXPECT_EQ(test, nits, 123U); + /* Reports above the panel ceiling clamp to it. */ + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(525, 525999, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 525U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), 0); + KUNIT_EXPECT_EQ(test, nits, 509U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 100U); KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); From bb01376beee4e32c626d10108c2aee643ec52886 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:14:54 +1100 Subject: [PATCH 241/631] drm: apple: register the H17P backlight without a loader level H17P never publishes the panel level. The driver registers its backlight only when the system endpoint forwards a powerlog report with the loader's level. That report is optional: when it does not arrive, or arrives in a form the parser does not accept, no backlight device is ever registered and desktops offer no brightness control. Two seconds after the first pipe enable, register the backlight anyway if no report has configured it. Expose the middle of the panel range as the current level, but present nothing: the policy starts clean, so the panel keeps the loader's level until userspace writes one. A report that arrives later is ignored, as are reports after a hint-based registration. The worker is cancelled with the other backlight work on unbind and shutdown. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 38 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 926edcbc849653..ace1d32fbdad56 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -396,6 +396,8 @@ struct apple_dcp { struct mutex bl_register_mutex; /* Workqueue for updating the brightness */ struct work_struct bl_update_wq; + /* Registers the H17P backlight when the loader reports no level. */ + struct delayed_work bl_fallback_wq; /* integrated panel if present */ struct dcp_panel panel; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0f18c50baba297..89be21810c71df 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3223,6 +3223,8 @@ int dcp_typec_prepare_route(struct apple_dcp *dcp) if (generation != dcp->typec_generation || !dcp->typec_cable_connected) return -ESTALE; return ready ? 0 : -ETIMEDOUT; +#define DCP_BL_FALLBACK_DELAY msecs_to_jiffies(2000) + static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || @@ -3248,6 +3250,10 @@ void dcp_poweron(struct platform_device *pdev) WRITE_ONCE(dcp->crashed, true); return; } + /* The powerlog takeover report normally precedes this. */ + if (dcp_has_panel(dcp)) + schedule_delayed_work(&dcp->bl_fallback_wq, + DCP_BL_FALLBACK_DELAY); } ret = dcp_backlight_dpms(dcp, true); if (ret) @@ -3452,6 +3458,34 @@ static void dcp_work_update_backlight(struct work_struct *work) dcp_backlight_update(dcp); } +/* + * H17P never publishes the panel level, and the loader's level arrives only + * as an optional powerlog report. Without one, register the backlight at + * the middle of the panel range. That value is only reported: no level is + * presented until userspace writes one, so the panel keeps the loader's + * level. + */ +static void dcp_work_backlight_fallback(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, bl_fallback_wq); + u32 nits = dcp->brightness.maximum / 2; + int ret; + + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || + dcp_backlight_active(dcp)) + return; + + ret = iomfb_configure_backlight_h17p(dcp, dcp->brightness.maximum, + false, 0, true, nits); + if (!ret) + dev_info(dcp->dev, + "no loader brightness reported; backlight starts at %u nits without a panel change\n", + nits); + else if (ret != -EBUSY) + dev_warn(dcp->dev, "backlight registration failed: %d\n", ret); +} + static void dcp_release_piodma_iommu_dev(struct apple_dcp *dcp) { if (dcp->retain_dma) { @@ -4288,6 +4322,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -4358,6 +4393,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_queue_stop(dcp); dcp_release_piodma_iommu_dev(dcp); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -4547,6 +4583,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) disable_work(&dcp->vblank_wq); disable_delayed_work(&dcp->swap_watchdog_wq); } + INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); @@ -4916,6 +4953,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) * its DMA resources alive until the system resets. */ iomfb_queue_stop(dcp); + cancel_delayed_work_sync(&dcp->bl_fallback_wq); cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); return; From 63a2d5f66617e45fe17fdaf85ac6b6633fc3fdbf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:15:31 +1100 Subject: [PATCH 242/631] drm: apple: drain H17P display updates before system sleep H17P keeps its pipe and display domains running across system sleep, and blanks the panel with a brightness present instead. The DRM and backlight suspend handlers only queue that present: the serialized IOMFB queue sends it, and finishes it when the firmware reports the completion. The DCP suspend handler returned without waiting, so the blanking present could still be in flight when interrupts are suspended, leaving the panel lit through the sleep and the transaction stalled until resume. Before the DCP suspends, flush the brightness worker and wait up to half a second for the queue to go idle, twice, since a completed present can schedule one more brightness present. A crashed or stopped queue counts as idle. If the queue does not drain, log it and continue suspending. Older firmware does not use the queue and is unaffected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 22 ++++++++++++++++++++++ drivers/gpu/drm/apple/iomfb.c | 26 ++++++++++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ace1d32fbdad56..d20cc9b3a46a81 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -574,6 +574,7 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool default_valid, u32 default_nits); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); +bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 89be21810c71df..e7352e1da52127 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5043,6 +5043,27 @@ static void dcp_platform_complete(struct device *dev) mutex_lock(&dcp->hpd_mutex); WRITE_ONCE(dcp->external_suspended, false); mutex_unlock(&dcp->hpd_mutex); + * The display stays powered across system sleep, and the blanking present + * queued by the DRM and backlight suspend handlers must reach the firmware + * before its mailbox interrupt is suspended. + */ +static void dcp_drain_for_sleep(struct apple_dcp *dcp) +{ + int pass; + + if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + return; + + /* A completed present can schedule one more brightness present. */ + for (pass = 0; pass < 2; pass++) { + if (dcp_has_panel(dcp)) + flush_work(&dcp->bl_update_wq); + if (!iomfb_queue_drain(dcp, msecs_to_jiffies(500))) { + dev_warn(dcp->dev, "display updates still pending at suspend\n"); + return; + } + } } static int dcp_platform_suspend(struct device *dev) @@ -5086,6 +5107,7 @@ static int dcp_platform_suspend(struct device *dev) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + dcp_drain_for_sleep(dcp); /* * The Type-C route reports cable removal through * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 0c6d6fdc0dbb72..1b18ecdda9ed0f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -388,6 +388,32 @@ void iomfb_queue_stop(struct apple_dcp *dcp) cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } +/* + * Wait until every queued transaction, including a present's completion, has + * finished. A stopped or crashed queue has nothing left to deliver. + */ +bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + bool idle; + + if (!iomfb_uses_queue(dcp)) + return true; + + for (;;) { + mutex_lock(&dcp->iomfb.lock); + idle = READ_ONCE(dcp->crashed) || dcp->iomfb.stopped || + (!dcp->iomfb.active && list_empty(&dcp->iomfb.pending) && + !dcp->present_state_h17p.pending); + mutex_unlock(&dcp->iomfb.lock); + if (idle) + return true; + if (time_after(jiffies, deadline)) + return false; + usleep_range(2000, 4000); + } +} + int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { int ret = 0; From 983541665b315f5f4fc358c521ff1099471a717d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:15:58 +1100 Subject: [PATCH 243/631] drm: apple: signal pending DRM events when H17P firmware crashes When the H17P firmware crashes, the driver marks the coprocessor failed and reports the connector disconnected. A present in flight at that moment never completes, so its page-flip event is only released by the DRM core's flip timeout, and the compositor stalls on it first. Schedule the existing delayed vblank work from the crash handler on H17P so the pending event is signalled. Framebuffers stay retained, since the display engine may still be reading the last surface. H17P has no in-place restart, so also log that a reboot is needed to restore the display. Older firmware keeps its existing crash handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e7352e1da52127..ea78a60d1eff84 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1833,6 +1833,12 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + /* No in-place restart exists; the last scanout stays retained. */ + dev_err(dcp->dev, "a reboot is required to restore the display\n"); + /* Nothing will complete a present that was in flight. */ + schedule_work(&dcp->vblank_wq); + } if (dcp->connector) { dcp->connector->connected = 0; apple_connector_edid_set_live(dcp->connector, false); From 60f756e6d5dc4f919bc89e3efc0b642d1a535dcc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 00:16:49 +1100 Subject: [PATCH 244/631] drm: apple: keep the internal panel when the T8140 external DCP is enabled The display subsystem builds its component match from every available DCP node and binds only when all of them have bound. The T8140 external DCP runs firmware this driver cannot drive yet: its node falls back to the generic DCP match, which has no firmware profile for it. Its probe then either fails, leaving the component master waiting forever, or selects an older firmware interface from the loader's properties. Enabling that node in a device tree would therefore take the internal panel down with it. Leave T8140 external DCP nodes out of the component match and the pipeline scan, and refuse them in the DCP probe with an informational message, so they are never touched. Other SoCs are unaffected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 8 ++++++++ drivers/gpu/drm/apple/dcp.c | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index ed2aa49abb70cf..bf0c176af3e3dc 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -804,6 +804,14 @@ static void apple_drm_quiesce_connectors(struct drm_device *drm) disable_work_sync(&apple_connector->bl_commit_wq); } drm_connector_list_iter_end(&iter); + * The T8140 external display coprocessor needs firmware support this driver + * does not have yet. Leave it out of the display subsystem, so that an + * enabled node cannot hold back the internal panel's component master. + */ +static bool apple_dcp_usable(struct device_node *np) +{ + return of_device_is_available(np) && + !of_device_is_compatible(np, "apple,t8140-dcpext"); } static int apple_drm_init_dcp(struct device *dev) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index ea78a60d1eff84..3d90372b5362e1 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4494,6 +4494,12 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (dcp_session_retained(dev)) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); + /* Not part of the display subsystem yet; see apple_dcp_usable(). */ + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) { + dev_info(dev, "external display coprocessor not supported yet\n"); + return -ENODEV; + } + fw_compat = dcp_check_firmware_version(dev); if (fw_compat == DCP_FIRMWARE_UNKNOWN) return -ENODEV; From 274b566cbad3f268b5f3f4696f0d9d5e61875f9c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:20:12 +1100 Subject: [PATCH 245/631] drm: apple: only signal crash events on a linked H17P pipeline The H17P crash handler schedules the delayed vblank work, so that a present in flight at the time of the crash completes its page-flip event. That work dereferences dcp->crtc, which is set only when the display subsystem links the pipeline. RTKit is already running before that: component binding wakes the coprocessor before the DRM device creates its CRTCs, and a pipeline whose CRTC setup failed stays bound with no CRTC at all. A firmware crash in either state then oopses the workqueue on a NULL CRTC. Schedule the work from the crash handler only once a CRTC is linked, and make the delayed vblank work itself return early without one, which also covers its other callers. Publish the CRTC with WRITE_ONCE() to match the lockless readers. Also limit the crash-time completion to the measured H17P profile, as every other H17P change in this series is. The H17G profile (T8142) keeps its existing crash handling. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3d90372b5362e1..5a2c6f491e2647 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1642,9 +1642,15 @@ static void dcp_delayed_vblank(struct work_struct *work) { struct apple_dcp *dcp; + struct apple_crtc *crtc; + dcp = container_of(work, struct apple_dcp, vblank_wq); + /* RTKit can report a crash before the pipeline has a CRTC. */ + crtc = READ_ONCE(dcp->crtc); + if (!crtc) + return; mdelay(5); - dcp_drm_crtc_vblank(dcp->crtc); + dcp_drm_crtc_vblank(crtc); } #define DCP_SWAP_WATCHDOG_MS 1000 @@ -1833,11 +1839,16 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ dcp->crashed = true; dev_err(dcp->dev, "DCP has crashed\n"); - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { /* No in-place restart exists; the last scanout stays retained. */ dev_err(dcp->dev, "a reboot is required to restore the display\n"); - /* Nothing will complete a present that was in flight. */ - schedule_work(&dcp->vblank_wq); + /* + * Nothing will complete a present that was in flight. Before + * the pipeline is linked there is no event to complete. + */ + if (READ_ONCE(dcp->crtc)) + schedule_work(&dcp->vblank_wq); } if (dcp->connector) { dcp->connector->connected = 0; From da8da8ff242041d93ffcc856265e835059836bc3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:20:22 +1100 Subject: [PATCH 246/631] drm: apple: keep the H17P backlight policy when registration fails When the backlight class device cannot be registered, the worker zeroes the panel maximum so that older firmware stops encoding brightness. On H17P the brightness policy is already active by then, and the present path needs that maximum for the ceiling it writes into every present: with it zeroed, every present is refused and the panel freezes on its last frame. Registration is attempted on every H17P boot now, so this error path is always reachable. Keep the maximum while the H17P policy is active. Presents keep their ceiling and the policy keeps applying DPMS and the firmware's requests; only the sysfs control is missing. Log the error code in either case. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5a2c6f491e2647..d95cffef3c473a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3458,8 +3458,14 @@ static void dcp_work_register_backlight(struct work_struct *work) if (ret == -ENODATA) goto out_unlock; if (ret) { - dev_err(dcp->dev, "Unable to register backlight device\n"); - dcp->brightness.maximum = 0; + dev_err(dcp->dev, "Unable to register backlight device: %d\n", ret); + /* + * The H17P policy encodes the panel ceiling into every present + * and keeps working without a class device; only older + * firmware stops sending brightness here. + */ + if (!dcp_backlight_active(dcp)) + dcp->brightness.maximum = 0; } out_unlock: From 43af7d63d9d445665a2fabadbbe1b240e74b2bbc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:21:34 +1100 Subject: [PATCH 247/631] drm: apple: never re-send a guessed H17P backlight level Without a loader brightness report, the H17P backlight is registered with a mid-range default that is only reported to userspace; the panel keeps the loader's level. Registration also marks the policy as controlled, which is the only condition the firmware's re-send request checked. A re-send request before userspace writes a level therefore presented the default and moved the panel from the loader's level to mid-range, contrary to the intent of the re-send and fallback registration changes. Track whether the level the panel shows is known: the loader's reported level, or one a completed present carried. Require it for a re-send, so a re-send request now leaves a panel showing an unknown level alone. A rejected present does not make a level known. A DPMS blank and wake, or a suspend and resume, after a fallback registration still has to light the panel at some level and uses the reported default. The panel keeps the loader's level only until the first such transition or userspace write. Also reject powerlog reports above twice the panel ceiling instead of clamping them, so a corrupt sample cannot become a known level. Add a KUnit case for the fallback registration, the re-send request, rejected and completed presents, and the DPMS wake level. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 29 ++++++++++---- drivers/gpu/drm/apple/dcp_backlight_test.c | 46 +++++++++++++++++++++- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 75242111bd1dea..73683cb0322a39 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -24,6 +24,12 @@ struct dcp_backlight_state { bool suspended; bool dirty; bool in_flight; + /* + * The panel shows a level Linux knows: the loader's reported level, or + * one a completed present carried. A default used for registration + * without a loader report is not known until a present carries a level. + */ + bool level_known; }; struct dcp_backlight_present { @@ -32,18 +38,24 @@ struct dcp_backlight_present { }; /* - * Public powerlog reports use millinits. A report above the panel ceiling - * describes the brightest level the panel can show, so clamp it. + * Public powerlog reports use millinits. A report slightly above the panel + * ceiling describes the brightest level the panel can show, so clamp it; one + * more than twice the ceiling is not a credible panel level. */ static inline int dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) { - if (!maximum || maximum > INT_MAX) + if (!maximum || maximum > INT_MAX / 2000) return -EINVAL; + if (millinits / 1000 > 2 * maximum) + return -ERANGE; *nits = min(millinits / 1000, maximum); return 0; } -/* Inputs must come from admitted panel/loader records, never a guess. */ +/* + * An inherited level comes from a loader report. A default level is only + * reported to userspace; it is never presented on its own (see level_known). + */ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits) @@ -63,6 +75,7 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, .target = nits, .actual = nits, .ready = true, + .level_known = inherited_valid, }; return 0; } @@ -118,12 +131,13 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) /* * The firmware asked for the level again. Re-send the effective level once - * Linux controls it, unless a present already carries or will carry it. + * Linux controls it and knows it, unless a present already carries or will + * carry it. A default level is never sent this way. */ static inline bool dcp_bl_resend(struct dcp_backlight_state *state) { - if (!state->ready || !state->controlled || state->dirty || - state->in_flight) + if (!state->ready || !state->controlled || !state->level_known || + state->dirty || state->in_flight) return false; state->dirty = true; return true; @@ -168,6 +182,7 @@ static inline bool dcp_bl_complete(struct dcp_backlight_state *state, if (accepted) { state->actual = state->sent_nits; state->retries = 0; + state->level_known = true; } else { state->dirty = true; if (state->retries < 4) diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 12e039d5ce0a10..0b94b41b414827 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -19,8 +19,13 @@ static void backlight_millinits_takeover_test(struct kunit *test) KUNIT_EXPECT_EQ(test, nits, 525U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 509001, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 509U); - KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 1018999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 509U); + /* More than twice the ceiling is not a panel level. */ + nits = 123; + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, 1019000, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); + KUNIT_EXPECT_EQ(test, nits, 123U); KUNIT_ASSERT_EQ(test, dcp_bl_takeover_nits(509, 100999, &nits), 0); KUNIT_EXPECT_EQ(test, nits, 100U); KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 509, true, nits, false, 0), 0); @@ -249,6 +254,44 @@ static void backlight_resend_test(struct kunit *test) KUNIT_EXPECT_EQ(test, state.target, 140U); } +static void backlight_fallback_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A registration default is reported, never presented on its own. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EALREADY); + + /* A rejected userspace level does not become known. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 300, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 300U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_FALSE(test, state.level_known); + + /* Once a present carrying it completes, the level is re-sendable. */ + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.level_known); + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + + /* DPMS has to light the panel at some level: wake uses the default. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + state.controlled = true; + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 262U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -260,6 +303,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_reject_test), KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), + KUNIT_CASE(backlight_fallback_test), {} }; From a0ae48a03679dec15394bbee2a49969554b463cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:22:15 +1100 Subject: [PATCH 248/631] drm: apple: tolerate late H17P completions of aborted presents The H17P abort callback is handled as "this present never completes", but no capture confirms that. If the firmware still completes an aborted present, the completion matches no present in flight and the driver declares the coprocessor crashed, turning the first real abort into a dead display. Remember the most recently aborted present. A completion for it is logged and otherwise ignored: its displaced framebuffers were already unarmed and its DRM event signalled when it was aborted, and it does not disturb a later present in flight. Also settle the swap watchdog when an accepted present is aborted. Otherwise a Type-C output would report a missing completion and retrain one second later. Rate-limit the abort messages, so that a firmware that aborts every frame cannot flood the log. Extend the KUnit present-state case with stale completions. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p.h | 13 +++++++++++++ drivers/gpu/drm/apple/iomfb_h17p_test.c | 9 +++++++++ drivers/gpu/drm/apple/iomfb_template.c | 24 ++++++++++++++++++++---- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_h17p.h b/drivers/gpu/drm/apple/iomfb_h17p.h index 94ac90be4247a9..75856d1921562b 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.h +++ b/drivers/gpu/drm/apple/iomfb_h17p.h @@ -23,6 +23,9 @@ struct dcp_present_state_h17p { bool accepted; /* The firmware aborted the present before its submit reply. */ bool aborted; + /* The most recent aborted present, whose late completion is ignored. */ + bool has_last_aborted; + u32 last_aborted; }; static inline bool @@ -64,9 +67,19 @@ dcp_present_abort_h17p(struct dcp_present_state_h17p *state, u32 swap_id) } else { state->aborted = true; } + state->has_last_aborted = true; + state->last_aborted = swap_id; return true; } +/* Whether the firmware aborted @swap_id; a completion for it is stale. */ +static inline bool +dcp_present_was_aborted_h17p(const struct dcp_present_state_h17p *state, + u32 swap_id) +{ + return state->has_last_aborted && state->last_aborted == swap_id; +} + static inline bool dcp_present_complete_h17p(struct dcp_present_state_h17p *state, u32 swap_id) { diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index 70fecfe52c0ffc..c017651945a4e4 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -104,9 +104,13 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 7, true)); KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 6)); KUNIT_EXPECT_TRUE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 7)); KUNIT_EXPECT_TRUE(test, dcp_present_abort_h17p(&state, 7)); KUNIT_EXPECT_FALSE(test, state.pending); KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 7)); + /* A late completion for it is recognisable as stale, not unexpected. */ + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 7)); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 6)); KUNIT_EXPECT_FALSE(test, dcp_present_abort_h17p(&state, 7)); /* An abort before the submit reply leaves the reply to finish it. */ @@ -123,8 +127,13 @@ static void h17p_present_abort_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, state.aborted); KUNIT_ASSERT_TRUE(test, dcp_present_submit_h17p(&state, 9, true)); KUNIT_EXPECT_TRUE(test, state.pending); + /* A stale completion for 8 neither matches nor disturbs present 9. */ + KUNIT_EXPECT_FALSE(test, dcp_present_complete_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, dcp_present_was_aborted_h17p(&state, 8)); + KUNIT_EXPECT_TRUE(test, state.pending); KUNIT_EXPECT_TRUE(test, dcp_present_complete_h17p(&state, 9)); KUNIT_EXPECT_FALSE(test, state.pending); + KUNIT_EXPECT_FALSE(test, dcp_present_was_aborted_h17p(&state, 9)); } static struct kunit_case h17p_present_cases[] = { diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 60d66ff966cd81..74cde4c5e2d672 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -250,6 +250,18 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + /* + * Abort semantics are not captured on H17P. If an aborted + * present completes after all, its framebuffers were already + * unarmed and its event signalled; record it and carry on. + */ + if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + resp->swap_id)) { + dev_warn_ratelimited(dcp->dev, + "completion for aborted present %u ignored\n", + resp->swap_id); + return; + } dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); dcp->crashed = true; return; @@ -1459,7 +1471,8 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted /* An abort received before this reply turns acceptance into failure. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && accepted && dcp->present_state_h17p.aborted) { - dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", + swap_id); return false; } #endif @@ -1885,15 +1898,18 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) { if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { - dev_warn(dcp->dev, "abort for present %u, which is not in flight\n", - swap_id); + dev_warn_ratelimited(dcp->dev, + "abort for present %u, which is not in flight\n", + swap_id); return; } /* The submit reply has not arrived; it finishes the present. */ if (dcp->present_state_h17p.pending) return; - dev_warn(dcp->dev, "firmware aborted present %u\n", swap_id); + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", swap_id); + /* No completion follows, so the swap watchdog must not wait for one. */ + dcp_swap_watchdog_complete(dcp); dcp_present_failed(dcp); if (dcp_present_retires(dcp)) { dcp_unarm_retained_framebuffers(dcp, swap_id); From c309f03f8bab7d7579b3c85b1ce44fbdac2c6b06 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:22:39 +1100 Subject: [PATCH 249/631] drm: apple: settle H17P brightness retries before system sleep The sleep drain waited for the IOMFB queue to go idle, but a rejected blanking present leaves the queue idle with a delayed retry scheduled. The drain then returned, and the retry ran after the coprocessor's mailbox had suspended, leaving the panel lit through sleep. Run a pending brightness retry immediately in each drain pass, and keep draining while the brightness policy still has a level to send. Four passes cover the first attempt and the policy's three retries, each bounded as before; a level that is still pending after them is logged. Also queue the fallback backlight registration on the freezable system workqueue, so that it cannot register a class device in the middle of a suspend that starts within two seconds of the first pipe enable. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d95cffef3c473a..8a15145c068217 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3269,8 +3269,9 @@ void dcp_poweron(struct platform_device *pdev) } /* The powerlog takeover report normally precedes this. */ if (dcp_has_panel(dcp)) - schedule_delayed_work(&dcp->bl_fallback_wq, - DCP_BL_FALLBACK_DELAY); + queue_delayed_work(system_freezable_wq, + &dcp->bl_fallback_wq, + DCP_BL_FALLBACK_DELAY); } ret = dcp_backlight_dpms(dcp, true); if (ret) @@ -5084,15 +5085,24 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) return; - /* A completed present can schedule one more brightness present. */ - for (pass = 0; pass < 2; pass++) { + /* + * A completed present can schedule one more brightness present, and a + * rejected one a delayed retry; run a pending retry now instead of + * letting it fire after the mailbox has suspended. Four passes cover + * the first attempt and the policy's three retries. + */ + for (pass = 0; pass < 4; pass++) { + flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); if (!iomfb_queue_drain(dcp, msecs_to_jiffies(500))) { dev_warn(dcp->dev, "display updates still pending at suspend\n"); return; } + if (!dcp_backlight_active(dcp) || !dcp_backlight_pending(dcp)) + return; } + dev_warn(dcp->dev, "brightness change still pending at suspend\n"); } static int dcp_platform_suspend(struct device *dev) From 4abc2a41e51d2c6d19d10233916eac0d26c0bf65 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:23:14 +1100 Subject: [PATCH 250/631] drm: apple: report H17P runtime callback sizes that exceed their bounds The runtime callbacks accepted within reply bounds are validated against what their handlers read and write, because no capture of their record lengths exists. A longer record is accepted silently, so a layout that differs from the handler's assumption, for instance a status word at another offset, would go unnoticed. The first time each of these callbacks arrives with lengths other than its handler's, log the lengths seen next to the handler's. Each tag is reported at most once per boot, and the measured startup callbacks are unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/iomfb.c | 2 +- drivers/gpu/drm/apple/iomfb.h | 2 ++ drivers/gpu/drm/apple/iomfb_h17p.c | 22 ++++++++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index d20cc9b3a46a81..109c3ef75a6733 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -537,6 +537,8 @@ struct apple_dcp { struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); + /* Runtime callbacks whose record size has been reported once. */ + DECLARE_BITMAP(sized_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 1b18ecdda9ed0f..8b7a91cfe5014f 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -850,7 +850,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && - !iomfb_validate_callback_h17p(tag, hdr->in_len, hdr->out_len)))) { + !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 19293941f7b5d0..09f56ced0a1f54 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -210,6 +210,8 @@ struct dcp_method_entry { bool iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); struct apple_dcp; +bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, + u32 out_len); typedef bool (*iomfb_cb_handler)(struct apple_dcp *, int, void *, void *); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index cf708655b3b348..682a92879f8ef5 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -694,6 +694,28 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [599] = trampoline_nop, /* find_swap_function_gated */ }; +/* + * Validate a callback record and, the first time each runtime callback whose + * length is only bounded arrives with a different length, report the length + * actually seen. Those reports are the first capture of these records. + */ +bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, + u32 out_len) +{ + const struct dcp_callback_size *size; + + if (!iomfb_validate_callback_h17p(tag, in_len, out_len)) + return false; + size = &callback_sizes[tag]; + if (size->minimum && + (in_len != size->in_len || out_len != size->out_len) && + !test_and_set_bit(tag, dcp->sized_callbacks)) + dev_info(dcp->dev, + "callback D%03d record is %#x/%#x bytes; its handler uses %#x/%#x\n", + tag, in_len, out_len, size->in_len, size->out_len); + return true; +} + void iomfb_start_h17p(struct apple_dcp *dcp) { WRITE_ONCE(dcp->pipe_enabled_h17p, false); From d8cdb8d335eda5a6d4244618b1670ec08d8d604f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:46:35 +1100 Subject: [PATCH 251/631] PCI: apple: run the J700 radio bootstrap by default The J700 Wi-Fi and Bluetooth functions sit behind port 0 of the T8140 PCIe host, which only opens downstream configuration access after the PIODMA bootstrap has completed. The bootstrap provider is already selected per board: it only binds on J700, and only when the device tree points the host's "apple,piodma" property at the engine. It nevertheless also required pcie_apple_piodma_diag.enumerate=1 on the kernel command line, so a kernel built with the provider still left both radio functions undiscovered unless the boot entry carried that option. Make the device tree the switch and run the bootstrap by default. Keep the parameter so enumerate=0 still selects the root-port-only path for diagnosis, and update the Kconfig help and documentation to match. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/PCI/controller/apple-t8140.rst | 11 +++++++---- .../networking/device_drivers/wifi/mt7932-neo.rst | 7 +++---- drivers/pci/controller/Kconfig | 6 ++++-- drivers/pci/controller/pcie-apple-piodma-diag.c | 7 +++++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 6f41484ccf84a6..5e336b3a384812 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -7,10 +7,13 @@ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. The root bridge forwards their BARs and DART translates endpoint DMA. -This is an opt-in bring-up implementation, not general T8140 PCIe support. -Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y`` and pass -``pcie_apple_piodma_diag.enumerate=1`` to the kernel. Without the opt-in, the -root-only diagnostic path continues to reject downstream configuration access. +This is a bring-up implementation, not general T8140 PCIe support. +Build ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. The board device tree selects the +bootstrap by pointing the host's ``apple,piodma`` property at the engine, and +the kernel then runs it during the host probe. Passing +``pcie_apple_piodma_diag.enumerate=0`` keeps the root-only diagnostic path, +which rejects downstream configuration access. A kernel built without the +option always uses that root-only path. Build the host and its DART in as well: the supplier retains published DMA and has no removal contract. Keeping the supplier and host built in avoids a module dependency on a provider whose lifetime cannot end at unload. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index a36998f1010e9b..113f43884155c1 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -17,10 +17,9 @@ Platform and configuration -------------------------- Use the separately reviewed T8140 PCIe bootstrap described in -``Documentation/PCI/controller/apple-t8140.rst``. Enumeration is enabled by -default only on the J700/T8140; ``pcie_apple_piodma_diag.enumerate=0`` disables -it. Public m1n1 must populate the J700 ``wifi0`` endpoint's own -``local-mac-address``; a zero placeholder is rejected. +``Documentation/PCI/controller/apple-t8140.rst``. It runs during boot when +``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. Public m1n1 must populate the J700 +``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. The original hardware-tested configuration included ``CONFIG_MT7932_FULLMAC=m``, ``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 9b4839a2d07d36..505315c68ff3ee 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -63,8 +63,10 @@ config PCIE_APPLE_PIODMA_DIAG depends on PCIEASPM help Build the experimental SID17 retained bootstrap provider. It submits - one fixed request, validates typed ECAM and permits enumeration of - the two Neo functions only with the explicit enumerate=1 parameter. + one fixed request, validates typed ECAM and then enumerates the two + Neo radio functions when the board device tree links the PCIe host + to this engine. The enumerate=0 parameter keeps the host on its + root-port-only path instead. The host and supplier must both be built in: supplier removal cannot release a published DMA arena safely. Suspend-to-idle sleep retains that arena and restores the configuration without repeating a request. diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index b41a462456aaa1..30e7a5b5782cc5 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -58,6 +58,13 @@ static bool enumerate = true; module_param(enumerate, bool, 0444); MODULE_PARM_DESC(enumerate, "Boot-time radio gate (read-only, default on; no effect off J700/T8140)"); +/* + * The board DT opts in by pointing the host's "apple,piodma" at this engine. + * enumerate=0 keeps the host on its root-port-only path for diagnosis. + */ +static bool enumerate = true; +module_param(enumerate, bool, 0400); +MODULE_PARM_DESC(enumerate, "Prime once, validate native ECAM, and enumerate the two Neo functions (default: on)"); struct apple_piodma_request { u16 offset; From 35b748bde324c371903954926aa540b500b8c190 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:47:09 +1100 Subject: [PATCH 252/631] PCI: apple: let drivers bind the J700 radio functions After the J700 bootstrap, the host adds both radio functions to the bus and only then checks that neither is bus mastering and that neither has a driver. If either function has a driver by then, the host reports -EBUSY and closes downstream configuration access again, under a driver that has already bound and started using the function. That happens with a built-in driver, or with a module that udev loaded while the bus was being scanned. Check the cold state, with bus mastering off, before the functions are added and their drivers can probe. Treat drivers binding during publication as the normal outcome. Do not report the host as root-port-only once a bootstrap supplier has been found. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 39 ++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index de04e144dc5c07..80f2338cb82b0d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2602,12 +2602,40 @@ static int apple_pcie_neo_check_memory(struct pci_dev *root) return 0; } +/* + * Both functions must come out of the bootstrap cold. Check that before they + * are published, because publication lets their drivers bind and take over. + */ +static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *root) +{ + struct pci_dev *endpoint; + u16 command; + int function, ret = 0; + + for (function = 0; function < 2 && !ret; function++) { + endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); + if (!endpoint) + return -ENODEV; + if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { + ret = -EIO; + } else { + dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", + pci_name(endpoint), endpoint->vendor, endpoint->device, command); + if (command & PCI_COMMAND_MASTER) { + pci_clear_master(endpoint); + ret = -EIO; + } + } + pci_dev_put(endpoint); + } + return ret; +} + static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) { struct apple_pcie *pcie = pci_host_bridge_priv(bridge); - struct pci_dev *root, *endpoint; - u16 command; - int function, ret; + struct pci_dev *root; + int ret; root = pci_get_slot(bridge->bus, PCI_DEVFN(0, 0)); if (!root) @@ -2638,6 +2666,10 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) dev_err(pcie->dev, "root memory forwarding unavailable after assignment: %d\n", ret); goto close_config; } + ret = apple_pcie_neo_check_cold(pcie, root); + if (ret) + goto close_config; + pci_bus_add_devices(root->subordinate); for (function = 0; function < 2; function++) { endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); if (!endpoint) { @@ -2754,6 +2786,7 @@ static int apple_pcie_probe(struct platform_device *pdev) dev_info_once(dev, "Neo radio host: native AER/DPC recovery disabled for retained-DMA bootstrap\n"); } if (hw->root_bus_only) + if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(pcie->base)) From eab8dae7468f02ef063ff24b8cf80982942020e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:48:09 +1100 Subject: [PATCH 253/631] wifi: mt7932: disable ASPM on the J700 radio link On devicetree platforms the PCI core enables ASPM L0s/L1 by default. The station driver was qualified with the PCIe ASPM "performance" policy selected before it was loaded, so the documented admission sequence had userspace change that global policy first. If the module is autoloaded, nothing changes the policy. Disable all ASPM and clock power management states on the link before the driver starts the device, as other MediaTek PCIe Wi-Fi drivers do. This removes the userspace step and leaves the policy for other links alone. The function is probed before the root filesystem is mounted, and the driver reads its firmware while probing, so recommend building it as a module. Document that, and how cfg80211 finds the regulatory database. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 37 ++++++++----------- drivers/net/wireless/mediatek/mt7932/Kconfig | 2 + drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 + 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 113f43884155c1..062c16e943a50f 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -28,27 +28,22 @@ The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle restrictions. Kexec is excluded while either retained experiment is enabled. Use the ordinary cfg80211 regulatory database and applicable country policy. -Kernel country 00 uses firmware XZ and its mandatory ``world-XZ.bin`` package. -An explicit country requires its own package; there is no world fallback. - -The host now disables link power states on the two radio endpoints before -driver binding; no global PCIe ASPM performance policy is required. Load -``mt7932-fullmac`` after the root filesystem and local firmware packages are -available. Bluetooth's gate defaults closed; load ``mt7932_bt_pcie`` with the -default ``enable=0``, validate the cold, unbound ``14c3:793b`` function, enable -``/sys/module/mt7932_bt_pcie/parameters/enable``, then request its PCI probe:: - - echo 1 > /sys/module/mt7932_bt_pcie/parameters/enable - echo YOUR_BT_PCI_BDF > /sys/bus/pci/drivers_probe - -Use the actual ``14c3:793b`` PCI address for ``YOUR_BT_PCI_BDF``. The driver -suppresses its individual bind/unbind attributes, so there is no driver -``bind`` file. For an intentionally enabled cold boot, a file under -``/etc/modprobe.d/`` may instead contain:: - - options mt7932_bt_pcie enable=1 - -Install the target unit's Bluetooth inputs before opening the gate. +The validated first-admission fallback is kernel country 00 with firmware XZ. +A modular cfg80211 loads ``regulatory.db`` from the root filesystem; a +built-in one tries before the root filesystem is mounted and then needs +``iw reg reload``. + +The radio functions appear while the kernel boots, before the root +filesystem is mounted, and ``mt7932-fullmac`` reads its firmware while +probing. Build it as a module so that udev loads it from the root +filesystem, or provide its firmware in the initramfs. The driver disables +ASPM and clock power management on its link before starting the device, +matching the tested admission sequence, which used the PCIe ASPM +performance policy. Bluetooth's gate defaults closed; +validate the cold, unbound ``14c3:793b`` function, then load +``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. +A built-in transport can instead open its enable parameter before requesting +the PCI probe. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index bed6279895a48e..323b5d9999fe81 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -10,6 +10,8 @@ config MT7932_FULLMAC at 20/40/80 MHz, with a software bandwidth limit of 80 MHz. Requires original firmware, own calibration and original-derived country policy packages in addition to the cfg80211 regulatory database. + The device is probed before the root filesystem is mounted, so say M + unless those files are in the initramfs. Requires a trained retained PCI link and a translated SID1 DMA domain. Firmware, runtime DMA and calibration negotiation start automatically; cfg80211 and userspace select the country and network. No test boot diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 3e783ec85dd916..9564ebc37328ca 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1252,6 +1252,8 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); + /* Station operation was qualified with ASPM and clock PM off on this link. */ + pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); ret = pcim_enable_device(pdev); if (ret) return ret; From 659a54a57339a1970d378ff09d1a0c574a63a39f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 01:48:39 +1100 Subject: [PATCH 254/631] Bluetooth: mt7932: probe the J700 transport by default The transport only probes after userspace loads it with enable=1, so an autoloaded module leaves the Bluetooth function unbound. The gate was meant to keep the probe away until the root filesystem holds the unit's firmware and calibration. A module already loads from the root filesystem, and the probe validates a cold, unbound function before it starts. Open the gate by default and keep enable=0 to leave the function unbound. The documented admission sequence also selected the PCIe ASPM "performance" policy and loaded the Wi-Fi driver first. Disable ASPM and clock power management on the link in this probe as well, so the transport does not depend on the Wi-Fi driver for it. Add a module soft dependency so that modprobe loads mt7932_fullmac, which probes function 0, before the transport, keeping the tested firmware start order when udev loads both. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../networking/device_drivers/wifi/mt7932-neo.rst | 13 ++++++++----- drivers/bluetooth/mt7932_bt_pcie.c | 4 ++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 062c16e943a50f..02ab76d537c65a 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -39,11 +39,14 @@ probing. Build it as a module so that udev loads it from the root filesystem, or provide its firmware in the initramfs. The driver disables ASPM and clock power management on its link before starting the device, matching the tested admission sequence, which used the PCIe ASPM -performance policy. Bluetooth's gate defaults closed; -validate the cold, unbound ``14c3:793b`` function, then load -``mt7932_bt_pcie`` with ``enable=1`` after its local inputs are installed. -A built-in transport can instead open its enable parameter before requesting -the PCI probe. +performance policy. + +``mt7932_bt_pcie`` probes the cold ``14c3:793b`` function when it loads and +rejects a function that is not cold; ``enable=0`` leaves it unbound. Its +module soft dependency loads ``mt7932-fullmac`` first, so the Bluetooth +firmware starts after the Wi-Fi firmware, as in the tested sequence. The +transport also disables ASPM on the shared link. Like the Wi-Fi driver, it +reads its firmware while probing, so build it as a module. Do not reprobe after a failed or uncertain Bluetooth admission. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 5f812cc42039e6..29a9e9950233c1 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1083,6 +1083,8 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); if (ret) dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); + /* Admission was qualified with ASPM and clock PM off on this link. */ + pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); bt = kzalloc_obj(*bt); if (!bt) return -ENOMEM; @@ -1423,6 +1425,8 @@ module_pci_driver(bt7932_driver); MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); MODULE_LICENSE("GPL"); +/* Admission was qualified after the Wi-Fi function had started its firmware. */ +MODULE_SOFTDEP("pre: mt7932_fullmac"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); From 8e643e871481e8b697bf19a94dab1c2c136ec968 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:15:45 +1100 Subject: [PATCH 255/631] wifi: mt7932: depend on cfg80211 instead of selecting it MT7932_FULLMAC selects CFG80211. Built in, that forces CFG80211=y even when RFKILL=m, and Kconfig then warns about an unmet dependency, because CFG80211 depends on RFKILL || !RFKILL. Wireless drivers normally depend on CFG80211 rather than select it. Do the same, so that cfg80211's own constraints decide whether the driver can be built in. The J700 radio fragment relied on the select. Enable cfg80211 there as a module, which also lets it load regulatory.db from the root filesystem. Record the modular cfg80211 in the tested configuration. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 4 ++-- arch/arm64/configs/j700-radio.config | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 02ab76d537c65a..92db4f6d852ef4 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -21,8 +21,8 @@ Use the separately reviewed T8140 PCIe bootstrap described in ``CONFIG_PCIE_APPLE_PIODMA_DIAG=y``. Public m1n1 must populate the J700 ``wifi0`` endpoint's own ``local-mac-address``; a zero placeholder is rejected. -The original hardware-tested configuration included ``CONFIG_MT7932_FULLMAC=m``, -``CONFIG_CFG80211=y``, ``CONFIG_BT_MTK7932_PCIE=y``, ``CONFIG_BT_BREDR=y``, +The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, +``CONFIG_CFG80211=m``, ``CONFIG_BT_MTK7932_PCIE=m``, ``CONFIG_BT_BREDR=y``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config index 87b09a42d3cc68..f5c5d3c3ec0bfe 100644 --- a/arch/arm64/configs/j700-radio.config +++ b/arch/arm64/configs/j700-radio.config @@ -8,6 +8,7 @@ CONFIG_PCI=y CONFIG_PCIE_APPLE=y CONFIG_APPLE_DART=y CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_CFG80211=m CONFIG_WLAN=y CONFIG_WLAN_VENDOR_MEDIATEK=y CONFIG_MT7932_FULLMAC=m From 0587bc5e56c37e5d4cddb333998b6653cb10fae9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:35:10 +1100 Subject: [PATCH 256/631] wifi: mt7932: support the 5 GHz channels above channel 48 The station only advertises 5 GHz channels 36 to 48, so access points on UNII-2, UNII-2e or UNII-3 channels are invisible. On the MacBook Neo used for testing, the network's 5 GHz radio is on channel 149, and the station falls back to its 2.4 GHz radio. Advertise every 20 MHz primary from 36 to 64, 100 to 144 and 149 to 165, and let the cfg80211 regulatory domain decide which are usable. Each gets its 40 and 80 MHz pairing (165 stays at 20 MHz). The same pairing is used to decode association channel events and to pick the unit's own calibration records, which cover the whole band. Radar channels were previously rejected by refusing the whole regulatory domain. Pass them to the firmware with cfg80211's flags instead, and keep the station passive there. cfg80211 marks them RADAR, not NO_IR, so scans listen on them without probing. Scans made while associated skip them, and connects never target them: DFS client operation is not qualified, and a passive dwell only takes time away from the home channel. A channel whose limits the driver cannot apply, such as a power cap below the qualified 20 dBm, is now disabled on its own rather than blocking every channel. The firmware scan request has 32 channel slots, which holds the larger 5 GHz list in one batch. A scan request left with no eligible channel now completes at once. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 7 ++ .../net/wireless/mediatek/mt7932/bandwidth.h | 9 +-- .../wireless/mediatek/mt7932/calibration.h | 7 +- .../net/wireless/mediatek/mt7932/channels.h | 66 ++++++++++++++++++- .../net/wireless/mediatek/mt7932/frontend.c | 23 +++++-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 +- .../net/wireless/mediatek/mt7932/regulatory.h | 6 +- drivers/net/wireless/mediatek/mt7932/scan.h | 5 +- .../net/wireless/mediatek/mt7932/startup.c | 41 ++++++++---- 10 files changed, 139 insertions(+), 31 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 92db4f6d852ef4..b113f5b3319cf3 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -27,6 +27,13 @@ The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle restrictions. Kexec is excluded while either retained experiment is enabled. +The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, +100-144 and 149-165 that the cfg80211 regulatory domain permits. Radar (DFS) +channels stay passive: the firmware listens on them while the station is not +associated, but never probes or joins an access point there, and scans made +while associated skip them. A channel whose restrictions the driver cannot +apply (reduced power, no OFDM, PSD limits) is disabled rather than used. + Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. A modular cfg80211 loads ``regulatory.db`` from the root filesystem; a diff --git a/drivers/net/wireless/mediatek/mt7932/bandwidth.h b/drivers/net/wireless/mediatek/mt7932/bandwidth.h index c5efe6599e0a46..07c281e0fc5893 100644 --- a/drivers/net/wireless/mediatek/mt7932/bandwidth.h +++ b/drivers/net/wireless/mediatek/mt7932/bandwidth.h @@ -13,13 +13,14 @@ static inline u8 mt7932_domain_5g_bw(void) } /* EE4d width 0 combines 20/40; primary and extension select the center. - * Keep all operational primaries inside 36..48, and 2 GHz at 20 MHz. + * Extension 1 places the secondary above the primary, 3 below it. Keep 2 GHz + * at 20 MHz, and 165, which has no 40 MHz pair, at 20 MHz too. */ static inline bool mt7932_channel_layout(u8 primary, u8 width, u8 extension, u8 maximum, u8 *center, u8 *bw) { u8 band = mt7932_channel_band(primary); - u8 expected = primary == 36 || primary == 44 ? 1 : 3; + u8 pair = mt7932_channel_ht40_centre(primary); if (!band || maximum > 2 || width > 1) return false; @@ -28,12 +29,12 @@ static inline bool mt7932_channel_layout(u8 primary, u8 width, u8 extension, *bw = 0; return true; } - if (band != 2 || extension != expected) + if (band != 2 || !pair || extension != (pair > primary ? 1 : 3)) return false; *bw = width ? 2 : 1; if (*bw > maximum) return false; - *center = width ? 42 : extension == 1 ? primary + 2 : primary - 2; + *center = width ? mt7932_channel_vht80_centre(primary) : pair; return true; } diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index cd53e0be435096..85075bce7b255c 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -2,6 +2,7 @@ #ifndef MT7932_CALIBRATION_H #define MT7932_CALIBRATION_H +#include "channels.h" #include "protocol.h" /* Protected by the RX/procedure lock. One active logical procedure only; @@ -159,7 +160,7 @@ static inline int mt7932_cal_association(const u8 *data, size_t size, u8 channel *input_tag = 0; if (channel >= 1 && channel <= 13) group = 0; - else if (channel >= 36 && channel <= 48 && !(channel % 2)) + else if (mt7932_channel_5g_cal(channel)) group = 1; else return -EOPNOTSUPP; @@ -245,7 +246,7 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, *input_tag = 0; if (upper != 0x1000 || version != 12 || module != 0x89 || - channel < 36 || channel > 48 || channel % 2) + !mt7932_channel_5g_cal(channel)) return -EOPNOTSUPP; ret = mt7932_cal_validate(data, size); if (ret) @@ -260,7 +261,7 @@ static inline int mt7932_cal_requested_5g(const u8 *data, size_t size, if (segment.data[i]) return -EOPNOTSUPP; /* Own row610f07 does not remap center selectors1/2; only primaries. */ - selected = channel % 4 ? channel : channel <= 40 ? 38 : 46; + selected = mt7932_channel_band(channel) ? mt7932_channel_cal_centre(channel) : channel; params[0] = 0x10000001; params[1] = 0x10001000 | channel; params[2] = 0x10001000 | (channel << 17) | selected; diff --git a/drivers/net/wireless/mediatek/mt7932/channels.h b/drivers/net/wireless/mediatek/mt7932/channels.h index 8c4b61e4a3ec75..bae04c245d72a5 100644 --- a/drivers/net/wireless/mediatek/mt7932/channels.h +++ b/drivers/net/wireless/mediatek/mt7932/channels.h @@ -4,14 +4,78 @@ /* Bounded operational whitelist, not the full calibration channel namespace. * Firmware RF bands1/2 are distinct from single-interface MAC index0. + * 5 GHz covers the 20 MHz primaries of UNII-1/2 (36..64), UNII-2e (100..144) + * and UNII-3 (149..165); regulatory flags decide which of them are usable. */ +#define MT7932_CHANNELS_2G 13 +#define MT7932_CHANNELS_5G 25 +#define MT7932_CHANNELS (MT7932_CHANNELS_2G + MT7932_CHANNELS_5G) + static inline unsigned int mt7932_channel_band(unsigned int channel) { if (channel >= 1 && channel <= 13) return 1; - if (channel >= 36 && channel <= 48 && !(channel % 4)) + if (((channel >= 36 && channel <= 64) || (channel >= 100 && channel <= 144)) && + !(channel % 4)) + return 2; + if (channel >= 149 && channel <= 165 && !((channel - 149) % 4)) return 2; return 0; } +/* The nth 5 GHz primary, in the order the wiphy advertises them. */ +static inline unsigned int mt7932_channel_5g(unsigned int index) +{ + if (index < 8) + return 36 + 4 * index; + if (index < 20) + return 100 + 4 * (index - 8); + return 149 + 4 * (index - 20); +} + +/* Offset of a 5 GHz primary within its 80 MHz block numbering. */ +static inline unsigned int mt7932_channel_slot(unsigned int primary, unsigned int *base) +{ + *base = primary >= 149 ? 149 : primary >= 100 ? 100 : 36; + return (primary - *base) / 4; +} + +/* 40 MHz pair centre of a 5 GHz primary; 0 where no pair exists (165). */ +static inline unsigned int mt7932_channel_ht40_centre(unsigned int primary) +{ + unsigned int base, slot; + + if (mt7932_channel_band(primary) != 2 || primary == 165) + return 0; + slot = mt7932_channel_slot(primary, &base); + return base + 4 * (slot & ~1U) + 2; +} + +/* 80 MHz block centre of a 5 GHz primary; 0 where no block exists (165). */ +static inline unsigned int mt7932_channel_vht80_centre(unsigned int primary) +{ + unsigned int base, slot; + + if (mt7932_channel_band(primary) != 2 || primary == 165) + return 0; + slot = mt7932_channel_slot(primary, &base); + return base + 4 * (slot & ~3U) + 6; +} + +/* A 5 GHz primary or a 40/80 MHz centre that calibration can be asked for. */ +static inline bool mt7932_channel_5g_cal(unsigned int channel) +{ + if ((channel >= 36 && channel <= 64) || (channel >= 100 && channel <= 144)) + return !(channel % 2); + return channel >= 149 && channel <= 167 && (channel % 2); +} + +/* Calibration centre for a 5 GHz primary. 165 has no 40 MHz pair; its own + * calibration package carries the record for the 165/169 centre, 167. + */ +static inline unsigned int mt7932_channel_cal_centre(unsigned int primary) +{ + return primary == 165 ? 167 : mt7932_channel_ht40_centre(primary); +} + #endif diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index f303afe2246f4d..7c92e10592ceb5 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -146,7 +146,7 @@ static int mt_scan_submit_next(struct mt7932 *m) { struct cfg80211_scan_request *request = m->scan_request; const struct cfg80211_ssid *ssid = NULL; - u8 body[0x4d4], channels[13]; + u8 body[0x4d4], channels[MT7932_CHANNELS_5G]; unsigned int count = 0, i; unsigned long flags; bool passive = false; @@ -160,11 +160,15 @@ static int mt_scan_submit_next(struct mt7932 *m) passive = !(m->scan_batch & 1); for (i = 0; i < request->n_channels; i++) { struct ieee80211_channel *channel = request->channels[i]; - bool listen = !request->n_ssids || + bool radar = channel->flags & IEEE80211_CHAN_RADAR; + bool listen = !request->n_ssids || radar || (channel->flags & IEEE80211_CHAN_NO_IR); + /* Radar channels are never joined; while associated, + * a passive dwell there only delays the home channel. + */ if (mt7932_channel_band(channel->hw_value) != m->scan_band || - listen != passive) + listen != passive || (radar && m->scan_home_channel)) continue; if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ) || count == ARRAY_SIZE(channels)) @@ -216,7 +220,7 @@ static int mt_scan_submit_next(struct mt7932 *m) */ int mt_connect_discover(struct mt7932 *m) { - u8 body[0x4d4], channels[13], cancel[4] = {}; + u8 body[0x4d4], channels[MT7932_CHANNELS_5G], cancel[4] = {}; unsigned long flags; unsigned int attempt, band, i, count; int ret = 0; @@ -237,7 +241,8 @@ int mt_connect_discover(struct mt7932 *m) if (m->connect_channel_req && channel != m->connect_channel_req) continue; - if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ)) + if (channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_RADAR)) continue; passive |= !!(channel->flags & IEEE80211_CHAN_NO_IR); channels[count++] = channel->hw_value; @@ -330,7 +335,10 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) dev_info_ratelimited(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", request->n_channels, request->n_ssids, request->ie_len, request->flags, request->duration, request->duration_mandatory); - if (!request->n_channels || request->n_channels > 17 || request->ie_len > 600 || + /* cfg80211 lists each advertised channel at most once. */ + if (request->n_channels > MT7932_CHANNELS) + return -EOPNOTSUPP; + if (!request->n_channels || request->ie_len > 600 || request->duration_mandatory || (request->flags & ~(NL80211_SCAN_FLAG_FLUSH | NL80211_SCAN_FLAG_COLOCATED_6GHZ)) || request->n_ssids > 1 || (request->n_ssids && request->ssids[0].ssid_len > 32)) @@ -364,6 +372,9 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) spin_unlock_irqrestore(&m->response_lock, flags); cancel_delayed_work(&m->scan_timeout_work); } else { + /* Nothing eligible to submit: complete the request. */ + if (ret) + schedule_work(&m->scan_finish_work); ret = 0; } out: diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 3360e8c18c0cbe..e81fb052c40d79 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -402,7 +402,7 @@ void mt_connect_work(struct work_struct *work) goto free; } if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | - IEEE80211_CHAN_NO_20MHZ)) { + IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_RADAR)) { ret = -EINVAL; goto free; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 89a58bd9c1b5d2..162f52075dace1 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -99,8 +99,8 @@ struct mt7932 { struct ieee80211_supported_band band2; struct cfg80211_chan_def connect_chandef; u8 connect_center; - struct ieee80211_channel channels[13]; - struct ieee80211_channel channels5[4]; + struct ieee80211_channel channels[MT7932_CHANNELS_2G]; + struct ieee80211_channel channels5[MT7932_CHANNELS_5G]; struct ieee80211_supported_band band5; u8 scan_band, scan_batch; struct ieee80211_rate rates[12]; diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 45cb3d6ce91f0d..fd65e07d9506ae 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -5,6 +5,8 @@ /* Private, versioned packaging of original-derived country policy. This is * not firmware code and does not replace cfg80211's regulatory database. */ +#include "channels.h" + #define MT7932_POLICY_TABLES 13 #define MT7932_POLICY_MODES 3 @@ -20,12 +22,12 @@ struct mt7932_reg_snapshot { * profile. Stock unknown-country startup uses XZ while Linux stays 00. */ u8 alpha2[2]; - u8 domain[148]; + u8 domain[12 + 8 * MT7932_CHANNELS]; u16 length; /* Keep numeric limits in equality checks even though CID0f contains only * flags. An unsupported reduction must not be mistaken for no change. */ - s32 power[17]; + s32 power[MT7932_CHANNELS]; int error; }; diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 4899cbb6c1654f..ae26645cc36932 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -5,6 +5,9 @@ #include "protocol.h" #include "channels.h" +/* CID03 carries this many band/channel slots, from 0xa0 up to 0xe0. */ +#define MT7932_SCAN_CHANNELS 32 + /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR * channels require mt7932_scan_passive() before submission. @@ -17,7 +20,7 @@ static inline int mt7932_scan_body(u8 *body, size_t capacity, u8 seq, 0x6b,7,0x0f,0xff,0xff,0xff,0xff,0xff,0xff}; unsigned int i, j; - if (capacity < 0x4d4 || !seq || seq > 127 || !count || count > 13) + if (capacity < 0x4d4 || !seq || seq > 127 || !count || count > MT7932_SCAN_CHANNELS) return -EINVAL; for (i = 0; i < count; i++) { if (!mt7932_channel_band(channels[i]) || diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 3dd160694c57f6..25327474f9962c 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -418,6 +418,9 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request memcpy(reg.domain, "XZ", 2); reg.domain[4] = 1; /* 2 GHz remains 20 MHz. */ reg.domain[5] = mt7932_domain_5g_bw(); + for (i = 0; i < MT7932_CHANNELS; i++) { + struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? + &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; for (i = 0; i < 17; i++) { struct ieee80211_channel *channel = mt_channel(m, i); u32 flags = channel->flags; @@ -425,15 +428,21 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request reg.power[i] = channel->max_power; if (flags & IEEE80211_CHAN_DISABLED) continue; - if (flags & (IEEE80211_CHAN_RADAR | IEEE80211_CHAN_NO_OFDM | - IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_PSD | IEEE80211_CHAN_IR_CONCURRENT)) - reg.error = -EOPNOTSUPP; /* Native per-rate units are not a proven dBm conversion. Do not * silently ignore a numeric reduction below the qualified 20 dBm * channel envelope or reinterpret sentinel bytes as power caps. + * A channel with a restriction the driver cannot honour is left + * out of the firmware domain and disabled for cfg80211 instead. + */ + if (flags & (IEEE80211_CHAN_NO_OFDM | IEEE80211_CHAN_NO_20MHZ | + IEEE80211_CHAN_PSD | IEEE80211_CHAN_IR_CONCURRENT) || + channel->max_power < 20) { + channel->flags |= IEEE80211_CHAN_DISABLED; + continue; + } + /* Radar channels carry cfg80211's RADAR flag: the firmware may + * listen there, but the station never probes or joins them. */ - if (channel->max_power < 20) - reg.error = -EOPNOTSUPP; put_unaligned_le16(channel->hw_value, reg.domain + 12 + count * 8); /* Driver bandwidth limits are already in channel->flags. Adding * the old blanket 20 MHz mask here would contradict the wiphy @@ -441,7 +450,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request */ put_unaligned_le32(flags, reg.domain + 16 + count * 8); count++; - reg.domain[i < 13 ? 8 : 9]++; + reg.domain[i < MT7932_CHANNELS_2G ? 8 : 9]++; } reg.length = 12 + count * 8; if (!count) @@ -526,7 +535,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) * Global hints from other radios still reach the notifier and are handled. */ m->wiphy->regulatory_flags |= REGULATORY_COUNTRY_IE_IGNORE; - for (i = 0; i < 13; i++) { + for (i = 0; i < MT7932_CHANNELS_2G; i++) { m->channels[i].band = NL80211_BAND_2GHZ; m->channels[i].center_freq = 2412 + 5 * i; m->channels[i].hw_value = i + 1; @@ -544,12 +553,22 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->wiphy->bands[NL80211_BAND_2GHZ] = &m->band2; { for (i = 0; i < ARRAY_SIZE(m->channels5); i++) { + unsigned int primary = mt7932_channel_5g(i); + unsigned int pair = mt7932_channel_ht40_centre(primary); + u32 flags = IEEE80211_CHAN_NO_160MHZ; + + /* The secondary channel sits on the far side of the pair centre. */ + if (!pair) + flags |= IEEE80211_CHAN_NO_HT40 | IEEE80211_CHAN_NO_80MHZ; + else if (pair < primary) + flags |= IEEE80211_CHAN_NO_HT40PLUS; + else + flags |= IEEE80211_CHAN_NO_HT40MINUS; m->channels5[i].band = NL80211_BAND_5GHZ; - m->channels5[i].center_freq = 5180 + 20 * i; - m->channels5[i].hw_value = 36 + 4 * i; + m->channels5[i].center_freq = 5000 + 5 * primary; + m->channels5[i].hw_value = primary; m->channels5[i].max_power = 20; - m->channels5[i].flags = IEEE80211_CHAN_NO_160MHZ | - (i % 2 ? IEEE80211_CHAN_NO_HT40PLUS : IEEE80211_CHAN_NO_HT40MINUS); + m->channels5[i].flags = flags; if (MT7932_MAX_5G_BW < 2) m->channels5[i].flags |= IEEE80211_CHAN_NO_80MHZ; if (!MT7932_MAX_5G_BW) From e277239f5afc59c757e3ca47752b49f8a6ad905f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:35:51 +1100 Subject: [PATCH 257/631] wifi: mt7932: honour the channel permissions of the country package The country package that the driver sends to the firmware has a per-rate power limit table for every channel. Where a country does not permit a channel, every limit of that channel is undefined. Examples are UNII-3 in Japan, UNII-2e in China, 120 to 128 in Australia and 144 in most of Europe. The driver still listed such channels in the firmware domain whenever the cfg80211 regulatory domain allowed them. Drop channels the loaded package does not permit from the firmware domain, and disable them in cfg80211 too, so that scans and connects do not offer them. cfg80211 recomputes channel flags on each regulatory change, and the package is reloaded for each change, so the two stay in step. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 4 +++- .../net/wireless/mediatek/mt7932/startup.c | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index b113f5b3319cf3..516d9ee25c358d 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -32,7 +32,9 @@ The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, channels stay passive: the firmware listens on them while the station is not associated, but never probes or joins an access point there, and scans made while associated skip them. A channel whose restrictions the driver cannot -apply (reduced power, no OFDM, PSD limits) is disabled rather than used. +apply (reduced power, no OFDM, PSD limits) is disabled rather than used, and +so is a channel the country package does not permit: every power limit of +that channel in ``policy/.bin`` is left undefined. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 25327474f9962c..8275282c885b42 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -286,6 +286,26 @@ void mt_retry_missing_policy(struct mt7932 *m) schedule_work(&m->startup_work); } spin_unlock_irqrestore(&m->response_lock, flags); +/* Keep cfg80211's channel list in step with the channels the country + * package leaves out of the firmware domain. cfg80211 recomputes the flags + * on its next regulatory change, which runs this again. + */ +static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy) +{ + unsigned int i; + + rtnl_lock(); + wiphy_lock(m->wiphy); + for (i = 0; i < MT7932_CHANNELS; i++) { + struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? + &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + + if (!(channel->flags & IEEE80211_CHAN_DISABLED) && + !mt7932_policy_permits(policy, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_DISABLED; + } + wiphy_unlock(m->wiphy); + rtnl_unlock(); } static void mt_startup_work(struct work_struct *work) @@ -351,6 +371,9 @@ static void mt_startup_work(struct work_struct *work) if (!ret) { mt7932_policy_filter(®, &policy); mt_policy_disable(m, &policy, generation); + if (!ret) { + mt7932_policy_filter(®, &policy); + mt_policy_disable(m, &policy); } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) From cdccb1f349c67901d9020d3fbd101c65d48c2f63 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 03:37:23 +1100 Subject: [PATCH 258/631] wifi: mt7932: let an address change wait for a retiring scan NetworkManager sets a per-network station address before each connection by default: Fedora uses a stable per-SSID address. It takes the interface down, changes the address and brings the interface back up. Taking the interface down cancels any running scan, but the firmware completes the cancellation asynchronously. Until then the driver refuses the address change with -EBUSY, so the activation fails with "set-hw-addr: failed to set-cloned MAC address" and NetworkManager moves on to another profile. When the interface is down, wait for the scan to retire, for up to two seconds, before checking whether the address can change. Changes on a running interface are still refused, as eth_mac_addr() requires. Log the busy state at debug level when a change is still refused. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/frontend.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 7c92e10592ceb5..7700ae6a96ca57 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -461,12 +461,25 @@ static int mt_set_mac_address(struct net_device *netdev, void *address) /* No live-MAC change: the next BSS activation and connect commands use * this address together. A failed retirement must not change identity. + * NetworkManager takes the interface down before changing the address, + * which cancels any scan; let that scan retire instead of failing. */ + if (!netif_running(netdev)) { + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + mt_scan_quiesce(m); + while (READ_ONCE(m->retired_scan_seq) && time_before(jiffies, deadline)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(100)); + } mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); if (m->stopping || m->link_failed || m->bss_active || m->connecting || m->connected || m->disconnecting || m->peer_valid || m->scan_request || m->retired_scan_seq || m->discovering) { + dev_dbg(&m->pdev->dev, + "address change refused: bss=%u connecting=%u connected=%u peer=%u scan=%u retired=%u\n", + m->bss_active, m->connecting, m->connected, m->peer_valid, + !!m->scan_request, m->retired_scan_seq); ret = -EBUSY; } else { ret = eth_mac_addr(netdev, address); From 75ac49938d2a061ba0414ec4dc57660a6ad17580 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:02:25 +1100 Subject: [PATCH 259/631] PCI: apple: keep T8140 hosts without a bootstrap link root-port-only With the J700 bootstrap on by default, every T8140 host now looks up its "apple,piodma" engine. A host whose device tree has no such property fails its probe with -EINVAL instead of falling back to the root-port-only path. That is the opposite of the intent that the board device tree selects the bootstrap. Only look up a supplier when the property is present. A property that names a missing or wrong engine still fails the probe. Also stop logging "downstream config blocked" for the port of a host whose bootstrap is about to open it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple-piodma-diag.c | 5 +++-- drivers/pci/controller/pcie-apple.c | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 30e7a5b5782cc5..e39172b97b9389 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -59,8 +59,9 @@ static bool enumerate = true; module_param(enumerate, bool, 0444); MODULE_PARM_DESC(enumerate, "Boot-time radio gate (read-only, default on; no effect off J700/T8140)"); /* - * The board DT opts in by pointing the host's "apple,piodma" at this engine. - * enumerate=0 keeps the host on its root-port-only path for diagnosis. + * The board DT opts in by pointing the host's "apple,piodma" at this engine; + * a host without that property stays root-port-only. enumerate=0 keeps every + * host on its root-port-only path for diagnosis. */ static bool enumerate = true; module_param(enumerate, bool, 0400); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 80f2338cb82b0d..add3e415e3fba4 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -1870,10 +1870,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (pcie->hw->tunneled) WRITE_ONCE(port->started, true); if (pcie->hw->root_bus_only) - dev_info(pcie->dev, - "port %pOF RUN=%#x link=%#x; downstream config blocked\n", + dev_info(pcie->dev, "port %pOF RUN=%#x link=%#x; downstream config %s\n", np, apple_pcie_port_readl(port, PORT_STATUS), - apple_pcie_port_readl(port, PORT_LINKSTS)); + apple_pcie_port_readl(port, PORT_LINKSTS), + pcie->piodma_supplier ? "waits for the bootstrap" : "blocked"); return 0; @@ -2759,9 +2759,9 @@ static int apple_pcie_probe(struct platform_device *pdev) } } - if (hw->root_bus_only && - of_device_is_compatible(dev->of_node, "apple,t8140-pcie") && - apple_piodma_bootstrap_enabled()) { + /* Only a host that the board links to a bootstrap engine opens bus 1. */ + if (hw->root_bus_only && apple_piodma_bootstrap_enabled() && + of_property_present(dev->of_node, "apple,piodma")) { ret = apple_piodma_bootstrap_get(dev, &piodma_supplier); if (ret) return dev_err_probe(dev, ret, "PIODMA supplier unavailable\n"); From a057aab32a0a9b3dc038bd1b542b8d5b934046d8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:02:43 +1100 Subject: [PATCH 260/631] PCI: apple: remove the J700 radio functions when publication fails If the root memory-window check or the cold check fails after the downstream scan, the host closes downstream configuration access but leaves both functions on the bus, scanned and unpublished. A later bus rescan would publish them and let their drivers probe through a closed config window, with memory decoding off. Remove the scanned functions, while still holding the rescan lock and before closing configuration access. A later rescan then finds nothing. Also check and quiesce both functions in the cold check instead of stopping at the first failure, so that a second function left bus mastering is cleared too. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pci/controller/pcie-apple.c | 30 +++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index add3e415e3fba4..2e2b76f6037f6f 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -2612,18 +2612,21 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro u16 command; int function, ret = 0; - for (function = 0; function < 2 && !ret; function++) { + /* Check and quiesce both functions; keep the first error. */ + for (function = 0; function < 2; function++) { endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); - if (!endpoint) - return -ENODEV; + if (!endpoint) { + ret = ret ?: -ENODEV; + continue; + } if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { - ret = -EIO; + ret = ret ?: -EIO; } else { dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x\n", pci_name(endpoint), endpoint->vendor, endpoint->device, command); if (command & PCI_COMMAND_MASTER) { pci_clear_master(endpoint); - ret = -EIO; + ret = ret ?: -EIO; } } pci_dev_put(endpoint); @@ -2631,6 +2634,19 @@ static int apple_pcie_neo_check_cold(struct apple_pcie *pcie, struct pci_dev *ro return ret; } +/* + * Rescan lock held, downstream config still open. Remove functions that were + * scanned but will not be published, so that a later rescan cannot publish + * them after config access has been closed again. + */ +static void apple_pcie_neo_unscan(struct pci_dev *root) +{ + struct pci_dev *dev, *tmp; + + list_for_each_entry_safe_reverse(dev, tmp, &root->subordinate->devices, bus_list) + pci_stop_and_remove_bus_device(dev); +} + static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) { struct apple_pcie *pcie = pci_host_bridge_priv(bridge); @@ -2717,8 +2733,10 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) pci_bus_add_devices(root->subordinate); } close_config: - if (ret) + if (ret) { + apple_pcie_neo_unscan(root); WRITE_ONCE(pcie->neo_config_ready, false); + } pci_unlock_rescan_remove(); if (!ret) dev_info(pcie->dev, "NEO_RADIO_READY; functions0/1 admitted with link power states disabled\n"); From d165a6a50d54bc6b68ee943192364e226ad04740 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:03:41 +1100 Subject: [PATCH 261/631] Bluetooth: mt7932: wait for the Wi-Fi function before starting Bluetooth admission was qualified with the Wi-Fi function already running its firmware. The module soft dependency added for that only orders a modprobe of both modules. A built-in transport, insmod, asynchronous probing or a failed Wi-Fi probe all bypass it. Removal is affected too: "modprobe -r mt7932_bt_pcie" now also unloads the Wi-Fi driver, as kmod removes pre-softdeps after the module. Enforce the order in the driver instead. Defer the probe until mt7932-fullmac is bound to function 0, and add a device link so that the driver core unbinds Bluetooth before Wi-Fi. Drop the soft dependency. Bluetooth now stays unbound while the Wi-Fi driver is missing or failed, which matches the qualified sequence. Also warn when the link's power management cannot be disabled, and update the help text and documentation, which still called the transport opt-in. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 23 +++++++++++-------- drivers/bluetooth/mt7932_bt_pcie.c | 21 ++++++++++++++--- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 516d9ee25c358d..1ac94a339ece2b 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -4,8 +4,8 @@ MT7932 radio bring-up on the MacBook Neo ======================================== J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed -WPA2-CCMP authentication or open networks. Bluetooth uses an opt-in PCIe -transport and remains disabled by default pending physical qualification. +WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second +PCIe function. The Wi-Fi import is pinned to ``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, commit @@ -50,14 +50,17 @@ ASPM and clock power management on its link before starting the device, matching the tested admission sequence, which used the PCIe ASPM performance policy. -``mt7932_bt_pcie`` probes the cold ``14c3:793b`` function when it loads and -rejects a function that is not cold; ``enable=0`` leaves it unbound. Its -module soft dependency loads ``mt7932-fullmac`` first, so the Bluetooth -firmware starts after the Wi-Fi firmware, as in the tested sequence. The -transport also disables ASPM on the shared link. Like the Wi-Fi driver, it -reads its firmware while probing, so build it as a module. -Do not reprobe after a failed or uncertain Bluetooth admission. The tested -Wi-Fi driver owns function 0 and Bluetooth owns function 1. +``mt7932_bt_pcie`` defers its probe until ``mt7932-fullmac`` is bound to +function 0, so the Bluetooth firmware always starts after the Wi-Fi firmware, +as in the tested sequence, however the drivers are loaded. Bluetooth therefore +stays unbound while the Wi-Fi driver is missing or has failed. A device link +makes the driver core unbind Bluetooth before Wi-Fi; unloading +``mt7932_bt_pcie`` does not affect Wi-Fi. The probe rejects a function that is +not cold, which is what now prevents a reprobe after a failed or uncertain +admission; ``enable=0`` leaves the function unbound. The transport also +disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware +while probing, so build it as a module. The tested Wi-Fi driver owns function +0 and Bluetooth owns function 1. Use a saved NetworkManager open or WPA2 profile matching the current interface. Audio in the desktop session requires BlueZ, PipeWire, its PulseAudio diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 29a9e9950233c1..a0480b36fbf02a 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1056,6 +1056,21 @@ static int bt7932_wait_for_wifi(struct pci_dev *pdev) else if (!mt7932_fullmac_ready(wifi)) ret = dev_err_probe(&pdev->dev, -ENODEV, "the Wi-Fi function failed to initialize\n"); + * Admission was only qualified with the Wi-Fi function already running its + * firmware. Wait until mt7932-fullmac is bound to function 0, then link the + * two functions so that the driver core unbinds this one first. + */ +static int bt7932_wait_for_wifi(struct pci_dev *pdev) +{ + struct pci_dev *wifi = pci_get_slot(pdev->bus, PCI_DEVFN(0, 0)); + int ret = 0; + + if (!wifi) + return -ENODEV; + if (!device_is_bound(&wifi->dev) || + strcmp(dev_driver_string(&wifi->dev), "mt7932-fullmac")) + ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "waiting for the Wi-Fi function\n"); else if (!device_link_add(&pdev->dev, &wifi->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) ret = -EINVAL; pci_dev_put(wifi); @@ -1084,7 +1099,9 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); /* Admission was qualified with ASPM and clock PM off on this link. */ - pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); bt = kzalloc_obj(*bt); if (!bt) return -ENOMEM; @@ -1425,8 +1442,6 @@ module_pci_driver(bt7932_driver); MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); MODULE_LICENSE("GPL"); -/* Admission was qualified after the Wi-Fi function had started its firmware. */ -MODULE_SOFTDEP("pre: mt7932_fullmac"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); From a4f1e7adea16cc13d4b1a0ec50361a640930bfdf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 04:03:48 +1100 Subject: [PATCH 262/631] wifi: mt7932: warn when link power management cannot be disabled pci_disable_link_state() fails when the OS has no ASPM control, for example with pcie_aspm=off. The link is then left as the firmware set it up, while station operation was qualified with it disabled. Report that instead of ignoring the error. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 9564ebc37328ca..d24bb674cafc91 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1253,7 +1253,9 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); /* Station operation was qualified with ASPM and clock PM off on this link. */ - pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); + if (ret) + dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); ret = pcim_enable_device(pdev); if (ret) return ret; From e20df18c88904ad312068c328f73909ea4fb58d0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:03:36 +1100 Subject: [PATCH 263/631] wifi: mt7932: bound the address-change wait to two seconds An address change on a stopped interface waits for a cancelled scan to retire before it checks whether the address can change. The change runs under RTNL, and the wait is documented as lasting at most two seconds, but it is longer in two cases: - mt_scan_quiesce() first waits up to five seconds for the scan to complete, and the two-second loop only starts after that, so a firmware that never retires the scan holds RTNL for about seven seconds; - once the driver is stopping, mt_stop_host() has completed scan_done for good, so every wait in the loop returns at once and the loop spins for two seconds. The interface is already down at this point, so its scan has been cancelled and only the retirement is left to wait for. Cancel any scan that is still recorded, then wait for the retirement against a single two-second deadline, and stop waiting as soon as the driver is stopping. The address change itself is unchanged: it is still refused with -EBUSY while a retirement is outstanding. Fixes: 22fc65ce6709 ("wifi: mt7932: let an address change wait for a retiring scan") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/frontend.c | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 7700ae6a96ca57..fc783ebe6b5bcd 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -138,6 +138,24 @@ int mt_scan_quiesce(struct mt7932 *m) return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; } +/* Wait at most @timeout in total for a cancelled scan to retire. Callers + * may hold RTNL, so this never waits longer than asked. A stopping driver + * completes scan_done for good, which would turn the wait into a spin. + */ +static void mt_scan_retire_wait(struct mt7932 *m, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + long left; + + mt_abort_scan(m->wiphy, &m->wdev); + flush_work(&m->scan_finish_work); + while (READ_ONCE(m->retired_scan_seq) && !READ_ONCE(m->stopping)) { + left = (long)(deadline - jiffies); + if (left <= 0 || !wait_for_completion_timeout(&m->scan_done, left)) + break; + } +} + /* command_mutex held: one RF-band/type batch at a time, with matching EID0d * completion before the next batch. A request without SSIDs is passive; * active requests must also listen passively on NO_IR channels. @@ -464,13 +482,8 @@ static int mt_set_mac_address(struct net_device *netdev, void *address) * NetworkManager takes the interface down before changing the address, * which cancels any scan; let that scan retire instead of failing. */ - if (!netif_running(netdev)) { - unsigned long deadline = jiffies + msecs_to_jiffies(2000); - - mt_scan_quiesce(m); - while (READ_ONCE(m->retired_scan_seq) && time_before(jiffies, deadline)) - wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(100)); - } + if (!netif_running(netdev)) + mt_scan_retire_wait(m, msecs_to_jiffies(2000)); mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); if (m->stopping || m->link_failed || m->bss_active || m->connecting || From 56e1f55c14499ec045e5cbf88d7100719bebcc4e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:07:06 +1100 Subject: [PATCH 264/631] drm: apple: skip the H17P sleep drain on a crashed or stopped DCP Before system sleep, the H17P drain runs up to four passes to settle a pending brightness change. A crashed or stopped DCP rejects every present, and the queue drain reports it idle at once, so a level that was dirty when it stopped stays pending: all four passes run and the suspend logs "brightness change still pending at suspend", although there is nothing left that could be sent. Return from the drain as soon as the DCP has crashed or its queue has stopped. The crash itself is already reported. Fixes: 6d4dc0a94dc9 ("drm: apple: settle H17P brightness retries before system sleep") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 8a15145c068217..bf64aeddcf6b65 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5092,6 +5092,12 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * the first attempt and the policy's three retries. */ for (pass = 0; pass < 4; pass++) { + /* + * A crashed or stopped DCP takes no more presents, so a level + * still pending there can never be sent and is not waited for. + */ + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return; flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); From e090b356b8f9c59d219570f226bcad7f55968057 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:07:35 +1100 Subject: [PATCH 265/631] drm: apple: treat a seeded H17P backlight level as known The H17P backlight policy re-sends a level on the firmware's request only when Linux knows what the panel shows: the loader's reported level or one that a completed present carried. dcp_bl_seed() puts a decoded powerlog sample, which is the loader's level, into the state before Linux takes control, but it does not mark the level as known. A seed over a registration default therefore leaves a real level that cannot be re-sent, and the state disagrees with its own definition. The seeding helper has no caller in the driver yet, so nothing goes wrong today, but wiring it up later, for example to let a late powerlog report correct a registration default, would hit this. Mark a seeded level as known and cover both the accepted and the refused seed in KUnit. Fixes: 73dd5dedcc77 ("drm: apple: never re-send a guessed H17P backlight level") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 6 ++++- drivers/gpu/drm/apple/dcp_backlight_test.c | 26 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 73683cb0322a39..2c9a0a7ab97be3 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -80,13 +80,17 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, return 0; } -/* A decoded powerlog sample can seed takeover only before Linux control. */ +/* + * A decoded powerlog sample can seed takeover only before Linux control. + * It reports the level the panel shows, so the level is then known. + */ static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) { if (!state->ready || state->controlled || nits > state->maximum) return false; state->target = nits; state->actual = nits; + state->level_known = true; return true; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 0b94b41b414827..a0aaad3ddd232d 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -292,6 +292,31 @@ static void backlight_fallback_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 262U); } +static void backlight_seed_known_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A loader sample seeded over a registration default is known. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, state.level_known); + KUNIT_EXPECT_TRUE(test, dcp_bl_seed(&state, 180)); + KUNIT_EXPECT_TRUE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_TRUE(test, dcp_bl_resend(&state)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 180U); + + /* A refused sample leaves the default unknown. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 526)); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.controlled = true; + KUNIT_EXPECT_FALSE(test, dcp_bl_seed(&state, 180)); + KUNIT_EXPECT_FALSE(test, state.level_known); + KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -304,6 +329,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), KUNIT_CASE(backlight_fallback_test), + KUNIT_CASE(backlight_seed_known_test), {} }; From f7abed8adebadc188490d09d0e09dfbed577e433 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:20 +1100 Subject: [PATCH 266/631] drm: apple: stop dropping DCP node references inside the match loops for_each_matching_node() hands each node to the loop body with a reference held and drops that reference itself when it moves on to the next node. apple_drm_init_dcp() also calls of_node_put() on every node it visits, and add_dcp_components() does the same, so every probe of the display subsystem drops one reference more than it took on each DCP node, and binding it drops another. The nodes are static and their platform devices keep further references, so nothing has been freed, but the counts are wrong, and repeated probe and bind attempts walk them towards a "Bad of_node_put()" error with CONFIG_OF_DYNAMIC. Leave the references to the iterator. Fixes: f2bc92ff49a9 ("drm/apple: Add DCP display driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index bf0c176af3e3dc..ced558ce38e3f1 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -831,12 +831,10 @@ static int apple_drm_init_dcp(struct device *dev) if (!of_device_is_available(np)) { of_node_put(np); continue; - } dcp_ext = of_device_is_compatible(np, "apple,dcpext") || of_property_present(np, "phys"); dcp[num_dcp] = of_find_device_by_node(np); - of_node_put(np); if (!dcp[num_dcp]) continue; /* A failed T8122 dcpext: bound, but no pipeline. */ @@ -1055,7 +1053,6 @@ static int add_dcp_components(struct device *dev, of_node_put(port); } } - of_node_put(np); } return num; From c836492c777c5f22565380319e6ba282a491fb99 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 05:08:39 +1100 Subject: [PATCH 267/631] drm: apple: do not log a missing EDID source for internal panels dcp_get_modes() used to ask for an EDID only once the DCP had announced its AV service. Since the EDID read pins the service itself, it is asked on every mode probe and returns -ENODEV when there is no service. Internal panels never have one, so every mode probe on a laptop now logs "copy_edid failed: -ENODEV" at info level. Treat -ENODEV on a DCP that drives an internal panel as the expected "no EDID" answer and log it at debug level. External outputs, and any other failure, are reported as before, and the EDID read itself is unchanged. Fixes: 5837706b10e3 ("drm/apple: pin DP service owners before recycling retired AFK slots") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8b7a91cfe5014f..b04bac0e9d864b 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1208,6 +1208,19 @@ int dcp_get_modes(struct drm_connector *connector) if (count) { const struct drm_edid *edid; + edid = dcpavserv_copy_edid(dcp); + if (IS_ERR_OR_NULL(edid)) { + /* + * An internal panel has no AV service and so no EDID; + * do not report that on every mode probe. + */ + if (dcp_has_panel(dcp) && PTR_ERR(edid) == -ENODEV) + dev_dbg(dcp->dev, "no EDID source for the panel\n"); + else + dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + } else { + drm_edid_free(apple_connector->drm_edid); + apple_connector->drm_edid = edid; u64 generation; /* The RPC can sleep while a disconnect or route change clears EDID. */ From 7880a2d3f47bdc7504704554bbd5ab70232a2cd2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 08:38:36 -0400 Subject: [PATCH 268/631] mm: reuse backing folios for small-granule anonymous COW Closing a subpage pool after each COW fault leaves every remaining slot unavailable for later faults. On a 16K kernel with a 4K process ABI, a 64MiB densely modified fork child therefore uses 256MiB of backing. Retain bounded, address-group pools for eligible faults in each mm. Keep charge domains and memory policy explicit, preserve independent fork ownership, and close pools after exit_mmap while referenced owners remain alive. Copy into a reserved slot under the folio lock, decide first rmap/LRU publication after relocking, and fall back to private backing if nonlinear rmap metadata cannot be allocated. Distribute concurrent faults across 16 address-group pools, skip busy backing on the normal path, and use exact per-offset free counts to avoid repeatedly scanning incompatible partial owners. Speculative allocation and charge do not invoke OOM; under pressure, retry reusable backing before ordinary allocation. Existing policy-sensitive, locked, droppable and multi-node faults keep private backing. Add a user4k selftest covering concurrent, sparse, phased and random COW, mlock and lazy-free transitions, and a descendant that retains data after the pool-owning mm exits. Each case checks untouched bytes and parent isolation. Physical density and reclaim were tested separately on Neo hardware; the selftest does not infer them from logical RSS. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5469651a0899f3bf776c90f4a24e25e952fa0ba2) --- include/linux/mm_subpage.h | 15 + include/linux/mm_types.h | 2 + kernel/fork.c | 3 + mm/memory.c | 152 +++++++-- mm/user-subpage.c | 297 +++++++++++++++++- tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../testing/selftests/arm64/user4k/cow-pool.c | 256 +++++++++++++++ 7 files changed, 690 insertions(+), 39 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/cow-pool.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 13344e3a92384c..c4ac2eb6c4236b 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -45,6 +45,21 @@ struct mm_subpage_pool *mm_subpage_pool_create_granule(gfp_t gfp, unsigned int s void mm_subpage_pool_get(struct mm_subpage_pool *pool); void mm_subpage_pool_put(struct mm_subpage_pool *pool); void mm_subpage_pool_close(struct mm_subpage_pool *pool); +#ifdef CONFIG_MM_SUBPAGE +/* Optional per-mm pool: default policy, one memory node, matching memcg. */ +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address); +void mm_subpage_cow_pool_exit(struct mm_struct *mm); +bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* Return an initialized slot with its folio locked; may wait without a pool lock. */ +struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, + unsigned int offset); +struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset); +#else +static inline void mm_subpage_cow_pool_exit(struct mm_struct *mm) { } +#endif /* Success consumes the folio reference. On error the caller retains it. */ int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 7f201dda7a13ba..46d6f13f6f3030 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1303,6 +1303,8 @@ struct mm_struct { #ifdef CONFIG_MM_SUBPAGE /* VM_ACCOUNT granules; global commitment rounds this sum once. */ unsigned long committed_user_pages; + /* Shared COW slots; never inherited by a forked mm. */ + struct mm_subpage_cow_pools *cow_subpage_pool; #endif unsigned long locked_vm; /* Locked bytes >> MM_ACCOUNT_SHIFT */ atomic64_t pinned_vm; /* Pinned native physical pages */ diff --git a/kernel/fork.c b/kernel/fork.c index 88788baaff964c..a9fedd8b8abacb 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -1097,6 +1098,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm->map_count = 0; #ifdef CONFIG_MM_SUBPAGE mm->committed_user_pages = 0; + mm->cow_subpage_pool = NULL; #endif mm->locked_vm = 0; atomic64_set(&mm->pinned_vm, 0); @@ -1212,6 +1214,7 @@ static inline void __mmput(struct mm_struct *mm) ksm_exit(mm); khugepaged_exit(mm); /* must run before exit_mmap */ exit_mmap(mm); + mm_subpage_cow_pool_exit(mm); mm_put_huge_zero_folio(mm); set_mm_exe_file(mm, NULL); if (!list_empty(&mm->mmlist)) { diff --git a/mm/memory.c b/mm/memory.c index ade3dbadb409a4..2fb430e8426035 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -88,6 +88,7 @@ #include "pgalloc-track.h" #include +#include "user-subpage-internal.h" #include "internal.h" #include "swap.h" @@ -1325,6 +1326,98 @@ static inline struct folio *folio_prealloc(struct mm_struct *src_mm, } #ifdef CONFIG_MM_SUBPAGE +/* Skipping a busy owner must not trigger an avoidable global or memcg OOM. */ +static struct folio *cow_subpage_speculative_folio(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr) +{ + gfp_t alloc_gfp = GFP_HIGHUSER_MOVABLE | __GFP_NORETRY | __GFP_NOWARN; + gfp_t charge_gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; + unsigned int order = max_t(unsigned int, mm_page_shift(mm), PAGE_SHIFT) - PAGE_SHIFT; + struct folio *folio = vma_alloc_folio(alloc_gfp, order, vma, addr); + + if (!folio) + return NULL; + if (mem_cgroup_charge(folio, mm, charge_gfp)) { + folio_put(folio); + return NULL; + } + folio_throttle_swaprate(folio, charge_gfp); + return folio; +} + +/* The returned initialized slot owns a reference and has its folio locked. */ +static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, + struct mm_subpage *source, bool force_private, bool *pooled) +{ + struct mm_subpage_pool *pool; + struct mm_subpage *slot; + struct folio *folio; + unsigned int offset = vma_page_offset_at(vma, addr).offset; + int err; + +retry_private: + pool = force_private ? NULL : mm_subpage_cow_pool_get(mm, vma, addr); + *pooled = pool != NULL; + if (!pool) + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return ERR_PTR(-ENOMEM); + for (;;) { + slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : + mm_subpage_alloc_at_locked(pool, offset); + if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) + break; + folio = NULL; + if (*pooled) { + folio = cow_subpage_speculative_folio(mm, vma, addr); + if (!folio && mm_subpage_cow_wait_busy_at(pool, offset)) { + cond_resched(); + continue; + } + } + /* No reusable owner exists after speculative allocation failed. + * Ordinary allocation/charge semantics are now appropriate. + */ + if (!folio) + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) { + slot = ERR_PTR(-ENOMEM); + break; + } + /* A concurrent cgroup move may change the fresh folio's charge. */ + if (*pooled && !mm_subpage_cow_folio_matches(pool, folio)) { + folio_put(folio); + mm_subpage_pool_put(pool); + force_private = true; + goto retry_private; + } + err = mm_subpage_pool_add_folio_at(pool, folio, offset, + *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + if (*pooled && err == -ENOMEM) { + if (mm_subpage_cow_wait_busy_at(pool, offset)) { + folio_put(folio); + cond_resched(); + continue; + } + err = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + } + if (err) { + folio_put(folio); + if (err == -EEXIST) { + cond_resched(); + continue; + } + slot = ERR_PTR(err); + break; + } + } + if (!*pooled) + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return slot; +} + static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { @@ -4541,11 +4634,11 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; struct mm_subpage *source = NULL, *dest = NULL; - struct mm_subpage_pool *pool = NULL; struct folio *old = vmf->page ? page_folio(vmf->page) : NULL; struct folio *new = NULL; struct mmu_notifier_range range; bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; + bool force_private = false, pooled, fresh, retry_private = false; unsigned int offset; vm_fault_t ret = VM_FAULT_SIGBUS; pte_t entry; @@ -4598,35 +4691,29 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto out; offset = vma_page_offset_at(vma, vmf->address).offset; ret = VM_FAULT_OOM; - pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!pool) - goto out; - new = folio_prealloc(mm, vma, vmf->address, false); - if (!new) - goto out; - err = mm_subpage_pool_add_folio(pool, new, GFP_KERNEL); - if (err) { - folio_put(new); - goto out; - } - dest = source ? mm_subpage_copy_at(pool, source, offset) : - mm_subpage_alloc_at(pool, offset); +retry_copy: + dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); if (IS_ERR(dest)) { if (PTR_ERR(dest) == -EHWPOISON) ret = VM_FAULT_HWPOISON; dest = NULL; goto out; } + new = mm_subpage_folio(dest); if (old && !source) { err = copy_user_subpage_range(&new->page, offset, vmf->page, pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, mm_page_size(mm)); if (err) { + folio_unlock(new); ret = VM_FAULT_HWPOISON; goto out; } } - mm_subpage_pool_close(pool); + /* Initialisation is complete; the slot reference prevents migration/free. + * Do not carry a folio lock through MMU notifier callbacks. + */ + folio_unlock(new); mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, vmf->address, vmf->address + mm_page_size(mm)); mmu_notifier_invalidate_range_start(&range); @@ -4634,6 +4721,19 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) folio_get(new); folio_lock(new); ret = 0; + /* A different fault may publish this owner's first PTE while we wait. + * Only the first publisher establishes folio rmap/LRU. Preflight nonlinear + * slot identity before taking PTL, with a fresh-folio fallback on ENOMEM. + */ + fresh = !mm_subpage_anon_root(dest); + if (!fresh) { + err = mm_subpage_bind_rmap(dest, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; + } + } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) goto unlock_folio; @@ -4642,7 +4742,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto unlock_pte; } mm_subpage_set_exclusive(dest); - err = mm_subpage_add_new_anon_rmap(dest, vma, vmf->address); + err = fresh ? mm_subpage_add_new_anon_rmap(dest, vma, vmf->address) : + mm_subpage_add_anon_rmap(dest, vma, vmf->address); if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; goto unlock_pte; @@ -4660,7 +4761,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); /* Break-before-make also orders removal of the old slot's rmap. */ ptep_clear_flush(vma, vmf->address, vmf->pte); - subpage_add_new_lru(new, vma, 1); + if (fresh) + subpage_add_new_lru(new, vma, 1); set_pte_at(mm, vmf->address, vmf->pte, entry); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); if (source) { @@ -4683,13 +4785,17 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) folio_unlock(new); folio_put(new); mmu_notifier_invalidate_range_end(&range); -out: - if (pool) { - mm_subpage_pool_close(pool); - if (dest) - mm_subpage_put(dest); - mm_subpage_pool_put(pool); + if (retry_private) { + mm_subpage_put(dest); + dest = NULL; + force_private = true; + retry_private = false; + ret = VM_FAULT_OOM; + goto retry_copy; } +out: + if (dest) + mm_subpage_put(dest); if (source) { if (copied) mm_subpage_put(source); /* Retired PTE reference, after TLBI. */ diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 319d792ab5e9a1..3adfb04ccdca16 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -222,6 +224,17 @@ struct mm_subpage_pool { unsigned long all_slots; u8 shift; bool closed; + /* Only a per-mm COW pool owns this charge-domain reference. */ + struct mem_cgroup *cow_memcg; + int cow_nid; + bool cow_shared; + /* Allocated only for shared COW pools; protected by pool->lock. */ + unsigned long *cow_free_slots; +}; + +#define COW_POOL_BITS 4 +struct mm_subpage_cow_pools { + struct mm_subpage_pool *pool[1U << COW_POOL_BITS]; }; static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, @@ -277,13 +290,147 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) if (!refcount_dec_and_test(&pool->refs)) return; WARN_ON_ONCE(!pool->closed || !list_empty(&pool->owners)); + mem_cgroup_put(pool->cow_memcg); + kfree(pool->cow_free_slots); kfree_rcu(pool, rcu); } +/* Keep the first implementation within one unambiguous allocation domain. + * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. + * An mm can move cgroups; old charged slots must not satisfy its new faults. + */ +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address) +{ + struct mm_subpage_cow_pools *pools, *new_pools; + struct mm_subpage_pool *pool, *new; + unsigned int bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + struct mem_cgroup *memcg; + bool matches; + int nid; + + if ((vma->vm_flags & (VM_LOCKED | VM_DROPPABLE | VM_SPECIAL)) || + num_online_nodes() != 1 || num_node_state(N_MEMORY) != 1) + return NULL; +#ifdef CONFIG_NUMA + if (vma_policy(vma) || current->mempolicy) + return NULL; +#endif + nid = first_node(node_states[N_MEMORY]); + if (!node_isset(nid, cpuset_current_mems_allowed)) + return NULL; + /* Publication pairs with cmpxchg: every bucket starts empty. Adjacent + * 4K leaves in one native virtual group always choose the same pool. + * Faulting threads can migrate CPUs without changing that identity. + */ + pools = smp_load_acquire(&mm->cow_subpage_pool); + if (!pools) { + new_pools = kzalloc(sizeof(*new_pools), GFP_KERNEL | __GFP_ACCOUNT); + if (!new_pools) + return NULL; + pools = cmpxchg(&mm->cow_subpage_pool, NULL, new_pools); + if (!pools) + pools = new_pools; + else + kfree(new_pools); + } + memcg = get_mem_cgroup_from_mm(mm); + /* Acquire the immutable charge domain published with this bucket. */ + pool = smp_load_acquire(&pools->pool[bucket]); + if (!pool) { + new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!new) { + mem_cgroup_put(memcg); + return NULL; + } + new->cow_free_slots = kcalloc(SUBPAGES_PER_FOLIO, + sizeof(*new->cow_free_slots), + GFP_KERNEL | __GFP_ACCOUNT); + if (!new->cow_free_slots) { + mm_subpage_pool_close(new); + mm_subpage_pool_put(new); + mem_cgroup_put(memcg); + return NULL; + } + new->cow_memcg = memcg; + new->cow_nid = nid; + new->cow_shared = true; + pool = cmpxchg(&pools->pool[bucket], NULL, new); + if (!pool) { + pool = new; + /* The mm owns the initial pool reference and its memcg ref. */ + mm_subpage_pool_get(pool); + return pool; + } + matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + mm_subpage_pool_close(new); + mm_subpage_pool_put(new); + } else { + matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + mem_cgroup_put(memcg); + } + if (!matches) + return NULL; + mm_subpage_pool_get(pool); + return pool; +} + +bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) +{ + return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); +} + +void mm_subpage_cow_pool_exit(struct mm_struct *mm) +{ + struct mm_subpage_cow_pools *pools = xchg(&mm->cow_subpage_pool, NULL); + unsigned int i; + + /* No faults remain once mm_users reaches zero. Slot owners can outlive mm. */ + if (!pools) + return; + for (i = 0; i < ARRAY_SIZE(pools->pool); i++) { + if (!pools->pool[i]) + continue; + mm_subpage_pool_close(pools->pool[i]); + mm_subpage_pool_put(pools->pool[i]); + } + kfree(pools); +} + +/* Counts include unavailable (busy/migrating) owners, so zero is conclusive. + * A nonzero count still requires all the normal owner eligibility checks. + */ +static void cow_update_free(struct mm_subpage_pool *pool, unsigned long mask, int delta) +{ + unsigned int i; + + lockdep_assert_held(&pool->lock); + if (!pool->cow_shared) + return; + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + VM_BUG_ON(delta < 0 && !pool->cow_free_slots[i]); + pool->cow_free_slots[i] += delta; + } +} + +static bool cow_has_free(struct mm_subpage_pool *pool, unsigned long eligible) +{ + unsigned int i; + + lockdep_assert_held(&pool->lock); + if (!pool->cow_shared) + return true; + for_each_set_bit(i, &eligible, SUBPAGES_PER_FOLIO) + if (pool->cow_free_slots[i]) + return true; + return false; +} + static void owner_remove(struct mm_subpage_owner *owner) { lockdep_assert_held(&owner->pool->lock); owner->detached = true; + cow_update_free(owner->pool, owner->pool->all_slots & ~owner->used, -1); list_del(&owner->all); list_del_init(&owner->available); if (owner->pool->unissued == owner) @@ -677,17 +824,81 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) owner_free(owner, true); } +static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio) +{ + if (!pool->cow_shared) + return true; + /* A previously unlocked owner can be mlocked or fully MADV_FREE'd + * after entering the pool. Never place a new live slot in such backing. + */ + return !folio_test_mlocked(folio) && !folio_test_unevictable(folio) && + !folio_test_ksm(folio) && + (!folio_test_anon(folio) || folio_test_swapbacked(folio)) && + mm_subpage_cow_folio_matches(pool, folio); +} + +/* Pressure-only fallback after a non-OOM speculative allocation failed. + * Return true when existing compatible backing can be retried. Unlike the + * normal bounded search, inspect all owners before allowing an OOM-capable + * allocation: a busy slot farther down the list may require no new charge. + */ +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + struct mm_subpage_owner *owner; + struct folio *wait = NULL; + unsigned long flags, eligible = offset_mask(pool->shift, offset); + + spin_lock_irqsave(&pool->lock, flags); + if (!pool->closed && pool->cow_shared && cow_has_free(pool, eligible)) { + list_for_each_entry(owner, &pool->available, available) { + if (!owner->migrating && (~owner->used & eligible) && + cow_owner_eligible(pool, owner->folio) && + !folio_test_swapcache(owner->folio) && + !folio_test_writeback(owner->folio)) { + wait = owner->folio; + folio_get(wait); + break; + } + } + } + spin_unlock_irqrestore(&pool->lock, flags); + if (wait) { + folio_wait_locked(wait); + folio_put(wait); + return true; + } + return false; +} + static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, unsigned long eligible) { struct mm_subpage_owner *owner; + unsigned int visited = 0; lockdep_assert_held(&pool->lock); - list_for_each_entry(owner, &pool->available, available) + if (!cow_has_free(pool, eligible)) + return NULL; + list_for_each_entry(owner, &pool->available, available) { + /* A fixed-offset stream must not scan every partially used owner + * on each fault. New COW supply is inserted first, so a bounded + * miss can always make progress using a fresh folio. + */ + if (pool->cow_shared && visited++ == 32) { + /* The next search starts past this incompatible prefix. A + * phased offset stream must not strand older reusable slots. + */ + list_rotate_to_front(&owner->available, &pool->available); + break; + } /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && - !folio_test_swapcache(owner->folio)) + (!pool->cow_shared || !folio_test_locked(owner->folio)) && + cow_owner_eligible(pool, owner->folio) && + !folio_test_swapcache(owner->folio) && + !folio_test_writeback(owner->folio)) return owner; + } return NULL; } @@ -720,13 +931,20 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (find_available(pool, eligible)) - /* Another allocator supplied backing while we allocated metadata. */ + else if (pool->unissued || find_available(pool, eligible)) + /* Another allocator supplied backing while we allocated metadata. + * Preserve the single-unissued-owner invariant even when a shared + * COW lookup skips a concurrently locked or rotating list prefix. + */ ret = -EEXIST; else { mm_subpage_pool_get(pool); list_add_tail(&owner->all, &pool->owners); - list_add_tail(&owner->available, &pool->available); + cow_update_free(pool, pool->all_slots, 1); + if (pool->cow_shared) + list_add(&owner->available, &pool->available); + else + list_add_tail(&owner->available, &pool->available); pool->backing_pages++; pool->unissued = owner; } @@ -765,13 +983,14 @@ unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) } static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, - unsigned long eligible, bool *needs_zero) + unsigned long eligible, bool *needs_zero, bool lock_folio) { struct mm_subpage_owner *owner; struct mm_subpage *subpage; unsigned long flags; unsigned int index; +retry: spin_lock_irqsave(&pool->lock, flags); if (pool->closed) { subpage = ERR_PTR(-ESHUTDOWN); @@ -782,6 +1001,34 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-EAGAIN); goto out; } + if (lock_folio) { + if (!folio_trylock(owner->folio)) { + struct folio *wait; + + if (pool->cow_shared) { + /* Another COW fault won this folio after the lookup. + * Try other backing, or supply another folio, rather + * than making parallel faults queue on its lock. + */ + spin_unlock_irqrestore(&pool->lock, flags); + cond_resched(); + goto retry; + } + wait = owner->folio; + folio_get(wait); + spin_unlock_irqrestore(&pool->lock, flags); + folio_wait_locked(wait); + folio_put(wait); + goto retry; + } + /* Reclaim may have acquired the folio after find_available's check. */ + if (folio_test_swapcache(owner->folio) || folio_test_writeback(owner->folio) || + !cow_owner_eligible(pool, owner->folio)) { + folio_unlock(owner->folio); + spin_unlock_irqrestore(&pool->lock, flags); + return ERR_PTR(-EAGAIN); + } + } if (pool->unissued == owner) /* Transfer the supplied reference to the first reservation. */ pool->unissued = NULL; @@ -789,6 +1036,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, folio_get(owner->folio); index = __ffs(~owner->used & eligible); __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); if (needs_zero) *needs_zero = !test_bit(index, &owner->zeroed); __clear_bit(index, &owner->zeroed); @@ -804,10 +1052,10 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return subpage; } -static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible) +static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; - struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero); + struct mm_subpage *subpage = subpage_reserve(pool, eligible, &needs_zero, lock_folio); if (IS_ERR(subpage)) return subpage; @@ -822,7 +1070,7 @@ static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned l struct mm_subpage *mm_subpage_alloc(struct mm_subpage_pool *pool) { - return subpage_alloc(pool, pool->all_slots); + return subpage_alloc(pool, pool->all_slots, false); } struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned int offset) @@ -831,7 +1079,7 @@ struct mm_subpage *mm_subpage_alloc_at(struct mm_subpage_pool *pool, unsigned in if (!eligible) return ERR_PTR(-EINVAL); - return subpage_alloc(pool, eligible); + return subpage_alloc(pool, eligible, false); } static void subpage_release_locked(struct mm_subpage *subpage, unsigned long flags) @@ -851,6 +1099,7 @@ static void subpage_release_locked(struct mm_subpage *subpage, unsigned long fla subpage->rmap_bound = false; } __clear_bit(subpage->index, &owner->used); + cow_update_free(pool, BIT(subpage->index), 1); release = !owner->used && !owner->migrating; if (release) owner_remove(owner); @@ -872,14 +1121,14 @@ static void subpage_release(struct mm_subpage *subpage) } static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, - const struct mm_subpage *source, unsigned long eligible) + const struct mm_subpage *source, unsigned long eligible, bool lock_folio) { struct mm_subpage *dest; int err; if (pool->shift != mm_subpage_shift(source)) return ERR_PTR(-EINVAL); - dest = subpage_reserve(pool, eligible, NULL); + dest = subpage_reserve(pool, eligible, NULL, lock_folio); if (IS_ERR(dest)) return dest; @@ -888,6 +1137,8 @@ static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, mm_subpage_size(source)); if (err) { /* Never publish a reservation whose copy did not finish. */ + if (lock_folio) + folio_unlock(dest->owner->folio); subpage_release(dest); return ERR_PTR(err); } @@ -898,7 +1149,7 @@ static struct mm_subpage *subpage_copy(struct mm_subpage_pool *pool, struct mm_subpage *mm_subpage_copy(struct mm_subpage_pool *pool, const struct mm_subpage *source) { - return subpage_copy(pool, source, pool->all_slots); + return subpage_copy(pool, source, pool->all_slots, false); } struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, @@ -908,7 +1159,23 @@ struct mm_subpage *mm_subpage_copy_at(struct mm_subpage_pool *pool, if (!eligible) return ERR_PTR(-EINVAL); - return subpage_copy(pool, source, eligible); + return subpage_copy(pool, source, eligible, false); +} + +struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, + unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + return eligible ? subpage_alloc(pool, eligible, true) : ERR_PTR(-EINVAL); +} + +struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source, unsigned int offset) +{ + unsigned long eligible = offset_mask(pool->shift, offset); + + return eligible ? subpage_copy(pool, source, eligible, true) : ERR_PTR(-EINVAL); } /* The page-extension RCU section also protects the pool until its lock is held. */ @@ -956,6 +1223,7 @@ static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int o else folio_get(folio); __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); __clear_bit(index, &owner->zeroed); if (owner->used == pool->all_slots) list_del_init(&owner->available); @@ -1117,6 +1385,7 @@ int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) mm_subpage_pool_get(pool); spin_lock_irqsave(&pool->lock, flags); list_add_tail(&new->all, &pool->owners); + cow_update_free(pool, pool->all_slots, 1); pool->backing_pages++; ret = owner_bind(new); if (ret) diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index cc89cc93293fed..4b7c4cf17006d2 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -2,7 +2,7 @@ TEST_GEN_PROGS := page-contract proc-args TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init -TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init +TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init cow-pool CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -12,7 +12,7 @@ include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init: LDLIBS += -pthread +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/cow-pool.c b/tools/testing/selftests/arm64/user4k/cow-pool.c new file mode 100644 index 00000000000000..bb25904d68c04a --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/cow-pool.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Exercise shared COW backing under a selected 4K userspace ABI. Run through + * the platform's ABI launcher; this test never changes its own page contract. + * Every selected leaf changes one byte, and all other bytes must survive. + * Physical density and reclaim require privileged, platform-specific controls + * and are deliberately not inferred from this unprivileged contents test. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define BYTES (64UL << 20) +#define LEAF 4096UL +#define LEAVES (BYTES / LEAF) +#define WORKERS 8 + +enum mode { DENSE, SPARSE, PHASED, RANDOM, LOCKED, LATE_LOCKED, LAZYFREE }; +struct scenario { + const char *name; + enum mode mode; + unsigned int workers; +}; + +static unsigned char *data; +static size_t order[LEAVES]; +static pthread_barrier_t barrier; +static const struct scenario *current; + +static unsigned char initial(size_t leaf, size_t byte) +{ + return leaf * 17 + byte * 3 + 0x35; +} + +static unsigned char changed(size_t leaf) +{ + return initial(leaf, 0) ^ 0xa5; +} + +static bool verify(unsigned int generation, bool discard) +{ + unsigned int stride = current->mode == SPARSE ? 4 : 1; + size_t leaf, byte; + + for (leaf = 0; leaf < LEAVES; leaf++) { + if (generation && discard && leaf % 4 == 0) + continue; + for (byte = 0; byte < LEAF; byte++) { + unsigned char want = initial(leaf, byte); + + if (generation && byte == 0 && leaf % stride == 0) + want ^= generation == 1 ? 0xa5 : 0x5a; + if (data[leaf * LEAF + byte] != want) { + ksft_print_msg("pid=%d leaf=%zu byte=%zu got=%u want=%u\n", + getpid(), leaf, byte, data[leaf * LEAF + byte], want); + return false; + } + } + } + return true; +} + +static bool wait_ok(pid_t pid) +{ + pid_t result; + int status; + + do { + result = waitpid(pid, &status, 0); + } while (result < 0 && errno == EINTR); + return result > 0 && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +static void *writer(void *arg) +{ + unsigned int id = (uintptr_t)arg; + size_t leaf, index; + unsigned int phase; + + pthread_barrier_wait(&barrier); + if (current->mode == PHASED || current->mode == LATE_LOCKED || + current->mode == LAZYFREE) { + for (phase = 0; phase < 4; phase++) { + for (leaf = phase + 4UL * id; leaf < LEAVES; + leaf += 4UL * current->workers) + data[leaf * LEAF] = changed(leaf); + pthread_barrier_wait(&barrier); + if (phase == 0 && id == 0) { + if (current->mode == LATE_LOCKED && mlock(data, BYTES)) + _exit(2); + if (current->mode == LAZYFREE) + for (leaf = 0; leaf < LEAVES; leaf += 4) + if (madvise(data + leaf * LEAF, LEAF, MADV_FREE)) + _exit(2); + } + pthread_barrier_wait(&barrier); + } + } else { + unsigned int stride = current->mode == SPARSE ? 4 : 1; + + for (index = id; index < LEAVES / stride; index += current->workers) { + leaf = current->mode == RANDOM ? order[index] : index * stride; + data[leaf * LEAF] = changed(leaf); + } + } + return NULL; +} + +static bool exercise(const struct scenario *scenario) +{ + pthread_t workers[WORKERS]; + struct timespec start, end; + unsigned int t; + pid_t child; + + current = scenario; + child = fork(); + if (child < 0) + return false; + if (!child) { + if (current->mode == LOCKED && mlock(data, BYTES)) + _exit(2); + if (pthread_barrier_init(&barrier, NULL, current->workers)) + _exit(2); + clock_gettime(CLOCK_MONOTONIC, &start); + for (t = 0; t < current->workers; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + _exit(2); + for (t = 0; t < current->workers; t++) + if (pthread_join(workers[t], NULL)) + _exit(2); + clock_gettime(CLOCK_MONOTONIC, &end); + ksft_print_msg("%s fault/write phase %.6f s\n", current->name, + end.tv_sec - start.tv_sec + (end.tv_nsec - start.tv_nsec) * 1e-9); + _exit(verify(1, current->mode == LAZYFREE) ? 0 : 3); + } + return wait_ok(child) && verify(0, false); +} + +static bool owner_exit(void) +{ + static const struct scenario scenario = { "owner-mm exit", DENSE, 1 }; + int ready[2]; + pid_t child, descendant; + size_t leaf; + char signal; + + current = &scenario; + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(ready)) + return false; + child = fork(); + if (child < 0) { + close(ready[0]); + close(ready[1]); + return false; + } + if (!child) { + close(ready[1]); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = changed(leaf); + if (!verify(1, false)) + _exit(3); + descendant = fork(); + if (descendant < 0) + _exit(2); + if (descendant) + _exit(0); + /* Root signals only after the original pool-owning mm has exited. */ + if (read(ready[0], &signal, 1) != 1 || signal != 'g') + _exit(2); + close(ready[0]); + if (!verify(1, false)) + _exit(3); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] ^= 0xff; + _exit(verify(2, false) ? 0 : 3); + } + close(ready[0]); + if (!wait_ok(child)) { + close(ready[1]); + return false; + } + if (write(ready[1], "g", 1) != 1) { + close(ready[1]); + return false; + } + close(ready[1]); + return wait_ok(-1) && verify(0, false); +} + +int main(void) +{ + static const struct scenario scenarios[] = { + { "dense single-thread COW", DENSE, 1 }, + { "dense concurrent COW", DENSE, WORKERS }, + { "single-offset sparse COW", SPARSE, 1 }, + { "phased-offset concurrent COW", PHASED, WORKERS }, + { "random-order concurrent COW", RANDOM, WORKERS }, + { "locked COW", LOCKED, WORKERS }, + { "mlock after first offset", LATE_LOCKED, WORKERS }, + { "MADV_FREE after first offset", LAZYFREE, WORKERS }, + }; + struct rlimit limit; + uint32_t seed = 0x9148140; + size_t leaf, byte, i, j, tmp; + bool can_lock; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("run under a selected 4K userspace ABI\n"); + ksft_set_plan(9); + alarm(180); + data = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (data == MAP_FAILED) + ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + for (leaf = 0; leaf < LEAVES; leaf++) + for (byte = 0; byte < LEAF; byte++) + data[leaf * LEAF + byte] = initial(leaf, byte); + for (i = 0; i < LEAVES; i++) + order[i] = i; + for (i = LEAVES; i > 1; i--) { + seed = seed * 1664525 + 1013904223; + j = seed % i; + tmp = order[j]; + order[j] = order[i - 1]; + order[i - 1] = tmp; + } + can_lock = !getrlimit(RLIMIT_MEMLOCK, &limit) && limit.rlim_cur >= BYTES; + for (i = 0; i < sizeof(scenarios) / sizeof(scenarios[0]); i++) { + if (!can_lock && (scenarios[i].mode == LOCKED || + scenarios[i].mode == LATE_LOCKED)) { + ksft_test_result_skip("%s needs a 64MiB memlock limit\n", scenarios[i].name); + continue; + } + ksft_test_result(exercise(&scenarios[i]), "%s preserves contents and parent\n", + scenarios[i].name); + } + ksft_test_result(owner_exit(), "descendant retains COW data after owner mm exits\n"); + munmap(data, BYTES); + ksft_finished(); +} From 81a74349713979ad3481ef8b7a8d0ccfce065fdc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:11 +1100 Subject: [PATCH 269/631] dt-bindings: soc: apple: Add the T8140 memory cache controller The memory cache controller of the Apple T8140 holds the system level cache in front of DRAM, split into planes. The boot firmware enables and powers the cache ways of every plane but leaves allocation disabled, so the operating system has to allow it. Describe one register block per plane, and the devices that take over DMA the bootloader left running: allocation must wait until they are bound. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/soc/apple/apple,t8140-mcc.yaml | 58 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 59 insertions(+) create mode 100644 Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml diff --git a/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml b/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml new file mode 100644 index 00000000000000..e4c5688a01d7f9 --- /dev/null +++ b/Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/soc/apple/apple,t8140-mcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple T8140 memory cache controller + +maintainers: + - Ryan Murray + +description: + The memory cache controller of the Apple T8140 SoC holds the system level + cache in front of DRAM, split into planes. The boot firmware enables and + powers the cache ways of every plane but leaves allocation disabled, so the + cache holds no data until the operating system allows allocation. + +properties: + compatible: + const: apple,t8140-mcc + + reg: + description: The register block of each cache plane. + minItems: 1 + maxItems: 8 + + apple,boot-dma-devices: + $ref: /schemas/types.yaml#/definitions/phandle-array + items: + maxItems: 1 + description: + Devices that take over DMA the bootloader left running. Cache allocation + is enabled only once each of them is bound to its driver. + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + display: display-subsystem { + }; + + memory-controller@220000000 { + compatible = "apple,t8140-mcc"; + reg = <0x2 0x20000000 0x0 0x4000>, + <0x2 0x20040000 0x0 0x4000>, + <0x2 0x20080000 0x0 0x4000>, + <0x2 0x200c0000 0x0 0x4000>; + apple,boot-dma-devices = <&display>; + }; + }; diff --git a/MAINTAINERS b/MAINTAINERS index 29d99053ebb184..2ff5f3d0c33467 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1999,6 +1999,7 @@ F: Documentation/devicetree/bindings/serial/apple,dockchannel-uart.yaml F: Documentation/devicetree/bindings/soc/apple/apple,dockchannel.yaml F: Documentation/devicetree/bindings/soc/apple/apple,rtk-helper-asc4.yaml F: Documentation/devicetree/bindings/soc/apple/apple,t8140-aop.yaml +F: Documentation/devicetree/bindings/soc/apple/apple,t8140-mcc.yaml F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst From 53c8297f323d8994a86b0d4c7c16121fb1936cac Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 15:16:21 +1100 Subject: [PATCH 270/631] soc: apple: Enable T8140 system level cache allocation On T8140 the boot firmware enables and powers the 12 ways of each of the 4 system level cache planes, but leaves the allocation-disable word of every plane set. The cache therefore holds no data, and every GPU and CPU access that misses the core caches goes to DRAM. Clear the allocation-disable word of each plane once every plane reports as many ways powered as enabled and no flush in progress, and read it back. Do it only after the devices named in the device tree as having taken over DMA from the bootloader are bound, so the cache starts allocating after their hand-off rather than during it. On shutdown, stop allocating and flush the cache before the next boot stage runs. With allocation disabled the cache holds no data, so no maintenance is needed before enabling it. Measured on J700 (MacBook Neo), toggling only this word on a running system: glmark2 build+texture 902 -> 1886, vkcube 2141 -> 5584 fps, glmark2 terrain at 1920x1200 41 -> 66 fps, and the submit-to-completion latency of an isolated GPU job 560 -> 210 us. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 2ff5f3d0c33467..0b792ed332e4e2 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2015,6 +2015,7 @@ F: drivers/iio/common/aop_sensors/ F: drivers/rtc/rtc-apple-pmu.c F: drivers/pci/controller/pcie-apple-piodma* F: drivers/soc/apple/aop.rs +F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* F: include/linux/soc/apple/pci-apple-piodma.h F: include/uapi/linux/apple-sep.h From 383a8f831717741eea91380c4cda2b49698559e6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 11:12:21 +1100 Subject: [PATCH 271/631] dt-bindings: cpu: apple,t8140-cpuidle: Correct the idle description The description calls the T8140 WFI state "clock-gated" and says the power-down states are "not enabled". On a J700, WFI already lets the hardware power-gate idle cores: the PMGR power-state registers of the idle E cores and of the second P core read as power-gated in 94% to 99.9% of 1 kHz samples. What T8140 lacks is a way for software to ask for power-down, so there is no separate state to enable. Describe that instead. No property changes. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/cpu/apple,t8140-cpuidle.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml index aa27ec55309c8b..7d4f9cdbc91829 100644 --- a/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml +++ b/Documentation/devicetree/bindings/cpu/apple,t8140-cpuidle.yaml @@ -10,9 +10,10 @@ maintainers: - Ryan Murray description: - Selects the Apple CPU idle driver on T8140 systems. Only the WFI - clock-gated state is supported; CPU and cluster power-down states - are not enabled on this SoC. + Selects the Apple CPU idle driver on T8140 systems. On this SoC the + hardware decides when an idle core is power-gated after WFI; software + cannot request core or cluster power-down, so WFI is the only idle + state. properties: $nodename: From a4f175ed1802f4fe2e1353ca9e10ec7a4b5b64a7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:22 -0400 Subject: [PATCH 272/631] dt-bindings: power: apple,t8140-pmp-report: describe report entries The PMP chooses DRAM and fabric performance states partly from the power state that the host reports for the devices it tracks. The host sets a device's bit in the PMP's power-state request word after it powers the device on, clears it before it powers the device off, and waits for the PMP to acknowledge the devices that require it. The pmp-report driver already provides such report entries as power domains on earlier SoCs. Allow them as children of the T8140 report aperture, with the device's PMP soc-device index as the unit address. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../power/apple,t8140-pmp-report.yaml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 9b515685eef00e..0cddaf6136379c 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -27,6 +27,51 @@ properties: power-domains: maxItems: 1 + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + +patternProperties: + '^report@[0-9a-f]+$': + type: object + description: | + A device whose power state the host reports to the PMP, provided as a + power domain. The host sets the device's request bit after its parent + domain powers on and clears it before the parent powers off, and waits + for the PMP to acknowledge while the PMP runs. + additionalProperties: false + + properties: + compatible: + items: + - const: apple,t8140-pmp-v2-report-entry + - const: apple,t6000-pmp-v2-report-entry + + reg: + maxItems: 1 + description: PMP soc-device index of the device. + + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + + apple,always-on: + type: boolean + description: Report the device as on from probe. + + required: + - compatible + - reg + - label + - '#power-domain-cells' + required: - compatible - reg @@ -42,5 +87,15 @@ examples: pmp-report@3003c0000 { compatible = "apple,t8140-pmp-v2-report"; reg = <3 0x003c0000 0 0x14000>; + #address-cells = <1>; + #size-cells = <0>; + + report@9 { + compatible = "apple,t8140-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + }; }; }; From cef8b165da46939f9a6adbf87d494ecbd574d800 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:22 -0400 Subject: [PATCH 273/631] dt-bindings: power: add Apple PMC voter interface The PMC arbitrates the performance states of shared rails, such as the DRAM controller and the SoC fabric, between voter agents: the PMP, the host performance controller (CLPC), the GPU and the application processor. Each agent has a perf-state floor word with one 4-bit state per rail, and each rail has an interface enable word with one bit per agent; the PMC ignores the floor of an agent that is not enabled on a rail. Describe the voter and rail windows, the static agents, the dynamic agents as power domains, and the host performance controller's agent. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../devicetree/bindings/power/apple,pmc.yaml | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 Documentation/devicetree/bindings/power/apple,pmc.yaml diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml new file mode 100644 index 00000000000000..69cdc1cb87197a --- /dev/null +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/power/apple,pmc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple SoC PMC voter interface + +maintainers: + - Diljit Singh + +description: | + The PMC arbitrates the performance states of shared rails, such as the + DRAM controller and the SoC fabric, between voter agents. Each agent has a + perf-state floor word with one 4-bit state per rail, and each rail has an + interface enable word with one bit per agent. The PMC only honours an + agent's floor on rails where its interface is enabled. + + Static agents are enabled for as long as the PMC is bound. A dynamic + agent is a power domain: it is enabled after its parent domain powers on + and disabled before its parent domain powers off. The host performance + controller's agent is described by a child node that raises its floor + while the CPU clusters or the GPU are busy. + +properties: + compatible: + const: apple,t8140-pmc + + reg: + items: + - description: Perf-state floor words of the voter agents + - description: Rail state and interface enable words + + reg-names: + items: + - const: voters + - const: rails + + power-domains: + maxItems: 1 + + '#address-cells': + const: 1 + + '#size-cells': + const: 0 + + apple,static-voters: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: Voter agents to enable on every rail. + +patternProperties: + '^voter@[0-9a-f]+$': + type: object + description: A dynamic voter agent, provided as a power domain. + additionalProperties: false + + properties: + reg: + maxItems: 1 + description: Voter agent index. + + label: + description: Name of the power domain. + + '#power-domain-cells': + const: 0 + + power-domains: + maxItems: 1 + description: Domain that must be powered while the agent is enabled. + + required: + - reg + - '#power-domain-cells' + + '^performance-controller@[0-9a-f]+$': + type: object + description: The host performance controller's voter agent. + additionalProperties: false + + properties: + compatible: + const: apple,t8140-clpc + + reg: + maxItems: 1 + description: Voter agent index. + + apple,gpu-power-state: + $ref: /schemas/types.yaml#/definitions/phandle + description: PMGR power state of the GPU, used as a demand input. + + required: + - compatible + - reg + +required: + - compatible + - reg + - reg-names + +additionalProperties: false + +examples: + - | + soc { + #address-cells = <2>; + #size-cells = <2>; + + power-management@302800000 { + compatible = "apple,t8140-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x028aa000 0x0 0x3000>; + reg-names = "voters", "rails"; + #address-cells = <1>; + #size-cells = <0>; + apple,static-voters = <0 1 3>; + + performance-controller@1 { + compatible = "apple,t8140-clpc"; + reg = <1>; + }; + + voter@2 { + reg = <2>; + label = "gfx"; + #power-domain-cells = <0>; + }; + }; + }; From 8141ca59f5a01c96cbf4919587d9964e7aa0ccff Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:39 -0400 Subject: [PATCH 274/631] pmdomain: apple: add PMC voter interface driver The PMC only honours a voter agent's perf-state floor on rails where that agent's interface is enabled. The static agents (PMP, CLPC and AP) need it enabled on all their rails once the PMGR is restored, and a dynamic agent such as the GPU needs it while the host keeps the agent's PMGR device powered: after the power-on state write and before the power-off one. Without these enables, the PMC ignores those agents' floors. Enable the static agents listed in the device tree for as long as the PMC is bound, and restore the saved enable words when it is unbound. Provide each dynamic agent as a power domain whose parent is its device's power domain, so that genpd enables the agent after the parent powers on and disables it before the parent powers off. Every enable write is read back. Export the floor words so that a host performance controller can set its agent's floor. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/pmdomain/apple/Kconfig | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 2fc24c49ddaf01..08921e63891fa9 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -20,6 +20,18 @@ config APPLE_PMP_REPORT depends on PM select PM_GENERIC_DOMAINS +config APPLE_PMC + bool "Apple PMC voter interface" + depends on OF + depends on PM + select PM_GENERIC_DOMAINS + help + The PMC in some Apple SoCs arbitrates the performance states of + shared rails, such as the DRAM controller and the SoC fabric, + between voter agents. This driver enables the static agents and + provides power domains that enable a dynamic agent, such as the + GPU, while its device is powered on. + endif config APPLE_PMP_REPORT_KUNIT_TEST From 79893c4ff4fe33041185c60eadc98ebf07cb5ea0 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:11:40 -0400 Subject: [PATCH 275/631] arm64: dts: apple: t8140: describe the PMC and report GFX to the PMP Describe the PMC voter interface with the PMP, CLPC and AP agents as static voters, the GPU's dynamic agent as a power domain, and the host performance controller's agent. Add the PMC's PMGR power state. Report the GPU to the PMP through a report entry for PMP soc-device 9, whose type in the Apple device tree requires an acknowledgment. The GPU now sits on the PMC GFX agent, which sits on the report entry, which sits on the GFX PMGR state. Powering the GPU on therefore writes the PMGR state, then reports it to the PMP, then enables its PMC agent, and powering it off undoes them in reverse order. Enable the PMC on J700. Assisted-by: LLM Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t8140.dtsi | 39 +++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index e793d7ed2ee418..80b5ee60ddf271 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -565,6 +565,12 @@ /* PMP soc-device 9, which the PMP acknowledges */ pmp_report_gfx: report@9 { + compatible = "apple,t8140-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x9>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; compatible = "apple,t8140-pmp-v2-report-entry"; reg = <0x9>; label = "pmp-gfx"; @@ -1213,7 +1219,7 @@ <0x4 0x80000000 0x0 0x4000000>; reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; - power-domains = <&ps_gfx>; + power-domains = <&pmc_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; @@ -1338,6 +1344,37 @@ status = "disabled"; }; + /* + * Perf-state floors of voter agents 0-3, and the rail state and + * interface enable words, in the PMC window at 0x302800000. + */ + pmc: power-management@302800000 { + compatible = "apple,t8140-pmc"; + reg = <0x3 0x02800000 0x0 0x10000>, + <0x3 0x028aa000 0x0 0x3000>; + reg-names = "voters", "rails"; + power-domains = <&ps_pmc>; + #address-cells = <1>; + #size-cells = <0>; + /* PMP, CLPC and AP agents */ + apple,static-voters = <0 1 3>; + status = "disabled"; + + clpc: performance-controller@1 { + compatible = "apple,t8140-clpc"; + reg = <1>; + apple,gpu-power-state = <&ps_gfx>; + }; + + /* Dynamic agent of the GFX PMGR device */ + pmc_gfx: voter@2 { + reg = <2>; + label = "gfx"; + #power-domain-cells = <0>; + power-domains = <&pmp_report_gfx>; + }; + }; + /* * Bandwidth scratch and request registers, 0x20000 and 0x68000 * into the display clock window at 0x302800000 (the sixth disp0 From cc37bf3028dd094c1a8d0bcb74960bcc21a6318a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:41:18 +1100 Subject: [PATCH 276/631] arm64: dts: apple: t8140: keep the GPU on its PMGR state With the previous commit the GPU's only power domain is the PMC's GFX voter domain, which only the new PMC driver provides. A kernel without that driver, or built without CONFIG_APPLE_PMC, never probes the GPU: platform 482600000.gpu: deferred probe pending: platform: wait for supplier /soc/power-management@302800000/voter@2 A device tree has to keep working with kernels that predate a driver it describes, so leave the GPU on its PMGR power state as before. The GFX voter does nothing measurable on the J700 anyway: its floor stayed 0 under every GPU load we sampled, and the GPU firmware has no mapping of the PMC window to write it through. The GPU throughput gain comes from the report entry alone. Once the PMP is told that the GPU is on, it raises its own DRAM controller (DCS) vote under GPU load, which the boot firmware already lets through on the DCS rail. With only the report bit set, a kernel without any PMC driver runs glmark2's build scene at 2199 instead of 1453; enabling every PMC agent on every rail on top of that changes it to 2218. Report the GPU as on from probe instead, as t8112 does for its display. The host never turns the GPU's PMGR state off at runtime, and hardware auto-PM already gates the GPU while it stays requested on, so the PMP sees the same thing it does with the report in the GPU's power path: a reported-on GPU that is gated whenever it idles. It votes the DCS up only under GPU load. Every kernel with the PMP report driver supports the entry. Drop the unused GFX voter node. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 80b5ee60ddf271..2857d8e7ed9aab 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1219,7 +1219,7 @@ <0x4 0x80000000 0x0 0x4000000>; reg-names = "asc", "sgx"; mboxes = <&gfx_mbox>, <&gfx1_mbox>; - power-domains = <&pmc_gfx>; + power-domains = <&ps_gfx>; memory-region = <&uat_ttbs>, <&uat_pagetables>, <&uat_handoff>; memory-region-names = "ttbs", "pagetables", "handoff"; @@ -1365,14 +1365,6 @@ reg = <1>; apple,gpu-power-state = <&ps_gfx>; }; - - /* Dynamic agent of the GFX PMGR device */ - pmc_gfx: voter@2 { - reg = <2>; - label = "gfx"; - #power-domain-cells = <0>; - power-domains = <&pmp_report_gfx>; - }; }; /* From 6fce437584fc81b5d572f239c2e1ced9301a770f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:48:27 +1100 Subject: [PATCH 277/631] dt-bindings: power: apple,pmc: describe the DCS limit The second register region holds one word per voter agent with the floor the PMC honours for it, followed by one interface enable word per rail; say so instead of calling the first words rail states. The PMC can also limit the DRAM controller (DCS) rail on behalf of the host, which makes it a thermal cooling device. Add #cooling-cells, and an optional power model for the power allocator governor: the system power at each DCS state above the lowest one. Bound the static voter list by the 32 agents an enable word can hold. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/power/apple,pmc.yaml | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 69cdc1cb87197a..91c77695178c49 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -29,7 +29,9 @@ properties: reg: items: - description: Perf-state floor words of the voter agents - - description: Rail state and interface enable words + - description: | + Per-agent words holding the floor the PMC honours for each agent, + and per-rail interface enable words reg-names: items: @@ -48,6 +50,26 @@ properties: apple,static-voters: $ref: /schemas/types.yaml#/definitions/uint32-array description: Voter agents to enable on every rail. + minItems: 1 + maxItems: 32 + items: + maximum: 31 + + '#cooling-cells': + const: 2 + description: | + The PMC can limit the DRAM controller (DCS) rail. Cooling state 0 leaves + the rail to the agents' votes; each further state lowers the highest + DCS state the PMC honours for them by one, down to the lowest state. + + apple,dcs-power-microwatt: + description: | + Power the system draws at each DCS performance state above what it + draws at the lowest one, from the lowest state up, measured under a + load whose speed follows the DCS state. With it, the DCS limit is a + power actor that the power allocator thermal governor can drive. + minItems: 2 + maxItems: 16 patternProperties: '^voter@[0-9a-f]+$': @@ -115,7 +137,9 @@ examples: reg-names = "voters", "rails"; #address-cells = <1>; #size-cells = <0>; + #cooling-cells = <2>; apple,static-voters = <0 1 3>; + apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; performance-controller@1 { compatible = "apple,t8140-clpc"; From 7cdd10deb4f8e0397c84ab3e5d9f84b238a16b35 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:44 +1100 Subject: [PATCH 278/631] nvme: apple: keep the post-M4 ANS link domains powered across sleep A post-M4 ANS cannot be started again once CPU_CONTROL.RUN is cleared, so apple_nvme_suspend() only disables the NVMe controller and leaves the coprocessor running. The running firmware keeps using its PCIe link, whose power domains on T8140 are apcie_sys_st and apcie_phy_sw. genpd powers a domain off in the noirq phase once every device attached to it has suspended, whatever the runtime PM state. The controller holds its domains through per-domain virtual devices, which suspend right after the controller, so both link domains went down underneath the running ANS. On the J700, a pm_test=platform cycle, which holds the devices suspended for five seconds, then hung the SoC during resume until the watchdog reset it. Put the devices through which the controller holds its domains on the awake path when the ANS is left running. The pmgr power-state driver marks these boot-active domains GENPD_FLAG_ACTIVE_WAKEUP, so genpd keeps them powered until resume, as it already does for the domains of the display coprocessor. With this change the same cycle resumes with the root filesystem intact, and genpd records no power-off of either link domain across it or across real suspend-to-idle cycles. Controllers before M4 shut the ANS down and are not affected. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index fe504d9bc702a8..094d11e88411e8 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2584,6 +2584,28 @@ static int apple_nvme_resume(struct device *dev) return nvme_reset_ctrl(&anv->ctrl); } +/* + * The ANS stays up across system sleep on post-M4 hardware, and it keeps + * using its PCIe link while it runs. genpd, however, switches a domain off + * in the noirq phase once every device attached to it has suspended, no + * matter what runtime PM says. Put the devices that hold the controller's + * domains on the awake path so that genpd leaves those domains powered + * until resume. + */ +static void apple_nvme_keep_link_powered(struct apple_nvme *anv) +{ + int i; + + /* A single domain is attached to the controller device itself. */ + if (anv->pd_count <= 1) { + device_set_awake_path(anv->dev); + return; + } + + for (i = 0; i < anv->pd_count; i++) + device_set_awake_path(anv->pd_dev[i]); +} + static int apple_nvme_suspend(struct device *dev) { struct apple_nvme *anv = dev_get_drvdata(dev); @@ -2603,6 +2625,7 @@ static int apple_nvme_suspend(struct device *dev) */ if (anv->hw->needs_ioq_registers) { cancel_work_sync(&anv->recovery_work); + apple_nvme_keep_link_powered(anv); return 0; } From b0da2d60573f1079098554d544e1d0ef49f65f97 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:48 +1100 Subject: [PATCH 279/631] wifi: mt7932: keep the function running across suspend-to-idle The driver has no dev_pm_ops, so the PCI core's default system-suspend callback clears bus mastering on the function while its firmware keeps running. On the J700 a pm_test=devices cycle then hangs the SoC until the watchdog resets it, every time; with the function removed, the same cycle resumes. Stopping the firmware for sleep would take the remove and probe sequence on every suspend: retiring its DMA, a function-level reset and a new firmware start. Leave it running across suspend-to-idle instead. Saving the PCI state in .suspend keeps the PCI core from changing the function's power state or command register, and putting the device on the awake path keeps the PCIe controller's power state on until resume. Refuse any other sleep state. With this change, the pm_test=devices cycle, a one-minute pm_test=platform cycle and real 30 and 90 second suspend-to-idle cycles all resume with the station still associated and passing traffic, and genpd records no power-off of the controller's power state. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index d24bb674cafc91..ab120796646a47 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -7,6 +7,9 @@ #include "mt7932.h" +#include +#include + static int mt_ring_alloc(struct mt7932 *m, struct mt7932_ring *q, unsigned int n, bool rx, unsigned int count); static void mt_rx_post(struct mt7932_ring *q, unsigned int index); @@ -1352,6 +1355,25 @@ static void mt_shutdown(struct pci_dev *pdev) /* Storage remains allocated through reset, even if the engine is stuck. */ } +/* + * Stopping the firmware for sleep would take the whole remove and probe + * sequence, so leave it running across suspend-to-idle. A saved state keeps + * the PCI core from changing the function's power state or command register, + * and the awake path keeps the PCIe controller powered until resume. + */ +static int mt_suspend(struct device *dev) +{ + if (!pm_suspend_no_platform()) + return -EOPNOTSUPP; + pci_save_state(to_pci_dev(dev)); + device_set_awake_path(dev); + return 0; +} + +static const struct dev_pm_ops mt_pm_ops = { + .suspend = mt_suspend, +}; + static const struct pci_device_id mt_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_MEDIATEK, 0x7932) }, {} @@ -1365,6 +1387,7 @@ static struct pci_driver mt_driver = { .remove = mt_remove, .shutdown = mt_shutdown, .driver.suppress_bind_attrs = true, + .driver.pm = pm_sleep_ptr(&mt_pm_ops), }; /** * mt7932_fullmac_ready() - check that the Wi-Fi function is up From ef245650a20d4e637fa1e913a05680307c89e4f3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:51 +1100 Subject: [PATCH 280/631] rtc: apple-pmu: take over an alarm that firmware left armed The probe refused to bind with -EBUSY whenever it found the alarm armed, so that it would not overwrite a wake scheduled by firmware. On the J700 the alarm is armed at every boot, with the comparator at about 2^32 - 1 day in wall-clock seconds, a time in early 2106. Nothing is due then, but the check means the driver never binds. Keep the alarm only when it is unmasked and due within the range the driver supports, so that the RTC core picks it up at registration. Otherwise disarm it, as probe already did for an alarm it found disarmed. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-apple-pmu.c | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index a6842332058032..8c7e4156b015f0 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -247,6 +247,32 @@ static irqreturn_t baku_rtc_interrupt(int irq, void *context) return IRQ_HANDLED; } +/* + * Firmware leaves the alarm armed even when it has no wake scheduled, with + * the comparator set close to the end of 32-bit time. Keep an alarm that is + * still pending within the supported range, so that the RTC core picks it + * up at registration, and disarm anything else. + */ +static int baku_rtc_init_alarm(struct baku_rtc *baku) +{ + unsigned int control, mask; + __le32 comparator; + int error; + + error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); + if (!error) + error = regmap_read(baku->map, BAKU_RTC_MASK, &mask); + if (!error) + error = regmap_bulk_read(baku->map, BAKU_RTC_COMPARE, + &comparator, sizeof(comparator)); + if (error) + return error; + if ((control & BAKU_RTC_ARM) && !(mask & BAKU_RTC_ALARM) && + !baku_rtc_validate_alarm(baku, le32_to_cpu(comparator))) + return 0; + return baku_rtc_stop_alarm(baku); +} + static const struct rtc_class_ops baku_rtc_ops = { .read_time = baku_rtc_read_time, .read_alarm = baku_rtc_read_alarm, @@ -284,7 +310,6 @@ static int baku_rtc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct baku_rtc *baku; - unsigned int control; int error; /* Retention has only been measured on J700 with the explicit opt-in. */ @@ -304,12 +329,6 @@ static int baku_rtc_probe(struct platform_device *pdev) return baku->interrupt; mutex_init(&baku->transaction); platform_set_drvdata(pdev, baku); - error = regmap_read(baku->map, BAKU_RTC_CONTROL, &control); - if (error) - return error; - /* Respect an alarm left armed by firmware. */ - if (control & BAKU_RTC_ARM) - return -EBUSY; baku->clock = devm_rtc_allocate_device(dev); if (IS_ERR(baku->clock)) return PTR_ERR(baku->clock); @@ -332,7 +351,7 @@ static int baku_rtc_probe(struct platform_device *pdev) dev_name(dev), dev); if (error) goto disable_wakeup; - error = baku_rtc_stop_alarm(baku); + error = baku_rtc_init_alarm(baku); if (error) goto disable_wakeup; error = devm_rtc_register_device(baku->clock); From 7460be638b4fe7df3b7a56de36be016a250a126c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 01:29:52 +1100 Subject: [PATCH 281/631] rtc: apple-pmu: enable the alarm without a module parameter The driver only probed with experimental=1, because alarm delivery during suspend-to-idle had not been measured. Nothing on that path is switched off by s2idle on the J700: the SPMI controller and the AIC sit in always-on power domains, the SPMI controller's chained interrupt is never disabled by the suspend core, and the alarm interrupt is requested with IRQF_NO_SUSPEND and reports a hard wakeup. Measured on the J700: alarms set 30 and 90 seconds ahead ended suspend-to-idle 29.8 to 30.5 and 90.3 seconds after it began. Drop the parameter so that the PMU RTC binds by default. It is the only wake source on the J700 that does not need a person at the machine. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/Kconfig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/rtc/Kconfig b/drivers/rtc/Kconfig index 5488a2dd5c86e9..e6077d109afb40 100644 --- a/drivers/rtc/Kconfig +++ b/drivers/rtc/Kconfig @@ -2104,10 +2104,10 @@ config RTC_DRV_APPLE_PMU tristate "Apple J700 Baku PMU RTC alarm" depends on ARCH_APPLE && NVMEM_APPLE_SPMI && PM_SLEEP help - Read the Baku PMU clock and use alarms with retained-power - suspend-to-idle. The module requires experimental=1. Alarms are - limited to one hour; deeper suspend states are not supported. - Continue setting the shared clock through the SMC RTC. + Read the Baku PMU clock and use its alarm to wake the system + from suspend-to-idle. Alarms are limited to one hour; deeper + suspend states are not supported. Continue setting the shared + clock through the SMC RTC. This driver can be built as the module rtc-apple-pmu. From 855b60b75684912957ac13d89145d0a0f9db88ec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:07:58 +1100 Subject: [PATCH 282/631] drm: apple: check only the lengths of a legacy reply record Commit 2b8d51be4f0a ("drm: apple: locate legacy command replies in the saved record") validates a reply on firmware older than H17P by comparing the whole record header, tag included, with the one that was sent. The lengths are what the check before it compared; requiring the firmware to leave the tag alone as well is a new assumption, and it was only confirmed by traces of 13.5 firmware on t8112 and t6000. The same path serves the 12.3 and 13.3 firmware interfaces. If either rewrote the tag when writing its reply, every ack would fail the check, the DCP would be marked crashed during start-up, and the display would stay dark. Compare only the in and out lengths, which is enough to tell that the record was overwritten. Fixes: 2b8d51be4f0a ("drm: apple: locate legacy command replies in the saved record") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b04bac0e9d864b..a422ac7d2ce63a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -933,13 +933,15 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, * acked on the callback context, whose receive area holds the callback * record rather than the reply, so the ack cannot be used to find it. * - * Older firmware leaves the record header untouched. If it no longer - * matches what was sent, the record was overwritten and its output - * cannot be trusted. + * On older firmware the lengths in the record header must still be + * the ones that were sent; otherwise the record was overwritten and + * its output cannot be trusted. The tag is left out of the check: it + * has only been seen preserved on 13.5 firmware. */ if (dcp->fw_compat != DCP_FIRMWARE_H17P) { record = out - sent->in_len - sizeof(*record); - if (memcmp(record, sent, sizeof(*record))) { + if (record->in_len != sent->in_len || + record->out_len != sent->out_len) { dev_err(dcp->dev, "corrupt %c%c%c%c command record on context %X\n", sent->tag[3], sent->tag[2], sent->tag[1], From c85ec415680a9549691ca5497120f8cc998e58b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:08:33 +1100 Subject: [PATCH 283/631] drm: apple: complete a legacy command whose reply record is corrupt When the reply record of a command on firmware older than H17P fails validation, dcpep_handle_ack() marks the DCP crashed and returns without calling the command's callback. The callback owns a reference to its cookie: for power and modeset commands that is a wait cookie whose caller then waits out its full timeout (1 to 10 seconds), and the reference is never dropped, so each such command leaks its cookie. The start-up wait in dcp_wait_ready() likewise runs to its timeout when the is_main_display reply is the one rejected. Call the callback with no output instead. The callbacks used on this firmware either ignore their output or already treat a missing one as a failed command, except two that dereferenced it unconditionally: make dcp_swapped() report a failed swap, as it already does on H17P, and make res_is_main_display() end the start-up wait, which then returns -ENODEV for the crashed DCP. Fixes: c529eadaf40a ("drm: apple: retain displaced framebuffers until confirmed completion") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 7 ++++++- drivers/gpu/drm/apple/iomfb_template.c | 4 ---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a422ac7d2ce63a..11eed4b5fb1cb0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -947,7 +947,12 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, sent->tag[3], sent->tag[2], sent->tag[1], sent->tag[0], context); WRITE_ONCE(dcp->crashed, true); - return; + /* + * Still complete the command, without an output, so + * that its callback drops the cookie reference and + * wakes the waiter instead of leaving it to time out. + */ + out = NULL; } } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 74cde4c5e2d672..01d1efc1528535 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2021,11 +2021,7 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER u32 status = resp ? resp->ret : ~0U; -#else - u32 status = resp->ret; -#endif if (resp->ret) { dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); From 5b4debf4a05d06fe0166ab366133188470b2bf25 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:14:46 +1100 Subject: [PATCH 284/631] drm: apple: present a backlight write while the H17P level is unknown Without a loader brightness report the H17P backlight registers with the middle of the panel range as its reported level, while the panel keeps the loader's level until a present carries one. dcp_bl_request() only marks a new request dirty when it differs from the target, so a write of exactly the reported level, as systemd-backlight makes when it restores a level saved on an earlier boot without a report, presents nothing. The panel then stays at the loader's level, the reported level stays wrong, and the first DPMS or suspend cycle jumps to it. Also mark a request dirty while the level is not known, so the first write is always presented and makes the level known. Add a KUnit case. Fixes: 34cd8f3032b3 ("drm: apple: register the H17P backlight without a loader level") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp_backlight.h | 6 +++- drivers/gpu/drm/apple/dcp_backlight_test.c | 36 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 2c9a0a7ab97be3..1232ff40a84ba2 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -110,8 +110,12 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, return -ERANGE; state->controlled = true; state->retries = 0; + /* + * Until a present carries a level, the panel does not show the target, + * so even a request for the reported default has to be presented. + */ if (state->target != nits || state->core_blank != core_blank || - state->suspended != suspended) { + state->suspended != suspended || !state->level_known) { state->target = nits; state->core_blank = core_blank; state->suspended = suspended; diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index a0aaad3ddd232d..41ac118f26cee9 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -292,6 +292,41 @@ static void backlight_fallback_test(struct kunit *test) KUNIT_EXPECT_EQ(test, present.nits, 262U); } +static void backlight_fallback_same_level_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* A userspace write of the reported default is still presented. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 262U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_TRUE(test, state.level_known); + + /* Once the level is known, repeating it presents nothing. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EALREADY); + + /* A rejected present leaves it unknown, so the same write is retried. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, false)); + KUNIT_EXPECT_FALSE(test, state.level_known); + state.dirty = false; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 262, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.dirty); + + /* An inherited level is known from the start. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, true, 180, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 180, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.dirty); +} + static void backlight_seed_known_test(struct kunit *test) { struct dcp_backlight_state state; @@ -329,6 +364,7 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_inflight_dpms_test), KUNIT_CASE(backlight_resend_test), KUNIT_CASE(backlight_fallback_test), + KUNIT_CASE(backlight_fallback_same_level_test), KUNIT_CASE(backlight_seed_known_test), {} }; From d65b96f0e5628cea05fb06387b4e16788f499835 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:18:29 +1100 Subject: [PATCH 285/631] drm: apple: do not report an aborted H17P present as a failed swap When H17P firmware aborts a present before its submit reply arrives, dcp_present_submit() logs the abort, rate-limited, and the reply handler then turns the accepted reply into a failure and logs it again, without a rate limit, as "swap failed! status 4294967295". A firmware that aborted every frame would log that error once per frame, with a status the firmware never returned. Let dcp_present_submit() return why a present does not stand, and skip the failure message for an abort that has already been reported. The present is still finished as a failure. Fixes: 239607cb12c3 ("drm: apple: finish H17P presents that the firmware aborts") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_template.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 01d1efc1528535..2d6a10243e5d93 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1459,7 +1459,12 @@ static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) return true; } -static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) +/* + * Returns 0 if the submitted present stands, -ECANCELED if the firmware + * aborted it before replying (already reported), or -EPROTO if no present + * was expecting this reply. + */ +static int dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && @@ -1476,7 +1481,7 @@ static bool dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted return false; } #endif - return true; + return 0; } static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) @@ -2019,6 +2024,7 @@ TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_se static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { + int ret; struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; u32 status = resp ? resp->ret : ~0U; From 82ed9f1a898d9ea254ccd0abf5a914a9af93183e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:18:52 +1100 Subject: [PATCH 286/631] drm: apple: check for a dead DCP after each H17P sleep drain pass Commit 744c2c7abc15 ("drm: apple: skip the H17P sleep drain on a crashed or stopped DCP") checks for a crashed or stopped DCP at the start of each drain pass. A crash during the last pass is not seen: the queue drain then reports the DCP idle at once, the level is still pending, and the loop ends with the "brightness change still pending at suspend" warning that commit set out to avoid. Returning before the first pass also skips the queue drain, whose lock orders the suspend after a queued transaction that is still being started. Check after the queue drain instead, before the pending test. A crashed or stopped DCP now always goes through the drain, which returns at once for it. Fixes: 744c2c7abc15 ("drm: apple: skip the H17P sleep drain on a crashed or stopped DCP") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index bf64aeddcf6b65..a9ab04913df3a9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5092,12 +5092,6 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * the first attempt and the policy's three retries. */ for (pass = 0; pass < 4; pass++) { - /* - * A crashed or stopped DCP takes no more presents, so a level - * still pending there can never be sent and is not waited for. - */ - if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) - return; flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) flush_work(&dcp->bl_update_wq); @@ -5105,6 +5099,14 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dev_warn(dcp->dev, "display updates still pending at suspend\n"); return; } + /* + * A crashed or stopped DCP takes no more presents, so a level + * still pending there can never be sent and is not waited for. + * Checking after the drain, which returns at once for such a + * DCP, also covers a crash during the pass. + */ + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return; if (!dcp_backlight_active(dcp) || !dcp_backlight_pending(dcp)) return; } From 03e3fcf9a9729c22bb070b55074db9b62ba59292 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:23:20 +1100 Subject: [PATCH 287/631] wifi: mt7932: report whether the Wi-Fi function initialized When initialization fails and the DMA of function 0 cannot be stopped, mt7932-fullmac keeps the binding to retain that DMA. Whether the Wi-Fi firmware is running therefore cannot be told from the binding, which is what the Bluetooth transport on function 1 waits for before it starts its own firmware on the shared radio. Record when the probe has finished initialization and export mt7932_fullmac_ready() to report it. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 ++ 2 files changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 0b792ed332e4e2..e37ddc37885aed 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1983,6 +1983,7 @@ F: Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml F: Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml F: drivers/pmdomain/apple/pmp-report.c F: drivers/soc/apple/pmp.rs +F: include/linux/mt7932.h F: include/linux/soc/apple/pmp-report.h F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml F: Documentation/devicetree/bindings/display/apple,dcp.yaml diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 162f52075dace1..5a3cb31c19bd2b 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -81,6 +81,8 @@ struct mt7932 { u64 interrupts, packets; bool running; bool stopping; + /* Probe finished initialization; the Bluetooth function may start. */ + bool ready; bool irq_requested, vectors_allocated, wiphy_registered; bool dma_owned, module_pinned; struct completion reset_retry; From fa40813efb960f30ea10274eaffb7b5dcee820c3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:24:23 +1100 Subject: [PATCH 288/631] Bluetooth: mt7932: start only after the Wi-Fi function initialized The Bluetooth transport waits until function 0 is bound to mt7932-fullmac, so that its firmware starts after the Wi-Fi firmware as in the tested sequence. Being bound does not mean the Wi-Fi firmware is running: when the Wi-Fi initialization fails and its DMA cannot be stopped, mt7932-fullmac keeps the binding to retain that DMA. Bluetooth then probes anyway and starts its firmware on the shared radio next to a failed Wi-Fi function. Once function 0 is bound, also require mt7932_fullmac_ready(). The Wi-Fi probe finishes initialization before it returns, so a bound function that is not ready has failed, and the Bluetooth probe fails with -ENODEV. The transport now calls into the Wi-Fi module, so it depends on MT7932_FULLMAC plainly: it can no longer be built in while the Wi-Fi driver is a module. Fixes: 920e89a2e3df ("Bluetooth: mt7932: wait for the Wi-Fi function before starting") Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 1ac94a339ece2b..582c43dc5578df 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -53,7 +53,8 @@ performance policy. ``mt7932_bt_pcie`` defers its probe until ``mt7932-fullmac`` is bound to function 0, so the Bluetooth firmware always starts after the Wi-Fi firmware, as in the tested sequence, however the drivers are loaded. Bluetooth therefore -stays unbound while the Wi-Fi driver is missing or has failed. A device link +stays unbound while the Wi-Fi driver is missing or has failed, including when +a failed Wi-Fi function stays bound to keep DMA it could not stop. A device link makes the driver core unbind Bluetooth before Wi-Fi; unloading ``mt7932_bt_pcie`` does not affect Wi-Fi. The probe rejects a function that is not cold, which is what now prevents a reprobe after a failed or uncertain From e72097ebd73a6e80e74cdb4279d231ffaed55ac4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 9 Oct 2026 19:27:33 +1100 Subject: [PATCH 289/631] wifi: mt7932: keep the country package's channel disables The channels a country package leaves out of the firmware domain are disabled only in cfg80211's channel flags, once per package pass. cfg80211 rebuilds those flags from the original ones on every regulatory update it applies, and then calls the notifier. The notifier skips an update whose snapshot matches the request already applied, so if an update ever leaves the snapshot unchanged, no pass runs and the disables are gone: the driver again offers channels its firmware domain excludes, where scans and joins fail. Do not depend on every such update producing a new snapshot. Record the channels each package pass disables and apply them again when the notifier finds the package still current. A new request clears the set; its own pass records a new one. Fixes: 6e3838e91a3b ("wifi: mt7932: honour the channel permissions of the country package") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 +-- .../net/wireless/mediatek/mt7932/startup.c | 25 +++++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 5a3cb31c19bd2b..9e7ae595cbd1c3 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -113,8 +113,8 @@ struct mt7932 { bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; - /* Applied package exclusions; protected by RTNL. */ - DECLARE_BITMAP(policy_disabled, 17); + /* Channels the applied country package leaves out; under RTNL. */ + DECLARE_BITMAP(policy_disabled, MT7932_CHANNELS); struct net_device *netdev; struct wireless_dev wdev; struct cfg80211_scan_request *scan_request; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 8275282c885b42..6f4efe3d247530 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -236,10 +236,15 @@ static int mt_startup_once(struct mt7932 *m) static struct ieee80211_channel *mt_channel(struct mt7932 *m, unsigned int i) { - return i < 13 ? &m->channels[i] : - &m->channels5[i - 13]; + return i < MT7932_CHANNELS_2G ? &m->channels[i] : + &m->channels5[i - MT7932_CHANNELS_2G]; } +/* Keep cfg80211's channel list in step with the channels the country + * package leaves out of the firmware domain. cfg80211 rebuilds the flags + * on every regulatory update; the notifier applies the recorded set again + * when the update keeps this package, or starts a new pass otherwise. + */ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy, u32 generation) { @@ -297,11 +302,11 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli rtnl_lock(); wiphy_lock(m->wiphy); for (i = 0; i < MT7932_CHANNELS; i++) { - struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? - &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; + struct ieee80211_channel *channel = mt_channel(m, i); + bool forbidden = !mt7932_policy_permits(policy, channel->hw_value); - if (!(channel->flags & IEEE80211_CHAN_DISABLED) && - !mt7932_policy_permits(policy, channel->hw_value)) + __assign_bit(i, m->policy_disabled, forbidden); + if (forbidden) channel->flags |= IEEE80211_CHAN_DISABLED; } wiphy_unlock(m->wiphy); @@ -485,13 +490,17 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request } if (!memcmp(®, &m->reg_desired, sizeof(reg)) && !m->reg_pending) { spin_unlock_irqrestore(&m->response_lock, irqflags); - for_each_set_bit(i, m->policy_disabled, 17) + /* The applied package still holds, but cfg80211 has just + * rebuilt the channel flags without its disables. + */ + for_each_set_bit(i, m->policy_disabled, MT7932_CHANNELS) mt_channel(m, i)->flags |= IEEE80211_CHAN_DISABLED; return; } m->reg_desired = reg; m->reg_generation++; - bitmap_zero(m->policy_disabled, 17); + /* The next package pass records its own set. */ + bitmap_zero(m->policy_disabled, MT7932_CHANNELS); WRITE_ONCE(m->reg_pending, true); if (m->netdev) { netif_stop_queue(m->netdev); From ba4e3fdecff5807f3d7f45157d18c1da37b9a196 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 03:44:23 -0400 Subject: [PATCH 290/631] drm/asahi: pack G17 tiler heap blocks contiguously in their backing Each tiler heap (TVB) block is 128 KiB and is mapped at a 160 KiB VA stride, so that an unmapped 32 KiB guard follows it. The backing object used the same stride: block i was mapped from offset i * 160 KiB, and an object of n blocks was (n - 1) * 160 KiB + 128 KiB long. Every block but the last therefore carried 32 KiB of allocated, zeroed and pinned backing that no GPU mapping covers, 20 % of all tiler heap memory. Map block i from offset i * 128 KiB and size the backing to n * 128 KiB. The VA placement, the guards and the addresses handed to the firmware are unchanged. At the in-world peak of a measured Subnautica session the render pairs held 2,057 blocks in 49 objects, so this returns 62.7 MiB. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 5f6490414a683eef149ba6de38cbc27ba62f9a26) --- drivers/gpu/drm/asahi/g17/buffer.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 90ab8ae21aa14f..e3387228093f73 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -340,21 +340,17 @@ pub(crate) struct Tvb { extensions: KVec, } impl Tvb { + /// Blocks are packed in the backing; only their VA placement keeps the + /// stride and its trailing guard. fn bytes(blocks: usize) -> Result { - blocks - .checked_sub(1) - .and_then(|v| v.checked_mul(cfg::TVB_BLOCK_STRIDE as usize)) - .and_then(|v| v.checked_add(cfg::TVB_BLOCK_SIZE)) - .ok_or(EOVERFLOW) + blocks.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW) } fn map_block( object: &KernelObject, vm: &mmu::Vm, relative: usize, ) -> Result { - let start = relative - .checked_mul(cfg::TVB_BLOCK_STRIDE as usize) - .ok_or(EOVERFLOW)?; + let start = relative.checked_mul(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; let mapping = compact_alias( object, From f08c09bf3c9c24e1e1a5e39af75be9bc8d0ea90f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 03:51:06 -0400 Subject: [PATCH 291/631] drm/asahi: allocate only the G17 boot pool blocks that are used Boot allocates three groups of 1 MiB pool blocks that no GPU mapping ever covers and nothing writes: - render blocks 23..27: the render pool maps blocks 0..21 and keeps the compute page list in block 22, but allocated 28 blocks; - the 23rd block of the device-wide compute backing: compute VMs map 22 of the 23 blocks; - the backing of the two zeroed compute blocks: their VAs fall inside the compute pool's page-list mapping, and the objects were never aliased anywhere. Each block is a zeroed, pinned kernel object for the life of the device. Allocate only the mapped render blocks plus the page-list block, only the mapped compute blocks, and no zero-block backing. Every firmware-visible address and table is unchanged; this returns 8 MiB. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit d2a2c2fa97b994595209843720ad78b99b93f16a) --- drivers/gpu/drm/asahi/g17/initdata.rs | 8 ++------ drivers/gpu/drm/asahi/hw/t8140.rs | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 5993be786874e7..46a38c5ad16ec3 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -23,7 +23,6 @@ const CONTEXT_SIZE: usize = 0x20000; const REGION_A_SIZE: usize = core::mem::size_of::(); const REGION_C_SIZE: usize = 0x1000; -const COMPUTE_BLOCKS_ALLOCATED: usize = 23; const COMPUTE_BLOCKS_MAPPED: usize = 22; const COMPUTE_BLOCKS_POPULATED: usize = 21; const COMPUTE_CONTROL_BLOCKS: u32 = 25; @@ -65,8 +64,8 @@ impl ComputeShared { KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; let runs = KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; - let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_ALLOCATED, GFP_KERNEL)?; - for _ in 0..COMPUTE_BLOCKS_ALLOCATED { + let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_MAPPED, GFP_KERNEL)?; + for _ in 0..COMPUTE_BLOCKS_MAPPED { blocks.push( KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, GFP_KERNEL, @@ -867,9 +866,6 @@ impl InitData { let compute_state = self.add(KernelObject::backing(env.dev, mmu::UAT_PGSZ, WB)?)?; self.object_mut(compute_state)? .write(0, abi::FreeListState::COMPUTE)?; - for _ in &low::COMPUTE_ZERO_BLOCKS { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; - } let compute_runs = self.add(KernelObject::new( env.dev, env.lower, diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 848f753a6d03d3..1c3e46cd60ded6 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -196,8 +196,9 @@ pub(crate) mod low { pub(crate) const RENDER_BLOCKS: u64 = 0x70_0022_0000; /// Distance between consecutive blocks. pub(crate) const BLOCK_STRIDE: u64 = 0x10_8000; - /// Render blocks allocated at boot. - pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = 28; + /// Render blocks allocated at boot: the mapped blocks and the block that + /// holds the compute page list. + pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = COMPUTE_PAGE_LIST_BLOCK + 1; /// Render blocks mapped at boot. pub(crate) const RENDER_BLOCKS_MAPPED: usize = 22; /// Render blocks in the free list at boot. From 2244c0c9b5c4c0f80a2c3ea0bf2bcfc210a6e062 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:49:41 -0400 Subject: [PATCH 292/631] drm/asahi: give back idle G17 tiler heap growth beyond recent use A render pair's tiler heap (TVB) grows on demand, by firmware request or from the sampled page high-water, and never shrinks. A pair that grew for one heavy scene keeps that growth for its whole life, also after the client returns to a light scene or goes idle; in a measured Subnautica session the game's pairs held 260 MiB of growth until it exited. A shrink policy for such heaps folds the page high-water into an average once per check, average = max((sample + 4 * average) / 5, sample). For each check in which the average stays below a quarter of the capacity it removes one 512 KiB block, and a buffer that has been idle for ten seconds returns to its minimum. Apply the same policy from the event worker, with a minimum that keeps what recent work used, so a shrink never makes the next pass at the recent load grow again: - once per second, sample the page high-water (sharing the sample with the growth check) and update the average; a period with a sample or a prepared pass also enters a window of the last ten such periods; - each second the average is below a quarter of the capacity earns 512 KiB of shrink; - a pair with no pass prepared or in flight for ten seconds may shrink to its floor; - the floor is the largest of the primary backing, the blocks of the last pass layout, and the size below which the automatic growth check would grow again for the largest sample in the window. No shrink goes below it. A shrink removes whole trailing growth extensions only, and only while the pair is idle and every block is back in the inventory ring. The kept blocks are laid out as a fresh heap is (ring entries and page list from the read cursor, state counts, both inventory cursors), and the PB descriptor row is reloaded, as an idle-host growth does. The removed blocks' client aliases leave the VM cache under the device mutex; the worker unmaps and frees them after it. A shrink is best effort and leaves the heap unchanged if anything fails. While some heap can still shrink, the worker wakes at least once per second. If the inventory of an idle heap does not hold every block, the heap is left as it is and the first such heaps are reported once each with their inventory cursors. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 272955b009fe844d4d06361365f166d75b6b451b) --- drivers/gpu/drm/asahi/g17/buffer.rs | 259 +++++++++++++++++++- drivers/gpu/drm/asahi/g17/queue/render.rs | 1 + drivers/gpu/drm/asahi/g17/runtime.rs | 24 +- drivers/gpu/drm/asahi/g17/runtime/events.rs | 14 ++ drivers/gpu/drm/asahi/g17/runtime/render.rs | 74 ++++++ drivers/gpu/drm/asahi/mmu/lifetime.rs | 34 +++ 6 files changed, 401 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index e3387228093f73..652f49a99258b2 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -333,6 +333,13 @@ pub(crate) struct TvbGrowth { /// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their /// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. +/// Aliases and backing of growth extensions a shrink removed from an idle +/// heap. Field order unmaps the aliases before freeing their backing. +pub(crate) struct TvbRelease { + _mappings: KVec, + _extensions: KVec, +} + pub(crate) struct Tvb { addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. @@ -394,6 +401,18 @@ impl Tvb { pub(crate) fn addresses(&self) -> &[u64] { &self.addresses } + /// Blocks of each growth extension; extensions are packed like the primary. + fn extension_blocks(extension: &KernelObject) -> usize { + extension.size() / cfg::TVB_BLOCK_SIZE + } + /// Blocks of the primary backing, which a heap never gives back. + fn primary_blocks(&self) -> usize { + self.extensions + .iter() + .fold(self.blocks(), |blocks, extension| { + blocks.saturating_sub(Self::extension_blocks(extension)) + }) + } pub(crate) fn prepare_growth( &mut self, @@ -536,8 +555,50 @@ pub(crate) struct Manager { scratch: u64, buffer_id: u8, generation: u64, + shrink: Shrink, +} + +/// Tiler-heap shrink policy for growth beyond what recent work used. +/// Once per +/// period the page high-water folds into an average, +/// `max((sample + 4 * average) / 5, sample)`. Each period whose average +/// stays below a quarter of the capacity earns one 512 KiB block of shrink, +/// and a heap idle for ten seconds returns to its minimum. Only an idle heap +/// is rebuilt here, by whole growth extensions. +/// +/// The minimum keeps what the recent work needs: never below +/// the size at which the automatic growth check would grow the heap again +/// for the largest high-water sample of the last ten active periods. A +/// shrink then never makes the next pass at the recent load grow back. +const SHRINK_PERIOD_NS: u64 = 1_000_000_000; +const SHRINK_IDLE_NS: u64 = 10_000_000_000; +/// Blocks in one 512 KiB parameter-buffer block. +const SHRINK_STEP_BLOCKS: usize = 0x8_0000 / cfg::TVB_BLOCK_SIZE; +/// Active periods in the recent-load window. +const SHRINK_WINDOW: usize = (SHRINK_IDLE_NS / SHRINK_PERIOD_NS) as usize; + +#[derive(Default)] +struct Shrink { + last_busy: u64, + period_start: u64, + /// High-water samples since the last period and the last growth check. + period_max: u32, + growth_max: u32, + average: u32, + credit: usize, + /// Blocks the most recent pass layout required. + required: usize, + /// High-water samples of the last active periods (with a sample or a + /// pass) and their largest value. Idle periods leave both unchanged. + window: [u32; SHRINK_WINDOW], + window_next: usize, + busy_peak: u32, + reported: bool, } +/// Bounds the one-line report of heaps that could not be rebuilt. +static SHRINK_REPORTS: AtomicU32 = AtomicU32::new(0); + pub(crate) struct Submitted<'a> { word: &'a AtomicU32, next: u32, @@ -599,6 +660,7 @@ impl Manager { scratch: args.scratch, buffer_id: args.buffer_id, generation: args.generation, + shrink: Shrink::default(), }) } pub(crate) fn buffer_id(&self) -> u8 { @@ -658,7 +720,9 @@ impl Manager { next: word.load(Ordering::Relaxed).wrapping_add(1), }) } - pub(crate) fn automatic_target(&self) -> Result> { + /// Reads the firmware page high-water and requests its reset. Both the + /// growth check and the shrink period see every sample. + fn sample_max_pages(&mut self) -> Result { let max_pages = self .graph .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; @@ -669,8 +733,83 @@ impl Manager { let pages = max_pages.load(Ordering::Relaxed); fence(Ordering::Release); reset.store(1, Ordering::Relaxed); + self.shrink.period_max = self.shrink.period_max.max(pages); + self.shrink.growth_max = self.shrink.growth_max.max(pages); + Ok(pages) + } + pub(crate) fn automatic_target(&mut self) -> Result> { + self.sample_max_pages()?; + let pages = core::mem::take(&mut self.shrink.growth_max); Ok(automatic_growth_target(pages as usize, self.blocks())) } + /// Records the blocks a pass layout requires, the floor of any shrink. + /// A pass is being prepared at `now`; the heap counts as busy. + pub(crate) fn note_required(&mut self, blocks: usize, now: u64) { + self.shrink.required = blocks; + self.shrink.last_busy = now; + } + /// The smallest heap a shrink may leave: the primary backing, the blocks + /// of the last pass layout, and the size below which the automatic growth + /// check would grow again for the recent peak. + fn shrink_floor(&self) -> usize { + let recent = (self.shrink.busy_peak as usize) + .saturating_mul(2) + .div_ceil(cfg::TVB_PAGES_PER_BLOCK); + self.tvb.primary_blocks().max(self.shrink.required).max(recent) + } + /// Whether the heap holds growth beyond its primary backing. + pub(crate) fn grown(&self) -> bool { + !self.tvb.extensions.is_empty() + } + /// Whether the heap holds growth that a shrink could give back. + pub(crate) fn shrinkable(&self) -> bool { + self.tvb.extensions.last().is_some_and(|extension| { + self.blocks() - Tvb::extension_blocks(extension) >= self.shrink_floor() + }) + } + /// Advances the shrink policy at `now` and returns the block count an idle + /// heap should shrink to. `busy` is true while the pair has work in flight + /// or is borrowed; a busy heap only updates its average and recent peak. + /// Call it for every grown heap, so the recent peak can also fall. + pub(crate) fn shrink_target(&mut self, now: u64, busy: bool) -> Result> { + if busy || self.shrink.last_busy == 0 { + self.shrink.last_busy = now; + } + if self.shrink.period_start == 0 { + self.shrink.period_start = now; + } + if now.saturating_sub(self.shrink.period_start) >= SHRINK_PERIOD_NS { + self.sample_max_pages()?; + let sample = core::mem::take(&mut self.shrink.period_max); + let average = (u64::from(sample) + 4 * u64::from(self.shrink.average)) / 5; + self.shrink.average = (average as u32).max(sample); + if sample != 0 || self.shrink.last_busy >= self.shrink.period_start { + let next = self.shrink.window_next; + self.shrink.window[next] = sample; + self.shrink.window_next = (next + 1) % SHRINK_WINDOW; + self.shrink.busy_peak = self.shrink.window.iter().copied().max().unwrap_or(0); + } + self.shrink.period_start = now; + let capacity = self.blocks() * cfg::TVB_PAGES_PER_BLOCK; + if (self.shrink.average as usize).saturating_mul(4) < capacity { + self.shrink.credit = self.shrink.credit.saturating_add(SHRINK_STEP_BLOCKS); + } else { + self.shrink.credit = 0; + } + } + if !self.shrinkable() { + self.shrink.credit = 0; + return Ok(None); + } + if busy { + return Ok(None); + } + if now.saturating_sub(self.shrink.last_busy) >= SHRINK_IDLE_NS { + return Ok(Some(self.shrink_floor())); + } + let last = self.tvb.extensions.last().map_or(0, Tvb::extension_blocks); + Ok((last != 0 && self.shrink.credit >= last).then(|| self.blocks() - last)) + } fn cursors(&self) -> Result { Ok(Cursors { total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), @@ -872,6 +1011,124 @@ impl Manager { } } +impl Manager { + /// Rebuilds an idle heap with whole trailing growth extensions removed, + /// keeping at least `target` blocks. The kept blocks are laid out as + /// `Graph::init` lays out a fresh heap, starting at the firmware read + /// cursor. This runs only while every block is back in the inventory, as + /// an idle-host growth requires the complete snapshot unchanged. + /// + /// `truncate(blocks, kept, out)` moves the removed blocks' client aliases + /// out of the VM cache. Returns the reloaded PB row and the released + /// aliases and backing, which the caller frees after the device mutex. + /// None leaves the heap, the firmware records and the VM unchanged. + pub(crate) fn shrink( + &mut self, + target: usize, + descriptor: [u32; 4], + truncate: impl FnOnce(usize, usize, &mut KVec) -> Result, + ) -> Result> { + let blocks = self.blocks(); + let mut kept = blocks; + let mut dropped = 0; + for extension in self.tvb.extensions.iter().rev() { + let size = Tvb::extension_blocks(extension); + if kept - size < target.max(self.tvb.primary_blocks()) { + break; + } + kept -= size; + dropped += 1; + } + if dropped == 0 { + return Ok(None); + } + let before = self.cursors()?; + let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) + % cfg::TVB_BLOCK_SLOTS; + if !before.valid() || before.total != before.committed || retained != blocks { + if !self.shrink.reported && SHRINK_REPORTS.fetch_add(1, Ordering::Relaxed) < 8 { + self.shrink.reported = true; + pr_info!( + "G17: tiler heap {} kept {} blocks: inventory {}/{}/{} holds {}\n", + self.buffer_id, + blocks, + before.total, + before.committed, + before.read, + retained + ); + } + return Ok(None); + } + // Allocation failure only postpones the shrink to a later pass. + let (Ok(mappings), Ok(extensions)) = ( + KVec::with_capacity(blocks - kept, GFP_NOWAIT), + KVec::with_capacity(dropped, GFP_NOWAIT), + ) else { + return Ok(None); + }; + let mut release = TvbRelease { + _mappings: mappings, + _extensions: extensions, + }; + for &address in &self.tvb.addresses[..kept] { + fw::compact_page(address)? + .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) + .ok_or(EOVERFLOW)?; + } + let base = self.graph.pointer(0, fw::SIZE)?.cast::(); + self.graph.word(0)?; + // The callback moves aliases only after checking the cached owner, + // its block count and the spare capacity. + truncate(blocks, kept, &mut release._mappings)?; + + // From here every index, address and CPU view has been checked. + let write = |offset: usize, value: u32| { + // SAFETY: The full graph and base alignment were checked above. + // Every offset is an aligned field or a checked array index of + // Graph. No mutable reference aliases firmware-owned fields. + unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; + }; + for (index, &address) in self.tvb.addresses[..kept].iter().enumerate() { + // The same pure conversion succeeded for every kept block above. + let Ok(first) = fw::compact_page(address) else { + continue; + }; + let slot = (before.read as usize + index) % cfg::TVB_BLOCK_SLOTS; + let at = fw::BLOCK_TABLE + slot * size_of::(); + write(at, first); + write(at + 4, 0); + for page in 0..cfg::TVB_PAGES_PER_BLOCK { + let entry = index * cfg::TVB_PAGES_PER_BLOCK + page; + write(fw::PAGE_LIST + entry * 4, first + page as u32); + } + } + let total = ((before.read as usize + kept) % cfg::TVB_BLOCK_SLOTS) as u32; + let pages = (kept * cfg::TVB_PAGES_PER_BLOCK) as u32; + write(core::mem::offset_of!(fw::State, pages), pages); + write(core::mem::offset_of!(fw::State, producer), total); + write(core::mem::offset_of!(fw::State, consumer), before.read); + write(core::mem::offset_of!(fw::State, last_page), pages - 1); + write(fw::CONTROL, total); + fence(Ordering::SeqCst); + write(fw::CONTROL + 4, total); + fence(Ordering::SeqCst); + + self.tvb.addresses.truncate(kept); + for _ in 0..dropped { + if let Some(extension) = self.tvb.extensions.pop() { + append_reserved(&mut release._extensions, extension); + } + } + self.shrink.credit = self.shrink.credit.saturating_sub(blocks - kept); + self.shrink.reported = false; + Ok(Some(( + fw::reload_descriptor(descriptor, self.page_list_client, pages), + release, + ))) + } +} + /// Caller reserves all needed entries before any firmware-visible mutation. fn append_reserved(values: &mut KVec, value: T) { let length = values.len(); diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index d82ee84c1283ff..0bf477c0344fb6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -909,6 +909,7 @@ impl Pair { return Err(EINVAL); }; let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; + self.memory.manager.note_required(required, crate::g17::now_ns()); if self.in_flight() { return Ok(None); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 4c7aa6a442cc10..e4f19397e5f923 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -656,6 +656,8 @@ pub(super) struct DeferredBatch { vacated: KVVec, /// Backing of render pools vacated for evicted idle clients. render: KVVec, + /// Tiler-heap extensions removed from idle heaps, with their aliases. + tvb: KVVec, } impl DeferredBatch { const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize @@ -676,14 +678,17 @@ impl DeferredBatch { Self::with_capacity(2 * Self::DEPTH, 0) } fn with_capacity(capacity: usize, detached: usize) -> Result { + let render_slots = if detached != 0 { + crate::hw::t8140::queues::RENDER_SLOTS + } else { + 0 + }; Ok(Self { values: KVVec::with_capacity(capacity, GFP_KERNEL)?, bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, - render: KVVec::with_capacity( - if detached != 0 { crate::hw::t8140::queues::RENDER_SLOTS } else { 0 }, - GFP_KERNEL, - )?, + render: KVVec::with_capacity(render_slots, GFP_KERNEL)?, + tvb: KVVec::with_capacity(render_slots, GFP_KERNEL)?, }) } /// Keeps the detached owner's unmapping and VM teardown out of the device @@ -702,6 +707,14 @@ impl DeferredBatch { drop(backing); } } + pub(super) fn tvb_room(&self) -> bool { + self.tvb.len() < self.tvb.capacity() + } + pub(super) fn defer_tvb(&mut self, release: super::buffer::TvbRelease) { + if let Err(release) = self.tvb.push_within_capacity(release) { + drop(release); + } + } pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { if let Err(vacated) = self.vacated.push_within_capacity(vacated) { drop(vacated); @@ -753,6 +766,9 @@ impl DeferredBatch { for backing in self.render.drain_all() { drop(backing); } + for release in self.tvb.drain_all() { + drop(release); + } } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 99f3caf66bebe2..ee4278c4204557 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -13,6 +13,9 @@ use crate::g17::{ }; use kernel::{prelude::*, sync::Arc}; +/// Wake interval while a tiler heap may still shrink: the shrink check period. +const SHRINK_WAKE_NS: u64 = 1_000_000_000; + struct Service<'a> { firmware: &'a mut Firmware, deferred: &'a mut DeferredBatch, @@ -387,10 +390,14 @@ impl crate::g17::Shared { firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; firmware.service_compute_pools(deferred)?; + let mut shrinking = false; if !firmware.recovery.pending() { firmware .queues .vacate_evicted_render_pools(&firmware.init, deferred)?; + shrinking = firmware + .queues + .shrink_render_heaps(&firmware.init, now, deferred); } { let mut reclaim = self.reclaim.lock(); @@ -414,6 +421,13 @@ impl crate::g17::Shared { if polling { self.queue_poll(); } + // A heap with growth to give back is re-evaluated at least once per + // shrink period, also while the device is otherwise idle. + let idle = if shrinking { + Some(idle.map_or(SHRINK_WAKE_NS, |delay| delay.min(SHRINK_WAKE_NS))) + } else { + idle + }; if let Some(delay) = idle { self.queue_idle(delay); } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index 71b21f98479a6c..bd68fb50c38fc5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -647,6 +647,80 @@ impl Registry { || !self.render.retired.is_empty() || !self.teardown.is_empty() } + /// Applies the tiler-heap shrink policy (`buffer::Manager:: + /// shrink_target`) to every grown heap at `now` and rebuilds the heaps of + /// idle pairs that are due, removing whole growth extensions. The removed + /// aliases and backing go to the worker's deferred batch, which frees them + /// after the device mutex. A shrink is best effort: any failure leaves the + /// heap as it was. Returns whether a heap still holds growth it could give + /// back, so the worker keeps a periodic wake. + pub(in crate::g17) fn shrink_render_heaps( + &mut self, + init: &crate::g17::initdata::InitData, + now: u64, + deferred: &mut super::DeferredBatch, + ) -> bool { + let mut pending = false; + for (slot, entry) in self.render.entries.iter_mut().enumerate() { + let Some(entry) = entry.as_mut() else { + continue; + }; + let borrowed = entry.constructing || entry.growing || entry.preparing.is_some(); + let Some(pair) = entry.pair.as_mut() else { + continue; + }; + if !pair.manager().grown() { + continue; + } + let busy = borrowed || !pair.idle_for_context(pair.context()).unwrap_or(false); + if let Ok(Some(target)) = pair.manager().shrink_target(now, busy) { + if deferred.tvb_room() { + if let Ok(slot) = u8::try_from(slot) { + // Ignoring the result keeps the heap unchanged on failure. + let _ = Self::shrink_pair(init, slot, pair, target, deferred); + } + } + } + pending |= pair.manager().shrinkable(); + } + pending + } + + fn shrink_pair( + init: &crate::g17::initdata::InitData, + slot: u8, + pair: &mut Pair, + target: usize, + deferred: &mut super::DeferredBatch, + ) -> Result { + use core::sync::atomic::{fence, Ordering}; + let offset = usize::from(pair.buffer_id()) * 16; + let table = init.pb_descriptor_table()?; + let words = [ + table.word(offset)?, + table.word(offset + 4)?, + table.word(offset + 8)?, + table.word(offset + 12)?, + ]; + fence(Ordering::Acquire); + let descriptor = words.each_ref().map(|word| word.load(Ordering::Relaxed)); + let vm = pair.context().vm().clone(); + let pool = pair.pool_id(); + let shrunk = pair + .manager() + .shrink(target, descriptor, |blocks, kept, out| { + vm.truncate_render_pool_mappings(slot, pool, blocks, kept, out) + })?; + if let Some((reloaded, release)) = shrunk { + for (word, value) in words.into_iter().zip(reloaded) { + word.store(value, Ordering::Relaxed); + } + fence(Ordering::SeqCst); + deferred.defer_tvb(release); + } + Ok(()) + } + /// Gives back the USC backing of render pools whose client VM pressure /// reclaim evicted (an idle VM) and whose final free-list release was /// consumed. The vacated backings go to the worker's deferred batch, which diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index d53c19b93bfdcd..975126e0cc76b0 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -238,6 +238,40 @@ impl Vm { Ok(()) } + /// Moves the aliases of the last `blocks - kept` heap blocks of the exact + /// cached owner into `out`, which the caller drops after its locks. The + /// growth path appends one alias per added block, so these are the + /// blocks a shrink removed. Nothing moves unless all of them fit in `out`. + pub(crate) fn truncate_render_pool_mappings( + &self, + slot: u8, + id: u64, + blocks: usize, + kept: usize, + out: &mut KVec, + ) -> Result { + let count = blocks.checked_sub(kept).ok_or(EINVAL)?; + if id == 0 || count == 0 || out.capacity() - out.len() < count { + return Err(EINVAL); + } + let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; + let mut driver = lifetime.driver.lock(); + let pool = Option::as_mut(&mut *driver) + .and_then(|driver| driver.render.get_mut(usize::from(slot))) + .and_then(Option::as_mut) + .filter(|pool| pool.id == id) + .ok_or(EINVAL)?; + if pool.blocks != blocks || pool.mappings.len() < count { + return Err(EBUSY); + } + for _ in 0..count { + let mapping = pool.mappings.pop().ok_or(EIO)?; + out.push_within_capacity(mapping).map_err(|_| EIO)?; + } + pool.blocks = kept; + Ok(()) + } + /// A delayed teardown must not remove a successor generation. Remove /// under the cache lock, then unmap after dropping that lock. pub(crate) fn clear_render_pool_mappings_if_owner(&self, slot: u8, id: u64) { From 14e504d64aa23484bd0ade39cdddddd86882ceb6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:50:48 -0400 Subject: [PATCH 293/631] drm/asahi: give back warm G17 compute pools after ten idle seconds A compute queue whose owner closed or exited stays installed for reuse, and up to four such queues (the warm cap) keep their 23 MiB USC private-memory pool populated for as long as no newer queue displaces them. On J700, the kernel's GEM after each game exit sat 54-78 MiB above its level before the game started, mostly in these warm pools. Releasing every pool as soon as its owner is gone (a cap of zero) would make each new client re-arm fresh backing in its first submission, which costs about 2.3 ms on a cold start (measured at cap 0 on hardware). Keep the warm cap for bursts of short-lived clients, and additionally release a retained pool once it has been idle for ten seconds, the same period as the idle tiler-heap shrink. The event worker schedules its wake for the next expiry, and retries 100 ms later when a pass could not release every due pool. Queues without a release time stay bounded by the cap only, as before. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c01af7a9e73c789c6a9f98547585a7048927af30) --- drivers/gpu/drm/asahi/g17/runtime.rs | 66 +++++++++++++++------ drivers/gpu/drm/asahi/g17/runtime/events.rs | 7 ++- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index e4f19397e5f923..65b58d477dfa5d 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -176,30 +176,42 @@ impl Registry { queue.bind_owner(owner, context.clone()) } - /// Publishes releases for the oldest idle retained pools beyond `cap`, at - /// most four per pass. Returns the number published. + /// Delay before retrying pools a turn could not release. + const POOL_RETRY_NS: u64 = 100_000_000; + /// Publishes releases for the oldest idle retained pools beyond `cap` and + /// for every pool released at least `expiry` before `now`, at most four + /// per pass. Returns the number published and the delay until the next + /// retained pool expires or a retry is due. pub(super) fn vacate_idle_pools( &mut self, init: &super::initdata::InitData, cap: usize, - ) -> Result { + now: u64, + expiry: u64, + ) -> Result<(usize, Option)> { // Count first: event turns are frequent and the steady state has no excess. - let forced = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.vacate_candidate() && queue.needs_fresh_backing()) - .count(); - let count = self + let mut forced = 0; + let mut count = 0; + let mut next = None::; + for queue in self .compute .iter() .flatten() .filter_map(|entry| entry.queue.as_deref()) .filter(|queue| queue.vacate_candidate()) - .count(); + { + count += 1; + let released = queue.released_at(); + let age = now.saturating_sub(released); + if queue.needs_fresh_backing() || (released != 0 && age >= expiry) { + forced += 1; + } else if released != 0 { + let left = expiry - age; + next = Some(next.map_or(left, |next| next.min(left))); + } + } if count <= cap && forced == 0 { - return Ok(0); + return Ok((0, next)); } let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; for queue in self @@ -213,6 +225,7 @@ impl Registry { let key = if queue.needs_fresh_backing() { 0 } else { queue.released_at() }; candidates.push((key, queue.qid()), GFP_KERNEL)?; } + // Recycled and expired pools are the oldest keys and sort first. candidates.sort_unstable(); let excess = candidates.len().saturating_sub(cap).max(forced); let mut published = 0; @@ -230,7 +243,11 @@ impl Registry { } published += 1; } - Ok(published) + if published < excess { + // Beyond this turn's bound, or not yet releasable: retry soon. + next = Some(next.map_or(Self::POOL_RETRY_NS, |next| next.min(Self::POOL_RETRY_NS))); + } + Ok((published, next)) } /// Frees the backing of every pool whose release the firmware consumed. @@ -1161,16 +1178,24 @@ impl super::Shared { /// Idle retained compute queues that keep their USC pool populated; older idle /// pools give their backing back. const COMPUTE_WARM_CAP: usize = 4; +/// A retained pool idle this long gives its backing back regardless of the +/// cap: a burst of short-lived clients still finds warm pools, but the pools +/// of clients that are gone do not stay resident. Ten seconds matches the +/// idle period of the tiler-heap shrink, so idle compute pools and idle +/// tiler heaps are given back on the same schedule. +const COMPUTE_WARM_EXPIRY_NS: u64 = 10_000_000_000; impl super::Firmware { /// Event-worker service step: witness consumed pool releases, then release the /// oldest idle retained pools beyond the warm cap. /// Detached owner bindings and vacated pool backing go to `deferred`, which /// the worker finishes after releasing the device mutex. + /// Returns the delay until a retained pool expires, for the worker's wake. pub(in crate::g17) fn service_compute_pools( &mut self, deferred: &mut DeferredBatch, - ) -> Result { + now: u64, + ) -> Result> { for queue in self .queues .compute @@ -1185,14 +1210,19 @@ impl super::Firmware { } self.queues.observe_pool_releases(&self.init, deferred)?; if self.render_control_backpressured() { - return Ok(()); + return Ok(None); } - let published = self.queues.vacate_idle_pools(&self.init, COMPUTE_WARM_CAP)?; + let (published, next) = self.queues.vacate_idle_pools( + &self.init, + COMPUTE_WARM_CAP, + now, + COMPUTE_WARM_EXPIRY_NS, + )?; if published != 0 { fence(Ordering::SeqCst); self.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; } - Ok(()) + Ok(next) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index ee4278c4204557..81ca10328b42a4 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -389,7 +389,7 @@ impl crate::g17::Shared { let idle = firmware.idle_check(now, in_flight)?; firmware.reclaim_drained_renders(deferred)?; firmware.service_compute_exit(deferred)?; - firmware.service_compute_pools(deferred)?; + let pools = firmware.service_compute_pools(deferred, now)?; let mut shrinking = false; if !firmware.recovery.pending() { firmware @@ -428,6 +428,11 @@ impl crate::g17::Shared { } else { idle }; + // A retained compute pool gives its backing back when it expires. + let idle = match (idle, pools) { + (Some(delay), Some(expiry)) => Some(delay.min(expiry)), + (delay, expiry) => delay.or(expiry), + }; if let Some(delay) = idle { self.queue_idle(delay); } From a847bf7d815d12d3787202921a452f62bb1aa01e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 04:26:27 -0400 Subject: [PATCH 294/631] drm/asahi: keep compaction away from GEM pages held for the GPU Every populated asahi GEM object keeps its shmem pages referenced: by the device SG table, a kernel vmap or a PFN CPU mapping. Those pages sit on the unevictable LRU and cannot migrate, but the compaction migrate scanner still isolates them (compact_unevictable_allowed) and every migration fails. In-world on J700, with about 5 GB of GEM held, background compaction isolated about 3,400 pages a second and 97 % of the migrations failed (pgmigrate_fail 3,203/s against pgmigrate_success 104/s). Mark the shmem mapping of a held object inaccessible. Compaction skips folios of inaccessible mappings before isolating them, and migration refuses them; the flag comes with unevictable, as mapping_set_inaccessible() sets it. Nothing reads or writes a GEM shmem file through its mapping. Kernel objects are marked when created; other objects when they take a device SG table. Releasing idle backing to reclaim, or purging, clears the mark, so released pages can be swapped and migrated again. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 2d7de7690cc0f9c8d3a74b08c33999593b720aac) --- drivers/gpu/drm/asahi/gem.rs | 48 +++++++++++++++++++++++++- drivers/gpu/drm/asahi/mmu.rs | 6 ++-- drivers/gpu/drm/asahi/mmu/bind.rs | 2 +- drivers/gpu/drm/asahi/mmu/residency.rs | 4 ++- 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index fb3d5a7f91ce34..a9a67c83c5e80d 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -13,7 +13,8 @@ use kernel::{ shmem, shmem::VMap, BaseObject, - DriverObject, // + DriverObject, + IntoGEMObject, // }, error::Result, prelude::*, @@ -24,6 +25,7 @@ use kernel::{ use core::ops::Range; use core::sync::atomic::{ AtomicU64, + AtomicUsize, Ordering, // }; @@ -60,6 +62,50 @@ pub(crate) struct AsahiObject { /// Type alias for the shmem GEM object type for this driver. pub(crate) type Object = shmem::Object; +/// Marks whether the pages of `gem` are held for the device. +/// +/// Populated asahi objects keep their pages referenced (device SG table, kernel vmap or a PFN +/// CPU mapping), so the pages cannot migrate. Unmarked, the compaction migrate scanner still +/// isolates them from the unevictable LRU and every migration fails. The shmem mapping of a +/// held object is therefore marked inaccessible, which compaction and migration check and which +/// comes with unevictable, as `mapping_set_inaccessible()` sets it; nothing reads or writes a GEM +/// shmem file through its mapping. Releasing idle backing or purging clears the mark, so the +/// released pages can be swapped and migrated again. +pub(crate) fn set_mapping_held(gem: &Object, held: bool) { + let raw = gem.as_raw(); + // SAFETY: `gem` keeps the GEM object alive. A shmem-backed object's `filp` and its + // `f_mapping` are set at initialization and never change; imported objects have none. + let mapping = unsafe { + let filp = (*raw).filp; + if filp.is_null() { + return; + } + (*filp).f_mapping + }; + if mapping.is_null() { + return; + } + // SAFETY: `flags` is the aligned flag word of the live mapping. The kernel updates it only + // with atomic bit operations on the same word. + let flags = + unsafe { AtomicUsize::from_ptr(core::ptr::addr_of_mut!((*mapping).flags).cast::()) }; + let inaccessible = 1usize << kernel::bindings::mapping_flags_AS_INACCESSIBLE; + let unevictable = 1usize << kernel::bindings::mapping_flags_AS_UNEVICTABLE; + if held { + flags.fetch_or(inaccessible | unevictable, Ordering::Relaxed); + } else { + flags.fetch_and(!inaccessible, Ordering::Relaxed); + } +} + +/// Takes an owned SG table of `gem`, which holds its pages for the device, and marks the +/// object held (see [`set_mapping_held`]). +pub(crate) fn held_sg_table(gem: &Object) -> Result> { + let sgt = gem.owned_sg_table()?; + set_mapping_held(gem, true); + Ok(sgt) +} + /// VM-private BOs share their assigned VM's reservation object. Validate this /// before acquiring mapping state; generic GPUVM accepts external shareable BOs. pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 0b1d5b66bb69b9..f81e845ac1d6cf 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -2710,7 +2710,7 @@ impl Vm { } let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut inner = self.inner.exec_lock(Some(gem), false)?; let vm_bo = self.inner.obtain_bo(gem)?; @@ -2792,7 +2792,7 @@ impl Vm { .checked_add(if guard { UAT_PGSZ } else { 0 }) .ok_or(EOVERFLOW)?; let _residency = self.enter_residency()?; - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut inner = self.inner.exec_lock(Some(&gem), false)?; let vm_bo = self.inner.obtain_bo(&gem)?; @@ -2886,7 +2886,7 @@ impl Vm { if needs_sgt { // Pin without the BO mutex: alias construction can hold the GEM // reservation while acquiring this mutex. - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut vm_bo_guard = vm_bo.inner().inner.lock(); if vm_bo_guard.sgt.is_none() { if vm_bo_guard.sg_vec.is_none() { diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index df189e9bc0f5fc..47a3421fa04b34 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -82,7 +82,7 @@ impl Vm { if needs_sgt { // Pin without the BO mutex: alias construction can hold the GEM // reservation while acquiring this mutex. - let sgt = gem.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&gem)?; let mut bo = vm_bo.inner().inner.lock(); if bo.sgt.is_none() { if bo.sg_vec.is_none() { diff --git a/drivers/gpu/drm/asahi/mmu/residency.rs b/drivers/gpu/drm/asahi/mmu/residency.rs index 73a1d163eca826..afc9255100fa2b 100644 --- a/drivers/gpu/drm/asahi/mmu/residency.rs +++ b/drivers/gpu/drm/asahi/mmu/residency.rs @@ -440,6 +440,7 @@ impl Vm { bo_inner.sgt = None; } if object.purge_locked().is_ok() { + crate::gem::set_mapping_held(object, false); purged = purged.saturating_add(object.size() / kernel::page::PAGE_SIZE); } else { gate.note_evicted(); @@ -519,6 +520,7 @@ impl Vm { // tables above. Kernel mappings and other SG owners are excluded; // the C/Rust backing helper rechecks exclusion before release. if object.release_idle_pages_locked().is_ok() { + crate::gem::set_mapping_held(object, false); unpinned = unpinned.saturating_add(object.size() / kernel::page::PAGE_SIZE); } Ok(unpinned < target) @@ -566,7 +568,7 @@ impl Vm { if needs_sgt { // Preserve review1's reservation/BO lock order: the getter // may acquire the object reservation; never hold BO here. - let sgt = map.object.owned_sg_table()?; + let sgt = crate::gem::held_sg_table(&map.object)?; let mut bo = bo_owner.inner().inner.lock(); if bo.sgt.is_none() { bo.sgt = Some(sgt); } drop(bo); From 536e4a5a6fe38b3f5d26836b73851652e184cc6e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 00:18:43 -0400 Subject: [PATCH 295/631] drm/asahi: complete page-table stores before GPU TLB invalidation Use the existing DSB helper before outer-shareable TLB invalidations. A sequentially consistent Rust fence emits DMB and does not provide the completion required before the GPU discards its cached translations. Publish new page-table leaves even when no context currently names the root, so a later binding cannot expose incomplete table stores. Keep invalidation ranges, queue admission and submission batching intact. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6dd0f73fd3b2bade5415b31346aadeb2f38d446e) --- drivers/gpu/drm/asahi/g17/kick.rs | 2 +- drivers/gpu/drm/asahi/mmu.rs | 36 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/kick.rs b/drivers/gpu/drm/asahi/g17/kick.rs index f109c8261476e3..3b78b9443a86de 100644 --- a/drivers/gpu/drm/asahi/g17/kick.rs +++ b/drivers/gpu/drm/asahi/g17/kick.rs @@ -308,7 +308,7 @@ impl Queue { if self.id.engine != DataMaster::Compute { return Err(EINVAL); } - fence(Ordering::SeqCst); + mem::sync(); for asid in 0..4 { mem::tlbi_range_or_asid(asid, self.low_va() as usize, KICK_RING_SIZE); mem::tlbi_range_or_asid(asid, self.firmware_va() as usize, KICK_RING_SIZE); diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index f81e845ac1d6cf..0088fcf90228b8 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -689,7 +689,7 @@ impl gpuvm::DriverGpuVm for VmInner { } if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, unmap_start, unmap_range as usize); mod_dev_dbg!( self.dev, @@ -748,7 +748,7 @@ impl VmInner { } if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, va.addr(), va.range() as usize); mod_dev_dbg!( self.dev, @@ -773,7 +773,7 @@ impl VmInner { let result = self.page_table.discard_partial_map(addr..addr + size); if self.uat_inner.firmware == UatFirmware::Handoff { if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); + mem::sync(); self.tlbi_range(asid as u8, addr, size as usize); mem::sync(); } @@ -922,6 +922,7 @@ impl VmInner { /// already invalidate. fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { if self.uat_inner.firmware == UatFirmware::Handoff { + mem::sync(); return; } @@ -930,11 +931,14 @@ impl VmInner { } fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { + // Complete new leaves even if no context currently names this root: + // a later context can publish it without repeating the map operation. + // DMB orders accesses but does not complete stores before TLBI; the + // GPU uses outer-shareable invalidation and the barrier covers it. + mem::sync(); if contexts == 0 { return; } - - fence(Ordering::SeqCst); for ctx in 0..UAT_NUM_CTX { if contexts & (1 << ctx) != 0 { mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); @@ -1182,7 +1186,7 @@ impl ExecutionRoot { owned || (low == 0 && high == 0) }; if release { - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(self.id); mem::sync(); self.inner.lock().execution_contexts &= !(1u64 << id); @@ -1440,7 +1444,7 @@ impl KernelMapping { self.iova()..self.iova() + prefix as u64, PROT_GPU_FW_PRIV_RW, )?; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); let mut after = [0; MAX_PAGES]; @@ -1495,7 +1499,7 @@ impl KernelMapping { self.size() ); } - fence(Ordering::SeqCst); + mem::sync(); // If we don't have (and have never had) a VM slot, just return let slot = match owner.slot() { @@ -1642,7 +1646,7 @@ impl KernelMapping { } if let Some(asid) = owner.slot() { - fence(Ordering::SeqCst); + mem::sync(); owner.tlbi_range(asid as u8, self.iova(), self.size()); mod_dev_dbg!( owner.dev, @@ -3218,6 +3222,7 @@ impl Drop for VmInner { self.id, idx ); + mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); } @@ -3306,7 +3311,7 @@ impl Uat { if contexts == 0 { return Err(ENOENT); } - fence(Ordering::SeqCst); + mem::sync(); for context in 0..UAT_NUM_CTX { if contexts & (1u64 << context) != 0 { mem::tlbi_asid(context as u8); @@ -3422,6 +3427,7 @@ impl Uat { core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone + mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); slot.generation = slot.generation.wrapping_add(1).max(1); @@ -3490,7 +3496,7 @@ impl Uat { roots.ttb0.store(low, Ordering::Release); roots.ttb1.store(high, Ordering::Release); } - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(id as u8); mem::sync(); Ok(context) @@ -3537,7 +3543,7 @@ impl Uat { }) }?; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); Ok(snapshot) @@ -3582,7 +3588,7 @@ impl Uat { }) }??; - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); Ok(()) @@ -3607,7 +3613,7 @@ impl Uat { .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); inner.handoff().unlock(); core::mem::drop(inner); - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); mem::sync(); Ok(()) @@ -3916,7 +3922,7 @@ impl Drop for Uat { } // Make sure we flush the TLBs - fence(Ordering::SeqCst); + mem::sync(); mem::tlbi_all(); mem::sync(); } From 55f77e4fabd1c8e514b67bd9e4b6f364f32e3412 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 00:18:43 -0400 Subject: [PATCH 296/631] drm/asahi: remove tiler-shrink diagnostic logging Keep an incomplete idle inventory unchanged without a global report counter or per-heap diagnostic state. Correct the backing-layout comment and place it on Tvb rather than the deferred release container. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 019ad857ddfa08ed2473eae34db5b8b8d98b068e) --- drivers/gpu/drm/asahi/g17/buffer.rs | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/buffer.rs b/drivers/gpu/drm/asahi/g17/buffer.rs index 652f49a99258b2..9294ee818daf57 100644 --- a/drivers/gpu/drm/asahi/g17/buffer.rs +++ b/drivers/gpu/drm/asahi/g17/buffer.rs @@ -331,8 +331,6 @@ pub(crate) struct TvbGrowth { extension: KernelObject, } -/// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their -/// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. /// Aliases and backing of growth extensions a shrink removed from an idle /// heap. Field order unmaps the aliases before freeing their backing. pub(crate) struct TvbRelease { @@ -340,6 +338,8 @@ pub(crate) struct TvbRelease { _extensions: KVec, } +/// Packed 128-KiB data blocks mapped at guarded 160-KiB virtual strides. +/// Firmware owns their contents; CPU zeroing occurs only on fresh backing. pub(crate) struct Tvb { addresses: KVec, // Retain sparse-block backing until the pool and its aliases retire. @@ -593,12 +593,8 @@ struct Shrink { window: [u32; SHRINK_WINDOW], window_next: usize, busy_peak: u32, - reported: bool, } -/// Bounds the one-line report of heaps that could not be rebuilt. -static SHRINK_REPORTS: AtomicU32 = AtomicU32::new(0); - pub(crate) struct Submitted<'a> { word: &'a AtomicU32, next: u32, @@ -1046,18 +1042,6 @@ impl Manager { let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) % cfg::TVB_BLOCK_SLOTS; if !before.valid() || before.total != before.committed || retained != blocks { - if !self.shrink.reported && SHRINK_REPORTS.fetch_add(1, Ordering::Relaxed) < 8 { - self.shrink.reported = true; - pr_info!( - "G17: tiler heap {} kept {} blocks: inventory {}/{}/{} holds {}\n", - self.buffer_id, - blocks, - before.total, - before.committed, - before.read, - retained - ); - } return Ok(None); } // Allocation failure only postpones the shrink to a later pass. @@ -1121,7 +1105,6 @@ impl Manager { } } self.shrink.credit = self.shrink.credit.saturating_sub(blocks - kept); - self.shrink.reported = false; Ok(Some(( fw::reload_descriptor(descriptor, self.page_list_client, pages), release, From 0ad1574a5ebb9c264b9ace66c90b83852897cf73 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 17:37:36 -0400 Subject: [PATCH 297/631] mm: reuse subpage backing when refilling anonymous holes A missing 4K PTE in a partially populated native 16K window currently gets a fresh private folio. Existing neighbors are skipped and the unused quarters are stranded when its pool closes. Repeated per-quarter hole refills can leave 16 MiB of logical data backed by 64 MiB of native pages. Reserve the missing-quarter mask from one compatible shared owner, keeping whole-window private allocation for initial dense faults. Initialize and hold the selected folio through reverse-map setup and PTE publication. Revalidate the missing mask under the PTE lock, release surplus or losing reservations, and retain the private fallback for metadata allocation failure. Existing pool charge, NUMA, locking and droppable exclusions apply. Add density, neighbor-prefault, concurrent refill, owner-exit, and controlled reclaim selftests. On native 16K Neo hardware with the 4K userspace ABI, the 16 MiB refill fixture finishes at 16.09 MiB instead of 64 MiB, with byte-exact contents. Initial dense backing stays at 16 MiB. Native/4K ABI, fork/exec, mprotect, existing concurrent COW and GPU quarter-refault controls pass. Controlled reclaim swaps all 4096 test leaves after original-owner exit, then passes byte-exact readback. Subnautica menu/world backing snapshots show lower native backing overhead; this is not an FPS qualification. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e8fb3863c6103fa0150ba98979756e6685e533d9) --- include/linux/mm_subpage.h | 9 + mm/memory.c | 183 ++++++++--- mm/user-subpage.c | 133 ++++++-- tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../selftests/arm64/user4k/README.hole-refill | 29 ++ .../arm64/user4k/README.hole-refill-race | 33 ++ .../arm64/user4k/README.hole-refill-reclaim | 36 +++ .../arm64/user4k/anon-hole-refill-race.c | 298 +++++++++++++++++ .../arm64/user4k/anon-hole-refill-reclaim.c | 299 ++++++++++++++++++ .../selftests/arm64/user4k/anon-hole-refill.c | 179 +++++++++++ 10 files changed, 1140 insertions(+), 63 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill-race create mode 100644 tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c create mode 100644 tools/testing/selftests/arm64/user4k/anon-hole-refill.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index c4ac2eb6c4236b..9a1d33166db30c 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -52,6 +52,15 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* Exact physical-slot mask, one owner and folio lock for the entire batch. + * slots is indexed by physical offset >> pool shift; only mask bits are set. + * Success returns initialized owned slots; errors leave slots untouched. + */ +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots); +int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long mask, gfp_t gfp); +bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask); /* Return an initialized slot with its folio locked; may wait without a pool lock. */ struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index 2fb430e8426035..4006d3cef29ffd 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6375,17 +6375,95 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, } #ifdef CONFIG_MM_SUBPAGE +/* One native-window batch, initialized and locked before PTE publication. */ +static int anon_subpage_prealloc_mask(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long addr, + unsigned long mask, bool use_pool, bool *pooled, + struct mm_subpage **slots) +{ + struct mm_subpage_pool *pool; + struct folio *folio; + int err; + +retry_private: + pool = use_pool ? mm_subpage_cow_pool_get(mm, vma, addr) : NULL; + *pooled = pool != NULL; + if (!pool) + pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!pool) + return -ENOMEM; + for (;;) { + err = mm_subpage_alloc_mask_locked(pool, mask, slots); + if (err != -EAGAIN) + break; + folio = NULL; + if (*pooled) { + folio = cow_subpage_speculative_folio(mm, vma, addr); + if (!folio && mm_subpage_cow_wait_busy_mask(pool, mask)) { + cond_resched(); + continue; + } + } + if (!folio) + folio = folio_prealloc(mm, vma, addr, false); + if (!folio) { + err = -ENOMEM; + break; + } + if (*pooled && !mm_subpage_cow_folio_matches(pool, folio)) { + folio_put(folio); + mm_subpage_pool_put(pool); + use_pool = false; + goto retry_private; + } + err = mm_subpage_pool_add_folio_mask(pool, folio, mask, + *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + if (*pooled && err == -ENOMEM) { + if (mm_subpage_cow_wait_busy_mask(pool, mask)) { + folio_put(folio); + cond_resched(); + continue; + } + err = mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); + } + if (err) { + folio_put(folio); + if (err == -EEXIST) { + cond_resched(); + continue; + } + break; + } + } + if (!*pooled) + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); + return err; +} + +static unsigned long anon_subpage_missing_mask(pte_t *first, unsigned int nr, + unsigned int target, unsigned int offset) +{ + unsigned long mask = 0; + unsigned int i; + + for (i = 0; i < nr; i++) + if (i == target || pte_none(ptep_get(first + i))) + mask |= BIT(offset + i); + return mask; +} + static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct mm_struct *mm = vma->vm_mm; struct mm_subpage *slots[PAGE_SIZE / MM_SUBPAGE_SIZE] = {}; struct vm_page_offset pos = vma_page_offset_at(vma, vmf->address); - struct mm_subpage_pool *pool; struct folio *folio; - unsigned long start, end, lower; - unsigned int first_offset, nr, target, i, mapped = 0; - vm_fault_t ret = VM_FAULT_OOM; + unsigned long start, end, lower, mask, current_mask, window_mask; + unsigned int first_offset, nr, target, i, index, mapped; + bool pooled, fresh, force_private = false, retry_private; + vm_fault_t ret; pte_t *first; int err; @@ -6394,34 +6472,48 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) start = vmf->address - min_t(unsigned long, pos.offset, vmf->address - lower); end = vmf->address + min(PAGE_SIZE - pos.offset, pmd_addr_end_mm(mm, vmf->address, vma->vm_end) - vmf->address); - first_offset = pos.offset - (vmf->address - start); + first_offset = (pos.offset - (vmf->address - start)) >> mm_page_shift(mm); nr = (end - start) >> mm_page_shift(mm); target = (vmf->address - start) >> mm_page_shift(mm); - - pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!pool) + window_mask = GENMASK(first_offset + nr - 1, first_offset); +retry: + /* This optimistic snapshot only selects backing; PTL revalidates it. + * Initial dense faults retain one-folio, whole-window faultaround. + */ + vmf->pte = pte_offset_map_mm(mm, vmf->pmd, vmf->address); + if (!vmf->pte) + return 0; + mask = anon_subpage_missing_mask(vmf->pte - target, nr, target, first_offset); + pte_unmap(vmf->pte); + vmf->pte = NULL; + err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, + mask != window_mask && !force_private && !userfaultfd_armed(vma), + &pooled, slots); + if (err) return VM_FAULT_OOM; - /* Supply initialises every byte once, including the unissued slots. */ - folio = folio_prealloc(mm, vma, vmf->address, false); - if (!folio) - goto release; - err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); - if (err) { - folio_put(folio); - goto release; - } - for (i = 0; i < nr; i++) { - slots[i] = mm_subpage_alloc_at(pool, first_offset + i * mm_page_size(mm)); - if (IS_ERR(slots[i])) { - slots[i] = NULL; - goto release; - } - } - + folio = mm_subpage_folio(slots[__ffs(mask)]); /* PTE references may disappear as soon as we drop the PTL. */ folio_get(folio); - folio_lock(folio); + fresh = !mm_subpage_anon_root(slots[__ffs(mask)]); + mapped = 0; + retry_private = false; ret = 0; + /* Existing owners may need nonlinear identities. Allocate metadata before + * PTL and keep the established fresh-folio fallback on metadata ENOMEM. + */ + if (!fresh) { + for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { + unsigned long address = start + + ((index - first_offset) << mm_page_shift(mm)); + + err = mm_subpage_bind_rmap(slots[index], vma, address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; + } + } + } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) goto unlock_folio; @@ -6442,20 +6534,30 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) } first = vmf->pte - target; + current_mask = anon_subpage_missing_mask(first, nr, target, first_offset); + /* Do not lose newly missing neighbors or publish sparse dense-path supply. + * Return to the fault handler on a racing mask change instead of spinning + * under repeated zaps. No mapping has been published yet. + */ + if ((current_mask & ~mask) || + (mask == window_mask && current_mask != mask && + !force_private && !userfaultfd_armed(vma))) + goto unlock_pte; for (i = 0; i < nr; i++) { unsigned long address = start + i * mm_page_size(mm); pte_t entry; - /* Do not replace existing mappings or userfaultfd markers nearby. */ - if (i != target && !pte_none(ptep_get(first + i))) + index = first_offset + i; + if (!(current_mask & BIT(index))) continue; - mm_subpage_set_exclusive(slots[i]); - err = mm_subpage_add_new_anon_rmap(slots[i], vma, address); + mm_subpage_set_exclusive(slots[index]); + err = fresh ? mm_subpage_add_new_anon_rmap(slots[index], vma, address) : + mm_subpage_add_anon_rmap(slots[index], vma, address); if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; break; } - entry = phys_pte_mm(mm, mm_subpage_phys(slots[i]), vma->vm_page_prot); + entry = phys_pte_mm(mm, mm_subpage_phys(slots[index]), vma->vm_page_prot); entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry), vma); @@ -6463,11 +6565,12 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) entry = pte_mkuffd_wp(entry); set_pte_at(mm, address, first + i, entry); update_mmu_cache(vma, address, first + i); - slots[i] = NULL; /* The PTE now owns the reference. */ + slots[index] = NULL; /* The PTE now owns the reference. */ mapped++; } if (mapped) { - subpage_add_new_lru(folio, vma, mapped); + if (fresh) + subpage_add_new_lru(folio, vma, mapped); add_mm_counter(mm, MM_ANONPAGES, mapped); count_mthp_stat(0, MTHP_STAT_ANON_FAULT_ALLOC); } @@ -6478,13 +6581,19 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) folio_unlock(folio); folio_put(folio); release: - mm_subpage_pool_close(pool); - for (i = 0; i < ARRAY_SIZE(slots); i++) - if (slots[i]) + for (i = 0; i < ARRAY_SIZE(slots); i++) { + if (slots[i]) { mm_subpage_put(slots[i]); - mm_subpage_pool_put(pool); + slots[i] = NULL; + } + } + if (retry_private) { + force_private = true; + goto retry; + } return ret; } + #endif /* diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 3adfb04ccdca16..c7876fd34a091b 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -842,16 +842,18 @@ static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio * normal bounded search, inspect all owners before allowing an OOM-capable * allocation: a busy slot farther down the list may require no new charge. */ -bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, + bool require_all) { struct mm_subpage_owner *owner; struct folio *wait = NULL; - unsigned long flags, eligible = offset_mask(pool->shift, offset); + unsigned long flags; spin_lock_irqsave(&pool->lock, flags); if (!pool->closed && pool->cow_shared && cow_has_free(pool, eligible)) { list_for_each_entry(owner, &pool->available, available) { if (!owner->migrating && (~owner->used & eligible) && + (!require_all || !(owner->used & eligible)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && !folio_test_writeback(owner->folio)) { @@ -870,8 +872,18 @@ bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offs return false; } +bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) +{ + return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); +} + +bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask) +{ + return cow_wait_busy(pool, mask, true); +} + static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, - unsigned long eligible) + unsigned long eligible, bool require_all) { struct mm_subpage_owner *owner; unsigned int visited = 0; @@ -893,6 +905,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && + (!require_all || !(owner->used & eligible)) && (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && @@ -903,7 +916,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, - unsigned long eligible, gfp_t gfp) + unsigned long eligible, bool require_all, gfp_t gfp) { struct mm_subpage_owner *owner; unsigned long flags; @@ -931,7 +944,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (pool->unissued || find_available(pool, eligible)) + else if (pool->unissued || find_available(pool, eligible, require_all)) /* Another allocator supplied backing while we allocated metadata. * Preserve the single-unissued-owner invariant even when a shared * COW lookup skips a concurrently locked or rotating list prefix. @@ -959,7 +972,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, gfp_t gfp) { - return pool_add_folio(pool, folio, pool->all_slots, gfp); + return pool_add_folio(pool, folio, pool->all_slots, false, gfp); } int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, @@ -969,7 +982,15 @@ int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *fol if (!eligible) return -EINVAL; - return pool_add_folio(pool, folio, eligible, gfp); + return pool_add_folio(pool, folio, eligible, false, gfp); +} + +int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, + unsigned long mask, gfp_t gfp) +{ + if (!mask || (mask & ~pool->all_slots)) + return -EINVAL; + return pool_add_folio(pool, folio, mask, true, gfp); } unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) @@ -982,6 +1003,34 @@ unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) return pages; } +/* Caller holds pool lock; each issued slot owns one native folio reference. */ +static struct mm_subpage *reserve_owner_slot(struct mm_subpage_owner *owner, + unsigned int index, bool *needs_zero) +{ + struct mm_subpage_pool *pool = owner->pool; + struct mm_subpage *subpage; + + lockdep_assert_held(&pool->lock); + if (pool->unissued == owner) + /* Transfer the supplied reference to the first reservation. */ + pool->unissued = NULL; + else + folio_get(owner->folio); + __set_bit(index, &owner->used); + cow_update_free(pool, BIT(index), -1); + if (needs_zero) + *needs_zero = !test_bit(index, &owner->zeroed); + __clear_bit(index, &owner->zeroed); + if (owner->used == pool->all_slots) + list_del_init(&owner->available); + subpage = &owner->slots[index]; + WRITE_ONCE(subpage->lazyfree, false); + /* The used bit keeps backing alive while data is initialised. */ + VM_BUG_ON(refcount_read(&subpage->refs)); + VM_BUG_ON(atomic_read(&subpage->mapcount)); + return subpage; +} + static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, unsigned long eligible, bool *needs_zero, bool lock_folio) { @@ -996,7 +1045,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-ESHUTDOWN); goto out; } - owner = find_available(pool, eligible); + owner = find_available(pool, eligible, false); if (!owner) { subpage = ERR_PTR(-EAGAIN); goto out; @@ -1029,29 +1078,63 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return ERR_PTR(-EAGAIN); } } - if (pool->unissued == owner) - /* Transfer the supplied reference to the first reservation. */ - pool->unissued = NULL; - else - folio_get(owner->folio); index = __ffs(~owner->used & eligible); - __set_bit(index, &owner->used); - cow_update_free(pool, BIT(index), -1); - if (needs_zero) - *needs_zero = !test_bit(index, &owner->zeroed); - __clear_bit(index, &owner->zeroed); - if (owner->used == pool->all_slots) - list_del_init(&owner->available); - subpage = &owner->slots[index]; - WRITE_ONCE(subpage->lazyfree, false); - /* The used bit keeps backing alive while data is initialised. */ - VM_BUG_ON(refcount_read(&subpage->refs)); - VM_BUG_ON(atomic_read(&subpage->mapcount)); + subpage = reserve_owner_slot(owner, index, needs_zero); out: spin_unlock_irqrestore(&pool->lock, flags); return subpage; } +/* Reserve one exact-offset batch from one owner, with one folio lock. */ +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + struct mm_subpage_owner *owner; + unsigned long flags, zero = 0; + unsigned int i; + bool needs_zero; + + if (!mask || (mask & ~pool->all_slots)) + return -EINVAL; +retry: + spin_lock_irqsave(&pool->lock, flags); + if (pool->closed) { + spin_unlock_irqrestore(&pool->lock, flags); + return -ESHUTDOWN; + } + owner = find_available(pool, mask, true); + if (!owner) { + spin_unlock_irqrestore(&pool->lock, flags); + return -EAGAIN; + } + if (!folio_trylock(owner->folio)) { + /* Parallel faults use another owner rather than queue behind it. */ + spin_unlock_irqrestore(&pool->lock, flags); + cond_resched(); + goto retry; + } + if (folio_test_swapcache(owner->folio) || folio_test_writeback(owner->folio) || + !cow_owner_eligible(pool, owner->folio)) { + folio_unlock(owner->folio); + spin_unlock_irqrestore(&pool->lock, flags); + return -EAGAIN; + } + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + slots[i] = reserve_owner_slot(owner, i, &needs_zero); + if (needs_zero) + zero |= BIT(i); + } + spin_unlock_irqrestore(&pool->lock, flags); + for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { + if (zero & BIT(i)) + clear_user_subpage_range(&owner->folio->page, + i << pool->shift, 1U << pool->shift); + /* Zeroing must precede publication, including reused slots. */ + refcount_set_release(&slots[i]->refs, 1); + } + return 0; +} + static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 4b7c4cf17006d2..083ce9dc56bfdb 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -7,12 +7,14 @@ CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-numa-balancing-probe +TEST_GEN_PROGS_EXTENDED += anon-hole-refill anon-hole-refill-race anon-hole-refill-reclaim +TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-reclaim TEST_FILES += compat-script runtime-app.py include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool: LDLIBS += -pthread +$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill b/tools/testing/selftests/arm64/user4k/README.hole-refill new file mode 100644 index 00000000000000..e547a6bfc05ebe --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill @@ -0,0 +1,29 @@ +Anonymous hole-refill regression +================================ + +Run anon-hole-refill through the platform's 4K userspace ABI launcher on a +kernel with a native 16K page size. The test does not change its page ABI. + + ./anon-hole-refill + sudo ./anon-hole-refill --native-16k + +The optional --native-16k argument enables physical density checks and requires +visible pagemap PFNs (CAP_SYS_ADMIN). Without it, four density results are +skipped. Do not use it with a different native page size. Pagemap must be +readable even without privileged PFNs: present bits verify neighbor prefault. + +The test owns 16 MiB of logical data. It discards individual 4K leaves and +writes before reading them, exercising missing-PTE refill rather than zero-page +COW. It checks zero initialization, all retained data and neighboring mappings. +Two phased rounds check slot reuse. A two-hole case checks that one fault +prefaults both missing neighbors before the second is accessed. + +Initial backing must be within one native page of 16 MiB. Final backing after +each refill sequence must be at most 20 MiB, allowing bounded pool slack. The +old fresh-private-folio path consumes 64 MiB after single-hole refill rounds +and 32 MiB after the paired-hole sequence. Temporary coexistence of original +and replacement backing is allowed; the test does not require 16 MiB at every +intermediate step. + +Expect eight TAP results. A 30-second alarm bounds the test. It does not fork, +induce pressure or test concurrent faults; see the companion lifetime tests. diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill-race b/tools/testing/selftests/arm64/user4k/README.hole-refill-race new file mode 100644 index 00000000000000..96f2994a4765b0 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill-race @@ -0,0 +1,33 @@ +Concurrent hole-refill lifetime test +=================================== + + sudo ./anon-hole-refill-race + +Run with a 4K userspace ABI on the native 16K kernel. CAP_SYS_ADMIN is required +to distinguish real swap entries from migration and poison markers. The target +configuration has MAX_SWAPFILES=27; types 27 through 31 are excluded and counted +separately. + +Four workers own disjoint 16K virtual windows in a 16 MiB region. Two phased +rounds discard and refill individual 4K leaves. Barriers synchronize quarter +phases, exercising concurrent shared-pool reservations without a userspace data +race. Every refill checks zeroing and unchanged neighbors. This covers pool +contention across windows, not simultaneous PTE changes within one window. + +After joining workers and verifying the region, the allocating process forks a +survivor, modifies one native window privately and exits. A supervisor waits +for that exit before releasing the survivor. The survivor verifies original +contents, rewrites all leaf tags, and verifies them again. This checks that +backing survives its original allocating mm and preserves fork isolation. + +The survivor then applies MADV_COLD/PAGEOUT only to its own region, at most 20 +times with 10 ms between attempts. At least one real swapped leaf and complete +byte-exact readback are required for the reclaim result. The reported count +states coverage; one swapped leaf is not broad reclaim qualification. These +advice operations can skip nonlinear pooled folios because their mappings are +not contiguous. Use README.hole-refill-reclaim for broader controlled coverage. + +Expect three TAP passes and no skips or unexpected signals. The supervisor has +a 35-second alarm and children have 30-second alarms. Failure or timeout kills +the dedicated process group; the supervisor reaps surviving descendants. An +external runner should also bound the entire fixture cgroup. diff --git a/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim b/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim new file mode 100644 index 00000000000000..6ede9503192b8b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.hole-refill-reclaim @@ -0,0 +1,36 @@ +Controlled reclaim of hole-refill backing +======================================== + + sudo ./anon-hole-refill-reclaim --external-reclaim + +This separate fixture repeats the concurrent allocation and owner-exit checks +in README.hole-refill-race. It requires the same page ABI, native granule and +visible swap metadata. After the survivor rewrites and verifies its region, it +prints: + + RECLAIM_READY pid= base= bytes=16777216 owner_mm_exited=1 + +It then waits up to 20 seconds for byte r on stdin. Place only the fixture and +its descendants in a fresh cgroup, with the controller outside that cgroup. +The controller may make at most three writes of "24M swappiness=200" to that +cgroup's memory.reclaim. Leave memory.max/high unchanged and do not reclaim +other cgroups or create global pressure. Bound controller work within the +20-second window. + +Record memory.current, memory.swap.current, memory.stat and memory.events, plus +present and valid-swap PTE counts for the reported region. EAGAIN or a partial +reclaim result does not determine success: actual coverage does. Send r after +the bounded attempts, even if coverage is below the required threshold. + +Before reading data back, the survivor snapshots pagemap. It requires at least +3072 of 4096 leaves (75%) to contain real swap entries and verifies every byte +after refault. Insufficient coverage fails this qualification rather than +skipping it. Expect three TAP passes, no skips and no unexpected signals or +kernel warnings. Supervisor and child alarms are 35 and 30 seconds; the +fixture's dedicated process group is killed on failure. + +MADV_COLD/PAGEOUT compares native folio mapcount with a contiguous PTE batch. +Nonlinear pooled quarters can belong to the same requested region while +occupying separate virtual locations, so those advice operations skip them. +This fixture exercises ordinary cgroup reclaim and nonlinear reverse mapping +without removing the advice path's protection for other mappings. diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c new file mode 100644 index 00000000000000..965141e52e6f77 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill-race.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Bounded partial-window sharing, original-owner exit and real pageout. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WORKERS 4 +#define ROUNDS 2 + +struct result { + bool concurrent, lifetime, reclaimed; + size_t swapped, non_swap_markers; + unsigned int attempts; +}; + +static unsigned char *data; +static pthread_barrier_t barrier; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int signal_number) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + signal_number); +} + +static unsigned char tag(size_t leaf, unsigned int generation) +{ + unsigned char value = leaf % 251 + 1; + + if (generation == 1 && leaf < 4) + value ^= 0xa5; + else if (generation == 2) + value ^= 0x5a; + return value; +} + +static bool verify_leaf(size_t leaf, unsigned int generation) +{ + size_t byte; + + if (data[leaf * LEAF] != tag(leaf, generation)) + return false; + for (byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + return true; +} + +static bool verify(unsigned int generation) +{ + size_t leaf; + + for (leaf = 0; leaf < LEAVES; leaf++) + if (!verify_leaf(leaf, generation)) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg, round, quarter, q; + size_t group, leaf, byte; + bool ok = true; + + for (round = 0; round < ROUNDS; round++) { + for (quarter = 0; quarter < 4; quarter++) { + pthread_barrier_wait(&barrier); + for (group = id; group < BYTES / NATIVE; group += WORKERS) { + unsigned char *p; + + leaf = group * 4 + quarter; + p = data + leaf * LEAF; + if (madvise(p, LEAF, MADV_DONTNEED)) { + ok = false; + continue; + } + /* A write fault, without a preceding zero-page read. */ + *(volatile unsigned char *)p = tag(leaf, 0); + for (byte = 1; byte < LEAF; byte++) + if (p[byte]) + ok = false; + memset(p + 1, 0x5a, LEAF - 1); + /* This worker owns the entire native virtual window. */ + for (q = 0; q < 4; q++) + ok &= verify_leaf(group * 4 + q, 0); + } + } + } + return (void *)(uintptr_t)!ok; +} + +static size_t swapped_leaves(size_t *markers) +{ + uint64_t entries[LEAVES]; + size_t i, count = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return SIZE_MAX; + n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != sizeof(entries)) + return SIZE_MAX; + *markers = 0; + for (i = 0; i < LEAVES; i++) { + if ((entries[i] & (UINT64_C(1) << 63)) || + !(entries[i] & (UINT64_C(1) << 62))) + continue; + /* Audited target config:27..31 are poison/migration/markers. */ + if ((entries[i] & 31) < 27) + count++; + else + (*markers)++; + } + return count; +} + +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void owner(int go, int result_fd) +{ + struct result result = {}; + pthread_t threads[WORKERS]; + void *mapping, *status; + unsigned int i; + bool advice_ok = true, readback_ok; + size_t leaf; + pid_t survivor; + char token; + + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 0); + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + _exit(2); + for (i = 0; i < WORKERS; i++) + if (pthread_create(&threads[i], NULL, worker, (void *)(uintptr_t)i)) + _exit(2); + result.concurrent = true; + for (i = 0; i < WORKERS; i++) { + if (pthread_join(threads[i], &status)) + _exit(2); + result.concurrent &= !status; + } + pthread_barrier_destroy(&barrier); + result.concurrent &= verify(0); + if (!result.concurrent) + _exit(3); + survivor = fork(); + if (survivor < 0) + _exit(2); + if (survivor) { + /* The survivor must retain the original contents across this COW. */ + for (leaf = 0; leaf < 4; leaf++) + data[leaf * LEAF] = tag(leaf, 1); + _exit(verify(1) ? 0 : 3); + } + alarm(30); + /* Supervisor waits for the original allocator's mm to disappear first. */ + if (read(go, &token, 1) != 1 || token != 'g') + _exit(2); + close(go); + result.lifetime = verify(0); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 2); + result.lifetime &= verify(2); + /* No memory.max reduction, memory.reclaim or unrelated memory pressure. */ + for (i = 0; i < 20; i++) { + result.attempts = i + 1; + if (madvise(data, BYTES, MADV_COLD) || madvise(data, BYTES, MADV_PAGEOUT)) { + advice_ok = false; + break; + } + usleep(10000); + result.swapped = swapped_leaves(&result.non_swap_markers); + if (result.swapped == SIZE_MAX) + break; + if (result.swapped == LEAVES) + break; + } + readback_ok = verify(2); + result.reclaimed = advice_ok && result.swapped && + result.swapped != SIZE_MAX && readback_ok; + if (write(result_fd, &result, sizeof(result)) != sizeof(result)) + _exit(2); + close(result_fd); + munmap(mapping, BYTES + NATIVE); + _exit(result.concurrent && result.lifetime && result.reclaimed ? 0 : 3); +} + +static bool visible_swap_metadata(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + + if (!file) + return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) + break; + fclose(file); + return caps & (1ULL << 21); /* CAP_SYS_ADMIN, needed for swap type bits. */ +} + +int main(void) +{ + struct result result = {}; + int go[2], results[2]; + pid_t child; + bool owner_ok, survivor_ok = false; + ssize_t n = 0; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (!visible_swap_metadata()) + ksft_exit_skip("CAP_SYS_ADMIN required to distinguish real swap entries\n"); + signal(SIGPIPE, SIG_IGN); + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + alarm(35); + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(go) || pipe(results)) + ksft_exit_fail_msg("setup: %s\n", strerror(errno)); + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + if (setpgid(0, 0)) + _exit(2); + close(go[1]); + close(results[0]); + owner(go[0], results[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) + stop_fixture(SIGTERM); + close(go[0]); + close(results[1]); + owner_ok = wait_ok(child); + if (owner_ok && write(go[1], "g", 1) == 1) { + n = read(results[0], &result, sizeof(result)); + survivor_ok = wait_ok(-1); + } + close(go[1]); + close(results[0]); + if (!owner_ok || !survivor_ok) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) + ; + } + fixture_group = 0; + alarm(0); + ksft_set_plan(3); + ksft_test_result(owner_ok && n == sizeof(result) && result.concurrent, + "four-worker partial-hole refill zeroing and neighbors\n"); + ksft_test_result(owner_ok && n == sizeof(result) && result.lifetime, + "original pool owner exits; descendant fork isolation/readwrite\n"); + ksft_print_msg("pageout attempts=%u real swapped=%zu/%lu excluded markers=%zu\n", + result.attempts, result.swapped, LEAVES, result.non_swap_markers); + ksft_test_result(survivor_ok && result.reclaimed, + "actual bounded pageout and full survivor readback\n"); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c new file mode 100644 index 00000000000000..4b15b076b0c1c2 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill-reclaim.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Opt-in own-cgroup reclaim of shared partial-hole backing after owner exit. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WORKERS 4 +#define ROUNDS 2 + +struct result { + bool concurrent, lifetime, reclaimed; + size_t swapped, non_swap_markers; + unsigned int attempts; +}; + +static unsigned char *data; +static pthread_barrier_t barrier; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int signal_number) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + signal_number); +} + +static unsigned char tag(size_t leaf, unsigned int generation) +{ + unsigned char value = leaf % 251 + 1; + + if (generation == 1 && leaf < 4) + value ^= 0xa5; + else if (generation == 2) + value ^= 0x5a; + return value; +} + +static bool verify_leaf(size_t leaf, unsigned int generation) +{ + size_t byte; + + if (data[leaf * LEAF] != tag(leaf, generation)) + return false; + for (byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + return true; +} + +static bool verify(unsigned int generation) +{ + size_t leaf; + + for (leaf = 0; leaf < LEAVES; leaf++) + if (!verify_leaf(leaf, generation)) + return false; + return true; +} + +static void *worker(void *arg) +{ + unsigned int id = (uintptr_t)arg, round, quarter, q; + size_t group, leaf, byte; + bool ok = true; + + for (round = 0; round < ROUNDS; round++) { + for (quarter = 0; quarter < 4; quarter++) { + pthread_barrier_wait(&barrier); + for (group = id; group < BYTES / NATIVE; group += WORKERS) { + unsigned char *p; + + leaf = group * 4 + quarter; + p = data + leaf * LEAF; + if (madvise(p, LEAF, MADV_DONTNEED)) { + ok = false; + continue; + } + /* A write fault, without a preceding zero-page read. */ + *(volatile unsigned char *)p = tag(leaf, 0); + for (byte = 1; byte < LEAF; byte++) + if (p[byte]) + ok = false; + memset(p + 1, 0x5a, LEAF - 1); + /* This worker owns the entire native virtual window. */ + for (q = 0; q < 4; q++) + ok &= verify_leaf(group * 4 + q, 0); + } + } + } + return (void *)(uintptr_t)!ok; +} + +static size_t swapped_leaves(size_t *markers) +{ + uint64_t entries[LEAVES]; + size_t i, count = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + ssize_t n; + + if (fd < 0) + return SIZE_MAX; + n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != sizeof(entries)) + return SIZE_MAX; + *markers = 0; + for (i = 0; i < LEAVES; i++) { + if ((entries[i] & (UINT64_C(1) << 63)) || + !(entries[i] & (UINT64_C(1) << 62))) + continue; + /* Audited target config:27..31 are poison/migration/markers. */ + if ((entries[i] & 31) < 27) + count++; + else + (*markers)++; + } + return count; +} + +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + + do { + got = waitpid(pid, &status, 0); + } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && !WEXITSTATUS(status); +} + +static void owner(int go, int result_fd) +{ + struct result result = {}; + pthread_t threads[WORKERS]; + void *mapping, *status; + unsigned int i; + bool readback_ok; + struct pollfd control = { .fd = STDIN_FILENO, .events = POLLIN }; + size_t leaf; + pid_t survivor; + char token; + + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 0); + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + _exit(2); + for (i = 0; i < WORKERS; i++) + if (pthread_create(&threads[i], NULL, worker, (void *)(uintptr_t)i)) + _exit(2); + result.concurrent = true; + for (i = 0; i < WORKERS; i++) { + if (pthread_join(threads[i], &status)) + _exit(2); + result.concurrent &= !status; + } + pthread_barrier_destroy(&barrier); + result.concurrent &= verify(0); + if (!result.concurrent) + _exit(3); + survivor = fork(); + if (survivor < 0) + _exit(2); + if (survivor) { + /* The survivor must retain the original contents across this COW. */ + for (leaf = 0; leaf < 4; leaf++) + data[leaf * LEAF] = tag(leaf, 1); + _exit(verify(1) ? 0 : 3); + } + alarm(30); + /* Supervisor waits for the original allocator's mm to disappear first. */ + if (read(go, &token, 1) != 1 || token != 'g') + _exit(2); + close(go); + result.lifetime = verify(0); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, 2); + result.lifetime &= verify(2); + /* Controller acts only on this fixture's cgroup. No pressure here. */ + printf("RECLAIM_READY pid=%d base=%p bytes=%lu owner_mm_exited=1\n", + getpid(), data, BYTES); + fflush(stdout); + if (poll(&control, 1, 20000) != 1 || + read(STDIN_FILENO, &token, 1) != 1 || token != 'r') + _exit(2); + result.attempts = 1; /* One post-controller pagemap snapshot. */ + result.swapped = swapped_leaves(&result.non_swap_markers); + readback_ok = verify(2); + result.reclaimed = result.swapped != SIZE_MAX && + result.swapped >= LEAVES * 3 / 4 && readback_ok; + + if (write(result_fd, &result, sizeof(result)) != sizeof(result)) + _exit(2); + close(result_fd); + munmap(mapping, BYTES + NATIVE); + _exit(result.concurrent && result.lifetime && result.reclaimed ? 0 : 3); +} + +static bool visible_swap_metadata(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + + if (!file) + return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) + break; + fclose(file); + return caps & (1ULL << 21); /* CAP_SYS_ADMIN, needed for swap type bits. */ +} + +int main(int argc, char **argv) +{ + struct result result = {}; + int go[2], results[2]; + pid_t child; + bool owner_ok, survivor_ok = false; + ssize_t n = 0; + + ksft_print_header(); + fflush(stdout); + if (argc != 2 || strcmp(argv[1], "--external-reclaim")) + ksft_exit_skip("Needs explicit --external-reclaim and a bounded own-cgroup controller\n"); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (!visible_swap_metadata()) + ksft_exit_skip("CAP_SYS_ADMIN required to distinguish real swap entries\n"); + signal(SIGPIPE, SIG_IGN); + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + alarm(35); + if (prctl(PR_SET_CHILD_SUBREAPER, 1) || pipe(go) || pipe(results)) + ksft_exit_fail_msg("setup: %s\n", strerror(errno)); + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + if (setpgid(0, 0)) + _exit(2); + close(go[1]); + close(results[0]); + owner(go[0], results[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) + stop_fixture(SIGTERM); + close(go[0]); + close(results[1]); + owner_ok = wait_ok(child); + if (owner_ok && write(go[1], "g", 1) == 1) { + n = read(results[0], &result, sizeof(result)); + survivor_ok = wait_ok(-1); + } + close(go[1]); + close(results[0]); + if (!owner_ok || !survivor_ok) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) + ; + } + fixture_group = 0; + alarm(0); + ksft_set_plan(3); + ksft_test_result(owner_ok && n == sizeof(result) && result.concurrent, + "four-worker partial-hole refill zeroing and neighbors\n"); + ksft_test_result(owner_ok && n == sizeof(result) && result.lifetime, + "original pool owner exits; descendant fork isolation/readwrite\n"); + ksft_print_msg("post-controller snapshots=%u real swapped=%zu/%lu excluded markers=%zu\n", + result.attempts, result.swapped, LEAVES, result.non_swap_markers); + ksft_test_result(survivor_ok && result.reclaimed, + "own-cgroup reclaim swaps at least75percent, full survivor readback\n"); + ksft_finished(); +} diff --git a/tools/testing/selftests/arm64/user4k/anon-hole-refill.c b/tools/testing/selftests/arm64/user4k/anon-hole-refill.c new file mode 100644 index 00000000000000..e2d71e5cce602e --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/anon-hole-refill.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Missing-PTE refill must retain contents, faultaround and native density. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) + +static unsigned char *data; +static int pm = -1; +static bool density; + +static unsigned char tag(size_t leaf) +{ + return leaf % 251 + 1; +} + +static bool refill(size_t leaf) +{ + unsigned char *p = data + leaf * LEAF; + size_t i; + + /* Write first: a read would test zero-page COW instead of hole refill. */ + *(volatile unsigned char *)p = tag(leaf); + for (i = 1; i < LEAF; i++) + if (p[i]) + return false; + memset(p + 1, 0x5a, LEAF - 1); + return true; +} + +static bool contents(void) +{ + size_t leaf, i; + + for (leaf = 0; leaf < LEAVES; leaf++) { + if (data[leaf * LEAF] != tag(leaf)) + return false; + for (i = 1; i < LEAF; i++) + if (data[leaf * LEAF + i] != 0x5a) + return false; + } + return true; +} + +static int cmp(const void *a, const void *b) +{ + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + + return (x > y) - (x < y); +} + +static int native_bytes(size_t *bytes) +{ + uint64_t entries[LEAVES]; + size_t i, unique = 0; + off_t offset = (uintptr_t)data / LEAF * sizeof(uint64_t); + + if (pm < 0 || pread(pm, entries, sizeof(entries), offset) != sizeof(entries)) + return -1; + for (i = 0; i < LEAVES; i++) { + uint64_t pfn = entries[i] & ((UINT64_C(1) << 55) - 1); + + if (!(entries[i] & (UINT64_C(1) << 63)) || !pfn) + return -1; + entries[i] = pfn / (NATIVE / LEAF); + } + qsort(entries, LEAVES, sizeof(*entries), cmp); + for (i = 0; i < LEAVES; i++) + unique += !i || entries[i] != entries[i - 1]; + *bytes = unique * NATIVE; + return 0; +} + +static void check_density(const char *stage, size_t limit) +{ + size_t bytes; + + if (!density) { + ksft_test_result_skip("%s physical density (needs --native-16k and visible PFNs)\n", stage); + return; + } + if (native_bytes(&bytes)) { + ksft_test_result_fail("%s pagemap unavailable, swapped or hidden\n", stage); + return; + } + ksft_print_msg("%s logical=%lu native=%zu limit=%zu\n", stage, BYTES, bytes, limit); + ksft_test_result(bytes <= limit, "%s physical density\n", stage); +} + +static bool phased_refill(void) +{ + unsigned int quarter; + size_t leaf; + + for (quarter = 0; quarter < 4; quarter++) { + for (leaf = quarter; leaf < LEAVES; leaf += 4) + if (madvise(data + leaf * LEAF, LEAF, MADV_DONTNEED) || !refill(leaf)) + return false; + if (!contents()) + return false; + } + return true; +} + +static bool paired_refill(void) +{ + unsigned int parity; + size_t leaf; + + for (parity = 0; parity < 2; parity++) { + for (leaf = parity; leaf < LEAVES; leaf += 4) { + uint64_t entry; + + if (madvise(data + leaf * LEAF, LEAF, MADV_DONTNEED) || + madvise(data + (leaf + 2) * LEAF, LEAF, MADV_DONTNEED) || + !refill(leaf)) + return false; + /* Present is visible without privileged PFNs. Prove the other + * missing neighbor was prefaulted before touching it ourselves. + */ + if (pm < 0 || pread(pm, &entry, sizeof(entry), + ((uintptr_t)data / LEAF + leaf + 2) * 8) != sizeof(entry) || + !(entry & (UINT64_C(1) << 63)) || !refill(leaf + 2)) + return false; + } + } + return contents(); +} + +int main(int argc, char **argv) +{ + void *mapping; + size_t leaf; + + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("Run through the established 4K ABI launcher\n"); + if (argc == 2 && !strcmp(argv[1], "--native-16k")) + density = true; + else if (argc != 1) + ksft_exit_fail_msg("Usage: %s [--native-16k]\n", argv[0]); + alarm(30); + mapping = mmap(NULL, BYTES + NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + ksft_exit_fail_msg("mmap: %s\n", strerror(errno)); + data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + pm = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + memset(data, 0x5a, BYTES); + for (leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf); + ksft_set_plan(8); + ksft_test_result(contents(), "initial dense content\n"); + check_density("initial dense", BYTES + NATIVE); + ksft_test_result(phased_refill(), "phased hole refill zeroing and neighbors\n"); + /* Allow bounded pool-bucket slop, but reject the old 64MiB outcome. */ + check_density("phased refill", BYTES + (4UL << 20)); + ksft_test_result(phased_refill(), "reused-slot refill zeroing and neighbors\n"); + check_density("reused-slot refill", BYTES + (4UL << 20)); + ksft_test_result(paired_refill(), "two-hole mask retains missing-neighbor prefault\n"); + check_density("two-hole refill", BYTES + (4UL << 20)); + munmap(mapping, BYTES + NATIVE); + if (pm >= 0) + close(pm); + ksft_finished(); +} From 90d2713f7bd04759e297efbf49ad87cc8cde9534 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:32:47 -0400 Subject: [PATCH 298/631] mm: pack single-slot COW across native page quarters A workload writing the same logical quarter in each native window used 16 MiB of backing for 4 MiB of changed data because COW only reused a matching physical quarter. Allow eligible pooled single-slot copies to use any free quarter, retaining exact-offset private fallbacks and dense anonymous batching. Prepare nonlinear reverse mappings before taking PTE locks, including anonymous hole and swap-in publication. Restore swapoff from a locked swap-PTE snapshot and recheck it before installation. Use actual source and destination slot offsets for file and anonymous copies. On Neo, the same-quarter case now uses 4.09375 MiB with byte-exact parent isolation. Hardware checks cover same-native-folio COW, typed GUP and shifted mremap, complete owned-cgroup reclaim/readback, selected-page swapoff, native/4K ABI, hole refill, GPU quarter refault and normal game correctness. Maintained regression binaries match those tested exactly. Ongoing game memory pressure and other first-write allocation paths are separate unresolved issues; this change makes no FPS claim. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6748c940b0e8fd5287a85ad9bda11e254c8901cf) --- include/linux/mm_subpage.h | 7 + mm/memory.c | 58 +-- mm/swapfile.c | 34 +- mm/user-subpage.c | 32 ++ .../testing/selftests/arm64/user4k/.gitignore | 3 + tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/README.any-quarter | 37 ++ .../arm64/user4k/check-any-quarter.py | 24 ++ .../arm64/user4k/file-cow-consumers.c | 319 +++++++++++++++++ .../selftests/arm64/user4k/same-native.c | 176 +++++++++ .../selftests/arm64/user4k/same-quarter.c | 335 ++++++++++++++++++ 11 files changed, 991 insertions(+), 36 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.any-quarter create mode 100644 tools/testing/selftests/arm64/user4k/check-any-quarter.py create mode 100644 tools/testing/selftests/arm64/user4k/file-cow-consumers.c create mode 100644 tools/testing/selftests/arm64/user4k/same-native.c create mode 100644 tools/testing/selftests/arm64/user4k/same-quarter.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 9a1d33166db30c..e892ae3e70bee2 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -52,6 +52,13 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); +/* COW slots may use any physical quarter; rmap retains the virtual position. */ +bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_alloc_locked(struct mm_subpage_pool *pool); +struct mm_subpage *mm_subpage_copy_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source); +int mm_subpage_prepare_anon_rmap(struct mm_subpage *slot, + struct vm_area_struct *vma, unsigned long address); /* Exact physical-slot mask, one owner and folio lock for the entire batch. * slots is indexed by physical offset >> pool shift; only mask bits are set. * Success returns initialized owned slots; errors leave slots untouched. diff --git a/mm/memory.c b/mm/memory.c index 4006d3cef29ffd..cde03950db1e15 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1364,14 +1364,18 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, if (!pool) return ERR_PTR(-ENOMEM); for (;;) { - slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : - mm_subpage_alloc_at_locked(pool, offset); + if (*pooled) + slot = source ? mm_subpage_copy_locked(pool, source) : + mm_subpage_alloc_locked(pool); + else + slot = source ? mm_subpage_copy_at_locked(pool, source, offset) : + mm_subpage_alloc_at_locked(pool, offset); if (!IS_ERR(slot) || PTR_ERR(slot) != -EAGAIN) break; folio = NULL; if (*pooled) { folio = cow_subpage_speculative_folio(mm, vma, addr); - if (!folio && mm_subpage_cow_wait_busy_at(pool, offset)) { + if (!folio && mm_subpage_cow_wait_busy(pool)) { cond_resched(); continue; } @@ -1392,15 +1396,16 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, force_private = true; goto retry_private; } - err = mm_subpage_pool_add_folio_at(pool, folio, offset, - *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); + err = *pooled ? mm_subpage_pool_add_folio(pool, folio, + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN) : + mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); if (*pooled && err == -ENOMEM) { - if (mm_subpage_cow_wait_busy_at(pool, offset)) { + if (mm_subpage_cow_wait_busy(pool)) { folio_put(folio); cond_resched(); continue; } - err = mm_subpage_pool_add_folio_at(pool, folio, offset, GFP_KERNEL); + err = mm_subpage_pool_add_folio(pool, folio, GFP_KERNEL); } if (err) { folio_put(folio); @@ -4639,7 +4644,6 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) struct mmu_notifier_range range; bool copied = false, unshare = vmf->flags & FAULT_FLAG_UNSHARE; bool force_private = false, pooled, fresh, retry_private = false; - unsigned int offset; vm_fault_t ret = VM_FAULT_SIGBUS; pte_t entry; int err; @@ -4689,7 +4693,6 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) ret = vmf_anon_prepare(vmf); if (ret) goto out; - offset = vma_page_offset_at(vma, vmf->address).offset; ret = VM_FAULT_OOM; retry_copy: dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); @@ -4701,7 +4704,7 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) } new = mm_subpage_folio(dest); if (old && !source) { - err = copy_user_subpage_range(&new->page, offset, vmf->page, + err = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), vmf->page, pte_phys_mm(mm, vmf->orig_pte) & ~PAGE_MASK, mm_page_size(mm)); if (err) { @@ -4726,13 +4729,11 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) * slot identity before taking PTL, with a fresh-folio fallback on ENOMEM. */ fresh = !mm_subpage_anon_root(dest); - if (!fresh) { - err = mm_subpage_bind_rmap(dest, vma, vmf->address); - if (err) { - ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; - retry_private = pooled && err == -ENOMEM; - goto unlock_folio; - } + err = mm_subpage_prepare_anon_rmap(dest, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) @@ -5552,6 +5553,11 @@ static vm_fault_t do_swap_subpage(struct vm_fault *vmf, swp_entry_t entry) ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; goto unlock_folio; } + err = mm_subpage_prepare_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } folio_throttle_swaprate(folio, GFP_KERNEL); vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); if (!vmf->pte) @@ -6501,17 +6507,15 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) /* Existing owners may need nonlinear identities. Allocate metadata before * PTL and keep the established fresh-folio fallback on metadata ENOMEM. */ - if (!fresh) { - for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { - unsigned long address = start + - ((index - first_offset) << mm_page_shift(mm)); + for_each_set_bit(index, &mask, ARRAY_SIZE(slots)) { + unsigned long address = start + + ((index - first_offset) << mm_page_shift(mm)); - err = mm_subpage_bind_rmap(slots[index], vma, address); - if (err) { - ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; - retry_private = pooled && err == -ENOMEM; - goto unlock_folio; - } + err = mm_subpage_prepare_anon_rmap(slots[index], vma, address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + retry_private = pooled && err == -ENOMEM; + goto unlock_folio; } } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); diff --git a/mm/swapfile.c b/mm/swapfile.c index e5faba70227c07..c2c49b9fb3f366 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -2453,33 +2453,50 @@ static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte) #ifdef CONFIG_MM_SUBPAGE /* Caller holds the swap-cache folio lock and the VMA's mmap read lock. */ static int unuse_subpage(struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr, swp_entry_t entry, struct folio *folio, - unsigned int offset) + unsigned long addr, swp_entry_t entry, struct folio *folio) { struct mm_struct *mm = vma->vm_mm; + unsigned int offset; struct mm_subpage *slot = NULL; spinlock_t *ptl; - pte_t *ptep, old, pte; + pte_t *ptep, old, snapshot, pte; bool poisoned = PageHWPoison(&folio->page) || !folio_test_uptodate(folio); int ret = 0; if (folio_order(folio) || folio_test_ksm(folio)) return -EOPNOTSUPP; + /* A swap PTE records the physical quarter, independently of the VMA. + * Snapshot under PTL, then allocate metadata while only folio is locked. + */ + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!ptep) + return 0; + snapshot = ptep_get(ptep); + if (!softleaf_is_swap(softleaf_from_pte(snapshot)) || + softleaf_from_pte(snapshot).val != entry.val) { + pte_unmap_unlock(ptep, ptl); + return 0; + } + offset = pte_swp_subpage_offset(snapshot); + pte_unmap_unlock(ptep, ptl); + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(mm))) + return -EINVAL; if (!poisoned) { arch_swap_restore(folio_swap(entry, folio), folio); /* Allocate metadata before taking the PTE spinlock. */ slot = mm_subpage_restore_granule(folio, offset, mm_page_shift(mm), GFP_KERNEL); if (IS_ERR(slot)) return PTR_ERR(slot) == -EAGAIN ? 0 : PTR_ERR(slot); + ret = mm_subpage_prepare_anon_rmap(slot, vma, addr); + if (ret) + goto out; } ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); if (!ptep) goto out; old = ptep_get(ptep); - if (softleaf_from_pte(old).val != entry.val) - goto unlock; - /* A moved swap PTE retains its backing offset, not its VMA offset. */ - if (pte_swp_subpage_offset(old) != offset) + /* Includes quarter and flag changes while metadata was prepared. */ + if (!pte_same(old, snapshot)) goto unlock; if (poisoned) { pte = swp_entry_to_pte(PageHWPoison(&folio->page) ? @@ -2540,8 +2557,7 @@ static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd, #ifdef CONFIG_MM_SUBPAGE if (mm_page_size(vma->vm_mm) < PAGE_SIZE) - return unuse_subpage(vma, pmd, addr, entry, folio, - pte_swp_subpage_offset(orig_pte)); + return unuse_subpage(vma, pmd, addr, entry, folio); #endif swapcache = folio; diff --git a/mm/user-subpage.c b/mm/user-subpage.c index c7876fd34a091b..30fb3669458cb6 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -630,6 +630,22 @@ int mm_subpage_bind_rmap(struct mm_subpage *slot, struct vm_area_struct *vma, return ret; } +/* Preflight publication without making an unexposed owner look published. + * A fresh linear slot needs no sidecar: add_new_anon_rmap will set its anchor. + * Otherwise bind the independent virtual position before any PTE lock. + */ +int mm_subpage_prepare_anon_rmap(struct mm_subpage *slot, + struct vm_area_struct *vma, unsigned long address) +{ + struct mm_subpage_owner *owner = slot->owner; + + VM_BUG_ON_FOLIO(!folio_test_locked(owner->folio), owner->folio); + if (!owner->anon_root && !owner->rmap && + mm_subpage_offset(slot) == vma_page_offset_at(vma, address).offset) + return 0; + return mm_subpage_bind_rmap(slot, vma, address); +} + static bool subpage_move_allowed(struct mm_subpage *slot, bool swap) { lockdep_assert_held(&slot->owner->pool->lock); @@ -872,6 +888,11 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, return false; } +bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool) +{ + return cow_wait_busy(pool, pool->all_slots, false); +} + bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) { return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); @@ -1261,6 +1282,17 @@ struct mm_subpage *mm_subpage_copy_at_locked(struct mm_subpage_pool *pool, return eligible ? subpage_copy(pool, source, eligible, true) : ERR_PTR(-EINVAL); } +struct mm_subpage *mm_subpage_alloc_locked(struct mm_subpage_pool *pool) +{ + return subpage_alloc(pool, pool->all_slots, true); +} + +struct mm_subpage *mm_subpage_copy_locked(struct mm_subpage_pool *pool, + const struct mm_subpage *source) +{ + return subpage_copy(pool, source, pool->all_slots, true); +} + /* The page-extension RCU section also protects the pool until its lock is held. */ static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, unsigned int shift) diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 2e6224b832f98e..86bb27433258e3 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -1,3 +1,6 @@ page-contract swap-fixture-init +same-quarter +file-cow-consumers +same-native diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 083ce9dc56bfdb..0c22734f5ad03d 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -10,6 +10,8 @@ TEST_GEN_FILES += compat-shm-probe compat-perf-probe compat-numa-probe compat-nu TEST_GEN_PROGS_EXTENDED += anon-hole-refill anon-hole-refill-race anon-hole-refill-reclaim TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-reclaim TEST_FILES += compat-script runtime-app.py +TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native +TEST_FILES += README.any-quarter check-any-quarter.py include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S diff --git a/tools/testing/selftests/arm64/user4k/README.any-quarter b/tools/testing/selftests/arm64/user4k/README.any-quarter new file mode 100644 index 00000000000000..305441c8a9db47 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.any-quarter @@ -0,0 +1,37 @@ +Arbitrary-quarter anonymous COW qualification +============================================= + +Run on the audited native16K kernel through an established4K ABI launcher, +with CAP_SYS_ADMIN for visible pagemap PFNs: + userpages 4k ./same-quarter --native-16k > same-quarter.log + python3 check-any-quarter.py same-quarter.log + userpages 4k ./file-cow-consumers --native-16k + userpages 4k ./same-native --native-16k + +same-quarter: two fresh owner+child arms compare4MiB changed data in same versus +balanced virtual quarters of a16MiB dense parent. Exact nativePFN union, no +modified-child alias to parent/unchangedchild, all-byte isolation and histograms. +The binary reports measurements; the checker requires both arms <=6MiB native +for4MiB changed and actual nonzero physical-quarter use in the same-quarter arm. +This catches the old16MiB same-quarter result while allowing bounded pool slack. +30s supervisor/20s child alarms, group/subreaper cleanup, no pressure. + +file-cow-consumers:1MiB owned memfd,two private views; child explicitly READ- +prefaults both views AFTER fork (fork may omit reconstructible file PTEs). +CPU COW, remote typed GUP read-unshare/write-COW each require actual physical/ +logical quarter mismatch and correct file-to-anonymous transition. Then shifted +VA mremap must preserve every actualPFN and byte; post-move GUP and parent/file +isolation are checked.30s supervisor/20s child. Exact stage/byte/PTE diagnostics. +This is typed GUP coverage, not long-term DMA pin or migration qualification. + +same-native:16fresh three-generation trials/16KiB logical payload. B's first COW +is retained by held C while B COWs again. Required distinct4KPFNs sharing native +PFN/4 prove source/destination use one nativefolio; every generation and neighbor +byte stays correct. At least1actual pair is required for PASS; exit77 means +UNEXERCISED, never a coverage pass.20/5/4s supervisor/owner/held-child alarms. + +No fixture changes memory policy, system swap, kernel or GPU state. Native16K +assumption is explicit; do not run density math on an unaudited kernel. Metadata +--selftest modes (where provided) are not kernel-path qualification. UFFD_MOVE +and targeted migration are separate gates; disabled UFFD/single-node systems do +not acquire those qualifications merely by passing these tests. diff --git a/tools/testing/selftests/arm64/user4k/check-any-quarter.py b/tools/testing/selftests/arm64/user4k/check-any-quarter.py new file mode 100644 index 00000000000000..1b71b3922783f5 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/check-any-quarter.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Validate native backing density from the unchanged same-quarter oracle.""" +import json,sys +from pathlib import Path + +def check(path): + rows=[json.loads(x) for x in Path(path).read_text().splitlines() if x.startswith('{')] + assert len(rows)==2, 'Require exactly two measured arms' + arms={x['arm']:x for x in rows} + assert set(arms)=={'same-quarter-zero','balanced'} + for name,r in arms.items(): + assert r['logical_parent_bytes']==16*1024*1024 and r['modified_child_bytes']==4*1024*1024 + assert r['content_ok'] and r['disjoint_ok'] + assert r['modified_parent_native_overlap']==0 and r['modified_unchanged_native_overlap']==0 + assert r['modified_child']['represented_bytes']==4*1024*1024 + n=r['modified_child']['native_bytes'] + assert 4*1024*1024<=n<=6*1024*1024, (name,'native backing outside packed bound',n) + assert sum(arms['same-quarter-zero']['modified_physical_quarter_counts'][1:])>0 + print('PASS same-quarter/balanced native density, actual relocation and isolation') + +if __name__=='__main__': + if len(sys.argv)!=2:sys.exit('usage: check-any-quarter.py same-quarter.log') + try:check(sys.argv[1]) + except (AssertionError,KeyError,ValueError,OSError) as e:sys.exit('FAIL '+str(e)) diff --git a/tools/testing/selftests/arm64/user4k/file-cow-consumers.c b/tools/testing/selftests/arm64/user4k/file-cow-consumers.c new file mode 100644 index 00000000000000..7d2f82500f6374 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/file-cow-consumers.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Owned memfd only: file COW, typed remote GUP and shifted mremap. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define WINDOWS 64UL +#define BYTES (WINDOWS * NATIVE) +#define LEAVES (BYTES / LEAF) +#define PRESENT (UINT64_C(1) << 63) +#define FILE_PAGE (UINT64_C(1) << 61) +#define PFN_MASK ((UINT64_C(1) << 55) - 1) + +struct message { uintptr_t address; uint64_t phase, detail; }; +static volatile sig_atomic_t child_group; +static const char *stage = "startup"; +static bool child_reaped; +static int child_status; +static void report_wait(pid_t pid, int status) +{ + fprintf(stderr, "CHILD_WAIT pid=%ld raw_status=%d exited=%d exit=%d signaled=%d signal=%d\n", + (long)pid, status, WIFEXITED(status), WIFEXITED(status) ? WEXITSTATUS(status) : -1, + WIFSIGNALED(status), WIFSIGNALED(status) ? WTERMSIG(status) : 0); +} + +static void stop(int sig) +{ + if (child_group > 0) kill(-child_group, SIGKILL); + _exit(128 + sig); +} +static bool io_full(int fd, void *buf, size_t n, bool write_it) +{ + char *p = buf; + while (n) { + ssize_t got = write_it ? write(fd, p, n) : read(fd, p, n); + if (got < 0 && errno == EINTR) continue; + if (got <= 0) return false; + p += got; n -= (size_t)got; + } + return true; +} +static unsigned char original(size_t offset) +{ + size_t leaf = offset / LEAF, byte = offset % LEAF; + return (unsigned char)(leaf * 17 + byte * 13 + byte / 256); +} +static bool verify(const unsigned char *data, unsigned kind, size_t gup_a_leaf) +{ + for (size_t at = 0; at < BYTES; at++) { + size_t leaf = at / LEAF, byte = at % LEAF; + unsigned char value = original(at); + if (kind == 1 && leaf % 4 == 3 && byte == 0) value ^= 0x5a; + if (kind == 1 && leaf == gup_a_leaf && byte >= 17 && byte < 49) value = 0xe3; + if (kind == 2 && leaf % 4 == 2 && byte >= 37 && byte < 69) value = 0xc7; + if (data[at] != value) { + fprintf(stderr, "VERIFY_FAIL stage=%s kind=%u offset=0x%zx leaf=%zu byte=%zu expected=%02x actual=%02x\n", + stage, kind, at, leaf, byte, value, data[at]); + return false; + } + } + return true; +} +static bool visible_pfns(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; unsigned long long caps = 0; + if (!f) return false; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) break; + fclose(f); return !!(caps & (1ULL << 21)); +} +static bool entries(pid_t pid, const void *base, uint64_t out[LEAVES]) +{ + char path[64]; snprintf(path, sizeof(path), "/proc/%ld/pagemap", (long)pid); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { fprintf(stderr, "PAGEMAP_OPEN_FAIL stage=%s pid=%ld errno=%d\n", stage, (long)pid, errno); return false; } + ssize_t n = pread(fd, out, LEAVES * 8, (uintptr_t)base / LEAF * 8); + close(fd); + if (n != LEAVES * 8) { fprintf(stderr, "PAGEMAP_READ_FAIL stage=%s bytes=%zd errno=%d\n", stage, n, errno); return false; } + for (size_t i = 0; i < LEAVES; i++) + if (!(out[i] & PRESENT) || !(out[i] & PFN_MASK)) { + fprintf(stderr, "PAGEMAP_ENTRY_FAIL stage=%s pid=%ld leaf=%zu raw=%016llx present=%d file=%d pfn_visible=%d\n", + stage, (long)pid, i, (unsigned long long)out[i], !!(out[i] & PRESENT), + !!(out[i] & FILE_PAGE), !!(out[i] & PFN_MASK)); + return false; + } + return true; +} +static bool quarter_file_state(const uint64_t *map, unsigned quarter, bool file) +{ + for (size_t leaf = quarter; leaf < LEAVES; leaf += 4) + if (!!(map[leaf] & FILE_PAGE) != file) { + fprintf(stderr, "PTE_KIND_FAIL stage=%s leaf=%zu expected_file=%d raw=%016llx\n", + stage, leaf, file, (unsigned long long)map[leaf]); + return false; + } + return true; +} +static unsigned mismatches(const uint64_t *map, unsigned logical_quarter, size_t *first) +{ + unsigned count = 0; + for (size_t leaf = logical_quarter; leaf < LEAVES; leaf += 4) { + if ((map[leaf] & PFN_MASK) % 4 != logical_quarter) { + if (!count && first) *first = leaf; + count++; + } + } + fprintf(stderr, "MISMATCH_WITNESS stage=%s logical_quarter=%u mismatches=%u/%lu\n", + stage, logical_quarter, count, WINDOWS); + return count; +} +/* Does not fault payload or require present entries: proves the fork premise. */ +static bool entry_summary(const void *base, const char *view) +{ + uint64_t map[LEAVES]; unsigned present = 0, file = 0; + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (fd < 0) return false; + ssize_t n = pread(fd, map, sizeof(map), (uintptr_t)base / LEAF * 8); + close(fd); + if (n != sizeof(map)) return false; + for (size_t i = 0; i < LEAVES; i++) { + present += !!(map[i] & PRESENT); + file += !!(map[i] & PRESENT) && !!(map[i] & FILE_PAGE); + } + fprintf(stderr, "PREFLIGHT stage=%s view=%s present=%u file_present=%u leaves=%lu first_raw=%016llx\n", + stage, view, present, file, LEAVES, (unsigned long long)map[0]); + return true; +} +static bool token(int fd, char wanted) +{ + char got; + return io_full(fd, &got, 1, false) && got == wanted; +} +#define CHILD_CHECK(test, name, code) do { stage = name; if (!(test)) { \ + fprintf(stderr, "CHILD_FAIL stage=%s exit=%d errno=%d\n", stage, code, errno); _exit(code); } } while (0) +static void child(unsigned char *a, unsigned char *b, int control, int status) +{ + struct message msg; + uint64_t before[LEAVES], after[LEAVES]; + alarm(20); + CHILD_CHECK(entry_summary(a, "A") && entry_summary(b, "B"), "before-child-prefault", 2); + /* Fork may skip reconstructible file PTEs. Establish them in this mm. */ + CHILD_CHECK(verify(a, 0, SIZE_MAX), "child-prefault-A-original", 3); + CHILD_CHECK(verify(b, 0, SIZE_MAX), "child-prefault-B-original", 3); + CHILD_CHECK(entry_summary(a, "A") && entry_summary(b, "B"), "after-child-prefault", 2); + CHILD_CHECK(entries(getpid(), a, before), "CPU-source-present", 3); + CHILD_CHECK(quarter_file_state(before, 3, true), "CPU-source-file-PTE", 3); + for (size_t window = 0; window < WINDOWS; window++) { + size_t at = (window * 4 + 3) * LEAF; + *(volatile unsigned char *)(a + at) = original(at) ^ 0x5a; + } + CHILD_CHECK(verify(a, 1, SIZE_MAX), "CPU-COW-full-bytes", 3); + CHILD_CHECK(entries(getpid(), a, after), "CPU-destination-present", 3); + CHILD_CHECK(quarter_file_state(after, 3, false), "CPU-destination-anon-PTE", 3); + size_t first = SIZE_MAX; + unsigned cpu_mismatch = mismatches(after, 3, &first); + CHILD_CHECK(cpu_mismatch, "CPU-COW-actual-quarter-mismatch", 5); + msg = (struct message){ .address = first, .phase = 1, .detail = cpu_mismatch }; + CHILD_CHECK(io_full(status, &msg, sizeof(msg), true), "CPU-result-write", 2); + CHILD_CHECK(token(control, 'm'), "wait-mremap-command", 2); + CHILD_CHECK(verify(a, 1, first), "post-GUP-A-full-bytes", 3); + CHILD_CHECK(verify(b, 2, SIZE_MAX), "post-GUP-B-full-bytes", 3); + CHILD_CHECK(entries(getpid(), b, before), "before-mremap-PFNs", 3); + void *reserve = mmap(NULL, BYTES + 2 * NATIVE, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + CHILD_CHECK(reserve != MAP_FAILED, "mremap-reservation", 2); + unsigned long desired = ((uintptr_t)b + LEAF) % NATIVE; + unsigned char *destination = (void *)((((uintptr_t)reserve + NATIVE - 1) & ~(NATIVE - 1)) + desired); + unsigned char *moved = mremap(b, BYTES, BYTES, MREMAP_MAYMOVE | MREMAP_FIXED, destination); + CHILD_CHECK(moved != MAP_FAILED && moved == destination && + (uintptr_t)moved % NATIVE != (uintptr_t)b % NATIVE, "mremap-shifted-VA", 3); + CHILD_CHECK(verify(moved, 2, SIZE_MAX), "mremap-full-bytes", 3); + CHILD_CHECK(entries(getpid(), moved, after), "after-mremap-PFNs", 3); + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + if ((before[leaf] & PFN_MASK) != (after[leaf] & PFN_MASK)) { + fprintf(stderr, "PFN_CHANGE stage=mremap leaf=%zu before=%016llx after=%016llx\n", + leaf, (unsigned long long)before[leaf], (unsigned long long)after[leaf]); + CHILD_CHECK(false, "mremap-preserved-PFNs", 6); + } + } + msg = (struct message){ .address = (uintptr_t)moved, .phase = 2, .detail = desired }; + CHILD_CHECK(io_full(status, &msg, sizeof(msg), true), "mremap-result-write", 2); + CHILD_CHECK(token(control, 'x'), "wait-final-release", 2); + CHILD_CHECK(verify(a, 1, first), "final-A-full-bytes", 3); + CHILD_CHECK(verify(moved, 2, SIZE_MAX), "final-B-full-bytes", 3); + munmap(reserve, BYTES + 2 * NATIVE); munmap(a, BYTES); + close(control); close(status); _exit(0); +} +static bool remote_one(pid_t pid, void *local, void *remote, size_t bytes, bool write_it) +{ + struct iovec l = { .iov_base = local, .iov_len = bytes }; + struct iovec r = { .iov_base = remote, .iov_len = bytes }; + ssize_t n = write_it ? process_vm_writev(pid, &l, 1, &r, 1, 0) : + process_vm_readv(pid, &l, 1, &r, 1, 0); + return n == (ssize_t)bytes; +} +static bool remote_quarters(pid_t pid, void *local, unsigned char *remote, + unsigned quarter, size_t offset, size_t bytes, bool write_it) +{ + struct iovec l = { .iov_base = local, .iov_len = WINDOWS * bytes }; + struct iovec r[WINDOWS]; + for (size_t window = 0; window < WINDOWS; window++) + r[window] = (struct iovec){ .iov_base = remote + (window * 4 + quarter) * LEAF + offset, .iov_len = bytes }; + ssize_t n = write_it ? process_vm_writev(pid, &l, 1, r, WINDOWS, 0) : + process_vm_readv(pid, &l, 1, r, WINDOWS, 0); + return n == (ssize_t)(WINDOWS * bytes); +} +static bool wait_ok(pid_t pid) +{ + int status; pid_t n; + do { n = waitpid(pid, &status, 0); } while (n < 0 && errno == EINTR); + if (n == pid) { child_reaped = true; child_status = status; report_wait(pid, status); } + else fprintf(stderr, "CHILD_WAIT_ERROR pid=%ld errno=%d\n", (long)pid, errno); + return n == pid && WIFEXITED(status) && !WEXITSTATUS(status); +} +#define CHECK(test, message) do { stage = message; if (!(test)) { fprintf(stderr, "FAIL: %s (errno=%d)\n", message, errno); goto cleanup; } } while (0) +int main(int argc, char **argv) +{ + unsigned char *buffer = NULL, *a = MAP_FAILED, *b = MAP_FAILED; + int fd = -1, control[2] = {-1,-1}, status[2] = {-1,-1}; + pid_t pid = -1; bool ok = false; + uint64_t map[LEAVES]; struct message msg; + unsigned cpu_mismatch = 0, read_mismatch = 0, write_mismatch = 0; + size_t first_a = SIZE_MAX, first_b = SIZE_MAX; + if (argc != 2 || strcmp(argv[1], "--native-16k")) { + fprintf(stderr, "usage: %s --native-16k\n", argv[0]); return 2; + } + if (getauxval(AT_PAGESZ) != LEAF || !visible_pfns()) { + fprintf(stderr, "SKIP: established4K launcher/native16K kernel/visiblePFNs required\n"); return 4; + } + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop); signal(SIGTERM, stop); signal(SIGINT, stop); + alarm(30); + buffer = malloc(BYTES); + CHECK(buffer, "buffer allocation"); + for (size_t i = 0; i < BYTES; i++) buffer[i] = original(i); + fd = memfd_create("own-file-cow-consumers", MFD_CLOEXEC); + CHECK(fd >= 0 && !ftruncate(fd, BYTES), "owned memfd"); + CHECK(pwrite(fd, buffer, BYTES, 0) == BYTES, "initialize memfd"); + a = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + b = mmap(NULL, BYTES, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); + CHECK(a != MAP_FAILED && b != MAP_FAILED, "private file views"); + /* Full reads establish file PTEs before COW and typed pin operations. */ + CHECK(verify(a, 0, SIZE_MAX) && verify(b, 0, SIZE_MAX), "prefault originals"); + CHECK(!pipe(control) && !pipe(status), "control pipes"); + pid = fork(); CHECK(pid >= 0, "fork"); + if (!pid) { + CHILD_CHECK(!setpgid(0, 0), "set-child-group", 2); + close(control[1]); close(status[0]); close(fd); + child(a, b, control[0], status[1]); + } + child_group = pid; + if (setpgid(pid, pid) && errno != EACCES && errno != ESRCH) CHECK(false, "child group"); + close(control[0]); control[0] = -1; close(status[1]); status[1] = -1; + bool received = io_full(status[0], &msg, sizeof(msg), false); + if (!received) { fprintf(stderr, "RESULT_EOF stage=CPU-COW; waiting for exact child status\n"); (void)wait_ok(pid); } + CHECK(received && msg.phase == 1, "CPU COW result"); + cpu_mismatch = msg.detail; first_a = msg.address; + CHECK(cpu_mismatch && first_a < LEAVES && first_a % 4 == 3, "actual CPU COW offset mismatch"); + CHECK(remote_one(pid, buffer, a + first_a * LEAF, LEAF, false), "GUP read of mismatched anonymous slot"); + for (size_t i = 0; i < LEAF; i++) + CHECK(buffer[i] == (unsigned char)(original(first_a * LEAF + i) ^ (i == 0 ? 0x5a : 0)), "GUP read bytes"); + memset(buffer, 0xe3, 32); + CHECK(remote_one(pid, buffer, a + first_a * LEAF + 17, 32, true), "GUP write of mismatched anonymous slot"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 3, true), "read-pin sources remain file PTEs"); + CHECK(remote_quarters(pid, buffer, b, 3, 0, LEAF, false), "typed read-pin file unshare"); + for (size_t w = 0; w < WINDOWS; w++) + for (size_t i = 0; i < LEAF; i++) + CHECK(buffer[w * LEAF + i] == original((w * 4 + 3) * LEAF + i), "read-pin copied bytes"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 3, false), "read-pin produced anonymous slots"); + read_mismatch = mismatches(map, 3, &first_b); + CHECK(read_mismatch && first_b < LEAVES, "actual read-pin file COW offset mismatch"); + CHECK(quarter_file_state(map, 2, true), "write-pin sources remain file PTEs"); + memset(buffer, 0xc7, WINDOWS * 32); + CHECK(remote_quarters(pid, buffer, b, 2, 37, 32, true), "typed write-pin file COW"); + CHECK(entries(pid, b, map) && quarter_file_state(map, 2, false), "write-pin produced anonymous slots"); + write_mismatch = mismatches(map, 2, NULL); + CHECK(write_mismatch, "actual write-pin file COW offset mismatch"); + char command = 'm'; CHECK(io_full(control[1], &command, 1, true), "mremap command"); + received = io_full(status[0], &msg, sizeof(msg), false); + if (!received) { fprintf(stderr, "RESULT_EOF stage=mremap; waiting for exact child status\n"); (void)wait_ok(pid); } + CHECK(received && msg.phase == 2, "mremap content/PFN result"); + unsigned char *moved = (void *)msg.address; + CHECK(remote_one(pid, buffer, moved + first_b * LEAF, LEAF, false), "GUP read after shifted mremap"); + for (size_t i = 0; i < LEAF; i++) CHECK(buffer[i] == original(first_b * LEAF + i), "moved GUP bytes"); + CHECK(verify(a, 0, SIZE_MAX) && verify(b, 0, SIZE_MAX), "parent private views unchanged"); + CHECK(pread(fd, buffer, BYTES, 0) == BYTES && verify(buffer, 0, SIZE_MAX), "original memfd byte exact"); + command = 'x'; CHECK(io_full(control[1], &command, 1, true), "child release"); + CHECK(wait_ok(pid), "child final integrity"); pid = -1; child_group = 0; + printf("{\"file_bytes\":%lu,\"cpu_cow_mismatches\":%u,\"read_pin_cow_mismatches\":%u," + "\"write_pin_cow_mismatches\":%u,\"targets_per_path\":%lu,\"mremap_shifted_va_quarter\":true," + "\"mremap_preserved_pfns\":true,\"gup_after_mremap\":true,\"parent_and_file_unchanged\":true,\"content_ok\":true}\n", + BYTES, cpu_mismatch, read_mismatch, write_mismatch, WINDOWS); + ok = true; +cleanup: + if (pid > 0 && !child_reaped) { + pid_t got = waitpid(pid, &child_status, WNOHANG); + if (got == pid) { child_reaped = true; report_wait(pid, child_status); } + else { kill(-pid, SIGKILL); (void)wait_ok(pid); } + } + child_group = 0; alarm(0); + for (unsigned i = 0; i < 2; i++) { if (control[i] >= 0) close(control[i]); if (status[i] >= 0) close(status[i]); } + if (a != MAP_FAILED) munmap(a, BYTES); + if (b != MAP_FAILED) munmap(b, BYTES); + if (fd >= 0) close(fd); + free(buffer); + return !ok; +} diff --git a/tools/testing/selftests/arm64/user4k/same-native.c b/tools/testing/selftests/arm64/user4k/same-native.c new file mode 100644 index 00000000000000..9c592ee98c3a73 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/same-native.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Three generations; actual PFNs distinguish same-native-folio COW coverage. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define TRIALS 16 +#define PFN_MASK ((UINT64_C(1) << 55) - 1) +struct result { uint64_t old_pfn, new_pfn, held_pfn; unsigned quarter; }; +static volatile sig_atomic_t group; +static unsigned char *data; +static void stop(int sig) +{ + if (group > 0) kill(-group, SIGKILL); + _exit(128 + sig); +} +static bool io_full(int fd, void *buf, size_t bytes, bool output) +{ + unsigned char *p = buf; + while (bytes) { + ssize_t n = output ? write(fd, p, bytes) : read(fd, p, bytes); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) return false; + p += n; bytes -= n; + } + return true; +} +static unsigned char pattern(size_t at, unsigned generation, unsigned quarter) +{ + unsigned salt = at / LEAF == quarter ? generation * 53 : 0; + return (unsigned char)((at * 17 + at / LEAF * 29 + at / 256 + salt) % 251 + 1); +} +static void change(unsigned generation, unsigned quarter) +{ + size_t at; + for (at = quarter * LEAF; at < (quarter + 1) * LEAF; at++) + data[at] = pattern(at, generation, quarter); +} +static bool verify(unsigned generation, unsigned quarter) +{ + size_t at; + for (at = 0; at < NATIVE; at++) + if (data[at] != pattern(at, generation, quarter)) return false; + return true; +} +static bool pfn(int fd, unsigned quarter, uint64_t *out) +{ + uint64_t entry; + off_t where = ((uintptr_t)data / LEAF + quarter) * 8; + if (pread(fd, &entry, 8, where) != 8 || !(entry & (UINT64_C(1) << 63)) || + (entry & (UINT64_C(1) << 62)) || !(entry & PFN_MASK)) return false; + *out = entry & PFN_MASK; + return true; +} +static bool wait_ok(pid_t child) +{ + int status; pid_t got; + do { got = waitpid(child, &status, 0); } while (got < 0 && errno == EINTR); + return got == child && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} +static bool caps(void) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256]; unsigned long long effective = 0; + if (!f) return false; + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "CapEff: %llx", &effective) == 1) break; + fclose(f); return !!(effective & (1ULL << 21)); +} +static void owner(int output, unsigned quarter) +{ + struct result result = { .quarter = quarter }; + int go[2], answer[2], mapfd; + char token; + pid_t held; + alarm(5); + if (pipe(go) || pipe(answer)) _exit(10); + /* Open in this mm, never inherit an ancestor's already-open pagemap fd. */ + mapfd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (mapfd < 0) _exit(11); + change(1, quarter); + if (!pfn(mapfd, quarter, &result.old_pfn) || !verify(1, quarter)) _exit(12); + held = fork(); + if (held < 0) _exit(13); + if (!held) { + uint64_t held_pfn; + close(go[1]); close(answer[0]); close(output); close(mapfd); + alarm(4); + if (!io_full(go[0], &token, 1, false) || token != 'r') _exit(14); + mapfd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (mapfd < 0 || !pfn(mapfd, quarter, &held_pfn) || + held_pfn != result.old_pfn || !verify(1, quarter) || + !io_full(answer[1], &held_pfn, sizeof(held_pfn), true)) _exit(15); + close(mapfd); close(go[0]); close(answer[1]); _exit(0); + } + /* Before libc/pipe bookkeeping, consume a free sibling if allocator picks it. */ + change(2, quarter); + if (!pfn(mapfd, quarter, &result.new_pfn) || + result.new_pfn == result.old_pfn || !verify(2, quarter)) _exit(16); + close(go[0]); close(answer[1]); token = 'r'; + if (!io_full(go[1], &token, 1, true) || + !io_full(answer[0], &result.held_pfn, sizeof(result.held_pfn), false) || + !wait_ok(held) || !verify(2, quarter)) _exit(17); + close(go[1]); close(answer[0]); close(mapfd); + if (!io_full(output, &result, sizeof(result), true)) _exit(18); + close(output); _exit(0); +} +int main(int argc, char **argv) +{ + unsigned same = 0; + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + for (unsigned q = 0; q < 4; q++) + for (size_t at = 0; at < NATIVE; at++) + if ((pattern(at, 1, q) != pattern(at, 2, q)) != (at / LEAF == q)) return 1; + puts("PASS generation pattern selftest"); return 0; + } + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF || !caps()) { + fputs("Requires established4K launcher/native16K kernel and visible PFNs\n", stderr); return 77; + } + if (prctl(PR_SET_CHILD_SUBREAPER, 1)) return 1; + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop); signal(SIGTERM, stop); signal(SIGINT, stop); + setvbuf(stdout, NULL, _IONBF, 0); alarm(20); + void *map = mmap(NULL, 2 * NATIVE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (map == MAP_FAILED) return 1; + data = (void *)(((uintptr_t)map + NATIVE - 1) & ~(NATIVE - 1)); + for (size_t at = 0; at < NATIVE; at++) data[at] = pattern(at, 0, 0); + for (unsigned trial = 0; trial < TRIALS; trial++) { + int output[2]; struct result result; pid_t pid; + if (pipe(output)) return 1; + pid = fork(); + if (pid < 0) return 1; + if (!pid) { + group = 0; + if (setpgid(0, 0)) _exit(1); + close(output[0]); owner(output[1], trial % 4); + } + group = pid; + if (setpgid(pid, pid) && errno != EACCES && errno != ESRCH) { + kill(pid, SIGKILL); stop(SIGTERM); + } + close(output[1]); + bool received = io_full(output[0], &result, sizeof(result), false); + close(output[0]); + bool good = wait_ok(pid); + /* Reap any orphaned held generation on failure; no child can escape group. */ + kill(-pid, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) {} + group = 0; + if (!received || !good || result.held_pfn != result.old_pfn || + result.new_pfn == result.old_pfn || !verify(0, 0)) { + fprintf(stderr, "FAIL trial %u generation/PFN/isolation/cleanup\n", trial); return 1; + } + bool native_equal = result.old_pfn / 4 == result.new_pfn / 4; + same += native_equal; + printf("{\"trial\":%u,\"logical_quarter\":%u,\"old_pfn4k\":%"PRIu64",\"new_pfn4k\":%"PRIu64",\"held_pfn4k\":%"PRIu64",\"same_native\":%s,\"three_generation_bytes_ok\":true}\n", + trial, result.quarter, result.old_pfn, result.new_pfn, result.held_pfn, native_equal ? "true" : "false"); + } + munmap(map, 2 * NATIVE); alarm(0); + printf("{\"result\":\"%s\",\"trials\":%u,\"same_native_exercised\":%u,\"payload_bytes\":%lu}\n", same ? "PASS" : "UNEXERCISED", TRIALS, same, NATIVE); + return same ? 0 : 77; +} diff --git a/tools/testing/selftests/arm64/user4k/same-quarter.c b/tools/testing/selftests/arm64/user4k/same-quarter.c new file mode 100644 index 00000000000000..ce1f7e80081d04 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/same-quarter.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Metadata-only native-backing comparison. Never changes memory policy. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define NATIVE 16384UL +#define BYTES (16UL << 20) +#define LEAVES (BYTES / LEAF) +#define WINDOWS (BYTES / NATIVE) +#define MAX_FRAMES (LEAVES * 2) + +struct coverage { + uint64_t native_units, quarters, occupancy[4]; +}; +struct result { + unsigned arm; + bool content_ok, disjoint_ok; + struct coverage parent, child, modified, unchanged, combined; + uint64_t modified_parent_native_overlap; + uint64_t modified_unchanged_native_overlap; + uint64_t modified_quarter_counts[4]; +}; +static volatile sig_atomic_t fixture_group; + +static void stop_fixture(int sig) +{ + if (fixture_group > 0) + kill(-fixture_group, SIGKILL); + _exit(128 + sig); +} +static bool full_io(int fd, void *buffer, size_t bytes, bool writing) +{ + char *p = buffer; + while (bytes) { + ssize_t n = writing ? write(fd, p, bytes) : read(fd, p, bytes); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + p += n; + bytes -= (size_t)n; + } + return true; +} +static bool wait_ok(pid_t pid) +{ + int status; + pid_t got; + do { got = waitpid(pid, &status, 0); } while (got < 0 && errno == EINTR); + return got > 0 && WIFEXITED(status) && WEXITSTATUS(status) == 0; +} +static int compare_u64(const void *a, const void *b) +{ + uint64_t x = *(const uint64_t *)a, y = *(const uint64_t *)b; + return (x > y) - (x < y); +} +/* Input is actual4K PFN, never VMA-derived quarter position. Duplicates count once. */ +static struct coverage summarize(uint64_t *frames, size_t count) +{ + struct coverage out = {}; + uint64_t last = UINT64_MAX; + unsigned mask = 0; + qsort(frames, count, sizeof(*frames), compare_u64); + for (size_t i = 0; i < count; i++) { + uint64_t native = frames[i] / 4; + if (native != last) { + if (mask) { + unsigned n = __builtin_popcount(mask); + out.native_units++; + out.quarters += n; + out.occupancy[n - 1]++; + } + last = native; + mask = 0; + } + mask |= 1U << (frames[i] % 4); + } + if (mask) { + unsigned n = __builtin_popcount(mask); + out.native_units++; + out.quarters += n; + out.occupancy[n - 1]++; + } + return out; +} +/* Both arrays were sorted by summarize. Count each common nativeunit once. */ +static uint64_t native_overlap(const uint64_t *a, size_t an, const uint64_t *b, size_t bn) +{ + size_t ai = 0, bi = 0; + uint64_t count = 0; + while (ai < an && bi < bn) { + uint64_t av = a[ai] / 4, bv = b[bi] / 4; + if (av == bv) + count++; + if (av <= bv) + while (ai < an && a[ai] / 4 == av) ai++; + if (bv <= av) + while (bi < bn && b[bi] / 4 == bv) bi++; + } + return count; +} +static bool read_frames(pid_t pid, const unsigned char *data, uint64_t frames[LEAVES]) +{ + char path[64]; + uint64_t entries[LEAVES]; + snprintf(path, sizeof(path), "/proc/%ld/pagemap", (long)pid); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return false; + ssize_t n = pread(fd, entries, sizeof(entries), (uintptr_t)data / LEAF * 8); + close(fd); + if (n != (ssize_t)sizeof(entries)) + return false; + for (size_t i = 0; i < LEAVES; i++) { + uint64_t pfn = entries[i] & ((UINT64_C(1) << 55) - 1); + if (!(entries[i] & (UINT64_C(1) << 63)) || !pfn) + return false; /* Swap/hiddenPFNs invalidate this bounded resident comparison. */ + frames[i] = pfn; + } + return true; +} +static unsigned chosen_quarter(unsigned arm, size_t window) +{ + return arm ? window % 4 : 0; +} +static unsigned char tag(size_t leaf, bool changed) +{ + unsigned char value = (unsigned char)(leaf % 251 + 1); + return changed ? value ^ 0xa5 : value; +} +static bool contents(const unsigned char *data, unsigned arm, bool child) +{ + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + bool changed = child && leaf % 4 == chosen_quarter(arm, leaf / 4); + if (data[leaf * LEAF] != tag(leaf, changed)) + return false; + for (size_t byte = 1; byte < LEAF; byte++) + if (data[leaf * LEAF + byte] != 0x5a) + return false; + } + return true; +} +static void owner(unsigned arm, int output) +{ + uint64_t parent[LEAVES], child_frames[LEAVES], modified[WINDOWS]; + uint64_t unchanged[LEAVES - WINDOWS], combined[MAX_FRAMES]; + struct result result = {.arm = arm}; + int ready[2], release[2]; + char token; + size_t mi = 0, ui = 0; + alarm(20); + void *mapping = mmap(NULL, BYTES + 2 * NATIVE, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapping == MAP_FAILED) + _exit(2); + unsigned char *data = (void *)(((uintptr_t)mapping + NATIVE - 1) & ~(NATIVE - 1)); + if (mprotect(data, BYTES, PROT_READ | PROT_WRITE)) + _exit(2); + memset(data, 0x5a, BYTES); + for (size_t leaf = 0; leaf < LEAVES; leaf++) + data[leaf * LEAF] = tag(leaf, false); + if (!contents(data, arm, false) || !read_frames(getpid(), data, parent)) + _exit(3); + struct coverage initial = summarize(parent, LEAVES); + if (initial.native_units * NATIVE != BYTES || initial.quarters != LEAVES) + _exit(4); /* Stop beforefork if the32MiB payload ceiling is not established. */ + if (pipe(ready) || pipe(release)) + _exit(2); + pid_t child = fork(); + if (child < 0) + _exit(2); + if (!child) { + alarm(20); + close(ready[0]); close(release[1]); close(output); + for (size_t window = 0; window < WINDOWS; window++) { + size_t leaf = window * 4 + chosen_quarter(arm, window); + *(volatile unsigned char *)(data + leaf * LEAF) = tag(leaf, true); + } + if (!contents(data, arm, true)) + _exit(3); + token = 'r'; + if (!full_io(ready[1], &token, 1, true)) + _exit(2); + close(ready[1]); + if (!full_io(release[0], &token, 1, false) || token != 'x') + _exit(2); + close(release[0]); + _exit(contents(data, arm, true) ? 0 : 3); + } + close(ready[1]); close(release[0]); + if (!full_io(ready[0], &token, 1, false) || token != 'r') + _exit(2); + close(ready[0]); + result.content_ok = contents(data, arm, false); + if (!result.content_ok || !read_frames(getpid(), data, parent) || + !read_frames(child, data, child_frames)) + _exit(3); + result.disjoint_ok = true; + for (size_t leaf = 0; leaf < LEAVES; leaf++) { + bool changed = leaf % 4 == chosen_quarter(arm, leaf / 4); + if (changed) { + modified[mi++] = child_frames[leaf]; + result.modified_quarter_counts[child_frames[leaf] % 4]++; + result.disjoint_ok &= parent[leaf] != child_frames[leaf]; + } else { + unchanged[ui++] = child_frames[leaf]; + result.disjoint_ok &= parent[leaf] == child_frames[leaf]; + } + } + memcpy(combined, parent, sizeof(parent)); + memcpy(combined + LEAVES, child_frames, sizeof(child_frames)); + result.parent = summarize(parent, LEAVES); + result.child = summarize(child_frames, LEAVES); + result.modified = summarize(modified, mi); + result.unchanged = summarize(unchanged, ui); + result.combined = summarize(combined, MAX_FRAMES); + result.modified_parent_native_overlap = native_overlap(modified, mi, parent, LEAVES); + result.modified_unchanged_native_overlap = native_overlap(modified, mi, unchanged, ui); + result.disjoint_ok &= mi == WINDOWS && result.modified.quarters == WINDOWS && + result.combined.quarters == LEAVES + WINDOWS && + result.modified_parent_native_overlap == 0 && + result.modified_unchanged_native_overlap == 0 && + result.combined.native_units * NATIVE <= 2 * BYTES; + token = 'x'; + if (!full_io(release[1], &token, 1, true)) + _exit(2); + close(release[1]); + bool child_ok = wait_ok(child); + result.content_ok &= child_ok && contents(data, arm, false); + munmap(mapping, BYTES + 2 * NATIVE); + if (!full_io(output, &result, sizeof(result), true)) + _exit(2); + close(output); + _exit(result.content_ok && result.disjoint_ok ? 0 : 3); +} +static bool visible_pfns(void) +{ + FILE *file = fopen("/proc/self/status", "re"); + char line[256]; + unsigned long long caps = 0; + if (!file) return false; + while (fgets(line, sizeof(line), file)) + if (sscanf(line, "CapEff: %llx", &caps) == 1) break; + fclose(file); + return !!(caps & (1ULL << 21)); +} +static void print_coverage(const char *name, struct coverage c) +{ + printf("\"%s\":{\"native_bytes\":%llu,\"represented_bytes\":%llu," + "\"quarter_occupancy\":[%llu,%llu,%llu,%llu]}", name, + (unsigned long long)(c.native_units * NATIVE), + (unsigned long long)(c.quarters * LEAF), + (unsigned long long)c.occupancy[0], (unsigned long long)c.occupancy[1], + (unsigned long long)c.occupancy[2], (unsigned long long)c.occupancy[3]); +} +static int selftest(void) +{ + uint64_t a[] = {400, 401, 401, 403, 404}, b[] = {402, 405}; + uint64_t combined[] = {400, 401, 401, 403, 404, 402, 405}; + struct coverage ca = summarize(a, 5), cb = summarize(b, 2); + struct coverage cc = summarize(combined, 7); + bool ok = ca.native_units == 2 && ca.quarters == 4 && ca.occupancy[0] == 1 && + ca.occupancy[2] == 1 && cb.native_units == 2 && cb.quarters == 2 && + cc.native_units == 2 && cc.quarters == 6 && cc.occupancy[1] == 1 && + cc.occupancy[3] == 1 && native_overlap(a, 5, b, 2) == 2; + uint64_t sparse[] = {400, 404, 408, 412}, dense[] = {400, 401, 402, 403}; + ca = summarize(sparse, 4); cb = summarize(dense, 4); + ok &= ca.native_units == 4 && cb.native_units == 1 && ca.quarters == cb.quarters; + printf("metadata selftest: %s\n", ok ? "PASS" : "FAIL"); + return !ok; +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) return selftest(); + if (argc != 2 || strcmp(argv[1], "--native-16k")) { fprintf(stderr, "usage: %s --native-16k | --selftest\n", argv[0]); return 2; } + if (getauxval(AT_PAGESZ) != LEAF || !visible_pfns()) { + fprintf(stderr, "SKIP: established4K launcher, native16K kernel and visiblePFNs required\n"); + return 4; + } + signal(SIGPIPE, SIG_IGN); signal(SIGALRM, stop_fixture); signal(SIGTERM, stop_fixture); + signal(SIGINT, stop_fixture); + if (prctl(PR_SET_CHILD_SUBREAPER, 1)) return 2; + alarm(30); + for (unsigned arm = 0; arm < 2; arm++) { + int pipefd[2]; struct result result = {}; bool copied, success; + if (pipe(pipefd)) return 2; + pid_t child = fork(); + if (child < 0) return 2; + if (!child) { + if (setpgid(0, 0)) _exit(2); + close(pipefd[0]); owner(arm, pipefd[1]); + } + fixture_group = child; + if (setpgid(child, child) && errno != EACCES && errno != ESRCH) stop_fixture(SIGTERM); + close(pipefd[1]); + copied = full_io(pipefd[0], &result, sizeof(result), false); + close(pipefd[0]); success = wait_ok(child); + if (!copied || !success) { + kill(-child, SIGKILL); + while (waitpid(-1, NULL, 0) > 0 || errno == EINTR) {} + fprintf(stderr, "FAIL: arm%u incomplete or integrity/coverage check failed\n", arm); + return 1; + } + fixture_group = 0; + printf("{\"arm\":\"%s\",\"logical_parent_bytes\":%lu,\"modified_child_bytes\":%lu,", + arm ? "balanced" : "same-quarter-zero", BYTES, WINDOWS * LEAF); + print_coverage("parent", result.parent); printf(","); + print_coverage("child", result.child); printf(","); + print_coverage("modified_child", result.modified); printf(","); + print_coverage("unchanged_child", result.unchanged); printf(","); + print_coverage("parent_child_union", result.combined); + printf(",\"modified_parent_native_overlap\":%llu,\"modified_unchanged_native_overlap\":%llu," + "\"modified_physical_quarter_counts\":[%llu,%llu,%llu,%llu],\"content_ok\":true,\"disjoint_ok\":true}\n", + (unsigned long long)result.modified_parent_native_overlap, + (unsigned long long)result.modified_unchanged_native_overlap, + (unsigned long long)result.modified_quarter_counts[0], (unsigned long long)result.modified_quarter_counts[1], + (unsigned long long)result.modified_quarter_counts[2], (unsigned long long)result.modified_quarter_counts[3]); + fflush(stdout); + } + alarm(0); + return 0; +} From b5a783aa5dc70a1c206ed4b7130e7ec5003238f3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 18:49:51 -0400 Subject: [PATCH 299/631] mm: share backing for clipped anonymous and file first writes Short anonymous windows and untouched private-file writes allocated a fresh 16 KiB folio for individual 4 KiB destinations. They consumed 64 MiB of backing for 16 MiB of written data on a 16 KiB kernel. Reserve clipped anonymous batches from one shared owner using a matching free-slot count throughout lookup, waiting and supply. Keep slots indexed by their logical mask while zeroing and publishing their actual physical quarters. Preserve full native-window batching and private fallbacks. Pool private-file first writes only for known filemap and shmem callbacks. Release destination locks before invoking the source callback, serialize publication and nonlinear rmap preparation, and mark shared folios uptodate atomically. Retry publication metadata OOM once with private backing, without internally retrying callbacks that dropped the VMA lock. Hardware first-write cases now use 16.06-16.19 MiB for 16 MiB of data. Clipped-neighbor prefaulting, full contents, source isolation, repeated eight-writer first faults, COW/GUP/mremap and actual reclaim/readback pass. Dense allocation remains 16 MiB; sparse wide-VMA prefaulting is unchanged. Maintained regression binaries match the hardware-tested artifacts. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit a0e28af0b29548a42148ce7e2c5054e24b89883a) --- include/linux/mm_subpage.h | 9 + mm/memory.c | 104 +++++++-- mm/user-subpage.c | 73 ++++-- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../selftests/arm64/user4k/README.first-write | 28 +++ .../arm64/user4k/clipped-firstwrite.c | 141 ++++++++++++ .../selftests/arm64/user4k/first-write.c | 216 ++++++++++++++++++ 8 files changed, 538 insertions(+), 39 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.first-write create mode 100644 tools/testing/selftests/arm64/user4k/clipped-firstwrite.c create mode 100644 tools/testing/selftests/arm64/user4k/first-write.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index e892ae3e70bee2..910ed2b36bcfd1 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -68,6 +68,15 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, unsigned long mask, gfp_t gfp); bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask); +/* Arbitrary physical offsets, one owner/folio lock for the whole logical mask. + * Output slots are indexed by logical mask bits; use each slot's actual offset. + * Count-aware supply/wait must accompany this allocation to guarantee progress. + */ +int mm_subpage_alloc_any_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots); +int mm_subpage_pool_add_folio_count(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int count, gfp_t gfp); +bool mm_subpage_cow_wait_busy_count(struct mm_subpage_pool *pool, unsigned int count); /* Return an initialized slot with its folio locked; may wait without a pool lock. */ struct mm_subpage *mm_subpage_alloc_at_locked(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index cde03950db1e15..77735be88aed57 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6384,28 +6384,33 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte, /* One native-window batch, initialized and locked before PTE publication. */ static int anon_subpage_prealloc_mask(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, - unsigned long mask, bool use_pool, bool *pooled, - struct mm_subpage **slots) + unsigned long mask, bool use_pool, bool any_offset, + bool *pooled, struct mm_subpage **slots) { struct mm_subpage_pool *pool; struct folio *folio; + unsigned int count = hweight_long(mask); + bool arbitrary; int err; retry_private: pool = use_pool ? mm_subpage_cow_pool_get(mm, vma, addr) : NULL; *pooled = pool != NULL; + arbitrary = *pooled && any_offset; if (!pool) pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); if (!pool) return -ENOMEM; for (;;) { - err = mm_subpage_alloc_mask_locked(pool, mask, slots); + err = arbitrary ? mm_subpage_alloc_any_mask_locked(pool, mask, slots) : + mm_subpage_alloc_mask_locked(pool, mask, slots); if (err != -EAGAIN) break; folio = NULL; if (*pooled) { folio = cow_subpage_speculative_folio(mm, vma, addr); - if (!folio && mm_subpage_cow_wait_busy_mask(pool, mask)) { + if (!folio && (arbitrary ? mm_subpage_cow_wait_busy_count(pool, count) : + mm_subpage_cow_wait_busy_mask(pool, mask))) { cond_resched(); continue; } @@ -6422,15 +6427,20 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, use_pool = false; goto retry_private; } - err = mm_subpage_pool_add_folio_mask(pool, folio, mask, + err = arbitrary ? mm_subpage_pool_add_folio_count(pool, folio, count, + GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN) : + mm_subpage_pool_add_folio_mask(pool, folio, mask, *pooled ? GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN : GFP_KERNEL); if (*pooled && err == -ENOMEM) { - if (mm_subpage_cow_wait_busy_mask(pool, mask)) { + if (arbitrary ? mm_subpage_cow_wait_busy_count(pool, count) : + mm_subpage_cow_wait_busy_mask(pool, mask)) { folio_put(folio); cond_resched(); continue; } - err = mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); + err = arbitrary ? + mm_subpage_pool_add_folio_count(pool, folio, count, GFP_KERNEL) : + mm_subpage_pool_add_folio_mask(pool, folio, mask, GFP_KERNEL); } if (err) { folio_put(folio); @@ -6493,8 +6503,9 @@ static vm_fault_t do_anonymous_subpage(struct vm_fault *vmf) pte_unmap(vmf->pte); vmf->pte = NULL; err = anon_subpage_prealloc_mask(mm, vma, vmf->address, mask, - mask != window_mask && !force_private && !userfaultfd_armed(vma), - &pooled, slots); + (mask != window_mask || nr < (PAGE_SIZE >> mm_page_shift(mm))) && + !force_private && !userfaultfd_armed(vma), + nr < (PAGE_SIZE >> mm_page_shift(mm)), &pooled, slots); if (err) return VM_FAULT_OOM; folio = mm_subpage_folio(slots[__ffs(mask)]); @@ -7092,7 +7103,9 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) struct mm_subpage *slot = vmf->cow_subpage; bool is_cow = (vmf->flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED); struct folio *folio = page_folio(is_cow ? vmf->cow_page : vmf->page); + bool fresh = false; vm_fault_t ret; + int err; pte_t entry; if (!(vma->vm_flags & VM_SHARED)) { @@ -7106,10 +7119,18 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) else if (pte_alloc(mm, vmf->pmd)) return VM_FAULT_OOM; } - /* The private destination is not yet visible through a PTE. */ + /* A reserved destination slot is private; sibling slots may be published. */ if (is_cow) { folio_get(folio); folio_lock(folio); + if (slot) { + fresh = !mm_subpage_anon_root(slot); + err = mm_subpage_prepare_anon_rmap(slot, vma, vmf->address); + if (err) { + ret = err == -ENOMEM ? VM_FAULT_OOM : VM_FAULT_SIGBUS; + goto unlock_folio; + } + } } vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl); ret = VM_FAULT_NOPAGE; @@ -7121,7 +7142,9 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) } if (slot) { mm_subpage_set_exclusive(slot); - if (WARN_ON_ONCE(mm_subpage_add_new_anon_rmap(slot, vma, vmf->address))) { + err = fresh ? mm_subpage_add_new_anon_rmap(slot, vma, vmf->address) : + mm_subpage_add_anon_rmap(slot, vma, vmf->address); + if (WARN_ON_ONCE(err)) { ret = VM_FAULT_SIGBUS; goto unlock; } @@ -7129,7 +7152,8 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) entry = maybe_mkwrite(pte_mkdirty(pte_sw_mkyoung(entry)), vma); if (vmf_orig_pte_uffd_wp(vmf)) entry = pte_mkuffd_wp(entry); - subpage_add_new_lru(folio, vma, 1); + if (fresh) + subpage_add_new_lru(folio, vma, 1); set_pte_at(mm, vmf->address, vmf->pte, entry); update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1); inc_mm_counter(mm, MM_ANONPAGES); @@ -7476,6 +7500,10 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct folio *folio; vm_fault_t ret; +#ifdef CONFIG_MM_SUBPAGE + bool pooled = false, force_private = false; + vm_fault_t earlier = 0; +#endif ret = vmf_can_call_fault(vmf); if (!ret) @@ -7484,11 +7512,24 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) return ret; #ifdef CONFIG_MM_SUBPAGE +retry_subpage: if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { - vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + /* Known callbacks only supply a locked source page for the core. + * Custom/DAX callbacks may fill a native cow_page or finish COW. + */ + if (!force_private && !userfaultfd_armed(vma) && + (vma->vm_ops->fault == filemap_fault || vma_is_shmem(vma))) { + vmf->cow_subpage = cow_subpage_prealloc(vma->vm_mm, vma, + vmf->address, NULL, false, &pooled); + if (!IS_ERR(vmf->cow_subpage)) + folio_unlock(mm_subpage_folio(vmf->cow_subpage)); + } else { + pooled = false; + vmf->cow_subpage = anon_subpage_prealloc(vma->vm_mm, vma, vmf->address); + } if (IS_ERR(vmf->cow_subpage)) { vmf->cow_subpage = NULL; - return VM_FAULT_OOM; + return earlier | VM_FAULT_OOM; } folio = mm_subpage_folio(vmf->cow_subpage); goto allocated; @@ -7513,12 +7554,22 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) #ifdef CONFIG_MM_SUBPAGE if (vmf->cow_subpage) { - if (copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), - vmf->page, vma_page_offset_at(vma, vmf->address).offset, - mm_page_size(vma->vm_mm))) { + int err; + + /* No destination lock crosses the source fault callback or retry. */ + if (pooled) + folio_lock(folio); + err = copy_user_subpage_range(vmf->cow_page, mm_subpage_offset(vmf->cow_subpage), + vmf->page, vma_page_offset_at(vma, vmf->address).offset, + mm_page_size(vma->vm_mm)); + if (pooled) + folio_unlock(folio); + if (err) { ret = VM_FAULT_HWPOISON; goto unlock; } + /* Preserve concurrent sibling lock/LRU/waiter flag updates. */ + folio_mark_uptodate(folio); goto copied; } #endif @@ -7539,21 +7590,36 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) goto unlock; } } + __folio_mark_uptodate(folio); #ifdef CONFIG_MM_SUBPAGE copied: #endif - __folio_mark_uptodate(folio); - ret |= finish_fault(vmf); unlock: unlock_page(vmf->page); put_page(vmf->page); +#ifdef CONFIG_MM_SUBPAGE + vmf->page = NULL; + /* Publication OOM left the PTE untouched. Retry with private metadata, + * outside the source lock; never loop internally on callback RETRY. + */ + if (pooled && (ret & VM_FAULT_OOM)) { + mm_subpage_put(vmf->cow_subpage); + vmf->cow_subpage = NULL; + vmf->cow_page = NULL; + earlier |= ret & VM_FAULT_MAJOR; + force_private = true; + goto retry_subpage; + } + ret |= earlier; +#endif if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) goto uncharge_out; return ret; uncharge_out: #ifdef CONFIG_MM_SUBPAGE + ret |= earlier; if (vmf->cow_subpage) { mm_subpage_put(vmf->cow_subpage); vmf->cow_subpage = NULL; diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 30fb3669458cb6..fd553493b3de4e 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -859,7 +859,7 @@ static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio * allocation: a busy slot farther down the list may require no new charge. */ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, - bool require_all) + bool require_all, unsigned int count) { struct mm_subpage_owner *owner; struct folio *wait = NULL; @@ -870,6 +870,7 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, list_for_each_entry(owner, &pool->available, available) { if (!owner->migrating && (~owner->used & eligible) && (!require_all || !(owner->used & eligible)) && + (count == 1 || hweight_long(~owner->used & eligible) >= count) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && !folio_test_writeback(owner->folio)) { @@ -890,21 +891,29 @@ static bool cow_wait_busy(struct mm_subpage_pool *pool, unsigned long eligible, bool mm_subpage_cow_wait_busy(struct mm_subpage_pool *pool) { - return cow_wait_busy(pool, pool->all_slots, false); + return cow_wait_busy(pool, pool->all_slots, false, 1); } bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset) { - return cow_wait_busy(pool, offset_mask(pool->shift, offset), false); + return cow_wait_busy(pool, offset_mask(pool->shift, offset), false, 1); } bool mm_subpage_cow_wait_busy_mask(struct mm_subpage_pool *pool, unsigned long mask) { - return cow_wait_busy(pool, mask, true); + return cow_wait_busy(pool, mask, true, 1); +} + +bool mm_subpage_cow_wait_busy_count(struct mm_subpage_pool *pool, unsigned int count) +{ + if (!count || count > hweight_long(pool->all_slots)) + return false; + return cow_wait_busy(pool, pool->all_slots, false, count); } static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, - unsigned long eligible, bool require_all) + unsigned long eligible, bool require_all, + unsigned int count) { struct mm_subpage_owner *owner; unsigned int visited = 0; @@ -927,6 +936,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, /* A nonpresent PTE may still need any of this folio's slot bytes. */ if (!owner->migrating && (~owner->used & eligible) && (!require_all || !(owner->used & eligible)) && + (count == 1 || hweight_long(~owner->used & eligible) >= count) && (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && @@ -937,7 +947,8 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, } static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, - unsigned long eligible, bool require_all, gfp_t gfp) + unsigned long eligible, bool require_all, + unsigned int count, gfp_t gfp) { struct mm_subpage_owner *owner; unsigned long flags; @@ -965,7 +976,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, spin_lock_irqsave(&pool->lock, flags); if (pool->closed) ret = -ESHUTDOWN; - else if (pool->unissued || find_available(pool, eligible, require_all)) + else if (pool->unissued || find_available(pool, eligible, require_all, count)) /* Another allocator supplied backing while we allocated metadata. * Preserve the single-unissued-owner invariant even when a shared * COW lookup skips a concurrently locked or rotating list prefix. @@ -993,7 +1004,7 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, int mm_subpage_pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, gfp_t gfp) { - return pool_add_folio(pool, folio, pool->all_slots, false, gfp); + return pool_add_folio(pool, folio, pool->all_slots, false, 1, gfp); } int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *folio, @@ -1003,7 +1014,7 @@ int mm_subpage_pool_add_folio_at(struct mm_subpage_pool *pool, struct folio *fol if (!eligible) return -EINVAL; - return pool_add_folio(pool, folio, eligible, false, gfp); + return pool_add_folio(pool, folio, eligible, false, 1, gfp); } int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *folio, @@ -1011,7 +1022,15 @@ int mm_subpage_pool_add_folio_mask(struct mm_subpage_pool *pool, struct folio *f { if (!mask || (mask & ~pool->all_slots)) return -EINVAL; - return pool_add_folio(pool, folio, mask, true, gfp); + return pool_add_folio(pool, folio, mask, true, 1, gfp); +} + +int mm_subpage_pool_add_folio_count(struct mm_subpage_pool *pool, struct folio *folio, + unsigned int count, gfp_t gfp) +{ + if (!count || count > hweight_long(pool->all_slots)) + return -EINVAL; + return pool_add_folio(pool, folio, pool->all_slots, false, count, gfp); } unsigned long mm_subpage_pool_backing_pages(struct mm_subpage_pool *pool) @@ -1066,7 +1085,7 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, subpage = ERR_PTR(-ESHUTDOWN); goto out; } - owner = find_available(pool, eligible, false); + owner = find_available(pool, eligible, false, 1); if (!owner) { subpage = ERR_PTR(-EAGAIN); goto out; @@ -1106,13 +1125,13 @@ static struct mm_subpage *subpage_reserve(struct mm_subpage_pool *pool, return subpage; } -/* Reserve one exact-offset batch from one owner, with one folio lock. */ -int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, - struct mm_subpage **slots) +/* One owner/folio lock; output remains indexed by the caller's logical mask. */ +static int subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + bool any_offset, struct mm_subpage **slots) { struct mm_subpage_owner *owner; - unsigned long flags, zero = 0; - unsigned int i; + unsigned long flags, zero = 0, available; + unsigned int i, physical, count = hweight_long(mask); bool needs_zero; if (!mask || (mask & ~pool->all_slots)) @@ -1123,7 +1142,8 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas spin_unlock_irqrestore(&pool->lock, flags); return -ESHUTDOWN; } - owner = find_available(pool, mask, true); + owner = find_available(pool, any_offset ? pool->all_slots : mask, + !any_offset, any_offset ? count : 1); if (!owner) { spin_unlock_irqrestore(&pool->lock, flags); return -EAGAIN; @@ -1140,8 +1160,11 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas spin_unlock_irqrestore(&pool->lock, flags); return -EAGAIN; } + available = ~owner->used & pool->all_slots; for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { - slots[i] = reserve_owner_slot(owner, i, &needs_zero); + physical = any_offset ? __ffs(available) : i; + available &= ~BIT(physical); + slots[i] = reserve_owner_slot(owner, physical, &needs_zero); if (needs_zero) zero |= BIT(i); } @@ -1149,13 +1172,25 @@ int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mas for_each_set_bit(i, &mask, SUBPAGES_PER_FOLIO) { if (zero & BIT(i)) clear_user_subpage_range(&owner->folio->page, - i << pool->shift, 1U << pool->shift); + mm_subpage_offset(slots[i]), 1U << pool->shift); /* Zeroing must precede publication, including reused slots. */ refcount_set_release(&slots[i]->refs, 1); } return 0; } +int mm_subpage_alloc_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + return subpage_alloc_mask_locked(pool, mask, false, slots); +} + +int mm_subpage_alloc_any_mask_locked(struct mm_subpage_pool *pool, unsigned long mask, + struct mm_subpage **slots) +{ + return subpage_alloc_mask_locked(pool, mask, true, slots); +} + static struct mm_subpage *subpage_alloc(struct mm_subpage_pool *pool, unsigned long eligible, bool lock_folio) { bool needs_zero; diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 86bb27433258e3..85e45b6ce0399d 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -4,3 +4,5 @@ swap-fixture-init same-quarter file-cow-consumers same-native +first-write +clipped-firstwrite diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 0c22734f5ad03d..dd9d4f233b0562 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -12,11 +12,13 @@ TEST_FILES += README.hole-refill README.hole-refill-race README.hole-refill-recl TEST_FILES += compat-script runtime-app.py TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native TEST_FILES += README.any-quarter check-any-quarter.py +TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite +TEST_FILES += README.first-write include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.first-write b/tools/testing/selftests/arm64/user4k/README.first-write new file mode 100644 index 00000000000000..2ec69368a6c704 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.first-write @@ -0,0 +1,28 @@ +First-write and clipped-native-window backing tests +================================================== + +Run only on the audited native16K kernel through the established4K ABI launcher: + cd /var/tmp/owned-test-dir + userpages 4k ./first-write --native-16k + userpages 4k ./clipped-firstwrite --native-16k + +first-write: six fresh child arms (anonymous,shmem,regular file x dense/guarded), +eight threads, absent-PTE direct first writes, full untouched-byte verification, +DONTNEED and distinct-byte same-PTE refault racing, all source-file bytes verified. +Regular-file working directory MUST be non-tmpfs; otherwise that arm SKIPs. +All six must pass for qualification.4MiB useful payload/arm,<=16MiB native private +backing under a wasteful baseline plus source file.40s supervisor/15s child alarm +kills the fixture child/threads on timeout. No global pressure or kernel change. +This content/race fixture does not itself assert physical density. + +clipped-firstwrite: root CAP_SYS_ADMIN pagemap visibility required.11 fresh child +arms cover all valid starts for1/2/3leaf windows,3+1tail fill and full4dense control; +two rounds with DONTNEED between. A single initial byte write must prefault every +neighbor in its clipped batch, proven by pagemap before any neighbor read. Full +zero/content checks then native PFN union enforce density.256groups/arm, +<=8MiB payload backing even on baseline;16MiB reserved VA.1MiB native slack bounds +pool sharding/background COW. Count3 alone allows natural16K/12K packing; mixed3+1 +must fill tails.40s supervisor/8s child alarm, no GPU/reclaim/pressure. + +The fixture metadata selftest (--selftest) checks tags/sorting only. It is not a +kernel-path test and does not replace running both fixtures on the target kernel. diff --git a/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c b/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c new file mode 100644 index 00000000000000..c07cb1a8a980f1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/clipped-firstwrite.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Real clipped-window batching, zeroing, reuse and physical backing density. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define GROUPS 256UL +#define STRIDE (4 * NATIVE) +#define PFN_MASK ((UINT64_C(1)<<55)-1) +#define PRESENT (UINT64_C(1)<<63) +#define SWAPPED (UINT64_C(1)<<62) +static volatile sig_atomic_t child; +static void stop(int sig) { if(child>0)kill(child,SIGKILL);_exit(128+sig); } +static bool visible(void) +{ + FILE *f=fopen("/proc/self/status","re");char line[256];unsigned long long caps=0; + if(!f)return false; + while(fgets(line,sizeof(line),f))if(sscanf(line,"CapEff: %llx",&caps)==1)break; + fclose(f);return !!(caps&(1ULL<<21)); +} +static bool entries(int fd,unsigned char *base,unsigned n,uint64_t *out) +{ + if(pread(fd,out,n*8,(uintptr_t)base/LEAF*8)!=(ssize_t)n*8)return false; + for(unsigned i=0;iy)-(x>2; + if(mix){ + p=base+g*STRIDE+2*NATIVE; + if(!check_window(p,1,g+GROUPS,round)||!entries(fd,p,1,map))return 4; + native[used++]=(map[0]&PFN_MASK)>>2; + } + } + qsort(native,used,sizeof(*native),compare);size_t unique=0; + for(size_t i=0;itarget+(1UL<<20)){fprintf(stderr,"FAIL physical density\n");return 5;} + if(!round)for(unsigned g=0;g +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#define LEAF 4096UL +#define LEAVES 1024UL +#define WORKERS 8 +#define TMPFS_MAGIC 0x01021994 + +enum backing { ANONYMOUS, SHMEM, REGULAR }; +static unsigned char *data; +static size_t stride; +static pthread_barrier_t barrier; +static bool race; +static volatile sig_atomic_t running_child; +static void stop_fixture(int sig) +{ + if (running_child > 0) + kill(running_child, SIGKILL); + _exit(128 + sig); +} + +static unsigned char initial(size_t leaf, size_t byte) +{ + return leaf * 17 + byte * 3 + 0x39; +} + +static void *writer(void *arg) +{ + size_t leaf, id = (uintptr_t)arg; + + pthread_barrier_wait(&barrier); + if (race) { + /* Distinct bytes avoid a C data race while faulting the same PTE. */ + for (leaf = 0; leaf < LEAVES; leaf += 3) + data[leaf * stride + id] = initial(leaf, id) ^ 0xa5; + } else { + for (leaf = id; leaf < LEAVES; leaf += WORKERS) + data[leaf * stride] = initial(leaf, 0) ^ 0xa5; + } + return NULL; +} + +static bool verify(enum backing backing) +{ + size_t leaf, byte; + + for (leaf = 0; leaf < LEAVES; leaf++) { + for (byte = 0; byte < LEAF; byte++) { + unsigned char want = backing == ANONYMOUS ? 0 : initial(leaf, byte); + + if (!byte || (race && leaf % 3 == 0 && byte < WORKERS)) + want = initial(leaf, byte) ^ 0xa5; + if (data[leaf * stride + byte] != want) { + ksft_print_msg("leaf=%zu byte=%zu got=%u want=%u\n", leaf, byte, + data[leaf * stride + byte], want); + return false; + } + } + } + return true; +} + +static int exercise(enum backing backing, bool small) +{ + unsigned char buffer[LEAF]; + char name[] = "first-write-XXXXXX"; + pthread_t workers[WORKERS]; + struct statfs fs; + int fd = -1, flags = MAP_PRIVATE; + size_t leaf, byte, span; + unsigned int t; + + stride = small ? 2 * LEAF : LEAF; + span = LEAVES * stride; + if (backing != ANONYMOUS) { + fd = backing == SHMEM ? memfd_create("first-write", MFD_CLOEXEC) : + mkstemp(name); + if (fd < 0) + return 2; + if (backing == REGULAR) + unlink(name); + if (fstatfs(fd, &fs)) + return 2; + if (backing == REGULAR && fs.f_type == TMPFS_MAGIC) { + ksft_print_msg("regular-file arm needs a non-tmpfs working directory\n"); + return KSFT_SKIP; + } + if (ftruncate(fd, span)) + return 2; + for (leaf = 0; leaf < LEAVES; leaf++) { + for (byte = 0; byte < LEAF; byte++) + buffer[byte] = initial(leaf, byte); + if (pwrite(fd, buffer, LEAF, leaf * stride) != LEAF) + return 2; + } + } else { + flags |= MAP_ANONYMOUS; + } + data = mmap(NULL, span, small ? PROT_NONE : PROT_READ | PROT_WRITE, + small ? MAP_PRIVATE | MAP_ANONYMOUS : flags, small ? -1 : fd, 0); + if (data == MAP_FAILED) + return 2; + if (small) + for (leaf = 0; leaf < LEAVES; leaf++) + if (mmap(data + leaf * stride, LEAF, PROT_READ | PROT_WRITE, + flags | MAP_FIXED, fd, backing == ANONYMOUS ? 0 : leaf * stride) == + MAP_FAILED) + return 2; + + /* No reads through the mapping before concurrent direct first writes. */ + if (pthread_barrier_init(&barrier, NULL, WORKERS)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_join(workers[t], NULL)) + return 2; + if (!verify(backing)) + return 3; + /* Drop selected private slots, leaving the other slots live. */ + for (leaf = 0; leaf < LEAVES; leaf += 3) + if (madvise(data + leaf * stride, LEAF, MADV_DONTNEED)) + return 2; + /* Again no reads before the first writes: exercise abort and slot reuse. */ + race = true; + for (t = 0; t < WORKERS; t++) + if (pthread_create(&workers[t], NULL, writer, (void *)(uintptr_t)t)) + return 2; + for (t = 0; t < WORKERS; t++) + if (pthread_join(workers[t], NULL)) + return 2; + pthread_barrier_destroy(&barrier); + if (!verify(backing)) + return 3; + if (backing != ANONYMOUS) { + for (leaf = 0; leaf < LEAVES; leaf++) { + if (pread(fd, buffer, LEAF, leaf * stride) != LEAF) + return 2; + for (byte = 0; byte < LEAF; byte++) + if (buffer[byte] != initial(leaf, byte)) + return 3; + } + close(fd); + } + munmap(data, span); + return 0; +} + +int main(int argc, char **argv) +{ + static const char * const names[] = { "anonymous", "shmem", "regular-file" }; + unsigned int backing, small; + int status; + pid_t child, result; + + if (argc != 2 || strcmp(argv[1], "--native-16k")) + return 2; + signal(SIGALRM, stop_fixture); + signal(SIGTERM, stop_fixture); + signal(SIGINT, stop_fixture); + ksft_print_header(); + if (getauxval(AT_PAGESZ) != LEAF) + ksft_exit_skip("run under a selected 4K userspace ABI\n"); + ksft_set_plan(6); + fflush(NULL); + alarm(40); + for (backing = ANONYMOUS; backing <= REGULAR; backing++) { + for (small = 0; small <= 1; small++) { + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork: %s\n", strerror(errno)); + if (!child) { + running_child = 0; + alarm(15); + _exit(exercise(backing, small)); + } + running_child = child; + do { + result = waitpid(child, &status, 0); + } while (result < 0 && errno == EINTR); + running_child = 0; + if (result > 0 && WIFEXITED(status) && WEXITSTATUS(status) == KSFT_SKIP) + ksft_test_result_skip("%s %s first writes\n", names[backing], + small ? "guarded" : "dense"); + else + ksft_test_result(result > 0 && WIFEXITED(status) && + !WEXITSTATUS(status), + "%s %s first writes and slot reuse preserve contents\n", + names[backing], small ? "guarded" : "dense"); + } + } + ksft_finished(); +} From bbfbff2a670a4b71bfe5385af50b179dfc5596d8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 19:28:15 -0400 Subject: [PATCH 300/631] mm: pack pinned-fork subpage copies without changing their charge domain A fork that must copy pinned 4 KiB leaves previously allocated one native 16 KiB folio per leaf. Reserve copies from the child mm's subpage pools, while retaining the source mm for the existing fork allocation charge. Preflight non-linear rmap before taking the page-table locks and copy from the actual source offset after revalidating the parent PTE. Add a bounded real io_uring buffer-pin regression that verifies distinct parent/child native owners, full two-way byte isolation and packed density. On Neo, a 1 MiB copy fell from 4 MiB to 1 MiB of native backing. A held non-root cgroup check verified every payload native PFN retained the correct charge. Retained MM, actual reclaim/readback and GPU controls pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 6ec0621cc97679bfa90f005c4e0666b319783842) --- include/linux/mm_subpage.h | 4 + mm/memory.c | 74 ++++++-- mm/user-subpage.c | 13 +- .../testing/selftests/arm64/user4k/.gitignore | 1 + tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/README.pinned-fork | 37 ++++ .../selftests/arm64/user4k/pinned-fork.c | 172 ++++++++++++++++++ 7 files changed, 285 insertions(+), 18 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.pinned-fork create mode 100644 tools/testing/selftests/arm64/user4k/pinned-fork.c diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 910ed2b36bcfd1..4a6224f8cc95a4 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -49,6 +49,10 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool); /* Optional per-mm pool: default policy, one memory node, matching memcg. */ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address); +/* Fork's child owns the pool before it joins a cgroup; charge the source mm. */ +struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, + struct mm_struct *charge_mm, struct vm_area_struct *vma, + unsigned long address); void mm_subpage_cow_pool_exit(struct mm_struct *mm); bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio); bool mm_subpage_cow_wait_busy_at(struct mm_subpage_pool *pool, unsigned int offset); diff --git a/mm/memory.c b/mm/memory.c index 77735be88aed57..ea603a72be93d3 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1125,7 +1125,7 @@ static void subpage_add_new_lru(struct folio *folio, struct vm_area_struct *vma, mlock_folio(folio); } -/* The destination slot and its charge/metadata were allocated outside PTLs. */ +/* Destination allocation, folio locking and rmap preflight precede both PTLs. */ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, pte_t source_pte, unsigned long addr, int *rss, struct mm_subpage *source, struct mm_subpage **prealloc) @@ -1133,23 +1133,26 @@ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, struct mm_subpage *dest = *prealloc; struct folio *new; pte_t pte; + bool fresh; int ret; if (!dest) return -EAGAIN; new = mm_subpage_folio(dest); + VM_BUG_ON_FOLIO(!folio_test_locked(new), new); ret = copy_user_subpage_range(&new->page, mm_subpage_offset(dest), &mm_subpage_folio(source)->page, mm_subpage_offset(source), mm_subpage_size(source)); if (ret) return ret; - /* Fresh, unmapped backing is private to this preallocation. */ - if (WARN_ON_ONCE(!folio_trylock(new))) - return -EBUSY; + fresh = !mm_subpage_anon_root(dest); + mm_subpage_set_exclusive(dest); - ret = mm_subpage_add_new_anon_rmap(dest, dst_vma, addr); + ret = fresh ? mm_subpage_add_new_anon_rmap(dest, dst_vma, addr) : + mm_subpage_add_anon_rmap(dest, dst_vma, addr); if (!ret) { - subpage_add_new_lru(new, dst_vma, 1); + if (fresh) + subpage_add_new_lru(new, dst_vma, 1); rss[MM_ANONPAGES]++; pte = phys_pte_mm(dst_vma->vm_mm, mm_subpage_phys(dest), dst_vma->vm_page_prot); pte = maybe_mkwrite(pte_mkdirty(pte), dst_vma); @@ -1157,8 +1160,9 @@ static int copy_pinned_subpage(struct vm_area_struct *dst_vma, pte_t *dst_pte, pte = pte_mkuffd_wp(pte); set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); *prealloc = NULL; + folio_unlock(new); } - folio_unlock(new); + /* An unconsumed reservation remains locked until the caller drops PTLs. */ return ret; } #endif @@ -1348,7 +1352,8 @@ static struct folio *cow_subpage_speculative_folio(struct mm_struct *mm, /* The returned initialized slot owns a reference and has its folio locked. */ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, - struct mm_subpage *source, bool force_private, bool *pooled) + struct mm_subpage *source, bool force_private, bool *pooled, + struct mm_struct *charge_mm) { struct mm_subpage_pool *pool; struct mm_subpage *slot; @@ -1357,7 +1362,8 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, int err; retry_private: - pool = force_private ? NULL : mm_subpage_cow_pool_get(mm, vma, addr); + pool = force_private ? NULL : + mm_subpage_cow_pool_get_charged(mm, charge_mm, vma, addr); *pooled = pool != NULL; if (!pool) pool = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); @@ -1374,7 +1380,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, break; folio = NULL; if (*pooled) { - folio = cow_subpage_speculative_folio(mm, vma, addr); + folio = cow_subpage_speculative_folio(charge_mm, vma, addr); if (!folio && mm_subpage_cow_wait_busy(pool)) { cond_resched(); continue; @@ -1384,7 +1390,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, * Ordinary allocation/charge semantics are now appropriate. */ if (!folio) - folio = folio_prealloc(mm, vma, addr, false); + folio = folio_prealloc(charge_mm, vma, addr, false); if (!folio) { slot = ERR_PTR(-ENOMEM); break; @@ -1423,6 +1429,30 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, return slot; } +/* The child owns the pool; fork retains the source mm allocation charge. */ +static struct mm_subpage *fork_subpage_prealloc(struct vm_area_struct *dst_vma, + struct mm_struct *src_mm, unsigned long addr) +{ + struct mm_subpage *slot; + bool pooled, force_private = false; + int err; + + for (;;) { + slot = cow_subpage_prealloc(dst_vma->vm_mm, dst_vma, addr, + NULL, force_private, &pooled, src_mm); + if (IS_ERR(slot)) + return slot; + err = mm_subpage_prepare_anon_rmap(slot, dst_vma, addr); + if (!err) + return slot; + folio_unlock(mm_subpage_folio(slot)); + mm_subpage_put(slot); + if (!pooled || err != -ENOMEM) + return ERR_PTR(err); + force_private = true; + } +} + static struct mm_subpage *anon_subpage_prealloc(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr) { @@ -1467,6 +1497,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, softleaf_t entry = softleaf_mk_none(); struct folio *prealloc = NULL; struct mm_subpage *subprealloc = NULL; + unsigned long subprealloc_addr __maybe_unused = 0; int nr; again: @@ -1507,6 +1538,14 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, do { nr = 1; +#ifdef CONFIG_MM_SUBPAGE + /* A vanished source PTE must not move a prepared slot to another VA. */ + if (subprealloc && addr != subprealloc_addr) { + folio_unlock(mm_subpage_folio(subprealloc)); + mm_subpage_put(subprealloc); + subprealloc = NULL; + } +#endif /* * We are holding two locks at this point - either of them @@ -1560,6 +1599,7 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, break; #ifdef CONFIG_MM_SUBPAGE if (subprealloc) { + folio_unlock(mm_subpage_folio(subprealloc)); mm_subpage_put(subprealloc); subprealloc = NULL; } @@ -1597,7 +1637,8 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, } else if (ret == -EAGAIN) { #ifdef CONFIG_MM_SUBPAGE if (mm_page_size(src_mm) < PAGE_SIZE) { - subprealloc = anon_subpage_prealloc(src_mm, src_vma, addr); + subprealloc_addr = addr; + subprealloc = fork_subpage_prealloc(dst_vma, src_mm, addr); if (IS_ERR(subprealloc)) return PTR_ERR(subprealloc); ret = 0; @@ -1621,8 +1662,10 @@ copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, goto again; out: #ifdef CONFIG_MM_SUBPAGE - if (subprealloc) + if (subprealloc) { + folio_unlock(mm_subpage_folio(subprealloc)); mm_subpage_put(subprealloc); + } #endif if (unlikely(prealloc)) folio_put(prealloc); @@ -4695,7 +4738,8 @@ static vm_fault_t wp_subpage_copy(struct vm_fault *vmf) goto out; ret = VM_FAULT_OOM; retry_copy: - dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, &pooled); + dest = cow_subpage_prealloc(mm, vma, vmf->address, source, force_private, + &pooled, mm); if (IS_ERR(dest)) { if (PTR_ERR(dest) == -EHWPOISON) ret = VM_FAULT_HWPOISON; @@ -7520,7 +7564,7 @@ static vm_fault_t do_cow_fault(struct vm_fault *vmf) if (!force_private && !userfaultfd_armed(vma) && (vma->vm_ops->fault == filemap_fault || vma_is_shmem(vma))) { vmf->cow_subpage = cow_subpage_prealloc(vma->vm_mm, vma, - vmf->address, NULL, false, &pooled); + vmf->address, NULL, false, &pooled, vma->vm_mm); if (!IS_ERR(vmf->cow_subpage)) folio_unlock(mm_subpage_folio(vmf->cow_subpage)); } else { diff --git a/mm/user-subpage.c b/mm/user-subpage.c index fd553493b3de4e..acfa80848e6e35 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -299,8 +299,9 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. * An mm can move cgroups; old charged slots must not satisfy its new faults. */ -struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, - struct vm_area_struct *vma, unsigned long address) +struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, + struct mm_struct *charge_mm, struct vm_area_struct *vma, + unsigned long address) { struct mm_subpage_cow_pools *pools, *new_pools; struct mm_subpage_pool *pool, *new; @@ -334,7 +335,7 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, else kfree(new_pools); } - memcg = get_mem_cgroup_from_mm(mm); + memcg = get_mem_cgroup_from_mm(charge_mm); /* Acquire the immutable charge domain published with this bucket. */ pool = smp_load_acquire(&pools->pool[bucket]); if (!pool) { @@ -375,6 +376,12 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, return pool; } +struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, + struct vm_area_struct *vma, unsigned long address) +{ + return mm_subpage_cow_pool_get_charged(mm, mm, vma, address); +} + bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) { return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 85e45b6ce0399d..a5b225dbaf61db 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -6,3 +6,4 @@ file-cow-consumers same-native first-write clipped-firstwrite +pinned-fork diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index dd9d4f233b0562..a6e4854f454c27 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -14,6 +14,8 @@ TEST_GEN_PROGS_EXTENDED += same-quarter file-cow-consumers same-native TEST_FILES += README.any-quarter check-any-quarter.py TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite TEST_FILES += README.first-write +TEST_GEN_PROGS_EXTENDED += pinned-fork +TEST_FILES += README.pinned-fork include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S diff --git a/tools/testing/selftests/arm64/user4k/README.pinned-fork b/tools/testing/selftests/arm64/user4k/README.pinned-fork new file mode 100644 index 00000000000000..129adde8b005c1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.pinned-fork @@ -0,0 +1,37 @@ +Pinned anonymous fork density and pin isolation +============================================= + +Build pinned-fork with the other arm64/user4k tests. Run as root through the +established 4K userspace launcher on a native16K kernel: + + userpages 4k ./pinned-fork --native-16k --expect-packed + +The fixture initializes one native-aligned1MiB anonymous range, registers real +writable long-term io_uring buffer pins, then forks while the parent retains the +registration. It requires every copied child native PFN to be disjoint from the +pinned parent's native PFNs. It checks every byte before and after distinct +parent/child writes; registration and PFN disjointness, not mlock or VmPin alone, +establish copy-path coverage. Kernel/pin-policy limitations return77 UNEXERCISED. + +--expect-packed requires child backing <=1.25MiB for1MiB copied data, with bounded +pool-shard slack. Without this option the fixture also accepts the old4MiB +backing so a baseline can record both content and density separately. Successful +contents alone do not establish packed density. --selftest checks metadata PFN +grouping only, with no dynamic pin/fork qualification. + +Optional --hold reports the parent PID, child PID and range base after fork PFN +verification, then waits for the byte 'c' on stdin. A controller can hold both +processes in one fresh uncapped non-root cgroup and verify exact payload charges: +read each target's pagemap in4KiB units; normalize PFN>>2 to native16KiB PFN; +read /proc/kpagecgroup at native_PFN*8 and require each returned inode equals +stat(the fixture cgroup).st_ino. Re-read pagemap to reject migration during the +sample. This directly catches incorrectly root-charged fork copies; total +memory.current includes unrelated fixture code/stack/table costs and is not an +exact payload charge oracle. Verify both PIDs belong to the same owned cgroup, +record memory.stat/status, and release promptly (within3seconds). + +Parent15s/child10s alarms bound the protocol; parent exit kills/reaps the child +and the child uses PDEATHSIG. Payload native backing is at most5MiB on the old +path. No forced reclaim, memory.max/high changes, GPU operations, or swap-area +changes are used. Controllers should additionally enforce their own process-group +or service-unit lifetime and preserve logs on any failure. diff --git a/tools/testing/selftests/arm64/user4k/pinned-fork.c b/tools/testing/selftests/arm64/user4k/pinned-fork.c new file mode 100644 index 00000000000000..ec91136af8de42 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pinned-fork.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define NATIVE 16384UL +#define SIZE (1024UL * 1024) +#define COUNT (SIZE / LEAF) +#define PFNMASK ((1ULL << 55) - 1) +static pid_t child = -1; +static uint64_t parent_pfn[COUNT], child_pfn[COUNT]; +static void die(const char *s) { perror(s); exit(1); } +static void cleanup(void) +{ + if (child > 0) { + kill(child, SIGKILL); + while (waitpid(child, NULL, 0) < 0 && errno == EINTR) {} + } +} +static void timeout(int sig) +{ + (void)sig; + if (child > 0) kill(child, SIGKILL); + _exit(124); +} +static void transfer(int fd, void *buffer, size_t len, int writing) +{ + unsigned char *p = buffer; + while (len) { + ssize_t n = writing ? write(fd, p, len) : read(fd, p, len); + if (n < 0 && errno == EINTR) continue; + if (n <= 0) die(writing ? "write protocol" : "read protocol"); + p += n; len -= n; + } +} +static unsigned char pattern(size_t i) { return (unsigned char)(1 + (i * 17 + i / LEAF) % 251); } +static void check(unsigned char *p, int mode) +{ + for (size_t i = 0; i < SIZE; i++) { + unsigned char expected = pattern(i); + if ((mode == 1 && i % LEAF == 0) || (mode == 2 && i % LEAF == 1)) expected ^= 0x80; + if (p[i] != expected) { + fprintf(stderr, "FAIL bytes mode=%d offset=%zu expected=%u actual=%u\n", mode, i, expected, p[i]); + exit(1); + } + } +} +static void pfns(unsigned char *p, uint64_t *out) +{ + int fd = open("/proc/self/pagemap", O_RDONLY | O_CLOEXEC); + if (fd < 0) die("pagemap open"); + for (size_t i = 0; i < COUNT; i++) { + uint64_t e; + if (pread(fd, &e, sizeof(e), (((uintptr_t)p / LEAF) + i) * sizeof(e)) != sizeof(e)) die("pagemap read"); + if (!(e >> 63) || (e & (1ULL << 62)) || !(e & PFNMASK)) { + fprintf(stderr, "FAIL pagemap leaf=%zu entry=%016llx (root PFNs required)\n", i, (unsigned long long)e); exit(1); + } + out[i] = e & PFNMASK; + } + close(fd); +} +static size_t native_count(uint64_t *pf) +{ + size_t n = 0; + for (size_t i = 0; i < COUNT; i++) { + size_t j; + for (j = 0; j < i; j++) if (pf[j] / 4 == pf[i] / 4) break; + if (j == i) n++; + } + return n; +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + for (size_t i = 0; i < COUNT; i++) parent_pfn[i] = 4 + i; + if (native_count(parent_pfn) != COUNT / 4) return 1; + for (size_t i = 0; i < COUNT; i++) parent_pfn[i] = 4 + 4 * i; + if (native_count(parent_pfn) != COUNT) return 1; + puts("PASS native PFN grouping selftest (no pin/fork coverage)"); return 0; + } + int packed = 0, hold = 0; + if (argc < 2 || strcmp(argv[1], "--native-16k") || getauxval(AT_PAGESZ) != LEAF) return 77; + for (int i = 2; i < argc; i++) { + if (!strcmp(argv[i], "--expect-packed")) packed = 1; + else if (!strcmp(argv[i], "--hold")) hold = 1; + else return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + signal(SIGALRM, timeout); signal(SIGPIPE, SIG_IGN); alarm(15); atexit(cleanup); + unsigned char *raw = mmap(NULL, SIZE + NATIVE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) die("mmap"); + unsigned char *p = (void *)(((uintptr_t)raw + NATIVE - 1) & ~(NATIVE - 1)); + for (size_t i = 0; i < SIZE; i++) p[i] = pattern(i); + check(p, 0); pfns(p, parent_pfn); + if (native_count(parent_pfn) * NATIVE != SIZE) die("initial dense native backing"); + struct io_uring_params params = {0}; + int ring = syscall(__NR_io_uring_setup, 2, ¶ms); + if (ring < 0) { perror("UNEXERCISED io_uring_setup"); return 77; } + struct iovec iov = { .iov_base = p, .iov_len = SIZE }; + if (syscall(__NR_io_uring_register, ring, IORING_REGISTER_BUFFERS, &iov, 1)) { + perror("UNEXERCISED register real long-term pins"); close(ring); return 77; + } + /* Long-term pin setup can migrate backing; capture the pinned identity. */ + pfns(p, parent_pfn); check(p, 0); + if (native_count(parent_pfn) * NATIVE != SIZE) die("pinned dense native backing"); + int to_child[2], from_child[2]; + if (pipe(to_child) || pipe(from_child)) die("pipe"); + pid_t parent = getpid(); + child = fork(); + if (child < 0) die("fork"); + if (!child) { + prctl(PR_SET_PDEATHSIG, SIGKILL); + if (getppid() != parent) _exit(1); + alarm(10); close(ring); close(to_child[1]); close(from_child[0]); + pfns(p, child_pfn); check(p, 0); + transfer(from_child[1], child_pfn, sizeof(child_pfn), 1); + char token; transfer(to_child[0], &token, 1, 0); + if (token != 'w') die("child token"); + check(p, 0); + for (size_t i = 0; i < COUNT; i++) p[i * LEAF + 1] ^= 0x80; + check(p, 2); + token = 'c'; transfer(from_child[1], &token, 1, 1); + transfer(to_child[0], &token, 1, 0); + if (token != 'e') die("child exit token"); + check(p, 2); _exit(0); + } + close(to_child[0]); close(from_child[1]); + transfer(from_child[0], child_pfn, sizeof(child_pfn), 0); + for (size_t i = 0; i < COUNT; i++) + for (size_t j = 0; j < COUNT; j++) + if (child_pfn[i] / 4 == parent_pfn[j] / 4) die("child copy overlaps pinned parent backing"); + printf("METADATA real_registered_pin=1 parent_native=%zu child_native=%zu copied_leaves=%lu native_overlap=0\n", native_count(parent_pfn) * NATIVE, native_count(child_pfn) * NATIVE, COUNT); + if (hold) { + printf("HOLD parent=%d child=%d base=%lx bytes=%lu (write c to continue)\n", getpid(), child, (unsigned long)p, SIZE); + char ack; transfer(STDIN_FILENO, &ack, 1, 0); + if (ack != 'c') die("controller hold token"); + } + size_t child_bytes = native_count(child_pfn) * NATIVE; + if (child_bytes < SIZE || child_bytes > COUNT * NATIVE || + (packed && child_bytes > SIZE + 256 * 1024)) die("child density bound"); + for (size_t i = 0; i < COUNT; i++) p[i * LEAF] ^= 0x80; + check(p, 1); + char token = 'w'; transfer(to_child[1], &token, 1, 1); + transfer(from_child[0], &token, 1, 0); + if (token != 'c') die("parent token"); + check(p, 1); + token = 'e'; transfer(to_child[1], &token, 1, 1); + int status; + if (waitpid(child, &status, 0) != child) die("waitpid"); + child = -1; + if (!WIFEXITED(status) || WEXITSTATUS(status)) die("child failed"); + if (syscall(__NR_io_uring_register, ring, IORING_UNREGISTER_BUFFERS, NULL, 0)) die("unregister"); + close(ring); close(to_child[1]); close(from_child[0]); + if (munmap(raw, SIZE + NATIVE)) die("munmap"); + puts("PASS real pin/fork copy and full-byte two-way isolation; density is separately reported"); + return 0; +} From 88aa35edcc05ebfdbc6961cb4697e6af5c33c645 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 19:29:34 -0400 Subject: [PATCH 301/631] arm64: reuse cached table fragments after allocation races Concurrent small-table allocators can all miss the fragment cache before allocating a native owner. Recheck the cache after construction and consume its reserved fragment if another allocator supplied it. Destroy the unused fresh owner instead of returning one table and stranding its other slots. Add bounded serial and concurrent backing fixtures with separate logical VmPTE and physical-charge checkpoints. On Neo the same concurrent fixture fell from 5632 to 4192 KiB physical pagetables for approximately 4160 KiB logical tables. All contents, retained MM, reclaim/readback and GPU controls pass. Issued-fragment retention after unmap is a separate unresolved case. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit fd8b40cc03751f3d4128be7d09c855ae1c7e5498) --- arch/arm64/mm/user4k-pgalloc.c | 26 ++- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../arm64/user4k/README.pagetable-backing | 33 ++++ .../arm64/user4k/pagetable-concurrent.c | 161 ++++++++++++++++++ .../arm64/user4k/pagetable-retention.c | 118 +++++++++++++ 6 files changed, 339 insertions(+), 5 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.pagetable-backing create mode 100644 tools/testing/selftests/arm64/user4k/pagetable-concurrent.c create mode 100644 tools/testing/selftests/arm64/user4k/pagetable-retention.c diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c index 0bcaab5c3a8d28..21d27c1656f769 100644 --- a/arch/arm64/mm/user4k-pgalloc.c +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -98,12 +98,30 @@ void *arm64_user_pt_alloc_granule(struct mm_struct *mm, atomic_set(&pt->pt_frag_refcount, 1); table = ptdesc_address(pt); spin_lock(&mm->page_table_lock); - if (nr_fragments > 1 && !mm->context.user4k_pt_frag[level]) { - atomic_set(&pt->pt_frag_refcount, nr_fragments); - mm->context.user4k_pt_frag[level] = table + frag_size; + if (nr_fragments > 1) { + next = mm->context.user4k_pt_frag[level]; + if (next) { + /* Another allocator supplied backing while we constructed + * a table owner. Consume its unused fragment rather than + * stranding the rest of our new native page. + */ + if (virt_to_ptdesc(next)->pt_frag_shift != shift) { + spin_unlock(&mm->page_table_lock); + fragment_owner_free(pt); + return NULL; + } + table = next; + next += frag_size; + mm->context.user4k_pt_frag[level] = + ((unsigned long)next & ~PAGE_MASK) ? next : NULL; + } else { + atomic_set(&pt->pt_frag_refcount, nr_fragments); + mm->context.user4k_pt_frag[level] = table + frag_size; + } } - /* A racing allocation may own the cache; this page then has one user. */ spin_unlock(&mm->page_table_lock); + if (table != ptdesc_address(pt)) + fragment_owner_free(pt); return table; } diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index a5b225dbaf61db..3c4c8c86eefe2c 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -7,3 +7,5 @@ same-native first-write clipped-firstwrite pinned-fork +pagetable-retention +pagetable-concurrent diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index a6e4854f454c27..d306deb1f00544 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -16,11 +16,13 @@ TEST_GEN_PROGS_EXTENDED += first-write clipped-firstwrite TEST_FILES += README.first-write TEST_GEN_PROGS_EXTENDED += pinned-fork TEST_FILES += README.pinned-fork +TEST_GEN_PROGS_EXTENDED += pagetable-retention pagetable-concurrent +TEST_FILES += README.pagetable-backing include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.pagetable-backing b/tools/testing/selftests/arm64/user4k/README.pagetable-backing new file mode 100644 index 00000000000000..2c6b16b17465a1 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.pagetable-backing @@ -0,0 +1,33 @@ +Page-table physical backing and fragment retention +================================================== + +These bounded fixtures reserve 2 GiB of virtual address space and write one +4 KiB leaf in each of 1,024 distinct 2 MiB table windows. Native payload +backing is at most 16 MiB, with approximately 4 MiB of logical PTE tables. +The concurrent variant uses 32 threads, 32 synchronized fault rounds and +128 KiB thread stacks. Both verify all payload bytes across six checkpoints: +baseline, full, quarter_survivors, refilled, quarter_survivors_again, unmapped. + +Run through the 4 KiB ABI launcher on a native 16 KiB kernel. Each JSON +checkpoint requires one byte 'c' on stdin before continuing. For contents +only, with no physical charge observation: + + yes c | userpages 4k ./pagetable-retention --native-16k + yes c | userpages 4k ./pagetable-concurrent --native-16k + +For physical accounting, run each fixture alone in a fresh uncapped cgroup. +At each checkpoint verify the fixture is the only member, record memory.stat +pagetables and memory.swap.current, and sample at 0, 1 and 3 seconds before +acknowledging. Record logical VmPTE separately: it is not physical backing. +Use an external service lifetime below the fixture's 60-second alarm and +verify all processes exit. Do not change memory.max/high or force reclaim. + +Compare the same complete fixture across kernels. Concurrent thread stacks +and upper-level tables contribute to the measurements; do not subtract them +as though they were pure payload. Deferred table releases can occur between +samples. Contents PASS does not assert a density or performance result. + +The allocation-race fix reduces initial concurrent physical backing. It does +not reclaim partially occupied table owners or reuse issued fragments after +unmap. Retention/refill checkpoints deliberately preserve that separate +coverage gap rather than declaring it fixed. diff --git a/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c b/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c new file mode 100644 index 00000000000000..2573a8882e1651 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pagetable-concurrent.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define WINDOW (2UL * 1024 * 1024) +#define WINDOWS 1024UL +#define LENGTH (WINDOWS * WINDOW) + +static void fail(const char *why) +{ + perror(why); + exit(1); +} + +static unsigned char tag(size_t i) +{ + return (unsigned char)(1 + i % 251); +} + +static void checkpoint(const char *phase) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256], ack; + unsigned long pte = 0; + if (!f) + fail("status"); + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPTE: %lu kB", &pte) == 1) + break; + fclose(f); + printf("{\"phase\":\"%s\",\"pid\":%d,\"VmPTE_bytes\":%lu}\n", + phase, getpid(), pte * 1024); + if (read(STDIN_FILENO, &ack, 1) != 1 || ack != 'c') + fail("controller ack"); +} + +static void touch_serial(unsigned char *base, int missing_only) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (missing_only && !(i % 4)) + continue; + if (missing_only) { + void *p = mmap(base + i * WINDOW, WINDOW, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + if (p != base + i * WINDOW) + fail("refill mmap"); + } + memset(base + i * WINDOW, tag(i), LEAF); + } +} + +#define WORKERS 32 +static pthread_barrier_t step; +static atomic_int start_workers; +struct worker { unsigned char *base; unsigned id; }; +static void *fault_worker(void *arg) +{ + struct worker *w = arg; + int start; + do { start = atomic_load_explicit(&start_workers, memory_order_acquire); + if (!start) sched_yield(); } while (!start); + if (start != 1) return NULL; + for (unsigned round = 0; round < WINDOWS / WORKERS; round++) { + int ret = pthread_barrier_wait(&step); + if (ret && ret != PTHREAD_BARRIER_SERIAL_THREAD) abort(); + size_t i = round * WORKERS + w->id; + memset(w->base + i * WINDOW, tag(i), LEAF); + } + return NULL; +} +static void touch(unsigned char *base, int missing_only) +{ + if (missing_only) { touch_serial(base, missing_only); return; } + pthread_t threads[WORKERS]; + struct worker args[WORKERS]; + pthread_attr_t attr; + if (pthread_attr_init(&attr) || pthread_attr_setstacksize(&attr, 128 * 1024) || + pthread_barrier_init(&step, NULL, WORKERS)) fail("thread setup"); + unsigned made = 0; + for (; made < WORKERS; made++) { + args[made] = (struct worker){.base = base, .id = made}; + int err = pthread_create(&threads[made], &attr, fault_worker, &args[made]); + if (err) { errno = err; break; } + } + pthread_attr_destroy(&attr); + atomic_store_explicit(&start_workers, made == WORKERS ? 1 : 2, memory_order_release); + for (unsigned i = 0; i < made; i++) if (pthread_join(threads[i], NULL)) abort(); + pthread_barrier_destroy(&step); + if (made != WORKERS) fail("pthread_create"); +} + +static void verify(unsigned char *base, int sparse) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (sparse && i % 4) + continue; + for (size_t j = 0; j < LEAF; j++) + if (base[i * WINDOW + j] != tag(i)) { + fprintf(stderr, "FAIL content window=%zu byte=%zu\n", i, j); + exit(1); + } + } +} + +static void thin(unsigned char *base) +{ + for (size_t i = 0; i < WINDOWS; i++) + if (i % 4 && munmap(base + i * WINDOW, WINDOW)) + fail("thin munmap"); + verify(base, 1); +} + +int main(int argc, char **argv) +{ + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF) { + fputs("Requires established 4K launcher on native16K kernel\n", stderr); + return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + alarm(60); + checkpoint("baseline"); + unsigned char *raw = mmap(NULL, LENGTH + WINDOW, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) + fail("reserve mmap"); + unsigned char *base = (void *)(((uintptr_t)raw + WINDOW - 1) & ~(WINDOW - 1)); + size_t head = base - raw, tail = WINDOW - head; + if (head && munmap(raw, head)) + fail("trim head"); + if (tail && munmap(base + LENGTH, tail)) + fail("trim tail"); + touch(base, 0); + verify(base, 0); + checkpoint("full"); + thin(base); + checkpoint("quarter_survivors"); + touch(base, 1); + verify(base, 0); + checkpoint("refilled"); + thin(base); + checkpoint("quarter_survivors_again"); + if (munmap(base, LENGTH)) + fail("final munmap"); + checkpoint("unmapped"); + puts("PASS contents, unmap and refill; concurrent firstfaults, accounting is a separate observation"); + return 0; +} diff --git a/tools/testing/selftests/arm64/user4k/pagetable-retention.c b/tools/testing/selftests/arm64/user4k/pagetable-retention.c new file mode 100644 index 00000000000000..09f18c5424ef8f --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/pagetable-retention.c @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +#define LEAF 4096UL +#define WINDOW (2UL * 1024 * 1024) +#define WINDOWS 1024UL +#define LENGTH (WINDOWS * WINDOW) + +static void fail(const char *why) +{ + perror(why); + exit(1); +} + +static unsigned char tag(size_t i) +{ + return (unsigned char)(1 + i % 251); +} + +static void checkpoint(const char *phase) +{ + FILE *f = fopen("/proc/self/status", "re"); + char line[256], ack; + unsigned long pte = 0; + if (!f) + fail("status"); + while (fgets(line, sizeof(line), f)) + if (sscanf(line, "VmPTE: %lu kB", &pte) == 1) + break; + fclose(f); + printf("{\"phase\":\"%s\",\"pid\":%d,\"VmPTE_bytes\":%lu}\n", + phase, getpid(), pte * 1024); + if (read(STDIN_FILENO, &ack, 1) != 1 || ack != 'c') + fail("controller ack"); +} + +static void touch(unsigned char *base, int missing_only) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (missing_only && !(i % 4)) + continue; + if (missing_only) { + void *p = mmap(base + i * WINDOW, WINDOW, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, + -1, 0); + if (p != base + i * WINDOW) + fail("refill mmap"); + } + memset(base + i * WINDOW, tag(i), LEAF); + } +} + +static void verify(unsigned char *base, int sparse) +{ + for (size_t i = 0; i < WINDOWS; i++) { + if (sparse && i % 4) + continue; + for (size_t j = 0; j < LEAF; j++) + if (base[i * WINDOW + j] != tag(i)) { + fprintf(stderr, "FAIL content window=%zu byte=%zu\n", i, j); + exit(1); + } + } +} + +static void thin(unsigned char *base) +{ + for (size_t i = 0; i < WINDOWS; i++) + if (i % 4 && munmap(base + i * WINDOW, WINDOW)) + fail("thin munmap"); + verify(base, 1); +} + +int main(int argc, char **argv) +{ + if (argc != 2 || strcmp(argv[1], "--native-16k") || + getauxval(AT_PAGESZ) != LEAF) { + fputs("Requires established 4K launcher on native16K kernel\n", stderr); + return 77; + } + setvbuf(stdout, NULL, _IONBF, 0); + alarm(60); + checkpoint("baseline"); + unsigned char *raw = mmap(NULL, LENGTH + WINDOW, + PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) + fail("reserve mmap"); + unsigned char *base = (void *)(((uintptr_t)raw + WINDOW - 1) & ~(WINDOW - 1)); + size_t head = base - raw, tail = WINDOW - head; + if (head && munmap(raw, head)) + fail("trim head"); + if (tail && munmap(base + LENGTH, tail)) + fail("trim tail"); + touch(base, 0); + verify(base, 0); + checkpoint("full"); + thin(base); + checkpoint("quarter_survivors"); + touch(base, 1); + verify(base, 0); + checkpoint("refilled"); + thin(base); + checkpoint("quarter_survivors_again"); + if (munmap(base, LENGTH)) + fail("final munmap"); + checkpoint("unmapped"); + puts("PASS contents, unmap and refill; accounting is a separate observation"); + return 0; +} From b464748a1b3d2dd21922e1c028030f80d83211db Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 20:29:26 -0400 Subject: [PATCH 302/631] arm64: reuse retired small page-table fragments Keep an independent per-mm, per-level pool of native table owners so holes left by unmap can satisfy subsequent small-table allocations. Retain issued references until RCU retirement, clear dirty slots only when reused, and release empty owners. Pool references outlive mm teardown for callbacks. On Neo, serial refill physical tables fall from 7,421,952 to 4,276,224 bytes; concurrent refill falls from 6,275,072 to 4,276,224. Contents, teardown, pinned-copy charges, repeated MM, actual swap readback and GPU controls pass. Host allocator checks cover native 4K/16K/64K primitives; maintained KUnit code compiles but was not enabled in the hardware kernel. Seven-round operation timing has overlapping ranges and about 1% higher medians; no zero-overhead claim. Live-table compaction remains separate. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit e08fea2abe7ea51544718e47d1f14706ce4c630e) --- arch/arm64/include/asm/mmu.h | 2 +- arch/arm64/mm/user4k-pgalloc.c | 282 +++++++++++++----- arch/arm64/mm/user4k-test.c | 160 +++------- .../arm64/user4k/README.pagetable-backing | 20 +- 4 files changed, 255 insertions(+), 209 deletions(-) diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 15943bec53392e..da82ed5f0c41ff 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -39,7 +39,7 @@ typedef struct { unsigned long flags; u8 pkey_allocation_map; #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL - /* Unissued PTE/PMD/PUD fragments; protected by page_table_lock. */ + /* PTE/PMD/PUD fragment pools; initialized before atomic publication. */ void *user4k_pt_frag[3]; #endif } mm_context_t; diff --git a/arch/arm64/mm/user4k-pgalloc.c b/arch/arm64/mm/user4k-pgalloc.c index 21d27c1656f769..7a3e34205760c0 100644 --- a/arch/arm64/mm/user4k-pgalloc.c +++ b/arch/arm64/mm/user4k-pgalloc.c @@ -1,20 +1,38 @@ // SPDX-License-Identifier: GPL-2.0-only -/* - * Hardware tables have immutable granule/owner metadata. Smaller tables share - * a native page; larger tables own a compound allocation. Only unissued slots - * in a shared native page are cached by the mm. - */ +/* Smaller hardware tables share native owners, including safely retired slots. */ #include #include #include #include +#include #include +struct fragment_pool { + spinlock_t lock; + refcount_t refs; + struct list_head available; + unsigned int shift; + bool closed; +}; + +struct fragment_slot { + struct rcu_head rcu; + void *table; +}; + +struct fragment_owner { + struct fragment_pool *pool; + struct ptdesc *pt; + struct list_head available; + unsigned long free_mask; + unsigned long dirty_mask; + struct fragment_slot slots[]; +}; + unsigned int arm64_ptep_page_shift(pte_t *ptep) { struct ptdesc *pt = virt_to_ptdesc(ptep); - /* Table owners retain geometry through their last RCU reader. */ return test_bit(PT_fragmented, &pt->pt_flags.f) ? pt->pt_frag_shift : PAGE_SHIFT; } EXPORT_SYMBOL_GPL(arm64_ptep_page_shift); @@ -30,49 +48,47 @@ static void fragment_owner_free_rcu(struct rcu_head *rcu) fragment_owner_free(container_of(rcu, struct ptdesc, pt_rcu_head)); } -static void fragment_put(struct ptdesc *pt, unsigned int nr) +static void fragment_pool_put(struct fragment_pool *pool) { - if (!atomic_sub_and_test(nr, &pt->pt_frag_refcount)) - return; - if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) - call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); - else - fragment_owner_free(pt); + if (refcount_dec_and_test(&pool->refs)) { + WARN_ON_ONCE(!list_empty(&pool->available)); + kfree(pool); + } } -void *arm64_user_pt_alloc_granule(struct mm_struct *mm, - enum arm64_user4k_pt_level level, unsigned int shift) +/* Active table allocation already holds the mm alive. Its cache reference + * remains until destroy_context, after all allocators have stopped. Owners + * independently retain the pool for callbacks after that point. + */ +static struct fragment_pool *fragment_pool_get(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) { - struct ptdesc *pt; - void *table, *next; - bool constructed; - unsigned int frag_size, nr_fragments, order; + struct fragment_pool *pool, *new; - if (shift != 12 && shift != 14 && shift != 16) - return NULL; - frag_size = 1U << shift; - order = shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0; - nr_fragments = (PAGE_SIZE << order) >> shift; - if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + pool = smp_load_acquire(&mm->context.user4k_pt_frag[level]); + if (pool) + return pool->shift == shift ? pool : NULL; + new = kzalloc(sizeof(*new), GFP_KERNEL | __GFP_ACCOUNT); + if (!new) return NULL; + spin_lock_init(&new->lock); + refcount_set(&new->refs, 1); + INIT_LIST_HEAD(&new->available); + new->shift = shift; + pool = cmpxchg(&mm->context.user4k_pt_frag[level], NULL, new); + if (!pool) + return new; + fragment_pool_put(new); + return pool->shift == shift ? pool : NULL; +} - spin_lock(&mm->page_table_lock); - table = mm->context.user4k_pt_frag[level]; - if (table) { - /* An mm's geometry is immutable; never consume another granule. */ - if (virt_to_ptdesc(table)->pt_frag_shift != shift) { - spin_unlock(&mm->page_table_lock); - return NULL; - } - next = table + frag_size; - mm->context.user4k_pt_frag[level] = - ((unsigned long)next & ~PAGE_MASK) ? next : NULL; - } - spin_unlock(&mm->page_table_lock); - if (table) - return table; +static struct ptdesc *fragment_pt_alloc(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct ptdesc *pt; + bool constructed; - pt = pagetable_alloc(GFP_PGTABLE_USER, order); + pt = pagetable_alloc(GFP_PGTABLE_USER, shift > PAGE_SHIFT ? shift - PAGE_SHIFT : 0); if (!pt) return NULL; switch (level) { @@ -96,32 +112,95 @@ void *arm64_user_pt_alloc_granule(struct mm_struct *mm, pt->pt_frag_shift = shift; set_bit(PT_fragmented, &pt->pt_flags.f); atomic_set(&pt->pt_frag_refcount, 1); - table = ptdesc_address(pt); - spin_lock(&mm->page_table_lock); - if (nr_fragments > 1) { - next = mm->context.user4k_pt_frag[level]; - if (next) { - /* Another allocator supplied backing while we constructed - * a table owner. Consume its unused fragment rather than - * stranding the rest of our new native page. - */ - if (virt_to_ptdesc(next)->pt_frag_shift != shift) { - spin_unlock(&mm->page_table_lock); - fragment_owner_free(pt); - return NULL; - } - table = next; - next += frag_size; - mm->context.user4k_pt_frag[level] = - ((unsigned long)next & ~PAGE_MASK) ? next : NULL; - } else { - atomic_set(&pt->pt_frag_refcount, nr_fragments); - mm->context.user4k_pt_frag[level] = table + frag_size; - } + return pt; +} + +/* The caller holds pool->lock; its active mm retains the pool. */ +static void *fragment_take(struct fragment_pool *pool, bool *zero) +{ + struct fragment_owner *owner; + unsigned int slot; + + if (pool->closed || list_empty(&pool->available)) + return NULL; + owner = list_first_entry(&pool->available, struct fragment_owner, available); + slot = __ffs(owner->free_mask); + owner->free_mask &= ~BIT(slot); + *zero = owner->dirty_mask & BIT(slot); + owner->dirty_mask &= ~BIT(slot); + atomic_inc(&owner->pt->pt_frag_refcount); + if (!owner->free_mask) + list_del_init(&owner->available); + return owner->slots[slot].table; +} + +void *arm64_user_pt_alloc_granule(struct mm_struct *mm, + enum arm64_user4k_pt_level level, unsigned int shift) +{ + struct fragment_pool *pool; + struct fragment_owner *owner; + struct ptdesc *pt; + void *table; + unsigned int nr, i; + unsigned long flags; + bool zero = false; + + if (shift != 12 && shift != 14 && shift != 16) + return NULL; + if (WARN_ON_ONCE((unsigned int)level >= ARRAY_SIZE(mm->context.user4k_pt_frag))) + return NULL; + if (shift >= PAGE_SHIFT) { + pt = fragment_pt_alloc(mm, level, shift); + return pt ? ptdesc_address(pt) : NULL; + } + pool = fragment_pool_get(mm, level, shift); + if (!pool) + return NULL; + spin_lock_irqsave(&pool->lock, flags); + table = fragment_take(pool, &zero); + spin_unlock_irqrestore(&pool->lock, flags); + if (table) + goto out; + + nr = PAGE_SIZE >> shift; + owner = kzalloc(struct_size(owner, slots, nr), GFP_KERNEL | __GFP_ACCOUNT); + if (!owner) + goto out; + pt = fragment_pt_alloc(mm, level, shift); + if (!pt) { + kfree(owner); + goto out; + } + owner->pool = pool; + owner->pt = pt; + INIT_LIST_HEAD(&owner->available); + owner->free_mask = GENMASK(nr - 1, 1); + for (i = 0; i < nr; i++) + owner->slots[i].table = ptdesc_address(pt) + (i << shift); + /* This word is unused by arm64 page-table constructors and walkers. */ + pt->__page_mapping = (unsigned long)owner; + + spin_lock_irqsave(&pool->lock, flags); + /* Reuse supply published while allocation slept, including retired slots. */ + table = fragment_take(pool, &zero); + if (!table && !pool->closed) { + refcount_inc(&pool->refs); /* Native owner's lifetime reference. */ + list_add_tail(&owner->available, &pool->available); + table = owner->slots[0].table; + owner = NULL; } - spin_unlock(&mm->page_table_lock); - if (table != ptdesc_address(pt)) + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) { + pt->__page_mapping = 0; fragment_owner_free(pt); + kfree(owner); + } +out: + /* Clear only recycled slots that are actually allocated again. The issued + * reference excludes other allocators and keeps this owner alive here. + */ + if (table && zero) + memset(table, 0, 1UL << shift); return table; } @@ -130,14 +209,45 @@ void *arm64_user4k_pt_alloc(struct mm_struct *mm, enum arm64_user4k_pt_level lev return arm64_user_pt_alloc_granule(mm, level, 12); } +/* Only unpublished tables or tables past the required walker grace arrive here. */ void arm64_user4k_pt_free(void *table) { struct ptdesc *pt = virt_to_ptdesc(table); + struct fragment_owner *owner; + struct fragment_pool *pool; + unsigned int slot; + unsigned long flags; + bool last; VM_BUG_ON(!test_bit(PT_fragmented, &pt->pt_flags.f)); VM_BUG_ON(!IS_ALIGNED((unsigned long)table, 1UL << pt->pt_frag_shift)); VM_BUG_ON(atomic_read(&pt->pt_frag_refcount) <= 0); - fragment_put(pt, 1); + if (pt->pt_frag_shift >= PAGE_SHIFT) { + if (test_and_clear_bit(PT_frag_defer, &pt->pt_flags.f)) + call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); + else + fragment_owner_free(pt); + return; + } + owner = (void *)pt->__page_mapping; + pool = owner->pool; + slot = ((unsigned long)table & ~PAGE_MASK) >> pt->pt_frag_shift; + spin_lock_irqsave(&pool->lock, flags); + VM_BUG_ON(owner->free_mask & BIT(slot)); + owner->free_mask |= BIT(slot); + owner->dirty_mask |= BIT(slot); + last = atomic_dec_and_test(&pt->pt_frag_refcount); + if (last) + list_del_init(&owner->available); + else if (!pool->closed && list_empty(&owner->available)) + list_add_tail(&owner->available, &pool->available); + spin_unlock_irqrestore(&pool->lock, flags); + if (last) { + pt->__page_mapping = 0; + fragment_owner_free(pt); + kfree(owner); + fragment_pool_put(pool); + } } void arm64_pgtable_free(void *table) @@ -154,32 +264,46 @@ void arm64_pgtable_free(void *table) void arm64_user4k_pt_cache_destroy(struct mm_struct *mm) { + struct fragment_pool *pool; unsigned int level; + unsigned long flags; - /* The mm is private or dying; no allocator may still access its cache. */ + /* No allocator may still access this mm; deferred owners are independent. */ for (level = 0; level < ARRAY_SIZE(mm->context.user4k_pt_frag); level++) { - void *next = mm->context.user4k_pt_frag[level]; - struct ptdesc *pt; - unsigned int issued, shift; - - if (!next) + pool = mm->context.user4k_pt_frag[level]; + if (!pool) continue; mm->context.user4k_pt_frag[level] = NULL; - pt = virt_to_ptdesc(next); - shift = pt->pt_frag_shift; - issued = ((unsigned long)next & ~PAGE_MASK) >> shift; - fragment_put(pt, (PAGE_SIZE >> shift) - issued); + spin_lock_irqsave(&pool->lock, flags); + pool->closed = true; + spin_unlock_irqrestore(&pool->lock, flags); + fragment_pool_put(pool); } } +static void fragment_slot_free_rcu(struct rcu_head *rcu) +{ + struct fragment_slot *slot = container_of(rcu, struct fragment_slot, rcu); + + arm64_user4k_pt_free(slot->table); +} + void arm64_pte_free_defer(struct mm_struct *mm, pgtable_t table) { struct ptdesc *pt = virt_to_ptdesc(table); if (test_bit(PT_fragmented, &pt->pt_flags.f)) { - /* No freed slot is reused. Delay the final owner release for RCU. */ - set_bit(PT_frag_defer, &pt->pt_flags.f); - arm64_user4k_pt_free(table); + if (pt->pt_frag_shift < PAGE_SHIFT) { + struct fragment_owner *owner = (void *)pt->__page_mapping; + unsigned int slot = ((unsigned long)table & ~PAGE_MASK) >> + pt->pt_frag_shift; + + /* Pending callbacks retain their issued references and contents. */ + call_rcu(&owner->slots[slot].rcu, fragment_slot_free_rcu); + } else { + set_bit(PT_frag_defer, &pt->pt_flags.f); + arm64_user4k_pt_free(table); + } } else { call_rcu(&pt->pt_rcu_head, fragment_owner_free_rcu); } diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index e406b97ea6235d..4e5b2befe18f5d 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -858,6 +857,7 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) unsigned int i, slots = PAGE_SIZE >> shift; unsigned long size = 1UL << shift; unsigned int read_shift = 0; + int pending_refs; bool live; KUNIT_ASSERT_NOT_NULL(test, f); @@ -925,20 +925,33 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) KUNIT_EXPECT_PTR_NE(test, f->pinned[1], f->pinned[2]); pte_owner = page_ptdesc(f->pinned[0]); KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots); + /* Refill holes in the same owner, without retaining old table contents. */ + for (i = 1; i < slots; i++) { + void *old = f->table[i]; + + memset(old, 0x6a, size); + arm64_pgtable_free(old); + f->table[i] = arm64_user_pt_alloc_granule(f->mm, USER4K_PT_PTE, shift); + KUNIT_ASSERT_NOT_NULL(test, f->table[i]); + KUNIT_EXPECT_PTR_EQ(test, f->table[i], old); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(f->table[i], 0, size), NULL); + } + /* Begin the reader before queuing a callback, so it cannot finish early. */ + rcu_read_lock(); arm64_pte_free_defer(f->mm, f->table[0]); f->table[0] = NULL; - KUNIT_EXPECT_EQ(test, atomic_read(&pte_owner->pt_frag_refcount), (int)slots - 1); + pending_refs = atomic_read(&pte_owner->pt_frag_refcount); for (i = 1; i < slots - 1; i++) { arm64_pgtable_free(f->table[i]); f->table[i] = NULL; } - /* A pre-existing software walker must survive the last fragment release. */ - rcu_read_lock(); + /* Pending slot zero keeps the owner alive through this reader. */ arm64_pgtable_free(f->table[slots - 1]); f->table[slots - 1] = NULL; live = folio_test_pgtable(page_folio(f->pinned[0])); read_shift = arm64_ptep_page_shift(page_address(f->pinned[0])); rcu_read_unlock(); + KUNIT_EXPECT_EQ(test, pending_refs, (int)slots); KUNIT_EXPECT_TRUE(test, live); KUNIT_EXPECT_EQ(test, read_shift, shift); rcu_barrier(); @@ -949,8 +962,9 @@ static void test_fragment_lifetime(struct kunit *test, unsigned int shift) arm64_pgtable_free(f->table[i]); f->table[i] = NULL; - KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), (int)slots - 1); - KUNIT_EXPECT_TRUE(test, folio_test_pgtable(ptdesc_folio(pt))); + /* A wholly empty owner is released without waiting for mm teardown. */ + KUNIT_EXPECT_EQ(test, atomic_read(&pt->pt_frag_refcount), 0); + KUNIT_EXPECT_FALSE(test, folio_test_pgtable(ptdesc_folio(pt))); } mmput(f->mm); f->mm = NULL; @@ -5213,14 +5227,13 @@ static void user4k_exec_arguments_test(struct kunit *test) } for (native = 0; native < 2; native++) { struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); - struct mm_struct *mm; + struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); unsigned long size, mask, initial, copied, new_top; unsigned long a_len = 2 * PAGE_SIZE + SZ_4K + 31, b_len = SZ_4K + 43; char *a, *b, *buf, *too_long; int ret; KUNIT_ASSERT_NOT_NULL(test, bprm); - mm = native ? mm_alloc() : test_vma_mm(test); KUNIT_ASSERT_NOT_NULL(test, mm); if (native) KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, mm), 0); @@ -5293,88 +5306,6 @@ static void user4k_exec_arguments_test(struct kunit *test) } } -/* The transfer can replace bprm->mm, so release its final owner. */ -static void test_exec_mm_free(void *data) -{ - struct linux_binprm *bprm = data; - - mmput(bprm->mm); -} - -static void user64k_exec_arguments_test(struct kunit *test) -{ - unsigned int shifts[] = { 16, 12, PAGE_SHIFT }; - unsigned int mode; - - if (PAGE_SHIFT >= 16 || - (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN))) { - kunit_skip(test, "requires internal larger user granule and MM opt-in"); - return; - } - for (mode = 0; mode < ARRAY_SIZE(shifts); mode++) { - struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); - unsigned long a_len = mode ? SZ_64K + SZ_16K + 31 : 32 * PAGE_SIZE + 31; - unsigned long b_len = PAGE_SIZE + 17, initial, copied; - char *a, *b, *buf; - unsigned long i; - - KUNIT_ASSERT_NOT_NULL(test, bprm); - bprm->mm = mm_alloc(); - KUNIT_ASSERT_NOT_NULL(test, bprm->mm); - KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_exec_mm_free, bprm), 0); - KUNIT_ASSERT_EQ(test, test_mm_select_granule(bprm->mm, 16), 0); - KUNIT_ASSERT_EQ(test, create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p), 0); - initial = bprm->p; - bprm->argmin = initial - SZ_2M; - bprm->rlim_stack.rlim_cur = SZ_8M; - bprm->rlim_stack.rlim_max = RLIM_INFINITY; - a = kunit_kmalloc(test, a_len + 1, GFP_KERNEL); - b = kunit_kmalloc(test, b_len + 1, GFP_KERNEL); - buf = kunit_kmalloc(test, a_len + b_len + 7, GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, a); - KUNIT_ASSERT_NOT_NULL(test, b); - KUNIT_ASSERT_NOT_NULL(test, buf); - for (i = 0; i < a_len; i++) - a[i] = 'A' + i % 23; - a[a_len] = 0; - memset(b, 'b', b_len); - b[b_len] = 0; - KUNIT_ASSERT_EQ(test, copy_string_kernel("tail", bprm), 0); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, 5, 0), 5); - KUNIT_ASSERT_STREQ(test, buf, "tail"); - KUNIT_ASSERT_EQ(test, copy_string_kernel(a, bprm), 0); - KUNIT_ASSERT_EQ(test, copy_string_kernel(b, bprm), 0); - bprm->argc = 3; - copied = initial - bprm->p; - KUNIT_ASSERT_EQ(test, copied, a_len + b_len + 7); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), - (int)copied); - KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); - KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); - KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); - /* Exercise copying out of every native page within a coarse leaf. */ - if (shifts[mode] != 16) { - KUNIT_ASSERT_EQ(test, bprm_set_page_shift(bprm, shifts[mode]), 0); - KUNIT_ASSERT_EQ(test, mm_page_shift(bprm->mm), shifts[mode]); - memset(buf, 0, copied); - KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), - (int)copied); - KUNIT_ASSERT_EQ(test, memcmp(buf, b, b_len + 1), 0); - KUNIT_ASSERT_EQ(test, memcmp(buf + b_len + 1, a, a_len + 1), 0); - KUNIT_ASSERT_STREQ(test, buf + b_len + a_len + 2, "tail"); - } - initial = bprm->p; - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + 1); - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + b_len + a_len + 2); - KUNIT_ASSERT_EQ(test, remove_arg_zero(bprm), 0); - KUNIT_EXPECT_EQ(test, bprm->p, initial + copied); - KUNIT_EXPECT_EQ(test, bprm->argc, 0); - KUNIT_EXPECT_EQ(test, bprm->vma_pages, 0UL); - } -} - static struct vm_area_struct *test_special_vma(struct mm_struct *mm, const struct vm_special_mapping *sm, unsigned long start, unsigned long len, vm_flags_t flags) @@ -6286,10 +6217,10 @@ static void user4k_file_mapcounts_test(struct kunit *test) folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 2 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 2 }, 1); KUNIT_ASSERT_EQ(test, test_file_fault(private, TEST_VA + 2 * SZ_4K, true) & VM_FAULT_ERROR, 0U); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); child = test_vma_mm(test); KUNIT_ASSERT_NOT_NULL(test, child); uprobe_start_dup_mmap(); @@ -6298,17 +6229,17 @@ static void user4k_file_mapcounts_test(struct kunit *test) KUNIT_ASSERT_EQ(test, ret, 0); cvma = test_vma_lookup(child, TEST_VA); KUNIT_ASSERT_NOT_NULL(test, cvma); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 3, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 3, 1 }, 1); KUNIT_ASSERT_EQ(test, test_file_fault(cvma, TEST_VA + SZ_4K, true) & VM_FAULT_ERROR, 0U); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 3, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 3, 2, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, child); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 2, 2, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 2, 2, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, private->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 1); kunit_release_action(test, user4k_vma_mm_free, native->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 1, 1, 1 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 1, 1, 1 }, 0); kunit_release_action(test, user4k_vma_mm_free, shared->vm_mm); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); } /* A unique RAM-backed test inode: exercise generic file faults without @@ -8890,10 +8821,6 @@ static void user_subpage_numa_cow_batch_test(struct kunit *test) { unsigned int shift; - if (!(READ_ONCE(sysctl_numa_balancing_mode) & NUMA_BALANCING_NORMAL)) { - kunit_skip(test, "requires normal NUMA balancing; use numa_balancing=enable"); - return; - } for (shift = 12; shift < PAGE_SHIFT; shift += 2) { unsigned long size = 1UL << shift, start = TEST_VA; struct vm_area_struct *vma; @@ -10092,7 +10019,7 @@ static void user4k_file_migration_test(struct kunit *test) lru_add_drain_all(); folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); old_phys = PFN_PHYS(folio_pfn(folio)); isolated = isolate_folio_to_list(folio, &folios); if (!fail || !isolated) @@ -10107,7 +10034,7 @@ static void user4k_file_migration_test(struct kunit *test) KUNIT_EXPECT_EQ(test, succeeded, fail ? 0U : 1U); folio = filemap_get_folio(file->f_mapping, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folio); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { [0 ... TEST_SLOTS - 1] = 1 }, 1); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ [0 ... TEST_SLOTS - 1] = 1 }, 1); for (i = 0; i < 4; i++) { pte_t entry = test_fault_entry(test, small->vm_mm, small->vm_start + i * SZ_4K); @@ -10125,7 +10052,7 @@ static void user4k_file_migration_test(struct kunit *test) folio_lock(folio); try_to_unmap(folio, TTU_SYNC); KUNIT_EXPECT_FALSE(test, folio_mapped(folio)); - test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]) { 0, 0, 0, 0 }, 0); + test_file_mapcounts(test, folio, (unsigned int[TEST_SLOTS]){ 0, 0, 0, 0 }, 0); folio_unlock(folio); folio_put(folio); for (i = 0; i < 4; i++) { @@ -14103,7 +14030,6 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_mremap_resize_test), KUNIT_CASE(user4k_stack_relocate_test), KUNIT_CASE(user4k_exec_arguments_test), - KUNIT_CASE(user64k_exec_arguments_test), KUNIT_CASE(user4k_special_mapping_test), KUNIT_CASE(user4k_insert_fragment_test), KUNIT_CASE(user16k_insert_shared_test), @@ -14142,6 +14068,9 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_vm_mmap_test), KUNIT_CASE(user4k_mprotect_test), KUNIT_CASE(user4k_mprotect_cow_batch_test), +#ifdef CONFIG_NUMA_BALANCING + KUNIT_CASE(user_subpage_numa_cow_batch_test), +#endif KUNIT_CASE(user4k_mprotect_accounting_test), KUNIT_CASE(user4k_file_cow_test), KUNIT_CASE(user4k_file_eof_test), @@ -14202,25 +14131,6 @@ static struct kunit_case user4k_test_cases[] = { {} }; -#ifdef CONFIG_NUMA_BALANCING -/* - * The COW hinting expectations require a stable normal-balancing policy. - * Runtime sysctl writers do not take the fixture's mm or PTE locks, so run - * this case at boot and do not expose it through the debugfs rerun control. - */ -static struct kunit_case user4k_numa_init_cases[] = { - KUNIT_CASE(user_subpage_numa_cow_batch_test), - {} -}; - -static struct kunit_suite user4k_numa_init_suite = { - .name = "arm64-user4k-numa-init", - .test_cases = user4k_numa_init_cases, -}; - -kunit_test_init_section_suite(user4k_numa_init_suite); -#endif - static struct kunit_suite user4k_test_suite = { .name = "arm64-user4k", .test_cases = user4k_test_cases, diff --git a/tools/testing/selftests/arm64/user4k/README.pagetable-backing b/tools/testing/selftests/arm64/user4k/README.pagetable-backing index 2c6b16b17465a1..cf87f2a64f5aec 100644 --- a/tools/testing/selftests/arm64/user4k/README.pagetable-backing +++ b/tools/testing/selftests/arm64/user4k/README.pagetable-backing @@ -27,7 +27,19 @@ and upper-level tables contribute to the measurements; do not subtract them as though they were pure payload. Deferred table releases can occur between samples. Contents PASS does not assert a density or performance result. -The allocation-race fix reduces initial concurrent physical backing. It does -not reclaim partially occupied table owners or reuse issued fragments after -unmap. Retention/refill checkpoints deliberately preserve that separate -coverage gap rather than declaring it fixed. +The allocator reuses safely retired fragments, including slots from owners +that still contain live tables. RCU-pending slots remain unavailable until +their callback runs; an empty owner is freed. A reused dirty slot is cleared +before it is returned. This avoids allocating new owners while old ones have +reusable holes. It does not compact surviving live tables. + +On Neo, the serial refill checkpoint fell from 7,421,952 to 4,276,224 bytes of +physical page tables; the concurrent refill fell from 6,275,072 to 4,276,224. +The same six-phase contents and post-unmap cleanup passed. Whole-cgroup slab +usage increased and includes allocator metadata plus other kernel objects. +Do not report page-table bytes as the complete memory cost. + +Seven operation-timed rounds after one warmup had serial/concurrent total +medians of 15.625/16.865 ms, versus 15.434/16.679 ms before reuse. The ranges +substantially overlap; a small cost remains possible. These measurements do +not establish a speedup or zero overhead. They exclude checkpoint I/O. From a96108ecd19d850d79729ea30080d83e03b3ec22 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Thu, 8 Oct 2026 20:29:40 -0400 Subject: [PATCH 303/631] mm: retain compatible subpage pools across allocation-domain changes Index per-mm subpage pools by charge domain, node and VA shard instead of keeping only the first domain. Retain discoverable pools while their native owners survive, so A-to-B-to-A moves can reuse each domain's free quarters. Pin charge identities and registry lifetime, retire empty pools safely, and retry lookup when allocation races with retirement. Preserve private fallback and existing allocation policy restrictions. Add a bounded fixture for bulk moves, alternating single-slot tails and concurrent moves followed by steady allocation. On Neo, the previously 4 MiB backing for a 1 MiB steady allocation becomes approximately 1 MiB; alternating eight 4K leaves per domain fit two 16K owners. Actual kpagecgroup charges, old/new PFN stability, disjoint quarters and complete bytes pass. The corrected race oracle measures added owners rather than counting reused live backing again. Retained MM, pin, real swap readback and GPU controls also pass. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 453b475f491683d207313c66736ca4eef4028c16) --- mm/memory.c | 12 + mm/user-subpage.c | 225 ++++++++++++++---- .../testing/selftests/arm64/user4k/.gitignore | 2 + tools/testing/selftests/arm64/user4k/Makefile | 4 +- .../arm64/user4k/README.domain-cycle | 78 ++++++ .../selftests/arm64/user4k/domain-cycle.c | 155 ++++++++++++ 6 files changed, 432 insertions(+), 44 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/README.domain-cycle create mode 100644 tools/testing/selftests/arm64/user4k/domain-cycle.c diff --git a/mm/memory.c b/mm/memory.c index ea603a72be93d3..22e3ea5356680f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1359,6 +1359,7 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, struct mm_subpage *slot; struct folio *folio; unsigned int offset = vma_page_offset_at(vma, addr).offset; + unsigned int retired_retries = 0; int err; retry_private: @@ -1426,6 +1427,11 @@ static struct mm_subpage *cow_subpage_prealloc(struct mm_struct *mm, if (!*pooled) mm_subpage_pool_close(pool); mm_subpage_pool_put(pool); + if (*pooled && IS_ERR(slot) && PTR_ERR(slot) == -ESHUTDOWN) { + if (++retired_retries > 1) + force_private = true; + goto retry_private; + } return slot; } @@ -6434,6 +6440,7 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, struct mm_subpage_pool *pool; struct folio *folio; unsigned int count = hweight_long(mask); + unsigned int retired_retries = 0; bool arbitrary; int err; @@ -6498,6 +6505,11 @@ static int anon_subpage_prealloc_mask(struct mm_struct *mm, if (!*pooled) mm_subpage_pool_close(pool); mm_subpage_pool_put(pool); + if (*pooled && err == -ESHUTDOWN) { + if (++retired_retries > 1) + use_pool = false; + goto retry_private; + } return err; } diff --git a/mm/user-subpage.c b/mm/user-subpage.c index acfa80848e6e35..cff26e5b5c8036 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -213,6 +214,12 @@ struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) return slot; } +struct cow_pool_key { + struct mem_cgroup *memcg; + int nid; + u32 bucket; +}; + struct mm_subpage_pool { struct rcu_head rcu; spinlock_t lock; @@ -230,13 +237,101 @@ struct mm_subpage_pool { bool cow_shared; /* Allocated only for shared COW pools; protected by pool->lock. */ unsigned long *cow_free_slots; + struct cow_pool_key cow_key; + struct rhash_head cow_hash; + struct list_head cow_domain; + /* Protected by this pool's lock, never used after registry detachment. */ + struct mm_subpage_cow_pools *cow_registry; }; #define COW_POOL_BITS 4 struct mm_subpage_cow_pools { - struct mm_subpage_pool *pool[1U << COW_POOL_BITS]; + struct rhashtable domains; + spinlock_t lock; + struct list_head pools; + refcount_t refs; + struct rcu_head rcu; + struct work_struct free_work; }; +static const struct rhashtable_params cow_domain_params = { + .head_offset = offsetof(struct mm_subpage_pool, cow_hash), + .key_offset = offsetof(struct mm_subpage_pool, cow_key), + .key_len = sizeof(struct cow_pool_key), + .nelem_hint = 16, + .automatic_shrinking = true, +}; + +static void cow_registry_free_work(struct work_struct *work) +{ + struct mm_subpage_cow_pools *pools = + container_of(work, struct mm_subpage_cow_pools, free_work); + + WARN_ON_ONCE(!list_empty(&pools->pools)); + rhashtable_destroy(&pools->domains); + kfree(pools); +} + +static void cow_registry_free_rcu(struct rcu_head *rcu) +{ + struct mm_subpage_cow_pools *pools = + container_of(rcu, struct mm_subpage_cow_pools, rcu); + + queue_work(system_unbound_wq, &pools->free_work); +} + +static void cow_registry_put(struct mm_subpage_cow_pools *pools) +{ + if (refcount_dec_and_test(&pools->refs)) + call_rcu(&pools->rcu, cow_registry_free_rcu); +} + +static void pool_put_raw(struct mm_subpage_pool *pool); +static void owner_remove(struct mm_subpage_owner *owner); +static void owner_free(struct mm_subpage_owner *owner, bool put_folio); + +/* The caller retains a pool reference independently of registry membership. */ +static void cow_pool_detach_locked(struct mm_subpage_pool *pool) +{ + struct mm_subpage_cow_pools *pools = pool->cow_registry; + + lockdep_assert_held(&pool->lock); + if (!pools) + return; + pool->cow_registry = NULL; + /* A failed prospective insertion may not have entered the hash/list. */ + rhashtable_remove_fast(&pools->domains, &pool->cow_hash, cow_domain_params); + spin_lock(&pools->lock); + list_del_init(&pool->cow_domain); + spin_unlock(&pools->lock); + pool_put_raw(pool); /* Removed (or unpublished prospective) membership. */ + cow_registry_put(pools); +} + +static void cow_pool_retire_empty(struct mm_subpage_pool *pool) +{ + struct mm_subpage_owner *owner = NULL; + unsigned long flags; + + if (!pool->cow_shared) + return; + spin_lock_irqsave(&pool->lock, flags); + if (!pool->backing_pages || + (pool->backing_pages == 1 && pool->unissued)) { + pool->closed = true; + cow_pool_detach_locked(pool); + /* A failed allocation may leave wholly unissued supply, not a + * live domain. Retire it under the same reservation lock. + */ + owner = pool->unissued; + if (owner) + owner_remove(owner); + } + spin_unlock_irqrestore(&pool->lock, flags); + if (owner) + owner_free(owner, true); +} + static void owner_init(struct mm_subpage_owner *owner, struct mm_subpage_pool *pool, struct folio *folio) { @@ -285,7 +380,7 @@ void mm_subpage_pool_get(struct mm_subpage_pool *pool) refcount_inc(&pool->refs); } -void mm_subpage_pool_put(struct mm_subpage_pool *pool) +static void pool_put_raw(struct mm_subpage_pool *pool) { if (!refcount_dec_and_test(&pool->refs)) return; @@ -295,19 +390,25 @@ void mm_subpage_pool_put(struct mm_subpage_pool *pool) kfree_rcu(pool, rcu); } -/* Keep the first implementation within one unambiguous allocation domain. - * Policy-sensitive, locked, droppable and multi-node faults keep fresh backing. - * An mm can move cgroups; old charged slots must not satisfy its new faults. - */ +void mm_subpage_pool_put(struct mm_subpage_pool *pool) +{ + /* Failed allocations must not leave an empty per-domain cache behind. */ + cow_pool_retire_empty(pool); + pool_put_raw(pool); +} + +/* Per-mm pools retain each still-backed charge/node/shard domain separately. */ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, struct mm_struct *charge_mm, struct vm_area_struct *vma, unsigned long address) { struct mm_subpage_cow_pools *pools, *new_pools; struct mm_subpage_pool *pool, *new; - unsigned int bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + struct cow_pool_key key = {}; struct mem_cgroup *memcg; - bool matches; + unsigned long flags; + unsigned int attempt; + void *result; int nid; if ((vma->vm_flags & (VM_LOCKED | VM_DROPPABLE | VM_SPECIAL)) || @@ -320,60 +421,85 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, nid = first_node(node_states[N_MEMORY]); if (!node_isset(nid, cpuset_current_mems_allowed)) return NULL; - /* Publication pairs with cmpxchg: every bucket starts empty. Adjacent - * 4K leaves in one native virtual group always choose the same pool. - * Faulting threads can migrate CPUs without changing that identity. - */ + /* Acquire initialization published by the mm's cmpxchg below. */ pools = smp_load_acquire(&mm->cow_subpage_pool); if (!pools) { new_pools = kzalloc(sizeof(*new_pools), GFP_KERNEL | __GFP_ACCOUNT); if (!new_pools) return NULL; + spin_lock_init(&new_pools->lock); + INIT_LIST_HEAD(&new_pools->pools); + refcount_set(&new_pools->refs, 1); + INIT_WORK(&new_pools->free_work, cow_registry_free_work); + if (rhashtable_init(&new_pools->domains, &cow_domain_params)) { + kfree(new_pools); + return NULL; + } pools = cmpxchg(&mm->cow_subpage_pool, NULL, new_pools); if (!pools) pools = new_pools; else - kfree(new_pools); + cow_registry_put(new_pools); } memcg = get_mem_cgroup_from_mm(charge_mm); - /* Acquire the immutable charge domain published with this bucket. */ - pool = smp_load_acquire(&pools->pool[bucket]); - if (!pool) { - new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); - if (!new) { + key.memcg = memcg; + key.nid = nid; + key.bucket = hash_long(address >> PAGE_SHIFT, COW_POOL_BITS); + for (attempt = 0; attempt < 3; attempt++) { + rcu_read_lock(); + pool = rhashtable_lookup(&pools->domains, &key, cow_domain_params); + if (pool && !refcount_inc_not_zero(&pool->refs)) + pool = NULL; + rcu_read_unlock(); + if (pool) { mem_cgroup_put(memcg); - return NULL; + return pool; } + new = mm_subpage_pool_create_granule(GFP_KERNEL, mm_page_shift(mm)); + if (!new) + break; new->cow_free_slots = kcalloc(SUBPAGES_PER_FOLIO, sizeof(*new->cow_free_slots), GFP_KERNEL | __GFP_ACCOUNT); if (!new->cow_free_slots) { mm_subpage_pool_close(new); mm_subpage_pool_put(new); - mem_cgroup_put(memcg); - return NULL; + break; } + if (memcg) + css_get(&memcg->css); new->cow_memcg = memcg; new->cow_nid = nid; new->cow_shared = true; - pool = cmpxchg(&pools->pool[bucket], NULL, new); - if (!pool) { - pool = new; - /* The mm owns the initial pool reference and its memcg ref. */ - mm_subpage_pool_get(pool); - return pool; + new->cow_key = key; + INIT_LIST_HEAD(&new->cow_domain); + new->cow_registry = pools; + /* Hold caller, prospective membership and registry lifetime refs + * before a concurrent lookup can observe this pool. + */ + mm_subpage_pool_get(new); + refcount_inc(&pools->refs); + spin_lock_irqsave(&new->lock, flags); + result = rhashtable_lookup_get_insert_fast(&pools->domains, + &new->cow_hash, cow_domain_params); + if (!result) { + spin_lock(&pools->lock); + list_add_tail(&new->cow_domain, &pools->pools); + spin_unlock(&pools->lock); } - matches = pool->cow_memcg == memcg && pool->cow_nid == nid; + spin_unlock_irqrestore(&new->lock, flags); + if (!result) { + mem_cgroup_put(memcg); + return new; + } + /* Do not dereference an unreferenced object returned by insert. */ mm_subpage_pool_close(new); mm_subpage_pool_put(new); - } else { - matches = pool->cow_memcg == memcg && pool->cow_nid == nid; - mem_cgroup_put(memcg); + if (IS_ERR(result)) + break; } - if (!matches) - return NULL; - mm_subpage_pool_get(pool); - return pool; + mem_cgroup_put(memcg); + return NULL; } struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, @@ -390,18 +516,26 @@ bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *fo void mm_subpage_cow_pool_exit(struct mm_struct *mm) { struct mm_subpage_cow_pools *pools = xchg(&mm->cow_subpage_pool, NULL); - unsigned int i; + struct mm_subpage_pool *pool; + unsigned long flags; - /* No faults remain once mm_users reaches zero. Slot owners can outlive mm. */ + /* No faults remain; outstanding owners/callbacks never use this mm. */ if (!pools) return; - for (i = 0; i < ARRAY_SIZE(pools->pool); i++) { - if (!pools->pool[i]) - continue; - mm_subpage_pool_close(pools->pool[i]); - mm_subpage_pool_put(pools->pool[i]); + for (;;) { + spin_lock_irqsave(&pools->lock, flags); + pool = list_first_entry_or_null(&pools->pools, + struct mm_subpage_pool, cow_domain); + if (pool) + mm_subpage_pool_get(pool); + spin_unlock_irqrestore(&pools->lock, flags); + if (!pool) + break; + /* Lock order is pool -> registry list, never the reverse. */ + mm_subpage_pool_close(pool); + mm_subpage_pool_put(pool); } - kfree(pools); + cow_registry_put(pools); } /* Counts include unavailable (busy/migrating) owners, so zero is conclusive. @@ -443,6 +577,10 @@ static void owner_remove(struct mm_subpage_owner *owner) if (owner->pool->unissued == owner) owner->pool->unissued = NULL; owner->pool->backing_pages--; + if (!owner->pool->backing_pages && owner->pool->cow_shared) { + owner->pool->closed = true; + cow_pool_detach_locked(owner->pool); + } } static void owner_free_rmap(struct mm_subpage_owner *owner) @@ -838,6 +976,7 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) spin_lock_irqsave(&pool->lock, flags); pool->closed = true; + cow_pool_detach_locked(pool); /* At most one newly supplied owner has no issued slots. Close is O(1). */ owner = pool->unissued; if (owner) diff --git a/tools/testing/selftests/arm64/user4k/.gitignore b/tools/testing/selftests/arm64/user4k/.gitignore index 3c4c8c86eefe2c..e3e47c93059017 100644 --- a/tools/testing/selftests/arm64/user4k/.gitignore +++ b/tools/testing/selftests/arm64/user4k/.gitignore @@ -9,3 +9,5 @@ clipped-firstwrite pinned-fork pagetable-retention pagetable-concurrent + +/domain-cycle diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index d306deb1f00544..0350be7a579128 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -18,11 +18,13 @@ TEST_GEN_PROGS_EXTENDED += pinned-fork TEST_FILES += README.pinned-fork TEST_GEN_PROGS_EXTENDED += pagetable-retention pagetable-concurrent TEST_FILES += README.pagetable-backing +TEST_GEN_PROGS_EXTENDED += domain-cycle +TEST_FILES += README.domain-cycle include ../../lib.mk $(OUTPUT)/page-contract: page-contract.c x18-probe.S -$(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread +$(OUTPUT)/domain-cycle $(OUTPUT)/pagetable-concurrent $(OUTPUT)/first-write $(OUTPUT)/thread-fixture-init $(OUTPUT)/user16k-smoke-init $(OUTPUT)/cow-pool $(OUTPUT)/anon-hole-refill-race $(OUTPUT)/anon-hole-refill-reclaim: LDLIBS += -pthread # Freestanding ELF32 probes exercise the compat ABI without an armhf sysroot. AARCH32_CC ?= clang --target=arm-linux-gnueabihf diff --git a/tools/testing/selftests/arm64/user4k/README.domain-cycle b/tools/testing/selftests/arm64/user4k/README.domain-cycle new file mode 100644 index 00000000000000..3e7425f4dbffbf --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/README.domain-cycle @@ -0,0 +1,78 @@ +Cgroup allocation-domain pool reuse +================================== + +Build domain-cycle with -pthread. Run in the established4K userspace ABI on a +native16K kernel. Root-visible pagemap and kpagecgroup are required for the +controller's density/charge proof. The fixture itself checks every payload byte. +No memory.high/max limits, reclaim, swap-area or GPU operations are needed. + + userpages 4k ./domain-cycle --native-16k [--tail | --race] + +The program prints READY and waits for stdin byte 'c' before allocating payload. +A controller places only this fresh fixture in owned cgroup A, then acknowledges. +Each PHASE prints its index/PID/base/leaf count/stride and waits for 'c'. Both +cgroups must remain alive until all prior ranges have been checked and unmapped. + +Default: three1MiB useful guarded allocation phases. Run in A, move to B, move +back to A between phases. Check all256 written leaves/range (4K every8K), excluding +the inaccessible guards. The old sticky-domain implementation can allocate16K +native backing per4K leaf after a move. Packed acceptance is1MiB..1.25MiB native +PFN union per1MiB phase; the allowance covers bounded shard/partial-owner tails. + +--tail:16 independent4K allocations, switching A/B after EVERY leaf. All16VAs are +selected in the same current16-way allocator shard using its documented hash. +Eight leaves/domain should occupy at most3native folios: two useful plus one +possible preexisting partial boundary. This catches a latest-domain-only cache +that closes old A pools when switching to B, then cannot use A's surviving free +quarters on return. Preserve every earlier leaf throughout all switches. Useful +payload64KiB, maximum256KiB private backing, about8MiB virtual reservation. + +--race:4workers allocate64guarded leaves each, with bounded test-only2ms pacing +per leaf. RACE prints four bases and waits for 'c'. After acknowledgement, move +only this thread group between A/B32times while allocation is active; record +monotonic move intervals. RACE_DONE prints the allocation interval and waits. +Require at least one move overlapping that interval before claiming concurrency +coverage (this does not prove a particular RCU/CAS interleaving occurred). Check +all race PFNs are charged to A or B, never outside those owned domains. Then move +the stopped fixture to B and acknowledge: it allocates one additional1MiB guarded +PHASE. Require packed density after transitions stop, plus unchanged earlier +race PFNs/charges and all bytes. Worst native payload8MiB, useful2MiB; threads +have128KiB stacks. No tight density claim applies during arbitrary in-flight moves. + +For each held range, read pagemap indexed by target4K VA. A present PFN is in4K +units; native PFN=PFN>>2. Read /proc/kpagecgroup at native_PFN*8 and compare with +stat(the intended cgroup).st_ino. Old phases keep their original allocation-time +charge after process moves. During racing allocations either A/B is valid. +Re-read PFNs to reject migration during the charge sample. Do not use total +memory.current as exact payload accounting: it includes code/stacks/page tables. +Normal8KiB guard stride and --tail4KiB stride are emitted explicitly. + +Use an owned fresh uncapped cgroup hierarchy and a bounded controller. If using +a parent unit and childA/B groups, move the held PID into childA BEFORE enabling +memory in that owned parent's subtree_control (parent must be empty). Do not +change unrelated cgroups or global controller settings. Parent15s alarm kills +all fixture threads; partial thread creation releases waiting workers before +joining. Controller should independently terminate/reap the owned process/unit +on protocol error, timeout or failed metadata, and retain the evidence. + +--selftest only validates byte patterns and same-shard address selection; it does +not exercise cgroup moves, kernel pool allocation or RCU. Hardware qualification +must include bulk/tail/race content and exact-charge/density gates, mm-exit cleanup, +existing COW/fork/reclaim controls, and dense/concurrent timing. No source or +parser-only pass closes these runtime/performance requirements. + +Race density must count newly added native owners. Before and after sampling +the final steady range, recheck the original race PFNs and allocation charges. +Reject old/new physical 4K aliases and inconsistent charges in shared native +owners. Subtract only those native owners actually observed in both stable +snapshots. The 80-owner/1,310,720-byte limit applies to the added native PFN +union for the new 1 MiB payload. Reusing an old owner's free quarters should +not count its existing live quarters as additional backing. Record total, +reused and added owners separately, plus combined per-domain quarter masks. +This is payload accounting, not proof of whole-mm alias closure. + +On Neo the corrected controller observed 77 owners in the final range, of +which 14 were reused and 63 new (1,032,192 added bytes). Old mappings and +charges stayed unchanged, reused slots were disjoint and all bytes passed. +The earlier range-only oracle failed at 81 seen owners before its final +checks; that run remains failed, not retrospectively qualified. diff --git a/tools/testing/selftests/arm64/user4k/domain-cycle.c b/tools/testing/selftests/arm64/user4k/domain-cycle.c new file mode 100644 index 00000000000000..e67b9f1b22b332 --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/domain-cycle.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define LEAF 4096UL +#define NATIVE 16384UL +#define COUNT 256UL +#define STRIDE (2 * LEAF) +#define LENGTH (COUNT * STRIDE) +#define TAIL_SPAN (8UL * 1024 * 1024) +static unsigned char *maps[16]; +static unsigned long leaves = COUNT, stride = STRIDE; +static void fail(const char *s) { perror(s); exit(1); } +static unsigned bucket(uintptr_t p) +{ return ((uint64_t)(p >> 14) * UINT64_C(0x61c8864680b583eb)) >> 60; } +static unsigned char value(unsigned phase, size_t leaf, size_t byte) +{ return (unsigned char)(1 + (phase * 53 + leaf * 7 + byte) % 251); } +static void check(unsigned count) +{ + for (unsigned phase = 0; phase < count; phase++) + for (size_t leaf = 0; leaf < leaves; leaf++) + for (size_t byte = 0; byte < LEAF; byte++) + if (maps[phase][leaf * stride + byte] != value(phase, leaf, byte)) { + fprintf(stderr, "FAIL bytes phase=%u leaf=%zu byte=%zu\n", phase, leaf, byte); exit(1); + } +} +static void ack(void) +{ + char c; ssize_t n; + do { n = read(0, &c, 1); } while (n < 0 && errno == EINTR); + if (n != 1 || c != 'c') fail("controller ack"); +} +#define WORKERS 4 +#define RACE_LEAVES 64UL +static unsigned char *race_maps[WORKERS]; +static atomic_int race_start; +static uint64_t monotonic_ns(void) +{ + struct timespec t; + if (clock_gettime(CLOCK_MONOTONIC, &t)) fail("clock_gettime"); + return (uint64_t)t.tv_sec * 1000000000 + t.tv_nsec; +} +static void race_check(void) +{ + for (unsigned w = 0; w < WORKERS; w++) + for (size_t leaf = 0; leaf < RACE_LEAVES; leaf++) + for (size_t byte = 0; byte < LEAF; byte++) + if (race_maps[w][leaf * STRIDE + byte] != value(w + 32, leaf, byte)) fail("race bytes"); +} +static void *race_worker(void *arg) +{ + unsigned w = (uintptr_t)arg; + int start; + do { start = atomic_load_explicit(&race_start, memory_order_acquire); if (!start) sched_yield(); } while (!start); + if (start != 1) return NULL; + for (size_t leaf = 0; leaf < RACE_LEAVES; leaf++) { + unsigned char *p = race_maps[w] + leaf * STRIDE; + if (mprotect(p, LEAF, PROT_READ | PROT_WRITE)) fail("race mprotect"); + for (size_t byte = 0; byte < LEAF; byte++) p[byte] = value(w + 32, leaf, byte); + /* Test-only pacing keeps allocation active during bounded controller moves. */ + struct timespec delay = {.tv_nsec = 2000000}; + while (nanosleep(&delay, &delay) && errno == EINTR) {} + } + return NULL; +} +static void race_run(void) +{ + pthread_t threads[WORKERS]; + pthread_attr_t attr; + if (pthread_attr_init(&attr) || pthread_attr_setstacksize(&attr, 128 * 1024)) fail("thread attributes"); + unsigned made = 0; + for (; made < WORKERS; made++) { + race_maps[made] = mmap(NULL, RACE_LEAVES * STRIDE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (race_maps[made] == MAP_FAILED) fail("race mmap"); + int err = pthread_create(&threads[made], &attr, race_worker, (void *)(uintptr_t)made); + if (err) { errno = err; break; } + } + pthread_attr_destroy(&attr); + if (made != WORKERS) { + atomic_store_explicit(&race_start, 2, memory_order_release); + for (unsigned i = 0; i < made; i++) pthread_join(threads[i], NULL); + fail("pthread_create"); + } + printf("RACE pid=%d base0=%lx base1=%lx base2=%lx base3=%lx leaves=%lu stride=%lu\n", getpid(), (unsigned long)race_maps[0], (unsigned long)race_maps[1], (unsigned long)race_maps[2], (unsigned long)race_maps[3], RACE_LEAVES, STRIDE); + ack(); + uint64_t begin = monotonic_ns(); + atomic_store_explicit(&race_start, 1, memory_order_release); + for (unsigned i = 0; i < WORKERS; i++) if (pthread_join(threads[i], NULL)) abort(); + uint64_t end = monotonic_ns(); + race_check(); + printf("RACE_DONE pid=%d begin_ns=%llu end_ns=%llu\n", getpid(), (unsigned long long)begin, (unsigned long long)end); + ack(); +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "--selftest")) { + unsigned count = 0; + for (uintptr_t p = 0x100000; p < 0x100000 + TAIL_SPAN; p += NATIVE) count += bucket(p) == 0; + if (count < 16 || value(0, 0, 0) == value(1, 0, 0)) return 1; + puts("PASS patterns/hash selection only; no domain allocation coverage"); return 0; + } + int tail = argc == 3 && !strcmp(argv[2], "--tail"); + int race = argc == 3 && !strcmp(argv[2], "--race"); + if ((argc != 2 && !tail && !race) || strcmp(argv[1], "--native-16k") || getauxval(AT_PAGESZ) != LEAF) return 77; + unsigned phases = tail ? 16 : race ? 1 : 3; + unsigned char *reservation = NULL; + alarm(15); setvbuf(stdout, NULL, _IONBF, 0); + printf("READY pid=%d\n", getpid()); ack(); + if (race) race_run(); + if (tail) { + leaves = 1; stride = LEAF; + reservation = mmap(NULL, TAIL_SPAN + NATIVE, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) fail("tail reserve"); + uintptr_t begin = ((uintptr_t)reservation + NATIVE - 1) & ~(NATIVE - 1); + unsigned found = 0; + for (uintptr_t p = begin; p < begin + TAIL_SPAN && found < phases; p += NATIVE) + if (!bucket(p)) maps[found++] = (void *)p; + if (found != phases) fail("same-shard tail addresses"); + } + for (unsigned phase = 0; phase < phases; phase++) { + check(phase); + if (!tail) { + maps[phase] = mmap(NULL, LENGTH, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (maps[phase] == MAP_FAILED) fail("mmap"); + } + for (size_t leaf = 0; leaf < leaves; leaf++) { + unsigned char *p = maps[phase] + leaf * stride; + if (mprotect(p, LEAF, PROT_READ | PROT_WRITE)) fail("mprotect"); + for (size_t byte = 0; byte < LEAF; byte++) p[byte] = value(phase, leaf, byte); + } + check(phase + 1); + printf("PHASE index=%u pid=%d base=%lx leaves=%lu stride=%lu\n", phase, getpid(), (unsigned long)maps[phase], leaves, stride); + ack(); + } + check(phases); + if (race) { + race_check(); + for (unsigned w = 0; w < WORKERS; w++) if (munmap(race_maps[w], RACE_LEAVES * STRIDE)) fail("race munmap"); + } + if (tail) { + if (munmap(reservation, TAIL_SPAN + NATIVE)) fail("tail munmap"); + } else for (unsigned phase = 0; phase < phases; phase++) if (munmap(maps[phase], LENGTH)) fail("munmap"); + puts("PASS domain allocation phases and all prior bytes; charge/density separately verified"); + return 0; +} From 82d2b049575e9528837f02f075eb75677ff10d2d Mon Sep 17 00:00:00 2001 From: Brent Kearney <1550934+brentkearney@users.noreply.github.com> Date: Thu, 8 Oct 2026 18:28:29 -0600 Subject: [PATCH 304/631] wifi: brcmfmac: log AWDL tx completions behind a runtime parameter Commit bc1823a897a7 ("wifi: brcmfmac: bound AWDL creation and validate its data path") moved the per-frame AWDL tx-completion line from pr_info() to brcmf_dbg(MSGBUF). brcmf_dbg() only prints with DEBUG or CONFIG_BRCM_TRACING; with neither, as in the linux-aurora config, it is no_printk() and the line is not compiled in at all. That line is not only a measurement trace. omdrop-discoverable sends a burst of frames on awdl0 and counts these completions to decide whether the firmware is transmitting. A firmware that has parked the AWDL data path parses peers, fills its peer table and elects normally while completing nothing, so no other signal tells the two apart. Without the line, every window fails the check. Add an awdl_trace module parameter, off by default and writable at runtime (0644), and print the completion only while it is set. A default build stays quiet, and userspace turns the trace on only for the seconds it needs it. This is the same parameter the out-of-tree omdrop-awdl driver already carries, so userspace works with either module. Fixes: bc1823a897a7 ("wifi: brcmfmac: bound AWDL creation and validate its data path") Signed-off-by: Brent Kearney <1550934+brentkearney@users.noreply.github.com> --- .../broadcom/brcm80211/brcmfmac/common.c | 9 +++++++++ .../broadcom/brcm80211/brcmfmac/debug.h | 1 + .../broadcom/brcm80211/brcmfmac/msgbuf.c | 17 +++++++++-------- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index e3a9ca5526c459..2c99a990315682 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -43,6 +43,15 @@ int brcmf_msg_level; module_param_named(debug, brcmf_msg_level, int, 0600); MODULE_PARM_DESC(debug, "Level of debug output"); +/* AWDL tx completions are logged per frame. Userspace counts them to tell a + * radio that is transmitting from one whose firmware has parked the AWDL data + * path: a parked radio completes nothing, and it looks healthy by every other + * measure. Off by default, writable at runtime so no reload is needed. + */ +int brcmf_awdl_trace; +module_param_named(awdl_trace, brcmf_awdl_trace, int, 0644); +MODULE_PARM_DESC(awdl_trace, "Log AWDL tx completions"); + static int brcmf_p2p_enable; module_param_named(p2pon, brcmf_p2p_enable, int, 0); MODULE_PARM_DESC(p2pon, "Enable legacy p2p management functionality"); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h index 432d93ae8fb854..4c7bd41c704c0c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/debug.h @@ -117,6 +117,7 @@ do { \ } while (0) extern int brcmf_msg_level; +extern int brcmf_awdl_trace; struct brcmf_pub; #ifdef DEBUG diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index d6a312e7f151df..d841d9e80637d2 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -985,14 +985,15 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) */ ring_ifidx = msgbuf->flow->rings[flowid] ? brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; - if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) - brcmf_dbg(MSGBUF, "awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - ring_ifidx, tx_status->msg.ifidx, - ifp ? ifp->is_awdl : -1, flowid, - (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), - le16_to_cpu(tx_status->tx_status), - le16_to_cpu(tx_status->metadata_len), - skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + if (ring_ifidx && READ_ONCE(brcmf_awdl_trace) && + __ratelimit(&brcmf_awdl_txs_rs)) + pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + ring_ifidx, tx_status->msg.ifidx, + ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); brcmf_txfinalize(ifp, skb, true); } From 25b2a327c4c3c0df91282561af41ce6e9a581b4e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 10 Oct 2026 04:45:59 -0400 Subject: [PATCH 305/631] drm/asahi: reuse the render state mapping for auxiliary framebuffers Each render pair keeps sixteen generations, with a 32 KiB auxiliary framebuffer in each generation. Allocating a second alias from the fixed 4 MiB auxiliary arena exhausts that arena as logical contexts accumulate, returning ENOSPC and stale GL readback despite spare backing memory. Use the existing full state mapping and its auxiliary offset. The state owner already retains both the pages and their client-VM mapping for the required lifetime. This preserves queue depth and concurrent contexts. Tested on A18 Pro with 16 retained contexts, 128 immediate context recreations, targeted native GL/Zink rendering, native/x86-64/i686 compute oracles and MRT, and GL/GLES glmark2 scene validation. Unknown scene validations remain unknown; this is not a CTS conformance claim. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/gpu/drm/asahi/g17/queue/render/memory.rs | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs index fc5a05e9bf8a91..85319585b3d135 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/memory.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/memory.rs @@ -259,7 +259,6 @@ impl Compact { pub(super) struct Generation { deflake: mmu::KernelMapping, - auxiliary: mmu::KernelMapping, fragment_client: mmu::KernelMapping, state: KernelObject, fragment: KernelObject, @@ -285,15 +284,6 @@ impl Generation { mmu::PROT_GPU_FW_SHARED_RW, mmu::UAT_PGSZ, )?; - let auxiliary_range = cfg::AUXILIARY; - let auxiliary = state.map_range( - vm, - storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE, - auxiliary_range, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - 0, - )?; let fragment = alloc.kernel( mmu::UAT_PGSZ, mmu::UAT_PGSZ as u64, @@ -311,7 +301,6 @@ impl Generation { KBox::new( Self { deflake, - auxiliary, fragment_client, state, fragment, @@ -349,7 +338,10 @@ impl Generation { self.state.gpu_va() + storage::TA_STATUS as u64, self.fragment_client.iova(), self.fragment.gpu_va(), - self.auxiliary.iova(), + // The retained state already maps these pages into the client VM. + // The auxiliary framebuffer accepts a full GPU address; reusing this + // view avoids exhausting the fixed 4 MiB alias arena across pairs. + self.state.gpu_va() + storage::AUXILIARY as u64, ] } } From 3dc1aee8c525ee22fe41c81d2904274da2588537 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 22:12:20 +1100 Subject: [PATCH 306/631] treewide: describe hardware behaviour in comments Rewrite a set of comments so they state the hardware and firmware behaviour directly: - soc/apple SEP (sks.rs, image.rs, profile.rs, sep.rs): key-store request layouts, timeouts and endpoint init described as the enclave protocol. - soc/apple mailbox: the ASC wrapper CPU start/stop sequences. - accel/ane: register, handshake, ring and command-format notes for the T602x/T8112 Neural Engine, plus the scratch3_ack parameter text. - drm/asahi G17: the tiler-heap shrink policy. - mt7932, reset-apple-cio and two documentation pages: keep the descriptions self-contained. Every edit keeps its line count, so line-dependent data (panic locations, WARN/BUG and dynamic-debug records) is unchanged. Apart from the scratch3_ack module parameter description, the change is comments only; there is no functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 6 ++---- drivers/net/wireless/mediatek/mt7932/mt7932.c | 1 + drivers/net/wireless/mediatek/mt7932/reset.c | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 582c43dc5578df..404051b3e6e05e 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -7,10 +7,8 @@ J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second PCIe function. -The Wi-Fi import is pinned to -``aurora-silicon/linux-neo-cleanroom-eryk-with-wifi``, commit -``610cb463c03f9bc68a5d020e7ca443af1fda4856``, with configuration, scan IE and -WPA2 AKM compatibility fixes. Existing source notices are retained. +The Wi-Fi driver carries configuration, scan IE and WPA2 AKM compatibility +fixes on top of its original import. Existing source notices are retained. Co-authored by DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. Platform and configuration diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index ab120796646a47..6de5245864ae53 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* J700 firmware-owned Wi-Fi: native transport and cfg80211 station support. + * Covers firmware download and the NIC capability exchange. * All addresses here are BAR0 offsets, never chip/CPU physical addresses. */ #include diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c index 488a3ba5414b9b..114da7afd69eb5 100644 --- a/drivers/net/wireless/mediatek/mt7932/reset.c +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* J700 MT7932 mapped preparation and function-scoped reset. - * Clean-room contracts: NEO_MT7932_DEVICE_RESET_UNLOAD_CONTRACT.md and - * NEO_MT7932_FLR_PREPARATION_RESOLVED.md. No secondary-bus reset fallback. + * Covers device reset, firmware unload and FLR preparation. + * No secondary-bus reset fallback. */ #include "mt7932.h" From 110f57e4813cdd350a9be3d0e9c51710e20f58ec Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 01:44:15 +1100 Subject: [PATCH 307/631] dt-bindings: soc: apple: sep: Allow a power domain the SEP depends on On T8140 the SEP depends on the GPIO power state while it runs: once that state is switched off, the enclave stops reading its mailbox and does not recover until the next boot. Let the node list the power domain the SEP depends on after its own. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/soc/apple/apple,sep.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml index 989ee4b9e9078a..83ad3aa31a4b7e 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,sep.yaml @@ -33,7 +33,13 @@ properties: maxItems: 1 power-domains: - maxItems: 1 + minItems: 1 + items: + - description: The SEP's own power state. + - description: + A power state the SEP depends on while it runs. On T8140 this is + the GPIO power state; the SEP stops servicing its mailbox for good + once that is switched off. memory-region: maxItems: 1 From 576f88917d5bb7e7e5a2842c075e614f62d431c9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:05:02 +1100 Subject: [PATCH 308/631] ASoC: apple: t8140-aop-audio: count the holders of the lpai run state The low-power microphone's runn state is requested by its own PCM and, around every back-end and microphone-array startup, by with_fabric_raised(), which runs the stream to bring a gated audio fabric back. Each request reads the state back from the firmware and fails on a mismatch, but nothing orders the requests. When two services start at the same time, one raise returns lpai to idle between the other's run request and its readback, and that open fails with EIO: audio.3.auto: lpai power state 0x69646c65 after requesting 0x72756e6e A raise that finishes while the PCM starts likewise idles the capture the PCM just started, and a raise that skips the run request because the PCM holds it powers the leaves while the PCM's close idles the stream under it. Count the holders of runn under a mutex: the first holder requests runn, the last one requests idle, and every lpai power request is made by one caller at a time. Fixes: 656867bfe3ec ("ASoC: apple: t8140-aop-audio: raise a gated audio fabric before the leaves") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 78 ++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 069656e5e6d98b..2d657cc62e5ef0 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -52,6 +52,7 @@ use kernel::{ }, error::from_err_ptr, module_platform_driver, + new_mutex, new_spinlock, of, platform, @@ -73,6 +74,7 @@ use kernel::{ }, Arc, ArcBorrow, + Mutex, SpinLock, // }, types::{ForeignOwnable, Opaque}, @@ -432,6 +434,14 @@ struct StreamState { // sets it and close clears it under the same lock. unsafe impl Send for StreamState {} +/// The holders of the low-power microphone's `runn` state: its own PCM +/// while prepared, and every service startup that raises the fabric. +struct LpaiRun { + users: u32, + /// `runn` is the last state the firmware confirmed. + running: bool, +} + #[pin_data] struct SndSocT8140AopData { dev: ARef, @@ -440,7 +450,7 @@ struct SndSocT8140AopData { ring: Arc, ring_bytes: Atomic, sequence: Atomic, - /// lpai is in `runn`. + /// The low-power microphone PCM holds a reference on `lpai_run`. powered: AtomicFlag, /// A low-power microphone report arrived since the last run request. reported: AtomicFlag, @@ -478,6 +488,10 @@ struct SndSocT8140AopData { hpai_start_work: Work, #[pin] stream: SpinLock, + /// Every lpai power request goes through here, see + /// [`SndSocT8140AopData::lpai_run_get`]. + #[pin] + lpai_run: Mutex, } const SPKR_START_WORK: u64 = 0; @@ -680,6 +694,39 @@ impl SndSocT8140AopData { Ok(()) } + /// Takes a reference on the low-power microphone's `runn` state, + /// requesting it for the first holder. + /// + /// The PCM and the fabric raises share the stream, and every power + /// request is checked against the state the firmware reports back: a + /// raise that returns lpai to `idle` while another service starts up + /// makes that one read `idle` after requesting `runn`, and idles a + /// capture the PCM just started. So they count their holds here and + /// only the first and last holder change the state. + fn lpai_run_get(&self) -> Result<()> { + let mut run = self.lpai_run.lock(); + if !run.running { + self.reported.store(false, Relaxed); + self.set_power(POWER_STATE_RUN)?; + run.running = true; + } + run.users += 1; + Ok(()) + } + + /// Drops a reference taken by [`Self::lpai_run_get`], returning lpai to + /// `idle` after the last holder. If that fails, the next holder finds + /// it still running. + fn lpai_run_put(&self) -> Result<()> { + let mut run = self.lpai_run.lock(); + run.users -= 1; + if run.users == 0 && run.running { + self.set_power(POWER_STATE_IDLE)?; + run.running = false; + } + Ok(()) + } + // ---- the back-end services ---- /// Property 202 with the service's sequence and a 24 MHz timestamp; @@ -785,14 +832,10 @@ impl SndSocT8140AopData { /// does not do that (measured: `audio_p` stays gated for the whole leaf /// retry window), only a running stream does, after which a service in /// `pw0 ` keeps it up only while its stream runs. The low-power - /// microphone is attached for the firmware's lifetime, so run it briefly - /// around `f` when it is idle, once its first report has arrived. + /// microphone is attached for the firmware's lifetime, so hold it in + /// `runn` around `f`, once its first report has arrived. fn with_fabric_raised(&self, f: impl FnOnce() -> Result) -> Result { - if self.powered.load(Relaxed) { - return f(); - } - self.reported.store(false, Relaxed); - self.set_power(POWER_STATE_RUN)?; + self.lpai_run_get()?; let mut polls = 0; while !self.reported.load(Relaxed) && polls < FABRIC_RAISE_POLLS { kernel::time::delay::fsleep(kernel::time::Delta::from_millis(FABRIC_RAISE_POLL_MS)); @@ -802,7 +845,7 @@ impl SndSocT8140AopData { dev_warn!(self.dev, "lpai: no report while raising the fabric\n"); } let ret = f(); - if let Err(e) = self.set_power(POWER_STATE_IDLE) { + if let Err(e) = self.lpai_run_put() { dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); } ret @@ -1192,12 +1235,11 @@ unsafe extern "C" fn lpai_pcm_close(substream: *mut bindings::snd_pcm_substream) let mut st = data.stream.lock(); st.substream = ptr::null_mut(); } - if data.powered.load(Relaxed) { - if let Err(e) = data.set_power(POWER_STATE_IDLE) { + if data.powered.xchg(false, Relaxed) { + if let Err(e) = data.lpai_run_put() { dev_err!(data.dev, "unable to return lpai to idle\n"); return e.to_errno(); } - data.powered.store(false, Relaxed); } 0 } @@ -1221,7 +1263,7 @@ unsafe extern "C" fn lpai_pcm_prepare(substream: *mut bindings::snd_pcm_substrea data.hw_ptr_frames.store(0, Relaxed); } if !data.powered.load(Relaxed) { - if let Err(e) = data.set_power(POWER_STATE_RUN) { + if let Err(e) = data.lpai_run_get() { dev_err!(data.dev, "unable to run lpai\n"); return e.to_errno(); } @@ -2620,8 +2662,10 @@ impl SndSocT8140AopDriver { // SAFETY: the card is freed once, while the binding remains live. unsafe { bindings::snd_card_free(self.card) }; } - if self.data.powered.load(Relaxed) && self.data.set_power(POWER_STATE_IDLE).is_ok() { - self.data.powered.store(false, Relaxed); + // Every stream is closed now; retry an idle request that failed then. + let mut run = self.data.lpai_run.lock(); + if run.running && self.data.set_power(POWER_STATE_IDLE).is_ok() { + run.running = false; } } } @@ -2750,6 +2794,10 @@ impl platform::Driver for SndSocT8140AopDriver { period_acc: 0, buffer_bytes: 0, }), + lpai_run <- new_mutex!(LpaiRun { + users: 0, + running: false, + }), }), GFP_KERNEL, )?; From 7f3d53d348f74ff4128ade73831bdb7a1a9a297a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:02:56 +1100 Subject: [PATCH 309/631] rust: drm: gem: shmem: let drivers allocate the objects the core creates The DRM core allocates some GEM objects itself. drm_gem_shmem_prime_import_sg_table() and drm_gem_shmem_dumb_create() both go through __drm_gem_shmem_create(), which allocates a bare struct drm_gem_shmem_object unless the driver provides gem_create_object. The shmem abstraction offers both helpers but no gem_create_object, while lookup_handle() and the GEM object callbacks take every object of the driver to be an Object. An imported dma-buf or a dumb buffer therefore reaches the driver as an Object whose Rust fields lie past the end of its allocation. Add an optional DriverObject::core_object_args() that gives the arguments for such objects. When a driver provides it, a gem_create_object callback allocates the whole Object and leaves the shmem part to the core, as the C shmem drivers do with their own gem_create_object. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/drm/gem/mod.rs | 9 +++++++ rust/kernel/drm/gem/shmem.rs | 48 +++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/rust/kernel/drm/gem/mod.rs b/rust/kernel/drm/gem/mod.rs index bea304395d67c5..42892c71f84d01 100644 --- a/rust/kernel/drm/gem/mod.rs +++ b/rust/kernel/drm/gem/mod.rs @@ -109,6 +109,15 @@ pub trait DriverObject: Sync + Send + Sized { ) -> Result::Object>> { unimplemented!() } + + /// Arguments for an object that the DRM core allocates on the driver's behalf, such as a + /// dma-buf imported from another device or a dumb buffer. + /// + /// The shmem helpers build the driver data of such an object from these arguments. If the + /// core then fails to initialize the object, it frees it without dropping the driver data. + fn core_object_args(_dev: &drm::Device, _size: usize) -> Result { + build_error!(crate::error::VTABLE_DEFAULT_ERROR) + } } /// Trait that represents a GEM object subtype diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index 2b3a3c2c4a1860..a99997114e8053 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -235,6 +235,48 @@ impl Object { let _ = unsafe { KBox::from_raw(this) }; } + /// `gem_create_object` callback, for the objects the DRM core allocates on the driver's + /// behalf (dma-buf imports and dumb buffers) before it initializes their shmem part. + /// + /// Every GEM object of the driver is taken to be an `Object`, these included, so build one + /// with the driver data from [`DriverObject::core_object_args`]. + extern "C" fn create_object_callback( + raw_dev: *mut bindings::drm_device, + size: usize, + ) -> *mut bindings::drm_gem_object { + // SAFETY: The DRM core only calls this for a device of `T::Driver`, which outlives the + // call. + let dev = unsafe { device::Device::::from_raw(raw_dev) }; + + let new = T::core_object_args(dev, size).and_then(|args| { + KBox::try_pin_init( + try_pin_init!(Self { + obj <- Opaque::init_zeroed(), + parent_resv_obj: None, + sgt_access: AtomicUsize::new(0), + inner <- T::new(dev, size, args), + }), + GFP_KERNEL, + ) + }); + let new = match new { + Ok(new) => new, + Err(e) => return e.to_ptr(), + }; + + // SAFETY: We never move out of `self`. The DRM core owns the object from here on and + // releases it through `free_callback()`. An object the core then fails to initialize is + // freed with `kfree()`, which the `KBox` allocator allows, without dropping `inner`. + let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); + + // SAFETY: `new` is a valid object that nothing else references yet. + unsafe { + let obj = (*new).obj.get(); + (*obj).base.funcs = &Self::VTABLE; + &raw mut (*obj).base + } + } + /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA /// pages for this object. /// @@ -479,7 +521,11 @@ impl driver::AllocImpl for Object { type Driver = T::Driver; const ALLOC_OPS: driver::AllocOps = driver::AllocOps { - gem_create_object: None, + gem_create_object: if T::HAS_CORE_OBJECT_ARGS { + Some(Self::create_object_callback) + } else { + None + }, prime_handle_to_fd: None, prime_fd_to_handle: None, gem_prime_import: None, From 35c86ea57564fd57eafb548a30ea5c63603a1428 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sat, 10 Oct 2026 19:02:58 +1100 Subject: [PATCH 310/631] drm/asahi: allocate imported and dumb GEM objects as asahi objects A dma-buf imported from another device comes back from drm_gem_shmem_prime_import_sg_table() as a bare 456-byte struct drm_gem_shmem_object. gnome-shell imports one at startup and binds it on every frame. ASAHI_VM_BIND then reads the object's flags and the SG lease counter behind its end, and updates the counter there. KASAN reports it on every boot: BUG: KASAN: slab-out-of-bounds in ...ASAHI_VM_BIND+0xb7c/0x1840 [asahi] Read of size 4 at addr ffff00002ac179f0 by task gnome-shell/1175 BUG: KASAN: slab-out-of-bounds in ...Object::owned_sg_table+0x48/0x11c [asahi] Write of size 8 at addr ffff00002ac179d0 by task gnome-shell/1175 ... The buggy address is located 40 bytes to the right of allocated 456-byte region [ffff00002ac17800, ffff00002ac179c8) A dumb buffer created on the card node has the same problem. Give the DRM core the arguments for the objects it allocates, so that imports and dumb buffers are allocated as whole asahi objects: shared, neither VM-private nor kernel-owned, like a GEM_CREATE object without flags. They now also get the driver's GEM callbacks, so closing the last handle in a file unbinds them from that file's VMs, as it does for every other object. Fixes: 80df573af9ef ("rust: drm: gem: shmem: Add DRM shmem helper abstraction") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/asahi/gem.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index a9a67c83c5e80d..290df11e428494 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -345,6 +345,16 @@ impl DriverObject for AsahiObject { }) } + /// Objects the DRM core allocates for us (dma-bufs imported from other devices and dumb + /// buffers) are shared user objects, like a GEM_CREATE object without flags. + fn core_object_args(_dev: &AsahiDevice, _size: usize) -> Result { + Ok(AsahiObjConfig { + flags: 0, + exportable: true, + kernel: false, + }) + } + /// Callback to drop all mappings for a GEM object owned by a given `File` fn close(obj: &::Object, file: &drm::gem::DriverFile) { // fn close(obj: &Object, file: &DrmFile) { From 515aa905d7bfcf9d6a8f3f8e5b39f310ba26109e Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sun, 11 Oct 2026 04:06:15 -0400 Subject: [PATCH 311/631] drm/asahi: preserve TA prefix dependencies across render pairs A fragment-only dependency leaves TA with its physical ring parent. When covered renders migrate between pairs, that parent does not order their TA work. In the Neo Canvas Arcs reproducer, normal two-pair execution stalls, while retaining the cross-pair TA predecessor allows progress. Collect the actual TA QID and kick timestamp alongside the fragment frontier, including predecessors whose graphs are borrowed for preparation. Attach foreign TA predecessors to TA while keeping the fragment frontier on FRG. Same-pair TA dependencies remain implicit; independent prefixes, compute dependencies, pair admission and queue depth are unchanged. Neo validation with production Mesa: the clean four-file patch completed 60 seconds of the failing 18,000-arc workload, and 2,196 targeted CTS outcomes produced 1,026 Pass / 1,170 NotSupported / zero failures or GPU faults. Host checks cover borrowed/resident frontiers, nonadjacent QIDs, unequal and wrapped timestamps, failed producers and barrier capacity. Same-binary opt-in comparison completed pixel-checked resolve repetitions with overlapping timing ranges (median 6.203s off / 6.178s on); capped 2,000-arc browser runs retained their 60 Hz pacing. The tested module uses the deployed651 kernel/PRIME API. Its four changed dependency files and their pre-patch bases match this public tree exactly. The precise firmware resource cycle is not independently established. --- drivers/gpu/drm/asahi/g17/queue/render.rs | 19 ++++++++-- .../asahi/g17/queue/render/dependencies.rs | 36 +++++++++++++++++++ .../asahi/g17/runtime/render/dependencies.rs | 25 +++++++++++++ .../drm/asahi/g17/runtime/render/submit.rs | 13 +++++++ 4 files changed, 91 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 0bf477c0344fb6..24a274e1af62a9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -43,6 +43,8 @@ use retirement::{Ticket, Tracker}; pub(crate) struct RenderDependencies { pub(crate) render: [KickDependency; 2], pub(crate) render_count: usize, + pub(crate) tiling: [KickDependency; 2], + pub(crate) tiling_count: usize, pub(crate) compute: Option, } @@ -50,6 +52,8 @@ impl RenderDependencies { pub(crate) const EMPTY: Self = Self { render: [KickDependency::ZERO; 2], render_count: 0, + tiling: [KickDependency::ZERO; 2], + tiling_count: 0, compute: None, }; pub(crate) fn render(&self) -> &[KickDependency] { @@ -403,8 +407,8 @@ impl Pair { /// Kick barriers of one render. The fragment kick always waits for its own tiling kick; /// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the -/// tiling kick keeps only its implicit parent, otherwise the prefix orders the tiling kick. -/// A compute dependency always orders the tiling kick. +/// tiling kick preserves the TA prefix across physical pairs. Otherwise the render prefix +/// orders the tiling kick. A compute dependency always orders the tiling kick. struct RenderBarriers { tiling: [KickDependency; 3], tiling_count: usize, @@ -430,6 +434,17 @@ fn render_barriers( tiling_count += 1; } } + if fragment_stage { + // The implicit parent covers only this physical ring. Preserve the logical + // TA prefix when a predecessor was published on another pair, without + // making this TA kick wait for that predecessor's fragment completion. + for dependency in &dependencies.tiling[..dependencies.tiling_count] { + if dependency.qid() != own_tiling.qid() { + tiling_barriers[tiling_count] = *dependency; + tiling_count += 1; + } + } + } if let Some(compute) = dependencies.compute { tiling_barriers[tiling_count] = compute; tiling_count += 1; diff --git a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs index be1d9fc9c902ce..53a0fa6e449ed7 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/dependencies.rs @@ -15,6 +15,7 @@ use kernel::{prelude::*, sync::Arc}; struct Predecessor { packet: Arc, word: u64, + ta_word: u64, } /// A loan of the graph cannot hide work already published on that pair. Fence @@ -68,6 +69,21 @@ impl Dependencies { } Ok(()) } + pub(crate) fn collect_tiling( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + for entry in self.entries.iter().flatten() { + include(frontier, status, prefix, &entry.packet, entry.ta_word)?; + } + Ok(()) + } } impl Pair { @@ -81,6 +97,8 @@ impl Pair { entries: core::array::from_fn(|_| { active.next().map(|entry| Predecessor { packet: entry.packet.clone(), + ta_word: KickDependency::new(self.qids()[0], entry.kicks[0]) + .map_or(u64::MAX, KickDependency::word), word: KickDependency::new(qid, entry.kicks[1]) .map_or(u64::MAX, KickDependency::word), }) @@ -106,4 +124,22 @@ impl Pair { } Ok(()) } + pub(crate) fn collect_tiling_dependencies( + &self, + owner: u64, + status: &Arc, + prefix: Option, + frontier: &mut Frontier, + ) -> Result { + if self.owner != Some(owner) { + return Ok(()); + } + let qid = self.qids()[0]; + for entry in self.active.iter() { + let word = + KickDependency::new(qid, entry.kicks[0]).map_or(u64::MAX, KickDependency::word); + include(frontier, status, prefix, &entry.packet, word)?; + } + Ok(()) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs index 09e4322bf42cef..2b14b4548e0db8 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/dependencies.rs @@ -31,4 +31,29 @@ impl Registry { } Ok(frontier) } + pub(in crate::g17::runtime) fn tiling_prefix( + &self, + owner: u64, + packet: &Arc, + ) -> Result { + let mut frontier = Frontier::new(); + for entry in self.render.entries.iter().flatten() { + if let Some(pair) = entry.pair.as_ref() { + pair.collect_tiling_dependencies( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { + dependencies.collect_tiling( + owner, + packet.completion.status(), + packet.order.wait_through[0], + &mut frontier, + )?; + } + } + Ok(frontier) + } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs index dbf7175c5cd4bd..13381b62da86c5 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/submit.rs @@ -187,6 +187,19 @@ impl Registry { .ok_or(EOVERFLOW)? = *dependency; dependencies.render_count += 1; } + if dependencies.render_count != 0 + && matches!(&packet.command, crate::g17::command::Validated::Render { pass, .. } + if pass.fragment_barriers) + { + let tiling = self.tiling_prefix(owner, packet)?; + for dependency in tiling.as_slice() { + *dependencies + .tiling + .get_mut(dependencies.tiling_count) + .ok_or(EOVERFLOW)? = *dependency; + dependencies.tiling_count += 1; + } + } if let Some(queue) = self.compute(owner) { dependencies.compute = queue.dependency( owner, From 276c7a93d6ab67712993235ed464239fbe11f994 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:23:53 +1100 Subject: [PATCH 312/631] Input: macsmc-input - unregister the SMC event notifier on unbind Probe adds a notifier block to the SMC's event chain, but nothing ever takes it off again. The block is embedded in devm-allocated driver data, so after an unbind the chain still links freed memory, and every SMC notification walks through it: a power button press, a lid change or a charger event is then a use-after-free, and once the module has been unloaded the chain calls into unloaded module text. A rebind can also reuse the freed memory for the new block, which zeroes the old link and cuts any handler registered after it, such as macsmc-power's, off the chain. Unregister the notifier from a devm action added right after the registration, so it comes off the chain before the input device and the driver data are released, and stop ignoring the registration result. Fixes: e89b625fdbce ("input: macsmc-input: New driver to handle the Apple Mac SMC buttons/lid") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/input/misc/macsmc-input.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/input/misc/macsmc-input.c b/drivers/input/misc/macsmc-input.c index 2cead3b7f45fed..e149b043fef820 100644 --- a/drivers/input/misc/macsmc-input.c +++ b/drivers/input/misc/macsmc-input.c @@ -106,6 +106,13 @@ static int macsmc_input_event(struct notifier_block *nb, unsigned long event, vo } } +static void macsmc_input_unregister_notifier(void *data) +{ + struct macsmc_input *smcin = data; + + blocking_notifier_chain_unregister(&smcin->smc->event_handlers, &smcin->nb); +} + static int macsmc_input_probe(struct platform_device *pdev) { struct apple_smc *smc = dev_get_drvdata(pdev->dev.parent); @@ -168,7 +175,19 @@ static int macsmc_input_probe(struct platform_device *pdev) input_sync(smcin->input); smcin->nb.notifier_call = macsmc_input_event; - blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + error = blocking_notifier_chain_register(&smc->event_handlers, &smcin->nb); + if (error) + return error; + + /* + * The notifier block lives in devm-allocated memory and the callback + * reports to a devm-managed input device: take it off the SMC's chain + * before either goes away on unbind. + */ + error = devm_add_action_or_reset(&pdev->dev, + macsmc_input_unregister_notifier, smcin); + if (error) + return error; device_init_wakeup(&pdev->dev, true); From f4b4f0ccc1f0216cb0630e0b00e503e608e89e07 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:30:48 +1100 Subject: [PATCH 313/631] dmaengine: apple-admac: Wait for the channel to stop in synchronize device_synchronize() must not return while the controller can still access the memory of terminated descriptors, but admac_synchronize() only kills the tasklet and frees the descriptors. terminate_all() posts the channel's STOP write and resets the rings without waiting for the channel to wind down, so a client that reuses or frees its buffer right after dmaengine_synchronize() races the channel's last burst, and nothing even ensures that the STOP write has reached the controller by then. The START registers hold the per-channel enable bits that the STOP registers clear (m1n1's register map calls them TX_EN/RX_EN and TX_EN_CLR/RX_EN_CLR). Poll the channel's bit until it reads back clear before synchronize goes on; the read also pushes out the posted STOP write. A channel only has a burst to finish, so bound the wait generously and warn instead of hanging the caller if it never clears. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/dma/apple-admac.c | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index a1aeaf3e4dc8f5..dfaeeba9d8e83e 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,10 @@ #define T603X_UNK_28_VAL 0x200000 +/* + * One bit per channel: a write to START sets the channel's enable bit, a write + * to STOP clears it, and START reads back the channels that are enabled. + */ #define REG_TX_START 0x0000 #define REG_TX_STOP 0x0004 #define REG_RX_START 0x0008 @@ -474,6 +479,35 @@ static void admac_stop_chan(struct admac_chan *adchan) dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no); } +/* + * Generous bound for a channel to wind down after its enable bit was cleared: + * it only has to finish the burst in flight, a few frames at audio rates. + */ +#define ADMAC_STOP_TIMEOUT_US 100000 + +static void admac_wait_stopped(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + u32 bit = BIT(adchan->no / 2); + void __iomem *reg; + u32 val; + + switch (admac_chan_direction(adchan->no)) { + case DMA_MEM_TO_DEV: + reg = ad->base + REG_TX_START; + break; + case DMA_DEV_TO_MEM: + reg = ad->base + REG_RX_START; + break; + default: + return; + } + + if (readl_poll_timeout(reg, val, !(val & bit), 10, ADMAC_STOP_TIMEOUT_US)) + dev_warn_ratelimited(ad->dev, "ch%d still enabled %d us after stop\n", + adchan->no, ADMAC_STOP_TIMEOUT_US); +} + static void admac_reset_rings(struct admac_chan *adchan) { struct admac_data *ad = adchan->host; @@ -564,6 +598,13 @@ static void admac_synchronize(struct dma_chan *chan) unsigned long flags; LIST_HEAD(head); + /* + * terminate_all() only requests the stop: make sure the channel no + * longer touches the memory of the terminated descriptors before the + * client is allowed to reuse or free it. + */ + admac_wait_stopped(adchan); + spin_lock_irqsave(&adchan->lock, flags); list_splice_tail_init(&adchan->to_free, &head); spin_unlock_irqrestore(&adchan->lock, flags); From 64cef8942fb389aa55ba98a79a379ab1fb8be49a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:35:02 +1100 Subject: [PATCH 314/631] ASoC: apple: t8140-aop-audio: wait for stopped streams in sync_stop None of the driver's PCMs has a sync_stop op, so after a stop ALSA moves on to hw_params, hw_free or prepare without waiting for the stream to have really stopped. The microphone array and the front-ends stop their ADMAC channel from the trigger with dmaengine_terminate_async(), and the channel is only synchronized at close. hw_free comes first and releases the buffer, which these PCMs allocate at every hw_params, so the channel's last burst and a pending period callback can land after the buffer is gone. Give them the dmaengine helper's sync_stop, as the generic dmaengine PCM does. The low-power microphone has no DMA: its report handler copies each reported span into the ALSA buffer under the stream state lock, after checking the run flag, and the trigger only clears that flag. A report that passed the check before the stop can still be copying when hw_free releases a buffer larger than the preallocated one. Its sync_stop takes the state lock once, which waits for such a copy; any later report sees the flag clear. Fixes: eeb112648f2a ("ASoC: apple: Add T8140 AOP audio driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 58 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 2d657cc62e5ef0..f726c88f0ea041 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -1298,6 +1298,24 @@ unsafe extern "C" fn lpai_pcm_trigger( } } +/// Called after a stop, without the ALSA stream lock, before the buffer is +/// reallocated or freed (hw_params, hw_free) or the counters are reset +/// (prepare). The trigger only clears the run flag, so a report that saw +/// the stream running may still be copying into the buffer: it holds the +/// stream lock for the whole copy, so taking that lock waits for it, and +/// every report after it sees the flag clear. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM, whose private +/// data is the driver's `Arc`. +unsafe extern "C" fn lpai_pcm_sync_stop(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: ALSA calls this op with a live substream of the LPAI PCM. + let data = unsafe { pcm_data(substream) }; + drop(data.stream.lock()); + 0 +} + /// # Safety /// /// Called by ALSA with a live substream of this card's PCM, whose private @@ -1416,6 +1434,19 @@ unsafe extern "C" fn hpai_pcm_trigger( 0 } +/// The trigger stops the DMA asynchronously: wait for the channel to stop +/// and for its completion callbacks to finish before ALSA reuses or frees +/// the buffer. +/// +/// # Safety +/// +/// Called by ALSA with a live substream of this card's PCM. +unsafe extern "C" fn hpai_pcm_sync_stop(substream: *mut bindings::snd_pcm_substream) -> i32 { + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `hpai_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_sync_stop(substream) } +} + /// # Safety /// /// Called once by ALSA when the PCM goes away, with the `Arc` leaked into @@ -1995,6 +2026,28 @@ unsafe extern "C" fn asoc_pcm_trigger( unsafe { bindings::snd_dmaengine_pcm_trigger(substream, cmd) } } +/// Front-end stop synchronization, as for the microphone array: the trigger +/// stops the DMA asynchronously, so wait for the channel to stop and for its +/// completion callbacks to finish before ALSA reuses or frees the buffer. +/// +/// # Safety +/// +/// Called by ASoC on this driver's registered component, with live +/// arguments belonging to one of its links. +unsafe extern "C" fn asoc_pcm_sync_stop( + _component: *mut bindings::snd_soc_component, + substream: *mut bindings::snd_pcm_substream, +) -> i32 { + // SAFETY: ASoC hands its ops a live substream whose private data is the + // `snd_soc_pcm_runtime` of one of this component's links. + if unsafe { rtd_is_be(substream_rtd(substream)) } { + return 0; + } + // SAFETY: the substream is live and its dmaengine runtime was set up by + // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. + unsafe { bindings::snd_dmaengine_pcm_sync_stop(substream) } +} + /// # Safety /// /// Called by ASoC on this driver's registered component, with live @@ -2431,6 +2484,7 @@ impl AsocContext { hw_params: Some(asoc_pcm_hw_params), prepare: Some(asoc_pcm_prepare), trigger: Some(asoc_pcm_trigger), + sync_stop: Some(asoc_pcm_sync_stop), pointer: Some(asoc_pcm_pointer), copy: Some(asoc_pcm_copy), ..Default::default() @@ -2474,7 +2528,7 @@ impl SndSocT8140AopDriver { ioctl: None, hw_params: None, hw_free: None, - sync_stop: None, + sync_stop: Some(lpai_pcm_sync_stop), get_time_info: None, fill_silence: None, copy: None, @@ -2492,7 +2546,7 @@ impl SndSocT8140AopDriver { ioctl: None, hw_params: Some(hpai_pcm_hw_params), hw_free: None, - sync_stop: None, + sync_stop: Some(hpai_pcm_sync_stop), get_time_info: None, fill_silence: None, copy: None, From db5d66f29d415399888c99fd03da3c33849a9756 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:25:16 +1100 Subject: [PATCH 315/631] wifi: mt7932: report the receive bitrate on 40 and 80 MHz channels The station's receive rate is discarded whenever the RX vector's bandwidth field is not 20 MHz. Associations on 5 GHz use 40 and 80 MHz channels where the access point offers them, and most frames on those channels are received at the full width, so "iw dev wlan0 station dump" shows no rx bitrate on most 5 GHz networks. mt7932_rx_rate() already decodes the bandwidth field into rate_info.bw, and cfg80211 computes the bitrate for every width it produces. Keep the rate unless it is wider than the channel the association uses, which a frame from the associated access point cannot be, and keep refusing a CCK rate on 5 GHz. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 34 ++++++++++++++------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 4d89f8925f86d0..80801991c0dbdc 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -164,6 +164,23 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; } +/* Width in MHz of a rate decoded by mt7932_rx_rate(). */ +static int mt_rate_width(enum rate_info_bw bw) +{ + switch (bw) { + case RATE_INFO_BW_20: + return 20; + case RATE_INFO_BW_40: + return 40; + case RATE_INFO_BW_80: + return 80; + case RATE_INFO_BW_160: + return 160; + default: + return INT_MAX; + } +} + void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) { struct sk_buff *skb; @@ -202,17 +219,12 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->station_signal_valid = true; } if (frame->unicast && frame->rate_valid) { - m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], - &m->station_rx_rate); - /* Report only RX widths admitted by the negotiated channel. - * The qualified decoder already bounds mode/MCS/NSS/GI; the - * association remains 20 MHz on 2 GHz and at most 80 MHz on 5 GHz. - */ - if (((frame->rxv >> 12) & 7) > - (m->connect_chandef.width == NL80211_CHAN_WIDTH_80 ? 2 : - m->connect_chandef.width == NL80211_CHAN_WIDTH_40 ? 1 : 0) || - (frame->channel <= 14 && ((frame->rxv >> 12) & 7)) || - (frame->channel > 14 && !((frame->rxv >> 24) & 15))) + struct rate_info *rate = &m->station_rx_rate; + + m->station_rx_rate_valid = mt7932_rx_rate(frame->rxv, m->phy_cap[4], rate); + /* No CCK on 5 GHz, and no rate wider than the associated channel. */ + if ((frame->channel > 14 && !((frame->rxv >> 24) & 15)) || + mt_rate_width(rate->bw) > cfg80211_chandef_get_width(&m->connect_chandef)) m->station_rx_rate_valid = false; m->station_rx_rate_time = jiffies; } From 03a41c0f4d277ef8b20cf38d55187c03eb0a88cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:29:01 +1100 Subject: [PATCH 316/631] wifi: mt7932: monitor the connection's signal against a threshold The driver does not implement set_cqm_rssi_config, so userspace cannot ask to be told when the signal crosses a threshold. wpa_supplicant's "simple" background scan module, which NetworkManager enables, then picks its scan interval from one signal reading at association and never revisits it: when that reading is below the threshold it keeps scanning at the short interval, 30 seconds in NetworkManager's configuration, for as long as the long interval lasts. Each scan takes the station off channel for about a second, which shows up as a latency spike every 31 seconds or so. iwd uses the same events to decide when to look for a better access point. The firmware reports no signal threshold events, so run the monitor on the host, the way mac80211 runs it for drivers without one: average the signal of the frames the associated access point sends, both its beacons and its data, and report a crossing of the threshold through cfg80211_cqm_rssi_notify(). Take at most one sample per 100 ms, about a beacon interval, so that the average follows time rather than traffic, and wait for four samples before the first event. As in mac80211, the first event after a new association or configuration reports the side of the threshold the signal is on, and a later event needs the average to be across the threshold and more than the hysteresis away from the level last reported. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 1 + .../net/wireless/mediatek/mt7932/frontend.c | 11 ++-- drivers/net/wireless/mediatek/mt7932/link.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 13 ++++ .../net/wireless/mediatek/mt7932/startup.c | 1 + .../net/wireless/mediatek/mt7932/station.c | 66 +++++++++++++++++++ 6 files changed, 89 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 80801991c0dbdc..e11d4567595944 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -217,6 +217,7 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) if (frame->signal_valid && signal > -128) { m->station_signal = signal; m->station_signal_valid = true; + mt_cqm_signal(m, signal); } if (frame->unicast && frame->rate_valid) { struct rate_info *rate = &m->station_rx_rate; diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index fc783ebe6b5bcd..aa36c68a106912 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -669,11 +669,14 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) unsigned long flags; spin_lock_irqsave(&m->response_lock, flags); - if (tim && tim->datalen >= 4 && m->connected && !m->disconnecting && - !m->stopping && !m->power_tim && frame.channel == m->connect_channel && + if (m->connected && !m->disconnecting && !m->stopping && + frame.channel == m->connect_channel && ether_addr_equal(frame.data + 16, m->connect_bssid)) { - m->power_tim = true; - schedule_work(&m->power_work); + mt_cqm_signal(m, mt7932_rx_signal(&frame, m->antenna_mask)); + if (tim && tim->datalen >= 4 && !m->power_tim) { + m->power_tim = true; + schedule_work(&m->power_work); + } } spin_unlock_irqrestore(&m->response_lock, flags); } diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index e81fb052c40d79..da471022d7da3e 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -535,6 +535,7 @@ void mt_connect_work(struct work_struct *work) m->station_rx_rate_valid = false; m->station_tx_rate_valid = false; spin_unlock(&m->data_lock); + mt_cqm_reset(m); } m->connected = !ret; m->disconnecting = false; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 9e7ae595cbd1c3..e2b430d2db9356 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -3,6 +3,7 @@ #define MT7932_H +#include #include #include #include @@ -40,6 +41,8 @@ #define STRIDE 8192 #define RX_STRIDE ALIGN(MT7932_RX_CAPACITY, 64) +DECLARE_EWMA(mt7932_signal, 4, 4) + struct mt7932; int mt_dma_stop(struct mt7932 *m); int mt_function_reset(struct mt7932 *m); @@ -162,6 +165,12 @@ struct mt7932 { struct rate_info station_tx_rate; unsigned long station_tx_rate_time, station_rate_query_time; bool station_tx_rate_valid, station_rate_query_failed; + /* Connection quality monitor, protected by response_lock. */ + struct ewma_mt7932_signal cqm_signal; + unsigned long cqm_sample_time; + s32 cqm_threshold, cqm_last_event; + u32 cqm_hysteresis; + u8 cqm_samples; u8 *data_headers, *data_payloads; dma_addr_t data_headers_dma, data_payloads_dma; bool data_live[512], data_token[512], data_retired[512]; @@ -232,6 +241,10 @@ int mt_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, const u8 *mac, struct station_info *sinfo); int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, int idx, u8 *mac, struct station_info *sinfo); +int mt_set_cqm_rssi_config(struct wiphy *wiphy, struct net_device *netdev, + s32 rssi_thold, u32 rssi_hyst); +void mt_cqm_reset(struct mt7932 *m); +void mt_cqm_signal(struct mt7932 *m, int signal); int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 6f4efe3d247530..883eb03e9204cf 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -527,6 +527,7 @@ static const struct cfg80211_ops mt_cfg_ops = { .get_channel = mt_get_channel, .dump_station = mt_dump_station, .set_power_mgmt = mt_set_power_mgmt, + .set_cqm_rssi_config = mt_set_cqm_rssi_config, }; int mt_register_regulatory_gate(struct mt7932 *m) diff --git a/drivers/net/wireless/mediatek/mt7932/station.c b/drivers/net/wireless/mediatek/mt7932/station.c index c9d388e50d8d52..83cee4b650926e 100644 --- a/drivers/net/wireless/mediatek/mt7932/station.c +++ b/drivers/net/wireless/mediatek/mt7932/station.c @@ -103,3 +103,69 @@ int mt_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, spin_unlock_irqrestore(&m->response_lock, flags); return ret; } + +/* + * The firmware reports no signal threshold events, so the connection quality + * monitor runs on the host, as mac80211 runs it for drivers without one: + * average the signal of the frames the associated access point sends and + * report when the average crosses the threshold. Sample at most once per + * beacon interval, so that the average follows time rather than traffic. + */ +#define MT7932_CQM_INTERVAL (HZ / 10) +#define MT7932_CQM_MIN_SAMPLES 4 + +/* Caller holds response_lock; a new association starts a new average. */ +void mt_cqm_reset(struct mt7932 *m) +{ + ewma_mt7932_signal_init(&m->cqm_signal); + m->cqm_samples = 0; + m->cqm_last_event = 0; +} + +/* Caller holds response_lock; @signal is a frame from the associated AP. */ +void mt_cqm_signal(struct mt7932 *m, int signal) +{ + enum nl80211_cqm_rssi_threshold_event event; + int average, last = m->cqm_last_event; + int hysteresis = m->cqm_hysteresis; + + if (signal >= 0 || signal <= -128) + return; + if (m->cqm_samples && + time_before(jiffies, m->cqm_sample_time + MT7932_CQM_INTERVAL)) + return; + m->cqm_sample_time = jiffies; + ewma_mt7932_signal_add(&m->cqm_signal, -signal); + if (m->cqm_samples < MT7932_CQM_MIN_SAMPLES) + m->cqm_samples++; + if (!m->cqm_threshold || m->cqm_samples < MT7932_CQM_MIN_SAMPLES) + return; + + /* The first event after (re)configuration reports the current side. */ + average = -(int)ewma_mt7932_signal_read(&m->cqm_signal); + if (average < m->cqm_threshold && (!last || average < last - hysteresis)) + event = NL80211_CQM_RSSI_THRESHOLD_EVENT_LOW; + else if (average > m->cqm_threshold && (!last || average > last + hysteresis)) + event = NL80211_CQM_RSSI_THRESHOLD_EVENT_HIGH; + else + return; + m->cqm_last_event = average; + cfg80211_cqm_rssi_notify(m->netdev, event, average, GFP_ATOMIC); +} + +int mt_set_cqm_rssi_config(struct wiphy *wiphy, struct net_device *netdev, + s32 rssi_thold, u32 rssi_hyst) +{ + struct mt7932 *m = mt_from_wiphy(wiphy); + unsigned long flags; + + if (netdev != m->netdev) + return -ENODEV; + spin_lock_irqsave(&m->response_lock, flags); + /* A zero threshold disables the monitor. */ + m->cqm_threshold = rssi_thold; + m->cqm_hysteresis = min_t(u32, rssi_hyst, S8_MAX); + m->cqm_last_event = 0; + spin_unlock_irqrestore(&m->response_lock, flags); + return 0; +} From ad2de091a5c6ed9176ad479cd66dbfec519e361a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:35:42 +1100 Subject: [PATCH 317/631] Bluetooth: mt7932: reset the controller instead of faulting the transport Any controller error ends the Bluetooth function for the rest of the boot. A command timeout, a Hardware Error event from the controller, the core's own recovery when the controller stops accepting commands, and a write to the device's "reset" sysfs attribute all reach bt7932_reset() or bt7932_hw_error(), which mark the transport faulted, stop bus mastering and leave the adapter dead until a cold boot. A failed setup command does the same. None of these mean that the transport is broken: the DMA rings were consistent when the controller stopped responding. Recover the way other Bluetooth drivers do. On a command timeout, inject a Hardware Error event with hci_reset_dev(); the core then closes the device, which resets the controller, and opens it again, which repeats setup and sends the calibration, PTX and address again. A Hardware Error event from the controller takes the same path. Let a setup that fails return its error, so that a later power-on can try again. Bound it so that a controller that no longer answers cannot cause a reset loop: while a recovery is pending or the device is being reopened, further timeouts do not start another one, and if the setup that follows a recovery fails, the transport faults as before. The next successful setup ends the recovery. Transport faults, where the ring counters or the firmware state are inconsistent, still fault the function: reloading the firmware needs a reset of the PCI function, which this driver does not perform. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 7 ++- drivers/bluetooth/Kconfig | 5 +- drivers/bluetooth/mt7932_bt_pcie.c | 61 ++++++++++++++++++- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 404051b3e6e05e..00a608fab482da 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -161,8 +161,11 @@ Limitations Removal first quiesces firmware, then stops work and interrupts, disables bus mastering and drains pending PCI transactions before freeing DMA. An unconfirmed drain retains storage - until reset. Firmware recovery and warm reprobe are unsupported; use a fresh - cold boot after removing or faulting an active transport. + until reset. A controller that stops answering commands, or reports a + hardware error, is reset over HCI and set up again; if that setup fails, + the transport faults. Firmware reloading and warm reprobe are + unsupported; use a fresh cold boot after removing or faulting an active + transport. * The Bluetooth software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. * Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 975801c66fe30c..10ffa3f3f27dcd 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -37,8 +37,9 @@ config BT_MTK7932_PCIE is mounted and read their firmware when probed, so say M, like the Wi-Fi driver. System suspend uses the firmware sleep handshake and preserves DMA - storage. The PCIe bootstrap retains its own PM and kexec blockers. - Hibernation and automatic firmware recovery are unsupported. + storage. Hibernation and firmware reloading are unsupported. A + controller that stops answering commands is reset over HCI and set + up again; if that setup fails, the transport stays faulted. Removal drains the PCI function before releasing device-visible memory; an unconfirmed drain retains it until a full hardware reset. diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index a0480b36fbf02a..fc1b2f636ba782 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -103,6 +103,14 @@ enum bt7932_pm_state { BT7932_RESTORING, }; +enum bt7932_recovery { + BT7932_RECOVERY_NONE, + /* A Hardware Error event was injected and not yet handled. */ + BT7932_RECOVERY_REQUESTED, + /* The core is closing and reopening the device. */ + BT7932_RECOVERY_REOPENING, +}; + struct bt7932 { struct pci_dev *pdev; void __iomem *bar; @@ -132,6 +140,7 @@ struct bt7932 { bool enabled, regions, vectors, irq_requested, retained; bool ipc_ready, opened, awake, waking, fault, registered; bool stopping, runtime_forbidden; + enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; u64 tx_packets, rx_packets, diagnostics; }; @@ -832,6 +841,9 @@ static int bt7932_open(struct hci_dev *hdev) ret = -EHOSTDOWN; else WRITE_ONCE(bt->opened, true); + /* Closing the device drops an injected event that was still queued. */ + if (bt->recovery == BT7932_RECOVERY_REQUESTED) + bt->recovery = BT7932_RECOVERY_NONE; mutex_unlock(&bt->lock); return ret; } @@ -939,28 +951,71 @@ static int bt7932_setup(struct hci_dev *hdev) if (ret) goto fault; dev_info(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); + mutex_lock(&bt->lock); + if (bt->recovery == BT7932_RECOVERY_REOPENING) + dev_info(&bt->pdev->dev, "controller recovered\n"); + bt->recovery = BT7932_RECOVERY_NONE; + mutex_unlock(&bt->lock); return 0; fault: dev_err(&bt->pdev->dev, "BT_HCI_SETUP_FAILED stage=%s opcode=%04x status=%d\n", stage, opcode, ret); + /* + * The transport is intact, so a requested recovery or a later power-on + * may try again. A controller that fails the setup of its recovery, + * though, does not come back without a reset of the PCI function. + */ mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "HCI setup command failed; no retry"); + if (bt->recovery == BT7932_RECOVERY_REOPENING) + bt7932_fault_locked(bt, "controller recovery failed"); mutex_unlock(&bt->lock); return ret; } +/* + * Called when a command times out, and through the "reset" sysfs attribute. + * While the transport rings are consistent, recover the controller the + * standard way: inject a Hardware Error event, so that the core closes the + * device, resetting the controller, and opens it again, repeating setup. + * Allow one attempt until a setup succeeds, so that a controller that no + * longer answers ends in a fault instead of a reset loop. + */ static void bt7932_reset(struct hci_dev *hdev) { struct bt7932 *bt = hci_get_drvdata(hdev); + bool recover; mutex_lock(&bt->lock); - bt7932_fault_locked(bt, "HCI requested recovery; cold reset required"); + recover = !bt->fault && !bt->stopping && + bt->recovery == BT7932_RECOVERY_NONE && + bt->pm_state == BT7932_RUNNING; + if (recover) + bt->recovery = BT7932_RECOVERY_REQUESTED; mutex_unlock(&bt->lock); + if (!recover) + return; + dev_warn(&bt->pdev->dev, "resetting the controller\n"); + if (hci_reset_dev(hdev)) { + mutex_lock(&bt->lock); + if (bt->recovery == BT7932_RECOVERY_REQUESTED) + bt->recovery = BT7932_RECOVERY_NONE; + mutex_unlock(&bt->lock); + } } +/* + * The core closes and reopens the device after this, for a Hardware Error + * event from the controller as for one injected above. That reopening is + * the recovery attempt. + */ static void bt7932_hw_error(struct hci_dev *hdev, u8 code) { - bt7932_reset(hdev); + struct bt7932 *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->recovery = BT7932_RECOVERY_REOPENING; + mutex_unlock(&bt->lock); + bt_dev_err(hdev, "hardware error 0x%2.2x", code); } static int bt7932_iommu(struct bt7932 *bt) From a100b3d2f82a69f71ec4943a9588ccabf22384cf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:20:16 +1100 Subject: [PATCH 318/631] drm: apple: fade between H17P backlight levels The H17P backlight presents every new level in one step, so each press of a brightness key and every change from an ambient light policy shows as a visible jump on the panel. Fade between levels instead. The policy now keeps the level it presents separate from the requested target: a new target for a panel that stays lit at a known level starts a linear fade from the level presented last, and a delayed work item moves it every 16 ms for 250 ms, the last step being the target itself. Each step reuses the existing brightness-only present, so steps that come faster than presents complete merge into the next one, and a step does not bypass the backoff of a rejected present. A new level during a fade restarts it from where it is. Blanking, unblanking, suspend and a level the panel may not show yet still jump as before, and a fade still running at system sleep ends at its target before the queue is drained. The KUnit cases cover the fade steps, restarts, merging, overflow and the cases that still jump. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 6 + drivers/gpu/drm/apple/dcp.c | 12 +- drivers/gpu/drm/apple/dcp_backlight.c | 94 ++++++++++++- drivers/gpu/drm/apple/dcp_backlight.h | 76 ++++++++++- drivers/gpu/drm/apple/dcp_backlight_test.c | 152 +++++++++++++++++++++ 6 files changed, 330 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index c15bf2a340f8c8..14b5531c2b55fd 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -64,7 +64,8 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST default KUNIT_ALL_TESTS help Build tests for the Apple display driver's AFK ring parser and - backlight policy (takeover, DPMS, suspend and present completion). + backlight policy (takeover, DPMS, suspend, fades and present + completion). The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 109c3ef75a6733..c8b46a57a69539 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -519,6 +519,9 @@ struct apple_dcp { spinlock_t lock; struct dcp_backlight_state state; void (*kick)(struct apple_dcp *dcp); + /* Steps a running fade; fade_start is its start in jiffies. */ + struct delayed_work fade_work; + unsigned long fade_start; /* Rate limit for firmware re-send requests. */ unsigned long resent_at; bool resent; @@ -583,6 +586,9 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); +int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); +void dcp_backlight_stop(struct apple_dcp *dcp); +void dcp_backlight_finish_fade(struct apple_dcp *dcp); int dcp_backlight_register(struct apple_dcp *dcp); int dcp_backlight_update(struct apple_dcp *dcp); s32 dcp_ext_backlight_value(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a9ab04913df3a9..c2b6638aab9ce9 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4347,6 +4347,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -4418,6 +4419,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp_release_piodma_iommu_dev(dcp); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); @@ -4543,7 +4545,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) INIT_LIST_HEAD(&dcp->swapped_out_fbs); mutex_init(&dcp->swapped_out_fbs_lock); - spin_lock_init(&dcp->backlight.lock); + ret = dcp_backlight_init(dcp, dev); + if (ret) + return ret; iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; @@ -4984,6 +4988,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) */ iomfb_queue_stop(dcp); cancel_delayed_work_sync(&dcp->bl_fallback_wq); + dcp_backlight_stop(dcp); cancel_work_sync(&dcp->bl_register_wq); cancel_work_sync(&dcp->bl_update_wq); return; @@ -5089,8 +5094,11 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) * A completed present can schedule one more brightness present, and a * rejected one a delayed retry; run a pending retry now instead of * letting it fire after the mailbox has suspended. Four passes cover - * the first attempt and the policy's three retries. + * the first attempt and the policy's three retries. A fade still + * running ends at its target first. */ + if (dcp_has_panel(dcp)) + dcp_backlight_finish_fade(dcp); for (pass = 0; pass < 4; pass++) { flush_delayed_work(&dcp->iomfb.backlight_retry); if (dcp_has_panel(dcp)) diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index f4e76830a71f6c..24fc2077e238aa 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -85,6 +85,14 @@ static u32 brightness_part2[] = { }; +/* + * A new level fades in over DCP_BL_FADE_MS, one brightness-only present per + * step at most. Steps that come faster than presents complete merge, so the + * step only bounds how smooth the fade can be. + */ +#define DCP_BL_FADE_MS 250U +#define DCP_BL_FADE_STEP_MS 16U + bool dcp_backlight_active(struct apple_dcp *dcp) { return dcp->fw_compat == DCP_FIRMWARE_H17P && @@ -148,8 +156,10 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, ret = dcp_bl_init(&dcp->backlight.state, maximum, inherited_valid, inherited, default_valid, default_nits); - if (!ret) + if (!ret) { + dcp->backlight.state.fade = true; dcp->backlight.kick = kick; + } } spin_unlock_irqrestore(&dcp->backlight.lock, flags); mutex_unlock(&dcp->bl_register_mutex); @@ -177,6 +187,71 @@ static void dcp_backlight_kick(struct apple_dcp *dcp) kick(dcp); } +static void dcp_backlight_fade_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), + struct apple_dcp, + backlight.fade_work); + unsigned long flags; + bool running, kick; + u32 elapsed; + + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || + READ_ONCE(dcp->iomfb.stopped)) + return; + + spin_lock_irqsave(&dcp->backlight.lock, flags); + elapsed = jiffies_to_msecs(jiffies - dcp->backlight.fade_start); + running = dcp_bl_fade_step(&dcp->backlight.state, elapsed, + DCP_BL_FADE_MS); + /* A rejected present is retried with backoff, carrying the new level. */ + kick = dcp->backlight.state.dirty && !dcp->backlight.state.retries; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + + if (kick) + dcp_backlight_kick(dcp); + if (running) + queue_delayed_work(system_wq, &dcp->backlight.fade_work, + msecs_to_jiffies(DCP_BL_FADE_STEP_MS)); +} + +void dcp_backlight_stop(struct apple_dcp *dcp) +{ + cancel_delayed_work_sync(&dcp->backlight.fade_work); +} + +static void dcp_backlight_release(void *data) +{ + dcp_backlight_stop(data); +} + +/* + * The backlight device is a devres of @dev, registered after probe, so it is + * unregistered before this action runs, and no request can start a fade + * afterwards. + */ +int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev) +{ + spin_lock_init(&dcp->backlight.lock); + INIT_DELAYED_WORK(&dcp->backlight.fade_work, dcp_backlight_fade_work); + return devm_add_action_or_reset(dev, dcp_backlight_release, dcp); +} + +/* Present the target of a running fade at once, e.g. before system sleep. */ +void dcp_backlight_finish_fade(struct apple_dcp *dcp) +{ + unsigned long flags; + bool kick; + + cancel_delayed_work_sync(&dcp->backlight.fade_work); + spin_lock_irqsave(&dcp->backlight.lock, flags); + dcp_bl_fade_step(&dcp->backlight.state, DCP_BL_FADE_MS, DCP_BL_FADE_MS); + kick = dcp->backlight.state.dirty; + spin_unlock_irqrestore(&dcp->backlight.lock, flags); + if (kick) + dcp_backlight_kick(dcp); +} + int dcp_backlight_dpms(struct apple_dcp *dcp, bool on) { unsigned long flags; @@ -413,14 +488,25 @@ static int dcp_set_brightness(struct backlight_device *bd) return -ENODEV; if (dcp_backlight_active(dcp)) { + struct dcp_backlight_state *state = &dcp->backlight.state; + bool fade; + u32 old; + /* Preserve the requested level even while the core forces zero. */ spin_lock_irqsave(&dcp->backlight.lock, flags); - ret = dcp_bl_request(&dcp->backlight.state, - bd->props.brightness, + old = state->target; + ret = dcp_bl_request(state, bd->props.brightness, backlight_is_blank(bd), bd->props.state & BL_CORE_SUSPENDED); + /* A new level restarts the fade from the level presented last. */ + fade = !ret && state->fading && state->target != old; + if (fade) + dcp->backlight.fade_start = jiffies; spin_unlock_irqrestore(&dcp->backlight.lock, flags); - if (!ret) + if (fade) + queue_delayed_work(system_wq, &dcp->backlight.fade_work, + msecs_to_jiffies(DCP_BL_FADE_STEP_MS)); + else if (!ret) dcp_backlight_kick(dcp); return ret; } diff --git a/drivers/gpu/drm/apple/dcp_backlight.h b/drivers/gpu/drm/apple/dcp_backlight.h index 1232ff40a84ba2..92ac07b2e9d9d4 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.h +++ b/drivers/gpu/drm/apple/dcp_backlight.h @@ -6,13 +6,20 @@ #include #include +#include #include #include -/* Commanded nits, independent of firmware encoding. No fade policy. */ +/* + * Commanded nits, independent of firmware encoding. target is the requested + * level and level the one presented while the panel is lit. They differ only + * while a fade runs; the caller supplies the fade's clock. + */ struct dcp_backlight_state { u32 maximum; u32 target; + u32 level; + u32 fade_from; u32 actual; u32 sent_nits; u64 sequence; @@ -24,6 +31,10 @@ struct dcp_backlight_state { bool suspended; bool dirty; bool in_flight; + /* Fade between lit levels instead of jumping to them. */ + bool fade; + /* A fade from fade_from to target is running. */ + bool fading; /* * The panel shows a level Linux knows: the loader's reported level, or * one a completed present carried. A default used for registration @@ -73,6 +84,7 @@ static inline int dcp_bl_init(struct dcp_backlight_state *state, u32 maximum, *state = (struct dcp_backlight_state) { .maximum = maximum, .target = nits, + .level = nits, .actual = nits, .ready = true, .level_known = inherited_valid, @@ -89,21 +101,40 @@ static inline bool dcp_bl_seed(struct dcp_backlight_state *state, u32 nits) if (!state->ready || state->controlled || nits > state->maximum) return false; state->target = nits; + state->level = nits; state->actual = nits; state->level_known = true; return true; } +static inline bool dcp_bl_lit(const struct dcp_backlight_state *state) +{ + return !state->dpms_off && !state->core_blank && !state->suspended; +} + static inline u32 dcp_bl_effective(const struct dcp_backlight_state *state) { - if (state->dpms_off || state->core_blank || state->suspended) - return 0; - return state->target; + return dcp_bl_lit(state) ? state->level : 0; } +/* Stop a running fade and present the target directly. */ +static inline void dcp_bl_jump(struct dcp_backlight_state *state) +{ + state->fading = false; + state->level = state->target; + state->dirty = true; +} + +/* + * With fades enabled, a new level for a panel that stays lit at a known level + * starts a fade from the level presented last; dcp_bl_fade_step() moves it. + * Blanking, unblanking and a level the panel may not show still jump. + */ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, bool core_blank, bool suspended) { + bool was_lit = dcp_bl_lit(state); + if (!state->ready) return -ENODATA; if (nits > state->maximum) @@ -119,11 +150,44 @@ static inline int dcp_bl_request(struct dcp_backlight_state *state, u32 nits, state->target = nits; state->core_blank = core_blank; state->suspended = suspended; - state->dirty = true; + if (state->fade && state->level_known && was_lit && + dcp_bl_lit(state) && state->level != nits) { + state->fade_from = state->level; + state->fading = true; + } else { + dcp_bl_jump(state); + } } return 0; } +/* + * Move a running fade to @elapsed of @duration, in any one unit. The level + * follows a straight line from the level presented when the fade started, + * and the last step is the target itself. Returns true while the fade runs. + */ +static inline bool dcp_bl_fade_step(struct dcp_backlight_state *state, + u32 elapsed, u32 duration) +{ + u32 from = state->fade_from, to = state->target, level; + + if (!state->fading) + return false; + if (elapsed >= duration) { + level = to; + state->fading = false; + } else if (to >= from) { + level = from + div_u64(mul_u32_u32(to - from, elapsed), duration); + } else { + level = from - div_u64(mul_u32_u32(from - to, elapsed), duration); + } + if (state->level != level) { + state->level = level; + state->dirty = true; + } + return state->fading; +} + static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) { if (!state->ready) @@ -132,7 +196,7 @@ static inline int dcp_bl_dpms(struct dcp_backlight_state *state, bool on) state->retries = 0; if (state->dpms_off != !on) { state->dpms_off = !on; - state->dirty = true; + dcp_bl_jump(state); } return 0; } diff --git a/drivers/gpu/drm/apple/dcp_backlight_test.c b/drivers/gpu/drm/apple/dcp_backlight_test.c index 41ac118f26cee9..a6fb45958b1684 100644 --- a/drivers/gpu/drm/apple/dcp_backlight_test.c +++ b/drivers/gpu/drm/apple/dcp_backlight_test.c @@ -352,6 +352,155 @@ static void backlight_seed_known_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, dcp_bl_resend(&state)); } +static void backlight_fade_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_EQ(test, state.target, 400U); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 100U); + + /* The start of the fade is the level already shown. */ + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 0, 250)); + KUNIT_EXPECT_FALSE(test, state.dirty); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 250U); + + /* Steps during a present merge into the next one. */ + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 150, 250)); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 200, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_prepare(&state, true, &present), -EBUSY); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 250U); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 340U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + + /* The last step is exactly the target, however late it runs. */ + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 1000, 250)); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_EXPECT_EQ(test, state.actual, 400U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_FALSE(test, state.dirty); + + /* Fading down reaches the target too, including zero. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 0, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 249, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 2U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); +} + +static void backlight_fade_restart_test(struct kunit *test) +{ + struct dcp_backlight_state state; + + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 100, 250)); + KUNIT_EXPECT_EQ(test, state.level, 260U); + + /* Repeating the target keeps the running fade. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_EQ(test, state.fade_from, 100U); + + /* A new level replaces it, starting from the level presented last. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 60, false, false), 0); + KUNIT_EXPECT_TRUE(test, state.fading); + KUNIT_EXPECT_EQ(test, state.fade_from, 260U); + KUNIT_EXPECT_EQ(test, state.level, 260U); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_EQ(test, state.level, 160U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, state.level, 60U); + + /* A new level equal to the one presented ends the fade there. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 300, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_EXPECT_EQ(test, state.level, 180U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 180, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 180U); + + /* No overflow over the whole range a panel ceiling may have. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, INT_MAX, true, 0, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, INT_MAX, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 249, 250)); + KUNIT_EXPECT_EQ(test, state.level, (u32)div_u64((u64)INT_MAX * 249, 250)); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 250, 250)); + KUNIT_EXPECT_EQ(test, state.level, (u32)INT_MAX); +} + +static void backlight_fade_blank_test(struct kunit *test) +{ + struct dcp_backlight_state state; + struct dcp_backlight_present present; + + /* DPMS off and on during a fade jump, as without fades. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 125, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_TRUE(test, state.dirty); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_EXPECT_FALSE(test, dcp_bl_fade_step(&state, 200, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 0U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + KUNIT_ASSERT_EQ(test, dcp_bl_dpms(&state, true), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 400U); + KUNIT_EXPECT_TRUE(test, dcp_bl_complete(&state, present.sequence, true)); + + /* A level set while blanked is shown at once on unblank. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, true, false), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 200, true, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 200, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 200U); + + /* Suspend during a fade blanks at once; resume shows the target. */ + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_TRUE(test, dcp_bl_fade_step(&state, 50, 250)); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, true, true), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 0U); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 500, false, false), 0); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 500U); + + /* A level the panel may not show yet is presented directly. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 525, false, 0, true, 262), 0); + state.fade = true; + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 100, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_ASSERT_EQ(test, dcp_bl_prepare(&state, true, &present), 0); + KUNIT_EXPECT_EQ(test, present.nits, 100U); + + /* Without fades a new level is presented directly. */ + KUNIT_ASSERT_EQ(test, dcp_bl_init(&state, 600, true, 100, false, 0), 0); + KUNIT_ASSERT_EQ(test, dcp_bl_request(&state, 400, false, false), 0); + KUNIT_EXPECT_FALSE(test, state.fading); + KUNIT_EXPECT_EQ(test, dcp_bl_effective(&state), 400U); +} + static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_millinits_takeover_test), KUNIT_CASE(backlight_takeover_test), @@ -366,6 +515,9 @@ static struct kunit_case backlight_cases[] = { KUNIT_CASE(backlight_fallback_test), KUNIT_CASE(backlight_fallback_same_level_test), KUNIT_CASE(backlight_seed_known_test), + KUNIT_CASE(backlight_fade_test), + KUNIT_CASE(backlight_fade_restart_test), + KUNIT_CASE(backlight_fade_blank_test), {} }; From 8cb4cfafddc59ec9787dfb2adf7792fcc98090db Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:37:55 +1100 Subject: [PATCH 319/631] btrfs: keep the extent map referenced until a compressed read is submitted When btrfs_do_readpage() adds blocks of a compressed extent to a bio it drops its reference on the extent map, and btrfs_submit_compressed_read() looks the extent map up again in the inode's extent map tree, by the bio's file offset, to learn where the extent starts in the file and how it is compressed. If the lookup misses, the whole bio fails with -EIO. The lookup can miss. The bio is submitted when a later block does not fit it, or at the end of the read_folio/readahead call after the range has been unlocked, and the extent map shrinker can remove the extent map from the tree in between: apart from pinned extent maps it drops whatever it finds, regardless of locked io tree ranges or of references held by readers. Under memory pressure, in particular in a memory cgroup at its limit whose reclaim keeps invoking the fs shrinker, this shows up as spurious -EIO on reads of compressed files, and on buffered writes that have to read a partial folio first. Have the bio_ctrl hold a reference on the extent map of the compressed extent its bio reads, handed over by btrfs_do_readpage() once the block is in the bio and dropped once the bio is submitted, and pass it to btrfs_submit_compressed_read() instead of looking it up again. Fixes: 956a17d9d050 ("btrfs: add a shrinker for extent maps") Assisted-by: LLM Signed-off-by: Ryan Murray --- fs/btrfs/compression.c | 18 +++++++--------- fs/btrfs/compression.h | 3 ++- fs/btrfs/extent_io.c | 47 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 2ceb5661e0718a..96ae95c3029bc3 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -540,11 +540,10 @@ static noinline int add_ra_bio_pages(struct inode *inode, * After the compressed pages are read, we copy the bytes into the * bio we were passed and then call the bio end_io calls */ -void btrfs_submit_compressed_read(struct btrfs_bio *bbio) +void btrfs_submit_compressed_read(struct btrfs_bio *bbio, struct extent_map *em) { struct btrfs_inode *inode = bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_map_tree *em_tree = &inode->extent_tree; struct compressed_bio *cb; unsigned int compressed_len; const u32 min_folio_size = btrfs_min_folio_size(fs_info); @@ -552,7 +551,6 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) gfp_t gfp; u64 em_len; u64 em_start; - struct extent_map *em; unsigned long pflags; int memstall = 0; int ret; @@ -568,16 +566,18 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) else gfp = GFP_NOFS; - /* we need the actual starting offset of this extent in the file */ - read_lock(&em_tree->lock); - em = btrfs_lookup_extent_mapping(em_tree, file_offset, fs_info->sectorsize); - read_unlock(&em_tree->lock); - if (!em) { + /* + * We need the actual starting offset of this extent in the file. The + * caller holds a reference on the extent map the bio was built from; + * it may no longer be in the inode's extent map tree. + */ + if (WARN_ON_ONCE(!em)) { ret = -EIO; goto out; } ASSERT(btrfs_extent_map_is_compressed(em)); + ASSERT(em->start <= file_offset && file_offset < btrfs_extent_map_end(em)); compressed_len = em->disk_num_bytes; cb = alloc_compressed_bio(inode, file_offset, REQ_OP_READ, @@ -592,8 +592,6 @@ void btrfs_submit_compressed_read(struct btrfs_bio *bbio) cb->orig_bbio = bbio; cb->bbio.csum_search_commit_root = bbio->csum_search_commit_root; - btrfs_free_extent_map(em); - for (int i = 0; i * min_folio_size < compressed_len; i++) { struct folio *folio; u32 cur_len = min(compressed_len - i * min_folio_size, min_folio_size); diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index 1022dc53ec51ea..6ccbcfaf3280e9 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -20,6 +20,7 @@ struct address_space; struct inode; struct btrfs_inode; struct btrfs_ordered_extent; +struct extent_map; /* * We want to make sure that amount of RAM required to uncompress an extent is @@ -94,7 +95,7 @@ struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, u64 start, u64 len); void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, struct compressed_bio *cb); -void btrfs_submit_compressed_read(struct btrfs_bio *bbio); +void btrfs_submit_compressed_read(struct btrfs_bio *bbio, struct extent_map *em); int btrfs_compress_str2level(unsigned int type, const char *str, int *level_ret); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 741fd1d4ab2ead..eddc6a834f214b 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -150,6 +150,16 @@ struct btrfs_bio_ctrl { * inside the same inode. */ u64 last_em_start; + + /* + * For a compressed read, the extent map of the compressed extent that + * @bbio reads, referenced until the bio is submitted. + * + * btrfs_submit_compressed_read() needs it, but must not look it up in + * the inode's extent map tree again: by then the extent map shrinker + * may have removed it from the tree. + */ + struct extent_map *compressed_em; }; /* @@ -176,8 +186,10 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) { struct btrfs_bio *bbio = bio_ctrl->bbio; - if (!bbio) + if (!bbio) { + ASSERT(!bio_ctrl->compressed_em); return; + } /* Caller should ensure the bio has at least some range added */ ASSERT(bbio->bio.bi_iter.bi_size); @@ -186,12 +198,14 @@ static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ && bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) - btrfs_submit_compressed_read(bbio); + btrfs_submit_compressed_read(bbio, bio_ctrl->compressed_em); else btrfs_submit_bbio(bbio, 0); /* The bbio is owned by the end_io handler now */ bio_ctrl->bbio = NULL; + btrfs_free_extent_map(bio_ctrl->compressed_em); + bio_ctrl->compressed_em = NULL; /* * We used the generation to decide whether to lookup csums in the * commit_root or not when we called bio_set_csum_search_commit_root() @@ -963,6 +977,26 @@ static struct extent_map *get_extent_map(struct btrfs_inode *inode, return em; } +/* + * A block of the compressed extent mapped by @em has just been added to + * @bio_ctrl->bbio, which only ever reads blocks of one compressed extent. + * Take over the caller's reference on @em: bio_ctrl keeps one reference on + * the extent map of its bio until submit_one_bio() passes it on. + */ +static void bio_ctrl_hold_compressed_em(struct btrfs_bio_ctrl *bio_ctrl, + struct extent_map *em) +{ + struct extent_map *old = bio_ctrl->compressed_em; + + /* + * Any reference held so far is for an earlier block of the same bio, + * on the same extent map or on a copy of it loaded again after the + * shrinker dropped it: one is enough. + */ + bio_ctrl->compressed_em = em; + btrfs_free_extent_map(old); +} + static void btrfs_readahead_expand(struct readahead_control *ractl, const struct extent_map *em) { @@ -1121,17 +1155,17 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, bio_ctrl->last_em_start = em->start; em_gen = em->generation; - btrfs_free_extent_map(em); - em = NULL; /* we've found a hole, just zero and go on */ if (block_start == EXTENT_MAP_HOLE) { + btrfs_free_extent_map(em); folio_zero_range(folio, pg_offset, blocksize); end_folio_read(vi, folio, true, cur, blocksize); continue; } /* the get_extent function already copied into the folio */ if (block_start == EXTENT_MAP_INLINE) { + btrfs_free_extent_map(em); end_folio_read(vi, folio, true, cur, blocksize); continue; } @@ -1145,6 +1179,11 @@ static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, submit_one_bio(bio_ctrl); submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize, pg_offset, em_gen); + + if (compress_type != BTRFS_COMPRESS_NONE) + bio_ctrl_hold_compressed_em(bio_ctrl, em); + else + btrfs_free_extent_map(em); } return 0; } From 32932612fe808afee55ecdb4ecf3fdbff8162e4e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:23:41 +1100 Subject: [PATCH 320/631] wifi: mt7932: acknowledge interrupts before servicing the rings mt_service() drains the RX rings and retires TX descriptors first and only then writes the causes it read back to the interrupt status register. A completion that the device posts after a ring has been drained but before that write sets a status bit that the write clears again. Its descriptor then waits, unserviced, until some unrelated interrupt arrives; for a command reply that can be the full command timeout. Acknowledge the causes, then service the rings, as other MediaTek PCIe drivers do. Read the status register back after the write so that the acknowledgment has reached the device, and the descriptors written before it are visible, before the rings are read. A completion that arrives while the rings are being serviced now leaves its status bit set, and the device interrupts again once the mask is restored. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 6de5245864ae53..b96f8d50f64d81 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -287,16 +287,27 @@ static void mt_service(struct mt7932 *m) u32 causes = mt_read(m, W + 0x200) & m->irq_mask; unsigned int i; - mt_data_clean(m); - for (i = 0; i < ARRAY_SIZE(m->rx); i++) - if (m->rx[i].count) - mt_rx_drain(m, &m->rx[i]); + /* + * Acknowledge before servicing the rings. A completion that lands + * while they are being drained then sets its status bit again and + * raises a new interrupt once the mask is restored, instead of being + * cleared by a late acknowledgment and left waiting for an unrelated + * interrupt. Read the status back so that the acknowledgment has + * reached the device, and every descriptor it covers is visible, + * before the rings are read. + */ if (causes & BIT(29)) { u32 sw = mt_read(m, W + 0x1f0); mt_write(m, W + 0x1f0, sw); } mt_write(m, W + 0x200, causes); + mt_read(m, W + 0x200); + + mt_data_clean(m); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_rx_drain(m, &m->rx[i]); if (READ_ONCE(m->running) && !poll_test) mt_write(m, W + 0x204, m->irq_mask); } From 28b2fc80b2d56a3ca845a414aa31b63d1ef3dee8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:31:42 +1100 Subject: [PATCH 321/631] media: apple: isp: Stop the coprocessor before unwinding a failed boot A firmware boot that fails after the first handshake unwinds stage by stage: it frees the channel table and the extra heap, and only then stops the coprocessor and powers its domains down. Stage 2 also frees the extra heap itself when the second handshake does not come. The firmware is running from the first handshake on, so it can still access the heap while its pages are released; the pages are unmapped from the DART first, so a late access faults instead of reaching freed memory, but it should not happen at all. isp_firmware_shutdown() already stops the coprocessor before freeing anything. Share that order with the failed boot: stop the coprocessor and power its domains down, then free the channel table, the extra heap and the surfaces the firmware requested. Stage 2 leaves the heap to the caller, and the heap pointer is cleared once it has been freed, so an unwind that runs before the heap is allocated has nothing to free. The runtime PM reference of the ISP and the references on its other power domains are dropped as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 57 +++++++++++++---------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index 7f6dc9d8cca2d7..cd04f8f2a77a69 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -456,7 +456,11 @@ void apple_isp_free_firmware_surface(struct apple_isp *isp) static void isp_firmware_shutdown_stage2(struct apple_isp *isp) { + if (!isp->extra_surf) + return; + isp_free_surface(isp, isp->extra_surf); + isp->extra_surf = NULL; } static void isp_write_bootargs(struct apple_isp *isp, void *virt, @@ -505,7 +509,6 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) size_t cmd_size = ISP_CMD_AREA_SIZE(isp_num_capmeta(isp)); dma_addr_t args_iova, cmd_iova; void *args_virt, *cmd_virt; - int err; u32 num_ipc_chans = isp_gpio_read32(isp, ISP_GPIO_0); u32 args_offset = isp_gpio_read32(isp, ISP_GPIO_1); @@ -578,19 +581,18 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) /* Wait for ISP_GPIO_7 to 0xf7fbdff9 -> 0x8042006 */ isp_gpio_write32(isp, ISP_GPIO_7, 0xf7fbdff9); + /* + * The firmware may already use the extra heap: it is freed only once + * the coprocessor has stopped, see isp_firmware_boot(). + */ if (isp_gpio_wait(isp, ISP_GPIO_7, 0x8042006)) { isp_err(isp, "never received second magic number from firmware\n"); - err = -ENODEV; - goto free_extra; + return -ENODEV; } isp_dbg(isp, "got second magic number from firmware\n"); return 0; - -free_extra: - isp_free_surface(isp, isp->extra_surf); - return err; } static inline struct isp_channel *isp_get_chan_index(struct apple_isp *isp, @@ -893,32 +895,49 @@ static void isp_collect_gc_surface(struct apple_isp *isp) } } +/* + * Stop the coprocessor and power its domains down, then release the memory + * the firmware may still be using: its channel table, extra heap and the + * surfaces it requested. + */ +static void isp_firmware_stop(struct apple_isp *isp) +{ + isp_firmware_shutdown_stage1(isp); + + isp_firmware_shutdown_stage3(isp); + isp_firmware_shutdown_stage2(isp); + isp_collect_gc_surface(isp); +} + static int isp_firmware_boot(struct apple_isp *isp) { int err; err = isp_firmware_boot_stage1(isp); if (err < 0) { + /* Stage 1 stops the coprocessor and powers it down itself. */ isp_err(isp, "failed firmware boot stage 1: %d\n", err); - goto garbage_collect; + isp_collect_gc_surface(isp); + return err; } + /* From here on the firmware runs, and may use what it was given. */ err = isp_firmware_boot_stage2(isp); if (err < 0) { isp_err(isp, "failed firmware boot stage 2: %d\n", err); - goto shutdown_stage1; + goto stop; } err = isp_firmware_boot_stage3(isp); if (err < 0) { isp_err(isp, "failed firmware boot stage 3: %d\n", err); - goto shutdown_stage2; + goto stop; } err = isp_enable_irq(isp); if (err < 0) { isp_err(isp, "failed to enable interrupts: %d\n", err); - goto shutdown_stage3; + goto stop; } err = isp_start_command_processor(isp); @@ -933,14 +952,8 @@ static int isp_firmware_boot(struct apple_isp *isp) disable_irqs: isp_disable_irq(isp); -shutdown_stage3: - isp_firmware_shutdown_stage3(isp); -shutdown_stage2: - isp_firmware_shutdown_stage2(isp); -shutdown_stage1: - isp_firmware_shutdown_stage1(isp); -garbage_collect: - isp_collect_gc_surface(isp); +stop: + isp_firmware_stop(isp); return err; } @@ -955,11 +968,7 @@ static void isp_firmware_shutdown(struct apple_isp *isp) if (isp_stop_command_processor(isp)) dev_warn(isp->dev, "firmware did not suspend, stopping it\n"); isp_disable_irq(isp); - isp_firmware_shutdown_stage1(isp); - - isp_firmware_shutdown_stage3(isp); - isp_firmware_shutdown_stage2(isp); - isp_collect_gc_surface(isp); + isp_firmware_stop(isp); } /* From 69b000e1eaffa35e942795c8c8df71c60c96959e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:32:41 +1100 Subject: [PATCH 322/631] media: apple: isp: Report a power domain that fails to go down as such apple_isp_power_down_domains() reports a failed runtime PM put of an ISP power domain as a failure to power it up, which points at the wrong transition when a domain stays on after the firmware has stopped. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index cd04f8f2a77a69..d65c035924a8dd 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -96,7 +96,7 @@ static void apple_isp_power_down_domains(struct apple_isp *isp) ret = pm_runtime_put_sync(isp->pd_dev[i]); if (ret < 0) dev_err(isp->dev, - "Failed to power up power domain %d: %d\n", i, ret); + "Failed to power down power domain %d: %d\n", i, ret); } isp->pds_active = false; From 9b35f9ad45af1b0e48821f4793fce0612efa561e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 14:03:58 +1100 Subject: [PATCH 323/631] media: apple: isp: Keep the driver state until the last file is closed The driver state, which embeds the video device, its queue and lock, the control handler and the media device, is allocated with devm_kzalloc() and freed as soon as the driver is unbound. A process that still has the video device open at unbind keeps a reference to the video device, and when it closes the file, vb2_fop_release() takes the queue lock and v4l2_fh_release() walks the file handle lists, all in freed memory. A control event subscription also reaches the freed control handler. Tie the state to the v4l2_device reference count as V4L2 drivers do: allocate it with kzalloc(), free it, the control handler and the media device from the v4l2_device release callback, and drop the driver's own reference at the end of remove. The video device holds the other one while it is registered or open, so the state lives until the last file is closed. Before the video device is registered nothing else can reach the state, so a failed probe still frees it directly. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 17 ++++++++++---- drivers/media/platform/apple/isp/isp-v4l2.c | 25 +++++++++++++++++++-- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 9f5f8f58b3748e..71ba9c79394705 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -380,7 +380,11 @@ static int apple_isp_probe(struct platform_device *pdev) if (err) return err; - isp = devm_kzalloc(dev, sizeof(*isp), GFP_KERNEL); + /* + * Not devres: an open video device file keeps this until it is + * closed, also after unbind. See apple_isp_release(). + */ + isp = kzalloc_obj(*isp); if (!isp) return -ENOMEM; @@ -399,7 +403,7 @@ static int apple_isp_probe(struct platform_device *pdev) if (err) { dev_err(dev, "failed to get 'apple,platform-id' property: %d\n", err); - return err; + goto free_isp; } err = of_property_read_u32(dev->of_node, "apple,temporal-filter", @@ -410,13 +414,13 @@ static int apple_isp_probe(struct platform_device *pdev) err = apple_isp_init_presets(isp); if (err) { dev_err(dev, "failed to initialize presets\n"); - return err; + goto free_isp; } err = apple_isp_attach_genpd(isp); if (err) { dev_err(dev, "failed to attatch power domains\n"); - return err; + goto free_isp; } isp->coproc = devm_platform_ioremap_resource_byname(pdev, "coproc"); @@ -529,6 +533,9 @@ static int apple_isp_probe(struct platform_device *pdev) destroy_workqueue(isp->wq); detach_genpd: apple_isp_detach_genpd(isp); +free_isp: + /* Nothing else can reach it before the video device is registered. */ + kfree(isp); return err; } @@ -545,6 +552,8 @@ static void apple_isp_remove(struct platform_device *pdev) apple_isp_free_iommu(isp); destroy_workqueue(isp->wq); apple_isp_detach_genpd(isp); + /* Freed now, or when the last open file is closed. */ + v4l2_device_put(&isp->v4l2_dev); } static const struct apple_isp_hw apple_isp_hw_t8103 = { diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 299177b9075cff..f3056e78a0a171 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -969,6 +969,22 @@ static const struct media_device_ops isp_media_device_ops = { .link_notify = v4l2_pipeline_link_notify, }; +/* + * The video device's file operations can run until its last open file is + * closed, after the driver has been unbound, so the state they reach is freed + * only with the last reference to the v4l2_device: the one remove drops, or + * the one the video device holds while it is open. + */ +static void apple_isp_release(struct v4l2_device *v4l2_dev) +{ + struct apple_isp *isp = container_of(v4l2_dev, struct apple_isp, + v4l2_dev); + + v4l2_ctrl_handler_free(&isp->ctrl_handler); + media_device_cleanup(&isp->mdev); + kfree(isp); +} + static void isp_free_meta_surfaces(struct apple_isp *isp) { for (int i = 0; i < ARRAY_SIZE(isp->capmeta_surfs); i++) { @@ -1093,6 +1109,9 @@ int apple_isp_setup_video(struct apple_isp *isp) goto v4l2_unregister; } + /* From here on, the last v4l2_device reference frees the state. */ + isp->v4l2_dev.release = apple_isp_release; + return 0; v4l2_unregister: @@ -1108,13 +1127,15 @@ int apple_isp_setup_video(struct apple_isp *isp) return err; } +/* + * Unregister the device nodes. An open file keeps the video device, and + * with it the driver state, until it is closed: see apple_isp_release(). + */ void apple_isp_remove_video(struct apple_isp *isp) { vb2_video_unregister_device(&isp->vdev); v4l2_device_unregister(&isp->v4l2_dev); - v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_unregister(&isp->mdev); - media_device_cleanup(&isp->mdev); } /* After apple_isp_remove_video() and with the firmware stopped */ From 8d6a6fc18b6d02bd1ebe93d0eb3824482be6f68d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 13:31:58 +1100 Subject: [PATCH 324/631] wifi: mt7932: ring the TX doorbell once per burst mt_net_xmit() writes the TX0 CPU index after every packet, even when the stack is handing over a burst. Each write is an uncached MMIO store across the PCIe link, and each makes the device fetch descriptors for a single packet. Use netdev_xmit_more() to publish a burst with one doorbell write: ring it when the stack has nothing more queued, when the queue has just stopped, and at least every 64 packets so that the device starts on a long burst early. Descriptors stay published in order behind the doorbell, and the TX0 accounting is unchanged. The queue can also stop from outside ndo_start_xmit(), when the access point goes absent or the link goes down, which ends a burst early. Ring the doorbell in those paths too, before the connection teardown waits for TX0 to drain, and on every interrupt, so that no published descriptor waits behind a doorbell that is never written. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/data.c | 41 +++++++++++++++++-- drivers/net/wireless/mediatek/mt7932/link.c | 6 ++- drivers/net/wireless/mediatek/mt7932/mt7932.h | 3 ++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index e11d4567595944..9e6a51f8e39878 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -35,6 +35,25 @@ void mt_data_clean_locked(struct mt7932 *m) netif_wake_queue(m->netdev); } +/* + * The stack marks a packet that has another one right behind it with + * netdev_xmit_more(), so the TX0 doorbell is rung once per burst instead of + * once per packet, and at least every MT7932_TX_BATCH packets so that the + * device starts on a long burst early. + */ +#define MT7932_TX_BATCH 64 + +/* data_lock held: hand every published TX0 descriptor to the device. */ +void mt_data_kick_locked(struct mt7932 *m) +{ + struct mt7932_ring *q = &m->tx[0]; + + if (!m->data_unkicked) + return; + m->data_unkicked = 0; + mt_write(m, q->reg + 8, q->head); +} + void mt_data_clean(struct mt7932 *m) { unsigned long flags; @@ -42,6 +61,10 @@ void mt_data_clean(struct mt7932 *m) if (!READ_ONCE(m->data_ready)) return; spin_lock_irqsave(&m->data_lock, flags); + /* The queue can be stopped from outside ndo_start_xmit() in the middle + * of a burst; no descriptor waits longer than the next interrupt. + */ + mt_data_kick_locked(m); mt_data_clean_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); } @@ -59,10 +82,12 @@ void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event) spin_lock_irqsave(&m->data_lock, flags); m->bss_absent = !!body[1]; m->bss_quota = body[2]; - if (!mt_bss_tx_allowed(m)) + if (!mt_bss_tx_allowed(m)) { netif_stop_queue(m->netdev); - else if (READ_ONCE(m->data_ready)) + mt_data_kick_locked(m); + } else if (READ_ONCE(m->data_ready)) { mt_data_clean_locked(m); + } spin_unlock_irqrestore(&m->data_lock, flags); dev_dbg_ratelimited(&m->pdev->dev, "BSS_PRESENCE: absent=%u quota=%u\n", !!body[1], body[2]); @@ -122,6 +147,7 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) slot = q->head; if (!mt_bss_tx_allowed(m) || m->data_live[slot] || q->queued >= q->count - 1) { netif_stop_queue(netdev); + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); return NETDEV_TX_BUSY; } @@ -147,18 +173,27 @@ netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev) q->head = (slot + 1) % q->count; q->queued++; dma_wmb(); - mt_write(m, q->reg + 8, q->head); + m->data_unkicked++; mt_bss_tx_account(m); m->station_tx_packets++; m->station_tx_bytes += skb->len; if (!mt_bss_tx_allowed(m) || m->data_live[q->head] || q->queued >= q->count - 1) netif_stop_queue(netdev); + if (!netdev_xmit_more() || netif_queue_stopped(netdev) || + m->data_unkicked >= MT7932_TX_BATCH) + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); netdev->stats.tx_packets++; netdev->stats.tx_bytes += skb->len; dev_kfree_skb_any(skb); return NETDEV_TX_OK; drop: + /* The burst ends here: publish what earlier packets queued. */ + if (!netdev_xmit_more() && READ_ONCE(m->data_ready)) { + spin_lock_irqsave(&m->data_lock, flags); + mt_data_kick_locked(m); + spin_unlock_irqrestore(&m->data_lock, flags); + } netdev->stats.tx_dropped++; dev_kfree_skb_any(skb); return NETDEV_TX_OK; diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index da471022d7da3e..cfe71b2305598b 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -283,8 +283,12 @@ static int mt_retire_connection(struct mt7932 *m) return -ESHUTDOWN; netif_carrier_off(m->netdev); netif_tx_disable(m->netdev); - /* ndo_start_xmit rechecks the closed admission gate under data_lock. */ + /* ndo_start_xmit rechecks the closed admission gate under data_lock. + * The queue may have stopped in the middle of a burst; hand the device + * what it already holds so that the drain below can complete. + */ spin_lock_irqsave(&m->data_lock, flags); + mt_data_kick_locked(m); spin_unlock_irqrestore(&m->data_lock, flags); flush_work(&m->cal_work); mutex_lock(&m->command_mutex); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index e2b430d2db9356..1568592b8a34b1 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -176,6 +176,8 @@ struct mt7932 { bool data_live[512], data_token[512], data_retired[512]; struct mt7932_duplicate duplicates[17]; u64 tx_released, rx_ethernet; + /* TX0 descriptors published since the last doorbell; data_lock. */ + unsigned int data_unkicked; bool data_ready; bool bss_absent; u8 bss_quota; @@ -208,6 +210,7 @@ int mt_calibration_gate(struct mt7932 *m); int mt_cal_drain_requests(struct mt7932 *m, const struct firmware *oca); void mt_cal_input_error(struct mt7932 *m, int error, unsigned int tag, const char *stage); void mt_data_clean_locked(struct mt7932 *m); +void mt_data_kick_locked(struct mt7932 *m); void mt_data_clean(struct mt7932 *m); void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event); void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length); From f0fe3f9ac72d5d89ab2f0ec80972c5823e259c99 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:27:16 +1100 Subject: [PATCH 325/631] wifi: mt7932: ring the TX doorbell when a regulatory change stops the queue A regulatory change stops the data queue from the cfg80211 notifier, outside ndo_start_xmit() and outside data_lock. If that lands in the middle of a burst, the TX0 descriptors the burst already published stay behind the doorbell until the next interrupt rings it. Ring the doorbell there as well, under data_lock nested inside response_lock in the same order as the receive path, so that every path that stops the queue from outside ndo_start_xmit() hands the device what it already holds. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/startup.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 883eb03e9204cf..9c908ad551f7cb 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -504,6 +504,14 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request WRITE_ONCE(m->reg_pending, true); if (m->netdev) { netif_stop_queue(m->netdev); + /* The stop can land in the middle of a burst: hand the device the + * TX0 descriptors already published behind the doorbell. + */ + if (READ_ONCE(m->data_ready)) { + spin_lock(&m->data_lock); + mt_data_kick_locked(m); + spin_unlock(&m->data_lock); + } if (m->connecting) { m->connect_error = -ECANCELED; complete(&m->assoc_start); From 2a320dd8e0116a68967b036b3c15a9440399e88e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:03:09 +1100 Subject: [PATCH 326/631] soc: apple: sep: let a start wait for the last capture to stop The enrol and verify work items report their result and wake the poller before they return and clear the session's busy marker. After a cancel or a close they stop only once the capture loop notices. An ENROL_START or VERIFY_START in that window, or an open right after a close, is refused with EBUSY. Userspace cannot tell that from a device in use by someone else: libfprint reports a protocol error, and fprintd turns it into verify-disconnected. fprintd starts a match again as soon as a capture is reported unusable, so it lands in that window. Hold such an open or start until the work item has stopped, for as long as a capture may take to drain before sleep. Only an open with no other opener, or a start whose last operation has ended, waits; a start while an operation is still running is refused at once, as before. If the work item has not stopped in time, the caller is refused as before, and the driver logs it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/bio.rs | 7 ++++++ drivers/soc/apple/sbio.rs | 47 +++++++++++++++++++++++++++++++++++++-- drivers/soc/apple/sep.rs | 19 +++++++++++----- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/drivers/soc/apple/bio.rs b/drivers/soc/apple/bio.rs index 09935233924553..6b4e1edbe29664 100644 --- a/drivers/soc/apple/bio.rs +++ b/drivers/soc/apple/bio.rs @@ -849,6 +849,13 @@ pub(crate) fn worker_stopped(session: &mut Session, epoch: u64) { } } +/// Whether an open (`opening`) or a capture start by the current opener would +/// be refused only because the last enrol or verify work item has not stopped +/// yet, although its operation has ended or its opener has gone. +pub(crate) fn waiting_on_worker(session: &Session, opening: bool) -> bool { + session.worker_epoch.is_some() && session.op.may_start() && session.open != opening +} + fn verify_poll(ctx: &mut Context<'_>, user: UserPtr) -> Result { let mut out = VerifyPoll::default(); // Spend the token only after the copy to user succeeds; a fault must not lose a match. diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 7e49ff2013bdf6..430bb4f136c377 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2846,6 +2846,7 @@ impl SepData { return Err(ENODEV); } let mut session = self.bio_session.lock(); + self.wait_for_worker_exit(&mut session, true)?; bio::open(&mut session)?; drop(session); if let Err(e) = self.prepare_bio_open() { @@ -2937,6 +2938,45 @@ impl SepData { self.captures_running.fetch_sub(1, Relaxed); } + /// The enrol or verify work item `epoch` has stopped. + pub(crate) fn capture_worker_stopped(&self, epoch: u64) { + bio::worker_stopped(&mut self.bio_session.lock(), epoch); + self.worker_gone.notify_all(); + } + + /// Holds an open or a capture start while the last enrol or verify work + /// item is still on its way out. It reports its result before it stops, + /// and after a cancel or a close it stops only once the capture notices, + /// so an open or a start straight after either would be refused with + /// EBUSY, which the caller cannot tell from a device in use. If it has + /// not stopped in time, the caller refuses as before. + fn wait_for_worker_exit( + &self, + session: &mut MutexGuard<'_, bio::Session>, + opening: bool, + ) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(WORKER_EXIT_WAIT_MS); + while bio::waiting_on_worker(session, opening) { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + if remaining == 0 { + dev_warn!( + self.dev, + "Touch ID: the last capture had not stopped after {} ms\n", + WORKER_EXIT_WAIT_MS + ); + return Ok(()); + } + match self.worker_gone.wait_interruptible_timeout(session, remaining) { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Timeout => remaining = 0, + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + } + } + Ok(()) + } + /// The system is about to suspend. End any capture in progress, hold new /// ones, and wait for the enrol or verify work to stop touching the sensor /// before devices go down. @@ -3001,7 +3041,7 @@ impl SepData { .is_err() { this.finish_enrolment(Err(ENROL_STATUS_SENSOR)); - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -3018,7 +3058,7 @@ impl SepData { bio::VerifyOutcome::Failed(ENROL_STATUS_SENSOR), [0u8; bio::TOKEN_LEN], ); - bio::worker_stopped(&mut this.bio_session.lock(), epoch); + this.capture_worker_stopped(epoch); } } @@ -3084,6 +3124,9 @@ impl SepData { let sensor_present = sensor::is_bound(); let handled = { let mut session = self.bio_session.lock(); + if bio::starts_capture(cmd) { + self.wait_for_worker_exit(&mut session, false)?; + } let index = self.bio_index.lock(); let mut ctx = bio::Context { session: &mut session, diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 89f24ed62e60bc..5cf939a6721e6b 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -64,7 +64,8 @@ use kernel::{ Arc, CondVar, CondVarTimeoutResult, - Mutex, // + Mutex, + MutexGuard, // }, time, types::ForeignOwnable, @@ -114,6 +115,11 @@ const SBIO_TIMEOUT_MS: time::Msecs = 5000; const SLEEP_DRAIN_MS: u32 = 6000; static_assert!(SLEEP_DRAIN_MS as u64 > SBIO_TIMEOUT_MS as u64); +// How long an open or a capture start waits for the last enrol or verify work +// item to stop once its operation has ended. It is stopped in the same way, +// so it gets the same bound as a capture draining before sleep. +const WORKER_EXIT_WAIT_MS: time::Msecs = SLEEP_DRAIN_MS; + const SKS_ALLOC: usize = 0x8000; const SKS_SECRET_LEN: usize = 32; @@ -711,6 +717,9 @@ struct SepData { // deletion durable failed; the next deletion retries the save. Changed // only under `catacomb_writer`. deletion_unsaved: Atomic, + // Signalled when an enrol or verify work item has stopped. + #[pin] + worker_gone: CondVar, #[pin] rx_work: Work, @@ -888,6 +897,7 @@ impl SepData { captures_running: Atomic::new(0), catacomb_writer <- new_mutex!(()), deletion_unsaved: Atomic::new(false), + worker_gone <- new_condvar!("SepData::worker_gone"), rx_work <- new_work!("SepData::rx_work"), enrol_work <- new_work!("SepData::enrol_work"), verify_work <- new_work!("SepData::verify_work"), @@ -2067,6 +2077,7 @@ impl SepData { fn remove(&self) { self.shutting_down.store(true, Relaxed); bio::release(&mut self.bio_session.lock()); + self.worker_gone.notify_all(); self.rng_shutdown.store(true, Relaxed); self.control_wq.notify_all(); self.sbio_wq.notify_all(); @@ -2265,8 +2276,7 @@ impl WorkItem for SepData { if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_enrolment(); } - this.run_enrolment(); - this.capture_end(); + this.capture_worker_stopped(epoch); } } @@ -2278,8 +2288,7 @@ impl WorkItem for SepData { if !this.shutting_down.load(Relaxed) && this.capture_begin() { this.run_verify(); } - this.run_verify(); - this.capture_end(); + this.capture_worker_stopped(epoch); } } From 9299b5606f1570e299cfa13ab5f226aac2aa43dc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:42:14 +1100 Subject: [PATCH 327/631] pmdomain: apple: Undo a power state transition that times out apple_pmgr_ps_set() writes the new target state and polls the actual state for 100 us. When the poll times out it logs and returns the error, but leaves the new target requested, and a power-on also sets AUTO_ENABLE. genpd then keeps the domain in its previous state: off after a failed ->power_on(), on after a failed ->power_off(). From then on the register and genpd disagree, and genpd does not write the domain again until its next transition. On T8140 (J700), rebinding the ISP after an unbind fails because the resident firmware does not boot a second time, and the CPU core domain stops short of ACTIVE: [ 760.990199] apple-pmgr-pwrstate 300700000.power-management:power-controller@4008: PS isp_cpu_core0: Failed to reach power state 0xf (requested 0xf00000f, now: 0xf000b4f) genpd and the ISP driver unwind as if the domain were off, but its register keeps requesting ACTIVE with auto-PM enabled, so the core sits clock-gated instead of power-gated until the next boot. A failed power-off is the opposite case: genpd still treats the domain as on and will not power it on again for its next consumer, while the register asks for it to be power-gated. Follow a failed transition with the opposite one: power the domain back off after a failed power-on, and back on after a failed power-off. Both are the normal sequences, so a power-off still sets the force-disable and force-reset bits first, and the domain is down before genpd powers off its parents. genpd ignores power-on errors in the noirq and syscore phases of system sleep (genpd_sync_power_on()) and records the domain as on there, so powering it back off would leave a domain off that genpd and its consumers believe is on. Slow domains do time out there: the gfx domain misses the poll window on resume from s2idle on M1 Pro (t6000), and did on T8140 in genpd's noirq resume before it became always-on, then reaches ACTIVE, and the GPU keeps working. The PM core runs those phases between the suspend_late and resume_early callbacks of every device. Add these callbacks to mark the window and keep requesting the domain on inside it, as before. The probe-time power-on of an always-on domain also keeps the current behaviour, as genpd never powers those domains. Transitions that succeed are unchanged. Fixes: 6df9d38f9146 ("soc: apple: Add driver for Apple PMGR power state controls") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmgr-pwrstate.c | 68 ++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 4d5c584d1152b7..4a12b16c95c64c 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -49,6 +49,7 @@ struct apple_pmgr_ps { bool force_reset; bool externally_clocked; bool retained_25g83; + bool system_sleep; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -147,14 +148,62 @@ static bool apple_pmgr_ps_is_active(struct apple_pmgr_ps *ps) reg & APPLE_PMGR_AUTO_ENABLE)); } +/* + * genpd keeps a domain off when ->power_on() fails and on when ->power_off() + * fails. Follow a failed transition with the opposite one, so the domain is + * not left requesting a state genpd does not know it is in. + * + * The exception is a failed power-on in the noirq and syscore phases of a + * system sleep transition: genpd ignores the error there and records the + * domain as on. The PM core runs those phases after the suspend_late callback + * of every device and before any resume_early callback, so keep requesting + * the domain on between this device's two callbacks. + */ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) { - return apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); + int ret; + + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + if (ret && !ps->system_sleep) + apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + + return ret; } static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) { - return apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + int ret; + + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); + if (ret) + apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + + return ret; +} + +static void apple_pmgr_ps_set_system_sleep(struct device *dev, bool sleep) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + unsigned long flags; + + spin_lock_irqsave(&ps->genpd.slock, flags); + ps->system_sleep = sleep; + spin_unlock_irqrestore(&ps->genpd.slock, flags); +} + +static int apple_pmgr_ps_suspend_late(struct device *dev) +{ + apple_pmgr_ps_set_system_sleep(dev, true); + + return 0; +} + +static int apple_pmgr_ps_resume_early(struct device *dev) +{ + apple_pmgr_ps_set_system_sleep(dev, false); + + return 0; } static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) @@ -285,6 +334,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->dev = dev; ps->regmap = regmap; + platform_set_drvdata(pdev, ps); ret = of_property_read_string(node, "label", &name); if (ret < 0) { @@ -328,8 +378,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { dev_warn(dev, "always-on domain %s is not on at boot\n", name); - /* Turn it on so pm_genpd_init does not fail */ - active = apple_pmgr_ps_power_on(&ps->genpd) == 0; + /* + * Turn it on so pm_genpd_init does not fail. genpd never + * powers an always-on domain, so keep requesting it on + * even if it does not get there in time. + */ + active = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, + true) == 0; } } else if (active) { ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; @@ -404,11 +459,16 @@ static const struct of_device_id apple_pmgr_ps_of_match[] = { MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); +static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) +}; + static struct platform_driver apple_pmgr_ps_driver = { .probe = apple_pmgr_ps_probe, .driver = { .name = "apple-pmgr-pwrstate", .of_match_table = apple_pmgr_ps_of_match, + .pm = pm_sleep_ptr(&apple_pmgr_ps_pm_ops), }, }; From d322f674ad3f18468c2c979399b9470c6b039782 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:36:15 +1100 Subject: [PATCH 328/631] tools: aurora-sep: give the SEP loader time for a slow enclave load-driver tries to seal and unseal a trusted key 300 times, 0.1 s apart, then waits as long again for /dev/sep-bio. On J700 the enclave's key store arrives around 43 s after boot, and the driver itself waits up to 90 s for it, so the loader can give up before Touch ID comes up. The service then fails, and fprintd, which requires it, fails with it. Measure the waits in seconds of uptime rather than in attempts, wait up to 120 s for the trusted key and 60 s for the device, and let AURORA_SEP_READY_TIMEOUT and AURORA_SEP_DEVICE_TIMEOUT override them. Log why the loader gives up and how long each step took, so a slow boot can be told from a broken one. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 11 +++++++ tools/aurora-sep/load-driver | 59 ++++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 03c03f5596b23c..23981ed1a20c36 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -34,6 +34,17 @@ systemctl daemon-reload systemctl enable aurora-sep.service ``` +The loader waits up to 120 s for the enclave to seal and unseal a trusted key, +then up to 60 s for `/dev/sep-bio`, and logs how long each took. On J700 the +enclave's key store arrives around 43 s after boot. To change a wait, set +`AURORA_SEP_READY_TIMEOUT` or `AURORA_SEP_DEVICE_TIMEOUT` in whole seconds, +for example with `systemctl edit aurora-sep.service`: + +```ini +[Service] +Environment=AURORA_SEP_READY_TIMEOUT=180 +``` + `xart_writes=0` remains the default. `load-driver` explicitly opts into writes and must only be enabled on a machine whose APFS `.gl` extent passes the driver's narrow ownership checks: one physical extent, no snapshots or revert diff --git a/tools/aurora-sep/load-driver b/tools/aurora-sep/load-driver index a70db75e6b25b1..fbf43e5093f005 100755 --- a/tools/aurora-sep/load-driver +++ b/tools/aurora-sep/load-driver @@ -4,6 +4,34 @@ set -eu keybag=/var/lib/aurora-sep-keybag.bin refkey=/var/lib/aurora-sep-refkey.bin +# Seconds to wait for the enclave to seal and unseal a trusted key, then for +# /dev/sep-bio. On J700 the key store arrives around 43 s after boot, and the +# driver waits up to 90 s for it before it brings Touch ID up, so the first +# wait has to outlast that. Set either in the environment, for example with a +# drop-in for aurora-sep.service. +ready_timeout=${AURORA_SEP_READY_TIMEOUT:-120} +device_timeout=${AURORA_SEP_DEVICE_TIMEOUT:-60} + +log() { + echo "aurora-sep-load: $*" >&2 +} + +for budget in "$ready_timeout" "$device_timeout"; do + case "$budget" in + '' | *[!0-9]*) + log "timeouts must be whole seconds, not '$budget'" + exit 2 + ;; + esac +done + +# Seconds since boot. Unlike the wall clock it cannot jump when the time is +# first synchronised during boot. +uptime_s() { + read -r up _ /dev/null || true) if [ -n "$key_id" ]; then if keyctl pipe "$key_id" >"$work/sealed"; then @@ -37,17 +66,27 @@ while [ "$attempts" -gt 0 ]; do keyctl revoke "$key_id" >/dev/null 2>&1 || true fi [ "$verified" = true ] && break + [ "$(uptime_s)" -lt "$deadline" ] || break sleep 0.1 - attempts=$((attempts - 1)) done -[ "$verified" = true ] || exit 1 -[ -s "$keybag" ] || exit 1 -[ -s "$refkey" ] || exit 1 +if [ "$verified" != true ]; then + log "the enclave did not seal and unseal a trusted key within ${ready_timeout} s" + exit 1 +fi +log "trusted keys work after $(($(uptime_s) - start)) s" + +if [ ! -s "$keybag" ] || [ ! -s "$refkey" ]; then + log "$keybag or $refkey is missing or empty" + exit 1 +fi -attempts=300 -while [ "$attempts" -gt 0 ] && [ ! -c /dev/sep-bio ]; do +deadline=$(($(uptime_s) + device_timeout)) +while [ ! -c /dev/sep-bio ] && [ "$(uptime_s)" -lt "$deadline" ]; do sleep 0.1 - attempts=$((attempts - 1)) done -[ -c /dev/sep-bio ] || exit 1 +if [ ! -c /dev/sep-bio ]; then + log "/dev/sep-bio did not appear within ${device_timeout} s" + exit 1 +fi +log "/dev/sep-bio is ready after $(($(uptime_s) - start)) s" From b134722fb7f0632b628d78216295760193cf70d5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 15:39:02 +1100 Subject: [PATCH 329/631] tools: aurora-sep: add Fedora and Arch Linux packaging A manual install leaves libfprint under /usr/local, where fprintd only finds it through a library path drop-in, and puts the loader and the units under /usr/local and /etc, where no package manager tracks them. Add packaging for the distributions the driver runs on: - packaging/fedora/build.sh rebuilds the release's own libfprint source package with the aurora patch added and an ".aurora1" release suffix, so it replaces the distribution's library in place. It also builds aurora-sep.spec. - packaging/arch has libfprint-aurora, which provides and conflicts with libfprint, and aurora-sep, for Arch Linux and Arch Linux ARM. - aurora-sep installs the loader, its service pointed at the packaged loader, and the fprintd policy as a drop-in under /usr/lib. On Fedora a preset enables the service at boot when the package is first installed; Arch leaves that to the user. The PKGBUILDs take their local sources through links into this tree, and a test checks their checksums against it. The README gives the build, install and rollback steps. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 52 ++++++++++ .../packaging/arch/aurora-sep/PKGBUILD | 44 +++++++++ .../packaging/arch/aurora-sep/README.md | 1 + .../packaging/arch/aurora-sep/XART-SAFETY.md | 1 + .../arch/aurora-sep/aurora-sep.service | 1 + .../arch/aurora-sep/fprintd-aurora.conf | 1 + .../packaging/arch/aurora-sep/load-driver | 1 + .../packaging/fedora/aurora-sep.spec | 74 ++++++++++++++ tools/aurora-sep/packaging/fedora/build.sh | 98 +++++++++++++++++++ tools/aurora-sep/tests/test_packaging.py | 72 ++++++++++++++ 10 files changed, 345 insertions(+) create mode 100644 tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/README.md create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/load-driver create mode 100644 tools/aurora-sep/packaging/fedora/aurora-sep.spec create mode 100755 tools/aurora-sep/packaging/fedora/build.sh create mode 100644 tools/aurora-sep/tests/test_packaging.py diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 23981ed1a20c36..dc08097f36b8f1 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -156,6 +156,58 @@ lock screen then selects `omarchy-lock-fingerprint` automatically and accepts Touch ID through fprintd. Hyprlock's separate fingerprint switch is not used by the Quickshell lock screen. +## Packages + +`packaging/` builds the same pieces as distribution packages: libfprint with +the aurora driver, which replaces the distribution's libfprint so fprintd needs +no library path, and `aurora-sep` with the loader, its service and the fprintd +policy. Installing `aurora-sep` lets fprintd load the driver with xART writes +on, as above. + +On Fedora, `packaging/fedora/build.sh` rebuilds the release's own libfprint +source package with the patch added (1.94.100 on Fedora 45) and builds +`aurora-sep`. It installs build dependencies, so run it in a throwaway +container from the top of this tree: + +```sh +mkdir rpms +podman run --rm -v "$PWD":/src:ro,z -v "$PWD/rpms":/out:z \ + registry.fedoraproject.org/fedora:45 \ + /src/tools/aurora-sep/packaging/fedora/build.sh /out +sudo dnf install rpms/libfprint-1.94.100-*.aurora1.*.rpm \ + rpms/aurora-sep-1-*.noarch.rpm +``` + +A later Fedora libfprint update replaces the rebuilt one; rebuild after it, or +hold it with `dnf versionlock add libfprint`. + +On Arch Linux and Arch Linux ARM, build `libfprint-aurora` (it replaces +`libfprint`) and then `aurora-sep`: + +```sh +cd tools/aurora-sep/packaging/arch/libfprint-aurora && makepkg -si +cd ../aurora-sep && makepkg -si +``` + +fprintd starts the loader the first time it runs. Loading the driver at boot +instead has the enclave ready before the login screen asks for a fingerprint. +The Fedora package enables that with a systemd preset when it is first +installed; on Arch, enable it yourself: + +```sh +sudo systemctl enable aurora-sep.service +``` + +The driver attaches to the enclave once per boot, so reboot rather than +restarting the service or reloading `apple_sep`. To move from the manual +install above, remove `/usr/local/sbin/aurora-sep-load`, +`/etc/systemd/system/aurora-sep.service`, +`/etc/systemd/system/fprintd.service.d/aurora.conf` and any drop-in that points +fprintd at another libfprint, then run `systemctl daemon-reload` and +`systemctl reenable aurora-sep.service`. To go back to the distribution's +libfprint, remove `aurora-sep` and run `dnf distro-sync libfprint` or +`pacman -S libfprint`. + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD new file mode 100644 index 00000000000000..d7a921a5501cf6 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# The loader, service and fprintd policy for the Apple SEP fingerprint driver. +# The local sources are links into tools/aurora-sep; after changing any of +# them, run updpkgsums here. libfprint-aurora provides the libfprint driver. + +pkgname=aurora-sep +pkgver=1 +pkgrel=1 +pkgdesc="Loader and fprintd policy for the Apple SEP (Touch ID) driver" +url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" +arch=(any) +license=(GPL-2.0-only) +depends=( + coreutils + fprintd + keyutils + kmod + sh +) +source=(load-driver + aurora-sep.service + fprintd-aurora.conf + README.md + XART-SAFETY.md) +sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' + 'd3441eeb06c930fcc13314d18012b1f5d71cc1a821ceaf44de13caafb2931d73' + 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' + 'd6e5c68bab7e0d71da48820da75d4ff7e67a06e7970961377ed28d30c2b66c08' + '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') + +package() { + install -Dm755 load-driver "$pkgdir/usr/lib/aurora-sep/aurora-sep-load" + + install -Dm644 aurora-sep.service "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + sed -i 's|^ExecStart=.*|ExecStart=/usr/lib/aurora-sep/aurora-sep-load|' \ + "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + grep -qx 'ExecStart=/usr/lib/aurora-sep/aurora-sep-load' \ + "$pkgdir/usr/lib/systemd/system/aurora-sep.service" + + install -Dm644 fprintd-aurora.conf \ + "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" + install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md +} diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/README.md b/tools/aurora-sep/packaging/arch/aurora-sep/README.md new file mode 120000 index 00000000000000..8a33348c7d811a --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/README.md @@ -0,0 +1 @@ +../../../README.md \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md new file mode 120000 index 00000000000000..45a7d77d7e92e2 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/XART-SAFETY.md @@ -0,0 +1 @@ +../../../XART-SAFETY.md \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service new file mode 120000 index 00000000000000..72bc9aad30d16b --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep.service @@ -0,0 +1 @@ +../../../aurora-sep.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf new file mode 120000 index 00000000000000..41efea20ba890e --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-aurora.conf @@ -0,0 +1 @@ +../../../fprintd-aurora.conf \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/load-driver b/tools/aurora-sep/packaging/arch/aurora-sep/load-driver new file mode 120000 index 00000000000000..3f9fc69e380426 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/load-driver @@ -0,0 +1 @@ +../../../load-driver \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec new file mode 100644 index 00000000000000..9d09b927274794 --- /dev/null +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Built from tools/aurora-sep: rpmbuild --define "_sourcedir " +# (build.sh does this). + +Name: aurora-sep +Version: 1 +Release: 1%{?dist} +Summary: Loader and fprintd policy for the Apple SEP fingerprint driver + +License: GPL-2.0-only +URL: https://github.com/aurora-silicon/linux +Source0: load-driver +Source1: aurora-sep.service +Source2: fprintd-aurora.conf +Source3: README.md +Source4: XART-SAFETY.md + +BuildArch: noarch +BuildRequires: systemd-rpm-macros +Requires: coreutils +Requires: fprintd +Requires: keyutils +Requires: kmod +%{?systemd_requires} + +%description +Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until +the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, +and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd +also pulls it in when it starts. libfprint needs the aurora driver, which +this package does not provide. Read XART-SAFETY.md before installing it on +a machine that also runs macOS. + +%prep +%setup -q -c -T +cp -p %{SOURCE3} %{SOURCE4} . + +%build + +%install +install -Dpm 0755 %{SOURCE0} %{buildroot}%{_libexecdir}/aurora-sep-load +install -Dpm 0644 %{SOURCE1} %{buildroot}%{_unitdir}/aurora-sep.service +sed -i 's|^ExecStart=.*|ExecStart=%{_libexecdir}/aurora-sep-load|' \ + %{buildroot}%{_unitdir}/aurora-sep.service +grep -qx 'ExecStart=%{_libexecdir}/aurora-sep-load' \ + %{buildroot}%{_unitdir}/aurora-sep.service +install -Dpm 0644 %{SOURCE2} \ + %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf +# Load the driver at boot, so the enclave is up before the login screen asks. +install -d %{buildroot}%{_presetdir} +echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset + +%post +%systemd_post aurora-sep.service + +%preun +%systemd_preun aurora-sep.service + +%postun +# Never restart it: the driver attaches to the enclave once per boot. +%systemd_postun aurora-sep.service + +%files +%doc README.md XART-SAFETY.md +%{_libexecdir}/aurora-sep-load +%{_unitdir}/aurora-sep.service +%dir %{_unitdir}/fprintd.service.d +%{_unitdir}/fprintd.service.d/aurora-sep.conf +%{_presetdir}/80-aurora-sep.preset + +%changelog +* Sun Oct 11 2026 Ryan Murray - 1-1 +- Package the loader, its service and the fprintd policy diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh new file mode 100755 index 00000000000000..32d9f30761d027 --- /dev/null +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Build the Fedora packages for Touch ID on the Apple SEP: +# +# - libfprint: Fedora's own source package, rebuilt with the aurora driver +# patch from ../../patches and nothing else changed. The release gains an +# ".aurora1" suffix, so it replaces the distribution's build in place. +# - aurora-sep: the driver loader, its service and the fprintd policy. +# +# It installs the build dependencies with dnf, so run it as root in a +# throwaway container of the Fedora release you are building for: +# +# podman run --rm -v :/src:ro,z -v :/out:z \ +# registry.fedoraproject.org/fedora:45 /src/ /out +# +# The binary and source packages land in the output directory. + +set -eu + +out=${1:?usage: build.sh } +here=$(cd "$(dirname "$0")" && pwd) +sep=$(cd "$here/../.." && pwd) +patch=$sep/patches/libfprint-1.94.100-apple-sep.patch +header=${APPLE_SEP_HEADER:-$sep/../../include/uapi/linux/apple-sep.h} +patched_version=1.94.100 + +for f in "$patch" "$header"; do + [ -f "$f" ] || { echo "build.sh: $f is missing" >&2; exit 1; } +done +mkdir -p "$out" + +top=$(mktemp -d) +trap 'rm -rf "$top"' EXIT +rpmtop=$top/rpmbuild + +dnf -y install rpm-build systemd-rpm-macros + +# Fedora's libfprint, exactly as the distribution builds it. +(cd "$top" && dnf -y download --source libfprint) +srpm=$(ls "$top"/libfprint-*.src.rpm) +rpm -i --define "_topdir $rpmtop" "$srpm" +spec=$rpmtop/SPECS/libfprint.spec + +version=$(rpmspec -q --srpm --qf '%{version}\n' "$spec") +if [ "$version" != "$patched_version" ]; then + echo "build.sh: this Fedora ships libfprint $version, the patch is for $patched_version" >&2 + exit 1 +fi +cp "$patch" "$header" "$rpmtop/SOURCES/" + +# Add the patch and the header, mark the release, and point the compiler at +# the header: is not in the distribution's kernel headers. +# %autosetup applies every PatchN. Stop if any anchor has moved. +awk -v patch="$(basename "$patch")" -v date="$(LC_ALL=C date -u '+%a %b %d %Y')" ' + /^Release:/ && !release { + sub(/[[:space:]]*$/, ".aurora1"); print; release = 1; next + } + /^Source0:/ && !source { + print + print "Source9000: apple-sep.h" + print "Patch9000: " patch + source = 1; next + } + /^%meson[[:space:]]/ && !build { + print "# The aurora driver includes from the Aurora kernel." + print "mkdir -p aurora-include/linux" + print "cp -p %{SOURCE9000} aurora-include/linux/" + print "export CFLAGS=\"%{build_cflags} -I$PWD/aurora-include\"" + print; build = 1; next + } + /^%changelog/ && !changelog { + print + print "* " date " Aurora Silicon - " "%{version}-%{release}" + print "- Add the Apple SEP (aurora) driver from aurora-silicon/linux" + print " tools/aurora-sep/patches/" patch + print "" + changelog = 1; next + } + { print } + END { + if (!(release && source && build && changelog)) { + print "build.sh: libfprint.spec changed shape; update the edit" > "/dev/stderr" + exit 1 + } + } +' "$spec" >"$spec.aurora" +mv "$spec.aurora" "$spec" + +dnf -y builddep "$spec" +rpmbuild --define "_topdir $rpmtop" -ba "$spec" + +# The loader, its service and the fprintd policy, from this tree. +rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ + -ba "$here/aurora-sep.spec" + +cp -p "$rpmtop"/RPMS/*/*.rpm "$rpmtop"/SRPMS/*.rpm "$out/" +ls -l "$out" diff --git a/tools/aurora-sep/tests/test_packaging.py b/tools/aurora-sep/tests/test_packaging.py new file mode 100644 index 00000000000000..f5aa1b124d060f --- /dev/null +++ b/tools/aurora-sep/tests/test_packaging.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""The packaging must describe the files in this tree, not stale copies.""" + +import hashlib +import re +import unittest +from pathlib import Path + +SEP = Path(__file__).resolve().parents[1] +ARCH = SEP / "packaging" / "arch" +FEDORA = SEP / "packaging" / "fedora" + + +def bash_array(text, name): + match = re.search(r"^%s=\((.*?)\)" % re.escape(name), text, re.M | re.S) + if match is None: + raise AssertionError("no %s array" % name) + return re.findall(r"'([^']*)'|\"([^\"]*)\"|(\S+)", match.group(1)) + + +def words(text, name): + return ["".join(parts) for parts in bash_array(text, name)] + + +class PkgbuildTest(unittest.TestCase): + def check(self, directory): + text = (directory / "PKGBUILD").read_text() + sources = words(text, "source") + sums = words(text, "sha256sums") + self.assertEqual(len(sources), len(sums)) + pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) + for source, expected in zip(sources, sums): + if "://" in source: + continue + name = source.replace("$pkgver", pkgver) + path = directory / name + self.assertTrue(path.is_file(), "%s is not a file" % path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + self.assertEqual( + digest, expected, "%s changed: run updpkgsums in %s" % (name, directory) + ) + + def test_libfprint_sums_match_tree(self): + self.check(ARCH / "libfprint-aurora") + + def test_aurora_sep_sums_match_tree(self): + self.check(ARCH / "aurora-sep") + + def test_patch_version_matches_pkgver(self): + text = (ARCH / "libfprint-aurora" / "PKGBUILD").read_text() + pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) + self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % pkgver)).is_file()) + + +class FedoraSpecTest(unittest.TestCase): + def test_sources_are_in_tree(self): + text = (FEDORA / "aurora-sep.spec").read_text() + sources = re.findall(r"^Source\d+:\s+(\S+)$", text, re.M) + self.assertTrue(sources) + for source in sources: + self.assertTrue((SEP / source).is_file(), source) + + def test_build_script_patches_the_shipped_version(self): + text = (FEDORA / "build.sh").read_text() + version = re.search(r"^patched_version=(\S+)$", text, re.M).group(1) + self.assertIn("libfprint-%s-apple-sep.patch" % version, text) + self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % version)).is_file()) + + +if __name__ == "__main__": + unittest.main() From c6e4dc3e2f715b1d1ed3fe5aaab0293cb8e8fadf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:34:30 +1100 Subject: [PATCH 330/631] tools: aurora-sep: activate Touch ID at boot, before fprintd The first open of /dev/sep-bio activates Touch ID: the driver restores the enrolled fingerprints from the enclave before the open returns, which takes a few seconds. That open is fprintd's first probe of the reader, so the first fingerprint request after boot, at the lock screen or for sudo, waits for the restore. Add aurora-sep-ready.service, which opens the device once after the loader and before fprintd, and pull it in from aurora-sep.service. It ends as soon as the open returns; a failed open is only logged, and a job timeout stops anything ordered after it from waiting beyond 90 s after boot. Install it in the Fedora and Arch packages. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 16 +++++++ tools/aurora-sep/aurora-sep-ready | 44 +++++++++++++++++++ tools/aurora-sep/aurora-sep-ready.service | 18 ++++++++ tools/aurora-sep/aurora-sep.service | 2 + .../packaging/arch/aurora-sep/PKGBUILD | 36 ++++++++++----- .../arch/aurora-sep/aurora-sep-ready | 1 + .../arch/aurora-sep/aurora-sep-ready.service | 1 + .../packaging/fedora/aurora-sep.spec | 41 +++++++++++------ 8 files changed, 133 insertions(+), 26 deletions(-) create mode 100644 tools/aurora-sep/aurora-sep-ready create mode 100644 tools/aurora-sep/aurora-sep-ready.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index dc08097f36b8f1..bf81fb9663f5b4 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -208,6 +208,22 @@ fprintd at another libfprint, then run `systemctl daemon-reload` and libfprint, remove `aurora-sep` and run `dnf distro-sync libfprint` or `pacman -S libfprint`. +## Desktop integration + +The first open of `/dev/sep-bio` activates Touch ID: the driver restores the +enrolled fingerprints from the enclave before the open returns, which takes +a few seconds. `aurora-sep-ready.service`, which `aurora-sep.service` pulls +in, makes that open at boot, before fprintd starts. The lock screen, sudo +and polkit then find the reader and the fingerprints ready, and the first +fingerprint request does not wait for the restore. A failed open is only +logged. The packages install it; by hand: + +```sh +install -Dm755 aurora-sep-ready /usr/local/sbin/aurora-sep-ready +install -Dm644 aurora-sep-ready.service /etc/systemd/system/aurora-sep-ready.service +systemctl daemon-reload +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/aurora-sep-ready b/tools/aurora-sep/aurora-sep-ready new file mode 100644 index 00000000000000..0363a3a298ecbb --- /dev/null +++ b/tools/aurora-sep/aurora-sep-ready @@ -0,0 +1,44 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Bring Touch ID up before anyone asks for a fingerprint. The first open of +# /dev/sep-bio activates it: the driver restores the enrolled fingerprints +# from the enclave and returns only when that is done. Opened here at boot, +# ahead of fprintd, the restore no longer delays the first fingerprint +# request. A failure is only logged. +set -u + +dev=/dev/sep-bio + +log() { + echo "aurora-sep-ready: $*" >&2 +} + +uptime_s() { + read -r up _ &1); then + log "Touch ID is ready after $(($(uptime_s) - start)) s" + exit 0 + fi + case "$err" in + # Another opener is activating it or using it; the unit's start + # timeout bounds this. + *[Bb]usy*) sleep 0.2 ;; + *) + log "could not open $dev: ${err##*: }" + exit 0 + ;; + esac +done diff --git a/tools/aurora-sep/aurora-sep-ready.service b/tools/aurora-sep/aurora-sep-ready.service new file mode 100644 index 00000000000000..91e5a7bce38154 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-ready.service @@ -0,0 +1,18 @@ +[Unit] +Description=Activate Touch ID and restore its fingerprints +Documentation=file:///usr/share/doc/aurora-sep/README.md +Requires=aurora-sep.service +After=aurora-sep.service +Before=fprintd.service +ConditionPathExists=/dev/sep-bio + +# Whatever is ordered after this waits for the enclave, which can take most +# of a minute after boot. Stop waiting 90 s after boot, so that a broken +# enclave cannot hold anything back for longer. +JobTimeoutSec=90 + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/aurora-sep-ready +TimeoutStartSec=60 diff --git a/tools/aurora-sep/aurora-sep.service b/tools/aurora-sep/aurora-sep.service index 4287247cba0da1..34a9f1cd69eb76 100644 --- a/tools/aurora-sep/aurora-sep.service +++ b/tools/aurora-sep/aurora-sep.service @@ -2,6 +2,8 @@ Description=Aurora SEP services After=local-fs.target RequiresMountsFor=/var/lib +# Activate Touch ID as soon as the driver is up. +Wants=aurora-sep-ready.service [Service] Type=oneshot diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index d7a921a5501cf6..e33322557b247d 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -1,13 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-only # -# The loader, service and fprintd policy for the Apple SEP fingerprint driver. +# The loader and desktop integration for the Apple SEP fingerprint driver. # The local sources are links into tools/aurora-sep; after changing any of # them, run updpkgsums here. libfprint-aurora provides the libfprint driver. pkgname=aurora-sep pkgver=1 -pkgrel=1 -pkgdesc="Loader and fprintd policy for the Apple SEP (Touch ID) driver" +pkgrel=2 +pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) license=(GPL-2.0-only) @@ -21,22 +21,34 @@ depends=( source=(load-driver aurora-sep.service fprintd-aurora.conf + aurora-sep-ready + aurora-sep-ready.service README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' - 'd3441eeb06c930fcc13314d18012b1f5d71cc1a821ceaf44de13caafb2931d73' + '447f99cc3c7e683134f31b993aa825fadde24e4854954a3035c96e6631d4d4d0' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' - 'd6e5c68bab7e0d71da48820da75d4ff7e67a06e7970961377ed28d30c2b66c08' + '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' + '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' + '29a8fb4970f7e550310a1c11d6aa0ad2b840b0307f4be0d1ad76875bed1a4ec2' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') -package() { - install -Dm755 load-driver "$pkgdir/usr/lib/aurora-sep/aurora-sep-load" +# A unit and the helper it runs ($1, installed as $3), pointed at the +# packaged helper. +_unit() { + local unit="$pkgdir/usr/lib/systemd/system/$2" + + install -Dm755 "$1" "$pkgdir/usr/lib/aurora-sep/$3" + install -Dm644 "$2" "$unit" + sed -i "s|^\(Exec[A-Za-z]*=\)/usr/local/sbin/$3|\1/usr/lib/aurora-sep/$3|" "$unit" + if grep -q /usr/local "$unit"; then + return 1 + fi +} - install -Dm644 aurora-sep.service "$pkgdir/usr/lib/systemd/system/aurora-sep.service" - sed -i 's|^ExecStart=.*|ExecStart=/usr/lib/aurora-sep/aurora-sep-load|' \ - "$pkgdir/usr/lib/systemd/system/aurora-sep.service" - grep -qx 'ExecStart=/usr/lib/aurora-sep/aurora-sep-load' \ - "$pkgdir/usr/lib/systemd/system/aurora-sep.service" +package() { + _unit load-driver aurora-sep.service aurora-sep-load + _unit aurora-sep-ready aurora-sep-ready.service aurora-sep-ready install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready new file mode 120000 index 00000000000000..55fb2a87e5d434 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready @@ -0,0 +1 @@ +../../../aurora-sep-ready \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service new file mode 120000 index 00000000000000..7ab622ca2d8339 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-ready.service @@ -0,0 +1 @@ +../../../aurora-sep-ready.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 9d09b927274794..57ae4ba402c1ab 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -5,8 +5,8 @@ Name: aurora-sep Version: 1 -Release: 1%{?dist} -Summary: Loader and fprintd policy for the Apple SEP fingerprint driver +Release: 2%{?dist} +Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only URL: https://github.com/aurora-silicon/linux @@ -15,6 +15,8 @@ Source1: aurora-sep.service Source2: fprintd-aurora.conf Source3: README.md Source4: XART-SAFETY.md +Source5: aurora-sep-ready +Source6: aurora-sep-ready.service BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -28,8 +30,9 @@ Requires: kmod Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd -also pulls it in when it starts. libfprint needs the aurora driver, which -this package does not provide. Read XART-SAFETY.md before installing it on +also pulls it in when it starts. Touch ID is then activated, before fprintd +starts. libfprint needs the aurora driver, which this package does not +provide. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -39,12 +42,17 @@ cp -p %{SOURCE3} %{SOURCE4} . %build %install -install -Dpm 0755 %{SOURCE0} %{buildroot}%{_libexecdir}/aurora-sep-load -install -Dpm 0644 %{SOURCE1} %{buildroot}%{_unitdir}/aurora-sep.service -sed -i 's|^ExecStart=.*|ExecStart=%{_libexecdir}/aurora-sep-load|' \ - %{buildroot}%{_unitdir}/aurora-sep.service -grep -qx 'ExecStart=%{_libexecdir}/aurora-sep-load' \ - %{buildroot}%{_unitdir}/aurora-sep.service +# A unit and the helper it runs ($1, installed as $3), pointed at the +# packaged helper. +unit() { + install -Dpm 0755 "$1" %{buildroot}%{_libexecdir}/"$3" + install -Dpm 0644 "$2" %{buildroot}%{_unitdir}/"${2##*/}" + sed -i "s|^\(Exec[A-Za-z]*=\)/usr/local/sbin/$3|\1%{_libexecdir}/$3|" \ + %{buildroot}%{_unitdir}/"${2##*/}" + if grep -q /usr/local %{buildroot}%{_unitdir}/"${2##*/}"; then exit 1; fi +} +unit %{SOURCE0} %{SOURCE1} aurora-sep-load +unit %{SOURCE5} %{SOURCE6} aurora-sep-ready install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf # Load the driver at boot, so the enclave is up before the login screen asks. @@ -52,23 +60,28 @@ install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %post -%systemd_post aurora-sep.service +%systemd_post aurora-sep.service aurora-sep-ready.service %preun -%systemd_preun aurora-sep.service +%systemd_preun aurora-sep.service aurora-sep-ready.service %postun -# Never restart it: the driver attaches to the enclave once per boot. -%systemd_postun aurora-sep.service +# Never restart them: the driver attaches to the enclave once per boot. +%systemd_postun aurora-sep.service aurora-sep-ready.service %files %doc README.md XART-SAFETY.md %{_libexecdir}/aurora-sep-load +%{_libexecdir}/aurora-sep-ready %{_unitdir}/aurora-sep.service +%{_unitdir}/aurora-sep-ready.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_presetdir}/80-aurora-sep.preset %changelog +* Sun Oct 11 2026 Ryan Murray - 1-2 +- Activate Touch ID at boot, before fprintd starts + * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From 12984b0c3a7dfe5c3831a7f405c2954663405fce Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:34:51 +1100 Subject: [PATCH 331/631] tools: aurora-sep: keep fprintd running fprintd exits after 30 s without a client, and each new start probes the reader again. The login and lock screens ask fprintd for the reader as they come up, so they wait for that start, and for a reader that is still being probed. Add a drop-in that runs fprintd with --no-timeout. Its path differs between distributions, so the drop-in is a template: the packages fill it in and check it against fprintd's own unit at build time. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 11 +++++++++++ tools/aurora-sep/fprintd-no-timeout.conf.in | 7 +++++++ tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 7 ++++++- .../arch/aurora-sep/fprintd-no-timeout.conf.in | 1 + tools/aurora-sep/packaging/fedora/aurora-sep.spec | 8 ++++++++ tools/aurora-sep/packaging/fedora/build.sh | 3 ++- 6 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tools/aurora-sep/fprintd-no-timeout.conf.in create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index bf81fb9663f5b4..8189bbea0f15c4 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -224,6 +224,17 @@ install -Dm644 aurora-sep-ready.service /etc/systemd/system/aurora-sep-ready.ser systemctl daemon-reload ``` +fprintd exits after 30 s without a client, and each new start probes the +reader again, while the login and lock screens ask for the reader as they +come up. `fprintd-no-timeout.conf.in` keeps it running so that it answers +them at once. The packages fill in where the distribution keeps fprintd; by +hand, with `/usr/lib/fprintd` on Arch: + +```sh +sed 's|@FPRINTD@|/usr/libexec/fprintd|' fprintd-no-timeout.conf.in \ + >/etc/systemd/system/fprintd.service.d/aurora-sep-no-timeout.conf +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/fprintd-no-timeout.conf.in b/tools/aurora-sep/fprintd-no-timeout.conf.in new file mode 100644 index 00000000000000..35f70927831fa9 --- /dev/null +++ b/tools/aurora-sep/fprintd-no-timeout.conf.in @@ -0,0 +1,7 @@ +# Keep fprintd running instead of letting it exit after 30 s without a +# client. Each new start probes the reader again, and the login and lock +# screens ask it for the reader when they come up, so a resident fprintd +# answers them at once. The packages fill in fprintd's path. +[Service] +ExecStart= +ExecStart=@FPRINTD@ --no-timeout diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index e33322557b247d..01d0b410d2fff2 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -23,6 +23,7 @@ source=(load-driver fprintd-aurora.conf aurora-sep-ready aurora-sep-ready.service + fprintd-no-timeout.conf.in README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -30,7 +31,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' 'f8d0c4b9385e6716d5ac5dd9941a57459fbc551a8238383bcde36cd929d0523f' '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' - '29a8fb4970f7e550310a1c11d6aa0ad2b840b0307f4be0d1ad76875bed1a4ec2' + '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' + '9d8caa6ad9be48cdb9331a5498eafad2b7db82764fe41b74bc97ba3fcccfbb2b' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -52,5 +54,8 @@ package() { install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" + grep -qx 'ExecStart=/usr/lib/fprintd' /usr/lib/systemd/system/fprintd.service + sed 's|@FPRINTD@|/usr/lib/fprintd|' fprintd-no-timeout.conf.in \ + >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in new file mode 120000 index 00000000000000..2e8a7a43c8830d --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/fprintd-no-timeout.conf.in @@ -0,0 +1 @@ +../../../fprintd-no-timeout.conf.in \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 57ae4ba402c1ab..59a0ade7ed2d4f 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -17,9 +17,12 @@ Source3: README.md Source4: XART-SAFETY.md Source5: aurora-sep-ready Source6: aurora-sep-ready.service +Source7: fprintd-no-timeout.conf.in BuildArch: noarch BuildRequires: systemd-rpm-macros +# Only to check where fprintd's unit runs it from. +BuildRequires: fprintd Requires: coreutils Requires: fprintd Requires: keyutils @@ -55,6 +58,9 @@ unit %{SOURCE0} %{SOURCE1} aurora-sep-load unit %{SOURCE5} %{SOURCE6} aurora-sep-ready install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf +grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service +sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ + >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -77,11 +83,13 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %{_unitdir}/aurora-sep-ready.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf +%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf %{_presetdir}/80-aurora-sep.preset %changelog * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts +- Keep fprintd running * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh index 32d9f30761d027..27fb64e28aafda 100755 --- a/tools/aurora-sep/packaging/fedora/build.sh +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -90,7 +90,8 @@ mv "$spec.aurora" "$spec" dnf -y builddep "$spec" rpmbuild --define "_topdir $rpmtop" -ba "$spec" -# The loader, its service and the fprintd policy, from this tree. +# The loader, its units and the desktop integration, from this tree. +dnf -y builddep "$here/aurora-sep.spec" rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ -ba "$here/aurora-sep.spec" From 700c97c09d72d2b6c0a501fb51b55c4965d18e92 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:35:10 +1100 Subject: [PATCH 332/631] tools: aurora-sep: suspend rather than power off on the Touch ID button On Macs with Touch ID the sensor is the power button. Touching it to authenticate goes to the enclave and never produces a key event, but pressing it does, and logind cannot tell a press meant for the sensor from a press meant for the button. Its default action is to power off, so pressing while authenticating outside GNOME ends the session. Add a logind drop-in that suspends on a short press instead: a press while authenticating then costs nothing, and the next press wakes the machine. Holding the button for 5 s powers off cleanly. GNOME sessions and the GDM login screen take a short press over and suspend by default already, so this only changes consoles and other desktops. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 14 ++++++++++++++ tools/aurora-sep/logind-aurora-sep.conf | 11 +++++++++++ .../aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++++- .../arch/aurora-sep/logind-aurora-sep.conf | 1 + tools/aurora-sep/packaging/fedora/aurora-sep.spec | 9 +++++++-- 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 tools/aurora-sep/logind-aurora-sep.conf create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 8189bbea0f15c4..ab212c5b7bf47c 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -235,6 +235,20 @@ sed 's|@FPRINTD@|/usr/libexec/fprintd|' fprintd-no-timeout.conf.in \ >/etc/systemd/system/fprintd.service.d/aurora-sep-no-timeout.conf ``` +The Touch ID sensor is the power button. A touch to authenticate goes to the +enclave and never reaches logind as a key press, but a press does, and logind +cannot tell a press meant for the sensor from one meant for the button. Its +default is to power off. `logind-aurora-sep.conf` makes a short press +suspend instead, so pressing while authenticating loses nothing and the next +press wakes the machine, and holding the button for 5 s powers off cleanly. +GNOME sessions and the GDM login screen handle a short press themselves +(`power-button-action`, `suspend` by default); the setting covers consoles +and other desktops. It takes effect at the next boot. By hand: + +```sh +install -Dm644 logind-aurora-sep.conf /etc/systemd/logind.conf.d/aurora-sep.conf +``` + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/logind-aurora-sep.conf b/tools/aurora-sep/logind-aurora-sep.conf new file mode 100644 index 00000000000000..a175c39f6ca73c --- /dev/null +++ b/tools/aurora-sep/logind-aurora-sep.conf @@ -0,0 +1,11 @@ +# The Touch ID sensor is the power button. Touching it to authenticate goes to +# the enclave and is never a key press, but pressing it is, and logind cannot +# tell a press meant for the sensor from one meant for the button. Suspend +# rather than power off on a short press, so a press while authenticating +# costs nothing: the next press wakes the machine. Holding the button for +# 5 s still powers off, cleanly. GNOME sessions and the GDM login screen +# handle a short press themselves (power-button-action, suspend by default); +# this covers everything else. +[Login] +HandlePowerKey=suspend +HandlePowerKeyLongPress=poweroff diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 01d0b410d2fff2..3c093add292bac 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -24,6 +24,7 @@ source=(load-driver aurora-sep-ready aurora-sep-ready.service fprintd-no-timeout.conf.in + logind-aurora-sep.conf README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -32,7 +33,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '2db07d3a0191e8d14581a3056505a8c4225b2fc8e6ea4a1956ae5493abcbb0ae' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' - '9d8caa6ad9be48cdb9331a5498eafad2b7db82764fe41b74bc97ba3fcccfbb2b' + '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' + '3f5a8a4a19e15268552264da95d4848be31629c49f3f64f377c590cf6ef9e1cb' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -57,5 +59,7 @@ package() { grep -qx 'ExecStart=/usr/lib/fprintd' /usr/lib/systemd/system/fprintd.service sed 's|@FPRINTD@|/usr/lib/fprintd|' fprintd-no-timeout.conf.in \ >"$pkgdir/usr/lib/systemd/system/fprintd.service.d/50-aurora-sep-no-timeout.conf" + install -Dm644 logind-aurora-sep.conf \ + "$pkgdir/usr/lib/systemd/logind.conf.d/50-aurora-sep.conf" install -Dm644 -t "$pkgdir/usr/share/doc/$pkgname" README.md XART-SAFETY.md } diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf b/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf new file mode 120000 index 00000000000000..db6dc4e9530226 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/logind-aurora-sep.conf @@ -0,0 +1 @@ +../../../logind-aurora-sep.conf \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 59a0ade7ed2d4f..a916df31cc0a5f 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -18,6 +18,7 @@ Source4: XART-SAFETY.md Source5: aurora-sep-ready Source6: aurora-sep-ready.service Source7: fprintd-no-timeout.conf.in +Source8: logind-aurora-sep.conf BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -34,8 +35,8 @@ Loads the Apple SEP driver (apple_sep) with xART writes enabled, waits until the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd also pulls it in when it starts. Touch ID is then activated, before fprintd -starts. libfprint needs the aurora driver, which this package does not -provide. Read XART-SAFETY.md before installing it on +starts, and pressing the Touch ID button suspends rather than powers off. +libfprint needs the aurora driver, which this package does not provide. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -61,6 +62,8 @@ install -Dpm 0644 %{SOURCE2} \ grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service sed 's|@FPRINTD@|%{_libexecdir}/fprintd|' %{SOURCE7} \ >%{buildroot}%{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf +install -Dpm 0644 %{SOURCE8} \ + %{buildroot}%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf # Load the driver at boot, so the enclave is up before the login screen asks. install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset @@ -84,12 +87,14 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf +%{_prefix}/lib/systemd/logind.conf.d/50-aurora-sep.conf %{_presetdir}/80-aurora-sep.preset %changelog * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts - Keep fprintd running +- Suspend rather than power off on the Touch ID button * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From 9320212d5fcee1ff1e5ed83f9afb9fa25294f29c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 16:35:38 +1100 Subject: [PATCH 333/631] tools: aurora-sep: offer fingerprint at the GNOME login screen GNOME Shell 51 connects the greeter's fingerprint reader only when it re-authenticates a session that is already running, so the login screen never offers fingerprint, whatever org.gnome.login-screen says. That is deliberate: a fingerprint cannot unlock the login keyring, and the first application that wants a stored secret then asks for the password. Some people want fingerprint login anyway. Add aurora-sep-greeter-fingerprint.service, off by default. At boot it takes the greeter source file that holds the check from the installed GNOME Shell, drops the check, and serves the copy to the login screen's shell alone, through a GResource overlay set in a runtime drop-in for org.gnome.Shell@gdm.service. When the check is not exactly as expected, it changes nothing, so a GNOME update that moves it leaves the login screen as GNOME ships it. enable-fingerprint-authentication still turns fingerprint off. The login screen asks fprintd for the reader once, as it comes up, and offers no fingerprint until the user is selected again if Touch ID is not up yet. The unit therefore also orders the display manager after aurora-sep-ready.service. That holds the login screen back by a few seconds on a healthy boot, and up to 90 s after boot when the enclave is slow, which only makes sense when it offers fingerprint, so both come with the one switch. It needs gresource, which the packages suggest rather than require. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 30 +++++++ .../aurora-sep/aurora-sep-greeter-fingerprint | 85 +++++++++++++++++++ .../aurora-sep-greeter-fingerprint.service | 17 ++++ .../packaging/arch/aurora-sep/PKGBUILD | 9 +- .../aurora-sep/aurora-sep-greeter-fingerprint | 1 + .../aurora-sep-greeter-fingerprint.service | 1 + .../packaging/fedora/aurora-sep.spec | 14 ++- 7 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 tools/aurora-sep/aurora-sep-greeter-fingerprint create mode 100644 tools/aurora-sep/aurora-sep-greeter-fingerprint.service create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint create mode 120000 tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index ab212c5b7bf47c..4037743d4d5723 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -249,6 +249,36 @@ and other desktops. It takes effect at the next boot. By hand: install -Dm644 logind-aurora-sep.conf /etc/systemd/logind.conf.d/aurora-sep.conf ``` +GNOME Shell 51 offers fingerprint only on the lock screen, never on the +login screen, and `org.gnome.login-screen enable-fingerprint-authentication` +(true by default) cannot change that: the greeter connects its fingerprint +reader only when it re-authenticates a running session. That is GNOME's +choice, because a fingerprint cannot unlock the login keyring: after a +fingerprint login, the first application that wants a stored secret asks +for the password. `aurora-sep-greeter-fingerprint.service` lifts the +restriction for those who want it. At boot it copies the one greeter source +file that holds the check out of the installed GNOME Shell, drops the check, +and serves the copy to the login screen's shell alone through a GResource +overlay. When the check is not exactly as expected, for example after a +GNOME update, it changes nothing. The enable key above still applies. + +The login screen asks fprintd for the reader once, as it comes up, so the +same unit also holds it back until `aurora-sep-ready.service` has finished: +a few seconds on a healthy boot, but as long as the enclave takes when it is +slow, up to 90 s after boot. Without fingerprint at the login screen that +wait buys nothing, which is why both are one switch, off by default. It +needs `gresource` from glib2-devel. With the packages, or after installing +it by hand, turn it on with `systemctl enable`: + +```sh +install -Dm755 aurora-sep-greeter-fingerprint /usr/local/sbin/aurora-sep-greeter-fingerprint +install -Dm644 aurora-sep-greeter-fingerprint.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable aurora-sep-greeter-fingerprint.service +``` + +Disabling it and rebooting returns the login screen to GNOME's behaviour. + ## Biometric userspace header The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this diff --git a/tools/aurora-sep/aurora-sep-greeter-fingerprint b/tools/aurora-sep/aurora-sep-greeter-fingerprint new file mode 100644 index 00000000000000..7f04a19d24d20d --- /dev/null +++ b/tools/aurora-sep/aurora-sep-greeter-fingerprint @@ -0,0 +1,85 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +# +# Offer fingerprint at the GNOME login screen, or with "stop", stop offering +# it from the next time the login screen starts. +# +# GNOME Shell 51 connects the fingerprint reader only when it re-authenticates +# a session that is already running, which is the lock screen. The login +# screen never offers fingerprint, whatever org.gnome.login-screen says: a +# fingerprint cannot unlock the login keyring, so a session started with one +# asks for the password the first time an application wants a stored secret. +# +# This takes the greeter source file that holds that check from the installed +# GNOME Shell, drops the check, and serves the copy to the login screen's +# shell alone through a GResource overlay. When the check is not exactly as +# expected, for example after a GNOME update, it changes nothing and the login +# screen stays as GNOME ships it. Run it before the display manager starts. +set -eu + +res=/org/gnome/shell/gdm/authServicesLegacy.js +out=/run/aurora-sep/greeter +dropin=/run/systemd/user/org.gnome.Shell@gdm.service.d/aurora-sep-fingerprint.conf + +log() { + echo "aurora-sep-greeter-fingerprint: $*" >&2 +} + +rm -rf "$out" "$dropin" +[ "${1:-}" != stop ] || exit 0 + +if ! command -v gresource >/dev/null; then + log "gresource is not installed (it comes with glib2-devel); the login screen stays as it is" + exit 0 +fi + +lib= +for f in /usr/lib64/gnome-shell/libshell-*.so /usr/lib/gnome-shell/libshell-*.so \ + /usr/lib/*/gnome-shell/libshell-*.so; do + [ -f "$f" ] || continue + if gresource list "$f" 2>/dev/null | grep -qxF "$res"; then + lib=$f + break + fi +done +if [ -z "$lib" ]; then + log "no installed GNOME Shell carries $res; the login screen stays as it is" + exit 0 +fi + +install -d -m 0755 "$out" +gresource extract "$lib" "$res" >"$out/stock.js" +if ! awk ' + state == 0 && $0 == " _connectFingerprintManager() {" { + print + state = 1 + next + } + state == 1 { + if ($0 == " if (!this._reauthOnly)") { + state = 2 + next + } + state = 9 + } + state == 2 { + if ($0 == " return;") { + state = 3 + next + } + state = 9 + } + { print } + END { exit state != 3 } +' "$out/stock.js" >"$out/authServicesLegacy.js"; then + log "$lib does not have the expected check; the login screen stays as it is" + rm -rf "$out" + exit 0 +fi +rm -f "$out/stock.js" +chmod 0644 "$out/authServicesLegacy.js" + +install -d -m 0755 "${dropin%/*}" +printf '[Service]\nEnvironment=G_RESOURCE_OVERLAYS=%s=%s\n' \ + "$res" "$out/authServicesLegacy.js" >"$dropin" +log "the login screen offers fingerprint ($res from $lib)" diff --git a/tools/aurora-sep/aurora-sep-greeter-fingerprint.service b/tools/aurora-sep/aurora-sep-greeter-fingerprint.service new file mode 100644 index 00000000000000..108b1e4ccbc875 --- /dev/null +++ b/tools/aurora-sep/aurora-sep-greeter-fingerprint.service @@ -0,0 +1,17 @@ +[Unit] +Description=Offer fingerprint at the GNOME login screen +Documentation=file:///usr/share/doc/aurora-sep/README.md +# The login screen asks fprintd for the reader once, as it comes up. Hold it +# back until Touch ID is up, so that it finds the reader and the fingerprints. +Wants=aurora-sep-ready.service +After=aurora-sep-ready.service +Before=gdm.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/aurora-sep-greeter-fingerprint +ExecStop=/usr/local/sbin/aurora-sep-greeter-fingerprint stop + +[Install] +WantedBy=gdm.service diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 3c093add292bac..d27422443be355 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -18,6 +18,7 @@ depends=( kmod sh ) +optdepends=('glib2-devel: fingerprint at the GNOME login screen') source=(load-driver aurora-sep.service fprintd-aurora.conf @@ -25,6 +26,8 @@ source=(load-driver aurora-sep-ready.service fprintd-no-timeout.conf.in logind-aurora-sep.conf + aurora-sep-greeter-fingerprint + aurora-sep-greeter-fingerprint.service README.md XART-SAFETY.md) sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' @@ -34,7 +37,9 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '5a5e5c7e93d1462d9d1023446fdb297926aeb93d8988867b515aca7adafe4487' '8d8df0ba2a6861b2edb2110e606906d84ca4848617621ad3cb48493a1522971d' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' - '3f5a8a4a19e15268552264da95d4848be31629c49f3f64f377c590cf6ef9e1cb' + '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' + 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' + '226a5f5f2244583b5b10d9a5b3e134f5b155065fd8dce850c828f98f085fc931' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the @@ -53,6 +58,8 @@ _unit() { package() { _unit load-driver aurora-sep.service aurora-sep-load _unit aurora-sep-ready aurora-sep-ready.service aurora-sep-ready + _unit aurora-sep-greeter-fingerprint aurora-sep-greeter-fingerprint.service \ + aurora-sep-greeter-fingerprint install -Dm644 fprintd-aurora.conf \ "$pkgdir/usr/lib/systemd/system/fprintd.service.d/aurora-sep.conf" diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint new file mode 120000 index 00000000000000..dd1bb065453132 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint @@ -0,0 +1 @@ +../../../aurora-sep-greeter-fingerprint \ No newline at end of file diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service new file mode 120000 index 00000000000000..bf6aeca94eb787 --- /dev/null +++ b/tools/aurora-sep/packaging/arch/aurora-sep/aurora-sep-greeter-fingerprint.service @@ -0,0 +1 @@ +../../../aurora-sep-greeter-fingerprint.service \ No newline at end of file diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index a916df31cc0a5f..648409fb7c463d 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -19,6 +19,8 @@ Source5: aurora-sep-ready Source6: aurora-sep-ready.service Source7: fprintd-no-timeout.conf.in Source8: logind-aurora-sep.conf +Source9: aurora-sep-greeter-fingerprint +Source10: aurora-sep-greeter-fingerprint.service BuildArch: noarch BuildRequires: systemd-rpm-macros @@ -28,6 +30,8 @@ Requires: coreutils Requires: fprintd Requires: keyutils Requires: kmod +# Only for aurora-sep-greeter-fingerprint.service, which is off by default. +Suggests: /usr/bin/gresource %{?systemd_requires} %description @@ -57,6 +61,7 @@ unit() { } unit %{SOURCE0} %{SOURCE1} aurora-sep-load unit %{SOURCE5} %{SOURCE6} aurora-sep-ready +unit %{SOURCE9} %{SOURCE10} aurora-sep-greeter-fingerprint install -Dpm 0644 %{SOURCE2} \ %{buildroot}%{_unitdir}/fprintd.service.d/aurora-sep.conf grep -qx 'ExecStart=%{_libexecdir}/fprintd' %{_unitdir}/fprintd.service @@ -69,21 +74,23 @@ install -d %{buildroot}%{_presetdir} echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %post -%systemd_post aurora-sep.service aurora-sep-ready.service +%systemd_post aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %preun -%systemd_preun aurora-sep.service aurora-sep-ready.service +%systemd_preun aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %postun # Never restart them: the driver attaches to the enclave once per boot. -%systemd_postun aurora-sep.service aurora-sep-ready.service +%systemd_postun aurora-sep.service aurora-sep-ready.service aurora-sep-greeter-fingerprint.service %files %doc README.md XART-SAFETY.md %{_libexecdir}/aurora-sep-load %{_libexecdir}/aurora-sep-ready +%{_libexecdir}/aurora-sep-greeter-fingerprint %{_unitdir}/aurora-sep.service %{_unitdir}/aurora-sep-ready.service +%{_unitdir}/aurora-sep-greeter-fingerprint.service %dir %{_unitdir}/fprintd.service.d %{_unitdir}/fprintd.service.d/aurora-sep.conf %{_unitdir}/fprintd.service.d/50-aurora-sep-no-timeout.conf @@ -95,6 +102,7 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset - Activate Touch ID at boot, before fprintd starts - Keep fprintd running - Suspend rather than power off on the Touch ID button +- Offer fingerprint at the GNOME 51 login screen, off by default * Sun Oct 11 2026 Ryan Murray - 1-1 - Package the loader, its service and the fprintd policy From b553b2ff3466f5fd95ba29a15eaa26017a8984d6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 23:08:47 +1100 Subject: [PATCH 334/631] tools: aurora-sep: leave libfprint to aurora-sep-userspace libfprint with the aurora driver, as a source patch and as Fedora and Arch Linux packages, is now maintained in https://github.com/aurora-silicon/aurora-sep-userspace. Two copies of the userspace driver, built from different trees with colliding package releases, would drift apart. Drop the libfprint patch, the Arch libfprint-aurora package and the libfprint rebuild from the Fedora build script. This tree keeps the kernel side's userspace pieces: the loader, its services, the fprintd policy and the desktop integration, packaged as aurora-sep. The README and the package descriptions point to aurora-sep-userspace for the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/README.md | 31 +++------ .../packaging/arch/aurora-sep/PKGBUILD | 7 +- .../packaging/fedora/aurora-sep.spec | 8 ++- tools/aurora-sep/packaging/fedora/build.sh | 69 ++----------------- tools/aurora-sep/tests/test_packaging.py | 15 +--- 5 files changed, 28 insertions(+), 102 deletions(-) diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 4037743d4d5723..4c0540bf351a35 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -158,35 +158,26 @@ by the Quickshell lock screen. ## Packages -`packaging/` builds the same pieces as distribution packages: libfprint with -the aurora driver, which replaces the distribution's libfprint so fprintd needs -no library path, and `aurora-sep` with the loader, its service and the fprintd -policy. Installing `aurora-sep` lets fprintd load the driver with xART writes -on, as above. +`packaging/` builds `aurora-sep`, with the loader, its service and the fprintd +policy, as a distribution package. libfprint with the aurora driver comes from +https://github.com/aurora-silicon/aurora-sep-userspace; install both. Installing +`aurora-sep` lets fprintd load the driver with xART writes on, as above. -On Fedora, `packaging/fedora/build.sh` rebuilds the release's own libfprint -source package with the patch added (1.94.100 on Fedora 45) and builds -`aurora-sep`. It installs build dependencies, so run it in a throwaway -container from the top of this tree: +On Fedora, `packaging/fedora/build.sh` builds `aurora-sep`. It installs build +dependencies, so run it in a throwaway container from the top of this tree: ```sh mkdir rpms podman run --rm -v "$PWD":/src:ro,z -v "$PWD/rpms":/out:z \ registry.fedoraproject.org/fedora:45 \ /src/tools/aurora-sep/packaging/fedora/build.sh /out -sudo dnf install rpms/libfprint-1.94.100-*.aurora1.*.rpm \ - rpms/aurora-sep-1-*.noarch.rpm +sudo dnf install rpms/aurora-sep-1-*.noarch.rpm ``` -A later Fedora libfprint update replaces the rebuilt one; rebuild after it, or -hold it with `dnf versionlock add libfprint`. - -On Arch Linux and Arch Linux ARM, build `libfprint-aurora` (it replaces -`libfprint`) and then `aurora-sep`: +On Arch Linux and Arch Linux ARM, build `aurora-sep`: ```sh -cd tools/aurora-sep/packaging/arch/libfprint-aurora && makepkg -si -cd ../aurora-sep && makepkg -si +cd tools/aurora-sep/packaging/arch/aurora-sep && makepkg -si ``` fprintd starts the loader the first time it runs. Loading the driver at boot @@ -284,5 +275,5 @@ Disabling it and rebooting returns the login screen to GNOME's behaviour. The canonical ioctl ABI is `include/uapi/linux/apple-sep.h`. Export this checkout's headers with `make headers_install INSTALL_HDR_PATH=` and add that directory's `include` to the libfprint build include path. The -libfprint patch uses `` directly; it has no copied ABI -header. Interface version, ioctl numbers and fixed-width layouts are unchanged. +aurora driver in aurora-sep-userspace uses `` directly; it +has no copied ABI header. Interface version, ioctl numbers and fixed-width layouts are unchanged. diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index d27422443be355..15712b3bdf4e25 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -2,11 +2,12 @@ # # The loader and desktop integration for the Apple SEP fingerprint driver. # The local sources are links into tools/aurora-sep; after changing any of -# them, run updpkgsums here. libfprint-aurora provides the libfprint driver. +# them, run updpkgsums here. libfprint with the aurora driver comes from +# https://github.com/aurora-silicon/aurora-sep-userspace. pkgname=aurora-sep pkgver=1 -pkgrel=2 +pkgrel=3 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -39,7 +40,7 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - '226a5f5f2244583b5b10d9a5b3e134f5b155065fd8dce850c828f98f085fc931' + 'd7a09f31367c181a2cbda63cd58104d6117136fb5db5b20f4a3996ea3af7025e' '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') # A unit and the helper it runs ($1, installed as $3), pointed at the diff --git a/tools/aurora-sep/packaging/fedora/aurora-sep.spec b/tools/aurora-sep/packaging/fedora/aurora-sep.spec index 648409fb7c463d..b4dd9ca4391b6e 100644 --- a/tools/aurora-sep/packaging/fedora/aurora-sep.spec +++ b/tools/aurora-sep/packaging/fedora/aurora-sep.spec @@ -5,7 +5,7 @@ Name: aurora-sep Version: 1 -Release: 2%{?dist} +Release: 3%{?dist} Summary: Loader and desktop integration for the Apple SEP fingerprint driver License: GPL-2.0-only @@ -40,7 +40,8 @@ the secure enclave seals and unseals a trusted key and /dev/sep-bio appears, and lets fprintd reach /dev/sep-bio. The loader runs at boot, and fprintd also pulls it in when it starts. Touch ID is then activated, before fprintd starts, and pressing the Touch ID button suspends rather than powers off. -libfprint needs the aurora driver, which this package does not provide. Read XART-SAFETY.md before installing it on +libfprint needs the aurora driver, which this package does not provide; it is +packaged in https://github.com/aurora-silicon/aurora-sep-userspace. Read XART-SAFETY.md before installing it on a machine that also runs macOS. %prep @@ -98,6 +99,9 @@ echo 'enable aurora-sep.service' >%{buildroot}%{_presetdir}/80-aurora-sep.preset %{_presetdir}/80-aurora-sep.preset %changelog +* Sun Oct 11 2026 Ryan Murray - 1-3 +- libfprint with the aurora driver is now built in aurora-sep-userspace + * Sun Oct 11 2026 Ryan Murray - 1-2 - Activate Touch ID at boot, before fprintd starts - Keep fprintd running diff --git a/tools/aurora-sep/packaging/fedora/build.sh b/tools/aurora-sep/packaging/fedora/build.sh index 27fb64e28aafda..a1331bbdb266b1 100755 --- a/tools/aurora-sep/packaging/fedora/build.sh +++ b/tools/aurora-sep/packaging/fedora/build.sh @@ -1,13 +1,13 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0-only # -# Build the Fedora packages for Touch ID on the Apple SEP: +# Build the Fedora package for Touch ID on the Apple SEP: # -# - libfprint: Fedora's own source package, rebuilt with the aurora driver -# patch from ../../patches and nothing else changed. The release gains an -# ".aurora1" suffix, so it replaces the distribution's build in place. # - aurora-sep: the driver loader, its service and the fprintd policy. # +# libfprint with the aurora driver is built and packaged in +# https://github.com/aurora-silicon/aurora-sep-userspace. +# # It installs the build dependencies with dnf, so run it as root in a # throwaway container of the Fedora release you are building for: # @@ -21,13 +21,6 @@ set -eu out=${1:?usage: build.sh } here=$(cd "$(dirname "$0")" && pwd) sep=$(cd "$here/../.." && pwd) -patch=$sep/patches/libfprint-1.94.100-apple-sep.patch -header=${APPLE_SEP_HEADER:-$sep/../../include/uapi/linux/apple-sep.h} -patched_version=1.94.100 - -for f in "$patch" "$header"; do - [ -f "$f" ] || { echo "build.sh: $f is missing" >&2; exit 1; } -done mkdir -p "$out" top=$(mktemp -d) @@ -36,60 +29,6 @@ rpmtop=$top/rpmbuild dnf -y install rpm-build systemd-rpm-macros -# Fedora's libfprint, exactly as the distribution builds it. -(cd "$top" && dnf -y download --source libfprint) -srpm=$(ls "$top"/libfprint-*.src.rpm) -rpm -i --define "_topdir $rpmtop" "$srpm" -spec=$rpmtop/SPECS/libfprint.spec - -version=$(rpmspec -q --srpm --qf '%{version}\n' "$spec") -if [ "$version" != "$patched_version" ]; then - echo "build.sh: this Fedora ships libfprint $version, the patch is for $patched_version" >&2 - exit 1 -fi -cp "$patch" "$header" "$rpmtop/SOURCES/" - -# Add the patch and the header, mark the release, and point the compiler at -# the header: is not in the distribution's kernel headers. -# %autosetup applies every PatchN. Stop if any anchor has moved. -awk -v patch="$(basename "$patch")" -v date="$(LC_ALL=C date -u '+%a %b %d %Y')" ' - /^Release:/ && !release { - sub(/[[:space:]]*$/, ".aurora1"); print; release = 1; next - } - /^Source0:/ && !source { - print - print "Source9000: apple-sep.h" - print "Patch9000: " patch - source = 1; next - } - /^%meson[[:space:]]/ && !build { - print "# The aurora driver includes from the Aurora kernel." - print "mkdir -p aurora-include/linux" - print "cp -p %{SOURCE9000} aurora-include/linux/" - print "export CFLAGS=\"%{build_cflags} -I$PWD/aurora-include\"" - print; build = 1; next - } - /^%changelog/ && !changelog { - print - print "* " date " Aurora Silicon - " "%{version}-%{release}" - print "- Add the Apple SEP (aurora) driver from aurora-silicon/linux" - print " tools/aurora-sep/patches/" patch - print "" - changelog = 1; next - } - { print } - END { - if (!(release && source && build && changelog)) { - print "build.sh: libfprint.spec changed shape; update the edit" > "/dev/stderr" - exit 1 - } - } -' "$spec" >"$spec.aurora" -mv "$spec.aurora" "$spec" - -dnf -y builddep "$spec" -rpmbuild --define "_topdir $rpmtop" -ba "$spec" - # The loader, its units and the desktop integration, from this tree. dnf -y builddep "$here/aurora-sep.spec" rpmbuild --define "_topdir $rpmtop" --define "_sourcedir $sep" \ diff --git a/tools/aurora-sep/tests/test_packaging.py b/tools/aurora-sep/tests/test_packaging.py index f5aa1b124d060f..961985758ba1e5 100644 --- a/tools/aurora-sep/tests/test_packaging.py +++ b/tools/aurora-sep/tests/test_packaging.py @@ -41,17 +41,9 @@ def check(self, directory): digest, expected, "%s changed: run updpkgsums in %s" % (name, directory) ) - def test_libfprint_sums_match_tree(self): - self.check(ARCH / "libfprint-aurora") - def test_aurora_sep_sums_match_tree(self): self.check(ARCH / "aurora-sep") - def test_patch_version_matches_pkgver(self): - text = (ARCH / "libfprint-aurora" / "PKGBUILD").read_text() - pkgver = re.search(r"^pkgver=(\S+)$", text, re.M).group(1) - self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % pkgver)).is_file()) - class FedoraSpecTest(unittest.TestCase): def test_sources_are_in_tree(self): @@ -61,11 +53,10 @@ def test_sources_are_in_tree(self): for source in sources: self.assertTrue((SEP / source).is_file(), source) - def test_build_script_patches_the_shipped_version(self): + def test_build_script_builds_only_aurora_sep(self): text = (FEDORA / "build.sh").read_text() - version = re.search(r"^patched_version=(\S+)$", text, re.M).group(1) - self.assertIn("libfprint-%s-apple-sep.patch" % version, text) - self.assertTrue((SEP / "patches" / ("libfprint-%s-apple-sep.patch" % version)).is_file()) + self.assertIn("aurora-sep.spec", text) + self.assertNotIn("dnf -y download --source libfprint", text) if __name__ == "__main__": From 7eff30e42955742a739bc538411d9d515df30776 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:57:42 +1100 Subject: [PATCH 335/631] arm64: configs: add a J700 base fragment defconfig, asahi.config and j700-radio.config give a J700 kernel that boots, but not the configuration it is tested with. The boot path (NVMe/ANS, btrfs, the DockChannel keyboard and trackpad, SMC, SPMI, the USB-C PHY and Type-C) ends up modular and so depends on what the initramfs carries, the per-process 4K user page mode stays off, and several J700 driver choices differ. Add j700.config, to be merged after j700-radio.config, with each group commented with its reason, and cover the J700 fragments with one MAINTAINERS pattern. Together with the device-parity fragment added next, the fragments reproduce the tested configuration. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 +- arch/arm64/configs/j700.config | 113 +++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/configs/j700.config diff --git a/MAINTAINERS b/MAINTAINERS index e37ddc37885aed..46b22192afaa69 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2008,7 +2008,7 @@ F: Documentation/hid/apple-j700-multitouch.rst F: Documentation/soc/apple/t8140-pmp-dvfs.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* -F: arch/arm64/configs/j700-radio.config +F: arch/arm64/configs/j700*.config F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config new file mode 100644 index 00000000000000..976fd65b7161cf --- /dev/null +++ b/arch/arm64/configs/j700.config @@ -0,0 +1,113 @@ +# J700 (MacBook Neo, T8140) base fragment. +# +# Merge after defconfig, asahi.config and j700-radio.config, then add the device-parity fragment: +# make defconfig +# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/asahi.config arch/arm64/configs/j700-radio.config +# make olddefconfig +# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/j700.config arch/arm64/configs/j700-device-parity.config +# make olddefconfig +# The result is the configuration the J700 kernel is tested with. + +# Boot NOR (nvram) access for boot-volume switching, and the PMU RTC wake alarm +CONFIG_SPI_APPLE_QSPI=m +CONFIG_RTC_DRV_APPLE_PMU=m + +# Classic Bluetooth profiles (headsets, tethering) +CONFIG_BT_RFCOMM=m +CONFIG_BT_RFCOMM_TTY=y +CONFIG_BT_BNEP=m +CONFIG_BT_BNEP_MC_FILTER=y +CONFIG_BT_BNEP_PROTO_FILTER=y + +# Distribution firmware files are compressed +CONFIG_FW_LOADER_COMPRESS=y +CONFIG_FW_LOADER_COMPRESS_ZSTD=y + +# 48-bit virtual addresses; KASLR is untested on this machine +CONFIG_ARM64_VA_BITS_48=y +# CONFIG_RANDOMIZE_BASE is not set + +# Boot path and platform built in, so storage, keyboard, SMC and USB do not depend on the initramfs +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_APPLE=y +CONFIG_APPLE_SART=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_BTRFS_FS=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_SPMI_APPLE=y +CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_MFD_MACSMC=y +CONFIG_GPIO_MACSMC=y +CONFIG_POWER_RESET_MACSMC=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_I2C_APPLE=y +CONFIG_PINCTRL_APPLE_GPIO=y +CONFIG_APPLE_WATCHDOG=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_MUX_APPLE_DPXBAR=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_SYSFB_SIMPLEFB=y +CONFIG_ZRAM=y +CONFIG_UHID=y +CONFIG_RFKILL=y +CONFIG_THERMAL_NETLINK=y +CONFIG_NVME_HWMON=y + +# J700 driver choices +CONFIG_APPLE_CLPC=m +CONFIG_APPLE_PMP=y +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE=m +# CONFIG_HID_APPLE_MTP_HAPTIC is not set +CONFIG_IIO=m +# CONFIG_IIO_AOP_SENSOR_LAS is not set +# CONFIG_USB4 is not set + +# Per-process 4K user pages on the 16K kernel (x86 emulation) +CONFIG_ARM64_USER4K_EXPERIMENTAL=y + +# USB Ethernet built in, for a wired link before modules load +CONFIG_USB_USBNET=y +CONFIG_USB_NET_AX8817X=y +CONFIG_USB_NET_AX88179_178A=y +CONFIG_USB_NET_CDC_NCM=y +CONFIG_USB_RTL8152=y +CONFIG_USB_UAS=m + +# Crypto that Bluetooth and Wi-Fi need early +CONFIG_CRYPTO_CCM=y +CONFIG_CRYPTO_ECDH=y +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_SHA256=y + +# Utilisation clamping +CONFIG_UCLAMP_TASK=y +CONFIG_UCLAMP_TASK_GROUP=y + +# LSMs named in CONFIG_LSM +CONFIG_SECURITY_LANDLOCK=y +CONFIG_SECURITY_YAMA=y + +# dma-buf heaps and udmabuf (camera and GPU buffer sharing) +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_SYSTEM=y +CONFIG_UDMABUF=y + +# Debugging aids kept in the tested configuration +CONFIG_DYNAMIC_DEBUG=y +CONFIG_FTRACE=y +CONFIG_ENABLE_DEFAULT_TRACERS=y +CONFIG_PM_DEBUG=y +CONFIG_PM_ADVANCED_DEBUG=y + +# Match the tested kernel image: no relative relocation packing, no SCM version suffix +# CONFIG_RELR is not set +# CONFIG_LOCALVERSION_AUTO is not set From 052a0439f2ab5106fb0e326a84559d4e5b8052e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:57:43 +1100 Subject: [PATCH 336/631] arm64: configs: add a J700 device-parity fragment The J700 configuration lacks much of what a general-purpose distribution kernel provides: game controllers and other USB or Bluetooth HID devices, most USB-serial and USB network adapters, filesystems found on removable media, dm-crypt, VPN protocols, the nftables and xtables extensions used by firewall and container tools, and binfmt_misc. Add j700-device-parity.config, to be merged after j700.config. It takes the Fedora Asahi Remix 44 kernel configuration as the reference and enables what can attach to or run on the J700, mostly as modules. It changes no option that the other fragments set. Options that change scheduler, memory, security or boot behaviour are left out. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/configs/j700-device-parity.config | 1053 ++++++++++++++++++ 1 file changed, 1053 insertions(+) create mode 100644 arch/arm64/configs/j700-device-parity.config diff --git a/arch/arm64/configs/j700-device-parity.config b/arch/arm64/configs/j700-device-parity.config new file mode 100644 index 00000000000000..55bce7a9ac0af5 --- /dev/null +++ b/arch/arm64/configs/j700-device-parity.config @@ -0,0 +1,1053 @@ +# J700 (MacBook Neo, T8140) device-parity fragment. +# +# Brings the J700 configuration to device and feature parity with a general-purpose +# distribution kernel (reference: Fedora Asahi Remix 44, 7.1.13-402.asahi.fc44.aarch64+16k): +# drivers for devices that can attach over USB or Bluetooth, and common protocols, +# filesystems and features. Most entries are modules. Nothing here changes an option +# that defconfig, asahi.config, j700-radio.config or j700.config set. +# +# Merge after j700.config; see that file for the full sequence. + +# Prerequisites of the entries below: nftables core, hidraw, uinput and the ALSA sequencer +CONFIG_NF_TABLES=m +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NF_TABLES_NETDEV=y +CONFIG_NFT_CT=m +CONFIG_NFT_LOG=m +CONFIG_NFT_LIMIT=m +CONFIG_NFT_MASQ=m +CONFIG_NFT_NAT=m +CONFIG_NFT_REJECT=m +CONFIG_NFT_REJECT_INET=m +CONFIG_NFT_FIB_INET=m +CONFIG_NFT_FIB_IPV4=m +CONFIG_NFT_FIB_IPV6=m +CONFIG_HIDRAW=y +CONFIG_INPUT_UINPUT=m +CONFIG_SND_SEQUENCER=m +CONFIG_SND_HRTIMER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_SEQ_HRTIMER_DEFAULT=y + +# HID / input / game controllers (USB + Bluetooth HID devices, xpad, joydev, USB tablets/touchscreens) +CONFIG_DRAGONRISE_FF=y +CONFIG_GREENASIA_FF=y +CONFIG_HID_A4TECH=m +CONFIG_HID_ACCUTOUCH=m +CONFIG_HID_ACRUX=m +CONFIG_HID_ACRUX_FF=y +CONFIG_HID_ALPS=m +CONFIG_HID_APPLEIR=m +CONFIG_HID_AUREAL=m +CONFIG_HID_BATTERY_STRENGTH=y +CONFIG_HID_BELKIN=m +CONFIG_HID_BETOP_FF=m +CONFIG_HID_BIGBEN_FF=m +CONFIG_HID_CHERRY=m +CONFIG_HID_CHICONY=m +CONFIG_HID_CMEDIA=m +CONFIG_HID_CORSAIR=m +CONFIG_HID_COUGAR=m +CONFIG_HID_CP2112=m +CONFIG_HID_CREATIVE_SB0540=m +CONFIG_HID_CYPRESS=m +CONFIG_HID_DRAGONRISE=m +CONFIG_HID_ELAN=m +CONFIG_HID_ELECOM=m +CONFIG_HID_ELO=m +CONFIG_HID_EMS_FF=m +CONFIG_HID_EVISION=m +CONFIG_HID_EZKEY=m +CONFIG_HID_FT260=m +CONFIG_HID_GEMBIRD=m +CONFIG_HID_GFRM=m +CONFIG_HID_GLORIOUS=m +CONFIG_HID_GOOGLE_STADIA_FF=m +CONFIG_HID_GREENASIA=m +CONFIG_HID_GT683R=m +CONFIG_HID_GYRATION=m +CONFIG_HID_HOLTEK=m +CONFIG_HID_HUAWEI=m +CONFIG_HID_ICADE=m +CONFIG_HID_ITE=m +CONFIG_HID_JABRA=m +CONFIG_HID_KENSINGTON=m +CONFIG_HID_KEYTOUCH=m +CONFIG_HID_KYE=m +CONFIG_HID_KYSONA=m +CONFIG_HID_LCPOWER=m +CONFIG_HID_LED=m +CONFIG_HID_LENOVO=m +CONFIG_HID_LENOVO_GO=m +CONFIG_HID_LENOVO_GO_S=m +CONFIG_HID_LETSKETCH=m +CONFIG_HID_LOGITECH=m +CONFIG_HID_LOGITECH_DJ=m +CONFIG_HID_LOGITECH_HIDPP=m +CONFIG_HID_MACALLY=m +CONFIG_HID_MALTRON=m +CONFIG_HID_MAYFLASH=m +CONFIG_HID_MCP2221=m +CONFIG_HID_MEGAWORLD_FF=m +CONFIG_HID_MICROSOFT=m +CONFIG_HID_MONTEREY=m +CONFIG_HID_NINTENDO=m +CONFIG_HID_NTI=m +CONFIG_HID_NTRIG=m +CONFIG_HID_NVIDIA_SHIELD=m +CONFIG_HID_ORTEK=m +CONFIG_HID_PANTHERLORD=m +CONFIG_HID_PENMOUNT=m +CONFIG_HID_PETALYNX=m +CONFIG_HID_PICOLCD=m +CONFIG_HID_PICOLCD_BACKLIGHT=y +CONFIG_HID_PICOLCD_FB=y +CONFIG_HID_PICOLCD_LEDS=y +CONFIG_HID_PID=y +CONFIG_HID_PLANTRONICS=m +CONFIG_HID_PLAYSTATION=m +CONFIG_HID_PRIMAX=m +CONFIG_HID_PRODIKEYS=m +CONFIG_HID_PXRC=m +CONFIG_HID_RAPOO=m +CONFIG_HID_RAZER=m +CONFIG_HID_RETRODE=m +CONFIG_HID_RMI=m +CONFIG_HID_ROCCAT=m +CONFIG_HID_SAITEK=m +CONFIG_HID_SAMSUNG=m +CONFIG_HID_SEMITEK=m +CONFIG_HID_SIGMAMICRO=m +CONFIG_HID_SMARTJOYPLUS=m +CONFIG_HID_SONY=m +CONFIG_HID_SPEEDLINK=m +CONFIG_HID_STEAM=m +CONFIG_HID_STEELSERIES=m +CONFIG_HID_SUNPLUS=m +CONFIG_HID_THINGM=m +CONFIG_HID_THRUSTMASTER=m +CONFIG_HID_TIVO=m +CONFIG_HID_TOPRE=m +CONFIG_HID_TOPSEED=m +CONFIG_HID_TWINHAN=m +CONFIG_HID_U2FZERO=m +CONFIG_HID_UCLOGIC=m +CONFIG_HID_UDRAW_PS3=m +CONFIG_HID_UNIVERSAL_PIDFF=m +CONFIG_HID_VIEWSONIC=m +CONFIG_HID_VIVALDI=m +CONFIG_HID_WACOM=m +CONFIG_HID_WALTOP=m +CONFIG_HID_WIIMOTE=m +CONFIG_HID_WINWING=m +CONFIG_HID_XIAOMI=m +CONFIG_HID_XINMO=m +CONFIG_HID_ZEROPLUS=m +CONFIG_HID_ZYDACRON=m +CONFIG_HOLTEK_FF=y +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_JOYSTICK=y +CONFIG_INPUT_TABLET=y +CONFIG_JOYSTICK_IFORCE=m +CONFIG_JOYSTICK_IFORCE_USB=m +CONFIG_JOYSTICK_PXRC=m +CONFIG_JOYSTICK_XPAD=m +CONFIG_JOYSTICK_XPAD_FF=y +CONFIG_JOYSTICK_XPAD_LEDS=y +CONFIG_LOGIG940_FF=y +CONFIG_LOGIRUMBLEPAD2_FF=y +CONFIG_LOGITECH_FF=y +CONFIG_LOGIWHEELS_FF=y +CONFIG_MOUSE_SYNAPTICS_USB=m +CONFIG_NINTENDO_FF=y +CONFIG_NVIDIA_SHIELD_FF=y +CONFIG_PANTHERLORD_FF=y +CONFIG_PLAYSTATION_FF=y +CONFIG_RMI4_CORE=m +CONFIG_RMI4_F03=y +CONFIG_RMI4_F11=y +CONFIG_RMI4_F12=y +CONFIG_RMI4_F1A=y +CONFIG_RMI4_F21=y +CONFIG_RMI4_F30=y +CONFIG_RMI4_F34=y +CONFIG_RMI4_F3A=y +CONFIG_RMI4_F55=y +CONFIG_SMARTJOYPLUS_FF=y +CONFIG_SONY_FF=y +CONFIG_STEAM_FF=y +CONFIG_TABLET_USB_ACECAD=m +CONFIG_TABLET_USB_AIPTEK=m +CONFIG_TABLET_USB_HANWANG=m +CONFIG_TABLET_USB_KBTAB=m +CONFIG_TABLET_USB_PEGASUS=m +CONFIG_THRUSTMASTER_FF=y +CONFIG_TOUCHSCREEN_USB_3M=y +CONFIG_TOUCHSCREEN_USB_COMPOSITE=m +CONFIG_TOUCHSCREEN_USB_DMC_TSC10=y +CONFIG_TOUCHSCREEN_USB_E2I=y +CONFIG_TOUCHSCREEN_USB_EASYTOUCH=y +CONFIG_TOUCHSCREEN_USB_EGALAX=y +CONFIG_TOUCHSCREEN_USB_ELO=y +CONFIG_TOUCHSCREEN_USB_ETT_TC45USB=y +CONFIG_TOUCHSCREEN_USB_ETURBO=y +CONFIG_TOUCHSCREEN_USB_GENERAL_TOUCH=y +CONFIG_TOUCHSCREEN_USB_GOTOP=y +CONFIG_TOUCHSCREEN_USB_GUNZE=y +CONFIG_TOUCHSCREEN_USB_IDEALTEK=y +CONFIG_TOUCHSCREEN_USB_IRTOUCH=y +CONFIG_TOUCHSCREEN_USB_ITM=y +CONFIG_TOUCHSCREEN_USB_JASTEC=y +CONFIG_TOUCHSCREEN_USB_NEXIO=y +CONFIG_TOUCHSCREEN_USB_PANJIT=y +CONFIG_TOUCHSCREEN_USB_ZYTRONIC=y +CONFIG_USB_HIDDEV=y +CONFIG_ZEROPLUS_FF=y + +# USB device classes: serial adapters, storage sub-drivers, printers/TMC/WDM, misc USB gadgets, usbmon, USB/IP, card readers, maker adapters +CONFIG_APPLE_MFI_FASTCHARGE=m +CONFIG_DLN2_ADC=m +CONFIG_GNSS_USB=m +CONFIG_GPIB=m +CONFIG_GPIB_AGILENT_82357A=m +CONFIG_GPIB_COMMON=m +CONFIG_GPIB_LPVO=m +CONFIG_GPIB_NI_USB=m +CONFIG_GPIO_DLN2=m +CONFIG_GPIO_MPSSE=m +CONFIG_I2C_CP2615=m +CONFIG_I2C_DIOLAN_U2C=m +CONFIG_I2C_DLN2=m +CONFIG_I2C_TINY_USB=m +CONFIG_MEMSTICK=m +CONFIG_MEMSTICK_REALTEK_USB=m +CONFIG_MFD_DLN2=m +CONFIG_MISC_RTSX_USB=m +CONFIG_MMC_REALTEK_USB=m +CONFIG_MMC_USHC=m +CONFIG_MMC_VUB300=m +CONFIG_MSPRO_BLOCK=m +CONFIG_REALTEK_AUTOPM=y +CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m +CONFIG_SENSORS_ASUS_ROG_RYUJIN=m +CONFIG_SENSORS_CORSAIR_CPRO=m +CONFIG_SENSORS_CORSAIR_PSU=m +CONFIG_SENSORS_GIGABYTE_WATERFORCE=m +CONFIG_SENSORS_NZXT_KRAKEN2=m +CONFIG_SENSORS_NZXT_KRAKEN3=m +CONFIG_SENSORS_NZXT_SMART2=m +CONFIG_SENSORS_POWERZ=m +CONFIG_SPI_CH341=m +CONFIG_SPI_DLN2=m +CONFIG_USBIP_CORE=m +CONFIG_USBIP_HOST=m +CONFIG_USBIP_VHCI_HCD=m +CONFIG_USBPCWATCHDOG=m +CONFIG_USB_ADUTUX=m +CONFIG_USB_APPLEDISPLAY=m +CONFIG_USB_CHAOSKEY=m +CONFIG_USB_EMI26=m +CONFIG_USB_EMI62=m +CONFIG_USB_EZUSB_FX2=m +CONFIG_USB_IDMOUSE=m +CONFIG_USB_IOWARRIOR=m +CONFIG_USB_ISIGHTFW=m +CONFIG_USB_LCD=m +CONFIG_USB_LD=m +CONFIG_USB_LEGOTOWER=m +CONFIG_USB_MDC800=m +CONFIG_USB_MICROTEK=m +CONFIG_USB_MON=m +CONFIG_USB_PRINTER=m +CONFIG_USB_SERIAL_AIRCABLE=m +CONFIG_USB_SERIAL_ARK3116=m +CONFIG_USB_SERIAL_BELKIN=m +CONFIG_USB_SERIAL_CH341=m +CONFIG_USB_SERIAL_CYBERJACK=m +CONFIG_USB_SERIAL_CYPRESS_M8=m +CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m +CONFIG_USB_SERIAL_EDGEPORT=m +CONFIG_USB_SERIAL_EDGEPORT_TI=m +CONFIG_USB_SERIAL_EMPEG=m +CONFIG_USB_SERIAL_F81232=m +CONFIG_USB_SERIAL_F8153X=m +CONFIG_USB_SERIAL_GARMIN=m +CONFIG_USB_SERIAL_GENERIC=y +CONFIG_USB_SERIAL_IPAQ=m +CONFIG_USB_SERIAL_IPW=m +CONFIG_USB_SERIAL_IR=m +CONFIG_USB_SERIAL_IUU=m +CONFIG_USB_SERIAL_KEYSPAN=m +CONFIG_USB_SERIAL_KEYSPAN_PDA=m +CONFIG_USB_SERIAL_KLSI=m +CONFIG_USB_SERIAL_KOBIL_SCT=m +CONFIG_USB_SERIAL_MCT_U232=m +CONFIG_USB_SERIAL_MOS7720=m +CONFIG_USB_SERIAL_MOS7840=m +CONFIG_USB_SERIAL_NAVMAN=m +CONFIG_USB_SERIAL_OMNINET=m +CONFIG_USB_SERIAL_OPTICON=m +CONFIG_USB_SERIAL_OTI6858=m +CONFIG_USB_SERIAL_PL2303=m +CONFIG_USB_SERIAL_QCAUX=m +CONFIG_USB_SERIAL_QT2=m +CONFIG_USB_SERIAL_QUALCOMM=m +CONFIG_USB_SERIAL_SAFE=m +CONFIG_USB_SERIAL_SAFE_PADDED=y +CONFIG_USB_SERIAL_SIERRAWIRELESS=m +CONFIG_USB_SERIAL_SIMPLE=m +CONFIG_USB_SERIAL_SPCP8X5=m +CONFIG_USB_SERIAL_SSU100=m +CONFIG_USB_SERIAL_SYMBOL=m +CONFIG_USB_SERIAL_TI=m +CONFIG_USB_SERIAL_UPD78F0730=m +CONFIG_USB_SERIAL_VISOR=m +CONFIG_USB_SERIAL_WHITEHEAT=m +CONFIG_USB_SERIAL_XR=m +CONFIG_USB_SERIAL_XSENS_MT=m +CONFIG_USB_SEVSEG=m +CONFIG_USB_SISUSBVGA=m +CONFIG_USB_STORAGE_ALAUDA=m +CONFIG_USB_STORAGE_CYPRESS_ATACB=m +CONFIG_USB_STORAGE_DATAFAB=m +CONFIG_USB_STORAGE_ENE_UB6250=m +CONFIG_USB_STORAGE_FREECOM=m +CONFIG_USB_STORAGE_ISD200=m +CONFIG_USB_STORAGE_JUMPSHOT=m +CONFIG_USB_STORAGE_KARMA=m +CONFIG_USB_STORAGE_ONETOUCH=m +CONFIG_USB_STORAGE_REALTEK=m +CONFIG_USB_STORAGE_SDDR09=m +CONFIG_USB_STORAGE_SDDR55=m +CONFIG_USB_STORAGE_USBAT=m +CONFIG_USB_TMC=m +CONFIG_USB_TRANCEVIBRATOR=m +CONFIG_USB_WDM=m +CONFIG_USB_YUREX=m +CONFIG_W1=m +CONFIG_W1_MASTER_DS2490=m +CONFIG_W1_SLAVE_DS2405=m +CONFIG_W1_SLAVE_DS2406=m +CONFIG_W1_SLAVE_DS2408=m +CONFIG_W1_SLAVE_DS2413=m +CONFIG_W1_SLAVE_DS2423=m +CONFIG_W1_SLAVE_DS2430=m +CONFIG_W1_SLAVE_DS2431=m +CONFIG_W1_SLAVE_DS2433=m +CONFIG_W1_SLAVE_DS2433_CRC=y +CONFIG_W1_SLAVE_DS2438=m +CONFIG_W1_SLAVE_DS2780=m +CONFIG_W1_SLAVE_DS2781=m +CONFIG_W1_SLAVE_DS2805=m +CONFIG_W1_SLAVE_DS28E04=m +CONFIG_W1_SLAVE_SMEM=m +CONFIG_W1_SLAVE_THERM=m +CONFIG_XILLYUSB=m + +# USB network adapters: USB Ethernet/tethering/mobile broadband, USB Wi-Fi dongles, USB CAN adapters +CONFIG_AR5523=m +CONFIG_ATH10K_USB=m +CONFIG_ATH6KL=m +CONFIG_ATH6KL_USB=m +CONFIG_ATH9K_HTC=m +CONFIG_BRCMFMAC_USB=y +CONFIG_CAN_8DEV_USB=m +CONFIG_CAN_EMS_USB=m +CONFIG_CAN_ESD_USB=m +CONFIG_CAN_F81604=m +CONFIG_CAN_GS_USB=m +CONFIG_CAN_ISOTP=m +CONFIG_CAN_J1939=m +CONFIG_CAN_KVASER_USB=m +CONFIG_CAN_MCBA_USB=m +CONFIG_CAN_PEAK_USB=m +CONFIG_CARL9170=m +CONFIG_CARL9170_LEDS=y +CONFIG_LIBERTAS_THINFIRM=m +CONFIG_LIBERTAS_THINFIRM_USB=m +CONFIG_MT7601U=m +CONFIG_MT7663U=m +CONFIG_MT76x0U=m +CONFIG_MT76x2U=m +CONFIG_MT7921U=m +CONFIG_MT7925U=m +CONFIG_RT2500USB=m +CONFIG_RT2800USB=m +CONFIG_RT2800USB_RT33XX=y +CONFIG_RT2800USB_RT3573=y +CONFIG_RT2800USB_RT35XX=y +CONFIG_RT2800USB_RT53XX=y +CONFIG_RT2800USB_RT55XX=y +CONFIG_RT2800USB_UNKNOWN=y +CONFIG_RT2X00=m +CONFIG_RT73USB=m +CONFIG_RTL8187=m +CONFIG_RTL8192DU=m +CONFIG_RTL8XXXU=m +CONFIG_RTW88=m +CONFIG_RTW88_8723DU=m +CONFIG_RTW88_8812AU=m +CONFIG_RTW88_8814AU=m +CONFIG_RTW88_8821AU=m +CONFIG_RTW88_8821CU=m +CONFIG_RTW88_8822BU=m +CONFIG_RTW88_8822CU=m +CONFIG_RTW89=m +CONFIG_RTW89_8851BU=m +CONFIG_RTW89_8852AU=m +CONFIG_RTW89_8852BU=m +CONFIG_RTW89_8852CU=m +CONFIG_USB_ALI_M5632=y +CONFIG_USB_AN2720=y +CONFIG_USB_CATC=m +CONFIG_USB_EPSON2888=y +CONFIG_USB_HSO=m +CONFIG_USB_IPHETH=m +CONFIG_USB_KAWETH=m +CONFIG_USB_KC2190=y +CONFIG_USB_NET_AQC111=m +CONFIG_USB_NET_CDC_EEM=m +CONFIG_USB_NET_CDC_MBIM=m +CONFIG_USB_NET_CH9200=m +CONFIG_USB_NET_CX82310_ETH=m +CONFIG_USB_NET_GL620A=m +CONFIG_USB_NET_HUAWEI_CDC_NCM=m +CONFIG_USB_NET_INT51X1=m +CONFIG_USB_NET_KALMIA=m +CONFIG_USB_NET_QMI_WWAN=m +CONFIG_USB_NET_RNDIS_HOST=m +CONFIG_USB_NET_SR9700=m +CONFIG_USB_SIERRA_NET=m +CONFIG_USB_VL600=m +CONFIG_ZD1211RW=m + +# Storage: optical/SCSI generic, device-mapper (LUKS, LVM, verity, integrity), MD RAID, partition tables +CONFIG_AIX_PARTITION=y +CONFIG_BLK_DEV_SR=m +CONFIG_BSD_DISKLABEL=y +CONFIG_CHR_DEV_SG=m +CONFIG_DM_CACHE=m +CONFIG_DM_CACHE_SMQ=m +CONFIG_DM_CRYPT=m +CONFIG_DM_INTEGRITY=m +CONFIG_DM_RAID=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_THIN_PROVISIONING=m +CONFIG_DM_UEVENT=y +CONFIG_DM_VERITY=m +CONFIG_DM_VERITY_FEC=y +CONFIG_DM_WRITECACHE=m +CONFIG_LDM_PARTITION=y +CONFIG_MAC_PARTITION=y +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_RAID456=m +CONFIG_MINIX_SUBPARTITION=y +CONFIG_OSF_PARTITION=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_SGI_PARTITION=y +CONFIG_SOLARIS_X86_PARTITION=y +CONFIG_SUN_PARTITION=y +CONFIG_UNIXWARE_DISKLABEL=y + +# Filesystems for removable media and images, binfmt_misc, SMB client +CONFIG_BINFMT_MISC=m +CONFIG_CIFS=m +CONFIG_CIFS_ALLOW_INSECURE_LEGACY=y +CONFIG_CIFS_DFS_UPCALL=y +CONFIG_CIFS_POSIX=y +CONFIG_CIFS_SWN_UPCALL=y +CONFIG_CIFS_UPCALL=y +CONFIG_CIFS_XATTR=y +CONFIG_EROFS_FS=m +CONFIG_EROFS_FS_BACKED_BY_FILE=y +CONFIG_EROFS_FS_POSIX_ACL=y +CONFIG_EROFS_FS_SECURITY=y +CONFIG_EROFS_FS_XATTR=y +CONFIG_EROFS_FS_ZIP=y +CONFIG_EROFS_FS_ZIP_ACCEL=y +CONFIG_EROFS_FS_ZIP_DEFLATE=y +CONFIG_EROFS_FS_ZIP_LZMA=y +CONFIG_EROFS_FS_ZIP_ZSTD=y +CONFIG_EXFAT_FS=m +CONFIG_F2FS_FS=m +CONFIG_F2FS_FS_COMPRESSION=y +CONFIG_F2FS_FS_LZ4=y +CONFIG_F2FS_FS_LZ4HC=y +CONFIG_F2FS_FS_LZO=y +CONFIG_F2FS_FS_LZORLE=y +CONFIG_F2FS_FS_POSIX_ACL=y +CONFIG_F2FS_FS_SECURITY=y +CONFIG_F2FS_FS_XATTR=y +CONFIG_F2FS_FS_ZSTD=y +CONFIG_F2FS_IOSTAT=y +CONFIG_F2FS_STAT_FS=y +CONFIG_F2FS_UNFAIR_RWSEM=y +CONFIG_HFSPLUS_FS=m +CONFIG_HFS_FS=m +CONFIG_ISO9660_FS=m +CONFIG_JOLIET=y +CONFIG_MSDOS_FS=m +CONFIG_NLS_ASCII=m +CONFIG_NLS_CODEPAGE_1250=m +CONFIG_NLS_CODEPAGE_1251=m +CONFIG_NLS_CODEPAGE_737=m +CONFIG_NLS_CODEPAGE_775=m +CONFIG_NLS_CODEPAGE_850=m +CONFIG_NLS_CODEPAGE_852=m +CONFIG_NLS_CODEPAGE_855=m +CONFIG_NLS_CODEPAGE_857=m +CONFIG_NLS_CODEPAGE_860=m +CONFIG_NLS_CODEPAGE_861=m +CONFIG_NLS_CODEPAGE_862=m +CONFIG_NLS_CODEPAGE_863=m +CONFIG_NLS_CODEPAGE_864=m +CONFIG_NLS_CODEPAGE_865=m +CONFIG_NLS_CODEPAGE_866=m +CONFIG_NLS_CODEPAGE_869=m +CONFIG_NLS_CODEPAGE_874=m +CONFIG_NLS_CODEPAGE_932=m +CONFIG_NLS_CODEPAGE_936=m +CONFIG_NLS_CODEPAGE_949=m +CONFIG_NLS_CODEPAGE_950=m +CONFIG_NLS_ISO8859_13=m +CONFIG_NLS_ISO8859_14=m +CONFIG_NLS_ISO8859_15=m +CONFIG_NLS_ISO8859_2=m +CONFIG_NLS_ISO8859_3=m +CONFIG_NLS_ISO8859_4=m +CONFIG_NLS_ISO8859_5=m +CONFIG_NLS_ISO8859_6=m +CONFIG_NLS_ISO8859_7=m +CONFIG_NLS_ISO8859_8=m +CONFIG_NLS_ISO8859_9=m +CONFIG_NLS_KOI8_R=m +CONFIG_NLS_KOI8_U=m +CONFIG_NLS_MAC_CELTIC=m +CONFIG_NLS_MAC_CENTEURO=m +CONFIG_NLS_MAC_CROATIAN=m +CONFIG_NLS_MAC_CYRILLIC=m +CONFIG_NLS_MAC_GAELIC=m +CONFIG_NLS_MAC_GREEK=m +CONFIG_NLS_MAC_ICELAND=m +CONFIG_NLS_MAC_INUIT=m +CONFIG_NLS_MAC_ROMAN=m +CONFIG_NLS_MAC_ROMANIAN=m +CONFIG_NLS_MAC_TURKISH=m +CONFIG_NLS_UTF8=m +CONFIG_NTFS3_FS=m +CONFIG_NTFS3_FS_POSIX_ACL=y +CONFIG_NTFS3_LZX_XPRESS=y +CONFIG_SQUASHFS_LZ4=y +CONFIG_SQUASHFS_LZO=y +CONFIG_SQUASHFS_XATTR=y +CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y +CONFIG_UDF_FS=m +CONFIG_XFS_FS=m +CONFIG_XFS_ONLINE_REPAIR=y +CONFIG_XFS_ONLINE_SCRUB=y +CONFIG_XFS_ONLINE_SCRUB_STATS=y +CONFIG_XFS_POSIX_ACL=y +CONFIG_XFS_QUOTA=y +CONFIG_XFS_RT=y +CONFIG_XFS_SUPPORT_ASCII_CI=y +CONFIG_XFS_SUPPORT_V4=y +CONFIG_ZISOFS=y + +# Network protocols: WireGuard/OpenVPN-DCO/L2TP/PPP/IPsec/kTLS, tunnels, policy routing, nftables+xtables+ipset+ebtables, tc, TCP CC, ss diag, virtual links +CONFIG_BONDING=m +CONFIG_BRIDGE_EBT_802_3=m +CONFIG_BRIDGE_EBT_AMONG=m +CONFIG_BRIDGE_EBT_ARP=m +CONFIG_BRIDGE_EBT_ARPREPLY=m +CONFIG_BRIDGE_EBT_BROUTE=m +CONFIG_BRIDGE_EBT_DNAT=m +CONFIG_BRIDGE_EBT_IP=m +CONFIG_BRIDGE_EBT_IP6=m +CONFIG_BRIDGE_EBT_LIMIT=m +CONFIG_BRIDGE_EBT_LOG=m +CONFIG_BRIDGE_EBT_MARK=m +CONFIG_BRIDGE_EBT_MARK_T=m +CONFIG_BRIDGE_EBT_NFLOG=m +CONFIG_BRIDGE_EBT_PKTTYPE=m +CONFIG_BRIDGE_EBT_REDIRECT=m +CONFIG_BRIDGE_EBT_SNAT=m +CONFIG_BRIDGE_EBT_STP=m +CONFIG_BRIDGE_EBT_T_FILTER=m +CONFIG_BRIDGE_EBT_T_NAT=m +CONFIG_BRIDGE_EBT_VLAN=m +CONFIG_BRIDGE_NF_EBTABLES=m +CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m +CONFIG_CLS_U32_MARK=y +CONFIG_CLS_U32_PERF=y +CONFIG_DEFAULT_CUBIC=y +CONFIG_DUMMY=m +CONFIG_GACT_PROB=y +CONFIG_GENEVE=m +CONFIG_IFB=m +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_ESPINTCP=y +CONFIG_INET6_ESP_OFFLOAD=m +CONFIG_INET6_IPCOMP=m +CONFIG_INET_AH=m +CONFIG_INET_DIAG_DESTROY=y +CONFIG_INET_ESP=m +CONFIG_INET_ESPINTCP=y +CONFIG_INET_ESP_OFFLOAD=m +CONFIG_INET_IPCOMP=m +CONFIG_INET_RAW_DIAG=m +CONFIG_INET_UDP_DIAG=m +CONFIG_IP6_NF_FILTER=m +CONFIG_IP6_NF_IPTABLES_LEGACY=m +CONFIG_IP6_NF_MANGLE=m +CONFIG_IP6_NF_MATCH_AH=m +CONFIG_IP6_NF_MATCH_EUI64=m +CONFIG_IP6_NF_MATCH_FRAG=m +CONFIG_IP6_NF_MATCH_HL=m +CONFIG_IP6_NF_MATCH_IPV6HEADER=m +CONFIG_IP6_NF_MATCH_MH=m +CONFIG_IP6_NF_MATCH_OPTS=m +CONFIG_IP6_NF_MATCH_RPFILTER=m +CONFIG_IP6_NF_MATCH_RT=m +CONFIG_IP6_NF_MATCH_SRH=m +CONFIG_IP6_NF_NAT=m +CONFIG_IP6_NF_RAW=m +CONFIG_IP6_NF_TARGET_HL=m +CONFIG_IP6_NF_TARGET_MASQUERADE=m +CONFIG_IP6_NF_TARGET_NPT=m +CONFIG_IP6_NF_TARGET_REJECT=m +CONFIG_IPV6_GRE=m +CONFIG_IPV6_MULTIPLE_TABLES=y +CONFIG_IPV6_TUNNEL=m +CONFIG_IPV6_VTI=m +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IP_NF_ARPFILTER=m +CONFIG_IP_NF_ARPTABLES=m +CONFIG_IP_NF_ARP_MANGLE=m +CONFIG_IP_NF_FILTER=m +CONFIG_IP_NF_IPTABLES_LEGACY=m +CONFIG_IP_NF_MANGLE=m +CONFIG_IP_NF_MATCH_AH=m +CONFIG_IP_NF_MATCH_ECN=m +CONFIG_IP_NF_MATCH_RPFILTER=m +CONFIG_IP_NF_MATCH_TTL=m +CONFIG_IP_NF_NAT=m +CONFIG_IP_NF_RAW=m +CONFIG_IP_NF_TARGET_ECN=m +CONFIG_IP_NF_TARGET_MASQUERADE=m +CONFIG_IP_NF_TARGET_NETMAP=m +CONFIG_IP_NF_TARGET_REDIRECT=m +CONFIG_IP_NF_TARGET_REJECT=m +CONFIG_IP_NF_TARGET_TTL=m +CONFIG_IP_ROUTE_MULTIPATH=y +CONFIG_IP_SET=m +CONFIG_IP_SET_BITMAP_IP=m +CONFIG_IP_SET_BITMAP_IPMAC=m +CONFIG_IP_SET_BITMAP_PORT=m +CONFIG_IP_SET_HASH_IP=m +CONFIG_IP_SET_HASH_IPMAC=m +CONFIG_IP_SET_HASH_IPMARK=m +CONFIG_IP_SET_HASH_IPPORT=m +CONFIG_IP_SET_HASH_IPPORTIP=m +CONFIG_IP_SET_HASH_IPPORTNET=m +CONFIG_IP_SET_HASH_MAC=m +CONFIG_IP_SET_HASH_NET=m +CONFIG_IP_SET_HASH_NETIFACE=m +CONFIG_IP_SET_HASH_NETNET=m +CONFIG_IP_SET_HASH_NETPORT=m +CONFIG_IP_SET_HASH_NETPORTNET=m +CONFIG_IP_SET_LIST_SET=m +CONFIG_L2TP=m +CONFIG_L2TP_ETH=m +CONFIG_L2TP_IP=m +CONFIG_L2TP_V3=y +CONFIG_NETFILTER_NETLINK_ACCT=m +CONFIG_NETFILTER_NETLINK_GLUE_CT=y +CONFIG_NETFILTER_NETLINK_HOOK=m +CONFIG_NETFILTER_NETLINK_LOG=m +CONFIG_NETFILTER_NETLINK_OSF=m +CONFIG_NETFILTER_NETLINK_QUEUE=m +CONFIG_NETFILTER_XTABLES_LEGACY=y +CONFIG_NETFILTER_XT_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_BPF=m +CONFIG_NETFILTER_XT_MATCH_CGROUP=m +CONFIG_NETFILTER_XT_MATCH_CLUSTER=m +CONFIG_NETFILTER_XT_MATCH_COMMENT=m +CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m +CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m +CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m +CONFIG_NETFILTER_XT_MATCH_CONNMARK=m +CONFIG_NETFILTER_XT_MATCH_CPU=m +CONFIG_NETFILTER_XT_MATCH_DCCP=m +CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m +CONFIG_NETFILTER_XT_MATCH_DSCP=m +CONFIG_NETFILTER_XT_MATCH_ECN=m +CONFIG_NETFILTER_XT_MATCH_ESP=m +CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m +CONFIG_NETFILTER_XT_MATCH_HELPER=m +CONFIG_NETFILTER_XT_MATCH_HL=m +CONFIG_NETFILTER_XT_MATCH_IPCOMP=m +CONFIG_NETFILTER_XT_MATCH_IPRANGE=m +CONFIG_NETFILTER_XT_MATCH_L2TP=m +CONFIG_NETFILTER_XT_MATCH_LENGTH=m +CONFIG_NETFILTER_XT_MATCH_LIMIT=m +CONFIG_NETFILTER_XT_MATCH_MAC=m +CONFIG_NETFILTER_XT_MATCH_MARK=m +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m +CONFIG_NETFILTER_XT_MATCH_NFACCT=m +CONFIG_NETFILTER_XT_MATCH_OSF=m +CONFIG_NETFILTER_XT_MATCH_OWNER=m +CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m +CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m +CONFIG_NETFILTER_XT_MATCH_POLICY=m +CONFIG_NETFILTER_XT_MATCH_QUOTA=m +CONFIG_NETFILTER_XT_MATCH_RATEEST=m +CONFIG_NETFILTER_XT_MATCH_REALM=m +CONFIG_NETFILTER_XT_MATCH_RECENT=m +CONFIG_NETFILTER_XT_MATCH_SCTP=m +CONFIG_NETFILTER_XT_MATCH_SOCKET=m +CONFIG_NETFILTER_XT_MATCH_STATE=m +CONFIG_NETFILTER_XT_MATCH_STATISTIC=m +CONFIG_NETFILTER_XT_MATCH_STRING=m +CONFIG_NETFILTER_XT_MATCH_TCPMSS=m +CONFIG_NETFILTER_XT_MATCH_TIME=m +CONFIG_NETFILTER_XT_MATCH_U32=m +CONFIG_NETFILTER_XT_NAT=m +CONFIG_NETFILTER_XT_SET=m +CONFIG_NETFILTER_XT_TARGET_AUDIT=m +CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m +CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m +CONFIG_NETFILTER_XT_TARGET_CONNMARK=m +CONFIG_NETFILTER_XT_TARGET_CT=m +CONFIG_NETFILTER_XT_TARGET_DSCP=m +CONFIG_NETFILTER_XT_TARGET_HL=m +CONFIG_NETFILTER_XT_TARGET_HMARK=m +CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m +CONFIG_NETFILTER_XT_TARGET_LED=m +CONFIG_NETFILTER_XT_TARGET_MARK=m +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m +CONFIG_NETFILTER_XT_TARGET_NETMAP=m +CONFIG_NETFILTER_XT_TARGET_NFLOG=m +CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m +CONFIG_NETFILTER_XT_TARGET_NOTRACK=m +CONFIG_NETFILTER_XT_TARGET_RATEEST=m +CONFIG_NETFILTER_XT_TARGET_REDIRECT=m +CONFIG_NETFILTER_XT_TARGET_TCPMSS=m +CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m +CONFIG_NETFILTER_XT_TARGET_TEE=m +CONFIG_NETFILTER_XT_TARGET_TPROXY=m +CONFIG_NETFILTER_XT_TARGET_TRACE=m +CONFIG_NETLINK_DIAG=m +CONFIG_NET_ACT_BPF=m +CONFIG_NET_ACT_CONNMARK=m +CONFIG_NET_ACT_CSUM=m +CONFIG_NET_ACT_CT=m +CONFIG_NET_ACT_CTINFO=m +CONFIG_NET_ACT_MPLS=m +CONFIG_NET_ACT_NAT=m +CONFIG_NET_ACT_PEDIT=m +CONFIG_NET_ACT_POLICE=m +CONFIG_NET_ACT_SAMPLE=m +CONFIG_NET_ACT_SIMP=m +CONFIG_NET_ACT_SKBEDIT=m +CONFIG_NET_ACT_SKBMOD=m +CONFIG_NET_ACT_TUNNEL_KEY=m +CONFIG_NET_ACT_VLAN=m +CONFIG_NET_CLS_BPF=m +CONFIG_NET_CLS_FLOW=m +CONFIG_NET_CLS_FW=m +CONFIG_NET_CLS_MATCHALL=m +CONFIG_NET_CLS_ROUTE4=m +CONFIG_NET_CLS_U32=m +CONFIG_NET_EMATCH=y +CONFIG_NET_EMATCH_CMP=m +CONFIG_NET_EMATCH_IPSET=m +CONFIG_NET_EMATCH_IPT=m +CONFIG_NET_EMATCH_META=m +CONFIG_NET_EMATCH_NBYTE=m +CONFIG_NET_EMATCH_TEXT=m +CONFIG_NET_EMATCH_U32=m +CONFIG_NET_FOU=m +CONFIG_NET_FOU_IP_TUNNELS=y +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NET_IPIP=m +CONFIG_NET_IPVTI=m +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_NET_SCH_CAKE=m +CONFIG_NET_SCH_CHOKE=m +CONFIG_NET_SCH_CODEL=m +CONFIG_NET_SCH_DRR=m +CONFIG_NET_SCH_DUALPI2=m +CONFIG_NET_SCH_ETS=m +CONFIG_NET_SCH_FQ=m +CONFIG_NET_SCH_FQ_CODEL=m +CONFIG_NET_SCH_FQ_PIE=m +CONFIG_NET_SCH_GRED=m +CONFIG_NET_SCH_HFSC=m +CONFIG_NET_SCH_HHF=m +CONFIG_NET_SCH_HTB=m +CONFIG_NET_SCH_MULTIQ=m +CONFIG_NET_SCH_NETEM=m +CONFIG_NET_SCH_PIE=m +CONFIG_NET_SCH_PLUG=m +CONFIG_NET_SCH_PRIO=m +CONFIG_NET_SCH_QFQ=m +CONFIG_NET_SCH_RED=m +CONFIG_NET_SCH_SFB=m +CONFIG_NET_SCH_SFQ=m +CONFIG_NET_SCH_TBF=m +CONFIG_NET_SCH_TEQL=m +CONFIG_NET_TEAM=m +CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m +CONFIG_NET_TEAM_MODE_BROADCAST=m +CONFIG_NET_TEAM_MODE_LOADBALANCE=m +CONFIG_NET_TEAM_MODE_RANDOM=m +CONFIG_NET_TEAM_MODE_ROUNDROBIN=m +CONFIG_NFT_BRIDGE_META=m +CONFIG_NFT_BRIDGE_REJECT=m +CONFIG_NFT_COMPAT=m +CONFIG_NFT_CONNLIMIT=m +CONFIG_NFT_DUP_IPV4=m +CONFIG_NFT_DUP_IPV6=m +CONFIG_NFT_DUP_NETDEV=m +CONFIG_NFT_FIB_NETDEV=m +CONFIG_NFT_FLOW_OFFLOAD=m +CONFIG_NFT_FWD_NETDEV=m +CONFIG_NFT_HASH=m +CONFIG_NFT_NUMGEN=m +CONFIG_NFT_OSF=m +CONFIG_NFT_QUEUE=m +CONFIG_NFT_QUOTA=m +CONFIG_NFT_REDIR=m +CONFIG_NFT_REJECT_NETDEV=m +CONFIG_NFT_SOCKET=m +CONFIG_NFT_TPROXY=m +CONFIG_NFT_TUNNEL=m +CONFIG_NFT_XFRM=m +CONFIG_NF_CONNTRACK_AMANDA=m +CONFIG_NF_CONNTRACK_BRIDGE=m +CONFIG_NF_CONNTRACK_FTP=m +CONFIG_NF_CONNTRACK_H323=m +CONFIG_NF_CONNTRACK_IRC=m +CONFIG_NF_CONNTRACK_LABELS=y +CONFIG_NF_CONNTRACK_MARK=y +CONFIG_NF_CONNTRACK_NETBIOS_NS=m +CONFIG_NF_CONNTRACK_PPTP=m +CONFIG_NF_CONNTRACK_PROCFS=y +CONFIG_NF_CONNTRACK_SANE=m +CONFIG_NF_CONNTRACK_SIP=m +CONFIG_NF_CONNTRACK_SNMP=m +CONFIG_NF_CONNTRACK_TFTP=m +CONFIG_NF_CONNTRACK_TIMEOUT=y +CONFIG_NF_CONNTRACK_TIMESTAMP=y +CONFIG_NF_CONNTRACK_ZONES=y +CONFIG_NF_CT_NETLINK=m +CONFIG_NF_CT_NETLINK_HELPER=m +CONFIG_NF_CT_NETLINK_TIMEOUT=m +CONFIG_NF_DUP_IPV4=m +CONFIG_NF_DUP_IPV6=m +CONFIG_NF_DUP_NETDEV=m +CONFIG_NF_FLOW_TABLE=m +CONFIG_NF_FLOW_TABLE_INET=m +CONFIG_NF_FLOW_TABLE_PROCFS=y +CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m +CONFIG_NF_NAT_SNMP_BASIC=m +CONFIG_NF_SOCKET_IPV4=m +CONFIG_NF_SOCKET_IPV6=m +CONFIG_NF_TABLES_ARP=y +CONFIG_NF_TABLES_BRIDGE=m +CONFIG_NF_TPROXY_IPV4=m +CONFIG_NF_TPROXY_IPV6=m +CONFIG_OVPN=m +CONFIG_PACKET_DIAG=m +CONFIG_PPP=m +CONFIG_PPPOE=m +CONFIG_PPPOE_HASH_BITS_4=y +CONFIG_PPPOL2TP=m +CONFIG_PPP_ASYNC=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_FILTER=y +CONFIG_PPP_MPPE=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPP_SYNC_TTY=m +CONFIG_PPTP=m +CONFIG_PSAMPLE=m +CONFIG_TCP_CONG_ADVANCED=y +CONFIG_TCP_CONG_BBR=m +CONFIG_TCP_CONG_BIC=m +CONFIG_TCP_CONG_CDG=m +CONFIG_TCP_CONG_DCTCP=m +CONFIG_TCP_CONG_HSTCP=m +CONFIG_TCP_CONG_HTCP=m +CONFIG_TCP_CONG_HYBLA=m +CONFIG_TCP_CONG_ILLINOIS=m +CONFIG_TCP_CONG_LP=m +CONFIG_TCP_CONG_NV=m +CONFIG_TCP_CONG_SCALABLE=m +CONFIG_TCP_CONG_VEGAS=m +CONFIG_TCP_CONG_VENO=m +CONFIG_TCP_CONG_WESTWOOD=m +CONFIG_TCP_CONG_YEAH=m +CONFIG_TLS=m +CONFIG_UNIX_DIAG=m +CONFIG_VXLAN=m +CONFIG_WIREGUARD=m +CONFIG_XFRM_INTERFACE=m +CONFIG_XFRM_MIGRATE=y +CONFIG_XFRM_STATISTICS=y +CONFIG_XFRM_USER=m + +# Bluetooth: USB dongle drivers, vhci, MSFT/AOSP vendor extensions (USB controllers only) +CONFIG_BT_AOSPEXT=y +CONFIG_BT_ATH3K=m +CONFIG_BT_HCIBCM203X=m +CONFIG_BT_HCIBFUSB=m +CONFIG_BT_HCIBPA10X=m +CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y +CONFIG_BT_HCIVHCI=m +CONFIG_BT_MSFTEXT=y + +# Containers / virtualisation: CRIU/checkpoint, userfaultfd, misc cgroup, KVM host vhost-net/vhost-vsock/vsock +CONFIG_CGROUP_MISC=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_USERFAULTFD=y +CONFIG_VHOST_NET=m +CONFIG_VHOST_VSOCK=m +CONFIG_VSOCKETS=m +CONFIG_VSOCKETS_DIAG=m +CONFIG_VSOCKETS_LOOPBACK=m + +# Crypto: AF_ALG user API (cryptsetup/iwd), dm-crypt/fscrypt modes, VeraCrypt/TrueCrypt ciphers+hashes, IPsec AEADs +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_AEGIS128_SIMD=y +CONFIG_CRYPTO_BLAKE2B=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_CRC32=m +CONFIG_CRYPTO_CRC32C=m +CONFIG_CRYPTO_CRYPTD=m +CONFIG_CRYPTO_CTS=m +CONFIG_CRYPTO_ESSIV=m +CONFIG_CRYPTO_HCTR2=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_LZ4HC=m +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_NULL=m +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SEQIV=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_STREEBOG=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_USER_API_AEAD=m +CONFIG_CRYPTO_USER_API_HASH=m +CONFIG_CRYPTO_USER_API_SKCIPHER=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_XXHASH=m + +# Sound: USB audio/MIDI device drivers, MIDI 2.0, virtual MIDI, dummy card, userspace timers +CONFIG_SND_BCD2000=m +CONFIG_SND_DUMMY=m +CONFIG_SND_SEQ_UMP=y +CONFIG_SND_UMP_LEGACY_RAWMIDI=y +CONFIG_SND_USB_6FIRE=m +CONFIG_SND_USB_AUDIO_MIDI_V2=y +CONFIG_SND_USB_CAIAQ=m +CONFIG_SND_USB_CAIAQ_INPUT=y +CONFIG_SND_USB_HIFACE=m +CONFIG_SND_USB_POD=m +CONFIG_SND_USB_PODHD=m +CONFIG_SND_USB_TONEPORT=m +CONFIG_SND_USB_UA101=m +CONFIG_SND_USB_VARIAX=m +CONFIG_SND_UTIMER=y +CONFIG_SND_VIRMIDI=m + +# Video: non-UVC USB webcams (gspca family, Philips pwc), USB displays (DisplayLink USB2 udl, GUD, GM12U320) +CONFIG_DRM_GM12U320=m +CONFIG_DRM_GUD=m +CONFIG_DRM_UDL=m +CONFIG_USB_GL860=m +CONFIG_USB_GSPCA=m +CONFIG_USB_GSPCA_BENQ=m +CONFIG_USB_GSPCA_CONEX=m +CONFIG_USB_GSPCA_CPIA1=m +CONFIG_USB_GSPCA_DTCS033=m +CONFIG_USB_GSPCA_ETOMS=m +CONFIG_USB_GSPCA_FINEPIX=m +CONFIG_USB_GSPCA_JEILINJ=m +CONFIG_USB_GSPCA_JL2005BCD=m +CONFIG_USB_GSPCA_KINECT=m +CONFIG_USB_GSPCA_KONICA=m +CONFIG_USB_GSPCA_MARS=m +CONFIG_USB_GSPCA_MR97310A=m +CONFIG_USB_GSPCA_NW80X=m +CONFIG_USB_GSPCA_OV519=m +CONFIG_USB_GSPCA_OV534=m +CONFIG_USB_GSPCA_OV534_9=m +CONFIG_USB_GSPCA_PAC207=m +CONFIG_USB_GSPCA_PAC7302=m +CONFIG_USB_GSPCA_PAC7311=m +CONFIG_USB_GSPCA_SE401=m +CONFIG_USB_GSPCA_SN9C2028=m +CONFIG_USB_GSPCA_SN9C20X=m +CONFIG_USB_GSPCA_SONIXB=m +CONFIG_USB_GSPCA_SONIXJ=m +CONFIG_USB_GSPCA_SPCA1528=m +CONFIG_USB_GSPCA_SPCA500=m +CONFIG_USB_GSPCA_SPCA501=m +CONFIG_USB_GSPCA_SPCA505=m +CONFIG_USB_GSPCA_SPCA506=m +CONFIG_USB_GSPCA_SPCA508=m +CONFIG_USB_GSPCA_SPCA561=m +CONFIG_USB_GSPCA_SQ905=m +CONFIG_USB_GSPCA_SQ905C=m +CONFIG_USB_GSPCA_SQ930X=m +CONFIG_USB_GSPCA_STK014=m +CONFIG_USB_GSPCA_STK1135=m +CONFIG_USB_GSPCA_STV0680=m +CONFIG_USB_GSPCA_SUNPLUS=m +CONFIG_USB_GSPCA_T613=m +CONFIG_USB_GSPCA_TOPRO=m +CONFIG_USB_GSPCA_TOUPTEK=m +CONFIG_USB_GSPCA_TV8532=m +CONFIG_USB_GSPCA_VC032X=m +CONFIG_USB_GSPCA_VICAM=m +CONFIG_USB_GSPCA_XIRLINK_CIT=m +CONFIG_USB_GSPCA_ZC3XX=m +CONFIG_USB_M5602=m +CONFIG_USB_PWC=m +CONFIG_USB_PWC_INPUT_EVDEV=y +CONFIG_USB_STV06XX=m + +# Other user-facing: Kerberos persistent keyrings, Speakup screen reader, Wine/Proton ntsync +CONFIG_ACCESSIBILITY=y +CONFIG_NTSYNC=m +CONFIG_PERSISTENT_KEYRINGS=y +CONFIG_SPEAKUP=m +CONFIG_SPEAKUP_SYNTH_ACNTSA=m +CONFIG_SPEAKUP_SYNTH_APOLLO=m +CONFIG_SPEAKUP_SYNTH_AUDPTR=m +CONFIG_SPEAKUP_SYNTH_BNS=m +CONFIG_SPEAKUP_SYNTH_DECTLK=m +CONFIG_SPEAKUP_SYNTH_LTLK=m +CONFIG_SPEAKUP_SYNTH_SOFT=m +CONFIG_SPEAKUP_SYNTH_SPKOUT=m +CONFIG_SPEAKUP_SYNTH_TXPRT=m From 7c9e2ce2228b73eed0a95931b03f8019d7ebe595 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Sun, 11 Oct 2026 20:31:59 +1100 Subject: [PATCH 337/631] wifi: mt7932: keep RX0 status in the RAM-config acknowledgment Since "acknowledge interrupts before servicing the rings", mt_service() clears the interrupt causes it is about to service, RX0 included, before it drains the rings, and restores the interrupt mask after the last ring. The RAM-config acknowledgment in mt_rx_drain() still clears RX0's status bit and rewrites the mask in the middle of the drain. The status write can clear the bit of an RX0 completion that arrived after the up-front acknowledgment, so the device no longer shows work that is still pending, and the unmask lets the interrupt fire again while the rings are still being serviced, with the fixed IRQ_MASK rather than m->irq_mask. Clear only the acknowledgment's other two status bits there, and leave the mask to mt_service(). The diagnostic drain on a command timeout runs with the interrupt line disabled; a bit it leaves set is serviced once the line is enabled again. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index b96f8d50f64d81..fbaa8e8fd1fc32 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -251,12 +251,17 @@ static void mt_rx_drain(struct mt7932 *m, struct mt7932_ring *q) if (READ_ONCE(m->ram_config) && length >= 33 && length <= MT7932_RX_CAPACITY) { u8 *packet = q->buffers + q->tail * RX_STRIDE; - /* Source-specific RAM-config acknowledgment before RX0 refill. */ + /* + * Source-specific RAM-config acknowledgment before RX0 + * refill. mt_service() has already acknowledged this + * pass's causes, RX0 (bit 0) included, and restores the + * mask once every ring is drained. Leave bit 0 alone, so + * that an RX0 completion arriving now keeps its status, + * and leave the mask alone while the rings are serviced. + */ if (q == &m->rx[0] && packet[28] == 1 && packet[29] == m->waiting && !packet[30] && packet[32] == 1) { - mt_write(m, W + 0x204, 0); - mt_write(m, W + 0x200, 0x0c000001); - mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + mt_write(m, W + 0x200, 0x0c000000); mt_write(m, W + 0x1f4, 0xffff); } } From c9e9b022ddd3cd5b05cab14dbd8056f4e63a69aa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:46:20 -0400 Subject: [PATCH 338/631] mm: preserve clean subpage lazyfree state on retained reads When a retained reference prevents discarding a clean lazyfree slot, try_to_unmap_subpage() restores the PTE but then marks the native folio swap-backed. The extra reference alone does not cancel MADV_FREE, so this can prevent reclaim without swap and cause unnecessary swap traffic. Leave the classification unchanged after restoring a clean PTE. Dirty entries still cancel lazyfree and follow the existing swap-backed path. Extend the lazyfree KUnit case to retain a reader after whole-folio advice, check the clean classification, then release the reader and reclaim. The expanded test passes on 16K and 64K KASAN guests without warnings. Signed-off-by: Diljit Singh (cherry picked from commit e9a25aae4e7509bc40554dac0a23240d77363110) --- arch/arm64/mm/user4k-test.c | 18 ++++++++++++++---- mm/rmap.c | 12 +++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 4e5b2befe18f5d..f9dcaf932557f5 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9380,7 +9380,7 @@ static void user4k_lazyfree_test(struct kunit *test) return; } /* Partial, redirty, retained GET, typed write, all leaves, migration. */ - for (mode = 0; mode < 10; mode++) { + for (mode = 0; mode < 11; mode++) { struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); struct mm_struct *mm; struct user_page_fragment held = {}; @@ -9401,7 +9401,8 @@ static void user4k_lazyfree_test(struct kunit *test) } kthread_use_mm(mm); ret = test_madvise(TEST_VA, - (mode == 4 || mode == 9) ? PAGE_SIZE : SZ_4K, MADV_FREE); + (mode == 4 || mode == 9 || mode == 10) ? PAGE_SIZE : SZ_4K, + MADV_FREE); kthread_unuse_mm(mm); KUNIT_ASSERT_EQ(test, ret, 0L); pte = test_fault_entry(test, mm, TEST_VA); @@ -9413,7 +9414,7 @@ static void user4k_lazyfree_test(struct kunit *test) if (mode == 1) KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); - if (mode == 2 || mode == 3 || mode == 8) { + if (mode == 2 || mode == 3 || mode == 8 || mode == 10) { ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, mode == 3 ? FOLL_WRITE : 0, &held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); @@ -9444,11 +9445,20 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, folio); folio_lock(folio); try_to_unmap(folio, TTU_SYNC); + if (mode == 10) { + /* A temporary reader leaves clean whole-folio advice intact. */ + KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); + KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); + KUNIT_EXPECT_FALSE(test, pte_none(test_fault_entry(test, mm, TEST_VA))); + release_user_fragments(&held, 1, false); + try_to_unmap(folio, TTU_SYNC); + } folio_unlock(folio); folio_put(folio); for (i = 0; i < 4; i++) { pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); - KUNIT_EXPECT_EQ(test, pte_none(pte), (mode == 4 || mode == 9) || + KUNIT_EXPECT_EQ(test, pte_none(pte), + (mode == 4 || mode == 9 || mode == 10) || (i == 0 && (mode == 0 || mode == 5))); } if (mode == 2 || mode == 6 || mode == 7) diff --git a/mm/rmap.c b/mm/rmap.c index 795b568b9f7c5a..bf11cc857cb45b 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2176,16 +2176,14 @@ static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma goto out; } set_pte_at(mm, address, pvmw->pte, pte); - /* Retained references must not turn a droppable folio into swap data. */ - if (vma->vm_flags & VM_DROPPABLE) + /* A retained reference alone does not cancel clean lazyfree data. */ + if ((vma->vm_flags & VM_DROPPABLE) || !pte_dirty(pte)) goto out; - if (pte_dirty(pte)) { - mm_subpage_cancel_lazyfree(slot); - folio_mark_dirty(folio); - } + mm_subpage_cancel_lazyfree(slot); + folio_mark_dirty(folio); } if (!folio_test_swapbacked(folio)) { - /* A write or retained reference cancelled whole-folio lazy reclaim. */ + /* A write cancelled whole-folio lazy reclaim. */ folio_set_swapbacked(folio); goto out; } From 3cb93fe475d0d0a3c314db6c851359168584fab6 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:56:16 -0400 Subject: [PATCH 339/631] arm64: preserve dependencies in mixed-page test selectors The IOMMUFD harness selects IOMMU_PT even when IOMMU_SUPPORT is disabled, excluding the page allocation implementation its selected code requires. The boolean VFIO harness selectors can also force VFIO built-in while IOMMUFD is modular, bypassing VFIO's own tristate constraint. Require IOMMU_SUPPORT for the IOMMUFD harness. Require non-modular IOMMUFD for both VFIO selectors, including the ioctl selector that selects the accounting harness. Built-in and disabled IOMMUFD configurations remain available for VFIO testing. Six before/after configuration checks cover the invalid combinations and the retained valid selections. Integrated 16K/64K KASAN images build. Signed-off-by: Diljit Singh (cherry picked from commit cf3e9884efb7a5cd06cb06d7ca632bd1ceef22d4) --- arch/arm64/Kconfig.debug | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 58d7a1dc98405b..52fced2ab8355d 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -65,6 +65,7 @@ config ARM64_USER4K_IOMMUFD_TEST bool "IOMMUFD ioctl harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && DEBUG_KERNEL depends on FAULT_INJECTION && RUNTIME_TESTING_MENU + depends on IOMMU_SUPPORT select GENERIC_PT select IOMMU_PT select IOMMU_PT_AMDV1 @@ -80,6 +81,7 @@ config ARM64_USER4K_IOMMUFD_TEST config ARM64_USER4K_VFIO_ACCOUNT_TEST bool "VFIO type1 pinning harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y + depends on IOMMUFD != m select VFIO select VFIO_GROUP select VFIO_CONTAINER @@ -96,6 +98,7 @@ config ARM64_USER4K_VFIO_IOCTL_TEST bool "Private VFIO type1 ioctl harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL depends on RUNTIME_TESTING_MENU + depends on IOMMUFD != m select ARM64_USER4K_VFIO_ACCOUNT_TEST default n help From 812510d5ba11fbe189eb2c310a6b67207a3ba3e2 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 04:56:16 -0400 Subject: [PATCH 340/631] arm64/mm: avoid table metadata lookup at the exclusive PTE endpoint contpte_try_unfold_partial() checks its exclusive endpoint with contpte_align_down(). That helper derives granule geometry from the PTE's backing metadata, but ptep + nr can already lie beyond that backing. Use the mm's contiguous-group size to test alignment at both ends. This preserves partial-group unfolding while avoiding a metadata lookup for a pointer that need not identify an allocated page-table entry. 16K/64K KASAN images build and their existing native/4K contiguous-PTE functional tests pass without warnings. The endpoint defect is established by source inspection; no runtime reproduction is claimed. Signed-off-by: Diljit Singh (cherry picked from commit f6e776e813d9dec5da55a0b2eb779eeca145c471) --- arch/arm64/mm/contpte.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index f8faf1a99eeca7..e78e0d4265218b 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -68,15 +68,18 @@ static inline pte_t *contpte_align_addr_ptep(struct mm_struct *mm, unsigned long static void contpte_try_unfold_partial(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned int nr) { + unsigned int cont_bytes = sizeof(*ptep) * cont_ptes_mm(mm); + /* * Unfold any partially covered contpte block at the beginning and end * of the range. */ - if (ptep != contpte_align_down(ptep) || nr < cont_ptes_mm(mm)) + if (!IS_ALIGNED((unsigned long)ptep, cont_bytes) || nr < cont_ptes_mm(mm)) contpte_try_unfold(mm, addr, ptep, __ptep_get(ptep)); - if (ptep + nr != contpte_align_down(ptep + nr)) { + /* The exclusive end can lie beyond the allocated page-table backing. */ + if (!IS_ALIGNED((unsigned long)(ptep + nr), cont_bytes)) { unsigned long last_addr = addr + mm_page_size(mm) * (nr - 1); pte_t *last_ptep = ptep + nr - 1; From 5b9c5b67cda4102f2b33c2d78c134619829e9d89 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:09:14 -0400 Subject: [PATCH 341/631] perf: sample page sizes using the target mm geometry perf_get_pgtable_size() starts with an mm-aware PGD lookup but continues with native table offsets and leaf sizes. A smaller userspace granule can therefore select an unrelated entry or report the native page size. Add mm-aware lockless offset and leaf-size helpers with native fallbacks. Use the target mm throughout perf's walk, including runtime folding and contiguous mappings, while preserving the existing interrupt exclusion and descriptor-snapshot semantics. Expose the walker only for KUnit testing and cover ordinary mapped and unmapped addresses across table levels. Extend contiguous-PTE fixtures to check perf's reported size and cover 16K userspace on a 64K kernel. The tests pass in 16K/64K KASAN guests for native, 4K and supported 16K userspace granules. An x86_64 perf-object build checks the native fallbacks. Hardware PMU event delivery was not tested. Signed-off-by: Diljit Singh (cherry picked from commit 4538d92c925d9adc7af502c9df8aec52c56df322) --- arch/arm64/include/asm/user-pgtable.h | 23 +++++++ arch/arm64/mm/user4k-test.c | 91 +++++++++++++++++++++++++-- include/linux/perf_event.h | 5 ++ include/linux/pgtable.h | 29 +++++++++ kernel/events/core.c | 24 +++---- 5 files changed, 156 insertions(+), 16 deletions(-) diff --git a/arch/arm64/include/asm/user-pgtable.h b/arch/arm64/include/asm/user-pgtable.h index b244f2387a7b47..7b1d1a94515d00 100644 --- a/arch/arm64/include/asm/user-pgtable.h +++ b/arch/arm64/include/asm/user-pgtable.h @@ -113,6 +113,17 @@ static inline pmd_t *pmd_offset_mm(struct mm_struct *mm, pud_t *pud, #define pmd_offset_mm pmd_offset_mm +/* Use the caller's descriptor snapshots throughout a lockless walk. */ +#define p4d_offset_lockless_mm(mm, pgdp, pgd, addr) \ + (arm64_mm_alt_granule(mm) ? p4d_offset_mm(mm, &(pgd), addr) : \ + p4d_offset_lockless(pgdp, pgd, addr)) +#define pud_offset_lockless_mm(mm, p4dp, p4d, addr) \ + (arm64_mm_alt_granule(mm) ? pud_offset_mm(mm, &(p4d), addr) : \ + pud_offset_lockless(p4dp, p4d, addr)) +#define pmd_offset_lockless_mm(mm, pudp, pud, addr) \ + (arm64_mm_alt_granule(mm) ? pmd_offset_mm(mm, &(pud), addr) : \ + pmd_offset_lockless(pudp, pud, addr)) + static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, unsigned long addr) { @@ -133,6 +144,18 @@ static inline pte_t *pte_offset_kernel_mm(struct mm_struct *mm, pmd_t *pmd, #define pud_size_mm(mm) (1UL << pud_shift_mm(mm)) #define p4d_size_mm(mm) (1UL << p4d_shift_mm(mm)) #define pgd_size_mm(mm) (arm64_mm_alt_granule(mm) ? (1UL << arm64_user_level_shift(mm, 0)) : PGDIR_SIZE) +#define pgd_leaf_size_mm(mm, pgd) \ + (arm64_mm_alt_granule(mm) ? pgd_size_mm(mm) : pgd_leaf_size(pgd)) +#define p4d_leaf_size_mm(mm, p4d) \ + (arm64_mm_alt_granule(mm) ? p4d_size_mm(mm) : p4d_leaf_size(p4d)) +#define pud_leaf_size_mm(mm, pud) \ + (arm64_mm_alt_granule(mm) ? pud_size_mm(mm) : pud_leaf_size(pud)) +#define pmd_leaf_size_mm(mm, pmd) \ + (arm64_mm_alt_granule(mm) ? pmd_size_mm(mm) * \ + (pmd_cont(pmd) ? (mm_page_shift(mm) == 12 ? 16 : 32) : 1) : pmd_leaf_size(pmd)) +#define __pte_leaf_size_mm(mm, pmd, pte) \ + (arm64_mm_alt_granule(mm) ? \ + (pte_cont(pte) ? cont_pte_size_mm(mm) : mm_page_size(mm)) : __pte_leaf_size(pmd, pte)) #define pte_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PTE * sizeof(pte_t)) #define pmd_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PMD * sizeof(pmd_t)) #define pud_table_bytes_mm(mm) (arm64_mm_alt_granule(mm) ? mm_page_size(mm) : PTRS_PER_PUD * sizeof(pud_t)) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index f9dcaf932557f5..69456ba03bac5e 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1324,6 +1325,58 @@ static int test_map_alias(struct test_mm *ctx, unsigned int index, return 0; } +#ifdef CONFIG_PERF_EVENTS +static void test_perf_page_size(struct kunit *test, unsigned int shift) +{ + const unsigned long addresses[] = { + SZ_1M, SZ_1G + SZ_1M, (1UL << 39) + SZ_1M, + }; + struct test_mm *ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); + void *data = test_alloc_granule(test, PAGE_SHIFT); + unsigned long flags; + u64 present, absent; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, ctx); + KUNIT_ASSERT_NOT_NULL(test, data); + ctx->test = test; + ctx->mm = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, ctx->mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_mm_destroy, ctx), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(ctx->mm, shift), 0); + for (i = 0; i < ARRAY_SIZE(addresses); i++) + KUNIT_ASSERT_EQ(test, test_map_alias(ctx, i, addresses[i], virt_to_phys(data)), 0); + for (i = 0; i < ARRAY_SIZE(addresses); i++) { + local_irq_save(flags); + present = perf_get_pgtable_size(ctx->mm, addresses[i]); + absent = perf_get_pgtable_size(ctx->mm, addresses[i] + (1UL << shift)); + local_irq_restore(flags); + KUNIT_EXPECT_EQ(test, present, 1ULL << shift); + KUNIT_EXPECT_EQ(test, absent, 0ULL); + } +} + +static void user4k_perf_page_size_test(struct kunit *test) +{ + test_perf_page_size(test, 12); +} + +static void user16k_perf_page_size_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_perf_page_size(test, 14); +} + +static void native_perf_page_size_test(struct kunit *test) +{ + test_perf_page_size(test, PAGE_SHIFT); +} +#endif + /* Physical-range lookup only; coarse anonymous ownership is a separate step. */ static void user64k_rmap_lookup_test(struct kunit *test) { @@ -7569,7 +7622,7 @@ static void contpte_fixture_free(void *arg) folio_put(f->folio); } -static void test_contpte_granule(struct kunit *test, bool small) +static void test_contpte_granule(struct kunit *test, unsigned int shift) { struct contpte_fixture *f = kunit_kzalloc(test, sizeof(*f), GFP_KERNEL); struct vm_area_struct vma = {}; @@ -7585,13 +7638,17 @@ static void test_contpte_granule(struct kunit *test, bool small) spinlock_t *ptl; unsigned int i; int access_ret, access_same_ret, access_noop_ret; +#ifdef CONFIG_PERF_EVENTS + unsigned long irqflags; + u64 reported_size; +#endif KUNIT_ASSERT_NOT_NULL(test, f); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, contpte_fixture_free, f), 0); f->mm = mm_alloc(); KUNIT_ASSERT_NOT_NULL(test, f->mm); - if (small) - KUNIT_ASSERT_EQ(test, test_mm_select4k(f->mm), 0); + if (shift != PAGE_SHIFT) + KUNIT_ASSERT_EQ(test, test_mm_select_granule(f->mm, shift), 0); f->nr = cont_ptes_mm(f->mm); size = mm_page_size(f->mm); f->folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, @@ -7621,6 +7678,11 @@ static void test_contpte_granule(struct kunit *test, bool small) ok.addresses &= pte_phys_mm(f->mm, observed) == phys + i * size; ok.hints &= pte_batch_hint(pte + i, observed) == f->nr - i; } +#ifdef CONFIG_PERF_EVENTS + local_irq_save(irqflags); + reported_size = perf_get_pgtable_size(f->mm, TEST_VA + size); + local_irq_restore(irqflags); +#endif __set_pte(pte + 3, pte_mkdirty(__ptep_get(pte + 3))); observed = ptep_get(pte + 1); ok.gathered &= pte_dirty(observed) && pte_young(observed); @@ -7693,16 +7755,28 @@ static void test_contpte_granule(struct kunit *test, bool small) KUNIT_EXPECT_TRUE(test, ok.holes); KUNIT_EXPECT_TRUE(test, ok.full_clear); KUNIT_EXPECT_TRUE(test, ok.sentinel); +#ifdef CONFIG_PERF_EVENTS + KUNIT_EXPECT_EQ(test, reported_size, (u64)cont_pte_size_mm(f->mm)); +#endif } static void user4k_contpte_test(struct kunit *test) { - test_contpte_granule(test, true); + test_contpte_granule(test, 12); +} + +static void user16k_contpte_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "requires native backing larger than 16K"); + return; + } + test_contpte_granule(test, 14); } static void native_contpte_test(struct kunit *test) { - test_contpte_granule(test, false); + test_contpte_granule(test, PAGE_SHIFT); } #endif @@ -14121,7 +14195,13 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_anon_remote_zap_test), #ifdef CONFIG_ARM64_CONTPTE KUNIT_CASE(user4k_contpte_test), + KUNIT_CASE(user16k_contpte_test), KUNIT_CASE(native_contpte_test), +#endif +#ifdef CONFIG_PERF_EVENTS + KUNIT_CASE(user4k_perf_page_size_test), + KUNIT_CASE(user16k_perf_page_size_test), + KUNIT_CASE(native_perf_page_size_test), #endif KUNIT_CASE(user4k_mm_tables_test), KUNIT_CASE(user16k_mm_tables_test), @@ -14155,3 +14235,4 @@ kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap #endif MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 48d851fbd8ea58..14a21f187e9f4d 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1209,6 +1209,11 @@ perf_cgroup_from_task(struct task_struct *task, struct perf_event_context *ctx) extern struct perf_event_context *perf_cpu_task_ctx(void); +#if IS_ENABLED(CONFIG_KUNIT) +/* The caller must disable IRQs, as for the sampling path. */ +u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr); +#endif + extern void *perf_aux_output_begin(struct perf_output_handle *handle, struct perf_event *event); extern void perf_aux_output_end(struct perf_output_handle *handle, diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index f5ef44f25c52b9..5c67d9b5d2f8df 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -2385,6 +2385,19 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) #define pmd_offset_lockless(pudp, pud, address) pmd_offset(&(pud), address) #endif +#ifndef p4d_offset_lockless_mm +#define p4d_offset_lockless_mm(mm, pgdp, pgd, address) \ + p4d_offset_lockless(pgdp, pgd, address) +#endif +#ifndef pud_offset_lockless_mm +#define pud_offset_lockless_mm(mm, p4dp, p4d, address) \ + pud_offset_lockless(p4dp, p4d, address) +#endif +#ifndef pmd_offset_lockless_mm +#define pmd_offset_lockless_mm(mm, pudp, pud, address) \ + pmd_offset_lockless(pudp, pud, address) +#endif + /* * pXd_leaf() is the API to check whether a pgtable entry is a huge page * mapping. It should work globally across all archs, without any @@ -2438,6 +2451,22 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) #define __pte_leaf_size(x,y) pte_leaf_size(y) #endif +#ifndef pgd_leaf_size_mm +#define pgd_leaf_size_mm(mm, pgd) pgd_leaf_size(pgd) +#endif +#ifndef p4d_leaf_size_mm +#define p4d_leaf_size_mm(mm, p4d) p4d_leaf_size(p4d) +#endif +#ifndef pud_leaf_size_mm +#define pud_leaf_size_mm(mm, pud) pud_leaf_size(pud) +#endif +#ifndef pmd_leaf_size_mm +#define pmd_leaf_size_mm(mm, pmd) pmd_leaf_size(pmd) +#endif +#ifndef __pte_leaf_size_mm +#define __pte_leaf_size_mm(mm, pmd, pte) __pte_leaf_size(pmd, pte) +#endif + /* * We always define pmd_pfn for all archs as it's used in lots of generic * code. Now it happens too for pud_pfn (and can happen for larger diff --git a/kernel/events/core.c b/kernel/events/core.c index 343dc1af5a3085..4b1b0f8b15cf29 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8,6 +8,7 @@ * Copyright © 2009 Paul Mackerras, IBM Corp. */ +#include #include #include #include @@ -8491,7 +8492,7 @@ static u64 perf_virt_to_phys(u64 virt) /* * Return the pagetable size of a given virtual address. */ -static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) +VISIBLE_IF_KUNIT u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) { u64 size = 0; @@ -8508,45 +8509,46 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) return 0; if (pgd_leaf(pgd)) - return pgd_leaf_size(pgd); + return pgd_leaf_size_mm(mm, pgd); - p4dp = p4d_offset_lockless(pgdp, pgd, addr); + p4dp = p4d_offset_lockless_mm(mm, pgdp, pgd, addr); p4d = p4dp_get(p4dp); - if (!p4d_present(p4d)) + if (!p4d_present_mm(mm, p4d)) return 0; if (p4d_leaf(p4d)) - return p4d_leaf_size(p4d); + return p4d_leaf_size_mm(mm, p4d); - pudp = pud_offset_lockless(p4dp, p4d, addr); + pudp = pud_offset_lockless_mm(mm, p4dp, p4d, addr); pud = pudp_get(pudp); if (!pud_present(pud)) return 0; if (pud_leaf(pud)) - return pud_leaf_size(pud); + return pud_leaf_size_mm(mm, pud); - pmdp = pmd_offset_lockless(pudp, pud, addr); + pmdp = pmd_offset_lockless_mm(mm, pudp, pud, addr); again: pmd = pmdp_get_lockless(pmdp); if (!pmd_present(pmd)) return 0; if (pmd_leaf(pmd)) - return pmd_leaf_size(pmd); + return pmd_leaf_size_mm(mm, pmd); - ptep = pte_offset_map(&pmd, addr); + ptep = pte_offset_map_mm(mm, &pmd, addr); if (!ptep) goto again; pte = ptep_get_lockless(ptep); if (pte_present(pte)) - size = __pte_leaf_size(pmd, pte); + size = __pte_leaf_size_mm(mm, pmd, pte); pte_unmap(ptep); #endif /* CONFIG_HAVE_GUP_FAST */ return size; } +EXPORT_SYMBOL_IF_KUNIT(perf_get_pgtable_size); static u64 perf_get_page_size(unsigned long addr) { From 51825f76615b7beb7bc4a9e191c5a364452154f5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:23:12 -0400 Subject: [PATCH 342/631] uprobes: preserve architecture metadata when comparing subpage ranges The smaller-page orig_page_is_identical() path compares bytes directly, bypassing the architecture checks used by pages_identical(). On arm64, byte equality does not make distinct tagged ranges interchangeable. Add memcmp_pages_range() with a generic mapped-byte implementation and an arm64 implementation retaining the existing MTE policy. Distinct tagged ranges remain non-identical, including different offsets within one page. Keep full-page comparison as the zero-offset full-length case, and use the range API for the smaller-page uprobe comparison. Add direct tests for offset ranges, bytes outside the compared range, unequal contents, empty ranges and tagged-range identity. These and the existing ownership tests pass in 16K/64K KASAN guests. The generic util and uprobe objects also compile for x86. Actual uprobe removal with private tag state was not runtime reproduced. Signed-off-by: Diljit Singh (cherry picked from commit 0c0dccf36f0a795fb0e52aa74ca1eb51c75f659e) --- arch/arm64/kernel/mte.c | 21 ++++++---- arch/arm64/mm/user4k-data-test.c | 72 ++++++++++++++++++++++++++++++++ include/linux/mm.h | 2 + kernel/events/uprobes.c | 6 +-- mm/util.c | 11 ++++- 5 files changed, 99 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 6874b16d065767..645f549d9fed90 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -69,21 +69,23 @@ void mte_sync_tags(pte_t pte, unsigned int nr_pages) smp_wmb(); } -int memcmp_pages(struct page *page1, struct page *page2) +int memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len) { char *addr1, *addr2; int ret; - addr1 = page_address(page1); - addr2 = page_address(page2); - ret = memcmp(addr1, addr2, PAGE_SIZE); + addr1 = page_address(page1) + offset1; + addr2 = page_address(page2) + offset2; + ret = memcmp(addr1, addr2, len); - if (!system_supports_mte() || ret) + if (!len || !system_supports_mte() || ret) return ret; /* - * If the page content is identical but at least one of the pages is - * tagged, return non-zero to avoid KSM merging. If only one of the + * If the byte ranges are identical but at least one page is tagged, + * only the same physical range is interchangeable. Avoid merging or + * replacing independently tagged ranges. If only one of the * pages is tagged, __set_ptes() may zero or change the tags of the * other page via mte_sync_tags(). */ @@ -93,6 +95,11 @@ int memcmp_pages(struct page *page1, struct page *page2) return ret; } +int memcmp_pages(struct page *page1, struct page *page2) +{ + return memcmp_pages_range(page1, 0, page2, 0, PAGE_SIZE); +} + static inline void __mte_enable_kernel(const char *mode, unsigned long tcf) { /* Enable MTE Sync Mode for EL1. */ diff --git a/arch/arm64/mm/user4k-data-test.c b/arch/arm64/mm/user4k-data-test.c index d7d550c9e14205..76656736074b1c 100644 --- a/arch/arm64/mm/user4k-data-test.c +++ b/arch/arm64/mm/user4k-data-test.c @@ -1084,6 +1084,76 @@ static void extent_folio_put(void *arg) folio_put(arg); } +static struct folio *compare_folio_alloc(struct kunit *test) +{ + struct folio *folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + + if (folio && kunit_add_action_or_reset(test, extent_folio_put, folio)) + return NULL; + return folio; +} + +static void user_page_range_compare_test(struct kunit *test) +{ + const unsigned int sizes[] = { SZ_4K, SZ_16K, PAGE_SIZE }; + struct folio *a = compare_folio_alloc(test), *b = compare_folio_alloc(test); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + unsigned int len = sizes[i], offset = PAGE_SIZE - len; + char *from = folio_address(a), *to = folio_address(b); + + if (len > PAGE_SIZE) + continue; + memset(from, 0x11, PAGE_SIZE); + memset(to, 0x22, PAGE_SIZE); + memset(from, 0x5a, len); + memset(to + offset, 0x5a, len); + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &b->page, offset, len), 0); + KUNIT_EXPECT_EQ(test, !memcmp_pages(&a->page, &b->page), len == PAGE_SIZE); + to[PAGE_SIZE - 1] ^= 1; + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, offset, len), 0); + } +} + +static void user_page_tagged_range_compare_test(struct kunit *test) +{ +#ifdef CONFIG_ARM64_MTE + struct folio *a, *b; + unsigned int len; + + if (!system_supports_mte()) { + kunit_skip(test, "MTE is not available on this CPU"); + return; + } + a = compare_folio_alloc(test); + b = compare_folio_alloc(test); + KUNIT_ASSERT_NOT_NULL(test, a); + KUNIT_ASSERT_NOT_NULL(test, b); + KUNIT_ASSERT_EQ(test, memcmp_pages(&a->page, &b->page), 0); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&a->page)); + mte_clear_page_tags(folio_address(a)); + set_page_mte_tagged(&a->page); + KUNIT_EXPECT_NE(test, memcmp_pages(&a->page, &b->page), 0); + for (len = SZ_4K; len <= PAGE_SIZE; len *= 4) { + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &a->page, 0, len), 0); + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, 0, len), 0); + if (len < PAGE_SIZE) + KUNIT_EXPECT_NE(test, + memcmp_pages_range(&a->page, 0, &a->page, len, len), 0); + } + KUNIT_EXPECT_EQ(test, memcmp_pages_range(&a->page, 0, &b->page, 0, 0), 0); + KUNIT_ASSERT_TRUE(test, try_page_mte_tagging(&b->page)); + mte_clear_page_tags(folio_address(b)); + set_page_mte_tagged(&b->page); + KUNIT_EXPECT_NE(test, memcmp_pages_range(&a->page, 0, &b->page, 0, SZ_4K), 0); +#else + kunit_skip(test, "MTE is not configured"); +#endif +} + static void user4k_data_copy_extent_test(struct kunit *test) { const unsigned int sizes[] = { SZ_4K, SZ_16K, SZ_64K }; @@ -1798,6 +1868,8 @@ static struct kunit_case user4k_data_cases[] = { #endif KUNIT_CASE(user4k_data_mte_test), KUNIT_CASE(user4k_data_copy_extent_test), + KUNIT_CASE(user_page_range_compare_test), + KUNIT_CASE(user_page_tagged_range_compare_test), KUNIT_CASE(user16k_data_owner_test), KUNIT_CASE(user16k_file_coverage_test), KUNIT_CASE(user16k_data_refset_test), diff --git a/include/linux/mm.h b/include/linux/mm.h index 6c01377914fe8c..9e621134c8fb19 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -5256,6 +5256,8 @@ static inline void setup_nr_node_ids(void) {} #endif extern int memcmp_pages(struct page *page1, struct page *page2); +int memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len); static inline int pages_identical(struct page *page1, struct page *page2) { diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 640991b1c25cb8..0d8dc2ae9fdd57 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -425,12 +425,10 @@ static bool orig_page_is_identical(struct vm_area_struct *vma, folio_test_pmd_mappable(orig_folio); if (mm_page_size(vma->vm_mm) < PAGE_SIZE) { unsigned int offset = vma_page_offset_at(vma, vaddr).offset; - void *a = kmap_local_page(page), *b = kmap_local_page(orig_page); identical = folio_test_uptodate(orig_folio) && - !memcmp(a + page_offset, b + offset, mm_page_size(vma->vm_mm)); - kunmap_local(b); - kunmap_local(a); + !memcmp_pages_range(page, page_offset, orig_page, offset, + mm_page_size(vma->vm_mm)); } else { identical = folio_test_uptodate(orig_folio) && pages_identical(page, orig_page); diff --git a/mm/util.c b/mm/util.c index 1f41dfdd3ad13e..2adb3d08660d68 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1094,19 +1094,26 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen) return res; } -int __weak memcmp_pages(struct page *page1, struct page *page2) +/* Both byte ranges must fit within their respective native pages. */ +int __weak memcmp_pages_range(struct page *page1, unsigned int offset1, + struct page *page2, unsigned int offset2, size_t len) { char *addr1, *addr2; int ret; addr1 = kmap_local_page(page1); addr2 = kmap_local_page(page2); - ret = memcmp(addr1, addr2, PAGE_SIZE); + ret = memcmp(addr1 + offset1, addr2 + offset2, len); kunmap_local(addr2); kunmap_local(addr1); return ret; } +int __weak memcmp_pages(struct page *page1, struct page *page2) +{ + return memcmp_pages_range(page1, 0, page2, 0, PAGE_SIZE); +} + #ifdef CONFIG_PRINTK /** * mem_dump_obj - Print available provenance information From 73ff08747dab00864912d15f01ffd0a74e2bc528 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:44:23 -0400 Subject: [PATCH 343/631] arm64: preserve subpage offsets and lengths in MTE core dumps Size MTE tag segments from dumped bytes rather than a truncated native page count. For alternative userspace geometry, retain the selected fragment and save only its tags, preserving the physical subpage offset. Keep the native page path and share its assembly packing loop through a bounded tag-range helper. Add a disposable-guest core-file fixture checking actual ELF data and tag headers and bytes for split read-only, sparse and excluded mappings. Native and 4K processes pass on a 16K KASAN kernel; native, 4K and 16K processes pass on a 64K KASAN kernel. These are emulated MTE checks, not physical-hardware validation. Signed-off-by: Diljit Singh (cherry picked from commit 886de7154672d656b031d3ee5a1cda03b7f3168f) --- arch/arm64/include/asm/mte.h | 2 + arch/arm64/kernel/elfcore.c | 60 ++++- arch/arm64/lib/mte.S | 20 +- tools/testing/selftests/arm64/user4k/Makefile | 2 + .../selftests/arm64/user4k/mte-core-init.c | 206 ++++++++++++++++++ 5 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 tools/testing/selftests/arm64/user4k/mte-core-init.c diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index 890d49a4398b28..c454f5ea8c3d15 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -25,6 +25,8 @@ unsigned long mte_copy_tags_to_user(void __user *to, void *from, unsigned long n); int mte_save_tags(struct page *page); void mte_save_page_tags(const void *page_addr, void *tag_storage); +/* Address and size must be aligned to 256 bytes; size may be zero. */ +void mte_save_tags_range(const void *addr, void *tag_storage, unsigned long size); void mte_restore_tags(swp_entry_t entry, struct page *page); void mte_restore_page_tags(void *page_addr, const void *tag_storage); void mte_invalidate_tags(int type, pgoff_t offset); diff --git a/arch/arm64/kernel/elfcore.c b/arch/arm64/kernel/elfcore.c index b735f4c2fe5ea8..32ddc36838fe35 100644 --- a/arch/arm64/kernel/elfcore.c +++ b/arch/arm64/kernel/elfcore.c @@ -2,8 +2,10 @@ #include #include +#include #include #include +#include #include #include @@ -17,9 +19,60 @@ static unsigned long mte_vma_tag_dump_size(struct core_vma_metadata *m) { - return (m->dump_size >> PAGE_SHIFT) * MTE_PAGE_TAG_STORAGE; + return (m->dump_size / MTE_GRANULE_SIZE) * MTE_TAG_SIZE / 8; } +#ifdef CONFIG_MM_SUBPAGE +static int mte_dump_fragment_tags(struct coredump_params *cprm, + unsigned long start, unsigned long len) +{ + unsigned long size = mm_user_fragment_size(current->mm); + unsigned long tag_size = (size / MTE_GRANULE_SIZE) * MTE_TAG_SIZE / 8; + unsigned long addr; + void *tags = NULL; + int ret = 1; + + for (addr = start; addr < start + len; addr += size) { + struct user_page_fragment fragment; + struct page *page; + + if (get_dump_fragment(addr, &fragment) != 1) { + dump_skip(cprm, tag_size); + goto next; + } + page = folio_page(fragment.folio, fragment.offset >> PAGE_SHIFT); + if (!page_mte_tagged(page)) { + release_user_fragments(&fragment, 1, false); + dump_skip(cprm, tag_size); + goto next; + } + if (!tags) { + tags = mte_allocate_tag_storage(); + if (!tags) { + release_user_fragments(&fragment, 1, false); + ret = 0; + break; + } + } + mte_save_tags_range(folio_address(fragment.folio) + fragment.offset, tags, size); + release_user_fragments(&fragment, 1, false); + if (!dump_emit(cprm, tags, tag_size)) { + ret = 0; + break; + } +next: + if (fatal_signal_pending(current) || freezing(current)) { + ret = 0; + break; + } + cond_resched(); + } + if (tags) + mte_free_tag_storage(tags); + return ret; +} +#endif + /* Derived from dump_user_range(); start/end must be page-aligned */ static int mte_dump_tag_range(struct coredump_params *cprm, unsigned long start, unsigned long len) @@ -29,6 +82,11 @@ static int mte_dump_tag_range(struct coredump_params *cprm, void *tags = NULL; int locked = 0; +#ifdef CONFIG_MM_SUBPAGE + if (mm_page_size(current->mm) != PAGE_SIZE) + return mte_dump_fragment_tags(cprm, start, len); +#endif + for (addr = start; addr < start + len; addr += PAGE_SIZE) { struct page *page = get_dump_page(addr, &locked); diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index c1ce370068cf44..8402e81b714749 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -183,6 +183,19 @@ SYM_FUNC_END(mte_copy_tags_to_user) * x1 - tag storage, MTE_PAGE_TAG_STORAGE bytes */ SYM_FUNC_START(mte_save_page_tags) + mov x2, #PAGE_SIZE + b mte_save_tags_range +SYM_FUNC_END(mte_save_page_tags) + +/* + * Save tags for a 256-byte-aligned range. + * x0 - start address + * x1 - tag storage, x2 / 32 bytes + * x2 - byte length, a multiple of 256 + */ +SYM_FUNC_START(mte_save_tags_range) + cbz x2, 3f + add x6, x0, x2 multitag_transfer_size x7, x5 1: mov x2, #0 @@ -195,11 +208,12 @@ SYM_FUNC_START(mte_save_page_tags) str x2, [x1], #8 - tst x0, #(PAGE_SIZE - 1) - b.ne 1b + cmp x0, x6 + b.lo 1b +3: ret -SYM_FUNC_END(mte_save_page_tags) +SYM_FUNC_END(mte_save_tags_range) /* * Restore the tags in a page diff --git a/tools/testing/selftests/arm64/user4k/Makefile b/tools/testing/selftests/arm64/user4k/Makefile index 0350be7a579128..889386ca79ffcc 100644 --- a/tools/testing/selftests/arm64/user4k/Makefile +++ b/tools/testing/selftests/arm64/user4k/Makefile @@ -3,6 +3,7 @@ TEST_GEN_PROGS := page-contract proc-args TEST_GEN_PROGS_EXTENDED := iommufd-init coarse-shmem-init bpf-uptr-init vhost-log-init uprobes-init user-events-init uffd-move-init swap-fixture-init exec-fixture-init mixed-process memory-interfaces advice reclaim-advice proc-pages proc-memory core-fixture-init context-stress-init mlock-fixture-init lazyfree-fixture-init thread-fixture-init splice-fixture-init dio-fixture-init uring-fixture-init gem-fixture-init compat-fixture-init granule-proc-init granule-swap-init aslr-fixture-init cpu-granule-init user16k-smoke-init cpu-granule16-init granule-threeway-init compat-mlock-init mglru-granule-init memcg-pressure-init memcg-accounting-init runtime-app-init fex-contract-init exec-policy-init uffd-wp-init shmem-uffd-swap-init compat-uffd-init shmem-hole-init shmem-thp-init shmem-collapse-init shmem-pin-init shmem-race-init shmem-pressure-init TEST_GEN_PROGS_EXTENDED += vfio-fixture-init sysv-shm-init perf-ring-init TEST_GEN_PROGS_EXTENDED += aio-ring-init numa-policy-init numa-balancing-init cow-pool +TEST_GEN_PROGS_EXTENDED += mte-core-init CFLAGS += -Wall -Wextra -O2 -ffixed-x18 TEST_GEN_FILES += compat-probe-4k compat-probe-64k compat-probe-dynamic compat-interp32 compat-aslr-probe compat-mlock-probe TEST_GEN_FILES += fex-x86-contract libfex-abi.so compat-policy-probe compat-uffd-probe compat-vfio-probe @@ -60,6 +61,7 @@ $(OUTPUT)/libfex-abi.so: fex-native-abi.c $(OUTPUT)/exec-policy-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/exec-policy-init: LDLIBS += -pthread +$(OUTPUT)/mte-core-init: LDFLAGS += -static -Wl,-z,max-page-size=65536 $(OUTPUT)/compat-policy-probe: compat-policy-probe.c $(AARCH32_CC) $(AARCH32_FLAGS) -Wl,-z,max-page-size=65536 $< -o $@ diff --git a/tools/testing/selftests/arm64/user4k/mte-core-init.c b/tools/testing/selftests/arm64/user4k/mte-core-init.c new file mode 100644 index 00000000000000..3a6cf5e0723b7b --- /dev/null +++ b/tools/testing/selftests/arm64/user4k/mte-core-init.c @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* MTE core-file validation: PID 1 and its children in a disposable VM only. */ +#define _GNU_SOURCE +#include "../../../../../include/uapi/linux/prctl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PT_AARCH64_MEMTAG_MTE +#define PT_AARCH64_MEMTAG_MTE (PT_LOPROC + 0x2) +#endif +#define TAGGED_BASE 0x200000000UL +#define ZERO_BASE 0x210000000UL +#define EXCLUDED_BASE 0x220000000UL +#define REQUIRE(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "FAIL line %d: %s errno=%d\n", __LINE__, #condition, errno); \ + exit(1); \ + } \ +} while (0) + +static void write_setting(const char *path, const char *value) +{ + int fd = open(path, O_WRONLY); + + REQUIRE(fd >= 0); + REQUIRE(write(fd, value, strlen(value)) == (ssize_t)strlen(value)); + REQUIRE(!close(fd)); +} + +static unsigned char *tagged_map(unsigned long address, size_t size) +{ + void *p = mmap((void *)address, size, PROT_READ | PROT_WRITE | PROT_MTE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + + REQUIRE(p == (void *)address); + return p; +} + +static void set_tags(unsigned char *p, size_t size, unsigned int tag) +{ + for (size_t i = 0; i < size; i += 16) { + uintptr_t address = (uintptr_t)(p + i) | ((uintptr_t)tag << 56); + + asm volatile(".arch_extension memtag\n\tstg %0, [%0]" + : : "r" (address) : "memory"); + } +} + +static void child(unsigned long ps) +{ + struct stat self, init; + struct rlimit limit = { 64UL << 20, 64UL << 20 }; + unsigned char *p; + + REQUIRE(getppid() == 1); + REQUIRE(!stat("/proc/self/exe", &self) && !stat("/proc/1/exe", &init)); + REQUIRE(self.st_dev == init.st_dev && self.st_ino == init.st_ino); + REQUIRE(getauxval(AT_PAGESZ) == ps); + REQUIRE(!setrlimit(RLIMIT_CORE, &limit)); + REQUIRE(!prctl(PR_SET_DUMPABLE, 1UL, 0UL, 0UL, 0UL)); + REQUIRE(!prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0UL, 0UL, 0UL)); + write_setting("/proc/self/coredump_filter", "1"); + p = tagged_map(TAGGED_BASE, 3 * ps); + for (unsigned int i = 0; i < 3; i++) { + memset(p + i * ps, 0x40 + i, ps); + set_tags(p + i * ps, ps, i + 1); + } + /* Split metadata at a non-native boundary and retain the physical offset. */ + REQUIRE(!mprotect(p + ps, ps, PROT_READ | PROT_MTE)); + p = tagged_map(ZERO_BASE, ps); + *p = 0; + /* Keep anonymous dump eligibility while leaving the leaf absent. */ + REQUIRE(!madvise(p, ps, MADV_DONTNEED)); + p = tagged_map(EXCLUDED_BASE, ps); + memset(p, 0x55, ps); + set_tags(p, ps, 7); + REQUIRE(!madvise(p, ps, MADV_DONTDUMP)); + raise(SIGABRT); + _exit(2); +} + +static void inspect_core(pid_t pid, unsigned long ps) +{ + char path[80]; + Elf64_Ehdr eh; + struct stat st; + unsigned int tagged = 0, loads = 0, zero = 0, excluded = 0; + unsigned char *bytes = malloc(ps); + int fd; + + REQUIRE(bytes); + snprintf(path, sizeof(path), "/tmp/core.%d", pid); + fd = open(path, O_RDONLY); + REQUIRE(fd >= 0 && !fstat(fd, &st)); + REQUIRE(pread(fd, &eh, sizeof(eh), 0) == sizeof(eh)); + REQUIRE(!memcmp(eh.e_ident, ELFMAG, SELFMAG)); + REQUIRE(eh.e_ident[EI_CLASS] == ELFCLASS64 && eh.e_type == ET_CORE); + REQUIRE(eh.e_phentsize == sizeof(Elf64_Phdr) && eh.e_phnum < 4096); + for (unsigned int n = 0; n < eh.e_phnum; n++) { + Elf64_Phdr ph; + unsigned char expected; + unsigned int index; + size_t length; + + REQUIRE(pread(fd, &ph, sizeof(ph), eh.e_phoff + n * sizeof(ph)) == sizeof(ph)); + if (ph.p_type != PT_LOAD && ph.p_type != PT_AARCH64_MEMTAG_MTE) + continue; + if (ph.p_vaddr >= TAGGED_BASE && ph.p_vaddr < TAGGED_BASE + 3 * ps) { + index = (ph.p_vaddr - TAGGED_BASE) / ps; + REQUIRE(ph.p_vaddr == TAGGED_BASE + index * ps && ph.p_memsz == ps); + if (ph.p_type == PT_LOAD) { + loads++; + length = ps; + expected = 0x40 + index; + } else { + tagged++; + length = ps / 32; + expected = (index + 1) * 0x11; + } + } else if (ph.p_vaddr == ZERO_BASE && ph.p_type == PT_AARCH64_MEMTAG_MTE) { + zero++; + length = ps / 32; + expected = 0; + REQUIRE(ph.p_memsz == ps); + } else if (ph.p_vaddr == EXCLUDED_BASE && ph.p_type == PT_AARCH64_MEMTAG_MTE) { + excluded++; + REQUIRE(ph.p_filesz == 0 && ph.p_memsz == ps); + continue; + } else { + continue; + } + if (ph.p_filesz != length) + fprintf(stderr, "segment type=%#x address=%#lx size=%lu expected=%zu\n", + ph.p_type, (unsigned long)ph.p_vaddr, + (unsigned long)ph.p_filesz, length); + REQUIRE(ph.p_filesz == length); + REQUIRE(ph.p_offset <= (uint64_t)st.st_size && length <= st.st_size - ph.p_offset); + REQUIRE(pread(fd, bytes, length, ph.p_offset) == (ssize_t)length); + for (size_t i = 0; i < length; i++) + REQUIRE(bytes[i] == expected); + } + REQUIRE(tagged == 3 && loads == 3 && zero == 1 && excluded == 1); + REQUIRE(!close(fd) && !unlink(path)); + free(bytes); + printf("MTE CORE CASE PASS page-size=%lu\n", ps); +} + +int main(int argc, char **argv) +{ + unsigned long native = getauxval(AT_PAGESZ); + unsigned long sizes[] = { native, 4096, 16384 }; + unsigned int count = native > 16384 ? 3 : native > 4096 ? 2 : 1; + + setbuf(stdout, NULL); + setbuf(stderr, NULL); + if (argc == 3 && !strcmp(argv[1], "--child")) { + child(strtoul(argv[2], NULL, 10)); + return 1; + } + if (getpid() != 1) { + fprintf(stderr, "SKIP: disposable guest PID1 only\n"); + return 4; + } + REQUIRE(getauxval(AT_HWCAP2) & HWCAP2_MTE); + mkdir("/proc", 0755); + mkdir("/tmp", 0700); + REQUIRE(!mount("proc", "/proc", "proc", 0, NULL)); + REQUIRE(!mount("tmpfs", "/tmp", "tmpfs", 0, "mode=0700")); + write_setting("/proc/sys/kernel/core_pattern", "/tmp/core.%p"); + for (unsigned int i = 0; i < count; i++) { + pid_t pid = fork(); + int status; + + REQUIRE(pid >= 0); + if (!pid) { + char value[32]; + + snprintf(value, sizeof(value), "%lu", sizes[i]); + REQUIRE(!prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, sizes[i], 0UL, 0UL, 0UL)); + execl("/init", "/init", "--child", value, NULL); + _exit(3); + } + REQUIRE(waitpid(pid, &status, 0) == pid); + REQUIRE(WIFSIGNALED(status) && WTERMSIG(status) == SIGABRT && WCOREDUMP(status)); + inspect_core(pid, sizes[i]); + } + printf("MTE CORE PASS cases=%u\n", count); + reboot(RB_POWER_OFF); + return 0; +} From f2403ac59066f32884acf43d735e479667b90d6a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:52:13 -0400 Subject: [PATCH 344/631] mm: distinguish callback-supplied pages from page-cache faults A VMA can retain vm_file while its fault callback returns independent pages. Use cache-indexed fault installation only when the returned folio belongs to that file mapping; otherwise preserve the callback page and its requested subpage offset. Allow ordinary non-anonymous fragment references to follow these mappings without requiring page-cache membership. Keep permission checks and private read-pin COW rules unchanged. Add native, 4K and supported 16K callback tests with reversed backing pages, nonzero offsets and remote reads. They pass on 16K/64K KASAN guests, along with existing file-fragment lifetime/COW and special-mapping tests. This tests the callback contract, not a physical usbmon device. Signed-off-by: Diljit Singh (cherry picked from commit 09dae19a275153e1bbfe13e4b31f3fa8d94f72e8) --- arch/arm64/mm/user4k-test.c | 81 +++++++++++++++++++++++++++++++++++++ mm/gup.c | 4 -- mm/memory.c | 4 +- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 69456ba03bac5e..de2d47a5e569ed 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5439,6 +5439,84 @@ static void user4k_special_mapping_test(struct kunit *test) } } +static vm_fault_t test_callback_fault(struct vm_fault *vmf) +{ + struct page **pages = vmf->vma->vm_private_data; + + if (vmf->pgoff >= 2) + return VM_FAULT_SIGBUS; + + vmf->page = pages[vmf->pgoff]; + get_page(vmf->page); + return 0; +} + +static const struct vm_operations_struct test_callback_vm_ops = { + .fault = test_callback_fault, +}; + +static void test_callback_mapping(struct kunit *test, unsigned int shift) +{ + void *data[2] = { test_alloc_page(test), test_alloc_page(test) }; + struct page **pages = kunit_kcalloc(test, 2, sizeof(*pages), GFP_KERNEL); + struct file *file = test_cache_file(test, 0, false); + struct vm_area_struct *vma; + unsigned long size = 1UL << shift; + unsigned long offset = shift == PAGE_SHIFT ? 0 : size; + unsigned long len = 2 * PAGE_SIZE - offset; + unsigned long addr; + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, data[0]); + KUNIT_ASSERT_NOT_NULL(test, data[1]); + KUNIT_ASSERT_NOT_NULL(test, pages); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + pages[0] = virt_to_page(data[1]); + pages[1] = virt_to_page(data[0]); + for (i = 0; i < 2 * PAGE_SIZE / size; i++) + memset(page_address(pages[(i * size) >> PAGE_SHIFT]) + + offset_in_page(i * size), 0x51 + i, size); + + /* The file identifies the VMA; the callback supplies independent pages. */ + vma = test_file_vma_granule(test, file, shift, false, TEST_VA, len, offset); + KUNIT_ASSERT_NOT_NULL(test, vma); + mmap_write_lock(vma->vm_mm); + vma->vm_ops = &test_callback_vm_ops; + vma->vm_private_data = pages; + mmap_write_unlock(vma->vm_mm); + for (addr = vma->vm_start; addr < vma->vm_end; addr += size) { + unsigned long source = offset + addr - vma->vm_start; + u8 byte; + + KUNIT_ASSERT_EQ(test, test_file_fault(vma, addr, false) & VM_FAULT_ERROR, 0U); + KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, + test_fault_entry(test, vma->vm_mm, addr)), + page_to_phys(pages[source >> PAGE_SHIFT]) + offset_in_page(source)); + KUNIT_ASSERT_EQ(test, access_remote_vm(vma->vm_mm, addr + 7, &byte, 1, 0), 1); + KUNIT_EXPECT_EQ(test, byte, (u8)(0x51 + source / size)); + } + KUNIT_EXPECT_EQ(test, get_mm_counter_sum(vma->vm_mm, MM_FILEPAGES), len / size); +} + +static void native_callback_mapping_test(struct kunit *test) +{ + test_callback_mapping(test, PAGE_SHIFT); +} + +static void user4k_callback_mapping_test(struct kunit *test) +{ + test_callback_mapping(test, 12); +} + +static void user16k_callback_mapping_test(struct kunit *test) +{ + if (PAGE_SHIFT <= 14) { + kunit_skip(test, "16K alternative requires a larger native granule"); + return; + } + test_callback_mapping(test, 14); +} + static void user4k_insert_range_test(struct kunit *test) { unsigned int native; @@ -14115,6 +14193,9 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_stack_relocate_test), KUNIT_CASE(user4k_exec_arguments_test), KUNIT_CASE(user4k_special_mapping_test), + KUNIT_CASE(native_callback_mapping_test), + KUNIT_CASE(user4k_callback_mapping_test), + KUNIT_CASE(user16k_callback_mapping_test), KUNIT_CASE(user4k_insert_fragment_test), KUNIT_CASE(user16k_insert_shared_test), KUNIT_CASE(user4k_insert_range_test), diff --git a/mm/gup.c b/mm/gup.c index 8d9bbc80546fa2..b7738d87d55b83 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -3521,10 +3521,6 @@ static int follow_subpage_fragment(struct vm_area_struct *vma, unsigned long add goto acquired; } if (!folio_test_anon(folio)) { - if (vma->vm_file && folio_mapping(folio) != vma->vm_file->f_mapping) { - ret = -EOPNOTSUPP; - goto out; - } if ((flags & FOLL_WRITE) && !pte_write(pte)) goto out; /* Private non-anonymous read pins must first acquire COW ownership. */ diff --git a/mm/memory.c b/mm/memory.c index 22e3ea5356680f..a2ca51ea019398 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -7096,7 +7096,7 @@ vm_fault_t map_file_user_pages(struct vm_fault *vmf, struct folio *folio, } /* - * A non-file fault callback supplies one native page, not a cache-indexed + * A non-page-cache fault callback supplies one native page, not a cache-indexed * folio. In particular, VVAR callbacks may reorder native pages. Only map * quarters of this supplied page; never infer adjacent pages from its folio. */ @@ -7232,7 +7232,7 @@ static vm_fault_t finish_user_page_fault(struct vm_fault *vmf) inc_mm_counter(mm, MM_ANONPAGES); ret = 0; } else { - if (vma->vm_file) + if (vma->vm_file && folio_mapping(folio) == vma->vm_file->f_mapping) ret = map_file_user_pages(vmf, folio, folio->index, folio_next_index(folio) - 1, true); else if (mm_page_size(mm) < PAGE_SIZE) From c759a6bc31e52e433c4dd63c0b85d273d28f86c5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 05:58:29 -0400 Subject: [PATCH 345/631] mm: clip mapping walks to exact VMA byte intervals walk_page_mapping() truncates a VMA length to native pages and ignores its subpage file offset. Short mappings can disappear from the walk and partial first/last native pages can produce incorrect virtual bounds. Use the existing byte-offset translation and last-index helpers to clip the requested native file indices to the VMA. Handle an empty request and reject an overflowing index interval before the interval-tree walk. Validated native and smaller-granule walks over short mappings and partial first/last pages in 16K/64K KASAN guests. The temporary fixture and results remain in the local validation records instead of adding another permanent arm64 test block. The generic x86 pagewalk object also compiles. Signed-off-by: Diljit Singh (cherry picked from commit 0a085675da3ec34e40f94c8d9af6194cfd133c91) --- mm/pagewalk.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index c3216b4852399e..54bc7ac30befa5 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -809,27 +810,26 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, .private = private, }; struct vm_area_struct *vma; - pgoff_t vba, vea, cba, cea; + pgoff_t last_index; unsigned long start_addr, end_addr; int err = 0; if (!check_ops_safe(ops)) return -EINVAL; + if (!nr) + return 0; + if (check_add_overflow(first_index, nr - 1, &last_index)) + return -EINVAL; lockdep_assert_held(&mapping->i_mmap_rwsem); vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, - first_index + nr - 1) { - /* Clip to the vma */ - vba = vma->vm_pgoff; - vea = vba + vma_pages(vma); - cba = first_index; - cba = max(cba, vba); - cea = first_index + nr; - cea = min(cea, vea); - - start_addr = ((cba - vba) << PAGE_SHIFT) + vma->vm_start; - end_addr = ((cea - vba) << PAGE_SHIFT) + vma->vm_start; - if (start_addr >= end_addr) + last_index) { + /* Clip native file indices to the VMA's exact byte interval. */ + start_addr = first_index <= vma->vm_pgoff ? vma->vm_start : + vma_address_at_offset(vma, (struct vm_page_offset) { first_index, 0 }); + end_addr = last_index >= vma_last_pgoff(vma) ? vma->vm_end : + vma_address_at_offset(vma, (struct vm_page_offset) { last_index + 1, 0 }); + if (IS_ERR_VALUE(start_addr) || IS_ERR_VALUE(end_addr) || start_addr >= end_addr) continue; walk.vma = vma; From 6d839e38659bc1fbe889227e53a8276d3219f430 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:02:48 -0400 Subject: [PATCH 346/631] mm: round up mapping lengths when checking I/O resource bounds A mapping can be shorter than a native page when its MM uses a smaller granule. Count a partial final native page in __simple_ioremap_prep() instead of truncating it away. Native-aligned mappings retain their existing calculation, and the quotient/remainder form avoids a rounding addition overflow. Both 16K/64K KASAN kernels build and pass the existing PFN-remap suite; the x86 memory object also builds. No device-boundary reproducer was run. This bounds correction does not qualify every device offset interface. Signed-off-by: Diljit Singh (cherry picked from commit d4e50b58b49627c95920d2f106da600ead1254ff) --- mm/memory.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/memory.c b/mm/memory.c index a2ca51ea019398..dd2916612f9c7e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3696,7 +3696,7 @@ static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, pages -= vm_pgoff; /* Can we fit all of the mapping? */ - if ((vm_len >> PAGE_SHIFT) > pages) + if ((vm_len >> PAGE_SHIFT) + !!offset_in_page(vm_len) > pages) return -EINVAL; *pfnp = pfn; From 5fb568c334c654493ffb4bd1011a46c6af797475 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:07:24 -0400 Subject: [PATCH 347/631] arm64: retain exact fragments for remote MTE tag access The legacy remote-page interface cannot represent smaller-granule backing. Use the stable-VMA fragment interface for alternative MMs, take the actual physical offset from the returned record, and bound each transfer by its length. Release the same retained fragment and dirty its ownership record on writes. Preserve the native page path and existing ptrace permission and partial-transfer semantics. Reused the local core-file fixture to validate own-child ptrace tag reads, writes and restoration across three differently tagged mappings, including a read-only mapping. Native/4K cases pass in a 16K KASAN guest and native/4K/16K cases pass in a 64K guest; resulting core files also pass. The feature-disabled native MTE object builds. No additional submitted test program is introduced, and physical MTE hardware remains untested. Signed-off-by: Diljit Singh (cherry picked from commit f41afd84696d9b7381b25f105f58caa2c0590365) --- arch/arm64/kernel/mte.c | 44 +++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 645f549d9fed90..fb4eba0626e5a5 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -463,11 +464,24 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, while (len) { struct vm_area_struct *vma; - unsigned long tags, offset; + unsigned long tags = 0, offset; void *maddr; - struct page *page = get_user_page_vma_remote(mm, addr, - gup_flags, &vma); + struct page *page; struct folio *folio; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment fragment = {}; + + if (mm_page_size(mm) != PAGE_SIZE) { + unsigned long bytes = min(len, PAGE_SIZE / MTE_GRANULE_SIZE) * + MTE_GRANULE_SIZE; + int got = get_user_fragment_vma_remote(mm, addr, bytes, + gup_flags, &fragment, &vma); + + page = got > 0 ? folio_page(fragment.folio, + fragment.offset >> PAGE_SHIFT) : ERR_PTR(got ?: -EFAULT); + } else +#endif + page = get_user_page_vma_remote(mm, addr, gup_flags, &vma); if (IS_ERR(page)) { err = PTR_ERR(page); @@ -483,8 +497,7 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, */ if (!(vma->vm_flags & VM_MTE)) { err = -EOPNOTSUPP; - put_page(page); - break; + goto release; } folio = page_folio(page); @@ -497,18 +510,33 @@ static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, /* limit access to the end of the page */ offset = offset_in_page(addr); tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE); +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) { + offset = offset_in_page(fragment.offset); + tags = fragment.length / MTE_GRANULE_SIZE; + } +#endif maddr = page_address(page); if (write) { tags = mte_copy_tags_from_user(maddr + offset, buf, tags); - set_page_dirty_lock(page); +#ifdef CONFIG_MM_SUBPAGE + if (!fragment.folio) +#endif + set_page_dirty_lock(page); } else { tags = mte_copy_tags_to_user(buf, maddr + offset, tags); } - put_page(page); +release: +#ifdef CONFIG_MM_SUBPAGE + if (fragment.folio) + release_user_fragments(&fragment, 1, write && !err); + else +#endif + put_page(page); /* error accessing the tracer's buffer */ - if (!tags) + if (err || !tags) break; len -= tags; From 2eea115b0612d4bdb18d200e417e41f4d5ffec22 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:18:24 -0400 Subject: [PATCH 348/631] iommu/arm-smmu-v3: use the bound mm granule for SVA Build the SVA context descriptor and invalidation geometry from the bound MM rather than PAGE_SIZE. Check the target device's granule support at allocation and attachment, and record the same granule in the domain so range invalidation commands agree with the CPU page-table geometry. Extend the existing SVA descriptor transition test to cover the enabled alternative granules. The existing 25-case SMMUv3 suite passes in both 16K and 64K KASAN guests; native feature-disabled SVA and test objects also compile. Actual SVA device DMA and invalidation delivery remain untested. The driver's existing global native-granule capability prerequisite is preserved. Signed-off-by: Diljit Singh (cherry picked from commit 4ccea57e713c22800c059cad4662abef494dd054) --- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 16 ++++++++----- .../iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c | 23 +++++++++++++++---- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index f1f8e01a7e9142..57069672545172 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -36,13 +36,13 @@ arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); } -static u64 page_size_to_cd(void) +static u64 page_size_to_cd(unsigned long page_size) { static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || PAGE_SIZE == SZ_64K); - if (PAGE_SIZE == SZ_64K) + if (page_size == SZ_64K) return ARM_LPAE_TCR_TG0_64K; - if (PAGE_SIZE == SZ_16K) + if (page_size == SZ_16K) return ARM_LPAE_TCR_TG0_16K; return ARM_LPAE_TCR_TG0_4K; } @@ -83,7 +83,7 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, target->data[0] |= cpu_to_le64( FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | - FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | + FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd(mm_page_size(mm))) | FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, @@ -138,7 +138,7 @@ static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, */ size = end - start; - arm_smmu_domain_inv_range(smmu_domain, start, size, PAGE_SIZE, false); + arm_smmu_domain_inv_range(smmu_domain, start, size, mm_page_size(mm), false); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -256,6 +256,8 @@ static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return -EOPNOTSUPP; + if (!(master->smmu->pgsize_bitmap & mm_page_size(domain->mm))) + return -EOPNOTSUPP; /* Prevent arm_smmu_mm_release from being called while we are attaching */ if (!mmget_not_zero(domain->mm)) @@ -312,6 +314,8 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return ERR_PTR(-EOPNOTSUPP); + if (!(smmu->pgsize_bitmap & mm_page_size(mm))) + return ERR_PTR(-EOPNOTSUPP); smmu_domain = arm_smmu_domain_alloc(); if (IS_ERR(smmu_domain)) @@ -323,7 +327,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, * Choose page_size as the leaf page size for invalidation when * ARM_SMMU_FEAT_RANGE_INV is present */ - smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; + smmu_domain->domain.pgsize_bitmap = mm_page_size(mm); smmu_domain->stage = ARM_SMMU_DOMAIN_SVA; smmu_domain->smmu = smmu; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c index add671363c828c..fa1c7b03394d68 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-test.c @@ -616,12 +616,25 @@ static void arm_smmu_v3_write_cd_test_sva_clear(struct kunit *test) { struct arm_smmu_cd cd = {}; struct arm_smmu_cd cd_2; + const unsigned long flags[] = { 0, MMCF_USER_4K, MMCF_USER_16K }; + /* TG0 encodings: 4K = 0, 64K = 1, 16K = 2. */ + const unsigned int tg[] = { PAGE_SHIFT == 16 ? 1 : PAGE_SHIFT == 14 ? 2 : 0, 0, 2 }; + unsigned int i; - arm_smmu_test_make_sva_cd(&cd_2, 1997); - arm_smmu_v3_test_cd_expect_non_hitless_transition( - test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); - arm_smmu_v3_test_cd_expect_non_hitless_transition( - test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); + for (i = 0; i < ARRAY_SIZE(flags); i++) { + if (i && (!IS_ENABLED(CONFIG_ARM64_USER4K_EXPERIMENTAL) || + (i == 2 && PAGE_SHIFT <= 14))) + continue; + sva_mm.context.flags = flags[i]; + arm_smmu_test_make_sva_cd(&cd_2, 1997); + KUNIT_EXPECT_EQ(test, FIELD_GET(CTXDESC_CD_0_TCR_TG0, + le64_to_cpu(cd_2.data[0])), tg[i]); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd, &cd_2, NUM_EXPECTED_SYNCS(2)); + arm_smmu_v3_test_cd_expect_non_hitless_transition( + test, &cd_2, &cd, NUM_EXPECTED_SYNCS(2)); + } + sva_mm.context.flags = 0; } static void arm_smmu_v3_write_cd_test_sva_release(struct kunit *test) From e93d412807b184533470b81b603d4850b0f2d883 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:21:46 -0400 Subject: [PATCH 349/631] ocfs2: use truncate_pagecache after shrinking an extent-backed file After ocfs2_orphan_for_truncate() updates i_size, use the generic pagecache truncation helper. Its invalidation covers smaller userspace leaves in the last native page and repeats after pagecache truncation to remove COW mappings created during the operation. Keep the inline-data path's ordering unchanged. OCFS2 blocks are at most 4K, so inline data cannot span multiple leaves of the exposed alternative userspace configurations. The changed OCFS2 file object builds for 16K and 64K arm64 configurations. A mounted OCFS2 filesystem has not been runtime-tested for this change. Signed-off-by: Diljit Singh (cherry picked from commit a6c5a0042f88694ebedb8b622cd0ca17896ba474) --- fs/ocfs2/file.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 7df9921c1a3899..21153bd53f66ea 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -504,8 +504,7 @@ int ocfs2_truncate_file(struct inode *inode, goto bail_unlock_sem; } - unmap_mapping_range(inode->i_mapping, new_i_size + PAGE_SIZE - 1, 0, 1); - truncate_inode_pages(inode->i_mapping, new_i_size); + truncate_pagecache(inode, new_i_size); status = ocfs2_commit_truncate(osb, inode, di_bh); if (status < 0) { From bfe56b941044f1dbe85438d350c437e605fd4e12 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:35:39 -0400 Subject: [PATCH 350/631] dma-buf: retain subpage offsets in CMA and udmabuf faults CMA and udmabuf fault handlers identify the native backing page but lose the smaller leaf's byte offset. Pass the explicit file subpage offset to PFN insertion. In udmabuf, use the same translation for demand faults and prefaults, advancing by the target MM granule so prefault coverage remains complete. Validated CPU mappings with the existing dma-buf heap and udmabuf selftests in 16K/64K KASAN guests for native, 4K and supported 16K processes. The local udmabuf test uses native-aligned buffer sizes and distinct per-leaf bytes; its two hugetlb cases are explicitly skipped without a reserved pool. Both CMA heap names and vgem import pass. No additional test source is submitted and physical device DMA remains unqualified. Signed-off-by: Diljit Singh (cherry picked from commit ac963beadc4754a08163d643e26b23f0d1c31bd5) --- drivers/dma-buf/heaps/cma_heap.c | 5 ++++- drivers/dma-buf/udmabuf.c | 38 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index a359aac4557990..37ec7bc84a24ee 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -174,7 +174,10 @@ static vm_fault_t cma_heap_vm_fault(struct vm_fault *vmf) if (vmf->pgoff >= buffer->pagecount) return VM_FAULT_SIGBUS; - return vmf_insert_pfn(vma, vmf->address, page_to_pfn(buffer->pages[vmf->pgoff])); + return vmf_insert_pfn_prot_mkwrite_offset(vma, vmf->address, + page_to_pfn(buffer->pages[vmf->pgoff]), + vma_page_offset_at(vma, vmf->address).offset, + vma->vm_page_prot, false); } static const struct vm_operations_struct dma_heap_vm_ops = { diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index 2e7b5d4d11f429..71efcf40d91695 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -45,45 +45,45 @@ struct udmabuf { pgoff_t *offsets; }; -static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) +static vm_fault_t udmabuf_insert_pfn(struct vm_area_struct *vma, unsigned long addr) { - struct vm_area_struct *vma = vmf->vma; struct udmabuf *ubuf = vma->vm_private_data; - pgoff_t pgoff = vmf->pgoff; - unsigned long addr, pfn; - vm_fault_t ret; + struct vm_page_offset pos = vma_page_offset_at(vma, addr); + unsigned long pfn; - if (pgoff >= ubuf->pagecount) + if (pos.index >= ubuf->pagecount) return VM_FAULT_SIGBUS; - pfn = folio_pfn(ubuf->folios[pgoff]); - pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; + pfn = folio_pfn(ubuf->folios[pos.index]); + pfn += ubuf->offsets[pos.index] >> PAGE_SHIFT; + return vmf_insert_pfn_prot_mkwrite_offset(vma, addr, pfn, pos.offset, + vma->vm_page_prot, false); +} + +static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + unsigned long addr; + vm_fault_t ret; - ret = vmf_insert_pfn(vma, vmf->address, pfn); + ret = udmabuf_insert_pfn(vma, vmf->address); if (ret & VM_FAULT_ERROR) return ret; /* pre fault */ - pgoff = vma->vm_pgoff; addr = vma->vm_start; - for (; addr < vma->vm_end; pgoff++, addr += PAGE_SIZE) { + for (; addr < vma->vm_end; addr += mm_page_size(vma->vm_mm)) { if (addr == vmf->address) continue; - if (WARN_ON(pgoff >= ubuf->pagecount)) - break; - - pfn = folio_pfn(ubuf->folios[pgoff]); - pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; - /** - * If the below vmf_insert_pfn() fails, we do not return an + * If the below insertion fails, we do not return an * error here during this pre-fault step. However, an error * will be returned if the failure occurs when the addr is * truly accessed. */ - if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR) + if (udmabuf_insert_pfn(vma, addr) & VM_FAULT_ERROR) break; } From e7eabafc37ba178e240322ee991daf318d3cd388 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:51:16 -0400 Subject: [PATCH 351/631] arm64/mm: preserve neighboring subpage dirty state when clearing PTEs Contiguous-PTE access/dirty clearing can expand to the full group because native MM tracks those bits per folio. Smaller userspace leaves can be reclaimed independently, so extending a dirty clear can erase evidence belonging to a different retained fragment. Unfold partially covered groups for smaller-granule dirty clearing and operate on the requested entries only. Keep native handling, full-group operations and the existing folding policy. Update the existing contiguous-PTE expectations for fragment isolation. The lazyfree migration case accepts dirty retention because migration can refold a group and conservatively aggregate a neighbor's dirty state; it still rejects a leftover migration entry or clean retained slot. The existing contiguous-PTE and lazyfree suites pass on 16K/64K KASAN kernels. No data-loss reproducer was run. Signed-off-by: Diljit Singh (cherry picked from commit 425488825ee3b258100181b75295215cf50397fe) --- arch/arm64/mm/contpte.c | 12 +++++++++--- arch/arm64/mm/user4k-test.c | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index e78e0d4265218b..b3fcca267148bc 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -614,6 +614,15 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, unsigned int nr, cydp_t flags) { struct mm_struct *mm = vma->vm_mm; + unsigned long start = addr; + unsigned long end = start + nr * mm_page_size(mm); + + /* Smaller leaves may be discarded independently within the folio. */ + if (mm_page_size(mm) < PAGE_SIZE && (flags & CYDP_CLEAR_DIRTY)) { + contpte_try_unfold_partial(mm, addr, ptep, nr); + __clear_young_dirty_ptes(vma, addr, ptep, nr, flags); + return; + } /* * We can safely clear access/dirty without needing to unfold from @@ -625,9 +634,6 @@ void contpte_clear_young_dirty_ptes(struct vm_area_struct *vma, * when it is covered by a single folio, we can get away with * clearing access/dirty for the whole block. */ - unsigned long start = addr; - unsigned long end = start + nr * mm_page_size(mm); - ptep = contpte_align_addr_ptep(vma->vm_mm, &start, &end, ptep, nr); __clear_young_dirty_ptes(vma, start, ptep, (end - start) >> mm_page_shift(mm), flags); } diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index de2d47a5e569ed..bad3eee2c79eec 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -7771,14 +7771,19 @@ static void test_contpte_granule(struct kunit *test, unsigned int shift) CYDP_CLEAR_YOUNG | CYDP_CLEAR_DIRTY); for (i = 0; i < f->nr; i++) { observed = __ptep_get(pte + i); - ok.aged &= !pte_young(observed) && !pte_dirty(observed) && pte_cont(observed); + if (size < PAGE_SIZE) + ok.aged &= !pte_cont(observed) && + pte_young(observed) == (i != 3) && pte_dirty(observed) == (i != 3); + else + ok.aged &= !pte_young(observed) && !pte_dirty(observed) && + pte_cont(observed); } observed = pte_mkdirty(pte_mkyoung(__ptep_get(pte + 3))); access_same_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, pte + 3, observed, 1); for (i = 0; i < f->nr; i++) { observed = __ptep_get(pte + i); - ok.access_all &= pte_cont(observed) && pte_young(observed) && + ok.access_all &= pte_cont(observed) == (size >= PAGE_SIZE) && pte_young(observed) && pte_dirty(observed) && !pte_write(observed); } access_noop_ret = ptep_set_access_flags(&vma, TEST_VA + 3 * size, @@ -9609,9 +9614,16 @@ static void user4k_lazyfree_test(struct kunit *test) folio_put(folio); for (i = 0; i < 4; i++) { pte = test_fault_entry(test, mm, TEST_VA + i * SZ_4K); - KUNIT_EXPECT_EQ(test, pte_none(pte), + /* Migration may refold dirty neighbors and retain an advised slot. */ + if (mode == 5 && !i && pte_present(pte)) { + KUNIT_EXPECT_TRUE(test, pte_dirty(pte)); + continue; + } + KUNIT_EXPECT_EQ_MSG(test, pte_none(pte), (mode == 4 || mode == 9 || mode == 10) || - (i == 0 && (mode == 0 || mode == 5))); + (i == 0 && (mode == 0 || mode == 5)), + "mode=%u slot=%u pte=%llx", mode, i, + (unsigned long long)pte_val(pte)); } if (mode == 2 || mode == 6 || mode == 7) release_user_fragments(&held, 1, false); From b56a2605683f61787199acf57062197876fcc2fa Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 06:58:30 -0400 Subject: [PATCH 352/631] mm: determine swap allocation requirements before taking the local lock The minimum anonymous folio order lookup temporarily holds an anon-vma reference. Dropping its last reference may sleep, so eligibility checks must not repeat that lookup under the per-CPU swap-cluster local lock. Reuse the mandatory-allocation decision already calculated by folio_alloc_swap() and pass it through the fast and slow allocation paths. The required fallback uses the same decision; hibernation has no folio. Document the lookup's sleepable context. 16K/64K KASAN builds and the existing swap/swapoff and UFFD move contract checks pass. The x86 swapfile object compiles. These checks do not reproduce the destructor lifetime race or qualify regular-file swap transport. Signed-off-by: Diljit Singh (cherry picked from commit 5f06041cbb3a178674d5aae5eb40483980175fca) --- mm/rmap.c | 5 ++++- mm/swapfile.c | 29 ++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index bf11cc857cb45b..9241060489f580 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -629,7 +629,10 @@ struct anon_vma *folio_get_anon_vma(const struct folio *folio) } #ifdef CONFIG_MM_SUBPAGE -/* The caller retains the folio; splitting rechecks the order under its lock. */ +/* + * The caller retains the folio; splitting rechecks the order under its lock. + * May sleep when dropping the temporary anon-vma reference. + */ unsigned int folio_anon_min_order(struct folio *folio) { struct anon_vma *anon_vma = NULL; diff --git a/mm/swapfile.c b/mm/swapfile.c index c2c49b9fb3f366..ca11d95c2d0371 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1083,7 +1083,8 @@ static void swap_reclaim_work(struct work_struct *work) swap_reclaim_full_clusters(si, true); } -static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio) +static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio *folio, + bool mandatory) { unsigned int flags; @@ -1093,7 +1094,7 @@ static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio * if (flags & SWP_BLKDEV) return true; /* Filesystem swap_rw implementations retain their existing contract. */ - return si->bdev && !(flags & SWP_FS_OPS) && folio_anon_min_order(folio); + return si->bdev && !(flags & SWP_FS_OPS) && mandatory; } /* @@ -1101,13 +1102,13 @@ static bool swap_folio_alloc_allowed(struct swap_info_struct *si, struct folio * * cluster for current CPU too. */ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, - struct folio *folio) + struct folio *folio, bool mandatory) { struct swap_cluster_info *ci; unsigned int order = likely(folio) ? folio_order(folio) : 0; unsigned int offset = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID; - if (!swap_folio_alloc_allowed(si, folio)) + if (!swap_folio_alloc_allowed(si, folio, mandatory)) return 0; if (!(si->flags & SWP_SOLIDSTATE)) { @@ -1371,7 +1372,7 @@ static bool get_swap_device_info(struct swap_info_struct *si) * Fast path try to get swap entries with specified order from current * CPU's swap entry pool (a cluster). */ -static bool swap_alloc_fast(struct folio *folio) +static bool swap_alloc_fast(struct folio *folio, bool mandatory) { unsigned int order = folio_order(folio); struct swap_cluster_info *ci; @@ -1386,7 +1387,7 @@ static bool swap_alloc_fast(struct folio *folio) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; - if (!swap_folio_alloc_allowed(si, folio)) { + if (!swap_folio_alloc_allowed(si, folio, mandatory)) { put_swap_device(si); return false; } @@ -1405,7 +1406,7 @@ static bool swap_alloc_fast(struct folio *folio) } /* Rotate the device and switch to a new cluster */ -static void swap_alloc_slow(struct folio *folio) +static void swap_alloc_slow(struct folio *folio, bool mandatory) { struct swap_info_struct *si, *next; @@ -1416,7 +1417,7 @@ static void swap_alloc_slow(struct folio *folio) plist_requeue(&si->avail_list, &swap_avail_head); spin_unlock(&swap_avail_lock); if (get_swap_device_info(si)) { - cluster_alloc_swap_entry(si, folio); + cluster_alloc_swap_entry(si, folio, mandatory); put_swap_device(si); if (folio_test_swapcache(folio)) return; @@ -1808,7 +1809,7 @@ static void swap_alloc_required_from_device(struct swap_info_struct *si, unsigned int order = folio_order(folio), nr = folio_nr_pages(folio); unsigned long start; - if (!swap_folio_alloc_allowed(si, folio)) + if (!swap_folio_alloc_allowed(si, folio, true)) return; for (start = 0; start < si->max; start += SWAPFILE_CLUSTER) { struct swap_cluster_info *ci = __swap_offset_to_cluster(si, start); @@ -1886,10 +1887,12 @@ int folio_alloc_swap(struct folio *folio) { unsigned int order = folio_order(folio); unsigned int size = 1 << order; - bool mandatory = order && folio_anon_min_order(folio); + bool mandatory; VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); VM_BUG_ON_FOLIO(!folio_test_uptodate(folio), folio); + /* The temporary anon-vma reference may require sleepable final release. */ + mandatory = order && folio_anon_min_order(folio); if (order) { /* @@ -1911,8 +1914,8 @@ int folio_alloc_swap(struct folio *folio) again: local_lock(&percpu_swap_cluster.lock); - if (!swap_alloc_fast(folio)) - swap_alloc_slow(folio); + if (!swap_alloc_fast(folio, mandatory)) + swap_alloc_slow(folio, mandatory); local_unlock(&percpu_swap_cluster.lock); if (mandatory && unlikely(!folio_test_swapcache(folio))) @@ -2318,7 +2321,7 @@ swp_entry_t swap_alloc_hibernation_slot(int type) swap_cluster_unlock(ci); } if (!offset) - offset = cluster_alloc_swap_entry(si, NULL); + offset = cluster_alloc_swap_entry(si, NULL, false); local_unlock(&percpu_swap_cluster.lock); if (offset) entry = swp_entry(si->type, offset); From ae29303a7aa4dd32abba4aeac63d5657a627b789 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:39 -0400 Subject: [PATCH 353/631] mm: return the mapped fragment address from page_mapped_in_vma Supply the native page offset when bounding the reverse-map walk. For a smaller-granule MM, return the matching leaf found by the walk instead of the initial address, which can refer to an unmapped leading fragment. Keep the byte address within a covering leaf for native and coarser MMs. Extend the existing boundary case with a nonzero file offset and an absent leading fragment. Reverse-map tests pass in 16K/64K KASAN guests; native x86 objects build with CONFIG_MEMORY_FAILURE enabled. Signed-off-by: Diljit Singh (cherry picked from commit c11e970bc8c123095f2deb62b40abe29cb8d129d) --- arch/arm64/mm/user4k-test.c | 16 ++++++++++++++++ mm/page_vma_mapped.c | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index bad3eee2c79eec..42719000b5a1d9 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -1507,11 +1507,14 @@ static void user4k_rmap_boundary_test(struct kunit *test) KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_free_vma, vma), 0); vma->vm_start = SZ_2M - SZ_4K; vma->vm_end = vma->vm_start + PAGE_SIZE; + vma->vm_pgoff = 37; + page_folio(pfn_to_page(ctx->phys[0] >> PAGE_SHIFT))->index = vma->vm_pgoff; for (i = 0; i < 3; i++) KUNIT_ASSERT_EQ(test, test_map_alias(ctx, 4 + i, vma->vm_start + i * SZ_4K, ctx->phys[i]), 0); walk = (struct page_vma_mapped_walk) { .pfn = ctx->phys[0] >> PAGE_SHIFT, + .pgoff = vma->vm_pgoff, .nr_pages = 1, .vma = vma, .address = vma->vm_start, @@ -1529,6 +1532,19 @@ static void user4k_rmap_boundary_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, valid); KUNIT_EXPECT_EQ(test, count, 3U); KUNIT_EXPECT_TRUE(test, walk.flags & PVMW_PGTABLE_CROSSED); +#ifdef CONFIG_MEMORY_FAILURE + { + spinlock_t *ptl; + pte_t *pte = test_mm_lookup(ctx->mm, vma->vm_start, &ptl); + + KUNIT_ASSERT_NOT_NULL(test, pte); + ptep_get_and_clear(ctx->mm, vma->vm_start, pte); + pte_unmap_unlock(pte, ptl); + KUNIT_EXPECT_EQ(test, + page_mapped_in_vma(pfn_to_page(ctx->phys[0] >> PAGE_SHIFT), vma), + vma->vm_start + SZ_4K); + } +#endif } static void native_rmap_walk_test(struct kunit *test) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index e3b0b64fbfba21..9377be895563a8 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -378,20 +378,24 @@ unsigned long page_mapped_in_vma(const struct page *page, unsigned long address; struct page_vma_mapped_walk pvmw = { .pfn = page_to_pfn(page), + .pgoff = page_pgoff(folio, page), .nr_pages = 1, .vma = vma, .flags = PVMW_SYNC, }; - address = vma_address(vma, page_pgoff(folio, page), 1); + address = vma_address(vma, pvmw.pgoff, 1); pvmw.address = address; if (pvmw.address == -EFAULT) goto out; if (!page_vma_mapped_walk(&pvmw)) return -EFAULT; + /* A smaller leaf can map the page after an unmapped leading fragment. */ + if (mm_page_size(vma->vm_mm) < PAGE_SIZE) + address = pvmw.address; page_vma_mapped_walk_done(&pvmw); out: - /* Report this native page's byte address, not the covering leaf start. */ + /* Coarser leaves retain this native page's byte address within the leaf. */ return address; } #endif From c5bde1524c1a786e89b09692d884caa92a69cf66 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:39 -0400 Subject: [PATCH 354/631] mm: clear poison entries on subpage MADV_FREE Match the native MADV_FREE handling of hardware-poison and software poison markers. Clearing these non-swap entries must not change swap accounting. Extend the existing userfaultfd marker case to check MADV_FREE recovery and neighboring data. The suite passes for native/4K processes on 16K and native/4K/16K processes on 64K KASAN kernels. Hardware memory errors were not injected. Signed-off-by: Diljit Singh (cherry picked from commit f86041e8396472c60b6a4adf063c4d93b99082d5) --- mm/madvise.c | 3 +++ tools/testing/selftests/arm64/user4k/uffd-wp-init.c | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index 1d15da9d1200d2..e746437c6b1f3f 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -690,6 +690,9 @@ static int madvise_free_subpage_range(pmd_t *pmd, unsigned long addr, pte_swp_subpage_offset(pte), true); pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); swapped--; + } else if (softleaf_is_hwpoison(entry) || + softleaf_is_poison_marker(entry)) { + pte_clear_not_present_full(mm, addr, ptep, tlb->fullmm); } continue; } diff --git a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c index 19747755c4c6c9..f34ab8328721d0 100644 --- a/tools/testing/selftests/arm64/user4k/uffd-wp-init.c +++ b/tools/testing/selftests/arm64/user4k/uffd-wp-init.c @@ -432,8 +432,12 @@ static void markers(size_t ps) CHECK(!sigaction(SIGBUS, &old, NULL)); p[17] = 0x81; p[2 * ps + 17] = 0x82; - CHECK(!close(fd) && !munmap(p, 3 * ps)); - printf("ok - uffd ABI %zuK poisoned leaf signals SIGBUS and preserves neighbours\n", + CHECK(!madvise(p + ps, ps, MADV_FREE)); + CHECK(!close(fd)); + CHECK(p[ps + 17] == 0); + CHECK(p[17] == 0x81 && p[2 * ps + 17] == 0x82); + CHECK(!munmap(p, 3 * ps)); + printf("ok - uffd ABI %zuK poison: SIGBUS, MADV_FREE recovery, neighbours\n", ps / 1024); } struct pager_request { From 87d4a4c091f32d06ca85c830586004e316d656b9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:17:40 -0400 Subject: [PATCH 355/631] arm64: retain the fragment pin test workers through cleanup A pin worker can exit on an error before cleanup calls kthread_stop(). Create it stopped, acquire a task reference, register cleanup, and then wake it. Release the reference after kthread_stop(), including the cleanup registration failure path. The existing retained-pin test passes on 16K and 64K KASAN kernels. No additional test fixture is introduced. Signed-off-by: Diljit Singh (cherry picked from commit e7d10fd228c70c65e932a002296d07a48a0bcdfd) --- arch/arm64/mm/user4k-test.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 42719000b5a1d9..369eb0d459ee5a 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -10406,6 +10406,7 @@ static void fragment_pin_worker_stop(void *arg) if (worker->task) { kthread_stop(worker->task); + put_task_struct(worker->task); worker->task = NULL; } } @@ -10433,13 +10434,16 @@ static void user4k_fragment_pin_race_test(struct kunit *test) for (i = 0; i < 2; i++) { workers[i].mm = vma->vm_mm; init_completion(&workers[i].ready); - workers[i].task = kthread_run(fragment_pin_worker_run, &workers[i], "user4k-pin-%u", i); + workers[i].task = kthread_create(fragment_pin_worker_run, &workers[i], + "user4k-pin-%u", i); if (IS_ERR(workers[i].task)) { workers[i].task = NULL; KUNIT_FAIL(test, "pin worker creation failed"); return; } + get_task_struct(workers[i].task); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, fragment_pin_worker_stop, &workers[i]), 0); + wake_up_process(workers[i].task); KUNIT_ASSERT_NE(test, wait_for_completion_timeout(&workers[i].ready, 10 * HZ), 0UL); } for (round = 0; round < 32; round++) { From 721d6f37662b83c953f7298f3eba66f2d3421022 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 07:39:15 -0400 Subject: [PATCH 356/631] mm: preserve physical subpage offsets in batched PFN remaps Add an explicit physical offset to the PFN range mapping path. Carry it through the existing page-table batching and error cleanup, record it in COW mapping offsets, and track the complete covering native-page range. The existing remap_pfn_range() interface selects offset zero. Use the offset-aware range interface in the system dma-buf heap. Shorten the first scatterlist extent by the starting offset and reset that offset when advancing to the next extent. Physical identity remains independent of the target virtual address. Extend the existing PFN mapping test with noncongruent physical/virtual offsets. 16K/64K KASAN builds and tests pass. Locally adapted existing dma-buf heap tests also verify offset aliases across backing segments for native/4K/16K processes, including coherent writes. Native x86 PFN tracking and system-heap objects build. Physical DMA hardware remains unqualified. Signed-off-by: Diljit Singh (cherry picked from commit eadcabe164613222b98854a43e75b1b35d8c8e0a) --- arch/arm64/mm/user4k-test.c | 15 ++++-- drivers/dma-buf/heaps/system_heap.c | 10 ++-- include/linux/mm.h | 2 + mm/memory.c | 72 ++++++++++++++++++++--------- 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 369eb0d459ee5a..887b2b599bc80d 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5650,6 +5650,7 @@ static void user4k_remap_pfn_range_test(struct kunit *test) for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { unsigned long start = boundaries[shape] - size; unsigned long length = PAGE_SIZE + size; + unsigned int physical_offset = native || !shape ? 0 : size; struct vm_area_struct *vma; int ret; @@ -5661,8 +5662,12 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); mmap_write_lock(mm); - ret = remap_pfn_range(vma, start, folio_pfn(folio), length, - vma->vm_page_prot); + if (physical_offset) + ret = remap_pfn_range_offset(vma, start, folio_pfn(folio), + physical_offset, length, vma->vm_page_prot); + else + ret = remap_pfn_range(vma, start, folio_pfn(folio), length, + vma->vm_page_prot); mmap_write_unlock(mm); KUNIT_ASSERT_EQ(test, ret, 0); for (unsigned long offset = 0; offset < length; offset += size) { @@ -5671,13 +5676,15 @@ static void user4k_remap_pfn_range_test(struct kunit *test) struct follow_pfnmap_args args = { .vma = vma, .address = start + offset + 31, }; - phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + offset + 31; + phys_addr_t expected = PFN_PHYS(folio_pfn(folio)) + + physical_offset + offset + 31; int ret; KUNIT_EXPECT_TRUE(test, pte_present(pte)); KUNIT_EXPECT_TRUE(test, pte_special(pte)); KUNIT_EXPECT_TRUE(test, pte_write(pte)); KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, pte), - PFN_PHYS(folio_pfn(folio)) + offset); + PFN_PHYS(folio_pfn(folio)) + + physical_offset + offset); mmap_read_lock(mm); ret = follow_pfnmap_start(&args); diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index 03c2b87cb11126..a1797ae903004e 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -228,6 +228,7 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) struct sg_table *table = &buffer->sg_table; unsigned long addr = vma->vm_start; unsigned long pgoff = vma->vm_pgoff; + unsigned int offset = vma_subpage_offset(vma); struct scatterlist *sg; pgprot_t prot; int i, ret; @@ -247,17 +248,18 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) for (; sg && addr < vma->vm_end; sg = sg_next(sg)) { unsigned long n = (sg->length >> PAGE_SHIFT) - pgoff; struct page *page = sg_page(sg) + pgoff; - unsigned long size = n << PAGE_SHIFT; + unsigned long size = (n << PAGE_SHIFT) - offset; - if (addr + size > vma->vm_end) - size = vma->vm_end - addr; + size = min(size, vma->vm_end - addr); - ret = remap_pfn_range(vma, addr, page_to_pfn(page), size, prot); + ret = remap_pfn_range_offset(vma, addr, page_to_pfn(page), offset, + size, prot); if (ret) return ret; addr += size; pgoff = 0; + offset = 0; } return 0; diff --git a/include/linux/mm.h b/include/linux/mm.h index 9e621134c8fb19..a49f279b934ff5 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4591,6 +4591,8 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr); int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); +int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, diff --git a/mm/memory.c b/mm/memory.c index dd2916612f9c7e..e302129c5ba49e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3466,13 +3466,13 @@ static int get_remap_pgoff(bool is_cow, unsigned long addr, } static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { pgd_t *pgd; unsigned long next; struct mm_struct *mm = vma->vm_mm; unsigned long end = addr + mm_page_align(mm, size); - phys_addr_t phys = PFN_PHYS(pfn); + phys_addr_t phys = PFN_PHYS(pfn) + offset; int err; if (WARN_ON_ONCE(!IS_ALIGNED(addr, mm_page_size(mm)))) @@ -3500,9 +3500,9 @@ static int remap_pfn_range_internal(struct vm_area_struct *vma, unsigned long ad * must have pre-validated the caching bits of the pgprot_t. */ static int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - int error = remap_pfn_range_internal(vma, addr, pfn, size, prot); + int error = remap_pfn_range_internal(vma, addr, pfn, offset, size, prot); if (!error) return 0; @@ -3546,12 +3546,17 @@ void pfnmap_track_ctx_release(struct kref *ref) } static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { struct pfnmap_track_ctx *ctx = NULL; + unsigned long track_size; int err; size = mm_page_align(vma->vm_mm, size); + if (check_add_overflow(size, (unsigned long)offset, &track_size) || + track_size > ULONG_MAX - (PAGE_SIZE - 1)) + return -EINVAL; + track_size = PAGE_ALIGN(track_size); /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3565,14 +3570,14 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, if (addr == vma->vm_start && addr + size == vma->vm_end) { if (vma->pfnmap_track_ctx) return -EINVAL; - ctx = pfnmap_track_ctx_alloc(pfn, size, &prot); + ctx = pfnmap_track_ctx_alloc(pfn, track_size, &prot); if (IS_ERR(ctx)) return PTR_ERR(ctx); - } else if (pfnmap_setup_cachemode(pfn, size, &prot)) { + } else if (pfnmap_setup_cachemode(pfn, track_size, &prot)) { return -EINVAL; } - err = remap_pfn_range_notrack(vma, addr, pfn, size, prot); + err = remap_pfn_range_notrack(vma, addr, pfn, offset, size, prot); if (ctx) { if (err) kref_put(&ctx->kref, pfnmap_track_ctx_release); @@ -3583,15 +3588,15 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, } static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - return remap_pfn_range_track(vma, addr, pfn, size, prot); + return remap_pfn_range_track(vma, addr, pfn, offset, size, prot); } #else static int do_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, - unsigned long pfn, unsigned long size, pgprot_t prot) + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) { - return remap_pfn_range_notrack(vma, addr, pfn, size, prot); + return remap_pfn_range_notrack(vma, addr, pfn, offset, size, prot); } #endif @@ -3618,7 +3623,7 @@ int remap_pfn_range_prepare(struct vm_area_desc *desc) static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, - unsigned long size) + unsigned int offset, unsigned long size) { const unsigned long end = addr + mm_page_align(vma->vm_mm, size); const bool is_cow = is_cow_mapping(vma->vm_flags); @@ -3629,10 +3634,41 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, if (err) return err; + if (is_cow) + vma_set_page_offset(vma, (struct vm_page_offset) { pfn, offset }); vma_set_flags_mask(vma, VMA_REMAP_FLAGS); return 0; } +/** + * remap_pfn_range_offset - remap a physical range starting within a native page + * @vma: user VMA to map to + * @addr: target address, aligned to the VMA's mapping granule + * @pfn: native page frame containing the first byte + * @offset: byte offset within @pfn, aligned to the mapping granule + * @size: size of the mapping, in bytes + * @prot: page protection flags + * + * The caller must hold the mmap lock. The physical offset is independent of + * the target virtual address. References and error cleanup match remap_pfn_range(). + * + * Return: 0 on success, a negative error code otherwise. + */ +int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) +{ + int err; + + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(vma->vm_mm))) + return -EINVAL; + err = remap_pfn_range_prepare_vma(vma, addr, pfn, offset, size); + if (err) + return err; + + return do_remap_pfn_range(vma, addr, pfn, offset, size, prot); +} +EXPORT_SYMBOL(remap_pfn_range_offset); + /** * remap_pfn_range - remap kernel memory to userspace * @vma: user vma to map to @@ -3648,13 +3684,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma, int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t prot) { - int err; - - err = remap_pfn_range_prepare_vma(vma, addr, pfn, size); - if (err) - return err; - - return do_remap_pfn_range(vma, addr, pfn, size, prot); + return remap_pfn_range_offset(vma, addr, pfn, 0, size, prot); } EXPORT_SYMBOL(remap_pfn_range); @@ -3666,7 +3696,7 @@ int remap_pfn_range_complete(struct vm_area_struct *vma, const unsigned long size = action->remap.size; const pgprot_t prot = action->remap.pgprot; - return do_remap_pfn_range(vma, start, pfn, size, prot); + return do_remap_pfn_range(vma, start, pfn, 0, size, prot); } static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, From 633e2c562b2c0b625b717e8e0944d8a33c5265a8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:00:36 -0400 Subject: [PATCH 357/631] mm: retain physical offsets through I/O mapping preparation Round resource coverage in the target MM's granule and combine the resource start with the VMA's complete byte offset. Validate lengths and offset arithmetic before publishing mappings. Native-page rounding is unchanged. Carry the resulting physical offset through both vm_iomap_memory() and mmap_prepare actions. Keep the native architecture PFN conversion and protection hooks, include the offset in their range, and preserve COW physical identity. The action field is absent without CONFIG_MM_SUBPAGE. Extend the existing PFN mapping test across native, 4K and supported 16K process geometries. Direct and descriptor I/O mappings check noncongruent offsets and reject insufficient resource ranges. 16K/64K KASAN builds and tests pass; native x86 mapping/action objects compile. Physical MMIO device access is not qualified by these RAM-backed fixtures. Signed-off-by: Diljit Singh (cherry picked from commit fa6b5cff34588782b063a10d40df4a4ecf882db8) --- arch/arm64/mm/user4k-test.c | 35 +++++++++++++++---- include/linux/mm.h | 30 +++++++++++++--- include/linux/mm_types.h | 3 ++ mm/internal.h | 6 +++- mm/memory.c | 70 +++++++++++++++++++++++-------------- 5 files changed, 106 insertions(+), 38 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 887b2b599bc80d..89b2bb46f6dc23 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -5635,18 +5635,17 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, folio); KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, test_folio_put, folio), 0); - for (unsigned int native = 0; native < 2; native++) { - struct mm_struct *mm = native ? mm_alloc() : test_vma_mm(test); - unsigned long size, boundaries[3]; + for (unsigned int shift = 12; shift <= PAGE_SHIFT; shift += 2) { + struct mm_struct *mm = test_vma_mm_granule(test, shift); + bool native = shift == PAGE_SHIFT; + unsigned long size, boundaries[4]; KUNIT_ASSERT_NOT_NULL(test, mm); - if (native) - KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, - user4k_vma_mm_free, mm), 0); size = mm_page_size(mm); boundaries[0] = pmd_size_mm(mm); boundaries[1] = pud_size_mm(mm); boundaries[2] = pgd_size_mm(mm); + boundaries[3] = 2 * pmd_size_mm(mm); for (unsigned int shape = 0; shape < ARRAY_SIZE(boundaries); shape++) { unsigned long start = boundaries[shape] - size; unsigned long length = PAGE_SIZE + size; @@ -5662,7 +5661,29 @@ static void user4k_remap_pfn_range_test(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vma); vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); mmap_write_lock(mm); - if (physical_offset) + if (shape >= 2) { + phys_addr_t base = PFN_PHYS(folio_pfn(folio)) + physical_offset; + unsigned long available = folio_size(folio) - physical_offset; + + vma_set_page_offset(vma, (struct vm_page_offset) { + .offset = physical_offset, + }); + if (shape == 2) { + KUNIT_EXPECT_EQ(test, + vm_iomap_memory(vma, base, PAGE_SIZE), -EINVAL); + ret = vm_iomap_memory(vma, base, available); + } else { + struct vm_area_desc desc; + + compat_set_desc_from_vma(&desc, NULL, vma); + mmap_action_simple_ioremap(&desc, base, PAGE_SIZE); + KUNIT_EXPECT_EQ(test, + __compat_vma_mmap(&desc, vma), -EINVAL); + mmap_action_simple_ioremap(&desc, base, available); + ret = __compat_vma_mmap(&desc, vma); + } + physical_offset *= 2; + } else if (physical_offset) ret = remap_pfn_range_offset(vma, start, folio_pfn(folio), physical_offset, length, vma->vm_page_prot); else diff --git a/include/linux/mm.h b/include/linux/mm.h index a49f279b934ff5..86241653559ad4 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4353,6 +4353,15 @@ static inline unsigned long vma_desc_pages(const struct vm_area_desc *desc) return vma_desc_size(desc) >> PAGE_SHIFT; } +static inline unsigned int mmap_action_remap_offset(const struct mmap_action *action) +{ +#ifdef CONFIG_MM_SUBPAGE + return action->remap.offset; +#else + return 0; +#endif +} + /** * mmap_action_remap - helper for mmap_prepare hook to specify that a pure PFN * remap is required. @@ -4378,6 +4387,9 @@ static inline void mmap_action_remap(struct vm_area_desc *desc, action->remap.start_pfn = start_pfn; action->remap.size = size; action->remap.pgprot = desc->page_prot; +#ifdef CONFIG_MM_SUBPAGE + action->remap.offset = 0; +#endif } /** @@ -4728,14 +4740,24 @@ static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, } #endif +static inline int io_remap_pfn_range_offset(struct vm_area_struct *vma, + unsigned long addr, unsigned long orig_pfn, unsigned int offset, + unsigned long size, pgprot_t orig_prot) +{ + const pgprot_t prot = pgprot_decrypted(orig_prot); + unsigned long pfn; + + if (size > ULONG_MAX - offset) + return -EINVAL; + pfn = io_remap_pfn_range_pfn(orig_pfn, size + offset); + return remap_pfn_range_offset(vma, addr, pfn, offset, size, prot); +} + static inline int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long orig_pfn, unsigned long size, pgprot_t orig_prot) { - const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); - const pgprot_t prot = pgprot_decrypted(orig_prot); - - return remap_pfn_range(vma, addr, pfn, size, prot); + return io_remap_pfn_range_offset(vma, addr, orig_pfn, 0, size, orig_prot); } static inline vm_fault_t vmf_error(int err) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 46d6f13f6f3030..180207618dbaea 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -838,6 +838,9 @@ struct mmap_action { unsigned long start_pfn; unsigned long size; pgprot_t pgprot; +#ifdef CONFIG_MM_SUBPAGE + unsigned int offset; +#endif } remap; struct { phys_addr_t start_phys_addr; diff --git a/mm/internal.h b/mm/internal.h index 5ad4e57daedc6a..bd953128ed6f1e 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1923,9 +1923,13 @@ static inline int io_remap_pfn_range_prepare(struct vm_area_desc *desc) const unsigned long orig_pfn = action->remap.start_pfn; const pgprot_t orig_pgprot = action->remap.pgprot; const unsigned long size = action->remap.size; - const unsigned long pfn = io_remap_pfn_range_pfn(orig_pfn, size); + const unsigned int offset = mmap_action_remap_offset(action); + unsigned long pfn; int err; + if (size > ULONG_MAX - offset) + return -EINVAL; + pfn = io_remap_pfn_range_pfn(orig_pfn, size + offset); action->remap.start_pfn = pfn; action->remap.pgprot = pgprot_decrypted(orig_pgprot); err = remap_pfn_range_prepare(desc); diff --git a/mm/memory.c b/mm/memory.c index e302129c5ba49e..8e58ea71b88c0f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3606,16 +3606,23 @@ int remap_pfn_range_prepare(struct vm_area_desc *desc) const unsigned long start = action->remap.start; const unsigned long end = start + action->remap.size; const unsigned long pfn = action->remap.start_pfn; + const unsigned int offset = mmap_action_remap_offset(action); const bool is_cow = vma_desc_is_cow_mapping(desc); int err; if (!range_in_vma_desc(desc, start, end)) return -EFAULT; + if (offset >= PAGE_SIZE || !IS_ALIGNED(offset, mm_page_size(desc->mm))) + return -EINVAL; err = get_remap_pgoff(is_cow, start, end, desc->start, desc->end, pfn, &desc->pgoff); if (err) return err; +#ifdef CONFIG_MM_SUBPAGE + if (is_cow) + desc->subpage_offset = offset; +#endif vma_desc_set_flags_mask(desc, VMA_REMAP_FLAGS); return 0; @@ -3696,40 +3703,43 @@ int remap_pfn_range_complete(struct vm_area_struct *vma, const unsigned long size = action->remap.size; const pgprot_t prot = action->remap.pgprot; - return do_remap_pfn_range(vma, start, pfn, 0, size, prot); + return do_remap_pfn_range(vma, start, pfn, mmap_action_remap_offset(action), size, prot); } -static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, +static int __simple_ioremap_prep(struct mm_struct *mm, unsigned long vm_len, + struct vm_page_offset pos, phys_addr_t start_phys, unsigned long size, - unsigned long *pfnp) + struct vm_page_offset *physical) { - unsigned long pfn, pages; + unsigned long granule = mm_page_size(mm); + unsigned long offset, bias = start_phys & (granule - 1); /* Check that the physical memory area passed in looks valid */ if (start_phys + size < start_phys) return -EINVAL; /* - * You *really* shouldn't map things that aren't page-aligned, - * but we've historically allowed it because IO memory might - * just have smaller alignment. + * Preserve the historical allowance for unaligned I/O resources, using + * the target MM's granule for the prefix and final partial mapping. */ - size += start_phys & ~PAGE_MASK; - pfn = start_phys >> PAGE_SHIFT; - pages = (size + ~PAGE_MASK) >> PAGE_SHIFT; - if (pfn + pages < pfn) + if (check_add_overflow(size, bias, &size) || size > ULONG_MAX - (granule - 1)) return -EINVAL; - - /* We start the mapping 'vm_pgoff' pages into the area */ - if (vm_pgoff > pages) + size = ALIGN(size, granule); + start_phys -= bias; + if (start_phys + size < start_phys) + return -EINVAL; + if (pos.offset >= PAGE_SIZE || !IS_ALIGNED(pos.offset, granule) || + check_mul_overflow(pos.index, PAGE_SIZE, &offset) || + check_add_overflow(offset, (unsigned long)pos.offset, &offset)) return -EINVAL; - pfn += vm_pgoff; - pages -= vm_pgoff; - /* Can we fit all of the mapping? */ - if ((vm_len >> PAGE_SHIFT) + !!offset_in_page(vm_len) > pages) + if (offset > size || vm_len > size - offset) return -EINVAL; - *pfnp = pfn; + start_phys += offset; + physical->index = PHYS_PFN(start_phys); + physical->offset = offset_in_page(start_phys); + if (PFN_PHYS(physical->index) + physical->offset != start_phys) + return -EINVAL; return 0; } @@ -3738,16 +3748,22 @@ int simple_ioremap_prepare(struct vm_area_desc *desc) struct mmap_action *action = &desc->action; const phys_addr_t start = action->simple_ioremap.start_phys_addr; const unsigned long size = action->simple_ioremap.size; - unsigned long pfn; + struct vm_page_offset physical, pos = { .index = desc->pgoff }; int err; - err = __simple_ioremap_prep(vma_desc_size(desc), desc->pgoff, - start, size, &pfn); +#ifdef CONFIG_MM_SUBPAGE + pos.offset = desc->subpage_offset; +#endif + err = __simple_ioremap_prep(desc->mm, vma_desc_size(desc), pos, + start, size, &physical); if (err) return err; /* The I/O remap logic does the heavy lifting. */ - mmap_action_ioremap_full(desc, pfn); + mmap_action_ioremap_full(desc, physical.index); +#ifdef CONFIG_MM_SUBPAGE + action->remap.offset = physical.offset; +#endif return io_remap_pfn_range_prepare(desc); } @@ -3771,15 +3787,17 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long const unsigned long vm_start = vma->vm_start; const unsigned long vm_end = vma->vm_end; const unsigned long vm_len = vm_end - vm_start; - unsigned long pfn; + struct vm_page_offset physical; int err; - err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); + err = __simple_ioremap_prep(vma->vm_mm, vm_len, vma_page_offset_at(vma, vm_start), + start, len, &physical); if (err) return err; /* Ok, let it rip */ - return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); + return io_remap_pfn_range_offset(vma, vm_start, physical.index, physical.offset, + vm_len, vma->vm_page_prot); } EXPORT_SYMBOL(vm_iomap_memory); From 84b72f9dbdf1584aee420bf0d59a863706a09ffd Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:05:14 -0400 Subject: [PATCH 358/631] mm: preserve the PFN remap interface on no-MMU kernels Provide a zero-offset wrapper for remap_pfn_range_offset() without an MMU. It retains the existing identity-mapping checks and rejects offsets that this configuration cannot represent. MMU kernels keep the range mapper. ARM MPS2 no-MMU builds of mm/nommu.o and the system heap pass; their symbols contain no unresolved offset-remap reference. Native x86 mapping and action objects also build. No no-MMU runtime test was performed. Signed-off-by: Diljit Singh (cherry picked from commit 207036cd20c2f5b43958146873fdba2d0d1d6b3b) --- include/linux/mm.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index 86241653559ad4..0536bb678c2e99 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4603,8 +4603,18 @@ struct vm_area_struct *find_extend_vma_locked(struct mm_struct *, unsigned long addr); int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t pgprot); +#ifdef CONFIG_MMU int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot); +#else +static inline int remap_pfn_range_offset(struct vm_area_struct *vma, unsigned long addr, + unsigned long pfn, unsigned int offset, unsigned long size, pgprot_t prot) +{ + if (offset) + return -EINVAL; + return remap_pfn_range(vma, addr, pfn, size, prot); +} +#endif int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages_range(struct vm_area_struct *vma, unsigned long addr, From 879b30e99e2f757b6bcbc7fd116d2bab60ede9e3 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:13:02 -0400 Subject: [PATCH 359/631] userfaultfd: update published folio flags atomically Shared shmem backing and neighboring retained slots can already expose the folio while a fill operation updates it. Holding the folio lock and PTL does not exclude other atomic changes to the folio flags word. Use folio_mark_uptodate() for both published-backing paths, preserving the write barrier while avoiding a non-atomic flags update. Keep private allocation initialization unchanged. 16K/64K KASAN builds and the existing userfaultfd contract suite pass for native/4K and supported 16K processes. No new fixture was added and the specific flag-word race was not forced at runtime. Signed-off-by: Diljit Singh (cherry picked from commit f2928b082c7a0e6e5b7588180daebce0f1efbea8) --- mm/userfaultfd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 8d72d991b05e50..e7327a67499629 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -438,7 +438,7 @@ static int mfill_atomic_install_pte(pmd_t *dst_pmd, } kunmap_local(dst); flush_dcache_folio(folio); - __folio_mark_uptodate(folio); + folio_mark_uptodate(folio); folio_mark_dirty(folio); } #endif @@ -813,7 +813,7 @@ static int mfill_atomic_subpage(struct mfill_state *state) } } /* The allocator zeroed every byte, including unissued neighbours. */ - __folio_mark_uptodate(folio); + folio_mark_uptodate(folio); vma = state->vma; pte = pte_offset_map_lock(mm, state->pmd, state->dst_addr, &ptl); err = -EAGAIN; From 61358693dfa6ec28c38971ec88cd50a3f8062b31 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:48:13 -0400 Subject: [PATCH 360/631] tomoyo: inspect exec arguments through virtual-byte access Argument staging can use a smaller userspace granule, where legacy page-array GUP cannot represent the backing. Read the requested virtual bytes through access_remote_vm(), retaining the existing native-page buffer and caller iteration units. Keep the no-MMU page-array path. Cache the first copied virtual position instead of a backing-page pointer. Refill on a backward access outside the cached tail and invalidate before a potentially partial copy. Multiple fragments may contribute to one cached native-sized virtual range. Local extensions of existing exec tests verify argument bytes and cache rewinds on 16K/64K KASAN kernels for all supported process granules. Native x86 and ARM no-MMU TOMOYO objects compile. The temporary test hooks are not submitted; full policy enforcement was not exercised. Signed-off-by: Diljit Singh (cherry picked from commit 3001cf4667e728ce38f3453aa98e5995af1ca098) --- security/tomoyo/common.h | 4 ++-- security/tomoyo/domain.c | 38 +++++++++++++++----------------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h index d098cf8aae6109..b859f099ae9eac 100644 --- a/security/tomoyo/common.h +++ b/security/tomoyo/common.h @@ -575,8 +575,8 @@ struct tomoyo_mini_stat { /* Structure for dumping argv[] and envp[] of "struct linux_binprm". */ struct tomoyo_page_dump { - struct page *page; /* Previously dumped page. */ - char *data; /* Contents of "page". Size is PAGE_SIZE. */ + unsigned long pos; /* First cached virtual byte, or ULONG_MAX. */ + char *data; /* Buffer for one native page of virtual bytes. */ }; /* Structure for attribute checks in addition to pathname checks. */ diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c index eeaad421f15baa..2ff8d783f54cc2 100644 --- a/security/tomoyo/domain.c +++ b/security/tomoyo/domain.c @@ -906,45 +906,37 @@ int tomoyo_find_next_domain(struct linux_binprm *bprm) bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, struct tomoyo_page_dump *dump) { - struct page *page; -#ifdef CONFIG_MMU - int ret; -#endif + const unsigned int offset = offset_in_page(pos); + const unsigned int length = PAGE_SIZE - offset; /* dump->data is released by tomoyo_find_next_domain(). */ if (!dump->data) { dump->data = kzalloc(PAGE_SIZE, GFP_NOFS); if (!dump->data) return false; + dump->pos = ULONG_MAX; } - /* Same with get_arg_page(bprm, pos, 0) in fs/exec.c */ + if (pos >= dump->pos && (pos & PAGE_MASK) == (dump->pos & PAGE_MASK)) + return true; + /* A failed partial copy must not leave the previous cache valid. */ + dump->pos = ULONG_MAX; #ifdef CONFIG_MMU /* - * This is called at execve() time in order to dig around - * in the argv/environment of the new process - * (represented by bprm). + * Inspect the nascent MM's virtual bytes. Smaller userspace mappings + * can use several independently retained fragments within this range. */ - mmap_read_lock(bprm->mm); - ret = get_user_pages_remote(bprm->mm, pos, 1, - FOLL_FORCE, &page, NULL); - mmap_read_unlock(bprm->mm); - if (ret <= 0) + if (access_remote_vm(bprm->mm, pos, dump->data + offset, length, + FOLL_FORCE) != length) return false; #else - page = bprm->page[pos / PAGE_SIZE]; -#endif - if (page != dump->page) { - const unsigned int offset = pos % PAGE_SIZE; + { + struct page *page = bprm->page[pos / PAGE_SIZE]; char *kaddr = kmap_local_page(page); - dump->page = page; - memcpy(dump->data + offset, kaddr + offset, - PAGE_SIZE - offset); + memcpy(dump->data + offset, kaddr + offset, length); kunmap_local(kaddr); } - /* Same with put_arg_page(page) in fs/exec.c */ -#ifdef CONFIG_MMU - put_page(page); #endif + dump->pos = pos; return true; } From 26a0ca12700395b2f29cd1fe2bb615f27e60100f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 08:48:13 -0400 Subject: [PATCH 361/631] exec: retain the selected user namespace when replacing the staging mm would_dump() can refine the staging MM's namespace for the interpreter before final ELF geometry selection. A replacement MM allocated from the current credentials must not discard that earlier policy decision. Transfer a namespace reference from the old MM after preparation succeeds, before publishing the replacement. Failed preparation leaves the old MM and namespace untouched. Local extensions of existing exec tests verify retention of a distinct namespace during granule changes on 16K/64K KASAN kernels. Native x86 and ARM no-MMU exec objects build. No privileged-exec experiment was performed. Signed-off-by: Diljit Singh (cherry picked from commit a8100f86f408900dc7f2a75c0b0f62a1b3021d8f) --- fs/exec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/exec.c b/fs/exec.c index cb5cda32f8e304..c9e6ecf187b7b6 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -439,6 +439,8 @@ int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift) } old_mm = bprm->mm; + put_user_ns(replacement.mm->user_ns); + replacement.mm->user_ns = get_user_ns(old_mm->user_ns); acct_arg_size(bprm, 0); bprm->mm = replacement.mm; bprm->vma = replacement.vma; From dd567354beae38e11e67c4808ead19f9d2a1337e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 09:37:17 -0400 Subject: [PATCH 362/631] block: estimate user vectors with the process fragment size Fragment pinning emits records in mm_user_fragment_size() intervals. Count the same intervals when reserving vectors, including unaligned segments and saturation at the caller's limit. Fixed 4K counting otherwise overestimates 16K userspace buffers on a 64K kernel, affecting allocation and direct-I/O path selection. Local extensions of the existing BIO tests cover native and smaller granules on 16K/64K KASAN kernels. The native x86 BIO object builds. Signed-off-by: Diljit Singh (cherry picked from commit e139642c1fac2cfce399d3b6cd8d229fbe7cf93c) --- block/bio.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/block/bio.c b/block/bio.c index 846417dec7d164..37fed255563987 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1247,10 +1247,12 @@ static bool bio_subpage_iter(const struct iov_iter *iter) int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) { struct iov_iter walk = *iter; + unsigned long size; unsigned int count = 0; if (!bio_subpage_iter(iter)) return iov_iter_npages(iter, max_vecs); + size = mm_user_fragment_size(current->mm); while (iov_iter_count(&walk) && count < max_vecs) { size_t len, offset, nr; @@ -1258,9 +1260,8 @@ int bio_iov_user_vecs(const struct iov_iter *iter, int max_vecs) len = min(iter_iov_len(&walk), iov_iter_count(&walk)); if (WARN_ON_ONCE(!len)) return max_vecs; - offset = (unsigned long)iter_iov_addr(&walk) & (MM_SUBPAGE_SIZE - 1); - nr = (len >> MM_SUBPAGE_SHIFT) + - DIV_ROUND_UP(offset + (len & (MM_SUBPAGE_SIZE - 1)), MM_SUBPAGE_SIZE); + offset = (unsigned long)iter_iov_addr(&walk) & (size - 1); + nr = len / size + DIV_ROUND_UP(offset + (len & (size - 1)), size); if (nr >= max_vecs - count) return max_vecs; count += nr; From ec64dea9da3b8e28c7972319ace379ffa22cfd9d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 09:37:17 -0400 Subject: [PATCH 363/631] block: preserve nofault when pinning user fragments Pass FOLL_NOFAULT when the iterator forbids faulting, matching ordinary iterator extraction. Filesystems set this flag while holding locks that must not be reacquired through a user-buffer fault. Keep existing partial-progress and cleanup behavior. Local extensions of the existing BIO tests check missing and resident mappings with nofault set on 16K/64K KASAN kernels. Filesystem-specific locking paths have not been exercised. Signed-off-by: Diljit Singh (cherry picked from commit 1de4d3c0c6bdd60c98f992aa259ab3ed1eb4c397) --- block/bio.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/block/bio.c b/block/bio.c index 37fed255563987..19805b79cf0b26 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1283,6 +1283,8 @@ static int bio_iov_iter_get_fragments(struct bio *bio, struct iov_iter *iter, if (WARN_ON_ONCE(bio->bi_vcnt || bio->bi_user_fragments || !capacity)) return -EINVAL; + if (iter->nofault) + flags |= FOLL_NOFAULT; bio->bi_user_fragments = kmalloc_array(capacity, sizeof(*bio->bi_user_fragments), GFP_KERNEL); if (!bio->bi_user_fragments) From 63acaf1db6c22dfc761f41546121d2e899b684a8 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 364/631] net: allocate a new stream zerocopy owner after subpage splice A queued TCP skb can carry retained subpage references in its ubuf_info. A subsequent MSG_ZEROCOPY send cannot extend that owner as a message completion record. Allocate a fresh record for stream sockets and let the existing incompatible-owner handling select another skb, as it already does when a completion record reaches its size limit. Keep the datagram behavior unchanged. Mixed-granule pipe producers and native TCP consumers passed both send orders on IPv4/IPv6 loopback in 16K/64K KASAN guests, including completion notifications. Signed-off-by: Diljit Singh (cherry picked from commit 4043902ee814ee5c94e1bd9c97f35dd81493a743) --- net/core/skbuff.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 0bc6467aae5d4b..91ee22a7c6cba7 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1767,8 +1767,11 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, u32 bytelen, next; /* there might be non MSG_ZEROCOPY users */ - if (uarg->ops != &msg_zerocopy_ubuf_ops) + if (uarg->ops != &msg_zerocopy_ubuf_ops) { + if (sk->sk_type == SOCK_STREAM) + goto new_alloc; return NULL; + } /* realloc only when socket is locked (TCP, UDP cork), * so uarg->len and sk_zckey access is serialized From c0ee18d99be6eab83080d403139da2284fcf8e5e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 365/631] tcp: start a new segment for incompatible splice ownership Splicing a retained subpage into a queued MSG_ZEROCOPY skb needs its own fragment owner. Report this conflict as EEXIST and retry on a new TCP segment, matching the reverse zerocopy transition. Preserve the iterator position and the existing owner's completion references. IPv4/IPv6 loopback checks cover both operation orders, exact bytes and completion notifications on 16K/64K KASAN guests. Existing SKB clone, trim and retained-reference tests also pass. No NIC throughput claim is made by these checks. Signed-off-by: Diljit Singh (cherry picked from commit f22a44fa973b4b96923246686d8d277ef13f8785) --- net/core/skbuff.c | 5 +++-- net/ipv4/tcp.c | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 91ee22a7c6cba7..090a26ed9f2db2 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -7427,7 +7427,7 @@ static int skb_splice_hold_subpages(struct sk_buff *skb, struct page *page, return 0; mm_subpage_put(probe); if (old && (skb_zcopy_is_nouarg(skb) || old->ops != &skb_subpage_ops)) - return -EOPNOTSUPP; + return -EEXIST; refs = old ? container_of(old, struct skb_subpage_refs, ubuf) : NULL; if (!refs || refcount_read(&old->refcnt) != 1) { new = kzalloc(sizeof(*new), GFP_ATOMIC); @@ -7481,7 +7481,8 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page, * pages). * * Returns the amount of data spliced/copied or -EMSGSIZE if there's - * insufficient space in the buffer to transfer anything. + * insufficient space in the buffer to transfer anything, or -EEXIST if the + * buffer already has an incompatible fragment owner. */ ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter, ssize_t maxsize) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 389a7cc17110da..05c8c9504bb3c0 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1356,7 +1356,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) err = skb_splice_from_iter(skb, &msg->msg_iter, copy); if (err < 0) { - if (err == -EMSGSIZE) { + if (err == -EMSGSIZE || err == -EEXIST) { tcp_mark_push(tp, skb); goto new_segment; } From 9bac64311280c1e6e640fad7aa9a5da882035c00 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 10:02:10 -0400 Subject: [PATCH 366/631] selftests/arm64: derive splice fixture sizes from runtime limits The splice fixture assumes a 16K native pipe page, a corked payload that fits below TCP's MSS, and a process page smaller than a UDP datagram. Those assumptions fail on a 64K kernel even for native processes. Discover the native pipe allocation unit, size the test mapping to hold it, and bound corked TCP and UDP payloads by their protocol limits. Retain the data, ownership, teardown and capacity assertions. The same existing fixture now passes its 148 checks on both 16K and 64K kernels. Signed-off-by: Diljit Singh (cherry picked from commit 260567108121b13af59df7430fb2137b6558e2cb) --- .../arm64/user4k/splice-fixture-init.c | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c index 282ba6268b25d9..bac72da15f40ec 100644 --- a/tools/testing/selftests/arm64/user4k/splice-fixture-init.c +++ b/tools/testing/selftests/arm64/user4k/splice-fixture-init.c @@ -336,8 +336,12 @@ static void tls_checks(size_t ps, unsigned char *p, size_t len, static int exercise(void) { - const size_t ps = getauxval(AT_PAGESZ), len = 8 * ps; - unsigned char *p, *back = malloc(len); + const size_t ps = getauxval(AT_PAGESZ); + size_t len; + unsigned char *p, *back; + int native; + unsigned int mss = 0; + socklen_t mss_len = sizeof(mss); int fds[2], dup[2], file, sockets[2]; struct iovec v[3]; struct stat self, init; @@ -348,9 +352,17 @@ static int exercise(void) if (stat("/proc/self/exe", &self) || stat("/proc/1/exe", &init) || self.st_dev != init.st_dev || self.st_ino != init.st_ino) return 2; + if (pipe(fds) || pipe(dup)) + ksft_exit_fail_msg("pipe setup\n"); + /* A pipe has at least one native page, independently of this MM. */ + native = fcntl(fds[1], F_SETPIPE_SZ, 1); + if (native <= 0 || fcntl(fds[1], F_SETPIPE_SZ, 16 * native) < 0) + ksft_exit_fail_msg("pipe capacity setup\n"); + len = 8 * ps > (size_t)native ? 8 * ps : (size_t)native; + back = malloc(len); p = mmap((void *)0x300000000UL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); - if (p == MAP_FAILED || !back || pipe(fds) || pipe(dup)) + if (p == MAP_FAILED || !back) ksft_exit_fail_msg("setup\n"); ksft_print_header(); ksft_set_plan(use_tcp ? 22 : 10); @@ -370,21 +382,21 @@ static int exercise(void) ok = read_exact(fds[0], back, result) && check(back, ps - 17, 17) && check(back + ps - 17, 2 * ps - 31, 2 * ps + 31); report(ok, ps, "unaligned fragmented iovecs and empty segment"); - v[0] = (struct iovec){ p, 16384 }; - ok = fcntl(fds[1], F_SETPIPE_SZ, 16384) == 16384; + v[0] = (struct iovec){ p, native }; + ok = fcntl(fds[1], F_SETPIPE_SZ, native) == native; result = vmsplice(fds[1], v, 1, SPLICE_F_NONBLOCK); { int pending = 0; - ok &= result == 16384 && !ioctl(fds[0], FIONREAD, &pending) && - pending == 16384; + ok &= result == native && !ioctl(fds[0], FIONREAD, &pending) && + pending == native; } if (result > 0) ok &= read_exact(fds[0], back, result) && check(back, result, 0); report(ok, ps, - "contiguous quarters retain one-native-page pipe capacity"); - fcntl(fds[1], F_SETPIPE_SZ, 16 * 16384); + "contiguous fragments retain one-native-page pipe capacity"); + fcntl(fds[1], F_SETPIPE_SZ, 16 * native); v[0] = (struct iovec){ p + 11, 3 * ps - 11 }; result = vmsplice(fds[1], v, 1, 0); if (result != (ssize_t)(3 * ps - 11) || @@ -422,14 +434,22 @@ static int exercise(void) ksft_exit_fail_msg("socketpair errno=%d\n", errno); setsockopt(sockets[1], SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + if (use_tcp && + (getsockopt(sockets[0], IPPROTO_TCP, TCP_MAXSEG, &mss, &mss_len) || mss <= 1)) + ksft_exit_fail_msg("TCP_MAXSEG setup\n"); for (unsigned int mode = 0; mode < 3; mode++) { - const size_t bytes = mode == 1 ? ps : + size_t bytes = mode == 1 ? ps : (use_udp && ps == 4096 ? 6 * ps - 256 : 2 * ps - 256); const size_t offset = mode == 1 ? 0 : ps + 29; int cork = 1; + /* Keep the corked payload below one segment; UDP has a byte limit. */ + if (use_tcp && mode < 2 && bytes >= mss) + bytes = mss - 1; + if (use_udp && bytes > 65507) + bytes = 65507; if ((use_tcp || use_udp) && mode < 2 && setsockopt(sockets[0], use_tcp ? IPPROTO_TCP : IPPROTO_UDP, use_tcp ? TCP_CORK : UDP_CORK, &cork, @@ -497,12 +517,14 @@ static int exercise(void) close(sockets[1]); ok = true; for (unsigned int round = 0; round < 128 && ok; round++) { + const size_t bytes = use_udp && ps > 65507 ? 65507 : ps; + fill(p, len); ok = !connected_pair(sockets); - v[0] = (struct iovec){ p + ps, ps }; - ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)ps && - splice(fds[0], NULL, sockets[0], NULL, ps, 0) == - (ssize_t)ps; + v[0] = (struct iovec){ p + ps, bytes }; + ok &= vmsplice(fds[1], v, 1, 0) == (ssize_t)bytes && + splice(fds[0], NULL, sockets[0], NULL, bytes, 0) == + (ssize_t)bytes; close(sockets[1]); close(sockets[0]); ok &= !madvise(p + ps, ps, MADV_DONTNEED); From 0057d36c16df84c21e3d1f75d2b37a234055c72d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 12:24:35 -0400 Subject: [PATCH 367/631] arm64: align mixed-granule KUnit cases with migration dependencies The migration guard accidentally encloses the VMA split and brk tests, but their registrations are unconditional. Keep those definitions available without migration and guard the reclaim cases which explicitly exercise migration. Guard the userfaultfd MOVE/migration fixture and its registration too; a migration-disabled build exposed its unguarded reference to the migration allocator. The existing KUnit object builds on 16K and 64K configurations with migration both enabled and disabled. No new tests are added. Signed-off-by: Diljit Singh (cherry picked from commit 76088e05aa1e391fdcf2d19944c2bf2f2412e3a7) --- arch/arm64/mm/user4k-test.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 89b2bb46f6dc23..38a6897e090e2a 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9387,6 +9387,7 @@ static long test_madvise(unsigned long addr, unsigned long size, unsigned long b return __arm64_sys_madvise(®s); } +#endif static void user_granule_large_vma_split_test(struct kunit *test) { @@ -9475,6 +9476,7 @@ static void user_granule_brk_extent_test(struct kunit *test) } } +#ifdef CONFIG_MIGRATION static void user4k_droppable_reclaim_test(struct kunit *test) { unsigned int mode; @@ -11708,6 +11710,7 @@ static ssize_t test_uffd_move(struct vm_area_struct *dst, unsigned long dst_addr return ret; } +#ifdef CONFIG_MIGRATION struct move_migration_worker { struct userfaultfd_ctx ctx; struct vm_area_struct *vma[2]; @@ -11863,6 +11866,8 @@ static void user4k_move_migration_race_test(struct kunit *test) } } +#endif + static void user4k_move_swap_get_test(struct kunit *test) { unsigned int variant; @@ -13420,7 +13425,9 @@ static struct kunit_suite user4k_shmem_swap_suite = { static struct kunit_case user4k_nonlinear_swap_cases[] = { #ifdef CONFIG_USERFAULTFD KUNIT_CASE(user4k_move_swap_get_test), +#ifdef CONFIG_MIGRATION KUNIT_CASE(user4k_move_migration_race_test), +#endif #endif KUNIT_CASE(user4k_nonlinear_swap_cached_test), KUNIT_CASE(user4k_nonlinear_swap_disk_test), @@ -14214,8 +14221,10 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_mlock_repair_test), KUNIT_CASE(user_granule_large_vma_split_test), KUNIT_CASE(user_granule_brk_extent_test), +#ifdef CONFIG_MIGRATION KUNIT_CASE(user4k_droppable_reclaim_test), KUNIT_CASE(user4k_lazyfree_test), +#endif KUNIT_CASE(user4k_zero_cow_test), KUNIT_CASE(user16k_zero_cow_test), KUNIT_CASE(user4k_shmem_swap_usage_test), From ef5a791d3a0a49fb54358a03f894f5aaae41112d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:40:54 -0400 Subject: [PATCH 368/631] io_uring: retain shared fragment charges through buffer teardown - Track fragment-backed registrations in a refcounted account for each user/MM counter pair. Charge each backing folio once and release its charge with the final retained fragment, including cloned buffers. - Keep the account alive independently of the originating ring. Roll back only the references acquired by failed preparation. - Serialize account updates locally; cache the registry lookup per ring. Preserve existing ring-context member offsets and keep native backing owners in their separate accounting path. - Validate with 16K/64K KASAN and lockdep builds, 110 guest lifecycle checks, and an x86 io_uring build. Warmed emulated registration measurements do not reproduce the earlier native timing discrepancy; hardware overhead and total metadata costs remain unqualified. Signed-off-by: Diljit Singh (cherry picked from commit d90fab78cd1b2044701c39b8593891a9b514b05f) --- include/linux/io_uring_types.h | 3 + io_uring/rsrc.c | 161 ++++++++++++++++++++++++++++++--- 2 files changed, 151 insertions(+), 13 deletions(-) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index f61b6cad229b4a..e29fcd83060eb6 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -512,6 +512,9 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; +#ifdef CONFIG_MM_SUBPAGE + struct io_fragment_account *fragment_account; +#endif }; /* diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 33cc1e26884801..b442d5cf6c7f5d 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,10 @@ struct io_rsrc_update { static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov, struct page **last_hpage); +#ifdef CONFIG_MM_SUBPAGE +static void io_fragment_account_put(struct io_fragment_account *account); +#endif + /* only define max */ #define IORING_MAX_FIXED_FILES (1U << 20) #define IORING_MAX_REG_BUFFERS (1U << 14) @@ -168,6 +173,10 @@ bool io_rsrc_cache_init(struct io_ring_ctx *ctx) void io_rsrc_cache_free(struct io_ring_ctx *ctx) { +#ifdef CONFIG_MM_SUBPAGE + io_fragment_account_put(ctx->fragment_account); + ctx->fragment_account = NULL; +#endif io_alloc_cache_free(&ctx->node_cache, kfree); io_alloc_cache_free(&ctx->imu_cache, kvfree); } @@ -635,6 +644,9 @@ static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, if (!node) continue; imu = node->buf; + /* Other backing owners have independent charge lifetimes. */ + if (imu->release != io_release_ubuf) + continue; for (j = 0; j < imu->nr_bvecs; j++) { if (!PageCompound(imu->bvec[j].bv_page)) continue; @@ -764,14 +776,145 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages, } #ifdef CONFIG_MM_SUBPAGE +/* + * Shared by rings charging the same counters, including cloned buffers. + * uring_lock nests outside lock; the registry lock is never held with lock. + * Long-term pins keep each indexed folio and its size stable until release. + */ +struct io_fragment_account { + struct hlist_node node; + refcount_t refs; + struct mutex lock; + struct xarray folios; + struct user_struct *user; + struct mm_struct *mm; +}; + +static DEFINE_HASHTABLE(io_fragment_accounts, 6); +static DEFINE_MUTEX(io_fragment_accounts_lock); + +static void io_fragment_account_put(struct io_fragment_account *account) +{ + if (!account || !refcount_dec_and_mutex_lock(&account->refs, + &io_fragment_accounts_lock)) + return; + hash_del(&account->node); + mutex_unlock(&io_fragment_accounts_lock); + WARN_ON_ONCE(!xa_empty(&account->folios)); + xa_destroy(&account->folios); + if (account->mm) + mmdrop(account->mm); + if (account->user) + free_uid(account->user); + kfree(account); +} + +static struct io_fragment_account *io_fragment_account_get(struct io_ring_ctx *ctx) +{ + unsigned long key = (unsigned long)ctx->mm_account ^ (unsigned long)ctx->user; + struct io_fragment_account *account; + + lockdep_assert_held(&ctx->uring_lock); + account = ctx->fragment_account; + if (account) + goto found; + + mutex_lock(&io_fragment_accounts_lock); + hash_for_each_possible(io_fragment_accounts, account, node, key) { + if (account->user == ctx->user && account->mm == ctx->mm_account) { + refcount_inc(&account->refs); + goto install; + } + } + account = kzalloc(sizeof(*account), GFP_KERNEL_ACCOUNT); + if (!account) { + mutex_unlock(&io_fragment_accounts_lock); + return NULL; + } + refcount_set(&account->refs, 1); + mutex_init(&account->lock); + xa_init(&account->folios); + account->user = ctx->user ? get_uid(ctx->user) : NULL; + account->mm = ctx->mm_account; + if (account->mm) + mmgrab(account->mm); + hash_add(io_fragment_accounts, &account->node, key); +install: + ctx->fragment_account = account; + mutex_unlock(&io_fragment_accounts_lock); +found: + /* The buffer retains its account after the originating ring exits. */ + refcount_inc(&account->refs); + return account; +} + struct io_fragment_buffer { unsigned int count; + unsigned int accounted; + struct io_fragment_account *account; struct user_page_fragment fragments[]; }; +static int io_account_fragment_buffer(struct io_fragment_buffer *buffer) +{ + struct io_fragment_account *account = buffer->account; + unsigned int i; + int ret = 0; + + mutex_lock(&account->lock); + for (i = 0; i < buffer->count; i++) { + struct folio *folio = buffer->fragments[i].folio; + unsigned long pfn = folio_pfn(folio); + void *entry = xa_load(&account->folios, pfn); + unsigned long refs = entry ? xa_to_value(entry) : 0; + + if (refs == LONG_MAX) { + ret = -EOVERFLOW; + break; + } + ret = xa_err(xa_store(&account->folios, pfn, xa_mk_value(refs + 1), + GFP_KERNEL_ACCOUNT)); + if (ret) + break; + if (!refs) { + ret = io_account_mem(account->user, account->mm, + folio_nr_pages(folio)); + if (ret) { + xa_erase(&account->folios, pfn); + break; + } + } + buffer->accounted++; + } + mutex_unlock(&account->lock); + return ret; +} + static void io_release_fragment_buffer(void *priv) { struct io_fragment_buffer *buffer = priv; + struct io_fragment_account *account = buffer->account; + unsigned int i; + + if (account) { + mutex_lock(&account->lock); + for (i = 0; i < buffer->accounted; i++) { + struct folio *folio = buffer->fragments[i].folio; + unsigned long pfn = folio_pfn(folio); + unsigned long refs = xa_to_value(xa_load(&account->folios, pfn)); + + if (refs == 1) { + xa_erase(&account->folios, pfn); + io_unaccount_mem(account->user, account->mm, + folio_nr_pages(folio)); + } else { + xa_store(&account->folios, pfn, xa_mk_value(refs - 1), + GFP_NOWAIT); + } + } + mutex_unlock(&account->lock); + io_fragment_account_put(account); + } release_user_fragments(buffer->fragments, buffer->count, false); kvfree(buffer); @@ -786,9 +929,7 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, struct io_fragment_buffer *buffer; struct io_mapped_ubuf *imu = NULL; struct io_rsrc_node *node; - unsigned long charged = 0; unsigned int i; - struct xarray seen; long nr; int ret = -ENOMEM; @@ -799,6 +940,8 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, if (!buffer) goto free_node; buffer->count = 0; + buffer->accounted = 0; + buffer->account = NULL; nr = pin_user_fragments_remote(current->mm, (unsigned long)iov->iov_base, iov->iov_len, FOLL_WRITE | FOLL_LONGTERM, buffer->fragments, count); if (nr > 0) @@ -812,26 +955,18 @@ static struct io_rsrc_node *io_register_fragment_buffer(struct io_ring_ctx *ctx, goto free_buffer; imu->nr_bvecs = count; imu->acct_pages = 0; - /* Several user leaves can share native backing, including aliases. */ - xa_init(&seen); for (i = 0; i < count; i++) { struct user_page_fragment *f = &buffer->fragments[i]; - ret = xa_insert(&seen, folio_pfn(f->folio), xa_mk_value(1), GFP_KERNEL); - if (!ret) - charged += folio_nr_pages(f->folio); - else if (ret != -EBUSY) - break; bvec_set_page(&imu->bvec[i], folio_page(f->folio, f->offset >> PAGE_SHIFT), f->length, offset_in_page(f->offset)); } - xa_destroy(&seen); - if (ret && ret != -EBUSY) + buffer->account = io_fragment_account_get(ctx); + if (!buffer->account) goto free_imu; - ret = io_account_mem(ctx->user, ctx->mm_account, charged); + ret = io_account_fragment_buffer(buffer); if (ret) goto free_imu; - imu->acct_pages = charged; imu->ubuf = (unsigned long)iov->iov_base; imu->len = iov->iov_len; imu->folio_shift = mm_page_shift(current->mm); From 0497243d42f942268ffa6dd4f49cfb95890ec253 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:46:31 -0400 Subject: [PATCH 369/631] mm: serialize subpage lazyfree cancellation with discard checks - Take the pool lock when cancelling lazyfree so discard eligibility cannot combine an earlier lazyfree state with a later reference count. - Keep cancellation ordered before releasing the caller's retained reference. Existing callers acquire the helper outside the pool lock. - Validate with 16K/64K KASAN and lockdep builds and existing lazyfree and ownership suites; no concurrent-race or hardware reproduction claimed. Signed-off-by: Diljit Singh (cherry picked from commit 808513ac0c8a339799e0eb8650d2e2b0fd388e38) --- include/linux/mm_subpage.h | 1 + mm/user-subpage.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/include/linux/mm_subpage.h b/include/linux/mm_subpage.h index 4a6224f8cc95a4..81f70bed1f2c89 100644 --- a/include/linux/mm_subpage.h +++ b/include/linux/mm_subpage.h @@ -251,6 +251,7 @@ bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage); bool mm_subpage_all_lazyfree(struct mm_subpage *subpage); bool mm_subpage_discardable(struct mm_subpage *subpage); bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage); +/* Hold a slot reference through cancellation; serializes with discard checks. */ void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage); /* * Caller holds a live slot reference and its PTE lock. Success takes an diff --git a/mm/user-subpage.c b/mm/user-subpage.c index cff26e5b5c8036..c08a816b90b204 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -1884,7 +1884,12 @@ bool mm_subpage_is_lazyfree(const struct mm_subpage *subpage) void mm_subpage_cancel_lazyfree(struct mm_subpage *subpage) { + struct mm_subpage_pool *pool = subpage->owner->pool; + unsigned long flags; + + spin_lock_irqsave(&pool->lock, flags); WRITE_ONCE(subpage->lazyfree, false); + spin_unlock_irqrestore(&pool->lock, flags); } bool mm_subpage_mark_lazyfree(struct mm_subpage *subpage) From 26583284c4c4d79740f92738e688352fb19ec13e Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 13:46:32 -0400 Subject: [PATCH 370/631] arm64: manage lazyfree test fragments through assertion cleanup - Keep retained fragments in KUnit-managed storage with a release action so a fatal assertion cannot strand the backing reference. - Use the already-retained folio when exercising reclaim; transient migration PTEs must not turn a successful GET into a failed lookup. - Keep the existing eleven test modes. Both 16K/64K lazyfree runs pass; no concurrent memory-offline experiment was performed. Signed-off-by: Diljit Singh (cherry picked from commit 5c9f905211e7abb99fe60c9eb21872938d1e67b2) --- arch/arm64/mm/user4k-test.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 38a6897e090e2a..6b5aeef7be61db 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -9574,6 +9574,11 @@ static void user4k_droppable_reclaim_test(struct kunit *test) } } +static void test_release_fragment(void *arg) +{ + release_user_fragments(arg, 1, false); +} + static void user4k_lazyfree_test(struct kunit *test) { unsigned int mode; @@ -9586,7 +9591,7 @@ static void user4k_lazyfree_test(struct kunit *test) for (mode = 0; mode < 11; mode++) { struct vm_area_struct *vma = test_fault_vma(test, TEST_VA, PAGE_SIZE); struct mm_struct *mm; - struct user_page_fragment held = {}; + struct user_page_fragment *held; struct folio *folio; struct mm_subpage *slot; unsigned int i, succeeded = 0; @@ -9595,11 +9600,15 @@ static void user4k_lazyfree_test(struct kunit *test) LIST_HEAD(folios); KUNIT_ASSERT_NOT_NULL(test, vma); + held = kunit_kzalloc(test, sizeof(*held), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, held); + KUNIT_ASSERT_EQ(test, + kunit_add_action_or_reset(test, test_release_fragment, held), 0); mm = vma->vm_mm; KUNIT_ASSERT_EQ(test, test_file_fault(vma, TEST_VA, true) & VM_FAULT_ERROR, 0U); lru_add_drain_all(); if (mode == 6) { - ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); } kthread_use_mm(mm); @@ -9619,15 +9628,15 @@ static void user4k_lazyfree_test(struct kunit *test) if (mode == 2 || mode == 3 || mode == 8 || mode == 10) { ret = get_user_fragments_remote(mm, TEST_VA, SZ_4K, - mode == 3 ? FOLL_WRITE : 0, &held, 1); + mode == 3 ? FOLL_WRITE : 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); if (mode == 3 || mode == 8) { - memset(folio_address(held.folio) + held.offset, 0xd3, SZ_4K); - release_user_fragments(&held, 1, true); + memset(folio_address(held->folio) + held->offset, 0xd3, SZ_4K); + release_user_fragments(held, 1, true); } } if (mode == 7) { - ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, &held, 1); + ret = pin_user_fragments_remote(mm, TEST_VA, SZ_4K, 0, held, 1); KUNIT_ASSERT_EQ(test, ret, 1L); } if (mode == 5 || mode == 9) { @@ -9644,7 +9653,12 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_EXPECT_EQ(test, succeeded, 1U); } lru_add_drain_all(); - folio = test_fault_folio(mm, TEST_VA); + /* A retained GET can coexist with a temporary migration PTE. */ + folio = held->folio; + if (folio) + folio_get(folio); + else + folio = test_fault_folio(mm, TEST_VA); KUNIT_ASSERT_NOT_NULL(test, folio); folio_lock(folio); try_to_unmap(folio, TTU_SYNC); @@ -9653,7 +9667,7 @@ static void user4k_lazyfree_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, folio_test_swapbacked(folio)); KUNIT_EXPECT_FALSE(test, folio_test_dirty(folio)); KUNIT_EXPECT_FALSE(test, pte_none(test_fault_entry(test, mm, TEST_VA))); - release_user_fragments(&held, 1, false); + release_user_fragments(held, 1, false); try_to_unmap(folio, TTU_SYNC); } folio_unlock(folio); @@ -9672,7 +9686,9 @@ static void user4k_lazyfree_test(struct kunit *test) (unsigned long long)pte_val(pte)); } if (mode == 2 || mode == 6 || mode == 7) - release_user_fragments(&held, 1, false); + release_user_fragments(held, 1, false); + kunit_release_action(test, test_release_fragment, held); + kunit_kfree(test, held); kunit_release_action(test, user4k_vma_mm_free, mm); } } From 97000932c1f92d09b1c56e9c3fff44576a3a0bce Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:22:28 -0400 Subject: [PATCH 371/631] xen/gntdev: retain typed fragments for mixed-granule grant copies Pin alternative-granule local ranges with the fragment API and derive the Xen frame from the retained backing offset. Keep native fast GUP and the existing hypercall batch size. Cache the fragment array with each batch, selecting its use from the calling MM on every ioctl. Track write intent per local pin so a later source-only segment cannot suppress dirty release for an earlier destination. Release adjacent pins with the same intent together, including on preparation failures. Signed-off-by: Diljit Singh (cherry picked from commit acebd9789c415af2f3e5492d839b8231e930671f) --- drivers/xen/gntdev.c | 82 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 1dcc4675580ed1..c35d81d26e981f 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -64,7 +65,11 @@ struct gntdev_copy_batch { s16 __user *status[GNTDEV_COPY_BATCH]; unsigned int nr_ops; unsigned int nr_pages; - bool writeable; + unsigned long writeable; +#ifdef CONFIG_MM_SUBPAGE + struct user_page_fragment *fragments; + bool use_fragments; +#endif struct gntdev_copy_batch *next; }; @@ -631,6 +636,9 @@ static int gntdev_release(struct inode *inode, struct file *flip) while (priv->batch) { batch = priv->batch; priv->batch = batch->next; +#ifdef CONFIG_MM_SUBPAGE + kfree(batch->fragments); +#endif kfree(batch); } mutex_unlock(&priv->batch_lock); @@ -805,20 +813,45 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u) } static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, - unsigned long *gfn) + size_t len, bool writeable, unsigned long *gfn) { unsigned long addr = (unsigned long)virt; struct page *page; unsigned long xen_pfn; int ret; - ret = pin_user_pages_fast(addr, 1, batch->writeable ? FOLL_WRITE : 0, &page); - if (ret < 0) - return ret; - - batch->pages[batch->nr_pages++] = page; +#ifdef CONFIG_MM_SUBPAGE + if (batch->use_fragments) { + struct user_page_fragment *fragment; - xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(addr & ~PAGE_MASK); + if (!batch->fragments) { + batch->fragments = kcalloc(GNTDEV_COPY_BATCH, + sizeof(*fragment), GFP_KERNEL); + if (!batch->fragments) + return -ENOMEM; + } + fragment = &batch->fragments[batch->nr_pages]; + ret = pin_user_fragments_remote(current->mm, addr, len, + writeable ? FOLL_WRITE : 0, fragment, 1); + if (ret != 1) + return ret < 0 ? ret : -EFAULT; + if (fragment->length != len) { + release_user_fragments(fragment, 1, false); + return -EFAULT; + } + xen_pfn = XEN_PFN_DOWN(PFN_PHYS(folio_pfn(fragment->folio)) + fragment->offset); + } else +#endif + { + ret = pin_user_pages_fast(addr, 1, writeable ? FOLL_WRITE : 0, &page); + if (ret != 1) + return ret < 0 ? ret : -EFAULT; + batch->pages[batch->nr_pages] = page; + xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(addr & ~PAGE_MASK); + } + if (writeable) + __set_bit(batch->nr_pages, &batch->writeable); + batch->nr_pages++; *gfn = pfn_to_gfn(xen_pfn); return 0; @@ -826,9 +859,24 @@ static int gntdev_get_page(struct gntdev_copy_batch *batch, void __user *virt, static void gntdev_put_pages(struct gntdev_copy_batch *batch) { - unpin_user_pages_dirty_lock(batch->pages, batch->nr_pages, batch->writeable); + unsigned int first = 0; + + while (first < batch->nr_pages) { + bool dirty = test_bit(first, &batch->writeable); + unsigned int next = dirty ? + find_next_zero_bit(&batch->writeable, batch->nr_pages, first) : + find_next_bit(&batch->writeable, batch->nr_pages, first); + +#ifdef CONFIG_MM_SUBPAGE + if (batch->use_fragments) + release_user_fragments(batch->fragments + first, next - first, dirty); + else +#endif + unpin_user_pages_dirty_lock(batch->pages + first, next - first, dirty); + first = next; + } batch->nr_pages = 0; - batch->writeable = false; + batch->writeable = 0; } static int gntdev_copy(struct gntdev_copy_batch *batch) @@ -917,9 +965,8 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->source.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); - batch->writeable = false; - ret = gntdev_get_page(batch, virt, &gfn); + ret = gntdev_get_page(batch, virt, len, false, &gfn); if (ret < 0) return ret; @@ -937,9 +984,8 @@ static int gntdev_grant_copy_seg(struct gntdev_copy_batch *batch, virt = seg->dest.virt + copied; off = (unsigned long)virt & ~XEN_PAGE_MASK; len = min(len, (size_t)XEN_PAGE_SIZE - off); - batch->writeable = true; - ret = gntdev_get_page(batch, virt, &gfn); + ret = gntdev_get_page(batch, virt, len, true, &gfn); if (ret < 0) return ret; @@ -971,6 +1017,10 @@ static long gntdev_ioctl_grant_copy(struct gntdev_priv *priv, void __user *u) mutex_lock(&priv->batch_lock); if (!priv->batch) { batch = kmalloc_obj(*batch); +#ifdef CONFIG_MM_SUBPAGE + if (batch) + batch->fragments = NULL; +#endif } else { batch = priv->batch; priv->batch = batch->next; @@ -981,6 +1031,10 @@ static long gntdev_ioctl_grant_copy(struct gntdev_priv *priv, void __user *u) batch->nr_ops = 0; batch->nr_pages = 0; + batch->writeable = 0; +#ifdef CONFIG_MM_SUBPAGE + batch->use_fragments = mm_page_size(current->mm) < PAGE_SIZE; +#endif for (i = 0; i < copy.count; i++) { struct gntdev_grant_copy_segment seg; From ed3eb1b9583e5a23451fd9b852e770e63338e7ca Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:34:29 -0400 Subject: [PATCH 372/631] mm: retire poisoned subpage mappings and exclude poisoned backing - Replace poisoned anonymous subpage PTEs with hardware-poison entries under the existing folio/PTE locks and MMU notifier scope. - Complete invalidation before dropping mapping ownership, preserving exact slot references retained by pins or asynchronous users. - Reject poisoned pool supply and skip poisoned owners during allocation; unused neighboring slots share the same native hardware-error granule. - Preserve the existing swap-cache containment path when TTU_HWPOISON is cleared by unmap_poisoned_folio(). Signed-off-by: Diljit Singh (cherry picked from commit ab3d927968dd9fb0563b26db4cc5fb7fb40a3d34) --- mm/rmap.c | 18 +++++++++++++++++- mm/user-subpage.c | 6 ++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/mm/rmap.c b/mm/rmap.c index 9241060489f580..84aba8ec5fa418 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -2157,11 +2157,27 @@ static bool try_to_unmap_subpage(struct folio *folio, struct vm_area_struct *vma swp_entry_t entry; bool exclusive, ret = false; - if (!pte_present(pte) || (flags & TTU_HWPOISON)) + if (!pte_present(pte)) return false; slot = mm_subpage_get_from_phys(pte_phys_mm(mm, pte)); if (WARN_ON_ONCE(!slot)) return false; + if (folio_test_hwpoison(folio) && (flags & TTU_HWPOISON)) { + flush_cache_range(vma, address, end); + pte = ptep_get_and_clear(mm, address, pvmw->pte); + /* Retained references keep their slot, but no CPU may keep its PTE. */ + flush_tlb_range(vma, address, end); + if (pte_dirty(pte)) + folio_mark_dirty(folio); + update_hiwater_rss(mm); + dec_mm_counter(mm, MM_ANONPAGES); + set_pte_at(mm, address, pvmw->pte, + swp_entry_to_pte(make_hwpoison_entry(&folio->page))); + mm_subpage_remove_anon_rmap(slot, vma); + mm_subpage_put(slot); /* Removed PTE reference, after invalidation. */ + ret = true; + goto out; + } if ((vma->vm_flags & VM_DROPPABLE) || mm_subpage_is_lazyfree(slot)) { flush_cache_range(vma, address, end); pte = ptep_get_and_clear(mm, address, pvmw->pte); diff --git a/mm/user-subpage.c b/mm/user-subpage.c index c08a816b90b204..bcb375cf53eca1 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -1086,7 +1086,8 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && - !folio_test_writeback(owner->folio)) + !folio_test_writeback(owner->folio) && + !folio_test_hwpoison(owner->folio)) return owner; } return NULL; @@ -1103,7 +1104,8 @@ static int pool_add_folio(struct mm_subpage_pool *pool, struct folio *folio, /* Only an unexposed order-0 allocation can become a slot owner. */ if (folio_order(folio) || folio_ref_count(folio) != 1 || folio->mapping || folio_mapped(folio) || folio_test_lru(folio) || - folio_test_private(folio) || folio_test_swapcache(folio)) + folio_test_private(folio) || folio_test_swapcache(folio) || + folio_test_hwpoison(folio)) return -EINVAL; owner = kzalloc(sizeof(*owner), gfp | __GFP_ACCOUNT); if (!owner) From 7e830b193d9fe544d4660b0858b712b4c001840c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 7 Oct 2026 14:50:27 -0400 Subject: [PATCH 373/631] mm: reject non-native geometry in native PFN array interfaces - HMM results represent whole native pages without fragment offsets or per-leaf permissions. Reject non-native MMs before walking or changing the output PFN array. - Device VMA migration collects, owns and restores native pages. Reject non-native MMs before rounding endpoints or modifying mappings/arrays. - Return -EOPNOTSUPP as a non-retry error. Native MMs keep the existing paths; enabling alternatives requires converting the complete API and its consumers together. Signed-off-by: Diljit Singh (cherry picked from commit 39c50fbe0687b81bdc2664da3dd051b3ebc43a5a) --- mm/hmm.c | 4 ++++ mm/migrate_device.c | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/mm/hmm.c b/mm/hmm.c index 5955f2f0c83db1..c4b96191ce09f9 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -652,6 +652,7 @@ static const struct mm_walk_ops hmm_walk_ops = { * the invalidation to finish. * -EFAULT: A page was requested to be valid and could not be made valid * ie it has no backing VMA or it is illegal to access + * -EOPNOTSUPP: The MM uses a non-native mapping granule. * * This is similar to get_user_pages(), except that it can read the page tables * without mutating them (ie causing faults). @@ -666,6 +667,9 @@ int hmm_range_fault(struct hmm_range *range) int ret; mmap_assert_locked(mm); + /* Each output PFN describes a whole native page, without a byte offset. */ + if (mm_page_size(mm) != PAGE_SIZE) + return -EOPNOTSUPP; do { /* If range is no longer valid force retry. */ diff --git a/mm/migrate_device.c b/mm/migrate_device.c index bae35a327cf1bb..4d72fb07810ab9 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -668,7 +668,7 @@ static void migrate_vma_unmap(struct migrate_vma *migrate) * @args: contains the vma, start, and pfns arrays for the migration * * Returns: negative errno on failures, 0 when 0 or more pages were migrated - * without an error. + * without an error. Non-native MM granules return -EOPNOTSUPP. * * Prepare to migrate a range of memory virtual address range by collecting all * the pages backing each virtual address in the range, saving them inside the @@ -731,6 +731,10 @@ int migrate_vma_setup(struct migrate_vma *args) { long nr_pages = (args->end - args->start) >> PAGE_SHIFT; + /* PFN arrays and migration ownership describe whole native pages. */ + if (args->vma && mm_page_size(args->vma->vm_mm) != PAGE_SIZE) + return -EOPNOTSUPP; + args->start &= PAGE_MASK; args->end &= PAGE_MASK; if (!args->vma || is_vm_hugetlb_page(args->vma) || From b91ecf4f5794458b893a2d80c4916c6c5d4d5dec Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:26:09 -0400 Subject: [PATCH 374/631] mm: exclude poisoned backing from COW pool retry and revalidation The published COW pool work added a pressure retry scan and folio-locked eligibility rechecks after the original poison-containment fix was prepared. Checking hwpoison only in find_available leaves those paths inconsistent: the pressure path can report unusable poisoned backing as retryable. Put the exclusion in the shared eligibility predicate, before the non-COW early return, so ordinary allocation, pressure retry and locked revalidation all use the same condition. Keep the fresh-supply rejection. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit c404199c7213c475f36788bd9336e2f9ebb89651) --- mm/user-subpage.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index bcb375cf53eca1..88f043bae28b25 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -988,6 +988,9 @@ void mm_subpage_pool_close(struct mm_subpage_pool *pool) static bool cow_owner_eligible(struct mm_subpage_pool *pool, struct folio *folio) { + /* All allocation and retry paths must exclude poisoned backing. */ + if (folio_test_hwpoison(folio)) + return false; if (!pool->cow_shared) return true; /* A previously unlocked owner can be mlocked or fully MADV_FREE'd @@ -1086,8 +1089,7 @@ static struct mm_subpage_owner *find_available(struct mm_subpage_pool *pool, (!pool->cow_shared || !folio_test_locked(owner->folio)) && cow_owner_eligible(pool, owner->folio) && !folio_test_swapcache(owner->folio) && - !folio_test_writeback(owner->folio) && - !folio_test_hwpoison(owner->folio)) + !folio_test_writeback(owner->folio)) return owner; } return NULL; From 92d67919c8bffb5924397bc6804998f69eab07a9 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:32:45 -0400 Subject: [PATCH 375/631] mm: protect COW backing charge comparisons with RCU The remote fragment-pin/COW test reaches mm_subpage_cow_folio_matches() without an RCU read lock. With lockdep enabled, folio_memcg() warns in obj_cgroup_memcg() because the charge's memcg can be reparented. Hold RCU across the comparison as required by folio_memcg(). The pool's retained memcg reference does not protect a different memcg obtained from the folio. No returned memcg pointer escapes the read-side section. Fixes: 5469651a0899 ("mm: reuse backing folios for small-granule anonymous COW") Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit 613963008f40faa6787e0a678afd7a008ca52554) --- mm/user-subpage.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 88f043bae28b25..68e410e546302c 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -510,7 +510,14 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get(struct mm_struct *mm, bool mm_subpage_cow_folio_matches(struct mm_subpage_pool *pool, struct folio *folio) { - return pool->cow_memcg == folio_memcg(folio) && pool->cow_nid == folio_nid(folio); + bool matches; + + /* The folio charge may be reparented while this pool remains alive. */ + rcu_read_lock(); + matches = pool->cow_memcg == folio_memcg(folio) && + pool->cow_nid == folio_nid(folio); + rcu_read_unlock(); + return matches; } void mm_subpage_cow_pool_exit(struct mm_struct *mm) From 381769bf1dfcb20c947722c6824331d8a2b92209 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 17:38:01 -0400 Subject: [PATCH 376/631] arm64: validate COW tests against the mapped physical quarter COW backing can now use a physical quarter unrelated to the fault's virtual quarter. Update the zero, fork, file and fragment-permission fixtures to validate the actual PTE and slot rather than requiring the old fixed-offset allocation policy. Read zeroed bytes and write child-private bytes at the actual slot offset. Keep the checks for untouched neighboring PTEs, copied data, parent and file-cache isolation, mapping ownership and access permissions. Assisted-by: LLM Signed-off-by: Diljit Singh (cherry picked from commit ef7f72d427299dd14bfed784ba917d19967f0bb3) --- arch/arm64/mm/user4k-test.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 6b5aeef7be61db..a18801a3fa44ec 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -7096,7 +7096,7 @@ static void user4k_file_cow_test(struct kunit *test) KUNIT_ASSERT_TRUE(test, pte_present(entry)); KUNIT_EXPECT_TRUE(test, pte_write(entry)); KUNIT_EXPECT_TRUE(test, folio_test_anon(page_folio(pte_page(entry)))); - KUNIT_EXPECT_EQ(test, pte_phys_mm(vma->vm_mm, entry) & ~PAGE_MASK, 2UL * SZ_4K); + /* Private COW can pack into any physical quarter of its backing. */ KUNIT_EXPECT_PTR_EQ(test, memchr_inv(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x33, SZ_4K), NULL); memset(phys_to_virt(pte_phys_mm(vma->vm_mm, entry)), 0x91, SZ_4K); @@ -7314,11 +7314,12 @@ static void test_zero_cow_granule(struct kunit *test, unsigned int shift) KUNIT_ASSERT_TRUE(test, pte_write(after)); slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); KUNIT_ASSERT_NOT_NULL(test, slot); - KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), + (unsigned int)offset_in_page(pte_phys_mm(mm, after))); KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + - size, 0, size), NULL); + mm_subpage_offset(slot), 0, size), NULL); mm_subpage_put(slot); } KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 1UL); @@ -7459,13 +7460,15 @@ static void test_fork_cow_granule(struct kunit *test, unsigned int shift) KUNIT_EXPECT_EQ(test, pte_phys_mm(mm, after) != mm_subpage_phys(held[i]), changed); slot = mm_subpage_get_from_phys(pte_phys_mm(mm, after)); KUNIT_ASSERT_NOT_NULL(test, slot); - KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), i * (unsigned int)size); + KUNIT_EXPECT_EQ(test, mm_subpage_offset(slot), + (unsigned int)offset_in_page(pte_phys_mm(mm, after))); KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(slot)) + mm_subpage_offset(slot), 0x31 + i, size), NULL); if (changed) { KUNIT_EXPECT_EQ(test, mm_subpage_mapcount(slot), 1); KUNIT_EXPECT_EQ(test, folio_ref_count(mm_subpage_folio(slot)), 2); - memset(folio_address(mm_subpage_folio(slot)) + size, 0x77, size); + memset(folio_address(mm_subpage_folio(slot)) + + mm_subpage_offset(slot), 0x77, size); } mm_subpage_put(slot); } @@ -9225,7 +9228,9 @@ static void user4k_fragment_permissions_test(struct kunit *test) ret = fragment_test_pin(pins, mm, start + 17, 31, FOLL_WRITE, 8); KUNIT_ASSERT_EQ(test, ret, 1L); KUNIT_EXPECT_NOT_NULL(test, fragment->subpage); - KUNIT_EXPECT_EQ(test, fragment->offset, SZ_4K + 17UL); + /* COW backing may use a different quarter from the virtual address. */ + KUNIT_EXPECT_EQ(test, page_to_phys(&fragment->folio->page) + fragment->offset, + pte_phys_mm(mm, test_fault_entry(test, mm, start)) + 17); memset(folio_address(fragment->folio) + fragment->offset, 0xa4, fragment->length); fragment_test_unpin(pins); mmap_write_lock(mm); From 9c894500278580d836b7686e6f5bc88c971dbfd4 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 19:36:01 -0400 Subject: [PATCH 377/631] mm: guard subpage COW pool charge references without MEMCG get_mem_cgroup_from_mm() returns NULL without CONFIG_MEMCG, but the COW pool still references the incomplete struct mem_cgroup in the css_get() expression. Guard the retained-reference operation so mixed-granule kernels build without memory cgroups. Keep the reference acquisition unchanged for enabled memory cgroups. --- mm/user-subpage.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/user-subpage.c b/mm/user-subpage.c index 68e410e546302c..c9101990099df9 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -466,8 +466,10 @@ struct mm_subpage_pool *mm_subpage_cow_pool_get_charged(struct mm_struct *mm, mm_subpage_pool_put(new); break; } +#ifdef CONFIG_MEMCG if (memcg) css_get(&memcg->css); +#endif new->cow_memcg = memcg; new->cow_nid = nid; new->cow_shared = true; From 94848202735f4b9a9a55b3dff5958ddc5ac35f5d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 19:42:05 -0400 Subject: [PATCH 378/631] arm64: validate packed fragments through mapped physical offsets The shared COW pool may place a private fragment in another physical quarter of the same native folio. Sparse anonymous allocations likewise need not match their virtual-page index. Follow actual PTE/subpage offsets when checking pinned copies, live COW writes and restored swap data; record pre-migration and pre-swap offsets where the operation must preserve them. Verify tags at the actual copied slot and require child read pins and swap COW writes to retain distinct physical backing from the parent. Keep data, permissions, ownership and accounting assertions intact. Validated with 233 passing ARM64 KUnit tests (39 skips), all 17 disposable swap tests, and native16K/4K exec and thread VM fixtures. A separate memory-cgroup-enabled COW object build also passes. --- arch/arm64/mm/user4k-test.c | 72 +++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index a18801a3fa44ec..d5349f54f6d7c4 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -3019,7 +3019,9 @@ static void test_anon_write_fault_granule(struct kunit *test, unsigned int shift if (!backing) backing = folio; identity = folio == backing && - mm_subpage_offset(slot) == i * size; + mm_subpage_phys(slot) == + pte_phys_mm(vma->vm_mm, ptep_get(pte)) && + (shape || mm_subpage_offset(slot) == i * size); zero = !memchr_inv(folio_address(folio) + mm_subpage_offset(slot), 0, size); mm_subpage_put(slot); @@ -8553,7 +8555,8 @@ static void test_swap_tags_set(struct folio *folio) #endif } -static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +static bool test_swap_tags_match_at(struct folio *folio, unsigned int start, + unsigned int slot) { #ifdef CONFIG_ARM64_MTE unsigned int offset; @@ -8562,7 +8565,7 @@ static bool test_swap_tags_match(struct folio *folio, unsigned int slot) return true; if (!page_mte_tagged(&folio->page)) return false; - for (offset = slot * SZ_4K; offset < (slot + 1) * SZ_4K; offset += MTE_GRANULE_SIZE) { + for (offset = start; offset < start + SZ_4K; offset += MTE_GRANULE_SIZE) { unsigned long tag = 0; asm volatile(".arch_extension memtag\n\tldg %0, [%1]" @@ -8574,6 +8577,11 @@ static bool test_swap_tags_match(struct folio *folio, unsigned int slot) return true; } +static bool test_swap_tags_match(struct folio *folio, unsigned int slot) +{ + return test_swap_tags_match_at(folio, slot * SZ_4K, slot); +} + struct fragment_test_pins { struct user_page_fragment fragments[2 * TEST_SLOTS]; unsigned long count; @@ -8681,9 +8689,14 @@ static void user4k_fragment_pin_test(struct kunit *test) KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(f->folio) + f->offset, 0x71 + i, f->length), NULL); if (!native) { + pte_t parent_pte = test_fault_entry(test, mm, TEST_VA + i * size); + KUNIT_EXPECT_TRUE(test, mm_subpage_is_exclusive(f->subpage)); + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), + pte_phys_mm(mm, parent_pte)); if (i) - KUNIT_EXPECT_PTR_NE(test, f->folio, pins->fragments[0].folio); + KUNIT_EXPECT_NE(test, mm_subpage_phys(f->subpage), + mm_subpage_phys(pins->fragments[0].subpage)); } } fragment_test_unpin(pins); @@ -9337,9 +9350,10 @@ static void user4k_slot_pin_test(struct kunit *test) copy = mm_subpage_get_from_phys(pte_phys_mm(child, pte)); KUNIT_ASSERT_NOT_NULL(test, copy); KUNIT_EXPECT_EQ(test, mm_subpage_is_exclusive(copy), i == 1); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + i * SZ_4K, + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(mm_subpage_folio(copy)) + mm_subpage_offset(copy), 0x61 + i, SZ_4K), NULL); - KUNIT_EXPECT_TRUE(test, test_swap_tags_match(mm_subpage_folio(copy), i)); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match_at(mm_subpage_folio(copy), + mm_subpage_offset(copy), i)); mm_subpage_put(copy); } /* A read pin on a shared slot must request unsharing, not pin shared COW data. */ @@ -10324,6 +10338,7 @@ static void user4k_migration_test(struct kunit *test) unsigned long start = edge ? boundaries[mode - 6] : TEST_VA + (sparse ? SZ_4K : 0); unsigned int nr = sparse ? 1 : 4, i, side, succeeded = 0; + unsigned int offsets[TEST_SLOTS]; struct vm_area_struct *vma = test_fault_vma(test, start, nr * SZ_4K); struct mm_struct *child = NULL; struct folio *folio; @@ -10350,6 +10365,9 @@ static void user4k_migration_test(struct kunit *test) for (i = 0; i < TEST_SLOTS; i++) memset(folio_address(folio) + i * SZ_4K, 0x41 + i + mode, SZ_4K); test_swap_tags_set(folio); + for (i = 0; i < nr; i++) + offsets[i] = offset_in_page(pte_phys_mm(vma->vm_mm, + test_fault_entry(test, vma->vm_mm, start + i * SZ_4K))); folio_put(folio); if (forked) { child = test_vma_mm(test); @@ -10379,7 +10397,7 @@ static void user4k_migration_test(struct kunit *test) struct mm_struct *mm = side ? child : vma->vm_mm; for (i = 0; i < nr; i++) { - unsigned int index = sparse ? offset_in_page(start) / SZ_4K : i; + unsigned int index = offsets[i] / SZ_4K; pte_t pte = test_fault_entry(test, mm, start + i * SZ_4K); struct mm_subpage *slot; @@ -10866,13 +10884,27 @@ static void test_user4k_swap_roundtrip(struct kunit *test, unsigned int mode, u8 KUNIT_EXPECT_TRUE(test, pte_write(pte)); folio = test_fault_folio(child, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); - KUNIT_EXPECT_TRUE(test, test_swap_tags_match(folio, 1)); - memset(folio_address(folio) + SZ_4K, 0xa7, SZ_4K); + { + unsigned int offset = pte_phys_mm(child, pte) - PFN_PHYS(folio_pfn(folio)); + + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + base + 1, SZ_4K), NULL); + KUNIT_EXPECT_TRUE(test, test_swap_tags_match_at(folio, offset, 1)); + memset(folio_address(folio) + offset, 0xa7, SZ_4K); + } folio_put(folio); folio = test_fault_folio(parent->vm_mm, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + SZ_4K, base + 1, SZ_4K), NULL); + { + pte_t parent_pte = test_fault_entry(test, parent->vm_mm, TEST_VA + SZ_4K); + unsigned int offset = pte_phys_mm(parent->vm_mm, parent_pte) - + PFN_PHYS(folio_pfn(folio)); + + KUNIT_EXPECT_NE(test, pte_phys_mm(child, pte), + pte_phys_mm(parent->vm_mm, parent_pte)); + KUNIT_EXPECT_PTR_EQ(test, memchr_inv(folio_address(folio) + offset, + base + 1, SZ_4K), NULL); + } folio_put(folio); KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_ANONPAGES), 1UL); KUNIT_EXPECT_EQ(test, get_mm_counter_sum(child, MM_SWAPENTS), TEST_SLOTS - 1UL); @@ -11247,12 +11279,13 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) struct vm_area_struct *parent = test_fault_vma(test, start, 2 * SZ_4K); struct mm_struct *child; unsigned int i, side; + unsigned int offsets[2]; int ret; KUNIT_ASSERT_NOT_NULL(test, parent); for (i = 0; i < 2; i++) { unsigned long addr = start + i * SZ_4K; - unsigned int offset = vma_page_offset_at(parent, addr).offset; + unsigned int offset; struct folio *folio; vm_fault_t fault; swp_entry_t entry; @@ -11262,6 +11295,9 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) fault = handle_mm_fault(parent, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE, NULL); mmap_read_unlock(parent->vm_mm); KUNIT_ASSERT_EQ(test, fault, (vm_fault_t)0); + offset = offset_in_page(pte_phys_mm(parent->vm_mm, + test_fault_entry(test, parent->vm_mm, addr))); + offsets[i] = offset; lru_add_drain_all(); folio = test_fault_folio(parent->vm_mm, addr); KUNIT_ASSERT_NOT_NULL(test, folio); @@ -11293,7 +11329,7 @@ static void user4k_swapoff_boundaries_test(struct kunit *test) KUNIT_EXPECT_EQ(test, get_mm_counter_sum(mm, MM_ANONPAGES), 2UL); for (i = 0; i < 2; i++) { unsigned long addr = start + i * SZ_4K; - unsigned int offset = vma_page_offset_at(parent, addr).offset; + unsigned int offset = offsets[i]; pte_t pte = test_fault_entry(test, mm, addr); struct folio *folio; @@ -11520,8 +11556,14 @@ static void test_live_fault_roundtrip(struct kunit *test, unsigned int mode) } folio = test_fault_folio(child, TEST_VA + SZ_4K); KUNIT_ASSERT_NOT_NULL(test, folio); - for (i = 0; i < 2; i++) - KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + SZ_4K + 64 + i), (u8)(0xe0 + i)); + { + pte_t pte = test_fault_entry(test, child, TEST_VA + SZ_4K); + unsigned long offset = pte_phys_mm(child, pte) - PFN_PHYS(folio_pfn(folio)); + + for (i = 0; i < 2; i++) + KUNIT_EXPECT_EQ(test, *((u8 *)folio_address(folio) + offset + 64 + i), + (u8)(0xe0 + i)); + } folio_put(folio); mmap_read_lock(parent->vm_mm); fault = handle_mm_fault(parent, TEST_VA + SZ_4K, FAULT_FLAG_REMOTE, NULL); From 6767bb48f3e3cbc4c6d0950f67028bfc4a045b53 Mon Sep 17 00:00:00 2001 From: Dj <43033058+DjDeveloperr@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:48:24 -0400 Subject: [PATCH 379/631] drm/asahi: provision G17P ray traversal pools from Gen1 SCS demand Program the Gen1 scratch and intersection-table registers from validated compute requests. Publish the optional compute command tail in the UAPI and retain zero-tail compatibility for ordinary compute submissions. Include each enabled USC's SCS small-page, large-page and tail inventories in physical compute pool admission. Allocate the admitted capacity for new or vacant queues, reuse retained pools only when they fit, and grow owned queues after prior work drains and firmware consumes the free-list release. Keep old aliases and backing until the replacement is published. Preserve the pending capacity high-water mark across competing requests. This replaces the private fixed 64MiB RT pool experiment. Ordinary compute keeps its existing 21MiB floor, while the captured 304-byte Gen1 profile requires 41MiB on Neo's five enabled cores. Diagnostic logging and RT-unit control experiments are excluded. Validation: seven actual-source host capacity/lifecycle tests; matched651 module import/modinfo audit; Neo hardware 99 RT CTS passes (two unsupported), repeated 16384-ray dispatches at local 64/256 and four concurrent clients, with fourteen checked runtime outputs and no new captured kernel faults. --- drivers/gpu/drm/asahi/g17/command.rs | 22 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 26 ++- drivers/gpu/drm/asahi/g17/fw/compute.rs | 187 +++++++++++++++++-- drivers/gpu/drm/asahi/g17/fw/compute_pool.rs | 25 +++ drivers/gpu/drm/asahi/g17/fw/initdata.rs | 6 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 1 + drivers/gpu/drm/asahi/g17/initdata.rs | 9 +- drivers/gpu/drm/asahi/g17/job.rs | 2 +- drivers/gpu/drm/asahi/g17/mod.rs | 3 + drivers/gpu/drm/asahi/g17/queue/compute.rs | 61 +++++- drivers/gpu/drm/asahi/g17/queue/mod.rs | 7 +- drivers/gpu/drm/asahi/g17/runtime.rs | 69 ++++++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 +- include/uapi/drm/asahi_drm.h | 22 +++ 14 files changed, 388 insertions(+), 56 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/compute_pool.rs diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 83191ddbcb1c36..dd3be1a81e1d83 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -3,7 +3,7 @@ //! Decode a submission and validate its user resources before publishing any work. use super::fw::{ - compute::ScratchRequest, + compute::{IntersectionTables, ScratchRequest}, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, }; @@ -14,6 +14,7 @@ use kernel::{prelude::*, uapi}; const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; +const COMPUTE_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) + size_of::(); const SAMPLER_SIZE: u64 = 8; const PAGE_SIZE: u64 = 0x4000; const ZLS_METADATA_STRIDE: u64 = 0x80; @@ -78,7 +79,7 @@ impl Attachments { #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), - Compute(uapi::drm_asahi_cmd_compute, Attachments), + Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), } pub(crate) struct Command { @@ -130,7 +131,11 @@ impl<'a> Parser<'a> { Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) } else { self.counts[1] += 1; - Payload::Compute(payload(bytes)?, self.attachments[2]) + Payload::Compute( + payload(&bytes[..bytes.len().min(COMPUTE_BASE_SIZE)])?, + self.attachments[2], + IntersectionTables::parse(bytes.get(COMPUTE_BASE_SIZE..).unwrap_or(&[]))?, + ) }; return Ok(Some(Command { barriers, payload })); } @@ -283,6 +288,7 @@ pub(crate) enum Validated { timestamps: [uapi::drm_asahi_timestamps; 2], }, Compute { + intersection: IntersectionTables, scratch: ScratchRequest, usc_base: u64, cdm_va: u64, @@ -297,7 +303,7 @@ impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], - Self::Compute(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], } } @@ -415,7 +421,7 @@ impl Payload { timestamps: [cmd.ts_vtx, cmd.ts_frag], }) } - Self::Compute(cmd, attachments) => { + Self::Compute(cmd, attachments, intersection) => { if cmd.flags != 0 || cmd.cdm_ctrl_stream_base & 3 != 0 || cmd.cdm_ctrl_stream_end & 3 != 0 @@ -432,7 +438,13 @@ impl Payload { )?; sampler(space, cmd.sampler_heap, cmd.sampler_count)?; attachments.validate(space)?; + if intersection.enabled() { + require_range(space, intersection.pool, intersection.pool_bytes, Access::Read)?; + let code = intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } Ok(Validated::Compute { + intersection: *intersection, scratch: ScratchRequest::new(cmd.scs_layout)?, usc_base: window.base, cdm_va: cmd.cdm_ctrl_stream_base, diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c2c9470d10aa3e..2612a9302ce28c 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -8,7 +8,7 @@ //! reference and are released only after the owner's teardown is acknowledged. use super::fw::freelist::*; -use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_SIZE}; +use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::object::{Allocator, CpuMap, KernelObject}; use crate::mmu; use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; @@ -90,9 +90,15 @@ impl Pages { layout, }) } - /// Fresh zeroed compute backing, built without the device mutex. - pub(crate) fn compute(alloc: &Allocator<'_>) -> Result { - Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute) + /// Fresh compute backing at the admitted capacity, built off-lock. + pub(crate) fn compute_sized(alloc: &Allocator<'_>, blocks: usize) -> Result { + if blocks < COMPUTE_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { + return Err(EINVAL); + } + Self::new(alloc, blocks, RunLayout::Compute) + } + pub(crate) fn map_client(&self, vm: &mmu::Vm) -> Result { + self.backing.map_alias(vm, self.page_list_va(), mmu::PROT_GPU_SHARED_RW) } fn page_list_va(&self) -> u64 { self.backing.gpu_va() @@ -437,12 +443,16 @@ pub(crate) struct ComputePool { id: u16, } impl ComputePool { - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8) -> Result { + pub(crate) fn fits(&self, blocks: usize) -> bool { + self.pages.as_ref().is_some_and(|pages| pages.blocks >= blocks) + } + + pub(crate) fn new(alloc: &Allocator<'_>, qid: u8, blocks: usize) -> Result { if qid > super::fw::kick::QID_MAX { return Err(EINVAL); } let id = u16::from(qid) + 1; - let pages = Pages::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute)?; + let pages = Pages::compute_sized(alloc, blocks)?; let mut control = alloc.kernel( CONTROL_SIZE, mmu::UAT_PGSZ as u64, @@ -462,7 +472,7 @@ impl ComputePool { class: FreeListClass::One, page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, + blocks: blocks as u32, state_va: state.gpu_va(), }), )?; @@ -517,7 +527,7 @@ impl ComputePool { class: FreeListClass::One, page_list_va: pages.page_list_va(), run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, + blocks: pages.blocks as u32, state_va: self.state.gpu_va(), }); header.unk_54[..8].copy_from_slice(&bytes); diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 5b7489cfe93f4b..829b519c545fb4 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -23,57 +23,114 @@ pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; /// Event mask of compute kicks. pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; -const COMPUTE_REGISTERS: usize = 40; +const COMPUTE_REGISTERS: usize = 41; const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; /// Validated per-command shader-context scratch request. +/// +/// The layout word is the packed SCS layout: depth (bits 0:15), levels +/// (16:23), cliques (24:31), user (32:47) and driver (48:63). The region the +/// hardware needs is derived with G17P's Gen1 section model: fixed prefixes +/// totalling [`Self::FIXED_BYTES`], then per-thread stack, per-thread driver +/// and per-thread user bytes. Gen1 has no per-clique level payload. The scratch +/// registers the program writes are computed from that layout, and the +/// disabled program writes the fixed disabled window and stack words +/// defined below. #[derive(Clone, Copy)] pub(crate) struct ScratchRequest { pages: u16, + window: u64, + stack: u64, + driver: u64, + tail: u64, } impl ScratchRequest { + /// End of the fixed Gen1 prefix sections. const FIXED_BYTES: u64 = 0x58000; const THREADS_PER_CLIQUE: u64 = 32; const BYTES_PER_STACK_ENTRY: u64 = 8; const PAGE_SHIFT: u32 = 12; const PAGE_SIZE: u64 = 1 << Self::PAGE_SHIFT; const PAGES_PER_MIB: u64 = 1 << (20 - Self::PAGE_SHIFT); + /// Window word when enabled: bit 24 plus the clique count at 16:23. const WINDOW_ENABLED: u64 = 1 << 24; + /// Disabled window word, [`UNK_10791`]. + const WINDOW_DISABLED: u64 = UNK_10791; + /// Disabled stack word, [`UNK_1A0E9`]. + const STACK_DISABLED: u64 = UNK_1A0E9; + /// Tail reservation when enabled. + const TAIL_ENABLED: u64 = 8; /// Payload-free layouts request no scratch even if depth/cliques are set. - /// Otherwise round the fixed and per-thread sections and add a guard page. + /// Otherwise derive the Gen1 end and round it up with a two-page guard. + /// The per-register words are computed from it and range-checked here + /// before they reach hardware. pub(crate) fn new(layout: u64) -> Result { let depth = layout & u16::MAX as u64; let cliques = (layout >> 24) & u8::MAX as u64; let user = (layout >> 32) & u16::MAX as u64; let driver = (layout >> 48) & u16::MAX as u64; if user == 0 && driver == 0 { - return Ok(Self { pages: 0 }); + return Ok(Self { + pages: 0, + window: Self::WINDOW_DISABLED, + stack: Self::STACK_DISABLED, + driver: 0, + tail: 0, + }); } let threads = cliques.checked_mul(Self::THREADS_PER_CLIQUE).ok_or(EINVAL)?; - let per_thread = depth.checked_mul(Self::BYTES_PER_STACK_ENTRY) - .and_then(|bytes| bytes.checked_add(user)) - .and_then(|bytes| bytes.checked_add(driver)).ok_or(EINVAL)?; - let bytes = threads.checked_mul(per_thread) - .and_then(|bytes| bytes.checked_add(Self::FIXED_BYTES)) + let stack_bytes = threads + .checked_mul(depth.checked_mul(Self::BYTES_PER_STACK_ENTRY).ok_or(EINVAL)?) + .ok_or(EINVAL)?; + let driver_bytes = threads.checked_mul(driver).ok_or(EINVAL)?; + let user_bytes = threads.checked_mul(user).ok_or(EINVAL)?; + let end10 = Self::FIXED_BYTES + .checked_add(stack_bytes) + .and_then(|bytes| bytes.checked_add(driver_bytes)) + .ok_or(EINVAL)?; + let end = end10 + .checked_add(user_bytes) .and_then(|bytes| bytes.checked_add(2 * Self::PAGE_SIZE - 1)) .ok_or(EINVAL)?; - let pages = bytes >> Self::PAGE_SHIFT; - Ok(Self { pages: u16::try_from(pages).map_err(|_| EINVAL)? }) + let pages = end >> Self::PAGE_SHIFT; + Ok(Self { + pages: u16::try_from(pages).map_err(|_| EINVAL)?, + window: Self::WINDOW_ENABLED | (cliques << 16), + stack: depth & !7, + // G17P's register list does not include the Gen2 level + // region register. Preserve the historical zero slot. + driver: 0, + tail: Self::TAIL_ENABLED, + }) } pub(crate) fn enabled(self) -> bool { self.pages != 0 } + pub(crate) fn pages(self) -> u16 { self.pages } + fn size_word(self) -> u64 { let pages = u64::from(self.pages); pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) } fn window_word(self) -> u64 { - if self.enabled() { Self::WINDOW_ENABLED } else { UNK_10791 } + self.window + } + + fn stack_word(self) -> u64 { + self.stack + } + + fn driver_word(self) -> u64 { + self.driver + } + + fn tail_word(self) -> u64 { + self.tail } fn checkpoints(self) -> [u16; 4] { @@ -106,10 +163,94 @@ const OPERAND_STATE_0A5C9: u64 = 0x9000; /// Checkpoints into the main register array, as entry counts: after the /// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full /// count. -const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; +const CHECKPOINTS: [u16; 4] = [39, 40, 40, COMPUTE_REGISTERS as u16]; + +#[derive(Clone, Copy)] +pub(crate) struct IntersectionTables { + pub(crate) pool: u64, + pub(crate) pool_bytes: u64, + auxiliary: u32, + first: u32, + count: u32, + pub(crate) default_program: u64, + default_control: u64, +} + +impl IntersectionTables { + pub(crate) const NONE: Self = Self { + pool: 0, pool_bytes: 0, auxiliary: 0, first: 0, count: 0, + default_program: 0, default_control: 0, + }; + + pub(crate) fn new(pool: u64, pool_bytes: u64, first: u32, count: u32, + default_program: u64, default_control: u64) -> Result { + if pool == 0 { + if pool_bytes != 0 || first != 0 || count != 0 + || default_program != 0 || default_control != 0 { + return Err(EINVAL); + } + return Ok(Self::NONE); + } + // G17P HAL200 Gen1 publishes a byte-sized table length (25 bits), + // with a plain aligned address. Unlike G17X HAL300, its address has + // no bit 51 and its two pool registers use fixed values. + // Keep the shared UAPI's 128-byte blocks and 16 KiB allocation units. + let end = pool.checked_add(pool_bytes).ok_or(EINVAL)?; + let blocks_end = u64::from(first) + u64::from(count); + let code = default_program & 0x0000_ffff_ffff_ffff; + if pool & 0x3fff != 0 || pool_bytes == 0 || pool_bytes & 0x3fff != 0 + || end > (1u64 << 48) || (pool_bytes >> 14) >= (1u64 << 25) + || count == 0 || count >= (1 << 18) || blocks_end * 128 > pool_bytes + || code & 63 != 0 + || default_control & !0x0000_7fff_0000_3fff != 0 + || (code == 0 && (default_program != 0 || default_control != 0)) + || (code != 0 && default_control & (1u64 << 40) == 0) { + return Err(EINVAL); + } + Ok(Self { pool, pool_bytes, auxiliary: 0, first, count, default_program, default_control }) + } + + pub(crate) fn parse(bytes: &[u8]) -> Result { + if bytes.get(48..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) { + return Err(EINVAL); + } + let mut tail = [0u8; 48]; + let count = bytes.len().min(tail.len()); + tail[..count].copy_from_slice(&bytes[..count]); + let auxiliary = u32::from_le_bytes(tail[..4].try_into().unwrap()); + if tail[4..8].iter().any(|&b| b != 0) + || (auxiliary != 0 && !(8..=32).contains(&auxiliary)) { + return Err(EINVAL); + } + let mut tables = Self::new( + u64::from_le_bytes(tail[8..16].try_into().unwrap()), + u64::from_le_bytes(tail[16..24].try_into().unwrap()), + u32::from_le_bytes(tail[24..28].try_into().unwrap()), + u32::from_le_bytes(tail[28..32].try_into().unwrap()), + u64::from_le_bytes(tail[32..40].try_into().unwrap()), + u64::from_le_bytes(tail[40..48].try_into().unwrap()), + )?; + if auxiliary != 0 && !tables.enabled() { return Err(EINVAL); } + tables.auxiliary = auxiliary; + Ok(tables) + } + + pub(crate) fn enabled(self) -> bool { self.pool != 0 } + + pub(crate) fn words(self) -> [u64; 6] { + if !self.enabled() { return [0; 6]; } + // The six intersection-table words of both command encodings; + // the firmware passes them to the hardware unchanged. + [self.pool + u64::from(self.first) * 128, + 0xc0000000 | (u64::from(self.count) << 7), 0x6f00000000, + 0x3feff, self.default_program, self.default_control] + } +} + /// Inputs of one compute descriptor. pub(crate) struct ComputeArgs { + pub(crate) intersection: IntersectionTables, /// Scratch request validated with the userspace command. pub(crate) scratch: ScratchRequest, /// Base of the queue's USC window. @@ -166,6 +307,7 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS let operand = args.operand_state_va; let key = work_key(args.context_id, args.work_state); let ids = OBJECT_IDS.word(); + let intersection = args.intersection.words(); Ok([ RegisterWrite::new(0x017d9, 1), @@ -183,15 +325,20 @@ fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGIS RegisterWrite::new(0x1a440, UNK_1A440), RegisterWrite::new(0x1a458, UNK_1A458), RegisterWrite::new(0x101d9, UNK_101D9), - RegisterWrite::new(0x1a089, 0), - RegisterWrite::new(0x1a091, 0), - RegisterWrite::new(0x1a059, 0), - RegisterWrite::new(0x1a061, 0), - RegisterWrite::new(0x1a0b9, 0), - RegisterWrite::new(0x1a0c1, 0), + RegisterWrite::new(0x1a089, intersection[0]), + RegisterWrite::new(0x1a091, intersection[1]), + RegisterWrite::new(0x1a059, intersection[2]), + RegisterWrite::new(0x1a061, intersection[3]), + RegisterWrite::new(0x1a0b9, intersection[4]), + RegisterWrite::new(0x1a0c1, intersection[5]), RegisterWrite::new(0x101d1, args.scratch.size_word()), - RegisterWrite::new(0x0d479, 0), - RegisterWrite::new(0x1a0e9, UNK_1A0E9), + RegisterWrite::new(0x0d479, if args.intersection.auxiliary == 0 { + args.scratch.tail_word() + } else { + u64::from(args.intersection.auxiliary) + }), + RegisterWrite::new(0x1a0e9, args.scratch.stack_word()), + RegisterWrite::new(0x1a141, args.scratch.driver_word()), RegisterWrite::new(0x107a1, args.scratch.window_word()), RegisterWrite::new(0x0a599, UNK_0A599), RegisterWrite::new(0x0d411, UNK_0D411), diff --git a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs new file mode 100644 index 00000000000000..4005daa4e98324 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +//! G17P compute pool capacity for a validated Gen1 SCS request. + +use kernel::prelude::*; +use super::initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; +use super::freelist::{COMPUTE_BLOCKS, PAGE_LIST_SIZE}; + +pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result { + if active_cores == 0 { return Err(EINVAL); } + if scs_pages == 0 { return Ok(COMPUTE_BLOCKS); } + // Register0x101d1 counts 4-KiB pages and their 1-MiB large-page inventory. + // Register0x0d479 reserves eight further pages for each enabled USC. + let small = u64::from(scs_pages); + let per_core = small.checked_add(small.div_ceil(256)) + .and_then(|pages| pages.checked_add(8)).ok_or(EOVERFLOW)?; + let extra = per_core.checked_mul(u64::from(active_cores)) + .and_then(|pages| pages.checked_mul(4096)).ok_or(EOVERFLOW)?; + let bytes = extra.checked_add((COMPUTE_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) + .map_err(|_| EOVERFLOW)?; + let maximum = PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8); + if blocks > maximum { return Err(E2BIG); } + Ok(blocks) +} diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 94b03ebe284dba..1f25bd0fc1c0dc 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -2239,7 +2239,11 @@ static_assert!(size_of::() == 0x20); impl PagePoolDescriptor { /// Complete row for a physical compute queue's independent 21-block pool. pub(crate) const fn compute(page_list_va: u64) -> Self { - let pages = 21 * FREE_LIST_BLOCK_PAGES as u64; + Self::compute_blocks(page_list_va, 21) + } + + pub(crate) const fn compute_blocks(page_list_va: u64, blocks: usize) -> Self { + let pages = blocks as u64 * FREE_LIST_BLOCK_PAGES as u64; Self::new(page_list_va, pages << PAGE_POOL_FREE_PAGES_SHIFT, pages) } diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 4f0ae8aa253ed0..479bc91d21ca5f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -14,6 +14,7 @@ use kernel::prelude::*; pub(crate) mod buffer; pub(crate) mod channels; pub(crate) mod compute; +pub(crate) mod compute_pool; pub(crate) mod context; pub(crate) mod freelist; pub(crate) mod initdata; diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index 46a38c5ad16ec3..4df78cca06d461 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -285,11 +285,12 @@ impl InitData { /// Initialize a fresh physical compute QID's row before exposing its queue. An installed /// or rebound queue keeps the row's firmware-owned counters and must never call this again. - pub(super) fn initialize_compute_pool(&self, slot: u16, page_list_va: u64) -> Result { + /// Called for a fresh row or after its free-list release was consumed. + pub(super) fn initialize_compute_pool_blocks(&self, slot: u16, page_list_va: u64, blocks: usize) -> Result { if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { return Err(EINVAL); } - let entry = abi::PagePoolDescriptor::compute(page_list_va); + let entry = abi::PagePoolDescriptor::compute_blocks(page_list_va, blocks); let size = core::mem::size_of::(); let offset = usize::from(slot) * size; let object = self.object(self.page_pool)?; @@ -299,8 +300,8 @@ impl InitData { object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, counters))?; let count = object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, page_count))?; - // SAFETY: the whole fresh row was bounds checked, the QID has never been published, - // and the device mutex excludes another host publisher. No other row is touched. + // SAFETY: the whole fresh row was bounds checked, this is a fresh QID or its release + // was consumed, and the device mutex excludes another host publisher. No other row is touched. unsafe { pointer.write_bytes(0, size); } diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 34b6308620644c..b85d794ace5120 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -238,7 +238,7 @@ pub(crate) trait Backend: Send + Sync { } /// Lazily create or rebind this logical queue's physical compute graph. - fn ensure_compute(&self) -> Result; + fn ensure_compute(&self, scs_pages: u16) -> Result; /// Settle outstanding work of a terminal VM before an ioctl error is published. fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index ea45c44ecd74d0..aae466429e0078 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -604,6 +604,7 @@ struct Shared { #[pin] drm: Mutex>, clusters: u32, + active_cores: u32, descriptor_flags: [u32; 2], crashed: AtomicBool, feed: Arc, @@ -881,11 +882,13 @@ impl Gpu { let preparations = preparation::Gate::new()?; let reclaim = runtime::teardown::ReclaimBatch::new()?; let feed = feed::Feed::new()?; + let active_cores = id.core_masks.iter().map(|mask| mask.count_ones()).sum(); let shared = Arc::pin_init( pin_init!(Shared { drm <- new_mutex!(Some(dev.into()), "g17::Shared::drm"), dev: dev.as_ref().into(), clusters: id.num_clusters, + active_cores, descriptor_flags: [id.perf_control, id.perf_map[0]], crashed: AtomicBool::new(false), feed, diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index 10b66c9ae811ba..cae576fe183af5 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -416,8 +416,8 @@ pub(crate) struct PoolBacking { } impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id) -> Result { - let pool = ComputePool::new(alloc, id.qid())?; + pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id, blocks: usize) -> Result { + let pool = ComputePool::new(alloc, id.qid(), blocks)?; Ok(Self { pool, id }) } @@ -459,6 +459,8 @@ pub(crate) struct Queue { /// the installed queue record still names them until the next owner binds. retained_owner: Option, pool_phase: PoolPhase, + growing: bool, + growth_blocks: usize, /// A killed queue was recycled: its pool/ring must be vacated and re-armed /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). needs_fresh_backing: bool, @@ -518,6 +520,8 @@ impl Queue { owner: Some(owner), retained_owner: None, pool_phase: PoolPhase::Populated, + growing: false, + growth_blocks: 0, needs_fresh_backing: false, released_at: 0, free_list_generation: 0, @@ -1042,6 +1046,7 @@ impl Queue { Ok(true) } pub(crate) fn reusable(&self) -> bool { + if self.growing { return false; } self.installed // A pool release in flight must be witnessed before any successor. && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) @@ -1238,6 +1243,7 @@ impl Queue { consumer: u32, producer: u32, ) -> Result> { + if self.growing { return Ok(None); } let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { return Ok(None); }; @@ -1274,6 +1280,55 @@ impl Queue { self.needs_fresh_backing = false; Ok((self.pool.id(), self.pool.page_list_va()?)) } + pub(crate) fn pool_fits(&self, required: usize) -> bool { + !self.growing && self.pool_phase == PoolPhase::Populated && self.pool.fits(required) + } + + /// Only a capacity increase waits for this queue's previously published work. + /// Other queues and ordinary submissions keep their existing concurrency. + pub(crate) fn begin_growth(&mut self, init: &crate::g17::initdata::InitData, blocks: usize) -> Result { + if self.growing { + if blocks < self.growth_blocks { return Err(EAGAIN); } + self.growth_blocks = blocks; + return Ok(false); + } + if self.quarantined || self.released || self.exit.is_some() + || self.active.len != 0 || self.replays.len != 0 || self.previous.is_some() + || (self.installed && !self.retirement_ready) + || self.pool_phase != PoolPhase::Populated { return Err(EAGAIN); } + if self.installed { + let record = ControlRecord::FreeListRelease(self.pool.release_record(self.free_list_generation)); + let cursor = Rings::publish_control(init, &record)?; + self.pool_phase = PoolPhase::ReleaseSent(cursor); + } else { self.pool_phase = PoolPhase::Vacant; } + self.growing = true; + self.growth_blocks = blocks; + Ok(self.installed) + } + + pub(crate) fn finish_growth(&mut self, init: &crate::g17::initdata::InitData, + pages: &mut Option, + alias: &mut Option, blocks: usize) -> Result { + if !self.growing || blocks < self.growth_blocks { return Err(EAGAIN); } + if let PoolPhase::ReleaseSent(cursor) = self.pool_phase { + let consumer = init.control_consumer()?.load(Ordering::Acquire); + let producer = init.control_producer()?.load(Ordering::Acquire); + if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { + return Err(EAGAIN); + } + } + let binding = self.binding.as_mut().ok_or(EFAULT)?; + let old_alias = binding.pool_alias.take(); + let old_pages = self.pool.take_pages(); + self.pool.rearm(pages.take().ok_or(EINVAL)?)?; + init.initialize_compute_pool_blocks(self.pool.id(), self.pool.page_list_va()?, blocks)?; + binding.pool_alias = alias.take(); + self.pool_phase = PoolPhase::Populated; + self.growing = false; + self.growth_blocks = 0; + Ok(Vacated { _alias: old_alias, _pages: old_pages, _descriptors: None, _ring: None }) + } + /// Exact host identity remains valid even when several commands share an ioctl ID. pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { self.active @@ -1506,6 +1561,7 @@ impl Queue { }; let (epoch, addresses) = self.publication_preflight(&packet, host)?; let crate::g17::command::Validated::Compute { + intersection, scratch, usc_base, cdm_va, @@ -1540,6 +1596,7 @@ impl Queue { self.graph.ensure_record_slot(slot)?; } let args = ComputeArgs { + intersection: *intersection, scratch: *scratch, usc_base: *usc_base, cdm_va: *cdm_va, diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 5f535a8321cf4b..31f2fcd6d016a9 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -127,9 +127,9 @@ impl Queue { }) } - fn ensure_compute(&mut self) -> Result { + fn ensure_compute(&mut self, scs_pages: u16) -> Result { // Admission is checked even when this queue already owns a scheduler. - self.backend.ensure_compute()?; + self.backend.ensure_compute(scs_pages)?; if self.compute.is_none() { self.compute = Some(Scheduler::new( &self.dev, @@ -174,7 +174,8 @@ impl Queue { self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { - self.ensure_compute()?; + let Validated::Compute { scratch, .. } = &command else { return Err(EINVAL); }; + self.ensure_compute(scratch.pages())?; } // Only earlier render ordinals covered by the explicit logical prefix // can delay host publication. These links never become GPU dependencies. diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 65b58d477dfa5d..93e089827ebd09 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -121,10 +121,10 @@ impl Registry { /// replacement context must succeed before its old binding is displaced. /// A vacant queue (USC backing given back) is offered for re-arming only /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { + pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc, required: usize) -> Result { if let Some(queue) = self.compute(owner) { return Ok(if queue.matches(owner, queue.qid(), context)? { - Rebind::Done + if queue.pool_fits(required) { Rebind::Done } else { Rebind::Grow } } else { Rebind::Fresh }); @@ -141,6 +141,7 @@ impl Registry { vacant.get_or_insert(queue.qid()); continue; } + if !queue.pool_fits(required) { continue; } queue.bind_owner(owner, context.clone())?; return Ok(Rebind::Done); } @@ -365,6 +366,7 @@ impl Registry { pub(super) enum Rebind { /// The owner now holds an installed, populated queue. Done, + Grow, /// Only a vacant queue is available; re-arm `qid` with fresh backing. Rearm(u8), /// No retained queue suits the owner: reserve a fresh QID. @@ -549,7 +551,52 @@ impl Backend { result } - fn build_compute(&self) -> Result { + fn grow_compute(&self, required: usize) -> Result { + use kernel::time::{delay::fsleep, Delta}; + { + let state = self.shared.state.lock(); + let firmware = state.as_ref().ok_or(ENODEV)?; + if firmware.queues.compute(self.owner).is_some_and(|queue| queue.pool_fits(required)) { + return Ok(()); + } + } + let lease = self.shared.preparations.enter(false)?; + let dev = self.shared.drm()?; + let uat = self.shared.state.lock().as_ref().ok_or(ENODEV)?.uat.clone(); + let alloc = super::object::Allocator { dev: &dev, uat: &uat }; + let mut pages = Some(super::freelist::Pages::compute_sized(&alloc, required)?); + let mut alias = Some(pages.as_ref().ok_or(EIO)?.map_client(self.context.vm())?); + let mut deferred = DeferredBatch::publication()?; + let started = now_ns(); + loop { + let result = (|| { + let mut state = self.shared.state.lock(); + let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; + if !lease.is_current() { return Err(EAGAIN); } + if self.shared.crashed.load(Ordering::Acquire) { return Err(EIO); } + firmware.scan_compute(None, &mut deferred)?; + let queue = firmware.queues.compute_mut(self.owner).ok_or(EIO)?; + if queue.pool_fits(required) { return Ok(None); } + if queue.begin_growth(&firmware.init, required)? { + fence(Ordering::SeqCst); + firmware.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; + } + queue.finish_growth(&firmware.init, &mut pages, &mut alias, required).map(Some) + })(); + deferred.finish(); + match result { + Ok(old) => { drop(old); return Ok(()); } + Err(error) if error == EAGAIN && lease.is_current() + && now_ns().saturating_sub(started) < WORK_RING_WAIT_NS => { + self.shared.queue_events(); + fsleep(Delta::from_millis(1)); + } + Err(error) => return Err(error), + } + } + } + + fn build_compute(&self, required: usize) -> Result { loop { match self.shared_compute() { Err(error) if error == EAGAIN => continue, @@ -572,8 +619,9 @@ impl Backend { { return Err(EIO); } - match firmware.queues.rebind_compute(self.owner, &self.context)? { + match firmware.queues.rebind_compute(self.owner, &self.context, required)? { Rebind::Done => return Ok(()), + Rebind::Grow => { drop(state); return self.grow_compute(required); } Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), Rebind::Fresh => { let reservation = firmware.queues.reserve_compute(self.owner)?; @@ -590,7 +638,7 @@ impl Backend { Err(qid) => { // Fresh zeroed backing is built without the device mutex; the // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute(&alloc)?; + let pages = crate::g17::freelist::Pages::compute_sized(&alloc, required)?; let ring = compute::DescriptorRing::new(&alloc)?; let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -603,13 +651,13 @@ impl Backend { }; // The firmware consumed this slot's release before the backing // was dropped; the row is rewritten before the queue is kicked. - firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; return Ok(()); } }; let built = (|| { - let pool = compute::PoolBacking::new(&alloc, reservation.id)?; + let pool = compute::PoolBacking::new(&alloc, reservation.id, required)?; { let mut state = self.shared.state.lock(); let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; @@ -618,7 +666,7 @@ impl Backend { } firmware.queues.reserved(reservation)?; let (slot, page_list) = pool.pool_descriptor()?; - firmware.init.initialize_compute_pool(slot, page_list)?; + firmware.init.initialize_compute_pool_blocks(slot, page_list, required)?; } let backing = compute::Backing::new(&alloc, pool)?; // The global scheduler is created after pool/kick backing and @@ -1005,7 +1053,8 @@ impl super::Firmware { } impl Backend { - fn ensure_compute(&self) -> Result { + fn ensure_compute(&self, scs_pages: u16) -> Result { + let required = super::fw::compute_pool::required_blocks(self.shared.active_cores, scs_pages)?; use kernel::time::{delay::fsleep, Delta}; const SPARED_WAIT_MS: usize = 200; let mut deferred = DeferredBatch::publication()?; @@ -1021,7 +1070,7 @@ impl Backend { deferred.finish(); let pending = pending?; if !pending { - return self.build_compute(); + return self.build_compute(required); } // Other owners continue submitting while this owner gives its // queue-local drain witness time to arrive. diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 7f204b9328ffb7..36b075b4521270 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -70,8 +70,8 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } - fn ensure_compute(&self) -> Result { - Backend::ensure_compute(self) + fn ensure_compute(&self, scs_pages: u16) -> Result { + Backend::ensure_compute(self, scs_pages) } fn fail_vm(&self, error: Error) { diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index dd70d2fe228576..2fae6bb473b763 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -1639,6 +1639,11 @@ enum drm_asahi_compute_flags { DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), /** @scs_layout: Reserved; must be zero. */ + /** + * @scs_layout: Packed G17P Gen1 shader-context scratch layout, or zero. + * Bits 0:15 contain stack depth, 16:23 levels, 24:31 cliques, + * 32:47 user bytes and 48:63 driver bytes. Non-G17P paths require zero. + */ __u64 scs_layout; /** @@ -1658,6 +1663,23 @@ enum drm_asahi_compute_flags { * queue's compute command count and the header's cdm_barrier instead. */ __u64 prior_cdm; + + /** @cdm_auxiliary_allocation: G17P auxiliary allocation, zero or 8..32. */ + __u32 cdm_auxiliary_allocation; + /** @reserved: MBZ. */ + __u32 reserved; + /** @intersection_pool: 16-KiB aligned G17P intersection-table address. */ + __u64 intersection_pool; + /** @intersection_pool_bytes: Mapped pool size in bytes, a multiple of 16 KiB. */ + __u64 intersection_pool_bytes; + /** @intersection_first: First 128-byte entry in @intersection_pool. */ + __u32 intersection_first; + /** @intersection_count: Number of 128-byte entries, less than 2^18. */ + __u32 intersection_count; + /** @intersection_default_program: Encoded default G17P program address. */ + __u64 intersection_default_program; + /** @intersection_default_control: Default G17P program control word. */ + __u64 intersection_default_control; }; /** From bd722f536941a9fa8f2afb92b05beba64632854d Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sun, 11 Oct 2026 20:06:54 -0400 Subject: [PATCH 380/631] Merge the running hotplug and mixed-ABI kernel into aurora-wip Integrate source commit 5c9c4446fd2f, used by 7.1.12-aurora11.31-hotplug-mixed2+, into the current upstream tree. Keep the latest public mixed-userspace-ABI series and its validation fixes, G17 GPU work, newer display/RTKit/NVMe code, and current J700 sleep and Bluetooth quiesce/drain teardown. The older fixed-hub DWC3 sleep callbacks are superseded by 1e79a47b63f and are deliberately not reinstated. Bring across data-port display isolation, busy CD321x revalidation, chained PCIe IRQ teardown gating, and activation-work draining. Include PCIe-C cold initialization, scan/readiness and runtime-PM policy, missing SEP board enablement and M1 reload protection, passive SEP status, Neo retained-DMA/kexec/PCI/VFIO admission guards, RF failure and policy retry handling, shared config additions, and existing release/Touch ID tooling. Reconcile retained-DMA admission with the current board-linked provider: keep the newer admitted Neo s2idle path, refuse sleep during incomplete admission and unsafe restoration, retain the runtime kexec veto, and allow Bluetooth and the provider to coexist with KEXEC_CORE. Preserve current SEP xART/provisioning opt-in defaults and authenticated capture lifetimes. Interpret apple,pciec-kernel-init as the t8103 sequence only on t8103; T600x/T602x board properties use the separate cold-init policy. Validation: affected C/Rust driver and kexec/VFIO objects compile with the repository's privately built, layout-tested bindgen; seven affected Apple DTBs compile. QEMU arm64 native-16K KUnit: 257 passed, 39 skipped, 0 failed (296 total), including PCI, kexec and MT7932 RF tests. Native 16K/4K exec and thread fixtures pass, along with all 17 swap cases and 16 userspace radio-runner unit tests. Shell syntax and git diff whitespace checks pass. Hardware hotplug and Neo sleep still require testing of this integrated revision. The installed kernel and boot configuration are unchanged. --- Documentation/PCI/controller/apple-t8140.rst | 7 +++- .../device_drivers/wifi/mt7932-neo.rst | 10 ++++-- MAINTAINERS | 1 - arch/arm64/configs/asahi.config | 5 ++- drivers/bluetooth/mt7932_bt_pcie.c | 4 +++ drivers/gpu/drm/apple/dcp.c | 8 ++--- drivers/net/wireless/mediatek/mt7932/mt7932.c | 3 +- .../net/wireless/mediatek/mt7932/startup.c | 25 +++++++++++++++ drivers/pci/controller/Kconfig | 2 +- .../pci/controller/pcie-apple-piodma-diag.c | 12 ++++--- drivers/pci/controller/pcie-apple.c | 32 +++++++++++++++++++ drivers/thunderbolt/apple.c | 3 ++ 12 files changed, 92 insertions(+), 20 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 5e336b3a384812..bd8b69242b7375 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -21,7 +21,12 @@ a module dependency on a provider whose lifetime cannot end at unload. Suspend-to-idle suspend retains the arena after checking the completed request, idle FIFO and command state. Resume restores the existing configuration banks without issuing a second bootstrap. Hibernation, deeper suspend and -kexec are unsupported after activation. Failed or uncertain bootstrap state +kexec are refused while admission or retained DMA ownership is active. The +runtime kexec guard permits the kernel to support kexec on other machines. +Host removal, sysfs reset, hotplug, VFIO assignment and native AER/DPC +recovery are withheld on this retained hierarchy. Sleep is refused until +radio admission completes; the existing suspend-to-idle path remains available +after successful admission. Failed or uncertain bootstrap state requires a full external hardware reset; there is no command retry. The PIODMA arena remains allocated until external reset. Controller removal, diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 00a608fab482da..8a9ff2b845bc7a 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -24,7 +24,8 @@ The tested configuration includes ``CONFIG_MT7932_FULLMAC=m``, ``CONFIG_BT_LE=y``, ``CONFIG_CRYPTO_AES=y`` and ``CONFIG_CRYPTO_CMAC=y``. The Bluetooth transport can be built with modular Bluetooth and system sleep. The separately selected bootstrap provider has its own lifecycle -restrictions. Kexec is excluded while either retained experiment is enabled. +restrictions. A runtime guard refuses kexec while radio admission or retained +bootstrap DMA ownership is active. Kernels may still support kexec elsewhere. The station uses 2.4 GHz channels 1-13 and the 5 GHz channels 36-64, 100-144 and 149-165 that the cfg80211 regulatory domain permits. Radar (DFS) channels stay passive: the firmware listens on them while the station is not @@ -54,7 +55,9 @@ as in the tested sequence, however the drivers are loaded. Bluetooth therefore stays unbound while the Wi-Fi driver is missing or has failed, including when a failed Wi-Fi function stays bound to keep DMA it could not stop. A device link makes the driver core unbind Bluetooth before Wi-Fi; unloading -``mt7932_bt_pcie`` does not affect Wi-Fi. The probe rejects a function that is +``mt7932_bt_pcie`` does not affect Wi-Fi. Wi-Fi pins its module before publishing +DMA pointers, and refuses function reset while its Bluetooth sibling is live. +The probe rejects a function that is not cold, which is what now prevents a reprobe after a failed or uncertain admission; ``enable=0`` leaves the function unbound. The transport also disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware @@ -155,7 +158,8 @@ Limitations ----------- * PCI bootstrap memory remains retained until external reset. Controller - removal, memory reuse, suspend and kexec are unqualified. + removal and memory reuse are unqualified. Runtime guards refuse reset and + kexec; suspend-to-idle retains the successfully admitted arena. * Bluetooth system sleep uses a bounded firmware quiesce/restore handshake around HCI suspend/resume, preserving its DMA arena. Hibernation is rejected. Removal first quiesces firmware, then stops work and interrupts, disables diff --git a/MAINTAINERS b/MAINTAINERS index 46b22192afaa69..f7f7badf5f8f0e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2672,7 +2672,6 @@ F: include/dt-bindings/interrupt-controller/apple-aic.h F: include/dt-bindings/pinctrl/apple.h F: include/linux/mfd/macsmc.h F: include/linux/soc/apple/* -F: tools/testing/selftests/drivers/apple/ F: include/uapi/drm/asahi_drm.h F: include/uapi/linux/apple-sep.h F: net/bluetooth/brcm.c diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index e332098f244ca0..41797e7774899f 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -12,13 +12,12 @@ CONFIG_BT_BRCMEXT=y CONFIG_BT_HCIBCM4377=m CONFIG_BT_MTK7932_PCIE=m CONFIG_PCIE_APPLE=y -CONFIG_PCIE_APPLE=m -CONFIG_PCIE_APPLE_PIODMA_DIAG=y -CONFIG_PCIEASPM=y CONFIG_NVME_APPLE=m CONFIG_MMC_SDHCI_PCI=m CONFIG_BRCMFMAC=m CONFIG_BRCMFMAC_PCIE=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_PCIEASPM=y CONFIG_NET_VENDOR_INTEL=y CONFIG_IGB=m CONFIG_IGC=m diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index fc1b2f636ba782..dd689d78e98783 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -1146,6 +1147,9 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) !of_machine_is_compatible("apple,t8140") || pci_domain_nr(pdev->bus) || pdev->bus->number != 1 || pdev->devfn != PCI_DEVFN(0, 1)) return -ENODEV; + ret = apple_piodma_radio_check(pdev); + if (ret) + return ret; ret = bt7932_wait_for_wifi(pdev); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c2b6638aab9ce9..5841c6c9d77c26 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -708,7 +708,7 @@ static void dcp_typec_route_waiting(void) if (!route || dcp_typec_route_activate(route, route->xbar)) continue; port->owner = route; - port->dp_release_deadline = 0; + dcp_typec_port_attach(port); } } @@ -804,7 +804,7 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, } port->owner = route; port->preferred_route = route; - port->dp_release_deadline = 0; + dcp_typec_port_attach(port); } @@ -4067,14 +4067,12 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); - disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); } static void dcp_enable_typec_work(struct apple_dcp *dcp) { enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); - enable_delayed_work(&dcp->typec_fabric_retrain_wq); /* A cable can be routed before the DRM component binds. */ if (READ_ONCE(dcp->typec_cable_connected)) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); @@ -4353,7 +4351,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) cancel_work_sync(&dcp->bl_update_wq); } cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); cancel_work_sync(&dcp->vblank_wq); if (dcp->connector) cancel_work_sync(&dcp->connector->hotplug_wq); @@ -4621,7 +4618,6 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); - disable_delayed_work(&dcp->typec_fabric_retrain_wq); platform_set_drvdata(pdev, dcp); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index fbaa8e8fd1fc32..6c5de74b39b33e 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -3,8 +3,8 @@ * Covers firmware download and the NIC capability exchange. * All addresses here are BAR0 offsets, never chip/CPU physical addresses. */ -#include #include +#include #include "mt7932.h" @@ -1406,6 +1406,7 @@ static struct pci_driver mt_driver = { .driver.suppress_bind_attrs = true, .driver.pm = pm_sleep_ptr(&mt_pm_ops), }; + /** * mt7932_fullmac_ready() - check that the Wi-Fi function is up * @pdev: function 0 of the radio diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 9c908ad551f7cb..ec9b70ad6ea148 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -313,6 +313,24 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli rtnl_unlock(); } +/* Retry only a completed file lookup that failed before any policy SET. + * Consume the flag under the same lock as the worker publication, so repeated + * userspace requests cannot continually invalidate an in-flight attempt. + */ +void mt_retry_missing_policy(struct mt7932 *m) +{ + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (m->reg_retryable && m->reg_pending && m->interface_registered && + !m->stopping && !m->policy_failed && !m->link_failed && !m->cal_state.error) { + m->reg_retryable = false; + m->reg_generation++; + schedule_work(&m->startup_work); + } + spin_unlock_irqrestore(&m->response_lock, flags); +} + static void mt_startup_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, startup_work); @@ -379,6 +397,13 @@ static void mt_startup_work(struct work_struct *work) if (!ret) { mt7932_policy_filter(®, &policy); mt_policy_disable(m, &policy); + ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); + if (!ret) { + mt7932_policy_filter(®, &policy); + mt_policy_disable(m, &policy, generation); + } else { + dev_err_ratelimited(&m->pdev->dev, "local input %s has invalid policy: %d\n", path, ret); + } } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 505315c68ff3ee..7e2f0198da0d7d 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -59,7 +59,7 @@ config PCIE_APPLE config PCIE_APPLE_PIODMA_DIAG bool "J700 PIODMA diagnostic (requires full reset after use)" - depends on PCIE_APPLE != n && ARCH_APPLE && IOMMU_DMA + depends on PCIE_APPLE=y && ARCH_APPLE && IOMMU_DMA depends on PCIEASPM help Build the experimental SID17 retained bootstrap provider. It submits diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index e39172b97b9389..1ba2dcc13cab3f 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -171,6 +171,10 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, switch (action) { case PM_SUSPEND_PREPARE: + if (pm_suspend_no_platform() && READ_ONCE(diag->ready) && + smp_load_acquire(&diag->radio_admitted)) + return NOTIFY_DONE; + fallthrough; case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: dev_warn_ratelimited(diag->dev, @@ -188,6 +192,9 @@ static int apple_piodma_pm_prepare(struct device *dev) /* Acquire admission before either PCI children or DMA pointers escape. */ if (!diag || !smp_load_acquire(&diag->guarded)) return 0; + if (pm_suspend_no_platform() && READ_ONCE(diag->ready) && + smp_load_acquire(&diag->radio_admitted)) + return 0; /* * A /dev/snapshot file may have sent its PREPARE notifier before this * supplier probed. Its later image/restore/suspend ioctls go straight @@ -198,10 +205,6 @@ static int apple_piodma_pm_prepare(struct device *dev) return -EOPNOTSUPP; } -static const struct dev_pm_ops apple_piodma_pm_ops = { - .prepare = apple_piodma_pm_prepare, -}; - static int apple_piodma_diag_root(struct apple_piodma_diag *diag, struct pci_dev *root) { int ret; @@ -914,6 +917,7 @@ static int apple_piodma_diag_freeze(struct device *dev) } static const struct dev_pm_ops apple_piodma_diag_pm_ops = { + .prepare = apple_piodma_pm_prepare, .suspend_noirq = apple_piodma_diag_suspend_noirq, .resume_noirq = apple_piodma_diag_resume_noirq, .freeze = apple_piodma_diag_freeze, diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 2e2b76f6037f6f..3d870c5de4bf7f 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -843,6 +843,21 @@ static void apple_pcie_port_disable_irq(struct apple_pcie_port *port) raw_spin_lock_irqsave(&port->irq_lock, flags); port->irq_disabled = true; raw_spin_unlock_irqrestore(&port->irq_lock, flags); + * A chained handler is not tracked by IRQD_IRQ_INPROGRESS, and AIC does not + * report IRQCHIP_STATE_ACTIVE. Gate and drain MMIO explicitly before masking + * the parent; disable_irq() alone cannot synchronize this chained handler. + */ +static void apple_pcie_port_disable_irq(struct apple_pcie_port *port) +{ + unsigned long flags; + + if (!port->irq || !port->domain || port->irq_disabled) + return; + + raw_spin_lock_irqsave(&port->irq_lock, flags); + port->irq_disabled = true; + raw_spin_unlock_irqrestore(&port->irq_lock, flags); + disable_irq(port->irq); } static void apple_pcie_port_enable_irq(struct apple_pcie_port *port) @@ -2436,6 +2451,20 @@ static int apple_pcie_tunnel_pci_notify(struct notifier_block *nb, pm_runtime_get_noresume(&pdev->dev); if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) pcie_aspm_remove_cap(pdev, PCI_EXP_LNKCAP_ASPM_L1); + */ +static int apple_pcie_tunnel_pci_notify(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct apple_pcie *pcie = container_of(nb, struct apple_pcie, + tunnel_pci_nb); + struct pci_dev *pdev = to_pci_dev(data); + + if (pci_host_bridge_priv(pci_find_host_bridge(pdev->bus)) != pcie) + return NOTIFY_DONE; + + switch (action) { + case BUS_NOTIFY_ADD_DEVICE: + pm_runtime_get_noresume(&pdev->dev); break; case BUS_NOTIFY_DEL_DEVICE: pm_runtime_put_noidle(&pdev->dev); @@ -2685,6 +2714,7 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) ret = apple_pcie_neo_check_cold(pcie, root); if (ret) goto close_config; + apple_piodma_bootstrap_admitted(pcie->piodma_supplier); pci_bus_add_devices(root->subordinate); for (function = 0; function < 2; function++) { endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); @@ -3080,6 +3110,8 @@ int apple_pcie_tunnel_restore(struct device *dev) pci_rescan_bus(bridge->bus); pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); pcie->bus_stopped = false; + list_for_each_entry(port, &pcie->ports, entry) + apple_pcie_port_enable_irq(port); pci_unlock_rescan_remove(); dev_info(dev, "PCIe-C hierarchy restored after tunnel activation\n"); diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index fd5c4eabcf5aaf..de7087cd8bac6f 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -3202,6 +3202,9 @@ static void apple_cio_remove(struct platform_device *pdev) /* A set may hold the switch read lock while waiting for acio->lock. */ typec_thunderbolt_switch_unregister(acio->tbt_switch); guard(mutex)(&acio->lock); + /* The worker can notify this switch; stop current and future enqueues. */ + disable_delayed_work_sync(&acio->pcie_tunnel_work); + typec_thunderbolt_switch_unregister(acio->tbt_switch); if (acio->current_cable_info) apple_cio_stop(acio); } From 554960b43d1c89b2b57cdea26915cf727474bb7f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 14:30:09 -0400 Subject: [PATCH 381/631] arm64: dts: apple: t8140-j700: keep the DCS rail out of thermal control The soc-dcs-thermal zone let the power allocator cap the DRAM controller (DCS) rail through the PMC whenever Tp0t passed 95 C. Most of that heat comes from the GPU and CPUs rather than from the DCS rail, so the governor could not settle. Under a sustained Minecraft load it stepped the cap between states 1 and 4 every 1-2 s, and at state 4 the rail was held at its lowest state while the PMP and CLPC both voted the highest. The game's GPU fence rate followed the cap (Pearson r = -0.98): about 2,000/s at state 4 and 5,200/s at state 1. In game the frame rate was seen swinging from about 150 FPS down to 30-50 FPS and back every few seconds. With the zone disabled and the cap lifted on the same session, the rate held between 4,500 and 4,900/s. Tp0t settled near 100 C (peak 105 C), and the CPU die sensors stayed at 88-97 C, under the 110 C critical trips of the CPU zones. The PMP already drops its DCS vote when the GPU idles, so the rail only needs to follow demand. Drop the zone. The PMC keeps its cooling cells and power model. Assisted-by: LLM Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t8140-j700.dts | 56 ++++-------------------- 1 file changed, 9 insertions(+), 47 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 842e1b2bcf5723..066175edbc83a7 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -381,52 +381,6 @@ }; }; }; - - /* - * Once the PMP knows the GPU is on, it votes the DRAM controller - * (DCS) up with GPU load, and the GPU and memory system draw up - * to 2.4 W more. The CPU zones above cannot hold that back: on a - * GPU-bound load the CPUs request little power, so with the - * performance governor the die passed 95 C within 35 s and - * reached 104 C before they throttled. This zone limits the DCS - * vote instead, through the PMC. With it the same load peaks at - * 97 C, as it does without any DCS vote, and a ten-minute - * balanced GPU load loses 1-2% of its throughput. - * - * Tp0t reads at or above every CPU die sensor under GPU load, - * and leads them by about 1 C when the CPUs also run hot. The - * sustainable power is the whole DCS range, so the vote is only - * trimmed above the control trip; the PID gains follow from it - * and the 6 C between the trips. - */ - soc-dcs-thermal { - polling-delay-passive = <500>; - polling-delay = <500>; - thermal-sensors = <&smc_hwmon 6>; /* Tp0t */ - sustainable-power = <2407>; - - trips { - soc_dcs_alert: trip-alert { - temperature = <89000>; - hysteresis = <2000>; - type = "passive"; - }; - - soc_dcs_control: trip-control { - temperature = <95000>; - hysteresis = <2000>; - type = "passive"; - }; - }; - - cooling-maps { - map0 { - trip = <&soc_dcs_control>; - cooling-device = - <&pmc THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; - }; - }; - }; }; }; @@ -472,6 +426,14 @@ /* * System power above DCS state 0 at each DCS state, measured with * glmark2's terrain scene, GPU-bound, on the J700. + * + * No thermal zone limits the DCS rail. The PMP already drops its + * DCS vote when the GPU idles, and under load the rail must stay + * where the PMP puts it: a power allocator zone on Tp0t at 95 C + * stepped the limit between states 1 and 4 every 1-2 s in + * Minecraft, swinging the game's GPU submission rate about 2.5x. + * With the limit lifted the rate held within 8% and Tp0t settled + * near 100 C, under the 110 C critical trips of the CPU zones. */ apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; @@ -676,7 +638,7 @@ * the SMC key groups and are not otherwise verified. */ - /* Channel 6, the sensor of soc-dcs-thermal: keep it first here. */ + /* Channel 6: reads at or above every CPU die sensor under GPU load. */ temperature-Tp0t { apple,key-id = "Tp0t"; label = "SMC Tp0t"; From 79560287b43246fc73912d588609b84a000c7230 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:15:50 +1100 Subject: [PATCH 382/631] drm: apple: isolate an experimental external H17P transport Add a separate external H17P method profile and compact service serializer for bounded trials. Keep the existing internal H17P send rejection and boot-mode inheritance unchanged. Restrict the new serializer to the existing DisplayPort and EDID service calls, validate ring occupancy, and reject unknown callbacks without fabricating success. The layout of the compact transmit header and existing command descriptor is untested on hardware. No hardware compatibility is claimed. This commit does not enable or match an external controller. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 130 +++++++++++++++++++++------ drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/dcp-internal.h | 3 + drivers/gpu/drm/apple/dcp.c | 10 +++ drivers/gpu/drm/apple/dptxep.c | 6 +- drivers/gpu/drm/apple/iomfb.c | 19 +++- 6 files changed, 140 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index d5d5f3430d0532..e63de19a078566 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -651,8 +652,10 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", ep->endpoint, channel, type, eshdr->category); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); + if (ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, + payload, payload_size, true); } static bool afk_validate_h17p_header(const u8 *data, size_t size) @@ -728,6 +731,16 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, subtype = le16_to_cpu(eshdr->type); trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); + if (ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + ep->experimental_rx_logs < 32) { + ep->experimental_rx_logs++; + dev_info(ep->dcp->dev, + "external AFK ep=%u channel=%u type=%u subtype=%u category=%u bytes=%zu\n", + ep->endpoint, channel, type, subtype, eshdr->category, + data_size); + } + service = afk_epic_find_service(ep, channel); @@ -790,8 +803,10 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); + if (ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, + payload, payload_size, true); return; too_small: @@ -974,6 +989,8 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, struct epic_sub_hdr *eshdr; unsigned long flags; size_t total_epic_size, total_size; + bool compact = ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; /* @@ -981,30 +998,73 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, * 8-byte sub-header, with the service channel carried in the EPIC * header on queue channel 0. Only the receive side of that format is * known, and the 24-byte framing below would be misparsed, so send - * nothing. The display path needs no AP-initiated EPIC traffic: the + * nothing for the established internal profile. The external-only + * experiment below tests receive-layout symmetry. Internal callers are: the * callers are optional (verbose firmware logging, EDID copy, audio, * DPTX, debugfs) or replies to calls only DPTX services make. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); return -EOPNOTSUPP; } + /* Only existing DPTX/EDID service commands participate in this trial. */ + if (compact && + (channel > U16_MAX || payload_len > SZ_1M || + (payload_len && !payload) || stype != EPIC_SUBTYPE_STD_SERVICE || + (ep->endpoint != DPTX_ENDPOINT && ep->endpoint != DPAVSERV_ENDPOINT) || + !((ecat == EPIC_CAT_COMMAND && etype == EPIC_TYPE_COMMAND && + payload_len == sizeof(struct epic_cmd)) || + (ecat == EPIC_CAT_REPLY && + (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) + return -EOPNOTSUPP; + spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { ret = -ESHUTDOWN; goto out; } + if (compact && (!ep->txbfr.ready || !ep->txbfr.rptr || + !ep->txbfr.wptr || !ep->txbfr.buf)) { + ret = -ESHUTDOWN; + goto out; + } + dma_rmb(); rptr = le32_to_cpu(*ep->txbfr.rptr); wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); - total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; + total_epic_size = sizeof(*ehdr) + + (compact ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + payload_len; total_size = sizeof(*hdr) + total_epic_size; + if (compact) { + u32 available, needed; + + if (!ep->txbfr.ready || !ep->txbfr.stride || + (ep->txbfr.stride & (ep->txbfr.stride - 1)) || + rptr >= ep->txbfr.bufsz || wptr >= ep->txbfr.bufsz || + !IS_ALIGNED(rptr, ep->txbfr.stride) || + !IS_ALIGNED(wptr, ep->txbfr.stride) || + total_size >= ep->txbfr.bufsz) { + ret = -EPROTO; + goto out; + } + available = rptr > wptr ? rptr - wptr : ep->txbfr.bufsz - wptr + rptr; + needed = ALIGN(total_size, ep->txbfr.stride); + if (total_size > ep->txbfr.bufsz - wptr) + needed += ep->txbfr.bufsz - wptr; + /* Retain an empty slot; equal read/write pointers mean empty. */ + if (needed >= available) { + ret = -ENOSPC; + goto out; + } + } + hdr = hdr2 = NULL; /* @@ -1085,31 +1145,45 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(channel); + hdr->channel = cpu_to_le32(compact ? 0 : channel); hdr->type = cpu_to_le32(etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); - ehdr = ep->txbfr.buf + wptr; - memset(ehdr, 0, sizeof(*ehdr)); - ehdr->version = 2; - ehdr->seq = cpu_to_le16(ep->qe_seq++); - ehdr->timestamp = cpu_to_le64(0); - wptr += sizeof(*ehdr); - - eshdr = ep->txbfr.buf + wptr; - memset(eshdr, 0, sizeof(*eshdr)); - eshdr->length = cpu_to_le32(payload_len); - eshdr->version = 4; - eshdr->category = ecat; - eshdr->type = cpu_to_le16(stype); - eshdr->timestamp = cpu_to_le64(0); - eshdr->tag = cpu_to_le16(tag); - if (ecat == EPIC_CAT_REPLY) - eshdr->inline_len = cpu_to_le32(payload_len - 4); - else - eshdr->inline_len = cpu_to_le32(0); - wptr += sizeof(*eshdr); + if (compact) { + u8 *wire = ep->txbfr.buf + wptr; + + /* Experimental symmetry with the published H17P receive layout. */ + memset(wire, 0, sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p)); + wire[0] = ep->qe_seq++; + put_unaligned_le16(channel, wire + 2); + put_unaligned_le32(total_epic_size - 8, wire + 4); + wire[16] = stype; + wire[17] = ecat; + put_unaligned_le32(tag, wire + 20); + wptr += sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p); + } else { + ehdr = ep->txbfr.buf + wptr; + memset(ehdr, 0, sizeof(*ehdr)); + ehdr->version = 2; + ehdr->seq = cpu_to_le16(ep->qe_seq++); + ehdr->timestamp = cpu_to_le64(0); + wptr += sizeof(*ehdr); + + eshdr = ep->txbfr.buf + wptr; + memset(eshdr, 0, sizeof(*eshdr)); + eshdr->length = cpu_to_le32(payload_len); + eshdr->version = 4; + eshdr->category = ecat; + eshdr->type = cpu_to_le16(stype); + eshdr->timestamp = cpu_to_le64(0); + eshdr->tag = cpu_to_le16(tag); + if (ecat == EPIC_CAT_REPLY) + eshdr->inline_len = cpu_to_le32(payload_len - 4); + else + eshdr->inline_len = cpu_to_le32(0); + wptr += sizeof(*eshdr); + } memcpy(ep->txbfr.buf + wptr, payload, payload_len); wptr += payload_len; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 04c612d8e6e17b..0e36317192cc55 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -294,6 +294,8 @@ struct afk_ringbuffer { struct apple_dcp_afkep { struct apple_dcp *dcp; + /* Bounded external protocol observations, owned by the RX worker. */ + u8 experimental_rx_logs; u32 endpoint; struct workqueue_struct *wq; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index c8b46a57a69539..9265ab28a98156 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -169,6 +169,7 @@ struct dcp_panel { enum dcp_iomfb_method_profile { DCP_IOMFB_METHODS_DEFAULT, DCP_IOMFB_METHODS_H17G, + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, }; struct apple_dcp_hw_data { @@ -208,6 +209,8 @@ struct apple_dcp { struct apple_connector *connector; struct apple_dcp_hw_data hw; + /* Protected by the H17P IOMFB receive mutex. */ + u8 experimental_callback_logs; /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5841c6c9d77c26..3fba84b6361ec6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -146,6 +146,16 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) +static bool experimental_h17p_external; +module_param(experimental_h17p_external, bool, 0444); +MODULE_PARM_DESC(experimental_h17p_external, + "Enable experimental T8140 external display protocol"); + +bool dcp_h17p_external_enabled(void) +{ + return experimental_h17p_external; +} + static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 2fc97c208d8f39..a057e09dea63f6 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -749,7 +749,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, trace_dptxport_apcall(dptx, idx, data_size); dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); - if (data_size) + if (data_size && service->ep->dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) print_hex_dump_debug("DPTXPort: apcall data: ", DUMP_PREFIX_OFFSET, 16, 1, data, min(data_size, (size_t)64), true); @@ -862,6 +863,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, memset(reply, 0, 4); return 0; default: + if (service->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return -EOPNOTSUPP; /* just try to ACK and hope for the best... */ dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", idx); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 11eed4b5fb1cb0..64ed3552cae747 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -843,6 +843,14 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + dcp->experimental_callback_logs < 64) { + dcp->experimental_callback_logs++; + dev_info(dev, "external IOMFB callback=%d in=%u out=%u context=%u\n", + tag, hdr->in_len, hdr->out_len, context); + } + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && dcp->cb_handlers[tag]; if (dcp->fw_compat == DCP_FIRMWARE_H17P && @@ -865,6 +873,13 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, !test_and_set_bit(tag, dcp->unknown_callbacks)) dev_warn(dev, "received unknown callback %c%c%c%c\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Retain the failed session without inventing a successful reply. */ + dcp->crashed = true; + complete(&dcp->start_done); + return; + } /* * Leaving a callback unanswered wedges the coprocessor: it * waits for the ack forever and the outer call never returns. @@ -1311,7 +1326,9 @@ static bool dcp_mode_settable(struct apple_dcp *dcp, if (!dcp_mode) return false; - if (dcp->fw_compat == DCP_FIRMWARE_H17P) + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; return true; From 2f52ccd5072c197cb597aa9bbccb1e386c43dbc5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:15:50 +1100 Subject: [PATCH 383/631] drm: apple: prepare optional external H17P before component matching Add a default-off T8140 external-controller experiment. Require the inherited START, STOPPED and IDLE state before the standard start/wake path and pin its DMA lifetime before any startup write. Complete this optional preparation during platform probe so failed or deferred controllers do not become prerequisites for the internal DRM master. Only successfully bound, prepared controllers are included. Preserve started-session mappings on failure and quiesce retained external work during shutdown. Internal H17P lifecycle remains unchanged. Hardware qualification and validated external resource mappings remain required before this experiment can be used as a supported display path. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 99 +++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 2 + 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index ced558ce38e3f1..8c588f7f2bcd72 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -811,7 +811,8 @@ static void apple_drm_quiesce_connectors(struct drm_device *drm) static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && - !of_device_is_compatible(np, "apple,t8140-dcpext"); + (!of_device_is_compatible(np, "apple,t8140-dcpext") || + dcp_h17p_external_ready(np)); } static int apple_drm_init_dcp(struct device *dev) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 9265ab28a98156..bcee4609c7add8 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -211,6 +211,7 @@ struct apple_dcp { struct apple_dcp_hw_data hw; /* Protected by the H17P IOMFB receive mutex. */ u8 experimental_callback_logs; + bool external_prepared; /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3fba84b6361ec6..36e2421c5c058a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -66,6 +67,9 @@ #define APPLE_DCP_COPROC_CPU_CONTROL 0x44 #define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) +#define APPLE_DCP_COPROC_CPU_STATUS 0x48 +#define APPLE_DCP_COPROC_CPU_STATUS_STOPPED BIT(1) +#define APPLE_DCP_COPROC_CPU_STATUS_IDLE BIT(5) struct dcp_session_ref { struct list_head link; @@ -96,7 +100,10 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) { struct dcp_session_ref *session; - if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || + if ((!dcp->hw.adopt_live_session && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) || + dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) return 0; session = kzalloc_obj(*session); @@ -156,6 +163,27 @@ bool dcp_h17p_external_enabled(void) return experimental_h17p_external; } +bool dcp_h17p_external_ready(struct device_node *np) +{ + struct platform_device *pdev; + struct apple_dcp *dcp; + bool ready = false; + + if (!dcp_h17p_external_enabled()) + return false; + pdev = of_find_device_by_node(np); + if (!pdev) + return false; + /* Failed and deferred probes must never become required components. */ + if (device_is_bound(&pdev->dev)) { + dcp = platform_get_drvdata(pdev); + ready = dcp && READ_ONCE(dcp->external_prepared) && + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); + } + put_device(&pdev->dev); + return ready; +} + static bool show_notch; module_param(show_notch, bool, 0644); MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); @@ -4088,6 +4116,17 @@ static void dcp_enable_typec_work(struct apple_dcp *dcp) mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, 0); } +static bool dcp_external_is_idle(struct apple_dcp *dcp) +{ + u32 control = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + u32 status = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_STATUS); + + /* Same externally inherited idle state accepted by the public loader. */ + return (control & APPLE_DCP_COPROC_CPU_CONTROL_RUN) && + (status & APPLE_DCP_COPROC_CPU_STATUS_STOPPED) && + (status & APPLE_DCP_COPROC_CPU_STATUS_IDLE); +} + static int dcp_comp_bind(struct device *dev, struct device *main, void *data) static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *data) { @@ -4100,6 +4139,9 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat return dev_err_probe(dev, -EBUSY, "Live DCP session cannot be rebound; reboot required\n"); + if (dcp->external_prepared) + return 0; + /* A timed-out prior session may still scan these mappings. */ mutex_lock(&dcp->swapped_out_fbs_lock); ret = list_empty(&dcp->swapped_out_fbs) ? 0 : -EBUSY; @@ -4134,6 +4176,12 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat "DCP was not left running by the bootloader\n"); } + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + !dcp_external_is_idle(dcp)) + return dev_err_probe(dev, -EBUSY, + "External DCP is not in the inherited idle state\n"); + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); @@ -4240,6 +4288,19 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Pin before the first write or mailbox wake can expose DMA. */ + ret = dcp_pin_live_session(dcp); + if (ret) + goto err_piodma; + if (!dcp_external_is_idle(dcp)) { + ret = dev_err_probe(dev, -EBUSY, + "External DCP changed state before startup\n"); + goto err_piodma; + } + } + dcp->swapped_out_fbs = (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); spin_lock_init(&dcp->swapped_out_lock); @@ -4522,7 +4583,8 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); /* Not part of the display subsystem yet; see apple_dcp_usable(). */ - if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) { + if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext") && + !dcp_h17p_external_enabled()) { dev_info(dev, "external display coprocessor not supported yet\n"); return -ENODEV; } @@ -4958,6 +5020,24 @@ static int dcp_platform_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dcp); dcp_ext_give_up(dcp, ret, "probe"); return component_add(dev, &dcp_comp_ops); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + /* Complete optional startup before the DRM master can depend on it. */ + ret = dcp_comp_bind(dev, NULL, NULL); + if (ret) { + dcp_disable_typec_work(dcp, true); + disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + return ret; + } + WRITE_ONCE(dcp->external_prepared, true); + } + + ret = component_add(&pdev->dev, &dcp_comp_ops); + if (ret && dcp->external_prepared) { + dcp_disable_typec_work(dcp, true); + disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + } + return ret; } static void dcp_platform_remove(struct platform_device *pdev) @@ -4982,6 +5062,12 @@ static void dcp_platform_shutdown(struct platform_device *pdev) if (dcp && dcp->external && !dcp->external_native) return; + if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + dcp_comp_unbind(&pdev->dev, NULL, NULL); + return; + } + if (dcp && dcp->hw.adopt_live_session && dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { @@ -5254,6 +5340,14 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .firmware_request = 0x68000, }; +/* Separate from the established internal-panel H17P profile. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { + .num_dptx_ports = 1, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + .adopt_live_session = false, + .firmware_compat = DCP_FIRMWARE_H17P, +}; + /* * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering * and is left running by the bootloader. @@ -5304,6 +5398,7 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6031-dcp", .data = &apple_dcp_hw_t6031, }, { .compatible = "apple,t8122-dcpext", .data = &apple_dcp_hw_t8122_dcpext, }, { .compatible = "apple,t8140-dcp", .data = &apple_dcp_hw_t8140, }, + { .compatible = "apple,t8140-dcpext", .data = &apple_dcp_hw_t8140_external, }, { .compatible = "apple,t8142-dcp", .data = &apple_dcp_hw_t8142, }, { .compatible = "apple,dcp", .data = &apple_dcp_hw_dcp, }, { .compatible = "apple,dcpext", .data = &apple_dcp_hw_dcpext, }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index cb45204398917c..2edc08e95f7349 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -20,6 +20,8 @@ struct apple_atomic_state { }; #define to_apple_atomic_state(s) container_of(s, struct apple_atomic_state, base) +bool dcp_h17p_external_enabled(void); +bool dcp_h17p_external_ready(struct device_node *np); struct apple_crtc { struct drm_crtc base; From 4a3d7a33e3ae9e03180f26e6e63909931463ee33 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:20:49 +1100 Subject: [PATCH 384/631] arm64: dts: apple: describe disabled J700 external resources Describe the external display controller and four display apertures plus its fifth clock aperture from bounded read-only ADT observations. Select the first observed DCP mapper and the observed PIODMA stream. Add the reciprocal rear Type-C graph while retaining disabled controller and route status. Register names follow existing public bindings. Mailbox, firmware handoff, routing and bandwidth integration remain qualification requirements. This resource description is not a display readiness claim. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../boot/dts/apple/t8140-j700-dcpext.dtsi | 69 +++++++++++++++++++ arch/arm64/boot/dts/apple/t8140-j700.dts | 3 + 2 files changed, 72 insertions(+) create mode 100644 arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi new file mode 100644 index 00000000000000..7f3adb838473fb --- /dev/null +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T8140 external display resources measured through the public ADT API. + * Generated-by: LLM + * Assisted-by: LLM + * + * The controller and route remain disabled. Their live firmware + * profile, bandwidth request, routing selector and handoff must be qualified + * before enabling the controller. The existing T8140 refusal remains in force. + */ + +&{/soc} { + dcpext: dcp@416e00000 { + compatible = "apple,t8140-dcpext"; + reg = <0x4 0x16e00000 0x0 0x4000>, + <0x4 0x14000000 0x0 0xafc000>, + <0x4 0x16240000 0x0 0x10000>, + <0x4 0x163ac000 0x0 0x4000>, + <0x4 0x16800000 0x0 0x800000>, + <0x3 0x02800000 0x0 0xbc000>, + <0x4 0x16850000 0x0 0x4000>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + mboxes = <&dcpext_mbox>; + mbox-names = "mbox"; + /* First mapper in the measured ADT iommu-parent list. */ + iommus = <&dcpext0_dart 23>; + power-domains = <&ps_dispext0_cpu>; + apple,connector-type = "USB-C"; + status = "disabled"; + + #address-cells = <2>; + #size-cells = <2>; + ranges; + + dispext0_piodma: piodma { + iommus = <&dispext0_dart 16>; + }; + + typec-routes { + #address-cells = <1>; + #size-cells = <0>; + + route@0 { + reg = <0>; + status = "disabled"; + mode-switch; + + port { + dcpext_typec0: endpoint { + remote-endpoint = <&typec0_dcp>; + }; + }; + }; + }; + }; +}; + +&typec0 { + ports { + port@3 { + reg = <3>; + + typec0_dcp: endpoint { + remote-endpoint = <&dcpext_typec0>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 066175edbc83a7..beacbf31689573 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1880,3 +1880,6 @@ local-mac-address = [00 00 00 00 00 00]; }; }; + +/* External resources stay disabled pending the separate firmware profile. */ +#include "t8140-j700-dcpext.dtsi" From f6da453b0d3658ee7007ee09122b7f74930ea492 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:30:45 +1100 Subject: [PATCH 385/631] drm: apple: recognize the T8140 external binding in the display master Use the generic DCP fallback required by the published T8140 external binding and explicitly classify that specific compatible as an external pipeline. Otherwise the display master never discovers a successfully prepared external node. The device and route remain disabled by default. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi | 2 +- drivers/gpu/drm/apple/apple_drv.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 7f3adb838473fb..cb3e7ea4ebdc82 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -11,7 +11,7 @@ &{/soc} { dcpext: dcp@416e00000 { - compatible = "apple,t8140-dcpext"; + compatible = "apple,t8140-dcpext", "apple,dcp"; reg = <0x4 0x16e00000 0x0 0x4000>, <0x4 0x14000000 0x0 0xafc000>, <0x4 0x16240000 0x0 0x10000>, diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8c588f7f2bcd72..38a3dd901d6047 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -833,6 +833,7 @@ static int apple_drm_init_dcp(struct device *dev) of_node_put(np); continue; dcp_ext = of_device_is_compatible(np, "apple,dcpext") || + of_device_is_compatible(np, "apple,t8140-dcpext") || of_property_present(np, "phys"); dcp[num_dcp] = of_find_device_by_node(np); From 8fd734b6454bf8579b424b82a2e25cb57861bdae Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:31:34 +1100 Subject: [PATCH 386/631] arm64: dts: apple: retain the inherited external display supply Keep the external CPU supply on after the loader validates and publishes its power-domain chain. The supply remains disabled in the static tree, so a refused handoff does not register or power it. This preserves firmware ownership while allowing mailbox consumers to bind. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index cb3e7ea4ebdc82..6a6538dc6dba68 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -67,3 +67,8 @@ }; }; }; + +/* Retain the inherited firmware supply once the loader publishes it. */ +&ps_dispext0_cpu { + apple,always-on; +}; From 779f9b2f53f697e63c7baa6eb1ac8a5922aa73e7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:40:04 +1100 Subject: [PATCH 387/631] drm: apple: validate external display clock and bandwidth resources Separate the number of display windows from the shared firmware clock aperture. Preserve the internal five-window profile and validate the external four-window profile against its existing final clock resource. Require explicit read-only experimental bandwidth offsets for the external profile, with alignment, distinctness and aperture bounds checks before startup. Keep both offsets unset by default and reject missing external clock rates rather than guessing them. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-external-bw.h | 24 ++++++++ drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 56 ++++++++++++++++++- .../drivers/gpu/test_t8140_external_bw.py | 45 +++++++++++++++ 4 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-external-bw.h create mode 100644 tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h new file mode 100644 index 00000000000000..439db008b50846 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_DCP_EXTERNAL_BW_H +#define APPLE_DCP_EXTERNAL_BW_H + +#include + +static inline bool apple_dcp_external_bw_valid(u64 size, int scratch, int request) +{ + if (size < sizeof(u32) || scratch < 0 || request < 0) + return false; + if ((scratch & 3) || (request & 3) || scratch == request) + return false; + return (u64)scratch <= size - sizeof(u32) && + (u64)request <= size - sizeof(u32); +} + +static inline bool apple_dcp_external_clock_window(u64 start, u64 size, + u64 expected_start, u64 expected_size) +{ + return start == expected_start && size == expected_size; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index bcee4609c7add8..bbe74879e1f346 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -184,6 +184,7 @@ struct apple_dcp_hw_data { enum dcp_firmware_version firmware_compat; /* Firmware-visible clock aperture, independent of the AP PMGR window. */ const struct resource *firmware_clock; + u32 firmware_display_windows; u32 firmware_scratch; u32 firmware_request; }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 36e2421c5c058a..a3990eaf06aa3f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -60,6 +60,7 @@ #include "dcp-hdmi.h" #include "dcpext_scanout.h" #include "dcp-internal.h" +#include "dcp-external-bw.h" #include "iomfb.h" #include "ibootep.h" #include "parser.h" @@ -158,6 +159,16 @@ module_param(experimental_h17p_external, bool, 0444); MODULE_PARM_DESC(experimental_h17p_external, "Enable experimental T8140 external display protocol"); +static int experimental_h17p_external_bw_scratch = -1; +module_param(experimental_h17p_external_bw_scratch, int, 0444); +MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, + "Experimental external bandwidth scratch offset; unset by default"); + +static int experimental_h17p_external_bw_request = -1; +module_param(experimental_h17p_external_bw_request, int, 0444); +MODULE_PARM_DESC(experimental_h17p_external_bw_request, + "Experimental external bandwidth request offset; unset by default"); + bool dcp_h17p_external_enabled(void) { return experimental_h17p_external; @@ -3822,8 +3833,25 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) } if (dcp->hw.firmware_clock) { - /* The internal firmware's last aperture follows five display windows. */ - if (count != 5 || resource_size(dcp->hw.firmware_clock) < 4 || + unsigned int windows = dcp->hw.firmware_display_windows; + bool external = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + + if (!windows || windows >= MAX_DISP_REGISTERS) + return -EINVAL; + if (external) { + /* The external clock aperture is already described by disp-4. */ + if (count != windows + 1) + return -EINVAL; + res = dcp->disp_registers[windows]; + if (!apple_dcp_external_clock_window(res->start, + resource_size(res), + dcp->hw.firmware_clock->start, + resource_size(dcp->hw.firmware_clock))) + return -EINVAL; + count = windows; + } + if (count != windows || resource_size(dcp->hw.firmware_clock) < 4 || dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) return -EINVAL; @@ -4281,6 +4309,15 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (!dcp->clk_194 || !clk_get_rate(dcp->clk) || + !clk_get_rate(dcp->clk_194))) { + ret = dev_err_probe(dev, -EINVAL, + "External display clocks are unavailable\n"); + goto err_piodma; + } + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry set_bit(0, dcp->memdesc_map); @@ -4641,6 +4678,18 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + if (!dcp->hw.firmware_clock || + !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), + experimental_h17p_external_bw_scratch, + experimental_h17p_external_bw_request)) + return dev_err_probe(dev, -EINVAL, + "External bandwidth offsets are unset or invalid\n"); + dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; + dcp->hw.firmware_request = experimental_h17p_external_bw_request; + } + /* The domain use must survive driver-core DMA cleanup for a live session. */ ret = iommu_device_use_default_domain(dev); @@ -5336,6 +5385,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { .adopt_live_session = true, .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, + .firmware_display_windows = 5, .firmware_scratch = 0x20000, .firmware_request = 0x68000, }; @@ -5346,6 +5396,8 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, .adopt_live_session = false, .firmware_compat = DCP_FIRMWARE_H17P, + .firmware_clock = &t8140_firmware_clock, + .firmware_display_windows = 4, }; /* diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py new file mode 100644 index 00000000000000..73f0dba8d4d225 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: MIT +import pathlib +import subprocess +import tempfile +import unittest + + +class BandwidthBounds(unittest.TestCase): + def test_shared_header_boundaries(self): + root = pathlib.Path(__file__).resolve().parents[5] + code = r""" +#include +#include +#include "dcp-external-bw.h" +int main(void) { + const u64 n=0xbc000; + assert(!apple_dcp_external_bw_valid(n,-1,-1)); + assert(!apple_dcp_external_bw_valid(n,0,-1)); + assert(!apple_dcp_external_bw_valid(n,-1,0)); + assert(!apple_dcp_external_bw_valid(n,4,4)); + assert(!apple_dcp_external_bw_valid(n,2,4)); + assert(!apple_dcp_external_bw_valid(n,4,6)); + assert(!apple_dcp_external_bw_valid(n,n,4)); + assert(!apple_dcp_external_bw_valid(n,0,n)); + assert(!apple_dcp_external_bw_valid(3,0,4)); + assert(!apple_dcp_external_bw_valid(n,INT_MAX,0)); + assert(apple_dcp_external_bw_valid(n,0,4)); + assert(apple_dcp_external_bw_valid(n,n-4,0)); + assert(apple_dcp_external_bw_valid(n,0,n-4)); + assert(apple_dcp_external_clock_window(0x302800000ULL,n,0x302800000ULL,n)); + assert(!apple_dcp_external_clock_window(0x302800000ULL,n-4,0x302800000ULL,n)); + assert(!apple_dcp_external_clock_window(0x302800004ULL,n,0x302800000ULL,n)); + return 0; +} +""" + with tempfile.TemporaryDirectory() as directory: + tmp=pathlib.Path(directory);(tmp/'linux').mkdir() + (tmp/'linux/types.h').write_text('#include \n#include \ntypedef uint64_t u64; typedef uint32_t u32;\n') + (tmp/'test.c').write_text(code) + subprocess.run(['clang','-std=c11','-Wall','-Wextra','-Werror','-pedantic','-I'+str(tmp),'-I'+str(root/'drivers/gpu/drm/apple'),str(tmp/'test.c'),'-o',str(tmp/'test')],check=True) + subprocess.run([str(tmp/'test')],check=True) + + +if __name__=='__main__': + unittest.main() From ace89fa32feac4cfb3b3d14138bbcee8345d9c0f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:47:51 +1100 Subject: [PATCH 388/631] soc: apple: rtkit: identify failed boot handshake stages Report which bounded startup wait failed so endpoint discovery, host readiness and IOP acknowledgment failures can be distinguished without logging message payloads. Successful handshakes remain unchanged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 6e1fd292468a3c..e5e112ea1f4b71 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1324,6 +1324,7 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) ret = apple_rtkit_wait(rtk, &rtk->epmap_completion); if (ret) return ret; + } if (rtk->boot_result) return rtk->boot_result; @@ -1332,12 +1333,14 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); if (ret) return ret; + } } dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait(rtk, &rtk->iop_pwr_ack_completion); if (ret) return ret; + } return early_ap_power ? 0 : apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); From 447ed5b4d183c4df60d209e2ed8b5362252cf082 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:47:51 +1100 Subject: [PATCH 389/631] drm/apple: announce AP readiness early for experimental external H17P The external H17P trial completes endpoint setup but its normal RTKit wake sequence times out. Use the existing early-AP wake helper for this experimental profile, which announces host readiness before waiting for IOP readiness. Keep the normal wake sequence for other profiles. This ordering is a hardware qualification candidate. The previous trial retained the internal panel but did not expose an external connector. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a3990eaf06aa3f..4b58b0b4f62f1e 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -4400,6 +4400,12 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) dcp_ext_give_up(dcp, ret, "bind"); return 0; } + /* Announce host readiness before awaiting external IOP readiness. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ret = apple_rtkit_wake_early_ap(dcp->rtk); + else + ret = apple_rtkit_wake(dcp->rtk); if (ret) { ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; From abeaccf64487d23ce33dc3f1c70cb2b9a96c8e76 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 16:54:13 +1100 Subject: [PATCH 390/631] drm/apple: acknowledge firmware-owned external H17P buffers The external H17P startup requests a shared buffer at a firmware-provided IOVA. Its setup callback maps that buffer, but the standard RTKit ops do not acknowledge mapped buffers. Select the existing co-processor-owned buffer protocol for this experimental profile so RTKit acknowledges the request and starts the crashlog endpoint before its endpoint-map reply. Keep other display profiles on their existing ops. Restore the normal wake ordering after the earlier AP-readiness trial also timed out, while retaining diagnostics for subsequent qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4b58b0b4f62f1e..9068e1ee335a14 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1997,6 +1997,14 @@ static struct apple_rtkit_ops rtkit_ops = { .shmem_destroy = dcp_rtk_shmem_destroy, }; +static const struct apple_rtkit_ops external_h17p_rtkit_ops = { + .flags = APPLE_RTKIT_COPROC_PLACES_BUFFERS, + .crashed = dcp_rtk_crashed, + .recv_message = dcp_recv_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { if (READ_ONCE(dcp->quiescing)) @@ -4359,7 +4367,13 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat ret = dcp_pin_live_session(dcp); if (ret) goto err_piodma; - dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + /* External H17P supplies shared-buffer addresses during RTKit startup. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, + &external_h17p_rtkit_ops); + else + dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); if (IS_ERR(dcp->rtk)) { ret = dev_err_probe(dev, PTR_ERR(dcp->rtk), "Failed to initialize RTKit\n"); From 86c758038f8e6177e09ff0e9e23cde22f2ce563a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:07:14 +1100 Subject: [PATCH 391/631] drm/apple: recognize the external H17P display service The external H17P firmware announces dispext-service on the existing IBOOT endpoint. Match that measured service name for the experimental external profile, keeping the normal disp0-service table unchanged. Propagate endpoint allocation and startup failures to the caller. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/ibootep.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index beb40ff3113ccb..153971148f1559 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -734,6 +734,14 @@ static const struct apple_epic_service_ops ibootep_ops[] = { {} }; +static const struct apple_epic_service_ops external_ibootep_ops[] = { + { + .name = "dispext-service", + .init = disp_service_init, + }, + {} +}; + int ibootep_init(struct apple_dcp *dcp) { int ret; @@ -756,6 +764,13 @@ int ibootep_init(struct apple_dcp *dcp) dcp->ibootep = NULL; return ret; } + const struct apple_epic_service_ops *ops = ibootep_ops; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + ops = external_ibootep_ops; + dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); + if (IS_ERR(dcp->ibootep)) + return PTR_ERR(dcp->ibootep); return afk_start(dcp->ibootep); } From 2c82b4623ec3f62d6bd3eaf17b0e68969d4342f0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:07:15 +1100 Subject: [PATCH 392/631] drm/apple: report external AFK startup progress Report external-profile control-message types and ring readiness when startup times out. Exclude normal receive notifications and rate-limit control diagnostics; no payload contents or addresses are logged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index e63de19a078566..c24cb5048ff771 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -196,6 +196,11 @@ int afk_start(struct apple_dcp_afkep *ep) afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); + if (ret <= 0 && ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + dev_warn(ep->dcp->dev, + "external AFK startup ep=%#x tx_ready=%u rx_ready=%u bytes=%zu\n", + ep->endpoint, ep->txbfr.ready, ep->rxbfr.ready, ep->bfr_size); if (ret <= 0) return -ETIMEDOUT; else @@ -910,6 +915,11 @@ static void afk_receive_message_worker(struct work_struct *work_) work = container_of(work_, struct afk_receive_message_work, work); type = FIELD_GET(RBEP_TYPE, work->message); + if (work->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && type != RBEP_RECV) + dev_info_ratelimited(work->ep->dcp->dev, + "external AFK control ep=%#x type=%#x\n", + work->ep->endpoint, type); switch (type) { case RBEP_INIT_ACK: break; From d54ef5098fa7df1772612d50a8262f4cf90d8550 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:11:52 +1100 Subject: [PATCH 393/631] drm/apple: start the external H17P allocator before service discovery External H17P reaches AFK transport readiness but does not announce the DPTX port service. Its existing allocator endpoint is currently started only later with IOMFB. Make that known endpoint available before external service discovery, and skip its duplicate start during IOMFB startup. Keep normal H17P ordering unchanged. This external-only ordering change requires hardware validation; no new endpoint or message is introduced. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 7 +++++++ drivers/gpu/drm/apple/iomfb.c | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9068e1ee335a14..c67a5a031f9752 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3095,6 +3095,13 @@ int dcp_start(struct platform_device *pdev) return -ENODEV; } + /* Make the H17P allocator available before external service discovery. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); + if (ret) + return dev_err_probe(dcp->dev, ret, "Failed to start remote allocator\n"); + } + /* start RTKit endpoints */ ret = systemep_init(dcp); if (ret) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 64ed3552cae747..cddec040a382e9 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1929,7 +1929,8 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * has nothing answering it on the AP side, and without this one the * display DART faults. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) return ret; From a0b1584180d86b3ef824e617ad3e72a894011e90 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:11:52 +1100 Subject: [PATCH 394/631] drm/apple: isolate DPTX discovery diagnostics Limit external AFK control diagnostics to DPTX so unrelated endpoint startup cannot consume their rate limit. Distinguish a ready transport with no announced port service from a transport-start timeout. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 ++- drivers/gpu/drm/apple/dptxep.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index c24cb5048ff771..772219605ad0d1 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -916,7 +916,8 @@ static void afk_receive_message_worker(struct work_struct *work_) type = FIELD_GET(RBEP_TYPE, work->message); if (work->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && type != RBEP_RECV) + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + work->ep->endpoint == DPTX_ENDPOINT && type != RBEP_RECV) dev_info_ratelimited(work->ep->dcp->dev, "external AFK control ep=%#x type=%#x\n", work->ep->endpoint, type); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index a057e09dea63f6..6467bb833e31b4 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -986,8 +986,9 @@ int dptxep_init(struct apple_dcp *dcp) return 0; } return -ETIMEDOUT; - } else if (ret < 0) + } else if (ret < 0) { return ret; + } timeout = ret; } From 7db62cb8a3f4ca254221d0b3d20f6455742f4536 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:17:13 +1100 Subject: [PATCH 395/631] drm/apple: initialize the external DPTX controller before port discovery The public m1n1 display startup initializes the DPTX controller endpoint before the port endpoint. Add that existing AFK controller interface to the experimental external H17P path, wait for its service announcement, and then continue port discovery. Send no controller service commands. Use a dedicated endpoint pointer and route its incoming messages through the existing AFK worker. Cover both retained-session quiescing and normal teardown, and never publish an allocation error as an endpoint pointer. Normal display profiles retain their existing startup sequence. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 3 +++ drivers/gpu/drm/apple/dcp.c | 19 ++++++++++++++++ drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dptxep.c | 34 ++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index bbe74879e1f346..ba2cbdd05721a4 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -83,6 +83,7 @@ enum { DISP0_ENDPOINT = 0x23, /* DPAV controller: publishes the port service used by DPTX 0x2a. */ DPAV_CTRL_ENDPOINT = 0x24, + DPTX_CONTROLLER_ENDPOINT = 0x24, DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, @@ -418,6 +419,8 @@ struct apple_dcp { struct audiosrv_data *audiosrv; struct apple_dcp_afkep *dptxep; + struct apple_dcp_afkep *dptx_controller_ep; + struct completion dptx_controller_ready; struct apple_dcp_afkep *dpavctrlep; struct apple_epic_service *dpav_ctrl; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c67a5a031f9752..4ac966a7521fd7 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1727,6 +1727,8 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dcpavservep; case DPTX_ENDPOINT: return dcp->dptxep; + case DPTX_CONTROLLER_ENDPOINT: + return dcp->dptx_controller_ep; default: return NULL; } @@ -1841,6 +1843,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case DPTX_ENDPOINT: ep = dcp->dptxep; break; + case DPTX_CONTROLLER_ENDPOINT: + ep = dcp->dptx_controller_ep; + break; default: ep = NULL; break; @@ -3120,6 +3125,14 @@ int dcp_start(struct platform_device *pdev) dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", ret); + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + ret = dptx_controller_init(dcp); + if (ret) + return dev_err_probe(dcp->dev, ret, + "Failed to start DPTX controller\n"); + } + ret = dptxep_init(dcp); if (ret) { dev_warn(dcp->dev, "Failed to start DPTX endpoint: %d\n", @@ -4476,6 +4489,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_queue_stop(dcp); afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); + afk_quiesce(dcp->dptx_controller_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -4528,6 +4542,11 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp_fabric_shutdown_dptx(dcp); + if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { + afk_shutdown(dcp->dptx_controller_ep); + dcp->dptx_controller_ep = NULL; + } + if (dcp->ibootep) { afk_shutdown(dcp->ibootep); dcp->ibootep = NULL; diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 2edc08e95f7349..47cbfbbeedd6a7 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -101,6 +101,7 @@ void iomfb_recv_msg(struct apple_dcp *dcp, u64 message); int systemep_init(struct apple_dcp *dcp); int dptxep_init(struct apple_dcp *dcp); +int dptx_controller_init(struct apple_dcp *dcp); int ibootep_init(struct apple_dcp *dcp); int dpavservep_init(struct apple_dcp *dcp); int avep_init(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6467bb833e31b4..1f3c00f3db6984 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -929,6 +929,40 @@ static const struct apple_epic_service_ops dptxep_ops[] = { {} }; +static void dptx_controller_service_init(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + complete(&service->ep->dcp->dptx_controller_ready); +} + +static const struct apple_epic_service_ops dptx_controller_ops[] = { + { + .name = "AppleDCPDPTXController", + .init = dptx_controller_service_init, + }, + {} +}; + +int dptx_controller_init(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *ep; + int ret; + + init_completion(&dcp->dptx_controller_ready); + ep = afk_init(dcp, DPTX_CONTROLLER_ENDPOINT, dptx_controller_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dptx_controller_ep = ep; + + ret = afk_start(ep); + if (ret) + return ret; + if (!wait_for_completion_timeout(&dcp->dptx_controller_ready, + msecs_to_jiffies(1000))) + return -ETIMEDOUT; + + return 0; +} /* 14.7 also announces the port under its EPIC names. */ static const struct apple_epic_service_ops dptxep_v14_7_ops[] = { { From afd79d1e10fb8d55763e6416dd8f8faaec09402f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:27:24 +1100 Subject: [PATCH 396/631] drm/apple: discover the version-two external DPTX service The external H17P test reaches transport readiness on the legacy DPTX endpoint without receiving a port announcement. The public version-two firmware implementation uses endpoint 0x2c, which this firmware advertises. Use that endpoint for an isolated service-discovery test. Keep a separate endpoint object with an initialization-only service table, RX routing and lifecycle handling. This does not enable operational port callbacks or transmit service commands; the discovered identity and framing must be validated before extending the command path. Normal profiles retain the legacy DPTX endpoint. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 +- drivers/gpu/drm/apple/dcp-internal.h | 3 ++ drivers/gpu/drm/apple/dcp.c | 12 ++++++++ drivers/gpu/drm/apple/dptxep.c | 41 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 772219605ad0d1..448a892ced1ffe 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -917,7 +917,8 @@ static void afk_receive_message_worker(struct work_struct *work_) type = FIELD_GET(RBEP_TYPE, work->message); if (work->ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - work->ep->endpoint == DPTX_ENDPOINT && type != RBEP_RECV) + (work->ep->endpoint == DPTX_ENDPOINT || + work->ep->endpoint == DPTX_ENDPOINT_V2) && type != RBEP_RECV) dev_info_ratelimited(work->ep->dcp->dev, "external AFK control ep=%#x type=%#x\n", work->ep->endpoint, type); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index ba2cbdd05721a4..444e636cfb26d8 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -87,6 +87,7 @@ enum { DPAVSERV_ENDPOINT = 0x28, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, + DPTX_ENDPOINT_V2 = 0x2c, HDCP_ENDPOINT = 0x2b, REMOTE_ALLOC_ENDPOINT = 0x2d, IOMFB_ENDPOINT = 0x37, @@ -421,6 +422,8 @@ struct apple_dcp { struct apple_dcp_afkep *dptxep; struct apple_dcp_afkep *dptx_controller_ep; struct completion dptx_controller_ready; + struct apple_dcp_afkep *dptx_v2_probe_ep; + struct completion dptx_v2_probe_ready; struct apple_dcp_afkep *dpavctrlep; struct apple_epic_service *dpav_ctrl; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4ac966a7521fd7..f354d16ffdb407 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1727,6 +1727,9 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dcpavservep; case DPTX_ENDPOINT: return dcp->dptxep; + case DPTX_ENDPOINT_V2: + return dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptx_v2_probe_ep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -1843,6 +1846,9 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) case DPTX_ENDPOINT: ep = dcp->dptxep; break; + case DPTX_ENDPOINT_V2: + ep = dcp->dptx_v2_probe_ep; + break; case DPTX_CONTROLLER_ENDPOINT: ep = dcp->dptx_controller_ep; break; @@ -4490,6 +4496,7 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); afk_quiesce(dcp->dptx_controller_ep); + afk_quiesce(dcp->dptx_v2_probe_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -4542,6 +4549,11 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp_fabric_shutdown_dptx(dcp); + if (!IS_ERR_OR_NULL(dcp->dptx_v2_probe_ep)) { + afk_shutdown(dcp->dptx_v2_probe_ep); + dcp->dptx_v2_probe_ep = NULL; + } + if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); dcp->dptx_controller_ep = NULL; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 1f3c00f3db6984..649b4488fa1dc3 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -983,12 +983,53 @@ static const struct apple_epic_service_ops dptxep_v14_7_ops[] = { {} }; +static void dptx_v2_probe_service(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + dev_info(service->ep->dcp->dev, + "DPTX v2 discovery name=%s class=%s unit=%lld channel=%u\n", + name ?: "(none)", class ?: "(none)", unit, service->channel); + complete(&service->ep->dcp->dptx_v2_probe_ready); +} + +static const struct apple_epic_service_ops dptx_v2_probe_ops[] = { + { + .name = "AppleDCPDPTXRemotePort", + .init = dptx_v2_probe_service, + }, + {} +}; + +static int dptx_v2_discover(struct apple_dcp *dcp) +{ + struct apple_dcp_afkep *ep; + int ret; + + init_completion(&dcp->dptx_v2_probe_ready); + ep = afk_init(dcp, DPTX_ENDPOINT_V2, dptx_v2_probe_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + dcp->dptx_v2_probe_ep = ep; + ret = afk_start(ep); + if (ret) + return ret; + if (!wait_for_completion_timeout(&dcp->dptx_v2_probe_ready, + msecs_to_jiffies(1000))) + return -ETIMEDOUT; + + /* Discovery does not enable port callbacks or transmit service commands. */ + return -EOPNOTSUPP; +} + int dptxep_init(struct apple_dcp *dcp) { int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return dptx_v2_discover(dcp); + init_completion(&dcp->dptxport[0].enable_completion); init_completion(&dcp->dptxport[1].enable_completion); init_completion(&dcp->dptxport[0].linkcfg_completion); From 3d93628c910085c703e217cc43c545abe4f10f07 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:39:11 +1100 Subject: [PATCH 397/631] drm/apple: operate the external DPTX service over AFKV2 The external T8140 controller announces its port on endpoint 0x2c using version-two frames. Select that transport per endpoint, open announced interfaces and map the single physical DCP unit to logical port zero. Encode bounded command descriptors and validate replies against pending owned buffers before completing them. Preserve callback tokens and limit peer-buffer access to active allocations of the same service. Keep the legacy transport for other display profiles. Add host selftests for frame encoding, malformed records, response correlation, command identifiers and the production owned-copy helper. These pass 905 codec assertions and owned-copy cases with sanitizers; hardware qualification of the operational transport remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/afk.c | 356 +++++++++++++++++- drivers/gpu/drm/apple/afk.h | 6 + drivers/gpu/drm/apple/afkv2-codec.h | 270 +++++++++++++ drivers/gpu/drm/apple/dcp.c | 10 +- drivers/gpu/drm/apple/dptxep.c | 70 ++-- .../drivers/gpu/apple-afkv2/Makefile | 9 + .../selftests/drivers/gpu/apple-afkv2/README | 24 ++ .../selftests/drivers/gpu/apple-afkv2/codec.c | 183 +++++++++ .../drivers/gpu/apple-afkv2/extract-owned.py | 28 ++ .../apple-afkv2/host-include/linux/errno.h | 5 + .../apple-afkv2/host-include/linux/string.h | 2 + .../apple-afkv2/host-include/linux/types.h | 8 + .../drivers/gpu/apple-afkv2/owned-prefix.c | 50 +++ .../drivers/gpu/apple-afkv2/owned-suffix.c | 56 +++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 40 ++ 16 files changed, 1046 insertions(+), 72 deletions(-) create mode 100644 drivers/gpu/drm/apple/afkv2-codec.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/README create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c create mode 100755 tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh diff --git a/MAINTAINERS b/MAINTAINERS index f7f7badf5f8f0e..9ecfd2681ff6e1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1960,6 +1960,7 @@ F: drivers/gpu/drm/apple/ F: drivers/gpu/drm/apple/dcp-fabric* F: include/linux/soc/apple/dp-tunnel.h F: tools/testing/apple/ +F: tools/testing/selftests/drivers/gpu/apple-afkv2/ APPLE PCIE CONTROLLER DRIVER M: Marc Zyngier diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 448a892ced1ffe..77dc69935348af 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -18,6 +18,7 @@ #include "afk.h" #include "afk_reply.h" +#include "afkv2-codec.h" #include "trace.h" struct afk_receive_message_work { @@ -355,6 +356,10 @@ static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep * return NULL; } +static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len, u64 token); + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { @@ -440,6 +445,22 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } ep->num_channels = max(ep->num_channels, ch_idx + 1); + if (ep->wire_v2) { + int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, + EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); + + if (ret) { + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); + dev_err(ep->dcp->dev, "AFKV2 interface %u OPEN failed: %d\n", + channel, ret); + goto free; + } + } ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); @@ -489,7 +510,7 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, struct epic_cmd *cmd = payload; struct apple_epic_service *service; unsigned long flags; - u8 idx = tag & 0xff; + u8 idx = tag & (ep->wire_v2 ? 0xf : 0xff); void *rxbuf, *txbuf; dma_addr_t rxbuf_dma, txbuf_dma; size_t rxlen, txlen; @@ -671,6 +692,233 @@ static bool afk_validate_h17p_header(const u8 *data, size_t size) get_unaligned_le32(data + 4) == size - 8; } +static void afk_v2_status(struct apple_dcp_afkep *ep, + const struct afkv2_frame *frame, + const struct afkv2_command *command, u32 status) +{ + u8 record[AFKV2_HEADER_SIZE + AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; + size_t size; + int ret; + + ret = afkv2_encode_error_reply(record, sizeof(record), 0, frame, command, + status, &size); + if (!ret) + ret = afk_send_epic_token(ep, frame->interface, command->id, + EPIC_TYPE_REPLY, EPIC_CAT_REPLY, frame->type, + record + AFKV2_HEADER_SIZE, + size - AFKV2_HEADER_SIZE, frame->token); + if (ret) + dev_warn(ep->dcp->dev, "AFKV2 error reply failed: %d\n", ret); +} + +static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer, bool copy) +{ + unsigned long flags; + unsigned int i; + int ret = -EACCES; + + if (!service || !service->ep->wire_v2 || size > SZ_1M || (copy && size && !buffer)) + return -EINVAL; + if (!size) + return 0; + spin_lock_irqsave(&service->lock, flags); + for (i = 0; i < MAX_PENDING_CMDS; i++) { + struct epic_cmd_info *cmd = &service->cmds[i]; + void *owned = NULL; + u64 offset; + + if (!test_bit(i, service->cmd_map) || cmd->done) + continue; + if (cmd->rxbuf && afkv2_owned_span(cmd->rxbuf_dma, cmd->rxlen, dva, size)) { + offset = dva - cmd->rxbuf_dma; + owned = cmd->rxbuf + offset; + } else if (cmd->txbuf && afkv2_owned_span(cmd->txbuf_dma, cmd->txlen, dva, size)) { + offset = dva - cmd->txbuf_dma; + owned = cmd->txbuf + offset; + } + if (!owned) + continue; + if (!copy) { + ret = 0; + break; + } + if (to_peer) { + memcpy(owned, buffer, size); + dma_wmb(); + } else { + dma_rmb(); + memcpy(buffer, owned, size); + } + ret = 0; + break; + } + spin_unlock_irqrestore(&service->lock, flags); + if (ret) + dev_warn_ratelimited(service->ep->dcp->dev, + "AFKV2 unowned buffer: ep=%u interface=%u size=%zu write=%u\n", + service->ep->endpoint, service->channel, size, to_peer); + return ret; +} + +int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer) +{ + return afk_v2_access_owned(service, dva, buffer, size, to_peer, true); +} + +static void afk_v2_callback(struct apple_dcp_afkep *ep, + const struct afkv2_frame *frame, + const struct afkv2_command *command) +{ + struct apple_epic_service *service; + const struct epic_service_call *call; + const u8 *request = command->inline_data; + u8 *owned_request = NULL, *reply = NULL; + size_t request_size = command->oob ? command->tx_size : command->inline_size; + size_t body_size, reply_size; + u32 status = 0xe00002c7; + int ret; + + if (frame->type != EPIC_SUBTYPE_STD_SERVICE || + request_size < sizeof(*call) + sizeof(u32) || + request_size > SZ_1M - AFKV2_COMMAND_SIZE || + request_size > command->value || + (command->oob && request_size > command->rx_size)) + goto reject; + service = afk_service_get(afk_epic_find_service(ep, frame->interface)); + if (!service || READ_ONCE(service->torndown) || !service->ops->call) { + if (service) + afk_service_put(service); + goto reject; + } + if (command->oob) { + /* Reject an unowned response destination before any callback side effect. */ + ret = afk_v2_access_owned(service, command->rx_address, NULL, + request_size, true, false); + if (ret) + goto release; + owned_request = kmalloc(request_size, GFP_KERNEL); + if (!owned_request) { + status = 0xe00002bd; + goto release; + } + ret = afk_v2_copy_owned(service, command->tx_address, + owned_request, request_size, false); + if (ret) + goto release; + request = owned_request; + } + call = (const void *)request; + body_size = request_size - sizeof(*call); + if (le16_to_cpu(call->group) || + le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + le32_to_cpu(call->data_len) != body_size) + goto release; + reply = kzalloc(AFKV2_COMMAND_SIZE + request_size, GFP_KERNEL); + if (!reply) { + status = 0xe00002bd; + goto release; + } + reply[1] = command->id; + memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); + ret = service->ops->call(service, le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + if (ret) + goto release; + reply_size = AFKV2_COMMAND_SIZE + request_size; + if (command->oob) { + ret = afk_v2_copy_owned(service, command->rx_address, + reply + AFKV2_COMMAND_SIZE, request_size, true); + if (ret) + goto release; + ret = afkv2_encode_oob_command(reply, reply_size, command->id, + command->rx_address, request_size, + command->tx_address, command->tx_size); + if (ret) + goto release; + afkv2_put32(reply + 4, 0); + reply_size = AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE; + } + ret = afk_send_epic_token(ep, frame->interface, command->id, EPIC_TYPE_REPLY, + EPIC_CAT_REPLY, frame->type, reply, reply_size, frame->token); + kfree(reply); + kfree(owned_request); + afk_service_put(service); + if (ret) + dev_warn(ep->dcp->dev, "AFKV2 callback reply failed: %d\n", ret); + return; +release: + kfree(reply); + kfree(owned_request); + afk_service_put(service); +reject: + afk_v2_status(ep, frame, command, status); +} + +static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, + const void *data, size_t size) +{ + struct afkv2_frame frame; + struct afkv2_command response; + struct apple_epic_service *service; + struct epic_cmd legacy = {}; + unsigned long flags; + unsigned int slot; + int ret; + + if (channel || type || afkv2_decode_frame(data, size, false, &frame) || + !frame.interface) + return; + if (frame.category == AFKV2_REPORT) { + if (frame.type == EPIC_SUBTYPE_ANNOUNCE_H17P) + afk_recv_handle_init(ep, frame.interface, (void *)frame.payload, + frame.payload_size); + else if (frame.type == AFKV2_CLOSE && frame.interface_flags == 1) + afk_recv_handle_teardown(ep, frame.interface); + return; + } + if (afkv2_decode_command(frame.payload, frame.payload_size, &response)) + return; + if (frame.category == AFKV2_COMMAND) { + afk_v2_callback(ep, &frame, &response); + return; + } + service = afk_epic_find_service(ep, frame.interface); + if (!service) + return; + spin_lock_irqsave(&service->lock, flags); + if (bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)) { + spin_unlock_irqrestore(&service->lock, flags); + return; + } + service->users++; + spin_unlock_irqrestore(&service->lock, flags); + slot = response.id & 0xf; + spin_lock_irqsave(&service->lock, flags); + ret = -EPROTO; + if (slot < MAX_PENDING_CMDS && test_bit(slot, service->cmd_map) && + !service->cmds[slot].done && response.id == service->cmds[slot].tag) + ret = afkv2_check_response(&frame, &response, service->channel, + service->cmds[slot].type, service->cmds[slot].tag, + service->cmds[slot].rxbuf_dma, + service->cmds[slot].rxlen, + service->cmds[slot].txbuf_dma, + service->cmds[slot].txlen); + if (!ret) + service->cmds[slot].reply_length = response.value ? 0 : response.rx_size; + spin_unlock_irqrestore(&service->lock, flags); + if (!ret) { + legacy.retcode = cpu_to_le32(response.value); + afk_recv_handle_reply(ep, frame.interface, response.id, + &legacy, sizeof(legacy)); + } else { + dev_warn(ep->dcp->dev, "AFKV2 reply rejected: %d\n", ret); + } + afk_service_put(service); +} + static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *data, size_t data_size) { @@ -686,6 +934,11 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); size_t payload_size; + if (ep->wire_v2) { + afk_v2_receive(ep, channel, type, data, data_size); + return; + } + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P) { const struct epic_sub_hdr_h17p *c = (const struct epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); @@ -991,9 +1244,9 @@ int afk_receive_message(struct apple_dcp_afkep *ep, u64 message) return 0; } -int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, - enum epic_type etype, enum epic_category ecat, u8 stype, - const void *payload, size_t payload_len) +static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len, u64 token) { u32 rptr, wptr; struct afk_qe *hdr, *hdr2; @@ -1001,7 +1254,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, struct epic_sub_hdr *eshdr; unsigned long flags; size_t total_epic_size, total_size; - bool compact = ep->dcp->hw.iomfb_method_profile == + u8 v2_command[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; + u8 v2_category = 0; + u16 v2_flags = 0; + bool compact = !ep->wire_v2 && ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; @@ -1015,7 +1271,7 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, * callers are optional (verbose firmware logging, EDID copy, audio, * DPTX, debugfs) or replies to calls only DPTX services make. */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact) { + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact && !ep->wire_v2) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); @@ -1033,14 +1289,44 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) return -EOPNOTSUPP; + if (ep->wire_v2) { + if (!channel || channel > U16_MAX || tag > U8_MAX || payload_len > SZ_1M) + return -EINVAL; + if (ecat == EPIC_CAT_REPORT) { + if ((stype != AFKV2_OPEN && stype != AFKV2_CLOSE) || payload_len) + return -EOPNOTSUPP; + v2_flags = 1; + } else if (ecat == EPIC_CAT_COMMAND) { + const struct epic_cmd *cmd = payload; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || + payload_len != sizeof(*cmd)) + return -EINVAL; + ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, + le64_to_cpu(cmd->rxbuf), + le32_to_cpu(cmd->rxlen), + le64_to_cpu(cmd->txbuf), + le32_to_cpu(cmd->txlen)); + if (ret) + return ret; + payload = v2_command; + payload_len = sizeof(v2_command); + v2_category = AFKV2_COMMAND; + } else if (ecat == EPIC_CAT_REPLY) { + v2_category = AFKV2_RESPONSE; + } else { + return -EOPNOTSUPP; + } + } + spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { ret = -ESHUTDOWN; goto out; } - if (compact && (!ep->txbfr.ready || !ep->txbfr.rptr || - !ep->txbfr.wptr || !ep->txbfr.buf)) { + if ((compact || ep->wire_v2) && + (!ep->txbfr.ready || !ep->txbfr.rptr || !ep->txbfr.wptr || !ep->txbfr.buf)) { ret = -ESHUTDOWN; goto out; } @@ -1050,11 +1336,11 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + - (compact ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + ((compact || ep->wire_v2) ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + payload_len; total_size = sizeof(*hdr) + total_epic_size; - if (compact) { + if (compact || ep->wire_v2) { u32 available, needed; if (!ep->txbfr.ready || !ep->txbfr.stride || @@ -1157,12 +1443,21 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(compact ? 0 : channel); - hdr->type = cpu_to_le32(etype); + hdr->channel = cpu_to_le32((compact || ep->wire_v2) ? 0 : channel); + hdr->type = cpu_to_le32(ep->wire_v2 ? 0 : etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); - if (compact) { + if (ep->wire_v2) { + size_t written; + + ret = afkv2_encode_frame(ep->txbfr.buf + wptr, total_epic_size, + ep->qe_seq++, channel, token, stype, v2_category, + v2_flags, payload, payload_len, &written); + if (ret) + goto out; + wptr += written; + } else if (compact) { u8 *wire = ep->txbfr.buf + wptr; /* Experimental symmetry with the published H17P receive layout. */ @@ -1197,8 +1492,10 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr += sizeof(*eshdr); } - memcpy(ep->txbfr.buf + wptr, payload, payload_len); - wptr += payload_len; + if (!ep->wire_v2) { + memcpy(ep->txbfr.buf + wptr, payload, payload_len); + wptr += payload_len; + } wptr = afk_ring_advance(wptr, 0, ep->txbfr.bufsz, ep->txbfr.stride); trace_afk_send_rwptr_post(ep, rptr, wptr); @@ -1216,6 +1513,17 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, return ret; } +int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, + enum epic_type etype, enum epic_category ecat, u8 stype, + const void *payload, size_t payload_len) +{ + return afk_send_epic_token(ep, channel, tag, etype, ecat, stype, + payload, payload_len, 0); +} + +int afk_send_command(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, void *output, + size_t output_len, u32 *retcode) /* * Send one EPIC command and wait up to @timeout_ms for its reply. With * @reply_len set, only the bytes the firmware reported are copied to @output @@ -1265,6 +1573,12 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_put_service; } + if (ep->wire_v2 && (payload_len > SZ_1M || output_len > SZ_1M || + (payload_len && !payload))) { + ret = -EINVAL; + goto err_put_service; + } + rxbuf = dma_alloc_coherent(ep->dcp->dev, output_len, &rxbuf_dma, GFP_KERNEL); if (!rxbuf) { @@ -1298,11 +1612,13 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_unlock; } - tag = (service->cmd_tag & 0xff) << 8; - tag |= idx & 0xff; + tag = ep->wire_v2 ? afkv2_command_id(service->cmd_tag, idx) : + ((service->cmd_tag & 0xff) << 8) | idx; service->cmd_tag++; service->cmds[idx].tag = tag; + service->cmds[idx].type = type; + service->cmds[idx].reply_length = 0; service->cmds[idx].rxbuf = rxbuf; service->cmds[idx].txbuf = txbuf; service->cmds[idx].rxbuf_dma = rxbuf_dma; @@ -1380,6 +1696,12 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, memcpy(output, rxbuf, *reply_len); } } else if (output && output_len) { + if (ep->wire_v2 && !service->cmds[idx].retcode && + service->cmds[idx].reply_length < output_len) + ret = -EMSGSIZE; + else if (output && output_len && (!ep->wire_v2 || !service->cmds[idx].retcode)) { + if (ep->wire_v2) + dma_rmb(); memcpy(output, rxbuf, output_len); } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 0e36317192cc55..12cdcde420ce54 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -25,6 +25,8 @@ struct apple_dcp_afkep; struct epic_cmd_info { u16 tag; + u8 type; + size_t reply_length; void *rxbuf; void *txbuf; @@ -298,6 +300,8 @@ struct apple_dcp_afkep { u8 experimental_rx_logs; u32 endpoint; + /* Set before afk_start; protocol selection is per endpoint. */ + bool wire_v2; struct workqueue_struct *wq; struct completion started; @@ -328,6 +332,8 @@ struct apple_dcp_afkep { struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops); +int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, + void *buffer, size_t size, bool to_peer); int afk_start(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); void afk_quiesce(struct apple_dcp_afkep *ep); diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h new file mode 100644 index 00000000000000..cdd5d5ad20b222 --- /dev/null +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +/* Wire layout: public Linux PR254 and m1n1 AFKV2 implementations. */ +#ifndef AURORA_AFKV2_CODEC_H +#define AURORA_AFKV2_CODEC_H + +#include +#include +#include + +#define AFKV2_HEADER_SIZE 24U +#define AFKV2_COMMAND_SIZE 8U +#define AFKV2_OOB_SIZE 24U +#define AFKV2_MAX_PAYLOAD 1048576U +#define AFKV2_REPORT 0U +#define AFKV2_COMMAND 1U +#define AFKV2_RESPONSE 2U +#define AFKV2_OPEN 0x12U +#define AFKV2_CLOSE 0x13U + +struct afkv2_frame { + u8 sequence; + u8 transport_flags; + u16 interface; + u64 token; + u8 type; + u8 category; + u16 interface_flags; + u32 reserved; + const u8 *payload; + size_t payload_size; + size_t padding_size; +}; + +struct afkv2_command { + u8 id; + u32 value; + bool oob; + u64 rx_address; + u64 tx_address; + u32 rx_size; + u32 tx_size; + const u8 *inline_data; + size_t inline_size; +}; + +static inline u16 afkv2_get16(const u8 *p) +{ + return (u16)p[0] | (u16)p[1] << 8; +} + +static inline u32 afkv2_get32(const u8 *p) +{ + return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16 | (u32)p[3] << 24; +} + +static inline u64 afkv2_get64(const u8 *p) +{ + return (u64)afkv2_get32(p) | (u64)afkv2_get32(p + 4) << 32; +} + +static inline void afkv2_put16(u8 *p, u16 value) +{ + p[0] = value; + p[1] = value >> 8; +} + +static inline void afkv2_put32(u8 *p, u32 value) +{ + p[0] = value; + p[1] = value >> 8; + p[2] = value >> 16; + p[3] = value >> 24; +} + +static inline void afkv2_put64(u8 *p, u64 value) +{ + afkv2_put32(p, value); + afkv2_put32(p + 4, value >> 32); +} + +/* Decode metadata only. Never follow an address supplied by the peer. */ +static inline int afkv2_decode_frame(const void *record, size_t size, + bool allow_padding, struct afkv2_frame *out) +{ + const u8 *p = record; + u32 length; + struct afkv2_frame result = {0}; + + if (!record || !out || size < AFKV2_HEADER_SIZE) + return -EINVAL; + if (size > AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD) + return -E2BIG; + length = afkv2_get32(p + 4); + if (length < 16 || length - 16 > AFKV2_MAX_PAYLOAD || length > size - 8) + return -EINVAL; + if (!allow_padding && length != size - 8) + return -EINVAL; + if (p[17] > AFKV2_RESPONSE) + return -EPROTO; + result.sequence = p[0]; + result.transport_flags = p[1]; + result.interface = afkv2_get16(p + 2); + result.token = afkv2_get64(p + 8); + result.type = p[16]; + result.category = p[17]; + result.interface_flags = afkv2_get16(p + 18); + result.reserved = afkv2_get32(p + 20); + result.payload = p + AFKV2_HEADER_SIZE; + result.payload_size = length - 16; + result.padding_size = size - 8 - length; + *out = result; + return 0; +} + +static inline int afkv2_encode_frame(void *record, size_t capacity, u8 sequence, + u16 interface, u64 token, u8 type, u8 category, + u16 flags, const void *payload, size_t payload_size, + size_t *written) +{ + u8 *p = record; + + if (!record || !written || !interface || category > AFKV2_RESPONSE || + (payload_size && !payload)) + return -EINVAL; + if (payload_size > AFKV2_MAX_PAYLOAD) + return -E2BIG; + if (capacity < AFKV2_HEADER_SIZE || payload_size > capacity - AFKV2_HEADER_SIZE) + return -ENOSPC; + /* Move first so an in-buffer payload is safe even when it overlaps. */ + if (payload_size) + memmove(p + AFKV2_HEADER_SIZE, payload, payload_size); + memset(p, 0, AFKV2_HEADER_SIZE); + p[0] = sequence; + afkv2_put16(p + 2, interface); + afkv2_put32(p + 4, payload_size + 16); + afkv2_put64(p + 8, token); + p[16] = type; + p[17] = category; + afkv2_put16(p + 18, flags); + *written = AFKV2_HEADER_SIZE + payload_size; + return 0; +} + +static inline int afkv2_encode_open(void *record, size_t capacity, u8 sequence, + u16 interface, size_t *written) +{ + return afkv2_encode_frame(record, capacity, sequence, interface, 0, + AFKV2_OPEN, AFKV2_REPORT, 1, NULL, 0, written); +} + +static inline bool afkv2_valid_span(u64 address, u32 size) +{ + if (!size) + return !address; + return address && size <= AFKV2_MAX_PAYLOAD && + address <= ~(u64)0 - (size - 1); +} + +static inline int afkv2_encode_oob_command(void *payload, size_t capacity, + u8 id, u64 rx_address, u32 rx_size, + u64 tx_address, u32 tx_size) +{ + u8 *p = payload; + + if (!payload || !afkv2_valid_span(rx_address, rx_size) || + !afkv2_valid_span(tx_address, tx_size)) + return -EINVAL; + if (capacity < AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE) + return -ENOSPC; + memset(p, 0, AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE); + p[0] = 1; + p[1] = id; + afkv2_put32(p + 4, rx_size); + afkv2_put64(p + 8, rx_address); + afkv2_put64(p + 16, tx_address); + afkv2_put32(p + 24, rx_size); + afkv2_put32(p + 28, tx_size); + return 0; +} + +static inline int afkv2_decode_command(const void *payload, size_t size, + struct afkv2_command *out) +{ + const u8 *p = payload; + struct afkv2_command result = {0}; + size_t header = AFKV2_COMMAND_SIZE; + + if (!payload || !out || size < header || p[0] & ~1U || afkv2_get16(p + 2)) + return -EINVAL; + result.id = p[1]; + result.value = afkv2_get32(p + 4); + result.oob = p[0] & 1; + if (result.oob) { + header += AFKV2_OOB_SIZE; + if (size < header) + return -EINVAL; + result.rx_address = afkv2_get64(p + 8); + result.tx_address = afkv2_get64(p + 16); + result.rx_size = afkv2_get32(p + 24); + result.tx_size = afkv2_get32(p + 28); + /* A failed response may retain addresses with zero lengths. */ + if ((result.rx_size && !afkv2_valid_span(result.rx_address, result.rx_size)) || + (result.tx_size && !afkv2_valid_span(result.tx_address, result.tx_size))) + return -EINVAL; + } + result.inline_data = p + header; + result.inline_size = size - header; + if (result.inline_size > AFKV2_MAX_PAYLOAD) + return -E2BIG; + *out = result; + return 0; +} + +/* Correlate only; no buffer is read, written or freed by this helper. */ +static inline int afkv2_check_response(const struct afkv2_frame *frame, + const struct afkv2_command *response, + u16 interface, u8 type, u8 id, + u64 rx_address, u32 rx_capacity, + u64 tx_address, u32 tx_capacity) +{ + if (!frame || !response || frame->category != AFKV2_RESPONSE || + frame->interface != interface || frame->type != type || response->id != id) + return -EPROTO; + if (response->value) + return 0; + if (!response->oob || response->inline_size || + response->rx_address != rx_address || response->tx_address != tx_address || + response->rx_size > rx_capacity || response->tx_size > tx_capacity) + return -EPROTO; + return 0; +} + +/* Reject a peer command without touching its advertised DMA buffers. */ +static inline int afkv2_encode_error_reply(void *record, size_t capacity, u8 sequence, + const struct afkv2_frame *request, + const struct afkv2_command *command, + u32 status, size_t *written) +{ + u8 payload[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE] = {0}; + size_t size = AFKV2_COMMAND_SIZE; + + if (!request || !command || request->category != AFKV2_COMMAND || !status) + return -EINVAL; + payload[0] = command->oob ? 1 : 0; + payload[1] = command->id; + afkv2_put32(payload + 4, status); + if (command->oob) { + afkv2_put64(payload + 8, command->rx_address); + afkv2_put64(payload + 16, command->tx_address); + size += AFKV2_OOB_SIZE; + } + return afkv2_encode_frame(record, capacity, sequence, request->interface, + request->token, request->type, AFKV2_RESPONSE, 0, + payload, size, written); +} + +static inline u8 afkv2_command_id(u8 generation, u8 slot) +{ + return ((generation & 15) << 4) | (slot & 15); +} + +static inline bool afkv2_owned_span(u64 base, size_t length, u64 address, size_t size) +{ + if (!length || base > ~(u64)0 - (length - 1)) + return false; + return address >= base && size <= length && address - base <= length - size; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index f354d16ffdb407..0a62a44302ce06 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1729,7 +1729,7 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dptxep; case DPTX_ENDPOINT_V2: return dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptx_v2_probe_ep : NULL; + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptxep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -1847,7 +1847,7 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) ep = dcp->dptxep; break; case DPTX_ENDPOINT_V2: - ep = dcp->dptx_v2_probe_ep; + ep = dcp->dptxep; break; case DPTX_CONTROLLER_ENDPOINT: ep = dcp->dptx_controller_ep; @@ -4496,7 +4496,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) afk_quiesce(dcp->avep); afk_quiesce(dcp->dptxep); afk_quiesce(dcp->dptx_controller_ep); - afk_quiesce(dcp->dptx_v2_probe_ep); afk_quiesce(dcp->ibootep); afk_quiesce(dcp->systemep); afk_quiesce(dcp->dcpavservep); @@ -4549,11 +4548,6 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp_fabric_shutdown_dptx(dcp); - if (!IS_ERR_OR_NULL(dcp->dptx_v2_probe_ep)) { - afk_shutdown(dcp->dptx_v2_probe_ep); - dcp->dptx_v2_probe_ep = NULL; - } - if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); dcp->dptx_controller_ep = NULL; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 649b4488fa1dc3..48109403812c19 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -882,19 +882,33 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, static void dptxport_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { + struct apple_dcp *dcp = service->ep->dcp; + s64 port = unit; bool v14_7 = dcp_fw_is_14_7(service->ep->dcp); if (!name) return; if (strcmp(name, "dcpdptx-port-epic") && (!v14_7 || strcmp(name, "dcp-lpdptx-port-epic"))) + if (!name || !class) + return; + if (strcmp(name, "dcpdptx-port-epic")) return; /* 14.7 may announce the port without its class. */ if (v14_7 ? class && class[0] && strcmp(class, "AppleDCPDPTXRemotePort") : !class || strcmp(class, "AppleDCPDPTXRemotePort")) return; - trace_dptxport_init(service->ep->dcp, unit); + trace_dptxport_init(dcp, unit); + if (service->ep->wire_v2) { + /* The single external service reports its physical DCP unit. */ + if (dcp->hw.num_dptx_ports != 1 || unit != dcp->index) { + dev_err(dcp->dev, "Unexpected external DPTX unit %lld\n", unit); + return; + } + port = 0; + dev_info(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); + } switch (unit) { case 0: @@ -983,63 +997,25 @@ static const struct apple_epic_service_ops dptxep_v14_7_ops[] = { {} }; -static void dptx_v2_probe_service(struct apple_epic_service *service, - const char *name, const char *class, s64 unit) -{ - dev_info(service->ep->dcp->dev, - "DPTX v2 discovery name=%s class=%s unit=%lld channel=%u\n", - name ?: "(none)", class ?: "(none)", unit, service->channel); - complete(&service->ep->dcp->dptx_v2_probe_ready); -} - -static const struct apple_epic_service_ops dptx_v2_probe_ops[] = { - { - .name = "AppleDCPDPTXRemotePort", - .init = dptx_v2_probe_service, - }, - {} -}; - -static int dptx_v2_discover(struct apple_dcp *dcp) -{ - struct apple_dcp_afkep *ep; - int ret; - - init_completion(&dcp->dptx_v2_probe_ready); - ep = afk_init(dcp, DPTX_ENDPOINT_V2, dptx_v2_probe_ops); - if (IS_ERR(ep)) - return PTR_ERR(ep); - dcp->dptx_v2_probe_ep = ep; - ret = afk_start(ep); - if (ret) - return ret; - if (!wait_for_completion_timeout(&dcp->dptx_v2_probe_ready, - msecs_to_jiffies(1000))) - return -ETIMEDOUT; - - /* Discovery does not enable port callbacks or transmit service commands. */ - return -EOPNOTSUPP; -} - int dptxep_init(struct apple_dcp *dcp) { + struct apple_dcp_afkep *ep; + bool wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) - return dptx_v2_discover(dcp); - init_completion(&dcp->dptxport[0].enable_completion); init_completion(&dcp->dptxport[1].enable_completion); init_completion(&dcp->dptxport[0].linkcfg_completion); init_completion(&dcp->dptxport[1].linkcfg_completion); - dcp->dptxep = afk_init(dcp, DPTX_ENDPOINT, - dcp_fw_is_14_7(dcp) ? - dptxep_v14_7_ops : dptxep_ops); - if (IS_ERR(dcp->dptxep)) - return PTR_ERR(dcp->dptxep); + ep = afk_init(dcp, wire_v2 ? DPTX_ENDPOINT_V2 : DPTX_ENDPOINT, dptxep_ops); + if (IS_ERR(ep)) + return PTR_ERR(ep); + ep->wire_v2 = wire_v2; + dcp->dptxep = ep; ret = afk_start(dcp->dptxep); if (ret) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile new file mode 100644 index 00000000000000..65780f3d5003f9 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +HOSTCC ?= cc +export HOSTCC +export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) + +TEST_PROGS := run-afkv2-tests.sh +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py host-include + +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README new file mode 100644 index 00000000000000..42eb152d716a22 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -0,0 +1,24 @@ +AFKV2 host selftests +=================== + +Run ./run-afkv2-tests.sh from this directory, or use make run_tests in a +complete kernel source checkout. HOSTCC, CFLAGS and PYTHON select host tools +(standalone CC is also accepted). +AFKV2_SANITIZERS=1 additionally enables AddressSanitizer and UBSan. +AFKV2_SOURCE_ROOT can name the source checkout when running a copied test +package; the Makefile exports it automatically for separate output builds. + +The codec tests include the production header directly and compare invented, +independently specified golden bytes. They cover truncation, category ranges, +length/padding/overflow, unaligned input, OOB descriptor ownership, response +correlation, token preservation and command ID packing. + +The owned-copy test extracts the actual production function from afk.c, prints +its SHA256, and compiles it with synthetic command slots and CPU buffers. It +checks active/inactive/done slots, high device addresses, boundaries and copies. +Locks and DMA barriers are mocked: this test does not qualify concurrency, +DMA coherency, firmware, a graphics device or any hardware display behavior. + +All build products are temporary. No device node is opened and no GPU, firmware +or target machine is contacted. The extractor fails if production function +boundaries change, requiring review rather than silently testing a stale copy. diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c new file mode 100644 index 00000000000000..de49360bdabeae --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +#undef NDEBUG +#include +#include +#include +#include +#include "afkv2-codec.h" + +static unsigned int assertions; +#define CHECK(x) \ + do { \ + assertions++; \ + assert(x); \ + } while (0) + +/* Invented fixtures with independently specified wire bytes; no device data. */ +static const u8 open_fixture[24] = { 0x7b, 0, 0x34, 0x12, 0x10, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0x12, 0, 1, 0, 0, 0, 0, 0 }; +static const u8 descriptor_fixture[32] = { + 1, 0xab, 0, 0, 0x40, 0, 0, 0, 0x88, 0x77, 0x66, + 0x55, 0x44, 0x33, 0x22, 0x11, 8, 7, 6, 5, 4, 3, + 2, 1, 0x40, 0, 0, 0, 0x10, 0, 0, 0 +}; + +int main(void) +{ + u8 data[128], command[32], unaligned[129], saved[128]; + struct afkv2_frame frame, reply; + struct afkv2_command parsed, response; + size_t written = 999; + int ret; + + memset(data, 0xa5, sizeof(data)); + CHECK(!afkv2_encode_open(data, sizeof(data), 0x7b, 0x1234, &written)); + CHECK(written == sizeof(open_fixture)); + CHECK(!memcmp(data, open_fixture, sizeof(open_fixture))); + CHECK(data[24] == 0xa5); + CHECK(!afkv2_decode_frame(data, written, false, &frame)); + CHECK(frame.interface == 0x1234 && frame.type == AFKV2_OPEN); + CHECK(frame.category == AFKV2_REPORT && frame.interface_flags == 1); + CHECK(frame.payload_size == 0 && frame.padding_size == 0); + memcpy(unaligned + 1, open_fixture, sizeof(open_fixture)); + CHECK(!afkv2_decode_frame(unaligned + 1, sizeof(open_fixture), false, + &frame)); + CHECK(afkv2_decode_frame(data, SIZE_MAX, true, &frame) == -E2BIG); + CHECK(afkv2_decode_frame(NULL, 24, false, &frame) == -EINVAL); + CHECK(afkv2_decode_frame(data, 24, false, NULL) == -EINVAL); + for (size_t i = 0; i < 24; i++) + CHECK(afkv2_decode_frame(data, i, false, &frame) == -EINVAL); + CHECK(afkv2_decode_frame(data, 32, false, &frame) == -EINVAL); + CHECK(!afkv2_decode_frame(data, 32, true, &frame) && + frame.padding_size == 8); + for (u32 i = 0; i < 16; i++) { + afkv2_put32(data + 4, i); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + } + afkv2_put32(data + 4, 17); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + afkv2_put32(data + 4, UINT32_MAX); + CHECK(afkv2_decode_frame(data, 24, false, &frame) == -EINVAL); + memcpy(data, open_fixture, sizeof(open_fixture)); + for (unsigned int category = 0; category < 256; category++) { + data[17] = category; + ret = afkv2_decode_frame(data, 24, false, &frame); + CHECK(category <= 2 ? ret == 0 : ret == -EPROTO); + } + CHECK(!afkv2_encode_oob_command(command, sizeof(command), 0xab, + 0x1122334455667788ULL, 64, + 0x0102030405060708ULL, 16)); + CHECK(!memcmp(command, descriptor_fixture, sizeof(command))); + CHECK(!afkv2_decode_command(command, sizeof(command), &parsed)); + CHECK(parsed.oob && parsed.id == 0xab && parsed.value == 64); + CHECK(parsed.rx_address == 0x1122334455667788ULL && + parsed.tx_size == 16); + CHECK(parsed.inline_size == 0); + for (size_t i = 0; i < sizeof(command); i++) + CHECK(afkv2_decode_command(command, i, &response) == -EINVAL); + command[0] = 2; + CHECK(afkv2_decode_command(command, sizeof(command), &response) == + -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + command[2] = 1; + CHECK(afkv2_decode_command(command, sizeof(command), &response) == + -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + CHECK(afkv2_encode_oob_command(command, 31, 0, 1, 1, 2, 1) == -ENOSPC); + CHECK(afkv2_encode_oob_command(command, 32, 0, UINT64_MAX, 2, 0, 0) == + -EINVAL); + CHECK(afkv2_encode_oob_command(command, 32, 0, 1, AFKV2_MAX_PAYLOAD + 1, + 0, 0) == -EINVAL); + CHECK(afkv2_encode_oob_command(command, 32, 0, 1, 0, 0, 0) == -EINVAL); + memcpy(command, descriptor_fixture, sizeof(command)); + CHECK(!afkv2_encode_frame(data, sizeof(data), 1, 0x1234, + 0xaabbccddeeff0011ULL, 0xc0, AFKV2_COMMAND, 0, + command, sizeof(command), &written)); + CHECK(written == 56 && afkv2_get32(data + 4) == 48); + CHECK(!afkv2_decode_frame(data, written, false, &frame)); + CHECK(frame.token == 0xaabbccddeeff0011ULL && + frame.category == AFKV2_COMMAND); + CHECK(!afkv2_encode_error_reply(unaligned, sizeof(unaligned), 2, &frame, + &parsed, 0xe00002c7U, &written)); + CHECK(!afkv2_decode_frame(unaligned, written, false, &reply)); + CHECK(reply.token == frame.token && reply.interface == frame.interface); + CHECK(reply.type == frame.type && reply.category == AFKV2_RESPONSE); + CHECK(!afkv2_decode_command(reply.payload, reply.payload_size, + &response)); + CHECK(response.value == 0xe00002c7U && response.rx_size == 0 && + response.tx_size == 0); + CHECK(response.rx_address == parsed.rx_address && + response.tx_address == parsed.tx_address); + CHECK(!afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16)); + response.value = 0; + response.rx_size = 64; + response.tx_size = 16; + CHECK(!afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16)); + response.rx_address++; + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + response.rx_address--; + response.rx_size++; + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + response.rx_size--; + CHECK(afkv2_check_response(&reply, &response, 0x1235, 0xc0, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc1, 0xab, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + CHECK(afkv2_check_response(&reply, &response, 0x1234, 0xc0, 0xaa, + parsed.rx_address, 64, parsed.tx_address, + 16) == -EPROTO); + memset(data, 0xa5, sizeof(data)); + memcpy(saved, data, sizeof(data)); + CHECK(afkv2_encode_open(data, 23, 0, 1, &written) == -ENOSPC); + CHECK(!memcmp(data, saved, sizeof(data))); + CHECK(afkv2_encode_open(data, sizeof(data), 0, 0, &written) == -EINVAL); + CHECK(!memcmp(data, saved, sizeof(data))); + CHECK(afkv2_encode_frame(data, sizeof(data), 0, 1, 0, 0, 0, 0, data, + SIZE_MAX, &written) == -E2BIG); + CHECK(!memcmp(data, saved, sizeof(data))); + u8 *large = malloc(AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD); + + CHECK(large); + memset(large, 0x5a, AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD); + CHECK(!afkv2_encode_frame(large, AFKV2_HEADER_SIZE + AFKV2_MAX_PAYLOAD, + 0, 1, 0, 0xc0, 1, 0, large, AFKV2_MAX_PAYLOAD, + &written)); + CHECK(!afkv2_decode_frame(large, written, false, &frame)); + CHECK(frame.payload_size == AFKV2_MAX_PAYLOAD && + frame.payload[0] == 0x5a); + free(large); + bool ids[256] = { 0 }; + + for (unsigned int generation = 0; generation < 16; generation++) { + for (unsigned int slot = 0; slot < 16; slot++) { + u8 id = afkv2_command_id(generation, slot); + + CHECK(!ids[id]); + ids[id] = true; + CHECK((id & 15) == slot); + } + } + CHECK(afkv2_command_id(16, 3) == afkv2_command_id(0, 3)); + CHECK(afkv2_owned_span(0x1000, 32, 0x1000, 32)); + CHECK(afkv2_owned_span(0x1000, 32, 0x101c, 4)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x0fff, 1)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x101d, 4)); + CHECK(!afkv2_owned_span(0x1000, 32, 0x1020, 1)); + CHECK(!afkv2_owned_span(0x1000, 0, 0x1000, 0)); + CHECK(afkv2_owned_span(UINT64_MAX - 15, 16, UINT64_MAX, 1)); + CHECK(!afkv2_owned_span(UINT64_MAX - 15, 17, UINT64_MAX, 1)); + printf("PASS %u assertions\n", assertions); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py new file mode 100644 index 00000000000000..21fe63b6d3823c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-owned.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Compile the production ownership checker with host-only synchronization mocks.""" +import hashlib +from pathlib import Path +import sys + + +def main(): + source = Path(sys.argv[1]).read_text(encoding="utf-8") + output = Path(sys.argv[2]) + first = "static int afk_v2_access_owned(" + last = "static void afk_v2_callback(" + if source.count(first) != 1 or source.count(last) != 1: + raise SystemExit("FAIL: ownership function boundaries changed; review extractor") + start = source.index(first) + end = source.index(last, start) + body = source[start:end] + if "int afk_v2_copy_owned(" not in body: + raise SystemExit("FAIL: production ownership wrapper missing") + directory = Path(__file__).resolve().parent + output.write_text((directory / "owned-prefix.c").read_text() + body + + (directory / "owned-suffix.c").read_text(), encoding="utf-8") + print("Production ownership function SHA256:", hashlib.sha256(body.encode()).hexdigest()) + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h new file mode 100644 index 00000000000000..a5c3a3f44a5912 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#define EINVAL 22 +#define ENOSPC 28 +#define E2BIG 7 +#define EPROTO 71 diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h new file mode 100644 index 00000000000000..e74214a66ed831 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/string.h @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h new file mode 100644 index 00000000000000..4d490ef828ccdb --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/types.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c new file mode 100644 index 00000000000000..8ff88ec1ec5b00 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-prefix.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +#undef NDEBUG +#include +#include +#include "afkv2-codec.h" +#define MAX_PENDING_CMDS 16 +#define SZ_1M 1048576U +#define EACCES 13 +#define spin_lock_irqsave(lock, flags) \ + do { \ + (void)(lock); \ + (flags) = 0; \ + } while (0) +#define spin_unlock_irqrestore(lock, flags) \ + do { \ + (void)(lock); \ + (void)(flags); \ + } while (0) +#define dma_rmb() \ + do { \ + } while (0) +#define dma_wmb() \ + do { \ + } while (0) +#define dev_warn_ratelimited(...) \ + do { \ + } while (0) +static int test_bit(unsigned int bit, const unsigned long *map) +{ + return !!(*map & (1UL << bit)); +} + +struct epic_cmd_info { + bool done; + void *rxbuf, *txbuf; + u64 rxbuf_dma, txbuf_dma; + size_t rxlen, txlen; +}; + +struct apple_dcp_afkep { + bool wire_v2; +}; + +struct apple_epic_service { + int lock; + struct apple_dcp_afkep *ep; + unsigned long cmd_map[1]; + struct epic_cmd_info cmds[MAX_PENDING_CMDS]; +}; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c new file mode 100644 index 00000000000000..770b16bc2fa944 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/owned-suffix.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Aurora Silicon */ +int main(void) +{ + u8 rx[16], tx[16], output[16]; + struct apple_dcp_afkep ep = { .wire_v2 = true }; + struct apple_epic_service service = { .ep = &ep }; + struct epic_cmd_info *cmd = &service.cmds[0]; + + memset(rx, 0x12, sizeof(rx)); + memset(tx, 0x34, sizeof(tx)); + memset(output, 0x56, sizeof(output)); + cmd->rxbuf = rx; + cmd->rxbuf_dma = 0x10000001000ULL; + cmd->rxlen = sizeof(rx); + cmd->txbuf = tx; + cmd->txbuf_dma = 0x10000002000ULL; + cmd->txlen = sizeof(tx); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 4, false) == + -EACCES); + service.cmd_map[0] = 1; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 12, output, 4, + false) == 0); + assert(output[0] == 0x12 && output[3] == 0x12 && output[4] == 0x56); + assert(afk_v2_copy_owned(&service, cmd->txbuf_dma + 4, output, 4, + false) == 0); + assert(output[0] == 0x34 && output[4] == 0x56); + memset(output, 0x78, 4); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 8, output, 4, + true) == 0); + assert(rx[7] == 0x12 && rx[8] == 0x78 && rx[11] == 0x78 && + rx[12] == 0x12); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma - 1, output, 1, + true) == -EACCES); + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma + 13, output, 4, + true) == -EACCES); + assert(afk_v2_copy_owned(&service, cmd->txbuf_dma + 16, output, 1, + true) == -EACCES); + cmd->done = true; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 1, false) == + -EACCES); + cmd->done = false; + ep.wire_v2 = false; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, output, 1, false) == + -EINVAL); + ep.wire_v2 = true; + assert(afk_v2_copy_owned(&service, cmd->rxbuf_dma, NULL, 1, false) == + -EINVAL); + assert(afk_v2_copy_owned(&service, 0, NULL, 0, false) == 0); + assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 15, true, + false) == 0); + assert(afk_v2_access_owned(&service, cmd->rxbuf_dma + 1, NULL, 16, true, + false) == -EACCES); + puts("PASS owned-copy boundaries (actual extracted function; lock/barrier mocks)"); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh new file mode 100755 index 00000000000000..4e13f6d05d986c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only +set -eu + +afkv2_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +afkv2_root=${AFKV2_SOURCE_ROOT:-$(CDPATH= cd -- "$afkv2_dir/../../../../../.." && pwd)} +afkv2_cc=${HOSTCC:-${CC:-cc}} +afkv2_cc_command=${afkv2_cc%% *} +afkv2_python=${PYTHON:-python3} +if ! command -v "$afkv2_cc_command" >/dev/null 2>&1 || + ! command -v "$afkv2_python" >/dev/null 2>&1; then + echo "SKIP: a host C compiler and Python 3 are required" + exit 4 +fi +if [ ! -f "$afkv2_root/drivers/gpu/drm/apple/afkv2-codec.h" ]; then + echo "SKIP: run from a source tree containing the AFKV2 codec" + exit 4 +fi +afkv2_tmp=$(mktemp -d "${TMPDIR:-/tmp}/apple-afkv2.XXXXXX") +trap 'rm -rf -- "$afkv2_tmp"' EXIT +trap 'exit 1' HUP INT TERM +afkv2_sanitizers= +if [ "${AFKV2_SANITIZERS:-0}" = 1 ]; then + afkv2_sanitizers="-fsanitize=address,undefined -fno-omit-frame-pointer" +fi + +# Compiler arguments, CFLAGS and sanitizers use normal compiler word splitting. +# shellcheck disable=SC2086 +$afkv2_cc -std=c11 -Wall -Wextra -Werror -pedantic ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_dir/codec.c" -o "$afkv2_tmp/codec" +"$afkv2_tmp/codec" +"$afkv2_python" "$afkv2_dir/extract-owned.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/owned.c" +# The extracted kernel function uses GNU C void-pointer arithmetic. +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/owned.c" -o "$afkv2_tmp/owned" +"$afkv2_tmp/owned" From d1e809f95059cc1926b6b737e37e63f30e750341 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:44:13 +1100 Subject: [PATCH 398/631] dt-bindings: phy: apple,atcphy: describe T8140 DP setup lists Document the three bounded calibration lists used for T8140 DisplayPort setup. Describe their register regions and ordering, and reject these properties on other PHY variants. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/phy/apple,atcphy.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index 24841218704009..fef0605343e8f5 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -50,6 +50,9 @@ allOf: reg-names: maxItems: 5 apple,tunable-usb2phy-reg-dflt: false + apple,tunable-dp-common-pre: false + apple,tunable-dp-lpdptx: false + apple,tunable-dp-common-post: false $defs: apple,tunable: @@ -179,6 +182,27 @@ properties: Note that lane here refers to a USB RX and TX pair re-used for DisplayPort and not to an individual DisplayPort differential lane. + apple,tunable-dp-common-pre: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort core setup applied before the LPDPTX list. The + bootloader supplies device-specific values for the core register region. + + apple,tunable-dp-lpdptx: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort setup for the lpdptx register region, applied between + the common-pre and common-post lists. + + apple,tunable-dp-common-post: + $ref: "#/$defs/apple,tunable" + maxItems: 5461 + description: > + T8140 DisplayPort core setup applied after the LPDPTX list. All three + DisplayPort lists are needed before selecting a DisplayPort PHY mode. + apple,tunable-usb2phy-reg-dflt: $ref: "#/$defs/apple,tunable" description: > From f3e537eb9c162cf57a353c6bf510669a07768d4a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:44:13 +1100 Subject: [PATCH 399/631] phy: apple: apply the T8140 DisplayPort setup lists The T8140 variant currently returns from AUX setup without configuring it. The bootloader supplies three ordered DisplayPort setup lists, but the PHY driver does not read or apply them. Validate every list against its mapped register resource and apply the complete bundle at AUX setup. Reject DP selection if the bundle is missing or invalid, while keeping ordinary USB operation independent. Other PHY generations keep their existing setup paths. Drive-level training programming remains unchanged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-dp-tunables.h | 13 ++++++ drivers/phy/apple/atc.c | 67 ++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 drivers/phy/apple/atc-dp-tunables.h diff --git a/drivers/phy/apple/atc-dp-tunables.h b/drivers/phy/apple/atc-dp-tunables.h new file mode 100644 index 00000000000000..6d0cb00158c71f --- /dev/null +++ b/drivers/phy/apple/atc-dp-tunables.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_ATC_DP_TUNABLES_H +#define APPLE_ATC_DP_TUNABLES_H + +#include + +static inline bool atc_dp_list_length_valid(int length) +{ + return length >= 0 && length <= 65536 && length % 12 == 0; +} + +#endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 5485781b76a3d3..7a683337924db8 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -43,6 +43,7 @@ #include #include +#include "atc-dp-tunables.h" #include "atc-t8122-dp.h" #include "atc-tunnel.h" @@ -648,6 +649,7 @@ struct atcphy_hw { u32 pipe_mux_data; bool has_usb2phy_reg; bool optional_tunables; + bool has_t8140_dp; bool dp_t8122; bool park_pipe_unlocked; bool park_dummy_phy; @@ -664,6 +666,9 @@ struct atcphy_hw { * @tunables.lane_usb3: USB3 lane-specific tunables * @tunables.lane_dp: DisplayPort lane-specific tunables * @tunables.lane_usb4: USB4 lane-specific tunables + * @tunables.dp_common_pre: T8140 DisplayPort common setup + * @tunables.dp_lpdptx: T8140 DisplayPort LPDPTX setup + * @tunables.dp_common_post: T8140 DisplayPort common completion * @tunables.usb2phy_reg_dflt: Defaults for the secondary eUSB2 register bank * @hw: SoC-specific PHY description * @ss_tunables: The complete SuperSpeed tunable set was supplied @@ -708,6 +713,7 @@ struct atcphy_hw { * @regs.lpdptx: DisplayPort registers * @res: Resources for memory-mapped registers, used to verify that tunables aren't out of bounds * @res.core: Core register resource + * @res.lpdptx: DisplayPort register resource * @res.axi2af: AXI to Apple Fabric interface resource * @res.usb2phy_reg: Secondary eUSB2 register bank resource * @phys: PHY instances @@ -731,6 +737,9 @@ struct apple_atcphy { struct apple_tunable *lane_dp[2]; struct apple_tunable *lane_usb4[2]; struct apple_tunable *usb2phy_reg_dflt; + struct apple_tunable *dp_common_pre; + struct apple_tunable *dp_lpdptx; + struct apple_tunable *dp_common_post; } tunables; const struct atcphy_hw *hw; @@ -776,6 +785,7 @@ struct apple_atcphy { struct { struct resource *core; + struct resource *lpdptx; struct resource *axi2af; struct resource *usb2phy_reg; } res; @@ -1591,6 +1601,14 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { + if (atcphy->hw->has_t8140_dp) { + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); + apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); + return; + } + + /* FIXME */ /* T8122 PHYs with DP support enable AUX in atcphy_dp_aux_on_t8122() */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) return; @@ -3001,6 +3019,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; /* the mode change below may power the PHY down: stop the tunnel clock first */ if (apple_atc_tunnel_is_t6030(atcphy)) { ret = atc_t8122_tunnel_stop(atcphy); @@ -3776,6 +3797,11 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta case APPLE_ATCPHY_MODE_USB3: case APPLE_ATCPHY_MODE_USB3_DP: case APPLE_ATCPHY_MODE_DP: + if (atcphy->hw->has_t8140_dp && atcphy_modes[target_mode].enable_dp_aux && + !atcphy->dp_tunables) + return -EOPNOTSUPP; + /* Without the lane tunables the SuperSpeed lanes are not calibrated */ + if (!atcphy->ss_tunables) /* * Without the lane tunables the SuperSpeed lanes are not calibrated. * T8122 four-lane DP programs its lanes itself and needs only the @@ -3845,6 +3871,37 @@ static int atcphy_probe_mux(struct apple_atcphy *atcphy) return devm_add_action_or_reset(atcphy->dev, atcphy_typec_mux_unregister, mux); } +static int atcphy_load_t8140_dp(struct apple_atcphy *atcphy) +{ + struct { + const char *name; + struct resource *resource; + struct apple_tunable **out; + } lists[] = { + { "apple,tunable-dp-common-pre", atcphy->res.core, + &atcphy->tunables.dp_common_pre }, + { "apple,tunable-dp-lpdptx", atcphy->res.lpdptx, + &atcphy->tunables.dp_lpdptx }, + { "apple,tunable-dp-common-post", atcphy->res.core, + &atcphy->tunables.dp_common_post }, + }; + if (!of_machine_is_compatible("apple,j700")) + return -EOPNOTSUPP; + for (unsigned int i = 0; i < ARRAY_SIZE(lists); i++) { + struct property *property = of_find_property(atcphy->np, lists[i].name, NULL); + + if (!property || !atc_dp_list_length_valid(property->length)) + return -EINVAL; + *lists[i].out = devm_apple_tunable_parse(atcphy->dev, atcphy->np, + lists[i].name, lists[i].resource); + if (IS_ERR(*lists[i].out)) + return PTR_ERR(*lists[i].out); + } + + atcphy->dp_tunables = true; + return 0; +} + static int atcphy_load_tunables(struct apple_atcphy *atcphy) { size_t tunable_count; @@ -3937,6 +3994,13 @@ static int atcphy_load_tunables(struct apple_atcphy *atcphy) atcphy->tunables.usb2phy_reg_dflt = tunable; } + if (atcphy->hw->has_t8140_dp) { + int ret = atcphy_load_t8140_dp(atcphy); + + if (ret) + dev_warn(atcphy->dev, "DisplayPort tunable bundle unavailable: %d\n", ret); + } + atcphy->ss_tunables = !ss_missing; atcphy->dp_tunables = atcphy->hw->dp_t8122 && !common_missing; if (atcphy->dp_only && !atcphy->dp_tunables) @@ -3958,7 +4022,7 @@ static int atcphy_map_resources(struct platform_device *pdev, struct apple_atcph struct resource **res; } resources[] = { { "core", &atcphy->regs.core, &atcphy->res.core }, - { "lpdptx", &atcphy->regs.lpdptx, NULL }, + { "lpdptx", &atcphy->regs.lpdptx, &atcphy->res.lpdptx }, { "axi2af", &atcphy->regs.axi2af, &atcphy->res.axi2af }, { "usb2phy", &atcphy->regs.usb2phy, NULL }, { "pipehandler", &atcphy->regs.pipehandler, NULL }, @@ -4253,6 +4317,7 @@ static const struct atcphy_hw atcphy_hw_t8140 = { .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, .has_usb2phy_reg = true, + .has_t8140_dp = true, .optional_tunables = true, }; From dd5734a3eb49efb756a06beb368b0b626b431f11 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 17:54:12 +1100 Subject: [PATCH 400/631] drm/apple: open the version-two DPTX controller interfaces The experimental external profile uses AFKV2 for its DPTX port, but the companion controller endpoint still uses the legacy path. Its interfaces are discovered without receiving the version-two OPEN handshake. Select AFKV2 for the controller before starting it, so controller publication performs the same OPEN-before-ready sequence as the port. Add a one-time PHY setup message to distinguish successful calibration application from controller activation during hardware qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 3 +++ drivers/phy/apple/atc.c | 1 + 2 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 48109403812c19..8762b760563ff7 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -966,6 +966,9 @@ int dptx_controller_init(struct apple_dcp *dcp) ep = afk_init(dcp, DPTX_CONTROLLER_ENDPOINT, dptx_controller_ops); if (IS_ERR(ep)) return PTR_ERR(ep); + /* Controller interfaces require the same OPEN handshake as the port. */ + ep->wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; dcp->dptx_controller_ep = ep; ret = afk_start(ep); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 7a683337924db8..c8af44ad8175e4 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1605,6 +1605,7 @@ static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); + dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); return; } From fc728ac3019a045f91f4bcbfaab6b742c58c0344 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:12:31 +1100 Subject: [PATCH 401/631] phy: apple: sequence J700 AUX power separately from calibration J700 reaches DisplayPort mode with its setup-list values present, but readback shows AUX still clamped and both supply overrides unset. The calibration lists do not perform the AUX power transition. Request the small and large supplies in order, release the clamp, then release analog shutdown. Check each masked state with a bounded poll. Track incomplete startup so a later shutdown can unwind it, and refuse to stop the domain if AUX cleanup fails. Leave ordinary USB operation and other PHY generations on their existing paths. Add host tests of the production transition helpers with register mocks, covering phase ordering, unrelated bits and failure at every stage. These pass with AddressSanitizer and UBSan; hardware qualification is pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 94 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 7 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index c8af44ad8175e4..4eb84d7dd515f0 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -746,6 +746,7 @@ struct apple_atcphy { bool ss_tunables; bool dp_tunables; bool dp_only; + bool aux_power_requested; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -1599,20 +1600,82 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode } } -static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) +#define J700_AUX_ANALOG_CONTROL 0x16000 +#define J700_AUX_SUPPLY_CONTROL 0x16400 +#define J700_AUX_ANALOG_OFF BIT(0) +#define J700_AUX_CLAMP_SELECT BIT(8) +#define J700_AUX_CLAMP_ENABLE BIT(9) + +static int j700_aux_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) +{ + u32 observed; + + core_mask32(atcphy, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, + (observed & mask) == value, 1, 100); +} + +static int j700_aux_enable(struct apple_atcphy *atcphy) +{ + u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; + int ret; + + /* Even an incomplete power-up must be unwound before the domain is stopped. */ + atcphy->aux_power_requested = true; + for (unsigned int stage = 0; stage < 2; stage++) { + u32 fields = 3U << (2 * stage); + + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); + if (ret) + return ret; + } + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); + if (ret) + return ret; + + return j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); +} + +static int j700_aux_disable(struct apple_atcphy *atcphy) +{ + u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; + int ret; + + if (!atcphy->aux_power_requested) + return 0; + ret = j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, + J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); + if (ret) + return ret; + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); + if (ret) + return ret; + for (unsigned int remaining = 2; remaining; remaining--) { + u32 override = BIT(2 * remaining - 1); + + ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); + if (ret) + return ret; + } + atcphy->aux_power_requested = false; + return 0; +} + +static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { if (atcphy->hw->has_t8140_dp) { apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); - return; + return j700_aux_enable(atcphy); } /* FIXME */ /* T8122 PHYs with DP support enable AUX in atcphy_dp_aux_on_t8122() */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) - return; + return 0; core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N_OV); @@ -1662,10 +1725,15 @@ static void atcphy_enable_dp_aux(struct apple_atcphy *atcphy) FIELD_PREP(LPDPTX_AUX_MARGIN_RCAL_TXSWING, 12)); atcphy->dp_link_rate = -1; + return 0; } -static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) +static int atcphy_disable_dp_aux(struct apple_atcphy *atcphy) { + if (atcphy->hw->has_t8140_dp) + return j700_aux_disable(atcphy); + + /* FIXME */ /* T8122 PHYs with DP support stop AUX in atcphy_dp_stop_t8122() */ if (atcphy->hw->gen != ATCPHY_GENERATION_T8122) { set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); @@ -1680,6 +1748,7 @@ static void atcphy_disable_dp_aux(struct apple_atcphy *atcphy) core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPRX_PCLK_ENABLE); core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK2_ENABLE); + return 0; } static int atcphy_dp_configure_lane(struct apple_atcphy *atcphy, enum atcphy_lane lane, @@ -2046,7 +2115,9 @@ static int atcphy_power_off(struct apple_atcphy *atcphy) u32 reg; int ret; - atcphy_disable_dp_aux(atcphy); + ret = atcphy_disable_dp_aux(atcphy); + if (ret) + return ret; /* Enable all reset lines */ core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); @@ -3023,6 +3094,12 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; + if (atcphy->hw->has_t8140_dp && !atcphy_modes[mode].enable_dp_aux && + atcphy->aux_power_requested) { + ret = j700_aux_disable(atcphy); + if (ret) + return ret; + } /* the mode change below may power the PHY down: stop the tunnel clock first */ if (apple_atc_tunnel_is_t6030(atcphy)) { ret = atc_t8122_tunnel_stop(atcphy); @@ -3095,8 +3172,11 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) } /* Setup AUX channel if DP altmode is requested */ - if (atcphy_modes[mode].enable_dp_aux) - atcphy_enable_dp_aux(atcphy); + if (atcphy_modes[mode].enable_dp_aux) { + ret = atcphy_enable_dp_aux(atcphy); + if (ret) + return ret; + } /* Enable clocks and configure lanes */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8103) { From c4266e486e1095da9fa285a0d5d31d3dfb80b864 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:19:18 +1100 Subject: [PATCH 402/631] phy: apple: park the J700 DP lane before link training The selected DP lane still reports the active power-control nibble after its protocol changes, with both receive overrides clear. The J700 path requires this lane to reach its quiescent state before link training. Assert only the selected lane's receive overrides, then sequence its power-control field into idle with bounded readback checks. Preserve the USB lane and unrelated bits. Return failures to the mux caller instead of publishing a completed mode transition. Host tests of the production helpers cover both lanes, invalid indices, unrelated-bit preservation and a failed write at every phase, under AddressSanitizer and UBSan. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 89 ++++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 4eb84d7dd515f0..d03f85daae6c70 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1541,7 +1541,50 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) atcphy_enable_dummy_phy(atcphy); } -static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) +#define J700_PMA_RECEIVER_CONTROL 0x000c +#define J700_PMA_POWER_CONTROL 0x0200 + +static int j700_field_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) +{ + u32 observed; + + core_mask32(atcphy, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, + (observed & mask) == value, 1, 100); +} + +static int j700_pma_idle(struct apple_atcphy *atcphy, unsigned int lane) +{ + u32 field; + int ret; + + if (lane >= 2) + return -EINVAL; + for (unsigned int supply = 0; supply < 2; supply++) { + field = BIT(2 + lane + 4 * supply); + ret = j700_field_update(atcphy, J700_PMA_RECEIVER_CONTROL, field, field); + if (ret) + return ret; + } + field = BIT(4 * lane + 3); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, 0); + if (ret) + return ret; + field = BIT(4 * lane); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, field); + if (ret) + return ret; + for (unsigned int control = 1; control <= 2; control++) { + field = BIT(4 * lane + control); + ret = j700_field_update(atcphy, J700_PMA_POWER_CONTROL, field, 0); + if (ret) + return ret; + } + return 0; +} + +static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); @@ -1568,12 +1611,26 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode else core_clear32(atcphy, atcphy->hw->aciophy_crossbar, ACIOPHY_CROSSBAR_DP_BOTH_PMA); + if (atcphy->hw->has_t8140_dp) { + for (unsigned int lane = 0; lane < 2; lane++) { + int ret; + + if (!mode_cfg->dp_lane[lane]) + continue; + ret = j700_pma_idle(atcphy, lane); + if (ret) + return ret; + } + return 0; + } + + /* unclear if we need the remainder for t8122 */ /* * The PMA FSM override below is for the T8103 generation. T8122 DP lanes * are started by the link rate sequence, see atcphy_dp_set_rate_t8122(). */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) - return; + return 0; if (mode_cfg->dp_lane[0]) { core_set32(atcphy, LN0_AUSPMA_RX_TOP + LN_AUSPMA_RX_TOP_PMAFSM, @@ -1598,6 +1655,7 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode LN_AUSPMA_RX_TOP_PMAFSM_PCS_OV); udelay(10); } + return 0; } #define J700_AUX_ANALOG_CONTROL 0x16000 @@ -1606,15 +1664,6 @@ static void atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode #define J700_AUX_CLAMP_SELECT BIT(8) #define J700_AUX_CLAMP_ENABLE BIT(9) -static int j700_aux_update(struct apple_atcphy *atcphy, u32 offset, u32 mask, u32 value) -{ - u32 observed; - - core_mask32(atcphy, offset, mask, value); - usleep_range(10, 20); - return readl_poll_timeout_atomic(atcphy->regs.core + offset, observed, - (observed & mask) == value, 1, 100); -} static int j700_aux_enable(struct apple_atcphy *atcphy) { @@ -1626,15 +1675,15 @@ static int j700_aux_enable(struct apple_atcphy *atcphy) for (unsigned int stage = 0; stage < 2; stage++) { u32 fields = 3U << (2 * stage); - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, fields, fields); if (ret) return ret; } - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, J700_AUX_CLAMP_SELECT); if (ret) return ret; - return j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); + return j700_field_update(atcphy, J700_AUX_ANALOG_CONTROL, J700_AUX_ANALOG_OFF, 0); } static int j700_aux_disable(struct apple_atcphy *atcphy) @@ -1644,17 +1693,17 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) if (!atcphy->aux_power_requested) return 0; - ret = j700_aux_update(atcphy, J700_AUX_ANALOG_CONTROL, - J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); + ret = j700_field_update(atcphy, J700_AUX_ANALOG_CONTROL, + J700_AUX_ANALOG_OFF, J700_AUX_ANALOG_OFF); if (ret) return ret; - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, clamp, clamp); if (ret) return ret; for (unsigned int remaining = 2; remaining; remaining--) { u32 override = BIT(2 * remaining - 1); - ret = j700_aux_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); + ret = j700_field_update(atcphy, J700_AUX_SUPPLY_CONTROL, override, 0); if (ret) return ret; } @@ -3183,7 +3232,9 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_PCLK_EN); core_set32(atcphy, CIO3PLL_CLK_CTRL, CIO3PLL_CLK_REFCLK_EN); } - atcphy_configure_lanes(atcphy, mode); + ret = atcphy_configure_lanes(atcphy, mode); + if (ret) + return ret; /* Take the USB3 PHY out of reset */ core_set32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); From a72a11d04f3f5cb7f57f45c1337104007bd7b537 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 403/631] dt-bindings: phy: apple: describe T8140 DisplayPort drive presets Document the firmware-provided fallback and per-rate drive tables. Each has sixteen entries indexed by voltage swing and pre-emphasis; other PHYs do not accept these properties. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/phy/apple,atcphy.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml index fef0605343e8f5..3e36244ef4d5b3 100644 --- a/Documentation/devicetree/bindings/phy/apple,atcphy.yaml +++ b/Documentation/devicetree/bindings/phy/apple,atcphy.yaml @@ -53,6 +53,8 @@ allOf: apple,tunable-dp-common-pre: false apple,tunable-dp-lpdptx: false apple,tunable-dp-common-post: false + patternProperties: + "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": false $defs: apple,tunable: @@ -210,6 +212,16 @@ properties: T8140, copied from Apple's Device Tree by the bootloader. Unlike the other tunables these are applied to the usb2phy-reg region. +patternProperties: + "^apple,dp-training-table(-rbr|-hbr|-hbr2|-hbr3)?$": + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 16 + maxItems: 16 + description: > + Firmware-provided T8140 drive presets indexed by voltage swing times + four plus pre-emphasis. The unsuffixed table is the required fallback; + a link-rate suffix selects an optional rate-specific override. + required: - compatible - reg From 16ff2ea8e654a0ffc8a845817e3891de3c3adabc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:34:02 +1100 Subject: [PATCH 404/631] phy: apple: implement bounded J700 DisplayPort main-link training After correcting the crossbar route, J700 reaches SET_LINK_RATE but the legacy PHY path waits for the wrong command-ready register. Add an isolated T8140 main-link sequence, firmware drive presets, and cleanup ownership that survives failed setup or shutdown. Preserve the other PMA used for USB. Handle the version-two group-one drive command with complete bounds and range checks before changing the PHY. Cache training values only after a successful PHY operation and retain the legacy endpoint behavior. Validation: 29 PHY state scenarios, 112 callback boundary cases, 100 group routing cases, 905 AFK assertions and production owned-buffer/AUX tests under ASan/UBSan; schema validation and strict checkpatch. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 4 + drivers/gpu/drm/apple/afk.c | 17 +- drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/dptx-j700-training.h | 54 ++++ drivers/gpu/drm/apple/dptxep.c | 30 +- drivers/gpu/drm/apple/dptxep.h | 1 + drivers/phy/apple/atc-j700-dp.h | 333 +++++++++++++++++++++ drivers/phy/apple/atc.c | 46 ++- 8 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 drivers/gpu/drm/apple/dptx-j700-training.h create mode 100644 drivers/phy/apple/atc-j700-dp.h diff --git a/MAINTAINERS b/MAINTAINERS index 9ecfd2681ff6e1..11cffe50b0ed89 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2014,6 +2014,7 @@ F: arch/arm64/include/asm/apple-idle.h F: arch/arm64/kernel/idle-apple.S F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ +F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c F: drivers/pci/controller/pcie-apple-piodma* F: drivers/soc/apple/aop.rs @@ -2025,6 +2026,9 @@ F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs +F: tools/diagnostics/apple-dptx-training-test.c +F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-j700-dp.py F: tools/testing/apple/ APPLETALK NETWORK LAYER diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 77dc69935348af..ce56ce9a82e994 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -787,7 +787,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, (command->oob && request_size > command->rx_size)) goto reject; service = afk_service_get(afk_epic_find_service(ep, frame->interface)); - if (!service || READ_ONCE(service->torndown) || !service->ops->call) { + if (!service || READ_ONCE(service->torndown) || + (!service->ops->call && !service->ops->call_v2)) { if (service) afk_service_put(service); goto reject; @@ -811,7 +812,7 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, } call = (const void *)request; body_size = request_size - sizeof(*call); - if (le16_to_cpu(call->group) || + if ((le16_to_cpu(call->group) && !service->ops->call_v2) || le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le32_to_cpu(call->data_len) != body_size) goto release; @@ -822,9 +823,15 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, } reply[1] = command->id; memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); - ret = service->ops->call(service, le32_to_cpu(call->command), - request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + if (service->ops->call_v2) + ret = service->ops->call_v2(service, le16_to_cpu(call->group), + le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + else + ret = service->ops->call(service, le32_to_cpu(call->command), + request + sizeof(*call), body_size, + reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); if (ret) goto release; reply_size = AFKV2_COMMAND_SIZE + request_size; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 12cdcde420ce54..3a197fc1f360f4 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -79,6 +79,8 @@ struct apple_epic_service_ops { int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); + int (*call_v2)(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room); int (*report)(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size); void (*teardown)(struct apple_epic_service *service); diff --git a/drivers/gpu/drm/apple/dptx-j700-training.h b/drivers/gpu/drm/apple/dptx-j700-training.h new file mode 100644 index 00000000000000..09a5d1018a7fb1 --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-j700-training.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_DPTX_J700_TRAINING_H +#define APPLE_DPTX_J700_TRAINING_H + +static int dptx_j700_training(struct apple_epic_service *service, bool write, + const void *data, size_t size, void *reply, size_t room) +{ + struct dptx_port *port = service->cookie; + union phy_configure_opts options = {}; + u32 count; + int ret; + + if (!port || !port->atcphy) + return -ENODEV; + if (size < 32 || room < 32) + return -EINVAL; + count = get_unaligned_le32(data + (write ? 16 : 0)); + if (count > 4 || (write && (!count || count == 3)) || room < 32 + 12 * count || + (write && size < 32 + 12 * count)) + return -EINVAL; + if (write) { + ret = phy_validate(port->atcphy, PHY_MODE_DP, 0, &options); + if (ret) + return ret; + if (count > options.dp.lanes) + return -EINVAL; + for (unsigned int i = 0; i < count; i++) { + const u8 *entry = data + 32 + 12 * i; + + if (get_unaligned_le32(entry) || get_unaligned_le32(entry + 4) > 3 || + get_unaligned_le32(entry + 8) > 3) + return -EINVAL; + options.dp.voltage[i] = get_unaligned_le32(entry + 4); + options.dp.pre[i] = get_unaligned_le32(entry + 8); + } + options.dp.lanes = count; + options.dp.set_voltages = true; + ret = phy_configure(port->atcphy, &options); + if (ret) + return ret; + memcpy(port->trained_voltage, options.dp.voltage, sizeof(port->trained_voltage)); + memcpy(port->trained_pre, options.dp.pre, sizeof(port->trained_pre)); + } + memset(reply, 0, room); + put_unaligned_le32(count, reply + (write ? 16 : 0)); + for (unsigned int i = 0; i < count; i++) { + put_unaligned_le32(port->trained_voltage[i], reply + 36 + 12 * i); + put_unaligned_le32(port->trained_pre[i], reply + 40 + 12 * i); + } + return 0; +} + +#endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 8762b760563ff7..54fa41ef107e0a 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -20,6 +21,8 @@ #include "parser.h" #include "trace.h" +#include "dptx-j700-training.h" + struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -286,6 +289,13 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, return -EINVAL; reply->retcode = cpu_to_le32(0); + if (service->ep->wire_v2) { + memset(reply, 0, reply_size); + put_unaligned_le32(3, reply_ + 20); + put_unaligned_le32(3, reply_ + 24); + return 0; + } + reply->max_drive_settings[0] = cpu_to_le32(0x3); reply->max_drive_settings[1] = cpu_to_le32(0x3); @@ -427,6 +437,9 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); + if (service->ep->wire_v2 && lane_count > dptx->lane_count) + return -EINVAL; + if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " "count:%llu phy: %d\n", lane_count, dptx->lane_count); @@ -437,7 +450,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic dptx->phy_ops.dp.lanes = lane_count; /* USB4 DP IN has no ATC PHY; still accept the lane count. */ dptx->phy_ops.dp.set_lanes = - dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); + service->ep->wire_v2 || dcp_is_usb4_output(dcp) || (dcp->dptx_phy > 3); break; default: dev_err(dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); @@ -934,11 +947,26 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, } } +static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, + const void *data, size_t size, void *reply, size_t room) +{ + if (group == 1 && command == DPTX_APCALL_SET_DRIVE_SETTINGS) + return dptx_j700_training(service, true, data, size, reply, room); + if (group) + return -EOPNOTSUPP; + if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) + return -EOPNOTSUPP; + if (command == DPTX_APCALL_GET_DRIVE_SETTINGS) + return dptx_j700_training(service, false, data, size, reply, room); + return dptxport_call(service, command, data, size, reply, room); +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .init = dptxport_init, .call = dptxport_call, + .call_v2 = dptxport_group_call, }, {} }; diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index b6360621351db6..1e772d2436d10c 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -59,6 +59,7 @@ struct dptx_port { u32 lane_count; u32 link_rate, pending_link_rate; u32 drive_settings[2]; + u32 trained_voltage[4], trained_pre[4]; /* * Per-boot invocation counters for the validate/connect/request/ * release AFK calls, used together with the caller return address diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h new file mode 100644 index 00000000000000..185551e5a589e6 --- /dev/null +++ b/drivers/phy/apple/atc-j700-dp.h @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* Copyright 2026 Aurora Silicon */ +#ifndef APPLE_ATC_J700_DP_H +#define APPLE_ATC_J700_DP_H + +/* Register facts from the authorized J700 hardware audit. */ +struct j700_dp_step { + u32 offset, mask, value; + u16 delay_ns; +}; + +static void j700_dp_steps(struct apple_atcphy *p, u32 base, + const struct j700_dp_step *steps, size_t count) +{ + for (size_t i = 0; i < count; i++) { + core_mask32(p, base + steps[i].offset, steps[i].mask, steps[i].value); + if (steps[i].delay_ns) + ndelay(steps[i].delay_ns); + } +} + +static int j700_dp_wait(struct apple_atcphy *p, u32 offset, u32 mask, bool set) +{ + u32 value; + + return readl_poll_timeout_atomic(p->regs.core + offset, value, + !!(value & mask) == set, 1, 100); +} + +static void j700_dp_power(struct apple_atcphy *p, unsigned int lane) +{ + unsigned int shift = 4 * lane; + const u32 bits[] = { BIT(2), BIT(1), BIT(0), BIT(3) }; + + for (unsigned int i = 0; i < ARRAY_SIZE(bits); i++) { + u32 mask = bits[i] << shift; + + core_mask32(p, 0x200, mask, i == 2 ? 0 : mask); + udelay(1); + } + for (unsigned int bank = 0; bank < 2; bank++) { + core_clear32(p, 0xc, BIT(6 - 4 * bank + lane)); + udelay(1); + } +} + +static void j700_dp_output_power(struct apple_atcphy *p, unsigned int lane, bool tx) +{ + u32 reg = tx ? 0x10c : 0xc; + + core_set32(p, reg, (BIT(4) | BIT(6)) << lane); + udelay(1); + core_set32(p, reg, (BIT(0) | BIT(2)) << lane); + udelay(1); + if (!tx) { + core_set32(p, 0x8, (BIT(10) | BIT(12)) << lane); + udelay(1); + core_set32(p, 0x8, (BIT(6) | BIT(8)) << lane); + udelay(1); + } + reg = tx ? 0x10c : 0x8; + core_clear32(p, reg, BIT(tx ? 8 : 14) << lane); + core_set32(p, reg, BIT(tx ? 10 : 16) << lane); + udelay(1); +} + +static void j700_dp_equalize(struct apple_atcphy *p, unsigned int lane, + bool tx, u32 preset) +{ + u32 base = (tx ? 0xc000 : 0x9000) + lane * 0x7000; + u32 swing = ((preset & 3) << 14) | ((preset >> 2) & 15); + u32 taps = ((preset >> 6) & 0x3ffff) * 2 + 1; + + if (tx) + core_set32(p, base + 0xd0, BIT(20)); + core_mask32(p, base + (tx ? 0x50 : 0x210), 0xc00f, swing); + core_set32(p, base + (tx ? 0xd4 : 0x248), BIT(28)); + core_mask32(p, base + (tx ? 0xd0 : 0x244), 0x7ffff, taps); +} + +static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int rate) +{ + static const struct j700_dp_step start[] = { + { 0x44, ~0U, 0x0aafb800, 500 }, + { 0x44, 0x1000000, 0x1000000, 250 }, + { 0x44, 0x4000000, 0x4000000, 250 }, + { 0x44, 0x7c0, 0x300, 250 }, + { 0x44, 0x6c000, 0, 250 }, + }; + static const struct j700_dp_step divide[] = { + { 0x44, 0x100000, 0x100000, 250 }, + { 0x44, 0x400000, 0x400000, 0 }, + }; + static const struct j700_dp_step finish[] = { + { 0x04, 0x33000000, 0x32000000, 0 }, + { 0x10, 0x600, 0x600, 0 }, + { 0x1c, 0xc00000, 0x800000, 0 }, + }; + u32 base = 0xd000 + lane * 0x7000; + + j700_dp_output_power(p, lane, true); + j700_dp_steps(p, base, start, ARRAY_SIZE(start)); + j700_dp_equalize(p, lane, true, 0); + if (rate != ATCPHY_DP_LINK_RATE_HBR3) + j700_dp_steps(p, base, divide, ARRAY_SIZE(divide)); + j700_dp_steps(p, base, finish, ARRAY_SIZE(finish)); + core_mask32(p, base + 0xc, 0x7000000, + rate == ATCPHY_DP_LINK_RATE_RBR ? 0x5000000 : 0x4000000); +} + +static void j700_dp_receiver(struct apple_atcphy *p, unsigned int lane, int rate) +{ + static const struct j700_dp_step setup[] = { + { 0xa4, 3, 2, 0 }, { 0xa8, 0x18000, 0x10000, 0 }, + { 0x6c, 0x18000, 0x10000, 0 }, + { 0xd4, 0x1f1f80, 0x121180, 0 }, { 0x64, 0xfc00, 0xb800, 0 }, + { 0xd8, 7, 4, 0 }, { 0xdc, 0x18, 0, 0 }, + { 0xd4, 0xe000, 0xc000, 100 }, + }; + static const struct j700_dp_step settle[] = { + { 0x6c, 0x600, 0x400, 100 }, { 0xd4, 0xd0000, 0x50000, 0 }, + { 0xd8, 3, 1, 500 }, { 0xd4, 0xc0000, 0, 200 }, + { 0x68, 0x180, 0x180, 200 }, { 0x68, 0x60, 0x60, 200 }, + { 0xd8, 3, 3, 1000 }, { 0xd8, 3, 2, 1000 }, + { 0xd8, 3, 0, 750 }, { 0xa8, 0x60000, 0x60000, 1000 }, + { 0x6c, 0x6000, 0x4000, 0 }, + }; + u32 top = 0x9000 + lane * 0x7000; + u32 base = top + 0x2000; + + core_set32(p, top + 0x160, 1); + core_set32(p, top + 0x1dc, 0x18000); + j700_dp_steps(p, base, setup, ARRAY_SIZE(setup)); + j700_dp_output_power(p, lane, false); + core_set32(p, base + 0x6c, BIT(11)); + core_mask32(p, top + 0x1d4, 3, 1); + ndelay(10); + core_set32(p, top + 0x1d4, 2); + j700_dp_steps(p, base, settle, ARRAY_SIZE(settle)); + j700_dp_equalize(p, lane, false, 0x3f000); + core_mask32(p, base + 0xa8, 0x380000, + rate == ATCPHY_DP_LINK_RATE_RBR ? 0x280000 : 0x200000); + core_mask32(p, base + 0xa8, 0xc00000, 0x800000); + ndelay(50); + core_set32(p, base + 0x30, 0xc00); + ndelay(100); +} + +static void j700_dp_outputs_off(struct apple_atcphy *p, unsigned int lane) +{ + static const struct j700_dp_step tx[] = { + { 0x1c, 0xc00000, 0xc00000, 0 }, { 0x10, 0x600, 0x400, 0 }, + { 0x04, 0x30000000, 0x20000000, 250 }, + { 0x44, 0x100000, 0, 250 }, { 0x44, 0x400000, 0, 250 }, + { 0x44, 0x4000000, 0, 250 }, { 0x44, 0x1000000, 0, 250 }, + { 0x44, 0x1000, 0, 0 }, + }; + static const struct j700_dp_step rx[] = { + { 0x30, 0xc00, 0x800, 50 }, { 0xa8, 0xc00000, 0xc00000, 500 }, + { 0xa8, 0x60000, 0x40000, 0 }, { 0x68, 0x60, 0x40, 200 }, + { 0x68, 0x180, 0x100, 200 }, { 0xd4, 0x10000, 0, 100 }, + { 0x6c, 0x600, 0x600, 0 }, + }; + const u32 tx_bits[] = { BIT(8), BIT(0), BIT(4) }; + const u32 rx_bits[] = { BIT(14), BIT(6), BIT(10) }; + + j700_dp_steps(p, 0xd000 + lane * 0x7000, tx, ARRAY_SIZE(tx)); + for (unsigned int i = 0; i < ARRAY_SIZE(tx_bits); i++) { + core_mask32(p, 0x10c, tx_bits[i] << lane, i ? 0 : tx_bits[i] << lane); + udelay(1); + } + j700_dp_steps(p, 0xb000 + lane * 0x7000, rx, ARRAY_SIZE(rx)); + core_mask32(p, 0x91d4 + lane * 0x7000, 3, 1); + core_clear32(p, 0xb06c + lane * 0x7000, BIT(11)); + for (unsigned int i = 0; i < ARRAY_SIZE(rx_bits); i++) { + core_mask32(p, 0x8, rx_bits[i] << lane, i ? 0 : rx_bits[i] << lane); + udelay(1); + } + for (unsigned int i = 0; i < 2; i++) { + core_clear32(p, 0xc, BIT(4 * i + lane)); + udelay(1); + } +} + +static int j700_dp_stop(struct apple_atcphy *p) +{ + int ret; + + if (!p->dp_owned_pmas) + return 0; + core_set32(p, 0x7000, BIT(0)); + for (unsigned int i = 0; i < 2; i++) + if (p->dp_owned_pmas & BIT(i)) + j700_dp_outputs_off(p, i); + core_mask32(p, 0x7000, 0xc, 8); + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000001); + ret = j700_dp_wait(p, 0x2000, BIT(1), true); + if (ret) + return ret; + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000018); + ret = j700_dp_wait(p, 0x2000, BIT(1), false); + if (!ret) + ret = j700_dp_wait(p, 0x7034, BIT(3), false); + if (ret) + return ret; + core_clear32(p, 0x2200, BIT(3)); + for (unsigned int i = 0; i < 2; i++) { + if (!(p->dp_owned_pmas & BIT(i))) + continue; + ret = j700_pma_idle(p, i); + if (ret) + return ret; + } + p->dp_owned_pmas = 0; + p->dp_link_rate = -1; + return 0; +} + +static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bool ssc) +{ + static const u32 parameters[4][4] = { + { 0x1e0e021c, 0, 0x156600, 0x15671b }, + { 0x1e0e01c2, 0x07fffffe, 0x155200, 0x155316 }, + { 0x1e0e01c2, 0x07fffffe, 0x554800, 0x554916 }, + { 0x1e0e02a3, 0x0bff7ffc, 0x554810, 0x564922 }, + }; + int ret; + + if (!pmas || (pmas & ~3U) || rate < 0 || (unsigned int)rate >= ARRAY_SIZE(parameters)) + return -EINVAL; + /* Claim before the first write: failed setup must still be cleaned up. */ + p->dp_owned_pmas = pmas; + p->dp_link_rate = -1; + for (unsigned int i = 0; i < 2; i++) + if (pmas & BIT(i)) + j700_dp_power(p, i); + core_clear32(p, 0x2234, 3); + core_mask32(p, 0x7000, 0x1ff0, 0x490); + for (unsigned int i = 0; i < 3; i++) + writel(parameters[rate][i == 2 ? 2 + ssc : i], p->regs.core + 0x2080 + 4 * i); + core_mask32(p, 0x2208, 0x1f0000, 0x70000); + core_set32(p, 0x2218, 1); + core_set32(p, 0x2200, BIT(3)); + core_mask32(p, 0x2000, 0x1ffffff9, 0x10000001); + ret = j700_dp_wait(p, 0x2000, BIT(1), true); + if (!ret) + ret = j700_dp_wait(p, 0x7034, BIT(3), true); + if (ret) + return ret; + core_mask32(p, 0x2000, 0x1ffffff9, 0x10014000); + ret = j700_dp_wait(p, 0x2000, BIT(1), false); + if (ret) + return ret; + for (unsigned int i = 0; i < 2; i++) + if (pmas & BIT(i)) + core_mask32(p, 0xb064 + i * 0x7000, 0xc, 8); + core_set32(p, 0x7000, 0xc); + udelay(1); + for (unsigned int i = 0; i < 2; i++) { + if (!(pmas & BIT(i))) + continue; + j700_dp_transmitter(p, i, rate); + j700_dp_receiver(p, i, rate); + } + core_clear32(p, 0x7000, 1); + p->dp_link_rate = rate; + p->dp_spread_spectrum = ssc; + return 0; +} + +static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_opts_dp *o) +{ + const struct atcphy_mode_configuration *mode; + unsigned int pmas, capacity, first, table; + int rate = -1, ret; + + guard(mutex)(&p->lock); + if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) + return o->set_voltages ? -EINVAL : j700_dp_stop(p); + if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) + return -EINVAL; + mode = atcphy_get_mode_config(p, p->mode); + pmas = !!mode->dp_lane[0] | (!!mode->dp_lane[1] << 1); + capacity = 2 * hweight32(pmas); + if (o->lanes > capacity || (o->set_lanes && o->lanes != 1 && + o->lanes != 2 && o->lanes != 4)) + return -EINVAL; + if (!p->dp_presets_valid[0]) + return -ENOENT; + if (o->set_voltages) { + if (!o->lanes) + return -EINVAL; + for (unsigned int i = 0; i < o->lanes; i++) + if (o->voltage[i] > 3 || o->pre[i] > 3) + return -EINVAL; + } + if (o->set_rate) { + const unsigned int rates[] = { 1620, 2700, 5400, 8100 }; + + for (unsigned int i = 0; i < ARRAY_SIZE(rates); i++) + if (o->link_rate == rates[i]) + rate = i; + if (rate < 0) + return -EINVAL; + if (p->dp_link_rate != rate || p->dp_owned_pmas != pmas || + p->dp_spread_spectrum != o->ssc) { + ret = j700_dp_stop(p); + if (ret) + return ret; + ret = j700_dp_start(p, pmas, rate, o->ssc); + if (ret) { + int cleanup = j700_dp_stop(p); + + dev_err(p->dev, "DP setup failed: %d, cleanup: %d\n", ret, cleanup); + return ret; + } + } + } + if (!o->set_voltages) + return 0; + if (!p->dp_owned_pmas || p->dp_link_rate < 0) + return -EINVAL; + table = p->dp_link_rate + 1; + if (!p->dp_presets_valid[table]) + table = 0; + first = pmas == 3 ? !p->swap_lanes : (pmas == 2); + for (unsigned int i = 0; i < o->lanes; i++) + j700_dp_equalize(p, first ^ (i / 2), i % 2, + p->dp_presets[table][4 * o->voltage[i] + o->pre[i]]); + return 0; +} + +#endif diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d03f85daae6c70..c4f2393f620d5b 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -676,6 +676,12 @@ struct atcphy_hw { * mode is then allowed without the USB lane tunables * @dp_only: A DisplayPort-only instance without a USB side, such as the PHY * behind the T6030 HDMI port; it maps only its core window + * @dp_tunables: The complete T8140 DisplayPort bundle was supplied + * @aux_power_requested: AUX power-up began and needs matching shutdown + * @dp_owned_pmas: J700 PMAs requiring main-link cleanup + * @dp_spread_spectrum: J700 main-link spread-spectrum state + * @dp_presets: Firmware-derived per-rate drive presets + * @dp_presets_valid: Firmware supplied each drive-preset table * @fixed_usb2: The USB2 pairs go to a fixed hub on the USB controller, which * stays in host mode; the PHY provides USB2 from probe on * @typec_mode: Mode the Type-C mux last asked for; a fixed-hub port returns to @@ -747,6 +753,10 @@ struct apple_atcphy { bool dp_tunables; bool dp_only; bool aux_power_requested; + u8 dp_owned_pmas; + bool dp_spread_spectrum; + u32 dp_presets[5][16]; + bool dp_presets_valid[5]; bool fixed_usb2; enum atcphy_mode typec_mode; bool host_active; @@ -1584,6 +1594,8 @@ static int j700_pma_idle(struct apple_atcphy *atcphy, unsigned int lane) return 0; } +#include "atc-j700-dp.h" + static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); @@ -1779,8 +1791,11 @@ static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) static int atcphy_disable_dp_aux(struct apple_atcphy *atcphy) { - if (atcphy->hw->has_t8140_dp) - return j700_aux_disable(atcphy); + if (atcphy->hw->has_t8140_dp) { + int ret = j700_dp_stop(atcphy); + + return ret ? ret : j700_aux_disable(atcphy); + } /* FIXME */ /* T8122 PHYs with DP support stop AUX in atcphy_dp_stop_t8122() */ @@ -3140,6 +3155,11 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && mode != atcphy->mode) { + ret = j700_dp_stop(atcphy); + if (ret) + return ret; + } if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; @@ -3527,6 +3547,8 @@ static int atcphy_dpphy_configure(struct phy *phy, union phy_configure_opts *opt if (atcphy->hw->dp_t8122) return atcphy_dpphy_configure_t8122(atcphy, opts); + if (atcphy->hw->has_t8140_dp) + return j700_dp_configure(atcphy, opts); if (opts->set_voltages) return -EINVAL; @@ -4030,6 +4052,26 @@ static int atcphy_load_t8140_dp(struct apple_atcphy *atcphy) return PTR_ERR(*lists[i].out); } + for (unsigned int i = 0; i < ARRAY_SIZE(atcphy->dp_presets); i++) { + static const char * const properties[] = { + "apple,dp-training-table", "apple,dp-training-table-rbr", + "apple,dp-training-table-hbr", "apple,dp-training-table-hbr2", + "apple,dp-training-table-hbr3", + }; + int count = of_property_count_u32_elems(atcphy->np, properties[i]); + int ret; + + if (i && !of_find_property(atcphy->np, properties[i], NULL)) + continue; + if (count != ARRAY_SIZE(atcphy->dp_presets[i])) + return -EINVAL; + ret = of_property_read_u32_array(atcphy->np, properties[i], + atcphy->dp_presets[i], count); + if (ret) + return ret; + atcphy->dp_presets_valid[i] = true; + } + atcphy->dp_tunables = true; return 0; } From 79d41c061373f40d688066a6c749780639f97221 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:36:12 +1100 Subject: [PATCH 405/631] drm: apple: honor separate AFK V2 callback reply capacity A drive-settings read has a short request and a larger response. Validate both owned-buffer lengths before dispatch, allocate the response capacity, and report its body length without extending the input read. Keep the full response range ownership check before any callback side effect. The 917 codec assertions now include unequal request/response lengths, undersized buffers, limits, and invalid output pointers. Production owned copy tests and all J700 PHY/training/group tests pass under sanitizers. Strict checkpatch is clean. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 27 ++++++++++--------- drivers/gpu/drm/apple/afkv2-codec.h | 19 +++++++++++++ .../selftests/drivers/gpu/apple-afkv2/codec.c | 24 +++++++++++++++++ 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index ce56ce9a82e994..17351e4dd260a7 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -775,16 +775,15 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, const struct epic_service_call *call; const u8 *request = command->inline_data; u8 *owned_request = NULL, *reply = NULL; - size_t request_size = command->oob ? command->tx_size : command->inline_size; + size_t request_size, response_size, response_body; size_t body_size, reply_size; u32 status = 0xe00002c7; int ret; if (frame->type != EPIC_SUBTYPE_STD_SERVICE || - request_size < sizeof(*call) + sizeof(u32) || - request_size > SZ_1M - AFKV2_COMMAND_SIZE || - request_size > command->value || - (command->oob && request_size > command->rx_size)) + afkv2_callback_lengths(command, sizeof(*call) + sizeof(u32), + SZ_1M - AFKV2_COMMAND_SIZE, + &request_size, &response_size)) goto reject; service = afk_service_get(afk_epic_find_service(ep, frame->interface)); if (!service || READ_ONCE(service->torndown) || @@ -796,7 +795,7 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, if (command->oob) { /* Reject an unowned response destination before any callback side effect. */ ret = afk_v2_access_owned(service, command->rx_address, NULL, - request_size, true, false); + response_size, true, false); if (ret) goto release; owned_request = kmalloc(request_size, GFP_KERNEL); @@ -816,32 +815,36 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le32_to_cpu(call->data_len) != body_size) goto release; - reply = kzalloc(AFKV2_COMMAND_SIZE + request_size, GFP_KERNEL); + response_body = response_size - sizeof(*call); + reply = kzalloc(AFKV2_COMMAND_SIZE + response_size, GFP_KERNEL); if (!reply) { status = 0xe00002bd; goto release; } reply[1] = command->id; memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); + ((struct epic_service_call *)(reply + AFKV2_COMMAND_SIZE))->data_len = + cpu_to_le32(response_body); if (service->ops->call_v2) ret = service->ops->call_v2(service, le16_to_cpu(call->group), le32_to_cpu(call->command), request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + reply + AFKV2_COMMAND_SIZE + sizeof(*call), + response_body); else ret = service->ops->call(service, le32_to_cpu(call->command), request + sizeof(*call), body_size, - reply + AFKV2_COMMAND_SIZE + sizeof(*call), body_size); + reply + AFKV2_COMMAND_SIZE + sizeof(*call), response_body); if (ret) goto release; - reply_size = AFKV2_COMMAND_SIZE + request_size; + reply_size = AFKV2_COMMAND_SIZE + response_size; if (command->oob) { ret = afk_v2_copy_owned(service, command->rx_address, - reply + AFKV2_COMMAND_SIZE, request_size, true); + reply + AFKV2_COMMAND_SIZE, response_size, true); if (ret) goto release; ret = afkv2_encode_oob_command(reply, reply_size, command->id, - command->rx_address, request_size, + command->rx_address, response_size, command->tx_address, command->tx_size); if (ret) goto release; diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index cdd5d5ad20b222..89901822753544 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -212,6 +212,25 @@ static inline int afkv2_decode_command(const void *payload, size_t size, return 0; } +/* Requests and responses may have different owned-buffer capacities. */ +static inline int afkv2_callback_lengths(const struct afkv2_command *command, + size_t minimum, size_t limit, + size_t *request, size_t *response) +{ + size_t in, out; + + if (!command || !request || !response || minimum > limit) + return -EINVAL; + in = command->oob ? command->tx_size : command->inline_size; + out = command->oob ? command->rx_size : command->inline_size; + if (in < minimum || out < minimum || in > limit || out > limit || + in > command->value || out > command->value) + return -EINVAL; + *request = in; + *response = out; + return 0; +} + /* Correlate only; no buffer is read, written or freed by this helper. */ static inline int afkv2_check_response(const struct afkv2_frame *frame, const struct afkv2_command *response, diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c index de49360bdabeae..79598033c084ab 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/codec.c @@ -178,6 +178,30 @@ int main(void) CHECK(!afkv2_owned_span(0x1000, 0, 0x1000, 0)); CHECK(afkv2_owned_span(UINT64_MAX - 15, 16, UINT64_MAX, 1)); CHECK(!afkv2_owned_span(UINT64_MAX - 15, 17, UINT64_MAX, 1)); + struct afkv2_command callback = { .oob = true, .tx_size = 96, + .rx_size = 144, .value = 144 }; + size_t input = 0, output = 0; + + CHECK(!afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + CHECK(input == 96 && output == 144); + callback.value = 143; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.value = 2048; + callback.rx_size = 2048; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.rx_size = 67; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.rx_size = 144; + callback.tx_size = 67; + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + callback.oob = false; + callback.inline_size = 96; + CHECK(!afkv2_callback_lengths(&callback, 68, 1024, &input, &output)); + CHECK(input == 96 && output == 96); + CHECK(afkv2_callback_lengths(NULL, 68, 1024, &input, &output)); + CHECK(afkv2_callback_lengths(&callback, 68, 1024, NULL, &output)); + CHECK(afkv2_callback_lengths(&callback, 68, 1024, &input, NULL)); + CHECK(afkv2_callback_lengths(&callback, 1025, 1024, &input, &output)); printf("PASS %u assertions\n", assertions); return 0; } From e9a9fc6cad3960e0b8d4e9d6c5dc2a6763a3ea94 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:42:34 +1100 Subject: [PATCH 406/631] phy: apple: preserve J700 lane-routing state during DP setup Use the T8140 routing masks for USB3/DP and four-lane DP. The generic sequence changes the miscellaneous lane-swap control and clears routing state that this generation preserves. Update the low protocol fields in order and idle only the PMAs assigned to DisplayPort. Add opt-in callback and PHY completion diagnostics to distinguish firmware waiting from a blocked PHY operation. No calibration values are logged. Host checks pass: 33 PHY routing/state cases, 112 training bounds, 100 group routing cases, 917 codec assertions, owned-copy and AUX transition tests. Strict checkpatch passes. External-display qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 6 ++++++ drivers/phy/apple/atc-j700-dp.h | 27 +++++++++++++++++++++++++++ drivers/phy/apple/atc.c | 17 +++++------------ 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 17351e4dd260a7..d10658013cc52c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -780,6 +780,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, u32 status = 0xe00002c7; int ret; + dev_dbg(ep->dcp->dev, "AFKV2 callback sizes: tx=%u rx=%u value=%u inline=%zu\n", + command->tx_size, command->rx_size, command->value, command->inline_size); if (frame->type != EPIC_SUBTYPE_STD_SERVICE || afkv2_callback_lengths(command, sizeof(*call) + sizeof(u32), SZ_1M - AFKV2_COMMAND_SIZE, @@ -825,6 +827,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, memcpy(reply + AFKV2_COMMAND_SIZE, call, sizeof(*call)); ((struct epic_service_call *)(reply + AFKV2_COMMAND_SIZE))->data_len = cpu_to_le32(response_body); + dev_dbg(ep->dcp->dev, "AFKV2 callback begin: group=%u command=%u in=%zu out=%zu\n", + le16_to_cpu(call->group), le32_to_cpu(call->command), body_size, response_body); if (service->ops->call_v2) ret = service->ops->call_v2(service, le16_to_cpu(call->group), le32_to_cpu(call->command), @@ -835,6 +839,8 @@ static void afk_v2_callback(struct apple_dcp_afkep *ep, ret = service->ops->call(service, le32_to_cpu(call->command), request + sizeof(*call), body_size, reply + AFKV2_COMMAND_SIZE + sizeof(*call), response_body); + dev_dbg(ep->dcp->dev, "AFKV2 callback end: group=%u command=%u result=%d\n", + le16_to_cpu(call->group), le32_to_cpu(call->command), ret); if (ret) goto release; reply_size = AFKV2_COMMAND_SIZE + response_size; diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 185551e5a589e6..5ce78d085159d3 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,6 +3,29 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Preserve the upper routing state established by the platform firmware. */ +static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) +{ + u32 protocols = four_lanes ? 0x492 : (p->swap_lanes ? 0x252 : 0x489); + int ret; + + core_mask32(p, 0x60, 0xfff, protocols); + if (four_lanes) { + core_mask32(p, 0x64, 0x1f, 0x14); + core_mask32(p, 0x64, 0x3ffe0, p->swap_lanes ? 0x100 : 0x22000); + } else { + core_mask32(p, 0x64, 0x1ffff, 0x110 | p->swap_lanes); + } + for (unsigned int lane = 0; lane < 2; lane++) { + if (!four_lanes && lane != !p->swap_lanes) + continue; + ret = j700_pma_idle(p, lane); + if (ret) + return ret; + } + return 0; +} + /* Register facts from the authorized J700 hardware audit. */ struct j700_dp_step { u32 offset, mask, value; @@ -213,6 +236,7 @@ static int j700_dp_stop(struct apple_atcphy *p) } p->dp_owned_pmas = 0; p->dp_link_rate = -1; + dev_dbg(p->dev, "J700 DP main stopped\n"); return 0; } @@ -265,6 +289,7 @@ static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bo core_clear32(p, 0x7000, 1); p->dp_link_rate = rate; p->dp_spread_spectrum = ssc; + dev_dbg(p->dev, "J700 DP main ready: PMAs=%u rate=%d ssc=%u\n", pmas, rate, ssc); return 0; } @@ -274,7 +299,9 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ unsigned int pmas, capacity, first, table; int rate = -1, ret; + dev_dbg(p->dev, "J700 DP configure waiting for PHY lock\n"); guard(mutex)(&p->lock); + dev_dbg(p->dev, "J700 DP configure acquired PHY lock\n"); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index c4f2393f620d5b..1d228eeccf8e15 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1600,6 +1600,11 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode { const struct atcphy_mode_configuration *mode_cfg = atcphy_get_mode_config(atcphy, mode); + if (atcphy->hw->has_t8140_dp && + (mode == APPLE_ATCPHY_MODE_USB3 || mode == APPLE_ATCPHY_MODE_USB3_DP || + mode == APPLE_ATCPHY_MODE_DP)) + return j700_dp_route(atcphy, mode == APPLE_ATCPHY_MODE_DP); + core_mask32(atcphy, atcphy->hw->aciophy_lane_mode, ACIOPHY_LANE_MODE_RX0, FIELD_PREP(ACIOPHY_LANE_MODE_RX0, mode_cfg->lane_mode[0])); core_mask32(atcphy, atcphy->hw->aciophy_lane_mode, ACIOPHY_LANE_MODE_TX0, @@ -1623,18 +1628,6 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode else core_clear32(atcphy, atcphy->hw->aciophy_crossbar, ACIOPHY_CROSSBAR_DP_BOTH_PMA); - if (atcphy->hw->has_t8140_dp) { - for (unsigned int lane = 0; lane < 2; lane++) { - int ret; - - if (!mode_cfg->dp_lane[lane]) - continue; - ret = j700_pma_idle(atcphy, lane); - if (ret) - return ret; - } - return 0; - } /* unclear if we need the remainder for t8122 */ /* From 7cf2aefb173815d715db5b35a7d2b2e76c316b33 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:45:36 +1100 Subject: [PATCH 407/631] mux: apple: retain the physical J700 crossbar gate sequence Keep the newer T602x DP IN clock controls for existing users, but exclude them from the J700 physical DP output. That output uses the original T602x gate sequence and read-clock reset location. The previous generic path wrote the additional source/output gate registers on J700 too. Validate the actual production mux function for every source on all three outputs, both with and without the J700 condition: 54 select/disconnect scenarios pass under ASan/UBSan. Existing PHY, training and AFK checks pass. Strict checkpatch passes; external hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/mux/apple-display-crossbar.c | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 11cffe50b0ed89..968913ebcb0a58 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2028,6 +2028,7 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-dp.py F: tools/testing/apple/ diff --git a/drivers/mux/apple-display-crossbar.c b/drivers/mux/apple-display-crossbar.c index 560ec53aee828c..d08b8abe6358ce 100644 --- a/drivers/mux/apple-display-crossbar.c +++ b/drivers/mux/apple-display-crossbar.c @@ -337,6 +337,8 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) unsigned int dispext_bit_en; u32 atc_bit, mux_mask, mux_val; bool enable; + /* J700 physical DP retains the original T602x clock-gate sequence. */ + bool j700_physical = index == MUX_DPPHY && of_machine_is_compatible("apple,j700"); int ret = 0; if (index >= MUX_MAX) @@ -401,11 +403,14 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) dpxbar_set32(dpxbar, T602X_REG_014, 0x4); /* Native T602x DPIN0 teardown restores the read reset at +0x24. */ - dpxbar_set32(dpxbar, T602X_FIFO_RD_PCLK2_EN, atc_bit); + dpxbar_set32(dpxbar, j700_physical ? FIFO_RD_PCLK1_EN : + T602X_FIFO_RD_PCLK2_EN, atc_bit); dpxbar_clear32(dpxbar, T602X_REG_034, atc_bit); - dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); - dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + if (!j700_physical) { + dpxbar_clear32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + dpxbar_clear32(dpxbar, CROSSBAR_DISPEXT_EN, prev_dispext_bit); + } dpxbar_mask32(dpxbar, T602X_REG_030, mux_mask, 0); dpxbar->selected_dispext[index] = -1; @@ -450,13 +455,15 @@ static int apple_dpxbar_set_t602x(struct mux_control *mux, int state) index == MUX_DPIN1 ? FIFO_RD_PCLK1_EN : T602X_REG_01C, atc_bit); dpxbar_set32(dpxbar, T602X_REG_034, atc_bit); - dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); + if (!j700_physical) + dpxbar_set32(dpxbar, CROSSBAR_ATC_EN, atc_bit); /* * t8103 enables the dispext source at 0x050. T602x left * that register at 0, so dpin0 was selected with no * source clock into the ACIO analog PHY. */ - dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); + if (!j700_physical) + dpxbar_set32(dpxbar, CROSSBAR_DISPEXT_EN, dispext_bit); dpxbar_set32(dpxbar, T602X_FIFO_RD_UNK_EN, dispext_bit); From bc6e8a1e18de1fec708bf2a796460ddbe3c79a5f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:50:55 +1100 Subject: [PATCH 408/631] drm: apple: validate J700 active lanes against PHY capacity Firmware resets the active lane count to zero before setting its link rate. Comparing the following two-lane request against that cached zero rejected a valid activation with EINVAL and prevented link completion. Query the PHY capacity for each request instead. Validate impossible counts before changing state, and propagate validation failures. Cover the zero-to-two reset sequence, unsupported counts and missing PHY. Validation: J700 PHY/training tests, 917 AFK assertions and owned-buffer checks pass under sanitizers; strict checkpatch is clean. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptx-j700-training.h | 16 ++++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 7 +++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dptx-j700-training.h b/drivers/gpu/drm/apple/dptx-j700-training.h index 09a5d1018a7fb1..a4632cd363abdf 100644 --- a/drivers/gpu/drm/apple/dptx-j700-training.h +++ b/drivers/gpu/drm/apple/dptx-j700-training.h @@ -3,6 +3,22 @@ #ifndef APPLE_DPTX_J700_TRAINING_H #define APPLE_DPTX_J700_TRAINING_H +/* Active lanes may be zero during the firmware's training reset. */ +static int dptx_j700_validate_lanes(struct dptx_port *port, u64 requested) +{ + union phy_configure_opts capacity = {}; + int ret; + + if (!port || !port->atcphy) + return -ENODEV; + if (requested > 4 || requested == 3) + return -EINVAL; + ret = phy_validate(port->atcphy, PHY_MODE_DP, 0, &capacity); + if (ret) + return ret; + return requested <= capacity.dp.lanes ? 0 : -EINVAL; +} + static int dptx_j700_training(struct apple_epic_service *service, bool write, const void *data, size_t size, void *reply, size_t room) { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 54fa41ef107e0a..0c12eb7aa86cc9 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -437,8 +437,11 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic u64 lane_count = le64_to_cpu(request->lane_count); - if (service->ep->wire_v2 && lane_count > dptx->lane_count) - return -EINVAL; + if (service->ep->wire_v2) { + ret = dptx_j700_validate_lanes(dptx, lane_count); + if (ret) + return ret; + } if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " From a7a8a0f552b7e79879352b7c5a166eb1b7ffa20e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:52:49 +1100 Subject: [PATCH 409/631] drm: apple: avoid stale active-lane warnings after V2 validation The V2 request has already passed the physical-capacity check. Restrict the older cached-count warning to the legacy path so a valid zero-to-two transition does not emit an incorrect error. No state transition changes. Validation: J700 host regression tests and strict checkpatch. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 0c12eb7aa86cc9..7ca22c11deb8b2 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -441,9 +441,7 @@ static int dptxport_call_set_active_lane_count(struct apple_epic_service *servic ret = dptx_j700_validate_lanes(dptx, lane_count); if (ret) return ret; - } - - if (dptx->lane_count < lane_count) + } else if (dptx->lane_count < lane_count) dev_err(dcp->dev, "set_active_lane_count: unexpected lane " "count:%llu phy: %d\n", lane_count, dptx->lane_count); From 2930a924dcae5d57fc77f101573e670c9224edc2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:57:03 +1100 Subject: [PATCH 410/631] phy: apple: preserve J700 transmitter programming order Keep the separate supply request/override writes and program the link-rate field before the final transmitter output controls. Combining those writes and moving the rate field after output setup did not preserve the hardware sequence. The host test now checks the actual production register transaction order, including supply writes. It rejects the previous implementation and passes the corrected sequence for both PMAs and every supported rate/SSC pair. Strict checkpatch passes. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc-j700-dp.h | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 5ce78d085159d3..11d7b320037483 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -71,16 +71,18 @@ static void j700_dp_output_power(struct apple_atcphy *p, unsigned int lane, bool { u32 reg = tx ? 0x10c : 0xc; - core_set32(p, reg, (BIT(4) | BIT(6)) << lane); - udelay(1); - core_set32(p, reg, (BIT(0) | BIT(2)) << lane); - udelay(1); - if (!tx) { - core_set32(p, 0x8, (BIT(10) | BIT(12)) << lane); - udelay(1); - core_set32(p, 0x8, (BIT(6) | BIT(8)) << lane); + for (unsigned int pair = 0; pair < 2; pair++) { + core_set32(p, reg, BIT(4 - 4 * pair + lane)); + core_set32(p, reg, BIT(6 - 4 * pair + lane)); udelay(1); } + if (!tx) { + for (unsigned int pair = 0; pair < 2; pair++) { + core_set32(p, 0x8, BIT(10 - 4 * pair + lane)); + core_set32(p, 0x8, BIT(12 - 4 * pair + lane)); + udelay(1); + } + } reg = tx ? 0x10c : 0x8; core_clear32(p, reg, BIT(tx ? 8 : 14) << lane); core_set32(p, reg, BIT(tx ? 10 : 16) << lane); @@ -115,7 +117,8 @@ static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int r { 0x44, 0x400000, 0x400000, 0 }, }; static const struct j700_dp_step finish[] = { - { 0x04, 0x33000000, 0x32000000, 0 }, + { 0x04, 0x03000000, 0x02000000, 0 }, + { 0x04, 0x30000000, 0x30000000, 0 }, { 0x10, 0x600, 0x600, 0 }, { 0x1c, 0xc00000, 0x800000, 0 }, }; @@ -126,9 +129,10 @@ static void j700_dp_transmitter(struct apple_atcphy *p, unsigned int lane, int r j700_dp_equalize(p, lane, true, 0); if (rate != ATCPHY_DP_LINK_RATE_HBR3) j700_dp_steps(p, base, divide, ARRAY_SIZE(divide)); - j700_dp_steps(p, base, finish, ARRAY_SIZE(finish)); + j700_dp_steps(p, base, finish, 2); core_mask32(p, base + 0xc, 0x7000000, rate == ATCPHY_DP_LINK_RATE_RBR ? 0x5000000 : 0x4000000); + j700_dp_steps(p, base, finish + 2, ARRAY_SIZE(finish) - 2); } static void j700_dp_receiver(struct apple_atcphy *p, unsigned int lane, int rate) From dc73885f286aece01e94dc7c3f6c36a5681afd6f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 18:57:03 +1100 Subject: [PATCH 411/631] drm: apple: answer the V2 physical-device mode query The external firmware asks whether the connection is a virtual display. Return a bounded, zero-initialized physical-device answer instead of unsupported, matching the public m1n1 V2 DPTX protocol. Keep other unknown commands rejected and retain the callback group restrictions. Validation: query size/group checks, J700 training tests, 54 crossbar scenarios, 917 codec assertions and owned-buffer tests; checkpatch passes. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 6 ++++++ drivers/gpu/drm/apple/dptxep.h | 1 + 2 files changed, 7 insertions(+) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 7ca22c11deb8b2..6c85ad39bc6069 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -957,6 +957,12 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 return -EOPNOTSUPP; if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) return -EOPNOTSUPP; + if (command == DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE) { + if (room < sizeof(struct dptxport_apcall_get_support)) + return -EINVAL; + memset(reply, 0, room); + return 0; + } if (command == DPTX_APCALL_GET_DRIVE_SETTINGS) return dptx_j700_training(service, false, data, size, reply, room); return dptxport_call(service, command, data, size, reply, room); diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 1e772d2436d10c..5072a545ce070b 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -30,6 +30,7 @@ enum dptx_apcall { DPTX_APCALL_DEVICE_NOT_RESPONDING = 22, DPTX_APCALL_DEVICE_BUSY_TIMEOUT = 23, DPTX_APCALL_DEVICE_NOT_STARTED = 24, + DPTX_APCALL_GET_VIRTUAL_DEVICE_MODE = 26, }; #define DCPDPTX_REMOTE_PORT_CORE GENMASK(3, 0) From 11d1f6d81ac01926ef2717c115ecc2e4c0a697c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:02:26 +1100 Subject: [PATCH 412/631] phy: apple: apply J700 DP calibration before common power-up Move the firmware DP calibration bundle into the tunable-application phase, before the common supply/clamp sequence. Applying it later from AUX enable preserved the final register values but not the required initialization order. Full DP uses its own common bundle; shared USB/DP keeps the USB lane calibration and then applies the DP bundle. Existing tunable operations are reused. Actual production-dispatch tests cover full/shared/reversed DP, the legacy path and fixed USB2. Existing PHY, codec and crossbar tests pass. Strict checkpatch passes. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/phy/apple/atc.c | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 968913ebcb0a58..8eb92831c35900 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2030,6 +2030,7 @@ F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-dp.py +F: tools/diagnostics/test-apple-j700-tuning.py F: tools/testing/apple/ APPLETALK NETWORK LAYER diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 1d228eeccf8e15..d39a0df03ebcd1 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1056,6 +1056,14 @@ static const struct atcphy_mode_configuration *atcphy_get_mode_config(struct app return &atcphy_modes[mode].normal; } +static void j700_dp_tunables(struct apple_atcphy *atcphy) +{ + apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); + apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); +} + static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode mode) { const int lane0 = atcphy->swap_lanes ? 1 : 0; @@ -1070,6 +1078,11 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode if (mode == APPLE_ATCPHY_MODE_USB2 && atcphy->fixed_usb2) return; + if (atcphy->hw->has_t8140_dp && mode == APPLE_ATCPHY_MODE_DP) { + j700_dp_tunables(atcphy); + return; + } + apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[0]); apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); apple_tunable_apply(atcphy->regs.core, atcphy->tunables.common[1]); @@ -1106,6 +1119,9 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode case APPLE_ATCPHY_MODE_USB2: break; } + if (atcphy->hw->has_t8140_dp && mode == APPLE_ATCPHY_MODE_USB3_DP) + j700_dp_tunables(atcphy); + } static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) @@ -1719,9 +1735,6 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { if (atcphy->hw->has_t8140_dp) { - apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_pre); - apple_tunable_apply(atcphy->regs.lpdptx, atcphy->tunables.dp_lpdptx); - apple_tunable_apply(atcphy->regs.core, atcphy->tunables.dp_common_post); dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); return j700_aux_enable(atcphy); } From e0816d2b36af1639a0219e087ffb46c6577500cb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:12:35 +1100 Subject: [PATCH 413/631] drm: apple: allow the qualified J700 provider fallback on DCPEXT Missing power-LUT properties on the external controller took the provider failure path and set dcp->crashed before display modes could be published. The existing J700 compatibility payloads were accepted only on the internal controller. Allow those same validated payloads for the explicitly experimental J700 external profile. Keep all request, capacity and property-length checks, and keep malformed properties and other machines/profiles fail-closed. Validation: actual provider-callback host tests cover the six known properties, internal compatibility, experimental external compatibility, other profiles/machines, malformed lengths and unknown requests under ASan/UBSan. The old implementation rejects the external regression case. Strict checkpatch passes; hardware qualification is pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/iomfb_h17p.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 8eb92831c35900..3141982fc89e15 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2030,6 +2030,7 @@ F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-dp.py +F: tools/diagnostics/test-apple-j700-provider.py F: tools/diagnostics/test-apple-j700-tuning.py F: tools/testing/apple/ diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 682a92879f8ef5..49fcabaeb7c66e 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -445,7 +445,9 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, goto fail; } else if (!of_property_present(node, property->dt_name) && of_machine_is_compatible("apple,j700") && - of_device_is_compatible(node, "apple,t8140-dcp")) { + (of_device_is_compatible(node, "apple,t8140-dcp") || + (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_device_is_compatible(node, "apple,t8140-dcpext")))) { /* Qualified older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); From 591e344ee1403108f2e43f3e1f0306c2a24f867e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:19:22 +1100 Subject: [PATCH 414/631] drm/apple: add scoped J700 link-rate diagnostic ceiling Preserve HBR3 by default and on other outputs. Permit known lower DP rates only for the experimental J700 physical output so training failures can be isolated without changing PHY tables. Exercise the actual callback over all ceilings and scope combinations. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6c85ad39bc6069..ea878b28a6dd89 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,12 @@ #include "dptx-j700-training.h" +/* Diagnostic ceiling for the experimental physical J700 output only. */ +static unsigned int j700_dp_max_link_rate = LINK_RATE_HBR3; +module_param(j700_dp_max_link_rate, uint, 0600); +MODULE_PARM_DESC(j700_dp_max_link_rate, + "J700 external DP ceiling: 6=RBR, 10=HBR, 20=HBR2, 30=HBR3"); + struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -365,12 +372,22 @@ static int dptxport_call_get_max_link_rate(struct apple_epic_service *service, void *reply_, size_t reply_size) { struct dptxport_apcall_link_rate *reply = reply_; + struct apple_dcp *dcp = service->ep->dcp; + u32 ceiling = READ_ONCE(j700_dp_max_link_rate); if (reply_size < sizeof(*reply)) return -EINVAL; reply->retcode = cpu_to_le32(0); - reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); + if (service->cookie && ((struct dptx_port *)service->cookie)->atcphy && + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_machine_is_compatible("apple,j700") && + (ceiling == LINK_RATE_RBR || ceiling == LINK_RATE_HBR || + ceiling == LINK_RATE_HBR2 || ceiling == LINK_RATE_HBR3)) + reply->link_rate = cpu_to_le32(ceiling); + else + reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); return 0; } From edb1b11954149faf2b6cb4c6f087f787c3ade974 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:20:51 +1100 Subject: [PATCH 415/631] drm/apple: report unknown callback bounds for diagnosis Include validated input/output lengths and channel context in the existing warning. Retain the external fail-closed behavior while identifying the D134 callback observed after link setup. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index cddec040a382e9..698e4f1523cecd 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -871,8 +871,9 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, if (!handled) { if (!iomfb_uses_queue(dcp) || !test_and_set_bit(tag, dcp->unknown_callbacks)) - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + dev_warn(dev, "received unknown callback %c%c%c%c (%u, %u) context=%u\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], + hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { /* Retain the failed session without inventing a successful reply. */ From f43eb2ea66e67546d9026b97db98168621d85967 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:25:59 +1100 Subject: [PATCH 416/631] drm/apple: answer the measured J700 external D134 record The external controller reaches video-interface publication, then sends D134 on the async channel with no input and a four-byte output. Its unanswered callback marks the strict external session crashed. Permit the existing zero reply only for that exact record on J700 T8140 DCPEXT. Retain fail-closed behavior for every other unknown external callback and preserve legacy handling. Test the production dispatcher under ASan/UBSan across 15552 tag, size, context and device/profile combinations, including output bounds and unchanged legacy replies. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 698e4f1523cecd..3403d0ade8196d 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -823,6 +823,23 @@ void dcp_hotplug(struct work_struct *work) apple_connector_backlight_sync(connector); } +/* + * J700/25G83 asks D134 after publishing its external video interface. + * The observed async record has no input and a four-byte result. Use the + * existing zero reply for this record only; its feature remains unsupported. + */ +static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, + enum dcp_context_id context, + u32 in_len, u32 out_len) +{ + return tag == 134 && context == DCP_CONTEXT_ASYNC && + in_len == 0 && out_len == sizeof(u32) && + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + of_machine_is_compatible("apple,j700") && + of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); +} + static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, void *data, u32 length, u16 offset) { @@ -832,7 +849,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, int tag; struct dcp_channel *ch = dcp_get_channel(dcp, context); u8 depth; - bool handled; + bool handled, zero_reply; if (dcp->fw_compat == DCP_FIRMWARE_H17P && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || @@ -869,13 +886,16 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, out = in + hdr->in_len; if (!handled) { - if (!iomfb_uses_queue(dcp) || - !test_and_set_bit(tag, dcp->unknown_callbacks)) + zero_reply = dcp_external_zero_reply(dcp, tag, context, + hdr->in_len, hdr->out_len); + if (!zero_reply && + (!iomfb_uses_queue(dcp) || + !test_and_set_bit(tag, dcp->unknown_callbacks))) dev_warn(dev, "received unknown callback %c%c%c%c (%u, %u) context=%u\n", hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && !zero_reply) { /* Retain the failed session without inventing a successful reply. */ dcp->crashed = true; complete(&dcp->start_done); From 44c2585d7831478152b58eb179f8836d5a345bfa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:30:46 +1100 Subject: [PATCH 417/631] drm/apple: route native training readback during modeset R24 reaches external mode export and a visible but frozen image. Firmware uses group 1 command 4 to read training settings during modeset; route it to the bounded native readback helper instead of rejecting it. Test returned cached lane values, no PHY writes and short-reply rejection. Include pending command and present-state information in the existing transaction timeout diagnostic. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 2 ++ drivers/gpu/drm/apple/iomfb.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index ea878b28a6dd89..3864b37102b5cc 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -970,6 +970,8 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 { if (group == 1 && command == DPTX_APCALL_SET_DRIVE_SETTINGS) return dptx_j700_training(service, true, data, size, reply, room); + if (group == 1 && command == DPTX_APCALL_GET_DRIVE_SETTINGS) + return dptx_j700_training(service, false, data, size, reply, room); if (group) return -EOPNOTSUPP; if (command == DPTX_APCALL_SET_DRIVE_SETTINGS) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 3403d0ade8196d..a57f755e759451 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -312,7 +312,19 @@ static void iomfb_queue_timeout(struct work_struct *work) dcp->iomfb.deadline - jiffies); } else if (dcp->iomfb.active) { WRITE_ONCE(dcp->crashed, true); - dev_err(dcp->dev, "IOMFB transaction timed out\n"); + dev_err(dcp->dev, + "IOMFB transaction timed out: cmd=%u oob=%u present=%u accepted=%u swap=%u\n", + dcp->ch_cmd.depth, dcp->ch_oobcmd.depth, + dcp->present_state_h17p.pending, dcp->present_state_h17p.accepted, + dcp->present_state_h17p.swap_id); + if (dcp->ch_cmd.depth && dcp->ch_cmd.depth <= DCP_MAX_CALL_DEPTH) { + const struct dcp_packet_header *h = + &dcp->ch_cmd.header[dcp->ch_cmd.depth - 1]; + + dev_err(dcp->dev, "pending IOMFB %c%c%c%c in=%u out=%u\n", + h->tag[3], h->tag[2], h->tag[1], h->tag[0], + h->in_len, h->out_len); + } iomfb_discard_pending(dcp); schedule_work(&dcp->vblank_wq); } From b7fabfc2753340d653ad33265d52b240fa801ff1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:35:59 +1100 Subject: [PATCH 418/631] drm/apple: keep opaque X property off experimental DCPEXT The external controller rejects the post-first-present enableGPAlphaDiv property with status 4. Treating that panel workaround as required marks the external session crashed and freezes a visible image. Leave the experimental external controller on its native presentation path while preserving the existing internal-panel property queue. Cover 396 production-queue scope/state/capacity cases and allocation failure under ASan/UBSan. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index a57f755e759451..98fa680b4a31c0 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -193,6 +193,10 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) struct iomfb_opaque_x_transaction *opaque; lockdep_assert_held(&dcp->iomfb.lock); + /* DCPEXT rejects enableGPAlphaDiv; retain the panel-only workaround. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + return 0; if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; if (dcp->iomfb.queued >= 32) From 14be4b1451e6bd4271b3c4b441272c795914221d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:45:26 +1100 Subject: [PATCH 419/631] drm/apple: remove temporary external bring-up diagnostics Drop temporary protocol capture counters, PHY lock chatter, the global Type-C trace addition and unused link-rate override. Preserve standard tracepoints, failure diagnostics and the validated HBR3 behavior. Register the remaining focused tests and output-frame utility with MAINTAINERS. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 3 +++ drivers/gpu/drm/apple/afk.c | 17 +---------------- drivers/gpu/drm/apple/dptxep.c | 21 ++------------------- 3 files changed, 6 insertions(+), 35 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 3141982fc89e15..305f3e97c0d62a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2029,9 +2029,12 @@ F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-j700-crossbar.py +F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py +F: tools/diagnostics/test-apple-j700-opaque-scope.py F: tools/diagnostics/test-apple-j700-provider.py F: tools/diagnostics/test-apple-j700-tuning.py +F: tools/diagnostics/wayland-output-frame-test.py F: tools/testing/apple/ APPLETALK NETWORK LAYER diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index d10658013cc52c..9b93d384c5efd9 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -1005,15 +1005,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, subtype = le16_to_cpu(eshdr->type); trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); - if (ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - ep->experimental_rx_logs < 32) { - ep->experimental_rx_logs++; - dev_info(ep->dcp->dev, - "external AFK ep=%u channel=%u type=%u subtype=%u category=%u bytes=%zu\n", - ep->endpoint, channel, type, subtype, eshdr->category, - data_size); - } service = afk_epic_find_service(ep, channel); @@ -1184,13 +1175,7 @@ static void afk_receive_message_worker(struct work_struct *work_) work = container_of(work_, struct afk_receive_message_work, work); type = FIELD_GET(RBEP_TYPE, work->message); - if (work->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - (work->ep->endpoint == DPTX_ENDPOINT || - work->ep->endpoint == DPTX_ENDPOINT_V2) && type != RBEP_RECV) - dev_info_ratelimited(work->ep->dcp->dev, - "external AFK control ep=%#x type=%#x\n", - work->ep->endpoint, type); + switch (type) { case RBEP_INIT_ACK: break; diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 3864b37102b5cc..89437e69016e3a 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -24,12 +23,6 @@ #include "dptx-j700-training.h" -/* Diagnostic ceiling for the experimental physical J700 output only. */ -static unsigned int j700_dp_max_link_rate = LINK_RATE_HBR3; -module_param(j700_dp_max_link_rate, uint, 0600); -MODULE_PARM_DESC(j700_dp_max_link_rate, - "J700 external DP ceiling: 6=RBR, 10=HBR, 20=HBR2, 30=HBR3"); - struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -372,22 +365,12 @@ static int dptxport_call_get_max_link_rate(struct apple_epic_service *service, void *reply_, size_t reply_size) { struct dptxport_apcall_link_rate *reply = reply_; - struct apple_dcp *dcp = service->ep->dcp; - u32 ceiling = READ_ONCE(j700_dp_max_link_rate); if (reply_size < sizeof(*reply)) return -EINVAL; reply->retcode = cpu_to_le32(0); - if (service->cookie && ((struct dptx_port *)service->cookie)->atcphy && - dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - of_machine_is_compatible("apple,j700") && - (ceiling == LINK_RATE_RBR || ceiling == LINK_RATE_HBR || - ceiling == LINK_RATE_HBR2 || ceiling == LINK_RATE_HBR3)) - reply->link_rate = cpu_to_le32(ceiling); - else - reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); + reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); return 0; } @@ -938,7 +921,7 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, return; } port = 0; - dev_info(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); + dev_dbg(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); } switch (unit) { From e3ccadab7a4bdf14b6720765bffa0eca2b6a86ca Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:51:31 +1100 Subject: [PATCH 420/631] drm/apple: close external admission and recovery races Make explicitly enabled external providers participate in component matching before readiness, then validate prepared state after component binding. Reject unpublished or retired DPTX services and preserve stable ownership rather than reusing a published service generation. Retain failed J700 PHY transition state so SAFE retries cleanup, and reject normal training while that state is dirty. Add fault, admission/lifetime and provider-order regressions. A published DPTX service loss remains fail-closed until recovery; these changes do not claim physical hotplug qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 2 + drivers/gpu/drm/apple/afk-service-test.c | 99 ++++++++++++++++++++++++ drivers/gpu/drm/apple/afk.c | 27 +++++-- drivers/gpu/drm/apple/afk.h | 32 +++++++- drivers/gpu/drm/apple/apple_drv.c | 17 +++- drivers/gpu/drm/apple/dcp.c | 27 ++++--- drivers/gpu/drm/apple/dptx-service.h | 63 +++++++++++++++ drivers/gpu/drm/apple/dptxep.c | 29 ++++--- drivers/phy/apple/atc-j700-dp.h | 26 +++++-- drivers/phy/apple/atc.c | 26 ++++++- 10 files changed, 312 insertions(+), 36 deletions(-) create mode 100644 drivers/gpu/drm/apple/dptx-service.h diff --git a/MAINTAINERS b/MAINTAINERS index 305f3e97c0d62a..a48187d52f95fb 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2028,6 +2028,8 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-atc-unwind.py +F: tools/diagnostics/test-apple-dcp-readiness.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index 4a5f07fa9b15f2..a86df389deb177 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -3,6 +3,7 @@ #include #include "afk.h" +#include "dptx-service.h" static const struct apple_epic_service_ops reusable_ops = { .name = "reusable", @@ -122,7 +123,105 @@ static void afk_service_unconverted_owner_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); } +static int afk_test_prepare(struct apple_epic_service *service, + const char *name, const char *class, s64 unit) +{ + return 0; +} + +static const struct apple_epic_service_ops prepared_ops = { + .name = "prepared", + .prepare = afk_test_prepare, + .teardown = dptx_service_unbind, +}; + +static void afk_service_admission_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 1)); + /* Neither callback dispatch nor command callers can observe preparation. */ + KUNIT_EXPECT_FALSE(test, afk_service_matches(service, 1)); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(service), NULL); + /* A rejected announcement or failed OPEN consumes no service slot. */ + afk_service_discard(service); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &prepared_ops, 2)); + /* Cookie-less services are valid: admission does not test cookie != NULL. */ + afk_service_publish(service); + KUNIT_EXPECT_TRUE(test, afk_service_matches(service, 2)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); + afk_service_put(service); +} + +static void dptx_service_identity_test(struct kunit *test) +{ + const char *name = "dcpdptx-port-epic"; + const char *class = "AppleDCPDPTXRemotePort"; + + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 3, true, 3, 1), 0); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 0, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, -1, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 3, true, 3, 2), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(NULL, class, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, NULL, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port("wrong", class, 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, "wrong", 3, true, 3, 1), -EINVAL); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 1, false, 3, 2), 1); + KUNIT_EXPECT_EQ(test, dptx_service_port(name, class, 2, false, 3, 2), -EINVAL); +} + +static void dptx_service_duplicate_close_test(struct kunit *test) +{ + struct apple_epic_service *first = test->priv; + struct apple_epic_service *next; + struct dptx_port port = {}; + + next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, next); + spin_lock_init(&next->lock); + init_completion(&port.enable_completion); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(first, NULL, &prepared_ops, 1)); + KUNIT_ASSERT_EQ(test, dptx_service_bind(first, &port, 3), 0); + /* An observer during preparation/OPEN sees no available port or owner. */ + KUNIT_EXPECT_FALSE(test, dptx_service_available(&port)); + KUNIT_EXPECT_PTR_EQ(test, port.service, NULL); + KUNIT_EXPECT_PTR_EQ(test, first->cookie, &port); + /* Failed OPEN does not publish anything and permits a fresh attempt. */ + afk_service_discard(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + KUNIT_EXPECT_PTR_EQ(test, port.service, NULL); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(first, NULL, &prepared_ops, 1)); + KUNIT_ASSERT_EQ(test, dptx_service_bind(first, &port, 3), 0); + afk_service_publish(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + dptx_service_publish(first); + KUNIT_EXPECT_TRUE(test, dptx_service_available(&port)); + KUNIT_EXPECT_PTR_EQ(test, port.service->cookie, &port); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(next, NULL, &prepared_ops, 2)); + KUNIT_EXPECT_EQ(test, dptx_service_bind(next, &port, 3), -EBUSY); + afk_service_discard(next); + KUNIT_EXPECT_TRUE(test, port.enabled); + KUNIT_EXPECT_PTR_EQ(test, port.service, first); + /* CLOSE invalidates a connected session without publishing a replacement. */ + port.connected = true; + afk_test_retire(first); + dptx_service_unbind(first); + KUNIT_EXPECT_FALSE(test, port.enabled); + KUNIT_EXPECT_FALSE(test, port.connected); + KUNIT_EXPECT_PTR_EQ(test, first->cookie, &port); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(next, NULL, &prepared_ops, 3)); + KUNIT_EXPECT_EQ(test, dptx_service_bind(next, &port, 3), -EBUSY); + afk_service_discard(next); + /* Old work between validate/connect/request still names the old session. */ + KUNIT_EXPECT_PTR_EQ(test, port.service, first); + KUNIT_EXPECT_PTR_EQ(test, afk_service_get(port.service), NULL); + KUNIT_EXPECT_FALSE(test, port.enabled); +} + static struct kunit_case afk_service_cases[] = { + KUNIT_CASE(afk_service_admission_test), + KUNIT_CASE(dptx_service_identity_test), + KUNIT_CASE(dptx_service_duplicate_close_test), KUNIT_CASE(afk_service_many_generations_test), KUNIT_CASE(afk_service_reader_pin_test), KUNIT_CASE(afk_service_pending_reply_test), diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9b93d384c5efd9..81fe461a101c31 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -445,6 +445,16 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } ep->num_channels = max(ep->num_channels, ch_idx + 1); + if (ops->prepare) { + int ret = ops->prepare(service, epic_name, epic_class, epic_unit); + + if (ret) { + dev_err(ep->dcp->dev, "AFK service %s admission failed: %d\n", + service_name, ret); + afk_service_discard(service); + goto free; + } + } if (ep->wire_v2) { int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); @@ -452,16 +462,23 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, if (ret) { unsigned long flags; - spin_lock_irqsave(&service->lock, flags); - service->enabled = false; - service->torndown = true; - spin_unlock_irqrestore(&service->lock, flags); + if (ops->prepare) { + afk_service_discard(service); + } else { + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); + } dev_err(ep->dcp->dev, "AFKV2 interface %u OPEN failed: %d\n", channel, ret); goto free; } } - ops->init(service, epic_name, epic_class, epic_unit); + if (ops->prepare) + afk_service_publish(service); + if (ops->init) + ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", ep->endpoint, service_name, channel); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 3a197fc1f360f4..8b99a6259df5b3 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -74,6 +74,12 @@ struct apple_epic_service_ops { /* All retained pointers must be pinned before enabling slot reuse. */ bool reusable; + /* + * Optional admission, before OPEN/publication. Must not retain service + * on failure; teardown must accept a failed/partial preparation. + */ + int (*prepare)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); int (*call)(struct apple_epic_service *service, u32 idx, @@ -157,7 +163,7 @@ afk_service_reinit(struct apple_epic_service *service, spin_lock_irqsave(&service->lock, flags); reusable = afk_service_reclaimable(service); if (reusable) { - service->enabled = true; + service->enabled = !ops->prepare; service->torndown = false; service->ops = ops; service->ep = ep; @@ -169,6 +175,30 @@ afk_service_reinit(struct apple_epic_service *service, return reusable; } +/* Only used before a prepared service has been published to callers. */ +static inline void afk_service_discard(struct apple_epic_service *service) +{ + unsigned long flags; + + if (service->ops->teardown) + service->ops->teardown(service); + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + service->cookie = NULL; + service->ops = NULL; + spin_unlock_irqrestore(&service->lock, flags); +} + +static inline void afk_service_publish(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = true; + spin_unlock_irqrestore(&service->lock, flags); +} + struct afk_ringbuffer_header { __le32 bufsz; u32 unk; diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 38a3dd901d6047..c06ac2727e81de 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -812,7 +812,7 @@ static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && (!of_device_is_compatible(np, "apple,t8140-dcpext") || - dcp_h17p_external_ready(np)); + dcp_h17p_external_enabled()); } static int apple_drm_init_dcp(struct device *dev) @@ -836,6 +836,13 @@ static int apple_drm_init_dcp(struct device *dev) of_device_is_compatible(np, "apple,t8140-dcpext") || of_property_present(np, "phys"); + /* component_bind_all() established the matched provider lifetime. */ + if (of_device_is_compatible(np, "apple,t8140-dcpext") && + !dcp_h17p_external_ready(np)) { + of_node_put(np); + return -ENODEV; + } + dcp[num_dcp] = of_find_device_by_node(np); if (!dcp[num_dcp]) continue; @@ -852,8 +859,14 @@ static int apple_drm_init_dcp(struct device *dev) ret = apple_probe_per_dcp(dev, &apple->drm, dcp[num_dcp], num_dcp, dcp_ext, &encoder[num_dcp], &crtc_mask[num_dcp]); - if (ret) + if (ret) { + if (of_device_is_compatible(np, "apple,t8140-dcpext")) { + put_device(&dcp[num_dcp]->dev); + of_node_put(np); + return ret; + } continue; + } num_dcp++; } diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0a62a44302ce06..a9e74f3af5237d 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -61,6 +61,7 @@ #include "dcpext_scanout.h" #include "dcp-internal.h" #include "dcp-external-bw.h" +#include "dptx-service.h" #include "iomfb.h" #include "ibootep.h" #include "parser.h" @@ -185,12 +186,14 @@ bool dcp_h17p_external_ready(struct device_node *np) pdev = of_find_device_by_node(np); if (!pdev) return false; - /* Failed and deferred probes must never become required components. */ - if (device_is_bound(&pdev->dev)) { - dcp = platform_get_drvdata(pdev); - ready = dcp && READ_ONCE(dcp->external_prepared) && - !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); - } + /* + * Called after component binding. component_add() may synchronously + * bind the master before the platform probe itself returns, so the + * driver core's bound flag is not a readiness condition here. + */ + dcp = platform_get_drvdata(pdev); + ready = dcp && READ_ONCE(dcp->external_prepared) && + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); put_device(&pdev->dev); return ready; } @@ -2180,6 +2183,7 @@ static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, return -ESHUTDOWN; } if (!dcp->dptxport[port].enabled) { + if (!dptx_service_available(&dcp->dptxport[port])) { dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; goto out_unlock; @@ -2736,7 +2740,7 @@ static void dcp_placeholder_edid_work(struct work_struct *work) mutex_lock(&dcp->hpd_mutex); generation = dcp->placeholder_generation; if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || - !dcp->dptxport[0].enabled || generation != dcp->typec_generation) + !dptx_service_available(&dcp->dptxport[0]) || generation != dcp->typec_generation) goto out_unlock; service = dcp->dptxport[0].service; @@ -2840,6 +2844,9 @@ static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) ret = dptxport_release_display(dcp->dptxport[port].service); if (ret) return ret; + mutex_lock(&dcp->hpd_mutex); + if (dptx_service_available(&dcp->dptxport[port]) && dcp->dptxport[port].connected) { + dptxport_release_display(dcp->dptxport[port].service); dcp->dptxport[port].connected = false; } /* @@ -2969,6 +2976,8 @@ int dcp_dptx_recover_irq(struct apple_dcp *dcp) /* Another delivered IRQ must confirm release before a failed attempt retries. */ return dcp_dptx_connect_session(dcp, 0, session, true, true); + if (dptx_service_available(&dcp->dptxport[port])) + dptxport_set_hpd(dcp->dptxport[port].service, false); out_unlock: mutex_unlock(&dcp->hpd_mutex); @@ -3150,7 +3159,7 @@ int dcp_start(struct platform_device *pdev) * Since dcp0 is not used at the moment let's avoid this * since it is possibly the cause for startup issues. */ - } else if (dcp->dptxport[0].enabled) { + } else if (dptx_service_available(&dcp->dptxport[0])) { bool connected; /* force disconnect on start - necessary if the display * is already up from m1n1 @@ -3493,7 +3502,7 @@ void dcp_poweroff(struct platform_device *pdev) if (dcp_is_typec_output(dcp)) { ret = 0; /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ - if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); if (ret) dev_warn(dcp->dev, diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h new file mode 100644 index 00000000000000..f806e3253999af --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DPTX_SERVICE_H +#define APPLE_DPTX_SERVICE_H + +#include "afk.h" +#include "dptxep.h" + +static inline int dptx_service_port(const char *name, const char *class, + s64 unit, bool wire_v2, u32 index, + u32 num_ports) +{ + if (!name || !class || strcmp(name, "dcpdptx-port-epic") || + strcmp(class, "AppleDCPDPTXRemotePort")) + return -EINVAL; + if (wire_v2) + return num_ports == 1 && unit == index ? 0 : -EINVAL; + return unit >= 0 && unit < 2 ? unit : -EINVAL; +} + +static inline bool dptx_service_available(struct dptx_port *port) +{ + /* Pairs with publication before readers use the service and its cookie. */ + return smp_load_acquire(&port->enabled); +} + +static inline int dptx_service_bind(struct apple_epic_service *service, + struct dptx_port *port, s64 unit) +{ + if (port->service) + return -EBUSY; + port->unit = unit; + /* No owner is published until OPEN and AFK publication succeed. */ + service->cookie = port; + return 0; +} + +static inline void dptx_service_publish(struct apple_epic_service *service) +{ + struct dptx_port *port = service->cookie; + + port->service = service; + port->connected = false; + /* Pair with dcp.c readers before they dereference the service cookie. */ + smp_store_release(&port->enabled, true); +} + +static inline void dptx_service_unbind(struct apple_epic_service *service) +{ + struct dptx_port *port = service->cookie; + + if (port && port->service == service) { + WRITE_ONCE(port->enabled, false); + WRITE_ONCE(port->connected, false); + reinit_completion(&port->enable_completion); + } + /* + * Keep the published owner and cookie valid for retained callers. Once + * this session closes, bind fails closed until reboot: + * an in-flight connect must never switch to a new firmware session. + */ +} + +#endif diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 89437e69016e3a..7c169b060f3939 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -18,6 +18,7 @@ #include "dcp.h" #include "dcp-hdmi.h" #include "dptxep.h" +#include "dptx-service.h" #include "parser.h" #include "trace.h" @@ -893,10 +894,12 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, } } -static void dptxport_init(struct apple_epic_service *service, const char *name, - const char *class, s64 unit) +static int dptxport_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; + int port = dptx_service_port(name, class, unit, service->ep->wire_v2, + dcp->index, dcp->hw.num_dptx_ports); s64 port = unit; bool v14_7 = dcp_fw_is_14_7(service->ep->dcp); @@ -913,17 +916,19 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, !class || strcmp(class, "AppleDCPDPTXRemotePort")) return; + if (port < 0) + return port; trace_dptxport_init(dcp, unit); - if (service->ep->wire_v2) { - /* The single external service reports its physical DCP unit. */ - if (dcp->hw.num_dptx_ports != 1 || unit != dcp->index) { - dev_err(dcp->dev, "Unexpected external DPTX unit %lld\n", unit); - return; - } - port = 0; - dev_dbg(dcp->dev, "DPTX physical unit %lld maps to logical port 0\n", unit); - } + return dptx_service_bind(service, &dcp->dptxport[port], unit); +} + +static void dptxport_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct dptx_port *port = service->cookie; + dptx_service_publish(service); + complete(&port->enable_completion); switch (unit) { case 0: case 1: @@ -973,6 +978,8 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", + .prepare = dptxport_prepare, + .teardown = dptx_service_unbind, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 11d7b320037483..6643b97d0f001e 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,11 +3,26 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Retry only the idle operations claimed by an incomplete route change. */ +static int j700_dp_idle_pending(struct apple_atcphy *p) +{ + int ret; + + for (unsigned int lane = 0; lane < 2; lane++) { + if (!(p->dp_idle_pmas & BIT(lane))) + continue; + ret = j700_pma_idle(p, lane); + if (ret) + return ret; + p->dp_idle_pmas &= ~BIT(lane); + } + return 0; +} + /* Preserve the upper routing state established by the platform firmware. */ static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) { u32 protocols = four_lanes ? 0x492 : (p->swap_lanes ? 0x252 : 0x489); - int ret; core_mask32(p, 0x60, 0xfff, protocols); if (four_lanes) { @@ -19,11 +34,9 @@ static int j700_dp_route(struct apple_atcphy *p, bool four_lanes) for (unsigned int lane = 0; lane < 2; lane++) { if (!four_lanes && lane != !p->swap_lanes) continue; - ret = j700_pma_idle(p, lane); - if (ret) - return ret; + p->dp_idle_pmas |= BIT(lane); } - return 0; + return j700_dp_idle_pending(p); } /* Register facts from the authorized J700 hardware audit. */ @@ -308,6 +321,9 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ dev_dbg(p->dev, "J700 DP configure acquired PHY lock\n"); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); + /* The last completed mode cannot describe a partially changed lane route. */ + if (p->mode_dirty) + return -EBUSY; if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) return -EINVAL; mode = atcphy_get_mode_config(p, p->mode); diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index d39a0df03ebcd1..ff170336a91675 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -678,6 +678,8 @@ struct atcphy_hw { * behind the T6030 HDMI port; it maps only its core window * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @aux_power_requested: AUX power-up began and needs matching shutdown + * @mode_dirty: J700 mode transition needs cleanup before it can be retried + * @dp_idle_pmas: J700 PMAs with an incomplete routing idle sequence * @dp_owned_pmas: J700 PMAs requiring main-link cleanup * @dp_spread_spectrum: J700 main-link spread-spectrum state * @dp_presets: Firmware-derived per-rate drive presets @@ -753,6 +755,8 @@ struct apple_atcphy { bool dp_tunables; bool dp_only; bool aux_power_requested; + bool mode_dirty; + u8 dp_idle_pmas; u8 dp_owned_pmas; bool dp_spread_spectrum; u32 dp_presets[5][16]; @@ -3161,20 +3165,32 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); - if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && mode != atcphy->mode) { + if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && + (mode != atcphy->mode || atcphy->mode_dirty)) { + atcphy->mode_dirty = true; ret = j700_dp_stop(atcphy); if (ret) return ret; } + if (atcphy->hw->has_t8140_dp && atcphy->mode_dirty) { + ret = j700_dp_idle_pending(atcphy); + if (ret) + return ret; + } if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && !atcphy->dp_tunables) return -EOPNOTSUPP; - if (atcphy->hw->has_t8140_dp && !atcphy_modes[mode].enable_dp_aux && + if (atcphy->hw->has_t8140_dp && + (!atcphy_modes[mode].enable_dp_aux || atcphy->mode_dirty) && atcphy->aux_power_requested) { + atcphy->mode_dirty = true; ret = j700_aux_disable(atcphy); if (ret) return ret; } + /* Keep failed transitions retryable even when the requested mode is unchanged. */ + if (atcphy->hw->has_t8140_dp) + atcphy->mode_dirty = true; /* the mode change below may power the PHY down: stop the tunnel clock first */ if (apple_atc_tunnel_is_t6030(atcphy)) { ret = atc_t8122_tunnel_stop(atcphy); @@ -3189,8 +3205,11 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) if (mode == APPLE_ATCPHY_MODE_OFF) { ret = atcphy_power_off(atcphy); + if (ret) + return ret; atcphy->mode = mode; - return ret; + atcphy->mode_dirty = false; + return 0; } ret = atcphy_power_on(atcphy); @@ -3274,6 +3293,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) } atcphy->mode = mode; + atcphy->mode_dirty = false; /* T8122: the AUX block needs the PHY out of reset and calibrated */ if (atcphy->hw->dp_t8122 && atcphy_modes[mode].enable_dp_aux) From a783ea9736222b07472d537cccf40340d1fb4f22 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:56:39 +1100 Subject: [PATCH 421/631] tests: publish standalone AFK admission regression harness Make the host-extracted admission/lifetime checks reproducible alongside the kernel KUnit suite, with repository-relative source selection and an optional source-root override. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index a48187d52f95fb..057533aa2b565b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2028,6 +2028,7 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py F: tools/diagnostics/test-apple-j700-crossbar.py From 97f5b766062aa5533c99a2312c9c9f9c4a22ed04 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 19:59:19 +1100 Subject: [PATCH 422/631] drm/apple: add bounded external stop qualification path Pause new presentation work while callbacks and control RPCs remain available; settle the active queue, serialize external lifecycle operations and retain resources on failure. Preserve first-start preparation and generation-aware CRTC recovery. OFF replies are captured but deliberately return EOPNOTSUPP until their contract is qualified; no successful production teardown or DMA retirement is claimed. Host tests exercise admission, drain, exact response bounds, timeout ownership and lifecycle ordering. A hypothetical qualified-OFF adapter exists only inside the host test. Astra reviewed this revision for one initialized DPMS-OFF capture followed by reboot, excluding unplug/suspend/restart qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/apple_drv.c | 3 + drivers/gpu/drm/apple/dcp-internal.h | 25 +++ drivers/gpu/drm/apple/dcp.c | 293 ++++++++++++++++++++++++++- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/iomfb.c | 64 +++++- drivers/gpu/drm/apple/iomfb_h17p.c | 99 +++++++++ 7 files changed, 480 insertions(+), 6 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 057533aa2b565b..de3bbcec2276be 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2031,6 +2031,7 @@ F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py +F: tools/diagnostics/test-apple-external-lifecycle.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index c06ac2727e81de..cd9fbda6895d8b 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -192,6 +192,9 @@ static void apple_crtc_atomic_enable(struct drm_crtc *crtc, (crtc_state->active_changed || dcp_crtc_needs_route_start(dcp))) { dcp_poweron(apple_crtc->dcp); crtc_state->color_mgmt_changed = true; + } else if (crtc_state->active && drm_atomic_crtc_needs_modeset(crtc_state)) { + /* A cable generation can change while the CRTC stays active. */ + dcp_external_recover(to_apple_crtc(crtc)->dcp); } if (crtc_state->active) dcp_crtc_atomic_modeset(crtc, state); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 444e636cfb26d8..2db7fb21f57618 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -49,6 +49,7 @@ struct iomfb_transaction { bool backlight_reserved; bool backlight_failed; bool brightness_only; + bool presentation; bool completed; }; struct apple_dcp_typec_port; @@ -215,6 +216,25 @@ struct apple_dcp { /* Protected by the H17P IOMFB receive mutex. */ u8 experimental_callback_logs; bool external_prepared; + /* Serializes external connection, power, and CRTC generation changes. */ + struct mutex external_lock; + u64 external_generation; + u64 external_crtc_generation; + /* iomfb.lock: epoch used by receive-side deferred stop requests. */ + u64 external_rx_generation; + u64 external_stop_generation; + bool external_cable; + bool external_running; + bool external_power_attempted; + /* iomfb.lock: replay the latest callback after activation succeeds. */ + bool external_hotplug_pending; + bool external_hotplug_connected; + bool external_stopped; + int external_error; + u64 external_power_reply; + bool external_power_reply_valid; + struct work_struct external_stop_work; + /* firmware version and compatible firmware version */ enum dcp_firmware_version fw_compat; @@ -567,6 +587,7 @@ struct apple_dcp { unsigned long deadline; unsigned int queued; bool stopped; + bool presentation_paused; bool backlight_queued; atomic_t opaque_x_state; struct iomfb_scanout_h17p *scanout; @@ -591,11 +612,15 @@ int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); +int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout); +void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause); +int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on); int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction); void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id); void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now); void dcp_handle_hotplug_actions(struct apple_dcp *dcp, unsigned int action); +bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected); int dcp_backlight_init(struct apple_dcp *dcp, struct device *dev); void dcp_backlight_stop(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index a9e74f3af5237d..d626a29f942d77 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -343,9 +343,12 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, * the crossbar nor restart the tunnel pixel clock behind our back. * Ported from aurora-silicon/linux#8. */ - scoped_guard(mutex, &dcp->tb_lock) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); + ret = dcp_dptx_disconnect(dcp, 0); + if (ret) + return ret; + + if (dcp->fixed_route_selected) { + ret = mux_control_deselect(dcp->xbar); if (ret) return ret; route->xbar_up = false; @@ -407,6 +410,11 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) bool was_tunnel = route->tunnel; int ret = 0; + guard(mutex)(&dcp->external_lock); + if (dcp_is_external_h17p(dcp) && + (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) + return dcp->external_error ?: -EBUSY; + /* * Under tb_lock so a DCP link (re)configuration can neither select the * crossbar nor restart the tunnel pixel clock behind our back. @@ -1471,6 +1479,15 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) { struct dcp_session_ref *session; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; + port->owner = NULL; + } + if (port->secondary_owner == route) { + struct apple_dcp *dcp = route->dcp; if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) return 0; @@ -1478,6 +1495,19 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) if (!session) return -ENOMEM; + if (port->hpd || dcp->typec_cable_connected) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; + port->secondary_owner = NULL; + } + port->hpd = !!(port->owner || port->secondary_owner); + list_del(&route->port_link); + if (list_empty(&port->routes)) { + list_del(&port->link); + of_node_put(port->connector_np); + kfree(port); + } /* Keep the DMA device, domain usage and driver code valid until reset. */ session->dev = get_device(dcp->dev); session->group = iommu_group_get(dcp->dev); @@ -2695,6 +2725,186 @@ static void dcp_placeholder_edid_work(struct work_struct *work) mutex_unlock(&dcp->hpd_mutex); } +static void dcp_external_report_disconnected(struct apple_dcp *dcp) +{ + unsigned int action; + + guard(mutex)(&dcp->iomfb.lock); + action = dcp_mode_hotplug(&dcp->mode_state, false, + dcp->connector ? &dcp->connector->connected : NULL); + /* The existing worker delivers DRM notifications outside our locks. */ + dcp_handle_hotplug_actions(dcp, action); +} + +static void dcp_external_publish_generation(struct apple_dcp *dcp) +{ + lockdep_assert_held(&dcp->external_lock); + guard(mutex)(&dcp->iomfb.lock); + dcp->external_rx_generation = dcp->external_generation; +} + +/* external_lock is never taken by the serialized IOMFB receive callbacks. */ +static int dcp_external_link_locked(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + if (!dcp->external_prepared) + return -EAGAIN; + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable) + return -ENODEV; + ret = dcp_dptx_connect_raw(dcp, 0); + if (ret) + dcp->external_error = ret; + return ret; +} + +static int dcp_external_start_locked(struct apple_dcp *dcp) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + ret = dcp_external_link_locked(dcp); + if (ret || dcp->external_running) + return ret; + iomfb_pause_presentation(dcp, true); + /* Even a timed-out first activation can have changed firmware state. */ + dcp->external_power_attempted = true; + ret = iomfb_external_power_h17p(dcp, true); + if (ret) { + dcp->external_error = ret; + return ret; + } + dcp->external_running = true; + dcp->external_stopped = false; + WRITE_ONCE(dcp->pipe_enabled_h17p, true); + WRITE_ONCE(dcp->typec_crtc_off, false); + iomfb_pause_presentation(dcp, false); + return 0; +} + +static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +{ + if (!dcp_is_external_h17p(dcp)) + return dcp_dptx_connect_raw(dcp, port); + guard(mutex)(&dcp->external_lock); + return dcp_external_link_locked(dcp); +} + +/* The generation token belongs to the CRTC that was actually enabled. */ +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) +{ + int ret; + + lockdep_assert_held(&dcp->external_lock); + if (generation != dcp->external_generation) + return 0; + if (dcp->external_error) + return dcp->external_error; + if (dcp->external_stopped) + return 0; + iomfb_pause_presentation(dcp, true); + dcp->external_running = false; + /* Route setup can call this before IOMFB or its shared memory exists. */ + if (!dcp->external_power_attempted) + goto release_link; + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (!ret) + ret = iomfb_external_power_h17p(dcp, false); + if (!ret) + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (ret) + goto failed; + /* This is a transport acknowledgment, not permission to free scanout. */ + WRITE_ONCE(dcp->pipe_enabled_h17p, false); +release_link: + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->dptxport[0].connected) { + if (!dptx_service_available(&dcp->dptxport[0])) { + ret = -ENODEV; + goto failed; + } + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (!ret) + ret = dptxport_release_display(dcp->dptxport[0].service); + if (ret) + goto failed; + dcp->dptxport[0].connected = false; + } + } + dcp->external_stopped = true; + WRITE_ONCE(dcp->typec_crtc_off, true); + if (dcp->avep) + av_service_disconnect(dcp); + return 0; +failed: + /* A retry could race a late reply. Keep this session blocked and owned. */ + dcp->external_error = ret; + return ret; +} + +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) +{ + int ret; + + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_stop_locked(dcp, dcp->external_generation); + if (physical) { + dcp->external_cable = false; + dcp->external_generation++; + dcp_external_publish_generation(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } + /* A missing cable is reported even when its hardware remains owned. */ + dcp_external_report_disconnected(dcp); + } + return ret; +} + +static void dcp_external_stop_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_stop_work); + + u64 generation; + + scoped_guard(mutex, &dcp->iomfb.lock) + generation = dcp->external_stop_generation; + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + if (generation == dcp->external_generation) { + dcp_external_stop_locked(dcp, generation); + dcp_external_report_disconnected(dcp); + } +} + +bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) +{ + if (!dcp_is_external_h17p(dcp)) + return false; + /* No lifecycle or HPD mutex may be acquired from the RX callback. */ + lockdep_assert_held(&dcp->iomfb.lock); + if (READ_ONCE(dcp->quiescing)) + return true; + if (dcp->iomfb.presentation_paused) { + dcp->external_hotplug_pending = true; + dcp->external_hotplug_connected = connected; + return true; + } + if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { + dcp->external_stop_generation = dcp->external_rx_generation; + schedule_work(&dcp->external_stop_work); + return true; + } + return READ_ONCE(dcp->iomfb.presentation_paused); +} + static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) { const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); @@ -2834,6 +3044,10 @@ static void disconnected_hpd_event(struct apple_connector *con) static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) { int ret; + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, false); + + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); lockdep_assert_held(&dcp->hpd_mutex); if (dcp->external) { @@ -2880,6 +3094,23 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); + guard(mutex)(&dcp->external_lock); + if (!dcp->external_cable) + dcp->external_generation++; + dcp->external_cable = true; + dcp_external_publish_generation(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_generation++; + dcp->placeholder_retried = false; + } + return dcp_external_link_locked(dcp); + } + if (dcp_is_typec_output(dcp)) { cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->hpd_mutex); @@ -2906,6 +3137,9 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, true); + if (dcp_is_typec_output(dcp)) { scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); @@ -3349,6 +3583,19 @@ void dcp_poweron(struct platform_device *pdev) unsigned long remaining; int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_start_locked(dcp); + if (!ret) + dcp->external_crtc_generation = dcp->external_generation; + else + dev_err(dcp->dev, "external display activation failed: %d\n", ret); + return; + } + if (dcp_uses_soft_dpms(dcp)) { if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && @@ -3435,12 +3682,31 @@ void dcp_poweron(struct platform_device *pdev) av_service_connect(dcp); } +/* Same-CRTC helper dependencies order this token refresh before later disables. */ +void dcp_external_recover(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp_is_external_h17p(dcp)) + dcp_poweron(pdev); +} + void dcp_poweroff(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); bool display_gone = false; int ret; + if (dcp_is_external_h17p(dcp)) { + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + ret = dcp_external_stop_locked(dcp, dcp->external_crtc_generation); + if (ret) + dev_err(dcp->dev, "external display stop failed: %d\n", ret); + return; + } + cancel_delayed_work(&dcp->swap_watchdog_wq); /* Internal H17P DPMS is a brightness present, never a pipe stop. */ @@ -4176,6 +4442,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); } static void dcp_enable_typec_work(struct apple_dcp *dcp) @@ -4496,7 +4763,9 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) iomfb_v14_7_unbind(dcp); if (dcp->retain_dma) { /* No firmware stop is acknowledged: detach host users, never DMA. */ - WRITE_ONCE(dcp->quiescing, true); + scoped_guard(mutex, &dcp->iomfb.lock) + WRITE_ONCE(dcp->quiescing, true); + cancel_work_sync(&dcp->external_stop_work); if (dcp->crtc && !dcp->drm_retained) { drm_dev_get(dcp->crtc->base.dev); dcp->drm_retained = true; @@ -4735,6 +5004,8 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) mutex_init(&dcp->tb_lock); dcp_fabric_init(dcp); mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->external_lock); + INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->dcpavserv.lock); @@ -4747,6 +5018,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) experimental_h17p_external_bw_request)) return dev_err_probe(dev, -EINVAL, "External bandwidth offsets are unset or invalid\n"); + iomfb_pause_presentation(dcp, true); dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; dcp->hw.firmware_request = experimental_h17p_external_bw_request; } @@ -5364,6 +5636,19 @@ static int dcp_platform_suspend(struct device *dev) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + if (dcp_is_external_h17p(dcp)) { + int ret; + + dcp_disable_typec_work(dcp, false); + ret = dcp_external_disconnect(dcp, false); + if (ret) { + dcp_enable_typec_work(dcp); + return ret; + } + device_set_wakeup_path(dev); + return 0; + } + dcp_drain_for_sleep(dcp); /* * The Type-C route reports cable removal through diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 47cbfbbeedd6a7..aced157269e1c8 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -46,6 +46,7 @@ struct apple_encoder { void dcp_poweroff(struct platform_device *pdev); int dcp_typec_prepare_route(struct apple_dcp *dcp); void dcp_poweron(struct platform_device *pdev); +void dcp_external_recover(struct platform_device *pdev); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); int dcp_get_connector_type(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 98fa680b4a31c0..e5e16c4b839cc3 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -430,6 +430,45 @@ bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) } } +/* Admission is independent of transport: replies and control RPCs remain live. */ +void iomfb_pause_presentation(struct apple_dcp *dcp, bool pause) +{ + guard(mutex)(&dcp->iomfb.lock); + dcp->iomfb.presentation_paused = pause; + if (!pause && dcp->external_hotplug_pending) { + unsigned int action; + + dcp->external_hotplug_pending = false; + action = dcp_mode_hotplug(&dcp->mode_state, dcp->external_hotplug_connected, + dcp->connector ? &dcp->connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); + } +} + +/* A dead transport is not proof that DMA or a pending present has stopped. */ +int iomfb_queue_settle(struct apple_dcp *dcp, unsigned long timeout) +{ + unsigned long deadline = jiffies + timeout; + bool idle; + + for (;;) { + mutex_lock(&dcp->iomfb.lock); + if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { + mutex_unlock(&dcp->iomfb.lock); + return -EIO; + } + idle = !dcp->iomfb.active && !dcp->iomfb.queued && + list_empty(&dcp->iomfb.pending) && + !dcp->present_state_h17p.pending && iomfb_channels_idle(dcp); + mutex_unlock(&dcp->iomfb.lock); + if (idle) + return 0; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(2000, 4000); + } +} + int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) { int ret = 0; @@ -437,6 +476,8 @@ int iomfb_queue(struct apple_dcp *dcp, struct iomfb_transaction *transaction) mutex_lock(&dcp->iomfb.lock); if (READ_ONCE(dcp->crashed) || dcp->iomfb.stopped) { ret = -EIO; + } else if (transaction->presentation && dcp->iomfb.presentation_paused) { + ret = -EAGAIN; } else if (transaction->brightness_only && dcp->iomfb.backlight_queued) { ret = -EALREADY; } else if (dcp->iomfb.queued >= 32) { @@ -506,7 +547,7 @@ void iomfb_queue_crc_h17p(struct apple_dcp *dcp, u32 swap_id) lockdep_assert_held(&dcp->iomfb.lock); if (!READ_ONCE(dcp->crc_enabled) || READ_ONCE(dcp->crashed) || - dcp->iomfb.stopped || dcp->iomfb.queued >= 32) + dcp->iomfb.stopped || dcp->iomfb.presentation_paused || dcp->iomfb.queued >= 32) return; crc = kzalloc_obj(*crc); if (!crc) @@ -1398,6 +1439,13 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, int ret = -EIO; bool modeset; + /* Disconnect cannot pass the admission check before these RPCs finish. */ + guard(mutex)(&dcp->external_lock); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (!dcp->external_running || dcp->external_error || READ_ONCE(dcp->quiescing) || + READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) + return -EPIPE; + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); if (!crtc_state) return 0; @@ -1734,6 +1782,7 @@ static void iomfb_backlight_kick(struct apple_dcp *dcp) transaction->start = iomfb_backlight_start; transaction->release = iomfb_backlight_release; transaction->brightness_only = true; + transaction->presentation = true; if (iomfb_queue(dcp, transaction)) kfree(transaction); } @@ -1794,18 +1843,29 @@ static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state) { struct iomfb_atomic_transaction *atomic; + int ret; + if (READ_ONCE(dcp->iomfb.presentation_paused)) { + schedule_work(&dcp->vblank_wq); + return; + } atomic = kzalloc_obj(*atomic); if (!atomic) goto failed; atomic->transaction.start = iomfb_atomic_start; + atomic->transaction.presentation = true; atomic->transaction.release = iomfb_atomic_release; atomic->state = drm_atomic_state_get(state); atomic->crtc = crtc; atomic->dcp = dcp; - if (!iomfb_queue(dcp, &atomic->transaction)) + ret = iomfb_queue(dcp, &atomic->transaction); + if (!ret) return; iomfb_atomic_release(&atomic->transaction); + if (ret == -EAGAIN) { + schedule_work(&dcp->vblank_wq); + return; + } failed: WRITE_ONCE(dcp->crashed, true); schedule_work(&dcp->vblank_wq); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 49fcabaeb7c66e..b3ae896426e251 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -718,6 +718,105 @@ bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, return true; } +static_assert(sizeof(struct dcp_set_power_state_resp) == 8); + +struct dcp_external_power_cookie { + struct dcp_wait_cookie wait; + bool on; +}; + +/* A473's envelope is known; no OFF reply has been qualified as a stop. */ +static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_external_power_cookie *cookie = data; + const struct dcp_packet_header *header; + int ret = -EPROTO; + + if (out) { + header = out - sizeof(struct dcp_set_power_state_req_h17p) - sizeof(*header); + if (header->in_len == sizeof(struct dcp_set_power_state_req_h17p) && + header->out_len == sizeof(struct dcp_set_power_state_resp)) { + u64 reply = get_unaligned_le64(out); + + WRITE_ONCE(dcp->external_power_reply, reply); + WRITE_ONCE(dcp->external_power_reply_valid, true); + dev_info(dcp->dev, "A473 power=%u reply=%#llx%s\n", + cookie->on, reply, cookie->on ? "" : " (stop unqualified)"); + ret = cookie->on ? (reply == 1 ? 0 : -EREMOTEIO) : -EOPNOTSUPP; + } + } + WRITE_ONCE(cookie->wait.status, ret); + complete(&cookie->wait.done); + kref_put(&cookie->wait.refcount, release_wait_cookie); +} + +static bool iomfb_external_reply_size(void *out, u32 in_len, u32 out_len) +{ + const struct dcp_packet_header *header; + + if (!out) + return false; + header = out - in_len - sizeof(*header); + return header->in_len == in_len && header->out_len == out_len; +} + +static void iomfb_external_parameter_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_set_power_state_req_h17p request = { .unklong = 1 }; + + if (!iomfb_external_reply_size(out, sizeof(struct dcp_set_parameter_dcp), sizeof(u32))) { + iomfb_external_power_complete(dcp, NULL, data); + return; + } + dcp_set_power_state(dcp, true, &request, iomfb_external_power_complete, data); +} + +static void iomfb_external_device_complete(struct apple_dcp *dcp, void *out, void *data) +{ + struct dcp_set_parameter_dcp parameter = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .count = 3, + }; + + if (!iomfb_external_reply_size(out, sizeof(u32), sizeof(u32))) { + iomfb_external_power_complete(dcp, NULL, data); + return; + } + /* Preserve the existing external power-on preparation and payload semantics. */ + dcp_set_parameter_dcp(dcp, false, ¶meter, iomfb_external_parameter_complete, data); +} + +int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on) +{ + struct dcp_set_power_state_req_h17p request = { .unklong = on }; + struct dcp_external_power_cookie *cookie; + int ret; + + if (READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.stopped)) + return -EIO; + cookie = kzalloc_obj(*cookie); + if (!cookie) + return -ENOMEM; + cookie->on = on; + WRITE_ONCE(dcp->external_power_reply_valid, false); + cookie->wait.status = -ETIMEDOUT; + init_completion(&cookie->wait.done); + kref_init(&cookie->wait.refcount); + kref_get(&cookie->wait.refcount); + if (on) { + u32 handle = 2; + + iomfb_opaque_x_reset_h17p(dcp); + dcp_set_display_device(dcp, false, &handle, iomfb_external_device_complete, cookie); + } else { + dcp_set_power_state(dcp, true, &request, iomfb_external_power_complete, cookie); + } + ret = wait_for_completion_timeout(&cookie->wait.done, msecs_to_jiffies(500)); + ret = ret ? (int)READ_ONCE(cookie->wait.status) : -ETIMEDOUT; + kref_put(&cookie->wait.refcount, release_wait_cookie); + return ret; +} + void iomfb_start_h17p(struct apple_dcp *dcp) { WRITE_ONCE(dcp->pipe_enabled_h17p, false); From ec7942f2b4e5589bfade5ddc0a4657318187020b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:13:34 +1100 Subject: [PATCH 423/631] drm/apple: retry initial external DPTX service discovery Distinguish a service not yet published from terminal session closure before issuing firmware commands. Keep initial publication retryable, preserve sticky failures after real connection commands, and schedule asynchronous reconnect when the service arrives late. The R28 hardware trial exposed the startup ordering regression. Extracted production tests cover initial and partial publication, closure, late arrival, and retained firmware failures. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/dcp.c | 15 ++++++++++++--- drivers/gpu/drm/apple/dptx-service.h | 4 +++- drivers/gpu/drm/apple/dptxep.c | 8 ++++++++ drivers/gpu/drm/apple/dptxep.h | 2 ++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index de3bbcec2276be..7f464107bc0279 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2031,6 +2031,7 @@ F: tools/diagnostics/apple-j700-dp-test.c F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py +F: tools/diagnostics/test-apple-dptx-readiness.py F: tools/diagnostics/test-apple-external-lifecycle.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d626a29f942d77..27c5554f718b81 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2178,6 +2178,7 @@ void dcp_queue_typec_reconnect(struct apple_dcp *dcp, unsigned long delay) static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, struct dcp_fabric_session session, bool cable, bool recovery) +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_service) { /* T8122 dcpext: HPD only after the link wait, see dcp_ext_dptx_607() */ bool late_hpd = dcp_ext_dptx_607(dcp); @@ -2186,6 +2187,8 @@ static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, unsigned long timeout; int ret = 0; + if (waiting_service) + *waiting_service = false; if (!dcp->phy) { dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); return -ENODEV; @@ -2216,6 +2219,11 @@ static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, if (!dptx_service_available(&dcp->dptxport[port])) { dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); ret = -ENODEV; + /* No firmware request has been made; the first publication can retry. */ + if (waiting_service && !READ_ONCE(dcp->dptxport[port].closed)) { + *waiting_service = true; + ret = -EAGAIN; + } goto out_unlock; } @@ -2746,6 +2754,7 @@ static void dcp_external_publish_generation(struct apple_dcp *dcp) /* external_lock is never taken by the serialized IOMFB receive callbacks. */ static int dcp_external_link_locked(struct apple_dcp *dcp) { + bool waiting_service; int ret; lockdep_assert_held(&dcp->external_lock); @@ -2755,8 +2764,8 @@ static int dcp_external_link_locked(struct apple_dcp *dcp) return dcp->external_error; if (!dcp->external_cable) return -ENODEV; - ret = dcp_dptx_connect_raw(dcp, 0); - if (ret) + ret = dcp_dptx_connect_raw(dcp, 0, &waiting_service); + if (ret && !waiting_service) dcp->external_error = ret; return ret; } @@ -2788,7 +2797,7 @@ static int dcp_external_start_locked(struct apple_dcp *dcp) static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { if (!dcp_is_external_h17p(dcp)) - return dcp_dptx_connect_raw(dcp, port); + return dcp_dptx_connect_raw(dcp, port, NULL); guard(mutex)(&dcp->external_lock); return dcp_external_link_locked(dcp); } diff --git a/drivers/gpu/drm/apple/dptx-service.h b/drivers/gpu/drm/apple/dptx-service.h index f806e3253999af..3d990a9ca1ea11 100644 --- a/drivers/gpu/drm/apple/dptx-service.h +++ b/drivers/gpu/drm/apple/dptx-service.h @@ -49,7 +49,9 @@ static inline void dptx_service_unbind(struct apple_epic_service *service) struct dptx_port *port = service->cookie; if (port && port->service == service) { - WRITE_ONCE(port->enabled, false); + WRITE_ONCE(port->closed, true); + /* Make terminal state visible to readers of availability. */ + smp_store_release(&port->enabled, false); WRITE_ONCE(port->connected, false); reinit_completion(&port->enable_completion); } diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 7c169b060f3939..ff3ab11ebbccd7 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -951,6 +951,14 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", unit); } + /* Publication may follow the startup wait or the bounded retry window. */ + if (service->ep->dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + READ_ONCE(service->ep->dcp->typec_cable_connected) && + !READ_ONCE(service->ep->dcp->quiescing) && + !READ_ONCE(service->ep->dcp->external_error)) + mod_delayed_work(system_freezable_wq, + &service->ep->dcp->typec_reconnect_wq, 0); } static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 5072a545ce070b..db9cedaa48f4a6 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -50,6 +50,8 @@ struct apple_epic_service; struct dptx_port { bool enabled, connected; + /* A published session closed; unlike initial discovery, this is terminal. */ + bool closed; struct completion enable_completion; struct completion linkcfg_completion; u32 unit; From a9070a1920af5927c2ccc794f7c53b3c18cb9e3d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:14:49 +1100 Subject: [PATCH 424/631] drm/apple: select compatible SDR color modes for external H17P Choose advertised full-range 8/10-bit SDR RGB modes for the experimental external H17P profile. Prefer sRGB or BT.709 over default RGB, then use firmware scores. Reject timings that only offer incompatible color modes instead of selecting BT.2020 for native SDR scanout. Leave internal and legacy policy unchanged. The working J700 selected BT.2020 RGB while displaying an SDR desktop. Production-parser tests cover 65 valid, malformed, legacy and high-refresh cases. Hardware color appearance still requires qualification. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/parser.c | 43 ++++++++++++++++++++++++++++++---- drivers/gpu/drm/apple/parser.h | 1 + 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 7f464107bc0279..2f38d30821f097 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2033,6 +2033,7 @@ F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py F: tools/diagnostics/test-apple-dptx-readiness.py F: tools/diagnostics/test-apple-external-lifecycle.py +F: tools/diagnostics/test-apple-external-sdr.py F: tools/diagnostics/test-apple-j700-crossbar.py F: tools/diagnostics/test-apple-j700-d134.py F: tools/diagnostics/test-apple-j700-dp.py diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 73cd764226fbed..d103a97ddb6977 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -446,10 +446,32 @@ static int fill_color_mode(struct dcp_color_mode *color, return 0; } +/* External SDR scanout must not select a wide-gamut mode by score alone. */ +static unsigned int external_sdr_rank(const struct color_mode *color) +{ + if (color->id < 0 || color->id > U32_MAX || + color->eotf != DCP_EOTF_SDR_GAMMA || + color->pixel_encoding != DCP_COLOR_FORMAT_RGB || + color->dynamic_range != DCP_COLOR_YCBCR_RANGE_FULL || + (color->depth != 8 && color->depth != 10)) + return 0; + + switch (color->colorimetry) { + case DCP_COLORIMETRY_SRGB: + case DCP_COLORIMETRY_BT709: + return 2; + case DCP_COLORIMETRY_RGB: + return 1; + default: + return 0; + } +} + static int parse_color_modes(struct dcp_parse_ctx *handle, - struct dcp_display_mode *out) + struct dcp_display_mode *out, bool external_sdr) { struct iterator outer_it; + unsigned int best_sdr_rank = 0; int ret = 0; out->sdr_444.score = -1; out->sdr_rgb.score = -1; @@ -513,6 +535,18 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.eotf, cmode.dynamic_range, cmode.pixel_encoding); + if (external_sdr) { + unsigned int rank = external_sdr_rank(&cmode); + + if (!rank || rank < best_sdr_rank) + continue; + if (rank > best_sdr_rank) + out->sdr_rgb.score = -1; + fill_color_mode(&out->sdr_rgb, &cmode); + best_sdr_rank = rank; + continue; + } + if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && cmode.depth <= 10) @@ -544,7 +578,7 @@ static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, - int height_mm, unsigned notch_height, bool internal) + int height_mm, unsigned notch_height, bool internal, bool external_sdr) { int ret = 0; struct iterator it; @@ -584,7 +618,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out); + ret = parse_color_modes(it.handle, out, external_sdr); else if (!strcmp(key, "ID")) ret = parse_int(it.handle, &id); else if (!strcmp(key, "IsVirtual")) @@ -695,7 +729,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, - bool internal) + bool internal, bool external_sdr) { struct iterator it; int ret; @@ -730,6 +764,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, mode = &modes[*count]; ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, + notch_height, internal, external_sdr); notch_height, internal); *handle = next; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 0dfca73de4b4ef..88685e75129fbf 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -107,6 +107,7 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, + bool internal, bool external_sdr); bool internal); /* Caller holds modes_lock while checking connector ownership. */ bool dcp_modes_for_connector(struct apple_dcp *dcp, From e6a83fee623abe683671efb8e0734a7f3e1029b2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:18:06 +1100 Subject: [PATCH 425/631] drm/apple: retrieve external H17P EDID through bounded AFKV2 inline calls Use the external DPAV endpoint's inline framing with independent request length and response capacity. Validate actual service and EDID lengths before reading them, retain timeout ownership, and queue a connector reprobe when the service appears late. Preserve the legacy padded OOB path and EDID reader, and keep internal H17P excluded. Host tests exercise the production sender, service envelope, EDID reader, ring admission and late notification with malformed replies and timeout cases. Hardware identity and color qualification remain pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 128 ++++++++++++++---- drivers/gpu/drm/apple/afk.h | 55 ++++---- drivers/gpu/drm/apple/afkv2-codec.h | 52 +++++++ drivers/gpu/drm/apple/epic/dpavserv-edid.h | 48 +++++++ drivers/gpu/drm/apple/epic/dpavservep.c | 83 +++++++++++- drivers/gpu/drm/apple/iomfb.c | 4 + .../drivers/gpu/apple-afkv2/Makefile | 2 +- .../selftests/drivers/gpu/apple-afkv2/README | 16 +++ .../drivers/gpu/apple-afkv2/extract-edid.py | 79 +++++++++++ .../drivers/gpu/apple-afkv2/extract-notify.py | 50 +++++++ .../drivers/gpu/apple-afkv2/extract-ring.py | 50 +++++++ .../drivers/gpu/apple-afkv2/extract-send.py | 118 ++++++++++++++++ .../gpu/apple-afkv2/extract-service.py | 86 ++++++++++++ .../apple-afkv2/host-include/linux/errno.h | 5 + .../drivers/gpu/apple-afkv2/inline-edid.c | 113 ++++++++++++++++ .../gpu/apple-afkv2/run-afkv2-tests.sh | 42 ++++++ 16 files changed, 872 insertions(+), 59 deletions(-) create mode 100644 drivers/gpu/drm/apple/epic/dpavserv-edid.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 81fe461a101c31..ffc53901eda9ba 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -932,15 +932,23 @@ static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, spin_lock_irqsave(&service->lock, flags); ret = -EPROTO; if (slot < MAX_PENDING_CMDS && test_bit(slot, service->cmd_map) && - !service->cmds[slot].done && response.id == service->cmds[slot].tag) - ret = afkv2_check_response(&frame, &response, service->channel, - service->cmds[slot].type, service->cmds[slot].tag, - service->cmds[slot].rxbuf_dma, - service->cmds[slot].rxlen, - service->cmds[slot].txbuf_dma, - service->cmds[slot].txlen); - if (!ret) - service->cmds[slot].reply_length = response.value ? 0 : response.rx_size; + !service->cmds[slot].done && response.id == service->cmds[slot].tag) { + struct epic_cmd_info *cmd = &service->cmds[slot]; + + if (ep->inline_v2) + ret = afkv2_check_inline_response(&frame, &response, service->channel, + cmd->type, cmd->tag, cmd->rxlen); + else + ret = afkv2_check_response(&frame, &response, service->channel, + cmd->type, cmd->tag, cmd->rxbuf_dma, + cmd->rxlen, cmd->txbuf_dma, cmd->txlen); + if (!ret) { + cmd->reply_length = response.value ? 0 : + (ep->inline_v2 ? response.inline_size : response.rx_size); + if (ep->inline_v2 && !response.value && cmd->reply_length) + memcpy(cmd->rxbuf, response.inline_data, cmd->reply_length); + } + } spin_unlock_irqrestore(&service->lock, flags); if (!ret) { legacy.retcode = cpu_to_le32(response.value); @@ -1315,20 +1323,29 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, return -EOPNOTSUPP; v2_flags = 1; } else if (ecat == EPIC_CAT_COMMAND) { - const struct epic_cmd *cmd = payload; - - if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || - payload_len != sizeof(*cmd)) - return -EINVAL; - ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, - le64_to_cpu(cmd->rxbuf), - le32_to_cpu(cmd->rxlen), - le64_to_cpu(cmd->txbuf), - le32_to_cpu(cmd->txlen)); - if (ret) - return ret; - payload = v2_command; - payload_len = sizeof(v2_command); + if (ep->inline_v2) { + struct afkv2_command command; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || + afkv2_decode_command(payload, payload_len, &command) || + command.oob || command.id != tag) + return -EINVAL; + } else { + const struct epic_cmd *cmd = payload; + + if (stype != EPIC_SUBTYPE_STD_SERVICE || !cmd || + payload_len != sizeof(*cmd)) + return -EINVAL; + ret = afkv2_encode_oob_command(v2_command, sizeof(v2_command), tag, + le64_to_cpu(cmd->rxbuf), + le32_to_cpu(cmd->rxlen), + le64_to_cpu(cmd->txbuf), + le32_to_cpu(cmd->txlen)); + if (ret) + return ret; + payload = v2_command; + payload_len = sizeof(v2_command); + } v2_category = AFKV2_COMMAND; } else if (ecat == EPIC_CAT_REPLY) { v2_category = AFKV2_RESPONSE; @@ -1564,6 +1581,8 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, unsigned int timeout_ms) { struct epic_cmd cmd; + void *inline_command = NULL; + size_t copy_len = output_len; void *rxbuf, *txbuf; dma_addr_t rxbuf_dma, txbuf_dma; unsigned long flags; @@ -1611,6 +1630,17 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, } memcpy(txbuf, payload, payload_len); + if (ep->inline_v2) { + if (payload_len > AFKV2_MAX_PAYLOAD - AFKV2_COMMAND_SIZE) { + ret = -E2BIG; + goto err_free_txbuf; + } + inline_command = kmalloc(AFKV2_COMMAND_SIZE + payload_len, GFP_KERNEL); + if (!inline_command) { + ret = -ENOMEM; + goto err_free_txbuf; + } + } memset(&cmd, 0, sizeof(cmd)); cmd.retcode = cpu_to_le32(0); @@ -1650,9 +1680,21 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, service->cmds[idx].completion = &completion; init_completion(&completion); - ret = afk_send_epic(service->ep, service->channel, tag, - EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, - sizeof(cmd)); + if (ep->inline_v2) { + ret = afkv2_encode_inline_command(inline_command, + AFKV2_COMMAND_SIZE + payload_len, + tag, payload, payload_len, output_len); + if (!ret) + ret = afk_send_epic(ep, service->channel, tag, EPIC_TYPE_COMMAND, + EPIC_CAT_COMMAND, type, inline_command, + AFKV2_COMMAND_SIZE + payload_len); + } else { + ret = afk_send_epic(service->ep, service->channel, tag, + EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, + sizeof(cmd)); + } + kfree(inline_command); + inline_command = NULL; spin_unlock_irqrestore(&service->lock, flags); if (ret) goto err_free_cmd; @@ -1715,7 +1757,7 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, } } else if (output && output_len) { if (ep->wire_v2 && !service->cmds[idx].retcode && - service->cmds[idx].reply_length < output_len) + service->cmds[idx].reply_length < copy_len) ret = -EMSGSIZE; else if (output && output_len && (!ep->wire_v2 || !service->cmds[idx].retcode)) { if (ep->wire_v2) @@ -1729,6 +1771,8 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, bitmap_release_region(service->cmd_map, idx, 0); err_unlock: spin_unlock_irqrestore(&service->lock, flags); +err_free_txbuf: + kfree(inline_command); dma_free_coherent(ep->dcp->dev, payload_len, txbuf, txbuf_dma); err_free_rxbuf: dma_free_coherent(ep->dcp->dev, output_len, rxbuf, rxbuf_dma); @@ -1778,6 +1822,22 @@ static int afk_service_call_full(struct apple_epic_service *service, u16 group, output_pad, MSEC_PER_SEC); } +static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, unsigned int timeout_ms, + size_t *received); + +int afk_service_call_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *received) +{ + return afk_service_call_timeout_len(service, group, command, data, data_len, + data_pad, output, output_len, output_pad, + MSEC_PER_SEC, received); +} + int afk_service_call_timeout(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, @@ -1811,11 +1871,17 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, memset(call, 0, sizeof(*call)); call->group = cpu_to_le16(group); call->command = cpu_to_le32(command); - call->data_len = cpu_to_le32(data_len + data_pad); + call->data_len = cpu_to_le32(service->ep->inline_v2 ? data_len : + data_len + data_pad); call->magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC); memcpy(bfr + sizeof(*call), data, data_len); + /* Inline carries only the request; its response capacity is independent. */ + ret = afk_send_command_timeout_len(service, EPIC_SUBTYPE_STD_SERVICE, bfr, + service->ep->inline_v2 ? sizeof(*call) + data_len : + bfr_len, bfr, bfr_len, &retcode, timeout_ms, + received ? &wire_received : NULL); ret = afk_send_command_full(service, EPIC_SUBTYPE_STD_SERVICE, bfr, bfr_len, bfr, bfr_len, &retcode, reply_len ? &received : NULL, timeout_ms); @@ -1839,6 +1905,12 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, ret = afk_reply_size_valid(true, received, bfr_len, sizeof(*call)); if (ret) goto out; + if (received) { + ret = afkv2_service_reply_size(bfr, wire_received, group, command, + output_len, &result_size); + if (ret) + goto out; + *received = result_size; } if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le16_to_cpu(call->group) != group || diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 8b99a6259df5b3..df65822ad208ca 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -92,6 +92,29 @@ struct apple_epic_service_ops { void (*teardown)(struct apple_epic_service *service); }; +static inline int afk_ring_stride(u32 total, u32 bufsz, u32 *stride) +{ + u32 header, block; + + if (bufsz >= total) + return -EINVAL; + header = total - bufsz; + block = header / 3; + if (header % 3 || block < 0x40 || block % 0x40 || + bufsz < block || bufsz % block) + return -EINVAL; + + *stride = block; + return 0; +} + +static inline u32 afk_ring_advance(u32 ptr, u32 bytes, u32 bufsz, + u32 stride) +{ + u32 next = roundup(ptr + bytes, stride); + + return next == bufsz ? 0 : next; +} static inline struct apple_epic_service *afk_service_get(struct apple_epic_service *service) { unsigned long flags; @@ -208,29 +231,6 @@ struct afk_ringbuffer_header { __le32 wptr; u32 _pad3[15]; }; -static inline int afk_ring_stride(u32 total, u32 bufsz, u32 *stride) -{ - u32 header, block; - - if (bufsz >= total) - return -EINVAL; - header = total - bufsz; - block = header / 3; - if (header % 3 || block < 0x40 || block % 0x40 || - bufsz < block || bufsz % block) - return -EINVAL; - - *stride = block; - return 0; -} - -static inline u32 afk_ring_advance(u32 ptr, u32 bytes, u32 bufsz, - u32 stride) -{ - u32 next = roundup(ptr + bytes, stride); - - return next == bufsz ? 0 : next; -} struct afk_qe { #define QE_MAGIC 0x20504f49 // ' POI' @@ -328,12 +328,12 @@ struct afk_ringbuffer { struct apple_dcp_afkep { struct apple_dcp *dcp; - /* Bounded external protocol observations, owned by the RX worker. */ - u8 experimental_rx_logs; u32 endpoint; /* Set before afk_start; protocol selection is per endpoint. */ bool wire_v2; + /* External H17P DPAV only; other endpoints retain owned-DMA commands. */ + bool inline_v2; struct workqueue_struct *wq; struct completion started; @@ -395,4 +395,9 @@ int afk_service_call_with_reply_len(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, size_t *reply_len); +int afk_service_call_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *received); + #endif diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index 89901822753544..9ef52b2e70ea2d 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -179,6 +179,26 @@ static inline int afkv2_encode_oob_command(void *payload, size_t capacity, return 0; } +/* Inline requests carry a response capacity separately from their body. */ +static inline int afkv2_encode_inline_command(void *payload, size_t capacity, + u8 id, const void *body, + size_t size, u32 response_capacity) +{ + u8 *p = payload; + + if (!payload || (size && !body) || size > AFKV2_MAX_PAYLOAD - AFKV2_COMMAND_SIZE || + response_capacity > AFKV2_MAX_PAYLOAD) + return -EINVAL; + if (capacity < AFKV2_COMMAND_SIZE || size > capacity - AFKV2_COMMAND_SIZE) + return -ENOSPC; + if (size) + memmove(p + AFKV2_COMMAND_SIZE, body, size); + memset(p, 0, AFKV2_COMMAND_SIZE); + p[1] = id; + afkv2_put32(p + 4, response_capacity); + return 0; +} + static inline int afkv2_decode_command(const void *payload, size_t size, struct afkv2_command *out) { @@ -250,6 +270,38 @@ static inline int afkv2_check_response(const struct afkv2_frame *frame, return 0; } +/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. */ +static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, + const struct afkv2_command *response, + u16 interface, u8 type, u8 id, + size_t capacity) +{ + if (!frame || !response || frame->category != AFKV2_RESPONSE || + frame->interface != interface || frame->type != type || response->id != id || + frame->token || frame->transport_flags || frame->interface_flags || + frame->reserved || response->oob || response->inline_size > capacity) + return -EPROTO; + return 0; +} + +/* The standard service envelope precedes the actual, possibly short reply. */ +static inline int afkv2_service_reply_size(const void *body, size_t received, + u16 group, u32 command, size_t capacity, + size_t *size) +{ + const u8 *p = body; + u32 length; + + if (!body || !size || received < 64 || afkv2_get32(p + 12) != 0x69706378 || + afkv2_get16(p + 2) != group || afkv2_get32(p + 4) != command) + return -EPROTO; + length = afkv2_get32(p + 8); + if (length > received - 64 || length > capacity) + return -EMSGSIZE; + *size = length; + return 0; +} + /* Reject a peer command without touching its advertised DMA buffers. */ static inline int afkv2_encode_error_reply(void *record, size_t capacity, u8 sequence, const struct afkv2_frame *request, diff --git a/drivers/gpu/drm/apple/epic/dpavserv-edid.h b/drivers/gpu/drm/apple/epic/dpavserv-edid.h new file mode 100644 index 00000000000000..31baaaea7070d0 --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavserv-edid.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DPAVSERV_EDID_H +#define APPLE_DPAVSERV_EDID_H + +#include "../afkv2-codec.h" + +#define DPAV_EDID_HEADER_SIZE 48U +#define EDID_LEADING_DATA_SIZE 8U +#define EDID_BLOCK_SIZE 128U +#define EDID_MAX_SIZE 32768U +#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) + +/* Validate only bytes actually returned by firmware, never zero-fill padding. */ +static inline int dpavserv_edid_size(const void *response, size_t received, + size_t *edid_size) +{ + static const u8 header[] = { 0, 255, 255, 255, 255, 255, 255, 0 }; + const u8 *p = response; + u64 used; + size_t length, offset, i; + + if (!response || !edid_size || received < DPAV_EDID_HEADER_SIZE) + return -EIO; + if (afkv2_get64(p) != EDID_BUF_SIZE) + return -EIO; + used = afkv2_get64(p + 32); + if (used < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || used > EDID_BUF_SIZE || + used > received - DPAV_EDID_HEADER_SIZE) + return -EIO; + length = used - EDID_LEADING_DATA_SIZE; + p += DPAV_EDID_HEADER_SIZE + EDID_LEADING_DATA_SIZE; + if (length % EDID_BLOCK_SIZE || memcmp(p, header, sizeof(header)) || + ((size_t)p[126] + 1) * EDID_BLOCK_SIZE > length) + return -EIO; + /* Permit extra whole blocks for HF-EEODB, as drm_edid supports it. */ + for (offset = 0; offset < length; offset += EDID_BLOCK_SIZE) { + u8 sum = 0; + + for (i = 0; i < EDID_BLOCK_SIZE; i++) + sum += p[offset + i]; + if (sum) + return -EIO; + } + *edid_size = length; + return 0; +} + +#endif diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 2c90bebf9185fc..29dae044323699 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -3,6 +3,9 @@ #include "dpavservep.h" #include "dpavservep-edid.h" +#include "dpavserv-edid.h" + +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E #include @@ -13,8 +16,23 @@ #include "../afk.h" #include "../dcp.h" #include "../dcp-internal.h" +#include "../connector.h" #include "../trace.h" +static void dcpavserv_notify_edid(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct apple_connector *connector = READ_ONCE(dcp->connector); + + if (!service->ep->inline_v2 || READ_ONCE(dcp->quiescing) || !connector || + !READ_ONCE(dcp->nr_modes) || !READ_ONCE(connector->connected) || + READ_ONCE(connector->base.registration_state) != DRM_CONNECTOR_REGISTERED || + READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) + return; + /* get_modes may already have run before this service was announced. */ + schedule_work(&connector->hotplug_wq); +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -36,6 +54,11 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service->ep->inline_v2) { + dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", + service->channel); + dcpavserv_notify_edid(service); + } } } @@ -172,7 +195,7 @@ static int dcpdpserv_report(struct apple_epic_service *service, return parse_report(service, type, data, data_size); } -static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *service) +static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; @@ -203,6 +226,48 @@ static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *ser data_size > EDID_BUF_SIZE) return ERR_PTR(-EIO); + /* + * An HDMI 2.x sink using the HF-EEODB announces fewer extension blocks + * in the base block than it sends, so the payload may be longer than + * the base block says. Accept any whole number of blocks at least that + * long and leave the block count to drm_edid, which knows about + * HF-EEODB. + */ + num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; + if ((1 + num_blocks) * EDID_BLOCK_SIZE > data_size - EDID_LEADING_DATA_SIZE || + (data_size - EDID_LEADING_DATA_SIZE) % EDID_BLOCK_SIZE) + return ERR_PTR(-EIO); + + return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, + data_size - EDID_LEADING_DATA_SIZE); +} + +static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *service) +{ + struct dpavserv_copy_edid_cmd cmd; + struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; + size_t received, edid_size; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); + resp = kzalloc(resp_size, GFP_KERNEL); + if (!resp) + return ERR_PTR(-ENOMEM); + + ret = afk_service_call_len(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, + resp, resp_size, 0, &received); + if (ret < 0) + return ERR_PTR(ret); + + ret = dpavserv_edid_size(resp, received, &edid_size); + if (ret) + return ERR_PTR(ret); + + return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, edid_size); +} + const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) { struct apple_epic_service *service; @@ -210,7 +275,7 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) unsigned long flags; spin_lock_irqsave(&dcp->dcpavserv.lock, flags); - service = afk_service_get(dcp->dcpavserv.service); + service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); if (!service) return ERR_PTR(-ENODEV); @@ -248,6 +313,8 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) return ERR_PTR(-EIO); edid = dcpavserv_read_edid(service); + edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : + dcpavserv_read_edid_legacy(service); spin_lock_irqsave(&dcp->dcpavserv.lock, flags); if (dcp->dcpavserv.service != service) { if (!IS_ERR(edid)) @@ -287,6 +354,10 @@ int dpavservep_init(struct apple_dcp *dcp) ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); + /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ + ep->wire_v2 = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + ep->inline_v2 = ep->wire_v2; dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; @@ -301,8 +372,10 @@ int dpavservep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); - if (ret >= 0) - return 0; + if (!ret) { + dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); + return -ETIMEDOUT; + } - return ret; + return 0; } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e5e16c4b839cc3..99253dd7ee32d5 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1306,6 +1306,10 @@ int dcp_get_modes(struct drm_connector *connector) mutex_unlock(&dcp->modes_lock); drm_connector_set_vrr_capable_property(connector, vrr_capable); + /* Internal H17P still has no supported EDID command transport. */ + if (dcp->nr_modes && !apple_connector->drm_edid && + (dcp->fw_compat != DCP_FIRMWARE_H17P || + (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { if (count) { const struct drm_edid *edid; edid = dcpavserv_copy_edid(dcp); diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile index 65780f3d5003f9..df1b033d9cb324 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/Makefile @@ -4,6 +4,6 @@ export HOSTCC export AFKV2_SOURCE_ROOT := $(abspath ../../../../../..) TEST_PROGS := run-afkv2-tests.sh -TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py host-include +TEST_FILES := codec.c owned-prefix.c owned-suffix.c extract-owned.py inline-edid.c extract-service.py extract-send.py extract-edid.py extract-ring.py extract-notify.py host-include include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/README b/tools/testing/selftests/drivers/gpu/apple-afkv2/README index 42eb152d716a22..064cdcf0185956 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/README +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/README @@ -22,3 +22,19 @@ DMA coherency, firmware, a graphics device or any hardware display behavior. All build products are temporary. No device node is opened and no GPU, firmware or target machine is contacted. The extractor fails if production function boundaries change, requiring review rather than silently testing a stale copy. + +The inline EDID tests cover response context, status, declared/actual lengths, +EDID header/checksum/block limits, and continued rejection of inline responses +by the OOB matcher. The actual service-call body is extracted for short reply, +error status and timeout checks; its wire sender is mocked. + +The real sender is also extracted to test a 128-byte mocked command payload +with a 32 KiB response capacity, OOB behavior, send failures and timeout +retention. The two production EDID readers are tested independently to +prove legacy checksum tolerance remains unchanged. + +The complete encoded EDID request is 160 bytes including the queue entry, +consumes 192 bytes at a 64-byte stride, and needs a 256-byte empty ring. The +ring-admission block is extracted from the real transmitter to check this. +The late-publication notifier is extracted to test asynchronous hotplug +queueing and connector-registration, quiesce, routing and legacy guards. diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py new file mode 100644 index 00000000000000..0ab0f5eda8cac5 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Verify legacy EDID tolerance and stricter external-inline parsing separately.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() + +def function(name): + match = re.search(r"static const struct drm_edid \*" + name + r"\([^;]*?\)\n\{", source) + if not match: + raise RuntimeError("production reader missing: " + name) + end, depth = match.end(), 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[match.start():end] + +assert "service->ep->inline_v2 ? dcpavserv_read_edid_inline(service)" in source +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include "epic/dpavserv-edid.h" +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define cpu_to_le64(x) (x) +#define le64_to_cpu(x) (x) +#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7e +#define ERR_PTR(x) ((void*)(intptr_t)(x)) +#define IS_ERR(p) ((uintptr_t)(p)>(uintptr_t)-4096) +static void cleanup_free(void *p) { free(*(void**)p); } +#define __free(x) __attribute__((cleanup(cleanup_free))) +struct apple_epic_service { int unused; }; +struct drm_edid { size_t size; }; +struct dpavserv_copy_edid_cmd { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8]; }; +struct dpavserv_copy_edid_resp { u64 max_size;u8 pad1[24];u64 used_size;u8 pad2[8];u8 data[]; }; +static u8 fixture[48+8+256]; +static bool truncated; +static const struct drm_edid *drm_edid_alloc(const void *p,size_t n) +{ struct drm_edid *e=malloc(sizeof(*e));(void)p;e->size=n;return e; } +static int afk_service_call(struct apple_epic_service *s,u16 group,u32 command, + const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len,size_t output_pad) +{ + const struct dpavserv_copy_edid_cmd *cmd=data; + (void)s;(void)data_pad;(void)output_pad; + assert(group==1 && command==7 && data_len==48 && cmd->max_size==EDID_BUF_SIZE); + assert(output_len>=sizeof(fixture));memcpy(output,fixture,sizeof(fixture));return 0; +} +static int afk_service_call_len(struct apple_epic_service *s,u16 group,u32 command, + const void *data,size_t data_len,size_t data_pad,void *output,size_t output_len, + size_t output_pad,size_t *received) +{ int ret=afk_service_call(s,group,command,data,data_len,data_pad,output,output_len,output_pad); + *received=sizeof(fixture)-(truncated?1:0);return ret; } +static void fix_checksum(u8 *p) +{ unsigned int i;u8 sum=0;p[127]=0;for(i=0;i<127;i++)sum+=p[i];p[127]=-sum; } +''' +suffix = r''' +int main(void) +{ + struct apple_epic_service service={0};const struct drm_edid *e; + u8 *base=fixture+56,*extension=base+128; + afkv2_put64(fixture,EDID_BUF_SIZE);afkv2_put64(fixture+32,8+256); + memset(base+1,255,6);base[126]=1;fix_checksum(base); + extension[0]=2; /* deliberately invalid extension checksum */ + e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); + e=dcpavserv_read_edid_inline(&service);assert(IS_ERR(e)); + fix_checksum(extension); + e=dcpavserv_read_edid_inline(&service);assert(!IS_ERR(e) && e->size==256);free((void*)e); + truncated=true; + e=dcpavserv_read_edid_inline(&service);assert(IS_ERR(e)); + e=dcpavserv_read_edid_legacy(&service);assert(!IS_ERR(e));free((void*)e); + puts("PASS: actual legacy EDID checksum tolerance preserved; inline checksum/actual-length validation enforced"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + function("dcpavserv_read_edid_legacy") + function("dcpavserv_read_edid_inline") + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py new file mode 100644 index 00000000000000..26579f8399cb13 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise actual late-EDID publication notification guards with mock work.""" +import pathlib +import re +import sys +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static void dcpavserv_notify_edid\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("EDID notification helper missing") +end, depth = match.end(), 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#define READ_ONCE(x) (x) +#define DRM_CONNECTOR_REGISTERED 1 +struct base { int registration_state; }; +struct apple_connector { bool connected;struct base base;void *dcp;int hotplug_wq; }; +struct apple_dcp { struct apple_connector *connector;bool quiescing;unsigned int nr_modes;void *dev; }; +struct ep { struct apple_dcp *dcp;bool inline_v2; }; +struct apple_epic_service { struct ep *ep; }; +static unsigned int queued; +static void *to_platform_device(void *p) { return p; } +static void schedule_work(int *w) { (*w)++;queued++; } +''' +suffix = r''' +int main(void) +{ + int device,other;struct apple_dcp dcp={.dev=&device}; + struct ep ep={.dcp=&dcp,.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + struct apple_connector connector={.dcp=&device}; + dcpavserv_notify_edid(&service);assert(queued==0); + dcp.connector=&connector;dcpavserv_notify_edid(&service);assert(queued==0); + dcp.nr_modes=1;connector.connected=true;dcpavserv_notify_edid(&service);assert(queued==0); + connector.base.registration_state=DRM_CONNECTOR_REGISTERED; + dcpavserv_notify_edid(&service);assert(queued==1 && connector.hotplug_wq==1); + dcp.quiescing=true;dcpavserv_notify_edid(&service);assert(queued==1); + dcp.quiescing=false;connector.dcp=&other;dcpavserv_notify_edid(&service);assert(queued==1); + connector.dcp=&device;ep.inline_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); + ep.inline_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); + puts("PASS: actual late-EDID notification queues ready connector only; legacy/quiesce/unrouted/unregistered guarded"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[match.start():end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py new file mode 100644 index 00000000000000..65d6fdeca7a542 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Check full AFKV2 frame size against the production ring-admission block.""" +import pathlib +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +start = source.index("\ttotal_epic_size = sizeof(*ehdr) +", source.index("static int afk_send_epic_token", source.index("int afk_receive_message"))) +end = source.index("\n\thdr = hdr2 = NULL;", start) +block = source[start:end] +prefix = r''' +#undef NDEBUG +#include +#include +#include "afkv2-codec.h" +#define IS_ALIGNED(x,a) (!((x)&((a)-1))) +#define ALIGN(x,a) (((x)+(a)-1)&~((a)-1)) +struct epic_hdr { u8 data[16]; }; +struct epic_sub_hdr { u8 data[24]; }; +struct epic_sub_hdr_h17p { u8 data[8]; }; +struct afk_qe { u8 data[16]; }; +struct ring { bool ready; u32 stride,bufsz; }; +struct ep { bool wire_v2; struct ring txbfr; }; +static int check_ring(struct ep *ep,u32 rptr,u32 wptr,size_t payload_len,size_t *total) +{ + struct epic_hdr *ehdr;struct epic_sub_hdr *eshdr;struct afk_qe *hdr; + bool compact=false;size_t total_epic_size,total_size;int ret=0; +''' +suffix = r''' + *total=total_size; +out: + return ret; +} +int main(void) +{ + u8 request[112]={0},command[120],frame[144];size_t written,total; + struct ep ep={.wire_v2=true,.txbfr={.ready=true,.stride=64,.bufsz=256}}; + assert(!afkv2_encode_inline_command(command,sizeof(command),1,request,sizeof(request),32888)); + assert(!afkv2_encode_frame(frame,sizeof(frame),0,1,0,0xc0,AFKV2_COMMAND,0,command,sizeof(command),&written)); + assert(written==144 && written+sizeof(struct afk_qe)==160); + assert(!check_ring(&ep,0,0,sizeof(command),&total)); + assert(total==160 && ALIGN(total,64)==192); + ep.txbfr.bufsz=192;assert(check_ring(&ep,0,0,sizeof(command),&total)==-ENOSPC); + ep.txbfr.bufsz=128;assert(check_ring(&ep,0,0,sizeof(command),&total)==-EPROTO); + ep.txbfr.bufsz=256;assert(check_ring(&ep,192,0,sizeof(command),&total)==-ENOSPC); + assert(check_ring(&ep,0,1,sizeof(command),&total)==-EPROTO); + puts("PASS: full request 160 bytes / 192-byte aligned ring consumption; 256-byte empty ring accepted, smaller capacity rejected"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + block + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py new file mode 100644 index 00000000000000..92ca8857cc1045 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Compile the real command sender against synthetic ring and DMA operations.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static int afk_send_command_timeout_len\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("production command sender not found") +start, end, depth = match.start(), match.end(), 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include "afkv2-codec.h" +#define SZ_1M 1048576U +#define GFP_KERNEL 0 +#define MAX_PENDING_CMDS 16 +#define cpu_to_le32(x) (x) +#define cpu_to_le64(x) (x) +#define kmalloc(n,f) malloc(n) +#define kfree(p) free(p) +#define READ_ONCE(x) (x) +#define spin_lock_irqsave(l,f) ((void)(l),(f)=0) +#define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) +#define dma_rmb() do {} while(0) +#define dev_warn(...) do {} while(0) +#define EPIC_TYPE_COMMAND 3 +#define EPIC_CAT_COMMAND 0x30 +#define DECLARE_COMPLETION_ONSTACK(n) int n=0 +#define init_completion(p) (*(p)=0) +#define msecs_to_jiffies(x) (x) +#define wait_for_completion_timeout(p,t) ((void)(t),*(p)) +typedef u64 dma_addr_t; +struct epic_cmd { u32 retcode; u64 rxbuf,txbuf; u32 rxlen,txlen; u8 rxcookie,txcookie; }; +struct cmd_info { + u16 tag; u8 type; size_t reply_length; void *rxbuf,*txbuf; + dma_addr_t rxbuf_dma,txbuf_dma; size_t rxlen,txlen; u32 retcode; + bool done,free_on_ack; int *completion; +}; +struct dcp { void *dev; bool quiescing; }; +struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint; }; +struct ops { const char *name; }; +struct apple_epic_service { + struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; + int lock; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; +}; +static struct apple_epic_service *active; +static unsigned int frees, sends; +static int wire_fail, timeout, reply_status; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { return s->enabled?s:NULL; } +static void afk_service_put(struct apple_epic_service *s) { (void)s; } +static void *dma_alloc_coherent(void *dev,size_t n,dma_addr_t *address,int flags) +{ void *p=calloc(1,n);(void)dev;(void)flags;*address=(uintptr_t)p;return p; } +static void dma_free_coherent(void *dev,size_t n,void *p,dma_addr_t a) +{ (void)dev;(void)n;assert((uintptr_t)p==a);free(p);frees++; } +static int bitmap_find_free_region(unsigned long *map,unsigned int n,int order) +{ (void)n;(void)order;assert(!*map);*map=1;return 0; } +static void bitmap_release_region(unsigned long *map,int i,int order) +{ (void)i;(void)order;*map=0; } +static int afk_send_epic(struct apple_dcp_afkep *ep,u32 channel,u16 tag,int type,int category,u8 subtype,const void *payload,size_t length) +{ + struct cmd_info *c=&active->cmds[0];struct afkv2_command parsed; + assert(channel==4 && type==3 && category==0x30 && subtype==0xc0); + sends++; + /* The payload-envelope mock bounds the command separately from its response. */ + assert(length<=128); + if(ep->inline_v2) { + assert(length==120); + assert(!afkv2_decode_command(payload,length,&parsed)); + assert(!parsed.oob && parsed.id==tag && parsed.value==32888 && parsed.inline_size==112); + assert(parsed.inline_data[0]==0x5a); + } else { + const struct epic_cmd *legacy=payload; + assert(length==sizeof(*legacy) && legacy->txlen==112 && legacy->rxlen==32888); + } + if(wire_fail)return -ENOSPC; + if(timeout)return 0; + c->reply_length=ep->inline_v2?4:32888;c->retcode=reply_status; + memset(c->rxbuf,0x77,c->reply_length);c->done=true;*c->completion=1; + return 0; +} +''' +suffix = r''' +int main(void) +{ + struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; + struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true}; + u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; + active=&s; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==4 && status==0 && output[0]==0x77 && frees==2 && sends==1 && !s.cmd_map[0]); + /* Existing strict callers still reject a short inline result. */ + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,NULL)==-EMSGSIZE); + reply_status=5;received=99; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(status==5 && received==0);reply_status=0; + wire_fail=1; + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ENOSPC && !s.cmd_map[0]); + wire_fail=0;ep.inline_v2=false; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==32888 && status==0); + ep.inline_v2=true;timeout=1; + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ETIMEDOUT); + assert(s.cmd_map[0] && s.cmds[0].free_on_ack && !s.cmds[0].completion); + /* Mock the late-reply owner's release, avoiding leaks in this host test. */ + free(s.cmds[0].rxbuf);free(s.cmds[0].txbuf);free(output); + puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, strict OOB and timeout ownership"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py new file mode 100644 index 00000000000000..68e3f5e6ba980c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production service-call body with a mocked wire reply.""" +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]).read_text() +match = re.search(r"static int afk_service_call_timeout_len\([^;]*?\)\n\{", source) +if not match: + raise RuntimeError("production service-call definition not found") +start = match.start() +end = match.end() +depth = 1 +while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 +prefix = r''' +#undef NDEBUG +#include +#include +#include +#include "afkv2-codec.h" +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define max(a,b) ((a)>(b)?(a):(b)) +#define cpu_to_le16(x) (x) +#define cpu_to_le32(x) (x) +#define le16_to_cpu(x) (x) +#define le32_to_cpu(x) (x) +#define EPIC_SERVICE_CALL_MAGIC 0x69706378 +#define EPIC_SUBTYPE_STD_SERVICE 0xc0 +#define dev_warn(...) do {} while(0) +#define dev_dbg(...) do {} while(0) +struct epic_service_call { + u8 pad0[2]; u16 group; u32 command, data_len, magic; u8 pad1[48]; +}; +struct dcp { void *dev; }; +struct ops { const char *name; }; +struct ep { struct dcp *dcp; int endpoint; bool inline_v2; }; +struct apple_epic_service { struct ep *ep; struct ops *ops; u32 channel; }; +static int error_status, malformed, short_header, too_long, wire_error; +static int afk_send_command_timeout_len(struct apple_epic_service *s, u8 type, + const void *payload, size_t payload_len, void *output, size_t output_len, + u32 *retcode, unsigned int timeout_ms, size_t *received) +{ + struct epic_service_call *call=output; + (void)payload; + assert(payload_len==(s->ep->inline_v2?68:output_len)); + assert(call->data_len==(s->ep->inline_v2?4:4100)); + assert(type==0xc0 && timeout_ms==1000 && output_len>=68); + if(wire_error) return -ETIMEDOUT; + call->group=1;call->command=7;call->magic=EPIC_SERVICE_CALL_MAGIC; + call->data_len=too_long?5:4; + ((u8*)output)[64]=0xde;((u8*)output)[65]=0xad; + ((u8*)output)[66]=0xbe;((u8*)output)[67]=0xef; + if(malformed)call->command=8; + *retcode=error_status; + if(received)*received=short_header?63:68; + return 0; +} +''' +suffix = r''' +int main(void) +{ + struct ep ep={.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + u8 input[4]={0}, output[128]={0}; size_t received=999; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + assert(received==4 && output[0]==0xde && output[3]==0xef && output[4]==0); + error_status=5;received=999; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-EINVAL && received==0); + error_status=0;malformed=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + malformed=0;short_header=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + short_header=0;too_long=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + too_long=0;wire_error=1; + assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-ETIMEDOUT); + wire_error=0; ep.inline_v2=false; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,NULL)); + puts("PASS: extracted production service-call actual request / legacy padding / short reply / status / context / length tests"); +} +''' +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h index a5c3a3f44a5912..23e11801f1e763 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/host-include/linux/errno.h @@ -3,3 +3,8 @@ #define ENOSPC 28 #define E2BIG 7 #define EPROTO 71 +#define EIO 5 +#define EMSGSIZE 90 +#define ENODEV 19 +#define ETIMEDOUT 110 +#define ENOMEM 12 diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c new file mode 100644 index 00000000000000..7ce16fd2b6a8b7 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#undef NDEBUG +#include +#include +#include "epic/dpavserv-edid.h" + +static unsigned int assertions; +#define CHECK(x) do { assertions++; assert(x); } while (0) + +static void checksum(u8 *block) +{ + u8 sum = 0; + unsigned int i; + + block[127] = 0; + for (i = 0; i < 127; i++) + sum += block[i]; + block[127] = -sum; +} + +int main(void) +{ + u8 command[192], packet[216], body[136] = {0}; + u8 edid[DPAV_EDID_HEADER_SIZE + 8 + 3 * 128] = {0}; + struct afkv2_frame frame; + struct afkv2_command reply; + size_t written, length = 0; + u8 *base = edid + DPAV_EDID_HEADER_SIZE + 8; + + CHECK(!afkv2_encode_inline_command(command, sizeof(command), 0x91, + body, sizeof(body), 400)); + CHECK(command[0] == 0 && command[1] == 0x91); + CHECK(afkv2_get32(command + 4) == 400); + CHECK(!afkv2_encode_frame(packet, sizeof(packet), 5, 4, 0, 0xc0, + AFKV2_RESPONSE, 0, command, 8 + sizeof(body), &written)); + /* Response uses status in the request-capacity field. */ + afkv2_put32(packet + AFKV2_HEADER_SIZE + 4, 0); + CHECK(!afkv2_decode_frame(packet, written, false, &frame)); + CHECK(!afkv2_decode_command(frame.payload, frame.payload_size, &reply)); + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 135)); + CHECK(afkv2_check_inline_response(&frame, &reply, 5, 0xc0, 0x91, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x92, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc1, 0x91, 136)); + frame.category = AFKV2_COMMAND; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.category = AFKV2_RESPONSE; + frame.token = 1; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.token = 0; + frame.reserved = 1; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + frame.reserved = 0; + reply.oob = true; + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + reply.oob = false; + /* The original OOB matcher must continue rejecting inline replies. */ + CHECK(afkv2_check_response(&frame, &reply, 4, 0xc0, 0x91, 0, 136, 0, 0)); + reply.value = 7; + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + CHECK(reply.value != 0); /* Error status is preserved for the caller. */ + CHECK(afkv2_encode_inline_command(command, 7, 1, body, 1, 1)); + CHECK(afkv2_encode_inline_command(command, sizeof(command), 1, NULL, 1, 1)); + CHECK(afkv2_encode_inline_command(command, sizeof(command), 1, body, 1, + AFKV2_MAX_PAYLOAD + 1)); + + afkv2_put16(body + 2, 1); + afkv2_put32(body + 4, 7); + afkv2_put32(body + 8, 72); + afkv2_put32(body + 12, 0x69706378); + CHECK(!afkv2_service_reply_size(body, 136, 1, 7, 72, &length) && length == 72); + CHECK(afkv2_service_reply_size(body, 135, 1, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 1, 7, 71, &length)); + CHECK(afkv2_service_reply_size(body, 63, 1, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 2, 7, 72, &length)); + CHECK(afkv2_service_reply_size(body, 136, 1, 8, 72, &length)); + body[12] ^= 1; + CHECK(afkv2_service_reply_size(body, 136, 1, 7, 72, &length)); + + afkv2_put64(edid, EDID_BUF_SIZE); + afkv2_put64(edid + 32, 8 + 128); + memset(base + 1, 255, 6); + base[18] = 1; + base[19] = 4; + checksum(base); + CHECK(!dpavserv_edid_size(edid, 48 + 8 + 128, &length) && length == 128); + CHECK(dpavserv_edid_size(edid, 47, &length)); + CHECK(dpavserv_edid_size(edid, 48 + 8 + 127, &length)); + base[0] = 1; + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + base[0] = 0; + base[24] ^= 1; + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + base[24] ^= 1; + base[126] = 1; + checksum(base); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, 8 + 256); + CHECK(!dpavserv_edid_size(edid, 48 + 8 + 256, &length) && length == 256); + /* HF-EEODB can supply more complete blocks than base[126] declares. */ + base[126] = 0; + checksum(base); + CHECK(!dpavserv_edid_size(edid, sizeof(edid), &length) && length == 256); + afkv2_put64(edid + 32, 8 + 255); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, EDID_BUF_SIZE + 1); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + afkv2_put64(edid + 32, 8 + 128); + afkv2_put64(edid, EDID_BUF_SIZE - 1); + CHECK(dpavserv_edid_size(edid, sizeof(edid), &length)); + printf("PASS: inline AFKV2 and EDID validation (%u checks)\n", assertions); + return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh index 4e13f6d05d986c..4348eebc78246e 100755 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh @@ -38,3 +38,45 @@ $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ "$afkv2_tmp/owned.c" -o "$afkv2_tmp/owned" "$afkv2_tmp/owned" + +# Inline DPAV transport and actual-length EDID response validation. +# shellcheck disable=SC2086 +$afkv2_cc -std=c11 -Wall -Wextra -Werror -pedantic ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_dir/inline-edid.c" -o "$afkv2_tmp/inline-edid" +"$afkv2_tmp/inline-edid" +"$afkv2_python" "$afkv2_dir/extract-service.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/service.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/service.c" -o "$afkv2_tmp/service" +"$afkv2_tmp/service" + +"$afkv2_python" "$afkv2_dir/extract-send.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/send.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/send.c" -o "$afkv2_tmp/send" +"$afkv2_tmp/send" + +"$afkv2_python" "$afkv2_dir/extract-edid.py" \ + "$afkv2_root/drivers/gpu/drm/apple/epic/dpavservep.c" "$afkv2_tmp/edid.c" +# shellcheck disable=SC2086 +$afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/edid.c" -o "$afkv2_tmp/edid" +"$afkv2_tmp/edid" + +"$afkv2_python" "$afkv2_dir/extract-ring.py" \ + "$afkv2_root/drivers/gpu/drm/apple/afk.c" "$afkv2_tmp/ring.c" +"$afkv2_python" "$afkv2_dir/extract-notify.py" \ + "$afkv2_root/drivers/gpu/drm/apple/epic/dpavservep.c" "$afkv2_tmp/notify.c" +for afkv2_test in ring notify; do + # shellcheck disable=SC2086 + $afkv2_cc -std=gnu11 -Wall -Wextra -Werror ${CFLAGS:-} $afkv2_sanitizers \ + -I"$afkv2_dir/host-include" -I"$afkv2_root/drivers/gpu/drm/apple" \ + "$afkv2_tmp/$afkv2_test.c" -o "$afkv2_tmp/$afkv2_test" + "$afkv2_tmp/$afkv2_test" +done From 0e93d737d5255e61a04a43a7b738169fa46dee1a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:28:37 +1100 Subject: [PATCH 426/631] drm/apple: add bounded tracepoints for external output qualification Expose AFK headers before protocol decoding, submitted surface metadata, CTM values, and A423/A428 reply values through disabled-by-default tracepoints. Capture at most 32 received header bytes and do not dereference firmware-advertised addresses. Preserve all firmware requests and callback results. Host tests exercise the actual trace assignments and callback continuation with zero/short/null and unaligned inputs. These observers support investigating the unqualified DPAV announcement and output-color behavior without changing protocol policy. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 2 + drivers/gpu/drm/apple/iomfb_template.c | 11 +++- drivers/gpu/drm/apple/trace.h | 86 ++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index ffc53901eda9ba..428d5241b0f4b1 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -975,6 +975,8 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); size_t payload_size; + trace_afk_recv_header(ep, channel, type, data_size, data); + if (ep->wire_v2) { afk_v2_receive(ep, channel, type, data, data_size); return; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 2d6a10243e5d93..1c8b68ee3e440e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -69,6 +69,13 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, const void *data = request; if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + for (i = 0; i < SWAP_SURFACES; i++) { + const struct dcp_surface *surface = &request->surf[i].base; + + if (!request->surf_null[i]) + trace_iomfb_surface(dcp, i, surface->format, surface->xfer_func, + surface->colorspace, surface->is_premultiplied); + } iomfb_serialize_present_h17p(&dcp->present_h17p, request); if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { if (cb) @@ -2112,7 +2119,8 @@ static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) { u32 identity = 0x10000; - dcp_pipe_set_428(dcp, false, &identity, do_swap, cookie); + trace_iomfb_control_result(dcp, 423, !!out, out ? get_unaligned_le32(out) : 0); + dcp_pipe_set_428(dcp, false, &identity, dcp_post_swap_a428, cookie); } #endif @@ -2379,6 +2387,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } + trace_iomfb_ctm(dcp, mat.location, mat.matrix, mat.matrix_null); #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); #else diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index d3c5106e5d66d2..085f28ac44faeb 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -205,6 +205,92 @@ TRACE_EVENT( print_epic_category(__entry->category), __entry->subtype, __entry->tag)); +/* Raw prefix before any protocol-specific decode; no peer address is followed. */ +TRACE_EVENT(afk_recv_header, + TP_PROTO(struct apple_dcp_afkep *ep, u32 channel, u32 type, + size_t data_size, const void *data), + TP_ARGS(ep, channel, type, data_size, data), + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) + __field(u32, channel) + __field(u32, type) + __field(size_t, data_size) + __field(u8, captured) + __array(u8, header, 32)), + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->channel = channel; + __entry->type = type; + __entry->data_size = data_size; + __entry->captured = data ? min_t(size_t, data_size, 32) : 0; + memset(__entry->header, 0, sizeof(__entry->header)); + if (__entry->captured) + memcpy(__entry->header, data, __entry->captured); + ), + TP_printk("%s: endpoint 0x%x channel 0x%x type 0x%x size %zu header[%u]=%s", + __get_str(devname), __entry->endpoint, __entry->channel, + __entry->type, __entry->data_size, __entry->captured, + __print_hex(__entry->header, __entry->captured)) +); + +TRACE_EVENT(iomfb_ctm, + TP_PROTO(struct apple_dcp *dcp, u32 location, const void *matrix, u8 matrix_null), + TP_ARGS(dcp, location, matrix, matrix_null), + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u32, location) + __field(u8, matrix_null) + __array(u64, matrix, 9)), + TP_fast_assign(__assign_str(devname); + __entry->location = location; + __entry->matrix_null = matrix_null; + if (matrix) + memcpy(__entry->matrix, matrix, sizeof(__entry->matrix)); + else + memset(__entry->matrix, 0, sizeof(__entry->matrix)); + ), + TP_printk("%s: location %u null %u matrix=%s", + __get_str(devname), __entry->location, __entry->matrix_null, + __print_array(__entry->matrix, 9, sizeof(u64))) +); + +TRACE_EVENT(iomfb_control_result, + TP_PROTO(struct apple_dcp *dcp, u32 method, bool valid, u32 result), + TP_ARGS(dcp, method, valid, result), + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u32, method) + __field(bool, valid) + __field(u32, result)), + TP_fast_assign(__assign_str(devname); + __entry->method = method; + __entry->valid = valid; + __entry->result = result; + ), + TP_printk("%s: A%u reply_valid %u result 0x%08x", + __get_str(devname), __entry->method, __entry->valid, __entry->result) +); + +TRACE_EVENT(iomfb_surface, + TP_PROTO(struct apple_dcp *dcp, u32 index, u32 format, u8 xfer_func, + u8 colorspace, u8 premultiplied), + TP_ARGS(dcp, index, format, xfer_func, colorspace, premultiplied), + TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) + __field(u32, index) + __field(u32, format) + __field(u8, xfer_func) + __field(u8, colorspace) + __field(u8, premultiplied)), + TP_fast_assign(__assign_str(devname); + __entry->index = index; + __entry->format = format; + __entry->xfer_func = xfer_func; + __entry->colorspace = colorspace; + __entry->premultiplied = premultiplied; + ), + TP_printk("%s: surface %u format 0x%08x xfer %u colorspace %u source_premul %u", + __get_str(devname), __entry->index, __entry->format, + __entry->xfer_func, __entry->colorspace, __entry->premultiplied) +); + TRACE_EVENT(iomfb_callback, TP_PROTO(struct apple_dcp *dcp, int tag, const char *name), TP_ARGS(dcp, tag, name), From 9e2981c653e37c818e61a5981dfacf7cfae7a9e2 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:39:37 +1100 Subject: [PATCH 427/631] drm/apple: refine inline reply validation and trace silent DPAV rings Treat the transport/interface/reserved header fields as opaque, matching captured firmware responses and the owned-DMA validator, while retaining identity, type, tag, category and payload-bound checks. Add an optional read-only ring snapshot at the DPAV announcement timeout; it does not consume data or send another command. Host tests preserve response mismatch rejection, short-reply checks and timeout ownership, and verify the snapshot's enable, readiness, stop and pointer guards. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 31 +++++++++++++++++ drivers/gpu/drm/apple/afk.h | 1 + drivers/gpu/drm/apple/afkv2-codec.h | 7 ++-- drivers/gpu/drm/apple/epic/dpavservep.c | 1 + drivers/gpu/drm/apple/trace.h | 34 +++++++++++++++++++ .../drivers/gpu/apple-afkv2/inline-edid.c | 12 +++++-- 6 files changed, 81 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 428d5241b0f4b1..6a5377347a9776 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -208,6 +208,37 @@ int afk_start(struct apple_dcp_afkep *ep) return 0; } +/* Read only host-owned ring headers; never consume an entry or notify firmware. */ +void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep) +{ + unsigned long flags; + u32 rx_rptr = 0, rx_wptr = 0, tx_rptr = 0, tx_wptr = 0; + bool rx_ready, tx_ready; + + if (!ep || !ep->inline_v2 || !trace_afk_ring_snapshot_enabled()) + return; + spin_lock_irqsave(&ep->lock, flags); + if (ep->stopping) { + spin_unlock_irqrestore(&ep->lock, flags); + return; + } + rx_ready = READ_ONCE(ep->rxbfr.ready) && ep->rxbfr.rptr && ep->rxbfr.wptr; + tx_ready = READ_ONCE(ep->txbfr.ready) && ep->txbfr.rptr && ep->txbfr.wptr; + dma_rmb(); + if (rx_ready) { + rx_rptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.rptr)); + rx_wptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.wptr)); + } + if (tx_ready) { + tx_rptr = le32_to_cpu(READ_ONCE(*ep->txbfr.rptr)); + tx_wptr = le32_to_cpu(READ_ONCE(*ep->txbfr.wptr)); + } + trace_afk_ring_snapshot(ep, rx_ready, rx_rptr, rx_wptr, + READ_ONCE(ep->rxbfr.bufsz), tx_ready, tx_rptr, + tx_wptr, READ_ONCE(ep->txbfr.bufsz)); + spin_unlock_irqrestore(&ep->lock, flags); +} + static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) { u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index df65822ad208ca..68a8c0218c45b0 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -367,6 +367,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, void *buffer, size_t size, bool to_peer); int afk_start(struct apple_dcp_afkep *ep); +void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); void afk_quiesce(struct apple_dcp_afkep *ep); int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index 9ef52b2e70ea2d..a06a1efd5fc7e8 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -270,7 +270,9 @@ static inline int afkv2_check_response(const struct afkv2_frame *frame, return 0; } -/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. */ +/* Kept separate so DPAV inline replies cannot loosen owned-DMA checks. + * Transport/interface flags and the reserved word are opaque firmware data. + */ static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, const struct afkv2_command *response, u16 interface, u8 type, u8 id, @@ -278,8 +280,7 @@ static inline int afkv2_check_inline_response(const struct afkv2_frame *frame, { if (!frame || !response || frame->category != AFKV2_RESPONSE || frame->interface != interface || frame->type != type || response->id != id || - frame->token || frame->transport_flags || frame->interface_flags || - frame->reserved || response->oob || response->inline_size > capacity) + frame->token || response->oob || response->inline_size > capacity) return -EPROTO; return 0; } diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 29dae044323699..1fb9818c0e5933 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -373,6 +373,7 @@ int dpavservep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); if (!ret) { + afk_trace_ring_snapshot(ep); dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); return -ETIMEDOUT; } diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index 085f28ac44faeb..d19609f116f2c4 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -233,6 +233,40 @@ TRACE_EVENT(afk_recv_header, __print_hex(__entry->header, __entry->captured)) ); +TRACE_EVENT(afk_ring_snapshot, + TP_PROTO(struct apple_dcp_afkep *ep, bool rx_ready, u32 rx_rptr, + u32 rx_wptr, u32 rx_size, bool tx_ready, u32 tx_rptr, + u32 tx_wptr, u32 tx_size), + TP_ARGS(ep, rx_ready, rx_rptr, rx_wptr, rx_size, + tx_ready, tx_rptr, tx_wptr, tx_size), + TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) + __field(u8, endpoint) + __field(bool, rx_ready) + __field(u32, rx_rptr) + __field(u32, rx_wptr) + __field(u32, rx_size) + __field(bool, tx_ready) + __field(u32, tx_rptr) + __field(u32, tx_wptr) + __field(u32, tx_size)), + TP_fast_assign(__assign_str(devname); + __entry->endpoint = ep->endpoint; + __entry->rx_ready = rx_ready; + __entry->rx_rptr = rx_rptr; + __entry->rx_wptr = rx_wptr; + __entry->rx_size = rx_size; + __entry->tx_ready = tx_ready; + __entry->tx_rptr = tx_rptr; + __entry->tx_wptr = tx_wptr; + __entry->tx_size = tx_size; + ), + TP_printk("%s: endpoint 0x%x rx_ready %u rx_rptr %#x rx_wptr %#x rx_size %#x tx_ready %u tx_rptr %#x tx_wptr %#x tx_size %#x", + __get_str(devname), __entry->endpoint, __entry->rx_ready, + __entry->rx_rptr, __entry->rx_wptr, __entry->rx_size, + __entry->tx_ready, __entry->tx_rptr, __entry->tx_wptr, + __entry->tx_size) +); + TRACE_EVENT(iomfb_ctm, TP_PROTO(struct apple_dcp *dcp, u32 location, const void *matrix, u8 matrix_null), TP_ARGS(dcp, location, matrix, matrix_null), diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c index 7ce16fd2b6a8b7..b711adf8047944 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/inline-edid.c @@ -48,8 +48,16 @@ int main(void) frame.token = 1; CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); frame.token = 0; - frame.reserved = 1; - CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + /* R30 responses carry these opaque nonzero header fields. */ + frame.transport_flags = 0x18; + frame.interface_flags = 0x0400; + frame.reserved = 0x3607ebe0; + CHECK(!afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); + /* Context and bounds remain mandatory even with opaque header values. */ + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x92, 136)); + CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 135)); + frame.transport_flags = 0; + frame.interface_flags = 0; frame.reserved = 0; reply.oob = true; CHECK(afkv2_check_inline_response(&frame, &reply, 4, 0xc0, 0x91, 136)); From 0f2c605dfe3b5d3669ed211153f74eeb3ee8db6d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:42:08 +1100 Subject: [PATCH 428/631] drm/apple: honor external timing color exclusions before SDR selection Exclude firmware-listed unsafe and DSC-required color IDs before choosing an external H17P SDR candidate. Read restrictions independently of property order and reject malformed or duplicate lists instead of silently treating them as empty. Preserve internal and legacy parsing. The captured dock route permits 60 Hz at 10-bit and 120 Hz at 8-bit; its 144/180 Hz timings have no eligible nonvirtual SDR candidate after exclusions. This is metadata admission, not a new physical-link qualification or a global refresh cap. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/parser.c | 105 ++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index d103a97ddb6977..0cd3df7afed479 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -467,8 +467,92 @@ static unsigned int external_sdr_rank(const struct color_mode *color) } } +/* Firmware lists color element IDs which this timing cannot safely use. */ +#define DCP_TIMING_COLOR_ID_LIMIT 64 + +struct timing_color_ids { + u32 ids[DCP_TIMING_COLOR_ID_LIMIT]; + unsigned int count; +}; + +struct timing_color_restrictions { + struct timing_color_ids unsafe; + struct timing_color_ids dsc_required; +}; + +static bool timing_color_contains(const struct timing_color_ids *ids, u32 id) +{ + unsigned int i; + + for (i = 0; i < ids->count; i++) + if (ids->ids[i] == id) + return true; + return false; +} + +static int parse_timing_color_ids(struct dcp_parse_ctx *handle, + struct timing_color_ids *ids) +{ + struct iterator it; + int ret; + + ret = iterator_begin(handle, &it, false); + if (ret) + return ret; + if (it.len > DCP_TIMING_COLOR_ID_LIMIT) + return -E2BIG; + for (; it.idx < it.len; it.idx++) { + s64 id; + + ret = parse_int_bound(handle, &id, 0, U32_MAX); + if (ret) + return ret; + if (timing_color_contains(ids, id)) + return -EINVAL; + ids->ids[ids->count++] = id; + } + return 0; +} + +static int parse_timing_color_restrictions(struct dcp_parse_ctx *handle, + struct timing_color_restrictions *limits) +{ + struct iterator it; + bool unsafe_seen = false, dsc_seen = false, colors_seen = false; + int ret; + + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char *key = parse_string(handle); + + if (IS_ERR(key)) + return PTR_ERR(key); + if (!strcmp(key, "UnsafeColorElementIDs")) { + ret = unsafe_seen ? -EINVAL : + parse_timing_color_ids(handle, &limits->unsafe); + unsafe_seen = true; + } else if (!strcmp(key, "DSCRequiredColorElementIDs")) { + ret = dsc_seen ? -EINVAL : + parse_timing_color_ids(handle, &limits->dsc_required); + dsc_seen = true; + } else if (!strcmp(key, "ColorModes")) { + ret = colors_seen ? -EINVAL : skip(handle); + colors_seen = true; + } else { + ret = skip(handle); + } + kfree(key); + if (ret) + return ret; + } + return 0; +} + static int parse_color_modes(struct dcp_parse_ctx *handle, - struct dcp_display_mode *out, bool external_sdr) + struct dcp_display_mode *out, bool external_sdr, + const struct timing_color_restrictions *limits) { struct iterator outer_it; unsigned int best_sdr_rank = 0; @@ -536,8 +620,13 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.pixel_encoding); if (external_sdr) { - unsigned int rank = external_sdr_rank(&cmode); + unsigned int rank; + if (cmode.id <= U32_MAX && + (timing_color_contains(&limits->unsafe, cmode.id) || + timing_color_contains(&limits->dsc_required, cmode.id))) + continue; + rank = external_sdr_rank(&cmode); if (!rank || rank < best_sdr_rank) continue; if (rank > best_sdr_rank) @@ -582,6 +671,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, { int ret = 0; struct iterator it; + struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; + struct timing_color_restrictions limits = {}; struct dimension horiz = {}, vert = {}; u64 clock; s64 min_vrr = 0, max_vrr = 0; @@ -598,6 +689,14 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->best.score = -1; *score = -1; + if (external_sdr) { + struct dcp_parse_ctx scan = *handle; + + /* ColorModes can precede either list: collect restrictions first. */ + ret = parse_timing_color_restrictions(&scan, &limits); + if (ret) + return ret; + } ret = iterator_begin(handle, &it, true); if (ret) return ret; @@ -618,7 +717,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out, external_sdr); + ret = parse_color_modes(it.handle, out, external_sdr, &limits); else if (!strcmp(key, "ID")) ret = parse_int(it.handle, &id); else if (!strcmp(key, "IsVirtual")) From 004f493a0ce2d9fffec69662ff8dd3dc71f4730a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:49:18 +1100 Subject: [PATCH 429/631] drm/apple: derive bounded external monitor metadata for initial attachment Expose the firmware-reported identity, primaries and preferred timing while raw DPAV EDID is unavailable on the experimental H17P external path. Keep this synthetic base block out of the raw cache, require complete metadata from the same initial connection, and invalidate it on route or cable loss. Do not synthesize extra timings, audio, HDR, VRR or link capabilities. Serialize EDID property and cache changes with connector probing. Defer commit-context disconnect cleanup through a persistent invalidation flag, so rapid reconnection cannot lose it or invert DRM modeset lock ordering. Include bounded-parser, independent EDID decoding and concurrent lifetime tests. Hardware qualification remains pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/connector.h | 3 + drivers/gpu/drm/apple/dcp-display-metadata.c | 204 ++++++++++++++ drivers/gpu/drm/apple/dcp-display-metadata.h | 24 ++ drivers/gpu/drm/apple/dcp-internal.h | 23 ++ drivers/gpu/drm/apple/dcp-metadata-epoch.h | 48 ++++ drivers/gpu/drm/apple/dcp.c | 174 ++++++++++++ drivers/gpu/drm/apple/iomfb.c | 66 +++++ drivers/gpu/drm/apple/iomfb_template.c | 1 + drivers/gpu/drm/apple/parser.c | 266 +++++++++++++++++++ drivers/gpu/drm/apple/parser.h | 6 + 12 files changed, 817 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/apple/dcp-display-metadata.c create mode 100644 drivers/gpu/drm/apple/dcp-display-metadata.h create mode 100644 drivers/gpu/drm/apple/dcp-metadata-epoch.h diff --git a/MAINTAINERS b/MAINTAINERS index 2f38d30821f097..22f8615ea8f50f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2028,6 +2028,7 @@ F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs F: tools/diagnostics/apple-dptx-training-test.c F: tools/diagnostics/apple-j700-dp-test.c +F: tools/diagnostics/display-attributes-fixture.json F: tools/diagnostics/test-apple-afk-admission.py F: tools/diagnostics/test-apple-atc-unwind.py F: tools/diagnostics/test-apple-dcp-readiness.py diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 4a4b7dba9372b7..4619415a36ab00 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -7,7 +7,7 @@ appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o appledrm-y += dcp-hdmi.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o +appledrm-y += connector.o dcp-display-metadata.o appledrm-y += ibootep.o appledrm-y += dcpext_scanout.o appledrm-y += dcpext_drm.o diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index fe4a60d7155997..45f59d93b97671 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -55,6 +55,9 @@ struct apple_connector { struct mutex edid_lock; u64 edid_generation; bool edid_live; + bool drm_edid_firmware_derived; + bool edid_invalidation_pending; + u64 drm_edid_generation; /* Workqueue for sending hotplug events to the associated device */ struct work_struct hotplug_wq; diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.c b/drivers/gpu/drm/apple/dcp-display-metadata.c new file mode 100644 index 00000000000000..cc4c69736d0986 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-display-metadata.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include +#include + +#include "dcp-display-metadata.h" + +static int dcp_metadata_timing(struct detailed_timing *timing, + const struct drm_display_mode *mode, + const struct dcp_display_metadata *metadata) +{ + struct detailed_pixel_timing *pixel = &timing->data.pixel_data; + u32 hblank, vblank, hoff, voff, hsync, vsync; + u32 width = metadata->width_cm * 10, height = metadata->height_cm * 10; + u32 hflags = mode->flags & (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC); + u32 vflags = mode->flags & (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC); + + if (mode->clock <= 0 || mode->clock > 655350 || mode->clock % 10 || + mode->hdisplay <= 0 || mode->vdisplay <= 0 || + mode->hdisplay > 4095 || mode->vdisplay > 4095 || + mode->htotal <= mode->hdisplay || mode->vtotal <= mode->vdisplay || + mode->hsync_start < mode->hdisplay || mode->hsync_end <= mode->hsync_start || + mode->hsync_end > mode->htotal || mode->vsync_start < mode->vdisplay || + mode->vsync_end <= mode->vsync_start || mode->vsync_end > mode->vtotal || + (hflags != DRM_MODE_FLAG_PHSYNC && hflags != DRM_MODE_FLAG_NHSYNC) || + (vflags != DRM_MODE_FLAG_PVSYNC && vflags != DRM_MODE_FLAG_NVSYNC) || + (mode->flags & ~(DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC | + DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) + return -EINVAL; + hblank = mode->htotal - mode->hdisplay; + vblank = mode->vtotal - mode->vdisplay; + hoff = mode->hsync_start - mode->hdisplay; + voff = mode->vsync_start - mode->vdisplay; + hsync = mode->hsync_end - mode->hsync_start; + vsync = mode->vsync_end - mode->vsync_start; + if (hblank > 4095 || vblank > 4095 || hoff > 1023 || hsync > 1023 || + voff > 63 || vsync > 63 || width > 4095 || height > 4095) + return -EINVAL; + + timing->pixel_clock = cpu_to_le16(mode->clock / 10); + pixel->hactive_lo = mode->hdisplay; + pixel->hblank_lo = hblank; + pixel->hactive_hblank_hi = (mode->hdisplay >> 8) << 4 | hblank >> 8; + pixel->vactive_lo = mode->vdisplay; + pixel->vblank_lo = vblank; + pixel->vactive_vblank_hi = (mode->vdisplay >> 8) << 4 | vblank >> 8; + pixel->hsync_offset_lo = hoff; + pixel->hsync_pulse_width_lo = hsync; + pixel->vsync_offset_pulse_width_lo = (voff & 15) << 4 | (vsync & 15); + pixel->hsync_vsync_offset_pulse_width_hi = (hoff >> 8) << 6 | (hsync >> 8) << 4 | + (voff >> 4) << 2 | vsync >> 4; + pixel->width_mm_lo = width; + pixel->height_mm_lo = height; + pixel->width_height_mm_hi = (width >> 8) << 4 | height >> 8; + pixel->misc = DRM_EDID_PT_SEPARATE_SYNC; + if (hflags == DRM_MODE_FLAG_PHSYNC) + pixel->misc |= DRM_EDID_PT_HSYNC_POSITIVE; + if (vflags == DRM_MODE_FLAG_PVSYNC) + pixel->misc |= DRM_EDID_PT_VSYNC_POSITIVE; + return 0; +} + +static int dcp_metadata_text(struct detailed_timing *timing, u8 type, const char *text) +{ + size_t length = strnlen(text, 14); + u8 *out = timing->data.other_data.data.str.str; + + if (!length || length > 13) + return -EINVAL; + for (size_t i = 0; i < length; i++) + if (text[i] < 0x20 || text[i] > 0x7e) + return -EINVAL; + timing->data.other_data.type = type; + memset(out, ' ', 13); + memcpy(out, text, length); + if (length < 13) + out[length] = '\n'; + return 0; +} + +static s64 dcp_metadata_side(const u32 *a, const u32 *b, const u32 *point) +{ + return ((s64)b[0] - a[0]) * ((s64)point[1] - a[1]) - + ((s64)b[1] - a[1]) * ((s64)point[0] - a[0]); +} + +/* Build only a base block: the authoritative firmware mode list stays separate. */ +static int dcp_metadata_block(struct edid *edid, const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + u32 coordinate[8], manufacturer = 0, gamma; + u8 checksum = 0; + int ret; + + if (!metadata->width_cm || metadata->width_cm > 255 || + !metadata->height_cm || metadata->height_cm > 255 || + metadata->product > U16_MAX || metadata->week > 53 || + metadata->year < 1990 || metadata->year > 2245 || + metadata->gamma < 65536 || metadata->gamma > (354U << 16) / 100 || + strnlen(metadata->manufacturer, 4) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + if (metadata->manufacturer[i] < 'A' || metadata->manufacturer[i] > 'Z') + return -EINVAL; + manufacturer = manufacturer << 5 | (metadata->manufacturer[i] - '@'); + } + for (unsigned int i = 0; i < 8; i++) { + u32 value = i < 6 ? metadata->chromaticity[i / 2][i % 2] : metadata->white[i - 6]; + + if (!value || value >= 65536) + return -EINVAL; + coordinate[i] = DIV_ROUND_CLOSEST(value, 64); + if (!coordinate[i] || coordinate[i] > 1023) + return -EINVAL; + if ((i & 1) && coordinate[i - 1] + coordinate[i] > 1024) + return -EINVAL; + } + /* Degenerate primaries or a white point outside their triangle are not usable. */ + { + s64 area = dcp_metadata_side(coordinate, coordinate + 2, coordinate + 4); + + if (!area) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) { + s64 side = dcp_metadata_side(coordinate + 2 * i, + coordinate + 2 * ((i + 1) % 3), coordinate + 6); + + if ((area > 0 && side < 0) || (area < 0 && side > 0)) + return -EINVAL; + } + } + gamma = DIV_ROUND_CLOSEST(metadata->gamma * 100, 65536) - 100; + if (gamma > 254) + return -EINVAL; + + memset(edid, 0, sizeof(*edid)); + memcpy(edid->header, "\x00\xff\xff\xff\xff\xff\xff\x00", 8); + edid->product_id.manufacturer_name = cpu_to_be16(manufacturer); + edid->product_id.product_code = cpu_to_le16(metadata->product); + edid->product_id.serial_number = cpu_to_le32(metadata->serial); + edid->product_id.week_of_manufacture = metadata->week; + edid->product_id.year_of_manufacture = metadata->year - 1990; + edid->version = 1; + edid->revision = 4; + /* No claim about connector interface, bit depth, HDR, audio or VRR. */ + edid->input = DRM_EDID_INPUT_DIGITAL; + edid->width_cm = metadata->width_cm; + edid->height_cm = metadata->height_cm; + edid->gamma = gamma; + edid->features = DRM_EDID_FEATURE_PREFERRED_TIMING | DRM_EDID_FEATURE_RGB; + edid->red_green_lo = (coordinate[0] & 3) << 6 | (coordinate[1] & 3) << 4 | + (coordinate[2] & 3) << 2 | (coordinate[3] & 3); + edid->blue_white_lo = (coordinate[4] & 3) << 6 | (coordinate[5] & 3) << 4 | + (coordinate[6] & 3) << 2 | (coordinate[7] & 3); + edid->red_x = coordinate[0] >> 2; + edid->red_y = coordinate[1] >> 2; + edid->green_x = coordinate[2] >> 2; + edid->green_y = coordinate[3] >> 2; + edid->blue_x = coordinate[4] >> 2; + edid->blue_y = coordinate[5] >> 2; + edid->white_x = coordinate[6] >> 2; + edid->white_y = coordinate[7] >> 2; + memset(edid->standard_timings, 1, sizeof(edid->standard_timings)); + ret = dcp_metadata_timing(&edid->detailed_timings[0], mode, metadata); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[1], EDID_DETAIL_MONITOR_NAME, + metadata->name); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[2], EDID_DETAIL_MONITOR_STRING, + "DCP metadata"); + if (!ret) + ret = dcp_metadata_text(&edid->detailed_timings[3], + metadata->serial_text[0] ? EDID_DETAIL_MONITOR_SERIAL : + EDID_DETAIL_MONITOR_STRING, + metadata->serial_text[0] ? + metadata->serial_text : "FW derived"); + if (ret) + return ret; + for (unsigned int i = 0; i < sizeof(*edid) - 1; i++) + checksum += ((const u8 *)edid)[i]; + edid->checksum = -checksum; + return 0; +} + +const struct drm_edid *dcp_metadata_edid(const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode) +{ + struct edid block; + const struct drm_edid *edid; + int ret = dcp_metadata_block(&block, metadata, mode); + + if (ret) + return ERR_PTR(ret); + edid = drm_edid_alloc(&block, sizeof(block)); + if (!edid) + return ERR_PTR(-ENOMEM); + if (!drm_edid_valid(edid)) { + drm_edid_free(edid); + return ERR_PTR(-EINVAL); + } + return edid; +} diff --git a/drivers/gpu/drm/apple/dcp-display-metadata.h b/drivers/gpu/drm/apple/dcp-display-metadata.h new file mode 100644 index 00000000000000..537994420ebd6e --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-display-metadata.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_DISPLAY_METADATA_H +#define APPLE_DCP_DISPLAY_METADATA_H + +#include + +/* Values reported by DisplayAttributes, not a copy of the sink's EDID. */ +struct dcp_display_metadata { + u32 width_cm, height_cm; + u32 chromaticity[3][2]; + u32 white[2]; + u32 gamma; + u32 product, serial, week, year; + char manufacturer[4]; + char name[14]; + char serial_text[14]; +}; + +struct drm_edid; +struct drm_display_mode; +const struct drm_edid *dcp_metadata_edid(const struct dcp_display_metadata *metadata, + const struct drm_display_mode *mode); + +#endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 2db7fb21f57618..895b2afd2f421d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -20,6 +20,8 @@ #include "dcp-fabric-session.h" #include "dptxep.h" #include "dcp_backlight.h" +#include "dcp-display-metadata.h" +#include "dcp-metadata-epoch.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb-state.h" @@ -99,6 +101,9 @@ struct dcp_chunks { u64 modes_generation; size_t length; void *data; + size_t received; + bool contiguous; + struct dcp_metadata_ticket metadata_ticket; }; #define DCP_MAX_MAPPINGS (128) /* should be enough */ @@ -331,6 +336,15 @@ struct apple_dcp { /* Active chunked transfer. There can only be one at a time. */ struct dcp_chunks chunks; + struct { + /* Protects connection tickets and cached metadata snapshots. */ + spinlock_t lock; + struct dcp_metadata_epoch epoch; + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + struct dcp_metadata_ticket attributes_ticket, timing_ticket; + bool attributes_valid, timing_valid, hpd_seen; + } metadata; /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ union { @@ -595,6 +609,15 @@ struct apple_dcp { } iomfb; }; +void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); +void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected); +void dcp_firmware_metadata_begin(struct apple_dcp *dcp); +void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, + const struct dcp_display_metadata *attributes); +void dcp_firmware_metadata_timing(struct apple_dcp *dcp); +bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation); +const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation); + void iomfb_scanout_complete_h17p(struct apple_dcp *dcp); void iomfb_present_backlight_h17p(struct apple_dcp *dcp); bool iomfb_present_brightness_only_h17p(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp-metadata-epoch.h b/drivers/gpu/drm/apple/dcp-metadata-epoch.h new file mode 100644 index 00000000000000..c2ab0c22a74e0b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-metadata-epoch.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_METADATA_EPOCH_H +#define APPLE_DCP_METADATA_EPOCH_H + +#include + +/* Caller serializes these fields with the metadata snapshot lock. */ +struct dcp_metadata_epoch { + u64 generation; + bool connected; + bool associated_once; + bool invalidated; +}; + +struct dcp_metadata_ticket { + u64 generation; + bool associated; +}; + +static inline void dcp_metadata_epoch_connection(struct dcp_metadata_epoch *epoch, + u64 generation, bool connected) +{ + if (epoch->associated_once && + (!connected || generation != epoch->generation)) + epoch->invalidated = true; + if (connected) + epoch->associated_once = true; + epoch->generation = generation; + epoch->connected = connected; +} + +static inline struct dcp_metadata_ticket +dcp_metadata_begin(const struct dcp_metadata_epoch *epoch) +{ + return (struct dcp_metadata_ticket) { + .generation = epoch->generation, + .associated = epoch->connected && !epoch->invalidated, + }; +} + +static inline bool dcp_metadata_matches(const struct dcp_metadata_epoch *epoch, + struct dcp_metadata_ticket ticket) +{ + return ticket.associated && epoch->connected && !epoch->invalidated && + ticket.generation == epoch->generation; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 27c5554f718b81..07d0c676a16cbd 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1668,6 +1668,169 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } +static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; +} + +void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (generation != dcp->metadata.epoch.generation || !connected) { + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; + dcp->metadata.hpd_seen = false; + } + dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (connected) { + dcp->metadata.hpd_seen = true; + } else if (dcp->metadata.hpd_seen) { + dcp_metadata_epoch_connection(&dcp->metadata.epoch, + dcp->metadata.epoch.generation, false); + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +void dcp_firmware_metadata_begin(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp->chunks.received = 0; + dcp->chunks.contiguous = true; +} + +static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) +{ + unsigned long flags; + bool ready; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (ready && dcp->connector && READ_ONCE(dcp->connector->connected)) + schedule_work(&dcp->connector->hotplug_wq); +} + +static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) +{ + return dcp->chunks.contiguous && dcp->chunks.received == dcp->chunks.length && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); +} + +void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, + const struct dcp_display_metadata *attributes) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) { + dcp->metadata.attributes_valid = attributes && dcp_firmware_metadata_complete(dcp); + if (dcp->metadata.attributes_valid) { + dcp->metadata.attributes = *attributes; + dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; + } + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); +} + +void dcp_firmware_metadata_timing(struct apple_dcp *dcp) +{ + unsigned long flags; + unsigned int i; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (!dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) + goto unlock; + dcp->metadata.timing_valid = false; + if (!dcp_firmware_metadata_complete(dcp)) + goto unlock; + for (i = 0; i < dcp->nr_modes; i++) { + if (!(dcp->modes[i].mode.type & DRM_MODE_TYPE_PREFERRED) || + !dcp->modes[i].metadata_timing_valid) + continue; + dcp->metadata.preferred = dcp->modes[i].mode; + dcp->metadata.timing_ticket = dcp->chunks.metadata_ticket; + dcp->metadata.timing_valid = true; + break; + } +unlock: + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); +} + +bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation) +{ + unsigned long flags; + bool valid; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return valid; +} + +const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) +{ + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + const struct drm_edid *edid; + unsigned long flags; + bool valid; + + if (!dcp_firmware_metadata_enabled(dcp)) + return ERR_PTR(-ENODEV); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + if (valid) { + attributes = dcp->metadata.attributes; + preferred = dcp->metadata.preferred; + *generation = dcp->metadata.epoch.generation; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid) + return ERR_PTR(-ENODATA); + edid = dcp_metadata_edid(&attributes, &preferred); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + *generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid && !IS_ERR(edid)) { + drm_edid_free(edid); + return ERR_PTR(-ESTALE); + } + return edid; +} + +void dcp_set_dimensions(struct apple_dcp *dcp) void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation) { struct apple_connector *apple_connector = READ_ONCE(dcp->connector); @@ -3031,6 +3194,13 @@ static void dcp_typec_reconnect_work(struct work_struct *work) static void disconnected_hpd_event(struct apple_connector *con) { + if (!con) + return; + + /* May run with DRM modeset locks held; defer property/cache cleanup. */ + WRITE_ONCE(con->connected, false); + WRITE_ONCE(con->edid_invalidation_pending, true); + schedule_work(&con->hotplug_wq); apple_connector_edid_set_live(con, false); if (con && con->connected) { struct platform_device *pdev = READ_ONCE(con->dcp); @@ -3125,6 +3295,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) guard(mutex)(&dcp->hpd_mutex); dcp->typec_generation++; WRITE_ONCE(dcp->typec_cable_connected, true); + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); dcp->typec_reconnect_tries = 0; dcp->placeholder_retried = false; cancel_delayed_work(&dcp->typec_reconnect_wq); @@ -3157,6 +3328,7 @@ int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) /* Nothing left to retry for the display that went. */ if (dcp->external_native) cancel_delayed_work(&dcp->external_retry_wq); + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } WRITE_ONCE(dcp->typec_crtc_off, false); reinit_completion(&dcp->typec_iomfb_hpd_ready); @@ -4447,6 +4619,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) if (release_cable) WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); @@ -5017,6 +5190,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) INIT_WORK(&dcp->external_stop_work, dcp_external_stop_work); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); + spin_lock_init(&dcp->metadata.lock); spin_lock_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 99253dd7ee32d5..ef63e40b7422b5 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -822,6 +822,23 @@ void dcp_route_failure_notify(struct apple_connector *connector) schedule_work(&connector->hotplug_wq); } +static void dcp_clear_connector_edid(struct apple_connector *connector) +{ + lockdep_assert_held(&connector->base.dev->mode_config.mutex); + drm_edid_connector_update(&connector->base, NULL); + drm_edid_free(connector->drm_edid); + connector->drm_edid = NULL; + connector->drm_edid_firmware_derived = false; +} + +static void dcp_consume_edid_invalidation(struct apple_connector *connector) +{ + lockdep_assert_held(&connector->base.dev->mode_config.mutex); + /* Preserve a disconnect even if the port reconnected before probing. */ + if (xchg(&connector->edid_invalidation_pending, false)) + dcp_clear_connector_edid(connector); +} + void dcp_hotplug(struct work_struct *work) { struct apple_connector *connector; @@ -832,12 +849,21 @@ void dcp_hotplug(struct work_struct *work) int ret; connector = container_of(work, struct apple_connector, hotplug_wq); + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) + dcp_consume_edid_invalidation(connector); reasons = atomic_xchg(&connector->hotplug_reasons, 0); notify_only = (reasons & DCP_HOTPLUG_ROUTE_FAILURE) && !(reasons & DCP_HOTPLUG_READY); pdev = READ_ONCE(connector->dcp); if (!pdev) { /* a Type-C port unrouted after this was queued */ + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (!READ_ONCE(connector->dcp) && connector->drm_edid_firmware_derived) { + drm_edid_connector_update(&connector->base, NULL); + connector->drm_edid_firmware_derived = false; + } + } + drm_kms_helper_connector_hotplug_event(&connector->base); apple_connector_invalidate_edid(connector); dcp_notify_hotplug(connector); apple_connector_backlight_sync(connector); @@ -847,6 +873,10 @@ void dcp_hotplug(struct work_struct *work) dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); + scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (!READ_ONCE(connector->connected)) + dcp_clear_connector_edid(connector); + } if (!connector->connected) apple_connector_invalidate_edid(connector); @@ -1273,8 +1303,14 @@ int dcp_get_modes(struct drm_connector *connector) unsigned int count; int i; + dcp_consume_edid_invalidation(apple_connector); + /* A Type-C port has no pipeline while the fabric is moving it. */ if (!pdev) { + if (apple_connector->drm_edid_firmware_derived) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } drm_edid_connector_update(connector, NULL); return 0; } @@ -1336,6 +1372,36 @@ int dcp_get_modes(struct drm_connector *connector) apple_connector_edid_install(apple_connector, pdev, generation, edid); } + } + if (dcp->nr_modes && apple_connector->drm_edid) { + apple_connector->drm_edid_firmware_derived = false; + drm_edid_connector_update(connector, apple_connector->drm_edid); + dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + } else { + u64 generation = 0; + const struct drm_edid *derived = dcp->nr_modes ? + dcp_firmware_edid(dcp, &generation) : ERR_PTR(-ENODATA); + + if (!IS_ERR(derived)) { + bool log = !apple_connector->drm_edid_firmware_derived || + apple_connector->drm_edid_generation != generation; + int update; + + apple_connector->drm_edid_firmware_derived = true; + apple_connector->drm_edid_generation = generation; + update = drm_edid_connector_update(connector, derived); + drm_edid_free(derived); + if (update || !dcp_firmware_metadata_current(dcp, generation)) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } else if (log) { + dev_info(dcp->dev, "using firmware-derived EDID metadata for connection %llu\n", + generation); + } + } else if (apple_connector->drm_edid_firmware_derived) { + drm_edid_connector_update(connector, NULL); + apple_connector->drm_edid_firmware_derived = false; + } edid = apple_connector_edid_dup(apple_connector, pdev); drm_edid_connector_update(connector, edid); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 1c8b68ee3e440e..d4dfa8940ba23c 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -981,6 +981,7 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) dcp->chunks.modes_generation = dcp_modes_transfer_begin(dcp); dcp->chunks.length = *length; + dcp_firmware_metadata_begin(dcp); dcp->chunks.data = kzalloc(*length, GFP_KERNEL); if (!dcp->chunks.data) { diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 0cd3df7afed479..302cdaf74933dd 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -387,6 +387,18 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); else if (!strcmp(key, "PreciseSyncRate")) ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); + else if (polarity && !strcmp(key, "SyncPolarity")) { + if (dim->sync_polarity >= 0) + ret = -EINVAL; + else + ret = parse_int_bound(it.handle, &dim->sync_polarity, 0, 1); + } else if (polarity && !strcmp(key, "PixelRepetition")) { + if (dim->pixel_repetition >= 0) + ret = -EINVAL; + else + ret = parse_int_bound(it.handle, &dim->pixel_repetition, + 0, U32_MAX); + } else ret = skip(it.handle); @@ -680,6 +692,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; + bool preferred = false, interlaced = false; + bool have_preferred = false, have_interlaced = false; struct drm_display_mode *mode = &out->mode; *out = (struct dcp_display_mode) {}; @@ -722,6 +736,20 @@ static int parse_mode(struct dcp_parse_ctx *handle, ret = parse_int(it.handle, &id); else if (!strcmp(key, "IsVirtual")) ret = parse_bool(it.handle, &is_virtual); + else if (external_sdr && !strcmp(key, "IsPreferred")) { + if (have_preferred) + ret = -EINVAL; + else + ret = parse_bool(it.handle, &preferred); + have_preferred = true; + } else if (external_sdr && !strcmp(key, "IsInterlaced")) { + if (have_interlaced) + ret = -EINVAL; + else + ret = parse_bool(it.handle, &interlaced); + have_interlaced = true; + } else if (!strcmp(key, "Score")) + ret = parse_int_bound(it.handle, score, 0, S64_MAX); else if (!strcmp(key, "Score")) ret = parse_int(it.handle, score); else @@ -813,6 +841,12 @@ static int parse_mode(struct dcp_parse_ctx *handle, .height_mm = height_mm, }; + if (external_sdr && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0) + mode->flags = (horiz.sync_polarity ? DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC) | + (vert.sync_polarity ? DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC); + out->metadata_timing_valid = external_sdr && have_preferred && preferred && + have_interlaced && !interlaced && horiz.pixel_repetition == 0 && + vert.pixel_repetition == 0 && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0; drm_mode_set_name(mode); out->timing_mode_id = id; @@ -968,6 +1002,238 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, return 0; } +/* Metadata strings must fit an EDID text descriptor without hidden suffixes. */ +static int parse_metadata_string(struct dcp_parse_ctx *handle, char *out, size_t size) +{ + const struct dcp_parse_tag *tag = parse_tag_of_type(handle, DCP_TYPE_STRING); + const u8 *value; + + if (IS_ERR(tag)) + return PTR_ERR(tag); + if (!tag->size || tag->size >= size) + return -EINVAL; + value = parse_bytes(handle, tag->size); + if (IS_ERR(value)) + return PTR_ERR(value); + for (unsigned int i = 0; i < tag->size; i++) + if (value[i] < 0x20 || value[i] > 0x7e) + return -EINVAL; + memcpy(out, value, tag->size); + out[tag->size] = 0; + return 0; +} + +static int parse_metadata_number(struct dcp_parse_ctx *handle, u32 *out, u32 min, u32 max) +{ + s64 value; + int ret = parse_int_bound(handle, &value, min, max); + + if (!ret) + *out = value; + return ret; +} + +static int parse_metadata_point(struct dcp_parse_ctx *handle, u32 point[2], u32 *gamma) +{ + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "X")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_number(handle, &point[0], 1, 65535); + } else if (!strcmp(key, "Y")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_number(handle, &point[1], 1, 65535); + } else if (gamma && !strcmp(key, "Gamma")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_number(handle, gamma, 65536, (354U << 16) / 100); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if (seen != (gamma ? 7 : 3) || point[0] + point[1] > 65536) + return -EINVAL; + return 0; +} + +static int parse_metadata_chromaticity(struct dcp_parse_ctx *handle, + struct dcp_display_metadata *out) +{ + static const char * const names[] = { "Red", "Green", "Blue" }; + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + unsigned int color; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + for (color = 0; color < ARRAY_SIZE(names); color++) + if (!strcmp(key, names[color])) + break; + if (color < ARRAY_SIZE(names)) { + if (seen & 1U << color) + return -EINVAL; + seen |= 1U << color; + ret = parse_metadata_point(handle, out->chromaticity[color], NULL); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + return seen == 7 ? 0 : -EINVAL; +} + +static int parse_metadata_product(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out) +{ + struct iterator it; + unsigned int seen = 0; + u32 legacy = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "ManufacturerID")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_string(handle, out->manufacturer, + sizeof(out->manufacturer)); + } else if (!strcmp(key, "ProductName")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_string(handle, out->name, sizeof(out->name)); + } else if (!strcmp(key, "ProductID")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_number(handle, &out->product, 0, U16_MAX); + } else if (!strcmp(key, "WeekOfManufacture")) { + if (seen & 8) + return -EINVAL; + seen |= 8; + ret = parse_metadata_number(handle, &out->week, 0, 53); + } else if (!strcmp(key, "YearOfManufacture")) { + if (seen & 16) + return -EINVAL; + seen |= 16; + ret = parse_metadata_number(handle, &out->year, 1990, 2245); + } else if (!strcmp(key, "SerialNumber")) { + if (seen & 64) + return -EINVAL; + seen |= 64; + ret = parse_metadata_number(handle, &out->serial, 0, U32_MAX); + } else if (!strcmp(key, "AlphanumericSerialNumber")) { + if (seen & 128) + return -EINVAL; + seen |= 128; + ret = parse_metadata_string(handle, out->serial_text, + sizeof(out->serial_text)); + } else if (!strcmp(key, "LegacyManufacturerID")) { + if (seen & 32) + return -EINVAL; + seen |= 32; + ret = parse_metadata_number(handle, &legacy, 0, 0x7fff); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if ((seen & 31) != 31 || strlen(out->manufacturer) != 3) + return -EINVAL; + for (unsigned int i = 0; i < 3; i++) + if (out->manufacturer[i] < 'A' || out->manufacturer[i] > 'Z') + return -EINVAL; + if ((seen & 32) && legacy != ((out->manufacturer[0] - '@') << 10 | + (out->manufacturer[1] - '@') << 5 | + (out->manufacturer[2] - '@'))) + return -EINVAL; + return 0; +} + +int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out) +{ + struct dcp_display_metadata parsed = {}; + struct iterator it; + unsigned int seen = 0; + int ret = iterator_begin(handle, &it, true); + + if (ret) + return ret; + for (; it.idx < it.len; it.idx++) { + char key[64]; + + ret = parse_metadata_string(handle, key, sizeof(key)); + if (ret) + return ret; + if (!strcmp(key, "MaxHorizontalImageSize")) { + if (seen & 1) + return -EINVAL; + seen |= 1; + ret = parse_metadata_number(handle, &parsed.width_cm, 1, 255); + } else if (!strcmp(key, "MaxVerticalImageSize")) { + if (seen & 2) + return -EINVAL; + seen |= 2; + ret = parse_metadata_number(handle, &parsed.height_cm, 1, 255); + } else if (!strcmp(key, "Chromaticity")) { + if (seen & 4) + return -EINVAL; + seen |= 4; + ret = parse_metadata_chromaticity(handle, &parsed); + } else if (!strcmp(key, "DefaultWhitePoint")) { + if (seen & 8) + return -EINVAL; + seen |= 8; + ret = parse_metadata_point(handle, parsed.white, &parsed.gamma); + } else if (!strcmp(key, "ProductAttributes")) { + if (seen & 16) + return -EINVAL; + seen |= 16; + ret = parse_metadata_product(handle, &parsed); + } else { + ret = skip(handle); + } + if (ret) + return ret; + } + if (seen != 31) + return -EINVAL; + *out = parsed; + return 0; +} + int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, const char **class, s64 *unit, bool h17p_keys) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 88685e75129fbf..74f020814fd5c0 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -7,6 +7,8 @@ /* For mode parsing */ #include +#include "dcp-display-metadata.h" + struct apple_dcp; struct apple_connector; struct apple_dcp_typec_route; @@ -96,11 +98,14 @@ struct dcp_display_mode { struct dcp_color_mode sdr; struct dcp_color_mode best; bool vrr; + bool metadata_timing_valid; }; struct dimension { s64 total, front_porch, sync_width, active; s64 precise_sync_rate; + s64 sync_polarity; + s64 pixel_repetition; }; int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); @@ -108,6 +113,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, bool internal, bool external_sdr); +int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); bool internal); /* Caller holds modes_lock while checking connector ownership. */ bool dcp_modes_for_connector(struct apple_dcp *dcp, From 6855b636efe0430cb7da8040ff9d0b29fd5a2206 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:54:54 +1100 Subject: [PATCH 430/631] drm/apple: trace external metadata admission state Record bounded connection tickets and validity flags while holding the metadata lock. Trace initial association, property receipt and probing to distinguish an ineligible connection from incomplete or rejected metadata without logging monitor contents or weakening admission. The tracepoint is disabled by default. Exercise the observation path in the existing metadata lifetime test. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 25 ++++++++++++++++++++++++- drivers/gpu/drm/apple/trace.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 07d0c676a16cbd..1b1fddec8fe4a6 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1673,6 +1673,23 @@ static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; } +/* Called under metadata.lock; no monitor contents or pointers are exposed. */ +static void dcp_trace_metadata(struct apple_dcp *dcp, const char *phase, bool valid) +{ + u32 flags = (dcp->metadata.epoch.connected ? BIT(0) : 0) | + (dcp->metadata.epoch.associated_once ? BIT(1) : 0) | + (dcp->metadata.epoch.invalidated ? BIT(2) : 0) | + (dcp->metadata.attributes_valid ? BIT(3) : 0) | + (dcp->metadata.timing_valid ? BIT(4) : 0) | + (dcp->chunks.metadata_ticket.associated ? BIT(5) : 0) | + (dcp->chunks.contiguous ? BIT(6) : 0) | (valid ? BIT(7) : 0); + + if (dcp_firmware_metadata_enabled(dcp)) + trace_iomfb_metadata(dcp, phase, dcp->metadata.epoch.generation, + dcp->chunks.metadata_ticket.generation, flags, + dcp->chunks.received, dcp->chunks.length); +} + void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) { unsigned long flags; @@ -1686,6 +1703,7 @@ void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, boo dcp->metadata.hpd_seen = false; } dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); + dcp_trace_metadata(dcp, "connection", connected); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1704,6 +1722,7 @@ void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) dcp->metadata.attributes_valid = false; dcp->metadata.timing_valid = false; } + dcp_trace_metadata(dcp, "hotplug", connected); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1713,9 +1732,10 @@ void dcp_firmware_metadata_begin(struct apple_dcp *dcp) spin_lock_irqsave(&dcp->metadata.lock, flags); dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); - spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp->chunks.received = 0; dcp->chunks.contiguous = true; + dcp_trace_metadata(dcp, "begin", dcp->chunks.metadata_ticket.associated); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); } static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) @@ -1753,6 +1773,7 @@ void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; } } + dcp_trace_metadata(dcp, "attributes", !!attributes); spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_firmware_metadata_notify(dcp); } @@ -1780,6 +1801,7 @@ void dcp_firmware_metadata_timing(struct apple_dcp *dcp) break; } unlock: + dcp_trace_metadata(dcp, "timing", dcp->metadata.timing_valid); spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_firmware_metadata_notify(dcp); } @@ -1810,6 +1832,7 @@ const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + dcp_trace_metadata(dcp, "probe", valid); if (valid) { attributes = dcp->metadata.attributes; preferred = dcp->metadata.preferred; diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index d19609f116f2c4..0af0e0d0d654a9 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -76,6 +76,34 @@ { DPTX_APCALL_DEVICE_BUSY_TIMEOUT, "device_busy_timeout" }, \ { DPTX_APCALL_DEVICE_NOT_STARTED, "device_not_started" }) +TRACE_EVENT(iomfb_metadata, + TP_PROTO(struct apple_dcp *dcp, const char *phase, u64 generation, + u64 ticket_generation, u32 flags, u32 received, u32 length), + TP_ARGS(dcp, phase, generation, ticket_generation, flags, received, length), + TP_STRUCT__entry( + __string(devname, dev_name(dcp->dev)) + __string(phase, phase) + __field(u64, generation) + __field(u64, ticket_generation) + __field(u32, flags) + __field(u32, received) + __field(u32, length) + ), + TP_fast_assign( + __assign_str(devname); + __assign_str(phase); + __entry->generation = generation; + __entry->ticket_generation = ticket_generation; + __entry->flags = flags; + __entry->received = received; + __entry->length = length; + ), + TP_printk("%s: %s generation=%llu ticket=%llu flags=%#x received=%u length=%u", + __get_str(devname), __get_str(phase), __entry->generation, + __entry->ticket_generation, __entry->flags, + __entry->received, __entry->length) +); + TRACE_EVENT(dcp_recv_msg, TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), TP_ARGS(dcp, endpoint, message), From e237f1fc2c3177a08416be269f70a4e97ef4f126 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 20:59:53 +1100 Subject: [PATCH 431/631] drm/apple: validate delivered property extent before metadata admission H17P advertises one extra byte after the word-aligned serialized property, but does not send that byte. Requiring received == advertised discarded complete same-connection TimingElements and DisplayAttributes. Validate the complete serialized object against bytes actually received. Accept exact lengths or the observed one-byte framing convention only; reject gaps, excess trailing content and truncated values. Perform the bounded structural walk outside the metadata spinlock, then retain the existing connection-ticket checks. No absent byte is used as payload. Exercise captured property extents, truncation mutations and a missing final string byte alongside metadata lifetime and decoder tests. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 6 ++++-- drivers/gpu/drm/apple/parser.c | 17 +++++++++++++++++ drivers/gpu/drm/apple/parser.h | 1 + 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 895b2afd2f421d..e7bcd5d7dd3a9d 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -103,6 +103,7 @@ struct dcp_chunks { void *data; size_t received; bool contiguous; + bool payload_complete; struct dcp_metadata_ticket metadata_ticket; }; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 1b1fddec8fe4a6..d6d659551dda4a 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1682,7 +1682,8 @@ static void dcp_trace_metadata(struct apple_dcp *dcp, const char *phase, bool va (dcp->metadata.attributes_valid ? BIT(3) : 0) | (dcp->metadata.timing_valid ? BIT(4) : 0) | (dcp->chunks.metadata_ticket.associated ? BIT(5) : 0) | - (dcp->chunks.contiguous ? BIT(6) : 0) | (valid ? BIT(7) : 0); + (dcp->chunks.contiguous ? BIT(6) : 0) | (valid ? BIT(7) : 0) | + (dcp->chunks.payload_complete ? BIT(8) : 0); if (dcp_firmware_metadata_enabled(dcp)) trace_iomfb_metadata(dcp, phase, dcp->metadata.epoch.generation, @@ -1734,6 +1735,7 @@ void dcp_firmware_metadata_begin(struct apple_dcp *dcp) dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); dcp->chunks.received = 0; dcp->chunks.contiguous = true; + dcp->chunks.payload_complete = false; dcp_trace_metadata(dcp, "begin", dcp->chunks.metadata_ticket.associated); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1754,7 +1756,7 @@ static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) { - return dcp->chunks.contiguous && dcp->chunks.received == dcp->chunks.length && + return dcp->chunks.contiguous && dcp->chunks.payload_complete && dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); } diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 302cdaf74933dd..5cd3849d44d3dc 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -342,6 +342,23 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) return 0; } +bool dcp_property_complete(const void *blob, size_t received, size_t advertised) +{ + struct dcp_parse_ctx ctx; + + if (!blob || !received || received > U32_MAX - 3 || + received > advertised || advertised - received > 1) + return false; + /* H17P may count one unsent byte after the aligned serialized object. */ + if (advertised != received && (received & 3)) + return false; + /* Bound decoding to bytes actually delivered, never allocation padding. */ + if (parse(blob, received, &ctx) || skip(&ctx)) + return false; + return ctx.pos == received || round_up(ctx.pos, 4) == received; +} + +static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool polarity) static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool require_rate) { diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 74f020814fd5c0..93fb80b05bf845 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -109,6 +109,7 @@ struct dimension { }; int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx); +bool dcp_property_complete(const void *blob, size_t received, size_t advertised); struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, From 09e72685dfa403ad7a4e94efdd11d33a96025f55 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 21:15:03 +1100 Subject: [PATCH 432/631] Merge qualified output metadata into the external lifecycle candidate Combine the public R34 output branch with the existing stop-capture and initial service-discovery fixes. Keep timing restrictions, firmware-derived monitor identity and bounded property framing intact. Fence metadata in the external path before service discovery. Duplicate connect notifications retain the physical connection ticket; physical removal invalidates it. Preserve pending EDID cache invalidation across a rapid reconnect and defer connector property cleanup to the worker. Host ASan/UBSan tests cover these interleavings. OFF replies remain unqualified and retain resources; no shutdown/reconnect qualification or production readiness is claimed by this merge. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d6d659551dda4a..c21262d6f930fe 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2928,6 +2928,10 @@ static void dcp_external_report_disconnected(struct apple_dcp *dcp) guard(mutex)(&dcp->iomfb.lock); action = dcp_mode_hotplug(&dcp->mode_state, false, dcp->connector ? &dcp->connector->connected : NULL); + if (dcp->connector) { + WRITE_ONCE(dcp->connector->edid_invalidation_pending, true); + action |= DCP_HOTPLUG_NOTIFY; + } /* The existing worker delivers DRM notifications outside our locks. */ dcp_handle_hotplug_actions(dcp, action); } @@ -3057,6 +3061,7 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } /* A missing cable is reported even when its hardware remains owned. */ dcp_external_report_disconnected(dcp); @@ -3299,17 +3304,23 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) int ret; if (dcp_is_external_h17p(dcp)) { + bool new_cable; + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); cancel_work_sync(&dcp->external_stop_work); guard(mutex)(&dcp->external_lock); - if (!dcp->external_cable) + new_cable = !dcp->external_cable; + if (new_cable) dcp->external_generation++; dcp->external_cable = true; dcp_external_publish_generation(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, true); - dcp->typec_generation++; + if (new_cable) { + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + } dcp->placeholder_retried = false; } return dcp_external_link_locked(dcp); From ec6505042c199dc61c2e77ea974d8adff4f0c12b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Mon, 12 Oct 2026 21:49:16 +1100 Subject: [PATCH 433/631] drm/apple: publish initial external discovery while presents remain paused The lifecycle gate cached the first connected HPD while waiting for DRM activation to unpause presentation. DRM could not activate a connector that had never been published, leaving the R35 external output absent even with a connected DPTX link and valid mode metadata. Publish cable-backed initial discovery before the first power attempt, while retaining the presentation pause. Sticky failures, absent cables and OFF/restart callbacks remain withheld. Receive callbacks still acquire no lifecycle mutex or issue firmware control requests. The regression test fails on the prior production function and passes with this change; it verifies that discovery does not power or unpause the pipe. ASan/UBSan lifecycle, metadata and delayed-service tests pass. Hardware activation and OFF-response qualification remain pending. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index c21262d6f930fe..d38e9a5bcb25c4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3097,6 +3097,15 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; + /* Publish discovery so DRM can activate the first CRTC; keep presents paused. */ + if (connected && READ_ONCE(dcp->external_cable) && + !READ_ONCE(dcp->external_power_attempted) && !READ_ONCE(dcp->external_error)) { + unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, + dcp->connector ? + &dcp->connector->connected : NULL); + + dcp_handle_hotplug_actions(dcp, action); + } return true; } if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { From 589c9a3ea79821669b6d07c0f97cfba316f6bcc7 Mon Sep 17 00:00:00 2001 From: nekolyanich Date: Mon, 12 Oct 2026 22:46:12 +0300 Subject: [PATCH 434/631] drm: apple: answer the DPTX port's group 1 calls On t6020 with the 13.5 firmware, DCP's DPTX port sends AP calls of group 1, commands 3 and 4, around link (re)configuration, audio prepare included. The AFK v1 notify path dropped the group, so they reached the group 0 handlers of the same numbers, SET/GET_DRIVE_SETTINGS: DCP addresses from the payload were stored as drive settings. Behind a USB-C to HDMI adapter DCP then refused the audio start ("Capabilities changed") or never confirmed the stream. Pass the group through a call_v2 op, with the signature of call_v2 on aurora's J700 test branch (85d81b76, not in aurora-wip), and keep .call for group 0 only. A group call to a service without call_v2 gets a warning and no reply: AFK has no handler and does not guess one. The DPTX port answers group 1 commands 3 and 4 with an echo of the request, with +0x00..0x03 cleared for (1, 3), and +0x10..0x13 and +0x20 to the end cleared for (1, 4). The lane count sits at +0x10 in (1, 3) and at +0x00 in (1, 4); the reply keeps it and clears the other slot. The J700 code reads (1, 3) the same way, with per-lane levels from +0x20, which the (1, 3) reply keeps; it refuses (1, 4), and its group 0 GET (0, 4) has the count at +0x00. Other groups are echoed with a rate-limited warning. Testing: MacBook Pro 16" M2 Pro (j416s), DCP firmware 13.5, a TV on a USB-C to HDMI adapter and a Dell over Thunderbolt, 8 audio starts on the TV per boot. Cold boots under the m1n1 hypervisor, aurora-wip with PRs #275-#277, the AFK slots series and a bench DT change, interleaved: with this commit the first start was accepted in 5 of 5 boots, with 0 refusals and 8 of 8 starts played; without it, 3 of 3 boots refused the first start, 2 refusals each, 4 of 8 played. The Dell had no refusals in any boot, and no group warning was logged. Natively, on a downstream tree with the same reply rule, the boot connect and 3 of 8 TV replugs reached DCP (the rest failed earlier, in the PHY): 0 refusals in 4 connects; before, 10 refusals in 5 connects. Not tested: other SoCs, other firmware, AFK v2. Assisted-by: Claude:claude-opus-5-5 checkpatch Signed-off-by: nekolyanich --- drivers/gpu/drm/apple/afk.c | 48 ++++++++++++++++++++------- drivers/gpu/drm/apple/dptxep.c | 60 +++++++++++++++++++++++++++++++++- drivers/gpu/drm/apple/trace.h | 17 ++++++---- 3 files changed, 105 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 6a5377347a9776..995e5e7746e0b7 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -629,14 +629,12 @@ static void afk_recv_handle_reply(struct apple_dcp_afkep *ep, u32 channel, dma_free_coherent(ep->dcp->dev, txlen, txbuf, txbuf_dma); } +/* A call from DCP: one word ahead of the call header AP sends. */ struct epic_std_service_ap_call { __le32 unk0; - __le32 unk1; - __le32 type; - __le32 len; - __le32 magic; - u8 _unk[48]; + struct epic_service_call call; } __attribute__((packed)); +static_assert(sizeof(struct epic_std_service_ap_call) == 68); static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, u32 type, struct epic_hdr *ehdr, @@ -679,30 +677,56 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_NOTIFY) { struct epic_std_service_ap_call *call = payload; size_t call_size; + u32 command; void *reply; + u16 group; int ret; if (payload_size < sizeof(*call)) return; - call_size = le32_to_cpu(call->len); + call_size = le32_to_cpu(call->call.data_len); if (payload_size < sizeof(*call) + call_size) return; - if (!service->ops->call) + group = le16_to_cpu(call->call.group); + command = le32_to_cpu(call->call.command); + + /* + * Command numbers are per group: a group-less .call handler + * would take another group's call for a group 0 one. Without + * call_v2 there is no handler for it, and AFK cannot guess a + * reply, so it gets none, as a call without any handler. + */ + if (!service->ops->call_v2 && (group || !service->ops->call)) { + if (group) + dev_warn_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: %s (chan:%u) apcall %u:%u has no handler, sending NO reply\n", + ep->endpoint, service->ops->name, + channel, group, command); return; + } reply = kzalloc(payload_size, GFP_KERNEL); if (!reply) return; - ret = service->ops->call(service, le32_to_cpu(call->type), - payload + sizeof(*call), call_size, - reply + sizeof(*call), call_size); + if (service->ops->call_v2) + ret = service->ops->call_v2(service, group, command, + payload + sizeof(*call), + call_size, + reply + sizeof(*call), + call_size); + else + ret = service->ops->call(service, command, + payload + sizeof(*call), + call_size, + reply + sizeof(*call), + call_size); if (ret) { dev_warn(ep->dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) apcall %u handler returned %d, sending NO reply\n", + "AFK[ep:%02x]: %s (chan:%u) apcall %u:%u handler returned %d, sending NO reply\n", ep->endpoint, service->ops->name, channel, - le32_to_cpu(call->type), ret); + group, command, ret); kfree(reply); return; } diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index ff3ab11ebbccd7..e25510e23b5bb0 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -761,7 +761,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, size_t reply_size) { struct dptx_port *dptx = service->cookie; - trace_dptxport_apcall(dptx, idx, data_size); + trace_dptxport_apcall(dptx, 0, idx, data_size); dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); if (data_size && service->ep->dcp->hw.iomfb_method_profile != @@ -983,6 +983,63 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, u3 return dptxport_call(service, command, data, size, reply, room); } +/* + * Group 1 calls 3 and 4 come around link (re)configuration, audio prepare + * included. They carry a lane count at +0x10 in (1, 3) and at +0x00 in + * (1, 4); the J700 code reads (1, 3) as drive settings with levels per + * lane from +0x20, and its group 0 GET (0, 4) has the count at +0x00. + * Both are acked with the request, (1, 3) with +0x00..0x03 + * cleared and (1, 4) with +0x10..0x13 and +0x20 to the end cleared, so + * what DCP left there does not go back. Tested on t6020 with 13.5 + * firmware: TV through a USB-C to HDMI adapter, Dell over TB. + */ +static int dptxport_call_group1(u32 command, const void *data, + size_t data_size, void *reply, size_t reply_size) +{ + size_t size = min(reply_size, data_size); + + memcpy(reply, data, size); + if (command == 3) { + memset(reply, 0, min_t(size_t, size, 0x4)); + } else if (size > 0x10) { + memset(reply + 0x10, 0, min_t(size_t, size - 0x10, 0x4)); + if (size > 0x20) + memset(reply + 0x20, 0, size - 0x20); + } + return 0; +} + +static int dptxport_group_call(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t size, + void *reply, size_t room) +{ + struct dptx_port *dptx = service->cookie; + + if (!group) + return dptxport_call(service, command, data, size, reply, room); + + trace_dptxport_apcall(dptx, group, command, size); + dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u:%u (%zu bytes)\n", + group, command, size); + if (size) + print_hex_dump_debug("DPTXPort: apcall data: ", + DUMP_PREFIX_OFFSET, 16, 1, data, + min(size, (size_t)64), true); + + if (group == 1 && (command == 3 || command == 4)) + return dptxport_call_group1(command, data, size, reply, room); + + /* + * Echo it, as group 0 acks an unhandled call: no reply might stall + * DCP's caller. + */ + dev_warn_ratelimited(service->ep->dcp->dev, + "DPTXPort: unknown AP call %u:%u, echoing it\n", + group, command); + memcpy(reply, data, min(room, size)); + return 0; +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", @@ -1048,6 +1105,7 @@ static const struct apple_epic_service_ops dptxep_v14_7_ops[] = { .name = "dcp-lpdptx-port-epic", .init = dptxport_init, .call = dptxport_call, + .call_v2 = dptxport_group_call, }, {} }; diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index 0af0e0d0d654a9..03f23ecf0a78ed 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -546,18 +546,21 @@ TRACE_EVENT(dptxport_init, TP_PROTO(struct apple_dcp *dcp, u64 unit), TRACE_EVENT( dptxport_apcall, - TP_PROTO(struct dptx_port *dptx, int idx, size_t len), - TP_ARGS(dptx, idx, len), + TP_PROTO(struct dptx_port *dptx, u16 group, int idx, size_t len), + TP_ARGS(dptx, group, idx, len), TP_STRUCT__entry(__string(devname, dev_name(dptx->service->ep->dcp->dev)) - __field(u32, unit) __field(int, idx) __field(size_t, len)), + __field(u32, unit) __field(u16, group) __field(int, idx) + __field(size_t, len)), TP_fast_assign(__assign_str(devname); - __entry->unit = dptx->unit; __entry->idx = idx; __entry->len = len;), + __entry->unit = dptx->unit; __entry->group = group; + __entry->idx = idx; __entry->len = len;), - TP_printk("%s: dptx%d: AP Call %d (%s) with len %lu", __get_str(devname), - __entry->unit, - __entry->idx, show_dptxport_apcall(__entry->idx), __entry->len)); + TP_printk("%s: dptx%d: AP Call %u:%d (%s) with len %lu", __get_str(devname), + __entry->unit, __entry->group, __entry->idx, + __entry->group ? "-" : show_dptxport_apcall(__entry->idx), + __entry->len)); TRACE_EVENT( dptxport_validate_connection, From 23e8abc2946ab8f3e0ed02ddca48295c947022d6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 08:36:22 +1100 Subject: [PATCH 435/631] Merge public legacy DPTX group replies without changing native J700 training Preserve nekolyanichs public AFK-v1 reply handling and the reviewed AFK-v2 J700 path in separate dispatch bodies selected by wire_v2. Resolve duplicate definition from automatic merge; native group bounds and unknown-group refusals remain unchanged. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dptxep.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index e25510e23b5bb0..3e529a20115c13 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -961,7 +961,7 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, &service->ep->dcp->typec_reconnect_wq, 0); } -static int dptxport_group_call(struct apple_epic_service *service, u16 group, u32 command, +static int dptxport_v2_group_call(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t size, void *reply, size_t room) { if (group == 1 && command == DPTX_APCALL_SET_DRIVE_SETTINGS) @@ -1015,6 +1015,11 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, { struct dptx_port *dptx = service->cookie; + /* Keep native AFK-v2 training and its refusal policy distinct. */ + if (service->ep->wire_v2) + return dptxport_v2_group_call(service, group, command, data, + size, reply, room); + if (!group) return dptxport_call(service, command, data, size, reply, room); From df836345f72368a8f0428174189fab2930d786df Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 09:40:54 +1100 Subject: [PATCH 436/631] drm/vc4: convert encoded match data through pointer-width integer Clang22 rejects a direct void-pointer-to-enum cast under the existing Fedora WERROR configuration. Preserve numeric generation match data by converting through uintptr_t before the enum; retain all drivers and warnings. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/vc4/vc4_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c index a14ecb7694613f..2454ce0b0bc479 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.c +++ b/drivers/gpu/drm/vc4/vc4_drv.c @@ -299,7 +299,7 @@ static int vc4_drm_bind(struct device *dev) dev->coherent_dma_mask = DMA_BIT_MASK(32); - gen = (enum vc4_gen)of_device_get_match_data(dev); + gen = (enum vc4_gen)(uintptr_t)of_device_get_match_data(dev); if (gen > VC4_GEN_4) driver = &vc5_drm_driver; From b4bace59193aba3d2af871dc597b5ae988e63768 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 09:45:37 +1100 Subject: [PATCH 437/631] drm/pl111: convert encoded match data through pointer-width integer Preserve Versatile and IM-PD1 generation values while avoiding Clang22 direct pointer-to-enum errors under the unchanged Fedora WERROR configuration. Keep all driver selections and strict compiler checks. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/pl111/pl111_versatile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c index 0d8331a3909f4d..93ef58ffc91d38 100644 --- a/drivers/gpu/drm/pl111/pl111_versatile.c +++ b/drivers/gpu/drm/pl111/pl111_versatile.c @@ -486,7 +486,7 @@ int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *p return 0; } - versatile_clcd_type = (enum versatile_clcd)clcd_id->data; + versatile_clcd_type = (enum versatile_clcd)(uintptr_t)clcd_id->data; /* Versatile Express special handling */ if (versatile_clcd_type == VEXPRESS_CLCD_V2M) { @@ -506,7 +506,7 @@ int pl111_versatile_init(struct drm_device *dev, struct pl111_drm_dev_private *p np = of_find_matching_node_and_match(NULL, impd1_clcd_of_match, &clcd_id); if (np) - versatile_clcd_type = (enum versatile_clcd)clcd_id->data; + versatile_clcd_type = (enum versatile_clcd)(uintptr_t)clcd_id->data; } map = syscon_node_to_regmap(np); From 42a410ec2bd2d032c0ea499bee58b3fdeb14d47c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 16:39:47 -0400 Subject: [PATCH 438/631] soc: apple: rtkit-helper: adopt a running T8140 MTP session T8140 MTP firmware faults or times out when it is woken after a boot stage has taken it to QUIESCED or SLEEP, so a boot stage that drives the keyboard has to hand the session over running. When the node carries apple,rtkit-handoff = "mtp-live-v1" and the ASC CPU is still running, adopt the session with apple_rtkit_init_adopted() instead of setting RUN and waking the coprocessor. The boot stage reserves the buffers the firmware holds and maps them 1:1 through the DART (memory-region with iommu-addresses), so they stay valid; the RTKit core acknowledges the inherited system endpoints without a new handshake. An adopted session cannot be restarted, so remove() leaves it running and the transport is released with apple_rtkit_free_retaining_buffers(). Without the property, or with a stopped CPU, the cold-boot path is unchanged. Co-Authored-By: Claude Opus 5.5 --- drivers/soc/apple/rtkit-helper.c | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 9235a44dd9b851..313281372a5f82 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -12,13 +12,19 @@ #include #include #include +#include #define APPLE_ASC_CPU_CONTROL 0x44 #define APPLE_ASC_CPU_CONTROL_RUN BIT(4) +/* A running T8140 MTP session described by the boot stage that started it. */ +#define APPLE_RTKIT_HELPER_HANDOFF "mtp-live-v1" + struct apple_rtkit_helper { struct device *dev; struct apple_rtkit *rtk; + /* Inherited from the boot stage; it cannot be restarted. */ + bool adopted; void __iomem *asc_base; @@ -73,6 +79,39 @@ static const struct apple_rtkit_ops apple_rtkit_helper_ops = { .shmem_destroy = apple_rtkit_helper_shmem_destroy, }; +static void apple_rtkit_helper_release(void *rtk) +{ + apple_rtkit_free_retaining_buffers(rtk); +} + +/* + * T8140 MTP firmware cannot be restarted once it has been taken to QUIESCED + * or SLEEP, so a boot stage that used it hands the session over running. It + * reserves the buffers the firmware holds and maps them 1:1 through the + * DART. Only adopt a session it described whose CPU still runs. + */ +static bool apple_rtkit_helper_live(struct apple_rtkit_helper *helper) +{ + const char *handoff; + + if (of_property_read_string(helper->dev->of_node, "apple,rtkit-handoff", + &handoff)) + return false; + + if (strcmp(handoff, APPLE_RTKIT_HELPER_HANDOFF)) { + dev_warn(helper->dev, "unknown RTKit handoff \"%s\"\n", handoff); + return false; + } + + if (!(readl_relaxed(helper->asc_base + APPLE_ASC_CPU_CONTROL) & + APPLE_ASC_CPU_CONTROL_RUN)) { + dev_warn(helper->dev, "RTKit handoff without a running CPU\n"); + return false; + } + + return true; +} + static int apple_rtkit_helper_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -103,6 +142,23 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) "Failed to map SRAM region"); } + if (apple_rtkit_helper_live(helper)) { + helper->rtk = apple_rtkit_init_adopted(dev, helper, NULL, 0, + &apple_rtkit_helper_ops); + if (IS_ERR(helper->rtk)) + return dev_err_probe(dev, PTR_ERR(helper->rtk), + "Failed to adopt RTKit"); + + ret = devm_add_action_or_reset(dev, apple_rtkit_helper_release, + helper->rtk); + if (ret) + return ret; + + helper->adopted = true; + dev_info(dev, "adopted the running RTKit session\n"); + return 0; + } + helper->rtk = devm_apple_rtkit_init(dev, helper, NULL, 0, &apple_rtkit_helper_ops); if (IS_ERR(helper->rtk)) @@ -127,6 +183,10 @@ static void apple_rtkit_helper_remove(struct platform_device *pdev) { struct apple_rtkit_helper *helper = platform_get_drvdata(pdev); + /* An inherited session would not come back from QUIESCED. */ + if (helper->adopted) + return; + if (apple_rtkit_is_running(helper->rtk)) apple_rtkit_quiesce(helper->rtk); From b00aabd59573c6579f7a517b9fe6dbb9a8b0f54f Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 19:11:08 -0400 Subject: [PATCH 439/631] arm64: dts: apple: t8140-j700: drive the SPI NOR on a single data line With spi-rx-bus-width and spi-tx-bus-width set to 4, the first command spi-nor sends at probe (READ ID) times out on J700 with every byte moved but no completion: "transfer timed out: p=0x100000 c=0x700 ... tx=0 rx=0", so no MTD device and no NVRAM partition appear and asahi-bless cannot read or write the boot volume selection. The same driver probes the flash, creates the nvram partition and lets asahi-bless list volumes when the node describes single data lines, at either 25 or 40 MHz. The ADT's spinor node gives 25 MHz and does not describe quad lanes, and the J700 kernels that probed this flash never declared them. Drop the quad widths and keep the ADT frequency. Tested on J700 (7.1.12, QSPIMC 0x20000): 25 MHz single-line probes and creates "nvram" (0x1d0000 bytes); asahi-bless --list-volumes works. 25 MHz quad fails at the first command. Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index beacbf31689573..44143ae11092eb 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -462,8 +462,6 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <25000000>; - spi-rx-bus-width = <4>; - spi-tx-bus-width = <4>; partitions { compatible = "fixed-partitions"; From 8ce34a2cb42648243690f39398062ceb50da0db1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 10:24:00 +1100 Subject: [PATCH 440/631] drm/apple: accept exact H17P power replies and republish reconnected displays Live R36 evidence records OFF state zero with zero status and a stopped external pipeline. Validate that exact echo while retaining malformed reply, timeout, error, queue drain and ownership guards. Publish discovery only for a cleanly stopped new physical cable epoch, and release a reattached link if unplugged before CRTC activation. Exercise production reply decoding and reconnect paths without the previous test-only OFF policy shim. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 13 +++++++++---- drivers/gpu/drm/apple/iomfb_h17p.c | 8 ++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index d38e9a5bcb25c4..9b70114b406634 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3004,12 +3004,12 @@ static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) return 0; if (dcp->external_error) return dcp->external_error; - if (dcp->external_stopped) + if (dcp->external_stopped && !READ_ONCE(dcp->dptxport[0].connected)) return 0; iomfb_pause_presentation(dcp, true); dcp->external_running = false; /* Route setup can call this before IOMFB or its shared memory exists. */ - if (!dcp->external_power_attempted) + if (dcp->external_stopped || !dcp->external_power_attempted) goto release_link; ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); if (!ret) @@ -3097,9 +3097,14 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; dcp->external_hotplug_connected = connected; - /* Publish discovery so DRM can activate the first CRTC; keep presents paused. */ + /* Publish initial/new-cable discovery, never same-cable DPMS wake. */ if (connected && READ_ONCE(dcp->external_cable) && - !READ_ONCE(dcp->external_power_attempted) && !READ_ONCE(dcp->external_error)) { + !READ_ONCE(dcp->external_error) && + (!READ_ONCE(dcp->external_power_attempted) || + (READ_ONCE(dcp->external_stopped) && + READ_ONCE(dcp->external_rx_generation) != + READ_ONCE(dcp->external_crtc_generation) && + READ_ONCE(dcp->dptxport[0].connected)))) { unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, dcp->connector ? &dcp->connector->connected : NULL); diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index b3ae896426e251..02bb44d76c29b2 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -725,7 +725,7 @@ struct dcp_external_power_cookie { bool on; }; -/* A473's envelope is known; no OFF reply has been qualified as a stop. */ +/* A473 echoes the accepted state with zero status; DMA ownership is unchanged. */ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void *data) { struct dcp_external_power_cookie *cookie = data; @@ -740,9 +740,9 @@ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void WRITE_ONCE(dcp->external_power_reply, reply); WRITE_ONCE(dcp->external_power_reply_valid, true); - dev_info(dcp->dev, "A473 power=%u reply=%#llx%s\n", - cookie->on, reply, cookie->on ? "" : " (stop unqualified)"); - ret = cookie->on ? (reply == 1 ? 0 : -EREMOTEIO) : -EOPNOTSUPP; + dev_info(dcp->dev, "A473 power=%u reply=%#llx\n", + cookie->on, reply); + ret = reply == (u64)cookie->on ? 0 : -EREMOTEIO; } } WRITE_ONCE(cookie->wait.status, ret); From 0358b84ac9e95a4cbe18bab8c836258e950a00ea Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Mon, 12 Oct 2026 19:11:08 -0400 Subject: [PATCH 441/631] PCI: apple: piodma-diag: decide suspend-to-idle admission in prepare Commit 2c815e33ece ("PCI: apple: preserve the retained bootstrap across suspend-to-idle sleep") admits suspend-to-idle once the bootstrap has completed, but its PM_SUSPEND_PREPARE notifier tests pm_suspend_no_platform(). enter_state() clears the suspend flags and runs these notifiers in suspend_prepare() before suspend_devices_and_enter() marks the suspend as suspend-to-idle, so the test is false for every suspend and the guard refuses s2idle too: apple-piodma-diag 390030000.piodma: sleep refused: Neo guard active during radio admission or retained DMA lifetime Leave the suspend decision to apple_piodma_pm_prepare(), which runs after the target state is set, makes the same ready/admitted check and still refuses anything but suspend-to-idle. As a supplier of the radio's PCI devices it is prepared before them. Hibernation and restore remain refused in the notifier. Co-Authored-By: Claude Opus 5.5 --- drivers/pci/controller/pcie-apple-piodma-diag.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 1ba2dcc13cab3f..6c4237826884c7 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -171,10 +171,14 @@ static int apple_piodma_pm_notify(struct notifier_block *nb, switch (action) { case PM_SUSPEND_PREPARE: - if (pm_suspend_no_platform() && READ_ONCE(diag->ready) && - smp_load_acquire(&diag->radio_admitted)) - return NOTIFY_DONE; - fallthrough; + /* + * The suspend core marks suspend-to-idle only after these + * notifiers have run, so pm_suspend_no_platform() is still + * false here for every suspend. apple_piodma_pm_prepare() + * makes the same check once the target state is known, and + * still vetoes everything but suspend-to-idle. + */ + return NOTIFY_DONE; case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: dev_warn_ratelimited(diag->dev, From 53d11d54ca97dd1528dd436885343870a65b0dfc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 11:59:20 +1100 Subject: [PATCH 442/631] drm/apple: tag H17P external SDR RGB sources as sRGB Native-tagged RGB surfaces produce washed-out external output on J700 H17P despite correct KMS pixel values. The same MSI and cable render normally in macOS and M2 Fairydust. A controlled J700 trial changing only the source tag to sRGB restores normal colours. Apply that tag to native RGB surfaces on the external H17P profile. Preserve transfer, pixel format, output modes and other profiles. No diagnostic switch is needed. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/iomfb_h17p_test.c | 8 ++++++++ drivers/gpu/drm/apple/iomfb_template.c | 12 ++++++++++++ 2 files changed, 20 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb_h17p_test.c b/drivers/gpu/drm/apple/iomfb_h17p_test.c index c017651945a4e4..c5955836700ca4 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p_test.c +++ b/drivers/gpu/drm/apple/iomfb_h17p_test.c @@ -45,6 +45,8 @@ static void h17p_surface_wire_test(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, wire->swap, &request->swap, sizeof(wire->swap)); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x593), DCP_FORMAT_BGRA); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59b], (u8)DCP_XFER_FUNC_SDR); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59c], (u8)DCP_COLORSPACE_NATIVE); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x59d), 10176U); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5a9), 2408U); KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5ad), 1506U); @@ -58,6 +60,12 @@ static void h17p_surface_wire_test(struct kunit *test) KUNIT_EXPECT_EQ(test, get_unaligned_le64((u8 *)wire + 0xe38), iova); KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 2], 0U); KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 1], 0U); + + /* Source-tag changes must reach the wire without changing transfer. */ + surface->colorspace = DCP_COLORSPACE_BG_SRGB; + iomfb_serialize_present_h17p(wire, request); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59b], (u8)DCP_XFER_FUNC_SDR); + KUNIT_EXPECT_EQ(test, ((u8 *)wire)[0x59c], (u8)DCP_COLORSPACE_BG_SRGB); } static void h17p_opaque_x_parameter_test(struct kunit *test) diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index d4dfa8940ba23c..39af92ed6b5c59 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -2328,6 +2328,18 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P external SDR RGB needs an sRGB source tag. Native + * tagging washes out the same pixel values on this profile. + * Keep YUV, other source spaces and other profiles unchanged. + */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + !new_state->fb->format->is_yuv && + req->surf[l].base.colorspace == DCP_COLORSPACE_NATIVE) + req->surf[l].base.colorspace = DCP_COLORSPACE_BG_SRGB; +#endif } if (!has_surface && !crtc_state->color_mgmt_changed) { From 2a930c188a89ffa62cd084921d2775d9178a7a1c Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 13:47:16 +1100 Subject: [PATCH 443/631] drm/apple: use the observed H17P DPAV service for fresh EDID The external H17P DPAV service is on endpoint 0x2a with the identity dcpav-service-epic / AppleDCPDP2HDMI / unit 1. Select that endpoint only for the external profile and route AFK receives by the instantiated DPAV and DPTX endpoints, preserving legacy 0x28 DPAV and 0x2a DPTX routing and native H17P DPTX at 0x2c. Admit the exact service before the existing AFKV2 OPEN. Keep the bounded read-only group 1 command 7 EDID request and its response validation. Reject in-flight EDID and cached/publication results when the physical Type-C generation changes. Invalidate that generation before waiting for firmware disconnect teardown. Release hpd_mutex during EDID requests and hold it for publication under the existing mode_config.mutex ordering. Keep the conservative firmware metadata epoch guard and service owner/reader pins. Host tests exercise actual production routing, admission, reader ownership and publication with synthetic disconnected/delayed responses. Firmware wire behavior still requires owner qualification. Complete existing kernel-doc parameter descriptions for a clean W=1 compile without warning suppression. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 1 + drivers/gpu/drm/apple/dcp.c | 27 +++++++----- drivers/gpu/drm/apple/epic/dpavservep.c | 58 ++++++++++++++++++++++++- drivers/gpu/drm/apple/epic/dpavservep.h | 3 +- drivers/gpu/drm/apple/iomfb.c | 40 ++++++++++++++++- 5 files changed, 113 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index e7bcd5d7dd3a9d..913e3902b34b1b 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -88,6 +88,7 @@ enum { DPAV_CTRL_ENDPOINT = 0x24, DPTX_CONTROLLER_ENDPOINT = 0x24, DPAVSERV_ENDPOINT = 0x28, + DPAVSERV_ENDPOINT_V2 = 0x2a, AV_ENDPOINT = 0x29, DPTX_ENDPOINT = 0x2a, DPTX_ENDPOINT_V2 = 0x2c, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 9b70114b406634..4e16861d53b779 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1937,6 +1937,11 @@ static void dcp_delayed_vblank(struct work_struct *work) static void dcp_swap_watchdog(struct work_struct *work) static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) { + /* DPAV 0x2a on H17P overlaps the legacy DPTX endpoint number. */ + if (dcp->dcpavservep && endpoint == dcp->dcpavservep->endpoint) + return dcp->dcpavservep; + if (dcp->dptxep && endpoint == dcp->dptxep->endpoint) + return dcp->dptxep; switch (endpoint) { case AV_ENDPOINT: return dcp->avep; @@ -1944,13 +1949,6 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->systemep; case DISP0_ENDPOINT: return dcp->ibootep; - case DPAVSERV_ENDPOINT: - return dcp->dcpavservep; - case DPTX_ENDPOINT: - return dcp->dptxep; - case DPTX_ENDPOINT_V2: - return dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? dcp->dptxep : NULL; case DPTX_CONTROLLER_ENDPOINT: return dcp->dptx_controller_ep; default: @@ -2077,6 +2075,10 @@ static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) ep = NULL; break; } + if (endpoint == IOMFB_ENDPOINT) + return iomfb_recv_msg(dcp, message); + + ep = dcp_afkep(dcp, endpoint); if (!ep) { dev_warn_ratelimited(dcp->dev, @@ -3053,16 +3055,19 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) cancel_delayed_work_sync(&dcp->typec_reconnect_wq); cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); - ret = dcp_external_stop_locked(dcp, dcp->external_generation); + /* Invalidate physical EDID before waiting for firmware power/link teardown. */ if (physical) { - dcp->external_cable = false; - dcp->external_generation++; - dcp_external_publish_generation(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + } + ret = dcp_external_stop_locked(dcp, dcp->external_generation); + if (physical) { + dcp->external_cable = false; + dcp->external_generation++; + dcp_external_publish_generation(dcp); /* A missing cable is reported even when its hardware remains owned. */ dcp_external_report_disconnected(dcp); } diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 1fb9818c0e5933..39d2fe70f904b2 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -33,6 +33,29 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) schedule_work(&connector->hotplug_wq); } +/* Admission runs before AFKV2 OPEN and before readers can pin the service. */ +static int dcpavserv_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + unsigned long flags; + int ret; + + if (!name || strcmp(name, "dcpav-service-epic")) + return -ENODEV; + if (service->ep->inline_v2) { + if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || + strcmp(class, "AppleDCPDP2HDMI") || unit != 1) + return -ENODEV; + } else if (unit != 0) { + return -ENODEV; + } + spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + ret = dcp->dcpavserv.enabled ? -EBUSY : 0; + spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + return ret; +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -98,6 +121,13 @@ void dpavservep_detach(struct apple_dcp *dcp) } } +static int dcpdpserv_prepare(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + /* No production command transport is needed for this H17P service. */ + return service->ep->inline_v2 ? -ENODEV : 0; +} + static void dcpdpserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -268,12 +298,25 @@ static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_servi return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, edid_size); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) +bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) +{ + lockdep_assert_held(&dcp->hpd_mutex); + return dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL || + (dcp->typec_cable_connected && generation == dcp->typec_generation && + !READ_ONCE(dcp->quiescing)); +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation) { struct apple_epic_service *service; const struct drm_edid *edid; unsigned long flags; + scoped_guard(mutex, &dcp->hpd_mutex) { + if (!dcpavserv_edid_current(dcp, generation)) + return ERR_PTR(-ESTALE); + } spin_lock_irqsave(&dcp->dcpavserv.lock, flags); service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); @@ -323,6 +366,13 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp) } spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); afk_service_put(service); + scoped_guard(mutex, &dcp->hpd_mutex) { + if (!dcpavserv_edid_current(dcp, generation)) { + if (!IS_ERR(edid)) + drm_edid_free(edid); + edid = ERR_PTR(-ESTALE); + } + } return edid; } @@ -330,6 +380,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpav-service-epic", .reusable = true, + .prepare = dcpavserv_prepare, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, @@ -337,6 +388,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpdp-service-epic", .reusable = true, + .prepare = dcpdpserv_prepare, .init = dcpdpserv_init, .teardown = dcpdpserv_teardown, .report = dcpdpserv_report, @@ -351,7 +403,9 @@ int dpavservep_init(struct apple_dcp *dcp) init_completion(&dcp->dcpavserv.enable_completion); - ep = afk_init(dcp, DPAVSERV_ENDPOINT, dpavservep_ops); + ep = afk_init(dcp, dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? + DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index 319471d272d99a..24e9a6c23dbfa0 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -22,7 +22,8 @@ struct dcpavserv { struct apple_epic_service *opened; }; -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp); +bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation); void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index ef63e40b7422b5..47486a7c0bb909 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1300,6 +1300,8 @@ int dcp_get_modes(struct drm_connector *connector) struct drm_display_mode *mode; u16 min_vfreq = 0, max_vfreq = 0; bool vrr_capable = false; + bool physical_edid; + u64 edid_generation = 0; unsigned int count; int i; @@ -1342,13 +1344,41 @@ int dcp_get_modes(struct drm_connector *connector) mutex_unlock(&dcp->modes_lock); drm_connector_set_vrr_capable_property(connector, vrr_capable); + physical_edid = dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + /* get_modes holds mode_config.mutex; HPD paths never acquire it. */ + if (physical_edid) { + mutex_lock(&dcp->hpd_mutex); + edid_generation = dcp->typec_generation; + if (apple_connector->drm_edid && + apple_connector->drm_edid_generation != edid_generation) + dcp_clear_connector_edid(apple_connector); + if (!dcpavserv_edid_current(dcp, edid_generation)) { + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + } + /* Internal H17P still has no supported EDID command transport. */ if (dcp->nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { if (count) { const struct drm_edid *edid; - edid = dcpavserv_copy_edid(dcp); + /* Let physical disconnect advance the generation during the request. */ + if (physical_edid) + mutex_unlock(&dcp->hpd_mutex); + edid = dcpavserv_copy_edid(dcp, edid_generation); + if (physical_edid) { + mutex_lock(&dcp->hpd_mutex); + if (!dcpavserv_edid_current(dcp, edid_generation) || + READ_ONCE(apple_connector->dcp) != pdev) { + if (!IS_ERR_OR_NULL(edid)) + drm_edid_free(edid); + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + } if (IS_ERR_OR_NULL(edid)) { /* * An internal panel has no AV service and so no EDID; @@ -1361,6 +1391,7 @@ int dcp_get_modes(struct drm_connector *connector) } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; + apple_connector->drm_edid_generation = edid_generation; u64 generation; /* The RPC can sleep while a disconnect or route change clears EDID. */ @@ -1376,7 +1407,6 @@ int dcp_get_modes(struct drm_connector *connector) if (dcp->nr_modes && apple_connector->drm_edid) { apple_connector->drm_edid_firmware_derived = false; drm_edid_connector_update(connector, apple_connector->drm_edid); - dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); } else { u64 generation = 0; const struct drm_edid *derived = dcp->nr_modes ? @@ -1413,6 +1443,12 @@ int dcp_get_modes(struct drm_connector *connector) drm_edid_connector_update(connector, NULL); } +edid_done: + if (physical_edid) + mutex_unlock(&dcp->hpd_mutex); + if (apple_connector->drm_edid) + dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + /* * An internal panel has no EDID, so nothing fills in the refresh range * that userspace needs before it will drive VRR. Supply the range DCP From f4c0b4a3cd5d4e99d2261b3bbf62a316b6940187 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 13:55:51 +1100 Subject: [PATCH 444/631] drm/apple: fence EDID publication by service lifetime and unblock RX A service CLOSE/replacement can leave a raw EDID cache valid even when the physical Type-C generation does not change. Replacement after the reader's last pointer check can also publish data from the retired service. Give DPAV an epoch that advances on service publication and retirement. Carry it from the pinned reader into the raw connector cache and validate it again while serializing publication against teardown. Use a process context mutex for the DPAV owner state, nesting below the DRM and HPD mutexes and releasing it during firmware waits. Defer CLOSE cache/property cleanup through the existing hotplug worker. Preserve the physical connection checks, owner/reader pins and firmware metadata epoch guard. DisplayAttributes is handled synchronously on RTKit's ordered RX worker. Taking mode_config.mutex there can block dispatch of the AFK reply that a mode probe is waiting for while it holds that mutex. Defer connector physical-dimension updates to the existing hotplug worker instead. Add a threaded host regression through the actual RTKit, IOMFB receive, dimensions and AFK enqueue functions. It fails on the parent with RX blocked before the AFK reply, and passes with the deferred update. Extend service and publication tests for same-slot CLOSE/re-OPEN, unchanged physical generations and replacement after the reader's last check. No command ABI or activation-readiness assumption is changed. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/connector.h | 1 + drivers/gpu/drm/apple/dcp.c | 31 +++++++++++- drivers/gpu/drm/apple/dcp.h | 3 ++ drivers/gpu/drm/apple/epic/dpavservep.c | 67 +++++++++++++++++-------- drivers/gpu/drm/apple/epic/dpavservep.h | 12 +++-- drivers/gpu/drm/apple/iomfb.c | 32 ++++++++++-- 6 files changed, 117 insertions(+), 29 deletions(-) diff --git a/drivers/gpu/drm/apple/connector.h b/drivers/gpu/drm/apple/connector.h index 45f59d93b97671..01a113a5cfa648 100644 --- a/drivers/gpu/drm/apple/connector.h +++ b/drivers/gpu/drm/apple/connector.h @@ -58,6 +58,7 @@ struct apple_connector { bool drm_edid_firmware_derived; bool edid_invalidation_pending; u64 drm_edid_generation; + u64 drm_edid_service_epoch; /* Workqueue for sending hotplug events to the associated device */ struct work_struct hotplug_wq; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 4e16861d53b779..dea88aa35b3262 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1878,6 +1878,20 @@ void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation) connector->display_info.width_mm = width_mm; connector->display_info.height_mm = height_mm; } + + /* + * This callback runs on the ordered RTKit RX worker. A mode probe + * can hold mode_config.mutex while waiting for an AFK reply which + * that same worker still has to dispatch. Defer the DRM update. + */ + if (dcp->connector && !READ_ONCE(dcp->quiescing)) + schedule_work(&dcp->connector->hotplug_wq); + + /* + * Fix up any probed modes. Modes are created when parsing + * TimingElements, dimensions are calculated when parsing + * DisplayAttributes, and TimingElements may be sent first + */ for (i = 0; i < dcp->nr_modes; ++i) { dcp->modes[i].mode.width_mm = width_mm; dcp->modes[i].mode.height_mm = height_mm; @@ -1888,6 +1902,21 @@ void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation) mutex_unlock(&connector->dev->mode_config.mutex); } +void dcp_update_connector_dimensions(struct apple_dcp *dcp, + struct drm_connector *connector) +{ + int width_mm = READ_ONCE(dcp->width_mm); + int height_mm = READ_ONCE(dcp->height_mm); + + lockdep_assert_held(&connector->dev->mode_config.mutex); + if (!width_mm || !height_mm) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; +} + bool dcp_has_panel(struct apple_dcp *dcp) { return dcp->panel.width_mm > 0; @@ -5246,7 +5275,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->metadata.lock); - spin_lock_init(&dcp->dcpavserv.lock); + mutex_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index aced157269e1c8..ad6a76e22fe4c8 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -80,6 +80,9 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation); +void dcp_set_dimensions(struct apple_dcp *dcp); +void dcp_update_connector_dimensions(struct apple_dcp *dcp, + struct drm_connector *connector); void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 39d2fe70f904b2..0be0b5336dfc66 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -38,7 +38,6 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; int ret; if (!name || strcmp(name, "dcpav-service-epic")) @@ -50,9 +49,9 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam } else if (unit != 0) { return -ENODEV; } - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); ret = dcp->dcpavserv.enabled ? -EBUSY : 0; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); return ret; } @@ -60,23 +59,25 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; trace_dcpavserv_init(dcp, unit); if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + if (!dcpavserv_prepare(service, name, class, unit)) { + mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.enabled) { - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); dev_err(dcp->dev, "DCPAVSERV: unit %lld already exists\n", unit); return; } dcp->dcpavserv.service = afk_service_get(service); dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); if (service->ep->inline_v2) { dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", service->channel); @@ -85,21 +86,36 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, } } +/* Defer DRM cleanup: AFK callbacks must not take mode_config.mutex. */ +static void dcpavserv_invalidate_edid(struct apple_dcp *dcp) +{ + struct apple_connector *connector = READ_ONCE(dcp->connector); + + lockdep_assert_held(&dcp->dcpavserv.lock); + if (!connector || READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) + return; + WRITE_ONCE(connector->edid_invalidation_pending, true); + if (!READ_ONCE(dcp->quiescing) && + READ_ONCE(connector->base.registration_state) == DRM_CONNECTOR_REGISTERED) + schedule_work(&connector->hotplug_wq); +} + static void dcpavserv_teardown(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; - unsigned long flags; bool owned = false; - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.service == service) { dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; + dcp->dcpavserv.epoch++; + dcpavserv_invalidate_edid(dcp); dcp->dcpavserv.opened = NULL; reinit_completion(&dcp->dcpavserv.enable_completion); owned = true; } - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); if (owned) afk_service_put(service); } @@ -107,14 +123,16 @@ static void dcpavserv_teardown(struct apple_epic_service *service) void dpavservep_detach(struct apple_dcp *dcp) { struct apple_epic_service *service; - unsigned long flags; - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.service; dcp->dcpavserv.service = NULL; dcp->dcpavserv.enabled = false; - dcp->dcpavserv.opened = NULL; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + if (service) { + dcp->dcpavserv.epoch++; + dcpavserv_invalidate_edid(dcp); + } + mutex_unlock(&dcp->dcpavserv.lock); if (service) { afk_service_disable(service); afk_service_put(service); @@ -307,19 +325,27 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) !READ_ONCE(dcp->quiescing)); } -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation) +bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch) +{ + lockdep_assert_held(&dcp->dcpavserv.lock); + return dcp->dcpavserv.enabled && dcp->dcpavserv.service && + epoch == dcp->dcpavserv.epoch; +} + +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, + u64 *epoch) { struct apple_epic_service *service; const struct drm_edid *edid; - unsigned long flags; scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) return ERR_PTR(-ESTALE); } - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); + mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + *epoch = dcp->dcpavserv.epoch; + mutex_unlock(&dcp->dcpavserv.lock); if (!service) return ERR_PTR(-ENODEV); @@ -358,13 +384,14 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation edid = dcpavserv_read_edid(service); edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); - if (dcp->dcpavserv.service != service) { + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.service != service || + !dcpavserv_edid_service_current(dcp, *epoch)) { if (!IS_ERR(edid)) drm_edid_free(edid); edid = ERR_PTR(-ENODEV); } - spin_unlock_irqrestore(&dcp->dcpavserv.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); afk_service_put(service); scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) { diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index 24e9a6c23dbfa0..001e211feeb6b4 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -5,7 +5,7 @@ #define _DRM_APPLE_EPIC_DPAVSERV_H #include -#include +#include #include struct drm_edid; @@ -13,7 +13,11 @@ struct apple_epic_service; struct apple_dcp; struct dcpavserv { - spinlock_t lock; /* Protects the service pointer and its owner pin. */ + /* Process context only: serializes retirement and EDID publication. + * Nest below mode_config.mutex and hpd_mutex; never wait for firmware. + */ + struct mutex lock; + u64 epoch; /* Changes even when the same service slot is reused. */ bool enabled; struct completion enable_completion; u32 unit; @@ -23,7 +27,9 @@ struct dcpavserv { }; bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); -const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation); +bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch); +const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, + u64 *epoch); void dpavservep_detach(struct apple_dcp *dcp); #endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 47486a7c0bb909..8196b72bf005cb 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -874,6 +874,8 @@ void dcp_hotplug(struct work_struct *work) connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); scoped_guard(mutex, &connector->base.dev->mode_config.mutex) { + if (READ_ONCE(connector->dcp) == pdev) + dcp_update_connector_dimensions(dcp, &connector->base); if (!READ_ONCE(connector->connected)) dcp_clear_connector_edid(connector); } @@ -1302,6 +1304,7 @@ int dcp_get_modes(struct drm_connector *connector) bool vrr_capable = false; bool physical_edid; u64 edid_generation = 0; + u64 service_epoch = 0; unsigned int count; int i; @@ -1350,9 +1353,15 @@ int dcp_get_modes(struct drm_connector *connector) if (physical_edid) { mutex_lock(&dcp->hpd_mutex); edid_generation = dcp->typec_generation; - if (apple_connector->drm_edid && - apple_connector->drm_edid_generation != edid_generation) - dcp_clear_connector_edid(apple_connector); + } + mutex_lock(&dcp->dcpavserv.lock); + /* Consume CLOSE invalidation under the publication/retirement lock. */ + dcp_consume_edid_invalidation(apple_connector); + if (apple_connector->drm_edid && + (!dcpavserv_edid_service_current(dcp, apple_connector->drm_edid_service_epoch) || + (physical_edid && apple_connector->drm_edid_generation != edid_generation))) + dcp_clear_connector_edid(apple_connector); + if (physical_edid) { if (!dcpavserv_edid_current(dcp, edid_generation)) { dcp_clear_connector_edid(apple_connector); goto edid_done; @@ -1366,11 +1375,14 @@ int dcp_get_modes(struct drm_connector *connector) if (count) { const struct drm_edid *edid; /* Let physical disconnect advance the generation during the request. */ + mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); - edid = dcpavserv_copy_edid(dcp, edid_generation); - if (physical_edid) { + edid = dcpavserv_copy_edid(dcp, edid_generation, &service_epoch); + if (physical_edid) mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (physical_edid) { if (!dcpavserv_edid_current(dcp, edid_generation) || READ_ONCE(apple_connector->dcp) != pdev) { if (!IS_ERR_OR_NULL(edid)) @@ -1379,6 +1391,14 @@ int dcp_get_modes(struct drm_connector *connector) goto edid_done; } } + if (!IS_ERR_OR_NULL(edid) && + !dcpavserv_edid_service_current(dcp, service_epoch)) { + drm_edid_free(edid); + dcp_clear_connector_edid(apple_connector); + goto edid_done; + } + /* Retirement may have queued cleanup while this request waited. */ + dcp_consume_edid_invalidation(apple_connector); if (IS_ERR_OR_NULL(edid)) { /* * An internal panel has no AV service and so no EDID; @@ -1392,6 +1412,7 @@ int dcp_get_modes(struct drm_connector *connector) drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; apple_connector->drm_edid_generation = edid_generation; + apple_connector->drm_edid_service_epoch = service_epoch; u64 generation; /* The RPC can sleep while a disconnect or route change clears EDID. */ @@ -1444,6 +1465,7 @@ int dcp_get_modes(struct drm_connector *connector) } edid_done: + mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); if (apple_connector->drm_edid) From 8a5a0f187cc1d6610cf297698ce0854fa595d5a3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 14:32:53 +1100 Subject: [PATCH 445/631] drm/apple: read external H17P EDID through owned AFKV2 buffers Use the existing host-owned AFKV2 OOB command transport for the observed external H17P DPAV service. Select service admission, notification and the strict length-aware EDID reader by AFKV2 framing rather than inline payload placement. Keep legacy profiles on their existing endpoint and reader. Length-aware AFKV2 callers must copy only the validated actual reply length for both inline and owned OOB replies. Keep strict callers' full-length contract and non-AFKV2 behavior unchanged. Preserve exact original DMA address/bounds matching, command slots, timeout retention and late-ACK release, without following peer-provided pointers. Apply the public owned transport correction from 821e79667f4d935fb50e1c6bc95beb4710f316cd on the service-lifetime/RX-progress base. Retain physical generation and service epoch checks, strict EDID header/length/block/checksum validation, and the conservative firmware metadata guard. No one-shot diagnostic permission, enlarged trace capture, timeout snapshot or refresh-rate admission is included. Extend actual sender/service extraction fixtures for short owned OOB responses, strict and legacy contracts, and the real late-ACK completer. A mismatched or duplicate reply cannot release retained DMA; the matching late reply retires the pending slot even after service retirement. Adapt lifetime/notification fixtures to the framing guard. Validation: full AFKV2 suite with ASan/UBSan (917 codec assertions), EDID lifetime, threaded RX-progress and metadata regressions pass. Targeted ARM64 W=1 module compile/modpost/link passes without warnings. Initial boot-time reads and physical hotplug remain owner qualification work. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 2 +- drivers/gpu/drm/apple/epic/dpavservep.c | 15 ++--- drivers/gpu/drm/apple/iomfb.c | 1 + .../drivers/gpu/apple-afkv2/extract-edid.py | 2 +- .../drivers/gpu/apple-afkv2/extract-notify.py | 8 +-- .../drivers/gpu/apple-afkv2/extract-send.py | 60 +++++++++++++++---- .../gpu/apple-afkv2/extract-service.py | 5 +- 7 files changed, 69 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 995e5e7746e0b7..4d9bdc1cacb4db 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -215,7 +215,7 @@ void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep) u32 rx_rptr = 0, rx_wptr = 0, tx_rptr = 0, tx_wptr = 0; bool rx_ready, tx_ready; - if (!ep || !ep->inline_v2 || !trace_afk_ring_snapshot_enabled()) + if (!ep || !ep->wire_v2 || !trace_afk_ring_snapshot_enabled()) return; spin_lock_irqsave(&ep->lock, flags); if (ep->stopping) { diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 0be0b5336dfc66..f042c9df4435f2 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -24,7 +24,7 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) struct apple_dcp *dcp = service->ep->dcp; struct apple_connector *connector = READ_ONCE(dcp->connector); - if (!service->ep->inline_v2 || READ_ONCE(dcp->quiescing) || !connector || + if (!service->ep->wire_v2 || READ_ONCE(dcp->quiescing) || !connector || !READ_ONCE(dcp->nr_modes) || !READ_ONCE(connector->connected) || READ_ONCE(connector->base.registration_state) != DRM_CONNECTOR_REGISTERED || READ_ONCE(connector->dcp) != to_platform_device(dcp->dev)) @@ -42,7 +42,7 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam if (!name || strcmp(name, "dcpav-service-epic")) return -ENODEV; - if (service->ep->inline_v2) { + if (service->ep->wire_v2) { if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || strcmp(class, "AppleDCPDP2HDMI") || unit != 1) return -ENODEV; @@ -78,8 +78,8 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); mutex_unlock(&dcp->dcpavserv.lock); - if (service->ep->inline_v2) { - dev_info(dcp->dev, "DCPAVSERV: inline EDID service ready on channel %u\n", + if (service->ep->wire_v2) { + dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", service->channel); dcpavserv_notify_edid(service); } @@ -143,7 +143,7 @@ static int dcpdpserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { /* No production command transport is needed for this H17P service. */ - return service->ep->inline_v2 ? -ENODEV : 0; + return service->ep->wire_v2 ? -ENODEV : 0; } static void dcpdpserv_init(struct apple_epic_service *service, const char *name, @@ -383,6 +383,7 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation edid = dcpavserv_read_edid(service); edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : + edid = service->ep->wire_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.service != service || @@ -435,10 +436,10 @@ int dpavservep_init(struct apple_dcp *dcp) DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); - /* Only the external H17P DPAV endpoint uses inline AFKV2 service calls. */ + /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ ep->wire_v2 = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; - ep->inline_v2 = ep->wire_v2; + ep->inline_v2 = false; dcp->dcpavservep = ep; dcp->dcpavservep->match_epic_name = true; diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 8196b72bf005cb..298696d99700ba 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1371,6 +1371,7 @@ int dcp_get_modes(struct drm_connector *connector) /* Internal H17P still has no supported EDID command transport. */ if (dcp->nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || + (dcp->dcpavservep && dcp->dcpavservep->wire_v2))) { (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { if (count) { const struct drm_edid *edid; diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py index 0ab0f5eda8cac5..0d554d1c82b5da 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-edid.py @@ -17,7 +17,7 @@ def function(name): end += 1 return source[match.start():end] -assert "service->ep->inline_v2 ? dcpavserv_read_edid_inline(service)" in source +assert "service->ep->wire_v2 ? dcpavserv_read_edid_inline(service)" in source prefix = r''' #undef NDEBUG #include diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py index 26579f8399cb13..481aac7d0246a5 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-notify.py @@ -23,7 +23,7 @@ struct base { int registration_state; }; struct apple_connector { bool connected;struct base base;void *dcp;int hotplug_wq; }; struct apple_dcp { struct apple_connector *connector;bool quiescing;unsigned int nr_modes;void *dev; }; -struct ep { struct apple_dcp *dcp;bool inline_v2; }; +struct ep { struct apple_dcp *dcp;bool wire_v2; }; struct apple_epic_service { struct ep *ep; }; static unsigned int queued; static void *to_platform_device(void *p) { return p; } @@ -33,7 +33,7 @@ int main(void) { int device,other;struct apple_dcp dcp={.dev=&device}; - struct ep ep={.dcp=&dcp,.inline_v2=true};struct apple_epic_service service={.ep=&ep}; + struct ep ep={.dcp=&dcp,.wire_v2=true};struct apple_epic_service service={.ep=&ep}; struct apple_connector connector={.dcp=&device}; dcpavserv_notify_edid(&service);assert(queued==0); dcp.connector=&connector;dcpavserv_notify_edid(&service);assert(queued==0); @@ -42,8 +42,8 @@ dcpavserv_notify_edid(&service);assert(queued==1 && connector.hotplug_wq==1); dcp.quiescing=true;dcpavserv_notify_edid(&service);assert(queued==1); dcp.quiescing=false;connector.dcp=&other;dcpavserv_notify_edid(&service);assert(queued==1); - connector.dcp=&device;ep.inline_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); - ep.inline_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); + connector.dcp=&device;ep.wire_v2=false;dcpavserv_notify_edid(&service);assert(queued==1); + ep.wire_v2=true;connector.connected=false;dcpavserv_notify_edid(&service);assert(queued==1); puts("PASS: actual late-EDID notification queues ready connector only; legacy/quiesce/unrouted/unregistered guarded"); } ''' diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py index 92ca8857cc1045..0d80a729a33424 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -13,6 +13,13 @@ while depth: depth += (source[end] == "{") - (source[end] == "}") end += 1 +reply_match = re.search(r"static void afk_recv_handle_reply\([^;]*?\)\n\{", source) +if not reply_match: + raise RuntimeError("production reply completer not found") +reply_end, reply_depth = reply_match.end(), 1 +while reply_depth: + reply_depth += (source[reply_end] == "{") - (source[reply_end] == "}") + reply_end += 1 prefix = r''' #undef NDEBUG #include @@ -24,6 +31,7 @@ #define GFP_KERNEL 0 #define MAX_PENDING_CMDS 16 #define cpu_to_le32(x) (x) +#define le32_to_cpu(x) (x) #define cpu_to_le64(x) (x) #define kmalloc(n,f) malloc(n) #define kfree(p) free(p) @@ -32,6 +40,8 @@ #define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) #define dma_rmb() do {} while(0) #define dev_warn(...) do {} while(0) +#define dev_err(...) do {} while(0) +#define complete(p) (*(p)=1) #define EPIC_TYPE_COMMAND 3 #define EPIC_CAT_COMMAND 0x30 #define DECLARE_COMPLETION_ONSTACK(n) int n=0 @@ -50,13 +60,18 @@ struct ops { const char *name; }; struct apple_epic_service { struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; - int lock; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; + int lock; unsigned int users; bool enabled,torndown; u8 cmd_tag; struct cmd_info cmds[16]; }; static struct apple_epic_service *active; static unsigned int frees, sends; static int wire_fail, timeout, reply_status; -static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { return s->enabled?s:NULL; } -static void afk_service_put(struct apple_epic_service *s) { (void)s; } +static size_t owned_reply_size=32888; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) +{ if(!s->enabled) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) { assert(s->users); s->users--; } +static struct apple_epic_service *afk_epic_find_service(struct apple_dcp_afkep *ep,u32 channel) +{ return active->ep==ep && active->channel==channel && (active->enabled || active->cmd_map[0]) ? active : NULL; } +static int test_bit(unsigned int bit,const unsigned long *map) { return !!(*map & (1UL<reply_length=ep->inline_v2?4:32888;c->retcode=reply_status; + c->reply_length=ep->inline_v2?4:owned_reply_size;c->retcode=reply_status; memset(c->rxbuf,0x77,c->reply_length);c->done=true;*c->completion=1; return 0; } @@ -92,7 +107,7 @@ int main(void) { struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; - struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true}; + struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true,.users=1}; u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; active=&s; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); @@ -107,12 +122,37 @@ wire_fail=0;ep.inline_v2=false; assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); assert(received==32888 && status==0); - ep.inline_v2=true;timeout=1; + /* Length-aware owned OOB callers copy only the bounded actual reply. */ + owned_reply_size=4;memset(output,0x19,32888); + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==4 && output[0]==0x77 && output[3]==0x77 && output[4]==0x19 && s.users==1); + memset(output,0x19,32888); + assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,NULL)==-EMSGSIZE); + assert(output[0]==0x19 && !s.cmd_map[0]); + owned_reply_size=0;received=99; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==0 && output[0]==0x19); + /* Legacy framing retains its full-buffer copy contract. */ + ep.wire_v2=false;owned_reply_size=4; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(received==32888 && s.users==1); + ep.wire_v2=true;owned_reply_size=32888;timeout=1; assert(afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)==-ETIMEDOUT); assert(s.cmd_map[0] && s.cmds[0].free_on_ack && !s.cmds[0].completion); - /* Mock the late-reply owner's release, avoiding leaks in this host test. */ - free(s.cmds[0].rxbuf);free(s.cmds[0].txbuf);free(output); - puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, strict OOB and timeout ownership"); + /* Execute the real late-ACK completer: mismatch/duplicate cannot free DMA. */ + unsigned int before_late=frees;u16 retained_tag=s.cmds[0].tag; + struct epic_cmd reply={.retcode=0}; + s.enabled=false;s.torndown=true; /* Retired service still owns the pending slot. */ + afk_recv_handle_reply(&ep,4,retained_tag^0x10,&reply,sizeof(reply)); + assert(frees==before_late && s.cmd_map[0] && !s.cmds[0].done && s.users==1); + afk_recv_handle_reply(&ep,4,retained_tag,&reply,sizeof(reply)); + assert(frees==before_late+2 && !s.cmd_map[0] && s.cmds[0].done && s.users==1); + afk_recv_handle_reply(&ep,4,retained_tag,&reply,sizeof(reply)); + assert(frees==before_late+2); + s.enabled=true;s.torndown=false;timeout=0; + assert(!afk_send_command_timeout_len(&s,0xc0,request,112,output,32888,&status,1000,&received)); + assert(!s.cmd_map[0] && s.users==1);free(output); + puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers and actual late-ACK ownership"); } ''' -pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) +pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + source[reply_match.start():reply_end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py index 68e3f5e6ba980c..ab95f5c422710e 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-service.py @@ -79,8 +79,11 @@ too_long=0;wire_error=1; assert(afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)==-ETIMEDOUT); wire_error=0; ep.inline_v2=false; + received=999; + assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,&received)); + assert(received==4 && output[0]==0xde && output[3]==0xef && output[4]==0); assert(!afk_service_call_timeout_len(&service,1,7,input,4,4096,output,128,0,1000,NULL)); - puts("PASS: extracted production service-call actual request / legacy padding / short reply / status / context / length tests"); + puts("PASS: extracted production service-call actual request / OOB padding and short reply / legacy / status / context / length tests"); } ''' pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + suffix) From 8d6a34ca9625ab7691fa758ad6101816e9004902 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 14:48:51 +1100 Subject: [PATCH 446/631] drm/apple: admit external AFKV2 EDID services asynchronously External H17P service announcements can arrive after the AFK endpoint starts. Waiting synchronously for the EDID service can time out just before exact admission completes, leaving a misleading startup failure despite a usable service and delaying other endpoint initialization. After a successful afk_start, return for the external AFKV2 profile and let the existing guarded admission and late connector notification publish the service. Do not claim a missing service is enabled. Retain the legacy completion wait, timeout behavior and AFK-start failure cleanup unchanged. Add host tests executing actual endpoint startup, admission, notification and retirement for early, late, permanently missing and wrong-identity announcements, plus legacy waits and endpoint-start failure. No protocol command, service identity, EDID validation, generation or lifetime fence is changed. Persistent Offline replies after physical reconnect remain separate session/attachment qualification work. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index f042c9df4435f2..c4258469f8d6a8 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -452,6 +452,10 @@ int dpavservep_init(struct apple_dcp *dcp) return ret; } + /* Admission may arrive after startup; exact guards and late notify own it. */ + if (ep->wire_v2) + return 0; + ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); if (!ret) { From e609c0f2ff6795e7e51d004e2fa4ab712dec598b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 15:15:51 +1100 Subject: [PATCH 447/631] drm/apple: replace H17P AV bindings across physical reconnect H17P parse_epic_service_init maps interface-id into the variable named unit. That value is a dynamic AFKV2 interface identity, not a physical-port number. Requiring unit 1 rejects fresh AV announcements after reconnect while retaining the old offline interface. Require the exact DPAV endpoint/compartment/class and a nonzero bounded interface-id equal to the validated service channel. Preserve legacy EPICUnit 0 admission. Retire the current AV registry binding on physical disconnect, current-generation deferred disconnect, route withdrawal and cable-release teardown, outside hpd_mutex. Ordinary DPMS poweroff does not withdraw it. Carry physical-generation and registry-epoch tickets from prepare through OPEN/init so a disconnect or retirement during admission cannot publish stale work. Local retirement disables the service and marks its slot torn down; reader pins and outstanding commands still prevent reuse, and timed-out DMA remains owned until a matching late ACK. Old CLOSE/ACK handling cannot revoke a fresh binding. Recognize the observed DPAV CLOSED report (0x14, zero token, low flags byte 1, one zero payload byte) only on the instantiated DPAV endpoint. Keep the existing 0x13 report handling and other endpoints unchanged. No new host opcode or reopen of an offline interface is introduced. Host regressions execute the production parser, ownership helpers, prepare/ init, retirement, receive dispatch and late-ACK completer for dynamic interfaces 1/5/9/13, forged identities, independent generation/epoch changes, old pins/CLOSED/replies and legacy unit 0. Full AFKV2 ASan/UBSan suite, startup, EDID lifetime, external lifecycle, metadata and RX-progress tests pass. Targeted ARM64 W=1 module compile/modpost/link passes without warnings. Generated-by: LLM Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 3 +- drivers/gpu/drm/apple/afk.h | 13 ++++ drivers/gpu/drm/apple/afkv2-codec.h | 10 +++ drivers/gpu/drm/apple/dcp.c | 6 ++ drivers/gpu/drm/apple/epic/dpavservep.c | 92 ++++++++++++++++++------- 5 files changed, 100 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 4d9bdc1cacb4db..63a2c17a55ea7d 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -963,7 +963,8 @@ static void afk_v2_receive(struct apple_dcp_afkep *ep, u32 channel, u32 type, if (frame.type == EPIC_SUBTYPE_ANNOUNCE_H17P) afk_recv_handle_init(ep, frame.interface, (void *)frame.payload, frame.payload_size); - else if (frame.type == AFKV2_CLOSE && frame.interface_flags == 1) + else if ((frame.type == AFKV2_CLOSE && frame.interface_flags == 1) || + (ep == ep->dcp->dcpavservep && afkv2_closed_report(&frame))) afk_recv_handle_teardown(ep, frame.interface); return; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 68a8c0218c45b0..c26f4453082f6d 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -59,6 +59,8 @@ struct apple_epic_service { bool enabled; bool torndown; + /* Provider-owned prepare/init ticket; AFK core never interprets it. */ + struct { u64 generation, epoch; } admission; void *cookie; struct { @@ -151,6 +153,17 @@ static inline void afk_service_disable(struct apple_epic_service *service) spin_unlock_irqrestore(&service->lock, flags); } +/* Retire local ownership while keeping timed-out commands discoverable. */ +static inline void afk_service_retire(struct apple_epic_service *service) +{ + unsigned long flags; + + spin_lock_irqsave(&service->lock, flags); + service->enabled = false; + service->torndown = true; + spin_unlock_irqrestore(&service->lock, flags); +} + static inline bool afk_service_matches(struct apple_epic_service *service, u32 channel) { diff --git a/drivers/gpu/drm/apple/afkv2-codec.h b/drivers/gpu/drm/apple/afkv2-codec.h index a06a1efd5fc7e8..c42788a1b2a74e 100644 --- a/drivers/gpu/drm/apple/afkv2-codec.h +++ b/drivers/gpu/drm/apple/afkv2-codec.h @@ -17,6 +17,7 @@ #define AFKV2_RESPONSE 2U #define AFKV2_OPEN 0x12U #define AFKV2_CLOSE 0x13U +#define AFKV2_CLOSED 0x14U struct afkv2_frame { u8 sequence; @@ -113,6 +114,15 @@ static inline int afkv2_decode_frame(const void *record, size_t size, return 0; } +/* Closed notification: flags occupy the low byte; the high byte is opaque. */ +static inline bool afkv2_closed_report(const struct afkv2_frame *frame) +{ + return frame && !frame->token && frame->category == AFKV2_REPORT && + frame->type == AFKV2_CLOSED && + (frame->interface_flags & 0xff) == 1 && frame->payload_size == 1 && + frame->payload && !frame->payload[0]; +} + static inline int afkv2_encode_frame(void *record, size_t capacity, u8 sequence, u16 interface, u64 token, u8 type, u8 category, u16 flags, const void *payload, size_t payload_size, diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index dea88aa35b3262..5ccaf242f0d257 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -3091,6 +3091,8 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + /* HPD is fenced; retire the AV owner after releasing hpd_mutex. */ + dpavservep_detach(dcp); } ret = dcp_external_stop_locked(dcp, dcp->external_generation); if (physical) { @@ -3115,6 +3117,8 @@ static void dcp_external_stop_work(struct work_struct *work) cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->external_lock); if (generation == dcp->external_generation) { + /* The deferred disconnect also covers missing out-of-band HPD. */ + dpavservep_detach(dcp); dcp_external_stop_locked(dcp, generation); dcp_external_report_disconnected(dcp); } @@ -4705,6 +4709,8 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) dcp->typec_generation++; dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } + if (release_cable && dcp_is_external_h17p(dcp)) + dpavservep_detach(dcp); /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index c4258469f8d6a8..381a9dec635580 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -33,25 +33,51 @@ static void dcpavserv_notify_edid(struct apple_epic_service *service) schedule_work(&connector->hotplug_wq); } +static bool dcpavserv_identity(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + if (!name || strcmp(name, "dcpav-service-epic")) + return false; + if (!service->ep->wire_v2) + return unit == 0; + /* H17P's lower-case interface-id is dynamic, not a physical-port unit. */ + return service->ep->endpoint == DPAVSERV_ENDPOINT_V2 && class && + !strcmp(class, "AppleDCPDP2HDMI") && service->channel && + service->channel <= U16_MAX && unit == service->channel; +} + +static bool dcpavserv_admission_current(struct apple_dcp *dcp, u64 generation) +{ + lockdep_assert_held(&dcp->hpd_mutex); + /* Initial service discovery may precede the first Type-C attachment. */ + return !READ_ONCE(dcp->quiescing) && generation == dcp->typec_generation && + (!generation || dcp->typec_cable_connected); +} + /* Admission runs before AFKV2 OPEN and before readers can pin the service. */ static int dcpavserv_prepare(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; - int ret; + bool v2 = service->ep->wire_v2; + int ret = 0; - if (!name || strcmp(name, "dcpav-service-epic")) - return -ENODEV; - if (service->ep->wire_v2) { - if (service->ep->endpoint != DPAVSERV_ENDPOINT_V2 || !class || - strcmp(class, "AppleDCPDP2HDMI") || unit != 1) - return -ENODEV; - } else if (unit != 0) { + if (!dcpavserv_identity(service, name, class, unit)) return -ENODEV; - } + if (v2) + mutex_lock(&dcp->hpd_mutex); mutex_lock(&dcp->dcpavserv.lock); - ret = dcp->dcpavserv.enabled ? -EBUSY : 0; + if (dcp->dcpavserv.enabled) + ret = -EBUSY; + else if (v2 && !dcpavserv_admission_current(dcp, dcp->typec_generation)) + ret = -ENODEV; + else { + service->admission.generation = v2 ? dcp->typec_generation : 0; + service->admission.epoch = dcp->dcpavserv.epoch; + } mutex_unlock(&dcp->dcpavserv.lock); + if (v2) + mutex_unlock(&dcp->hpd_mutex); return ret; } @@ -59,8 +85,17 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; + bool v2 = service->ep->wire_v2; + bool published = false; trace_dcpavserv_init(dcp, unit); + if (v2) + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (dcpavserv_identity(service, name, class, unit) && + !dcp->dcpavserv.enabled && + service->admission.epoch == dcp->dcpavserv.epoch && + (!v2 || dcpavserv_admission_current(dcp, service->admission.generation))) { if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { spin_lock_irqsave(&dcp->dcpavserv.lock, flags); @@ -73,17 +108,27 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, return; } dcp->dcpavserv.service = afk_service_get(service); - dcp->dcpavserv.enabled = true; - dcp->dcpavserv.epoch++; - service->cookie = &dcp->dcpavserv; - complete(&dcp->dcpavserv.enable_completion); - mutex_unlock(&dcp->dcpavserv.lock); - if (service->ep->wire_v2) { - dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", - service->channel); - dcpavserv_notify_edid(service); + if (dcp->dcpavserv.service) { + dcp->dcpavserv.enabled = true; + dcp->dcpavserv.epoch++; + service->cookie = &dcp->dcpavserv; + complete(&dcp->dcpavserv.enable_completion); + published = true; } } + mutex_unlock(&dcp->dcpavserv.lock); + if (v2) + mutex_unlock(&dcp->hpd_mutex); + if (!published) { + /* A generation/registry change during OPEN cannot enable stale work. */ + afk_service_retire(service); + return; + } + if (v2) { + dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", + service->channel); + dcpavserv_notify_edid(service); + } } /* Defer DRM cleanup: AFK callbacks must not take mode_config.mutex. */ @@ -128,15 +173,16 @@ void dpavservep_detach(struct apple_dcp *dcp) service = dcp->dcpavserv.service; dcp->dcpavserv.service = NULL; dcp->dcpavserv.enabled = false; + /* Also reject an in-flight prepare/OPEN with no published owner yet. */ + dcp->dcpavserv.epoch++; + reinit_completion(&dcp->dcpavserv.enable_completion); if (service) { - dcp->dcpavserv.epoch++; + afk_service_retire(service); dcpavserv_invalidate_edid(dcp); } mutex_unlock(&dcp->dcpavserv.lock); - if (service) { - afk_service_disable(service); + if (service) afk_service_put(service); - } } static int dcpdpserv_prepare(struct apple_epic_service *service, const char *name, From 8d4fc59ff589c8d1e48dd55a282c15eb3cdde407 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Tue, 13 Oct 2026 07:38:47 -0400 Subject: [PATCH 448/631] drm/apple: enable the validated J700 external display profile by default Use native PMGR bandwidth scratch and clock-request offsets from the T8140 hardware profile. Keep diagnostic overrides bounded and retain the disable switch and loader firmware validation. Put the existing tested external route and scanout window in the normal J700 tree so users need no manual command line or alternate DTB. --- .../boot/dts/apple/t8140-j700-dcpext.dtsi | 5 ++-- arch/arm64/boot/dts/apple/t8140-j700.dts | 27 ++++++++++++++++++- drivers/gpu/drm/apple/dcp-external-bw.h | 15 +++++++++++ drivers/gpu/drm/apple/dcp.c | 23 +++++++++------- .../drivers/gpu/test_t8140_external_bw.py | 12 +++++++++ 5 files changed, 69 insertions(+), 13 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 6a6538dc6dba68..3b8be06ed22f50 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -4,9 +4,8 @@ * Generated-by: LLM * Assisted-by: LLM * - * The controller and route remain disabled. Their live firmware - * profile, bandwidth request, routing selector and handoff must be qualified - * before enabling the controller. The existing T8140 refusal remains in force. + * The controller remains disabled until the loader validates and preserves + * the inherited firmware and its complete dependency group. */ &{/soc} { diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 44143ae11092eb..033272cdb32d91 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -1879,5 +1879,30 @@ }; }; -/* External resources stay disabled pending the separate firmware profile. */ +/* The loader publishes external resources after validating their firmware. */ #include "t8140-j700-dcpext.dtsi" + +&display { + iommus = <&disp0_dart 0>, <&dispext0_dart 0>; +}; + +&dcpext { + clocks = <&clk_disp0_154>, <&clk_disp0_194>; + clock-names = "clock-154", "clock-194"; + phys = <&atcphy0 PHY_TYPE_DP>; + phy-names = "typec0"; + mux-controls = <&dpxbar0 0>; + mux-control-names = "typec0"; + apple,typec-mux-indices = <2>; + apple,typec-dptx-phys = <0>; + apple,dcp-index = <1>; +}; + +&{/soc/dcp@416e00000/typec-routes/route@0} { + status = "okay"; +}; + +/* Allocate scanout buffers in the window used by the inherited firmware. */ +&dcpext0_dart { + apple,dma-range = <0x100 0x0 0x10 0x0>; +}; diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h index 439db008b50846..a3a799ddefbd7b 100644 --- a/drivers/gpu/drm/apple/dcp-external-bw.h +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -21,4 +21,19 @@ static inline bool apple_dcp_external_clock_window(u64 start, u64 size, return start == expected_start && size == expected_size; } +/* Resolve optional diagnostic overrides without changing the SoC defaults. */ +static inline bool apple_dcp_external_bw_resolve(u64 size, int default_scratch, + int default_request, int override_scratch, + int override_request, u32 *scratch, u32 *request) +{ + int s = override_scratch == -1 ? default_scratch : override_scratch; + int r = override_request == -1 ? default_request : override_request; + + if (!apple_dcp_external_bw_valid(size, s, r)) + return false; + *scratch = s; + *request = r; + return true; +} + #endif diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5ccaf242f0d257..927df2d865e107 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -155,20 +155,20 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) -static bool experimental_h17p_external; +static bool experimental_h17p_external = true; module_param(experimental_h17p_external, bool, 0444); MODULE_PARM_DESC(experimental_h17p_external, - "Enable experimental T8140 external display protocol"); + "Enable T8140 external display protocol (default: enabled)"); static int experimental_h17p_external_bw_scratch = -1; module_param(experimental_h17p_external_bw_scratch, int, 0444); MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, - "Experimental external bandwidth scratch offset; unset by default"); + "Override external bandwidth scratch offset (-1 uses the hardware profile)"); static int experimental_h17p_external_bw_request = -1; module_param(experimental_h17p_external_bw_request, int, 0444); MODULE_PARM_DESC(experimental_h17p_external_bw_request, - "Experimental external bandwidth request offset; unset by default"); + "Override external bandwidth request offset (-1 uses the hardware profile)"); bool dcp_h17p_external_enabled(void) { @@ -5286,14 +5286,16 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { if (!dcp->hw.firmware_clock || - !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), + !apple_dcp_external_bw_resolve(resource_size(dcp->hw.firmware_clock), + dcp->hw.firmware_scratch, + dcp->hw.firmware_request, experimental_h17p_external_bw_scratch, - experimental_h17p_external_bw_request)) + experimental_h17p_external_bw_request, + &dcp->hw.firmware_scratch, + &dcp->hw.firmware_request)) return dev_err_probe(dev, -EINVAL, - "External bandwidth offsets are unset or invalid\n"); + "External bandwidth offsets are invalid\n"); iomfb_pause_presentation(dcp, true); - dcp->hw.firmware_scratch = experimental_h17p_external_bw_scratch; - dcp->hw.firmware_request = experimental_h17p_external_bw_request; } @@ -6017,6 +6019,9 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, .firmware_display_windows = 4, + /* PMGR BWR agent 2 and boost agent 3 for DISPEXT0. */ + .firmware_scratch = 0x28000, + .firmware_request = 0x70000, }; /* diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py index 73f0dba8d4d225..f55107c0ddaa67 100644 --- a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -14,6 +14,18 @@ def test_shared_header_boundaries(self): #include "dcp-external-bw.h" int main(void) { const u64 n=0xbc000; + u32 scratch=1, request=2; + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,-1,&scratch,&request)); + assert(scratch==0x28000 && request==0x70000); + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x30000,-1,&scratch,&request)); + assert(scratch==0x30000 && request==0x70000); + assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x74000,&scratch,&request)); + assert(scratch==0x28000 && request==0x74000); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-2,-1,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x28000,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x28002,-1,&scratch,&request)); + assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,n,&scratch,&request)); + assert(scratch==0x28000 && request==0x74000); assert(!apple_dcp_external_bw_valid(n,-1,-1)); assert(!apple_dcp_external_bw_valid(n,0,-1)); assert(!apple_dcp_external_bw_valid(n,-1,0)); From 12d9a979ecaaae0f21d88581bc3bc449fbf2a3d7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 21:51:16 +1100 Subject: [PATCH 449/631] drm/apple: admit the native DP EDID provider in the guarded AV service H17P can announce DCPDP13Service in the EDID AV compartment after an attachment change. The HDMI-only class check rejects that otherwise valid identity before OPEN, leaving the connector without an EDID owner. Accept that exact native class alongside AppleDCPDP2HDMI under the existing AFKV2 AV compartment, endpoint, channel and unit constraints. Keep legacy handling, auxiliary-service policy and all attachment, generation, epoch, ownership and retirement fences unchanged. Exercise the actual announcement and admission functions with both providers over 64 flip/revalidation/return cycles, including same-generation revalidation. Retain forged identity, stale OPEN, owner, auxiliary, reader/pending-command and legacy controls. The exact parent fails the first legitimate native publication; the correction passes. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/epic/dpavservep.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 381a9dec635580..926cf064040f54 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -42,7 +42,8 @@ static bool dcpavserv_identity(struct apple_epic_service *service, const char *n return unit == 0; /* H17P's lower-case interface-id is dynamic, not a physical-port unit. */ return service->ep->endpoint == DPAVSERV_ENDPOINT_V2 && class && - !strcmp(class, "AppleDCPDP2HDMI") && service->channel && + (!strcmp(class, "AppleDCPDP2HDMI") || + !strcmp(class, "DCPDP13Service")) && service->channel && service->channel <= U16_MAX && unit == service->channel; } From e82e308db7251a980363710e4c1dfda93b3ce653 Mon Sep 17 00:00:00 2001 From: Bartosz Nitka Date: Mon, 12 Oct 2026 21:32:58 +0200 Subject: [PATCH 450/631] cpuidle: apple: retain a selectable WFI state for T8140 s2idle T8140 exposes only the index-0 WFI fallback to avoid the unsupported CPU power-down path. The generic s2idle selector skips index 0, so this prevents tick_freeze() and timekeeping suspension from being reached. Copy the safe WFI state over the T8140 power-down entry instead of reducing state_count. This preserves an eligible nonzero s2idle entry without using CYC_OVRD, while leaving other Apple SoCs unchanged. On MacBook Neo, three 20-second RTC sleeps accounted for approximately zero suspended time before this change and 20.198, 19.808 and 20.419 seconds afterward using the identical BOOTTIME-minus-MONOTONIC test. Signed-off-by: Bartosz Nitka --- drivers/cpuidle/cpuidle-apple.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/cpuidle/cpuidle-apple.c b/drivers/cpuidle/cpuidle-apple.c index 6c02bd4197c263..64b45dcbafb31a 100644 --- a/drivers/cpuidle/cpuidle-apple.c +++ b/drivers/cpuidle/cpuidle-apple.c @@ -127,10 +127,12 @@ static int apple_cpuidle_probe(struct platform_device *pdev) /* * CPU PD writes CYC_OVRD, which is an undefined instruction on * T8140. Its WFI already lets the hardware power-gate idle - * secondary cores, so expose only the WFI state. + * secondary cores, so both entries must use WFI. Keep index 1: + * the generic s2idle selector skips fallback index 0, otherwise + * tick_freeze() and timekeeping suspension are never reached. */ if (of_machine_is_compatible("apple,t8140")) - apple_idle_driver.state_count = STATE_WFI + 1; + apple_idle_driver.states[STATE_PWRDOWN] = apple_idle_driver.states[STATE_WFI]; return cpuidle_register(&apple_idle_driver, NULL); } From e02f58bcbfa09c812ae4e89022abdd420e8f5f7a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 15:52:10 +0000 Subject: [PATCH 451/631] drm/apple: compose lean J700 display and automatic recovery candidate Replace the experimental profile and bandwidth/rate switches with the validated hardware and firmware profile, bounded advertised fixed-rate admission, guarded native monitor identity and physical disconnect handling. Resume the matching SuperSpeed root across Type-C PHY transitions and retain failed handoff ownership instead of reporting an untrained link as usable. Remove one-off diagnostics and qualification observers. Keep a compact standard selftest suite exercising actual admission, publication, PHY, ownership, retry and legacy paths. Include reviewed SEP readiness, producer sensor refusal and ISP metadata parsing corrections without changing their cryptographic or hardware command protocols. Include Bartosz Nitka's unchanged T8140 WFI/s2idle correction through its original PR298 commit as a merge parent, preserving its authorship and signoff. This is an integration candidate, not a hardware-qualified beta. Exact source tree7196655aef16cd39d9db2f17e0cbfbf5067526fe built Image/modules/dtbs with C -Werror and Rust -Dwarnings; affected objects passed W=1. Host and adverse controls passed. Automatic repeated orientation/hotplug, PM, reference-machine regressions and HDR qualification remain open. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + .../boot/dts/apple/t8140-j700-dcpext.dtsi | 2 - drivers/gpu/drm/apple/Makefile | 2 +- drivers/gpu/drm/apple/afk.c | 85 +- drivers/gpu/drm/apple/afk.h | 1 - drivers/gpu/drm/apple/apple_drv.c | 3 +- drivers/gpu/drm/apple/dcp-180hz-policy.h | 101 + drivers/gpu/drm/apple/dcp-180hz.c | 260 ++ drivers/gpu/drm/apple/dcp-180hz.h | 34 + drivers/gpu/drm/apple/dcp-external-bw.h | 27 +- drivers/gpu/drm/apple/dcp-internal.h | 16 +- drivers/gpu/drm/apple/dcp.c | 144 +- drivers/gpu/drm/apple/dcp.h | 1 + drivers/gpu/drm/apple/dptxep.c | 63 +- drivers/gpu/drm/apple/epic/dpavservep.c | 7 +- drivers/gpu/drm/apple/ibootep.c | 2 +- drivers/gpu/drm/apple/iomfb.c | 132 +- drivers/gpu/drm/apple/iomfb_h17p.c | 25 +- drivers/gpu/drm/apple/iomfb_template.c | 36 +- drivers/gpu/drm/apple/parser.c | 230 +- drivers/gpu/drm/apple/parser.h | 11 + drivers/gpu/drm/apple/trace.h | 148 - drivers/media/platform/apple/isp/isp-drv.c | 39 +- drivers/phy/apple/Kconfig | 1 + drivers/phy/apple/atc-j700-dp.h | 160 +- drivers/phy/apple/atc.c | 411 ++- drivers/soc/apple/dt.rs | 23 +- drivers/soc/apple/profile.rs | 2 +- drivers/soc/apple/sbio.rs | 26 +- drivers/soc/apple/sep.rs | 42 + drivers/usb/dwc3/core.c | 31 +- drivers/usb/dwc3/dwc3-apple.c | 62 +- drivers/usb/dwc3/glue.h | 20 + drivers/usb/typec/tipd/core.c | 47 +- tools/testing/selftests/Makefile | 1 + .../drivers/gpu/apple-afkv2/extract-ring.py | 2 +- .../selftests/drivers/gpu/apple-dcp/Makefile | 4 + .../selftests/drivers/gpu/apple-dcp/README | 56 + .../selftests/drivers/gpu/apple-dcp/common.py | 45 + .../drivers/gpu/apple-dcp/dwc3/cases.c | 304 +++ .../drivers/gpu/apple-dcp/dwc3/mocks.c | 125 + .../drivers/gpu/apple-dcp/dwc3/prefix.c | 54 + .../selftests/drivers/gpu/apple-dcp/edid.py | 789 ++++++ .../selftests/drivers/gpu/apple-dcp/idle.py | 185 ++ .../drivers/gpu/apple-dcp/lifecycle.py | 1194 ++++++++ .../selftests/drivers/gpu/apple-dcp/phy.py | 2399 +++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/pm.py | 453 ++++ .../selftests/drivers/gpu/apple-dcp/rate.py | 1125 ++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 112 + .../selftests/drivers/gpu/apple-dcp/run.sh | 4 + .../drivers/gpu/test_t8140_external_bw.py | 13 +- 51 files changed, 8577 insertions(+), 483 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-180hz-policy.h create mode 100644 drivers/gpu/drm/apple/dcp-180hz.c create mode 100644 drivers/gpu/drm/apple/dcp-180hz.h create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/Makefile create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/README create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/common.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/edid.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/idle.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/lifecycle.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/phy.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/pm.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/rate.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/run.py create mode 100755 tools/testing/selftests/drivers/gpu/apple-dcp/run.sh diff --git a/MAINTAINERS b/MAINTAINERS index 22f8615ea8f50f..50a28de436054c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2043,6 +2043,7 @@ F: tools/diagnostics/test-apple-j700-provider.py F: tools/diagnostics/test-apple-j700-tuning.py F: tools/diagnostics/wayland-output-frame-test.py F: tools/testing/apple/ +F: tools/testing/selftests/drivers/gpu/apple-dcp/ APPLETALK NETWORK LAYER L: netdev@vger.kernel.org diff --git a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi index 3b8be06ed22f50..69d6eff9aa756f 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-j700-dcpext.dtsi @@ -1,8 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ OR MIT /* * T8140 external display resources measured through the public ADT API. - * Generated-by: LLM - * Assisted-by: LLM * * The controller remains disabled until the loader validates and preserves * the inherited firmware and its complete dependency group. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 4619415a36ab00..41dfad9cfcd1ab 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -7,7 +7,7 @@ appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o appledrm-y += dcp-hdmi.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += av.o -appledrm-y += connector.o dcp-display-metadata.o +appledrm-y += connector.o dcp-display-metadata.o dcp-180hz.o appledrm-y += ibootep.o appledrm-y += dcpext_scanout.o appledrm-y += dcpext_drm.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 63a2c17a55ea7d..9600e27150d71c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -198,7 +198,7 @@ int afk_start(struct apple_dcp_afkep *ep) ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); if (ret <= 0 && ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) dev_warn(ep->dcp->dev, "external AFK startup ep=%#x tx_ready=%u rx_ready=%u bytes=%zu\n", ep->endpoint, ep->txbfr.ready, ep->rxbfr.ready, ep->bfr_size); @@ -208,37 +208,6 @@ int afk_start(struct apple_dcp_afkep *ep) return 0; } -/* Read only host-owned ring headers; never consume an entry or notify firmware. */ -void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep) -{ - unsigned long flags; - u32 rx_rptr = 0, rx_wptr = 0, tx_rptr = 0, tx_wptr = 0; - bool rx_ready, tx_ready; - - if (!ep || !ep->wire_v2 || !trace_afk_ring_snapshot_enabled()) - return; - spin_lock_irqsave(&ep->lock, flags); - if (ep->stopping) { - spin_unlock_irqrestore(&ep->lock, flags); - return; - } - rx_ready = READ_ONCE(ep->rxbfr.ready) && ep->rxbfr.rptr && ep->rxbfr.wptr; - tx_ready = READ_ONCE(ep->txbfr.ready) && ep->txbfr.rptr && ep->txbfr.wptr; - dma_rmb(); - if (rx_ready) { - rx_rptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.rptr)); - rx_wptr = le32_to_cpu(READ_ONCE(*ep->rxbfr.wptr)); - } - if (tx_ready) { - tx_rptr = le32_to_cpu(READ_ONCE(*ep->txbfr.rptr)); - tx_wptr = le32_to_cpu(READ_ONCE(*ep->txbfr.wptr)); - } - trace_afk_ring_snapshot(ep, rx_ready, rx_rptr, rx_wptr, - READ_ONCE(ep->rxbfr.bufsz), tx_ready, tx_rptr, - tx_wptr, READ_ONCE(ep->txbfr.bufsz)); - spin_unlock_irqrestore(&ep->lock, flags); -} - static void afk_getbuf(struct apple_dcp_afkep *ep, u64 message) { u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; @@ -751,7 +720,7 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", ep->endpoint, channel, type, eshdr->category); if (ep->dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); } @@ -1031,8 +1000,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); size_t payload_size; - trace_afk_recv_header(ep, channel, type, data_size, data); - if (ep->wire_v2) { afk_v2_receive(ep, channel, type, data, data_size); return; @@ -1152,7 +1119,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, dev_err(ep->dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); if (ep->dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, payload_size, true); return; @@ -1341,38 +1308,16 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, u8 v2_command[AFKV2_COMMAND_SIZE + AFKV2_OOB_SIZE]; u8 v2_category = 0; u16 v2_flags = 0; - bool compact = !ep->wire_v2 && ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; int ret; - /* - * H17P frames EPIC messages differently (see afk_recv_handle()): an - * 8-byte sub-header, with the service channel carried in the EPIC - * header on queue channel 0. Only the receive side of that format is - * known, and the 24-byte framing below would be misparsed, so send - * nothing for the established internal profile. The external-only - * experiment below tests receive-layout symmetry. Internal callers are: the - * callers are optional (verbose firmware logging, EDID copy, audio, - * DPTX, debugfs) or replies to calls only DPTX services make. - */ - if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !compact && !ep->wire_v2) { + /* External H17P services use AFKV2; the internal profile has no EPIC sender. */ + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && !ep->wire_v2) { dev_dbg_once(ep->dcp->dev, "AFK[ep:%02x]: not sending EPIC message on H17P\n", ep->endpoint); return -EOPNOTSUPP; } - /* Only existing DPTX/EDID service commands participate in this trial. */ - if (compact && - (channel > U16_MAX || payload_len > SZ_1M || - (payload_len && !payload) || stype != EPIC_SUBTYPE_STD_SERVICE || - (ep->endpoint != DPTX_ENDPOINT && ep->endpoint != DPAVSERV_ENDPOINT) || - !((ecat == EPIC_CAT_COMMAND && etype == EPIC_TYPE_COMMAND && - payload_len == sizeof(struct epic_cmd)) || - (ecat == EPIC_CAT_REPLY && - (etype == EPIC_TYPE_REPLY || etype == EPIC_TYPE_NOTIFY_ACK))))) - return -EOPNOTSUPP; - if (ep->wire_v2) { if (!channel || channel > U16_MAX || tag > U8_MAX || payload_len > SZ_1M) return -EINVAL; @@ -1418,7 +1363,7 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, goto out; } - if ((compact || ep->wire_v2) && + if (ep->wire_v2 && (!ep->txbfr.ready || !ep->txbfr.rptr || !ep->txbfr.wptr || !ep->txbfr.buf)) { ret = -ESHUTDOWN; goto out; @@ -1429,11 +1374,11 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, wptr = le32_to_cpu(*ep->txbfr.wptr); trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + - ((compact || ep->wire_v2) ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + + (ep->wire_v2 ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + payload_len; total_size = sizeof(*hdr) + total_epic_size; - if (compact || ep->wire_v2) { + if (ep->wire_v2) { u32 available, needed; if (!ep->txbfr.ready || !ep->txbfr.stride || @@ -1536,7 +1481,7 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, hdr->magic = cpu_to_le32(QE_MAGIC); hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32((compact || ep->wire_v2) ? 0 : channel); + hdr->channel = cpu_to_le32(ep->wire_v2 ? 0 : channel); hdr->type = cpu_to_le32(ep->wire_v2 ? 0 : etype); if (hdr2) memcpy(hdr2, hdr, sizeof(*hdr)); @@ -1550,18 +1495,6 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, if (ret) goto out; wptr += written; - } else if (compact) { - u8 *wire = ep->txbfr.buf + wptr; - - /* Experimental symmetry with the published H17P receive layout. */ - memset(wire, 0, sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p)); - wire[0] = ep->qe_seq++; - put_unaligned_le16(channel, wire + 2); - put_unaligned_le32(total_epic_size - 8, wire + 4); - wire[16] = stype; - wire[17] = ecat; - put_unaligned_le32(tag, wire + 20); - wptr += sizeof(*ehdr) + sizeof(struct epic_sub_hdr_h17p); } else { ehdr = ep->txbfr.buf + wptr; memset(ehdr, 0, sizeof(*ehdr)); diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index c26f4453082f6d..ecd7fbf279a9e3 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -380,7 +380,6 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, int afk_v2_copy_owned(struct apple_epic_service *service, u64 dva, void *buffer, size_t size, bool to_peer); int afk_start(struct apple_dcp_afkep *ep); -void afk_trace_ring_snapshot(struct apple_dcp_afkep *ep); void afk_shutdown(struct apple_dcp_afkep *ep); void afk_quiesce(struct apple_dcp_afkep *ep); int afk_receive_message(struct apple_dcp_afkep *ep, u64 message); diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index cd9fbda6895d8b..8e198c3cb1050e 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -811,11 +811,12 @@ static void apple_drm_quiesce_connectors(struct drm_device *drm) * does not have yet. Leave it out of the display subsystem, so that an * enabled node cannot hold back the internal panel's component master. */ +/* A supported firmware-described external provider participates in component matching. */ static bool apple_dcp_usable(struct device_node *np) { return of_device_is_available(np) && (!of_device_is_compatible(np, "apple,t8140-dcpext") || - dcp_h17p_external_enabled()); + dcp_h17p_external_supported(np)); } static int apple_drm_init_dcp(struct device *dev) diff --git a/drivers/gpu/drm/apple/dcp-180hz-policy.h b/drivers/gpu/drm/apple/dcp-180hz-policy.h new file mode 100644 index 00000000000000..a221c0e4a0773f --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-180hz-policy.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_180HZ_POLICY_H +#define APPLE_DCP_180HZ_POLICY_H + +#include +#include +#include + +/* Only this qualified uncompressed configuration is admitted. */ +static inline bool dcp_180hz_payload(u32 rate, u32 lanes, u32 clock) +{ + return rate == 0x1e && lanes == 2 && clock == 414090 && + (u64)clock * 24 <= (u64)8100000 * lanes * 8 / 10; +} + +static inline bool dcp_180hz_dtd(const u8 *d, const struct drm_display_mode *m) +{ + int ha = d[2] | (d[4] & 0xf0) << 4; + int hb = d[3] | (d[4] & 0x0f) << 8; + int va = d[5] | (d[7] & 0xf0) << 4; + int vb = d[6] | (d[7] & 0x0f) << 8; + int hf = d[8] | (d[11] & 0xc0) << 2; + int hs = d[9] | (d[11] & 0x30) << 4; + int vf = (d[10] >> 4) | (d[11] & 0x0c) << 2; + int vs = (d[10] & 0x0f) | (d[11] & 0x03) << 4; + u32 flags = (d[17] & 2 ? DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC) | + (d[17] & 4 ? DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC); + + return ((d[0] | d[1] << 8) * 10 == m->clock) && m->clock == 414090 && + (d[17] & 0x98) == 0x18 && ha == 1920 && va == 1080 && + ha + hb == 2080 && va + vb == 1106 && + m->hdisplay == ha && m->htotal == ha + hb && + m->hsync_start == ha + hf && m->hsync_end == ha + hf + hs && + m->vdisplay == va && m->vtotal == va + vb && + m->vsync_start == va + vf && m->vsync_end == va + vf + vs && + m->flags == flags; +} + +/* Bounded actual EDID only; no serial field is consulted. */ +static inline bool dcp_180hz_edid(const u8 *p, size_t len, + const struct drm_display_mode *mode) +{ + static const u8 header[] = { 0, 255, 255, 255, 255, 255, 255, 0 }; + bool name = false, timing = false, forum = false; + unsigned int names = 0, i, off, end; + + if (!p || !mode || len != 256 || memcmp(p, header, 8) || p[126] != 1 || + p[8] != 0x04 || p[9] != 0x72 || p[10] != 0x4f || p[11] != 0x0b || + p[18] != 1 || p[19] < 3 || p[19] > 4 || !(p[20] & 0x80)) + return false; + for (off = 0; off < len; off += 128) { + u8 sum = 0; + + for (i = 0; i < 128; i++) + sum += p[off + i]; + if (sum) + return false; + } + for (off = 54; off + 18 <= 126; off += 18) { + const u8 *d = p + off; + + if (d[0] || d[1]) { + timing |= dcp_180hz_dtd(d, mode); + } else if (!d[2] && d[3] == 0xfc && !d[4]) { + unsigned int n = 13; + + names++; + while (n && (d[5 + n - 1] == ' ' || d[5 + n - 1] == '\n' || + d[5 + n - 1] == 0)) + n--; + name = (n == 8 && !memcmp(d + 5, "QG270 S3", 8)) || + (n == 7 && !memcmp(d + 5, "QG270S3", 7)); + } + } + p += 128; + if (p[0] != 2 || p[1] < 3 || p[2] < 4 || p[2] > 127) + return false; + end = p[2]; + for (off = 4; off < end;) { + const u8 *db = p + off; + unsigned int size = (db[0] & 31) + 1; + + if (size > end - off) + return false; + if ((db[0] >> 5) == 3 && size >= 4 && + db[1] == 0xd8 && db[2] == 0x5d && db[3] == 0xc4) { + /* HF-VSDB overrides the older HDMI VSDB clock, as drm_edid does. */ + if (forum || size < 8 || db[4] != 1 || !(db[6] & 0x80) || + (u32)db[5] * 5000 < (u32)mode->clock) + return false; + forum = true; + } + off += size; + } + for (off = end; off + 18 <= 127; off += 18) + if (p[off] || p[off + 1]) + timing |= dcp_180hz_dtd(p + off, mode); + return name && names == 1 && timing && forum; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-180hz.c b/drivers/gpu/drm/apple/dcp-180hz.c new file mode 100644 index 00000000000000..8eb4166aa6dbb6 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-180hz.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +#include +#include +#include +#include "connector.h" +#include "afk.h" +#include "dcp.h" +#include "parser.h" +#include "dcp-180hz.h" +#include "dcp-180hz-policy.h" + +static bool dcp_180hz_direct_port(struct apple_epic_service *service) +{ + struct apple_dcp *dcp; + struct dptx_port *port; + + if (!service || !service->ep || !service->cookie) + return false; + dcp = service->ep->dcp; + port = service->cookie; + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + service->ep->endpoint == DPTX_ENDPOINT_V2 && service->ep->wire_v2 && + port == &dcp->dptxport[0] && READ_ONCE(port->enabled) && + !READ_ONCE(port->closed) && READ_ONCE(port->service) == service && + READ_ONCE(service->enabled) && port->atcphy && port->atcphy == dcp->phy && + dcp->active_typec_route && !dcp->dptx_tunnel && !dcp_is_usb4_output(dcp); +} + +void dcp_180hz_invalidate(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; + dcp->rate_proof.retries = 0; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); +} + +struct dcp_180hz_token dcp_180hz_config_begin(struct apple_epic_service *service, bool lanes) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct dcp_180hz_token token = {}; + unsigned long flags; + + /* RX callbacks never acquire lifecycle/HPD/AV mutexes. */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + token.generation = READ_ONCE(dcp->typec_generation); + token.sequence = dcp->rate_proof.sequence; + token.connection_sequence = dcp->rate_proof.connection_sequence; + token.phy = dcp->phy; + token.route = dcp->active_typec_route; + token.service = service; + token.valid = !READ_ONCE(dcp->quiescing) && dcp_180hz_direct_port(service) && + READ_ONCE(dcp->typec_cable_connected) && + token.generation == dcp->metadata.epoch.generation; + if (lanes) + dcp->rate_proof.lanes.valid = false; + else + dcp->rate_proof.rate.valid = false; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return token; +} + +/* Preserve the receipt-start registry ticket even when it awaited OPEN. */ +static bool dcp_180hz_timing_epoch(const struct dcp_display_mode *mode, u64 epoch) +{ + if (mode->automatic_service_pending) + return mode->unsafe_rgb8_service_epoch != U64_MAX && + epoch == mode->unsafe_rgb8_service_epoch + 1; + return epoch && mode->unsafe_rgb8_service_epoch == epoch; +} + +void dcp_180hz_config_complete(struct apple_epic_service *service, bool lanes, + struct dcp_180hz_token token, u32 value, bool success) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct dcp_180hz_fact *fact; + bool notify = false; + unsigned long flags; + + if (!success) { + dcp_180hz_invalidate(dcp); + return; + } + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (token.valid && token.service == service && + token.sequence == dcp->rate_proof.sequence && + token.generation == READ_ONCE(dcp->typec_generation) && + token.connection_sequence == dcp->rate_proof.connection_sequence && + READ_ONCE(dcp->typec_cable_connected) && + token.phy == dcp->phy && token.route == dcp->active_typec_route && + dcp_180hz_direct_port(service) && !READ_ONCE(dcp->quiescing)) { + fact = lanes ? &dcp->rate_proof.lanes : &dcp->rate_proof.rate; + *fact = (struct dcp_180hz_fact) { .token = token, .value = value, .valid = true }; + notify = dcp->rate_proof.rate.valid && dcp->rate_proof.lanes.valid; + dcp->rate_proof.retries = 0; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (notify) + dcp_180hz_notify(dcp); +} + +static bool dcp_180hz_fact_current(struct apple_dcp *dcp, const struct dcp_180hz_fact *fact) +{ + const struct dcp_180hz_token *t = &fact->token; + + return fact->valid && t->generation == dcp->typec_generation && + t->sequence == dcp->rate_proof.sequence && + t->connection_sequence == dcp->rate_proof.connection_sequence && t->phy == dcp->phy && + t->route == dcp->active_typec_route && t->service == dcp->dptxport[0].service; +} + +/* Catalog withdrawal needs a DRM refresh even when no exception is ready. */ +void dcp_180hz_catalog_changed(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || + !con || READ_ONCE(dcp->quiescing) || !READ_ONCE(con->connected) || + READ_ONCE(con->dcp) != to_platform_device(dcp->dev) || + READ_ONCE(con->base.registration_state) != DRM_CONNECTOR_REGISTERED) + return; + schedule_work(&con->hotplug_wq); +} + +/* Timing and configuration can arrive in either order, before AV admission. */ +void dcp_180hz_notify(struct apple_dcp *dcp) +{ + struct apple_connector *con = READ_ONCE(dcp->connector); + unsigned long flags; + bool ready; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp->metadata.epoch.connected && READ_ONCE(dcp->typec_cable_connected) && + !READ_ONCE(dcp->quiescing) && READ_ONCE(dcp->nr_modes) && + dcp_180hz_direct_port(dcp->dptxport[0].service) && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) && + dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) && + dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, 414090); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (ready && con && READ_ONCE(con->connected) && READ_ONCE(dcp->dptxport[0].connected) && + READ_ONCE(con->dcp) == to_platform_device(dcp->dev) && + READ_ONCE(con->base.registration_state) == DRM_CONNECTOR_REGISTERED) + schedule_work(&con->hotplug_wq); +} + +bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +{ + struct apple_connector *con = dcp->connector; + const struct edid *raw; + + lockdep_assert_held(&dcp->hpd_mutex); + lockdep_assert_held(&dcp->dcpavserv.lock); + lockdep_assert_held(&dcp->metadata.lock); + if (!con || dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || + !dcp_180hz_direct_port(dcp->dptxport[0].service) || + !mode->unsafe_rgb8_ticket.associated || + mode->unsafe_rgb8_ticket.generation != dcp->typec_generation || + mode->automatic_connection_sequence != dcp->rate_proof.connection_sequence || + mode->automatic_catalog_sequence != dcp->rate_proof.catalog_sequence || + !dcp->metadata.epoch.connected || !dcp->typec_cable_connected || dcp->quiescing || + READ_ONCE(con->dcp) != to_platform_device(dcp->dev) || + !dcpavserv_edid_service_current(dcp, con->drm_edid_service_epoch) || + !dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) || + !con->drm_edid || con->drm_edid_firmware_derived || + READ_ONCE(con->edid_invalidation_pending) || + con->drm_edid_generation != dcp->typec_generation || + !dcp->dptxport[0].connected || !dcp->dptxport[0].enabled || dcp->dptxport[0].closed || + !dcp_180hz_fact_current(dcp, &dcp->rate_proof.rate) || + !dcp_180hz_fact_current(dcp, &dcp->rate_proof.lanes) || + !dcp_180hz_payload(dcp->rate_proof.rate.value, dcp->rate_proof.lanes.value, mode->mode.clock) || + mode->sdr_rgb.depth != 8 || mode->sdr_rgb.format != DCP_COLOR_FORMAT_RGB || + mode->sdr_rgb.range != DCP_COLOR_YCBCR_RANGE_FULL || + mode->sdr_rgb.eotf != DCP_EOTF_SDR_GAMMA || mode->sdr_rgb.colorimetry != DCP_COLORIMETRY_SRGB) + return false; + raw = drm_edid_raw(con->drm_edid); + /* The transport validates extent/checksums; this quirk accepts only its 256-byte form. */ + return raw && raw->extensions == 1 && + dcp_180hz_edid((const u8 *)raw, 256, &mode->mode); +} + +bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode) +{ + struct apple_connector *con = dcp->connector; + const struct edid *raw; + + if (!con || !dcp->active_typec_route || dcp_is_usb4_output(dcp) || dcp->dptx_tunnel || + !mode->unsafe_rgb8_ticket.associated || + mode->unsafe_rgb8_ticket.generation != dcp->typec_generation || + mode->automatic_connection_sequence != dcp->rate_proof.connection_sequence || + mode->automatic_catalog_sequence != dcp->rate_proof.catalog_sequence || + (!dcp_180hz_timing_epoch(mode, dcp->dcpavserv.epoch) && + !(mode->automatic_service_pending && !dcp->dcpavserv.enabled && + mode->unsafe_rgb8_service_epoch == dcp->dcpavserv.epoch))) + return false; + if (!con->drm_edid) + return true; + raw = drm_edid_raw(con->drm_edid); + return raw && raw->extensions == 1 && dcp_180hz_edid((const u8 *)raw, 256, &mode->mode); +} + +static void dcp_180hz_retry_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, rate_retry_wq); + unsigned long flags; + bool current_generation; + + guard(mutex)(&dcp->hpd_mutex); + spin_lock_irqsave(&dcp->metadata.lock, flags); + current_generation = dcp->typec_cable_connected && !dcp->quiescing && + dcp->rate_proof.retry_generation == dcp->typec_generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (current_generation && dcp->connector && + READ_ONCE(dcp->connector->dcp) == to_platform_device(dcp->dev)) + schedule_work(&dcp->connector->hotplug_wq); +} + +void dcp_180hz_retry(struct apple_dcp *dcp) +{ + unsigned long flags; + bool retry = false; + + if (!dcp->rate_retry_initialized || READ_ONCE(dcp->quiescing)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (dcp->rate_proof.retry_generation != READ_ONCE(dcp->typec_generation)) { + dcp->rate_proof.retry_generation = READ_ONCE(dcp->typec_generation); + dcp->rate_proof.retries = 0; + } + if (READ_ONCE(dcp->typec_cable_connected) && dcp->rate_proof.retries < 8) { + dcp->rate_proof.retries++; + retry = true; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (retry) + mod_delayed_work(system_freezable_wq, &dcp->rate_retry_wq, msecs_to_jiffies(300)); +} + +static void dcp_180hz_cancel(void *data) +{ + struct apple_dcp *dcp = data; + + WRITE_ONCE(dcp->rate_retry_initialized, false); + cancel_delayed_work_sync(&dcp->rate_retry_wq); +} + +int dcp_180hz_init(struct apple_dcp *dcp) +{ + int ret; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) + return 0; + INIT_DELAYED_WORK(&dcp->rate_retry_wq, dcp_180hz_retry_work); + ret = devm_add_action_or_reset(dcp->dev, dcp_180hz_cancel, dcp); + if (!ret) + dcp->rate_retry_initialized = true; + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-180hz.h b/drivers/gpu/drm/apple/dcp-180hz.h new file mode 100644 index 00000000000000..4995dae4cf6489 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-180hz.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_180HZ_H +#define APPLE_DCP_180HZ_H +#include +struct apple_dcp; +struct apple_epic_service; +struct dcp_display_mode; +struct dcp_180hz_token { + /* Successful physical configuration is independent of AV discovery. */ + u64 generation, sequence, connection_sequence; + const void *phy, *route, *service; + bool valid; +}; +struct dcp_180hz_fact { + struct dcp_180hz_token token; + u32 value; + bool valid; +}; +struct dcp_180hz_proof { + u64 sequence, connection_sequence, catalog_sequence, retry_generation; + unsigned int retries; + struct dcp_180hz_fact rate, lanes; +}; +struct dcp_180hz_token dcp_180hz_config_begin(struct apple_epic_service *service, bool lanes); +void dcp_180hz_config_complete(struct apple_epic_service *service, bool lanes, + struct dcp_180hz_token token, u32 value, bool success); +void dcp_180hz_invalidate(struct apple_dcp *dcp); +void dcp_180hz_notify(struct apple_dcp *dcp); +void dcp_180hz_catalog_changed(struct apple_dcp *dcp); +bool dcp_180hz_mode_current(struct apple_dcp *dcp, const struct dcp_display_mode *mode); +bool dcp_180hz_pending(struct apple_dcp *dcp, const struct dcp_display_mode *mode); +void dcp_180hz_retry(struct apple_dcp *dcp); +int dcp_180hz_init(struct apple_dcp *dcp); +#endif diff --git a/drivers/gpu/drm/apple/dcp-external-bw.h b/drivers/gpu/drm/apple/dcp-external-bw.h index a3a799ddefbd7b..99430d3e4d883f 100644 --- a/drivers/gpu/drm/apple/dcp-external-bw.h +++ b/drivers/gpu/drm/apple/dcp-external-bw.h @@ -5,6 +5,19 @@ #include +#define DCP_EXTERNAL_FW_CELLS 3 + +/* The external H17P wire contract is supported only for this firmware release. */ +static inline bool +apple_dcp_external_firmware_supported(const u32 *firmware, int firmware_cells, + const u32 *compat, int compat_cells) +{ + return firmware && compat && firmware_cells == DCP_EXTERNAL_FW_CELLS && + compat_cells == DCP_EXTERNAL_FW_CELLS && + firmware[0] == 26 && firmware[1] == 6 && firmware[2] == 2 && + compat[0] == 26 && compat[1] == 6 && compat[2] == 2; +} + static inline bool apple_dcp_external_bw_valid(u64 size, int scratch, int request) { if (size < sizeof(u32) || scratch < 0 || request < 0) @@ -21,19 +34,5 @@ static inline bool apple_dcp_external_clock_window(u64 start, u64 size, return start == expected_start && size == expected_size; } -/* Resolve optional diagnostic overrides without changing the SoC defaults. */ -static inline bool apple_dcp_external_bw_resolve(u64 size, int default_scratch, - int default_request, int override_scratch, - int override_request, u32 *scratch, u32 *request) -{ - int s = override_scratch == -1 ? default_scratch : override_scratch; - int r = override_request == -1 ? default_request : override_request; - - if (!apple_dcp_external_bw_valid(size, s, r)) - return false; - *scratch = s; - *request = r; - return true; -} #endif diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 913e3902b34b1b..75d9f0ce748e16 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -22,6 +22,7 @@ #include "dcp_backlight.h" #include "dcp-display-metadata.h" #include "dcp-metadata-epoch.h" +#include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" #include "iomfb-state.h" @@ -106,6 +107,10 @@ struct dcp_chunks { bool contiguous; bool payload_complete; struct dcp_metadata_ticket metadata_ticket; + u64 av_service_epoch; /* Captured at property start, never at publication. */ + struct dcp_metadata_ticket mode_ticket; + u64 automatic_connection_sequence; + bool automatic_service_pending; }; #define DCP_MAX_MAPPINGS (128) /* should be enough */ @@ -179,7 +184,7 @@ struct dcp_panel { enum dcp_iomfb_method_profile { DCP_IOMFB_METHODS_DEFAULT, DCP_IOMFB_METHODS_H17G, - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + DCP_IOMFB_METHODS_H17P_EXTERNAL, }; struct apple_dcp_hw_data { @@ -231,6 +236,8 @@ struct apple_dcp { u64 external_rx_generation; u64 external_stop_generation; bool external_cable; + /* RX tombstone: initially unknown; only physical OOB events change it. */ + bool external_physically_absent; bool external_running; bool external_power_attempted; /* iomfb.lock: replay the latest callback after activation succeeds. */ @@ -238,8 +245,6 @@ struct apple_dcp { bool external_hotplug_connected; bool external_stopped; int external_error; - u64 external_power_reply; - bool external_power_reply_valid; struct work_struct external_stop_work; @@ -338,6 +343,9 @@ struct apple_dcp { /* Active chunked transfer. There can only be one at a time. */ struct dcp_chunks chunks; + struct dcp_180hz_proof rate_proof; + struct delayed_work rate_retry_wq; + bool rate_retry_initialized; struct { /* Protects connection tickets and cached metadata snapshots. */ spinlock_t lock; @@ -587,8 +595,6 @@ struct apple_dcp { struct dcp_present_h17p present_h17p; struct dcp_present_state_h17p present_state_h17p; DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); - /* Runtime callbacks whose record size has been reported once. */ - DECLARE_BITMAP(sized_callbacks, IOMFB_MAX_CB); /* Serializes H17P transmit preparation with RTKit receive callbacks. */ struct { diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 927df2d865e107..223b6e0daa0bdd 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -104,7 +104,7 @@ static int dcp_pin_live_session(struct apple_dcp *dcp) if ((!dcp->hw.adopt_live_session && dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) || + DCP_IOMFB_METHODS_H17P_EXTERNAL) || dcp->fw_compat != DCP_FIRMWARE_H17P || dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) return 0; @@ -155,24 +155,22 @@ static void dcp_release_rtkit(void *data) #define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) -static bool experimental_h17p_external = true; -module_param(experimental_h17p_external, bool, 0444); -MODULE_PARM_DESC(experimental_h17p_external, - "Enable T8140 external display protocol (default: enabled)"); - -static int experimental_h17p_external_bw_scratch = -1; -module_param(experimental_h17p_external_bw_scratch, int, 0444); -MODULE_PARM_DESC(experimental_h17p_external_bw_scratch, - "Override external bandwidth scratch offset (-1 uses the hardware profile)"); - -static int experimental_h17p_external_bw_request = -1; -module_param(experimental_h17p_external_bw_request, int, 0444); -MODULE_PARM_DESC(experimental_h17p_external_bw_request, - "Override external bandwidth request offset (-1 uses the hardware profile)"); - -bool dcp_h17p_external_enabled(void) +bool dcp_h17p_external_supported(struct device_node *np) { - return experimental_h17p_external; + u32 firmware[DCP_EXTERNAL_FW_CELLS], compat[DCP_EXTERNAL_FW_CELLS]; + int firmware_cells, compat_cells; + + if (!of_machine_is_compatible("apple,j700") || + !of_device_is_compatible(np, "apple,t8140-dcpext")) + return false; + firmware_cells = of_property_read_variable_u32_array( + np, "apple,firmware-version", firmware, + DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); + compat_cells = of_property_read_variable_u32_array( + np, "apple,firmware-compat", compat, + DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); + return apple_dcp_external_firmware_supported(firmware, firmware_cells, + compat, compat_cells); } bool dcp_h17p_external_ready(struct device_node *np) @@ -181,7 +179,7 @@ bool dcp_h17p_external_ready(struct device_node *np) struct apple_dcp *dcp; bool ready = false; - if (!dcp_h17p_external_enabled()) + if (!dcp_h17p_external_supported(np)) return false; pdev = of_find_device_by_node(np); if (!pdev) @@ -299,6 +297,14 @@ static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); } +static bool dcp_is_external_h17p(struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; +} + +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation); +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical); + static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, struct mux_control *xbar); static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); @@ -1670,25 +1676,7 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) { - return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; -} - -/* Called under metadata.lock; no monitor contents or pointers are exposed. */ -static void dcp_trace_metadata(struct apple_dcp *dcp, const char *phase, bool valid) -{ - u32 flags = (dcp->metadata.epoch.connected ? BIT(0) : 0) | - (dcp->metadata.epoch.associated_once ? BIT(1) : 0) | - (dcp->metadata.epoch.invalidated ? BIT(2) : 0) | - (dcp->metadata.attributes_valid ? BIT(3) : 0) | - (dcp->metadata.timing_valid ? BIT(4) : 0) | - (dcp->chunks.metadata_ticket.associated ? BIT(5) : 0) | - (dcp->chunks.contiguous ? BIT(6) : 0) | (valid ? BIT(7) : 0) | - (dcp->chunks.payload_complete ? BIT(8) : 0); - - if (dcp_firmware_metadata_enabled(dcp)) - trace_iomfb_metadata(dcp, phase, dcp->metadata.epoch.generation, - dcp->chunks.metadata_ticket.generation, flags, - dcp->chunks.received, dcp->chunks.length); + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; } void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) @@ -1699,17 +1687,21 @@ void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, boo return; spin_lock_irqsave(&dcp->metadata.lock, flags); if (generation != dcp->metadata.epoch.generation || !connected) { + dcp->rate_proof.connection_sequence++; + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; dcp->metadata.attributes_valid = false; dcp->metadata.timing_valid = false; dcp->metadata.hpd_seen = false; } dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); - dcp_trace_metadata(dcp, "connection", connected); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) { + struct apple_connector *con = READ_ONCE(dcp->connector); unsigned long flags; if (!dcp_firmware_metadata_enabled(dcp)) @@ -1717,14 +1709,29 @@ void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) spin_lock_irqsave(&dcp->metadata.lock, flags); if (connected) { dcp->metadata.hpd_seen = true; + /* + * Fresh proof may precede HPD assertion after a same-cable loss. + * Keep invalidated firmware metadata and old receipt tickets revoked. + */ + if (READ_ONCE(dcp->typec_cable_connected) && + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) + dcp->metadata.epoch.connected = true; } else if (dcp->metadata.hpd_seen) { + dcp->metadata.hpd_seen = false; + dcp->rate_proof.connection_sequence++; + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; + if (con && READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + WRITE_ONCE(con->edid_invalidation_pending, true); dcp_metadata_epoch_connection(&dcp->metadata.epoch, dcp->metadata.epoch.generation, false); dcp->metadata.attributes_valid = false; dcp->metadata.timing_valid = false; } - dcp_trace_metadata(dcp, "hotplug", connected); spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (connected) + dcp_180hz_notify(dcp); } void dcp_firmware_metadata_begin(struct apple_dcp *dcp) @@ -1736,7 +1743,6 @@ void dcp_firmware_metadata_begin(struct apple_dcp *dcp) dcp->chunks.received = 0; dcp->chunks.contiguous = true; dcp->chunks.payload_complete = false; - dcp_trace_metadata(dcp, "begin", dcp->chunks.metadata_ticket.associated); spin_unlock_irqrestore(&dcp->metadata.lock, flags); } @@ -1775,7 +1781,6 @@ void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; } } - dcp_trace_metadata(dcp, "attributes", !!attributes); spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_firmware_metadata_notify(dcp); } @@ -1803,7 +1808,6 @@ void dcp_firmware_metadata_timing(struct apple_dcp *dcp) break; } unlock: - dcp_trace_metadata(dcp, "timing", dcp->metadata.timing_valid); spin_unlock_irqrestore(&dcp->metadata.lock, flags); dcp_firmware_metadata_notify(dcp); } @@ -1834,7 +1838,6 @@ const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); - dcp_trace_metadata(dcp, "probe", valid); if (valid) { attributes = dcp->metadata.attributes; preferred = dcp->metadata.preferred; @@ -2305,6 +2308,11 @@ int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + /* External H17P does not program adaptive sync. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL && + crtc_state->vrr_enabled) + return -EINVAL; /* * A Type-C display routed to another pipeline: its route follows * this CRTC if it can (see dcp_typec_follow_check()), and the mode is @@ -2957,6 +2965,8 @@ static void dcp_external_report_disconnected(struct apple_dcp *dcp) unsigned int action; guard(mutex)(&dcp->iomfb.lock); + /* A paused callback from the old sink must not be replayed on wake. */ + dcp->external_hotplug_pending = false; action = dcp_mode_hotplug(&dcp->mode_state, false, dcp->connector ? &dcp->connector->connected : NULL); if (dcp->connector) { @@ -3086,6 +3096,7 @@ static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) guard(mutex)(&dcp->external_lock); /* Invalidate physical EDID before waiting for firmware power/link teardown. */ if (physical) { + WRITE_ONCE(dcp->external_physically_absent, true); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, false); dcp->typec_generation++; @@ -3130,7 +3141,7 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) return false; /* No lifecycle or HPD mutex may be acquired from the RX callback. */ lockdep_assert_held(&dcp->iomfb.lock); - if (READ_ONCE(dcp->quiescing)) + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->external_physically_absent)) return true; if (dcp->iomfb.presentation_paused) { dcp->external_hotplug_pending = true; @@ -3152,6 +3163,8 @@ bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) return true; } if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { + /* Fence admission in RX before a busy actor can delay teardown. */ + dcp_firmware_metadata_hotplug(dcp, false); dcp->external_stop_generation = dcp->external_rx_generation; schedule_work(&dcp->external_stop_work); return true; @@ -3366,6 +3379,7 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) if (new_cable) dcp->external_generation++; dcp->external_cable = true; + WRITE_ONCE(dcp->external_physically_absent, false); dcp_external_publish_generation(dcp); scoped_guard(mutex, &dcp->hpd_mutex) { WRITE_ONCE(dcp->typec_cable_connected, true); @@ -3619,7 +3633,7 @@ int dcp_start(struct platform_device *pdev) } /* Make the H17P allocator available before external service discovery. */ - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) return dev_err_probe(dcp->dev, ret, "Failed to start remote allocator\n"); @@ -3644,7 +3658,7 @@ int dcp_start(struct platform_device *pdev) ret); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = dptx_controller_init(dcp); if (ret) return dev_err_probe(dcp->dev, ret, @@ -4413,7 +4427,7 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) if (dcp->hw.firmware_clock) { unsigned int windows = dcp->hw.firmware_display_windows; bool external = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; if (!windows || windows >= MAX_DISP_REGISTERS) return -EINVAL; @@ -4787,7 +4801,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat } if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && !dcp_external_is_idle(dcp)) return dev_err_probe(dev, -EBUSY, "External DCP is not in the inherited idle state\n"); @@ -4892,7 +4906,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat } if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && (!dcp->clk_194 || !clk_get_rate(dcp->clk) || !clk_get_rate(dcp->clk_194))) { ret = dev_err_probe(dev, -EINVAL, @@ -4908,7 +4922,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Pin before the first write or mailbox wake can expose DMA. */ ret = dcp_pin_live_session(dcp); if (ret) @@ -4943,7 +4957,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; /* External H17P supplies shared-buffer addresses during RTKit startup. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) dcp->rtk = apple_rtkit_init(dev, dcp, "mbox", 0, &external_h17p_rtkit_ops); else @@ -4990,7 +5004,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) } /* Announce host readiness before awaiting external IOP readiness. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) ret = apple_rtkit_wake_early_ap(dcp->rtk); else ret = apple_rtkit_wake(dcp->rtk); @@ -5221,10 +5235,10 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (dcp_session_retained(dev)) return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); - /* Not part of the display subsystem yet; see apple_dcp_usable(). */ + /* Reject unsupported external firmware before replacing the boot display. */ if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext") && - !dcp_h17p_external_enabled()) { - dev_info(dev, "external display coprocessor not supported yet\n"); + !dcp_h17p_external_supported(dev->of_node)) { + dev_err(dev, "Unsupported external display hardware or firmware handoff\n"); return -ENODEV; } @@ -5284,17 +5298,13 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) mutex_init(&dcp->dcpavserv.lock); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { if (!dcp->hw.firmware_clock || - !apple_dcp_external_bw_resolve(resource_size(dcp->hw.firmware_clock), + !apple_dcp_external_bw_valid(resource_size(dcp->hw.firmware_clock), dcp->hw.firmware_scratch, - dcp->hw.firmware_request, - experimental_h17p_external_bw_scratch, - experimental_h17p_external_bw_request, - &dcp->hw.firmware_scratch, - &dcp->hw.firmware_request)) + dcp->hw.firmware_request)) return dev_err_probe(dev, -EINVAL, - "External bandwidth offsets are invalid\n"); + "Invalid external profile bandwidth resources\n"); iomfb_pause_presentation(dcp, true); } @@ -5678,7 +5688,7 @@ static int dcp_platform_probe(struct platform_device *pdev) dcp_ext_give_up(dcp, ret, "probe"); return component_add(dev, &dcp_comp_ops); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Complete optional startup before the DRM master can depend on it. */ ret = dcp_comp_bind(dev, NULL, NULL); if (ret) { @@ -5720,7 +5730,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) if (dcp && dcp->external && !dcp->external_native) return; if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + DCP_IOMFB_METHODS_H17P_EXTERNAL) { dcp_comp_unbind(&pdev->dev, NULL, NULL); return; } @@ -6014,7 +6024,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8140 = { /* Separate from the established internal-panel H17P profile. */ static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external = { .num_dptx_ports = 1, - .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL, + .iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL, .adopt_live_session = false, .firmware_compat = DCP_FIRMWARE_H17P, .firmware_clock = &t8140_firmware_clock, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index ad6a76e22fe4c8..243c5cb8504f96 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -21,6 +21,7 @@ struct apple_atomic_state { #define to_apple_atomic_state(s) container_of(s, struct apple_atomic_state, base) bool dcp_h17p_external_enabled(void); +bool dcp_h17p_external_supported(struct device_node *np); bool dcp_h17p_external_ready(struct device_node *np); struct apple_crtc { diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 3e529a20115c13..6fbdffa7802519 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -18,6 +18,7 @@ #include "dcp.h" #include "dcp-hdmi.h" #include "dptxep.h" +#include "dcp-180hz.h" #include "dptx-service.h" #include "parser.h" #include "trace.h" @@ -234,6 +235,7 @@ int dptxport_release_display(struct apple_epic_service *service) struct dptx_port *dptx = service->cookie; int ret; + dcp_180hz_invalidate(service->ep->dcp); dptx->release_calls++; dev_dbg(service->ep->dcp->dev, "DPTX release_display: call #%u this boot caller=%pS\n", @@ -522,6 +524,7 @@ dptxport_call_will_change_link_config(struct apple_epic_service *service) { struct dptx_port *dptx = service->cookie; + dcp_180hz_invalidate(service->ep->dcp); dptx->phy_ops.dp.set_lanes = 0; dptx->phy_ops.dp.set_rate = 0; dptx->phy_ops.dp.set_voltages = 0; @@ -756,6 +759,40 @@ dptxport_call_deactivate(struct apple_epic_service *service, return 0; } +static int dptxport_call_qualified_config(struct apple_epic_service *service, bool lanes, + const void *data, size_t size, void *reply, size_t room) +{ + struct dcp_180hz_token token = dcp_180hz_config_begin(service, lanes); + struct dptx_port *port = service->cookie; + bool success = false; + u32 value = 0; + int ret; + + if (lanes) { + ret = dptxport_call_set_active_lane_count(service, data, size, reply, room); + if (!ret && size >= sizeof(struct dptxport_apcall_set_active_lane_count) && + room >= sizeof(struct dptxport_apcall_set_active_lane_count)) { + const struct dptxport_apcall_set_active_lane_count *request = data; + const struct dptxport_apcall_set_active_lane_count *response = reply; + + value = le64_to_cpu(request->lane_count); + success = !le32_to_cpu(response->retcode) && value == 2 && port->lane_count == 2; + } + } else { + ret = dptxport_call_set_link_rate(service, data, size, reply, room); + if (!ret && size >= sizeof(struct dptxport_apcall_link_rate) && + room >= sizeof(struct dptxport_apcall_link_rate)) { + const struct dptxport_apcall_link_rate *request = data; + const struct dptxport_apcall_link_rate *response = reply; + + value = le32_to_cpu(request->link_rate); + success = !le32_to_cpu(response->retcode) && value == LINK_RATE_HBR3 && port->link_rate == value; + } + } + dcp_180hz_config_complete(service, lanes, token, value, success); + return ret; +} + static int dptxport_call(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size) @@ -765,7 +802,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, dev_dbg(service->ep->dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", idx, data_size); if (data_size && service->ep->dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) print_hex_dump_debug("DPTXPort: apcall data: ", DUMP_PREFIX_OFFSET, 16, 1, data, min(data_size, (size_t)64), true); @@ -799,8 +836,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_LINK_RATE: return dptxport_call_get_link_rate(service, reply, reply_size); case DPTX_APCALL_SET_LINK_RATE: - return dptxport_call_set_link_rate(service, data, data_size, - reply, reply_size); + return dptxport_call_qualified_config(service, false, data, data_size, reply, reply_size); case DPTX_APCALL_GET_MAX_LANE_COUNT: return dptxport_call_get_max_lane_count(service, reply, reply_size); case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { @@ -813,8 +849,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; } case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: - return dptxport_call_set_active_lane_count(service, data, data_size, - reply, reply_size); + return dptxport_call_qualified_config(service, true, data, data_size, reply, reply_size); case DPTX_APCALL_GET_SUPPORTS_HPD: return dptxport_call_get_supports_hpd(service, reply, reply_size); @@ -879,7 +914,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; default: if (service->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) return -EOPNOTSUPP; /* just try to ACK and hope for the best... */ dev_info(service->ep->dcp->dev, "DPTXPort: acking unhandled call %u\n", @@ -953,7 +988,7 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, } /* Publication may follow the startup wait or the bounded retry window. */ if (service->ep->dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && READ_ONCE(service->ep->dcp->typec_cable_connected) && !READ_ONCE(service->ep->dcp->quiescing) && !READ_ONCE(service->ep->dcp->external_error)) @@ -1045,11 +1080,17 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, return 0; } +static void dptxport_qualified_teardown(struct apple_epic_service *service) +{ + dcp_180hz_invalidate(service->ep->dcp); + dptx_service_unbind(service); +} + static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .prepare = dptxport_prepare, - .teardown = dptx_service_unbind, + .teardown = dptxport_qualified_teardown, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, @@ -1082,7 +1123,7 @@ int dptx_controller_init(struct apple_dcp *dcp) return PTR_ERR(ep); /* Controller interfaces require the same OPEN handshake as the port. */ ep->wire_v2 = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; dcp->dptx_controller_ep = ep; ret = afk_start(ep); @@ -1119,7 +1160,7 @@ int dptxep_init(struct apple_dcp *dcp) { struct apple_dcp_afkep *ep; bool wire_v2 = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; int ret; u32 port; unsigned long timeout = msecs_to_jiffies(1000); @@ -1161,7 +1202,7 @@ int dptxep_init(struct apple_dcp *dcp) timeout = ret; } - return 0; + return dcp_180hz_init(dcp); } static void dpav_ctrl_bind(struct apple_epic_service *service, const char *name, diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 926cf064040f54..dc37f4afa7f0f8 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -367,7 +367,7 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) { lockdep_assert_held(&dcp->hpd_mutex); return dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL || + DCP_IOMFB_METHODS_H17P_EXTERNAL || (dcp->typec_cable_connected && generation == dcp->typec_generation && !READ_ONCE(dcp->quiescing)); } @@ -479,13 +479,13 @@ int dpavservep_init(struct apple_dcp *dcp) init_completion(&dcp->dcpavserv.enable_completion); ep = afk_init(dcp, dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL ? + DCP_IOMFB_METHODS_H17P_EXTERNAL ? DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ ep->wire_v2 = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; ep->inline_v2 = false; dcp->dcpavservep = ep; @@ -506,7 +506,6 @@ int dpavservep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); if (!ret) { - afk_trace_ring_snapshot(ep); dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); return -ETIMEDOUT; } diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index 153971148f1559..630110bc8cd555 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -766,7 +766,7 @@ int ibootep_init(struct apple_dcp *dcp) } const struct apple_epic_service_ops *ops = ibootep_ops; - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) ops = external_ibootep_ops; dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); if (IS_ERR(dcp->ibootep)) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 298696d99700ba..e45a700c98df6a 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -195,7 +195,7 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) lockdep_assert_held(&dcp->iomfb.lock); /* DCPEXT rejects enableGPAlphaDiv; retain the panel-only workaround. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) return 0; if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; @@ -924,7 +924,7 @@ static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, return tag == 134 && context == DCP_CONTEXT_ASYNC && in_len == 0 && out_len == sizeof(u32) && dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && of_machine_is_compatible("apple,j700") && of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); } @@ -984,7 +984,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0], hdr->in_len, hdr->out_len, context); if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && !zero_reply) { + DCP_IOMFB_METHODS_H17P_EXTERNAL && !zero_reply) { /* Retain the failed session without inventing a successful reply. */ dcp->crashed = true; complete(&dcp->start_done); @@ -1290,6 +1290,53 @@ void dcp_modes_release(void *data) kfree(dcp->modes); dcp->modes = NULL; dcp->nr_modes = 0; +/* Retained arrays are not freed while mode readers can still refer to them. */ +static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, + unsigned int *count) +{ + struct dcp_display_mode *modes; + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + modes = dcp->modes; + *count = dcp->nr_modes; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return modes; +} + +/* Caller holds hpd_mutex -> dcpavserv.lock -> metadata.lock. */ +static bool dcp_exception_mode_current_locked(struct apple_dcp *dcp, + const struct dcp_display_mode *mode) +{ + if (!mode->automatic_rgb8_quirk) + return true; + return dcp_180hz_mode_current(dcp, mode); +} + +static bool dcp_publish_mode(struct apple_dcp *dcp, struct drm_connector *connector, + const struct dcp_display_mode *source, + struct drm_display_mode *copy) +{ + unsigned long flags; + bool eligible; + + /* Ordinary modes keep their existing publication policy. */ + if (!source->automatic_rgb8_quirk) { + drm_mode_probed_add(connector, copy); + return true; + } + /* External get_modes holds hpd_mutex after its final EDID generation check. */ + /* Allocation happened outside metadata.lock; recheck before linking. */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + eligible = dcp->connector && &dcp->connector->base == connector && + dcp_exception_mode_current_locked(dcp, source); + if (eligible) + /* drm_mode_probed_add only links a list under mode_config.mutex. */ + drm_mode_probed_add(connector, copy); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!eligible) + drm_mode_destroy(connector->dev, copy); + return eligible; } int dcp_get_modes(struct drm_connector *connector) @@ -1300,11 +1347,15 @@ int dcp_get_modes(struct drm_connector *connector) struct drm_device *dev = connector->dev; struct drm_display_mode *mode; + struct dcp_display_mode *modes; + unsigned int nr_modes, published = 0; u16 min_vfreq = 0, max_vfreq = 0; bool vrr_capable = false; bool physical_edid; + bool pending_180hz = false, published_180hz = false; u64 edid_generation = 0; u64 service_epoch = 0; + unsigned int i; unsigned int count; int i; @@ -1320,6 +1371,7 @@ int dcp_get_modes(struct drm_connector *connector) return 0; } dcp = platform_get_drvdata(pdev); + modes = dcp_modes_snapshot(dcp, &nr_modes); mutex_lock(&dcp->modes_lock); count = dcp_modes_for_connector(dcp, apple_connector) ? dcp->nr_modes : 0; @@ -1348,7 +1400,7 @@ int dcp_get_modes(struct drm_connector *connector) drm_connector_set_vrr_capable_property(connector, vrr_capable); physical_edid = dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL; + DCP_IOMFB_METHODS_H17P_EXTERNAL; /* get_modes holds mode_config.mutex; HPD paths never acquire it. */ if (physical_edid) { mutex_lock(&dcp->hpd_mutex); @@ -1369,7 +1421,7 @@ int dcp_get_modes(struct drm_connector *connector) } /* Internal H17P still has no supported EDID command transport. */ - if (dcp->nr_modes && !apple_connector->drm_edid && + if (nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || (dcp->dcpavservep && dcp->dcpavservep->wire_v2))) { (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { @@ -1426,12 +1478,18 @@ int dcp_get_modes(struct drm_connector *connector) generation, edid); } } - if (dcp->nr_modes && apple_connector->drm_edid) { - apple_connector->drm_edid_firmware_derived = false; - drm_edid_connector_update(connector, apple_connector->drm_edid); + if (nr_modes && apple_connector->drm_edid) { + int update = drm_edid_connector_update(connector, apple_connector->drm_edid); + + if (update) { + dev_info(dcp->dev, "raw EDID publication failed: %d\n", update); + dcp_clear_connector_edid(apple_connector); + } else { + apple_connector->drm_edid_firmware_derived = false; + } } else { u64 generation = 0; - const struct drm_edid *derived = dcp->nr_modes ? + const struct drm_edid *derived = nr_modes ? dcp_firmware_edid(dcp, &generation) : ERR_PTR(-ENODATA); if (!IS_ERR(derived)) { @@ -1466,9 +1524,38 @@ int dcp_get_modes(struct drm_connector *connector) } edid_done: + /* Publish after a possibly delayed raw read, under its physical generation. */ + for (i = 0; i < nr_modes; ++i) { + mode = drm_mode_duplicate(dev, &modes[i].mode); + if (!mode) { + dev_err(dev->dev, "Failed to duplicate display mode\n"); + published = 0; + goto modes_done; + } + pending_180hz |= modes[i].automatic_rgb8_quirk && + dcp_180hz_pending(dcp, &modes[i]); + if (!dcp_publish_mode(dcp, connector, &modes[i], mode)) + continue; + published_180hz |= modes[i].automatic_rgb8_quirk; + published++; + if (modes[i].vrr) { + u16 lo = modes[i].min_vrr >> 16; + u16 hi = modes[i].max_vrr >> 16; + + if (!min_vfreq || lo < min_vfreq) + min_vfreq = lo; + if (hi > max_vfreq) + max_vfreq = hi; + } + vrr_capable |= modes[i].vrr; + } + drm_connector_set_vrr_capable_property(connector, vrr_capable); +modes_done: mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); + if (pending_180hz && !published_180hz) + dcp_180hz_retry(dcp); if (apple_connector->drm_edid) dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); @@ -1484,6 +1571,7 @@ int dcp_get_modes(struct drm_connector *connector) connector->display_info.monitor_range.max_vfreq = max_vfreq; } + return published; return count; } @@ -1492,6 +1580,10 @@ static bool lookup_mode_locked(struct apple_dcp *dcp, const struct drm_display_mode *mode, struct dcp_display_mode *out) { + struct dcp_display_mode *modes; + unsigned int count, i; + unsigned long flags; + bool eligible; bool found = false; int i; @@ -1505,6 +1597,22 @@ static bool lookup_mode_locked(struct apple_dcp *dcp, *out = dcp->modes[i]; found = true; break; + modes = dcp_modes_snapshot(dcp, &count); + for (i = 0; i < count; ++i) { + if (drm_mode_match(mode, &modes[i].mode, + DRM_MODE_MATCH_TIMINGS | + DRM_MODE_MATCH_CLOCK)) { + if (!modes[i].automatic_rgb8_quirk) + return &modes[i]; + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + spin_lock_irqsave(&dcp->metadata.lock, flags); + eligible = dcp_exception_mode_current_locked(dcp, &modes[i]); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + mutex_unlock(&dcp->dcpavserv.lock); + mutex_unlock(&dcp->hpd_mutex); + if (eligible) + return &modes[i]; } } out_unlock: @@ -1535,7 +1643,7 @@ static bool dcp_mode_settable(struct apple_dcp *dcp, if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) + DCP_IOMFB_METHODS_H17P_EXTERNAL) return dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; return true; @@ -1570,7 +1678,7 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, /* Disconnect cannot pass the admission check before these RPCs finish. */ guard(mutex)(&dcp->external_lock); - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && (!dcp->external_running || dcp->external_error || READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->crashed) || READ_ONCE(dcp->iomfb.presentation_paused))) return -EPIPE; @@ -2156,7 +2264,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * display DART faults. */ if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL) { + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) return ret; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 02bb44d76c29b2..99948cab7158fb 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -446,7 +446,7 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, } else if (!of_property_present(node, property->dt_name) && of_machine_is_compatible("apple,j700") && (of_device_is_compatible(node, "apple,t8140-dcp") || - (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && of_device_is_compatible(node, "apple,t8140-dcpext")))) { /* Qualified older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); @@ -696,26 +696,10 @@ static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { [599] = trampoline_nop, /* find_swap_function_gated */ }; -/* - * Validate a callback record and, the first time each runtime callback whose - * length is only bounded arrives with a different length, report the length - * actually seen. Those reports are the first capture of these records. - */ bool iomfb_check_callback_h17p(struct apple_dcp *dcp, int tag, u32 in_len, u32 out_len) { - const struct dcp_callback_size *size; - - if (!iomfb_validate_callback_h17p(tag, in_len, out_len)) - return false; - size = &callback_sizes[tag]; - if (size->minimum && - (in_len != size->in_len || out_len != size->out_len) && - !test_and_set_bit(tag, dcp->sized_callbacks)) - dev_info(dcp->dev, - "callback D%03d record is %#x/%#x bytes; its handler uses %#x/%#x\n", - tag, in_len, out_len, size->in_len, size->out_len); - return true; + return iomfb_validate_callback_h17p(tag, in_len, out_len); } static_assert(sizeof(struct dcp_set_power_state_resp) == 8); @@ -738,10 +722,6 @@ static void iomfb_external_power_complete(struct apple_dcp *dcp, void *out, void header->out_len == sizeof(struct dcp_set_power_state_resp)) { u64 reply = get_unaligned_le64(out); - WRITE_ONCE(dcp->external_power_reply, reply); - WRITE_ONCE(dcp->external_power_reply_valid, true); - dev_info(dcp->dev, "A473 power=%u reply=%#llx\n", - cookie->on, reply); ret = reply == (u64)cookie->on ? 0 : -EREMOTEIO; } } @@ -798,7 +778,6 @@ int iomfb_external_power_h17p(struct apple_dcp *dcp, bool on) if (!cookie) return -ENOMEM; cookie->on = on; - WRITE_ONCE(dcp->external_power_reply_valid, false); cookie->wait.status = -ETIMEDOUT; init_completion(&cookie->wait.done); kref_init(&cookie->wait.refcount); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 39af92ed6b5c59..beeabad7267c13 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -69,13 +69,6 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, const void *data = request; if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - for (i = 0; i < SWAP_SURFACES; i++) { - const struct dcp_surface *surface = &request->surf[i].base; - - if (!request->surf_null[i]) - trace_iomfb_surface(dcp, i, surface->format, surface->xfer_func, - surface->colorspace, surface->is_premultiplied); - } iomfb_serialize_present_h17p(&dcp->present_h17p, request); if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { if (cb) @@ -981,7 +974,30 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) dcp->chunks.modes_generation = dcp_modes_transfer_begin(dcp); dcp->chunks.length = *length; - dcp_firmware_metadata_begin(dcp); + dcp->chunks.av_service_epoch = 0; + dcp->chunks.automatic_service_pending = false; + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { + /* Match the receipt's start, not a replacement service at its end. */ + mutex_lock(&dcp->dcpavserv.lock); + dcp->chunks.av_service_epoch = dcp->dcpavserv.epoch; + dcp->chunks.automatic_service_pending = + !dcp->dcpavserv.enabled && !dcp->dcpavserv.service; + dcp_firmware_metadata_begin(dcp); + { + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->chunks.automatic_connection_sequence = dcp->rate_proof.connection_sequence; + dcp->chunks.mode_ticket = dcp_metadata_begin(&dcp->metadata.epoch); + dcp->chunks.mode_ticket.associated = + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation) && + READ_ONCE(dcp->typec_cable_connected); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + } + mutex_unlock(&dcp->dcpavserv.lock); + } else { + dcp_firmware_metadata_begin(dcp); + } dcp->chunks.data = kzalloc(*length, GFP_KERNEL); if (!dcp->chunks.data) { @@ -2120,7 +2136,6 @@ static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) { u32 identity = 0x10000; - trace_iomfb_control_result(dcp, 423, !!out, out ? get_unaligned_le32(out) : 0); dcp_pipe_set_428(dcp, false, &identity, dcp_post_swap_a428, cookie); } #endif @@ -2335,7 +2350,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru * Keep YUV, other source spaces and other profiles unchanged. */ if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && + DCP_IOMFB_METHODS_H17P_EXTERNAL && !new_state->fb->format->is_yuv && req->surf[l].base.colorspace == DCP_COLORSPACE_NATIVE) req->surf[l].base.colorspace = DCP_COLORSPACE_BG_SRGB; @@ -2400,7 +2415,6 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } - trace_iomfb_ctm(dcp, mat.location, mat.matrix, mat.matrix_null); #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); #else diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 5cd3849d44d3dc..ad5563cf0b8e50 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -362,6 +362,8 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool require_rate) { + static const char * const fields[] = { "Active", "Total", "FrontPorch", "SyncWidth", "PreciseSyncRate" }; + unsigned int seen = 0; struct iterator it; unsigned int fields = 0; int ret = 0; @@ -372,6 +374,14 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); + unsigned int field; + + if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) + for (field = 0; field < 5; field++) + if (!strcmp(key, fields[field])) { + handle->ambiguous_180hz |= !!(seen & (1U << field)); + seen |= 1U << field; + } if (IS_ERR(key)) { ret = PTR_ERR(key); @@ -581,10 +591,11 @@ static int parse_timing_color_restrictions(struct dcp_parse_ctx *handle, static int parse_color_modes(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, bool external_sdr, - const struct timing_color_restrictions *limits) + const struct timing_color_restrictions *limits, + struct dcp_color_mode *unsafe_rgb8) { struct iterator outer_it; - unsigned int best_sdr_rank = 0; + unsigned int best_sdr_rank = 0, best_unsafe_rank = 0; int ret = 0; out->sdr_444.score = -1; out->sdr_rgb.score = -1; @@ -597,6 +608,8 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, for (; outer_it.idx < outer_it.len; ++outer_it.idx) { struct iterator it; + static const char * const fields[] = { "Colorimetry", "Depth", "DynamicRange", "EOTF", "ID", "PixelEncoding", "Score", "IsVirtual" }; + unsigned int seen = 0; bool is_virtual = true; struct color_mode cmode = { .colorimetry = -1, .depth = -1, .dynamic_range = -1, @@ -609,6 +622,14 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); + unsigned int field; + + if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) + for (field = 0; field < 8; field++) + if (!strcmp(key, fields[field])) { + handle->ambiguous_180hz |= !!(seen & (1U << field)); + seen |= 1U << field; + } if (IS_ERR(key)) ret = PTR_ERR(key); @@ -652,10 +673,25 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, unsigned int rank; if (cmode.id <= U32_MAX && - (timing_color_contains(&limits->unsafe, cmode.id) || - timing_color_contains(&limits->dsc_required, cmode.id))) + timing_color_contains(&limits->dsc_required, cmode.id)) continue; rank = external_sdr_rank(&cmode); + if (cmode.id <= U32_MAX && + timing_color_contains(&limits->unsafe, cmode.id)) { + /* + * Collect only a timing-local bounded RGB8 candidate. + * Admission still requires current EDID and link proof. + */ + if (unsafe_rgb8 && cmode.depth == 8 && rank && + cmode.colorimetry == DCP_COLORIMETRY_SRGB && + rank >= best_unsafe_rank) { + if (rank > best_unsafe_rank) + unsafe_rgb8->score = -1; + fill_color_mode(unsafe_rgb8, &cmode); + best_unsafe_rank = rank; + } + continue; + } if (!rank || rank < best_sdr_rank) continue; if (rank > best_sdr_rank) @@ -698,10 +734,17 @@ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, int height_mm, unsigned notch_height, bool internal, bool external_sdr) { + static const char * const fields[] = { + "ID", "Score", "MinimumVariableRefreshRate", "MaximumVariableRefreshRate", + }; + unsigned int seen = 0; int ret = 0; struct iterator it; struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; struct timing_color_restrictions limits = {}; + struct dcp_color_mode unsafe_rgb8 = { .score = -1 }; + bool have_horiz = false, have_vert = false, have_virtual = false; + bool duplicate_scope = false; struct dimension horiz = {}, vert = {}; u64 clock; s64 min_vrr = 0, max_vrr = 0; @@ -719,6 +762,7 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->sdr.score = -1; out->best.score = -1; *score = -1; + handle->ambiguous_180hz = false; if (external_sdr) { struct dcp_parse_ctx scan = *handle; @@ -734,26 +778,44 @@ static int parse_mode(struct dcp_parse_ctx *handle, for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); + unsigned int field; + + if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) + for (field = 0; field < ARRAY_SIZE(fields); field++) + if (!strcmp(key, fields[field])) { + handle->ambiguous_180hz |= !!(seen & (1U << field)); + seen |= 1U << field; + } if (IS_ERR(key)) ret = PTR_ERR(key); else if (is_virtual) ret = skip(it.handle); - else if (!strcmp(key, "HorizontalAttributes")) - ret = parse_dimension(it.handle, &horiz, false); - else if (!strcmp(key, "VerticalAttributes")) - ret = parse_dimension(it.handle, &vert, true); - else if (!strcmp(key, "MinimumVariableRefreshRate")) + else if (!strcmp(key, "HorizontalAttributes")) { + duplicate_scope |= have_horiz; + have_horiz = true; + ret = parse_dimension(it.handle, &horiz, external_sdr); + } else if (!strcmp(key, "VerticalAttributes")) { + duplicate_scope |= have_vert; + have_vert = true; + ret = parse_dimension(it.handle, &vert, external_sdr); + } else if (!strcmp(key, "MinimumVariableRefreshRate")) ret = parse_int_bound(it.handle, &min_vrr, 0, U32_MAX); else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) - ret = parse_color_modes(it.handle, out, external_sdr, &limits); + ret = parse_color_modes(it.handle, out, external_sdr, &limits, + handle->automatic_1080p180_rgb8 ? + &unsafe_rgb8 : NULL); else if (!strcmp(key, "ID")) ret = parse_int(it.handle, &id); else if (!strcmp(key, "IsVirtual")) + ret = parse_int_bound(it.handle, &id, 0, U32_MAX); + else if (!strcmp(key, "IsVirtual")) { + duplicate_scope |= have_virtual; + have_virtual = true; ret = parse_bool(it.handle, &is_virtual); - else if (external_sdr && !strcmp(key, "IsPreferred")) { + } else if (external_sdr && !strcmp(key, "IsPreferred")) { if (have_preferred) ret = -EINVAL; else @@ -780,6 +842,27 @@ static int parse_mode(struct dcp_parse_ctx *handle, return ret; } } + /* + * Never replace a normal candidate or create a timing/color pairing. + * Only the bounded automatic policy may publish or use this candidate. + * Its EDID identity and successful link callbacks must remain current. + * The observed 2080x1106 total and 11796583/65536 Hz are deliberate: + * do not widen this to arbitrary nominal-180 Hz blanking or rates. + */ + if (external_sdr && !internal && + handle->automatic_1080p180_rgb8 && + !handle->ambiguous_180hz && + out->sdr_rgb.score < 0 && unsafe_rgb8.score >= 0 && + !duplicate_scope && have_horiz && have_vert && have_virtual && !is_virtual && + have_interlaced && !interlaced && !notch_height && + horiz.active == 1920 && vert.active == 1080 && + horiz.total == 2080 && vert.total == 1106 && + vert.precise_sync_rate == 11796583 && + horiz.pixel_repetition == 0 && vert.pixel_repetition == 0 && + horiz.sync_polarity >= 0 && vert.sync_polarity >= 0) { + out->sdr_rgb = unsafe_rgb8; + out->automatic_rgb8_quirk = handle->automatic_1080p180_rgb8; + } if (out->sdr_rgb.score >= 0) best_color_mode = out->sdr_rgb.id; else if (out->sdr_444.score >= 0) @@ -824,8 +907,12 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->vrr = true; } - /* Refresh rates are reported by DCP as 16.16 fixed-point Hz. */ - if (min_vrr && max_vrr > min_vrr) { + /* + * Refresh rates are reported by DCP as 16.16 fixed-point Hz. + * External H17P skips adaptive-sync programming: keep the fixed timing + * available, but do not advertise its sink VRR range as driver support. + */ + if (!external_sdr && min_vrr && max_vrr > min_vrr) { out->min_vrr = min_vrr; out->max_vrr = max_vrr; out->vrr = true; @@ -861,7 +948,8 @@ static int parse_mode(struct dcp_parse_ctx *handle, if (external_sdr && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0) mode->flags = (horiz.sync_polarity ? DRM_MODE_FLAG_PHSYNC : DRM_MODE_FLAG_NHSYNC) | (vert.sync_polarity ? DRM_MODE_FLAG_PVSYNC : DRM_MODE_FLAG_NVSYNC); - out->metadata_timing_valid = external_sdr && have_preferred && preferred && + out->metadata_timing_valid = external_sdr && !out->automatic_rgb8_quirk && + have_preferred && preferred && have_interlaced && !interlaced && horiz.pixel_repetition == 0 && vert.pixel_repetition == 0 && horiz.sync_polarity >= 0 && vert.sync_polarity >= 0; drm_mode_set_name(mode); @@ -876,11 +964,110 @@ static int parse_mode(struct dcp_parse_ctx *handle, return 0; } +/* + * Inspect command identities before filtering can discard conflicting siblings. + * Only automatic exceptions use this conservative receipt-wide check. + */ +static bool dcp_180hz_color_ids_current(struct dcp_parse_ctx *handle, u32 id) +{ + static const char * const fields[] = { + "ID", "Depth", "DynamicRange", "EOTF", "PixelEncoding", "Colorimetry", + }; + struct iterator colors; + + if (iterator_begin(handle, &colors, false)) + return false; + for (; colors.idx < colors.len; colors.idx++) { + struct color_mode color = { .id = -1, .depth = -1, .dynamic_range = -1, + .eotf = -1, .pixel_encoding = -1, .colorimetry = -1 }; + s64 *values[] = { &color.id, &color.depth, &color.dynamic_range, + &color.eotf, &color.pixel_encoding, &color.colorimetry }; + bool is_virtual = true, virtual_seen = false; + unsigned int seen = 0; + struct iterator entry; + + if (iterator_begin(handle, &entry, true)) + return false; + for (; entry.idx < entry.len; entry.idx++) { + char *key = parse_string(handle); + unsigned int field; + int ret; + + if (IS_ERR(key)) + return false; + for (field = 0; field < ARRAY_SIZE(fields); field++) + if (!strcmp(key, fields[field])) + break; + if (field < ARRAY_SIZE(fields)) { + ret = seen & (1U << field) ? -EINVAL : + parse_int_bound(handle, values[field], 0, U32_MAX); + seen |= 1U << field; + } else if (!strcmp(key, "IsVirtual")) { + ret = virtual_seen ? -EINVAL : parse_bool(handle, &is_virtual); + virtual_seen = true; + } else { + ret = skip(handle); + } + kfree(key); + if (ret) + return false; + } + if (color.id == id && + (is_virtual || !virtual_seen || color.depth != 8 || + color.dynamic_range != DCP_COLOR_YCBCR_RANGE_FULL || + color.eotf != DCP_EOTF_SDR_GAMMA || + color.pixel_encoding != DCP_COLOR_FORMAT_RGB || + color.colorimetry != DCP_COLORIMETRY_SRGB)) + return false; + } + return true; +} + +static bool dcp_180hz_ids_current(struct dcp_parse_ctx handle, + const struct dcp_display_mode *mode) +{ + struct iterator timings; + unsigned int matches = 0; + + if (iterator_begin(&handle, &timings, false)) + return false; + for (; timings.idx < timings.len; timings.idx++) { + struct iterator entry; + + if (iterator_begin(&handle, &entry, true)) + return false; + for (; entry.idx < entry.len; entry.idx++) { + char *key = parse_string(&handle); + int ret = 0; + + if (IS_ERR(key)) + return false; + if (!strcmp(key, "ID")) { + s64 id; + + ret = parse_int_bound(&handle, &id, 0, U32_MAX); + if (!ret && id == mode->timing_mode_id) + matches++; + } else if (!strcmp(key, "ColorModes")) { + if (!dcp_180hz_color_ids_current(&handle, mode->color_mode_id)) + ret = -EINVAL; + } else { + ret = skip(&handle); + } + kfree(key); + if (ret || matches > 1) + return false; + } + } + return matches == 1; +} + struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, unsigned int *count, int width_mm, int height_mm, unsigned notch_height, bool internal, bool external_sdr) { + struct dcp_parse_ctx catalog = *handle; struct iterator it; int ret; struct dcp_display_mode *mode, *modes; @@ -933,7 +1120,7 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, /* Process a successful mode */ (*count)++; - if (score > best_score) { + if (!mode->automatic_rgb8_quirk && score > best_score) { best_score = score; best_mode = mode; } @@ -942,6 +1129,17 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, if (best_mode != NULL) best_mode->mode.type |= DRM_MODE_TYPE_PREFERRED; + if (handle->automatic_1080p180_rgb8) { + unsigned int i, kept = 0; + + for (i = 0; i < *count; i++) { + if (modes[i].automatic_rgb8_quirk && + !dcp_180hz_ids_current(catalog, &modes[i])) + continue; + modes[kept++] = modes[i]; + } + *count = kept; + } return modes; } @@ -1192,7 +1390,7 @@ static int parse_metadata_product(struct dcp_parse_ctx *handle, struct dcp_displ for (unsigned int i = 0; i < 3; i++) if (out->manufacturer[i] < 'A' || out->manufacturer[i] > 'Z') return -EINVAL; - if ((seen & 32) && legacy != ((out->manufacturer[0] - '@') << 10 | + if ((seen & 32) && legacy != (u32)((out->manufacturer[0] - '@') << 10 | (out->manufacturer[1] - '@') << 5 | (out->manufacturer[2] - '@'))) return -EINVAL; diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 93fb80b05bf845..ef40247e7993fe 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -8,6 +8,7 @@ #include #include "dcp-display-metadata.h" +#include "dcp-metadata-epoch.h" struct apple_dcp; struct apple_connector; @@ -17,6 +18,9 @@ struct dcp_parse_ctx { struct apple_dcp *dcp; const void *blob; u32 pos, len; + /* External H17P candidates require separate publication/lookup proof. */ + bool automatic_1080p180_rgb8; + bool ambiguous_180hz; }; enum dcp_color_eotf { @@ -99,6 +103,13 @@ struct dcp_display_mode { struct dcp_color_mode best; bool vrr; bool metadata_timing_valid; + bool automatic_rgb8_quirk; + u64 automatic_connection_sequence; + u64 automatic_catalog_sequence; + bool automatic_service_pending; + /* Immutable receipt provenance; never retag a cached exception. */ + struct dcp_metadata_ticket unsafe_rgb8_ticket; + u64 unsafe_rgb8_service_epoch; }; struct dimension { diff --git a/drivers/gpu/drm/apple/trace.h b/drivers/gpu/drm/apple/trace.h index 03f23ecf0a78ed..cbb3f4080fd573 100644 --- a/drivers/gpu/drm/apple/trace.h +++ b/drivers/gpu/drm/apple/trace.h @@ -76,34 +76,6 @@ { DPTX_APCALL_DEVICE_BUSY_TIMEOUT, "device_busy_timeout" }, \ { DPTX_APCALL_DEVICE_NOT_STARTED, "device_not_started" }) -TRACE_EVENT(iomfb_metadata, - TP_PROTO(struct apple_dcp *dcp, const char *phase, u64 generation, - u64 ticket_generation, u32 flags, u32 received, u32 length), - TP_ARGS(dcp, phase, generation, ticket_generation, flags, received, length), - TP_STRUCT__entry( - __string(devname, dev_name(dcp->dev)) - __string(phase, phase) - __field(u64, generation) - __field(u64, ticket_generation) - __field(u32, flags) - __field(u32, received) - __field(u32, length) - ), - TP_fast_assign( - __assign_str(devname); - __assign_str(phase); - __entry->generation = generation; - __entry->ticket_generation = ticket_generation; - __entry->flags = flags; - __entry->received = received; - __entry->length = length; - ), - TP_printk("%s: %s generation=%llu ticket=%llu flags=%#x received=%u length=%u", - __get_str(devname), __get_str(phase), __entry->generation, - __entry->ticket_generation, __entry->flags, - __entry->received, __entry->length) -); - TRACE_EVENT(dcp_recv_msg, TP_PROTO(struct apple_dcp *dcp, u8 endpoint, u64 message), TP_ARGS(dcp, endpoint, message), @@ -233,126 +205,6 @@ TRACE_EVENT( print_epic_category(__entry->category), __entry->subtype, __entry->tag)); -/* Raw prefix before any protocol-specific decode; no peer address is followed. */ -TRACE_EVENT(afk_recv_header, - TP_PROTO(struct apple_dcp_afkep *ep, u32 channel, u32 type, - size_t data_size, const void *data), - TP_ARGS(ep, channel, type, data_size, data), - TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) - __field(u8, endpoint) - __field(u32, channel) - __field(u32, type) - __field(size_t, data_size) - __field(u8, captured) - __array(u8, header, 32)), - TP_fast_assign(__assign_str(devname); - __entry->endpoint = ep->endpoint; - __entry->channel = channel; - __entry->type = type; - __entry->data_size = data_size; - __entry->captured = data ? min_t(size_t, data_size, 32) : 0; - memset(__entry->header, 0, sizeof(__entry->header)); - if (__entry->captured) - memcpy(__entry->header, data, __entry->captured); - ), - TP_printk("%s: endpoint 0x%x channel 0x%x type 0x%x size %zu header[%u]=%s", - __get_str(devname), __entry->endpoint, __entry->channel, - __entry->type, __entry->data_size, __entry->captured, - __print_hex(__entry->header, __entry->captured)) -); - -TRACE_EVENT(afk_ring_snapshot, - TP_PROTO(struct apple_dcp_afkep *ep, bool rx_ready, u32 rx_rptr, - u32 rx_wptr, u32 rx_size, bool tx_ready, u32 tx_rptr, - u32 tx_wptr, u32 tx_size), - TP_ARGS(ep, rx_ready, rx_rptr, rx_wptr, rx_size, - tx_ready, tx_rptr, tx_wptr, tx_size), - TP_STRUCT__entry(__string(devname, dev_name(ep->dcp->dev)) - __field(u8, endpoint) - __field(bool, rx_ready) - __field(u32, rx_rptr) - __field(u32, rx_wptr) - __field(u32, rx_size) - __field(bool, tx_ready) - __field(u32, tx_rptr) - __field(u32, tx_wptr) - __field(u32, tx_size)), - TP_fast_assign(__assign_str(devname); - __entry->endpoint = ep->endpoint; - __entry->rx_ready = rx_ready; - __entry->rx_rptr = rx_rptr; - __entry->rx_wptr = rx_wptr; - __entry->rx_size = rx_size; - __entry->tx_ready = tx_ready; - __entry->tx_rptr = tx_rptr; - __entry->tx_wptr = tx_wptr; - __entry->tx_size = tx_size; - ), - TP_printk("%s: endpoint 0x%x rx_ready %u rx_rptr %#x rx_wptr %#x rx_size %#x tx_ready %u tx_rptr %#x tx_wptr %#x tx_size %#x", - __get_str(devname), __entry->endpoint, __entry->rx_ready, - __entry->rx_rptr, __entry->rx_wptr, __entry->rx_size, - __entry->tx_ready, __entry->tx_rptr, __entry->tx_wptr, - __entry->tx_size) -); - -TRACE_EVENT(iomfb_ctm, - TP_PROTO(struct apple_dcp *dcp, u32 location, const void *matrix, u8 matrix_null), - TP_ARGS(dcp, location, matrix, matrix_null), - TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) - __field(u32, location) - __field(u8, matrix_null) - __array(u64, matrix, 9)), - TP_fast_assign(__assign_str(devname); - __entry->location = location; - __entry->matrix_null = matrix_null; - if (matrix) - memcpy(__entry->matrix, matrix, sizeof(__entry->matrix)); - else - memset(__entry->matrix, 0, sizeof(__entry->matrix)); - ), - TP_printk("%s: location %u null %u matrix=%s", - __get_str(devname), __entry->location, __entry->matrix_null, - __print_array(__entry->matrix, 9, sizeof(u64))) -); - -TRACE_EVENT(iomfb_control_result, - TP_PROTO(struct apple_dcp *dcp, u32 method, bool valid, u32 result), - TP_ARGS(dcp, method, valid, result), - TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) - __field(u32, method) - __field(bool, valid) - __field(u32, result)), - TP_fast_assign(__assign_str(devname); - __entry->method = method; - __entry->valid = valid; - __entry->result = result; - ), - TP_printk("%s: A%u reply_valid %u result 0x%08x", - __get_str(devname), __entry->method, __entry->valid, __entry->result) -); - -TRACE_EVENT(iomfb_surface, - TP_PROTO(struct apple_dcp *dcp, u32 index, u32 format, u8 xfer_func, - u8 colorspace, u8 premultiplied), - TP_ARGS(dcp, index, format, xfer_func, colorspace, premultiplied), - TP_STRUCT__entry(__string(devname, dev_name(dcp->dev)) - __field(u32, index) - __field(u32, format) - __field(u8, xfer_func) - __field(u8, colorspace) - __field(u8, premultiplied)), - TP_fast_assign(__assign_str(devname); - __entry->index = index; - __entry->format = format; - __entry->xfer_func = xfer_func; - __entry->colorspace = colorspace; - __entry->premultiplied = premultiplied; - ), - TP_printk("%s: surface %u format 0x%08x xfer %u colorspace %u source_premul %u", - __get_str(devname), __entry->index, __entry->format, - __entry->xfer_func, __entry->colorspace, __entry->premultiplied) -); - TRACE_EVENT(iomfb_callback, TP_PROTO(struct apple_dcp *dcp, int tag, const char *name), TP_ARGS(dcp, tag, name), diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 71ba9c79394705..76c28c0dc9f7c9 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -309,12 +309,39 @@ static const char * isp_fw2str(enum isp_firmware_version version) #define ISP_FW_VERSION_MAX_LEN 5 static enum isp_firmware_version isp_read_fw_version(struct device *dev, - const char *name) + const char *name, + enum isp_fw_abi abi) { u32 ver[ISP_FW_VERSION_MAX_LEN]; - int len = of_property_read_variable_u32_array(dev->of_node, name, ver, - ISP_FW_VERSION_MIN_LEN, - ISP_FW_VERSION_MAX_LEN); + int size, len; + + /* The array reader truncates trailing bytes; require complete cells. */ + if (of_find_property(dev->of_node, name, &size) && size % sizeof(u32)) { + dev_warn(dev, "could not parse %s: invalid length %d\n", name, size); + return ISP_FIRMWARE_V_UNKNOWN; + } + + len = of_property_read_variable_u32_array(dev->of_node, name, ver, + ISP_FW_VERSION_MIN_LEN, + ISP_FW_VERSION_MAX_LEN); + + /* H17 command layouts come from match data, not the legacy version table. */ + if (abi == ISP_FW_ABI_H17) { + switch (len) { + case 3: + dev_info(dev, "%s: %u.%u.%u (H17 interface)\n", + name, ver[0], ver[1], ver[2]); + return ISP_FIRMWARE_V_UNKNOWN; + case 4: + dev_info(dev, "%s: %u.%u.%u.%u (H17 interface)\n", + name, ver[0], ver[1], ver[2], ver[3]); + return ISP_FIRMWARE_V_UNKNOWN; + case 5: + dev_info(dev, "%s: %u.%u.%u.%u.%u (H17 interface)\n", + name, ver[0], ver[1], ver[2], ver[3], ver[4]); + return ISP_FIRMWARE_V_UNKNOWN; + } + } switch (len) { case -EINVAL: @@ -355,8 +382,8 @@ isp_check_firmware_version(struct apple_isp *isp) enum isp_firmware_version version, compat; /* firmware version is just informative */ - version = isp_read_fw_version(dev, "apple,firmware-version"); - compat = isp_read_fw_version(dev, "apple,firmware-compat"); + version = isp_read_fw_version(dev, "apple,firmware-version", isp->hw->fw_abi); + compat = isp_read_fw_version(dev, "apple,firmware-compat", isp->hw->fw_abi); dev_dbg(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), isp_fw2str(version)); diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index de01c11160ca35..0e4ae9056e35b2 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -3,6 +3,7 @@ config PHY_APPLE_ATC tristate "Apple Type-C PHY" depends on (ARM64 && ARCH_APPLE) || (COMPILE_TEST && !GENERIC_ATOMIC64) depends on TYPEC + depends on USB || !USB select GENERIC_PHY select APPLE_TUNABLE help diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 6643b97d0f001e..5e1ccb7cadc5b8 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -3,6 +3,159 @@ #ifndef APPLE_ATC_J700_DP_H #define APPLE_ATC_J700_DP_H +/* Masks are the union of writes in the public J700 DP start/voltage/stop paths. */ +struct j700_usb_field { + u32 offset, mask; +}; + +static const struct j700_usb_field j700_usb_fields[] = { + { 0x0008, 0x00015540 }, + { 0x000c, 0x00000055 }, + { 0x010c, 0x00000555 }, + { 0x0200, 0x0000000f }, + { 0x9160, 0x00000001 }, + { 0x91d4, 0x00000003 }, + { 0x91dc, 0x00018000 }, + { 0x9210, 0x0000c00f }, + { 0x9244, 0x0007ffff }, + { 0x9248, 0x10000000 }, + { 0xb030, 0x00000c00 }, + { 0xb064, 0x0000fc0c }, + { 0xb068, 0x000001e0 }, + { 0xb06c, 0x0001ee00 }, + { 0xb0a4, 0x00000003 }, + { 0xb0a8, 0x00ff8000 }, + { 0xb0d4, 0x001fff80 }, + { 0xb0d8, 0x00000007 }, + { 0xb0dc, 0x00000018 }, + { 0xc050, 0x0000c00f }, + { 0xc0d0, 0x0017ffff }, + { 0xc0d4, 0x10000000 }, + { 0xd004, 0x33000000 }, + { 0xd00c, 0x07000000 }, + { 0xd010, 0x00000600 }, + { 0xd01c, 0x00c00000 }, + { 0xd044, 0xffffffff }, +}; + +static_assert(ARRAY_SIZE(j700_usb_fields) == 27, "USB context storage size"); + +static void j700_usb_field(unsigned int index, unsigned int lane, u32 *offset, u32 *mask) +{ + *offset = j700_usb_fields[index].offset; + *mask = j700_usb_fields[index].mask; + if (index < 4) + *mask <<= index == 3 ? 4 * lane : lane; + else + *offset += 0x7000 * lane; +} + +/* Capture before the first DP tunable or idle write; never replace a retained context. */ +static int j700_usb_capture(struct apple_atcphy *p, enum atcphy_mode mode) +{ + const struct atcphy_mode_configuration *cfg = atcphy_get_mode_config(p, mode); + + if (!p->hw->has_t8140_dp || !p->fixed_usb2) + return 0; + if (p->usb_off_pending || + p->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY || p->pipe_pending) + return -EBUSY; + for (unsigned int lane = 0; lane < 2; lane++) { + if (!cfg->dp_lane[lane] || (p->dp_usb_context_valid & BIT(lane))) + continue; + for (unsigned int i = 0; i < ARRAY_SIZE(j700_usb_fields); i++) { + u32 offset, mask; + + j700_usb_field(i, lane, &offset, &mask); + p->dp_usb_context[lane][i] = core_read32(p, offset) & mask; + } + p->dp_usb_context_valid |= BIT(lane); + } + return 0; +} + +/* Same control write/delay/poll sequence, retaining its final sample for diagnostics. */ +static int j700_usb_field_update(struct apple_atcphy *p, u32 offset, u32 mask, + u32 value, u32 *observed) +{ + core_mask32(p, offset, mask, value); + usleep_range(10, 20); + return readl_poll_timeout_atomic(p->regs.core + offset, *observed, + (*observed & mask) == value, 1, 100); +} + +/* Restore only DP-owned bits, with the PIPE parked and the DP PMA idle. */ +static int j700_usb_restore(struct apple_atcphy *p, enum atcphy_mode mode) +{ + const struct atcphy_mode_configuration *cfg = atcphy_get_mode_config(p, mode); + const unsigned int power_order[] = { 2, 1, 0, 3 }; + const unsigned int receiver_order[] = { 4, 0, 6, 2 }; + int ret; + + if (!p->hw->has_t8140_dp || !p->fixed_usb2) + return 0; + if (p->usb_off_pending && mode != APPLE_ATCPHY_MODE_OFF) + return -EBUSY; + if (p->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY || p->pipe_pending) + return -EBUSY; + if (p->dp_owned_pmas || p->dp_idle_pmas) + return -EBUSY; + if (mode == APPLE_ATCPHY_MODE_OFF && p->usb_off_drained) + return 0; + for (unsigned int lane = 0; lane < 2; lane++) { + if ((mode != APPLE_ATCPHY_MODE_OFF && + cfg->lane_mode[lane] != ACIOPHY_LANE_MODE_USB3) || + !(p->dp_usb_context_valid & BIT(lane))) + continue; + if (p->dp_usb_context_dirty & BIT(lane)) { + for (unsigned int i = 0; i < ARRAY_SIZE(j700_usb_fields); i++) { + u32 offset, mask, observed, value = p->dp_usb_context[lane][i]; + + if (i == 1 || i == 3) + continue; + j700_usb_field(i, lane, &offset, &mask); + core_mask32(p, offset, mask, value); + observed = core_read32(p, offset) & mask; + if (observed != value) { + dev_dbg(p->dev, "USB restore stage=static pma=%u reg=%#x mask=%#x expected=%#x observed=%#x\n", + lane, offset, mask, value, observed); + return -EIO; + } + } + } + /* Preserve the public PMA power bit order, using saved values. */ + for (unsigned int i = 0; i < ARRAY_SIZE(power_order); i++) { + u32 mask = BIT(4 * lane + power_order[i]); + u32 value = p->dp_usb_context[lane][3] & mask, observed; + + ret = j700_usb_field_update(p, 0x200, mask, value, &observed); + if (ret) { + dev_dbg(p->dev, "USB restore stage=power pma=%u reg=0x200 mask=%#x expected=%#x observed=%#x ret=%d\n", + lane, mask, value, observed & mask, ret); + return ret; + } + } + /* Restore receiver enables, then release its two idle controls. */ + for (unsigned int i = 0; i < ARRAY_SIZE(receiver_order); i++) { + u32 mask = BIT(lane + receiver_order[i]); + u32 value = p->dp_usb_context[lane][1] & mask, observed; + + ret = j700_usb_field_update(p, 0xc, mask, value, &observed); + if (ret) { + dev_dbg(p->dev, "USB restore stage=receiver pma=%u reg=0xc mask=%#x expected=%#x observed=%#x ret=%d\n", + lane, mask, value, observed & mask, ret); + return ret; + } + } + /* OFF keeps the shadow until domain shutdown succeeds. */ + if (mode != APPLE_ATCPHY_MODE_OFF) { + p->dp_usb_context_dirty &= ~BIT(lane); + p->dp_usb_context_valid &= ~BIT(lane); + } + } + return 0; +} + /* Retry only the idle operations claimed by an incomplete route change. */ static int j700_dp_idle_pending(struct apple_atcphy *p) { @@ -269,6 +422,11 @@ static int j700_dp_start(struct apple_atcphy *p, unsigned int pmas, int rate, bo if (!pmas || (pmas & ~3U) || rate < 0 || (unsigned int)rate >= ARRAY_SIZE(parameters)) return -EINVAL; + if (p->hw->has_t8140_dp && p->fixed_usb2) { + if ((p->dp_usb_context_valid & pmas) != pmas) + return -EIO; + p->dp_usb_context_dirty |= pmas; + } /* Claim before the first write: failed setup must still be cleaned up. */ p->dp_owned_pmas = pmas; p->dp_link_rate = -1; @@ -322,7 +480,7 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); /* The last completed mode cannot describe a partially changed lane route. */ - if (p->mode_dirty) + if (p->mode_dirty || p->pipe_pending || p->usb_off_pending) return -EBUSY; if (p->mode != APPLE_ATCPHY_MODE_DP && p->mode != APPLE_ATCPHY_MODE_USB3_DP) return -EINVAL; diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index ff170336a91675..c09bf398e83943 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -28,14 +28,18 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include #include +#include +#include #include #include #include @@ -678,9 +682,15 @@ struct atcphy_hw { * behind the T6030 HDMI port; it maps only its core window * @dp_tunables: The complete T8140 DisplayPort bundle was supplied * @aux_power_requested: AUX power-up began and needs matching shutdown - * @mode_dirty: J700 mode transition needs cleanup before it can be retried + * @mode_dirty: Mode transition needs cleanup before it can be retried * @dp_idle_pmas: J700 PMAs with an incomplete routing idle sequence * @dp_owned_pmas: J700 PMAs requiring main-link cleanup + * @dp_usb_context_valid: Physical PMAs retaining their pre-DP USB field values + * @dp_usb_context_dirty: PMAs whose static fields were programmed by DP + * @usb_off_pending: Failed full OFF must finish before a new powered epoch + * @usb_off_drained: Masked recovery data was discharged before OFF began + * @pipe_pending: PIPE handoff still requires acknowledged dummy parking + * @dp_usb_context: Saved bits from the public per-PMA DP write masks * @dp_spread_spectrum: J700 main-link spread-spectrum state * @dp_presets: Firmware-derived per-rate drive presets * @dp_presets_valid: Firmware supplied each drive-preset table @@ -758,6 +768,9 @@ struct apple_atcphy { bool mode_dirty; u8 dp_idle_pmas; u8 dp_owned_pmas; + u8 dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool usb_off_pending, usb_off_drained, pipe_pending; bool dp_spread_spectrum; u32 dp_presets[5][16]; bool dp_presets_valid[5]; @@ -1060,6 +1073,13 @@ static const struct atcphy_mode_configuration *atcphy_get_mode_config(struct app return &atcphy_modes[mode].normal; } +/* Readback only; callers hold the PHY mutex and keep the fixed host live. */ +static inline u32 core_read32(struct apple_atcphy *atcphy, u32 reg) +{ + return readl(atcphy->regs.core + reg); +} + +/* Static configuration fields already read/modified by the public DP path. */ static void j700_dp_tunables(struct apple_atcphy *atcphy) { apple_tunable_apply(atcphy->regs.axi2af, atcphy->tunables.axi2af); @@ -1133,13 +1153,15 @@ static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) int ret; u32 reg; - if (readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ) & PIPEHANDLER_LOCK_EN) { - dev_warn(atcphy->dev, "Pipehandler already locked\n"); - return 0; + dev_dbg(atcphy->dev, "PIPE request: req=%#x mode=%u pipe=%u host=%u dirty=%u\n", + requested, atcphy->mode, atcphy->pipe_state, + atcphy->host_active, atcphy->mode_dirty); + if (requested & PIPEHANDLER_LOCK_EN) { + dev_dbg(atcphy->dev, "Pipehandler lock already requested\n"); + } else { + set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); } - set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); - ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, reg & PIPEHANDLER_LOCK_EN, 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); if (ret) { @@ -1158,8 +1180,11 @@ static int atcphy_pipehandler_unlock(struct apple_atcphy *atcphy) clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, !(reg & PIPEHANDLER_LOCK_EN), 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); - if (ret) + if (ret) { + dev_dbg(atcphy->dev, "PIPE unlock timeout: ack=%#x mode=%u pipe=%u dirty=%u\n", + reg, atcphy->mode, atcphy->pipe_state, atcphy->mode_dirty); dev_warn(atcphy->dev, "Pipehandler lock release not acked.\n"); + } return ret; } @@ -1215,8 +1240,9 @@ static void atcphy_usb3_bist_step_dbg(struct apple_atcphy *atcphy, int step) } static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atcphy) +static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atcphy) { - int ret; + int ret, error = 0; u32 reg; core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_CLOCK_EN); @@ -1229,13 +1255,22 @@ static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST USB_EN phase=1 value=%#x mode=%u pipe=%u\n", + reg, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN\n"); + } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2, reg, !(reg & ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST STATUS2 phase=1 value=%#x swap=%u mode=%u pipe=%u\n", + reg, atcphy->swap_lanes, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2\n"); atcphy_usb3_bist_step_dbg(atcphy, 1); @@ -1251,13 +1286,22 @@ static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST USB_EN phase=2 value=%#x mode=%u pipe=%u\n", + reg, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN 2nd toggle\n"); + } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2, reg, !(reg & ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST STATUS2 phase=2 value=%#x swap=%u mode=%u pipe=%u\n", + reg, atcphy->swap_lanes, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2 2nd time\n"); atcphy_usb3_bist_step_dbg(atcphy, 2); @@ -1272,17 +1316,23 @@ static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; + dev_dbg(atcphy->dev, "BIST USB_EN phase=3 value=%#x mode=%u pipe=%u\n", + reg, atcphy->mode, atcphy->pipe_state); dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN 3rd toggle\n"); + } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); atcphy_usb3_bist_step_dbg(atcphy, 3); core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_CLOCK_EN); + return error; } -static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *atcphy) +static int atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *atcphy) { - int ret; + int ret, error = 0; u32 reg; core_set32(atcphy, ACIOPHY_TOP_BIST_PHY_CFG0, @@ -1290,9 +1340,12 @@ static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *at core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + } core_set32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); @@ -1313,15 +1366,21 @@ static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *at ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, (reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK0), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; dev_warn(atcphy->dev, "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK0\n"); + } ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); - if (ret) + if (ret) { + if (!error) + error = ret; dev_warn(atcphy->dev, "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); + } atcphy_clear_nonselected_phy_reset(atcphy); @@ -1331,6 +1390,7 @@ static void atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *at ACIOPHY_TOP_BIST_CIOPHY_CFG1_CLK_EN); core_set32(atcphy, ACIOPHY_TOP_BIST_CIOPHY_CFG1, ACIOPHY_TOP_BIST_CIOPHY_CFG1_BIST_EN); + return error; } static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool host) @@ -1362,8 +1422,21 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h /* BIST dance */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8103) - atcphy_configure_pipehandler_usb3_host_t8103(atcphy); + ret = atcphy_configure_pipehandler_usb3_host_t8103(atcphy); else + ret = atcphy_configure_pipehandler_usb3_host_t8122(atcphy); + if (ret) { + int cleanup; + + clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, + PIPEHANDLER_OVERRIDE_RXVALID | PIPEHANDLER_OVERRIDE_RXDETECT); + cleanup = atcphy_pipehandler_unlock(atcphy); + if (cleanup) + dev_warn(atcphy->dev, "PIPE cleanup after BIST: %d\n", cleanup); + /* Native BIST changed state; a retry must rebuild after parking. */ + atcphy->mode_dirty = true; + return ret; + } atcphy_configure_pipehandler_usb3_host_t8122(atcphy); if (atcphy->hw->dp_t8122) dev_dbg(atcphy->dev, "USB3 host BIST done: status2=%08x ov1=%08x ov2=%08x\n", @@ -1375,6 +1448,8 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h /* Configure PIPE mux to USB3 PHY */ atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_USB3, PIPEHANDLER_MUX_CTRL_CLK_USB3); + /* Track the selected backend even if its completion ACK fails. */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; /* Remove link detection override */ clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); @@ -1383,8 +1458,11 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h /* Pipehandler was only locked when the BIST sequence was applied for host mode */ if (host) { ret = atcphy_pipehandler_unlock(atcphy); - if (ret) + if (ret) { dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + atcphy->mode_dirty = true; + return ret; + } } return 0; @@ -1456,11 +1534,18 @@ static bool atcphy_needs_dummy_phy(struct apple_atcphy *atcphy) static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy, bool lock) { + bool live_usb3 = atcphy->pipe_pending || + (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->host_active && + atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); int ret; ret = atcphy_pipehandler_check(atcphy); - if (ret) + if (ret) { + if (live_usb3) + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; return ret; + } /* Force disable link detection */ clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE_VALUES, @@ -1468,6 +1553,15 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy, bool set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); + ret = atcphy_pipehandler_lock(atcphy); + if (ret) { + /* A live controller must acknowledge before its backend changes. */ + if (live_usb3) { + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; + return ret; + } + dev_warn(atcphy->dev, "Failed to lock pipehandler"); if (lock) { ret = atcphy_pipehandler_lock(atcphy); if (ret) @@ -1478,6 +1572,15 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy, bool atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) { + /* The mux changed, but the handoff has not completed. Retain ownership. */ + if (live_usb3) { + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; + return ret; + } + dev_warn(atcphy->dev, "Failed to unlock pipehandler"); if (lock) { ret = atcphy_pipehandler_unlock(atcphy); if (ret) @@ -1530,6 +1633,10 @@ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN), readl(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE)); atcphy->pipe_state = state; + if (!ret) { + atcphy->pipe_state = state; + atcphy->pipe_pending = false; + } return ret; } @@ -1549,15 +1656,24 @@ static void atcphy_park_pipehandler(struct apple_atcphy *atcphy, bool stopped) lockdep_assert_held(&atcphy->lock); - if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY) - return; + dev_dbg(atcphy->dev, "PIPE park: mode=%u pipe=%u host=%u dirty=%u owned=%u idle=%u\n", + atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty, + atcphy->dp_owned_pmas, atcphy->dp_idle_pmas); + if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !atcphy->pipe_pending) + return 0; + ret = atcphy_configure_pipehandler_dummy(atcphy); + if (ret) { if (!lock) dev_dbg(atcphy->dev, "Parking the PIPE of a stopped controller\n"); ret = atcphy_configure_pipehandler_dummy(atcphy, lock); if (ret) dev_warn(atcphy->dev, "Failed to switch PIPE to dummy: %d\n", ret); + return ret; + } atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->pipe_pending = false; + return 0; } static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) @@ -1568,6 +1684,8 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) PIPEHANDLER_MUX_CTRL_CLK_DUMMY); atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; if (atcphy_needs_dummy_phy(atcphy)) + atcphy->pipe_pending = false; + if (!atcphy->hw->has_usb4) atcphy_enable_dummy_phy(atcphy); } @@ -2068,6 +2186,9 @@ static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_ u32 reg; guard(mutex)(&atcphy->lock); + /* A failed transition may have asserted the PHY resets already. */ + if (atcphy->mode_dirty || atcphy->pipe_pending) + return -EBUSY; /* the AUSPLL is in use as the Thunderbolt DP tunnel pixel clock */ if (atcphy->tunnel_clock_on || atcphy->tunnel_saved) return -EBUSY; @@ -3165,6 +3286,13 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + if (atcphy->pipe_pending) + return -EBUSY; + if (atcphy->usb_off_pending && mode != APPLE_ATCPHY_MODE_OFF) + return -EBUSY; + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && + atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + return -EBUSY; if (atcphy->hw->has_t8140_dp && atcphy->dp_owned_pmas && (mode != atcphy->mode || atcphy->mode_dirty)) { atcphy->mode_dirty = true; @@ -3204,18 +3332,49 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) atcphy_dp_stop_t8122(atcphy); if (mode == APPLE_ATCPHY_MODE_OFF) { + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && + (atcphy->dp_usb_context_valid || atcphy->usb_off_pending)) { + atcphy->usb_off_pending = true; + if (!atcphy->usb_off_drained) { + ret = j700_usb_restore(atcphy, APPLE_ATCPHY_MODE_OFF); + if (ret) + return ret; + atcphy->usb_off_drained = true; + } + } ret = atcphy_power_off(atcphy); if (ret) return ret; + /* Recovery data was discharged before OFF, not discarded while dirty. */ + atcphy->dp_usb_context_valid = 0; + atcphy->dp_usb_context_dirty = 0; + atcphy->usb_off_pending = false; + atcphy->usb_off_drained = false; atcphy->mode = mode; atcphy->mode_dirty = false; return 0; } + /* + * Fixed-hub SAFE keeps USB2 up rather than taking the block OFF. + * With PIPE parked and DP owners released, reinitialize USB3 before + * programming its lanes; the existing sequence releases this reset. + */ + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) + core_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + ret = atcphy_power_on(atcphy); if (ret) return ret; + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) { + ret = j700_usb_capture(atcphy, mode); + if (!ret) + ret = j700_usb_restore(atcphy, mode); + if (ret) + return ret; + } + atcphy_apply_tunables(atcphy, mode); if (atcphy->hw->gen == ATCPHY_GENERATION_T8103) { @@ -3421,10 +3580,23 @@ static int atcphy_usb2_init(struct phy *phy) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); - if (!atcphy->fixed_usb2) - return 0; - guard(mutex)(&atcphy->lock); + if (atcphy->usb_off_pending) + return -EBUSY; + if (atcphy->pipe_pending && + (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty)) { + int ret = atcphy_park_pipehandler(atcphy); + + if (ret) + return ret; + } + + /* A delayed shutdown may have completed after the last Type-C event. */ + if (!atcphy->fixed_usb2) { + if (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) + return atcphy_configure(atcphy, atcphy->typec_mode); + return 0; + } atcphy_restore_after_pd_off(atcphy); /* @@ -3436,7 +3608,7 @@ static int atcphy_usb2_init(struct phy *phy) * the Type-C mux last asked for, USB2 until a cable event says otherwise. */ set32(atcphy->regs.usb2phy + USB2PHY_SIG, USB2PHY_SIG_HOST); - if (atcphy->mode == APPLE_ATCPHY_MODE_OFF) + if (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) return atcphy_configure(atcphy, atcphy->typec_mode); atcphy_usb2_power_on(atcphy); @@ -3453,10 +3625,19 @@ static const struct phy_ops apple_atc_usb2_phy_ops = { static int atcphy_usb3_power_off(struct phy *phy) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); + int ret; guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + if (atcphy->mode != APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) { + ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); + if (ret) + return ret; + } atcphy_park_pipehandler(atcphy, true); atcphy->host_active = false; @@ -3483,6 +3664,9 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); + if (atcphy->usb_off_pending) + return -EBUSY; + switch (mode) { case PHY_MODE_USB_HOST: atcphy->host_active = true; @@ -3500,7 +3684,17 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to * the backend the current mode uses, which includes the dummy backend of the USB2 modes. */ - if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state) + /* A failed lane/domain transition must be restored before routing PIPE. */ + if (atcphy->mode_dirty) + return -EBUSY; + if (atcphy->pipe_pending) { + int ret = atcphy_park_pipehandler(atcphy); + + if (ret) + return ret; + } + if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state && + !atcphy->pipe_pending) return 0; return atcphy_configure_pipehandler(atcphy, mode == PHY_MODE_USB_HOST); @@ -3800,10 +3994,16 @@ static void _atcphy_dwc3_reset_assert(struct apple_atcphy *atcphy) static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) { struct apple_atcphy *atcphy = container_of(rcdev, struct apple_atcphy, rcdev); + int ret; guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); + /* Do not reset the controller or USB2 PHY if PIPE parking fails. */ + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + _atcphy_dwc3_reset_assert(atcphy); atcphy_park_pipehandler(atcphy, true); atcphy->host_active = false; @@ -3852,18 +4052,26 @@ static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) { struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); + bool swap_lanes; guard(mutex)(&atcphy->lock); switch (orientation) { case TYPEC_ORIENTATION_NONE: - break; + return 0; case TYPEC_ORIENTATION_NORMAL: atcphy->swap_lanes = false; break; case TYPEC_ORIENTATION_REVERSE: atcphy->swap_lanes = true; break; + default: + return -EINVAL; + } + /* The mode alone cannot describe a route after its orientation changes. */ + if (atcphy->swap_lanes != swap_lanes) { + atcphy->swap_lanes = swap_lanes; + atcphy->mode_dirty = true; } dev_dbg(atcphy->dev, "orientation %d, lanes swapped %d\n", orientation, atcphy->swap_lanes); @@ -3892,7 +4100,109 @@ static int atcphy_probe_switch(struct apple_atcphy *atcphy) return devm_add_action_or_reset(atcphy->dev, atcphy_typec_switch_unregister, sw); } -static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +/* + * The fixed USB2 hub keeps the host controller active, but its independent + * SuperSpeed root hub may autosuspend while PIPE still needs it awake. + * Own a bounded root-hub resume reference across a Type-C handoff. Resume + * precedes the ATC lock: USB PM can call back into the PHY on recovery. + */ +struct atcphy_usb3_pm { + struct platform_device *controller; + struct usb_device *root; +}; + +#if IS_REACHABLE(CONFIG_USB) +static int atcphy_usb3_pm_find(struct usb_device *udev, void *data) +{ + struct atcphy_usb3_pm *pm = data; + + if (udev->parent || udev->speed < USB_SPEED_SUPER || + udev->bus->sysdev != &pm->controller->dev) + return 0; + + pm->root = usb_get_dev(udev); + return 1; +} +#endif + +static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm *pm) +{ +#if IS_REACHABLE(CONFIG_USB) + struct device_node *ep __free(device_node) = NULL; + struct device_node *ctrl __free(device_node) = NULL; + struct usb_hcd *hcd; + int ret; +#endif + + if (!atcphy->fixed_usb2 || !atcphy->hw->has_t8140_dp) + return 0; + +#if IS_REACHABLE(CONFIG_USB) + ep = of_graph_get_endpoint_by_regs(atcphy->np, 1, -1); + ctrl = ep ? of_graph_get_remote_port_parent(ep) : NULL; + if (!ctrl) + return -ENODEV; + pm->controller = of_find_device_by_node(ctrl); + if (!pm->controller) + return -EAGAIN; + + /* Firmware identity, not a bus number, binds this PHY to its root hub. */ + usb_for_each_dev(pm, atcphy_usb3_pm_find); + if (!pm->root) { + ret = -EAGAIN; + goto put_controller; + } + + /* Exclude root-hub teardown while its controller backs the handoff. */ + usb_lock_device(pm->root); + hcd = bus_to_hcd(pm->root->bus); + if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || HCD_DEAD(hcd)) { + ret = -ENODEV; + goto unlock_root; + } + ret = pm_runtime_resume_and_get(&pm->root->dev); + if (ret < 0) + goto unlock_root; + if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || + HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { + pm_runtime_put_noidle(&pm->root->dev); + ret = -ENODEV; + goto unlock_root; + } + return 0; + +unlock_root: + usb_unlock_device(pm->root); + usb_put_dev(pm->root); + pm->root = NULL; +put_controller: + put_device(&pm->controller->dev); + pm->controller = NULL; + return ret; +#else + /* typec_mux_set() skips EOPNOTSUPP, which would report a false success. */ + return -ENODEV; +#endif +} + +static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) +{ +#if IS_REACHABLE(CONFIG_USB) + if (!pm->root) + return; + + /* Restore normal runtime policy after both success and failed parking. */ + usb_mark_last_busy(pm->root); + pm_runtime_put_autosuspend(&pm->root->dev); + usb_unlock_device(pm->root); + usb_put_dev(pm->root); + put_device(&pm->controller->dev); + pm->root = NULL; + pm->controller = NULL; +#endif +} + +static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_state *state) { struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); enum atcphy_mode target_mode; @@ -3966,7 +4276,21 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta return -EOPNOTSUPP; if (atcphy->mode == target_mode) + dev_dbg(atcphy->dev, "mux target=%u\n", target_mode); + if (atcphy->mode == target_mode && !atcphy->mode_dirty) { + /* Retry a failed PIPE setup without disturbing the configured lanes. */ + if (atcphy->fixed_usb2 && atcphy->host_active && + (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || + atcphy->pipe_pending)) { + if (atcphy->pipe_pending) { + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + } + return atcphy_configure_pipehandler(atcphy, true); + } return 0; + } switch (target_mode) { case APPLE_ATCPHY_MODE_TBT: @@ -4008,19 +4332,28 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta * mode without a USB3 backend, and route it to the new backend * afterwards while dwc3 is up in host mode. */ - if (pipe_state != ATCPHY_PIPEHANDLER_STATE_USB3) - atcphy_park_pipehandler(atcphy, false); + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; ret = atcphy_configure(atcphy, target_mode); if (ret) return ret; + if (atcphy->host_active && atcphy->pipe_state != pipe_state) ret = atcphy_configure_pipehandler(atcphy, true); return ret; } + /* A failed USB3 handoff must finish parking before ordinary-port lane work. */ + if (atcphy->pipe_pending) { + ret = atcphy_park_pipehandler(atcphy); + if (ret) + return ret; + } + /* * If the pipehandler is still/already up here there's a bug somewhere so make sure to * complain loudly. We can still try to switch modes and hope for the best though, @@ -4030,6 +4363,20 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta return atcphy_configure(atcphy, target_mode); } +static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) +{ + struct apple_atcphy *atcphy = typec_mux_get_drvdata(mux); + struct atcphy_usb3_pm pm = {}; + int ret; + + ret = atcphy_usb3_pm_get(atcphy, &pm); + if (ret) + return ret; + ret = atcphy_mux_set_locked(mux, state); + atcphy_usb3_pm_put(&pm); + return ret; +} + static void atcphy_typec_mux_unregister(void *data) { typec_mux_unregister(data); diff --git a/drivers/soc/apple/dt.rs b/drivers/soc/apple/dt.rs index 48a4732008967e..51c83d1249cdf8 100644 --- a/drivers/soc/apple/dt.rs +++ b/drivers/soc/apple/dt.rs @@ -400,7 +400,14 @@ unsafe fn add_u32( } } -pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { +// The public J700 DTS enables SPI2. A disabled controller at handoff is a +// firmware hardware-presence decision; older platforms still need activation. +fn sensor_controller_refused(controller: &DtNode) -> bool { + crate::profile::machine_has(b"apple,j700") && !controller.is_available() +} + +/// Returns false when the bootloader explicitly withheld J700's controller. +pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result { // A machine that describes the sensor in its own device tree (compatible // apple,mesa-fingerprint) is handled machine-agnostically: enable that node // and its SPI controller wherever they sit, with the power/IRQ GPIOs coming @@ -411,6 +418,9 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { pr_err!("apple_sep: sensor node has no parent SPI controller\n"); ENODEV })?; + if sensor_controller_refused(&controller) { + return Ok(false); + } pr_info!( "apple_sep: sensor node present in the device tree; enabling it and its SPI bus\n" ); @@ -425,7 +435,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { queued = true; } Ok(queued) - }); + }).map(|()| true); } let controller = node_at_address(base).ok_or_else(|| { @@ -436,6 +446,9 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { ENODEV })?; + if sensor_controller_refused(&controller) { + return Ok(false); + } let controller_ok = controller.is_available(); let existing = controller.child_with_reg(cs); pr_info!( @@ -447,7 +460,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { ); if controller_ok && existing.is_some() { - return Ok(()); + return Ok(true); } if existing.is_some() && !controller_ok { @@ -456,7 +469,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { return with_changeset(|cs_handle| { queue_status_okay(cs_handle, &controller)?; Ok(true) - }); + }).map(|()| true); } pr_info!( @@ -470,7 +483,7 @@ pub(crate) fn enable_spi_sensor(base: u64, cs: u32) -> Result<()> { } queue_sensor_node(cs_handle, &controller)?; Ok(true) - }) + }).map(|()| true) } /// Resolve a reserved-memory region by name on the SEP node's device. diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index 472c712a862df6..b8e2d684649db2 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -447,7 +447,7 @@ static_assert!(T8122.wide_dma_mask); /// /// The root property is a NUL-separated list, so whole entries are compared /// instead of substrings: `apple,t8103` cannot match a longer unrelated value. -fn machine_has(compatible: &[u8]) -> bool { +pub(crate) fn machine_has(compatible: &[u8]) -> bool { let Some(root) = of::root() else { return false; }; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 430bb4f136c377..4441a94a285777 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -304,14 +304,24 @@ impl SepData { return; } - if let Err(e) = dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { - dev_warn!( - self.dev, - "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", - sensor::CONTROLLER_BASE, - e - ); - return; + match dt::enable_spi_sensor(sensor::CONTROLLER_BASE, sensor::CHIP_SELECT) { + Ok(true) => {}, + Ok(false) => { + dev_info!( + self.dev, + "sensor: SPI controller withheld by the J700 firmware handoff; Touch ID unavailable\n" + ); + return; + } + Err(e) => { + dev_warn!( + self.dev, + "sensor: could not enable the SPI bus at 0x{:x} or create the sensor node ({:?}); no sensor\n", + sensor::CONTROLLER_BASE, + e + ); + return; + } } let bound = sensor::is_bound(); diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5cf939a6721e6b..a1f8b90bbda857 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -565,6 +565,8 @@ struct SepData { #[pin] endpoints: Mutex, + #[pin] + trusted_ready: CondVar, #[pin] control: Mutex, @@ -2005,6 +2007,46 @@ impl SepData { Self::arm_settle(this); } + // Publish completion under the same lock that trusted consumers wait on. + // PHASE_READY ends discovery/initialisation; it is not proof of success. + fn finish_bringup(&self) { + let _guard = self.endpoints.lock(); + self.phase.store(PHASE_READY, Relaxed); + self.trusted_ready.notify_all(); + } + + /// Trusted-key callbacks can run as soon as probe publishes the source, + /// while endpoint discovery and key-store initialisation are still running. + /// Wait only in those consumers, leaving bringup free to establish the key. + pub(crate) fn wait_for_trusted_ready(&self) -> Result<()> { + let mut remaining = time::msecs_to_jiffies(EXCHANGE_TIMEOUT_MS); + let mut guard = self.endpoints.lock(); + loop { + if self.shutting_down.load(Relaxed) { + return Err(ENODEV); + } + if self.phase.load(Relaxed) == PHASE_READY { + // A completed but failed attach must not look ready. + if !guard.eps.iter().any(|e| e.id == proto::EP_SKS) { + return Err(ENODEV); + } + if !self.sks_initialized.load(Relaxed) { + return Err(EIO); + } + return Ok(()); + } + if remaining == 0 { + dev_err!(self.dev, "trusted-keys: key-store bringup timed out\n"); + return Err(ETIMEDOUT); + } + match self.trusted_ready.wait_interruptible_timeout(&mut guard, remaining) { + CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, + CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), + CondVarTimeoutResult::Timeout => remaining = 0, + } + } + } + fn endpoint_present(&self, id: u8) -> bool { self.endpoints.lock().eps.iter().any(|e| e.id == id) } diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 6f3c00b089371c..c2bbcbcf3797a2 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -995,14 +995,20 @@ static void dwc3_clk_disable(struct dwc3 *dwc) clk_disable_unprepare(dwc->bus_clk); } -void dwc3_core_exit(struct dwc3 *dwc) +void dwc3_core_exit_after_phy_power_off(struct dwc3 *dwc) { - dwc3_event_buffers_cleanup(dwc); - dwc3_phy_power_off(dwc); dwc3_phy_exit(dwc); dwc3_clk_disable(dwc); reset_control_assert(dwc->reset); } +EXPORT_SYMBOL_GPL(dwc3_core_exit_after_phy_power_off); + +void dwc3_core_exit(struct dwc3 *dwc) +{ + dwc3_event_buffers_cleanup(dwc); + dwc3_phy_power_off(dwc); + dwc3_core_exit_after_phy_power_off(dwc); +} EXPORT_SYMBOL_GPL(dwc3_core_exit); static bool dwc3_core_is_valid(struct dwc3 *dwc) @@ -2410,14 +2416,16 @@ static int dwc3_probe(struct platform_device *pdev) return dwc3_core_probe(&probe_data); } -void dwc3_core_remove(struct dwc3 *dwc) +static void dwc3_core_remove_common(struct dwc3 *dwc, bool core_exited) { pm_runtime_get_sync(dwc->dev); - dwc3_core_exit_mode(dwc); + if (!core_exited) + dwc3_core_exit_mode(dwc); dwc3_debugfs_exit(dwc); - dwc3_core_exit(dwc); + if (!core_exited) + dwc3_core_exit(dwc); dwc3_ulpi_exit(dwc); pm_runtime_allow(dwc->dev); @@ -2438,8 +2446,19 @@ void dwc3_core_remove(struct dwc3 *dwc) power_supply_put(dwc->usb_psy); } } + +void dwc3_core_remove(struct dwc3 *dwc) +{ + dwc3_core_remove_common(dwc, false); +} EXPORT_SYMBOL_GPL(dwc3_core_remove); +void dwc3_core_remove_no_shutdown(struct dwc3 *dwc) +{ + dwc3_core_remove_common(dwc, true); +} +EXPORT_SYMBOL_GPL(dwc3_core_remove_no_shutdown); + static void dwc3_remove(struct platform_device *pdev) { dwc3_core_remove(platform_get_drvdata(pdev)); diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 232a4de95bb75a..248ccbf90027c1 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -254,6 +254,52 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) return ret; } +/* Keep failed releases owned by the glue, without repeating completed teardown. */ +static int dwc3_apple_core_stop(struct dwc3_apple *appledwc) +{ + struct dwc3 *dwc = &appledwc->dwc; + int i, ret; + + lockdep_assert_held(&appledwc->lock); + if (appledwc->core_active) { + if (!appledwc->core_exit_pending) { + dwc3_enable_susphy(dwc, true); + dwc3_event_buffers_cleanup(dwc); + appledwc->core_exit_pending = true; + } + for (i = 0; i < dwc->num_usb3_ports; i++) { + if (!(appledwc->usb3_power_off_pending & BIT(i))) + continue; + ret = phy_power_off(dwc->usb3_generic_phy[i]); + if (ret) + return ret; + appledwc->usb3_power_off_pending &= ~BIT(i); + } + for (i = 0; i < dwc->num_usb2_ports; i++) { + if (!(appledwc->usb2_power_off_pending & BIT(i))) + continue; + ret = phy_power_off(dwc->usb2_generic_phy[i]); + if (ret) + return ret; + appledwc->usb2_power_off_pending &= ~BIT(i); + } + usb_phy_set_suspend(dwc->usb3_phy, 1); + usb_phy_set_suspend(dwc->usb2_phy, 1); + dwc3_core_exit_after_phy_power_off(dwc); + appledwc->core_active = false; + appledwc->core_exit_pending = false; + } + if (appledwc->reset_pending) { + ret = reset_control_assert(appledwc->reset); + if (ret) + return ret; + appledwc->reset_pending = false; + } + dev_pm_syscore_device(appledwc->dev, false); + + return 0; +} + /* * Bring the core up for the given role: configure the USB2 PHY, release the reset and probe * (on the first call) or initialise the core. @@ -265,6 +311,11 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st lockdep_assert_held(&appledwc->lock); + /* A failed exit must finish before any new PHY reference is acquired. */ + ret = dwc3_apple_core_stop(appledwc); + if (ret) + return ret; + /* * The USB2 PHY on this platform must be configured for device mode before dwc3 tries to * access it. Otherwise, the new configuration will sometimes only take affect after the @@ -297,6 +348,9 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st return ret; } + appledwc->reset_pending = true; + /* Keep register access powered until both PHY shutdown and reset finish. */ + dev_pm_syscore_device(appledwc->dev, true); ret = dwc3_apple_core_init(appledwc); if (ret) { ret_reset = reset_control_assert(appledwc->reset); @@ -596,9 +650,11 @@ static int dwc3_apple_probe(struct platform_device *pdev) scoped_guard(mutex, &appledwc->lock) ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); if (ret) { - /* dwc3_apple_init() has already undone the core initialisation */ + /* A failed cleanup remains visible even when probe cannot be retried. */ + if (appledwc->core_active || appledwc->reset_pending) + dev_err(dev, "Host startup left pending PHY/reset cleanup\n"); if (appledwc->state != DWC3_APPLE_PROBE_PENDING) - dwc3_core_remove(&appledwc->dwc); + dwc3_core_remove_no_shutdown(&appledwc->dwc); usb_role_switch_unregister(appledwc->role_sw); return dev_err_probe(dev, ret, "Failed to start host mode\n"); } @@ -622,7 +678,7 @@ static void dwc3_apple_remove(struct platform_device *pdev) */ dwc3_apple_exit(appledwc); if (appledwc->state != DWC3_APPLE_PROBE_PENDING) - dwc3_core_remove(&appledwc->dwc); + dwc3_core_remove_no_shutdown(&appledwc->dwc); } static const struct of_device_id dwc3_apple_of_match[] = { diff --git a/drivers/usb/dwc3/glue.h b/drivers/usb/dwc3/glue.h index d738e1739ae015..db5f32df712bf8 100644 --- a/drivers/usb/dwc3/glue.h +++ b/drivers/usb/dwc3/glue.h @@ -108,6 +108,26 @@ int dwc3_core_init(struct dwc3 *dwc); */ void dwc3_core_exit(struct dwc3 *dwc); +/** + * dwc3_core_exit_after_phy_power_off - Finish a glue-owned core shutdown + * @dwc: Pointer to DWC3 controller context + * + * The glue must first stop host/gadget operation, clean up event buffers, + * release every core-owned generic PHY power reference and suspend legacy + * USB PHYs. This finishes PHY exit, clock disable and reset exactly once. + */ +void dwc3_core_exit_after_phy_power_off(struct dwc3 *dwc); + +/** + * dwc3_core_remove_no_shutdown - Release resources without a second shutdown + * @dwc: Pointer to DWC3 controller context + * + * The glue owns host/gadget and core shutdown, including reporting any + * failure to finish it. This releases software resources without repeating + * those operations or discarding another generic PHY reference. + */ +void dwc3_core_remove_no_shutdown(struct dwc3 *dwc); + /** * dwc3_host_init - Initialize host mode operation * @dwc: Pointer to DWC3 controller context diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index bb7f471e4ba575..b936bb171fbd7f 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -899,6 +899,23 @@ static void cd321x_update_work(struct work_struct *work) goto retry; } + /* + * Report sink absence before any fallible hardware transition. A failed + * shutdown retains the route/PHY owner, but cannot keep the sink present. + * Every Type-C port references the same external DCP connector. Only the + * port which successfully acquired the shared display route may report a + * disconnect. A second DP sink can negotiate Alt Mode even though its mux + * request is rejected with -EBUSY; treating that sink as the connector + * owner would tear down the display carried by the first port. + */ + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + dp_route_was_active && + (!new_connected || was_disconnected || !dp_connected || !dp_hpd || + dp_hpd_changed || dp_mode_changed)) { + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, + connector_status_disconnected); + } + /* * ACIO carries the tunneled PCIe reset handshake over the still-live * Type-C data path. Close a disappearing USB4/TBT session before dropping @@ -921,20 +938,6 @@ static void cd321x_update_work(struct work_struct *work) goto retry; } - /* - * Every Type-C port references the same external DCP connector. Only the - * port which successfully acquired the shared display route may report a - * disconnect. A second DP sink can negotiate Alt Mode even though its mux - * request is rejected with -EBUSY; treating that sink as the connector - * owner would tear down the display carried by the first port. - */ - if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && - dp_route_was_active && - (!new_connected || was_disconnected || !dp_connected || !dp_hpd || - dp_hpd_changed || dp_mode_changed)) { - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_disconnected); - } - /* Process partner disconnection or change */ if (!new_connected || partner_changed) { if (!IS_ERR(tps->partner)) @@ -1753,7 +1756,17 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) cd321x->state.alt = NULL; cd321x->state.mode = TYPEC_STATE_SAFE; cd321x->state.data = NULL; - cd321x->state_valid = !typec_set_mode(tps->port, TYPEC_STATE_SAFE); + ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); + cd321x->state_valid = !ret; + /* + * A fixed host can register its SuperSpeed root after the mux provider. + * Wait for that consumer at probe instead of exhausting the cable retry + * budget before it exists. Host startup does not depend on this port. + */ + if (ret == -EAGAIN) { + ret = -EPROBE_DEFER; + goto err_put_switch; + } cd321x->tbt_notifier.notifier_call = cd321x_tbt_notify; ret = typec_thunderbolt_switch_register_notifier(cd321x->tbt_switch, @@ -1768,6 +1781,8 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; err_unregister_mux: + fwnode_handle_put(cd321x->connector_fwnode); + cd321x->connector_fwnode = NULL; typec_mux_put(cd321x->mux); cd321x->mux = NULL; err_unregister_altmodes: @@ -1792,6 +1807,8 @@ cd321x_unregister_port(struct tps6598x *tps) struct cd321x *cd321x = container_of(tps, struct cd321x, tps); cd321x_stop_work(cd321x); + fwnode_handle_put(cd321x->connector_fwnode); + cd321x->connector_fwnode = NULL; typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; typec_mux_put(cd321x->mux); diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index ac5717ad1078c9..6efdaf9c73cb1e 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -19,6 +19,7 @@ TARGETS += devices/probe TARGETS += dmabuf-heaps TARGETS += drivers/apple TARGETS += drivers/dma-buf +TARGETS += drivers/gpu/apple-dcp TARGETS += drivers/ntsync TARGETS += drivers/s390x/uvdevice TARGETS += drivers/net diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py index 65d6fdeca7a542..b55eaea38c8d78 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-ring.py @@ -24,7 +24,7 @@ static int check_ring(struct ep *ep,u32 rptr,u32 wptr,size_t payload_len,size_t *total) { struct epic_hdr *ehdr;struct epic_sub_hdr *eshdr;struct afk_qe *hdr; - bool compact=false;size_t total_epic_size,total_size;int ret=0; + size_t total_epic_size,total_size;int ret=0; ''' suffix = r''' *total=total_size; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile new file mode 100644 index 00000000000000..0d6d406ef5b9d9 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +TEST_PROGS := run.sh +TEST_FILES := run.py common.py edid.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README +include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README new file mode 100644 index 00000000000000..a61543d1168f8f --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -0,0 +1,56 @@ +Apple external display host regressions +====================================== + +Run from a kernel source checkout: + + make -C tools/testing/selftests TARGETS=drivers/gpu/apple-dcp run_tests + +Or keep translation units, commands, failure logs and source hashes: + + tools/testing/selftests/drivers/gpu/apple-dcp/run.sh --output /tmp/apple-dcp-results + +An installed copy needs --source /path/to/kernel. --group edid, rate, phy or +lifecycle selects one group. HOSTCC selects the host compiler (default clang). +Python 3 and a compiler supporting AddressSanitizer/UndefinedBehaviorSanitizer +are required. No target access, firmware command, captured fixture or kernel +configuration is required. + +One TAP entry point uses shared production extraction and compiler handling. +The modules contain public synthetic host boundaries and retained assertions: + +* edid: actual AFK admission and OPEN/CLOSE handling; native/HDMI providers + alternating over 64 epochs; foreign owners, stale replies, reader/command + pins, ordered receive progress and legacy service behavior. +* rate: actual binary parser, EDID/capacity policy, receipt ingress, getters + and lookup; all arrival orders, conflicting/discarded siblings, rejected + replacement catalogs, HPD retirement and recovery; malformed/duplicate + metadata and independent EDID decoding; eight bounded delayed retries. +* phy: actual parking/mux/init/reset/configure callbacks, masked PMA context, + faulted ACK/BIST cleanup, dirty-route retry, OFF/new-calibration epochs, + warm reset and legacy T8103/T8122/T8140/T8142 route controls. +* lifecycle: actual physical-disconnect authority and ownership despite failed + teardown; DWC3/core/PHY PM and reset-reference recovery; exact firmware + tuple/resource admission, unsupported VRR rejection and power reply bounds; + 15,552 D134 dispatcher scope/length/context and legacy reply cases. + +Transport, MMIO, kernel locks and work scheduling are host mocks. These tests +establish source behavior; electrical readiness, firmware/FDT ownership, +concurrency and system suspend require separate kernel/hardware qualification. +Kernel KUnit tests and apple-afkv2 remain separate tests in their existing +standard layouts. No archived diagnostic script is a dependency of this suite. + +The runner preserves host artifacts on failure as well as success. Existing +physical-disconnect fixture warnings for unused mock boundaries and signed +host/kernel ABI comparisons are handled only by those fixtures' original +compiler options; production source compilation uses the normal Kbuild policy. + +The bounded-PM group shares the PHY fixture and executes the actual PM get/put +and mux wrapper, usb_hc_died and typec_mux_set. The probe group executes real +CD321x register/unregister/rollback and retry-state functions. USB n/y/m uses +the actual optional dependency and a real Kconfig producer (KCONFIG_CONF may +select an existing producer). Legacy lane/teardown groups keep executing the +same locked mux body; they do not pretend their old USB mocks own runtime PM. +The cpuidle group executes the actual Apple table/probe, generic registration +and s2idle selector/entry, and tick_freeze/unfreeze. It checks source behavior +only; RTC, elapsed suspend time and simultaneous hardware execution remain +separate qualification gates. The original human cpuidle patch is preserved. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/common.py b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py new file mode 100644 index 00000000000000..1a61b02dac07e3 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/common.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Shared public-source extraction and strict host compilation.""" +from pathlib import Path +import hashlib, json, os, re, shlex, subprocess + +def block(source, marker): + start = source.index(marker) if isinstance(marker, str) else marker + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + +def source_defines(source): + lines = iter(source.splitlines()) + definitions = [] + for line in lines: + if not line.startswith('#define '): + continue + definition = [line] + while definition[-1].endswith(chr(92)): + definition.append(next(lines)) + definitions.append('\n'.join(definition)) + return definitions + +def run_unit(name, unit, output, cases=((),), includes=(), extra=()): + work = output / name + work.mkdir(parents=True, exist_ok=True) + source = work / 'host.c' + source.write_text(unit) + command = shlex.split(os.environ.get('HOSTCC', 'clang')) + [ + '-std=gnu11', '-Wall', '-Werror', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + ] + list(extra) + ['-I' + str(p) for p in includes] + [str(source), '-o', str(work / 'host')] + (work / 'command.json').write_text(json.dumps(command, indent=2) + '\n') + result = subprocess.run(command, capture_output=True, text=True, timeout=120) + (work / 'compile.log').write_text(result.stdout + result.stderr) + result.check_returncode() + for index, case in enumerate(cases): + result = subprocess.run([str(work / 'host'), *case], capture_output=True, text=True, timeout=120) + (work / ('case-' + str(index) + '.log')).write_text(result.stdout + result.stderr) + result.check_returncode() + return hashlib.sha256(unit.encode()).hexdigest() diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c new file mode 100644 index 00000000000000..db4f198307362d --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/cases.c @@ -0,0 +1,304 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +static struct atcphy_hw hardware; +static int counted_off(struct phy *p) { callback_calls++; return atcphy_usb3_power_off(p); } +static int other_off(struct phy *p) { (void)p; return other_off_error; } +static const struct phy_ops usb3_ops = { .power_off = counted_off }; +static const struct phy_ops usb2_ops = { .init = atcphy_usb2_init, .power_off = other_off }; +static const struct phy_ops plain_ops = { .power_off = other_off }; +static void dwc3_host_exit(struct dwc3 *d) +{ + host_exits++; + if (root_refs) { + assert(phy_power_off(d->usb3_generic_phy[0]) == 0); + assert(phy_exit(d->usb3_generic_phy[0]) == 0); + root_refs = 0; + } +} +/* Boundary mocks acquire references through the production generic PHY API. */ +static int dwc3_core_init(struct dwc3 *d) +{ + starts++; + if (core_init_error) return core_init_error; + for (int i = 0; i < d->num_usb2_ports; i++) assert(phy_init(d->usb2_generic_phy[i]) == 0); + for (int i = 0; i < d->num_usb3_ports; i++) assert(phy_init(d->usb3_generic_phy[i]) == 0); + return dwc3_phy_power_on(d); +} +static int dwc3_apple_core_probe(struct dwc3_apple *a) +{ + int ret = dwc3_core_init(&a->dwc); + if (!ret) a->state = DWC3_APPLE_NO_CABLE; + return ret; +} +static void dwc3_apple_set_role(struct dwc3_apple *a, enum dwc3_apple_state s) +{ assert(atcphy_usb3_set_mode(a->usb3_phy, s == DWC3_APPLE_HOST ? PHY_MODE_USB_HOST : PHY_MODE_USB_DEVICE, 0) == 0); } +struct fixture { struct apple_atcphy atc; struct phy p2, p3, extra; struct dwc3_apple apple; }; +static void initial(struct fixture *f, unsigned int gen, bool fixed, bool pm) +{ + memset(f, 0, sizeof(*f)); + memset(core_regs, 0, sizeof(core_regs)); + memset(pipe_regs, 0, sizeof(pipe_regs)); + memset(usb2_regs, 0, sizeof(usb2_regs)); + hardware = (struct atcphy_hw){ + .gen = gen == 0 ? ATCPHY_GENERATION_T8103 : ATCPHY_GENERATION_T8122, + .has_t8140_dp = gen == 2, .has_usb4 = gen != 2, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, + }; + f->atc = (struct apple_atcphy){ .hw = &hardware, + .regs.core = (unsigned char *)core_regs, .regs.lpdptx = (unsigned char *)core_regs, + .regs.pipehandler = (unsigned char *)pipe_regs, .regs.usb2phy = (unsigned char *)usb2_regs, + .mode = APPLE_ATCPHY_MODE_USB3, .typec_mode = APPLE_ATCPHY_MODE_USB3, + .pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3, + .host_active = true, .fixed_usb2 = fixed, .ss_tunables = true, .dp_tunables = true, + }; + f->p3 = (struct phy){ .data = &f->atc, .ops = &usb3_ops, .power_count = 1, .init_count = 1, + .pm_enabled = pm, .pm_refs = pm ? 1 : 0 }; + f->p2 = (struct phy){ .data = &f->atc, .ops = &usb2_ops, .power_count = 1, .init_count = 1, + .pm_enabled = pm, .pm_refs = pm ? 1 : 0 }; + f->extra = (struct phy){ .ops = &plain_ops, .power_count = 1, .init_count = 1, + .pm_enabled = pm, .pm_refs = pm ? 1 : 0 }; + f->apple = (struct dwc3_apple){ .state = DWC3_APPLE_HOST, .reset = &f->atc.rcdev, + .usb2_phy = &f->p2, .usb3_phy = &f->p3, .core_active = true, .reset_pending = true, + .usb3_power_off_pending = 1, .usb2_power_off_pending = 1, + .dwc = { .num_usb3_ports = 1, .num_usb2_ports = 1, + .usb3_generic_phy = { &f->p3 }, .usb2_generic_phy = { &f->p2 } }, + }; + core_regs[ATCPHY_POWER_CTRL / 4] = ATCPHY_POWER_PHY_RESET_N | ATCPHY_POWER_SLEEP_BIG | + ATCPHY_POWER_SLEEP_SMALL | ATCPHY_POWER_APB_RESET_N; + core_regs[ATCPHY_POWER_STAT / 4] = core_regs[ATCPHY_POWER_CTRL / 4]; + core_regs[AUS_COMMON_DIG_RCAL1 / 4] = AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + core_regs[ACIOPHY_TOP_PHY_STAT / 4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + pipe_regs[PIPEHANDLER_AON_GEN / 4] = PIPEHANDLER_AON_GEN_DWC3_RESET_N; + polls = fail_poll = reset_calls = reset_writes = core_writes = warnings = 0; + generic_warnings = 0; + host_exits = clocks_disabled = callback_calls = gadget_exits = event_cleanups = 0; + legacy_suspends = core_mode_exits = free_buffers = starts = deasserts = setup_cio = root_refs = 0; + host_init_error = gadget_init_error = core_init_error = reset_error = other_off_error = 0; + init_cleanup_fault = false; + syscore = true; + stuck_ack = false; +} +static void recover(void) { fail_poll = 0; stuck_ack = false; pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = 0; } +static int exit_apple(struct fixture *f) +{ mutex_lock(&f->apple.lock); int ret = dwc3_apple_exit(&f->apple); mutex_unlock(&f->apple.lock); return ret; } +static int role(struct fixture *f, enum usb_role r) +{ struct usb_role_switch sw = { &f->apple }; return dwc3_usb_role_switch_set(&sw, r); } +static void clean(struct fixture *f) +{ + assert(!f->p3.power_count && !f->p2.power_count && !f->p3.init_count && !f->p2.init_count); + assert(!f->p3.pm_refs && !f->p2.pm_refs && !f->apple.core_active && !f->apple.reset_pending); + assert(!f->apple.usb3_power_off_pending && !f->apple.usb2_power_off_pending); + assert(!f->atc.host_active && !f->atc.mode_dirty && f->atc.mode == APPLE_ATCPHY_MODE_OFF); + assert(!syscore); + assert(!f->atc.lock && !f->p3.mutex && !f->p2.mutex && !f->apple.lock); + assert(!(pipe_regs[PIPEHANDLER_AON_GEN / 4] & PIPEHANDLER_AON_GEN_DWC3_RESET_N)); +} +static void cycles(void) +{ + unsigned int cases = 0; + for (unsigned int gen = 0; gen < 3; gen++) + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int pm = 0; pm < 2; pm++) + for (unsigned int fault = 0; fault < 4; fault++) { + struct fixture f; initial(&f, gen, fixed, pm); + if (fault == 1) { + stuck_ack = true; pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + } else if (fault > 1) { + f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = fault - 1; + } + int ret = exit_apple(&f); + printf("CYCLE gen=%u fixed=%u pm=%u fault=%u exit=%d refs=%d init=%d\n", + gen, fixed, pm, fault, ret, f.p3.power_count, f.p3.init_count); fflush(stdout); + if (fault) { + assert(ret == -ETIMEDOUT && f.p3.power_count == 1 && f.p3.init_count == 1); + assert(f.apple.core_active && clocks_disabled == 0 && event_cleanups == 1 && host_exits == 1); + assert(syscore); + if (fault > 1) assert(f.atc.mode_dirty); + if (fault == 1) { + assert(role(&f, USB_ROLE_HOST) == -ETIMEDOUT && starts == 0 && deasserts == 0); + assert(host_exits == 1 && event_cleanups == 1 && f.p3.power_count == 1); + } + recover(); + assert(exit_apple(&f) == 0); + } else assert(ret == 0); + clean(&f); + assert(host_exits == 1 && clocks_disabled == 1 && event_cleanups == 1); + assert(exit_apple(&f) == 0 && host_exits == 1 && clocks_disabled == 1); + assert(role(&f, USB_ROLE_HOST) == 0 && f.p3.power_count == 1 && f.p3.init_count == 1); + assert(f.atc.mode == APPLE_ATCPHY_MODE_USB3 && !f.atc.mode_dirty); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); + assert(role(&f, USB_ROLE_NONE) == 0); + clean(&f); + assert(!generic_warnings); + if (!fault) assert(!warnings); + cases++; + } + printf("PASS: %u normal/parking/domain shutdown and restart scenarios\n", cases); +} +static void pending_reset(void) +{ + for (unsigned int gen = 0; gen < 3; gen++) { + struct fixture f; initial(&f, gen, false, true); + reset_error = -ETIMEDOUT; + assert(exit_apple(&f) == -ETIMEDOUT); + assert(!f.apple.core_active && f.apple.reset_pending && !f.p3.power_count && !f.p3.init_count); + assert(syscore); + assert(role(&f, USB_ROLE_HOST) == -ETIMEDOUT && !starts && !deasserts); + assert(host_exits == 1 && clocks_disabled == 1 && event_cleanups == 1 && callback_calls == 1); + reset_error = 0; + assert(exit_apple(&f) == 0); + clean(&f); + assert(host_exits == 1 && clocks_disabled == 1 && callback_calls == 1); + assert(reset_calls == 3); + } + puts("PASS: pending reset retry does not repeat host, PHY or core teardown"); +} +static void partial_release(void) +{ + struct fixture f; initial(&f, 1, false, true); + other_off_error = -EIO; + assert(exit_apple(&f) == -EIO); + assert(!f.p3.power_count && f.p3.init_count == 1 && f.p2.power_count == 1); + assert(!f.apple.usb3_power_off_pending && f.apple.usb2_power_off_pending == 1); + assert(callback_calls == 1 && !clocks_disabled); + other_off_error = 0; + assert(exit_apple(&f) == 0); clean(&f); + assert(callback_calls == 1 && event_cleanups == 1 && clocks_disabled == 1); + initial(&f, 1, false, false); + f.apple.dwc.num_usb3_ports = 2; f.apple.dwc.usb3_generic_phy[1] = &f.extra; + f.apple.usb3_power_off_pending = 3; + other_off_error = -EIO; + assert(exit_apple(&f) == -EIO && !f.p3.power_count && f.extra.power_count == 1); + other_off_error = 0; + assert(exit_apple(&f) == 0 && !f.extra.power_count && !f.extra.init_count); + clean(&f); assert(callback_calls == 1); + puts("PASS: successful USB3 references are not released twice after another PHY fails"); +} +static void init_failures(void) +{ + for (unsigned int device = 0; device < 2; device++) { + struct fixture f; initial(&f, 2, false, true); + assert(exit_apple(&f) == 0); + if (device) gadget_init_error = -EIO; else host_init_error = -EIO; + reset_error = -ETIMEDOUT; + assert(role(&f, device ? USB_ROLE_DEVICE : USB_ROLE_HOST) == -EIO); + assert(f.apple.reset_pending && !f.apple.core_active && !f.p3.power_count); + assert(syscore); + host_init_error = gadget_init_error = reset_error = 0; + assert(role(&f, device ? USB_ROLE_DEVICE : USB_ROLE_HOST) == 0); + assert(f.p3.power_count == 1 && f.p3.init_count == 1); + assert(role(&f, USB_ROLE_NONE) == 0); clean(&f); + } + struct fixture f; initial(&f, 1, false, false); assert(exit_apple(&f) == 0); + core_init_error = -EIO; reset_error = -ETIMEDOUT; + assert(role(&f, USB_ROLE_HOST) == -EIO && f.apple.reset_pending && !f.apple.core_active); + core_init_error = reset_error = 0; + assert(role(&f, USB_ROLE_HOST) == 0); + assert(role(&f, USB_ROLE_NONE) == 0); clean(&f); + initial(&f, 1, false, true); assert(exit_apple(&f) == 0); + host_init_error = -EIO; init_cleanup_fault = true; + assert(role(&f, USB_ROLE_HOST) == -EIO && f.apple.core_active); + assert(syscore); + assert(f.p3.power_count == 1 && f.p3.init_count == 1); + recover(); host_init_error = 0; init_cleanup_fault = false; + assert(role(&f, USB_ROLE_HOST) == 0 && f.p3.power_count == 1 && f.p3.init_count == 1); + assert(role(&f, USB_ROLE_NONE) == 0); clean(&f); + puts("PASS: host/gadget/core startup failure reset obligations drain before retry"); +} +static void pending_restart(void) +{ + unsigned int cases = 0; + for (unsigned int gen = 0; gen < 3; gen++) + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int pm = 0; pm < 2; pm++) + for (unsigned int fault = 1; fault < 4; fault++) { + struct fixture f; initial(&f, gen, fixed, pm); + if (fault == 1) { + stuck_ack = true; pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + } else { f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = fault - 1; } + assert(exit_apple(&f) == -ETIMEDOUT); + recover(); + assert(role(&f, USB_ROLE_HOST) == 0 && f.p3.power_count == 1 && f.p3.init_count == 1); + assert(f.atc.mode == APPLE_ATCPHY_MODE_USB3 && !f.atc.mode_dirty); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); + assert(!generic_warnings); + assert(role(&f, USB_ROLE_NONE) == 0); clean(&f); cases++; + } + struct fixture f; initial(&f, 2, true, true); f.apple.fixed_hub = true; + assert(role(&f, USB_ROLE_NONE) == 0 && f.apple.state == DWC3_APPLE_HOST && f.p3.power_count == 1); + assert(exit_apple(&f) == 0); + mutex_lock(&f.apple.lock); + assert(dwc3_apple_init(&f.apple, DWC3_APPLE_HOST) == 0); + mutex_unlock(&f.apple.lock); + assert(f.p3.power_count == 1 && exit_apple(&f) == 0); clean(&f); + printf("PASS: %u pending-failure to HOST transitions and actual fixed-hub role no-op\n", cases); +} +static void negative_power(void) +{ + struct fixture f; initial(&f, 0, false, false); + f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = 1; + assert(exit_apple(&f) == -ETIMEDOUT && f.p3.power_count == 1 && f.p3.init_count == 1); + recover(); assert(role(&f, USB_ROLE_HOST) == 0 && f.p3.power_count == 1); +} +static void negative_reset(void) +{ + struct fixture f; initial(&f, 0, false, false); reset_error = -ETIMEDOUT; + assert(exit_apple(&f) == -ETIMEDOUT && reset_calls == 1); + reset_error = 0; assert(exit_apple(&f) == 0 && reset_calls == 2); + assert(!(pipe_regs[PIPEHANDLER_AON_GEN / 4] & PIPEHANDLER_AON_GEN_DWC3_RESET_N)); +} +static void negative_legacy(void) +{ + struct fixture f; initial(&f, 0, false, false); + f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = 1; + assert(phy_power_off(&f.p3) == -ETIMEDOUT && f.atc.mode_dirty); + recover(); + struct typec_mux_dev mux = { &f.atc }; + struct typec_mux_state state = { .mode = TYPEC_STATE_USB }; + assert(atcphy_mux_set(&mux, &state) == 0); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); +} +static void legacy_restore(void) +{ + for (unsigned int gen = 0; gen < 3; gen++) { + struct fixture f; initial(&f, gen, false, false); + f.atc.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; fail_poll = 1; + assert(phy_power_off(&f.p3) == -ETIMEDOUT && f.atc.mode_dirty); + unsigned int before = core_writes; + assert(atcphy_usb3_set_mode(&f.p3, PHY_MODE_USB_HOST, 0) == -EBUSY && core_writes == before); + if (gen < 2) { + f.atc.dp_link_rate = ATCPHY_DP_LINK_RATE_RBR; + assert(atcphy_dp_configure(&f.atc, ATCPHY_DP_LINK_RATE_RBR) == -EBUSY && core_writes == before); + } + recover(); + struct typec_mux_dev mux = { &f.atc }; + struct typec_mux_state state = { .mode = TYPEC_STATE_USB }; + assert(atcphy_mux_set(&mux, &state) == 0 && !f.atc.mode_dirty && core_writes > before); + assert(core_regs[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); + assert(phy_power_off(&f.p3) == 0 && !f.p3.power_count); + } + puts("PASS: M1/M2/J700 aborted shutdown blocks PIPE and restores saved lane mode"); +} +static void removal_and_generic(void) +{ + struct fixture f; initial(&f, 0, false, false); + root_refs = 1; assert(phy_init(&f.p3) == 0 && phy_power_on(&f.p3) == 0); + assert(exit_apple(&f) == 0); clean(&f); + unsigned int callbacks = callback_calls, clocks = clocks_disabled, events = event_cleanups; + dwc3_core_remove_no_shutdown(&f.apple.dwc); + assert(callbacks == callback_calls && clocks == clocks_disabled && events == event_cleanups); + assert(!core_mode_exits && free_buffers == 1 && !f.p3.power_count && !f.p3.init_count); + initial(&f, 1, false, false); + dwc3_core_exit(&f.apple.dwc); + assert(!f.p3.power_count && !f.p3.init_count && !f.p2.power_count && !f.p2.init_count); + assert(event_cleanups == 1 && clocks_disabled == 1 && callback_calls == 1); + puts("PASS: root-hub references, already-exited removal and ordinary generic core exit"); +} +int main(int argc, char **argv) +{ + if (argc == 2 && !strcmp(argv[1], "power")) { negative_power(); return 0; } + if (argc == 2 && !strcmp(argv[1], "reset")) { negative_reset(); return 0; } + if (argc == 2 && !strcmp(argv[1], "legacy")) { negative_legacy(); return 0; } + cycles(); pending_restart(); pending_reset(); partial_release(); init_failures(); legacy_restore(); removal_and_generic(); + puts("PASS: Apple teardown actual-function qualification"); return 0; +} diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c new file mode 100644 index 00000000000000..e54e73146cbad6 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/mocks.c @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *lpdptx, *pipehandler, *usb2phy; } regs; + struct reset_controller_dev rcdev; + void *dev; + int lock, dp_link_rate; + bool swap_lanes, tunnel_clock_on, tunnel_saved; + bool fixed_usb2, host_active, mode_dirty, ss_tunables, dp_tunables, aux_power_requested; + unsigned int dp_owned_pmas, dp_idle_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +struct phy; +struct phy_ops { int (*init)(struct phy *); int (*exit)(struct phy *); + int (*power_on)(struct phy *); int (*power_off)(struct phy *); }; +struct phy { struct apple_atcphy *data; const struct phy_ops *ops; + int mutex, power_count, init_count, pm_refs, pwr, dev; bool pm_enabled; }; +#define phy_get_drvdata(p) ((p)->data) +static int phy_pm_runtime_get_sync(struct phy *p) +{ if (!p->pm_enabled) return -ENOTSUPP; p->pm_refs++; return 0; } +static void phy_pm_runtime_put(struct phy *p) { if (p->pm_enabled) p->pm_refs--; } +static void phy_pm_runtime_put_sync(struct phy *p) { phy_pm_runtime_put(p); } +static int regulator_enable(int p) { assert(!p); return 0; } +static void regulator_disable(int p) { assert(!p); } +struct dwc3 { struct phy *usb2_generic_phy[3], *usb3_generic_phy[3]; + int num_usb2_ports, num_usb3_ports, usb2_phy, usb3_phy; + struct reset_controller_dev *reset; + int dev, usb_psy, vbus_draw_work; }; +struct dwc3_apple { struct dwc3 dwc; enum dwc3_apple_state state; + struct reset_controller_dev *reset; int lock; void *dev; + struct phy *usb2_phy, *usb3_phy; + bool fixed_hub, core_active, core_exit_pending, reset_pending; + unsigned long usb2_power_off_pending, usb3_power_off_pending; }; +static unsigned int polls, fail_poll, reset_calls, reset_writes, core_writes; +static unsigned int host_exits, clocks_disabled, callback_calls; +static unsigned int gadget_exits, event_cleanups, legacy_suspends, core_mode_exits, free_buffers; +static unsigned int starts, deasserts, setup_cio, root_refs; +static int host_init_error, gadget_init_error, core_init_error, reset_error, other_off_error; +static bool init_cleanup_fault; +static bool syscore; +static bool stuck_ack; +static u32 core_regs[0x30000 / 4], pipe_regs[0x1000 / 4], usb2_regs[0x1000 / 4]; +static u32 readl(void *p) { return *(u32 *)p; } +static void mask32(void *p, u32 m, u32 v) +{ + *(u32 *)p = (*(u32 *)p & ~m) | v; + if (p == &pipe_regs[PIPEHANDLER_LOCK_REQ / 4] && !stuck_ack) + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = *(u32 *)p & PIPEHANDLER_LOCK_EN; + if (p == &pipe_regs[PIPEHANDLER_AON_GEN / 4]) reset_writes++; + if ((uintptr_t)p >= (uintptr_t)core_regs && + (uintptr_t)p < (uintptr_t)core_regs + sizeof(core_regs)) core_writes++; +} +static void writel(u32 v, void *p) { mask32(p, ~0U, v); } +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +static void core_mask32(struct apple_atcphy *p, u32 o, u32 m, u32 v) +{ + assert(o < sizeof(core_regs)); + mask32(p->regs.core + o, m, v); + if (o == ATCPHY_POWER_CTRL) core_regs[ATCPHY_POWER_STAT / 4] = core_regs[o / 4]; +} +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +#define readl_poll_timeout(a,v,c,i,t) \ + ({ (v) = readl(a); ++polls; polls == fail_poll || !(c) ? -ETIMEDOUT : 0; }) +#define readl_poll_timeout_atomic readl_poll_timeout +/* These dependencies are outside the cross-layer test; no DP ownership is seeded. */ +static int j700_dp_stop(struct apple_atcphy *p) { assert(!p->dp_owned_pmas); return 0; } +static int j700_dp_idle_pending(struct apple_atcphy *p) { (void)p; return 0; } +static void atc_tunnel_stop_t8103(struct apple_atcphy *p) { (void)p; } +static void atc_tunnel_restore(struct apple_atcphy *p) { (void)p; } +/* USB2 power-on is outside this teardown/recovery control. */ +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p; } +static void atcphy_apply_tunables(struct apple_atcphy *p, enum atcphy_mode m) { (void)p; (void)m; } +static int atcphy_configure_lanes(struct apple_atcphy *p, enum atcphy_mode m) { (void)p; (void)m; return 0; } +static int atcphy_enable_dp_aux(struct apple_atcphy *p) { (void)p; return 0; } +static int atcphy_auspll_apb_command(struct apple_atcphy *p, int command) +{ (void)p; (void)command; return 0; } +static int atcphy_dp_configure_lane(struct apple_atcphy *p, int lane, + const struct atcphy_dp_link_rate_configuration *config) +{ (void)p; (void)lane; (void)config; return 0; } +static void usb_phy_set_suspend(int p, int s) { (void)p; (void)s; legacy_suspends++; } +static void usb_phy_shutdown(int p) { (void)p; } +static void dwc3_event_buffers_cleanup(struct dwc3 *d) { (void)d; event_cleanups++; } +static void dwc3_clk_disable(struct dwc3 *d) { (void)d; clocks_disabled++; } +static void dwc3_host_exit(struct dwc3 *d); +static void dwc3_gadget_exit(struct dwc3 *d) { (void)d; gadget_exits++; } +static void dev_pm_syscore_device(void *d, bool s) { (void)d; syscore = s; } +static void dwc3_enable_susphy(struct dwc3 *d, bool s) { (void)d; (void)s; } +static int atcphy_dwc3_reset_assert(struct reset_controller_dev *, unsigned long); +static int reset_control_assert(struct reset_controller_dev *r) +{ if (!r) return 0; reset_calls++; if (reset_error) return reset_error; + return atcphy_dwc3_reset_assert(r, 0); } +static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *, unsigned long); +static int reset_control_deassert(struct reset_controller_dev *r) +{ deasserts++; return atcphy_dwc3_reset_deassert(r, 0); } +static int phy_set_mode(struct phy *p, int m) { (void)p; (void)m; return 0; } +static int dwc3_apple_core_probe(struct dwc3_apple *a); +static int dwc3_core_init(struct dwc3 *d); +static void dwc3_apple_setup_cio(struct dwc3_apple *a) { (void)a; setup_cio++; } +static int dwc3_apple_usb2_mode_before_reset(enum dwc3_apple_state s, enum phy_mode *m) +{ if (s != DWC3_APPLE_HOST && s != DWC3_APPLE_DEVICE) return -EINVAL; + *m = s == DWC3_APPLE_HOST ? PHY_MODE_INVALID : PHY_MODE_USB_DEVICE; return 0; } +static void dwc3_apple_set_role(struct dwc3_apple *a, enum dwc3_apple_state s); +static int dwc3_host_init(struct dwc3 *d) +{ (void)d; if (init_cleanup_fault) { stuck_ack = true; + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; } return host_init_error; } +static int dwc3_gadget_init(struct dwc3 *d) { (void)d; return gadget_init_error; } +#define pm_runtime_get_sync(...) ((void)0) +#define pm_runtime_allow(...) ((void)0) +#define pm_runtime_disable(...) ((void)0) +#define pm_runtime_dont_use_autosuspend(...) ((void)0) +#define pm_runtime_put_noidle(...) ((void)0) +#define pm_runtime_set_suspended(...) ((void)0) +#define dev_set_drvdata(...) ((void)0) +#define cancel_work_sync(...) ((void)0) +#define power_supply_put(...) ((void)0) +static void dwc3_core_exit_mode(struct dwc3 *d) { (void)d; core_mode_exits++; } +static void dwc3_debugfs_exit(struct dwc3 *d) { (void)d; } +static void dwc3_ulpi_exit(struct dwc3 *d) { (void)d; } +static void dwc3_free_event_buffers(struct dwc3 *d) { (void)d; free_buffers++; } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c new file mode 100644 index 00000000000000..a3e62f8a4f12da --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/dwc3/prefix.c @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint8_t u8; +#define ENOTSUPP 524 +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define field_prep FIELD_PREP +#define __iomem +#define udelay(n) ((void)(n)) +#define usleep_range(a,b) ((void)(a), (void)(b)) +#define fallthrough __attribute__((fallthrough)) +static unsigned int warnings, generic_warnings; +static void warning(const char *message) +{ warnings++; if (strstr(message, "phy_power_on was called before phy_init")) generic_warnings++; } +#define dev_warn(d, message, ...) warning(message) +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define lockdep_assert_held(p) assert(*(p)) +#define WARN_ON_ONCE(v) assert(!(v)) +static void mutex_lock(int *m) { assert(!*m); *m = 1; } +static void mutex_unlock(int *m) { assert(*m); *m = 0; } +struct held { int *m; }; +static struct held hold(int *m) { mutex_lock(m); return (struct held){m}; } +static void release(struct held *h) { mutex_unlock(h->m); } +#define guard(kind) struct held held_guard __attribute__((cleanup(release))) = hold +enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE }; +enum { APPLE_ATCPHY_LANE_0, APPLE_ATCPHY_LANE_1 }; +enum usb_role { USB_ROLE_NONE, USB_ROLE_HOST, USB_ROLE_DEVICE }; +struct usb_role_switch { struct dwc3_apple *data; }; +#define usb_role_switch_get_drvdata(s) ((s)->data) +#define PHY_MODE_INVALID (-1) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +struct reset_controller_dev { int unused; }; +#define container_of(p,t,m) ((t *)((char *)(p) - offsetof(t,m))) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py new file mode 100644 index 00000000000000..55a16678b97be2 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -0,0 +1,789 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Actual native EDID admission, service epochs, stale replies and RX progress.""" +from pathlib import Path +from common import block, source_defines + +def rebind(root): + base = root / 'drivers/gpu/drm/apple' + def function(file,signature): + text=(base/file).read_text();start=text.index(signature);opening=text.index('{',start) + while ';' in text[start:opening]: + start=text.index(signature,start+1);opening=text.index('{',start) + end,depth=opening+1,1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}');end+=1 + return text[start:end] + prefix=r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include "afkv2-codec.h" +typedef int64_t s64; +typedef u64 dma_addr_t; +#define U16_MAX UINT16_MAX +#define EBUSY 16 +#define ENOENT 2 +#define MAX_PENDING_CMDS 16 +#define AFK_MAX_CHANNEL 16 +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define WARN_ON(x) (x) +#define bitmap_empty(p,n) (!*(p)) +#define spin_lock_irqsave(p,f) do { assert(!*(p));*(p)=true;(f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f);assert(*(p));*(p)=false; } while(0) +static int test_bit(unsigned int bit,const unsigned long *p) { return !!(*p & (1UL<(b)?(a):(b)) +#define strscpy(dst,src,n) snprintf(dst,n,"%s",(char *)(src)) +#define trace_dcpavserv_init(...) ((void)0) +#define dma_rmb() ((void)0) +#define DRM_CONNECTOR_REGISTERED 1 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define EPIC_SUBTYPE_STD_SERVICE 0xc0 +#define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 +#define to_platform_device(x) (x) +#define ERR_PTR(x) ((void *)(intptr_t)(x)) +#define IS_ERR(x) ((uintptr_t)(x)>(uintptr_t)-4096) +#define lockdep_assert_held(p) assert(*(p)) +static bool *hpd_lock,*av_lock; +static void mutex_lock(bool *p) { if(p==hpd_lock) assert(!*av_lock);assert(!*p);*p=true; } +static void mutex_unlock(bool *p) { assert(*p);*p=false; } +static void complete(int *p) { (*p)++; } +static void reinit_completion(int *p) { *p=0; } +static void schedule_work(unsigned int *p) { (*p)++; } +struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; +struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; +struct apple_epic_service { + struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; + bool lock,enabled,torndown;unsigned int users;u32 channel;u8 cmd_tag; + unsigned long cmd_map[1];struct epic_cmd_info cmds[MAX_PENDING_CMDS]; + struct { u64 generation,epoch; } admission;void *cookie; +}; +struct apple_dcp_afkep { struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; +struct apple_connector { void *dcp;bool connected,edid_invalidation_pending;struct { int registration_state; } base;unsigned int hotplug_wq; }; +struct apple_dcp { + void *dev;int fw_compat;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; + bool hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; + struct { int iomfb_method_profile; } hw; + struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; +}; +struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; +static unsigned int dma_frees,opens; +static int open_error,open_action; +void dpavservep_detach(struct apple_dcp *dcp); +static void inject_owner(struct apple_dcp_afkep *ep,u32 pending_channel); +static void dma_free_coherent(void *dev,size_t size,void *p,dma_addr_t address) +{ (void)dev;assert(size && (uintptr_t)p==address);free(p);dma_frees++; } +static void afk_remove_service_debugfs(struct apple_epic_service *s) { (void)s; } +static void afk_populate_service_debugfs(struct apple_epic_service *s) { (void)s; } +static int afk_send_epic(struct apple_dcp_afkep *e,u32 channel,u16 tag,int type,int category,u8 subtype,const void *p,size_t n) +{ + assert(channel && !tag && type==EPIC_TYPE_NOTIFY && category==EPIC_CAT_REPORT && subtype==AFKV2_OPEN && !p && !n); + opens++; + assert(!e->dcp->dcpavserv.enabled && !e->dcp->dcpavserv.service); + for(unsigned int i=0;inum_channels;i++) { + struct apple_epic_service *s=&e->services[i]; + if(s->channel==channel && !s->torndown) assert(!s->enabled && !s->users); + } + if(open_action==1) e->dcp->typec_generation++; + if(open_action==2) dpavservep_detach(e->dcp); + if(open_action==4) inject_owner(e,channel); + if(open_action==5) e->dcp->typec_cable_connected=false; + if(open_action==6) e->dcp->quiescing=true; + if(open_action==3) { + for(unsigned int i=0;inum_channels;i++) + if(e->services[i].channel==channel && !e->services[i].torndown) e->services[i].channel++; + } + return open_error; +} +static void afk_v2_callback(struct apple_dcp_afkep *e,const struct afkv2_frame *f,const struct afkv2_command *c) { (void)e;(void)f;(void)c;assert(!"unexpected callback"); } + +struct item { const char *key,*value; s64 integer; }; +struct dcp_parse_ctx { struct item *items;unsigned int count,index;bool key; }; +struct iterator { struct dcp_parse_ctx *handle;unsigned int idx,len; }; +#define kfree(p) free((void *)(p)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +static struct item announcement_items[3]; +static int parse(u8 *p,size_t n,struct dcp_parse_ctx *ctx) +{ (void)p;assert(n==37);*ctx=(struct dcp_parse_ctx){.items=announcement_items,.count=3};return 0; } +static int iterator_begin(struct dcp_parse_ctx *h,struct iterator *it,bool dict) +{ assert(dict);h->index=0;h->key=true;*it=(struct iterator){.handle=h,.len=h->count};return 0; } +static char *parse_string(struct dcp_parse_ctx *h) +{ + struct item *i=&h->items[h->index];const char *p; + if(h->key) { p=i->key;h->key=false; } else { p=i->value;h->index++;h->key=true; } + if(!p) return ERR_PTR(-EINVAL);char *copy=malloc(strlen(p)+1);assert(copy);strcpy(copy,p);return copy; +} +static int parse_int(struct dcp_parse_ctx *h,s64 *out) +{ assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } +static int skip(struct dcp_parse_ctx *h) { assert(!h->key);h->index++;h->key=true;return 0; } +''' + parts=[function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), + function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('),function('afk.h','static inline void afk_service_disable('), + function('afk.h','static inline bool afk_service_matches('),function('afk.h','static inline bool afk_service_reclaimable('), + function('afk.h','static inline bool\nafk_service_reinit('),function('afk.h','static inline void afk_service_discard('), + function('afk.h','static inline void afk_service_publish('), + function('epic/dpavservep.c','static void dcpavserv_notify_edid('),function('epic/dpavservep.c','static bool dcpavserv_identity('), + function('epic/dpavservep.c','static bool dcpavserv_admission_current('),function('epic/dpavservep.c','static int dcpavserv_prepare('), + function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), + function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), + function('afk.c','static struct apple_epic_service *afk_epic_find_service('), + function('afk.c','static void afk_recv_handle_init('), + function('epic/dpavservep.c','static int dcpdpserv_prepare('), + function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), + function('afk.c','static void afk_recv_handle_reply('),function('afk.c','static void afk_v2_receive(')] + suffix=r''' +static struct apple_dcp d; +static struct apple_dcp_afkep ep; +static struct apple_connector con; +static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; +static const struct apple_epic_service_ops announcement_ops[]={ + {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, + {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, + {0} +}; +static const char *name="dcpav-service-epic",*class="AppleDCPDP2HDMI"; +static const char *native_class="DCPDP13Service"; +static struct apple_epic_service *prepare(u32 channel,s64 unit,const char *n,const char *c) +{ + for(unsigned int i=0;iusers==1); +} +static void repeated_announcements(void) +{ + ep.ops=announcement_ops;ep.match_epic_name=true;ep.num_channels=0;d.fw_compat=DCP_FIRMWARE_H17P; + /* More than the entire slot budget; wrap channel identity as well. */ + for(unsigned int i=0;i<4*AFK_MAX_CHANNEL;i++) { + u16 channel=1+4*(i%4);unsigned int before=opens; + /* Flip, same-attachment revalidation, return flip; swap both providers. */ + bool native=((i%4==1 || i%4==2) != !!((i/4)%2)); + const char *provider=native?native_class:class; + announce(channel,name,provider,channel); + struct apple_epic_service *owner=d.dcpavserv.service; + if(native) assert(owner && "native DP EDID provider must publish"); + assert(owner->ops==&announcement_ops[0]); + assert(owner && owner->channel==channel && owner->users==1 && opens==before+1); + assert(ep.num_channels<=2); /* Real allocator reuses, no slot exhaustion. */ + u64 epoch=d.dcpavserv.epoch; + announce(channel,name,provider,channel); /* Active duplicate: no extra OPEN/pin. */ + assert(opens==before+1 && owner->users==1 && d.dcpavserv.epoch==epoch); + announce(channel+2,"dcpdp-service-epic",provider,channel+2); + assert(opens==before+1 && d.dcpavserv.service==owner && d.dcpavserv.epoch==epoch); + if(i%4==1) { + u64 generation=d.typec_generation; + closed(channel,0x6a01,0); /* Revalidation retires service, not attachment. */ + assert(!d.dcpavserv.service && d.typec_generation==generation && d.dcpavserv.epoch>epoch); + } else { + disconnect();connect(); + } + } + unsigned int missing_ops_before=opens; + assert(afk_match_service(&ep,"missing-service-epic")==NULL); + assert(afk_match_service(&ep,"dcpav-service-epic")==&announcement_ops[0]); + announce(13,"missing-service-epic",native_class,13); + announce(13,"dcpav-service-epic-extra",native_class,13); + assert(opens==missing_ops_before && !d.dcpavserv.service); + assert(ep.num_channels==2); /* Auxiliary rejection leaves a reusable slot. */ + unsigned int before=opens; + announce(13,name,"synthetic-wrong-class",13);assert(opens==before && !d.dcpavserv.service); + char long_class[161];memset(long_class,'x',160);long_class[160]=0; + announce(13,name,long_class,13);assert(opens==before && !d.dcpavserv.service); + for(unsigned int variant=0;variant<2;variant++) { + const char *provider=variant?native_class:class; + assert(!prepare(0,0,name,provider));assert(!prepare(65536,65536,name,provider)); + assert(!prepare(13,-1,name,provider));assert(!prepare(13,0,name,provider)); + assert(!prepare(13,1,name,provider));assert(!prepare(13,13,NULL,provider)); + assert(!prepare(13,13,"dcpdp-service-epic",provider)); + announce(13,"wrong-compartment",provider,13);assert(opens==before && !d.dcpavserv.service); + ep.endpoint=0x2c;announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);ep.endpoint=0x2a; + d.quiescing=true;announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);d.quiescing=false; + disconnect();announce(13,name,provider,13);assert(opens==before && !d.dcpavserv.service);connect(); + } + for(unsigned int variant=0;variant<4;variant++) { + const char *forged[]={"DCPDP13ServiceExtra","dcpdp13service","AppleDCPDP2HDMIExtra","unrelated-provider"}; + announce(13,name,forged[variant],13);assert(opens==before && !d.dcpavserv.service); + } + assert(!prepare(13,13,name,NULL)); + open_error=-EIO;announce(13,name,native_class,13);assert(opens==++before && !d.dcpavserv.service);open_error=0; + for(unsigned int window=0;window<5;window++) { + const int actions[]={1,2,3,5,6};open_action=actions[window]; + announce(13,name,native_class,13); + assert(opens==++before && !d.dcpavserv.service && !d.dcpavserv.enabled); + d.typec_cable_connected=true;d.quiescing=false; + } + open_action=4;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service==injected_owner); + assert(injected_owner->users==1);dpavservep_detach(&d);assert(!injected_owner->users); + open_action=0;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); + struct apple_epic_service *old=d.dcpavserv.service; + assert(afk_service_get(old)==old);old->cmd_map[0]=1; + disconnect();connect(); + announce(13,name,native_class,13); /* Same channel still owns a pending reply. */ + assert(opens==before && !d.dcpavserv.service && old->users==1 && old->channel==13); + announce(1,name,class,1);assert(opens==++before && d.dcpavserv.service!=old); + struct apple_epic_service *fresh=d.dcpavserv.service;u64 epoch=d.dcpavserv.epoch; + dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + old->cmd_map[0]=0;afk_service_put(old);disconnect();connect(); + announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); + struct apple_epic_service *owner=d.dcpavserv.service; + u64 owner_epoch=d.dcpavserv.epoch; + announce(5,name,class,5); /* A second admitted identity cannot displace the owner. */ + assert(opens==before && d.dcpavserv.service==owner && d.dcpavserv.epoch==owner_epoch); + dpavservep_detach(&d); + /* All slots held by retired readers: fail safely, then recover after puts. */ + for(unsigned int i=0;itorndown); + legacy=prepare(13,0,name,native_class);assert(legacy); + d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); + assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); + assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); + ep.wire_v2=true;ep.endpoint=0x2a; + announce(U16_MAX,name,native_class,U16_MAX);assert(d.dcpavserv.service && d.dcpavserv.service->channel==U16_MAX); + dpavservep_detach(&d); + puts("PASS: actual AFKV2 HDMI/native DP alternating over 64 cycles: flip/revalidation/return, strict identity, stale OPEN generation/epoch/owner, auxiliary isolation, reader/command pins and legacy controls"); +} +int main(void) +{ + d.hw.iomfb_method_profile=2;d.typec_cable_connected=true;d.typec_generation=1;d.nr_modes=1;d.connector=&con;d.dev=&d; + con.dcp=&d;con.connected=true;con.base.registration_state=1; + ep.dcp=&d;ep.wire_v2=true;ep.endpoint=0x2a;ep.num_channels=AFK_MAX_CHANNEL;d.dcpavservep=&ep; + hpd_lock=&d.hpd_mutex;av_lock=&d.dcpavserv.lock; + struct item h17p[]={{"compartment","dcpav-service-epic",0},{"name","AppleDCPDP2HDMI",0},{"interface-id",NULL,13}}; + struct dcp_parse_ctx parse_ctx={.items=h17p,.count=3};const char *parsed_name,*parsed_class;s64 parsed_unit=-1; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,true)); + assert(parsed_unit==13 && !strcmp(parsed_name,name) && !strcmp(parsed_class,class)); + struct apple_epic_service *proof=prepare(13,parsed_unit,parsed_name,parsed_class);assert(proof);afk_service_discard(proof); + free((void *)parsed_name);free((void *)parsed_class); + struct item legacy[]={{"EPICName","dcpav-service-epic",0},{"EPICProviderClass","legacy-class",0},{"EPICUnit",NULL,0}}; + parse_ctx=(struct dcp_parse_ctx){.items=legacy,.count=3}; + assert(!parse_epic_service_init(&parse_ctx,&parsed_name,&parsed_class,&parsed_unit,false));assert(parsed_unit==0); + free((void *)parsed_name);free((void *)parsed_class); + struct apple_epic_service *old=prepare(1,1,name,class);assert(old);publish(old,1); + assert(d.dcpavserv.service==old && old->users==1); + assert(afk_service_get(old)==old); /* Delayed EDID reader pins the retired slot. */ + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32);assert(old->cmds[0].rxbuf && old->cmds[0].txbuf); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf;old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf;old->cmd_map[0]=1; + disconnect();assert(!d.dcpavserv.service && !d.dcpavserv.enabled && old->users==1 && old->torndown && !old->enabled); + assert(con.edid_invalidation_pending && !afk_service_get(old)); + assert(!afk_service_reinit(old,&ep,&ops,5)); + assert(!prepare(5,5,name,class)); /* No admission while physically disconnected. */ + connect();struct apple_epic_service *fresh=prepare(5,5,name,class);assert(fresh && fresh!=old);publish(fresh,5); + assert(d.dcpavserv.service==fresh && fresh->users==1); + u64 epoch=d.dcpavserv.epoch; + dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + closed(1,0x6a01,0);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch && !dma_frees); + u8 oob[32],reply[56];size_t written; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32));afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(reply,sizeof(reply),1,1,0,0xc0,AFKV2_RESPONSE,0,oob,sizeof(oob),&written)); + afk_v2_receive(&ep,0,0,reply,written); + assert(dma_frees==2 && !old->cmd_map[0] && old->users==1 && d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); + assert(!afk_service_reinit(old,&ep,&ops,9));afk_service_put(old); + afk_v2_receive(&ep,0,0,reply,written);assert(dma_frees==2); /* Duplicate late ACK. */ + disconnect();connect();struct apple_epic_service *next=prepare(9,9,name,class);assert(next);publish(next,9); + assert(d.dcpavserv.service==next); + closed(1,0x6a01,0);closed(5,0x6a01,0);assert(d.dcpavserv.service==next); + disconnect();connect();next=prepare(13,13,name,class);assert(next);publish(next,13);assert(d.dcpavserv.service==next); + closed(13,0x6a01,1);assert(d.dcpavserv.service==next); /* Nonzero result rejected. */ + closed(13,0x6a00,0);assert(d.dcpavserv.service==next); /* Wrong low flags rejected. */ + d.dcpavservep=NULL;closed(13,0x6a01,0);assert(d.dcpavserv.service==next);d.dcpavservep=&ep; + closed(13,0x6a01,0);assert(!d.dcpavserv.service && !d.dcpavserv.enabled); + u8 result_byte=0;struct afkv2_frame notification={.category=AFKV2_REPORT,.type=AFKV2_CLOSED,.interface_flags=1,.payload=&result_byte,.payload_size=1}; + assert(afkv2_closed_report(¬ification));notification.token=1;assert(!afkv2_closed_report(¬ification)); + notification.token=0;notification.payload_size=0;assert(!afkv2_closed_report(¬ification)); + assert(!prepare(0,0,name,class));assert(!prepare(65536,65536,name,class)); + assert(!prepare(9,1,name,class));assert(!prepare(9,9,"wrong",class));assert(!prepare(9,9,name,"wrong")); + struct apple_epic_service *pending=prepare(9,9,name,class);assert(pending); + disconnect();connect();publish(pending,9); + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled && !pending->users); + pending=prepare(9,9,name,class);assert(pending); + mutex_lock(&d.hpd_mutex);d.typec_generation++;mutex_unlock(&d.hpd_mutex); + publish(pending,9); /* Generation change alone, without registry retirement. */ + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled); + pending=prepare(13,13,name,class);assert(pending); + dpavservep_detach(&d);publish(pending,13); /* Registry retirement with no physical change. */ + assert(!d.dcpavserv.service && pending->torndown && !pending->enabled); + ep.wire_v2=false;ep.endpoint=0x28;d.hw.iomfb_method_profile=0; + assert(!prepare(7,7,name,class));pending=prepare(7,0,name,NULL);assert(pending);publish(pending,0); + assert(d.dcpavserv.service==pending && pending->users==1);dpavservep_detach(&d); + assert(!pending->users && !d.hpd_mutex && !d.dcpavserv.lock); + memset(&ep.services,0,sizeof(ep.services));ep.wire_v2=true;ep.endpoint=0x2a;d.hw.iomfb_method_profile=2; + repeated_announcements(); + puts("PASS: actual dynamic 1/5/9/13 binding, forged identities, prepare/init epochs, physical retirement, reader pins, real old CLOSED/late ACK isolation and legacy unit 0"); +} +''' + return prefix + '\n'.join(parts) + suffix + +def admission(root): + + base = root / 'drivers/gpu/drm/apple' + + def function(file, signature): + source = (base / file).read_text() + start = source.index(signature) + opening = source.index('{', start) + while ';' in source[start:opening]: + start = source.index(signature, start + 1) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + av = (base / 'epic/dpavservep.c').read_text() + af = (base / 'afk.c').read_text() + announce = af[af.index('static void afk_recv_handle_init('):af.index('static void afk_recv_handle_init(') + 6500] + assert announce.index('ops->prepare(service,') < announce.index('EPIC_CAT_REPORT, AFKV2_OPEN') + assert '.prepare = dcpavserv_prepare,' in av and '.prepare = dcpdpserv_prepare,' in av + prefix = r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint64_t u64; +typedef int64_t s64; +#define U16_MAX UINT16_MAX +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define ERR_PTR(x) ((void *)(intptr_t)(x)) +#define PTR_ERR(x) ((int)(intptr_t)(x)) +#define IS_ERR(x) ((uintptr_t)(x) > (uintptr_t)-4096) +#define spin_lock_irqsave(p,f) do { assert(!*(p)); *(p)=true; (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); assert(*(p)); *(p)=false; } while(0) +#define lockdep_assert_held(p) assert(*(p)) +static bool *acquire(bool *p) { assert(!*p); *p=true; return p; } +static void release(bool **p) { if(*p) { assert(**p); **p=false; } } +static bool *once(bool *p) { assert(*p); *p=false; return NULL; } +#define scoped_guard(k,p) for(bool *_lock __attribute__((cleanup(release)))=acquire(p); _lock; _lock=once(_lock)) +static void mutex_lock(bool *p) { (void)acquire(p); } +static void mutex_unlock(bool *p) { (void)once(p); } +#define dev_info(...) ((void)0) +#define dev_err(...) ((void)0) +#define trace_dcpavserv_init(...) ((void)0) +#define DRM_CONNECTOR_REGISTERED 1 +#define to_platform_device(x) (x) +static void reinit_completion(int *p) { *p=0; } +static void complete(int *p) { *p=1; } +static void schedule_work(int *p) { (*p)++; } +#define dev_dbg_ratelimited(...) ((void)0) +#define dev_warn_ratelimited(...) ((void)0) +#define trace_dcp_recv_msg(...) ((void)0) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DCP_FIRMWARE_H17P 2 +#define AV_ENDPOINT 0x29 +#define SYSTEM_ENDPOINT 0x20 +#define DISP0_ENDPOINT 0x23 +#define DPAVSERV_ENDPOINT 0x28 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DPTX_ENDPOINT 0x2a +#define DPTX_ENDPOINT_V2 0x2c +#define DPTX_CONTROLLER_ENDPOINT 0x24 +#define IOMFB_ENDPOINT 0x37 +struct apple_dcp; +struct apple_connector { void *dcp; struct { int registration_state; } base; bool connected, edid_invalidation_pending; int hotplug_wq; }; +struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; bool wire_v2; }; +struct apple_epic_service { struct apple_dcp_afkep *ep; unsigned int users; bool enabled,torndown; struct { u64 generation,epoch; } admission; void *cookie; unsigned int channel; }; +struct apple_dcp { + struct apple_dcp_afkep *dcpavservep, *dptxep, *avep, *systemep, *ibootep, *dptx_controller_ep; + struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; + struct apple_connector *connector; unsigned int nr_modes; + struct { int iomfb_method_profile; } hw; + bool hpd_mutex, typec_cable_connected, quiescing; + u64 typec_generation; + int fw_compat; + void *dev; +}; +struct drm_edid { int identity; }; +static unsigned int reads, allocations, legacy_reads, receives; +static unsigned int event; +static struct apple_dcp_afkep *received_ep; +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) +{ if(!s || !s->enabled || s->torndown) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) +{ assert(s->users); s->users--; } +static void afk_service_retire(struct apple_epic_service *s) { s->enabled=false;s->torndown=true; } +static void drm_edid_free(const struct drm_edid *e) +{ if(e) { assert(allocations); allocations--; free((void *)e); } } +static void dcpavserv_teardown(struct apple_epic_service *s); +static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *s) +{ + struct apple_dcp *d=s->ep->dcp; + assert(!d->hpd_mutex && !d->dcpavserv.lock && s->users==2); + reads++; + if(event==1 || event==2) { d->typec_generation++; d->typec_cable_connected=(event==2); } + if(event==3) { s->enabled=false; dcpavserv_teardown(s); } + struct drm_edid *e=malloc(sizeof(*e)); assert(e); e->identity=42; allocations++; return e; +} +static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *s) +{ legacy_reads++; return dcpavserv_read_edid_inline(s); } +static void iomfb_recv_msg(struct apple_dcp *d, u64 message) { (void)d; (void)message; } +static void afk_receive_message(struct apple_dcp_afkep *ep, u64 message) +{ (void)message; received_ep=ep; receives++; } +''' + parts = [function('epic/dpavservep.c', 'static void dcpavserv_notify_edid('), + function('epic/dpavservep.c', 'static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c', 'static void dcpavserv_teardown('), + function('dcp.c', 'static struct apple_dcp_afkep *dcp_afkep('), + function('dcp.c', 'static void dcp_recv_msg('), + function('epic/dpavservep.c', 'static bool dcpavserv_identity('), + function('epic/dpavservep.c', 'static bool dcpavserv_admission_current('), + function('epic/dpavservep.c', 'static int dcpavserv_prepare('), + function('epic/dpavservep.c', 'static int dcpdpserv_prepare('), + function('epic/dpavservep.c', 'static void dcpavserv_init('), + function('epic/dpavservep.c', 'bool dcpavserv_edid_current('), + function('epic/dpavservep.c', 'bool dcpavserv_edid_service_current('), + function('epic/dpavservep.c', 'const struct drm_edid *dcpavserv_copy_edid(')] + suffix = r''' +int main(void) +{ + u64 epoch=0; + int device; + struct apple_connector connector={.dcp=&device,.base.registration_state=DRM_CONNECTOR_REGISTERED,.connected=true}; + struct apple_dcp d={.fw_compat=DCP_FIRMWARE_H17P,.dev=&device,.connector=&connector,.nr_modes=1}; + struct apple_dcp_afkep av={.dcp=&d,.endpoint=0x28}, tx={.dcp=&d,.endpoint=0x2a}; + d.dcpavservep=&av; d.dptxep=&tx; + assert(dcp_afkep(&d,0x28)==&av && dcp_afkep(&d,0x2a)==&tx && !dcp_afkep(&d,0x2c)); + dcp_recv_msg(&d,0x28,1); assert(received_ep==&av && receives==1); + dcp_recv_msg(&d,0x2a,2); assert(received_ep==&tx && receives==2); + d.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL; + av.endpoint=0x2a; av.wire_v2=true; tx.endpoint=0x2c; + assert(!dcp_afkep(&d,0x28) && dcp_afkep(&d,0x2a)==&av && dcp_afkep(&d,0x2c)==&tx); + dcp_recv_msg(&d,0x28,3); assert(receives==2); + dcp_recv_msg(&d,0x2a,4); assert(received_ep==&av && receives==3); + dcp_recv_msg(&d,0x2c,5); assert(received_ep==&tx && receives==4); + d.dcpavservep=NULL; dcp_recv_msg(&d,0x2a,6); assert(receives==4); d.dcpavservep=&av; + struct apple_epic_service s={.ep=&av,.users=1,.enabled=true,.channel=1}; + const char *name="dcpav-service-epic", *class="AppleDCPDP2HDMI"; + assert(!dcpavserv_prepare(&s,name,class,1)); + assert(dcpavserv_prepare(&s,name,class,0)==-ENODEV); + assert(dcpavserv_prepare(&s,name,class,3)==-ENODEV); + assert(dcpavserv_prepare(&s,name,class,-1)==-ENODEV); + assert(dcpavserv_prepare(&s,"dcpdp-service-epic",class,1)==-ENODEV); + assert(dcpavserv_prepare(&s,name,"wrong",1)==-ENODEV); + assert(dcpavserv_prepare(&s,NULL,class,1)==-ENODEV); + assert(dcpavserv_prepare(&s,name,NULL,1)==-ENODEV); + av.endpoint=0x28; assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); av.endpoint=0x2a; + assert(dcpdpserv_prepare(&s,"dcpdp-service-epic",class,3)==-ENODEV); + d.dcpavserv.enabled=true; assert(dcpavserv_prepare(&s,name,class,1)==-EBUSY); + d.dcpavserv.service=&s; d.dcpavserv.epoch=1; d.typec_generation=1; d.typec_cable_connected=true; + const struct drm_edid *e=dcpavserv_copy_edid(&d,1,&epoch); + assert(!IS_ERR(e) && s.users==1 && reads==1); drm_edid_free(e); + d.typec_cable_connected=false; + assert(PTR_ERR(dcpavserv_copy_edid(&d,1,&epoch))==-ESTALE && reads==1); + d.typec_cable_connected=true; + assert(PTR_ERR(dcpavserv_copy_edid(&d,0,&epoch))==-ESTALE && reads==1); + event=1; assert(PTR_ERR(dcpavserv_copy_edid(&d,1,&epoch))==-ESTALE && !allocations && s.users==1); + event=2; d.typec_cable_connected=true; + assert(PTR_ERR(dcpavserv_copy_edid(&d,2,&epoch))==-ESTALE && !allocations && s.users==1); + event=0; e=dcpavserv_copy_edid(&d,3,&epoch); assert(!IS_ERR(e)); drm_edid_free(e); + event=3; assert(PTR_ERR(dcpavserv_copy_edid(&d,3,&epoch))==-ENODEV && !allocations && !s.users); + assert(PTR_ERR(dcpavserv_copy_edid(&d,3,&epoch))==-ENODEV); + assert(connector.edid_invalidation_pending && connector.hotplug_wq && d.dcpavserv.epoch==2); + /* Same slot and physical generation: CLOSE/re-OPEN gets a fresh ticket. */ + s.enabled=true; assert(!dcpavserv_prepare(&s,name,class,1)); dcpavserv_init(&s,name,class,1); + assert(d.dcpavserv.service==&s && s.users==1 && d.dcpavserv.epoch==3); + mutex_lock(&d.dcpavserv.lock); + assert(!dcpavserv_edid_service_current(&d,1) && dcpavserv_edid_service_current(&d,3)); + mutex_unlock(&d.dcpavserv.lock); + event=0; e=dcpavserv_copy_edid(&d,3,&epoch); assert(!IS_ERR(e) && epoch==3); + /* Simulate retirement in the final validation-to-publication gap. */ + s.enabled=false; dcpavserv_teardown(&s); s.enabled=true; assert(!dcpavserv_prepare(&s,name,class,1)); dcpavserv_init(&s,name,class,1); + mutex_lock(&d.dcpavserv.lock); assert(!dcpavserv_edid_service_current(&d,epoch)); mutex_unlock(&d.dcpavserv.lock); + drm_edid_free(e); assert(d.typec_generation==3 && d.dcpavserv.epoch==5 && s.users==1); + s.enabled=false; dcpavserv_teardown(&s); + av.wire_v2=false; av.endpoint=0x28; d.hw.iomfb_method_profile=0; + d.dcpavserv.enabled=false; assert(!dcpavserv_prepare(&s,name,NULL,0)); + assert(dcpavserv_prepare(&s,name,class,1)==-ENODEV); + assert(!dcpdpserv_prepare(&s,NULL,NULL,3)); + d.dcpavserv.service=&s; d.dcpavserv.enabled=true; s.enabled=true; s.users=1; event=0; + d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0,&epoch); + assert(!IS_ERR(e) && legacy_reads==1 && s.users==1); drm_edid_free(e); + assert(!allocations && !d.hpd_mutex && !d.dcpavserv.lock); + puts("PASS: production old/new lookup and receive, exact service admission before OPEN, delayed disconnect/replug, service detach pins, legacy transport"); +} +''' + return prefix + '\n'.join(parts) + suffix + +def rx_progress(root): + + def source(path): + return (root / path).read_text() + + def function(text, signature): + start = text.index(signature) + opening = text.index('{', start) + while ';' in text[start:opening]: + start = text.index(signature, start+1) + opening = text.index('{', start) + end, depth = opening+1, 1 + while depth: + depth += (text[end]=='{')-(text[end]=='}') + end += 1 + return text[start:end] + + dcp=source('drivers/gpu/drm/apple/dcp.c') + iomfb=source('drivers/gpu/drm/apple/iomfb.c') + af=source('drivers/gpu/drm/apple/afk.c') + rtkit=source('drivers/soc/apple/rtkit.c') + template=source('drivers/gpu/drm/apple/iomfb_template.c') + assert 'alloc_ordered_workqueue("rtkit-%s"' in rtkit + assert '.recv_message = dcp_recv_msg' in dcp + assert 'dcp_set_dimensions(dcp);' in template[template.index('"DisplayAttributes"'):] + assert 'queue_work(ep->wq, &work->work)' in af + prefix=r''' +#undef NDEBUG +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x)=(v)) +#define GFP_KERNEL 0 +#define kzalloc(n,f) calloc(1,n) +#define kfree(p) free(p) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define INIT_WORK(p,f) ((p)->fn=(f)) +#define dev_warn(...) ((void)0) +#define DCP_FW_VER 26 +#define DCP_FW_VERSION(a,b,c) (a) +#define SZ_1M (1024*1024) +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define dev_warn_ratelimited(...) ((void)0) +#define dev_dbg_ratelimited(...) ((void)0) +#define trace_dcp_recv_msg(...) ((void)0) +#define DCP_FIRMWARE_H17P 2 +#define IOMFB_ENDPOINT 0x37 +#define AV_ENDPOINT 0x29 +#define SYSTEM_ENDPOINT 0x20 +#define DISP0_ENDPOINT 0x23 +#define DPTX_CONTROLLER_ENDPOINT 0x24 +#define APPLE_RTKIT_EP_MGMT 0 +#define APPLE_RTKIT_EP_CRASHLOG 1 +#define APPLE_RTKIT_EP_SYSLOG 2 +#define APPLE_RTKIT_EP_IOREPORT 4 +#define APPLE_RTKIT_EP_OSLOG 8 +#define APPLE_RTKIT_APP_ENDPOINT_START 0x20 +struct mutex { pthread_mutex_t native; }; +static _Thread_local struct mutex *locks[4]; +static _Thread_local unsigned int depth; +static bool held(struct mutex *m) { for(unsigned int i=0;inative)); locks[depth++]=m; } +static void mutex_unlock(struct mutex *m) { assert(depth && locks[depth-1]==m); depth--; assert(!pthread_mutex_unlock(&m->native)); } +#define lockdep_assert_held(p) assert(held(p)) +#define spin_lock_irqsave(p,f) do { mutex_lock(p); (f)=0; } while(0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); mutex_unlock(p); } while(0) +struct work_struct { void (*fn)(struct work_struct *); }; +struct workqueue_struct { struct work_struct *pending; pthread_cond_t ready; pthread_mutex_t lock; }; +struct drm_device { struct { struct mutex mutex; } mode_config; }; +struct drm_connector { struct drm_device *dev; struct { int width_mm,height_mm; } display_info; }; +struct apple_connector { struct drm_connector base; struct work_struct hotplug_wq; }; +struct apple_dcp; +struct apple_dcp_afkep { struct apple_dcp *dcp; u8 endpoint; struct mutex lock; bool stopping; struct workqueue_struct *wq; }; +struct test_mode_ticket { u64 generation; bool associated; }; +struct apple_dcp { + bool quiescing; int fw_compat; void *dev; + struct { int iomfb_method_profile; } hw; + struct { struct mutex lock; u64 epoch; bool enabled; void *service; } dcpavserv; + struct test_mode_ticket dummy_ticket; + struct { u64 connection_sequence; } rate_proof; + struct { struct mutex lock; struct { u64 generation; bool connected; } epoch; } metadata; + u64 typec_generation; bool typec_cable_connected; + struct { size_t length; void *data; u64 av_service_epoch, automatic_connection_sequence; bool automatic_service_pending; struct test_mode_ticket mode_ticket; } chunks; + int width_mm,height_mm; struct { int width_mm,height_mm; } panel; + unsigned int nr_modes; struct { struct { int width_mm,height_mm; } mode; } modes[1]; + struct apple_connector *connector; + struct apple_dcp_afkep *dcpavservep,*dptxep,*avep,*systemep,*ibootep,*dptx_controller_ep; + struct { struct mutex lock; bool stopped; void *owner; } iomfb; +}; +struct afk_receive_message_work { struct apple_dcp_afkep *ep; u64 message; struct work_struct work; }; +struct apple_rtkit_ops { void (*recv_message)(void *,u8,u64); }; +struct apple_rtkit { bool shutting_down; const struct apple_rtkit_ops *ops; void *cookie,*dev; }; +struct apple_rtkit_rx_work { struct apple_rtkit *rtk; u8 ep; u64 msg; struct work_struct work; }; +#define current ((void *)1) +static pthread_mutex_t reply_lock=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t reply_ready=PTHREAD_COND_INITIALIZER; +static bool reply_done; +static unsigned int advances, dimension_updates; +static struct workqueue_struct dimensions_queue={.ready=PTHREAD_COND_INITIALIZER,.lock=PTHREAD_MUTEX_INITIALIZER}; +static struct workqueue_struct afk_queue={.ready=PTHREAD_COND_INITIALIZER,.lock=PTHREAD_MUTEX_INITIALIZER}; +static void queue_work(struct workqueue_struct *q,struct work_struct *w) +{ assert(!pthread_mutex_lock(&q->lock)); assert(!q->pending); q->pending=w; pthread_cond_signal(&q->ready); assert(!pthread_mutex_unlock(&q->lock)); } +static void schedule_work(struct work_struct *w) { queue_work(&dimensions_queue,w); } +static void afk_receive_message_worker(struct work_struct *w) +{ + struct afk_receive_message_work *a=container_of(w,struct afk_receive_message_work,work); + assert(a->ep->endpoint==0x2a && a->message==7); + assert(!pthread_mutex_lock(&reply_lock)); reply_done=true; + pthread_cond_signal(&reply_ready); assert(!pthread_mutex_unlock(&reply_lock)); free(a); +} +static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return true; } +static void iomfb_queue_advance(struct apple_dcp *d) { assert(held(&d->iomfb.lock)); advances++; } +static void dcp_set_dimensions(struct apple_dcp *d); +static u8 dcpep_cb_prop_start(struct apple_dcp *d,u32 *length); +static void dcp_firmware_metadata_begin(struct apple_dcp *d) { (void)d; } +#define dcp_metadata_begin(e) ((struct test_mode_ticket){.generation=(e)->generation,.associated=(e)->connected}) +static void iomfb_recv_message(struct apple_dcp *d,u64 message) +{ + assert(message==1 && held(&d->iomfb.lock)); dcp_set_dimensions(d); + u32 length=64; + assert(dcpep_cb_prop_start(d,&length)); +} +#define rx_unused(name) static void name(struct apple_rtkit *r,u64 m) { (void)r;(void)m; } +rx_unused(apple_rtkit_management_rx) +rx_unused(apple_rtkit_crashlog_rx) +rx_unused(apple_rtkit_syslog_rx) +rx_unused(apple_rtkit_ioreport_rx) +rx_unused(apple_rtkit_oslog_rx) +''' + parts=[function(dcp,'void dcp_set_dimensions('), function(template,'static u8 dcpep_cb_prop_start(')] + if 'void dcp_update_connector_dimensions(' in dcp: + parts.append(function(dcp,'void dcp_update_connector_dimensions(')) + else: + parts.append('static void dcp_update_connector_dimensions(struct apple_dcp *d, struct drm_connector *c) { (void)d; (void)c; }') + parts += [function(iomfb,'void iomfb_recv_msg('),function(af,'int afk_receive_message('), + function(dcp,'static struct apple_dcp_afkep *dcp_afkep('), + function(dcp,'static void dcp_recv_msg('),function(rtkit,'static void apple_rtkit_rx_work(')] + suffix=r''' +static struct drm_device drm={.mode_config.mutex.native=PTHREAD_MUTEX_INITIALIZER}; +static struct apple_connector connector={.base.dev=&drm}; +static struct apple_dcp d={.connector=&connector,.width_mm=600,.height_mm=340,.nr_modes=1,.fw_compat=DCP_FIRMWARE_H17P,.iomfb.lock.native=PTHREAD_MUTEX_INITIALIZER}; +static struct apple_dcp_afkep av={.dcp=&d,.endpoint=0x2a,.lock.native=PTHREAD_MUTEX_INITIALIZER,.wq=&afk_queue}; +static void dimension_work(struct work_struct *w) +{ + (void)w; + mutex_lock(&drm.mode_config.mutex); + dcp_update_connector_dimensions(&d,&connector.base); dimension_updates++; + mutex_unlock(&drm.mode_config.mutex); +} +static void *worker(void *arg) +{ + struct workqueue_struct *q=arg; + assert(!pthread_mutex_lock(&q->lock)); + while(!q->pending) assert(!pthread_cond_wait(&q->ready,&q->lock)); + struct work_struct *w=q->pending; q->pending=NULL; + assert(!pthread_mutex_unlock(&q->lock)); w->fn(w); return NULL; +} +static const struct apple_rtkit_ops ops={.recv_message=dcp_recv_msg}; +static struct apple_rtkit rtk={.ops=&ops,.cookie=&d}; +static void *ordered_rx(void *arg) +{ + (void)arg; + /* One ordered RX worker must complete DisplayAttributes before AFK. */ + struct apple_rtkit_rx_work *a=calloc(1,sizeof(*a)); assert(a); + a->rtk=&rtk; a->ep=IOMFB_ENDPOINT; a->msg=1; apple_rtkit_rx_work(&a->work); + a=calloc(1,sizeof(*a)); assert(a); + a->rtk=&rtk; a->ep=0x2a; a->msg=7; apple_rtkit_rx_work(&a->work); + return NULL; +} +int main(void) +{ + pthread_t rx,afk,dimensions; + (void)schedule_work; /* The negative baseline does not defer dimensions. */ + d.dcpavservep=&av; connector.hotplug_wq.fn=dimension_work; + d.hw.iomfb_method_profile=DCP_IOMFB_METHODS_H17P_EXTERNAL; + assert(!pthread_mutex_init(&d.dcpavserv.lock.native,NULL)); d.dcpavserv.epoch=1; + assert(!pthread_mutex_init(&d.metadata.lock.native,NULL)); d.metadata.epoch.generation=1; + d.metadata.epoch.connected=true; d.typec_generation=1; d.typec_cable_connected=true; + mutex_lock(&drm.mode_config.mutex); /* DRM get_modes holds this while waiting. */ + assert(!pthread_create(&afk,NULL,worker,&afk_queue)); + assert(!pthread_create(&dimensions,NULL,worker,&dimensions_queue)); + assert(!pthread_mutex_lock(&reply_lock)); + assert(!pthread_create(&rx,NULL,ordered_rx,NULL)); + struct timespec deadline; assert(!clock_gettime(CLOCK_REALTIME,&deadline)); deadline.tv_sec+=2; + int rc=0; + while(!reply_done && !rc) rc=pthread_cond_timedwait(&reply_ready,&reply_lock,&deadline); + if(!reply_done) { fputs("FAIL: ordered RTKit RX blocked before AFK reply while mode_config.mutex held\n",stderr); fflush(stderr); _Exit(1); } + assert(!pthread_mutex_unlock(&reply_lock)); + assert(!dimension_updates); /* Deferred worker cannot take the probe's mutex. */ + mutex_unlock(&drm.mode_config.mutex); + assert(!pthread_join(rx,NULL) && !pthread_join(afk,NULL) && !pthread_join(dimensions,NULL)); + assert(advances==1 && dimension_updates==1 && connector.base.display_info.width_mm==600 && connector.base.display_info.height_mm==340); + assert(d.modes[0].mode.width_mm==600 && d.modes[0].mode.height_mm==340); + assert(d.chunks.length==64 && d.chunks.data); free(d.chunks.data); + assert(!pthread_mutex_destroy(&d.dcpavserv.lock.native)); + assert(!pthread_mutex_destroy(&d.metadata.lock.native)); + puts("PASS: actual ordered RTKit RX -> synchronous IOMFB dimensions/property-start callbacks -> AFK enqueue makes reply progress while DRM mutex held; deferred dimensions update completes after release"); +} +''' + return prefix + '\n'.join(parts) + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py b/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py new file mode 100644 index 00000000000000..7a0666fbe92ef6 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/idle.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual cpuidle driver/registration/selector and tick suspension source replay.""" +import re +def function(text, name): + match=re.search(r'^((?:static\s+)?(?:inline\s+)?(?:__cpuidle\s+|noinstr\s+)?(?:int|void)\s+'+re.escape(name)+r'\([^;{}]*?\))\s*\{',text,re.M) + if not match: raise ValueError(name) + start=match.start(); pos=match.end(); depth=1 + while depth: + if text[pos]=='{': depth+=1 + elif text[pos]=='}': depth-=1 + pos+=1 + return text[start:pos]+'\n' + +MOCKS = r''' +#include +#include +#include +#include +#include +#include +typedef uint64_t u64; typedef int64_t s64; typedef int64_t ktime_t; +#define CONFIG_SUSPEND 1 +#define CPUIDLE_NAME_LEN 16 +#define CPUIDLE_DESC_LEN 32 +#define CPUIDLE_STATE_MAX 10 +#define BIT(n) (1U<<(n)) +#define NSEC_PER_USEC 1000 +#define __cpuidle +#define noinstr +#define THIS_MODULE NULL +#define SYSTEM_SUSPEND 1 +#define SYSTEM_RUNNING 0 +#define TPS(x) x +#define instrumentation_begin() ((void)0) +#define instrumentation_end() ((void)0) +#define raw_spin_lock(x) ((void)0) +#define raw_spin_unlock(x) ((void)0) +#define lock_map_acquire_try(x) ((void)0) +#define lock_map_release(x) ((void)0) +#define trace_suspend_resume(...) ((void)0) +#define WARN_ON(x) ((x)?(warnings++,1):0) +#define WARN_ON_ONCE(x) WARN_ON(x) +#define pr_err(...) ((void)0) +#define for_each_cpu(cpu,mask) for(cpu=0;cpu<(mask)->count;cpu++) +#define per_cpu(v,cpu) v[cpu] +struct platform_device {int dummy;}; +struct module; struct cpumask {int count;}; +struct cpuidle_device; struct cpuidle_driver; +struct cpuidle_device {int cpu; struct cpuidle_state_usage states_usage[10];}; +static struct cpumask possible={6}; static const struct cpumask *cpu_possible_mask=&possible; +static struct cpuidle_device cpuidle_dev[6]; +static int current, irq[6], warnings, wfi, deep, pm, pm_fail, reg_fail=-1, drv_fail; +static int registered[6], unregistered, rcu[6], frozen, stopped[6], online=6; +static int tick_freeze_depth, tick_freeze_lock, tick_freeze_map, system_state; +static int tk_suspend, tk_resume, local_suspend, local_resume, sched_suspend, sched_resume; +static int fanout, faulty_irq, checks; +static uint64_t clock_ns; +static void check(bool b) {checks++; assert(b);} +static int num_online_cpus(void){return online;} +static void sched_clock_suspend(void){sched_suspend++;} +static void sched_clock_resume(void){sched_resume++;} +static void timekeeping_suspend(void){check(tick_freeze_depth==online);tk_suspend++;frozen=1;} +static void timekeeping_resume(void){check(frozen);tk_resume++;frozen=0;} +static void tick_suspend_local(void){local_suspend++;} +static void tick_resume_local(void){local_resume++;} +static void touch_softlockup_watchdog(void){} +static void ct_cpuidle_enter(void){check(irq[current]);rcu[current]++;} +static void ct_cpuidle_exit(void){check(irq[current]);rcu[current]--;} +static bool irqs_disabled(void){return irq[current];} +static void raw_local_irq_disable(void){irq[current]=1;} +static void local_irq_enable(void){check(!frozen);irq[current]=0;} +static void stop_critical_timings(void){stopped[current]++;} +static void start_critical_timings(void){stopped[current]--;} +static ktime_t local_clock_noinstr(void){return clock_ns+=1000;} +static ktime_t ns_to_ktime(ktime_t t){return t;} +static ktime_t ktime_us_delta(ktime_t a,ktime_t b){return (a-b)/1000;} +static u64 div_u64(u64 a,u64 b){return a/b;} +static u64 mul_u32_u32(unsigned a,unsigned b){return (u64)a*b;} +static bool t8140; +static bool of_machine_is_compatible(const char *s){return t8140&&!strcmp(s,"apple,t8140");} +static int cpu_pm_enter(void){pm++;return pm_fail;} +static void cpu_pm_exit(void){pm--;} +static void apple_cpu_deep_wfi(void){deep++;} +static void cpu_do_idle(void); +static int cpuidle_register_driver(struct cpuidle_driver *drv); +static int cpuidle_register_device(struct cpuidle_device *dev){if(dev->cpu==reg_fail)return -5; registered[dev->cpu]=1;return 0;} +static void cpuidle_unregister(struct cpuidle_driver *drv){(void)drv;memset(registered,0,sizeof(registered));unregistered++;} +''' +MAIN = r''' +static int cpuidle_register_driver(struct cpuidle_driver *drv){if(drv_fail)return -5;__cpuidle_driver_init(drv);return 0;} +static void cpu_do_idle(void){ + check(irq[current]);wfi++; + if(fanout && current<5){int previous=current;current++;irq[current]=1; + cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[current],100000);current=previous;} + if(faulty_irq)irq[current]=0; +} +static struct cpuidle_driver original; +static void reset(bool is_t8140){ + apple_idle_driver=original;t8140=is_t8140; current=0; + memset(cpuidle_dev,0,sizeof(cpuidle_dev)); memset(registered,0,sizeof(registered)); + for(int i=0;i<6;i++){irq[i]=1;rcu[i]=stopped[i]=0;} + wfi=deep=pm=pm_fail=warnings=unregistered=0;reg_fail=-1;drv_fail=0; + tick_freeze_depth=frozen=tk_suspend=tk_resume=local_suspend=local_resume=sched_suspend=sched_resume=0; + fanout=faulty_irq=0; +} +int main(void){ + original=apple_idle_driver; + reset(true);check(!apple_cpuidle_probe(NULL)); + check(apple_idle_driver.cpumask==cpu_possible_mask);check(!apple_idle_driver.bctimer); + for(int i=0;i<6;i++)check(registered[i]&&cpuidle_dev[i].cpu==i); + int expected=EXPECT_FIXED?1:0; + check(apple_idle_driver.state_count==(EXPECT_FIXED?2:1)); + check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,0,true)==expected); + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==expected); + check(tick_freeze_depth==0&&!frozen);check(deep==0&&pm==0); + check(local_suspend==expected&&local_resume==expected); + if(EXPECT_FIXED){ + check(apple_idle_driver.states[1].enter==apple_enter_wfi); + check(apple_idle_driver.states[1].enter_s2idle==apple_enter_wfi); + check(apple_idle_driver.states[1].flags==0); + check(apple_idle_driver.states[1].exit_latency_ns==1000); + check(apple_idle_driver.states[1].target_residency_ns==1000); + check(apple_idle_driver.states[1].power_usage==apple_idle_driver.states[0].power_usage); + reset(true);apple_cpuidle_probe(NULL);fanout=1; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==1); + check(wfi==6&&deep==0&&pm==0);check(tk_suspend==1&&tk_resume==1); + check(sched_suspend==1&&sched_resume==1&&local_suspend==5&&local_resume==5); + check(tick_freeze_depth==0&&!frozen); + for(int i=0;i<6;i++)check(cpuidle_dev[i].states_usage[1].s2idle_usage==1&&rcu[i]==0&&stopped[i]==0); + reset(true);apple_cpuidle_probe(NULL);fanout=1;cpuidle_dev[5].states_usage[1].disable=1; + cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000); + check(wfi==5&&tk_suspend==0&&tk_resume==0&&tick_freeze_depth==0); + reset(true);apple_cpuidle_probe(NULL);cpuidle_dev[0].states_usage[1].disable=1; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==0&&wfi==0); + cpuidle_dev[0].states_usage[1].disable=0;apple_idle_driver.states[1].enter_s2idle=NULL; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==0&&wfi==0); + reset(true);apple_cpuidle_probe(NULL); + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],999)==0); + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],1000)==1); + reset(true);apple_cpuidle_probe(NULL);apple_idle_driver.states[1].exit_latency_ns=0; + check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,0,true)==0); + reset(true);apple_cpuidle_probe(NULL);apple_idle_driver.states[1].flags=CPUIDLE_FLAG_TIMER_STOP; + check(find_deepest_state(&apple_idle_driver,&cpuidle_dev[0],100000,CPUIDLE_FLAG_TIMER_STOP,false)==0); + reset(true);apple_cpuidle_probe(NULL);faulty_irq=1; + check(cpuidle_enter_s2idle(&apple_idle_driver,&cpuidle_dev[0],100000)==1); + check(warnings==1&&tick_freeze_depth==0&&rcu[0]==0&&stopped[0]==0); + } + reset(true);apple_cpuidle_probe(NULL); + for(int i=0;i', '') + core = 'drivers/usb/typec/tipd/core.c' + dcp = 'drivers/gpu/drm/apple/dcp.c' + funcs = '\n'.join(function(f, n) for f, n in ( + ('drivers/gpu/drm/apple/iomfb.c', 'iomfb_pause_presentation'), + (dcp, 'dcp_external_report_disconnected'), + (dcp, 'dcp_external_publish_generation'), + (dcp, 'dcp_external_stop_locked'), + (dcp, 'dcp_external_disconnect'), + (dcp, 'dcp_external_stop_work'), + (dcp, 'dcp_external_hotplug'), + (dcp, 'dcp_external_link_locked'), + (dcp, 'dcp_dptx_connect_oob'), + (core, 'cd321x_retry_revalidation'), + (core, 'cd321x_update_work'))) + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; typedef uint64_t u64; +#define guard(x) fake_guard +#define fake_guard(x) ((void)(x)) +#define scoped_guard(x,y) for (int once = ((void)(y), 1); once; once = 0) +#define lockdep_assert_held(x) ((void)(x)) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x,v) ((x) = (v)) +#define container_of(p,t,m) ((t *)((char *)(p) - offsetof(t,m))) +#define to_delayed_work(w) container_of(w,struct delayed_work,work) +#define msecs_to_jiffies(x) (x) +#define IS_REACHABLE(x) 1 +#define CONFIG_DRM 1 +#define IS_ERR(x) false +#define PTR_ERR(x) (-EINVAL) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define CD321X_DEBOUNCE_DELAY_MS 500 +#define TPS_STATUS_PLUG_PRESENT (1U << 0) +#define TPS_STATUS_PLUG_UPSIDE_DOWN (1U << 1) +#define TPS_DATA_STATUS_USB2_CONNECTION (1U << 0) +#define TPS_DATA_STATUS_USB3_CONNECTION (1U << 1) +#define TPS_DATA_STATUS_TBT_CONNECTION (1U << 2) +#define TPS_DATA_STATUS_DP_CONNECTION (1U << 3) +#define TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK (3U << 4) +#define TPS_DATA_STATUS_USB_DATA_ROLE (1U << 6) +#define CD321X_DATA_STATUS_USB4_CONNECTION (1U << 7) +#define CD321X_DATA_STATUS_HPD_LEVEL (1U << 8) +#define TPS_POWER_STATUS_PWROPMODE(x) (x) +#define TPS_STATUS_TO_UPSIDE_DOWN(x) ((x)&TPS_STATUS_PLUG_UPSIDE_DOWN) +#define TPS_STATUS_TO_TYPEC_PORTROLE(x) 0 +#define TPS_STATUS_TO_TYPEC_VCONN(x) 0 +#define TPS_STATUS_TO_TYPEC_DATAROLE(x) 0 +#define TYPEC_STATE_SAFE 0 +#define TYPEC_STATE_USB 1 +#define TYPEC_THUNDERBOLT_SWITCH_OFF 0 +#define TYPEC_ACCESSORY_NONE 0 +#define DCP_HOTPLUG_NOTIFY 1 +struct work_struct { bool pending; }; +struct delayed_work { struct work_struct work; }; +static int system_freezable_wq; +static int schedule_work(struct work_struct *w) { w->pending = true; return 1; } +static void cancel_work_sync(struct work_struct *w) { w->pending = false; } +static void cancel_delayed_work_sync(struct delayed_work *w) { w->work.pending = false; } +#define cancel_delayed_work cancel_delayed_work_sync +static void mod_delayed_work(int q, struct delayed_work *w, int t) +{ (void)q; (void)t; w->work.pending = true; } +struct apple_connector { bool connected, edid_invalidation_pending; }; +struct apple_dcp { + int external_lock, hpd_mutex; + struct { int lock; bool presentation_paused; } iomfb; + struct { bool connected; void *service; } dptxport[1]; + struct apple_connector *connector; + struct delayed_work typec_reconnect_wq, placeholder_edid_wq; + struct work_struct external_stop_work; + bool external_running, external_stopped, external_power_attempted; + bool external_cable, typec_cable_connected, typec_crtc_off, pipe_enabled_h17p; + bool external_physically_absent, external_prepared, placeholder_retried; + int typec_reconnect_tries; + bool external_hotplug_pending, external_hotplug_connected, quiescing; + u64 external_generation, external_rx_generation, external_crtc_generation; + u64 external_stop_generation, typec_generation; + int external_error, mode_state; void *avep; +}; +struct platform_device { struct apple_dcp *data; }; +static struct apple_dcp *platform_get_drvdata(struct platform_device *p) { return p->data; } +static int raw_error, raw_links; +static int dcp_dptx_connect_raw(struct apple_dcp *d, u32 port, bool *waiting) +{ assert(port == 0); *waiting = false; raw_links++; + if (!raw_error) d->dptxport[0].connected = true; + return raw_error; } +static bool dcp_is_typec_output(struct apple_dcp *d) { (void)d; return true; } +static int dcp_dptx_connect(struct apple_dcp *d, u32 port) { (void)d; (void)port; return 0; } +#define DPTX_RECONNECT_DELAY 500 +static int settle_error, power_error, hpd_error, release_error; +static int settles, powers, releases, detaches, notices; +static bool dcp_is_external_h17p(struct apple_dcp *d) { (void)d; return true; } +static unsigned dcp_mode_hotplug(int *mode, bool connected, bool *sink) +{ (void)mode; if (sink) *sink = connected; return 1; } +static void dcp_handle_hotplug_actions(struct apple_dcp *d, unsigned act) +{ (void)d; notices += !!act; } +static int iomfb_queue_settle(struct apple_dcp *d, int t) +{ (void)d; (void)t; settles++; return settle_error; } +static int iomfb_external_power_h17p(struct apple_dcp *d, bool on) +{ (void)d; assert(!on); powers++; return power_error; } +static bool dptx_service_available(void *p) { (void)p; return true; } +static int dptxport_set_hpd(void *p, bool on) +{ (void)p; assert(!on); return hpd_error; } +static int dptxport_release_display(void *p) +{ (void)p; releases++; return release_error; } +static void av_service_disconnect(struct apple_dcp *d) { (void)d; } +static void dpavservep_detach(struct apple_dcp *d) { (void)d; detaches++; } +static void dcp_firmware_metadata_hotplug(struct apple_dcp *d, bool c) { (void)d; (void)c; } +static void dcp_firmware_metadata_connection(struct apple_dcp *d, u64 g, bool c) +{ (void)d; (void)g; (void)c; } +static int dcp_external_disconnect(struct apple_dcp *, bool); +static struct apple_dcp display; +static void drm_connector_oob_hotplug_event(void *node, int status) +{ assert(node); if (!status) dcp_external_disconnect(&display, true); } +#define connector_status_disconnected 0 +#define connector_status_connected 1 +''' + shim = r''' +enum usb_role { USB_ROLE_NONE, USB_ROLE_HOST, USB_ROLE_DEVICE }; +enum typec_pwr_opmode { TYPEC_PWR_MODE_USB, TYPEC_PWR_MODE_PD }; +enum typec_orientation { TYPEC_ORIENTATION_NONE, TYPEC_ORIENTATION_NORMAL, TYPEC_ORIENTATION_REVERSE }; +struct usb_pd_identity { int id; }; +struct typec_mux_state { void *alt, *data; int mode; }; +struct typec_thunderbolt_switch_data { int state; }; +struct typec_partner_desc { bool usb_pd; int accessory; struct usb_pd_identity *identity; }; +struct cd321x_status { u32 status, data_status, status_changed, data_status_changed, pwr_status; + struct usb_pd_identity partner_identity; }; +struct tps6598x { void *partner, *dev, *port, *psy; int lock; u32 data_status; enum usb_role *role_sw; }; +struct cd321x { struct tps6598x tps; struct cd321x_status update_status; + struct cd321x_pm_state pm; struct delayed_work update_work, resume_work; + struct typec_mux_state state; bool state_valid, display_route_active, cur_partner_is_pd; + struct usb_pd_identity cur_partner_identity; void *connector_fwnode, *tbt_switch; }; +static int tbt_error, role_error, safe_error, mode_error; +static int tbts, roles, safes; +static enum usb_role usb_role_switch_get_role(enum usb_role *r) { return *r; } +static int usb_role_switch_set_role(enum usb_role *r, enum usb_role n) +{ roles++; if (role_error) return role_error; *r = n; return 0; } +static int typec_thunderbolt_switch_set(void *p, void *s) +{ (void)p; (void)s; tbts++; return tbt_error; } +static int typec_set_mode(void *p, int s) +{ (void)p; assert(s == TYPEC_STATE_SAFE); safes++; return safe_error; } +static int cd321x_typec_update_mode(struct tps6598x *t, struct cd321x_status *s, bool *busy) +{ (void)t; (void)s; *busy = mode_error == -EBUSY; return mode_error; } +static void typec_unregister_partner(void *p) { (void)p; } +static void *typec_register_partner(void *p, struct typec_partner_desc *d) +{ (void)p; (void)d; return (void *)1; } +static void typec_partner_set_identity(void *p) { (void)p; } +#define typec_set_pwr_opmode(...) ((void)0) +#define typec_set_pwr_role(...) ((void)0) +#define typec_set_vconn_role(...) ((void)0) +#define typec_set_orientation(...) ((void)0) +#define typec_set_data_role(...) ((void)0) +#define power_supply_changed(...) ((void)0) +''' + tests = r''' +static int failures, checks; +#define CHECK(x) do { checks++; if (!(x)) { printf("FAIL line %d: %s\n", __LINE__, #x); failures++; } } while (0) +static struct apple_connector sink; +static enum usb_role role; +static struct cd321x port; +static void init(void) +{ + memset(&display,0,sizeof(display)); memset(&sink,0,sizeof(sink)); memset(&port,0,sizeof(port)); + sink.connected = true; display.connector = &sink; display.external_prepared = true; + raw_error = raw_links = 0; + display.external_running = display.external_power_attempted = true; + display.external_cable = display.typec_cable_connected = true; + display.dptxport[0].connected = display.pipe_enabled_h17p = true; + display.external_generation = display.external_rx_generation = display.external_crtc_generation = 7; + display.typec_generation = 4; + port.tps.partner = (void *)1; port.tps.role_sw = &role; role = USB_ROLE_HOST; + port.pm.phase = CD321X_PM_RUNNING; port.state_valid = true; port.state.mode = 2; + port.display_route_active = true; port.connector_fwnode = (void *)1; + settle_error = power_error = hpd_error = release_error = 0; + tbt_error = role_error = safe_error = mode_error = 0; + settles = powers = releases = detaches = notices = tbts = roles = safes = 0; +} +static void callback(bool connected) +{ + if (!dcp_external_hotplug(&display,connected)) + dcp_mode_hotplug(&display.mode_state,connected,&sink.connected); +} +int main(void) +{ + /* Unplug at each fallible transition with failing DCP power teardown. */ + for (int which = 0; which < 3; which++) { + init(); power_error = -ETIMEDOUT; + if (which == 0) tbt_error = -ETIMEDOUT; + if (which == 1) role_error = -EIO; + if (which == 2) safe_error = -ETIMEDOUT; + cd321x_update_work(&port.update_work.work); + CHECK(!sink.connected); CHECK(!display.external_cable); + CHECK(display.external_error == -ETIMEDOUT); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p); + CHECK(!display.external_stopped && !display.typec_crtc_off); + CHECK(port.display_route_active && !port.state_valid); + CHECK(port.pm.phase == CD321X_PM_REVALIDATE && port.pm.attempts_left == CD321X_RESUME_ATTEMPTS); + CHECK(notices > 0 && releases == 0); + callback(true); CHECK(!sink.connected); + printf("case transition-%d complete\n",which); + } + /* A rejected/nonowner port cannot disconnect the shared owner's sink. */ + for (int which = 0; which < 3; which++) { + init(); port.display_route_active = false; + if (which == 0) tbt_error = -EIO; + if (which == 1) role_error = -EIO; + if (which == 2) safe_error = -EIO; + cd321x_update_work(&port.update_work.work); + CHECK(sink.connected && display.external_cable && !notices); + } + init(); port.display_route_active = false; role = USB_ROLE_NONE; + port.update_status.status = TPS_STATUS_PLUG_PRESENT; + port.update_status.data_status = TPS_DATA_STATUS_DP_CONNECTION | CD321X_DATA_STATUS_HPD_LEVEL; + mode_error = -EBUSY; cd321x_update_work(&port.update_work.work); + CHECK(sink.connected && !notices && !port.display_route_active); + /* Two ports share the connector: busy-port unplug leaves its owner live. */ + init(); + struct cd321x second = port; + enum usb_role second_role = USB_ROLE_NONE; + second.tps.role_sw = &second_role; second.display_route_active = false; + second.update_status.status = TPS_STATUS_PLUG_PRESENT; + second.update_status.data_status = TPS_DATA_STATUS_DP_CONNECTION | CD321X_DATA_STATUS_HPD_LEVEL; + mode_error = -EBUSY; cd321x_update_work(&second.update_work.work); + CHECK(!second.display_route_active && sink.connected && !notices); + second.update_status.status = second.update_status.data_status = 0; + tbt_error = -EIO; cd321x_update_work(&second.update_work.work); + CHECK(port.display_route_active && sink.connected && !notices); + power_error = -EIO; cd321x_update_work(&port.update_work.work); + CHECK(!sink.connected && port.display_route_active); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p); + /* Revalidation is bounded even though the failed hardware remains owned. */ + init(); tbt_error = -EIO; + cd321x_update_work(&port.update_work.work); + for (int i = 0; i < CD321X_RESUME_ATTEMPTS; i++) { + CHECK(cd321x_pm_begin_read(&port.pm)); cd321x_pm_snapshot_ready(&port.pm); + cd321x_update_work(&port.update_work.work); + } + CHECK(port.pm.phase == CD321X_PM_STALE && !port.pm.attempts_left); + CHECK(!sink.connected && port.display_route_active); + /* Stop failure stages preserve all state whose shutdown was not acked. */ + for (int which = 0; which < 4; which++) { + init(); + if (which == 0) settle_error = -EIO; + if (which == 1) power_error = -EIO; + if (which == 2) hpd_error = -EIO; + if (which == 3) release_error = -EIO; + CHECK(dcp_external_disconnect(&display,true) == -EIO); + CHECK(!sink.connected && !display.typec_cable_connected); + CHECK(display.external_generation == 8 && display.external_rx_generation == 8); + CHECK(display.external_error == -EIO && display.dptxport[0].connected); + CHECK(!display.external_stopped && !display.typec_crtc_off); + CHECK(display.pipe_enabled_h17p == (which < 2)); + callback(true); CHECK(!sink.connected); + } + /* An existing error returns before pause: late RX still cannot resurrect. */ + init(); display.external_error = -EIO; + CHECK(dcp_external_disconnect(&display,true) == -EIO); + CHECK(!display.iomfb.presentation_paused); + callback(true); CHECK(!sink.connected); + CHECK(display.dptxport[0].connected && display.pipe_enabled_h17p); + /* A queued callback from the old sink cannot be replayed after unplug. */ + init(); display.external_error = -EIO; + display.iomfb.presentation_paused = true; callback(true); + CHECK(display.external_hotplug_pending); + CHECK(dcp_external_disconnect(&display,true) == -EIO); + iomfb_pause_presentation(&display,false); CHECK(!sink.connected); + /* An old stop generation cannot touch a later session. */ + init(); display.external_stop_generation = 6; + dcp_external_stop_work(&display.external_stop_work); + CHECK(sink.connected && settles == 0 && powers == 0 && releases == 0); + CHECK(dcp_external_stop_locked(&display,6) == 0 && sink.connected); + /* Same-cable CRTC off remains distinct from physical absence. */ + init(); CHECK(dcp_external_disconnect(&display,false) == 0); + CHECK(display.external_cable && display.typec_cable_connected); + CHECK(display.external_generation == 7); + /* Genuine discovery and the paused DPMS generation gate still work. */ + init(); sink.connected = false; callback(true); CHECK(sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_power_attempted = false; callback(true); CHECK(sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_stopped = true; display.external_rx_generation = 8; + callback(true); CHECK(sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_stopped = true; callback(true); CHECK(!sink.connected); + init(); sink.connected = false; display.iomfb.presentation_paused = true; + display.external_error = -EIO; callback(true); CHECK(!sink.connected); + /* Fresh firmware discovery precedes OOB: no second RX is required. */ + init(); sink.connected = false; + display.external_cable = display.typec_cable_connected = false; + display.external_generation = display.external_rx_generation = 37; + display.external_power_attempted = false; display.external_stopped = true; + display.dptxport[0].connected = false; + callback(true); CHECK(sink.connected); CHECK(!display.external_physically_absent); + struct platform_device device = { .data = &display }; + CHECK(dcp_dptx_connect_oob(&device,0) == 0); + CHECK(raw_links == 1 && sink.connected); + CHECK(display.external_generation == 38); + /* Initially paused discovery keeps the original pending-replay behavior. */ + init(); sink.connected = false; display.external_cable = false; + display.external_power_attempted = false; display.iomfb.presentation_paused = true; + callback(true); CHECK(display.external_hotplug_pending); CHECK(!sink.connected); + CHECK(dcp_dptx_connect_oob(&device,0) == 0 && raw_links == 1); + iomfb_pause_presentation(&display,false); CHECK(sink.connected); + /* Physical absence consumes late RX and clears an already queued replay. */ + init(); display.iomfb.presentation_paused = true; callback(true); + CHECK(display.external_hotplug_pending); + CHECK(dcp_external_disconnect(&display,true) == 0); + CHECK(display.external_physically_absent && !display.external_hotplug_pending); + callback(true); CHECK(!sink.connected && !display.external_hotplug_pending); + /* A real next OOB connect removes the tombstone and permits discovery. */ + CHECK(dcp_dptx_connect_oob(&device,0) == 0); + CHECK(!display.external_physically_absent && display.external_cable); + CHECK(display.external_generation == 9 && raw_links == 1); + callback(true); CHECK(sink.connected); + /* Same-cable DPMS does not create a physical-absence tombstone. */ + init(); CHECK(dcp_external_disconnect(&display,false) == 0); + CHECK(!display.external_physically_absent && display.external_cable); + sink.connected = false; callback(true); CHECK(!sink.connected); + CHECK(display.external_hotplug_pending); + iomfb_pause_presentation(&display,false); CHECK(sink.connected); + /* A failed hardware stop remains owned, even after a real OOB reconnect. */ + init(); display.external_error = -EIO; + CHECK(dcp_external_disconnect(&display,true) == -EIO); + CHECK(display.external_physically_absent); callback(true); CHECK(!sink.connected); + CHECK(dcp_dptx_connect_oob(&device,0) == -EIO); + CHECK(!display.external_physically_absent && display.external_error == -EIO); + CHECK(raw_links == 0 && display.dptxport[0].connected && display.pipe_enabled_h17p); + /* Successful physical teardown may release ownership normally. */ + init(); CHECK(dcp_external_disconnect(&display,true) == 0); + CHECK(!sink.connected && !display.dptxport[0].connected); + CHECK(display.external_stopped && display.typec_crtc_off && !display.pipe_enabled_h17p); + printf("%d assertions, %d failures\n", checks, failures); return !!failures; +} +''' + source = prefix + pm + shim + funcs + tests + return source + +def teardown(root): + + files = {n: (root / f).read_text() for n, f in { + 'atc': 'drivers/phy/apple/atc.c', + 'dp': 'drivers/phy/apple/atc-j700-dp.h', + 'phy': 'drivers/phy/phy-core.c', + 'core': 'drivers/usb/dwc3/core.c', + 'apple': 'drivers/usb/dwc3/dwc3-apple.c', + 'state': 'drivers/usb/dwc3/dwc3-apple-state.h', + }.items()} + + + def fn(n, name): + s = files[n] + normalized = n == 'atc' and name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in s + if normalized: + name = 'atcphy_mux_set_locked' + m = re.search(r'^(?:static [^\n]*\b|(?:int|void) )' + name + r'\(', s, re.M) + assert m, (n, name) + result = block(s, m.start()) + return result.replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) if normalized else result + + decls = [] + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', 'atcphy_dp_link_rate'): + decls.append(block(files['atc'], files['atc'].index('enum ' + name + ' {')) + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw', 'atcphy_dp_link_rate_configuration'): + decls.append(block(files['atc'], files['atc'].index('struct ' + name + ' {')) + ';') + decls.append(block(files['state'], files['state'].index('enum dwc3_apple_state {')) + ';') + start = files['atc'].index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + end = files['atc'].index('\n};', start) + 3 + decls.append(files['atc'][start:end]) + start = files['atc'].index('static const struct atcphy_dp_link_rate_configuration dp_lr_config[]') + end = files['atc'].index('\n};', start) + 3 + decls.append(files['atc'][start:end]) + + names = { + 'atc': ('atcphy_pipehandler_lock', 'atcphy_pipehandler_unlock', + 'atcphy_pipehandler_check', 'atcphy_pipehandler_set_mux', + 'atcphy_clear_nonselected_phy_reset', + 'atcphy_configure_pipehandler_usb3_host_t8103', + 'atcphy_configure_pipehandler_usb3_host_t8122', + 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', + 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', + 'atcphy_get_mode_config', 'atcphy_dp_configure', + 'j700_field_update', 'j700_aux_disable', 'atcphy_disable_dp_aux', + 'atcphy_power_off', 'atcphy_power_on', 'atcphy_configure', 'atcphy_usb2_init', 'atcphy_usb3_power_off', + 'atcphy_usb3_set_mode', 'atcphy_usb2_power_off', + '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert', 'atcphy_mux_set'), + 'phy': ('phy_init', 'phy_exit', 'phy_power_on', 'phy_power_off'), + 'core': ('dwc3_phy_power_on', 'dwc3_phy_power_off', 'dwc3_phy_exit', 'dwc3_core_exit'), + 'apple': ('dwc3_apple_core_stop', 'dwc3_apple_core_init', 'dwc3_apple_core_start', + 'dwc3_apple_init', 'dwc3_apple_exit', 'dwc3_usb_role_switch_set'), + } + names['core'] = ('dwc3_phy_power_on', 'dwc3_phy_power_off', 'dwc3_phy_exit', + 'dwc3_core_exit_after_phy_power_off', 'dwc3_core_exit', + 'dwc3_core_remove_common', 'dwc3_core_remove', 'dwc3_core_remove_no_shutdown') + names['atc'] += ('atcphy_dwc3_reset_deassert',) + names['atc'] = tuple(name for name in names['atc'] if name not in ('atcphy_get_mode_config', 'j700_field_update')) + names = {n: tuple(name for name in group if re.search(r'^(?:static [^\n]*\b|(?:int|void) )' + name + r'\(', files[n], re.M)) + for n, group in names.items()} + context = (fn('atc', 'core_read32') + '\n' + fn('atc', 'atcphy_get_mode_config') + '\n' + fn('atc', 'j700_field_update') + '\n' + files['dp'][files['dp'].index('/* Masks are the union'):files['dp'].index('/* Retry only the idle operations')]) + functions = '\n\n'.join(fn(n, name) for n, group in names.items() for name in group) + definitions = [] + lines = iter(files['atc'].splitlines()) + for line in lines: + if not line.startswith('#define '): + continue + definition = [line] + while definition[-1].endswith('\\'): + definition.append(next(lines)) + definitions.append('\n'.join(definition)) + definitions = '\n'.join(definitions) + here = Path(__file__).resolve().parent + fallback = '' if 'dwc3_core_remove_no_shutdown' in names['core'] else '#define dwc3_core_remove_no_shutdown dwc3_core_remove\n' + unit = ((here / 'dwc3/prefix.c').read_text() + '\n' + definitions + '\n' + + '\n'.join(decls) + '\n' + (here / 'dwc3/mocks.c').read_text() + '\n' + + context + '\n' + functions + '\n' + fallback + (here / 'dwc3/cases.c').read_text()) + return unit + +def profile(root): + + base = root / "drivers/gpu/drm/apple" + source = (base / "dcp.c").read_text() + + + def block(marker): + start = source.index(marker) + opening = source.index("{", start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + + + clock = source[source.index("static const struct resource t8140_firmware_clock"): + source.index("static const struct apple_dcp_hw_data apple_dcp_hw_t8140 =")] + profiles = clock + "\n" + "\n".join(block(marker) + ";" for marker in ( + "static const struct apple_dcp_hw_data apple_dcp_hw_t8140 =", + "static const struct apple_dcp_hw_data apple_dcp_hw_t8140_external =", + )) + admission = block("static int dcp_get_disp_regs(") + headers = "\n".join(line for name in ("dcp-external-bw.h", "dcp-link.h") + for line in (base / name).read_text().splitlines() + if not line.startswith("#include")) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; typedef uint64_t u64; +typedef uint32_t __le32; typedef uint64_t __le64; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(condition, ...) _Static_assert(condition, "wire layout") +#define le32_to_cpu(v) (v) +#define cpu_to_le32(v) (v) +#define cpu_to_le64(v) (v) +#define IORESOURCE_MEM 1 +#define MAX_DISP_REGISTERS 7 +#define DCP_FIRMWARE_H17P 3 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DEFINE_RES_MEM(s,n) { .start=(s), .end=(s)+(n)-1 } +#define resource_size(r) ((r)->end-(r)->start+1) +struct resource { u64 start,end; }; +struct device_node { bool named; unsigned count; }; +struct device { struct device_node *of_node; }; +struct platform_device { struct device dev; int num_resources; struct resource resources[10]; }; +#define to_platform_device(d) ((struct platform_device *)(d)) +struct apple_dcp_hw_data { + unsigned num_dptx_ports; bool adopt_live_session; + int iomfb_method_profile,firmware_compat; + const struct resource *firmware_clock; + u32 firmware_display_windows,firmware_scratch,firmware_request; +}; +struct apple_dcp { + struct device *dev; struct apple_dcp_hw_data hw; + struct resource *disp_registers[10],disp_bw_scratch_res,disp_bw_doorbell_res; + u32 disp_bw_scratch_index,disp_bw_scratch_offset,disp_bw_doorbell_index,nr_disp_registers; +}; +static bool of_property_present(struct device_node *n,const char *name) +{ return !strcmp(name,"reg-names") && n->named; } +static int of_property_count_strings(struct device_node *n,const char *name) +{ assert(!strcmp(name,"reg-names"));return n->count+1; } +static int of_property_read_string_index(struct device_node *n,const char *name,int i,const char **v) +{ + static const char *names[]={"coproc","disp-0","disp-1","disp-2","disp-3","disp-4","disp-5"}; + assert(!strcmp(name,"reg-names"));if(i<0 || (unsigned)i>n->count)return -EINVAL; + *v=names[i];return 0; +} +static struct resource *platform_get_resource(struct platform_device *p,int type,int i) +{ assert(type==IORESOURCE_MEM);return i>=1 && inum_resources?&p->resources[i-1]:NULL; } +static struct resource *platform_get_resource_byname(struct platform_device *p,int type,const char *name) +{ int i=-1;assert(sscanf(name,"disp-%d",&i)==1);return platform_get_resource(p,type,i+1); } +static int dcp_get_bw_scratch_reg(struct apple_dcp *p,u32 count) { (void)p;(void)count;return -ENOENT; } +static int dcp_get_bw_doorbell_reg(struct apple_dcp *p,u32 count) { (void)p;(void)count;return -ENOENT; } +''' + tests = r''' +static struct platform_device provider; +static struct device_node node; +static struct apple_dcp d; +static void reset(bool external,bool named) +{ + node=(struct device_node){.named=named,.count=5}; + provider=(struct platform_device){.dev.of_node=&node,.num_resources=6}; + for(unsigned i=0;i<5;i++)provider.resources[i]=(struct resource)DEFINE_RES_MEM(0x1000ULL*i,0x1000); + provider.resources[4]=t8140_firmware_clock; + d=(struct apple_dcp){.dev=&provider.dev,.hw=external?apple_dcp_hw_t8140_external:apple_dcp_hw_t8140}; +} +int main(void) +{ + const u32 fw[]={26,6,2},unknown[]={0,0,0}; + assert(apple_dcp_external_bw_valid(0xbc000,0x28000,0x70000)); + assert(!apple_dcp_external_bw_valid(0xbc000,0x28002,0x70000)); + struct apple_dcp_h17p_clock_request clock={.clock_id=APPLE_DCP_H17P_CLOCK_ID_154};u64 rate; + assert(!apple_dcp_h17p_clock_rate(&clock,123,456,&rate) && rate==123); + assert(apple_dcp_external_firmware_supported(fw,3,fw,3)); + assert(!apple_dcp_external_firmware_supported(NULL,3,fw,3)); + assert(!apple_dcp_external_firmware_supported(fw,3,NULL,3)); + assert(!apple_dcp_external_firmware_supported(unknown,3,fw,3)); + assert(apple_dcp_hw_t8140.num_dptx_ports==0 && apple_dcp_hw_t8140.adopt_live_session); + assert(apple_dcp_hw_t8140.firmware_display_windows==5); + assert(apple_dcp_hw_t8140.firmware_scratch==0x20000 && apple_dcp_hw_t8140.firmware_request==0x68000); + assert(apple_dcp_hw_t8140_external.num_dptx_ports==1 && !apple_dcp_hw_t8140_external.adopt_live_session); + assert(apple_dcp_hw_t8140_external.firmware_display_windows==4); + assert(apple_dcp_hw_t8140_external.firmware_scratch==0x28000 && apple_dcp_hw_t8140_external.firmware_request==0x70000); + for(unsigned named=0;named<2;named++) { + reset(true,named);assert(dcp_get_disp_regs(&d)==0); + assert(d.nr_disp_registers==5 && d.disp_bw_scratch_index==4); + assert(d.disp_bw_scratch_res.start+d.disp_bw_scratch_offset==0x302828000ULL); + assert(d.disp_bw_doorbell_res.start==0x302870000ULL && resource_size(&d.disp_bw_doorbell_res)==4); + struct apple_dcp_h17p_rt_bw_request request={0};struct apple_dcp_h17p_rt_bw_reply reply; + assert(!apple_dcp_h17p_rt_bw_encode_reply(&request,&reply,d.disp_bw_scratch_res.start+d.disp_bw_scratch_offset,d.disp_bw_doorbell_res.start)); + assert(reply.config.scratch==0x302828000ULL && reply.config.clock_request==0x302870000ULL && !reply.status); + request.config_null=1;assert(apple_dcp_h17p_rt_bw_encode_reply(&request,&reply,0,0)==-EINVAL); + reset(false,named);assert(dcp_get_disp_regs(&d)==0 && d.nr_disp_registers==6); + assert(d.disp_bw_scratch_offset==0x20000 && d.disp_bw_doorbell_res.start==0x302868000ULL); + reset(true,named);provider.resources[4].start+=4;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);provider.resources[4].end-=4;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);node.count=4;provider.num_resources=5;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);d.hw.firmware_scratch=0xbc000;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);d.hw.firmware_request=0xbc000;assert(dcp_get_disp_regs(&d)==-EINVAL); + reset(true,named);d.hw.firmware_display_windows=0;assert(dcp_get_disp_regs(&d)==-EINVAL); + } + puts("PASS: actual internal/external profiles, resource admission and D003 bandwidth reply"); +} +''' + return prefix + headers + "\n" + profiles + admission + tests + +def firmware(root): + + source = (root / "drivers/gpu/drm/apple/iomfb_h17p.c").read_text() + body = source[source.index("struct dcp_h17p_provider_request {"): + source.index("static bool trampoline_luma_span_h17p")] + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint32_t __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(cond) _Static_assert(cond, "") +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define sizeof_field(t, f) sizeof(((t *)0)->f) +#define le32_to_cpu(x) (x) +#define cpu_to_le32(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define DCP_IOMFB_METHODS_H17G 1 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define trace_iomfb_callback(d, t, n) ((void)(d), (void)(t)) +#define dev_err(...) ((void)0) +#define dev_warn_once(...) ((void)0) +struct device_node { bool internal, external, present; int length; }; +struct device { struct device_node *of_node; }; +struct apple_dcp { struct { int iomfb_method_profile; } hw; struct device *dev; bool crashed; }; +static bool j700 = true; +static bool of_machine_is_compatible(const char *name) +{ + assert(!strcmp(name, "apple,j700")); + return j700; +} +static bool of_device_is_compatible(struct device_node *n, const char *name) +{ + return (!strcmp(name, "apple,t8140-dcp") && n->internal) || + (!strcmp(name, "apple,t8140-dcpext") && n->external); +} +static bool of_property_present(struct device_node *n, const char *name) +{ + (void)name; + return n->present; +} +static int of_property_count_u8_elems(struct device_node *n, const char *name) +{ + (void)name; + return n->present ? n->length : -EINVAL; +} +static int of_property_read_u8_array(struct device_node *n, const char *name, void *p, unsigned int len) +{ + (void)n; (void)name; + memset(p, 0x5a, len); + return 0; +} +static const void *memchr_inv(const void *p, int value, size_t size) +{ + const u8 *b = p; + for (size_t i = 0; i < size; i++) + if (b[i] != value) + return b + i; + return NULL; +} +''' + suffix = r''' +int main(void) +{ + struct device_node node = { .external = true }; + struct device dev = { .of_node = &node }; + struct apple_dcp dcp = { .dev = &dev, .hw.iomfb_method_profile = 2 }; + struct dcp_h17p_provider_request req = { .scope = { 'V', 'O', 'R', 'P' }, .capacity = 0xc00 }; + struct dcp_h17p_provider_reply reply; + unsigned int cases = 0; + for (unsigned int i = 0; i < ARRAY_SIZE(provider_properties); i++) { + memset(&reply, 0xff, sizeof(reply)); + strcpy(req.name, provider_properties[i].name); + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req)); + assert(!dcp.crashed && reply.length == provider_properties[i].length); + if (reply.length) + assert(!memcmp(reply.data, provider_properties[i].legacy_data, reply.length)); + cases++; + } + strcpy(req.name, "power-lut-data-x"); + j700 = false; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + j700 = true; dcp.crashed = false; dcp.hw.iomfb_method_profile = 3; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + node.external = false; node.internal = true; dcp.crashed = false; + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && !dcp.crashed); + node.internal = false; node.external = true; node.present = true; node.length = 3; + dcp.hw.iomfb_method_profile = 2; dcp.crashed = false; + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && dcp.crashed); + node.length = 4; dcp.crashed = false; + assert(trampoline_provider_property_h17p(&dcp, 400, &reply, &req) && !dcp.crashed); + assert(reply.length == 4 && reply.data[0] == 0x5a); + strcpy(req.name, "unqualified-property"); + assert(!trampoline_provider_property_h17p(&dcp, 400, &reply, &req)); + printf("PASS %u provider payloads and scope/length rejection cases\n", cases); + return 0; +} +''' + return prefix + body + suffix + +def readiness(root): + + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + drm = (root / 'drivers/gpu/drm/apple/apple_drv.c').read_text() + + + def function(source, marker): + start = source.index(marker) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + + # This slice is the production probe's publication boundary, including its + # failure exits and prepared-before-component_add ordering, not a copied model. + probe = function(dcp, 'static int dcp_platform_probe(') + publish = probe[probe.rindex('\tif (dcp->hw.iomfb_method_profile ==\n\t DCP_IOMFB_METHODS_H17P_EXTERNAL) {'):] + production = '\n'.join(( + function(dcp, 'bool dcp_h17p_external_supported('), + function(dcp, 'bool dcp_h17p_external_ready('), + function(drm, 'static bool apple_dcp_usable('), + function(drm, 'static int add_dcp_components('), + function(drm, 'static int apple_drm_init_dcp('), + 'static int publish_provider(struct platform_device *pdev, struct apple_dcp *dcp)\n' + '{\n\tstruct device *dev = &pdev->dev;\n\tint ret;\n' + publish, + )) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define IS_ENABLED(x) 0 +#define MAX_COPROCESSORS 4 +#define DL_FLAG_AUTOREMOVE_SUPPLIER 0 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define dev_warn(...) ((void)0) +#define msleep(x) ((void)(x)) +#define component_compare_of 0 +#define apple_dcp_id_tbl 0 +static int dcp_comp_ops; +struct device_node { bool available, external, phys; unsigned int index; u32 firmware[3], compat[3]; int firmware_cells, compat_cells; }; +struct device { struct device_node *of_node; void *data; }; +struct platform_device { struct device dev; bool bound, registered; }; +struct apple_dcp { + bool external_prepared, quiescing, crashed; + struct { int iomfb_method_profile; } hw; +}; +struct drm_device { int unused; }; +struct drm_encoder { int unused; }; +struct apple_drm_private { struct drm_device drm; }; +struct component_match { bool includes[2]; }; +static struct device_node nodes[2]; +static struct platform_device devices[2]; +static struct apple_dcp dcps[2]; +static struct apple_drm_private priv; +static struct device master = { .data = &priv }; +static struct component_match matches; +static bool machine_supported, absent_external; +static unsigned int references, released, node_puts, probes[2], bind_attempts, disables; +static int preflight_error, external_probe_error, bind_result; +static void try_bind(void); +static bool of_machine_is_compatible(const char *s) +{ assert(!strcmp(s,"apple,j700")); return machine_supported; } +static int of_property_read_variable_u32_array(struct device_node *n,const char *s,u32 *v,int min,int max) +{ + assert(min==3 && max==3); + bool fw=!strcmp(s,"apple,firmware-version"); + assert(fw || !strcmp(s,"apple,firmware-compat")); + int count=fw?n->firmware_cells:n->compat_cells; + if(count!=3) return -EINVAL; + memcpy(v,fw?n->firmware:n->compat,3*sizeof(*v)); return count; +} +static bool of_device_is_available(struct device_node *n) { return n->available; } +static bool of_device_is_compatible(struct device_node *n, const char *s) +{ + return n->external && (!strcmp(s, "apple,t8140-dcpext") || !strcmp(s, "apple,dcpext")); +} +static bool of_property_present(struct device_node *n, const char *s) +{ assert(!strcmp(s, "phys")); return n->phys; } +#define for_each_matching_node(np, table) \ + for (unsigned int node_i = 0; node_i < 2 && ((np) = &nodes[node_i]); node_i++) +#define for_each_endpoint_of_node(np, endpoint) \ + for ((endpoint) = NULL; (endpoint) != NULL; (endpoint) = NULL) +static struct device_node *of_graph_get_remote_port_parent(struct device_node *n) +{ return n; } +static void of_node_put(struct device_node *n) { (void)n; node_puts++; } +static struct platform_device *of_find_device_by_node(struct device_node *n) +{ + if (n->external && absent_external) + return NULL; + references++; + return &devices[n->index]; +} +static void *platform_get_drvdata(struct platform_device *p) { return p->dev.data; } +static void *dev_get_drvdata(struct device *d) { return d->data; } +static void put_device(struct device *d) { (void)d; released++; } +/* Deliberately provided: using this during synchronous bind is a regression. */ +#define device_is_bound(dev) (devices[(dev)->of_node->index].bound) +static void drm_of_component_match_add(struct device *d, struct component_match **m, + int compare, struct device_node *n) +{ (void)d; (void)compare; *m = &matches; matches.includes[n->index] = true; } +static void device_link_add(struct device *a, struct device *b, int flags) +{ (void)a; (void)b; (void)flags; } +static int apple_probe_per_dcp(struct device *d, struct drm_device *drm, + struct platform_device *p, int index, bool external, + struct drm_encoder **encoder, u32 *mask) +{ + (void)d; (void)drm; (void)index; (void)encoder; (void)mask; + assert(external == p->dev.of_node->external); + probes[p->dev.of_node->index]++; + return external ? external_probe_error : 0; +} +static int apple_probe_typec_ports(struct drm_device *d, struct platform_device **p, + u32 *masks, int count) +{ (void)d; (void)p; (void)masks; (void)count; return 0; } +static int dcp_start(struct platform_device *p) { (void)p; return 0; } +static int dcp_wait_ready(struct platform_device *p, u64 timeout) +{ (void)p; (void)timeout; return 0; } +static u64 get_jiffies_64(void) { return 0; } +static u64 msecs_to_jiffies(unsigned int ms) { return ms; } +static bool time_after_eq64(u64 a, u64 b) { return a >= b; } +static int dcp_comp_bind(struct device *d, void *master_data, void *data) +{ (void)d; (void)master_data; (void)data; return preflight_error; } +static void dcp_disable_typec_work(struct apple_dcp *d, bool quiesce) +{ disables++; d->quiescing = quiesce; } +static int component_add(struct device *d, const void *ops) +{ + (void)ops; + struct platform_device *p = &devices[d->of_node->index]; + assert(!p->bound); + p->registered = true; + try_bind(); /* Linux component_add may bind the master before probe returns. */ + return 0; +} +''' + suffix = r''' +static void try_bind(void) +{ + for (unsigned int i = 0; i < 2; i++) + if (matches.includes[i] && !devices[i].registered) + return; + bind_attempts++; + bind_result = apple_drm_init_dcp(&master); +} +static void initial(bool supported_firmware) +{ + memset(nodes, 0, sizeof(nodes)); + memset(devices, 0, sizeof(devices)); + memset(dcps, 0, sizeof(dcps)); + memset(&matches, 0, sizeof(matches)); + memset(probes, 0, sizeof(probes)); + for (unsigned int i = 0; i < 2; i++) { + nodes[i] = (struct device_node){ .available = true, .external = i == 1, .index = i }; + devices[i].dev = (struct device){ .of_node = &nodes[i], .data = &dcps[i] }; + } + dcps[1].hw.iomfb_method_profile = DCP_IOMFB_METHODS_H17P_EXTERNAL; + devices[0].registered = devices[0].bound = true; + machine_supported = true; + nodes[1].firmware_cells=nodes[1].compat_cells=3; + if(supported_firmware) { + u32 version[]={26,6,2}; + memcpy(nodes[1].firmware,version,sizeof(version)); + memcpy(nodes[1].compat,version,sizeof(version)); + } + absent_external = false; + references = released = node_puts = bind_attempts = disables = 0; + preflight_error = external_probe_error = bind_result = 0; +} +static int match_providers(void) +{ + struct component_match *match = NULL; + return add_dcp_components(&master, &match); +} +int main(void) +{ + assert(apple_dcp_external_bw_valid(0xbc000,0x28000,0x70000)); + assert(apple_dcp_external_clock_window(0x302800000ULL,0xbc000,0x302800000ULL,0xbc000)); + /* No command line state: exact J700 firmware selects the external profile. */ + initial(true); assert(dcp_h17p_external_supported(&nodes[1])); + machine_supported=false; + assert(!dcp_h17p_external_supported(&nodes[1]) && match_providers()==1); + for(unsigned i=0;i<3;i++) { + initial(true); nodes[1].firmware[i]++; + assert(!dcp_h17p_external_supported(&nodes[1]) && match_providers()==1); + initial(true); nodes[1].compat[i]++; + assert(!dcp_h17p_external_supported(&nodes[1]) && match_providers()==1); + } + for(int count=-1;count<=5;count++) if(count!=3) { + initial(true); nodes[1].firmware_cells=count; + assert(!dcp_h17p_external_supported(&nodes[1])); + initial(true); nodes[1].compat_cells=count; + assert(!dcp_h17p_external_supported(&nodes[1])); + } + + /* Unknown external firmware preserves the normal internal provider. */ + initial(false); + assert(match_providers() == 1 && matches.includes[0] && !matches.includes[1]); + assert(!dcp_h17p_external_ready(&nodes[1]) && references == 0); + try_bind(); + assert(bind_attempts == 1 && bind_result == 0 && probes[0] == 1 && !probes[1]); + + /* Delayed external provider remains in the match; the master must wait. */ + initial(true); + assert(match_providers() == 2 && matches.includes[1]); + try_bind(); + assert(bind_attempts == 0 && !probes[0] && !probes[1]); + /* Last provider is prepared, but driver core bound is still false here. */ + assert(publish_provider(&devices[1], &dcps[1]) == 0); + assert(dcps[1].external_prepared && !devices[1].bound); + assert(bind_attempts == 1 && bind_result == 0 && probes[0] == 1 && probes[1] == 1); + devices[1].bound = true; + + /* Permanent preflight failure cannot silently drop a requested provider. */ + initial(true); + assert(match_providers() == 2); + preflight_error = -EIO; + assert(publish_provider(&devices[1], &dcps[1]) == -EIO); + assert(!devices[1].registered && !dcps[1].external_prepared && disables == 1); + try_bind(); + assert(matches.includes[1] && bind_attempts == 0 && !probes[0]); + + /* An unavailable DT node is excluded; a matched but absent device is fatal. */ + initial(true); + nodes[1].available = false; + assert(!apple_dcp_usable(&nodes[1]) && match_providers() == 1); + initial(true); + absent_external = true; + assert(match_providers() == 2); + assert(apple_drm_init_dcp(&master) == -ENODEV && !probes[1]); + + /* Readiness rejects every unavailable provider state and releases lookup refs. */ + for (unsigned int state = 0; state < 4; state++) { + initial(true); + dcps[1].external_prepared = true; + if (state == 0) devices[1].dev.data = NULL; + if (state == 1) dcps[1].external_prepared = false; + if (state == 2) dcps[1].quiescing = true; + if (state == 3) dcps[1].crashed = true; + assert(!dcp_h17p_external_ready(&nodes[1])); + assert(references == 1 && released == 1); + assert(apple_drm_init_dcp(&master) == -ENODEV && !probes[1]); + } + initial(true); + dcps[1].external_prepared = true; + external_probe_error = -EIO; + assert(apple_drm_init_dcp(&master) == -EIO); + assert(probes[1] == 1 && node_puts == 1); + puts("PASS: automatic hardware/firmware matching, delayed/preflight-failed providers, synchronous publication, readiness rejection"); + return 0; +} +''' + + header = (root / 'drivers/gpu/drm/apple/dcp-external-bw.h').read_text().replace('#include ', '') + return prefix + header + production + suffix + +def vrr(root): + + source = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + start = source.index('int dcp_crtc_atomic_check(') + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + check = source[start:end] + prefix = r""" +#undef NDEBUG +#include +#include +#include +#include +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define READ_ONCE(x) (x) +#define dev_dbg(...) ((void)0) +struct apple_connector { bool connected; }; +struct apple_dcp { + bool crashed; + struct { int iomfb_method_profile; } hw; + struct { bool valid; } mode_state; + struct apple_connector *connector; + void *dev; +}; +struct platform_device { struct apple_dcp *data; }; +struct drm_crtc { struct platform_device *dcp; }; +struct drm_crtc_state { bool vrr_enabled, modeset; }; +struct drm_atomic_state { struct drm_crtc_state crtc; }; +#define to_apple_crtc(x) (x) +#define platform_get_drvdata(x) ((x)->data) +static struct drm_crtc_state *drm_atomic_get_new_crtc_state( + struct drm_atomic_state *s, struct drm_crtc *c) +{ (void)c; return &s->crtc; } +static bool drm_atomic_crtc_needs_modeset(struct drm_crtc_state *s) +{ return s->modeset; } +""" + suffix = r""" +int main(void) +{ + struct apple_connector connector = { .connected = true }; + struct apple_dcp dcp = { .connector = &connector }; + struct platform_device pdev = { .data = &dcp }; + struct drm_crtc crtc = { .dcp = &pdev }; + struct drm_atomic_state state = {0}; + unsigned int cases = 0; + /* Check initial modesets, plane-only commits and link recovery states. */ + for (unsigned int profile = 0; profile < 3; profile++) + for (unsigned int valid = 0; valid < 2; valid++) + for (unsigned int modeset = 0; modeset < 2; modeset++) + for (unsigned int link = 0; link < 3; link++) + for (unsigned int vrr = 0; vrr < 2; vrr++) { + dcp.hw.iomfb_method_profile = profile; + dcp.mode_state.valid = valid; + state.crtc.modeset = modeset; + state.crtc.vrr_enabled = vrr; + dcp.connector = link == 2 ? NULL : &connector; + connector.connected = link == 1; + int expected = profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && vrr ? -EINVAL : 0; + int actual = dcp_crtc_atomic_check(&crtc, &state); + if (actual != expected) { + fprintf(stderr, "profile=%u valid=%u modeset=%u link=%u vrr=%u: got %d expected %d\n", + profile, valid, modeset, link, vrr, actual, expected); + return 1; + } + cases++; + } + dcp.crashed = true; + state.crtc.vrr_enabled = false; + assert(dcp_crtc_atomic_check(&crtc, &state) == -EINVAL); + printf("PASS: %u production atomic VRR admission cases and crashed-device rejection\n", cases); + return 0; +} +""" + return prefix + check + suffix + + +def power_reply(root): + """Actual external power callback validation with host completion/refcount mocks.""" + source = (root / 'drivers/gpu/drm/apple/iomfb_h17p.c').read_text() + production = block(source, 'struct dcp_external_power_cookie {') + ';\n' + production += block(source, 'static void iomfb_external_power_complete(') + production += block(source, 'static bool iomfb_external_reply_size(') + prefix = r""" +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; typedef uint64_t u64; +#define WRITE_ONCE(p,v) ((p)=(v)) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +struct completion { bool done; }; +struct kref { unsigned count; }; +struct dcp_wait_cookie { struct kref refcount; struct completion done; u32 status; }; +struct dcp_packet_header { char tag[4]; u32 in_len,out_len; }; +struct dcp_set_power_state_req_h17p { u64 unklong; u32 unkint; unsigned char null,padding[3]; } __attribute__((packed)); +struct dcp_set_power_state_resp { u32 unknown,ret; } __attribute__((packed)); +struct apple_dcp { int unused; }; +static unsigned released; +static u64 get_unaligned_le64(const void *p) { u64 v; memcpy(&v,p,sizeof(v)); return v; } +static void complete(struct completion *c) { assert(!c->done); c->done=true; } +static void release_wait_cookie(struct kref *r) { assert(!r->count); released++; } +static void kref_put(struct kref *r,void (*release)(struct kref *)) +{ assert(r->count); if (!--r->count) release(r); } +""" + controls = r""" +int main(void) +{ + struct apple_dcp d={}; + struct { struct dcp_packet_header h; struct dcp_set_power_state_req_h17p q; unsigned char reply[8]; } packet={}; + for(unsigned on=0;on<2;on++) { + const u64 replies[]={0,1,0x123400000000ULL}; + for(unsigned i=0;i<3;i++) { + packet.h.in_len=sizeof(packet.q);packet.h.out_len=sizeof(packet.reply);memcpy(packet.reply,&replies[i],sizeof(packet.reply)); + struct dcp_external_power_cookie c={.on=on,.wait.refcount.count=2}; + assert(iomfb_external_reply_size(&packet.reply,sizeof(packet.q),sizeof(packet.reply))); + iomfb_external_power_complete(&d,&packet.reply,&c); + assert(c.wait.done.done && c.wait.refcount.count==1); + assert((int)c.wait.status==(replies[i]==on?0:-EREMOTEIO)); + kref_put(&c.wait.refcount,release_wait_cookie); + } + } + for(unsigned fault=0;fault<3;fault++) { + packet.h.in_len=sizeof(packet.q);packet.h.out_len=sizeof(packet.reply); + void *out=&packet.reply; + if(fault==0) out=NULL; + if(fault==1) packet.h.in_len--; + if(fault==2) packet.h.out_len--; + struct dcp_external_power_cookie c={.wait.refcount.count=2}; + assert(!iomfb_external_reply_size(out,sizeof(packet.q),sizeof(packet.reply))); + iomfb_external_power_complete(&d,out,&c); + assert(c.wait.done.done && c.wait.refcount.count==1 && (int)c.wait.status==-EPROTO); + kref_put(&c.wait.refcount,release_wait_cookie); + } + assert(released==9); + puts("PASS: actual exact ON/OFF echoes, nonzero status and null/truncated reply rejection preserve callback completion and references"); +} +""" + return prefix + production + controls + + +def dispatcher(root): + source = (root / "drivers/gpu/drm/apple/iomfb.c").read_text() + start = source.index("static bool dcp_external_zero_reply(") + end = source.index("static void dcpep_handle_ack(", start) + prefix = r""" +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +#define DCP_FIRMWARE_H17P 1 +#define DCP_IOMFB_METHODS_H17P_EXTERNAL 2 +#define DCP_IOMFB_METHODS_H17G 3 +#define DCP_MAX_CALL_DEPTH 8 +#define IOMFB_MAX_CB 600 +#define DCP_PACKET_ALIGNMENT 64 +#define ALIGN(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define dev_err(...) ((void)dev) +#define dev_info(...) ((void)dev) +#define dev_warn(...) ((void)dev) +enum dcp_context_id { DCP_CONTEXT_CB = 0, DCP_CONTEXT_ASYNC = 3 }; +struct device_node { bool external; }; +struct device { struct device_node *of_node; }; +struct dcp_packet_header { char tag[4]; u32 in_len, out_len; }; +struct dcp_channel { unsigned depth; void *output[8]; unsigned end[8]; }; +struct apple_dcp { + struct { int iomfb_method_profile; } hw; + struct device *dev; + int fw_compat; + bool crashed; + bool (**cb_handlers)(struct apple_dcp *, int, void *, void *); + unsigned long unknown_callbacks[16]; + int start_done; + struct dcp_channel channel; +}; +static bool machine; +static unsigned acks, case_tag; +static struct dcp_channel *dcp_get_channel(struct apple_dcp *d, enum dcp_context_id c) +{ (void)c; return &d->channel; } +static int dcp_parse_tag(char *p) { (void)p; return case_tag; } +static bool of_machine_is_compatible(const char *n) { (void)n; return machine; } +static bool of_device_is_compatible(struct device_node *n, const char *s) +{ (void)s; return n->external; } +static bool iomfb_check_callback_h17p(struct apple_dcp *d, int t, u32 i, u32 o) +{ (void)d; (void)t; (void)i; (void)o; return true; } +static bool iomfb_uses_queue(struct apple_dcp *d) { (void)d; return false; } +static bool test_and_set_bit(int t, unsigned long *b) { (void)t; (void)b; return false; } +static void complete(int *p) { *p = 1; } +static u8 dcp_push_depth(unsigned *d) { return (*d)++; } +static void dcp_ack(struct apple_dcp *d, enum dcp_context_id c) +{ (void)c; acks++; d->channel.depth--; } +""" + suffix = r""" +int main(void) +{ + struct device_node node; + struct device dev = { &node }; + bool (*handlers[IOMFB_MAX_CB])(struct apple_dcp *, int, void *, void *) = {}; + unsigned cases = 0; + for (unsigned scope = 0; scope < 8; scope++) + for (unsigned tag = 133; tag <= 135; tag++) + for (unsigned context = 0; context < 8; context++) + for (unsigned in = 0; in <= 8; in++) + for (unsigned out = 0; out <= 8; out++) { + struct apple_dcp d = { + .dev = &dev, .fw_compat = 1, + .cb_handlers = handlers, .hw.iomfb_method_profile = 2, + }; + union { uint64_t align; unsigned char data[64]; } packet; + struct dcp_packet_header *h = (void *)packet.data; + memset(&packet, 0xa5, sizeof(packet)); + h->in_len = in; h->out_len = out; + machine = scope & 1; node.external = scope & 2; + if (!(scope & 4)) + d.hw.iomfb_method_profile = 0; + case_tag = tag; acks = 0; + dcpep_handle_cb(&d, context, h, sizeof(*h) + in + out, 0); + bool accepted = scope == 7 && tag == 134 && context == 3 && !in && out == 4; + if (scope & 4) { + assert(acks == accepted && d.crashed != accepted); + for (unsigned i = 0; i < out; i++) + assert(packet.data[sizeof(*h) + in + i] == (accepted ? 0 : 0xa5)); + } else { + assert(acks == 1 && !d.crashed); /* Existing legacy behavior. */ + } + assert(packet.data[sizeof(*h) + in + out] == 0xa5); + cases++; + } + return cases != 15552; +} +""" + + return prefix + source[start:end] + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py new file mode 100644 index 00000000000000..fbb1951c0f0b1c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/phy.py @@ -0,0 +1,2399 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual PHY parking, failure/retry, masked context and OFF controls.""" +from pathlib import Path +import re +from common import block, source_defines + +def parking(root): + + atc = (root / 'drivers/phy/apple/atc.c').read_text() + dp = (root / 'drivers/phy/apple/atc-j700-dp.h').read_text() + + + def function(source, name): + if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: + marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() + return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) + + + # Use the production mode table and wrappers too, so SAFE mapping and fast paths + # are tested rather than mirrored by a second implementation. + parts = source_defines(atc) + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', + 'atcphy_dp_link_rate'): + parts.append(block(atc, 'enum ' + name + ' {') + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw'): + parts.append(block(atc, 'struct ' + name + ' {') + ';') + parts.append(r''' +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *lpdptx, *pipehandler, *usb2phy; } regs; + struct reset_controller_dev rcdev; + void *dev; + int lock, dp_link_rate; + bool fixed_usb2, host_active, swap_lanes, ss_tunables, dp_tunables; + bool aux_power_requested, mode_dirty, dp_spread_spectrum, dp_presets_valid[5]; + u32 dp_presets[5][16]; + unsigned int dp_idle_pmas, dp_owned_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +''') + start = atc.index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + parts.append(atc[start:atc.index('\n};', start) + 3]) + parts.append(r''' +struct phy_configure_opts_dp { + unsigned int link_rate, lanes, voltage[4], pre[4]; + bool set_rate, set_lanes, set_voltages, ssc; +}; +static u32 registers[0x30000 / 4]; +static u32 pipe_regs[0x1000 / 4], usb2_regs[0x1000 / 4]; +static unsigned int polls, fail_poll, writes, usb2_on, usb2_writes, reset_writes, mux_writes; +static bool stuck_ack; +static bool hold_unlock_ack; +static const char *poll_caller[256], *fail_caller; +static bool bist_status_stuck; +static unsigned int bist_faults; +static bool require_parked, expected_j700_swap; +static u32 readl(void *address) { return *(u32 *)address; } +static void core_mask32(struct apple_atcphy *p, u32 offset, u32 mask, u32 value) +{ + (void)p; + assert(offset < sizeof(registers) && offset % 4 == 0); + if (require_parked) { + assert(p->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(FIELD_GET(p->hw->pipe_mux_data, pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) == + PIPEHANDLER_MUX_CTRL_DATA_DUMMY); + assert(FIELD_GET(p->hw->pipe_mux_clk, pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) == + PIPEHANDLER_MUX_CTRL_CLK_DUMMY); + } + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + writes++; + if (offset == ATCPHY_POWER_CTRL) + registers[ATCPHY_POWER_STAT / 4] = registers[offset / 4]; + if (offset == 0x2000) { + if (value == 0x10000001) { + registers[offset / 4] |= BIT(1); + registers[0x7034 / 4] |= BIT(3); + } else { + registers[offset / 4] &= ~BIT(1); + registers[0x7034 / 4] &= ~BIT(3); + } + } +} +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +static void mask32(void *p, u32 m, u32 v) +{ + *(u32 *)p = (*(u32 *)p & ~m) | v; + if (p == (void *)&pipe_regs[PIPEHANDLER_LOCK_REQ / 4] && !stuck_ack) + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = hold_unlock_ack && + !(*(u32 *)p & PIPEHANDLER_LOCK_EN) ? PIPEHANDLER_LOCK_EN : + *(u32 *)p & PIPEHANDLER_LOCK_EN; + if (p == (void *)&pipe_regs[PIPEHANDLER_AON_GEN / 4]) reset_writes++; + if (p == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) mux_writes++; + if ((uintptr_t)p >= (uintptr_t)usb2_regs && + (uintptr_t)p < (uintptr_t)usb2_regs + sizeof(usb2_regs)) usb2_writes++; +} +static void writel(u32 value, void *address) { mask32(address, ~0U, value); } +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = readl(address); assert(polls < ARRAY_SIZE(poll_caller)); \ + poll_caller[polls++] = __func__; \ + bool bist_fault = bist_status_stuck && (void *)(address) == \ + (void *)®isters[ACIOPHY_BIST_MAC_USBPHY_STATUS2 / 4]; \ + bist_faults += bist_fault; \ + polls == fail_poll || bist_fault || \ + (fail_caller && !strcmp(fail_caller, __func__)) || !(condition) ? -ETIMEDOUT : 0; }) +#define readl_poll_timeout readl_poll_timeout_atomic +/* Observation is read-only and separately tested; retain production caller control flow. */ +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p; usb2_on++; } +static void atcphy_apply_tunables(struct apple_atcphy *p, enum atcphy_mode mode) +{ (void)p; (void)mode; } +static void atc_tunnel_stop_t8103(struct apple_atcphy *p) { (void)p; } +static void atc_tunnel_restore(struct apple_atcphy *p) { (void)p; } +''') + for name in ('atcphy_pipehandler_lock', 'atcphy_pipehandler_unlock', 'atcphy_pipehandler_check', + 'atcphy_pipehandler_set_mux', 'atcphy_clear_nonselected_phy_reset', + 'atcphy_configure_pipehandler_usb3_host_t8122', + 'atcphy_configure_pipehandler_usb3_host_t8103', + 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', + 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', + 'atcphy_usb2_power_off', '_atcphy_dwc3_reset_assert', 'atcphy_dwc3_reset_assert'): + parts.append(function(atc, name)) + for name in ('core_read32', 'atcphy_get_mode_config', 'j700_field_update', 'j700_pma_idle'): + parts.append(function(atc, name)) + parts.append(dp) + for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', + 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', + 'atcphy_power_on', 'atcphy_configure', 'atcphy_sw_set', 'atcphy_mux_set', + 'atcphy_usb3_set_mode', 'atcphy_usb3_power_off'): + parts.append(function(atc, name)) + production = '\n'.join(parts) + if 'static int j700_usb_capture(' in dp: + production = '#define HAS_USB_CONTEXT\n' + production + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define field_prep FIELD_PREP +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define hweight32(v) __builtin_popcount(v) +#define dev_info_once(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +static void report(void *dev, const char *fmt, ...) { (void)dev; (void)fmt; } +#define dev_err report +#define lockdep_assert_held(...) ((void)0) +#define WARN_ON_ONCE(v) assert(!(v)) +#define guard(kind) (void) +#define fallthrough __attribute__((fallthrough)) +#define ndelay(n) ((void)(n)) +#define udelay(n) ((void)(n)) +#define usleep_range(a, b) ((void)(a), (void)(b)) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +enum typec_orientation { TYPEC_ORIENTATION_NONE, TYPEC_ORIENTATION_NORMAL, + TYPEC_ORIENTATION_REVERSE }; +enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +struct typec_switch_dev { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +#define typec_switch_get_drvdata(sw) ((sw)->data) +struct phy { struct apple_atcphy *data; }; +#define phy_get_drvdata(phy) ((phy)->data) +struct reset_controller_dev { int unused; }; +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +''' + suffix = r''' +static struct atcphy_hw hw = { + .has_t8140_dp = true, .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, + .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, +}; +static void cold_initial(struct apple_atcphy *p, bool fixed, bool swapped) +{ + memset(registers, 0, sizeof(registers)); + memset(pipe_regs, 0, sizeof(pipe_regs)); + memset(usb2_regs, 0, sizeof(usb2_regs)); + registers[0x200 / 4] = 0xee; /* public PMA powered state, synthetic cold fixture */ + registers[AUS_COMMON_DIG_RCAL1 / 4] = AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + pipe_regs[PIPEHANDLER_AON_GEN / 4] = PIPEHANDLER_AON_GEN_DWC3_RESET_N; + usb2_regs[USB2PHY_USBCTL / 4] = USB2PHY_USBCTL_RUN; + *p = (struct apple_atcphy) { + .hw = &hw, .regs.core = (unsigned char *)registers, + .regs.lpdptx = (unsigned char *)registers, + .regs.pipehandler = (unsigned char *)pipe_regs, + .regs.usb2phy = (unsigned char *)usb2_regs, + .fixed_usb2 = fixed, .host_active = true, .swap_lanes = swapped, + .ss_tunables = true, .dp_tunables = true, .dp_presets_valid = { true }, + .mode = APPLE_ATCPHY_MODE_USB3_DP, .typec_mode = APPLE_ATCPHY_MODE_USB3_DP, + .pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, .mode_dirty = false, + .dp_owned_pmas = 0, .dp_idle_pmas = 0, + .aux_power_requested = false, + }; + polls = fail_poll = writes = usb2_on = usb2_writes = reset_writes = mux_writes = 0; + stuck_ack = bist_status_stuck = false; + hold_unlock_ack = false; + fail_caller = NULL; + bist_faults = 0; + require_parked = false; +} +static void initial(struct apple_atcphy *p, bool fixed, bool dirty, bool swapped) +{ + cold_initial(p, fixed, swapped); +#ifdef HAS_USB_CONTEXT + if (fixed && hw.has_t8140_dp) { + /* Two-PMA cleanup stress setup: capture before ownership and actual DP writes. */ + assert(j700_usb_capture(p, APPLE_ATCPHY_MODE_DP) == 0); + assert(atcphy_configure_pipehandler(p, true) == 0); + assert(j700_dp_start(p, 3, 3, false) == 0); + } else +#endif + { + p->pipe_state = ATCPHY_PIPEHANDLER_STATE_USB3; + p->dp_owned_pmas = 3; /* legacy synthetic cleanup ownership, no J700 context path */ + } + p->mode_dirty = dirty; + p->dp_idle_pmas = dirty ? 1 : 0; + p->aux_power_requested = true; + polls = fail_poll = writes = usb2_on = usb2_writes = reset_writes = mux_writes = 0; +} + +static void inject_parking(unsigned int fault) +{ + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + if (fault) + fail_poll = 1; + else + stuck_ack = true; +} +static void recover(void) +{ + stuck_ack = bist_status_stuck = false; + hold_unlock_ack = false; + fail_caller = NULL; + fail_poll = polls = 0; +} +static int request(struct apple_atcphy *p, bool safe) +{ + struct typec_mux_dev mux = { p }; + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = { + .mode = safe ? TYPEC_STATE_SAFE : TYPEC_DP_STATE_C, + .alt = safe ? NULL : &dp, + }; + return atcphy_mux_set(&mux, &state); +} +static void unchanged(struct apple_atcphy *p, bool dirty, bool host) +{ + bool expected_dirty = dirty || (p->hw->has_t8140_dp && p->fixed_usb2 && host); + assert(p->mode == APPLE_ATCPHY_MODE_USB3_DP && p->mode_dirty == expected_dirty); + assert(p->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && p->host_active == host); + assert(p->dp_owned_pmas == 3 && p->dp_idle_pmas == (dirty ? 1U : 0U)); + assert(p->aux_power_requested); + assert(writes == 0 && usb2_on == 0 && usb2_writes == 0 && reset_writes == 0); + assert(mux_writes == 0); + assert(pipe_regs[PIPEHANDLER_AON_GEN / 4] == PIPEHANDLER_AON_GEN_DWC3_RESET_N); + assert(usb2_regs[USB2PHY_USBCTL / 4] == USB2PHY_USBCTL_RUN); + if (expected_dirty) { + struct phy_configure_opts_dp training = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + assert(j700_dp_configure(p, &training) == -EBUSY && writes == 0); + } +} +static void observed(const char *caller, int result) +{ + printf("%s: result=%d core_writes=%u usb2_writes=%u reset_writes=%u\n", + caller, result, writes, usb2_writes, reset_writes); + fflush(stdout); +} +static void mux_parking(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int dirty = 0; dirty < 2; dirty++) + for (unsigned int swapped = 0; swapped < 2; swapped++) + for (unsigned int host = 0; host < 2; host++) + for (unsigned int safe = 0; safe < 2; safe++) + for (unsigned int fault = 0; fault < 2; fault++) { + initial(&p, true, dirty, swapped); + p.host_active = host; + inject_parking(fault); + int ret = request(&p, safe); + observed("mux parking failure", ret); + assert(ret == -ETIMEDOUT); + unchanged(&p, dirty, host); + if (!fault) { + assert(request(&p, safe) == -ETIMEDOUT); + unchanged(&p, dirty, host); + } + recover(); + assert(request(&p, safe) == 0); + assert(p.mode == (safe ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_DP)); + assert(!p.mode_dirty && p.host_active == host); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !p.dp_owned_pmas); + assert(!p.dp_idle_pmas && p.aux_power_requested == !safe); + assert(usb2_on == 1 && usb2_writes == 0 && reset_writes == 0); + assert(usb2_regs[USB2PHY_USBCTL / 4] == USB2PHY_USBCTL_RUN); + unsigned int before = writes; + assert(request(&p, safe) == 0 && writes == before && usb2_on == 1); + cases++; + } + printf("PASS: %u actual mux parking/retry scenarios preserve fixed USB2 and ownership\n", cases); +} +static void power_parking(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int dirty = 0; dirty < 2; dirty++) + for (unsigned int swapped = 0; swapped < 2; swapped++) + for (unsigned int fault = 0; fault < 2; fault++) { + initial(&p, fixed, dirty, swapped); + struct phy phy = { &p }; + inject_parking(fault); + int ret = atcphy_usb3_power_off(&phy); + observed("power-off parking failure", ret); + assert(ret == -ETIMEDOUT); + unchanged(&p, dirty, true); + if (!fault) { + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + unchanged(&p, dirty, true); + } + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + assert(usb2_writes == 0 && reset_writes == 0); + unsigned int before = writes; + assert(atcphy_usb3_power_off(&phy) == 0 && writes == before); + cases++; + } + printf("PASS: %u actual power-off parking/retry scenarios\n", cases); +} +static void reset_parking(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) + for (unsigned int dirty = 0; dirty < 2; dirty++) + for (unsigned int fault = 0; fault < 2; fault++) { + initial(&p, fixed, dirty, false); + inject_parking(fault); + int ret = atcphy_dwc3_reset_assert(&p.rcdev, 0); + observed("reset parking failure", ret); + assert(ret == -ETIMEDOUT); + unchanged(&p, dirty, true); + if (!fault) { + assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == -ETIMEDOUT); + unchanged(&p, dirty, true); + } + recover(); + assert(atcphy_dwc3_reset_assert(&p.rcdev, 0) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty == (dirty || fixed)); + assert(p.dp_owned_pmas == 3 && p.dp_idle_pmas == (dirty ? 1U : 0U)); + assert(p.aux_power_requested && writes == 0); + assert(reset_writes == 2 && usb2_writes > 0); + assert(!(pipe_regs[PIPEHANDLER_AON_GEN / 4] & PIPEHANDLER_AON_GEN_DWC3_RESET_N)); + assert(pipe_regs[PIPEHANDLER_AON_GEN / 4] & PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); + assert(!(usb2_regs[USB2PHY_USBCTL / 4] & USB2PHY_USBCTL_RUN)); + cases++; + } + printf("PASS: %u actual reset parking/retry scenarios retain DP ownership\n", cases); +} +static void power_cleanup(void) +{ + struct apple_atcphy p; + unsigned int phases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + initial(&p, fixed, false, false); + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + struct phy phy = { &p }; + fail_poll = 1; /* The first DP stop ACK cannot complete. */ + int ret = atcphy_usb3_power_off(&phy); + observed("power-off DP cleanup failure", ret); + assert(ret == -ETIMEDOUT && p.mode_dirty && p.host_active); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.dp_owned_pmas == 3); + assert(p.aux_power_requested && usb2_writes == 0 && reset_writes == 0); + struct phy_configure_opts_dp training = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + unsigned int before = writes; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(!p.mode_dirty && !p.host_active && !p.dp_owned_pmas && !p.aux_power_requested); + /* A failed initial transition can leave OFF recorded with dirty resources. */ + initial(&p, fixed, true, false); + p.mode = APPLE_ATCPHY_MODE_OFF; + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = 1; + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + assert(p.mode_dirty && p.host_active && p.dp_owned_pmas == 3); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + for (unsigned int fault = 1; fault <= 2; fault++) { + initial(&p, fixed, false, false); + p.mode = APPLE_ATCPHY_MODE_USB2; + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + p.dp_owned_pmas = p.dp_idle_pmas = 0; + p.aux_power_requested = false; + fail_poll = fault; /* BIG/SMALL sleep ACK. */ + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty && p.host_active); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0 && !p.mode_dirty && !p.host_active); + } + /* Inject every poll in DP stop, pending PMA idle, AUX and domain sleep. */ + initial(&p, fixed, true, false); + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + assert(atcphy_usb3_power_off(&phy) == 0); + unsigned int total = polls; + for (unsigned int fault = 1; fault <= total; fault++) { + initial(&p, fixed, true, false); + p.pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_poll = fault; + assert(atcphy_usb3_power_off(&phy) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty && p.host_active); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(usb2_writes == 0 && reset_writes == 0); + unsigned int owned = p.dp_owned_pmas, idle = p.dp_idle_pmas; + bool aux = p.aux_power_requested; + unsigned int before = writes; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(p.dp_owned_pmas == owned && p.dp_idle_pmas == idle); + assert(p.aux_power_requested == aux); + recover(); + assert(atcphy_usb3_power_off(&phy) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty && !p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + phases++; + } + } + printf("PASS: %u cleanup poll faults retain retry/ownership; dirty OFF retries cleanup\n", phases); +} +static int select_mode(struct apple_atcphy *p, enum atcphy_mode mode) +{ + struct typec_mux_dev mux = { p }; + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + + if (mode == APPLE_ATCPHY_MODE_USB3) + state.mode = TYPEC_STATE_USB; + else if (mode == APPLE_ATCPHY_MODE_USB3_DP || mode == APPLE_ATCPHY_MODE_DP) { + state.alt = &dp; + state.mode = mode == APPLE_ATCPHY_MODE_DP ? TYPEC_DP_STATE_C : TYPEC_DP_STATE_D; + } + return atcphy_mux_set(&mux, &state); +} +static void orient(struct apple_atcphy *p, bool reverse) +{ + struct typec_switch_dev sw = { p }; + assert(atcphy_sw_set(&sw, reverse ? TYPEC_ORIENTATION_REVERSE : + TYPEC_ORIENTATION_NORMAL) == 0); +} +static void orientation_initial(struct apple_atcphy *p, bool fixed) +{ + cold_initial(p, fixed, false); + p->mode = p->typec_mode = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF; + p->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + p->dp_owned_pmas = p->dp_idle_pmas = 0; + p->aux_power_requested = false; + p->dp_link_rate = -1; + p->host_active = fixed; + registers[ACIOPHY_TOP_PHY_STAT / 4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + registers[ATCPHY_MISC / 4] = BIT(7) | ATCPHY_MISC_LANE_SWAP; + expected_j700_swap = true; + registers[hw.aciophy_lane_mode / 4] = 0xa5000000; + registers[hw.aciophy_crossbar / 4] = 0x80000000; + pipe_regs[PIPEHANDLER_MUX_CTRL / 4] = + FIELD_PREP(hw.pipe_mux_data, PIPEHANDLER_MUX_CTRL_DATA_DUMMY) | + FIELD_PREP(hw.pipe_mux_clk, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); +} +static void check_route(struct apple_atcphy *p, enum atcphy_mode mode, bool reverse) +{ + assert(p->mode == mode && p->swap_lanes == reverse && !p->mode_dirty); + assert(registers[ATCPHY_MISC / 4] & BIT(7)); + /* Fixed public route expectations, independent of the production mode table. */ + assert(mode == APPLE_ATCPHY_MODE_USB3 || mode == APPLE_ATCPHY_MODE_USB3_DP || + mode == APPLE_ATCPHY_MODE_DP); + u32 lane = registers[hw.aciophy_lane_mode / 4]; + u32 crossbar = registers[hw.aciophy_crossbar / 4]; + u32 mask = hw.has_t8140_dp && mode != APPLE_ATCPHY_MODE_DP ? 0x1ffff : 0x3ffff; + assert((lane & 0xfff) == (mode == APPLE_ATCPHY_MODE_DP ? 0x492 : + (reverse ? 0x252 : 0x489))); + assert((lane & 0xfffff000) == 0xa5000000); + assert((crossbar & mask) == (mode == APPLE_ATCPHY_MODE_DP ? + (reverse ? 0x114 : 0x22014) : (0x110 | reverse))); + assert(crossbar & 0x80000000); + /* J700 preserves the boot swap bit; generic routes apply their explicit swap. */ + assert(!!(registers[ATCPHY_MISC / 4] & ATCPHY_MISC_LANE_SWAP) == + (hw.has_t8140_dp ? expected_j700_swap : + (mode != APPLE_ATCPHY_MODE_DP && reverse))); + assert(p->pipe_state == (p->fixed_usb2 && mode != APPLE_ATCPHY_MODE_DP ? + ATCPHY_PIPEHANDLER_STATE_USB3 : ATCPHY_PIPEHANDLER_STATE_DUMMY)); +} +static void orientation_roundtrip(void) +{ + struct apple_atcphy p; + const enum atcphy_mode modes[] = { + APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_DP, + }; + unsigned int cases = 0; + + for (unsigned int generation = 0; generation < 4; generation++) { + hw.gen = generation == 0 ? ATCPHY_GENERATION_T8103 : ATCPHY_GENERATION_T8122; + hw.has_t8140_dp = generation == 2; + hw.has_usb4 = !hw.has_t8140_dp; + hw.aciophy_lane_mode = generation == 0 ? ACIOPHY_LANE_MODE_T8103 : + ACIOPHY_LANE_MODE_T8122; + hw.aciophy_crossbar = generation == 0 ? ACIOPHY_CROSSBAR_T8103 : + ACIOPHY_CROSSBAR_T8122; + hw.pipe_mux_clk = generation == 3 ? PIPEHANDLER_MUX_CTRL_CLK_T8142 : + PIPEHANDLER_MUX_CTRL_CLK_T8103; + hw.pipe_mux_data = generation == 3 ? PIPEHANDLER_MUX_CTRL_DATA_T8142 : + PIPEHANDLER_MUX_CTRL_DATA_T8103; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int m = 0; m < ARRAY_SIZE(modes); m++) { + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, fixed); + for (unsigned int turn = 0; turn < 3; turn++) { + bool reverse = first ^ (turn & 1); + orient(&p, reverse); + if (turn) { + assert(p.mode_dirty); + struct phy phy = { &p }; + unsigned int before = writes; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(writes == before); + } + require_parked = true; + assert(select_mode(&p, modes[m]) == 0); + require_parked = false; + check_route(&p, modes[m], reverse); + unsigned int before = writes, before_mux = mux_writes; + orient(&p, reverse); + assert(select_mode(&p, modes[m]) == 0); + assert(writes == before && mux_writes == before_mux); + struct typec_switch_dev sw = { &p }; + assert(atcphy_sw_set(&sw, TYPEC_ORIENTATION_NONE) == 0); + assert(!p.mode_dirty && p.swap_lanes == reverse); + cases++; + } + /* A USB3 protocol change also rebuilds the live backend. */ + if (m == 0) { + require_parked = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + require_parked = false; + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, first); + } + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == (fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF)); + assert(!p.mode_dirty && !p.aux_power_requested && !p.dp_idle_pmas); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + } + } + } + } + printf("PASS: %u production orientation routes across T8103/T8122/T8140/T8142, fixed/ordinary ports, both flip roundtrips\n", cases); +} +static void orientation_faults(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(!first)); + orient(&p, !first); + unsigned int before = writes; + struct phy_configure_opts_dp training = { .set_voltages = true, .lanes = 2 }; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + /* Derive all actual poll phases, then inject each in a fresh transition. */ + polls = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int total = polls; + bool advisory[256] = { false }; + for (unsigned int i = 0; i < total; i++) { + const char *caller = poll_caller[i]; + /* Every observed BIST and USB3 handoff failure is now fatal. */ + if (fixed && i + 1 == total) { + assert(!strcmp(caller, "atcphy_pipehandler_unlock")); + advisory[i] = false; + } else if (!strcmp(caller, "atcphy_configure_pipehandler_usb3_host_t8122")) { + assert(fixed); + advisory[i] = false; + } else { + assert(!strcmp(caller, "j700_dp_wait") || + !strcmp(caller, "j700_field_update") || + !strcmp(caller, "j700_usb_field_update") || + !strcmp(caller, "atcphy_power_on") || + !strcmp(caller, "atcphy_configure") || + (fixed && (!strcmp(caller, "atcphy_pipehandler_lock") || + !strcmp(caller, "atcphy_pipehandler_unlock")))); + } + } + for (unsigned int fault = 1; fault <= total; fault++) { + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + unsigned int owned = p.dp_owned_pmas; + orient(&p, !first); + polls = 0; + fail_poll = fault; + int ret = select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP); + /* A fatal phase must never be accepted as advisory success. */ + assert((ret == 0) == advisory[fault - 1]); + if (!ret) { + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !first); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas); + cases++; + continue; + } + assert(ret == -ETIMEDOUT); + if (p.mode_dirty || p.pipe_pending) { + before = writes; + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + if (fault == 1 && !fixed) + assert(p.dp_owned_pmas == owned && p.aux_power_requested); + } else { + assert(fixed && p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + } + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !first); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas); + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(first)); + assert(j700_dp_configure(&p, &training) == 0); + cases++; + } + /* A fatal parking check preserves the old owner and all lane words. */ + if (fixed) { + orientation_initial(&p, true); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, true); + before = writes; + inject_parking(0); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(writes == before && p.dp_owned_pmas == 2 && p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && p.host_active); + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, true); + } + /* Reversing a pending flip cannot erase cleanup obligations. */ + orientation_initial(&p, fixed); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, true); + orient(&p, false); + assert(p.mode_dirty && p.dp_owned_pmas == 2); + p.ss_tunables = false; + before = writes; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -EOPNOTSUPP); + assert(writes == before && p.mode_dirty && p.dp_owned_pmas == 2); + p.ss_tunables = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, false); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas); + } + } + printf("PASS: %u orientation transition poll injections, dirty training guard, ownership and retry controls\n", cases); +} +static void backend_change(void) +{ + struct apple_atcphy p; + + orientation_initial(&p, true); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + require_parked = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + require_parked = false; + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, false); + puts("PASS: a live fixed-hub USB3 backend parks before same-orientation protocol changes"); +} + + +static void failed_flip_return(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, !first); + polls = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int route_fault = 0; + for (unsigned int i = 0; i < polls; i++) + if (!strcmp(poll_caller[i], "j700_field_update")) route_fault = i + 1; + assert(route_fault); + + orientation_initial(&p, fixed); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + orient(&p, !first); + polls = 0; + fail_poll = route_fault; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + /* The route changed before PMA idle completed; completed mode is still old. */ + assert((registers[0x60 / 4] & 0xfff) == (first ? 0x489 : 0x252)); + assert(p.mode_dirty && p.dp_idle_pmas == BIT(first) && !p.dp_owned_pmas); + orient(&p, first); + assert(p.mode_dirty && p.dp_idle_pmas == BIT(first)); + unsigned int before = writes; + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, first); + assert(!p.dp_idle_pmas && !p.dp_owned_pmas); + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(!first)); + cases++; + } + } + printf("PASS: %u partially applied flip cancellations preserve pending PMA ownership\n", cases); +} + +static void safe_first_recovery(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + + for (unsigned int first = 0; first < 2; first++) { + orientation_initial(&p, true); + orient(&p, first); + bist_status_stuck = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == -ETIMEDOUT); + assert(bist_faults == 2 && p.mode_dirty && p.pipe_pending); + bist_status_stuck = false; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3, first); + polls = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int host_lock_poll = 0; + for (unsigned int i = 0; i < polls; i++) + if (!strcmp(poll_caller[i], "atcphy_pipehandler_lock")) host_lock_poll = i + 1; + assert(host_lock_poll > 1); /* Inject the host lock, after acknowledged parking. */ + orientation_initial(&p, true); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3) == 0); + polls = 0; + fail_poll = host_lock_poll; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && !p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && p.host_active); + assert(p.aux_power_requested); + + /* Persistent AUX cleanup failure blocks SAFE before orientation publication. */ + fail_caller = "j700_field_update"; + u32 route = registers[0x60 / 4], crossbar = registers[0x64 / 4]; + for (unsigned int attempt = 0; attempt < 3; attempt++) { + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == -ETIMEDOUT); + assert(p.swap_lanes == first && p.mode_dirty && p.aux_power_requested); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.host_active); + assert(registers[0x60 / 4] == route && registers[0x64 / 4] == crossbar); + assert(usb2_regs[USB2PHY_USBCTL / 4] == USB2PHY_USBCTL_RUN); + } + /* Returning the plug cannot clear an unavailable cleanup handshake. */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == -ETIMEDOUT); + assert(p.swap_lanes == first && p.mode_dirty); + recover(); /* Model hardware readiness returning; production performs the retry. */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && !p.mode_dirty && !p.aux_power_requested); + /* SAFE uses the existing generic USB2 route and clears its swap control. */ + assert(!(registers[ATCPHY_MISC / 4] & ATCPHY_MISC_LANE_SWAP)); + expected_j700_swap = false; + orient(&p, !first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !first); + + /* Real reconnect order: SAFE must succeed before the new orientation is published. */ + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(first)); + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, first); + assert(j700_dp_configure(&p, &rate) == 0); + assert(p.dp_owned_pmas == BIT(!first)); + unsigned int before = writes, before_mux = mux_writes; + orient(&p, first); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(writes == before && mux_writes == before_mux); + cases++; + } + printf("PASS: %u chained BIST/host-ACK failure, persistent SAFE failure and SAFE-first roundtrip controls\n", cases); +} + +static void live_parking_handshake(void) +{ + struct apple_atcphy p; + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + struct phy_configure_opts_dp training = { .set_voltages = true, .lanes = 2 }; + unsigned int cases = 0; + for (unsigned int reverse = 0; reverse < 2; reverse++) { + for (unsigned int fault = 0; fault < 3; fault++) { + orientation_initial(&p, true); + orient(&p, reverse); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + u32 lanes = registers[0x60 / 4], route = registers[0x64 / 4]; + u32 native = pipe_regs[PIPEHANDLER_NONSELECTED_OVERRIDE / 4]; + unsigned int before = writes, owned = p.dp_owned_pmas; + if (fault == 2) { + /* A previous host unlock left ACK asserted before parking. */ + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + hold_unlock_ack = true; + } else if (fault == 1) { + hold_unlock_ack = true; + } else { + stuck_ack = true; + } + for (unsigned int retry = 0; retry < 3; retry++) { + unsigned int before_mux = mux_writes; + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_USB3_DP && p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && p.host_active); + assert(p.dp_owned_pmas == owned && p.aux_power_requested); + assert(writes == before && registers[0x60 / 4] == lanes && registers[0x64 / 4] == route); + assert(pipe_regs[PIPEHANDLER_NONSELECTED_OVERRIDE / 4] == native); + if (fault != 1) assert(mux_writes == before_mux); + /* A failed SAFE cannot be forgotten by returning to the cached mode. */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(j700_dp_configure(&p, &training) == -EBUSY && writes == before); + assert(!usb2_writes && !reset_writes); + } + recover(); + assert(select_mode(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && !p.mode_dirty && p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + expected_j700_swap = false; + orient(&p, !reverse); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + check_route(&p, APPLE_ATCPHY_MODE_USB3_DP, !reverse); + assert(j700_dp_configure(&p, &rate) == 0 && p.dp_owned_pmas == BIT(reverse)); + cases++; + } + } + printf("PASS: %u live parking entry-ACK/LOCK/UNLOCK failure, ownership, write ordering and retry roundtrips\n", cases); +} + +int main(int argc, char **argv) +{ + assert(argc == 2); + if (!strcmp(argv[1], "mux")) mux_parking(); + else if (!strcmp(argv[1], "power")) power_parking(); + else if (!strcmp(argv[1], "reset")) reset_parking(); + else if (!strcmp(argv[1], "cleanup")) power_cleanup(); + else if (!strcmp(argv[1], "orientation")) orientation_roundtrip(); + else if (!strcmp(argv[1], "orientation-faults")) orientation_faults(); + else if (!strcmp(argv[1], "backend-change")) backend_change(); + else if (!strcmp(argv[1], "safe-first")) safe_first_recovery(); + else if (!strcmp(argv[1], "live-park")) live_parking_handshake(); + else if (!strcmp(argv[1], "failed-flip-return")) failed_flip_return(); + else assert(false); + return 0; +} +''' + return {**locals(), "block": block} + +def adverse(root): + + # Preserve continued source macros; the parking fixture does not use the + # continued PLL clock macro exercised by the legacy rate helper here. + ns = parking(root) + production = ns['production'] + '\n' + ns['function'](ns['atc'], 'atcphy_usb2_init') + production = production.replace('bool pipe_pending, usb_off_pending, usb_off_drained;', + 'bool pipe_pending, usb_off_pending, usb_off_drained, tunnel_clock_on, tunnel_saved;', 1) + production += '\n' + ns['block'](ns['atc'], 'enum atcphy_lane {') + ';' + production += '\n' + ns['block'](ns['atc'], 'struct atcphy_dp_link_rate_configuration {') + ';' + production += '\n' + ns['block'](ns['atc'], 'static const struct atcphy_dp_link_rate_configuration dp_lr_config[] = {') + ';' + # These leaf operations are outside the pending/cache gate under test. + production += r''' +static int atcphy_auspll_apb_command(struct apple_atcphy *p, u32 command) +{ (void)p; (void)command; assert(false); return -EIO; } +static int atcphy_dp_configure_lane(struct apple_atcphy *p, enum atcphy_lane lane, + const struct atcphy_dp_link_rate_configuration *cfg) +{ (void)p; (void)lane; (void)cfg; assert(false); return -EIO; } +''' + production += '\n' + ns['function'](ns['atc'], 'atcphy_dp_configure') + for profile in ('t8103', 't8122', 't8140', 't8142'): + production += '\n' + ns['block'](ns['atc'], + 'static const struct atcphy_hw atcphy_hw_' + profile + ' = {') + ';' + production = production.replace('static unsigned int polls,', 'static int poll_errors[256];\nstatic unsigned int polls,', 1) + production = production.replace('polls == fail_poll || bist_fault ||', + 'poll_errors[polls] ? poll_errors[polls] : polls == fail_poll || bist_fault ||', 1) + production = production.replace('static u32 readl(void *address) { return *(u32 *)address; }', + '''static unsigned int mmio_reads; +static bool only_power_mmio; +static u32 readl(void *address) +{ + mmio_reads++; + if (only_power_mmio && (uintptr_t)address >= (uintptr_t)registers && + (uintptr_t)address < (uintptr_t)registers + sizeof(registers)) { + uintptr_t offset=(uintptr_t)address-(uintptr_t)registers; + assert(offset==ATCPHY_POWER_CTRL || offset==ATCPHY_POWER_STAT || offset==ATCPHY_MISC); + } + return *(u32 *)address; +}''', 1) + production = production.replace(' (void)p;\n assert(offset < sizeof(registers)', + ' (void)p;\n if (only_power_mmio) assert(offset==ATCPHY_POWER_CTRL || offset==ATCPHY_MISC);\n assert(offset < sizeof(registers)', 1) + controls = r''' +static void legacy_setup(struct apple_atcphy *p, unsigned int gen, bool fixed) +{ + hw = gen==0 ? atcphy_hw_t8103 : gen==1 ? atcphy_hw_t8122 : + gen==2 ? atcphy_hw_t8140 : atcphy_hw_t8142; + cold_initial(p, fixed, false); + p->mode = p->typec_mode = APPLE_ATCPHY_MODE_USB3; + registers[ACIOPHY_TOP_PHY_STAT/4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + memset(poll_errors, 0, sizeof(poll_errors)); +} +static void shared_bist(void) +{ + for (unsigned int gen=0; gen<2; gen++) { + unsigned int count=gen ? 5 : 3; + for (unsigned int fault=1; fault<=count; fault++) { + struct apple_atcphy p; + legacy_setup(&p,gen,false); + poll_errors[fault]=-EIO; + if (fault1 && !strcmp(argv[1],"epoch")) { epoch_controls(); return 0; } + if(argc>1 && !strcmp(argv[1],"bist")) { bist_controls(); return 0; } + if(argc>1)return original_fixture_main(argc,argv); + epoch_controls();bist_controls();return 0; +} +''' + return ns['prefix'] + production + ns['suffix'].replace('int main(', 'int original_fixture_main(', 1) + controls + +def warm_reset(root): + + ns = parking(root) + production = ns['production'] + reset_block = '''\t/* +\t * Fixed-hub SAFE keeps USB2 up rather than taking the block OFF. +\t * With PIPE parked and DP owners released, reinitialize USB3 before +\t * programming its lanes; the existing sequence releases this reset. +\t */ +\tif (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2) +\t\tcore_clear32(atcphy, ATCPHY_POWER_CTRL, ATCPHY_POWER_PHY_RESET_N); + +''' + assert production.count(reset_block) == 1 + production = production.replace('static u32 registers[', '''static unsigned int warm_reset_edges, warm_unreset_lanes, warm_ownership_errors; +static u32 registers[''', 1) + production = production.replace(' registers[offset / 4] = (registers[offset / 4] & ~mask) | value;', ''' if (offset == ATCPHY_POWER_CTRL && (mask & ATCPHY_POWER_PHY_RESET_N) && + !(value & ATCPHY_POWER_PHY_RESET_N)) { + warm_reset_edges++; + if (p->fixed_usb2 && p->hw->has_t8140_dp && + (p->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY || p->dp_owned_pmas || + p->dp_idle_pmas || p->aux_power_requested)) warm_ownership_errors++; + } + if (offset == p->hw->aciophy_lane_mode && p->fixed_usb2 && p->hw->has_t8140_dp && + (registers[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N)) warm_unreset_lanes++; + registers[offset / 4] = (registers[offset / 4] & ~mask) | value;''', 1) + controls = r''' +static void warm_initial(struct apple_atcphy *p) +{ + orientation_initial(p, true); + registers[ATCPHY_POWER_CTRL / 4] |= ATCPHY_POWER_PHY_RESET_N; + warm_reset_edges = warm_unreset_lanes = warm_ownership_errors = 0; +} +static void warm_roundtrip(void) +{ + bool failed = false; + for (unsigned int first = 0; first < 2; first++) { + struct apple_atcphy p; + const struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + warm_initial(&p); + for (unsigned int attach = 0; attach < 3; attach++) { + orient(&p, first ^ (attach & 1)); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && !p.mode_dirty); + assert(j700_dp_configure(&p, &rate) == 0); + unsigned int edges = warm_reset_edges; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(warm_reset_edges == edges); /* cached same-mode must not reset */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && p.host_active); + assert(!p.dp_owned_pmas && !p.dp_idle_pmas && !p.aux_power_requested); + assert(registers[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); + assert(!reset_writes && !usb2_writes); + } + assert(usb2_on == 6); /* no extra USB2 power-on call introduced */ + printf("first=%u reset_assertions=%u unreset_lane_writes=%u ownership_errors=%u\n", + first, warm_reset_edges, warm_unreset_lanes, warm_ownership_errors); + failed |= warm_reset_edges != 6 || warm_unreset_lanes || warm_ownership_errors; + } + fflush(stdout); + assert(!failed); +} +static void warm_failure_controls(void) +{ + struct apple_atcphy p; + const struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 2, + }; + warm_initial(&p); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + unsigned int edges = warm_reset_edges, owned = p.dp_owned_pmas; + stuck_ack = true; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == -ETIMEDOUT); + assert(warm_reset_edges == edges && p.dp_owned_pmas == owned && p.mode_dirty); + assert(!usb2_writes && !reset_writes); + + warm_initial(&p); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(j700_dp_configure(&p, &rate) == 0); + edges = warm_reset_edges; owned = p.dp_owned_pmas; + fail_caller = "j700_field_update"; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == -ETIMEDOUT); + assert(warm_reset_edges == edges && p.dp_owned_pmas == owned && p.mode_dirty); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert(!usb2_writes && !reset_writes); + + warm_initial(&p); + fail_caller = "atcphy_configure"; /* existing RCAL poll, after reset release */ + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(warm_reset_edges == 1 && !warm_ownership_errors && p.mode_dirty); + assert(registers[ATCPHY_POWER_CTRL / 4] & ATCPHY_POWER_PHY_RESET_N); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + struct phy phy = { &p }; + unsigned int before = writes; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -EBUSY); + assert(j700_dp_configure(&p, &rate) == -EBUSY && writes == before); + fail_caller = NULL; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(!p.mode_dirty && !warm_ownership_errors); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + assert(!usb2_writes && !reset_writes); + puts("PASS: park/DP-cleanup failures precede reset; released-reset RCAL failure stays dirty and retryable"); +} +static void warm_scope_controls(void) +{ + struct apple_atcphy p; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int t8140 = 0; t8140 < 2; t8140++) { + orientation_initial(&p, fixed); + hw.has_t8140_dp = t8140; + registers[ATCPHY_POWER_CTRL / 4] |= ATCPHY_POWER_PHY_RESET_N; + warm_reset_edges = warm_unreset_lanes = warm_ownership_errors = 0; + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + assert(warm_reset_edges == (fixed && t8140)); + assert(!warm_ownership_errors && !usb2_writes && !reset_writes); + } + } + hw.has_t8140_dp = true; + puts("PASS: reset scope requires both fixed hub and T8140; ordinary/legacy paths unchanged"); +} +int main(int argc, char **argv) +{ + if (argc > 1) return original_fixture_main(argc, argv); + warm_roundtrip(); warm_failure_controls(); warm_scope_controls(); + puts("PASS: warm reset source ordering; no electrical recovery is modeled"); + return 0; +} +''' + suffix = ns['suffix'].replace('int main(', 'int original_fixture_main(', 1) + controls + return ns['prefix'] + production + suffix + +def usb_context(root): + ns = parking(root) + production=ns['production'].replace('static u32 registers[', 'static u32 blocked_offset;\nstatic unsigned int mmio_reads;\nstatic bool track_dp_writes;\nstatic u32 dp_write_masks[0x30000 / 4];\nstatic u32 registers[', 1) + production=production.replace('static u32 readl(void *address) { return *(u32 *)address; }', 'static u32 readl(void *address) { mmio_reads++; return *(u32 *)address; }', 1) + production=production.replace(' registers[offset / 4] = (registers[offset / 4] & ~mask) | value;', ' if (track_dp_writes) dp_write_masks[offset / 4] |= mask | value;\n if (blocked_offset && offset == blocked_offset) { writes++; return; }\n registers[offset / 4] = (registers[offset / 4] & ~mask) | value;', 1) + controls = r''' +static void seed_usb(struct apple_atcphy *p, unsigned int first) +{ + orientation_initial(p, true); + registers[0x200 / 4] = 0xee; + registers[0xc / 4] = 0; + /* Synthetic USB context, never a private factory table or captured register image. */ + for (unsigned int offset = 0x9000; offset < 0x15000; offset += 4) + registers[offset / 4] = 0x5a5a1234 ^ offset; + registers[0x8 / 4] = 0xa5000000; + registers[0x10c / 4] = 0xa5000000; + orient(p, first); + assert(select_mode(p, APPLE_ATCPHY_MODE_USB3_DP) == 0); +} + +static void mask_coverage(void) +{ + struct apple_atcphy p; + memset(dp_write_masks, 0, sizeof(dp_write_masks)); + seed_usb(&p, 1); + for (unsigned int rate = 0; rate < 4; rate++) { + track_dp_writes = true; + assert(j700_dp_start(&p, 1, rate, false) == 0); + j700_dp_equalize(&p, 0, true, 0x123); + j700_dp_equalize(&p, 0, false, 0x123); + assert(j700_dp_stop(&p) == 0); + track_dp_writes = false; + } + unsigned int entries = 0; + for (unsigned int offset = 0; offset < sizeof(dp_write_masks); offset += 4) { + if (!((offset >= 0x9000 && offset <= 0xdfff) || offset == 8 || + offset == 0xc || offset == 0x10c || offset == 0x200)) continue; + u32 expected = 0; + for (unsigned int i = 0; i < ARRAY_SIZE(j700_usb_fields); i++) + if (j700_usb_fields[i].offset == offset) expected = j700_usb_fields[i].mask; + assert(dp_write_masks[offset / 4] == expected); + entries += !!expected; + } + assert(entries == 27); + puts("PASS: 27 masks equal the actual public DP start/voltage/stop write union at every rate"); +} +static void masked_roundtrip(void) +{ + const struct phy_configure_opts_dp rate = { + .set_rate=true, .set_lanes=true, .link_rate=8100, .lanes=2, + }; + bool failure=false; + for (unsigned int first=0; first<2; first++) { + struct apple_atcphy p; + seed_usb(&p, first); + unsigned int old_dp=!first; +#ifdef PROPOSAL + assert(p.dp_usb_context_valid & BIT(old_dp)); + assert(p.dp_usb_context[old_dp][1] == 0); + assert(p.dp_usb_context[old_dp][3] == (0xeU << (4 * old_dp))); + assert(((registers[0x200 / 4] >> (4 * old_dp)) & 0xf) == 1); +#endif + u32 saved[27], outside[27]; + for (unsigned int i=0;i<27;i++) { + u32 offset,mask; + j700_usb_field(i,old_dp,&offset,&mask); + saved[i]=core_read32(&p,offset)&mask; + } + /* Power/receiver were saved before route idle, not at this post-route sample. */ + saved[1]=0; saved[3]=0xeU<<(4*old_dp); + assert(j700_dp_configure(&p,&rate)==0); + unsigned int edges_before=writes; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(writes==edges_before); /* same mode never resets or recaptures */ + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + for(unsigned int i=0;i<27;i++) { + u32 offset,mask;j700_usb_field(i,old_dp,&offset,&mask); + u32 other_offset,other_mask; j700_usb_field(i,!old_dp,&other_offset,&other_mask); + outside[i]=registers[offset/4]&~(mask | (i<4 ? other_mask : 0)); + } + orient(&p,!first); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + unsigned int mismatch=0; + for(unsigned int i=0;i<27;i++) { + u32 offset,mask;j700_usb_field(i,old_dp,&offset,&mask); + if((registers[offset/4]&mask)!=saved[i]) mismatch++; + u32 other_offset,other_mask; j700_usb_field(i,!old_dp,&other_offset,&other_mask); + assert((registers[offset/4]&~(mask | (i<4 ? other_mask : 0)))==outside[i]); + } + printf("first=%u former_dp_pma=%u masked_mismatches=%u receiver=%#x power=%#x\n", + first,old_dp,mismatch,registers[0xc/4],registers[0x200/4]); + failure |= mismatch!=0; + assert(!usb2_writes&&!reset_writes); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,first); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(!p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + } + fflush(stdout);assert(!failure); +} +static void restore_controls(void) +{ +#ifdef PROPOSAL + struct apple_atcphy p; + const struct phy_configure_opts_dp rate={.set_rate=true,.set_lanes=true,.link_rate=8100,.lanes=2}; + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + unsigned int valid=p.dp_usb_context_valid,dirty=p.dp_usb_context_dirty; + u32 snapshot[2][27];memcpy(snapshot,p.dp_usb_context,sizeof(snapshot)); + stuck_ack=true; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==-ETIMEDOUT); + assert(p.dp_owned_pmas==1&&p.dp_usb_context_valid==valid&&p.dp_usb_context_dirty==dirty); + assert(!memcmp(snapshot,p.dp_usb_context,sizeof(snapshot))); + recover(); + fail_caller="j700_field_update"; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==-ETIMEDOUT); + assert(p.dp_owned_pmas==1&&p.dp_usb_context_valid==valid); + assert(!memcmp(snapshot,p.dp_usb_context,sizeof(snapshot))); + recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + assert(j700_usb_capture(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + p.dp_owned_pmas=1; + unsigned int before=writes; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EBUSY&&writes==before); + p.dp_owned_pmas=0;p.dp_idle_pmas=1; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EBUSY&&writes==before); + p.dp_idle_pmas=0;p.pipe_state=ATCPHY_PIPEHANDLER_STATE_USB3; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EBUSY&&writes==before); + p.pipe_state=ATCPHY_PIPEHANDLER_STATE_DUMMY; + fail_caller="j700_usb_field_update"; + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT); + assert(p.dp_usb_context_valid&1); + assert(p.dp_usb_context_dirty&1); + recover();assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(!(p.dp_usb_context_valid&1)&&!(p.dp_usb_context_dirty&1)); + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + u32 old_context[27];memcpy(old_context,p.dp_usb_context[0],sizeof(old_context)); + fail_caller="j700_usb_field_update"; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT); + assert(p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert((p.dp_usb_context_valid&1)&&(p.dp_usb_context_dirty&1)); + assert(!memcmp(old_context,p.dp_usb_context[0],sizeof(old_context))); + before=writes; + assert(j700_dp_configure(&p,&rate)==-EBUSY&&writes==before); + recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(!p.mode_dirty&&!(p.dp_usb_context_valid&1)); + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + blocked_offset=0x9244; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-EIO); + assert(p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY); + assert((p.dp_usb_context_valid&1)&&(p.dp_usb_context_dirty&1)); + before=writes; + assert(j700_dp_configure(&p,&rate)==-EBUSY&&writes==before); + blocked_offset=0; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&!p.mode_dirty); + puts("PASS: actual mux static/power restore failures keep dirty route/context, block training and retry"); + seed_usb(&p,1); + assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); + orient(&p,0); + registers[ACIOPHY_BIST_MAC_USBPHY_STATUS2/4]=BIT(7); + bist_status_stuck=true; bist_faults=0; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT); + assert(p.mode_dirty && p.pipe_pending); + assert(bist_faults==2 && (registers[ACIOPHY_BIST_MAC_USBPHY_STATUS2/4]&BIT(7))); + assert((registers[0x200/4]&0xf)==0xe && !(registers[0xc/4]&0x44)); + stuck_ack=true; + before=writes; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==-ETIMEDOUT && writes==before); + puts("PASS: restoration never clears simulated BIST/ACK faults or forces readiness"); + for(unsigned int fixed=0;fixed<2;fixed++) { + seed_usb(&p,0);p.fixed_usb2=fixed;hw.has_t8140_dp=false; + before=writes; + assert(j700_usb_capture(&p,APPLE_ATCPHY_MODE_USB3_DP)==0); + assert(j700_usb_restore(&p,APPLE_ATCPHY_MODE_USB3_DP)==0&&writes==before); + hw.has_t8140_dp=true; + } + puts("PASS: park/DP-idle failure retention, immutable snapshot, live/pending-owner restore guards, power readback failure/retry, legacy exclusions"); +#endif +} + +static void inactive_host_gates(void) +{ +#ifdef PROPOSAL + const struct phy_configure_opts_dp rate = { + .set_rate=true, .set_lanes=true, .link_rate=8100, .lanes=2, + }; + unsigned int violations = 0; + for (unsigned int backend = ATCPHY_PIPEHANDLER_STATE_USB3; + backend <= ATCPHY_PIPEHANDLER_STATE_USB4; backend++) { + for (unsigned int entry = 0; entry < 3; entry++) { + struct apple_atcphy p; + seed_usb(&p, 1); + assert(j700_dp_configure(&p, &rate) == 0); + assert(select_mode(&p, APPLE_ATCPHY_MODE_USB2) == 0); + orient(&p, 0); + p.host_active = false; + p.pipe_state = backend; + unsigned int before_writes = writes, before_reads = mmio_reads; + unsigned int valid = p.dp_usb_context_valid, dirty = p.dp_usb_context_dirty; + u32 snapshot[2][27]; memcpy(snapshot, p.dp_usb_context, sizeof(snapshot)); + int ret = entry == 0 ? j700_usb_capture(&p, APPLE_ATCPHY_MODE_USB3_DP) : + entry == 1 ? j700_usb_restore(&p, APPLE_ATCPHY_MODE_USB3_DP) : + atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP); + bool blocked = ret == -EBUSY && writes == before_writes && + mmio_reads == before_reads && valid == p.dp_usb_context_valid && + dirty == p.dp_usb_context_dirty && + !memcmp(snapshot, p.dp_usb_context, sizeof(snapshot)); + printf("inactive-host backend=%u entry=%u ret=%d new_writes=%u new_reads=%u blocked=%u\n", + backend, entry, ret, writes-before_writes, mmio_reads-before_reads, blocked); + violations += !blocked; + } + } + fflush(stdout); assert(!violations); + puts("PASS: six inactive-host/live-PIPE entry gates retain snapshots and perform zero MMIO"); +#endif +} +static void failure_diagnostics(void) +{ +#ifdef PROPOSAL + struct apple_atcphy p; + const struct phy_configure_opts_dp rate={.set_rate=true,.set_lanes=true,.link_rate=8100,.lanes=2}; + const char *stages[] = { "stage=static", "stage=power", "stage=receiver" }; + for (unsigned int stage=0; stage<3; stage++) { + seed_usb(&p,1); assert(j700_dp_configure(&p,&rate)==0); + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB2)==0); orient(&p,0); + diagnostic_records=0; last_diagnostic[0]=0; + blocked_offset=stage==0 ? 0x9244 : stage==1 ? 0x200 : 0xc; + int ret=select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP); + assert(ret==(stage==0 ? -EIO : -ETIMEDOUT)); + assert(diagnostic_records==1 && strstr(last_diagnostic,stages[stage])); + assert(strstr(last_diagnostic,"pma=0") && strstr(last_diagnostic,"reg=")); + assert(strstr(last_diagnostic,"mask=") && strstr(last_diagnostic,"expected=")); + assert(strstr(last_diagnostic,"observed=")); + assert(p.mode_dirty && (p.dp_usb_context_valid&1) && (p.dp_usb_context_dirty&1)); + blocked_offset=0; + } + puts("PASS: static/power/receiver errors preserve state and report stage/PMA/register/mask/expected/final observed"); +#endif +} +int main(int argc,char **argv) +{ + if(argc>1 && !strcmp(argv[1],"inactive-host")) { inactive_host_gates(); return 0; } + if(argc>1)return original_fixture_main(argc,argv); + mask_coverage();masked_roundtrip();restore_controls();inactive_host_gates();failure_diagnostics(); + puts("PASS: masked restoration roundtrips; no BIST readiness or USB speed is modeled"); + return 0; +} +''' + logger = r''' +#include +static unsigned int diagnostic_records; +static char last_diagnostic[512]; +static void record_debug(const char *format, ...) +{ + if (!strstr(format, "USB restore stage=")) return; + va_list args; va_start(args,format); + vsnprintf(last_diagnostic,sizeof(last_diagnostic),format,args); + va_end(args); diagnostic_records++; + fputs(last_diagnostic,stdout); +} +#undef dev_dbg +#define dev_dbg(dev, format, ...) record_debug(format, ##__VA_ARGS__) +''' + return ns["prefix"] + logger + "#define PROPOSAL\n" + production + ns["suffix"].replace("int main(", "int original_fixture_main(", 1) + controls + +def pipe(root): + + source = (root / 'drivers/phy/apple/atc.c').read_text() + dp = (root / 'drivers/phy/apple/atc-j700-dp.h').read_text() + + + def block(marker): + start = source.index(marker) + opening = source.index('{', start) + end, depth = opening + 1, 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[start:end] + + + def function(name): + if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: + marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() + return block(marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return block(re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) + + + parts = re.findall(r'^#define [^\n]*(?:\\\n[^\n]*)*', source, re.M) + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode'): + parts.append(block('enum ' + name + ' {') + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw'): + parts.append(block('struct ' + name + ' {') + ';') + parts.append(r''' +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *pipehandler; } regs; + void *dev; + int lock; + bool fixed_usb2, host_active, ss_tunables, dp_tunables, mode_dirty; + unsigned int dp_owned_pmas, dp_idle_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + bool swap_lanes; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +''') + start = source.index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + parts.append(source[start:source.index('\n};', start) + 3]) + parts.append(r''' +static u32 pipe_regs[0x1000 / 4], core_regs[0x30000 / 4]; +static unsigned int polls, fail_poll, mux_writes, lane_calls; +static bool stuck_ack; +static u32 readl(void *address) { return *(u32 *)address; } +static void mask32(void *address, u32 mask, u32 value) +{ + *(u32 *)address = (*(u32 *)address & ~mask) | value; + if (address == (void *)&pipe_regs[PIPEHANDLER_LOCK_REQ / 4] && !stuck_ack) + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = *(u32 *)address & PIPEHANDLER_LOCK_EN; + if (address == (void *)&pipe_regs[PIPEHANDLER_MUX_CTRL / 4]) + mux_writes++; +} +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +static void writel(u32 v, void *p) { mask32(p, ~0U, v); } +static void core_mask32(struct apple_atcphy *p, u32 o, u32 m, u32 v) +{ assert(o < sizeof(core_regs) && o % 4 == 0); mask32(p->regs.core + o, m, v); } +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +#define readl_poll_timeout_atomic readl_poll_timeout +#define readl_poll_timeout(address, value, condition, interval, timeout) \ + ({ (value) = readl(address); polls++; \ + polls == fail_poll || !(condition) ? -ETIMEDOUT : 0; }) +/* Lane configuration is outside this test: count calls and commit its result. */ +/* Observation is read-only and separately tested; retain production caller control flow. */ +static int atcphy_configure(struct apple_atcphy *p, enum atcphy_mode mode); +''') + for name in ('core_read32', 'atcphy_get_mode_config'): + parts.append(function(name)) + # Actual restoration functions and gates; analog lane setup stays outside this PIPE unit. + parts.append(dp[dp.index('/* Masks are the union'):dp.index('/* Retry only the idle operations')]) + parts.append(r''' +static int atcphy_configure(struct apple_atcphy *p, enum atcphy_mode mode) +{ + int ret; + lane_calls++; p->mode_dirty = true; + ret = j700_usb_capture(p, mode); + if (!ret) ret = j700_usb_restore(p, mode); + if (ret) return ret; + p->mode = mode; p->mode_dirty = false; return 0; +} +''') + for name in ('atcphy_pipehandler_lock', 'atcphy_pipehandler_unlock', 'atcphy_pipehandler_check', + 'atcphy_pipehandler_set_mux', 'atcphy_clear_nonselected_phy_reset', + 'atcphy_configure_pipehandler_usb3_host_t8122', + 'atcphy_configure_pipehandler_usb3_host_t8103', + 'atcphy_configure_pipehandler_usb3', 'atcphy_configure_pipehandler_usb4', + 'atcphy_enable_dummy_phy', 'atcphy_configure_pipehandler_dummy', + 'atcphy_configure_pipehandler', 'atcphy_park_pipehandler', + 'atcphy_usb3_set_mode', 'atcphy_mux_set'): + parts.append(function(name)) + production = '\n'.join(parts) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define field_prep FIELD_PREP +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define lockdep_assert_held(...) ((void)0) +#define WARN_ON_ONCE(v) assert(!(v)) +#define guard(kind) (void) +#define fallthrough __attribute__((fallthrough)) +#define usleep_range(...) ((void)0) +#define udelay(n) ((void)(n)) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE, PHY_MODE_INVALID }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +struct phy { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +#define phy_get_drvdata(phy) ((phy)->data) +''' + suffix = r''' +static struct atcphy_hw hw = { + .gen = ATCPHY_GENERATION_T8122, .has_t8140_dp = true, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, +}; +static void initial(struct apple_atcphy *p, enum atcphy_mode mode, + enum atcphy_pipehandler_state backend) +{ + memset(pipe_regs, 0, sizeof(pipe_regs)); + memset(core_regs, 0, sizeof(core_regs)); + core_regs[ACIOPHY_TOP_PHY_STAT / 4] = ACIOPHY_TOP_PHY_STAT_LN0_UNK0; + *p = (struct apple_atcphy) { + .hw = &hw, .regs.core = (unsigned char *)core_regs, + .regs.pipehandler = (unsigned char *)pipe_regs, + .mode = mode, .typec_mode = mode, .pipe_state = backend, + .ss_tunables = true, .dp_tunables = true, + }; + polls = fail_poll = mux_writes = lane_calls = 0; + stuck_ack = false; +} +static int request(struct apple_atcphy *p, enum atcphy_mode mode) +{ + struct typec_mux_dev mux = { p }; + struct typec_altmode dp = { .svid = USB_TYPEC_DP_SID }; + struct enter_usb_data data = { .eudo = EUDO_USB_MODE_USB4 }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + if (mode == APPLE_ATCPHY_MODE_USB3) + state.mode = TYPEC_STATE_USB; + else if (mode == APPLE_ATCPHY_MODE_USB3_DP || mode == APPLE_ATCPHY_MODE_DP) { + state.mode = mode == APPLE_ATCPHY_MODE_DP ? TYPEC_DP_STATE_C : TYPEC_DP_STATE_D; + state.alt = &dp; + } else if (mode == APPLE_ATCPHY_MODE_USB4) { + state.mode = TYPEC_MODE_USB4; + state.data = &data; + } + return atcphy_mux_set(&mux, &state); +} +static void backend_failure(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (enum atcphy_mode mode = APPLE_ATCPHY_MODE_OFF; mode <= APPLE_ATCPHY_MODE_DP; mode++) { + enum atcphy_pipehandler_state target = atcphy_modes[mode].pipehandler_state; + enum atcphy_pipehandler_state old = target == ATCPHY_PIPEHANDLER_STATE_DUMMY ? + ATCPHY_PIPEHANDLER_STATE_USB3 : ATCPHY_PIPEHANDLER_STATE_DUMMY; + for (unsigned int host = 0; host < 2; host++) { + initial(&p, mode, old); + /* A previous lock ACK cannot be cleared: fatal before mux writes. */ + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_configure_pipehandler(&p, host) == -ETIMEDOUT); + assert(p.pipe_state == old && mux_writes == 0); + stuck_ack = false; + assert(atcphy_configure_pipehandler(&p, host) == 0); + assert(p.pipe_state == target && mux_writes == 3); + const unsigned int data[] = { PIPEHANDLER_MUX_CTRL_DATA_DUMMY, + PIPEHANDLER_MUX_CTRL_DATA_USB3, PIPEHANDLER_MUX_CTRL_DATA_USB4 }; + const unsigned int clk[] = { PIPEHANDLER_MUX_CTRL_CLK_DUMMY, + PIPEHANDLER_MUX_CTRL_CLK_USB3, PIPEHANDLER_MUX_CTRL_CLK_USB4 }; + u32 mux = pipe_regs[PIPEHANDLER_MUX_CTRL / 4]; + assert(FIELD_GET(hw.pipe_mux_data, mux) == data[target]); + assert(FIELD_GET(hw.pipe_mux_clk, mux) == clk[target]); + cases++; + } + } + printf("PASS: %u backend/role check-failure and retry cases\n", cases); +} +static void host_lock_failure(void) +{ + struct apple_atcphy p; + for (unsigned int gen = 0; gen < 2; gen++) { + hw.gen = gen ? ATCPHY_GENERATION_T8122 : ATCPHY_GENERATION_T8103; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + struct phy phy = { &p }; + stuck_ack = true; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == -ETIMEDOUT); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && mux_writes == 0); + assert(!(pipe_regs[PIPEHANDLER_LOCK_REQ / 4] & PIPEHANDLER_LOCK_EN)); + stuck_ack = false; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && mux_writes == 6); + unsigned int before = polls; + assert(atcphy_usb3_set_mode(&phy, PHY_MODE_USB_HOST, 0) == 0); + assert(polls == before && mux_writes == 6); + } + puts("PASS: both USB3 host BIST generations retain fatal lock ACK and retry"); +} +static void mux_retry(void) +{ + struct apple_atcphy p; + const enum atcphy_mode modes[] = { APPLE_ATCPHY_MODE_USB3, APPLE_ATCPHY_MODE_USB3_DP }; + for (unsigned int m = 0; m < 2; m++) { + initial(&p, APPLE_ATCPHY_MODE_USB2, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + stuck_ack = true; + assert(request(&p, modes[m]) == -ETIMEDOUT); + assert(p.mode == modes[m] && !p.mode_dirty && lane_calls == 1); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && mux_writes == 0); + p.dp_owned_pmas = 3; /* PIPE retry must not reconfigure an owned DP mode. */ + assert(request(&p, modes[m]) == -ETIMEDOUT); + assert(lane_calls == 1 && p.dp_owned_pmas == 3 && mux_writes == 0); + stuck_ack = false; + assert(request(&p, modes[m]) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && lane_calls == 1); + assert(p.dp_owned_pmas == 3 && mux_writes == 6); + unsigned int before = polls; + assert(request(&p, modes[m]) == 0 && polls == before && lane_calls == 1); + } + puts("PASS: fixed-hub same-mode retry preserves lanes and DP ownership"); +} +static void same_mode_pending(void) +{ + struct apple_atcphy p; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + stuck_ack = true; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == -ETIMEDOUT); + assert(lane_calls == 0 && mux_writes == 0); + stuck_ack = false; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && lane_calls == 0); + puts("PASS: same-mode pending PIPE setup independently retries"); +} +static void parking_and_guards(void) +{ + struct apple_atcphy p; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_USB3); + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_park_pipehandler(&p) == -ETIMEDOUT); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && mux_writes == 0); + stuck_ack = false; + assert(atcphy_park_pipehandler(&p) == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && mux_writes == 3); + assert(atcphy_park_pipehandler(&p) == 0); + assert(mux_writes == 3); + initial(&p, APPLE_ATCPHY_MODE_USB2, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + assert(request(&p, APPLE_ATCPHY_MODE_OFF) == 0); + assert(p.mode == APPLE_ATCPHY_MODE_USB2 && lane_calls == 0 && mux_writes == 0); + p.mode_dirty = true; + assert(request(&p, APPLE_ATCPHY_MODE_OFF) == 0 && lane_calls == 1 && !p.mode_dirty); + /* Inactive hosts and ordinary ports retain their deferred PIPE setup. */ + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = true; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == 0 && mux_writes == 0); + p.fixed_usb2 = false; + p.host_active = true; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == 0 && mux_writes == 0); + /* Capability rejection must still precede mode configuration. */ + initial(&p, APPLE_ATCPHY_MODE_USB2, ATCPHY_PIPEHANDLER_STATE_DUMMY); + p.fixed_usb2 = p.host_active = true; + p.ss_tunables = false; + assert(request(&p, APPLE_ATCPHY_MODE_USB3) == -EOPNOTSUPP && lane_calls == 0); + p.ss_tunables = true; + p.dp_tunables = false; + assert(request(&p, APPLE_ATCPHY_MODE_USB3_DP) == -EOPNOTSUPP && lane_calls == 0); + assert(request(&p, APPLE_ATCPHY_MODE_USB4) == -EOPNOTSUPP && lane_calls == 0); + puts("PASS: parking retry, SAFE fixed USB2, dirty-mode and capability guards"); +} +static void requested_lock(void) +{ + struct apple_atcphy p; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_DUMMY); + pipe_regs[PIPEHANDLER_LOCK_REQ / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_pipehandler_lock(&p) == -ETIMEDOUT && polls == 1); + assert(!(pipe_regs[PIPEHANDLER_LOCK_REQ / 4] & PIPEHANDLER_LOCK_EN)); + pipe_regs[PIPEHANDLER_LOCK_REQ / 4] = PIPEHANDLER_LOCK_EN; + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + assert(atcphy_pipehandler_lock(&p) == 0 && polls == 2); + puts("PASS: an existing LOCK request must still have actual ACK before success"); +} +static void live_policy_scope(void) +{ + struct apple_atcphy p; + for (unsigned int t8140 = 0; t8140 < 2; t8140++) { + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int active = 0; active < 2; active++) { + hw.has_t8140_dp = t8140; + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_USB3); + p.fixed_usb2 = fixed; + p.host_active = active; + stuck_ack = true; /* Request exists, but ACK never asserts. */ + int ret = atcphy_park_pipehandler(&p); + if (t8140 && fixed && active) { + assert(ret == -ETIMEDOUT && p.mode_dirty && mux_writes == 0); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); + assert(!pipe_regs[PIPEHANDLER_NONSELECTED_OVERRIDE / 4]); + } else { + assert(ret == 0 && !p.mode_dirty && mux_writes == 3); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY); + } + /* Entry ACK release is fatal everywhere, but dirties only live T8140 parking. */ + initial(&p, APPLE_ATCPHY_MODE_USB3, ATCPHY_PIPEHANDLER_STATE_USB3); + p.fixed_usb2 = fixed; + p.host_active = active; + pipe_regs[PIPEHANDLER_LOCK_ACK / 4] = PIPEHANDLER_LOCK_EN; + stuck_ack = true; + assert(atcphy_park_pipehandler(&p) == -ETIMEDOUT); + assert(p.mode_dirty == (t8140 && fixed && active)); + assert(p.pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && mux_writes == 0); + } + } + } + puts("PASS: eight live/inactive, fixed/ordinary and T8140/legacy LOCK and entry-ACK policy controls"); +} +int main(int argc, char **argv) +{ + assert(argc == 2); + if (!strcmp(argv[1], "backend")) backend_failure(); + else if (!strcmp(argv[1], "host")) host_lock_failure(); + else if (!strcmp(argv[1], "mux")) mux_retry(); + else if (!strcmp(argv[1], "same")) same_mode_pending(); + else if (!strcmp(argv[1], "park")) parking_and_guards(); + else if (!strcmp(argv[1], "requested")) requested_lock(); + else if (!strcmp(argv[1], "scope")) live_policy_scope(); + else assert(false); + return 0; +} +''' + return prefix + production + suffix + +def unwind(root): + + atc = (root / 'drivers/phy/apple/atc.c').read_text() + dp = (root / 'drivers/phy/apple/atc-j700-dp.h').read_text() + + + def function(source, name): + if name == 'atcphy_mux_set' and 'static int atcphy_mux_set_locked(' in source: + marker = re.search(r'^static [^\n]+\batcphy_mux_set_locked\(', source, re.M).group() + return block(source, marker).replace('atcphy_mux_set_locked(', 'atcphy_mux_set(', 1) + return block(source, re.search(r'^static [^\n]+\b' + name + r'\(', source, re.M).group()) + + + # Use the production mode table and wrappers too, so SAFE mapping and fast paths + # are tested rather than mirrored by a second implementation. + parts = re.findall(r'^#define [^\n]*(?:\\\n[^\n]*)*', atc, re.M) + for name in ('atcphy_generation', 'atcphy_lane_mode', 'atcphy_pipehandler_state', 'atcphy_mode', + 'atcphy_dp_link_rate'): + parts.append(block(atc, 'enum ' + name + ' {') + ';') + for name in ('atcphy_mode_configuration', 'atcphy_hw'): + parts.append(block(atc, 'struct ' + name + ' {') + ';') + parts.append(r''' +struct apple_atcphy { + const struct atcphy_hw *hw; + struct { unsigned char *core, *lpdptx; } regs; + void *dev; + int lock, dp_link_rate; + bool fixed_usb2, host_active, swap_lanes, ss_tunables, dp_tunables; + bool aux_power_requested, mode_dirty, dp_spread_spectrum, dp_presets_valid[5]; + u32 dp_presets[5][16]; + unsigned int dp_idle_pmas, dp_owned_pmas; + unsigned int dp_usb_context_valid, dp_usb_context_dirty; + u32 dp_usb_context[2][27]; + bool pipe_pending, usb_off_pending, usb_off_drained; + enum atcphy_mode mode, typec_mode; + enum atcphy_pipehandler_state pipe_state; +}; +''') + start = atc.index('static const struct {\n\tconst struct atcphy_mode_configuration normal;') + parts.append(atc[start:atc.index('\n};', start) + 3]) + parts.append(r''' +struct phy_configure_opts_dp { + unsigned int link_rate, lanes, voltage[4], pre[4]; + bool set_rate, set_lanes, set_voltages, ssc; +}; +static u32 registers[0x30000 / 4]; +static unsigned int polls, fail_poll, writes, usb2_on; +static u32 readl(void *address) { return *(u32 *)address; } +static void core_mask32(struct apple_atcphy *p, u32 offset, u32 mask, u32 value) +{ + (void)p; + assert(offset < sizeof(registers) && offset % 4 == 0); + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + writes++; + if (offset == ATCPHY_POWER_CTRL) + registers[ATCPHY_POWER_STAT / 4] = registers[offset / 4]; + if (offset == 0x2000) { + if (value == 0x10000001) + registers[offset / 4] |= BIT(1); + else { + registers[offset / 4] &= ~BIT(1); + registers[0x7034 / 4] &= ~BIT(3); + } + } +} +static void core_set32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, 0, v); } +static void core_clear32(struct apple_atcphy *p, u32 o, u32 v) { core_mask32(p, o, v, 0); } +static void writel(u32 value, void *address) { *(u32 *)address = value; } +static void mask32(void *p, u32 m, u32 v) { * (u32 *)p = (*(u32 *)p & ~m) | v; } +static void set32(void *p, u32 v) { mask32(p, 0, v); } +static void clear32(void *p, u32 v) { mask32(p, v, 0); } +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = readl(address); polls++; \ + polls == fail_poll || !(condition) ? -ETIMEDOUT : 0; }) +#define readl_poll_timeout readl_poll_timeout_atomic +static void atcphy_usb2_power_on(struct apple_atcphy *p) { (void)p; usb2_on++; } +static void atcphy_apply_tunables(struct apple_atcphy *p, enum atcphy_mode mode) +{ (void)p; (void)mode; } +static void atc_tunnel_stop_t8103(struct apple_atcphy *p) { (void)p; } +static void atc_tunnel_restore(struct apple_atcphy *p) { (void)p; } +static int atcphy_park_pipehandler(struct apple_atcphy *p) +{ p->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; return 0; } +/* Observation is read-only and separately tested; retain production caller control flow. */ +static int atcphy_configure_pipehandler(struct apple_atcphy *p, bool host) +{ (void)host; p->pipe_state = atcphy_modes[p->mode].pipehandler_state; return 0; } +''') + for name in ('core_read32', 'atcphy_get_mode_config', 'j700_field_update', 'j700_pma_idle'): + parts.append(function(atc, name)) + parts.append(dp) + for name in ('atcphy_configure_lanes', 'j700_aux_enable', 'j700_aux_disable', + 'atcphy_enable_dp_aux', 'atcphy_disable_dp_aux', 'atcphy_power_off', + 'atcphy_power_on', 'atcphy_configure', 'atcphy_mux_set'): + parts.append(function(atc, name)) + production = '\n'.join(parts) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +typedef uint16_t u16; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U << (l)) & (~0U >> (31 - (h)))) +#define FIELD_PREP(m, v) (((u32)(v) << __builtin_ctz(m)) & (m)) +#define FIELD_GET(m, v) (((v) & (m)) >> __builtin_ctz(m)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define hweight32(v) __builtin_popcount(v) +#define dev_info_once(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +static void report(void *dev, const char *fmt, ...) { (void)dev; (void)fmt; } +#define dev_err report +#define lockdep_assert_held(...) ((void)0) +#define WARN_ON_ONCE(v) assert(!(v)) +#define guard(kind) (void) +#define fallthrough __attribute__((fallthrough)) +#define ndelay(n) ((void)(n)) +#define udelay(n) ((void)(n)) +#define usleep_range(a, b) ((void)(a), (void)(b)) +enum { TYPEC_STATE_SAFE, TYPEC_STATE_USB, TYPEC_MODE_USB4, + TYPEC_DP_STATE_C, TYPEC_DP_STATE_D, TYPEC_DP_STATE_E }; +enum { USB_TYPEC_TBT_SID = 0x8087, USB_TYPEC_DP_SID = 0xff01 }; +#define EUDO_USB_MODE_MASK 3U +enum { EUDO_USB_MODE_USB2, EUDO_USB_MODE_USB3, EUDO_USB_MODE_USB4 }; +struct enter_usb_data { u32 eudo; }; +struct typec_altmode { unsigned int svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +#define typec_mux_get_drvdata(mux) ((mux)->data) +''' + suffix = r''' +static struct atcphy_hw hw = { + .has_t8140_dp = true, .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = 0x60, .aciophy_crossbar = 0x64, +}; +static void initial(struct apple_atcphy *p, bool fixed, bool swapped) +{ + memset(registers, 0, sizeof(registers)); + registers[AUS_COMMON_DIG_RCAL1 / 4] = AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + *p = (struct apple_atcphy) { + .hw = &hw, .regs.core = (unsigned char *)registers, + .regs.lpdptx = (unsigned char *)registers, + .fixed_usb2 = fixed, .swap_lanes = swapped, + .ss_tunables = true, .dp_tunables = true, .dp_presets_valid = { true }, + .mode = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF, + .pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY, + }; + polls = fail_poll = writes = usb2_on = 0; +} +static int safe(struct apple_atcphy *p) +{ + struct typec_mux_dev mux = { p }; + struct typec_mux_state state = { .mode = TYPEC_STATE_SAFE }; + return atcphy_mux_set(&mux, &state); +} +int main(void) +{ + struct apple_atcphy p; + unsigned int cases = 0; + for (unsigned int fixed = 0; fixed < 2; fixed++) { + for (unsigned int swapped = 0; swapped < 2; swapped++) { + const enum atcphy_mode modes[] = { + APPLE_ATCPHY_MODE_DP, APPLE_ATCPHY_MODE_USB3_DP, APPLE_ATCPHY_MODE_USB3, + }; + for (unsigned int m = 0; m < ARRAY_SIZE(modes); m++) { + unsigned int aux = modes[m] != APPLE_ATCPHY_MODE_USB3 ? 4 : 0; + unsigned int idle = modes[m] == APPLE_ATCPHY_MODE_DP ? 12 : 6; + for (unsigned int fault = 3; fault <= 2 + aux + idle; fault++) { + enum atcphy_mode old = fixed ? APPLE_ATCPHY_MODE_USB2 : APPLE_ATCPHY_MODE_OFF; + initial(&p, fixed, swapped); + fail_poll = fault; + assert(atcphy_configure(&p, modes[m]) == -ETIMEDOUT); + assert(p.mode == old && p.mode_dirty); + if (fault <= 2 + aux) + assert(p.aux_power_requested && !p.dp_idle_pmas); + else + assert(p.dp_idle_pmas); + bool owned_aux = p.aux_power_requested; + unsigned int owned_idle = p.dp_idle_pmas; + polls = 0; + fail_poll = 1; + usb2_on = 0; + assert(safe(&p) == -ETIMEDOUT); + assert(p.mode == old && p.mode_dirty); + assert(p.aux_power_requested == owned_aux && p.dp_idle_pmas == owned_idle); + assert(usb2_on == 0); /* Cleanup failure must not continue into power changes. */ + polls = fail_poll = 0; + assert(safe(&p) == 0); + assert(p.mode == old && !p.mode_dirty); + assert(!p.aux_power_requested && !p.dp_idle_pmas && !p.dp_owned_pmas); + assert(usb2_on == fixed); /* Fixed-hub SAFE restores USB2, never powers it off. */ + polls = writes = 0; + assert(safe(&p) == 0 && polls == 0 && writes == 0); + cases++; + } + } + } + } + /* A failed route change must not expose the old DP mode to link training. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == 0); + unsigned int last_idle_poll = polls - 1; /* The final poll is RCAL, after route idle. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = last_idle_poll; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode == APPLE_ATCPHY_MODE_DP && p.mode_dirty && p.dp_idle_pmas); + struct phy_configure_opts_dp rate = { + .set_rate = true, .set_lanes = true, .link_rate = 8100, .lanes = 4, + }; + struct phy_configure_opts_dp training = { .set_voltages = true, .lanes = 4 }; + unsigned int writes_before = writes, idle_before = p.dp_idle_pmas; + unsigned int owned_before = p.dp_owned_pmas; + assert(j700_dp_configure(&p, &rate) == -EBUSY); + assert(j700_dp_configure(&p, &training) == -EBUSY); + assert(writes == writes_before && p.dp_idle_pmas == idle_before); + assert(p.dp_owned_pmas == owned_before && p.aux_power_requested && p.mode_dirty); + /* Explicit stop is allowed, but must leave the route cleanup obligation. */ + struct phy_configure_opts_dp stop = { .set_rate = true }; + assert(j700_dp_configure(&p, &stop) == 0); + assert(p.dp_idle_pmas == idle_before && p.mode_dirty); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.dp_idle_pmas && !p.aux_power_requested); + /* If teardown itself fails, reject training without dropping live ownership. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = 1; + assert(atcphy_configure(&p, APPLE_ATCPHY_MODE_USB3_DP) == -ETIMEDOUT); + assert(p.mode_dirty && p.dp_owned_pmas == 3); + writes_before = writes; + assert(j700_dp_configure(&p, &rate) == -EBUSY); + assert(j700_dp_configure(&p, &training) == -EBUSY); + assert(writes == writes_before && p.dp_owned_pmas == 3 && p.aux_power_requested); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.dp_owned_pmas && !p.aux_power_requested); + /* A cleanup failure also dirties an initially successful mode transition. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_DP; + p.aux_power_requested = true; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT && p.mode_dirty && p.aux_power_requested); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.aux_power_requested); + /* Existing main-link ownership survives a failed cleanup of a dirty mode. */ + initial(&p, true, false); + p.mode_dirty = p.aux_power_requested = true; + p.dp_owned_pmas = 3; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT); + assert(p.mode_dirty && p.aux_power_requested && p.dp_owned_pmas == 3); + fail_poll = polls = 0; + assert(safe(&p) == 0 && !p.mode_dirty && !p.aux_power_requested && !p.dp_owned_pmas); + /* OFF is not committed when domain sleep fails, and SAFE retries it. */ + initial(&p, false, false); + p.mode = APPLE_ATCPHY_MODE_USB2; + fail_poll = 1; + assert(safe(&p) == -ETIMEDOUT && p.mode == APPLE_ATCPHY_MODE_USB2 && p.mode_dirty); + fail_poll = polls = 0; + assert(safe(&p) == 0 && p.mode == APPLE_ATCPHY_MODE_OFF && !p.mode_dirty); + printf("PASS: %u AUX/PMA fault scenarios, failed cleanup, dirty training guard, SAFE retry, fixed USB2, OFF commit\n", cases); + return 0; +} +''' + return prefix + production + suffix + +def aux(root): + + source = root / 'drivers/phy/apple/atc.c' + text = source.read_text() + parts = [line for line in text.splitlines() if line.startswith(('#define J700_AUX_', '#define J700_PMA_'))] + for name in ('j700_field_update', 'j700_pma_idle', 'j700_aux_enable', 'j700_aux_disable'): + start = text.index('static int ' + name + '(') + begin = text.index('{', start) + depth = 1 + end = begin + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + parts.append(text[start:end]) + production = '\n'.join(parts) + prefix = r''' +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +struct apple_atcphy { struct { unsigned char *core; } regs; bool aux_power_requested; }; +struct event { u32 offset, mask, value; } events[32]; +static u32 registers[0x18000 / 4]; +static unsigned int writes, reject; +static void core_mask32(struct apple_atcphy *phy, u32 offset, u32 mask, u32 value) +{ + assert(offset < sizeof(registers) && offset % 4 == 0 && writes < 32); + assert(!(value & ~mask)); + events[writes] = (struct event){offset, mask, value}; + writes++; + if (writes != reject) + registers[offset / 4] = (registers[offset / 4] & ~mask) | value; + (void)phy; +} +static void usleep_range(unsigned int low, unsigned int high) +{ + assert(low && high >= low); +} +#define readl_poll_timeout_atomic(address, value, condition, interval, timeout) \ + ({ (value) = *(u32 *)(address); (condition) ? 0 : -110; }) +''' + suffix = r''' +static void initial(struct apple_atcphy *phy) +{ + memset(registers, 0, sizeof(registers)); + memset(events, 0, sizeof(events)); + registers[J700_AUX_SUPPLY_CONTROL / 4] = 0x80000200; + registers[J700_AUX_ANALOG_CONTROL / 4] = 0x40000001; + *phy = (struct apple_atcphy){ .regs.core = (unsigned char *)registers }; + writes = reject = 0; +} +int main(void) +{ + struct apple_atcphy phy; + initial(&phy); + assert(j700_aux_disable(&phy) == 0 && writes == 0); + assert(j700_aux_enable(&phy) == 0 && phy.aux_power_requested && writes == 4); + assert(registers[J700_AUX_SUPPLY_CONTROL / 4] == 0x8000010f); + assert(registers[J700_AUX_ANALOG_CONTROL / 4] == 0x40000000); + assert(events[0].mask == 3 && events[1].mask == 12); + assert(events[2].mask == 0x300 && events[2].value == 0x100); + assert(events[3].offset == J700_AUX_ANALOG_CONTROL); + writes = 0; + assert(j700_aux_disable(&phy) == 0 && !phy.aux_power_requested && writes == 4); + assert(events[0].offset == J700_AUX_ANALOG_CONTROL); + assert(events[1].mask == 0x300 && events[2].mask == 8 && events[3].mask == 2); + assert(registers[J700_AUX_SUPPLY_CONTROL / 4] == 0x80000305); + assert(registers[J700_AUX_ANALOG_CONTROL / 4] == 0x40000001); + for (unsigned int fault = 1; fault <= 4; fault++) { + initial(&phy); + reject = fault; + assert(j700_aux_enable(&phy) == -110 && phy.aux_power_requested); + reject = 0; + writes = 0; + assert(j700_aux_disable(&phy) == 0 && !phy.aux_power_requested); + initial(&phy); + assert(j700_aux_enable(&phy) == 0); + writes = 0; + reject = fault; + assert(j700_aux_disable(&phy) == -110 && phy.aux_power_requested); + } + initial(&phy); + assert(j700_pma_idle(&phy, 2) == -EINVAL && writes == 0); + for (unsigned int lane = 0; lane < 2; lane++) { + initial(&phy); + registers[J700_PMA_POWER_CONTROL / 4] = 0x800000ee; + registers[J700_PMA_RECEIVER_CONTROL / 4] = 0x5000; + assert(j700_pma_idle(&phy, lane) == 0 && writes == 6); + assert(((registers[J700_PMA_POWER_CONTROL / 4] >> (4 * lane)) & 15) == 1); + assert(((registers[J700_PMA_POWER_CONTROL / 4] >> (4 * (1 - lane))) & 15) == 14); + assert(registers[J700_PMA_POWER_CONTROL / 4] & 0x80000000); + assert(registers[J700_PMA_RECEIVER_CONTROL / 4] == (0x5000U | (0x44U << lane))); + for (unsigned int fault = 1; fault <= 6; fault++) { + initial(&phy); + registers[J700_PMA_POWER_CONTROL / 4] = 0xee; + reject = fault; + assert(j700_pma_idle(&phy, lane) == -110); + } + } + puts("PASS: sequencing, unrelated bits, every failed phase, retained cleanup ownership"); + return 0; +} +''' + return prefix + production + suffix diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py new file mode 100644 index 00000000000000..66c58178be9d49 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -0,0 +1,453 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual PM/HCD/mux/probe boundaries using the shared PHY extraction fixture.""" +from pathlib import Path +import os, re, subprocess +import phy +from common import block + +def ownership(root, usb_unreachable=False): + ns=phy.parking(root) + s=ns['atc'];has_pm='static int atcphy_usb3_pm_get(' in s + prod=ns['production'].replace('struct apple_atcphy {','struct apple_atcphy {\n struct device_node *np;',1) + if has_pm: prod=prod.replace('static int atcphy_mux_set(', 'static int atcphy_mux_set_locked(',1) + # Non-mux fixture entry points continue to exercise actual source. Only the new + # wrapper owns PM; the fixture's original helpers call it through this declaration. + prefix=ns['prefix']+'\nstruct device_node;\nstatic int atcphy_mux_set(struct typec_mux_dev *, struct typec_mux_state *);\n' + prefix=prefix.replace('struct typec_mux_dev { struct apple_atcphy *data; };', + '''struct typec_mux_dev { struct apple_atcphy *data; + int (*set)(struct typec_mux_dev *, struct typec_mux_state *); }; + struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; }; + #define IS_ERR_OR_NULL(p) (!(p))''') + prefix=prefix.replace('#define guard(kind) (void)',r''' +struct lock_guard { int *lock; }; +static struct lock_guard lock_take(int *lock) { assert(!*lock); *lock=1; return (struct lock_guard){lock}; } +static void lock_release(struct lock_guard *g) { assert(*g->lock==1); *g->lock=0; } +#define guard(kind) struct lock_guard __attribute__((cleanup(lock_release))) _guard = lock_take +''') + # Basic USB/device APIs are modeled, but get/put/wrapper and all parking/ + # restoration bodies are extracted verbatim from the chosen public source. + pmfixture=r''' +#define CONFIG_USB 1 +#define READ_ONCE(x) (x) +#define IS_REACHABLE(x) USB_REACHABLE +struct device_node { int id; }; +struct device { int refs; }; +struct platform_device { struct device dev; }; +struct usb_device; +struct usb_hcd; +struct usb_bus { struct device *sysdev, *controller; struct usb_hcd *hcd; struct usb_device *root_hub; }; +struct usb_hcd { unsigned int rh_registered:1; + struct usb_bus self; unsigned long flags; + struct usb_hcd *shared_hcd,*primary_hcd; struct {bool pending;} died_work; }; +enum { HCD_FLAG_RH_RUNNING,HCD_FLAG_DEAD,HCD_FLAG_POLL_RH,HCD_FLAG_HW_ACCESSIBLE }; +static int __attribute__((unused)) hcd_root_hub_lock; +#define spin_lock_irqsave(lock, flags) do { assert(!*(lock));*(lock)=1;(flags)=0; } while(0) +#define spin_unlock_irqrestore(lock, flags) do { assert(*(lock));*(lock)=0;(void)(flags); } while(0) +#define clear_bit(bit, p) (*(p)&=~(1UL<<(bit))) +#define set_bit(bit, p) (*(p)|=1UL<<(bit)) +#define usb_hcd_is_primary_hcd(hcd) (!(hcd)->primary_hcd||(hcd)->primary_hcd==(hcd)) +#define usb_set_device_state(u, s) ((u)->state=(s)) +#define usb_kick_hub_wq(u) ((void)(u)) +#define schedule_work(w) ((w)->pending=true) +#define bus_to_hcd(bus) ((bus)->hcd) +#define HCD_DEAD(hcd) (!!((hcd)->flags&(1UL<flags&(1UL<refs>0);d->refs--; } +static struct usb_device *usb_get_dev(struct usb_device *u) { u->refs++;return u; } +static void usb_put_dev(struct usb_device *u) { assert(u->refs>0);u->refs--; } +static int usb_for_each_dev(void *data,int (*fn)(struct usb_device *,void *)) { if(root_missing)return 0;for(unsigned int i=0;i<4;i++){int r=fn(&roots[i],data);if(r)return r;}return 0; } +static void usb_lock_device(struct usb_device *u) { assert(u==&roots[3]&&!root_locked);root_locked=true;if(detach_on_lock)u->state=0; } +static void usb_unlock_device(struct usb_device *u) { assert(u==&roots[3]&&root_locked);root_locked=false; } +static int pm_runtime_resume_and_get(struct device *d) { + assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock);resumes++; + /* A re-entrant PHY lock acquisition during resume must remain possible. */ + struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); + if(resume_error)return resume_error; + pm_refs++;stuck_ack=false;if(detach_on_resume)roots[3].state=0; + if(inaccessible_on_resume)clear_bit(HCD_FLAG_HW_ACCESSIBLE,&hcd.flags); + if(dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); + return 0; +} +static void pm_runtime_put_noidle(struct device *d) { assert(d==&roots[3].dev&&pm_refs==1);pm_refs--; } +static void usb_mark_last_busy(struct usb_device *u) { assert(u==&roots[3]&&root_locked&&!pm_phy->lock); } +static void pm_runtime_put_autosuspend(struct device *d) { assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock&&pm_refs==1);pm_refs--; } +''' + suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) + controls=r''' +static void prepare(struct apple_atcphy *p,bool fixed,bool t8140,bool swap) +{ + hw.has_t8140_dp=t8140;initial(p,fixed,false,swap); + p->np=&ctrl_node;pm_phy=p; + hcd=(struct usb_hcd){.rh_registered=true,.flags=1UL<lock&&!root_locked&&!pm_refs&&!node_refs&&!controller.dev.refs); + for(unsigned int i=0;i<4;i++)assert(!roots[i].refs); +} +int main(void) +{ + struct apple_atcphy p; +#if !USB_REACHABLE + { prepare(&p,true,true,false);unsigned int before=writes; + struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; + struct typec_mux_dev *providers[]={&provider};struct typec_mux mux={1,providers}; + struct typec_mux_state state={.mode=TYPEC_STATE_SAFE}; + assert(typec_mux_set(&mux,&state)==-ENODEV&&writes==before);balanced(&p); + puts("PASS USB-unreachable fixed route blocks before MMIO");return 0; } +#endif + /* HC death before rh_registered is set need not mark the root NOTATTACHED. */ + prepare(&p,true,true,false);hcd.rh_registered=false;usb_hc_died(&hcd); + assert(roots[3].state!=USB_STATE_NOTATTACHED&&HCD_DEAD(&hcd));unsigned int before_hcd=writes; + int hcd_ret=safe(&p);fprintf(stderr,"dead HCD admission returned %d, PM resumes %d\n",hcd_ret,resumes); + assert(hcd_ret==-ENODEV&&writes==before_hcd&&resumes==0);balanced(&p); + if(HCD_DEAD_ONLY){puts("PASS dead HCD rejected before PM/MMIO");return 0;} + for(unsigned int gate=0;gate<2;gate++){ + prepare(&p,true,true,false);before_hcd=writes; + if(gate==0)inaccessible_on_resume=1; + if(gate==1)dead_on_resume=1; + assert(safe(&p)==-ENODEV&&writes==before_hcd);balanced(&p); + } + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);stuck_ack=true; + assert(safe(&p)==0);assert(resumes==1);balanced(&p); + } + /* Resume failure must return before ANY lane/core programming. */ + prepare(&p,true,true,false);unsigned int before=writes;resume_error=-EIO; + unsigned int valid=p.dp_usb_context_valid,dirty=p.dp_usb_context_dirty,owned=p.dp_owned_pmas; + assert(safe(&p)==-EIO&&writes==before&&p.dp_usb_context_valid==valid&&p.dp_usb_context_dirty==dirty&&p.dp_owned_pmas==owned);balanced(&p); + resume_error=0;assert(safe(&p)==0);balanced(&p); + /* PM resume is NOT proof of ACK. Real parking timeout stays fatal. */ + prepare(&p,true,true,false);fail_caller="atcphy_pipehandler_lock";before=writes; + assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3&&p.mode_dirty);balanced(&p); + fail_caller=NULL;assert(safe(&p)==0);balanced(&p); + for(unsigned int missing=0;missing<5;missing++){ + prepare(&p,true,true,false);before=writes; + if(missing==0)graph_missing=true; + if(missing==1)controller_missing=true; + if(missing==2)root_missing=true; + if(missing==3)detach_on_lock=1; + if(missing==4)detach_on_resume=1; + assert(safe(&p)==(missing==0||missing>=3?-ENODEV:-EAGAIN)&&writes==before);balanced(&p); + } + /* Root appears after independent fixed-host probe: missing root is retryable. */ + prepare(&p,true,true,false);root_missing=true;before=writes; + assert(safe(&p)==-EAGAIN&&writes==before);balanced(&p); + root_missing=false;assert(safe(&p)==0);balanced(&p); + /* BIST faults are independent of the runtime clock wake; never force readiness. */ + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);assert(safe(&p)==0);balanced(&p); + orient(&p,!swap);bist_status_stuck=true; + assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==-ETIMEDOUT&&p.mode_dirty&&p.pipe_pending);balanced(&p); + bist_status_stuck=false;recover();assert(select_mode(&p,APPLE_ATCPHY_MODE_USB3_DP)==0);balanced(&p); + } + /* Scope and clean same-mode preserve MMIO guards; fixed no-op briefly owns PM. */ + struct typec_altmode alt={.svid=USB_TYPEC_DP_SID}; + struct typec_mux_dev mux={&p};struct typec_mux_state same={.mode=TYPEC_DP_STATE_D,.alt=&alt}; + prepare(&p,false,true,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); + prepare(&p,true,false,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); + prepare(&p,true,true,false);before=writes; + assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==1);balanced(&p); + puts("PASS actual PM wrapper + actual parking/restore: both PMAs, gated clock, PM error/retry, fatal ACK/retry, graph/controller/root/removal gates, legacy/ordinary and cached mode"); + return 0; +} +''' + helpers='' + if has_pm: + helpers=ns['block'](s,'struct atcphy_usb3_pm {')+';\n' + helpers+='#if IS_REACHABLE(CONFIG_USB)\n'+ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' + helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' + # The fixture extractor adapts the locked body; extract the wrapper directly. + marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() + helpers+=ns['block'](s,marker)+'\n' + else: + # Parent runs its ACTUAL mux callback from production, without a substituted wrapper. + pmfixture=pmfixture.replace('static ', 'static __attribute__((unused)) ') + if usb_unreachable: + pmfixture=pmfixture.replace('static ', 'static __attribute__((unused)) ') + typec_source=(root/'drivers/usb/typec/mux.c').read_text() + typec_caller=ns['block'](typec_source,'int typec_mux_set(')+'\n' + hcd_source=(root/'drivers/usb/core/hcd.c').read_text() + hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' + text=prefix+prod+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls + return text + +def probe(root): + def source(path): + return (root/path).read_text() + + + def function(text, name): + match = re.search(r'^static (?:int|void)\s*\n?' + name + r'\(', text, re.M) + assert match, name + start = match.start() + opening = text.index('{', start) + depth = 1 + end = opening + 1 + while depth: + depth += (text[end] == '{') - (text[end] == '}') + end += 1 + return text[start:end] + + + prefix = r''' +#include +#include +#include +#include +#include +#include +#include +#define EPROBE_DEFER 517 +#define IS_ERR(p) ((intptr_t)(p)<0) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR_OR_NULL(p) (!(p)||IS_ERR(p)) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define guard(kind) (void) +#define TYPEC_STATE_SAFE 0 +#define INIT_DELAYED_WORK(p,fn) ((p)->pending=false) +#define atomic_set(p,n) (*(p)=(n)) +struct work { bool pending; }; +struct fwnode_handle { int refs; }; +struct typec_mux_state { void *alt,*data; unsigned long mode; }; +struct tps6598x { int lock; void *port; }; +struct notifier_block { int (*notifier_call)(struct notifier_block *,unsigned long,void *); }; +''' + pm = source('drivers/usb/typec/tipd/cd321x-pm.h').replace('#include ', '') + mocks = r''' +struct cd321x { + struct tps6598x tps; + struct work update_work,resume_work; + int link_event; + struct cd321x_pm_state pm; + void *mux,*tbt_switch,*port_altmode_dp,*port_altmode_tbt; + struct fwnode_handle *connector_fwnode; + struct typec_mux_state state; + struct notifier_block tbt_notifier; + bool state_valid,tbt_notifier_registered; +}; +static struct fwnode_handle connector; +static int mux_error, notifier_error, switch_error; +static int ports,altmodes,muxes,switches,notifiers,stop_calls; +static bool displayport=true; +static int tps6598x_register_port(struct tps6598x *t,struct fwnode_handle *f) { + (void)f;t->port=t;ports++;return 0; +} +static int cd321x_register_port_altmodes(struct cd321x *c) { + c->port_altmode_dp=c;c->port_altmode_tbt=c;altmodes+=2;return 0; +} +static void *fwnode_typec_mux_get(struct fwnode_handle *f) { (void)f;muxes++;return &muxes; } +static bool fwnode_property_present(struct fwnode_handle *f,const char *s) { (void)f;(void)s;return displayport; } +static struct fwnode_handle *fwnode_find_reference(struct fwnode_handle *f,const char *s,int n) { + (void)f;(void)s;(void)n;connector.refs++;return &connector; +} +static void __attribute__((unused)) fwnode_handle_put(struct fwnode_handle *f) { + if(f){assert(f->refs>0);f->refs--;} +} +static void *fwnode_typec_thunderbolt_switch_get(struct fwnode_handle *f) { + (void)f;if(switch_error)return (void *)(intptr_t)switch_error;switches++;return &switches; +} +static int typec_set_mode(void *p,unsigned long m) { (void)p;assert(m==TYPEC_STATE_SAFE);return mux_error; } +static int cd321x_tbt_notify(struct notifier_block *n,unsigned long e,void *d) { (void)n;(void)e;(void)d;return 0; } +static int typec_thunderbolt_switch_register_notifier(void *s,struct notifier_block *n) { + (void)s;(void)n;if(notifier_error)return notifier_error;notifiers++;return 0; +} +static void typec_thunderbolt_switch_put(void *s) { assert(s&&switches>0);switches--; } +static void typec_mux_put(void *m) { assert(m&&muxes>0);muxes--; } +static void typec_unregister_altmode(void *a) { if(a){assert(altmodes>0);altmodes--;} } +static void typec_unregister_port(void *p) { assert(p&&ports>0);ports--; } +static void cd321x_stop_work(struct cd321x *c) { + stop_calls++;if(c->tbt_notifier_registered){notifiers--;c->tbt_notifier_registered=false;} + cd321x_pm_remove(&c->pm);c->resume_work.pending=c->update_work.pending=false; +} +''' + core = source('drivers/usb/typec/tipd/core.c') + production = function(core, 'cd321x_register_port') + '\n' + function(core, 'cd321x_unregister_port') + controls = r''' +static void reset(struct cd321x *c) { + memset(c,0,sizeof(*c));connector.refs=0; + mux_error=notifier_error=switch_error=0;stop_calls=0; + assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers); +} +static void balanced(void) { + assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers&&!connector.refs); +} +int main(void) { + struct cd321x c; + struct cd321x_pm_state budget; + cd321x_pm_init(&budget);assert(cd321x_pm_ready(&budget)); + for(unsigned int i=0;iport=t;ports++;return 0;', '(void)f;t->port=&probe_port;ports++;return 0;') + # Real class dispatch requires the port's mux; its allocator remains a mock. + declarations = 'struct typec_port { struct typec_mux *mux; };\nstatic struct typec_port probe_port;\n' + dispatch = block((root / 'drivers/usb/typec/class.c').read_text(), 'int typec_set_mode(') + marker = port.index('static int tps6598x_register_port(') + port = port[:marker] + declarations + dispatch + '\n' + port[marker:] + import re + # Only helpers used at this boundary belong in this translation unit; + # the separate probe group exercises the complete retry state machine. + used = {'cd321x_pm_new_connection', 'cd321x_pm_init', 'cd321x_pm_remove', 'cd321x_pm_ready', + 'cd321x_pm_begin_read', 'cd321x_pm_snapshot_ready', + 'cd321x_pm_complete', 'cd321x_pm_can_update'} + for match in list(re.finditer(r'^static inline (?:void|bool) (cd321x_\w+)\(', port, re.M))[::-1]: + if match.group(1) not in used: + body = block(port, match.group()) + port = port[:match.start()] + port[match.start() + len(body):] + port = port.replace('static int mux_error, notifier_error, switch_error;', + 'static int notifier_error, switch_error;') + existing = set(re.findall(r'^\s*#define (\w+)', unit, re.M)) + port = '\n'.join(line for line in port.splitlines() + if not (line.startswith('#define ') and line.split()[1].split('(')[0] in existing)) + controls = r""" +static void port_balance(void) +{ + assert(!ports&&!altmodes&&!muxes&&!switches&&!notifiers&&!connector.refs); +} +int main(void) +{ + struct apple_atcphy p; + struct cd321x c={}; + struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; + struct typec_mux_dev *providers[]={&provider}; + struct typec_mux route={1,providers};probe_port.mux=&route; + prepare(&p,true,true,false);root_missing=true;unsigned int before=writes; + assert(cd321x_register_port(&c.tps,&connector)==-EPROBE_DEFER); + assert(!c.state_valid&&!resumes&&writes==before);balanced(&p);port_balance(); + memset(&c,0,sizeof(c));root_missing=false; + assert(cd321x_register_port(&c.tps,&connector)==0 && c.state_valid && resumes==1); + assert(cd321x_pm_ready(&c.pm) && cd321x_pm_begin_read(&c.pm)); + cd321x_pm_snapshot_ready(&c.pm);cd321x_pm_complete(&c.pm); + assert(cd321x_pm_can_update(&c.pm)); + balanced(&p);cd321x_unregister_port(&c.tps);port_balance(); + memset(&c,0,sizeof(c));prepare(&p,true,true,false);notifier_error=-ENOMEM; + assert(cd321x_register_port(&c.tps,&connector)==-ENOMEM); + balanced(&p);port_balance();notifier_error=0; + memset(&c,0,sizeof(c));prepare(&p,true,true,false);resume_error=-EIO;before=writes; + assert(cd321x_register_port(&c.tps,&connector)==0 && !c.state_valid && writes==before); + balanced(&p);cd321x_unregister_port(&c.tps);port_balance(); + puts("PASS: actual CD321x probe -> typec_set_mode -> typec_mux_set -> bounded PM wrapper; delayed root reprobe and rollback references"); +} +""" + return unit + '\n' + port + '\n' + controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py new file mode 100644 index 00000000000000..32585f1755845f --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -0,0 +1,1125 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual parser, bounded rate admission, catalog replacement and HPD.""" +from pathlib import Path +import copy, struct +from common import block, source_defines + +def parser_fixture(root): + + base = root / 'drivers/gpu/drm/apple' + fixture = {'MaxHorizontalImageSize': 60, 'MaxVerticalImageSize': 33, 'Chromaticity': {'Red': {'X': 44608, 'Y': 20288}, 'Green': {'X': 16960, 'Y': 40448}, 'Blue': {'X': 9344, 'Y': 5184}}, 'DefaultWhitePoint': {'X': 20544, 'Y': 21568, 'Gamma': 144179}, 'WhitePoints': [{'X': 20544, 'Y': 21568, 'Gamma': 144179}], 'DefaultColorSpaceIsSRGB': True, 'SupportsPQEOTF': True, 'ProductAttributes': {'LegacyManufacturerID': 1138, 'ProductID': 2895, 'SerialNumber': 305419896, 'WeekOfManufacture': 7, 'YearOfManufacture': 2013, 'AlphanumericSerialNumber': 'ANON000000001', 'ProductName': 'QG270 S3', 'ManufacturerID': 'ACR'}} + + + class ObjectPairs(list): + pass + + + def wire(value, out): + out += b'\0' * (-len(out) % 4) + if isinstance(value, bool): + out += struct.pack('', '') + timing_fixture = [{ + "HorizontalAttributes": {"Total": 2200, "FrontPorch": 88, "SyncWidth": 44, + "Active": 1920, "PreciseSyncRate": 4423680, "SyncPolarity": 1, "PixelRepetition": 0}, + "VerticalAttributes": {"Total": 1125, "FrontPorch": 4, "SyncWidth": 5, + "Active": 1080, "PreciseSyncRate": 3932160, "SyncPolarity": 1, "PixelRepetition": 0}, + "ColorModes": [{"ID": 85, "Score": 1, "IsVirtual": False, "PixelEncoding": 0, + "Colorimetry": 10, "Depth": 10, "DynamicRange": 0, "EOTF": 0}], + "ID": 62, "Score": 16224, "IsPreferred": True, "IsInterlaced": False, "IsVirtual": False, + }] + timing_raw = bytearray(struct.pack(' +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; +typedef int64_t s64; +typedef u16 __le16; +typedef u16 __be16; +typedef u32 __le32; +#define __packed __attribute__((packed)) +#undef static_assert +#define static_assert(c) _Static_assert(c, #c) +#define IS_ENABLED(x) 0 +#define U32_MAX UINT32_MAX +#define U16_MAX UINT16_MAX +#define S64_MAX INT64_MAX +#define S64_MIN INT64_MIN +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +#define round_up(n, a) (((n) + (a) - 1) & ~((a) - 1)) +#define ERR_PTR(n) ((void *)(intptr_t)(n)) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IS_ERR(p) ((uintptr_t)(p) > (uintptr_t)-4096) +#define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) +#define GFP_KERNEL 0 +#define kmalloc(n, f) malloc(n) +#define kcalloc(n, s, f) calloc(n,s) +#define kfree(p) free(p) +#define mul_u32_u32(a, b) ((u64)(u32)(a) * (u32)(b)) +#define DIV_ROUND_CLOSEST_ULL(n, d) (((n) + (d) / 2) / (d)) +#define DIV_ROUND_CLOSEST(n, d) (((n) + (d) / 2) / (d)) +#define trace_iomfb_color_mode(...) ((void)0) +#define trace_iomfb_parse_mode_fail(...) ((void)0) +#define trace_iomfb_parse_mode_success(...) ((void)0) +#define trace_iomfb_timing_mode(...) ((void)0) +#define DRM_MODE_TYPE_DRIVER 1 +#define DRM_MODE_TYPE_PREFERRED 2 +#define DRM_MODE_FLAG_PHSYNC 1 +#define DRM_MODE_FLAG_NHSYNC 2 +#define DRM_MODE_FLAG_PVSYNC 4 +#define DRM_MODE_FLAG_NVSYNC 8 +#define cpu_to_le16(v) (v) +#define cpu_to_le32(v) (v) +#define cpu_to_be16(v) __builtin_bswap16(v) +struct drm_display_mode { + unsigned int type, flags; + int clock, vdisplay, vsync_start, vsync_end, vtotal; + int hdisplay, hsync_start, hsync_end, htotal, width_mm, height_mm; +}; +static void drm_mode_set_name(struct drm_display_mode *mode) { (void)mode; } +''' + stubs = r''' +struct drm_edid { struct edid raw; }; +static const struct drm_edid *drm_edid_alloc(const void *data, size_t size) +{ + assert(size == 128); + struct drm_edid *result = malloc(sizeof(*result)); + memcpy(&result->raw, data, size); + return result; +} +static void drm_edid_free(const struct drm_edid *edid) { free((void *)edid); } +static bool drm_edid_valid(const struct drm_edid *edid) +{ + const u8 *raw = (const u8 *)&edid->raw; + unsigned int sum = 0; + for (unsigned int i = 0; i < 128; i++) sum += raw[i]; + return !(sum & 255) && !raw[126] && !memcmp(raw, "\0\xff\xff\xff\xff\xff\xff\0", 8); +} +''' + suffix = r''' +static void epochs(void) +{ + struct dcp_metadata_epoch e = {0}; + struct dcp_metadata_ticket before = dcp_metadata_begin(&e); + assert(!dcp_metadata_matches(&e, before)); + dcp_metadata_epoch_connection(&e, 1, true); + struct dcp_metadata_ticket current = dcp_metadata_begin(&e); + assert(dcp_metadata_matches(&e, current)); + assert(!dcp_metadata_matches(&e, before)); /* Never retag pre-association metadata. */ + dcp_metadata_epoch_connection(&e, 1, true); + assert(dcp_metadata_matches(&e, current)); + dcp_metadata_epoch_connection(&e, 2, false); + assert(!dcp_metadata_matches(&e, current)); + dcp_metadata_epoch_connection(&e, 3, true); + assert(!dcp_metadata_matches(&e, dcp_metadata_begin(&e))); /* Later attachment ineligible. */ + e = (struct dcp_metadata_epoch){0}; + dcp_metadata_epoch_connection(&e, 9, true); + current = dcp_metadata_begin(&e); + dcp_metadata_epoch_connection(&e, 10, true); /* A route change also invalidates. */ + assert(!dcp_metadata_matches(&e, current)); +} +int main(int argc, char **argv) +{ + if (argc == 3 && !strcmp(argv[1], "--property")) { + FILE *f=fopen(argv[2],"rb"); assert(f && !fseek(f,0,SEEK_END)); + long n=ftell(f); assert(n>20 && !fseek(f,0,SEEK_SET)); + unsigned char *b=malloc(n); assert(fread(b,n,1,f)==1); fclose(f); + assert(dcp_property_complete(b,n-1,n)); + assert(dcp_property_complete(b,n-1,n-1)); + for(unsigned int cut=1;cut<=16;cut++) + assert(!dcp_property_complete(b,n-1-cut,n-cut)); + assert(!dcp_property_complete(b,n-1,n+1)); + free(b); return 0; + } + assert(argc == 4 || argc == 5); + /* A missing final string byte must not be supplied by zero allocation. */ + unsigned char incomplete[]={0xd3,0,0,0,1,0,0,0x82,1,0,0,0x89,0x42}; + assert(dcp_property_complete(incomplete,sizeof(incomplete),sizeof(incomplete))); + assert(!dcp_property_complete(incomplete,12,13)); + assert(!dcp_property_complete(NULL,0,0)); + epochs(); + timing_proof(); + FILE *f = fopen(argv[1], "rb"); + assert(f && !fseek(f, 0, SEEK_END)); + long length = ftell(f); + assert(length > 0 && !fseek(f, 0, SEEK_SET)); + void *data = malloc(length); + assert(fread(data, length, 1, f) == 1); + fclose(f); + struct dcp_parse_ctx ctx; + struct dcp_display_metadata metadata, unchanged; + memset(&metadata, 0xa5, sizeof(metadata)); unchanged = metadata; + assert(!parse(data, length, &ctx)); + int ret = parse_display_metadata(&ctx, &metadata); + if (atoi(argv[2])) { + assert(ret == -EINVAL && !memcmp(&metadata, &unchanged, sizeof(metadata))); + free(data); return 0; + } + assert(!ret); + assert(dcp_property_complete(data,length,length)); + size_t aligned=round_up((size_t)length,4); + unsigned char *padded=malloc(aligned+4); + memset(padded,0xa5,aligned+4); memcpy(padded,data,length); + assert(dcp_property_complete(padded,aligned,aligned+1)); + assert(!dcp_property_complete(data,length-1,length)); + assert(!dcp_property_complete(padded,aligned,aligned+2)); + assert(!dcp_property_complete(padded,aligned+4,aligned+5)); + assert(!dcp_property_complete(padded,aligned-1,aligned+1)); + free(padded); + struct dcp_parse_ctx timing_ctx; + unsigned int count = 0; + assert(!parse(timing_blob, sizeof(timing_blob), &timing_ctx)); + struct dcp_display_mode *timings = enumerate_modes(&timing_ctx, &count, + metadata.width_cm * 10, metadata.height_cm * 10, 0, false, true); + assert(!IS_ERR(timings) && count == 1 && timings[0].metadata_timing_valid); + struct drm_display_mode mode = timings[0].mode; + assert(mode.flags == (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC)); + assert(mode.clock == 148500 && mode.hdisplay == 1920 && mode.vdisplay == 1080); + free(timings); + const struct drm_edid *edid = dcp_metadata_edid(&metadata, &mode); + assert(!IS_ERR(edid)); + f = fopen(argv[3], "wb"); assert(f); + assert(fwrite(&edid->raw, 128, 1, f) == 1); fclose(f); + drm_edid_free(edid); + mode.flags = 0; + assert(IS_ERR(dcp_metadata_edid(&metadata, &mode))); + mode.flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC; + mode.clock++; + assert(IS_ERR(dcp_metadata_edid(&metadata, &mode))); + free(data); + if (argc == 5) { + f=fopen(argv[4],"rb"); assert(f && !fseek(f,0,SEEK_END)); + length=ftell(f); assert(length>0 && !fseek(f,0,SEEK_SET)); + data=malloc(length); assert(fread(data,length,1,f)==1); fclose(f); + assert(!parse(data,length,&timing_ctx)); count=0; + timings=enumerate_modes(&timing_ctx,&count,600,330,0,false,true); + assert(!IS_ERR(timings)); bool found60=false, found120=false; + for(unsigned int i=0;iusb4; } +static void schedule_work(struct work_struct *w) { w->scheduled++; } +static const struct edid *drm_edid_raw(const struct drm_edid *e) { return (const void *)e->data; } +static bool dcpavserv_edid_service_current(struct apple_dcp *d, u64 e) { + assert(d->dcpavserv.lock); return e==d->dcpavserv.epoch && d->dcpavserv.enabled && d->dcpavserv.service; +} +''' + callback_stubs=r''' +#define le64_to_cpu(v) (v) +#define le32_to_cpu(v) (v) +#define LINK_RATE_HBR3 0x1e +static int configure_error; +static int dptxport_call_set_active_lane_count(struct apple_epic_service *s, const void *data, size_t size, void *reply, size_t room) { + if (configure_error) return configure_error; + if(sizecookie)->lane_count=((const struct dptxport_apcall_set_active_lane_count *)data)->lane_count; return 0; +} +static int dptxport_call_set_link_rate(struct apple_epic_service *s, const void *data, size_t size, void *reply, size_t room) { + if(configure_error) return configure_error; + if(sizecookie)->link_rate=((const struct dptxport_apcall_link_rate *)data)->link_rate; return 0; +} +static bool dcp_firmware_metadata_enabled(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } +#define dcp_trace_metadata(d,p,v) ((void)0) +''' + callback_types='typedef u64 __le64;\n'+callback_types + test=r''' +static void checksum(u8 *p) { unsigned sum=0; for(unsigned i=0;i<127;i++) sum+=p[i]; p[127]=-sum; } +static void qualify(struct apple_dcp *d, struct apple_epic_service *s) { + struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, rate_reply={}; + struct dptxport_apcall_set_active_lane_count lanes={.lane_count=2}, lanes_reply={}; + assert(!dptxport_call_qualified_config(s,false,&rate,sizeof(rate),&rate_reply,sizeof(rate_reply))); + assert(!dptxport_call_qualified_config(s,true,&lanes,sizeof(lanes),&lanes_reply,sizeof(lanes_reply))); + (void)d; +} +static bool current(struct apple_dcp *d,const struct dcp_display_mode *m) { + d->hpd_mutex=d->dcpavserv.lock=d->metadata.lock=true; + bool result=dcp_180hz_mode_current(d,m); + d->hpd_mutex=d->dcpavserv.lock=d->metadata.lock=false; + return result; +} +int main(int argc,char **argv) { + struct dcp_metadata_epoch epoch={}; + dcp_metadata_epoch_connection(&epoch,7,true); + assert(dcp_metadata_matches(&epoch,dcp_metadata_begin(&epoch))); + PARSER_CHECKS + struct drm_edid raw={}; u8 *p=raw.data; + struct dcp_display_mode m={}; + if(argc==2) { FILE *f=fopen(argv[1],"rb"); assert(f); assert(fread(p,1,256,f)==256 && fgetc(f)==EOF); fclose(f); } + else { + memcpy(p,"\0\xff\xff\xff\xff\xff\xff\0",8); + p[8]=4;p[9]=0x72;p[10]=0x4f;p[11]=0x0b;p[18]=1;p[19]=4;p[20]=0xa0;p[126]=1; + u8 *d=p+54; unsigned clock=41409;d[0]=clock;d[1]=clock>>8; + d[2]=1920&255;d[3]=160;d[4]=0x70;d[5]=1080&255;d[6]=26;d[7]=0x40; + d[8]=48;d[9]=32;d[10]=0x35;d[17]=0x1e; + p[75]=0xfc;memcpy(p+77,"QG270 S3\n ",13); + u8 *e=p+128;e[0]=2;e[1]=3;e[2]=12;e[4]=0x67;e[5]=0xd8;e[6]=0x5d;e[7]=0xc4;e[8]=1;e[9]=120;e[10]=0x80; + checksum(p);checksum(e); + } + /* Get the advertised DTD's porch/polarity, without copying identity data. */ + const u8 *dt=NULL; + for(unsigned off=54;off+18<=126;off+=18) if((p[off]|p[off+1]<<8)==41409)dt=p+off; + for(unsigned off=p[130];off+18<=127;off+=18) if((p[128+off]|p[129+off]<<8)==41409)dt=p+128+off; + assert(dt); + m.mode=(struct drm_display_mode){.clock=414090,.hdisplay=1920,.htotal=2080,.vdisplay=1080,.vtotal=1106}; + m.mode.hsync_start=1920+(dt[8]|(dt[11]&0xc0)<<2);m.mode.hsync_end=m.mode.hsync_start+(dt[9]|(dt[11]&0x30)<<4); + m.mode.vsync_start=1080+(dt[10]>>4|(dt[11]&0xc)<<2);m.mode.vsync_end=m.mode.vsync_start+((dt[10]&15)|(dt[11]&3)<<4); + m.mode.flags=(dt[17]&2?DRM_MODE_FLAG_PHSYNC:DRM_MODE_FLAG_NHSYNC)|(dt[17]&4?DRM_MODE_FLAG_PVSYNC:DRM_MODE_FLAG_NVSYNC); + assert(dcp_180hz_edid(p,256,&m.mode)); + assert(!dcp_180hz_edid(p,255,&m.mode)); + u8 saved[256];memcpy(saved,p,256); + p[10]^=1;checksum(p);assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + p[8]^=1;checksum(p);assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + unsigned hf=0;for(unsigned off=132;off<128+p[130];off+=(p[off]&31)+1) if(p[off+1]==0xd8 && p[off+2]==0x5d && p[off+3]==0xc4)hf=off; + assert(hf);p[hf+5]=82;checksum(p+128);assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + p[127]^=1;assert(!dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + /* Serial changes do not participate in identity matching. */ + p[12]^=0xff;checksum(p);assert(dcp_180hz_edid(p,256,&m.mode));memcpy(p,saved,256); + assert(dcp_180hz_payload(0x1e,2,414090));assert(!dcp_180hz_payload(0x14,2,414090));assert(!dcp_180hz_payload(0x1e,1,414090)); + struct apple_dcp d={.dev=&d,.phy=&raw,.active_typec_route=&m,.typec_cable_connected=true,.typec_generation=7,.nr_modes=1}; + struct apple_connector con={.dcp=&d,.connected=true,.drm_edid=&raw,.drm_edid_generation=7,.drm_edid_service_epoch=1,.base.registration_state=1}; + struct apple_epic ep={.dcp=&d,.endpoint=0x2c,.wire_v2=true}; + struct apple_epic_service s={.ep=&ep,.cookie=&d.dptxport[0],.enabled=true}; + d.connector=&con;d.hw.iomfb_method_profile=2;d.metadata.epoch.connected=true;d.metadata.epoch.generation=7; + d.dcpavserv.enabled=true;d.dcpavserv.service=&raw;d.dcpavserv.epoch=1; + d.dptxport[0]=(struct dptx_port){.enabled=true,.connected=true,.atcphy=d.phy,.service=&s}; + m.unsafe_rgb8_ticket=(struct dcp_metadata_ticket){.associated=true,.generation=7};m.unsafe_rgb8_service_epoch=1; + m.sdr_rgb=(struct dcp_color_mode){.depth=8,.format=0,.range=0,.eotf=0,.colorimetry=10}; + assert(!current(&d,&m));qualify(&d,&s);assert(current(&d,&m));assert(con.hotplug_wq.scheduled==1); + #define DENY(field,value) do { __typeof__(field) old=field;field=value;assert(!current(&d,&m));field=old;assert(current(&d,&m)); } while(0) + DENY(d.typec_generation,8);DENY(con.drm_edid_generation,6);DENY(d.dcpavserv.epoch,3);DENY(con.drm_edid_service_epoch,3); + /* Timings may precede the very first AV announcement. Only fresh epoch-1 + * EDID plus epoch-1 callback proof may admit this initial receipt. */ + m.unsafe_rgb8_service_epoch=0;m.automatic_service_pending=true;assert(current(&d,&m)); + d.dcpavserv.epoch=3;con.drm_edid_service_epoch=3;qualify(&d,&s); + assert(!current(&d,&m)); /* Initial receipt never survives retirement. */ + m.unsafe_rgb8_service_epoch=3;m.automatic_service_pending=false;assert(current(&d,&m)); + d.dcpavserv.epoch=1;con.drm_edid_service_epoch=1;m.unsafe_rgb8_service_epoch=1;qualify(&d,&s); + DENY(con.drm_edid_firmware_derived,true);DENY(d.usb4,true);DENY(d.dptx_tunnel,true); + DENY(d.quiescing,true);DENY(d.metadata.epoch.connected,false);DENY(d.rate_proof.connection_sequence,1); + DENY(d.rate_proof.rate.value,0x14);DENY(d.rate_proof.lanes.value,1);DENY(ep.endpoint,0x28); + DENY(m.sdr_rgb.depth,10);DENY(m.sdr_rgb.colorimetry,1);DENY(m.sdr_rgb.range,1);DENY(m.mode.clock,414091); + struct dptxport_apcall_link_rate rate={.link_rate=0x1e}, response={}; + configure_error=-EIO; + assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate),&response,sizeof(response))==-EIO); + assert(!current(&d,&m));configure_error=0;qualify(&d,&s); + assert(dptxport_call_qualified_config(&s,false,&rate,sizeof(rate)-1,&response,sizeof(response))==-EINVAL); + assert(!current(&d,&m));qualify(&d,&s); + dcp_firmware_metadata_connection(&d,7,false);dcp_firmware_metadata_connection(&d,7,true); + assert(!current(&d,&m));qualify(&d,&s);assert(!current(&d,&m)); + m.automatic_connection_sequence=d.rate_proof.connection_sequence;assert(current(&d,&m)); + dcp_firmware_metadata_hotplug(&d,true);dcp_firmware_metadata_hotplug(&d,false); + assert(!current(&d,&m)); + dcp_firmware_metadata_connection(&d,7,true);m.automatic_connection_sequence=d.rate_proof.connection_sequence;con.edid_invalidation_pending=false;qualify(&d,&s);assert(current(&d,&m)); + struct dcp_180hz_token old=dcp_180hz_config_begin(&s,false);d.typec_generation++; + dcp_180hz_config_complete(&s,false,old,0x1e,true);assert(!d.rate_proof.rate.valid);d.typec_generation--;qualify(&d,&s); + old=dcp_180hz_config_begin(&s,true);d.dcpavserv.epoch+=2; + dcp_180hz_config_complete(&s,true,old,2,true);assert(d.rate_proof.lanes.valid);assert(!current(&d,&m));d.dcpavserv.epoch-=2;qualify(&d,&s); + old=dcp_180hz_config_begin(&s,false);dcp_180hz_invalidate(&d); + dcp_180hz_config_complete(&s,false,old,0x1e,true);assert(!current(&d,&m));qualify(&d,&s); + old=dcp_180hz_config_begin(&s,false);dcp_180hz_config_complete(&s,false,old,0x1e,false);assert(!current(&d,&m)); + d.dcpavserv.epoch=0;old=dcp_180hz_config_begin(&s,false);d.dcpavserv.epoch=1; + dcp_180hz_config_complete(&s,false,old,0x1e,true);assert(d.rate_proof.rate.valid);qualify(&d,&s);assert(current(&d,&m)); + puts("PASS: actual binary parser, Acer EDID, capacity, callback success, late-proof event, generation/AV lifetime and foreign-sink controls"); +} +'''.replace('PARSER_CHECKS', checks) + unit=namespace['prefix']+namespace['metadata']+namespace['epoch']+namespace['header']+namespace['edid_header']+namespace['parser']+blobs+policy+proof+state+source+callback_types+callback_stubs+callback+connection+test + return {**locals(), "block": block} + +def publication(root): + + n = policy(root) + n.update({key:n['namespace'][key] for key in ('prefix','metadata','epoch','header','edid_header','parser')}) + base = n['base'] + block = n['block'] + av = (base/'epic/dpavservep.c').read_text() + iomfb = (base/'iomfb.c').read_text() + template = (base/'iomfb_template.c').read_text() + + state = n['state'] + state = state.replace('struct apple_connector {', '''struct drm_device { void *dev; struct { bool mutex; } mode_config; }; +struct drm_connector { struct drm_device *dev; int registration_state; void *state; + struct { struct { unsigned min_vfreq,max_vfreq; } monitor_range; } display_info; +}; +struct apple_connector {''') + state = state.replace(' struct { int registration_state; } base;', ' struct drm_connector base;') + state = state.replace('struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled; };', '''struct apple_epic_service { struct apple_epic *ep; void *cookie; bool enabled,torndown; + unsigned users,channel; struct { u64 generation,epoch; } admission; +};''') + state = state.replace('struct { bool lock, enabled; void *service; u64 epoch; } dcpavserv;', 'struct { bool lock, enabled; void *service; u64 epoch; int enable_completion; } dcpavserv;') + state = state.replace('u64 typec_generation; unsigned int nr_modes;', '''u64 typec_generation; unsigned int nr_modes; + struct dcp_display_mode *modes; + struct apple_epic *dcpavservep; + int fw_compat,width_mm,height_mm,fixed_connector_type; unsigned notch_height; + struct { size_t length,received; void *data; bool contiguous,payload_complete; + u64 av_service_epoch,automatic_connection_sequence; bool automatic_service_pending; + struct dcp_metadata_ticket metadata_ticket,mode_ticket; + } chunks;''') + + start = state.index('static bool dcpavserv_edid_service_current(') + state = state[:start] + + stubs = r''' +#define WRITE_ONCE(p,v) ((p)=(v)) +#define platform_device apple_dcp +#define platform_get_drvdata(p) ((struct apple_dcp *)(p)) +#define to_apple_connector(p) ((struct apple_connector *)((char *)(p)-offsetof(struct apple_connector,base))) +#define xchg(p,v) __extension__ ({ __typeof__(*(p)) old=*(p); *(p)=(v); old; }) +#define DRM_MODE_MATCH_TIMINGS 1 +#define DRM_MODE_MATCH_CLOCK 2 +#define DCP_FIRMWARE_H17P 2 +#define DPAVSERV_ENDPOINT_V2 0x2a +#define DRM_CONNECTOR_eDP 14 +#define DRM_MODE_CONNECTOR_eDP 14 +#define DCP_FW_VER 26 +#define DCP_FW_VERSION(a,b,c) (a) +#define SZ_1M (1024*1024) +#define kzalloc(n,f) calloc(1,n) +#define dev_err(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define trace_dcpavserv_init(...) ((void)0) +#include +static bool *hpd_lock,*av_lock; +static void mutex_lock(bool *p) { assert(!*p); if(p==hpd_lock) assert(!*av_lock); *p=true; } +static void mutex_unlock(bool *p) { assert(*p); *p=false; } +static void complete(int *p) { (*p)++; } +static void reinit_completion(int *p) { *p=0; } +static struct apple_epic_service *afk_service_get(struct apple_epic_service *s) { if(!s->enabled) return NULL; s->users++; return s; } +static void afk_service_put(struct apple_epic_service *s) { assert(s->users); s->users--; } +static void afk_service_retire(struct apple_epic_service *s) { assert(!*hpd_lock); s->enabled=false; s->torndown=true; } +static void dpavservep_detach(struct apple_dcp *d); +static struct apple_dcp *probing; +static struct drm_edid transport_edid; +static unsigned raw_allocations,published,reads,retirement_during_read; +static bool fail_update,invalidate_during_duplicate,replace_catalog_during_duplicate; +static void conflicting_catalog_arrived(void); +static const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *d,u64 generation,u64 *epoch) { + assert(!d->hpd_mutex && !d->dcpavserv.lock && !d->metadata.lock); + if(!d->dcpavserv.enabled || !d->dcpavserv.service || generation!=d->typec_generation) return ERR_PTR(-ENODEV); + *epoch=d->dcpavserv.epoch;reads++; + struct drm_edid *e=malloc(sizeof(*e)); assert(e); *e=transport_edid;raw_allocations++; + if(retirement_during_read) { retirement_during_read--;dpavservep_detach(d); } + return e; +} +static void drm_edid_free(const struct drm_edid *e) { if(e) { assert(raw_allocations);raw_allocations--;free((void *)e); } } +static int drm_edid_connector_update(struct drm_connector *c,const struct drm_edid *e) { + assert(c->dev->mode_config.mutex); (void)e; return fail_update?-EIO:0; +} +static const struct drm_edid *dcp_firmware_edid(struct apple_dcp *d,u64 *generation) { (void)d; (void)generation; return ERR_PTR(-ENODATA); } +static bool dcp_firmware_metadata_current(struct apple_dcp *d,u64 generation) { (void)d; (void)generation; return false; } +static bool dcp_has_panel(struct apple_dcp *d) { return d->fixed_connector_type==DRM_CONNECTOR_eDP; } +static struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,const struct drm_display_mode *m) { + assert(dev->mode_config.mutex);struct drm_display_mode *copy=malloc(sizeof(*copy));assert(copy);*copy=*m; + if(invalidate_during_duplicate) { invalidate_during_duplicate=false;dcp_180hz_invalidate(probing); } + if(replace_catalog_during_duplicate) { replace_catalog_during_duplicate=false;conflicting_catalog_arrived(); } + return copy; +} +static void drm_mode_probed_add(struct drm_connector *c,struct drm_display_mode *m) { + assert(c->dev->mode_config.mutex && probing->hpd_mutex && probing->dcpavserv.lock && probing->metadata.lock); + published++;free(m); +} +static void drm_mode_destroy(struct drm_device *dev,struct drm_display_mode *m) { (void)dev; free(m); } +static bool drm_mode_match(const struct drm_display_mode *a,const struct drm_display_mode *b,unsigned flags) { + (void)flags;return a->clock==b->clock && a->hdisplay==b->hdisplay && a->vdisplay==b->vdisplay && a->htotal==b->htotal && a->vtotal==b->vtotal && a->hsync_start==b->hsync_start && a->hsync_end==b->hsync_end && a->vsync_start==b->vsync_start && a->vsync_end==b->vsync_end && a->flags==b->flags; +} +static void drm_connector_set_vrr_capable_property(struct drm_connector *c,bool capable) { (void)c; (void)capable; } +static void dcp_retry_placeholder_edid(struct apple_dcp *d,const struct drm_edid *e) { (void)d; (void)e; } +static unsigned pending_probes; +void dcp_180hz_retry(struct apple_dcp *d) { (void)d;pending_probes++; } +static void dcp_firmware_metadata_begin(struct apple_dcp *d) { + unsigned long flags;spin_lock_irqsave(&d->metadata.lock,flags); + d->chunks.metadata_ticket=dcp_metadata_begin(&d->metadata.epoch);spin_unlock_irqrestore(&d->metadata.lock,flags); +} +static void dcp_firmware_metadata_timing(struct apple_dcp *d) { (void)d; } +static void dcp_firmware_metadata_attributes(struct apple_dcp *d,const struct dcp_display_metadata *m) { (void)d; (void)m; } +static void dcp_set_dimensions(struct apple_dcp *d) { (void)d; } +struct dcp_set_dcpav_prop_end_req { char key[64]; }; +''' + transport = '\n'.join(block(av,marker) for marker in [ + 'static void dcpavserv_notify_edid(', 'static bool dcpavserv_identity(', + 'static bool dcpavserv_admission_current(', 'static int dcpavserv_prepare(', + 'static void dcpavserv_init(', 'static void dcpavserv_invalidate_edid(', + 'static void dcpavserv_teardown(', 'void dpavservep_detach(', + 'bool dcpavserv_edid_current(', 'bool dcpavserv_edid_service_current(', + ]) + mode_helpers = iomfb[iomfb.index('static struct dcp_display_mode *dcp_modes_snapshot('):iomfb.index('int dcp_get_modes(')] + publication = block(iomfb,'static void dcp_clear_connector_edid(')+'\n'+block(iomfb,'static void dcp_consume_edid_invalidation(')+mode_helpers+block(iomfb,'int dcp_get_modes(')+block(iomfb,'struct dcp_display_mode *lookup_mode(') + # Drive malformed replacements through the same chunk/end ingress as firmware. + # The dictionary cache is a boundary substitute and takes ownership of the blob. + receipt = r''' +struct dcp_set_dcpav_prop_chunk_req { u32 offset,length; u8 data[4096]; }; +#define dcp_connector_update_dict(con,key,chunks) do { (void)(con); (void)(key); free((chunks)->data); } while (0) +''' + '\n'.join(block(template,marker) for marker in [ + 'static u8 dcpep_cb_prop_start(', 'static u8 dcpep_cb_prop_chunk(', + 'static bool dcpep_process_chunks(', 'static u8 dcpep_cb_prop_end(']) + + main_prefix = n['test'].split('int main(int argc,char **argv) {',1)[1].split(' struct apple_dcp d={',1)[0] + helpers = n['test'].split('int main(int argc,char **argv) {',1)[0] + helpers = helpers.replace(block(helpers, 'static void qualify('), '') + fixture = r''' +static struct apple_dcp d; +static struct apple_connector con; +static struct drm_device drm; +static struct apple_epic ep,av_ep; +static struct apple_epic_service s,av_services[3]; +static void reset(void) { + assert(!raw_allocations); + d=(struct apple_dcp){.dev=&d,.phy=&transport_edid,.active_typec_route=&con,.typec_cable_connected=true,.typec_generation=7,.fw_compat=2,.width_mm=600,.height_mm=330}; + drm=(struct drm_device){}; + con=(struct apple_connector){.dcp=&d,.connected=true,.base={.dev=&drm,.registration_state=1}}; + ep=(struct apple_epic){.dcp=&d,.endpoint=0x2c,.wire_v2=true}; + av_ep=(struct apple_epic){.dcp=&d,.endpoint=0x2a,.wire_v2=true}; + s=(struct apple_epic_service){.ep=&ep,.cookie=&d.dptxport[0],.enabled=true}; + for(unsigned i=0;i<3;i++) av_services[i]=(struct apple_epic_service){.ep=&av_ep,.enabled=true,.channel=1+4*i}; + d.connector=&con;d.hw.iomfb_method_profile=2;d.metadata.epoch.connected=true;d.metadata.epoch.generation=7;dcp_metadata_epoch_connection(&d.metadata.epoch,7,true); + d.dptxport[0]=(struct dptx_port){.enabled=true,.connected=true,.atcphy=d.phy,.service=&s}; + d.dcpavservep=&av_ep;hpd_lock=&d.hpd_mutex;av_lock=&d.dcpavserv.lock;probing=&d; + published=reads=pending_probes=0;fail_update=invalidate_during_duplicate=false; +} +static void admit(unsigned index) { + struct apple_epic_service *a=&av_services[index]; + assert(!dcpavserv_prepare(a,"dcpav-service-epic","AppleDCPDP2HDMI",a->channel)); + dcpavserv_init(a,"dcpav-service-epic","AppleDCPDP2HDMI",a->channel); + assert(d.dcpavserv.enabled && d.dcpavserv.service==a); +} +static void start_receipt(void) { + u32 size=sizeof(input_0);assert(dcpep_cb_prop_start(&d,&size)); + memcpy(d.chunks.data,input_0,size);d.chunks.received=size;d.chunks.contiguous=true; +} +static void finish_receipt(void) { + struct dcp_set_dcpav_prop_end_req request={.key="TimingElements"}; + assert(dcpep_process_chunks(&d,&request));assert(d.nr_modes==1 && d.modes[0].automatic_rgb8_quirk); + free(d.chunks.data);d.chunks.data=NULL; +} +static void timings(void) { start_receipt();finish_receipt(); } +static void callbacks(bool lanes) { + if(lanes) { struct dptxport_apcall_set_active_lane_count a={.lane_count=2},b={}; + assert(!dptxport_call_qualified_config(&s,true,&a,sizeof(a),&b,sizeof(b))); } + else { struct dptxport_apcall_link_rate a={.link_rate=0x1e},b={}; + assert(!dptxport_call_qualified_config(&s,false,&a,sizeof(a),&b,sizeof(b))); } +} +static void probe(bool expected) { + drm.mode_config.mutex=true;unsigned before=published; + int count=dcp_get_modes(&con.base);drm.mode_config.mutex=false; + assert(count==(int)expected && published==before+(unsigned)expected); + if(d.nr_modes) assert(!!lookup_mode(&d,&d.modes[0].mode)==expected); + assert(!d.hpd_mutex && !d.dcpavserv.lock && !d.metadata.lock); +} +static void cleanup(void) { + drm_edid_free(con.drm_edid);con.drm_edid=NULL;free(d.modes);d.modes=NULL; + free(d.chunks.data);d.chunks.data=NULL; + if(d.dcpavserv.service)dpavservep_detach(&d); + assert(!raw_allocations); +} +static void permutations(void) { + unsigned runs=0; + for(unsigned a=0;a<4;a++) for(unsigned b=0;b<4;b++) for(unsigned c=0;c<4;c++) for(unsigned e=0;e<4;e++) { + if(a==b || a==c || a==e || b==c || b==e || c==e)continue; + reset();unsigned order[]={a,b,c,e};unsigned seen=0; + for(unsigned i=0;i<4;i++) { + unsigned notifications=con.hotplug_wq.scheduled; + switch(order[i]) { case 0:callbacks(false);break;case 1:callbacks(true);break;case 2:timings();break;case 3:admit(0);break; } + seen|=1u<notifications); + } + assert(current(&d,&d.modes[0]));cleanup();runs++; + } + assert(runs==24); +} +''' + main=r''' + transport_edid=raw; + /* Runtime EDID porches are encoded into the timing fixture by this + * harness, without embedding identity bytes or fabricating tickets. */ + permutations(); + reset();callbacks(false);callbacks(true);timings();probe(false);admit(0);probe(true); + assert(d.rate_proof.rate.valid && d.rate_proof.lanes.valid); /* No second callback. */ + struct dcp_display_mode old=d.modes[0]; + dcpavserv_teardown(&av_services[0]);probe(false); + admit(1);probe(false);assert(!memcmp(&old,d.modes,sizeof(old))); /* No retagging. */ + free(d.modes);d.modes=NULL;timings();probe(true); /* Existing physical config survives AV-only CLOSE. */ + dcpavserv_teardown(&av_services[0]);probe(true); /* Old CLOSE cannot revoke the new owner. */ + cleanup(); + + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + dcp_firmware_metadata_connection(&d,8,false);d.typec_generation=8;d.typec_cable_connected=false;dpavservep_detach(&d);probe(false); + d.typec_generation=9;d.typec_cable_connected=true;dcp_firmware_metadata_connection(&d,9,true); + free(d.modes);d.modes=NULL;timings();admit(1);probe(false); /* Fresh EDID cannot rescue old PHY configuration. */ + callbacks(false);probe(false);callbacks(true);probe(true);cleanup(); + + reset();callbacks(false);callbacks(true);start_receipt();dpavservep_detach(&d);admit(0);finish_receipt();probe(false);cleanup(); + reset();callbacks(false);callbacks(true);admit(0);start_receipt();dcpavserv_teardown(&av_services[0]);admit(1);finish_receipt();probe(false);cleanup(); + reset();callbacks(false);callbacks(true);dpavservep_detach(&d);timings();admit(0);probe(true);cleanup(); /* Legitimate receipt awaiting post-retirement admission. */ + + reset();callbacks(false);callbacks(true);timings();admit(0);retirement_during_read=1;probe(false);cleanup(); + reset();callbacks(false);callbacks(true);timings();admit(0);invalidate_during_duplicate=true;probe(false);cleanup(); + reset();callbacks(false);callbacks(true);timings();admit(0);fail_update=true;probe(false);assert(!con.drm_edid);cleanup(); + reset();callbacks(false);callbacks(true);timings();admit(0);transport_edid.data[10]^=1;checksum(transport_edid.data);probe(false);cleanup();transport_edid=raw; + reset();callbacks(false);callbacks(true);timings();admit(0);configure_error=-EIO; + struct dptxport_apcall_link_rate req={.link_rate=0x1e},resp={};assert(dptxport_call_qualified_config(&s,false,&req,sizeof(req),&resp,sizeof(resp))==-EIO); + configure_error=0;probe(false);cleanup(); + puts("PASS: 24 callback/timing/AV arrival permutations and real get_modes/lookup publication; early configuration needs no second callback; CLOSE, physical reconnect, receipt epochs and publication races deny stale proof"); +} +''' + unit=n['prefix']+n['metadata']+n['epoch']+n['header']+n['edid_header']+n['parser']+n['blobs']+n['policy']+n['proof']+state+stubs+transport+n['source']+n['callback_types']+n['callback_stubs']+n['callback']+n['connection']+publication+receipt+helpers+fixture+'int main(int argc,char **argv) {'+main_prefix+main + # Conflicts are scanned across the full binary receipt, including filtered + # siblings. Exercise both orders, replacement receipts and equivalent aliases. + base_fixture=copy.deepcopy(n['fixture']) + conflict_blobs='' + conflict_checks='' + for kind in ('color', 'timing', 'other-color'): + for reverse in (False, True): + candidate=copy.deepcopy(base_fixture) + other=copy.deepcopy(candidate[0]) + if kind=='color': + other_color=copy.deepcopy(other['ColorModes'][0]) + other_color.update(Depth=10,EOTF=2,Score=999999) + candidate[0]['ColorModes'].insert(0 if reverse else 1,other_color) + elif kind=='other-color': + other.update(ID=79,DSCRequiredColorElementIDs=[80]) + other['ColorModes'][0].update(Depth=10,EOTF=2) + candidate.insert(0 if reverse else 1,other) + else: + other['HorizontalAttributes'].update(Total=2200,FrontPorch=88,SyncWidth=44) + other['VerticalAttributes'].update(Total=1125,FrontPorch=4,SyncWidth=5,PreciseSyncRate=60<<16) + other.update(DSCRequiredColorElementIDs=[80]) + candidate.insert(0 if reverse else 1,other) + raw_blob=bytearray(struct.pack('before);assert(!lookup_mode(&d,&saved));free(old); } + cleanup(); + if(late) { + reset();callbacks(false);callbacks(true);admit(0);timings();probe(true);old=d.modes;saved=old[0].mode; + u32 size=len;assert(dcpep_cb_prop_start(&d,&size));memcpy(d.chunks.data,blob,size); + d.chunks.received=size;d.chunks.contiguous=true; + before=con.hotplug_wq.scheduled; + replace_catalog_during_duplicate=true;probe(false); + assert(con.hotplug_wq.scheduled>before); + assert(!current(&d,old) && !lookup_mode(&d,&saved));free(old);cleanup(); + } +} +static void conflicting_receipts(void) { + CONFLICT_CHECKS + reset();callbacks(false);callbacks(true);admit(0);receipt_blob(equivalent_ids,sizeof(equivalent_ids));probe(true);cleanup(); + puts("PASS: conflicting color/timing IDs, discarded siblings, both receipt orders and late conflicts deny getter/lookup; equivalent color aliases remain eligible"); +} +""".replace('CONFLICT_CHECKS',conflict_checks) + unit=unit.replace('int main(int argc,char **argv) {',conflict_blobs+conflict_helper+'int main(int argc,char **argv) {') + unit=unit.replace(' permutations();',' permutations();conflicting_receipts();') + # Execute the production metadata notifier so a manual probe cannot hide a + # missing connector refresh after the last eligible mode is withdrawn. + unit=unit.replace('bool lock, attributes_valid, timing_valid, hpd_seen; struct dcp_metadata_epoch epoch;', + 'bool lock, attributes_valid, timing_valid, hpd_seen; struct dcp_metadata_ticket attributes_ticket,timing_ticket; struct drm_display_mode preferred; struct dcp_metadata_epoch epoch;') + dcp_source=(base/'dcp.c').read_text() + metadata_notifier='\n'.join(block(dcp_source,marker) for marker in ( + 'static void dcp_firmware_metadata_notify(', + 'static bool dcp_firmware_metadata_complete(', + 'void dcp_firmware_metadata_timing(')) + unit=unit.replace(block(unit,'static void dcp_firmware_metadata_timing('), + '#define dcp_trace_metadata(d,p,v) ((void)0)\nstatic bool dcp_firmware_metadata_enabled(struct apple_dcp *d);\n'+metadata_notifier) + + failed_receipts = r''' +static bool malformed_receipt(const char *key,unsigned fault) { + u32 size=fault==1 ? 3 : fault==2 ? 4 : sizeof(input_0); + assert(dcpep_cb_prop_start(&d,&size)); + struct dcp_set_dcpav_prop_chunk_req chunk={.length=size}; + assert(size<=sizeof(chunk.data));memcpy(chunk.data,input_0,size); + if(fault==0)chunk.data[0]^=1; + assert(dcpep_cb_prop_chunk(&d,&chunk)); + struct dcp_set_dcpav_prop_end_req request={}; + strcpy(request.key,key); + bool accepted=dcpep_cb_prop_end(&d,&request); + assert(!d.chunks.data && !d.chunks.length); + return accepted; +} +static void failed_replacements(void) { + for(unsigned fault=0;fault<3;fault++) { + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + struct dcp_display_mode *old=d.modes; + struct drm_display_mode saved=old[0].mode; + unsigned before=con.hotplug_wq.scheduled; + assert(!malformed_receipt("TimingElements",fault)); + assert(!current(&d,old) && !lookup_mode(&d,&saved)); + assert(con.hotplug_wq.scheduled>before);probe(false); + /* Fresh catalog can recover using still-current EDID and link proof. */ + timings();probe(true);free(old);cleanup(); + } + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + assert(!malformed_receipt("DisplayAttributes",0));probe(true);cleanup(); + puts("PASS: actual chunk/end callbacks withdraw malformed replacement exceptions and notify DRM; fresh catalogs recover; unrelated bad properties preserve admission"); +} +''' + unit=unit.replace('int main(int argc,char **argv) {',failed_receipts+'int main(int argc,char **argv) {') + unit=unit.replace(' permutations();conflicting_receipts();',' permutations();conflicting_receipts();failed_replacements();') + return {**locals(), "block": block} + +def hotplug(root): + n = publication(root) + unit = n['unit'] + block = n['block'] + base = root / 'drivers/gpu/drm/apple' + dptx = (base / 'dptxep.c').read_text() + training = (base / 'dptx-j700-training.h').read_text() + unit = unit.replace('struct dptx_port { u32 lane_count, link_rate;', ''' +union phy_configure_opts { struct { unsigned lanes,link_rate; bool set_lanes,set_rate,set_voltages; } dp; }; +struct dptx_port { union phy_configure_opts phy_ops; int linkcfg_completion; u32 pending_link_rate; +u32 lane_count, link_rate;''') + unit = unit.replace('int fw_compat,width_mm,height_mm,fixed_connector_type;', 'unsigned dptx_phy; int fw_compat,width_mm,height_mm,fixed_connector_type;') + real_handlers = r''' +#define cpu_to_le32(v) (v) +#define cpu_to_le64(v) (v) +#define LINK_RATE_RBR 0x06 +#define LINK_RATE_HBR 0x0a +#define LINK_RATE_HBR2 0x14 +#define PHY_MODE_DP 1 +#define trace_dptxport_call_set_link_rate(...) ((void)0) +static unsigned phy_capacity=2, phy_calls, hardware_rate, hardware_lanes; +static int validate_error; +static int phy_validate(void *phy,int mode,int submode,union phy_configure_opts *o) { + assert(phy);(void)mode;(void)submode; + if(validate_error)return validate_error; + o->dp.lanes=phy_capacity;return 0; +} +static int phy_configure(void *phy,union phy_configure_opts *o) { + assert(phy);phy_calls++; + if(configure_error)return configure_error; + if(o->dp.set_lanes)hardware_lanes=o->dp.lanes; + if(o->dp.set_rate)hardware_rate=o->dp.link_rate; + return 0; +} +static void dcp_tunnel_set_rate(struct apple_dcp *d,void *phy,unsigned rate) { + (void)d;(void)phy;(void)rate;assert(false); +} +''' + real_handlers += block(training, 'static int dptx_j700_validate_lanes(') + '\n' + real_handlers += block(dptx, 'static int dptxport_call_set_active_lane_count(') + '\n' + real_handlers += block(dptx, 'static int dptxport_call_set_link_rate(') + '\n' + for marker in ['static int dptxport_call_set_active_lane_count(', 'static int dptxport_call_set_link_rate(']: + unit = unit.replace(block(unit, marker), '') + unit = unit.replace('static int configure_error;', 'static int configure_error;\n' + real_handlers) + unit = unit.replace('published=reads=pending_probes=0;', 'phy_capacity=2;phy_calls=hardware_rate=hardware_lanes=0;validate_error=0;published=reads=pending_probes=0;') + unit = unit.replace('assert(current(&d,&d.modes[0]));cleanup();runs++;', 'assert(current(&d,&d.modes[0]));assert(hardware_rate==8100 && hardware_lanes==2 && phy_calls==2);cleanup();runs++;') + unit = unit.replace('struct apple_dcp {', '''struct apple_dcp { + struct { bool lock,presentation_paused; } iomfb; + struct { bool valid,changing; } mode_state; + struct work_struct external_stop_work,vblank_wq; + bool external_physically_absent,main_display,typec_crtc_off,external_cable,external_error,external_power_attempted,external_stopped; + bool external_hotplug_pending,external_hotplug_connected; + u64 external_stop_generation,external_rx_generation,external_crtc_generation; + unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready;''') + stubs = r''' +#define DCP_HOTPLUG_NOTIFY 1 +static bool dcp_is_external_h17p(struct apple_dcp *d) { return d->hw.iomfb_method_profile==2; } +static bool dcp_is_typec_output(struct apple_dcp *d) { (void)d;return true; } +static void complete_all(int *p) { (*p)++; } +#define dcp_mode_invalidate(s) ((s)->valid=false) +#define dcp_mode_hotplug(s,c,p) __extension__ ({ (void)(s);if(p)*(p)=(c);1u; }) +static void dcp_handle_hotplug_actions(struct apple_dcp *d,unsigned action) { (void)d;(void)action; } +''' + dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() + template = (root / 'drivers/gpu/drm/apple/iomfb_template.c').read_text() + functions = stubs + block(dcp, 'bool dcp_external_hotplug(') + '\n' + block(template, 'static void dcpep_cb_hotplug(') + + main=r"""int main(int argc,char **argv) { MAIN_PREFIX + transport_edid=raw;permutations();conflicting_receipts();failed_replacements(); + unsigned runs=0; + for(unsigned a=0;a<4;a++)for(unsigned b=0;b<4;b++)for(unsigned c=0;c<4;c++)for(unsigned e=0;e<4;e++) { + if(a==b||a==c||a==e||b==c||b==e||c==e)continue; + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + d.external_cable=true;d.external_power_attempted=true;d.external_rx_generation=7; + d.iomfb.lock=true;u64 connected=1;dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + struct dcp_display_mode *old=d.modes;struct drm_display_mode saved=old[0].mode; + u64 sequence=d.rate_proof.sequence,connection=d.rate_proof.connection_sequence; + d.iomfb.lock=true;connected=0;dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + assert(d.external_stop_work.scheduled==1 && d.external_stop_generation==7); + assert(sequence!=d.rate_proof.sequence && connection!=d.rate_proof.connection_sequence); + assert(!d.rate_proof.rate.valid && !d.rate_proof.lanes.valid); + assert(!current(&d,old) && !lookup_mode(&d,&saved)); + assert(con.edid_invalidation_pending);probe(false); /* Work deliberately not executed. */ + dpavservep_detach(&d);dcp_180hz_invalidate(&d); /* Deferred retirement/release. */ + unsigned order[]={a,b,c,e}; + for(unsigned i=0;i<4;i++) { + switch(order[i]) { case 0:callbacks(false);break;case 1:callbacks(true);break;case 2:timings();break;case 3:admit(1);break; } + probe(false);assert(!current(&d,old)); + } + assert(d.rate_proof.rate.valid && d.rate_proof.lanes.valid); + /* Repeated low reports before reassertion are not a new HPD loss. */ + sequence=d.rate_proof.sequence;d.iomfb.lock=true;connected=0; + dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + assert(d.rate_proof.sequence==sequence && d.rate_proof.rate.valid && d.rate_proof.lanes.valid); + d.iomfb.lock=true;connected=1;dcpep_cb_hotplug(&d,&connected);d.iomfb.lock=false; + assert(d.metadata.epoch.connected && d.metadata.epoch.invalidated); + probe(true);assert(!current(&d,old));assert(phy_calls==4 && hardware_rate==8100 && hardware_lanes==2); + free(old);cleanup();runs++; + } + assert(runs==24); + /* Initial bootstrap false and power/paused suppression keep proof intact. */ + reset();callbacks(false);callbacks(true);timings();admit(0);probe(true); + d.iomfb.lock=true;u64 connected=0;u64 sequence=d.rate_proof.sequence; + dcpep_cb_hotplug(&d,&connected);assert(d.rate_proof.sequence==sequence && !con.edid_invalidation_pending); + connected=1;dcpep_cb_hotplug(&d,&connected); + connected=0;d.typec_crtc_off=true;dcpep_cb_hotplug(&d,&connected);assert(d.rate_proof.sequence==sequence); + d.typec_crtc_off=false;d.iomfb.presentation_paused=true;dcpep_cb_hotplug(&d,&connected); + assert(d.rate_proof.sequence==sequence && !con.edid_invalidation_pending);d.iomfb.lock=false;probe(true); + dcpavserv_teardown(&av_services[0]);cleanup(); + puts("PASS: actual RX HPD loss immediately denies getter/lookup before deferred work; 24 fresh-proof recovery orders need only one rate/lane callback; bootstrap/power/paused suppression preserved"); +}""".replace('MAIN_PREFIX',n['main_prefix']) + unit=unit.replace(block(unit,'int main(int argc,char **argv) {'),functions+'\n'+main) + return unit + + +def metadata(root, output): + """Keep malformed/duplicate metadata cases with synthetic bounded wire input.""" + n = parser_fixture(root) + fixture = n['fixture'] + cases = [(fixture, False)] + for path, value in [ + (('MaxHorizontalImageSize',), 256), (('MaxVerticalImageSize',), '33'), + (('Chromaticity', 'Red', 'X'), True), (('Chromaticity', 'Blue', 'Y'), 65536), + (('Chromaticity', 'Green', 'Y'), 64000), (('DefaultWhitePoint', 'Gamma'), 65535), + (('DefaultWhitePoint', 'Gamma'), 240000), (('ProductAttributes', 'ManufacturerID'), 'AC'), + (('ProductAttributes', 'ManufacturerID'), 'A\0R'), (('ProductAttributes', 'LegacyManufacturerID'), 1), + (('ProductAttributes', 'ProductName'), 'x' * 14), (('ProductAttributes', 'ProductID'), 65536), + (('ProductAttributes', 'YearOfManufacture'), 1989), (('ProductAttributes', 'WeekOfManufacture'), 54), + ]: + bad = copy.deepcopy(fixture) + target = bad + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + cases.append((bad, True)) + for key in ('Chromaticity', 'DefaultWhitePoint', 'ProductAttributes'): + bad = copy.deepcopy(fixture) + del bad[key] + cases.append((bad, True)) + second = copy.deepcopy(fixture[key]) + cases.append((n['ObjectPairs'](list(fixture.items()) + [(key, second)]), True)) + for path, field in [(('Chromaticity', 'Red'), 'X'), (('ProductAttributes',), 'ProductName')]: + bad = copy.deepcopy(fixture) + target = bad + for key in path[:-1]: + target = target[key] + value = target[path[-1]] + target[path[-1]] = n['ObjectPairs'](list(value.items()) + [(field, value[field])]) + cases.append((bad, True)) + inputs = output / 'metadata-inputs' + inputs.mkdir(parents=True, exist_ok=True) + arguments = [] + for index, (value, bad) in enumerate(cases): + data = bytearray(struct.pack('H', 1138) + assert raw[21:24] == bytes([60, 33, 120]) + assert raw[38:54] == b'\x01' * 16 and raw[35:38] == b'\0' * 3 + assert struct.unpack_from('> 4) << 8) == 1920 + assert raw[57] + ((raw[58] & 15) << 8) == 280 + assert raw[59] + ((raw[61] >> 4) << 8) == 1080 + assert raw[60] + ((raw[61] & 15) << 8) == 45 + assert raw[62:65] == bytes([88, 44, 0x45]) and raw[71] == 0x1e + chroma = [raw[27] * 4 + ((raw[25] >> 6) & 3), raw[28] * 4 + ((raw[25] >> 4) & 3), + raw[29] * 4 + ((raw[25] >> 2) & 3), raw[30] * 4 + (raw[25] & 3), + raw[31] * 4 + ((raw[26] >> 6) & 3), raw[32] * 4 + ((raw[26] >> 4) & 3), + raw[33] * 4 + ((raw[26] >> 2) & 3), raw[34] * 4 + (raw[26] & 3)] + assert chroma == [697, 317, 265, 632, 146, 81, 321, 337] + assert raw[72:77] == bytes.fromhex('000000fc00') + + +def retry(root): + source = (root / 'drivers/gpu/drm/apple/dcp-180hz.c').read_text() + production = '\n'.join(block(source, marker) for marker in ( + 'static void dcp_180hz_retry_work(', 'void dcp_180hz_retry(', 'static void dcp_180hz_cancel(')) + prefix = r""" +#include +#include +#include +#include +#include +typedef uint64_t u64; +#define READ_ONCE(x) (x) +#define WRITE_ONCE(p,v) ((p)=(v)) +#define guard(kind) (void) +#define spin_lock_irqsave(p,f) do { assert(!*(p)); *(p)=true; (f)=0; } while (0) +#define spin_unlock_irqrestore(p,f) do { (void)(f); assert(*(p)); *(p)=false; } while (0) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_delayed_work(p) container_of(p,struct delayed_work,work) +#define to_platform_device(p) (p) +#define msecs_to_jiffies(x) (x) +struct work_struct { unsigned scheduled; }; +struct delayed_work { struct work_struct work; unsigned delay; bool pending; }; +struct apple_connector { void *dcp; struct work_struct hotplug_wq; }; +struct apple_dcp { + bool hpd_mutex, typec_cable_connected, quiescing, rate_retry_initialized; + struct { bool lock; } metadata; + struct { u64 retry_generation; unsigned retries; } rate_proof; + u64 typec_generation; + struct apple_connector *connector; + void *dev; + struct delayed_work rate_retry_wq; +}; +static int system_freezable_wq; +static unsigned submissions, cancellations; +static void schedule_work(struct work_struct *w) { w->scheduled++; } +static void mod_delayed_work(int q,struct delayed_work *w,unsigned delay) +{ (void)q; submissions++; w->pending=true; w->delay=delay; } +static void cancel_delayed_work_sync(struct delayed_work *w) +{ cancellations++; w->pending=false; } +""" + controls = r""" +int main(void) +{ + struct apple_dcp d={.typec_cable_connected=true,.typec_generation=7}; + struct apple_connector c={.dcp=&d}; d.connector=&c; d.dev=&d; + dcp_180hz_retry(&d); assert(!submissions); + d.rate_retry_initialized=true; + for(unsigned i=0;i<100;i++) dcp_180hz_retry(&d); + assert(submissions==8 && d.rate_proof.retries==8 && d.rate_retry_wq.delay==300); + dcp_180hz_retry_work(&d.rate_retry_wq.work); assert(c.hotplug_wq.scheduled==1); + d.typec_generation++; dcp_180hz_retry_work(&d.rate_retry_wq.work); + assert(c.hotplug_wq.scheduled==1); + d.typec_cable_connected=false; dcp_180hz_retry(&d); assert(submissions==8); + d.typec_cable_connected=true; dcp_180hz_retry(&d); assert(submissions==9 && d.rate_proof.retries==1); + d.quiescing=true; dcp_180hz_retry(&d); dcp_180hz_retry_work(&d.rate_retry_wq.work); + assert(submissions==9 && c.hotplug_wq.scheduled==1); + d.quiescing=false; c.dcp=NULL; dcp_180hz_retry_work(&d.rate_retry_wq.work); + assert(c.hotplug_wq.scheduled==1); + c.dcp=&d; dcp_180hz_retry_work(&d.rate_retry_wq.work); assert(c.hotplug_wq.scheduled==2); + dcp_180hz_cancel(&d); assert(!d.rate_retry_initialized && !d.rate_retry_wq.pending && cancellations==1); + dcp_180hz_retry(&d); assert(submissions==9); + puts("PASS: actual eight-retry bound, 300ms delay, stale-generation/disconnect/quiesce/ownership gates and cancellation"); +} +""" + return prefix + production + controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py new file mode 100644 index 00000000000000..fe62df8d781299 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Host regressions for Apple external display ownership and recovery. + +Only public synthetic fixtures and extracted production functions are used. +The transport, MMIO, locks and work queues are mocks, not hardware validation. +""" +import argparse +import copy +import hashlib +import json +import os +from pathlib import Path +import struct +import tempfile +import traceback + +from common import run_unit +import edid +import lifecycle +import phy +import pm +import idle +import rate + + +def parking_unit(root): + fixture = phy.parking(root) + return ''.join(fixture[k] for k in ('prefix', 'production', 'suffix')) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[6]) + parser.add_argument('--output', type=Path, help='retain generated C, commands and results here') + parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle'), default='all') + args = parser.parse_args() + root = args.source.resolve() + output = args.output or Path(tempfile.mkdtemp(prefix='apple-dcp-selftest-')) + output.mkdir(parents=True, exist_ok=True) + apple = root / 'drivers/gpu/drm/apple' + includes = (apple, root / 'tools/testing/selftests/drivers/gpu/apple-afkv2/host-include') + parking_cases = tuple((c,) for c in ('mux', 'power', 'reset', 'cleanup', 'orientation', + 'orientation-faults', 'backend-change', 'safe-first', + 'failed-flip-return', 'live-park')) + tests = [ + ('edid', 'service-rebind', edid.rebind, ((),), ()), + ('edid', 'provider-admission', edid.admission, ((),), ()), + ('edid', 'ordered-rx', edid.rx_progress, ((),), ('-pthread',)), + ('rate', 'parser-policy', lambda r: rate.policy(r)['unit'], ((),), ()), + ('rate', 'publication-hpd', rate.hotplug, ((),), ()), + ('rate', 'metadata-decoder', lambda r: rate.metadata(r, output), ((),), ('-Wno-sign-compare',)), + ('rate', 'bounded-retry', rate.retry, ((),), ()), + ('phy', 'pm-ownership', pm.ownership, ((),), ()), + ('phy', 'usb-unreachable', lambda r: pm.ownership(r, True), ((),), ()), + ('lifecycle', 'cpuidle-s2idle', idle.s2idle, ((),), ('-Wno-unused-parameter', '-Wno-unused-variable')), + ('lifecycle', 'typec-probe', pm.probe, ((),), ()), + ('lifecycle', 'probe-mux-pm-dispatch', pm.probe_dispatch, ((),), ()), + ('phy', 'parking', lambda r: parking_unit(r), parking_cases, ()), + ('phy', 'adverse-retry', phy.adverse, tuple((c,) for c in ('bist', 'init', 'mux', 'gate', 'dp-cache', 'off', 'success')), ()), + ('phy', 'masked-context', phy.usb_context, ((),), ()), + ('phy', 'off-hardening', phy.hardening, ((),), ()), + ('phy', 'warm-reset', phy.warm_reset, ((),), ()), + ('phy', 'pipe', phy.pipe, tuple((c,) for c in ('backend', 'host', 'mux', 'same', 'park', 'requested', 'scope')), ()), + ('phy', 'unwind', phy.unwind, ((),), ()), + ('phy', 'aux', phy.aux, ((),), ()), + ('lifecycle', 'physical-disconnect', lifecycle.disconnect, ((),), + ('-Wno-unused-variable', '-Wno-unused-function', '-Wno-unused-but-set-variable')), + ('lifecycle', 'dwc3-teardown', lifecycle.teardown, (('all',),), ()), + ('lifecycle', 'resource-profile', lifecycle.profile, ((),), ('-Wno-sign-compare',)), + ('lifecycle', 'firmware-provider', lifecycle.firmware, ((),), ()), + ('lifecycle', 'readiness', lifecycle.readiness, ((),), ()), + ('lifecycle', 'vrr-rejection', lifecycle.vrr, ((),), ()), + ('lifecycle', 'power-reply', lifecycle.power_reply, ((),), ()), + ('lifecycle', 'dispatcher-scope', lifecycle.dispatcher, ((),), ()), + ] + tests.append(('phy', 'usb-kconfig', lambda r: pm.kconfig(r, output), (), ())) + tests = [t for t in tests if args.group in ('all', t[0])] + print('TAP version 13', flush=True) + print('1..' + str(len(tests)), flush=True) + results = [] + for index, (group, name, build, cases, flags) in enumerate(tests, 1): + result = {'name': name, 'group': group, 'passed': False} + try: + unit = build(root) + if unit is None: + result['producer_passed'] = True + elif isinstance(unit, tuple): + unit, cases = unit + if unit is not None: + result['translation_unit_sha256'] = run_unit(name, unit, output, cases, includes if name == 'service-rebind' else (), flags) + if name == 'metadata-decoder': + rate.check_metadata(output) + result['passed'] = True + except Exception: + result['failure'] = traceback.format_exc() + (output / (name + '-failure.log')).write_text(result['failure']) + print(('ok ' if result['passed'] else 'not ok ') + str(index) + ' - ' + name, flush=True) + results.append(result) + sources = {} + for area in ('drivers/gpu/drm/apple', 'drivers/phy/apple', 'drivers/usb/dwc3', 'drivers/usb/typec/tipd'): + for path in sorted((root / area).rglob('*')): + if path.is_file() and path.suffix in ('.c', '.h'): + sources[str(path.relative_to(root))] = hashlib.sha256(path.read_bytes()).hexdigest() + (output / 'results.json').write_text(json.dumps({'source': str(root), 'sources': sources, 'results': results}, indent=2) + '\n') + print('# Artifacts: ' + str(output), flush=True) + return int(any(not r['passed'] for r in results)) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.sh b/tools/testing/selftests/drivers/gpu/apple-dcp/run.sh new file mode 100755 index 00000000000000..f4052ba6632f1c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only OR MIT +PYTHONDONTWRITEBYTECODE=1; export PYTHONDONTWRITEBYTECODE +exec python3 "$(dirname "$0")/run.py" "$@" diff --git a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py index f55107c0ddaa67..5be56f76a68598 100644 --- a/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py +++ b/tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py @@ -14,18 +14,7 @@ def test_shared_header_boundaries(self): #include "dcp-external-bw.h" int main(void) { const u64 n=0xbc000; - u32 scratch=1, request=2; - assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,-1,&scratch,&request)); - assert(scratch==0x28000 && request==0x70000); - assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x30000,-1,&scratch,&request)); - assert(scratch==0x30000 && request==0x70000); - assert(apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x74000,&scratch,&request)); - assert(scratch==0x28000 && request==0x74000); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-2,-1,&scratch,&request)); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,0x28000,&scratch,&request)); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,0x28002,-1,&scratch,&request)); - assert(!apple_dcp_external_bw_resolve(n,0x28000,0x70000,-1,n,&scratch,&request)); - assert(scratch==0x28000 && request==0x74000); + assert(apple_dcp_external_bw_valid(n,0x28000,0x70000)); assert(!apple_dcp_external_bw_valid(n,-1,-1)); assert(!apple_dcp_external_bw_valid(n,0,-1)); assert(!apple_dcp_external_bw_valid(n,-1,0)); From 55ed507566b1359d61f0112cf5643f8b45e61612 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 19:16:00 +0000 Subject: [PATCH 452/631] drm/apple: scope deferred DPAV recovery to current interfaces Retain admitted reannouncements while old commands drain, cancel pending OPEN when CLOSED supersedes it, and preserve interface tag history across reader-pinned slot transitions. Fence explicit physical absence during admission, publication and EDID-current validation. Preserve old command buffers, reader pins and existing endpoint teardown ownership. Keep actual-function regressions in the compact standard selftests. Independent review verified corrected CLOSED/history/absence/fixture controls and strict ARM objects. Exact source tree f14f599948a8ccae7362c4d328dddea623442e54 passed the full strict R66 Image/modules/dtbs build. This is a functional qualification candidate. The inherited four-bit command identity still cannot distinguish a firmware-republished consumed reply when all correlation values recur; retained command-17 counterexamples remain unresolved. No new peer-token semantics, packet experiment, cached EDID override or production diagnostic knob is introduced. No release or hardware qualification is claimed for this new candidate. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk.c | 170 ++++++- drivers/gpu/drm/apple/afk.h | 18 +- drivers/gpu/drm/apple/epic/dpavservep.c | 30 +- .../drivers/gpu/apple-afkv2/extract-send.py | 16 +- .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../selftests/drivers/gpu/apple-dcp/README | 14 + .../selftests/drivers/gpu/apple-dcp/edid.py | 135 +++++- .../drivers/gpu/apple-dcp/recovery.py | 439 ++++++++++++++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 3 + 9 files changed, 800 insertions(+), 27 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 9600e27150d71c..611b7d47967a9f 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -56,6 +56,21 @@ enum rbep_msg_type { #define SEND_WPTR GENMASK(31, 0) +static void afk_service_worker(struct work_struct *work); +static void afk_discard_pending_open(struct apple_epic_service *service); + +void afk_service_retry(struct apple_dcp_afkep *ep) +{ + unsigned long flags; + + if (!ep || !ep->wire_v2 || ep != ep->dcp->dcpavservep) + return; + spin_lock_irqsave(&ep->lock, flags); + if (!ep->stopping) + queue_work(ep->wq, &ep->service_work); + spin_unlock_irqrestore(&ep->lock, flags); +} + static void afk_send(struct apple_dcp_afkep *ep, u64 message) { dcp_send_message(ep->dcp, ep->endpoint, message); @@ -66,8 +81,10 @@ static void afk_release_context(void *data) struct apple_dcp_afkep *afkep = data; afk_quiesce(afkep); - if (!afkep->dcp->retain_dma) + if (!afkep->dcp->retain_dma) { + kvfree(afkep->command_tags); kfree(afkep); + } } struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, @@ -90,6 +107,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, goto out_free_afkep; } + INIT_WORK(&afkep->service_work, afk_service_worker); init_completion(&afkep->started); init_completion(&afkep->stopped); spin_lock_init(&afkep->lock); @@ -108,12 +126,21 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, void afk_quiesce(struct apple_dcp_afkep *afkep) { + unsigned long flags; + u32 i; + if (IS_ERR_OR_NULL(afkep)) return; + spin_lock_irqsave(&afkep->lock, flags); + afkep->stopping = true; + spin_unlock_irqrestore(&afkep->lock, flags); if (afkep->wq) { destroy_workqueue(afkep->wq); afkep->wq = NULL; } + for (i = 0; i < afkep->num_channels; i++) + if (afkep->services[i].pending_open) + afk_discard_pending_open(&afkep->services[i]); debugfs_remove_recursive(afkep->debugfs_entry); afkep->debugfs_entry = NULL; } @@ -188,6 +215,14 @@ int afk_start(struct apple_dcp_afkep *ep) { int ret; + /* Allocate before firmware can announce services or issue commands. */ + if (ep->wire_v2 && !ep->command_tags) { + ep->command_tags = kvcalloc(AFKV2_INTERFACES, sizeof(*ep->command_tags), + GFP_KERNEL); + if (!ep->command_tags) + return -ENOMEM; + } + reinit_completion(&ep->started); ret = apple_rtkit_start_ep(ep->dcp->rtk, ep->endpoint); if (ret) @@ -360,12 +395,74 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, enum epic_type etype, enum epic_category ecat, u8 stype, const void *payload, size_t payload_len, u64 token); +/* Pending identities are owned only by the ordered endpoint workqueue. */ +static void afk_discard_pending_open(struct apple_epic_service *service) +{ + service->pending_open = false; + kfree(service->pending_name); + kfree(service->pending_class); + service->pending_name = NULL; + service->pending_class = NULL; + afk_service_discard(service); +} + +static void afk_retry_service_init(struct apple_dcp_afkep *ep) +{ + u32 i; + + for (i = 0; i < ep->num_channels; i++) { + struct apple_epic_service *service = &ep->services[i]; + const struct apple_epic_service_ops *ops = service->ops; + int ret; + + if (!service->pending_open) + continue; + ret = ops->revalidate(service, service->pending_name, + service->pending_class, service->pending_unit); + if (ret) { + afk_discard_pending_open(service); + continue; + } + /* Old replies keep their original slot, tag and buffers until drained. */ + if (afk_epic_find_service(ep, service->channel)) + continue; + ret = afk_send_epic(ep, service->channel, 0, EPIC_TYPE_NOTIFY, + EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); + if (ret) { + dev_err(ep->dcp->dev, "AFKV2 interface %u deferred OPEN failed: %d\n", + service->channel, ret); + afk_discard_pending_open(service); + continue; + } + service->pending_open = false; + afk_service_publish(service); + ops->init(service, service->pending_name, service->pending_class, + service->pending_unit); + if (service->enabled) + afk_populate_service_debugfs(service); + kfree(service->pending_name); + kfree(service->pending_class); + service->pending_name = NULL; + service->pending_class = NULL; + } +} + +static void afk_service_worker(struct work_struct *work) +{ + struct apple_dcp_afkep *ep = container_of(work, struct apple_dcp_afkep, service_work); + + if (!READ_ONCE(ep->stopping)) + afk_retry_service_init(ep); +} + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { char name[32]; s64 epic_unit = -1; - struct apple_epic_service *service = NULL; + struct apple_epic_service *service = NULL, *previous; + unsigned long flags; + bool retired = false; u32 ch_idx; const char *service_name = name; const char *epic_name = NULL, *epic_class = NULL; @@ -374,8 +471,25 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *props = payload + sizeof(name); size_t props_size = payload_size - sizeof(name); - if (afk_epic_find_service(ep, channel)) - return; + previous = afk_epic_find_service(ep, channel); + if (previous) { + spin_lock_irqsave(&previous->lock, flags); + retired = previous->torndown; + spin_unlock_irqrestore(&previous->lock, flags); + if (!ep->wire_v2 || ep != ep->dcp->dcpavservep || + !retired || !previous->ops->revalidate) + return; + } + for (ch_idx = 0; ch_idx < ep->num_channels; ch_idx++) { + struct apple_epic_service *pending = &ep->services[ch_idx]; + + if (!pending->pending_open || pending->channel != channel) + continue; + if (!pending->ops->revalidate(pending, pending->pending_name, + pending->pending_class, pending->pending_unit)) + return; + afk_discard_pending_open(pending); + } if (payload_size < sizeof(name)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", @@ -455,6 +569,15 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, goto free; } } + if (previous && retired && ops->revalidate) { + /* Preserve the admitted identity and its original provider ticket. */ + service->pending_open = true; + service->pending_name = epic_name; + service->pending_class = epic_class; + service->pending_unit = epic_unit; + afk_service_retry(ep); + return; + } if (ep->wire_v2) { int ret = afk_send_epic(ep, channel, 0, EPIC_TYPE_NOTIFY, EPIC_CAT_REPORT, AFKV2_OPEN, NULL, 0); @@ -495,6 +618,14 @@ static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel) const struct apple_epic_service_ops *ops; unsigned long flags; + /* Accepted CLOSE/CLOSED supersedes unpublished recovery for this interface. */ + for (u32 i = 0; i < ep->num_channels; i++) { + struct apple_epic_service *pending = &ep->services[i]; + + if (pending->pending_open && pending->channel == channel) + afk_discard_pending_open(pending); + } + service = afk_epic_find_service(ep, channel); if (!service) { dev_warn(ep->dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", @@ -1566,6 +1697,28 @@ static int afk_send_command_full(struct apple_epic_service *service, u8 type, MSEC_PER_SEC); } +static u16 afk_command_tag(struct apple_epic_service *service, u8 slot) +{ + struct apple_dcp_afkep *ep = service->ep; + unsigned long flags; + u16 tag; + + lockdep_assert_held(&service->lock); + if (!ep->wire_v2) + return ((service->cmd_tag++ & 0xff) << 8) | slot; + + /* Send already nests the endpoint lock below the service lock. */ + spin_lock_irqsave(&ep->lock, flags); + tag = afkv2_command_id(ep->command_tags[service->channel]++, slot); + spin_unlock_irqrestore(&ep->lock, flags); + return tag; +} + +static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + unsigned int timeout_ms, size_t *received); + int afk_send_command_timeout(struct apple_epic_service *service, u8 type, const void *payload, size_t payload_len, void *output, size_t output_len, u32 *retcode, @@ -1596,6 +1749,11 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, if (!service) return -ENODEV; ep = service->ep; + if (ep->wire_v2 && (!service->channel || service->channel >= AFKV2_INTERFACES || + !ep->command_tags)) { + ret = -ENODEV; + goto err_put_service; + } if (READ_ONCE(ep->dcp->quiescing)) { ret = -ENODEV; goto err_put_service; @@ -1651,9 +1809,7 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, goto err_unlock; } - tag = ep->wire_v2 ? afkv2_command_id(service->cmd_tag, idx) : - ((service->cmd_tag & 0xff) << 8) | idx; - service->cmd_tag++; + tag = afk_command_tag(service, idx); service->cmds[idx].tag = tag; service->cmds[idx].type = type; diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index ecd7fbf279a9e3..b03be04259e706 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -19,10 +19,13 @@ #define AFK_MAX_CHANNEL 16 #define MAX_PENDING_CMDS 16 +#define AFKV2_INTERFACES BIT(16) struct apple_epic_service_ops; struct apple_dcp_afkep; +void afk_service_retry(struct apple_dcp_afkep *ep); + struct epic_cmd_info { u16 tag; u8 type; @@ -50,6 +53,7 @@ struct apple_epic_service { struct epic_cmd_info cmds[MAX_PENDING_CMDS]; DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); + /* Legacy EPIC sequence; AFKV2 history belongs to the interface. */ u8 cmd_tag; spinlock_t lock; /* Pins held by owners and synchronous callers; protected by lock. */ @@ -63,6 +67,11 @@ struct apple_epic_service { struct { u64 generation, epoch; } admission; void *cookie; + /* Admitted reannouncement waiting for the previous channel to drain. */ + bool pending_open; + const char *pending_name, *pending_class; + s64 pending_unit; + struct { struct dentry *entry; u8 *scratch; @@ -82,6 +91,9 @@ struct apple_epic_service_ops { */ int (*prepare)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); + /* Check the original prepare ticket before a deferred OPEN. */ + int (*revalidate)(struct apple_epic_service *service, const char *name, + const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); int (*call)(struct apple_epic_service *service, u32 idx, @@ -142,6 +154,7 @@ static inline void afk_service_put(struct apple_epic_service *service) if (!WARN_ON(!service->users)) service->users--; spin_unlock_irqrestore(&service->lock, flags); + afk_service_retry(service->ep); } static inline void afk_service_disable(struct apple_epic_service *service) @@ -205,7 +218,7 @@ afk_service_reinit(struct apple_epic_service *service, service->ep = ep; service->channel = channel; service->cookie = NULL; - /* Keep the command-tag sequence across service generations. */ + /* Keep legacy slot history; AFKV2 history belongs to the interface. */ } spin_unlock_irqrestore(&service->lock, flags); return reusable; @@ -347,7 +360,10 @@ struct apple_dcp_afkep { bool wire_v2; /* External H17P DPAV only; other endpoints retain owned-DMA commands. */ bool inline_v2; + /* Fixed 16-bit interface domain; survives slot and channel reuse. */ + u8 *command_tags; struct workqueue_struct *wq; + struct work_struct service_work; struct completion started; struct completion stopped; diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index dc37f4afa7f0f8..012ce4824a3bc8 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -51,7 +51,8 @@ static bool dcpavserv_admission_current(struct apple_dcp *dcp, u64 generation) { lockdep_assert_held(&dcp->hpd_mutex); /* Initial service discovery may precede the first Type-C attachment. */ - return !READ_ONCE(dcp->quiescing) && generation == dcp->typec_generation && + return !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->external_physically_absent) && + generation == dcp->typec_generation && (!generation || dcp->typec_cable_connected); } @@ -82,6 +83,26 @@ static int dcpavserv_prepare(struct apple_epic_service *service, const char *nam return ret; } +static int dcpavserv_revalidate(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + int ret = 0; + + if (!dcpavserv_identity(service, name, class, unit)) + return -ENODEV; + mutex_lock(&dcp->hpd_mutex); + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.enabled) + ret = -EBUSY; + else if (service->admission.epoch != dcp->dcpavserv.epoch || + !dcpavserv_admission_current(dcp, service->admission.generation)) + ret = -ENODEV; + mutex_unlock(&dcp->dcpavserv.lock); + mutex_unlock(&dcp->hpd_mutex); + return ret; +} + static void dcpavserv_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -369,14 +390,16 @@ bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation) return dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || (dcp->typec_cable_connected && generation == dcp->typec_generation && - !READ_ONCE(dcp->quiescing)); + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->external_physically_absent)); } bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch) { lockdep_assert_held(&dcp->dcpavserv.lock); return dcp->dcpavserv.enabled && dcp->dcpavserv.service && - epoch == dcp->dcpavserv.epoch; + epoch == dcp->dcpavserv.epoch && + (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL || + !READ_ONCE(dcp->external_physically_absent)); } const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, @@ -456,6 +479,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { .name = "dcpav-service-epic", .reusable = true, .prepare = dcpavserv_prepare, + .revalidate = dcpavserv_revalidate, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, diff --git a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py index 0d80a729a33424..70dbd5d3a69f4f 100644 --- a/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py +++ b/tools/testing/selftests/drivers/gpu/apple-afkv2/extract-send.py @@ -30,6 +30,8 @@ #define SZ_1M 1048576U #define GFP_KERNEL 0 #define MAX_PENDING_CMDS 16 +#define AFKV2_INTERFACES (1U << 16) +#define lockdep_assert_held(l) ((void)(l)) #define cpu_to_le32(x) (x) #define le32_to_cpu(x) (x) #define cpu_to_le64(x) (x) @@ -56,7 +58,7 @@ bool done,free_on_ack; int *completion; }; struct dcp { void *dev; bool quiescing; }; -struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint; }; +struct apple_dcp_afkep { struct dcp *dcp; bool wire_v2,inline_v2; int endpoint,lock;u8 *command_tags; }; struct ops { const char *name; }; struct apple_epic_service { struct apple_dcp_afkep *ep; struct ops *ops; unsigned int channel; unsigned long cmd_map[1]; @@ -106,7 +108,8 @@ suffix = r''' int main(void) { - struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true}; + u8 tags[AFKV2_INTERFACES]={0}; + struct dcp dcp={0};struct apple_dcp_afkep ep={.dcp=&dcp,.wire_v2=true,.inline_v2=true,.command_tags=tags}; struct ops ops={.name="test"};struct apple_epic_service s={.ep=&ep,.ops=&ops,.channel=4,.enabled=true,.users=1}; u8 request[112]={0x5a},*output=calloc(1,32888);size_t received;u32 status=99; active=&s; @@ -155,4 +158,11 @@ puts("PASS: extracted production sender, 128-byte command envelope / 32888-byte reply, short OOB, strict/legacy callers and actual late-ACK ownership"); } ''' -pathlib.Path(sys.argv[2]).write_text(prefix + source[start:end] + source[reply_match.start():reply_end] + suffix) +tag_match = re.search(r"static u16 afk_command_tag\([^;]*?\)\n\{", source) +if not tag_match: + raise RuntimeError("production interface tag allocator not found") +tag_end, tag_depth = tag_match.end(), 1 +while tag_depth: + tag_depth += (source[tag_end] == "{") - (source[tag_end] == "}") + tag_end += 1 +pathlib.Path(sys.argv[2]).write_text(prefix + source[tag_match.start():tag_end] + source[start:end] + source[reply_match.start():reply_end] + suffix) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index 0d6d406ef5b9d9..cdc821ce4be259 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh -TEST_FILES := run.py common.py edid.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README +TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index a61543d1168f8f..e9bf9bdabcb67b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -54,3 +54,17 @@ The cpuidle group executes the actual Apple table/probe, generic registration and s2idle selector/entry, and tick_freeze/unfreeze. It checks source behavior only; RTC, elapsed suspend time and simultaneous hardware execution remain separate qualification gates. The original human cpuidle patch is preserved. + +The service-rebind group also runs retained reviewer controls through the +actual sender, receive worker, recovery, quiesce and EDID wrapper. It covers +CLOSED cancellation before lookup, first late ACK versus already consumed +duplicates, reader-pinned cross-slot/channel reuse, and the explicit physical +absence window. Per-interface sequence history has a fixed 64-KiB bound per +AFKV2 endpoint; reply routing still ignores reader-only retired services. + +Two wrap controls intentionally retain the command-17 replay assertion +failure, for OOB and inline. The group verifies that these counterexamples +still reproduce, alongside nonvacuous mutations and passing corrected safety +controls. A passing host suite does not establish a replay-exclusion contract +or admit a hardware trial. Token/sequence semantics remain a separate protocol +fact to establish; the tests do not manufacture a firmware echo guarantee. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py index 55a16678b97be2..7d892663ff6756 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/edid.py @@ -21,6 +21,8 @@ def function(file,signature): #include #include #include +#include +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) #include "afkv2-codec.h" typedef int64_t s64; typedef u64 dma_addr_t; @@ -29,6 +31,7 @@ def function(file,signature): #define ENOENT 2 #define MAX_PENDING_CMDS 16 #define AFK_MAX_CHANNEL 16 +#define AFKV2_INTERFACES (1U << 16) #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) #define WARN_ON(x) (x) @@ -65,25 +68,30 @@ def function(file,signature): static void complete(int *p) { (*p)++; } static void reinit_completion(int *p) { *p=0; } static void schedule_work(unsigned int *p) { (*p)++; } +struct work_struct { bool pending; }; +static bool queue_work(void *wq,struct work_struct *w) { assert(wq);bool fresh=!w->pending;w->pending=true;return fresh; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; -struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +void afk_service_retry(struct apple_dcp_afkep *ep); +struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; bool lock,enabled,torndown;unsigned int users;u32 channel;u8 cmd_tag; unsigned long cmd_map[1];struct epic_cmd_info cmds[MAX_PENDING_CMDS]; struct { u64 generation,epoch; } admission;void *cookie; + bool pending_open;const char *pending_name,*pending_class;s64 pending_unit; }; -struct apple_dcp_afkep { struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; +struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;void *wq;struct work_struct service_work;struct apple_dcp *dcp;const struct apple_epic_service_ops *ops;bool wire_v2,inline_v2,match_epic_name;u32 endpoint;unsigned int num_channels;struct apple_epic_service services[AFK_MAX_CHANNEL]; }; struct apple_connector { void *dcp;bool connected,edid_invalidation_pending;struct { int registration_state; } base;unsigned int hotplug_wq; }; struct apple_dcp { void *dev;int fw_compat;struct apple_dcp_afkep *dcpavservep;struct apple_connector *connector; - bool hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; + bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;u64 typec_generation;unsigned int nr_modes; struct { int iomfb_method_profile; } hw; struct { bool lock,enabled;u64 epoch;int enable_completion;struct apple_epic_service *service; } dcpavserv; }; struct epic_cmd { u32 retcode;u64 rxbuf,txbuf;u32 rxlen,txlen; }; static unsigned int dma_frees,opens; + static int open_error,open_action; void dpavservep_detach(struct apple_dcp *dcp); static void inject_owner(struct apple_dcp_afkep *ep,u32 pending_channel); @@ -105,6 +113,7 @@ def function(file,signature): if(open_action==4) inject_owner(e,channel); if(open_action==5) e->dcp->typec_cable_connected=false; if(open_action==6) e->dcp->quiescing=true; + if(open_action==7) e->dcp->external_physically_absent=true; if(open_action==3) { for(unsigned int i=0;inum_channels;i++) if(e->services[i].channel==channel && !e->services[i].torndown) e->services[i].channel++; @@ -133,18 +142,18 @@ def function(file,signature): { assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } static int skip(struct dcp_parse_ctx *h) { assert(!h->key);h->index++;h->key=true;return 0; } ''' - parts=[function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), + parts=[function('afk.c','static u16 afk_command_tag('),function('afk.c','void afk_service_retry('),function('parser.c','int parse_epic_service_init('),function('afk.h','static inline struct apple_epic_service *afk_service_get('), function('afk.h','static inline void afk_service_put('),function('afk.h','static inline void afk_service_retire('),function('afk.h','static inline void afk_service_disable('), function('afk.h','static inline bool afk_service_matches('),function('afk.h','static inline bool afk_service_reclaimable('), function('afk.h','static inline bool\nafk_service_reinit('),function('afk.h','static inline void afk_service_discard('), function('afk.h','static inline void afk_service_publish('), function('epic/dpavservep.c','static void dcpavserv_notify_edid('),function('epic/dpavservep.c','static bool dcpavserv_identity('), function('epic/dpavservep.c','static bool dcpavserv_admission_current('),function('epic/dpavservep.c','static int dcpavserv_prepare('), - function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), + function('epic/dpavservep.c','static int dcpavserv_revalidate('),function('epic/dpavservep.c','static void dcpavserv_init('),function('epic/dpavservep.c','static void dcpavserv_invalidate_edid('), function('epic/dpavservep.c','static void dcpavserv_teardown('),function('epic/dpavservep.c','void dpavservep_detach('), function('afk.c','static const struct apple_epic_service_ops *\nafk_match_service('), function('afk.c','static struct apple_epic_service *afk_epic_find_service('), - function('afk.c','static void afk_recv_handle_init('), + function('afk.c','static void afk_discard_pending_open('),function('afk.c','static void afk_retry_service_init('),function('afk.c','static void afk_service_worker('),function('afk.c','static void afk_recv_handle_init('), function('epic/dpavservep.c','static int dcpdpserv_prepare('), function('epic/dpavservep.c','static void dcpdpserv_teardown('),function('afk.c','static void afk_recv_handle_teardown('), function('afk.c','static void afk_recv_handle_reply('),function('afk.c','static void afk_v2_receive(')] @@ -152,9 +161,10 @@ def function(file,signature): static struct apple_dcp d; static struct apple_dcp_afkep ep; static struct apple_connector con; -static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; +static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } } +static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; static const struct apple_epic_service_ops announcement_ops[]={ - {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, + {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, {0} }; @@ -268,14 +278,39 @@ def function(file,signature): assert(injected_owner->users==1);dpavservep_detach(&d);assert(!injected_owner->users); open_action=0;announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); struct apple_epic_service *old=d.dcpavserv.service; - assert(afk_service_get(old)==old);old->cmd_map[0]=1; + assert(afk_service_get(old)==old);ep.command_tags[13]=1;old->cmd_map[0]=1; disconnect();connect(); announce(13,name,native_class,13); /* Same channel still owns a pending reply. */ assert(opens==before && !d.dcpavserv.service && old->users==1 && old->channel==13); + /* The firmware need not announce again after the old reply drains. */ + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf; + old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf; + u8 drain_oob[32],drain_reply[56];size_t drain_written; + assert(!afkv2_encode_oob_command(drain_oob,sizeof(drain_oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32)); + afkv2_put32(drain_oob+4,0); + assert(!afkv2_encode_frame(drain_reply,sizeof(drain_reply),1,13,0,0xc0,AFKV2_RESPONSE,0,drain_oob,sizeof(drain_oob),&drain_written)); + afk_v2_receive(&ep,0,0,drain_reply,drain_written); + run_retries(); + assert(d.dcpavserv.service && "a blocked reannouncement must recover after the old ACK without another announcement"); + assert(d.dcpavserv.service!=old && opens==++before && old->users==1 && !old->cmd_map[0]); + struct apple_epic_service *resumed=d.dcpavserv.service; + resumed->cmd_map[0]=1; + /* The DMA allocator can reuse both old addresses for the new command. */ + resumed->lock=true; + resumed->cmds[0]=(struct epic_cmd_info){.tag=afk_command_tag(resumed,0),.type=0xc0, + .rxbuf_dma=old->cmds[0].rxbuf_dma,.txbuf_dma=old->cmds[0].txbuf_dma,.rxlen=32,.txlen=32}; + resumed->lock=false; + afk_v2_receive(&ep,0,0,drain_reply,drain_written); + assert(!resumed->cmds[0].done && resumed->cmd_map[0] && "old ACK must not complete the next owner's command"); + resumed->cmd_map[0]=0; + dpavservep_detach(&d); announce(1,name,class,1);assert(opens==++before && d.dcpavserv.service!=old); struct apple_epic_service *fresh=d.dcpavserv.service;u64 epoch=d.dcpavserv.epoch; dcpavserv_teardown(old);assert(d.dcpavserv.service==fresh && d.dcpavserv.epoch==epoch); old->cmd_map[0]=0;afk_service_put(old);disconnect();connect(); + run_retries(); announce(13,name,native_class,13);assert(opens==++before && d.dcpavserv.service); struct apple_epic_service *owner=d.dcpavserv.service; u64 owner_epoch=d.dcpavserv.epoch; @@ -294,7 +329,7 @@ def function(file,signature): dpavservep_detach(&d); assert(!d.hpd_mutex && !d.dcpavserv.lock); /* Legacy remains unit-zero and class-agnostic, with its original epoch fence. */ - ep.wire_v2=false;ep.endpoint=0x28; + ep.wire_v2=false;ep.endpoint=0x28;d.external_physically_absent=true; assert(!prepare(13,13,name,native_class)); struct apple_epic_service *legacy=prepare(13,0,name,native_class);assert(legacy); dpavservep_detach(&d);afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); @@ -303,11 +338,82 @@ def function(file,signature): d.typec_generation++;afk_service_publish(legacy);dcpavserv_init(legacy,name,native_class,0); assert(d.dcpavserv.service==legacy && legacy->users==1);dpavservep_detach(&d); assert(dcpdpserv_prepare(legacy,"dcpdp-service-epic",native_class,0)==0); - ep.wire_v2=true;ep.endpoint=0x2a; + ep.wire_v2=true;ep.endpoint=0x2a;d.external_physically_absent=false; announce(U16_MAX,name,native_class,U16_MAX);assert(d.dcpavserv.service && d.dcpavserv.service->channel==U16_MAX); dpavservep_detach(&d); puts("PASS: actual AFKV2 HDMI/native DP alternating over 64 cycles: flip/revalidation/return, strict identity, stale OPEN generation/epoch/owner, auxiliary isolation, reader/command pins and legacy controls"); } + +static void deferred_announcements(void) +{ + for(unsigned int cycle=0;cycle<64;cycle++) { + unsigned int event=cycle%10,before=opens,freed=dma_frees; + u16 channel=1+4*(cycle%4); + const char *provider=(cycle%2)?native_class:class; + announce(channel,name,provider,channel); + struct apple_epic_service *old=d.dcpavserv.service; + assert(old && opens==++before && afk_service_get(old)==old); + old->cmd_map[0]=1; + old->cmds[0]=(struct epic_cmd_info){.tag=0,.type=0xc0,.rxlen=32,.txlen=32,.free_on_ack=true}; + old->cmds[0].rxbuf=calloc(1,32);old->cmds[0].txbuf=calloc(1,32); + old->cmds[0].rxbuf_dma=(uintptr_t)old->cmds[0].rxbuf; + old->cmds[0].txbuf_dma=(uintptr_t)old->cmds[0].txbuf; + u8 oob[32],reply[56];size_t written; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),0,old->cmds[0].rxbuf_dma,32,old->cmds[0].txbuf_dma,32)); + afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(reply,sizeof(reply),1,channel,0,0xc0,AFKV2_RESPONSE,0,oob,sizeof(oob),&written)); + u64 generation=d.typec_generation; + closed(channel,0x6a01,0); + assert(!d.dcpavserv.service && old->users==1 && old->torndown && d.typec_generation==generation); + u64 epoch=d.dcpavserv.epoch; + announce(channel,name,provider,channel); + run_retries(); + assert(opens==before && !d.dcpavserv.service && dma_frees==freed); + for(unsigned int i=0;ipending_open) continue; + assert(dcpavserv_revalidate(pending,pending->pending_name,pending->pending_class,pending->pending_unit)==0); + d.dcpavserv.enabled=true; + assert(dcpavserv_revalidate(pending,pending->pending_name,pending->pending_class,pending->pending_unit)==-EBUSY); + d.dcpavserv.enabled=false; + } + if(event==1) d.typec_generation++; + if(event==2) dpavservep_detach(&d); + if(event==3) d.typec_cable_connected=false; + if(event==4) d.quiescing=true; + struct apple_epic_service *other=NULL; + if(event==5) { other=prepare(channel+2,channel+2,name,class);publish(other,channel+2); } + if(event==6) assert(!ep.stopping); + if(event==7) open_error=-EIO; + if(event==8) announce(channel,name,provider,channel); + if(event==9) { d.typec_generation++;announce(channel,name,provider,channel); } + afk_v2_receive(&ep,0,0,reply,written); + if(event==6) { assert(ep.service_work.pending);ep.stopping=true; } + run_retries(); + assert(dma_frees==freed+2 && old->users==1 && !old->cmd_map[0]); + if(event==0 || event==8 || event==9) { + assert(d.dcpavserv.service && d.dcpavserv.service!=old); + assert(d.dcpavserv.service->channel==channel && opens==++before); + assert(d.dcpavserv.epoch>epoch && d.dcpavserv.service->users==1); + } else if(event==5) { + assert(d.dcpavserv.service==other && opens==before); + } else { + assert(!d.dcpavserv.service && opens==before+(event==7)); + } + afk_v2_receive(&ep,0,0,reply,written); /* Duplicate late ACK cannot touch the new owner. */ + assert(dma_frees==freed+2); + afk_service_put(old);run_retries(); + assert(!old->users); + if(event==6) { + for(unsigned int i=0;iusers==1); assert(afk_service_get(old)==old); /* Delayed EDID reader pins the retired slot. */ @@ -375,6 +482,8 @@ def function(file,signature): assert(!pending->users && !d.hpd_mutex && !d.dcpavserv.lock); memset(&ep.services,0,sizeof(ep.services));ep.wire_v2=true;ep.endpoint=0x2a;d.hw.iomfb_method_profile=2; repeated_announcements(); + deferred_announcements(); + free(ep.command_tags);ep.command_tags=NULL; puts("PASS: actual dynamic 1/5/9/13 binding, forged identities, prepare/init epochs, physical retirement, reader pins, real old CLOSED/late ACK isolation and legacy unit 0"); } ''' @@ -461,7 +570,7 @@ def function(file, signature): struct { bool lock, enabled; u64 epoch; int enable_completion; struct apple_epic_service *service; } dcpavserv; struct apple_connector *connector; unsigned int nr_modes; struct { int iomfb_method_profile; } hw; - bool hpd_mutex, typec_cable_connected, quiescing; + bool external_physically_absent,hpd_mutex, typec_cable_connected, quiescing; u64 typec_generation; int fw_compat; void *dev; @@ -571,6 +680,8 @@ def function(file, signature): d.dcpavserv.service=&s; d.dcpavserv.enabled=true; s.enabled=true; s.users=1; event=0; d.typec_cable_connected=false; e=dcpavserv_copy_edid(&d,0,&epoch); assert(!IS_ERR(e) && legacy_reads==1 && s.users==1); drm_edid_free(e); + d.hw.iomfb_method_profile=0;av.wire_v2=false;d.external_physically_absent=true; + mutex_lock(&d.hpd_mutex);assert(dcpavserv_edid_current(&d,0));mutex_unlock(&d.hpd_mutex); assert(!allocations && !d.hpd_mutex && !d.dcpavserv.lock); puts("PASS: production old/new lookup and receive, exact service admission before OPEN, delayed disconnect/replug, service detach pins, legacy transport"); } diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py new file mode 100644 index 00000000000000..310af408b0f96e --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/recovery.py @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Retained reviewer controls: actual AFK sender/receive, queue and EDID functions. + +The wrap controls deliberately retain an expected replay assertion failure. +They document an unresolved peer contract, not successful replay protection. +""" +from pathlib import Path +import json, hashlib, subprocess + +def check(root, output): + import edid + out = output / 'service-recovery-controls' + def extract(text,signature): + start=text.index(signature);opening=text.index('{',start) + while ';' in text[start:opening]: + start=text.index(signature,start+1);opening=text.index('{',start) + end,depth=opening+1,1 + while depth: + depth+=(text[end]=='{')-(text[end]=='}');end+=1 + return text[start:end] + + unit=edid.rebind(root) + # Retain the original exact fixture separately. Correct only its causal ordering. + corrected=unit.replace('assert(afk_service_get(old)==old);old->cmd_map[0]=1;', + 'assert(afk_service_get(old)==old);old->cmd_tag=1;old->cmd_map[0]=1;') + corrected=corrected.replace(' old->cmd_tag=1;\n','') + + sender=extract((root/'drivers/gpu/drm/apple/afk.c').read_text(), + 'static int afk_send_command_timeout_len(') + unit=unit[:unit.index('int main(void)')] + unit=unit.replace('static unsigned int dma_frees,opens;',r''' + static unsigned int dma_frees,opens; + #define SZ_1M (1U<<20) + #define GFP_KERNEL 0 + #define DECLARE_COMPLETION_ONSTACK(c) int c=0 + #define init_completion(c) (*(c)=0) + #define msecs_to_jiffies(x) (x) + #define cpu_to_le64(x) (x) + #define kmalloc(n,g) malloc(n) + #define EPIC_TYPE_COMMAND 3 + #define EPIC_CAT_COMMAND 3 + static int bitmap_find_free_region(unsigned long *p,int n,int order) { + (void)order;for(int i=0;iinline_v2) { + struct afkv2_command decoded;u8 response[40]={0}; + assert(!afkv2_decode_command(p,n,&decoded) && !decoded.oob && decoded.value==32 && decoded.id==tag); + response[1]=decoded.id; + assert(!afkv2_encode_frame(current_reply,sizeof(current_reply),1,channel,0,subtype,AFKV2_RESPONSE,0,response,sizeof(response),¤t_size)); + current_channel=channel;current_tag=tag;return 0; + } + assert(n==sizeof(struct epic_cmd)); + const struct epic_cmd *cmd=p;u8 oob[32]; + assert(!afkv2_encode_oob_command(oob,sizeof(oob),tag,cmd->rxbuf,cmd->rxlen,cmd->txbuf,cmd->txlen)); + afkv2_put32(oob+4,0); + assert(!afkv2_encode_frame(current_reply,sizeof(current_reply),1,channel,0,subtype,AFKV2_RESPONSE,0,oob,sizeof(oob),¤t_size)); + current_channel=channel;current_tag=tag;return 0; + } + '''+marker) + + tests=r''' + static void setup(void) { + d.hw.iomfb_method_profile=2;d.typec_cable_connected=true;d.typec_generation=1; + d.nr_modes=1;d.connector=&con;d.dev=&d; + con.dcp=&d;con.connected=true;con.base.registration_state=1; + ep.dcp=&d;ep.wire_v2=true;ep.endpoint=0x2a;ep.wq=&ep;ep.command_tags=calloc(AFKV2_INTERFACES,1);assert(ep.command_tags); + d.dcpavservep=&ep;ep.ops=announcement_ops;ep.match_epic_name=true; + d.fw_compat=DCP_FIRMWARE_H17P;hpd_lock=&d.hpd_mutex;av_lock=&d.dcpavserv.lock;active_ep=&ep; + } + static int send_actual(struct apple_epic_service *s,bool ack,bool late) { + u8 payload[32]={0},output[32]={0};u32 code; + ack_current=ack;inject_late=late; + int result=afk_send_command_timeout_len(s,0xc0,payload,32,output,32,&code,1,NULL); + inject_late=false;return result; + } + static void save_ack(void) { memcpy(saved_reply,current_reply,current_size);saved_size=current_size; } + static void cleanup(void) { + dpavservep_detach(&d);run_retries(); + for(unsigned int i=0;ichannel==5); + assert(send_actual(middle,true,true)==0); + closed(5,0x6a01,0);announce(13,name,native_class,13);run_retries(); + assert(d.dcpavserv.service && d.dcpavserv.service==middle && d.dcpavserv.service!=old); + assert(send_actual(d.dcpavserv.service,true,true)==0); + afk_service_put(old); + } else if(!strcmp(argv[1],"multiple-first-late-acks")) { + reuse_dma=false;assert(afk_service_get(old)==old); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + u8 first[56];size_t first_size=saved_size;memcpy(first,saved_reply,first_size); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + assert(old->cmd_map[0]==3); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries();assert(!d.dcpavserv.service); + afk_v2_receive(&ep,0,0,first,first_size);run_retries(); + assert(!d.dcpavserv.service && old->cmd_map[0]==2 && old->users==1); + unsigned int freed=dma_frees; + afk_v2_receive(&ep,0,0,first,first_size);run_retries(); + assert(dma_frees==freed && old->cmd_map[0]==2 && !d.dcpavserv.service); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(d.dcpavserv.service && d.dcpavserv.service!=old && !old->cmd_map[0]); + afk_service_put(old); + } else if(!strcmp(argv[1],"tag-wrap") || !strcmp(argv[1],"inline-tag-wrap")) { + assert(send_actual(old,true,false)==0);save_ack(); + for(unsigned int i=0;i<15;i++) assert(send_actual(old,true,true)==0); + assert(send_actual(old,true,true)==0); + } else if(!strcmp(argv[1],"pending-closed")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + assert(!d.dcpavserv.service); + closed(13,0x6a01,0); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(!d.dcpavserv.service && "CLOSED during pending recovery must invalidate the pending OPEN"); + } else if(!strcmp(argv[1],"actual-tag-before-announcement")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();assert(ep.command_tags[13]==1); + closed(13,0x6a01,0);announce(13,name,native_class,13); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(d.dcpavserv.service && send_actual(d.dcpavserv.service,true,true)==0); + } else if(!strncmp(argv[1],"open-race-",10)) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + if(!strcmp(argv[1],"open-race-generation")) open_action=1; + else if(!strcmp(argv[1],"open-race-epoch")) open_action=2; + else if(!strcmp(argv[1],"open-race-owner")) open_action=4; + else if(!strcmp(argv[1],"open-race-cable")) open_action=5; + else if(!strcmp(argv[1],"open-race-quiescing")) open_action=6; + else if(!strcmp(argv[1],"open-race-absence")) open_action=7; + else assert(!"unknown OPEN race"); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + if(open_action==4) assert(d.dcpavserv.service==injected_owner); + else assert(!d.dcpavserv.service); + open_action=0;d.quiescing=false;d.typec_cable_connected=true;d.external_physically_absent=false; + } else if(!strcmp(argv[1],"physical-absence")) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + d.external_physically_absent=true; + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(!d.dcpavserv.service && "physical absence authority must fence deferred OPEN before cable/generation update"); + } else if(!strncmp(argv[1],"fence-",6)) { + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack(); + closed(13,0x6a01,0);announce(13,name,native_class,13);run_retries(); + unsigned int before=opens; + if(!strcmp(argv[1],"fence-epoch")) dpavservep_detach(&d); + else if(!strcmp(argv[1],"fence-generation")) d.typec_generation++; + else if(!strcmp(argv[1],"fence-cable")) d.typec_cable_connected=false; + else if(!strcmp(argv[1],"fence-quiescing")) d.quiescing=true; + else if(!strcmp(argv[1],"fence-owner")) inject_owner(&ep,13); + else if(!strcmp(argv[1],"fence-open-failure")) open_error=-EIO; + else assert(!"unknown fence"); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);run_retries(); + assert(opens==before+(!strcmp(argv[1],"fence-open-failure"))); + if(!strcmp(argv[1],"fence-owner")) assert(d.dcpavserv.service==injected_owner); + else assert(!d.dcpavserv.service); + for(unsigned int i=0;ifn=(f);(w)->pending=false;(w)->next=NULL; } while(0) + static struct work_struct *fifo_head,*fifo_tail;static bool destroyed,rx_running; + ''') + queued=queued.replace('static bool queue_work(void *wq,struct work_struct *w) { assert(wq);bool fresh=!w->pending;w->pending=true;return fresh; }',r''' + static bool queue_work(void *wq,struct work_struct *w) { + assert(wq && !destroyed);if(w->pending) return false; + w->pending=true;w->next=NULL;if(fifo_tail) fifo_tail->next=w;else fifo_head=w;fifo_tail=w;return true; + } + static void run_one_work(void) { + assert(fifo_head);struct work_struct *w=fifo_head;fifo_head=w->next;if(!fifo_head) fifo_tail=NULL; + w->pending=false;w->next=NULL;assert(w->fn);w->fn(w); + } + static void destroy_workqueue(void *wq) { assert(wq && !destroyed);while(fifo_head) run_one_work();destroyed=true; } + #define IS_ERR_OR_NULL(p) (!(p) || IS_ERR(p)) + #define debugfs_remove_recursive(p) ((void)(p)) + #define kzalloc(n,g) calloc(1,n) + #define kvfree(p) free(p) + #define ESHUTDOWN 108 + #define RBEP_TYPE 0xffff + #define FIELD_GET(mask,val) ((val)&(mask)) + #define RBEP_INIT_ACK 0xa0 + #define RBEP_START_ACK 0x86 + #define RBEP_SHUTDOWN_ACK 0xc1 + #define RBEP_GETBUF 0x89 + #define RBEP_INIT_TX 0x8a + #define RBEP_INIT_RX 0x8b + #define RBEP_RECV 0x85 + #define complete_all(p) complete(p) + ''') + queued=queued.replace('struct apple_dcp_afkep { u8 *command_tags;bool lock,stopping;', + 'struct apple_dcp_afkep { u8 *command_tags;int started,stopped,txbfr,rxbfr;void *debugfs_entry;bool lock,stopping;') + queued=queued.replace('bool external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;', + 'bool retain_dma,external_physically_absent,hpd_mutex,quiescing,typec_cable_connected;') + queued=queued.replace('static void run_retries(void) { assert(!ep.lock);if(ep.service_work.pending) { ep.service_work.pending=false;afk_service_worker(&ep.service_work); } }', + 'static void run_retries(void) { assert(!ep.lock);while(fifo_head) run_one_work(); }') + queued=queued.replace(' if(type==EPIC_TYPE_COMMAND) {', + ' if(e->stopping) return -ESHUTDOWN;\n if(type==EPIC_TYPE_COMMAND) {') + queued=queued[:queued.index('int main(int argc')] + queued+=r''' + struct afk_receive_message_work { struct apple_dcp_afkep *ep;u64 message;struct work_struct work; }; + static void afk_getbuf(struct apple_dcp_afkep *e,u64 m) { (void)e;(void)m;assert(!"unexpected GETBUF"); } + static void afk_init_rxtx(struct apple_dcp_afkep *e,u64 m,int *r) { (void)e;(void)m;(void)r;assert(!"unexpected INIT"); } + static unsigned int rx_step;static bool close_pending; + static bool afk_recv(struct apple_dcp_afkep *e) { + assert(e==&ep);rx_running=true; + if(rx_step==0) closed(13,0x6a01,0); + else if(rx_step==1) announce(13,name,native_class,13); + else if(rx_step==2) afk_v2_receive(e,0,0,saved_reply,saved_size); + else if(rx_step==3 && close_pending) closed(13,0x6a01,0); + else { assert(!d.dcpavserv.service && "recovery must wait for the RX drain loop to finish");rx_running=false;return false; } + assert(!d.dcpavserv.service && "recovery must remain behind RX on the ordered queue");rx_step++;return true; + } + ''' + af=(root/'drivers/gpu/drm/apple/afk.c').read_text() + for signature in ['static void afk_receive_message_worker(', 'int afk_receive_message(', 'void afk_quiesce(', 'static void afk_release_context(']: + queued+=extract(af,signature)+'\n' + queued+=r''' + int main(int argc,char **argv) { + assert(argc==2);setup();reuse_dma=true;INIT_WORK(&ep.service_work,afk_service_worker); + announce(13,name,native_class,13);struct apple_epic_service *old=d.dcpavserv.service;assert(old); + assert(send_actual(old,false,false)==-ETIMEDOUT);save_ack();run_retries(); + unsigned int before=opens; + if(!strncmp(argv[1],"ordered-",8)) { + close_pending=!strcmp(argv[1],"ordered-pending-closed"); + assert(!afk_receive_message(&ep,RBEP_RECV));assert(fifo_head); + run_one_work();assert(!rx_running && opens==before && !d.dcpavserv.service); + assert(fifo_head==&ep.service_work);run_retries(); + if(close_pending) assert(!d.dcpavserv.service && "ordered CLOSED must cancel pending OPEN"); + else { assert(d.dcpavserv.service && opens==before+1);assert(send_actual(d.dcpavserv.service,true,true)==0); } + cleanup(); + } else { + assert(!strcmp(argv[1],"quiesce-pending"));assert(afk_service_get(old)==old); + closed(13,0x6a01,0);announce(13,name,native_class,13); + afk_v2_receive(&ep,0,0,saved_reply,saved_size);assert(ep.service_work.pending); + afk_quiesce(&ep);assert(destroyed && !ep.wq && ep.stopping && !fifo_head); + assert(opens==before && !d.dcpavserv.service && old->users==1); + for(unsigned int i=0;iusers); + assert(afk_receive_message(&ep,RBEP_RECV)==-ESHUTDOWN);assert(!fifo_head); + afk_quiesce(&ep);assert(!fifo_head); + struct apple_dcp_afkep *empty=calloc(1,sizeof(*empty));assert(empty);empty->dcp=&d; + afk_release_context(empty); + cleanup(); + } + printf("PASS actual RX worker/quiesce: %s\n",argv[1]); + } + ''' + run('actual-queue-quiesce',queued,[[c] for c in ['ordered-recovery','ordered-pending-closed','quiesce-pending']]) + + # Actual EDID wrapper exercises its owner/reader pins, epoch and generation exits. + reader=unit[:unit.index('int main(int argc')] + reader+=r''' + #define ESTALE 116 + struct drm_edid { int marker; }; + static unsigned int edid_allocations,edid_frees;static int read_action; + struct reader_guard { bool *lock;bool once; }; + static struct reader_guard lock_reader_guard(bool *lock) { mutex_lock(lock);return (struct reader_guard){lock,true}; } + static void unlock_reader_guard(struct reader_guard *guard) { mutex_unlock(guard->lock); } + #define scoped_guard(kind,p) for(struct reader_guard _guard __attribute__((cleanup(unlock_reader_guard)))=lock_reader_guard(p);_guard.once;_guard.once=false) + static void drm_edid_free(const struct drm_edid *edid) { assert(edid && !IS_ERR(edid));free((void *)edid);edid_frees++; } + static const struct drm_edid *dcpavserv_read_edid_inline(struct apple_epic_service *service) { + assert(service->users==2 && "owner plus actual EDID wrapper reader pin"); + assert(send_actual(service,true,false)==0); + if(read_action==1) { closed(service->channel,0x6a01,0);announce(5,name,class,5);assert(d.dcpavserv.service!=service); } + if(read_action==2) d.typec_generation++; + if(read_action==3) d.external_physically_absent=true; + if(read_action==4) { dpavservep_detach(&d);d.quiescing=true; } + struct drm_edid *edid=malloc(sizeof(*edid));assert(edid);edid->marker=1;edid_allocations++;return edid; + } + static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) { return dcpavserv_read_edid_inline(service); } + ''' + dpav=(root/'drivers/gpu/drm/apple/epic/dpavservep.c').read_text() + for signature in ['bool dcpavserv_edid_current(', 'bool dcpavserv_edid_service_current(', 'const struct drm_edid *dcpavserv_copy_edid(']: + reader+=extract(dpav,signature)+'\n' + reader+=r''' + int main(int argc,char **argv) { + assert(argc==2);setup();reuse_dma=true;announce(13,name,class,13); + struct apple_epic_service *old=d.dcpavserv.service;assert(old); + if(!strcmp(argv[1],"reader-owner")) read_action=1; + else if(!strcmp(argv[1],"reader-generation")) read_action=2; + else if(!strcmp(argv[1],"reader-physical-absence")) read_action=3; + else if(!strcmp(argv[1],"reader-quiescing")) read_action=4; + else assert(!strcmp(argv[1],"reader-success")); + u64 epoch;const struct drm_edid *edid=dcpavserv_copy_edid(&d,1,&epoch); + if(!read_action) { assert(!IS_ERR(edid));drm_edid_free(edid); } + else { assert(IS_ERR(edid) && "actual EDID wrapper must reject invalidated ownership/physical state"); } + if(read_action==1 || read_action==4) assert(!old->users);else assert(old->users==1); + assert(edid_allocations==edid_frees && !d.hpd_mutex && !d.dcpavserv.lock); + d.quiescing=false;cleanup();printf("PASS actual copy_edid: %s\n",argv[1]); + } + ''' + run('actual-edid-reader',reader,[[c] for c in ['reader-success','reader-owner','reader-generation','reader-quiescing','reader-physical-absence']]) + mutations=[ + ('restart-tags','afkv2_command_id(ep->command_tags[service->channel]++, slot)','afkv2_command_id(0, slot)'), + ('no-drain-retry','\tafk_service_retry(service->ep);',''), + ('ignore-old-command','\t\tif (afk_epic_find_service(ep, service->channel))\n\t\t\tcontinue;',''), + ('ignore-epoch','service->admission.epoch != dcp->dcpavserv.epoch ||\n\t\t ','false ||\n\t\t '), + ('refresh-generation','!dcpavserv_admission_current(dcp, service->admission.generation))\n\t\tret = -ENODEV;','!dcpavserv_admission_current(dcp, dcp->typec_generation))\n\t\tret = -ENODEV;'), + ('ignore-owner','if (dcp->dcpavserv.enabled)\n\t\tret = -EBUSY;\n\telse if (service->admission.epoch','if (false)\n\t\tret = -EBUSY;\n\telse if (service->admission.epoch'), + ('ignore-stop','if (!READ_ONCE(ep->stopping))\n\t\tafk_retry_service_init(ep);','if (true)\n\t\tafk_retry_service_init(ep);'), + ] + + for name,text,case in [ + ('cancel-closed',unit.replace('if (pending->pending_open && pending->channel == channel)','if (false && pending->pending_open && pending->channel == channel)'), 'pending-closed'), + ('reader-lookup',unit.replace('(service->enabled ||','(service->users || service->enabled ||'), 'drained-reader-rebind'), + ('reset-interface-history',unit.replace('service->cookie = NULL;', 'service->cookie = NULL; if(ep->wire_v2 && ep->command_tags) ep->command_tags[channel]=0;',1), 'channel-reuse'), + ('absence-admission',unit.replace('!READ_ONCE(dcp->external_physically_absent) &&','true &&',1), 'initial-absence'), + ('absence-publication',unit.replace('!READ_ONCE(dcp->external_physically_absent) &&','true &&',1), 'initial-open-absence'), + ('absence-reader',reader.replace('!READ_ONCE(dcp->external_physically_absent)', 'true'), 'reader-physical-absence'), + ]: + run('mutation-'+name,text,[[case]]) + + for name,old,new in mutations: + assert corrected.count(old)==1,(name,corrected.count(old)) + run('mutation-'+name,corrected.replace(old,new),[[]]) + negative={ + 'actual-sender':{'tag-wrap','inline-tag-wrap'}, + 'actual-queue-quiesce':set(), + 'actual-edid-reader':set(), + } + for result in results: + case=result.get('case',[]) + fail_expected=bool('case' in result and (result['name'].startswith('mutation-') or (case and case[0] in negative.get(result['name'],set())))) + result['assertion_failure_expected']=fail_expected + result['expectation_met']=result['exit']==(-6 if fail_expected else 0) + if fail_expected: result['expectation_met'] = result['expectation_met'] and 'Assertion' in result.get('output','') + if case and case[0] in {'tag-wrap','inline-tag-wrap'}: + result['expectation_met'] = result['expectation_met'] and 'late ACK completed an actual new production command' in result.get('output','') + (out/'results.json').write_text(json.dumps(results,indent=2)+'\n') + + assert all(r['expectation_met'] for r in results), "retained recovery control expectation failed; see results.json" diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index fe62df8d781299..94349823fa03cb 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -18,6 +18,7 @@ from common import run_unit import edid +import recovery import lifecycle import phy import pm @@ -90,6 +91,8 @@ def main(): unit, cases = unit if unit is not None: result['translation_unit_sha256'] = run_unit(name, unit, output, cases, includes if name == 'service-rebind' else (), flags) + if name == 'service-rebind': + recovery.check(root, output) if name == 'metadata-decoder': rate.check_metadata(output) result['passed'] = True From 69ce25094921d82b84198654b9801a2215869203 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 22:11:04 +0000 Subject: [PATCH 453/631] drm/apple: integrate reviewed fixed-host lifecycle and scoped boot fixes Own the attached upstream USB3 link across the fixed-host PHY handoff, reject detached or quiescing roots, and serialize synchronous DWC3/xHCI removal before SUSPHY/controller teardown. Preserve fatal ACK errors, normal autosuspend and the compact source regressions. Keep required AFK faults visible while distinguishing the intentionally unsupported auxiliary role and known opaque OSLog events. The published source tree exactly matches compiled checkpoint 1183f73bc58972456acf6ff5fcfe094a2427364e and its R68 full build. Preserve the original Bartosz cpuidle commit through the current aurora-wip merge parent. Source review and builds pass; hardware trial, repeated physical orientations/hotplug and full beta qualification remain open. This draft is not a release or merge qualification claim. Signed-off-by: Ryan Murray Assisted-by: LLM --- drivers/gpu/drm/apple/afk.c | 9 +- drivers/gpu/drm/apple/afk.h | 2 + drivers/gpu/drm/apple/epic/dpavservep.c | 3 +- drivers/gpu/drm/apple/iomfb_h17p.c | 4 +- drivers/phy/apple/atc.c | 101 +++++++- .../selftests/drivers/gpu/apple-dcp/Makefile | 2 +- .../selftests/drivers/gpu/apple-dcp/README | 9 +- .../selftests/drivers/gpu/apple-dcp/boot.py | 56 +++++ .../selftests/drivers/gpu/apple-dcp/edid.py | 48 +++- .../selftests/drivers/gpu/apple-dcp/pm.py | 223 ++++++++++++++++-- .../selftests/drivers/gpu/apple-dcp/run.py | 7 +- .../drivers/gpu/apple-dcp/teardown_pm.py | 156 ++++++++++++ 12 files changed, 586 insertions(+), 34 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/boot.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 611b7d47967a9f..7e23edc851b2da 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -563,8 +563,13 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, int ret = ops->prepare(service, epic_name, epic_class, epic_unit); if (ret) { - dev_err(ep->dcp->dev, "AFK service %s admission failed: %d\n", - service_name, ret); + if (ops->optional && ret == -EOPNOTSUPP) + dev_info(ep->dcp->dev, + "AFK optional %s (%s) is unsupported on channel %u\n", + ops->name, service_name, channel); + else + dev_err(ep->dcp->dev, "AFK service %s admission failed: %d\n", + service_name, ret); afk_service_discard(service); goto free; } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index b03be04259e706..94926860ba33f6 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -84,6 +84,8 @@ struct apple_epic_service_ops { const char name[32]; /* All retained pointers must be pinned before enabling slot reuse. */ bool reusable; + /* EOPNOTSUPP from prepare means intentional absence only for this role. */ + bool optional; /* * Optional admission, before OPEN/publication. Must not retain service diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 012ce4824a3bc8..62a30a23914436 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -211,7 +211,7 @@ static int dcpdpserv_prepare(struct apple_epic_service *service, const char *nam const char *class, s64 unit) { /* No production command transport is needed for this H17P service. */ - return service->ep->wire_v2 ? -ENODEV : 0; + return service->ep->wire_v2 ? -EOPNOTSUPP : 0; } static void dcpdpserv_init(struct apple_epic_service *service, const char *name, @@ -487,6 +487,7 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpdp-service-epic", .reusable = true, + .optional = true, .prepare = dcpdpserv_prepare, .init = dcpdpserv_init, .teardown = dcpdpserv_teardown, diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 99948cab7158fb..45743b41736999 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -452,7 +452,9 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); reply->length = cpu_to_le32(property->length); - dev_warn_once(dcp->dev, "using J700 provider data for an older loader\n"); + dev_warn_once(dcp->dev, + "missing provider property %s (%u bytes); using qualified J700 compatibility data\n", + property->dt_name, property->length); return true; } else if (length != (int)property->length) { dev_err(dcp->dev, "invalid provider property %s: %d bytes\n", diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index c09bf398e83943..3e7ab85eaa95d6 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -4108,10 +4108,58 @@ static int atcphy_probe_switch(struct apple_atcphy *atcphy) */ struct atcphy_usb3_pm { struct platform_device *controller; + struct device *host; struct usb_device *root; + struct usb_device *link; }; #if IS_REACHABLE(CONFIG_USB) +/* The root's runtime state does not cover selective suspend of its port. */ +static int atcphy_usb3_link_get(struct atcphy_usb3_pm *pm) +{ + struct usb_device *link; + int ret; + + /* The graph identifies the controller, not a port in a multiport root. */ + if (pm->root->maxchild != 1) + return pm->root->maxchild ? -ENODEV : -EAGAIN; + link = usb_hub_find_child(pm->root, 1); + if (!link) + return 0; + pm->link = usb_get_dev(link); + usb_lock_device(pm->link); + if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { + ret = 0; + goto unlock_link; + } + ret = pm_runtime_resume_and_get(&pm->link->dev); + if (ret < 0) + goto unlock_link; + if (READ_ONCE(pm->link->state) == USB_STATE_NOTATTACHED) { + pm_runtime_put_noidle(&pm->link->dev); + ret = -ENODEV; + goto unlock_link; + } + return 0; + +unlock_link: + usb_unlock_device(pm->link); + usb_put_dev(pm->link); + pm->link = NULL; + return ret; +} + +static void atcphy_usb3_link_put(struct atcphy_usb3_pm *pm) +{ + if (!pm->link) + return; + usb_mark_last_busy(pm->link); + pm_runtime_put_autosuspend(&pm->link->dev); + usb_unlock_device(pm->link); + usb_put_dev(pm->link); + pm->link = NULL; +} + static int atcphy_usb3_pm_find(struct usb_device *udev, void *data) { struct atcphy_usb3_pm *pm = data; @@ -4146,6 +4194,17 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm if (!pm->controller) return -EAGAIN; + /* Driver core takes this lock before DWC3 removal touches SUSPHY. */ + device_lock(&pm->controller->dev); + if (!pm->controller->dev.driver) { + ret = -EAGAIN; + goto put_controller; + } + if (strcmp(pm->controller->dev.driver->name, "dwc3-apple")) { + ret = -ENODEV; + goto put_controller; + } + /* Firmware identity, not a bus number, binds this PHY to its root hub. */ usb_for_each_dev(pm, atcphy_usb3_pm_find); if (!pm->root) { @@ -4153,7 +4212,20 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm goto put_controller; } - /* Exclude root-hub teardown while its controller backs the handoff. */ + /* Also serialize direct xHCI unbind, before it marks the HCD removing. */ + pm->host = get_device(pm->root->bus->controller); + device_lock(pm->host); + if (!pm->host->driver) { + ret = -EAGAIN; + goto put_root; + } + if (pm->host->parent != &pm->controller->dev || + strcmp(pm->host->driver->name, "xhci-hcd")) { + ret = -ENODEV; + goto put_root; + } + + /* USB destruction follows the platform lifecycle locks, not vice versa. */ usb_lock_device(pm->root); hcd = bus_to_hcd(pm->root->bus); if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || HCD_DEAD(hcd)) { @@ -4169,13 +4241,33 @@ static int atcphy_usb3_pm_get(struct apple_atcphy *atcphy, struct atcphy_usb3_pm ret = -ENODEV; goto unlock_root; } + /* Resume the attached upstream link before taking the ATC mutex. */ + ret = atcphy_usb3_link_get(pm); + if (ret) { + usb_mark_last_busy(pm->root); + pm_runtime_put_autosuspend(&pm->root->dev); + goto unlock_root; + } + if (READ_ONCE(pm->root->state) == USB_STATE_NOTATTACHED || + READ_ONCE(hcd->state) != HC_STATE_RUNNING || + HCD_DEAD(hcd) || !HCD_HW_ACCESSIBLE(hcd)) { + atcphy_usb3_link_put(pm); + pm_runtime_put_noidle(&pm->root->dev); + ret = -ENODEV; + goto unlock_root; + } return 0; unlock_root: usb_unlock_device(pm->root); +put_root: + device_unlock(pm->host); + put_device(pm->host); + pm->host = NULL; usb_put_dev(pm->root); pm->root = NULL; put_controller: + device_unlock(&pm->controller->dev); put_device(&pm->controller->dev); pm->controller = NULL; return ret; @@ -4191,12 +4283,19 @@ static void atcphy_usb3_pm_put(struct atcphy_usb3_pm *pm) if (!pm->root) return; + /* Release child ownership before the root: keep the lock order hierarchical. */ + atcphy_usb3_link_put(pm); + /* Restore normal runtime policy after both success and failed parking. */ usb_mark_last_busy(pm->root); pm_runtime_put_autosuspend(&pm->root->dev); usb_unlock_device(pm->root); usb_put_dev(pm->root); + device_unlock(pm->host); + put_device(pm->host); + device_unlock(&pm->controller->dev); put_device(&pm->controller->dev); + pm->host = NULL; pm->root = NULL; pm->controller = NULL; #endif diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile index cdc821ce4be259..bbb39cada55bf8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only OR MIT TEST_PROGS := run.sh -TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py dwc3 README +TEST_FILES := run.py common.py edid.py recovery.py rate.py phy.py lifecycle.py pm.py idle.py boot.py dwc3 README include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/README b/tools/testing/selftests/drivers/gpu/apple-dcp/README index e9bf9bdabcb67b..b04b6b9934193b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/README +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/README @@ -10,7 +10,7 @@ Or keep translation units, commands, failure logs and source hashes: tools/testing/selftests/drivers/gpu/apple-dcp/run.sh --output /tmp/apple-dcp-results An installed copy needs --source /path/to/kernel. --group edid, rate, phy or -lifecycle selects one group. HOSTCC selects the host compiler (default clang). +lifecycle or boot selects one group. HOSTCC selects the host compiler (default clang). Python 3 and a compiler supporting AddressSanitizer/UndefinedBehaviorSanitizer are required. No target access, firmware command, captured fixture or kernel configuration is required. @@ -68,3 +68,10 @@ still reproduce, alongside nonvacuous mutations and passing corrected safety controls. A passing host suite does not establish a replay-exclusion contract or admit a hardware trial. Token/sequence semantics remain a separate protocol fact to establish; the tests do not manufacture a firmware echo guarantee. + +The boot group executes the actual OSLOG dispatcher with synthetic fixed-width +words. Known opaque event types remain visible, unknown types retain warnings, +and only the existing buffer-request path can allocate/ACK buffers. The EDID +service-rebind group also distinguishes intentional optional EOPNOTSUPP from +required admission failures and genuine optional-service errors, and proves +refused services never OPEN or publish. No captured firmware values are used. diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py new file mode 100644 index 00000000000000..bf95df845443d9 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/boot.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0-only OR MIT +# Copyright 2026 Ryan Murray +"""Opaque RTKit event dispatch: no firmware addresses or captured payloads.""" +from common import block + +def oslog(root): + source = (root / 'drivers/soc/apple/rtkit.c').read_text() + mask = next(line for line in source.splitlines() + if line.startswith('#define APPLE_RTKIT_OSLOG_TYPE ')) + return r''' +#include +#include +#include +typedef uint64_t u64; +typedef uint8_t u8; +#define GENMASK_ULL(h,l) ((~0ULL << (l)) & (~0ULL >> (63-(h)))) +#define FIELD_GET(mask,value) (((value) & (mask)) >> __builtin_ctzll(mask)) +#define APPLE_RTKIT_OSLOG_BUFFER_REQUEST 1 +#define APPLE_RTKIT_EP_OSLOG 8 +struct buffer { unsigned sentinel; }; +struct apple_rtkit { void *dev; struct buffer oslog_buffer; }; +static unsigned warnings, information, requests; +static u64 observed; +static u8 observed_type; +#define dev_warn(dev,fmt,msg) do { (void)(dev); (void)(fmt); warnings++; observed=(msg); } while (0) +#define dev_info(dev,fmt,type,msg) do { (void)(dev); (void)(fmt); information++; observed_type=(type); observed=(msg); } while (0) +static void apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, struct buffer *b, u8 ep, u64 msg) +{ assert(b==&rtk->oslog_buffer && ep==8); requests++; observed=msg; } +''' + mask + '\n' + block(source, 'static void apple_rtkit_oslog_rx(') + r''' +static void reset(void) { warnings=information=requests=0; observed=0; observed_type=0; } +int main(void) +{ + struct apple_rtkit rtk={.oslog_buffer={.sentinel=0x12345678}}; + for (u8 type=3; type<=5; type++) { + for (unsigned edge=0; edge<2; edge++) { + u64 msg=((u64)type<<56) | (edge ? ((1ULL<<56)-1) : 0); + reset(); apple_rtkit_oslog_rx(&rtk,msg); + assert(!warnings && information==1 && !requests); + assert(observed==msg && observed_type==type); + assert(rtk.oslog_buffer.sentinel==0x12345678); + } + } + puts("PASS recognized opaque types preserve both payload bounds without buffer/ACK operations"); + const u8 unknown[]={0,2,6,0xff}; + for (unsigned i=0;ipending;w->pending=true;return fresh; } struct apple_dcp;struct apple_dcp_afkep;struct apple_epic_service; void afk_service_retry(struct apple_dcp_afkep *ep); -struct apple_epic_service_ops { char name[32];bool reusable;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; +struct apple_epic_service_ops { char name[32];bool reusable,optional;void (*init)(struct apple_epic_service *,const char *,const char *,s64);int (*prepare)(struct apple_epic_service *,const char *,const char *,s64);int (*revalidate)(struct apple_epic_service *,const char *,const char *,s64);void (*teardown)(struct apple_epic_service *); }; struct epic_cmd_info { u16 tag;u8 type;size_t reply_length;void *rxbuf,*txbuf;dma_addr_t rxbuf_dma,txbuf_dma;size_t rxlen,txlen;u32 retcode;bool done,free_on_ack;int *completion; }; struct apple_epic_service { struct apple_dcp_afkep *ep;const struct apple_epic_service_ops *ops; @@ -136,7 +138,9 @@ def function(file,signature): { struct item *i=&h->items[h->index];const char *p; if(h->key) { p=i->key;h->key=false; } else { p=i->value;h->index++;h->key=true; } - if(!p) return ERR_PTR(-EINVAL);char *copy=malloc(strlen(p)+1);assert(copy);strcpy(copy,p);return copy; + if(!p) + return ERR_PTR(-EINVAL); + char *copy=malloc(strlen(p)+1);assert(copy);strcpy(copy,p);return copy; } static int parse_int(struct dcp_parse_ctx *h,s64 *out) { assert(!h->key);*out=h->items[h->index++].integer;h->key=true;return 0; } @@ -165,8 +169,8 @@ def function(file,signature): static const struct apple_epic_service_ops ops={.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}; static const struct apple_epic_service_ops announcement_ops[]={ {.name="dcpav-service-epic",.reusable=true,.prepare=dcpavserv_prepare,.revalidate=dcpavserv_revalidate,.init=dcpavserv_init,.teardown=dcpavserv_teardown}, - {.name="dcpdp-service-epic",.reusable=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, - {0} + {.name="dcpdp-service-epic",.reusable=true,.optional=true,.prepare=dcpdpserv_prepare,.teardown=dcpdpserv_teardown}, + {.name=""} }; static const char *name="dcpav-service-epic",*class="AppleDCPDP2HDMI"; static const char *native_class="DCPDP13Service"; @@ -414,12 +418,42 @@ def function(file,signature): } puts("PASS: 64 actual CLOSED/reannounce/late-ACK cycles with deferred OPEN, unchanged attachment, duplicate announcement/reply, stale generation/epoch, cable/quiesce/owner/stop and OPEN-error controls"); } +static int injected_prepare_error; +static int failed_prepare(struct apple_epic_service *s,const char *n,const char *c,s64 u) +{ (void)s;(void)n;(void)c;(void)u;return injected_prepare_error; } +static void optional_admission_controls(void) +{ + ep.ops=announcement_ops;ep.match_epic_name=true;ep.num_channels=0;d.fw_compat=DCP_FIRMWARE_H17P; + diagnostic_errors=diagnostic_information=0; + unsigned before=opens; + announce(3,"dcpdp-service-epic",class,3); + announce(5,"dcpdp-service-epic",native_class,5); + assert(!diagnostic_errors && diagnostic_information==2 && opens==before); + assert(!d.dcpavserv.enabled && !d.dcpavserv.service); + for(unsigned i=0;ihcd) #define HCD_DEAD(hcd) (!!((hcd)->flags&(1UL<flags&(1UL<refs++;return d; } +static void device_lock(struct device *d) { + if(d==&controller.dev){assert(!controller_locked&&!host_locked&&!root_locked);controller_locked=true;} + else {assert(d==&host_device&&controller_locked&&!host_locked&&!root_locked);host_locked=true;} +} +static void device_unlock(struct device *d) { + if(d==&host_device){assert(host_locked&&!root_locked);host_locked=false;} + else {assert(d==&controller.dev&&controller_locked&&!host_locked&&!root_locked);controller_locked=false;} +} static void put_device(struct device *d) { assert(d->refs>0);d->refs--; } static struct usb_device *usb_get_dev(struct usb_device *u) { u->refs++;return u; } static void usb_put_dev(struct usb_device *u) { assert(u->refs>0);u->refs--; } static int usb_for_each_dev(void *data,int (*fn)(struct usb_device *,void *)) { if(root_missing)return 0;for(unsigned int i=0;i<4;i++){int r=fn(&roots[i],data);if(r)return r;}return 0; } -static void usb_lock_device(struct usb_device *u) { assert(u==&roots[3]&&!root_locked);root_locked=true;if(detach_on_lock)u->state=0; } -static void usb_unlock_device(struct usb_device *u) { assert(u==&roots[3]&&root_locked);root_locked=false; } +static struct usb_device *usb_hub_find_child(struct usb_device *u, int port) { assert(u==&roots[3]&&root_locked&&port==1); + if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED); + if(quiesce_at_lookup)hcd.state=HC_STATE_QUIESCING; + return link_present?&link_device:NULL; } +static void usb_lock_device(struct usb_device *u) { + if(u==&roots[3]){assert(!root_locked);if(REQUIRES_PLATFORM_LOCKS)assert(controller_locked&&host_locked);root_locked=true;if(detach_on_lock)u->state=0;} + else {assert(u==&link_device&&root_locked&&!link_locked);link_locked=true;} +} +static void usb_unlock_device(struct usb_device *u) { + if(u==&roots[3]){assert(root_locked&&!link_locked);root_locked=false;} + else {assert(u==&link_device&&link_locked&&root_locked);link_locked=false;} +} static int pm_runtime_resume_and_get(struct device *d) { - assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock);resumes++; + assert(root_locked&&!pm_phy->lock); + if(d==&link_device.dev){ + assert(link_locked);link_resumes++; + struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); + if(link_error)return link_error; + link_pm_refs++;link_suspended=false;stuck_ack=false; + if(link_detach_on_resume)link_device.state=0; + if(link_dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); + return 0; + } + assert(d==&roots[3].dev);resumes++; /* A re-entrant PHY lock acquisition during resume must remain possible. */ struct lock_guard g=lock_take(&pm_phy->lock);lock_release(&g); if(resume_error)return resume_error; - pm_refs++;stuck_ack=false;if(detach_on_resume)roots[3].state=0; + pm_refs++;stuck_ack=link_present&&link_device.state!=USB_STATE_NOTATTACHED&&link_suspended;if(detach_on_resume)roots[3].state=0; if(inaccessible_on_resume)clear_bit(HCD_FLAG_HW_ACCESSIBLE,&hcd.flags); if(dead_on_resume)set_bit(HCD_FLAG_DEAD,&hcd.flags); return 0; } -static void pm_runtime_put_noidle(struct device *d) { assert(d==&roots[3].dev&&pm_refs==1);pm_refs--; } -static void usb_mark_last_busy(struct usb_device *u) { assert(u==&roots[3]&&root_locked&&!pm_phy->lock); } -static void pm_runtime_put_autosuspend(struct device *d) { assert(d==&roots[3].dev&&root_locked&&!pm_phy->lock&&pm_refs==1);pm_refs--; } +static void pm_runtime_put_noidle(struct device *d) { + if(d==&link_device.dev){assert(link_pm_refs==1);link_pm_refs--;} + else {assert(d==&roots[3].dev&&pm_refs==1);pm_refs--;} +} +static void usb_mark_last_busy(struct usb_device *u) { + assert(root_locked&&!pm_phy->lock); + if(u==&link_device)assert(link_locked);else assert(u==&roots[3]&&!link_locked); +} +static void pm_runtime_put_autosuspend(struct device *d) { + assert(root_locked&&!pm_phy->lock); + if(d==&link_device.dev){assert(link_locked&&link_pm_refs==1);link_pm_refs--;} + else {assert(d==&roots[3].dev&&!link_locked&&pm_refs==1);pm_refs--;} +} ''' suffix=ns['suffix'].replace('int main(', 'int __attribute__((unused)) fixture_main(',1) controls=r''' @@ -96,16 +143,23 @@ def ownership(root, usb_unreachable=False): { hw.has_t8140_dp=t8140;initial(p,fixed,false,swap); p->np=&ctrl_node;pm_phy=p; - hcd=(struct usb_hcd){.rh_registered=true,.flags=1UL<lock&&!root_locked&&!pm_refs&&!node_refs&&!controller.dev.refs); + assert(!controller_locked&&!host_locked&&!host_device.refs&&!p->lock&&!root_locked&&!link_locked&&!link_pm_refs&&!link_device.refs&&!pm_refs&&!node_refs&&!controller.dev.refs); for(unsigned int i=0;i<4;i++)assert(!roots[i].refs); } int main(void) @@ -180,6 +234,49 @@ def ownership(root, usb_unreachable=False): prepare(&p,true,false,false);before=writes;assert(atcphy_mux_set(&mux,&same)==0&&resumes==0&&writes==before);balanced(&p); prepare(&p,true,true,false);before=writes; assert(atcphy_mux_set(&mux,&same)==0&&writes==before&&resumes==1);balanced(&p); + /* Binding/identity failure must not reach USB PM or PHY programming. */ + for(unsigned int missing=0;missing<3;missing++){ + prepare(&p,true,true,false);before=writes; + if(missing==0)controller.dev.driver=NULL; + if(missing==1)host_device.driver=NULL; + if(missing==2)host_device.parent=&wrong_controller; + assert(safe(&p)==(missing==2?-ENODEV:-EAGAIN)&&writes==before&&resumes==0);balanced(&p); + } + /* Root state can change during lookup despite USB device locks. */ + for(unsigned int event=0;event<2;event++){ + prepare(&p,true,true,false);before=writes; + if(event==0)detach_at_lookup=1;else quiesce_at_lookup=1; + int ret=safe(&p);fprintf(stderr,"post-child admission event %u returned %d writes %u\n",event,ret,writes-before); + assert(ret==-ENODEV&&writes==before);balanced(&p); + } + /* Selectively suspended upstream link is not resumed by root ownership alone. */ + for(unsigned int swap=0;swap<2;swap++){ + prepare(&p,true,true,swap);link_present=link_suspended=stuck_ack=true; + int link_ret=safe(&p);fprintf(stderr,"selective link SAFE returned %d, root resumes %d, link resumes %d\n",link_ret,resumes,link_resumes); + assert(link_ret==0&&resumes==1&&link_resumes==1&&!link_suspended);balanced(&p); + } + for(unsigned int fault=0;fault<3;fault++){ + prepare(&p,true,true,false);link_present=link_suspended=true;before=writes; + unsigned int retained=p.dp_usb_context_valid,owners=p.dp_owned_pmas; + if(fault==0)link_error=-EIO; + if(fault==1)link_detach_on_resume=1; + if(fault==2)link_dead_on_resume=1; + assert(safe(&p)==(fault?-ENODEV:-EIO)&&writes==before&&p.dp_usb_context_valid==retained&&p.dp_owned_pmas==owners);balanced(&p); + } + prepare(&p,true,true,false);link_present=link_suspended=true;link_error=-EIO; + assert(safe(&p)==-EIO);balanced(&p);link_error=0;assert(safe(&p)==0);balanced(&p); + prepare(&p,true,true,false);link_present=link_suspended=true;fail_caller="atcphy_pipehandler_lock";before=writes; + assert(safe(&p)==-ETIMEDOUT&&writes==before&&p.mode_dirty&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3);balanced(&p); + fail_caller=NULL;assert(safe(&p)==0);balanced(&p); + for(unsigned int count=0;count<2;count++){ + prepare(&p,true,true,false);roots[3].maxchild=count?2:0;before=writes; + assert(safe(&p)==(count?-ENODEV:-EAGAIN)&&writes==before);balanced(&p); + } + prepare(&p,true,true,false);link_present=true;link_device.state=USB_STATE_NOTATTACHED; + assert(safe(&p)==0&&link_resumes==0);balanced(&p); + /* Startup with no old child is allowed; no fake dock or link is required. */ + prepare(&p,true,true,false);assert(safe(&p)==0&&link_resumes==0);balanced(&p); + puts("PASS actual upstream-link ownership: both PMAs, selective suspend, resume/removal/death/ACK failures, retry, unknown port map and no-child startup"); puts("PASS actual PM wrapper + actual parking/restore: both PMAs, gated clock, PM error/retry, fatal ACK/retry, graph/controller/root/removal gates, legacy/ordinary and cached mode"); return 0; } @@ -187,7 +284,10 @@ def ownership(root, usb_unreachable=False): helpers='' if has_pm: helpers=ns['block'](s,'struct atcphy_usb3_pm {')+';\n' - helpers+='#if IS_REACHABLE(CONFIG_USB)\n'+ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' + helpers+='#if IS_REACHABLE(CONFIG_USB)\n' + if 'static int atcphy_usb3_link_get(' in s: + helpers+=ns['function'](s,'atcphy_usb3_link_get')+'\n'+ns['function'](s,'atcphy_usb3_link_put')+'\n' + helpers+=ns['function'](s,'atcphy_usb3_pm_find')+'\n#endif\n' helpers+=ns['function'](s,'atcphy_usb3_pm_get')+'\n'+ns['function'](s,'atcphy_usb3_pm_put')+'\n' # The fixture extractor adapts the locked body; extract the wrapper directly. marker=re.search(r'^static int atcphy_mux_set\(',s,re.M).group() @@ -201,7 +301,7 @@ def ownership(root, usb_unreachable=False): typec_caller=ns['block'](typec_source,'int typec_mux_set(')+'\n' hcd_source=(root/'drivers/usb/core/hcd.c').read_text() hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' - text=prefix+prod+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls + text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls return text def probe(root): @@ -368,7 +468,7 @@ def kconfig(root, output): compiler = Path(os.environ.get('KCONFIG_CONF', root / 'scripts/kconfig/conf')) if not compiler.is_file(): subprocess.run(['make', '-C', str(root), 'O=' + str(directory / 'producer'), - 'ARCH=arm64', 'LLVM=1', 'scripts/kconfig/conf'], check=True) + 'ARCH=arm64', 'LLVM=1', 'allnoconfig'], check=True) compiler = directory / 'producer/scripts/kconfig/conf' for usb, phy_setting in [('n', 'y'), ('y', 'y'), ('m', 'y'), ('m', 'm')]: case = directory / (usb + '-' + phy_setting) @@ -451,3 +551,88 @@ def probe_dispatch(root): } """ return unit + '\n' + port + '\n' + controls + + +def upstream_resume_boundary(root): + """Actual usbcore dispatch: root bus resume does not resume its child.""" + source = (root / 'drivers/usb/core/generic.c').read_text() + function = block(source, 'int usb_generic_driver_resume(') + prefix = r""" +#include +#include +#include +#include +typedef int pm_message_t; +struct usb_device { struct usb_device *parent; }; +static int bus_resumes, port_resumes, notifications, port_error; +static bool child_suspended = true; +static int hcd_bus_resume(struct usb_device *u, pm_message_t m) +{ (void)m; assert(!u->parent); bus_resumes++; return 0; } +static int usb_port_resume(struct usb_device *u, pm_message_t m) +{ (void)m; assert(u->parent); port_resumes++; if(port_error)return port_error; child_suspended=false; return 0; } +static void usbfs_notify_resume(struct usb_device *u) +{ (void)u; notifications++; } +""" + controls = r""" +int main(void) { + struct usb_device root={0}, child={.parent=&root}; + assert(usb_generic_driver_resume(&root,1)==0); + assert(bus_resumes==1&&port_resumes==0&&child_suspended&¬ifications==1); + assert(usb_generic_driver_resume(&child,1)==0); + assert(bus_resumes==1&&port_resumes==1&&!child_suspended&¬ifications==2); + child_suspended=true;port_error=-EIO; + assert(usb_generic_driver_resume(&child,1)==-EIO&&child_suspended&¬ifications==2); + puts("PASS actual usbcore root/child resume dispatch and failed child resume"); + return 0; +} +""" + return prefix + function + controls + + +def controller_lifecycle(root): + from teardown_pm import unit_for + return unit_for(root) + + +def asynchronous_root_state(root): + """Actual USB state API does not use the USB device mutex as its fence.""" + source = (root / 'drivers/usb/core/hub.c').read_text() + prefix = r""" +#include +#include +#include +enum usb_device_state { USB_STATE_NOTATTACHED, USB_STATE_CONFIGURED, USB_STATE_SUSPENDED }; +struct usb_device; +struct usb_port { struct usb_device *child; }; +struct usb_hub { struct usb_port **ports; }; +struct device { int unused; }; +struct config { struct { unsigned bmAttributes; } desc; }; +struct usb_device { struct device dev; struct usb_device *parent; struct usb_hub *hub; enum usb_device_state state; int maxchild; unsigned quirks; struct config *actconfig; unsigned long active_duration; bool device_locked; }; +static unsigned long jiffies=10; +static int device_state_lock; +#define USB_QUIRK_IGNORE_REMOTE_WAKEUP 1 +#define USB_CONFIG_ATT_WAKEUP 2 +#define spin_lock_irqsave(l,f) do { assert(!*(l));*(l)=1;(f)=0; } while(0) +#define spin_unlock_irqrestore(l,f) do { assert(*(l));*(l)=0;(void)(f); } while(0) +static struct usb_hub *usb_hub_to_struct_hub(struct usb_device *u) { return u->hub; } +static void update_port_device_state(struct usb_device *u) { (void)u; } +static void trace_usb_set_device_state(struct usb_device *u) { (void)u; } +static void device_set_wakeup_capable(struct device *d,int w) { (void)d;(void)w; } +""" + controls = r""" +int main(void) { + struct usb_device child={.state=USB_STATE_CONFIGURED,.device_locked=true}; + struct usb_port port={.child=&child};struct usb_port *ports[]={&port}; + struct usb_hub hub={.ports=ports}; + struct usb_device root={.state=USB_STATE_CONFIGURED,.maxchild=1,.hub=&hub,.device_locked=true}; + child.parent=&root; + usb_set_device_state(&root,USB_STATE_NOTATTACHED); + assert(root.state==USB_STATE_NOTATTACHED&&child.state==USB_STATE_NOTATTACHED); + assert(root.device_locked&&child.device_locked&&!device_state_lock); + usb_set_device_state(&child,USB_STATE_CONFIGURED); + assert(child.state==USB_STATE_NOTATTACHED); + puts("PASS actual asynchronous USB detach while device locks are held; no resurrection"); + return 0; +} +""" + return prefix + block(source, 'static void update_usb_device_state(') + block(source, 'static void recursively_mark_NOTATTACHED(') + block(source, 'void usb_set_device_state(') + controls diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py index 94349823fa03cb..3365394b513e4b 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/run.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/run.py @@ -24,6 +24,7 @@ import pm import idle import rate +import boot def parking_unit(root): @@ -35,7 +36,7 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--source', type=Path, default=Path(__file__).resolve().parents[6]) parser.add_argument('--output', type=Path, help='retain generated C, commands and results here') - parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle'), default='all') + parser.add_argument('--group', choices=('all', 'edid', 'rate', 'phy', 'lifecycle', 'boot'), default='all') args = parser.parse_args() root = args.source.resolve() output = args.output or Path(tempfile.mkdtemp(prefix='apple-dcp-selftest-')) @@ -46,6 +47,7 @@ def main(): 'orientation-faults', 'backend-change', 'safe-first', 'failed-flip-return', 'live-park')) tests = [ + ('boot', 'oslog-events', boot.oslog, ((),), ()), ('edid', 'service-rebind', edid.rebind, ((),), ()), ('edid', 'provider-admission', edid.admission, ((),), ()), ('edid', 'ordered-rx', edid.rx_progress, ((),), ('-pthread',)), @@ -54,6 +56,9 @@ def main(): ('rate', 'metadata-decoder', lambda r: rate.metadata(r, output), ((),), ('-Wno-sign-compare',)), ('rate', 'bounded-retry', rate.retry, ((),), ()), ('phy', 'pm-ownership', pm.ownership, ((),), ()), + ('phy', 'controller-lifecycle', pm.controller_lifecycle, ((),), ('-pthread',)), + ('phy', 'asynchronous-root-state', pm.asynchronous_root_state, ((),), ()), + ('phy', 'upstream-resume-boundary', pm.upstream_resume_boundary, ((),), ()), ('phy', 'usb-unreachable', lambda r: pm.ownership(r, True), ((),), ()), ('lifecycle', 'cpuidle-s2idle', idle.s2idle, ((),), ('-Wno-unused-parameter', '-Wno-unused-variable')), ('lifecycle', 'typec-probe', pm.probe, ((),), ()), diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py new file mode 100644 index 00000000000000..1f17261136a9c3 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/teardown_pm.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual wrapper/driver-core/DWC3/xHCI teardown order with modeled scheduling. + +Device mutex APIs use pthread mutexes. Clock, unregister, PM and hardware leaves +are mocks; this is not lockdep, system-PM or electrical atomicity qualification. +""" +import pm +from common import block + +def unit_for(source): + unit=pm.ownership(source) + unit=unit.replace('int main(void)','int __attribute__((unused)) ownership_main(void)',1) + unit=unit.replace('struct device_driver { const char *name; };', 'struct device;\nstruct device_driver { const char *name; void (*remove)(struct device *); void *dev_groups; };') + unit=unit.replace('struct device { int refs;', 'struct bus_type { bool need_parent_lock; void (*remove)(struct device *); void (*dma_cleanup)(struct device *); };\nstruct device_private { int knode_driver; };\nstruct device { struct bus_type *bus; struct device_private *p; int kobj; int refs;') + unit=unit.replace('static struct device host_device;', 'static struct platform_device host_platform_device;\n#define host_device host_platform_device.dev') + unit=unit.replace('static bool controller_locked, host_locked;', 'static _Atomic bool controller_locked, host_locked;') + pre=r''' +#include +#include +static pthread_mutex_t controller_mutex=PTHREAD_MUTEX_INITIALIZER, host_mutex=PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t schedule_mutex=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; +static _Thread_local bool remover; +static atomic_bool handoff_active, early_write; +static bool removal_started, blocked_attempt, release_remove, inject_remove; +static int remove_target, susphy_calls, host_remove_calls; +static pthread_t remove_thread; +static void start_remove(void); +static void removal_lock_attempt(bool held) { + if(!remover||!held)return; + pthread_mutex_lock(&schedule_mutex);blocked_attempt=true; + pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); +} +static void observed_controller_write(void) { + if(atomic_load(&handoff_active))atomic_store(&early_write,true); + pthread_mutex_lock(&schedule_mutex); + removal_started=true;pthread_cond_broadcast(&schedule_cond); + while(atomic_load(&handoff_active)&&!release_remove)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); +} +''' + unit=unit.replace('#define CONFIG_USB 1',pre+'\n#define CONFIG_USB 1',1) + old=block(unit,'static void device_lock(') + new=r'''static void device_lock(struct device *d) { + if(d==&controller.dev){ + removal_lock_attempt(atomic_load(&controller_locked)); + pthread_mutex_lock(&controller_mutex);controller_locked=true; + }else{ + assert(d==&host_device);removal_lock_attempt(atomic_load(&host_locked)); + pthread_mutex_lock(&host_mutex);host_locked=true; + } +}''' + unit=unit.replace(old,new) + old=block(unit,'static void device_unlock(') + new=r'''static void device_unlock(struct device *d) { + if(d==&host_device){assert(host_locked);host_locked=false;pthread_mutex_unlock(&host_mutex);} + else {assert(d==&controller.dev&&controller_locked);if(!remover)atomic_store(&handoff_active,false);controller_locked=false;pthread_mutex_unlock(&controller_mutex);} +}''' + unit=unit.replace(old,new) + # The scheduler injection is a child-lookup interleaving, as in the review. + unit=unit.replace('if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);','if(inject_remove){inject_remove=false;start_remove();}\n if(detach_at_lookup)usb_set_device_state(u,USB_STATE_NOTATTACHED);',1) + unit=unit.replace('static void pm_runtime_put_noidle(struct device *d) {','static void pm_runtime_put_noidle(struct device *d) {\n if(remover&&d==&host_device)return;',1) + # Old source lacks platform-lock helpers. Keep fixture-only API definitions + # available without claiming they are invoked by the parent. + if 'device_lock(&pm->controller->dev)' not in (source/'drivers/phy/apple/atc.c').read_text(): + unit=unit.replace('static struct device *get_device(', 'static __attribute__((unused)) struct device *get_device(') + prefix=r''' +struct clk { int unused; }; struct reset_control { int unused; }; +struct xhci_hcd { atomic_int xhc_state; struct clk *clk,*reg_clk; struct reset_control *reset; struct usb_hcd *shared_hcd; }; +static struct xhci_hcd xhci_context; +#define XHCI_STATE_REMOVING 1 +#define hcd_to_xhci(h) (&xhci_context) +struct dwc3 { bool sys_wakeup; struct platform_device *xhci; }; +enum { DWC3_APPLE_PROBE_PENDING, DWC3_APPLE_NO_CABLE, DWC3_APPLE_HOST, DWC3_APPLE_DEVICE }; +struct dwc3_apple { struct dwc3 dwc; struct device *dev; int lock,state; void *role_sw; }; +static struct dwc3_apple apple_context; +#define to_dwc3_apple(d) container_of(d,struct dwc3_apple,dwc) +static bool inside_host_remove; +static void *platform_get_drvdata(struct platform_device *d) { return d==&controller?(void *)&apple_context.dwc:(void *)&hcd; } +static void dwc3_enable_susphy(struct dwc3 *d,bool v) { (void)d;assert(v);susphy_calls++;observed_controller_write(); } +static void device_init_wakeup(struct device *d,bool b) { (void)d;(void)b; } +static void platform_device_unregister(struct platform_device *d); +static void dwc3_gadget_exit(struct dwc3 *d) { (void)d;assert(0); } +static int dwc3_apple_core_stop(struct dwc3_apple *d) { (void)d;return 0; } +static void usb_role_switch_unregister(void *s) { (void)s; } +static void dwc3_core_remove_no_shutdown(struct dwc3 *d) { (void)d; } +static void usb_remove_hcd(struct usb_hcd *h) { host_remove_calls++;h->state=HC_STATE_QUIESCING;usb_set_device_state(h->self.root_hub,USB_STATE_NOTATTACHED); } +static void usb_phy_shutdown(void *p) { (void)p; } +static void usb_put_hcd(struct usb_hcd *h) { (void)h; } +static void clk_disable_unprepare(struct clk *c) { (void)c; } +static void reset_control_assert(struct reset_control *r) { (void)r; } +static void pm_runtime_disable(struct device *d) { (void)d; } +static void pm_runtime_set_suspended(struct device *d) { (void)d; } +static int pm_runtime_get_sync(struct device *d) { if(inside_host_remove&&d==&host_device){assert(xhci_context.xhc_state&XHCI_STATE_REMOVING);observed_controller_write();}return 0; } +static void pm_runtime_put_sync(struct device *d) { (void)d; } +static void pm_runtime_put(struct device *d) { (void)d; } +#define dev_attr_state_synced 0 +#define BUS_NOTIFY_UNBIND_DRIVER 0 +#define BUS_NOTIFY_UNBOUND_DRIVER 1 +#define KOBJ_UNBIND 0 +static bool device_links_busy(struct device *d) { (void)d;return false; } +static void device_links_unbind_consumers(struct device *d) { (void)d;assert(0); } +static void driver_sysfs_remove(struct device *d) { (void)d; } +static void bus_notify(struct device *d,int n) { (void)d;(void)n; } +static void device_remove_file(struct device *d,void *a) { (void)d;(void)a; } +static void device_remove_groups(struct device *d,void *a) { (void)d;(void)a; } +static void device_unbind_cleanup(struct device *d) { d->driver=NULL; } +static void device_links_driver_cleanup(struct device *d) { (void)d; } +static void klist_remove(void *k) { (void)k; } +static void device_pm_check_callbacks(struct device *d) { (void)d; } +static void kobject_uevent(void *k,int n) { (void)k;(void)n; } +''' + prefix=prefix.replace('#define dev_attr_state_synced 0','static int dev_attr_state_synced;') + unit=unit.replace('struct usb_hcd { int state;', 'struct usb_hcd { void *usb_phy; int state;',1) + # Extract actual callback order; clock/USB/reset leaves remain mocks. + host=(source/'drivers/usb/dwc3/host.c').read_text() + glue=(source/'drivers/usb/dwc3/dwc3-apple.c').read_text() + plat=(source/'drivers/usb/host/xhci-plat.c').read_text() + dd=(source/'drivers/base/dd.c').read_text() + functions=block(host,'void dwc3_host_exit(')+'\n'+block(glue,'static int dwc3_apple_exit(')+'\n'+block(glue,'static void dwc3_apple_remove(')+'\n'+block(plat,'void xhci_plat_remove(')+'\n' + functions+='\n'.join(block(dd,x) for x in ['static void __device_driver_lock(', 'static void __device_driver_unlock(', 'static void device_remove(', 'static void __device_release_driver(', 'void device_release_driver_internal(']) + controls=r''' +static void controller_remove(struct device *d) { assert(d==&controller.dev);dwc3_apple_remove(&controller); } +static void host_remove(struct device *d) { assert(d==&host_device);inside_host_remove=true;xhci_plat_remove(&host_platform_device);inside_host_remove=false; } +static void platform_device_unregister(struct platform_device *d) { assert(d==&host_platform_device);device_release_driver_internal(&d->dev,NULL,NULL); } +static void *run_remove(void *unused) { (void)unused;remover=true;device_release_driver_internal(remove_target?&host_device:&controller.dev,NULL,NULL);return NULL; } +static void start_remove(void) { + pthread_mutex_lock(&schedule_mutex); + assert(pthread_create(&remove_thread,NULL,run_remove,NULL)==0); + while(!blocked_attempt&&!removal_started)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); +} +int main(void) { + for(remove_target=0;remove_target<2;remove_target++) { + struct apple_atcphy p;prepare(&p,true,true,false); + struct device_private private={0};controller.dev.p=&private;host_device.p=&private; + controller_driver.remove=controller_remove;host_driver.remove=host_remove; + apple_context=(struct dwc3_apple){.dwc={.xhci=&host_platform_device},.dev=&controller.dev,.state=DWC3_APPLE_HOST}; + xhci_context=(struct xhci_hcd){0}; + inject_remove=true;removal_started=blocked_attempt=release_remove=false; + atomic_store(&early_write,false);atomic_store(&handoff_active,true); + susphy_calls=host_remove_calls=0; + int ret=safe(&p); + atomic_store(&handoff_active,false); + pthread_mutex_lock(&schedule_mutex);release_remove=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + assert(pthread_join(remove_thread,NULL)==0); + fprintf(stderr,"connected teardown target=%d ret=%d early_write=%d susphy=%d host_remove=%d\n",remove_target,ret,atomic_load(&early_write),susphy_calls,host_remove_calls); + assert(ret==0&&!atomic_load(&early_write)&&host_remove_calls==1); + assert(susphy_calls==(remove_target?0:1));balanced(&p); + } + puts("PASS actual PM wrapper connected to driver-core/DWC3/xHCI removal; synchronous teardown waits until owned transaction ends"); + return 0; +} +''' + return unit+prefix+functions+controls From 7256ade0e7f3fe962b5afe4ce84d39352a1bbcfd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 23:21:04 +0000 Subject: [PATCH 454/631] dt-bindings: display: apple: allow T8140 provider data on DCPEXT The shared H17P provider callback consumes the same five opaque, exact-size table inputs for T8140 DCP and DCPEXT. Permit them on both controllers; keep the existing byte lengths and restrictions on every other compatible. No requiredness or runtime consumer change is made. Independent validation reproduces five DCPEXT errors on the parent and passes the correction, 113 contract controls and dt-doc validation. Live firmware presence and loader/FDT qualification remain separate. Signed-off-by: Ryan Murray Assisted-by: LLM --- Documentation/devicetree/bindings/display/apple,dcp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index bba5d64f504517..0fe31267f99e16 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -320,7 +320,7 @@ allOf: properties: compatible: contains: - const: apple,t8140-dcp + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] then: properties: apple,power-lut-data-x: false From 34429e575bd78987dcba9dcc433e69062ae3d08d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Wed, 14 Oct 2026 01:22:15 +0000 Subject: [PATCH 455/631] usb: typec: serialize display teardown and preserve recovery ownership Quiesce the display before SAFE provider writes and role-clock shutdown. Serialize connector transitions across mux handles, retain failed SAFE ownership, and check finalization before admitting a new DP route. Keep SRCU reader accounting local and preserve module/provider lifetime. Track consumed mux release separately so retries cannot release twice. Refuse reset when provider acquisition cannot establish SAFE. Retain compact actual-source regression coverage for transaction ordering, error paths, reference lifetime and release accounting. Independent review passes all five retained failure witnesses. Complete ARM64 Image/modules/ DTB build passes. Target recovery qualification is pending; this is not a hardware hotplug or suspend/wake pass. Signed-off-by: Ryan Murray Assisted-by: LLM --- drivers/gpu/drm/apple/dcp.c | 221 +++++++++- drivers/usb/typec/mux.c | 178 +++++++- drivers/usb/typec/mux.h | 6 + drivers/usb/typec/tipd/core.c | 97 ++++- drivers/usb/typec/tipd/tps6598x.h | 3 + include/linux/usb/typec_mux.h | 9 + .../drivers/gpu/apple-dcp/accounting.py | 318 ++++++++++++++ .../selftests/drivers/gpu/apple-dcp/pm.py | 24 +- .../drivers/gpu/apple-dcp/quiesce.py | 147 +++++++ .../selftests/drivers/gpu/apple-dcp/run.py | 11 +- .../drivers/gpu/apple-dcp/transaction.py | 406 ++++++++++++++++++ 11 files changed, 1375 insertions(+), 45 deletions(-) create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py create mode 100644 tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 223b6e0daa0bdd..115be64dc8503f 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -426,13 +426,22 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * crossbar nor restart the tunnel pixel clock behind our back. */ scoped_guard(mutex, &dcp->tb_lock) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else - dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - if (ret) - return ret; - route->xbar_up = false; + if (route->deactivate_error) + return route->deactivate_error; + if (!route->deactivate_xbar_done) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); + /* Deselect always unlocks: never consume that selection twice. */ + route->deactivate_xbar_done = true; + route->xbar_up = false; + if (ret) { + /* Idle is unproven; retain the owner and the real error. */ + route->deactivate_error = ret; + return ret; + } + } route->selected = false; if (dcp->active_typec_route == route) @@ -826,6 +835,8 @@ dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, struct apple_dcp_typec_route *owner = port->owner; struct apple_dcp *dcp; + if (port->safe_pending) + continue; if (!owner || owner->tunnel || owner == port->target) continue; @@ -914,19 +925,192 @@ void dcp_typec_reorder(void) return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); } -/* - * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate - * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that - * the tunnel pixel clock runs, and re-assert the DP IN adapter's - * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; - * after a re-link only the clocks are brought back (the mux selection and - * the ATC output enable are kept, see dcp_tunnel_crossbar_down()). Runs - * inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. - */ -int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +static int dcp_typec_route_prepare(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp *dcp; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE) + return port->safe_pending ? -EBUSY : 0; + if (!port->owner || port->owner->tunnel) + return 0; + dcp = port->owner->dcp; + /* Stop new admission, but retain the owner until the set phase succeeds. */ + if (!port->safe_pending) + port->safe_finish_ready = false; + port->safe_pending = true; + port->applied_valid = false; + port->dp_wanted = false; + port->dp_hpd = false; + /* HPD absence alone is not proof of a completed firmware shutdown. */ + if (dcp_is_external_h17p(dcp)) { + scoped_guard(mutex, &dcp->external_lock) { + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable && dcp->external_stopped && + !READ_ONCE(dcp->dptxport[0].connected)) { + port->hpd = false; + return 0; + } + } + } + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (ret) + return ret; + port->hpd = false; + return 0; +} + +static int dcp_typec_route_finish(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending || + port->safe_finish_ready) + return 0; + ret = port->owner ? dcp_typec_route_deactivate(port->owner) : 0; + if (ret) + return ret; + port->safe_finish_ready = true; + return 0; +} + +static void dcp_typec_route_commit(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending) + return; + if (port->owner) + port->preferred_route = port->owner; + port->owner = NULL; + port->safe_pending = false; + port->safe_finish_ready = false; + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = 0; + port->applied_conf = 0; + port->applied_valid = true; + dcp_typec_pipeline_freed(); +} + +static int dcp_typec_route_set(struct typec_mux_dev *mux, + struct typec_mux_state *state) { struct apple_dcp_typec_route *route; int ret; + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp_typec_route *best = NULL; + bool is_dp = dcp_typec_route_is_dp(state); + struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; + u32 dp_status = dp_data ? dp_data->status : 0; + u32 dp_conf = dp_data ? dp_data->conf : 0; + bool hpd, was_counted; + int ret = 0; + + guard(mutex)(&dcp_typec_fabric_lock); + + if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) + return 0; + if (port->safe_pending) + return -EBUSY; + + /* + * Every candidate route for this port is notified with the same state, + * so only the first to arrive does the work; the others return early. + * + * Deliberately not a nominated coordinator: fwnode_typec_mux_get() + * caps the providers one connector may have and drops the remainder + * without a word, so a nominated route might never be called at all -- + * and the port would then never be routed. + */ + if (port->applied_valid && port->applied_alt == state->alt && + port->applied_mode == state->mode && + port->applied_status == dp_status && port->applied_conf == dp_conf) + return 0; + + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = dp_status; + port->applied_conf = dp_conf; + /* Failed route acquisition must remain retryable on the next update. */ + port->applied_valid = false; + + /* did the pairing pass count this port's direct stream so far? */ + was_counted = port->dp_wanted && port->dp_hpd; + + if (!is_dp) { + port->dp_wanted = false; + port->dp_hpd = false; + + /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ + if (port->owner && port->owner->tunnel) { + port->applied_valid = true; + return 0; + } + if (port->owner) { + struct apple_dcp *dcp = port->owner->dcp; + + port->preferred_route = port->owner; + if (port->hpd || dcp->typec_cable_connected || + (dcp->typec_connector && + dcp->typec_connector->connected)) + dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + port->hpd = false; + ret = dcp_typec_route_deactivate(port->owner); + if (ret) + return ret; + port->owner = NULL; + + if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) + dcp_dptx_connect(dcp, 0); + dcp_typec_pipeline_freed(); + } else if (was_counted && dcp_typec_keep_order()) { + /* a stream the plan left dark is gone: plan the rest */ + dcp_typec_rebalance_locked(NULL, 0); + } + + /* Data-only USB4 changes do not invalidate other display routes. */ + port->applied_valid = true; + return 0; + } + + /* + * A Thunderbolt DP tunnel still owns the port (its teardown is on the + * way): don't act on or remember this state; the tunnel teardown drops + * what was recorded so the next update is applied. + */ + if (port->owner && port->owner->tunnel) { + port->applied_valid = false; + return 0; + } + + hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); + port->dp_wanted = true; + port->dp_hpd = hpd; + + /* + * Until a compositor owns the display, a direct DP-alt stream has a + * pipeline only while its sink asserts HPD, and the pairing pass + * places it: HPD coming or going is the stream connecting or + * disconnecting as far as a compositor can tell. Otherwise the port + * takes the lowest free pipeline on DP entry, as a compositor does. + */ + if (dcp_typec_keep_order()) { + struct apple_dcp_typec_route *owner = port->owner; guard(mutex)(&dcp->tb_lock); route = dcp->active_typec_route; @@ -1037,7 +1221,8 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) guard(mutex)(&dcp->tb_lock); route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar) + if (!route || !route->tunnel || !route->active_xbar || + route->deactivate_xbar_done) return -ENODEV; if (!dcp->tb_clock_ok) { dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index a7cf8c46c541e8..31241ed2beb643 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -264,6 +264,37 @@ struct typec_mux { unsigned int num_mux_devs; }; +static struct typec_mux_context *typec_mux_context_get(struct fwnode_handle *fwnode) +{ + struct typec_mux_context *context; + + guard(mutex)(&typec_mux_contexts_lock); + list_for_each_entry(context, &typec_mux_contexts, list) { + if (context->fwnode == fwnode) { + context->users++; + return context; + } + } + context = kzalloc_obj(*context); + if (!context) + return ERR_PTR(-ENOMEM); + context->fwnode = fwnode_handle_get(fwnode); + context->users = 1; + mutex_init(&context->lock); + list_add_tail(&context->list, &typec_mux_contexts); + return context; +} + +static void typec_mux_context_put(struct typec_mux_context *context) +{ + guard(mutex)(&typec_mux_contexts_lock); + if (--context->users) + return; + list_del(&context->list); + fwnode_handle_put(context->fwnode); + kfree(context); +} + static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -355,7 +386,13 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) } for (i = 0; i < count; i++) { - WARN_ON(!try_module_get(mux_devs[i]->dev.parent->driver->owner)); + if (READ_ONCE(mux_devs[i]->removed) || + !try_module_get(mux_devs[i]->owner)) { + err = -ENODEV; + while (i--) + module_put(mux_devs[i]->owner); + goto put_mux_devs; + } mux->mux_devs[i] = mux_devs[i]; } @@ -400,22 +437,148 @@ EXPORT_SYMBOL_GPL(typec_mux_put); int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) { - struct typec_mux_dev *mux_dev; + int index = mux->srcu_index; + + mux->prepared = 0; + mux->applied = false; + mux->owner = NULL; + mutex_unlock(&mux->context->lock); + srcu_read_unlock(&typec_mux_srcu, index); +} + +void typec_mux_abort(struct typec_mux *mux) +{ + unsigned int i; + + if (IS_ERR_OR_NULL(mux)) + return; + if (WARN_ON_ONCE(mux->owner != current)) + return; + for (i = mux->prepared; i > 0; i--) { + struct typec_mux_dev *dev = mux->mux_devs[i - 1]; + + if (dev->abort) + dev->abort(dev, &mux->transaction); + } + typec_mux_unlock(mux); +} +EXPORT_SYMBOL_GPL(typec_mux_abort); + +int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) +{ + unsigned int i; + int index, ret; + + if (IS_ERR_OR_NULL(mux)) + return 0; + index = srcu_read_lock(&typec_mux_srcu); + /* Nonblocking acquisition also rejects same-domain forwarding recursion. */ + if (!mutex_trylock(&mux->context->lock)) { + srcu_read_unlock(&typec_mux_srcu, index); + return -EBUSY; + } + mux->srcu_index = index; + mux->owner = current; + mux->transaction = *state; + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + + if (READ_ONCE(dev->removed)) { + typec_mux_unlock(mux); + return -ENODEV; + } + } + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + + mux->prepared = i + 1; + if (!dev->prepare) + continue; + ret = dev->prepare(dev, &mux->transaction); + if (ret) + goto abort; + } + return 0; +abort: + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_begin); + +int typec_mux_apply(struct typec_mux *mux) +{ unsigned int i; int ret; if (IS_ERR_OR_NULL(mux)) return 0; + if (WARN_ON_ONCE(mux->owner != current)) + return -EINVAL; + if (mux->applied) + return 0; + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; + ret = dev->set(dev, &mux->transaction); + if (ret && ret != -EOPNOTSUPP) { + typec_mux_abort(mux); + return ret; + } + } + mux->applied = true; + return 0; +} +EXPORT_SYMBOL_GPL(typec_mux_apply); + +int typec_mux_end(struct typec_mux *mux) +{ + unsigned int i; + int ret; + + if (IS_ERR_OR_NULL(mux)) + return 0; + ret = typec_mux_apply(mux); + if (ret) + return ret; + /* Finalizers can fail, but must not release their software reservation. */ for (i = 0; i < mux->num_mux_devs; i++) { mux_dev = mux->mux_devs[i]; - ret = mux_dev->set(mux_dev, state); - if (ret && ret != -EOPNOTSUPP) - return ret; + if (!dev->finish) + continue; + ret = dev->finish(dev, &mux->transaction); + if (ret) + goto abort; } + /* Only this no-fail publication phase may release owner reservations. */ + for (i = 0; i < mux->num_mux_devs; i++) { + struct typec_mux_dev *dev = mux->mux_devs[i]; return 0; +abort: + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_end); + +/* Preparation-only callers leave recovery ownership pending, never committed. */ +int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state) +{ + int ret = typec_mux_begin(mux, state); + + if (!ret) + typec_mux_abort(mux); + return ret; +} +EXPORT_SYMBOL_GPL(typec_mux_prepare); + +int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) +{ + int ret = typec_mux_begin(mux, state); + + if (ret) + return ret; + return typec_mux_end(mux); } EXPORT_SYMBOL_GPL(typec_mux_set); @@ -453,6 +616,10 @@ typec_mux_register(struct device *parent, const struct typec_mux_desc *desc) return ERR_PTR(-ENOMEM); mux_dev->set = desc->set; + mux_dev->prepare = desc->prepare; + mux_dev->finish = desc->finish; + mux_dev->commit = desc->commit; + mux_dev->abort = desc->abort; device_initialize(&mux_dev->dev); mux_dev->dev.parent = parent; @@ -487,6 +654,7 @@ void typec_mux_unregister(struct typec_mux_dev *mux_dev) { if (!IS_ERR_OR_NULL(mux_dev)) device_unregister(&mux_dev->dev); + } } EXPORT_SYMBOL_GPL(typec_mux_unregister); diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 0f1547fdbd11ae..1dc9d90d2cdaad 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -13,8 +13,14 @@ struct typec_switch_dev { }; struct typec_mux_dev { + struct module *owner; struct device dev; typec_mux_set_fn_t set; + typec_mux_set_fn_t prepare; + typec_mux_set_fn_t finish; + typec_mux_notify_fn_t commit; + typec_mux_notify_fn_t abort; + bool removed; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index b936bb171fbd7f..a431cbe431ffba 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -217,6 +217,7 @@ static void tps6598x_set_data_role(struct tps6598x *tps, enum typec_data_role role, bool connected) { enum usb_role role_val; + int ret; if (role == TYPEC_HOST) role_val = USB_ROLE_HOST; @@ -226,8 +227,11 @@ static void tps6598x_set_data_role(struct tps6598x *tps, if (!connected) role_val = USB_ROLE_NONE; - usb_role_switch_set_role(tps->role_sw, role_val); + ret = usb_role_switch_set_role(tps->role_sw, role_val); + if (ret) + return ret; typec_set_data_role(tps->port, role); + return 0; } static int tps6598x_connect(struct tps6598x *tps, u32 status) @@ -260,7 +264,9 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status) else typec_set_orientation(tps->port, TYPEC_ORIENTATION_NORMAL); typec_set_mode(tps->port, TYPEC_STATE_USB); - tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); + ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); + if (ret) + return ret; tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) @@ -282,11 +288,37 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status) typec_set_pwr_opmode(tps->port, TYPEC_PWR_MODE_USB); typec_set_pwr_role(tps->port, TPS_STATUS_TO_TYPEC_PORTROLE(status)); typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status)); - typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); - typec_set_mode(tps->port, TYPEC_STATE_SAFE); - tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); +} + +static int tps6598x_disconnect(struct tps6598x *tps, u32 status) +{ + struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; + bool prepared = tps->remove_safe_prepared; + int ret; + + tps->remove_safe_prepared = false; + tps6598x_drop_partner(tps, status); + if (!prepared) { + typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); + ret = typec_mux_begin(tps->mux, &safe); + if (ret) + return ret; + /* Park while the host still supplies the PIPE backend clock. */ + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + } + ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); + if (ret) { + typec_mux_abort(tps->mux); + return ret; + } + ret = typec_mux_end(tps->mux); + if (ret) + return ret; power_supply_changed(tps->psy); + return 0; } int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, @@ -538,7 +570,9 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) if (ret) dev_err(tps->dev, "failed to register partner\n"); } else { - tps6598x_disconnect(tps, status); + ret = tps6598x_disconnect(tps, status); + if (ret) + dev_err(tps->dev, "hardware disconnect failed: %d\n", ret); } } @@ -2248,10 +2282,24 @@ static void cd321x_remove(struct tps6598x *tps) ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) - dev_warn(tps->dev, - "failed to close Thunderbolt/USB4 session on remove: %d\n", - ret); - usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + return ret; + /* Display quiescence must precede even an ordinary host-role shutdown. */ + typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); + ret = typec_mux_begin(tps->mux, &safe); + if (ret) + return ret; + /* Park while the host still supplies the PIPE backend clock. */ + ret = typec_mux_apply(tps->mux); + if (ret) + return ret; + ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); + if (ret) { + typec_mux_abort(tps->mux); + return ret; + } + /* Same task/handle completes without another post-shutdown set callback. */ + tps->remove_safe_prepared = true; + return 0; } @@ -2333,9 +2381,20 @@ int tipd_init(struct tps6598x *tps) if (ret) goto err_role_put; - ret = tps->data->register_port(tps, fwnode); - if (ret) + tps->mux = fwnode_typec_mux_get(fwnode); + if (IS_ERR(tps->mux)) { + ret = PTR_ERR(tps->mux); + /* No provider reference means rollback cannot prove SAFE. */ + reset_allowed = false; goto err_role_put; + } + + ret = tps->data->register_port(tps, fwnode); + if (ret) { + /* No live port remains to prove SAFE on failed registration. */ + reset_allowed = false; + goto err_mux_put; + } if (status & TPS_STATUS_PLUG_PRESENT) { ret = -EINVAL; @@ -2344,7 +2403,7 @@ int tipd_init(struct tps6598x *tps) if (!tps->data->read_data_status(tps)) goto err_unregister_port; ret = tps->data->connect(tps, status); - if (ret) + if (ret) { dev_err(tps->dev, "failed to register partner\n"); } @@ -2397,6 +2456,8 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { + int ret = 0; + tipd_debugfs_unregister(tps); if (!tps->irq) @@ -2407,12 +2468,18 @@ void tipd_remove(struct tps6598x *tps) if (tps->data->remove) tps->data->remove(tps); - tps6598x_disconnect(tps, 0); + if (!ret) + ret = tps6598x_disconnect(tps, 0); + else + tps6598x_drop_partner(tps, 0); + if (ret) + dev_err(tps->dev, "hardware teardown failed on remove: %d\n", ret); tps->data->unregister_port(tps); usb_role_switch_put(tps->role_sw); /* Reset PD controller to remove any applied patch */ - tps->data->reset(tps); + if (!ret) + tps->data->reset(tps); if (tps->reset) gpiod_set_value_cansleep(tps->reset, 1); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 435d2a8969c4a8..c47556384a4383 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -320,6 +320,9 @@ struct tipd_data { }; struct tps6598x { + struct typec_mux *mux; + /* CD remove leaves one open SAFE transaction for generic completion. */ + bool remove_safe_prepared; struct device *dev; struct regmap *regmap; struct mutex lock; /* device lock */ diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index b99701e512527a..2afc5e896835f4 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -86,6 +86,9 @@ struct typec_mux_state { void *data; }; +typedef void (*typec_mux_notify_fn_t)(struct typec_mux_dev *mux, + struct typec_mux_state *state); + typedef int (*typec_mux_set_fn_t)(struct typec_mux_dev *mux, struct typec_mux_state *state); @@ -94,6 +97,12 @@ struct typec_mux_desc { typec_mux_set_fn_t set; const char *name; void *drvdata; + /* Quiesce consumers before any provider changes the shared route. */ + typec_mux_set_fn_t prepare; + /* Fallible finalization retains ownership; commit cannot fail. */ + typec_mux_set_fn_t finish; + typec_mux_notify_fn_t commit; + typec_mux_notify_fn_t abort; }; #if IS_ENABLED(CONFIG_TYPEC) diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py b/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py new file mode 100644 index 00000000000000..ce2791cc3a379c --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/accounting.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Source-connected accounting regressions retaining adverse-review assertions. + +SRCU, module and hardware leaves are models; the transaction, DCP and mux core +functions are extracted from the selected public source. No device is accessed. +""" +import quiesce +import transaction +from common import block + +def srcu_index(ROOT): + unit=quiesce.make_unit(ROOT) + main=block(unit,'int main(') + unit=unit.replace(main,main.replace('int main(','int __attribute__((unused)) prior_main(',1),1) + old_lock=block(unit,'static int srcu_read_lock(') + old_unlock=block(unit,'static void srcu_read_unlock(') + old_mutex_unlock=block(unit,'static void mutex_unlock(') + leaves=r''' + static pthread_mutex_t schedule_mutex=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; + static int srcu_bank,bank_readers[2],released_index=-1; + static bool pause_unlock,pause_seen,resume_unlock; + static _Thread_local int expected_index=-1; + static int srcu_read_lock(int *s) { + (void)s;pthread_mutex_lock(&schedule_mutex);int i=srcu_bank;bank_readers[i]++;expected_index=i;pthread_mutex_unlock(&schedule_mutex);return i; + } + static void srcu_read_unlock(int *s,int i) { + (void)s;pthread_mutex_lock(&schedule_mutex);released_index=i; + fprintf(stderr,"SRCU unlock: acquired=%d released=%d bank0=%d bank1=%d\n",expected_index,i,bank_readers[0],bank_readers[1]); + assert(i==expected_index&&bank_readers[i]>0);bank_readers[i]--;expected_index=-1; + pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + } + static void mutex_unlock(struct mutex *m) { + assert(!pthread_mutex_unlock(&m->value)); + if(m==&shared_context.lock&&pause_unlock) { + pthread_mutex_lock(&schedule_mutex);pause_seen=true;pause_unlock=false;pthread_cond_broadcast(&schedule_cond); + while(!resume_unlock)pthread_cond_wait(&schedule_cond,&schedule_mutex); + pthread_mutex_unlock(&schedule_mutex); + } + } + ''' + unit=unit.replace(old_lock,'',1).replace(old_unlock,'',1).replace(old_mutex_unlock,leaves,1) + unit+=r''' + static void *first_actor(void *p) { + struct typec_mux *m=p;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};assert(typec_mux_set(m,&safe)==0);return NULL; + } + int main(void) { + struct cd321x c;initial(&c);mux.num_mux_devs=0;pause_unlock=true; + pthread_t thread;assert(!pthread_create(&thread,NULL,first_actor,&mux)); + pthread_mutex_lock(&schedule_mutex);while(!pause_seen)pthread_cond_wait(&schedule_cond,&schedule_mutex); + srcu_bank=1;pthread_mutex_unlock(&schedule_mutex); + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};assert(typec_mux_begin(&mux,&safe)==0); + assert(mux.srcu_index==1&&bank_readers[0]==1&&bank_readers[1]==1); + pthread_mutex_lock(&schedule_mutex);resume_unlock=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_mutex); + assert(!pthread_join(thread,NULL));typec_mux_abort(&mux);assert(bank_readers[0]==0&&bank_readers[1]==0); + puts("PASS same-handle SRCU read-index balance across context unlock"); + } + ''' + return unit + +def resources(ROOT): + actual=next(u for n,u,c,f in transaction.controls(ROOT) if n=='actual-dcp-normal-safe') + main=block(actual,'int main(') + actual=actual.replace(main,main.replace('int main(','int __attribute__((unused)) prior_route_main(',1),1) + actual=actual.replace('struct mux_control { int unused; };',r''' + struct mux_control; + struct mux_ops { int (*set)(struct mux_control *,int); }; + struct mux_chip { const struct mux_ops *ops; }; + struct mux_control { int lock,idle_state,cached_state; unsigned last_change;struct mux_chip *chip; }; + ''',1) + muxcore=(ROOT/'drivers/mux/core.c').read_text() + resource=r''' + #define MUX_IDLE_AS_IS -1 + #define MUX_CACHE_UNKNOWN -2 + static int hw_calls,hw_failures,fixed_restore_calls,fixed_restore_failures; + static unsigned ktime_get(void) { return 0; } + static void up(int *s) { (*s)++; } + static int hw_set(struct mux_control *m,int state) { (void)m;(void)state;hw_calls++;return hw_calls<=hw_failures?-EIO:0; } + ''' + resource+=block(muxcore,'static int mux_control_set(')+'\n'+block(muxcore,'int mux_control_deselect(')+'\n' + actual=actual.replace(block(actual,'static int mux_control_deselect('),resource,1) + actual=actual.replace(block(actual,'static int mux_control_select('),r'''static int mux_control_select(struct mux_control *m,unsigned n) { (void)m;(void)n;fixed_restore_calls++;return fixed_restore_calls<=fixed_restore_failures?-EIO:0; }''',1) + actual=actual.replace('static int gpiod_get_value_cansleep(void *p) { (void)p;return 0; }', 'static int gpio_level;\nstatic int gpiod_get_value_cansleep(void *p) { (void)p;return gpio_level; }',1) + actual+=r''' + static int peer_error; + static int peer_finish(struct typec_mux_dev *p,struct typec_mux_state *s) { (void)p;(void)s;return peer_error; } + int main(int argc,char **argv) { + (void)argc;struct cd321x c;initial(&c); + struct mux_ops ops={.set=hw_set};struct mux_chip chip={.ops=&ops}; + struct mux_control xbar={.lock=0,.idle_state=0,.cached_state=1,.chip=&chip}; + struct mux_control fixed_xbar={.lock=1,.idle_state=0,.cached_state=0,.chip=&chip}; + route.xbar=route.active_xbar=&xbar; + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + if(argv[1][0]=='n') { + assert(typec_mux_set(&mux,&safe)==0&&xbar.lock==1&&!port.owner&&!port.safe_pending); + puts("PASS actual DCP finalizer + real mux deselect resource balance");return 0; + } + if(argv[1][0]=='l') { + phy_provider.finish=peer_finish;mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;peer_error=-EIO; + assert(typec_mux_set(&mux,&safe)==-EIO&&xbar.lock==1&&port.owner==&route&&port.safe_pending&&port.safe_finish_ready); + peer_error=0;assert(typec_mux_set(&mux,&safe)==0&&xbar.lock==1&&!port.owner&&!port.safe_pending); + puts("PASS successful DCP finalizer is not repeated after a later provider failure");return 0; + } + if(argv[1][0]=='h')hw_failures=1; + if(argv[1][0]=='f') {display.fixed_connector_type=2;display.hdmi_hpd=(void *)1;gpio_level=1;display.xbar=&fixed_xbar;fixed_restore_failures=1;} + assert(typec_mux_set(&mux,&safe)==-EIO&&xbar.lock==1&&port.owner==&route&&port.safe_pending&&!port.safe_finish_ready); + bool selected_after_error=route.selected; + int ret=typec_mux_set(&mux,&safe); + fprintf(stderr,"finalizer retry: kind=%c ret=%d permits=%d selected_after_error=%d owner=%p pending=%d hw_calls=%d fixed_calls=%d\n",argv[1][0],ret,xbar.lock,selected_after_error,(void *)port.owner,port.safe_pending,hw_calls,fixed_restore_calls); + assert(xbar.lock==1); + if(argv[1][0]=='h')assert(ret==-EIO&&hw_calls==1&&port.owner==&route&&port.safe_pending); + if(argv[1][0]=='f')assert(ret==0&&!port.owner&&!port.safe_pending&&fixed_restore_calls==2); + return 0; + } + ''' + return actual + +def getter_unit(ROOT): + getter=transaction.lifetime(ROOT) + main=block(getter,'int main(') + renamed=main.replace('int main(','int __attribute__((unused)) prior_lifetime_main(',1) + renamed=renamed[:-1]+'return 0;\n}' + getter=getter.replace(main,renamed,1) + getter=getter.replace('struct typec_mux_dev {', 'struct module { unsigned refs;bool going; };\nstruct device { unsigned refs; };\nstruct typec_mux_dev { struct module *owner;struct device dev;',1) + getter=getter.replace(block(getter,'static void typec_mux_put('),'void typec_mux_put(struct typec_mux *mux);',1) + getter=getter.replace('#include ',r''' + #include + static int allocation_count,allocation_failure; + static void *review_alloc(size_t size) { if(++allocation_count==allocation_failure)return NULL;return calloc(1,size); } + ''',1) + getter=getter.replace('#define kzalloc_obj(x) calloc(1,sizeof(x))','#define kzalloc_obj(x) review_alloc(sizeof(x))',1) + getter+=r''' + #undef IS_ERR + #define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) + #define PTR_ERR(p) ((int)(intptr_t)(p)) + #define TYPEC_MUX_MAX_DEVS 3 + #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) + static struct typec_mux_dev graph_providers[2]; + static struct module modules[2]; + static bool graph_second_error; + static void *typec_mux_match(const struct fwnode_handle *f,const char *id,void *data) { (void)f;(void)id;(void)data;return NULL; } + static int fwnode_connection_find_matches(struct fwnode_handle *f,const char *id,void *data,void *(*fn)(const struct fwnode_handle *,const char *,void *),void **matches,unsigned max) { + (void)f;(void)id;(void)data;(void)fn;assert(max>=2); + graph_providers[0].dev.refs++;matches[0]=&graph_providers[0]; + if(graph_second_error)matches[1]=ERR_PTR(-EPROBE_DEFER); + else {graph_providers[1].dev.refs++;matches[1]=&graph_providers[1];} + return 2; + } + static bool try_module_get(struct module *m) { if(!m)return true;if(m->going)return false;m->refs++;return true; } + static void module_put(struct module *m) { if(m){assert(m->refs);m->refs--;} } + static void put_device(struct device *d) { assert(d->refs);d->refs--; } + ''' + getter='#define EPROBE_DEFER 517\n'+getter + mux=(ROOT/'drivers/usb/typec/mux.c').read_text() + getter+='\n'+block(mux,'struct typec_mux *fwnode_typec_mux_get(')+'\n'+block(mux,'void typec_mux_put(')+'\n' + getter+=r''' + static void graph_setup(void) { + graph_second_error=false;allocation_count=allocation_failure=0; + for(unsigned i=0;i<2;i++){modules[i]=(struct module){0};graph_providers[i]=(struct typec_mux_dev){.owner=&modules[i],.set=phy_set};} + } + static void graph_balanced(struct fwnode_handle *f) { + assert(!f->refs&&typec_mux_contexts.next==&typec_mux_contexts); + for(unsigned i=0;i<2;i++)assert(!modules[i].refs&&!graph_providers[i].dev.refs); + } + int main(void) { + struct fwnode_handle f={0};graph_setup(); + struct typec_mux *a=fwnode_typec_mux_get(&f),*b=fwnode_typec_mux_get(&f); + assert(!IS_ERR(a)&&!IS_ERR(b)&&a->context==b->context&&a->context->users==2&&f.refs==1); + for(unsigned i=0;i<2;i++)assert(modules[i].refs==2&&graph_providers[i].dev.refs==2); + typec_mux_put(a);typec_mux_put(b);graph_balanced(&f); + graph_setup();modules[1].going=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);graph_balanced(&f); + graph_setup();graph_second_error=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-EPROBE_DEFER);graph_balanced(&f); + graph_setup();allocation_failure=2;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENOMEM);graph_balanced(&f); + graph_setup();a=fwnode_typec_mux_get(&f);graph_providers[0].removed=true; + struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};unsigned before=phy_calls; + assert(typec_mux_set(a,&safe)==-ENODEV&&phy_calls==before);typec_mux_put(a);graph_balanced(&f); + puts("PASS actual getter/context/module/device balance: same fwnode, failed module conversion, deferred provider, context allocation and stale-provider handle"); + return 0; + } + ''' + return getter + +def controls(root): + clock = transaction.clock_order(root).replace('if(parked)return 0;if(!backend_clock)', 'if(!backend_clock)', 1) + clock = clock.replace('assert(!port.owner&&!port.safe_pending);', 'assert(!port.owner&&!port.safe_pending&&phy_calls==1&&role_calls==2);', 1) + return [ + ('same-handle-srcu-index', srcu_index(root), ((),), ('-pthread',)), + ('mux-release-stages', resources(root), (('normal',), ('late',), ('hardware',), ('fixed',)), ('-pthread',)), + ('single-safe-before-role-exit', clock, ((),), ('-pthread',)), + ('mux-get-put-balance', getter_unit(root), ((),), ('-pthread',)), + ('lookup-unregister-module-lifetime', lookup_lifetime(root), ((),), ('-pthread',)), + ('kernel-module-conversion-balance', module_conversion(root), ((),), ('-pthread',)), + ] + +def lookup_lifetime(ROOT): + getter = getter_unit(ROOT) + mux = (ROOT / "drivers/usb/typec/mux.c").read_text() + module_unit=getter.replace(block(getter,'int main('),block(getter,'int main(').replace('int main(','int __attribute__((unused)) prior_getter_main(',1),1) + module_unit=module_unit.replace('struct module { unsigned refs;bool going; };','struct module { unsigned refs,refcnt;bool going; };',1) + module_unit=module_unit.replace('struct device { unsigned refs; };','struct device { unsigned refs;bool unregistered; };',1) + sync=r''' + static pthread_mutex_t conversion_mutex=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t conversion_cond=PTHREAD_COND_INITIALIZER; + static bool lookup_ready,lookup_resume; + static unsigned active_readers; + static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);assert(active_readers==0);pthread_mutex_unlock(&conversion_mutex); } + static void device_unregister(struct device *d) { d->unregistered=true;assert(d->refs);d->refs--; } + static bool module_is_live(struct module *m) { return !m->going; } + static bool atomic_inc_not_zero(unsigned *p) { if(!*p)return false;(*p)++;return true; } + #define likely(x) (x) + #define trace_module_get(...) ((void)0) + #define _RET_IP_ 0 + ''' + pos=module_unit.index('static bool try_module_get(') + module_unit=module_unit[:pos]+sync+module_unit[pos:] + modulemain=(ROOT/'kernel/module/main.c').read_text() + module_unit=module_unit.replace(block(module_unit,'static bool try_module_get('),block(modulemain,'bool try_module_get('),1) + needle='return 2;\n }' + assert module_unit.count(needle)==1 + module_unit=module_unit.replace(needle,r''' + pthread_mutex_lock(&conversion_mutex);lookup_ready=true;pthread_cond_broadcast(&conversion_cond); + while(!lookup_resume)pthread_cond_wait(&conversion_cond,&conversion_mutex); + pthread_mutex_unlock(&conversion_mutex);return 2; + }''',1) + + declarations = r''' +static pthread_mutex_t conversion_mutex=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t conversion_cond=PTHREAD_COND_INITIALIZER; +static bool lookup_ready,lookup_resume,draining; +static unsigned active_readers; +#define WRITE_ONCE(x,v) ((x)=(v)) +''' + start = module_unit.index('static pthread_mutex_t conversion_mutex=') + end = module_unit.index('static void synchronize_srcu(', start) + module_unit = module_unit[:start] + module_unit[end:] + pos = module_unit.index('static int srcu_read_lock(') + module_unit = module_unit[:pos] + declarations + module_unit[pos:] + module_unit = module_unit.replace(block(module_unit, 'static int srcu_read_lock('), r''' +static int srcu_read_lock(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);active_readers++;pthread_mutex_unlock(&conversion_mutex);return 0; } +''', 1) + module_unit = module_unit.replace(block(module_unit, 'static void srcu_read_unlock('), r''' +static void srcu_read_unlock(int *s,int index) { (void)s;assert(index==0);pthread_mutex_lock(&conversion_mutex);assert(active_readers);active_readers--;pthread_cond_broadcast(&conversion_cond);pthread_mutex_unlock(&conversion_mutex); } +''', 1) + module_unit = module_unit.replace(block(module_unit, 'static void synchronize_srcu('), r''' +static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&conversion_mutex);draining=true;pthread_cond_broadcast(&conversion_cond);while(active_readers)pthread_cond_wait(&conversion_cond,&conversion_mutex);pthread_mutex_unlock(&conversion_mutex); } +''', 1) + module_unit += '\n' + block(mux, 'void typec_mux_unregister(') + r''' +static struct typec_mux *lookup_result; +static void *getter_actor(void *node) { lookup_result=fwnode_typec_mux_get(node);return NULL; } +static void *unregister_actor(void *provider) { typec_mux_unregister(provider);return NULL; } +int main(void) { + struct fwnode_handle f={0};graph_setup(); + struct module *departing=calloc(1,sizeof(*departing));departing->going=true;graph_providers[0].owner=departing;graph_providers[0].dev.refs=1; + pthread_t getter_thread,remove_thread;assert(!pthread_create(&getter_thread,NULL,getter_actor,&f)); + pthread_mutex_lock(&conversion_mutex);while(!lookup_ready)pthread_cond_wait(&conversion_cond,&conversion_mutex); + assert(active_readers==1&&graph_providers[0].dev.refs==2);pthread_mutex_unlock(&conversion_mutex); + assert(!pthread_create(&remove_thread,NULL,unregister_actor,&graph_providers[0])); + pthread_mutex_lock(&conversion_mutex);while(!draining)pthread_cond_wait(&conversion_cond,&conversion_mutex); + assert(active_readers==1&&!graph_providers[0].dev.unregistered); + lookup_resume=true;pthread_cond_broadcast(&conversion_cond);pthread_mutex_unlock(&conversion_mutex); + assert(!pthread_join(getter_thread,NULL));assert(!pthread_join(remove_thread,NULL)); + assert(PTR_ERR(lookup_result)==-ENODEV&&graph_providers[0].dev.unregistered&&graph_providers[0].dev.refs==0&&active_readers==0); + assert(graph_providers[1].dev.refs==0&&!f.refs); + free(departing); + puts("PASS lookup-to-module conversion holds a reader until removed-provider rejection and balanced cleanup"); + return 0; +} +''' + return module_unit + + +def module_conversion(ROOT): + unit = getter_unit(ROOT) + main = block(unit, 'int main(') + unit = unit.replace(main, main.replace('int main(', 'int __attribute__((unused)) prior_getter_main(', 1), 1) + unit = unit.replace('struct module { unsigned refs;bool going; };', 'struct module { unsigned refs,refcnt;bool going; };', 1) + source = (ROOT / 'kernel/module/main.c').read_text() + pos = unit.index('static bool try_module_get(') + leaves = r""" +static bool module_is_live(struct module *m) { return !m->going; } +static bool atomic_inc_not_zero(unsigned *p) { if(!*p)return false;(*p)++;return true; } +static int atomic_dec_if_positive(unsigned *p) { if(!*p)return -1;return --(*p); } +#define likely(x) (x) +#define trace_module_get(...) ((void)0) +#define trace_module_put(...) ((void)0) +#define preempt_disable() ((void)0) +#define preempt_enable() ((void)0) +#define WARN_ON(v) assert(!(v)) +#define _RET_IP_ 0 +""" + unit = unit[:pos] + leaves + unit[pos:] + unit = unit.replace(block(unit, 'static bool try_module_get('), block(source, 'bool try_module_get('), 1) + unit = unit.replace(block(unit, 'static void module_put('), block(source, 'void module_put('), 1) + old = block(unit, 'static int srcu_read_lock(') + '\n' + block(unit, 'static void srcu_read_unlock(') + unit = unit.replace(old, r""" +static unsigned conversion_readers; +static int srcu_read_lock(int *s) { (void)s;conversion_readers++;return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;assert(i==0&&conversion_readers);conversion_readers--; } +""", 1) + unit += r""" +static void live_graph(void) { graph_setup();for(unsigned i=0;i<2;i++)modules[i].refcnt=1; } +static void module_balance(struct fwnode_handle *f) { graph_balanced(f);assert(!conversion_readers);for(unsigned i=0;i<2;i++)assert(modules[i].refcnt==1); } +int main(void) { + struct fwnode_handle f={0};live_graph(); + struct typec_mux *a=fwnode_typec_mux_get(&f),*b=fwnode_typec_mux_get(&f); + assert(!IS_ERR(a)&&!IS_ERR(b)&&a->context==b->context&&!conversion_readers); + for(unsigned i=0;i<2;i++)assert(modules[i].refcnt==3&&graph_providers[i].dev.refs==2); + typec_mux_put(a);typec_mux_put(b);module_balance(&f); + live_graph();modules[1].going=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);module_balance(&f); + live_graph();allocation_failure=2;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENOMEM);module_balance(&f); + live_graph();graph_second_error=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-EPROBE_DEFER);module_balance(&f); + live_graph();graph_providers[0].removed=true;a=fwnode_typec_mux_get(&f);assert(PTR_ERR(a)==-ENODEV);module_balance(&f); + puts("PASS actual kernel module get/put and mux get/put: shared handles and every failed conversion unwind"); + return 0; +} +""" + return unit diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py index 3c2913281ba5e5..6125dbf312b8f8 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/pm.py @@ -16,9 +16,22 @@ def ownership(root, usb_unreachable=False): prefix=ns['prefix']+'\nstruct device_node;\nstatic int atcphy_mux_set(struct typec_mux_dev *, struct typec_mux_state *);\n' prefix=prefix.replace('struct typec_mux_dev { struct apple_atcphy *data; };', '''struct typec_mux_dev { struct apple_atcphy *data; - int (*set)(struct typec_mux_dev *, struct typec_mux_state *); }; - struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; }; + int (*set)(struct typec_mux_dev *, struct typec_mux_state *); + int (*prepare)(struct typec_mux_dev *, struct typec_mux_state *); + int (*finish)(struct typec_mux_dev *, struct typec_mux_state *); + void (*commit)(struct typec_mux_dev *, struct typec_mux_state *); + void (*abort)(struct typec_mux_dev *, struct typec_mux_state *);bool removed; }; + struct typec_mux_context { struct { bool held; } lock; }; + struct task_struct { int unused; };static struct task_struct test_task; + #define current (&test_task) + static int typec_mux_srcu; + static int srcu_read_lock(int *s) { (void)s;return 0; } + static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } + #define mutex_trylock(m) ((m)->held ? 0 : ((m)->held=true)) + #define mutex_unlock(m) ((m)->held=false) + struct typec_mux { unsigned int num_mux_devs; struct typec_mux_dev **mux_devs; struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; }; #define IS_ERR_OR_NULL(p) (!(p))''') + prefix=prefix.replace('#define WARN_ON_ONCE(v) assert(!(v))', 'static bool checked_warn(bool v) { assert(!v);return v; }\n#define WARN_ON_ONCE(v) checked_warn(v)') prefix=prefix.replace('#define guard(kind) (void)',r''' struct lock_guard { int *lock; }; static struct lock_guard lock_take(int *lock) { assert(!*lock); *lock=1; return (struct lock_guard){lock}; } @@ -177,7 +190,7 @@ def ownership(root, usb_unreachable=False): #if !USB_REACHABLE { prepare(&p,true,true,false);unsigned int before=writes; struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; - struct typec_mux_dev *providers[]={&provider};struct typec_mux mux={1,providers}; + struct typec_mux_dev *providers[]={&provider};struct typec_mux_context typec_test_context={0};struct typec_mux mux={.num_mux_devs=1,.mux_devs=providers,.context=&typec_test_context}; struct typec_mux_state state={.mode=TYPEC_STATE_SAFE}; assert(typec_mux_set(&mux,&state)==-ENODEV&&writes==before);balanced(&p); puts("PASS USB-unreachable fixed route blocks before MMIO");return 0; } @@ -298,7 +311,8 @@ def ownership(root, usb_unreachable=False): if usb_unreachable: pmfixture=pmfixture.replace('static ', 'static __attribute__((unused)) ') typec_source=(root/'drivers/usb/typec/mux.c').read_text() - typec_caller=ns['block'](typec_source,'int typec_mux_set(')+'\n' + typec_caller='\n'.join(ns['block'](typec_source, marker) for marker in + ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_set('))+'\n' hcd_source=(root/'drivers/usb/core/hcd.c').read_text() hcd_death=ns['block'](hcd_source,'void usb_hc_died (')+'\n' text=prefix+prod+'\n#define REQUIRES_PLATFORM_LOCKS '+str(int('device_lock(&pm->controller->dev)' in s))+'\n#define USB_REACHABLE '+str(int(not usb_unreachable))+'\n#define HCD_DEAD_ONLY '+str(int(False))+'\n'+pmfixture+helpers+typec_caller+hcd_death+suffix+controls @@ -531,7 +545,7 @@ def probe_dispatch(root): struct cd321x c={}; struct typec_mux_dev provider={.data=&p,.set=atcphy_mux_set}; struct typec_mux_dev *providers[]={&provider}; - struct typec_mux route={1,providers};probe_port.mux=&route; + struct typec_mux_context typec_test_context={0};struct typec_mux route={.num_mux_devs=1,.mux_devs=providers,.context=&typec_test_context};probe_port.mux=&route; prepare(&p,true,true,false);root_missing=true;unsigned int before=writes; assert(cd321x_register_port(&c.tps,&connector)==-EPROBE_DEFER); assert(!c.state_valid&&!resumes&&writes==before);balanced(&p);port_balance(); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py new file mode 100644 index 00000000000000..6e7b9f69b2a254 --- /dev/null +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/quiesce.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright 2026 Ryan Murray +"""Actual mux prepare/set and TPS removal ordering; firmware/PHY leaves are mocks.""" +from common import block + +def make_unit(source, parent=False, reference=None): + reference=source if reference is None else reference + core=(source/'drivers/usb/typec/tipd/core.c').read_text();mux=(source/'drivers/usb/typec/mux.c').read_text() + dcp=(reference/'drivers/gpu/drm/apple/dcp.c').read_text() + prefix=r''' +#include +#include +#include +#include +#include +#include +typedef unsigned u32; +#define IS_ERR(p) false +#define IS_ERR_OR_NULL(p) (!(p)) +#define EXPORT_SYMBOL_GPL(x) +#define guard(k) (void) +#define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0) +#define READ_ONCE(x) (x) +#define WARN_ON_ONCE(x) (x) +#define container_of(p,t,m) ((t *)((char *)(p)-offsetof(t,m))) +#define to_platform_device(p) (p) +#define typec_mux_get_drvdata(m) ((m)->data) +#define dev_warn(...) ((void)0) +#define dev_info(...) ((void)0) +#define dev_name(p) "device" +#define list_for_each_entry(p,h,m) for((void)(h),(p)=sole_port();(p);(p)=NULL) +#define lockdep_assert_held(p) ((void)(p)) +#define dev_err(dev,fmt,ret) do { (void)(dev);last_error=(ret); } while(0) +enum {TYPEC_STATE_SAFE,TYPEC_STATE_USB,TYPEC_ORIENTATION_NONE,TYPEC_PWR_MODE_USB,TYPEC_THUNDERBOLT_SWITCH_OFF}; +enum usb_role {USB_ROLE_NONE,USB_ROLE_HOST,USB_ROLE_DEVICE}; +enum typec_data_role {TYPEC_HOST,TYPEC_DEVICE}; +#define TPS_STATUS_TO_TYPEC_PORTROLE(s) (s) +#define TPS_STATUS_TO_TYPEC_VCONN(s) (s) +#define TPS_STATUS_TO_TYPEC_DATAROLE(s) (s) +struct typec_mux_state { void *alt; unsigned long mode; void *data; }; +struct typec_mux_dev { bool removed; void (*commit)(struct typec_mux_dev *,struct typec_mux_state *); void (*abort)(struct typec_mux_dev *,struct typec_mux_state *); int (*finish)(struct typec_mux_dev *,struct typec_mux_state *); void *data; int (*set)(struct typec_mux_dev *,struct typec_mux_state *); int (*prepare)(struct typec_mux_dev *,struct typec_mux_state *); }; +struct task_struct { int unused; }; +static _Thread_local struct task_struct this_task; +#define current (&this_task) +struct mutex { pthread_mutex_t value; }; +struct typec_mux_context { struct mutex lock; }; +static struct typec_mux_context shared_context={.lock={.value=PTHREAD_MUTEX_INITIALIZER}}; +static int typec_mux_srcu; +static int srcu_read_lock(int *s) { (void)s;return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;(void)i; } +static int mutex_trylock(struct mutex *m) { return pthread_mutex_trylock(&m->value)==0; } +static void mutex_unlock(struct mutex *m) { assert(!pthread_mutex_unlock(&m->value)); } +struct typec_mux { struct typec_mux_context *context; struct typec_mux_state transaction; struct task_struct *owner; int srcu_index; unsigned prepared; bool applied; unsigned num_mux_devs; struct typec_mux_dev *mux_devs[3]; }; +struct connector { bool connected; }; +struct apple_dcp { int external_lock,external_error; bool external_cable,external_stopped; struct { bool connected; } dptxport[1]; void *hdmi_hpd; bool active; bool external,typec_cable_connected; void *dev; struct connector *typec_connector; }; +struct apple_dcp_typec_route; +struct apple_dcp_typec_port { int link; void *connector_np; struct apple_dcp_typec_route *owner,*target,*secondary_target,*preferred_route; void *applied_alt; unsigned long applied_mode; unsigned applied_status,applied_conf; bool applied_valid,safe_pending,safe_finish_ready,dp_wanted,dp_hpd,hpd; }; +struct apple_dcp_typec_route { struct apple_dcp_typec_port *port; struct apple_dcp *dcp; void *xbar; bool deactivate_xbar_done; int deactivate_error; bool selected,tunnel; }; +static int dcp_typec_fabric_lock,dcp_typec_ports,deactivate_calls; +struct drm_device { int unused; }; +static struct apple_dcp display; +static struct connector connector; +static struct apple_dcp_typec_port port; +static struct apple_dcp_typec_route route; +static struct apple_dcp_typec_port *sole_port(void) { return &port; } +static char events[100];static unsigned count; +static int power_error,phy_error,tbt_error,role_error,last_error,phy_calls,reset_calls,gpio_calls,role_calls,stops,unregisters; +static void event(char c) { assert(count+1external; } +static int dcp_dptx_disconnect_oob(void *dev,u32 index) { (void)dev;assert(index==0);event('Q');stops++;if(power_error){display.external_error=power_error;return power_error;}display.external_cable=false;display.external_stopped=true;display.dptxport[0].connected=false;display.typec_cable_connected=false;connector.connected=false;return 0; } +static struct drm_device *dcp_typec_drm(void) { return NULL; } +static void dcp_typec_plan(struct drm_device *d,struct apple_dcp_typec_port *p,unsigned n) { (void)d;(void)p;(void)n; } +static bool dcp_typec_tunnel_held(struct apple_dcp *d) { (void)d;return false; } +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *r) { deactivate_calls++;r->selected=false;return 0; } +static void dcp_typec_port_attach(struct apple_dcp_typec_port *p) { (void)p; } +static int gpiod_get_value_cansleep(void *p) { (void)p;return 0; } +static int dcp_dptx_connect(struct apple_dcp *d,unsigned n) { (void)d;(void)n;return 0; } +static bool dcp_typec_route_available(struct apple_dcp_typec_route *r) { return !r->selected; } +static int dcp_typec_route_activate(struct apple_dcp_typec_route *r,void *x) { (void)x;r->selected=true;return 0; } +struct tps6598x; +struct tipd_data { REMOVE_RETURN (*remove)(struct tps6598x *); void (*unregister_port)(struct tps6598x *); int (*reset)(struct tps6598x *); }; +struct typec_port { struct typec_mux *mux; }; +struct tps6598x { bool remove_safe_prepared; struct typec_mux *mux; void *partner,*dev,*psy,*role_sw,*reset; int irq,wq_poll; struct typec_port *port; struct tipd_data *data; }; +struct cd321x { struct tps6598x tps; struct typec_mux *mux; void *tbt_switch; }; +struct typec_thunderbolt_switch_data { int state; }; +static void cd321x_stop_work(struct cd321x *c) { (void)c;event('W'); } +static int typec_thunderbolt_switch_set(void *p,void *s) { (void)p;(void)s;event('B');return tbt_error; } +static int usb_role_switch_set_role(void *p,int r) { (void)p;assert(r==USB_ROLE_NONE);event('R');role_calls++;return role_error; } +static void typec_unregister_partner(void *p) { (void)p;event('P'); } +#define typec_set_pwr_opmode(...) ((void)0) +#define typec_set_pwr_role(...) ((void)0) +#define typec_set_vconn_role(...) ((void)0) +static int typec_set_orientation(void *p,int o) { (void)p;assert(o==TYPEC_ORIENTATION_NONE);return 0; } +static void typec_set_data_role(struct typec_port *p,enum typec_data_role r) { (void)p;(void)r;event('D'); } +static void dcp_typec_pipeline_freed(void) { event('F'); } +static void power_supply_changed(void *p) { (void)p; } +static void tipd_debugfs_unregister(struct tps6598x *t) { (void)t; } +static void cancel_delayed_work_sync(void *w) { (void)w; } +static void devm_free_irq(void *d,int i,void *t) { (void)d;(void)i;(void)t; } +static void typec_mux_put(struct typec_mux *m) { (void)m; } +static void usb_role_switch_put(void *p) { (void)p;event('V'); } +static void gpiod_set_value_cansleep(void *p,int v) { (void)p;assert(v==1);gpio_calls++;event('G'); } + +'''.replace('REMOVE_RETURN','void' if parent else 'int') + functions='\n'.join(block(mux, marker) for marker in ('static void typec_mux_unlock(', 'void typec_mux_abort(', 'int typec_mux_begin(', 'int typec_mux_apply(', 'int typec_mux_end(', 'int typec_mux_prepare(')) if not parent else block((reference/'drivers/usb/typec/mux.c').read_text(),'int typec_mux_prepare(')+'\n' + functions+=block(mux,'int typec_mux_set(')+'\n'+block(dcp,'static int dcp_typec_route_prepare(')+'\n'+block(dcp,'static int dcp_typec_route_finish(')+'\n'+block(dcp,'static void dcp_typec_route_commit(')+'\n' + functions+=block(dcp,'static struct apple_dcp_typec_route *\ndcp_typec_rebalance_locked(')+'\n' + if not parent:functions+=block(core,'static void tps6598x_drop_partner(')+'\n'+block(core,'static int tps6598x_set_data_role(')+'\n' + functions+=block(core,'static '+('void' if parent else 'int')+' tps6598x_disconnect(')+'\n'+block(core,'static '+('void' if parent else 'int')+' cd321x_remove(')+'\n'+block(core,'void tipd_remove(')+'\n' + controls=r''' +static struct typec_mux_dev phy_provider,display_provider; +static struct typec_mux mux; +static struct typec_port class_port; +static struct tipd_data data; + +static int phy_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;assert(s->mode==TYPEC_STATE_SAFE);event('L');phy_calls++;return phy_error; } +static int display_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;event('S');if(port.owner&&display.external&&connector.connected) {int r=dcp_dptx_disconnect_oob(display.dev,0);if(r)return r;}if(port.safe_pending)return 0;port.owner=NULL;return 0; } +static void unregister_port(struct tps6598x *t) { (void)t;unregisters++;event('U'); } +static int reset(struct tps6598x *t) { (void)t;reset_calls++;event('X');return 0; } +static void initial(struct cd321x *c) { + count=0;events[0]=0;power_error=phy_error=tbt_error=role_error=last_error=phy_calls=reset_calls=gpio_calls=role_calls=stops=unregisters=deactivate_calls=0; + connector=(struct connector){.connected=true};display=(struct apple_dcp){.external=true,.external_cable=true,.dptxport={{.connected=true}},.typec_cable_connected=true,.typec_connector=&connector}; + port=(struct apple_dcp_typec_port){.owner=&route,.applied_valid=true,.dp_wanted=true,.dp_hpd=true,.hpd=true};route=(struct apple_dcp_typec_route){.port=&port,.dcp=&display,.selected=true}; + phy_provider=(struct typec_mux_dev){.set=phy_set};display_provider=(struct typec_mux_dev){.data=&route,.set=display_set,.prepare=dcp_typec_route_prepare,.finish=dcp_typec_route_finish,.commit=dcp_typec_route_commit}; + mux=(struct typec_mux){.context=&shared_context,.num_mux_devs=2,.mux_devs={&phy_provider,&display_provider}};class_port.mux=&mux; + data=(struct tipd_data){.remove=cd321x_remove,.unregister_port=unregister_port,.reset=reset}; + *c=(struct cd321x){.tps={.mux=&mux,.partner=(void *)1,.port=&class_port,.data=&data,.reset=(void *)1},.mux=&mux}; +} +static int position(char c) { for(unsigned i=0;i=0&&position('Q')mode==TYPEC_STATE_SAFE);event('L');", "(void)m;(void)s;event('L');",1) + actual=actual.replace('connector=(struct connector){.connected=true};','connector=(struct connector){.connected=true};',1) + actual=actual.replace('phy_provider=(struct typec_mux_dev){.set=phy_set};','display.typec_routes=&route;display.fixed_connector_type=DRM_MODE_CONNECTOR_USB;display.active_typec_route=&route;\n phy_provider=(struct typec_mux_dev){.set=phy_set};',1) + controls=r''' +static int late_finish_error(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;return -EIO; } + int main(int argc,char **argv) { + (void)argc;struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + if(argv[1][0]=='o') { + assert(typec_mux_set(&mux,&safe)==0); + assert(stops==1&&!port.owner&&!port.safe_pending&&!route.selected); + puts("PASS actual DCP prepare + route_set + route_deactivate + framework normal SAFE"); + } + if(argv[1][0]=='r') { + mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;phy_error=-ETIMEDOUT; + assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT); + fprintf(stderr,"display-first failed SAFE: owner=%p lease=%d selected=%d events=%s\n",(void *)port.owner,port.safe_pending,route.selected,events); + assert(port.owner==&route&&port.safe_pending&&route.selected); + } + if(argv[1][0]=='d') { + phy_error=-ETIMEDOUT;assert(typec_mux_set(&mux,&safe)==-ETIMEDOUT&&port.safe_pending&&port.owner==&route); + phy_error=0;struct typec_altmode alt={.svid=USB_TYPEC_DP_SID}; + struct typec_displayport_data data={.status=DP_STATUS_HPD_STATE}; + struct typec_mux_state dp={.alt=&alt,.mode=TYPEC_DP_STATE_A,.data=&data}; + int ret=typec_mux_set(&mux,&dp); + fprintf(stderr,"DP after failed SAFE: ret=%d connects=%d pending=%d wanted=%d hpd=%d\n",ret,connect_calls,port.safe_pending,port.dp_wanted,port.hpd); + assert(ret<0&&connect_calls==0&&!port.dp_wanted); + } + if(argv[1][0]=='a') { + struct altmode alt={.mux=&mux,.adev={.svid=USB_TYPEC_DP_SID}};phy_error=-ETIMEDOUT; + int ret=typec_altmode_set_mux(&alt,TYPEC_STATE_SAFE,NULL); + fprintf(stderr,"altmode SAFE: ret=%d stops=%d phy_calls=%d lease=%d\n",ret,stops,phy_calls,port.safe_pending); + assert(ret==-ETIMEDOUT&&stops==1&&port.safe_pending); + } + if(argv[1][0]=='x') { + xbar_error=-EIO;assert(typec_mux_set(&mux,&safe)==-EIO); + assert(port.owner==&route&&port.safe_pending&&!port.safe_finish_ready&&route.selected&&route.deactivate_xbar_done&&route.deactivate_error==-EIO); + xbar_error=0;assert(typec_mux_set(&mux,&safe)==-EIO&&port.owner==&route&&port.safe_pending&&xbar_calls==1); + } + if(argv[1][0]=='f') { + mux.mux_devs[0]=&display_provider;mux.mux_devs[1]=&phy_provider;phy_provider.finish=late_finish_error; + assert(typec_mux_set(&mux,&safe)==-EIO); + assert(port.owner==&route&&port.safe_pending&&port.safe_finish_ready&&!route.selected&&xbar_calls==1); + assert(!dcp_typec_route_available(&route)); + struct apple_dcp peer={0};struct apple_dcp_typec_route peer_route={.dcp=&peer}; + assert(dcp_typec_route_available(&peer_route)); + phy_provider.finish=NULL; + assert(typec_mux_set(&mux,&safe)==0&&!port.owner&&!port.safe_pending&&xbar_calls==1); + } + return 0; + } + ''' + bus=(ROOT/'drivers/usb/typec/bus.c').read_text() + actual+='\nstruct altmode { struct typec_mux *mux; struct typec_altmode adev; };\n'+block(bus,'static inline int\ntypec_altmode_set_mux(')+'\n'+controls + check('actual-dcp-normal-safe',actual,(('ordered',),)) + check('actual-dcp-finalizer-recovery',actual,(('xbar-error',),('finalizer',))) + check('provider-order-releases-failed-lease',actual,(('release',),)) + check('failed-safe-allows-new-dp-admission',actual,(('dp',),)) + check('altmode-safe-skips-quiescence',actual,(('altmode',),)) + + # Real pthread fabric/consumer mutexes. Pause the SAFE actor after its actual + # DCP prepare has returned and released its locks, before any set callback. + # A second handle can reference the same providers (class/altmode/CD handles + # are separately acquired in the real framework). + concurrent=actual.replace(block(actual,'int main('),block(actual,'int main(').replace('int main(','int __attribute__((unused)) route_controls_main(',1),1) + concurrent=concurrent.replace('#define guard(k) (void)','') + concurrent=concurrent.replace('#define scoped_guard(k,l) for(int once=((void)(l),1);once;once=0)','') + locks=r''' + #include + static pthread_mutex_t fabric_mutex=PTHREAD_MUTEX_INITIALIZER,external_mutex=PTHREAD_MUTEX_INITIALIZER,tb_mutex=PTHREAD_MUTEX_INITIALIZER; + struct real_guard { pthread_mutex_t *mutex; bool held; }; + static struct real_guard take_lock(int *key) { + pthread_mutex_t *m=key==&dcp_typec_fabric_lock?&fabric_mutex:key==&display.external_lock?&external_mutex:&tb_mutex; + assert(pthread_mutex_lock(m)==0);return (struct real_guard){m,true}; + } + static void drop_lock(struct real_guard *g) { if(g->held){g->held=false;assert(pthread_mutex_unlock(g->mutex)==0);} } + #define guard(k) struct real_guard __attribute__((cleanup(drop_lock))) _guard = take_lock + #define scoped_guard(k,l) for(struct real_guard __attribute__((cleanup(drop_lock))) _scope=take_lock(l);_scope.held;drop_lock(&_scope)) + static bool observed_unquiesced_safe; + ''' + pos=concurrent.index('static int dcp_typec_route_deactivate(') + concurrent=concurrent[:pos]+locks+concurrent[pos:] + concurrent=concurrent.replace("(void)m;(void)s;event('L');phy_calls++;", "(void)m;if(s->mode==TYPEC_STATE_SAFE&&display.dptxport[0].connected)observed_unquiesced_safe=true;event('L');phy_calls++;",1) + concurrent+=r''' + static pthread_mutex_t schedule_lock=PTHREAD_MUTEX_INITIALIZER; + static pthread_cond_t schedule_cond=PTHREAD_COND_INITIALIZER; + static bool prepared,continue_safe; + static int safe_ret,dp_ret; + static int scheduled_prepare(struct typec_mux_dev *m,struct typec_mux_state *s) { + int ret=dcp_typec_route_prepare(m,s); + if(s->mode==TYPEC_STATE_SAFE&&!s->alt) { + pthread_mutex_lock(&schedule_lock);prepared=true;pthread_cond_broadcast(&schedule_cond); + while(!continue_safe)pthread_cond_wait(&schedule_cond,&schedule_lock); + pthread_mutex_unlock(&schedule_lock); + } + return ret; + } + static void *safe_actor(void *unused) { + (void)unused;struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE};safe_ret=typec_mux_set(&mux,&safe);return NULL; + } + int main(void) { + struct cd321x c;initial(&c);display_provider.prepare=scheduled_prepare; + pthread_t thread;assert(pthread_create(&thread,NULL,safe_actor,NULL)==0); + pthread_mutex_lock(&schedule_lock);while(!prepared)pthread_cond_wait(&schedule_cond,&schedule_lock);pthread_mutex_unlock(&schedule_lock); + struct typec_mux second_handle=mux; + struct typec_altmode alt={.svid=USB_TYPEC_DP_SID};struct typec_displayport_data data={.status=DP_STATUS_HPD_STATE}; + struct typec_mux_state dp={.alt=&alt,.mode=TYPEC_DP_STATE_A,.data=&data}; + dp_ret=typec_mux_set(&second_handle,&dp); + pthread_mutex_lock(&schedule_lock);continue_safe=true;pthread_cond_broadcast(&schedule_cond);pthread_mutex_unlock(&schedule_lock); + assert(pthread_join(thread,NULL)==0); + fprintf(stderr,"overlapping handles: dp_ret=%d safe_ret=%d connected_at_SAFE_PHY=%d events=%s\n",dp_ret,safe_ret,observed_unquiesced_safe,events); + assert(!observed_unquiesced_safe); + } + ''' + check('concurrent-prepare-set-stale-quiescence',concurrent,flags=('-pthread',)) + check('connector-context-lifetime', lifetime(ROOT), flags=('-pthread',)) + check('probe-transaction-rollback', probe(ROOT)) + check('provider-unregister-drain', unregister(ROOT), flags=('-pthread',)) + check('park-before-host-role-exit', clock_order(ROOT)) + return tests + + +def lifetime(root): + mux = (root / 'drivers/usb/typec/mux.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), + block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct typec_mux_context { struct mutex lock; };', r""" +struct list_head { struct list_head *next,*prev; }; +struct fwnode_handle { unsigned refs; }; +struct typec_mux_context { struct list_head list; struct fwnode_handle *fwnode; struct mutex lock; unsigned users; }; +""") + helpers = r""" +#include +#include +#undef guard +#undef list_for_each_entry +static struct mutex typec_mux_contexts_lock={.value=PTHREAD_MUTEX_INITIALIZER}; +static struct list_head typec_mux_contexts={&typec_mux_contexts,&typec_mux_contexts}; +static struct fwnode_handle *fwnode_handle_get(struct fwnode_handle *f) { f->refs++;return f; } +static void fwnode_handle_put(struct fwnode_handle *f) { assert(f->refs);f->refs--; } +static void mutex_init(struct mutex *m) { assert(!pthread_mutex_init(&m->value,NULL)); } +static struct mutex *registry_lock(struct mutex *m) { assert(!pthread_mutex_lock(&m->value));return m; } +static void registry_unlock(struct mutex **m) { mutex_unlock(*m); } +#define guard(k) struct mutex * __attribute__((cleanup(registry_unlock))) held=registry_lock +#define kzalloc_obj(x) calloc(1,sizeof(x)) +#define kfree(x) free(x) +#define ERR_PTR(e) ((void *)(intptr_t)(e)) +#define list_for_each_entry(p,h,m) for(p=container_of((h)->next,__typeof__(*p),m); &(p)->m!=(h);p=container_of((p)->m.next,__typeof__(*p),m)) +static void list_add_tail(struct list_head *n,struct list_head *h) { n->prev=h->prev;n->next=h;h->prev->next=n;h->prev=n; } +static void list_del(struct list_head *n) { n->prev->next=n->next;n->next->prev=n->prev; } +""" + unit += helpers + block(mux, 'static struct typec_mux_context *typec_mux_context_get(') + '\n' + unit += block(mux, 'static void typec_mux_context_put(') + '\n' + unit += r""" +static struct typec_mux downstream; +static int forwarded,commits,aborts,finish_error; +static int forward_set(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;forwarded++;return typec_mux_set(&downstream,s); } +static int finish(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;return finish_error; } +static void commit(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;commits++; } +static void abort_callback(struct typec_mux_dev *m,struct typec_mux_state *s) { (void)m;(void)s;aborts++; } +int main(void) { + struct fwnode_handle a={0},b={0}; + struct typec_mux_context *one=typec_mux_context_get(&a),*two=typec_mux_context_get(&a),*other=typec_mux_context_get(&b); + assert(one==two&&one!=other&&one->users==2&&a.refs==1&&b.refs==1); + struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + struct typec_mux second=mux; mux.context=one;second.context=two; + assert(typec_mux_begin(&mux,&safe)==0); + unsigned before=phy_calls;assert(typec_mux_set(&second,&safe)==-EBUSY&&phy_calls==before); + typec_mux_abort(&mux);assert(port.safe_pending&&port.owner==&route); + phy_error=-ETIMEDOUT;assert(typec_mux_set(&second,&safe)==-ETIMEDOUT&&port.owner==&route); + phy_error=0;assert(typec_mux_set(&second,&safe)==0&&!port.owner&&!port.safe_pending); + struct typec_mux_dev leaf={.set=phy_set,.finish=finish,.commit=commit,.abort=abort_callback}; + downstream=(struct typec_mux){.context=other,.num_mux_devs=1,.mux_devs={&leaf}}; + struct typec_mux_dev forward={.set=forward_set}; + mux=(struct typec_mux){.context=one,.num_mux_devs=1,.mux_devs={&forward}}; + assert(typec_mux_set(&mux,&safe)==0&&forwarded==1&&commits==1); + finish_error=-EIO;assert(typec_mux_set(&downstream,&safe)==-EIO&&commits==1&&aborts==1); + finish_error=0;assert(typec_mux_set(&downstream,&safe)==0&&commits==2); + downstream.context=one;before=phy_calls; + assert(typec_mux_set(&mux,&safe)==-EBUSY&&phy_calls==before); // cyclic same-connector forwarding + leaf.removed=true;downstream.context=other;before=phy_calls; + assert(typec_mux_set(&downstream,&safe)==-ENODEV&&phy_calls==before&&aborts==1); + typec_mux_context_put(one);assert(a.refs==1&&two->users==1); + typec_mux_context_put(two);assert(a.refs==0); + typec_mux_context_put(other);assert(b.refs==0&&typec_mux_contexts.next==&typec_mux_contexts); + puts("PASS actual connector registry, shared handles, forwarding, finalize/abort and removed-provider guards"); +} +""" + return unit + + +def probe(root): + core = (root / 'drivers/usb/typec/tipd/core.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct tipd_data {', 'struct fwnode_handle;\nstruct tipd_data { unsigned irq_mask1; int (*init)(struct tps6598x *); int (*switch_power_state)(struct tps6598x *, int); int (*register_port)(struct tps6598x *, struct fwnode_handle *); bool (*read_data_status)(struct tps6598x *); int (*connect)(struct tps6598x *,u32); void (*ready)(struct tps6598x *); void *irq_handler;') + unit = unit.replace('struct tps6598x {', 'struct tps6598x { bool dbma_at_probe,wakeup;') + helpers = r""" +#undef dev_err +#define dev_err(...) ((void)0) +#define TPS_SETUP_MS 1000 +#define TPS_SYSTEM_POWER_STATE_S0 0 +#define TPS_MODE_PTCH 2 +#define TPS_MODE_DBMA 3 +#define TPS_REG_VID 0 +#define TPS_REG_INT_MASK1 1 +#define TPS_STATUS_PLUG_PRESENT 1 +#define GPIOD_OUT_LOW 0 +#include +#undef IS_ERR +#define IS_ERR(p) ((uintptr_t)(p)>=(uintptr_t)-4095) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define IRQF_SHARED 0 +#define IRQF_ONESHOT 0 +#define POLL_INTERVAL 1000 +#define INIT_DELAYED_WORK(...) ((void)0) +#define tps6598x_poll_work 0 +#define devm_device_init_wakeup(...) ((void)0) +#define enable_irq_wake(...) ((void)0) +#define dev_err_probe(...) (-EINVAL) +struct fwnode_handle { int unused; }; +static int connect_error,irq_error,register_error,node_puts,mux_get_error; +static void *devm_gpiod_get_optional(void *d,const char *s,int v) { (void)d;(void)s;(void)v;return NULL; } +static void msleep(unsigned ms) { (void)ms; } +static bool device_is_compatible(void *d,const char *s) { (void)d;(void)s;return false; } +static int tps6598x_read32(struct tps6598x *t,int reg,u32 *v) { (void)t;(void)reg;*v=1;return 0; } +static int tps6598x_check_mode(struct tps6598x *t) { (void)t;return 0; } +static int tps6598x_write64(struct tps6598x *t,int reg,unsigned v) { (void)t;(void)reg;(void)v;return 0; } +static bool tps6598x_read_status(struct tps6598x *t,u32 *s) { (void)t;*s=TPS_STATUS_PLUG_PRESENT;return true; } +static bool tps6598x_read_power_status(struct tps6598x *t) { (void)t;return true; } +static struct fwnode_handle node; +static struct fwnode_handle *device_get_named_child_node(void *d,const char *s) { (void)d;(void)s;return &node; } +static void fw_devlink_purge_absent_suppliers(struct fwnode_handle *f) { (void)f; } +static void *fwnode_usb_role_switch_get(struct fwnode_handle *f) { (void)f;return (void *)1; } +static struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *f) { (void)f;return mux_get_error?(struct typec_mux *)(intptr_t)mux_get_error:&mux; } +static void fwnode_handle_put(struct fwnode_handle *f) { (void)f;node_puts++; } +static int devm_tps6598_psy_register(struct tps6598x *t) { (void)t;return 0; } +static int devm_request_threaded_irq(void *d,int irq,void *h,void *th,int flags,const char *name,void *arg) { (void)d;(void)irq;(void)h;(void)th;(void)flags;(void)name;(void)arg;return irq_error; } +static int system_power_efficient_wq; +static unsigned msecs_to_jiffies(unsigned ms) { return ms; } +static void queue_delayed_work(int wq,int *work,unsigned delay) { (void)wq;(void)work;(void)delay; } +static bool device_property_read_bool(void *d,const char *p) { (void)d;(void)p;return false; } +static void tipd_debugfs_register(struct tps6598x *t) { (void)t; } +static int register_port(struct tps6598x *t,struct fwnode_handle *f) { (void)t;(void)f;return register_error; } +static bool read_data_status(struct tps6598x *t) { (void)t;return true; } +static int connect(struct tps6598x *t,u32 status) { (void)t;(void)status;return connect_error; } +""" + unit += helpers + block(core, 'int tipd_init(') + r""" +static void probe_initial(struct cd321x *c) { + initial(c);c->tps.irq=1;data.register_port=register_port;data.read_data_status=read_data_status;data.connect=connect; + connect_error=irq_error=register_error=node_puts=mux_get_error=0; +} +int main(void) { + struct cd321x c; + probe_initial(&c);irq_error=-ENOSPC;phy_error=-ETIMEDOUT; + assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==0&&gpio_calls==0&&port.owner==&route&&port.safe_pending&&unregisters==1&&node_puts==1); + probe_initial(&c);connect_error=-EAGAIN;role_error=-EBUSY; + assert(tipd_init(&c.tps)==-EAGAIN&&reset_calls==0&&gpio_calls==0&&phy_calls==1&&port.owner==&route&&port.safe_pending); + probe_initial(&c);irq_error=-ENOSPC; + assert(tipd_init(&c.tps)==-ENOSPC&&reset_calls==1&&!port.owner&&!port.safe_pending&&unregisters==1); + probe_initial(&c);mux_get_error=-ENODEV; + assert(tipd_init(&c.tps)==-ENODEV&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0&&node_puts==1); + probe_initial(&c);register_error=-EIO; + assert(tipd_init(&c.tps)==-EIO&&reset_calls==0&&stops==0&&phy_calls==0&&unregisters==0); + puts("PASS actual probe rollback preserves primary errors and blocks reset after SAFE/role/registration errors"); +} +""" + return unit + + +def unregister(root): + """Actual unregister/begin/end with sleeping SRCU leaves and two threads.""" + source = (root / 'drivers/usb/typec/mux.c').read_text() + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('struct typec_mux_dev {', 'struct device { bool unregistered; };\nstruct typec_mux_dev { struct device dev;') + old = block(unit, 'static int srcu_read_lock(') + '\n' + block(unit, 'static void srcu_read_unlock(') + leaves = r""" +static pthread_mutex_t lifetime_lock=PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t lifetime_cond=PTHREAD_COND_INITIALIZER; +static unsigned readers; +static bool draining; +static int srcu_read_lock(int *s) { (void)s;pthread_mutex_lock(&lifetime_lock);readers++;pthread_mutex_unlock(&lifetime_lock);return 0; } +static void srcu_read_unlock(int *s,int i) { (void)s;(void)i;pthread_mutex_lock(&lifetime_lock);assert(readers);readers--;pthread_cond_broadcast(&lifetime_cond);pthread_mutex_unlock(&lifetime_lock); } +static void synchronize_srcu(int *s) { (void)s;pthread_mutex_lock(&lifetime_lock);draining=true;pthread_cond_broadcast(&lifetime_cond);while(readers)pthread_cond_wait(&lifetime_cond,&lifetime_lock);pthread_mutex_unlock(&lifetime_lock); } +static void device_unregister(struct device *d) { assert(!readers);d->unregistered=true; } +#define WRITE_ONCE(x,v) ((x)=(v)) +""" + unit = unit.replace(old, leaves, 1) + unit += block(source, 'void typec_mux_unregister(') + r""" +static void *remove_provider(void *arg) { typec_mux_unregister(arg);return NULL; } +int main(void) { + struct cd321x c;initial(&c);struct typec_mux_state safe={.mode=TYPEC_STATE_SAFE}; + assert(typec_mux_begin(&mux,&safe)==0&&readers==1); + pthread_t thread;assert(!pthread_create(&thread,NULL,remove_provider,&phy_provider)); + pthread_mutex_lock(&lifetime_lock);while(!draining)pthread_cond_wait(&lifetime_cond,&lifetime_lock);pthread_mutex_unlock(&lifetime_lock); + assert(phy_provider.removed&&!phy_provider.dev.unregistered&&readers==1); + assert(typec_mux_end(&mux)==0); // pre-existing transaction finishes before removal returns + assert(!pthread_join(thread,NULL)&&phy_provider.dev.unregistered&&readers==0); + unsigned before=phy_calls; + assert(typec_mux_set(&mux,&safe)==-ENODEV&&phy_calls==before&&readers==0); + puts("PASS actual unregister drains the open transaction and tombstone blocks stale handles"); +} +""" + return unit + + +def clock_order(root): + unit = quiesce.make_unit(root) + unit = unit.replace(block(unit, 'int main('), block(unit, 'int main(').replace('int main(', 'int __attribute__((unused)) prior_main(', 1), 1) + unit = unit.replace('static int power_error,', 'static bool backend_clock=true,parked;\nstatic int power_error,') + unit = unit.replace("event('R');role_calls++;return role_error;", "event('R');role_calls++;if(!role_error)backend_clock=false;return role_error;", 1) + unit = unit.replace("event('L');phy_calls++;return phy_error;", "event('L');phy_calls++;if(parked)return 0;if(!backend_clock)return -ENODEV;if(!phy_error)parked=true;return phy_error;", 1) + unit += r""" +int main(void) { + (void)current;struct cd321x c;initial(&c); + tipd_remove(&c.tps); + assert(!last_error&&reset_calls==1&&gpio_calls==1&&parked&&!backend_clock); + assert(position('Q') Date: Wed, 14 Oct 2026 04:00:16 -0400 Subject: [PATCH 456/631] soc: apple: enable SEP autoload and writable xART defaults Select the SEP driver as a module by default and export its OF device table for modpost. The missing alias left SEP unbound on a fresh Neo install until apple_sep was loaded manually. Default xart_writes to 1 while preserving APFS extent ownership, record validation and write checks. Keep xart_writes=0 as the read-only override and leave identity-keybag provisioning opt-in. Update policy documentation and the Arch package checksums. Validation: 12 SEP host tests pass; the module builds with W=1 using the prepared J700 kernel build. Neo rebooted with the exact candidate module loading automatically, attached, and reporting validated writable xART. Two 64-byte hardware RNG reads passed. The shared iBoot-container hash remained unchanged; no actual xART write or Touch ID operation is claimed. Assisted-by: LLM Signed-off-by: DjDeveloperr --- drivers/soc/apple/Kconfig | 41 ++++++++++--------- drivers/soc/apple/sep.rs | 8 +++- tools/aurora-sep/README.md | 11 +++-- tools/aurora-sep/XART-SAFETY.md | 11 +++++ .../packaging/arch/aurora-sep/PKGBUILD | 6 +-- 5 files changed, 48 insertions(+), 29 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 5cec2b22a5de87..22a4bd2ba5d8e4 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -25,6 +25,18 @@ config APPLE_MAILBOX Say Y here if you have an Apple SoC. +config APPLE_MCC + tristate "Apple T8140 system level cache" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF + default ARCH_APPLE + help + The boot firmware of the Apple T8140 SoC enables the system level + cache without letting it allocate. This driver allows allocation + once the devices that took over DMA from the bootloader are bound. + + Say 'y' or 'm' here if you have an Apple T8140 SoC. + config APPLE_PMGR_MISC bool "Apple SoC PMGR miscellaneous support" depends on PM @@ -104,6 +116,7 @@ config APPLE_SEP_FV_INTERFACE config APPLE_SEP tristate "Apple SEP (Secure Enclave Processor)" + default m if ARCH_APPLE depends on ARCH_APPLE depends on PM depends on RUST @@ -129,9 +142,10 @@ config APPLE_SEP the iBoot system container, locating its .gl extent through read-only, checksum-validated APFS metadata. It validates the complete store and both root records before registering with SEP. - Shared xART writes and identity-keybag creation default on. A write - still runs only when APFS proves a single unsnapshotted .gl extent. - The first boot creates the Linux identity keybag when it is absent. + xART writes are enabled by default, but accepted only when the driver + validates the physical extent ownership invariants. Unsupported APFS + layouts fail closed. Set xart_writes=0 for read-only operation. + Identity-keybag provisioning remains a separate opt-in. With writes off the driver still attaches, but refuses Touch ID enrol and verify: the enclave saves anti-replay state after every match. @@ -199,23 +213,6 @@ config APPLE_DISPLAY_GATE_KUNIT_TEST nothing changed when a check fails. On T6030: on by default, as before, whatever the T6031 option says. -endmenu - -endif - -config APPLE_MCC - tristate "Apple T8140 system level cache" - depends on ARCH_APPLE || COMPILE_TEST - depends on OF - default ARCH_APPLE - help - The boot firmware of the Apple T8140 SoC enables the system level - cache without letting it allocate. This driver allows allocation - once the devices that took over DMA from the bootloader are bound. - - Say 'y' or 'm' here if you have an Apple T8140 SoC. - - config APPLE_CLPC tristate "Apple host performance controller floors" depends on ARCH_APPLE || COMPILE_TEST @@ -228,3 +225,7 @@ config APPLE_CLPC are busy, and releases them at idle. Say 'y' or 'm' here if you have an Apple SoC with a PMC. + +endmenu + +endif diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index a1f8b90bbda857..aa35477b4904ad 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2337,8 +2337,12 @@ impl WorkItem for SepData { struct SepDriver(Arc); -const OF_TABLE: kernel::device_id::IdArray = - kernel::device_id::IdArray::new([(of::DeviceId::new(c"apple,sep"), ())]); +kernel::of_device_table!( + OF_TABLE, + MODULE_OF_TABLE, + (), + [(of::DeviceId::new(c"apple,sep"), ())] +); impl platform::Driver for SepDriver { type IdInfo = (); diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 4c0540bf351a35..24117623b2ba0e 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -45,12 +45,15 @@ for example with `systemctl edit aurora-sep.service`: Environment=AURORA_SEP_READY_TIMEOUT=180 ``` -`xart_writes=0` remains the default. `load-driver` explicitly opts into writes -and must only be enabled on a machine whose APFS `.gl` extent passes the -driver's narrow ownership checks: one physical extent, no snapshots or revert +The driver defaults to `xart_writes=1` and exports the `apple,sep` module +alias for automatic loading. Writes are accepted only when the APFS `.gl` +extent passes the driver's narrow ownership checks: one physical extent, no snapshots or revert state, and a single extent reference owned by the `.gl` inode. The kernel resolves that extent from checksum-validated APFS metadata; unsupported layouts -fail closed. The opt-in has been tested on m2 with identity-keybag provisioning, +fail closed. Set `options apple_sep xart_writes=0 provision_keybag=0` in a +modprobe configuration to retain read-only operation. Identity-keybag creation +still requires `provision_keybag=1`; `load-driver` supplies that option. +Enabling writes has been tested on m2 with identity-keybag provisioning, trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not change in those tests, so an actual xART write was **not** validated. Do not infer that all SEP or Touch ID operations work with a read-only GigaLocker. diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index e4578a0ec29425..5d9de0390e08de 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -1,5 +1,16 @@ # Shared xART gigalocker safety gate +## Current default + +`xart_writes=1` is the driver default. The APFS extent-ownership checks, +record validation and write-ordering checks remain mandatory; unsupported +layouts fail closed. `xart_writes=0` explicitly selects read-only operation. +Identity-keybag creation remains a separate `provision_keybag=1` opt-in. +The historical evidence below records the earlier opt-in deployments and +does not establish qualification on every supported Mac or APFS layout. + +## Earlier deployment policy + The XART `.gl` file occupies APFS-managed storage. Writes require the explicit `xart_writes=1` opt-in and an APFS-resolved extent with a single owner, no snapshots or clones, and valid existing records. `xart_start_sector` is only diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 15712b3bdf4e25..c0a935fa93e184 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=3 +pkgrel=4 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,8 +40,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - 'd7a09f31367c181a2cbda63cd58104d6117136fb5db5b20f4a3996ea3af7025e' - '83ae33b34d4f4ff42be7ba22b6d5557c3c5776075753ea288bef986645687f23') + '7777f399b05f9ed860687140efdb0e07ea7b2d23939fe59ec4894690dff299cc' + '6628ae9c63265914e7206edf61e42c613810d9d9a61adc6c7c02327504911a42') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From a1ce86685a9ee5305c5c6b515a41b3bc5b820a73 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 04:28:57 -0400 Subject: [PATCH 457/631] soc: apple: provision the Linux SEP identity automatically A fresh installation can attach SEP and use its RNG yet fail keyctl with ENODEV because no Linux identity keybag or machine reference key exists. Enable existing first-boot identity provisioning by default. Recover valid persisted identities on later boots and retain the existing refusal to replace unreadable identities or reference keys. Keep the explicit xart_writes=0 provision_keybag=0 read-only configuration and APFS checks. Update configuration help, documentation and package checksums. Validation: 12 SEP host tests pass and the module builds with W=1. Neo provisioned its missing identity using the existing provisioning path, then loaded the same trusted-key blob after three reboots. An encrypted payload authenticated under that recovered master key, corrupted trusted blobs were rejected, and identity/reference-key hashes stayed unchanged. The final module loaded automatically, with no provisioning override, and recovered the existing identity successfully. Touch ID was not tested. Assisted-by: LLM Signed-off-by: DjDeveloperr --- drivers/soc/apple/Kconfig | 5 +++-- drivers/soc/apple/sep.rs | 2 +- tools/aurora-sep/README.md | 6 ++++-- tools/aurora-sep/XART-SAFETY.md | 5 +++-- tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 6 +++--- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 22a4bd2ba5d8e4..938cfc26b03d68 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -144,8 +144,9 @@ config APPLE_SEP both root records before registering with SEP. xART writes are enabled by default, but accepted only when the driver validates the physical extent ownership invariants. Unsupported APFS - layouts fail closed. Set xart_writes=0 for read-only operation. - Identity-keybag provisioning remains a separate opt-in. + layouts fail closed. A Linux identity keybag is created automatically + when none exists; existing identities and reference keys are retained. + Set xart_writes=0 provision_keybag=0 for read-only operation. With writes off the driver still attaches, but refuses Touch ID enrol and verify: the enclave saves anti-replay state after every match. diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index aa35477b4904ad..f5b9048109a95d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2476,7 +2476,7 @@ module! { }, provision_keybag: u8 { default: 1, - description: "Create the Linux identity keybag when none exists; requires xart_writes=1", + description: "Create the Linux identity keybag when none exists; set to 0 to disable first-boot provisioning; requires xart_writes=1", }, os_uuid_hi: u64 { default: 0, diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 24117623b2ba0e..c9a1d32996422d 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -51,8 +51,10 @@ extent passes the driver's narrow ownership checks: one physical extent, no snap state, and a single extent reference owned by the `.gl` inode. The kernel resolves that extent from checksum-validated APFS metadata; unsupported layouts fail closed. Set `options apple_sep xart_writes=0 provision_keybag=0` in a -modprobe configuration to retain read-only operation. Identity-keybag creation -still requires `provision_keybag=1`; `load-driver` supplies that option. +modprobe configuration to retain read-only operation. `provision_keybag=1` +is also the default: first boot creates the Linux identity needed for trusted +keys, and later boots recover that identity and its machine reference key. +An existing unreadable or invalid identity is not silently replaced. Enabling writes has been tested on m2 with identity-keybag provisioning, trusted-key seal/unseal, and reboot reload. The raw iBoot container hash did not change in those tests, so an actual xART write was **not** validated. Do not diff --git a/tools/aurora-sep/XART-SAFETY.md b/tools/aurora-sep/XART-SAFETY.md index 5d9de0390e08de..4258fc6db06657 100644 --- a/tools/aurora-sep/XART-SAFETY.md +++ b/tools/aurora-sep/XART-SAFETY.md @@ -4,8 +4,9 @@ `xart_writes=1` is the driver default. The APFS extent-ownership checks, record validation and write-ordering checks remain mandatory; unsupported -layouts fail closed. `xart_writes=0` explicitly selects read-only operation. -Identity-keybag creation remains a separate `provision_keybag=1` opt-in. +layouts fail closed. `xart_writes=0 provision_keybag=0` explicitly selects +read-only operation. `provision_keybag=1` is the default: create the Linux +identity if missing and recover an existing identity without replacing it. The historical evidence below records the earlier opt-in deployments and does not establish qualification on every supported Mac or APFS layout. diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index c0a935fa93e184..5c0f665f9dcfe1 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=4 +pkgrel=5 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,8 +40,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - '7777f399b05f9ed860687140efdb0e07ea7b2d23939fe59ec4894690dff299cc' - '6628ae9c63265914e7206edf61e42c613810d9d9a61adc6c7c02327504911a42') + 'ed507587d3356ff2713e988e024b183772bab14061bf787bcaef28691867af0d' + '3d92b3be06bc1b670666dec3d0032893fe8c7be428a4f1200d7c9803980c769a') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. From 0f40e8c576b012839812b57bc311292e90fc2ff6 Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 13 Oct 2026 02:54:20 +0200 Subject: [PATCH 458/631] pmdomain: apple: leave firmware-owned power domains untouched On MacBook Neo (J700) m1n1 hands the display coprocessor (DCP) over running, and its firmware owns the DISP_CPU power state: writing that register raises an SError. The domain is marked apple,always-on, but the driver still writes it at probe: it sets automatic gating when the domain is on, or forces it on when it is not. The kernel panicked with an asynchronous SError in apple_pmgr_ps_probe before any console. Add apple,firmware-owned: the driver only reads such a domain, keeps it on for its consumers (the boot framebuffer), and never writes its minimum state, power target, automatic gating or reset. Mark J700's disp_cpu with it. Co-Authored-By: Claude Opus 5.5 --- .../bindings/power/apple,pmgr-pwrstate.yaml | 7 ++++++ arch/arm64/boot/dts/apple/t8140-pmgr.dtsi | 1 + drivers/pmdomain/apple/pmgr-pwrstate.c | 23 +++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index d131441214db14..d86c8b68d1b48a 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -79,6 +79,13 @@ properties: Forces this power domain to always be powered up. type: boolean + apple,firmware-owned: + description: + The boot loader hands over the coprocessor in this domain running, and its + firmware owns the domain's power state; writing it faults. The OS keeps the + domain on and never changes its power target, automatic gating or reset. + type: boolean + apple,min-state: description: Specifies the minimum power state for auto-PM. diff --git a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi index 6e6098144b757e..8438e4f17e81f9 100644 --- a/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t8140-pmgr.dtsi @@ -397,6 +397,7 @@ * firmware owns this state; writing it faults. */ apple,always-on; + apple,firmware-owned; }; diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index 4a12b16c95c64c..f66b9837c01e86 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -50,6 +50,7 @@ struct apple_pmgr_ps { bool externally_clocked; bool retained_25g83; bool system_sleep; + bool firmware_owned; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -164,6 +165,10 @@ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); int ret; + /* Its coprocessor's firmware owns the state; Linux only holds a vote. */ + if (ps->firmware_owned) + return 0; + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); if (ret && !ps->system_sleep) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); @@ -213,6 +218,8 @@ static int apple_pmgr_reset_assert(struct reset_controller_dev *rcdev, unsigned if (ps->retained_25g83) return -EBUSY; + if (ps->firmware_owned) + return -EPERM; spin_lock_irqsave(&ps->genpd.slock, flags); @@ -239,6 +246,8 @@ static int apple_pmgr_reset_deassert(struct reset_controller_dev *rcdev, unsigne if (ps->retained_25g83) return -EBUSY; + if (ps->firmware_owned) + return -EPERM; spin_lock_irqsave(&ps->genpd.slock, flags); @@ -354,6 +363,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->genpd.power_on = apple_pmgr_ps_power_on; ps->genpd.power_off = apple_pmgr_ps_power_off; + /* + * A domain whose coprocessor the boot loader hands over running (the DCP) is + * owned by that coprocessor's firmware: writing its state faults. Only read + * it, keep it on and never change its target or automatic gating. + */ + ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); + ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); if (!ps->retained_25g83 && ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, @@ -375,6 +391,13 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; } if (of_property_read_bool(node, "apple,always-on")) { + if (ps->firmware_owned) { + /* Consumers (the boot framebuffer) need it; never try to turn it on. */ + if (!active) + dev_warn(dev, "firmware-owned domain %s does not read as on\n", name); + ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; + active = true; + } else if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; if (!active) { dev_warn(dev, "always-on domain %s is not on at boot\n", name); From 5a2a4d773ec69118071f8df96cd393f039d19bfb Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 19:46:45 -0400 Subject: [PATCH 459/631] drm/asahi: recycle settled quarantined G17 compute queues A failed client's compute queue is quarantined. Once its owner closes and the retirement witness has settled every retained command, the queue has no active command, so exit_candidate() never issues a kill for it, and reusable() refuses it because it is still quarantined. It therefore keeps its QID and its USC pool backing until the processor stops. Each Gen1-capable compute pool takes about 43 MiB of the T8140 dynamic lower VA window. On Neo, after roughly 42 failed clients the window is exhausted and every new compute submission fails with ENOSPC until reboot. Return such a queue to service the way recycle_killed() already does after an acknowledged kill: reset the item ring at the firmware consumer cursor and require fresh backing, so the existing vacate path publishes the free-list release and frees the VA. A kill acknowledged before the scheduler page showed the killed state was left Unproven forever, retaining its queue as well. Re-read such kills on later passes and settle them only once the page shows the killed state, also when their commands were retired by other means. Validation on Neo with deliberately failing compute clients: unmodified aurora-wip reaches ENOSPC after about 42 failed clients. With this change, a fresh boot handled 84 failed clients, including a replay of 28 failed device creations, with no ENOSPC and no firmware crash, and the same build passed more than 150 failed clients on one boot. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/queue/compute.rs | 29 ++++++++++++++++++- .../gpu/drm/asahi/g17/queue/compute/exit.rs | 10 +++++-- drivers/gpu/drm/asahi/g17/runtime.rs | 1 + 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index cae576fe183af5..f3f050d482bef6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1192,6 +1192,33 @@ impl Queue { { return Ok(false); } + self.recycle_settled()?; + Ok(true) + } + /// A quarantined queue whose owner closed and whose every retained command was + /// already settled, by the exact retirement witness or by an acknowledged kill, + /// holds no pending command: the firmware consumed its item ring exactly as after + /// a kill. Neither `reusable` nor `exit_candidate` matches it, so without this it + /// keeps its QID and USC pool backing until processor stop, and repeated client + /// failures exhaust the compute pool VA range. Return it to service as + /// `recycle_killed` does. + pub(crate) fn recycle_quarantined_idle(&mut self) -> Result { + if !self.installed + || !self.quarantined + || !self.released + || self.growing + || self.exit.is_some() + || self.awaiting_witness + || self.active.len != 0 + || self.replays.len != 0 + || self.previous.is_some() + { + return Ok(false); + } + self.recycle_settled()?; + Ok(true) + } + fn recycle_settled(&mut self) -> Result { self.graph.reset_items()?; self.exit = None; self.quarantined = false; @@ -1213,7 +1240,7 @@ impl Queue { self.retirement_ready = true; self.released_at = now_ns(); self.needs_fresh_backing = true; - Ok(true) + Ok(()) } pub(crate) fn released_at(&self) -> u64 { self.released_at diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index a5fda94d6783a9..a4b1b8b03fc939 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -156,10 +156,16 @@ impl Queue { complete_qos: &mut impl FnMut(crate::g17::qos::Owner) -> Result, defer: &mut impl FnMut(Deferred) -> Result, ) -> Result { - if !self.exit.as_ref().is_some_and(|exit| exit.phase == Phase::Acknowledged) { + let Some(phase) = self.exit.as_ref().map(|exit| exit.phase) else { + return Ok(()); + }; + if !matches!(phase, Phase::Acknowledged | Phase::Unproven) { return Ok(()); } - if self.active.len != 0 { + // An unproven kill (acknowledged before the scheduler page showed the killed + // state) is re-read on later passes and settles only once the page proves it, + // even after its commands were retired by other means. + if self.active.len != 0 || phase == Phase::Unproven { if !recovery_closed { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index 93e089827ebd09..bf3260685d51ff 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -1253,6 +1253,7 @@ impl super::Firmware { .filter_map(|entry| entry.queue.as_deref_mut()) { queue.recycle_killed()?; + queue.recycle_quarantined_idle()?; if let Some(binding) = queue.detach_exited_owner() { deferred.defer_binding(binding); } From b6e4ba38f05cde4235da35f7447c8404004e3b8a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 20:05:00 -0400 Subject: [PATCH 460/631] drm/asahi: g17: replay render groups firmware retired without executing A render group can be retired by firmware without its fragment pass ever running: the tiler start record is written, the fragment end record is not, both completion stamps already cover the group, its job list is empty and no recovery is reported. On Neo this happens to an innocent owner (the compositor) at the instant another client's compute command fails with a zero-duration completion record. The front group then polls Pending forever, the 2 s scheduler timeout fires, fail_after_observation takes the strict abandonment witness and fails the owner's VM with EIO. The compositor's device is lost and the desktop freezes. Use the same witness to recognise the abandoned group as soon as it retires, and publish the same command again instead of failing it. In Pair::retire_front a lone, outer-published front group that polls Pending is checked with retirement::abandoned (executed groups are never accepted); the proof has to hold on two observations at least 2 ms apart so a single sample straddling a firmware update cannot trigger it. The dead generation's free-list and QoS accounting is completed exactly as the timeout path does, the record is retired and the packet is parked on the pair with its publication state reset; its completion stays unsignalled and no VM status is recorded. Firmware::replay_render, run by the event worker next to the compute replay and never during recovery, prepares and publishes the parked packet again through the ordinary path (fresh ordinal, descriptors, scene and timestamp record). A closed owner, a cancelled packet or a stale context settle the replay with ECANCELED, which does not poison the VM; other publication failures settle it with their error as before. Completion::begin_replay bounds the attempts to three, after which the previous EIO settlement applies. The original submission's scheduler timeout is ignored while its packet is parked, and a processor stop drains a parked replay like active groups. Nothing of an abandoned group reached its attachments, which only the fragment stage writes, so re-running the tiler and fragment passes from the same command is the first execution as far as render targets are concerned. Tiler-side pipeline statistics may count twice; vertex-stage side effects exist only behind an opt-in diagnostic. Tested on Neo (A18 Pro, G17P) with failing ray-query clients next to an animating compositor: the first incident at 81.455 s was replayed at 81.468 s (slot 4, qids [8, 9], pass [tiler start, 0], stamps covering the group) and the compositor kept rendering with no device loss, where the previous kernels lost the desktop in three out of three runs. With this change and VerboseFaults set, two further boots logged 2 and 3 replayed groups over 13 and 24 failing clients (the first boot differed only in the log string), with no device loss and the compositor still rendering. Boots of earlier builds with the same replay logic ran about 1,500 ray-query, ray-tracing pipeline and mesh conformance cases without failing clients and logged no replay. With the VerboseFaults debug flag, each replay is logged with its slot and the pass record that proved the group was not executed. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/job.rs | 5 + drivers/gpu/drm/asahi/g17/queue/render.rs | 134 +++++++++++++++++- .../gpu/drm/asahi/g17/queue/render/parked.rs | 1 + drivers/gpu/drm/asahi/g17/runtime/events.rs | 2 + drivers/gpu/drm/asahi/g17/runtime/render.rs | 1 + .../drm/asahi/g17/runtime/render/replay.rs | 105 ++++++++++++++ .../drm/asahi/g17/runtime/render/settle.rs | 2 +- 7 files changed, 247 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/runtime/render/replay.rs diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index b85d794ace5120..9ad0370edfa2fe 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -185,6 +185,11 @@ impl Packet { pub(crate) fn is_published(&self) -> bool { self.publication.load(Ordering::Acquire) == 2 } + /// A published group retired by firmware without executing is published + /// again from the same packet. Publication state returns to "begun". + pub(crate) fn reset_publication_for_replay(&self) { + self.publication.store(1, Ordering::Release); + } pub(crate) fn cancel_requested(&self) -> bool { self.cancelled.load(Ordering::Acquire) } diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 24a274e1af62a9..bbf85695970a04 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -130,8 +130,14 @@ struct Active { context_update: bool, failed: bool, drained_result: Option>, + /// First observation of this group retired and unlinked without execution. + abandoned_since: Option, } +/// A firmware-retired, unexecuted group must hold on two observations this far +/// apart before it is republished; a single sample may straddle an update. +const ABANDON_GRACE_NS: u64 = 2_000_000; + /// Installed kick backing remains owned even when this pair has no logical /// owner. Commands retain their exact descriptor and allocation generations. pub(crate) struct Pair { @@ -162,6 +168,8 @@ pub(crate) struct Pair { previous: Option, displaced: Option, context_update: bool, + /// A retired, never-executed group awaiting republication on this pair. + replay: Option>, } impl Pair { #[inline(never)] @@ -243,6 +251,7 @@ impl Pair { terminal: None, previous: None, displaced: None, + replay: None, context_update: true, })) } @@ -271,7 +280,21 @@ impl Pair { &mut self.memory.manager } pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 + self.active.len() != 0 || self.replay.is_some() + } + /// A replay or an abandonment grace period needs the worker to poll. + pub(crate) fn replay_polling(&self) -> bool { + self.replay.is_some() || self.active.iter().any(|a| a.abandoned_since.is_some()) + } + pub(crate) fn replay_front(&self) -> Option> { + self.replay.clone() + } + pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { + if !self.replay.as_ref().is_some_and(|front| Arc::ptr_eq(front, packet)) { + return Err(EIO); + } + self.replay = None; + Ok(()) } pub(crate) fn published_in_flight(&self) -> bool { !self.quarantined && self.active.iter().any(|active| active.outer_published) @@ -682,6 +705,7 @@ impl Pair { context_update: self.context_update, failed: false, drained_result: None, + abandoned_since: None, }) .map_err(|_| EBUSY)?; self.prepared = None; @@ -904,6 +928,9 @@ impl Pair { .as_ref() .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) } + pub(crate) fn replay_contains(&self, packet: &Arc) -> bool { + self.replay.as_ref().is_some_and(|replay| Arc::ptr_eq(replay, packet)) + } pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { self.active.iter().find_map(|active| { active @@ -1079,7 +1106,8 @@ impl Pair { }; let timestamps = match result { Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Retry | retirement::Poll::Pending) => return Ok(false), + Ok(retirement::Poll::Pending) => return self.replay_abandoned(host, defer), + Ok(retirement::Poll::Retry) => return Ok(false), Err(error) => { self.quarantine(error, defer)?; return Ok(false); @@ -1109,6 +1137,105 @@ impl Pair { } } + /// The front group was retired and unlinked by firmware without executing + /// (job list empty, stamps cover it, no recovery, fragment end never + /// written). Nothing of this group reached its attachments, so the same + /// command is published again instead of failing the owner's VM. The + /// proof is the strict timeout witness, taken twice ABANDON_GRACE_NS apart, + /// and only for a lone group whose completion still accepts a replay. + fn replay_abandoned( + &mut self, + host: &mut impl Host, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + use retirement::Memory; + if self.quarantined || self.active.len() != 1 || self.replay.is_some() { + return Ok(false); + } + let front = self.active.front().ok_or(EIO)?; + if !front.outer_published || front.failed { + return Ok(false); + } + let mut observer = Observer { + memory: &self.memory, + qids: self.qids(), + host, + }; + core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); + let snapshot = observer.snapshot(front.ticket.ordinal)?; + let stamps = observer.stamps()?; + let (_, state) = observer.host.epoch()?; + let abandoned = + retirement::abandoned(&front.ticket, snapshot, stamps, state, false)?; + let now = crate::g17::now_ns(); + let front = self.active.front_mut().ok_or(EIO)?; + if !abandoned { + front.abandoned_since = None; + return Ok(false); + } + let Some(since) = front.abandoned_since else { + front.abandoned_since = Some(now); + return Ok(false); + }; + if now.saturating_sub(since) < ABANDON_GRACE_NS { + return Ok(false); + } + if crate::debug::debug_enabled(crate::debug::DebugFlags::VerboseFaults) { + pr_info!( + "render slot {} retired without execution (pass {:?}); replaying\n", + self.slot, + snapshot.pass + ); + } + let qos = self.qos_owners(); + let buffer = self.buffer_id(); + let pool = self.free_list.id(); + let front = self.active.front_mut().ok_or(EIO)?; + if !front.flist_completed { + host.complete_render( + (!front.qos_completed).then_some(qos), + pool, + front.generation, + buffer, + )?; + front.flist_completed = true; + front.qos_completed = true; + } + let ordinal = front.ticket.ordinal; + let packet = front.packet.clone(); + if !packet.completion.begin_replay() { + if !packet.completion.spared() { + packet.completion.status().record(EIO); + if self.terminal.is_none() { + self.terminal = Some((packet.completion.status().clone(), EIO)); + } + } + defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + return Ok(true); + } + self.memory.retire(ordinal)?; + self.active.pop().ok_or(EIO)?; + packet.reset_publication_for_replay(); + self.replay = Some(packet); + Ok(true) + } + + /// The pending replay cannot proceed (closed owner, cancellation or a + /// publication refusal). ECANCELED leaves the VM usable; any other error + /// is the ordinary device-loss outcome the replay tried to avoid. + pub(crate) fn fail_replay( + &mut self, + error: Error, + defer: &mut impl FnMut(Deferred) -> Result, + ) -> Result { + let Some(packet) = self.replay.take() else { + return Ok(()); + }; + defer(Deferred::Retired(packet.completion.clone(), Err(error))) + } + /// A closed preparation epoch invalidates only this unpublished descriptor. /// Its completion remains available for a retry under the next epoch. pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { @@ -1292,6 +1419,9 @@ impl Pair { self.memory.retire(ordinal)?; self.active.pop().ok_or(EIO)?; } + if let Some(packet) = self.replay.take() { + defer(Deferred::Drained(packet.completion.clone(), Err(EIO)))?; + } if let Some(previous) = self.previous.as_mut() { previous.ready = true; } diff --git a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs index f712bb7b1b87dd..39941de815e077 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render/parked.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render/parked.rs @@ -213,6 +213,7 @@ impl ParkedPair { terminal: None, previous: self.previous.take(), displaced: None, + replay: None, context_update: true, })) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/events.rs b/drivers/gpu/drm/asahi/g17/runtime/events.rs index 81ca10328b42a4..04b2ae7457120b 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/events.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/events.rs @@ -368,6 +368,7 @@ impl crate::g17::Shared { // Replay only at this outer boundary, after recovery has settled. // Publication may service events but must never recurse into replay. firmware.replay_compute(deferred)?; + firmware.replay_render(deferred)?; drop(state); self.observe_renders(masks, deferred)?; let mut state = self.state.lock(); @@ -409,6 +410,7 @@ impl crate::g17::Shared { let backpressured = firmware.render_control_backpressured(); let polling = backpressured || firmware.queues.replay_outer_wait.is_some() + || firmware.queues.render.replay_polling() || firmware.queues.teardown.polling(now) || firmware.compute_exit_polling(now) || firmware diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index bd68fb50c38fc5..2d87679c58110c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -11,6 +11,7 @@ mod observe; mod pools; mod prepare; mod recovery; +mod replay; mod settle; mod submit; pub(super) mod teardown; diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs new file mode 100644 index 00000000000000..ce9771778acd4d --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/runtime/render/replay.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Republication of render groups that firmware retired without executing. +//! The pair proves the abandonment (retired, unlinked, stamps covering it, no +//! recovery, fragment end never written) and parks the packet; this pass +//! prepares and publishes the same command again under the device mutex. + +use super::super::DeferredBatch; +use crate::g17::{object::Allocator, queue::render::RenderAddresses, Firmware}; +use kernel::prelude::*; + +impl super::RenderState { + pub(in crate::g17) fn replay_polling(&self) -> bool { + self.entries + .iter() + .flatten() + .filter_map(|entry| entry.pair.as_ref()) + .any(|pair| pair.replay_polling()) + } +} + +impl Firmware { + pub(in crate::g17) fn replay_render(&mut self, deferred: &mut DeferredBatch) -> Result { + if self.recovery.pending() + || crate::g17::recovery::Memory::recovery_state(&self.init)? != 0 + { + return Ok(()); + } + for slot in 0..self.queues.render.entries.len() { + let Some((packet, owner, qids, closed, busy)) = + self.queues.render.entries[slot].as_ref().and_then(|entry| { + let pair = entry.pair.as_ref()?; + let packet = pair.replay_front()?; + Some(( + packet, + entry.reservation.owner, + pair.qids(), + entry.closed, + entry.preparing.is_some() + || entry.growing + || entry.pending_timeout + || entry.pending_error.is_some(), + )) + }) + else { + continue; + }; + if busy { + continue; + } + if closed || packet.cancel_requested() || !packet.context.is_current() { + self.with_render(slot, |pair, _| { + pair.fail_replay(ECANCELED, &mut |event| deferred.push(event)) + })?; + continue; + } + if packet.completion.status().get() != 0 { + self.with_render(slot, |pair, _| { + pair.fail_replay(EIO, &mut |event| deferred.push(event)) + })?; + continue; + } + let addresses = RenderAddresses { + stamps: [ + self.init.stamp_addresses(qids[0])?, + self.init.stamp_addresses(qids[1])?, + ], + }; + let dev = self.primary.state.shared.drm()?; + let uat = self.uat.clone(); + let prepared = self.with_render(slot, |pair, _| { + if !pair.room() { + return Err(EBUSY); + } + let alloc = Allocator { + dev: &dev, + uat: &uat, + }; + pair.prepare(&alloc, &packet, addresses) + }); + match prepared { + Ok(()) => {} + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + continue; + } + } + match self.publish_render(owner, slot as u8, &packet, deferred) { + Ok(()) => { + self.with_render(slot, |pair, _| pair.finish_replay(&packet))?; + } + Err(error) if error == EBUSY || error == EAGAIN => continue, + Err(error) => { + self.with_render(slot, |pair, _| { + pair.fail_replay(error, &mut |event| deferred.push(event)) + })?; + } + } + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index 0a4634bccaea52..a51a42021c4680 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -70,7 +70,7 @@ impl crate::g17::Firmware { .and_then(Option::as_ref) .and_then(|entry| entry.pair.as_ref()) .ok_or(EIO)?; - if !pair.contains_packet(packet) { + if pair.replay_contains(packet) || !pair.contains_packet(packet) { return Ok(()); } let generation = self.recovery.generation(); From 3bd17221b5fb47ef79d03528d681c7bb48a01778 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 21:40:42 -0400 Subject: [PATCH 461/631] drm/asahi: g17: never kill compute commands with ray traversal scratch A closed client's quarantined or cancelled compute queue is ended with a firmware context kill once its owner is gone. For commands that use ray traversal scratch this goes wrong on Neo (T8140) when the traversal stopped making progress: - releasing the killed queue's pool right after the kill was acknowledged crashed the GPU firmware about 40 ms later (3 of 3 cases), taking the whole GPU and the desktop down; - keeping the pool instead avoided the crash, but about 12 s later every following render on the device timed out and the compositor lost its context. Leaving such a queue alone, as a diagnostic build that skipped the kill did on five boots, kept the rest of the GPU working each time. Do not issue the kill while any active command of the queue has traversal scratch enabled. The client still sees its device lost through the existing timeout and quarantine path, the queue stays retained until its commands retire, and every other compute queue keeps the existing kill and recycle behaviour. Tested on Neo with this exact change: a ray tracing dispatch that stopped making progress timed out at 69.7 s and was left unkilled; the compositor kept rendering for the following 40 s, and new compute, ray tracing pipeline and amber conformance work ran normally afterwards. A raygen shader looping forever was also left unkilled; later ray tracing dispatches still completed in 20 ms each and the desktop stayed live. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/g17/queue/compute/exit.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index a4b1b8b03fc939..6557d8999ba438 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -83,11 +83,25 @@ impl Transaction { } impl Queue { + /// A command with ray traversal scratch is never killed. On Neo, killing a + /// traversal that stopped making progress either crashed the GPU firmware + /// when the queue's pool was then released, or stalled every later render + /// when it was retained; left alone, the rest of the GPU kept working. Such + /// a queue stays quarantined and retained until its commands retire. + fn active_traversal(&self) -> bool { + self.active.iter().any(|active| { + matches!( + &active.packet.command, + crate::g17::command::Validated::Compute { scratch, .. } if scratch.enabled() + ) + }) + } pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) && self.released && self.active.len != 0 && self.exit.is_none() + && !self.active_traversal() && (self.quarantined || self.retired_by_teardown) && age >= if self.quarantined { From 16144c20c808ae60d9a0f00b5669915deb30054a Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Wed, 14 Oct 2026 20:46:34 -0400 Subject: [PATCH 462/631] drm/asahi: g17: ray queries in render stages Let render commands carry ray query state for their vertex (tiling) and fragment stages, as compute commands already do. UAPI: - DRM_ASAHI_FEATURE_RENDER_RAY_QUERY advertises the feature. - struct drm_asahi_render_ray_query_stage describes one stage: its ray query scratch layout and intersection tables. drm_asahi_cmd_render gains ray_query[2] at its end; shorter commands from older userspace are read with both stages zeroed. - drm_asahi_queue_create gains render_scs_layout[2], the largest layouts the queue will submit, which sizes the queue's render pool. Zero keeps the existing pool size. Firmware programming: for each stage with ray query scratch, the tiling or fragment register program carries the scratch size, tail, stack and window words and the intersection table registers, and enables SCS with register 0x017d9; its kick sets the SCS-enabled header bit. Stages without ray queries keep the existing register programs unchanged. Validation rejects intersection pools and default programs outside the VM and a scratch layout whose enable state disagrees with the intersection tables. The render pool is sized from both stages' scratch demand at queue creation, and a command needing more than was reserved is rejected. Tested on Neo (A18 Pro) with the matching Mesa series: ray_query builtin.flow and advanced.null_as cases 41 of 48 pass across vertex, tessellation, geometry, fragment, compute and ray tracing stages (the failures are tessellation evaluation and three null acceleration structure cases); position fetch 60 of 60 runnable cases; earlier builds of the same render path passed 883 ray query stage cases. Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/file.rs | 2 + drivers/gpu/drm/asahi/g17/command.rs | 23 ++++- drivers/gpu/drm/asahi/g17/context.rs | 14 ++- drivers/gpu/drm/asahi/g17/freelist.rs | 19 ++-- drivers/gpu/drm/asahi/g17/fw/compute.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/compute_pool.rs | 9 +- drivers/gpu/drm/asahi/g17/fw/kick.rs | 8 +- drivers/gpu/drm/asahi/g17/fw/mod.rs | 1 + drivers/gpu/drm/asahi/g17/fw/render.rs | 92 +++++++++++++------- drivers/gpu/drm/asahi/g17/fw/render_rt.rs | 48 ++++++++++ drivers/gpu/drm/asahi/g17/job.rs | 4 + drivers/gpu/drm/asahi/g17/mod.rs | 3 +- drivers/gpu/drm/asahi/g17/queue/compute.rs | 2 +- drivers/gpu/drm/asahi/g17/queue/mod.rs | 11 +++ drivers/gpu/drm/asahi/g17/queue/render.rs | 13 ++- drivers/gpu/drm/asahi/g17/runtime/backend.rs | 4 + drivers/gpu/drm/asahi/queue/mod.rs | 5 ++ include/uapi/drm/asahi_drm.h | 55 ++++++++++++ 18 files changed, 261 insertions(+), 60 deletions(-) create mode 100644 drivers/gpu/drm/asahi/g17/fw/render_rt.rs diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index ae40b1ca612cb3..2d518e16adffdc 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -1334,6 +1334,8 @@ impl File { data.usc_exec_base, )?; + queue.reserve_render_scs(data.render_scs_layout)?; + data.queue_id = resv.index().try_into()?; resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index dd3be1a81e1d83..671e513852c96a 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -6,6 +6,7 @@ use super::fw::{ compute::{IntersectionTables, ScratchRequest}, kick::WriteRange, render::{RenderPass, UscProgram, ZlsSurface}, + render_rt::RayQueryStage, }; use crate::util::{AnyBitPattern, Reader}; use core::ops::Range; @@ -14,6 +15,7 @@ use kernel::{prelude::*, uapi}; const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; +const RENDER_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render, prior_cdm) + size_of::(); const COMPUTE_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) + size_of::(); const SAMPLER_SIZE: u64 = 8; const PAGE_SIZE: u64 = 0x4000; @@ -78,7 +80,7 @@ impl Attachments { // Commands stay inline in the submission plan, avoiding an allocation per render pass. #[allow(clippy::large_enum_variant)] pub(crate) enum Payload { - Render(uapi::drm_asahi_cmd_render, [Attachments; 2]), + Render(uapi::drm_asahi_cmd_render, [Attachments; 2], [RayQueryStage; 2]), Compute(uapi::drm_asahi_cmd_compute, Attachments, IntersectionTables), } @@ -128,7 +130,11 @@ impl<'a> Parser<'a> { == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; - Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) + Payload::Render( + payload(&bytes[..bytes.len().min(RENDER_BASE_SIZE)])?, + [self.attachments[0], self.attachments[1]], + RayQueryStage::parse_pair(bytes.get(RENDER_BASE_SIZE..).unwrap_or(&[]))?, + ) } else { self.counts[1] += 1; Payload::Compute( @@ -284,6 +290,7 @@ fn surface( pub(crate) enum Validated { Render { pass: RenderPass, + ray_query: [RayQueryStage; 2], attachments: [Attachments; 2], timestamps: [uapi::drm_asahi_timestamps; 2], }, @@ -302,7 +309,7 @@ pub(crate) enum Validated { impl Payload { pub(crate) fn prior(&self) -> [u64; 2] { match self { - Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], + Self::Render(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], Self::Compute(cmd, _, _) => [cmd.prior_vdm, cmd.prior_cdm], } } @@ -313,7 +320,7 @@ impl Payload { space: &impl AddressSpace, ) -> Result { match self { - Self::Render(cmd, attachments) => { + Self::Render(cmd, attachments, ray_query) => { if cmd.flags & !RENDER_FLAGS != 0 || !(1..=MAX_DIMENSION).contains(&cmd.width_px) || !(1..=MAX_DIMENSION).contains(&cmd.height_px) @@ -415,8 +422,16 @@ impl Payload { | (u64::from(cmd.bg_eot_partial_rsrc_spec_hi) << 32), rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, }; + for stage in ray_query { + if stage.intersection.enabled() { + require_range(space, stage.intersection.pool, stage.intersection.pool_bytes, Access::Read)?; + let code = stage.intersection.default_program & 0x0000_ffff_ffff_ffff; + if code != 0 { require_range(space, code, 64, Access::Read)?; } + } + } Ok(Validated::Render { pass, + ray_query: *ray_query, attachments: *attachments, timestamps: [cmd.ts_vtx, cmd.ts_frag], }) diff --git a/drivers/gpu/drm/asahi/g17/context.rs b/drivers/gpu/drm/asahi/g17/context.rs index afd0d1d22f5bbb..6bc62a7d44beef 100644 --- a/drivers/gpu/drm/asahi/g17/context.rs +++ b/drivers/gpu/drm/asahi/g17/context.rs @@ -14,7 +14,7 @@ use super::{ status::VmStatus, }; use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, Ordering}; +use core::sync::atomic::{fence, AtomicBool, AtomicU32, AtomicUsize, Ordering}; use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; use kernel::{ new_mutex, @@ -120,6 +120,7 @@ pub(crate) struct Context { work: Arc>, work_base: u64, render: Arc>>>, + render_blocks: AtomicUsize, /// Fixed for the context's life. Physical queues bound to it carry this /// profile in every record, ring, doorbell and registration. policy: Policy, @@ -192,6 +193,7 @@ impl Context { work, work_base, render, + render_blocks: AtomicUsize::new(super::fw::freelist::RENDER_BLOCKS), policy, }) } @@ -277,6 +279,14 @@ impl Context { } Ok(result) } + pub(crate) fn reserve_render_blocks(&self, blocks: usize) -> Result { + if self.is_published() || self.render.lock().is_some() { return Err(EBUSY); } + self.render_blocks.store(blocks, Ordering::Release); + Ok(()) + } + pub(crate) fn render_blocks(&self) -> usize { + self.render_blocks.load(Ordering::Acquire) + } pub(crate) fn render_pool( &self, alloc: &Allocator<'_>, @@ -293,7 +303,7 @@ impl Context { return Ok(pool.clone()); } let pool = Arc::new( - RenderPool::new(alloc, ids, render_global, self.vm())?, + RenderPool::new(alloc, ids, render_global, self.vm(), self.render_blocks())?, GFP_KERNEL, )?; *render = Some(pool.clone()); diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index 2612a9302ce28c..c9c9ebda7f6bc0 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -139,8 +139,12 @@ impl RenderBacking { alloc: &Allocator<'_>, render_global: &mmu::Vm, client: &mmu::Vm, + blocks: usize, ) -> Result { - let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; + if blocks < RENDER_BLOCKS || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) { + return Err(EINVAL); + } + let pages = Pages::new(alloc, blocks, RunLayout::Render)?; let address = pages.page_list_va(); let global = pages .backing @@ -161,6 +165,7 @@ impl RenderBacking { /// consumed and the client VM is evicted; a fresh backing is installed before /// the pool is staged again. pub(crate) struct RenderPool { + blocks: usize, backing: Arc>>, /// Page-list VA of the installed backing; zero while vacant. page_list: AtomicU64, @@ -182,9 +187,10 @@ impl RenderPool { ids: &Arc, render_global: &mmu::Vm, client: &mmu::Vm, + blocks: usize, ) -> Result { let id = RenderId::new(ids)?; - let backing = RenderBacking::new(alloc, render_global, client)?; + let backing = RenderBacking::new(alloc, render_global, client, blocks)?; let address = backing.pages.page_list_va(); let run_list_va = backing.pages.run_list_va(); let state = alloc.kernel( @@ -207,7 +213,7 @@ impl RenderPool { class: FreeListClass::Two, page_list_va: address, run_list_va, - blocks: RENDER_BLOCKS as u32, + blocks: blocks as u32, state_va: state.gpu_va(), }, 0, @@ -222,6 +228,7 @@ impl RenderPool { GFP_KERNEL, )?; Ok(Self { + blocks, backing, page_list: AtomicU64::new(address), epoch: AtomicU64::new(0), @@ -279,7 +286,7 @@ impl RenderPool { if !self.vacant() { return Ok(()); } - self.rearm(RenderBacking::new(alloc, render_global, client)?) + self.rearm(RenderBacking::new(alloc, render_global, client, self.blocks)?) } pub(crate) fn serial(&self) -> u64 { @@ -303,7 +310,7 @@ impl RenderPool { } pub(crate) fn end_va(&self) -> u64 { self.run_list_va() - + (RENDER_BLOCKS * size_of::()) as u64 + + (self.blocks * size_of::()) as u64 } /// Fresh pools are already populated. Reuse requires the previous release @@ -319,7 +326,7 @@ impl RenderPool { class: FreeListClass::Two, page_list_va: self.page_list_va(), run_list_va: self.run_list_va(), - blocks: RENDER_BLOCKS as u32, + blocks: self.blocks as u32, state_va: self.state.gpu_va(), }, submitted, diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 829b519c545fb4..f47b31a4eb1b60 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -112,16 +112,16 @@ impl ScratchRequest { pub(crate) fn pages(self) -> u16 { self.pages } - fn size_word(self) -> u64 { + pub(crate) fn size_word(self) -> u64 { let pages = u64::from(self.pages); pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) } - fn window_word(self) -> u64 { + pub(crate) fn window_word(self) -> u64 { self.window } - fn stack_word(self) -> u64 { + pub(crate) fn stack_word(self) -> u64 { self.stack } @@ -129,7 +129,7 @@ impl ScratchRequest { self.driver } - fn tail_word(self) -> u64 { + pub(crate) fn tail_word(self) -> u64 { self.tail } diff --git a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs index 4005daa4e98324..0f0614f6155a17 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute_pool.rs @@ -5,9 +5,9 @@ use kernel::prelude::*; use super::initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; use super::freelist::{COMPUTE_BLOCKS, PAGE_LIST_SIZE}; -pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result { +pub(crate) fn extra_bytes(active_cores: u32, scs_pages: u16) -> Result { if active_cores == 0 { return Err(EINVAL); } - if scs_pages == 0 { return Ok(COMPUTE_BLOCKS); } + if scs_pages == 0 { return Ok(0); } // Register0x101d1 counts 4-KiB pages and their 1-MiB large-page inventory. // Register0x0d479 reserves eight further pages for each enabled USC. let small = u64::from(scs_pages); @@ -15,6 +15,11 @@ pub(crate) fn required_blocks(active_cores: u32, scs_pages: u16) -> Result Result { + let extra = extra_bytes(active_cores, scs_pages)?; let bytes = extra.checked_add((COMPUTE_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) .ok_or(EOVERFLOW)?; let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index 47cdce3ca09e5c..e07fbfb485a10a 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -370,7 +370,7 @@ pub(crate) struct KickArgs<'a> { /// Register arrays bound to the kick. pub(crate) register_arrays: [Option; 4], /// The compute register program includes a shader-context scratch request. - pub(crate) compute_scratch: bool, + pub(crate) scs_enabled: bool, /// Priority class of the queue. pub(crate) priority: u8, } @@ -423,7 +423,7 @@ impl KickEntry { const QID_SHIFT: u32 = 42; const MCACHE_PRESENT: u64 = 1 << 58; const PRIORITY_SHIFT: u32 = 59; - const COMPUTE_SCRATCH: u64 = 1 << 61; + const SCS_ENABLED: u64 = 1 << 61; const QOS_SLOT_SHIFT: u32 = 32; const QOS_CLASS_SHIFT: u32 = 40; const QOS_CLASS_MAX: u8 = 0x1f; @@ -446,8 +446,8 @@ impl KickEntry { let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT | (args.qid as u64) << Self::QID_SHIFT | (args.priority as u64) << Self::PRIORITY_SHIFT; - if args.compute_scratch { - header |= Self::COMPUTE_SCRATCH; + if args.scs_enabled { + header |= Self::SCS_ENABLED; } let mcache = match args.mcache { Some(table) => { diff --git a/drivers/gpu/drm/asahi/g17/fw/mod.rs b/drivers/gpu/drm/asahi/g17/fw/mod.rs index 479bc91d21ca5f..b6bb6aaea860ff 100644 --- a/drivers/gpu/drm/asahi/g17/fw/mod.rs +++ b/drivers/gpu/drm/asahi/g17/fw/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod initdata; pub(crate) mod kick; pub(crate) mod queue; pub(crate) mod render; +pub(crate) mod render_rt; pub(crate) mod qos; /// Clears a firmware structure in place. diff --git a/drivers/gpu/drm/asahi/g17/fw/render.rs b/drivers/gpu/drm/asahi/g17/fw/render.rs index 23eaa7e7dfba9e..85a2385b421b81 100644 --- a/drivers/gpu/drm/asahi/g17/fw/render.rs +++ b/drivers/gpu/drm/asahi/g17/fw/render.rs @@ -23,7 +23,7 @@ use super::kick::{ use super::queue::CommandTag; use super::{ clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, - UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, + UNK_0D411, UNK_101D9, UNK_1C8F8, }; /// Size of a tiling descriptor. @@ -54,7 +54,11 @@ const DEFLAKE_20: u64 = 0x20; const DEFLAKE_2A0: u64 = 0x2a0; const TA_REGISTERS: usize = 73; +const TA_REGISTERS_MAX: usize = TA_REGISTERS + 1; +const TA_SCS_ENABLE_INDEX: usize = 1; const FRAGMENT_REGISTERS: usize = 89; +const FRAGMENT_REGISTERS_MAX: usize = FRAGMENT_REGISTERS + 1; +const FRAGMENT_SCS_ENABLE_INDEX: usize = 1; const PARTIAL_STORE_REGISTERS: usize = 16; const PARTIAL_RESUME_REGISTERS: usize = 23; const PARTIAL_LOAD_REGISTERS: usize = 10; @@ -122,8 +126,6 @@ const UNK_15359_PARTIAL: u64 = 0x20; /// Required value of register 0x15049 and of fragment descriptor field /// +0x1f50. const UNK_15049: u64 = 0x10_0000; -/// Value of the fragment register 0x10791. -const UNK_10791_FRAGMENT: u64 = UNK_10791 | 1 << 9; /// USC program offsets carry tag bits that are not part of the address. const USC_OFFSET_TAG_MASK: u32 = 7; @@ -303,6 +305,7 @@ pub(crate) struct RenderStage { pub(crate) struct RenderArgs<'a> { /// Render-pass state. pub(crate) pass: &'a RenderPass, + pub(crate) ray_query: [super::render_rt::RayQueryStage; 2], /// Kernel-owned objects. pub(crate) resources: &'a RenderResources, /// Submission ordinal of the render queue pair, from zero. @@ -481,7 +484,7 @@ fn ta_registers( args: &RenderArgs<'_>, g: &Geometry, w: &PassWords, -) -> Result<[RegisterWrite; TA_REGISTERS]> { +) -> Result<[RegisterWrite; TA_REGISTERS_MAX]> { let p = args.pass; let r = args.resources; let tilemap = user_offset(r.tilemap_va)?; @@ -497,9 +500,12 @@ fn ta_registers( } let key = work_key(r.context_id, args.ta.state_word); let ids = r.ta_ids.word(); + let rt = args.ray_query[0]; + let ift = rt.intersection.words(); Ok([ RegisterWrite::new(0x01748, 1), + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x10141, 0x200), RegisterWrite::new(0x1c039, tilemap), RegisterWrite::new(0x1c9c8, tilemap), @@ -545,16 +551,16 @@ fn ta_registers( RegisterWrite::new(0x1c898, 0), RegisterWrite::new(0x101e1, UNK_101D9), RegisterWrite::new(0x1c9e8, w.layer_mode & LAYER_MODE_PIPE_MASK), - RegisterWrite::new(0x1a099, 0), - RegisterWrite::new(0x1a0a1, 0), - RegisterWrite::new(0x1a069, 0), - RegisterWrite::new(0x1a071, 0), - RegisterWrite::new(0x1a0c9, 0), - RegisterWrite::new(0x1a0d1, 0), - RegisterWrite::new(0x101c9, 0), - RegisterWrite::new(0x0d471, 0), - RegisterWrite::new(0x1a0f1, UNK_1A0E9), - RegisterWrite::new(0x10799, UNK_10791), + RegisterWrite::new(0x1a099, ift[0]), + RegisterWrite::new(0x1a0a1, ift[1]), + RegisterWrite::new(0x1a069, ift[2]), + RegisterWrite::new(0x1a071, ift[3]), + RegisterWrite::new(0x1a0c9, ift[4]), + RegisterWrite::new(0x1a0d1, ift[5]), + RegisterWrite::new(0x101c9, rt.scratch.size_word()), + RegisterWrite::new(0x0d471, rt.scratch.tail_word()), + RegisterWrite::new(0x1a0f1, rt.scratch.stack_word()), + RegisterWrite::new(0x10799, rt.scratch.window_word()), RegisterWrite::new(0x1c830, r.pb_slot as u64), RegisterWrite::new(0x1ca30, r.pm_scratch), RegisterWrite::new(0x16c39, r.pm_scratch), @@ -579,7 +585,7 @@ fn fragment_registers( args: &RenderArgs<'_>, g: &Geometry, w: &PassWords, -) -> [RegisterWrite; FRAGMENT_REGISTERS] { +) -> [RegisterWrite; FRAGMENT_REGISTERS_MAX] { let p = args.pass; let r = args.resources; let mut tile_state = TILE_STATE_BASE @@ -591,9 +597,12 @@ fn fragment_registers( } let key = work_key(r.context_id, args.fragment.state_word); let ids = r.fragment_ids.word(); + let rt = args.ray_query[1]; + let ift = rt.intersection.words(); [ RegisterWrite::new(0x01739, 1), + RegisterWrite::new(0x017d9, 1), RegisterWrite::new(0x10009, w.utile_config), // End-of-tile resource specifier. RegisterWrite::new(0x15379, w.eot_rsrc_spec), @@ -611,7 +620,7 @@ fn fragment_registers( RegisterWrite::new(0x16030, g.macro_size), RegisterWrite::new(0x100d9, g.screen), RegisterWrite::new(0x0a301, 0), - RegisterWrite::new(0x10791, UNK_10791_FRAGMENT), + RegisterWrite::new(0x10791, rt.scratch.window_word() | 1 << 9), RegisterWrite::new(0x16098, r.heapmeta_va), RegisterWrite::new(0x15109, p.scissor_va), RegisterWrite::new(0x15101, p.depth_bias_va), @@ -659,15 +668,15 @@ fn fragment_registers( RegisterWrite::new(0x101e9, UNK_101D9), RegisterWrite::new(0x160a8, 0), RegisterWrite::new(0x16068, tile_state), - RegisterWrite::new(0x1a0a9, 0), - RegisterWrite::new(0x1a0b1, 0), - RegisterWrite::new(0x1a079, 0), - RegisterWrite::new(0x1a081, 0), - RegisterWrite::new(0x1a0d9, 0), - RegisterWrite::new(0x1a0e1, 0), - RegisterWrite::new(0x101c1, 0), - RegisterWrite::new(0x0d469, 0), - RegisterWrite::new(0x1a0f9, UNK_1A0E9), + RegisterWrite::new(0x1a0a9, ift[0]), + RegisterWrite::new(0x1a0b1, ift[1]), + RegisterWrite::new(0x1a079, ift[2]), + RegisterWrite::new(0x1a081, ift[3]), + RegisterWrite::new(0x1a0d9, ift[4]), + RegisterWrite::new(0x1a0e1, ift[5]), + RegisterWrite::new(0x101c1, rt.scratch.size_word()), + RegisterWrite::new(0x0d469, rt.scratch.tail_word()), + RegisterWrite::new(0x1a0f9, rt.scratch.stack_word()), RegisterWrite::new(0x0a5a9, UNK_0A599), RegisterWrite::new(0x0d429, UNK_0D411), RegisterWrite::new(0x160e0, ids), @@ -907,9 +916,16 @@ impl TaDescriptor { self.pb_va = resources.pb_va; self.scene_va = resources.scene_va; self.scene_end_va = resources.scene_end_va; - self.checkpoints = TA_CHECKPOINTS; + let scs_register = args.ray_query[0].scratch.enabled(); + self.checkpoints = TA_CHECKPOINTS.map(|count| count + u16::from(scs_register)); self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - self.registers.set(array_va, ®isters)?; + if scs_register { + self.registers.set(array_va, ®isters)?; + } else { + let mut trimmed = [registers[0]; TA_REGISTERS]; + trimmed[TA_SCS_ENABLE_INDEX..].copy_from_slice(®isters[TA_SCS_ENABLE_INDEX + 1..]); + self.registers.set(array_va, &trimmed)?; + } self.tpc_va = resources.tpc_va; self.tpc_size = resources.tpc_size; self.fragment_qid = args.fragment.qid.into(); @@ -963,10 +979,11 @@ impl TaDescriptor { /// descriptor's address as seen by the pass's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scs_register: bool, ) -> Result<[Option; 4]> { let array = RegisterArrayBinding::new( offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8, + TA_CHECKPOINTS[0] as u8 + u8::from(scs_register), )?; Ok([Some(array), None, None, None]) } @@ -1153,13 +1170,14 @@ impl FragmentDescriptor { /// the descriptor's address as seen by the pass's VM. pub(crate) fn register_bindings( descriptor_va: u64, + scs_register: bool, ) -> Result<[Option; 4]> { let array = |index, checkpoint: usize| -> Result> { let va = offset_va(descriptor_va, Self::array_offset(index))?; Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) }; Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize)?, + array(0, FRAGMENT_CHECKPOINTS[0] as usize + usize::from(scs_register))?, array(1, PARTIAL_STORE_REGISTERS)?, array(2, PARTIAL_RESUME_REGISTERS)?, array(3, PARTIAL_LOAD_REGISTERS)?, @@ -1226,9 +1244,19 @@ impl FragmentSlot { desc.merge_upper_x = pass.merge_upper_x; desc.merge_upper_y = pass.merge_upper_y; desc.tiles = geometry.tiles_x * geometry.tiles_y; - desc.checkpoints = FRAGMENT_CHECKPOINTS; - desc.entries = FRAGMENT_REGISTERS as u16; - desc.arrays[0].set(arrays[0], &fragment_registers(args, &geometry, &words))?; + let scs_register = args.ray_query[1].scratch.enabled(); + let extra = u16::from(scs_register); + desc.checkpoints = FRAGMENT_CHECKPOINTS.map(|count| count + extra); + desc.entries = FRAGMENT_REGISTERS as u16 + extra; + let registers = fragment_registers(args, &geometry, &words); + if scs_register { + desc.arrays[0].set(arrays[0], ®isters)?; + } else { + let mut trimmed = [registers[0]; FRAGMENT_REGISTERS]; + trimmed[FRAGMENT_SCS_ENABLE_INDEX..] + .copy_from_slice(®isters[FRAGMENT_SCS_ENABLE_INDEX + 1..]); + desc.arrays[0].set(arrays[0], &trimmed)?; + } desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; desc.arrays[3].set(arrays[3], &partial_load_registers(pass, &words))?; diff --git a/drivers/gpu/drm/asahi/g17/fw/render_rt.rs b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs new file mode 100644 index 00000000000000..5ae23d84400eb0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g17/fw/render_rt.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +use super::{compute::{IntersectionTables, ScratchRequest}, compute_pool, + freelist::{PAGE_LIST_SIZE, RENDER_BLOCKS}, + initdata::{FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}}; +use kernel::prelude::*; + +#[derive(Clone, Copy)] +pub(crate) struct RayQueryStage { + pub(crate) intersection: IntersectionTables, + pub(crate) scratch: ScratchRequest, +} + +impl RayQueryStage { + fn parse(bytes: &[u8]) -> Result { + let mut wire = [0u8; 56]; + wire[..bytes.len()].copy_from_slice(bytes); + let layout = u64::from_le_bytes(wire[..8].try_into().unwrap()); + let intersection = IntersectionTables::parse(&wire[8..])?; + let scratch = ScratchRequest::new(layout)?; + if scratch.enabled() != intersection.enabled() { return Err(EINVAL); } + Ok(Self { intersection, scratch }) + } + + pub(crate) fn parse_pair(bytes: &[u8]) -> Result<[Self; 2]> { + if bytes.get(112..).is_some_and(|tail| tail.iter().any(|&b| b != 0)) { + return Err(EINVAL); + } + let mut wire = [0u8; 112]; + let count = bytes.len().min(wire.len()); + wire[..count].copy_from_slice(&bytes[..count]); + Ok([Self::parse(&wire[..56])?, Self::parse(&wire[56..])?]) + } +} + +pub(crate) fn required_blocks(active_cores: u32, pages: [u16; 2]) -> Result { + let extra = compute_pool::extra_bytes(active_cores, pages[0])? + .checked_add(compute_pool::extra_bytes(active_cores, pages[1])?) + .and_then(|bytes| bytes.checked_mul(crate::hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u64)) + .ok_or(EOVERFLOW)?; + let bytes = extra.checked_add((RENDER_BLOCKS * FREE_LIST_BLOCK_SIZE) as u64) + .ok_or(EOVERFLOW)?; + let blocks = usize::try_from(bytes.div_ceil(FREE_LIST_BLOCK_SIZE as u64)) + .map_err(|_| EOVERFLOW)?; + let maximum = PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8); + if blocks > maximum { return Err(E2BIG); } + Ok(blocks) +} diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9ad0370edfa2fe..9293c83c55b613 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -244,6 +244,10 @@ pub(crate) trait Backend: Send + Sync { /// Lazily create or rebind this logical queue's physical compute graph. fn ensure_compute(&self, scs_pages: u16) -> Result; + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { + if pages != [0; 2] { return Err(EINVAL); } + Ok(super::fw::freelist::RENDER_BLOCKS) + } /// Settle outstanding work of a terminal VM before an ioctl error is published. fn fail_vm(&self, error: Error); /// Called after both scheduler entities have cancelled their queued jobs. diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index aae466429e0078..355cf9c5a83119 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -1103,7 +1103,8 @@ impl gpu::Gpu for Gpu { uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64 | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64; + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY as u64; params.gpu_generation = hw::GpuGen::G17 as u32; params.gpu_variant = self.cfg.identity.gpu_variant as u32; params.gpu_revision = self.id.gpu_rev as u32; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index f3f050d482bef6..b78598cfbb7609 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1678,7 +1678,7 @@ impl Queue { mcache: None, event_mask: COMPUTE_KICK_EVENT_MASK, register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, - compute_scratch: scratch.enabled(), + scs_enabled: scratch.enabled(), priority: policy.priority(), }; let published = host.next_compute_publication()?; diff --git a/drivers/gpu/drm/asahi/g17/queue/mod.rs b/drivers/gpu/drm/asahi/g17/queue/mod.rs index 31f2fcd6d016a9..326405f05fdf90 100644 --- a/drivers/gpu/drm/asahi/g17/queue/mod.rs +++ b/drivers/gpu/drm/asahi/g17/queue/mod.rs @@ -171,6 +171,10 @@ impl Queue { Validated::Compute { .. } => Engine::Compute, }; let index = engine.index(); + if let Validated::Render { ray_query, .. } = &command { + let blocks = self.backend.render_scs_blocks(ray_query.map(|stage| stage.scratch.pages()))?; + if blocks > self.context.render_blocks() { return Err(EINVAL); } + } self.prune_frontier(index); self.frontiers[index].reserve(1, GFP_KERNEL)?; if engine == Engine::Compute { @@ -354,6 +358,13 @@ impl Queue { } impl crate::queue::Queue for Queue { + fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { + let pages = [super::fw::compute::ScratchRequest::new(layouts[0])?.pages(), + super::fw::compute::ScratchRequest::new(layouts[1])?.pages()]; + let blocks = self.backend.render_scs_blocks(pages)?; + self.context.reserve_render_blocks(blocks) + } + fn submit( &mut self, id: u64, diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index bbf85695970a04..99be455b48e402 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -321,7 +321,7 @@ impl Pair { return Err(EBUSY); } let Validated::Render { - pass, attachments, .. + pass, attachments, ray_query, .. } = &packet.command else { return Err(EINVAL); @@ -385,6 +385,7 @@ impl Pair { let qids = self.qids(); let args = abi::RenderArgs { pass, + ray_query: *ray_query, resources: &resources, ordinal, ta: abi::RenderStage { @@ -585,7 +586,7 @@ impl Pair { self.memory.graph.queues.gpu_va() + u64::from(qid) * size_of::() as u64 }); - let Validated::Render { pass, .. } = &packet.command else { + let Validated::Render { pass, ray_query, .. } = &packet.command else { return Err(EINVAL); }; let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; @@ -609,13 +610,16 @@ impl Pair { mcache, event_mask: abi::RENDER_KICK_EVENT_MASK, register_arrays: arrays, - compute_scratch: false, + scs_enabled: ray_query[stage].scratch.enabled(), priority: policy.priority(), }; let ta_args = args( 0, tiling_barriers, - abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, + abi::TaDescriptor::register_bindings( + self.memory.graph.descriptor_client(0, ordinal), + ray_query[0].scratch.enabled(), + )?, None, ); let fragment_args = args( @@ -623,6 +627,7 @@ impl Pair { &barriers.fragment[..barriers.fragment_count], abi::FragmentDescriptor::register_bindings( self.memory.graph.descriptor_client(1, ordinal), + ray_query[1].scratch.enabled(), )?, mcache, ); diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index 36b075b4521270..c36c0add4f87ae 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -70,6 +70,10 @@ impl job::Backend for Backend { Some(self.shared.feed.clone()) } + fn render_scs_blocks(&self, pages: [u16; 2]) -> Result { + crate::g17::fw::render_rt::required_blocks(self.shared.active_cores, pages) + } + fn ensure_compute(&self, scs_pages: u16) -> Result { Backend::ensure_compute(self, scs_pages) } diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 46c55e22b74dc0..1ea9860b193d33 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -61,6 +61,11 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { + fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { + if layouts != [0; 2] { return Err(EINVAL); } + Ok(()) + } + fn submit( &mut self, id: u64, diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 2fae6bb473b763..7e858f0be4ad4f 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -418,6 +418,12 @@ struct drm_asahi_queue_limits { * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. */ DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, + /** + * @DRM_ASAHI_FEATURE_RENDER_RAY_QUERY: Render commands accept per-stage + * ray query state (struct drm_asahi_render_ray_query_stage) and queues + * accept render_scs_layout. + */ + DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 8, }; /** @@ -947,6 +953,13 @@ struct drm_asahi_queue_create { * fix a 4GiB VA carveout for USC memory and pass its base address here. */ __u64 usc_exec_base; + + /** + * @render_scs_layout: Largest ray query scratch layouts of the vertex and + * fragment stages this queue will submit, used to size its render pool. + * Requires DRM_ASAHI_FEATURE_RENDER_RAY_QUERY; zero otherwise. + */ + __u64 render_scs_layout[2]; }; /** @@ -1374,6 +1387,42 @@ struct drm_asahi_bg_eot { __u32 rsrc_spec; }; +/** + * struct drm_asahi_render_ray_query_stage - Ray query state of one render stage + * + * Index 0 describes the vertex (tiling) stage and index 1 the fragment stage + * of a render command. An all-zero entry disables ray queries in that stage. + * Requires DRM_ASAHI_FEATURE_RENDER_RAY_QUERY. + */ +struct drm_asahi_render_ray_query_stage { + /** @scs_layout: Per-thread ray query scratch layout, as for compute. */ + __u64 scs_layout; + + /** @intersection_aux: Auxiliary intersection table word. */ + __u32 intersection_aux; + + /** @intersection_pad: MBZ */ + __u32 intersection_pad; + + /** @intersection_pool: GPU VA of the intersection program pool. */ + __u64 intersection_pool; + + /** @intersection_pool_bytes: Size of the intersection program pool. */ + __u64 intersection_pool_bytes; + + /** @intersection_first: First intersection table entry. */ + __u32 intersection_first; + + /** @intersection_count: Number of intersection table entries. */ + __u32 intersection_count; + + /** @intersection_default: GPU VA of the default intersection program. */ + __u64 intersection_default; + + /** @intersection_default_control: Control word of the default program. */ + __u64 intersection_default_control; +}; + /** * struct drm_asahi_cmd_render - Command to submit 3D * @@ -1585,6 +1634,12 @@ struct drm_asahi_cmd_render { * queue's compute command count and the header's cdm_barrier instead. */ __u64 prior_cdm; + + /** + * @ray_query: Ray query state of the vertex and fragment stages. Older + * userspace omits it; a shorter command is read with both stages zeroed. + */ + struct drm_asahi_render_ray_query_stage ray_query[2]; }; /** From 5c99f69765dd842bbac75d5ea5f7b7607f4e6b7a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:56:30 +1100 Subject: [PATCH 463/631] usb: typec: tipd: Tidy up the CD321x additions to tps6598x.h The CD321x support placed the mux handle and its remove flag ahead of the device pointer in struct tps6598x, and included cd321x-pm.h in the middle of the header between a register define and the bit fields. Keep the includes together at the top, document why TPS_REG_MODE lives in the header, and move the mux handle next to the other port objects. Describe what remove_safe_prepared actually means instead of the previous shorthand. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/tps6598x.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index c47556384a4383..1d0d22f3545996 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -15,12 +15,14 @@ #include #include +#include "cd321x-pm.h" + #ifndef __TPS6598X_H__ #define __TPS6598X_H__ #include "cd321x-pm.h" +/* Also read by the SPMI glue to tell when the controller has woken up */ #define TPS_REG_MODE 0x03 -#include "cd321x-pm.h" #define TPS_FIELD_GET(_mask, _reg) ((typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask))) @@ -320,9 +322,6 @@ struct tipd_data { }; struct tps6598x { - struct typec_mux *mux; - /* CD remove leaves one open SAFE transaction for generic completion. */ - bool remove_safe_prepared; struct device *dev; struct regmap *regmap; struct mutex lock; /* device lock */ @@ -336,6 +335,12 @@ struct tps6598x { struct usb_pd_identity partner_identity; struct usb_role_switch *role_sw; struct typec_capability typec_cap; + struct typec_mux *mux; + /* + * Set by ->remove() once it has parked the mux in SAFE, so that + * tps6598x_disconnect() only ends that mux transaction. + */ + bool remove_safe_prepared; struct power_supply *psy; struct power_supply_desc psy_desc; From fe935b239876e85a6e7e3c76871d9f349c65eeb7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:57:02 +1100 Subject: [PATCH 464/631] usb: typec: tipd: Fix coding style in the CD321x update path Fix the issues checkpatch --strict reports in code added for CD321x: misaligned continuation lines, two lines over 100 columns, an else branch after a goto, and a double blank line. Reword the comment on the shared display connector, which used double spaces and repeated itself, and the comment on the SAFE transaction left open by cd321x_remove(). Move the DRM include below the linux/ ones, where the DisplayPort Alt Mode driver has it, instead of ahead of them. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/core.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index a431cbe431ffba..6b5e567eece341 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -6,8 +6,6 @@ * Author: Heikki Krogerus */ -#include - #include #include #include @@ -22,6 +20,7 @@ #include #include #include +#include #include "tps6598x.h" #include "cd321x-setup.h" @@ -934,21 +933,19 @@ static void cd321x_update_work(struct work_struct *work) } /* - * Report sink absence before any fallible hardware transition. A failed - * shutdown retains the route/PHY owner, but cannot keep the sink present. - * Every Type-C port references the same external DCP connector. Only the - * port which successfully acquired the shared display route may report a - * disconnect. A second DP sink can negotiate Alt Mode even though its mux - * request is rejected with -EBUSY; treating that sink as the connector - * owner would tear down the display carried by the first port. + * Report the sink as gone before any hardware transition that can fail: + * a failed teardown keeps the route, but the sink must not stay present. + * All ports reference the same external display connector, so only the + * port that owns the display route may report a disconnect. A second DP + * sink can enter Alt Mode while its mux request fails with -EBUSY, and + * must not tear down the display carried by the other port. */ if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && dp_route_was_active && (!new_connected || was_disconnected || !dp_connected || !dp_hpd || dp_hpd_changed || dp_mode_changed)) { drm_connector_oob_hotplug_event(cd321x->connector_fwnode, - connector_status_disconnected); - } + connector_status_disconnected); /* * ACIO carries the tunneled PCIe reset handshake over the still-live @@ -2297,12 +2294,11 @@ static void cd321x_remove(struct tps6598x *tps) typec_mux_abort(tps->mux); return ret; } - /* Same task/handle completes without another post-shutdown set callback. */ + /* Leave the SAFE transaction open for tps6598x_disconnect() to end. */ tps->remove_safe_prepared = true; return 0; } - int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; From 1defdad684249cf4cbd261818247479407c23e9e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:59:58 +1100 Subject: [PATCH 465/631] usb: typec: tipd: Use guard() for the locks in the debugfs interface core.c takes tps->lock with guard() and scoped_guard(), but the debugfs interface added for CD321x open-codes mutex_lock()/mutex_unlock() and needs a goto label to unlock on the -EBUSY path. Use the same guards there, which removes the label and the extra return-value juggling. The critical sections are unchanged: user copies still happen outside tps->lock except for the read of the saved DATA1 result, as before. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/typec/tipd/debugfs.c | 56 +++++++++++++------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c index 5a8d576eff0dc6..8f35394624fe69 100644 --- a/drivers/usb/typec/tipd/debugfs.c +++ b/drivers/usb/typec/tipd/debugfs.c @@ -122,10 +122,10 @@ static int tipd_debugfs_command_release(struct inode *inode, struct file *file) * may be gone. tipd_debugfs_unregister() has then left "DBMa" itself. */ if (f->holds_dbma && !debugfs_file_get(dentry)) { - mutex_lock(&tps->lock); - if (!--tps->debugfs->dbma_holders) - tipd_debugfs_leave_dbma(tps); - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) { + if (!--tps->debugfs->dbma_holders) + tipd_debugfs_leave_dbma(tps); + } debugfs_file_put(dentry); } @@ -166,17 +166,15 @@ static ssize_t tipd_debugfs_command_write(struct file *file, if (!memcmp(cmd->cmd, "DBMa", 4) && buf[4] > 1) return -EINVAL; - mutex_lock(&tps->lock); + guard(mutex)(&tps->lock); /* Preserve inherited mode and the leases held by other open files. */ if ((!memcmp(cmd->cmd, "DBMa", 4) && !buf[4] && (tps->dbma_at_probe || tps->debugfs->dbma_holders > (unsigned int)f->holds_dbma)) || (!memcmp(cmd->cmd, "LOCK", 4) && !memchr_inv(buf + 4, 0, 4) && - (tps->dbma_at_probe || tps->debugfs->dbma_holders))) { - ret = -EBUSY; - goto out; - } + (tps->dbma_at_probe || tps->debugfs->dbma_holders))) + return -EBUSY; ret = tps6598x_exec_cmd_tmo(tps, cmd->cmd, len, len ? &buf[4] : NULL, sizeof(out), out, cmd->timeout_ms, 0); @@ -198,27 +196,20 @@ static ssize_t tipd_debugfs_command_write(struct file *file, } } -out: - mutex_unlock(&tps->lock); - - return ret ? ret : count; + return ret ?: count; } static ssize_t tipd_debugfs_command_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { struct tipd_debugfs_file *f = file->private_data; - ssize_t ret; - mutex_lock(&f->tps->lock); + guard(mutex)(&f->tps->lock); if (!f->have_result) - ret = -ENODATA; - else - ret = simple_read_from_buffer(ubuf, count, ppos, f->result, - sizeof(f->result)); - mutex_unlock(&f->tps->lock); + return -ENODATA; - return ret; + return simple_read_from_buffer(ubuf, count, ppos, f->result, + sizeof(f->result)); } static const struct file_operations tipd_debugfs_command_fops = { @@ -244,9 +235,8 @@ static ssize_t tipd_debugfs_regs_read(struct file *file, char __user *ubuf, if (count > sizeof(buf)) return -EINVAL; - mutex_lock(&tps->lock); - ret = tps6598x_block_read(tps, *ppos, buf, count); - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) + ret = tps6598x_block_read(tps, *ppos, buf, count); if (ret) return ret; @@ -268,10 +258,10 @@ void tipd_debugfs_register(struct tps6598x *tps) { struct tipd_debugfs *dbg; - mutex_lock(&tipd_debugfs_root_lock); - if (!tipd_debugfs_root) - tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); - mutex_unlock(&tipd_debugfs_root_lock); + scoped_guard(mutex, &tipd_debugfs_root_lock) { + if (!tipd_debugfs_root) + tipd_debugfs_root = debugfs_create_dir("tipd", usb_debug_root); + } dbg = kzalloc_obj(*dbg); if (!dbg) @@ -295,11 +285,11 @@ void tipd_debugfs_unregister(struct tps6598x *tps) /* Waits for running file operations, later ones fail with -EIO */ debugfs_remove_recursive(dbg->dentry); - mutex_lock(&tps->lock); - if (dbg->dbma_holders) - tipd_debugfs_leave_dbma(tps); - tps->debugfs = NULL; - mutex_unlock(&tps->lock); + scoped_guard(mutex, &tps->lock) { + if (dbg->dbma_holders) + tipd_debugfs_leave_dbma(tps); + tps->debugfs = NULL; + } kfree(dbg); } From 643d21ace4b9741de51493c5db412e6c60489d10 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:09:59 +1100 Subject: [PATCH 466/631] HID: dockchannel: Drop reports that arrive before the device is open The MTP firmware starts sending reports for an interface as soon as the host enables it. dchid_create_interface_work() enables the interface first and only then allocates and registers the HID device, while the reports are handled on the interface's own workqueue. The keyboard sends its first report in that window on every boot, so the driver logs Report received but keyboard is not initialized! and reads iface->hid while the create work is still writing it. A report that arrives before the HID device is open has nobody to go to either way: the existing open check already dropped it once the device was registered. Make the open flag the only gate, so this expected state no longer warns, and use release/acquire on it so a reader that sees the device open also sees iface->hid. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index bd529afd3d4a80..ce82e25ce052bc 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -675,7 +675,8 @@ static int dchid_open(struct hid_device *hdev) } } - iface->open = true; + /* Pairs with the acquire in dchid_handle_report(); publishes iface->hid. */ + smp_store_release(&iface->open, true); return 0; } @@ -683,7 +684,7 @@ static void dchid_close(struct hid_device *hdev) { struct dchid_iface *iface = hdev->driver_data; - iface->open = false; + WRITE_ONCE(iface->open, false); } static int dchid_parse(struct hid_device *hdev) @@ -1063,14 +1064,12 @@ static void dchid_handle_event(struct dockchannel_hid *dchid, void *data, size_t static void dchid_handle_report(struct dchid_iface *iface, void *data, size_t length) { - struct dockchannel_hid *dchid = iface->dchid; - - if (!iface->hid) { - dev_warn(dchid->dev, "Report received but %s is not initialized!\n", iface->name); - return; - } - - if (!iface->open) + /* + * An interface starts reporting as soon as it is enabled, which is + * before its HID device has been registered and opened. Until then + * there is nobody to deliver the reports to. + */ + if (!smp_load_acquire(&iface->open)) return; hid_input_report(iface->hid, HID_INPUT_REPORT, data, length, 1); From ec1fcb1f08cdb86debac83730ab20ed7027b95cf Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:13 +1100 Subject: [PATCH 467/631] soc: apple: rtkit: Handle oslog notifications quietly Besides the buffer request, RTKit firmware sends oslog messages of types 2 to 5 during normal operation. Types 3 to 5 arrive in a burst when SMC, DCP and AOP firmware boot, and were logged at info level, four or five lines per co-processor on every boot. Type 2 arrives once per boot from the AOP and the MTP, and was logged as an unknown message with a warning. None of them expects a reply: the co-processors keep running, and keyboard, trackpad and AOP sensors keep working after they arrive. The payload is not decoded, so the lines say nothing actionable. Log all four types at debug level and keep the warning for types that have not been seen. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index e5e112ea1f4b71..b78ae6c2afb02a 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -770,6 +770,7 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) msg, NULL, false); if (err) dev_err(rtk->dev, "RTKit: oslog acknowledgment failed: %d\n", err); + dev_dbg(rtk->dev, "RTKit: oslog notification: %016llx\n", msg); break; default: dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", From 481a77c146c2bdf631d709f433bdcaaf350995b5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:10:28 +1100 Subject: [PATCH 468/631] HID: dockchannel: Terminate log messages and fix two typos Eight messages lack a trailing newline, so the next message can be appended to the same console line, and two carry typos ("firmwareS", "expFected"). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hid/dockchannel-hid/dockchannel-hid.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index ce82e25ce052bc..6a5e412887082a 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -580,7 +580,7 @@ static int dchid_start_interface(struct dchid_iface *iface) int ret; if (iface->starting) { - dev_warn(iface->dchid->dev, "Interface %s is already starting", iface->name); + dev_warn(iface->dchid->dev, "Interface %s is already starting\n", iface->name); return -EINPROGRESS; } @@ -607,7 +607,7 @@ static int dchid_start_interface(struct dchid_iface *iface) dev_info(iface->dchid->dev, "Sending firmware for %s\n", iface->name); ret = dchid_send_firmware(iface, fw, size); if (ret < 0) { - dev_err(iface->dchid->dev, "Failed to send %s firmwareS", iface->name); + dev_err(iface->dchid->dev, "Failed to send %s firmware\n", iface->name); goto err; } @@ -640,7 +640,7 @@ static int dchid_start(struct hid_device *hdev) if (iface->keyboard_layout_id) { int ret = device_create_file(&hdev->dev, &dev_attr_apple_layout_id); if (ret) { - dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d", ret); + dev_warn(iface->dchid->dev, "Failed to create apple_layout_id: %d\n", ret); iface->keyboard_layout_id = 0; } } @@ -802,7 +802,7 @@ static void dchid_create_interface_work(struct work_struct *ws) if (ret < 0) { iface->hid = NULL; hid_destroy_device(hid); - dev_warn(iface->dchid->dev, "Failed to register hid device %s", iface->name); + dev_warn(iface->dchid->dev, "Failed to register hid device %s\n", iface->name); } } @@ -1133,7 +1133,7 @@ static void dchid_handle_ack(struct dchid_iface *iface, struct dchid_hdr *hdr, v } if (shdr->flags != iface->out_flags) { dev_err(iface->dchid->dev, - "Received unexpected flags 0x%x on ACK channel (expFected 0x%x)\n", + "Received unexpected flags 0x%x on ACK channel (expected 0x%x)\n", shdr->flags, iface->out_flags); return; } @@ -1344,14 +1344,14 @@ static int dockchannel_hid_probe(struct platform_device *pdev) */ helper = of_parse_phandle(dev->of_node, "apple,helper-cpu", 0); if (!helper) { - dev_err(dev, "Missing apple,helper-cpu property"); + dev_err(dev, "Missing apple,helper-cpu property\n"); return -EINVAL; } helper_pdev = of_find_device_by_node(helper); of_node_put(helper); if (!helper_pdev) { - dev_err(dev, "Failed to find helper device"); + dev_err(dev, "Failed to find helper device\n"); return -EINVAL; } @@ -1359,7 +1359,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) DL_FLAG_AUTOREMOVE_CONSUMER); put_device(&helper_pdev->dev); if (!dchid->helper_link) { - dev_err(dev, "Failed to link to helper device"); + dev_err(dev, "Failed to link to helper device\n"); return -EINVAL; } @@ -1388,7 +1388,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) dchid->comm = dchid_get_interface(dchid, IFACE_COMM, "comm"); if (!dchid->comm) { - dev_err(dchid->dev, "Failed to initialize comm interface"); + dev_err(dchid->dev, "Failed to initialize comm interface\n"); destroy_workqueue(dchid->new_iface_wq); return -EIO; } From a19692dd3937a366cb96b36d3f58f2bce8ba50a1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 14:08:53 +1100 Subject: [PATCH 469/631] rust: iosys_map: let the KUnit helper compare with assert_eq VecIoSysMap::assert_eq() uses assert_eq!, which needs T: Debug, but the helper only bounds T by AsBytes + FromBytes + Clone + PartialEq. With CONFIG_KUNIT=y the kernel crate then fails to build (E0277, "T doesn't implement Debug"), which takes every Rust KUnit suite down with it. Add the Debug bound to the helper struct and its impl. Assisted-by: LLM Signed-off-by: Ryan Murray --- rust/kernel/iosys_map.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/kernel/iosys_map.rs b/rust/kernel/iosys_map.rs index 3fad1c1aebbbc8..629f40de3dbf2c 100644 --- a/rust/kernel/iosys_map.rs +++ b/rust/kernel/iosys_map.rs @@ -536,7 +536,7 @@ pub mod tests { use super::*; /// A helper struct for managed IoSysMapRef structs which point to a [`Vec`]. - pub struct VecIoSysMap { + pub struct VecIoSysMap { map: RawIoSysMap, vec: KVec, } From 80c174b8a4a40cb34c143b4289154ec442df8de9 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 14:08:54 +1100 Subject: [PATCH 470/631] ASoC: apple: t8140-aop-audio: test the speaker ceiling with KUnit The copy-time conversion is the only digital limit between userspace and the J700 speaker amplifiers: the top of "Speaker Playback Volume" must be the amplifier's full scale (a float of 0.5 on the LEAP; above it the words wrap), and macaudio's lock level (87) must sit 20 dB below it. Nothing checked either outside the compile-time assertion on the table's top entry. Add a KUnit suite, snd_soc_t8140_aop_audio_speaker: - the top of the volume range is the amplifier full scale; - the table falls in 0.5 dB steps to mute; - the locked level is 20 dB below full scale; - no S32 sample at any volume converts above full scale (with the exact results for i32::MAX/MIN at full volume and at the lock); - the float conversion is exact on powers of two. kunit.py on arm64 qemu: 5/5 pass. With the conversion's shift changed from 31 to 30 (6 dB over the ceiling) the full-scale test fails. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 87 ++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index f726c88f0ea041..57dad8c960331e 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -2869,3 +2869,90 @@ module_platform_driver! { description: "Apple T8140 AOP audio: microphones, headphone jack and speakers", license: "Dual MIT/GPL", } + +/// The speaker ceiling and the copy-path conversion are the whole digital +/// protection of the J700 speakers below the daemon: check them where they +/// are built, without the hardware. +#[kunit_tests(snd_soc_t8140_aop_audio_speaker)] +mod tests { + use super::*; + + /// macaudio's level for the J700 while no daemon holds the lock: + /// MAX98360A_DB(safe_vol = -20). + const LOCKED_VOLUME: usize = 87; + /// 10^(0.5 / 20) in parts per 10^9. + const HALF_DB_PPB: u128 = 1_059_253_725; + /// 0.5 as a float32: the amplifier's full scale on the wire. + const F32_HALF: u32 = 0x3f00_0000; + + #[test] + fn the_top_of_the_volume_range_is_the_amplifier_full_scale() { + assert_eq!( + SPKR_VOLUME_Q31[SPKR_VOLUME_MAX as usize] as i64, + SPKR_CEILING_Q31 + ); + assert_eq!(SPKR_CEILING_Q31, 1 << 30); + assert_eq!(f32_bits_of_q31(SPKR_CEILING_Q31 as i32), F32_HALF); + } + + #[test] + fn the_volume_table_falls_in_half_db_steps_to_mute() { + assert_eq!(SPKR_VOLUME_Q31[0], 0); + for i in 2..=SPKR_VOLUME_MAX as usize { + let (lo, hi) = (SPKR_VOLUME_Q31[i - 1] as u128, SPKR_VOLUME_Q31[i] as u128); + assert!(hi > lo); + // hi / lo == 10^(0.5/20) to within the table's rounding + let expect = lo * HALF_DB_PPB / 1_000_000_000; + assert!(hi.abs_diff(expect) <= 2); + } + } + + #[test] + fn the_locked_level_is_20_db_below_full_scale() { + let locked = SPKR_VOLUME_Q31[LOCKED_VOLUME] as i64; + // 10 * locked == ceiling to within the table's rounding (0.1 ppm) + assert!((10 * locked - SPKR_CEILING_Q31).abs() <= 10); + } + + #[test] + fn no_sample_at_any_volume_exceeds_the_full_scale() { + for &gain in SPKR_VOLUME_Q31.iter() { + for sample in [ + i32::MAX, + i32::MIN, + i32::MAX - 1, + i32::MIN + 1, + 1 << 30, + -(1 << 30), + 1, + -1, + 0, + ] { + let bits = spkr_f32_bits(sample, gain); + assert!(bits & 0x7fff_ffff <= F32_HALF); + } + } + // the extremes at full volume: just under and exactly at full scale + assert_eq!( + spkr_f32_bits(i32::MAX, SPKR_CEILING_Q31 as u32), + 0x3eff_ffff + ); + assert_eq!( + spkr_f32_bits(i32::MIN, SPKR_CEILING_Q31 as u32), + 0x8000_0000 | F32_HALF + ); + // and 20 dB down at the locked level + let locked = spkr_f32_bits(i32::MIN, SPKR_VOLUME_Q31[LOCKED_VOLUME]); + // 0.05 as a float32 is 0x3d4c_cccd; the conversion truncates + assert!(locked & 0x7fff_ffff <= 0x3d4c_cccd && locked & 0x7fff_ffff > 0x3d4c_cccd - 64); + } + + #[test] + fn the_float_conversion_is_exact_on_powers_of_two() { + assert_eq!(f32_bits_of_q31(0), 0); + assert_eq!(f32_bits_of_q31(1 << 29), 0x3e80_0000); // 0.25 + assert_eq!(f32_bits_of_q31(-(1 << 30)), 0xbf00_0000); // -0.5 + assert_eq!(f32_bits_of_q31(1), 0x3000_0000); // 2^-31 + assert_eq!(f32_bits_of_q31(i32::MIN), 0xbf80_0000); // -1.0 + } +} From a5e31622c7f96895cd1ffb04a40d6b7cda283c43 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 08:41:03 +1100 Subject: [PATCH 471/631] soc: apple: recheck sensor status after SEP registration Registration can clear and authenticate the sensor session after the wake status was sampled. Read status again before trusting the patch acknowledgement, refuse unreadable or silent status, and send an explicit needs-patch state through the existing patch sequence. Preserve capture failures and enrolled identities. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 4441a94a285777..64c3b0ddcd53bc 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1946,8 +1946,18 @@ impl SepData { return None; } - // The patch decision is made on status[8], not on state 9. - if st.patch_ack() == sensor::PATCH_ACCEPTED { + // Registration can reset the sensor session. Do not reuse the + // acknowledgement sampled before CLEAR_STATE and authentication. + let st = match sensor::status() { + Ok(st) if !st.is_silent() => st, + _ => { + dev_warn!(self.dev, "sensor: status unavailable after registration\n"); + return None; + } + }; + if st.patch_ack() == sensor::PATCH_ACCEPTED + && st.state != sensor::STATE_NEEDS_PATCH + { return Some(PatchLoaded(())); } return self.apply_sensor_patch(st.patch_ack()); From 81b5893452f943d658b308dbe001b1e2d1386a07 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 08:47:09 +1100 Subject: [PATCH 472/631] tools: aurora-sep: exercise post-registration wake status Run the actual wake_sensor body with synthetic registration and status mocks. Check register-before-status ordering, stale accepted acknowledgement followed by needs-patch, unreadable and silent status, fresh accepted readiness, and missing acknowledgement. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/tests/test_sensor_wake.py | 96 ++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tools/aurora-sep/tests/test_sensor_wake.py diff --git a/tools/aurora-sep/tests/test_sensor_wake.py b/tools/aurora-sep/tests/test_sensor_wake.py new file mode 100644 index 00000000000000..c9bb8c6af75792 --- /dev/null +++ b/tools/aurora-sep/tests/test_sensor_wake.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: MIT +"""Execute the actual wake_sensor body with synthetic status/registration mocks.""" +from pathlib import Path +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +class SensorWakeTest(unittest.TestCase): + def test_post_registration_status_drives_patch_readiness(self): + source = (APPLE / 'sbio.rs').read_text() + start = source.index(' fn wake_sensor(') + opening = source.index('{', start) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + function = source[start:end] + program = r''' +#![allow(dead_code, unused_variables)] +use std::cell::RefCell; +macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_err { ($($args:tt)*) => {} } +mod kernel { pub mod time { + pub struct Delta; + impl Delta { pub fn from_millis(_:i64)->Self {Self} } + pub mod delay { pub fn fsleep(_:super::Delta) {} } +}} +#[derive(Clone, Copy)] struct PatchLoaded(()); +#[derive(Default)] struct Mock { + reads: usize, registered: bool, patch: Option, next: Option, +} +thread_local! { static MOCK: RefCell = RefCell::new(Mock::default()); } +mod sensor { + pub const POWER_ON_READ_DELAYS_MS: [u32;1] = [0]; + pub const KNOWN_IDENTIFIERS: &[u16] = &[1]; + pub const PATCH_ACCEPTED: u8 = 0x29; + pub const STATE_NEEDS_PATCH: u8 = 9; + #[derive(Clone,Copy)] pub struct Status { pub state:u8, pub ack:u8, pub silent:bool } + pub struct Identifier; + pub enum Offset12 { Identifier(u16), Other } + pub struct Timing; + impl Timing { pub fn is_hardware(&self)->bool {true} pub fn name(&self)->&'static str {"mock"} } + pub fn power_source() {} + pub fn power_cycle()->bool {true} + pub fn cs_timing()->Timing {Timing} + pub fn status()->Result { + super::MOCK.with(|m| { let mut m=m.borrow_mut(); m.reads+=1; + if m.reads==1 {return Ok(Status{state:0,ack:PATCH_ACCEPTED,silent:false})} + assert!(m.registered, "post-registration read preceded registration"); + m.next.ok_or(()) + }) + } + impl Status { + pub fn is_silent(&self)->bool {self.silent} + pub fn offset12(&self)->Offset12 {Offset12::Identifier(1)} + pub fn identifier_to_register(&self)->Option {Some(Identifier)} + pub fn patch_ack(&self)->u8 {self.ack} + } +} +struct Driver; +impl Driver { + fn register_sensor(&self,_:&sensor::Identifier)->bool { + MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,1);m.registered=true;}); true + } + fn apply_sensor_patch(&self,ack:u8)->Option { + MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,2);m.patch=Some(ack);}); + Some(PatchLoaded(())) + } +''' + program += function + '\n}\n' + r''' +fn main() { + for (next,ready,patch) in [ + (Some(sensor::Status{state:9,ack:0x29,silent:false}),true,Some(0x29)), + (None,false,None), + (Some(sensor::Status{state:0,ack:0x29,silent:true}),false,None), + (Some(sensor::Status{state:0,ack:0x29,silent:false}),true,None), + (Some(sensor::Status{state:0,ack:0,silent:false}),true,Some(0)), + ] { + MOCK.with(|m| *m.borrow_mut()=Mock{next,..Mock::default()}); + assert_eq!(Driver.wake_sensor().is_some(),ready); + MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,2);assert!(m.registered);assert_eq!(m.patch,patch);}); + } +} +''' + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From e97505336910e88d64f215acb21a56cd40d647c8 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:06:47 +1100 Subject: [PATCH 473/631] soc: apple: sep: serialize SBIO exchanges The capture work item and the ioctl path both talk to the SBIO endpoint, which has one out-of-line buffer and one reassembly state. GET_INFO and ENROL_START list the enclave's identities before waiting for the last work item, so libfprint cancelling an identify and starting the next one lands that request in the middle of the previous work item's sensor registration. The second caller is then refused with EBUSY, or, when it starts between a completed reply and its collection, clears that reply and the first caller takes the second caller's answer. Hold an exchange lock for the whole request and reply, as the key store already does with sks_exchange_lock. The receive path does not take it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 +++++ drivers/soc/apple/sep.rs | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 64c3b0ddcd53bc..ffa06b06a08b77 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2695,6 +2695,11 @@ impl SepData { name: &'static CStr, payload: &[u8], ) -> Result { + // The capture work item and the ioctl path both reach this endpoint. + // Without this, the second caller is refused with EBUSY, or clears a + // reply the first has not collected yet and takes the next one. Taken + // before the shutdown check, so a waiter never starts after remove(). + let _exchange = self.sbio_exchange_lock.lock(); if self.shutting_down.load(Relaxed) || !self.sbio_ready.load(Relaxed) { return Err(ENODEV); } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index f5b9048109a95d..5af1a76c70e254 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -595,6 +595,11 @@ struct SepData { #[pin] dma_ring: Mutex>, + // One SBIO request/reply exchange at a time: the endpoint has a single + // out-of-line buffer and `sbio_rx` tracks a single transfer. + #[pin] + sbio_exchange_lock: Mutex<()>, + #[pin] sbio_rx: Mutex, @@ -871,6 +876,7 @@ impl SepData { scrd_probe <- new_mutex!(ScrdProbe::new()), scrd_wq <- new_condvar!("SepData::scrd_wq"), dma_ring <- new_mutex!(Some(dma_ring)), + sbio_exchange_lock <- new_mutex!(()), sbio_rx <- new_mutex!(transfer::Reassembly::new()), sbio_wq <- new_condvar!("SepData::sbio_wq"), sbio_ready: Atomic::new(false), From ee71f5e24721d6a3099f18a0f3b1a5b62bb5ddbd Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:06:48 +1100 Subject: [PATCH 474/631] tools: aurora-sep: run concurrent SBIO exchanges Compile the driver's sbio_transfer_raw against a threaded enclave mock and run a work-item exchange and an ioctl exchange at the same time. Each must get its own reply. Without the exchange lock the ioctl side is refused with EBUSY in every round. Assisted-by: LLM Signed-off-by: Ryan Murray --- tools/aurora-sep/tests/test_sbio_exchange.py | 174 +++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tools/aurora-sep/tests/test_sbio_exchange.py diff --git a/tools/aurora-sep/tests/test_sbio_exchange.py b/tools/aurora-sep/tests/test_sbio_exchange.py new file mode 100644 index 00000000000000..177246001f7593 --- /dev/null +++ b/tools/aurora-sep/tests/test_sbio_exchange.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: MIT +"""Run the driver's actual SBIO transfer method from two threads at once. + +The verify/enrol work item and the ioctl path (GET_INFO and ENROL_START list +the enclave's identities, DELETE removes one) can reach the single SBIO +endpoint at the same time, e.g. when libfprint cancels an identify and starts +the next one while the previous work item is still registering the sensor. +Each exchange must complete with its own reply; none may be refused with +EBUSY or receive another caller's answer. +""" +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +APPLE = Path(__file__).resolve().parents[3] / 'drivers/soc/apple' + + +def method(file, name): + source = (APPLE / file).read_text() + match = re.search(r'^ (?:pub\(crate\) )?fn ' + name + r'\(', source, re.M) + if match is None: + raise AssertionError(f'Missing driver method {name}') + opening = source.index('{', match.start()) + depth, end = 1, opening + 1 + while depth: + depth += (source[end] == '{') - (source[end] == '}') + end += 1 + return source[match.start():end] + + +HARNESS = r''' +#![allow(dead_code, unused_variables, unused_imports)] +use std::sync::atomic::{AtomicBool, Ordering::Relaxed}; +use std::sync::mpsc; +use std::time::Duration; + +type CStr = str; +type Result = std::result::Result; +const ENODEV: i32 = -19; +const EBUSY: i32 = -16; +const ETIMEDOUT: i32 = -110; +const SBIO_TIMEOUT_MS: u64 = 2000; + +mod time { pub fn msecs_to_jiffies(ms: u64) -> u64 { ms } } + +pub enum CondVarTimeoutResult { Woken { jiffies: u64 }, Signal { jiffies: u64 }, Timeout } + +/// Kernel-shaped mutex: lock() returns a guard a CondVar can wait on. +struct Mutex(std::sync::Mutex); +struct Guard<'a, T>(Option>); +impl Mutex { + fn new(v: T) -> Self { Mutex(std::sync::Mutex::new(v)) } + fn lock(&self) -> Guard<'_, T> { Guard(Some(self.0.lock().unwrap())) } +} +impl<'a, T> std::ops::Deref for Guard<'a, T> { + type Target = T; + fn deref(&self) -> &T { self.0.as_ref().unwrap() } +} +impl<'a, T> std::ops::DerefMut for Guard<'a, T> { + fn deref_mut(&mut self) -> &mut T { self.0.as_mut().unwrap() } +} +struct CondVar(std::sync::Condvar); +impl CondVar { + fn wait_interruptible_timeout(&self, g: &mut Guard<'_, T>, j: u64) -> CondVarTimeoutResult { + let inner = g.0.take().unwrap(); + let (inner, r) = self.0.wait_timeout(inner, Duration::from_millis(j)).unwrap(); + g.0 = Some(inner); + if r.timed_out() { CondVarTimeoutResult::Timeout } else { CondVarTimeoutResult::Woken { jiffies: j / 2 } } + } + fn notify_all(&self) { self.0.notify_all() } +} + +mod transfer { + pub struct Completed { pub status: u32, pub payload: Vec } + /// Same state rules as drivers/soc/apple/transfer.rs Reassembly. + pub struct Reassembly { pub active: Option, pub done: Option } + impl Reassembly { + pub fn begin(&mut self, opcode: u32) -> super::Result<()> { + if self.active.is_some() { return Err(super::EBUSY); } + self.done = None; + self.active = Some(opcode); + Ok(()) + } + pub fn take_done(&mut self) -> Option { self.done.take() } + pub fn abort(&mut self) { + if self.active.take().is_some() { + self.done = Some(Completed { status: 0xdead, payload: Vec::new() }); + } + } + /// The enclave's reply: it echoes the opcode in the payload. + pub fn on_reply(&mut self, opcode: u32) { + if self.active == Some(opcode) { + self.active = None; + self.done = Some(Completed { status: 0, payload: opcode.to_le_bytes().to_vec() }); + } + } + } +} + +struct SepData { + shutting_down: AtomicBool, + sbio_ready: AtomicBool, + sbio_rx: Mutex, + sbio_wq: CondVar, + sbio_exchange_lock: Mutex<()>, + to_enclave: std::sync::Mutex>, +} + +impl SepData { + fn sbio_start(&self, opcode: u16, _name: &'static CStr, _payload: &[u8]) -> Result<()> { + self.to_enclave.lock().unwrap().send(u32::from(opcode)).unwrap(); + Ok(()) + } +METHOD +} + +fn main() { + let (tx, rx) = mpsc::channel::(); + let sep: &'static SepData = Box::leak(Box::new(SepData { + shutting_down: AtomicBool::new(false), + sbio_ready: AtomicBool::new(true), + sbio_rx: Mutex::new(transfer::Reassembly { active: None, done: None }), + sbio_wq: CondVar(std::sync::Condvar::new()), + sbio_exchange_lock: Mutex::new(()), + to_enclave: std::sync::Mutex::new(tx), + })); + // The enclave answers each request after a short delay, as the mailbox does. + std::thread::spawn(move || { + for opcode in rx { + std::thread::sleep(Duration::from_millis(30)); + sep.sbio_rx.lock().on_reply(opcode); + sep.sbio_wq.notify_all(); + } + }); + let mut failures = 0; + for round in 0..20u16 { + let worker = std::thread::spawn(move || { + // e.g. CLEAR_STATE from the work item's sensor registration + sep.sbio_transfer_raw(0x10 + round, "WORKER", &[]) + .map(|d| (d.status, d.payload)) + }); + std::thread::sleep(Duration::from_millis(5)); + // e.g. LIST_IDENTITIES from GET_INFO on the next identify + let ioctl = sep.sbio_transfer_raw(0x40 + round, "IOCTL", &[]).map(|d| (d.status, d.payload)); + let worker = worker.join().unwrap(); + let want_w = Ok((0, u32::from(0x10 + round).to_le_bytes().to_vec())); + let want_i = Ok((0, u32::from(0x40 + round).to_le_bytes().to_vec())); + if worker != want_w || ioctl != want_i { + eprintln!("round {round}: worker {:?} ioctl {:?}", worker, ioctl); + failures += 1; + } + } + assert_eq!(failures, 0, "concurrent SBIO exchanges interfered"); +} +''' + + +@unittest.skipUnless(shutil.which('rustc'), 'rustc required for driver method tests') +class SbioExchangeTest(unittest.TestCase): + def test_concurrent_exchanges_each_get_their_own_reply(self): + program = HARNESS.replace('METHOD', method('sbio.rs', 'sbio_transfer_raw')) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'test.rs').write_text(program) + subprocess.run(['rustc', '--edition=2021', str(root / 'test.rs'), + '-o', str(root / 'test')], check=True) + subprocess.run([str(root / 'test')], check=True) + + +if __name__ == '__main__': + unittest.main() From 24cb82ff129e3d3d2ebe564a5fd454d80f9cfe6d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 11:55:26 +1100 Subject: [PATCH 475/631] drm/apple: Resume a CRTC whose mode needs a new link disabled Every resume with the external display at 1920x1080@180 logged [drm:drm_mode_config_helper_resume] *ERROR* Failed to resume (-22) The 180 Hz mode is admitted only while the DPTX link training that proves its bandwidth is current. System sleep releases the link, so when drm_atomic_helper_resume() commits the saved state, dcp_crtc_mode_fixup() rejects the mode and the whole commit fails with -EINVAL. Neither CRTC is restored; both outputs come back only when the compositor reacts to the hotplug that follows the relink. Before resuming, disable each CRTC of the saved state whose mode no longer passes the mode fixup, detaching its planes and connectors. The rest of the state is restored as before, and the DCP reports the external display again once its link is back, which is when userspace re-enables it, as after a replug. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 57 ++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 8e198c3cb1050e..68da176c5049f5 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -1119,12 +1120,66 @@ static int apple_platform_suspend(struct device *dev) return 0; } +/* + * A mode can be admissible only while the link it was proven on is up: the + * external H17P 180 Hz mode needs the current DPTX link training, and system + * sleep releases that link. Restoring such a CRTC as it was fails the whole + * resume commit in its mode fixup, which leaves every output dark until + * userspace reacts to the next hotplug. Restore that CRTC disabled instead. + * The DCP reports the display again once its link is back, and userspace + * re-enables it on that hotplug, as it does after a cable is replugged. + * + * The suspend state holds every CRTC, plane and connector already, so it is + * edited directly rather than through helpers that need the modeset locks. + */ +static void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) +{ + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_connector *connector; + struct drm_plane *plane; + struct drm_crtc *crtc; + int i, j; + + for_each_new_crtc_in_state(state, crtc, crtc_state, i) { + if (!crtc_state->enable || + dcp_crtc_mode_fixup(crtc, &crtc_state->mode, + &crtc_state->adjusted_mode)) + continue; + + drm_dbg_kms(crtc->dev, + "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", + crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); + + for_each_new_plane_in_state(state, plane, plane_state, j) { + if (plane_state->crtc != crtc) + continue; + crtc_state->plane_mask &= ~drm_plane_mask(plane); + plane_state->crtc = NULL; + drm_atomic_set_fb_for_plane(plane_state, NULL); + } + for_each_new_connector_in_state(state, connector, conn_state, j) { + /* Detaching from a CRTC already in the state cannot fail. */ + if (conn_state->crtc == crtc && + WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + return; + } + crtc_state->active = false; + if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) + return; + } +} + static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); - if (apple) + if (apple) { + if (apple->drm.mode_config.suspend_state) + apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); drm_mode_config_helper_resume(&apple->drm); + } return 0; } From d6020f1eeb5529df2b198c96da462221c3b6bb43 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:13:05 +1100 Subject: [PATCH 476/631] drm/apple: Test the resume of a CRTC whose mode no longer fixes up Resume restores a CRTC whose mode needs a link that system sleep released by bringing it back disabled. That edit of the suspend state detaches planes, connectors and the framebuffer by hand, outside the helpers, and nothing exercises it. Move it into its own file and have it ask the CRTC's own mode fixup, which for every apple-drm CRTC is dcp_crtc_mode_fixup(), so that KUnit can run it against a mock device. Add tests that take that device through drm_mode_config_helper_suspend() and _resume(): an inadmissible mode fails the resume when it is restored as is, and resumes disabled, with its planes, connector and framebuffer reference released, when it goes through the filter; an admissible or DPMS-off CRTC is handled the same way as before. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/Kconfig | 4 +- drivers/gpu/drm/apple/Makefile | 3 + drivers/gpu/drm/apple/apple_drv.c | 53 +---- drivers/gpu/drm/apple/apple_resume.c | 66 ++++++ drivers/gpu/drm/apple/apple_resume.h | 11 + drivers/gpu/drm/apple/apple_resume_test.c | 261 ++++++++++++++++++++++ 6 files changed, 345 insertions(+), 53 deletions(-) create mode 100644 drivers/gpu/drm/apple/apple_resume.c create mode 100644 drivers/gpu/drm/apple/apple_resume.h create mode 100644 drivers/gpu/drm/apple/apple_resume_test.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 14b5531c2b55fd..ba374848bbffaf 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -61,6 +61,7 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST machine, and which DCP firmware image each record admits. tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS depends on KUNIT + select DRM_KUNIT_TEST_HELPERS if DRM_APPLE default KUNIT_ALL_TESTS help Build tests for the Apple display driver's AFK ring parser and @@ -69,4 +70,5 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST The tests cover the header strides used by older firmware and larger rings, as well as malformed ring size combinations. This option is intended for developer test kernels. - Also test the Type-C modeset, hotplug and service ownership state. + Also test the Type-C modeset, hotplug and service ownership state, + and the system resume of a CRTC whose mode is no longer admissible. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 41dfad9cfcd1ab..4f71f3ea0256f3 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -2,6 +2,8 @@ CFLAGS_trace.o = -I$(src) +appledrm-y := apple_drv.o apple_resume.o +appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-y := apple_drv.o appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-y += dcp-hdmi.o @@ -18,6 +20,7 @@ appledrm-y += iomfb_v14_7.o iomfb_v14_7_link.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o appledrm-y += iomfb_h17p.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += iomfb_h17p_test.o +appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple_resume_test.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 68da176c5049f5..1b0f57a1718746 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -43,6 +42,7 @@ #include +#include "apple_resume.h" #include "dcp.h" #include "plane.h" @@ -1120,57 +1120,6 @@ static int apple_platform_suspend(struct device *dev) return 0; } -/* - * A mode can be admissible only while the link it was proven on is up: the - * external H17P 180 Hz mode needs the current DPTX link training, and system - * sleep releases that link. Restoring such a CRTC as it was fails the whole - * resume commit in its mode fixup, which leaves every output dark until - * userspace reacts to the next hotplug. Restore that CRTC disabled instead. - * The DCP reports the display again once its link is back, and userspace - * re-enables it on that hotplug, as it does after a cable is replugged. - * - * The suspend state holds every CRTC, plane and connector already, so it is - * edited directly rather than through helpers that need the modeset locks. - */ -static void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) -{ - struct drm_connector_state *conn_state; - struct drm_plane_state *plane_state; - struct drm_crtc_state *crtc_state; - struct drm_connector *connector; - struct drm_plane *plane; - struct drm_crtc *crtc; - int i, j; - - for_each_new_crtc_in_state(state, crtc, crtc_state, i) { - if (!crtc_state->enable || - dcp_crtc_mode_fixup(crtc, &crtc_state->mode, - &crtc_state->adjusted_mode)) - continue; - - drm_dbg_kms(crtc->dev, - "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", - crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); - - for_each_new_plane_in_state(state, plane, plane_state, j) { - if (plane_state->crtc != crtc) - continue; - crtc_state->plane_mask &= ~drm_plane_mask(plane); - plane_state->crtc = NULL; - drm_atomic_set_fb_for_plane(plane_state, NULL); - } - for_each_new_connector_in_state(state, connector, conn_state, j) { - /* Detaching from a CRTC already in the state cannot fail. */ - if (conn_state->crtc == crtc && - WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return; - } - crtc_state->active = false; - if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) - return; - } -} - static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c new file mode 100644 index 00000000000000..4d17d693fa59bb --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "apple_resume.h" + +/* + * A mode can be admissible only while the link it was proven on is up: the + * external H17P 180 Hz mode needs the current DPTX link training, and system + * sleep releases that link. Restoring such a CRTC as it was fails the whole + * resume commit in its mode fixup, which leaves every output dark until + * userspace reacts to the next hotplug. Restore that CRTC disabled instead. + * The DCP reports the display again once its link is back, and userspace + * re-enables it on that hotplug, as it does after a cable is replugged. + * + * The suspend state holds every CRTC, plane and connector already, so it is + * edited directly rather than through helpers that need the modeset locks. + */ +void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) +{ + const struct drm_crtc_helper_funcs *funcs; + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_connector *connector; + struct drm_plane *plane; + struct drm_crtc *crtc; + int i, j; + + for_each_new_crtc_in_state(state, crtc, crtc_state, i) { + funcs = crtc->helper_private; + if (!crtc_state->enable || !funcs || !funcs->mode_fixup || + funcs->mode_fixup(crtc, &crtc_state->mode, + &crtc_state->adjusted_mode)) + continue; + + drm_dbg_kms(crtc->dev, + "[CRTC:%d:%s] " DRM_MODE_FMT " needs a new link, resuming disabled\n", + crtc->base.id, crtc->name, DRM_MODE_ARG(&crtc_state->mode)); + + for_each_new_plane_in_state(state, plane, plane_state, j) { + if (plane_state->crtc != crtc) + continue; + crtc_state->plane_mask &= ~drm_plane_mask(plane); + plane_state->crtc = NULL; + drm_atomic_set_fb_for_plane(plane_state, NULL); + } + for_each_new_connector_in_state(state, connector, conn_state, j) { + /* Detaching from a CRTC already in the state cannot fail. */ + if (conn_state->crtc == crtc && + WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + return; + } + crtc_state->active = false; + if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) + return; + } +} diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h new file mode 100644 index 00000000000000..313e02c12b0649 --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Copyright 2026 Ryan Murray */ + +#ifndef __APPLE_RESUME_H__ +#define __APPLE_RESUME_H__ + +struct drm_atomic_state; + +void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); + +#endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c new file mode 100644 index 00000000000000..2dd1431bc065f2 --- /dev/null +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* Copyright 2026 Ryan Murray */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "apple_resume.h" + +/* One pipe: a primary plane scanning a framebuffer out to one connector. */ +struct resume_test { + struct drm_device drm; + struct drm_plane *plane; + struct drm_crtc *crtc; + struct drm_encoder encoder; + struct drm_connector connector; + struct drm_framebuffer fb; + /* Whether the mode fixup admits the mode, as a current link would. */ + bool admissible; +}; + +static struct resume_test *to_resume_test(struct drm_device *drm) +{ + return container_of(drm, struct resume_test, drm); +} + +static bool resume_test_mode_fixup(struct drm_crtc *crtc, + const struct drm_display_mode *mode, + struct drm_display_mode *adjusted_mode) +{ + return to_resume_test(crtc->dev)->admissible; +} + +static const struct drm_crtc_helper_funcs resume_test_crtc_helpers = { + .mode_fixup = resume_test_mode_fixup, +}; + +/* Scanout itself is not under test; the plane helpers require an update. */ +static void resume_test_plane_update(struct drm_plane *plane, + struct drm_atomic_state *state) +{ +} + +static const struct drm_plane_helper_funcs resume_test_plane_helpers = { + .atomic_update = resume_test_plane_update, +}; + +static const struct drm_connector_helper_funcs resume_test_connector_helpers = { +}; + +static const struct drm_connector_funcs resume_test_connector_funcs = { + .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, + .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, + .reset = drm_atomic_helper_connector_reset, +}; + +static void resume_test_fb_destroy(struct drm_framebuffer *fb) +{ + drm_framebuffer_cleanup(fb); +} + +static const struct drm_framebuffer_funcs resume_test_fb_funcs = { + .destroy = resume_test_fb_destroy, +}; + +static void resume_test_teardown(void *data) +{ + struct resume_test *priv = data; + + /* Release the scanout reference before the device goes away. */ + drm_atomic_helper_shutdown(&priv->drm); + drm_framebuffer_put(&priv->fb); +} + +static struct resume_test *resume_test_init(struct kunit *test) +{ + struct drm_modeset_acquire_ctx ctx; + struct drm_connector_state *conn_state; + struct drm_plane_state *plane_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; + struct drm_display_mode *mode; + struct resume_test *priv; + struct drm_device *drm; + struct device *dev; + int ret; + + dev = drm_kunit_helper_alloc_device(test); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + priv = drm_kunit_helper_alloc_drm_device(test, dev, struct resume_test, drm, + DRIVER_MODESET | DRIVER_ATOMIC); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv); + drm = &priv->drm; + priv->admissible = true; + + priv->plane = drm_kunit_helper_create_primary_plane(test, drm, NULL, + &resume_test_plane_helpers, + NULL, 0, NULL); + priv->crtc = drm_kunit_helper_create_crtc(test, drm, priv->plane, NULL, NULL, + &resume_test_crtc_helpers); + ret = drmm_encoder_init(drm, &priv->encoder, NULL, DRM_MODE_ENCODER_TMDS, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + priv->encoder.possible_crtcs = drm_crtc_mask(priv->crtc); + ret = drmm_connector_init(drm, &priv->connector, &resume_test_connector_funcs, + DRM_MODE_CONNECTOR_USB, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_connector_helper_add(&priv->connector, &resume_test_connector_helpers); + ret = drm_connector_attach_encoder(&priv->connector, &priv->encoder); + KUNIT_ASSERT_EQ(test, ret, 0); + drm_mode_config_reset(drm); + + priv->fb.dev = drm; + priv->fb.format = drm_format_info(DRM_FORMAT_XRGB8888); + priv->fb.width = 1920; + priv->fb.height = 1080; + priv->fb.pitches[0] = 1920 * 4; + priv->fb.modifier = DRM_FORMAT_MOD_LINEAR; + ret = drm_framebuffer_init(drm, &priv->fb, &resume_test_fb_funcs); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = kunit_add_action_or_reset(test, resume_test_teardown, priv); + KUNIT_ASSERT_EQ(test, ret, 0); + + mode = drm_kunit_display_mode_from_cea_vic(test, drm, 16); + KUNIT_ASSERT_NOT_NULL(test, mode); + + /* Put by hand: a state kept to the end of the test pins old scanout. */ + drm_modeset_acquire_init(&ctx, 0); + state = drm_atomic_state_alloc(drm); + KUNIT_ASSERT_NOT_NULL(test, state); + state->acquire_ctx = &ctx; + conn_state = drm_atomic_get_connector_state(state, &priv->connector); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, conn_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_crtc_for_connector(conn_state, priv->crtc), 0); + crtc_state = drm_atomic_get_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_mode_for_crtc(crtc_state, mode), 0); + crtc_state->active = true; + plane_state = drm_atomic_get_plane_state(state, priv->plane); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, plane_state); + KUNIT_ASSERT_EQ(test, drm_atomic_set_crtc_for_plane(plane_state, priv->crtc), 0); + drm_atomic_set_fb_for_plane(plane_state, &priv->fb); + plane_state->crtc_w = 1920; + plane_state->crtc_h = 1080; + plane_state->src_w = 1920 << 16; + plane_state->src_h = 1080 << 16; + ret = drm_atomic_commit(state); + drm_atomic_state_put(state); + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_ASSERT_TRUE(test, priv->crtc->state->active); + + return priv; +} + +/* The failure the stale CRTC filter exists for: one bad mode fails resume. */ +static void resume_stale_mode_fails_unfiltered_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), -EINVAL); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); +} + +static void resume_stale_crtc_disabled_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->active); + KUNIT_EXPECT_EQ(test, priv->crtc->state->plane_mask, 0U); + KUNIT_EXPECT_EQ(test, priv->crtc->state->connector_mask, 0U); + KUNIT_EXPECT_NULL(test, priv->plane->state->crtc); + KUNIT_EXPECT_NULL(test, priv->plane->state->fb); + KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + /* Only the creator's reference is left: the saved scanout was dropped. */ + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); +} + +static void resume_admissible_crtc_restored_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + + KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); + KUNIT_EXPECT_TRUE(test, priv->crtc->state->active); + KUNIT_EXPECT_PTR_EQ(test, priv->plane->state->crtc, priv->crtc); + KUNIT_EXPECT_PTR_EQ(test, priv->plane->state->fb, &priv->fb); + KUNIT_EXPECT_PTR_EQ(test, priv->connector.state->crtc, priv->crtc); + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_GOOD); + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 2U); +} + +/* A DPMS-off CRTC keeps its mode, so it is checked like an active one. */ +static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) +{ + struct resume_test *priv = resume_test_init(test); + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; + int ret; + + drm_modeset_acquire_init(&ctx, 0); + state = drm_atomic_state_alloc(&priv->drm); + KUNIT_ASSERT_NOT_NULL(test, state); + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state); + crtc_state->active = false; + ret = drm_atomic_commit(state); + drm_atomic_state_put(state); + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + KUNIT_ASSERT_EQ(test, ret, 0); + + KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); + priv->admissible = false; + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); + KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); + KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); +} + +static struct kunit_case resume_cases[] = { + KUNIT_CASE(resume_stale_mode_fails_unfiltered_test), + KUNIT_CASE(resume_stale_crtc_disabled_test), + KUNIT_CASE(resume_admissible_crtc_restored_test), + KUNIT_CASE(resume_stale_dpms_off_crtc_disabled_test), + {} +}; + +static struct kunit_suite resume_suite = { + .name = "apple-drm-resume", + .test_cases = resume_cases, +}; + +kunit_test_suite(resume_suite); From 028ae74d5b02dca2ad381cd0811e556edcaeaf74 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:06 +1100 Subject: [PATCH 477/631] drm/apple: Release the encoder of a connector detached for resume A CRTC restored disabled after system sleep has its connectors detached from it in the suspend state, but each keeps the encoder it was routed through. The commit that restores the state then finds a connector with an encoder and no CRTC, which the atomic helpers only expect from a broken duplicate, and every such resume logs WARNING: drivers/gpu/drm/drm_atomic_helper.c:238 at set_best_encoder The CRTC also keeps the encoder in its encoder mask. Release the encoder together with the CRTC, as the helpers do when they route a connector to no CRTC. The KUnit test now checks the edited suspend state. Fixes: 229d38cd44fd ("drm/apple: Resume a CRTC whose mode needs a new link disabled") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_resume.c | 14 ++++++++++++-- drivers/gpu/drm/apple/apple_resume_test.c | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index 4d17d693fa59bb..ec49db7d1d924c 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -54,10 +55,19 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) drm_atomic_set_fb_for_plane(plane_state, NULL); } for_each_new_connector_in_state(state, connector, conn_state, j) { + if (conn_state->crtc != crtc) + continue; /* Detaching from a CRTC already in the state cannot fail. */ - if (conn_state->crtc == crtc && - WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) + if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) return; + /* + * Nor does a connector without a CRTC keep an encoder: + * the resume check warns about one left routed. + */ + if (conn_state->best_encoder) + crtc_state->encoder_mask &= + ~drm_encoder_mask(conn_state->best_encoder); + conn_state->best_encoder = NULL; } crtc_state->active = false; if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index 2dd1431bc065f2..94f539986e6638 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -179,10 +179,23 @@ static void resume_stale_mode_fails_unfiltered_test(struct kunit *test) static void resume_stale_crtc_disabled_test(struct kunit *test) { struct resume_test *priv = resume_test_init(test); + struct drm_connector_state *conn_state; + struct drm_crtc_state *crtc_state; + struct drm_atomic_state *state; KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + state = priv->drm.mode_config.suspend_state; + apple_drm_drop_stale_crtcs(state); + + /* The resume check warns about an encoder routed without a CRTC. */ + conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); + crtc_state = drm_atomic_get_new_crtc_state(state, priv->crtc); + KUNIT_ASSERT_NOT_NULL(test, conn_state); + KUNIT_ASSERT_NOT_NULL(test, crtc_state); + KUNIT_EXPECT_NULL(test, conn_state->best_encoder); + KUNIT_EXPECT_EQ(test, crtc_state->encoder_mask, 0U); + KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); From 50d67ae91aee87034d2872bcd56a4ecf88894b6f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:23:56 +1100 Subject: [PATCH 478/631] drm/apple: Have userspace restore a CRTC resumed disabled A CRTC whose mode needs a link that system sleep released is restored disabled, on the expectation that the hotplug following the relink makes userspace enable it again. Nothing guarantees that hotplug: the DCP reports a display that comes back on the same cable only for a new cable or the first attachment, and the 180 Hz mode is announced again only if its rate proof is renewed. Even when a hotplug arrives, the connector is still connected with the same modes, so a compositor that tracks its own state sees no change, believes the output is still lit, and has every flip rejected on the disabled CRTC. Use the link-status contract instead: mark the link of each connector detached from such a CRTC bad in the state being restored, and send a hotplug event once resume has committed it. Userspace then sets the mode again, with whatever modes the connector offers at that point, as it does after a failed link retraining. Fixes: 229d38cd44fd ("drm/apple: Resume a CRTC whose mode needs a new link disabled") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/apple_drv.c | 6 +++++- drivers/gpu/drm/apple/apple_resume.c | 19 ++++++++++++++----- drivers/gpu/drm/apple/apple_resume.h | 4 +++- drivers/gpu/drm/apple/apple_resume_test.c | 14 +++++++++++--- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 1b0f57a1718746..3d76e6fdf8734b 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -1123,11 +1123,15 @@ static int apple_platform_suspend(struct device *dev) static int apple_platform_resume(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); + bool dropped = false; if (apple) { if (apple->drm.mode_config.suspend_state) - apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); + dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); drm_mode_config_helper_resume(&apple->drm); + /* Have userspace set the mode of a CRTC resumed disabled again. */ + if (dropped) + drm_kms_helper_hotplug_event(&apple->drm); } return 0; diff --git a/drivers/gpu/drm/apple/apple_resume.c b/drivers/gpu/drm/apple/apple_resume.c index ec49db7d1d924c..52dabd11d3e510 100644 --- a/drivers/gpu/drm/apple/apple_resume.c +++ b/drivers/gpu/drm/apple/apple_resume.c @@ -19,13 +19,17 @@ * sleep releases that link. Restoring such a CRTC as it was fails the whole * resume commit in its mode fixup, which leaves every output dark until * userspace reacts to the next hotplug. Restore that CRTC disabled instead. - * The DCP reports the display again once its link is back, and userspace - * re-enables it on that hotplug, as it does after a cable is replugged. + * Userspace still believes the CRTC is on, and the DCP need not report the + * same display again when its link comes back, so mark the link of each of + * its connectors bad: userspace then sets the mode again, on the hotplug + * event the caller sends, as after a failed link retraining. * * The suspend state holds every CRTC, plane and connector already, so it is * edited directly rather than through helpers that need the modeset locks. + * + * Returns true if a CRTC is to be resumed disabled. */ -void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) +bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) { const struct drm_crtc_helper_funcs *funcs; struct drm_connector_state *conn_state; @@ -34,6 +38,7 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) struct drm_connector *connector; struct drm_plane *plane; struct drm_crtc *crtc; + bool dropped = false; int i, j; for_each_new_crtc_in_state(state, crtc, crtc_state, i) { @@ -59,7 +64,7 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) continue; /* Detaching from a CRTC already in the state cannot fail. */ if (WARN_ON(drm_atomic_set_crtc_for_connector(conn_state, NULL))) - return; + return true; /* * Nor does a connector without a CRTC keep an encoder: * the resume check warns about one left routed. @@ -68,9 +73,13 @@ void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state) crtc_state->encoder_mask &= ~drm_encoder_mask(conn_state->best_encoder); conn_state->best_encoder = NULL; + conn_state->link_status = DRM_LINK_STATUS_BAD; } crtc_state->active = false; + dropped = true; if (WARN_ON(drm_atomic_set_mode_for_crtc(crtc_state, NULL))) - return; + break; } + + return dropped; } diff --git a/drivers/gpu/drm/apple/apple_resume.h b/drivers/gpu/drm/apple/apple_resume.h index 313e02c12b0649..de0dffe2dc8bd5 100644 --- a/drivers/gpu/drm/apple/apple_resume.h +++ b/drivers/gpu/drm/apple/apple_resume.h @@ -4,8 +4,10 @@ #ifndef __APPLE_RESUME_H__ #define __APPLE_RESUME_H__ +#include + struct drm_atomic_state; -void apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); +bool apple_drm_drop_stale_crtcs(struct drm_atomic_state *state); #endif /* __APPLE_RESUME_H__ */ diff --git a/drivers/gpu/drm/apple/apple_resume_test.c b/drivers/gpu/drm/apple/apple_resume_test.c index 94f539986e6638..7f89c07c152946 100644 --- a/drivers/gpu/drm/apple/apple_resume_test.c +++ b/drivers/gpu/drm/apple/apple_resume_test.c @@ -186,7 +186,8 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; state = priv->drm.mode_config.suspend_state; - apple_drm_drop_stale_crtcs(state); + /* The caller tells userspace about the CRTC it had and lost. */ + KUNIT_EXPECT_TRUE(test, apple_drm_drop_stale_crtcs(state)); /* The resume check warns about an encoder routed without a CRTC. */ conn_state = drm_atomic_get_new_connector_state(state, &priv->connector); @@ -205,6 +206,9 @@ static void resume_stale_crtc_disabled_test(struct kunit *test) KUNIT_EXPECT_NULL(test, priv->plane->state->crtc); KUNIT_EXPECT_NULL(test, priv->plane->state->fb); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + /* Userspace must modeset again rather than flip onto a dead CRTC. */ + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_BAD); /* Only the creator's reference is left: the saved scanout was dropped. */ KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); } @@ -214,7 +218,8 @@ static void resume_admissible_crtc_restored_test(struct kunit *test) struct resume_test *priv = resume_test_init(test); KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_FALSE(test, + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_TRUE(test, priv->crtc->state->enable); @@ -251,10 +256,13 @@ static void resume_stale_dpms_off_crtc_disabled_test(struct kunit *test) KUNIT_ASSERT_EQ(test, drm_mode_config_helper_suspend(&priv->drm), 0); priv->admissible = false; - apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state); + KUNIT_EXPECT_TRUE(test, + apple_drm_drop_stale_crtcs(priv->drm.mode_config.suspend_state)); KUNIT_EXPECT_EQ(test, drm_mode_config_helper_resume(&priv->drm), 0); KUNIT_EXPECT_FALSE(test, priv->crtc->state->enable); KUNIT_EXPECT_NULL(test, priv->connector.state->crtc); + KUNIT_EXPECT_EQ(test, priv->connector.state->link_status, + DRM_LINK_STATUS_BAD); KUNIT_EXPECT_EQ(test, drm_framebuffer_read_refcount(&priv->fb), 1U); } From b971be11f0b106d69630a8273dd0251476300d7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:15:37 +1100 Subject: [PATCH 479/631] soc: apple: mailbox, rtkit: Allow holding received messages The receive interrupt of an ASC mailbox is requested IRQF_NO_SUSPEND so that RTKit co-processors stay serviceable during system sleep. A co-processor that keeps sending while the system sleeps then wakes a CPU for every message. Add apple_mbox_hold_rx() and apple_mbox_release_rx(), which mask and unmask the receive interrupt so that messages stay queued in the hardware FIFO, and the RTKit wrappers apple_rtkit_hold_rx() and apple_rtkit_resume_rx() for co-processor drivers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mailbox.c | 12 ++++++++++++ drivers/soc/apple/rtkit.c | 11 +++++++++++ include/linux/soc/apple/mailbox.h | 6 ++++++ include/linux/soc/apple/rtkit.h | 7 +++++++ 4 files changed, 36 insertions(+) diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 58d177c435bd8d..53be121ac970a3 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -512,6 +512,18 @@ void apple_mbox_stop(struct apple_mbox *mbox) } EXPORT_SYMBOL(apple_mbox_stop); +void apple_mbox_hold_rx(struct apple_mbox *mbox) +{ + disable_irq(mbox->irq_recv_not_empty); +} +EXPORT_SYMBOL(apple_mbox_hold_rx); + +void apple_mbox_release_rx(struct apple_mbox *mbox) +{ + enable_irq(mbox->irq_recv_not_empty); +} +EXPORT_SYMBOL(apple_mbox_release_rx); + /* * Stop the provider-owned CPU, then quiesce every transport edge before * devres releases IRQs or MMIO mappings. Device links order consumer removal diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index b78ae6c2afb02a..20e3255cdd8c95 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -966,6 +966,17 @@ void apple_rtkit_flush_rx(struct apple_rtkit *rtk) flush_work(&rtk->sync_ack_work); } EXPORT_SYMBOL_GPL(apple_rtkit_flush_rx); +void apple_rtkit_hold_rx(struct apple_rtkit *rtk) +{ + apple_mbox_hold_rx(rtk->mbox); +} +EXPORT_SYMBOL_GPL(apple_rtkit_hold_rx); + +void apple_rtkit_resume_rx(struct apple_rtkit *rtk) +{ + apple_mbox_release_rx(rtk->mbox); +} +EXPORT_SYMBOL_GPL(apple_rtkit_resume_rx); bool apple_rtkit_has_endpoint(struct apple_rtkit *rtk, u8 ep) { diff --git a/include/linux/soc/apple/mailbox.h b/include/linux/soc/apple/mailbox.h index e5697975a50b50..8813438326ffcf 100644 --- a/include/linux/soc/apple/mailbox.h +++ b/include/linux/soc/apple/mailbox.h @@ -93,6 +93,12 @@ struct apple_mbox *apple_mbox_get_byname(struct device *dev, const char *name); int apple_mbox_start(struct apple_mbox *mbox); void apple_mbox_stop(struct apple_mbox *mbox); int apple_mbox_poll(struct apple_mbox *mbox); +/* + * Leave received messages in the hardware FIFO, without an interrupt, until + * apple_mbox_release_rx(). For consumers that must not wake the system. + */ +void apple_mbox_hold_rx(struct apple_mbox *mbox); +void apple_mbox_release_rx(struct apple_mbox *mbox); int apple_mbox_send(struct apple_mbox *mbox, struct apple_mbox_msg msg, bool atomic); diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index c11f1585f3a4e4..11d13a4a9cd461 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -240,6 +240,13 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message, int apple_rtkit_poll(struct apple_rtkit *rtk); void apple_rtkit_flush_rx(struct apple_rtkit *rtk); +/* + * Stop taking interrupts for messages from the co-processor, which then + * stay queued in the mailbox until apple_rtkit_resume_rx(). + */ +void apple_rtkit_hold_rx(struct apple_rtkit *rtk); +void apple_rtkit_resume_rx(struct apple_rtkit *rtk); + /* * Checks if an endpoint with a given index exists * From c46f48a505e1ca7c4ffc899dc4a9685710a68caa Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:15:37 +1100 Subject: [PATCH 480/631] drm/apple: Hold internal DCP messages during system sleep DPMS off on the internal H17P panel is a brightness present and leaves the pipe scanning out. While it scans out, the firmware publishes a property to the AP on every frame (D300 pr_publish, id 108, about 60 per second), and the DCP mailbox interrupt is IRQF_NO_SUSPEND, so each of them wakes a CPU out of suspend-to-idle: a 60 s RTC sleep took about 3650 interrupts from 412e08000.mailbox-recv, a 360 s one 21676. Hold the coprocessor's messages in the mailbox as the last step of the DCP suspend handler, after presentation is paused and the command queue has settled, and process them first thing on resume. A 360 s sleep now takes 97 interrupts, all of them in the transitions, and the panel and the firmware carry on normally after it. apple-drm is a device-link consumer of each DCP, so its suspend handler has blanked the panel before this runs and its resume handler restores the CRTC after the messages are processed. Powering the pipe off with A473 or idling the coprocessor through RTKit instead are not usable: the first left the firmware crashed after the power on, and the firmware does not acknowledge the second. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 37 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 75d9f0ce748e16..7bc2ab6f7336a3 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -393,6 +393,8 @@ struct apple_dcp { bool active; /* A successful initial enable keeps the H17P pipe running across DPMS. */ bool pipe_enabled_h17p; + /* Messages from the coprocessor are held in the mailbox for sleep. */ + bool rx_held; /* eDP display without DP-HDMI conversion */ bool main_display; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 115be64dc8503f..96a2dfd0511435 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -6065,6 +6065,40 @@ static void dcp_drain_for_sleep(struct apple_dcp *dcp) dev_warn(dcp->dev, "brightness change still pending at suspend\n"); } +/* + * DPMS off leaves the internal H17P pipe scanning out (see dcp_poweroff()), + * and while it scans out the firmware publishes a property to the AP on every + * frame. The mailbox interrupt is not suspended, so each of those messages + * wakes a CPU out of suspend-to-idle about 60 times a second. Hold the + * messages in the mailbox for system sleep and process them at resume. + * apple-drm is a consumer of this device, so its suspend handler has blanked + * the panel before this runs and its resume handler restores the CRTC after. + */ +static void dcp_hold_for_sleep(struct apple_dcp *dcp) +{ + if (!dcp_uses_soft_dpms(dcp) || dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || + READ_ONCE(dcp->crashed)) + return; + + iomfb_pause_presentation(dcp, true); + if (iomfb_queue_settle(dcp, msecs_to_jiffies(500))) { + iomfb_pause_presentation(dcp, false); + return; + } + apple_rtkit_hold_rx(dcp->rtk); + dcp->rx_held = true; +} + +static void dcp_release_after_sleep(struct apple_dcp *dcp) +{ + if (!dcp->rx_held) + return; + dcp->rx_held = false; + apple_rtkit_resume_rx(dcp->rtk); + iomfb_pause_presentation(dcp, false); +} + static int dcp_platform_suspend(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -6145,6 +6179,8 @@ static int dcp_platform_suspend(struct device *dev) * shutdown properly yet. */ device_set_wakeup_path(dev); + /* Last: the steps above still talk to the coprocessor. */ + dcp_hold_for_sleep(dcp); return 0; } @@ -6161,6 +6197,7 @@ static int dcp_platform_resume(struct device *dev) if (dcp->external_native) dcp_fabric_hdmi_reinit(dcp, "after system sleep"); /* Observe future edges before sampling any edges lost in sleep. */ + dcp_release_after_sleep(dcp); dcp_enable_typec_work(dcp); if (dcp->hdmi_hpd_irq) dcp_fabric_run_resume(&dcp_resume_ops, dcp); From ac3144901f4b1b481610e56145b61d96a2c85790 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:32:20 +1100 Subject: [PATCH 481/631] drm/apple, usb: typec: tipd: Retry a display mux that is only busy The CD321x driver treats -EBUSY from typec_mux_set() as "the shared external display is driven by the other port": it sets up the port's USB path without the display and does not retry. -EBUSY has other sources on the same mux chain, though. The Apple Type-C PHY returns it while a mode or pipe transition is pending, and the DCP route returned it while the previous SAFE transition was still running and while the external DCP had not finished stopping. Those conditions clear on their own, but the port was left without its display until the next cable event. Have the DCP route report the lasting condition, every pipeline driving another port's display, as -ENOSPC, and the transient ones as -EAGAIN. The CD321x driver now skips the retry only for -ENOSPC, through a helper with a KUnit case; -EBUSY and -EAGAIN are retried like any other failure, -EAGAIN without counting against the port. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 +++--- drivers/usb/typec/tipd/cd321x-pm-test.c | 10 ++++++++++ drivers/usb/typec/tipd/cd321x-pm.h | 2 ++ drivers/usb/typec/tipd/core.c | 4 ++-- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 96a2dfd0511435..57f5c4059c81a4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -419,7 +419,7 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) guard(mutex)(&dcp->external_lock); if (dcp_is_external_h17p(dcp) && (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) - return dcp->external_error ?: -EBUSY; + return dcp->external_error ?: -EAGAIN; /* * Under tb_lock so a DCP link (re)configuration can neither select the @@ -935,7 +935,7 @@ static int dcp_typec_route_prepare(struct typec_mux_dev *mux, guard(mutex)(&dcp_typec_fabric_lock); if (state->mode != TYPEC_STATE_SAFE) - return port->safe_pending ? -EBUSY : 0; + return port->safe_pending ? -EAGAIN : 0; if (!port->owner || port->owner->tunnel) return 0; dcp = port->owner->dcp; @@ -1025,7 +1025,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) return 0; if (port->safe_pending) - return -EBUSY; + return -EAGAIN; /* * Every candidate route for this port is notified with the same state, diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 92b2b9cbeed2cc..9c1e3f608b6308 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -752,6 +752,15 @@ static void cd321x_restore_ignores_fault_budget_test(struct kunit *test) /* A prepared port revalidates on its own resume instead. */ cd321x_pm_prepare(&pm); KUNIT_EXPECT_EQ(test, cd321x_pm_notifications(&pm, false, false, true), 0U); +/* Only a pipeline owned by another port skips the retry. */ +static void cd321x_display_taken_test(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, cd321x_display_taken(-ENOSPC)); + /* A PHY mid-transition or a pending SAFE state must be retried. */ + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EBUSY)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EAGAIN)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-ETIMEDOUT)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(0)); } static struct kunit_case cd321x_pm_cases[] = { @@ -790,6 +799,7 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_status_event_after_exhaustion_test), KUNIT_CASE(cd321x_runtime_failure_revalidates_test), KUNIT_CASE(cd321x_restore_ignores_fault_budget_test), + KUNIT_CASE(cd321x_display_taken_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index de35c14b7b6b75..a717c95a2d920c 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -2,6 +2,8 @@ #ifndef __CD321X_PM_H__ #define __CD321X_PM_H__ +#include + #include #define CD321X_RESUME_ATTEMPTS 3 diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 6b5e567eece341..01b3c6fb2fa715 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -736,7 +736,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, if (ret) goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); - *display_busy = ret == -EBUSY; + *display_busy = cd321x_display_taken(ret); if (!ret) { cd321x->dp_status = dp_data.status; cd321x->dp_conf = dp_data.conf; @@ -937,7 +937,7 @@ static void cd321x_update_work(struct work_struct *work) * a failed teardown keeps the route, but the sink must not stay present. * All ports reference the same external display connector, so only the * port that owns the display route may report a disconnect. A second DP - * sink can enter Alt Mode while its mux request fails with -EBUSY, and + * sink can enter Alt Mode while its mux request fails with -ENOSPC, and * must not tear down the display carried by the other port. */ if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && From 42a1c22c61bd094c8c5c1bae49507bd7cc18820a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:34:20 +1100 Subject: [PATCH 482/631] drm/apple: Log routine DPTX and EDID service events at debug level Every hotplug, link training and resume of the external display logged a dozen info lines: each DPTX port call (SET_LINK_RATE, GET_SUPPORTS_HPD, DEACTIVATE, FORCE_HOTPLUG_DETECT, INACTIVE_SINK_DETECTED, the lane count calls and the validate_connection attributes), the AV service announcing its EDID channel, every connect attempt, every boot property and read_edt_data request, and "copy_edid failed: -ENODEV" whenever the EDID was probed while the link was down. None of them is a problem; they are the normal protocol flow. Log them with dev_dbg() instead, so dynamic debug still shows them. -ENODEV from the EDID copy now means no EDID source for any connector, not only the internal panel. A copy or publication failure with any other error is a real problem and is now a warning. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp.c | 6 ++++++ drivers/gpu/drm/apple/epic/dpavservep.c | 4 ++-- drivers/gpu/drm/apple/iomfb.c | 11 ++++++----- drivers/gpu/drm/apple/iomfb_template.c | 8 ++++---- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 57f5c4059c81a4..0cf6b20b7d7114 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -2617,6 +2617,12 @@ static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_s dcp->dptx_tunnel, dcp_is_typec_output(dcp), dcp->active_typec_route ? "borrowed" : "fixed", dcp->connector_type, dcp->dptxport[port].connected); + dev_dbg(dcp->dev, + "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", + __func__, port, dcp->dptx_die, dcp->dptx_phy, + dcp_is_typec_output(dcp), + dcp->active_typec_route ? "borrowed" : "fixed", + dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 62a30a23914436..21b0ee790369b3 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -147,8 +147,8 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, return; } if (v2) { - dev_info(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", - service->channel); + dev_dbg(dcp->dev, "DCPAVSERV: AFKV2 EDID service ready on channel %u\n", + service->channel); dcpavserv_notify_edid(service); } } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e45a700c98df6a..32e7bfa406e450 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -1454,13 +1454,14 @@ int dcp_get_modes(struct drm_connector *connector) dcp_consume_edid_invalidation(apple_connector); if (IS_ERR_OR_NULL(edid)) { /* - * An internal panel has no AV service and so no EDID; + * There is no AV service for an internal panel, nor + * for an external display while its link is down; * do not report that on every mode probe. */ - if (dcp_has_panel(dcp) && PTR_ERR(edid) == -ENODEV) - dev_dbg(dcp->dev, "no EDID source for the panel\n"); + if (PTR_ERR(edid) == -ENODEV) + dev_dbg(dcp->dev, "no EDID source\n"); else - dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + dev_warn(dcp->dev, "copy_edid failed: %pe\n", edid); } else { drm_edid_free(apple_connector->drm_edid); apple_connector->drm_edid = edid; @@ -1482,7 +1483,7 @@ int dcp_get_modes(struct drm_connector *connector) int update = drm_edid_connector_update(connector, apple_connector->drm_edid); if (update) { - dev_info(dcp->dev, "raw EDID publication failed: %d\n", update); + dev_warn(dcp->dev, "raw EDID publication failed: %d\n", update); dcp_clear_connector_edid(apple_connector); } else { apple_connector->drm_edid_firmware_derived = false; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index beeabad7267c13..f26aa6877bda05 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -912,8 +912,8 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) { /* Observe boot-property requests without inventing firmware timings. */ if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(dcp->dev, - "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", + dev_dbg(dcp->dev, + "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", (int)sizeof(req->key), req->key, req->count, req->value[0]); @@ -1097,8 +1097,8 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, resp, dcp->nr_modes); if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(dcp->dev, - "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", + dev_dbg(dcp->dev, + "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", (int)sizeof(req->key), req->key, dcp->chunks.length, resp, dcp->nr_modes); From 4431ee749006f8efaf61fdb67b71e25dfb5bd030 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:35:16 +1100 Subject: [PATCH 483/631] drm/apple: Describe the H17P support without bring-up terms Comments, function names and a few messages in the H17P support still use the language of its bring-up: "measured" profiles, "qualified" loaders and sequences, an "unqualified" callback, the "USB4 PHY experiment". They tell a reader nothing beyond the code next to them, and "unqualified" in an error suggests a state the driver could get out of rather than an unsupported request. Say what is meant instead: the H17P profile, a known or unknown sequence, an unsupported callback or request, the J700 defaults. Rename dptxport_call_qualified_config() and dptxport_qualified_teardown() after what they do. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/dcp-180hz-policy.h | 2 +- drivers/gpu/drm/apple/dcp.c | 6 +++--- drivers/gpu/drm/apple/dptxep.c | 14 ++++++++------ drivers/gpu/drm/apple/iomfb.c | 2 +- drivers/gpu/drm/apple/iomfb_h17p.c | 16 ++++++++-------- drivers/gpu/drm/apple/iomfb_template.c | 8 ++++---- drivers/gpu/drm/apple/plane.c | 2 +- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-180hz-policy.h b/drivers/gpu/drm/apple/dcp-180hz-policy.h index a221c0e4a0773f..d23aa57c685e28 100644 --- a/drivers/gpu/drm/apple/dcp-180hz-policy.h +++ b/drivers/gpu/drm/apple/dcp-180hz-policy.h @@ -6,7 +6,7 @@ #include #include -/* Only this qualified uncompressed configuration is admitted. */ +/* Only this verified uncompressed configuration is admitted. */ static inline bool dcp_180hz_payload(u32 rate, u32 lanes, u32 clock) { return rate == 0x1e && lanes == 2 && clock == 414090 && diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 0cf6b20b7d7114..3719ddf3c718ff 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -5603,7 +5603,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) set_bit(surf, dcp->iomfb_surfaces); surf++; } - /* The qualified J700 loader names surface zero rather than enabling it. */ + /* The J700 loader names surface zero rather than enabling it. */ if (of_machine_is_compatible("apple,j700") && of_device_is_compatible(dev->of_node, "apple,t8140-dcp") && num_surfs == 1 && @@ -5931,7 +5931,7 @@ static void dcp_platform_shutdown(struct platform_device *pdev) dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { /* * This firmware keeps scanning across soft DPMS and has no - * qualified stop sequence. Component unbind releases the RTKit + * known stop sequence. Component unbind releases the RTKit * devres group and the piodma domain while firmware may still * access them. Stop host submissions but keep the component and * its DMA resources alive until the system resets. @@ -6235,7 +6235,7 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; -/* The internal T8140 endpoint uses the measured H17P method profile. */ +/* The internal T8140 endpoint uses the H17P method profile. */ static const struct resource t8140_firmware_clock = DEFINE_RES_MEM(0x302800000, 0xbc000); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 6fbdffa7802519..52d9e1cf3edac4 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -759,8 +759,8 @@ dptxport_call_deactivate(struct apple_epic_service *service, return 0; } -static int dptxport_call_qualified_config(struct apple_epic_service *service, bool lanes, - const void *data, size_t size, void *reply, size_t room) +static int dptxport_call_link_config(struct apple_epic_service *service, bool lanes, + const void *data, size_t size, void *reply, size_t room) { struct dcp_180hz_token token = dcp_180hz_config_begin(service, lanes); struct dptx_port *port = service->cookie; @@ -836,7 +836,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_GET_LINK_RATE: return dptxport_call_get_link_rate(service, reply, reply_size); case DPTX_APCALL_SET_LINK_RATE: - return dptxport_call_qualified_config(service, false, data, data_size, reply, reply_size); + return dptxport_call_link_config(service, false, data, data_size, + reply, reply_size); case DPTX_APCALL_GET_MAX_LANE_COUNT: return dptxport_call_get_max_lane_count(service, reply, reply_size); case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { @@ -849,7 +850,8 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return 0; } case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: - return dptxport_call_qualified_config(service, true, data, data_size, reply, reply_size); + return dptxport_call_link_config(service, true, data, data_size, + reply, reply_size); case DPTX_APCALL_GET_SUPPORTS_HPD: return dptxport_call_get_supports_hpd(service, reply, reply_size); @@ -1080,7 +1082,7 @@ static int dptxport_group_call(struct apple_epic_service *service, u16 group, return 0; } -static void dptxport_qualified_teardown(struct apple_epic_service *service) +static void dptxport_teardown(struct apple_epic_service *service) { dcp_180hz_invalidate(service->ep->dcp); dptx_service_unbind(service); @@ -1090,7 +1092,7 @@ static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .prepare = dptxport_prepare, - .teardown = dptxport_qualified_teardown, + .teardown = dptxport_teardown, .init = dptxport_init, .call = dptxport_call, .call_v2 = dptxport_group_call, diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index 32e7bfa406e450..b6bc2bb76c027d 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -965,7 +965,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { - dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", + dev_err(dev, "unsupported IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; return; diff --git a/drivers/gpu/drm/apple/iomfb_h17p.c b/drivers/gpu/drm/apple/iomfb_h17p.c index 45743b41736999..6c0b780674206d 100644 --- a/drivers/gpu/drm/apple/iomfb_h17p.c +++ b/drivers/gpu/drm/apple/iomfb_h17p.c @@ -311,7 +311,7 @@ static bool trampoline_analytics_h17p(struct apple_dcp *dcp, int tag, return trampoline_zero(dcp, tag, out, in); trace_iomfb_callback(dcp, tag, __func__); - /* The dispatcher has validated the measured D114 reply size. */ + /* The dispatcher has validated the D114 reply size. */ memset(out, 0, 0x1004); *(u8 *)out = 'd'; return true; @@ -390,7 +390,7 @@ static const u8 j700_provider_data_4[] = { 0xa2, 0x19, 0x00, 0x00 }; -/* Only these provider transfers have been measured against the disp0 ADT. */ +/* Only these provider transfers are known to match the disp0 ADT. */ static const struct dcp_h17p_provider_property provider_properties[] = { { "power-lut-data-x", "apple,power-lut-data-x", 4, j700_provider_data_0 }, { "power-lut-data-y", "apple,power-lut-data-y", 4, j700_provider_data_1 }, @@ -434,13 +434,13 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, trace_iomfb_callback(dcp, tag, __func__); property = dcp_provider_property_h17p(in); if (!property) { - dev_err(dcp->dev, "unqualified D400 provider property request\n"); + dev_err(dcp->dev, "unsupported D400 provider property request\n"); goto fail; } length = of_property_count_u8_elems(node, property->dt_name); if (!property->length) { - /* Only this absent property has a measured empty reply. */ + /* Only this absent property has a known empty reply. */ if (of_property_present(node, property->dt_name)) goto fail; } else if (!of_property_present(node, property->dt_name) && @@ -448,12 +448,12 @@ static bool trampoline_provider_property_h17p(struct apple_dcp *dcp, int tag, (of_device_is_compatible(node, "apple,t8140-dcp") || (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && of_device_is_compatible(node, "apple,t8140-dcpext")))) { - /* Qualified older loaders lack the newly described table properties. */ + /* Older loaders lack the newly described table properties. */ memset(reply, 0, sizeof(*reply)); memcpy(reply->data, property->legacy_data, property->length); reply->length = cpu_to_le32(property->length); dev_warn_once(dcp->dev, - "missing provider property %s (%u bytes); using qualified J700 compatibility data\n", + "missing provider property %s (%u bytes); using the J700 defaults\n", property->dt_name, property->length); return true; } else if (length != (int)property->length) { @@ -486,7 +486,7 @@ static bool trampoline_luma_span_h17p(struct apple_dcp *dcp, int tag, trace_iomfb_callback(dcp, tag, __func__); if (get_unaligned_le32(in)) { - dev_err(dcp->dev, "unqualified D574 range selector\n"); + dev_err(dcp->dev, "unsupported D574 range selector\n"); WRITE_ONCE(dcp->crashed, true); return false; } @@ -510,7 +510,7 @@ static bool trampoline_get_uint_prop_h17p(struct apple_dcp *dcp, int tag, strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || strcmp(request->key, "Temperature") || request->value_null || memchr_inv(request->padding, 0, sizeof(request->padding))) { - dev_err(dcp->dev, "unqualified D401 integer property query\n"); + dev_err(dcp->dev, "unsupported D401 integer property query\n"); WRITE_ONCE(dcp->crashed, true); return false; } diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index f26aa6877bda05..03ce35d540bf2e 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1394,7 +1394,7 @@ static void release_swap_cookie(struct kref *ref) * format two bytes late and its surface check rejects the swap as an * unsupported format. * - * H17G retains this public packing path. The measured H17P profile uses + * H17G retains this public packing path. The H17P profile uses * a separate wire record with the SPEC's 0x588 swap boundary. */ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) @@ -1417,7 +1417,7 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) req->swap.h17p_pre_swap_id[0x00] = 2; req->swap.h17p_pre_swap_id[0x28] = 3; - /* The measured H17P layout is serialized without shifting its surfaces. */ + /* The H17P layout is serialized without shifting its surfaces. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; @@ -1699,7 +1699,7 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) { - /* H17P stop semantics still require hardware qualification. */ + /* H17P does not confirm that scanout has stopped. */ #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER return false; #else @@ -1737,7 +1737,7 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* The measured profile has no safe pipe-off or surface-free transition. */ + /* H17P has no known safe pipe-off or surface-free transition. */ if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; #endif diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index de525e2b80ba76..a298a5bf923e05 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -427,7 +427,7 @@ static const u32 dcp_primary_formats[] = { #endif }; -/* H17P 10-bit XRGB alpha semantics have not been qualified. */ +/* The H17P alpha semantics of 10-bit XRGB are unknown. */ static const u32 dcp_primary_formats_h17p[] = { DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, From 2b9ca8cca00f471bc4b62e1b85a0dee3a2ea78c1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 484/631] phy: apple: atc: Quiet the DisplayPort AUX and tunnel clock logging atcphy_enable_dp_aux() logged "Applied T8140 DisplayPort setup lists" once per boot. It comes from enabling the AUX channel, not from where the setup lists are applied, and it reports nothing a user needs. atc_tunnel_start_t602x() logged four raw register values at info level on every tunnel clock preflight. They only help when a preflight is refused, so log them with dev_dbg(). No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 3e7ab85eaa95d6..02ed125e6fc45e 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1856,10 +1856,8 @@ static int j700_aux_disable(struct apple_atcphy *atcphy) static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) { - if (atcphy->hw->has_t8140_dp) { - dev_info_once(atcphy->dev, "Applied T8140 DisplayPort setup lists\n"); + if (atcphy->hw->has_t8140_dp) return j700_aux_enable(atcphy); - } /* FIXME */ /* T8122 PHYs with DP support enable AUX in atcphy_dp_aux_on_t8122() */ From 4b147e56a4f89c562c141467221da77085526408 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 485/631] phy: apple: atc: Use the register field names in the tunnel clock code atc_tunnel_restore(), atc_tunnel_start_t602x() and the T602X tunnel rate code write raw masks where the fields already have names: 0x54 and BIT(2), BIT(4), BIT(6) are the AUSPLL_CLKOUT_MASTER output drivers, the ACIOPHY_SLEEP_CTRL values are the TX_SMALL, TX_BIG and TX_CLAMP fields and their overrides, and 0xc00, 0xc0 and 0xc000 in ACIOPHY_CFG0 are RX_BIG, RX_SMALL and RX_CLAMP. The comments calling the last three "additional override bits" of unconfirmed meaning were wrong; the sequence is the same one atc_tunnel_wake_t8103() writes for the TX and RX lanes. Use the names. No functional change: every write is the same value. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 02ed125e6fc45e..8e7d53a2c8111c 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2906,25 +2906,24 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x30); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc0); + /* same for the TX lanes */ + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x03); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x0c); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); udelay(2); - core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, 0x300); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc00); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); udelay(2); - /* Additional override bit alongside RX_BIG_OV; exact meaning not independently confirmed. */ - core_set32(atcphy, ACIOPHY_CFG0, 0xc00); + /* and for the RX lanes */ + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); udelay(2); - /* Additional override bit alongside RX_SMALL_OV; exact meaning not independently confirmed. */ - core_set32(atcphy, ACIOPHY_CFG0, 0xc0); + core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); udelay(2); - /* Additional override bit alongside RX_CLAMP_OV; exact meaning not independently confirmed. */ - core_clear32(atcphy, ACIOPHY_CFG0, 0xc000); + core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); udelay(2); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); From 9b5d40350ea2364fe7e864039172c8801eef2ba3 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:11:02 +1100 Subject: [PATCH 486/631] phy: apple: atc: Share the tunnel rate selector table atc_tunnel_start_t8103() mapped the DP link rate code to the pixel clock selector with its own switch, a copy of atc_tunnel_rate_selector() used by the T602X code. Move the helper up and use it for both. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 8e7d53a2c8111c..359eb62f32e5bb 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -2656,7 +2656,8 @@ static void atc_tunnel_wake_t8103(struct apple_atcphy *atcphy) udelay(2); } -static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) +/* The pixel clock selector for a DP link rate code. */ +static int atc_tunnel_rate_selector(u8 rate) { u32 selector, value; int ret; @@ -2664,20 +2665,28 @@ static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) lockdep_assert_held(&atcphy->lock); switch (rate) { case 0x06: /* RBR */ - selector = 4; - break; + return 4; case 0x0a: /* HBR */ - selector = 3; - break; + return 3; case 0x14: /* HBR2 */ - selector = 1; - break; + return 1; case 0x1e: /* HBR3 */ selector = 0; break; default: return -EINVAL; } +} + +static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) +{ + int selector, ret; + u32 value; + + lockdep_assert_held(&atcphy->lock); + selector = atc_tunnel_rate_selector(rate); + if (selector < 0) + return selector; if (atcphy->tunnel_clock_on) { if (atcphy->tunnel_rate == rate) return 0; From 8fadc267306d922a9df9f30a78fb69492e3ac61f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:12:38 +1100 Subject: [PATCH 487/631] phy: apple: atc: Fix coding style in the AUX and tunnel clock code Align the readl_poll_timeout() continuations in the tunnel clock code with the open parenthesis and drop a doubled blank line after the J700 AUX register definitions, as checkpatch asks. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/phy/apple/atc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 359eb62f32e5bb..a45973d05fc8e9 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -1807,7 +1807,6 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode #define J700_AUX_CLAMP_SELECT BIT(8) #define J700_AUX_CLAMP_ENABLE BIT(9) - static int j700_aux_enable(struct apple_atcphy *atcphy) { u32 clamp = J700_AUX_CLAMP_SELECT | J700_AUX_CLAMP_ENABLE; From 02fd81ffbaf00c30a91a030b3d94508e828bfc93 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:40 +1100 Subject: [PATCH 488/631] spi: apple: Reword the hardware CS delay comment Describe what the pre- and post-delay register values do. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/spi/spi-apple.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-apple.c b/drivers/spi/spi-apple.c index 21937bb50ba08d..f18cc6e1b8d0ee 100644 --- a/drivers/spi/spi-apple.c +++ b/drivers/spi/spi-apple.c @@ -112,10 +112,9 @@ #define APPLE_SPI_DELAY_CYCLES GENMASK(31, 16) /* - * These are the values used by AppleSPIMCController's hardware-delay path. - * The pre-delay asserts active-low CS and the post-delay releases it. The - * otherwise undocumented bit 11 is named for its observed role rather than - * assigning it a speculative electrical meaning. + * Hardware CS delays: the pre-delay asserts the active-low CS and the + * post-delay releases it. Bit 11 is undocumented and named for its observed + * role only. */ #define APPLE_SPI_DELAY_PRE_FLAGS (APPLE_SPI_DELAY_PRE_ARM | \ APPLE_SPI_DELAY_SET_CS | \ From b976e71343ab195b5cb4a024d4991fbef348fe56 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:40 +1100 Subject: [PATCH 489/631] dt-bindings: gpu: apple,agx: Match the T8140 register layout On T8140 the GFX and GFX1 coprocessor windows are owned by their "apple,t8140-ascwrap-v6" mailbox providers, and the GPU node only keeps the GFX coprocessor control registers ("asc") and the GPU block ("sgx"), which is all the driver requests. The binding still required a third "asc1" region, so dtbs_check rejected t8140-j700.dtb. Describe the two regions the device tree and the driver use. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/gpu/apple,agx.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Documentation/devicetree/bindings/gpu/apple,agx.yaml b/Documentation/devicetree/bindings/gpu/apple,agx.yaml index cc9c78a7624f8d..e7afc4dd87e8f1 100644 --- a/Documentation/devicetree/bindings/gpu/apple,agx.yaml +++ b/Documentation/devicetree/bindings/gpu/apple,agx.yaml @@ -30,12 +30,10 @@ properties: - const: apple,agx-g14s reg: - minItems: 2 - maxItems: 3 + maxItems: 2 reg-names: - minItems: 2 - maxItems: 3 + maxItems: 2 power-domains: maxItems: 1 @@ -93,12 +91,10 @@ allOf: reg: items: - description: GFX coprocessor control registers - - description: GFX1 coprocessor control registers - description: GPU block MMIO registers reg-names: items: - const: asc - - const: asc1 - const: sgx mboxes: items: From f7c22a922c90b6fd5a4c0476c18b8c1d16e1b55e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:41 +1100 Subject: [PATCH 490/631] arm64: dts: apple: t8140: Fold the trailing soc override into the soc node The display, PMC, pinctrl, I2C, SPI, AOP, ISP and ANE nodes were added through a second "&soc { }" block at the end of the same file, indented as if they were still inside the soc node. Move them into the soc node itself, as the other Apple SoC dtsi files do. The compiled DTB is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 2857d8e7ed9aab..9244c4ab065942 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -1294,8 +1294,6 @@ #mbox-cells = <0>; status = "disabled"; }; - }; -}; &soc { disp0_dart: iommu@412300000 { @@ -1998,6 +1996,7 @@ power-domains = <&ps_ane_sys>; status = "disabled"; }; + }; }; #include "t8140-pmgr.dtsi" From 967f03f59988d8931e07f6b2e35a7ff426280e19 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:18:12 +1100 Subject: [PATCH 491/631] arm64: dts: apple: t8140: Tidy the mailbox and pinctrl comments Shorten the AOP mailbox comments to what the nodes need: the register layout of the setup mailbox lives in the driver, and the interrupt and register numbering of the firmware's own device tree does not help a reader of this one. Also drop a stray blank line in the pinctrl and pmgr nodes. The compiled DTB is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 9244c4ab065942..00640dc340f360 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -753,7 +753,6 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; - aic: interrupt-controller@301000000 { compatible = "apple,t8140-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; @@ -1543,7 +1542,6 @@ pinmux = , ; }; - }; avd: avd@381080000 { From 0f8fb21346cc0d8e934b3cf8755a9e98d9cb4b90 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:49:11 +1100 Subject: [PATCH 492/631] iommu: apple-dart: Fix the KUnit test after deferred root publication apple_dart_hw_sync_locked() gained a defer_unknown argument when the early TLB sync learnt to skip locked streams whose roots are not mapped yet, but the KUnit test kept calling it with two arguments, so CONFIG_APPLE_DART_KUNIT_TEST no longer builds. The before-attach case also still expected that early sync to fail with -EIO, which is exactly what the change removed. Pass false where the test exercises the attach-time publication, expect the early sync to succeed, and add a case that checks both sides of the new argument: the early sync leaves a stream with a half-known root alone, while the attach-time sync still refuses it. With this the whole suite passes again under QEMU. Fixes: c361702757bf ("iommu/apple-dart: defer publication until locked roots are known") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/iommu/apple-dart-test.c | 39 +++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 2998c873747222..86ed79c8d0c1d4 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -47,9 +47,12 @@ static void apple_dart_test_before_attach(struct kunit *test) KUNIT_EXPECT_EQ(test, atomic_long_read(&domain->stream_maps[0].sidmap[0]), 0L); KUNIT_EXPECT_EQ(test, readl(dart->regs + DART_T8020_STREAM_COMMAND), 0U); - /* Once attached, missing root mappings must still report an error. */ + /* + * A stream whose locked roots are not mapped yet is left to attach, + * which publishes the complete page table. + */ KUNIT_EXPECT_EQ(test, apple_dart_domain_add_streams(domain, &master), 0); - KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), -EIO); + KUNIT_EXPECT_EQ(test, apple_dart_iotlb_sync_map(&domain->domain, 0, PAGE_SIZE), 0); pm_runtime_disable(dart->dev); free_io_pgtable_ops(ops); @@ -93,6 +96,37 @@ static void apple_dart_test_locked_handoff(struct kunit *test) KUNIT_EXPECT_MEMEQ(test, live, ours, sizeof(live)); } +static void apple_dart_test_sync_unknown_roots(struct kunit *test) +{ + struct apple_dart_stream_map stream = {}; + struct io_pgtable_cfg cfg = {}; + struct apple_dart *dart; + u64 live[] = { 0x1001, 0, 0, 0 }; + u64 ours[] = { 0, 0x2001, 0, 0 }; + + dart = kunit_kzalloc(test, sizeof(*dart), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, dart); + dart->hw = &apple_dart_hw_t8110; + dart->version = 0x0202; + dart->num_streams = 1; + dart->pgsize = sizeof(live); + spin_lock_init(&dart->lock); + /* The live root is mapped, its ownership shadow is not yet. */ + dart->locked_ttbr[0][0] = live; + stream.dart = dart; + __set_bit(0, stream.sidmap); + cfg.apple_dart_cfg.n_ttbrs = 1; + cfg.apple_dart_cfg.ttbr[0] = ours; + + /* An early TLB sync skips the stream and leaves the root alone... */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, true), 0); + KUNIT_EXPECT_EQ(test, live[0], 0x1001ULL); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); + /* ...but publishing at attach still requires both. */ + KUNIT_EXPECT_EQ(test, apple_dart_hw_sync_locked(&cfg, &stream, false), -EIO); + KUNIT_EXPECT_EQ(test, live[1], 0ULL); +} + static void apple_dart_test_firmware_roots(struct kunit *test) { struct apple_dart_stream_map stream = {}; @@ -377,6 +411,7 @@ static void apple_dart_test_gated_irq(struct kunit *test) static struct kunit_case apple_dart_test_cases[] = { KUNIT_CASE(apple_dart_test_before_attach), KUNIT_CASE(apple_dart_test_locked_handoff), + KUNIT_CASE(apple_dart_test_sync_unknown_roots), KUNIT_CASE(apple_dart_test_firmware_roots), KUNIT_CASE(apple_dart_test_gated_commands), KUNIT_CASE(apple_dart_test_resume_commands_failure), From 3a6c8f5257301432cacb94d46590ef4b84387c6a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 15:48:35 +1100 Subject: [PATCH 493/631] soc: apple: rtkit: Do not warn when buffers are retained on purpose apple_rtkit_free_retaining_buffers() prints a warning every time it is called. Its callers use it deliberately for firmware that is left running: the T8140 NVMe controller on every unbind after its firmware reached RUN, the DCP when it has pinned a live firmware session, and the MTP helper for a session adopted from the bootloader. None of these is a problem, yet each unbind left a warning in the log. A caller that gets here after a failed shutdown is better placed to report that failure, so log the retained buffer sizes at debug level and say in the kernel-doc that a running co-processor is an expected case. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/rtkit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 20e3255cdd8c95..6ca0ed30003993 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -1514,10 +1514,10 @@ void apple_rtkit_free_retaining_buffers(struct apple_rtkit *rtk) * Retain shared buffers, including custom allocation contexts, rather * than returning their storage to an allocator while it may be live. */ - dev_warn(rtk->dev, - "RTKit: retaining the shared buffers of an unstopped co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", - rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, - rtk->oslog_buffer.size, rtk->syslog_buffer.size); + dev_dbg(rtk->dev, + "RTKit: retaining the shared buffers of a running co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", + rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, + rtk->oslog_buffer.size, rtk->syslog_buffer.size); kfree(rtk->syslog_msg_buffer); apple_rtkit_release_rx(rtk); kfree(rtk); From ffdbd6258296c142406df820d331aed0cf135e96 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:40 +1100 Subject: [PATCH 494/631] ASoC: apple: t8140-aop-audio: Format with rustfmt make rustfmtcheck flags one dev_warn!() call that is wider than the kernel's rustfmt configuration allows. Reformat it. Also move the comment on the float conversion's exponent onto its own line above the statement. rustfmt had aligned its continuation line under the end of the statement, where it reads as a stray comment. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- sound/soc/apple/t8140_aop_audio.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index 57dad8c960331e..b63c6c8baa0849 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -846,7 +846,11 @@ impl SndSocT8140AopData { } let ret = f(); if let Err(e) = self.lpai_run_put() { - dev_warn!(self.dev, "lpai: unable to idle after raising the fabric: {:?}\n", e); + dev_warn!( + self.dev, + "lpai: unable to idle after raising the fabric: {:?}\n", + e + ); } ret } @@ -1907,8 +1911,8 @@ fn f32_bits_of_q31(v: i32) -> u32 { } let sign = if v < 0 { 1u32 << 31 } else { 0 }; let m = v.unsigned_abs(); - let p = 31 - m.leading_zeros(); // position of the leading one, 0..=31 - // value = m * 2^-31 = 1.f * 2^(p - 31) + // The position of the leading one, 0..=31: |v| / 2^31 = 1.f * 2^(p - 31). + let p = 31 - m.leading_zeros(); let exp = (p as i32 - 31 + 127) as u32; let mant = if p >= 23 { (m >> (p - 23)) & 0x7f_ffff From 49ef3c956345a081c30c55e1f3bd6217ffbfb7fc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:44:17 +1100 Subject: [PATCH 495/631] ASoC: dt-bindings: cirrus,cs42l42: Reference dai-common.yaml The binding lists '#sound-dai-cells' and sound-name-prefix by hand. Reference dai-common.yaml for the common DAI properties instead, as the other codec bindings do, and use unevaluatedProperties so that what it defines is accepted. While at it, put the note on the CS42L83 supplies of Apple Silicon machines in a paragraph of its own and say why they are optional there. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/sound/cirrus,cs42l42.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml index 1d839e28c0f8a7..31cdd309f33ee3 100644 --- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml +++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml @@ -14,8 +14,9 @@ description: It provides a high-dynamic range, stereo DAC for audio playback and a mono high-dynamic-range ADC for audio capture. There is an integrated headset detection block. - CS42L83 systems may have platform-managed supplies without regulator - phandles, as on Apple Silicon boards. + + On Apple Silicon machines the supplies of the CS42L83 are platform-managed + rails that are not described as regulators, so they are optional there. properties: compatible: @@ -31,8 +32,6 @@ properties: '#sound-dai-cells': const: 0 - sound-name-prefix: true - VP-supply: description: VP power supply. @@ -194,7 +193,9 @@ properties: required: - compatible - reg + allOf: + - $ref: dai-common.yaml# - if: properties: compatible: @@ -207,7 +208,7 @@ allOf: - VL-supply - VA-supply -additionalProperties: false +unevaluatedProperties: false examples: - | From 151cd697a76e1a09166920e7f59adf97a9d7c295 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:44:18 +1100 Subject: [PATCH 496/631] ASoC: dt-bindings: apple,t8140-aop-audio: Reference dai-common.yaml The node is a DAI provider, like the MCA. Reference dai-common.yaml as apple,mca.yaml does, and use unevaluatedProperties so that the common DAI properties it defines are accepted. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../devicetree/bindings/sound/apple,t8140-aop-audio.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml index a4e10cbb4deea3..b023080e48c7fc 100644 --- a/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml +++ b/Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml @@ -20,6 +20,9 @@ description: | The remaining audio data moves over the named ADMAC channels while its AOP service is active. The parent AOP binding describes the firmware transport. +allOf: + - $ref: dai-common.yaml# + properties: compatible: const: apple,t8140-aop-audio @@ -96,7 +99,7 @@ required: - apple,chassis-name - apple,machine-kind -additionalProperties: false +unevaluatedProperties: false examples: - | From c87ab9b325cce5e1273b55d3987e9d640ddd14bb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:12:43 +1100 Subject: [PATCH 497/631] soc: apple: pmp: log tunables the boot loader does not provide at debug level While it starts, the PMP firmware asks for tunables by name, and the driver answers from the apple,tunable-* properties the boot loader copied from the Apple Device Tree. On T8140 the firmware asks for four names that the Apple Device Tree does not carry (pmc-max-bwr-latency-us, mc-quota-policy, mc-cue-dsid0-quota and lts-config), so every boot logs four "unknown property" lines, although the firmware starts and runs without them. A missing tunable is not something the user can act on. Log it at debug level, where it is still available when bringing up a new SoC. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/pmp.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index be248317cdc783..07baa134460b6c 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -339,7 +339,9 @@ impl PmpData { } size = len as u32; } else { - dev_info!(self.dev, "unknown property {:?}", name_str); + // Not every boot loader provides every tunable the firmware + // asks for, and the firmware starts without them. + dev_dbg!(self.dev, "unknown property {:?}", name_str); } } state.ioreg_entries.push(size, GFP_KERNEL)?; From 35d401cf6b9524de70dc43ec1c98be6a51f90a96 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:31 +1100 Subject: [PATCH 498/631] hwmon: macsmc: drop the check for the undefined apple,cpu-thermal-policy Temperature discovery refuses to run when the hwmon node has an apple,cpu-thermal-policy property. No binding defines that property and no device tree sets it, so the check can never fire. The real guard is the one next to it: discovered channels are not mixed with channels the device tree describes, which thermal zones refer to by index. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index fcf381aa2c1947..8e839373486409 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -698,9 +698,8 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) if (!of_property_read_bool(hwmon->dev->of_node, "apple,discover-temperatures")) return 0; - /* Dynamic channel numbers must not replace a DT-described thermal policy. */ - if (hwmon->temp.count || - of_property_read_bool(hwmon->dev->of_node, "apple,cpu-thermal-policy")) + /* Dynamic channel numbers must not replace DT-described channels. */ + if (hwmon->temp.count) return -EINVAL; /* Bound both allocation and mailbox transactions for invalid firmware data. */ From 2ebea1705b67e515c735b99dc887bc1904cb663e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:17:04 +1100 Subject: [PATCH 499/631] cpufreq: apple-soc: tidy includes and the ->exit() callback The headers the fast die-temperature limit needs were added as a second, separately sorted block. Merge them into the existing alphabetical list. ->exit() is a wrapper that stops the limit and then calls apple_soc_cpufreq_exit(), which needs a forward declaration. Stop the limit at the start of apple_soc_cpufreq_exit() instead and use that as the callback. The order of teardown is unchanged. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/cpufreq/apple-soc-cpufreq.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index c3c13950cbea72..5cbf5d4a8d6b37 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -7,32 +7,31 @@ * Based on scpi-cpufreq.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include #include +#include #include #include #include +#include #include #include +#include +#include +#include #include #include #include #include +#include #include +#include +#include +#include +#include +#include #define APPLE_DVFS_CMD 0x20 #define APPLE_DVFS_CMD_BUSY BIT(31) From 9d5b025537862843c0f6bcdb0f47f8f91d796171 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:20:16 +1100 Subject: [PATCH 500/631] pmdomain: apple: pmp-report: remove the raw DVFS debugfs dump On T8140 the driver creates /sys/kernel/debug/apple-pmp-/dvfs, which prints four undocumented 16-byte entries of the report region as raw hex. It was a debugging aid: nothing reads it, it sits in a second debugfs root next to apple-pmp-report/, and it contradicts both the comment on the status file and the binding, which only allow documented entries to be read. The status file already shows what the driver knows about (whether the PMP runs and the fast die-temperature efforts). Remove it together with its forward declarations. While at it, drop a duplicate include and a dev_set_drvdata() call whose value nothing reads since report entries look their region up in the RCU list. No functional change outside debugfs. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmp-report.c | 75 ----------------------------- 1 file changed, 75 deletions(-) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index b6e7e4f0c3ce0a..d6da36bc402066 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -68,10 +67,6 @@ struct apple_pmp_report_offsets { u32 fast_die_lanes; }; -struct apple_pmp_report; - -static int apple_pmp_dvfs_create(struct apple_pmp_report *report); - struct apple_pmp_report { struct device *dev; const struct apple_pmp_report_offsets *offsets; @@ -1481,76 +1476,6 @@ static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) return apple_pmp_report_set_state(genpd, false); } -/* ADT DVFS-STATE: four 16-byte entries beginning at entry 8. */ -#define APPLE_PMP_DVFS_OFFSET 0x80 -#define APPLE_PMP_DVFS_STRIDE 16 -#define APPLE_PMP_DVFS_COUNT 4 - -static int apple_pmp_dvfs_show(struct seq_file *seq, void *unused) -{ - struct apple_pmp_report *report = seq->private; - u64 values[APPLE_PMP_DVFS_COUNT][2]; - unsigned int index; - - if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) - return -EAGAIN; - - for (index = 0; index < ARRAY_SIZE(values); index++) { - void __iomem *entry = report->base + APPLE_PMP_DVFS_OFFSET + - index * APPLE_PMP_DVFS_STRIDE; - - values[index][0] = readq(entry); - values[index][1] = readq(entry + sizeof(values[index][0])); - } - - if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) - return -EAGAIN; - - for (index = 0; index < ARRAY_SIZE(values); index++) - seq_printf(seq, "entry%u: 0x%016llx 0x%016llx\n", index + 8, - values[index][0], values[index][1]); - - return 0; -} -DEFINE_SHOW_ATTRIBUTE(apple_pmp_dvfs); - -static void apple_pmp_dvfs_remove(void *directory) -{ - debugfs_remove_recursive(directory); -} - -static int apple_pmp_dvfs_create(struct apple_pmp_report *report) -{ - struct device *dev = report->dev; - struct resource *resource; - struct dentry *directory; - char *name; - int ret; - - if (!of_device_is_compatible(dev->of_node, "apple,t8140-pmp-v2-report")) - return 0; - - resource = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 0); - if (resource_size(resource) < APPLE_PMP_DVFS_OFFSET + - APPLE_PMP_DVFS_COUNT * APPLE_PMP_DVFS_STRIDE) - return dev_err_probe(dev, -EINVAL, "DVFS report exceeds resource\n"); - - name = devm_kasprintf(dev, GFP_KERNEL, "apple-pmp-%s", dev_name(dev)); - if (!name) - return -ENOMEM; - - directory = debugfs_create_dir(name, NULL); - if (IS_ERR(directory)) - return 0; - - ret = devm_add_action_or_reset(dev, apple_pmp_dvfs_remove, directory); - if (ret) - return ret; - - debugfs_create_file("dvfs", 0400, directory, report, &apple_pmp_dvfs_fops); - return 0; -} - static void apple_pmp_report_entry_free(struct apple_pmp_report_entry *ent) { of_node_put(ent->report_node); From 5842994ba4adb770ea5d7f7c116ffd2d8460673f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:20:35 +1100 Subject: [PATCH 501/631] dt-bindings: apple: use the canonical SPDX form in PMP, PMC and report bindings Upstream bindings, and checkpatch, expect the dual licence to be written as "(GPL-2.0-only OR BSD-2-Clause)". The PMP, PMP report and PMC bindings leave out the parentheses; the other T8140 power bindings already use them. Add them. The licence is unchanged. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/devicetree/bindings/power/apple,pmc.yaml | 2 +- .../devicetree/bindings/power/apple,t8140-pmp-report.yaml | 2 +- Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,pmc.yaml b/Documentation/devicetree/bindings/power/apple,pmc.yaml index 91c77695178c49..a57b90806511b1 100644 --- a/Documentation/devicetree/bindings/power/apple,pmc.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmc.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/power/apple,pmc.yaml# diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 0cddaf6136379c..781afa1ad9d9f1 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/power/apple,t8140-pmp-report.yaml# diff --git a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml index 9784c62fba89ec..997761ed5ad4de 100644 --- a/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml +++ b/Documentation/devicetree/bindings/soc/apple/apple,pmp.yaml @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- $id: http://devicetree.org/schemas/soc/apple/apple,pmp.yaml# From 2d11d6ba42eb093a5a4ff8d3e95051a8e0886dd4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:27:48 +1100 Subject: [PATCH 502/631] rtc: macsmc: correct the comment on the whole-second offset With apple,rtc-offset-seconds the offset is stored in whole seconds. set_time() subtracts the counter's whole seconds from the requested time, so the fraction of a second the counter had reached is dropped and the next second starts early: the clock runs up to a second ahead. The comment says the fraction "carries into the new time", which reads as the opposite. Say what happens. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/rtc/rtc-macsmc.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/rtc/rtc-macsmc.c b/drivers/rtc/rtc-macsmc.c index 25d5af5cfbbcd3..1c1c529df82966 100644 --- a/drivers/rtc/rtc-macsmc.c +++ b/drivers/rtc/rtc-macsmc.c @@ -113,7 +113,10 @@ static int macsmc_rtc_set_time(struct device *dev, struct rtc_time *tm) if (rtc->offset_in_seconds) { __le64 off_secs; - /* The counter's fraction of a second carries into the new time */ + /* + * The offset holds whole seconds, so the counter's fraction of a + * second is lost: the clock may run up to a second ahead. + */ off_secs = cpu_to_le64(rtc_tm_to_time64(tm) - (ctr >> RTC_SEC_SHIFT)); ret = nvmem_cell_write(rtc->rtc_offset, &off_secs, sizeof(off_secs)); return ret < 0 ? ret : 0; From 3652af641e65c66ae76098f3504387f7e193d8e1 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:05:55 +1100 Subject: [PATCH 503/631] hwmon: macsmc: account for temperature channels in one helper Move the code that turns a temperature node into a channel into macsmc_hwmon_add_temp_channel(), so that the next patch can change how a node that fails to set up is accounted for in one place, and a unit test can exercise it without an SMC. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/hwmon/macsmc-hwmon.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 8e839373486409..55cf4b09554c49 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -685,6 +685,21 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } +/* Account for the temperature sensor just set up at the end of @temp. */ +static void macsmc_hwmon_add_temp_channel(struct macsmc_hwmon_sensors *temp, + int create_ret) +{ + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + + if (create_ret) + return; + + sensor->attrs = HWMON_T_INPUT; + if (*sensor->label) + sensor->attrs |= HWMON_T_LABEL; + temp->count++; +} + /* Discover reporting-only temperatures without a board-specific key table. */ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) { @@ -970,14 +985,9 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { sensor = &hwmon->temp.sensors[hwmon->temp.count]; - if (!macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor)) { - sensor->attrs = HWMON_T_INPUT; - - if (*sensor->label) - sensor->attrs |= HWMON_T_LABEL; - - hwmon->temp.count++; - } + ret = macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, + key_node, sensor); + macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } for (list = die_keys; list && (ret = macsmc_hwmon_next_key(&list, key));) { From 36994e2b331518db69b809258be8650dd2596895 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:09:26 +1100 Subject: [PATCH 504/631] hwmon: macsmc: keep the channel of a temperature sensor that fails to set up Since the SMC temperatures became thermal sensors, thermal zones refer to them by channel index, and the binding defines that index as the position of the node among the temperature-* nodes. The driver only counts nodes whose key it could look up, so when one fails, for instance because the SMC firmware does not have the key, every later channel moves down by one. Every zone that refers to a later channel then silently reads a different sensor. On T8140 the six CPU die zones use the first six channels, so a missing die key would move every later CPU zone onto its neighbour's sensor and the last one onto a sensor outside the CPU clusters. On a fanless machine that can mean running hot without throttling. Give every temperature node its channel. A sensor that fails to set up keeps its slot with no key; is_visible() hides it, so hwmon creates no attributes and no thermal sensor for it, and only the zones that use that channel fail to register. The config entry stays non-zero, since a zero entry would end the channel list. Say which node failed. The KUnit test sets up three channels with the middle one failing and checks that the third keeps its index and key and that the middle one is hidden. It fails before this change (two channels, the third key in slot 1) and passes after it. Assisted-by: LLM Signed-off-by: Ryan Murray --- MAINTAINERS | 1 + drivers/hwmon/Kconfig | 10 ++++ drivers/hwmon/macsmc-hwmon-test.c | 87 +++++++++++++++++++++++++++++++ drivers/hwmon/macsmc-hwmon.c | 28 ++++++++-- 4 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 drivers/hwmon/macsmc-hwmon-test.c diff --git a/MAINTAINERS b/MAINTAINERS index 50a28de436054c..c0f155b3949711 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2650,6 +2650,7 @@ F: drivers/clk/clk-apple-nco.c F: drivers/cpufreq/apple-soc-cpufreq.c F: drivers/dma/apple-admac.c F: drivers/gpio/gpio-macsmc.c +F: drivers/hwmon/macsmc-hwmon*.c F: drivers/hwmon/macsmc-hwmon.c F: drivers/pmdomain/apple/ F: drivers/pmdomain/apple/.kunitconfig.apple-pmp diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index a2bd15114d3f77..0e60c5e4c8de73 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -1189,6 +1189,16 @@ config SENSORS_MACSMC_HWMON on Apple Silicon devices. Say Y here if you have an Apple Silicon device. +config SENSORS_MACSMC_HWMON_KUNIT_TEST + bool "KUnit tests for the Apple SMC hwmon driver" if !KUNIT_ALL_TESTS + depends on SENSORS_MACSMC_HWMON && KUNIT=y + default KUNIT_ALL_TESTS + help + Unit tests for the channel numbering of the Apple SMC hwmon driver. + Thermal zones refer to temperature channels by index, so a sensor + that cannot be set up must keep its channel; the tests check that. + No SMC is needed. Only useful for kernel developers. + This driver can also be built as a module. If so, the module will be called macsmc-hwmon. diff --git a/drivers/hwmon/macsmc-hwmon-test.c b/drivers/hwmon/macsmc-hwmon-test.c new file mode 100644 index 00000000000000..e7239986ce3c9e --- /dev/null +++ b/drivers/hwmon/macsmc-hwmon-test.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the Apple SMC hwmon driver: channel numbering of the + * temperature sensors, which thermal zones depend on. + * + * Included by macsmc-hwmon.c. + */ + +#include + +/* What macsmc_hwmon_create_sensor() leaves behind for one node. */ +static void macsmc_hwmon_test_add(struct macsmc_hwmon_sensors *temp, + const char *key, int create_ret) +{ + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + + sensor->macsmc_key = _SMC_KEY(key); + strscpy_pad(sensor->label, key, sizeof(sensor->label)); + macsmc_hwmon_add_temp_channel(temp, create_ret); +} + +/* A node that fails to set up keeps its channel; later channels do not move. */ +static void macsmc_hwmon_test_failed_temp_keeps_slot(struct kunit *test) +{ + struct macsmc_hwmon_sensor sensors[3] = {}; + struct macsmc_hwmon hwmon = { + .temp = { .sensors = sensors }, + }; + u32 configs[ARRAY_SIZE(sensors) + 1] = {}; + + macsmc_hwmon_test_add(&hwmon.temp, "Te05", 0); + macsmc_hwmon_test_add(&hwmon.temp, "Te0S", -EINVAL); + macsmc_hwmon_test_add(&hwmon.temp, "Tp01", 0); + + KUNIT_ASSERT_EQ(test, hwmon.temp.count, 3); + KUNIT_EXPECT_EQ(test, sensors[0].macsmc_key, _SMC_KEY("Te05")); + KUNIT_EXPECT_EQ(test, sensors[2].macsmc_key, _SMC_KEY("Tp01")); + + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 0), 0444); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 2), 0444); + + /* A zero entry would end the channel list at the failed sensor. */ + macsmc_hwmon_populate_configs(configs, &hwmon.temp); + KUNIT_EXPECT_NE(test, configs[0], 0); + KUNIT_EXPECT_NE(test, configs[1], 0); + KUNIT_EXPECT_NE(test, configs[2], 0); + KUNIT_EXPECT_EQ(test, configs[3], 0); +} + +static void macsmc_hwmon_test_temp_label(struct kunit *test) +{ + struct macsmc_hwmon_sensor sensors[2] = {}; + struct macsmc_hwmon hwmon = { + .temp = { .sensors = sensors }, + }; + + macsmc_hwmon_test_add(&hwmon.temp, "Tp05", 0); + sensors[1].macsmc_key = _SMC_KEY("Tp09"); + macsmc_hwmon_add_temp_channel(&hwmon.temp, 0); + + KUNIT_ASSERT_EQ(test, hwmon.temp.count, 2); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 0), 0444); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_label, 1), 0); + KUNIT_EXPECT_EQ(test, macsmc_hwmon_is_visible(&hwmon, hwmon_temp, + hwmon_temp_input, 1), 0444); +} + +static struct kunit_case macsmc_hwmon_test_cases[] = { + KUNIT_CASE(macsmc_hwmon_test_failed_temp_keeps_slot), + KUNIT_CASE(macsmc_hwmon_test_temp_label), + {} +}; + +static struct kunit_suite macsmc_hwmon_test_suite = { + .name = "macsmc-hwmon", + .test_cases = macsmc_hwmon_test_cases, +}; + +kunit_test_suite(macsmc_hwmon_test_suite); diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 55cf4b09554c49..94358065a44f16 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -602,6 +602,10 @@ static umode_t macsmc_hwmon_is_visible(const void *data, return 0; } + /* A sensor without a key could not be set up; see macsmc_hwmon_add_temp_channel(). */ + if (!sensor->macsmc_key) + return 0; + /* Sensors only register ro attributes */ if (sensor->attrs & BIT(attr)) return 0444; @@ -685,19 +689,28 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } -/* Account for the temperature sensor just set up at the end of @temp. */ +/* + * Account for the temperature sensor just set up at the end of @temp. + * + * Thermal zones refer to a temperature channel by the index of its node + * among the temperature-* nodes, so every node takes a channel. A sensor + * that could not be set up keeps its channel with no key, which hides it: + * hwmon creates no attributes and no thermal sensor for it, and only the + * zones that use it fail to register, instead of every later channel + * moving onto the wrong sensor. + */ static void macsmc_hwmon_add_temp_channel(struct macsmc_hwmon_sensors *temp, int create_ret) { - struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count]; + struct macsmc_hwmon_sensor *sensor = &temp->sensors[temp->count++]; if (create_ret) - return; + memset(sensor, 0, sizeof(*sensor)); + /* A zero entry would end the list of channels here. */ sensor->attrs = HWMON_T_INPUT; if (*sensor->label) sensor->attrs |= HWMON_T_LABEL; - temp->count++; } /* Discover reporting-only temperatures without a board-specific key table. */ @@ -987,6 +1000,9 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, sensor = &hwmon->temp.sensors[hwmon->temp.count]; ret = macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor); + if (ret) + dev_warn(hwmon->dev, "%pOFn unavailable (%d), temp%u hidden\n", + key_node, ret, hwmon->temp.count + 1); macsmc_hwmon_add_temp_channel(&hwmon->temp, ret); } @@ -1390,6 +1406,10 @@ static struct platform_driver macsmc_hwmon_driver = { }; module_platform_driver(macsmc_hwmon_driver); +#if IS_ENABLED(CONFIG_SENSORS_MACSMC_HWMON_KUNIT_TEST) +#include "macsmc-hwmon-test.c" +#endif + MODULE_DESCRIPTION("Apple Silicon SMC hwmon driver"); MODULE_AUTHOR("James Calligeros "); MODULE_LICENSE("Dual MIT/GPL"); From 9e55fa2b1b7832e9ffbb8aa74a734bfc4dfbf82e Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:00 +1100 Subject: [PATCH 505/631] soc: apple: mcc: factor out disabling and flushing planes Move the loop that stops allocation and flushes each plane out of apple_mcc_quiesce() into apple_mcc_disable_planes(), which takes the number of planes to act on, so that the next patch can use it on the planes enabled before a failure. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index 10f5e6e01e8cd1..d0f6c1362b308e 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -72,6 +72,24 @@ static bool apple_mcc_plane_ready(struct apple_mcc *mcc, unsigned int plane) return false; } +/* Stop allocating on the first @count planes, then write back and invalidate them. */ +static void apple_mcc_disable_planes(struct apple_mcc *mcc, unsigned int count) +{ + unsigned int i; + u32 val; + + for (i = 0; i < count; i++) { + writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); + writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); + } + + for (i = 0; i < count; i++) + if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, + !(val & MCC_CACHE_FLUSH_BUSY), 10, + MCC_FLUSH_TIMEOUT_US)) + dev_err(mcc->dev, "plane %u flush timed out\n", i); +} + static int apple_mcc_enable(struct apple_mcc *mcc) { unsigned int i; @@ -91,26 +109,12 @@ static int apple_mcc_enable(struct apple_mcc *mcc) return 0; } -/* Stop allocating, then write back and invalidate the cache. */ static void apple_mcc_quiesce(struct apple_mcc *mcc) { - unsigned int i; - u32 val; - if (!mcc->enabled) return; - for (i = 0; i < mcc->nr_planes; i++) { - writel(1, mcc->planes[i] + MCC_ALLOC_DISABLE); - writel(1, mcc->planes[i] + MCC_CACHE_FLUSH); - } - - for (i = 0; i < mcc->nr_planes; i++) - if (readl_poll_timeout_atomic(mcc->planes[i] + MCC_CACHE_FLUSH, val, - !(val & MCC_CACHE_FLUSH_BUSY), 10, - MCC_FLUSH_TIMEOUT_US)) - dev_err(mcc->dev, "plane %u flush timed out\n", i); - + apple_mcc_disable_planes(mcc, mcc->nr_planes); mcc->enabled = false; } From 9ee126966f7fec9609961b2de71b9917175e7eda Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:03 +1100 Subject: [PATCH 506/631] soc: apple: mcc: disable the planes already enabled when enabling fails apple_mcc_enable() clears the allocation-disable bit plane by plane and reads it back. If a later plane does not take it, probe returns -EIO with the earlier planes allocating. The driver does not bind, so neither remove() nor shutdown() runs, and nothing stops allocation or writes the cache back before the next kernel or the firmware takes over, which is what the driver promises to do on shutdown. Disable and flush the planes enabled so far, and the one that failed, whose state is unknown, before returning the error. The failure needs a plane whose register does not take the write, which cannot be produced on a working machine or with memory standing in for the registers, so there is no automated test; the normal path is unchanged, and the on-device check is that probe still reports allocation enabled on every plane. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/mcc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/soc/apple/mcc.c b/drivers/soc/apple/mcc.c index d0f6c1362b308e..7b8c2e2220c8ba 100644 --- a/drivers/soc/apple/mcc.c +++ b/drivers/soc/apple/mcc.c @@ -102,6 +102,12 @@ static int apple_mcc_enable(struct apple_mcc *mcc) writel(0, mcc->planes[i] + MCC_ALLOC_DISABLE); if (readl(mcc->planes[i] + MCC_ALLOC_DISABLE)) { dev_err(mcc->dev, "plane %u did not enable allocation\n", i); + /* + * The driver will not bind, so nothing disables these + * planes on shutdown: disable the ones enabled so far, + * and this one, whose state is unknown, now. + */ + apple_mcc_disable_planes(mcc, i + 1); return -EIO; } } From 55630476d074f3db64ff13cee85c4466df4c6d0b Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:10:45 +1100 Subject: [PATCH 507/631] pmdomain: apple: pmc: undo a partial agent enable apple_pmc_agent_enable() changes the agent's bit rail by rail and stops at the first rail that does not take the write. The rails before it keep the change, and pmc->enable[] records the failed one as changed too. For a dynamic voter domain genpd treats a failed power_on() as the domain staying off and never calls power_off(), so the agent stays enabled on those rails for good; a failed disable likewise leaves it half disabled while genpd keeps the domain on. Snapshot the enables before the change and, on failure, put back the rails changed so far and the one that failed, so the agent is in the same state on every rail and that state matches what genpd records. The failure needs a rail enable word that does not take the write, which memory standing in for the registers cannot produce, so the KUnit test added with the next patch covers the successful path only. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/pmc.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 397fce7fc8ad26..5ae62280734b0b 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -151,18 +151,46 @@ static int apple_pmc_enable_update(struct apple_pmc *pmc, unsigned int rail, return apple_pmc_enable_write(pmc, rail); } +/* + * Caller holds pmc->lock. Put the rails up to and including @last back to + * the enables in @prev, after a change failed on @last. + */ +static void apple_pmc_enable_restore(struct apple_pmc *pmc, const u32 *prev, + unsigned int last) +{ + unsigned int rail; + + for (rail = 0; rail <= last; rail++) { + pmc->enable[rail] = prev[rail]; + apple_pmc_enable_write(pmc, rail); + } +} + +/* + * Enable or disable @agent on every rail. If a rail does not take the + * change, undo it on the rails changed so far: genpd keeps a domain whose + * power_on() fails off, and never calls its power_off(), so an agent left + * enabled on some rails would stay enabled for good. + */ static int apple_pmc_agent_enable(struct apple_pmc *pmc, unsigned int agent, bool enable) { + u32 prev[APPLE_PMC_MAX_RAILS]; unsigned long flags; unsigned int rail; int ret = 0; raw_spin_lock_irqsave(&pmc->lock, flags); - for (rail = 0; rail < pmc->hw->rails && !ret; rail++) + memcpy(prev, pmc->enable, sizeof(prev)); + for (rail = 0; rail < pmc->hw->rails; rail++) { ret = apple_pmc_enable_update(pmc, rail, enable ? BIT(agent) : 0, enable ? 0 : BIT(agent)); + if (ret) { + apple_pmc_enable_restore(pmc, prev, rail); + break; + } + } raw_spin_unlock_irqrestore(&pmc->lock, flags); return ret; From 4462a692cba6d95891133825be5843e40b2d21fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:13:13 +1100 Subject: [PATCH 508/631] pmdomain: apple: pmc: restore the AP's own DCS vote when a limit is lifted The DCS limit votes through the AP's agent on behalf of the agents it stops honouring. When the limit is lifted, and when the driver is torn down, apple_pmc_dcs_vote() writes 0 into the AP agent's DCS field. That discards whatever the AP agent voted before Linux took over, so a single limit cycle changes the boot configuration of the memory controller. Read the AP agent's DCS vote at probe, before anything writes it, and put it back instead of 0 whenever no limit applies. Add a KUnit test with memory standing in for the registers. It checks that agent enables only change the agent's own bit, and that after a limit cycle the AP agent's DCS vote is the one it started with; the latter fails without this change (the vote reads 0) and passes with it. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/pmdomain/apple/Kconfig | 11 ++++ drivers/pmdomain/apple/pmc-test.c | 95 +++++++++++++++++++++++++++++++ drivers/pmdomain/apple/pmc.c | 22 ++++++- 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 drivers/pmdomain/apple/pmc-test.c diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 08921e63891fa9..74907bf6e55eb9 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -65,3 +65,14 @@ config APPLE_PMP_REPORT_NEO depends on ARCH_APPLE && OF && PM select PM_GENERIC_DOMAINS default ARCH_APPLE +config APPLE_PMC_KUNIT_TEST + bool "KUnit tests for the Apple PMC voter interface" if !KUNIT_ALL_TESTS + depends on APPLE_PMC && KUNIT=y + default KUNIT_ALL_TESTS + help + Unit tests for the Apple PMC driver. They check that enabling and + disabling a voter agent only changes that agent on each rail, and + that lifting a DCS limit gives the AP agent its own vote back. The + registers are simulated in memory. Only useful for kernel developers. + +endif diff --git a/drivers/pmdomain/apple/pmc-test.c b/drivers/pmdomain/apple/pmc-test.c new file mode 100644 index 00000000000000..3f0e277b212551 --- /dev/null +++ b/drivers/pmdomain/apple/pmc-test.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * KUnit tests for the Apple PMC voter interface: agent enables and the + * AP's DCS vote, with memory standing in for the registers. + * + * Included by pmc.c. + */ + +#include + +#define APPLE_PMC_TEST_RAILS_SIZE 0x4000 + +static struct apple_pmc *apple_pmc_test_alloc(struct kunit *test) +{ + const struct apple_pmc_hw *hw = &apple_pmc_hw_t8140; + struct apple_pmc *pmc; + + pmc = kunit_kzalloc(test, sizeof(*pmc), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pmc); + pmc->hw = hw; + raw_spin_lock_init(&pmc->lock); + pmc->voters = (void __iomem *)kunit_kzalloc(test, + hw->agents * hw->agent_stride, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, (void __force *)pmc->voters); + pmc->rails = (void __iomem *)kunit_kzalloc(test, + APPLE_PMC_TEST_RAILS_SIZE, + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, (void __force *)pmc->rails); + + return pmc; +} + +static void apple_pmc_test_set_dcs_vote(struct apple_pmc *pmc, + unsigned int agent, unsigned int state) +{ + void __iomem *reg = pmc->voters + agent * pmc->hw->agent_stride; + + writel(state << APPLE_PMC_RAIL_SHIFT(pmc->hw->dcs_rail), reg); +} + +/* Enabling and disabling an agent changes its bit on every rail only. */ +static void apple_pmc_test_agent_enable(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + unsigned int rail; + + pmc->enable[0] = BIT(0); + writel(BIT(0), pmc->rails + pmc->hw->rail_enable); + + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, true), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + BIT(1) | (rail ? 0 : BIT(0))); + + KUNIT_ASSERT_EQ(test, apple_pmc_agent_enable(pmc, 1, false), 0); + for (rail = 0; rail < pmc->hw->rails; rail++) + KUNIT_EXPECT_EQ(test, apple_pmc_enable_read(pmc, rail), + rail ? 0 : BIT(0)); +} + +/* Lifting a DCS limit gives the AP's agent back its own vote, not 0. */ +static void apple_pmc_test_dcs_limit_restores_ap_vote(struct kunit *test) +{ + struct apple_pmc *pmc = apple_pmc_test_alloc(test); + const struct apple_pmc_hw *hw = pmc->hw; + + apple_pmc_test_set_dcs_vote(pmc, hw->ap_agent, 2); + apple_pmc_test_set_dcs_vote(pmc, 0, 4); + pmc->enable[hw->dcs_rail] = BIT(0) | BIT(hw->ap_agent); + pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); + KUNIT_ASSERT_EQ(test, pmc->ap_dcs_vote, 2); + + /* A limit of 3 lets the AP vote at most DCS 1 for the PMP's 4. */ + pmc->dcs_state = 3; + apple_pmc_dcs_vote(pmc); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 1); + + pmc->dcs_state = 0; + apple_pmc_dcs_vote(pmc); + KUNIT_EXPECT_EQ(test, apple_pmc_ap_dcs_read(pmc), 2); +} + +static struct kunit_case apple_pmc_test_cases[] = { + KUNIT_CASE(apple_pmc_test_agent_enable), + KUNIT_CASE(apple_pmc_test_dcs_limit_restores_ap_vote), + {} +}; + +static struct kunit_suite apple_pmc_test_suite = { + .name = "apple-pmc", + .test_cases = apple_pmc_test_cases, +}; + +kunit_test_suite(apple_pmc_test_suite); diff --git a/drivers/pmdomain/apple/pmc.c b/drivers/pmdomain/apple/pmc.c index 5ae62280734b0b..d1d37a9917ff40 100644 --- a/drivers/pmdomain/apple/pmc.c +++ b/drivers/pmdomain/apple/pmc.c @@ -92,6 +92,8 @@ struct apple_pmc { struct dentry *debugfs; /* Cooling state of the DCS limit: 0 is none, dcs_max holds DCS at 0. */ unsigned long dcs_state; + /* DCS state the AP's agent voted for before probe, kept without a limit. */ + unsigned int ap_dcs_vote; /* Power cost of each DCS state in mW, if a model is described. */ u32 dcs_power[APPLE_PMC_MAX_STATES]; struct delayed_work dcs_work; @@ -315,16 +317,27 @@ static unsigned int apple_pmc_dcs_demand(struct apple_pmc *pmc) return min(demand, hw->dcs_max); } +/* The DCS state the AP's agent votes for. */ +static unsigned int apple_pmc_ap_dcs_read(struct apple_pmc *pmc) +{ + const struct apple_pmc_hw *hw = pmc->hw; + u32 floor = readl(pmc->voters + hw->ap_agent * hw->agent_stride); + + return (floor & APPLE_PMC_RAIL_MASK(hw->dcs_rail)) >> + APPLE_PMC_RAIL_SHIFT(hw->dcs_rail); +} + /* * Caller holds pmc->lock. Vote through the AP's agent for what the other - * agents ask for, capped by the limit; vote nothing without a limit. + * agents ask for, capped by the limit; without a limit, put back the vote + * the AP's agent had before probe. */ static void apple_pmc_dcs_vote(struct apple_pmc *pmc) { const struct apple_pmc_hw *hw = pmc->hw; void __iomem *reg = pmc->voters + hw->ap_agent * hw->agent_stride; u32 mask = APPLE_PMC_RAIL_MASK(hw->dcs_rail); - unsigned int state = 0; + unsigned int state = pmc->ap_dcs_vote; u32 old, val; if (pmc->dcs_state) @@ -703,6 +716,7 @@ static int apple_pmc_probe(struct platform_device *pdev) "PMC is not responding\n"); pmc->enable[rail] = pmc->saved_enable[rail]; } + pmc->ap_dcs_vote = apple_pmc_ap_dcs_read(pmc); count = of_property_count_u32_elems(np, "apple,static-voters"); if (count > 0) { @@ -797,3 +811,7 @@ module_init(apple_pmc_init); MODULE_DESCRIPTION("Apple SoC PMC voter interface driver"); MODULE_LICENSE("Dual MIT/GPL"); + +#if IS_ENABLED(CONFIG_APPLE_PMC_KUNIT_TEST) +#include "pmc-test.c" +#endif From b8486e0757afd70806f67f27ba2eb23cb1b374a4 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 16:13:40 +1100 Subject: [PATCH 509/631] arm64: dts: apple: t8140-j700: trim measurement notes from comments The PMC node's comment records how the DCS power model and the decision not to limit the DCS rail were measured, down to the benchmark and the game used. Keep what the values are and why no zone uses the limit; the measurements belong in the history, not in the device tree. Two PCIe comments call themselves experimental; describe what the nodes do instead. No functional change: the compiled device tree is identical. Assisted-by: LLM Signed-off-by: Ryan Murray --- arch/arm64/boot/dts/apple/t8140-j700.dts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 033272cdb32d91..942e360f57c4ec 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -424,16 +424,12 @@ &pmc { /* - * System power above DCS state 0 at each DCS state, measured with - * glmark2's terrain scene, GPU-bound, on the J700. + * System power above DCS state 0 at each DCS state, GPU-bound. * - * No thermal zone limits the DCS rail. The PMP already drops its - * DCS vote when the GPU idles, and under load the rail must stay - * where the PMP puts it: a power allocator zone on Tp0t at 95 C - * stepped the limit between states 1 and 4 every 1-2 s in - * Minecraft, swinging the game's GPU submission rate about 2.5x. - * With the limit lifted the rate held within 8% and Tp0t settled - * near 100 C, under the 110 C critical trips of the CPU zones. + * No thermal zone limits the DCS rail: the PMP drops its DCS vote + * when the GPU idles, and a power allocator zone on Tp0t made the + * limit oscillate between states 1 and 4 under GPU load. Without + * it Tp0t stays below the 110 C critical trips of the CPU zones. */ apple,dcs-power-microwatt = <0 919000 1453000 2058000 2407000>; status = "okay"; @@ -1826,7 +1822,7 @@ #include "hwmon-laptop.dtsi" &pcie0 { - /* Experiment: guard stays closed until the retained bootstrap succeeds. */ + /* The PIODMA guard stays closed until the retained bootstrap succeeds. */ apple,piodma = <&pcie0_piodma_diag>; iommu-map = <0x100 &pcie0_dart 1 2>; iommu-map-mask = <0xffff>; @@ -1848,7 +1844,7 @@ }; }; -/* Experimental SID17 bootstrap; arena ownership lasts until full reset. */ +/* SID17 bootstrap; the arena stays owned until a full reset. */ &pcie0_dart { status = "okay"; apple,dma-range = /bits/ 64 <0 0x1000000000>; From 928a56130d44d1157d8de60f5e34c35491bf1e61 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 510/631] wifi: mt7932: drop the diagnostic module parameters legacy_irq_test, poll_test and reset_test were bring-up switches. They forced INTx, replaced interrupts with polling of the rings, and ran an extra function reset and firmware boot before the interface was registered. All three default to off, nothing sets them, and none of them is needed to run or debug the station. Remove them with the code only they reached: the polled completion wait, mt_transport_polled() and mt_reset_retained(). With the defaults the driver behaves as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 3 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 106 ++---------------- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 - .../net/wireless/mediatek/mt7932/recovery.c | 2 +- 4 files changed, 11 insertions(+), 102 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 323b5d9999fe81..3489c75681e7ad 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -14,8 +14,7 @@ config MT7932_FULLMAC unless those files are in the initramfs. Requires a trained retained PCI link and a translated SID1 DMA domain. Firmware, runtime DMA and calibration negotiation start automatically; - cfg80211 and userspace select the country and network. No test boot - parameters are required for ordinary station operation. + cfg80211 and userspace select the country and network. Once DMA pointers are published the module is pinned until hardware reset. Normal unload is refused, including with active Bluetooth. A failed probe reset retains the binding and DMA; a successful checked diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 6c5de74b39b33e..658edef3723941 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -41,22 +41,6 @@ static int mt_initialize(struct mt7932 *m); static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id); static void mt_shutdown(struct pci_dev *pdev); -static bool legacy_irq_test; -module_param(legacy_irq_test, bool, 0444); -MODULE_PARM_DESC(legacy_irq_test, "Explicit diagnostic use of standard PCI INTx instead of MSI"); -static bool poll_test; -module_param(poll_test, bool, 0444); -MODULE_PARM_DESC(poll_test, "Diagnostic ROM/firmware polling with endpoint IRQ generation masked"); -static bool reset_test; -module_param(reset_test, bool, 0444); -MODULE_PARM_DESC(reset_test, "Qualify one FLR and fresh firmware epoch before publishing an interface, retaining old DMA"); - -bool mt_transport_polled(void) -{ - return poll_test; -} - - bool mt_rf_allowed(struct mt7932 *m) { return !READ_ONCE(m->stopping) && READ_ONCE(m->rf_ready) && !READ_ONCE(m->reg_pending) && @@ -313,7 +297,7 @@ static void mt_service(struct mt7932 *m) for (i = 0; i < ARRAY_SIZE(m->rx); i++) if (m->rx[i].count) mt_rx_drain(m, &m->rx[i]); - if (READ_ONCE(m->running) && !poll_test) + if (READ_ONCE(m->running)) mt_write(m, W + 0x204, m->irq_mask); } @@ -395,22 +379,9 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo m->reply_length = 0; spin_unlock_irqrestore(&m->response_lock, flags); ret = mt_submit(m, q, ret); - if (!ret && wait) { - if (poll_test) { - unsigned long deadline = jiffies + msecs_to_jiffies(5000); - - do { - mt_service(m); - if (completion_done(&m->response)) - break; - usleep_range(100, 200); - } while (time_before(jiffies, deadline)); - if (!completion_done(&m->response)) - ret = -ETIMEDOUT; - } else if (!wait_for_completion_timeout(&m->response, msecs_to_jiffies(5000))) { - ret = -ETIMEDOUT; - } - } + if (!ret && wait && + !wait_for_completion_timeout(&m->response, msecs_to_jiffies(5000))) + ret = -ETIMEDOUT; if (READ_ONCE(m->stopping)) ret = -ESHUTDOWN; if (ret == -ETIMEDOUT) { @@ -504,7 +475,7 @@ static int mt_dma_setup(struct mt7932 *m) u32 glo; mt_write(m, W + 0x204, 0); - mt_write(m, 0x10188, poll_test ? 0 : 0xff); + mt_write(m, 0x10188, 0xff); mt_rmw(m, W + 0x208, BIT(0) | BIT(2) | BIT(15) | BIT(21) | BIT(27) | BIT(28), 0); ret = mt_poll(m, W + 0x208, BIT(1) | BIT(3), 0, 100000); if (ret) @@ -598,7 +569,7 @@ static int mt_dma_setup(struct mt7932 *m) mt_write(m, W + 0x2f0, 0x8032800a); mt_rmw(m, 0x2120, 0, BIT(1)); mt_write(m, W + 0x1f4, 0xffff); - mt_write(m, W + 0x204, poll_test ? 0 : IRQ_MASK); + mt_write(m, W + 0x204, IRQ_MASK); dev_info(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); return 0; @@ -636,8 +607,8 @@ static int mt_rom_gate(struct mt7932 *m) dev_info(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); } - dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK mode=%s IRQ=%llu (no firmware download or RF yet)\n", - poll_test ? "polled-test" : "interrupt", m->interrupts); + dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", + m->interrupts); return 0; } @@ -1113,63 +1084,6 @@ static int mt_initialize(struct mt7932 *m) return ret; } -static int mt_reset_retained(struct mt7932 **epoch) -{ - struct mt7932 *old = *epoch, *fresh; - struct pci_dev *pdev = old->pdev; - int irq = pci_irq_vector(pdev, 0), ret; - - /* No interface/work producers have been published at this test gate. */ - WRITE_ONCE(old->running, false); - mt_write(old, W + 0x204, 0); - devm_free_irq(&pdev->dev, irq, old); - old->irq_requested = false; - ret = mt_ownership(old); - if (!ret) - ret = mt_dma_stop(old); - if (!ret) - ret = mt_function_reset(old); - if (ret) - return ret; - old->dma_owned = false; - if (old->module_pinned) { - old->module_pinned = false; - module_put(THIS_MODULE); - } - dev_info(&pdev->dev, "RESET_TEST: function reset complete; retaining old DMA\n"); - fresh = devm_kzalloc(&pdev->dev, sizeof(*fresh), GFP_KERNEL); - if (!fresh) - return -ENOMEM; - fresh->pdev = pdev; - fresh->bar = old->bar; - fresh->irq_mask = IRQ_MASK; - fresh->vectors_allocated = true; - ether_addr_copy(fresh->mac, old->mac); - init_completion(&fresh->response); - init_completion(&fresh->cal_response); - init_completion(&fresh->reset_retry); - mutex_init(&fresh->command_mutex); - spin_lock_init(&fresh->response_lock); - *epoch = fresh; - pci_set_drvdata(pdev, fresh); - ret = mt_ownership(fresh); - if (ret) - return ret; - ret = mt_poll(fresh, 0xf0140, BIT(4), BIT(4), 200000); - if (ret) - return ret; - mt_write(fresh, W + 0x204, 0); - ret = devm_request_threaded_irq(&pdev->dev, irq, mt_irq, mt_irq_thread, - IRQF_ONESHOT | IRQF_SHARED, "mt7932", fresh); - if (ret) - return ret; - fresh->irq_requested = true; - ret = mt_initialize(fresh); - if (!ret) - dev_info(&pdev->dev, "RESET_TEST: fresh ROM and firmware gates passed\n"); - return ret; -} - /* A failed PCI reset cannot be turned into successful removal: devres would * free still-owned DMA. Keep the remove callback (and therefore its device, * mappings and module text) alive. This explicit retry does not run hardware @@ -1307,7 +1221,7 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto retain; mt_write(m, W + 0x204, 0); - ret = pci_alloc_irq_vectors(pdev, 1, 1, legacy_irq_test ? PCI_IRQ_INTX : PCI_IRQ_ALL_TYPES); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); if (ret < 0) goto retain; m->vectors_allocated = true; @@ -1338,8 +1252,6 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) irq->chip->name, irq->hwirq); } ret = mt_initialize(m); - if (!ret && reset_test) - ret = mt_reset_retained(&m); if (!ret) ret = mt_register_regulatory_gate(m); if (!ret) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 1568592b8a34b1..b6e7925d48763d 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -254,6 +254,4 @@ void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); void mt_station_update_rate(struct mt7932 *m, const u8 *mac); -bool mt_transport_polled(void); - #endif diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index f0b7cc9714c241..cdb327bb27e39f 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -192,7 +192,7 @@ int mt_recovery_gate(struct mt7932 *m) int ret; /* mt_runtime_start() completed configuration before entering this stage. */ - if (mt_transport_polled() || m->runtime_epoch) + if (m->runtime_epoch) return -EINVAL; for (i = 0; i < ARRAY_SIZE(queue_config); i++) { ret = mt_request(m, 0x6b, true, true, false, queue_config[i], 32); From 8833786cbaad7251d605f0d65cb37043ba37c595 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 511/631] wifi: mt7932: remove the unused station rate query mt_station_update_rate() queried the firmware for the transmit rate of the associated access point. It has never been called: the reply comes from a transmit vector that the firmware shares between stations, so station reports leave the transmit bitrate out. Remove it, the reply parser and the station fields that only it used. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Makefile | 2 +- drivers/net/wireless/mediatek/mt7932/link.c | 3 - drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 -- drivers/net/wireless/mediatek/mt7932/rate.h | 29 --------- .../wireless/mediatek/mt7932/station_query.c | 65 ------------------- 5 files changed, 1 insertion(+), 102 deletions(-) delete mode 100644 drivers/net/wireless/mediatek/mt7932/station_query.c diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 9e76679c46a8a6..f854d21e5db813 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o -mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o station_query.o +mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index cfe71b2305598b..443a07c7708ca1 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -537,7 +537,6 @@ void mt_connect_work(struct work_struct *work) m->station_connected = jiffies; m->station_signal_valid = false; m->station_rx_rate_valid = false; - m->station_tx_rate_valid = false; spin_unlock(&m->data_lock); mt_cqm_reset(m); } @@ -648,8 +647,6 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, m->connect_center = 0; memset(&m->connect_chandef, 0, sizeof(m->connect_chandef)); m->connection_generation++; - m->station_rate_query_failed = false; - m->station_rate_query_time = jiffies - HZ; m->connect_cancelled = false; m->peer_valid = false; m->connecting = true; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index b6e7925d48763d..c8f1be570b112a 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -162,9 +162,6 @@ struct mt7932 { struct rate_info station_rx_rate; unsigned long station_rx_rate_time; bool station_rx_rate_valid; - struct rate_info station_tx_rate; - unsigned long station_tx_rate_time, station_rate_query_time; - bool station_tx_rate_valid, station_rate_query_failed; /* Connection quality monitor, protected by response_lock. */ struct ewma_mt7932_signal cqm_signal; unsigned long cqm_sample_time; @@ -252,6 +249,5 @@ int mt_set_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, bool enabled, int timeout); void mt_power_work(struct work_struct *work); int mt_power_init(struct mt7932 *m); -void mt_station_update_rate(struct mt7932 *m, const u8 *mac); #endif diff --git a/drivers/net/wireless/mediatek/mt7932/rate.h b/drivers/net/wireless/mediatek/mt7932/rate.h index d570a69f81f441..7e7c5685d40fdb 100644 --- a/drivers/net/wireless/mediatek/mt7932/rate.h +++ b/drivers/net/wireless/mediatek/mt7932/rate.h @@ -65,33 +65,4 @@ static inline bool mt7932_rx_rate(u32 rxv, u8 max_nss, struct rate_info *rate) return true; } -/* Stock CID85 reply: exact version1 body, owned BSS/MAC and pending sequence. - * Byte8's station identity and byte0a remain opaque; no invented WTBL test. - */ -static inline bool mt7932_tx_rate_reply(const struct mt7932_event *event, - u8 sequence, const u8 *peer, u8 max_nss, - struct rate_info *rate) -{ - const u8 *body; - u32 a, b, c, rxv; - - if (!mt7932_event_matches(event, sequence) || event->eid != 0x21 || - event->length != 344) - return false; - body = event->packet + 36; - if (body[0] != 1 || - !(get_unaligned_le32(body + 4) & 1) || body[9] || - memcmp(body + 12, peer, 6)) - return false; - a = get_unaligned_le32(body + 0x9c); - b = get_unaligned_le32(body + 0xa0); - c = get_unaligned_le32(body + 0xa4); - if (((a >> 12) & 15) >= 5) - return false; /* TX HE GI/DCM/allocation fields not qualified by this reply. */ - /* Normalize the independently qualified TX words to the shared fields. */ - rxv = (c & 127) | (((c >> 8) & 7) << 7) | - (((a >> 8) & 7) << 12) | (((b >> 26) & 3) << 15) | - (((a >> 6) & 3) << 22) | (((a >> 12) & 15) << 24); - return mt7932_rx_rate(rxv, max_nss, rate); -} #endif diff --git a/drivers/net/wireless/mediatek/mt7932/station_query.c b/drivers/net/wireless/mediatek/mt7932/station_query.c deleted file mode 100644 index 8382768898172b..00000000000000 --- a/drivers/net/wireless/mediatek/mt7932/station_query.c +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include "mt7932.h" - -/* Research helper, deliberately not called by production station reporting: - * the reply identity is per-peer, but its vector is shared and unattributed. - * Retained while qualifying a real per-peer rate source, not a public metric. - * One serialized query for the owned peer. Keep the command owner until the - * reply has been checked; disconnect may close admission in IRQ context but - * cannot reuse BSS/station resources across this critical section. - */ -void mt_station_update_rate(struct mt7932 *m, const u8 *mac) -{ - struct mt7932_event event; - struct rate_info rate; - u8 body[28] = {}, peer[ETH_ALEN]; - u32 generation; - unsigned long flags; - bool valid = false; - int ret; - - mutex_lock(&m->command_mutex); - spin_lock_irqsave(&m->response_lock, flags); - if (!m->connected || m->disconnecting || !m->peer_valid || !mt_rf_allowed(m) || - m->station_rate_query_failed || (mac && !ether_addr_equal(mac, m->connect_bssid)) || - time_before(jiffies, m->station_rate_query_time + HZ / 2)) { - spin_unlock_irqrestore(&m->response_lock, flags); - goto unlock; - } - generation = m->connection_generation; - ether_addr_copy(peer, m->connect_bssid); - body[0] = m->peer_ids.station[0]; - memcpy(body + 4, peer, ETH_ALEN); - m->station_rate_query_time = jiffies; - spin_unlock_irqrestore(&m->response_lock, flags); - - ret = mt_request(m, 0x85, true, false, true, body, sizeof(body)); - if (!ret) - ret = mt7932_event_parse(m->reply, m->reply_length, &event); - if (!ret) - valid = mt7932_tx_rate_reply(&event, m->seq, peer, m->phy_cap[4], &rate); - if (valid && (rate.bw != RATE_INFO_BW_20 || - (m->connect_channel > 14 && !rate.flags && - (rate.legacy == 10 || rate.legacy == 20 || rate.legacy == 55 || rate.legacy == 110)))) - valid = false; - - spin_lock_irqsave(&m->response_lock, flags); - if (generation == m->connection_generation && m->connected && - !m->disconnecting && m->peer_valid && ether_addr_equal(peer, m->connect_bssid)) { - /* A timed-out command must not turn every station dump into another - * five-second wait. A new connection/module epoch may try again. - */ - if (ret) - m->station_rate_query_failed = true; - spin_lock(&m->data_lock); - m->station_tx_rate_valid = valid; - if (valid) { - m->station_tx_rate = rate; - m->station_tx_rate_time = jiffies; - } - spin_unlock(&m->data_lock); - } - spin_unlock_irqrestore(&m->response_lock, flags); -unlock: - mutex_unlock(&m->command_mutex); -} From 1cb4b7d44c9cb187ecdbe5a43c313dc1cf549c49 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 512/631] wifi: mt7932: report driver progress at debug level The driver logged each firmware boot step, every scan request and completion, every association event and each calibration request at info level. A desktop scans every few seconds, so this filled the kernel log with SCAN_REQUEST, SCAN_SUBMITTED, LINK_EVENT and DISCONNECT_EVENT lines, plus hex dumps of firmware replies and DMA addresses at probe. Log routine progress with dev_dbg() and print_hex_dump_debug(), so it is still available through dynamic debug. Errors and warnings keep their level and text. Also drop two diagnostics only used during bring-up: the dump of the MSI configuration and interrupt hierarchy at probe, and the register and descriptor dump on a command timeout. A timed-out command still collects a late reply before reporting the error, and the hex dump of a rejected calibration upload reply goes away, since the error line already has the fields that matter. A few of the demoted messages read device status registers. Those reads now happen only while the message is enabled; each follows the driver's own accesses to the same block, so nothing depends on them. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../mediatek/mt7932/calibration_transport.c | 13 +- drivers/net/wireless/mediatek/mt7932/data.c | 6 +- .../net/wireless/mediatek/mt7932/frontend.c | 24 +-- drivers/net/wireless/mediatek/mt7932/link.c | 64 ++++--- drivers/net/wireless/mediatek/mt7932/mt7932.c | 156 +++++++----------- drivers/net/wireless/mediatek/mt7932/mt7932.h | 1 - .../net/wireless/mediatek/mt7932/recovery.c | 6 +- .../net/wireless/mediatek/mt7932/startup.c | 22 +-- 8 files changed, 132 insertions(+), 160 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index 1d6d4d6985a45b..fb5a33dd5aa61e 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -124,12 +124,10 @@ int mt_calibration_gate(struct mt7932 *m) get_unaligned_le32(m->reply + 40)) { dev_err(&m->pdev->dev, "WCAL_REPLY_REJECTED: length=%zu eid=%02x ext=%02x\n", m->reply_length, m->reply[28], m->reply[32]); - print_hex_dump(KERN_INFO, "WCAL_RESULT: ", DUMP_PREFIX_OFFSET, 16, 1, - m->reply, min_t(size_t, m->reply_length, 64), false); ret = -EPROTO; goto out; } - dev_info(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); + dev_dbg(&m->pdev->dev, "OWN_WCAL_ACCEPTED: %zu bytes\n", wcal->size); phase = "power-on D6 command/completion"; for (at = 0, group = 0; at < planned; at += count, group++) { count = pieces[at].fragment >> 4; @@ -140,10 +138,10 @@ int mt_calibration_gate(struct mt7932 *m) ret = mt_cal_procedure(m, pieces + at, count, 1, m->preload_version); if (ret) goto out; - dev_info(&m->pdev->dev, "POWER_ON_CAL_GROUP_OK: %u type=%u parameter=%08x fragments=%u\n", - group, pieces[at].type, pieces[at].parameter, count); + dev_dbg(&m->pdev->dev, "POWER_ON_CAL_GROUP_OK: %u type=%u parameter=%08x fragments=%u\n", + group, pieces[at].type, pieces[at].parameter, count); } - dev_info(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); + dev_dbg(&m->pdev->dev, "POWER_ON_CAL_COMPLETE: %u groups, %d fragments\n", group, planned); phase = "queued D7 planning/command/completion"; ret = mt_cal_drain_requests(m, oca); out: @@ -187,7 +185,8 @@ int mt_cal_drain_requests(struct mt7932 *m, const struct firmware *oca) mt7932_cal_request_replies(request), m->preload_version); if (ret) return ret; - dev_info(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", get_unaligned_le32(request + 12)); + dev_dbg(&m->pdev->dev, "CAL_REQUEST_SERVICED: channel=%u\n", + get_unaligned_le32(request + 12)); } return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 9e6a51f8e39878..e4f9d8a9f9cc88 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -313,10 +313,10 @@ int mt_data_prepare(struct mt7932 *m) /* The selected response parser establishes sequence completion only, * not a command-specific firmware status. No payload-status invention. */ - dev_info(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", - m->reply[28], m->reply_length); + dev_dbg(&m->pdev->dev, "MAC_RTS_RESPONSE: sequence matched, EID=%02x length=%zu\n", + m->reply[28], m->reply_length); /* Publish the initialized data_lock and DMA state to failure handlers. */ smp_store_release(&m->data_ready, true); - dev_info(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); + dev_dbg(&m->pdev->dev, "ETHERNET_PREPARED: TX0 retained coherent slots=512, RX translation enabled\n"); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index aa36c68a106912..7b0259bde004a6 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -64,8 +64,8 @@ static void mt_scan_finish_work(struct work_struct *work) } mutex_unlock(&m->command_mutex); if (request) { - dev_info(&m->pdev->dev, "SCAN_FINISHED: seq=%u aborted=%u received-BSS=%llu\n", - cancel[0], info.aborted, m->beacons); + dev_dbg(&m->pdev->dev, "SCAN_FINISHED: seq=%u aborted=%u received-BSS=%llu\n", + cancel[0], info.aborted, m->beacons); cfg80211_scan_done(request, &info); } } @@ -226,8 +226,8 @@ static int mt_scan_submit_next(struct mt7932 *m) ret = mt_request(m, 3, true, true, false, body, sizeof(body)); } if (!ret) - dev_info(&m->pdev->dev, "SCAN_SUBMITTED: seq=%u band=%u channels=%u passive=%u\n", - m->scan_seq, m->scan_band, count, passive); + dev_dbg(&m->pdev->dev, "SCAN_SUBMITTED: seq=%u band=%u channels=%u passive=%u\n", + m->scan_seq, m->scan_band, count, passive); return ret; } @@ -350,9 +350,10 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) if (request->wdev != &m->wdev) return -ENODEV; mt_retry_missing_policy(m); - dev_info_ratelimited(&m->pdev->dev, "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", - request->n_channels, request->n_ssids, request->ie_len, request->flags, - request->duration, request->duration_mandatory); + dev_dbg(&m->pdev->dev, + "SCAN_REQUEST: channels=%u ssids=%d ies=%zu flags=%08x duration=%u/%u\n", + request->n_channels, request->n_ssids, request->ie_len, request->flags, + request->duration, request->duration_mandatory); /* cfg80211 lists each advertised channel at most once. */ if (request->n_channels > MT7932_CHANNELS) return -EOPNOTSUPP; @@ -547,7 +548,8 @@ static void mt_cal_work(struct work_struct *work) m->preload_version); if (ret) break; - dev_info(&m->pdev->dev, "D7_CALIBRATION_COMPLETE: channel=%u\n", get_unaligned_le32(request + 12)); + dev_dbg(&m->pdev->dev, "D7_CALIBRATION_COMPLETE: channel=%u\n", + get_unaligned_le32(request + 12)); } release_firmware(oca); if (!ret) @@ -601,7 +603,8 @@ int mt_register_interface(struct mt7932 *m) free_netdev(netdev); return ret; } - dev_info(&m->pdev->dev, "CFG80211_INTERFACE_READY: %s, RF awaits country policy\n", netdev->name); + dev_dbg(&m->pdev->dev, "CFG80211_INTERFACE_READY: %s, RF awaits country policy\n", + netdev->name); return 0; } @@ -637,7 +640,8 @@ int mt_enable_scan(struct mt7932 *m) return ret; } WRITE_ONCE(m->rf_ready, true); - dev_info(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", m->netdev->name); + dev_dbg(&m->pdev->dev, "CFG80211_SCAN_READY: %s, carrier off until WPA2 authorization\n", + m->netdev->name); schedule_work(&m->cal_work); spin_unlock_irqrestore(&m->response_lock, flags); return 0; diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 443a07c7708ca1..48353f762c22f2 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -113,11 +113,11 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) selector = get_unaligned_le32(event->packet + 40); body = event->packet + 48; length = event->length - 48; - dev_info(&m->pdev->dev, "LINK_EVENT: selector=%02x length=%zu\n", selector, length); + dev_dbg(&m->pdev->dev, "LINK_EVENT: selector=%02x length=%zu\n", selector, length); if (selector == 0x44) { if (length < 5 || (!m->connecting && !m->connected && !m->disconnecting)) return; - dev_info(&m->pdev->dev, "DISCONNECT_EVENT: metadata=%u deferred=%u reason-index=%u\n", + dev_dbg(&m->pdev->dev, "DISCONNECT_EVENT: metadata=%u deferred=%u reason-index=%u\n", body[0], body[3], body[4]); if (m->disconnect_seen) return; @@ -179,18 +179,18 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) * rate-control setting. Bounds were checked by the exporter. */ if (he) - dev_info(&m->pdev->dev, - "ASSOC_HE_CAPABILITIES: epoch=%u rx-map=%04x tx-map=%04x\n", - m->connection_generation, - get_unaligned_le16(he->data + 18), - get_unaligned_le16(he->data + 20)); + dev_dbg(&m->pdev->dev, + "ASSOC_HE_CAPABILITIES: epoch=%u rx-map=%04x tx-map=%04x\n", + m->connection_generation, + get_unaligned_le16(he->data + 18), + get_unaligned_le16(he->data + 20)); } break; } case 0x4d: if (length >= 20) - dev_info(&m->pdev->dev, "ASSOC_START: band=%u width=%u channel=%u extension=%u\n", - get_unaligned_le32(body + 12), body[16], body[17], body[19]); + dev_dbg(&m->pdev->dev, "ASSOC_START: band=%u width=%u channel=%u extension=%u\n", + get_unaligned_le32(body + 12), body[16], body[17], body[19]); if (!mt_channel_event(m, body, length)) { m->connect_error = -EPROTO; complete(&m->assoc_done); @@ -199,9 +199,10 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) break; case 0x40: if (length >= 52) - dev_info(&m->pdev->dev, "ASSOC_REPORT: status=%u band=%u channel=%u bss=%u bmc=%u station=%u peer=%u cipher=%08x\n", - get_unaligned_le16(body), body[11], body[12], body[16], body[17], body[20], body[21], - get_unaligned_le32(body + 48)); + dev_dbg(&m->pdev->dev, + "ASSOC_REPORT: status=%u band=%u channel=%u bss=%u bmc=%u station=%u peer=%u cipher=%08x\n", + get_unaligned_le16(body), body[11], body[12], body[16], body[17], + body[20], body[21], get_unaligned_le32(body + 48)); /* OPEN reports retain cipher as numeric metadata only; it neither * installs a key nor changes the immutable requested security mode. */ @@ -215,12 +216,13 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) m->peer_valid = true; /* HOST decoder confirms the RSN suite namespace, not request enum10. */ m->event_cipher = get_unaligned_le32(body + 48); - dev_info(&m->pdev->dev, "ASSOCIATED: peer=%u station=%u cipher-raw=%u\n", - m->peer_wtbl, body[20], m->event_cipher); + dev_dbg(&m->pdev->dev, "ASSOCIATED: peer=%u station=%u cipher-raw=%u\n", + m->peer_wtbl, body[20], m->event_cipher); break; case 0x42: if (length >= 2) - dev_info(&m->pdev->dev, "CONNECT_REPORT: status=%u peer-owned=%u\n", get_unaligned_le16(body), m->peer_valid); + dev_dbg(&m->pdev->dev, "CONNECT_REPORT: status=%u peer-owned=%u\n", + get_unaligned_le16(body), m->peer_valid); if (length < 2 || get_unaligned_le16(body) || !m->peer_valid) m->connect_error = -ECONNREFUSED; complete(&m->assoc_done); @@ -354,8 +356,9 @@ static int mt_retire_connection(struct mt7932 *m) m->link_failed = true; /* uncertain ownership: recovery required */ spin_unlock_irqrestore(&m->response_lock, flags); mutex_unlock(&m->command_mutex); - dev_info(&m->pdev->dev, "CONNECTION_RETIRED: status=%d%s\n", ret, - ret ? "; recovery required, DMA ownership retained" : "; old TX drained, six IDs released"); + dev_dbg(&m->pdev->dev, "CONNECTION_RETIRED: status=%d%s\n", ret, + ret ? "; recovery required, DMA ownership retained" : + "; old TX drained, six IDs released"); return ret; } @@ -400,7 +403,7 @@ void mt_connect_work(struct work_struct *work) goto free; } if (!m->connect_bss) { - dev_info(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); + dev_dbg(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); ret = mt_connect_discover(m); if (ret) goto free; @@ -446,7 +449,8 @@ void mt_connect_work(struct work_struct *work) memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); if (ret) goto report; - dev_info(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", m->connect_bssid, m->connect_channel); + dev_dbg(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", + m->connect_bssid, m->connect_channel); /* Genuine D7 requests must remain serviceable during asynchronous waits. */ mutex_unlock(&m->command_mutex); ret = wait_for_completion_timeout(&m->assoc_start, msecs_to_jiffies(5000)) ? 0 : -ETIMEDOUT; @@ -462,8 +466,8 @@ void mt_connect_work(struct work_struct *work) ret = ret ?: READ_ONCE(m->connect_error); if (ret) goto report; - dev_info(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", - m->connect_channel, m->connect_center); + dev_dbg(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", + m->connect_channel, m->connect_center); mt7932_peer_queue(queue, m->peer_ids.station[0], true); ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); if (ret) @@ -556,10 +560,14 @@ void mt_connect_work(struct work_struct *work) netif_wake_queue(m->netdev); } if (m->connect_open) { - dev_info(&m->pdev->dev, "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", m->data_ready); + dev_dbg(&m->pdev->dev, + "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", + m->data_ready); } else { cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_ATOMIC); - dev_info(&m->pdev->dev, "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", m->data_ready); + dev_dbg(&m->pdev->dev, + "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", + m->data_ready); } } else { dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, m->link_failed); @@ -601,9 +609,11 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, mt_retry_missing_policy(m); if (!mt_rf_allowed(m) || !m->bss_active || netdev != m->netdev) return -EAGAIN; - dev_info_ratelimited(&m->pdev->dev, "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", - params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, params->crypto.cipher_group, - params->crypto.n_akm_suites, !!params->crypto.psk, params->mfp, params->ie_len); + dev_dbg(&m->pdev->dev, + "CONNECT_REQUEST: WPA=%u pairwise-count=%d group=%08x AKM-count=%d PMK-present=%u MFP=%u IE-bytes=%zu\n", + params->crypto.wpa_versions, params->crypto.n_ciphers_pairwise, + params->crypto.cipher_group, params->crypto.n_akm_suites, + !!params->crypto.psk, params->mfp, params->ie_len); if (!params->ssid_len || params->ssid_len > 32) return -EINVAL; if (!open && (params->crypto.wpa_versions != NL80211_WPA_VERSION_2 || @@ -680,7 +690,7 @@ void mt_disconnect_work(struct work_struct *work) m->disconnecting = false; cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); spin_unlock_irqrestore(&m->response_lock, flags); - dev_info(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); + dev_dbg(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); } int mt_disconnect(struct wiphy *wiphy, struct net_device *netdev, u16 reason) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 658edef3723941..be636069559b44 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -169,8 +169,8 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) ret = mt7932_cal_null(&m->cal_state, packet + 36, event.length >= 36 ? event.length - 36 : 0, m->smart_version); - dev_info(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", - event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); + dev_dbg(&m->pdev->dev, "CAL_EVENT: seq=%u length=%zu result=%d credits=%u/%u\n", + event.seq, event.length, ret, m->cal_state.received, m->cal_state.expected); if (ret < 0) mt_rf_fail_locked(m, ret); if (ret) @@ -196,9 +196,10 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) memcpy(m->cal_requests[at], packet + 36, 16); m->cal_request_seq[at] = event.seq; m->cal_request_count++; - dev_info(&m->pdev->dev, "CAL_REQUEST: seq=%u words=%08x/%08x/%08x/%08x queued=%u\n", - event.seq, get_unaligned_le32(packet + 36), get_unaligned_le32(packet + 40), - get_unaligned_le32(packet + 44), get_unaligned_le32(packet + 48), m->cal_request_count); + dev_dbg(&m->pdev->dev, "CAL_REQUEST: seq=%u words=%08x/%08x/%08x/%08x queued=%u\n", + event.seq, get_unaligned_le32(packet + 36), + get_unaligned_le32(packet + 40), get_unaligned_le32(packet + 44), + get_unaligned_le32(packet + 48), m->cal_request_count); if (READ_ONCE(m->rf_ready)) schedule_work(&m->cal_work); } @@ -385,38 +386,18 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo if (READ_ONCE(m->stopping)) ret = -ESHUTDOWN; if (ret == -ETIMEDOUT) { - /* Diagnostic only: never turn a polled late reply into IRQ success. */ unsigned int i; - u16 command; - u32 mask, pending; - unsigned int cap = pci_find_capability(m->pdev, PCI_CAP_ID_MSI); - - pci_read_config_word(m->pdev, PCI_COMMAND, &command); - pci_read_config_dword(m->pdev, cap + PCI_MSI_MASK_64, &mask); - pci_read_config_dword(m->pdev, cap + PCI_MSI_PENDING_64, &pending); - dev_info(&m->pdev->dev, "timeout PCI command=%04x MSI mask=%08x pending=%08x\n", - command, mask, pending); - - dev_info(&m->pdev->dev, "timeout gates MAC=%08x WFDMA=%08x SW=%08x\n", - mt_read(m, 0x10188), mt_read(m, W + 0x204), mt_read(m, W + 0x1f4)); + + /* + * Collect a reply that arrived without its interrupt, so that the + * error below reports it. It does not turn the timeout into success. + */ disable_irq(pci_irq_vector(m->pdev, 0)); - for (i = 0; i < ARRAY_SIZE(m->rx); i++) { - struct mt7932_ring *rx = &m->rx[i]; - - if (!rx->count) - continue; - dev_info(&m->pdev->dev, "timeout RX%u ctrl=%08x CPU=%u DMA=%u\n", i, - le32_to_cpu(rx->desc[rx->tail].control), - mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12)); - if (le32_to_cpu(rx->desc[rx->tail].control) & DONE) { - dma_rmb(); - print_hex_dump(KERN_INFO, "ROM RX prefix: ", DUMP_PREFIX_OFFSET, - 16, 1, rx->buffers + rx->tail * RX_STRIDE, 40, false); - } - mt_rx_drain(m, rx); - } - dev_info(&m->pdev->dev, "late polled reply length=%zu eid=%02x (not IRQ success)\n", - m->reply_length, m->reply_length > 28 ? m->reply[28] : 0); + for (i = 0; i < ARRAY_SIZE(m->rx); i++) + if (m->rx[i].count) + mt_rx_drain(m, &m->rx[i]); + dev_dbg(&m->pdev->dev, "late reply length=%zu eid=%02x\n", + m->reply_length, m->reply_length > 28 ? m->reply[28] : 0); enable_irq(pci_irq_vector(m->pdev, 0)); } spin_lock_irqsave(&m->response_lock, flags); @@ -459,7 +440,7 @@ static int mt_ownership(struct mt7932 *m) udelay(2); if ((mt_read(m, 0xfe250) >> 16) != 0x7000) return -EIO; - dev_info(&m->pdev->dev, "DRIVER_OWN_OK power=%08x\n", mt_read(m, 0xe00f0)); + dev_dbg(&m->pdev->dev, "DRIVER_OWN_OK power=%08x\n", mt_read(m, 0xe00f0)); return 0; } @@ -570,8 +551,8 @@ static int mt_dma_setup(struct mt7932 *m) mt_rmw(m, 0x2120, 0, BIT(1)); mt_write(m, W + 0x1f4, 0xffff); mt_write(m, W + 0x204, IRQ_MASK); - dev_info(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", - mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); + dev_dbg(&m->pdev->dev, "DMA_PUBLISHED glo=%08x TX17=%pad RX0=%pad IPC=%pad\n", + mt_read(m, W + 0x208), &m->tx[17].dma, &m->rx[0].dma, &m->ipc_dma); return 0; } @@ -604,11 +585,11 @@ static int mt_rom_gate(struct mt7932 *m) if (m->reply_length < 40 || m->reply[28] != 2 || get_unaligned_le32(m->reply + 32) != addresses[i]) return -EPROTO; - dev_info(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", - addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); + dev_dbg(&m->pdev->dev, "ROM_READ_OK %08x=%08x seq=%u IRQ=%llu\n", + addresses[i], get_unaligned_le32(m->reply + 36), m->seq, m->interrupts); } - dev_info(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", - m->interrupts); + dev_dbg(&m->pdev->dev, "ROM_TRANSPORT_OK IRQ=%llu (no firmware download or RF yet)\n", + m->interrupts); return 0; } @@ -663,8 +644,8 @@ static int mt_download(struct mt7932 *m, const u8 *data, struct mt7932_region *r if (ret) return ret; ret = mt_scatter(m, data + r->offset, r->length); - dev_info(&m->pdev->dev, "%s region target=%08x length=%u scatter=%d\n", - ram ? "RAM" : "PATCH", r->target, r->length, ret); + dev_dbg(&m->pdev->dev, "%s region target=%08x length=%u scatter=%d\n", + ram ? "RAM" : "PATCH", r->target, r->length, ret); return ret; } @@ -728,8 +709,8 @@ static int mt_capability_gate(struct mt7932 *m) count = mt7932_capabilities(body, m->reply_length - 36); if (count < 0) { dev_err(&m->pdev->dev, "invalid capability reply len=%zu parse=%d\n", m->reply_length, count); - print_hex_dump(KERN_INFO, "NIC_CAP_REJECTED: ", DUMP_PREFIX_OFFSET, - 16, 1, m->reply, m->reply_length, false); + print_hex_dump_debug("NIC_CAP_REJECTED: ", DUMP_PREFIX_OFFSET, + 16, 1, m->reply, m->reply_length, false); return count; } for (i = 0; i < count; i++) { @@ -737,9 +718,9 @@ static int mt_capability_gate(struct mt7932 *m) u32 length = get_unaligned_le32(body + offset + 4); const u8 *value = body + offset + 8; - dev_info(&m->pdev->dev, "capability TLV type=%x bytes=%u\n", type, length); + dev_dbg(&m->pdev->dev, "capability TLV type=%x bytes=%u\n", type, length); if (type == 7) { - dev_info(&m->pdev->dev, "firmware MAC %pM\n", value); + dev_dbg(&m->pdev->dev, "firmware MAC %pM\n", value); mac = true; } else if (type == 8) { if (!value[4] || value[4] > 4) { @@ -748,13 +729,13 @@ static int mt_capability_gate(struct mt7932 *m) } memcpy(m->phy_cap, value, 12); m->antenna_mask = (1U << value[4]) - 1; - dev_info(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", - value[4], value[10], value[0], value[1], value[11]); + dev_dbg(&m->pdev->dev, "firmware PHY NSS=%u paths=%02x HT=%u VHT=%u HE=%u\n", + value[4], value[10], value[0], value[1], value[11]); phy = true; } else if (type == 0x38 && length) { m->runtime_cal = value[0]; m->runtime_cal_valid = true; - dev_info(&m->pdev->dev, "RUNTIME_CAL_CAPABILITY: %u\n", value[0]); + dev_dbg(&m->pdev->dev, "RUNTIME_CAL_CAPABILITY: %u\n", value[0]); } else if (type == 0x18 && length >= 4) { m->six_ghz = value[0]; m->six_ghz_valid = value[0] <= 1; @@ -767,10 +748,10 @@ static int mt_capability_gate(struct mt7932 *m) dev_err(&m->pdev->dev, "CAPABILITY_MISSING: MAC=%u PHY=%u\n", mac, phy); return -ENODATA; } - print_hex_dump(KERN_INFO, "NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, - m->reply, m->reply_length, false); - dev_info(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", - m->reply[28], m->seq); + print_hex_dump_debug("NIC_CAP: ", DUMP_PREFIX_OFFSET, 16, 1, + m->reply, m->reply_length, false); + dev_dbg(&m->pdev->dev, "MCU_CAPABILITY_OK eid=%02x seq=%u (no calibration or RF yet)\n", + m->reply[28], m->seq); return 0; } @@ -792,7 +773,7 @@ static int mt_config_gate(struct mt7932 *m) do { mt_tx_clean(&m->tx[17]); if (!m->tx[17].queued) { - dev_info(&m->pdev->dev, "BASIC_MLME_SUBMITTED: TX retired, no semantic ACK or RF startup\n"); + dev_dbg(&m->pdev->dev, "BASIC_MLME_SUBMITTED: TX retired, no semantic ACK or RF startup\n"); return 0; } usleep_range(100, 200); @@ -816,7 +797,7 @@ static int mt_board_gate(struct mt7932 *m) return ret; if (m->reply_length < 44 || m->reply[28] != 0xed || m->reply[32] != 0xc1) return -EPROTO; - dev_info(&m->pdev->dev, "CHIP_UID_QUERY_OK: 8 opaque bytes received, not published\n"); + dev_dbg(&m->pdev->dev, "CHIP_UID_QUERY_OK: 8 opaque bytes received, not published\n"); for (i = 0; i < ARRAY_SIZE(addresses); i++) { put_unaligned_le32(addresses[i], request); ret = mt_request_ext(m, 0xed, 1, true, false, true, request, sizeof(request)); @@ -834,8 +815,8 @@ static int mt_board_gate(struct mt7932 *m) candidate = !(module & 15) ? 2 : !board_byte ? 3 : 1; m->board_type = candidate; m->module_byte = module; - dev_info(&m->pdev->dev, "BOARD_CLASSIFICATION_CAPTURED: candidate=%u module=%02x board-byte=%02x status=%08x/%08x; no buffer branch selected\n", - candidate, module, board_byte, status[0], status[1]); + dev_dbg(&m->pdev->dev, "BOARD_CLASSIFICATION_CAPTURED: candidate=%u module=%02x board-byte=%02x status=%08x/%08x; no buffer branch selected\n", + candidate, module, board_byte, status[0], status[1]); return 0; } @@ -874,7 +855,7 @@ static int mt_precal_gate(struct mt7932 *m) return -EPROTO; } status = get_unaligned_le32(m->reply + 40); - dev_info(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); + dev_dbg(&m->pdev->dev, "PPR_SOURCE2_RESULT: status=%08x\n", status); if (status) return -EREMOTEIO; memset(request, 0, sizeof(request)); @@ -884,17 +865,17 @@ static int mt_precal_gate(struct mt7932 *m) if (m->reply_length < 40 || m->reply[28] != 0xed || m->reply[32] != 0xaa) return -EPROTO; m->smart_version = m->reply[38] & 15; - dev_info(&m->pdev->dev, "SMART_VERSION_REPLY: eid=%02x ext=%02x version=%u\n", - m->reply[28], m->reply[32], m->smart_version); + dev_dbg(&m->pdev->dev, "SMART_VERSION_REPLY: eid=%02x ext=%02x version=%u\n", + m->reply[28], m->reply[32], m->smart_version); request[3] = 9; ret = mt_request(m, 0xbf, true, false, true, request, 20); if (ret) return ret; if (m->reply_length < 56) return -EPROTO; - dev_info(&m->pdev->dev, "CALTYPE_REPLY: eid=%02x ext=%02x H=%u P=%u O=%u V=%u marker=%08x\n", - m->reply[28], m->reply[32], m->runtime_cal, m->reply[37], - m->reply[44], m->reply[38], get_unaligned_le32(m->reply + 48)); + dev_dbg(&m->pdev->dev, "CALTYPE_REPLY: eid=%02x ext=%02x H=%u P=%u O=%u V=%u marker=%08x\n", + m->reply[28], m->reply[32], m->runtime_cal, m->reply[37], + m->reply[44], m->reply[38], get_unaligned_le32(m->reply + 48)); /* M=0/T=3 exact host predicate. Stage metadata only after acceptance; * do not select the legacy absent-marker D6 compatibility fallback. */ @@ -902,7 +883,7 @@ static int mt_precal_gate(struct mt7932 *m) m->reply[37] != 1 || m->reply[44] != 1) return -EPROTO; m->preload_version = m->reply[38]; - dev_info(&m->pdev->dev, "PRECAL_HANDSHAKE_OK: own calibration and RF policy still required\n"); + dev_dbg(&m->pdev->dev, "PRECAL_HANDSHAKE_OK: own calibration and RF policy still required\n"); return 0; } @@ -958,8 +939,8 @@ static int mt_firmware_gate(struct mt7932 *m) dev_err(&m->pdev->dev, "local input mediatek/mt7932/IZUBA_W7932_2.bin has invalid region %d\n", i); goto out; } - dev_info(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", - patch->size, patches, ram->size, regions); + dev_dbg(&m->pdev->dev, "stock containers patch=%zu/%d regions RAM=%zu/%d regions\n", + patch->size, patches, ram->size, regions); phase = "patch semaphore command/reply"; put_unaligned_le32(2, request); ret = mt_command(m, 0x10, request, 4); @@ -986,7 +967,7 @@ static int mt_firmware_gate(struct mt7932 *m) goto out; mt_secure_release(m); } - dev_info(&m->pdev->dev, "PATCH_READY\n"); + dev_dbg(&m->pdev->dev, "PATCH_READY\n"); phase = "OTP command/profile"; ret = mt_otp_byte(m, 0x7a, &value); if (!ret) @@ -1041,7 +1022,7 @@ static int mt_firmware_gate(struct mt7932 *m) ret = mt_poll(m, 0xe00f0, 3, 3, 5000000); if (ret) goto out; - dev_info(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); + dev_dbg(&m->pdev->dev, "FIRMWARE_READY entry=%08x\n", entry); phase = "runtime capability/transport gates"; ret = mt_runtime_start(m); out: @@ -1058,16 +1039,16 @@ void mt_transport_snapshot(struct mt7932 *m) unsigned int i; struct mt7932_ring *tx = &m->tx[17]; - dev_info(&m->pdev->dev, "TRANSPORT: interrupts=%llu packets=%llu pending=%08x mask=%08x TX17 head=%u tail=%u queued=%u DMA=%u\n", - m->interrupts, m->packets, mt_read(m, W + 0x200), mt_read(m, W + 0x204), - tx->head, tx->tail, tx->queued, mt_read(m, tx->reg + 12)); + dev_dbg(&m->pdev->dev, "TRANSPORT: interrupts=%llu packets=%llu pending=%08x mask=%08x TX17 head=%u tail=%u queued=%u DMA=%u\n", + m->interrupts, m->packets, mt_read(m, W + 0x200), mt_read(m, W + 0x204), + tx->head, tx->tail, tx->queued, mt_read(m, tx->reg + 12)); for (i = 0; i < ARRAY_SIZE(m->rx); i++) { struct mt7932_ring *rx = &m->rx[i]; if (rx->count) - dev_info(&m->pdev->dev, "TRANSPORT_RX%u: tail=%u producer=%u DMA=%u control=%08x\n", - i, rx->tail, mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12), - le32_to_cpu(READ_ONCE(rx->desc[rx->tail].control))); + dev_dbg(&m->pdev->dev, "TRANSPORT_RX%u: tail=%u producer=%u DMA=%u control=%08x\n", + i, rx->tail, mt_read(m, rx->reg + 8), mt_read(m, rx->reg + 12), + le32_to_cpu(READ_ONCE(rx->desc[rx->tail].control))); } } @@ -1155,7 +1136,7 @@ static void mt_remove(struct pci_dev *pdev) wiphy_free(m->wiphy); if (m->vectors_allocated) pci_free_irq_vectors(pdev); - dev_info(&pdev->dev, "REMOVE_COMPLETE: host users joined, PCI function reset, DMA retired\n"); + dev_dbg(&pdev->dev, "REMOVE_COMPLETE: host users joined, PCI function reset, DMA retired\n"); /* All coherent allocations/BAR mappings are devres-owned and now safe * to release. No default-domain ownership manipulation is necessary. */ @@ -1230,27 +1211,6 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) goto retain; m->irq_requested = true; - { - struct irq_data *irq = irq_get_irq_data(pci_irq_vector(pdev, 0)); - unsigned int cap = pci_find_capability(pdev, PCI_CAP_ID_MSI); - u16 control, command; - u32 lo, hi, data, mask; - - pci_read_config_word(pdev, PCI_COMMAND, &command); - pci_read_config_word(pdev, cap + PCI_MSI_FLAGS, &control); - pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_LO, &lo); - pci_read_config_dword(pdev, cap + PCI_MSI_ADDRESS_HI, &hi); - pci_read_config_dword(pdev, cap + PCI_MSI_DATA_64, &data); - pci_read_config_dword(pdev, cap + PCI_MSI_MASK_64, &mask); - dev_dbg(&pdev->dev, "PCI command=%04x MSI cap=%x control=%04x addr=%08x:%08x data=%08x\n", - command, cap, control, hi, lo, data); - dev_dbg(&pdev->dev, "MSI mask=%08x MSI-X cap=%x chosen=%s\n", mask, - pci_find_capability(pdev, PCI_CAP_ID_MSIX), - pdev->msix_enabled ? "MSI-X" : pdev->msi_enabled ? "MSI" : "INTx"); - for (; irq; irq = irq->parent_data) - dev_dbg(&pdev->dev, "IRQ %u chip=%s hwirq=%lu\n", irq->irq, - irq->chip->name, irq->hwirq); - } ret = mt_initialize(m); if (!ret) ret = mt_register_regulatory_gate(m); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index c8f1be570b112a..28597a36e90acd 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index cdb327bb27e39f..91ae4b03426088 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -274,7 +274,7 @@ int mt_recovery_gate(struct mt7932 *m) mt_read(m, m->tx[17].reg + 12)); } else { WRITE_ONCE(m->running, true); - dev_info(&m->pdev->dev, "RUNTIME_DMA_READY: full RX epoch, TX17 reset, firmware NORMAL_STATE\n"); + dev_dbg(&m->pdev->dev, "RUNTIME_DMA_READY: full RX epoch, TX17 reset, firmware NORMAL_STATE\n"); } enable_irq(pci_irq_vector(m->pdev, 0)); if (!ret) { @@ -284,8 +284,8 @@ int mt_recovery_gate(struct mt7932 *m) mt7932_capabilities(m->reply + 36, m->reply_length - 36) < 0)) ret = -EPROTO; if (!ret) - dev_info(&m->pdev->dev, "RUNTIME_QUERY_OK: TX17 capability response, IRQ=%llu\n", - m->interrupts); + dev_dbg(&m->pdev->dev, "RUNTIME_QUERY_OK: TX17 capability response, IRQ=%llu\n", + m->interrupts); } return ret; } diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index ec9b70ad6ea148..c29512a4c518ad 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -123,7 +123,7 @@ static int mt_stock_config(struct mt7932 *m) if (ret) goto free; } - dev_info(&m->pdev->dev, "STOCK_CONFIG_SUBMITTED: %u original/PHY-derived entries\n", count); + dev_dbg(&m->pdev->dev, "STOCK_CONFIG_SUBMITTED: %u original/PHY-derived entries\n", count); free: kfree(records); out: @@ -165,9 +165,9 @@ static int mt_publish_policy(struct mt7932 *m, const struct mt7932_reg_snapshot /* Native SETs have no response/atomic policy-commit event. Command * submission is not misrepresented as firmware readback or an ACK. */ - dev_info(&m->pdev->dev, "REGULATORY_POLICY_SUBMITTED: %c%c firmware=%c%c channels=%u (native no-response SETs)\n", - reg->alpha2[0], reg->alpha2[1], reg->domain[0], reg->domain[1], - (reg->length - 12) / 8); + dev_dbg(&m->pdev->dev, "REGULATORY_POLICY_SUBMITTED: %c%c firmware=%c%c channels=%u (native no-response SETs)\n", + reg->alpha2[0], reg->alpha2[1], reg->domain[0], reg->domain[1], + (reg->length - 12) / 8); return 0; } @@ -226,7 +226,7 @@ static int mt_startup_once(struct mt7932 *m) ret = mt_calibration_gate(m); if (!ret) ret = mt_request(m, 0x3e, true, true, false, roam_lock, sizeof(roam_lock)); - dev_info(&m->pdev->dev, "OWN_CALIBRATION_STARTUP_RESULT: %d\n", ret); + dev_dbg(&m->pdev->dev, "OWN_CALIBRATION_STARTUP_RESULT: %d\n", ret); if (!ret) ret = mt_enable_scan(m); if (ret) @@ -433,9 +433,9 @@ static void mt_startup_work(struct work_struct *work) reg.alpha2[0], reg.alpha2[1], generation, ret, m->policy_failed || m->link_failed); else - dev_info(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", - reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], - generation, !m->reg_pending); + dev_dbg(&m->pdev->dev, "REGULATORY_READY: %c%c firmware=%c%c generation=%u current=%u\n", + reg.alpha2[0], reg.alpha2[1], reg.domain[0], reg.domain[1], + generation, !m->reg_pending); spin_unlock_irqrestore(&m->response_lock, flags); next: mutex_unlock(&m->command_mutex); @@ -547,8 +547,8 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request if (READ_ONCE(m->interface_registered)) schedule_work(&m->startup_work); spin_unlock_irqrestore(&m->response_lock, irqflags); - dev_info_ratelimited(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", - reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); + dev_dbg(&m->pdev->dev, "REGULATORY_REQUEST: %c%c initiator=%u generation=%u\n", + reg.alpha2[0], reg.alpha2[1], request->initiator, m->reg_generation); } static const struct cfg80211_ops mt_cfg_ops = { @@ -662,6 +662,6 @@ int mt_register_regulatory_gate(struct mt7932 *m) return ret; WRITE_ONCE(m->interface_registered, true); schedule_work(&m->startup_work); - dev_info(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); + dev_dbg(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); return 0; } From 00e5ad771743bb3a12646a4e5faec13a8fe448c5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 513/631] wifi: mt7932: disable link power management only once Probe disabled ASPM and clock PM on the link twice. The first call fails the probe, so that a remove and rescan cannot leave the radio on a link policy it was never tested with. The second call could then only succeed, and its warning for a failure was unreachable. Keep the first call and merge the two comments. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index be636069559b44..0d1e7de5eb71a8 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1160,17 +1160,16 @@ static int mt_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = apple_piodma_radio_check(pdev); if (ret) return ret; - /* A remove/rescan may have recreated link state under the global policy. */ + /* + * Station operation was tested with ASPM and clock PM off on this link, + * and a remove/rescan may have restored them under the global policy. + */ ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); if (ret) return ret; domain = iommu_get_domain_for_dev(&pdev->dev); if (!domain || (domain->type != IOMMU_DOMAIN_DMA && domain->type != IOMMU_DOMAIN_DMA_FQ)) return dev_err_probe(&pdev->dev, -EINVAL, "translated DMA domain required\n"); - /* Station operation was qualified with ASPM and clock PM off on this link. */ - ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); - if (ret) - dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); ret = pcim_enable_device(pdev); if (ret) return ret; From 8091334917efbc43748ed11a9096d7851b2a2639 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 514/631] wifi: mt7932: drop internal document names from comments Several comments named internal design notes (NEO_*_CONTRACT and similar) or used bring-up wording such as "campaign" and "explicit experiment". These names mean nothing outside the project. State the facts they stood for instead. Comment-only change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/association.h | 9 ++++----- drivers/net/wireless/mediatek/mt7932/calibration.h | 2 +- drivers/net/wireless/mediatek/mt7932/frontend.c | 4 ++-- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/packet.h | 4 ++-- drivers/net/wireless/mediatek/mt7932/recovery.c | 10 +++++----- drivers/net/wireless/mediatek/mt7932/reset.c | 2 +- drivers/net/wireless/mediatek/mt7932/scan.h | 6 +++--- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h index eb207921906e14..ba733a885e2617 100644 --- a/drivers/net/wireless/mediatek/mt7932/association.h +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -91,8 +91,8 @@ static inline int mt7932_assoc_capabilities(u8 *out, size_t capacity, } /* Original25G83 pure WPA2 binary-key result: structured type6, not a string. - * NEO_TYPE6_BINARY_PMK_NATIVE_CONTRACT: standard nl80211 PMK maps directly; - * no passphrase API, key derivation, duplicate independent PMK or extra flags. + * The standard nl80211 PMK maps directly; no passphrase API, key derivation, + * duplicate independent PMK or extra flags. */ static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, const u8 *pmk, size_t length) @@ -111,8 +111,8 @@ static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, } /* Explicit OPEN or PMK-only WPA2/CCMP; never an automatic security fallback. - * Caller owns identities/resources and validated BSS state. - * NEO_WPA2_FULLMAC_ASSOCIATION_AND_ETHERNET_CONTRACT. Never log this body. + * Caller owns identities/resources and validated BSS state. Never log this + * body: it carries the PMK. */ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, size_t ssid_length, const u8 pmk[32], const u8 bssid[6], @@ -141,7 +141,6 @@ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, /* NORMAL25G83 original strict keyless OPEN branch, without optional * automatic-authentication policy. Keep this coherent version/auth/ * policy tuple together; no dummy keys or stale security state. - * NEO_NORMAL_STRICT_OPEN_PRODUCER_CONTRACT. */ if (open) { body[1] = 1; diff --git a/drivers/net/wireless/mediatek/mt7932/calibration.h b/drivers/net/wireless/mediatek/mt7932/calibration.h index 85075bce7b255c..f13ee11866bade 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration.h +++ b/drivers/net/wireless/mediatek/mt7932/calibration.h @@ -80,7 +80,7 @@ struct mt7932_cal_segment { }; /* Validate the entire original container before selecting any data to send. - * NEO_MT7932_STOCK_ARTIFACTS_AND_CAL_CONVERSION: raw BLOB, not a transcript. + * The input is the raw BLOB container, not a converted transcript. */ static inline int mt7932_cal_validate(const u8 *data, size_t size) { diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 7b0259bde004a6..0a5e700d16ccd3 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -89,7 +89,7 @@ void mt_scan_event(struct mt7932 *m, const struct mt7932_event *event) if (m->stopping) return; /* Abort completes userspace locally, not by an invented cancel ACK. - * Keep the sequence reserved until an actual old-campaign done arrives. + * Keep the sequence reserved until the old scan's own done event arrives. */ if (event->length >= 37 && m->retired_scan_seq && event->packet[36] == m->retired_scan_seq) { @@ -127,7 +127,7 @@ void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) } /* No command/response lock held. The caller closes new scan admission first. - * Natural EID0d completion fences the campaign; a local cancel alone does not. + * Natural EID0d completion fences the scan; a local cancel alone does not. */ int mt_scan_quiesce(struct mt7932 *m) { diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 48353f762c22f2..033ce178483e6e 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -425,7 +425,7 @@ void mt_connect_work(struct work_struct *work) goto free; /* Initial station policy: framework M=0 branch and no optional flags. * This yields flag2 independently of credentials; no scan-result count - * belongs in this field. The recovered framework uses listen interval20. + * belongs in this field. The original driver uses listen interval 20. * Other platform flag policies remain unsupported, not guessed mappings. */ ret = mt7932_connect_body(body, 856, m->connect_ssid, m->connect_ssid_length, diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index 303cc87fef8b04..a93d04d91fd412 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -14,8 +14,8 @@ struct mt7932_rx_frame { bool translated, group4, signal_valid, rate_valid, unicast; }; -/* NEO_PACKET_RX_GROUPS_AND_DATA_TXP_CONTRACT. Strict non-monitor subset; - * descriptor groups are ordered4,1,2,3,5, not by numerical group identifier. +/* Strict non-monitor subset of the RX descriptor; its groups are ordered + * 4, 1, 2, 3, 5, not by numerical group identifier. */ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_rx_frame *out) { diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index 91ae4b03426088..fc60bc05b13d63 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -6,9 +6,6 @@ static int mt_runtime_scheduler(struct mt7932 *m, bool publish); static int mt_runtime_rebuild(struct mt7932 *m); static void mt_runtime_publish(struct mt7932 *m); static int mt_runtime_ack(struct mt7932 *m, u32 bit); -/* First runtime epoch only. Contracts: NEO_IRQ_SELECTION_AND_RUNTIME_HANDSHAKE - * and NEO_RUNTIME_RING_REBUILD_AND_OBSERVED_PRECAL. - */ static int mt_tx_retire(struct mt7932 *m) { @@ -179,7 +176,7 @@ static int mt_runtime_ack(struct mt7932 *m, u32 bit) int mt_recovery_gate(struct mt7932 *m) { /* Original observed 25F84 CID6b bodies, not PL country tables. - * Explicit experiment; first two action2 field meanings remain opaque. + * The meaning of the first two action2 fields is unknown. */ static const u8 queue_config[3][32] = { {0x10,0x11,2,0,3,0,7,0,0x2f,0,2}, @@ -191,7 +188,10 @@ int mt_recovery_gate(struct mt7932 *m) unsigned int i, phase = 0; int ret; - /* mt_runtime_start() completed configuration before entering this stage. */ + /* + * mt_runtime_start() completed configuration before entering this stage, + * which switches to the runtime ring layout once per firmware boot. + */ if (m->runtime_epoch) return -EINVAL; for (i = 0; i < ARRAY_SIZE(queue_config); i++) { diff --git a/drivers/net/wireless/mediatek/mt7932/reset.c b/drivers/net/wireless/mediatek/mt7932/reset.c index 114da7afd69eb5..d93080513c77f4 100644 --- a/drivers/net/wireless/mediatek/mt7932/reset.c +++ b/drivers/net/wireless/mediatek/mt7932/reset.c @@ -125,7 +125,7 @@ int mt_function_reset(struct mt7932 *m) if (!peer) return mt_function_reset_locked(m); /* The radio functions may share state beyond PCI's advertised FLR scope. - * No qualified contract permits resetting Wi-Fi while Bluetooth is live. + * Resetting Wi-Fi while Bluetooth is live has never been tested. * Trylock avoids inversion with a concurrent sibling probe/remove; hold * it across FLR to prevent Bluetooth starting after the admission check. */ diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index ae26645cc36932..217fc5c9d1e96c 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -10,8 +10,8 @@ /* Selected ordinary TX17/fullmac wildcard scan, not a pre-RF gate. * Caller must own BSS0 and filter cfg80211 DISABLED channels first. NO_IR - * channels require mt7932_scan_passive() before submission. - * NEO_FIRST_CFG80211_SCAN_CONTRACT: 20ms dwell and meaningful MT7932 tail. + * channels require mt7932_scan_passive() before submission. The body uses + * a 20 ms dwell and fills the MT7932-specific tail. */ static inline int mt7932_scan_body(u8 *body, size_t capacity, u8 seq, const u8 *channels, unsigned int count) @@ -87,7 +87,7 @@ static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) } /* This firmware requires its native wildcard template. Directed scans use - * no probe IEs. Forwarding the normal supplicant extras stalled the campaign + * no probe IEs. Forwarding the normal supplicant extras stalled the scan * on J700; arbitrary caller probe-IE carriage remains unsupported here. */ static inline int mt7932_scan_options(u8 body[0x4d4], const u8 *ssid, From a5be96512feffc9911a0c863b1a6e346b7281322 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 515/631] wifi: mt7932: tidy the Kconfig help and file headers Describe what the driver supports and what it needs in the Kconfig help instead of listing bring-up details, use C++ comment style for the SPDX line of every C file, and remove repeated blank lines. Rename the documentation title, which still called the driver a bring-up. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../device_drivers/wifi/mt7932-neo.rst | 68 ++++--------------- drivers/net/wireless/mediatek/mt7932/Kconfig | 35 +++++----- .../mediatek/mt7932/calibration_transport.c | 4 +- drivers/net/wireless/mediatek/mt7932/data.c | 2 +- .../net/wireless/mediatek/mt7932/frontend.c | 5 +- drivers/net/wireless/mediatek/mt7932/link.c | 2 +- drivers/net/wireless/mediatek/mt7932/mt7932.c | 2 - drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 - .../net/wireless/mediatek/mt7932/recovery.c | 2 +- .../net/wireless/mediatek/mt7932/startup.c | 2 +- 10 files changed, 38 insertions(+), 86 deletions(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 8a9ff2b845bc7a..58e97162bdba3e 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -1,6 +1,6 @@ .. SPDX-License-Identifier: GPL-2.0-only -MT7932 radio bring-up on the MacBook Neo +MediaTek MT7932 radio on the MacBook Neo ======================================== J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed @@ -64,10 +64,9 @@ disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware while probing, so build it as a module. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. -Use a saved NetworkManager open or WPA2 profile matching the current interface. -Audio in the desktop session requires BlueZ, PipeWire, its PulseAudio -compatibility service, WirePlumber and Bluetooth audio plugins in the logged-in -user's session. +Use a saved NetworkManager WPA2 profile matching the current interface. +KDE audio requires BlueZ, PipeWire, its PulseAudio compatibility service, +WirePlumber and Bluetooth audio plugins in the logged-in user's session. Local firmware inputs --------------------- @@ -85,70 +84,31 @@ Wi-Fi requests the following files under ``mediatek/mt7932/``: its original BLOB container. * ``config-original.bin``: the bounded J7CF package with the original 64 or 65 configuration records, including all required keys. -* ``policy/world-XZ.bin`` for cfg80211 country 00, or ``policy/.bin`` for - the matching explicit uppercase country code: J7RP containers with - original-derived modes and power tables. A country package is mandatory - whenever that country is set; there is no fallback to the world package. - Relabeling a different country's payload is unsupported. +* ``policy/world-XZ.bin`` and, when available, the matching country package + under ``policy/``: J7RP containers with original-derived modes and power + tables. Relabeling a different country's payload is unsupported. Bluetooth requests these files under ``mediatek/``: * ``MT7932B1_OS_TypeB_2.0.177.0_260706180356.bin`` or the supported fallback - ``MT7932B1_OS_TypeB_0.1.133.0_260128190103.bin`` for ROM1. The fallback is - tried only when the preferred file is absent. ROM0 instead requests - ``MT7932B0_OS_TypeB_0.1.44.0_241001003711.bin``; admitting that variant in - software does not qualify it on physical Neo hardware; -* ``j700-mt7932-btcal.bin`` (the measured input is 388 bytes; the driver accepts - nonempty inputs up to 65535 bytes and relies on HCI setup acceptance); + ``MT7932B1_OS_TypeB_0.1.133.0_260128190103.bin``; +* ``j700-mt7932-btcal.bin`` (388 bytes); * ``MT7932_PTB_IzubaA_0.1.0.0_20251021141303.ptx`` (198 bytes); -* ``j700-mt7932-bdaddr.bin`` (the unit's six-byte Bluetooth address, - most-significant byte first; the driver converts it to HCI byte order). +* ``j700-mt7932-bdaddr.bin`` (the unit's six-byte Bluetooth address). Extraction and packaging require the original local assets. Keep these assets and unit identities out of commits and public test reports. -Keep the installed calibration inputs stable while the Wi-Fi driver is bound: -it requests ``oca2.bin`` again for runtime D7 requests and each association. -``config-original.bin`` and country policies require J7CF/J7RP packaging; copying -unconverted source files to those names does not satisfy the format checks. -If loading Wi-Fi in the initramfs, include every fixed input and the selected -country package there explicitly. Firmware metadata includes a country-file -pattern, but an initramfs builder's glob handling may select only one match; -inspect the resulting image rather than assuming every country is included. - -Load the radio drivers after these inputs are available on the real root. -Wi-Fi reports these distinct failure classes: - -* An initialization probe failure can occur after DMA is published but before - an interface is registered. Only ``initialization failed; DMA retired`` - confirms that the checked reset completed and the unbound driver may be - loaded again. A failed - reset retains the binding and DMA; do not reprobe it. -* ``REGULATORY_BLOCKED`` with ``error=-2`` and ``recovery-required=0`` means - the requested policy file is absent before policy submission. Install that - exact file and bring the interface up or retry a scan/connection. An identical - ``iw reg set`` request alone does not trigger a retry in cfg80211. -* A malformed country package reports ``error=-22`` before submission and - is not retried on interface up/scan/connect. Replace the exact named package - with a valid one, then shut down and start again. An unchanged - ``iw reg set`` request is ignored by cfg80211. If ``recovery-required=1`` - is also present, use the external-reset recovery below instead. -* ``recovery-required=1`` or ``RF_FAILED`` is a latched startup/calibration - failure. Installing a file does not clear it in the bound epoch. A full - external reset is required after a failed or uncertain admission. - Repeatable physical network test -------------------------------- -The manual test activates a saved profile and checks scan, open or WPA2 -authentication, DHCP, gateway replies, DNS, HTTPS and kernel diagnostics. -For a desktop session using Hyprland, use ``--desktop none`` for this network test. -The helper restores the profile's original band even after a failure. It is -not run automatically. Run as root, with a fresh attempt name:: +The manual test activates a saved profile and checks scan, WPA2, DHCP, +gateway replies, DNS, HTTPS, KDE and kernel diagnostics. It is not run +automatically. Run as root, with a fresh attempt name:: python3 tools/testing/selftests/drivers/net/mt7932_e2e.py \ --expected-release "$(uname -r)" --profile "YOUR SAVED PROFILE" \ - --connect --band a --attempt five-ghz --desktop none + --connect --band a --attempt five-ghz Use ``--band bg`` for 2.4 GHz. Require ``WIFI_PHYSICAL_NETWORK_E2E_PASS`` in the JSON artifact printed on completion. The artifact records three traffic diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 3489c75681e7ad..c6120372d1f3b8 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -1,25 +1,26 @@ # SPDX-License-Identifier: GPL-2.0-only config MT7932_FULLMAC - tristate "Experimental J700 MT7932 fullmac wireless support" - depends on PCI && OF && ARM64 && ARCH_APPLE && IOMMU_DMA + tristate "MediaTek MT7932 Wi-Fi on the Apple MacBook Neo (J700)" + depends on PCI && OF && ARM64 && IOMMU_DMA depends on PCIE_APPLE_PIODMA_DIAG && CFG80211 select FW_LOADER help - Native support for the built-in J700 Wi-Fi endpoint: managed - open-network or WPA2-CCMP station operation on 2.4 GHz at 20 MHz and channels 36..48 - at 20/40/80 MHz, with a software bandwidth limit of 80 MHz. - Requires original firmware, own calibration and original-derived - country policy packages in addition to the cfg80211 regulatory database. - The device is probed before the root filesystem is mounted, so say M - unless those files are in the initramfs. - Requires a trained retained PCI link and a translated SID1 DMA domain. - Firmware, runtime DMA and calibration negotiation start automatically; - cfg80211 and userspace select the country and network. - Once DMA pointers are published the module is pinned until hardware - reset. Normal unload is refused, including with active Bluetooth. - A failed probe reset retains the binding and DMA; a successful checked - reset retires ownership before releasing probe resources. - Other MediaTek devices and other machines are not matched. + Support for the Wi-Fi function of the MediaTek MT7932 radio in the + Apple MacBook Neo (J700). The firmware runs association and the + WPA2-PSK handshake; the driver is a cfg80211 station on 2.4 GHz and + on the 5 GHz UNII-1, -2, -2e and -3 channels that the regulatory + domain permits. Radar (DFS) channels are scanned passively but not + joined. Other MediaTek devices and other machines are not matched. + + The driver needs the original firmware, the unit's own calibration + and the country policy packages under mediatek/mt7932/, besides the + cfg80211 regulatory database. The device is probed before the root + filesystem is mounted, so say M unless those files are in the + initramfs. + + Once the device has been given DMA addresses, the module cannot be + unloaded. A function that fails to initialize and cannot be reset + stays bound and keeps its DMA memory. config MT7932_RF_KUNIT_TEST bool "KUnit tests for MT7932 terminal RF failure" if !KUNIT_ALL_TESTS diff --git a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c index fb5a33dd5aa61e..4efd5bed774798 100644 --- a/drivers/net/wireless/mediatek/mt7932/calibration_transport.c +++ b/drivers/net/wireless/mediatek/mt7932/calibration_transport.c @@ -1,8 +1,6 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" - - /* Called only by the serialized command owner, never from the RX consumer. */ int mt_cal_procedure(struct mt7932 *m, struct mt7932_cal_piece *pieces, unsigned int count, unsigned int logical, u8 context_version) diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index e4f9d8a9f9cc88..99ae52977b5823 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static bool mt_bss_tx_allowed(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 0a5e700d16ccd3..8d873a4379bfa2 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static void mt_scan_finish_work(struct work_struct *work); @@ -15,9 +15,6 @@ struct mt7932 *mt_from_wiphy(struct wiphy *wiphy) return *(struct mt7932 **)wiphy_priv(wiphy); } - - - static void mt_scan_finish_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, scan_finish_work); diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 033ce178483e6e..c7aeec6f50d6c8 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static int mt_peer_reserve(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 0d1e7de5eb71a8..6e5409af6253d3 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -781,7 +781,6 @@ static int mt_config_gate(struct mt7932 *m) return -ETIMEDOUT; } - static int mt_board_gate(struct mt7932 *m) { const u32 addresses[] = {0x70, 0x160}; @@ -1052,7 +1051,6 @@ void mt_transport_snapshot(struct mt7932 *m) } } - static int mt_initialize(struct mt7932 *m) { int ret; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 28597a36e90acd..fc5c70c6796ae6 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -2,7 +2,6 @@ #ifndef MT7932_H #define MT7932_H - #include #include #include @@ -31,7 +30,6 @@ #include "rate.h" #include "capabilities.h" - #define W 0xd4000 #define H 0xd6000 #define C 0xd7000 diff --git a/drivers/net/wireless/mediatek/mt7932/recovery.c b/drivers/net/wireless/mediatek/mt7932/recovery.c index fc60bc05b13d63..31168bfa2b489a 100644 --- a/drivers/net/wireless/mediatek/mt7932/recovery.c +++ b/drivers/net/wireless/mediatek/mt7932/recovery.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" static int mt_tx_retire(struct mt7932 *m); diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index c29512a4c518ad..6cd0afc035abd2 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +// SPDX-License-Identifier: GPL-2.0-only #include #include "mt7932.h" From 3c7054cdb57969962ec5644ea415e328588291cc Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 516/631] Bluetooth: mt7932: drop the enable module parameter enable=0 made probe return -ENODEV, to keep the Bluetooth function unbound while the transport was being brought up. Nothing sets it any more, and a function can be kept unbound with the driver core's driver_override or by blacklisting the module instead. Remove the parameter and the documentation that mentions it. Assisted-by: LLM Signed-off-by: Ryan Murray --- Documentation/networking/device_drivers/wifi/mt7932-neo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index 58e97162bdba3e..a5f0f16a9d46fa 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -59,7 +59,7 @@ makes the driver core unbind Bluetooth before Wi-Fi; unloading DMA pointers, and refuses function reset while its Bluetooth sibling is live. The probe rejects a function that is not cold, which is what now prevents a reprobe after a failed or uncertain -admission; ``enable=0`` leaves the function unbound. The transport also +admission. The transport also disables ASPM on the shared link. Like the Wi-Fi driver, it reads its firmware while probing, so build it as a module. The tested Wi-Fi driver owns function 0 and Bluetooth owns function 1. From ce76446506166b3aa00a78d6c079d2c5341f0c52 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 517/631] Bluetooth: mt7932: report setup progress at debug level The driver logged eight lines at info level while probing, among them DMA addresses, and another whenever the HCI device was set up, which happens on every power-on because the setup is not persistent. On a transport fault it also printed eleven error lines, one per ring. Log routine progress and the ring state at debug level. Errors keep their level, and a successful controller recovery is still reported at info level. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index dd689d78e98783..4bccdaa92ada17 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -180,7 +180,7 @@ static void bt7932_fault_locked(struct bt7932 *bt, const char *reason) dev_err(&bt->pdev->dev, "terminal transport fault: %s; owned DMA retained, external reset required\n", reason); for (channel = 0; channel < BT7932_CHANNELS; channel++) - dev_err(&bt->pdev->dev, "ring%u host=%u firmware=%u previous=%u\n", channel, + dev_dbg(&bt->pdev->dev, "ring%u host=%u firmware=%u previous=%u\n", channel, bt7932_counter(bt, bt->host_transfer, channel), bt7932_counter(bt, bt->fw_transfer, channel), bt->rings[channel].previous); pci_clear_master(bt->pdev); @@ -412,8 +412,8 @@ static int bt7932_arena(struct bt7932 *bt) context[0x51] = 13; context[0x53] = 2; dma_wmb(); - dev_info(&bt->pdev->dev, "owned SID2 arena=%pad bytes=%zu image=%pad bytes=%zu, both low32\n", - &bt->arena_dma, bt->arena_size, &bt->image_dma, bt->image_size); + dev_dbg(&bt->pdev->dev, "owned SID2 arena=%pad bytes=%zu image=%pad bytes=%zu, both low32\n", + &bt->arena_dma, bt->arena_size, &bt->image_dma, bt->image_size); return 0; } @@ -727,7 +727,7 @@ static int bt7932_download(struct bt7932 *bt) } writel(0x188d1807, bt->bar + BT7932_CONNECTION_SELECTOR); writel(1, bt->bar + BT7932_SEMAPHORE_RELEASE); - dev_info(&bt->pdev->dev, "BT_IMAGE_ACCEPTED response1 boot0->2; image remains retained\n"); + dev_dbg(&bt->pdev->dev, "BT_IMAGE_ACCEPTED response1 boot0->2; image remains retained\n"); return 0; } @@ -828,7 +828,7 @@ static void bt7932_channels(struct bt7932 *bt) bt->last_sleep = 0; WRITE_ONCE(bt->awake, true); WRITE_ONCE(bt->ipc_ready, true); - dev_info(&bt->pdev->dev, "BT_IPC_READY ten retained advertisements;127 RX offers per channel\n"); + dev_dbg(&bt->pdev->dev, "BT_IPC_READY ten retained advertisements;127 RX offers per channel\n"); } static int bt7932_open(struct hci_dev *hdev) @@ -951,7 +951,7 @@ static int bt7932_setup(struct hci_dev *hdev) HCI_INIT_TIMEOUT); if (ret) goto fault; - dev_info(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); + dev_dbg(&bt->pdev->dev, "BT_HCI_SETUP_OK Reset/calibration/PTX/address completed in order\n"); mutex_lock(&bt->lock); if (bt->recovery == BT7932_RECOVERY_REOPENING) dev_info(&bt->pdev->dev, "controller recovered\n"); @@ -1059,7 +1059,7 @@ static int bt7932_iommu(struct bt7932 *bt) if (ret) dev_err(dev, "BT_IOMMU_REJECT 32-bit DMA mask status=%d\n", ret); else - dev_info(dev, "BT_IOMMU_ADMITTED RID101->SID2 matched translated DART\n"); + dev_dbg(dev, "BT_IOMMU_ADMITTED RID101->SID2 matched translated DART\n"); } of_node_put(target); return ret; @@ -1198,7 +1198,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = -ENOMEM; goto fail; } - dev_info(&pdev->dev, "BT_BAR0 resource=%pR\n", &pdev->resource[0]); + dev_dbg(&pdev->dev, "BT_BAR0 resource=%pR\n", &pdev->resource[0]); ret = pci_read_config_dword(pdev, BT7932_BOOT_CONFIG, &boot); if (ret || boot == U32_MAX || (boot & 0xf) || ((boot >> 4) & 7) > 1) { dev_err(&pdev->dev, "BT_IDENTITY_REJECT config_status=%d boot=%#x\n", ret, boot); @@ -1215,7 +1215,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = -ENODEV; goto fail; } - dev_info(&pdev->dev, "BT_COLD_ADMISSION ROM=%u bootstate0 chip7932 SID2\n", bt->rom); + dev_dbg(&pdev->dev, "BT_COLD_ADMISSION ROM=%u bootstate0 chip7932 SID2\n", bt->rom); ret = bt7932_read_inputs(bt); if (ret) goto fail; @@ -1276,7 +1276,7 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret < 0) goto retained_fault; bt->registered = true; - dev_info(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); + dev_dbg(&pdev->dev, "BT_HCI_REGISTERED; setup/discovery/connection remain separate gates\n"); return 0; retained_fault: mutex_lock(&bt->lock); From 1bf729bea5666a8db6a7a5e210ecd9ffa9805b4d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 518/631] Bluetooth: mt7932: remove write-only packet counters tx_packets, rx_packets and diagnostics were counted but never read. The HCI core keeps its own statistics, which the driver updates as before. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 4bccdaa92ada17..63121683026054 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -143,7 +143,6 @@ struct bt7932 { bool stopping, runtime_forbidden; enum bt7932_recovery recovery; enum bt7932_pm_state pm_state; - u64 tx_packets, rx_packets, diagnostics; }; static void bt7932_ring_doorbell(struct bt7932 *bt, unsigned int bit) @@ -586,7 +585,6 @@ static void bt7932_tx_work(struct work_struct *work) ring->host = (ring->host + 1) & BT7932_MASK; bt7932_publish_counter(bt, bt->host_transfer, channel, ring->host); bt7932_ring_doorbell(bt, geometry[channel].doorbell); - bt->tx_packets++; bt->hdev->stat.byte_tx += skb->len; mutex_unlock(&bt->lock); kfree_skb(skb); @@ -631,10 +629,7 @@ static int bt7932_rx_locked(struct bt7932 *bt, unsigned int channel) hci_skb_pkt_type(skb) = geo->packet; skb_put_data(skb, descriptor + 16, length); bt->hdev->stat.byte_rx += length; - bt->rx_packets++; hci_recv_frame(bt->hdev, skb); - } else if (!geo->packet) { - bt->diagnostics++; } slot = (slot + 1) & BT7932_MASK; } From 43c9bda19dc4f9e0406454f461c5b15e75517d42 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 519/631] Bluetooth: mt7932: describe the driver without bring-up wording The file header, two comments and the module description used bring-up wording. Describe the hardware and the behaviour instead, and keep the credit for the transport research. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/mt7932_bt_pcie.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 63121683026054..85e543e11e70d4 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * J700 MT7932 Bluetooth PCIe transport. - * Authors: DJ (DjDeveloperr), Ace (Acelogic), and Ryan Murray. + * Bluetooth transport for the MediaTek MT7932 radio in the Apple MacBook Neo + * (J700), whose Bluetooth controller is the second function of the radio's + * PCIe device. Transport research by DJ (DjDeveloperr), Ace (Acelogic) and + * Ryan Murray. * * Firmware and board inputs are supplied locally. System sleep preserves the * transport arena and uses the firmware quiesce/restore handshake. Unload @@ -736,7 +738,7 @@ static int bt7932_ipc_state(struct bt7932 *bt, u8 *state) value = readl(bt->bar + BT7932_STATE); if (value == U32_MAX) return -EIO; - /* The reviewed early-state rule replaces memory with the BAR byte. */ + /* Until the state in memory passes 1, the BAR register holds it. */ *state = value & 0xff; } return 0; @@ -819,7 +821,7 @@ static void bt7932_channels(struct bt7932 *bt) bt7932_publish_counter(bt, bt->host_transfer, channel, BT7932_DEPTH - 1); bt7932_ring_doorbell(bt, geometry[channel].doorbell); } - /* The reviewed cold IPC transition starts ACTIVE; no initial sleep ACK. */ + /* The IPC starts active after a cold start; no sleep ACK is due. */ bt->last_sleep = 0; WRITE_ONCE(bt->awake, true); WRITE_ONCE(bt->ipc_ready, true); @@ -1494,7 +1496,7 @@ static struct pci_driver bt7932_driver = { }; module_pci_driver(bt7932_driver); -MODULE_DESCRIPTION("Experimental standalone J700 MT7932 Bluetooth PCIe transport"); +MODULE_DESCRIPTION("MediaTek MT7932 Bluetooth PCIe transport for the Apple MacBook Neo"); MODULE_LICENSE("GPL"); MODULE_FIRMWARE(BT7932_FW_B0); MODULE_FIRMWARE(BT7932_FW_B1); From c1619f4ed9ed534a09361063b26f02672eec86ac Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 520/631] Bluetooth: mt7932: add KUnit tests for packet and board file checks bt7932_hci_length() decides whether a packet the controller placed in a receive ring is handed to the HCI core, and bt7932_validate_ptx() whether a board file is sent to the controller. Cover both with KUnit, including truncated and over-long packets and corrupted board files built from synthetic data. The tests are built into the driver when BT_MTK7932_PCIE_KUNIT_TEST is enabled, so that they reach the static helpers. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/bluetooth/Kconfig | 13 +++ drivers/bluetooth/mt7932_bt_pcie.c | 4 + drivers/bluetooth/mt7932_bt_pcie_test.c | 133 ++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 drivers/bluetooth/mt7932_bt_pcie_test.c diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 10ffa3f3f27dcd..38482dff3ea25e 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -45,6 +45,19 @@ config BT_MTK7932_PCIE To build as a module, choose M. The module is mt7932_bt_pcie. +config BT_MTK7932_PCIE_KUNIT_TEST + bool "KUnit tests for the MT7932 Bluetooth transport" if !KUNIT_ALL_TESTS + depends on KUNIT=y && BT_MTK7932_PCIE + default KUNIT_ALL_TESTS + help + Build KUnit tests into the MT7932 Bluetooth transport. They check, + without radio hardware, that HCI packets whose length does not + match their header are rejected in both directions, and that a + board file with a bad layout or checksum is never sent to the + controller. + + If unsure, say N. + config BT_HCIBTUSB tristate "HCI USB driver" depends on USB diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 85e543e11e70d4..8adff8e60dd020 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1504,3 +1504,7 @@ MODULE_FIRMWARE(BT7932_FW_B1_FALLBACK); MODULE_FIRMWARE(BT7932_PTX); MODULE_FIRMWARE(BT7932_CAL); MODULE_FIRMWARE(BT7932_ADDR); + +#if IS_ENABLED(CONFIG_BT_MTK7932_PCIE_KUNIT_TEST) +#include "mt7932_bt_pcie_test.c" +#endif diff --git a/drivers/bluetooth/mt7932_bt_pcie_test.c b/drivers/bluetooth/mt7932_bt_pcie_test.c new file mode 100644 index 00000000000000..748dcabde731cc --- /dev/null +++ b/drivers/bluetooth/mt7932_bt_pcie_test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the checks mt7932_bt_pcie applies to HCI packets and to the + * board file before either reaches the controller or the HCI core. Built as + * part of mt7932_bt_pcie.c, so the tests reach its static helpers. + */ +#include + +static void bt7932_hci_length_command_test(struct kunit *test) +{ + /* HCI_Reset (0x0c03) and a command carrying two parameter bytes. */ + static const u8 reset[] = { 0x03, 0x0c, 0x00 }; + static const u8 command[] = { 0x1a, 0xfc, 0x02, 0x11, 0x22, 0x33 }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_COMMAND_PKT, reset, sizeof(reset))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, reset, 2)); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 5)); + /* One byte short of, and one byte past, the declared length. */ + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 4)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_COMMAND_PKT, command, 6)); +} + +static void bt7932_hci_length_event_test(struct kunit *test) +{ + /* Command Complete for HCI_Reset with status 0, then one spare byte. */ + static const u8 event[] = { 0x0e, 0x04, 0x01, 0x03, 0x0c, 0x00, 0xff }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 6)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 5)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 7)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_EVENT_PKT, event, 1)); +} + +static void bt7932_hci_length_data_test(struct kunit *test) +{ + /* ACL and ISO lengths are 16-bit; ISO keeps two flag bits on top. */ + static const u8 acl[] = { 0x01, 0x20, 0x02, 0x00, 0xaa, 0xbb }; + static const u8 sco[] = { 0x01, 0x00, 0x01, 0xcc }; + static const u8 iso[] = { 0x01, 0x00, 0x01, 0xc0, 0xdd }; + static const u8 long_acl[] = { 0x01, 0x20, 0x00, 0x01 }; + + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, sizeof(acl))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, 5)); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, acl, 3)); + /* A 256-byte ACL payload must not be satisfied by its header alone. */ + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ACLDATA_PKT, long_acl, sizeof(long_acl))); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_SCODATA_PKT, sco, sizeof(sco))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_SCODATA_PKT, sco, 3)); + KUNIT_EXPECT_TRUE(test, bt7932_hci_length(HCI_ISODATA_PKT, iso, sizeof(iso))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_ISODATA_PKT, iso, 4)); +} + +static void bt7932_hci_length_type_test(struct kunit *test) +{ + static const u8 packet[] = { 0x00, 0x00, 0x00, 0x00 }; + + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(HCI_VENDOR_PKT, packet, sizeof(packet))); + KUNIT_EXPECT_FALSE(test, bt7932_hci_length(0, packet, sizeof(packet))); +} + +#define BT7932_TEST_PTX_SIZE 198 + +/* A well-formed container with synthetic record contents. */ +static void bt7932_test_ptx(u8 *data) +{ + static const u32 types[] = { 0x101, 0x201, 0x301, 0x401 }; + static const u32 sizes[] = { 14, 44, 44, 0 }; + u32 offset = 96, sum, i, j; + + memset(data, 0, BT7932_TEST_PTX_SIZE); + memcpy(data, "BLOB", 4); + put_unaligned_le32(96, data + 4); + put_unaligned_le16(1, data + 8); + put_unaligned_le16(ARRAY_SIZE(types), data + 10); + for (i = 0; i < ARRAY_SIZE(types); i++) { + u8 *record = data + 16 + i * 20; + + for (j = 0, sum = 0; j < sizes[i]; j++) { + data[offset + j] = 0x40 + i * 16 + j; + sum += data[offset + j]; + } + put_unaligned_le32(types[i], record); + put_unaligned_le32(offset, record + 4); + put_unaligned_le32(sizes[i], record + 8); + put_unaligned_le32(sum, record + 12); + offset += sizes[i]; + } +} + +static void bt7932_validate_ptx_test(struct kunit *test) +{ + u8 *data = kunit_kzalloc(test, BT7932_TEST_PTX_SIZE + 1, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, data); + bt7932_test_ptx(data); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), 0); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE - 1), -EINVAL); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE + 1), -EINVAL); + + /* A payload byte that no longer matches its record's checksum. */ + data[120]++; + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + bt7932_test_ptx(data); + data[0] = 'b'; + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + /* A record of the wrong type in the first slot. */ + bt7932_test_ptx(data); + put_unaligned_le32(0x201, data + 16); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); + + /* A record length other than the one its type has. */ + bt7932_test_ptx(data); + put_unaligned_le32(45, data + 16 + 2 * 20 + 8); + KUNIT_EXPECT_EQ(test, bt7932_validate_ptx(data, BT7932_TEST_PTX_SIZE), -EINVAL); +} + +static struct kunit_case bt7932_test_cases[] = { + KUNIT_CASE(bt7932_hci_length_command_test), + KUNIT_CASE(bt7932_hci_length_event_test), + KUNIT_CASE(bt7932_hci_length_data_test), + KUNIT_CASE(bt7932_hci_length_type_test), + KUNIT_CASE(bt7932_validate_ptx_test), + {} +}; + +static struct kunit_suite bt7932_test_suite = { + .name = "mt7932-bt-pcie", + .test_cases = bt7932_test_cases, +}; + +kunit_test_suite(bt7932_test_suite); From 8e1a7cdc39fb4325e16e072e84ce8bd6aface01f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:25:23 +1100 Subject: [PATCH 521/631] selftests: mt7932: fail on routine driver messages at info level The radio drivers report scans, association and setup steps at debug level. Make the physical network test fail when either driver logs at info or notice level while it runs, so that a return of per-scan log spam is caught on the hardware. The check reads "dmesg --level=notice,info" and counts the messages of mt7932-fullmac and mt7932_bt_pcie newer than the last kernel message before the test. Add unit tests for the parsing. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../selftests/drivers/net/mt7932_e2e.py | 27 ++++++++++++++++++- .../selftests/drivers/net/mt7932_e2e_test.py | 22 +++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e.py b/tools/testing/selftests/drivers/net/mt7932_e2e.py index 258baebce12f34..f66ccaac201a67 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e.py @@ -25,6 +25,12 @@ r'SCAN_INTERFACE_SETUP_FAILED|RF_FAILED|recovery-required=1|WARNING:|' r'UBSAN:|hung_task|blocked for more than|WIFI_FLR_REFUSED_OR_FAILED', re.I) +# Both radio drivers report progress at debug level, so a message of theirs at +# info or notice level during scans, association or traffic is log spam. +driver_message = re.compile(r'^\[\s*(\d+\.\d+)\] (?:mt7932-fullmac|mt7932_bt_pcie) ' + r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]: ', re.M) +log_stamp = re.compile(r'^\[\s*(\d+\.\d+)\]', re.M) + def run(argv, timeout=30): return subprocess.run(argv, text=True, stdout=subprocess.PIPE, @@ -42,6 +48,17 @@ def kernel_ok(): return p.stdout +def last_log_time(log): + """Timestamp of the newest kernel log message, or 0.0 if there is none.""" + stamps = log_stamp.findall(log) + return float(stamps[-1]) if stamps else 0.0 + + +def driver_info_messages(log, since): + """Radio driver messages in a ``dmesg --level=notice,info`` log after @since.""" + return [m.group(0) for m in driver_message.finditer(log) if float(m.group(1)) > since] + + def dns_name(packet, offset): """Decode a bounded DNS name, rejecting bad offsets and compression cycles.""" labels, seen, end, size = [], set(), None, 1 @@ -178,8 +195,9 @@ def main(): compatible = Path('/sys/firmware/devicetree/base/compatible').read_bytes().split(b'\0') require(b'apple,j700' in compatible and b'apple,t8140' in compatible, 'Wrong device') out.mkdir(mode=0o700) + started = None try: - kernel_ok() + started = last_log_time(kernel_ok()) profile = run(['nmcli', '-g', 'connection.uuid', 'connection', 'show', 'id', args.profile]) profile_uuid = profile.stdout.strip() require(profile.returncode == 0 and re.fullmatch(r'[0-9a-f-]{36}', profile_uuid), @@ -272,6 +290,13 @@ def main(): final_log.returncode != 0 or result['kernel_faults']): result['result'] = 'WIFI_NETWORK_E2E_FAILED' result['error'] = 'Final kernel log unavailable or contains a driver fault' + info_log = run(['dmesg', '--level=notice,info']) + if started is not None and info_log.returncode == 0: + result['driver_info_messages'] = len(driver_info_messages(info_log.stdout, started)) + if result['result'] in ('WIFI_SCAN_OBSERVED', 'WIFI_PHYSICAL_NETWORK_E2E_PASS') and ( + info_log.returncode != 0 or result.get('driver_info_messages')): + result['result'] = 'WIFI_NETWORK_E2E_FAILED' + result['error'] = 'Radio drivers logged routine events at info level' (out / 'result.json').write_text(json.dumps(result, indent=2) + '\n') print(json.dumps(result, indent=2)) print('ARTIFACT=' + str(out / 'result.json')) diff --git a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py index 22b45b154f3655..75d341cad257ff 100644 --- a/tools/testing/selftests/drivers/net/mt7932_e2e_test.py +++ b/tools/testing/selftests/drivers/net/mt7932_e2e_test.py @@ -182,5 +182,27 @@ def test_new_fault_markers_are_terminal(self): self.assertIsNotNone(e2e.fault.search(message)) +class DriverLogTest(unittest.TestCase): + LOG = ('[ 10.000001] mt7932-fullmac 0000:01:00.0: probe message\n' + '[ 20.500000] usb 1-1: new high-speed USB device\n' + '[ 21.000000] mt7932-fullmac 0000:01:00.0: SCAN_REQUEST: channels=38\n' + '[ 21.250000] mt7932_bt_pcie 0000:01:00.1: controller recovered\n' + '[ 22.000000] Bluetooth: hci0: unrelated core message\n') + + def test_last_log_time(self): + self.assertEqual(e2e.last_log_time(self.LOG), 22.0) + self.assertEqual(e2e.last_log_time(''), 0.0) + + def test_only_radio_messages_after_start_count(self): + self.assertEqual(len(e2e.driver_info_messages(self.LOG, 0.0)), 3) + self.assertEqual(e2e.driver_info_messages(self.LOG, 20.5), + ['[ 21.000000] mt7932-fullmac 0000:01:00.0: ', + '[ 21.250000] mt7932_bt_pcie 0000:01:00.1: ']) + self.assertEqual(e2e.driver_info_messages(self.LOG, 22.0), []) + + def test_quiet_log_has_no_driver_messages(self): + self.assertEqual(e2e.driver_info_messages('[ 1.000000] usb 1-1: reset\n', 0.0), []) + + if __name__ == '__main__': unittest.main() From f6900242a2dbe643ac215d2c3151d8931a36c346 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:15:34 +1100 Subject: [PATCH 522/631] media: apple: isp: Refuse format changes while buffers are allocated S_FMT changes the format of the current channel and the type of the vb2 queue at any time, also while buffers are allocated or the stream runs, for example from a second open file. The running stream keeps the preset it was configured with, but buffer preparation and submission then use the new frame size. With the multiplanar API enabled, an S_FMT for the other buffer type also changes the queue type under the allocated buffers. Return -EBUSY from S_FMT while the queue has buffers, as V4L2 specifies and as uvcvideo does. TRY_FMT and G_FMT are unchanged. Fixes: 329bf83de183 ("media: apple: Add Apple ISP driver") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index f3056e78a0a171..29e51f8166c898 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -757,6 +757,10 @@ static int isp_vidioc_set_format(struct file *file, void *fh, struct isp_preset *preset; int err; + /* The buffers and the queue type depend on the format. */ + if (vb2_is_busy(&isp->vbq)) + return -EBUSY; + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); err = isp_set_preset(isp, fmt, preset); if (err) @@ -812,6 +816,9 @@ static int isp_vidioc_set_format_mplane(struct file *file, void *fh, if (!isp->multiplanar) return -ENOTTY; + if (vb2_is_busy(&isp->vbq)) + return -EBUSY; + preset = isp_select_preset(isp, f->fmt.pix_mp.width, f->fmt.pix_mp.height); err = isp_set_preset(isp, fmt, preset); From 8ae00df355891dda69d68bfc915d0e992f3e681a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:41:51 +1100 Subject: [PATCH 523/631] media: apple: isp: Count existing buffers against the buffer pool size The firmware's buffer pools are configured for ISP_MAX_BUFFERS buffers, and queue_setup() caps the number of buffers to allocate to that many. The cap only looks at the current request. REQBUFS frees the old buffers before it calls queue_setup(), but CREATE_BUFS adds to them: REQBUFS for 16 buffers followed by CREATE_BUFS gives the queue more buffers than the pool holds, and once they are all queued the driver hands them all to the firmware. Count the buffers the queue already has. CREATE_BUFS on a full queue now fails with -ENOBUFS, and one that asks for more than the remaining space gets what fits. REQBUFS behaves as before. vb2_queue.max_num_buffers cannot express the limit: vb2_queue_init() refuses any value below VB2_MAX_FRAME (32). Fixes: 6f0201787b3a ("media: apple: isp: Limit maximal number of buffers") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 29e51f8166c898..ff7e5abca35f78 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -292,13 +292,20 @@ static int isp_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, { struct apple_isp *isp = vb2_get_drv_priv(vq); struct isp_format *fmt = isp_get_current_format(isp); + unsigned int q_num_bufs = vb2_get_num_buffers(vq); /* This is not strictly neccessary but makes it easy to enforce that * at most 16 buffers are submitted at once. ISP on t6001 (FW 12.3) * times out if more buffers are submitted than set in the buffer pool * config before streaming is started. + * + * REQBUFS frees the old buffers first, CREATE_BUFS adds to them, so + * count the buffers the queue already has. */ - *nbuffers = min_t(unsigned int, *nbuffers, ISP_MAX_BUFFERS); + if (q_num_bufs >= ISP_MAX_BUFFERS) + return -ENOBUFS; + *nbuffers = min_t(unsigned int, *nbuffers, + ISP_MAX_BUFFERS - q_num_bufs); if (*num_planes) { if (sizes[0] < fmt->total_size) From d76cc84587990e9721c2d631030e1c28bfd7cdda Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:11 +1100 Subject: [PATCH 524/631] media: apple: isp: Use V4L2_CAP_VIDEO_CAPTURE in the device capabilities device_caps is built from V4L2_BUF_TYPE_VIDEO_CAPTURE, a buffer type, instead of the V4L2_CAP_VIDEO_CAPTURE capability flag. Both are 1, so the reported capabilities are right, but only by coincidence. Use the capability flag. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-v4l2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index ff7e5abca35f78..fc9d8409982e73 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1106,7 +1106,7 @@ int apple_isp_setup_video(struct apple_isp *isp) vdev->queue = vbq; vdev->fops = &isp_v4l2_fops; vdev->ioctl_ops = &isp_v4l2_ioctl_ops; - vdev->device_caps = V4L2_BUF_TYPE_VIDEO_CAPTURE | V4L2_CAP_STREAMING; + vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING; if (isp->multiplanar) vdev->device_caps |= V4L2_CAP_VIDEO_CAPTURE_MPLANE; vdev->v4l2_dev = &isp->v4l2_dev; From 69046897900fdda6ecdaa0c75f9e3eefde7e1e77 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:39 +1100 Subject: [PATCH 525/631] media: apple: isp: Skip the firmware version check for the H17 interface On T8140 the command layouts come from the match data, and the firmware version properties are never compared against the version table. isp_read_fw_version() still parsed them, then returned "unknown" through a separate H17 branch that printed each property at info level, two lines on every boot that the other SoCs log only at debug level. Return from isp_check_firmware_version() before reading the properties on H17. isp_read_fw_version() loses its interface argument and the H17 branch. fw_compat stays ISP_FIRMWARE_V_UNKNOWN on T8140, as before; the other SoCs are unchanged. The bootloader-provided versions remain readable in the device tree. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-drv.c | 33 ++++++---------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 76c28c0dc9f7c9..d11ce3eff54ba0 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -309,8 +309,7 @@ static const char * isp_fw2str(enum isp_firmware_version version) #define ISP_FW_VERSION_MAX_LEN 5 static enum isp_firmware_version isp_read_fw_version(struct device *dev, - const char *name, - enum isp_fw_abi abi) + const char *name) { u32 ver[ISP_FW_VERSION_MAX_LEN]; int size, len; @@ -325,24 +324,6 @@ static enum isp_firmware_version isp_read_fw_version(struct device *dev, ISP_FW_VERSION_MIN_LEN, ISP_FW_VERSION_MAX_LEN); - /* H17 command layouts come from match data, not the legacy version table. */ - if (abi == ISP_FW_ABI_H17) { - switch (len) { - case 3: - dev_info(dev, "%s: %u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2]); - return ISP_FIRMWARE_V_UNKNOWN; - case 4: - dev_info(dev, "%s: %u.%u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2], ver[3]); - return ISP_FIRMWARE_V_UNKNOWN; - case 5: - dev_info(dev, "%s: %u.%u.%u.%u.%u (H17 interface)\n", - name, ver[0], ver[1], ver[2], ver[3], ver[4]); - return ISP_FIRMWARE_V_UNKNOWN; - } - } - switch (len) { case -EINVAL: /* not provided, see isp_check_firmware_version() */ @@ -381,16 +362,18 @@ isp_check_firmware_version(struct apple_isp *isp) struct device *dev = isp->dev; enum isp_firmware_version version, compat; + /* The H17 interface is selected by match data, not by version. */ + if (isp->hw->fw_abi == ISP_FW_ABI_H17) + return ISP_FIRMWARE_V_UNKNOWN; + /* firmware version is just informative */ - version = isp_read_fw_version(dev, "apple,firmware-version", isp->hw->fw_abi); - compat = isp_read_fw_version(dev, "apple,firmware-compat", isp->hw->fw_abi); + version = isp_read_fw_version(dev, "apple,firmware-version"); + compat = isp_read_fw_version(dev, "apple,firmware-compat"); dev_dbg(dev, "ISP firmware-compat: %s (FW: %s)\n", isp_fw2str(compat), isp_fw2str(version)); - /* The H17 interface does not depend on it. */ - if (isp->hw->fw_abi == ISP_FW_ABI_LEGACY && - !of_property_present(dev->of_node, "apple,firmware-compat")) + if (!of_property_present(dev->of_node, "apple,firmware-compat")) dev_warn(dev, "firmware compatibility version not provided, assuming 12.x\n"); return compat; From 5b9720a4604f42132e9c5484c405e70a800ca310 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:16:54 +1100 Subject: [PATCH 526/631] media: apple: isp: Name the H17 flag in boot stage 2 after the interface isp_firmware_boot_stage2() keeps "is this the H17 interface" in a variable called h16, after the H16 boot descriptor that interface takes. Everywhere else the same test is called h17. Rename it so that the condition reads the same in every function; the descriptor keeps its H16 name. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/isp-fw.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index d65c035924a8dd..6d8f4b91416b4c 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -503,8 +503,8 @@ static void isp_write_bootargs_h16(struct apple_isp *isp, void *virt, static int isp_firmware_boot_stage2(struct apple_isp *isp) { - bool h16 = isp->hw->fw_abi == ISP_FW_ABI_H17; - size_t args_size = h16 ? sizeof(struct isp_firmware_bootargs_h16) : + bool h17 = isp->hw->fw_abi == ISP_FW_ABI_H17; + size_t args_size = h17 ? sizeof(struct isp_firmware_bootargs_h16) : sizeof(struct isp_firmware_bootargs); size_t cmd_size = ISP_CMD_AREA_SIZE(isp_num_capmeta(isp)); dma_addr_t args_iova, cmd_iova; @@ -527,7 +527,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) num_ipc_chans); /* Only hand the H16 descriptor to firmware that asks for it. */ - if (h16 && !(desc_flags & ISP_GPIO2_H16_DESCRIPTOR)) { + if (h17 && !(desc_flags & ISP_GPIO2_H16_DESCRIPTOR)) { dev_err(isp->dev, "firmware did not request the H16 boot descriptor (0x%x)\n", desc_flags); @@ -565,7 +565,7 @@ static int isp_firmware_boot_stage2(struct apple_isp *isp) isp->ipc_boot_size = args_size + 0x40 + cmd_size; isp->cmd_virt = cmd_virt; - if (h16) + if (h17) isp_write_bootargs_h16(isp, args_virt, args_offset); else isp_write_bootargs(isp, args_virt, args_iova); From 82b7f829116af96a73b5a8881617d57edf044e45 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:22:56 +1100 Subject: [PATCH 527/631] dt-bindings: media: apple,t8140-isp: Add an example The binding has no example, so dt_binding_check never validates a node against it. Add one that follows the T8140 device tree, with a single sensor preset. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/media/apple,t8140-isp.yaml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml index dea138b415b7ae..59c6c91fb19d72 100644 --- a/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml +++ b/Documentation/devicetree/bindings/media/apple,t8140-isp.yaml @@ -145,3 +145,45 @@ required: - sensor-presets additionalProperties: false + +examples: + - | + #include + #include + + soc { + #address-cells = <2>; + #size-cells = <2>; + + isp@320000000 { + compatible = "apple,t8140-isp", "apple,isp"; + reg = <0x3 0x20000000 0x0 0x1801000>, + <0x3 0x22920000 0x0 0x1000>, + <0x3 0x22921000 0x0 0x100>, + <0x3 0x63b50000 0x0 0x8000>; + reg-names = "coproc", "mbox", "gpio", "wdt"; + interrupt-parent = <&aic>; + interrupts = ; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, + <&ps_isp_cpu_core0>, <&ps_isp_cpu_core1>, + <&ps_isp_fe>, <&ps_isp_dprx>, <&ps_isp_secure>, + <&ps_isp_sts0>, <&ps_isp_sts1>, <&ps_isp_pearl>, + <&ps_isp_be>, <&ps_isp_clr>, <&ps_isp_cvd>, + <&ps_isp_cvm>; + memory-region = <&isp_heap>; + memory-region-names = "heap"; + apple,dart-vm-size = <0x0 0xa0000000>; + apple,platform-id = <0x70>; + apple,temporal-filter = <1>; + + sensor-presets { + preset0 { + apple,config-index = <1>; + apple,input-size = <1920 1080>; + apple,output-size = <1920 1080>; + apple,crop = <0 0 1920 1080>; + }; + }; + }; + }; From 25a7e9d875886f7c455a8e0d9b728ddf41292737 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 17:45:34 +1100 Subject: [PATCH 528/631] media: apple: isp: Add KUnit tests for the videobuf2 queue Move the vb2 queue set-up out of apple_isp_setup_video() into isp_vb2_queue_init() and test it, together with the buffer limit in queue_setup(), without hardware: - the queue initialises with the parameters the driver probes with, so a parameter that vb2_queue_init() rejects fails here instead of at probe, where it leaves the machine without a camera; - REQBUFS gets at most ISP_MAX_BUFFERS buffers; - CREATE_BUFS counts the buffers the queue already has, fails with -ENOBUFS on a full queue and still refuses planes smaller than a frame. The tests are built into isp-v4l2.c, as they call its static functions, and run with tools/testing/kunit/kunit.py run --arch=arm64 \ --kunitconfig= No functional change to the driver. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/media/platform/apple/isp/Kconfig | 10 ++ .../media/platform/apple/isp/isp-v4l2-test.c | 122 ++++++++++++++++++ drivers/media/platform/apple/isp/isp-v4l2.c | 35 +++-- 3 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 drivers/media/platform/apple/isp/isp-v4l2-test.c diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index 942509978fab8c..79a7a075bd54e8 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -27,3 +27,13 @@ config VIDEO_APPLE_ISP_KUNIT_TEST overflow, alignment and overlap with the channel table or other rings. These tests run without starting or accessing the ISP and do not require a supported Apple device or the ISP driver itself. + bool "KUnit tests for the Apple ISP driver" if !KUNIT_ALL_TESTS + depends on VIDEO_APPLE_ISP && KUNIT + depends on KUNIT=y || VIDEO_APPLE_ISP=m + default KUNIT_ALL_TESTS + help + Builds unit tests for the videobuf2 queue set-up of the Apple ISP + driver into the driver. They need no hardware. Only useful for + kernel developers running KUnit. + + If unsure, say N. diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c new file mode 100644 index 00000000000000..895a6d1299c220 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the videobuf2 queue of the Apple ISP driver, built into + * isp-v4l2.c. They check the queue parameters vb2 accepts and the buffer + * limit queue_setup() enforces, without any hardware. + */ + +#include +#include +#include + +#define ISP_TEST_FRAME_SIZE 0x1000 + +static struct apple_isp *isp_test_alloc(struct kunit *test) +{ + struct apple_isp *isp; + + isp = kunit_kzalloc(test, sizeof(*isp), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp); + + mutex_init(&isp->video_lock); + isp_get_current_format(isp)->total_size = ISP_TEST_FRAME_SIZE; + + return isp; +} + +/* The queue must initialise with the parameters the driver probes with. */ +static void isp_test_queue_init(struct kunit *test) +{ + struct apple_isp *isp = isp_test_alloc(test); + + isp->dev = kunit_device_register(test, "apple-isp-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp->dev); + + KUNIT_ASSERT_EQ(test, isp_vb2_queue_init(isp), 0); + KUNIT_EXPECT_GE(test, isp->vbq.max_num_buffers, VB2_MAX_FRAME); + vb2_queue_release(&isp->vbq); +} + +/* Pretend the queue already has @num buffers, as vb2_get_num_buffers() sees. */ +static void isp_test_set_num_buffers(struct kunit *test, struct vb2_queue *q, + unsigned int num) +{ + q->max_num_buffers = VB2_MAX_FRAME; + q->bufs_bitmap = kunit_kzalloc(test, + BITS_TO_LONGS(VB2_MAX_FRAME) * + sizeof(unsigned long), + GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, q->bufs_bitmap); + bitmap_set(q->bufs_bitmap, 0, num); +} + +static int isp_test_setup(struct kunit *test, unsigned int existing, + unsigned int *nbuffers, unsigned int *num_planes, + unsigned int sizes[]) +{ + struct apple_isp *isp = isp_test_alloc(test); + + isp->vbq.drv_priv = isp; + isp_test_set_num_buffers(test, &isp->vbq, existing); + + return isp_vb2_queue_setup(&isp->vbq, nbuffers, num_planes, sizes, + NULL); +} + +/* REQBUFS: an empty queue gets at most one pool of buffers. */ +static void isp_test_reqbufs_limit(struct kunit *test) +{ + unsigned int sizes[VB2_MAX_PLANES] = {}; + unsigned int nbuffers = VB2_MAX_FRAME, num_planes = 0; + + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, ISP_MAX_BUFFERS); + KUNIT_EXPECT_EQ(test, num_planes, 1); + KUNIT_EXPECT_EQ(test, sizes[0], ISP_TEST_FRAME_SIZE); + + nbuffers = 4; + num_planes = 0; + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, 4); +} + +/* CREATE_BUFS: the buffers the queue has count against the pool. */ +static void isp_test_create_bufs_limit(struct kunit *test) +{ + unsigned int sizes[VB2_MAX_PLANES] = { ISP_TEST_FRAME_SIZE }; + unsigned int nbuffers, num_planes; + + nbuffers = 10; + num_planes = 1; + KUNIT_ASSERT_EQ(test, isp_test_setup(test, 10, &nbuffers, &num_planes, + sizes), 0); + KUNIT_EXPECT_EQ(test, nbuffers, ISP_MAX_BUFFERS - 10); + + nbuffers = 1; + num_planes = 1; + KUNIT_EXPECT_EQ(test, isp_test_setup(test, ISP_MAX_BUFFERS, &nbuffers, + &num_planes, sizes), -ENOBUFS); + + /* A plane smaller than a frame is still refused. */ + nbuffers = 1; + num_planes = 1; + sizes[0] = ISP_TEST_FRAME_SIZE - 1; + KUNIT_EXPECT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), -EINVAL); +} + +static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_queue_init), + KUNIT_CASE(isp_test_reqbufs_limit), + KUNIT_CASE(isp_test_create_bufs_limit), + {} +}; + +static struct kunit_suite isp_v4l2_test_suite = { + .name = "apple-isp-v4l2", + .test_cases = isp_v4l2_test_cases, +}; + +kunit_test_suite(isp_v4l2_test_suite); diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index fc9d8409982e73..f321cbcb3368d9 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1014,6 +1014,24 @@ static void isp_free_meta_surfaces(struct apple_isp *isp) } } +static int isp_vb2_queue_init(struct apple_isp *isp) +{ + struct vb2_queue *vbq = &isp->vbq; + + vbq->drv_priv = isp; + vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + vbq->io_modes = VB2_MMAP; + vbq->dev = isp->dev; + vbq->ops = &isp_vb2_ops; + vbq->mem_ops = &vb2_dma_sg_memops; + vbq->buf_struct_size = sizeof(struct isp_buffer); + vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; + vbq->min_queued_buffers = ISP_MIN_FRAMES; + vbq->lock = &isp->video_lock; + + return vb2_queue_init(vbq); +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1086,18 +1104,7 @@ int apple_isp_setup_video(struct apple_isp *isp) goto ctrl_free; } - vbq->drv_priv = isp; - vbq->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - vbq->io_modes = VB2_MMAP; - vbq->dev = isp->dev; - vbq->ops = &isp_vb2_ops; - vbq->mem_ops = &vb2_dma_sg_memops; - vbq->buf_struct_size = sizeof(struct isp_buffer); - vbq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC; - vbq->min_queued_buffers = ISP_MIN_FRAMES; - vbq->lock = &isp->video_lock; - - err = vb2_queue_init(vbq); + err = isp_vb2_queue_init(isp); if (err) { dev_err(isp->dev, "failed to init vb2 queue: %d\n", err); goto v4l2_unregister; @@ -1157,3 +1164,7 @@ void apple_isp_free_video(struct apple_isp *isp) { isp_free_meta_surfaces(isp); } + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include "isp-v4l2-test.c" +#endif From ddd49f33d2880d30d74ab13b2d36d25e3cdd68c6 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:03 +1100 Subject: [PATCH 529/631] nvme-apple: Drop the unused stopped-controller handoff Post-M4 ANS2 firmware can only be adopted while it still runs. The driver accepts two hand-over states: a controller the bootloader left enabled and ready, and a stopped controller that the bootloader vouches for with the "apple,rtkit-handoff" = "nvme-disabled-v1" device tree property. No bootloader produces that property. U-Boot, which the T8140 boot chain uses to load Linux, deletes its I/O queues and hands the controller over enabled and ready, and m1n1 does not publish the property either. The second state is therefore unreachable, and its property is device tree ABI that nothing produces. Remove the property parser and the stopped-controller branch of apple_nvme_can_adopt_rtkit(). The live-controller predicate is unchanged: RUN, BOOT_STATUS, CC.EN and CSTS.RDY must be set and CSTS.CFS clear. A device tree without the property, which is every device tree in use, takes exactly the same paths as before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 094d11e88411e8..ef650af9de140c 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -209,7 +209,6 @@ struct apple_nvme { /* Protects reset_ready against RTKit crash callbacks. */ spinlock_t event_lock; bool reset_ready; - bool handoff_disabled; struct list_head quarantine_node; resource_size_t controller_start; bool quarantined; @@ -336,33 +335,15 @@ static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) return false; cc = readl(anv->mmio_nvme + NVME_REG_CC); csts = readl(anv->mmio_nvme + NVME_REG_CSTS); - if (csts & NVME_CSTS_CFS) - return false; /* - * CPU_RUN/BOOT_STATUS alone survive inactive sessions. A disabled - * controller requires the producer's explicit live-session contract. - * Borrowed buffers still require reserved RAM and protected SART ranges. + * CPU_RUN and BOOT_STATUS alone survive a session that has gone + * inactive. Only a controller that the previous owner left enabled + * and ready identifies a live session. Borrowed buffers still require + * reserved RAM and protected SART ranges. */ - if (anv->handoff_disabled) - return !(cc & NVME_CC_ENABLE) && !(csts & NVME_CSTS_RDY); - return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY); -} - -static int apple_nvme_read_handoff(struct apple_nvme *anv) -{ - const char *handoff; - int ret; - - ret = of_property_read_string(anv->dev->of_node, "apple,rtkit-handoff", - &handoff); - if (ret == -EINVAL) - return 0; - if (ret || !anv->hw->needs_ioq_registers || - strcmp(handoff, "nvme-disabled-v1")) - return -EINVAL; - anv->handoff_disabled = true; - return apple_nvme_can_adopt_rtkit(anv) ? 0 : -EIO; + return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY) && + !(csts & NVME_CSTS_CFS); } static inline struct apple_nvme *queue_to_apple_nvme(struct apple_nvme_queue *q) @@ -2317,12 +2298,6 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) anv->mmio_nvmmu = anv->mmio_nvme; } - ret = apple_nvme_read_handoff(anv); - if (ret) { - dev_err_probe(dev, ret, "Invalid live firmware handoff\n"); - goto out_detach_genpd; - } - if (anv->hw->has_lsq_nvmmu) { anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; From 8323c693a837b92ce5829254e6a38e606c7d4fef Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:20 +1100 Subject: [PATCH 530/631] dt-bindings: nvme: apple,nvme-ans: Drop apple,rtkit-handoff The "apple,rtkit-handoff" property was meant to be added at boot by a bootloader that stops the ANS2 NVMe controller but leaves its firmware running. No bootloader does that: U-Boot hands the T8140 controller over enabled and ready, which the driver recognises from the hardware state alone, and the driver no longer parses the property. Remove the property and the condition that limited it to T8140 before it gains a user. No board device tree source carries it. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../bindings/nvme/apple,nvme-ans.yaml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml index 5449db3b250802..fc8a4afd5add47 100644 --- a/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml +++ b/Documentation/devicetree/bindings/nvme/apple,nvme-ans.yaml @@ -85,30 +85,7 @@ properties: The SART address filter is documented in iommu/apple,sart.yaml. - apple,rtkit-handoff: - $ref: /schemas/types.yaml#/definitions/string - const: nvme-disabled-v1 - description: | - Added at boot by a bootloader that ran the ANS2 firmware and hands it - over still running instead of shutting it down. It describes the - state that bootloader left behind, not the hardware, and must not be - written into a board device tree source. Its presence states that the - firmware keeps a negotiated RTKit session, that all NVMe controller - DMA has stopped (CC.EN and CSTS.RDY clear), that the SART entries the - bootloader programmed are kept, and that every buffer shared with the - firmware is reserved in the memory map without no-map. - allOf: - - if: - properties: - compatible: - not: - contains: - const: apple,t8140-nvme-ans2 - then: - properties: - apple,rtkit-handoff: false - - if: properties: compatible: From 6e103470bb5148fb255e8ecc7c145a0ee1664ca7 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 13:38:45 +1100 Subject: [PATCH 531/631] nvme-apple: Name the NVMMU TCB inline encryption flag The TCB flag that enables inline AES for a command is open-coded as BIT(2) where the command is submitted and where its key material is wiped on completion. Define it next to the DMA direction flags of the same field, and fold the key and DUN lookup into the block that programs them. No functional change. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/nvme/host/apple.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index ef650af9de140c..0b71973f33622f 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -116,6 +116,7 @@ struct apple_nvmmu_tcb { #define APPLE_ANS_TCB_DMA_FROM_DEVICE BIT(0) #define APPLE_ANS_TCB_DMA_TO_DEVICE BIT(1) +#define APPLE_ANS_TCB_AES BIT(2) u8 dma_flags; u8 command_id; @@ -566,8 +567,6 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, struct request *req) { struct apple_nvme *anv = queue_to_apple_nvme(q); - const u8 *key = NULL; - u64 dun = 0; u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; @@ -585,16 +584,10 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; if (unlikely(req->crypt_ctx)) { - const struct blk_crypto_key *blk_key = req->crypt_ctx->bc_key; + const u8 *key = req->crypt_ctx->bc_key->bytes; + __le64 dun_le = cpu_to_le64(req->crypt_ctx->bc_dun[0]); - key = blk_key->bytes; - dun = req->crypt_ctx->bc_dun[0]; - } - - if (key) { - __le64 dun_le = cpu_to_le64(dun); - - tcb->dma_flags |= BIT(2); + tcb->dma_flags |= APPLE_ANS_TCB_AES; memcpy(tcb->aes_iv, &dun_le, sizeof(dun_le)); memset(tcb->_aes_unk, 0, sizeof(tcb->_aes_unk)); memcpy(tcb->_aes_unk, key, 4); @@ -862,10 +855,10 @@ static __always_inline void apple_nvme_unmap_rq(struct request *req) tag = nvme_tag_from_cid(iod->cmd.common.command_id); tcb = &iod->q->tcbs[tag]; - if (unlikely(tcb->dma_flags & BIT(2))) { + if (unlikely(tcb->dma_flags & APPLE_ANS_TCB_AES)) { memzero_explicit(tcb->aes_iv, sizeof(tcb->aes_iv)); memzero_explicit(tcb->_aes_unk, sizeof(tcb->_aes_unk)); - tcb->dma_flags &= ~BIT(2); + tcb->dma_flags &= ~APPLE_ANS_TCB_AES; } } From fd2d3a2aa650ba5999d5dfc0bde0fedc7033b191 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 21:25:06 +1100 Subject: [PATCH 532/631] usb: dwc3: Export dwc3_event_buffers_cleanup() The Apple glue calls dwc3_event_buffers_cleanup() when it stops the core, but the function is not exported. With USB_DWC3=y and USB_DWC3_APPLE=m (or both modular) modpost fails: ERROR: modpost: "dwc3_event_buffers_cleanup" [drivers/usb/dwc3/dwc3-apple.ko] undefined! Export it like the other core helpers the glue drivers use (dwc3_enable_susphy(), dwc3_core_exit()). Fixes: f5afac94ca29 ("drm/apple: compose lean J700 display and automatic recovery candidate") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/usb/dwc3/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index c2bbcbcf3797a2..498325fa1001da 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -610,6 +610,7 @@ void dwc3_event_buffers_cleanup(struct dwc3 *dwc) reg = dwc3_readl(dwc, DWC3_GEVNTCOUNT(0)); dwc3_writel(dwc, DWC3_GEVNTCOUNT(0), reg); } +EXPORT_SYMBOL_GPL(dwc3_event_buffers_cleanup); static void dwc3_core_num_eps(struct dwc3 *dwc) { From af59453a516454b1f134fc619499ee325e88ee5f Mon Sep 17 00:00:00 2001 From: Eryk Wieliczko Date: Tue, 13 Oct 2026 14:20:26 +0200 Subject: [PATCH 533/631] arm64: dts: apple: t8140-j700: give the boot framebuffer its panel simpledrm reads the panel's physical size through the framebuffer's panel phandle. Without it the boot display reports 96 dpi, and compositors pick a scale of 1 for the 219 ppi panel. Link it as the other Apple laptops do. Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140-j700.dts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/boot/dts/apple/t8140-j700.dts b/arch/arm64/boot/dts/apple/t8140-j700.dts index 942e360f57c4ec..affdd0149be647 100644 --- a/arch/arm64/boot/dts/apple/t8140-j700.dts +++ b/arch/arm64/boot/dts/apple/t8140-j700.dts @@ -52,6 +52,8 @@ power-domains = <&ps_disp_cpu>; /* Format properties will be added by loader */ status = "disabled"; + panel = <&panel>; + post-init-providers = <&panel>; }; }; From 1a95b4eadd40d91a05fb19c294cc9d834d846915 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:24:02 +1100 Subject: [PATCH 534/631] wifi: mt7932: drop received data the hardware did not decrypt On a WPA2 connection the driver hands every header-translated data frame from the associated peer to the network stack. Nothing checks that the hardware decrypted it: a frame that arrived in plaintext, or that was decrypted with a cipher or key length other than the one the key expects, is delivered like any other. The receive descriptor reports both: word1 carries the cipher the frame was decrypted with (zero when it was not) and the cipher and cipher-length mismatch flags, in the same layout as the connac2 devices that mt76 handles. Only a cipher mismatch on a translated frame was rejected; the cipher-length mismatch flag was ignored. Keep the cipher and both mismatch flags in the parsed frame and, on a protected connection, deliver only frames that were decrypted without a mismatch. The firmware runs the WPA2 handshake itself, so the host never needs to accept an unprotected EAPOL frame. An open connection is unchanged. Add KUnit tests with synthetic descriptors: decrypted, plaintext, cipher-length and cipher mismatch, integrity errors, every truncation of the descriptor and its Ethernet header, and a length beyond the buffer. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 10 ++ drivers/net/wireless/mediatek/mt7932/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/data.c | 7 + .../net/wireless/mediatek/mt7932/input-test.c | 133 ++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/packet.h | 29 +++- 5 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/input-test.c diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index c6120372d1f3b8..f09fb1f32012bc 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -31,3 +31,13 @@ config MT7932_RF_KUNIT_TEST admission paths with synthetic devices, without radio hardware. Check association wakeups, retained peer ownership and refusal to reopen RF admission after a terminal calibration error. + +config MT7932_KUNIT_TEST + bool "KUnit tests for MT7932 firmware input validation" if !KUNIT_ALL_TESTS + depends on KUNIT=y && MT7932_FULLMAC + default KUNIT_ALL_TESTS + help + Exercise the validation of what the firmware and the device hand + to the driver, with synthetic, truncated and corrupted input and + without radio hardware: receive descriptors, including frames the + hardware did not decrypt, and command replies. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index f854d21e5db813..97b1b760372c16 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -2,3 +2,4 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index 99ae52977b5823..ba06db134414ea 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -234,6 +234,13 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) m->netdev->stats.rx_dropped++; goto unlock; } + if (!mt7932_rx_protection_ok(frame, m->connect_open)) { + m->netdev->stats.rx_dropped++; + dev_dbg_ratelimited(&m->pdev->dev, + "RX_UNPROTECTED_DROPPED: cipher=%u mismatch=%u\n", + frame->sec_mode, frame->cipher_mismatch); + goto unlock; + } skb = netdev_alloc_skb_ip_align(m->netdev, frame->length); if (!skb) { m->netdev->stats.rx_dropped++; diff --git a/drivers/net/wireless/mediatek/mt7932/input-test.c b/drivers/net/wireless/mediatek/mt7932/input-test.c new file mode 100644 index 00000000000000..409103ed187402 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/input-test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the validation of firmware input: RX descriptors and + * command replies. The firmware and the device are treated as untrusted. + */ +#include + +#include "mt7932.h" + +#define RX_HEADER 24 +#define RX_ETHERNET 14 +#define RX_PAYLOAD 40 +#define RX_SIZE (RX_HEADER + RX_ETHERNET + RX_PAYLOAD) + +/* A translated (Ethernet) unicast data frame on channel 36 for WTBL 5. */ +static void rx_build(u8 *packet, u32 extra_w1) +{ + memset(packet, 0, RX_SIZE); + put_unaligned_le32(RX_SIZE | (2U << 27), packet); + put_unaligned_le32(5 | extra_w1, packet + 4); + put_unaligned_le32(BIT(13), packet + 8); + put_unaligned_le32((36U << 8) | (1U << 16), packet + 12); + memset(packet + RX_HEADER, 0xa5, RX_ETHERNET + RX_PAYLOAD); + put_unaligned_be16(ETH_P_IP, packet + RX_HEADER + 12); +} + +static u32 rx_cipher(unsigned int mode) +{ + return mode << MT7932_RXD1_SEC_MODE_SHIFT; +} + +static void mt7932_rx_decrypted_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, rx_cipher(4)); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_TRUE(test, frame.translated); + KUNIT_EXPECT_EQ(test, frame.sec_mode, 4); + KUNIT_EXPECT_EQ(test, frame.wtbl, 5); + KUNIT_EXPECT_EQ(test, frame.length, (size_t)(RX_ETHERNET + RX_PAYLOAD)); + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, false)); + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, true)); +} + +/* A plaintext frame on a protected connection is never delivered. */ +static void mt7932_rx_plaintext_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, 0); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_EQ(test, frame.sec_mode, 0); + KUNIT_EXPECT_FALSE(test, mt7932_rx_protection_ok(&frame, false)); + /* An open connection has nothing to decrypt. */ + KUNIT_EXPECT_TRUE(test, mt7932_rx_protection_ok(&frame, true)); +} + +/* Decrypted with the wrong cipher or key length: refused on WPA2. */ +static void mt7932_rx_mismatch_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + + rx_build(packet, rx_cipher(4) | MT7932_RXD1_CIPHER_LEN_MISMATCH); + KUNIT_ASSERT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), 0); + KUNIT_EXPECT_TRUE(test, frame.cipher_mismatch); + KUNIT_EXPECT_FALSE(test, mt7932_rx_protection_ok(&frame, false)); + /* A translated frame with a cipher mismatch is rejected by the parser. */ + rx_build(packet, rx_cipher(4) | MT7932_RXD1_CIPHER_MISMATCH); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), -EBADMSG); +} + +/* ICV, TKIP MIC and FCS errors are rejected before any field is used. */ +static void mt7932_rx_integrity_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE]; + unsigned int bit; + + for (bit = 25; bit <= 27; bit++) { + rx_build(packet, rx_cipher(4) | BIT(bit)); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, sizeof(packet), &frame), + -EBADMSG); + } +} + +/* Short buffers and length fields beyond the buffer never parse. */ +static void mt7932_rx_short_test(struct kunit *test) +{ + struct mt7932_rx_frame frame; + u8 packet[RX_SIZE], *copy; + size_t cut; + + rx_build(packet, rx_cipher(4)); + for (cut = 0; cut < RX_HEADER + RX_ETHERNET; cut++) { + /* An exact-size copy lets KASAN catch any read past the end. */ + copy = kmemdup(packet, cut, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, copy || !cut); + if (cut >= 4) + put_unaligned_le32(cut | (2U << 27), copy); + KUNIT_EXPECT_LT_MSG(test, mt7932_rx_frame(copy, cut, &frame), 0, + "length %zu", cut); + kfree(copy); + } + /* The descriptor claims more than the DMA buffer holds. */ + copy = kmemdup(packet, RX_SIZE - 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(copy, RX_SIZE - 1, &frame), -EMSGSIZE); + kfree(copy); + /* Optional groups (3 and 5) that do not fit. */ + rx_build(packet, rx_cipher(4) | BIT(13) | BIT(15)); + put_unaligned_le32((RX_HEADER + 20) | (2U << 27), packet); + KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, RX_HEADER + 20, &frame), -EMSGSIZE); +} + +static struct kunit_case mt7932_input_test_cases[] = { + KUNIT_CASE(mt7932_rx_decrypted_test), + KUNIT_CASE(mt7932_rx_plaintext_test), + KUNIT_CASE(mt7932_rx_mismatch_test), + KUNIT_CASE(mt7932_rx_integrity_test), + KUNIT_CASE(mt7932_rx_short_test), + {} +}; + +static struct kunit_suite mt7932_input_test_suite = { + .name = "mt7932-input", + .test_cases = mt7932_input_test_cases, +}; + +kunit_test_suite(mt7932_input_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index a93d04d91fd412..d7480c81872b26 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -2,6 +2,8 @@ #ifndef MT7932_PACKET_H #define MT7932_PACKET_H +#include + #include "protocol.h" #include "channels.h" @@ -10,10 +12,20 @@ struct mt7932_rx_frame { size_t length; u32 timestamp, rcpi, rxv; u16 frame_control, sequence_control, qos_control, wtbl; - u8 channel, amsdu; + u8 channel, amsdu, sec_mode; bool translated, group4, signal_valid, rate_valid, unicast; + bool cipher_mismatch; }; +/* RX descriptor word1: the cipher the hardware decrypted the frame with (0 + * when it did not decrypt it), and the cipher and cipher-length mismatch + * flags. The layout is the connac2 one. + */ +#define MT7932_RXD1_SEC_MODE_SHIFT 16 +#define MT7932_RXD1_SEC_MODE_MASK 0x1f +#define MT7932_RXD1_CIPHER_MISMATCH BIT(23) +#define MT7932_RXD1_CIPHER_LEN_MISMATCH BIT(24) + /* Strict non-monitor subset of the RX descriptor; its groups are ordered * 4, 1, 2, 3, 5, not by numerical group identifier. */ @@ -54,6 +66,9 @@ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_r out->wtbl = w1 & 0x3ff; out->unicast = ((w3 >> 16) & 3) == 1; out->amsdu = w4 & 3; + out->sec_mode = (w1 >> MT7932_RXD1_SEC_MODE_SHIFT) & MT7932_RXD1_SEC_MODE_MASK; + out->cipher_mismatch = w1 & (MT7932_RXD1_CIPHER_MISMATCH | + MT7932_RXD1_CIPHER_LEN_MISMATCH); if (!mt7932_channel_band(out->channel)) return -EOPNOTSUPP; if ((out->translated && (w2 & (1U << 27))) || (!out->translated && out->amsdu)) @@ -103,6 +118,18 @@ static inline int mt7932_rx_frame(const u8 *packet, size_t size, struct mt7932_r return 0; } +/* + * On a protected connection, deliver only frames the hardware decrypted with + * the cipher and key length it expected. The firmware runs the WPA2 handshake + * itself, so the host has no unprotected EAPOL to accept. Nothing is + * decrypted on an open connection. + */ +static inline bool mt7932_rx_protection_ok(const struct mt7932_rx_frame *frame, + bool open) +{ + return open || (frame->sec_mode && !frame->cipher_mismatch); +} + static inline int mt7932_rx_signal(const struct mt7932_rx_frame *frame, u8 antenna_mask) { int signal = -128; From 2ea98cb8b092d30003c00f0ebcb81a922b3ead9a Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:24:57 +1100 Subject: [PATCH 535/631] wifi: mt7932: complete a command only with the event that answers it A command that waits for a reply is completed by the first event, not otherwise dispatched, that carries its sequence number. Nothing checks that the event answers that command. A delayed event after the 8-bit sequence has wrapped, or a malformed one, can complete a different command; most callers then check the reply, but the RTS setup accepts any matching sequence and goes on to publish data readiness. Record the event that answers the command being sent and require it as well as the sequence. Extended commands answer with the extended event (0xed), the runtime capability query (0x8a) with 0xec, other runtime queries with an event of their own ID, the boot ROM's register read with 0x02, its patch semaphore with 0x04 and its status commands with 0x01, the event numbers other connac devices use. These are also the events the callers already check for, and the RTS reply is the extended event. The ROM's OTP read keeps sequence-only matching for now: no public source documents its reply event. Log the event it receives at debug level so that it can be measured and pinned. Add KUnit tests for the matcher and the reply table, including same-sequence events with the wrong event ID. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/input-test.c | 57 +++++++++++++++++++ drivers/net/wireless/mediatek/mt7932/mt7932.c | 5 +- drivers/net/wireless/mediatek/mt7932/mt7932.h | 2 + .../net/wireless/mediatek/mt7932/protocol.h | 32 ++++++++++- 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/input-test.c b/drivers/net/wireless/mediatek/mt7932/input-test.c index 409103ed187402..9ee9253f34a307 100644 --- a/drivers/net/wireless/mediatek/mt7932/input-test.c +++ b/drivers/net/wireless/mediatek/mt7932/input-test.c @@ -116,12 +116,69 @@ static void mt7932_rx_short_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt7932_rx_frame(packet, RX_HEADER + 20, &frame), -EMSGSIZE); } +static void event_build(u8 *packet, u8 eid, u8 seq, u8 option) +{ + memset(packet, 0, 64); + put_unaligned_le32(64 | (7U << 27), packet); + put_unaligned_le16(40, packet + 24); + packet[28] = eid; + packet[29] = seq; + packet[30] = option; +} + +/* Only the event that answers a command completes it. */ +static void mt7932_event_reply_test(struct kunit *test) +{ + struct mt7932_event event; + u8 packet[64]; + int eid = mt7932_reply_eid(0xed, true); + + event_build(packet, 0xed, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_TRUE(test, mt7932_event_matches(&event, 9, eid)); + /* Another sequence, the no-reply option bit, or nothing waiting. */ + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 10, eid)); + event.option = 4; + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, eid)); + event.option = 0; + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 0, eid)); + /* An unrelated event that happens to carry the waiting sequence. */ + event_build(packet, 0x99, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, eid)); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x8a, true))); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x03, false))); + /* The ROM's patch semaphore answers with its own event only. */ + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); + event_build(packet, 0x04, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_TRUE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); + event_build(packet, 0x01, 9, 0); + KUNIT_ASSERT_EQ(test, mt7932_event_parse(packet, sizeof(packet), &event), 0); + KUNIT_EXPECT_FALSE(test, mt7932_event_matches(&event, 9, mt7932_reply_eid(0x10, false))); +} + +static void mt7932_event_reply_eid_test(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xed, false), 0xed); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xed, true), 0xed); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x8a, true), 0xec); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0xbf, true), 0xbf); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x03, false), 0x02); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x01, false), 0x01); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x05, false), 0x01); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x10, false), 0x04); + KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x50, false), -1); +} + static struct kunit_case mt7932_input_test_cases[] = { KUNIT_CASE(mt7932_rx_decrypted_test), KUNIT_CASE(mt7932_rx_plaintext_test), KUNIT_CASE(mt7932_rx_mismatch_test), KUNIT_CASE(mt7932_rx_integrity_test), KUNIT_CASE(mt7932_rx_short_test), + KUNIT_CASE(mt7932_event_reply_test), + KUNIT_CASE(mt7932_event_reply_eid_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index 6e5409af6253d3..d7fcffb71d1c6f 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -205,7 +205,7 @@ static void mt_receive(struct mt7932 *m, const u8 *packet, size_t length) } goto unlock; } - if (mt7932_event_matches(&event, m->waiting) && !m->reply_length) { + if (mt7932_event_matches(&event, m->waiting, m->waiting_eid) && !m->reply_length) { memcpy(m->reply, packet, event.length); m->reply_length = event.length; complete(&m->response); @@ -377,6 +377,7 @@ int mt_request_ext(struct mt7932 *m, u8 cid, u8 ext, bool runtime, bool set, boo reinit_completion(&m->response); spin_lock_irqsave(&m->response_lock, flags); m->waiting = wait ? m->seq : 0; + m->waiting_eid = mt7932_reply_eid(cid, runtime); m->reply_length = 0; spin_unlock_irqrestore(&m->response_lock, flags); ret = mt_submit(m, q, ret); @@ -679,6 +680,8 @@ static int mt_otp_byte(struct mt7932 *m, u32 address, u8 *value) if (m->reply_length < 40) return -EPROTO; *value = m->reply[36]; + dev_dbg(&m->pdev->dev, "OTP_REPLY: eid=%02x word=%08x address=%08x length=%zu\n", + m->reply[28], get_unaligned_le32(m->reply + 32), address, m->reply_length); return 0; } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index fc5c70c6796ae6..d7430eee379bc9 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -75,6 +75,8 @@ struct mt7932 { unsigned int cal_request_head, cal_request_count; spinlock_t response_lock; u8 seq, waiting; + /* Event that answers the waiting command; see mt7932_reply_eid(). */ + int waiting_eid; u8 mac[ETH_ALEN]; u8 reply[MT7932_RX_CAPACITY]; size_t reply_length; diff --git a/drivers/net/wireless/mediatek/mt7932/protocol.h b/drivers/net/wireless/mediatek/mt7932/protocol.h index 00751e130ef44a..e7f8820a9ff8c6 100644 --- a/drivers/net/wireless/mediatek/mt7932/protocol.h +++ b/drivers/net/wireless/mediatek/mt7932/protocol.h @@ -65,9 +65,37 @@ static inline int mt7932_event_parse(const u8 *packet, size_t length, return 0; } -static inline bool mt7932_event_matches(const struct mt7932_event *e, u8 seq) +/* + * The event that answers a command waiting for a reply, or -1 for any event. + * Extended commands answer with the extended event, the capability query + * with its own event, and the boot ROM's status commands with the generic + * one. The ROM's register read answers with the access-register event and + * its patch semaphore with the patch-semaphore event, as on other connac + * devices. Other runtime commands answer with an event of their own ID. + * The ROM's OTP read alone still matches on the sequence: its reply event + * is not documented, and mt_otp_byte() records the one it receives. + */ +static inline int mt7932_reply_eid(u8 cid, bool runtime) { - return seq && e->seq == seq && !(e->option & 4); + if (cid == 0xed) + return 0xed; + if (runtime) + return cid == 0x8a ? 0xec : cid; + switch (cid) { + case 0x03: + return 0x02; + case 0x10: + return 0x04; + case 0x50: + return -1; + default: + return 0x01; + } +} + +static inline bool mt7932_event_matches(const struct mt7932_event *e, u8 seq, int eid) +{ + return seq && e->seq == seq && !(e->option & 4) && (eid < 0 || e->eid == eid); } #endif From cd0a50c4eb8de1236df0f7b8c70fbcf51f4dac15 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:25:22 +1100 Subject: [PATCH 536/631] wifi: mt7932: refuse hibernation The driver supports only suspend-to-idle and refuses other suspend states in its .suspend callback. Hibernation does not go through .suspend: its freeze phase calls .freeze, which the driver does not provide, so the PCI core lets it pass with the firmware still running. The firmware keeps writing to the receive rings while the memory image is saved, and after a restore the kernel resumes with ring state that the firmware started by the booting kernel does not have. Refuse the freeze and poweroff phases. The PM core then aborts hibernation before the device changes state, so there is nothing to undo. Supporting hibernation needs DMA retirement on freeze and a full reinitialisation, including the regulatory pass, on restore. Fixes: b4d89a96d6b0 ("wifi: mt7932: keep the function running across suspend-to-idle") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/mt7932.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.c b/drivers/net/wireless/mediatek/mt7932/mt7932.c index d7fcffb71d1c6f..b90c262a92177b 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.c +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.c @@ -1259,8 +1259,24 @@ static int mt_suspend(struct device *dev) return 0; } +/* + * Hibernation is refused. Its freeze phase does not call .suspend, so without + * this the firmware would keep writing to memory while the image is saved, + * and the restored kernel would resume with ring state the firmware of the + * booting kernel does not have. Supporting it needs DMA retirement and a full + * reinitialisation on restore. Returning an error here has no side effect to + * undo: the PM core aborts hibernation before this device changes state. + */ +static int mt_freeze(struct device *dev) +{ + dev_err(dev, "hibernation is not supported\n"); + return -EOPNOTSUPP; +} + static const struct dev_pm_ops mt_pm_ops = { .suspend = mt_suspend, + .freeze = mt_freeze, + .poweroff = mt_freeze, }; static const struct pci_device_id mt_ids[] = { From ef66121f714d9bc748e92d14a9588bd918983014 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:11 +1100 Subject: [PATCH 537/631] wifi: mt7932: let a cancelled scan batch retire before a disconnect gives up The disconnect worker quiesces scanning before it retires the connection. mt_scan_quiesce() waits for the current batch to finish, aborts the scan and returns -EBUSY if a cancelled batch has not yet been reported finished. The disconnect worker treats that as fatal and sets link_failed, which nothing clears: the radio stays unusable until the next boot. A multi-batch scan makes this reachable without any firmware fault. When the awaited batch finishes, the scan completion work may submit the next batch before the abort is seen. The abort then cancels that batch, and its end is reported by the firmware a little later, after the quiesce has already returned -EBUSY. cfg80211 disconnects on every suspend, so a background scan at suspend time is enough. Wait for the cancelled batch to retire, bounded to two seconds, as mt_scan_retire_wait() already does for an address change. Only a scan that the firmware never ends still fails the link. Add a KUnit test that drives mt_scan_quiesce() with a synthetic scan: the completion work has already submitted the next batch when the abort arrives, and the firmware reports its end 300 ms later (success) or never (-EBUSY after the two-second bound). Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- .../net/wireless/mediatek/mt7932/frontend.c | 26 +++--- .../net/wireless/mediatek/mt7932/rf-test.c | 91 +++++++++++++++++++ 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 8d873a4379bfa2..84170c8f613265 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -123,18 +123,6 @@ void mt_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) spin_unlock_irqrestore(&m->response_lock, flags); } -/* No command/response lock held. The caller closes new scan admission first. - * Natural EID0d completion fences the scan; a local cancel alone does not. - */ -int mt_scan_quiesce(struct mt7932 *m) -{ - if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) - wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); - mt_abort_scan(m->wiphy, &m->wdev); - flush_work(&m->scan_finish_work); - return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; -} - /* Wait at most @timeout in total for a cancelled scan to retire. Callers * may hold RTNL, so this never waits longer than asked. A stopping driver * completes scan_done for good, which would turn the wait into a spin. @@ -153,6 +141,20 @@ static void mt_scan_retire_wait(struct mt7932 *m, unsigned long timeout) } } +/* No command/response lock held. The caller closes new scan admission first. + * Natural EID0d completion fences the scan; a local cancel alone does not. + * The batch awaited here may already have been followed by the next one, + * which the abort then cancels: give the firmware time to end that one too, + * rather than declaring the link unrecoverable while its cancel is in flight. + */ +int mt_scan_quiesce(struct mt7932 *m) +{ + if (READ_ONCE(m->scan_request) || READ_ONCE(m->retired_scan_seq)) + wait_for_completion_timeout(&m->scan_done, msecs_to_jiffies(5000)); + mt_scan_retire_wait(m, msecs_to_jiffies(2000)); + return READ_ONCE(m->retired_scan_seq) ? -EBUSY : 0; +} + /* command_mutex held: one RF-band/type batch at a time, with matching EID0d * completion before the next batch. A request without SSIDs is passive; * active requests must also listen passively on NO_IR channels. diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index ede549842f7b2a..0b88ef72bdf111 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -412,6 +412,96 @@ static void mt_late_association_channel_preserves_terminal_test(struct kunit *te } } +struct mt_scan_retire_test_context { + struct delayed_work retire; + struct mt7932 *m; + bool late; +}; + +static struct mt_scan_retire_test_context *mt_scan_retire_ctx; + +/* The firmware reports the end of the cancelled batch a little later. */ +static void mt_scan_retire_test_late(struct work_struct *work) +{ + struct mt_scan_retire_test_context *ctx = + container_of(to_delayed_work(work), struct mt_scan_retire_test_context, retire); + struct mt7932 *m = ctx->m; + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + m->retired_scan_seq = 0; + complete(&m->scan_done); + spin_unlock_irqrestore(&m->response_lock, flags); +} + +/* + * The scan completion work had already submitted the next batch when the + * abort arrived: the abort cancels it, and its end is still to come. + */ +static void mt_scan_retire_test_finish(struct work_struct *work) +{ + struct mt_scan_retire_test_context *ctx = mt_scan_retire_ctx; + struct mt7932 *m = ctx->m; + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (m->scan_request && m->scan_aborted) { + m->scan_request = NULL; + m->retired_scan_seq = 8; + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (ctx->late) + schedule_delayed_work(&ctx->retire, msecs_to_jiffies(300)); +} + +static void mt_scan_quiesce_retirement_test(struct kunit *test) +{ + static const struct cfg80211_ops ops = {}; + struct mt_scan_retire_test_context ctx = {}; + struct cfg80211_scan_request *scan; + struct mt7932 *m = test->priv; + unsigned long start; + unsigned int round; + + scan = kunit_kzalloc(test, sizeof(*scan), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, scan); + m->wiphy = wiphy_new(&ops, sizeof(m)); + KUNIT_ASSERT_NOT_NULL(test, m->wiphy); + *(struct mt7932 **)wiphy_priv(m->wiphy) = m; + init_completion(&m->scan_done); + INIT_WORK(&m->scan_finish_work, mt_scan_retire_test_finish); + INIT_DELAYED_WORK(&ctx.retire, mt_scan_retire_test_late); + ctx.m = m; + mt_scan_retire_ctx = &ctx; + for (round = 0; round < 2; round++) { + ctx.late = !round; + reinit_completion(&m->scan_done); + m->scan_request = scan; + scan->wdev = &m->wdev; + m->scan_aborted = false; + m->retired_scan_seq = 0; + /* The batch the quiesce waits for has just finished. */ + complete(&m->scan_done); + start = jiffies; + if (ctx.late) { + /* A late but genuine end retires the scan: no failure. */ + KUNIT_EXPECT_EQ(test, mt_scan_quiesce(m), 0); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 0); + } else { + /* A batch the firmware never ends still fails, after the bound. */ + KUNIT_EXPECT_EQ(test, mt_scan_quiesce(m), -EBUSY); + KUNIT_EXPECT_EQ(test, m->retired_scan_seq, 8); + KUNIT_EXPECT_GE(test, jiffies - start, msecs_to_jiffies(1900)); + KUNIT_EXPECT_LE(test, jiffies - start, msecs_to_jiffies(3000)); + } + cancel_delayed_work_sync(&ctx.retire); + flush_work(&m->scan_finish_work); + } + mt_scan_retire_ctx = NULL; + wiphy_free(m->wiphy); + m->wiphy = NULL; +} + static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_invalid_association_channel_wakes_both_test), KUNIT_CASE(mt_valid_repeated_association_start_test), @@ -422,6 +512,7 @@ static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_rf_failure_wakes_association_test), KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), + KUNIT_CASE(mt_scan_quiesce_retirement_test), {} }; From 9617db8ba7bcdbcffa0f0220e388a62c6b3f9ea0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 23:34:59 +1100 Subject: [PATCH 538/631] wifi: mt7932: report the connection outcome outside the response lock The connect worker publishes its outcome with response_lock held and interrupts disabled, so that a late firmware success cannot raise the carrier after a cancellation. Some of those cfg80211 calls take bottom-half spinlocks: - a connect cancelled while it was in progress, which happens whenever the system suspends during a connect, drops the BSS reference with cfg80211_put_bss(), which takes rdev->bss_lock with spin_lock_bh(); - cfg80211_connect_done() looks the BSS up again, under the same lock, when it has dropped off the scan list, and drops the reference when it cannot allocate its event. Re-enabling bottom halves with interrupts disabled is not allowed: lockdep reports it, and pending softirqs would then run with response_lock held. Keep deciding the outcome, the connection flags and the carrier under the lock, then make the cfg80211 calls after dropping it, with GFP_KERNEL. The ordering against cancellation still holds: - mt_disconnect() and a firmware disconnect event both leave the report to the disconnect worker, which retires the connection under command_mutex first, and this worker holds command_mutex until its own report is queued; - an RF failure reports a lost connection directly. While the worker is publishing, the failure now marks the connection lost instead, and the worker reports it once its own result has been queued, so cfg80211 never sees the disconnect before the connect. A connect cancelled by userspace or by suspend is not a driver failure: log it at debug level instead of as an error. Add a KUnit test for an RF failure during publication: the failure marks the connection lost and leaves the report to the worker. Fixes: 8709ef5c7d40 ("wifi: add existing J700 MT7932 fullmac implementation") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/link.c | 72 +++++++++++++------ drivers/net/wireless/mediatek/mt7932/mt7932.h | 4 ++ .../net/wireless/mediatek/mt7932/rf-test.c | 25 +++++++ 3 files changed, 78 insertions(+), 23 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index c7aeec6f50d6c8..927807e2e8ad37 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -71,11 +71,17 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) complete_all(&m->assoc_done); complete_all(&m->discovery_done); } else if (m->connected) { - /* Hardware ownership is uncertain: report failure, retain the peer. */ + /* Hardware ownership is uncertain: report failure, retain the peer. + * A connect worker still publishing the connection reports this + * after it, so that cfg80211 sees the two in order. + */ m->connected = false; m->disconnecting = true; - cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, - NULL, 0, true, GFP_ATOMIC); + if (m->connect_publishing) + m->connect_lost = true; + else + cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, true, GFP_ATOMIC); } if (m->scan_request) { m->scan_aborted = true; @@ -390,7 +396,8 @@ void mt_connect_work(struct work_struct *work) struct cfg80211_connect_resp_params response = {}; u8 *body, mac[8] = {}, queue[32]; unsigned long flags; - bool submitted = false, peer_owned = false; + bool submitted = false, peer_owned = false, cancelled; + u16 reason; int ret; mutex_lock(&m->command_mutex); @@ -506,7 +513,10 @@ void mt_connect_work(struct work_struct *work) } /* Serialize the final outcome with cancellation. In particular, a late * firmware success must not publish carrier/authorization after .deauth. - * cfg80211 queues these notifications; GFP_ATOMIC keeps this IRQ-safe. + * The cfg80211 calls may take bottom-half locks, so they are made after + * response_lock is dropped. command_mutex keeps the disconnect worker, + * which retires the connection before reporting its end, behind them; + * an RF failure in the meantime leaves its report to this worker. */ spin_lock_irqsave(&m->response_lock, flags); if (!ret && (m->connect_error || !mt_rf_allowed(m))) { @@ -546,33 +556,49 @@ void mt_connect_work(struct work_struct *work) } m->connected = !ret; m->disconnecting = false; - if (m->connect_cancelled) { - if (response.links[0].bss) - cfg80211_put_bss(m->wiphy, response.links[0].bss); - cfg80211_disconnected(m->netdev, m->disconnect_reason, NULL, 0, true, GFP_ATOMIC); - } else { - cfg80211_connect_done(m->netdev, &response, GFP_ATOMIC); - } + cancelled = m->connect_cancelled; + reason = m->disconnect_reason; + m->connect_publishing = true; if (!ret) { schedule_work(&m->power_work); if (READ_ONCE(m->data_ready)) { netif_carrier_on(m->netdev); netif_wake_queue(m->netdev); } - if (m->connect_open) { - dev_dbg(&m->pdev->dev, - "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", - m->data_ready); - } else { - cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_ATOMIC); - dev_dbg(&m->pdev->dev, - "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", - m->data_ready); - } + } + spin_unlock_irqrestore(&m->response_lock, flags); + + if (cancelled) { + if (response.links[0].bss) + cfg80211_put_bss(m->wiphy, response.links[0].bss); + cfg80211_disconnected(m->netdev, reason, NULL, 0, true, GFP_KERNEL); } else { - dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", ret, m->link_failed); + cfg80211_connect_done(m->netdev, &response, GFP_KERNEL); + } + if (!ret && !m->connect_open) + cfg80211_port_authorized(m->netdev, m->connect_bssid, NULL, 0, GFP_KERNEL); + + spin_lock_irqsave(&m->response_lock, flags); + m->connect_publishing = false; + if (m->connect_lost) { + m->connect_lost = false; + cfg80211_disconnected(m->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, true, GFP_ATOMIC); } spin_unlock_irqrestore(&m->response_lock, flags); + if (!ret && m->connect_open) + dev_dbg(&m->pdev->dev, + "OPEN_CONNECTED: owned peer, firmware completion; Ethernet=%u\n", + READ_ONCE(m->data_ready)); + else if (!ret) + dev_dbg(&m->pdev->dev, + "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", + READ_ONCE(m->data_ready)); + else if (cancelled) + dev_dbg(&m->pdev->dev, "CONNECT_CANCELLED: %d\n", ret); + else + dev_err(&m->pdev->dev, "CONNECT_FAILED: %d; recovery-required=%u\n", + ret, READ_ONCE(m->link_failed)); mutex_unlock(&m->command_mutex); } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index d7430eee379bc9..8f041cb20d0bfe 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -148,6 +148,10 @@ struct mt7932 { u8 connect_ssid_length, connect_channel, peer_wtbl; bool connecting, connected, peer_valid, link_failed, connect_open; bool disconnecting, disconnect_seen, disconnect_event_received, disconnect_local; + /* The connect worker is reporting its outcome; an RF failure meanwhile + * sets connect_lost for the worker to report. Under response_lock. + */ + bool connect_publishing, connect_lost; u16 disconnect_reason; int connect_error; u32 event_cipher; diff --git a/drivers/net/wireless/mediatek/mt7932/rf-test.c b/drivers/net/wireless/mediatek/mt7932/rf-test.c index 0b88ef72bdf111..5f9a34825cc231 100644 --- a/drivers/net/wireless/mediatek/mt7932/rf-test.c +++ b/drivers/net/wireless/mediatek/mt7932/rf-test.c @@ -502,6 +502,30 @@ static void mt_scan_quiesce_retirement_test(struct kunit *test) m->wiphy = NULL; } +/* + * An RF failure while the connect worker is publishing a new connection + * must not report the disconnect ahead of the connect: it leaves the report + * to the worker. (cfg80211 is not called here: the synthetic netdev has no + * wireless device, so a direct report would crash the test.) + */ +static void mt_rf_failure_during_publication_test(struct kunit *test) +{ + struct mt7932 *m = test->priv; + + m->connecting = false; + m->connected = true; + m->connect_publishing = true; + mt_rf_test_fail(m, -EIO); + KUNIT_EXPECT_TRUE(test, m->connect_lost); + KUNIT_EXPECT_FALSE(test, m->connected); + KUNIT_EXPECT_TRUE(test, m->disconnecting); + KUNIT_EXPECT_FALSE(test, netif_carrier_ok(m->netdev)); + /* Ownership is uncertain after an RF failure: keep the peer. */ + KUNIT_EXPECT_TRUE(test, m->peer_valid); + m->connect_lost = false; + m->connect_publishing = false; +} + static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_invalid_association_channel_wakes_both_test), KUNIT_CASE(mt_valid_repeated_association_start_test), @@ -513,6 +537,7 @@ static struct kunit_case mt_rf_test_cases[] = { KUNIT_CASE(mt_rf_failure_blocks_startup_test), KUNIT_CASE(mt_rf_failure_preserves_first_error_test), KUNIT_CASE(mt_scan_quiesce_retirement_test), + KUNIT_CASE(mt_rf_failure_during_publication_test), {} }; From 0cb518bf6de5343b5e387b3a0d5d3ba2042ee866 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 01:27:46 +1100 Subject: [PATCH 539/631] wifi: mt7932: keep the link when a beacon hint only lifts NO_IR In the world regulatory domain, cfg80211 lifts NO_IR on a channel once it has heard a beacon there, and calls the notifier because the driver asks for channel-change callbacks. The changed channel flags make a new snapshot, which starts a full regulatory pass, and the pass ends any connection before it programs the firmware. So the first beacon heard on each new channel during a background scan drops a working connection, only to permit more than the firmware currently allows. While associated, keep the stricter domain the firmware already has when the new snapshot differs from the applied one only by NO_IR lifted on some channels: same country, firmware domain, channel list and limits. Record it as deferred and run the pass when the connection has ended, before the disconnect worker lets a new connect start. Any other change, including a restriction, is applied at once as before and replaces a deferred relaxation. Until the deferred pass runs, a scan may probe on such a channel, which cfg80211 now permits, while the firmware domain still marks it passive. Add KUnit tests for the relaxation check. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/net/wireless/mediatek/mt7932/Kconfig | 3 +- drivers/net/wireless/mediatek/mt7932/Makefile | 1 + drivers/net/wireless/mediatek/mt7932/link.c | 1 + drivers/net/wireless/mediatek/mt7932/mt7932.h | 6 + .../mediatek/mt7932/regulatory-test.c | 103 ++++++++++++++++++ .../net/wireless/mediatek/mt7932/regulatory.h | 32 ++++++ .../net/wireless/mediatek/mt7932/startup.c | 32 ++++++ 7 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 drivers/net/wireless/mediatek/mt7932/regulatory-test.c diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index f09fb1f32012bc..49a6fbad0e67e4 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -40,4 +40,5 @@ config MT7932_KUNIT_TEST Exercise the validation of what the firmware and the device hand to the driver, with synthetic, truncated and corrupted input and without radio hardware: receive descriptors, including frames the - hardware did not decrypt, and command replies. + hardware did not decrypt, command replies, regulatory snapshots + and country power packages. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 97b1b760372c16..781d4530fbee16 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -3,3 +3,4 @@ obj-$(CONFIG_MT7932_FULLMAC) += mt7932-fullmac.o mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend.o link.o startup.o reset.o lifecycle.o station.o power.o mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 927807e2e8ad37..1d6ef0596de367 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -715,6 +715,7 @@ void mt_disconnect_work(struct work_struct *work) spin_lock_irqsave(&m->response_lock, flags); m->disconnecting = false; cfg80211_disconnected(m->netdev, reason, NULL, 0, local, GFP_ATOMIC); + mt_reg_apply_deferred_locked(m); spin_unlock_irqrestore(&m->response_lock, flags); dev_dbg(&m->pdev->dev, "DISCONNECTED: teardown=%d local=%u\n", ret, local); } diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 8f041cb20d0bfe..11efdce6b7a5b4 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -115,6 +115,11 @@ struct mt7932 { bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; + /* A relaxation kept from the firmware while associated; applied by the + * next pass once the connection has ended. Under response_lock. + */ + struct mt7932_reg_snapshot reg_deferred; + bool reg_deferral; /* Channels the applied country package leaves out; under RTNL. */ DECLARE_BITMAP(policy_disabled, MT7932_CHANNELS); struct net_device *netdev; @@ -188,6 +193,7 @@ struct mt7932 { bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); +void mt_reg_apply_deferred_locked(struct mt7932 *m); int mt_net_open(struct net_device *netdev); int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name); /* response_lock held; closes RF admission and reports terminal host state. */ diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c new file mode 100644 index 00000000000000..1292c450ff54bb --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the regulatory snapshot logic and, as local files the + * driver must not trust, the country power packages. + */ +#include + +#include "mt7932.h" + +/* A world-domain snapshot: channels 1, 6, 36 and 149, the 5 GHz ones NO_IR. */ +static void reg_build(struct mt7932_reg_snapshot *reg) +{ + static const u16 channels[] = {1, 6, 36, 149}; + unsigned int i; + + memset(reg, 0, sizeof(*reg)); + memcpy(reg->alpha2, "00", 2); + memcpy(reg->domain, "XZ", 2); + reg->domain[4] = 1; + reg->domain[5] = 2; + reg->domain[8] = 2; + reg->domain[9] = 2; + for (i = 0; i < ARRAY_SIZE(channels); i++) { + put_unaligned_le16(channels[i], reg->domain + 12 + i * 8); + put_unaligned_le32(channels[i] > 14 ? IEEE80211_CHAN_NO_IR : 0, + reg->domain + 16 + i * 8); + reg->power[i] = 20; + } + reg->length = 12 + ARRAY_SIZE(channels) * 8; +} + +static void reg_set_flags(struct mt7932_reg_snapshot *reg, unsigned int entry, u32 flags) +{ + put_unaligned_le32(flags, reg->domain + 16 + entry * 8); +} + +/* A beacon hint lifting NO_IR only relaxes; nothing else does. */ +static void mt7932_reg_relaxation_test(struct kunit *test) +{ + struct mt7932_reg_snapshot *prev, *next; + + prev = kunit_kzalloc(test, sizeof(*prev), GFP_KERNEL); + next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, prev); + KUNIT_ASSERT_NOT_NULL(test, next); + reg_build(prev); + + reg_build(next); + reg_set_flags(next, 2, 0); + KUNIT_EXPECT_TRUE(test, mt7932_reg_only_relaxes(prev, next)); + /* An identical snapshot is not treated as a change by the caller, + * but it is trivially within the applied domain. + */ + KUNIT_EXPECT_TRUE(test, mt7932_reg_only_relaxes(prev, prev)); + + /* NO_IR added: a restriction. */ + reg_build(next); + reg_set_flags(next, 0, IEEE80211_CHAN_NO_IR); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + /* Radar or a bandwidth limit appears, or another flag goes away. */ + reg_build(next); + reg_set_flags(next, 2, IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_set_flags(prev, 3, IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_NO_HT40); + reg_build(next); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(prev); + /* A new country, power limit, channel or channel order. */ + reg_build(next); + memcpy(next->alpha2, "AU", 2); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + next->power[1] = 19; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + put_unaligned_le16(40, next->domain + 12 + 3 * 8); + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + put_unaligned_le16(44, next->domain + 12 + 4 * 8); + next->domain[9] = 3; + next->length += 8; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + /* A failed snapshot never relaxes, and a corrupt length is refused. */ + reg_build(next); + next->error = -EOPNOTSUPP; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); + reg_build(next); + prev->length = sizeof(prev->domain) + 8; + next->length = prev->length; + KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); +} + +static struct kunit_case mt7932_policy_test_cases[] = { + KUNIT_CASE(mt7932_reg_relaxation_test), + {} +}; + +static struct kunit_suite mt7932_policy_test_suite = { + .name = "mt7932-regulatory", + .test_cases = mt7932_policy_test_cases, +}; + +kunit_test_suite(mt7932_policy_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index fd65e07d9506ae..3a8c1ffc740151 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -5,6 +5,8 @@ /* Private, versioned packaging of original-derived country policy. This is * not firmware code and does not replace cfg80211's regulatory database. */ +#include + #include "channels.h" #define MT7932_POLICY_TABLES 13 @@ -102,6 +104,36 @@ static inline int mt7932_policy_parse(struct mt7932_policy *out, return at == size ? 0 : -EINVAL; } +/* + * Whether @next differs from the applied @prev only by NO_IR lifted on some + * channels, as a beacon hint does in the world domain: same country, firmware + * domain header, channel list and limits, and every channel's flags equal to + * the applied ones less, at most, NO_IR. Such a change only permits more; the + * firmware may keep the stricter domain until it is idle. + */ +static inline bool mt7932_reg_only_relaxes(const struct mt7932_reg_snapshot *prev, + const struct mt7932_reg_snapshot *next) +{ + unsigned int i, count; + + if (prev->error || next->error || prev->length != next->length || + prev->length < 12 || prev->length > sizeof(prev->domain) || + memcmp(prev->alpha2, next->alpha2, 2) || + memcmp(prev->domain, next->domain, 12) || + memcmp(prev->power, next->power, sizeof(prev->power))) + return false; + count = (prev->length - 12) / 8; + for (i = 0; i < count; i++) { + const u8 *a = prev->domain + 12 + i * 8, *b = next->domain + 12 + i * 8; + u32 was = get_unaligned_le32(a + 4), now = get_unaligned_le32(b + 4); + + if (memcmp(a, b, 4) || (now & ~was) || + ((was & ~now) & ~IEEE80211_CHAN_NO_IR)) + return false; + } + return true; +} + /* Every entry of a validated package starts with its channel number and is * followed by per-rate power limits. A country that does not permit a * channel sets all of that channel's limits to this marker. diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 6cd0afc035abd2..fda897e941cbc2 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -317,6 +317,20 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli * Consume the flag under the same lock as the worker publication, so repeated * userspace requests cannot continually invalidate an in-flight attempt. */ +/* response_lock held: start the pass a deferred relaxation is waiting for. */ +void mt_reg_apply_deferred_locked(struct mt7932 *m) +{ + lockdep_assert_held(&m->response_lock); + if (!m->reg_deferral || m->stopping) + return; + m->reg_deferral = false; + m->reg_desired = m->reg_deferred; + m->reg_generation++; + WRITE_ONCE(m->reg_pending, true); + if (READ_ONCE(m->interface_registered)) + schedule_work(&m->startup_work); +} + void mt_retry_missing_policy(struct mt7932 *m) { unsigned long flags; @@ -513,7 +527,24 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request spin_unlock_irqrestore(&m->response_lock, irqflags); return; } + if (!m->reg_pending && m->connected && !m->disconnecting && + mt7932_reg_only_relaxes(&m->reg_desired, ®) && + memcmp(®, &m->reg_desired, sizeof(reg))) { + /* A pass would end the connection only to permit more than the + * firmware now allows. Keep the stricter applied domain and run + * the pass once the connection has ended. + */ + m->reg_deferred = reg; + m->reg_deferral = true; + spin_unlock_irqrestore(&m->response_lock, irqflags); + for_each_set_bit(i, m->policy_disabled, MT7932_CHANNELS) + mt_channel(m, i)->flags |= IEEE80211_CHAN_DISABLED; + dev_dbg(&m->pdev->dev, "REGULATORY_RELAXATION_DEFERRED: %c%c until disconnected\n", + reg.alpha2[0], reg.alpha2[1]); + return; + } if (!memcmp(®, &m->reg_desired, sizeof(reg)) && !m->reg_pending) { + m->reg_deferral = false; spin_unlock_irqrestore(&m->response_lock, irqflags); /* The applied package still holds, but cfg80211 has just * rebuilt the channel flags without its disables. @@ -523,6 +554,7 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request return; } m->reg_desired = reg; + m->reg_deferral = false; m->reg_generation++; /* The next package pass records its own set. */ bitmap_zero(m->policy_disabled, MT7932_CHANNELS); From ef18b58fb6d5777c1375491e1b1f7c9f1d2ce13d Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:50:13 +1100 Subject: [PATCH 540/631] wifi: mt7932: test the power package parser The power packages are files from the local system, so the parser and the channel filter treat them as untrusted input. Add KUnit tests with synthetic country and world packages: valid packages, a package that names another country, every truncation, trailing data, header, mode, table length, entry count and table type bounds, the world package's 2.4 GHz channel order, the not-permitted marker, the channel filter and 2,000 random corruptions. No driver behaviour changes. Assisted-by: LLM Signed-off-by: Ryan Murray --- .../mediatek/mt7932/regulatory-test.c | 330 +++++++++++++++++- 1 file changed, 328 insertions(+), 2 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c index 1292c450ff54bb..225f3b8f35a0c0 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * KUnit tests for the regulatory snapshot logic and, as local files the - * driver must not trust, the country power packages. + * KUnit tests for the regulatory snapshot logic and the power package + * parser and channel filter. The packages are local files, so + * the parser treats them as untrusted. */ #include +#include #include "mt7932.h" @@ -90,8 +92,332 @@ static void mt7932_reg_relaxation_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, mt7932_reg_only_relaxes(prev, next)); } +#define PKG_MODE_SIZE 328 +#define PKG_ENTRY_SIZE 122 + +/* Channels of the synthetic 2.4 and 5 GHz tables 1..9, in table order. */ +static const u8 pkg_channels[] = { + 1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, + 36, 40, 44, 48, 52, 56, 60, 64, + 100, 104, 108, 112, 116, 120, 124, 128, + 132, 136, 140, 144, 149, 153, 157, 161, + 38, 46, 54, 62, 102, 110, 118, 126, + 134, 142, 151, 159, 42, 58, 106, 122, + 138, 155, 50, 114, 130, 146, 163, 167, + 165, 169, 171, 173, 175, +}; + +static unsigned int pkg_count(unsigned int table, bool world) +{ + if (table == 2) + return world ? 6 : 5; + return table == 9 ? 5 : 8; +} + +static size_t pkg_size(bool world) +{ + static const u16 others[] = {532, 144, 58, 265}; + size_t size = 16 + (world ? 2 : 3) * PKG_MODE_SIZE; + unsigned int i; + + for (i = 1; i <= 9; i++) + size += 44 + pkg_count(i, world) * PKG_ENTRY_SIZE; + for (i = 0; i < ARRAY_SIZE(others); i++) + size += others[i]; + return size; +} + +/* Build a structurally valid package. Channel 14 is only in the world one. */ +static u8 *pkg_build(struct kunit *test, const char *alpha2, size_t *size) +{ + static const u16 lengths[] = {532, 144, 58, 265}; + static const u8 types[] = {5, 1, 4, 2}; + bool world = !memcmp(alpha2, "XZ", 2); + unsigned int i, j, modes = world ? 2 : 3, channel = 0; + u8 *data, *p; + + *size = pkg_size(world); + data = kunit_kzalloc(test, *size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + memcpy(data, "J7RP", 4); + put_unaligned_le16(1, data + 4); + memcpy(data + 6, alpha2, 2); + data[8] = modes; + data[9] = MT7932_POLICY_TABLES; + put_unaligned_le32(*size, data + 12); + p = data + 16; + for (i = 0; i < modes; i++, p += PKG_MODE_SIZE) + put_unaligned_le32(17 + i, p + 4); + for (i = 0; i < MT7932_POLICY_TABLES; i++) { + if (i >= 1 && i <= 9) { + unsigned int count = pkg_count(i, world); + + p[0] = 3; + p[1] = 0x0a; + put_unaligned_le16(44 + count * PKG_ENTRY_SIZE, p + 2); + p[4] = count; + p[5] = i <= 2 ? 1 : 2; + p[6] = i == 2 || i == 9; + memcpy(p + 8, alpha2, 2); + for (j = 0; j < count; j++) { + u8 *entry = p + 44 + j * PKG_ENTRY_SIZE; + + /* A country package has no channel 14 entry. */ + if (!world && pkg_channels[channel] == 14) + channel++; + entry[0] = pkg_channels[channel++]; + memset(entry + 1, 30, PKG_ENTRY_SIZE - 1); + } + p += 44 + count * PKG_ENTRY_SIZE; + } else { + unsigned int k = i ? i - 9 : 0; + + p[1] = 2; + put_unaligned_le16(lengths[k], p + 2); + p[7] = types[k]; + if (i == 10) + memcpy(p + 8, alpha2, 2); + p += lengths[k]; + } + } + KUNIT_ASSERT_PTR_EQ(test, p, data + *size); + return data; +} + +/* The entry of @channel in a parsed package, or NULL. */ +static u8 *pkg_entry(const struct mt7932_policy *policy, unsigned int channel) +{ + unsigned int i, j; + + for (i = 1; i <= 9; i++) + for (j = 0; j < policy->table[i][4]; j++) + if (policy->table[i][44 + j * PKG_ENTRY_SIZE] == channel) + return (u8 *)policy->table[i] + 44 + j * PKG_ENTRY_SIZE; + return NULL; +} + +static void mt7932_policy_valid_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "AU", &size); + KUNIT_EXPECT_EQ(test, size, 10447); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "AU"), 0); + KUNIT_EXPECT_EQ(test, policy.modes, 3); + KUNIT_EXPECT_PTR_EQ(test, policy.table[0], data + 16 + 3 * PKG_MODE_SIZE); + KUNIT_EXPECT_EQ(test, policy.length[12], 265); + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 36)); + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 14)); + + data = pkg_build(test, "XZ", &size); + KUNIT_EXPECT_EQ(test, size, 10241); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), 0); + KUNIT_EXPECT_EQ(test, policy.modes, 2); + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 14)); +} + +/* A package serves only the country it names, in the header and table 10. */ +static void mt7932_policy_country_mismatch_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "AU", &size); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); + /* A country package renamed to the world one is not the world one. */ + memcpy(data + 6, "XZ", 2); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); +} + +/* Every prefix of a valid package, with a consistent header, is rejected. */ +static void mt7932_policy_truncation_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size, cut; + u8 *data, *copy; + + data = pkg_build(test, "PL", &size); + for (cut = 0; cut < size; cut++) { + /* An exact-size copy lets KASAN catch any read past the end. */ + copy = kmemdup(data, cut, GFP_KERNEL); + KUNIT_ASSERT_TRUE(test, copy || !cut); + if (cut >= 16) + put_unaligned_le32(cut, copy + 12); + KUNIT_EXPECT_EQ_MSG(test, mt7932_policy_parse(&policy, copy, cut, "PL"), + -EINVAL, "length %zu", cut); + kfree(copy); + } +} + +static void mt7932_policy_header_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *longer; + + data = pkg_build(test, "PL", &size); + /* Trailing bytes after the last table. */ + longer = kunit_kzalloc(test, size + 4, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, longer); + memcpy(longer, data, size); + put_unaligned_le32(size + 4, longer + 12); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, longer, size + 4, "PL"), -EINVAL); + + data[4] = 2; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[4] = 1; + data[8] = 4; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[8] = 3; + data[9] = 12; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + data[9] = MT7932_POLICY_TABLES; + put_unaligned_le32(size - 1, data + 12); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(size, data + 12); + /* Mode command lengths are bounded by the 320-byte body. */ + put_unaligned_le32(321, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(0, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le32(17, data + 16 + 4); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); +} + +/* Table lengths and entry counts must agree with the table type. */ +static void mt7932_policy_table_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *table; + + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + table = (u8 *)policy.table[3]; + /* An entry count that would run past the table. */ + table[4] = 9; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + table[4] = 8; + /* A table length that claims the rest of the file. */ + put_unaligned_le16(0xffff, table + 2); + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); + put_unaligned_le16(44 + 8 * PKG_ENTRY_SIZE, table + 2); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + table = (u8 *)policy.table[10]; + table[7] = 2; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), -EINVAL); +} + +/* The world package's 2.4 GHz tables must list channels 1..14 in order. */ +static void mt7932_policy_world_order_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data; + + data = pkg_build(test, "XZ", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), 0); + pkg_entry(&policy, 9)[0] = 10; + KUNIT_EXPECT_EQ(test, mt7932_policy_parse(&policy, data, size, "XZ"), -EINVAL); +} + +static void mt7932_policy_permits_test(struct kunit *test) +{ + struct mt7932_policy policy; + size_t size; + u8 *data, *entry; + + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + entry = pkg_entry(&policy, 144); + KUNIT_ASSERT_NOT_NULL(test, entry); + memset(entry + 1, MT7932_POLICY_NOT_PERMITTED, PKG_ENTRY_SIZE - 1); + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 144)); + /* One rate with a limit is enough to permit the channel. */ + entry[PKG_ENTRY_SIZE - 1] = 8; + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&policy, 144)); + /* A channel the package does not list is not permitted. */ + KUNIT_EXPECT_FALSE(test, mt7932_policy_permits(&policy, 177)); +} + +static void mt7932_policy_filter_test(struct kunit *test) +{ + static const u16 channels[] = {1, 6, 13, 36, 144, 149, 165}; + struct mt7932_reg_snapshot *reg; + struct mt7932_policy policy; + unsigned int i; + size_t size; + u8 *data; + + reg = kunit_kzalloc(test, sizeof(*reg), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, reg); + data = pkg_build(test, "PL", &size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&policy, data, size, "PL"), 0); + memset(pkg_entry(&policy, 144) + 1, MT7932_POLICY_NOT_PERMITTED, + PKG_ENTRY_SIZE - 1); + memset(pkg_entry(&policy, 6) + 1, MT7932_POLICY_NOT_PERMITTED, + PKG_ENTRY_SIZE - 1); + memcpy(reg->domain, "PL", 2); + for (i = 0; i < ARRAY_SIZE(channels); i++) { + put_unaligned_le16(channels[i], reg->domain + 12 + i * 8); + put_unaligned_le32(i, reg->domain + 16 + i * 8); + } + reg->domain[8] = 3; + reg->domain[9] = 4; + reg->length = 12 + ARRAY_SIZE(channels) * 8; + + mt7932_policy_filter(reg, &policy); + KUNIT_EXPECT_EQ(test, reg->length, 12 + 5 * 8); + KUNIT_EXPECT_EQ(test, reg->domain[8], 2); + KUNIT_EXPECT_EQ(test, reg->domain[9], 3); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(reg->domain + 12 + 8), 13); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(reg->domain + 12 + 3 * 8), 149); + /* Each entry keeps its own channel flags. */ + KUNIT_EXPECT_EQ(test, get_unaligned_le32(reg->domain + 16 + 3 * 8), 5); + /* The vacated tail is cleared. */ + KUNIT_EXPECT_TRUE(test, !memchr_inv(reg->domain + 12 + 5 * 8, 0, 2 * 8)); +} + +/* Random corruption must never be accepted in a way that reads out of bounds. */ +static void mt7932_policy_corruption_test(struct kunit *test) +{ + struct rnd_state rnd; + struct mt7932_policy policy; + unsigned int round, flips, channel; + size_t size; + u8 *data, *copy; + + prandom_seed_state(&rnd, 7932); + data = pkg_build(test, "AU", &size); + copy = kunit_kzalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, copy); + for (round = 0; round < 2000; round++) { + memcpy(copy, data, size); + for (flips = 1 + prandom_u32_state(&rnd) % 4; flips; flips--) + copy[prandom_u32_state(&rnd) % size] ^= 1 << (prandom_u32_state(&rnd) % 8); + if (mt7932_policy_parse(&policy, copy, size, "AU")) + continue; + for (channel = 0; channel < 256; channel++) + mt7932_policy_permits(&policy, channel); + } +} + static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_reg_relaxation_test), + KUNIT_CASE(mt7932_policy_valid_test), + KUNIT_CASE(mt7932_policy_country_mismatch_test), + KUNIT_CASE(mt7932_policy_truncation_test), + KUNIT_CASE(mt7932_policy_header_test), + KUNIT_CASE(mt7932_policy_table_test), + KUNIT_CASE(mt7932_policy_world_order_test), + KUNIT_CASE(mt7932_policy_permits_test), + KUNIT_CASE(mt7932_policy_filter_test), + KUNIT_CASE(mt7932_policy_corruption_test), {} }; From 02cdaa79c7e4f48c35b8b27c06c3295734fb8b55 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 06:50:57 +1100 Subject: [PATCH 541/631] drm/apple: afk: Export afk_service_retry() to the AFK service test afk_service_put() is an inline helper in afk.h that ends by calling afk_service_retry(), which lives in afk.c inside appledrm. The AFK service lifetime test is built as its own module and uses afk_service_put(), so with DRM_APPLE=m and DRM_APPLE_KUNIT_TEST=m (as KUNIT_ALL_TESTS=m selects) modpost fails: ERROR: modpost: "afk_service_retry" [drivers/gpu/drm/apple/afk-service-test.ko] undefined! Export the function to the KUnit test namespace and import that namespace in the test, as other drivers do for tests that live in their own module. The export is compiled out without CONFIG_KUNIT. Fixes: 7357e51053e4 ("drm/apple: scope deferred DPAV recovery to current interfaces") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/gpu/drm/apple/afk-service-test.c | 1 + drivers/gpu/drm/apple/afk.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index a86df389deb177..1cf589ae7a4172 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -238,4 +238,5 @@ static struct kunit_suite afk_service_suite = { }; kunit_test_suite(afk_service_suite); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index 7e23edc851b2da..e85c36d1b87704 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -16,6 +16,8 @@ #include #include +#include + #include "afk.h" #include "afk_reply.h" #include "afkv2-codec.h" @@ -70,6 +72,7 @@ void afk_service_retry(struct apple_dcp_afkep *ep) queue_work(ep->wq, &ep->service_work); spin_unlock_irqrestore(&ep->lock, flags); } +EXPORT_SYMBOL_IF_KUNIT(afk_service_retry); static void afk_send(struct apple_dcp_afkep *ep, u64 message) { From a09a20899a3e32bc0fed859112427c66e0d37e93 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:11:50 +1100 Subject: [PATCH 542/631] soc: apple: sep: Name the failed step of a Touch ID sensor registration When the enclave did not register the sensor, the log said so and nothing else. Eight steps can fail on the way: an enclave refusal of REGISTER_SENSOR, REGISTER_SENSOR_SERIAL, the session share or its commit, and a failed serial read, nonce read or session exchange on the sensor side. Most of them returned false without a word, so a failure could not be told apart from any other. Name the step that failed with the enclave status or the error, and how long after the sensor was powered on. Then report what the sensor shows afterwards: its state, the patch acknowledgment, whether it presents an identifier and how long a frame it holds. The sensor data itself is never logged. The sensor_trace parameter also logs each step as it completes, with its time since power-on. The host test that runs wake_sensor() gets the new fields in its mock. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 137 ++++++++++++++++----- drivers/soc/apple/sensor.rs | 8 ++ drivers/soc/apple/sep.rs | 9 ++ tools/aurora-sep/tests/test_sensor_wake.py | 15 ++- 4 files changed, 134 insertions(+), 35 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index ffa06b06a08b77..c4cbd2edb18f94 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1765,6 +1765,64 @@ impl SepData { } } + fn sensor_ms(&self) -> u64 { + shim::boottime_ns().saturating_sub(self.sensor_power_ns.load(Relaxed)) / 1_000_000 + } + + fn sensor_trace(&self, step: &CStr) { + if *module_parameters::sensor_trace.value() != 0 { + dev_info!(self.dev, "sensor: {} done at {} ms\n", step, self.sensor_ms()); + } + } + + /// Reports a failed registration step with what the sensor shows + /// afterwards: its state, patch acknowledgment, whether it presents an + /// identifier and the length of any frame it holds. Never the + /// identifier, the serial or any key material. + fn sensor_step_failed(&self, step: &CStr) { + let ms = self.sensor_ms(); + match sensor::status() { + Ok(st) => dev_warn!( + self.dev, + "sensor: {} failed {} ms after power-on; sensor now state {} ack {:#04x} identifier {} holding {} bytes{}\n", + step, + ms, + st.state, + st.patch_ack(), + st.identifier_to_register().is_some(), + st.available().unwrap_or(0), + if st.is_silent() { c" (silent)" } else { c"" } + ), + Err(e) => dev_warn!( + self.dev, + "sensor: {} failed {} ms after power-on; its status read failed too ({:?})\n", + step, + ms, + e + ), + } + } + + /// One enclave request of the registration. A refusal is named. + fn sensor_enclave_step(&self, op: &crate::sbio::SbioOp) -> Option> { + match self.sbio_transfer(op) { + Ok(done) if done.status.is_ok() => { + self.sensor_trace(op.name()); + Some(done.payload) + } + Ok(done) => { + dev_warn!(self.dev, "sensor: {} refused with status {}\n", op.name(), done.status); + self.sensor_step_failed(op.name()); + None + } + Err(e) => { + dev_warn!(self.dev, "sensor: {} transfer failed: {:?}\n", op.name(), e); + self.sensor_step_failed(op.name()); + None + } + } + } + fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); @@ -1778,9 +1836,12 @@ impl SepData { return false; } }; - if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) - || self.sbio_expect_ok(&crate::sbio::sbio_register_sensor(id)).is_none() - { + if !self.sensor_session_request(0x48, c"PLATFORM_IDENTITY", &platform) { + self.sensor_step_failed(c"PLATFORM_IDENTITY"); + return false; + } + self.sensor_trace(c"PLATFORM_IDENTITY"); + if self.sensor_enclave_step(&crate::sbio::sbio_register_sensor(id)).is_none() { return false; } } @@ -1788,7 +1849,7 @@ impl SepData { if stage >= BRINGUP_ESTABLISHED { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self - .sbio_expect_ok(&crate::sbio::sbio_clear_state()) + .sensor_enclave_step(&crate::sbio::sbio_clear_state()) .is_none() { return false; @@ -1816,23 +1877,24 @@ impl SepData { let serial = match sensor::read_sensor_serial() { Ok(serial) => serial, - Err(_) => { + Err(e) => { + dev_warn!(self.dev, "sensor: serial read failed: {:?}\n", e); + self.sensor_step_failed(c"serial read"); return false; } }; + self.sensor_trace(c"serial read"); let op = crate::sbio::sbio_register_sensor_serial(&serial); - match self.sbio_call(&op) { - SbioOutcome::Ok(_payload) => { - if self.profile.nonce_session && !self.authenticate_sensor_session() { - return false; - } - self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); - let _ = self.log_identity_count(c"after sensor serial registration"); - true - } - _ => false, + if self.sensor_enclave_step(&op).is_none() { + return false; + } + if self.profile.nonce_session && !self.authenticate_sensor_session() { + return false; } + self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + let _ = self.log_identity_count(c"after sensor serial registration"); + true } fn sensor_platform_identity() -> Result<[u8; 12]> { @@ -1866,12 +1928,16 @@ impl SepData { Ok(nonce) => nonce, Err(error) => { dev_err!(self.dev, "sensor: nonce read failed: {:?}\n", error); + self.sensor_step_failed(c"nonce read"); return false; } }; + self.sensor_trace(c"nonce read"); if !self.sensor_session_request(0x81, c"SENSOR_NONCE", &nonce) { + self.sensor_step_failed(c"SENSOR_NONCE"); return false; } + self.sensor_trace(c"SENSOR_NONCE"); if !self.sensor_key_loaded.load(Relaxed) { let firmware = match kernel::firmware::Firmware::request( @@ -1890,9 +1956,11 @@ impl SepData { if firmware.data().is_empty() || !self.sensor_session_request(0x86, c"SENSOR_ROOT_KEY", firmware.data()) { + self.sensor_step_failed(c"SENSOR_ROOT_KEY"); return false; } self.sensor_key_loaded.store(true, Relaxed); + self.sensor_trace(c"SENSOR_ROOT_KEY"); } // Each identification starts a new sensor-side exchange, even though // the signed key remains loaded for the lifetime of the enclave. @@ -1901,6 +1969,7 @@ impl SepData { fn wake_sensor(&self) -> Option { let _source = sensor::power_source(); + self.sensor_power_ns.store(shim::boottime_ns(), Relaxed); let cycled = sensor::power_cycle(); if !sensor::cs_timing().is_hardware() { dev_warn!( @@ -1941,6 +2010,13 @@ impl SepData { answered_without_identifier = true; continue; }; + if *module_parameters::sensor_trace.value() != 0 { + dev_info!( + self.dev, + "sensor: identifier present {} ms after power-on\n", + self.sensor_ms() + ); + } if !self.register_sensor(&id) { dev_warn!(self.dev, "sensor: the enclave did not register the sensor\n"); return None; @@ -2161,43 +2237,37 @@ impl SepData { } fn establish_session(&self) -> bool { - let share = match self.sbio_call(&crate::sbio::sbio_request_session_share()) { - SbioOutcome::Ok(sh) => sh, - SbioOutcome::Status16 => { - return false; - } - SbioOutcome::PrerequisiteMissing => { - return false; - } - SbioOutcome::Other => { - return false; - } + let Some(share) = self.sensor_enclave_step(&crate::sbio::sbio_request_session_share()) + else { + return false; }; // 0x30 reserved for this reply; length is a floor, take the first 40 if share.len() < sensor::SESSION_SHARE_LEN { + dev_warn!(self.dev, "sensor: short session share ({} bytes)\n", share.len()); return false; } let mut out = [0u8; sensor::SESSION_SHARE_LEN]; out.copy_from_slice(&share[..sensor::SESSION_SHARE_LEN]); // relay it: class 0x72, no CRC, no padding - if sensor::send_session_share(&out).is_err() { + if let Err(e) = sensor::send_session_share(&out) { + dev_warn!(self.dev, "sensor: sending the session share failed: {:?}\n", e); + self.sensor_step_failed(c"session share send"); return false; } let reply = match sensor::read_session_reply() { Ok(r) => r, - Err(_) => { + Err(e) => { + dev_warn!(self.dev, "sensor: session reply read failed: {:?}\n", e); + self.sensor_step_failed(c"session reply read"); return false; } }; - match self.sbio_transfer(&crate::sbio::sbio_commit_session_share(&reply)) { - Ok(done) if done.status.is_ok() => true, - Ok(_) => false, - Err(_) => false, - } + self.sensor_enclave_step(&crate::sbio::sbio_commit_session_share(&reply)) + .is_some() } // Each failure names its step: a report that only says "sequence counter @@ -2251,6 +2321,7 @@ impl SepData { Ok(done) if done.status.is_ok() => true, Ok(done) => { dev_warn!(self.dev, "sequence counter: the enclave refused the sensor's reply, status {}\n", done.status); + self.sensor_step_failed(c"challenge commit"); false } Err(e) => { diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index 3fcba35e39428f..dc3df5c6b1d348 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -635,6 +635,14 @@ impl Identifier { } impl Status { + /// The length of the frame the sensor offers, if it offers one. + pub(crate) fn available(&self) -> Option { + match self.offset12() { + Offset12::Available(n) => Some(n), + _ => None, + } + } + pub(crate) fn offset12(&self) -> Offset12 { match self.state { STATE_IDLE => Offset12::Identifier(u16::from_le_bytes([ diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 5af1a76c70e254..3c6179ee6c586d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -630,6 +630,10 @@ struct SepData { last_capture_end_ns: Atomic, + // Boot-time clock at the start of the last sensor power cycle, for + // reporting how far into a bring-up a step failed. + sensor_power_ns: Atomic, + device_view_synced: Atomic, keybag_designated: Atomic, @@ -864,6 +868,7 @@ impl SepData { enrol_material <- new_mutex!(None), enrol_identity_candidates <- new_mutex!(KVec::new()), last_capture_end_ns: Atomic::new(0), + sensor_power_ns: Atomic::new(0), device_view_synced: Atomic::new(false), phase: Atomic::new(PHASE_ATTACH), ool_xarm <- new_mutex!(Some(ool_xarm)), @@ -2492,6 +2497,10 @@ module! { default: 0, description: "Low 64 bits of an explicit xART OS UUID", }, + sensor_trace: u8 { + default: 0, + description: "Trace Touch ID sensor registration steps and timing, never sensor data", + }, xarm_trace: u8 { default: 0, description: "Opt-in xART opcode/status trace without payloads or key material", diff --git a/tools/aurora-sep/tests/test_sensor_wake.py b/tools/aurora-sep/tests/test_sensor_wake.py index c9bb8c6af75792..cd1c9c74a793eb 100644 --- a/tools/aurora-sep/tests/test_sensor_wake.py +++ b/tools/aurora-sep/tests/test_sensor_wake.py @@ -22,6 +22,15 @@ def test_post_registration_status_drives_patch_readiness(self): #![allow(dead_code, unused_variables)] use std::cell::RefCell; macro_rules! dev_warn { ($($args:tt)*) => {} } +macro_rules! dev_info { ($($args:tt)*) => {} } +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +mod shim { pub fn boottime_ns()->u64 {0} } +mod module_parameters { + pub struct Param(u8); + impl Param { pub fn value(&self)->&u8 {&self.0} } + #[allow(non_upper_case_globals)] pub static sensor_trace: Param = Param(0); +} +struct Profile { nonce_session: bool } macro_rules! dev_err { ($($args:tt)*) => {} } mod kernel { pub mod time { pub struct Delta; @@ -60,8 +69,10 @@ def test_post_registration_status_drives_patch_readiness(self): pub fn patch_ack(&self)->u8 {self.ack} } } -struct Driver; +struct Driver { profile: Profile, sensor_power_ns: AtomicU64 } impl Driver { + fn new()->Self { Driver { profile: Profile { nonce_session: false }, sensor_power_ns: AtomicU64::new(0) } } + fn sensor_ms(&self)->u64 {0} fn register_sensor(&self,_:&sensor::Identifier)->bool { MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,1);m.registered=true;}); true } @@ -80,7 +91,7 @@ def test_post_registration_status_drives_patch_readiness(self): (Some(sensor::Status{state:0,ack:0,silent:false}),true,Some(0)), ] { MOCK.with(|m| *m.borrow_mut()=Mock{next,..Mock::default()}); - assert_eq!(Driver.wake_sensor().is_some(),ready); + assert_eq!(Driver::new().wake_sensor().is_some(),ready); MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,2);assert!(m.registered);assert_eq!(m.patch,patch);}); } } From d4c27628103efd917b41652229eda3c473b16ee5 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:29:43 +1100 Subject: [PATCH 543/631] soc: apple: sep: Clear a partial sensor registration before retrying it On a profile with a nonce session, every registration identifies the sensor first: PLATFORM_IDENTITY, then REGISTER_SENSOR. The enclave state is only cleared once the bring-up stage reads ESTABLISHED. If a registration fails after REGISTER_SENSOR was accepted, the stage stays FRESH while the enclave holds the partial registration. Each retry then identified the sensor again without clearing it. On boot d1962b3a, all 42 retries over 800 s failed that way, and Touch ID stayed unusable until a reboot. Count the enclave as holding the sensor once REGISTER_SENSOR has been submitted, answered or not, since a lost or late reply does not mean the enclave did not act on it. Clear its sensor state before registering again whenever it may hold one. Restarting the bring-up from FRESH does not change that, because only CLEAR_STATE empties the enclave. Profiles without a nonce session keep their flow. The decision is a plain function, with KUnit tests for the first registration, the retry after a partial one, re-registration and the other profiles. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/registration.rs | 116 ++++++++++++++++++++++++++++++ drivers/soc/apple/sbio.rs | 26 +++++-- drivers/soc/apple/sep.rs | 11 ++- 3 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 drivers/soc/apple/registration.rs diff --git a/drivers/soc/apple/registration.rs b/drivers/soc/apple/registration.rs new file mode 100644 index 00000000000000..95edd6a0b65b59 --- /dev/null +++ b/drivers/soc/apple/registration.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Copyright 2026 Ryan Murray + +//! What a Touch ID sensor registration has to send to the enclave. +//! +//! A registration identifies the sensor (REGISTER_SENSOR), registers its +//! serial and, on profiles with a nonce session, authenticates it. Any of +//! those steps can fail after the enclave has already accepted an earlier +//! one, and then the enclave holds a partial registration. On boot +//! d1962b3a every retry that registered the sensor again without clearing +//! it failed, 42 of them over 800 s. Re-registration that identifies the +//! sensor and then clears the enclave's sensor state, the order every +//! verify uses, has not failed on an idle machine. So the next attempt +//! clears whenever the enclave may hold any registration, complete or not. + +/// Nothing is registered with the enclave yet. +pub(crate) const FRESH: u32 = 0; +/// The sensor is identified; profiles without a nonce session register the +/// serial next. +pub(crate) const IDENTIFIED: u32 = 1; +/// The registration completed. +pub(crate) const ESTABLISHED: u32 = 2; + +/// The enclave requests one registration attempt makes, in order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Plan { + /// PLATFORM_IDENTITY and REGISTER_SENSOR, before anything else. A + /// nonce-session profile identifies the sensor on every reset. + pub(crate) identify_first: bool, + /// CLEAR_STATE, after any identification. + pub(crate) clear_state: bool, + /// REGISTER_SENSOR after the clear, on a profile without a nonce session. + pub(crate) register_sensor: bool, +} + +/// Plans one registration attempt. +/// +/// `stage` is the bring-up stage and `holds_sensor` whether an +/// identification of the sensor was submitted to the enclave since it +/// booted, answered or not: a lost reply does not mean the enclave did not +/// act on it. `sepos13` says whether the profile starts from the serial, as +/// the 13.5 key store does. +pub(crate) fn plan(stage: u32, holds_sensor: bool, nonce_session: bool, sepos13: bool) -> Plan { + Plan { + identify_first: nonce_session, + clear_state: stage >= ESTABLISHED || (nonce_session && holds_sensor), + register_sensor: stage == FRESH && !nonce_session && !sepos13, + } +} + +#[kernel::macros::kunit_tests(apple_sep_registration)] +mod tests { + use super::*; + + #[test] + fn first_registration_after_boot_does_not_clear() { + assert_eq!( + plan(FRESH, false, true, false), + Plan { + identify_first: true, + clear_state: false, + register_sensor: false + } + ); + } + + #[test] + fn retry_after_a_partial_registration_clears() { + // REGISTER_SENSOR was accepted, then the serial, nonce or session + // step failed: the bring-up stage is still FRESH. + let retry = plan(FRESH, true, true, false); + assert!(retry.identify_first); + assert!(retry.clear_state); + } + + #[test] + fn re_registration_after_a_complete_one_clears() { + assert!(plan(ESTABLISHED, true, true, false).clear_state); + assert!(plan(ESTABLISHED, false, false, false).clear_state); + } + + #[test] + fn a_fresh_restart_still_clears_what_the_enclave_holds() { + // Restarting the bring-up from FRESH does not erase the enclave's + // registration; only CLEAR_STATE does. + assert!(plan(FRESH, true, true, false).clear_state); + } + + #[test] + fn profiles_without_a_nonce_session_are_unchanged() { + assert_eq!( + plan(FRESH, true, false, false), + Plan { + identify_first: false, + clear_state: false, + register_sensor: true + } + ); + assert_eq!( + plan(IDENTIFIED, true, false, false), + Plan { + identify_first: false, + clear_state: false, + register_sensor: false + } + ); + assert_eq!( + plan(FRESH, false, false, true), + Plan { + identify_first: false, + clear_state: false, + register_sensor: false + } + ); + } +} diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index c4cbd2edb18f94..8ec5de59ffeeec 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1825,9 +1825,24 @@ impl SepData { fn register_sensor(&self, id: &sensor::Identifier) -> bool { let stage = self.bringup.load(Relaxed); - dev_info!(self.dev, "sbio: registering sensor at bring-up stage {}\n", stage); + let plan = registration::plan( + stage, + self.sensor_held.load(Relaxed), + self.profile.nonce_session, + matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }), + ); + dev_info!( + self.dev, + "sbio: registering sensor at bring-up stage {}{}\n", + stage, + if plan.clear_state && stage < BRINGUP_ESTABLISHED { + c", clearing the enclave's partial registration first" + } else { + c"" + } + ); - if self.profile.nonce_session { + if plan.identify_first { // J700 needs identification before CLEAR_STATE on every reset. let platform = match Self::sensor_platform_identity() { Ok(platform) => platform, @@ -1841,12 +1856,15 @@ impl SepData { return false; } self.sensor_trace(c"PLATFORM_IDENTITY"); + // Counted as held from submission on: a reply that is lost or + // late does not mean the enclave did not act on the request. + self.sensor_held.store(true, Relaxed); if self.sensor_enclave_step(&crate::sbio::sbio_register_sensor(id)).is_none() { return false; } } - if stage >= BRINGUP_ESTABLISHED { + if plan.clear_state { let _ = self.log_identity_count(c"before CLEAR_STATE for sensor re-registration"); if self .sensor_enclave_step(&crate::sbio::sbio_clear_state()) @@ -1863,7 +1881,7 @@ impl SepData { // The 13.5 J313 flow begins with REGISTER_SENSOR_SERIAL (0x17). // The newer REGISTER_SENSOR (0x80) returned 0x16 on this firmware. self.bringup.store(BRINGUP_IDENTIFIED, Relaxed); - } else if stage == BRINGUP_FRESH { + } else if plan.register_sensor { let op = crate::sbio::sbio_register_sensor(id); match self.sbio_call(&op) { SbioOutcome::Ok(_payload) => { diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 3c6179ee6c586d..697268980a0836 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -25,6 +25,7 @@ mod profile; mod proto; mod refkey; mod refkey_seal; +mod registration; mod rxring; mod sbio; mod scrd; @@ -251,9 +252,9 @@ enum SbioOutcome { Other, } -const BRINGUP_FRESH: u32 = 0; -const BRINGUP_IDENTIFIED: u32 = 1; -const BRINGUP_ESTABLISHED: u32 = 2; +const BRINGUP_FRESH: u32 = registration::FRESH; +const BRINGUP_IDENTIFIED: u32 = registration::IDENTIFIED; +const BRINGUP_ESTABLISHED: u32 = registration::ESTABLISHED; pub(crate) struct PatchLoaded(()); @@ -643,6 +644,9 @@ struct SepData { touchid_failed: Atomic, bringup: Atomic, + // The enclave accepted an identification of the sensor since it booted, + // even if the registration it began then failed. + sensor_held: Atomic, #[pin] ool_sks: Mutex>, @@ -854,6 +858,7 @@ impl SepData { sks_initialized: Atomic::new(false), sks_wedged: Atomic::new(0), bringup: Atomic::new(BRINGUP_FRESH), + sensor_held: Atomic::new(false), keybag_designated: Atomic::new(false), bringup_started: Atomic::new(false), touchid_started: Atomic::new(false), From 760302abaf457d2593bf2f14f426a842c88cb108 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:30:12 +1100 Subject: [PATCH 544/631] soc: apple: sep: Re-read a sensor frame that is still on offer A read of a sensor frame can fail on the bus in two ways. The bytes come back garbled, or the sensor does not take the frame. Either way it keeps offering the frame at the same length. Boot 0874c215 showed both: the nonce read failed its CRC, and afterwards the sensor still offered the complete 43-byte nonce frame with a valid CRC. On boot 7f0c01be the serial read succeeded but the serial frame was not taken. The nonce command that followed then waited behind it until it timed out. Check the CRC of the serial, nonce, session and challenge replies. The session and challenge replies were relayed unchecked, but they carry the same CRC; it held on every boot measured. The calibration module reply keeps its own handling. When a reply fails its CRC and the sensor still offers a frame of that length, read it again, at most twice. Before a command that expects a reply, take any stale frame the sensor still offers, so the reply is not queued behind it. Two such frames at most: if one is still on offer after that, fail the step rather than send the command, because a read does not prove the sensor took the frame. Report both, since either means an exchange went wrong on the bus. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 15 +++++ drivers/soc/apple/sensor.rs | 130 +++++++++++++++++++++++++++--------- 2 files changed, 112 insertions(+), 33 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 8ec5de59ffeeec..1c41b25cbf64a1 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1775,11 +1775,25 @@ impl SepData { } } + /// Reports frames the sensor exchange had to read again or discard. + fn note_sensor_recoveries(&self) { + let (reread, drained) = sensor::take_recoveries(); + if reread != 0 || drained != 0 { + dev_warn!( + self.dev, + "sensor: re-read {} frame(s) the sensor still offered after a garbled read, discarded {} stale frame(s)\n", + reread, + drained + ); + } + } + /// Reports a failed registration step with what the sensor shows /// afterwards: its state, patch acknowledgment, whether it presents an /// identifier and the length of any frame it holds. Never the /// identifier, the serial or any key material. fn sensor_step_failed(&self, step: &CStr) { + self.note_sensor_recoveries(); let ms = self.sensor_ms(); match sensor::status() { Ok(st) => dev_warn!( @@ -1911,6 +1925,7 @@ impl SepData { return false; } self.bringup.store(BRINGUP_ESTABLISHED, Relaxed); + self.note_sensor_recoveries(); let _ = self.log_identity_count(c"after sensor serial registration"); true } diff --git a/drivers/soc/apple/sensor.rs b/drivers/soc/apple/sensor.rs index dc3df5c6b1d348..c0f3a6159f74c9 100644 --- a/drivers/soc/apple/sensor.rs +++ b/drivers/soc/apple/sensor.rs @@ -5,6 +5,7 @@ //! The fingerprint sensor over SPI. use kernel::prelude::*; +use kernel::sync::atomic::{Atomic, Relaxed}; use kernel::time::delay::fsleep; use kernel::time::Delta; @@ -118,6 +119,7 @@ const CMD_START_CAPTURE: [u8; CMD_LEN] = [0x80, 0x40, 0x00, 0x07, 0x00, 0x00, 0x // Precedes the patch blob. const CMD_SETUP_PATCH_ENABLE: [u8; CMD_LEN] = [0x80, 0x60, 0x00, 0x07, 0x00, 0x00, 0x00]; const CMD_GET_SENSOR_SERIAL: [u8; CMD_LEN] = [0x80, 0x70, 0x00, 0x07, 0x00, 0x00, 0x00]; +const CMD_GET_NONCE: [u8; CMD_LEN] = [0x80, 0x35, 0x00, 0x07, 0x00, 0x00, 0x00]; const READ_CMD_LEN: usize = 11; const CMD_READ_PREFIX: [u8; CMD_LEN] = [0x80, 0x13, 0x00, 0x0b, 0x00, 0x00, 0x00]; @@ -475,26 +477,18 @@ fn poll_until_ready_any(accepted: &[usize]) -> Result { } pub(crate) fn send_session_share(share: &[u8; SESSION_SHARE_LEN]) -> Result<()> { + drain_stale_frames()?; send_framed(TYPE_SESSION, share) } pub(crate) fn send_challenge(challenge: &[u8; CHALLENGE_LEN]) -> Result<()> { + drain_stale_frames()?; send_framed(TYPE_SEQUENCE, challenge) } static_assert!(FRAME_HEADER + SESSION_SHARE_LEN == 0x2f); static_assert!(FRAME_HEADER + CHALLENGE_LEN == 0x47); -fn read_framed(total: usize, at: usize, out: &mut [u8]) -> Result<()> { - let mut buf = [0xffu8; MAX_REPLY]; - read_reply(total, &mut buf)?; - if at + out.len() > total { - return Err(EINVAL); - } - out.copy_from_slice(&buf[at..at + out.len()]); - Ok(()) -} - fn poll_until_ready(expected: usize) -> Result<()> { for _ in 0..READ_POLL_ATTEMPTS { let st = status()?; @@ -511,11 +505,6 @@ fn poll_until_ready(expected: usize) -> Result<()> { const READ_POLL_MS: i64 = 5; const READ_POLL_ATTEMPTS: usize = 200; -fn read_reply(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { - poll_until_ready(total)?; - read_reply_ready(total, buf) -} - fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { if total > MAX_REPLY { return Err(EINVAL); @@ -538,22 +527,99 @@ fn read_reply_ready(total: usize, buf: &mut [u8; MAX_REPLY]) -> Result<()> { }) } -fn read_framed_verified(total: usize, at: usize, out: &mut [u8]) -> Result<()> { - if total < CRC_LEN || at + out.len() > total - CRC_LEN { - return Err(EINVAL); +/// Frames the sensor still offered after the read that should have taken +/// them, and stale frames found on offer before a command. Both mean an +/// exchange went wrong on the bus; the caller reports them. +static REREADS: Atomic = Atomic::new(0); +static DRAINED: Atomic = Atomic::new(0); + +/// Takes the counts of re-read and drained frames since the last call. +pub(crate) fn take_recoveries() -> (u32, u32) { + (REREADS.xchg(0, Relaxed), DRAINED.xchg(0, Relaxed)) +} + +// A garbled read the sensor did not take is read once more, twice at most. +const FRAME_READS: usize = 3; +// Stale frames taken before a command; the sensor queues at most a couple. +const DRAIN_FRAMES: usize = 2; + +/// A reply buffer, wiped however the read that fills it ends. +struct FrameBuf([u8; MAX_REPLY]); + +impl FrameBuf { + fn new() -> Self { + FrameBuf([0xff; MAX_REPLY]) } - let mut buf = [0xffu8; MAX_REPLY]; - read_reply(total, &mut buf)?; +} - let split = total - CRC_LEN; - let expected = u16::from_le_bytes([buf[split], buf[split + 1]]); - let computed = crc16_ansi(&buf[..split]); - if expected != computed { - return Err(EIO); +impl Drop for FrameBuf { + fn drop(&mut self) { + for b in self.0.iter_mut() { + // SAFETY: a valid, uniquely borrowed byte. + unsafe { core::ptr::write_volatile(b, 0) }; + } + } +} + +fn frame_crc_valid(frame: &[u8]) -> bool { + let split = frame.len() - CRC_LEN; + u16::from_le_bytes([frame[split], frame[split + 1]]) == crc16_ansi(&frame[..split]) +} + +/// Takes any frame the sensor still offers before a new command, so the +/// reply to that command is not mistaken for, or queued behind, a stale one. +/// Fails if the sensor still offers a frame once the budget is spent: a +/// read is no proof the sensor took the frame. +fn drain_stale_frames() -> Result<()> { + for _ in 0..DRAIN_FRAMES { + let Some(count) = status()?.available().filter(|n| *n != 0) else { + return Ok(()); + }; + let count = count as usize; + if count > MAX_REPLY { + return Err(EIO); + } + let mut buf = FrameBuf::new(); + read_reply_ready(count, &mut buf.0)?; + DRAINED.fetch_add(1, Relaxed); + } + match status()?.available() { + None | Some(0) => Ok(()), + Some(_) => Err(EBUSY), + } +} + +/// Sends `cmd` and reads its `total`-byte reply, checking the reply's CRC. +fn command_frame(cmd: &[u8; CMD_LEN], total: usize, at: usize, out: &mut [u8]) -> Result<()> { + drain_stale_frames()?; + command(cmd)?; + read_frame(total, at, out) +} + +/// Reads a `total`-byte reply once the sensor offers it and checks its CRC. +/// A read the sensor did not take leaves the frame on offer at the same +/// length: only then is it read again. +fn read_frame(total: usize, at: usize, out: &mut [u8]) -> Result<()> { + if total < CRC_LEN || total > MAX_REPLY || at + out.len() > total - CRC_LEN { + return Err(EINVAL); } + poll_until_ready(total)?; - out.copy_from_slice(&buf[at..at + out.len()]); - Ok(()) + let mut buf = FrameBuf::new(); + for attempt in 0..FRAME_READS { + if attempt > 0 { + if status()?.available() != Some(total as u32) { + break; + } + REREADS.fetch_add(1, Relaxed); + } + read_reply_ready(total, &mut buf.0)?; + if frame_crc_valid(&buf.0[..total]) { + out.copy_from_slice(&buf.0[at..at + out.len()]); + return Ok(()); + } + } + Err(EIO) } const MAX_REPLY: usize = 0x4f; @@ -576,17 +642,15 @@ impl SensorSerial { } pub(crate) fn read_sensor_serial() -> Result { - command(&CMD_GET_SENSOR_SERIAL)?; let mut serial = [0u8; SENSOR_SERIAL_LEN]; - read_framed_verified(SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; + command_frame(&CMD_GET_SENSOR_SERIAL, SENSOR_SERIAL_REPLY_LEN, SENSOR_SERIAL_AT, &mut serial)?; Ok(SensorSerial(serial)) } /// Read the nonce as one frame; a separate acknowledgment would consume it. pub(crate) fn session_nonce() -> Result<[u8; 32]> { - command(&[0x80, 0x35, 0, 7, 0, 0, 0])?; let mut value = [0; 32]; - read_framed_verified(9 + value.len() + CRC_LEN, 9, &mut value)?; + command_frame(&CMD_GET_NONCE, 9 + value.len() + CRC_LEN, 9, &mut value)?; if value.iter().all(|byte| *byte == 0) { return Err(EIO); } @@ -595,13 +659,13 @@ pub(crate) fn session_nonce() -> Result<[u8; 32]> { pub(crate) fn read_session_reply() -> Result<[u8; SESSION_SHARE_LEN]> { let mut share = [0u8; SESSION_SHARE_LEN]; - read_framed(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; + read_frame(SESSION_REPLY_LEN, SESSION_REPLY_AT, &mut share)?; Ok(share) } pub(crate) fn read_challenge_reply() -> Result<[u8; CHALLENGE_LEN]> { let mut reply = [0u8; CHALLENGE_LEN]; - read_framed(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; + read_frame(CHALLENGE_REPLY_LEN, CHALLENGE_REPLY_AT, &mut reply)?; Ok(reply) } From cc534b157112d060e1e10354eb7424762f73f5f0 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Thu, 15 Oct 2026 22:30:52 +1100 Subject: [PATCH 545/631] soc: apple: sep: Retry a failed attach-time sensor bring-up When the bring-up at attach failed, nothing retried it until the first verify, minutes later. That verify was then refused before it touched the sensor, and gdm offered no fingerprint at login. A bring-up that fails on the bus leaves nothing a new attempt cannot undo. Each attempt power-cycles the sensor and, after a failed one, clears the enclave partial registration first. That is the same path every verify takes, and it has never failed on an idle machine. Try up to three times at attach, logging each failed attempt. A power cycle that fails ends the attempt before registration: a sensor that was not reset may still hold the session of the attempt before. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 1c41b25cbf64a1..5a03e24e0d602f 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1100,12 +1100,28 @@ impl SepData { fn attach_bringup(&self) { if self.profile.nonce_session { - if !self.bring_sensor_online() { - dev_warn!( - self.dev, - "sensor: authenticated attach incomplete; capture unavailable\n" - ); + // A bring-up that fails on the bus leaves nothing a new attempt + // cannot undo: each attempt power-cycles the sensor and, after a + // failed one, clears the enclave's partial registration first, + // as every verify does. + const ATTEMPTS: u32 = 3; + for attempt in 1..=ATTEMPTS { + if self.bring_sensor_online() { + return; + } + if attempt < ATTEMPTS { + dev_warn!( + self.dev, + "sensor: attach-time bring-up attempt {} of {} failed; starting again\n", + attempt, + ATTEMPTS + ); + } } + dev_warn!( + self.dev, + "sensor: authenticated attach incomplete; capture unavailable\n" + ); return; } if !sensor::is_bound() { @@ -2004,6 +2020,12 @@ impl SepData { let _source = sensor::power_source(); self.sensor_power_ns.store(shim::boottime_ns(), Relaxed); let cycled = sensor::power_cycle(); + if !cycled && self.profile.nonce_session { + // A sensor that was not reset may still hold the session of an + // earlier attempt; registering it again cannot be trusted. + dev_err!(self.dev, "sensor: the power cycle failed; not registering the sensor\n"); + return None; + } if !sensor::cs_timing().is_hardware() { dev_warn!( self.dev, From ede975e8c46483e80f6a6a77ddf939a925597e23 Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 02:17:40 +1100 Subject: [PATCH 546/631] soc: apple: sep: Drop an SBIO reply that belongs to an earlier exchange An SBIO reply is matched to the waiting exchange by opcode alone, and an error marker is matched before the opcode is checked. The receive work reads a reply before it takes the reassembly lock. If the requester times out in between and a retry begins the same opcode, the late reply completes the retry with the earlier attempt result. Bring-up now retries the registration, so the window is real. Number the exchanges. The receive work samples the number before it reads a reply and passes it with the chunk; a chunk sampled before the current exchange began is dropped. Replies to the current exchange can only arrive after it began, since its request is sent after that. Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/sbio.rs | 5 ++- drivers/soc/apple/transfer.rs | 83 ++++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 5a03e24e0d602f..6137215f57b305 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -2725,6 +2725,9 @@ impl SepData { if !self.sbio_ready.load(Relaxed) { return; } + // Before reading the reply: a requester that times out meanwhile and + // begins a new exchange must not receive it. + let generation = self.sbio_rx.lock().generation(); // notification and the enclave's DMA are not ordered; buffer may still be poison if !self.ool_await_written(&self.ool_sbio, 0, transfer::HEADER_LEN) { @@ -2763,7 +2766,7 @@ impl SepData { } }; - let progress = self.sbio_rx.lock().on_chunk(marker, &packet, &payload); + let progress = self.sbio_rx.lock().on_chunk(generation, marker, &packet, &payload); match progress { transfer::Progress::NeedMore(cont) => { diff --git a/drivers/soc/apple/transfer.rs b/drivers/soc/apple/transfer.rs index 1f77f451625760..a4f5f0df3e308b 100644 --- a/drivers/soc/apple/transfer.rs +++ b/drivers/soc/apple/transfer.rs @@ -177,6 +177,10 @@ pub(crate) struct Reassembly { done: Option, sending: Option, grants: u32, + // Counts exchanges begun. A reply is matched only by its opcode, so a + // late reply to an exchange that timed out would otherwise complete the + // next exchange with the same opcode. + generation: u64, } impl Reassembly { @@ -186,14 +190,22 @@ impl Reassembly { done: None, sending: None, grants: 0, + generation: 0, } } + /// The exchange a message arriving now can belong to. Sampled before the + /// message is read and passed back to [`Self::on_chunk`]. + pub(crate) fn generation(&self) -> u64 { + self.generation + } + pub(crate) fn begin(&mut self, opcode: u32) -> Result<()> { if self.active.is_some() { return Err(EBUSY); } self.done = None; + self.generation = self.generation.wrapping_add(1); self.active = Some(Active { opcode, total: 0, @@ -242,7 +254,16 @@ impl Reassembly { self.abort(); } - pub(crate) fn on_chunk(&mut self, marker: u8, packet: &Packet, payload: &[u8]) -> Progress { + /// Takes one chunk of a reply. `generation` is [`Self::generation`] as + /// sampled when the message arrived: a chunk that arrived before the + /// current exchange began belongs to an earlier one and is dropped. + pub(crate) fn on_chunk( + &mut self, + generation: u64, + marker: u8, + packet: &Packet, + payload: &[u8], + ) -> Progress { if marker < MARKER_FIRST { return Progress::Notification; } @@ -262,6 +283,9 @@ impl Reassembly { let Some(active_opcode) = self.active.as_ref().map(|a| a.opcode) else { return Progress::Ignored; }; + if generation != self.generation { + return Progress::Ignored; + } // Checked before the opcode test below, deliberately. if marker == MARKER_ERROR { @@ -371,3 +395,60 @@ impl Reassembly { self.done = Some(Completed { status, payload }); } } + +#[kernel::macros::kunit_tests(apple_sep_transfer)] +mod tests { + use super::*; + + const OPCODE: u32 = 0x48; + + fn reply(err: u32) -> Packet { + Packet { + version: VERSION, + total: 0, + offset: 0, + flags: 0, + err, + opcode: OPCODE, + chunk: 0, + } + } + + #[test] + fn a_reply_to_the_current_exchange_completes_it() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + let generation = rx.generation(); + assert!(matches!( + rx.on_chunk(generation, MARKER_FIRST, &reply(0), &[]), + Progress::Complete + )); + assert!(rx.take_done().is_some_and(|done| done.status.is_ok())); + } + + #[test] + fn a_late_reply_does_not_complete_the_next_exchange() { + let mut rx = Reassembly::new(); + assert!(rx.begin(OPCODE).is_ok()); + // The reply to the first exchange is being read when it times out. + let late = rx.generation(); + rx.abort(); + assert!(rx.take_done().is_some()); + // A retry begins the same opcode before the late reply is delivered. + assert!(rx.begin(OPCODE).is_ok()); + assert!(matches!( + rx.on_chunk(late, MARKER_FIRST, &reply(0), &[]), + Progress::Ignored + )); + assert!(matches!( + rx.on_chunk(late, MARKER_ERROR, &reply(0x16), &[]), + Progress::Ignored + )); + assert!(rx.take_done().is_none()); + let current = rx.generation(); + assert!(matches!( + rx.on_chunk(current, MARKER_FIRST, &reply(0), &[]), + Progress::Complete + )); + } +} From 4ecd4330795cbde5f1d3ef983eb8caa30938ca7f Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Fri, 16 Oct 2026 07:27:23 +1100 Subject: [PATCH 547/631] soc: apple: sep: Refuse a damaged host-state store instead of erasing it Store::open() initialises the host-state store whenever the file is shorter than the store or its header does not match: it zeroes all 72 blocks and writes an empty superblock. That was by design while the store held records that a seed import restored after a reset. Since the SEP's anti-replay records moved to the shared xART gigalocker, though, the file holds Linux state that nothing regenerates: the Touch ID identity index and the lockout state. A truncated file, a damaged header or a store in a format this driver does not know is now erased on the next probe, together with the evidence needed to recover it. Initialise the store only when this open created the file and it is empty, and refuse an existing file that is short or has an unknown header without writing to it. That includes what an interrupted creation leaves: in general the driver cannot tell it from a damaged store, so the decision is the administrator's. The log line says when the file is empty or all zero: that is all an interrupted creation leaves, and such a file holds nothing to keep. Creating a new store is logged too: on a machine that has used Touch ID it means the old store was removed. Opening also has two ways to lose or leak a record of a store with a valid header. A pending write intent that names a used block makes it free that block, discarding the committed record, and a record length beyond its 32 KiB block makes read() allocate up to 4 GiB and return the following blocks. No writer of the format produces either; refuse them as well instead of acting on them. The on-disk format does not change. Fixes: 7165ae9d98bf ("soc: apple: use shared xART gigalocker") Assisted-by: LLM Signed-off-by: Ryan Murray --- drivers/soc/apple/shim.h | 1 + drivers/soc/apple/shim.rs | 16 ++++++- drivers/soc/apple/store.rs | 84 ++++++++++++++++++++++++++++++++-- drivers/soc/apple/store_shim.c | 17 +++++++ 4 files changed, 113 insertions(+), 5 deletions(-) diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 66b39903c6d81f..40aae82b111016 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -44,6 +44,7 @@ void sep_hwrng_unregister(void *mem); /* -- store_shim.c ------------------------------------------------------- */ void *sep_store_open(const char *path); +void *sep_store_open_host(const char *path, bool *created); void *sep_store_open_trunc(const char *path); void *sep_store_open_ro(const char *path); void *sep_store_open_block(const char *path, int writable); diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 4e25060d15d25a..7e7f489e594818 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -7,6 +7,7 @@ use kernel::prelude::*; extern "C" { fn sep_store_open(path: *const c_char) -> *mut c_void; + fn sep_store_open_host(path: *const c_char, created: *mut bool) -> *mut c_void; fn sep_store_open_trunc(path: *const c_char) -> *mut c_void; fn sep_store_open_ro(path: *const c_char) -> *mut c_void; fn sep_store_open_block(path: *const c_char, writable: c_int) -> *mut c_void; @@ -29,7 +30,7 @@ pub(crate) fn random_bytes(buf: &mut [u8]) -> Result<()> { /// Backing-store file handle. /// /// # Invariants -/// `handle` is a live `struct file *` from `sep_store_open`. +/// `handle` is a live `struct file *` from one of the store-open shims. pub(crate) struct StoreFile { handle: *mut c_void, } @@ -56,6 +57,19 @@ impl StoreFile { Ok(StoreFile { handle }) } + /// Opens the file like [`StoreFile::open`], and also reports whether this + /// call created it. + pub(crate) fn open_host(path: &CStr) -> Result<(StoreFile, bool)> { + let mut created = false; + // SAFETY: `path` is NUL-terminated and `created` is valid for writes + // for the duration of the call. The shim returns a live file or an + // error pointer. + let handle = unsafe { + kernel::error::from_err_ptr(sep_store_open_host(path.as_char_ptr(), &mut created))? + }; + Ok((StoreFile { handle }, created)) + } + pub(crate) fn open_trunc(path: &CStr) -> Result { // SAFETY: `path` is NUL-terminated; the shim returns NULL on failure. let handle = unsafe { sep_store_open_trunc(path.as_char_ptr()) }; diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index dc37025ce3b8ad..03bd6f1bf0a01d 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -73,6 +73,20 @@ const SLOT_ENTRY_SIZE: usize = 24; const INTENT_NONE: u8 = 0; const INTENT_WRITE: u8 = 1; +/// Why an existing file is refused. It is logged, so it never carries record +/// contents. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Damage { + /// The file is shorter than the store. + Size, + /// The magic, version or geometry is not this format's. + Header, + /// The slot's record is longer than its block. + Slot(usize), + /// The pending write intent names a block that holds a committed record. + Intent, +} + // Not internally locked; the caller holds a mutex around all access. pub(crate) struct Store { file: shim::StoreFile, @@ -92,9 +106,34 @@ fn le64(buf: &[u8], off: usize) -> u64 { u64::from_le_bytes(v) } +/// Describes an existing file that holds no store as "empty" or "all zero". +/// That is all an interrupted creation leaves, and such a file holds nothing +/// to keep; anything else may be what is left of a store. Reads at most a +/// store's worth of the file. +fn describe(file: &shim::StoreFile, size: u64) -> Option<&'static str> { + if size == 0 { + return Some("empty"); + } + if size > STORE_SIZE as u64 { + return None; + } + let mut buf = KVec::new(); + buf.resize(BLOCK_SIZE, 0, GFP_KERNEL).ok()?; + let mut off = 0; + while off < size { + let len = (size - off).min(BLOCK_SIZE as u64) as usize; + file.read_exact(off, &mut buf[..len]).ok()?; + if buf[..len].iter().any(|&b| b != 0) { + return None; + } + off += len as u64; + } + Some("all zero") +} + impl Store { pub(crate) fn open() -> Result { - let file = shim::StoreFile::open(STORE_PATH)?; + let (file, created) = shim::StoreFile::open_host(STORE_PATH)?; let mut store = Store { file, @@ -104,11 +143,18 @@ impl Store { fresh: false, }; + // Only a new, empty file is initialised. An existing file may hold the + // only copy of the host state, so one that this driver cannot use is + // refused and left as it is. let size = store.file.size()?; - if size < STORE_SIZE as u64 { + if created && size == 0 { store.create()?; + pr_info!("created a new host-state store\n"); return Ok(store); } + if size < STORE_SIZE as u64 { + return Err(store.refuse(Damage::Size, size)); + } let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; sb.resize(BLOCK_SIZE, 0, GFP_KERNEL)?; @@ -119,8 +165,7 @@ impl Store { || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE { - store.create()?; - return Ok(store); + return Err(store.refuse(Damage::Header, size)); } store.generation = le64(&sb, SB_GENERATION); @@ -134,6 +179,10 @@ impl Store { uuid, len: le32(&sb, off + 18), }; + // A longer record would be read from the blocks of the next slots. + if store.slots[i].used && store.slots[i].len as usize > MAX_VALUE { + return Err(store.refuse(Damage::Slot(i), size)); + } } // Crash recovery: writes are copy-on-write, so discard the intent, never complete it. @@ -142,6 +191,12 @@ impl Store { store.recovered = true; let slot = u16::from_le_bytes([sb[SB_INTENT_SLOT], sb[SB_INTENT_SLOT + 1]]) as usize; if intent_kind == INTENT_WRITE && slot < SLOT_COUNT { + // The intent names the free block that was being filled. One + // that names a committed record is damage, and freeing that + // block would discard the record. + if store.slots[slot].used { + return Err(store.refuse(Damage::Intent, size)); + } store.slots[slot] = Slot::FREE; } store.commit()?; @@ -150,6 +205,27 @@ impl Store { Ok(store) } + /// Logs why the existing file is refused, and returns the error. Nothing + /// is written to the file. + fn refuse(&self, damage: Damage, size: u64) -> Error { + let what = match damage { + Damage::Size | Damage::Header => describe(&self.file, size), + _ => None, + }; + match what { + Some(what) => pr_err!( + "host-state store failed validation ({:?}, {}), refusing it\n", + damage, + what + ), + None => pr_err!( + "host-state store failed validation ({:?}), refusing it\n", + damage + ), + } + EIO + } + fn create(&mut self) -> Result<()> { self.slots = [Slot::FREE; SLOT_COUNT]; self.generation = 1; diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index bdfd5dfbb6391d..4c89f92eb5f13b 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -51,6 +51,23 @@ void *sep_store_open(const char *path) return IS_ERR(f) ? NULL : f; } +/* + * As sep_store_open(), but reports whether this call created the file, so that + * only a new file is ever initialised. Returns an error pointer on failure. + */ +void *sep_store_open_host(const char *path, bool *created) +{ + struct file *f; + + *created = false; + f = open_as_kernel(path, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE, 0600); + if (PTR_ERR_OR_ZERO(f) == -EEXIST) + return open_as_kernel(path, O_RDWR | O_LARGEFILE, 0); + if (!IS_ERR(f)) + *created = true; + return f; +} + /* * Opens truncated to empty, for callers that rewrite the whole file each time, * so a shorter record cannot leave stale trailing bytes from a larger one. From 0c986f352044c361a4dc9976c3c982fb8bb31102 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:01 -0400 Subject: [PATCH 548/631] drm/asahi: serve M1/M2/M3 and G17P from one driver Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. Integration's driver is the base. It covers the M1/M2 firmware, the M3 (G15S) runtime and the J613 HAL200 runtime. The G17P runtime from aurora-wip joins it as another backend of the DRM-facing GPU interface, and the shared files select the G17 paths per device or VM, so M1/M2/M3 behaviour does not change. The g17/ code is aurora-wip's. The UAPI keeps integration's numbers and layouts, which existing M3 userspace depends on. The G17P additions move past them: - feature bits 10 to 15; - the render and compute command fields after integration's 64-byte timestamp-copy arrays. G17P rejects nonzero timestamp copies. Neo Mesa must use the matching header. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../gpu/drm/asahi/{asahi_core.rs => asahi.rs} | 2 +- drivers/gpu/drm/asahi/driver.rs | 92 +- drivers/gpu/drm/asahi/drm_gpu.rs | 119 +- drivers/gpu/drm/asahi/file.rs | 125 +- drivers/gpu/drm/asahi/g16_submit.rs | 14 +- drivers/gpu/drm/asahi/g17/command.rs | 17 + drivers/gpu/drm/asahi/g17/job.rs | 1 + drivers/gpu/drm/asahi/g17/mod.rs | 1 - .../gpu/drm/asahi/g17/queue/compute/exit.rs | 1 - drivers/gpu/drm/asahi/g17/runtime/backend.rs | 6 - .../drm/asahi/g17/runtime/render/settle.rs | 3 - drivers/gpu/drm/asahi/g17/runtime/teardown.rs | 6 - drivers/gpu/drm/asahi/gem.rs | 35 +- drivers/gpu/drm/asahi/gpu.rs | 180 +- drivers/gpu/drm/asahi/hw/mod.rs | 19 - drivers/gpu/drm/asahi/identity.rs | 1 + drivers/gpu/drm/asahi/m3_submit.rs | 14 +- drivers/gpu/drm/asahi/mmu.rs | 1826 ++++++++++++----- drivers/gpu/drm/asahi/mmu/bind.rs | 1 - drivers/gpu/drm/asahi/mmu/lifetime.rs | 9 + drivers/gpu/drm/asahi/pgtable.rs | 489 +++-- drivers/gpu/drm/asahi/queue/mod.rs | 21 +- drivers/gpu/drm/asahi/regs.rs | 42 +- include/uapi/drm/asahi_drm.h | 278 +-- rust/kernel/drm/gem/shmem.rs | 4 + rust/kernel/drm/gpuvm.rs | 206 +- rust/kernel/drm/sched.rs | 69 +- tools/asahi/neo-auxiliary-tests.py | 4 +- 28 files changed, 2293 insertions(+), 1292 deletions(-) rename drivers/gpu/drm/asahi/{asahi_core.rs => asahi.rs} (98%) diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi.rs similarity index 98% rename from drivers/gpu/drm/asahi/asahi_core.rs rename to drivers/gpu/drm/asahi/asahi.rs index beb85ace60ada9..46d73120a918cd 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi.rs @@ -111,7 +111,6 @@ mod agx_resources; mod agx_status; mod agx_uapi; mod gem; -mod gem_stats; mod g17; mod gpu; mod hw; @@ -140,6 +139,7 @@ struct DriverModule { // Field drop order unregisters/joins driver producers before queue teardown. #[pin] _driver: kernel::driver::Registration>, + // Deferred G17P cleanup work; allocated at module load on every platform. _cleanup: cleanup::QueueOwner, } diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index fd2c5430d40ca5..b7a02abb7a0a15 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -45,6 +45,9 @@ pub(crate) struct AsahiData { pub(crate) completion: SetOnce>, pub(crate) pdev: ARef, pub(crate) resources: Option, + /// Kernel GEM objects are marked held for the device (`gem::set_mapping_held`). Only the + /// G17P runtime sets this. + pub(crate) hold_kernel_pages: bool, #[pin] g16_completion_work: Work, @@ -59,10 +62,33 @@ impl AsahiData { completion: SetOnce::new(), pdev, resources, + hold_kernel_pages: false, g16_completion_work <- new_work!("AsahiData::g16_completion_work"), }) } + /// Driver data of a G17P device. + fn new_g17(pdev: &platform::Device, resources: regs::Resources) -> impl PinInit { + let pdev: ARef = pdev.into(); + try_pin_init!(Self { + gpu: SetOnce::new(), + is_m3: false, + completion: SetOnce::new(), + pdev, + resources: Some(resources), + hold_kernel_pages: true, + g16_completion_work <- new_work!("AsahiData::g16_completion_work"), + }) + } + + /// The G17P runtime, if this device runs it. + pub(crate) fn g17(&self) -> Option<&Arc> { + match self.gpu.as_ref() { + Some(drm_gpu::Backend::G17(gpu)) => Some(gpu), + _ => None, + } + } + /// Runtime construction can use GEM before the GPU backend exists. All /// DRM operations fail with ENODEV until probe publishes the backend once. pub(crate) fn gpu(&self) -> Result<&dyn drm_gpu::DrmGpu> { @@ -90,6 +116,7 @@ enum AsahiRuntime { M3(crate::m3_drm::Registered), G16(crate::g16_drm::Registered), G15(crate::g15_probe::G15Manager), + G17(ARef>, Arc), } pub(crate) struct AsahiDriver { @@ -104,6 +131,8 @@ pub(crate) enum ProbeConfig { /// topology decode run at probe time, then the probe fails closed before /// DMA setup, ASC start, or any MMIO write. Agx3Diagnostic(&'static hw::agx3::SocConfig), + /// G17 (T8140): the G17P firmware runtime. + G17(&'static g17::Config), } unsafe impl Send for AsahiDriver {} @@ -133,6 +162,7 @@ impl drm::driver::Driver for AsahiDriver { type Object = drm::gem::shmem::Object; const INFO: drm::driver::DriverInfo = INFO; + /// Open files and exported buffers pin the module while G17P deferred cleanup can run. const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); const FEATURES: u32 = drm::driver::FEAT_GEM | drm::driver::FEAT_RENDER @@ -168,22 +198,12 @@ impl drm::driver::Driver for AsahiDriver { } } -/// Firmware interface selected by the hardware compatible. -pub(crate) enum ProbeConfig { - Legacy(&'static hw::HwConfig), - G17(&'static g17::Config), -} - -// OF Device ID table. +// OF Device ID table.s kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, ::IdInfo, [ - ( - of::DeviceId::new(c_str!("apple,agx-t8140")), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), ( of::DeviceId::new(c_str!("apple,agx-t8103")), ProbeConfig::Supported(&hw::t8103::HWCONFIG) @@ -223,8 +243,8 @@ kernel::of_device_table!( // while its table is incomplete; T6031 starts that runtime only with // asahi.t6031_start=1 and otherwise fails closed the same way; the // others fail closed before any firmware handoff. - // The M4 (G16) and M5/A18 Pro (G17) bring-up runtimes are not bound - // in this kernel. + // The M4 (G16) and M5 bring-up runtimes are not bound in this kernel. + // The A18 Pro (T8140, G17P) runtime is bound below. ( of::DeviceId::new(c_str!("apple,agx-t8122")), ProbeConfig::Agx3Diagnostic(&hw::agx3::T8122) @@ -241,6 +261,10 @@ kernel::of_device_table!( of::DeviceId::new(c_str!("apple,agx-t6032")), ProbeConfig::Agx3Diagnostic(&hw::agx3::T6032) ), + ( + of::DeviceId::new(c_str!("apple,agx-t8140")), + ProbeConfig::G17(&hw::t8140::CONFIG) + ), ] ); @@ -431,10 +455,9 @@ impl platform::Driver for AsahiDriver { match &this.runtime { AsahiRuntime::M3(runtime) => runtime.stop(), AsahiRuntime::G16(runtime) => runtime.stop(), - _ => {}, - if let Some(gpu) = (*this.drm).gpu.as_any().downcast_ref::() { // Driver data is dropped after devres, so CPU control must happen in unbind. - gpu.shutdown(); + AsahiRuntime::G17(_, gpu) => gpu.shutdown(), + _ => {}, } } @@ -500,7 +523,6 @@ impl platform::Driver for AsahiDriver { } } ProbeConfig::Agx3Diagnostic(soc) => return Err(refuse_agx3_probe(pdev, soc)), - ProbeConfig::Legacy(cfg) => *cfg, ProbeConfig::G17(cfg) => return Self::probe_g17(pdev, cfg), }; @@ -569,43 +591,27 @@ impl AsahiDriver { // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; let res = regs::Resources::new(pdev)?; - // SAFETY: GPU construction cannot access driver data before init_data(). - // Failed construction releases the allocation without dropping uninitialized data. - let drm: ARef = unsafe { drm::device::Device::new_uninit(pdev.as_ref())? }; - let gpu = g17::Gpu::new(pdev, &drm, cfg, res.clone())?; - let data_gpu = gpu.clone() as Arc; - let data = try_pin_init!(AsahiData { - gpu: data_gpu, - pdev: pdev.into(), - resources: res, - }); - // SAFETY: This is the sole initializer; no driver-data users have been published. - if let Err(error) = unsafe { drm.init_data(data) } { + // No userspace or GPU backend is published before the runtime is constructed. + let drm: ARef = + drm::device::Device::new(pdev.as_ref(), AsahiData::new_g17(pdev, res.clone()))?; + let gpu = g17::Gpu::new(pdev, &drm, cfg, res)?; + if !drm.gpu.populate(drm_gpu::Backend::G17(gpu.clone())) { gpu.shutdown(); - return Err(error); + return Err(EBUSY); } if let Err(error) = drm::driver::Registration::new_foreign_owned(&drm, pdev.as_ref(), 0) { gpu.shutdown(); return Err(error); } - // G17's firmware channel format has no supported stats decoder yet. - let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register( - raw_dev, core::ptr::null(), false, - ) { - Ok(handle) => handle, - Err(error) => { - gpu.shutdown(); - return Err(error); - } - }; - Ok(Self { drm, raw_dev, raw_stats, stats: None }) + Ok(Self { + runtime: AsahiRuntime::G17(drm, gpu), + }) } } impl Drop for AsahiDriver { fn drop(&mut self) { - if let Some(gpu) = (*self.drm).gpu.as_any().downcast_ref::() { + if let AsahiRuntime::G17(_, gpu) = &self.runtime { // Also covers failure to allocate platform driver data after the probe body returned. gpu.shutdown(); } diff --git a/drivers/gpu/drm/asahi/drm_gpu.rs b/drivers/gpu/drm/asahi/drm_gpu.rs index c9d997186d12a6..cd1b720b55f6b7 100644 --- a/drivers/gpu/drm/asahi/drm_gpu.rs +++ b/drivers/gpu/drm/asahi/drm_gpu.rs @@ -4,7 +4,8 @@ //! //! The legacy manager remains unchanged behind [`LegacyDrmGpu`]. M3 uses //! the same file and ioctl layer through its own adapter without pretending to -//! be an AGX2 firmware manager. +//! be an AGX2 firmware manager. The G17P runtime reaches this layer through its +//! [`gpu::Gpu`] implementation, adapted below. use core::ops::Range; @@ -14,7 +15,7 @@ use kernel::{ uapi, }; -use crate::{alloc, file, fw, gem, gpu, mmu, queue}; +use crate::{alloc, file, fw, g17, gem, gpu, mmu, queue}; /// Hardware facts returned through `DRM_ASAHI_GET_PARAMS`. pub(crate) struct DrmGpuParams { @@ -71,6 +72,16 @@ pub(crate) trait DrmGpu: Send + Sync { fn fwctl(&self, _msg: fw::channels::FwCtlMsg) -> Result { Err(ENODEV) } + + /// Independent-queue limits; only the G17P runtime has them. + fn queue_limits(&self) -> Option { + None + } + /// Temporary capacity hint and validity in milliseconds for subsequent syncobj waits; + /// only the G17P runtime gives one. + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + None + } } /// Publish an existing manager without allocating another reference-counted adapter. @@ -78,6 +89,7 @@ pub(crate) enum Backend { Legacy(LegacyDrmGpu), M3(Arc), G16(Arc), + G17(Arc), } impl Backend { @@ -85,7 +97,110 @@ impl Backend { match self { Self::Legacy(manager) => manager, Self::M3(runtime) | Self::G16(runtime) => &**runtime, + Self::G17(runtime) => &**runtime, + } + } +} + +/// The G17P runtime's `gpu::Gpu` operations, as the file layer sees them. +impl DrmGpu for g17::Gpu { + fn init(&self) -> Result { + gpu::Gpu::init(self) + } + + fn ids(&self) -> &gpu::SequenceIDs { + gpu::Gpu::ids(self) + } + + fn is_crashed(&self) -> bool { + gpu::Gpu::is_crashed(self) + } + + fn supports_vm_status(&self) -> bool { + gpu::Gpu::supports_vm_status(self) + } + + /// G17P has no G13/G14 globals. + fn update_globals(&self) {} + + fn extra_features(&self) -> u64 { + // The out-of-line bindgen `Default` is not exported to modules; zero-fill instead. + let mut params: uapi::drm_asahi_params_global = pin_init::zeroed(); + if gpu::Gpu::get_params(self, &mut params).is_err() { + return 0; + } + let mut features = params.features; + if gpu::Gpu::supports_vm_status(self) { + features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + } + features + } + + fn params(&self) -> Result { + // The out-of-line bindgen `Default` is not exported to modules; zero-fill instead. + let mut params: uapi::drm_asahi_params_global = pin_init::zeroed(); + gpu::Gpu::get_params(self, &mut params)?; + let mut core_masks = [0u32; uapi::DRM_ASAHI_MAX_CLUSTERS as usize]; + for (mask, reported) in core_masks.iter_mut().zip(params.core_masks.iter()) { + *mask = u32::try_from(*reported).map_err(|_| EIO)?; } + Ok(DrmGpuParams { + gpu_generation: params.gpu_generation, + gpu_variant: params.gpu_variant, + gpu_revision: params.gpu_revision, + chip_id: params.chip_id, + num_dies: params.num_dies, + num_clusters_total: params.num_clusters_total, + num_cores_per_cluster: params.num_cores_per_cluster, + core_masks, + max_frequency_khz: params.max_frequency_khz, + usc_generation: params.usc_generation, + gpu_hal_generation: params.gpu_hal_generation, + max_commands_per_submission: file::MAX_COMMANDS_PER_SUBMISSION, + }) + } + + fn user_range(&self) -> Result> { + Ok(gpu::Gpu::uat_geometry(self).user_usable_range()) + } + + fn unknown_page(&self) -> Result { + Ok(gpu::Gpu::uat_geometry(self).unk_page()) + } + + fn base_clock_hz(&self) -> u32 { + gpu::Gpu::base_clock_hz(self) + } + + fn new_vm(&self, kernel_range: Range) -> Result { + gpu::Gpu::new_vm(self, kernel_range) + } + + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + gpu::Gpu::new_queue(self, vm, ualloc, ualloc_priv, priority, usc_exec_base) + } + + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result { + gpu::Gpu::map_timestamp_buffer(self, bo, range) + } + + fn queue_limits(&self) -> Option { + gpu::Gpu::queue_limits(self) + } + + fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { + gpu::Gpu::syncobj_wait_hint(self) } } diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 2d518e16adffdc..e977f63617bd3f 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -38,7 +38,6 @@ use kernel::error::code::*; use kernel::new_mutex; use kernel::prelude::*; use kernel::sync::{ - aref::ARef, Arc, Mutex, // }; @@ -64,11 +63,11 @@ const LEGACY_VM_USER_RANGE: Range = mmu::IOVA_USER_BASE..mmu::IOVA_UNK_PAGE /// A client instance of an `mmu::Vm` address space. struct Vm { + /// Residency reclaim of a G17 VM; None elsewhere. shrinker: Option>, ualloc: Arc>, ualloc_priv: Arc>, vm: mmu::Vm, - user_range: Range, kernel_range: Range, user_range: Range, _dummy_mapping: mmu::KernelMapping, @@ -77,6 +76,14 @@ struct Vm { impl Drop for Vm { fn drop(&mut self) { if self.vm.status().is_some() { + // G17 VM. + self.shrinker = None; + self.vm.close_idle_reclaim(); + if let Some(status) = self.vm.status() { + // Retained graphs may keep this status alive; the client's mirror + // object must not outlive the client's VM handle. + status.release_mirror(); + } // Tracked jobs can outlive their file's VM handle. Their last guard performs // these unmaps once no accepted work can still dereference the mappings. if let Err(error) = self @@ -88,6 +95,7 @@ impl Drop for Vm { self.vm.bo_deferred_cleanup(); return; } + if !self.vm.is_m3() { // When the user Vm is dropped, unmap everything in the user range @@ -236,8 +244,6 @@ pub(crate) enum Object { // #[pin_data] pub(crate) struct File { id: u64, - /// Available VM range for the user - user_range: Range, // #[pin] vms: xarray::XArray>, // #[pin] @@ -331,15 +337,11 @@ impl File { let gpu = device.gpu()?; - if data.param_group == crate::agx_status::PARAM_GROUP_VM_STATUS { - return Self::get_vm_status(device, data, file); - } - match data.param_group { - uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS if gpu.supports_vm_status() => { - return Self::get_vm_status(data, file); + crate::agx_status::PARAM_GROUP_VM_STATUS => { + return Self::get_vm_status(device, data, file); } - uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR if gpu.supports_vm_status() => { + uapi::DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR if device.g17().is_some() => { return Self::register_vm_status_mirror(data, file); } uapi::DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS => { @@ -402,15 +404,10 @@ impl File { params.features |= crate::agx_status::FEATURE_VM_STATUS; } - if gpu.supports_vm_status() { - params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; - } if gpu.queue_limits().is_some() { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; } - if *module_parameters::fault_control.value() == 0xb { if gpu.supports_scheduled_queues() { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SCHEDULED_QUEUES as u64; } @@ -435,6 +432,10 @@ impl File { { params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; } + // The G17P runtime keeps its own soft-fault rule. + if device.g17().is_some() && *module_parameters::fault_control.value() == 0xb { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + } let size = core::mem::size_of::().min(data.size.try_into()?); @@ -462,36 +463,29 @@ impl File { let mut bytes = [0u8; VM_STATUS_SIZE]; UserSlice::new(user, VM_STATUS_SIZE).reader().read_slice(&mut bytes)?; let vm_id = decode_request(&bytes).ok_or(EINVAL)?; - let status = { + let (g17_status, status) = { let vms = file.inner().vms(); let guard = vms.lock(); let vm = guard.get(vm_id as usize).ok_or(ENOENT)?; - vm.vm.status()?.clone() + match vm.vm.status() { + // G17 VMs keep their own sticky status object. + Some(status) => (Some(status.clone()), None), + None => (None, Some(vm.vm.agx_status()?.clone())), + } + }; + let error = match (g17_status, status) { + (Some(status), _) => status.reported_error(), + (None, Some(status)) => { + status.record(device.gpu()?.submission_error()); + status.get() + } + (None, None) => return Err(EINVAL), }; - status.record(device.gpu()?.submission_error()); - let bytes = encode_response(vm_id, status.get()); + let bytes = encode_response(vm_id, error); UserSlice::new(user, VM_STATUS_SIZE).writer().write_slice(&bytes)?; Ok(0) } - /// Query only file-owned VM state; this does not start recovery or wait for work. - fn get_vm_status(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { - let size = core::mem::size_of::(); - if data.pad != 0 || data.size < size as u64 { - return Err(EINVAL); - } - let user = UserPtr::from_addr(data.pointer as _); - let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; - if flags != 0 || pad != 0 { - return Err(EINVAL); - } - let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; - Ok(0) - } - /// Register a file-owned GEM word as the mirror of a file-owned VM's status. fn register_vm_status_mirror(data: &uapi::drm_asahi_get_params, file: &DrmFile) -> Result { let size = core::mem::size_of::(); @@ -529,9 +523,16 @@ impl File { data: &mut uapi::drm_asahi_vm_create, file: &DrmFile, ) -> Result { - let hal = device.gpu()?.params()?.gpu_hal_generation; - let expected = if hal == 200 { uapi::drm_asahi_vm_create_flags_DRM_ASAHI_VM_CREATE_HAL200 } else { 0 }; - if data.flags != expected { return Err(EINVAL); } + if device.g17().is_some() { + // G17P reports HAL200; it accepts the acknowledgement but does not require it. + if data.flags & !uapi::drm_asahi_vm_create_flags_DRM_ASAHI_VM_CREATE_HAL200 != 0 { + return Err(EINVAL); + } + } else { + let hal = device.gpu()?.params()?.gpu_hal_generation; + let expected = if hal == 200 { uapi::drm_asahi_vm_create_flags_DRM_ASAHI_VM_CREATE_HAL200 } else { 0 }; + if data.flags != expected { return Err(EINVAL); } + } if let Some(gate) = &file.inner().m3_client { gate.admit_vm(device, data.flags)?; } let kernel_range = data.kernel_start..data.kernel_end; let user_range = device.gpu()?.user_range()?; @@ -625,7 +626,6 @@ impl File { ualloc, ualloc_priv, vm, - user_range, kernel_range, user_range, _dummy_mapping: dummy_mapping, @@ -686,6 +686,7 @@ impl File { let vms = file.inner().vms(); let vms = vms.lock(); let vm = &vms.get(data.vm_id.try_into()?).ok_or(ENOENT)?.vm; + // Only G17 VMs enable residency reclaim. reclaim_cpu = vm.residency_reclaim_enabled(); resv_gem = vm.get_resv_obj(); Some(resv_gem.deref()) @@ -734,10 +735,14 @@ impl File { /// IOCTL: gem_madvise: Mark a VM-private object purgeable or needed. pub(crate) fn gem_madvise( - _device: &AsahiDevice, + device: &AsahiDevice, data: &mut uapi::drm_asahi_gem_madvise, file: &DrmFile, ) -> Result { + // Only the G17P runtime implements purgeable objects. + if device.g17().is_none() { + return Err(EINVAL); + } if data.pad != 0 || data.madv > uapi::drm_asahi_gem_madv_DRM_ASAHI_GEM_MADV_DONTNEED { return Err(EINVAL); } @@ -767,7 +772,7 @@ impl File { return Err(EINVAL); } - if device.gpu.as_any().is::() { + if device.g17().is_some() { return Self::vm_bind_g17(data, file); } @@ -818,7 +823,6 @@ impl File { ) }; let mut batch = mmu::PreparedUserBindBatch::new(count)?; - let mut updates = KVec::with_capacity(count, GFP_KERNEL)?; let mut unmaps = 0; // A fresh reader per declared stride also skips extension bytes in longer records. for bytes in bytes.chunks_exact(stride) { @@ -841,7 +845,6 @@ impl File { return Err(EINVAL); } batch.push_unmap(vm.prepare_user_unmap(op.addr, op.range)?)?; - updates.push(MappingUpdate::Unmap(range), GFP_KERNEL)?; unmaps += 1; continue; } @@ -881,31 +884,9 @@ impl File { prot, single_page, )?)?; - updates.push( - MappingUpdate::Map { - object, - address: op.addr, - size: op.range, - offset: op.offset, - }, - GFP_KERNEL, - )?; } vm.reserve_deferred_user_unmaps(unmaps)?; vm.commit_prepared_user_bind_batch(&mut batch)?; - for update in updates { - match update { - MappingUpdate::Map { - object, - address, - size, - offset, - } => { - vm.track_context_binding(object, address, size, offset)?; - } - MappingUpdate::Unmap(range) => vm.untrack_context_range(range), - } - } vm.bo_deferred_cleanup(); Ok(0) } @@ -915,9 +896,6 @@ impl File { data: &uapi::drm_asahi_gem_bind_op, file: &DrmFile, ) -> Result { - if data.range == 0 { - return Err(EINVAL); - } if (data.flags & uapi::drm_asahi_bind_flags_DRM_ASAHI_BIND_UNBIND) != 0 { Self::do_gem_unbind(vm_id, data, file) } else { @@ -1325,7 +1303,7 @@ impl File { let _ = file_vm; core::mem::drop(guard); - let queue = device.gpu()?.new_queue( + let mut queue = device.gpu()?.new_queue( vm, ualloc, ualloc_priv, @@ -1334,6 +1312,7 @@ impl File { data.usc_exec_base, )?; + // Only G17P queues accept a nonzero render ray query scratch layout. queue.reserve_render_scs(data.render_scs_layout)?; data.queue_id = resv.index().try_into()?; @@ -1407,7 +1386,7 @@ impl File { .reader() .read_all(bytes, GFP_KERNEL) }; - let guarded = gpu.as_any().is::(); + let guarded = device.g17().is_some(); // Once G17 output syncobjs are resolved, its queue must install a fence // on every failure. Take the fallible userspace snapshot before that point. if guarded { diff --git a/drivers/gpu/drm/asahi/g16_submit.rs b/drivers/gpu/drm/asahi/g16_submit.rs index 00b35afb9b699a..e5913424398894 100644 --- a/drivers/gpu/drm/asahi/g16_submit.rs +++ b/drivers/gpu/drm/asahi/g16_submit.rs @@ -115,7 +115,7 @@ impl Packet { if let Err(error) = result { let _ = self.failure.compare_exchange(0, error.to_errno(), Ordering::Relaxed, Ordering::Relaxed); - if let Ok(status) = self.vm.status() { status.record(error.to_errno()); } + if let Ok(status) = self.vm.agx_status() { status.record(error.to_errno()); } self.completion.set_error(error); } if result.is_ok() { @@ -134,8 +134,14 @@ impl Packet { } pub(crate) struct Job { shared: Shared, packet: Arc, dev: driver::AsahiDevRef } +/// Timeout state of a [`Job`], owned by the scheduler's timeout worker. +pub(crate) struct JobTimeout { shared: Shared, packet: Arc, dev: driver::AsahiDevRef } impl sched::JobImpl for Job { - fn run(job: &mut sched::Job) -> Result> { + type TimeoutData = JobTimeout; + fn timeout_data(&self) -> JobTimeout { + JobTimeout { shared: self.shared.clone(), packet: self.packet.clone(), dev: self.dev.clone() } + } + fn run(job: &mut Self) -> Result> { let failure = job.packet.failure.load(Ordering::Relaxed); if failure != 0 { let error = Error::from_errno(failure); @@ -153,7 +159,7 @@ impl sched::JobImpl for Job { driver::queue_g16_completion_worker(job.dev.clone()); Ok(Some(Fence::from_fence(&job.packet.completion))) } - fn timed_out(job: &mut sched::Job) -> sched::Status { + fn timed_out(job: &mut JobTimeout, _finished: bool) -> sched::Status { let mut guard = job.shared.lock(); let Some(runtime) = Option::as_mut(&mut *guard) else { job.packet.finish(Err(ENODEV)); @@ -169,7 +175,7 @@ impl sched::JobImpl for Job { runtime.fail_job(job.packet.id, ETIMEDOUT); sched::Status::NoDevice } - fn cancel(job: &mut sched::Job) { + fn cancel(job: &mut Self) { let mut guard = job.shared.lock(); if let Some(runtime) = Option::as_mut(&mut *guard) { runtime.fail_job(job.packet.id, ECANCELED); diff --git a/drivers/gpu/drm/asahi/g17/command.rs b/drivers/gpu/drm/asahi/g17/command.rs index 671e513852c96a..3dfa760969224e 100644 --- a/drivers/gpu/drm/asahi/g17/command.rs +++ b/drivers/gpu/drm/asahi/g17/command.rs @@ -17,6 +17,11 @@ const USC_WINDOW_SIZE: u64 = 1 << 32; const MAX_SAMPLERS: u32 = 1024; const RENDER_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render, prior_cdm) + size_of::(); const COMPUTE_BASE_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, prior_cdm) + size_of::(); +/// The timestamp-copy arrays of the common command layout, which G17P does not implement. +const RENDER_TIMESTAMP_COPIES: Range = core::mem::offset_of!(uapi::drm_asahi_cmd_render, ts_frag_end_copies) + ..core::mem::offset_of!(uapi::drm_asahi_cmd_render, prior_vdm); +const COMPUTE_TIMESTAMP_COPIES: Range = core::mem::offset_of!(uapi::drm_asahi_cmd_compute, ts_end_copies) + ..core::mem::offset_of!(uapi::drm_asahi_cmd_compute, scs_layout); const SAMPLER_SIZE: u64 = 8; const PAGE_SIZE: u64 = 0x4000; const ZLS_METADATA_STRIDE: u64 = 0x80; @@ -130,6 +135,7 @@ impl<'a> Parser<'a> { == uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER { self.counts[0] += 1; + no_timestamp_copies(bytes, RENDER_TIMESTAMP_COPIES)?; Payload::Render( payload(&bytes[..bytes.len().min(RENDER_BASE_SIZE)])?, [self.attachments[0], self.attachments[1]], @@ -137,6 +143,7 @@ impl<'a> Parser<'a> { ) } else { self.counts[1] += 1; + no_timestamp_copies(bytes, COMPUTE_TIMESTAMP_COPIES)?; Payload::Compute( payload(&bytes[..bytes.len().min(COMPUTE_BASE_SIZE)])?, self.attachments[2], @@ -169,6 +176,16 @@ impl<'a> Parser<'a> { } } +/// Reject a command that asks for timestamp copies. +fn no_timestamp_copies(bytes: &[u8], copies: Range) -> Result { + let end = copies.end.min(bytes.len()); + let start = copies.start.min(end); + if bytes[start..end].iter().any(|&b| b != 0) { + return Err(EINVAL); + } + Ok(()) +} + fn payload(bytes: &[u8]) -> Result { if bytes .get(size_of::()..) diff --git a/drivers/gpu/drm/asahi/g17/job.rs b/drivers/gpu/drm/asahi/g17/job.rs index 9293c83c55b613..07db5d32e92c1e 100644 --- a/drivers/gpu/drm/asahi/g17/job.rs +++ b/drivers/gpu/drm/asahi/g17/job.rs @@ -288,6 +288,7 @@ impl Drop for Job { impl sched::JobImpl for Job { type TimeoutData = Timeout; const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + const SYNCHRONIZED_CANCEL: bool = true; fn timeout_data(&self) -> Timeout { Timeout { diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 355cf9c5a83119..0ffa568d6d4b3d 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -44,7 +44,6 @@ mod runtime; pub(crate) mod status; mod teardown; mod timeout; -mod queue_stats; mod validation; use core::any::Any; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs index 6557d8999ba438..187fd27f8e78e6 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute/exit.rs @@ -227,7 +227,6 @@ impl Queue { self.spared_quarantine = false; self.retire_pending = false; self.retirement_ready = false; - self.update_slot_accounting(); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/g17/runtime/backend.rs b/drivers/gpu/drm/asahi/g17/runtime/backend.rs index c36c0add4f87ae..3d5f441d8428f1 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/backend.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/backend.rs @@ -142,12 +142,6 @@ impl job::Backend for Backend { Engine::Render => self.render_packet(&packet, &mut deferred), } }; - if result.as_ref().is_err_and(|error| *error == ENOSPC) { - crate::g17::queue_stats::note_submit_enospc(match packet.engine() { - Engine::Compute => "compute-publish", - Engine::Render => "render-publish", - }); - } self.release_backlog(&packet); self.finish_publication(&packet, result, &mut deferred) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs index a51a42021c4680..2e88848b932161 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render/settle.rs @@ -164,7 +164,6 @@ impl crate::g17::Firmware { return Ok(()); } let mut serviced = false; - let mut draining = 0; for slot in 1..self.queues.render.entries.len() { if !self.queues.render.entries[slot] .as_ref() @@ -173,7 +172,6 @@ impl crate::g17::Firmware { { continue; } - draining += 1; if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { dev_warn!( self.primary.state.shared.dev.as_ref(), @@ -183,7 +181,6 @@ impl crate::g17::Firmware { ); } } - crate::g17::queue_stats::note_draining_pairs(draining); Ok(()) } diff --git a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs index f310f8154aae70..8a637e0b3ed01c 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/teardown.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/teardown.rs @@ -158,9 +158,6 @@ impl Host for Service<'_> { context.id(), error ), - Failure::Stalled => { - dev_warn!(dev, "Scheduler {} release is still pending\n", context.id()) - } } } } @@ -169,7 +166,6 @@ impl Firmware { /// The event worker has already reclaimed drained pairs and reannounced /// pending controls. Recovery blocks new releases, never an observed ack. pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { - crate::g17::queue_stats::note_teardowns(self.queues.teardown.len()); self.queues.render.collect_reclaimable(reclaim)?; if self.queues.teardown.is_empty() { return Ok(()); @@ -198,8 +194,6 @@ impl Firmware { .take_consumed(consumer, producer, &mut service)? .is_some() {} - teardown.report_stalled(super::now_ns(), &mut service); - crate::g17::queue_stats::note_teardowns(teardown.len()); Ok(()) } } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 290df11e428494..58eed45bf43c77 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -117,6 +117,7 @@ pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { Ok(()) } + unsafe impl Send for AsahiObject {} unsafe impl Sync for AsahiObject {} @@ -228,6 +229,10 @@ pub(crate) struct AsahiObjConfig { flags: u32, exportable: bool, kernel: bool, + /// Created by this driver rather than by the DRM core for a dma-buf import or a dumb + /// buffer. Only driver-created objects enter the M3 memory accounting, as before core + /// objects were allocated as asahi objects. + driver_created: bool, } /// Validate before rounding: wrapping a user size can create a much smaller @@ -245,26 +250,12 @@ fn new_kernel_object_with_cpu_mapping( size: usize, map_wc: bool, ) -> Result { -/// Create a new kernel-owned GEM object. -pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { - new_kernel_object_mapped(dev, size, false) -} - -/// Create a new kernel-owned GEM object whose kernel mapping is write-combined. -/// -/// Used for memory that a non-coherent agent reads as soon as the host has stored to it and -/// issued a barrier, without cache maintenance. -pub(crate) fn new_kernel_object_wc(dev: &AsahiDevice, size: usize) -> Result { - new_kernel_object_mapped(dev, size, true) -} - -fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Result { let gem = shmem::Object::::new( dev, checked_object_size(size)?, shmem::ObjectConfig:: { - reclaimable_cpu_mappings: false, map_wc, + reclaimable_cpu_mappings: false, parent_resv_obj: None, synchronous_reclaim: dev.is_m3, }, @@ -272,9 +263,15 @@ fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Res flags: 0, exportable: false, kernel: true, + driver_created: true, }, )?; + // Only the G17P runtime marks its kernel objects held (see `set_mapping_held`). + if dev.hold_kernel_pages { + set_mapping_held(&gem, true); + } + mod_pr_debug!("AsahiObject new kernel object id={}\n", gem.id); Ok(ObjectRef::new(gem)) } @@ -314,6 +311,7 @@ pub(crate) fn new_object( flags, exportable: parent_object.is_none(), kernel: false, + driver_created: true, }, )?; @@ -334,13 +332,13 @@ impl DriverObject for AsahiObject { mod_pr_debug!("AsahiObject::new id={}\n", id); try_pin_init!(AsahiObject { id, + _allocation: if dev.is_m3 && args.driver_created { + Some(crate::agx_memory_stats::Allocation::gem(size, args.kernel)?) + } else { None }, size, flags: args.flags, exportable: args.exportable, kernel: args.kernel, - _allocation: if dev.is_m3 { - Some(crate::agx_memory_stats::Allocation::gem(size, args.kernel)?) - } else { None }, kernel_mappings: AtomicU64::new(0), }) } @@ -352,6 +350,7 @@ impl DriverObject for AsahiObject { flags: 0, exportable: true, kernel: false, + driver_created: false, }) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index 0d7de75042993f..1f87dfa479cab1 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -552,18 +552,21 @@ pub(crate) struct GpuManager { garbage_contexts: Mutex>>>, } -/// Operations the DRM driver performs on a GPU, independent of its firmware interface. +/// Operations the G17P runtime (`g17::Gpu`) provides to the DRM driver. +/// +/// `drm_gpu` adapts it to [`crate::drm_gpu::DrmGpu`]; the G13/G14 and M3 runtimes use that +/// trait directly. pub(crate) trait Gpu: Send + Sync { - /// Return a snapshot only for firmware families with a supported decoder. - fn stats_snapshot(&self) -> Option> { - None - } - + // `g17::Gpu` implements these three; the DRM layer selects that runtime through + // `drm_gpu::Backend::G17` instead of downcasting, so nothing calls them. /// Cast as an Any type. + #[allow(dead_code)] fn as_any(&self) -> &dyn Any; /// Cast Arc as an Any type. + #[allow(dead_code)] fn arc_as_any(self: Arc) -> Arc; /// Return the G13/G14 firmware manager, if this GPU uses that firmware interface. + #[allow(dead_code)] fn manager(&self) -> Option<&dyn GpuManager>; /// Initialize the GPU. fn init(&self) -> Result; @@ -572,13 +575,13 @@ pub(crate) trait Gpu: Send + Sync { /// Check whether the GPU is crashed fn is_crashed(&self) -> bool; /// Fill in the hardware description reported to userspace. - fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result; + fn get_params(&self, params: &mut kernel::uapi::drm_asahi_params_global) -> Result; /// Whether user VMs retain a sticky error and support a userspace mirror. fn supports_vm_status(&self) -> bool { false } /// Limits for independently executing queues, if supported by this GPU. - fn queue_limits(&self) -> Option { + fn queue_limits(&self) -> Option { None } /// Temporary capacity hint and validity in milliseconds for subsequent syncobj waits. @@ -610,14 +613,33 @@ pub(crate) trait Gpu: Send + Sync { /// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. pub(crate) trait GpuManager: Send + Sync { + /// Cast as an Any type. + fn as_any(&self) -> &dyn Any; + /// Cast Arc as an Any type. + fn arc_as_any(self: Arc) -> Arc; + /// Initialize the GPU. + fn init(&self) -> Result; /// Update the GPU globals from global info /// /// TODO: Unclear what can and cannot be updated like this. fn update_globals(&self); /// Get a reference to the KernelAllocators. fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; + /// Create a new `Vm` given a unique `File` ID. + fn new_vm(&self, kernel_range: Range) -> Result; /// Bind a `Vm` to an available slot and return the `VmBind`. fn bind_vm(&self, vm: &mmu::Vm) -> Result; + /// Create a new user command queue. + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result>; + /// Return a reference to the global `SequenceIDs` instance. + fn ids(&self) -> &SequenceIDs; /// Kick the firmware (wake it up if asleep). /// /// This should be useful to reduce latency on work submission, so we can ask the firmware to @@ -651,6 +673,14 @@ pub(crate) trait GpuManager: Send + Sync { fn get_dyncfg(&self) -> &hw::DynConfig; /// Register an unused context as garbage fn free_context(&self, data: KBox>); + /// Check whether the GPU is crashed + fn is_crashed(&self) -> bool; + /// Map a BO as a timestamp buffer + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result; } /// Private generic trait for functions that don't need to escape this module. @@ -1489,14 +1519,7 @@ impl GpuManager::ver { let map_kernel_to_user = false; Ok(KBox::new( - mmu::Uat::new( - dev, - mmu::UatGeometry { - ias: cfg.uat_ias, - oas: cfg.uat_oas, - }, - map_kernel_to_user, - )?, + mmu::Uat::new_for_config(dev, cfg, map_kernel_to_user)?, GFP_KERNEL, )?) } @@ -1570,12 +1593,7 @@ impl GpuManager::ver { )?, fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, - stats: channel::StatsChannel::ver::new( - dev, - alloc_ref, - stats_snap_clone, - u64::from(cfg.base_clock_hz), - )?, + stats: channel::StatsChannel::ver::new(dev, alloc_ref)?, }), GFP_KERNEL, )?; @@ -2943,89 +2961,6 @@ impl GpuManager for GpuManager::ver { Ok(()) } - fn ids(&self) -> &SequenceIDs { - &self.ids - } - - fn is_crashed(&self) -> bool { - self.crashed.load(Ordering::Relaxed) - } - - fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { - params.gpu_generation = self.dyncfg.id.gpu_gen as u32; - params.gpu_variant = self.dyncfg.id.gpu_variant as u32; - params.gpu_revision = self.dyncfg.id.gpu_rev as u32; - params.chip_id = self.cfg.chip_id; - params.num_dies = self.cfg.num_dies; - params.num_clusters_total = self.dyncfg.id.num_clusters; - params.num_cores_per_cluster = self.dyncfg.id.num_cores; - params.max_frequency_khz = self.dyncfg.pwr.max_frequency_khz(); - // G13 and G14 share the second shader ISA generation and predate the numbered command - // stream generations. - params.usc_generation = 2; - params.gpu_hal_generation = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY; - - for (i, mask) in self.dyncfg.id.core_masks.iter().enumerate() { - *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); - } - - Ok(()) - } - - fn base_clock_hz(&self) -> u32 { - self.cfg.base_clock_hz - } - - fn new_vm(&self, kernel_range: Range) -> Result { - self.uat.new_vm(self.ids.vm.next(), kernel_range) - } - - fn new_queue( - &self, - vm: mmu::Vm, - ualloc: Arc>, - ualloc_priv: Arc>, - priority: u32, - usc_exec_base: u64, - ) -> Result> { - let mut kalloc = self.alloc(); - let id = self.ids.queue.next(); - Ok(KBox::new( - queue::Queue::ver::new( - &self.dev, - self, - vm, - &mut kalloc, - ualloc, - ualloc_priv, - self.event_manager.clone(), - &self.buffer_mgr, - id, - priority, - usc_exec_base, - )?, - GFP_KERNEL, - )?) - } - - fn map_timestamp_buffer( - &self, - mut bo: gem::ObjectRef, - range: Range, - ) -> Result { - bo.map_range_into_range( - self.uat.kernel_vm(), - range, - IOVA_KERN_TIMESTAMP_RANGE, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - ) - } -} - -#[versions(AGX)] -impl GpuManager for GpuManager::ver { fn update_globals(&self) { let mut timeout: u32 = 2; if debug_enabled(DebugFlags::WaitForPowerOff) { @@ -3264,10 +3199,41 @@ impl GpuManager for GpuManager::ver { guard } + fn new_vm(&self, kernel_range: Range) -> Result { + self.uat.new_vm(self.ids.vm.next(), kernel_range) + } + fn bind_vm(&self, vm: &mmu::Vm) -> Result { self.uat.bind(vm) } + fn new_queue( + &self, + vm: mmu::Vm, + ualloc: Arc>, + ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + let mut kalloc = self.alloc(); + let id = self.ids.queue.next(); + Ok(KBox::new( + queue::Queue::ver::new( + &self.dev, + vm, + &mut kalloc, + ualloc, + ualloc_priv, + self.event_manager.clone(), + &self.buffer_mgr, + id, + priority, + usc_exec_base, + )?, + GFP_KERNEL, + )?) + } + fn kick_firmware(&self) -> Result { if self.is_crashed() { return Err(ENODEV); @@ -3331,6 +3297,10 @@ impl GpuManager for GpuManager::ver { Ok(()) } + fn ids(&self) -> &SequenceIDs { + &self.ids + } + fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { dev_err!(self.dev.as_ref(), " (\\________/) \n"); dev_err!(self.dev.as_ref(), " | | \n"); diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index 360f4f8844d96b..c90156bd1244bd 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -21,25 +21,6 @@ pub(crate) mod t8112; pub(crate) mod t8122; pub(crate) mod t8140; -/// GPU generation enumeration. Note: Part of the UABI. -#[derive(Debug, PartialEq, Copy, Clone)] -#[repr(u32)] -pub(crate) enum GpuGen { - G13 = 13, - G14 = 14, -} - -/// GPU variant enumeration. Note: Part of the UABI. -#[derive(Debug, PartialEq, Copy, Clone)] -#[repr(u32)] -pub(crate) enum GpuVariant { - P = 'P' as u32, - G = 'G' as u32, - S = 'S' as u32, - C = 'C' as u32, - D = 'D' as u32, -} - /// GPU revision enumeration. Note: Part of the UABI. #[derive(Debug, PartialEq, Copy, Clone)] #[repr(u32)] diff --git a/drivers/gpu/drm/asahi/identity.rs b/drivers/gpu/drm/asahi/identity.rs index eb62a32da07dc8..f468272c5680ed 100644 --- a/drivers/gpu/drm/asahi/identity.rs +++ b/drivers/gpu/drm/asahi/identity.rs @@ -8,6 +8,7 @@ pub(crate) enum GpuGen { G13 = 13, G14 = 14, G15 = 15, + G17 = 17, } /// GPU variant enumeration. Note: Part of the UABI. diff --git a/drivers/gpu/drm/asahi/m3_submit.rs b/drivers/gpu/drm/asahi/m3_submit.rs index f1c6e561b9aeb8..9bd08d0257b0c9 100644 --- a/drivers/gpu/drm/asahi/m3_submit.rs +++ b/drivers/gpu/drm/asahi/m3_submit.rs @@ -84,7 +84,7 @@ impl Packet { // Claim error/signal ownership once; this flag is NOT proof that the // fence is signaled or that hardware DMA has stopped. if self.finish_claimed.swap(true, Ordering::Relaxed) { return; } - if let Err(e)=result {if let Ok(status) = self.vm.status() { status.record(e.to_errno()); }self.completion.set_error(e);} + if let Err(e)=result {if let Ok(status) = self.vm.agx_status() { status.record(e.to_errno()); }self.completion.set_error(e);} // A scheduler job/fence and the runtime's last packet can outlive // hardware retirement. They must not keep the VM's active-job count // nonzero: GEM closes otherwise accumulate across completed frames. @@ -103,8 +103,14 @@ impl Packet { } } pub(crate) struct Job {shared:Shared,packet:Arc} +/// Timeout state of a [`Job`], owned by the scheduler's timeout worker. +pub(crate) struct JobTimeout {shared:Shared,packet:Arc} impl sched::JobImpl for Job { - fn run(job:&mut sched::Job)->Result> { + type TimeoutData = JobTimeout; + fn timeout_data(&self)->JobTimeout { + JobTimeout {shared:self.shared.clone(),packet:self.packet.clone()} + } + fn run(job:&mut Self)->Result> { // Queue the packet and return its fence; the completion worker publishes and retires // it, so the scheduler can hand over further ready jobs meanwhile. if let Err(e)=crate::m3_runtime::Runtime::submit(&job.shared,job.packet.clone()) { @@ -114,7 +120,7 @@ impl sched::JobImpl for Job { } Ok(Some(Fence::from_fence(&job.packet.completion))) } - fn timed_out(job:&mut sched::Job)->sched::Status { + fn timed_out(job:&mut JobTimeout,_finished:bool)->sched::Status { // Retain the publishing branch's explicit diagnostic override. if !crate::m3_params::timeout_nohang() { if let Some(r)=Option::as_mut(&mut *job.shared.lock()) {r.health().mark_failed();} @@ -136,7 +142,7 @@ impl sched::JobImpl for Job { job.packet.finish(Err(EIO)); sched::Status::NoDevice } - fn cancel(job:&mut sched::Job) { + fn cancel(job:&mut Self) { crate::m3_runtime::Runtime::withdraw(&job.shared,&job.packet); job.packet.finish(Err(ECANCELED)); } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 0088fcf90228b8..06acbcef391ce6 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -12,7 +12,9 @@ //! //! G17 firmware does not take part in the handoff protocol. Instead, each of its two coprocessor //! instances publishes the top-level entries it owns into its own kernel root while it boots, and -//! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. +//! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. G17 UATs +//! and their VMs ("published-roots" UATs) run the G17 paths of this module; every other UAT keeps +//! the handoff/M3/J613 paths. use core::fmt::Debug; use core::mem::{size_of, ManuallyDrop}; @@ -73,7 +75,7 @@ use crate::{ mem, pgtable, slotalloc, - uat::UatGeometry, + uat::UatGeometry as RootGeometry, util::RangeExt, // }; @@ -186,6 +188,18 @@ impl UatConfig { }) } + /// The UAT of a published-roots (G17) GPU. No chip-specific owner applies to it. + fn published(geometry: UatGeometry) -> Result { + let ias = u8::try_from(geometry.ias)?; + RootGeometry::new(ias).ok_or(ENODEV)?; + Ok(Self { + chip_id: 0, + ias, + oas: geometry.oas, + reserved_tables: false, + }) + } + /// Whether an M3 owner holds this UAT: the M3 runtime (reserved tables) on any SoC, or the /// G15 manager backend, which runs on T6030 only. These owners look their reserved regions /// up through `m3_resources`, track faults and VM status, and keep the AP side of the @@ -195,10 +209,10 @@ impl UatConfig { } } -fn uat_geometry(cfg: UatConfig) -> Result { +fn uat_geometry(cfg: UatConfig) -> Result { // 39-bit (G13/G14) and 42-bit (G15+) roots are the two recovered // geometries; anything else is undefined. - UatGeometry::new(cfg.ias).ok_or(ENODEV) + RootGeometry::new(cfg.ias).ok_or(ENODEV) } /// Lower/user VA range for this SoC's root geometry (one page up to the @@ -228,7 +242,7 @@ const IOVA_TTBR1_BASE_39: u64 = 0xffffff8000000000; /// Returns the upper/kernel (TTBR1) base VA for this GPU's root geometry. pub(crate) fn iova_ttbr1_base(cfg: &hw::HwConfig) -> u64 { - UatGeometry::new(cfg.uat_ias) + RootGeometry::new(cfg.uat_ias) .map(|g| g.upper_canonical_base()) .unwrap_or(IOVA_TTBR1_BASE_39) } @@ -248,6 +262,13 @@ pub(crate) fn kern_iova_range(cfg: &hw::HwConfig, range: Range) -> Range r, + None => panic!("unsupported root geometry"), + }; + assert!(g.user_top() == r.user_va_top()); + assert!(g.upper_base() == r.upper_canonical_base()); + assert!(g.kernel_range().start == r.kernel_va_base()); + assert!(g.kernel_range().end == r.kernel_va_top()); + i += 1; + } + assert!(UatGeometry { ias: 39, oas: 40 }.unk_page() == IOVA_UNK_PAGE); +}; + /// A pre-allocated memory region for UAT management struct UatRegion { base: PhysicalAddr, @@ -441,6 +479,7 @@ struct VmInner { fixed_slot: Option, va_range: Range, page_table: UatPageTable, + /// Mapping epoch of a published-roots (G17) VM; `None` on every other VM. mapping_epoch: Option>, epoch: Option, mm: mm::Allocator<(), KernelMappingInner>, @@ -507,8 +546,10 @@ impl gpuvm::DriverGpuVm for VmInner { op: &mut gpuvm::OpMap, ctx: &mut Self::StepContext, ) -> Result { + if self.published() { + return Self::step_map_published(self, op, ctx); + } if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } - let _mutation = self.mapping_mutation(); let mut iova = op.addr(); let mut left = op.range() as usize; let mut offset = op.offset() as usize; @@ -525,6 +566,242 @@ impl gpuvm::DriverGpuVm for VmInner { .prepare_map(op.addr()..op.addr().checked_add(op.range()).ok_or(EOVERFLOW)?)?; } + let mut do_map = |mut addr: usize, mut len: usize, offset: &mut usize| -> Result { + if left == 0 { + return Ok(false); + } + + if *offset > 0 { + let skip = len.min(*offset); + addr += skip; + len -= skip; + *offset -= skip; + } + if len == 0 { + return Ok(true); + } + assert!(*offset == 0); + + if one_page { + len = left; + } else { + len = len.min(left); + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x} [OP={}]\n", + addr, + len, + iova, + one_page + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + ctx.prot, + one_page, + )?; + + left -= len; + iova += len as u64; + Ok(true) + }; + + let guard = bo.inner().inner.lock(); + if let Some(sg_vec) = guard.sg_vec.as_ref() { + let start_idx = sg_vec + .binary_search_by(|range| { + if range.0 > offset { + cmp::Ordering::Greater + } else if (range.0 + range.1.len()) <= offset { + cmp::Ordering::Less + } else { + cmp::Ordering::Equal + } + }) + .expect("sg_vec does not contain offset???"); + + offset -= sg_vec[start_idx].0 as usize; + + for cur in start_idx..sg_vec.len() { + let addr = sg_vec[cur].1.start as usize; + let len: usize = sg_vec[cur].1.len() as usize; + if do_map(addr, len, &mut offset)? == false { + break; + } + } + } else { + for range in guard.sgt.as_ref().expect("step_map with no SGT").iter() { + // TODO: proper DMA address/length handling + let addr = range.dma_address() as usize; + let len: usize = range.dma_len() as usize; + if do_map(addr, len, &mut offset)? == false { + break; + } + } + } + + let gpuva = ctx.new_va.take().expect("Multiple step_map calls"); + + if op + .map_and_link_va( + self, + gpuva, + ctx.vm_bo.as_ref().expect("step_map with no BO"), + ) + .is_err() + { + dev_err!( + self.dev.as_ref(), + "map_and_link_va failed: {:#x} [{:#x}] -> {:#x}\n", + op.offset(), + op.range(), + op.addr() + ); + return Err(EINVAL); + } + Ok(()) + } + fn step_unmap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpUnMap, + _ctx: &mut Self::StepContext, + ) -> Result { + if self.published() { + return Self::step_unmap_published(self, op, _ctx); + } + if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } + let va = op.va().expect("step_unmap: missing VA"); + + mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); + + self.page_table + .unmap_pages(va.addr()..(va.addr() + va.range()))?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, va.addr() as usize, va.range() as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + va.addr(), + va.range(), + ); + mem::sync(); + } + + if op.unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + Ok(()) + } + fn step_remap( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpReMap, + vm_bo: &gpuvm::GpuVmBo, + ctx: &mut Self::StepContext, + ) -> Result { + if self.published() { + return Self::step_remap_published(self, op, vm_bo, ctx); + } + if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } + let va = op.unmap().va().expect("No previous VA"); + let orig_addr = va.addr(); + let orig_range = va.range(); + + // Only unmap the hole between prev/next, if they exist + let unmap_start = if let Some(op) = op.prev_map() { + op.addr() + op.range() + } else { + orig_addr + }; + + let unmap_end = if let Some(op) = op.next_map() { + op.addr() + } else { + orig_addr + orig_range + }; + + mod_dev_dbg!( + self.dev, + "MMU: unmap for remap: {:#x}..{:#x} (from {:#x}:{:#x})\n", + unmap_start, + unmap_end, + orig_addr, + orig_range + ); + + let unmap_range = unmap_end - unmap_start; + + self.page_table.unmap_pages(unmap_start..unmap_end)?; + + if let Some(asid) = self.slot() { + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, unmap_start as usize, unmap_range as usize); + mod_dev_dbg!( + self.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + unmap_start, + unmap_range, + ); + mem::sync(); + } + + if op.unmap().unmap_and_unlink_va_defer().is_none() { + dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); + } + + if let Some(prev_op) = op.prev_map() { + let prev_gpuva = ctx + .prev_va + .take() + .expect("Multiple step_remap calls with prev_op"); + if prev_op.map_and_link_va(self, prev_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink prev gpuva"); + return Err(EINVAL); + } + } + + if let Some(next_op) = op.next_map() { + let next_gpuva = ctx + .next_va + .take() + .expect("Multiple step_remap calls with next_op"); + if next_op.map_and_link_va(self, next_gpuva, vm_bo).is_err() { + dev_err!(self.dev.as_ref(), "step_remap: could not relink next gpuva"); + return Err(EINVAL); + } + } + + Ok(()) + } +} + +impl VmInner { + /// Whether this VM belongs to a published-roots (G17) UAT. + fn published(&self) -> bool { + self.uat_inner.firmware != UatFirmware::Handoff + } + + /// `DriverGpuVm::step_map` of a published-roots (G17) VM. + fn step_map_published( + self: &mut gpuvm::UpdatingGpuVm<'_, Self>, + op: &mut gpuvm::OpMap, + ctx: &mut StepContext, + ) -> Result { + let _mutation = self.mapping_mutation(); + let mut iova = op.addr(); + let mut left = op.range() as usize; + let mut offset = op.offset() as usize; + + let bo = ctx.vm_bo.as_ref().expect("step_map with no BO"); + + let one_page = op.flags().contains(gpuvm::GpuVaFlags::REPEAT); + let mut do_map = |mut addr: usize, mut len: usize, offset: &mut usize| -> Result { if left == 0 { return Ok(false); @@ -629,12 +906,12 @@ impl gpuvm::DriverGpuVm for VmInner { self.tlbi_contexts_naming_root(op.addr(), op.range() as usize); Ok(()) } - fn step_unmap( + /// `DriverGpuVm::step_unmap` of a published-roots (G17) VM. + fn step_unmap_published( self: &mut gpuvm::UpdatingGpuVm<'_, Self>, op: &mut gpuvm::OpUnMap, - _ctx: &mut Self::StepContext, + _ctx: &mut StepContext, ) -> Result { - if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } let va = op.va().expect("step_unmap: missing VA"); self.unmap_gpuva(va)?; @@ -644,13 +921,13 @@ impl gpuvm::DriverGpuVm for VmInner { } Ok(()) } - fn step_remap( + /// `DriverGpuVm::step_remap` of a published-roots (G17) VM. + fn step_remap_published( self: &mut gpuvm::UpdatingGpuVm<'_, Self>, op: &mut gpuvm::OpReMap, vm_bo: &gpuvm::GpuVmBo, - ctx: &mut Self::StepContext, + ctx: &mut StepContext, ) -> Result { - if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } let va = op.unmap().va().expect("No previous VA"); let orig_addr = va.addr(); let orig_range = va.range(); @@ -733,9 +1010,7 @@ impl gpuvm::DriverGpuVm for VmInner { Ok(()) } -} -impl VmInner { /// Remove hardware leaves for one complete user GPUVA without allocating. fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); @@ -801,17 +1076,140 @@ impl VmInner { MappingMutation(state) } - /// Returns the slot index, if this VM is bound. - fn slot(&self) -> Option { - if let Some(slot) = self.fixed_slot { return Some(slot); } - if self.is_kernel { - // The GFX ASC does not care about the ASID. Pick an arbitrary one. - // TODO: This needs to be a persistently reserved ASID once we integrate - // with the ARM64 kernel ASID machinery to avoid overlap. - Some(0) - } else { - // We don't check whether we lost the slot, which could cause unnecessary - // invalidations against another Vm. However, this situation should be very + /// Invalidate a VA range of this Vm's address space in the TLB of one ASID. + fn tlbi_range(&self, asid: u8, iova: u64, size: usize) { + match self.uat_inner.firmware { + UatFirmware::Handoff => mem::tlbi_range(asid, iova as usize, size), + UatFirmware::PublishedRoots { .. } => { + mem::tlbi_range_or_asid(asid, iova as usize, size) + } + } + } + + /// Invalidate a VA range in every TTBAT context that names this Vm's root as its lower half. + /// + /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root + /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping + /// change must reach all of them. Other firmware only uses the bind slot, which the callers + /// already invalidate. + fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { + if self.uat_inner.firmware == UatFirmware::Handoff { + mem::sync(); + return; + } + + let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); + self.tlbi_context_mask(iova, size, contexts); + } + + fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { + // Complete new leaves even if no context currently names this root: + // a later context can publish it without repeating the map operation. + // DMB orders accesses but does not complete stores before TLBI; the + // GPU uses outer-shareable invalidation and the barrier covers it. + mem::sync(); + if contexts == 0 { + return; + } + for ctx in 0..UAT_NUM_CTX { + if contexts & (1 << ctx) != 0 { + mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); + } + } + mem::sync(); + } + + /// `map_node` of a published-roots (G17) VM. + fn map_node_published(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let end = node + .start() + .checked_add(node.mapped_size as u64) + .ok_or(EOVERFLOW)?; + let _mutation = self.mapping_mutation(); + self.page_table.prepare_map(node.start()..end)?; + let result = self.map_node_prepared(node, prot); + if result.is_err() { + self.rollback_failed_map(node.start(), node.mapped_size as u64); + } + result + } + + /// Install only into the empty range reserved by `map_node`. + fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { + let mut iova = node.start(); + let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); + let sgt = guard.sgt.as_ref().ok_or(EINVAL)?; + let mut offset = node.offset; + let mut left = node.mapped_size; + + for range in sgt.iter() { + if left == 0 { + break; + } + + // TODO: proper DMA address/length handling + let mut addr = range.dma_address() as usize; + let mut len: usize = range.dma_len() as usize; + + if (offset | addr | len | iova as usize) & UAT_PGMSK != 0 { + dev_err!( + self.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + addr, + len, + iova + ); + return Err(EINVAL); + } + + if offset > 0 { + let skip = len.min(offset); + addr += skip; + len -= skip; + offset -= skip; + } + + len = len.min(left); + + if len == 0 { + continue; + } + + mod_dev_dbg!( + self.dev, + "MMU: map: {:#x}:{:#x} -> {:#x}\n", + addr, + len, + iova + ); + + self.page_table.map_pages( + iova..(iova + len as u64), + addr as PhysicalAddr, + prot, + false, + )?; + + iova += len as u64; + left -= len; + } + + // A context naming this root may be live and hold a negative translation for the range. + self.tlbi_contexts_naming_root(node.start(), node.mapped_size); + Ok(()) + } + + /// Returns the slot index, if this VM is bound. + fn slot(&self) -> Option { + if let Some(slot) = self.fixed_slot { return Some(slot); } + if self.is_kernel { + // The GFX ASC does not care about the ASID. Pick an arbitrary one. + // TODO: This needs to be a persistently reserved ASID once we integrate + // with the ARM64 kernel ASID machinery to avoid overlap. + Some(0) + } else { + // We don't check whether we lost the slot, which could cause unnecessary + // invalidations against another Vm. However, this situation should be very // rare (e.g. a Vm lost its slot, which means 63 other Vms bound in the // interim, and then it gets killed / drops its mappings without doing any // final rendering). Anything doing active maps/unmaps is probably also @@ -904,75 +1302,21 @@ impl VmInner { done += chunk; } Ok(()) - /// Invalidate a VA range of this Vm's address space in the TLB of one ASID. - fn tlbi_range(&self, asid: u8, iova: u64, size: usize) { - match self.uat_inner.firmware { - UatFirmware::Handoff => mem::tlbi_range(asid, iova as usize, size), - UatFirmware::PublishedRoots { .. } => { - mem::tlbi_range_or_asid(asid, iova as usize, size) - } - } - } - - /// Invalidate a VA range in every TTBAT context that names this Vm's root as its lower half. - /// - /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root - /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. - fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { - mem::sync(); - return; - } - - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); - self.tlbi_context_mask(iova, size, contexts); - } - - fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { - // Complete new leaves even if no context currently names this root: - // a later context can publish it without repeating the map operation. - // DMB orders accesses but does not complete stores before TLBI; the - // GPU uses outer-shareable invalidation and the barrier covers it. - mem::sync(); - if contexts == 0 { - return; - } - for ctx in 0..UAT_NUM_CTX { - if contexts & (1 << ctx) != 0 { - mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); - } - } - mem::sync(); } /// Map an `mm::Node` representing an mapping in VA space. fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { - let end = node - .start() - .checked_add(node.mapped_size as u64) - .ok_or(EOVERFLOW)?; - let _mutation = self.mapping_mutation(); - self.page_table.prepare_map(node.start()..end)?; - let result = self.map_node_prepared(node, prot); - if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); + if self.published() { + return self.map_node_published(node, prot); } - result - } - - /// Install only into the empty range reserved by `map_node`. - fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { let mut iova = node.start(); - let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); let mapping_end = if self.uat_inner.fault.is_some() { iova.checked_add(node.mapped_size as u64).ok_or(EOVERFLOW)? } else { iova + node.mapped_size as u64 }; // The private fixed-mapping allocator cannot see GPUVM-owned leaves. // Reserve the complete page-table walk before installing any SG run. if self.uat_inner.fault.is_some() { self.page_table.prepare_map(iova..mapping_end)?; } - let guard = node.bo.as_ref().ok_or(EINVAL)?.inner().inner.lock(); + let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); let sgt = guard.sgt.as_ref().ok_or(EINVAL)?; let mut offset = node.offset; let mut left = node.mapped_size; @@ -1028,9 +1372,6 @@ impl VmInner { iova += len as u64; left -= len; } - - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); Ok(()) } } @@ -1040,18 +1381,19 @@ impl VmInner { pub(crate) struct Vm { fault: Option>, id: u64, + inner: ARef>, + dummy_obj: ARef, + binding: Arc>, + driver_mappings: Option>>>, + agx_status: Option>, + job_lifetime: Option>>, + // Published-roots (G17) VM state; `None` on every other VM. mapping_epoch: Option>, lifetime: Option>, residency: Option>, context_bindings: Option>, shared_bindings: Option>, status: Option>, - inner: ARef>, - dummy_obj: ARef, - binding: Arc>, - driver_mappings: Option>>>, - status: Option>, - job_lifetime: Option>>, } impl Drop for Vm { fn drop(&mut self) { @@ -1118,48 +1460,120 @@ where /// Allocation generation of a [`Vm`] hardware slot. pub(crate) struct SlotInner { generation: u8 } -/// An application TTBAT identity retained by a logical queue and its jobs. -/// The root lease is dropped before the VM, so no page table can be freed -/// while an active context still names it. Installed firmware queues retain -/// this object through replacement retirement or processor stop; a consumed -/// closed-scheduler release may return its root identity while keeping storage. -pub(crate) struct ExecutionContext { - root: ExecutionRoot, - vm: Vm, -} -struct ExecutionRoot { - released: AtomicBool, - inner: Arc, - id: u8, - generation: u8, - low: u64, - high: u64, +impl slotalloc::SlotItem for SlotInner { + type Data = (); } -impl ExecutionContext { - pub(crate) fn id(&self) -> u32 { - self.root.id.into() - } - pub(crate) fn generation(&self) -> u8 { - self.root.generation +/// Represents a single user of a binding of a [`Vm`] to a slot. +/// +/// The number of users is counted, and the slot will be freed when it drops to 0. +#[derive(Debug)] +pub(crate) struct VmBind(Vm, u32, u8); + +impl VmBind { + /// Returns the slot that this `Vm` is bound to. + pub(crate) fn slot(&self) -> u32 { + self.1 } + + /// Allocation generation retained with this ASID. + pub(crate) fn generation(&self) -> u8 { self.2 } + + /// The bound `Vm` itself, for diagnostics that need to walk its tables. pub(crate) fn vm(&self) -> &Vm { - &self.vm + &self.0 } - /// Called only after the closed scheduler's release was consumed and all - /// its commands retired. Installed storage may outlive this root lease. - pub(crate) fn release(&self) { - self.root.release(); + /// Whether this retained binding belongs to the supplied DRM VM. + pub(crate) fn matches(&self, vm: &Vm) -> bool { + core::ptr::eq(&*self.0.inner, &*vm.inner) } - pub(crate) fn is_current(&self) -> bool { - let inner = self.root.inner.lock(); - if self.root.released.load(Ordering::Acquire) { - return false; - } - let roots = &inner.ttbs()[usize::from(self.root.id)]; + /// Physical base of the bound VM's page-table root. + pub(crate) fn root(&self) -> u64 { + self.0.ttb() + } + + /// The bound VM's driver-assigned id. + pub(crate) fn vm_id(&self) -> u64 { + self.0.id + } +} + +impl Drop for VmBind { + fn drop(&mut self) { + let mut binding = self.0.binding.lock(); + + assert_ne!(binding.active_users, 0); + binding.active_users -= 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + if binding.active_users == 0 { + binding.binding = None; + } + } +} + +impl Clone for VmBind { + fn clone(&self) -> VmBind { + let mut binding = self.0.binding.lock(); + + binding.active_users += 1; + mod_pr_debug!( + "MMU: slot {} active users {}\n", + self.1, + binding.active_users + ); + VmBind(self.0.clone(), self.1, self.2) + } +} + +/// An application TTBAT identity retained by a logical queue and its jobs. +/// The root lease is dropped before the VM, so no page table can be freed +/// while an active context still names it. Installed firmware queues retain +/// this object through replacement retirement or processor stop; a consumed +/// closed-scheduler release may return its root identity while keeping storage. +pub(crate) struct ExecutionContext { + root: ExecutionRoot, + vm: Vm, +} + +struct ExecutionRoot { + released: AtomicBool, + inner: Arc, + id: u8, + generation: u8, + low: u64, + high: u64, +} + +impl ExecutionContext { + pub(crate) fn id(&self) -> u32 { + self.root.id.into() + } + pub(crate) fn generation(&self) -> u8 { + self.root.generation + } + pub(crate) fn vm(&self) -> &Vm { + &self.vm + } + + /// Called only after the closed scheduler's release was consumed and all + /// its commands retired. Installed storage may outlive this root lease. + pub(crate) fn release(&self) { + self.root.release(); + } + + pub(crate) fn is_current(&self) -> bool { + let inner = self.root.inner.lock(); + if self.root.released.load(Ordering::Acquire) { + return false; + } + let roots = &inner.ttbs()[usize::from(self.root.id)]; roots.ttb0.load(Ordering::Acquire) == self.root.low && roots.ttb1.load(Ordering::Acquire) == self.root.high } @@ -1173,7 +1587,7 @@ impl ExecutionRoot { let id = usize::from(self.id); let release = { let inner = self.inner.lock(); - inner.handoff().lock(); + let handoff = inner.handoff().lock_legacy(); let roots = &inner.ttbs()[id]; let low = roots.ttb0.load(Ordering::Acquire); let high = roots.ttb1.load(Ordering::Acquire); @@ -1182,7 +1596,7 @@ impl ExecutionRoot { roots.ttb0.store(0, Ordering::Release); roots.ttb1.store(0, Ordering::Release); } - inner.handoff().unlock(); + drop(handoff); owned || (low == 0 && high == 0) }; if release { @@ -1205,85 +1619,12 @@ impl Drop for ExecutionRoot { } } -/// Slot data for a [`Vm`] slot (nothing, we only care about the indices). -pub(crate) struct SlotInner(); - -impl slotalloc::SlotItem for SlotInner { - type Data = (); -} - -/// Represents a single user of a binding of a [`Vm`] to a slot. -/// -/// The number of users is counted, and the slot will be freed when it drops to 0. -#[derive(Debug)] -pub(crate) struct VmBind(Vm, u32, u8); - -impl VmBind { - /// Returns the slot that this `Vm` is bound to. - pub(crate) fn slot(&self) -> u32 { - self.1 - } - - /// Allocation generation retained with this ASID. - pub(crate) fn generation(&self) -> u8 { self.2 } - - /// The bound `Vm` itself, for diagnostics that need to walk its tables. - pub(crate) fn vm(&self) -> &Vm { - &self.0 - } - - /// Whether this retained binding belongs to the supplied DRM VM. - pub(crate) fn matches(&self, vm: &Vm) -> bool { - core::ptr::eq(&*self.0.inner, &*vm.inner) - } - - /// Physical base of the bound VM's page-table root. - pub(crate) fn root(&self) -> u64 { - self.0.ttb() - } - - /// The bound VM's driver-assigned id. - pub(crate) fn vm_id(&self) -> u64 { - self.0.id - } -} - -impl Drop for VmBind { - fn drop(&mut self) { - let mut binding = self.0.binding.lock(); - - assert_ne!(binding.active_users, 0); - binding.active_users -= 1; - mod_pr_debug!( - "MMU: slot {} active users {}\n", - self.1, - binding.active_users - ); - if binding.active_users == 0 { - binding.binding = None; - } - } -} - -impl Clone for VmBind { - fn clone(&self) -> VmBind { - let mut binding = self.0.binding.lock(); - - binding.active_users += 1; - mod_pr_debug!( - "MMU: slot {} active users {}\n", - self.1, - binding.active_users - ); - VmBind(self.0.clone(), self.1, self.2) - } -} - -/// A mapping BO reference released under the immediate-mode GPUVA lock. -/// Retain the owner until deferred cleanup completes, including its final BO. +/// A mapping BO reference. On published-roots (G17) VMs it is released under the +/// immediate-mode GPUVA lock, retaining the owner until deferred cleanup completes, including +/// its final BO. Other VMs drop the reference synchronously. struct MappingBo { bo: Option>>, - owner: ARef>, + owner: Option>>, } impl MappingBo { fn new( @@ -1292,15 +1633,22 @@ impl MappingBo { ) -> Self { Self { bo: Some(bo), - owner, + owner: Some(owner), } } + fn synchronous(bo: ARef>) -> Self { + Self { bo: Some(bo), owner: None } + } fn get(&self) -> Result<&gpuvm::GpuVmBo> { self.bo.as_deref().ok_or(EINVAL) } } impl Drop for MappingBo { fn drop(&mut self) { + let Some(owner) = self.owner.as_ref() else { + // The `bo` field drops normally. + return; + }; let Some(bo) = self.bo.take() else { return; }; @@ -1313,7 +1661,23 @@ impl Drop for MappingBo { raw.as_ptr().cast::(), ) }; - self.owner.bo_deferred_cleanup(); + owner.bo_deferred_cleanup(); + } +} + +/// The GEM object of a mapping: published-roots (G17) mappings pin it against idle reclaim. +enum MappingObject { + Shared(ARef), + Pinned(gem::KernelMappingPin), +} + +impl core::ops::Deref for MappingObject { + type Target = gem::Object; + fn deref(&self) -> &gem::Object { + match self { + Self::Shared(object) => object, + Self::Pinned(pin) => pin, + } } } @@ -1321,10 +1685,11 @@ impl Drop for MappingBo { pub(crate) struct KernelMappingInner { // Drop order matters: // - Drop the GpuVmBo first, which locks its BO GPUVA list and drops a GpuVm reference + // (published-roots VMs: deferred-put it under its immediate-mode GPUVA lock) // - Drop the GEM BO next, since BO free can take the resv lock itself // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 bo: Option, - _gem: Option, + _gem: Option, owner: ARef>, uat_inner: Arc, prot: Prot, @@ -1466,10 +1831,53 @@ impl KernelMapping { (result, mutated) } - /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the - /// coprocessor cache. This is required to safely unmap cached/private mappings. - fn remap_uncached_and_flush(&mut self) { + /// `Drop` of a published-roots (G17) mapping. That firmware does not use the handoff + /// protocol and cannot be asked to flush its cache; it only caches kernel mappings that live + /// until its coprocessors stop. + fn drop_published(&mut self) { + // Only VM page tables/binding state are touched here. The shared + // reservation is sufficient and avoids fallible exec allocation in Drop. let mut owner = self.0.owner.lock_inner(); + mod_dev_dbg!( + owner.dev, + "MMU: unmap {:#x}:{:#x}\n", + self.iova(), + self.size() + ); + + let mutation = owner.mapping_mutation(); + if owner.page_table.unmap_pages(self.iova_range()).is_err() { + dev_err!( + owner.dev.as_ref(), + "MMU: unmap {:#x}:{:#x} failed\n", + self.iova(), + self.size() + ); + } + + if let Some(asid) = owner.slot() { + mem::sync(); + owner.tlbi_range(asid as u8, self.iova(), self.size()); + mod_dev_dbg!( + owner.dev, + "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", + asid, + self.iova(), + self.size() + ); + mem::sync(); + } + + // The range returns to the VA allocator once this mapping is gone, so no context that + // names this root may keep a stale translation for it. + owner.tlbi_contexts_naming_root(self.iova(), self.size()); + drop(mutation); + drop(owner); + // SAFETY: The PTEs are gone and invalidated, and the VM reservation is released before + // the node drops its BO references. + unsafe { ManuallyDrop::drop(&mut self.0) }; + } + fn remap_uncached_and_flush_legacy(&mut self) { let mut owner = self .0 @@ -1499,7 +1907,7 @@ impl KernelMapping { self.size() ); } - mem::sync(); + fence(Ordering::SeqCst); // If we don't have (and have never had) a VM slot, just return let slot = match owner.slot() { @@ -1626,9 +2034,11 @@ impl KernelMapping { self.remap_uncached_and_flush_legacy(); } - // Only VM page tables/binding state are touched here. The shared - // reservation is sufficient and avoids fallible exec allocation in Drop. - let mut owner = self.0.owner.lock_inner(); + let mut owner = self + .0 + .owner + .exec_lock(None, false) + .expect("exec_lock failed in KernelMapping::drop"); mod_dev_dbg!( owner.dev, "MMU: unmap {:#x}:{:#x}\n", @@ -1646,8 +2056,8 @@ impl KernelMapping { } if let Some(asid) = owner.slot() { - mem::sync(); - owner.tlbi_range(asid as u8, self.iova(), self.size()); + fence(Ordering::SeqCst); + mem::tlbi_range(asid as u8, self.iova() as usize, self.size()); mod_dev_dbg!( owner.dev, "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", @@ -1808,6 +2218,10 @@ no_debug!(KernelMapping); impl Drop for KernelMapping { fn drop(&mut self) { + if self.0.uat_inner.firmware != UatFirmware::Handoff { + self.drop_published(); + return; + } if self.0.uat_inner.fault.is_none() { self.drop_legacy(); return; @@ -1888,10 +2302,6 @@ impl Drop for KernelMapping { // SAFETY: The PTEs are gone, required flushes completed, and the VM // execution lock is released before the node drops BO references. unsafe { ManuallyDrop::drop(&mut self.0) }; - - // The range returns to the VA allocator once this mapping is gone, so no context that - // names this root may keep a stale translation for it. - owner.tlbi_contexts_naming_root(self.iova(), self.size()); } } @@ -1899,6 +2309,7 @@ impl Drop for KernelMapping { struct UatShared { fault: Option>, kernel_ttb1: u64, + /// Execution contexts leased to logical queues (published-roots UATs). execution_contexts: u64, context_generations: [u8; UAT_NUM_CTX], /// Kernel root of the secondary firmware instance, if there is one. @@ -1951,10 +2362,10 @@ unsafe impl Send for UatShared {} /// Inner data for the top-level UAT instance. #[pin_data] struct UatInner { + firmware: UatFirmware, m3_running: bool, fault: Option>, firmware_cache_flush_ready: AtomicBool, - firmware: UatFirmware, #[pin] shared: Mutex, #[pin] @@ -2061,7 +2472,7 @@ impl Handoff { /// In [`HandoffMode::FirmwareDekker`] this blocks (bounded) until the /// firmware publishes `magic_fw`. The stopped M3 owner initializes /// its AP state before firmware starts. - fn init(&self, mode: HandoffMode, m3: bool) -> Result { + fn init(&self, mode: HandoffMode, m3: bool, firmware: UatFirmware) -> Result { self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); self.cur_slot.store(if m3 && mode != HandoffMode::FirmwareDekker { u32::MAX } else { 0 }, Ordering::Relaxed); self.unk3.store(0, Ordering::Relaxed); @@ -2077,24 +2488,10 @@ impl Handoff { } fence(Ordering::SeqCst); - if firmware == UatFirmware::Handoff { - self.wait_for_firmware()?; - } - - for i in 0..=UAT_NUM_CTX { - self.flush[i].state.store(0, Ordering::Relaxed); - self.flush[i].addr.store(0, Ordering::Relaxed); - self.flush[i].size.store(0, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - Ok(()) - } - - /// Wait for the firmware to publish its half of the handoff region. - fn wait_for_firmware(&self) -> Result { - let start = Instant::::now(); - const TIMEOUT: Delta = Delta::from_millis(1000); - if matches!(mode, HandoffMode::FirmwareDekker | HandoffMode::FirmwareM3) { + // Published-roots (G17) firmware never touches the handoff region. + if matches!(mode, HandoffMode::FirmwareDekker | HandoffMode::FirmwareM3) + && firmware == UatFirmware::Handoff + { let start = Instant::::now(); const TIMEOUT: Delta = Delta::from_millis(1000); @@ -2231,24 +2628,22 @@ impl Vm { pub(crate) fn same(&self, other: &Vm) -> bool { self.id == other.id } /// The DRM VM's id. pub(crate) fn id(&self) -> u64 { self.id } - pub(crate) fn status(&self) -> Result<&Arc> { self.status.as_ref().ok_or(EINVAL) } + pub(crate) fn agx_status(&self) -> Result<&Arc> { self.agx_status.as_ref().ok_or(EINVAL) } pub(crate) fn is_m3(&self) -> bool { self.fault.is_some() } + /// Whether this VM belongs to a published-roots (G17) UAT. + fn published(&self) -> bool { self.mapping_epoch.is_some() } - /// Create a new virtual memory address space - fn new( + /// Create a new virtual memory address space of a published-roots (G17) UAT. + fn new_published( dev: &driver::AsahiDevice, uat_inner: Arc, kernel_range: Range, geometry: UatGeometry, - cfg: UatConfig, ttb: Option, id: u64, ) -> Result { - if uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } - let fault = uat_inner.fault.clone(); let dummy_obj = gem::new_kernel_object(dev, UAT_PGSZ)?; let is_kernel = ttb.is_some(); - let iova_kern_range = iova_kern_range(cfg)?; let page_table = if let Some(ttb) = ttb { UatPageTable::new_with_ttb( @@ -2258,53 +2653,18 @@ impl Vm { geometry.oas, uat_inner.firmware != UatFirmware::Handoff, )? - if cfg.reserved_tables { - let node = dev.as_ref().of_node().ok_or(ENODEV)?; - let mut resources = KVec::new(); - for name in [c_str!("pagetables"), c_str!("shared-l2")] { - resources.push(crate::m3_resources::reserved_resource(&node, name)?, GFP_KERNEL)?; - } - // SAFETY: the owner retains these admitted reservations until ASC stops. - // Running firmware roots are preserved until host InitData is published. - let wc = cfg.reserved_tables - && [c_str!("pagetables"), c_str!("shared-l2")].iter().all(|name| crate::m3_board::region_is_nomap(&node, name)); - let tables = unsafe { if wc {crate::pgtable_memory::ReservedTables::new_wc(resources)} else {crate::pgtable_memory::ReservedTables::new(resources)} }?; - if cfg.reserved_tables && uat_inner.m3_running { - // RTKit has switched its private mapping regime. Preserve - // firmware root entries 0/1; the host window is unused - // until this owner publishes initdata. - unsafe { UatPageTable::new_with_m3_live_ttb(ttb,iova_kern_range.clone(),tables,dev) }? - } else { - let mut table = unsafe { UatPageTable::new_with_reserved_ttb(ttb, iova_kern_range.clone(), cfg.ias, cfg.oas, tables) }?; - let shared = crate::m3_resources::reserved_resource(&node, c_str!("shared-l2"))?; - table.install_reserved_middle(iova_kern_range.start, shared.start())?; - table - } - } else { - UatPageTable::new_with_ttb(ttb, iova_kern_range.clone(), cfg.ias, cfg.oas)? - } - } else if cfg.reserved_tables { - UatPageTable::new_m3_coherent(cfg.ias,cfg.oas,dev)? - } else { UatPageTable::new(geometry.ias, geometry.oas)? }; - let lower_start = IOVA_USER_BASE; - let driver_start = lower_start; let (va_range, gpuvm_range) = if is_kernel { (geometry.kernel_range(), kernel_range.clone()) - (iova_kern_range, kernel_range.clone()) } else { let mut range = geometry.user_range(); if uat_inner.firmware != UatFirmware::Handoff { range.start = 0; } (range, geometry.user_usable_range()) - ( - driver_start..uat_geometry(cfg)?.user_va_top(), - lower_start..(uat_geometry(cfg)?.user_va_top() - 2 * UAT_PGSZ as u64), - ) }; let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; @@ -2329,7 +2689,10 @@ impl Vm { }; let binding_clone = binding.clone(); Ok(Vm { - fault, + fault: uat_inner.fault.clone(), + driver_mappings: None, + agx_status: None, + job_lifetime: None, id, mapping_epoch: mapping_epoch.clone(), lifetime: None, @@ -2338,7 +2701,7 @@ impl Vm { shared_bindings: None, status: None, dummy_obj: dummy_obj.gem.clone(), - inner: gpuvm::GpuVm::new( + inner: gpuvm::GpuVm::new_deferred_bo_put( c_str!("Asahi::GpuVm"), // TODO: should we using DRM_GPUVM_RESV_PROTECTED as well? drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, @@ -2361,18 +2724,6 @@ impl Vm { }), )?, binding, - driver_mappings: if cfg.m3_owner() { - Some(Arc::pin_init(new_mutex!(None, "VmDriverMappings"), GFP_KERNEL)?) - } else { None }, - status: if cfg.m3_owner() { - Some(Arc::new(crate::agx_status::VmStatus::new(), GFP_KERNEL)?) - } else { None }, - job_lifetime: if cfg.m3_owner() { - Some(Arc::pin_init(new_mutex!(M3VmJobLifetime { - active: 0, closed: false, close_ranges: None, closed_objects: KVec::new(), - }, "M3VmJobLifetime"), GFP_KERNEL)?) - } else { None }, - }) } @@ -2423,13 +2774,6 @@ impl Vm { .unwrap_or(false) } - pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let Ok(mut inner) = self.inner.exec_lock(None, false) else { - return false; - }; - Self::covers_range_locked(&mut inner, address, size, read, write) - } - /// Checks all writable sparse blocks and their guard coverage under one /// GPUVM lock. A fully mapped guard is not a valid sparse allocation. pub(crate) fn covers_sparse_blocks_with_guards( @@ -2473,6 +2817,129 @@ impl Vm { self.mapping_epoch.as_ref()?.snapshot() } + /// Create a new virtual memory address space + fn new( + dev: &driver::AsahiDevice, + uat_inner: Arc, + kernel_range: Range, + cfg: UatConfig, + ttb: Option, + id: u64, + ) -> Result { + if uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { return Err(EIO); } + let fault = uat_inner.fault.clone(); + let dummy_obj = gem::new_kernel_object(dev, UAT_PGSZ)?; + let is_kernel = ttb.is_some(); + let iova_kern_range = iova_kern_range(cfg)?; + + let page_table = if let Some(ttb) = ttb { + if cfg.reserved_tables { + let node = dev.as_ref().of_node().ok_or(ENODEV)?; + let mut resources = KVec::new(); + for name in [c_str!("pagetables"), c_str!("shared-l2")] { + resources.push(crate::m3_resources::reserved_resource(&node, name)?, GFP_KERNEL)?; + } + // SAFETY: the owner retains these admitted reservations until ASC stops. + // Running firmware roots are preserved until host InitData is published. + let wc = cfg.reserved_tables + && [c_str!("pagetables"), c_str!("shared-l2")].iter().all(|name| crate::m3_board::region_is_nomap(&node, name)); + let tables = unsafe { if wc {crate::pgtable_memory::ReservedTables::new_wc(resources)} else {crate::pgtable_memory::ReservedTables::new(resources)} }?; + if cfg.reserved_tables && uat_inner.m3_running { + // RTKit has switched its private mapping regime. Preserve + // firmware root entries 0/1; the host window is unused + // until this owner publishes initdata. + unsafe { UatPageTable::new_with_m3_live_ttb(ttb,iova_kern_range.clone(),tables,dev) }? + } else { + let mut table = unsafe { UatPageTable::new_with_reserved_ttb(ttb, iova_kern_range.clone(), cfg.ias, cfg.oas, tables) }?; + let shared = crate::m3_resources::reserved_resource(&node, c_str!("shared-l2"))?; + table.install_reserved_middle(iova_kern_range.start, shared.start())?; + table + } + } else { + UatPageTable::new_with_ttb_by_geometry(ttb, iova_kern_range.clone(), cfg.ias, cfg.oas)? + } + } else if cfg.reserved_tables { + UatPageTable::new_m3_coherent(cfg.ias,cfg.oas,dev)? + + } else { + UatPageTable::new_by_geometry(cfg.ias, cfg.oas)? + }; + + let lower_start = IOVA_USER_BASE; + let driver_start = lower_start; + let (va_range, gpuvm_range) = if is_kernel { + (iova_kern_range, kernel_range.clone()) + } else { + ( + driver_start..uat_geometry(cfg)?.user_va_top(), + lower_start..(uat_geometry(cfg)?.user_va_top() - 2 * UAT_PGSZ as u64), + ) + }; + + let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; + + let binding = Arc::pin_init( + new_mutex!( + VmBinding { + binding: None, + bind_token: None, + active_users: 0, + ttb: page_table.ttb(), + }, + "VmBinding", + ), + GFP_KERNEL, + )?; + + let binding_clone = binding.clone(); + Ok(Vm { + fault, + id, + dummy_obj: dummy_obj.gem.clone(), + inner: gpuvm::GpuVm::new( + c_str!("Asahi::GpuVm"), + // TODO: should we using DRM_GPUVM_RESV_PROTECTED as well? + drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, + dev, + dummy_obj.gem.clone(), + gpuvm_range, + kernel_range, + init!(VmInner { + dev: dev.into(), + va_range, + is_kernel, + fixed_slot: None, + page_table, + mapping_epoch: None, + epoch: Some(0), + mm, + uat_inner, + binding: binding_clone, + id, + }), + )?, + binding, + mapping_epoch: None, + lifetime: None, + residency: None, + context_bindings: None, + shared_bindings: None, + status: None, + driver_mappings: if cfg.m3_owner() { + Some(Arc::pin_init(new_mutex!(None, "VmDriverMappings"), GFP_KERNEL)?) + } else { None }, + agx_status: if cfg.m3_owner() { + Some(Arc::new(crate::agx_status::VmStatus::new(), GFP_KERNEL)?) + } else { None }, + job_lifetime: if cfg.m3_owner() { + Some(Arc::pin_init(new_mutex!(M3VmJobLifetime { + active: 0, closed: false, close_ranges: None, closed_objects: KVec::new(), + }, "M3VmJobLifetime"), GFP_KERNEL)?) + } else { None }, + + }) + } + /// Get the translation table base for this Vm fn ttb(&self) -> u64 { self.binding.lock().ttb @@ -2481,7 +2948,9 @@ impl Vm { /// Retain hardware-required fixed aliases for this VM's complete /// lifetime. Installing them before the VM reaches userspace also makes /// the GPUVM allocator reserve the addresses against user mappings. - pub(crate) fn install_driver_mappings( + /// + /// `install_driver_mappings` of every VM except published-roots (G17) ones. + fn m3_install_driver_mappings( &self, mappings: KVec, reserved_ranges: KVec>, @@ -2497,7 +2966,8 @@ impl Vm { Ok(()) } - pub(crate) fn driver_range_overlaps(&self, range: Range) -> bool { + /// `driver_range_overlaps` of every VM except published-roots (G17) ones. + fn m3_driver_range_overlaps(&self, range: Range) -> bool { let Some(driver_mappings) = &self.driver_mappings else { return false; }; driver_mappings.lock().as_ref().is_some_and(|driver| { any_range_overlaps( @@ -2515,7 +2985,9 @@ impl Vm { } /// Remove userspace mappings while preserving VM-lifetime driver aliases. - pub(crate) fn unmap_user_ranges( + /// + /// `unmap_user_ranges` of every VM except published-roots (G17) ones. + fn m3_unmap_user_ranges( &self, user_range: Range, kernel_range: Range, @@ -2596,12 +3068,7 @@ impl Vm { inner.covers_mapped_range(address, size, need_read, need_write) } - /// Resolve an IOVA through this VM's current page table. Callers use this - /// for mapping-identity checks only; it does not read the mapped page. - pub(crate) fn translate_iova(&self, address: u64) -> Result { - let mut inner = self.inner.exec_lock(None, false)?; - inner.page_table.translate_iova(address) - } + // `translate_iova` (the same exec-locked page-table lookup) is defined in `context.rs`. /// Read GPU-visible bytes out of this VM. /// @@ -2681,14 +3148,346 @@ impl Vm { prot: Prot, guard: bool, ) -> Result { - self.map_in_range_with_guard_size( - gem, - object_range, + if self.published() { + return self.map_in_range_with_guard_size( + gem, + object_range, + alignment, + range, + prot, + if guard { UAT_PGSZ } else { 0 }, + ); + } + let size = object_range.range(); + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(gem), false)?; + let vm_bo = self.inner.obtain_bo(gem)?; + + let mut vm_bo_guard = vm_bo.inner().inner.lock(); + if vm_bo_guard.sgt.is_none() { + vm_bo_guard.sgt.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + // Reserve the VA before attaching objects whose destructors take the + // GEM reservation lock. The allocator drops its payload internally + // on an insertion error, while exec_lock still owns that lock. + let node_result = inner.mm.insert_node_in_range( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: None, + _gem: None, + offset: object_range.start, + mapped_size: size, + }, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page alignment, - range, + 0, + range.start, + range.end, + mm::InsertMode::Best, + ); + let mut node = match node_result { + Ok(node) => node, + Err(err) => { + core::mem::drop(inner); + return Err(err); + } + }; + { + let node_inner = node.as_mut().inner_mut(); + node_inner.bo = Some(MappingBo::synchronous(vm_bo)); + node_inner._gem = Some(MappingObject::Shared(gem.into())); + } + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(ManuallyDrop::new(node))) + } + + /// Map a GEM object into this Vm at a specific address. + #[allow(clippy::too_many_arguments)] + pub(crate) fn map_at( + &self, + addr: u64, + size: usize, + gem: ARef, + prot: Prot, + guard: bool, + ) -> Result { + if self.published() { + return self.map_at_published(addr, size, gem, prot, guard); + } + let sgt = gem.owned_sg_table()?; + let mut inner = self.inner.exec_lock(Some(&gem), false)?; + + let vm_bo = self.inner.obtain_bo(&gem)?; + + let mut vm_bo_guard = vm_bo.inner().inner.lock(); + if vm_bo_guard.sgt.is_none() { + vm_bo_guard.sgt.replace(sgt); + } + core::mem::drop(vm_bo_guard); + + let uat_inner = inner.uat_inner.clone(); + // See map_in_range(): keep dma_resv-owning references out of the + // allocator payload until the fixed-address reservation succeeds. + let node_result = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }, + addr, + (size + if guard { UAT_PGSZ } else { 0 }) as u64, // Add guard page + 0, + ); + let mut node = match node_result { + Ok(node) => node, + Err(err) => { + core::mem::drop(inner); + return Err(err); + } + }; + { + let node_inner = node.as_mut().inner_mut(); + node_inner.bo = Some(MappingBo::synchronous(vm_bo)); + node_inner._gem = Some(MappingObject::Shared(gem.clone())); + } + + let ret = inner.map_node(&node, prot); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(ManuallyDrop::new(node))) + } + + /// Map a range of a GEM object into this Vm using GPUVM. + #[allow(clippy::too_many_arguments)] + pub(crate) fn bind_object( + &self, + gem: &gem::Object, + addr: u64, + size: u64, + offset: u64, + prot: Prot, + single_page: bool, + ) -> Result { + if self.published() { + return self.bind_object_published(gem, addr, size, offset, prot, single_page); + } + // Mapping needs a complete context + let mut ctx = StepContext { + new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), prot, - if guard { UAT_PGSZ } else { 0 }, - ) + ..Default::default() + }; + + let vm_bo = self.inner.obtain_bo(gem)?; + { + let mut vm_bo_guard = vm_bo.inner().inner.lock(); + if vm_bo_guard.sgt.is_none() { + let sgt = gem.owned_sg_table()?; + + if vm_bo_guard.sg_vec.is_none() { + let mut sg_vec = KVVec::new(); + let mut offset = 0; + for range in sgt.iter() { + let addr = range.dma_address() as usize; + let len = range.dma_len() as usize; + sg_vec.push((offset, addr..(addr + len)), GFP_KERNEL)?; + offset += len; + } + vm_bo_guard.sg_vec.replace(sg_vec); + } + vm_bo_guard.sgt.replace(sgt); + } + core::mem::drop(vm_bo_guard); + } + + let mut inner = self.inner.exec_lock(Some(gem), true)?; + + // Legacy preallocation happens before GPUVM can unmap an old VA. + if self.fault.is_none() { + inner.page_table.alloc_pages(addr..(addr + size))?; + } + + ctx.vm_bo = Some(vm_bo); + + if (addr | size | offset) & (UAT_PGMSK as u64) != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: Map step {:#x} [{:#x}] -> {:#x} is not page-aligned\n", + offset, + size, + addr + ); + return Err(EINVAL); + } + + let (flags, gem_range) = if single_page { + (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) + } else { + (gpuvm::GpuVaFlags::NONE, 0u32) + }; + + mod_dev_dbg!( + inner.dev, + "MMU: sm_map: {:#x} [{:#x}] -> {:#x}\n", + offset, + size, + addr + ); + inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) + } + + /// Add a direct MMIO mapping to this Vm at a free address. + pub(crate) fn map_io( + &self, + iova: u64, + phys: usize, + size: usize, + prot: Prot, + ) -> Result { + if self.published() { + return self.map_io_published(iova, phys, size, prot); + } + self.map_io_internal(Some(iova), 0..0, phys, size, prot) + } + + /// Allocate a firmware MMIO VA through the same DRM range allocator as GEM. + pub(crate) fn map_io_in_range(&self, range: Range, phys: usize, + size: usize, prot: Prot) -> Result { + self.map_io_internal(None, range, phys, size, prot) + } + + fn map_io_internal(&self, fixed: Option, range: Range, phys: usize, + size: usize, prot: Prot) -> Result { + let mut inner = self.inner.exec_lock(None, false)?; + let iova = fixed.unwrap_or(0); + if size == 0 || (iova as usize | phys | size) & UAT_PGMSK != 0 { + dev_err!( + inner.dev.as_ref(), + "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", + phys, + size, + iova + ); + return Err(EINVAL); + } + + // Legacy GPUs only map fixed addresses here, and keep logging each + // firmware MMIO mapping at boot as they always have. + if self.fault.is_none() { + dev_info!( + inner.dev.as_ref(), + "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", + phys, + size, + iova + ); + } + + let payload = KernelMappingInner { + owner: self.inner.clone(), uat_inner: inner.uat_inner.clone(), prot, + bo: None, _gem: None, offset: 0, mapped_size: size, + }; + let node = if let Some(iova) = fixed { + inner.mm.reserve_node(payload, iova, size as u64, 0)? + } else { + inner.mm.insert_node_in_range(payload, size as u64, UAT_PGSZ as u64, + 0, range.start, range.end, mm::InsertMode::Best)? + }; + let iova = node.start(); + if self.fault.is_some() { + mod_dev_dbg!(inner.dev, "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", phys, size, iova); + } + + let prepared = if self.fault.is_some() { + inner.page_table.prepare_map(iova..iova.checked_add(size as u64).ok_or(EOVERFLOW)?) + } else { Ok(()) }; + if let Err(error) = prepared { + // Drop the VM lock before the reserved node and its owner refs. + core::mem::drop(inner); + return Err(error); + } + let ret = inner.page_table.map_pages( + iova..(iova + size as u64), + phys as PhysicalAddr, + prot, + false, + ); + // Drop the exec_lock first, so that if map_node failed the + // KernelMappingInner destructur does not deadlock. + core::mem::drop(inner); + ret?; + Ok(KernelMapping(ManuallyDrop::new(node))) + } + + /// Unmap everything in an address range. + pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { + if self.published() { + if self.defer_unmap(iova, size)? { + return Ok(()); + } + return self.unmap_range_commit(iova, size, true); + } + // Unmapping a range can only do a single split, so just preallocate + // the prev and next GpuVas + let mut ctx = StepContext { + prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), + ..Default::default() + }; + + let mut inner = self.inner.exec_lock(None, false)?; + + mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); + inner.sm_unmap(&mut ctx, iova, size) + } + + /// Drop mappings for a given bo. + /// + /// `drop_mappings` of every VM except published-roots (G17) ones. + fn m3_drop_mappings(&self, gem: &gem::Object) -> Result { + if let Some(job_lifetime) = &self.job_lifetime { + let mut state = job_lifetime.lock(); + if state.active != 0 { + if !state.closed_objects.iter().any(|object| core::ptr::eq(&**object, gem)) { + state.closed_objects.push(gem.into(), GFP_KERNEL)?; + } + return Ok(()); + } + } + + // Removing whole mappings only does unmaps, so no preallocated VAs + let mut ctx = Default::default(); + + let inner = self.inner.exec_lock(Some(gem), false)?; + + if let Some(bo) = self.inner.find_bo(gem) { + mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); + self.inner.bo_unmap(&mut ctx, &bo)?; + mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); + // We need to drop the exec_lock first, then the GpuVmBo since that will take the lock itself. + core::mem::drop(inner); + core::mem::drop(bo); + } + + Ok(()) } /// Reserves a whole-page unmapped guard without adding it to the mapped backing. @@ -2726,10 +3525,6 @@ impl Vm { let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); let uat_inner = inner.uat_inner.clone(); - // Reserve the VA before attaching objects whose destructors take the - // GEM reservation lock. The allocator drops its payload internally - // on an insertion error, while exec_lock still owns that lock. - let node_result = inner.mm.insert_node_in_range( // A failed reservation drops its payload while exec still owns GEM // reservations. Attach BO/GEM references only after success, since // their destructors may acquire the same reservation locks. @@ -2760,18 +3555,7 @@ impl Vm { { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); - payload._gem = Some(gem.into()); - let mut node = match node_result { - Ok(node) => node, - Err(err) => { - core::mem::drop(inner); - return Err(err); - } - }; - { - let node_inner = node.as_mut().inner_mut(); - node_inner.bo = Some(vm_bo); - node_inner._gem = Some(gem.into()); + payload._gem = Some(MappingObject::Pinned(gem::KernelMappingPin::new(gem))); } let ret = inner.map_node(&node, prot); @@ -2782,9 +3566,9 @@ impl Vm { Ok(KernelMapping(ManuallyDrop::new(node))) } - /// Map a GEM object into this Vm at a specific address. + /// `map_at` of a published-roots (G17) VM. #[allow(clippy::too_many_arguments)] - pub(crate) fn map_at( + fn map_at_published( &self, addr: u64, size: usize, @@ -2809,10 +3593,6 @@ impl Vm { let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); let uat_inner = inner.uat_inner.clone(); - // See map_in_range(): keep dma_resv-owning references out of the - // allocator payload until the fixed-address reservation succeeds. - let node_result = inner.mm.reserve_node( - // Published-root VMs defer BO/GEM attachment until reservation succeeds. // Keep reservation-locking destructors out of the allocator payload // until reservation succeeds. let result = inner.mm.reserve_node( @@ -2839,18 +3619,7 @@ impl Vm { { let payload = node.as_mut().inner_mut(); payload.bo = vm_bo.take(); - payload._gem = Some(gem.clone()); - let mut node = match node_result { - Ok(node) => node, - Err(err) => { - core::mem::drop(inner); - return Err(err); - } - }; - { - let node_inner = node.as_mut().inner_mut(); - node_inner.bo = Some(vm_bo); - node_inner._gem = Some(gem.clone()); + payload._gem = Some(MappingObject::Pinned(gem::KernelMappingPin::new(&gem))); } let ret = inner.map_node(&node, prot); @@ -2861,9 +3630,9 @@ impl Vm { Ok(KernelMapping(ManuallyDrop::new(node))) } - /// Map a range of a GEM object into this Vm using GPUVM. + /// `bind_object` of a published-roots (G17) VM. #[allow(clippy::too_many_arguments)] - pub(crate) fn bind_object( + fn bind_object_published( &self, gem: &gem::Object, addr: u64, @@ -2910,15 +3679,12 @@ impl Vm { // owner. The declaration order also preserves this on errors. drop(vm_bo_guard); } - core::mem::drop(vm_bo_guard); } let mut inner = self.inner.exec_lock(Some(gem), true)?; - // Legacy preallocation happens before GPUVM can unmap an old VA. - if self.fault.is_none() { - inner.page_table.alloc_pages(addr..(addr + size))?; - } + // Preallocate the page tables, to fail early if we ENOMEM + inner.page_table.alloc_pages(addr..(addr + size))?; ctx.vm_bo = Some(vm_bo); @@ -2950,28 +3716,17 @@ impl Vm { inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) } - /// Add a direct MMIO mapping to this Vm at a free address. - pub(crate) fn map_io( + /// `map_io` of a published-roots (G17) VM. + fn map_io_published( &self, iova: u64, phys: usize, size: usize, prot: Prot, ) -> Result { - self.map_io_internal(Some(iova), 0..0, phys, size, prot) - } - - /// Allocate a firmware MMIO VA through the same DRM range allocator as GEM. - pub(crate) fn map_io_in_range(&self, range: Range, phys: usize, - size: usize, prot: Prot) -> Result { - self.map_io_internal(None, range, phys, size, prot) - } - - fn map_io_internal(&self, fixed: Option, range: Range, phys: usize, - size: usize, prot: Prot) -> Result { let mut inner = self.inner.exec_lock(None, false)?; - let iova = fixed.unwrap_or(0); - if size == 0 || (iova as usize | phys | size) & UAT_PGMSK != 0 { + + if (iova as usize | phys | size) & UAT_PGMSK != 0 { dev_err!( inner.dev.as_ref(), "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", @@ -2982,41 +3737,30 @@ impl Vm { return Err(EINVAL); } - // Legacy GPUs only map fixed addresses here, and keep logging each - // firmware MMIO mapping at boot as they always have. - if self.fault.is_none() { - dev_info!( - inner.dev.as_ref(), - "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", - phys, - size, - iova - ); - } + dev_info!( + inner.dev.as_ref(), + "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", + phys, + size, + iova + ); - let payload = KernelMappingInner { - owner: self.inner.clone(), uat_inner: inner.uat_inner.clone(), prot, - bo: None, _gem: None, offset: 0, mapped_size: size, - }; - let node = if let Some(iova) = fixed { - inner.mm.reserve_node(payload, iova, size as u64, 0)? - } else { - inner.mm.insert_node_in_range(payload, size as u64, UAT_PGSZ as u64, - 0, range.start, range.end, mm::InsertMode::Best)? - }; - let iova = node.start(); - if self.fault.is_some() { - mod_dev_dbg!(inner.dev, "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", phys, size, iova); - } + let uat_inner = inner.uat_inner.clone(); + let node = inner.mm.reserve_node( + KernelMappingInner { + owner: self.inner.clone(), + uat_inner, + prot, + bo: None, + _gem: None, + offset: 0, + mapped_size: size, + }, + iova, + size as u64, + 0, + )?; - let prepared = if self.fault.is_some() { - inner.page_table.prepare_map(iova..iova.checked_add(size as u64).ok_or(EOVERFLOW)?) - } else { Ok(()) }; - if let Err(error) = prepared { - // Drop the VM lock before the reserved node and its owner refs. - core::mem::drop(inner); - return Err(error); - } let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; inner.page_table.prepare_map(iova..end)?; let mutation = inner.mapping_mutation(); @@ -3037,14 +3781,6 @@ impl Vm { Ok(KernelMapping(ManuallyDrop::new(node))) } - /// Unmap everything in an address range. - pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { - if self.defer_unmap(iova, size)? { - return Ok(()); - } - self.unmap_range_commit(iova, size, true) - } - fn unmap_prepared_cleanup(&self, unmap: &mut bind::PreparedUserUnmap) -> Result { let mut inner = self.inner.lock_inner(); // SAFETY: step_unmap/step_remap modify this VM's translations and @@ -3091,19 +3827,10 @@ impl Vm { Ok(()) } - /// Drop mappings for a given bo. - pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { - if let Some(job_lifetime) = &self.job_lifetime { - let mut state = job_lifetime.lock(); - if state.active != 0 { - if !state.closed_objects.iter().any(|object| core::ptr::eq(&**object, gem)) { - state.closed_objects.push(gem.into(), GFP_KERNEL)?; - } - return Ok(()); - } - } - fn drop_mappings_now(&self, gem: &gem::Object) -> Result { + if !self.published() { + return self.m3_drop_mappings(gem); + } let _residency = self.enter_retirement_residency(); // Removing whole mappings only does unmaps, so no preallocated VAs let mut ctx = Default::default(); @@ -3166,8 +3893,63 @@ impl Vm { } } +impl VmInner { + /// `Drop` of a published-roots (G17) VM. + fn drop_published(&mut self) { + let mut binding = self.binding.lock(); + assert_eq!(binding.active_users, 0); + + mod_pr_debug!( + "VmInner::Drop [{}]: bind_token={:?}\n", + self.id, + binding.bind_token + ); + + // Make sure this VM is not mapped to a TTB if it was + if let Some(token) = binding.bind_token.take() { + let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; + let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; + + let uat_inner = self.uat_inner.lock(); + let handoff = uat_inner.handoff().lock_legacy(); + let handoff_cur = uat_inner.handoff().current_slot(); + let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); + let inval = ttb_cur == ttb; + if inval { + if handoff_cur == Some(idx as u32) { + pr_err!( + "VmInner::drop owning slot {}, but it is currently in use by the ASC?\n", + idx + ); + } + uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); + uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); + } + drop(handoff); + core::mem::drop(uat_inner); + + // In principle we dropped all the KernelMappings already, but we might as + // well play it safe and invalidate the whole ASID. + if inval { + mod_pr_debug!( + "VmInner::Drop [{}]: need inval for ASID {:#x}\n", + self.id, + idx + ); + mem::sync(); + mem::tlbi_asid(idx as u8); + mem::sync(); + } + } + } +} + impl Drop for VmInner { fn drop(&mut self) { + if self.published() { + self.drop_published(); + return; + } if self.uat_inner.fault.as_ref().is_some_and(|fault| fault.load(Ordering::Acquire)) { self.page_table.quarantine(); core::mem::forget(self.uat_inner.clone()); @@ -3222,7 +4004,6 @@ impl Drop for VmInner { self.id, idx ); - mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); } @@ -3324,7 +4105,11 @@ impl Uat { #[cfg(CONFIG_DEV_COREDUMP)] pub(crate) fn dump_kernel_pages(&self) -> Result> { // Full upper-half range for this SoC's root geometry. - let full_range = uat_geometry(self.cfg)?.upper_canonical_base()..(!UAT_PGMSK as u64); + let full_range = if self.inner.firmware != UatFirmware::Handoff { + self.geometry.upper_range() + } else { + uat_geometry(self.cfg)?.upper_canonical_base()..(!UAT_PGMSK as u64) + }; let mut inner = self.kernel_vm.inner.exec_lock(None, false)?; inner.page_table.dump_pages(full_range) } @@ -3427,7 +4212,6 @@ impl Uat { core::mem::drop(uat_inner); // Make sure all TLB entries from the previous owner of this ASID are gone - mem::sync(); mem::tlbi_asid(idx as u8); mem::sync(); slot.generation = slot.generation.wrapping_add(1).max(1); @@ -3502,18 +4286,6 @@ impl Uat { Ok(context) } - /// Creates a new `Vm` linked to this UAT. - pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( - &self.dev, - self.inner.clone(), - kernel_range, - self.cfg, - None, - id, - ) - } - /// Returns the address space geometry of this UAT. pub(crate) fn geometry(&self) -> UatGeometry { self.geometry @@ -3598,20 +4370,20 @@ impl Uat { /// Both contexts must still contain their original kernel roots. pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { let inner = self.inner.lock(); - inner.handoff().lock(); + let handoff = inner.handoff().lock_legacy(); let ttbs = inner.ttbs(); let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) }); if !ready { - inner.handoff().unlock(); + drop(handoff); return Err(EIO); } ttbs[KERNEL_ALIAS_CTX] .ttb0 .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); + drop(handoff); core::mem::drop(inner); mem::sync(); mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); @@ -3619,9 +4391,36 @@ impl Uat { Ok(()) } + /// Creates a new `Vm` linked to this UAT. + pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { + if self.inner.firmware != UatFirmware::Handoff { + return Vm::new_published( + &self.dev, + self.inner.clone(), + kernel_range, + self.geometry, + None, + id, + ); + } + Vm::new( + &self.dev, + self.inner.clone(), + kernel_range, + self.cfg, + None, + id, + ) + } + /// Creates the reference-counted inner data for a new `Uat` instance. #[inline(never)] - fn make_inner(dev: &driver::AsahiDevice, handoff_mode: HandoffMode, m3: bool) -> Result> { + fn make_inner( + dev: &driver::AsahiDevice, + handoff_mode: HandoffMode, + m3: bool, + firmware: UatFirmware, + ) -> Result> { let cached = !matches!( handoff_mode, HandoffMode::StoppedFirmwareT6030 | HandoffMode::FirmwareM3 | HandoffMode::FirmwareG16 ); @@ -3718,11 +4517,10 @@ impl Uat { } #[inline(never)] - pub(crate) fn new( + pub(crate) fn new_for_config( dev: &driver::AsahiDevice, - geometry: UatGeometry, + cfg: &'static hw::HwConfig, map_kernel_to_user: bool, - firmware: UatFirmware, ) -> Result { Self::new_with_handoff_mode(dev, cfg, map_kernel_to_user, HandoffMode::FirmwareDekker) } @@ -3786,7 +4584,7 @@ impl Uat { } - let inner = Self::make_inner(dev, handoff_mode, cfg.m3_owner())?; + let inner = Self::make_inner(dev, handoff_mode, cfg.m3_owner(), UatFirmware::Handoff)?; let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; let res = if cfg.m3_owner() { @@ -3799,16 +4597,6 @@ impl Uat { let required_pagetables_size = PAGETABLES_SIZE; if ttb1size < required_pagetables_size { - let (pagetables_size, secondary_ttb1) = match firmware { - UatFirmware::Handoff => (PAGETABLES_SIZE, None), - UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), - }; - if ttb1size < pagetables_size { dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); return Err(ENOMEM); } @@ -3832,10 +4620,100 @@ impl Uat { kernel_lower_vm.inner.exec_lock(None, false)?.fixed_slot = Some(0); } let kernel_vm = Vm::new(dev, inner.clone(), kernel_range, cfg, Some(ttb1), 0)?; - let kernel_lower_vm = Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; + + dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); + + let ttb0 = kernel_lower_vm.ttb(); + let uat = Self { + dev: dev.into(), + cfg, + geometry: UatGeometry { + ias: u32::from(cfg.ias), + oas: cfg.oas, + }, + kernel_vm, + kernel_lower_vm, + inner, + slots: slotalloc::SlotAllocator::new( + UAT_USER_CTX as u32, + (), + |_inner, _slot| Some(SlotInner { generation: 0 }), + c_str!("Uat::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }; + + let mut inner = uat.inner.lock(); + + inner.map_kernel_to_user = map_kernel_to_user; + inner.kernel_ttb1 = ttb1; + + inner.handoff().init(handoff_mode, cfg.m3_owner(), UatFirmware::Handoff)?; + + dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); + + let handoff_guard = inner.lock_handoff()?; + + let ttbs = inner.ttbs(); + + ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::SeqCst); + ttbs[0].ttb1.store(ttb1 | TTBR_VALID, Ordering::SeqCst); + + for ctx in &ttbs[1..] { + ctx.ttb0.store(0, Ordering::Relaxed); + ctx.ttb1.store(0, Ordering::Relaxed); + } + + drop(handoff_guard); + + core::mem::drop(inner); + + dev_info!(dev.as_ref(), "MMU: initialized\n"); + + Ok(uat) + } + + /// Creates a new published-roots (G17) `Uat` for the given address space geometry. + #[inline(never)] + pub(crate) fn new( + dev: &driver::AsahiDevice, + geometry: UatGeometry, + map_kernel_to_user: bool, + firmware: UatFirmware, + ) -> Result { + dev_info!(dev.as_ref(), "MMU: Initializing...\n"); + + if firmware == UatFirmware::Handoff { + // Handoff UATs are built by `new_for_config` and its siblings. + return Err(EINVAL); + } + let cfg = UatConfig::published(geometry)?; + let inner = Self::make_inner(dev, HandoffMode::FirmwareDekker, false, firmware)?; + + let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; + let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; + let ttb1 = res.start(); + let ttb1size: usize = res.size().try_into()?; + + let (pagetables_size, secondary_ttb1) = match firmware { + UatFirmware::Handoff => (PAGETABLES_SIZE, None), + UatFirmware::PublishedRoots { + secondary_root_offset, + } => ( + secondary_root_offset as usize + UAT_PGSZ, + Some(ttb1 + secondary_root_offset), + ), + }; + if ttb1size < pagetables_size { + dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); + return Err(ENOMEM); + } + + dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); let kernel_lower_vm = - Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; - let kernel_vm = Vm::new( + Vm::new_published(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_vm = Vm::new_published( dev, inner.clone(), geometry.kernel_range(), @@ -3847,8 +4725,10 @@ impl Uat { dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); let ttb0 = kernel_lower_vm.ttb(); + let uat = Self { dev: dev.into(), + cfg, geometry, kernel_vm, kernel_lower_vm, @@ -3869,11 +4749,11 @@ impl Uat { inner.kernel_ttb1 = ttb1; inner.secondary_ttb1 = secondary_ttb1; - inner.handoff().init(handoff_mode, cfg.m3_owner())?; + inner.handoff().init(HandoffMode::FirmwareDekker, false, firmware)?; dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - let handoff_guard = inner.lock_handoff()?; + let handoff = inner.handoff().lock_legacy(); let ttbs = inner.ttbs(); @@ -3885,7 +4765,6 @@ impl Uat { ctx.ttb1.store(0, Ordering::Relaxed); } - drop(handoff_guard); if secondary_ttb1.is_some() { let alias = &ttbs[KERNEL_ALIAS_CTX]; alias @@ -3896,7 +4775,7 @@ impl Uat { .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); } - inner.handoff().unlock(); + drop(handoff); core::mem::drop(inner); @@ -3908,6 +4787,13 @@ impl Uat { impl Drop for Uat { fn drop(&mut self) { + if self.inner.firmware != UatFirmware::Handoff { + // Make sure we flush the TLBs + mem::sync(); + mem::tlbi_all(); + mem::sync(); + return; + } if self.cfg.reserved_tables { self.inner.firmware_cache_flush_ready.store(false, Ordering::Release); let inner = self.inner.lock(); @@ -3922,7 +4808,7 @@ impl Drop for Uat { } // Make sure we flush the TLBs - mem::sync(); + fence(Ordering::SeqCst); mem::tlbi_all(); mem::sync(); } diff --git a/drivers/gpu/drm/asahi/mmu/bind.rs b/drivers/gpu/drm/asahi/mmu/bind.rs index 47a3421fa04b34..824a960f6530b5 100644 --- a/drivers/gpu/drm/asahi/mmu/bind.rs +++ b/drivers/gpu/drm/asahi/mmu/bind.rs @@ -56,7 +56,6 @@ impl PreparedUserBindBatch { impl Vm { /// Prepares host state without taking DMA reservations or changing any leaf. - #[allow(clippy::too_many_arguments)] pub(crate) fn prepare_bind_object( &self, gem: &gem::Object, diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 975126e0cc76b0..42ac32bdae1302 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -134,6 +134,9 @@ impl Vm { mappings: KVec, reserved: KVec>, ) -> Result { + if !self.published() { + return self.m3_install_driver_mappings(mappings, reserved); + } let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; let mut driver = lifetime.driver.lock(); if driver.is_some() { @@ -335,6 +338,9 @@ impl Vm { /// Tests mappings and reservations which userspace must never replace. pub(crate) fn driver_range_overlaps(&self, range: Range) -> bool { + if !self.published() { + return self.m3_driver_range_overlaps(range); + } let Some(lifetime) = self.lifetime.as_ref() else { return false; }; @@ -561,6 +567,9 @@ impl Vm { /// Closes admission permanently and preserves mappings referenced by accepted jobs. pub(crate) fn unmap_user_ranges(&self, user: Range, kernel: Range) -> Result { + if !self.published() { + return self.m3_unmap_user_ranges(user, kernel); + } if let Some(lifetime) = self.lifetime.as_ref() { let mut state = lifetime.state.lock(); state.blocked = true; diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index 7d71ea0c226cb6..534af29461baf6 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -54,7 +54,6 @@ const UAT_LVSZ: usize = UAT_NPTE; const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; const UAT_LEVELS: usize = crate::uat::UAT_LEVELS; -const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; @@ -391,20 +390,43 @@ pub(crate) struct UatPageTable { noncoherent: bool, dma_tables: Option, reserved_tables: Option, - /// Tables below a root the driver did not allocate may belong to the bootloader or the - /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the - /// tables below the root, as with the handoff firmware interface. - driver_tables: Option>, va_range: Range, geometry: UatGeometry, oas_mask: u64, coverage: Option>, + /// G17 tables (see [`Self::new`]): every walk, including those of 42-bit roots, uses the + /// ordered walk with the leaf-level semantics of the 39-bit walk. + ordered_walk: bool, + /// Tables below a root the driver did not allocate may belong to the bootloader or the + /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the + /// tables below the root. Only used by the ordered walk. + driver_tables: Option>, } impl UatPageTable { + /// Allocate a root for a G17 user VM. Its tables use the ordered walk. pub(crate) fn new(ias: u32, oas: u32) -> Result { mod_pr_debug!("UATPageTable::new: ias={} oas={}\n", ias, oas); - pub(crate) fn new(ias: u8, oas: u32) -> Result { + let geometry = UatGeometry::new(u8::try_from(ias)?).ok_or(EINVAL)?; + let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let ttb = Page::into_phys(ttb_page); + Ok(UatPageTable { + ttb, + ttb_owned: true, + quarantined: false, + noncoherent: false, + dma_tables: None, + reserved_tables: None, + va_range: 0..(1u64 << ias), + geometry, + oas_mask: (1u64 << oas) - 1, + coverage: None, + ordered_walk: true, + driver_tables: None, + }) + } + + pub(crate) fn new_by_geometry(ias: u8, oas: u32) -> Result { mod_pr_debug!("UATPageTable::new: ias={} oas={}\n", ias, oas); let geometry = UatGeometry::new(ias).ok_or(EINVAL)?; let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; @@ -418,16 +440,15 @@ impl UatPageTable { reserved_tables: None, va_range: 0..geometry.root_size(), geometry, - driver_tables: None, - va_range: 0..(1u64 << ias), - ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, coverage: None, + ordered_walk: false, + driver_tables: None, }) } pub(crate) fn new_m3_coherent(ias:u8,oas:u32,dev:&crate::driver::AsahiDevice)->Result { - let mut table=Self::new(ias,oas)?; + let mut table=Self::new_by_geometry(ias,oas)?; let dma=DmaTables::new(dev);let root=dma.alloc()?; // SAFETY: this newly allocated, unpublished root is still empty. unsafe {Page::from_phys(table.ttb)}; @@ -439,6 +460,10 @@ impl UatPageTable { /// mapped memory. This is used to prove that separately named aliases /// refer to the same backing page before firmware can consume them. pub(crate) fn translate_iova(&mut self, iova: u64) -> Result { + if self.ordered_walk { + let entry = self.leaf_entry(iova & !(UAT_PGMSK as u64))?; + return Ok((entry & self.oas_mask & !(UAT_PGMSK as u64)) | (iova & UAT_PGMSK as u64)); + } let page_mask = UAT_PGMSK as u64; let page = iova & !page_mask; let end = page.checked_add(UAT_PGSZ as u64).ok_or(EOVERFLOW)?; @@ -456,12 +481,8 @@ impl UatPageTable { translated.ok_or(EFAULT) } - pub(crate) fn new_with_ttb( - ttb: PhysicalAddr, - va_range: Range, - ias: u8, /// Borrow a root; track individual child ownership only when other agents - /// can install tables below it. Handoff roots have driver-owned children. + /// can install tables below it. Used for G17 kernel roots (ordered walk). pub(crate) fn new_with_ttb( ttb: PhysicalAddr, va_range: Range, @@ -476,7 +497,7 @@ impl UatPageTable { ias, oas ); - let geometry = UatGeometry::new(ias).ok_or(EINVAL)?; + let geometry = UatGeometry::new(u8::try_from(ias)?).ok_or(EINVAL)?; if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { return Err(EINVAL); } @@ -499,11 +520,57 @@ impl UatPageTable { noncoherent: false, dma_tables: None, reserved_tables: None, + va_range, + geometry, + oas_mask: (1u64 << oas) - 1, + coverage: None, + ordered_walk: true, driver_tables: shared_tables.then(KVec::new), + }) + } + + pub(crate) fn new_with_ttb_by_geometry( + ttb: PhysicalAddr, + va_range: Range, + ias: u8, + oas: u32, + ) -> Result { + mod_pr_debug!( + "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", + ttb, + va_range, + ias, + oas + ); + let geometry = UatGeometry::new(ias).ok_or(EINVAL)?; + if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { + return Err(EINVAL); + } + if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { + return Err(EINVAL); + } + // SAFETY: The TTB is should remain valid (if properly mapped), as it is bootloader-managed. + if unsafe { Page::borrow_phys(&ttb) }.is_none() { + pr_err!( + "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", + ttb + ); + return Err(EIO); + } + + Ok(UatPageTable { + ttb, + ttb_owned: false, + quarantined: false, + noncoherent: false, + dma_tables: None, + reserved_tables: None, va_range, geometry, oas_mask: (1u64 << oas) - 1, coverage: None, + ordered_walk: false, + driver_tables: None, }) } @@ -538,6 +605,8 @@ impl UatPageTable { geometry, oas_mask: (1u64 << oas) - 1, coverage: None, + ordered_walk: false, + driver_tables: None, }) } @@ -555,7 +624,8 @@ impl UatPageTable { })?; crate::mem::tlbi_all();crate::mem::sync(); Ok(Self {ttb,ttb_owned:false,quarantined:false,noncoherent:true,dma_tables:Some(DmaTables::new(dev)), - reserved_tables:Some(tables),va_range,geometry,oas_mask:(1u64<<42)-1,coverage:None}) + reserved_tables:Some(tables),va_range,geometry,oas_mask:(1u64<<42)-1,coverage:None, + ordered_walk:false,driver_tables:None}) } /// Adopt the loader-designated shared middle table before ASC starts. @@ -676,7 +746,15 @@ impl UatPageTable { self.ttb } - fn with_pages_legacy( + /// INT's 42-bit (AGX3) walk with prepared, non-allocating leaf writes. 39-bit roots and G17 + /// tables use the leaf semantics of the 39-bit walk instead. + fn agx3_walk(&self) -> bool { + !self.ordered_walk && self.geometry.root_size() != 1u64 << 39 + } + + /// The G17 walk (`ordered_walk`): records driver-allocated tables below shared roots, + /// completes child-table initialization before linking it, and detaches tables bottom-up. + fn with_pages_ordered( &mut self, iova_range: Range, alloc: bool, @@ -732,11 +810,9 @@ impl UatPageTable { phys ); let parent = pt_parent[level].take(); - self.free_table(phys, parent)?; + self.free_detached_table(phys, parent)?; } else { pt_parent[level] = None; - // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). - unsafe { Page::from_phys(phys) }; } mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); } @@ -847,6 +923,218 @@ impl UatPageTable { // The final path is detached bottom-up for the same reason as the // boundary transitions above. for level in 0..UAT_LEVELS - 1 { + if let Some(phys) = pt_addr[level] { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + self.free_detached_table(phys, pt_parent[level].take())?; + } + } + } + + Ok(()) + } + + /// Free a page table, unless it was not allocated by the driver. + fn free_detached_table( + &mut self, + phys: PhysicalAddr, + parent: Option<(PhysicalAddr, usize)>, + ) -> Result { + let owned = if let Some(tables) = self.driver_tables.as_ref() { + match tables.iter().position(|&table| table == phys) { + Some(_) => true, + None => false, + } + } else { + true + }; + if !owned { + return Ok(()); + } + if let Some((parent_phys, parent_index)) = parent { + // SAFETY: The bottom-up traversal keeps the parent allocated while + // its child descriptor is detached. A changed descriptor indicates + // an ownership violation; fail closed instead of freeing a table + // that may still be reachable. + let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; + page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { + let pte = unsafe { &*(p as *const AtomicU64) }; + let current = pte.load(Ordering::Acquire); + let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); + if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { + return Err(EIO); + } + pte.store(0, Ordering::Release); + Ok(()) + })?; + } + if let Some(tables) = self.driver_tables.as_mut() { + let index = tables + .iter() + .position(|&table| table == phys) + .ok_or(EIO)?; + tables.swap_remove(index); + } + // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without + // `driver_tables`, every child table belongs to the driver. + unsafe { Page::from_phys(phys) }; + Ok(()) + } + + fn with_pages_legacy( + &mut self, + iova_range: Range, + alloc: bool, + free: bool, + mut cb: F, + ) -> Result + where + F: FnMut(u64, &[Pte]) -> Result, + { + if self.ordered_walk { + return self.with_pages_ordered(iova_range, alloc, free, cb); + } + mod_pr_debug!( + "UATPageTable::with_pages: {:#x?} alloc={} free={}\n", + iova_range, + alloc, + free + ); + if (iova_range.start | iova_range.end) & (UAT_PGMSK as u64) != 0 { + pr_err!( + "UATPageTable::with_pages: iova range not aligned: {:#x?}\n", + iova_range + ); + return Err(EINVAL); + } + + if iova_range.is_empty() { + return Ok(()); + } + + let mut iova = iova_range.start & self.geometry.root_mask(); + let mut last_iova = iova; + // Handle the case where iova_range.end is just at the top boundary of the IAS + let end = ((iova_range.end - 1) & self.geometry.root_mask()) + 1; + + let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); + pt_addr[UAT_LEVELS - 1] = Some(self.ttb); + + 'outer: while iova < end { + mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); + let addr_diff = last_iova ^ iova; + for level in (0..UAT_LEVELS - 1).rev() { + // If the iova has changed at this level or above, invalidate the physaddr + if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { + if let Some(phys) = pt_addr[level].take() { + if free { + mod_pr_debug!( + "UATPageTable::with_pages: free level {} {:#x?}\n", + level, + phys + ); + // SAFETY: Page tables for our VA ranges always come from Page::into_phys(). + unsafe { Page::from_phys(phys) }; + } + mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); + } + } + } + last_iova = iova; + for level in (0..UAT_LEVELS - 1).rev() { + // Fetch the page table base address for this level + if pt_addr[level].is_none() { + let phys = pt_addr[level + 1].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: need level {}, parent phys {:#x}\n", + level, + phys + ); + let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) + as usize; + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; + mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); + pt_addr[level] = + upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { + let uptep = p as *const _ as *const Pte; + // SAFETY: with_pointer_into_page() ensures the pointer is valid, + // and our index is aligned so it is safe to deref as an AtomicU64. + let upte = unsafe { &*uptep }; + let mut upte_val = upte.load(Ordering::Relaxed); + // Allocate if requested + if upte_val == 0 && alloc { + let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + let pt_paddr = Page::into_phys(pt_page); + upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; + upte.store(upte_val, Ordering::Relaxed); + } + if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { + Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) + } else if upte_val == 0 || (!alloc && !free) { + mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); + Ok(None) + } else { + pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, + iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); + Ok(None) + } + })?; + mod_pr_debug!( + "UATPageTable::with_pages: level {} PT {:#x?}\n", + level, + pt_addr[level] + ); + } + // If we don't have a page table, skip this entire level + if pt_addr[level].is_none() { + let block = 1 << (UAT_PGBIT + UAT_LVBIT * (level + 1)); + let old = iova; + iova = align(iova + 1, block); + mod_pr_debug!( + "UATPageTable::with_pages: skip {:#x} {:#x} -> {:#x}\n", + block, + old, + iova + ); + continue 'outer; + } + } + + let idx = ((iova >> UAT_PGBIT as u64) & UAT_LVMSK) as usize; + let max_count = UAT_NPTE - idx; + let count = (((end - iova) >> UAT_PGBIT) as usize).min(max_count); + let phys = pt_addr[0].unwrap(); + mod_pr_debug!( + "UATPageTable::with_pages: leaf PT at {:#x} idx {:#x} count {:#x} iova {:#x}\n", + phys, + idx, + count, + iova + ); + // SAFETY: Page table addresses are either allocated by us, or + // firmware-managed and safe to borrow a struct page from. + let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; + pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { + let ptep = p as *const _ as *const Pte; + // SAFETY: We know this is a valid pointer to PTEs and the range is valid and + // checked by with_pointer_into_page(). + let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; + cb(iova, ptes)?; + Ok(()) + })?; + + let block = 1 << (UAT_PGBIT + UAT_LVBIT); + iova = align(iova + 1, block); + } + + if free { + for level in (0..UAT_LEVELS - 1).rev() { if let Some(phys) = pt_addr[level] { mod_pr_debug!( "UATPageTable::with_pages: free level {} {:#x?}\n", @@ -873,7 +1161,7 @@ impl UatPageTable { where F: FnMut(u64, &[Pte]) -> Result, { - if self.geometry.root_size() == 1u64 << 39 { + if !self.agx3_walk() { return self.with_pages_legacy(iova_range, alloc, free, cb); } @@ -1027,107 +1315,9 @@ impl UatPageTable { } } - Ok(()) - /// Free a page table, unless it was not allocated by the driver. - fn free_table( - &mut self, - phys: PhysicalAddr, - parent: Option<(PhysicalAddr, usize)>, - ) -> Result { - let owned = if let Some(tables) = self.driver_tables.as_ref() { - match tables.iter().position(|&table| table == phys) { - Some(_) => true, - None => false, - } - } else { - true - }; - if !owned { - return Ok(()); - } - if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { - let pte = unsafe { &*(p as *const AtomicU64) }; - let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { - return Err(EIO); - } - pte.store(0, Ordering::Release); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - let index = tables - .iter() - .position(|&table| table == phys) - .ok_or(EIO)?; - tables.swap_remove(index); - } - // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without - // `driver_tables`, every child table belongs to the driver. - unsafe { Page::from_phys(phys) }; Ok(()) } - /// Checks every page, including holes skipped by the page-table walker. - pub(crate) fn covers_range( - &mut self, - range: Range, - read: bool, - write: bool, - ) -> Result { - if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { - return Ok(false); - } - let expected = (range.end - range.start) >> UAT_PGBIT; - let mut visited = 0u64; - let mut permitted = true; - self.with_pages(range, false, false, |_, ptes| { - visited = visited.checked_add(ptes.len() as u64).ok_or(EOVERFLOW)?; - for pte in ptes { - let value = pte.load(Ordering::Acquire); - if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE - || !Prot::from_pte(value).allows_gpu(read, write) - { - permitted = false; - } - } - Ok(()) - })?; - Ok(expected == visited && permitted) - } - - /// Resolves an existing leaf without creating page tables. - pub(crate) fn translate_iova(&mut self, iova: u64) -> Result { - let entry = self.leaf_entry(iova & !(UAT_PGMSK as u64))?; - Ok((entry & self.oas_mask & !(UAT_PGMSK as u64)) | (iova & UAT_PGMSK as u64)) - } - - /// Reads one mapped leaf entry without allocating missing tables. - pub(crate) fn leaf_entry(&mut self, iova: u64) -> Result { - if iova & UAT_PGMSK as u64 != 0 { - return Err(EINVAL); - } - let end = iova.checked_add(UAT_PGSZ as u64).ok_or(EINVAL)?; - let mut value = 0; - self.with_pages(iova..end, false, false, |_, ptes| { - if let Some(pte) = ptes.first() { - value = pte.load(Ordering::Relaxed); - } - Ok(()) - })?; - if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { - return Err(EFAULT); - } - Ok(value) - } - pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); self.with_pages(iova_range, true, false, false, |_, _| Ok(())) @@ -1140,6 +1330,20 @@ impl UatPageTable { /// can detect ownership held by the other one. The page table is the /// authoritative final arbiter. pub(crate) fn prepare_map(&mut self, iova_range: Range) -> Result { + if self.ordered_walk { + // G17: GPUVM and the fixed-address allocator may share a page table, so check + // that neither owns a destination leaf before allowing failure rollback. + if iova_range.is_empty() || (iova_range.start | iova_range.end) & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + self.with_pages(iova_range.clone(), false, false, false, |_, ptes| { + if ptes.iter().any(|pte| pte.load(Ordering::Acquire) != 0) { + return Err(EBUSY); + } + Ok(()) + })?; + return self.alloc_pages(iova_range); + } if self.geometry.root_size() == 1u64 << 39 { return self.alloc_pages(iova_range); } if iova_range.is_empty() @@ -1216,22 +1420,6 @@ impl UatPageTable { Ok(covered) } - /// Reserve all tables for a fixed mapping before publishing any leaves. - /// GPUVM and the fixed-address allocator may share a page table, so check - /// that neither owns a destination leaf before allowing failure rollback. - pub(crate) fn prepare_map(&mut self, range: Range) -> Result { - if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { - return Err(EINVAL); - } - self.with_pages(range.clone(), false, false, |_, ptes| { - if ptes.iter().any(|pte| pte.load(Ordering::Acquire) != 0) { - return Err(EBUSY); - } - Ok(()) - })?; - self.alloc_pages(range) - } - fn pte_bits(&self) -> u64 { if self.ttb_owned { // Owned page tables are userspace, so non-global @@ -1288,7 +1476,7 @@ impl UatPageTable { prot: Prot, one_page: bool, ) -> Result { - if self.geometry.root_size() == 1u64 << 39 { + if !self.agx3_walk() { return self.map_pages_legacy(iova_range, phys, prot, one_page); } @@ -1374,7 +1562,7 @@ impl UatPageTable { } pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { - if self.geometry.root_size() == 1u64 << 39 { + if !self.agx3_walk() { return self.reprot_pages_legacy(iova_range, prot); } @@ -1430,7 +1618,7 @@ impl UatPageTable { } pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { - if self.geometry.root_size() == 1u64 << 39 { + if !self.agx3_walk() { return self.unmap_pages_legacy(iova_range); } @@ -1460,15 +1648,36 @@ impl UatPageTable { }) } + /// Reads one mapped leaf entry without allocating missing tables. + pub(crate) fn leaf_entry(&mut self, iova: u64) -> Result { + if iova & UAT_PGMSK as u64 != 0 { + return Err(EINVAL); + } + let end = iova.checked_add(UAT_PGSZ as u64).ok_or(EINVAL)?; + let mut value = 0; + self.with_pages(iova..end, false, false, false, |_, ptes| { + if let Some(pte) = ptes.first() { + value = pte.load(Ordering::Relaxed); + } + Ok(()) + })?; + if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { + return Err(EFAULT); + } + Ok(value) + } + /// Discards an aligned map operation that failed before acquiring a GPUVA owner. /// Missing tables and leaves are expected; existing table storage stays allocated. pub(crate) fn discard_partial_map(&mut self, iova_range: Range) -> Result { - self.with_pages(iova_range, false, false, |_, ptes| { + self.with_pages(iova_range, false, false, true, |_, ptes| { for pte in ptes { pte.store(0, Ordering::Relaxed); } Ok(()) }) + } + /// Read one PTE from a page table page, without ever allocating or modifying anything. /// /// Fails with an error (instead of oopsing) if the table page is not mapped. diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 1ea9860b193d33..862e6ad92e752f 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -61,6 +61,8 @@ mod render; /// Trait implemented by all versioned queues. pub(crate) trait Queue: Send + Sync { + /// Reserve render ray query scratch for the given vertex and fragment layouts. Only the + /// G17P runtime supports render ray queries; every other queue accepts only zero. fn reserve_render_scs(&mut self, layouts: [u64; 2]) -> Result { if layouts != [0; 2] { return Err(EINVAL); } Ok(()) @@ -201,6 +203,7 @@ pub(crate) struct QueueJob { id: u64, } +/// Timeout state of a [`QueueJob`], owned by the scheduler's timeout worker. pub(crate) struct QueueTimeout { dev: AsahiDevRef, id: u64, @@ -251,7 +254,6 @@ impl QueueJob::ver { #[versions(AGX)] impl sched::JobImpl for QueueJob::ver { type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); fn timeout_data(&self) -> QueueTimeout { QueueTimeout { dev: self.dev.clone(), id: self.id } @@ -341,6 +343,10 @@ impl sched::JobImpl for QueueJob::ver { // are "invisible" to the firmware. Incrementing the notifier threshold // earlier than this violates that which leads to circles like the // above. Deferring the increment to submit solves the race. + job.notifier.threshold.with(|raw, _inner| { + raw.increase(job.notification_count); + }); + let gpu = match (*job.dev) .gpu()? .legacy_manager() @@ -392,13 +398,6 @@ impl sched::JobImpl for QueueJob::ver { } } - // Prepared submissions still hold their workqueue locks and have not - // published cpu_wptr. Only count them after every fallible submit() - // succeeded, before the first run_job exposes commands to firmware. - job.notifier.threshold.with(|raw, _inner| { - raw.increase(job.notification_count); - }); - // Now we fully commit to running the job mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); frag_sub.map(|a| gpu.run_job(a)).transpose()?; @@ -421,6 +420,8 @@ impl sched::JobImpl for QueueJob::ver { fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { // FIXME: Handle timeouts properly + // The timeout worker no longer sees the job itself, so it cannot tell whether run() + // completed. dev_err!( job.dev.as_ref(), "QueueJob {}: Job timed out on the DRM scheduler, things will probably break\n", @@ -727,11 +728,11 @@ impl Queue for Queue::ver { match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { - last_render = nr_commands; + last_compute = nr_commands; nr_render += 1; } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { - last_compute = nr_commands; + last_render = nr_commands; nr_compute += 1; } _ => {} diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index 639b9f54f46ac3..7b1bf698c054a4 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -11,7 +11,6 @@ //! coprocessors start, and enables dynamic power gating before the firmware is initialized. use crate::{hw, identity}; -use crate::hw; use core::mem::MaybeUninit; use kernel::{ bindings, c_str, @@ -26,7 +25,6 @@ use kernel::{ platform, prelude::*, sync::{aref::ARef, Arc}, // - sync::aref::ARef, // time::Delta, }; @@ -663,9 +661,43 @@ impl Resources { return Err(ENODEV); } - let (core_masks, total_active_cores) = - self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; - let (gpu_rev, gpu_rev_id) = self.gpu_revision(id_version)?; + let mut core_masks = KVec::new(); + let mut total_active_cores: u32 = 0; + + let max_core_mask = ((1u64 << num_cores) - 1) as u32; + for _ in 0..num_clusters { + let mask = core_mask_regs[0] & max_core_mask; + core_masks.push(mask, GFP_KERNEL)?; + for i in 0..core_mask_regs.len() { + core_mask_regs[i] >>= num_cores; + if i < (core_mask_regs.len() - 1) { + core_mask_regs[i] |= core_mask_regs[i + 1] << (32 - num_cores); + } + } + total_active_cores += mask.count_ones(); + } + + if core_mask_regs.iter().any(|a| *a != 0) { + dev_err!( + self.dev.as_ref(), + "Leftover core mask: {:#x?}\n", + core_mask_regs + ); + return Err(EIO); + } + + let (gpu_rev, gpu_rev_id) = match (id_version >> 8) & 0xff { + 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), + 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), + 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), + 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), + 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), + 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), + a => { + dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); + return Err(ENODEV); + } + }; // Preserve the public G13/G14 identity independently of the firmware // core identity: the legacy ABI distinguishes Max and Ultra by clusters. diff --git a/include/uapi/drm/asahi_drm.h b/include/uapi/drm/asahi_drm.h index 7e858f0be4ad4f..524e55d74849c9 100644 --- a/include/uapi/drm/asahi_drm.h +++ b/include/uapi/drm/asahi_drm.h @@ -94,7 +94,10 @@ enum drm_asahi_ioctl_id { /** @DRM_ASAHI_SUBMIT: Submit commands to a queue. */ DRM_ASAHI_SUBMIT, - /** @DRM_ASAHI_GEM_MADVISE: Mark a VM-private object purgeable or needed. */ + /** + * @DRM_ASAHI_GEM_MADVISE: Mark a VM-private object purgeable or needed. + * Only the G17P runtime accepts it; others fail with EINVAL. + */ DRM_ASAHI_GEM_MADVISE, }; @@ -208,7 +211,8 @@ struct drm_asahi_params_global { * of enum drm_asahi_gpu_hal_generation. * * G13-G15 on 14.x report DRM_ASAHI_GPU_HAL_LEGACY. - * The exact J613 25G83 runtime reports DRM_ASAHI_GPU_HAL_200. + * The exact J613 25G83 runtime and the G17P runtime report + * DRM_ASAHI_GPU_HAL_200. */ __u32 gpu_hal_generation; }; @@ -220,36 +224,13 @@ enum drm_asahi_gpu_hal_generation { /** @DRM_ASAHI_GPU_HAL_LEGACY: GPU predates the numbered HAL generations. */ DRM_ASAHI_GPU_HAL_LEGACY = 0, - /** @DRM_ASAHI_GPU_HAL_200: HAL200 command stream, reported by the J613 25G83 runtime. */ - DRM_ASAHI_GPU_HAL_200 = 200, - - /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream. Never reported by this kernel. */ - * @usc_generation: Instruction set generation of the unified shader - * cores: 2 for G13 and G14, 3 for G15 and newer. - */ - __u32 usc_generation; - /** - * @gpu_hal_generation: Command stream generation, from - * enum drm_asahi_gpu_hal_generation. + * @DRM_ASAHI_GPU_HAL_200: HAL200 command stream, reported by the J613 + * 25G83 runtime and by the G17P runtime. */ - __u32 gpu_hal_generation; -}; - -/** - * enum drm_asahi_gpu_hal_generation - GPU command stream generation - * - * The command stream generation determines the layout of the control streams - * and descriptors that userspace builds for the GPU. - */ -enum drm_asahi_gpu_hal_generation { - /** @DRM_ASAHI_GPU_HAL_LEGACY: Unnumbered generation of G13 to G15. */ - DRM_ASAHI_GPU_HAL_LEGACY = 0, - - /** @DRM_ASAHI_GPU_HAL_200: Generation 200, used by G17P. */ DRM_ASAHI_GPU_HAL_200 = 200, - /** @DRM_ASAHI_GPU_HAL_300: Generation 300. */ + /** @DRM_ASAHI_GPU_HAL_300: HAL300 command stream. Never reported by this kernel. */ DRM_ASAHI_GPU_HAL_300 = 300, }; @@ -291,8 +272,9 @@ enum drm_asahi_feature { /** * @DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES: GET_PARAMS group - * DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS is available. This kernel never - * sets this bit. + * DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS is available, and each queue + * reserves its own hardware execution state when it is created. Only + * the G17P runtime sets this bit. */ DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES = (1UL) << 2, @@ -308,7 +290,7 @@ enum drm_asahi_feature { DRM_ASAHI_FEATURE_SCHEDULED_QUEUES = (1UL) << 3, /* - * Bits 4, 5 and 7 are reported only by the HAL200 runtime. + * Bits 4, 5 and 7 are reported only by the J613 25G83 HAL200 runtime. * Bit 9 remains a diagnostic and is not advertised. */ @@ -318,66 +300,10 @@ enum drm_asahi_feature { DRM_ASAHI_FEATURE_FRAGMENT_DEPENDENCY = (1UL) << 6, DRM_ASAHI_FEATURE_COMPUTE_WIDE_VISIBILITY = (1UL) << 8, -}; - -#define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 -#define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 -/* GET_PARAMS group VM_STATUS is an in/out query on the calling file's VM. - * Set pointer to this structure and size to sizeof(structure). - * vm_id selects the VM; flags and pad must be zero. error is output only. - * error == 0 means no observed submission error; a negative Linux errno is - * sticky for this VM's lifetime. The first error is retained. Creating a new - * queue or successfully waiting another fence does not clear it. - * An execution-domain failure also marks other queried VMs lost when their - * hardware owner cannot progress. A query never recovers/resets firmware. - * The driver publishes an accepted job's failure before signaling its fence. - * Thus userspace must query after a successful DRM syncobj wait/poll before - * treating that fence as successful execution. Syncobj signal alone is not - * an execution-success indication. - * The feature bit is the availability contract; older kernels leave it zero. - * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or - * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. - */ -struct drm_asahi_vm_status { - __u32 vm_id; - __u32 flags; - __s32 error; - __u32 pad; -}; - -/** - * struct drm_asahi_queue_limits - GET_PARAMS group2 output, feature bit2. - * @max_queues: Maximum independent DRM queues on the physical device. This is - * a hardware resource bound, shared by all files/VMs; it is not a promise that - * all slots are currently free. QUEUE_CREATE reserves a queue's hardware state - * and execution context, returning an allocation error if resources run out. - * @max_in_flight_per_queue: Hardware scheduler credits per queue. Software - * may enqueue additional jobs, which wait behind preceding work/dependencies. - * @flags: Reserved, zero. - * @pad: Reserved, zero. - * - * The query is available only when DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES is - * set, which this kernel never does; otherwise it fails with EINVAL. A query - * does not allocate or reset anything. - */ -struct drm_asahi_queue_limits { - __u32 max_queues; - __u32 max_in_flight_per_queue; - __u32 flags; - __u32 pad; - /** - * @DRM_ASAHI_FEATURE_VM_STATUS: GET_PARAMS supports - * DRM_ASAHI_PARAM_GROUP_VM_STATUS. - */ - DRM_ASAHI_FEATURE_VM_STATUS = (1UL) << 1, - - /** - * @DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES: Each queue reserves its own - * hardware execution state when it is created, and GET_PARAMS - * supports DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS. + /* + * Bits 10 to 15 are reported only by the G17P runtime. */ - DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES = (1UL) << 2, /** * @DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS: Every hardware command of @@ -387,7 +313,7 @@ struct drm_asahi_queue_limits { * submissions and relies on this must pass its input syncs to each of * them. */ - DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS = (1UL) << 3, + DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS = (1UL) << 10, /** * @DRM_ASAHI_FEATURE_VM_STATUS_MIRROR: GET_PARAMS supports @@ -398,116 +324,88 @@ struct drm_asahi_queue_limits { * a successful wait a load of the word replaces the VM_STATUS query. * Registration may still be refused. */ - DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, + DRM_ASAHI_FEATURE_VM_STATUS_MIRROR = (1UL) << 11, /** * @DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS: Render and compute payloads * support exact queue-local historical prefixes in prior_vdm/prior_cdm. * These fields supplement the header barriers. */ - DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, + DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 12, /** * @DRM_ASAHI_FEATURE_FEW_PRIMITIVES: Render commands accept * DRM_ASAHI_RENDER_FEW_PRIMITIVES. */ - DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 6, + DRM_ASAHI_FEATURE_FEW_PRIMITIVES = (1UL) << 13, /** * @DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS: Render commands accept * DRM_ASAHI_RENDER_FRAGMENT_BARRIERS. */ - DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, + DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS = (1UL) << 14, + /** * @DRM_ASAHI_FEATURE_RENDER_RAY_QUERY: Render commands accept per-stage * ray query state (struct drm_asahi_render_ray_query_stage) and queues * accept render_scs_layout. */ - DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 8, + DRM_ASAHI_FEATURE_RENDER_RAY_QUERY = (1UL) << 15, }; -/** - * define DRM_ASAHI_PARAM_GROUP_VM_STATUS - GET_PARAMS group for - * struct drm_asahi_vm_status - */ #define DRM_ASAHI_PARAM_GROUP_VM_STATUS 1 - -/** - * define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS - GET_PARAMS group for - * struct drm_asahi_queue_limits - */ #define DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS 2 /** * define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR - GET_PARAMS group for - * struct drm_asahi_vm_status_mirror + * struct drm_asahi_vm_status_mirror, available with + * DRM_ASAHI_FEATURE_VM_STATUS_MIRROR */ #define DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR 3 -/** - * struct drm_asahi_vm_status - VM status, GET_PARAMS group - * DRM_ASAHI_PARAM_GROUP_VM_STATUS - * - * This is an in/out query on a VM of the calling file. Userspace sets - * &drm_asahi_get_params.pointer to this structure and - * &drm_asahi_get_params.size to its size. - * - * The kernel records the failure of an accepted job before it signals the - * job's fences, so userspace must query the status after a successful syncobj - * wait before it treats the work as successfully executed. A signalled - * syncobj alone does not indicate successful execution. An execution failure - * also marks other VMs whose work can no longer make progress. A query never - * resets or recovers the GPU. - * - * Invalid requests fail with EINVAL (non-zero @flags or @pad, wrong size), - * ENOENT (@vm_id unknown to the calling file) or EFAULT, and write nothing. +/* GET_PARAMS group VM_STATUS is an in/out query on the calling file's VM. + * Set pointer to this structure and size to sizeof(structure). + * vm_id selects the VM; flags and pad must be zero. error is output only. + * error == 0 means no observed submission error; a negative Linux errno is + * sticky for this VM's lifetime. The first error is retained. Creating a new + * queue or successfully waiting another fence does not clear it. + * An execution-domain failure also marks other queried VMs lost when their + * hardware owner cannot progress. A query never recovers/resets firmware. + * The driver publishes an accepted job's failure before signaling its fence. + * Thus userspace must query after a successful DRM syncobj wait/poll before + * treating that fence as successful execution. Syncobj signal alone is not + * an execution-success indication. + * The feature bit is the availability contract; older kernels leave it zero. + * Kernel errors for invalid requests: EINVAL flags/pad/size, ENOENT foreign or + * missing vm_id, EFAULT inaccessible pointer. No output on invalid request. */ struct drm_asahi_vm_status { - /** @vm_id: VM to query */ - __u32 vm_id; - - /** @flags: MBZ */ - __u32 flags; - - /** - * @error: Output: 0 if no submission error was observed, otherwise the - * first error as a negative errno. The error is sticky for the lifetime - * of the VM: neither creating queues nor waiting on other fences clears - * it. - */ - __s32 error; - - /** @pad: MBZ */ - __u32 pad; + __u32 vm_id; + __u32 flags; + __s32 error; + __u32 pad; }; /** - * struct drm_asahi_queue_limits - Queue limits, GET_PARAMS group - * DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS + * struct drm_asahi_queue_limits - GET_PARAMS group2 output, feature bit2. + * @max_queues: Maximum independent DRM queues on the physical device. This is + * a hardware resource bound, shared by all files/VMs; it is not a promise that + * all slots are currently free. QUEUE_CREATE reserves a queue's hardware state + * and execution context, returning an allocation error if resources run out. + * @max_in_flight_per_queue: Hardware scheduler credits per queue. Software + * may enqueue additional jobs, which wait behind preceding work/dependencies. + * @flags: Reserved, zero. + * @pad: Reserved, zero. * - * The query neither allocates nor resets anything. + * The query is available only when DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES is + * set, which only the G17P runtime does; otherwise it fails with EINVAL. A + * query does not allocate or reset anything. */ struct drm_asahi_queue_limits { - /** - * @max_queues: Maximum number of queues on the device, shared by all - * files and VMs. This is a hardware bound, not a promise that as many - * are currently available: QUEUE_CREATE reserves a queue's hardware - * state and fails once it runs out. - */ - __u32 max_queues; - - /** - * @max_in_flight_per_queue: Number of jobs of a queue that the - * hardware may run at once. Further jobs wait behind the preceding work - * and their dependencies. - */ - __u32 max_in_flight_per_queue; - - /** @flags: Reserved, zero */ - __u32 flags; - - /** @pad: Reserved, zero */ - __u32 pad; + __u32 max_queues; + __u32 max_in_flight_per_queue; + __u32 flags; + __u32 pad; }; /** @@ -544,7 +442,9 @@ struct drm_asahi_get_params { * drm_asahi_params_global, DRM_ASAHI_PARAM_GROUP_VM_STATUS when * DRM_ASAHI_FEATURE_VM_STATUS is set, or * DRM_ASAHI_PARAM_GROUP_QUEUE_LIMITS when - * DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES is set. Any other group, or a + * DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES is set, or + * DRM_ASAHI_PARAM_GROUP_VM_STATUS_MIRROR when + * DRM_ASAHI_FEATURE_VM_STATUS_MIRROR is set. Any other group, or a * group whose feature bit is clear, fails with EINVAL. */ __u32 param_group; @@ -553,8 +453,8 @@ struct drm_asahi_get_params { __u32 pad; /** - * @pointer: User pointer to the parameter struct. The VM status groups - * also read their input from it. + * @pointer: User pointer to parameter struct. The VM status groups also + * read their input from it. */ __u64 pointer; @@ -1175,7 +1075,7 @@ enum drm_asahi_render_flags { * @DRM_ASAHI_RENDER_RSRC_SPEC_HI: The appended resource-specifier high * dwords are present. A kernel that cannot use them rejects the flag * instead of silently truncating the values. This kernel accepts it on - * T6030 only. + * T6030 only. The G17P runtime also accepts it. */ DRM_ASAHI_RENDER_RSRC_SPEC_HI = (1U << 4), @@ -1191,16 +1091,9 @@ enum drm_asahi_render_flags { DRM_ASAHI_RENDER_FRAGMENT_DEPENDENCY = (1U << 5), /* - * Bits 6 and 7 are supported only by the HAL200 runtime. - * The legacy runtime rejects render commands that set them. - */ - * @DRM_ASAHI_RENDER_RSRC_SPEC_HI: The command carries the high halves - * of the background and end-of-tile resource specifiers - * (&drm_asahi_cmd_render.bg_rsrc_spec_hi and the following fields). - * Kernels that do not know the flag reject the command instead of - * silently truncating the specifiers. + * Bits 6 and 7 are supported only by the J613 25G83 HAL200 runtime. + * The legacy and G17P runtimes reject render commands that set them. */ - DRM_ASAHI_RENDER_RSRC_SPEC_HI = (1U << 4), /** * @DRM_ASAHI_RENDER_DBIAS_IS_INT: Use integer depth bias formula. @@ -1605,17 +1498,6 @@ struct drm_asahi_cmd_render { /** @ts_frag_end_copies: End timestamp copies, requiring the corresponding feature bit. */ struct drm_asahi_timestamp ts_frag_end_copies[8]; - /** @bg_eot_rsrc_spec_hi: High half of @eot.rsrc_spec */ - __u32 bg_eot_rsrc_spec_hi; - - /** @bg_eot_partial_rsrc_spec_hi: High half of @partial_eot.rsrc_spec */ - __u32 bg_eot_partial_rsrc_spec_hi; - - /** @bg_rsrc_spec_hi: High half of @bg.rsrc_spec */ - __u32 bg_rsrc_spec_hi; - - /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ - __u32 bg_partial_rsrc_spec_hi; /** * @prior_vdm: Historical render prefix, or zero for no added dependency. @@ -1680,20 +1562,7 @@ struct drm_asahi_cmd_compute { /** @ts_end_copies: End timestamp copies, requiring the corresponding feature bit. */ struct drm_asahi_timestamp ts_end_copies[8]; -}; -/** - * enum drm_asahi_compute_flags - Compute command flags - */ -enum drm_asahi_compute_flags { - /** - * @DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF: Reserved. This kernel rejects - * compute commands that set it. - */ - DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), - - DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), - /** @scs_layout: Reserved; must be zero. */ /** * @scs_layout: Packed G17P Gen1 shader-context scratch layout, or zero. * Bits 0:15 contain stack depth, 16:23 levels, 24:31 cliques, @@ -1737,6 +1606,19 @@ enum drm_asahi_compute_flags { __u64 intersection_default_control; }; +/** + * enum drm_asahi_compute_flags - Compute command flags + */ +enum drm_asahi_compute_flags { + /** + * @DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF: Reserved. This kernel rejects + * compute commands that set it. + */ + DRM_ASAHI_COMPUTE_G17P_ADD3_PROOF = (1L << 0), + + DRM_ASAHI_COMPUTE_WIDE_VISIBILITY = (1L << 1), +}; + /** * struct drm_asahi_get_time - Arguments passed to DRM_IOCTL_ASAHI_GET_TIME */ diff --git a/rust/kernel/drm/gem/shmem.rs b/rust/kernel/drm/gem/shmem.rs index a99997114e8053..01be8d1c032622 100644 --- a/rust/kernel/drm/gem/shmem.rs +++ b/rust/kernel/drm/gem/shmem.rs @@ -58,6 +58,7 @@ pub struct ObjectConfig<'a, T: DriverObject> { /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. pub reclaimable_cpu_mappings: bool, + /// Reclaim private backing pages before returning from the final free callback. /// The default retains the usual deferred file-release policy. pub synchronous_reclaim: bool, @@ -254,6 +255,9 @@ impl Object { obj <- Opaque::init_zeroed(), parent_resv_obj: None, sgt_access: AtomicUsize::new(0), + // Imports have no private shmem file; dumb buffers keep the + // default deferred file release. + synchronous_reclaim: false, inner <- T::new(dev, size, args), }), GFP_KERNEL, diff --git a/rust/kernel/drm/gpuvm.rs b/rust/kernel/drm/gpuvm.rs index f3562ccd17e1a8..7293df41f2dee1 100644 --- a/rust/kernel/drm/gpuvm.rs +++ b/rust/kernel/drm/gpuvm.rs @@ -363,30 +363,24 @@ unsafe impl AlwaysRefCounted for GpuVmBo { // must take the same lock as those operations. Keep a separate GEM // reference until after unlocking: the final vm_bo put can release // its GEM reference and otherwise free the mutex we still hold. - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: This owns a live BO reference. Immediate-mode GPUVA lists - // use the GEM's gpuva mutex, not its reservation. The deferred put - // releases that mutex before final GEM destruction. Keep the GPUVM - // alive across draining, which can drop its last BO-owned reference. unsafe { - let bo = &raw mut (*obj.as_ptr()).bo; - let vm = (*bo).vm; - if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE != 0 { + let vm = (*obj.as_ptr()).bo.vm; + // SAFETY: Every drm_gpuvm with Rust GpuVmBo objects is embedded in a GpuVm. + let gpuvm = crate::container_of!( + vm.cast::>(), GpuVm, gpuvm + ); + if (*gpuvm).deferred_bo_put { + // The VM was created by `GpuVm::new_deferred_bo_put`, which requires + // immediate mode. The deferred put releases the GEM's GPUVA mutex before + // final GEM destruction. Keep the GPUVM alive across draining, which can + // drop its last BO-owned reference. + let bo = &raw mut (*obj.as_ptr()).bo; bindings::drm_gpuvm_get(vm); bindings::drm_gpuvm_bo_put_deferred(bo); bindings::drm_gpuvm_bo_deferred_cleanup(vm); bindings::drm_gpuvm_put(vm); return; } - - // A final BO put also puts its GEM. Retain the GEM until after - // unlocking its reservation, which may be embedded in that GEM. - let gem = (*bo).obj; - bindings::drm_gem_object_get(gem); - let resv = (*gem).resv; - bindings::dma_resv_lock(resv, core::ptr::null_mut()); - bindings::drm_gpuvm_bo_put(bo); - bindings::dma_resv_unlock(resv); let bo = &mut obj.as_mut().bo; let gem = bo.obj; bindings::drm_gem_object_get(gem); @@ -418,6 +412,8 @@ pub struct GpuVm { gpuvm: Opaque, #[pin] inner: UnsafeCell, + /// Final VM/BO association puts are deferred (see [`GpuVm::new_deferred_bo_put`]). + deferred_bo_put: bool, #[pin] _p: PhantomPinned, } @@ -572,6 +568,47 @@ impl GpuVm { reserve_range: Range, inner: impl PinInit, ) -> Result>> + where + Error: From, + { + Self::new_inner(name, flags, dev, r_obj, range, reserve_range, inner, false) + } + + /// Create an immediate-mode GPUVM whose final VM/BO association puts are deferred. + /// + /// Dropping the last [`GpuVmBo`] reference puts the association with + /// `drm_gpuvm_bo_put_deferred()` and then drains the VM's deferred list, instead of + /// putting it synchronously under the GEM's GPUVA lock. The GEM GPUVA mutex is + /// released before final GEM destruction. + pub fn new_deferred_bo_put( + name: &'static CStr, + flags: bindings::drm_gpuvm_flags, + dev: &device::Device, + r_obj: ARef>, + range: Range, + reserve_range: Range, + inner: impl PinInit, + ) -> Result>> + where + Error: From, + { + if flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { + return Err(EINVAL); + } + Self::new_inner(name, flags, dev, r_obj, range, reserve_range, inner, true) + } + + #[allow(clippy::too_many_arguments)] + fn new_inner( + name: &'static CStr, + flags: bindings::drm_gpuvm_flags, + dev: &device::Device, + r_obj: ARef>, + range: Range, + reserve_range: Range, + inner: impl PinInit, + deferred_bo_put: bool, + ) -> Result>> where Error: From, { @@ -603,6 +640,7 @@ impl GpuVm { inner.__pinned_init(slot as *mut _) }) }, + deferred_bo_put, _p: PhantomPinned }), GFP_KERNEL, @@ -700,12 +738,6 @@ impl GpuVm { }), GFP_KERNEL, )?; - num_fences: 0, - }), - GFP_KERNEL, - )?, - obj, - }); // SAFETY: The object is valid and was initialized above to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *vm_exec) })?; @@ -713,61 +745,7 @@ impl GpuVm { Ok(LockedGpuVm { gpuvm: self, vm_exec, - objects: LockedObjects::Single(obj), - }) - } - - /// Lock the GPUVM and an array of additional GEM objects in one - /// wound/wait transaction. - pub fn exec_lock_array<'a, 'b>( - &'a self, - objects: &'b [ARef>], - interruptible: bool, - ) -> Result> { - let count = u32::try_from(objects.len()).map_err(|_| EINVAL)?; - let mut raw_objects = KVec::with_capacity(objects.len(), GFP_KERNEL)?; - for object in objects { - raw_objects.push(object.as_raw(), GFP_KERNEL)?; - } - - // vm_exec needs a stable address while locking. Construct the unlock - // guard only after success; C already finalizes the exec on error. - let mut vm_exec = KBox::init( - init!(bindings::drm_gpuvm_exec { - vm: self.gpuvm() as *mut _, - flags: (if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 - }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, - // SAFETY: both bindgen C structures are zero-valid; their generated - // `Default` methods perform the same zero initialization but are not - // exported from the kernel crate to loadable modules. - exec: unsafe { MaybeUninit::::zeroed().assume_init() }, - extra: unsafe { - MaybeUninit::::zeroed() - .assume_init() - }, - num_fences: 0, - }), - GFP_KERNEL, - )?; - - // SAFETY: Every raw pointer is backed by an ARef in `objects`, which - // outlives the returned guard. The helper consumes the temporary - // pointer array before returning and keeps its own object references. - to_result(unsafe { - bindings::drm_gpuvm_exec_lock_array( - &mut *vm_exec, - raw_objects.as_mut_ptr(), - count, - ) - })?; - - Ok(LockedGpuVm { - gpuvm: self, - vm_exec, - objects: LockedObjects::Array(objects), + obj, }) } @@ -852,28 +830,6 @@ unsafe impl AlwaysRefCounted for GpuVm { } } -enum LockedObjects<'a, T: DriverGpuVm> { - Single(Option<&'a Object>), - Array(&'a [ARef>]), -} - -impl LockedObjects<'_, T> { - fn single(&self) -> Option<&Object> { - match self { - Self::Single(object) => *object, - Self::Array(_) => None, - } - } - - fn indexed(&self, index: usize) -> Option<&Object> { - match self { - Self::Array(objects) => objects.get(index).map(|object| &**object), - Self::Single(_) => None, - } - } -} - -/// Access to VM driver data under its shared reservation, without external BO locks. pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { gpuvm: &'a GpuVm, // Reservation locks must be released by the task that acquired them. @@ -1238,55 +1194,7 @@ impl LockedGpuVm<'_, '_, T> { req_gem_range: u32, flags: GpuVaFlags, ) -> Result { - let obj = self.objects.single().ok_or(EINVAL)?.as_raw(); - self.sm_map_raw( - obj, - ctx, - req_addr, - req_range, - req_offset, - req_gem_range, - flags, - ) - } - - /// Map with an object acquired by [`GpuVm::exec_lock_array`]. - pub fn sm_map_indexed( - &mut self, - object_index: usize, - ctx: &mut T::StepContext, - req_addr: u64, - req_range: u64, - req_offset: u64, - req_gem_range: u32, - flags: GpuVaFlags, - ) -> Result { - let obj = self - .objects - .indexed(object_index) - .ok_or(EINVAL)? - .as_raw(); - self.sm_map_raw( - obj, - ctx, - req_addr, - req_range, - req_offset, - req_gem_range, - flags, - ) - } - - fn sm_map_raw( - &mut self, - obj: *mut bindings::drm_gem_object, - ctx: &mut T::StepContext, - req_addr: u64, - req_range: u64, - req_offset: u64, - req_gem_range: u32, - flags: GpuVaFlags, - ) -> Result { + let obj = self.obj.ok_or(EINVAL)?; let mut ctx = StepContext { ctx, gpuvm: self.gpuvm, diff --git a/rust/kernel/drm/sched.rs b/rust/kernel/drm/sched.rs index 965800645091ab..a76bf624237c83 100644 --- a/rust/kernel/drm/sched.rs +++ b/rust/kernel/drm/sched.rs @@ -55,21 +55,20 @@ pub trait JobImpl: Sized { /// through entity destruction; the core pins detached kill-job callbacks. const MODULE: Option<&'static crate::ThisModule> = None; - /// Handle a scoped timeout without resetting the C scheduler. + /// Opt in to synchronized cancellation and teardown. /// - /// The backend must settle/quarantine its exact hardware work, or retain - /// that owner until an in-progress host transaction can classify it. It - /// must not invoke drm_sched_stop/start or replay scheduler jobs. The core - /// reinserts the detached guilty job through NO_HANG, preserving every - /// parent callback and the backend's actual success/error fence result. - const MANAGED_TIMEOUT_RECOVERY: bool = false; + /// When set, scheduler teardown calls `drm_sched_fini()` without a preceding + /// `drm_sched_stop()`, and the cancel callback removes the job's parent fence + /// callback under the parent's lock before it settles the scheduler's credit and + /// score accounting exactly once. Backends that leave this unset keep the + /// stop-then-fini teardown and the unconditional cancel signal. + const SYNCHRONIZED_CANCEL: bool = false; /// Reconsider a timeout before stopping the scheduler or settling fences. /// Returning true asks the core to reinsert the retained job and rearm. /// The backend must not stop/start the scheduler from this hook. fn false_timeout(_data: &mut Self::TimeoutData, _finished: bool) -> bool { false } - /// Called when the scheduler is considering scheduling this job next, to get another Fence /// for this job to block on. Once it returns None, run() may be called. fn prepare(_job: &mut Self) -> Option { @@ -83,7 +82,8 @@ pub trait JobImpl: Sized { /// Called when a job has taken too long to execute, to trigger GPU recovery. /// - /// This method is called in a workqueue context. + /// This method is called in a workqueue context. `finished` reports whether the job's + /// scheduler fence had already signalled. fn timed_out(data: &mut Self::TimeoutData, finished: bool) -> Status; /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences @@ -129,30 +129,17 @@ unsafe extern "C" fn timedout_job_cb( // The C timeout worker owns this job outside pending_list until return. // Free work cannot release it even if its parent signals concurrently. + // SAFETY: An armed job has a valid scheduler fence for its lifetime. let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; + // SAFETY: `finished` stays valid while the timeout worker owns the job. let is_finished = || unsafe { bindings::dma_fence_get_status(finished) != 0 }; - if T::MANAGED_TIMEOUT_RECOVERY { - if is_finished() { - return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; - } - } // SAFETY: the C core retains the detached job and serializes its timeout // worker. This field is disjoint from inner, which run may still mutate. let data = unsafe { &mut *addr_of_mut!((*p).timeout_data) }; if T::false_timeout(data, is_finished()) { return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; } - let status = T::timed_out(data, is_finished()); - if T::MANAGED_TIMEOUT_RECOVERY { - // This is scoped fence handling, not C reset recovery. stop would - // detach and discard healthy parents; start would cancel those jobs. - // NO_HANG reinserts the exact job and queues free work if it completed - // during the callback. It does not change the parent's error or replay - // work. A deferred exact owner remains eligible for its real callback. - bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG - } else { - status as bindings::drm_gpu_sched_stat - } + T::timed_out(data, is_finished()) as bindings::drm_gpu_sched_stat } unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { @@ -168,11 +155,24 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch // SAFETY: All of our jobs are Job. let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; + if !T::SYNCHRONIZED_CANCEL { + // SAFETY: All of our jobs are Job; fini has stopped the scheduler workers. + T::cancel(unsafe { &mut *addr_of_mut!((*p).inner) }); + + // SAFETY: s_fence is valid for an armed job. + let fence = unsafe { Fence::get_raw(&mut (*(*sched_job).s_fence).finished) }; + fence.set_error(ECANCELED); + let _ = fence.signal(); + return; + } + // fini has stopped/synchronized submission and timeout workers. A parent // callback can still run on the device completion thread. The scheduler // fence retains the parent reference while we remove its callback under // the parent's lock; that also waits for an already-running callback. + // SAFETY: s_fence is valid for an armed job. let s_fence = unsafe { (*sched_job).s_fence }; + // SAFETY: s_fence is valid; parent is read once under fini's exclusion. let parent = unsafe { (*s_fence).parent }; let removed_parent_callback = if !parent.is_null() { // SAFETY: run_job installed (or attempted to install on an already @@ -180,6 +180,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) } } else { false }; let finish_here = removed_parent_callback || (parent.is_null() + // SAFETY: the finished fence is valid for an armed job. && unsafe { bindings::dma_fence_get_status(addr_of_mut!((*s_fence).finished)) == 0 }); // Null/error run_job calls job_done before submit work returns, so its // finished fence is already signaled. NULL+unfinished can instead survive @@ -195,7 +196,9 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch // Match drm_sched_job_done's accounting exactly once, only when its // callback did not do it. Submission/free work is stopped, so fini // itself unlinks and frees this job; no workqueue rearm is appropriate. + // SAFETY: the job's scheduler outlives fini's cancellation of its jobs. let sched = unsafe { (*sched_job).sched }; + // SAFETY: credit_count and score belong to the live scheduler. unsafe { if removed_parent_callback { bindings::atomic_sub((*sched_job).credits as i32, @@ -203,6 +206,7 @@ unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sch } bindings::atomic_sub(1, (*sched).score); } + // SAFETY: the finished fence is valid for an armed job. let fence = unsafe { Fence::get_raw(addr_of_mut!((*s_fence).finished)) }; fence.set_error(ECANCELED); let _ = fence.signal(); @@ -471,14 +475,18 @@ struct SchedulerOwner { impl Drop for SchedulerOwner { fn drop(&mut self) { + let sched = addr_of!(self.allocation.sched).cast_mut(); + if !T::SYNCHRONIZED_CANCEL { + // SAFETY: The scheduler is valid. This assumes drm_sched_fini() will take care of + // freeing all in-progress jobs. + unsafe { bindings::drm_sched_stop(sched, core::ptr::null_mut()) }; + } // SAFETY: fini stops submission, synchronizes the timeout worker and - // cancels/frees remaining jobs through our synchronized cancel callback. - // The C contract explicitly prohibits drm_sched_stop before fini. + // cancels/frees remaining jobs through the cancel callback. With + // SYNCHRONIZED_CANCEL, the C contract prohibits drm_sched_stop before fini. // Deferred entity-kill callbacks are on a different workqueue and may // survive this call. Their job guards retain allocation through cleanup. - unsafe { - bindings::drm_sched_fini(addr_of!(self.allocation.sched).cast_mut()) - }; + unsafe { bindings::drm_sched_fini(sched) }; } } @@ -501,7 +509,6 @@ impl Scheduler { Some(module) => module.as_ptr(), None => core::ptr::null_mut(), }, - owner: core::ptr::null_mut(), retain_job_on_enodev: false, }; /// Creates a new DRM Scheduler object diff --git a/tools/asahi/neo-auxiliary-tests.py b/tools/asahi/neo-auxiliary-tests.py index 276e4960e7490e..899fd9d8a78dc2 100644 --- a/tools/asahi/neo-auxiliary-tests.py +++ b/tools/asahi/neo-auxiliary-tests.py @@ -9,7 +9,7 @@ import subprocess ROOT = Path(__file__).resolve().parents[2] -PATH = 'drivers/gpu/drm/asahi-neo/g17/queue/render/memory.rs' +PATH = 'drivers/gpu/drm/asahi/g17/queue/render/memory.rs' PREFIX = r''' #![allow(dead_code)] use std::ops::{Deref,Range}; @@ -82,7 +82,7 @@ def main(): structure = item(source, 'pub(super) struct Generation').replace('pub(super)', '') constructor = item(source[source.index('impl Generation'):], ' fn new(') addresses = item(source, ' pub(super) fn addresses').replace('pub(super)', '') - storage=(ROOT/'drivers/gpu/drm/asahi-neo/g17/fw/render/storage.rs').read_text() + storage=(ROOT/'drivers/gpu/drm/asahi/g17/fw/render/storage.rs').read_text() constants=[] for name in ['STATE_SIZE','DEFLAKE','TA_STATUS','AUXILIARY','AUXILIARY_SIZE']: line=next(x for x in storage.splitlines() if 'const '+name+':' in x) From 8be1719a3e4c52bcb943397e764a811a461d46c1 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:01 -0400 Subject: [PATCH 549/631] drm/apple: serve M1/M2/M3, T8140 and T8142 from one driver Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. Integration's driver is the base, including its Type-C fabric, mode catalog and M3 display work. The H17-generation display support from aurora-wip sits behind the firmware the DCP runs: the T8140 internal and external DCPs and the T8142 DCP. It covers the external display lifecycle over AFKV2, DCPLink, the H17P iomfb template, the parser additions and the DPTX service. The H17P-only files are aurora-wip's. The driver binds the T8140 and T8142 display subsystems and DCPs again. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/gpu/drm/apple/Kconfig | 21 +- drivers/gpu/drm/apple/Makefile | 4 +- drivers/gpu/drm/apple/afk.c | 206 +- drivers/gpu/drm/apple/apple_drv.c | 75 +- drivers/gpu/drm/apple/audio.c | 15 +- drivers/gpu/drm/apple/dcp-fabric.c | 210 +- drivers/gpu/drm/apple/dcp-fabric.h | 6 + drivers/gpu/drm/apple/dcp-internal.h | 48 +- drivers/gpu/drm/apple/dcp-link.h | 107 +- drivers/gpu/drm/apple/dcp.c | 8037 +++++++++---------- drivers/gpu/drm/apple/dcp.h | 5 +- drivers/gpu/drm/apple/dcp_backlight.c | 72 +- drivers/gpu/drm/apple/dptxep.c | 133 +- drivers/gpu/drm/apple/epic/dpavservep.c | 213 +- drivers/gpu/drm/apple/external-hold-test.c | 2 +- drivers/gpu/drm/apple/ibootep.c | 18 +- drivers/gpu/drm/apple/iomfb.c | 569 +- drivers/gpu/drm/apple/iomfb.h | 3 + drivers/gpu/drm/apple/iomfb_internal.h | 3 + drivers/gpu/drm/apple/iomfb_template.c | 1118 +-- drivers/gpu/drm/apple/iomfb_template.h | 15 +- drivers/gpu/drm/apple/iomfb_template_h17p.c | 2723 +++++++ drivers/gpu/drm/apple/iomfb_v14_7.c | 2 +- drivers/gpu/drm/apple/iomfb_v14_7_tpl.c | 1 + drivers/gpu/drm/apple/iomfb_v14_7_tpl.h | 12 + drivers/gpu/drm/apple/iomfb_v26_6.c | 2 +- drivers/gpu/drm/apple/parser-test.c | 2 +- drivers/gpu/drm/apple/parser.c | 262 +- drivers/gpu/drm/apple/parser.h | 3 +- drivers/gpu/drm/apple/plane.c | 29 + 30 files changed, 7828 insertions(+), 6088 deletions(-) create mode 100644 drivers/gpu/drm/apple/iomfb_template_h17p.c diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index ba374848bbffaf..32173fef7127f3 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -35,8 +35,9 @@ config DRM_APPLE_DEBUG depends on EXPERT # only for developers config DRM_APPLE_KUNIT_TEST - tristate "KUnit tests for Apple display state" if !KUNIT_ALL_TESTS + tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS depends on KUNIT + select DRM_KUNIT_TEST_HELPERS if DRM_APPLE default KUNIT_ALL_TESTS help Test modeset and hotplug state transitions without display hardware, @@ -44,6 +45,11 @@ config DRM_APPLE_KUNIT_TEST failed modesets. These tests can run on other architectures and do not require the Apple display driver to be enabled. Also test service ownership and pending commands across disconnect and reannouncement. + With the Apple display driver enabled, also build tests for its AFK + ring parser (header strides used by older firmware and larger rings, + malformed ring size combinations), the H17P backlight policy + (takeover, DPMS, suspend, fades and present completion) and the + system resume of a CRTC whose mode is no longer admissible. config DRM_APPLE_DISPLAY_KUNIT_TEST bool "KUnit tests for Apple display mode probing" if !KUNIT_ALL_TESTS @@ -59,16 +65,3 @@ config DRM_APPLE_DISPLAY_KUNIT_TEST status updates before hotplug notifications and same-state retrains. Check which 14.x board record an internal display gets on which machine, and which DCP firmware image each record admits. - tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS - depends on KUNIT - select DRM_KUNIT_TEST_HELPERS if DRM_APPLE - default KUNIT_ALL_TESTS - help - Build tests for the Apple display driver's AFK ring parser and - backlight policy (takeover, DPMS, suspend, fades and present - completion). - The tests cover the header strides used by older firmware and - larger rings, as well as malformed ring size combinations. - This option is intended for developer test kernels. - Also test the Type-C modeset, hotplug and service ownership state, - and the system resume of a CRTC whose mode is no longer admissible. diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index 4f71f3ea0256f3..dfd6fde66c387f 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -3,8 +3,6 @@ CFLAGS_trace.o = -I$(src) appledrm-y := apple_drv.o apple_resume.o -appledrm-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o -appledrm-y := apple_drv.o appledrm-y += afk.o dcp.o dcp-fabric.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o appledrm-y += dcp-hdmi.o appledrm-$(CONFIG_DRM_APPLE_AUDIO) += audio.o @@ -35,5 +33,7 @@ appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += connector-edid-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += hotplug-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += external-hold-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += v14-board-test.o + +# afk-service-test.o is built from drivers/gpu/drm/Makefile. obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index e85c36d1b87704..5daa26990a02bb 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -93,10 +93,19 @@ static void afk_release_context(void *data) struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, const struct apple_epic_service_ops *ops) { + bool h17p = dcp->fw_compat == DCP_FIRMWARE_H17P; struct apple_dcp_afkep *afkep; int ret, i; - afkep = kzalloc_obj(*afkep); + /* + * H17P: an endpoint of a retained session outlives the binding, so it + * is freed by its own release action. Other firmware keeps the + * device-managed lifetime it always had. + */ + if (h17p) + afkep = kzalloc_obj(*afkep); + else + afkep = devm_kzalloc(dcp->dev, sizeof(*afkep), GFP_KERNEL); if (!afkep) return ERR_PTR(-ENOMEM); @@ -116,14 +125,19 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, spin_lock_init(&afkep->lock); for (i = 0; i < AFK_MAX_CHANNEL; i++) spin_lock_init(&afkep->services[i].lock); - ret = devm_add_action_or_reset(dcp->dev, afk_release_context, afkep); - if (ret) - return ERR_PTR(ret); + if (h17p) { + ret = devm_add_action_or_reset(dcp->dev, afk_release_context, afkep); + if (ret) + return ERR_PTR(ret); + } return afkep; out_free_afkep: - kfree(afkep); + if (h17p) + kfree(afkep); + else + devm_kfree(dcp->dev, afkep); return ERR_PTR(ret); } @@ -163,11 +177,10 @@ void afk_shutdown(struct apple_dcp_afkep *afkep) spin_lock_irqsave(&afkep->lock, flags); afkep->stopping = true; spin_unlock_irqrestore(&afkep->lock, flags); - afk_quiesce(afkep); - spin_lock_irqsave(&afkep->lock, flags); - afkep->stopping = true; - spin_unlock_irqrestore(&afkep->lock, flags); - destroy_workqueue(afkep->wq); + if (afkep->dcp->fw_compat == DCP_FIRMWARE_H17P) + afk_quiesce(afkep); + else + destroy_workqueue(afkep->wq); if (afkep->rx_scratch) { devm_kfree(afkep->dcp->dev, afkep->rx_scratch); afkep->rx_scratch = NULL; @@ -208,10 +221,6 @@ static int afk_start_polled(struct apple_dcp_afkep *ep) "RTKit AFK endpoint %02x did not acknowledge init\n", ep->endpoint); return -ETIMEDOUT; - if (afkep->rx_scratch) { - devm_kfree(afkep->dcp->dev, afkep->rx_scratch); - afkep->rx_scratch = NULL; - } } int afk_start(struct apple_dcp_afkep *ep) @@ -801,13 +810,17 @@ static void afk_recv_handle_std_service(struct apple_dcp_afkep *ep, u32 channel, command = le32_to_cpu(call->call.command); /* - * Command numbers are per group: a group-less .call handler - * would take another group's call for a group 0 one. Without - * call_v2 there is no handler for it, and AFK cannot guess a - * reply, so it gets none, as a call without any handler. + * H17P firmware: command numbers are per group, so a + * group-less .call handler would take another group's call + * for a group 0 one. Without call_v2 there is no handler for + * it, and AFK cannot guess a reply, so it gets none, as a call + * without any handler. Other firmware hands every group's + * command to .call, as it always has. */ - if (!service->ops->call_v2 && (group || !service->ops->call)) { - if (group) + if (!service->ops->call_v2 && + (!service->ops->call || + (group && ep->dcp->fw_compat == DCP_FIRMWARE_H17P))) { + if (group && ep->dcp->fw_compat == DCP_FIRMWARE_H17P) dev_warn_ratelimited(ep->dcp->dev, "AFK[ep:%02x]: %s (chan:%u) apcall %u:%u has no handler, sending NO reply\n", ep->endpoint, service->ops->name, @@ -1135,8 +1148,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, size_t hdr_len = sizeof(*ehdr) + sizeof(*eshdr); u16 subtype; u8 *payload; - u16 subtype; - u8 *payload = data + sizeof(*ehdr) + sizeof(*eshdr); size_t payload_size; if (ep->wire_v2) { @@ -1190,8 +1201,6 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, subtype = le16_to_cpu(eshdr->type); payload = data + hdr_len; payload_size = data_size - hdr_len; - payload_size = data_size - sizeof(*ehdr) - sizeof(*eshdr); - subtype = le16_to_cpu(eshdr->type); trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -1345,6 +1354,9 @@ static bool afk_recv(struct apple_dcp_afkep *ep) rptr = afk_ring_advance(rptr, sizeof(*hdr) + size, ep->rxbfr.bufsz, ep->rxbfr.stride); + if (ep->dcp->fw_compat != DCP_FIRMWARE_H17P && + WARN_ON(rptr > ep->rxbfr.bufsz)) + rptr = 0; dma_mb(); @@ -1667,7 +1679,6 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, /* Publish the completed entry before allowing the consumer to read it. */ dma_wmb(); - ep->txbfr.hdr->wptr = cpu_to_le32(wptr); *ep->txbfr.wptr = cpu_to_le32(wptr); afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | FIELD_PREP(SEND_WPTR, wptr)); @@ -1686,25 +1697,6 @@ int afk_send_epic(struct apple_dcp_afkep *ep, u32 channel, u16 tag, payload, payload_len, 0); } -int afk_send_command(struct apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, void *output, - size_t output_len, u32 *retcode) -/* - * Send one EPIC command and wait up to @timeout_ms for its reply. With - * @reply_len set, only the bytes the firmware reported are copied to @output - * (the rest is zeroed) and *@reply_len receives that count; without it the - * whole receive buffer is copied, as the original callers expect. - */ -static int afk_send_command_full(struct apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, - void *output, size_t output_len, u32 *retcode, - size_t *reply_len, unsigned int timeout_ms) -{ - return afk_send_command_timeout(service, type, payload, payload_len, - output, output_len, retcode, - MSEC_PER_SEC); -} - static u16 afk_command_tag(struct apple_epic_service *service, u8 slot) { struct apple_dcp_afkep *ep = service->ep; @@ -1722,15 +1714,21 @@ static u16 afk_command_tag(struct apple_epic_service *service, u8 slot) return tag; } -static int afk_send_command_timeout_len(struct apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, - void *output, size_t output_len, u32 *retcode, - unsigned int timeout_ms, size_t *received); - -int afk_send_command_timeout(struct apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, - void *output, size_t output_len, u32 *retcode, - unsigned int timeout_ms) +/* + * Send one EPIC command and wait up to @timeout_ms for its reply. With + * @reply_len set, only the bytes the firmware reported are copied to @output + * (the rest is zeroed) and *@reply_len receives that count; without it the + * whole receive buffer is copied, as the original callers expect. + * + * AFKV2 endpoints (@ep->wire_v2, external H17P) carry the reply length in + * the reply itself: with @reply_len set, only that many bytes are copied, + * a short successful reply fails with -EMSGSIZE, and nothing is copied for + * a nonzero status. + */ +static int afk_send_command_full(struct apple_epic_service *service, u8 type, + const void *payload, size_t payload_len, + void *output, size_t output_len, u32 *retcode, + size_t *reply_len, unsigned int timeout_ms) { struct epic_cmd cmd; void *inline_command = NULL; @@ -1748,11 +1746,6 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, if (payload_len > U32_MAX || output_len > U32_MAX) return -EOVERFLOW; - service = afk_service_get(service); - if (!service) - return -ENODEV; - ep = service->ep; - if (READ_ONCE(ep->dcp->quiescing)) service = afk_service_get(service); if (!service) return -ENODEV; @@ -1872,7 +1865,7 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, "AFK[ep:%02x]: %s (chan:%u) command type 0x%x tag 0x%04x timed out after %u ms\n", ep->endpoint, service->ops->name, service->channel, type, tag, timeout_ms); - if (ep->dcp->external) { + if (ep->dcp->external && ep->txbfr.ready && ep->rxbfr.ready) { /* * Shared ring state, not coprocessor MMIO: unequal * TX pointers mean the firmware has not drained @@ -1882,10 +1875,10 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, dev_warn(ep->dcp->dev, "AFK[ep:%02x]: TX r/w %#x/%#x RX r/w %#x/%#x running %d crashed %d\n", ep->endpoint, - le32_to_cpu(READ_ONCE(ep->txbfr.hdr->rptr)), - le32_to_cpu(READ_ONCE(ep->txbfr.hdr->wptr)), - le32_to_cpu(READ_ONCE(ep->rxbfr.hdr->rptr)), - le32_to_cpu(READ_ONCE(ep->rxbfr.hdr->wptr)), + le32_to_cpu(READ_ONCE(*ep->txbfr.rptr)), + le32_to_cpu(READ_ONCE(*ep->txbfr.wptr)), + le32_to_cpu(READ_ONCE(*ep->rxbfr.rptr)), + le32_to_cpu(READ_ONCE(*ep->rxbfr.wptr)), apple_rtkit_is_running(ep->dcp->rtk), apple_rtkit_is_crashed(ep->dcp->rtk)); } @@ -1898,7 +1891,19 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, ret = 0; if (retcode) *retcode = service->cmds[idx].retcode; - if (reply_len) { + if (ep->wire_v2) { + if (reply_len) + copy_len = service->cmds[idx].reply_length; + if (!service->cmds[idx].retcode && + service->cmds[idx].reply_length < copy_len) { + ret = -EMSGSIZE; + } else if (output && output_len && !service->cmds[idx].retcode) { + dma_rmb(); + memcpy(output, rxbuf, copy_len); + if (reply_len) + *reply_len = copy_len; + } + } else if (reply_len) { ret = afk_reply_size_valid(service->cmds[idx].reply_len_valid, service->cmds[idx].reply_len, output_len, 0); @@ -1911,12 +1916,6 @@ int afk_send_command_timeout(struct apple_epic_service *service, u8 type, memcpy(output, rxbuf, *reply_len); } } else if (output && output_len) { - if (ep->wire_v2 && !service->cmds[idx].retcode && - service->cmds[idx].reply_length < copy_len) - ret = -EMSGSIZE; - else if (output && output_len && (!ep->wire_v2 || !service->cmds[idx].retcode)) { - if (ep->wire_v2) - dma_rmb(); memcpy(output, rxbuf, output_len); } @@ -1966,41 +1965,21 @@ int afk_send_command_with_reply_len(struct apple_epic_service *service, u8 type, MSEC_PER_SEC); } +/* + * One EPIC standard-service call. With @reply_len set, the reply body + * length is validated and returned: against the received bytes on the + * legacy wire, with afkv2_service_reply_size() on AFKV2 endpoints. + */ static int afk_service_call_full(struct apple_epic_service *service, u16 group, u32 command, const void *data, size_t data_len, size_t data_pad, void *output, size_t output_len, size_t output_pad, size_t *reply_len, unsigned int timeout_ms) -{ - return afk_service_call_timeout(service, group, command, data, data_len, - data_pad, output, output_len, - output_pad, MSEC_PER_SEC); -} - -static int afk_service_call_timeout_len(struct apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms, - size_t *received); - -int afk_service_call_len(struct apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, size_t *received) -{ - return afk_service_call_timeout_len(service, group, command, data, data_len, - data_pad, output, output_len, output_pad, - MSEC_PER_SEC, received); -} - -int afk_service_call_timeout(struct apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms) { struct epic_service_call *call; void *bfr; - size_t bfr_len, input_size, output_size, received = 0; + size_t bfr_len, input_size, output_size, received = 0, result_size; + bool v2; int ret; u32 retcode; u32 retlen; @@ -2009,13 +1988,12 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, *reply_len = 0; if (!service) return -ENODEV; + v2 = service->ep->wire_v2; if (check_add_overflow(data_len, data_pad, &input_size) || check_add_overflow(output_len, output_pad, &output_size) || check_add_overflow(max(input_size, output_size), sizeof(*call), &bfr_len) || bfr_len > U32_MAX) return -EOVERFLOW; - if (!service) - return -ENODEV; bfr = kzalloc(bfr_len, GFP_KERNEL); if (!bfr) @@ -2033,11 +2011,8 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, memcpy(bfr + sizeof(*call), data, data_len); /* Inline carries only the request; its response capacity is independent. */ - ret = afk_send_command_timeout_len(service, EPIC_SUBTYPE_STD_SERVICE, bfr, - service->ep->inline_v2 ? sizeof(*call) + data_len : - bfr_len, bfr, bfr_len, &retcode, timeout_ms, - received ? &wire_received : NULL); ret = afk_send_command_full(service, EPIC_SUBTYPE_STD_SERVICE, bfr, + service->ep->inline_v2 ? sizeof(*call) + data_len : bfr_len, bfr, bfr_len, &retcode, reply_len ? &received : NULL, timeout_ms); if (ret) { @@ -2056,16 +2031,16 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, ret = -EINVAL; goto out; } - if (reply_len) { - ret = afk_reply_size_valid(true, received, bfr_len, sizeof(*call)); + if (reply_len && v2) { + ret = afkv2_service_reply_size(bfr, received, group, command, + output_len, &result_size); if (ret) goto out; - if (received) { - ret = afkv2_service_reply_size(bfr, wire_received, group, command, - output_len, &result_size); + *reply_len = result_size; + } else if (reply_len) { + ret = afk_reply_size_valid(true, received, bfr_len, sizeof(*call)); if (ret) goto out; - *received = result_size; } if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || le16_to_cpu(call->group) != group || @@ -2085,7 +2060,7 @@ int afk_service_call_timeout(struct apple_epic_service *service, u16 group, group, command, le32_to_cpu(call->data_len)); retlen = le32_to_cpu(call->data_len); - if (reply_len) { + if (reply_len && !v2) { ret = afk_service_body_size_valid(received, sizeof(*call), retlen, output_len); if (ret) @@ -2135,6 +2110,17 @@ int afk_service_call_with_reply_len(struct apple_epic_service *service, reply_len, MSEC_PER_SEC); } +/* The AFKV2 (external H17P) name of afk_service_call_with_reply_len(). */ +int afk_service_call_len(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t data_len, + size_t data_pad, void *output, size_t output_len, + size_t output_pad, size_t *received) +{ + return afk_service_call_with_reply_len(service, group, command, data, + data_len, data_pad, output, + output_len, output_pad, received); +} + #if IS_ENABLED(CONFIG_DRM_APPLE_DEBUG) #define AFK_DEBUGFS_MAX_REPLY 8192 diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index 3d76e6fdf8734b..50b19346d641c4 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -89,15 +89,6 @@ static int apple_drm_fbdev_probe(struct drm_fb_helper *helper, } return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); -/* - * A compositor or boot splash is closing the device. drm_release() has - * already taken its file off the file list, so the Type-C fabric can tell - * whether any other one still has the device open. - */ -static void apple_drm_postclose(struct drm_device *drm, struct drm_file *file) -{ - if (file->was_master) - dcp_typec_reorder(); } #endif @@ -119,7 +110,6 @@ static const struct drm_driver apple_drm_driver = { #else DRM_FBDEV_DMA_DRIVER_OPS, #endif - DRM_FBDEV_DMA_DRIVER_OPS, .postclose = apple_drm_postclose, .name = DRIVER_NAME, .desc = DRIVER_DESC, @@ -183,7 +173,8 @@ static void apple_crtc_atomic_enable(struct drm_crtc *crtc, } if (crtc_state->active && !dcp_crtc_route_ready(crtc, state, true)) goto failed; - if (crtc_state->active && !dcp->main_display && !apple_crtc->fabric_held) { + if (crtc_state->active && !dcp->main_display && !apple_crtc->fabric_held && + dcp->fw_compat != DCP_FIRMWARE_H17P) { /* Raise the fabric before external scanout starts. */ if (apple_pmgr_fabric_hold()) dev_warn(dcp->dev, "fabric raise not confirmed before scanout\n"); @@ -444,10 +435,18 @@ apple_connector_atomic_best_encoder(struct drm_connector *conn, * the fabric routes within: on dual-stream machines, and where the * route follows the CRTC (see dcp_typec_follow_crtc()). So it is the * only answer there is. + * + * H17P (T8140, T8142): Type-C tunnels are routed before the modeset + * and cannot move here. Reject stale topology and explicit assignments + * to another pipeline. */ if (apple_connector->port_encoder) { + struct apple_dcp *target; + conn_state = drm_atomic_get_new_connector_state(state, conn); - if (!conn_state || !conn_state->crtc || + target = conn_state && conn_state->crtc ? + platform_get_drvdata(to_apple_crtc(conn_state->crtc)->dcp) : NULL; + if (target && target->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(apple_connector->dcp) != to_apple_crtc(conn_state->crtc)->dcp) return NULL; @@ -547,6 +546,12 @@ static int apple_probe_per_dcp(struct device *dev, bool supports_xrgb2101010; enum drm_plane_type plane_type; + supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || + dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; + if (dcp_data->fw_compat == DCP_FIRMWARE_H17P && + bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) + return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); + if (dcp_fw_compat_is_14_7(dcp)) { planes[0] = apple_plane_init_v14_7(drm, 1U << num, 0, DRM_PLANE_TYPE_PRIMARY); @@ -557,10 +562,6 @@ static int apple_probe_per_dcp(struct device *dev, return ret; zpos = 1; } - supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || - dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; - if (bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) - return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { if (dcp_fw_compat_is_14_7(dcp)) @@ -808,10 +809,22 @@ static void apple_drm_quiesce_connectors(struct drm_device *drm) disable_work_sync(&apple_connector->bl_commit_wq); } drm_connector_list_iter_end(&iter); - * The T8140 external display coprocessor needs firmware support this driver - * does not have yet. Leave it out of the display subsystem, so that an - * enabled node cannot hold back the internal panel's component master. - */ +} + +/* Does any pipeline of @drm run H17-generation firmware (T8140, T8142)? */ +static bool apple_drm_has_h17p(struct drm_device *drm) +{ + struct drm_crtc *crtc; + + drm_for_each_crtc(crtc, drm) { + struct apple_dcp *dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + if (dcp && dcp->fw_compat == DCP_FIRMWARE_H17P) + return true; + } + return false; +} + /* A supported firmware-described external provider participates in component matching. */ static bool apple_dcp_usable(struct device_node *np) { @@ -832,10 +845,7 @@ static int apple_drm_init_dcp(struct device *dev) for_each_matching_node(np, apple_dcp_id_tbl) { bool dcp_ext; - if (!apple_dcp_in_drm(np)) - continue; - if (!of_device_is_available(np)) { - of_node_put(np); + if (!apple_dcp_in_drm(np) || !apple_dcp_usable(np)) continue; dcp_ext = of_device_is_compatible(np, "apple,dcpext") || of_device_is_compatible(np, "apple,t8140-dcpext") || @@ -1008,7 +1018,11 @@ static void apple_drm_uninit(struct device *dev) struct apple_drm_private *apple = dev_get_drvdata(dev); apple_drm_quiesce_connectors(&apple->drm); - drm_dev_unregister(&apple->drm); + /* H17P keeps its live scanout across unbind: unplug, as aurora-wip. */ + if (apple_drm_has_h17p(&apple->drm)) + drm_dev_unplug(&apple->drm); + else + drm_dev_unregister(&apple->drm); drm_atomic_helper_shutdown(&apple->drm); component_unbind_all(dev, NULL); @@ -1039,7 +1053,7 @@ static int add_dcp_components(struct device *dev, int num = 0; for_each_matching_node(np, apple_dcp_id_tbl) { - if (apple_dcp_in_drm(np) && of_device_is_available(np)) { + if (apple_dcp_in_drm(np) && apple_dcp_usable(np)) { drm_of_component_match_add(dev, matchptr, component_compare_of, np); num++; @@ -1080,9 +1094,6 @@ static int add_dcp_components(struct device *dev, static int apple_platform_probe(struct platform_device *pdev) { - if (of_machine_is_compatible("apple,t8140")) - return -ENODEV; - struct device *mdev = &pdev->dev; struct component_match *match = NULL; int num_dcp; @@ -1105,6 +1116,8 @@ static const struct of_device_id of_match[] = { { .compatible = "apple,t6030-display-subsystem" }, { .compatible = "apple,t8122-display-subsystem" }, { .compatible = "apple,t6031-display-subsystem" }, + { .compatible = "apple,t8140-display-subsystem" }, + { .compatible = "apple,t8142-display-subsystem" }, {} }; MODULE_DEVICE_TABLE(of, of_match); @@ -1126,7 +1139,9 @@ static int apple_platform_resume(struct device *dev) bool dropped = false; if (apple) { - if (apple->drm.mode_config.suspend_state) + /* H17P (T8140, T8142) only, as on aurora-wip. */ + if (apple->drm.mode_config.suspend_state && + apple_drm_has_h17p(&apple->drm)) dropped = apple_drm_drop_stale_crtcs(apple->drm.mode_config.suspend_state); drm_mode_config_helper_resume(&apple->drm); /* Have userspace set the mode of a CRTC resumed disabled again. */ diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index 4b8c4d8fdd7840..cfbb09863fa4b7 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -27,6 +27,7 @@ #include "av.h" #include "dcp.h" +#include "dcp-internal.h" #include "audio.h" #include "parser.h" @@ -101,6 +102,13 @@ static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { .periods_max = UINT_MAX, }; +static bool dcpaud_h17p(struct dcp_audio *dcpaud) +{ + struct apple_dcp *dcp = dev_get_drvdata(dcpaud->dcp_dev); + + return dcp && dcp->fw_compat == DCP_FIRMWARE_H17P; +} + static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) { int ret; @@ -283,7 +291,12 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) return ret; } - dcpaud->open_unplugged = !dcpaud->connected; + /* H17P (T8140): the PCM opens without a sink, as on aurora-wip. */ + dcpaud->open_unplugged = !dcpaud->connected && dcpaud_h17p(dcpaud); + if (!dcpaud->connected && !dcpaud->open_unplugged) { + mutex_unlock(&dcpaud->data_lock); + return -ENXIO; + } dcpaud->open_cookie = dcpaud->connection_cookie; mutex_unlock(&dcpaud->data_lock); diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 78a436b23193a6..49e759e6e11316 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -62,6 +62,12 @@ struct apple_dcp_typec_port { u32 applied_status; u32 applied_conf; bool applied_valid; + /* + * H17P: SAFE preparation retains this owner until the set phase + * completes (dcp_typec_route_prepare() and its finish and commit). + */ + bool safe_pending; + bool safe_finish_ready; bool hpd; /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ bool dp_wanted; @@ -162,6 +168,7 @@ static bool dcp_typec_route_busy(struct apple_dcp_typec_route *route) static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) { + struct apple_dcp_typec_port *port; struct dcp_fabric_pipeline pipeline = { .owned = !!route->dcp->active_typec_route || route->dcp->tb_retiring, .fixed_busy = dcp_typec_route_busy(route), @@ -169,6 +176,12 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_dual_stream() }; + /* Only H17P routes prepare SAFE: their pipeline stays owned until commit. */ + list_for_each_entry(port, &dcp_typec_ports, link) { + if (port->safe_pending && port->owner && + port->owner->dcp == route->dcp) + return false; + } pipeline.presence = dcp_hdmi_presence(route->dcp); return dcp_fabric_available(&pipeline, &policy); } @@ -274,6 +287,9 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, route->port->secondary_connector : route->port->connector; int ret; + /* External H17P: a new route starts a new firmware-metadata connection. */ + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + /* * The fixed output's HPD handler leaves disconnects to DCP, so the port * can still be marked connected to a display that is gone. Release it @@ -359,6 +375,8 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, } dcp->active_typec_route = route; scoped_guard(mutex, &dcp->tb_lock) { + route->deactivate_xbar_done = false; + route->deactivate_error = 0; route->active_xbar = xbar; route->tunnel = xbar != route->xbar; route->xbar_up = !route->tunnel; @@ -375,7 +393,7 @@ static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, return 0; } -static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) +static int dcp_typec_route_release(struct apple_dcp_typec_route *route) { struct apple_dcp *dcp = route->dcp; struct apple_connector *connector = dcp->typec_connector; @@ -389,16 +407,36 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) * crossbar nor restart the tunnel pixel clock behind our back. */ scoped_guard(mutex, &dcp->tb_lock) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else - dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - /* mux_control_deselect releases its semaphore even on failure. */ - route->xbar_up = false; - if (ret) - dev_warn(dcp->dev, "crossbar deselect failed: %d\n", ret); - /* Ownership is released; let the caller release its route owner too. */ - ret = 0; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (route->deactivate_error) + return route->deactivate_error; + if (!route->deactivate_xbar_done) { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + ret = dcp_dpxbar_preselect(route->active_xbar, + MUX_IDLE_DISCONNECT); + /* Deselect always unlocks: never consume that selection twice. */ + route->deactivate_xbar_done = true; + route->xbar_up = false; + if (ret) { + /* Idle is unproven; retain the owner and the real error. */ + route->deactivate_error = ret; + return ret; + } + } + } else { + if (!route->tunnel || route->xbar_up) + ret = mux_control_deselect(route->active_xbar ?: route->xbar); + else + dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); + /* mux_control_deselect releases its semaphore even on failure. */ + route->xbar_up = false; + if (ret) + dev_warn(dcp->dev, "crossbar deselect failed: %d\n", ret); + /* Ownership is released; let the caller release its route owner too. */ + ret = 0; + } if (route->tunnel && dcp->phy) { /* the tunnel pixel clock must not outlive the tunnel */ @@ -424,6 +462,8 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) if (sel && sel != -EOPNOTSUPP) dev_warn(dcp->dev, "DP tunnel source reset failed: %d\n", sel); } + if (dcp_is_external_h17p(dcp)) + dpavservep_detach(dcp); route->selected = false; ended_attachment = dcp_modes_end_typec(dcp, route); if (dcp->active_typec_route == route) @@ -495,6 +535,28 @@ static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) return 0; } +static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) +{ + struct apple_dcp *dcp = route->dcp; + int ret; + + if (dcp->fw_compat != DCP_FIRMWARE_H17P) + return dcp_typec_route_release(route); + + /* + * External H17P: the route stays owned until its firmware stop has + * completed; a failed stop keeps it, with its error. + */ + guard(mutex)(&dcp->external_lock); + if (dcp_is_external_h17p(dcp) && + (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) + return dcp->external_error ?: -EAGAIN; + + dcp_firmware_metadata_connection(dcp, READ_ONCE(dcp->typec_generation), false); + ret = dcp_typec_route_release(route); + return ret; +} + /* * The DRM device once it has bound, that is once every pipeline has its * CRTC and every port its connectors; NULL before. @@ -1239,6 +1301,10 @@ static int dcp_typec_port_busy(struct apple_dcp_typec_port *port, { const char *label = NULL; + /* H17P: every pipeline drives another display; -EBUSY means retry. */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return -ENOSPC; + if (port->busy_noted) return -EBUSY; port->busy_noted = true; @@ -1249,6 +1315,93 @@ static int dcp_typec_port_busy(struct apple_dcp_typec_port *port, return -EBUSY; } +/* + * H17P (T8140, T8142): the mux transaction's SAFE phases. Preparation + * disconnects the display and stops new admission, while the route keeps + * its owner until the PHY has gone SAFE (dcp_typec_route_finish()) and the + * transaction commits (dcp_typec_route_commit()). Other processors register + * only dcp_typec_route_set(). + */ +static int dcp_typec_route_prepare(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); + struct apple_dcp_typec_port *port = route->port; + struct apple_dcp *dcp; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE) + return port->safe_pending ? -EAGAIN : 0; + if (!port->owner || port->owner->tunnel) + return 0; + dcp = port->owner->dcp; + /* Stop new admission, but retain the owner until the set phase succeeds. */ + if (!port->safe_pending) + port->safe_finish_ready = false; + port->safe_pending = true; + port->applied_valid = false; + port->dp_wanted = false; + port->dp_hpd = false; + /* HPD absence alone is not proof of a completed firmware shutdown. */ + if (dcp_is_external_h17p(dcp)) { + scoped_guard(mutex, &dcp->external_lock) { + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable && dcp->external_stopped && + !READ_ONCE(dcp->dptxport[0].connected)) { + port->hpd = false; + return 0; + } + } + } + ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); + if (ret) + return ret; + port->hpd = false; + return 0; +} + +static int dcp_typec_route_finish(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + int ret; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending || + port->safe_finish_ready) + return 0; + ret = port->owner ? dcp_typec_route_deactivate(port->owner) : 0; + if (ret) + return ret; + port->safe_finish_ready = true; + return 0; +} + +static void dcp_typec_route_commit(struct typec_mux_dev *mux, + struct typec_mux_state *state) +{ + struct apple_dcp_typec_port *port = + ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + + guard(mutex)(&dcp_typec_fabric_lock); + if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending) + return; + if (port->owner) + port->preferred_route = port->owner; + port->owner = NULL; + port->safe_pending = false; + port->safe_finish_ready = false; + port->applied_alt = state->alt; + port->applied_mode = state->mode; + port->applied_status = 0; + port->applied_conf = 0; + port->applied_valid = true; + dcp_typec_pipeline_freed(); +} + static int dcp_typec_route_set(struct typec_mux_dev *mux, struct typec_mux_state *state) { @@ -1269,6 +1422,13 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, guard(mutex)(&dcp_typec_fabric_lock); + /* H17P only: a SAFE transition is still being prepared or finished. */ + if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) + return 0; + /* The previous SAFE transition has not finished; retry. */ + if (port->safe_pending) + return -EAGAIN; + /* * Every candidate route for this port is notified with the same state, * so only the first to arrive does the work; the others return early. @@ -1332,6 +1492,7 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_typec_rebalance_locked(NULL, 0); } + port->safe_pending = false; /* Data-only USB4 changes do not invalidate other display routes. */ port->applied_valid = true; return 0; @@ -1378,6 +1539,9 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, if (!port->owner) { best = dcp_typec_free_route(port); + /* H17P: every pipeline is driving another port's display. */ + if (!best && route->dcp->fw_compat == DCP_FIRMWARE_H17P) + return -ENOSPC; if (!best) return hpd ? dcp_typec_port_busy(port, route->dcp) : -EBUSY; @@ -1401,9 +1565,11 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, /* * A T6030 external processor services the request itself, and * reads the display again if it changed; elsewhere the active - * mode is applied again. + * mode is applied again. H17P always applies it again. */ - if (dcp->typec_connector && dcp->external_native) + if (dcp->typec_connector && dcp->fw_compat == DCP_FIRMWARE_H17P) + dcp_retrain_oob(dcp->typec_connector); + else if (dcp->typec_connector && dcp->external_native) dcp_external_sink_irq(dcp); else if (dcp->typec_connector && !READ_ONCE(dcp->typec_connector->connected)) { @@ -1591,7 +1757,8 @@ int dcp_tunnel_crossbar_up(struct apple_dcp *dcp, u32 unit) if (unit) return dcp_tunnel_split_up_locked(dcp); route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar) + if (!route || !route->tunnel || !route->active_xbar || + route->deactivate_xbar_done) return -ENODEV; if (!dcp_fabric_callback_valid(route->tunnel_generation, dcp->tb_generation)) return -ESTALE; @@ -3258,7 +3425,9 @@ static void dcp_typec_route_unregister(void *data) if (!route->tunnel && (port->hpd || dcp->typec_cable_connected)) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); port->hpd = false; - dcp_typec_route_deactivate(route); + /* External H17P: a failed firmware stop keeps the route owned. */ + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; port->owner = NULL; } if (port->secondary_owner == route) { @@ -3266,7 +3435,8 @@ static void dcp_typec_route_unregister(void *data) if (!route->tunnel && (port->hpd || dcp->typec_cable_connected)) dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - dcp_typec_route_deactivate(route); + if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) + return; port->secondary_owner = NULL; } WRITE_ONCE(route->dcp->tb_retiring, false); @@ -3444,6 +3614,11 @@ static int dcp_register_typec_routes_present(void *data) desc.fwnode = of_fwnode_handle(route_np); desc.set = dcp_typec_route_set; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + desc.prepare = dcp_typec_route_prepare; + desc.finish = dcp_typec_route_finish; + desc.commit = dcp_typec_route_commit; + } desc.name = mux_name; desc.drvdata = route; route->typec_mux = typec_mux_register(dev, &desc); @@ -3900,7 +4075,8 @@ void dcp_link(struct platform_device *pdev, struct apple_crtc *crtc, { struct apple_dcp *dcp = platform_get_drvdata(pdev); - dcp->crtc = crtc; + /* dcp_delayed_vblank() reads this after an early RTKit crash. */ + WRITE_ONCE(dcp->crtc, crtc); /* * Type-C connectors belong to physical ports and are bound by the diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 94ac176d96db3c..5bb70348091145 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -46,6 +46,12 @@ struct apple_dcp_typec_route { bool tunnel; /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ bool xbar_up; + /* + * H17P: the lower mux release is consumed even when idle programming + * fails, and that error retains the route's owner. + */ + bool deactivate_xbar_done; + int deactivate_error; }; bool dcp_is_typec_output(struct apple_dcp *dcp); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 7bc2ab6f7336a3..5392736e453126 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -225,8 +225,6 @@ struct apple_dcp { struct apple_connector *connector; struct apple_dcp_hw_data hw; - /* Protected by the H17P IOMFB receive mutex. */ - u8 experimental_callback_logs; bool external_prepared; /* Serializes external connection, power, and CRTC generation changes. */ struct mutex external_lock; @@ -434,8 +432,6 @@ struct apple_dcp { struct delayed_work swap_watchdog_wq; unsigned int swap_watchdog_retrains; - /* List of referenced drm_framebuffers which can be unreferenced - * on the next successfully completed swap. /* List of referenced framebuffers pending a completed replacement swap. * The commit and RTKit work queues share it under this lock. */ @@ -468,8 +464,6 @@ struct apple_dcp { struct apple_dcp_afkep *dptxep; struct apple_dcp_afkep *dptx_controller_ep; struct completion dptx_controller_ready; - struct apple_dcp_afkep *dptx_v2_probe_ep; - struct completion dptx_v2_probe_ready; struct apple_dcp_afkep *dpavctrlep; struct apple_epic_service *dpav_ctrl; @@ -537,24 +531,6 @@ struct apple_dcp { struct dcp_fabric_session typec_reconnect_session; /* hpd_mutex */ bool typec_work_enabled; /* hpd_mutex; IRQ recovery admission */ - /* Ported from aurora-silicon/linux#8. */ - /* DPTX feeds a Thunderbolt DP IN adapter, not the Type-C PHY lanes. */ - bool dptx_tunnel; - /* DFP port in the DPTX target: 0 = dpphy, 1 = dpin0, 2 = dpin1. */ - u8 dptx_dfp_port; - /* Wakes/sleeps the Thunderbolt DP IN adapter from DCP Activate/ - * Deactivate. Set by apple_dcp_tb_dp_tunnel(), cleared by - * dcp_typec_route_deactivate(). - */ - int (*tb_dpin_set_active)(void *ctx, bool active); - void *tb_dpin_ctx; - /* Serializes the Thunderbolt DP IN callback and tunnel crossbar state - * between DCP apcalls and tunnel teardown; never held while waiting - * for DCP. - */ - struct mutex tb_lock; - bool tb_clock_ok; - struct gpio_desc *hdmi_hpd; struct gpio_desc *hdmi_pwren; struct gpio_desc *dp2hdmi_pwren; @@ -569,6 +545,15 @@ struct apple_dcp { struct apple_connector *fixed_connector; struct apple_connector *typec_connector; int hdmi_hpd_irq; + /* Hardirq, resume and expiry share one generation under this lock. */ + spinlock_t hdmi_presence_lock; + struct dcp_fabric_presence hdmi_presence; + unsigned long hdmi_edge_jiffies; + bool hdmi_edge_seen; + struct delayed_work hdmi_settle_wq; + struct delayed_work hdmi_recheck_wq; + /* T6030 HDMI converter service requests, see dcp-hdmi.c */ + struct dcp_hdmi *hdmi; /* H17P policy shared by backlight, commit, and reply workers. */ struct { @@ -583,15 +568,6 @@ struct apple_dcp { unsigned long resent_at; bool resent; } backlight; - /* Hardirq, resume and expiry share one generation under this lock. */ - spinlock_t hdmi_presence_lock; - struct dcp_fabric_presence hdmi_presence; - unsigned long hdmi_edge_jiffies; - bool hdmi_edge_seen; - struct delayed_work hdmi_settle_wq; - struct delayed_work hdmi_recheck_wq; - /* T6030 HDMI converter service requests, see dcp-hdmi.c */ - struct dcp_hdmi *hdmi; /* Staging wire record; serialization never mutates the atomic inputs. */ struct dcp_present_h17p present_h17p; @@ -688,6 +664,12 @@ static inline bool dcp_ext_dptx_607(const struct apple_dcp *dcp) return dcp->fw_compat == DCP_FIRMWARE_V_14_7_TEMPLATE; } +/* The T8140 external processor (dispext0) running H17P firmware. */ +static inline bool dcp_is_external_h17p(const struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; +} + #define DCP_AUDIO_MAX_CHANS 15 #endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-link.h b/drivers/gpu/drm/apple/dcp-link.h index 2168d03dd53767..1bcca315c42578 100644 --- a/drivers/gpu/drm/apple/dcp-link.h +++ b/drivers/gpu/drm/apple/dcp-link.h @@ -761,98 +761,6 @@ apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, config->clock_request = cpu_to_le64(clock_request); } -static inline int -apple_dcp_h17p_rt_bw_encode_reply(struct apple_dcp_link_rpc_view *view, - u64 scratch, u64 clock_request) -{ - const struct apple_dcp_h17p_rt_bw_request *request; - struct apple_dcp_h17p_rt_bw_reply *reply; - - if (!view || view->call != APPLE_DCP_LINK_CALL_D003) - return -EINVAL; - if (view->input_size < sizeof(*request) || - view->output_size < sizeof(*reply)) - return -EMSGSIZE; - - request = view->input; - if (le32_to_cpu(request->config_null) & 1) - return -EINVAL; - - reply = view->output; -#include -#include -#include - -#include - -/* - * H17-generation DCP firmware uses the ordinary IOMFB transport once the - * INIT/SetShmem handshake is done: the same six fixed channels (CMD 0x00000, - * OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, CB 0x60000, - * OOBCB 0x68000), selected by the context field of the message. Records are - * allocated on each channel's stack at - * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) - * which is the rule dcp_packet_start() and DCP_PACKET_ALIGNMENT implement in - * iomfb.c. - * - * This header only describes the callback payloads whose layout differs from - * older firmware. - */ - -#define APPLE_DCP_H17P_CLOCK_ID_154 0x154 -#define APPLE_DCP_H17P_CLOCK_ID_194 0x194 - -struct apple_dcp_h17p_clock_request { - __le32 service; - __le32 clock_id; -} __packed; - -struct apple_dcp_h17p_rt_bw_config { - __le64 scratch; - __le64 clock_request; -} __packed; - -struct apple_dcp_h17p_rt_bw_request { - __le32 config_null; -} __packed; - -struct apple_dcp_h17p_rt_bw_reply { - struct apple_dcp_h17p_rt_bw_config config; - __le32 status; -} __packed; - -static_assert(sizeof(struct apple_dcp_h17p_clock_request) == 0x08); -static_assert(sizeof(struct apple_dcp_h17p_rt_bw_config) == 0x10); -static_assert(sizeof(struct apple_dcp_h17p_rt_bw_request) == 0x04); -static_assert(sizeof(struct apple_dcp_h17p_rt_bw_reply) == 0x14); - -static inline int -apple_dcp_h17p_clock_rate(const struct apple_dcp_h17p_clock_request *request, - u64 rate_154, u64 rate_194, u64 *rate) -{ - if (!request || !rate) - return -EINVAL; - - switch (le32_to_cpu(request->clock_id)) { - case APPLE_DCP_H17P_CLOCK_ID_154: - *rate = rate_154; - return 0; - case APPLE_DCP_H17P_CLOCK_ID_194: - *rate = rate_194; - return 0; - default: - return -ENOENT; - } -} - -static inline void -apple_dcp_h17p_rt_bw_encode(struct apple_dcp_h17p_rt_bw_config *config, - u64 scratch, u64 clock_request) -{ - config->scratch = cpu_to_le64(scratch); - config->clock_request = cpu_to_le64(clock_request); -} - /* D003: fill in the bandwidth config unless the firmware passed it as null */ static inline int apple_dcp_h17p_rt_bw_encode_reply(const struct apple_dcp_h17p_rt_bw_request *request, @@ -867,4 +775,19 @@ apple_dcp_h17p_rt_bw_encode_reply(const struct apple_dcp_h17p_rt_bw_request *req return 0; } +/* The same, for a call record seen through a DCPLink RPC view. */ +static inline int +apple_dcp_h17p_rt_bw_encode_reply_view(struct apple_dcp_link_rpc_view *view, + u64 scratch, u64 clock_request) +{ + if (!view || view->call != APPLE_DCP_LINK_CALL_D003) + return -EINVAL; + if (view->input_size < sizeof(struct apple_dcp_h17p_rt_bw_request) || + view->output_size < sizeof(struct apple_dcp_h17p_rt_bw_reply)) + return -EMSGSIZE; + + return apple_dcp_h17p_rt_bw_encode_reply(view->input, view->output, + scratch, clock_request); +} + #endif /* __APPLE_DCP_LINK_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 3719ddf3c718ff..5caf56299f8710 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -10,12 +10,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -28,15 +26,9 @@ #include #include #include -#include #include #include #include -#include -#include -#include -#include -#include #include #include #include @@ -47,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -73,3715 +64,4164 @@ #define APPLE_DCP_COPROC_CPU_STATUS_STOPPED BIT(1) #define APPLE_DCP_COPROC_CPU_STATUS_IDLE BIT(5) -struct dcp_session_ref { - struct list_head link; - struct device *dev; - struct iommu_group *group; -}; - -static LIST_HEAD(dcp_retained_sessions); -static DEFINE_MUTEX(dcp_retained_sessions_lock); +#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) -static bool dcp_session_retained(struct device *dev) -{ - struct dcp_session_ref *session; - bool retained = false; +static bool show_notch; +module_param(show_notch, bool, 0644); +MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); - mutex_lock(&dcp_retained_sessions_lock); - list_for_each_entry(session, &dcp_retained_sessions, link) { - if (dev_fwnode(session->dev) == dev_fwnode(dev)) { - retained = true; - break; - } - } - mutex_unlock(&dcp_retained_sessions_lock); - return retained; -} +static bool t6030_show_notch; +module_param(t6030_show_notch, bool, 0444); +MODULE_PARM_DESC(t6030_show_notch, + "M3 internal display (14.x and 26.6 IOMFB): use the full height and show the notch"); -static int dcp_pin_live_session(struct apple_dcp *dcp) -{ - struct dcp_session_ref *session; +bool hdmi_audio; +module_param(hdmi_audio, bool, 0644); +MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); - if ((!dcp->hw.adopt_live_session && - dcp->hw.iomfb_method_profile != - DCP_IOMFB_METHODS_H17P_EXTERNAL) || - dcp->fw_compat != DCP_FIRMWARE_H17P || - dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) - return 0; - session = kzalloc_obj(*session); - if (!session) - return -ENOMEM; +static bool unstable_edid = true; +module_param(unstable_edid, bool, 0644); +MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); - /* Keep the DMA device, domain usage and driver code valid until reset. */ - session->dev = get_device(dcp->dev); - session->group = iommu_group_get(dcp->dev); - __module_get(THIS_MODULE); - WRITE_ONCE(dcp->retain_dma, true); - mutex_lock(&dcp_retained_sessions_lock); - list_add_tail(&session->link, &dcp_retained_sessions); - mutex_unlock(&dcp_retained_sessions_lock); - dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); - return 0; -} +/* + * H17-generation firmware (T8140 H17P, T8142 H17G) runs aurora-wip's + * processor lifecycle, below in the H17P section. Shared entry points + * hand those processors over to it. + */ +static void dcp_recv_msg_h17p(struct apple_dcp *dcp, u8 endpoint, u64 message); +static void dcp_rtk_crashed_h17p(struct apple_dcp *dcp); +static int dcp_rtk_shmem_setup_h17p(struct apple_dcp *dcp, struct apple_rtkit_shmem *bfr); +static int dcp_crtc_atomic_check_h17p(struct drm_crtc *crtc, struct drm_atomic_state *state); +static int dcp_dptx_connect_h17p(struct apple_dcp *dcp, u32 port); +static int dcp_dptx_disconnect_h17p(struct apple_dcp *dcp, u32 port); +static int dcp_dptx_connect_oob_h17p(struct platform_device *pdev, u32 port); +static int dcp_dptx_disconnect_oob_h17p(struct platform_device *pdev, u32 port); +static int dcp_start_h17p(struct platform_device *pdev); +static void dcp_poweron_h17p(struct platform_device *pdev); +static void dcp_poweroff_h17p(struct platform_device *pdev); +static void dcp_disable_typec_work_h17p(struct apple_dcp *dcp, bool release_cable); +static void dcp_enable_typec_work_h17p(struct apple_dcp *dcp); +static int dcp_comp_bind_h17p(struct device *dev, struct device *main, void *data); +static void dcp_comp_unbind_h17p(struct device *dev, struct device *main, void *data); +static int dcp_platform_probe_h17p(struct platform_device *pdev); +static void dcp_platform_shutdown_h17p(struct platform_device *pdev); +static int dcp_platform_suspend_h17p(struct device *dev); +static int dcp_platform_resume_h17p(struct device *dev); -static void dcp_release_dma_domain(void *data) +/* copied and simplified from drm_vblank.c */ +static void send_vblank_event(struct drm_device *dev, + struct drm_pending_vblank_event *e, + u64 seq, ktime_t now) { - struct apple_dcp *dcp = data; + struct timespec64 tv; - if (!dcp->retain_dma) - iommu_device_unuse_default_domain(dcp->dev); -} + if (e->event.base.type != DRM_EVENT_FLIP_COMPLETE) + return; -static void dcp_release_context(void *data) -{ - struct apple_dcp *dcp = data; + tv = ktime_to_timespec64(now); + e->event.vbl.sequence = seq; + /* + * e->event is a user space structure, with hardcoded unsigned + * 32-bit seconds/microseconds. This is safe as we always use + * monotonic timestamps since linux-4.15 + */ + e->event.vbl.tv_sec = tv.tv_sec; + e->event.vbl.tv_usec = tv.tv_nsec / 1000; - if (!dcp->retain_dma) - kfree(dcp); + /* + * Use the same timestamp for any associated fence signal to avoid + * mismatch in timestamps for vsync & fence events triggered by the + * same HW event. Frameworks like SurfaceFlinger in Android expects the + * retire-fence timestamp to match exactly with HW vsync as it uses it + * for its software vsync modeling. + */ + drm_send_event_timestamp_locked(dev, &e->base, now); } -static void dcp_release_rtkit(void *data) +/** + * dcp_crtc_send_page_flip_event - helper to send vblank event after pageflip + * + * Compensate for unknown slack between page flip and arrival of the + * swap_complete callback. Minimal observed duration on DCP with HDMI output + * was around 2.3 ms. If the fb swap was submitted closer to the expected + * swap_complete it gets a penalty of one frame duration. This is on the border + * of unreasonable considering that Apple advertises support for 240 Hz (frame + * duration of 4.167 ms). + * It is unreasonable considering kwin's kms commit scheduling. Kwin commits + * 1.5 ms + the mode's vblank time before the expected next page flip + * completion. This results in presenting at half the display's rate for HDMI + * outputs. + * This might be a difference between dcp and dcpext. + */ +static void dcp_crtc_send_page_flip_event(struct apple_crtc *crtc, + struct drm_pending_vblank_event *e, + ktime_t now, ktime_t start) { - struct apple_dcp *dcp = data; + struct drm_device *dev = crtc->base.dev; + u64 seq; + unsigned int pipe = drm_crtc_index(&crtc->base); + ktime_t flip; - if (dcp->retain_dma) { - WRITE_ONCE(dcp->quiescing, true); - apple_rtkit_free_retaining_buffers(dcp->rtk); + seq = 0; + if (start != KTIME_MIN) { + s64 delta = ktime_us_delta(now, start); + if (delta <= 500) + flip = now; + else if (delta >= 2500) + flip = ktime_sub_us(now, 1000); + else + flip = ktime_sub_us(now, (delta - 500) / 2); } else { - apple_rtkit_free(dcp->rtk); + flip = now; } - dcp->rtk = NULL; + e->pipe = pipe; + send_vblank_event(dev, e, seq, flip); } -#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) - -bool dcp_h17p_external_supported(struct device_node *np) +/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ +void dcp_drm_crtc_vblank(struct apple_crtc *crtc) { - u32 firmware[DCP_EXTERNAL_FW_CELLS], compat[DCP_EXTERNAL_FW_CELLS]; - int firmware_cells, compat_cells; + unsigned long flags; - if (!of_machine_is_compatible("apple,j700") || - !of_device_is_compatible(np, "apple,t8140-dcpext")) - return false; - firmware_cells = of_property_read_variable_u32_array( - np, "apple,firmware-version", firmware, - DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); - compat_cells = of_property_read_variable_u32_array( - np, "apple,firmware-compat", compat, - DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); - return apple_dcp_external_firmware_supported(firmware, firmware_cells, - compat, compat_cells); + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + crtc->event = NULL; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } -bool dcp_h17p_external_ready(struct device_node *np) +void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) { - struct platform_device *pdev; - struct apple_dcp *dcp; - bool ready = false; + unsigned long flags; + struct apple_crtc *crtc = dcp->crtc; - if (!dcp_h17p_external_supported(np)) - return false; - pdev = of_find_device_by_node(np); - if (!pdev) - return false; - /* - * Called after component binding. component_add() may synchronously - * bind the master before the platform probe itself returns, so the - * driver core's bound flag is not a readiness condition here. - */ - dcp = platform_get_drvdata(pdev); - ready = dcp && READ_ONCE(dcp->external_prepared) && - !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); - put_device(&pdev->dev); - return ready; + spin_lock_irqsave(&crtc->base.dev->event_lock, flags); + if (crtc->event) { + if (crtc->event->event.base.type == DRM_EVENT_FLIP_COMPLETE) + dcp_crtc_send_page_flip_event(crtc, crtc->event, now, dcp->swap_start); + else + drm_crtc_send_vblank_event(&crtc->base, crtc->event); + crtc->event = NULL; + dcp->swap_start = KTIME_MIN; + } + spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } -static bool show_notch; -module_param(show_notch, bool, 0644); -MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); - -bool hdmi_audio; -module_param(hdmi_audio, bool, 0644); -MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); - -static bool unstable_edid = true; -module_param(unstable_edid, bool, 0644); -MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); - -struct apple_dcp_typec_port { - struct list_head link; - struct list_head routes; - struct device_node *connector_np; - struct apple_dcp_typec_route *owner; - /* DRM connector for this physical port, driven by whichever DCP owns it */ - struct apple_connector *connector; - /* last mux state acted on, to collapse the per-candidate notifications */ - struct typec_altmode *applied_alt; - unsigned long applied_mode; - u32 applied_status; - u32 applied_conf; - bool applied_valid; - bool hpd; -}; - -static DEFINE_MUTEX(dcp_typec_fabric_lock); -static LIST_HEAD(dcp_typec_ports); - -static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); -static void disconnected_hpd_event(struct apple_connector *connector); - -bool dcp_is_typec_output(struct apple_dcp *dcp) +void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation) { - return dcp->active_typec_route || - dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; + struct apple_connector *apple_connector = READ_ONCE(dcp->connector); + struct drm_connector *connector = apple_connector ? &apple_connector->base : NULL; + int width_mm, height_mm, i; + + if (connector) + mutex_lock(&connector->dev->mode_config.mutex); + mutex_lock(&dcp->modes_lock); + if (generation != dcp->modes_generation || + apple_connector != READ_ONCE(dcp->connector)) + goto out_unlock; + width_mm = dcp->width_mm; + height_mm = dcp->height_mm; + if (!width_mm || !height_mm) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } + if (connector) { + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; + } + for (i = 0; i < dcp->nr_modes; ++i) { + dcp->modes[i].mode.width_mm = width_mm; + dcp->modes[i].mode.height_mm = height_mm; + } +out_unlock: + mutex_unlock(&dcp->modes_lock); + if (connector) + mutex_unlock(&connector->dev->mode_config.mutex); } -static bool dcp_typec_route_is_dp(const struct typec_mux_state *state) +bool dcp_has_panel(struct apple_dcp *dcp) { - return state->alt && state->alt->svid == USB_TYPEC_DP_SID && - state->mode >= TYPEC_DP_STATE_A && - state->mode <= TYPEC_DP_STATE_F; + return dcp->panel.width_mm > 0; } -/* Keep a live fixed output on its own pipeline. */ -static bool dcp_typec_route_fixed_output_busy(struct apple_dcp_typec_route *route) +int dcp_set_crc(struct drm_crtc *crtc, bool enabled) { - struct apple_dcp *dcp = route->dcp; - - if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB) - return false; - if (dcp->fixed_connector && dcp->fixed_connector->connected) - return true; - if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) - return true; + struct apple_crtc *ac = to_apple_crtc(crtc); + struct apple_dcp *dcp = platform_get_drvdata(ac->dcp); - return false; -} + dcp->crc_enabled = enabled; -static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) -{ - return !route->dcp->active_typec_route && - !dcp_typec_route_fixed_output_busy(route); + return 0; } /* - * Machines whose USB4 docks carry two independent DP streams through one - * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and - * each port has a second connector for the DPIN1 stream. + * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp + * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks + * send a vblank event via a workqueue. */ -bool dcp_typec_dual_stream(void) +static void dcp_delayed_vblank(struct work_struct *work) { - return apple_dp_tunnel_t602x(); -} + struct apple_dcp *dcp; -bool dcp_is_typec_only(struct platform_device *pdev) -{ - struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_crtc *crtc; - return !dcp->fixed_phy; + dcp = container_of(work, struct apple_dcp, vblank_wq); + /* RTKit can report a crash before the pipeline has a CRTC. */ + crtc = READ_ONCE(dcp->crtc); + if (!crtc) + return; + mdelay(5); + dcp_drm_crtc_vblank(crtc); } +#define DCP_SWAP_WATCHDOG_MS 1000 +#define DCP_SWAP_WATCHDOG_RETRAINS 5 + /* - * Can this route feed the given connector without changing its - * possible_crtcs? On dual-stream machines those are fixed at probe, since - * compositors read them once when the connector appears and pair the - * connector with a CRTC before this fabric has routed it. + * DCP drops swaps without completing them while an external pipe is not + * enabled, for instance after a modeset that raced a Type-C sink which had + * not asserted HPD yet. Userspace would then wait for the flip until the + * commit times out, stalling every output it drives. Complete the flip, + * mark the mode invalid so later commits do not wait for DCP, and let the + * hotplug worker re-apply the active CRTC. */ -static bool dcp_typec_route_fits(struct apple_dcp_typec_route *route, - struct apple_connector *connector) +static void dcp_swap_watchdog(struct work_struct *work) { - struct apple_dcp *dcp = route->dcp; + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + swap_watchdog_wq); - if (!dcp_typec_dual_stream() || !connector || !dcp->crtc) - return true; - return connector->candidate_crtcs & drm_crtc_mask(&dcp->crtc->base); + dev_warn(dcp->dev, "swap not completed, retraining the display\n"); + dcp_mode_invalidate(&dcp->mode_state); + dcp_drm_crtc_vblank(dcp->crtc); + if (dcp->connector && + dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) + dcp_queue_hotplug(dcp->connector); } -static bool dcp_is_external_h17p(struct apple_dcp *dcp) +void dcp_swap_watchdog_arm(struct apple_dcp *dcp) { - return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; + if (dcp_is_typec_output(dcp)) + mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, + msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); } -static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation); -static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical); - -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, - struct mux_control *xbar); -static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route); -static int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); - -static int dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) +void dcp_swap_watchdog_complete(struct apple_dcp *dcp) { - typeof(&apple_dpxbar_tunnel_select_source) select = - symbol_get(apple_dpxbar_tunnel_select_source); - int ret; - - if (!select) - return -ENOENT; - ret = select(mux, state); - symbol_put(apple_dpxbar_tunnel_select_source); - return ret; + cancel_delayed_work(&dcp->swap_watchdog_wq); + dcp->swap_watchdog_retrains = 0; } -/* - * For a port without a prior owner, rank pipelines by CRTC index. A pipeline - * whose fixed output is live is not a candidate at all, so a hybrid is only - * ever ranked here when it is genuinely free. - */ -static unsigned int dcp_typec_route_score(struct apple_dcp_typec_route *route) +static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) { - struct apple_dcp *dcp = route->dcp; + struct apple_dcp *dcp = cookie; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_recv_msg_h17p(dcp, endpoint, message); + + trace_dcp_recv_msg(dcp, endpoint, message); - if (!dcp->crtc) - return UINT_MAX - 1; + /* A failed T8122 dcpext: its firmware may still talk, nobody listens. */ + if (READ_ONCE(dcp->ext_dead)) + return; - return drm_crtc_index(&dcp->crtc->base); + switch (endpoint) { + case IOMFB_ENDPOINT: + return iomfb_recv_msg(dcp, message); + case AV_ENDPOINT: + afk_receive_message(dcp->avep, message); + return; + case SYSTEM_ENDPOINT: + afk_receive_message(dcp->systemep, message); + return; + case DISP0_ENDPOINT: + afk_receive_message(dcp->ibootep, message); + return; + case DPAVSERV_ENDPOINT: + afk_receive_message(dcp->dcpavservep, message); + return; + case DPTX_ENDPOINT: + afk_receive_message(dcp->dptxep, message); + return; + default: + WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); + } } -static int dcp_typec_route_activate(struct apple_dcp_typec_route *route, - struct mux_control *xbar) +static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) { - struct apple_dcp *dcp = route->dcp; - int ret = 0; + struct apple_dcp *dcp = cookie; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_rtk_crashed_h17p(dcp); /* - * Under tb_lock so a DCP link (re)configuration can neither select - * the crossbar nor restart the tunnel pixel clock behind our back. - * Ported from aurora-silicon/linux#8. + * A failed T8122 dcpext: its start never ran (start_done is not set + * up) and it has no connector. Its firmware may still report a crash. */ - ret = dcp_dptx_disconnect(dcp, 0); - if (ret) - return ret; - - if (dcp->fixed_route_selected) { - ret = mux_control_deselect(dcp->xbar); - if (ret) - return ret; - route->xbar_up = false; - - if (route->tunnel && dcp->phy) { - /* The tunnel pixel clock must not outlive the tunnel. */ - typeof(&apple_atc_dp_tunnel_rate) stop = - symbol_get(apple_atc_dp_tunnel_rate); - - if (stop) { - stop(dcp->phy, 0); - symbol_put(apple_atc_dp_tunnel_rate); - } - } - route->active_xbar = NULL; - route->tunnel = false; - dcp->dptx_tunnel = false; - dcp->dptx_dfp_port = 0; - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - dcp->tb_clock_ok = false; + if (READ_ONCE(dcp->ext_dead)) { + dcp->crashed = true; + dev_err(dcp->dev, "unusable external display processor has crashed\n"); + return; } - - dcp->phy = route->phy; - dcp->dptx_phy = route->dptx_phy; - dcp->connector_type = DRM_MODE_CONNECTOR_USB; - if (connector) { - WRITE_ONCE(connector->dcp, to_platform_device(dcp->dev)); - dcp->typec_connector = connector; - dcp->connector = connector; - - /* - * Narrow the port to the pipeline now driving it. The encoder - * spans every pipeline that could, which is what lets userspace - * see the port as usable at all -- but only one of them is - * routed to the display, and userspace has no way to tell which. - * Offering it the choice makes it pair the port with a pipeline - * holding a different monitor's mode list, and the modeset is - * rejected with no way for it to recover. The hotplug that - * follows publishes the actual binding on dual-stream machines - * too. Clients must re-read encoder topology after a hotplug. - */ - if (connector->port_encoder && dcp->crtc) - connector->port_encoder->possible_crtcs = - drm_crtc_mask(&dcp->crtc->base); + dcp->crashed = true; + dev_err(dcp->dev, "DCP has crashed\n"); + if (dcp->connector) { + dcp->connector->connected = 0; + apple_connector_edid_set_live(dcp->connector, false); + dcp_queue_hotplug(dcp->connector); } - dcp->active_typec_route = route; - route->selected = true; - - dev_info(dcp->dev, "allocated Type-C DPTX PHY %u\n", route->dptx_phy); - return 0; + complete(&dcp->start_done); } -static int dcp_typec_route_deactivate(struct apple_dcp_typec_route *route) +static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) { - struct apple_dcp *dcp = route->dcp; - struct apple_connector *connector = dcp->typec_connector; - struct mux_control *active_xbar = route->active_xbar; - bool was_tunnel = route->tunnel; - int ret = 0; - - guard(mutex)(&dcp->external_lock); - if (dcp_is_external_h17p(dcp) && - (dcp->external_error || (dcp->external_power_attempted && !dcp->external_stopped))) - return dcp->external_error ?: -EAGAIN; + struct apple_dcp *dcp = cookie; - /* - * Under tb_lock so a DCP link (re)configuration can neither select the - * crossbar nor restart the tunnel pixel clock behind our back. - */ - scoped_guard(mutex, &dcp->tb_lock) { - if (route->deactivate_error) - return route->deactivate_error; - if (!route->deactivate_xbar_done) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else - ret = dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - /* Deselect always unlocks: never consume that selection twice. */ - route->deactivate_xbar_done = true; - route->xbar_up = false; - if (ret) { - /* Idle is unproven; retain the owner and the real error. */ - route->deactivate_error = ret; - return ret; - } - } + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_rtk_shmem_setup_h17p(dcp, bfr); - route->selected = false; - if (dcp->active_typec_route == route) - dcp->active_typec_route = NULL; + if (bfr->iova) { + struct iommu_domain *domain = + iommu_get_domain_for_dev(dcp->dev); + phys_addr_t phy_addr; - if (connector && connector->dcp == to_platform_device(dcp->dev)) { - /* - * Until the port is activated again it has no pipeline behind - * it, and nothing can read modes or EDID from it. Report it - * disconnected for that window: leaving a connected connector - * whose ->dcp is NULL lets anything probing it in between -- - * a compositor starting up while the fabric is still settling - * -- see an output it cannot get a mode for, and give up on - * it. The new pipeline marks it connected again once the - * display has come back up on it. - */ - WRITE_ONCE(connector->connected, false); - /* hotplug work queued before this checks for it */ - WRITE_ONCE(connector->dcp, NULL); + if (!domain) + return -ENOMEM; - /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder) - connector->port_encoder->possible_crtcs = - connector->candidate_crtcs; - } - dcp->typec_connector = NULL; - dcp->connector = dcp->fixed_connector; + // TODO: get map from device-tree + phy_addr = iommu_iova_to_phys(domain, bfr->iova); + if (!phy_addr) + return -ENOMEM; - if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_USB) { - dcp->connector_type = dcp->fixed_connector_type; + // TODO: verify phy_addr, cache attribute + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WB); + if (!bfr->buffer) + return -ENOMEM; - /* - * Only hand the pipeline back to its fixed output if that output - * is live. Re-targeting the DPTX endpoint at the fixed PHY while - * nothing is attached there leaves DCP unable to train a link on a - * later Type-C target: it answers DEVICE_NOT_STARTED and every - * following DPTX call times out. Park on a Type-C PHY instead, - * for the same reason the USB-C-only case does below. - */ - if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - dcp->phy = dcp->fixed_phy; - dcp->dptx_phy = dcp->fixed_dptx_phy; - struct dcp_session_ref *session; - bool retained = false; + bfr->is_mapped = true; + dev_info(dcp->dev, + "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)phy_addr, + (uintptr_t)bfr->buffer); + } else { + bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; - mutex_lock(&dcp_retained_sessions_lock); - list_for_each_entry(session, &dcp_retained_sessions, link) { - if (dev_fwnode(session->dev) == dev_fwnode(dev)) { - retained = true; - break; - } + dev_info(dcp->dev, "shmem_setup: iova: %lx, buffer: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); } - mutex_unlock(&dcp_retained_sessions_lock); - return retained; + + return 0; } -/* - * Thunderbolt DP tunnels, ported from aurora-silicon/linux#8. Crossbar - * connection up/down, looked up at runtime so appledrm does not require the - * crossbar driver to be built. - */ -static int dcp_dpxbar_link(struct mux_control *mux, bool up) +static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) { - typeof(&apple_dpxbar_link_up) fn; - int ret; + struct apple_dcp *dcp = cookie; - fn = up ? symbol_get(apple_dpxbar_link_up) : - symbol_get(apple_dpxbar_link_down); - if (!fn) - return -ENOENT; - ret = fn(mux); - if (up) - symbol_put(apple_dpxbar_link_up); + if (bfr->is_mapped) + memunmap(bfr->buffer); else - symbol_put(apple_dpxbar_link_down); - return ret; + dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); } -/* DP IN adapter handshake through the thunderbolt glue; tb_lock held. */ -static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) +static struct apple_rtkit_ops rtkit_ops = { + .crashed = dcp_rtk_crashed, + .recv_message = dcp_recv_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + +void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) { - struct apple_dcp_typec_port *port; + int ret; - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->owner) - mod_delayed_work(system_freezable_wq, - &port->owner->dcp->typec_fabric_retrain_wq, - msecs_to_jiffies(200)); - if (port->secondary_owner) - mod_delayed_work(system_freezable_wq, - &port->secondary_owner->dcp->typec_fabric_retrain_wq, - msecs_to_jiffies(200)); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (READ_ONCE(dcp->quiescing)) + return; + trace_dcp_send_msg(dcp, endpoint, message); + apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, + true); + return; } + + trace_dcp_send_msg(dcp, endpoint, message); + /* + * The adopted 14.7 session shares this mailbox with the panel link. + * A non-sleeping send fails while that FIFO is full, and the + * DisplayPort handshake then waits for a reply that was never sent. + */ + ret = apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, + dcp->fw_compat != DCP_FIRMWARE_V_14_7 || + in_atomic()); + if (ret) + dev_warn_ratelimited(dcp->dev, "DCP send ep %02x failed: %d\n", + endpoint, ret); } -static struct apple_dcp_typec_route * -dcp_typec_port_route(struct apple_dcp_typec_port *port, struct apple_dcp *dcp) +int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) { - struct apple_dcp_typec_route *route; + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_crtc_state *crtc_state; + bool needs_modeset; - list_for_each_entry(route, &port->routes, port_link) - if (route->dcp == dcp) - return route; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_crtc_atomic_check_h17p(crtc, state); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) + return iomfb_v26_6_atomic_check(dcp, crtc, state); + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) + return iomfb_v14_7_atomic_check(dcp, crtc, state); - return NULL; -} + if (dcp->crashed) + return -EINVAL; -/* - * The DRM device once it has bound, that is once every pipeline has its - * CRTC and every port its connectors; NULL before. - */ -static struct drm_device *dcp_typec_drm(void) -{ - struct apple_dcp_typec_port *port; - struct apple_dcp_typec_route *route; - struct drm_device *drm = NULL; - - list_for_each_entry(port, &dcp_typec_ports, link) { - if (!port->connector || !port->secondary_connector) - return NULL; - list_for_each_entry(route, &port->routes, port_link) { - if (!route->dcp->crtc) - return NULL; - drm = route->dcp->crtc->base.dev; - } - } + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - return drm; -} + /* + * A Type-C display routed to another pipeline: its route follows + * this CRTC if it can (see dcp_typec_follow_check()), and the mode is + * one of the display's, which only the pipeline driving it knows. + */ + if (crtc_state->enable && drm_atomic_crtc_needs_modeset(crtc_state) && + dcp_typec_follows_crtc(dcp)) { + int ret = dcp_typec_follow_check(dcp, crtc, state, + &crtc_state->mode); -/* - * Has a compositor (or boot splash) taken the display? It paired its - * connectors with CRTCs when it started and keeps that pairing, also - * across a VT switch, where it drops DRM master but keeps the device - * open; moving routes under it would hand connectors to pipelines driving - * other displays. So the routes are frozen while any open file has ever - * been master, and thaw when the last such file is closed. - * - * Taken under dcp_typec_fabric_lock, next to the routing it decides, so - * the order is the fabric lock, then filelist_mutex. Nothing nests them - * the other way: filelist_mutex only covers list edits and walks, and - * drm_release() drops it before drm_file_free() calls postclose, which is - * where a close takes the fabric lock. drm_open() makes a file master - * before adding it here, but its owner cannot have paired anything before - * open() returns, and it re-probes on the hotplugs the moves send. - */ -static bool dcp_typec_frozen(struct drm_device *drm) -{ - struct drm_file *file; + if (ret) + return ret; + } - guard(mutex)(&drm->filelist_mutex); - list_for_each_entry(file, &drm->filelist, lhead) { - /* set under master_mutex, and only ever from false to true */ - if (READ_ONCE(file->was_master)) - return true; + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || + !READ_ONCE(dcp->mode_state.valid); + if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { + /* + * Resume restores the mode before the firmware reports the + * display back, so a plane-only commit lands here while the + * connector is still marked disconnected. Rejecting it makes + * the compositor fail every flip and give up on the output; + * dcp_flush() defers the commit until the link returns. + */ + dev_dbg(dcp->dev, + "crtc_atomic_check: deferring commit, link still down\n"); } - return false; + return 0; } -/* Are the routes kept in compositor pairing order right now? */ -static bool dcp_typec_keep_order(void) +int dcp_get_connector_type(struct platform_device *pdev) { - struct drm_device *drm; - - if (!dcp_typec_dual_stream()) - return false; - drm = dcp_typec_drm(); + struct apple_dcp *dcp = platform_get_drvdata(pdev); - return drm && READ_ONCE(drm->registered) && !dcp_typec_frozen(drm); + return dcp->fixed_connector_type; } -/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ -static bool dcp_typec_tunnel_held(struct apple_dcp *dcp) -{ - return dcp->active_typec_route && dcp->active_typec_route->tunnel; -} - -/* - * Give @dcp's pipeline to the first stream, in connector order, that - * wants one, has none yet and has it in its possible_crtcs. A tunnel - * stream wants one once it is set up, or as @arriving/@dpin. A direct - * DP-alt stream wants one once its sink asserts HPD: only then can its - * connector read connected, and a compositor pairs only those. - */ -static void dcp_typec_plan_pipeline(struct apple_dcp *dcp, struct drm_crtc *crtc, - struct apple_dcp_typec_port *arriving, - unsigned int dpin) -{ - struct apple_dcp_typec_port *port; - bool secondary = false; - - /* primary connectors in port order, then the DPIN1 ones */ - do { - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route **slot = secondary ? - &port->secondary_target : &port->target; - struct apple_connector *connector = secondary ? - port->secondary_connector : port->connector; - struct apple_dcp_typec_route *route = - dcp_typec_port_route(port, dcp); - bool wants; - - if (secondary) - wants = port->secondary_owner || - (port == arriving && dpin == 1); - else - wants = (port->owner && port->owner->tunnel) || - (port->dp_wanted && port->dp_hpd && - !port->plan_dark) || - (port == arriving && dpin == 0); +#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) +#define DPTX_TUNNEL_CONNECT_TIMEOUT msecs_to_jiffies(8000) +#define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) +#define DPTX_RECONNECT_RETRIES 5 - if (!wants || *slot || !route || - !(connector->candidate_crtcs & drm_crtc_mask(crtc))) - continue; +static struct dcp_fabric_session dcp_session_locked(struct apple_dcp *dcp) +{ + struct dcp_fabric_session session = { .generation = dcp->typec_generation }; - *slot = route; - return; - } - secondary = !secondary; - } while (secondary); + lockdep_assert_held(&dcp->hpd_mutex); + scoped_guard(mutex, &dcp->tb_lock) + session.cookie = dcp->tb_generation; + return session; } -/* - * The pairing a compositor starting now would make, wherever the streams - * sit at the moment: pipelines in CRTC index order, each to the first - * stream that wants it. A pipeline whose fixed output is live is left to - * that output, unless a tunnel holds it: the output cannot have it back - * then, and its connector reads disconnected. - * - * Tunnels are planned in connector order like any other stream, not - * seated on the pipelines they hold: the compositor pairs their connectors - * by order too, and seating them would plan the direct streams around - * pairings it never makes. A direct stream planned onto a pipeline a - * tunnel holds cannot have it and stays dark, so it is left out and the - * pass re-run, or every stream after it would be planned one pipeline - * off. Streams only ever drop out, so that settles within one pass per - * port. It also brings a tunnel's plan onto the pipeline it holds - * wherever direct streams ahead of it were in the way. A tunnel still - * planned elsewhere cannot be brought there by any direct stream dropping - * out, and the compositor pairs its connector wrongly whatever they do. - */ -static void dcp_typec_plan(struct drm_device *drm, - struct apple_dcp_typec_port *arriving, unsigned int dpin) +static bool dcp_session_valid_locked(struct apple_dcp *dcp, + struct dcp_fabric_session session, bool cable) { - struct apple_dcp_typec_port *port; - struct drm_crtc *crtc; - bool again; - - list_for_each_entry(port, &dcp_typec_ports, link) - port->plan_dark = false; - - do { - list_for_each_entry(port, &dcp_typec_ports, link) { - port->target = NULL; - port->secondary_target = NULL; - } - - /* CRTCs are listed in index order */ - drm_for_each_crtc(crtc, drm) { - struct apple_dcp *dcp = - platform_get_drvdata(to_apple_crtc(crtc)->dcp); - - if (dcp->nr_typec_routes && - (dcp_typec_tunnel_held(dcp) || - !dcp_typec_route_fixed_output_busy(&dcp->typec_routes[0]))) - dcp_typec_plan_pipeline(dcp, crtc, arriving, dpin); - } + return dcp_fabric_session_valid(session, dcp_session_locked(dcp), + !cable || dcp->typec_cable_connected, + READ_ONCE(dcp->tb_retiring) || + READ_ONCE(dcp->typec_follow_retiring)); +} - again = false; - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port == arriving || !port->target || - (port->owner && port->owner->tunnel) || - !dcp_typec_tunnel_held(port->target->dcp)) - continue; - port->plan_dark = true; - again = true; - } - } while (again); +/* Recovery must not wake a deliberately parked or disabled output. */ +static bool dcp_connect_session_valid_locked(struct apple_dcp *dcp, + struct dcp_fabric_session session, + bool cable, bool recovery) +{ + return (!recovery || (dcp->typec_work_enabled && + !READ_ONCE(dcp->typec_crtc_off) && + !READ_ONCE(dcp->crashed))) && + dcp_session_valid_locked(dcp, session, cable); } -/* Replay HPD to the pipeline a direct DP-alt port has just been given. */ -static void dcp_typec_port_attach(struct apple_dcp_typec_port *port) +/* Capture at enqueue, not when a stale worker eventually starts running. */ +void dcp_queue_typec_reconnect(struct apple_dcp *dcp, unsigned long delay) { - struct apple_dcp *dcp = port->owner->dcp; + guard(mutex)(&dcp->hpd_mutex); - port->hpd = port->dp_hpd; - if (!port->hpd) + if (!dcp->typec_cable_connected || READ_ONCE(dcp->tb_retiring) || + READ_ONCE(dcp->typec_follow_retiring)) return; - - WRITE_ONCE(dcp->typec_cable_connected, true); - if (dcp->typec_connector) - dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + dcp->typec_reconnect_session = dcp_session_locked(dcp); + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, delay); } -static struct apple_dcp_typec_route * -dcp_typec_lowest_free(struct apple_dcp_typec_port *port) +static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, + struct dcp_fabric_session session, bool cable, + bool recovery) { - struct apple_dcp_typec_route *candidate, *best = NULL; - unsigned int best_score = UINT_MAX; - - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; + /* T8122 dcpext: HPD only after the link wait, see dcp_ext_dptx_607() */ + bool late_hpd = dcp_ext_dptx_607(dcp); + unsigned long timeout; + u8 dfp_port; + int ret = 0; - if (!dcp_typec_route_available(candidate)) - continue; - /* - * Prefer the lowest free CRTC index for a stable allocation order. - */ - score = dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; - } + if (!dcp->phy) { + dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); + return -ENODEV; } + /* A native external processor starts here; its start retries this. */ + if (dcp->external_native && !iomfb_v14_7_external_ready(dcp)) + return -EAGAIN; + /* @port selects the upstream RemotePort service/core. dptx_dfp_port + * is the downstream address: dpphy=0, dpin0=1, dpin1=2. + */ + dev_info(dcp->dev, + "%s(port=%d) die=%u atc=%u dfp_port=%u tunnel=%d typec=%d route=%s conn_type=%d connected=%d\n", + __func__, port, dcp->dptx_die, dcp->dptx_phy, dcp->dptx_dfp_port, + dcp->dptx_tunnel, dcp_is_typec_output(dcp), + dcp->active_typec_route ? "borrowed" : "fixed", + dcp->connector_type, dcp->dptxport[port].connected); - return best; -} - -/* - * Route the direct DP-alt ports left waiting for a pipeline. Their DP - * state is not reported again while it stays the same, so nothing else - * would. Each goes back to the pipeline it last had if that is free, as - * a compositor keeps a reconnected connector's CRTC, and otherwise takes - * the lowest free one, the CRTC a compositor gives a new connector. - */ -static void dcp_typec_route_waiting(void) -{ - struct apple_dcp_typec_port *port; - struct apple_dcp_typec_route *route; + mutex_lock(&dcp->hpd_mutex); + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } + if (dcp->external && dcpext_scanout_terminal(dcp)) { + mutex_unlock(&dcp->hpd_mutex); + return -ESHUTDOWN; + } + if (!dcp->dptxport[port].enabled) { + dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); + ret = -ENODEV; + goto out_unlock; + } - list_for_each_entry(port, &dcp_typec_ports, link) { - if (port->owner || !port->dp_wanted || !port->dp_hpd) - continue; + if (dcp->dptxport[port].connected) + goto out_unlock; + if (dcp->external) + smp_store_release(&dcp->external_link_ready, false); - route = port->preferred_route; - if (!route || !dcp_typec_route_available(route)) - route = dcp_typec_lowest_free(port); - if (!route || dcp_typec_route_activate(route, route->xbar)) - continue; - port->owner = route; + reinit_completion(&dcp->dptxport[port].linkcfg_completion); + WRITE_ONCE(dcp->dptxport[port].tile_hint, false); + dcp->dptxport[port].atcphy = dcp->phy; + /* a tiled display's second half comes in on the port's dpin1 */ + dfp_port = port && READ_ONCE(dcp->split.active) ? 2 : dcp->dptx_dfp_port; + if (!dcp_ext_dptx_607(dcp)) { + ret = dptxport_validate_connection(dcp->dptxport[port].service, + dfp_port, + dcp->dptx_phy, dcp->dptx_die); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); + goto out_unlock; + } + } - dcp_typec_port_attach(port); + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } + ret = dptxport_connect(dcp->dptxport[port].service, + dfp_port, + dcp->dptx_phy, dcp->dptx_die, + dcp_is_typec_output(dcp) && !dcp_ext_dptx_607(dcp)); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); + goto out_unlock; } -} -/* - * Prefer connector order when placing Type-C streams before a DRM master - * starts. This is an allocation policy, not a promise about the CRTC that - * userspace must choose: each connected encoder advertises only its actual - * route, and atomic encoder selection checks that ownership. Route changes - * publish the new mask through hotplug; the plan uses candidate_crtcs so - * narrowing the advertised mask does not remove hardware routing choices. - * - * Only direct DP-alt routes move: each goes to exactly its planned - * pipeline. A Thunderbolt tunnel never moves once set up, so where one - * sits on a pipeline planned for a direct stream, or the plan has nothing - * for it, that stream stays unrouted and its connector disconnected. - * Placing it on some other pipeline would have the compositor cross both - * displays; dark is the better failure. @arriving/@dpin is a tunnel - * stream asking for a pipeline, planned like any other; its route is - * returned for apple_dcp_tb_dp_tunnel() to set up, or NULL if the plan - * has none for it or another tunnel holds that one. - * - * A move is an unplug and replug. Every route that moves is taken down - * first, so that two never share a pipeline midway, then each goes up on - * its new pipeline with its HPD replayed; the hotplugs this sends make - * fbdev and userspace re-probe. - */ -static struct apple_dcp_typec_route * -dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, - unsigned int dpin) -{ - struct drm_device *drm = dcp_typec_drm(); - struct apple_dcp_typec_route *planned = NULL; - struct apple_dcp_typec_port *port; - - lockdep_assert_held(&dcp_typec_fabric_lock); - - dcp_typec_plan(drm, arriving, dpin); - if (arriving) { - planned = dpin ? arriving->secondary_target : arriving->target; - /* it will be refused: plan as if it had not asked */ - if (planned && dcp_typec_tunnel_held(planned->dcp)) { - planned = NULL; - dcp_typec_plan(drm, NULL, 0); + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } + ret = dptxport_request_display(dcp->dptxport[port].service); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to request display: %d\n", + ret); + goto out_release; + } + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_release; + } + dcp->dptxport[port].connected = true; + if (dcp_is_typec_output(dcp) && !late_hpd) { + if (dcp_uses_t6020_tunnel_flow(dcp)) + ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, + true, 8000); + else + ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to assert Type-C HPD: %d\n", + ret); + dcp->dptxport[port].connected = false; + goto out_release; } } - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route *owner = port->owner; - struct apple_dcp *dcp; - - if (port->safe_pending) - continue; - if (!owner || owner->tunnel || owner == port->target) - continue; - - dcp = owner->dcp; - dev_info(dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", - port->connector_np, dev_name(dcp->dev), - port->target ? dev_name(port->target->dcp->dev) : "none"); - if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - if (dcp_typec_route_deactivate(owner) && owner->selected) { - /* still routed: leave the display where it is */ - dcp_typec_port_attach(port); - continue; + mutex_unlock(&dcp->hpd_mutex); + /* + * The display interface of a native external processor opens once a + * display is attached; the firmware then describes that display. + */ + if (dcp->external_native) { + ret = iomfb_v14_7_external_open(dcp); + if (ret) { + mutex_lock(&dcp->hpd_mutex); + goto out_disconnect; } - port->owner = NULL; + } + timeout = dcp_uses_t6020_tunnel_flow(dcp) ? + DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; + ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, + timeout); + mutex_lock(&dcp->hpd_mutex); + /* A revoked wait must not disconnect or publish readiness for a new session. */ + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } + if (!ret && late_hpd) { + /* Without HPD this firmware does not train: assert it now. */ + dev_dbg(dcp->dev, + "dcp_dptx_connect: no link configuration on port %u yet, asserting HPD\n", + port); + } else if (!ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: timed out waiting for port %u link configuration\n", + port); + ret = -ETIMEDOUT; + goto out_disconnect; + } else { + dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", + jiffies_to_msecs(timeout - ret)); + } - /* as after a DP exit, hand the hybrid back to a live HDMI */ - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); + usleep_range(5, 10); + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; } - list_for_each_entry(port, &dcp_typec_ports, link) { - struct apple_dcp_typec_route *route = port->target; + if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort || late_hpd) { + ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + if (ret && late_hpd) + dev_err(dcp->dev, + "dcp_dptx_connect: failed to assert HPD after the link wait: %d\n", + ret); + if (ret && (dcp->external || late_hpd)) + goto out_disconnect; + } + if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { + ret = -ESTALE; + goto out_unlock; + } + if (dcp->external) { + /* + * The port must still have its display: a Type-C cable, or HPD + * on the fixed HDMI output, which has no cable flag. + */ + if (!dcp->dptxport[port].connected || + !(READ_ONCE(dcp->typec_cable_connected) || dcp_hdmi_port_attached(dcp)) || + READ_ONCE(dcp->crashed) || dcpext_scanout_terminal(dcp)) { + ret = -ENOLINK; + goto out_unlock; + } + smp_store_release(&dcp->external_link_ready, true); + dcpext_scanout_link_restored(dcp); + } - if (port->owner || !port->dp_wanted || !port->dp_hpd || - port == arriving) - continue; + if (dcp->avep) + av_service_connect(dcp); - if (!route || !dcp_typec_route_available(route) || - dcp_typec_route_activate(route, route->xbar)) { - /* - * Dark for now. Its unchanged DP state is not reported - * again, so it is placed when the plan next runs, or - * once a compositor owns the display, when a pipeline - * is freed (dcp_typec_route_waiting()). - */ - port->applied_valid = false; - continue; - } - port->owner = route; - port->preferred_route = route; + ret = 0; + goto out_unlock; - dcp_typec_port_attach(port); - } +out_disconnect: + dcp->dptxport[port].connected = false; +out_release: + dptxport_release_display(dcp->dptxport[port].service); - return planned; +out_unlock: + mutex_unlock(&dcp->hpd_mutex); + return ret; } /* - * A route has let its pipeline go. Until a compositor owns the display - * the plan places everything anew; after that the pipeline goes to a port - * left waiting for one. + * A T6030 external processor runs as a pipe of the main DRM device when the + * display gate handed it its display DART: a piodma child on stream 4 of + * that DART, and stream 0 among the display subsystem's iommus, so that + * framebuffers are mapped for it as for the panel. */ -static void dcp_typec_pipeline_freed(void) +bool dcp_t6030_ext_native(const struct device_node *np) { - if (!dcp_typec_dual_stream()) - return; + struct of_phandle_args dart, args; + struct device_node *piodma, *display; + bool native = false; + u32 marker; + int i, n; - if (dcp_typec_keep_order()) - dcp_typec_rebalance_locked(NULL, 0); - else - dcp_typec_route_waiting(); + if (!of_device_is_compatible(np, "apple,t6030-dcpext")) + return false; + piodma = of_get_child_by_name(np, "piodma"); + if (!piodma || !of_device_is_available(piodma) || + of_property_read_u32(piodma, "apple,t6030-dispext-handoff", &marker) || + marker != 1 || + of_parse_phandle_with_args(piodma, "iommus", "#iommu-cells", 0, &dart)) { + of_node_put(piodma); + return false; + } + of_node_put(piodma); + if (dart.args_count != 1 || dart.args[0] != 4 || !of_device_is_available(dart.np)) + goto out; + display = of_find_compatible_node(NULL, NULL, "apple,t6030-display-subsystem"); + n = display && of_device_is_available(display) ? + of_count_phandle_with_args(display, "iommus", "#iommu-cells") : 0; + for (i = 0; i < n && !native; i++) { + if (of_parse_phandle_with_args(display, "iommus", "#iommu-cells", i, &args)) + break; + native = args.np == dart.np && args.args_count == 1 && !args.args[0]; + of_node_put(args.np); + } + of_node_put(display); +out: + of_node_put(dart.np); + return native; } /* - * Re-run the pairing pass from outside the fabric: once DRM is registered, - * as ports routed before that could not follow it, and when the last - * compositor or boot splash has closed the device, as the next one pairs - * the connectors from scratch. + * A native external processor is running, or has announced a DPTX port: + * connect the display that is already waiting for it. */ -void dcp_typec_reorder(void) +void dcp_external_ready(struct apple_dcp *dcp) { - if (!dcp_typec_dual_stream()) - return; - - guard(mutex)(&dcp_typec_fabric_lock); + if (READ_ONCE(dcp->typec_cable_connected)) { + dcp->typec_reconnect_tries = 0; + dcp_queue_typec_reconnect(dcp, 0); + } else if (dcp->hdmi_hpd && dcp->active && + gpiod_get_value_cansleep(dcp->hdmi_hpd)) { + int ret = dcp_dptx_connect(dcp, 0); - if (dcp_typec_keep_order()) - dcp_typec_rebalance_locked(NULL, 0); - lockdep_assert_held(&dcp->tb_lock); - if (!dcp->tb_dpin_set_active) - return -ENODEV; - return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); + if (ret && ret != -EAGAIN && ret != -ESTALE && ret != -ESHUTDOWN) + dcp_external_retry(dcp, "HDMI display link not set up", ret, 1000); + } } -static int dcp_typec_route_prepare(struct typec_mux_dev *mux, - struct typec_mux_state *state) +#define DCP_EXTERNAL_RETRIES 3 + +static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port); + +/* + * Bounded recovery for a native external pipe: re-apply the display mode, + * or redo the display link, up to DCP_EXTERNAL_RETRIES times per attached + * display, @base_ms, then twice and four times as long apart. A stopped + * firmware session cannot be recovered, and is not retried. + */ +void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, + unsigned int base_ms) { - struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); - struct apple_dcp_typec_port *port = route->port; - struct apple_dcp *dcp; - int ret; + unsigned int n, delay; - guard(mutex)(&dcp_typec_fabric_lock); - if (state->mode != TYPEC_STATE_SAFE) - return port->safe_pending ? -EAGAIN : 0; - if (!port->owner || port->owner->tunnel) - return 0; - dcp = port->owner->dcp; - /* Stop new admission, but retain the owner until the set phase succeeds. */ - if (!port->safe_pending) - port->safe_finish_ready = false; - port->safe_pending = true; - port->applied_valid = false; - port->dp_wanted = false; - port->dp_hpd = false; - /* HPD absence alone is not proof of a completed firmware shutdown. */ - if (dcp_is_external_h17p(dcp)) { - scoped_guard(mutex, &dcp->external_lock) { - if (dcp->external_error) - return dcp->external_error; - if (!dcp->external_cable && dcp->external_stopped && - !READ_ONCE(dcp->dptxport[0].connected)) { - port->hpd = false; - return 0; - } - } + if (!dcp->external_native) + return; + if (iomfb_v14_7_external_failed(dcp)) { + dev_err(dcp->dev, "%s (%d): the external display processor stopped; no retry until reboot\n", + why, error); + return; } - ret = dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - if (ret) - return ret; - port->hpd = false; - return 0; + if (delayed_work_pending(&dcp->external_retry_wq)) { + dev_info(dcp->dev, "%s (%d): a retry is already queued\n", why, error); + return; + } + n = atomic_inc_return(&dcp->external_retries); + if (n > DCP_EXTERNAL_RETRIES) { + atomic_set(&dcp->external_retries, DCP_EXTERNAL_RETRIES); + dev_warn(dcp->dev, "%s (%d): no retry left after %u; replug the display\n", + why, error, DCP_EXTERNAL_RETRIES); + return; + } + delay = base_ms << (n - 1); + dev_info(dcp->dev, "%s (%d): retry %u of %u in %u ms\n", why, error, n, + DCP_EXTERNAL_RETRIES, delay); + /* An attach or detach after this makes it stale; see the work. */ + WRITE_ONCE(dcp->external_retry_generation, READ_ONCE(dcp->typec_generation)); + mod_delayed_work(system_freezable_wq, &dcp->external_retry_wq, msecs_to_jiffies(delay)); } -static int dcp_typec_route_finish(struct typec_mux_dev *mux, - struct typec_mux_state *state) +/* The active CRTC's mode is the one the firmware runs and shows swaps in. */ +static bool dcp_external_crtc_showing(struct apple_dcp *dcp) { - struct apple_dcp_typec_port *port = - ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; - int ret; + struct drm_crtc *crtc = dcp->crtc ? &dcp->crtc->base : NULL; + bool showing = false; - guard(mutex)(&dcp_typec_fabric_lock); - if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending || - port->safe_finish_ready) - return 0; - ret = port->owner ? dcp_typec_route_deactivate(port->owner) : 0; - if (ret) - return ret; - port->safe_finish_ready = true; - return 0; + if (!crtc) + return false; + drm_modeset_lock(&crtc->mutex, NULL); + if (crtc->state && crtc->state->active) + showing = iomfb_v14_7_external_showing(dcp, &crtc->state->mode); + drm_modeset_unlock(&crtc->mutex); + return showing; } -static void dcp_typec_route_commit(struct typec_mux_dev *mux, - struct typec_mux_state *state) +void dcp_external_retry_work(struct work_struct *work) { - struct apple_dcp_typec_port *port = - ((struct apple_dcp_typec_route *)typec_mux_get_drvdata(mux))->port; + struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, + external_retry_wq); + struct apple_connector *connector = READ_ONCE(dcp->connector); + u64 generation = READ_ONCE(dcp->external_retry_generation); + bool released; - guard(mutex)(&dcp_typec_fabric_lock); - if (state->mode != TYPEC_STATE_SAFE || !port->safe_pending) + if (READ_ONCE(dcp->typec_generation) != generation) { + dev_info(dcp->dev, "display retry: the display was attached or detached since; dropped\n"); return; - if (port->owner) - port->preferred_route = port->owner; - port->owner = NULL; - port->safe_pending = false; - port->safe_finish_ready = false; - port->applied_alt = state->alt; - port->applied_mode = state->mode; - port->applied_status = 0; - port->applied_conf = 0; - port->applied_valid = true; - dcp_typec_pipeline_freed(); -} - -static int dcp_typec_route_set(struct typec_mux_dev *mux, - struct typec_mux_state *state) -{ - struct apple_dcp_typec_route *route; - int ret; - struct apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); - struct apple_dcp_typec_port *port = route->port; - struct apple_dcp_typec_route *best = NULL; - bool is_dp = dcp_typec_route_is_dp(state); - struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; - u32 dp_status = dp_data ? dp_data->status : 0; - u32 dp_conf = dp_data ? dp_data->conf : 0; - bool hpd, was_counted; - int ret = 0; - - guard(mutex)(&dcp_typec_fabric_lock); - - if (state->mode == TYPEC_STATE_SAFE && port->safe_pending) - return 0; - if (port->safe_pending) - return -EAGAIN; - - /* - * Every candidate route for this port is notified with the same state, - * so only the first to arrive does the work; the others return early. - * - * Deliberately not a nominated coordinator: fwnode_typec_mux_get() - * caps the providers one connector may have and drops the remainder - * without a word, so a nominated route might never be called at all -- - * and the port would then never be routed. - */ - if (port->applied_valid && port->applied_alt == state->alt && - port->applied_mode == state->mode && - port->applied_status == dp_status && port->applied_conf == dp_conf) - return 0; - - port->applied_alt = state->alt; - port->applied_mode = state->mode; - port->applied_status = dp_status; - port->applied_conf = dp_conf; - /* Failed route acquisition must remain retryable on the next update. */ - port->applied_valid = false; - - /* did the pairing pass count this port's direct stream so far? */ - was_counted = port->dp_wanted && port->dp_hpd; - - if (!is_dp) { - port->dp_wanted = false; - port->dp_hpd = false; - - /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ - if (port->owner && port->owner->tunnel) { - port->applied_valid = true; - return 0; + } + /* Kept connected through a withdrawal, and not described again. */ + released = iomfb_v14_7_external_release(dcp); + if (iomfb_v14_7_external_failed(dcp)) { + dev_err(dcp->dev, "display retry: the external display processor stopped; no retry until reboot\n"); + return; + } + /* Described again: re-apply the mode if it is not set. */ + if (!released && connector && READ_ONCE(connector->connected)) { + if (READ_ONCE(dcp->mode_state.valid)) { + dev_info(dcp->dev, "display retry: the display is back and set\n"); + return; } - if (port->owner) { - struct apple_dcp *dcp = port->owner->dcp; - - port->preferred_route = port->owner; - if (port->hpd || dcp->typec_cable_connected || - (dcp->typec_connector && - dcp->typec_connector->connected)) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(port->owner); - if (ret) - return ret; - port->owner = NULL; - - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - dcp_typec_pipeline_freed(); - } else if (was_counted && dcp_typec_keep_order()) { - /* a stream the plan left dark is gone: plan the rest */ - dcp_typec_rebalance_locked(NULL, 0); + if (dcp_external_crtc_showing(dcp)) { + dev_info(dcp->dev, "display retry: the display is back in its mode and showing\n"); + dcp_mode_set_valid(&dcp->mode_state, true); + return; } - - /* Data-only USB4 changes do not invalidate other display routes. */ - port->applied_valid = true; - return 0; + dev_info(dcp->dev, "display retry: setting the display mode again\n"); + dcp_mode_invalidate(&dcp->mode_state); + dcp_queue_hotplug(connector); + return; } - - /* - * A Thunderbolt DP tunnel still owns the port (its teardown is on the - * way): don't act on or remember this state; the tunnel teardown drops - * what was recorded so the next update is applied. - */ - if (port->owner && port->owner->tunnel) { - port->applied_valid = false; - return 0; + /* An HDMI output connects through the fabric, as on its HPD. */ + if (!dcp_is_typec_output(dcp) && dcp->hdmi_hpd) { + dcp_fabric_hdmi_retry(dcp); + return; + } + if (!READ_ONCE(dcp->typec_cable_connected)) { + dev_info(dcp->dev, "display retry: the port has no display any more\n"); + return; } - - hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); - port->dp_wanted = true; - port->dp_hpd = hpd; - /* - * Until a compositor owns the display, a direct DP-alt stream has a - * pipeline only while its sink asserts HPD, and the pairing pass - * places it: HPD coming or going is the stream connecting or - * disconnecting as far as a compositor can tell. Otherwise the port - * takes the lowest free pipeline on DP entry, as a compositor does. + * The port holds a display that the firmware no longer describes: + * release the link if it is still up, and connect it again, which + * makes the firmware describe the display anew. */ - if (dcp_typec_keep_order()) { - struct apple_dcp_typec_route *owner = port->owner; + scoped_guard(mutex, &dcp->hpd_mutex) { + /* Not under a connect or release of another display. */ + if (dcp->typec_generation != generation || !dcp->typec_cable_connected) { + dev_info(dcp->dev, "display retry: the display was attached or detached since; dropped\n"); + return; + } + dev_info(dcp->dev, "display retry: reconnecting the display link\n"); + if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + int ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - guard(mutex)(&dcp->tb_lock); - route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar) - return -ENODEV; - if (!dcp->tb_clock_ok) { - dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); - return -EIO; - } - if (!route->xbar_up) { - /* Never block a DCP call on the mux semaphore. */ - ret = mux_control_try_select(route->active_xbar, route->mux_index); - if (!ret) - route->xbar_up = true; - } else { - ret = dcp_dpxbar_link(route->active_xbar, true); - } - if (ret) { - dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); - return ret; + if (ret) { + dev_warn(dcp->dev, "display retry: HPD deassert failed: %d\n", ret); + return; + } + ret = dcp_dptx_release_locked(dcp, 0); + if (ret) + return; + } + dcp->typec_reconnect_tries = 0; } - return dcp_tunnel_dpin_locked(dcp, true); + dcp_queue_typec_reconnect(dcp, 0); } /* - * Crossbar connection up/down, looked up at runtime so appledrm does not - * require the crossbar driver to be built. + * The sink of a native external pipe raised IRQ_HPD with HPD staying high, + * as a DP branch device does when the display behind it changes. Pass the + * request to the firmware, which services it as a DP source does. If it + * cannot take the request, connect the display link anew, which makes the + * firmware read the display again. */ -static int dcp_dpxbar_link(struct mux_control *mux, bool up) +void dcp_external_sink_irq(struct apple_dcp *dcp) { - typeof(&apple_dpxbar_link_up) fn; + struct platform_device *pdev = to_platform_device(dcp->dev); + struct apple_epic_service *service = READ_ONCE(dcp->dptxport[0].service); int ret; - fn = up ? symbol_get(apple_dpxbar_link_up) : - symbol_get(apple_dpxbar_link_down); - if (!fn) - return -ENOENT; - ret = fn(mux); - if (up) - symbol_put(apple_dpxbar_link_up); - else - symbol_put(apple_dpxbar_link_down); - return ret; + /* With no link up, the next connect reads the display anyway. */ + if (!dcp->external_native || !service || !READ_ONCE(dcp->dptxport[0].enabled) || + !READ_ONCE(dcp->dptxport[0].connected)) + return; + ret = dptxport_sink_irq(service); + if (!ret) { + dev_info(dcp->dev, "display sink IRQ_HPD passed to the firmware\n"); + return; + } + dev_info(dcp->dev, "display sink IRQ_HPD not passed (%d): connecting the display anew\n", + ret); + ret = dcp_dptx_disconnect_oob(pdev, 0); + if (ret) + return; + dcp_dptx_connect_oob(pdev, 0); } -static int dcp_dpxbar_preselect(struct mux_control *mux, int state) +/* + * tiled_split: DPTX port 1 carries the second half of a tiled display on + * this pipeline. Bring it back whenever port 0 (re)connects, as DCP pairs + * the tiles only while both ports are connected. + */ +static int dcp_dptx_connect_tile(struct apple_dcp *dcp) { - typeof(&apple_dpxbar_preselect) fn = symbol_get(apple_dpxbar_preselect); + bool split; int ret; - if (!fn) - return -ENOENT; - ret = fn(mux, state); - symbol_put(apple_dpxbar_preselect); + scoped_guard(mutex, &dcp->tb_lock) + split = dcp->split.active && dcp->split.ready && + dcp->split.generation; + if (!split) + return 0; + ret = dcp_dptx_connect(dcp, 1); + if (ret) + dev_warn(dcp->dev, "tiled: DPTX port 1 reconnect failed: %d\n", ret); return ret; } -/* - * Thunderbolt DP IN, before DCP hears about the display: point the DP IN - * output at this pipeline and wake the ATC's DP clock path. The crossbar - * connection itself still only comes up in dcp_tunnel_crossbar_up(), but an - * output left at its idle source (dispext0) only suits the pipeline that is - * dispext0: any other one activates the link, gets no answer from the sink - * and gives up with DEVICE_NOT_RESPONDING before it ever sets a link rate. - */ -static void dcp_tunnel_prepare(struct apple_dcp_typec_route *route, - struct mux_control *xbar) +int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) { - typeof(&apple_atc_dp_tunnel_open) open; - struct apple_dcp *dcp = route->dcp; + struct dcp_fabric_session session; int ret; - ret = dcp_dpxbar_preselect(xbar, route->mux_index); - if (ret && ret != -EOPNOTSUPP) - dev_warn(dcp->dev, "DP tunnel crossbar preselect failed: %d\n", ret); - - open = symbol_get(apple_atc_dp_tunnel_open); - if (!open) - return; - ret = open(dcp->phy); - symbol_put(apple_atc_dp_tunnel_open); - if (ret && ret != -EOPNOTSUPP) - dev_warn(dcp->dev, "DP tunnel PHY open failed: %d\n", ret); -} + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_dptx_connect_h17p(dcp, port); -/* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ -static int dcp_tunnel_dpin_locked(struct apple_dcp *dcp, bool active) -{ - lockdep_assert_held(&dcp->tb_lock); - if (!dcp->tb_dpin_set_active) - return -ENODEV; - return dcp->tb_dpin_set_active(dcp->tb_dpin_ctx, active); + scoped_guard(mutex, &dcp->hpd_mutex) + session = dcp_session_locked(dcp); + ret = dcp_dptx_connect_session(dcp, port, session, !!session.cookie, false); + if (!ret && !port) + ret = dcp_dptx_connect_tile(dcp); + return ret; } -/* - * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate - * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that - * the tunnel pixel clock runs, and re-assert the DP IN adapter's - * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; after - * a re-link only the clocks are brought back (the mux selection and the ATC - * output enable are kept, see dcp_tunnel_crossbar_down()). - * Runs inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. - */ -int dcp_tunnel_crossbar_up(struct apple_dcp *dcp) +static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) { - struct apple_dcp_typec_route *route; - int ret; + const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); + static const u8 name[] = "Non-PnP"; + unsigned int i; - guard(mutex)(&dcp->tb_lock); - route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar || - route->deactivate_xbar_done) - return -ENODEV; - if (!dcp->tb_clock_ok) { - dev_warn(dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); - return -EIO; - } - if (!route->xbar_up) { - /* never block a DCP call on the mux semaphore */ - ret = mux_control_try_select(route->active_xbar, route->mux_index); - if (!ret) - route->xbar_up = true; - } else { - ret = dcp_dpxbar_link(route->active_xbar, true); - } - if (ret) { - dev_warn(dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); - return ret; + if (!raw) + return false; + + /* EDID contains interior NUL bytes, so this cannot use strnstr(). */ + for (i = 0; i + sizeof(name) - 1 <= sizeof(struct edid); i++) { + if (!memcmp(raw + i, name, sizeof(name) - 1)) + return true; } - return dcp_tunnel_dpin_locked(dcp, true); + + return false; } -/* - * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an established - * link: DP IN inactive, crossbar clocks down (mux selection and ATC output - * enable kept). DP IN goes active again in dcp_tunnel_crossbar_up(). - */ -int dcp_tunnel_crossbar_down(struct apple_dcp *dcp) +void dcp_retry_placeholder_edid(struct apple_dcp *dcp, + const struct drm_edid *drm_edid) { - struct apple_dcp_typec_route *route; + guard(mutex)(&dcp->hpd_mutex); + if (!dcp_is_typec_output(dcp) || !dcp->typec_cable_connected || + dcp->placeholder_retried) + return; + if (!dcp_edid_is_placeholder(drm_edid)) + return; - guard(mutex)(&dcp->tb_lock); - route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->xbar_up) - return 0; - dcp_tunnel_dpin_locked(dcp, false); - return dcp_dpxbar_link(route->active_xbar, false); + dcp->placeholder_retried = true; + dcp->placeholder_generation = dcp->typec_generation; + schedule_delayed_work(&dcp->placeholder_edid_wq, msecs_to_jiffies(300)); } -/* - * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the - * tunnel pixel clock. A stopped clock also takes the crossbar connection down. - */ -int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) +static void dcp_placeholder_edid_work(struct work_struct *work) { - typeof(&apple_atc_dp_tunnel_rate) fn; - struct apple_dcp_typec_route *route; + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + placeholder_edid_wq); + struct apple_epic_service *service; + u64 generation; int ret; - guard(mutex)(&dcp->tb_lock); - if (!dcp->dptx_tunnel) - return -ENODEV; - fn = symbol_get(apple_atc_dp_tunnel_rate); - if (!fn) { - dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); - return -ENOENT; - } - route = dcp->active_typec_route; - if (!route) { - symbol_put(apple_atc_dp_tunnel_rate); - return -ENODEV; + mutex_lock(&dcp->hpd_mutex); + generation = dcp->placeholder_generation; + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + !dcp->dptxport[0].enabled || generation != dcp->typec_generation) + goto out_unlock; + service = dcp->dptxport[0].service; + + /* + * Some adapters answer the first connection with a 1024x768 + * placeholder and publish the panel EDID only after HPD drops + * and returns. One pulse; a second placeholder is left alone. + */ + ret = dptxport_set_hpd(service, false); + if (ret) { + dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", + ret); + goto out_unlock; } - if (!link_rate && route->xbar_up) - dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, route->tunnel_dpin, link_rate); - symbol_put(apple_atc_dp_tunnel_rate); - dcp->tb_clock_ok = !ret && link_rate; + mutex_unlock(&dcp->hpd_mutex); + + msleep(1000); + + mutex_lock(&dcp->hpd_mutex); + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + generation != dcp->typec_generation) + goto out_unlock; + + ret = dptxport_set_hpd(service, true); if (ret) - dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", - link_rate, ret); - return ret; + dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", + ret); +out_unlock: + mutex_unlock(&dcp->hpd_mutex); } -/* Thunderbolt DP IN: DCP Activate/Deactivate */ -int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +static void dcp_typec_reconnect_work(struct work_struct *work) { - guard(mutex)(&dcp->tb_lock); - if (!dcp->dptx_tunnel) - return 0; - return dcp_tunnel_dpin_locked(dcp, active); -} - -/* - * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the - * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt - * connection manager has set up a tunnel from one of them, route a free - * display pipeline there and tell DCP a display is attached, so it trains - * the link (and completes DPRX) through the tunnel. - */ -int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, - bool active, int (*set_active)(void *ctx, bool active), - void *ctx) -{ - struct apple_dcp_typec_port *port = NULL, *pos; - struct apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; - struct apple_dcp_typec_route **slot; - unsigned int best_score = UINT_MAX; - struct mux_control *ctl; - struct apple_dcp *dcp; - bool ordered; + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + typec_reconnect_wq); + struct dcp_fabric_session session; int ret; - if (!connector_np || dpin > 1) - return -EINVAL; - - guard(mutex)(&dcp_typec_fabric_lock); - - list_for_each_entry(pos, &dcp_typec_ports, link) { - if (pos->connector_np == connector_np) { - port = pos; - break; - } + scoped_guard(mutex, &dcp->hpd_mutex) { + session = dcp->typec_reconnect_session; + if ((dcp->external && dcpext_scanout_terminal(dcp)) || + !dcp_session_valid_locked(dcp, session, true)) + return; } - if (!port) - return -ENODEV; - slot = dpin ? &port->secondary_owner : &port->owner; - - if (!active) { - if (!*slot || !(*slot)->tunnel || - (*slot)->tunnel_dpin != dpin) - return 0; - dcp = (*slot)->dcp; - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - ret = dcp_typec_route_deactivate(*slot); - if (ret) { - /* the caller's context is going away regardless */ - scoped_guard(mutex, &dcp->tb_lock) { - dcp->dptx_tunnel = false; - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - return ret; - } - *slot = NULL; - port->hpd = !!(port->owner || port->secondary_owner); - /* re-apply the next Type-C mux state in full */ - port->applied_valid = false; - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - dcp_typec_pipeline_freed(); - return 0; + ret = dcp_dptx_connect_session(dcp, 0, session, true, false); + if (!ret) + ret = dcp_dptx_connect_tile(dcp); + guard(mutex)(&dcp->hpd_mutex); + if (!dcp_session_valid_locked(dcp, session, true)) + return; + if (!ret) { + dcp->typec_reconnect_tries = 0; + return; } - - if (*slot) { - if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) - return 0; - dev_warn((*slot)->dcp->dev, - "port already routed, not taking DP tunnel dpin%u\n", dpin); - return -EBUSY; + if (++dcp->typec_reconnect_tries < + (dcp_uses_t6020_tunnel_flow(dcp) ? 1 : DPTX_RECONNECT_RETRIES)) { + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + return; } - if (port->owner && !port->owner->tunnel) - return -EBUSY; + dev_err(dcp->dev, "Type-C DPTX reconnect failed after %u retries: %d\n", + dcp->typec_reconnect_tries, ret); +} - /* - * Until a compositor owns the display, the pairing pass decides where - * the stream goes, moving direct DP-alt routes out of its way, and - * whether it gets a pipeline at all. - */ - ordered = dcp_typec_keep_order(); - if (ordered) - planned = dcp_typec_rebalance_locked(port, dpin); +static void disconnected_hpd_event(struct apple_connector *con) +{ + apple_connector_edid_set_live(con, false); + if (con && con->connected) { + struct platform_device *pdev = READ_ONCE(con->dcp); - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; + if (pdev) { + struct apple_dcp *dcp = platform_get_drvdata(pdev); - if (ordered && candidate != planned) - continue; - if (!dcp_typec_route_available(candidate)) - continue; - if (!dcp_typec_route_fits(candidate, dpin ? - port->secondary_connector : - port->connector)) - continue; - score = dcp_typec_route_score(candidate); + WRITE_ONCE(dcp->ext_backlight, false); + } + con->connected = 0; + drm_kms_helper_connector_hotplug_event(&con->base); /* - * DPIN0 prefers the hybrid dcpext0, which completes tunneled - * link training. On dual-stream machines DPIN1 is confined to - * dcpext1 by its connector's possible_crtcs, so both pipelines - * drive independent streams through one dock. + * Drop the display's backlight, outside the caller's locks. Not + * the hotplug work, which would send a second hotplug event. */ - if (dcp_typec_dual_stream() && dpin == 0 && - !candidate->dcp->fixed_phy && score < UINT_MAX - 100) - score += 100; - if (score < best_score) { - best = candidate; - best_score = score; - } - } - if (!best) { - ret = -EBUSY; - goto err_reorder; + schedule_work(&con->bl_sync_wq); } +} - /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ - if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) { - ret = -EOPNOTSUPP; - goto err_reorder; - } - ctl = &best->xbar->chip->mux[1 + dpin]; +static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) +{ + int ret; - dcp = best->dcp; - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = set_active; - dcp->tb_dpin_ctx = ctx; + lockdep_assert_held(&dcp->hpd_mutex); + if (dcp->external) { + smp_store_release(&dcp->external_link_ready, false); + dcpext_scanout_invalidate(dcp); } - best->tunnel_dpin = dpin; - ret = dcp_typec_route_activate(best, ctl); - if (ret) { - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - goto err_reorder; + if (dcp->dptxport[port].enabled && dcp->dptxport[port].connected) { + ret = dptxport_release_display(dcp->dptxport[port].service); + if (ret) + return ret; + dcp->dptxport[port].connected = false; } - *slot = best; - /* the port is in USB4 mode, not DP-alt */ - port->dp_wanted = false; - dcp_tunnel_prepare(best, ctl); - - dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); - /* - * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP - * drives the DPTX, i.e. from DCP's Activate call; waking it earlier - * hangs the machine. dptxep calls set_active back from Activate and - * Deactivate (set above, before the route became a tunnel). + * T6030: the firmware takes a direct crossbar output down when it + * stops the link. Make sure of it for a display released here, also + * when the firmware did not get that far. */ - if (!dcp->typec_connector) - dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); - WRITE_ONCE(dcp->typec_cable_connected, true); - port->hpd = true; - if (dcp->typec_connector) - dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); - + if (dcp->external_native) + dcp_direct_crossbar_link(dcp, false); return 0; - -err_reorder: - /* the pass kept a pipeline for this stream: give it to the others */ - if (planned) - dcp_typec_rebalance_locked(NULL, 0); - return ret; } -EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); -static struct apple_dcp_typec_port * -dcp_typec_port_get(struct device_node *connector_np) +int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) { - struct apple_dcp_typec_route *route; + int ret; - guard(mutex)(&dcp->tb_lock); - route = dcp->active_typec_route; - if (!route || !route->tunnel || !route->xbar_up) - return 0; - dcp_tunnel_dpin_locked(dcp, false); - return dcp_dpxbar_link(route->active_xbar, false); -} + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_dptx_disconnect_h17p(dcp, port); -/* - * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the - * tunnel pixel clock. A stopped clock also takes the crossbar connection - * down. - */ -int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 link_rate) -{ - typeof(&apple_atc_dp_tunnel_rate) fn; - struct apple_dcp_typec_route *route; - int ret; + /* Release the caller's RemotePort service, not the downstream DFP port. */ + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + + mutex_lock(&dcp->hpd_mutex); + ret = dcp_dptx_release_locked(dcp, port); + if (!ret) + WRITE_ONCE(dcp->dptxport[port].tile_hint, false); + mutex_unlock(&dcp->hpd_mutex); - guard(mutex)(&dcp->tb_lock); - if (!dcp->dptx_tunnel) - return -ENODEV; - fn = symbol_get(apple_atc_dp_tunnel_rate); - if (!fn) { - dev_err(dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); - return -ENOENT; - } - route = dcp->active_typec_route; - if (!link_rate && route && route->xbar_up) - dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, link_rate); - symbol_put(apple_atc_dp_tunnel_rate); - dcp->tb_clock_ok = !ret && link_rate; - if (ret) - dev_warn(dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", - link_rate, ret); return ret; } -/* Thunderbolt DP IN: DCP Activate/Deactivate. */ -int dcp_tunnel_dpin_activate(struct apple_dcp *dcp, bool active) +int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) { - guard(mutex)(&dcp->tb_lock); - if (!dcp->dptx_tunnel) - return 0; - return dcp_tunnel_dpin_locked(dcp, active); -} - -/* - * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the - * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt - * connection manager has set up a tunnel from one of them, route a free - * display pipeline there and tell DCP a display is attached, so it trains - * the link (and completes DPRX) through the tunnel. - */ -int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, - bool active, int (*set_active)(void *ctx, bool active), - void *ctx) -{ - struct apple_dcp_typec_port *port = NULL, *pos; - struct apple_dcp_typec_route *candidate, *best = NULL; - unsigned int best_score = UINT_MAX; - struct mux_control *ctl; - struct apple_dcp *dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (!connector_np || dpin > 1) - return -EINVAL; - - guard(mutex)(&dcp_typec_fabric_lock); - - list_for_each_entry(pos, &dcp_typec_ports, link) { - if (pos->connector_np == connector_np) { - port = pos; - break; - } - } - if (!port) - return -ENODEV; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_dptx_connect_oob_h17p(pdev, port); - if (!active) { - if (!port->owner || !port->owner->tunnel || - port->owner->tunnel_dpin != dpin) - return 0; - dcp = port->owner->dcp; - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - ret = dcp_typec_route_deactivate(port->owner); - if (ret) { - /* The caller's context is going away regardless. */ - scoped_guard(mutex, &dcp->tb_lock) { - dcp->dptx_tunnel = false; - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - return ret; + if (dcp_is_typec_output(dcp)) { + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->hpd_mutex); + dcp->typec_generation++; + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_reconnect_tries = 0; + dcp->placeholder_retried = false; + cancel_delayed_work(&dcp->typec_reconnect_wq); + /* A newly attached display gets its own retries. */ + if (dcp->external_native) { + atomic_set(&dcp->external_retries, 0); + cancel_delayed_work(&dcp->external_retry_wq); } - port->owner = NULL; - /* Re-apply the next Type-C mux state in full. */ - port->applied_valid = false; - if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) - dcp_dptx_connect(dcp, 0); - return 0; } - if (port->owner) { - if (port->owner->tunnel && port->owner->tunnel_dpin == dpin) - return 0; - dev_warn(port->owner->dcp->dev, - "port already routed, not taking DP tunnel dpin%u\n", dpin); - return -EBUSY; - } + ret = dcp_dptx_connect(dcp, port); + if (ret && ret != -ESHUTDOWN && dcp_is_typec_output(dcp)) + dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; + return ret; +} - if (!dcp_typec_route_available(candidate)) - continue; - score = dcp_typec_route_score(candidate); - /* - * j416s-specific, not in the reference (t8103 has a single - * dcpext): prefer a pipeline with a fixed output of its own - * (dcpext0) for a Type-C tunnel route. dcpext1 (Type-C only, - * no fixed output of its own) never completes link training - * for a tunneled target on this hardware: DCP firmware - * accepts request_display but never issues another apcall, - * on every attempt, even though the driver-issued connect - * parameters are byte-identical between the two pipelines. - * Nothing in this driver's source explains the difference, - * so this is a firmware-internal decision on the dcpext1 - * coprocessor instance, not something fixable here. dcpext0, - * forced onto the same physical port and tunnel, reaches a - * full AUX/DPCD link (DPRX_DONE=1) and a working picture, so - * it is preferred unconditionally for a tunnel route on this - * hardware. - */ - if (!candidate->dcp->fixed_phy) - score += 100; - if (score < best_score) { - best = candidate; - best_score = score; - } - } - if (!best) - return -EBUSY; +int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); - /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ - if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) - return -EOPNOTSUPP; - ctl = &best->xbar->chip->mux[1 + dpin]; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_dptx_disconnect_oob_h17p(pdev, port); - dcp = best->dcp; - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = set_active; - dcp->tb_dpin_ctx = ctx; - } - best->tunnel_dpin = dpin; - ret = dcp_typec_route_activate(best, ctl); - if (ret) { - scoped_guard(mutex, &dcp->tb_lock) { - dcp->tb_dpin_set_active = NULL; - dcp->tb_dpin_ctx = NULL; - } - return ret; - } - port->owner = best; - - dev_info(dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); - - /* - * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP - * drives the DPTX, i.e. from DCP's Activate call; waking it earlier - * hangs the machine (confirmed independently by this project's own - * AFK ordered-workqueue deadlock finding). dptxep calls set_active - * back from Activate and Deactivate (set above, before the route - * became a tunnel). - */ - if (!dcp->typec_connector) - dev_warn(dcp->dev, "no Type-C connector for the DP tunnel\n"); - WRITE_ONCE(dcp->typec_cable_connected, true); - port->hpd = true; - if (dcp->typec_connector) - dcp_dptx_connect_oob(to_platform_device(dcp->dev), 0); + if (dcp_is_typec_output(dcp)) { + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + dcp->typec_unplugs++; + /* Nothing left to retry for the display that went. */ + if (dcp->external_native) + cancel_delayed_work(&dcp->external_retry_wq); + } + WRITE_ONCE(dcp->typec_crtc_off, false); + reinit_completion(&dcp->typec_iomfb_hpd_ready); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + } - return 0; + return dcp_dptx_disconnect_drained(dcp, port); } -EXPORT_SYMBOL_GPL(apple_dcp_tb_dp_tunnel); -static int dcp_typec_route_set(struct typec_mux_dev *mux, - struct typec_mux_state *state) -static int dcp_pin_live_session(struct apple_dcp *dcp) +/* A delivered sink IRQ can follow a firmware unplug with cable HPD still high. */ +int dcp_dptx_recover_irq(struct apple_dcp *dcp) { - struct dcp_session_ref *session; + struct dcp_fabric_session session; + int ret; - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - port->hpd = false; - if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) - return; - port->owner = NULL; + mutex_lock(&dcp->hpd_mutex); + if (dcp->external_native || !dcp_is_typec_output(dcp) || + dcp_is_usb4_output(dcp) || !dcp->typec_connector || + READ_ONCE(dcp->typec_connector->connected) || + READ_ONCE(dcp->typec_crtc_off)) { + ret = 0; + goto out_unlock; + } + if (!dcp->typec_work_enabled || READ_ONCE(dcp->crashed) || + !dcp->dptxport[0].enabled) { + ret = -EAGAIN; + goto out_unlock; + } + session = dcp_session_locked(dcp); + if (!dcp_connect_session_valid_locked(dcp, session, true, true)) { + ret = -ESTALE; + goto out_unlock; } - if (port->secondary_owner == route) { - struct apple_dcp *dcp = route->dcp; - if (!dcp->hw.adopt_live_session || dcp->fw_compat != DCP_FIRMWARE_H17P || - dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) - return 0; - session = kzalloc_obj(*session); - if (!session) - return -ENOMEM; - if (port->hpd || dcp->typec_cable_connected) - dcp_dptx_disconnect_oob(to_platform_device(dcp->dev), 0); - if (dcp_typec_route_deactivate(route) && dcp_is_external_h17p(dcp)) - return; - port->secondary_owner = NULL; + /* Revoke older reconnect waits without changing physical cable state. */ + dcp->typec_generation++; + session = dcp_session_locked(dcp); + cancel_delayed_work(&dcp->typec_reconnect_wq); + cancel_delayed_work(&dcp->placeholder_edid_wq); + if (dcp->avep) + av_service_disconnect(dcp); + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (ret) + goto out_unlock; + if (!dcp_connect_session_valid_locked(dcp, session, true, true)) { + ret = -ESTALE; + goto out_unlock; } - port->hpd = !!(port->owner || port->secondary_owner); - list_del(&route->port_link); - if (list_empty(&port->routes)) { - list_del(&port->link); - of_node_put(port->connector_np); - kfree(port); + /* A failed earlier connect may also have left an unconfirmed release. */ + ret = dptxport_release_display(dcp->dptxport[0].service); + if (ret) + goto out_unlock; + if (!dcp_connect_session_valid_locked(dcp, session, true, true)) { + ret = -ESTALE; + goto out_unlock; } - /* Keep the DMA device, domain usage and driver code valid until reset. */ - session->dev = get_device(dcp->dev); - session->group = iommu_group_get(dcp->dev); - __module_get(THIS_MODULE); - WRITE_ONCE(dcp->retain_dma, true); - mutex_lock(&dcp_retained_sessions_lock); - list_add_tail(&session->link, &dcp_retained_sessions); - mutex_unlock(&dcp_retained_sessions_lock); - dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); - return 0; -} + dcp->dptxport[0].connected = false; + mutex_unlock(&dcp->hpd_mutex); -static void dcp_release_dma_domain(void *data) -{ - struct apple_dcp *dcp = data; + /* Another delivered IRQ must confirm release before a failed attempt retries. */ + return dcp_dptx_connect_session(dcp, 0, session, true, true); - if (!dcp->retain_dma) - iommu_device_unuse_default_domain(dcp->dev); +out_unlock: + mutex_unlock(&dcp->hpd_mutex); + return ret; } -static void dcp_release_context(void *data) +/* + * Release the display link of a Type-C output while its display stays + * attached, as a CRTC power-off does (see dcp_poweroff()): the firmware's + * unplug for it is ignored, and dcp_poweron() connects the link again. + */ +int dcp_dptx_park(struct apple_dcp *dcp) { - struct apple_dcp *dcp = data; + int ret; - if (!dcp->retain_dma) - kfree(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_crtc_off, true); + dcp->typec_generation++; + } + /* Session checks around reconnect waits reject the revoked generation. */ + cancel_delayed_work(&dcp->typec_reconnect_wq); + cancel_delayed_work(&dcp->placeholder_edid_wq); + if (dcp->avep) + av_service_disconnect(dcp); + if (READ_ONCE(dcp->split.active) && dcp->dptxport[1].enabled && + dcp->dptxport[1].connected) { + ret = dptxport_set_hpd(dcp->dptxport[1].service, false); + if (ret) + return ret; + ret = dcp_dptx_disconnect(dcp, 1); + if (ret) + return ret; + } + if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (ret) { + dev_warn(dcp->dev, "failed to deassert Type-C DPTX HPD: %d\n", ret); + return ret; + } + return dcp_dptx_disconnect(dcp, 0); + } + return 0; } -static void dcp_release_rtkit(void *data) +int dcp_dptx_disconnect_drained(struct apple_dcp *dcp, u32 port) { - struct apple_dcp *dcp = data; - - if (dcp->retain_dma) { - WRITE_ONCE(dcp->quiescing, true); - apple_rtkit_free_retaining_buffers(dcp->rtk); - } else { - apple_rtkit_free(dcp->rtk); - } - dcp->rtk = NULL; -} + int ret; -#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) + WRITE_ONCE(dcp->typec_crtc_off, false); + reinit_completion(&dcp->typec_iomfb_hpd_ready); -static bool show_notch; -module_param(show_notch, bool, 0644); -MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); + disconnected_hpd_event(dcp->connector); -static bool t6030_show_notch; -module_param(t6030_show_notch, bool, 0444); -MODULE_PARM_DESC(t6030_show_notch, - "M3 internal display (14.x and 26.6 IOMFB): use the full height and show the notch"); + if (dcp->avep) + av_service_disconnect(dcp); -bool hdmi_audio; -module_param(hdmi_audio, bool, 0644); -MODULE_PARM_DESC(hdmi_audio, "Enable unstable HDMI audio support"); + if (dcp->dptxport[port].enabled) { + ret = dptxport_set_hpd(dcp->dptxport[port].service, false); + if (ret) + return ret; + } -static bool unstable_edid = true; -module_param(unstable_edid, bool, 0644); -MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); + return dcp_dptx_disconnect(dcp, port); +} -/* copied and simplified from drm_vblank.c */ -static void send_vblank_event(struct drm_device *dev, - struct drm_pending_vblank_event *e, - u64 seq, ktime_t now) +bool dcp_fw_compat_is_12_x(struct platform_device *pdev) { - struct timespec64 tv; + struct apple_dcp *dcp = platform_get_drvdata(pdev); - if (e->event.base.type != DRM_EVENT_FLIP_COMPLETE) - return; + return dcp->fw_compat == DCP_FIRMWARE_V_12_3; +} - tv = ktime_to_timespec64(now); - e->event.vbl.sequence = seq; - /* - * e->event is a user space structure, with hardcoded unsigned - * 32-bit seconds/microseconds. This is safe as we always use - * monotonic timestamps since linux-4.15 - */ - e->event.vbl.tv_sec = tv.tv_sec; - e->event.vbl.tv_usec = tv.tv_nsec / 1000; +/* A failed T8122 dcpext: no pipeline of the DRM device. */ +bool dcp_ext_unusable(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); - /* - * Use the same timestamp for any associated fence signal to avoid - * mismatch in timestamps for vsync & fence events triggered by the - * same HW event. Frameworks like SurfaceFlinger in Android expects the - * retire-fence timestamp to match exactly with HW vsync as it uses it - * for its software vsync modeling. - */ - drm_send_event_timestamp_locked(dev, &e->base, now); + return !dcp || READ_ONCE(dcp->ext_dead); } -/** - * dcp_crtc_send_page_flip_event - helper to send vblank event after pageflip - * - * Compensate for unknown slack between page flip and arrival of the - * swap_complete callback. Minimal observed duration on DCP with HDMI output - * was around 2.3 ms. If the fb swap was submitted closer to the expected - * swap_complete it gets a penalty of one frame duration. This is on the border - * of unreasonable considering that Apple advertises support for 240 Hz (frame - * duration of 4.167 ms). - * It is unreasonable considering kwin's kms commit scheduling. Kwin commits - * 1.5 ms + the mode's vblank time before the expected next page flip - * completion. This results in presenting at half the display's rate for HDMI - * outputs. - * This might be a difference between dcp and dcpext. - */ -static void dcp_crtc_send_page_flip_event(struct apple_crtc *crtc, - struct drm_pending_vblank_event *e, - ktime_t now, ktime_t start) +bool dcp_fw_compat_is_14_7(struct platform_device *pdev) { - struct drm_device *dev = crtc->base.dev; - u64 seq; - unsigned int pipe = drm_crtc_index(&crtc->base); - ktime_t flip; + struct apple_dcp *dcp = platform_get_drvdata(pdev); - seq = 0; - if (start != KTIME_MIN) { - s64 delta = ktime_us_delta(now, start); - if (delta <= 500) - flip = now; - else if (delta >= 2500) - flip = ktime_sub_us(now, 1000); - else - flip = ktime_sub_us(now, (delta - 500) / 2); - } else { - flip = now; - } - e->pipe = pipe; - send_vblank_event(dev, e, seq, flip); + return dcp->fw_compat == DCP_FIRMWARE_V_14_7 || + dcp->fw_compat == DCP_FIRMWARE_V_26_6; } -/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ -void dcp_drm_crtc_vblank(struct apple_crtc *crtc) +unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) { - unsigned long flags; + struct apple_dcp *dcp = platform_get_drvdata(pdev); - spin_lock_irqsave(&crtc->base.dev->event_lock, flags); - if (crtc->event) { - drm_crtc_send_vblank_event(&crtc->base, crtc->event); - crtc->event = NULL; - } - spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); + return dcp->iomfb_surfaces; } -void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) +int dcp_start(struct platform_device *pdev) { - unsigned long flags; - struct apple_crtc *crtc = dcp->crtc; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; - spin_lock_irqsave(&crtc->base.dev->event_lock, flags); - if (crtc->event) { - if (crtc->event->event.base.type == DRM_EVENT_FLIP_COMPLETE) - dcp_crtc_send_page_flip_event(crtc, crtc->event, now, dcp->swap_start); - else - drm_crtc_send_vblank_event(&crtc->base, crtc->event); - crtc->event = NULL; - dcp->swap_start = KTIME_MIN; - } - spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); -} + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_start_h17p(pdev); -static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) -{ - return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; -} + init_completion(&dcp->start_done); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) + return iomfb_v26_6_start(dcp); -void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) -{ - unsigned long flags; + /* + * The T6030 firmware session is adopted for the internal panel. + * Open DPTX on that same RTKit only when the firmware advertised it. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (dcp->external) { + /* A native pipe is usable now; its firmware starts on first use. */ + if (dcp->external_native) + dcp->active = true; + complete(&dcp->start_done); + return 0; + } + ret = iomfb_v14_7_start(dcp); + if (ret) + return ret; + return 0; + } - if (!dcp_firmware_metadata_enabled(dcp)) - return; - spin_lock_irqsave(&dcp->metadata.lock, flags); - if (generation != dcp->metadata.epoch.generation || !connected) { - dcp->rate_proof.connection_sequence++; - dcp->rate_proof.sequence++; - dcp->rate_proof.rate.valid = false; - dcp->rate_proof.lanes.valid = false; - dcp->metadata.attributes_valid = false; - dcp->metadata.timing_valid = false; - dcp->metadata.hpd_seen = false; + /* start RTKit endpoints */ + ret = systemep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start system endpoint: %d\n", ret); + + if (unstable_edid && !dcp_has_panel(dcp)) { + ret = dpavservep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start DPAVSERV endpoint: %d", + ret); } - dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); - spin_unlock_irqrestore(&dcp->metadata.lock, flags); -} -void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) -{ - struct apple_connector *con = READ_ONCE(dcp->connector); - unsigned long flags; + if (dcp->phy && dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { + ret = ibootep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", + ret); - if (!dcp_firmware_metadata_enabled(dcp)) - return; - spin_lock_irqsave(&dcp->metadata.lock, flags); - if (connected) { - dcp->metadata.hpd_seen = true; + ret = dptxep_init(dcp); + if (ret) { + dev_warn(dcp->dev, "Failed to start DPTX endpoint: %d\n", + ret); +#ifdef DCP_DPTX_DISCONNECT_ON_INIT /* - * Fresh proof may precede HPD assertion after a same-cable loss. - * Keep invalidated firmware metadata and old receipt tickets revoked. + * This disconnect / connect cycle on init is only necessary + * when using dcp0 on j473, j474s and presumedly j475c. + * Since dcp0 is not used at the moment let's avoid this + * since it is possibly the cause for startup issues. */ - if (READ_ONCE(dcp->typec_cable_connected) && - dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) - dcp->metadata.epoch.connected = true; - } else if (dcp->metadata.hpd_seen) { - dcp->metadata.hpd_seen = false; - dcp->rate_proof.connection_sequence++; - dcp->rate_proof.sequence++; - dcp->rate_proof.rate.valid = false; - dcp->rate_proof.lanes.valid = false; - if (con && READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) - WRITE_ONCE(con->edid_invalidation_pending, true); - dcp_metadata_epoch_connection(&dcp->metadata.epoch, - dcp->metadata.epoch.generation, false); - dcp->metadata.attributes_valid = false; - dcp->metadata.timing_valid = false; + } else if (dcp->dptxport[0].enabled) { + bool connected; + /* force disconnect on start - necessary if the display + * is already up from m1n1 + */ + dptxport_set_hpd(dcp->dptxport[0].service, false); + dptxport_release_display(dcp->dptxport[0].service); + usleep_range(10 * USEC_PER_MSEC, 25 * USEC_PER_MSEC); + + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); + + // necessary on j473/j474 but not on j314c + if (connected) + dcp_dptx_connect(dcp, 0); +#endif + } + } else if (dcp->phy) { + dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); } - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - if (connected) - dcp_180hz_notify(dcp); + ret = iomfb_start_rtkit(dcp); + if (ret) { + dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); + return ret; + } + +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + if (hdmi_audio) { + ret = avep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start AV endpoint: %d", ret); + ret = 0; + } +#endif + + return ret; } -void dcp_firmware_metadata_begin(struct apple_dcp *dcp) +static void _dcp_poweroff(struct apple_dcp *dcp) { - unsigned long flags; - - spin_lock_irqsave(&dcp->metadata.lock, flags); - dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); - dcp->chunks.received = 0; - dcp->chunks.contiguous = true; - dcp->chunks.payload_complete = false; - spin_unlock_irqrestore(&dcp->metadata.lock, flags); + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweroff_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_poweroff_v13_3(dcp); + break; + case DCP_FIRMWARE_V_14_7_TEMPLATE: + iomfb_poweroff_v14_7_0(dcp); + break; + case DCP_FIRMWARE_V_26_6: + iomfb_v26_6_poweroff(dcp); + break; + case DCP_FIRMWARE_V_14_7: + iomfb_v14_7_poweroff(dcp); + break; + case DCP_FIRMWARE_H17P: + iomfb_poweroff_h17p(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } } -static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) +static void dcp_resume_enable_irq(void *ctx) { - unsigned long flags; - bool ready; + struct apple_dcp *dcp = ctx; - spin_lock_irqsave(&dcp->metadata.lock, flags); - ready = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && - dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && - dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - if (ready && dcp->connector && READ_ONCE(dcp->connector->connected)) - schedule_work(&dcp->connector->hotplug_wq); + enable_irq(dcp->hdmi_hpd_irq); } -static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) +static void dcp_resume_sample(void *ctx) { - return dcp->chunks.contiguous && dcp->chunks.payload_complete && - dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); + dcp_fabric_hdmi_resume(ctx); } -void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, - const struct dcp_display_metadata *attributes) +static const struct dcp_fabric_resume_ops dcp_resume_ops = { + .enable_irq = dcp_resume_enable_irq, + .sample = dcp_resume_sample, +}; + +static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) { - unsigned long flags; + if (dcp_is_typec_output(dcp)) { + if (READ_ONCE(dcp->typec_cable_connected)) + dcp_dptx_connect(dcp, 0); + } else if (dcp->hdmi_hpd) { + /* Check HPD before enabling the edge-triggered IRQ. */ + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - if (!dcp_firmware_metadata_enabled(dcp)) - return; - spin_lock_irqsave(&dcp->metadata.lock, flags); - if (dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) { - dcp->metadata.attributes_valid = attributes && dcp_firmware_metadata_complete(dcp); - if (dcp->metadata.attributes_valid) { - dcp->metadata.attributes = *attributes; - dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; - } + if (connected) + dcp_dptx_connect(dcp, 0); + else + _dcp_poweroff(dcp); } - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - dcp_firmware_metadata_notify(dcp); -} -void dcp_firmware_metadata_timing(struct apple_dcp *dcp) -{ - unsigned long flags; - unsigned int i; + if (dcp->hdmi_hpd_irq) + dcp_fabric_run_resume(&dcp_resume_ops, dcp); - if (!dcp_firmware_metadata_enabled(dcp)) - return; - spin_lock_irqsave(&dcp->metadata.lock, flags); - if (!dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) - goto unlock; - dcp->metadata.timing_valid = false; - if (!dcp_firmware_metadata_complete(dcp)) - goto unlock; - for (i = 0; i < dcp->nr_modes; i++) { - if (!(dcp->modes[i].mode.type & DRM_MODE_TYPE_PREFERRED) || - !dcp->modes[i].metadata_timing_valid) - continue; - dcp->metadata.preferred = dcp->modes[i].mode; - dcp->metadata.timing_ticket = dcp->chunks.metadata_ticket; - dcp->metadata.timing_valid = true; - break; - } -unlock: - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - dcp_firmware_metadata_notify(dcp); + return 0; } -bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation) -{ - unsigned long flags; - bool valid; - - spin_lock_irqsave(&dcp->metadata.lock, flags); - valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && - generation == dcp->metadata.epoch.generation; - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - return valid; -} - -const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) +int dcp_wait_ready(struct platform_device *pdev, u64 timeout) { - struct dcp_display_metadata attributes; - struct drm_display_mode preferred; - const struct drm_edid *edid; - unsigned long flags; - bool valid; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; - if (!dcp_firmware_metadata_enabled(dcp)) - return ERR_PTR(-ENODEV); - spin_lock_irqsave(&dcp->metadata.lock, flags); - valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && - dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && - dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); - if (valid) { - attributes = dcp->metadata.attributes; - preferred = dcp->metadata.preferred; - *generation = dcp->metadata.epoch.generation; - } - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - if (!valid) - return ERR_PTR(-ENODATA); - edid = dcp_metadata_edid(&attributes, &preferred); - spin_lock_irqsave(&dcp->metadata.lock, flags); - valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && - *generation == dcp->metadata.epoch.generation; - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - if (!valid && !IS_ERR(edid)) { - drm_edid_free(edid); - return ERR_PTR(-ESTALE); - } - return edid; -} + if (dcp->crashed) + return -ENODEV; + if (dcp->active) + return dcp_enable_dp2hdmi_hpd(dcp); + if (timeout <= 0) + return -ETIMEDOUT; -void dcp_set_dimensions(struct apple_dcp *dcp) -void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation) -{ - struct apple_connector *apple_connector = READ_ONCE(dcp->connector); - struct drm_connector *connector = apple_connector ? &apple_connector->base : NULL; - int width_mm, height_mm, i; + ret = wait_for_completion_timeout(&dcp->start_done, timeout); + if (ret < 0) + return ret; - if (connector) - mutex_lock(&connector->dev->mode_config.mutex); - mutex_lock(&dcp->modes_lock); - if (generation != dcp->modes_generation || - apple_connector != READ_ONCE(dcp->connector)) - goto out_unlock; - width_mm = dcp->width_mm; - height_mm = dcp->height_mm; - if (!width_mm || !height_mm) { - width_mm = dcp->panel.width_mm; - height_mm = dcp->panel.height_mm; - } - if (connector) { - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; - } + if (dcp->crashed) + return -ENODEV; - /* - * This callback runs on the ordered RTKit RX worker. A mode probe - * can hold mode_config.mutex while waiting for an AFK reply which - * that same worker still has to dispatch. Defer the DRM update. - */ - if (dcp->connector && !READ_ONCE(dcp->quiescing)) - schedule_work(&dcp->connector->hotplug_wq); + if (dcp->active) + dcp_enable_dp2hdmi_hpd(dcp); - /* - * Fix up any probed modes. Modes are created when parsing - * TimingElements, dimensions are calculated when parsing - * DisplayAttributes, and TimingElements may be sent first - */ - for (i = 0; i < dcp->nr_modes; ++i) { - dcp->modes[i].mode.width_mm = width_mm; - dcp->modes[i].mode.height_mm = height_mm; - } -out_unlock: - mutex_unlock(&dcp->modes_lock); - if (connector) - mutex_unlock(&connector->dev->mode_config.mutex); + return dcp->active ? 0 : -ETIMEDOUT; } -void dcp_update_connector_dimensions(struct apple_dcp *dcp, - struct drm_connector *connector) +static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) { - int width_mm = READ_ONCE(dcp->width_mm); - int height_mm = READ_ONCE(dcp->height_mm); - - lockdep_assert_held(&connector->dev->mode_config.mutex); - if (!width_mm || !height_mm) { - width_mm = dcp->panel.width_mm; - height_mm = dcp->panel.height_mm; + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_sleep_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_sleep_v13_3(dcp); + break; + case DCP_FIRMWARE_V_14_7_TEMPLATE: + iomfb_sleep_v14_7_0(dcp); + break; + case DCP_FIRMWARE_V_26_6: + iomfb_v26_6_poweroff(dcp); + break; + case DCP_FIRMWARE_V_14_7: + iomfb_v14_7_poweroff(dcp); + break; + case DCP_FIRMWARE_H17P: + iomfb_sleep_h17p(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; } - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; } -bool dcp_has_panel(struct apple_dcp *dcp) +int dcp_typec_prepare_route(struct apple_dcp *dcp) { - return dcp->panel.width_mm > 0; + u64 generation; + unsigned long ready; + int ret; + + if (!dcp_crtc_needs_route_start(dcp)) + return 0; + scoped_guard(mutex, &dcp->hpd_mutex) + generation = dcp->typec_generation; + ret = dcp_dptx_connect(dcp, 0); + if (ret) + return ret; + ready = wait_for_completion_timeout(&dcp->typec_iomfb_hpd_ready, + msecs_to_jiffies(3000)); + guard(mutex)(&dcp->hpd_mutex); + if (generation != dcp->typec_generation || !dcp->typec_cable_connected) + return -ESTALE; + return ready ? 0 : -ETIMEDOUT; } -int dcp_set_crc(struct drm_crtc *crtc, bool enabled) +void dcp_poweron(struct platform_device *pdev) { - struct apple_crtc *ac = to_apple_crtc(crtc); - struct apple_dcp *dcp = platform_get_drvdata(ac->dcp); + struct apple_dcp *dcp = platform_get_drvdata(pdev); + bool wait_for_typec_hpd = false; + unsigned long remaining; + int ret; - dcp->crc_enabled = enabled; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_poweron_h17p(pdev); - return 0; -} + if (dcp_is_typec_output(dcp)) { + /* T8122 dcpext only, see dcp_poweroff() */ + if (dcp->ext_failsafe) { + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->crtc_unplugs = dcp->typec_unplugs; + } + wait_for_typec_hpd = READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected); + WRITE_ONCE(dcp->typec_crtc_off, false); + WRITE_ONCE(dcp->typec_follow_start, false); -/* - * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp - * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks - * send a vblank event via a workqueue. - */ -static void dcp_delayed_vblank(struct work_struct *work) -{ - struct apple_dcp *dcp; + /* + * A Type-C CRTC disable releases its DPTX session. Re-establish it + * synchronously before IOMFB is powered back on. + */ + if (READ_ONCE(dcp->typec_cable_connected)) { + cancel_delayed_work(&dcp->typec_reconnect_wq); + dcp->typec_reconnect_tries = 0; + ret = dcp_dptx_connect(dcp, 0); + if (ret) + dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); + else if (wait_for_typec_hpd) { + remaining = wait_for_completion_timeout( + &dcp->typec_iomfb_hpd_ready, + msecs_to_jiffies(3000)); + if (!remaining) + dev_warn(dcp->dev, + "Type-C IOMFB hotplug not ready on wake\n"); + } + } + } else if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - struct apple_crtc *crtc; + if (connected) + dcp_dptx_connect(dcp, 0); + } - dcp = container_of(work, struct apple_dcp, vblank_wq); - /* RTKit can report a crash before the pipeline has a CRTC. */ - crtc = READ_ONCE(dcp->crtc); - if (!crtc) - return; - mdelay(5); - dcp_drm_crtc_vblank(crtc); + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_poweron_v12_3(dcp); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_poweron_v13_3(dcp); + break; + case DCP_FIRMWARE_V_14_7_TEMPLATE: + iomfb_poweron_v14_7_0(dcp); + break; + case DCP_FIRMWARE_V_26_6: + iomfb_v26_6_poweron(dcp); + break; + case DCP_FIRMWARE_V_14_7: + iomfb_v14_7_poweron(dcp); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } + if (dcp->avep) + av_service_connect(dcp); } -#define DCP_SWAP_WATCHDOG_MS 1000 -#define DCP_SWAP_WATCHDOG_RETRAINS 5 - -/* - * DCP drops swaps without completing them while an external pipe is not - * enabled, for instance after a modeset that raced a Type-C sink which had - * not asserted HPD yet. Userspace would then wait for the flip until the - * commit times out, stalling every output it drives. Complete the flip, - * mark the mode invalid so later commits do not wait for DCP, and let the - * hotplug worker re-apply the active CRTC. - */ -static void dcp_swap_watchdog(struct work_struct *work) -static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) +void dcp_poweroff(struct platform_device *pdev) { - /* DPAV 0x2a on H17P overlaps the legacy DPTX endpoint number. */ - if (dcp->dcpavservep && endpoint == dcp->dcpavservep->endpoint) - return dcp->dcpavservep; - if (dcp->dptxep && endpoint == dcp->dptxep->endpoint) - return dcp->dptxep; - switch (endpoint) { - case AV_ENDPOINT: - return dcp->avep; - case SYSTEM_ENDPOINT: - return dcp->systemep; - case DISP0_ENDPOINT: - return dcp->ibootep; - case DPTX_CONTROLLER_ENDPOINT: - return dcp->dptx_controller_ep; - default: - return NULL; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + bool display_gone = false; + int ret; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_poweroff_h17p(pdev); + + cancel_delayed_work(&dcp->swap_watchdog_wq); + if (dcp->avep) + av_service_disconnect(dcp); + + /* + * The display this CRTC was powered on for has been unplugged since: + * its link went down with the cable (dcp_dptx_disconnect_oob()), and + * the desktop only now turns the CRTC off. By then the pipeline may + * already drive the next display, on this port or on another one that + * was waiting for it, and that connection is not the CRTC's to + * release: dropping its HPD makes the firmware report the new display + * inactive, and it stays dark. + * + * Only the T8122 dcpext (ext_failsafe) takes this path, the one + * pipeline where it was run. Every other Type-C output releases its + * connection below as before: the M1/M2 dcpext routes, and the T6030 + * split second port and external retry. + */ + if (dcp->ext_failsafe && dcp_is_typec_output(dcp)) { + scoped_guard(mutex, &dcp->hpd_mutex) + display_gone = dcp->typec_unplugs != dcp->crtc_unplugs; + if (display_gone) { + dev_dbg(dcp->dev, + "CRTC off after its display was unplugged: leaving the current connection alone\n"); + _dcp_poweroff(dcp); + return; + } } -} -#define DCP_SWAP_WATCHDOG_MS 1000 -#define DCP_SWAP_WATCHDOG_RETRAINS 5 + /* + * Powering a Type-C CRTC off drops DCP's synthetic HPD, and the firmware + * reports that as an unplug. The display is still attached: keep the + * connector connected (see dcpep_cb_hotplug()) and let dcp_poweron() + * re-establish the DPTX session. Recreating it here instead makes the + * display vanish and come back, and compositors light a returning + * display, so DPMS off never sticks. Cable removal is reported through + * the Type-C mux. + */ + if (dcp_is_typec_output(dcp)) { + reinit_completion(&dcp->typec_iomfb_hpd_ready); + if (READ_ONCE(dcp->typec_cable_connected)) + WRITE_ONCE(dcp->typec_crtc_off, true); + /* dcp_poweron() reconnects the link on DPMS wake. */ + cancel_delayed_work(&dcp->typec_reconnect_wq); + } -/* - * DCP drops swaps without completing them while an external pipe is not - * enabled, for instance after a modeset that raced a Type-C sink which had - * not asserted HPD yet. Userspace would then wait for the flip until the - * commit times out, stalling every output it drives. Complete the flip, - * mark the mode invalid so later commits do not wait for DCP, and let the - * hotplug worker re-apply the active CRTC. - */ -static void dcp_swap_watchdog(struct work_struct *work) -{ - struct apple_dcp *dcp = - container_of(to_delayed_work(work), struct apple_dcp, - swap_watchdog_wq); + _dcp_poweroff(dcp); - dev_warn(dcp->dev, "swap not completed, retraining the display\n"); - dcp_mode_invalidate(&dcp->mode_state); - dcp_drm_crtc_vblank(dcp->crtc); - if (dcp->connector && - dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) - schedule_work(&dcp->connector->hotplug_wq); + if (dcp_is_typec_output(dcp)) { + ret = 0; + /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ + if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (ret) + dev_warn(dcp->dev, + "failed to deassert Type-C DPTX HPD: %d\n", ret); + dcp_dptx_disconnect(dcp, 0); + } + /* + * A native external pipe whose display is no longer described + * (it went away during a hotplug bounce) is powered off by the + * desktop for good: with HPD still high, connect the link again + * so that the firmware describes the display anew. + */ + if (dcp->external_native && READ_ONCE(dcp->typec_cable_connected) && + (ret || !dcp->connector || !READ_ONCE(dcp->connector->connected))) + dcp_external_retry(dcp, ret ? "display link released with an HPD error" : + "display link released with no display described", + ret, 500); + /* and the second tile's, which dcp_poweron() brings back too */ + if (READ_ONCE(dcp->split.active) && dcp->dptxport[1].enabled && + dcp->dptxport[1].connected) { + dptxport_set_hpd(dcp->dptxport[1].service, false); + dcp_dptx_disconnect(dcp, 1); + } + } else if (dcp->hdmi_hpd) { + bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + if (!connected) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + } } -void dcp_swap_watchdog_arm(struct apple_dcp *dcp) +static void dcp_work_register_backlight(struct work_struct *work) { - if (dcp_is_typec_output(dcp)) - mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, - msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); -} + int ret; + struct apple_dcp *dcp; -void dcp_swap_watchdog_complete(struct apple_dcp *dcp) -{ - cancel_delayed_work(&dcp->swap_watchdog_wq); - dcp->swap_watchdog_retrains = 0; -} + dcp = container_of(work, struct apple_dcp, bl_register_wq); -static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) -{ - struct apple_dcp *dcp = - container_of(to_delayed_work(work), struct apple_dcp, - swap_watchdog_wq); + mutex_lock(&dcp->bl_register_mutex); + if (dcp->brightness.bl_dev) + goto out_unlock; - if (READ_ONCE(dcp->quiescing)) - return; + /* try to register backlight device, */ + ret = dcp_backlight_register(dcp); + if (ret) { + dev_err(dcp->dev, "Unable to register backlight device\n"); + dcp->brightness.maximum = 0; + } - trace_dcp_recv_msg(dcp, endpoint, message); - dev_warn(dcp->dev, "swap not completed, retraining the display\n"); - dcp_mode_invalidate(&dcp->mode_state); - dcp_drm_crtc_vblank(dcp->crtc); - if (dcp->connector && - dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) - dcp_queue_hotplug(dcp->connector); +out_unlock: + mutex_unlock(&dcp->bl_register_mutex); } -void dcp_swap_watchdog_arm(struct apple_dcp *dcp) +static void dcp_work_update_backlight(struct work_struct *work) { - if (dcp_is_typec_output(dcp)) - mod_delayed_work(system_wq, &dcp->swap_watchdog_wq, - msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); -} + struct apple_dcp *dcp; -void dcp_swap_watchdog_complete(struct apple_dcp *dcp) -{ - cancel_delayed_work(&dcp->swap_watchdog_wq); - dcp->swap_watchdog_retrains = 0; + dcp = container_of(work, struct apple_dcp, bl_update_wq); + + dcp_backlight_update(dcp); } -static void dcp_recv_msg(void *cookie, u8 endpoint, u64 message) +static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) { - struct apple_dcp *dcp = cookie; - struct apple_dcp_afkep *ep; + int ret; + struct device_node *node __free(device_node) = of_get_child_by_name(dcp->dev->of_node, "piodma"); - trace_dcp_recv_msg(dcp, endpoint, message); + if (!node) + return dev_err_probe(dcp->dev, -ENODEV, + "Failed to get piodma child DT node\n"); - /* A failed T8122 dcpext: its firmware may still talk, nobody listens. */ - if (READ_ONCE(dcp->ext_dead)) - return; - /* - * H17P starts endpoints that have no AFK instance behind them (the - * remote allocator endpoint, for one). Log their messages rather than - * dereferencing a NULL afkep or warning about an unknown endpoint. - */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P && endpoint != IOMFB_ENDPOINT && - !dcp_afkep(dcp, endpoint)) { - dev_dbg_ratelimited(dcp->dev, "ep %#04x: %#llx\n", endpoint, - message); - return; - } + dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); + if (!dcp->piodma) + return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); - switch (endpoint) { - case IOMFB_ENDPOINT: - return iomfb_recv_msg(dcp, message); - case AV_ENDPOINT: - ep = dcp->avep; - break; - case SYSTEM_ENDPOINT: - ep = dcp->systemep; - break; - case DISP0_ENDPOINT: - ep = dcp->ibootep; - break; - case DPAVSERV_ENDPOINT: - ep = dcp->dcpavservep; - break; - case DPTX_ENDPOINT: - ep = dcp->dptxep; - break; - case DPTX_ENDPOINT_V2: - ep = dcp->dptxep; - break; - case DPTX_CONTROLLER_ENDPOINT: - ep = dcp->dptx_controller_ep; - break; - default: - ep = NULL; - break; - } - if (endpoint == IOMFB_ENDPOINT) - return iomfb_recv_msg(dcp, message); + ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); + if (ret) + goto err_destroy_pdev; - ep = dcp_afkep(dcp, endpoint); + ret = of_dma_configure(&dcp->piodma->dev, node, true); + if (ret) { + ret = dev_err_probe(dcp->dev, ret, + "Failed to configure IOMMU child DMA\n"); + goto err_destroy_pdev; + } - if (!ep) { - dev_warn_ratelimited(dcp->dev, - "dropping message for unhandled endpoint %#x\n", - endpoint); - return; - case SYSTEM_ENDPOINT: - afk_receive_message(dcp->systemep, message); - return; - case DISP0_ENDPOINT: - afk_receive_message(dcp->ibootep, message); - return; - case DPAVSERV_ENDPOINT: - afk_receive_message(dcp->dcpavservep, message); - return; - case DPTX_ENDPOINT: - afk_receive_message(dcp->dptxep, message); - return; - default: - WARN(endpoint, "unknown DCP endpoint %hhu\n", endpoint); + dcp->iommu_dom = iommu_get_domain_for_dev(&dcp->piodma->dev); + if (IS_ERR(dcp->iommu_dom)) { + ret = dev_err_probe(dcp->dev, PTR_ERR(dcp->iommu_dom), + "Failed to get default iommu domain for " + "piodma device\n"); + dcp->iommu_dom = NULL; + goto err_destroy_pdev; } - afk_receive_message(ep, message); + return 0; +err_destroy_pdev: + of_platform_device_destroy(&dcp->piodma->dev, NULL); + return ret; } -static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) +static int dcp_get_bw_scratch_reg(struct apple_dcp *dcp, u32 expected) { - struct apple_dcp *dcp = cookie; + struct of_phandle_args ph_args; + u32 addr_idx, disp_idx, offset; + int ret; - /* - * A failed T8122 dcpext: its start never ran (start_done is not set - * up) and it has no connector. Its firmware may still report a crash. - */ - if (READ_ONCE(dcp->ext_dead)) { - dcp->crashed = true; - dev_err(dcp->dev, "unusable external display processor has crashed\n"); - return; + ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-scratch", + "#apple,bw-scratch-cells", 0, &ph_args); + if (ret < 0) { + dev_err(dcp->dev, "Failed to read 'apple,bw-scratch': %d\n", ret); + return ret; } - if (READ_ONCE(dcp->quiescing)) - return; - dcp->crashed = true; - dev_err(dcp->dev, "DCP has crashed\n"); - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - /* No in-place restart exists; the last scanout stays retained. */ - dev_err(dcp->dev, "a reboot is required to restore the display\n"); - /* - * Nothing will complete a present that was in flight. Before - * the pipeline is linked there is no event to complete. - */ - if (READ_ONCE(dcp->crtc)) - schedule_work(&dcp->vblank_wq); - } - if (dcp->connector) { - dcp->connector->connected = 0; - apple_connector_edid_set_live(dcp->connector, false); - dcp_queue_hotplug(dcp->connector); + if (ph_args.args_count != 3) { + dev_err(dcp->dev, "Unexpected 'apple,bw-scratch' arg count %d\n", + ph_args.args_count); + ret = -EINVAL; + goto err_of_node_put; } - complete(&dcp->start_done); -} -/* - * Check that every IOMMU page of [iova, iova + size) translates to the same - * offset from @phys as it has from @iova. - */ -static int dcp_check_iova_contiguous(struct iommu_domain *domain, - dma_addr_t iova, size_t size, - phys_addr_t phys) -{ - unsigned long pgsize = domain->pgsize_bitmap ? - 1UL << __ffs(domain->pgsize_bitmap) : PAGE_SIZE; - dma_addr_t end = iova + size; - dma_addr_t addr = iova; + addr_idx = ph_args.args[0]; + disp_idx = ph_args.args[1]; + offset = ph_args.args[2]; - while (addr < end) { - if (iommu_iova_to_phys(domain, addr) != phys + (addr - iova)) - return -EINVAL; - addr = ALIGN_DOWN(addr, pgsize) + pgsize; + if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { + dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-scratch': %d\n", + disp_idx); + ret = -EINVAL; + goto err_of_node_put; } - return 0; + ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_scratch_res); + if (ret < 0) { + dev_err(dcp->dev, "Failed to get 'apple,bw-scratch' resource %d from %pOF\n", + addr_idx, ph_args.np); + goto err_of_node_put; + } + if (offset > resource_size(&dcp->disp_bw_scratch_res) - 4) { + ret = -EINVAL; + goto err_of_node_put; + } + + dcp->disp_registers[disp_idx] = &dcp->disp_bw_scratch_res; + dcp->disp_bw_scratch_index = disp_idx; + dcp->disp_bw_scratch_offset = offset; + ret = 0; + +err_of_node_put: + of_node_put(ph_args.np); + return ret; } -static int dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) +static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) { - struct apple_dcp *dcp = cookie; + struct of_phandle_args ph_args; + u32 addr_idx, disp_idx; int ret; - if (bfr->iova) { - struct iommu_domain *domain = - iommu_get_domain_for_dev(dcp->dev); - phys_addr_t phy_addr; + ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-doorbell", + "#apple,bw-doorbell-cells", 0, &ph_args); + if (ret < 0) { + dev_err(dcp->dev, "Failed to read 'apple,bw-doorbell': %d\n", ret); + return ret; + } - if (!domain) - return -ENOMEM; + if (ph_args.args_count != 2) { + dev_err(dcp->dev, "Unexpected 'apple,bw-doorbell' arg count %d\n", + ph_args.args_count); + ret = -EINVAL; + goto err_of_node_put; + } - // TODO: get map from device-tree - phy_addr = iommu_iova_to_phys(domain, bfr->iova); - if (!phy_addr) - return -ENOMEM; + addr_idx = ph_args.args[0]; + disp_idx = ph_args.args[1]; - /* memremap() below needs the whole buffer physically contiguous */ - ret = dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, - phy_addr); - if (ret) { - dev_err(dcp->dev, - "shmem_setup: iova %pad (%#zx bytes) is not physically contiguous\n", - &bfr->iova, bfr->size); - return ret; - } + if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { + dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-doorbell': %d\n", + disp_idx); + ret = -EINVAL; + goto err_of_node_put; + } - /* Firmware writes without CPU cache maintenance. */ - bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WC); - if (!bfr->buffer) - return -ENOMEM; + ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_doorbell_res); + if (ret < 0) { + dev_err(dcp->dev, "Failed to get 'apple,bw-doorbell' resource %d from %pOF\n", + addr_idx, ph_args.np); + goto err_of_node_put; + } + dcp->disp_bw_doorbell_index = disp_idx; + dcp->disp_registers[disp_idx] = &dcp->disp_bw_doorbell_res; + ret = 0; - bfr->is_mapped = true; - dev_info(dcp->dev, - "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", - (uintptr_t)bfr->iova, (uintptr_t)phy_addr, - (uintptr_t)bfr->buffer); - } else { - bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, - &bfr->iova, GFP_KERNEL); - if (!bfr->buffer) - return -ENOMEM; +err_of_node_put: + of_node_put(ph_args.np); + return ret; +} - dev_info(dcp->dev, "shmem_setup: iova: %lx, buffer: %lx\n", - (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); +static int dcp_get_disp_regs(struct apple_dcp *dcp) +{ + struct platform_device *pdev = to_platform_device(dcp->dev); + int count = pdev->num_resources - 1; + int i, ret; + + if (count <= 0 || count > MAX_DISP_REGISTERS) + return -EINVAL; + + for (i = 0; i < count; ++i) { + dcp->disp_registers[i] = + platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + } + + /* load pmgr bandwidth scratch resource and offset */ + ret = dcp_get_bw_scratch_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; + + /* load pmgr bandwidth doorbell resource if present (only on t8103) */ + if (of_property_present(dcp->dev->of_node, "apple,bw-doorbell")) { + ret = dcp_get_bw_doorbell_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; } + dcp->nr_disp_registers = count; return 0; } -static void dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) +#define DCP_FW_VERSION_MIN_LEN 3 +#define DCP_FW_VERSION_MAX_LEN 5 +#define DCP_FW_VERSION_STR_LEN (DCP_FW_VERSION_MAX_LEN * 4) + +static int dcp_read_fw_version(struct device *dev, const char *name, + char *version_str) { - struct apple_dcp *dcp = cookie; + u32 ver[DCP_FW_VERSION_MAX_LEN]; + int len_str; + int len; - if (bfr->is_mapped) - memunmap(bfr->buffer); - else - dma_free_coherent(dcp->dev, bfr->size, bfr->buffer, bfr->iova); -} + len = of_property_read_variable_u32_array(dev->of_node, name, ver, + DCP_FW_VERSION_MIN_LEN, + DCP_FW_VERSION_MAX_LEN); -static struct apple_rtkit_ops rtkit_ops = { - .crashed = dcp_rtk_crashed, - .recv_message = dcp_recv_msg, - .shmem_setup = dcp_rtk_shmem_setup, - .shmem_destroy = dcp_rtk_shmem_destroy, -}; + switch (len) { + case 3: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d", ver[0], ver[1], ver[2]); + break; + case 4: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3]); + break; + case 5: + len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, + "%d.%d.%d.%d.%d", ver[0], ver[1], ver[2], + ver[3], ver[4]); + break; + default: + len_str = strscpy(version_str, "UNKNOWN", + DCP_FW_VERSION_STR_LEN); + if (len >= 0) + len = -EOVERFLOW; + break; + } -static const struct apple_rtkit_ops external_h17p_rtkit_ops = { - .flags = APPLE_RTKIT_COPROC_PLACES_BUFFERS, - .crashed = dcp_rtk_crashed, - .recv_message = dcp_recv_msg, - .shmem_setup = dcp_rtk_shmem_setup, - .shmem_destroy = dcp_rtk_shmem_destroy, -}; + if (len_str >= DCP_FW_VERSION_STR_LEN) + dev_warn(dev, "'%s' truncated: '%s'\n", name, version_str); -void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message) + return len; +} + +static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) { - if (READ_ONCE(dcp->quiescing)) - return; + const struct apple_dcp_hw_data *hw = of_device_get_match_data(dev); + const struct dcp_v14_board *v14_board; + char compat_str[DCP_FW_VERSION_STR_LEN]; + char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; - trace_dcp_send_msg(dcp, endpoint, message); /* - * The adopted 14.7 session shares this mailbox with the panel link. - * A non-sleeping send fails while that FIFO is full, and the - * DisplayPort handshake then waits for a reply that was never sent. + * SoCs introduced with H17-generation firmware pin its interface; the + * loader may not recognise their boot firmware version. */ - ret = apple_rtkit_send_message(dcp->rtk, endpoint, message, NULL, - dcp->fw_compat != DCP_FIRMWARE_V_14_7 || - in_atomic()); - if (ret) - dev_warn_ratelimited(dcp->dev, "DCP send ep %02x failed: %d\n", - endpoint, ret); -} + if (hw && hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) + return hw->firmware_compat; -int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - struct platform_device *pdev = to_apple_crtc(crtc)->dcp; - struct apple_dcp *dcp = platform_get_drvdata(pdev); - struct drm_crtc_state *crtc_state; - bool needs_modeset; + /* firmware version is just informative */ + dcp_read_fw_version(dev, "apple,firmware-version", fw_str); - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) - return iomfb_v26_6_atomic_check(dcp, crtc, state); - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) - return iomfb_v14_7_atomic_check(dcp, crtc, state); + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0) { + dev_err(dev, "Could not read 'apple,firmware-compat': %d\n", ret); + return DCP_FIRMWARE_UNKNOWN; + } - if (dcp->crashed) - return -EINVAL; + /* 25G83 (J613; J615 experimental) has a distinct callback table, never a v14 fallback. */ + if (of_device_is_compatible(dev->of_node, "apple,t8122-dcp") && + of_property_present(dev->of_node, "apple,j613-25g83-profile")) { + const char *uuid; + u32 profile; - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + if (apple_t8122_25g83_board() && + !of_property_read_u32(dev->of_node, "apple,j613-25g83-profile", &profile) && + profile == 1 && !strcmp(compat_str, "26.6.2") && + !of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) && + !strcmp(uuid, "C042E95C-B9D8-3F0E-94B3-582A08AA6FDD")) + return DCP_FIRMWARE_V_26_6; + return DCP_FIRMWARE_UNKNOWN; + } - /* External H17P does not program adaptive sync. */ - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL && - crtc_state->vrr_enabled) - return -EINVAL; /* - * A Type-C display routed to another pipeline: its route follows - * this CRTC if it can (see dcp_typec_follow_check()), and the mode is - * one of the display's, which only the pipeline driving it knows. + * The T8122 external processor: Linux starts it and drives 14.7 + * firmware through the classic IOMFB template, as on M1/M2. */ - if (crtc_state->enable && drm_atomic_crtc_needs_modeset(crtc_state) && - dcp_typec_follows_crtc(dcp)) { - int ret = dcp_typec_follow_check(dcp, crtc, state, - &crtc_state->mode); + if (of_device_is_compatible(dev->of_node, "apple,t8122-dcpext")) { + if (!strcmp(compat_str, "14.7.0")) + return DCP_FIRMWARE_V_14_7_TEMPLATE; + dev_err(dev, "T8122 external display not started: DCP firmware-compat %s is not 14.7.0\n", + compat_str); + return DCP_FIRMWARE_UNKNOWN; + } - if (ret) - return ret; + /* The T6030 external processors: the 14.x firmware IOMFB only. */ + if (of_device_is_compatible(dev->of_node, "apple,t6030-dcpext")) { + if (!strcmp(compat_str, "14.7.0")) + return DCP_FIRMWARE_V_14_7; + dev_err(dev, "T6030 display not started: DCP firmware-compat %s is not 14.7.0\n", + compat_str); + return DCP_FIRMWARE_UNKNOWN; } - needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || - !READ_ONCE(dcp->mode_state.valid); - if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { - /* - * Resume restores the mode before the firmware reports the - * display back, so a plane-only commit lands here while the - * connector is still marked disconnected. Rejecting it makes - * the compositor fail every flip and give up on the output; - * dcp_flush() defers the commit until the link returns. - */ - dev_dbg(dcp->dev, - "crtc_atomic_check: deferring commit, link still down\n"); + /* The internal panels with a board record (T6030, J613): the same. */ + v14_board = iomfb_v14_7_board(dev); + if (IS_ERR(v14_board)) + return DCP_FIRMWARE_UNKNOWN; + if (v14_board) { + if (!strcmp(compat_str, "14.7.0")) + return DCP_FIRMWARE_V_14_7; + dev_err(dev, "%s display not started: DCP firmware-compat %s is not 14.7.0\n", + iomfb_v14_7_board_name(v14_board), compat_str); + return DCP_FIRMWARE_UNKNOWN; } - return 0; + if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_12_3; + /* + * m1n1 reports firmware version 13.5 as compatible with 13.3. This is + * only true for the iomfb endpoint. The interface for the dptx-port + * endpoint changed between 13.3 and 13.5. The driver will only support + * firmware 13.5. Check the actual firmware version for compat version + * 13.3 until m1n1 reports 13.5 as "firmware-compat". + */ + else if ((strncmp(compat_str, "13.3.0", sizeof(compat_str)) == 0) && + (strncmp(fw_str, "13.5.0", sizeof(compat_str)) == 0)) + return DCP_FIRMWARE_V_13_5; + else if (strncmp(compat_str, "13.5.0", sizeof(compat_str)) == 0) + return DCP_FIRMWARE_V_13_5; + + dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", + compat_str, fw_str); + + return DCP_FIRMWARE_UNKNOWN; } -int dcp_get_connector_type(struct platform_device *pdev) +static int dcp_connector_type_from_dt(struct device_node *np) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (of_property_match_string(np, "apple,connector-type", "HDMI-A") >= 0) + return DRM_MODE_CONNECTOR_HDMIA; + if (of_property_match_string(np, "apple,connector-type", "DP") >= 0) + return DRM_MODE_CONNECTOR_DisplayPort; + if (of_property_match_string(np, "apple,connector-type", "USB-C") >= 0) + return DRM_MODE_CONNECTOR_USB; - return dcp->fixed_connector_type; + return DRM_MODE_CONNECTOR_Unknown; } -#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) -#define DPTX_TUNNEL_CONNECT_TIMEOUT msecs_to_jiffies(8000) -#define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) -#define DPTX_RECONNECT_RETRIES 5 - -static struct dcp_fabric_session dcp_session_locked(struct apple_dcp *dcp) +static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) { - struct dcp_fabric_session session = { .generation = dcp->typec_generation }; - - lockdep_assert_held(&dcp->hpd_mutex); - scoped_guard(mutex, &dcp->tb_lock) - session.cookie = dcp->tb_generation; - return session; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_disable_typec_work_h17p(dcp, release_cable); + scoped_guard(mutex, &dcp->hpd_mutex) { + dcp->typec_work_enabled = false; + if (release_cable) + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + } + /* Block new enqueues as well as draining users of the AFK endpoints. */ + disable_delayed_work_sync(&dcp->typec_reconnect_wq); + disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->hdmi_settle_wq); + disable_delayed_work_sync(&dcp->hdmi_recheck_wq); + dcp_hdmi_disable(dcp); } -static bool dcp_session_valid_locked(struct apple_dcp *dcp, - struct dcp_fabric_session session, bool cable) +static void dcp_enable_typec_work(struct apple_dcp *dcp) { - return dcp_fabric_session_valid(session, dcp_session_locked(dcp), - !cable || dcp->typec_cable_connected, - READ_ONCE(dcp->tb_retiring) || - READ_ONCE(dcp->typec_follow_retiring)); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_enable_typec_work_h17p(dcp); + enable_delayed_work(&dcp->typec_reconnect_wq); + enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->hdmi_settle_wq); + enable_delayed_work(&dcp->hdmi_recheck_wq); + dcp_hdmi_enable(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->typec_work_enabled = true; + /* A cable can be routed before the DRM component binds. */ + if (READ_ONCE(dcp->typec_cable_connected)) + dcp_queue_typec_reconnect(dcp, 0); } -/* Recovery must not wake a deliberately parked or disabled output. */ -static bool dcp_connect_session_valid_locked(struct apple_dcp *dcp, - struct dcp_fabric_session session, - bool cable, bool recovery) +/* + * The M3 internal displays (14.x and 26.6 IOMFB) also accept + * appledrm.t6030_show_notch=1, the switch existing J613/T6030 installs pass, + * so the panel comes up at its full height (2560x1664 on the J613) instead + * of the module refusing an unknown parameter. + */ +static bool dcp_show_notch(struct apple_dcp *dcp) { - return (!recovery || (dcp->typec_work_enabled && - !READ_ONCE(dcp->typec_crtc_off) && - !READ_ONCE(dcp->crashed))) && - dcp_session_valid_locked(dcp, session, cable); + if (dcp->fw_compat != DCP_FIRMWARE_V_14_7 && + dcp->fw_compat != DCP_FIRMWARE_V_26_6) + return show_notch; + return show_notch || t6030_show_notch; } -/* Capture at enqueue, not when a stale worker eventually starts running. */ -void dcp_queue_typec_reconnect(struct apple_dcp *dcp, unsigned long delay) +/* + * T8122 dcpext (DCP_FIRMWARE_V_14_7_TEMPLATE): if the PMP runs, the display + * gate added its DISPEXT request and pointed the dcpext at it + * (apple,pmp-report). As on T6030, run the processor only once the PMP has + * acknowledged that request: the running PMP owns the SoC power and + * performance states and treats a block whose request it has not taken as + * idle. The internal display started the PMP long before this, so the wait + * is short; it is bounded, and without an acknowledgement the processor is + * started anyway (as before the request existed), with a warning. + */ +#define DCP_EXT_PMP_TIMEOUT_MS 3000 + +static void dcp_ext_wait_pmp(struct apple_dcp *dcp) { - guard(mutex)(&dcp->hpd_mutex); + struct device_node *entry; + unsigned long deadline; + int ret; - if (!dcp->typec_cable_connected || READ_ONCE(dcp->tb_retiring) || - READ_ONCE(dcp->typec_follow_retiring)) + if (dcp->fw_compat != DCP_FIRMWARE_V_14_7_TEMPLATE) return; - dcp->typec_reconnect_session = dcp_session_locked(dcp); - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, delay); + entry = of_parse_phandle(dcp->dev->of_node, "apple,pmp-report", 0); + if (!entry) { + dev_info(dcp->dev, "no PMP DISPEXT request; starting without one\n"); + return; + } + deadline = jiffies + msecs_to_jiffies(DCP_EXT_PMP_TIMEOUT_MS); + for (;;) { + ret = apple_pmp_report_wait_ready(entry, + msecs_to_jiffies(DCP_EXT_PMP_TIMEOUT_MS)); + /* -EIO: the PMP runs, but has not acknowledged the request yet. */ + if (ret != -EIO || time_after(jiffies, deadline)) + break; + msleep(20); + } + of_node_put(entry); + if (ret) + dev_warn(dcp->dev, + "the PMP has not acknowledged the DISPEXT request (%d); starting anyway\n", + ret); + else + dev_info(dcp->dev, "PMP acknowledged the DISPEXT request\n"); } -#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) -#define DPTX_TUNNEL_CONNECT_TIMEOUT msecs_to_jiffies(8000) -#define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) -#define DPTX_RECONNECT_RETRIES 5 -static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, - struct dcp_fabric_session session, bool cable, - bool recovery) -static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_service) +static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *data) { - /* T8122 dcpext: HPD only after the link wait, see dcp_ext_dptx_607() */ - bool late_hpd = dcp_ext_dptx_607(dcp); - unsigned long timeout; - u8 dfp_port; - unsigned long timeout; - int ret = 0; + struct device_node *panel_np; + struct apple_dcp *dcp = dev_get_drvdata(dev); + u32 cpu_ctrl; + int ret; - if (waiting_service) - *waiting_service = false; - if (!dcp->phy) { - dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); - return -ENODEV; - } - /* A native external processor starts here; its start retries this. */ - if (dcp->external_native && !iomfb_v14_7_external_ready(dcp)) - return -EAGAIN; - /* @port selects the upstream RemotePort service/core. dptx_dfp_port - * is the downstream address: dpphy=0, dpin0=1, dpin1=2. - */ - dev_info(dcp->dev, - "%s(port=%d) die=%u atc=%u dfp_port=%u tunnel=%d typec=%d route=%s conn_type=%d connected=%d\n", - __func__, port, dcp->dptx_die, dcp->dptx_phy, dcp->dptx_dfp_port, - dcp->dptx_tunnel, dcp_is_typec_output(dcp), - dcp->active_typec_route ? "borrowed" : "fixed", - dcp->connector_type, dcp->dptxport[port].connected); - dev_dbg(dcp->dev, - "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", - __func__, port, dcp->dptx_die, dcp->dptx_phy, - dcp_is_typec_output(dcp), - dcp->active_typec_route ? "borrowed" : "fixed", - dcp->connector_type, dcp->dptxport[port].connected); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + if (ret) + return ret; - mutex_lock(&dcp->hpd_mutex); - if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { - ret = -ESTALE; - goto out_unlock; - } - if (dcp->external && dcpext_scanout_terminal(dcp)) { - mutex_unlock(&dcp->hpd_mutex); - return -ESHUTDOWN; - } - if (!dcp->dptxport[port].enabled) { - if (!dptx_service_available(&dcp->dptxport[port])) { - dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); - ret = -ENODEV; - /* No firmware request has been made; the first publication can retry. */ - if (waiting_service && !READ_ONCE(dcp->dptxport[port].closed)) { - *waiting_service = true; - ret = -EAGAIN; - } - goto out_unlock; + /* A native external processor mapped these at probe, for its start work. */ + if (!dcp->external_native) { + dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), + "coproc"); + if (IS_ERR(dcp->coproc_reg)) + return PTR_ERR(dcp->coproc_reg); } - if (dcp->dptxport[port].connected) - goto out_unlock; - if (dcp->external) - smp_store_release(&dcp->external_link_ready, false); + if (dcp->index || dcp->dptx_phy || dcp->dptx_die) + dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", + dcp->index, dcp->dptx_phy, dcp->dptx_die); - reinit_completion(&dcp->dptxport[port].linkcfg_completion); - WRITE_ONCE(dcp->dptxport[port].tile_hint, false); - dcp->dptxport[port].atcphy = dcp->phy; - /* a tiled display's second half comes in on the port's dpin1 */ - dfp_port = port && READ_ONCE(dcp->split.active) ? 2 : dcp->dptx_dfp_port; - if (!dcp_ext_dptx_607(dcp)) { - ret = dptxport_validate_connection(dcp->dptxport[port].service, - dfp_port, - dcp->dptx_phy, dcp->dptx_die); - if (ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", - dcp->dptx_die, dcp->dptx_phy, ret); - goto out_unlock; + if (!dcp_show_notch(dcp)) + ret = of_property_read_u32(dev->of_node, "apple,notch-height", + &dcp->notch_height); + + if (dcp->notch_height > MAX_NOTCH_HEIGHT) + dcp->notch_height = MAX_NOTCH_HEIGHT; + if (dcp->notch_height > 0) + dev_info(dev, "Detected display with notch of %u pixel\n", dcp->notch_height); + + /* initialize brightness scale to a sensible default to avoid divide by 0*/ + dcp->brightness.scale = 65536; + panel_np = of_get_compatible_child(dev->of_node, "apple,panel-mini-led"); + if (panel_np) + dcp->panel.has_mini_led = true; + else + panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); + + if (panel_np) { + const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; + + if (of_device_is_available(panel_np)) { + ret = of_property_read_u32(panel_np, "apple,max-brightness", + &dcp->brightness.maximum); + if (ret) + dev_err(dev, "Missing property 'apple,max-brightness'\n"); } - ret = dptxport_validate_connection(dcp->dptxport[port].service, - dcp->dptx_dfp_port, - dcp->dptx_phy, dcp->dptx_die); - if (ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", - dcp->dptx_die, dcp->dptx_phy, ret); - goto out_unlock; - } - if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { - ret = -ESTALE; - goto out_unlock; - } - ret = dptxport_connect(dcp->dptxport[port].service, - dfp_port, - ret = dptxport_connect(dcp->dptxport[port].service, - dcp->dptx_dfp_port, - dcp->dptx_phy, dcp->dptx_die, - dcp_is_typec_output(dcp) && !dcp_ext_dptx_607(dcp)); - if (ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", - dcp->dptx_die, dcp->dptx_phy, ret); - goto out_unlock; - } + of_property_read_u32(panel_np, "width-mm", &dcp->panel.width_mm); + /* use adjusted height as long as the notch is hidden */ + of_property_read_u32(panel_np, height_prop[!dcp->notch_height], + &dcp->panel.height_mm); - if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { - ret = -ESTALE; - goto out_unlock; - } - ret = dptxport_request_display(dcp->dptxport[port].service); - if (ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: failed to request display: %d\n", - ret); - goto out_release; - } - if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { - ret = -ESTALE; - goto out_release; + of_node_put(panel_np); + dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; + dcp->connector_type = DRM_MODE_CONNECTOR_eDP; + INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); + mutex_init(&dcp->bl_register_mutex); + INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); } - dcp->dptxport[port].connected = true; - if (dcp_is_typec_output(dcp) && !late_hpd) { - if (dcp_uses_t6020_tunnel_flow(dcp)) - ret = dptxport_set_hpd_timeout(dcp->dptxport[port].service, - true, 8000); - else - ret = dptxport_set_hpd(dcp->dptxport[port].service, true); - if (ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: failed to assert Type-C HPD: %d\n", - ret); - dcp->dptxport[port].connected = false; - goto out_release; - } + + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { + ret = iomfb_v26_6_bind(dcp); + if (!ret) + enable_work(&dcp->dimensions_wq); + return ret; } - mutex_unlock(&dcp->hpd_mutex); - /* - * The display interface of a native external processor opens once a - * display is attached; the firmware then describes that display. - */ - if (dcp->external_native) { - ret = iomfb_v14_7_external_open(dcp); - if (ret) { - mutex_lock(&dcp->hpd_mutex); - goto out_disconnect; + /* The running T6030 firmware is adopted as is. */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (dcp->external_native) { + WRITE_ONCE(dcp->external_detached, false); + enable_work(&dcp->vblank_wq); + enable_delayed_work(&dcp->swap_watchdog_wq); + enable_delayed_work(&dcp->external_retry_wq); + enable_work(&dcp->dimensions_wq); + dcp_enable_typec_work(dcp); + return 0; } - } - timeout = dcp_uses_t6020_tunnel_flow(dcp) ? - DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; - ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, - timeout); - mutex_lock(&dcp->hpd_mutex); - /* A revoked wait must not disconnect or publish readiness for a new session. */ - if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { - ret = -ESTALE; - goto out_unlock; - } - if (!ret && late_hpd) { - /* Without HPD this firmware does not train: assert it now. */ - dev_dbg(dcp->dev, - "dcp_dptx_connect: no link configuration on port %u yet, asserting HPD\n", - port); - } else if (!ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: timed out waiting for port %u link configuration\n", - port); - ret = -ETIMEDOUT; - goto out_disconnect; - } else { - dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", - jiffies_to_msecs(timeout - ret)); + if (dcp->external) + return 0; + ret = iomfb_v14_7_bind(dcp); + if (!ret) + enable_work(&dcp->dimensions_wq); + return ret; } - usleep_range(5, 10); - if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { - ret = -ESTALE; - goto out_unlock; - } + ret = dcp_create_piodma_iommu_dev(dcp); + if (ret || !dcp->iommu_dom) + return dev_err_probe(dev, ret, + "Failed to created PIODMA iommu child device"); - if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort || late_hpd) { - ret = dptxport_set_hpd(dcp->dptxport[port].service, true); - if (ret && late_hpd) - dev_err(dcp->dev, - "dcp_dptx_connect: failed to assert HPD after the link wait: %d\n", - ret); - if (ret && (dcp->external || late_hpd)) - goto out_disconnect; - } - if (!dcp_connect_session_valid_locked(dcp, session, cable, recovery)) { - ret = -ESTALE; - goto out_unlock; + ret = dcp_get_disp_regs(dcp); + if (ret) { + dev_err(dev, "failed to find display registers\n"); + return ret; } - if (dcp->external) { - /* - * The port must still have its display: a Type-C cable, or HPD - * on the fixed HDMI output, which has no cable flag. - */ - if (!dcp->dptxport[port].connected || - !(READ_ONCE(dcp->typec_cable_connected) || dcp_hdmi_port_attached(dcp)) || - READ_ONCE(dcp->crashed) || dcpext_scanout_terminal(dcp)) { - ret = -ENOLINK; - goto out_unlock; + + dcp->clk = devm_clk_get(dev, NULL); + if (IS_ERR(dcp->clk)) + return dev_err_probe(dev, PTR_ERR(dcp->clk), + "Unable to find clock\n"); + + bitmap_zero(dcp->memdesc_map, DCP_MAX_MAPPINGS); + // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry + set_bit(0, dcp->memdesc_map); + + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + + dcp->swapped_out_fbs = + (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); + + dcp_ext_wait_pmp(dcp); + + cpu_ctrl = + readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, + dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); + + dcp->rtk = devm_apple_rtkit_init(dev, dcp, "mbox", 0, &rtkit_ops); + if (IS_ERR(dcp->rtk)) + return dev_err_probe(dev, PTR_ERR(dcp->rtk), + "Failed to initialize RTKit\n"); + + ret = apple_rtkit_wake(dcp->rtk); + if (ret) + return dev_err_probe(dev, ret, + "Failed to boot RTKit: %d\n", ret); + enable_work(&dcp->dimensions_wq); + schedule_work(&dcp->dimensions_wq); + dcp_enable_typec_work(dcp); + return ret; +} + +static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when); + +/* + * The T8122 dcpext boots its firmware here. If that fails, it must not take + * the internal panel's DRM device down with it: keep it as an unusable pipe. + */ +static int dcp_comp_bind(struct device *dev, struct device *main, void *data) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + int ret; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_comp_bind_h17p(dev, main, data); + + if (READ_ONCE(dcp->ext_dead)) + return 0; + ret = dcp_comp_bind_pipe(dev, main, data); + if (ret && dcp->ext_failsafe) { + dcp_ext_give_up(dcp, ret, "bind"); + return 0; + } + return ret; +} + +/* + * We need to shutdown DCP before tearing down the display subsystem. Otherwise + * the DCP will crash and briefly flash a green screen of death. + */ +static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp && dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_comp_unbind_h17p(dev, main, data); + if (!dcp || READ_ONCE(dcp->ext_dead)) + return; + + disable_work_sync(&dcp->dimensions_wq); + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { + iomfb_v26_6_unbind(dcp); + return; + } + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { + if (dcp->external_native) { + /* Bind's wait_ready enabled it, as for any HDMI port. */ + if (dcp->hdmi_hpd_irq) + disable_irq(dcp->hdmi_hpd_irq); + dcp_disable_typec_work(dcp, true); + disable_delayed_work_sync(&dcp->external_retry_wq); + disable_delayed_work_sync(&dcp->swap_watchdog_wq); + disable_work_sync(&dcp->vblank_wq); + } + iomfb_v14_7_unbind(dcp); + return; + } + + if (dcp->hdmi_hpd_irq) + disable_irq(dcp->hdmi_hpd_irq); + + dcp_disable_typec_work(dcp, true); + /* RTKit is released after this unbind callback, and can still deliver + * a final swap completion or hotplug while its receive queue drains. + */ + disable_delayed_work_sync(&dcp->swap_watchdog_wq); + disable_work_sync(&dcp->vblank_wq); + typec_mux_put(dcp->typec_mux); + + if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP) { + /* Registration runs asynchronously and its devres can belong + * to the platform device rather than the component bind group. + * Stop callbacks and userspace writes while the CRTC, RTKit and + * IOMMU they use are still available. + */ + disable_work_sync(&dcp->bl_register_wq); + disable_work_sync(&dcp->bl_update_wq); + if (dcp->brightness.bl_dev) { + devm_backlight_device_unregister(dev, dcp->brightness.bl_dev); + dcp->brightness.bl_dev = NULL; + } + } + + if (dcp->avep) { + av_service_disconnect(dcp); + afk_shutdown(dcp->avep); + dcp->avep = NULL; + } + + dcp_fabric_shutdown_dptx(dcp); + + if (dcp->ibootep) { + afk_shutdown(dcp->ibootep); + dcp->ibootep = NULL; + } + + if (dcp->systemep) { + afk_shutdown(dcp->systemep); + dcp->systemep = NULL; + } + + if (dcp->dcpavservep) { + afk_shutdown(dcp->dcpavservep); + dpavservep_detach(dcp); + dcp->dcpavservep = NULL; + } + + if (dcp->shmem) + iomfb_shutdown(dcp); + + if (dcp->piodma) { + dcp->iommu_dom = NULL; + of_platform_device_destroy(&dcp->piodma->dev, NULL); + dcp->piodma = NULL; + } + + devm_clk_put(dev, dcp->clk); + dcp->clk = NULL; +} + +static const struct component_ops dcp_comp_ops = { + .bind = dcp_comp_bind, + .unbind = dcp_comp_unbind, +}; + +/* + * A T6030 external processor's fixed (HDMI) output is optional. Its Type-C + * routes must register whatever happens to that output: every USB-C port + * lists them and waits for them, as the panel's DRM device waits for this + * processor. Keep deferring while the HDMI parts may still appear, then go + * on as a Type-C-only pipe. Elsewhere the error ends the probe, as before. + */ +static int dcp_fixed_output_error(struct apple_dcp *dcp, int err, const char *what) +{ + if (!dcp->external_native) + return err; + if (err == -EPROBE_DEFER) { + err = driver_deferred_probe_check_state(dcp->dev); + if (err == -EPROBE_DEFER) + return err; + } + dev_err(dcp->dev, "HDMI output unusable (%s: %d); USB-C displays are not affected\n", + what, err); + if (dcp->fixed_route_selected) + mux_control_deselect(dcp->xbar); + dcp->fixed_route_selected = false; + dcp->xbar = NULL; + if (!IS_ERR_OR_NULL(dcp->typec_mux)) + typec_mux_put(dcp->typec_mux); + dcp->typec_mux = NULL; + dcp->phy_managed_by_typec = false; + /* A requested HPD interrupt stays disabled: nothing enables it now. */ + dcp->hdmi_hpd_irq = 0; + dcp->hdmi_hpd = NULL; + dcp->hdmi_pwren = NULL; + dcp->dp2hdmi_pwren = NULL; + dcp->phy = NULL; + dcp->fixed_phy = NULL; + dcp->fixed_connector_type = DRM_MODE_CONNECTOR_USB; + dcp->connector_type = DRM_MODE_CONNECTOR_USB; + return 0; +} + +/* + * A T6030 external processor whose Type-C routes did not register: disable + * them in the device tree so that the ports stop waiting for them, and add + * the processor to the DRM device as a pipe that never starts, so that the + * panel is not held up either. + */ +static int dcp_native_routes_error(struct apple_dcp *dcp, int err) +{ + if (!dcp->external_native) + return err; + if (err == -EPROBE_DEFER) { + err = driver_deferred_probe_check_state(dcp->dev); + if (err == -EPROBE_DEFER) + return err; + } + dev_err(dcp->dev, "Type-C display routes unusable: %d; USB-C ports go on without this processor\n", + err); + dcp_typec_routes_disable(dcp); + return 0; +} + +static int dcp_platform_probe_pipe(struct platform_device *pdev) +{ + enum dcp_firmware_version fw_compat; + struct device *dev = &pdev->dev; + struct apple_dcp *dcp; + int ret, surf, num_surfs; + bool routes_failed = false; + u32 surf_en; + u32 mux_index; + + fw_compat = dcp_check_firmware_version(dev); + if (fw_compat == DCP_FIRMWARE_UNKNOWN) + return -ENODEV; + + /* Check for "apple,bw-scratch" to avoid probing appledrm with outdated + * device trees. This prevents replacing simpledrm and ending up without + * display. + */ + if (fw_compat != DCP_FIRMWARE_V_14_7 && + fw_compat != DCP_FIRMWARE_V_26_6 && + !of_property_present(dev->of_node, "apple,bw-scratch")) + return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " + "Use devicetree matching this kernel.\n"); + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + + dcp->fw_compat = fw_compat; + dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext"); + dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); + dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); + dcp->dev = dev; + /* + * Type-C and Thunderbolt routes can be activated as soon as they are + * registered below, before the DRM device binds. + */ + dcp_modes_init(dcp); + /* Registered before callback resources, so they drain before the catalog. */ + ret = devm_add_action_or_reset(dev, dcp_modes_release, dcp); + if (ret) + return ret; + mutex_init(&dcp->hpd_mutex); + mutex_init(&dcp->tb_lock); + dcp_fabric_init(dcp); + spin_lock_init(&dcp->mode_state.lock); + mutex_init(&dcp->dcpavserv.lock); + dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); + dcp->fixed_connector_type = dcp_connector_type_from_dt(dev->of_node); + dcp->connector_type = dcp->fixed_connector_type; + of_property_read_u32(dev->of_node, "apple,dcp-index", &dcp->index); + of_property_read_u32(dev->of_node, "apple,dptx-phy", &dcp->dptx_phy); + of_property_read_u32(dev->of_node, "apple,dptx-die", &dcp->dptx_die); + dcp->fixed_dptx_phy = dcp->dptx_phy; + init_completion(&dcp->typec_iomfb_hpd_ready); + INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, + dcp_typec_reconnect_work); + INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, + dcp_placeholder_edid_work); + /* Balanced by enable at successful component bind. */ + disable_delayed_work(&dcp->typec_reconnect_wq); + disable_delayed_work(&dcp->placeholder_edid_wq); + /* + * A native external pipe's bind enables these; firmware callbacks may + * come before it. The 14.7 panel never uses them, but suspend cancels + * the watchdog of every processor. + */ + if (fw_compat == DCP_FIRMWARE_V_14_7 || fw_compat == DCP_FIRMWARE_V_26_6) { + INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); + INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); + disable_work(&dcp->vblank_wq); + disable_delayed_work(&dcp->swap_watchdog_wq); + } + + platform_set_drvdata(pdev, dcp); + + if (fw_compat == DCP_FIRMWARE_V_26_6) { + ret = iomfb_v26_6_probe(dcp); + if (ret) + return ret; + } + if (fw_compat == DCP_FIRMWARE_V_14_7 && !dcp->external) { + ret = iomfb_v14_7_probe(dcp); + if (ret) + return ret; + } + if (dcp->external) + dcp->hw.num_dptx_ports = 2; + + dcp->phy = devm_phy_optional_get(dev, "dp-phy"); + if (IS_ERR(dcp->phy) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->phy), "dp-phy"); + if (ret) + return ret; + } + if (IS_ERR(dcp->phy)) { + dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(dcp->phy)); + return PTR_ERR(dcp->phy); + } + dcp->fixed_phy = dcp->phy; + + bitmap_zero(dcp->iomfb_surfaces, DCP_MAX_PLANES); + if (!of_property_present(dev->of_node, "apple,iomfb-surfaces")) + num_surfs = 0; + else + num_surfs = of_property_count_elems_of_size(dev->of_node, + "apple,iomfb-surfaces", + sizeof(u32)); + + if (fw_compat == DCP_FIRMWARE_V_14_7 || fw_compat == DCP_FIRMWARE_V_26_6) { + set_bit(0, dcp->iomfb_surfaces); + } else if (num_surfs == 0 || num_surfs == -ENODATA) { + set_bit(0, dcp->iomfb_surfaces); + set_bit(1, dcp->iomfb_surfaces); + } else if (num_surfs < 0) { + return num_surfs; + } else if (num_surfs > DCP_MAX_PLANES) { + dev_err(dev, "Number of iomfb-surfaces (%d) exceeds DCP_MAX_PLANES\n", + num_surfs); + return -EINVAL; + } + + surf = 0; + of_property_for_each_u32(dev->of_node, "apple,iomfb-surfaces", surf_en) { + if (surf_en) + set_bit(surf, dcp->iomfb_surfaces); + surf++; + } + + if (dcp->phy) { + int ret; + /* + * Request DP2HDMI related GPIOs as optional for DP-altmode + * compatibility. J180D misses a dp2hdmi-pwren GPIO in the + * template ADT. TODO: check device ADT + */ + dcp->hdmi_hpd = devm_gpiod_get_optional(dev, "hdmi-hpd", GPIOD_IN); + if (IS_ERR(dcp->hdmi_hpd) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->hdmi_hpd), "hdmi-hpd"); + if (ret) + return ret; + goto fixed_done; + } + if (IS_ERR(dcp->hdmi_hpd)) + return PTR_ERR(dcp->hdmi_hpd); + ret = dcp_hdmi_init(dcp); + if (ret) + return ret; + if (dcp->hdmi_hpd) { + int irq = gpiod_to_irq(dcp->hdmi_hpd); + if (irq < 0 && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, irq, "hdmi-hpd interrupt"); + if (ret) + return ret; + goto fixed_done; + } + if (irq < 0) { + dev_err(dev, "failed to translate HDMI hpd GPIO to IRQ\n"); + return irq; + } + dcp->hdmi_hpd_irq = irq; + + ret = devm_request_threaded_irq(dev, dcp->hdmi_hpd_irq, + dcp_dp2hdmi_hpd_edge, dcp_dp2hdmi_hpd, + IRQF_ONESHOT | IRQF_NO_AUTOEN | + IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, + "dp2hdmi-hpd-irq", dcp); + if (ret < 0 && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, ret, "hdmi-hpd interrupt"); + if (ret) + return ret; + goto fixed_done; + } + if (ret < 0) { + dev_err(dev, "failed to request HDMI hpd irq %d: %d\n", + irq, ret); + return ret; + } + } + + /* + * Power DP2HDMI on as it is required for the HPD irq. + * TODO: check if one is sufficient for the hpd to save power + * on battery powered Macbooks. + */ + dcp->hdmi_pwren = devm_gpiod_get_optional(dev, "hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->hdmi_pwren) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->hdmi_pwren), "hdmi-pwren"); + if (ret) + return ret; + goto fixed_done; + } + if (IS_ERR(dcp->hdmi_pwren)) + return PTR_ERR(dcp->hdmi_pwren); + + dcp->dp2hdmi_pwren = devm_gpiod_get_optional(dev, "dp2hdmi-pwren", GPIOD_OUT_HIGH); + if (IS_ERR(dcp->dp2hdmi_pwren) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->dp2hdmi_pwren), "dp2hdmi-pwren"); + if (ret) + return ret; + goto fixed_done; + } + if (IS_ERR(dcp->dp2hdmi_pwren)) + return PTR_ERR(dcp->dp2hdmi_pwren); + + /* + * A DCP may have both a fixed HDMI/DP route and allocatable Type-C + * routes. Keep the fixed route selected until the allocator borrows + * this otherwise-idle pipeline for a Type-C display. + */ + ret = dcp->fixed_phy ? + of_property_read_u32(dev->of_node, "mux-index", &mux_index) : + -ENODATA; + if (!ret) { + dcp->fixed_mux_index = mux_index; + dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); + if (IS_ERR(dcp->xbar) && dcp->external_native) { + ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->xbar), "dp-xbar"); + if (ret) + return ret; + goto fixed_done; + } + if (IS_ERR(dcp->xbar)) { + dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); + return PTR_ERR(dcp->xbar); + } + ret = mux_control_select(dcp->xbar, mux_index); + if (ret) + dev_warn(dev, "mux_control_select failed: %d\n", ret); + else + dcp->fixed_route_selected = true; + + /* + * Switch atcphy to DP-only. should move to a Macbook Pro + * 14-/16-inch specific DP-to-HDMI drm_bridge. + */ + dcp->typec_mux = fwnode_typec_mux_get(dev_fwnode(dcp->dev)); + if (!IS_ERR_OR_NULL(dcp->typec_mux)) { + struct typec_altmode alt = { + .svid = USB_TYPEC_DP_SID, + }; + struct typec_mux_state state = { + .alt = &alt, + .mode = TYPEC_DP_STATE_C, + }; + int ret = typec_mux_set(dcp->typec_mux, &state); + dev_info(dev, "typec_mux_set() returned: %d\n", ret); + if (!ret) + dcp->phy_managed_by_typec = true; + } else { + dev_info(dev, "fwnode_typec_mux_get() returned: %ld\n", + IS_ERR(dcp->typec_mux) ? PTR_ERR(dcp->typec_mux) : 0); + dcp->typec_mux = NULL; + } + } + } + +fixed_done: + ret = dcp_register_typec_routes(dcp); + if (ret) { + ret = dcp_native_routes_error(dcp, ret); + if (ret) + return ret; + routes_failed = true; + } + + /* + * The manual external path stays out of the panel's DRM device. A + * native external processor joins it, and probe completes for it even + * when its handoff is unusable, so that the panel is never held up. + */ + if (dcp->external && !dcp->external_native) + return iomfb_v14_7_external_start(dcp); + if (dcp->external_native) { + ret = iomfb_v14_7_external_prepare(dcp); + if (ret) + return ret; + if (routes_failed) + iomfb_v14_7_external_refuse(dcp, "its Type-C display routes did not register"); + } + + ret = component_add(&pdev->dev, &dcp_comp_ops); + if (ret && dcp->fw_compat == DCP_FIRMWARE_V_26_6) + iomfb_v26_6_unbind(dcp); + /* A failed bind run from here may already have started RTKit. */ + if (ret && dcp->fw_compat == DCP_FIRMWARE_V_14_7) + iomfb_v14_7_remove(dcp); + return ret; +} + +/* + * Marks a T8122 dcpext unusable, for good: its Type-C routes are disabled in + * the device tree, so that a USB-C port controller probing later does not + * wait for them, and a route already registered ignores mode changes (see + * dcp_typec_route_set()), so that USB on the port goes on. + */ +static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when) +{ + WRITE_ONCE(dcp->ext_dead, true); + dev_err(dcp->dev, "external display processor unusable (%s failed: %d); the internal display goes on without it\n", + when, err); + dcp_typec_routes_disable(dcp); +} + +/* + * Whether the DP PHY and display crossbar of every Type-C route are there. + * Routes register one at a time, and one registered cannot be taken back, + * since a port controller may hold it: a later route waiting for its + * supplier would leave the dcpext unusable for the boot. Returns + * -EPROBE_DEFER until all of them are; other errors are left to the route + * probe, which reports them. + */ +static int dcp_ext_route_suppliers(struct device *dev) +{ + struct device_node *routes __free(device_node) = + of_get_child_by_name(dev->of_node, "typec-routes"); + char name[16]; + u32 index; + + if (!routes) + return 0; + for_each_available_child_of_node_scoped(routes, route_np) { + struct mux_control *xbar; + struct phy *phy; + + if (of_property_read_u32(route_np, "reg", &index)) + continue; + snprintf(name, sizeof(name), "typec%u", index); + phy = of_phy_get(dev->of_node, name); + if (PTR_ERR_OR_ZERO(phy) == -EPROBE_DEFER) + return -EPROBE_DEFER; + if (!IS_ERR(phy)) + of_phy_put(phy); + xbar = mux_control_get(dev, name); + if (PTR_ERR_OR_ZERO(xbar) == -EPROBE_DEFER) + return -EPROBE_DEFER; + if (!IS_ERR(xbar)) + mux_control_put(xbar); + } + return 0; +} + +/* + * The T8122 dcpext is a component of the panel's DRM device, so it must + * always register one, or the panel stays on the boot framebuffer. Keep + * deferring while its suppliers may still appear (bounded by the deferred + * probe timeout); on any other failure join as an unusable pipe. + */ +static void dcp_unuse_dma_domain(void *data) +{ + iommu_device_unuse_default_domain(data); +} + +static int dcp_platform_probe(struct platform_device *pdev) +{ + const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); + struct device *dev = &pdev->dev; + struct apple_dcp *dcp; + void *group; + int ret; + + if (hw && hw->firmware_compat == DCP_FIRMWARE_H17P) + return dcp_platform_probe_h17p(pdev); + + /* + * The driver manages DMA ownership itself (driver_managed_dma), so + * that a retained H17P session keeps its domain use across unbind. + * Every other processor uses its default domain while bound, as the + * driver core would. + */ + ret = iommu_device_use_default_domain(dev); + if (ret) + return ret; + ret = devm_add_action_or_reset(dev, dcp_unuse_dma_domain, dev); + if (ret) + return ret; + + if (!of_device_is_compatible(dev->of_node, "apple,t8122-dcpext")) + return dcp_platform_probe_pipe(pdev); + + ret = dcp_ext_route_suppliers(dev); + if (ret) + goto unusable; + + group = devres_open_group(dev, NULL, GFP_KERNEL); + if (!group) + return -ENOMEM; + ret = dcp_platform_probe_pipe(pdev); + if (!ret) { + devres_remove_group(dev, group); + return 0; + } + + dcp = platform_get_drvdata(pdev); + if (dcp && dcp->nr_typec_routes) { + /* Routes may be in use already: keep them, and the pipe, as they are. */ + devres_remove_group(dev, group); + dcp_ext_give_up(dcp, ret, "probe"); + return component_add(dev, &dcp_comp_ops); + } + + devres_release_group(dev, group); + platform_set_drvdata(pdev, NULL); +unusable: + if (ret == -EPROBE_DEFER) { + ret = driver_deferred_probe_check_state(dev); + if (ret == -EPROBE_DEFER) + return ret; + } + + dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); + if (!dcp) + return -ENOMEM; + dcp->dev = dev; + dcp->ext_failsafe = true; + platform_set_drvdata(pdev, dcp); + dcp_ext_give_up(dcp, ret, "probe"); + return component_add(dev, &dcp_comp_ops); +} + +static void dcp_platform_remove(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp && dcp->external && !dcp->external_native) { + iomfb_v14_7_remove(dcp); + return; + } + component_del(&pdev->dev, &dcp_comp_ops); + if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_26_6) + iomfb_v26_6_unbind(dcp); + /* Unbind does not wait for RTKit callbacks, or run if bind failed late. */ + if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_14_7) + iomfb_v14_7_remove(dcp); +} + +static void dcp_platform_shutdown(struct platform_device *pdev) +{ + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (dcp && dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_platform_shutdown_h17p(pdev); + + if (dcp && dcp->external && !dcp->external_native) + return; + component_del(&pdev->dev, &dcp_comp_ops); +} + +/* + * Why an external processor must keep the system awake, or NULL. The manual + * path refuses sleep once its firmware was ever started. A native processor + * refuses while its firmware starts or holds a display link, and while a + * display is attached to it at all, lit or not: resume does not bring an + * attached display back. Idle with nothing attached, its firmware stays + * powered across s2idle like the panel's (the gate holds its CPU domain at + * the active floor) and the next attach reconnects through it. + * + * A processor that failed (its start, or its session later) never refuses + * sleep: it drives no display until reboot, attached or not, and its power + * domains stay on across s2idle as they are, like those of a working one. + */ +static const char *dcp_external_sleep_vetoed(struct apple_dcp *dcp) +{ + const char *why; + + if (!dcp->external_native) + return (atomic_read(&dcp->external_requested) || + (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || + dcpext_scanout_requested(dcp)) ? + "external firmware/scanout attempted" : NULL; + why = iomfb_v14_7_external_busy(dcp); + if (why) + return why; + if (iomfb_v14_7_external_failed(dcp)) { + if (dcp->v14) + dev_info(dcp->dev, "external display processor failed earlier: sleeping with it as it is\n"); + return NULL; + } + if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) + return "a USB-C display is attached"; + if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd) > 0) + return "an HDMI display is attached"; + return NULL; +} + +/* dpm_prepare completes for every device before any dpm_suspend callback. + * Veto early so Thunderbolt cannot tear down a working tunnel first. The + * same HPD lock serializes this gate against explicit firmware startup. + */ +static int dcp_platform_prepare(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + const char *why; + + if (!dcp->external) + return 0; + mutex_lock(&dcp->hpd_mutex); + why = dcp_external_sleep_vetoed(dcp); + if (why) { + mutex_unlock(&dcp->hpd_mutex); + if (dcp->external_native) + dev_warn(dev, "refusing system sleep: %s\n", why); + else + dev_warn(dev, "external firmware/scanout attempted: refusing PM prepare; reboot required for retained DMA\n"); + return -EBUSY; + } + WRITE_ONCE(dcp->external_suspended, true); + mutex_unlock(&dcp->hpd_mutex); + return 0; +} + +static void dcp_platform_complete(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (!dcp->external) + return; + mutex_lock(&dcp->hpd_mutex); + WRITE_ONCE(dcp->external_suspended, false); + mutex_unlock(&dcp->hpd_mutex); +} + +static int dcp_platform_suspend(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_platform_suspend_h17p(dev); + if (READ_ONCE(dcp->ext_dead)) + return 0; + + /* Serialize PM against explicit bring-up, including queued startup work. + * Before a startup attempt the device remains suspendable. After any + * attempt we cannot prove DMA/firmware quiescence, even on failure. + */ + if (dcp->external) { + const char *why; + + mutex_lock(&dcp->hpd_mutex); + why = dcp_external_sleep_vetoed(dcp); + if (why) { + mutex_unlock(&dcp->hpd_mutex); + if (dcp->external_native) + dev_warn(dev, "suspend refused: %s\n", why); + else + dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); + return -EBUSY; } - smp_store_release(&dcp->external_link_ready, true); - dcpext_scanout_link_restored(dcp); + WRITE_ONCE(dcp->external_suspended, true); + mutex_unlock(&dcp->hpd_mutex); } + /* + * The Type-C route reports cable removal through + * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays + * connected, and resume powers the CRTC back up through dcp_poweron() + * as after DPMS off. + */ + dcp_disable_typec_work(dcp, false); + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); if (dcp->avep) - av_service_connect(dcp); + av_service_disconnect(dcp); - ret = 0; - goto out_unlock; + if (dcp->hdmi_hpd_irq) { + disable_irq(dcp->hdmi_hpd_irq); + if (!dcp->active_typec_route) { + disconnected_hpd_event(dcp->connector); + dcp_dptx_disconnect(dcp, 0); + } + } + /* + * Set the device as a wakeup device, which forces its power + * domains to stay on. We need this as we do not support full + * shutdown properly yet. + */ + device_set_wakeup_path(dev); -out_disconnect: - dcp->dptxport[port].connected = false; -out_release: - dptxport_release_display(dcp->dptxport[port].service); + return 0; +} -out_unlock: - mutex_unlock(&dcp->hpd_mutex); - return ret; +static int dcp_platform_resume(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_platform_resume_h17p(dev); + if (READ_ONCE(dcp->ext_dead)) + return 0; + + dcp_enable_typec_work(dcp); + /* The HDMI output's PHY and crossbar may have lost their setup in sleep. */ + if (dcp->external_native) + dcp_fabric_hdmi_reinit(dcp, "after system sleep"); + /* Observe future edges before sampling any edges lost in sleep. */ + if (dcp->hdmi_hpd_irq) + dcp_fabric_run_resume(&dcp_resume_ops, dcp); + + if (dcp->avep) + av_service_connect(dcp); + + return 0; } -/* - * A T6030 external processor runs as a pipe of the main DRM device when the - * display gate handed it its display DART: a piodma child on stream 4 of - * that DART, and stream 0 among the display subsystem's iommus, so that - * framebuffers are mapped for it as for the panel. - */ -bool dcp_t6030_ext_native(const struct device_node *np) + +/* --------------------------------------------------------------------------- + * H17-generation firmware: T8140 (H17P, internal and the external dispext0) + * and T8142 (H17G). aurora-wip's processor lifecycle. + * ------------------------------------------------------------------------- */ + +struct dcp_session_ref { + struct list_head link; + struct device *dev; + struct iommu_group *group; +}; + +static LIST_HEAD(dcp_retained_sessions); +static DEFINE_MUTEX(dcp_retained_sessions_lock); + +static bool dcp_session_retained(struct device *dev) { - struct of_phandle_args dart, args; - struct device_node *piodma, *display; - bool native = false; - u32 marker; - int i, n; + struct dcp_session_ref *session; + bool retained = false; - if (!of_device_is_compatible(np, "apple,t6030-dcpext")) - return false; - piodma = of_get_child_by_name(np, "piodma"); - if (!piodma || !of_device_is_available(piodma) || - of_property_read_u32(piodma, "apple,t6030-dispext-handoff", &marker) || - marker != 1 || - of_parse_phandle_with_args(piodma, "iommus", "#iommu-cells", 0, &dart)) { - of_node_put(piodma); - return false; - } - of_node_put(piodma); - if (dart.args_count != 1 || dart.args[0] != 4 || !of_device_is_available(dart.np)) - goto out; - display = of_find_compatible_node(NULL, NULL, "apple,t6030-display-subsystem"); - n = display && of_device_is_available(display) ? - of_count_phandle_with_args(display, "iommus", "#iommu-cells") : 0; - for (i = 0; i < n && !native; i++) { - if (of_parse_phandle_with_args(display, "iommus", "#iommu-cells", i, &args)) + mutex_lock(&dcp_retained_sessions_lock); + list_for_each_entry(session, &dcp_retained_sessions, link) { + if (dev_fwnode(session->dev) == dev_fwnode(dev)) { + retained = true; break; - native = args.np == dart.np && args.args_count == 1 && !args.args[0]; - of_node_put(args.np); + } } - of_node_put(display); -out: - of_node_put(dart.np); - return native; + mutex_unlock(&dcp_retained_sessions_lock); + return retained; } -/* - * A native external processor is running, or has announced a DPTX port: - * connect the display that is already waiting for it. - */ -void dcp_external_ready(struct apple_dcp *dcp) +static int dcp_pin_live_session(struct apple_dcp *dcp) { - if (READ_ONCE(dcp->typec_cable_connected)) { - dcp->typec_reconnect_tries = 0; - dcp_queue_typec_reconnect(dcp, 0); - } else if (dcp->hdmi_hpd && dcp->active && - gpiod_get_value_cansleep(dcp->hdmi_hpd)) { - int ret = dcp_dptx_connect(dcp, 0); + struct dcp_session_ref *session; - if (ret && ret != -EAGAIN && ret != -ESTALE && ret != -ESHUTDOWN) - dcp_external_retry(dcp, "HDMI display link not set up", ret, 1000); - } + if ((!dcp->hw.adopt_live_session && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL) || + dcp->fw_compat != DCP_FIRMWARE_H17P || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G || dcp->retain_dma) + return 0; + session = kzalloc_obj(*session); + if (!session) + return -ENOMEM; + + /* Keep the DMA device, domain usage and driver code valid until reset. */ + session->dev = get_device(dcp->dev); + session->group = iommu_group_get(dcp->dev); + __module_get(THIS_MODULE); + WRITE_ONCE(dcp->retain_dma, true); + mutex_lock(&dcp_retained_sessions_lock); + list_add_tail(&session->link, &dcp_retained_sessions); + mutex_unlock(&dcp_retained_sessions_lock); + dev_info(dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); + return 0; } -#define DCP_EXTERNAL_RETRIES 3 +static void dcp_release_dma_domain(void *data) +{ + struct apple_dcp *dcp = data; -static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port); + if (!dcp->retain_dma) + iommu_device_unuse_default_domain(dcp->dev); +} -/* - * Bounded recovery for a native external pipe: re-apply the display mode, - * or redo the display link, up to DCP_EXTERNAL_RETRIES times per attached - * display, @base_ms, then twice and four times as long apart. A stopped - * firmware session cannot be recovered, and is not retried. - */ -void dcp_external_retry(struct apple_dcp *dcp, const char *why, int error, - unsigned int base_ms) +static void dcp_release_context(void *data) { - unsigned int n, delay; + struct apple_dcp *dcp = data; - if (!dcp->external_native) - return; - if (iomfb_v14_7_external_failed(dcp)) { - dev_err(dcp->dev, "%s (%d): the external display processor stopped; no retry until reboot\n", - why, error); - return; - } - if (delayed_work_pending(&dcp->external_retry_wq)) { - dev_info(dcp->dev, "%s (%d): a retry is already queued\n", why, error); - return; - } - n = atomic_inc_return(&dcp->external_retries); - if (n > DCP_EXTERNAL_RETRIES) { - atomic_set(&dcp->external_retries, DCP_EXTERNAL_RETRIES); - dev_warn(dcp->dev, "%s (%d): no retry left after %u; replug the display\n", - why, error, DCP_EXTERNAL_RETRIES); - return; + if (!dcp->retain_dma) + kfree(dcp); +} + +static void dcp_release_rtkit(void *data) +{ + struct apple_dcp *dcp = data; + + if (dcp->retain_dma) { + WRITE_ONCE(dcp->quiescing, true); + apple_rtkit_free_retaining_buffers(dcp->rtk); + } else { + apple_rtkit_free(dcp->rtk); } - delay = base_ms << (n - 1); - dev_info(dcp->dev, "%s (%d): retry %u of %u in %u ms\n", why, error, n, - DCP_EXTERNAL_RETRIES, delay); - /* An attach or detach after this makes it stale; see the work. */ - WRITE_ONCE(dcp->external_retry_generation, READ_ONCE(dcp->typec_generation)); - mod_delayed_work(system_freezable_wq, &dcp->external_retry_wq, msecs_to_jiffies(delay)); + dcp->rtk = NULL; +} + +bool dcp_h17p_external_supported(struct device_node *np) +{ + u32 firmware[DCP_EXTERNAL_FW_CELLS], compat[DCP_EXTERNAL_FW_CELLS]; + int firmware_cells, compat_cells; + + if (!of_machine_is_compatible("apple,j700") || + !of_device_is_compatible(np, "apple,t8140-dcpext")) + return false; + firmware_cells = of_property_read_variable_u32_array( + np, "apple,firmware-version", firmware, + DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); + compat_cells = of_property_read_variable_u32_array( + np, "apple,firmware-compat", compat, + DCP_EXTERNAL_FW_CELLS, DCP_EXTERNAL_FW_CELLS); + return apple_dcp_external_firmware_supported(firmware, firmware_cells, + compat, compat_cells); +} + +bool dcp_h17p_external_ready(struct device_node *np) +{ + struct platform_device *pdev; + struct apple_dcp *dcp; + bool ready = false; + + if (!dcp_h17p_external_supported(np)) + return false; + pdev = of_find_device_by_node(np); + if (!pdev) + return false; + /* + * Called after component binding. component_add() may synchronously + * bind the master before the platform probe itself returns, so the + * driver core's bound flag is not a readiness condition here. + */ + dcp = platform_get_drvdata(pdev); + ready = dcp && READ_ONCE(dcp->external_prepared) && + !READ_ONCE(dcp->quiescing) && !READ_ONCE(dcp->crashed); + put_device(&pdev->dev); + return ready; +} + +static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) +{ + return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; } -/* The active CRTC's mode is the one the firmware runs and shows swaps in. */ -static bool dcp_external_crtc_showing(struct apple_dcp *dcp) +void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected) { - struct drm_crtc *crtc = dcp->crtc ? &dcp->crtc->base : NULL; - bool showing = false; + unsigned long flags; - if (!crtc) - return false; - drm_modeset_lock(&crtc->mutex, NULL); - if (crtc->state && crtc->state->active) - showing = iomfb_v14_7_external_showing(dcp, &crtc->state->mode); - drm_modeset_unlock(&crtc->mutex); - return showing; + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (generation != dcp->metadata.epoch.generation || !connected) { + dcp->rate_proof.connection_sequence++; + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; + dcp->metadata.hpd_seen = false; + } + dcp_metadata_epoch_connection(&dcp->metadata.epoch, generation, connected); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); } -void dcp_external_retry_work(struct work_struct *work) +void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected) { - struct apple_dcp *dcp = container_of(to_delayed_work(work), struct apple_dcp, - external_retry_wq); - struct apple_connector *connector = READ_ONCE(dcp->connector); - u64 generation = READ_ONCE(dcp->external_retry_generation); - bool released; + struct apple_connector *con = READ_ONCE(dcp->connector); + unsigned long flags; - if (READ_ONCE(dcp->typec_generation) != generation) { - dev_info(dcp->dev, "display retry: the display was attached or detached since; dropped\n"); - return; - } - /* Kept connected through a withdrawal, and not described again. */ - released = iomfb_v14_7_external_release(dcp); - if (iomfb_v14_7_external_failed(dcp)) { - dev_err(dcp->dev, "display retry: the external display processor stopped; no retry until reboot\n"); - return; - } - /* Described again: re-apply the mode if it is not set. */ - if (!released && connector && READ_ONCE(connector->connected)) { - if (READ_ONCE(dcp->mode_state.valid)) { - dev_info(dcp->dev, "display retry: the display is back and set\n"); - return; - } - if (dcp_external_crtc_showing(dcp)) { - dev_info(dcp->dev, "display retry: the display is back in its mode and showing\n"); - dcp_mode_set_valid(&dcp->mode_state, true); - return; - } - dev_info(dcp->dev, "display retry: setting the display mode again\n"); - dcp_mode_invalidate(&dcp->mode_state); - dcp_queue_hotplug(connector); - return; - } - /* An HDMI output connects through the fabric, as on its HPD. */ - if (!dcp_is_typec_output(dcp) && dcp->hdmi_hpd) { - dcp_fabric_hdmi_retry(dcp); - return; - } - if (!READ_ONCE(dcp->typec_cable_connected)) { - dev_info(dcp->dev, "display retry: the port has no display any more\n"); + if (!dcp_firmware_metadata_enabled(dcp)) return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (connected) { + dcp->metadata.hpd_seen = true; + /* + * Fresh proof may precede HPD assertion after a same-cable loss. + * Keep invalidated firmware metadata and old receipt tickets revoked. + */ + if (READ_ONCE(dcp->typec_cable_connected) && + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation)) + dcp->metadata.epoch.connected = true; + } else if (dcp->metadata.hpd_seen) { + dcp->metadata.hpd_seen = false; + dcp->rate_proof.connection_sequence++; + dcp->rate_proof.sequence++; + dcp->rate_proof.rate.valid = false; + dcp->rate_proof.lanes.valid = false; + if (con && READ_ONCE(con->dcp) == to_platform_device(dcp->dev)) + WRITE_ONCE(con->edid_invalidation_pending, true); + dcp_metadata_epoch_connection(&dcp->metadata.epoch, + dcp->metadata.epoch.generation, false); + dcp->metadata.attributes_valid = false; + dcp->metadata.timing_valid = false; } - /* - * The port holds a display that the firmware no longer describes: - * release the link if it is still up, and connect it again, which - * makes the firmware describe the display anew. - */ - scoped_guard(mutex, &dcp->hpd_mutex) { - /* Not under a connect or release of another display. */ - if (dcp->typec_generation != generation || !dcp->typec_cable_connected) { - dev_info(dcp->dev, "display retry: the display was attached or detached since; dropped\n"); - return; - } - dev_info(dcp->dev, "display retry: reconnecting the display link\n"); - if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { - int ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - - if (ret) { - dev_warn(dcp->dev, "display retry: HPD deassert failed: %d\n", ret); - return; - } - ret = dcp_dptx_release_locked(dcp, 0); - if (ret) - return; - } - dcp->typec_reconnect_tries = 0; - } - dcp_queue_typec_reconnect(dcp, 0); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (connected) + dcp_180hz_notify(dcp); } -/* - * The sink of a native external pipe raised IRQ_HPD with HPD staying high, - * as a DP branch device does when the display behind it changes. Pass the - * request to the firmware, which services it as a DP source does. If it - * cannot take the request, connect the display link anew, which makes the - * firmware read the display again. - */ -void dcp_external_sink_irq(struct apple_dcp *dcp) +void dcp_firmware_metadata_begin(struct apple_dcp *dcp) { - struct platform_device *pdev = to_platform_device(dcp->dev); - struct apple_epic_service *service = READ_ONCE(dcp->dptxport[0].service); - int ret; + unsigned long flags; - /* With no link up, the next connect reads the display anyway. */ - if (!dcp->external_native || !service || !READ_ONCE(dcp->dptxport[0].enabled) || - !READ_ONCE(dcp->dptxport[0].connected)) - return; - ret = dptxport_sink_irq(service); - if (!ret) { - dev_info(dcp->dev, "display sink IRQ_HPD passed to the firmware\n"); - return; - } - dev_info(dcp->dev, "display sink IRQ_HPD not passed (%d): connecting the display anew\n", - ret); - ret = dcp_dptx_disconnect_oob(pdev, 0); - if (ret) - return; - dcp_dptx_connect_oob(pdev, 0); + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->chunks.metadata_ticket = dcp_metadata_begin(&dcp->metadata.epoch); + dcp->chunks.received = 0; + dcp->chunks.contiguous = true; + dcp->chunks.payload_complete = false; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); } -/* - * tiled_split: DPTX port 1 carries the second half of a tiled display on - * this pipeline. Bring it back whenever port 0 (re)connects, as DCP pairs - * the tiles only while both ports are connected. - */ -static int dcp_dptx_connect_tile(struct apple_dcp *dcp) +static void dcp_firmware_metadata_notify(struct apple_dcp *dcp) { - bool split; - int ret; + unsigned long flags; + bool ready; - scoped_guard(mutex, &dcp->tb_lock) - split = dcp->split.active && dcp->split.ready && - dcp->split.generation; - if (!split) - return 0; - ret = dcp_dptx_connect(dcp, 1); - if (ret) - dev_warn(dcp->dev, "tiled: DPTX port 1 reconnect failed: %d\n", ret); - return ret; + spin_lock_irqsave(&dcp->metadata.lock, flags); + ready = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (ready && dcp->connector && READ_ONCE(dcp->connector->connected)) + schedule_work(&dcp->connector->hotplug_wq); } -int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) +static bool dcp_firmware_metadata_complete(struct apple_dcp *dcp) { - struct dcp_fabric_session session; - int ret; + return dcp->chunks.contiguous && dcp->chunks.payload_complete && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket); +} - scoped_guard(mutex, &dcp->hpd_mutex) - session = dcp_session_locked(dcp); - ret = dcp_dptx_connect_session(dcp, port, session, !!session.cookie, false); - if (!ret && !port) - ret = dcp_dptx_connect_tile(dcp); - return ret; +void dcp_firmware_metadata_attributes(struct apple_dcp *dcp, + const struct dcp_display_metadata *attributes) +{ + unsigned long flags; + + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) { + dcp->metadata.attributes_valid = attributes && dcp_firmware_metadata_complete(dcp); + if (dcp->metadata.attributes_valid) { + dcp->metadata.attributes = *attributes; + dcp->metadata.attributes_ticket = dcp->chunks.metadata_ticket; + } + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); } -static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) +void dcp_firmware_metadata_timing(struct apple_dcp *dcp) { - const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); - static const u8 name[] = "Non-PnP"; + unsigned long flags; unsigned int i; - if (!raw) - return false; - - /* EDID contains interior NUL bytes, so this cannot use strnstr(). */ - for (i = 0; i + sizeof(name) - 1 <= sizeof(struct edid); i++) { - if (!memcmp(raw + i, name, sizeof(name) - 1)) - return true; - timeout = dcp_is_usb4_output(dcp) && apple_dp_tunnel_t602x() ? - DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; - ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, - timeout); - if (!ret) { - dev_err(dcp->dev, - "dcp_dptx_connect: timed out waiting for port %u link configuration\n", - port); - ret = -ETIMEDOUT; - goto out_disconnect; + if (!dcp_firmware_metadata_enabled(dcp)) + return; + spin_lock_irqsave(&dcp->metadata.lock, flags); + if (!dcp_metadata_matches(&dcp->metadata.epoch, dcp->chunks.metadata_ticket)) + goto unlock; + dcp->metadata.timing_valid = false; + if (!dcp_firmware_metadata_complete(dcp)) + goto unlock; + for (i = 0; i < dcp->nr_modes; i++) { + if (!(dcp->modes[i].mode.type & DRM_MODE_TYPE_PREFERRED) || + !dcp->modes[i].metadata_timing_valid) + continue; + dcp->metadata.preferred = dcp->modes[i].mode; + dcp->metadata.timing_ticket = dcp->chunks.metadata_ticket; + dcp->metadata.timing_valid = true; + break; } +unlock: + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_firmware_metadata_notify(dcp); +} + +bool dcp_firmware_metadata_current(struct apple_dcp *dcp, u64 generation) +{ + unsigned long flags; + bool valid; - return false; + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + return valid; } - dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", - jiffies_to_msecs(timeout - ret)); -void dcp_retry_placeholder_edid(struct apple_dcp *dcp, - const struct drm_edid *drm_edid) +const struct drm_edid *dcp_firmware_edid(struct apple_dcp *dcp, u64 *generation) { - guard(mutex)(&dcp->hpd_mutex); - if (!dcp_is_typec_output(dcp) || !dcp->typec_cable_connected || - dcp->placeholder_retried) - return; - if (!dcp_edid_is_placeholder(drm_edid)) - return; + struct dcp_display_metadata attributes; + struct drm_display_mode preferred; + const struct drm_edid *edid; + unsigned long flags; + bool valid; - dcp->placeholder_retried = true; - dcp->placeholder_generation = dcp->typec_generation; - schedule_delayed_work(&dcp->placeholder_edid_wq, msecs_to_jiffies(300)); + if (!dcp_firmware_metadata_enabled(dcp)) + return ERR_PTR(-ENODEV); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = dcp->metadata.attributes_valid && dcp->metadata.timing_valid && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.attributes_ticket) && + dcp_metadata_matches(&dcp->metadata.epoch, dcp->metadata.timing_ticket); + if (valid) { + attributes = dcp->metadata.attributes; + preferred = dcp->metadata.preferred; + *generation = dcp->metadata.epoch.generation; + } + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid) + return ERR_PTR(-ENODATA); + edid = dcp_metadata_edid(&attributes, &preferred); + spin_lock_irqsave(&dcp->metadata.lock, flags); + valid = !dcp->metadata.epoch.invalidated && dcp->metadata.epoch.connected && + *generation == dcp->metadata.epoch.generation; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (!valid && !IS_ERR(edid)) { + drm_edid_free(edid); + return ERR_PTR(-ESTALE); + } + return edid; } -static void dcp_placeholder_edid_work(struct work_struct *work) +/* + * H17P: deferred to the connector hotplug work, which updates the connector + * with dcp_update_connector_dimensions(). + */ +void dcp_set_dimensions_h17p(struct apple_dcp *dcp) { - struct apple_dcp *dcp = - container_of(to_delayed_work(work), struct apple_dcp, - placeholder_edid_wq); - struct apple_epic_service *service; - u64 generation; - int ret; + int i; + int width_mm = dcp->width_mm; + int height_mm = dcp->height_mm; - mutex_lock(&dcp->hpd_mutex); - generation = dcp->placeholder_generation; - if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || - !dcp->dptxport[0].enabled || generation != dcp->typec_generation) - goto out_unlock; - service = dcp->dptxport[0].service; + if (width_mm == 0 || height_mm == 0) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; + } /* - * Some adapters answer the first connection with a 1024x768 - * placeholder and publish the panel EDID only after HPD drops - * and returns. One pulse; a second placeholder is left alone. + * This callback runs on the ordered RTKit RX worker. A mode probe + * can hold mode_config.mutex while waiting for an AFK reply which + * that same worker still has to dispatch. Defer the DRM update. */ - ret = dptxport_set_hpd(service, false); - if (ret) { - dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", - ret); - goto out_unlock; - } - mutex_unlock(&dcp->hpd_mutex); - - msleep(1000); - - mutex_lock(&dcp->hpd_mutex); - if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || - generation != dcp->typec_generation) - goto out_unlock; + if (dcp->connector && !READ_ONCE(dcp->quiescing)) + schedule_work(&dcp->connector->hotplug_wq); - ret = dptxport_set_hpd(service, true); - if (ret) - dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", - ret); -out_unlock: - mutex_unlock(&dcp->hpd_mutex); + /* + * Fix up any probed modes. Modes are created when parsing + * TimingElements, dimensions are calculated when parsing + * DisplayAttributes, and TimingElements may be sent first + */ + for (i = 0; i < dcp->nr_modes; ++i) { + dcp->modes[i].mode.width_mm = width_mm; + dcp->modes[i].mode.height_mm = height_mm; + } } -static void dcp_external_report_disconnected(struct apple_dcp *dcp) +void dcp_update_connector_dimensions(struct apple_dcp *dcp, + struct drm_connector *connector) { - unsigned int action; + int width_mm = READ_ONCE(dcp->width_mm); + int height_mm = READ_ONCE(dcp->height_mm); - guard(mutex)(&dcp->iomfb.lock); - /* A paused callback from the old sink must not be replayed on wake. */ - dcp->external_hotplug_pending = false; - action = dcp_mode_hotplug(&dcp->mode_state, false, - dcp->connector ? &dcp->connector->connected : NULL); - if (dcp->connector) { - WRITE_ONCE(dcp->connector->edid_invalidation_pending, true); - action |= DCP_HOTPLUG_NOTIFY; + lockdep_assert_held(&connector->dev->mode_config.mutex); + if (!width_mm || !height_mm) { + width_mm = dcp->panel.width_mm; + height_mm = dcp->panel.height_mm; } - /* The existing worker delivers DRM notifications outside our locks. */ - dcp_handle_hotplug_actions(dcp, action); + connector->display_info.width_mm = width_mm; + connector->display_info.height_mm = height_mm; } -static void dcp_external_publish_generation(struct apple_dcp *dcp) + +static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) { - lockdep_assert_held(&dcp->external_lock); - guard(mutex)(&dcp->iomfb.lock); - dcp->external_rx_generation = dcp->external_generation; + /* DPAV 0x2a on H17P overlaps the legacy DPTX endpoint number. */ + if (dcp->dcpavservep && endpoint == dcp->dcpavservep->endpoint) + return dcp->dcpavservep; + if (dcp->dptxep && endpoint == dcp->dptxep->endpoint) + return dcp->dptxep; + switch (endpoint) { + case AV_ENDPOINT: + return dcp->avep; + case SYSTEM_ENDPOINT: + return dcp->systemep; + case DISP0_ENDPOINT: + return dcp->ibootep; + case DPTX_CONTROLLER_ENDPOINT: + return dcp->dptx_controller_ep; + default: + return NULL; + } } -/* external_lock is never taken by the serialized IOMFB receive callbacks. */ -static int dcp_external_link_locked(struct apple_dcp *dcp) + +static void dcp_recv_msg_h17p(struct apple_dcp *dcp, u8 endpoint, u64 message) { - bool waiting_service; - int ret; + struct apple_dcp_afkep *ep; - lockdep_assert_held(&dcp->external_lock); - if (!dcp->external_prepared) - return -EAGAIN; - if (dcp->external_error) - return dcp->external_error; - if (!dcp->external_cable) - return -ENODEV; - ret = dcp_dptx_connect_raw(dcp, 0, &waiting_service); - if (ret && !waiting_service) - dcp->external_error = ret; - return ret; -} + if (READ_ONCE(dcp->quiescing)) + return; -static int dcp_external_start_locked(struct apple_dcp *dcp) -{ - int ret; + trace_dcp_recv_msg(dcp, endpoint, message); - lockdep_assert_held(&dcp->external_lock); - ret = dcp_external_link_locked(dcp); - if (ret || dcp->external_running) - return ret; - iomfb_pause_presentation(dcp, true); - /* Even a timed-out first activation can have changed firmware state. */ - dcp->external_power_attempted = true; - ret = iomfb_external_power_h17p(dcp, true); - if (ret) { - dcp->external_error = ret; - return ret; + /* + * H17P starts endpoints that have no AFK instance behind them (the + * remote allocator endpoint, for one). Log their messages rather than + * dereferencing a NULL afkep or warning about an unknown endpoint. + */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P && endpoint != IOMFB_ENDPOINT && + !dcp_afkep(dcp, endpoint)) { + dev_dbg_ratelimited(dcp->dev, "ep %#04x: %#llx\n", endpoint, + message); + return; } - dcp->external_running = true; - dcp->external_stopped = false; - WRITE_ONCE(dcp->pipe_enabled_h17p, true); - WRITE_ONCE(dcp->typec_crtc_off, false); - iomfb_pause_presentation(dcp, false); - return 0; -} -static int dcp_dptx_connect(struct apple_dcp *dcp, u32 port) -{ - if (!dcp_is_external_h17p(dcp)) - return dcp_dptx_connect_raw(dcp, port, NULL); - guard(mutex)(&dcp->external_lock); - return dcp_external_link_locked(dcp); -} + if (endpoint == IOMFB_ENDPOINT) + return iomfb_recv_msg(dcp, message); -/* The generation token belongs to the CRTC that was actually enabled. */ -static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) -{ - int ret; + ep = dcp_afkep(dcp, endpoint); - lockdep_assert_held(&dcp->external_lock); - if (generation != dcp->external_generation) - return 0; - if (dcp->external_error) - return dcp->external_error; - if (dcp->external_stopped && !READ_ONCE(dcp->dptxport[0].connected)) - return 0; - iomfb_pause_presentation(dcp, true); - dcp->external_running = false; - /* Route setup can call this before IOMFB or its shared memory exists. */ - if (dcp->external_stopped || !dcp->external_power_attempted) - goto release_link; - ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); - if (!ret) - ret = iomfb_external_power_h17p(dcp, false); - if (!ret) - ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); - if (ret) - goto failed; - /* This is a transport acknowledgment, not permission to free scanout. */ - WRITE_ONCE(dcp->pipe_enabled_h17p, false); -release_link: - scoped_guard(mutex, &dcp->hpd_mutex) { - if (dcp->dptxport[0].connected) { - if (!dptx_service_available(&dcp->dptxport[0])) { - ret = -ENODEV; - goto failed; - } - ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - if (!ret) - ret = dptxport_release_display(dcp->dptxport[0].service); - if (ret) - goto failed; - dcp->dptxport[0].connected = false; - } + if (!ep) { + dev_warn_ratelimited(dcp->dev, + "dropping message for unhandled endpoint %#x\n", + endpoint); + return; } - dcp->external_stopped = true; - WRITE_ONCE(dcp->typec_crtc_off, true); - if (dcp->avep) - av_service_disconnect(dcp); - return 0; -failed: - /* A retry could race a late reply. Keep this session blocked and owned. */ - dcp->external_error = ret; - return ret; + + afk_receive_message(ep, message); } -static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) +static void dcp_rtk_crashed_h17p(struct apple_dcp *dcp) { - int ret; + if (READ_ONCE(dcp->quiescing)) + return; - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - guard(mutex)(&dcp->external_lock); - /* Invalidate physical EDID before waiting for firmware power/link teardown. */ - if (physical) { - WRITE_ONCE(dcp->external_physically_absent, true); - scoped_guard(mutex, &dcp->hpd_mutex) { - WRITE_ONCE(dcp->typec_cable_connected, false); - dcp->typec_generation++; - dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); - } - /* HPD is fenced; retire the AV owner after releasing hpd_mutex. */ - dpavservep_detach(dcp); + dcp->crashed = true; + dev_err(dcp->dev, "DCP has crashed\n"); + if (dcp->fw_compat == DCP_FIRMWARE_H17P && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* No in-place restart exists; the last scanout stays retained. */ + dev_err(dcp->dev, "a reboot is required to restore the display\n"); + /* + * Nothing will complete a present that was in flight. Before + * the pipeline is linked there is no event to complete. + */ + if (READ_ONCE(dcp->crtc)) + schedule_work(&dcp->vblank_wq); } - ret = dcp_external_stop_locked(dcp, dcp->external_generation); - if (physical) { - dcp->external_cable = false; - dcp->external_generation++; - dcp_external_publish_generation(dcp); - /* A missing cable is reported even when its hardware remains owned. */ - dcp_external_report_disconnected(dcp); + if (dcp->connector) { + dcp->connector->connected = 0; + /* The cached EDID is owned under edid_lock. */ + apple_connector_invalidate_edid(dcp->connector); + schedule_work(&dcp->connector->hotplug_wq); } - return ret; + complete(&dcp->start_done); } -static void dcp_external_stop_work(struct work_struct *work) +/* + * Check that every IOMMU page of [iova, iova + size) translates to the same + * offset from @phys as it has from @iova. + */ +static int dcp_check_iova_contiguous(struct iommu_domain *domain, + dma_addr_t iova, size_t size, + phys_addr_t phys) { - struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_stop_work); - - u64 generation; + unsigned long pgsize = domain->pgsize_bitmap ? + 1UL << __ffs(domain->pgsize_bitmap) : PAGE_SIZE; + dma_addr_t end = iova + size; + dma_addr_t addr = iova; - scoped_guard(mutex, &dcp->iomfb.lock) - generation = dcp->external_stop_generation; - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - guard(mutex)(&dcp->external_lock); - if (generation == dcp->external_generation) { - /* The deferred disconnect also covers missing out-of-band HPD. */ - dpavservep_detach(dcp); - dcp_external_stop_locked(dcp, generation); - dcp_external_report_disconnected(dcp); + while (addr < end) { + if (iommu_iova_to_phys(domain, addr) != phys + (addr - iova)) + return -EINVAL; + addr = ALIGN_DOWN(addr, pgsize) + pgsize; } + + return 0; } -bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) +static int dcp_rtk_shmem_setup_h17p(struct apple_dcp *dcp, struct apple_rtkit_shmem *bfr) { - if (!dcp_is_external_h17p(dcp)) - return false; - /* No lifecycle or HPD mutex may be acquired from the RX callback. */ - lockdep_assert_held(&dcp->iomfb.lock); - if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->external_physically_absent)) - return true; - if (dcp->iomfb.presentation_paused) { - dcp->external_hotplug_pending = true; - dcp->external_hotplug_connected = connected; - /* Publish initial/new-cable discovery, never same-cable DPMS wake. */ - if (connected && READ_ONCE(dcp->external_cable) && - !READ_ONCE(dcp->external_error) && - (!READ_ONCE(dcp->external_power_attempted) || - (READ_ONCE(dcp->external_stopped) && - READ_ONCE(dcp->external_rx_generation) != - READ_ONCE(dcp->external_crtc_generation) && - READ_ONCE(dcp->dptxport[0].connected)))) { - unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, - dcp->connector ? - &dcp->connector->connected : NULL); + int ret; - dcp_handle_hotplug_actions(dcp, action); + if (bfr->iova) { + struct iommu_domain *domain = + iommu_get_domain_for_dev(dcp->dev); + phys_addr_t phy_addr; + + if (!domain) + return -ENOMEM; + + // TODO: get map from device-tree + phy_addr = iommu_iova_to_phys(domain, bfr->iova); + if (!phy_addr) + return -ENOMEM; + + /* memremap() below needs the whole buffer physically contiguous */ + ret = dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, + phy_addr); + if (ret) { + dev_err(dcp->dev, + "shmem_setup: iova %pad (%#zx bytes) is not physically contiguous\n", + &bfr->iova, bfr->size); + return ret; } - return true; - } - if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { - /* Fence admission in RX before a busy actor can delay teardown. */ - dcp_firmware_metadata_hotplug(dcp, false); - dcp->external_stop_generation = dcp->external_rx_generation; - schedule_work(&dcp->external_stop_work); - return true; - } - return READ_ONCE(dcp->iomfb.presentation_paused); -} -static bool dcp_edid_is_placeholder(const struct drm_edid *drm_edid) -{ - const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); - static const u8 name[] = "Non-PnP"; - unsigned int i; + /* Firmware writes without CPU cache maintenance. */ + bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WC); + if (!bfr->buffer) + return -ENOMEM; - if (!raw) - return false; + bfr->is_mapped = true; + dev_info(dcp->dev, + "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)phy_addr, + (uintptr_t)bfr->buffer); + } else { + bfr->buffer = dma_alloc_coherent(dcp->dev, bfr->size, + &bfr->iova, GFP_KERNEL); + if (!bfr->buffer) + return -ENOMEM; - /* EDID contains interior NUL bytes, so this cannot use strnstr(). */ - for (i = 0; i + sizeof(name) - 1 <= sizeof(struct edid); i++) { - if (!memcmp(raw + i, name, sizeof(name) - 1)) - return true; + dev_info(dcp->dev, "shmem_setup: iova: %lx, buffer: %lx\n", + (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); } - return false; + return 0; } -void dcp_retry_placeholder_edid(struct apple_dcp *dcp, - const struct drm_edid *drm_edid) +static const struct apple_rtkit_ops external_h17p_rtkit_ops = { + .flags = APPLE_RTKIT_COPROC_PLACES_BUFFERS, + .crashed = dcp_rtk_crashed, + .recv_message = dcp_recv_msg, + .shmem_setup = dcp_rtk_shmem_setup, + .shmem_destroy = dcp_rtk_shmem_destroy, +}; + +static int dcp_crtc_atomic_check_h17p(struct drm_crtc *crtc, struct drm_atomic_state *state) { - guard(mutex)(&dcp->hpd_mutex); - if (!dcp_is_typec_output(dcp) || !dcp->typec_cable_connected || - dcp->placeholder_retried) - return; - if (!dcp_edid_is_placeholder(drm_edid)) - return; + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct drm_crtc_state *crtc_state; + bool needs_modeset; - dcp->placeholder_retried = true; - dcp->placeholder_generation = dcp->typec_generation; - schedule_delayed_work(&dcp->placeholder_edid_wq, msecs_to_jiffies(300)); + if (dcp->crashed) + return -EINVAL; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + /* External H17P does not program adaptive sync. */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL && + crtc_state->vrr_enabled) + return -EINVAL; + + needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || + !READ_ONCE(dcp->mode_state.valid); + if (!needs_modeset && (!dcp->connector || !dcp->connector->connected)) { + /* + * Resume restores the mode before the firmware reports the + * display back, so a plane-only commit lands here while the + * connector is still marked disconnected. Rejecting it makes + * the compositor fail every flip and give up on the output; + * dcp_flush() defers the commit until the link returns. + */ + dev_dbg(dcp->dev, + "crtc_atomic_check: deferring commit, link still down\n"); + } + + return 0; } -static void dcp_placeholder_edid_work(struct work_struct *work) +static int dcp_dptx_connect_raw(struct apple_dcp *dcp, u32 port, bool *waiting_service) { - struct apple_dcp *dcp = - container_of(to_delayed_work(work), struct apple_dcp, - placeholder_edid_wq); - struct apple_epic_service *service; - u64 generation; - int ret; + unsigned long timeout; + int ret = 0; + + if (waiting_service) + *waiting_service = false; + if (!dcp->phy) { + dev_warn(dcp->dev, "dcp_dptx_connect: missing phy\n"); + return -ENODEV; + } + dev_dbg(dcp->dev, + "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", + __func__, port, dcp->dptx_die, dcp->dptx_phy, + dcp_is_typec_output(dcp), + dcp->active_typec_route ? "borrowed" : "fixed", + dcp->connector_type, dcp->dptxport[port].connected); mutex_lock(&dcp->hpd_mutex); - generation = dcp->placeholder_generation; - if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || - !dptx_service_available(&dcp->dptxport[0]) || generation != dcp->typec_generation) + if (!dptx_service_available(&dcp->dptxport[port])) { + dev_warn(dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); + ret = -ENODEV; + /* No firmware request has been made; the first publication can retry. */ + if (waiting_service && !READ_ONCE(dcp->dptxport[port].closed)) { + *waiting_service = true; + ret = -EAGAIN; + } + goto out_unlock; + } + + if (dcp->dptxport[port].connected) + goto out_unlock; + + reinit_completion(&dcp->dptxport[port].linkcfg_completion); + dcp->dptxport[port].atcphy = dcp->phy; + ret = dptxport_validate_connection(dcp->dptxport[port].service, + dcp->dptx_dfp_port, + dcp->dptx_phy, dcp->dptx_die); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); goto out_unlock; - service = dcp->dptxport[0].service; + } - /* - * Some adapters answer the first connection with a 1024x768 - * placeholder and publish the panel EDID only after HPD drops - * and returns. One pulse; a second placeholder is left alone. - */ - ret = dptxport_set_hpd(service, false); + ret = dptxport_connect(dcp->dptxport[port].service, + dcp->dptx_dfp_port, + dcp->dptx_phy, dcp->dptx_die, + dcp_is_typec_output(dcp)); if (ret) { - dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", - ret); + dev_err(dcp->dev, + "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", + dcp->dptx_die, dcp->dptx_phy, ret); goto out_unlock; } + + ret = dptxport_request_display(dcp->dptxport[port].service); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to request display: %d\n", + ret); + goto out_release; + } + dcp->dptxport[port].connected = true; + /* Not a T602x USB4 tunnel: the plain HPD assertion and timeout apply. */ + if (dcp_is_typec_output(dcp)) { + ret = dptxport_set_hpd(dcp->dptxport[port].service, true); + if (ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: failed to assert Type-C HPD: %d\n", + ret); + dcp->dptxport[port].connected = false; + goto out_release; + } + } + mutex_unlock(&dcp->hpd_mutex); + timeout = DPTX_CONNECT_TIMEOUT; + ret = wait_for_completion_timeout(&dcp->dptxport[port].linkcfg_completion, + timeout); + if (!ret) { + dev_err(dcp->dev, + "dcp_dptx_connect: timed out waiting for port %u link configuration\n", + port); + ret = -ETIMEDOUT; + goto out_disconnect; + } - msleep(1000); + dev_dbg(dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", + jiffies_to_msecs(timeout - ret)); + + usleep_range(5, 10); + + if (dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort) + dptxport_set_hpd(dcp->dptxport[port].service, true); + + if (dcp->avep) + av_service_connect(dcp); + return 0; + +out_disconnect: mutex_lock(&dcp->hpd_mutex); - if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || - generation != dcp->typec_generation) - goto out_unlock; + dcp->dptxport[port].connected = false; +out_release: + dptxport_release_display(dcp->dptxport[port].service); - ret = dptxport_set_hpd(service, true); - if (ret) - dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", - ret); out_unlock: mutex_unlock(&dcp->hpd_mutex); + return ret; } -static void dcp_typec_reconnect_work(struct work_struct *work) +static void dcp_external_report_disconnected(struct apple_dcp *dcp) { - struct apple_dcp *dcp = - container_of(to_delayed_work(work), struct apple_dcp, - typec_reconnect_wq); - struct dcp_fabric_session session; - int ret; - - scoped_guard(mutex, &dcp->hpd_mutex) { - session = dcp->typec_reconnect_session; - if ((dcp->external && dcpext_scanout_terminal(dcp)) || - !dcp_session_valid_locked(dcp, session, true)) - return; - } - ret = dcp_dptx_connect_session(dcp, 0, session, true, false); - if (!ret) - ret = dcp_dptx_connect_tile(dcp); - guard(mutex)(&dcp->hpd_mutex); - if (!dcp_session_valid_locked(dcp, session, true)) - return; - if (!ret) { - dcp->typec_reconnect_tries = 0; - return; - } - if (++dcp->typec_reconnect_tries < - (dcp_uses_t6020_tunnel_flow(dcp) ? 1 : DPTX_RECONNECT_RETRIES)) { + unsigned int action; - if (++dcp->typec_reconnect_tries < - (dcp_is_usb4_output(dcp) && of_machine_is_compatible("apple,j416s") ? - 1 : DPTX_RECONNECT_RETRIES)) { - mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - return; + guard(mutex)(&dcp->iomfb.lock); + /* A paused callback from the old sink must not be replayed on wake. */ + dcp->external_hotplug_pending = false; + action = dcp_mode_hotplug(&dcp->mode_state, false, + dcp->connector ? &dcp->connector->connected : NULL); + if (dcp->connector) { + WRITE_ONCE(dcp->connector->edid_invalidation_pending, true); + action |= DCP_HOTPLUG_NOTIFY; } - dev_err(dcp->dev, "Type-C DPTX reconnect failed after %u retries: %d\n", - dcp->typec_reconnect_tries, ret); + /* The existing worker delivers DRM notifications outside our locks. */ + dcp_handle_hotplug_actions(dcp, action); } -static void disconnected_hpd_event(struct apple_connector *con) +static void dcp_external_publish_generation(struct apple_dcp *dcp) { - if (!con) - return; - - /* May run with DRM modeset locks held; defer property/cache cleanup. */ - WRITE_ONCE(con->connected, false); - WRITE_ONCE(con->edid_invalidation_pending, true); - schedule_work(&con->hotplug_wq); - apple_connector_edid_set_live(con, false); - if (con && con->connected) { - struct platform_device *pdev = READ_ONCE(con->dcp); + lockdep_assert_held(&dcp->external_lock); + guard(mutex)(&dcp->iomfb.lock); + dcp->external_rx_generation = dcp->external_generation; +} - if (pdev) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); +/* external_lock is never taken by the serialized IOMFB receive callbacks. */ +static int dcp_external_link_locked(struct apple_dcp *dcp) +{ + bool waiting_service; + int ret; - WRITE_ONCE(dcp->ext_backlight, false); - } - con->connected = 0; - drm_kms_helper_connector_hotplug_event(&con->base); - /* - * Drop the display's backlight, outside the caller's locks. Not - * the hotplug work, which would send a second hotplug event. - */ - schedule_work(&con->bl_sync_wq); - } + lockdep_assert_held(&dcp->external_lock); + if (!dcp->external_prepared) + return -EAGAIN; + if (dcp->external_error) + return dcp->external_error; + if (!dcp->external_cable) + return -ENODEV; + ret = dcp_dptx_connect_raw(dcp, 0, &waiting_service); + if (ret && !waiting_service) + dcp->external_error = ret; + return ret; } -static int dcp_dptx_release_locked(struct apple_dcp *dcp, u32 port) +static int dcp_external_start_locked(struct apple_dcp *dcp) { int ret; - if (dcp_is_external_h17p(dcp)) - return dcp_external_disconnect(dcp, false); - - dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); - lockdep_assert_held(&dcp->hpd_mutex); - if (dcp->external) { - smp_store_release(&dcp->external_link_ready, false); - dcpext_scanout_invalidate(dcp); - } - if (dcp->dptxport[port].enabled && dcp->dptxport[port].connected) { - ret = dptxport_release_display(dcp->dptxport[port].service); - if (ret) - return ret; - mutex_lock(&dcp->hpd_mutex); - if (dptx_service_available(&dcp->dptxport[port]) && dcp->dptxport[port].connected) { - dptxport_release_display(dcp->dptxport[port].service); - dcp->dptxport[port].connected = false; + lockdep_assert_held(&dcp->external_lock); + ret = dcp_external_link_locked(dcp); + if (ret || dcp->external_running) + return ret; + iomfb_pause_presentation(dcp, true); + /* Even a timed-out first activation can have changed firmware state. */ + dcp->external_power_attempted = true; + ret = iomfb_external_power_h17p(dcp, true); + if (ret) { + dcp->external_error = ret; + return ret; } - /* - * T6030: the firmware takes a direct crossbar output down when it - * stops the link. Make sure of it for a display released here, also - * when the firmware did not get that far. - */ - if (dcp->external_native) - dcp_direct_crossbar_link(dcp, false); + dcp->external_running = true; + dcp->external_stopped = false; + WRITE_ONCE(dcp->pipe_enabled_h17p, true); + WRITE_ONCE(dcp->typec_crtc_off, false); + iomfb_pause_presentation(dcp, false); return 0; } -int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) +static int dcp_dptx_connect_h17p(struct apple_dcp *dcp, u32 port) { - int ret; + if (!dcp_is_external_h17p(dcp)) + return dcp_dptx_connect_raw(dcp, port, NULL); + guard(mutex)(&dcp->external_lock); + return dcp_external_link_locked(dcp); +} - /* Release the caller's RemotePort service, not the downstream DFP port. */ - dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); +/* The generation token belongs to the CRTC that was actually enabled. */ +static int dcp_external_stop_locked(struct apple_dcp *dcp, u64 generation) +{ + int ret; - mutex_lock(&dcp->hpd_mutex); - ret = dcp_dptx_release_locked(dcp, port); + lockdep_assert_held(&dcp->external_lock); + if (generation != dcp->external_generation) + return 0; + if (dcp->external_error) + return dcp->external_error; + if (dcp->external_stopped && !READ_ONCE(dcp->dptxport[0].connected)) + return 0; + iomfb_pause_presentation(dcp, true); + dcp->external_running = false; + /* Route setup can call this before IOMFB or its shared memory exists. */ + if (dcp->external_stopped || !dcp->external_power_attempted) + goto release_link; + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); if (!ret) - WRITE_ONCE(dcp->dptxport[port].tile_hint, false); - mutex_unlock(&dcp->hpd_mutex); - + ret = iomfb_external_power_h17p(dcp, false); + if (!ret) + ret = iomfb_queue_settle(dcp, msecs_to_jiffies(500)); + if (ret) + goto failed; + /* This is a transport acknowledgment, not permission to free scanout. */ + WRITE_ONCE(dcp->pipe_enabled_h17p, false); +release_link: + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->dptxport[0].connected) { + if (!dptx_service_available(&dcp->dptxport[0])) { + ret = -ENODEV; + goto failed; + } + ret = dptxport_set_hpd(dcp->dptxport[0].service, false); + if (!ret) + ret = dptxport_release_display(dcp->dptxport[0].service); + if (ret) + goto failed; + dcp->dptxport[0].connected = false; + } + } + dcp->external_stopped = true; + WRITE_ONCE(dcp->typec_crtc_off, true); + if (dcp->avep) + av_service_disconnect(dcp); + return 0; +failed: + /* A retry could race a late reply. Keep this session blocked and owned. */ + dcp->external_error = ret; return ret; } -int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) +static int dcp_external_disconnect(struct apple_dcp *dcp, bool physical) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; - if (dcp_is_external_h17p(dcp)) { - bool new_cable; - - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_work_sync(&dcp->external_stop_work); - guard(mutex)(&dcp->external_lock); - new_cable = !dcp->external_cable; - if (new_cable) - dcp->external_generation++; - dcp->external_cable = true; - WRITE_ONCE(dcp->external_physically_absent, false); - dcp_external_publish_generation(dcp); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + /* Invalidate physical EDID before waiting for firmware power/link teardown. */ + if (physical) { + WRITE_ONCE(dcp->external_physically_absent, true); scoped_guard(mutex, &dcp->hpd_mutex) { - WRITE_ONCE(dcp->typec_cable_connected, true); - if (new_cable) { - dcp->typec_generation++; - dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); - } - dcp->placeholder_retried = false; + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); } - return dcp_external_link_locked(dcp); + /* HPD is fenced; retire the AV owner after releasing hpd_mutex. */ + dpavservep_detach(dcp); } - - if (dcp_is_typec_output(dcp)) { - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - guard(mutex)(&dcp->hpd_mutex); - dcp->typec_generation++; - WRITE_ONCE(dcp->typec_cable_connected, true); - dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); - dcp->typec_reconnect_tries = 0; - dcp->placeholder_retried = false; - cancel_delayed_work(&dcp->typec_reconnect_wq); - /* A newly attached display gets its own retries. */ - if (dcp->external_native) { - atomic_set(&dcp->external_retries, 0); - cancel_delayed_work(&dcp->external_retry_wq); - } + ret = dcp_external_stop_locked(dcp, dcp->external_generation); + if (physical) { + dcp->external_cable = false; + dcp->external_generation++; + dcp_external_publish_generation(dcp); + /* A missing cable is reported even when its hardware remains owned. */ + dcp_external_report_disconnected(dcp); } - - ret = dcp_dptx_connect(dcp, port); - if (ret && ret != -ESHUTDOWN && dcp_is_typec_output(dcp)) - dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); - return ret; } -int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) +static void dcp_external_stop_work(struct work_struct *work) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_stop_work); - if (dcp_is_external_h17p(dcp)) - return dcp_external_disconnect(dcp, true); + u64 generation; - if (dcp_is_typec_output(dcp)) { - scoped_guard(mutex, &dcp->hpd_mutex) { - WRITE_ONCE(dcp->typec_cable_connected, false); - dcp->typec_generation++; - dcp->typec_unplugs++; - /* Nothing left to retry for the display that went. */ - if (dcp->external_native) - cancel_delayed_work(&dcp->external_retry_wq); - dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); - } - WRITE_ONCE(dcp->typec_crtc_off, false); - reinit_completion(&dcp->typec_iomfb_hpd_ready); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + scoped_guard(mutex, &dcp->iomfb.lock) + generation = dcp->external_stop_generation; + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->external_lock); + if (generation == dcp->external_generation) { + /* The deferred disconnect also covers missing out-of-band HPD. */ + dpavservep_detach(dcp); + dcp_external_stop_locked(dcp, generation); + dcp_external_report_disconnected(dcp); } +} - return dcp_dptx_disconnect_drained(dcp, port); +bool dcp_external_hotplug(struct apple_dcp *dcp, bool connected) +{ + if (!dcp_is_external_h17p(dcp)) + return false; + /* No lifecycle or HPD mutex may be acquired from the RX callback. */ + lockdep_assert_held(&dcp->iomfb.lock); + if (READ_ONCE(dcp->quiescing) || READ_ONCE(dcp->external_physically_absent)) + return true; + if (dcp->iomfb.presentation_paused) { + dcp->external_hotplug_pending = true; + dcp->external_hotplug_connected = connected; + /* Publish initial/new-cable discovery, never same-cable DPMS wake. */ + if (connected && READ_ONCE(dcp->external_cable) && + !READ_ONCE(dcp->external_error) && + (!READ_ONCE(dcp->external_power_attempted) || + (READ_ONCE(dcp->external_stopped) && + READ_ONCE(dcp->external_rx_generation) != + READ_ONCE(dcp->external_crtc_generation) && + READ_ONCE(dcp->dptxport[0].connected)))) { + unsigned int action = dcp_mode_hotplug(&dcp->mode_state, true, + dcp->connector ? + &dcp->connector->connected : NULL); + + dcp_handle_hotplug_actions(dcp, action); + } + return true; + } + if (!connected && !READ_ONCE(dcp->iomfb.presentation_paused)) { + /* Fence admission in RX before a busy actor can delay teardown. */ + dcp_firmware_metadata_hotplug(dcp, false); + dcp->external_stop_generation = dcp->external_rx_generation; + schedule_work(&dcp->external_stop_work); + return true; + } + return READ_ONCE(dcp->iomfb.presentation_paused); } -/* A delivered sink IRQ can follow a firmware unplug with cable HPD still high. */ -int dcp_dptx_recover_irq(struct apple_dcp *dcp) + +static void dcp_placeholder_edid_work_h17p(struct work_struct *work) { - struct dcp_fabric_session session; + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + placeholder_edid_wq); + struct apple_epic_service *service; + u64 generation; int ret; mutex_lock(&dcp->hpd_mutex); - if (dcp->external_native || !dcp_is_typec_output(dcp) || - dcp_is_usb4_output(dcp) || !dcp->typec_connector || - READ_ONCE(dcp->typec_connector->connected) || - READ_ONCE(dcp->typec_crtc_off)) { - ret = 0; - goto out_unlock; - } - if (!dcp->typec_work_enabled || READ_ONCE(dcp->crashed) || - !dcp->dptxport[0].enabled) { - ret = -EAGAIN; - goto out_unlock; - } - session = dcp_session_locked(dcp); - if (!dcp_connect_session_valid_locked(dcp, session, true, true)) { - ret = -ESTALE; + generation = dcp->placeholder_generation; + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + !dptx_service_available(&dcp->dptxport[0]) || generation != dcp->typec_generation) goto out_unlock; - } + service = dcp->dptxport[0].service; - /* Revoke older reconnect waits without changing physical cable state. */ - dcp->typec_generation++; - session = dcp_session_locked(dcp); - cancel_delayed_work(&dcp->typec_reconnect_wq); - cancel_delayed_work(&dcp->placeholder_edid_wq); - if (dcp->avep) - av_service_disconnect(dcp); - ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - if (ret) - goto out_unlock; - if (!dcp_connect_session_valid_locked(dcp, session, true, true)) { - ret = -ESTALE; - goto out_unlock; - } - /* A failed earlier connect may also have left an unconfirmed release. */ - ret = dptxport_release_display(dcp->dptxport[0].service); - if (ret) - goto out_unlock; - if (!dcp_connect_session_valid_locked(dcp, session, true, true)) { - ret = -ESTALE; + /* + * Some adapters answer the first connection with a 1024x768 + * placeholder and publish the panel EDID only after HPD drops + * and returns. One pulse; a second placeholder is left alone. + */ + ret = dptxport_set_hpd(service, false); + if (ret) { + dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", + ret); goto out_unlock; } - dcp->dptxport[0].connected = false; mutex_unlock(&dcp->hpd_mutex); - /* Another delivered IRQ must confirm release before a failed attempt retries. */ - return dcp_dptx_connect_session(dcp, 0, session, true, true); - if (dptx_service_available(&dcp->dptxport[port])) - dptxport_set_hpd(dcp->dptxport[port].service, false); + msleep(1000); + + mutex_lock(&dcp->hpd_mutex); + if (!dcp->typec_cable_connected || !dcp->dptxport[0].connected || + generation != dcp->typec_generation) + goto out_unlock; + ret = dptxport_set_hpd(service, true); + if (ret) + dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", + ret); out_unlock: mutex_unlock(&dcp->hpd_mutex); - return ret; -} - -/* - * Release the display link of a Type-C output while its display stays - * attached, as a CRTC power-off does (see dcp_poweroff()): the firmware's - * unplug for it is ignored, and dcp_poweron() connects the link again. - */ -int dcp_dptx_park(struct apple_dcp *dcp) -{ - int ret; - - scoped_guard(mutex, &dcp->hpd_mutex) { - WRITE_ONCE(dcp->typec_crtc_off, true); - dcp->typec_generation++; - } - /* Session checks around reconnect waits reject the revoked generation. */ - cancel_delayed_work(&dcp->typec_reconnect_wq); - cancel_delayed_work(&dcp->placeholder_edid_wq); - if (dcp->avep) - av_service_disconnect(dcp); - if (READ_ONCE(dcp->split.active) && dcp->dptxport[1].enabled && - dcp->dptxport[1].connected) { - ret = dptxport_set_hpd(dcp->dptxport[1].service, false); - if (ret) - return ret; - ret = dcp_dptx_disconnect(dcp, 1); - if (ret) - return ret; - } - if (dcp->dptxport[0].enabled && dcp->dptxport[0].connected) { - ret = dptxport_set_hpd(dcp->dptxport[0].service, false); - if (ret) { - dev_warn(dcp->dev, "failed to deassert Type-C DPTX HPD: %d\n", ret); - return ret; - } - return dcp_dptx_disconnect(dcp, 0); - } - return 0; } -int dcp_dptx_disconnect_drained(struct apple_dcp *dcp, u32 port) +static void dcp_typec_reconnect_work_h17p(struct work_struct *work) { + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + typec_reconnect_wq); int ret; - WRITE_ONCE(dcp->typec_crtc_off, false); - reinit_completion(&dcp->typec_iomfb_hpd_ready); - - disconnected_hpd_event(dcp->connector); + if (!READ_ONCE(dcp->typec_cable_connected)) + return; - if (dcp->avep) - av_service_disconnect(dcp); + ret = dcp_dptx_connect(dcp, 0); + if (!ret) { + dcp->typec_reconnect_tries = 0; + return; + } - if (dcp->dptxport[port].enabled) { - ret = dptxport_set_hpd(dcp->dptxport[port].service, false); - if (ret) - return ret; + /* Not a T602x USB4 tunnel: the full retry budget applies. */ + if (++dcp->typec_reconnect_tries < DPTX_RECONNECT_RETRIES) { + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + return; } - return dcp_dptx_disconnect(dcp, port); + dev_err(dcp->dev, "Type-C DPTX reconnect failed after %u retries: %d\n", + dcp->typec_reconnect_tries, ret); } -bool dcp_fw_compat_is_12_x(struct platform_device *pdev) +static void disconnected_hpd_event_h17p(struct apple_connector *con) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (!con) + return; - return dcp->fw_compat == DCP_FIRMWARE_V_12_3; + /* May run with DRM modeset locks held; defer property/cache cleanup. */ + WRITE_ONCE(con->connected, false); + WRITE_ONCE(con->edid_invalidation_pending, true); + schedule_work(&con->hotplug_wq); } -/* A failed T8122 dcpext: no pipeline of the DRM device. */ -bool dcp_ext_unusable(struct platform_device *pdev) +static int dcp_dptx_disconnect_h17p(struct apple_dcp *dcp, u32 port) { - struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, false); - return !dcp || READ_ONCE(dcp->ext_dead); + dev_info(dcp->dev, "%s(port=%d)\n", __func__, port); + + mutex_lock(&dcp->hpd_mutex); + if (dptx_service_available(&dcp->dptxport[port]) && dcp->dptxport[port].connected) { + dptxport_release_display(dcp->dptxport[port].service); + dcp->dptxport[port].connected = false; + } + mutex_unlock(&dcp->hpd_mutex); + + return 0; } -bool dcp_fw_compat_is_14_7(struct platform_device *pdev) +static int dcp_dptx_connect_oob_h17p(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); + int ret; - return dcp->fw_compat == DCP_FIRMWARE_V_14_7 || - dcp->fw_compat == DCP_FIRMWARE_V_26_6; + if (dcp_is_external_h17p(dcp)) { + bool new_cable; + + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + cancel_work_sync(&dcp->external_stop_work); + guard(mutex)(&dcp->external_lock); + new_cable = !dcp->external_cable; + if (new_cable) + dcp->external_generation++; + dcp->external_cable = true; + WRITE_ONCE(dcp->external_physically_absent, false); + dcp_external_publish_generation(dcp); + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, true); + if (new_cable) { + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + } + dcp->placeholder_retried = false; + } + return dcp_external_link_locked(dcp); + } + + if (dcp_is_typec_output(dcp)) { + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + guard(mutex)(&dcp->hpd_mutex); + dcp->typec_generation++; + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, true); + dcp->typec_reconnect_tries = 0; + dcp->placeholder_retried = false; + cancel_delayed_work(&dcp->typec_reconnect_wq); + } + + ret = dcp_dptx_connect(dcp, port); + if (ret && dcp_is_typec_output(dcp)) + mod_delayed_work(system_freezable_wq, &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); + + return ret; } -unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) +static int dcp_dptx_disconnect_oob_h17p(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - return dcp->iomfb_surfaces; + if (dcp_is_external_h17p(dcp)) + return dcp_external_disconnect(dcp, true); + + if (dcp_is_typec_output(dcp)) { + scoped_guard(mutex, &dcp->hpd_mutex) { + WRITE_ONCE(dcp->typec_cable_connected, false); + dcp->typec_generation++; + dcp_firmware_metadata_connection(dcp, dcp->typec_generation, false); + } + WRITE_ONCE(dcp->typec_crtc_off, false); + reinit_completion(&dcp->typec_iomfb_hpd_ready); + cancel_delayed_work_sync(&dcp->typec_reconnect_wq); + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); + } + + disconnected_hpd_event_h17p(dcp->connector); + + if (dcp->avep) + av_service_disconnect(dcp); + + if (dptx_service_available(&dcp->dptxport[port])) + dptxport_set_hpd(dcp->dptxport[port].service, false); + + return dcp_dptx_disconnect(dcp, port); } -int dcp_start(struct platform_device *pdev) +static int dcp_start_h17p(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); int ret; @@ -3790,26 +4230,6 @@ int dcp_start(struct platform_device *pdev) return -ENODEV; init_completion(&dcp->start_done); - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) - return iomfb_v26_6_start(dcp); - - /* - * The T6030 firmware session is adopted for the internal panel. - * Open DPTX on that same RTKit only when the firmware advertised it. - */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { - if (dcp->external) { - /* A native pipe is usable now; its firmware starts on first use. */ - if (dcp->external_native) - dcp->active = true; - complete(&dcp->start_done); - return 0; - } - ret = iomfb_v14_7_start(dcp); - if (ret) - return ret; - return 0; - } /* * The coprocessor start can fail (or be recovered from) without tearing @@ -3870,176 +4290,39 @@ int dcp_start(struct platform_device *pdev) } else if (dptx_service_available(&dcp->dptxport[0])) { bool connected; /* force disconnect on start - necessary if the display - * is already up from m1n1 - */ - dptxport_set_hpd(dcp->dptxport[0].service, false); - dptxport_release_display(dcp->dptxport[0].service); - usleep_range(10 * USEC_PER_MSEC, 25 * USEC_PER_MSEC); - - connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); - dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - - // necessary on j473/j474 but not on j314c - if (connected) - dcp_dptx_connect(dcp, 0); -#endif - } - } else if (dcp->phy) { - dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); - } - ret = iomfb_start_rtkit(dcp); - if (ret) { - dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); - return ret; - } - -#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) - if (hdmi_audio) { - ret = avep_init(dcp); - if (ret) - dev_warn(dcp->dev, "Failed to start AV endpoint: %d", ret); - ret = 0; - } -#endif - - return ret; -} - -static void _dcp_poweroff(struct apple_dcp *dcp) -{ - switch (dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - iomfb_poweroff_v12_3(dcp); - break; - case DCP_FIRMWARE_V_13_5: - iomfb_poweroff_v13_3(dcp); - break; - case DCP_FIRMWARE_V_14_7_TEMPLATE: - iomfb_poweroff_v14_7_0(dcp); - break; - case DCP_FIRMWARE_V_26_6: - iomfb_v26_6_poweroff(dcp); - break; - case DCP_FIRMWARE_V_14_7: - iomfb_v14_7_poweroff(dcp); - case DCP_FIRMWARE_H17P: - iomfb_poweroff_h17p(dcp); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); - break; - } -} - -static void dcp_resume_enable_irq(void *ctx) -{ - struct apple_dcp *dcp = ctx; - - enable_irq(dcp->hdmi_hpd_irq); -} - -static void dcp_resume_sample(void *ctx) -{ - dcp_fabric_hdmi_resume(ctx); -} - -static const struct dcp_fabric_resume_ops dcp_resume_ops = { - .enable_irq = dcp_resume_enable_irq, - .sample = dcp_resume_sample, -}; - -static int dcp_enable_dp2hdmi_hpd(struct apple_dcp *dcp) -{ - if (dcp_is_typec_output(dcp)) { - if (READ_ONCE(dcp->typec_cable_connected)) - dcp_dptx_connect(dcp, 0); - } else if (dcp->hdmi_hpd) { - /* Check HPD before enabling the edge-triggered IRQ. */ - bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); - dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - - if (connected) - dcp_dptx_connect(dcp, 0); - else - _dcp_poweroff(dcp); - } - - if (dcp->hdmi_hpd_irq) - dcp_fabric_run_resume(&dcp_resume_ops, dcp); - - return 0; -} - -int dcp_wait_ready(struct platform_device *pdev, u64 timeout) -{ - struct apple_dcp *dcp = platform_get_drvdata(pdev); - int ret; - - if (dcp->crashed) - return -ENODEV; - if (dcp->active) - return dcp_enable_dp2hdmi_hpd(dcp); - if (timeout <= 0) - return -ETIMEDOUT; - - ret = wait_for_completion_timeout(&dcp->start_done, timeout); - if (ret < 0) - return ret; - - if (dcp->crashed) - return -ENODEV; + * is already up from m1n1 + */ + dptxport_set_hpd(dcp->dptxport[0].service, false); + dptxport_release_display(dcp->dptxport[0].service); + usleep_range(10 * USEC_PER_MSEC, 25 * USEC_PER_MSEC); - if (dcp->active) - dcp_enable_dp2hdmi_hpd(dcp); + connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); + dev_info(dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - return dcp->active ? 0 : -ETIMEDOUT; -} + // necessary on j473/j474 but not on j314c + if (connected) + dcp_dptx_connect(dcp, 0); +#endif + } + } else if (dcp->phy) { + dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); + } + ret = iomfb_start_rtkit(dcp); + if (ret) + dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); -static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) -{ - switch (dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - iomfb_sleep_v12_3(dcp); - break; - case DCP_FIRMWARE_V_13_5: - iomfb_sleep_v13_3(dcp); - break; - case DCP_FIRMWARE_V_14_7_TEMPLATE: - iomfb_sleep_v14_7_0(dcp); - break; - case DCP_FIRMWARE_V_26_6: - iomfb_v26_6_poweroff(dcp); - break; - case DCP_FIRMWARE_V_14_7: - iomfb_v14_7_poweroff(dcp); - case DCP_FIRMWARE_H17P: - iomfb_sleep_h17p(dcp); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); - break; +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + if (hdmi_audio) { + ret = avep_init(dcp); + if (ret) + dev_warn(dcp->dev, "Failed to start AV endpoint: %d", ret); + ret = 0; } -} +#endif -int dcp_typec_prepare_route(struct apple_dcp *dcp) -{ - u64 generation; - unsigned long ready; - int ret; + return ret; +} - if (!dcp_crtc_needs_route_start(dcp)) - return 0; - scoped_guard(mutex, &dcp->hpd_mutex) - generation = dcp->typec_generation; - ret = dcp_dptx_connect(dcp, 0); - if (ret) - return ret; - ready = wait_for_completion_timeout(&dcp->typec_iomfb_hpd_ready, - msecs_to_jiffies(3000)); - guard(mutex)(&dcp->hpd_mutex); - if (generation != dcp->typec_generation || !dcp->typec_cable_connected) - return -ESTALE; - return ready ? 0 : -ETIMEDOUT; #define DCP_BL_FALLBACK_DELAY msecs_to_jiffies(2000) static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) @@ -4050,7 +4333,7 @@ static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) dcp->connector_type == DRM_MODE_CONNECTOR_eDP); } -void dcp_poweron(struct platform_device *pdev) +static void dcp_poweron_h17p(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); bool wait_for_typec_hpd = false; @@ -4093,15 +4376,9 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_is_typec_output(dcp)) { - /* T8122 dcpext only, see dcp_poweroff() */ - if (dcp->ext_failsafe) { - scoped_guard(mutex, &dcp->hpd_mutex) - dcp->crtc_unplugs = dcp->typec_unplugs; - } wait_for_typec_hpd = READ_ONCE(dcp->typec_crtc_off) && READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); - WRITE_ONCE(dcp->typec_follow_start, false); /* * A Type-C CRTC disable releases its DPTX session. Re-establish it @@ -4112,7 +4389,9 @@ void dcp_poweron(struct platform_device *pdev) dcp->typec_reconnect_tries = 0; ret = dcp_dptx_connect(dcp, 0); if (ret) - dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); + mod_delayed_work(system_freezable_wq, + &dcp->typec_reconnect_wq, + DPTX_RECONNECT_DELAY); else if (wait_for_typec_hpd) { remaining = wait_for_completion_timeout( &dcp->typec_iomfb_hpd_ready, @@ -4137,14 +4416,6 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_V_13_5: iomfb_poweron_v13_3(dcp); break; - case DCP_FIRMWARE_V_14_7_TEMPLATE: - iomfb_poweron_v14_7_0(dcp); - break; - case DCP_FIRMWARE_V_26_6: - iomfb_v26_6_poweron(dcp); - break; - case DCP_FIRMWARE_V_14_7: - iomfb_v14_7_poweron(dcp); case DCP_FIRMWARE_H17P: iomfb_poweron_h17p(dcp); break; @@ -4165,10 +4436,9 @@ void dcp_external_recover(struct platform_device *pdev) dcp_poweron(pdev); } -void dcp_poweroff(struct platform_device *pdev) +static void dcp_poweroff_h17p(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - bool display_gone = false; int ret; if (dcp_is_external_h17p(dcp)) { @@ -4191,35 +4461,9 @@ void dcp_poweroff(struct platform_device *pdev) return; } - cancel_delayed_work(&dcp->swap_watchdog_wq); if (dcp->avep) av_service_disconnect(dcp); - /* - * The display this CRTC was powered on for has been unplugged since: - * its link went down with the cable (dcp_dptx_disconnect_oob()), and - * the desktop only now turns the CRTC off. By then the pipeline may - * already drive the next display, on this port or on another one that - * was waiting for it, and that connection is not the CRTC's to - * release: dropping its HPD makes the firmware report the new display - * inactive, and it stays dark. - * - * Only the T8122 dcpext (ext_failsafe) takes this path, the one - * pipeline where it was run. Every other Type-C output releases its - * connection below as before: the M1/M2 dcpext routes, and the T6030 - * split second port and external retry. - */ - if (dcp->ext_failsafe && dcp_is_typec_output(dcp)) { - scoped_guard(mutex, &dcp->hpd_mutex) - display_gone = dcp->typec_unplugs != dcp->crtc_unplugs; - if (display_gone) { - dev_dbg(dcp->dev, - "CRTC off after its display was unplugged: leaving the current connection alone\n"); - _dcp_poweroff(dcp); - return; - } - } - /* * Powering a Type-C CRTC off drops DCP's synthetic HPD, and the firmware * reports that as an unplug. The display is still attached: keep the @@ -4240,7 +4484,6 @@ void dcp_poweroff(struct platform_device *pdev) _dcp_poweroff(dcp); if (dcp_is_typec_output(dcp)) { - ret = 0; /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ if (dptx_service_available(&dcp->dptxport[0]) && dcp->dptxport[0].connected) { ret = dptxport_set_hpd(dcp->dptxport[0].service, false); @@ -4249,33 +4492,16 @@ void dcp_poweroff(struct platform_device *pdev) "failed to deassert Type-C DPTX HPD: %d\n", ret); dcp_dptx_disconnect(dcp, 0); } - /* - * A native external pipe whose display is no longer described - * (it went away during a hotplug bounce) is powered off by the - * desktop for good: with HPD still high, connect the link again - * so that the firmware describes the display anew. - */ - if (dcp->external_native && READ_ONCE(dcp->typec_cable_connected) && - (ret || !dcp->connector || !READ_ONCE(dcp->connector->connected))) - dcp_external_retry(dcp, ret ? "display link released with an HPD error" : - "display link released with no display described", - ret, 500); - /* and the second tile's, which dcp_poweron() brings back too */ - if (READ_ONCE(dcp->split.active) && dcp->dptxport[1].enabled && - dcp->dptxport[1].connected) { - dptxport_set_hpd(dcp->dptxport[1].service, false); - dcp_dptx_disconnect(dcp, 1); - } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); if (!connected) { - disconnected_hpd_event(dcp->connector); + disconnected_hpd_event_h17p(dcp->connector); dcp_dptx_disconnect(dcp, 0); } } } -static void dcp_work_register_backlight(struct work_struct *work) +static void dcp_work_register_backlight_h17p(struct work_struct *work) { int ret; struct apple_dcp *dcp; @@ -4305,15 +4531,6 @@ static void dcp_work_register_backlight(struct work_struct *work) mutex_unlock(&dcp->bl_register_mutex); } -static void dcp_work_update_backlight(struct work_struct *work) -{ - struct apple_dcp *dcp; - - dcp = container_of(work, struct apple_dcp, bl_update_wq); - - dcp_backlight_update(dcp); -} - /* * H17P never publishes the panel level, and the loader's level arrives only * as an optional powerlog report. Without one, register the backlight at @@ -4430,7 +4647,8 @@ void dcp_release_all_retained_framebuffers(struct apple_dcp *dcp) } } -static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) + +static int dcp_create_piodma_iommu_dev_h17p(struct apple_dcp *dcp) { int ret; struct device_node *node __free(device_node) = of_get_child_by_name(dcp->dev->of_node, "piodma"); @@ -4446,137 +4664,36 @@ static int dcp_create_piodma_iommu_dev(struct apple_dcp *dcp) if (!dcp->piodma) return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); - dcp->piodma = of_platform_device_create(node, NULL, dcp->dev); - if (!dcp->piodma) - return dev_err_probe(dcp->dev, -ENODEV, "Failed to create piodma pdev for %pOF\n", node); ret = dma_set_mask_and_coherent(&dcp->piodma->dev, DMA_BIT_MASK(42)); - if (ret) - goto err_destroy_pdev; - - ret = of_dma_configure(&dcp->piodma->dev, node, true); - if (ret) { - ret = dev_err_probe(dcp->dev, ret, - "Failed to configure IOMMU child DMA\n"); - goto err_destroy_pdev; - } - - dcp->iommu_dom = iommu_get_domain_for_dev(&dcp->piodma->dev); - if (IS_ERR(dcp->iommu_dom)) { - ret = dev_err_probe(dcp->dev, PTR_ERR(dcp->iommu_dom), - "Failed to get default iommu domain for " - "piodma device\n"); - dcp->iommu_dom = NULL; - goto err_destroy_pdev; - } - - return 0; -err_destroy_pdev: - dcp_release_piodma_iommu_dev(dcp); - return ret; -} - -static int dcp_get_bw_scratch_reg(struct apple_dcp *dcp, u32 expected) -{ - struct of_phandle_args ph_args; - u32 addr_idx, disp_idx, offset; - int ret; - - ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-scratch", - "#apple,bw-scratch-cells", 0, &ph_args); - if (ret < 0) { - dev_err(dcp->dev, "Failed to read 'apple,bw-scratch': %d\n", ret); - return ret; - } - - if (ph_args.args_count != 3) { - dev_err(dcp->dev, "Unexpected 'apple,bw-scratch' arg count %d\n", - ph_args.args_count); - ret = -EINVAL; - goto err_of_node_put; - } - - addr_idx = ph_args.args[0]; - disp_idx = ph_args.args[1]; - offset = ph_args.args[2]; - - if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { - dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-scratch': %d\n", - disp_idx); - ret = -EINVAL; - goto err_of_node_put; - } - - ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_scratch_res); - if (ret < 0) { - dev_err(dcp->dev, "Failed to get 'apple,bw-scratch' resource %d from %pOF\n", - addr_idx, ph_args.np); - goto err_of_node_put; - } - if (offset > resource_size(&dcp->disp_bw_scratch_res) - 4) { - ret = -EINVAL; - goto err_of_node_put; - } - - dcp->disp_registers[disp_idx] = &dcp->disp_bw_scratch_res; - dcp->disp_bw_scratch_index = disp_idx; - dcp->disp_bw_scratch_offset = offset; - ret = 0; - -err_of_node_put: - of_node_put(ph_args.np); - return ret; -} - -static int dcp_get_bw_doorbell_reg(struct apple_dcp *dcp, u32 expected) -{ - struct of_phandle_args ph_args; - u32 addr_idx, disp_idx; - int ret; - - ret = of_parse_phandle_with_args(dcp->dev->of_node, "apple,bw-doorbell", - "#apple,bw-doorbell-cells", 0, &ph_args); - if (ret < 0) { - dev_err(dcp->dev, "Failed to read 'apple,bw-doorbell': %d\n", ret); - return ret; - } - - if (ph_args.args_count != 2) { - dev_err(dcp->dev, "Unexpected 'apple,bw-doorbell' arg count %d\n", - ph_args.args_count); - ret = -EINVAL; - goto err_of_node_put; - } - - addr_idx = ph_args.args[0]; - disp_idx = ph_args.args[1]; - - if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { - dev_err(dcp->dev, "Unexpected disp_reg value in 'apple,bw-doorbell': %d\n", - disp_idx); - ret = -EINVAL; - goto err_of_node_put; + if (ret) + goto err_destroy_pdev; + + ret = of_dma_configure(&dcp->piodma->dev, node, true); + if (ret) { + ret = dev_err_probe(dcp->dev, ret, + "Failed to configure IOMMU child DMA\n"); + goto err_destroy_pdev; } - ret = of_address_to_resource(ph_args.np, addr_idx, &dcp->disp_bw_doorbell_res); - if (ret < 0) { - dev_err(dcp->dev, "Failed to get 'apple,bw-doorbell' resource %d from %pOF\n", - addr_idx, ph_args.np); - goto err_of_node_put; + dcp->iommu_dom = iommu_get_domain_for_dev(&dcp->piodma->dev); + if (IS_ERR(dcp->iommu_dom)) { + ret = dev_err_probe(dcp->dev, PTR_ERR(dcp->iommu_dom), + "Failed to get default iommu domain for " + "piodma device\n"); + dcp->iommu_dom = NULL; + goto err_destroy_pdev; } - dcp->disp_bw_doorbell_index = disp_idx; - dcp->disp_registers[disp_idx] = &dcp->disp_bw_doorbell_res; - ret = 0; -err_of_node_put: - of_node_put(ph_args.np); + return 0; +err_destroy_pdev: + dcp_release_piodma_iommu_dev(dcp); return ret; } -static int dcp_get_disp_regs(struct apple_dcp *dcp) +static int dcp_get_disp_regs_h17p(struct apple_dcp *dcp) { struct platform_device *pdev = to_platform_device(dcp->dev); - int count = pdev->num_resources - 1; int count = 0; int i, ret; char name[16]; @@ -4635,278 +4752,40 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) count = windows; } if (count != windows || resource_size(dcp->hw.firmware_clock) < 4 || - dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || - dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) - return -EINVAL; - - dcp->disp_bw_scratch_res = *dcp->hw.firmware_clock; - dcp->disp_bw_scratch_index = count; - dcp->disp_bw_scratch_offset = dcp->hw.firmware_scratch; - dcp->disp_bw_doorbell_res = dcp->disp_bw_scratch_res; - dcp->disp_bw_doorbell_res.start += dcp->hw.firmware_request; - dcp->disp_bw_doorbell_res.end = dcp->disp_bw_doorbell_res.start + 3; - dcp->disp_registers[count] = &dcp->disp_bw_scratch_res; - dcp->nr_disp_registers = count + 1; - return 0; - } - - /* load pmgr bandwidth scratch resource and offset */ - ret = dcp_get_bw_scratch_reg(dcp, count); - if (ret < 0) - return ret; - count += 1; - - /* load pmgr bandwidth doorbell resource if present (only on t8103) */ - if (of_property_present(dcp->dev->of_node, "apple,bw-doorbell")) { - ret = dcp_get_bw_doorbell_reg(dcp, count); - if (ret < 0) - return ret; - count += 1; - } - - dcp->nr_disp_registers = count; - return 0; -} - -#define DCP_FW_VERSION_MIN_LEN 3 -#define DCP_FW_VERSION_MAX_LEN 5 -#define DCP_FW_VERSION_STR_LEN (DCP_FW_VERSION_MAX_LEN * 4) - -static int dcp_read_fw_version(struct device *dev, const char *name, - char *version_str) -{ - u32 ver[DCP_FW_VERSION_MAX_LEN]; - int len_str; - int len; - - len = of_property_read_variable_u32_array(dev->of_node, name, ver, - DCP_FW_VERSION_MIN_LEN, - DCP_FW_VERSION_MAX_LEN); - - switch (len) { - case 3: - len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, - "%d.%d.%d", ver[0], ver[1], ver[2]); - break; - case 4: - len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, - "%d.%d.%d.%d", ver[0], ver[1], ver[2], - ver[3]); - break; - case 5: - len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, - "%d.%d.%d.%d.%d", ver[0], ver[1], ver[2], - ver[3], ver[4]); - break; - default: - len_str = strscpy(version_str, "UNKNOWN", - DCP_FW_VERSION_STR_LEN); - if (len >= 0) - len = -EOVERFLOW; - break; - } - - if (len_str >= DCP_FW_VERSION_STR_LEN) - dev_warn(dev, "'%s' truncated: '%s'\n", name, version_str); - - return len; -} - -static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) -{ - const struct apple_dcp_hw_data *hw = of_device_get_match_data(dev); - const struct dcp_v14_board *v14_board; - char compat_str[DCP_FW_VERSION_STR_LEN]; - char fw_str[DCP_FW_VERSION_STR_LEN]; - int ret; - - /* firmware version is just informative */ - dcp_read_fw_version(dev, "apple,firmware-version", fw_str); - - ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); - if (ret < 0) { - dev_err(dev, "Could not read 'apple,firmware-compat': %d\n", ret); - return DCP_FIRMWARE_UNKNOWN; - } - - /* 25G83 (J613; J615 experimental) has a distinct callback table, never a v14 fallback. */ - if (of_device_is_compatible(dev->of_node, "apple,t8122-dcp") && - of_property_present(dev->of_node, "apple,j613-25g83-profile")) { - const char *uuid; - u32 profile; - - if (apple_t8122_25g83_board() && - !of_property_read_u32(dev->of_node, "apple,j613-25g83-profile", &profile) && - profile == 1 && !strcmp(compat_str, "26.6.2") && - !of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) && - !strcmp(uuid, "C042E95C-B9D8-3F0E-94B3-582A08AA6FDD")) - return DCP_FIRMWARE_V_26_6; - return DCP_FIRMWARE_UNKNOWN; - } - - /* - * The T8122 external processor: Linux starts it and drives 14.7 - * firmware through the classic IOMFB template, as on M1/M2. - */ - if (of_device_is_compatible(dev->of_node, "apple,t8122-dcpext")) { - if (!strcmp(compat_str, "14.7.0")) - return DCP_FIRMWARE_V_14_7_TEMPLATE; - dev_err(dev, "T8122 external display not started: DCP firmware-compat %s is not 14.7.0\n", - compat_str); - return DCP_FIRMWARE_UNKNOWN; - } - - /* The T6030 external processors: the 14.x firmware IOMFB only. */ - if (of_device_is_compatible(dev->of_node, "apple,t6030-dcpext")) { - if (!strcmp(compat_str, "14.7.0")) - return DCP_FIRMWARE_V_14_7; - dev_err(dev, "T6030 display not started: DCP firmware-compat %s is not 14.7.0\n", - compat_str); - return DCP_FIRMWARE_UNKNOWN; - } - /* - * SoCs introduced with H17-generation firmware pin its interface; the - * loader may not recognise their boot firmware version. - */ - if (hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) - return hw->firmware_compat; - - /* firmware version is just informative */ - dcp_read_fw_version(dev, "apple,firmware-version", fw_str); - - /* The internal panels with a board record (T6030, J613): the same. */ - v14_board = iomfb_v14_7_board(dev); - if (IS_ERR(v14_board)) - return DCP_FIRMWARE_UNKNOWN; - if (v14_board) { - if (!strcmp(compat_str, "14.7.0")) - return DCP_FIRMWARE_V_14_7; - dev_err(dev, "%s display not started: DCP firmware-compat %s is not 14.7.0\n", - iomfb_v14_7_board_name(v14_board), compat_str); - return DCP_FIRMWARE_UNKNOWN; - } - - if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) - return DCP_FIRMWARE_V_12_3; - /* - * m1n1 reports firmware version 13.5 as compatible with 13.3. This is - * only true for the iomfb endpoint. The interface for the dptx-port - * endpoint changed between 13.3 and 13.5. The driver will only support - * firmware 13.5. Check the actual firmware version for compat version - * 13.3 until m1n1 reports 13.5 as "firmware-compat". - */ - else if ((strncmp(compat_str, "13.3.0", sizeof(compat_str)) == 0) && - (strncmp(fw_str, "13.5.0", sizeof(compat_str)) == 0)) - return DCP_FIRMWARE_V_13_5; - else if (strncmp(compat_str, "13.5.0", sizeof(compat_str)) == 0) - return DCP_FIRMWARE_V_13_5; - - dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", - compat_str, fw_str); - - return DCP_FIRMWARE_UNKNOWN; -} - -static int dcp_connector_type_from_dt(struct device_node *np) -{ - if (of_property_match_string(np, "apple,connector-type", "HDMI-A") >= 0) - return DRM_MODE_CONNECTOR_HDMIA; - if (of_property_match_string(np, "apple,connector-type", "DP") >= 0) - return DRM_MODE_CONNECTOR_DisplayPort; - if (of_property_match_string(np, "apple,connector-type", "USB-C") >= 0) - return DRM_MODE_CONNECTOR_USB; - - return DRM_MODE_CONNECTOR_Unknown; -} - -static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) -{ - scoped_guard(mutex, &dcp->hpd_mutex) { - dcp->typec_work_enabled = false; - if (release_cable) - WRITE_ONCE(dcp->typec_cable_connected, false); - dcp->typec_generation++; - } - /* Block new enqueues as well as draining users of the AFK endpoints. */ - disable_delayed_work_sync(&dcp->typec_reconnect_wq); - disable_delayed_work_sync(&dcp->placeholder_edid_wq); - disable_delayed_work_sync(&dcp->hdmi_settle_wq); - disable_delayed_work_sync(&dcp->hdmi_recheck_wq); - dcp_hdmi_disable(dcp); -} - -static void dcp_enable_typec_work(struct apple_dcp *dcp) -{ - enable_delayed_work(&dcp->typec_reconnect_wq); - enable_delayed_work(&dcp->placeholder_edid_wq); - enable_delayed_work(&dcp->hdmi_settle_wq); - enable_delayed_work(&dcp->hdmi_recheck_wq); - dcp_hdmi_enable(dcp); - scoped_guard(mutex, &dcp->hpd_mutex) - dcp->typec_work_enabled = true; - /* A cable can be routed before the DRM component binds. */ - if (READ_ONCE(dcp->typec_cable_connected)) - dcp_queue_typec_reconnect(dcp, 0); -} - -/* - * The M3 internal displays (14.x and 26.6 IOMFB) also accept - * appledrm.t6030_show_notch=1, the switch existing J613/T6030 installs pass, - * so the panel comes up at its full height (2560x1664 on the J613) instead - * of the module refusing an unknown parameter. - */ -static bool dcp_show_notch(struct apple_dcp *dcp) -{ - if (dcp->fw_compat != DCP_FIRMWARE_V_14_7 && - dcp->fw_compat != DCP_FIRMWARE_V_26_6) - return show_notch; - return show_notch || t6030_show_notch; -} + dcp->hw.firmware_scratch > resource_size(dcp->hw.firmware_clock) - 4 || + dcp->hw.firmware_request > resource_size(dcp->hw.firmware_clock) - 4) + return -EINVAL; -/* - * T8122 dcpext (DCP_FIRMWARE_V_14_7_TEMPLATE): if the PMP runs, the display - * gate added its DISPEXT request and pointed the dcpext at it - * (apple,pmp-report). As on T6030, run the processor only once the PMP has - * acknowledged that request: the running PMP owns the SoC power and - * performance states and treats a block whose request it has not taken as - * idle. The internal display started the PMP long before this, so the wait - * is short; it is bounded, and without an acknowledgement the processor is - * started anyway (as before the request existed), with a warning. - */ -#define DCP_EXT_PMP_TIMEOUT_MS 3000 + dcp->disp_bw_scratch_res = *dcp->hw.firmware_clock; + dcp->disp_bw_scratch_index = count; + dcp->disp_bw_scratch_offset = dcp->hw.firmware_scratch; + dcp->disp_bw_doorbell_res = dcp->disp_bw_scratch_res; + dcp->disp_bw_doorbell_res.start += dcp->hw.firmware_request; + dcp->disp_bw_doorbell_res.end = dcp->disp_bw_doorbell_res.start + 3; + dcp->disp_registers[count] = &dcp->disp_bw_scratch_res; + dcp->nr_disp_registers = count + 1; + return 0; + } -static void dcp_ext_wait_pmp(struct apple_dcp *dcp) -{ - struct device_node *entry; - unsigned long deadline; - int ret; + /* load pmgr bandwidth scratch resource and offset */ + ret = dcp_get_bw_scratch_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; - if (dcp->fw_compat != DCP_FIRMWARE_V_14_7_TEMPLATE) - return; - entry = of_parse_phandle(dcp->dev->of_node, "apple,pmp-report", 0); - if (!entry) { - dev_info(dcp->dev, "no PMP DISPEXT request; starting without one\n"); - return; - } - deadline = jiffies + msecs_to_jiffies(DCP_EXT_PMP_TIMEOUT_MS); - for (;;) { - ret = apple_pmp_report_wait_ready(entry, - msecs_to_jiffies(DCP_EXT_PMP_TIMEOUT_MS)); - /* -EIO: the PMP runs, but has not acknowledged the request yet. */ - if (ret != -EIO || time_after(jiffies, deadline)) - break; - msleep(20); + /* load pmgr bandwidth doorbell resource if present (only on t8103) */ + if (of_property_present(dcp->dev->of_node, "apple,bw-doorbell")) { + ret = dcp_get_bw_doorbell_reg(dcp, count); + if (ret < 0) + return ret; + count += 1; } - of_node_put(entry); - if (ret) - dev_warn(dcp->dev, - "the PMP has not acknowledged the DISPEXT request (%d); starting anyway\n", - ret); - else - dev_info(dcp->dev, "PMP acknowledged the DISPEXT request\n"); + + dcp->nr_disp_registers = count; + return 0; } -static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) +static void dcp_disable_typec_work_h17p(struct apple_dcp *dcp, bool release_cable) { scoped_guard(mutex, &dcp->hpd_mutex) { if (release_cable) @@ -4922,7 +4801,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) cancel_work_sync(&dcp->external_stop_work); } -static void dcp_enable_typec_work(struct apple_dcp *dcp) +static void dcp_enable_typec_work_h17p(struct apple_dcp *dcp) { enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); @@ -4942,8 +4821,7 @@ static bool dcp_external_is_idle(struct apple_dcp *dcp) (status & APPLE_DCP_COPROC_CPU_STATUS_IDLE); } -static int dcp_comp_bind(struct device *dev, struct device *main, void *data) -static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *data) +static int dcp_comp_bind_h17p(struct device *dev, struct device *main, void *data) { struct device_node *panel_np; struct apple_dcp *dcp = dev_get_drvdata(dev); @@ -4969,13 +4847,9 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat if (ret) return ret; - /* A native external processor mapped these at probe, for its start work. */ - if (!dcp->external_native) { - dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), - "coproc"); - if (IS_ERR(dcp->coproc_reg)) - return PTR_ERR(dcp->coproc_reg); - } + dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), "coproc"); + if (IS_ERR(dcp->coproc_reg)) + return PTR_ERR(dcp->coproc_reg); /* * Display coprocessors running H17-generation firmware are left running @@ -5001,7 +4875,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", dcp->index, dcp->dptx_phy, dcp->dptx_die); - if (!dcp_show_notch(dcp)) + if (!show_notch) ret = of_property_read_u32(dev->of_node, "apple,notch-height", &dcp->notch_height); @@ -5036,38 +4910,12 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat of_node_put(panel_np); dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; dcp->connector_type = DRM_MODE_CONNECTOR_eDP; - INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight); + INIT_WORK(&dcp->bl_register_wq, dcp_work_register_backlight_h17p); mutex_init(&dcp->bl_register_mutex); INIT_WORK(&dcp->bl_update_wq, dcp_work_update_backlight); } - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { - ret = iomfb_v26_6_bind(dcp); - if (!ret) - enable_work(&dcp->dimensions_wq); - return ret; - } - - /* The running T6030 firmware is adopted as is. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { - if (dcp->external_native) { - WRITE_ONCE(dcp->external_detached, false); - enable_work(&dcp->vblank_wq); - enable_delayed_work(&dcp->swap_watchdog_wq); - enable_delayed_work(&dcp->external_retry_wq); - enable_work(&dcp->dimensions_wq); - dcp_enable_typec_work(dcp); - return 0; - } - if (dcp->external) - return 0; - ret = iomfb_v14_7_bind(dcp); - if (!ret) - enable_work(&dcp->dimensions_wq); - return ret; - } - - ret = dcp_create_piodma_iommu_dev(dcp); + ret = dcp_create_piodma_iommu_dev_h17p(dcp); if (ret) return dev_err_probe(dev, ret, "Failed to create PIODMA iommu child device\n"); @@ -5077,7 +4925,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } - ret = dcp_get_disp_regs(dcp); + ret = dcp_get_disp_regs_h17p(dcp); if (ret) { dev_err(dev, "failed to find display registers\n"); goto err_piodma; @@ -5125,16 +4973,6 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat } } - dcp->swapped_out_fbs = - (struct list_head)LIST_HEAD_INIT(dcp->swapped_out_fbs); - spin_lock_init(&dcp->swapped_out_lock); - - dcp_ext_wait_pmp(dcp); - - cpu_ctrl = - readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, - dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); if (!dcp->hw.adopt_live_session) { cpu_ctrl = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -5165,34 +5003,6 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat if (dcp->hw.adopt_live_session) dev_info(dev, "negotiating RTKit with the running DCP\n"); - ret = apple_rtkit_wake(dcp->rtk); - if (ret) - return dev_err_probe(dev, ret, - "Failed to boot RTKit: %d\n", ret); - enable_work(&dcp->dimensions_wq); - schedule_work(&dcp->dimensions_wq); - dcp_enable_typec_work(dcp); - return ret; -} - -static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when); - -/* - * The T8122 dcpext boots its firmware here. If that fails, it must not take - * the internal panel's DRM device down with it: keep it as an unusable pipe. - */ -static int dcp_comp_bind(struct device *dev, struct device *main, void *data) -{ - struct apple_dcp *dcp = dev_get_drvdata(dev); - int ret; - - if (READ_ONCE(dcp->ext_dead)) - return 0; - ret = dcp_comp_bind_pipe(dev, main, data); - if (ret && dcp->ext_failsafe) { - dcp_ext_give_up(dcp, ret, "bind"); - return 0; - } /* Announce host readiness before awaiting external IOP readiness. */ if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) @@ -5203,7 +5013,7 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); goto err_piodma; } - dcp_enable_typec_work(dcp); + dcp_enable_typec_work_h17p(dcp); return 0; err_piodma: @@ -5215,29 +5025,13 @@ static int dcp_comp_bind(struct device *dev, struct device *main, void *data) * We need to shutdown DCP before tearing down the display subsystem. Otherwise * the DCP will crash and briefly flash a green screen of death. */ -static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) +static void dcp_comp_unbind_h17p(struct device *dev, struct device *main, void *data) { struct apple_dcp *dcp = dev_get_drvdata(dev); - if (!dcp || READ_ONCE(dcp->ext_dead)) + if (!dcp) return; - disable_work_sync(&dcp->dimensions_wq); - if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { - iomfb_v26_6_unbind(dcp); - return; - } - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { - if (dcp->external_native) { - /* Bind's wait_ready enabled it, as for any HDMI port. */ - if (dcp->hdmi_hpd_irq) - disable_irq(dcp->hdmi_hpd_irq); - dcp_disable_typec_work(dcp, true); - disable_delayed_work_sync(&dcp->external_retry_wq); - disable_delayed_work_sync(&dcp->swap_watchdog_wq); - disable_work_sync(&dcp->vblank_wq); - } - iomfb_v14_7_unbind(dcp); if (dcp->retain_dma) { /* No firmware stop is acknowledged: detach host users, never DMA. */ scoped_guard(mutex, &dcp->iomfb.lock) @@ -5273,35 +5067,19 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) if (dcp->hdmi_hpd_irq) disable_irq(dcp->hdmi_hpd_irq); - dcp_disable_typec_work(dcp, true); - /* RTKit is released after this unbind callback, and can still deliver - * a final swap completion or hotplug while its receive queue drains. - */ - disable_delayed_work_sync(&dcp->swap_watchdog_wq); - disable_work_sync(&dcp->vblank_wq); + dcp_disable_typec_work_h17p(dcp, true); typec_mux_put(dcp->typec_mux); - if (dcp->fixed_connector_type == DRM_MODE_CONNECTOR_eDP) { - /* Registration runs asynchronously and its devres can belong - * to the platform device rather than the component bind group. - * Stop callbacks and userspace writes while the CRTC, RTKit and - * IOMMU they use are still available. - */ - disable_work_sync(&dcp->bl_register_wq); - disable_work_sync(&dcp->bl_update_wq); - if (dcp->brightness.bl_dev) { - devm_backlight_device_unregister(dev, dcp->brightness.bl_dev); - dcp->brightness.bl_dev = NULL; - } - } - if (dcp->avep) { av_service_disconnect(dcp); afk_shutdown(dcp->avep); dcp->avep = NULL; } - dcp_fabric_shutdown_dptx(dcp); + if (dcp->dptxep) { + afk_shutdown(dcp->dptxep); + dcp->dptxep = NULL; + } if (!IS_ERR_OR_NULL(dcp->dptx_controller_ep)) { afk_shutdown(dcp->dptx_controller_ep); @@ -5331,95 +5109,27 @@ static void dcp_comp_unbind(struct device *dev, struct device *main, void *data) dcp_release_piodma_iommu_dev(dcp); cancel_delayed_work_sync(&dcp->bl_fallback_wq); - dcp_backlight_stop(dcp); - if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { - cancel_work_sync(&dcp->bl_register_wq); - cancel_work_sync(&dcp->bl_update_wq); - } - cancel_delayed_work_sync(&dcp->swap_watchdog_wq); - cancel_work_sync(&dcp->vblank_wq); - - devm_clk_put(dev, dcp->clk); - dcp->clk = NULL; - /* optional and possibly NULL, released with the bind's devres group */ - dcp->clk_194 = NULL; -} - -static const struct component_ops dcp_comp_ops = { - .bind = dcp_comp_bind, - .unbind = dcp_comp_unbind, -}; - -/* - * A T6030 external processor's fixed (HDMI) output is optional. Its Type-C - * routes must register whatever happens to that output: every USB-C port - * lists them and waits for them, as the panel's DRM device waits for this - * processor. Keep deferring while the HDMI parts may still appear, then go - * on as a Type-C-only pipe. Elsewhere the error ends the probe, as before. - */ -static int dcp_fixed_output_error(struct apple_dcp *dcp, int err, const char *what) -{ - if (!dcp->external_native) - return err; - if (err == -EPROBE_DEFER) { - err = driver_deferred_probe_check_state(dcp->dev); - if (err == -EPROBE_DEFER) - return err; - } - dev_err(dcp->dev, "HDMI output unusable (%s: %d); USB-C displays are not affected\n", - what, err); - if (dcp->fixed_route_selected) - mux_control_deselect(dcp->xbar); - dcp->fixed_route_selected = false; - dcp->xbar = NULL; - if (!IS_ERR_OR_NULL(dcp->typec_mux)) - typec_mux_put(dcp->typec_mux); - dcp->typec_mux = NULL; - dcp->phy_managed_by_typec = false; - /* A requested HPD interrupt stays disabled: nothing enables it now. */ - dcp->hdmi_hpd_irq = 0; - dcp->hdmi_hpd = NULL; - dcp->hdmi_pwren = NULL; - dcp->dp2hdmi_pwren = NULL; - dcp->phy = NULL; - dcp->fixed_phy = NULL; - dcp->fixed_connector_type = DRM_MODE_CONNECTOR_USB; - dcp->connector_type = DRM_MODE_CONNECTOR_USB; - return 0; -} - -/* - * A T6030 external processor whose Type-C routes did not register: disable - * them in the device tree so that the ports stop waiting for them, and add - * the processor to the DRM device as a pipe that never starts, so that the - * panel is not held up either. - */ -static int dcp_native_routes_error(struct apple_dcp *dcp, int err) -{ - if (!dcp->external_native) - return err; - if (err == -EPROBE_DEFER) { - err = driver_deferred_probe_check_state(dcp->dev); - if (err == -EPROBE_DEFER) - return err; + dcp_backlight_stop(dcp); + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { + cancel_work_sync(&dcp->bl_register_wq); + cancel_work_sync(&dcp->bl_update_wq); } - dev_err(dcp->dev, "Type-C display routes unusable: %d; USB-C ports go on without this processor\n", - err); - dcp_typec_routes_disable(dcp); - return 0; + cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + cancel_work_sync(&dcp->vblank_wq); + + devm_clk_put(dev, dcp->clk); + dcp->clk = NULL; + /* optional and possibly NULL, released with the bind's devres group */ + dcp->clk_194 = NULL; } -static int dcp_platform_probe_pipe(struct platform_device *pdev) +static int dcp_platform_probe_h17p(struct platform_device *pdev) { - if (of_machine_is_compatible("apple,t8140")) - return -ENODEV; - const struct apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); enum dcp_firmware_version fw_compat; struct device *dev = &pdev->dev; struct apple_dcp *dcp; int ret, surf, num_surfs; - bool routes_failed = false; u32 surf_en; u32 mux_index; @@ -5441,9 +5151,6 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) * device trees. This prevents replacing simpledrm and ending up without * display. */ - if (fw_compat != DCP_FIRMWARE_V_14_7 && - fw_compat != DCP_FIRMWARE_V_26_6 && - !of_property_present(dev->of_node, "apple,bw-scratch")) if (!of_property_present(dev->of_node, "apple,bw-scratch") && !hw->firmware_clock) return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " @@ -5464,21 +5171,16 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) iomfb_queue_init(dcp); dcp->fw_compat = fw_compat; - dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext"); - dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); - dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); dcp->dev = dev; /* * Type-C and Thunderbolt routes can be activated as soon as they are * registered below, before the DRM device binds. */ + /* The shared Type-C fabric takes the catalog lock and HDMI presence. */ dcp_modes_init(dcp); - /* Registered before callback resources, so they drain before the catalog. */ ret = devm_add_action_or_reset(dev, dcp_modes_release, dcp); if (ret) return ret; - mutex_init(&dcp->hpd_mutex); - mutex_init(&dcp->tb_lock); dcp_fabric_init(dcp); mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->external_lock); @@ -5515,35 +5217,9 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->fixed_dptx_phy = dcp->dptx_phy; init_completion(&dcp->typec_iomfb_hpd_ready); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, - dcp_typec_reconnect_work); - INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, - dcp_placeholder_edid_work); - INIT_DELAYED_WORK(&dcp->typec_fabric_retrain_wq, - dcp_typec_retrain_work); - /* - * Type-C and Thunderbolt routes can be activated as soon as they are - * registered below (dcp_register_typec_routes()), before the DRM - * device binds and dcp_comp_bind() would otherwise init this. Ported - * from aurora-silicon/linux#8. - */ - mutex_init(&dcp->hpd_mutex); - mutex_init(&dcp->tb_lock); + dcp_typec_reconnect_work_h17p); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, - dcp_placeholder_edid_work); - /* Balanced by enable at successful component bind. */ - disable_delayed_work(&dcp->typec_reconnect_wq); - disable_delayed_work(&dcp->placeholder_edid_wq); - /* - * A native external pipe's bind enables these; firmware callbacks may - * come before it. The 14.7 panel never uses them, but suspend cancels - * the watchdog of every processor. - */ - if (fw_compat == DCP_FIRMWARE_V_14_7 || fw_compat == DCP_FIRMWARE_V_26_6) { - INIT_WORK(&dcp->vblank_wq, dcp_delayed_vblank); - INIT_DELAYED_WORK(&dcp->swap_watchdog_wq, dcp_swap_watchdog); - disable_work(&dcp->vblank_wq); - disable_delayed_work(&dcp->swap_watchdog_wq); - } + dcp_placeholder_edid_work_h17p); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, dcp_work_backlight_fallback); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); @@ -5551,25 +5227,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) platform_set_drvdata(pdev, dcp); - if (fw_compat == DCP_FIRMWARE_V_26_6) { - ret = iomfb_v26_6_probe(dcp); - if (ret) - return ret; - } - if (fw_compat == DCP_FIRMWARE_V_14_7 && !dcp->external) { - ret = iomfb_v14_7_probe(dcp); - if (ret) - return ret; - } - if (dcp->external) - dcp->hw.num_dptx_ports = 2; - dcp->phy = devm_phy_optional_get(dev, "dp-phy"); - if (IS_ERR(dcp->phy) && dcp->external_native) { - ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->phy), "dp-phy"); - if (ret) - return ret; - } if (IS_ERR(dcp->phy)) { dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(dcp->phy)); return PTR_ERR(dcp->phy); @@ -5584,9 +5242,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) "apple,iomfb-surfaces", sizeof(u32)); - if (fw_compat == DCP_FIRMWARE_V_14_7 || fw_compat == DCP_FIRMWARE_V_26_6) { - set_bit(0, dcp->iomfb_surfaces); - } else if (num_surfs == 0 || num_surfs == -ENODATA) { + if (num_surfs == 0 || num_surfs == -ENODATA) { set_bit(0, dcp->iomfb_surfaces); set_bit(1, dcp->iomfb_surfaces); } else if (num_surfs < 0) { @@ -5619,25 +5275,10 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) * template ADT. TODO: check device ADT */ dcp->hdmi_hpd = devm_gpiod_get_optional(dev, "hdmi-hpd", GPIOD_IN); - if (IS_ERR(dcp->hdmi_hpd) && dcp->external_native) { - ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->hdmi_hpd), "hdmi-hpd"); - if (ret) - return ret; - goto fixed_done; - } if (IS_ERR(dcp->hdmi_hpd)) return PTR_ERR(dcp->hdmi_hpd); - ret = dcp_hdmi_init(dcp); - if (ret) - return ret; if (dcp->hdmi_hpd) { int irq = gpiod_to_irq(dcp->hdmi_hpd); - if (irq < 0 && dcp->external_native) { - ret = dcp_fixed_output_error(dcp, irq, "hdmi-hpd interrupt"); - if (ret) - return ret; - goto fixed_done; - } if (irq < 0) { dev_err(dev, "failed to translate HDMI hpd GPIO to IRQ\n"); return irq; @@ -5645,16 +5286,10 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->hdmi_hpd_irq = irq; ret = devm_request_threaded_irq(dev, dcp->hdmi_hpd_irq, - dcp_dp2hdmi_hpd_edge, dcp_dp2hdmi_hpd, + NULL, dcp_dp2hdmi_hpd, IRQF_ONESHOT | IRQF_NO_AUTOEN | IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, "dp2hdmi-hpd-irq", dcp); - if (ret < 0 && dcp->external_native) { - ret = dcp_fixed_output_error(dcp, ret, "hdmi-hpd interrupt"); - if (ret) - return ret; - goto fixed_done; - } if (ret < 0) { dev_err(dev, "failed to request HDMI hpd irq %d: %d\n", irq, ret); @@ -5668,22 +5303,10 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) * on battery powered Macbooks. */ dcp->hdmi_pwren = devm_gpiod_get_optional(dev, "hdmi-pwren", GPIOD_OUT_HIGH); - if (IS_ERR(dcp->hdmi_pwren) && dcp->external_native) { - ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->hdmi_pwren), "hdmi-pwren"); - if (ret) - return ret; - goto fixed_done; - } if (IS_ERR(dcp->hdmi_pwren)) return PTR_ERR(dcp->hdmi_pwren); dcp->dp2hdmi_pwren = devm_gpiod_get_optional(dev, "dp2hdmi-pwren", GPIOD_OUT_HIGH); - if (IS_ERR(dcp->dp2hdmi_pwren) && dcp->external_native) { - ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->dp2hdmi_pwren), "dp2hdmi-pwren"); - if (ret) - return ret; - goto fixed_done; - } if (IS_ERR(dcp->dp2hdmi_pwren)) return PTR_ERR(dcp->dp2hdmi_pwren); @@ -5698,193 +5321,51 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (!ret) { dcp->fixed_mux_index = mux_index; dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); - if (IS_ERR(dcp->xbar) && dcp->external_native) { - ret = dcp_fixed_output_error(dcp, PTR_ERR(dcp->xbar), "dp-xbar"); - if (ret) - return ret; - goto fixed_done; - } if (IS_ERR(dcp->xbar)) { dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(dcp->xbar)); return PTR_ERR(dcp->xbar); } - ret = mux_control_select(dcp->xbar, mux_index); - if (ret) - dev_warn(dev, "mux_control_select failed: %d\n", ret); - else - dcp->fixed_route_selected = true; - - /* - * Switch atcphy to DP-only. should move to a Macbook Pro - * 14-/16-inch specific DP-to-HDMI drm_bridge. - */ - dcp->typec_mux = fwnode_typec_mux_get(dev_fwnode(dcp->dev)); - if (!IS_ERR_OR_NULL(dcp->typec_mux)) { - struct typec_altmode alt = { - .svid = USB_TYPEC_DP_SID, - }; - struct typec_mux_state state = { - .alt = &alt, - .mode = TYPEC_DP_STATE_C, - }; - int ret = typec_mux_set(dcp->typec_mux, &state); - dev_info(dev, "typec_mux_set() returned: %d\n", ret); - if (!ret) - dcp->phy_managed_by_typec = true; - } else { - dev_info(dev, "fwnode_typec_mux_get() returned: %ld\n", - IS_ERR(dcp->typec_mux) ? PTR_ERR(dcp->typec_mux) : 0); - dcp->typec_mux = NULL; - } - } - } - -fixed_done: - ret = dcp_register_typec_routes(dcp); - if (ret) { - ret = dcp_native_routes_error(dcp, ret); - if (ret) - return ret; - routes_failed = true; - } - - /* - * The manual external path stays out of the panel's DRM device. A - * native external processor joins it, and probe completes for it even - * when its handoff is unusable, so that the panel is never held up. - */ - if (dcp->external && !dcp->external_native) - return iomfb_v14_7_external_start(dcp); - if (dcp->external_native) { - ret = iomfb_v14_7_external_prepare(dcp); - if (ret) - return ret; - if (routes_failed) - iomfb_v14_7_external_refuse(dcp, "its Type-C display routes did not register"); - } - - ret = component_add(&pdev->dev, &dcp_comp_ops); - if (ret && dcp->fw_compat == DCP_FIRMWARE_V_26_6) - iomfb_v26_6_unbind(dcp); - /* A failed bind run from here may already have started RTKit. */ - if (ret && dcp->fw_compat == DCP_FIRMWARE_V_14_7) - iomfb_v14_7_remove(dcp); - return ret; -} - -/* - * Marks a T8122 dcpext unusable, for good: its Type-C routes are disabled in - * the device tree, so that a USB-C port controller probing later does not - * wait for them, and a route already registered ignores mode changes (see - * dcp_typec_route_set()), so that USB on the port goes on. - */ -static void dcp_ext_give_up(struct apple_dcp *dcp, int err, const char *when) -{ - WRITE_ONCE(dcp->ext_dead, true); - dev_err(dcp->dev, "external display processor unusable (%s failed: %d); the internal display goes on without it\n", - when, err); - dcp_typec_routes_disable(dcp); -} - -/* - * Whether the DP PHY and display crossbar of every Type-C route are there. - * Routes register one at a time, and one registered cannot be taken back, - * since a port controller may hold it: a later route waiting for its - * supplier would leave the dcpext unusable for the boot. Returns - * -EPROBE_DEFER until all of them are; other errors are left to the route - * probe, which reports them. - */ -static int dcp_ext_route_suppliers(struct device *dev) -{ - struct device_node *routes __free(device_node) = - of_get_child_by_name(dev->of_node, "typec-routes"); - char name[16]; - u32 index; - - if (!routes) - return 0; - for_each_available_child_of_node_scoped(routes, route_np) { - struct mux_control *xbar; - struct phy *phy; - - if (of_property_read_u32(route_np, "reg", &index)) - continue; - snprintf(name, sizeof(name), "typec%u", index); - phy = of_phy_get(dev->of_node, name); - if (PTR_ERR_OR_ZERO(phy) == -EPROBE_DEFER) - return -EPROBE_DEFER; - if (!IS_ERR(phy)) - of_phy_put(phy); - xbar = mux_control_get(dev, name); - if (PTR_ERR_OR_ZERO(xbar) == -EPROBE_DEFER) - return -EPROBE_DEFER; - if (!IS_ERR(xbar)) - mux_control_put(xbar); - } - return 0; -} - -/* - * The T8122 dcpext is a component of the panel's DRM device, so it must - * always register one, or the panel stays on the boot framebuffer. Keep - * deferring while its suppliers may still appear (bounded by the deferred - * probe timeout); on any other failure join as an unusable pipe. - */ -static int dcp_platform_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct apple_dcp *dcp; - void *group; - int ret; - - if (!of_device_is_compatible(dev->of_node, "apple,t8122-dcpext")) - return dcp_platform_probe_pipe(pdev); - - ret = dcp_ext_route_suppliers(dev); - if (ret) - goto unusable; - - group = devres_open_group(dev, NULL, GFP_KERNEL); - if (!group) - return -ENOMEM; - ret = dcp_platform_probe_pipe(pdev); - if (!ret) { - devres_remove_group(dev, group); - return 0; - } + ret = mux_control_select(dcp->xbar, mux_index); + if (ret) + dev_warn(dev, "mux_control_select failed: %d\n", ret); + else + dcp->fixed_route_selected = true; - dcp = platform_get_drvdata(pdev); - if (dcp && dcp->nr_typec_routes) { - /* Routes may be in use already: keep them, and the pipe, as they are. */ - devres_remove_group(dev, group); - dcp_ext_give_up(dcp, ret, "probe"); - return component_add(dev, &dcp_comp_ops); + /* + * Switch atcphy to DP-only. should move to a Macbook Pro + * 14-/16-inch specific DP-to-HDMI drm_bridge. + */ + dcp->typec_mux = fwnode_typec_mux_get(dev_fwnode(dcp->dev)); + if (!IS_ERR_OR_NULL(dcp->typec_mux)) { + struct typec_altmode alt = { + .svid = USB_TYPEC_DP_SID, + }; + struct typec_mux_state state = { + .alt = &alt, + .mode = TYPEC_DP_STATE_C, + }; + int ret = typec_mux_set(dcp->typec_mux, &state); + dev_info(dev, "typec_mux_set() returned: %d\n", ret); + if (!ret) + dcp->phy_managed_by_typec = true; + } else { + dev_info(dev, "fwnode_typec_mux_get() returned: %ld\n", + IS_ERR(dcp->typec_mux) ? PTR_ERR(dcp->typec_mux) : 0); + dcp->typec_mux = NULL; + } + } } - devres_release_group(dev, group); - platform_set_drvdata(pdev, NULL); -unusable: - if (ret == -EPROBE_DEFER) { - ret = driver_deferred_probe_check_state(dev); - if (ret == -EPROBE_DEFER) - return ret; - } + ret = dcp_register_typec_routes(dcp); + if (ret) + return ret; - dcp = devm_kzalloc(dev, sizeof(*dcp), GFP_KERNEL); - if (!dcp) - return -ENOMEM; - dcp->dev = dev; - dcp->ext_failsafe = true; - platform_set_drvdata(pdev, dcp); - dcp_ext_give_up(dcp, ret, "probe"); - return component_add(dev, &dcp_comp_ops); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { /* Complete optional startup before the DRM master can depend on it. */ - ret = dcp_comp_bind(dev, NULL, NULL); + ret = dcp_comp_bind_h17p(dev, NULL, NULL); if (ret) { - dcp_disable_typec_work(dcp, true); - disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + dcp_disable_typec_work_h17p(dcp, true); return ret; } WRITE_ONCE(dcp->external_prepared, true); @@ -5892,37 +5373,18 @@ static int dcp_platform_probe(struct platform_device *pdev) ret = component_add(&pdev->dev, &dcp_comp_ops); if (ret && dcp->external_prepared) { - dcp_disable_typec_work(dcp, true); - disable_delayed_work_sync(&dcp->typec_fabric_retrain_wq); + dcp_disable_typec_work_h17p(dcp, true); } return ret; } -static void dcp_platform_remove(struct platform_device *pdev) -{ - struct apple_dcp *dcp = platform_get_drvdata(pdev); - - if (dcp && dcp->external && !dcp->external_native) { - iomfb_v14_7_remove(dcp); - return; - } - component_del(&pdev->dev, &dcp_comp_ops); - if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_26_6) - iomfb_v26_6_unbind(dcp); - /* Unbind does not wait for RTKit callbacks, or run if bind failed late. */ - if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_14_7) - iomfb_v14_7_remove(dcp); -} - -static void dcp_platform_shutdown(struct platform_device *pdev) +static void dcp_platform_shutdown_h17p(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - if (dcp && dcp->external && !dcp->external_native) - return; if (dcp && dcp->retain_dma && dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { - dcp_comp_unbind(&pdev->dev, NULL, NULL); + dcp_comp_unbind_h17p(&pdev->dev, NULL, NULL); return; } @@ -5944,90 +5406,10 @@ static void dcp_platform_shutdown(struct platform_device *pdev) return; } - if (dcp) { - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - if (dcp) { - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); - } component_del(&pdev->dev, &dcp_comp_ops); } /* - * Why an external processor must keep the system awake, or NULL. The manual - * path refuses sleep once its firmware was ever started. A native processor - * refuses while its firmware starts or holds a display link, and while a - * display is attached to it at all, lit or not: resume does not bring an - * attached display back. Idle with nothing attached, its firmware stays - * powered across s2idle like the panel's (the gate holds its CPU domain at - * the active floor) and the next attach reconnects through it. - * - * A processor that failed (its start, or its session later) never refuses - * sleep: it drives no display until reboot, attached or not, and its power - * domains stay on across s2idle as they are, like those of a working one. - */ -static const char *dcp_external_sleep_vetoed(struct apple_dcp *dcp) -{ - const char *why; - - if (!dcp->external_native) - return (atomic_read(&dcp->external_requested) || - (dcp->rtk && apple_rtkit_is_running(dcp->rtk)) || - dcpext_scanout_requested(dcp)) ? - "external firmware/scanout attempted" : NULL; - why = iomfb_v14_7_external_busy(dcp); - if (why) - return why; - if (iomfb_v14_7_external_failed(dcp)) { - if (dcp->v14) - dev_info(dcp->dev, "external display processor failed earlier: sleeping with it as it is\n"); - return NULL; - } - if (READ_ONCE(dcp->active_typec_route) || READ_ONCE(dcp->typec_cable_connected)) - return "a USB-C display is attached"; - if (dcp->hdmi_hpd && gpiod_get_value_cansleep(dcp->hdmi_hpd) > 0) - return "an HDMI display is attached"; - return NULL; -} - -/* dpm_prepare completes for every device before any dpm_suspend callback. - * Veto early so Thunderbolt cannot tear down a working tunnel first. The - * same HPD lock serializes this gate against explicit firmware startup. - */ -static int dcp_platform_prepare(struct device *dev) -{ - struct apple_dcp *dcp = dev_get_drvdata(dev); - const char *why; - - if (!dcp->external) - return 0; - mutex_lock(&dcp->hpd_mutex); - why = dcp_external_sleep_vetoed(dcp); - if (why) { - mutex_unlock(&dcp->hpd_mutex); - if (dcp->external_native) - dev_warn(dev, "refusing system sleep: %s\n", why); - else - dev_warn(dev, "external firmware/scanout attempted: refusing PM prepare; reboot required for retained DMA\n"); - return -EBUSY; - } - WRITE_ONCE(dcp->external_suspended, true); - mutex_unlock(&dcp->hpd_mutex); - return 0; -} - -static void dcp_platform_complete(struct device *dev) -{ - struct apple_dcp *dcp = dev_get_drvdata(dev); - - if (!dcp->external) - return; - mutex_lock(&dcp->hpd_mutex); - WRITE_ONCE(dcp->external_suspended, false); - mutex_unlock(&dcp->hpd_mutex); * The display stays powered across system sleep, and the blanking present * queued by the DRM and backlight suspend handlers must reach the firmware * before its mailbox interrupt is suspended. @@ -6105,54 +5487,17 @@ static void dcp_release_after_sleep(struct apple_dcp *dcp) iomfb_pause_presentation(dcp, false); } -static int dcp_platform_suspend(struct device *dev) +static int dcp_platform_suspend_h17p(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); - if (READ_ONCE(dcp->ext_dead)) - return 0; - - /* Serialize PM against explicit bring-up, including queued startup work. - * Before a startup attempt the device remains suspendable. After any - * attempt we cannot prove DMA/firmware quiescence, even on failure. - */ - if (dcp->external) { - const char *why; - - mutex_lock(&dcp->hpd_mutex); - why = dcp_external_sleep_vetoed(dcp); - if (why) { - mutex_unlock(&dcp->hpd_mutex); - if (dcp->external_native) - dev_warn(dev, "suspend refused: %s\n", why); - else - dev_warn(dev, "external firmware/scanout attempted: suspend refused; retained DMA requires reboot\n"); - return -EBUSY; - } - WRITE_ONCE(dcp->external_suspended, true); - mutex_unlock(&dcp->hpd_mutex); - } - /* - * The Type-C route reports cable removal through - * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays - * connected, and resume powers the CRTC back up through dcp_poweron() - * as after DPMS off. - */ - dcp_disable_typec_work(dcp, false); - cancel_delayed_work_sync(&dcp->swap_watchdog_wq); - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - WRITE_ONCE(dcp->typec_cable_connected, false); - cancel_delayed_work_sync(&dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&dcp->placeholder_edid_wq); - cancel_delayed_work_sync(&dcp->typec_fabric_retrain_wq); if (dcp_is_external_h17p(dcp)) { int ret; - dcp_disable_typec_work(dcp, false); + dcp_disable_typec_work_h17p(dcp, false); ret = dcp_external_disconnect(dcp, false); if (ret) { - dcp_enable_typec_work(dcp); + dcp_enable_typec_work_h17p(dcp); return ret; } device_set_wakeup_path(dev); @@ -6166,7 +5511,7 @@ static int dcp_platform_suspend(struct device *dev) * connected, and resume powers the CRTC back up through dcp_poweron() * as after DPMS off. */ - dcp_disable_typec_work(dcp, false); + dcp_disable_typec_work_h17p(dcp, false); cancel_delayed_work_sync(&dcp->swap_watchdog_wq); if (dcp->avep) @@ -6175,8 +5520,8 @@ static int dcp_platform_suspend(struct device *dev) if (dcp->hdmi_hpd_irq) { disable_irq(dcp->hdmi_hpd_irq); if (!dcp->active_typec_route) { - disconnected_hpd_event(dcp->connector); - dcp_dptx_disconnect(dcp, 0); + disconnected_hpd_event_h17p(dcp->connector); + dcp_dptx_disconnect_h17p(dcp, 0); } } /* @@ -6191,22 +5536,14 @@ static int dcp_platform_suspend(struct device *dev) return 0; } -static int dcp_platform_resume(struct device *dev) +static int dcp_platform_resume_h17p(struct device *dev) { struct apple_dcp *dcp = dev_get_drvdata(dev); - if (READ_ONCE(dcp->ext_dead)) - return 0; - - dcp_enable_typec_work(dcp); - /* The HDMI output's PHY and crossbar may have lost their setup in sleep. */ - if (dcp->external_native) - dcp_fabric_hdmi_reinit(dcp, "after system sleep"); - /* Observe future edges before sampling any edges lost in sleep. */ dcp_release_after_sleep(dcp); - dcp_enable_typec_work(dcp); + dcp_enable_typec_work_h17p(dcp); if (dcp->hdmi_hpd_irq) - dcp_fabric_run_resume(&dcp_resume_ops, dcp); + enable_irq(dcp->hdmi_hpd_irq); if (dcp->avep) av_service_connect(dcp); @@ -6235,6 +5572,35 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8112 = { .num_dptx_ports = 2, }; +static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { + .num_dptx_ports = 0, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t6030 = { + .num_dptx_ports = 1, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t8122 = { + .num_dptx_ports = 2, +}; + +/* The internal panel only, through the 14.x IOMFB, as on T6030. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t6031 = { + .num_dptx_ports = 1, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t6030_dcpext = { + .num_dptx_ports = 1, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t8122_dcpext = { + .num_dptx_ports = 2, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { + .num_dptx_ports = 2, +}; + /* The internal T8140 endpoint uses the H17P method profile. */ static const struct resource t8140_firmware_clock = DEFINE_RES_MEM(0x302800000, 0xbc000); @@ -6273,35 +5639,6 @@ static const struct apple_dcp_hw_data apple_dcp_hw_t8142 = { .firmware_compat = DCP_FIRMWARE_H17P, }; -static const struct apple_dcp_hw_data apple_dcp_hw_dcp = { - .num_dptx_ports = 0, -}; - -static const struct apple_dcp_hw_data apple_dcp_hw_t6030 = { - .num_dptx_ports = 1, -}; - -static const struct apple_dcp_hw_data apple_dcp_hw_t8122 = { - .num_dptx_ports = 2, -}; - -/* The internal panel only, through the 14.x IOMFB, as on T6030. */ -static const struct apple_dcp_hw_data apple_dcp_hw_t6031 = { - .num_dptx_ports = 1, -}; - -static const struct apple_dcp_hw_data apple_dcp_hw_t6030_dcpext = { - .num_dptx_ports = 1, -}; - -static const struct apple_dcp_hw_data apple_dcp_hw_t8122_dcpext = { - .num_dptx_ports = 2, -}; - -static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { - .num_dptx_ports = 2, -}; - static const struct of_device_id of_match[] = { { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t6020-dcpext", .data = &apple_dcp_hw_t6020_dcpext, }, diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index 243c5cb8504f96..0fc12de4db11f7 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -20,7 +20,7 @@ struct apple_atomic_state { }; #define to_apple_atomic_state(s) container_of(s, struct apple_atomic_state, base) -bool dcp_h17p_external_enabled(void); + bool dcp_h17p_external_supported(struct device_node *np); bool dcp_h17p_external_ready(struct device_node *np); @@ -81,7 +81,8 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode); void dcp_set_dimensions(struct apple_dcp *dcp, u64 generation); -void dcp_set_dimensions(struct apple_dcp *dcp); +/* H17P (T8140, T8142) */ +void dcp_set_dimensions_h17p(struct apple_dcp *dcp); void dcp_update_connector_dimensions(struct apple_dcp *dcp, struct drm_connector *connector); void dcp_send_message(struct apple_dcp *dcp, u8 endpoint, u64 message); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 24fc2077e238aa..2bdd58652de0a8 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -407,6 +407,44 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) if (!dcp->brightness.update) goto done; + state = drm_atomic_state_alloc(crtc->dev); + if (!state) { + ret = -ENOMEM; + goto done; + } + + state->acquire_ctx = &ctx; + crtc_state = drm_atomic_get_crtc_state(state, crtc); + if (IS_ERR(crtc_state)) { + ret = PTR_ERR(crtc_state); + goto fail; + } + + crtc_state->color_mgmt_changed |= true; + + ret = drm_atomic_commit(state); + +fail: + drm_atomic_state_put(state); +done: + drm_modeset_drop_locks(&ctx); + drm_modeset_acquire_fini(&ctx); + + return ret; +} + +/* + * H17P (T8142 H17G): brightness-only commits re-present the scanout and back + * off within the call. + */ +static int drm_crtc_set_brightness_h17p(struct apple_dcp *dcp) +{ + struct drm_atomic_state *state; + struct drm_crtc_state *crtc_state; + struct drm_modeset_acquire_ctx ctx; + struct drm_crtc *crtc = &dcp->crtc->base; + int ret = 0; + state = drm_atomic_state_alloc(crtc->dev); if (!state) return -ENOMEM; @@ -442,7 +480,6 @@ static int drm_crtc_set_brightness(struct apple_dcp *dcp) } drm_atomic_state_put(state); -done: drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); @@ -474,10 +511,12 @@ int dcp_backlight_update(struct apple_dcp *dcp) * the brightness */ msleep((1001 + 23) / 24); // 42ms for 23.976 fps + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return drm_crtc_set_brightness_h17p(dcp); return drm_crtc_set_brightness(dcp); } -static int dcp_set_brightness(struct backlight_device *bd) +static int dcp_set_brightness_h17p(struct backlight_device *bd) { int ret = 0; struct apple_dcp *dcp = bl_get_data(bd); @@ -514,6 +553,25 @@ static int dcp_set_brightness(struct backlight_device *bd) ret = drm_modeset_lock_single_interruptible(&dcp->crtc->base.mutex); if (ret) return ret; + + dcp->brightness.dac = calculate_dac(dcp, brightness); + dcp->brightness.update = true; + + drm_modeset_unlock(&dcp->crtc->base.mutex); + + return dcp_backlight_update(dcp); +} + +static int dcp_set_brightness(struct backlight_device *bd) +{ + int ret = 0; + struct apple_dcp *dcp = bl_get_data(bd); + struct drm_modeset_acquire_ctx ctx; + int brightness = backlight_get_brightness(bd); + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_set_brightness_h17p(bd); + drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); ret = drm_modeset_lock(&dcp->crtc->base.mutex, &ctx); if (ret == -EDEADLK) { @@ -530,7 +588,6 @@ static int dcp_set_brightness(struct backlight_device *bd) drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); - drm_modeset_unlock(&dcp->crtc->base.mutex); return dcp_backlight_update(dcp); } @@ -548,6 +605,13 @@ static const struct backlight_ops dcp_backlight_ops = { .options = BL_CORE_SUSPENDRESUME, .get_brightness = dcp_get_brightness, .update_status = dcp_set_brightness, +}; + +/* H17P: fbdev blanking follows only this display. */ +static const struct backlight_ops dcp_backlight_ops_h17p = { + .options = BL_CORE_SUSPENDRESUME, + .get_brightness = dcp_get_brightness, + .update_status = dcp_set_brightness, .controls_device = dcp_backlight_controls_device, }; @@ -580,6 +644,8 @@ int dcp_backlight_register(struct apple_dcp *dcp) } bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, dcp, + dcp->fw_compat == DCP_FIRMWARE_H17P ? + &dcp_backlight_ops_h17p : &dcp_backlight_ops, &props); if (IS_ERR(bl_dev)) return PTR_ERR(bl_dev); diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 52d9e1cf3edac4..350a186b8ed037 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -25,6 +25,16 @@ #include "dptx-j700-training.h" +/* + * The 1080p180 rate proof (dcp-180hz.c) belongs to H17-generation + * firmware, whose probe sets up its lock; other firmware has none. + */ +static void dptx_180hz_invalidate(struct apple_dcp *dcp) +{ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + dcp_180hz_invalidate(dcp); +} + struct dcpdptx_connection_cmd { __le32 unk; __le32 target; @@ -235,7 +245,7 @@ int dptxport_release_display(struct apple_epic_service *service) struct dptx_port *dptx = service->cookie; int ret; - dcp_180hz_invalidate(service->ep->dcp); + dptx_180hz_invalidate(service->ep->dcp); dptx->release_calls++; dev_dbg(service->ep->dcp->dev, "DPTX release_display: call #%u this boot caller=%pS\n", @@ -524,7 +534,7 @@ dptxport_call_will_change_link_config(struct apple_epic_service *service) { struct dptx_port *dptx = service->cookie; - dcp_180hz_invalidate(service->ep->dcp); + dptx_180hz_invalidate(service->ep->dcp); dptx->phy_ops.dp.set_lanes = 0; dptx->phy_ops.dp.set_rate = 0; dptx->phy_ops.dp.set_voltages = 0; @@ -744,6 +754,7 @@ dptxport_call_deactivate(struct apple_epic_service *service, struct dptx_port *dptx = service->cookie; struct apple_dcp *dcp = service->ep->dcp; + dptx_180hz_invalidate(dcp); dev_info(dcp->dev, "DPTXPort: DEACTIVATE\n"); if (dcp->dptx_tunnel) dcp_tunnel_dpin_activate(dcp, dptx->unit, false); @@ -762,12 +773,20 @@ dptxport_call_deactivate(struct apple_epic_service *service, static int dptxport_call_link_config(struct apple_epic_service *service, bool lanes, const void *data, size_t size, void *reply, size_t room) { - struct dcp_180hz_token token = dcp_180hz_config_begin(service, lanes); + struct dcp_180hz_token token; struct dptx_port *port = service->cookie; bool success = false; u32 value = 0; int ret; + if (service->ep->dcp->fw_compat != DCP_FIRMWARE_H17P) + return lanes ? + dptxport_call_set_active_lane_count(service, data, size, + reply, room) : + dptxport_call_set_link_rate(service, data, size, reply, + room); + + token = dcp_180hz_config_begin(service, lanes); if (lanes) { ret = dptxport_call_set_active_lane_count(service, data, size, reply, room); if (!ret && size >= sizeof(struct dptxport_apcall_set_active_lane_count) && @@ -864,6 +883,7 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_DEVICE_NOT_RESPONDING: case DPTX_APCALL_DEVICE_BUSY_TIMEOUT: case DPTX_APCALL_DEVICE_NOT_STARTED: + dptx_180hz_invalidate(service->ep->dcp); dev_warn(service->ep->dcp->dev, "DPTXPort: firmware reports link fault %u on target %u:%u\n", idx, service->ep->dcp->dptx_die, @@ -937,35 +957,31 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name struct apple_dcp *dcp = service->ep->dcp; int port = dptx_service_port(name, class, unit, service->ep->wire_v2, dcp->index, dcp->hw.num_dptx_ports); - s64 port = unit; + + if (port < 0) + return port; + trace_dptxport_init(dcp, unit); + return dptx_service_bind(service, &dcp->dptxport[port], unit); +} + +/* M1/M2/M3 (12.3, 13.5 and 14.7 firmware): integration's port binding. */ +static void dptxport_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ bool v14_7 = dcp_fw_is_14_7(service->ep->dcp); if (!name) return; if (strcmp(name, "dcpdptx-port-epic") && (!v14_7 || strcmp(name, "dcp-lpdptx-port-epic"))) - if (!name || !class) - return; - if (strcmp(name, "dcpdptx-port-epic")) return; /* 14.7 may announce the port without its class. */ if (v14_7 ? class && class[0] && strcmp(class, "AppleDCPDPTXRemotePort") : !class || strcmp(class, "AppleDCPDPTXRemotePort")) return; - if (port < 0) - return port; - trace_dptxport_init(dcp, unit); - return dptx_service_bind(service, &dcp->dptxport[port], unit); -} - -static void dptxport_init(struct apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ - struct dptx_port *port = service->cookie; + trace_dptxport_init(service->ep->dcp, unit); - dptx_service_publish(service); - complete(&port->enable_completion); switch (unit) { case 0: case 1: @@ -988,6 +1004,16 @@ static void dptxport_init(struct apple_epic_service *service, const char *name, dev_err(service->ep->dcp->dev, "DPTXPort: invalid unit %lld\n", unit); } +} + +/* H17-generation firmware: dptxport_prepare() bound the port before OPEN. */ +static void dptxport_h17_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct dptx_port *port = service->cookie; + + dptx_service_publish(service); + complete(&port->enable_completion); /* Publication may follow the startup wait or the bounded retry window. */ if (service->ep->dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && @@ -1088,12 +1114,23 @@ static void dptxport_teardown(struct apple_epic_service *service) dptx_service_unbind(service); } +/* 12.3 and 13.5 firmware. */ static const struct apple_epic_service_ops dptxep_ops[] = { + { + .name = "AppleDCPDPTXRemotePort", + .init = dptxport_init, + .call = dptxport_call, + }, + {} +}; + +/* H17-generation firmware (T8140 H17P, T8142 H17G). */ +static const struct apple_epic_service_ops dptxep_h17_ops[] = { { .name = "AppleDCPDPTXRemotePort", .prepare = dptxport_prepare, .teardown = dptxport_teardown, - .init = dptxport_init, + .init = dptxport_h17_init, .call = dptxport_call, .call_v2 = dptxport_group_call, }, @@ -1114,6 +1151,26 @@ static const struct apple_epic_service_ops dptx_controller_ops[] = { {} }; +/* 14.7 also announces the port under its EPIC names. */ +static const struct apple_epic_service_ops dptxep_v14_7_ops[] = { + { + .name = "AppleDCPDPTXRemotePort", + .init = dptxport_init, + .call = dptxport_call, + }, + { + .name = "dcpdptx-port-epic", + .init = dptxport_init, + .call = dptxport_call, + }, + { + .name = "dcp-lpdptx-port-epic", + .init = dptxport_init, + .call = dptxport_call, + }, + {} +}; + int dptx_controller_init(struct apple_dcp *dcp) { struct apple_dcp_afkep *ep; @@ -1137,26 +1194,6 @@ int dptx_controller_init(struct apple_dcp *dcp) return 0; } -/* 14.7 also announces the port under its EPIC names. */ -static const struct apple_epic_service_ops dptxep_v14_7_ops[] = { - { - .name = "AppleDCPDPTXRemotePort", - .init = dptxport_init, - .call = dptxport_call, - }, - { - .name = "dcpdptx-port-epic", - .init = dptxport_init, - .call = dptxport_call, - }, - { - .name = "dcp-lpdptx-port-epic", - .init = dptxport_init, - .call = dptxport_call, - .call_v2 = dptxport_group_call, - }, - {} -}; int dptxep_init(struct apple_dcp *dcp) { @@ -1172,9 +1209,18 @@ int dptxep_init(struct apple_dcp *dcp) init_completion(&dcp->dptxport[0].linkcfg_completion); init_completion(&dcp->dptxport[1].linkcfg_completion); - ep = afk_init(dcp, wire_v2 ? DPTX_ENDPOINT_V2 : DPTX_ENDPOINT, dptxep_ops); - if (IS_ERR(ep)) + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + ep = afk_init(dcp, wire_v2 ? DPTX_ENDPOINT_V2 : DPTX_ENDPOINT, + dptxep_h17_ops); + else + ep = afk_init(dcp, DPTX_ENDPOINT, + dcp_fw_is_14_7(dcp) ? dptxep_v14_7_ops : dptxep_ops); + if (IS_ERR(ep)) { + /* As integration: other firmware keeps the error pointer. */ + if (dcp->fw_compat != DCP_FIRMWARE_H17P) + dcp->dptxep = ep; return PTR_ERR(ep); + } ep->wire_v2 = wire_v2; dcp->dptxep = ep; @@ -1197,6 +1243,9 @@ int dptxep_init(struct apple_dcp *dcp) "DPTX port %u not announced yet\n", port); return 0; } + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + dev_warn(dcp->dev, "DPTX transport ready but port %u service not announced\n", + port); return -ETIMEDOUT; } else if (ret < 0) { return ret; diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index 21b0ee790369b3..527b88160e32e6 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -103,8 +103,8 @@ static int dcpavserv_revalidate(struct apple_epic_service *service, const char * return ret; } -static void dcpavserv_init(struct apple_epic_service *service, const char *name, - const char *class, s64 unit) +static void dcpavserv_init_h17p(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) { struct apple_dcp *dcp = service->ep->dcp; bool v2 = service->ep->wire_v2; @@ -118,17 +118,6 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, !dcp->dcpavserv.enabled && service->admission.epoch == dcp->dcpavserv.epoch && (!v2 || dcpavserv_admission_current(dcp, service->admission.generation))) { - - if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { - spin_lock_irqsave(&dcp->dcpavserv.lock, flags); - if (!dcpavserv_prepare(service, name, class, unit)) { - mutex_lock(&dcp->dcpavserv.lock); - if (dcp->dcpavserv.enabled) { - mutex_unlock(&dcp->dcpavserv.lock); - dev_err(dcp->dev, - "DCPAVSERV: unit %lld already exists\n", unit); - return; - } dcp->dcpavserv.service = afk_service_get(service); if (dcp->dcpavserv.service) { dcp->dcpavserv.enabled = true; @@ -167,7 +156,7 @@ static void dcpavserv_invalidate_edid(struct apple_dcp *dcp) schedule_work(&connector->hotplug_wq); } -static void dcpavserv_teardown(struct apple_epic_service *service) +static void dcpavserv_teardown_h17p(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; bool owned = false; @@ -178,6 +167,51 @@ static void dcpavserv_teardown(struct apple_epic_service *service) dcp->dcpavserv.service = NULL; dcp->dcpavserv.epoch++; dcpavserv_invalidate_edid(dcp); + reinit_completion(&dcp->dcpavserv.enable_completion); + owned = true; + } + mutex_unlock(&dcp->dcpavserv.lock); + if (owned) + afk_service_put(service); +} + +/* + * Every firmware other than H17P (M1/M2/M3): integration's service + * lifetime. The H17P variants below add admission, service epochs and + * connector EDID invalidation. + */ +static void dcpavserv_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ + struct apple_dcp *dcp = service->ep->dcp; + + trace_dcpavserv_init(dcp, unit); + + if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.enabled) { + mutex_unlock(&dcp->dcpavserv.lock); + dev_err(dcp->dev, + "DCPAVSERV: unit %lld already exists\n", unit); + return; + } + dcp->dcpavserv.service = afk_service_get(service); + dcp->dcpavserv.enabled = true; + service->cookie = &dcp->dcpavserv; + complete(&dcp->dcpavserv.enable_completion); + mutex_unlock(&dcp->dcpavserv.lock); + } +} + +static void dcpavserv_teardown(struct apple_epic_service *service) +{ + struct apple_dcp *dcp = service->ep->dcp; + bool owned = false; + + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.service == service) { + dcp->dcpavserv.enabled = false; + dcp->dcpavserv.service = NULL; dcp->dcpavserv.opened = NULL; reinit_completion(&dcp->dcpavserv.enable_completion); owned = true; @@ -191,6 +225,20 @@ void dpavservep_detach(struct apple_dcp *dcp) { struct apple_epic_service *service; + if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + mutex_lock(&dcp->dcpavserv.lock); + service = dcp->dcpavserv.service; + dcp->dcpavserv.service = NULL; + dcp->dcpavserv.enabled = false; + dcp->dcpavserv.opened = NULL; + mutex_unlock(&dcp->dcpavserv.lock); + if (service) { + afk_service_disable(service); + afk_service_put(service); + } + return; + } + mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.service; dcp->dcpavserv.service = NULL; @@ -311,7 +359,8 @@ static int dcpdpserv_report(struct apple_epic_service *service, return parse_report(service, type, data, data_size); } -static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) +/* Integration's bounded read, for every firmware other than H17P. */ +static const struct drm_edid *dcpavserv_read_edid(struct apple_epic_service *service) { struct dpavserv_copy_edid_cmd cmd; struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; @@ -337,6 +386,33 @@ static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_servi return drm_edid_alloc(resp->data + DPAVSERV_EDID_LEADING_SIZE, edid_size); } + +static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_service *service) +{ + struct dpavserv_copy_edid_cmd cmd; + struct dpavserv_copy_edid_resp *resp __free(kfree) = NULL; + size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; + int num_blocks; + u64 data_size; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); + resp = kzalloc(resp_size, GFP_KERNEL); + if (!resp) + return ERR_PTR(-ENOMEM); + + ret = afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, + resp_size, 0); + if (ret < 0) + return ERR_PTR(ret); + + if (le64_to_cpu(resp->max_size) != EDID_BUF_SIZE) + return ERR_PTR(-EIO); + + // print_hex_dump(KERN_DEBUG, "dpavserv EDID cmd: ", DUMP_PREFIX_NONE, + // 16, 1, resp, 192, true); + data_size = le64_to_cpu(resp->used_size); if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || data_size > EDID_BUF_SIZE) @@ -408,6 +484,49 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation struct apple_epic_service *service; const struct drm_edid *edid; + if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + mutex_lock(&dcp->dcpavserv.lock); + service = afk_service_get(dcp->dcpavserv.service); + *epoch = dcp->dcpavserv.epoch; + mutex_unlock(&dcp->dcpavserv.lock); + if (!service) + return ERR_PTR(-ENODEV); + + /* + * The 14.7 firmware of an external processor answers EDID + * requests once its service has been opened; open each + * announced instance once. + * + * Only on Aurora's DCPLink session (T6030). The T8122 dcpext, + * which Linux starts and drives through the IOMFB template, + * reads the EDID directly, as in AsahiLinux/linux#607. Its + * firmware never answers the open call and drops the display + * link instead, about 50 ms after it, which brings the display + * back 4 s later: hotplug in a loop. + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && + READ_ONCE(dcp->dcpavserv.opened) != service) { + int ret = afk_service_call(service, 4, 6, NULL, 0, 32, NULL, 0, 32); + + if (ret) { + afk_service_put(service); + return ERR_PTR(ret); + } + WRITE_ONCE(dcp->dcpavserv.opened, service); + } + + edid = dcpavserv_read_edid(service); + mutex_lock(&dcp->dcpavserv.lock); + if (dcp->dcpavserv.service != service) { + if (!IS_ERR(edid)) + drm_edid_free(edid); + edid = ERR_PTR(-ENODEV); + } + mutex_unlock(&dcp->dcpavserv.lock); + afk_service_put(service); + return edid; + } + scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) return ERR_PTR(-ESTALE); @@ -419,40 +538,6 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation if (!service) return ERR_PTR(-ENODEV); - /* - * The 14.7 firmware of an external processor answers EDID requests - * once its service has been opened; open each announced instance once. - * - * Only on Aurora's DCPLink session (T6030). The T8122 dcpext, which - * Linux starts and drives through the IOMFB template, reads the EDID - * directly, as in AsahiLinux/linux#607. Its firmware never answers the - * open call and drops the display link instead, about 50 ms after it, - * which brings the display back 4 s later: hotplug in a loop. - */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && - READ_ONCE(dcp->dcpavserv.opened) != service) { - int ret = afk_service_call(service, 4, 6, NULL, 0, 32, NULL, 0, 32); - - if (ret) { - afk_service_put(service); - return ERR_PTR(ret); - } - WRITE_ONCE(dcp->dcpavserv.opened, service); - } - /* - * An HDMI 2.x sink using the HF-EEODB announces fewer extension blocks - * in the base block than it sends, so the payload may be longer than - * the base block says. Accept any whole number of blocks at least that - * long and leave the block count to drm_edid, which knows about - * HF-EEODB. - */ - num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; - if ((1 + num_blocks) * EDID_BLOCK_SIZE > data_size - EDID_LEADING_DATA_SIZE || - (data_size - EDID_LEADING_DATA_SIZE) % EDID_BLOCK_SIZE) - return ERR_PTR(-EIO); - - edid = dcpavserv_read_edid(service); - edid = service->ep->inline_v2 ? dcpavserv_read_edid_inline(service) : edid = service->ep->wire_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); mutex_lock(&dcp->dcpavserv.lock); @@ -478,12 +563,31 @@ static const struct apple_epic_service_ops dpavservep_ops[] = { { .name = "dcpav-service-epic", .reusable = true, - .prepare = dcpavserv_prepare, - .revalidate = dcpavserv_revalidate, .init = dcpavserv_init, .teardown = dcpavserv_teardown, .report = dcpavserv_report, }, + { + .name = "dcpdp-service-epic", + .reusable = true, + .init = dcpdpserv_init, + .teardown = dcpdpserv_teardown, + .report = dcpdpserv_report, + }, + {}, +}; + +/* H17P (T8140, T8142): admission before OPEN, and the optional DP service. */ +static const struct apple_epic_service_ops dpavservep_h17p_ops[] = { + { + .name = "dcpav-service-epic", + .reusable = true, + .prepare = dcpavserv_prepare, + .revalidate = dcpavserv_revalidate, + .init = dcpavserv_init_h17p, + .teardown = dcpavserv_teardown_h17p, + .report = dcpavserv_report, + }, { .name = "dcpdp-service-epic", .reusable = true, @@ -505,7 +609,9 @@ int dpavservep_init(struct apple_dcp *dcp) ep = afk_init(dcp, dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL ? - DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, dpavservep_ops); + DPAVSERV_ENDPOINT_V2 : DPAVSERV_ENDPOINT, + dcp->fw_compat == DCP_FIRMWARE_H17P ? + dpavservep_h17p_ops : dpavservep_ops); if (IS_ERR(ep)) return PTR_ERR(ep); /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ @@ -530,6 +636,9 @@ int dpavservep_init(struct apple_dcp *dcp) ret = wait_for_completion_timeout(&dcp->dcpavserv.enable_completion, msecs_to_jiffies(1000)); + /* Integration: a missing announcement is not an error elsewhere. */ + if (dcp->fw_compat != DCP_FIRMWARE_H17P) + return 0; if (!ret) { dev_warn(dcp->dev, "DCPAVSERV: timed out waiting for EDID service announcement\n"); return -ETIMEDOUT; diff --git a/drivers/gpu/drm/apple/external-hold-test.c b/drivers/gpu/drm/apple/external-hold-test.c index 1830c1846d06bd..88a4f8b742d1a6 100644 --- a/drivers/gpu/drm/apple/external-hold-test.c +++ b/drivers/gpu/drm/apple/external-hold-test.c @@ -151,7 +151,7 @@ static int hold_test_init(struct kunit *test) mutex_init(&dcp->hpd_mutex); mutex_init(&dcp->tb_lock); spin_lock_init(&dcp->mode_state.lock); - spin_lock_init(&dcp->dcpavserv.lock); + mutex_init(&dcp->dcpavserv.lock); init_completion(&dcp->typec_iomfb_hpd_ready); INIT_DELAYED_WORK(&dcp->external_retry_wq, dcp_external_retry_work); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, hold_relink_work); diff --git a/drivers/gpu/drm/apple/ibootep.c b/drivers/gpu/drm/apple/ibootep.c index 630110bc8cd555..d757ddc47edc74 100644 --- a/drivers/gpu/drm/apple/ibootep.c +++ b/drivers/gpu/drm/apple/ibootep.c @@ -744,8 +744,19 @@ static const struct apple_epic_service_ops external_ibootep_ops[] = { int ibootep_init(struct apple_dcp *dcp) { + const struct apple_epic_service_ops *ops = ibootep_ops; int ret; + /* H17P firmware (T8140, T8142): aurora-wip's service ops and errors. */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) + ops = external_ibootep_ops; + dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); + if (IS_ERR(dcp->ibootep)) + return PTR_ERR(dcp->ibootep); + return afk_start(dcp->ibootep); + } + if (!of_machine_is_compatible("apple,t6030") && !of_machine_is_compatible("apple,t6031") && !of_machine_is_compatible("apple,t6032") && @@ -764,13 +775,6 @@ int ibootep_init(struct apple_dcp *dcp) dcp->ibootep = NULL; return ret; } - const struct apple_epic_service_ops *ops = ibootep_ops; - - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) - ops = external_ibootep_ops; - dcp->ibootep = afk_init(dcp, DISP0_ENDPOINT, ops); - if (IS_ERR(dcp->ibootep)) - return PTR_ERR(dcp->ibootep); return afk_start(dcp->ibootep); } diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index b6bc2bb76c027d..e0aa8b2e8d29c5 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -616,6 +616,7 @@ static int iomfb_queue_command(struct apple_dcp *dcp, bool oob, return ret; } + /* Call a DCP function given by a tag */ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, u32 in_len, u32 out_len, void *data, dcp_callback_t cb, @@ -677,8 +678,11 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca if (in_len > 0) memcpy(out_data, data, in_len); - /* An unwritten status must not look like a successful response. */ - if (out_len) + /* + * H17P: an unwritten status must not look like a successful + * response. + */ + if (out_len && dcp->fw_compat == DCP_FIRMWARE_H17P) memset(out_data + in_len, 0xff, out_len); trace_iomfb_push(dcp, call, context, offset, depth); @@ -822,12 +826,34 @@ void dcp_route_failure_notify(struct apple_connector *connector) schedule_work(&connector->hotplug_wq); } + +/* Does @connector belong to a DRM device of H17-generation pipelines? */ +static bool dcp_connector_h17p(struct apple_connector *connector) +{ + struct platform_device *pdev = READ_ONCE(connector->dcp); + struct drm_crtc *crtc; + + if (pdev) + return ((struct apple_dcp *)platform_get_drvdata(pdev))->fw_compat == + DCP_FIRMWARE_H17P; + drm_for_each_crtc(crtc, connector->base.dev) { + struct apple_dcp *dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + if (dcp && dcp->fw_compat == DCP_FIRMWARE_H17P) + return true; + } + return false; +} + +/* + * The cached EDID itself is owned under edid_lock, which integration's + * route and sink changes take to drop it (apple_connector_invalidate_edid()). + */ static void dcp_clear_connector_edid(struct apple_connector *connector) { lockdep_assert_held(&connector->base.dev->mode_config.mutex); drm_edid_connector_update(&connector->base, NULL); - drm_edid_free(connector->drm_edid); - connector->drm_edid = NULL; + apple_connector_invalidate_edid(connector); connector->drm_edid_firmware_derived = false; } @@ -839,21 +865,15 @@ static void dcp_consume_edid_invalidation(struct apple_connector *connector) dcp_clear_connector_edid(connector); } -void dcp_hotplug(struct work_struct *work) +/* H17P (T8140, T8142): aurora-wip's hotplug work. */ +static void dcp_hotplug_h17p(struct apple_connector *connector) { - struct apple_connector *connector; struct platform_device *pdev; struct apple_dcp *dcp; - bool notify_only; - unsigned int reasons; int ret; - connector = container_of(work, struct apple_connector, hotplug_wq); scoped_guard(mutex, &connector->base.dev->mode_config.mutex) dcp_consume_edid_invalidation(connector); - reasons = atomic_xchg(&connector->hotplug_reasons, 0); - notify_only = (reasons & DCP_HOTPLUG_ROUTE_FAILURE) && - !(reasons & DCP_HOTPLUG_READY); pdev = READ_ONCE(connector->dcp); if (!pdev) { /* a Type-C port unrouted after this was queued */ @@ -864,9 +884,6 @@ void dcp_hotplug(struct work_struct *work) } } drm_kms_helper_connector_hotplug_event(&connector->base); - apple_connector_invalidate_edid(connector); - dcp_notify_hotplug(connector); - apple_connector_backlight_sync(connector); return; } dcp = platform_get_drvdata(pdev); @@ -879,6 +896,62 @@ void dcp_hotplug(struct work_struct *work) if (!READ_ONCE(connector->connected)) dcp_clear_connector_edid(connector); } + + /* + * DCP defers link training until we set a display mode. But we set + * display modes from atomic_flush, so userspace needs to trigger a + * flush, or the CRTC gets no signal. + */ + if (connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected) { + drm_connector_set_link_status_property(&connector->base, + DRM_MODE_LINK_STATUS_BAD); + + /* + * A short Type-C route interruption can leave the DRM CRTC active + * while DCP has discarded its display mode. Userspace is then free + * to keep submitting plane-only commits, none of which retrains the + * link. Re-apply the active CRTC state once the DPTX link-config + * callback has completed. + */ + ret = dcp_retrain_active_crtc(connector); + if (ret) + dev_warn(dcp->dev, + "failed to retrain active CRTC after hotplug: %d\n", + ret); + } + + drm_kms_helper_connector_hotplug_event(&connector->base); +} + +void dcp_hotplug(struct work_struct *work) +{ + struct apple_connector *connector; + struct platform_device *pdev; + struct apple_dcp *dcp; + bool notify_only; + unsigned int reasons; + int ret; + + connector = container_of(work, struct apple_connector, hotplug_wq); + if (dcp_connector_h17p(connector)) { + dcp_hotplug_h17p(connector); + return; + } + reasons = atomic_xchg(&connector->hotplug_reasons, 0); + notify_only = (reasons & DCP_HOTPLUG_ROUTE_FAILURE) && + !(reasons & DCP_HOTPLUG_READY); + + pdev = READ_ONCE(connector->dcp); + if (!pdev) { /* a Type-C port unrouted after this was queued */ + apple_connector_invalidate_edid(connector); + dcp_notify_hotplug(connector); + apple_connector_backlight_sync(connector); + return; + } + dcp = platform_get_drvdata(pdev); + dev_info(dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, + connector->connected, READ_ONCE(dcp->mode_state.valid), dcp->nr_modes); + if (!connector->connected) apple_connector_invalidate_edid(connector); @@ -929,8 +1002,8 @@ static bool dcp_external_zero_reply(struct apple_dcp *dcp, int tag, of_device_is_compatible(dcp->dev->of_node, "apple,t8140-dcpext"); } -static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length, u16 offset) +static void dcpep_handle_cb_h17p(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length, u16 offset) { struct device *dev = dcp->dev; struct dcp_packet_header *hdr = data; @@ -949,13 +1022,7 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, } tag = dcp_parse_tag(hdr->tag); - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL_EXPERIMENTAL && - dcp->experimental_callback_logs < 64) { - dcp->experimental_callback_logs++; - dev_info(dev, "external IOMFB callback=%d in=%u out=%u context=%u\n", - tag, hdr->in_len, hdr->out_len, context); - } + handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && dcp->cb_handlers[tag]; @@ -1020,10 +1087,9 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, dcp_ack(dcp, context); } -static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, - void *data, u32 length) +/* H17P: the reply is written in place into the record dcp_push() saved. */ +static void dcpep_handle_ack_h17p(struct apple_dcp *dcp, enum dcp_context_id context) { - struct dcp_packet_header *header = data; struct dcp_channel *ch = dcp_get_channel(dcp, context); const struct dcp_packet_header *record, *sent; void *cookie, *out; @@ -1085,6 +1151,63 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, cb(dcp, out, cookie); } +static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length, u16 offset) +{ + struct device *dev = dcp->dev; + struct dcp_packet_header *hdr = data; + void *in, *out; + int tag = dcp_parse_tag(hdr->tag); + struct dcp_channel *ch = dcp_get_channel(dcp, context); + u8 depth; + + if (tag < 0 || tag >= IOMFB_MAX_CB || !dcp->cb_handlers || !dcp->cb_handlers[tag]) { + dev_warn(dev, "received unknown callback %c%c%c%c\n", + hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); + return; + } + + in = data + sizeof(*hdr); + out = in + hdr->in_len; + + // TODO: verify that in_len and out_len match our prototypes + // for now just clear the out data to have at least consistent results + if (hdr->out_len) + memset(out, 0, hdr->out_len); + + depth = dcp_push_depth(&ch->depth); + ch->output[depth] = out; + ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); + + if (dcp->cb_handlers[tag](dcp, tag, out, in)) + dcp_ack(dcp, context); +} + +static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context, + void *data, u32 length) +{ + struct dcp_packet_header *header = data; + struct dcp_channel *ch = dcp_get_channel(dcp, context); + void *cookie; + dcp_callback_t cb; + + if (!ch) { + dev_warn(dcp->dev, "ignoring ack on context %X\n", context); + return; + } + + dcp_pop_depth(&ch->depth); + + cb = ch->callbacks[ch->depth]; + cookie = ch->cookies[ch->depth]; + + ch->callbacks[ch->depth] = NULL; + ch->cookies[ch->depth] = NULL; + + if (cb) + cb(dcp, data + sizeof(*header) + header->in_len, cookie); +} + static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) { enum dcp_context_id ctx_id; @@ -1106,6 +1229,14 @@ static void dcpep_got_msg(struct apple_dcp *dcp, u64 message) data = dcp->shmem + channel_offset + offset; + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (FIELD_GET(IOMFB_MSG_ACK, message)) + dcpep_handle_ack_h17p(dcp, ctx_id); + else + dcpep_handle_cb_h17p(dcp, ctx_id, data, length, offset); + return; + } + if (FIELD_GET(IOMFB_MSG_ACK, message)) dcpep_handle_ack(dcp, ctx_id, data, length); else @@ -1290,6 +1421,9 @@ void dcp_modes_release(void *data) kfree(dcp->modes); dcp->modes = NULL; dcp->nr_modes = 0; +} + + /* Retained arrays are not freed while mode readers can still refer to them. */ static struct dcp_display_mode *dcp_modes_snapshot(struct apple_dcp *dcp, unsigned int *count) @@ -1339,7 +1473,8 @@ static bool dcp_publish_mode(struct apple_dcp *dcp, struct drm_connector *connec return eligible; } -int dcp_get_modes(struct drm_connector *connector) +/* H17P (T8140, T8142): aurora-wip's mode and EDID publication. */ +static int dcp_get_modes_h17p(struct drm_connector *connector) { struct apple_connector *apple_connector = to_apple_connector(connector); struct platform_device *pdev = READ_ONCE(apple_connector->dcp); @@ -1355,9 +1490,8 @@ int dcp_get_modes(struct drm_connector *connector) bool pending_180hz = false, published_180hz = false; u64 edid_generation = 0; u64 service_epoch = 0; + const struct drm_edid *placeholder; unsigned int i; - unsigned int count; - int i; dcp_consume_edid_invalidation(apple_connector); @@ -1367,38 +1501,11 @@ int dcp_get_modes(struct drm_connector *connector) drm_edid_connector_update(connector, NULL); apple_connector->drm_edid_firmware_derived = false; } - drm_edid_connector_update(connector, NULL); return 0; } dcp = platform_get_drvdata(pdev); modes = dcp_modes_snapshot(dcp, &nr_modes); - mutex_lock(&dcp->modes_lock); - count = dcp_modes_for_connector(dcp, apple_connector) ? dcp->nr_modes : 0; - for (i = 0; i < count; ++i) { - if (dcp->modes[i].vrr) { - u16 lo = dcp->modes[i].min_vrr >> 16; - u16 hi = dcp->modes[i].max_vrr >> 16; - - if (!min_vfreq || lo < min_vfreq) - min_vfreq = lo; - if (hi > max_vfreq) - max_vfreq = hi; - } - vrr_capable |= dcp->modes[i].vrr; - mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); - - if (!mode) { - dev_err(dev->dev, "Failed to duplicate display mode\n"); - mutex_unlock(&dcp->modes_lock); - return 0; - } - - drm_mode_probed_add(connector, mode); - } - mutex_unlock(&dcp->modes_lock); - drm_connector_set_vrr_capable_property(connector, vrr_capable); - physical_edid = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; /* get_modes holds mode_config.mutex; HPD paths never acquire it. */ @@ -1424,8 +1531,6 @@ int dcp_get_modes(struct drm_connector *connector) if (nr_modes && !apple_connector->drm_edid && (dcp->fw_compat != DCP_FIRMWARE_H17P || (dcp->dcpavservep && dcp->dcpavservep->wire_v2))) { - (dcp->dcpavservep && dcp->dcpavservep->inline_v2))) { - if (count) { const struct drm_edid *edid; /* Let physical disconnect advance the generation during the request. */ mutex_unlock(&dcp->dcpavserv.lock); @@ -1463,24 +1568,22 @@ int dcp_get_modes(struct drm_connector *connector) else dev_warn(dcp->dev, "copy_edid failed: %pe\n", edid); } else { - drm_edid_free(apple_connector->drm_edid); - apple_connector->drm_edid = edid; + scoped_guard(mutex, &apple_connector->edid_lock) { + drm_edid_free(apple_connector->drm_edid); + apple_connector->drm_edid = edid; + } apple_connector->drm_edid_generation = edid_generation; apple_connector->drm_edid_service_epoch = service_epoch; - u64 generation; - - /* The RPC can sleep while a disconnect or route change clears EDID. */ - if (apple_connector_edid_begin(apple_connector, pdev, &generation)) { - edid = dcpavserv_copy_edid(dcp); - if (IS_ERR_OR_NULL(edid)) - dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); - else - apple_connector_edid_install(apple_connector, pdev, - generation, edid); } } if (nr_modes && apple_connector->drm_edid) { - int update = drm_edid_connector_update(connector, apple_connector->drm_edid); + int update; + + /* Integration's route changes drop the cached EDID under edid_lock. */ + scoped_guard(mutex, &apple_connector->edid_lock) + update = apple_connector->drm_edid ? + drm_edid_connector_update(connector, apple_connector->drm_edid) : + -ENODATA; if (update) { dev_warn(dcp->dev, "raw EDID publication failed: %d\n", update); @@ -1513,18 +1616,11 @@ int dcp_get_modes(struct drm_connector *connector) drm_edid_connector_update(connector, NULL); apple_connector->drm_edid_firmware_derived = false; } - - edid = apple_connector_edid_dup(apple_connector, pdev); - drm_edid_connector_update(connector, edid); - if (edid) { - dcp_retry_placeholder_edid(dcp, edid); - drm_edid_free(edid); - } - } else { - drm_edid_connector_update(connector, NULL); } edid_done: + /* The 180 Hz exception checks below read the cached EDID. */ + mutex_lock(&apple_connector->edid_lock); /* Publish after a possibly delayed raw read, under its physical generation. */ for (i = 0; i < nr_modes; ++i) { mode = drm_mode_duplicate(dev, &modes[i].mode); @@ -1552,13 +1648,19 @@ int dcp_get_modes(struct drm_connector *connector) } drm_connector_set_vrr_capable_property(connector, vrr_capable); modes_done: + mutex_unlock(&apple_connector->edid_lock); mutex_unlock(&dcp->dcpavserv.lock); if (physical_edid) mutex_unlock(&dcp->hpd_mutex); if (pending_180hz && !published_180hz) dcp_180hz_retry(dcp); - if (apple_connector->drm_edid) - dcp_retry_placeholder_edid(dcp, apple_connector->drm_edid); + scoped_guard(mutex, &apple_connector->edid_lock) + placeholder = apple_connector->drm_edid ? + drm_edid_dup(apple_connector->drm_edid) : NULL; + if (placeholder) { + dcp_retry_placeholder_edid(dcp, placeholder); + drm_edid_free(placeholder); + } /* * An internal panel has no EDID, so nothing fills in the refresh range @@ -1573,31 +1675,18 @@ int dcp_get_modes(struct drm_connector *connector) } return published; - return count; } /* The user may own drm_display_mode, so we need to search for our copy */ -static bool lookup_mode_locked(struct apple_dcp *dcp, - const struct drm_display_mode *mode, - struct dcp_display_mode *out) +struct dcp_display_mode *lookup_mode_h17p(struct apple_dcp *dcp, + const struct drm_display_mode *mode) { + struct apple_connector *con; struct dcp_display_mode *modes; unsigned int count, i; unsigned long flags; bool eligible; - bool found = false; - int i; - lockdep_assert_held(&dcp->modes_lock); - if (!dcp->modes_admitted) - goto out_unlock; - for (i = 0; i < dcp->nr_modes; ++i) { - if (drm_mode_match(mode, &dcp->modes[i].mode, - DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_CLOCK)) { - if (out) - *out = dcp->modes[i]; - found = true; - break; modes = dcp_modes_snapshot(dcp, &count); for (i = 0; i < count; ++i) { if (drm_mode_match(mode, &modes[i].mode, @@ -1607,24 +1696,23 @@ static bool lookup_mode_locked(struct apple_dcp *dcp, return &modes[i]; mutex_lock(&dcp->hpd_mutex); mutex_lock(&dcp->dcpavserv.lock); + con = READ_ONCE(dcp->connector); + /* The exception check reads the cached EDID. */ + if (con) + mutex_lock(&con->edid_lock); spin_lock_irqsave(&dcp->metadata.lock, flags); eligible = dcp_exception_mode_current_locked(dcp, &modes[i]); spin_unlock_irqrestore(&dcp->metadata.lock, flags); + if (con) + mutex_unlock(&con->edid_lock); mutex_unlock(&dcp->dcpavserv.lock); mutex_unlock(&dcp->hpd_mutex); if (eligible) return &modes[i]; } } -out_unlock: - return found; -} -bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, - struct dcp_display_mode *out) -{ - guard(mutex)(&dcp->modes_lock); - return lookup_mode_locked(dcp, mode, out); + return NULL; } /* @@ -1637,7 +1725,7 @@ bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, static bool dcp_mode_settable(struct apple_dcp *dcp, const struct drm_display_mode *mode) { - struct dcp_display_mode *dcp_mode = lookup_mode(dcp, mode); + struct dcp_display_mode *dcp_mode = lookup_mode_h17p(dcp, mode); if (!dcp_mode) return false; @@ -1650,6 +1738,133 @@ static bool dcp_mode_settable(struct apple_dcp *dcp, return true; } +int dcp_get_modes(struct drm_connector *connector) +{ + struct apple_connector *apple_connector = to_apple_connector(connector); + struct platform_device *pdev = READ_ONCE(apple_connector->dcp); + struct apple_dcp *dcp; + + struct drm_device *dev = connector->dev; + struct drm_display_mode *mode; + u16 min_vfreq = 0, max_vfreq = 0; + bool vrr_capable = false; + unsigned int count; + int i; + + if (dcp_connector_h17p(apple_connector)) + return dcp_get_modes_h17p(connector); + + /* A Type-C port has no pipeline while the fabric is moving it. */ + if (!pdev) { + drm_edid_connector_update(connector, NULL); + return 0; + } + dcp = platform_get_drvdata(pdev); + + mutex_lock(&dcp->modes_lock); + count = dcp_modes_for_connector(dcp, apple_connector) ? dcp->nr_modes : 0; + for (i = 0; i < count; ++i) { + if (dcp->modes[i].vrr) { + u16 lo = dcp->modes[i].min_vrr >> 16; + u16 hi = dcp->modes[i].max_vrr >> 16; + + if (!min_vfreq || lo < min_vfreq) + min_vfreq = lo; + if (hi > max_vfreq) + max_vfreq = hi; + } + vrr_capable |= dcp->modes[i].vrr; + mode = drm_mode_duplicate(dev, &dcp->modes[i].mode); + + if (!mode) { + dev_err(dev->dev, "Failed to duplicate display mode\n"); + mutex_unlock(&dcp->modes_lock); + return 0; + } + + drm_mode_probed_add(connector, mode); + } + mutex_unlock(&dcp->modes_lock); + drm_connector_set_vrr_capable_property(connector, vrr_capable); + + if (count) { + const struct drm_edid *edid; + u64 generation, epoch; + + /* The RPC can sleep while a disconnect or route change clears EDID. */ + if (apple_connector_edid_begin(apple_connector, pdev, &generation)) { + edid = dcpavserv_copy_edid(dcp, 0, &epoch); + if (IS_ERR_OR_NULL(edid)) + dev_info(dcp->dev, "copy_edid failed: %pe\n", edid); + else + apple_connector_edid_install(apple_connector, pdev, + generation, edid); + } + + edid = apple_connector_edid_dup(apple_connector, pdev); + drm_edid_connector_update(connector, edid); + if (edid) { + dcp_retry_placeholder_edid(dcp, edid); + drm_edid_free(edid); + } + } else { + drm_edid_connector_update(connector, NULL); + } + + /* + * An internal panel has no EDID, so nothing fills in the refresh range + * that userspace needs before it will drive VRR. Supply the range DCP + * reported for the mode. This has to follow the EDID update, which + * resets display_info. + */ + if (vrr_capable && max_vfreq && + !connector->display_info.monitor_range.max_vfreq) { + connector->display_info.monitor_range.min_vfreq = min_vfreq; + connector->display_info.monitor_range.max_vfreq = max_vfreq; + } + + return count; +} + +/* The user may own drm_display_mode, so we need to search for our copy */ +static bool lookup_mode_locked(struct apple_dcp *dcp, + const struct drm_display_mode *mode, + struct dcp_display_mode *out) +{ + bool found = false; + int i; + + lockdep_assert_held(&dcp->modes_lock); + if (!dcp->modes_admitted) + goto out_unlock; + for (i = 0; i < dcp->nr_modes; ++i) { + if (drm_mode_match(mode, &dcp->modes[i].mode, + DRM_MODE_MATCH_TIMINGS | DRM_MODE_MATCH_CLOCK)) { + if (out) + *out = dcp->modes[i]; + found = true; + break; + } + } +out_unlock: + return found; +} + +bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, + struct dcp_display_mode *out) +{ + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + struct dcp_display_mode *found = lookup_mode_h17p(dcp, mode); + + if (found && out) + *out = *found; + return found; + } + + guard(mutex)(&dcp->modes_lock); + return lookup_mode_locked(dcp, mode, out); +} + enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, const struct drm_display_mode *mode) { @@ -1661,14 +1876,18 @@ enum drm_mode_status dcp_mode_valid(struct drm_connector *connector, return MODE_ERROR; dcp = platform_get_drvdata(pdev); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_mode_settable(dcp, mode) ? MODE_OK : MODE_BAD; + guard(mutex)(&dcp->modes_lock); if (!dcp_modes_for_connector(dcp, apple_connector)) return MODE_BAD; return lookup_mode_locked(dcp, mode, NULL) ? MODE_OK : MODE_BAD; } -int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, - struct drm_atomic_state *state) +/* H17P (T8140, T8142): aurora-wip's modeset admission. */ +static int dcp_crtc_atomic_modeset_h17p(struct drm_crtc *crtc, + struct drm_atomic_state *state) { struct apple_crtc *apple_crtc = to_apple_crtc(crtc); struct apple_dcp *dcp = platform_get_drvdata(apple_crtc->dcp); @@ -1708,6 +1927,55 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, break; case DCP_FIRMWARE_H17P: ret = iomfb_modeset_h17p(dcp, crtc_state); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", + dcp->fw_compat); + break; + } + + action = dcp_mode_finish(&dcp->mode_state, !ret, dcp->connector ? + &dcp->connector->connected : NULL); + dcp_handle_hotplug_actions(dcp, action); + + return ret; +} + +int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, + struct drm_atomic_state *state) +{ + struct apple_crtc *apple_crtc = to_apple_crtc(crtc); + struct apple_dcp *dcp = platform_get_drvdata(apple_crtc->dcp); + struct drm_crtc_state *crtc_state; + unsigned int action; + int ret = -EIO; + bool modeset; + + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_crtc_atomic_modeset_h17p(crtc, state); + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + if (!crtc_state) + return 0; + + modeset = drm_atomic_crtc_needs_modeset(crtc_state) || + !READ_ONCE(dcp->mode_state.valid); + + if (!modeset) + return 0; + + /* ignore no mode, poweroff is handled elsewhere */ + if (crtc_state->mode.hdisplay == 0 && crtc_state->mode.vdisplay == 0) + return 0; + + dcp_mode_begin(&dcp->mode_state); + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + ret = iomfb_modeset_v12_3(dcp, crtc_state); + break; + case DCP_FIRMWARE_V_13_5: + ret = iomfb_modeset_v13_3(dcp, crtc_state); + break; case DCP_FIRMWARE_V_26_6: ret = iomfb_v26_6_modeset(dcp, crtc_state); break; @@ -1805,6 +2073,9 @@ bool dcp_crtc_mode_fixup(struct drm_crtc *crtc, struct platform_device *pdev = apple_crtc->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return dcp_mode_settable(dcp, mode); + /* * TODO: support synthesized modes through scaling * @@ -2109,19 +2380,79 @@ static void iomfb_queue_atomic(struct apple_dcp *dcp, struct drm_crtc *crtc, schedule_work(&dcp->vblank_wq); } +/* H17P (T8140, T8142): aurora-wip's flush. */ +static void dcp_flush_h17p(struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct platform_device *pdev = to_apple_crtc(crtc)->dcp; + struct apple_dcp *dcp = platform_get_drvdata(pdev); + + if (iomfb_uses_queue(dcp)) { + iomfb_queue_atomic(dcp, crtc, state); + return; + } + + /* + * DCP does not complete swaps after a link loss. A plane-only commit + * arriving between the reconnect callback and the required modeset + * would otherwise leave an unsignalled flip event in front of the + * recovery commit. Modesets set valid_mode before reaching flush. + * + * The same applies while a connector is still disconnected: resume + * re-runs the modeset, which marks the mode valid again, before the + * firmware has reported the display back. + */ + if (!READ_ONCE(dcp->mode_state.valid) || !dcp->connector || !dcp->connector->connected) { + schedule_work(&dcp->vblank_wq); + return; + } + + if (dcp_channel_busy(&dcp->ch_cmd)) + { + if (!dcp->ch_cmd.warned_busy) { + dev_err(dcp->dev, "unexpected busy command channel\n"); + dcp->ch_cmd.warned_busy = true; + } + /* HACK: issue a delayed vblank event to avoid timeouts in + * drm_atomic_helper_wait_for_vblanks(). + */ + schedule_work(&dcp->vblank_wq); + return; + } else if (dcp->ch_cmd.warned_busy) { + dcp->ch_cmd.warned_busy = false; + } + + switch (dcp->fw_compat) { + case DCP_FIRMWARE_V_12_3: + iomfb_flush_v12_3(dcp, crtc, state); + break; + case DCP_FIRMWARE_V_13_5: + iomfb_flush_v13_3(dcp, crtc, state); + break; + case DCP_FIRMWARE_H17P: + iomfb_flush_h17p(dcp, crtc, state); + break; + default: + WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); + break; + } +} + void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) { struct platform_device *pdev = to_apple_crtc(crtc)->dcp; struct apple_dcp *dcp = platform_get_drvdata(pdev); + if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + dcp_flush_h17p(crtc, state); + return; + } + if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { iomfb_v26_6_flush(dcp, crtc, state); return; } if (dcp->fw_compat == DCP_FIRMWARE_V_14_7) { iomfb_v14_7_flush(dcp, crtc, state); - if (iomfb_uses_queue(dcp)) { - iomfb_queue_atomic(dcp, crtc, state); return; } @@ -2164,8 +2495,6 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) case DCP_FIRMWARE_V_13_5: iomfb_flush_v13_3(dcp, crtc, state); break; - case DCP_FIRMWARE_H17P: - iomfb_flush_h17p(dcp, crtc, state); case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_flush_v14_7_0(dcp, crtc, state); break; @@ -2184,11 +2513,12 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_start_v13_3(dcp); break; - case DCP_FIRMWARE_H17P: - iomfb_start_h17p(dcp); case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_start_v14_7_0(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_start_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2276,7 +2606,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) dcp->shmem = dma_alloc_coherent(dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, GFP_KERNEL); - if (!dcp->shmem) + if (!dcp->shmem && dcp->fw_compat == DCP_FIRMWARE_H17P) return -ENOMEM; dcp_send_message(dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); @@ -2297,11 +2627,12 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_V_13_5: iomfb_shutdown_v13_3(dcp); break; - case DCP_FIRMWARE_H17P: - iomfb_shutdown_h17p(dcp); case DCP_FIRMWARE_V_14_7_TEMPLATE: iomfb_shutdown_v14_7_0(dcp); break; + case DCP_FIRMWARE_H17P: + iomfb_shutdown_h17p(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 09f56ced0a1f54..e4e6ea533da220 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -230,6 +230,7 @@ struct dcp_map_buf_req { u8 dva_null; } __packed; +/* 12.3, 13.5 and 14.7; H17P's is in iomfb_template.h. */ struct dcp_map_buf_resp { u64 vaddr; u64 dva; @@ -276,6 +277,7 @@ struct dcp_map_physical_resp { u32 mem_desc_id; } __packed; +/* 12.3, 13.5 and 14.7; H17P's are in iomfb_template.h. */ struct dcp_swap_start_req { u32 swap_id; struct dcp_iouserclient client; @@ -318,6 +320,7 @@ struct iomfb_set_fx_prop_req { u32 value; } __packed; +/* 12.3, 13.5 and 14.7; H17P's is in iomfb_template.h. */ struct dcp_set_power_state_req { u64 unklong; u8 unkbool; diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index bd55bfd475f3ed..ce0d7e0867d0be 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -112,3 +112,6 @@ void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context); /* The user may own drm_display_mode, so we need to search for our copy */ bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, struct dcp_display_mode *out); +/* H17P (T8140, T8142): the catalog entry itself, see iomfb.c. */ +struct dcp_display_mode *lookup_mode_h17p(struct apple_dcp *dcp, + const struct drm_display_mode *mode); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 03ce35d540bf2e..b11451c53eece8 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -4,6 +4,15 @@ * Copyright The Asahi Linux Contributors */ +/* + * H17-generation firmware (T8140 H17P, T8142 H17G) is driven by the template + * aurora-wip carries for it; the 12.3, 13.5 and 14.7 instances below keep + * integration's. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#include "iomfb_template_h17p.c" +#else + #include #include #include @@ -61,31 +70,9 @@ IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); IOMFB_THUNK_INOUT(abort_swaps_dcp); -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER -static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, - struct dcp_swap_submit_req_h17p *request, - dcp_callback_t cb, void *cookie) -{ - const void *data = request; - - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - iomfb_serialize_present_h17p(&dcp->present_h17p, request); - if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { - if (cb) - cb(dcp, NULL, cookie); - return; - } - data = &dcp->present_h17p; - } - - dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), - sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); -} -#else DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, struct DCP_FW_NAME(dcp_swap_submit_req), struct DCP_FW_NAME(dcp_swap_submit_resp)); -#endif DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct dcp_swap_start_req, struct dcp_swap_start_resp); @@ -94,43 +81,6 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, struct dcp_set_power_state_req, struct dcp_set_power_state_resp); -/* - * H17P firmware expects set_power_state (A473) on the out-of-band command - * channel. Sent on the ordinary command channel it is never answered, which - * strands the power transition and every later call behind it. - */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_POWER_OOB true -#else -#define DCP_POWER_OOB false -#endif - -/* - * H17P firmware is attached to a display the bootloader has already brought - * up, and it does not accept set_digital_out_mode (A413) for that display: - * the call fails (0x8000000b) and the timing generator never starts, so there - * is no vblank and no swap completion. Keep the mode the bootloader - * programmed, which is the panel's native mode, instead of requesting one. - */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_INHERIT_BOOT_MODE true -#else -#define DCP_INHERIT_BOOT_MODE false -#endif - -/* - * abort_swaps_dcp and last_client_close have no verified H17P method numbers, - * and the H17P table leaves them out. Rather than send the firmware a record - * it would decode as some other call, power-off and sleep skip both and go - * straight to set_power_state, whose number and sizes are known. Power-off - * still sends its clear swap first. - */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_HAS_CLIENT_TEARDOWN false -#else -#define DCP_HAS_CLIENT_TEARDOWN true -#endif - DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, struct dcp_set_digital_out_mode_req, u32); @@ -149,68 +99,6 @@ DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * Default-framebuffer queries issued from inside start_hardware_boot (D121). - * The getters take a zeroed input of the expected size; the firmware only - * checks the size. - */ -struct dcp_dfb_info_h17p { - u8 data[0x2c]; -} __packed; - -struct dcp_dfb_layout_req_h17p { - u8 data[0xc]; -} __packed; - -struct dcp_dfb_dimensions_h17p { - u32 width; - u32 height; -} __packed; - -DCP_THUNK_INOUT(dcp_get_dfb_compression_info, dcpep_get_dfb_compression_info, - u64, u64); -DCP_THUNK_INOUT(dcp_get_dfb_info, dcpep_get_dfb_info, - struct dcp_dfb_info_h17p, struct dcp_dfb_info_h17p); -DCP_THUNK_INOUT(dcp_get_dfb_layout, dcpep_get_dfb_layout, - struct dcp_dfb_layout_req_h17p, u64); -DCP_THUNK_INOUT(dcp_get_dfb_state, dcpep_get_dfb_state, u32, u64); -DCP_THUNK_IN(dcp_set_dfb_dimensions, dcpep_set_dfb_dimensions, - struct dcp_dfb_dimensions_h17p); -DCP_THUNK_VOID(dcp_commit_dfb_info, dcpep_commit_dfb_info); -DCP_THUNK_OUT(dcp_dfb_query, dcpep_dfb_query, u32); -DCP_THUNK_OUT(dcp_dfb_ready_query, dcpep_dfb_ready_query, u32); - -/* - * Pipe-configuration calls the firmware expects between the boot chain and - * the first swap. Most of them have no known name; they are identified by - * method number and wire size. - */ -struct dcp_pipe_cfg_415 { - u8 data[0xc]; -} __packed; - -/* - * A031 is update_notify_clients_dcp(u32[27]): an array of the DCP - * notification clients the AP wants delivered. Clients that are not - * registered here never send the callbacks they gate. - */ -struct dcp_pipe_cfg_031 { - __le32 clients[27]; -} __packed; -static_assert(sizeof(struct dcp_pipe_cfg_031) == 0x6c); - -DCP_THUNK_INOUT(dcp_pipe_cfg_415, dcpep_pipe_cfg_415, - struct dcp_pipe_cfg_415, struct dcp_pipe_cfg_415); -DCP_THUNK_IN(dcp_pipe_cfg_031, dcpep_pipe_cfg_031, struct dcp_pipe_cfg_031); -DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); -DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); -DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); -DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); -DCP_THUNK_INOUT(dcp_pipe_set_428, dcpep_pipe_set_428, u32, u32); -#endif - DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, @@ -246,27 +134,6 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, struct DCP_FW_NAME(dc_swap_complete_resp) *resp) { ktime_t now = ktime_get(); - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { - /* - * Abort semantics are not captured on H17P. If an aborted - * present completes after all, its framebuffers were already - * unarmed and its event signalled; record it and carry on. - */ - if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, - resp->swap_id)) { - dev_warn_ratelimited(dcp->dev, - "completion for aborted present %u ignored\n", - resp->swap_id); - return; - } - dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); - dcp->crashed = true; - return; - } -#endif trace_iomfb_swap_complete(dcp, resp->swap_id); dcp->last_swap_id = resp->swap_id; dcp_swap_watchdog_complete(dcp); @@ -296,94 +163,6 @@ static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *o return false; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P asks the AP to make the default framebuffer surface (D582). The - * firmware expects the AP to register a surface from inside that callback, - * with a nested A472 carrying a 0x22c-byte surface descriptor, and only then - * to report success. Without it the firmware has no surface, programs the - * display engine from a null base and the piodma DART faults. - */ -struct dcp_h17p_dfb_surface_req { - u8 surface[0x22c]; - u8 surface_null; - u8 padding[3]; -} __packed; -static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); - -/* - * A472 has a 4-byte output; with out_len 0 the firmware has nowhere to write - * its result and stops answering. - */ -DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, - struct dcp_h17p_dfb_surface_req, u32); - -static void dcp_ack_dfb_surface(struct apple_dcp *dcp) -{ - struct dcp_channel *ch = &dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - dcp_ack(dcp, DCP_CONTEXT_CB); -} - -/* - * Fill the default surface descriptor. Fields not set here are zero. The - * format at +0x0b is the fourcc 'b3a8', stored byte-reversed; +0x35 is the - * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose - * meaning is not known. - */ -static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) -{ - s[0x002] = 0x01; - memcpy(s + 0x00b, "8a3b", 4); - s[0x019] = 0x01; - s[0x01b] = 0x01; - s[0x01c] = 0x01; - s[0x02b] = 0x4d; - s[0x02c] = 0x01; - s[0x035] = 0x02; - s[0x051] = 0x01; - s[0x149] = 0x01; -} - -/* - * A471 (update_dfb) is never sent: on H17P it makes the coprocessor take an - * AXI read error. Scanout does not need it, since every swap carries its own - * surface. Just acknowledge the callback. - */ -static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_ack_dfb_surface(dcp); -} - -static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, - void *out, void *in) -{ - struct dcp_h17p_dfb_surface_req *req; - - trace_iomfb_callback(dcp, tag, __func__); - - /* - * The request is too large for the stack this deep in the RTKit - * receive work (rx work -> iomfb_recv_msg -> dcpep_handle_cb -> here -> - * dcp_push). dcp_push() copies the payload into shared memory before - * returning, so a short-lived allocation is enough. - */ - req = kzalloc_obj(*req); - if (!req) - return true; - - dcp_fill_dfb_surface(dcp, req->surface); - - dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); - kfree(req); - - /* deferred ACK: the A472 completion reports success for us */ - return false; -} -#endif - static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_channel *ch = &dcp->ch_cb; @@ -506,7 +285,6 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, { struct dcp_mem_descriptor *memdesc; struct sg_table *map; - size_t size; ssize_t ret; if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) @@ -514,7 +292,6 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, memdesc = &dcp->memdesc[req->buffer]; map = &memdesc->map; - size = ALIGN(memdesc->size, SZ_16K); if (!map->sgl) goto reject; @@ -531,87 +308,12 @@ static struct dcp_map_buf_resp dcpep_cb_map_piodma(struct apple_dcp *dcp, goto reject; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - memdesc->piodma_mapped = true; - - /* the H17P reply echoes the buffer index */ - *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { - .buffer = req->buffer, - .dva = memdesc->dva, - }; -#else - *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .dva = memdesc->dva }; -#endif - return true; + return (struct dcp_map_buf_resp){ .dva = memdesc->dva }; reject: dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", req->buffer); - return false; -} - -static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, - struct dcp_map_buf_req *req) -{ - struct DCP_FW_NAME(dcp_map_buf_resp) resp = { 0 }; - - if (dcpep_map_piodma(dcp, req, &resp)) - return resp; - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - resp.buffer = req->buffer; -#else - resp.ret = EINVAL; -#endif - return resp; -} - -static bool dcpep_unmap_piodma(struct apple_dcp *dcp, - struct dcp_unmap_buf_resp *request) -{ - struct dcp_mem_descriptor *memdesc; - size_t size, unmapped; - - if (request->buffer >= ARRAY_SIZE(dcp->memdesc)) { - dev_warn(dcp->dev, "unmap request for out of range buffer %llu\n", - request->buffer); - return false; - } - - memdesc = &dcp->memdesc[request->buffer]; - - if (!test_bit(request->buffer, dcp->memdesc_map) || !memdesc->buf || - !dcp->iommu_dom) - goto not_mapped; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (!memdesc->piodma_mapped) - goto not_mapped; -#endif - - if (memdesc->dva != request->dva) { - dev_warn(dcp->dev, - "unmap buffer %llu address mismatch memdesc.dva:%llx dva:%llx\n", - request->buffer, - memdesc->dva, request->dva); - return false; - } - - /* use the piodma iommu domain to unmap from the right IOMMU */ - size = ALIGN(memdesc->size, SZ_16K); - unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); - if (unmapped != size) { - dev_err(dcp->dev, "iommu_unmap() returned %zu instead of %zu\n", - unmapped, size); - return false; - } - memdesc->piodma_mapped = false; - return true; - -not_mapped: - dev_warn(dcp->dev, - "unmap for non-mapped buffer %llu iova:0x%08llx\n", - request->buffer, request->dva); - return false; + return (struct dcp_map_buf_resp){ .ret = EINVAL }; } static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, @@ -661,40 +363,18 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, struct dcp_mem_descriptor *memdesc; size_t size; u32 id; - int ret; - /* Validate both wire-size and IOMMU-page rounding before allocating. */ - if (!req->size || req->size > SIZE_MAX || - check_add_overflow((size_t)req->size, (size_t)4095, &rounded)) - return resp; - allocated.size = round_down(rounded, 4096); - if (check_add_overflow(allocated.size, (size_t)SZ_16K - 1, &rounded)) - return resp; - size = round_down(rounded, SZ_16K); - - id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + resp.dva_size = ALIGN(req->size, 4096); + resp.mem_desc_id = + find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - if (id >= DCP_MAX_MAPPINGS) { + if (resp.mem_desc_id >= DCP_MAX_MAPPINGS) { dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); resp.dva_size = 0; resp.mem_desc_id = 0; return resp; } - - allocated.buf = dma_alloc_coherent(dcp->dev, size, &allocated.dva, - GFP_KERNEL); - if (!allocated.buf) - return resp; - - ret = dma_get_sgtable(dcp->dev, &allocated.map, allocated.buf, - allocated.dva, size); - if (ret) { - dma_free_coherent(dcp->dev, size, allocated.buf, allocated.dva); - return resp; - } - - /* Callbacks are serialized; publish only a fully initialized descriptor. */ - dcp->memdesc[id] = allocated; + id = resp.mem_desc_id; set_bit(id, dcp->memdesc_map); memdesc = &dcp->memdesc[id]; @@ -709,15 +389,6 @@ dcpep_cb_allocate_buffer(struct apple_dcp *dcp, size); resp.dva = memdesc->dva; -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * Linux allocation traces correlate the first H17P reply address with - * the first SG page. The remaining pages need not be contiguous. - */ - if (allocated.map.sgl) - resp.paddr = sg_phys(allocated.map.sgl); -#endif - return resp; } @@ -741,31 +412,6 @@ static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) memdesc = &dcp->memdesc[id]; size = ALIGN(memdesc->size, SZ_16K); - - /* - * Tear down any translation still standing before the pages go back to - * the allocator. The firmware is expected to unmap piodma first, but - * if it releases a descriptor without doing so we would hand the pages - * back while the DART still resolves them -- the display engine would - * then read whoever owns them next. Only the H17P map_piodma handler - * sets piodma_mapped, so older firmware is unaffected. - */ - if (memdesc->piodma_mapped) { - dev_warn(dcp->dev, - "releasing buffer %u while still mapped for piodma; unmapping first\n", - id); - if (!dcp->iommu_dom) - return 0; - unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); - if (unmapped != size) { - dev_err(dcp->dev, - "failed to unmap buffer %u before release: %zu of %zu\n", - id, unmapped, size); - return 0; - } - memdesc->piodma_mapped = false; - } - if (memdesc->buf) { dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); memdesc->buf = NULL; @@ -814,18 +460,6 @@ dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) } id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); - if (id >= DCP_MAX_MAPPINGS) { - dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); - return (struct dcp_map_physical_resp){}; - } - - dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); - if (dma_mapping_error(dcp->dev, dva)) { - dev_err(dcp->dev, "failed to map phys address %llx size %llx\n", - req->paddr, size); - return (struct dcp_map_physical_resp){}; - } - set_bit(id, dcp->memdesc_map); dcp->memdesc[id].size = size; dcp->memdesc[id].reg = req->paddr; @@ -881,17 +515,6 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; } else { struct resource *rsrc = dcp->disp_registers[req->index]; -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P reaches these apertures physically, not through its - * DART, and expects dva == addr in the reply. Trying to - * dma_map_resource() them instead fails and hands the firmware - * DMA_MAPPING_ERROR as a device address. - */ - dma_addr_t dva = rsrc->start; -#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), - DMA_BIDIRECTIONAL, 0); #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) dma_addr_t dva = dcpep_map_reg_dva(dcp, req->index, rsrc); WARN_ON(dva == DMA_MAPPING_ERROR); @@ -912,8 +535,8 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) { /* Observe boot-property requests without inventing firmware timings. */ if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_dbg(dcp->dev, - "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", + dev_info(dcp->dev, + "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", (int)sizeof(req->key), req->key, req->count, req->value[0]); @@ -926,15 +549,6 @@ dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, u8 *enabled) { -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* H17P carries the level only in presents; ask for one more. */ - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - if (dcp_has_panel(dcp) && dcp_backlight_active(dcp) && - dcp_backlight_resend(dcp)) - schedule_work(&dcp->bl_update_wq); - return; - } -#endif /* * Reapply brightness on the next swap after DPMS wake on panels * without mini-LED backlights. @@ -956,48 +570,11 @@ static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) return false; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * This length comes straight off the wire. A bogus value would turn - * the allocation below into a multi-gigabyte GFP_KERNEL request. - * Property blobs are a few KiB, so bound it. - */ - if (*length == 0 || *length > SZ_1M) { - dev_warn(dcp->dev, "rejecting %u byte property transfer\n", - *length); - return false; - } -#endif - if (!dcp_property_size_valid(*length)) return false; dcp->chunks.modes_generation = dcp_modes_transfer_begin(dcp); dcp->chunks.length = *length; - dcp->chunks.av_service_epoch = 0; - dcp->chunks.automatic_service_pending = false; - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { - /* Match the receipt's start, not a replacement service at its end. */ - mutex_lock(&dcp->dcpavserv.lock); - dcp->chunks.av_service_epoch = dcp->dcpavserv.epoch; - dcp->chunks.automatic_service_pending = - !dcp->dcpavserv.enabled && !dcp->dcpavserv.service; - dcp_firmware_metadata_begin(dcp); - { - unsigned long flags; - - spin_lock_irqsave(&dcp->metadata.lock, flags); - dcp->chunks.automatic_connection_sequence = dcp->rate_proof.connection_sequence; - dcp->chunks.mode_ticket = dcp_metadata_begin(&dcp->metadata.epoch); - dcp->chunks.mode_ticket.associated = - dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation) && - READ_ONCE(dcp->typec_cable_connected); - spin_unlock_irqrestore(&dcp->metadata.lock, flags); - } - mutex_unlock(&dcp->dcpavserv.lock); - } else { - dcp_firmware_metadata_begin(dcp); - } dcp->chunks.data = kzalloc(*length, GFP_KERNEL); if (!dcp->chunks.data) { @@ -1096,12 +673,6 @@ static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, (int)sizeof(req->key), req->key, dcp->chunks.length, resp, dcp->nr_modes); - if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_dbg(dcp->dev, - "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", - (int)sizeof(req->key), req->key, dcp->chunks.length, - resp, dcp->nr_modes); - /* move chunked data to connector to provide it via debugfs */ dcp_connector_update_dict(dcp->connector, req->key, &dcp->chunks); dcp->chunks.data = NULL; @@ -1126,40 +697,16 @@ static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); } -static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) -{ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* H17P expects A380 right before set_display_refresh_properties */ - dcp_dfb_ready_query(dcp, false, boot_5b, NULL); -#else - boot_5b(dcp, out, cookie); -#endif -} - -static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - u32 value = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; -#else - u32 value = 1; -#endif - dcp_late_init_signal(dcp, false, &value, boot_5, NULL); + u32 v_true = 1; + dcp_late_init_signal(dcp, false, &v_true, boot_5, NULL); #else dcp_late_init_signal(dcp, false, boot_5, NULL); #endif } -static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) -{ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* H17P expects A033 between flush_supports_power and late_init_signal */ - dcp_dfb_query(dcp, false, boot_4b, NULL); -#else - boot_4b(dcp, out, cookie); -#endif -} - static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) { u32 v_true = true; @@ -1172,103 +719,11 @@ static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) dcp_setup_video_limits(dcp, false, boot_3, NULL); } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P expects the default framebuffer to be described between - * create_default_fb and setup_video_limits: - * get_dfb_compression_info -> get_dfb_info -> get_dfb_layout -> - * get_dfb_state -> set_dfb_dimensions(w, h) -> commit_dfb_info - * Without it the firmware never finishes its own default framebuffer setup. - */ -static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) -{ - dcp_commit_dfb_info(dcp, false, boot_2, NULL); -} - -static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_dfb_dimensions_h17p dim; - - /* - * get_dfb_state returns the default framebuffer dimensions, which are - * passed back unchanged to set_dfb_dimensions. - */ - memcpy(&dim, out, sizeof(dim)); - if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { - struct dcp_channel *ch = &dcp->ch_cb; - - dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", - dim.width, dim.height); - *(u8 *)ch->output[ch->depth - 1] = false; - dcp_ack(dcp, DCP_CONTEXT_CB); - return; - } - dev_dbg(dcp->dev, "default framebuffer dimensions %ux%u\n", - dim.width, dim.height); - - dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); -} - -static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) -{ - u32 zero = 0; - - dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); -} - -static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_dfb_layout_req_h17p req = {}; - - dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); -} - -static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_dfb_info_h17p info = {}; - - dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); -} - -static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) -{ - u64 zero = 0; - - dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); -} -#endif - static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) { -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - dcp_create_default_fb(dcp, false, boot_1_6, NULL); -#else dcp_create_default_fb(dcp, false, boot_2, NULL); -#endif } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -static void d000_done(struct apple_dcp *dcp, void *out, void *cookie) -{ - struct dcp_channel *ch = &dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - dcp_ack(dcp, DCP_CONTEXT_CB); -} - -/* - * The firmware expects D000 to be answered only after a nested A033 - * (dfb_query), and then with 1. Defer the ACK until that call completes. - */ -static bool dcpep_cb_d000_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) -{ - trace_iomfb_callback(dcp, tag, __func__); - dcp_dfb_query(dcp, false, d000_done, NULL); - return false; -} -#endif - /* Use special function signature to defer the ACK */ static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) { @@ -1316,46 +771,6 @@ static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) return rt_bw; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P's D006 is 0x54 in / 0x50 out, far wider than the 28-byte - * frame_sync_props of older firmware, so the fields that matter sit past - * anything the old struct could describe. - * - * The firmware expects zeros except for three u32 at 0x24, 0x28 and 0x2c, - * each 0x10000 -- 1.0 in 16.16 fixed point, the identity scale. An all-zero - * reply hands the pipe a zero scale, and it then programs a null surface - * base: the display DART faults at DVA 0 and the coprocessor takes an AXI - * error. - */ -#define DCP_FRAME_SYNC_IDENTITY 0x10000 - -struct dcp_set_frame_sync_props_req_h17p { - u8 unk[0x54]; -} __packed; - -struct dcp_set_frame_sync_props_resp_h17p { - u8 unk[0x50]; -} __packed; - -static struct dcp_set_frame_sync_props_resp_h17p -dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, - struct dcp_set_frame_sync_props_req_h17p *req) -{ - struct dcp_set_frame_sync_props_resp_h17p resp = {}; - static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(identity_at); i++) { - u32 v = DCP_FRAME_SYNC_IDENTITY; - - memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); - } - - return resp; -} -#endif - static struct dcp_set_frame_sync_props_resp dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, struct dcp_set_frame_sync_props_req *req) @@ -1383,179 +798,12 @@ static void release_swap_cookie(struct kref *ref) kfree(cookie); } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_SWAP_FLAGS1 0x34865202ULL -#define DCP_SWAP_FLAGS2 0x4ULL - -/* - * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: - * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier - * than on every other firmware. With the older layout the firmware reads the - * format two bytes late and its surface check rejects the swap as an - * unsupported format. - * - * H17G retains this public packing path. The H17P profile uses - * a separate wire record with the SPEC's 0x588 swap boundary. - */ -static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) -{ - struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); - unsigned int i; - - /* - * The firmware expects flags1 = 0x34865202 and flags2 = 4 (at +0x40 - * and +0x48) in every swap. - */ - req->swap.flags1 = DCP_SWAP_FLAGS1; - req->swap.flags2 = DCP_SWAP_FLAGS2; - - /* - * The H17P gap before swap_id carries two small counters the firmware - * expects to be 2 (at +0x50) and 3 (at +0x78). The gap starts at - * +0x50, so these are indices 0 and 0x28 of the blob. - */ - req->swap.h17p_pre_swap_id[0x00] = 2; - req->swap.h17p_pre_swap_id[0x28] = 3; - - /* The H17P layout is serialized without shifting its surfaces. */ - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return; - - for (i = 0; i < SWAP_SURFACES; i++) { - u8 *p = (u8 *)&req->surf[i]; - size_t n = sizeof(req->surf[i]); - - memmove(p + 1, p + 3, n - 3); - memset(p + n - 2, 0, 2); - } - - /* - * Everything from surf_iova onward also sits two bytes earlier on - * H17P: surf_iova[0] is at +0xe38 of the 0xe9c-byte record, not - * +0xe3a. Left in place, the firmware reads every IOVA two bytes off - * and the trailing null-flag block misaligned, so it faults on a bogus - * address or verifies layers that are not present. Shift the whole - * block down. - */ - { - u8 *r = (u8 *)req; - size_t n = sizeof(*req); - size_t off = offsetof(typeof(*req), surf_iova); - - static_assert(offsetof(typeof(*req), surf_iova) == 0xe3a, - "H17P source IOVA offset"); - static_assert(sizeof(*req) == 0xe9c, "H17P request size"); - memmove(r + off - 2, r + off, n - off); - memset(r + n - 2, 0, 2); - } -} -#else -static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } -#endif - -static bool dcp_present_retires(struct apple_dcp *dcp) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return !iomfb_present_brightness_only_h17p(dcp); -#endif - return true; -} - -static void dcp_present_failed(struct apple_dcp *dcp) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - iomfb_present_failed_h17p(dcp); -#endif -} - -static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !dcp_present_begin_h17p(&dcp->present_state_h17p, swap_id)) { - dev_err(dcp->dev, "overlapping present %u\n", swap_id); - dcp->crashed = true; - return false; - } -#endif - return true; -} - -/* - * Returns 0 if the submitted present stands, -ECANCELED if the firmware - * aborted it before replying (already reported), or -EPROTO if no present - * was expecting this reply. - */ -static int dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !dcp_present_submit_h17p(&dcp->present_state_h17p, swap_id, accepted)) { - dev_err(dcp->dev, "unexpected present submission %u\n", swap_id); - dcp->crashed = true; - return false; - } - /* An abort received before this reply turns acceptance into failure. */ - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - accepted && dcp->present_state_h17p.aborted) { - dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", - swap_id); - return false; - } -#endif - return 0; -} - -static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) -{ - typeof(DCP_FW_UNION(dcp->swap)) *swap = request; - - /* Clear surfaces. */ - memset(swap, 0, sizeof(*swap)); - - swap->swap.swap_enabled = - swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; - swap->swap.bg_color = 0xFF000000; - - /* - * Turn off the backlight. This matters because the DCP's idea of - * backlight brightness gets desynced after a power change, and it - * needs to be told it's going to turn off so it will consider the - * subsequent update on poweron an actual change and restore the - * brightness. - */ - if (dcp_has_panel(dcp)) { - swap->swap.bl_unk = 1; - swap->swap.bl_value = 0; - swap->swap.bl_power = 0; - } - - /* Null all surfaces */ - for (int l = 0; l < SWAP_SURFACES; l++) - swap->surf_null[l] = true; -#if DCP_FW_VERSION(13, 2, 0) <= DCP_FW_VER -#if DCP_FW_VERSION(26, 0, 0) > DCP_FW_VER - for (int l = 0; l < 5; l++) - swap->surf2_null[l] = true; -#endif - swap->unknown_pointer_null = true; - swap->unknown_output_null = true; -#endif -} - static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; - if (status) { - dev_err(dcp->dev, "swap_clear failed! status %u\n", status); - dcp_unarm_retained_framebuffers(dcp, swap_id); - dcp_drm_crtc_vblank(dcp->crtc); - } - if (info) { - WRITE_ONCE(info->status, status); + if (cookie) { + struct dcp_swap_cookie *info = cookie; complete(&info->done); kref_put(&info->refcount, release_swap_cookie); } @@ -1584,14 +832,12 @@ static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, { struct dcp_swap_start_resp *resp = data; DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - if (dcp_present_retires(dcp)) - dcp_arm_retained_framebuffers(dcp, resp->swap_id); if (cookie) { struct dcp_swap_cookie *info = cookie; info->swap_id = resp->swap_id; + } - dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); } @@ -1646,19 +892,9 @@ void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) kref_get(&cookie->refcount); if (dcp->main_display) { -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P takes set_display_device (A411) once, in the - * client-open sequence. Repeating it after the client is open - * is a call the firmware stops answering, which wedges the - * command channel and with it every later swap. - */ - dcp_on_set_power_state(dcp, NULL, cookie); -#else handle = 0; dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, cookie); -#endif } else { handle = 2; dcp_set_display_device(dcp, false, &handle, @@ -1697,16 +933,6 @@ static void complete_set_powerstate(struct apple_dcp *dcp, void *out, } } -static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) -{ - /* H17P does not confirm that scanout has stopped. */ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - return false; -#else - return READ_ONCE(cookie->status) == 0; -#endif -} - static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_set_power_state_req power_req = { @@ -1736,12 +962,6 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) struct dcp_swap_start_req swap_req = { 0 }; struct DCP_FW_NAME(dcp_swap_submit_req) *swap = &DCP_FW_UNION(dcp->swap); -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* H17P has no known safe pipe-off or surface-free transition. */ - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return; -#endif - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -1750,11 +970,34 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) /* increase refcount to ensure the receiver has a reference */ kref_get(&cookie->refcount); -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - dcp_prepare_clear_swap(dcp, swap); -#else - dcp_prepare_clear_swap(dcp, swap); + // clear surfaces + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + if (dcp_has_panel(dcp)) { + swap->swap.bl_unk = 1; + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + } + + /* Null all surfaces */ + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; + swap->unkU32Ptr_null = true; + swap->unkU32out_null = true; #endif dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); @@ -1825,11 +1068,6 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return; -#endif - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -1918,35 +1156,6 @@ dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, info->width, info->height); } -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER -/* - * An aborted H17P present never completes. Finish it like a rejected one: - * keep the displaced framebuffers for the next completed present and signal - * the DRM event, so neither the queue nor userspace waits for a completion. - */ -static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) -{ - if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { - dev_warn_ratelimited(dcp->dev, - "abort for present %u, which is not in flight\n", - swap_id); - return; - } - /* The submit reply has not arrived; it finishes the present. */ - if (dcp->present_state_h17p.pending) - return; - - dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", swap_id); - /* No completion follows, so the swap watchdog must not wait for one. */ - dcp_swap_watchdog_complete(dcp); - dcp_present_failed(dcp); - if (dcp_present_retires(dcp)) { - dcp_unarm_retained_framebuffers(dcp, swap_id); - dcp_drm_crtc_vblank(dcp->crtc); - } -} -#endif - static void dcpep_cb_abort_swap_ap_gated(struct apple_dcp *dcp, u32 *swap_id) { @@ -1965,19 +1174,7 @@ dcpep_cb_get_tiling_state(struct apple_dcp *dcp, static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) { -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P asks during start_ios_gated, before any mode is known, but - * dcp_has_panel() only becomes true once the panel dimensions have been - * parsed. An internal panel always has a backlight, so answer for the - * connector; answering false makes the firmware report that it could - * not find the backlight service. - */ - return dcp_has_panel(dcp) || - dcp->connector_type == DRM_MODE_CONNECTOR_eDP; -#else return dcp_has_panel(dcp); -#endif } TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); @@ -2020,12 +1217,6 @@ TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, struct dcp_set_frame_sync_props_req, struct dcp_set_frame_sync_props_resp); -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -TRAMPOLINE_INOUT(trampoline_set_frame_sync_props_h17p, - dcpep_cb_set_frame_sync_props_h17p, - struct dcp_set_frame_sync_props_req_h17p, - struct dcp_set_frame_sync_props_resp_h17p); -#endif TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); @@ -2048,10 +1239,7 @@ TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_se static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) { - int ret; struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; - u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; - u32 status = resp ? resp->ret : ~0U; if (resp->ret) { dev_err(dcp->dev, "swap failed! status %u\n", resp->ret); @@ -2079,21 +1267,9 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { struct dcp_swap_start_resp *resp = data; - if (!resp || resp->ret) { - dev_err(dcp->dev, "swap_start was rejected\n"); - dcp_drm_crtc_vblank(dcp->crtc); - return; - } - if (!dcp_present_begin(dcp, resp->swap_id)) { - dcp_drm_crtc_vblank(dcp->crtc); - return; - } DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; - if (dcp_present_retires(dcp)) - dcp_arm_retained_framebuffers(dcp, resp->swap_id); trace_iomfb_swap_submit(dcp, resp->swap_id); - dcp_h17p_prepare_swap(dcp); dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); } @@ -2127,19 +1303,6 @@ static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) } } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in - * 16.16 fixed point), the same identity value the D006 reply carries. - */ -static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) -{ - u32 identity = 0x10000; - - dcp_pipe_set_428(dcp, false, &identity, dcp_post_swap_a428, cookie); -} -#endif - static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, void *cookie) { @@ -2289,7 +1452,6 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) for (l = 0; l < 5; l++) req->surf2_null[l] = true; -#endif req->unkU32Ptr_null = true; req->unkU32out_null = true; #endif @@ -2315,6 +1477,27 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru l = apl_plane->iomfb_surf; req->swap.swap_enabled |= BIT(l); + if (old_state->fb && new_state->fb != old_state->fb) { + /* + * Race condition between a framebuffer unbind getting + * swapped out and GEM unreferencing a framebuffer. If + * we lose the race, the display gets IOVA faults and + * the DCP crashes. We need to extend the lifetime of + * the drm_framebuffer (and hence the GEM object) until + * after we get a swap complete for the swap unbinding + * it. + */ + struct dcp_fb_reference *entry = + kzalloc(sizeof(*entry), GFP_KERNEL); + if (entry) { + entry->fb = old_state->fb; + entry->swap_id = dcp->last_swap_id; + list_add_tail(&entry->head, + &dcp->swapped_out_fbs); + } + drm_framebuffer_get(old_state->fb); + } + if (!new_state->fb || !new_state->visible) { continue; } @@ -2343,18 +1526,6 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* H17P external SDR RGB needs an sRGB source tag. Native - * tagging washes out the same pixel values on this profile. - * Keep YUV, other source spaces and other profiles unchanged. - */ - if (dcp->hw.iomfb_method_profile == - DCP_IOMFB_METHODS_H17P_EXTERNAL && - !new_state->fb->format->is_yuv && - req->surf[l].base.colorspace == DCP_COLORSPACE_NATIVE) - req->surf[l].base.colorspace = DCP_COLORSPACE_BG_SRGB; -#endif } if (!has_surface && !crtc_state->color_mgmt_changed) { @@ -2415,11 +1586,7 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); -#else iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); -#endif } else do_swap(dcp, NULL, NULL); } @@ -2447,141 +1614,6 @@ static void init_3(struct apple_dcp *dcp, void *out, void *cookie) dcp_is_main_display(dcp, false, res_is_main_display, NULL); } -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * The client-open sequence H17P firmware expects before the first swap, in - * H17P method numbers (dcp_push() applies the H17G aliases): - * - * A445 A411 A445 A415 A450 A031 x9 A414 A478 A457 A474 A478 A476 A478 A412 - * - * A412 is not in the table: its result feeds res_is_main_display(), so it - * stays on the existing path. - */ -enum dcp_client_step_kind { - DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A415, DCP_STEP_A450, - DCP_STEP_A031, DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, - DCP_STEP_A474, DCP_STEP_A476, -}; - -static const u8 dcp_client_open_seq[] = { - DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A445, DCP_STEP_A415, - DCP_STEP_A450, - DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, - DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, - DCP_STEP_A031, - DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, DCP_STEP_A474, - DCP_STEP_A478, DCP_STEP_A476, DCP_STEP_A478, -}; - -static void dcp_client_open_run(struct apple_dcp *dcp, void *out, void *cookie) -{ - unsigned long idx = (unsigned long)cookie; - dcp_callback_t next = dcp_client_open_run; - void *ncookie = (void *)(idx + 1); - - if (idx >= ARRAY_SIZE(dcp_client_open_seq)) { - init_3(dcp, NULL, NULL); - return; - } - - switch (dcp_client_open_seq[idx]) { - case DCP_STEP_A445: { - u32 zero = 0; - - dcp_get_dfb_state(dcp, false, &zero, next, ncookie); - break; - } - case DCP_STEP_A411: { - u32 handle = 0; - - dcp_set_display_device(dcp, false, &handle, next, ncookie); - break; - } - case DCP_STEP_A415: { - struct dcp_pipe_cfg_415 cfg = {}; - - dcp_pipe_cfg_415(dcp, false, &cfg, next, ncookie); - break; - } - case DCP_STEP_A450: { - u32 val = 0; - - dcp_enable_disable_video_power_savings(dcp, false, &val, next, - ncookie); - break; - } - case DCP_STEP_A031: { - /* - * Nine cumulative states: each call repeats the previous set - * of notification clients and adds one, so state n lists n - * clients. - */ - static const u8 notify_clients[9][8] = { - {}, - { 6 }, - { 6, 7 }, - { 6, 7, 8 }, - { 6, 7, 8, 10 }, - { 6, 7, 8, 10, 11 }, - { 6, 7, 8, 10, 11, 12 }, - { 6, 7, 8, 10, 11, 12, 25 }, - { 6, 7, 8, 10, 11, 12, 25, 26 }, - }; - struct dcp_pipe_cfg_031 cfg = {}; - unsigned int step = dcp->a031_step; - unsigned int i; - - if (step >= ARRAY_SIZE(notify_clients)) - step = ARRAY_SIZE(notify_clients) - 1; - for (i = 0; i < step; i++) - cfg.clients[i] = cpu_to_le32(notify_clients[step][i]); - dcp->a031_step++; - - dcp_pipe_cfg_031(dcp, false, &cfg, next, ncookie); - break; - } - case DCP_STEP_A414: { - u64 val = 0; - - dcp_pipe_cfg_414(dcp, false, &val, next, ncookie); - break; - } - case DCP_STEP_A478: - dcp_pipe_query_478(dcp, false, next, ncookie); - break; - case DCP_STEP_A457: - dcp_first_client_open(dcp, false, next, ncookie); - break; - case DCP_STEP_A474: - dcp_pipe_query_474(dcp, false, next, ncookie); - break; - case DCP_STEP_A476: { - /* - * A476 takes a single u32, which the firmware expects to be 1. - * Its name is not known for certain: the shape (in 4, out 0) - * fits both set_dcp_clamshellstate() and - * set_has_frame_swap_function(). - */ - u32 state = 1; - - dcp_pipe_set_476(dcp, false, &state, next, ncookie); - break; - } - default: - dev_warn(dcp->dev, "unknown client-open step %u\n", - dcp_client_open_seq[idx]); - init_3(dcp, NULL, NULL); - break; - } -} - -static void init_1(struct apple_dcp *dcp, void *out, void *cookie) -{ - /* every client-open sequence starts from the empty A031 state */ - dcp->a031_step = 0; - dcp_client_open_run(dcp, NULL, (void *)0UL); -} -#else static void init_2(struct apple_dcp *dcp, void *out, void *cookie) { dcp_first_client_open(dcp, false, init_3, NULL); @@ -2592,7 +1624,6 @@ static void init_1(struct apple_dcp *dcp, void *out, void *cookie) u32 val = 0; dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); } -#endif static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) { @@ -2611,8 +1642,7 @@ void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) struct dcp_set_power_state_req req = { /* defaults are ok */ }; - struct dcp_wait_cookie *cookie; - int ret; dcp_set_power_state(dcp, false, &req, NULL, NULL); } +#endif /* DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) */ diff --git a/drivers/gpu/drm/apple/iomfb_template.h b/drivers/gpu/drm/apple/iomfb_template.h index e19397b21261cf..b9e3fe9e7f2aec 100644 --- a/drivers/gpu/drm/apple/iomfb_template.h +++ b/drivers/gpu/drm/apple/iomfb_template.h @@ -24,7 +24,7 @@ struct DCP_FW_NAME(dcp_swap) { u64 flags1; u64 flags2; -#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) && DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 unk_v14_7[0x48]; #endif @@ -52,7 +52,7 @@ struct DCP_FW_NAME(dcp_swap) { u32 bg_color; u8 unk_110[0x1b8]; u32 unk_2c8; -#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) || DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) u8 unk_2cc[0x14]; u32 unk_2e0; #else @@ -69,7 +69,7 @@ struct DCP_FW_NAME(dcp_swap) { #else u8 unk_2e2[3]; #endif -#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_VER < DCP_FW_VERSION(14, 7, 0) || DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) u64 bl_unk; #else u32 bl_unk_v14_7; /* the 13.x bl_unk position; not written */ @@ -79,7 +79,7 @@ struct DCP_FW_NAME(dcp_swap) { u8 unk_2f3[0x2d]; #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u8 unk_320[0x13f]; -#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) && DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 unk_14_7_2[0x30]; #endif u64 unk_1; @@ -130,7 +130,7 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) u32 unkU32Ptr; #endif -#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) && DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding_14_7[0x234]; #endif u8 swap_null; @@ -141,9 +141,12 @@ struct DCP_FW_NAME(dcp_swap_submit_req) { #endif #endif u8 unkoutbool_null; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) u8 unknown_pointer_null; u8 unknown_output_null; +#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + u8 unkU32Ptr_null; + u8 unkU32out_null; #endif #if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) u8 padding[1]; diff --git a/drivers/gpu/drm/apple/iomfb_template_h17p.c b/drivers/gpu/drm/apple/iomfb_template_h17p.c new file mode 100644 index 00000000000000..1dd691dd315b0f --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_template_h17p.c @@ -0,0 +1,2723 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Copyright 2021 Alyssa Rosenzweig + * Copyright The Asahi Linux Contributors + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "dcp.h" +#include "dcp-internal.h" +#include "iomfb.h" +#include "iomfb_internal.h" +#include "parser.h" +#include "trace.h" +#include "version_utils.h" + +/* Register defines used in bandwidth setup structure */ +#define REG_DOORBELL_BIT(idx) (2 + (idx)) + +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), timestamp[6]) == 0x30); +static_assert(offsetof(struct DCP_FW_NAME(dcp_swap), flags1) == 0x40); + +struct dcp_wait_cookie { + struct kref refcount; + struct completion done; + u32 status; +}; + +static void release_wait_cookie(struct kref *ref) +{ + struct dcp_wait_cookie *cookie; + cookie = container_of(ref, struct dcp_wait_cookie, refcount); + + kfree(cookie); +} + +DCP_THUNK_OUT(iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); +DCP_THUNK_OUT(iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); +DCP_THUNK_OUT(iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); + +IOMFB_THUNK_INOUT(set_matrix); +IOMFB_THUNK_INOUT(get_color_remap_mode); +IOMFB_THUNK_INOUT(last_client_close); +IOMFB_THUNK_INOUT(abort_swaps_dcp); + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, + struct dcp_swap_submit_req_h17p *request, + dcp_callback_t cb, void *cookie) +{ + const void *data = request; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_serialize_present_h17p(&dcp->present_h17p, request); + if (!iomfb_apply_backlight_h17p(dcp, request, &dcp->present_h17p)) { + if (cb) + cb(dcp, NULL, cookie); + return; + } + data = &dcp->present_h17p; + } + + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), + sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); +} +#else +DCP_THUNK_INOUT(dcp_swap_submit, dcpep_swap_submit, + struct DCP_FW_NAME(dcp_swap_submit_req), + struct DCP_FW_NAME(dcp_swap_submit_resp)); +#endif + +DCP_THUNK_INOUT(dcp_swap_start, dcpep_swap_start, struct DCP_FW_NAME(dcp_swap_start_req), + struct DCP_FW_NAME(dcp_swap_start_resp)); + +DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, + struct DCP_FW_NAME(dcp_set_power_state_req), + struct dcp_set_power_state_resp); + +/* + * H17P firmware expects set_power_state (A473) on the out-of-band command + * channel. Sent on the ordinary command channel it is never answered, which + * strands the power transition and every later call behind it. + */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +#define DCP_POWER_OOB true +#else +#define DCP_POWER_OOB false +#endif + +/* + * H17P firmware is attached to a display the bootloader has already brought + * up, and it does not accept set_digital_out_mode (A413) for that display: + * the call fails (0x8000000b) and the timing generator never starts, so there + * is no vblank and no swap completion. Keep the mode the bootloader + * programmed, which is the panel's native mode, instead of requesting one. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_INHERIT_BOOT_MODE true +#else +#define DCP_INHERIT_BOOT_MODE false +#endif + +/* + * abort_swaps_dcp and last_client_close have no verified H17P method numbers, + * and the H17P table leaves them out. Rather than send the firmware a record + * it would decode as some other call, power-off and sleep skip both and go + * straight to set_power_state, whose number and sizes are known. Power-off + * still sends its clear swap first. + */ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_HAS_CLIENT_TEARDOWN false +#else +#define DCP_HAS_CLIENT_TEARDOWN true +#endif + +DCP_THUNK_INOUT(dcp_set_digital_out_mode, dcpep_set_digital_out_mode, + struct dcp_set_digital_out_mode_req, u32); + +DCP_THUNK_INOUT(dcp_set_display_device, dcpep_set_display_device, u32, u32); + +DCP_THUNK_OUT(dcp_set_display_refresh_properties, + dcpep_set_display_refresh_properties, u32); + +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +DCP_THUNK_INOUT(dcp_late_init_signal, dcpep_late_init_signal, u32, u32); +#else +DCP_THUNK_OUT(dcp_late_init_signal, dcpep_late_init_signal, u32); +#endif +DCP_THUNK_IN(dcp_flush_supports_power, dcpep_flush_supports_power, u32); +DCP_THUNK_OUT(dcp_create_default_fb, dcpep_create_default_fb, u32); +DCP_THUNK_OUT(dcp_start_signal, dcpep_start_signal, u32); +DCP_THUNK_VOID(dcp_setup_video_limits, dcpep_setup_video_limits); +DCP_THUNK_VOID(dcp_set_create_dfb, dcpep_set_create_dfb); + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * Default-framebuffer queries issued from inside start_hardware_boot (D121). + * The getters take a zeroed input of the expected size; the firmware only + * checks the size. + */ +struct dcp_dfb_info_h17p { + u8 data[0x2c]; +} __packed; + +struct dcp_dfb_layout_req_h17p { + u8 data[0xc]; +} __packed; + +struct dcp_dfb_dimensions_h17p { + u32 width; + u32 height; +} __packed; + +DCP_THUNK_INOUT(dcp_get_dfb_compression_info, dcpep_get_dfb_compression_info, + u64, u64); +DCP_THUNK_INOUT(dcp_get_dfb_info, dcpep_get_dfb_info, + struct dcp_dfb_info_h17p, struct dcp_dfb_info_h17p); +DCP_THUNK_INOUT(dcp_get_dfb_layout, dcpep_get_dfb_layout, + struct dcp_dfb_layout_req_h17p, u64); +DCP_THUNK_INOUT(dcp_get_dfb_state, dcpep_get_dfb_state, u32, u64); +DCP_THUNK_IN(dcp_set_dfb_dimensions, dcpep_set_dfb_dimensions, + struct dcp_dfb_dimensions_h17p); +DCP_THUNK_VOID(dcp_commit_dfb_info, dcpep_commit_dfb_info); +DCP_THUNK_OUT(dcp_dfb_query, dcpep_dfb_query, u32); +DCP_THUNK_OUT(dcp_dfb_ready_query, dcpep_dfb_ready_query, u32); + +/* + * Pipe-configuration calls the firmware expects between the boot chain and + * the first swap. Most of them have no known name; they are identified by + * method number and wire size. + */ +struct dcp_pipe_cfg_415 { + u8 data[0xc]; +} __packed; + +/* + * A031 is update_notify_clients_dcp(u32[27]): an array of the DCP + * notification clients the AP wants delivered. Clients that are not + * registered here never send the callbacks they gate. + */ +struct dcp_pipe_cfg_031 { + __le32 clients[27]; +} __packed; +static_assert(sizeof(struct dcp_pipe_cfg_031) == 0x6c); + +DCP_THUNK_INOUT(dcp_pipe_cfg_415, dcpep_pipe_cfg_415, + struct dcp_pipe_cfg_415, struct dcp_pipe_cfg_415); +DCP_THUNK_IN(dcp_pipe_cfg_031, dcpep_pipe_cfg_031, struct dcp_pipe_cfg_031); +DCP_THUNK_INOUT(dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); +DCP_THUNK_OUT(dcp_pipe_query_478, dcpep_pipe_query_478, u32); +DCP_THUNK_OUT(dcp_pipe_query_474, dcpep_pipe_query_474, u32); +DCP_THUNK_IN(dcp_pipe_set_476, dcpep_pipe_set_476, u32); +DCP_THUNK_INOUT(dcp_pipe_set_428, dcpep_pipe_set_428, u32, u32); +#endif + +DCP_THUNK_VOID(dcp_first_client_open, dcpep_first_client_open); + +DCP_THUNK_INOUT(dcp_set_parameter_dcp, dcpep_set_parameter_dcp, + struct dcp_set_parameter_dcp, u32); + +DCP_THUNK_INOUT(dcp_enable_disable_video_power_savings, + dcpep_enable_disable_video_power_savings, u32, int); + +DCP_THUNK_OUT(dcp_is_main_display, dcpep_is_main_display, u32); + +/* DCP callback handlers */ +static void dcpep_cb_nop(struct apple_dcp *dcp) +{ + /* No operation */ +} + +static u8 dcpep_cb_true(struct apple_dcp *dcp) +{ + return true; +} + +static u8 dcpep_cb_false(struct apple_dcp *dcp) +{ + return false; +} + +static u32 dcpep_cb_zero(struct apple_dcp *dcp) +{ + return 0; +} + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +{ + ktime_t now = ktime_get(); + bool retire = true; + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + /* + * Abort semantics are not captured on H17P. If an aborted + * present completes after all, its framebuffers were already + * unarmed and its event signalled; record it and carry on. + */ + if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, + resp->swap_id)) { + dev_warn_ratelimited(dcp->dev, + "completion for aborted present %u ignored\n", + resp->swap_id); + return; + } + dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); + dcp->crashed = true; + return; + } + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + retire = iomfb_present_complete_h17p(dcp); +#endif + trace_iomfb_swap_complete(dcp, resp->swap_id); + dcp->last_swap_id = resp->swap_id; + dcp_swap_watchdog_complete(dcp); + if (retire) { + dcp_release_retained_framebuffers(dcp, resp->swap_id); + dcp_drm_crtc_page_flip(dcp, now); + } + if (READ_ONCE(dcp->crc_enabled)) { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_queue_crc_h17p(dcp, resp->swap_id); + return; + } +#endif + u32 crc32 = 0; + drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); + } +} + +/* special */ +static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) +{ + // ack D100 cb_match_pmu_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + iomfb_a358_vi_set_temperature_hint(dcp, false, + complete_vi_set_temperature_hint, + NULL); + + // return false for deferred ACK + return false; +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P asks the AP to make the default framebuffer surface (D582). The + * firmware expects the AP to register a surface from inside that callback, + * with a nested A472 carrying a 0x22c-byte surface descriptor, and only then + * to report success. Without it the firmware has no surface, programs the + * display engine from a null base and the piodma DART faults. + */ +struct dcp_h17p_dfb_surface_req { + u8 surface[0x22c]; + u8 surface_null; + u8 padding[3]; +} __packed; +static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); + +/* + * A472 has a 4-byte output; with out_len 0 the firmware has nowhere to write + * its result and stops answering. + */ +DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, + struct dcp_h17p_dfb_surface_req, u32); + +static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * Fill the default surface descriptor. Fields not set here are zero. The + * format at +0x0b is the fourcc 'b3a8', stored byte-reversed; +0x35 is the + * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose + * meaning is not known. + */ +static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) +{ + s[0x002] = 0x01; + memcpy(s + 0x00b, "8a3b", 4); + s[0x019] = 0x01; + s[0x01b] = 0x01; + s[0x01c] = 0x01; + s[0x02b] = 0x4d; + s[0x02c] = 0x01; + s[0x035] = 0x02; + s[0x051] = 0x01; + s[0x149] = 0x01; + + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + /* Internal H17P default registration differs from the older profile. */ + memcpy(s + 11, "ARGB", 4); + put_unaligned_le32(0x00e44000, s + 41); + s[53] = BIT(2); + } +} + +/* + * A471 (update_dfb) is never sent: on H17P it makes the coprocessor take an + * AXI read error. Scanout does not need it, since every swap carries its own + * surface. Just acknowledge the callback. + */ +static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_ack_dfb_surface(dcp); +} + +static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + struct dcp_h17p_dfb_surface_req *req; + + trace_iomfb_callback(dcp, tag, __func__); + + /* + * The request is too large for the stack this deep in the RTKit + * receive work (rx work -> iomfb_recv_msg -> dcpep_handle_cb -> here -> + * dcp_push). dcp_push() copies the payload into shared memory before + * returning, so a short-lived allocation is enough. + */ + req = kzalloc_obj(*req); + if (!req) + return true; + + dcp_fill_dfb_surface(dcp, req->surface); + + dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); + kfree(req); + + /* deferred ACK: the A472 completion reports success for us */ + return false; +} +#endif + +static void complete_pmu_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_pmu_service_2 + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_pmu_service_2(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + iomfb_a131_pmu_service_matched(dcp, false, complete_pmu_service_matched, + out); + + // return false for deferred ACK + return false; +} + +static void complete_backlight_service_matched(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + + // ack D206 cb_match_backlight_service + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + + if (!dcp_has_panel(dcp)) { + u8 *succ = out; + *succ = true; + return true; + } + + iomfb_a132_backlight_service_matched(dcp, false, complete_backlight_service_matched, out); + + // return false for deferred ACK + return false; +} + +static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) +{ + switch (prop->id) { + case IOMFB_PROPERTY_NITS: + { +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* The H17P takeover level comes from the powerlog hint interface. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + break; +#endif + if (dcp_has_panel(dcp)) { + dcp->brightness.nits = prop->value / dcp->brightness.scale; + /* notify backlight device of the initial brightness */ + if (!dcp->brightness.bl_dev && dcp->brightness.maximum > 0) + schedule_work(&dcp->bl_register_wq); + trace_iomfb_brightness(dcp, prop->value); + } + break; + } + default: + dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); + } +} + +static struct dcp_get_uint_prop_resp +dcpep_cb_get_uint_prop(struct apple_dcp *dcp, struct dcp_get_uint_prop_req *req) +{ + struct dcp_get_uint_prop_resp resp = (struct dcp_get_uint_prop_resp){ + .value = 0 + }; + + if (dcp->panel.has_mini_led && + memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ + if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { + /* + * TODO: value from j314c, find out if it is temperature in + * centigrade C and which temperature sensor reports it + */ + resp.value = 3029; + resp.ret = true; + } + } + + return resp; +} + +static u8 iomfbep_cb_sr_set_property_int(struct apple_dcp *dcp, + struct iomfb_sr_set_property_int_req *req) +{ + if (memcmp(req->obj, "FMOI", sizeof(req->obj)) == 0) { /* "IOMF */ + if (strncmp(req->key, "Brightness_Scale", sizeof(req->key)) == 0) { + if (!req->value_null) + dcp->brightness.scale = req->value; + } + } + + return 1; +} + +static void iomfbep_cb_set_fx_prop(struct apple_dcp *dcp, struct iomfb_set_fx_prop_req *req) +{ + // TODO: trace this, see if there properties which needs to used later +} + +/* + * Callback to map a buffer allocated with allocate_buf for PIODMA usage. + * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated + * stream of the display DART, rather than the expected DCP DART. + */ +static bool dcpep_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req, + struct DCP_FW_NAME(dcp_map_buf_resp) * resp) +{ + struct dcp_mem_descriptor *memdesc; + struct sg_table *map; + size_t size; + ssize_t ret; + + if (req->buffer >= ARRAY_SIZE(dcp->memdesc)) + goto reject; + + memdesc = &dcp->memdesc[req->buffer]; + map = &memdesc->map; + size = ALIGN(memdesc->size, SZ_16K); + + if (!test_bit(req->buffer, dcp->memdesc_map) || !map->sgl || + memdesc->piodma_mapped || !dcp->iommu_dom) + goto reject; + + /* use the piodma iommu domain to map against the right IOMMU */ + ret = iommu_map_sgtable(dcp->iommu_dom, memdesc->dva, map, + IOMMU_READ | IOMMU_WRITE); + + if (ret != (ssize_t)size) { + dev_err(dcp->dev, + "iommu_map_sgtable() returned %zd instead of %zu\n", + ret, size); + if (ret > 0) + iommu_unmap(dcp->iommu_dom, memdesc->dva, ret); + goto reject; + } + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + memdesc->piodma_mapped = true; + + /* the H17P reply echoes the buffer index */ + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { + .buffer = req->buffer, + .dva = memdesc->dva, + }; +#else + *resp = (struct DCP_FW_NAME(dcp_map_buf_resp)) { .dva = memdesc->dva }; +#endif + return true; + +reject: + dev_err(dcp->dev, "denying map of invalid buffer %llx for piodma\n", + req->buffer); + return false; +} + +static struct DCP_FW_NAME(dcp_map_buf_resp) dcpep_cb_map_piodma(struct apple_dcp *dcp, + struct dcp_map_buf_req *req) +{ + struct DCP_FW_NAME(dcp_map_buf_resp) resp = { 0 }; + + if (dcpep_map_piodma(dcp, req, &resp)) + return resp; + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + resp.buffer = req->buffer; +#else + resp.ret = EINVAL; +#endif + return resp; +} + +static bool dcpep_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *request) +{ + struct dcp_mem_descriptor *memdesc; + size_t size, unmapped; + + if (request->buffer >= ARRAY_SIZE(dcp->memdesc)) { + dev_warn(dcp->dev, "unmap request for out of range buffer %llu\n", + request->buffer); + return false; + } + + memdesc = &dcp->memdesc[request->buffer]; + + if (!test_bit(request->buffer, dcp->memdesc_map) || !memdesc->buf || + !dcp->iommu_dom) + goto not_mapped; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (!memdesc->piodma_mapped) + goto not_mapped; +#endif + + if (memdesc->dva != request->dva) { + dev_warn(dcp->dev, + "unmap buffer %llu address mismatch memdesc.dva:%llx dva:%llx\n", + request->buffer, + memdesc->dva, request->dva); + return false; + } + + /* use the piodma iommu domain to unmap from the right IOMMU */ + size = ALIGN(memdesc->size, SZ_16K); + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, "iommu_unmap() returned %zu instead of %zu\n", + unmapped, size); + return false; + } + memdesc->piodma_mapped = false; + return true; + +not_mapped: + dev_warn(dcp->dev, + "unmap for non-mapped buffer %llu iova:0x%08llx\n", + request->buffer, request->dva); + return false; +} + +static void dcpep_cb_unmap_piodma(struct apple_dcp *dcp, + struct dcp_unmap_buf_resp *request) +{ + dcpep_unmap_piodma(dcp, request); +} + +/* + * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be + * physically contiguous, however we should save the sgtable in case the + * buffer needs to be later mapped for PIODMA. + */ +static struct dcp_allocate_buffer_resp +dcpep_cb_allocate_buffer(struct apple_dcp *dcp, + struct dcp_allocate_buffer_req *req) +{ + struct dcp_allocate_buffer_resp resp = { 0 }; + struct dcp_mem_descriptor allocated = { 0 }; + size_t size, rounded; + u32 id; + int ret; + + /* Validate both wire-size and IOMMU-page rounding before allocating. */ + if (!req->size || req->size > SIZE_MAX || + check_add_overflow((size_t)req->size, (size_t)4095, &rounded)) + return resp; + allocated.size = round_down(rounded, 4096); + if (check_add_overflow(allocated.size, (size_t)SZ_16K - 1, &rounded)) + return resp; + size = round_down(rounded, SZ_16K); + + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); + return resp; + } + + allocated.buf = dma_alloc_coherent(dcp->dev, size, &allocated.dva, + GFP_KERNEL); + if (!allocated.buf) + return resp; + + ret = dma_get_sgtable(dcp->dev, &allocated.map, allocated.buf, + allocated.dva, size); + if (ret) { + dma_free_coherent(dcp->dev, size, allocated.buf, allocated.dva); + return resp; + } + + /* Callbacks are serialized; publish only a fully initialized descriptor. */ + dcp->memdesc[id] = allocated; + set_bit(id, dcp->memdesc_map); + resp.mem_desc_id = id; + resp.dva_size = allocated.size; + resp.dva = allocated.dva; + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * Linux allocation traces correlate the first H17P reply address with + * the first SG page. The remaining pages need not be contiguous. + */ + if (allocated.map.sgl) + resp.paddr = sg_phys(allocated.map.sgl); +#endif + + return resp; +} + +static u8 dcpep_cb_release_mem_desc(struct apple_dcp *dcp, u32 *mem_desc_id) +{ + struct dcp_mem_descriptor *memdesc; + size_t size, unmapped; + u32 id = *mem_desc_id; + + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, + "unmap request for out of range mem_desc_id %u", id); + return 0; + } + + if (!test_bit(id, dcp->memdesc_map)) { + dev_warn(dcp->dev, "unmap request for unused mem_desc_id %u\n", + id); + return 0; + } + + memdesc = &dcp->memdesc[id]; + size = ALIGN(memdesc->size, SZ_16K); + + /* + * Tear down any translation still standing before the pages go back to + * the allocator. The firmware is expected to unmap piodma first, but + * if it releases a descriptor without doing so we would hand the pages + * back while the DART still resolves them -- the display engine would + * then read whoever owns them next. Only the H17P map_piodma handler + * sets piodma_mapped, so older firmware is unaffected. + */ + if (memdesc->piodma_mapped) { + dev_warn(dcp->dev, + "releasing buffer %u while still mapped for piodma; unmapping first\n", + id); + if (!dcp->iommu_dom) + return 0; + unmapped = iommu_unmap(dcp->iommu_dom, memdesc->dva, size); + if (unmapped != size) { + dev_err(dcp->dev, + "failed to unmap buffer %u before release: %zu of %zu\n", + id, unmapped, size); + return 0; + } + memdesc->piodma_mapped = false; + } + clear_bit(id, dcp->memdesc_map); + + if (memdesc->buf) { + sg_free_table(&memdesc->map); + dma_free_coherent(dcp->dev, size, memdesc->buf, memdesc->dva); + memdesc->buf = NULL; + memset(&memdesc->map, 0, sizeof(memdesc->map)); + } else if (memdesc->size) { + /* A register range mapped by map_physical */ + dma_unmap_resource(dcp->dev, memdesc->dva, memdesc->size, + DMA_BIDIRECTIONAL, 0); + memdesc->reg = 0; + } + memdesc->dva = 0; + + memdesc->size = 0; + + return 1; +} + +/* Validate that the specified region is a display register */ +static bool is_disp_register(struct apple_dcp *dcp, u64 start, u64 end) +{ + int i; + + for (i = 0; i < dcp->nr_disp_registers; ++i) { + struct resource *r = dcp->disp_registers[i]; + + if ((start >= r->start) && (end <= r->end)) + return true; + } + + return false; +} + +/* + * Map contiguous physical memory into the DCP's address space. The firmware + * uses this to map the display registers we advertise in + * sr_map_device_memory_with_index, so we bounds check against that to guard + * safe against malicious coprocessors. + */ +static struct dcp_map_physical_resp +dcpep_cb_map_physical(struct apple_dcp *dcp, struct dcp_map_physical_req *req) +{ + u64 size, end; + dma_addr_t dva; + u32 id; + + /* Both values come from the coprocessor; reject wrapping ranges. */ + if (!req->size || check_add_overflow(req->size, 4095ULL, &size) || + check_add_overflow(req->paddr, round_down(size, 4096) - 1, &end) || + !is_disp_register(dcp, req->paddr, end)) { + dev_err(dcp->dev, "refusing to map phys address %llx size %llx\n", + req->paddr, req->size); + return (struct dcp_map_physical_resp){}; + } + size = round_down(size, 4096); + + id = find_first_zero_bit(dcp->memdesc_map, DCP_MAX_MAPPINGS); + if (id >= DCP_MAX_MAPPINGS) { + dev_warn(dcp->dev, "DCP overflowed mapping table, ignoring\n"); + return (struct dcp_map_physical_resp){}; + } + + dva = dma_map_resource(dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); + if (dma_mapping_error(dcp->dev, dva)) { + dev_err(dcp->dev, "failed to map phys address %llx size %llx\n", + req->paddr, size); + return (struct dcp_map_physical_resp){}; + } + + set_bit(id, dcp->memdesc_map); + dcp->memdesc[id].size = size; + dcp->memdesc[id].reg = req->paddr; + dcp->memdesc[id].dva = dva; + + return (struct dcp_map_physical_resp){ + .dva_size = size, + .mem_desc_id = id, + .dva = dva, + }; +} + +static u64 dcpep_cb_get_frequency(struct apple_dcp *dcp) +{ + return clk_get_rate(dcp->clk); +} + +static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *dcp, + struct DCP_FW_NAME(dcp_map_reg_req) *req) +{ + if (req->index >= dcp->nr_disp_registers) { + dev_warn(dcp->dev, "attempted to read invalid reg index %u\n", + req->index); + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ .ret = 1 }; + } else { + struct resource *rsrc = dcp->disp_registers[req->index]; +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P reaches these apertures physically, not through its + * DART, and expects dva == addr in the reply. Trying to + * dma_map_resource() them instead fails and hands the firmware + * DMA_MAPPING_ERROR as a device address. + */ + dma_addr_t dva = rsrc->start; +#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + dma_addr_t dva = dma_map_resource(dcp->dev, rsrc->start, resource_size(rsrc), + DMA_BIDIRECTIONAL, 0); + WARN_ON(dva == DMA_MAPPING_ERROR); +#endif + + return (struct DCP_FW_NAME(dcp_map_reg_resp)){ + .addr = rsrc->start, + .length = resource_size(rsrc), +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .dva = dva, +#endif + }; + } +} + +static struct dcp_read_edt_data_resp +dcpep_cb_read_edt_data(struct apple_dcp *dcp, struct dcp_read_edt_data_req *req) +{ + /* Observe boot-property requests without inventing firmware timings. */ + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) + dev_dbg(dcp->dev, + "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", + (int)sizeof(req->key), req->key, req->count, + req->value[0]); + + return (struct dcp_read_edt_data_resp){ + .value[0] = req->value[0], + .ret = 0, + }; +} + +static void iomfbep_cb_enable_backlight_message_ap_gated(struct apple_dcp *dcp, + u8 *enabled) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* H17P carries the level only in presents; ask for one more. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (dcp_has_panel(dcp) && dcp_backlight_active(dcp) && + dcp_backlight_resend(dcp)) + schedule_work(&dcp->bl_update_wq); + return; + } +#endif + /* + * update backlight brightness on next swap, on non mini-LED displays + * DCP seems to set an invalid iDAC value after coming out of DPMS. + * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" + */ + dcp->brightness.update = true; + /* Backlight handling is only set up for panels described in the DT. */ + if (dcp_has_panel(dcp)) + schedule_work(&dcp->bl_update_wq); +} + +/* Chunked data transfer for property dictionaries */ +static u8 dcpep_cb_prop_start(struct apple_dcp *dcp, u32 *length) +{ + if (dcp->chunks.data != NULL) { + dev_warn(dcp->dev, "ignoring spurious transfer start\n"); + return false; + } + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * This length comes straight off the wire. A bogus value would turn + * the allocation below into a multi-gigabyte GFP_KERNEL request. + * Property blobs are a few KiB, so bound it. + */ + if (*length == 0 || *length > SZ_1M) { + dev_warn(dcp->dev, "rejecting %u byte property transfer\n", + *length); + return false; + } +#endif + + dcp->chunks.length = *length; + dcp->chunks.av_service_epoch = 0; + dcp->chunks.automatic_service_pending = false; + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { + /* Match the receipt's start, not a replacement service at its end. */ + mutex_lock(&dcp->dcpavserv.lock); + dcp->chunks.av_service_epoch = dcp->dcpavserv.epoch; + dcp->chunks.automatic_service_pending = + !dcp->dcpavserv.enabled && !dcp->dcpavserv.service; + dcp_firmware_metadata_begin(dcp); + { + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->chunks.automatic_connection_sequence = dcp->rate_proof.connection_sequence; + dcp->chunks.mode_ticket = dcp_metadata_begin(&dcp->metadata.epoch); + dcp->chunks.mode_ticket.associated = + dcp->metadata.epoch.generation == READ_ONCE(dcp->typec_generation) && + READ_ONCE(dcp->typec_cable_connected); + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + } + mutex_unlock(&dcp->dcpavserv.lock); + } else { + dcp_firmware_metadata_begin(dcp); + } + dcp->chunks.data = kzalloc(*length, GFP_KERNEL); + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "failed to allocate chunks\n"); + return false; + } + + return true; +} + +static u8 dcpep_cb_prop_chunk(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_chunk_req *req) +{ + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious chunk\n"); + return false; + } + + if (req->offset > dcp->chunks.length || req->length > sizeof(req->data) || + req->length > dcp->chunks.length - req->offset) { + dcp->chunks.contiguous = false; + dev_warn(dcp->dev, "ignoring overflowing chunk\n"); + return false; + } + + if (req->offset != dcp->chunks.received) + dcp->chunks.contiguous = false; + if (dcp->chunks.contiguous) + dcp->chunks.received += req->length; + memcpy(dcp->chunks.data + req->offset, req->data, req->length); + return true; +} + +static bool dcpep_process_chunks(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + struct dcp_parse_ctx ctx; + int ret; + + if (!dcp->chunks.data) { + dev_warn(dcp->dev, "ignoring spurious end\n"); + return false; + } + + /* Validate serialized extent outside the metadata spinlock. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) + dcp->chunks.payload_complete = dcp->chunks.contiguous && + dcp_property_complete(dcp->chunks.data, dcp->chunks.received, + dcp->chunks.length); + + /* used just as opaque pointer for tracing */ + ctx.dcp = dcp; + + ret = parse(dcp->chunks.data, dcp->chunks.length, &ctx); + + if (ret) { + dev_warn(dcp->dev, "bad header on dcpav props\n"); + /* A failed replacement cannot keep an unsafe catalog admitted. */ + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + !strcmp(req->key, "TimingElements")) { + unsigned long flags; + + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->rate_proof.catalog_sequence++; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + dcp_180hz_catalog_changed(dcp); + } + return false; + } + + if (!strcmp(req->key, "TimingElements")) { + struct dcp_display_mode *modes; + struct dcp_metadata_ticket rate_ticket = {}; + u64 rate_service_epoch = 0; + unsigned int nr_modes = 0, i; + unsigned long flags; + + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL && + dcp->chunks.payload_complete && dcp->chunks.mode_ticket.associated) { + ctx.automatic_1080p180_rgb8 = true; + rate_ticket = dcp->chunks.mode_ticket; + rate_service_epoch = dcp->chunks.av_service_epoch; + } + modes = enumerate_modes(&ctx, &nr_modes, + dcp->width_mm, dcp->height_mm, + dcp->notch_height, + dcp->fixed_connector_type == + DRM_MODE_CONNECTOR_eDP, + dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL); + + if (IS_ERR(modes)) { + dev_warn(dcp->dev, "failed to parse modes\n"); + modes = NULL; + nr_modes = 0; + } else { + for (i = 0; i < nr_modes; i++) + if (modes[i].automatic_rgb8_quirk) { + modes[i].unsafe_rgb8_ticket = rate_ticket; + modes[i].unsafe_rgb8_service_epoch = rate_service_epoch; + modes[i].automatic_connection_sequence = dcp->chunks.automatic_connection_sequence; + modes[i].automatic_service_pending = dcp->chunks.automatic_service_pending; + } + } + /* Publish one catalog revision with its immutable receipt tickets. */ + spin_lock_irqsave(&dcp->metadata.lock, flags); + dcp->rate_proof.catalog_sequence++; + for (i = 0; i < nr_modes; i++) + if (modes[i].automatic_rgb8_quirk) + modes[i].automatic_catalog_sequence = dcp->rate_proof.catalog_sequence; + dcp->modes = modes; + dcp->nr_modes = nr_modes; + spin_unlock_irqrestore(&dcp->metadata.lock, flags); + /* Notify removals as well as newly eligible modes, outside the lock. */ + dcp_180hz_catalog_changed(dcp); + if (!modes) { + dcp_firmware_metadata_timing(dcp); + return false; + } + if (nr_modes == 0) + dev_warn(dcp->dev, "TimingElements without valid modes!\n"); + dcp_180hz_notify(dcp); + dcp_firmware_metadata_timing(dcp); + } else if (!strcmp(req->key, "DisplayAttributes")) { + struct dcp_parse_ctx metadata_ctx = ctx; + + ret = parse_display_attributes_h17p(&ctx, &dcp->width_mm, + &dcp->height_mm); + + if (ret) { + dev_warn(dcp->dev, "failed to parse display attribs\n"); + return false; + } + + dcp_set_dimensions_h17p(dcp); + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { + struct dcp_display_metadata metadata; + + ret = parse_display_metadata(&metadata_ctx, &metadata); + dcp_firmware_metadata_attributes(dcp, ret ? NULL : &metadata); + } + } + + return true; +} + +static u8 dcpep_cb_prop_end(struct apple_dcp *dcp, + struct dcp_set_dcpav_prop_end_req *req) +{ + u8 resp = dcpep_process_chunks(dcp, req); + + if (dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) + dev_dbg(dcp->dev, + "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", + (int)sizeof(req->key), req->key, dcp->chunks.length, + resp, dcp->nr_modes); + + /* move chunked data to connector to provide it via debugfs */ + dcp_connector_update_dict(dcp->connector, req->key, &dcp->chunks); + dcp->chunks.data = NULL; + dcp->chunks.length = 0; + + return resp; +} + +/* Boot sequence */ +static void boot_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + dev_dbg(dcp->dev, "boot done\n"); + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +static void boot_5b(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_set_display_refresh_properties(dcp, false, boot_done, NULL); +} + +static void boot_5(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A380 right before set_display_refresh_properties */ + dcp_dfb_ready_query(dcp, false, boot_5b, NULL); +#else + boot_5b(dcp, out, cookie); +#endif +} + +static void boot_4b(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + u32 value = dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; +#else + u32 value = 1; +#endif + dcp_late_init_signal(dcp, false, &value, boot_5, NULL); +#else + dcp_late_init_signal(dcp, false, boot_5, NULL); +#endif +} + +static void boot_4(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P expects A033 between flush_supports_power and late_init_signal */ + dcp_dfb_query(dcp, false, boot_4b, NULL); +#else + boot_4b(dcp, out, cookie); +#endif +} + +static void boot_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 v_true = true; + + dcp_flush_supports_power(dcp, false, &v_true, boot_4, NULL); +} + +static void boot_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_setup_video_limits(dcp, false, boot_3, NULL); +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects the default framebuffer to be described between + * create_default_fb and setup_video_limits: + * get_dfb_compression_info -> get_dfb_info -> get_dfb_layout -> + * get_dfb_state -> set_dfb_dimensions(w, h) -> commit_dfb_info + * Without it the firmware never finishes its own default framebuffer setup. + */ +static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_commit_dfb_info(dcp, false, boot_2, NULL); +} + +static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_dimensions_h17p dim; + + /* + * get_dfb_state returns the default framebuffer dimensions, which are + * passed back unchanged to set_dfb_dimensions. + */ + memcpy(&dim, out, sizeof(dim)); + if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); + return; + } + dev_dbg(dcp->dev, "default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); +} + +static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); +} + +static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); +} + +static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_dfb_info_h17p info = {}; + + dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); +} + +static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) +{ + u64 zero = 0; + + dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); +} +#endif + +static void boot_1_5(struct apple_dcp *dcp, void *out, void *cookie) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + dcp_create_default_fb(dcp, false, boot_1_6, NULL); +#else + dcp_create_default_fb(dcp, false, boot_2, NULL); +#endif +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +static void d000_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_channel *ch = &dcp->ch_cb; + u8 *succ = ch->output[ch->depth - 1]; + + *succ = true; + dcp_ack(dcp, DCP_CONTEXT_CB); +} + +/* + * The firmware expects D000 to be answered only after a nested A033 + * (dfb_query), and then with 1. Defer the ACK until that call completes. + */ +static bool dcpep_cb_d000_h17p(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_dfb_query(dcp, false, d000_done, NULL); + return false; +} +#endif + +/* Use special function signature to defer the ACK */ +static bool dcpep_cb_boot_1(struct apple_dcp *dcp, int tag, void *out, void *in) +{ + trace_iomfb_callback(dcp, tag, __func__); + dcp_set_create_dfb(dcp, false, boot_1_5, NULL); + return false; +} + +static struct dcp_allocate_bandwidth_resp dcpep_cb_allocate_bandwidth(struct apple_dcp *dcp, + struct dcp_allocate_bandwidth_req *req) +{ + return (struct dcp_allocate_bandwidth_resp){ + .unk1 = req->unk1, + .unk2 = req->unk2, + .ret = 1, + }; +} + +static struct dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct apple_dcp *dcp) +{ + struct dcp_rt_bandwidth rt_bw = (struct dcp_rt_bandwidth){ + .reg_scratch = 0, + .reg_doorbell = 0, + .doorbell_bit = 0, + }; + + if (dcp->disp_bw_scratch_index) { + u32 offset = dcp->disp_bw_scratch_offset; + u32 index = dcp->disp_bw_scratch_index; + rt_bw.reg_scratch = dcp->disp_registers[index]->start + offset; + } + + if (dcp->disp_bw_doorbell_index) { + u32 index = dcp->disp_bw_doorbell_index; + rt_bw.reg_doorbell = dcp->disp_registers[index]->start; + rt_bw.doorbell_bit = REG_DOORBELL_BIT(dcp->index); + /* + * This is most certainly not padding. t8103-dcp crashes without + * setting this immediately during modeset on 12.3 and 13.5 + * firmware. + */ + rt_bw.padding[3] = 0x4; + } + + return rt_bw; +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P's D006 is 0x54 in / 0x50 out, far wider than the 28-byte + * frame_sync_props of older firmware, so the fields that matter sit past + * anything the old struct could describe. + * + * The firmware expects zeros except for three u32 at 0x24, 0x28 and 0x2c, + * each 0x10000 -- 1.0 in 16.16 fixed point, the identity scale. An all-zero + * reply hands the pipe a zero scale, and it then programs a null surface + * base: the display DART faults at DVA 0 and the coprocessor takes an AXI + * error. + */ +#define DCP_FRAME_SYNC_IDENTITY 0x10000 + +struct dcp_set_frame_sync_props_req_h17p { + u8 unk[0x54]; +} __packed; + +struct dcp_set_frame_sync_props_resp_h17p { + u8 unk[0x50]; +} __packed; + +static struct dcp_set_frame_sync_props_resp_h17p +dcpep_cb_set_frame_sync_props_h17p(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req_h17p *req) +{ + struct dcp_set_frame_sync_props_resp_h17p resp = {}; + static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(identity_at); i++) { + u32 v = DCP_FRAME_SYNC_IDENTITY; + + memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); + } + + return resp; +} +#endif + +static struct dcp_set_frame_sync_props_resp +dcpep_cb_set_frame_sync_props(struct apple_dcp *dcp, + struct dcp_set_frame_sync_props_req *req) +{ + return (struct dcp_set_frame_sync_props_resp){}; +} + +/* Callback to get the current time as milliseconds since the UNIX epoch */ +static u64 dcpep_cb_get_time(struct apple_dcp *dcp) +{ + return ktime_to_ms(ktime_get_real()); +} + +struct dcp_swap_cookie { + struct kref refcount; + struct completion done; + u32 status; + u32 swap_id; +}; + +static void release_swap_cookie(struct kref *ref) +{ + struct dcp_swap_cookie *cookie; + cookie = container_of(ref, struct dcp_swap_cookie, refcount); + + kfree(cookie); +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +#define DCP_SWAP_FLAGS1 0x34865202ULL +#define DCP_SWAP_FLAGS2 0x4ULL + +/* + * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: + * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier + * than on every other firmware. With the older layout the firmware reads the + * format two bytes late and its surface check rejects the swap as an + * unsupported format. + * + * H17G retains this public packing path. The H17P profile uses + * a separate wire record with the SPEC's 0x588 swap boundary. + */ +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + unsigned int i; + + /* + * The firmware expects flags1 = 0x34865202 and flags2 = 4 (at +0x40 + * and +0x48) in every swap. + */ + req->swap.flags1 = DCP_SWAP_FLAGS1; + req->swap.flags2 = DCP_SWAP_FLAGS2; + + /* + * The H17P gap before swap_id carries two small counters the firmware + * expects to be 2 (at +0x50) and 3 (at +0x78). The gap starts at + * +0x50, so these are indices 0 and 0x28 of the blob. + */ + req->swap.h17p_pre_swap_id[0x00] = 2; + req->swap.h17p_pre_swap_id[0x28] = 3; + + /* The H17P layout is serialized without shifting its surfaces. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; + + for (i = 0; i < SWAP_SURFACES; i++) { + u8 *p = (u8 *)&req->surf[i]; + size_t n = sizeof(req->surf[i]); + + memmove(p + 1, p + 3, n - 3); + memset(p + n - 2, 0, 2); + } + + /* + * Everything from surf_iova onward also sits two bytes earlier on + * H17P: surf_iova[0] is at +0xe38 of the 0xe9c-byte record, not + * +0xe3a. Left in place, the firmware reads every IOVA two bytes off + * and the trailing null-flag block misaligned, so it faults on a bogus + * address or verifies layers that are not present. Shift the whole + * block down. + */ + { + u8 *r = (u8 *)req; + size_t n = sizeof(*req); + size_t off = offsetof(typeof(*req), surf_iova); + + static_assert(offsetof(typeof(*req), surf_iova) == 0xe3a, + "H17P source IOVA offset"); + static_assert(sizeof(*req) == 0xe9c, "H17P request size"); + memmove(r + off - 2, r + off, n - off); + memset(r + n - 2, 0, 2); + } +} +#else +static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } +#endif + +static bool dcp_present_retires(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return !iomfb_present_brightness_only_h17p(dcp); +#endif + return true; +} + +static void dcp_present_failed(struct apple_dcp *dcp) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_present_failed_h17p(dcp); +#endif +} + +static bool dcp_present_begin(struct apple_dcp *dcp, u32 swap_id) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_begin_h17p(&dcp->present_state_h17p, swap_id)) { + dev_err(dcp->dev, "overlapping present %u\n", swap_id); + dcp->crashed = true; + return false; + } +#endif + return true; +} + +/* + * Returns 0 if the submitted present stands, -ECANCELED if the firmware + * aborted it before replying (already reported), or -EPROTO if no present + * was expecting this reply. + */ +static int dcp_present_submit(struct apple_dcp *dcp, u32 swap_id, bool accepted) +{ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + !dcp_present_submit_h17p(&dcp->present_state_h17p, swap_id, accepted)) { + dev_err(dcp->dev, "unexpected present submission %u\n", swap_id); + dcp->crashed = true; + return -EPROTO; + } + /* An abort received before this reply turns acceptance into failure. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + accepted && dcp->present_state_h17p.aborted) { + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", + swap_id); + return -ECANCELED; + } +#endif + return 0; +} + +static void dcp_prepare_clear_swap(struct apple_dcp *dcp, void *request) +{ + typeof(DCP_FW_UNION(dcp->swap)) *swap = request; + + /* Clear surfaces. */ + memset(swap, 0, sizeof(*swap)); + + swap->swap.swap_enabled = + swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; + swap->swap.bg_color = 0xFF000000; + + /* + * Turn off the backlight. This matters because the DCP's idea of + * backlight brightness gets desynced after a power change, and it + * needs to be told it's going to turn off so it will consider the + * subsequent update on poweron an actual change and restore the + * brightness. + */ + if (dcp_has_panel(dcp)) { + swap->swap.bl_unk = 1; + swap->swap.bl_value = 0; + swap->swap.bl_power = 0; + } + + /* Null all surfaces */ + for (int l = 0; l < SWAP_SURFACES; l++) + swap->surf_null[l] = true; +#if DCP_FW_VERSION(13, 2, 0) <= DCP_FW_VER +#if DCP_FW_VERSION(26, 0, 0) > DCP_FW_VER + for (int l = 0; l < 5; l++) + swap->surf2_null[l] = true; +#endif + swap->unknown_pointer_null = true; + swap->unknown_output_null = true; +#endif +} + +static void dcp_swap_cleared(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + struct dcp_swap_cookie *info = cookie; + u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; + u32 status = resp ? resp->ret : ~0U; + int ret; + + ret = dcp_present_submit(dcp, swap_id, !status); + if (ret) + status = ~0U; + + if (status) { + /* An aborted present has already been reported. */ + if (ret != -ECANCELED) + dev_err(dcp->dev, "swap_clear failed! status %u\n", status); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } + } + if (info) { + WRITE_ONCE(info->status, status); + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } +} + +static void dcp_swap_clear_started(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; + struct dcp_swap_cookie *info = cookie; + + if (!resp || resp->ret) { + if (info) { + WRITE_ONCE(info->status, resp ? resp->ret : ~0U); + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + if (!dcp_present_begin(dcp, resp->swap_id)) { + if (info) { + WRITE_ONCE(info->status, ~0U); + complete(&info->done); + kref_put(&info->refcount, release_swap_cookie); + } + dcp_drm_crtc_vblank(dcp->crtc); + return; + } +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + dcp_prepare_clear_swap(dcp, &DCP_FW_UNION(dcp->swap)); +#endif + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); + + if (info) + info->swap_id = resp->swap_id; + + dcp_h17p_prepare_swap(dcp); + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swap_cleared, cookie); +} + +static void dcp_on_final(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + out && get_unaligned_le64(out) == 1) + WRITE_ONCE(dcp->pipe_enabled_h17p, true); +#endif + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct DCP_FW_NAME(dcp_set_power_state_req) req = { + .unklong = 1, + }; + + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, dcp_on_final, cookie); +} + +static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .value = { 0 }, +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .count = 3, +#else + .count = 1, +#endif + }; + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_power_state, cookie); +} + +void DCP_FW_NAME(iomfb_poweron)(struct apple_dcp *dcp) +{ + struct dcp_wait_cookie *cookie; + int ret; + u32 handle; + dev_info(dcp->dev, "dcp_poweron() starting\n"); + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + iomfb_opaque_x_reset_h17p(dcp); +#endif + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + if (dcp->main_display) { +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P takes set_display_device (A411) once, in the + * client-open sequence. Repeating it after the client is open + * is a call the firmware stops answering, which wedges the + * command channel and with it every later swap. + */ + dcp_on_set_power_state(dcp, NULL, cookie); +#else + handle = 0; + dcp_set_display_device(dcp, false, &handle, dcp_on_set_power_state, + cookie); +#endif + } else { + handle = 2; + dcp_set_display_device(dcp, false, &handle, + dcp_on_set_parameter, cookie); + } + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(10000)); + + if (ret == 0) { + dev_warn(dcp->dev, "wait for power timed out, connector will be broken\n"); + } else if (ret > 0) { + int msecs = jiffies_to_msecs(ret); + if (msecs > 6000) + dev_info(dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); + else + dev_warn(dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); + } else { + drm_connector_set_link_status_property(&dcp->connector->base, + DRM_MODE_LINK_STATUS_BAD); + dev_warn(dcp->dev, "wait for completion error: %d\n", ret); + } + + kref_put(&cookie->refcount, release_wait_cookie);; + + /* Force a brightness update after poweron, to restore the brightness */ + dcp->brightness.update = true; +} + +static void complete_set_powerstate(struct apple_dcp *dcp, void *out, + void *cookie) +{ + struct dcp_set_power_state_resp *resp = out; + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + WRITE_ONCE(wait->status, resp ? resp->ret : ~0U); + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +static bool dcp_power_stop_confirmed(struct dcp_wait_cookie *cookie) +{ + /* H17P does not confirm that scanout has stopped. */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + return false; +#else + return READ_ONCE(cookie->status) == 0; +#endif +} + +static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, + cookie); +} + +static void aborted_swaps_dcp_poff(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_last_client_close_req last_client_req = {}; + iomfb_last_client_close(dcp, false, &last_client_req, + last_client_closed_poff, cookie); +} + +void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) +{ + int ret, swap_id; + struct iomfb_abort_swaps_dcp_req abort_req = { + .client = { + .flag2 = 1, + }, + }; + struct dcp_swap_cookie *cookie; + struct dcp_wait_cookie *poff_cookie; + struct DCP_FW_NAME(dcp_swap_start_req) swap_req = { 0 }; + typeof(DCP_FW_UNION(dcp->swap)) *swap = &DCP_FW_UNION(dcp->swap); + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* H17P has no known safe pipe-off or surface-free transition. */ + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G) + dcp_prepare_clear_swap(dcp, swap); +#else + dcp_prepare_clear_swap(dcp, swap); +#endif + + dcp_swap_start(dcp, false, &swap_req, dcp_swap_clear_started, cookie); + + /* + * On unplug the firmware powers the external pipe down on its own and + * can take tens of milliseconds before it answers (and swallows) the + * clear swap. That is not a crash: a real one is reported through the + * RTKit crash callback. Wait longer and carry on with the power-off + * either way, otherwise every later modeset fails with -EINVAL. + */ + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); + swap_id = cookie->swap_id; + if (READ_ONCE(cookie->status)) + ret = 0; + kref_put(&cookie->refcount, release_swap_cookie); + if (ret <= 0) + dev_warn(dcp->dev, "%s: clear swap timed out\n", __func__); + else + dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, + swap_id); + + poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); + if (!poff_cookie) + return; + poff_cookie->status = ~0U; + init_completion(&poff_cookie->done); + kref_init(&poff_cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&poff_cookie->refcount); + + if (DCP_HAS_CLIENT_TEARDOWN) { + iomfb_abort_swaps_dcp(dcp, false, &abort_req, + aborted_swaps_dcp_poff, poff_cookie); + } else { + dev_dbg_once(dcp->dev, + "power-off: skipping abort_swaps/last_client_close\n"); + last_client_closed_poff(dcp, NULL, poff_cookie); + } + ret = wait_for_completion_timeout(&poff_cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setPowerState(0) timeout %u ms\n", 1000); + else if (ret > 0) { + dev_dbg(dcp->dev, + "setPowerState(0) finished with %d ms to spare", + jiffies_to_msecs(ret)); + if (dcp_power_stop_confirmed(poff_cookie)) + dcp_release_all_retained_framebuffers(dcp); + else + dev_warn(dcp->dev, "scanout stop was not confirmed\n"); + } + + kref_put(&poff_cookie->refcount, release_wait_cookie); + + dev_info(dcp->dev, "dcp_poweroff() done\n"); +} + +static void last_client_closed_sleep(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct DCP_FW_NAME(dcp_set_power_state_req) power_req = { + .unklong = 0, + }; + dcp_set_power_state(dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); +} + +static void aborted_swaps_dcp_sleep(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct iomfb_last_client_close_req req = { 0 }; + iomfb_last_client_close(dcp, false, &req, last_client_closed_sleep, cookie); +} + +void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) +{ + int ret; + struct iomfb_abort_swaps_dcp_req req = { + .client = { + .flag2 = 1, + }, + }; + + struct dcp_wait_cookie *cookie; + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + return; +#endif + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) + return; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + if (DCP_HAS_CLIENT_TEARDOWN) { + iomfb_abort_swaps_dcp(dcp, false, &req, aborted_swaps_dcp_sleep, + cookie); + } else { + dev_dbg_once(dcp->dev, + "sleep: skipping abort_swaps/last_client_close\n"); + last_client_closed_sleep(dcp, NULL, cookie); + } + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(1000)); + + if (ret == 0) + dev_warn(dcp->dev, "setDCPPower(0) timeout %u ms\n", 1000); + + kref_put(&cookie->refcount, release_wait_cookie); + dev_info(dcp->dev, "dcp_sleep() done\n"); +} + +static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) +{ + struct apple_connector *connector = dcp->connector; + unsigned int action; + + /* DCP issues hotplug_gated callbacks after SetPowerState() calls on + * devices with display (macbooks, imacs). This must not result in + * connector state changes on DRM side. Some applications won't enable + * a CRTC with a connector in disconnected state. Weston after DPMS off + * is one example. dcp_is_main_display() returns true on devices with + * integrated display. Ignore the hotplug_gated() callbacks there. + */ + if (dcp->main_display) + return; + /* + * Report firmware hotplug independently of the USB4 PHY. + * Reassigning lpdptxphy blanked eDP even with these callbacks ignored; + * suppressing connector notifications does not protect the panel. + * Mode probing still uses this DCP's firmware modes, and mode_valid + * rejects modes absent from that list. Do not synthesize a mode here. + */ + + /* + * Same for the unplug a Type-C output reports after its CRTC was + * powered off with the cable still attached (see dcp_poweroff()). + */ + if (!(*connected) && READ_ONCE(dcp->typec_crtc_off) && + READ_ONCE(dcp->typec_cable_connected)) { + dev_dbg(dcp->dev, "cb_hotplug() ignoring unplug of powered-off Type-C output\n"); + dcp_mode_invalidate(&dcp->mode_state); + schedule_work(&dcp->vblank_wq); + return; + } + if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) + complete_all(&dcp->typec_iomfb_hpd_ready); + + if (dcp_external_hotplug(dcp, !!(*connected))) + return; + + dcp_firmware_metadata_hotplug(dcp, !!(*connected)); + action = dcp_mode_hotplug(&dcp->mode_state, !!(*connected), + connector ? &connector->connected : NULL); + /* + * A Type-C sink can assert HPD only after a modeset has already failed, + * as a TV behind a DP-to-HDMI converter does when it wakes from standby. + * The connector state does not change then; re-apply the mode anyway. + */ + if (*connected && dcp_is_typec_output(dcp) && + !READ_ONCE(dcp->mode_state.valid) && + !READ_ONCE(dcp->mode_state.changing)) { + dcp->swap_watchdog_retrains = 0; + action |= DCP_HOTPLUG_NOTIFY; + } + dcp_handle_hotplug_actions(dcp, action); +} + +static void +dcpep_cb_swap_complete_intent_gated(struct apple_dcp *dcp, + struct dcp_swap_complete_intent_gated *info) +{ + trace_iomfb_swap_complete_intent_gated(dcp, info->swap_id, + info->width, info->height); +} + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +/* + * An aborted H17P present never completes. Finish it like a rejected one: + * keep the displaced framebuffers for the next completed present and signal + * the DRM event, so neither the queue nor userspace waits for a completion. + */ +static void dcp_present_aborted(struct apple_dcp *dcp, u32 swap_id) +{ + if (!dcp_present_abort_h17p(&dcp->present_state_h17p, swap_id)) { + dev_warn_ratelimited(dcp->dev, + "abort for present %u, which is not in flight\n", + swap_id); + return; + } + /* The submit reply has not arrived; it finishes the present. */ + if (dcp->present_state_h17p.pending) + return; + + dev_warn_ratelimited(dcp->dev, "firmware aborted present %u\n", swap_id); + /* No completion follows, so the swap watchdog must not wait for one. */ + dcp_swap_watchdog_complete(dcp); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } +} +#endif + +static void +dcpep_cb_abort_swap_ap_gated(struct apple_dcp *dcp, u32 *swap_id) +{ + trace_iomfb_abort_swap_ap_gated(dcp, *swap_id); +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + dcp_present_aborted(dcp, *swap_id); +#endif +} + +static struct dcpep_get_tiling_state_resp +dcpep_cb_get_tiling_state(struct apple_dcp *dcp, + struct dcpep_get_tiling_state_req *req) +{ + return (struct dcpep_get_tiling_state_resp){ + .value = 0, + .ret = 1, + }; +} + +static u8 dcpep_cb_create_backlight_service(struct apple_dcp *dcp) +{ +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* + * H17P asks during start_ios_gated, before any mode is known, but + * dcp_has_panel() only becomes true once the panel dimensions have been + * parsed. An internal panel always has a backlight, so answer for the + * connector; answering false makes the firmware report that it could + * not find the backlight service. + */ + return dcp_has_panel(dcp) || + dcp->connector_type == DRM_MODE_CONNECTOR_eDP; +#else + return dcp_has_panel(dcp); +#endif +} + +TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); +TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); +TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); +TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); +TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, + struct DCP_FW_NAME(dc_swap_complete_resp)); +TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, + struct dcp_get_uint_prop_req, struct dcp_get_uint_prop_resp); +TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, + struct iomfb_set_fx_prop_req) +TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, + struct dcp_map_buf_req, struct DCP_FW_NAME(dcp_map_buf_resp)); +TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, + struct dcp_unmap_buf_resp); +TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, + struct iomfb_sr_set_property_int_req, u8); +TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, + struct dcp_allocate_buffer_req, + struct dcp_allocate_buffer_resp); +TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, + struct dcp_map_physical_req, struct dcp_map_physical_resp); +TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, u32, + u8); +TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, + struct DCP_FW_NAME(dcp_map_reg_req), + struct DCP_FW_NAME(dcp_map_reg_resp)); +TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, + struct dcp_read_edt_data_req, struct dcp_read_edt_data_resp); +TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); +TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, + struct dcp_set_dcpav_prop_chunk_req, u8); +TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, + struct dcp_set_dcpav_prop_end_req, u8); +TRAMPOLINE_INOUT(trampoline_allocate_bandwidth, dcpep_cb_allocate_bandwidth, + struct dcp_allocate_bandwidth_req, struct dcp_allocate_bandwidth_resp); +TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, + struct dcp_rt_bandwidth); +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, + struct dcp_set_frame_sync_props_req, + struct dcp_set_frame_sync_props_resp); +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +TRAMPOLINE_INOUT(trampoline_set_frame_sync_props_h17p, + dcpep_cb_set_frame_sync_props_h17p, + struct dcp_set_frame_sync_props_req_h17p, + struct dcp_set_frame_sync_props_resp_h17p); +#endif +TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); +TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); +TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); +TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, + dcpep_cb_swap_complete_intent_gated, + struct dcp_swap_complete_intent_gated); +TRAMPOLINE_IN(trampoline_abort_swap_ap_gated, dcpep_cb_abort_swap_ap_gated, u32); +TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, + iomfbep_cb_enable_backlight_message_ap_gated, u8); +TRAMPOLINE_IN(trampoline_pr_publish, iomfb_cb_pr_publish, + struct iomfb_property); +TRAMPOLINE_INOUT(trampoline_get_tiling_state, dcpep_cb_get_tiling_state, + struct dcpep_get_tiling_state_req, struct dcpep_get_tiling_state_resp); +TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_service, u8); + +/* + * Callback for swap requests. If a swap failed, we'll never get a swap + * complete event so we need to fake a vblank event early to avoid a hang. + */ + +static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) +{ + int ret; + struct DCP_FW_NAME(dcp_swap_submit_resp) *resp = data; + u32 swap_id = DCP_FW_UNION(dcp->swap).swap.swap_id; + u32 status = resp ? resp->ret : ~0U; + + ret = dcp_present_submit(dcp, swap_id, !status); + if (ret) + status = ~0U; + + if (status) { + /* An aborted present has already been reported. */ + if (ret != -ECANCELED) + dev_err(dcp->dev, "swap failed! status %u\n", status); + dcp_present_failed(dcp); + if (dcp_present_retires(dcp)) { + dcp_unarm_retained_framebuffers(dcp, swap_id); + dcp_drm_crtc_vblank(dcp->crtc); + } + return; + } + dcp->swap_start = ktime_get(); + dcp->swap_submit_timestamp = arch_timer_read_counter(); + dcp_swap_watchdog_arm(dcp); +} + +static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; + + if (!resp || resp->ret) { + dev_err(dcp->dev, "swap_start was rejected\n"); + if (dcp_present_retires(dcp)) + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + if (!dcp_present_begin(dcp, resp->swap_id)) { + if (dcp_present_retires(dcp)) + dcp_drm_crtc_vblank(dcp->crtc); + return; + } + DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; + if (dcp_present_retires(dcp)) + dcp_arm_retained_framebuffers(dcp, resp->swap_id); + + trace_iomfb_swap_submit(dcp, resp->swap_id); + dcp_h17p_prepare_swap(dcp); + dcp_swap_submit(dcp, false, &DCP_FW_UNION(dcp->swap), dcp_swapped, NULL); +} + +/* Helpers to modeset and swap, used to flush */ +static void do_swap(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct DCP_FW_NAME(dcp_swap_start_req) start_req = { 0 }; + + if (READ_ONCE(dcp->mode_state.valid) && dcp->connector && + READ_ONCE(dcp->connector->connected)) + dcp_swap_start(dcp, false, &start_req, dcp_swap_started, NULL); + else + dcp_drm_crtc_vblank(dcp->crtc); +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in + * 16.16 fixed point), the same identity value the D006 reply carries. + */ +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +void iomfb_present_backlight_h17p(struct apple_dcp *dcp) +{ + do_swap(dcp, NULL, NULL); +} +#endif + +static void dcp_post_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + do_swap(dcp, out, cookie); +} + +static void dcp_pre_swap_a428(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 identity = 0x10000; + + dcp_pipe_set_428(dcp, false, &identity, dcp_post_swap_a428, cookie); +} +#endif + +static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, + void *cookie) +{ + struct dcp_wait_cookie *wait = cookie; + + if (wait) { + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + } +} + +/* DCP applies Adaptive Sync changes when the display mode is reselected. */ +static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, + void *cookie) +{ + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); +} + +static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 min_vrr, + void *cookie) +{ + struct dcp_set_parameter_dcp param = { + .param = IOMFBPARAM_ADAPTIVE_SYNC, + .value = { + min_vrr, /* minRR, 16.16 fixed-point Hz */ + 0, /* mediaTargetRate */ + 0, /* fractional rate */ + }, +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) + .count = 3, +#else + .count = 1, +#endif + }; + + dcp_set_parameter_dcp(dcp, false, ¶m, dcp_on_set_adaptive_sync, + cookie); +} + +int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, + struct drm_crtc_state *crtc_state) +{ + struct dcp_display_mode *mode; + struct dcp_wait_cookie *cookie; + struct dcp_color_mode *cmode = NULL; + int ret; + + mode = lookup_mode_h17p(dcp, &crtc_state->mode); + if (!mode) { + dev_err(dcp->dev, "no match for " DRM_MODE_FMT "\n", + DRM_MODE_ARG(&crtc_state->mode)); + return -EIO; + } + + dev_info(dcp->dev, + "set_digital_out_mode(color:%d timing:%d) " DRM_MODE_FMT "\n", + mode->color_mode_id, mode->timing_mode_id, + DRM_MODE_ARG(&crtc_state->mode)); + if (mode->color_mode_id == mode->sdr_rgb.id) + cmode = &mode->sdr_rgb; + else if (mode->color_mode_id == mode->sdr_444.id) + cmode = &mode->sdr_444; + else if (mode->color_mode_id == mode->sdr.id) + cmode = &mode->sdr; + else if (mode->color_mode_id == mode->best.id) + cmode = &mode->best; + if (cmode) + dev_info(dcp->dev, + "set_digital_out_mode() color mode depth:%hhu format:%u " + "colorimetry:%u eotf:%u range:%u vrr:%u\n", cmode->depth, + cmode->format, cmode->colorimetry, cmode->eotf, + cmode->range, mode->vrr); + + dcp->mode = (struct dcp_set_digital_out_mode_req){ + .color_mode_id = mode->color_mode_id, + .timing_mode_id = mode->timing_mode_id + }; + + /* Built-in ProMotion panels require timestamps even in fixed-120 mode. */ + dcp->use_timestamps = mode->vrr && dcp->main_display; + + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); + if (!cookie) { + return -ENOMEM; + } + + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + /* increase refcount to ensure the receiver has a reference */ + kref_get(&cookie->refcount); + + dcp->swap_submit_timestamp = 0; + + if (DCP_INHERIT_BOOT_MODE && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL) { + dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); + complete_set_digital_out_mode(dcp, NULL, cookie); + } else if (mode->vrr && + dcp->hw.iomfb_method_profile != + DCP_IOMFB_METHODS_H17P_EXTERNAL) + dcp_set_adaptive_sync(dcp, + crtc_state->vrr_enabled ? mode->min_vrr : 0, + cookie); + else + dcp_set_digital_out_mode(dcp, false, &dcp->mode, + complete_set_digital_out_mode, cookie); + + /* + * The DCP firmware has an internal timeout of ~8 seconds for + * modesets. Add an extra 500ms to safe side that the modeset + * call has returned. + */ + ret = wait_for_completion_timeout(&cookie->done, + msecs_to_jiffies(8500)); + + dev_info(dcp->dev, "set_digital_out_mode finished:%d\n", ret); + + if (ret == 0) { + dev_info(dcp->dev, "set_digital_out_mode timed out\n"); + kref_put(&cookie->refcount, release_wait_cookie); + return -EIO; + } else if (ret < 0) { + dev_info(dcp->dev, + "waiting on set_digital_out_mode failed:%d\n", ret); + kref_put(&cookie->refcount, release_wait_cookie); + return -EIO; + } else { + dev_dbg(dcp->dev, + "set_digital_out_mode finished with %d to spare\n", + jiffies_to_msecs(ret)); + } + kref_put(&cookie->refcount, release_wait_cookie); + dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; + + return 0; +} + +void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, struct drm_atomic_state *state) +{ + struct drm_plane *plane; + struct drm_plane_state *new_state, *old_state; + struct drm_crtc_state *crtc_state; + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); + int plane_idx, l; + int has_surface = 0; + bool update_brightness; + + crtc_state = drm_atomic_get_new_crtc_state(state, crtc); + + /* Reset all surfaces to defaults */ + memset(req, 0, sizeof(*req)); + for (l = 0; l < SWAP_SURFACES; l++) + req->surf_null[l] = true; +#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) +#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) + for (l = 0; l < 5; l++) + req->surf2_null[l] = true; +#endif + req->unknown_pointer_null = true; + req->unknown_output_null = true; +#endif + + /* + * Clear all surfaces on startup. The boot framebuffer in surface 0 + * sticks around. + */ + if (!dcp->surfaces_cleared) { + req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0x7; + req->swap.bg_color = 0xFF000000; + dcp->surfaces_cleared = true; + } + + for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { + struct apple_plane_state *apple_state = to_apple_plane_state(new_state); + struct apple_plane *apl_plane = to_apple_plane(plane); + + /* skip planes not for this crtc */ + if (old_state->crtc != crtc && new_state->crtc != crtc) + continue; + + l = apl_plane->iomfb_surf; + req->swap.swap_enabled |= BIT(l); + + if (!new_state->fb || !new_state->visible) { + continue; + } + req->surf_null[l] = false; + has_surface = 1; + + req->swap.src_rect[l] = apple_state->src_rect; + req->swap.dst_rect[l] = apple_state->dst_rect; + + if (dcp->notch_height > 0) + req->swap.dst_rect[l].y += dcp->notch_height; + + req->surf_iova[l] = apple_state->iova; + req->surf[l].base = apple_state->surf; + + /* Use sRGB colorspace only for internal panels. External + * displays are expected to have EDID and user space can use + * the contained colorimetry information to provide native + * colors. + */ + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP && + req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) + req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + /* H17P external SDR RGB needs an sRGB source tag. Native + * tagging washes out the same pixel values on this profile. + * Keep YUV, other source spaces and other profiles unchanged. + */ + if (dcp->hw.iomfb_method_profile == + DCP_IOMFB_METHODS_H17P_EXTERNAL && + !new_state->fb->format->is_yuv && + req->surf[l].base.colorspace == DCP_COLORSPACE_NATIVE) + req->surf[l].base.colorspace = DCP_COLORSPACE_BG_SRGB; +#endif + } + + if (!has_surface && !crtc_state->color_mgmt_changed) { + if (crtc_state->enable && crtc_state->active && + !crtc_state->planes_changed) { + schedule_work(&dcp->vblank_wq); + return; + } + + /* Set black background */ + req->swap.swap_enabled |= IOMFB_SET_BACKGROUND; + req->swap.bg_color = 0xFF000000; + req->clear = 1; + } + + if (has_surface && (dcp->use_timestamps || dcp->vrr_enabled)) { + u64 submit_timestamp = dcp->swap_submit_timestamp; + u64 timestamp = arch_timer_read_counter(); + + /* + * IOMobileFramebuffer uses Mach continuous-time values here. On + * Apple Silicon that is the ARM architectural counter. Empirical + * testing shows that using the current submission time together + * with the previous accepted swap makes DCP follow swap pacing. + */ + if (!submit_timestamp) + submit_timestamp = timestamp; + + /* Firmware 12.x/13.x requires timestamp types 1, 2, and 7. */ + req->swap.timestamp[0] = timestamp; + req->swap.timestamp[1] = submit_timestamp; + req->swap.timestamp[6] = timestamp; + + trace_iomfb_vrr_timestamps(dcp, dcp->vrr_enabled, timestamp, + submit_timestamp); + } + + /* These fields should be set together */ + req->swap.swap_completed = req->swap.swap_enabled; + + /* update brightness if changed */ + update_brightness = dcp_has_panel(dcp) && dcp->brightness.update; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + update_brightness = update_brightness && + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; +#endif + if (update_brightness) { + req->swap.bl_unk = 1; + req->swap.bl_value = dcp->brightness.dac; + req->swap.bl_power = 0x40; + dcp->brightness.update = false; + } + + if (crtc_state->color_mgmt_changed) { + struct iomfb_set_matrix_req mat = { + .location = 9, + }; + + if (crtc_state->ctm) { + struct drm_color_ctm *ctm = (struct drm_color_ctm *)crtc_state->ctm->data; + memcpy(mat.matrix, ctm->matrix, sizeof(mat.matrix)); + } else { + mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; + } + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) + iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); +#else + iomfb_set_matrix(dcp, false, &mat, do_swap, NULL); +#endif + } else + do_swap(dcp, NULL, NULL); +} + +static void res_is_main_display(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct apple_connector *connector; + bool result; + + /* No reply: the DCP is marked crashed, so only end the start-up wait. */ + if (!out) { + complete(&dcp->start_done); + return; + } + + /* + * The reply is a one-byte boolean. The firmware leaves the rest of + * the 32-bit slot alone, and dcp_push() fills that with 0xff. + */ + result = *(u8 *)out; + + dev_info(dcp->dev, "DCP is_main_display: %d\n", result); + + dcp->main_display = result; + + connector = dcp->connector; + if (connector) { + connector->connected = dcp->nr_modes > 0; + schedule_work(&connector->hotplug_wq); + } + + dcp->active = true; + complete(&dcp->start_done); +} + +static void init_3(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_is_main_display(dcp, false, res_is_main_display, NULL); +} + +#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) +/* + * The client-open sequence H17P firmware expects before the first swap, in + * H17P method numbers (dcp_push() applies the H17G aliases): + * + * A445 A411 A445 A415 A450 A031 x9 A414 A478 A457 A474 A478 A476 A478 A412 + * + * A412 is not in the table: its result feeds res_is_main_display(), so it + * stays on the existing path. + */ +enum dcp_client_step_kind { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A415, DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, + DCP_STEP_A474, DCP_STEP_A476, +}; + +static const u8 dcp_client_open_seq[] = { + DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A445, DCP_STEP_A415, + DCP_STEP_A450, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, + DCP_STEP_A031, + DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, DCP_STEP_A474, + DCP_STEP_A478, DCP_STEP_A476, DCP_STEP_A478, +}; + +static void dcp_client_open_run(struct apple_dcp *dcp, void *out, void *cookie) +{ + unsigned long idx = (unsigned long)cookie; + dcp_callback_t next = dcp_client_open_run; + void *ncookie = (void *)(idx + 1); + + if (idx >= ARRAY_SIZE(dcp_client_open_seq)) { + init_3(dcp, NULL, NULL); + return; + } + + switch (dcp_client_open_seq[idx]) { + case DCP_STEP_A445: { + u32 zero = 0; + + dcp_get_dfb_state(dcp, false, &zero, next, ncookie); + break; + } + case DCP_STEP_A411: { + u32 handle = 0; + + dcp_set_display_device(dcp, false, &handle, next, ncookie); + break; + } + case DCP_STEP_A415: { + struct dcp_pipe_cfg_415 cfg = {}; + + dcp_pipe_cfg_415(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A450: { + u32 val = 0; + + dcp_enable_disable_video_power_savings(dcp, false, &val, next, + ncookie); + break; + } + case DCP_STEP_A031: { + /* + * Nine cumulative states: each call repeats the previous set + * of notification clients and adds one, so state n lists n + * clients. + */ + static const u8 notify_clients[9][8] = { + {}, + { 6 }, + { 6, 7 }, + { 6, 7, 8 }, + { 6, 7, 8, 10 }, + { 6, 7, 8, 10, 11 }, + { 6, 7, 8, 10, 11, 12 }, + { 6, 7, 8, 10, 11, 12, 25 }, + { 6, 7, 8, 10, 11, 12, 25, 26 }, + }; + struct dcp_pipe_cfg_031 cfg = {}; + unsigned int step = dcp->a031_step; + unsigned int i; + + if (step >= ARRAY_SIZE(notify_clients)) + step = ARRAY_SIZE(notify_clients) - 1; + for (i = 0; i < step; i++) + cfg.clients[i] = cpu_to_le32(notify_clients[step][i]); + dcp->a031_step++; + + dcp_pipe_cfg_031(dcp, false, &cfg, next, ncookie); + break; + } + case DCP_STEP_A414: { + u64 val = 0; + + dcp_pipe_cfg_414(dcp, false, &val, next, ncookie); + break; + } + case DCP_STEP_A478: + dcp_pipe_query_478(dcp, false, next, ncookie); + break; + case DCP_STEP_A457: + dcp_first_client_open(dcp, false, next, ncookie); + break; + case DCP_STEP_A474: + dcp_pipe_query_474(dcp, false, next, ncookie); + break; + case DCP_STEP_A476: { + /* + * A476 takes a single u32, which the firmware expects to be 1. + * Its name is not known for certain: the shape (in 4, out 0) + * fits both set_dcp_clamshellstate() and + * set_has_frame_swap_function(). + */ + u32 state = 1; + + dcp_pipe_set_476(dcp, false, &state, next, ncookie); + break; + } + default: + dev_warn(dcp->dev, "unknown client-open step %u\n", + dcp_client_open_seq[idx]); + init_3(dcp, NULL, NULL); + break; + } +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + /* every client-open sequence starts from the empty A031 state */ + dcp->a031_step = 0; + dcp_client_open_run(dcp, NULL, (void *)0UL); +} +#else +static void init_2(struct apple_dcp *dcp, void *out, void *cookie) +{ + dcp_first_client_open(dcp, false, init_3, NULL); +} + +static void init_1(struct apple_dcp *dcp, void *out, void *cookie) +{ + u32 val = 0; + dcp_enable_disable_video_power_savings(dcp, false, &val, init_2, NULL); +} +#endif + +static void dcp_started(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct iomfb_get_color_remap_mode_req color_remap = + (struct iomfb_get_color_remap_mode_req){ + .mode = 6, + }; + + dev_info(dcp->dev, "DCP booted\n"); + + iomfb_get_color_remap_mode(dcp, false, &color_remap, init_1, cookie); +} + +void DCP_FW_NAME(iomfb_shutdown)(struct apple_dcp *dcp) +{ + struct DCP_FW_NAME(dcp_set_power_state_req) req = { + /* defaults are ok */ + }; + struct dcp_wait_cookie *cookie; + int ret; + +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + iomfb_queue_stop(dcp); + return; + } +#endif + + cookie = kzalloc_obj(*cookie); + if (!cookie) + return; + cookie->status = ~0U; + init_completion(&cookie->done); + kref_init(&cookie->refcount); + kref_get(&cookie->refcount); + dcp_set_power_state(dcp, DCP_POWER_OOB, &req, complete_set_powerstate, cookie); + ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); + if (ret > 0 && dcp_power_stop_confirmed(cookie)) + dcp_release_all_retained_framebuffers(dcp); + else + dev_warn(dcp->dev, "shutdown scanout stop was not confirmed\n"); + kref_put(&cookie->refcount, release_wait_cookie); +} diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index d7dcc2ca70fa66..0e39ab4b7478e6 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -2308,7 +2308,7 @@ static int dcp_v14_mode(struct apple_dcp *dcp, struct apple_dcp_v14 *v14) return ret; ctx.dcp = dcp; modes = enumerate_modes(&ctx, &count, dcp->panel.width_mm, dcp->panel.height_mm, - dcp->notch_height, true); + dcp->notch_height, true, false); if (IS_ERR(modes)) return PTR_ERR(modes); for (i = 0; i < count; i++) { diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_tpl.c b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.c index f59c016cd95c0a..b634d86bde2770 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_tpl.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.c @@ -11,6 +11,7 @@ #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7_tpl.h" +#include "iomfb_h17p.h" #include "version_utils.h" static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_tpl.h b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.h index 888ff9ef4dc966..47590b4354232a 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7_tpl.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7_tpl.h @@ -13,6 +13,15 @@ #include "version_utils.h" +/* + * dcp-internal.h includes this header, also from inside another template + * instantiation (iomfb_h17p.c): keep that one's DCP_FW. + */ +#pragma push_macro("DCP_FW") +#pragma push_macro("DCP_FW_VER") +#undef DCP_FW +#undef DCP_FW_VER + #define DCP_FW v14_7_0 #define DCP_FW_VER DCP_FW_VERSION(14, 7, 0) @@ -21,4 +30,7 @@ #undef DCP_FW_VER #undef DCP_FW +#pragma pop_macro("DCP_FW_VER") +#pragma pop_macro("DCP_FW") + #endif /* __APPLE_IOMFB_V14_7_TPL_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v26_6.c b/drivers/gpu/drm/apple/iomfb_v26_6.c index a65d88d89087e6..5a26c08d2d6ef6 100644 --- a/drivers/gpu/drm/apple/iomfb_v26_6.c +++ b/drivers/gpu/drm/apple/iomfb_v26_6.c @@ -1302,7 +1302,7 @@ static int dcp_v26_mode(struct apple_dcp *dcp, struct apple_dcp_v26 *v26) return ret; ctx.dcp = dcp; modes = enumerate_modes(&ctx, &count, dcp->panel.width_mm, dcp->panel.height_mm, - dcp->notch_height, true); + dcp->notch_height, true, false); if (IS_ERR(modes)) return PTR_ERR(modes); for (i = 0; i < count; i++) { diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index 26ef32e13a4796..96a119bff535e0 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -182,7 +182,7 @@ static struct dcp_display_mode *parse_blob(struct parser_blob *blob, if (ret) return ERR_PTR(ret); ctx.dcp = &parser_dcp; - return enumerate_modes(&ctx, count, 500, 300, notch, internal); + return enumerate_modes(&ctx, count, 500, 300, notch, internal, false); } static void parser_score_presence(struct kunit *test) diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index ad5563cf0b8e50..2dfd85a19be3cf 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -87,19 +87,14 @@ static const struct dcp_parse_tag *parse_tag_of_type(struct dcp_parse_ctx *ctx, static int skip_depth(struct dcp_parse_ctx *handle, unsigned int depth) { const struct dcp_parse_tag *tag; - const void *data; - unsigned int count, i; - int ret; - - if (depth >= DCP_PARSE_MAX_DEPTH) - return -EINVAL; + int ret = 0; + int i; if (depth > 64) return -E2BIG; tag = parse_tag(handle); - tag = parse_tag(handle); if (IS_ERR(tag)) return PTR_ERR(tag); @@ -123,12 +118,7 @@ static int skip_depth(struct dcp_parse_ctx *handle, unsigned int depth) return ret; } - for (i = 0; i < count; ++i) { - ret = skip_value(handle, depth + 1); - if (ret) - return ret; - } - return 0; + return ret; case DCP_TYPE_INT64: return PTR_ERR_OR_ZERO(parse_bytes(handle, sizeof(s64))); @@ -143,13 +133,6 @@ static int skip_depth(struct dcp_parse_ctx *handle, unsigned int depth) default: return -EINVAL; } - - return IS_ERR(data) ? PTR_ERR(data) : 0; -} - -static int skip(struct dcp_parse_ctx *handle) -{ - return skip_value(handle, 0); } static int skip(struct dcp_parse_ctx *handle) @@ -173,6 +156,7 @@ static bool consume_string(struct dcp_parse_ctx *ctx, const char *specimen) { const struct dcp_parse_tag *tag; const char *key; + ctx->pos = round_up(ctx->pos, 4); if (ctx->pos + sizeof(*tag) + strlen(specimen) - 1 > ctx->len) return false; @@ -324,7 +308,6 @@ int parse(const void *blob, size_t size, struct dcp_parse_ctx *ctx) if (size > U32_MAX) return -E2BIG; - return -EINVAL; *ctx = (struct dcp_parse_ctx) { .blob = blob, @@ -358,27 +341,31 @@ bool dcp_property_complete(const void *blob, size_t received, size_t advertised) return ctx.pos == received || round_up(ctx.pos, 4) == received; } -static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, bool polarity) +/* + * @polarity: external H17P only, also read SyncPolarity and PixelRepetition. + * An automatic 1080p180 exception catalog also records duplicate keys. + */ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, - bool require_rate) + bool require_rate, bool polarity) { - static const char * const fields[] = { "Active", "Total", "FrontPorch", "SyncWidth", "PreciseSyncRate" }; + static const char * const dup_fields[] = { + "Active", "Total", "FrontPorch", "SyncWidth", "PreciseSyncRate", + }; unsigned int seen = 0; struct iterator it; unsigned int fields = 0; int ret = 0; - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; + dim->sync_polarity = -1; + dim->pixel_repetition = -1; - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); unsigned int field; if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) - for (field = 0; field < 5; field++) - if (!strcmp(key, fields[field])) { + for (field = 0; field < ARRAY_SIZE(dup_fields); field++) + if (!strcmp(key, dup_fields[field])) { handle->ambiguous_180hz |= !!(seen & (1U << field)); seen |= 1U << field; } @@ -401,20 +388,7 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, ret = parse_int_bound(it.handle, &dim->precise_sync_rate, require_rate ? 1 : 0, U32_MAX); fields |= BIT(4); - } else { - ret = skip(it.handle); - } - else if (!strcmp(key, "Active")) - ret = parse_int_bound(it.handle, &dim->active, 0, U16_MAX); - else if (!strcmp(key, "Total")) - ret = parse_int_bound(it.handle, &dim->total, 0, U16_MAX); - else if (!strcmp(key, "FrontPorch")) - ret = parse_int_bound(it.handle, &dim->front_porch, 0, U16_MAX); - else if (!strcmp(key, "SyncWidth")) - ret = parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); - else if (!strcmp(key, "PreciseSyncRate")) - ret = parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); - else if (polarity && !strcmp(key, "SyncPolarity")) { + } else if (polarity && !strcmp(key, "SyncPolarity")) { if (dim->sync_polarity >= 0) ret = -EINVAL; else @@ -425,9 +399,9 @@ static int parse_dimension(struct dcp_parse_ctx *handle, struct dimension *dim, else ret = parse_int_bound(it.handle, &dim->pixel_repetition, 0, U32_MAX); - } - else + } else { ret = skip(it.handle); + } if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -457,9 +431,6 @@ struct color_mode { static int fill_color_mode(struct dcp_color_mode *color, struct color_mode *cmode) { - if (cmode->id < 0 || cmode->id > U32_MAX) - return -EINVAL; - if (color->score >= cmode->score) return 0; @@ -602,31 +573,26 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, out->sdr.score = -1; out->best.score = -1; - ret = iterator_begin(handle, &outer_it, false); - if (ret) - return ret; - - for (; outer_it.idx < outer_it.len; ++outer_it.idx) { - struct iterator it; - static const char * const fields[] = { "Colorimetry", "Depth", "DynamicRange", "EOTF", "ID", "PixelEncoding", "Score", "IsVirtual" }; + dcp_parse_foreach_in_array(handle, outer_it) { + static const char * const dup_fields[] = { + "Colorimetry", "Depth", "DynamicRange", "EOTF", "ID", + "PixelEncoding", "Score", "IsVirtual", + }; unsigned int seen = 0; + struct iterator it; bool is_virtual = true; struct color_mode cmode = { .colorimetry = -1, .depth = -1, .dynamic_range = -1, .eotf = -1, .id = -1, .pixel_encoding = -1, .score = -1, }; - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); unsigned int field; if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) - for (field = 0; field < 8; field++) - if (!strcmp(key, fields[field])) { + for (field = 0; field < ARRAY_SIZE(dup_fields); field++) + if (!strcmp(key, dup_fields[field])) { handle->ambiguous_180hz |= !!(seen & (1U << field)); seen |= 1U << field; } @@ -669,15 +635,14 @@ static int parse_color_modes(struct dcp_parse_ctx *handle, cmode.eotf, cmode.dynamic_range, cmode.pixel_encoding); + /* External H17P: bounded SDR selection and timing color limits. */ if (external_sdr) { unsigned int rank; - if (cmode.id <= U32_MAX && - timing_color_contains(&limits->dsc_required, cmode.id)) + if (timing_color_contains(&limits->dsc_required, cmode.id)) continue; rank = external_sdr_rank(&cmode); - if (cmode.id <= U32_MAX && - timing_color_contains(&limits->unsafe, cmode.id)) { + if (timing_color_contains(&limits->unsafe, cmode.id)) { /* * Collect only a timing-local bounded RGB8 candidate. * Admission still requires current EDID and link proof. @@ -730,25 +695,30 @@ static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) return DIV_ROUND_CLOSEST_ULL(clock >> 16, 1000); } +/* + * @external_sdr: the external H17P (T8140 dispext0): bounded SDR color + * selection, timing color restrictions, IsPreferred/IsInterlaced and sync + * polarity, and no adaptive-sync range. + */ static int parse_mode(struct dcp_parse_ctx *handle, struct dcp_display_mode *out, s64 *score, int width_mm, - int height_mm, unsigned notch_height, bool internal, bool external_sdr) + int height_mm, unsigned notch_height, bool internal, + bool external_sdr) { - static const char * const fields[] = { + static const char * const dup_fields[] = { "ID", "Score", "MinimumVariableRefreshRate", "MaximumVariableRefreshRate", }; unsigned int seen = 0; int ret = 0; struct iterator it; - struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; + struct dimension horiz = { .sync_polarity = -1, .pixel_repetition = -1 }; + struct dimension vert = { .sync_polarity = -1, .pixel_repetition = -1 }; struct timing_color_restrictions limits = {}; struct dcp_color_mode unsafe_rgb8 = { .score = -1 }; bool have_horiz = false, have_vert = false, have_virtual = false; bool duplicate_scope = false; - struct dimension horiz = {}, vert = {}; u64 clock; s64 min_vrr = 0, max_vrr = 0; - u64 clock; s64 id = -1; s64 best_color_mode = -1; bool is_virtual = false; @@ -772,17 +742,14 @@ static int parse_mode(struct dcp_parse_ctx *handle, if (ret) return ret; } - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); unsigned int field; if (!IS_ERR(key) && handle->automatic_1080p180_rgb8) - for (field = 0; field < ARRAY_SIZE(fields); field++) - if (!strcmp(key, fields[field])) { + for (field = 0; field < ARRAY_SIZE(dup_fields); field++) + if (!strcmp(key, dup_fields[field])) { handle->ambiguous_180hz |= !!(seen & (1U << field)); seen |= 1U << field; } @@ -790,27 +757,25 @@ static int parse_mode(struct dcp_parse_ctx *handle, if (IS_ERR(key)) ret = PTR_ERR(key); else if (is_virtual) - ret = skip(it.handle); + skip(it.handle); else if (!strcmp(key, "HorizontalAttributes")) { duplicate_scope |= have_horiz; have_horiz = true; - ret = parse_dimension(it.handle, &horiz, external_sdr); + ret = parse_dimension(it.handle, &horiz, false, external_sdr); } else if (!strcmp(key, "VerticalAttributes")) { duplicate_scope |= have_vert; have_vert = true; - ret = parse_dimension(it.handle, &vert, external_sdr); + ret = parse_dimension(it.handle, &vert, true, external_sdr); } else if (!strcmp(key, "MinimumVariableRefreshRate")) ret = parse_int_bound(it.handle, &min_vrr, 0, U32_MAX); else if (!strcmp(key, "MaximumVariableRefreshRate")) ret = parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); else if (!strcmp(key, "ColorModes")) ret = parse_color_modes(it.handle, out, external_sdr, &limits, - handle->automatic_1080p180_rgb8 ? - &unsafe_rgb8 : NULL); + handle->automatic_1080p180_rgb8 ? + &unsafe_rgb8 : NULL); else if (!strcmp(key, "ID")) ret = parse_int(it.handle, &id); - else if (!strcmp(key, "IsVirtual")) - ret = parse_int_bound(it.handle, &id, 0, U32_MAX); else if (!strcmp(key, "IsVirtual")) { duplicate_scope |= have_virtual; have_virtual = true; @@ -828,11 +793,9 @@ static int parse_mode(struct dcp_parse_ctx *handle, ret = parse_bool(it.handle, &interlaced); have_interlaced = true; } else if (!strcmp(key, "Score")) - ret = parse_int_bound(it.handle, score, 0, S64_MAX); - else if (!strcmp(key, "Score")) ret = parse_int(it.handle, score); else - ret = skip(it.handle); + skip(it.handle); if (!IS_ERR_OR_NULL(key)) kfree(key); @@ -918,10 +881,6 @@ static int parse_mode(struct dcp_parse_ctx *handle, out->vrr = true; } - clock = calculate_clock(&horiz, &vert); - if (!clock || clock > INT_MAX) - return -EINVAL; - vert.active -= notch_height; vert.sync_width += notch_height; @@ -1102,20 +1061,11 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, mode = &modes[*count]; ret = parse_mode(it.handle, mode, &score, width_mm, height_mm, notch_height, internal, external_sdr); - notch_height, internal); *handle = next; /* Errors for a single mode are recoverable -- just skip it. */ - if (ret) { - /* A failed parse can stop in the middle of the dictionary. */ - *it.handle = entry; - ret = skip(it.handle); - if (ret) { - kfree(modes); - return ERR_PTR(ret); - } + if (ret) continue; - } /* Process a successful mode */ (*count)++; @@ -1159,7 +1109,7 @@ int replace_modes(struct dcp_parse_ctx *handle, return ret; next = enumerate_modes(handle, &next_count, width_mm, height_mm, - notch_height, internal); + notch_height, internal, false); if (IS_ERR(next)) return PTR_ERR(next); if (it.len && !next_count) { @@ -1185,10 +1135,6 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, ret = iterator_begin(handle, &it, true); if (ret) return ret; - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - for (; it.idx < it.len; ++it.idx) { char *key = parse_string(it.handle); @@ -1217,6 +1163,44 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, return 0; } +/* H17P (T8140, T8142): DisplayAttributes as aurora-wip reads them. */ +int parse_display_attributes_h17p(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm) +{ + int ret = 0; + struct iterator it; + s64 width_cm = 0, height_cm = 0; + + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + + for (; it.idx < it.len; ++it.idx) { + char *key = parse_string(it.handle); + + if (IS_ERR(key)) + ret = PTR_ERR(key); + else if (!strcmp(key, "MaxHorizontalImageSize")) + ret = parse_int(it.handle, &width_cm); + else if (!strcmp(key, "MaxVerticalImageSize")) + ret = parse_int(it.handle, &height_cm); + else + ret = skip(it.handle); + + if (!IS_ERR_OR_NULL(key)) + kfree(key); + + if (ret) + return ret; + } + + /* 1cm = 10mm */ + *width_mm = 10 * width_cm; + *height_mm = 10 * height_cm; + + return 0; +} + /* Metadata strings must fit an EDID text descriptor without hidden suffixes. */ static int parse_metadata_string(struct dcp_parse_ctx *handle, char *out, size_t size) { @@ -1461,11 +1445,7 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, *name = ERR_PTR(-ENOENT); *class = ERR_PTR(-ENOENT); - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); if (IS_ERR(key)) { @@ -1507,7 +1487,7 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, if (!ret) parsed_unit = true; } else { - ret = skip(it.handle); + skip(it.handle); } kfree(key); @@ -1627,11 +1607,7 @@ static int parse_chmap(struct dcp_parse_ctx *handle, struct snd_pcm_chmap_elem * chmap->channels = 0; - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_array(handle, it) { for (i = 0; i < ARRAY_SIZE(chan_position_names); i++) if (consume_string(it.handle, chan_position_names[i].label)) break; @@ -1659,11 +1635,7 @@ static int parse_chan_layout_element(struct dcp_parse_ctx *handle, int ret; s64 nchans = 0; - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { if (consume_string(it.handle, "ActiveChannelCount")) ret = parse_int(it.handle, &nchans); else if (consume_string(it.handle, "ChannelLayout")) @@ -1688,11 +1660,7 @@ static int parse_nchans_mask(struct dcp_parse_ctx *handle, unsigned int *mask) *mask = 0; - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_array(handle, it) { int nchans; ret = parse_chan_layout_element(it.handle, &nchans, NULL); @@ -1712,11 +1680,7 @@ static int parse_avep_element(struct dcp_parse_ctx *handle, struct iterator it; int ret; - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { if (consume_string(handle, "StreamSampleRate")) ret = parse_sample_rate_bit(it.handle, &mask.rates); else if (consume_string(handle, "SampleSize")) @@ -1754,20 +1718,12 @@ static int parse_mode_in_avep_element(struct dcp_parse_ctx *handle, struct dcp_parse_ctx save_handle; int ret; - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { if (consume_string(it.handle, "AudioChannelLayoutElements")) { struct iterator inner_it; int nchans; - ret = iterator_begin(it.handle, &inner_it, false); - if (ret) - return ret; - - for (; inner_it.idx < inner_it.len; ++inner_it.idx) { + dcp_parse_foreach_in_array(it.handle, inner_it) { save_handle = *it.handle; ret = parse_chan_layout_element(inner_it.handle, &nchans, NULL); @@ -1820,11 +1776,7 @@ int parse_sound_constraints(struct dcp_parse_ctx *handle, hits->nchans = 0; } - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_array(handle, it) { ret = parse_avep_element(it.handle, sieve, hits); if (ret < 0) @@ -1843,11 +1795,7 @@ int parse_sound_mode(struct dcp_parse_ctx *handle, struct iterator it; int ret; - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_array(handle, it) { save_handle = *it.handle; ret = parse_avep_element(it.handle, sieve, NULL); @@ -1877,11 +1825,7 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn s64 timestamp = -1; bool type_match = false; - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { + dcp_parse_foreach_in_dict(handle, it) { char *key = parse_string(it.handle); if (IS_ERR(key)) { ret = PTR_ERR(key); @@ -1898,7 +1842,7 @@ int parse_system_log_mnits(struct dcp_parse_ctx *handle, struct dcp_system_ev_mn } else if (!strcmp(key, "timestamp")) { ret = parse_int(it.handle, ×tamp); } else { - ret = skip(it.handle); + skip(it.handle); } if (!IS_ERR_OR_NULL(key)) diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index ef40247e7993fe..610e601d947fd9 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -126,7 +126,6 @@ struct dcp_display_mode *enumerate_modes(struct dcp_parse_ctx *handle, int height_mm, unsigned notch_height, bool internal, bool external_sdr); int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_metadata *out); - bool internal); /* Caller holds modes_lock while checking connector ownership. */ bool dcp_modes_for_connector(struct apple_dcp *dcp, struct apple_connector *connector); @@ -153,6 +152,8 @@ int replace_modes(struct dcp_parse_ctx *handle, bool internal); int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm, bool *backlight_control); +int parse_display_attributes_h17p(struct dcp_parse_ctx *handle, int *width_mm, + int *height_mm); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, const char **class, s64 *unit, bool h17p_keys); diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index a298a5bf923e05..48fbca01f4f9b8 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -5,6 +5,7 @@ #include "plane.h" +#include "dcp.h" #include "dcp-internal.h" #include "iomfb_internal.h" @@ -308,6 +309,26 @@ static void apple_plane_cleanup_fb(struct drm_plane *plane, kfree(entry); } +/* Does @plane scan out on an H17-generation pipeline (T8140, T8142)? */ +static bool apple_plane_h17p(struct drm_plane *plane) +{ + struct drm_crtc *crtc; + + drm_for_each_crtc(crtc, plane->dev) { + struct apple_dcp *dcp; + + if (!(plane->possible_crtcs & drm_crtc_mask(crtc))) + continue; + dcp = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + return dcp && dcp->fw_compat == DCP_FIRMWARE_H17P; + } + return false; +} + +/* + * H17P: a displaced framebuffer stays referenced until the swap that + * replaces it completes (dcp_release_retained_framebuffers()). + */ static int apple_plane_prepare_fb(struct drm_plane *plane, struct drm_plane_state *state) { @@ -322,6 +343,8 @@ static int apple_plane_prepare_fb(struct drm_plane *plane, old = drm_atomic_get_old_plane_state(state->state, plane); if (!old || !old->crtc || !old->fb || old->fb == state->fb) return 0; + if (!apple_plane_h17p(plane)) + return 0; entry = kzalloc_obj(*entry); if (!entry) @@ -380,6 +403,12 @@ apple_plane_duplicate_state(struct drm_plane *plane) __drm_atomic_helper_plane_duplicate_state(plane, &apple_plane_state->base); apple_plane_state->surf = old_apple_plane_state->surf; + /* H17P presents re-use the cached scanout geometry. */ + if (apple_plane_h17p(plane)) { + apple_plane_state->src_rect = old_apple_plane_state->src_rect; + apple_plane_state->dst_rect = old_apple_plane_state->dst_rect; + apple_plane_state->iova = old_apple_plane_state->iova; + } return &apple_plane_state->base; } From 095df56f983d1bbea200348b33f52fe9869d92ab Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 11 Oct 2026 16:23:33 +0200 Subject: [PATCH 550/631] drm: apple: relink a Type-C sink that lost its link to a hub's USB3 A USB-C monitor with a built-in USB hub (EIZO EV2780) intermittently stays dark after a cold boot or a hot-plug on the M1 MacBook Air and the M1 Mac mini. DCP trains the link for the first modeset at about the time the monitor's hub brings its SuperSpeed link up. When the two overlap, the sink loses the link, but DCP still reports the modeset as linked: UnifiedPipeline::mode_set_gated(): 2560x1440@59 Hz link: 1 Then one of two things happens. DCP gives up on its own 2.008 s after asserting HPD: IOMFB: IOAVVideoInterface terminated VideoInterfaceIOAV::unplug_gated(): display HPD removed or it keeps driving the dead link without noticing, while the monitor pulses HPD about 2-4 s later to be detected again. In both cases the CD321x still reports DP alt mode with HPD high, and nothing re-detects the sink. With dwc3 kept off the port, 5 out of 5 boots brought the display up; with it, 3 out of 6 failed. Replay an unplug and replug of the sink when DCP drops a sink that still asserts HPD, or when the sink asks for attention (IRQ_HPD): the link is then retrained after the hub has settled, as a cable replug does. A request from the sink within 4 s after a relink is the sink's late reaction to the loss the relink already handled and is ignored; another relink would only interrupt the new link. A sink request must not be dropped as an unchanged route state either. A connection gets at most three relinks; the replay does not reset that count. Pulsing only DCP's HPD instead, with the connector kept connected, raced with DCP re-detecting the sink on its own and left the display dark on the Mac mini. H17P and native external pipes keep their own recovery. Once the relink budget is spent, a sink request falls back to dcp_dptx_recover_irq() or dcp_retrain_oob() as before. With this, 8 out of 8 cold boots of the MacBook Air and 8 out of 8 of the Mac mini (EIZO on the port that failed) brought the display up. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/dcp-fabric.c | 19 +-- drivers/gpu/drm/apple/dcp-internal.h | 8 ++ drivers/gpu/drm/apple/dcp.c | 108 ++++++++++++++++++ drivers/gpu/drm/apple/iomfb_template.c | 6 + .../selftests/drivers/gpu/apple-dcp/rate.py | 1 + .../drivers/gpu/apple-dcp/transaction.py | 1 + 6 files changed, 135 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 49e759e6e11316..80c205b8a1f67f 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1438,7 +1438,8 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, * without a word, so a nominated route might never be called at all -- * and the port would then never be routed. */ - if (port->applied_valid && port->applied_alt == state->alt && + if (port->applied_valid && !(dp_status & DP_STATUS_IRQ_HPD) && + port->applied_alt == state->alt && port->applied_mode == state->mode && port->applied_status == dp_status && port->applied_conf == dp_conf) return 0; @@ -1571,13 +1572,15 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, dcp_retrain_oob(dcp->typec_connector); else if (dcp->typec_connector && dcp->external_native) dcp_external_sink_irq(dcp); - else if (dcp->typec_connector && - !READ_ONCE(dcp->typec_connector->connected)) { - ret = dcp_dptx_recover_irq(dcp); - if (ret) - return ret; - } else if (dcp->typec_connector) - dcp_retrain_oob(dcp->typec_connector); + else if (dcp->typec_connector && !dcp_typec_sink_relink(dcp)) { + if (!READ_ONCE(dcp->typec_connector->connected)) { + ret = dcp_dptx_recover_irq(dcp); + if (ret) + return ret; + } else { + dcp_retrain_oob(dcp->typec_connector); + } + } } port->hpd = hpd; port->applied_valid = true; diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5392736e453126..699689a0770b85 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -61,6 +61,8 @@ void dcp_swap_watchdog_arm(struct apple_dcp *dcp); void dcp_swap_watchdog_complete(struct apple_dcp *dcp); void dcp_retry_placeholder_edid(struct apple_dcp *dcp, const struct drm_edid *drm_edid); +bool dcp_typec_relink(struct apple_dcp *dcp); +bool dcp_typec_sink_relink(struct apple_dcp *dcp); struct dcpav_service_epic; @@ -380,6 +382,12 @@ struct apple_dcp { u64 crtc_unplugs; u64 placeholder_generation; struct delayed_work placeholder_edid_wq; + /* replugs replayed for a Type-C sink that lost its link */ + u32 relink_tries; + bool relink_active; + unsigned long relink_done; + u64 relink_generation; + struct delayed_work typec_relink_wq; bool use_timestamps; bool vrr_enabled; struct dcp_set_digital_out_mode_req mode; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 5caf56299f8710..2d4f79364d2787 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1129,6 +1129,110 @@ static void dcp_typec_reconnect_work(struct work_struct *work) dcp->typec_reconnect_tries, ret); } +#define DPTX_RELINK_RETRIES 3 +#define DPTX_RELINK_UNPLUG_MS 1000 +/* a monitor pulses HPD up to about 4 s after it lost the link */ +#define DPTX_RELINK_SETTLE_MS 4000 + +/* + * A USB-C monitor with a built-in hub can lose the DP link when the hub + * brings its SuperSpeed link up right around DCP's link training for the + * first modeset. DCP may then report the sink as unplugged two seconds + * later although it still asserts HPD, or keep driving the dead link + * without noticing while the monitor pulses HPD to be detected again. + * Neither is recovered by anything else: replay an unplug and replug of + * the sink, which retrains the link once the hub has settled, as a cable + * replug does. + * + * The retries cover one connection: the replay itself does not reset them. + */ +static bool dcp_typec_relink_start(struct apple_dcp *dcp, const char *why) +{ + u64 generation = READ_ONCE(dcp->typec_generation); + + /* H17P and native external pipes recover through their own lifecycle */ + if (dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->external_native || + !dcp_is_typec_output(dcp) || dcp_is_usb4_output(dcp) || + dcp_is_external_h17p(dcp) || + !READ_ONCE(dcp->typec_cable_connected) || + READ_ONCE(dcp->typec_crtc_off) || !dcp->typec_connector) + return false; + + if (dcp->relink_generation != generation) { + dcp->relink_generation = generation; + dcp->relink_tries = 0; + WRITE_ONCE(dcp->relink_active, false); + } + /* the unplug a running relink replays makes DCP report one too */ + if (READ_ONCE(dcp->relink_active)) + return true; + if (dcp->relink_tries >= DPTX_RELINK_RETRIES) + return false; + + dcp->relink_tries++; + WRITE_ONCE(dcp->relink_active, true); + dev_info(dcp->dev, "%s, relinking the Type-C sink (%u/%u)\n", why, + dcp->relink_tries, DPTX_RELINK_RETRIES); + mod_delayed_work(system_freezable_wq, &dcp->typec_relink_wq, 0); + + return true; +} + +/* DCP reported an unplug of a Type-C sink that still asserts HPD */ +bool dcp_typec_relink(struct apple_dcp *dcp) +{ + return dcp_typec_relink_start(dcp, "DCP dropped a sink that asserts HPD"); +} + +/* + * The sink asked for attention: IRQ_HPD or a coalesced HPD pulse. Right + * after a relink that is the sink's late reaction to the loss the relink + * already handled; another relink would only interrupt the new link. + */ +bool dcp_typec_sink_relink(struct apple_dcp *dcp) +{ + unsigned long done = READ_ONCE(dcp->relink_done); + + if (done && time_before(jiffies, done + msecs_to_jiffies(DPTX_RELINK_SETTLE_MS)) && + READ_ONCE(dcp->relink_generation) == READ_ONCE(dcp->typec_generation)) { + dev_info(dcp->dev, "Type-C sink requested attention right after a relink, ignored\n"); + return true; + } + + return dcp_typec_relink_start(dcp, "Type-C sink requested attention"); +} + +static void dcp_typec_relink_work(struct work_struct *work) +{ + struct apple_dcp *dcp = + container_of(to_delayed_work(work), struct apple_dcp, + typec_relink_wq); + struct platform_device *pdev = to_platform_device(dcp->dev); + u64 generation; + + scoped_guard(mutex, &dcp->hpd_mutex) + generation = dcp->typec_generation; + if (generation != dcp->relink_generation || + !READ_ONCE(dcp->typec_cable_connected)) + goto out; + + dcp_dptx_disconnect_oob(pdev, 0); + msleep(DPTX_RELINK_UNPLUG_MS); + + /* a real unplug or replug meanwhile takes over */ + scoped_guard(mutex, &dcp->hpd_mutex) { + if (dcp->typec_generation != generation + 1) + goto out; + } + dcp_dptx_connect_oob(pdev, 0); + + scoped_guard(mutex, &dcp->hpd_mutex) + dcp->relink_generation = dcp->typec_generation; + WRITE_ONCE(dcp->relink_done, jiffies); +out: + WRITE_ONCE(dcp->relink_active, false); +} + static void disconnected_hpd_event(struct apple_connector *con) { apple_connector_edid_set_live(con, false); @@ -2158,6 +2262,7 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + disable_delayed_work_sync(&dcp->typec_relink_wq); disable_delayed_work_sync(&dcp->hdmi_settle_wq); disable_delayed_work_sync(&dcp->hdmi_recheck_wq); dcp_hdmi_disable(dcp); @@ -2169,6 +2274,7 @@ static void dcp_enable_typec_work(struct apple_dcp *dcp) return dcp_enable_typec_work_h17p(dcp); enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); + enable_delayed_work(&dcp->typec_relink_wq); enable_delayed_work(&dcp->hdmi_settle_wq); enable_delayed_work(&dcp->hdmi_recheck_wq); dcp_hdmi_enable(dcp); @@ -2614,9 +2720,11 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp_typec_reconnect_work); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); + INIT_DELAYED_WORK(&dcp->typec_relink_wq, dcp_typec_relink_work); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); + disable_delayed_work(&dcp->typec_relink_wq); /* * A native external pipe's bind enables these; firmware callbacks may * come before it. The 14.7 panel never uses them, but suspend cancels diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index b11451c53eece8..5e6a81ee6419af 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1121,6 +1121,12 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) schedule_work(&dcp->vblank_wq); return; } + /* DCP lost a sink which is still attached, see dcp_typec_relink() */ + if (!(*connected) && dcp_typec_relink(dcp)) { + dcp_mode_invalidate(&dcp->mode_state); + schedule_work(&dcp->vblank_wq); + return; + } if (dcp_is_typec_output(dcp) && *connected && dcp->nr_modes) complete_all(&dcp->typec_iomfb_hpd_ready); diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 32585f1755845f..7145d3b13d2298 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -936,6 +936,7 @@ def hotplug(root): #define dcp_mode_invalidate(s) ((s)->valid=false) #define dcp_mode_hotplug(s,c,p) __extension__ ({ (void)(s);if(p)*(p)=(c);1u; }) static void dcp_handle_hotplug_actions(struct apple_dcp *d,unsigned action) { (void)d;(void)action; } +static bool dcp_typec_relink(struct apple_dcp *d) { (void)d;return false; } ''' dcp = (root / 'drivers/gpu/drm/apple/dcp.c').read_text() template = (root / 'drivers/gpu/drm/apple/iomfb_template.c').read_text() diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py index 0ec0cc21c26c06..fe4143c84ac571 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/transaction.py @@ -72,6 +72,7 @@ def check(name, unit, cases=((),), negative=False, flags=()): static int connect_calls; static int dcp_dptx_connect_oob(void *dev,unsigned n) { (void)dev;(void)n;connect_calls++;display.external_cable=true;display.dptxport[0].connected=true;display.typec_cable_connected=true;connector.connected=true;return 0; } static void dcp_retrain_oob(struct connector *c) { (void)c; } + static bool dcp_typec_sink_relink(struct apple_dcp *d) { (void)d;return false; } ''' pos=actual.index('static int dcp_typec_route_deactivate(') actual=actual[:pos]+extra+actual[pos:] From 58b25ade0fee9bdb91093bf2f57cf965f61adf15 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:01 -0400 Subject: [PATCH 551/631] usb: typec: keep the H17 display route handshake Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. The DCP display routes on T8140 and T8142 register prepare, finish and commit callbacks, so that the DCP lets go before the PHY changes mode. Run those phases in typec_mux_set() for the mux devices that register them, with abort in reverse on failure. Devices with only a set callback behave as before. On T8140 and T8142 the CD321x driver keeps aurora-wip's error meaning: -ENOSPC means the display is taken and -EBUSY is retried. Integration's DP status interrupt and its HPD rules stay off on those SoCs. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/usb/typec/mux.c | 197 ++++++------------------ drivers/usb/typec/mux.h | 2 - drivers/usb/typec/tipd/cd321x-pm-test.c | 19 ++- drivers/usb/typec/tipd/cd321x-pm.h | 14 +- drivers/usb/typec/tipd/core.c | 181 +++++++++------------- drivers/usb/typec/tipd/tps6598x.h | 16 +- include/linux/usb/typec_mux.h | 7 + 7 files changed, 160 insertions(+), 276 deletions(-) diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 31241ed2beb643..70023d7342f892 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -264,37 +264,6 @@ struct typec_mux { unsigned int num_mux_devs; }; -static struct typec_mux_context *typec_mux_context_get(struct fwnode_handle *fwnode) -{ - struct typec_mux_context *context; - - guard(mutex)(&typec_mux_contexts_lock); - list_for_each_entry(context, &typec_mux_contexts, list) { - if (context->fwnode == fwnode) { - context->users++; - return context; - } - } - context = kzalloc_obj(*context); - if (!context) - return ERR_PTR(-ENOMEM); - context->fwnode = fwnode_handle_get(fwnode); - context->users = 1; - mutex_init(&context->lock); - list_add_tail(&context->list, &typec_mux_contexts); - return context; -} - -static void typec_mux_context_put(struct typec_mux_context *context) -{ - guard(mutex)(&typec_mux_contexts_lock); - if (--context->users) - return; - list_del(&context->list); - fwnode_handle_put(context->fwnode); - kfree(context); -} - static int mux_fwnode_match(struct device *dev, const void *fwnode) { if (!is_typec_mux_dev(dev)) @@ -386,13 +355,7 @@ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) } for (i = 0; i < count; i++) { - if (READ_ONCE(mux_devs[i]->removed) || - !try_module_get(mux_devs[i]->owner)) { - err = -ENODEV; - while (i--) - module_put(mux_devs[i]->owner); - goto put_mux_devs; - } + WARN_ON(!try_module_get(mux_devs[i]->dev.parent->driver->owner)); mux->mux_devs[i] = mux_devs[i]; } @@ -435,151 +398,84 @@ void typec_mux_put(struct typec_mux *mux) } EXPORT_SYMBOL_GPL(typec_mux_put); -int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) -{ - int index = mux->srcu_index; - - mux->prepared = 0; - mux->applied = false; - mux->owner = NULL; - mutex_unlock(&mux->context->lock); - srcu_read_unlock(&typec_mux_srcu, index); -} - -void typec_mux_abort(struct typec_mux *mux) +/* Abort, in reverse, the providers whose prepare stage was reached. */ +static void typec_mux_abort(struct typec_mux *mux, unsigned int prepared, + struct typec_mux_state *state) { - unsigned int i; - - if (IS_ERR_OR_NULL(mux)) - return; - if (WARN_ON_ONCE(mux->owner != current)) - return; - for (i = mux->prepared; i > 0; i--) { - struct typec_mux_dev *dev = mux->mux_devs[i - 1]; + struct typec_mux_dev *mux_dev; - if (dev->abort) - dev->abort(dev, &mux->transaction); + while (prepared--) { + mux_dev = mux->mux_devs[prepared]; + if (mux_dev->abort) + mux_dev->abort(mux_dev, state); } - typec_mux_unlock(mux); } -EXPORT_SYMBOL_GPL(typec_mux_abort); -int typec_mux_begin(struct typec_mux *mux, struct typec_mux_state *state) +/** + * typec_mux_set - Set the state of the muxes of a connector + * @mux: USB Type-C Connector Multiplexer/DeMultiplexer + * @state: New state + * + * Sets every mux of the connector. Providers that register transaction + * phases (struct typec_mux_desc) take part in one transaction: every + * provider's prepare runs before any provider's set, and every finish, then + * every commit, after all of them. A failure in prepare, set or finish calls + * abort, in reverse order, on the providers whose prepare stage was reached. + * A provider that registers only set is just set, in connection order, as + * before. + */ +int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) { + struct typec_mux_dev *mux_dev; + unsigned int prepared = 0; unsigned int i; - int index, ret; + int ret; if (IS_ERR_OR_NULL(mux)) return 0; - index = srcu_read_lock(&typec_mux_srcu); - /* Nonblocking acquisition also rejects same-domain forwarding recursion. */ - if (!mutex_trylock(&mux->context->lock)) { - srcu_read_unlock(&typec_mux_srcu, index); - return -EBUSY; - } - mux->srcu_index = index; - mux->owner = current; - mux->transaction = *state; - for (i = 0; i < mux->num_mux_devs; i++) { - struct typec_mux_dev *dev = mux->mux_devs[i]; - if (READ_ONCE(dev->removed)) { - typec_mux_unlock(mux); - return -ENODEV; - } - } + /* Quiesce every consumer before any provider changes the route. */ for (i = 0; i < mux->num_mux_devs; i++) { - struct typec_mux_dev *dev = mux->mux_devs[i]; - - mux->prepared = i + 1; - if (!dev->prepare) + mux_dev = mux->mux_devs[i]; + prepared = i + 1; + if (!mux_dev->prepare) continue; - ret = dev->prepare(dev, &mux->transaction); + ret = mux_dev->prepare(mux_dev, state); if (ret) goto abort; } - return 0; -abort: - typec_mux_abort(mux); - return ret; -} -EXPORT_SYMBOL_GPL(typec_mux_begin); - -int typec_mux_apply(struct typec_mux *mux) -{ - unsigned int i; - int ret; - if (IS_ERR_OR_NULL(mux)) - return 0; - if (WARN_ON_ONCE(mux->owner != current)) - return -EINVAL; - if (mux->applied) - return 0; for (i = 0; i < mux->num_mux_devs; i++) { - struct typec_mux_dev *dev = mux->mux_devs[i]; + mux_dev = mux->mux_devs[i]; - ret = dev->set(dev, &mux->transaction); - if (ret && ret != -EOPNOTSUPP) { - typec_mux_abort(mux); - return ret; - } + ret = mux_dev->set(mux_dev, state); + if (ret && ret != -EOPNOTSUPP) + goto abort; } - mux->applied = true; - return 0; -} -EXPORT_SYMBOL_GPL(typec_mux_apply); - -int typec_mux_end(struct typec_mux *mux) -{ - unsigned int i; - int ret; - if (IS_ERR_OR_NULL(mux)) - return 0; - ret = typec_mux_apply(mux); - if (ret) - return ret; - /* Finalizers can fail, but must not release their software reservation. */ + /* Finalizers can fail, but must not release their reservation. */ for (i = 0; i < mux->num_mux_devs; i++) { mux_dev = mux->mux_devs[i]; - - if (!dev->finish) + if (!mux_dev->finish) continue; - ret = dev->finish(dev, &mux->transaction); + ret = mux_dev->finish(mux_dev, state); if (ret) goto abort; } - /* Only this no-fail publication phase may release owner reservations. */ + + /* Only this phase, which cannot fail, may release a reservation. */ for (i = 0; i < mux->num_mux_devs; i++) { - struct typec_mux_dev *dev = mux->mux_devs[i]; + mux_dev = mux->mux_devs[i]; + if (mux_dev->commit) + mux_dev->commit(mux_dev, state); + } return 0; -abort: - typec_mux_abort(mux); - return ret; -} -EXPORT_SYMBOL_GPL(typec_mux_end); -/* Preparation-only callers leave recovery ownership pending, never committed. */ -int typec_mux_prepare(struct typec_mux *mux, struct typec_mux_state *state) -{ - int ret = typec_mux_begin(mux, state); - - if (!ret) - typec_mux_abort(mux); +abort: + typec_mux_abort(mux, prepared, state); return ret; } -EXPORT_SYMBOL_GPL(typec_mux_prepare); - -int typec_mux_set(struct typec_mux *mux, struct typec_mux_state *state) -{ - int ret = typec_mux_begin(mux, state); - - if (ret) - return ret; - return typec_mux_end(mux); -} EXPORT_SYMBOL_GPL(typec_mux_set); static void typec_mux_release(struct device *dev) @@ -654,7 +550,6 @@ void typec_mux_unregister(struct typec_mux_dev *mux_dev) { if (!IS_ERR_OR_NULL(mux_dev)) device_unregister(&mux_dev->dev); - } } EXPORT_SYMBOL_GPL(typec_mux_unregister); diff --git a/drivers/usb/typec/mux.h b/drivers/usb/typec/mux.h index 1dc9d90d2cdaad..b055705a03ffad 100644 --- a/drivers/usb/typec/mux.h +++ b/drivers/usb/typec/mux.h @@ -13,14 +13,12 @@ struct typec_switch_dev { }; struct typec_mux_dev { - struct module *owner; struct device dev; typec_mux_set_fn_t set; typec_mux_set_fn_t prepare; typec_mux_set_fn_t finish; typec_mux_notify_fn_t commit; typec_mux_notify_fn_t abort; - bool removed; }; #define to_typec_switch_dev(_dev_) container_of(_dev_, struct typec_switch_dev, dev) diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 9c1e3f608b6308..0e86ff1f42bd13 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -752,15 +752,22 @@ static void cd321x_restore_ignores_fault_budget_test(struct kunit *test) /* A prepared port revalidates on its own resume instead. */ cd321x_pm_prepare(&pm); KUNIT_EXPECT_EQ(test, cd321x_pm_notifications(&pm, false, false, true), 0U); +} + /* Only a pipeline owned by another port skips the retry. */ static void cd321x_display_taken_test(struct kunit *test) { - KUNIT_EXPECT_TRUE(test, cd321x_display_taken(-ENOSPC)); - /* A PHY mid-transition or a pending SAFE state must be retried. */ - KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EBUSY)); - KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EAGAIN)); - KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-ETIMEDOUT)); - KUNIT_EXPECT_FALSE(test, cd321x_display_taken(0)); + /* T8140/T8142: a PHY mid-transition or a pending SAFE state is retried. */ + KUNIT_EXPECT_TRUE(test, cd321x_display_taken(-ENOSPC, true)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EBUSY, true)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EAGAIN, true)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-ETIMEDOUT, true)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(0, true)); + /* Other SoCs keep -EBUSY as the taken display. */ + KUNIT_EXPECT_TRUE(test, cd321x_display_taken(-EBUSY, false)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-ENOSPC, false)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(-EAGAIN, false)); + KUNIT_EXPECT_FALSE(test, cd321x_display_taken(0, false)); } static struct kunit_case cd321x_pm_cases[] = { diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index a717c95a2d920c..7f7cae8166e776 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -3,7 +3,6 @@ #define __CD321X_PM_H__ #include - #include #define CD321X_RESUME_ATTEMPTS 3 @@ -213,4 +212,17 @@ static inline void cd321x_pm_remove(struct cd321x_pm_state *pm) pm->attempts_left = 0; } +/* + * Whether a display mux error means that every display pipeline drives + * another port. That lasts until the other display goes away, so the port's + * USB path is set up without the display and nothing is retried. T8140 and + * T8142 display routes report it as -ENOSPC (@enospc), and their PHY returns + * -EBUSY while a transition is pending, which must be retried. Elsewhere the + * display route reports it as -EBUSY. + */ +static inline bool cd321x_display_taken(int ret, bool enospc) +{ + return ret == (enospc ? -ENOSPC : -EBUSY); +} + #endif /* __CD321X_PM_H__ */ diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 01b3c6fb2fa715..492544f8568c67 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -6,6 +6,8 @@ * Author: Heikki Krogerus */ +#include + #include #include #include @@ -20,7 +22,6 @@ #include #include #include -#include #include "tps6598x.h" #include "cd321x-setup.h" @@ -216,7 +217,6 @@ static void tps6598x_set_data_role(struct tps6598x *tps, enum typec_data_role role, bool connected) { enum usb_role role_val; - int ret; if (role == TYPEC_HOST) role_val = USB_ROLE_HOST; @@ -226,11 +226,8 @@ static void tps6598x_set_data_role(struct tps6598x *tps, if (!connected) role_val = USB_ROLE_NONE; - ret = usb_role_switch_set_role(tps->role_sw, role_val); - if (ret) - return ret; + usb_role_switch_set_role(tps->role_sw, role_val); typec_set_data_role(tps->port, role); - return 0; } static int tps6598x_connect(struct tps6598x *tps, u32 status) @@ -263,9 +260,7 @@ static int tps6598x_connect(struct tps6598x *tps, u32 status) else typec_set_orientation(tps->port, TYPEC_ORIENTATION_NORMAL); typec_set_mode(tps->port, TYPEC_STATE_USB); - ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); - if (ret) - return ret; + tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), true); tps->partner = typec_register_partner(tps->port, &desc); if (IS_ERR(tps->partner)) @@ -287,37 +282,11 @@ static void tps6598x_disconnect(struct tps6598x *tps, u32 status) typec_set_pwr_opmode(tps->port, TYPEC_PWR_MODE_USB); typec_set_pwr_role(tps->port, TPS_STATUS_TO_TYPEC_PORTROLE(status)); typec_set_vconn_role(tps->port, TPS_STATUS_TO_TYPEC_VCONN(status)); -} - -static int tps6598x_disconnect(struct tps6598x *tps, u32 status) -{ - struct typec_mux_state safe = { .mode = TYPEC_STATE_SAFE }; - bool prepared = tps->remove_safe_prepared; - int ret; - - tps->remove_safe_prepared = false; - tps6598x_drop_partner(tps, status); - if (!prepared) { - typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); - ret = typec_mux_begin(tps->mux, &safe); - if (ret) - return ret; - /* Park while the host still supplies the PIPE backend clock. */ - ret = typec_mux_apply(tps->mux); - if (ret) - return ret; - } - ret = tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); - if (ret) { - typec_mux_abort(tps->mux); - return ret; - } - ret = typec_mux_end(tps->mux); - if (ret) - return ret; + typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); + typec_set_mode(tps->port, TYPEC_STATE_SAFE); + tps6598x_set_data_role(tps, TPS_STATUS_TO_TYPEC_DATAROLE(status), false); power_supply_changed(tps->psy); - return 0; } int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, @@ -569,12 +538,24 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) if (ret) dev_err(tps->dev, "failed to register partner\n"); } else { - ret = tps6598x_disconnect(tps, status); - if (ret) - dev_err(tps->dev, "hardware disconnect failed: %d\n", ret); + tps6598x_disconnect(tps, status); } } +/* + * The interrupt mask of this controller. T8140 and T8142 leave the SN201202x + * DP status event masked: the display handling these machines were brought + * up with never used it (see h17_display). + */ +static u64 tipd_irq_mask1(const struct tps6598x *tps) +{ + u64 mask = tps->data->irq_mask1; + + if (tps->h17_display) + mask &= ~SN201202X_INT_DP_SID_UPDATE; + return mask; +} + /* * The HPD level of a DisplayPort partner. DATA_STATUS reports it. An * SN201202x also signals every new DP status of the partner, and a DP-to-HDMI @@ -736,7 +717,7 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, if (ret) goto out; ret = typec_mux_set(cd321x->mux, &cd321x->state); - *display_busy = cd321x_display_taken(ret); + *display_busy = cd321x_display_taken(ret, tps->h17_display); if (!ret) { cd321x->dp_status = dp_data.status; cd321x->dp_conf = dp_data.conf; @@ -814,7 +795,13 @@ static int cd321x_dp_release_first(struct cd321x *cd321x) struct typec_mux_state state; int ret; - if (cd321x->tps.data != &tipd_sn201202x_data || !cd321x->display_route_active || + /* + * T8140/T8142 keep the teardown they were brought up with: there the + * display route releases the display itself, in the prepare phase of + * the SAFE transaction, before the PHY changes. + */ + if (cd321x->tps.data != &tipd_sn201202x_data || cd321x->tps.h17_display || + !cd321x->display_route_active || cd321x->state.alt != cd321x->port_altmode_dp || !(cd321x->dp_status & DP_STATUS_HPD_STATE)) return 0; @@ -932,21 +919,6 @@ static void cd321x_update_work(struct work_struct *work) goto retry; } - /* - * Report the sink as gone before any hardware transition that can fail: - * a failed teardown keeps the route, but the sink must not stay present. - * All ports reference the same external display connector, so only the - * port that owns the display route may report a disconnect. A second DP - * sink can enter Alt Mode while its mux request fails with -ENOSPC, and - * must not tear down the display carried by the other port. - */ - if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && - dp_route_was_active && - (!new_connected || was_disconnected || !dp_connected || !dp_hpd || - dp_hpd_changed || dp_mode_changed)) { - drm_connector_oob_hotplug_event(cd321x->connector_fwnode, - connector_status_disconnected); - /* * ACIO carries the tunneled PCIe reset handshake over the still-live * Type-C data path. Close a disappearing USB4/TBT session before dropping @@ -969,6 +941,21 @@ static void cd321x_update_work(struct work_struct *work) goto retry; } + /* + * Every Type-C port references the same external DCP connector. Only the + * port which successfully acquired the shared display route may report a + * disconnect. A second DP sink can negotiate Alt Mode even though its mux + * request is rejected with -EBUSY (-ENOSPC on T8140/T8142); treating that + * sink as the connector owner would tear down the display carried by the + * first port. + */ + if (IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode && + dp_route_was_active && + (!new_connected || was_disconnected || !dp_connected || !dp_hpd || + dp_hpd_changed || dp_mode_changed)) { + drm_connector_oob_hotplug_event(cd321x->connector_fwnode, connector_status_disconnected); + } + /* Process partner disconnection or change */ if (!new_connected || partner_changed) { if (!IS_ERR(tps->partner)) @@ -1262,9 +1249,9 @@ static int cd321x_check_setup(struct cd321x *cd321x, u64 event) lockdep_assert_held(&tps->lock); if (cd321x->pm.phase == CD321X_PM_REMOVED || - (!cd321x->pm.setup_pending && !(event & ~tps->data->irq_mask1))) + (!cd321x->pm.setup_pending && !(event & ~tipd_irq_mask1(tps)))) return 0; - ret = cd321x_setup_run(&cd321x->pm, event, tps->data->irq_mask1, + ret = cd321x_setup_run(&cd321x->pm, event, tipd_irq_mask1(tps), &cd321x_setup_ops, cd321x); if (cd321x->pm.phase == CD321X_PM_PREPARED || cd321x->pm.phase == CD321X_PM_INITIALIZING) @@ -1327,10 +1314,10 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) } if (event & (APPLE_CD_REG_INT_DATA_STATUS_UPDATE | - (tps->data->irq_mask1 & SN201202X_INT_DP_SID_UPDATE))) + (tipd_irq_mask1(tps) & SN201202X_INT_DP_SID_UPDATE))) if (!tps->data->read_data_status(tps)) goto err_unlock; - if (event & tps->data->irq_mask1 & SN201202X_INT_DP_SID_UPDATE) + if (event & tipd_irq_mask1(tps) & SN201202X_INT_DP_SID_UPDATE) cd321x_dp_sid_event(container_of(tps, struct cd321x, tps)); /* Can be called uncondtionally since it will check for any changes itself */ @@ -1790,9 +1777,10 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) ret = typec_set_mode(tps->port, TYPEC_STATE_SAFE); cd321x->state_valid = !ret; /* - * A fixed host can register its SuperSpeed root after the mux provider. - * Wait for that consumer at probe instead of exhausting the cable retry - * budget before it exists. Host startup does not depend on this port. + * A fixed host can register its SuperSpeed root after the mux provider + * (the T8140 ATC PHY reports this as -EAGAIN). Wait for that consumer at + * probe instead of exhausting the cable retry budget before it exists. + * Host startup does not depend on this port. */ if (ret == -EAGAIN) { ret = -EPROBE_DEFER; @@ -1812,6 +1800,7 @@ cd321x_register_port(struct tps6598x *tps, struct fwnode_handle *fwnode) typec_thunderbolt_switch_put(cd321x->tbt_switch); cd321x->tbt_switch = NULL; err_unregister_mux: + /* A deferred probe must not leak the connector reference on every retry. */ fwnode_handle_put(cd321x->connector_fwnode); cd321x->connector_fwnode = NULL; typec_mux_put(cd321x->mux); @@ -2279,26 +2268,13 @@ static void cd321x_remove(struct tps6598x *tps) ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) - return ret; - /* Display quiescence must precede even an ordinary host-role shutdown. */ - typec_set_orientation(tps->port, TYPEC_ORIENTATION_NONE); - ret = typec_mux_begin(tps->mux, &safe); - if (ret) - return ret; - /* Park while the host still supplies the PIPE backend clock. */ - ret = typec_mux_apply(tps->mux); - if (ret) - return ret; - ret = usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); - if (ret) { - typec_mux_abort(tps->mux); - return ret; - } - /* Leave the SAFE transaction open for tps6598x_disconnect() to end. */ - tps->remove_safe_prepared = true; - return 0; + dev_warn(tps->dev, + "failed to close Thunderbolt/USB4 session on remove: %d\n", + ret); + usb_role_switch_set_role(tps->role_sw, USB_ROLE_NONE); } + int tipd_init(struct tps6598x *tps) { struct fwnode_handle *fwnode; @@ -2313,6 +2289,14 @@ int tipd_init(struct tps6598x *tps) if (tps->reset) msleep(TPS_SETUP_MS); + /* + * The T8140 and T8142 display routes report a display taken by another + * port as -ENOSPC, and these machines were brought up without the + * SN201202x DP status event; keep that display handling there. + */ + tps->h17_display = of_machine_is_compatible("apple,t8140") || + of_machine_is_compatible("apple,t8142"); + if (!device_is_compatible(tps->dev, "ti,tps25750")) { ret = tps6598x_read32(tps, TPS_REG_VID, &vid); if (ret < 0 || !vid) @@ -2347,7 +2331,7 @@ int tipd_init(struct tps6598x *tps) if (tps->dbma_at_probe) dev_info(tps->dev, "controller in debug mode \"DBMa\", leaving it there\n"); - ret = tps6598x_write64(tps, TPS_REG_INT_MASK1, tps->data->irq_mask1); + ret = tps6598x_write64(tps, TPS_REG_INT_MASK1, tipd_irq_mask1(tps)); if (ret) goto err_reset_controller; @@ -2377,20 +2361,9 @@ int tipd_init(struct tps6598x *tps) if (ret) goto err_role_put; - tps->mux = fwnode_typec_mux_get(fwnode); - if (IS_ERR(tps->mux)) { - ret = PTR_ERR(tps->mux); - /* No provider reference means rollback cannot prove SAFE. */ - reset_allowed = false; - goto err_role_put; - } - ret = tps->data->register_port(tps, fwnode); - if (ret) { - /* No live port remains to prove SAFE on failed registration. */ - reset_allowed = false; - goto err_mux_put; - } + if (ret) + goto err_role_put; if (status & TPS_STATUS_PLUG_PRESENT) { ret = -EINVAL; @@ -2399,7 +2372,7 @@ int tipd_init(struct tps6598x *tps) if (!tps->data->read_data_status(tps)) goto err_unregister_port; ret = tps->data->connect(tps, status); - if (ret) { + if (ret) dev_err(tps->dev, "failed to register partner\n"); } @@ -2452,8 +2425,6 @@ EXPORT_SYMBOL_GPL(tipd_init); void tipd_remove(struct tps6598x *tps) { - int ret = 0; - tipd_debugfs_unregister(tps); if (!tps->irq) @@ -2464,18 +2435,12 @@ void tipd_remove(struct tps6598x *tps) if (tps->data->remove) tps->data->remove(tps); - if (!ret) - ret = tps6598x_disconnect(tps, 0); - else - tps6598x_drop_partner(tps, 0); - if (ret) - dev_err(tps->dev, "hardware teardown failed on remove: %d\n", ret); + tps6598x_disconnect(tps, 0); tps->data->unregister_port(tps); usb_role_switch_put(tps->role_sw); /* Reset PD controller to remove any applied patch */ - if (!ret) - tps->data->reset(tps); + tps->data->reset(tps); if (tps->reset) gpiod_set_value_cansleep(tps->reset, 1); diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 1d0d22f3545996..de74a55c3f8b1f 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -15,12 +15,11 @@ #include #include -#include "cd321x-pm.h" - #ifndef __TPS6598X_H__ #define __TPS6598X_H__ #include "cd321x-pm.h" + /* Also read by the SPMI glue to tell when the controller has woken up */ #define TPS_REG_MODE 0x03 @@ -328,6 +327,13 @@ struct tps6598x { int irq; u8 i2c_protocol:1; u8 dbma_at_probe:1; /* Apple "DBMa" debug mode was active at probe */ + /* + * T8140 (H17P) and T8142 (H17G) Type-C display handling: the display + * mux reports a display taken by another port as -ENOSPC (-EBUSY is a + * transient PHY state and is retried), and the SN201202x DP status + * event and its HPD rules stay off. + */ + u8 h17_display:1; struct gpio_desc *reset; struct typec_port *port; @@ -335,12 +341,6 @@ struct tps6598x { struct usb_pd_identity partner_identity; struct usb_role_switch *role_sw; struct typec_capability typec_cap; - struct typec_mux *mux; - /* - * Set by ->remove() once it has parked the mux in SAFE, so that - * tps6598x_disconnect() only ends that mux transaction. - */ - bool remove_safe_prepared; struct power_supply *psy; struct power_supply_desc psy_desc; diff --git a/include/linux/usb/typec_mux.h b/include/linux/usb/typec_mux.h index 2afc5e896835f4..0703ec219e08bd 100644 --- a/include/linux/usb/typec_mux.h +++ b/include/linux/usb/typec_mux.h @@ -97,6 +97,13 @@ struct typec_mux_desc { typec_mux_set_fn_t set; const char *name; void *drvdata; + /* + * Optional transaction phases, run by typec_mux_set() only for the + * providers that register them: every provider's prepare runs before + * any provider's set, and finish and then commit after all of them. + * A failure in any phase calls abort, in reverse order, on the + * providers whose prepare stage was reached. + */ /* Quiesce consumers before any provider changes the shared route. */ typec_mux_set_fn_t prepare; /* Fallible finalization retains ownership; commit cannot fail. */ From 3842a7424b6998cd46a6c4ec446c9a6ecbbb549e Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 11 Oct 2026 18:06:38 +0200 Subject: [PATCH 552/631] usb: typec: tipd: report a coalesced sink HPD pulse as IRQ_HPD A DisplayPort sink that lost the link drops HPD and raises it again to be detected anew. The EIZO EV2780 does so for about 250 ms, which falls within the 500 ms debounce of cd321x_update_work(). The worker then sees DP alt mode with HPD high, the state it already applied, and cd321x_typec_update_mode() returns without telling the mux, so the display driver never learns that the sink wants attention and the display stays dark. The out-of-band DRM hotplug event does not cover this either on the M1 MacBook Air and Mac mini: their connector nodes have no "displayport" reference, so connector_fwnode is NULL. When HPD changed within the debounce window and is high before and after, pass the state on with DP_STATUS_IRQ_HPD set, even though it otherwise matches the applied one. Signed-off-by: Thomas Glanzmann --- drivers/usb/typec/tipd/core.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 492544f8568c67..6c90eb3221adf0 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -704,7 +704,19 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, else dp_data.status &= ~DP_STATUS_HPD_STATE; + /* + * The sink dropped HPD and raised it again within the debounce + * window, as a monitor does after losing the link. The coalesced + * state looks unchanged, so the display side would never hear of + * it: report the pulse as IRQ_HPD. + */ + if ((st->data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL) && + (dp_data.status & DP_STATUS_HPD_STATE) && + (cd321x->dp_status & DP_STATUS_HPD_STATE)) + dp_data.status |= DP_STATUS_IRQ_HPD; + if (cd321x->state_valid && cd321x->state.alt == cd321x->port_altmode_dp && + !(dp_data.status & DP_STATUS_IRQ_HPD) && cd321x->state.mode == mode && cd321x->dp_status == dp_data.status && cd321x->dp_conf == dp_data.conf) From 0983fd26acb58f2173b33d98d6a08776de58279c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:01 -0400 Subject: [PATCH 553/631] soc: apple: rtkit: keep the T8140 and T8142 OSLog behaviour Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. Keep integration's handling of an adopted co-processor that has crashed and its warning when the shared buffers of an unstopped co-processor are retained. aurora-wip answers no OSLog notification 2 to 5: the T8140 and T8142 co-processors send them during normal operation and keep running without a reply, which is how the J700 and M5 stacks were tested. integration acknowledges type 2 so that the M1/M2/M3 DCP can drain its log transport before sleep. Keep integration's behaviour, and leave the notifications unanswered on T8140 and T8142. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/soc/apple/rtkit.c | 41 +++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 6ca0ed30003993..cce23cd167e0bb 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -552,6 +552,23 @@ static void apple_rtkit_crashlog_rx(struct apple_rtkit *rtk, u64 msg) return; } + if (rtk->crashlog_inherited && !rtk->crashlog_buffer.size) { + /* + * The previous owner negotiated the crashlog buffer, so this + * session never learned its geometry. A CRASH message on an + * inherited session is therefore a crash notification, not a + * buffer request, and must not be answered as one. + */ + dev_err(rtk->dev, + "RTKit: adopted co-processor has crashed (crashlog at %#llx, %#llx bytes, not mapped)\n", + (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg), + (u64)FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg) << 12); + rtk->crashed = true; + if (rtk->ops->crashed) + rtk->ops->crashed(rtk->cookie, NULL, 0); + return; + } + if (!rtk->crashlog_buffer.size) { if (apple_rtkit_common_rx_get_buffer(rtk, &rtk->crashlog_buffer, APPLE_RTKIT_EP_CRASHLOG, msg)) @@ -737,6 +754,17 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) u8 type = FIELD_GET(APPLE_RTKIT_OSLOG_TYPE, msg); int err; + /* + * T8140 and T8142 co-processors send notifications 2 to 5 during + * normal operation; they expect no reply and keep running without one. + */ + if (type >= APPLE_RTKIT_OSLOG_LOG && type <= 5 && + (of_machine_is_compatible("apple,t8140") || + of_machine_is_compatible("apple,t8142"))) { + dev_dbg(rtk->dev, "RTKit: oslog notification: %016llx\n", msg); + return; + } + switch (type) { case APPLE_RTKIT_OSLOG_BUFFER_REQUEST: apple_rtkit_common_rx_get_buffer(rtk, &rtk->oslog_buffer, @@ -770,7 +798,6 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) msg, NULL, false); if (err) dev_err(rtk->dev, "RTKit: oslog acknowledgment failed: %d\n", err); - dev_dbg(rtk->dev, "RTKit: oslog notification: %016llx\n", msg); break; default: dev_warn(rtk->dev, "RTKit: Unknown oslog message: %llx\n", @@ -966,6 +993,7 @@ void apple_rtkit_flush_rx(struct apple_rtkit *rtk) flush_work(&rtk->sync_ack_work); } EXPORT_SYMBOL_GPL(apple_rtkit_flush_rx); + void apple_rtkit_hold_rx(struct apple_rtkit *rtk) { apple_mbox_hold_rx(rtk->mbox); @@ -1336,7 +1364,6 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) ret = apple_rtkit_wait(rtk, &rtk->epmap_completion); if (ret) return ret; - } if (rtk->boot_result) return rtk->boot_result; @@ -1345,14 +1372,12 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) ret = apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); if (ret) return ret; - } } dev_dbg(rtk->dev, "RTKit: waiting for IOP power state ACK\n"); ret = apple_rtkit_wait(rtk, &rtk->iop_pwr_ack_completion); if (ret) return ret; - } return early_ap_power ? 0 : apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); @@ -1514,10 +1539,10 @@ void apple_rtkit_free_retaining_buffers(struct apple_rtkit *rtk) * Retain shared buffers, including custom allocation contexts, rather * than returning their storage to an allocator while it may be live. */ - dev_dbg(rtk->dev, - "RTKit: retaining the shared buffers of a running co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", - rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, - rtk->oslog_buffer.size, rtk->syslog_buffer.size); + dev_warn(rtk->dev, + "RTKit: retaining the shared buffers of an unstopped co-processor (ioreport %zu, crashlog %zu, oslog %zu, syslog %zu bytes)\n", + rtk->ioreport_buffer.size, rtk->crashlog_buffer.size, + rtk->oslog_buffer.size, rtk->syslog_buffer.size); kfree(rtk->syslog_msg_buffer); apple_rtkit_release_rx(rtk); kfree(rtk); From f1056eacff097e3bcb54cc8017b1b055f620a215 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 12 Oct 2026 13:04:11 +0200 Subject: [PATCH 554/631] drm: apple: do not report a bad link for the HPD a Type-C wake awaits Waking a DP alt mode display from DPMS off with sway ("output * power on") on the M1 Mac mini (Fujitsu E24T-7 Pro on USB-C, which keeps HPD asserted while blanked) power cycles the display about every 750 ms until reboot: dcp_hotplug() connected:1 valid_mode:0 nr_modes:1 dcp_poweron() starting set_digital_out_mode finished dcp_poweroff() done ... With the cable attached, dcp_poweron() reconnects DPTX and waits for IOMFB to report the sink before the commit does its modeset. At that point no mode is valid yet, so dcpep_cb_hotplug() takes the HPD for a sink that asserted HPD only after a failed modeset and sends DCP_HOTPLUG_NOTIFY. dcp_hotplug() then sets the connector's link-status to BAD. wlroots treats a bad link as a disconnect and recreates the output, which disables the CRTC and enables it again. That wake does the same, so the loop never ends. Each round also leaks AFK service slots ("AFK[ep:29]: no reusable service slots") and the system becomes sluggish. Do not take the HPD that dcp_poweron() waits for as a late one: the modeset of the same commit follows. A sink that asserts HPD only after dcp_poweron() stopped waiting is still handled as before. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/dcp-internal.h | 2 ++ drivers/gpu/drm/apple/dcp.c | 3 +++ drivers/gpu/drm/apple/iomfb_template.c | 9 ++++++++- tools/testing/selftests/drivers/gpu/apple-dcp/rate.py | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 699689a0770b85..4d706327793c04 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -532,6 +532,8 @@ struct apple_dcp { bool typec_follow_start; bool typec_follow_retiring; u64 typec_follow_gen; + /* dcp_poweron() reconnects DPTX; its own modeset follows the HPD */ + bool typec_poweron_hpd; /* IOMFB reports its video interface ready after DPTX link training. */ struct completion typec_iomfb_hpd_ready; struct delayed_work typec_reconnect_wq; diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 2d4f79364d2787..6ef18230c86967 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1764,6 +1764,8 @@ void dcp_poweron(struct platform_device *pdev) if (READ_ONCE(dcp->typec_cable_connected)) { cancel_delayed_work(&dcp->typec_reconnect_wq); dcp->typec_reconnect_tries = 0; + /* the modeset of this commit follows, see dcpep_cb_hotplug() */ + WRITE_ONCE(dcp->typec_poweron_hpd, true); ret = dcp_dptx_connect(dcp, 0); if (ret) dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); @@ -1775,6 +1777,7 @@ void dcp_poweron(struct platform_device *pdev) dev_warn(dcp->dev, "Type-C IOMFB hotplug not ready on wake\n"); } + WRITE_ONCE(dcp->typec_poweron_hpd, false); } } else if (dcp->hdmi_hpd) { bool connected = gpiod_get_value_cansleep(dcp->hdmi_hpd); diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index 5e6a81ee6419af..783ef63094734b 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -1091,6 +1091,8 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) { struct apple_connector *connector = dcp->connector; + /* sampled before complete_all() lets dcp_poweron() clear it */ + bool poweron_hpd = READ_ONCE(dcp->typec_poweron_hpd); unsigned int action; /* DCP issues hotplug_gated callbacks after SetPowerState() calls on @@ -1144,8 +1146,13 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) * A Type-C sink can assert HPD only after a modeset has already failed, * as a TV behind a DP-to-HDMI converter does when it wakes from standby. * The connector state does not change then; re-apply the mode anyway. + * + * Not for the HPD dcp_poweron() waits for: the modeset of that commit + * has not started yet, rather than failed. Reporting the link as bad + * then makes wlroots replace the output, which powers the CRTC off and + * on again, and the next wake does the same, forever. */ - if (*connected && dcp_is_typec_output(dcp) && + if (*connected && dcp_is_typec_output(dcp) && !poweron_hpd && !READ_ONCE(dcp->mode_state.valid) && !READ_ONCE(dcp->mode_state.changing)) { dcp->swap_watchdog_retrains = 0; diff --git a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py index 7145d3b13d2298..88943830f65441 100644 --- a/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py +++ b/tools/testing/selftests/drivers/gpu/apple-dcp/rate.py @@ -925,7 +925,7 @@ def hotplug(root): struct { bool valid,changing; } mode_state; struct work_struct external_stop_work,vblank_wq; bool external_physically_absent,main_display,typec_crtc_off,external_cable,external_error,external_power_attempted,external_stopped; - bool external_hotplug_pending,external_hotplug_connected; + bool external_hotplug_pending,external_hotplug_connected,typec_poweron_hpd; u64 external_stop_generation,external_rx_generation,external_crtc_generation; unsigned swap_watchdog_retrains;int typec_iomfb_hpd_ready;''') stubs = r''' From bd33a717aaef9b0b0e1838f8a424571b4138a3b5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:01 -0400 Subject: [PATCH 555/631] soc: apple: sep: finish combining the SEP work of both branches Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. integration's SEP driver is the base: its enrolment budget and stages, Catacomb save on deletion, entropy requests kept out of system sleep, the M2 Pro/Max key store chosen by system firmware, the T8122 profile and the private-key round trip for trusted keys. The J700 work from aurora-wip sits on top: authenticated sensor sessions and their persistence, capture draining and binding, serialized SBIO exchanges, and the sensor SPI mode and enable GPIO taken from the board. Remove the older duplicate profile and key-store definitions that the replay left behind. The aurora-wip defaults stay: the module builds by default and loads through its OF alias, with xART writes and first-boot identity provisioning enabled. Update the documentation to say so, and refresh the Arch package checksums of the documents it installs. The Neo GPU environment helper checks the asahi and apple-drm drivers again. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/soc/apple/pm.rs | 36 +---- drivers/soc/apple/pm_shim.c | 64 +------- drivers/soc/apple/profile.rs | 79 +--------- drivers/soc/apple/sbio.rs | 139 ++++++------------ drivers/soc/apple/sep.rs | 74 ++-------- drivers/soc/apple/shim.h | 2 - drivers/soc/apple/sks.rs | 128 ---------------- tools/aurora-sep/README.md | 4 + tools/aurora-sep/neo/mesa-neo-env | 4 +- .../packaging/arch/aurora-sep/PKGBUILD | 4 +- tools/aurora-sep/tests/test_neo_gpu.py | 4 +- tools/aurora-sep/tests/test_sensor_wake.py | 16 +- 12 files changed, 95 insertions(+), 459 deletions(-) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index 3c7a844b521344..a517e6e448732b 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -23,31 +23,6 @@ static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; fn sep_pm_unregister(); -} - -/// # Safety -/// Called by `pm_shim.c` from the PM notifier chain, in process context. -unsafe extern "C" fn pm_event(entering: bool) -> c_int { - let ptr = SEP.load(Ordering::Acquire); - if ptr.is_null() { - return 0; - } - // SAFETY: produced by `into_foreign` in `register`, and reclaimed by - // `unregister` only after `sep_pm_unregister` has waited out any running - // notifier call, so it is live for this borrow. - let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; - if entering { - if let Err(error) = sep.sleep_prepare() { - // A failing notifier is not included in the PM chain's rollback. - sep.sleep_finished(); - return error.to_errno(); - } - } else { - sep.sleep_finished(); - } - 0 - fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; - fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; } @@ -66,20 +41,25 @@ pub(crate) fn keep_domains(dev: &kernel::device::Device) -> Result<()> { /// # Safety /// Called by `pm_shim.c` from the PM notifier chain, in process context. -unsafe extern "C" fn pm_event(entering: bool) { +unsafe extern "C" fn pm_event(entering: bool) -> c_int { let ptr = SEP.load(Ordering::Acquire); if ptr.is_null() { - return; + return 0; } // SAFETY: produced by `into_foreign` in `register`, and reclaimed by // `unregister` only after `sep_pm_unregister` has waited out any running // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; if entering { - sep.sleep_prepare(); + if let Err(error) = sep.sleep_prepare() { + // A failing notifier is not included in the PM chain's rollback. + sep.sleep_finished(); + return error.to_errno(); + } } else { sep.sleep_finished(); } + 0 } pub(crate) fn register(sep: Arc) -> Result<()> { diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 21993dd25667f9..2ef96c5fd96ebe 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -14,12 +14,12 @@ #include "shim.h" -static void (*sep_pm_event)(bool entering); +static int (*sep_pm_event)(bool entering); static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) { - void (*event)(bool entering) = READ_ONCE(sep_pm_event); + int (*event)(bool entering) = READ_ONCE(sep_pm_event); if (!event) return NOTIFY_DONE; @@ -28,8 +28,7 @@ static int sep_pm_notify(struct notifier_block *nb, unsigned long action, case PM_SUSPEND_PREPARE: case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: - event(true); - return NOTIFY_OK; + return notifier_from_errno(event(true)); case PM_POST_SUSPEND: case PM_POST_HIBERNATION: case PM_POST_RESTORE: @@ -44,7 +43,7 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(void (*event)(bool entering)) +int sep_pm_register(int (*event)(bool entering)) { int ret; @@ -105,59 +104,4 @@ int sep_pm_keep_domains(void *ptr) dev_pm_syscore_device(list->pd_devs[i], true); return devm_add_action_or_reset(dev, sep_pm_release_domains, list); - * System-sleep notifier for the SEP driver. It only forwards "about to sleep" - * and "back from sleep" to Rust, which ends any Touch ID capture before the - * system suspends; see pm.rs. - */ -#include -#include -#include - -#include "shim.h" - -static int (*sep_pm_event)(bool entering); - -static int sep_pm_notify(struct notifier_block *nb, unsigned long action, - void *data) -{ - int (*event)(bool entering) = READ_ONCE(sep_pm_event); - - if (!event) - return NOTIFY_DONE; - - switch (action) { - case PM_SUSPEND_PREPARE: - case PM_HIBERNATION_PREPARE: - case PM_RESTORE_PREPARE: - return notifier_from_errno(event(true)); - case PM_POST_SUSPEND: - case PM_POST_HIBERNATION: - case PM_POST_RESTORE: - event(false); - return NOTIFY_OK; - default: - return NOTIFY_DONE; - } -} - -static struct notifier_block sep_pm_nb = { - .notifier_call = sep_pm_notify, -}; - -int sep_pm_register(int (*event)(bool entering)) -{ - int ret; - - WRITE_ONCE(sep_pm_event, event); - ret = register_pm_notifier(&sep_pm_nb); - if (ret) - WRITE_ONCE(sep_pm_event, NULL); - return ret; -} - -/* Returns only once no notifier call is still running. */ -void sep_pm_unregister(void) -{ - unregister_pm_notifier(&sep_pm_nb); - WRITE_ONCE(sep_pm_event, NULL); } diff --git a/drivers/soc/apple/profile.rs b/drivers/soc/apple/profile.rs index b8e2d684649db2..af803b13b7ca3b 100644 --- a/drivers/soc/apple/profile.rs +++ b/drivers/soc/apple/profile.rs @@ -118,14 +118,6 @@ pub(crate) struct KeybagCreate { /// (22G74) sepOS, whose request shapes are the 13.5 ones; the T6020 sepOS /// speaks them too before system firmware 26.3 (see [`t6020_profile`]). #[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum KeyStore { - /// The 13.5 key store. Before any other request the endpoint is - /// initialised with `0x4d`, then `set_env` carrying the ADT's `/defaults` - /// `cpx-encryption-mode`. -/// firmware hands the SEP, not the SoC alone: the T8103 stub boots the 13.5 -/// (22G74) sepOS, whose request shapes are the 13.5 ones; the T6020 sepOS -/// is newer. -#[derive(Clone, Copy, PartialEq, Eq)] pub(crate) enum KeyStore { /// The 13.5 key store. Before any other request the endpoint is /// initialised with `0x4d`, then `set_env` carrying the ADT's `/defaults` @@ -194,18 +186,6 @@ const T8103: PlatformProfile = PlatformProfile { cpx_encryption_mode: 2, }, persistent_enrol: true, - // 13.5 identity create: version 2, type 0x400000, parent -1 (the enclave - // accepts only -1 or <= -10). - keybag_create: KeybagCreate { - variant: 2, - bag_type: 0x40_0000, - arg: -1, - }, - // The j293 and j313 ADTs both carry `/defaults` `cpx-encryption-mode = 2`. - key_store: KeyStore::Sepos13 { - cpx_encryption_mode: 2, - }, - persistent_enrol: true, nonce_session: false, }; @@ -242,10 +222,6 @@ const T6000: PlatformProfile = PlatformProfile { key_store: KeyStore::Sepos13 { cpx_encryption_mode: 2, }, - // The J316s ADT carries `/defaults` `cpx-encryption-mode = 2`. - key_store: KeyStore::Sepos13 { - cpx_encryption_mode: 2, - }, persistent_enrol: true, nonce_session: false, }; @@ -277,6 +253,7 @@ const T6020: PlatformProfile = PlatformProfile { }, key_store: KeyStore::Variant5, persistent_enrol: true, + nonce_session: false, }; /// M2 Pro/Max on system firmware before 26.3. Their enclave refuses the @@ -295,7 +272,6 @@ const T6020_SEPOS13: PlatformProfile = PlatformProfile { cpx_encryption_mode: 2, }, ..T6020 - nonce_session: false, }; /// MacBook Neo. The J700 ADT records a pre-booted SEP, a spi2 Mesa sensor @@ -371,6 +347,7 @@ const T8112: PlatformProfile = PlatformProfile { }, key_store: KeyStore::Variant5, persistent_enrol: true, + nonce_session: false, }; /// MacBook Air M3 (J613, J615). The J613 and J615 ADTs (26.6.2, 25G83) are @@ -394,37 +371,6 @@ const T8122: PlatformProfile = PlatformProfile { }, wide_dma_mask: true, ..T8112 -/// warm-registration platforms (T6020, T8140). -const T8112: PlatformProfile = PlatformProfile { - name: "T8112/J415", - shmem_capacity: 0x4_0000, - shmem_first_item: b"CINP", - bootstrap: Bootstrap::WarmRegister, - identity: IdentitySource::Chosen, - sensor: SensorProfile { - controller_base: 0x2_3510_8000, - chip_select: 0, - max_hz: 8_000_000, - cs_setup_ns: 20, - cs_hold_ns: 20, - mode: SpiMode::Mode1, - expected_id: 0x3352, - capture_qualified: false, - }, - dart_range_required: true, - wide_dma_mask: false, - firmware_region: c"sepfw", - // J415 accepted the 13.5 GET_CAPABILITIES/SET_ENV init but refused the - // strict CREATE_KEYBAG (mailbox status -1, empty reply), as J700 did; - // use the T6020 form like the other warm-registration platforms. - keybag_create: KeybagCreate { - variant: 5, - bag_type: 0, - arg: -1, - }, - key_store: KeyStore::Variant5, - persistent_enrol: true, - nonce_session: false, }; static_assert!(T8103.shmem_capacity == 0x30000); @@ -574,27 +520,6 @@ pub(crate) fn detect() -> Result<&'static PlatformProfile> { if machine_has(b"apple,t6001") { return Ok(&T6000); } - // t6020.dtsi is defined as a cut-down t6021: it includes t6021.dtsi and - // disables the parts the smaller die lacks. Both pull in t602x-die0.dtsi, - // so the SEP, its DART and the SPI controller sit at the same addresses - // with the same interrupts on either part, and the T6020 constants apply - // to the M2 Max unchanged. - if machine_has(b"apple,t6021") { - return Ok(&T6020); - } - if machine_has(b"apple,t6000") { - return Ok(&T6000); - } - // t6000.dtsi is defined as a cut-down t6001: it includes t6001.dtsi and - // deletes the parts the smaller die lacks. Both pull in t600x-die0.dtsi, - // so the SEP, its mailbox, its DART and the SPI controller sit at the same - // addresses with the same interrupts on either part, and the T6000 - // constants apply to the M1 Max unchanged. The Mac Studio (j375c) is also - // a t6001 but leaves the SEP disabled, so nothing binds there; apple,t6002 - // (M1 Ultra, no built-in sensor) stays unmapped. - if machine_has(b"apple,t6001") { - return Ok(&T6000); - } if machine_has(b"apple,t8140") { return Ok(&T8140); } diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 6137215f57b305..434a8b42871495 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -1046,6 +1046,12 @@ impl SepData { self.register_ool(&self.ool_sbio)?; self.sbio_ready.store(true, Relaxed); + // The 13.5 sepOS needs request 0x73 + // with u32(1) before its first ordinary SBIO transaction. + // Without it, J313 can complete sensor setup and authenticate + // its SCRD context but rejects BEGIN_ENROL. Buffer readiness is + // required by this transaction; revoke it if initialization fails. + // The J700 nonce-session protocol sends the same request with u32(6). let init_word = if self.profile.nonce_session { Some(6u32) } else if matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) { @@ -1060,27 +1066,11 @@ impl SepData { match result { Ok(done) if done.status.is_ok() && (self.profile.nonce_session || done.payload.is_empty()) => { - dev_info!(self.dev, "sbio: communication initialization accepted\n"); - } - Ok(done) => { - self.sbio_ready.store(false, Relaxed); - dev_err!(self.dev, "sbio: communication init refused: status {}, {} bytes\n", - done.status, done.payload.len()); - return Err(EIO); - } - Err(e) => { - self.sbio_ready.store(false, Relaxed); - dev_err!(self.dev, "sbio: communication init transport failed: {:?}\n", e); - if matches!(self.profile.key_store, profile::KeyStore::Sepos13 { .. }) { - // 22G74 AppleMesaSEPDriver::initSbioCommunication sends 0x73 - // with u32(1) before its first ordinary SBIO transaction. - // Without it, J313 can complete sensor setup and authenticate - // its SCRD context but rejects BEGIN_ENROL. Buffer readiness is - // required by this transaction; revoke it if initialization fails. - let result = self.sbio_transfer_raw(0x73, c"INIT_SBIO_COMMUNICATION", &1u32.to_le_bytes()); - match result { - Ok(done) if done.status.is_ok() && done.payload.is_empty() => { - dev_info!(self.dev, "sbio: native 13.5 communication initialization accepted\n"); + if self.profile.nonce_session { + dev_info!(self.dev, "sbio: communication initialization accepted\n"); + } else { + dev_info!(self.dev, "sbio: native 13.5 communication initialization accepted\n"); + } } Ok(done) => { self.sbio_ready.store(false, Relaxed); @@ -1176,22 +1166,17 @@ impl SepData { self.sensor_calibrated.store(true, Relaxed); if parameters.is_none() { let Some(reloaded_patch) = self.wake_sensor() else { + diag::sensor_brought_up(bind, false); return false; }; patch = reloaded_patch; } - let Some(reloaded_patch) = self.wake_sensor() else { - diag::sensor_brought_up(bind, false); - return false; - }; - patch = reloaded_patch; } let ok = match parameters { Some(parameters) => self.finish_sensor_setup(patch, parameters), None => self.complete_bringup(patch), }; - let ok = self.complete_bringup(patch); diag::sensor_brought_up(bind, ok); // The sensor is patched and idle here -- the only safe moment to // configure the data-ready interrupt, which is then left alone for the @@ -2077,18 +2062,26 @@ impl SepData { return None; } - // Registration can reset the sensor session. Do not reuse the - // acknowledgement sampled before CLEAR_STATE and authentication. - let st = match sensor::status() { - Ok(st) if !st.is_silent() => st, - _ => { - dev_warn!(self.dev, "sensor: status unavailable after registration\n"); - return None; + if self.profile.nonce_session { + // Registration can reset the sensor session. Do not reuse the + // acknowledgement sampled before CLEAR_STATE and authentication. + let st = match sensor::status() { + Ok(st) if !st.is_silent() => st, + _ => { + dev_warn!(self.dev, "sensor: status unavailable after registration\n"); + return None; + } + }; + if st.patch_ack() == sensor::PATCH_ACCEPTED + && st.state != sensor::STATE_NEEDS_PATCH + { + return Some(PatchLoaded(())); } - }; - if st.patch_ack() == sensor::PATCH_ACCEPTED - && st.state != sensor::STATE_NEEDS_PATCH - { + return self.apply_sensor_patch(st.patch_ack()); + } + + // The patch decision is made on status[8], not on state 9. + if st.patch_ack() == sensor::PATCH_ACCEPTED { return Some(PatchLoaded(())); } return self.apply_sensor_patch(st.patch_ack()); @@ -2158,7 +2151,13 @@ impl SepData { return false; } }; - if self + if self.profile.nonce_session { + if !self.sensor_session_request(request.opcode(), request.name(), request.payload()) { + dev_err!(self.dev, "sensor: calibration refused (source {}, {} bytes)\n", + source, request.payload().len()); + return false; + } + } else if self .sbio_transfer_raw(request.opcode(), request.name(), request.payload()) .ok() .filter(|done| done.status.is_ok()) @@ -2515,7 +2514,7 @@ impl SepData { } CaptureWait::Abandon => { // A cancelled enrolment also abandons; only a live one is a fault. - if bio::enrol_is_live(&self.bio_session.lock()) { + if bio::enrol_is_live(&self.bio_session.lock(), self.capture_epoch.load(Relaxed)) { dev_err!( self.dev, "enrol: the sensor status became unreadable during capture {}\n", @@ -3040,58 +3039,6 @@ impl SepData { self.captures_running.fetch_sub(1, Relaxed); } - /// The system is about to suspend. End any capture in progress, hold new - /// ones, and wait for the enrol or verify work to stop touching the sensor - /// before devices go down. - pub(crate) fn sleep_prepare(&self) { - let woke = { - let mut session = self.bio_session.lock(); - self.suspending.store(true, Relaxed); - bio::abort_capture(&mut session) - }; - if woke { - dev_info!( - self.dev, - "Touch ID: ending the capture in progress before sleep\n" - ); - self.bio_wake(); - } - - let mut waited: u32 = 0; - while self.captures_running.load(Relaxed) != 0 { - if waited >= SLEEP_DRAIN_MS { - dev_warn!( - self.dev, - "Touch ID: a capture was still running {} ms after the system started to sleep; continuing\n", - SLEEP_DRAIN_MS - ); - return; - } - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); - waited += 10; - } - /// Called by the enrol and verify work items before they touch the sensor. - /// Returns false if the system is going to sleep; the session has then - /// been ended and the capture must not start. A true return must be - /// paired with [`Self::capture_end`]. - pub(crate) fn capture_begin(&self) -> bool { - let mut session = self.bio_session.lock(); - if self.suspending.load(Relaxed) { - let woke = bio::abort_capture(&mut session); - drop(session); - if woke { - self.bio_wake(); - } - return false; - } - self.captures_running.fetch_add(1, Relaxed); - true - } - - pub(crate) fn capture_end(&self) { - self.captures_running.fetch_sub(1, Relaxed); - } - /// The enrol or verify work item `epoch` has stopped. pub(crate) fn capture_worker_stopped(&self, epoch: u64) { bio::worker_stopped(&mut self.bio_session.lock(), epoch); @@ -3410,7 +3357,9 @@ impl SepData { self.templates_restored.store(false, Relaxed); // The nonce path synced the view before COMPLETE_INIT to refresh SCRD. // Re-arm the final proof and its bounded retry on every cold restore. - self.device_view_synced.store(false, Relaxed); + if self.profile.nonce_session { + self.device_view_synced.store(false, Relaxed); + } if !candidate { dev_err!( self.dev, @@ -4413,7 +4362,6 @@ impl IdentityV1 { pub(crate) struct MatchResult { user_id: i32, identity: [u8; IDENTITY_UUID_LEN], - flags: u32, } impl MatchResult { @@ -4431,7 +4379,6 @@ impl MatchResult { bytes[MR_USER_ID + 3], ]), identity, - flags: u32::from_le_bytes(bytes[MR_FLAGS..MR_FLAGS + 4].try_into().ok()?), }) } @@ -4440,7 +4387,7 @@ impl MatchResult { } pub(crate) fn matches(&self, user: UserId) -> bool { - self.flags & MR_FLAG_MATCH != 0 && self.user_id == user.value() + self.user_id == user.value() } } diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 697268980a0836..8ff12dfefab8ec 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -566,8 +566,6 @@ struct SepData { #[pin] endpoints: Mutex, - #[pin] - trusted_ready: CondVar, #[pin] control: Mutex, @@ -732,6 +730,7 @@ struct SepData { // deletion durable failed; the next deletion retries the save. Changed // only under `catacomb_writer`. deletion_unsaved: Atomic, + // Signalled when an enrol or verify work item has stopped. #[pin] worker_gone: CondVar, @@ -2023,46 +2022,6 @@ impl SepData { Self::arm_settle(this); } - // Publish completion under the same lock that trusted consumers wait on. - // PHASE_READY ends discovery/initialisation; it is not proof of success. - fn finish_bringup(&self) { - let _guard = self.endpoints.lock(); - self.phase.store(PHASE_READY, Relaxed); - self.trusted_ready.notify_all(); - } - - /// Trusted-key callbacks can run as soon as probe publishes the source, - /// while endpoint discovery and key-store initialisation are still running. - /// Wait only in those consumers, leaving bringup free to establish the key. - pub(crate) fn wait_for_trusted_ready(&self) -> Result<()> { - let mut remaining = time::msecs_to_jiffies(EXCHANGE_TIMEOUT_MS); - let mut guard = self.endpoints.lock(); - loop { - if self.shutting_down.load(Relaxed) { - return Err(ENODEV); - } - if self.phase.load(Relaxed) == PHASE_READY { - // A completed but failed attach must not look ready. - if !guard.eps.iter().any(|e| e.id == proto::EP_SKS) { - return Err(ENODEV); - } - if !self.sks_initialized.load(Relaxed) { - return Err(EIO); - } - return Ok(()); - } - if remaining == 0 { - dev_err!(self.dev, "trusted-keys: key-store bringup timed out\n"); - return Err(ETIMEDOUT); - } - match self.trusted_ready.wait_interruptible_timeout(&mut guard, remaining) { - CondVarTimeoutResult::Woken { jiffies } => remaining = jiffies, - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => remaining = 0, - } - } - } - fn endpoint_present(&self, id: u8) -> bool { self.endpoints.lock().eps.iter().any(|e| e.id == id) } @@ -2145,7 +2104,6 @@ impl SepData { // Release any capture start still held for a resume that will not be // reported now. self.sleep_finished(); - self.release_firmware_mapping(); trusted::unregister(); self.unregister_fv_kernel(); @@ -2331,8 +2289,14 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) && this.capture_begin() { - this.run_enrolment(); + { + // Taken before `capture_begin`, so the sleep drain never counts the + // wait for it. + let _writer = this.catacomb_writer.lock(); + if !this.shutting_down.load(Relaxed) && this.capture_begin() { + this.run_enrolment(); + this.capture_end(); + } } this.capture_worker_stopped(epoch); } @@ -2343,8 +2307,12 @@ impl WorkItem for SepData { fn run(this: Arc) { let epoch = this.capture_epoch.load(Relaxed); - if !this.shutting_down.load(Relaxed) && this.capture_begin() { - this.run_verify(); + { + let _writer = this.catacomb_writer.lock(); + if !this.shutting_down.load(Relaxed) && this.capture_begin() { + this.run_verify(); + this.capture_end(); + } } this.capture_worker_stopped(epoch); } @@ -2522,17 +2490,5 @@ module! { default: 0, description: "Opt-in J414s enrollment test: export owner from the fresh context, then save user before master at completion", }, - provision_keybag: u8 { - default: 0, - description: "Create the Linux identity keybag when none exists; requires xart_writes=1", - }, - os_uuid_hi: u64 { - default: 0, - description: "High 64 bits of an explicit xART OS UUID", - }, - os_uuid_lo: u64 { - default: 0, - description: "Low 64 bits of an explicit xART OS UUID", - }, }, } diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 40aae82b111016..83839b9353691e 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -27,8 +27,6 @@ void sep_cancel_delayed_work_sync(void *work); int sep_pm_register(int (*event)(bool entering)); void sep_pm_unregister(void); -int sep_pm_register(void (*event)(bool entering)); -void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); /* -- hwrng_shim.c ------------------------------------------------------- */ diff --git a/drivers/soc/apple/sks.rs b/drivers/soc/apple/sks.rs index 10b771e9801a4c..4020d089eb0019 100644 --- a/drivers/soc/apple/sks.rs +++ b/drivers/soc/apple/sks.rs @@ -46,36 +46,6 @@ impl Drop for KeybagCreateIntent<'_> { } } -struct KeybagCreateIntent<'a> { - dev: &'a device::Device, - slot: keybag::Slot, - sent: bool, -} - -impl<'a> KeybagCreateIntent<'a> { - fn new(dev: &'a device::Device, slot: keybag::Slot) -> Option { - if let Err(e) = keybag::write_intent(slot) { - dev_err!(dev, "sks: cannot persist keybag-create intent: {:?}\n", e); - return None; - } - Some(Self { dev, slot, sent: false }) - } - - fn sending(&mut self) { - self.sent = true; - } -} - -impl Drop for KeybagCreateIntent<'_> { - fn drop(&mut self) { - if !self.sent { - if let Err(e) = keybag::mark_refused(self.slot) { - dev_err!(self.dev, "sks: cannot mark unsent keybag create retryable: {:?}\n", e); - } - } - } -} - impl SepData { pub(crate) fn on_sks(&self, msg: Message) { let r = crate::sks::decode_sks(&msg); @@ -782,90 +752,6 @@ impl SepData { None => keybag::write_generated_uuid(slot, &wrapped, &uuid, &secret), }; if let Err(e) = committed { - let mut body = image::Body::new(); - body.put_u32(crate::sks::SKS_CREATE_VARIANT_IDENTITY)?; - body.put_u64(crate::sks::SKS_CLIENT_ID)?; - body.put_u32(crate::sks::CreateFlags::none().value())?; - body.put_i32(crate::sks::SpecialHandle::first_identity().value())?; - body.put_blob(secret)?; - body.put_blob(&[])?; - body.put_blob(uuid)?; - body.put_blob(&[])?; - body.put_u64(0)?; - body.put_u64(0)?; - body.put_blob(&[])?; - let img = image::build_request(image::Version::V1, self.sks_timestamp_us(), &body)?; - let len = self.sks_image_len(&img)?; - let msg = crate::sks::encode_sks_create(self.sks_next_seq(), len).ok_or(EINVAL)?; - Ok(SksRequest { name: crate::sks::SKS_CREATE_NAME, msg, img }) - } - - pub(crate) fn sks_provision_identity_keybag(&self) -> bool { - let proof = match keybag::read(keybag::Slot::Identity) { - Ok(keybag::State::Present(_)) => return true, - Ok(keybag::State::Absent(proof)) => proof, - Err(e) => { - dev_err!(self.dev, "sks: identity keybag state is ambiguous: {:?}\n", e); - return false; - } - }; - let slot = proof.slot(); - let mut intent = match KeybagCreateIntent::new(&self.dev, slot) { - Some(intent) => intent, - None => return false, - }; - let mut secret_bytes = KVec::new(); - if secret_bytes.resize(SKS_SECRET_LEN, 0, GFP_KERNEL).is_err() - || shim::random_bytes(&mut secret_bytes).is_err() - { - return false; - } - let secret = Secret(secret_bytes); - let mut uuid = [0u8; crate::sks::SKS_IDENTITY_UUID_LEN]; - if shim::random_bytes(&mut uuid).is_err() { - return false; - } - uuid[6] = (uuid[6] & 0x0f) | 0x40; - uuid[8] = (uuid[8] & 0x3f) | 0x80; - - let request = match self.sks_req_create_identity_keybag(&secret, &uuid, proof) { - Ok(request) => request, - Err(e) => { - dev_err!(self.dev, "sks: cannot build identity keybag request: {:?}\n", e); - return false; - } - }; - intent.sending(); - let Some(out) = self.sks_exchange(request.name, request.msg, &request.img) else { - return false; - }; - let Some(body) = self.sks_report_response(crate::sks::SKS_CREATE_NAME, &out) else { - return false; - }; - if out.reply.status != 0 || body.len() < 12 { - dev_err!(self.dev, "sks: CREATE_KEYBAG failed: mailbox {}, body {} bytes\n", out.reply.status, body.len()); - return false; - } - let variant = u32::from_le_bytes(body[0..4].try_into().unwrap()); - let raw_handle = i32::from_le_bytes(body[4..8].try_into().unwrap()); - let Some((_fv_data, end)) = image::read_blob(body, 8) else { - return false; - }; - if variant != crate::sks::SKS_CREATE_VARIANT_IDENTITY || raw_handle < 0 || end != body.len() { - dev_err!(self.dev, "sks: invalid CREATE_KEYBAG reply: variant {}, handle {}\n", variant, raw_handle); - return false; - } - let handle = crate::sks::KeyBagHandle::from_create_reply(raw_handle); - let Some(bag_uuid) = self.sks_read_uuid(handle) else { - return false; - }; - let Some(out) = self.sks_send(self.sks_req_copy_keybag(handle)) else { - return false; - }; - let Some(wrapped) = self.wrapped_from_copy_reply(&out, c"new identity keybag") else { - return false; - }; - if let Err(e) = keybag::write_bag_uuid(slot, &wrapped, &bag_uuid, &secret) { dev_err!(self.dev, "sks: cannot commit identity keybag: {:?}\n", e); return false; } @@ -1109,8 +995,6 @@ impl SepData { let Some(uuid) = self.sks_read_uuid(handle) else { dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); let _ = self.sks_send(self.sks_req_unload_keybag(handle)); - let Some(uuid) = self.sks_read_uuid(handle) else { - dev_warn!(self.dev, "sks: loaded keybag has no readable UUID\n"); return None; }; @@ -1290,9 +1174,6 @@ pub(crate) const SKS_UNWRAP_VEK_NAME: &CStr = c"FV_UNWRAP_VEK"; const OP_SKS_SET_PROTECTION: u8 = 0x47; pub(crate) const SKS_SET_PROTECTION_NAME: &CStr = c"FV_SET_PROTECTION"; -const OP_SKS_SET_PROTECTION: u8 = 0x47; -pub(crate) const SKS_SET_PROTECTION_NAME: &CStr = c"FV_SET_PROTECTION"; - const OP_SKS_GET_BLOB_STATE: u8 = 0x48; pub(crate) const SKS_GET_BLOB_STATE_NAME: &CStr = c"FV_GET_BLOB_STATE"; @@ -1379,11 +1260,6 @@ pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; const OP_SKS_DEVICE_STATE_TRANSITION: u8 = 0x18; pub(crate) const SKS_DEVICE_STATE_NAME: &CStr = c"DEVICE_STATE_TRANSITION"; -pub(crate) const SKS_LOAD_NAME: &CStr = c"LOAD_KEYBAG"; - -const OP_SKS_CHANGE_LOCK_STATE: u8 = 0x04; - -pub(crate) const SKS_LOCK_STATE_NAME: &CStr = c"CHANGE_LOCK_STATE"; pub(crate) fn encode_sks_device_state_transition(seq: Sequence, len: ImageLen) -> Message { encode_sks_raw(OP_SKS_DEVICE_STATE_TRANSITION, seq.value(), len.value()) } @@ -1408,10 +1284,6 @@ impl KeyBagHandle { KeyBagHandle(v) } - pub(crate) const fn from_create_reply(v: i32) -> KeyBagHandle { - KeyBagHandle(v) - } - pub(crate) const fn from_load_reply(v: i32) -> KeyBagHandle { KeyBagHandle(v) } diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index c9a1d32996422d..5de174ba4c7994 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -90,6 +90,10 @@ ninja -C build meson test -C build --print-errorlogs ``` +The aurora driver for libfprint is also maintained in +https://github.com/aurora-silicon/aurora-sep-userspace, as source and as +distribution packages. + Adjust the patch path for your checkout. The patch includes a fix for an upstream Meson test loop when introspection is disabled. In the Aurora `linux-build` guest, the additional packages needed for this build are diff --git a/tools/aurora-sep/neo/mesa-neo-env b/tools/aurora-sep/neo/mesa-neo-env index b36054c2ac1c56..0ea9eb32902e7a 100644 --- a/tools/aurora-sep/neo/mesa-neo-env +++ b/tools/aurora-sep/neo/mesa-neo-env @@ -48,7 +48,7 @@ mesa_neo_display_admit() { [ -e "$_neo_card" ] || continue if mesa_neo_has_compatible "$_neo_card/device/of_node/compatible" apple,agx-t8140; then [ "$(readlink -f "$_neo_card/device" 2>/dev/null)" = "$_neo_gpu_device" ] && - [ "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" = asahi_neo ] || return 1 + [ "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" = asahi ] || return 1 # The GPU primary node has no KMS connectors. for _neo_connector in "$_neo_card"-*; do [ ! -e "$_neo_connector" ] || return 1 @@ -57,7 +57,7 @@ mesa_neo_display_admit() { fi _neo_cards=$((_neo_cards+1)) # A firmware framebuffer cannot use the Asahi render-only KMS route. - [ "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" = apple-drm-neo ] || return 1 + [ "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" = apple-drm ] || return 1 mesa_neo_has_compatible "$_neo_card/device/of_node/compatible" apple,t8140-display-subsystem || return 1 [ -r "$_neo_root/dev/dri/${_neo_card##*/}" ] && [ -w "$_neo_root/dev/dri/${_neo_card##*/}" ] || return 1 for _neo_connector in "$_neo_card"-*; do diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index 5c0f665f9dcfe1..bcb0917d589869 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -40,8 +40,8 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - 'ed507587d3356ff2713e988e024b183772bab14061bf787bcaef28691867af0d' - '3d92b3be06bc1b670666dec3d0032893fe8c7be428a4f1200d7c9803980c769a') + 'a87584f87b8b49c19333fd9c5a8e469401756ec08519fd6cc655bef74b5d284f' + 'a950153f3f4e600f29b8e1976bb7de71f96bbf6b922cb88512c40409a8d1b6ef') # A unit and the helper it runs ($1, installed as $3), pointed at the # packaged helper. diff --git a/tools/aurora-sep/tests/test_neo_gpu.py b/tools/aurora-sep/tests/test_neo_gpu.py index 6b9af473d0d52d..a9dddb2488f977 100644 --- a/tools/aurora-sep/tests/test_neo_gpu.py +++ b/tools/aurora-sep/tests/test_neo_gpu.py @@ -144,7 +144,7 @@ def hook(self, intent=True, profile='j700-g17p-hal200', abi=0, display=True): (self.tmp / 'dev').mkdir(exist_ok=True) (self.tmp / 'dev/renderD128').touch() gpu = render.parent - driver = self.tmp / 'drivers/asahi_neo' + driver = self.tmp / 'drivers/asahi' driver.mkdir(parents=True, exist_ok=True) if not (gpu / 'driver').is_symlink(): (gpu / 'driver').symlink_to(driver) @@ -156,7 +156,7 @@ def hook(self, intent=True, profile='j700-g17p-hal200', abi=0, display=True): kms = self.tmp / 'sys/card2/device/of_node' kms.mkdir(parents=True, exist_ok=True) (kms / 'compatible').write_bytes(b'apple,t8140-display-subsystem\0') - dcp = self.tmp / 'drivers/apple-drm-neo' + dcp = self.tmp / 'drivers/apple-drm' dcp.mkdir(parents=True, exist_ok=True) if not (kms.parent / 'driver').is_symlink(): (kms.parent / 'driver').symlink_to(dcp) diff --git a/tools/aurora-sep/tests/test_sensor_wake.py b/tools/aurora-sep/tests/test_sensor_wake.py index cd1c9c74a793eb..64fe6898a37fca 100644 --- a/tools/aurora-sep/tests/test_sensor_wake.py +++ b/tools/aurora-sep/tests/test_sensor_wake.py @@ -1,5 +1,9 @@ # SPDX-License-Identifier: MIT -"""Execute the actual wake_sensor body with synthetic status/registration mocks.""" +"""Execute the actual wake_sensor body with synthetic status/registration mocks. + +The post-registration status read applies to nonce-session (J700) profiles; +other profiles keep the patch decision on the status read before registration. +""" from pathlib import Path import subprocess import tempfile @@ -77,7 +81,7 @@ def test_post_registration_status_drives_patch_readiness(self): MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,1);m.registered=true;}); true } fn apply_sensor_patch(&self,ack:u8)->Option { - MOCK.with(|m| {let mut m=m.borrow_mut();assert_eq!(m.reads,2);m.patch=Some(ack);}); + MOCK.with(|m| {let mut m=m.borrow_mut();m.patch=Some(ack);}); Some(PatchLoaded(())) } ''' @@ -91,9 +95,15 @@ def test_post_registration_status_drives_patch_readiness(self): (Some(sensor::Status{state:0,ack:0,silent:false}),true,Some(0)), ] { MOCK.with(|m| *m.borrow_mut()=Mock{next,..Mock::default()}); - assert_eq!(Driver::new().wake_sensor().is_some(),ready); + let driver = Driver { profile: Profile { nonce_session: true }, sensor_power_ns: AtomicU64::new(0) }; + assert_eq!(driver.wake_sensor().is_some(),ready); MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,2);assert!(m.registered);assert_eq!(m.patch,patch);}); } + // Without a nonce session the pre-registration status decides, as before. + MOCK.with(|m| *m.borrow_mut()=Mock::default()); + let driver = Driver::new(); + assert!(driver.wake_sensor().is_some()); + MOCK.with(|m| {let m=m.borrow();assert_eq!(m.reads,1);assert!(m.registered);assert_eq!(m.patch,None);}); } ''' with tempfile.TemporaryDirectory() as directory: From 4d51d600407d82604ddeb30c51957077d11192b3 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 12 Oct 2026 14:01:54 +0200 Subject: [PATCH 556/631] drm: apple: audio: play into the void without a sink On the M1 Mac mini with two Fujitsu E24T-7 Pro monitors, which have no audio (single-block EDID 1.3), PipeWire logs spa.alsa: hw:0p: snd_pcm_start: No such device or address about 130000 times a second, for hours, as soon as something plays to a DCP audio output: 4.1 million log lines in one boot, with journald dropping most of them, and a core kept busy. Without a sink, opening, configuring and preparing the PCM succeed, as userspace drops an output that fails its probe for good, but START fails with -ENXIO. That does not stop userspace from trying: PipeWire starts playback after its first write and ignores the error, and a stream that never starts never makes it wait, so the next wakeup is immediate and tries again. Any other error ends up in its recovery, which prepares and starts again. Play into the void instead, as HDMI PCMs on other platforms do: without a sink, or once the sink the stream was opened for is gone, START succeeds and the stream runs on the clock, without DMA and without a DCP audio link. The position follows the time since START and a work item reports the periods. The samples are dropped. A stream playing into the void is not stopped as disconnected when a sink goes away; a sink attached later is used by the next stream opened. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/audio.c | 113 ++++++++++++++++++++++++++++++---- 1 file changed, 100 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index cfbb09863fa4b7..b2706c7ac3f823 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -53,6 +53,11 @@ struct dcp_audio { struct snd_pcm_chmap_elem selected_chmap; struct dcp_sound_cookie selected_cookie; + /* stream runs without a sink, see dcpaud_void_start() */ + bool void_running; + ktime_t void_start; + u64 void_periods; + struct delayed_work void_work; void *elements; void *productattrs; @@ -82,7 +87,7 @@ static const struct snd_pcm_hardware dcp_pcm_hw = { * configured, as HDMI PCMs on other platforms can. Userspace (PipeWire, * PulseAudio) probes the PCM once when the card appears and drops the * output for good if that fails, so a monitor attached later would never - * show up. Starting the stream still fails until a sink is connected. + * show up. The stream then plays into the void, see dcpaud_void_start(). */ static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | @@ -365,12 +370,10 @@ static int dcp_pcm_hw_params(struct snd_pcm_substream *substream, struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); int ret; - if (dcpaud->open_unplugged) + /* no sink: the stream runs without DMA, see dcpaud_void_start() */ + if (dcpaud->open_unplugged || !dcpaud_connection_up(dcpaud)) return 0; - if (!dcpaud_connection_up(dcpaud)) - return -ENXIO; - ret = dcpaud_select_cookie(dcpaud, params); if (ret < 0) return ret; @@ -410,16 +413,79 @@ static int dcp_pcm_prepare(struct snd_pcm_substream *substream) struct dcp_audio *dcpaud = substream->pcm->private_data; /* alsa-lib prepares right after hw_params; see dcp_pcm_hw_unplugged */ - if (dcpaud->open_unplugged) + if (dcpaud->open_unplugged || !dcpaud_connection_up(dcpaud)) return 0; - if (!dcpaud_connection_up(dcpaud)) - return -ENXIO; - return dcp_audiosrv_prepare(dcpaud->dcp_dev, &dcpaud->selected_cookie); } +static u64 dcpaud_void_period_ns(struct snd_pcm_runtime *runtime) +{ + return div_u64((u64)runtime->period_size * NSEC_PER_SEC, runtime->rate); +} + +static void dcpaud_void_work(struct work_struct *work) +{ + struct dcp_audio *dcpaud = container_of(to_delayed_work(work), + struct dcp_audio, void_work); + struct snd_pcm_substream *substream = dcpaud->substream; + ktime_t next; + s64 delay; + + snd_pcm_period_elapsed(substream); + + snd_pcm_stream_lock_irq(substream); + if (dcpaud->void_running) { + /* the next period boundary, so that periods do not drift */ + next = ktime_add_ns(dcpaud->void_start, + ++dcpaud->void_periods * + dcpaud_void_period_ns(substream->runtime)); + delay = ktime_to_ns(ktime_sub(next, ktime_get())); + schedule_delayed_work(&dcpaud->void_work, + delay > 0 ? nsecs_to_jiffies(delay) : 0); + } + snd_pcm_stream_unlock_irq(substream); +} + +/* + * Without a sink, DCP has no audio link to start. Failing START does not + * stop userspace from trying: PipeWire retries it on every wakeup, and a + * stream that never starts never makes it wait, so it retries at about + * 130000 times a second for as long as something plays (a monitor + * without audio, or none attached). Play into the void instead, as HDMI + * PCMs elsewhere do: the stream runs on the clock, without DMA, and its + * samples are dropped. A sink attached later gets the next stream opened. + */ +static int dcpaud_void_start(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + u64 period_ns = dcpaud_void_period_ns(substream->runtime); + + dev_dbg(dcpaud->dev, "no sink, playing into the void\n"); + dcpaud->void_running = true; + dcpaud->void_start = ktime_get(); + dcpaud->void_periods = 0; + schedule_delayed_work(&dcpaud->void_work, nsecs_to_jiffies(period_ns)); + + return 0; +} + +static snd_pcm_uframes_t dcp_pcm_pointer(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + struct snd_pcm_runtime *runtime = substream->runtime; + u64 frames; + + if (!dcpaud->void_running) + return snd_dmaengine_pcm_pointer(substream); + + frames = mul_u64_u32_div(ktime_to_ns(ktime_sub(ktime_get(), + dcpaud->void_start)), + runtime->rate, NSEC_PER_SEC); + return do_div(frames, runtime->buffer_size); +} + static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) { struct dcp_audio *dcpaud = substream->pcm->private_data; @@ -428,8 +494,8 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) switch (cmd) { case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: - if (!dcpaud_connection_up(dcpaud)) - return -ENXIO; + if (dcpaud->open_unplugged || !dcpaud_connection_up(dcpaud)) + return dcpaud_void_start(substream); WARN_ON(pm_runtime_get_sync(dcpaud->dev) < 0); ret = dcp_audiosrv_startlink(dcpaud->dcp_dev, @@ -440,6 +506,11 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: + if (dcpaud->void_running) { + /* sync_stop waits for the work */ + dcpaud->void_running = false; + return 0; + } break; default: @@ -468,6 +539,19 @@ static int dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) return 0; } +/* + * ALSA calls this once a stream has stopped, outside the stream lock, before + * the next hw_params, prepare or hw_free and at close. + */ +static int dcp_pcm_sync_stop(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + cancel_delayed_work_sync(&dcpaud->void_work); + + return 0; +} + struct snd_pcm_ops dcp_playback_ops = { .open = dcp_pcm_open, .close = dcp_pcm_close, @@ -475,7 +559,8 @@ struct snd_pcm_ops dcp_playback_ops = { .hw_free = dcp_pcm_hw_free, .prepare = dcp_pcm_prepare, .trigger = dcp_pcm_trigger, - .pointer = snd_dmaengine_pcm_pointer, + .sync_stop = dcp_pcm_sync_stop, + .pointer = dcp_pcm_pointer, }; // Transitional workaround: for the chmap control TLV, advertise options @@ -557,7 +642,8 @@ static void dcpaud_report_hotplug(struct dcp_audio *dcpaud, bool connected) if (!connected) { snd_pcm_stream_lock(substream); - if (substream->runtime) + /* a stream without a sink plays on, see dcpaud_void_start() */ + if (substream->runtime && !dcpaud->void_running) snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); snd_pcm_stream_unlock(substream); } @@ -769,6 +855,7 @@ static int dcpaud_probe(struct platform_device *pdev) dcpaud->dev = &pdev->dev; mutex_init(&dcpaud->data_lock); + INIT_DELAYED_WORK(&dcpaud->void_work, dcpaud_void_work); platform_set_drvdata(pdev, dcpaud); return component_add(&pdev->dev, &dcpaud_comp_ops); From 21ea35508e22df1490b681d7006cdab3d6aeb0d5 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:01 -0400 Subject: [PATCH 557/631] phy, thunderbolt, PCI, usb: apple: finish the merge with integration Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. integration's ATC PHY, Thunderbolt and PCIe host code is the base, including its M3 PIPE parking, s2idle port power-off and tunnel cold-initialisation. The J700 DisplayPort tunables, AUX power, lane routing and main-link training, the J700 radio bootstrap and the M5 controller support from aurora-wip apply only on those SoCs. Drop the aurora-wip variants of shared code that integration's versions replace. The J700 fixed-hub USB controller is torn down without running the host and core exit twice. Assisted-by: LLM Signed-off-by: Diljit Singh --- Documentation/PCI/controller/apple-t8140.rst | 6 +- .../pci/controller/pcie-apple-piodma-diag.c | 3 - drivers/pci/controller/pcie-apple.c | 320 ---------------- drivers/phy/apple/atc-j700-dp.h | 2 - drivers/phy/apple/atc.c | 343 ++++++++---------- drivers/thunderbolt/apple.c | 20 +- drivers/usb/dwc3/dwc3-apple.c | 71 +--- 7 files changed, 187 insertions(+), 578 deletions(-) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index bd8b69242b7375..496e9b94d40f60 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only T8140/J700 experimental radio PCIe bootstrap -============================================ +========================================== The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. @@ -34,10 +34,6 @@ teardown, memory reuse and arbitrary downstream devices are unqualified. Fresh hardware qualification must check both radio functions across sleep. Do not remove the controller or its IOMMU while this experiment is active. These lifecycle contracts require further work before production support. -They protect against accidental teardown and are not a security boundary. -Root can still reset through raw configuration writes, including secondary-bus -reset, FLR and Link Disable, change per-link ASPM controls or unbind the root -port's ``pcieport`` driver. Do not perform these operations on an admitted Neo. J700's ``wifi0`` alias and PCI endpoint node allow the public bootloader to pass the unit's own Wi-Fi address through ``local-mac-address``. The zero diff --git a/drivers/pci/controller/pcie-apple-piodma-diag.c b/drivers/pci/controller/pcie-apple-piodma-diag.c index 6c4237826884c7..6f5d9bac0e64e4 100644 --- a/drivers/pci/controller/pcie-apple-piodma-diag.c +++ b/drivers/pci/controller/pcie-apple-piodma-diag.c @@ -55,9 +55,6 @@ #define PIODMA_FIFO_MASK GENMASK(5, 0) #define PIODMA_IRQ_LIMIT 16 -static bool enumerate = true; -module_param(enumerate, bool, 0444); -MODULE_PARM_DESC(enumerate, "Boot-time radio gate (read-only, default on; no effect off J700/T8140)"); /* * The board DT opts in by pointing the host's "apple,piodma" at this engine; * a host without that property stays root-port-only. enumerate=0 keeps every diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index 3d870c5de4bf7f..621739615d7da5 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -116,21 +116,6 @@ bool apple_pcie_tunnel_needs_cold_init(struct device_node *np) return tunables; } EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); -static bool tunnel_kernel_init; -module_param(tunnel_kernel_init, bool, 0644); -MODULE_PARM_DESC(tunnel_kernel_init, - "Cold-initialize apple,pciec-kernel-init PCIe-C ports (experimental, default off)"); - -static unsigned int tunnel_kernel_init_trace_ms; -module_param(tunnel_kernel_init_trace_ms, uint, 0644); -MODULE_PARM_DESC(tunnel_kernel_init_trace_ms, - "Log each PCIe-C cold init step and sleep this long after it (debug)"); - -bool apple_pcie_tunnel_kernel_init_enabled(void) -{ - return READ_ONCE(tunnel_kernel_init); -} -EXPORT_SYMBOL_GPL(apple_pcie_tunnel_kernel_init_enabled); /* T8103 (original M1) and related SoCs */ #define CORE_RC_PHYIF_CTL 0x00024 @@ -843,21 +828,6 @@ static void apple_pcie_port_disable_irq(struct apple_pcie_port *port) raw_spin_lock_irqsave(&port->irq_lock, flags); port->irq_disabled = true; raw_spin_unlock_irqrestore(&port->irq_lock, flags); - * A chained handler is not tracked by IRQD_IRQ_INPROGRESS, and AIC does not - * report IRQCHIP_STATE_ACTIVE. Gate and drain MMIO explicitly before masking - * the parent; disable_irq() alone cannot synchronize this chained handler. - */ -static void apple_pcie_port_disable_irq(struct apple_pcie_port *port) -{ - unsigned long flags; - - if (!port->irq || !port->domain || port->irq_disabled) - return; - - raw_spin_lock_irqsave(&port->irq_lock, flags); - port->irq_disabled = true; - raw_spin_unlock_irqrestore(&port->irq_lock, flags); - disable_irq(port->irq); } static void apple_pcie_port_enable_irq(struct apple_pcie_port *port) @@ -1013,74 +983,6 @@ static int apple_pcie_tunnel_reinitialize(struct apple_pcie_port *port) return 0; } -static void apple_pcie_tunnel_cold_init_step(struct apple_pcie_port *port, - const char *step) -{ - unsigned int ms = READ_ONCE(tunnel_kernel_init_trace_ms); - - if (!ms) - return; - dev_info(port->pcie->dev, "PCIe-C %pOF cold init: %s\n", port->np, step); - msleep(ms); -} - -/* - * Cold-initialize a PCIe-C port that no m1n1 handoff brought up. This - * follows the enablePortHardware() baseline of apple_pcie_tunnel_start() and - * the t8103 cold init of aurora-silicon/linux#8: the port leaves reset with - * APPCLK running and reports RUN before anything touches the root port's - * config space. The downstream (tunnel) reset stays asserted until - * apple_pcie_setup_port() releases it before link training. - */ -static int apple_pcie_tunnel_cold_init(struct apple_pcie_port *port) -{ - struct apple_pcie *pcie = port->pcie; - u32 stat; - int i, ret; - - apple_pcie_tunnel_cold_init_step(port, "debug and fabric tunables"); - apple_pcie_tunnel_apply_tunable(pcie->debug_base, pcie->debug_tunable); - apple_pcie_tunnel_apply_tunable(pcie->fabric_base, - pcie->fabric_tunable); - apple_pcie_tunnel_cold_init_step(port, "port reset table"); - apple_pcie_tunnel_reset_hardware(port); - apple_pcie_tunnel_cold_init_step(port, "port tunables"); - apple_pcie_tunnel_apply_tunable(port->base, port->tunable); - - apple_pcie_tunnel_cold_init_step(port, "PERST off, APPCLK on"); - apple_pcie_port_rmw_set(port, PORT_PERST_OFF, pcie->hw->port_perst); - apple_pcie_port_rmw_set(port, PORT_APPCLK_EN, PORT_APPCLK); - ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, - stat & PORT_STATUS_READY, - 10, 250000, false, port, PORT_STATUS); - if (ret) - return dev_err_probe(pcie->dev, ret, - "PCIe-C %pOF cold init: port did not enter RUN (status %#x)\n", - port->np, stat); - - apple_pcie_tunnel_cold_init_step(port, "root complex tunables"); - apple_pcie_tunnel_apply_tunable(pcie->cfg->win, pcie->rc_tunable); - apple_pcie_tunnel_cold_init_step(port, "APPCLK ungate, counters, Intr2AXI"); - apple_pcie_port_rmw_clear(port, PORT_APPCLK_CGDIS, PORT_APPCLK); - apple_pcie_port_writel(port, PORT_COUNTER_ENABLE, PORT_COUNTER_CTRL); - apple_pcie_port_writel(port, ~0, PORT_INTSTAT); - apple_pcie_port_writel(port, ~0, PORT_LINKCMDSTS); - apple_pcie_tunnel_writel(pcie->intr2axi_base, PCIEC_INTR2AXI_ENABLE, - PCIEC_INTR2AXI_CTRL); - - ret = read_poll_timeout_atomic(apple_pcie_port_readl, stat, - !(stat & PORT_LINKSTS_BUSY), - 10, 250000, false, port, PORT_LINKSTS); - if (ret) - return dev_err_probe(pcie->dev, ret, - "PCIe-C %pOF cold init: link still busy (status %#x)\n", - port->np, stat); - - for (i = 0; i < pcie->hw->max_rid2sid; i++) - apple_pcie_rid2sid_write(port, i, 0); - - dev_info(pcie->dev, "PCIe-C %pOF cold init done, status %#x tunstat %#x\n", - port->np, apple_pcie_port_readl(port, PORT_STATUS), /* * Bring up a port that no firmware handoff left running (t8103, or * T600x/T602x with pcie_apple.tunnel_kernel_init): tunables and the reset @@ -2451,20 +2353,6 @@ static int apple_pcie_tunnel_pci_notify(struct notifier_block *nb, pm_runtime_get_noresume(&pdev->dev); if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) pcie_aspm_remove_cap(pdev, PCI_EXP_LNKCAP_ASPM_L1); - */ -static int apple_pcie_tunnel_pci_notify(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct apple_pcie *pcie = container_of(nb, struct apple_pcie, - tunnel_pci_nb); - struct pci_dev *pdev = to_pci_dev(data); - - if (pci_host_bridge_priv(pci_find_host_bridge(pdev->bus)) != pcie) - return NOTIFY_DONE; - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: - pm_runtime_get_noresume(&pdev->dev); break; case BUS_NOTIFY_DEL_DEVICE: pm_runtime_put_noidle(&pdev->dev); @@ -2716,52 +2604,6 @@ static int apple_pcie_neo_enumerate(struct pci_host_bridge *bridge) goto close_config; apple_piodma_bootstrap_admitted(pcie->piodma_supplier); pci_bus_add_devices(root->subordinate); - for (function = 0; function < 2; function++) { - endpoint = pci_get_slot(root->subordinate, PCI_DEVFN(0, function)); - if (!endpoint) { - ret = -ENODEV; - break; - } - if (pci_read_config_word(endpoint, PCI_COMMAND, &command)) { - ret = -EIO; - } else if (endpoint->vendor != PCI_VENDOR_ID_MEDIATEK || - endpoint->device != (function ? 0x793b : 0x7932)) { - ret = -ENODEV; - } else { - dev_info(pcie->dev, "enumerated %s id=%04x:%04x command=%#x driver=%s\n", - pci_name(endpoint), endpoint->vendor, endpoint->device, command, - endpoint->driver ? endpoint->driver->name : "unbound"); - if (command & PCI_COMMAND_MASTER) { - /* Admission requires cold, unbound endpoints. */ - pci_clear_master(endpoint); - ret = -EIO; - } - if (endpoint->driver) - ret = -EBUSY; - if (!ret) { - /* - * The radios require the equivalent of ASPM performance - * policy before their drivers start. Keep this restriction - * on their shared link, including L1 substates and Clock - * PM, without changing policy on other PCIe links. The - * PCI core tracks the restriction across policy changes. - */ - ret = pci_disable_link_state(endpoint, - PCIE_LINK_STATE_ALL); - if (ret) - pci_err(endpoint, - "cannot disable radio link power states: %d\n", - ret); - } - } - pci_dev_put(endpoint); - if (ret) - break; - } - if (!ret) { - apple_piodma_bootstrap_admitted(pcie->piodma_supplier); - pci_bus_add_devices(root->subordinate); - } close_config: if (ret) { apple_pcie_neo_unscan(root); @@ -2833,7 +2675,6 @@ static int apple_pcie_probe(struct platform_device *pdev) bridge->native_dpc = false; dev_info_once(dev, "Neo radio host: native AER/DPC recovery disabled for retained-DMA bootstrap\n"); } - if (hw->root_bus_only) if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); pcie->base = devm_platform_ioremap_resource(pdev, 1); @@ -3110,8 +2951,6 @@ int apple_pcie_tunnel_restore(struct device *dev) pci_rescan_bus(bridge->bus); pci_walk_bus(bridge->bus, apple_pcie_tunnel_keep_d0, NULL); pcie->bus_stopped = false; - list_for_each_entry(port, &pcie->ports, entry) - apple_pcie_port_enable_irq(port); pci_unlock_rescan_remove(); dev_info(dev, "PCIe-C hierarchy restored after tunnel activation\n"); @@ -3265,13 +3104,6 @@ int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) if (!np) return 0; - /* A qualified bootloader handoff needs no cold initialization. */ - if (!of_property_read_u32(np, "apple,pciec-preinit-status", &stat) && - stat == 1) { - ret = 0; - goto out_np; - } - match = of_match_node(apple_pcie_of_match, np); if (!match) { ret = -ENODEV; @@ -3675,158 +3507,6 @@ static const struct of_device_id apple_pcie_of_match[] = { }; MODULE_DEVICE_TABLE(of, apple_pcie_of_match); -struct apple_pcie_tunnel_map { - struct resource res; - void __iomem *base; -}; - -static int apple_pcie_tunnel_map_named(struct device_node *np, - const char *name, - struct apple_pcie_tunnel_map *map) -{ - int index, ret; - - index = of_property_match_string(np, "reg-names", name); - if (index < 0) - return index; - ret = of_address_to_resource(np, index, &map->res); - if (ret) - return ret; - /* Tunneled PCIe-C MMIO is non-posted, as in the probe path. */ - map->base = ioremap_np(map->res.start, resource_size(&map->res)); - return map->base ? 0 : -ENOMEM; -} - -static void apple_pcie_tunnel_unmap(struct apple_pcie_tunnel_map *map) -{ - if (map->base) - iounmap(map->base); -} - -/* - * Cold-initialize an "apple,pciec-kernel-init" PCIe-C port before the tunnel - * children are populated. The DART is the first child and probes as soon as - * it is created, and its command engine never completes an invalidation - * while the port's APPCLK is gated. A port that is already running (m1n1 - * handoff, or retained from an earlier connection) is left alone. - * - * There is no host bridge yet, so build a transient one around temporary - * mappings and reuse the port enable sequence. - */ -int apple_pcie_tunnel_prepare(struct device *dev, struct device_node *tunnel) -{ - struct apple_pcie_tunnel_map config = {}, debug = {}, fabric = {}, - intr2axi = {}, portmap = {}; - struct device_node *np = NULL, *port_np = NULL; - const struct of_device_id *match; - struct pci_config_window cfg = {}; - struct apple_pcie_port port = {}; - struct apple_pcie pcie = {}; - void *group = NULL; - u32 stat; - int ret; - - for_each_available_child_of_node(tunnel, np) - if (of_property_read_bool(np, "apple,pciec-kernel-init")) - break; - if (!np) - return 0; - - /* A qualified bootloader handoff does not require experimental init. */ - if (!of_property_read_u32(np, "apple,pciec-preinit-status", &stat) && - stat == 1) { - ret = 0; - goto out; - } - - if (!apple_pcie_tunnel_kernel_init_enabled()) { - dev_err(dev, "PCIe-C %pOF: kernel init disabled (pcie_apple.tunnel_kernel_init=0)\n", - np); - ret = -EPERM; - goto out; - } - - match = of_match_node(apple_pcie_of_match, np); - port_np = of_get_next_available_child(np, NULL); - if (!match || !((const struct hw_info *)match->data)->tunneled || - !port_np) { - ret = -ENODEV; - goto out; - } - - ret = apple_pcie_tunnel_map_named(np, "port0", &portmap); - if (ret) - goto out; - - pcie.dev = dev; - pcie.hw = match->data; - port.pcie = &pcie; - port.np = port_np; - port.base = portmap.base; - - apple_pcie_tunnel_cold_init_step(&port, "read port status"); - stat = apple_pcie_port_readl(&port, PORT_STATUS); - if (stat & PORT_STATUS_READY) { - dev_info(dev, "PCIe-C port %pOF already running\n", port_np); - goto out; - } - - ret = apple_pcie_tunnel_map_named(np, "config", &config); - if (!ret) - ret = apple_pcie_tunnel_map_named(np, "debug", &debug); - if (!ret) - ret = apple_pcie_tunnel_map_named(np, "fabric", &fabric); - if (!ret) - ret = apple_pcie_tunnel_map_named(np, "intr2axi", &intr2axi); - if (ret) - goto out; - - group = devres_open_group(dev, NULL, GFP_KERNEL); - if (!group) { - ret = -ENOMEM; - goto out; - } - pcie.rc_tunable = devm_apple_tunable_parse(dev, np, "apple,tunable-rc", - &config.res); - pcie.debug_tunable = devm_apple_tunable_parse(dev, np, - "apple,tunable-debug", - &debug.res); - pcie.fabric_tunable = devm_apple_tunable_parse(dev, np, - "apple,tunable-fabric", - &fabric.res); - port.tunable = devm_apple_tunable_parse(dev, port_np, "apple,tunable", - &portmap.res); - if (IS_ERR(pcie.rc_tunable) || IS_ERR(pcie.debug_tunable) || - IS_ERR(pcie.fabric_tunable) || IS_ERR(port.tunable)) { - ret = dev_err_probe(dev, -ENODEV, - "PCIe-C %pOF tunables unavailable\n", np); - goto out; - } - - cfg.win = config.base; - pcie.cfg = &cfg; - pcie.debug_base = debug.base; - pcie.fabric_base = fabric.base; - pcie.intr2axi_base = intr2axi.base; - - dev_info(dev, "PCIe-C port %pOF cold init (status %#x)\n", - port_np, stat); - ret = apple_pcie_tunnel_cold_init(&port); - -out: - if (group) - devres_release_group(dev, group); - apple_pcie_tunnel_unmap(&intr2axi); - apple_pcie_tunnel_unmap(&fabric); - apple_pcie_tunnel_unmap(&debug); - apple_pcie_tunnel_unmap(&config); - apple_pcie_tunnel_unmap(&portmap); - of_node_put(port_np); - of_node_put(np); - return ret; -} -EXPORT_SYMBOL_GPL(apple_pcie_tunnel_prepare); - static struct platform_driver apple_pcie_driver = { .probe = apple_pcie_probe, .remove = apple_pcie_remove, diff --git a/drivers/phy/apple/atc-j700-dp.h b/drivers/phy/apple/atc-j700-dp.h index 5e1ccb7cadc5b8..a07f1125c2c43f 100644 --- a/drivers/phy/apple/atc-j700-dp.h +++ b/drivers/phy/apple/atc-j700-dp.h @@ -474,9 +474,7 @@ static int j700_dp_configure(struct apple_atcphy *p, const struct phy_configure_ unsigned int pmas, capacity, first, table; int rate = -1, ret; - dev_dbg(p->dev, "J700 DP configure waiting for PHY lock\n"); guard(mutex)(&p->lock); - dev_dbg(p->dev, "J700 DP configure acquired PHY lock\n"); if ((o->set_rate && !o->link_rate) || (o->set_lanes && !o->lanes)) return o->set_voltages ? -EINVAL : j700_dp_stop(p); /* The last completed mode cannot describe a partially changed lane route. */ diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index a45973d05fc8e9..1b6f5df70bf577 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -631,6 +631,9 @@ struct atcphy_mode_configuration { * @pipe_mux_clk: PIPE clock selection field mask * @pipe_mux_data: PIPE data selection field mask * @has_usb2phy_reg: The PHY has the secondary eUSB2 register bank (T8140) + * @has_t8140_dp: Uses the T8140 DisplayPort tunable bundle, AUX, lane routing + * and link training, and retains failed PIPE and mode transitions + * for a retry (the T8140 transition state below is used only here) * @optional_tunables: The bootloader may leave out the common-a tunables (this * generation has none) and the SuperSpeed tunables; USB2 * still works without the latter @@ -680,16 +683,16 @@ struct atcphy_hw { * mode is then allowed without the USB lane tunables * @dp_only: A DisplayPort-only instance without a USB side, such as the PHY * behind the T6030 HDMI port; it maps only its core window - * @dp_tunables: The complete T8140 DisplayPort bundle was supplied - * @aux_power_requested: AUX power-up began and needs matching shutdown - * @mode_dirty: Mode transition needs cleanup before it can be retried + * @t8140_dp_tunables: The complete T8140 DisplayPort bundle was supplied + * @aux_power_requested: T8140 AUX power-up began and needs matching shutdown + * @mode_dirty: T8140 mode transition needs cleanup before it can be retried * @dp_idle_pmas: J700 PMAs with an incomplete routing idle sequence * @dp_owned_pmas: J700 PMAs requiring main-link cleanup * @dp_usb_context_valid: Physical PMAs retaining their pre-DP USB field values * @dp_usb_context_dirty: PMAs whose static fields were programmed by DP * @usb_off_pending: Failed full OFF must finish before a new powered epoch * @usb_off_drained: Masked recovery data was discharged before OFF began - * @pipe_pending: PIPE handoff still requires acknowledged dummy parking + * @pipe_pending: T8140 PIPE handoff still requires acknowledged dummy parking * @dp_usb_context: Saved bits from the public per-PMA DP write masks * @dp_spread_spectrum: J700 main-link spread-spectrum state * @dp_presets: Firmware-derived per-rate drive presets @@ -764,6 +767,8 @@ struct apple_atcphy { bool ss_tunables; bool dp_tunables; bool dp_only; + /* T8140 only: no other PHY ever sets the transition state below. */ + bool t8140_dp_tunables; bool aux_power_requested; bool mode_dirty; u8 dp_idle_pmas; @@ -1145,7 +1150,6 @@ static void atcphy_apply_tunables(struct apple_atcphy *atcphy, enum atcphy_mode } if (atcphy->hw->has_t8140_dp && mode == APPLE_ATCPHY_MODE_USB3_DP) j700_dp_tunables(atcphy); - } static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) @@ -1153,11 +1157,12 @@ static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) int ret; u32 reg; - dev_dbg(atcphy->dev, "PIPE request: req=%#x mode=%u pipe=%u host=%u dirty=%u\n", - requested, atcphy->mode, atcphy->pipe_state, - atcphy->host_active, atcphy->mode_dirty); - if (requested & PIPEHANDLER_LOCK_EN) { - dev_dbg(atcphy->dev, "Pipehandler lock already requested\n"); + if (readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ) & PIPEHANDLER_LOCK_EN) { + /* T8140: a request left behind does not prove the ACK; wait for it. */ + if (!atcphy->hw->has_t8140_dp) { + dev_warn(atcphy->dev, "Pipehandler already locked\n"); + return 0; + } } else { set32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); } @@ -1180,11 +1185,8 @@ static int atcphy_pipehandler_unlock(struct apple_atcphy *atcphy) clear32(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); ret = readl_poll_timeout(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK, reg, !(reg & PIPEHANDLER_LOCK_EN), 10, PIPEHANDLER_LOCK_ACK_TIMEOUT_US); - if (ret) { - dev_dbg(atcphy->dev, "PIPE unlock timeout: ack=%#x mode=%u pipe=%u dirty=%u\n", - reg, atcphy->mode, atcphy->pipe_state, atcphy->mode_dirty); + if (ret) dev_warn(atcphy->dev, "Pipehandler lock release not acked.\n"); - } return ret; } @@ -1239,7 +1241,6 @@ static void atcphy_usb3_bist_step_dbg(struct apple_atcphy *atcphy, int step) readl(atcphy->regs.core + ACIOPHY_BIST_CFG1)); } -static void atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atcphy) static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atcphy) { int ret, error = 0; @@ -1255,22 +1256,17 @@ static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atc core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) { - if (!error) - error = ret; - dev_dbg(atcphy->dev, "BIST USB_EN phase=1 value=%#x mode=%u pipe=%u\n", - reg, atcphy->mode, atcphy->pipe_state); + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN\n"); - } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2, reg, !(reg & ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2), 10, 10000); - if (ret) { - if (!error) - error = ret; - dev_dbg(atcphy->dev, "BIST STATUS2 phase=1 value=%#x swap=%u mode=%u pipe=%u\n", - reg, atcphy->swap_lanes, atcphy->mode, atcphy->pipe_state); + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2\n"); atcphy_usb3_bist_step_dbg(atcphy, 1); @@ -1286,22 +1282,17 @@ static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atc core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) { - if (!error) - error = ret; - dev_dbg(atcphy->dev, "BIST USB_EN phase=2 value=%#x mode=%u pipe=%u\n", - reg, atcphy->mode, atcphy->pipe_state); + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN 2nd toggle\n"); - } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2, reg, !(reg & ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2), 10, 10000); - if (ret) { - if (!error) - error = ret; - dev_dbg(atcphy->dev, "BIST STATUS2 phase=2 value=%#x swap=%u mode=%u pipe=%u\n", - reg, atcphy->swap_lanes, atcphy->mode, atcphy->pipe_state); + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_MAC_USBPHY_STATUS2_LN0_PHY_STATUS2 2nd time\n"); atcphy_usb3_bist_step_dbg(atcphy, 2); @@ -1316,14 +1307,11 @@ static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *atc core_set32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_BIST_CFG1, reg, (reg & ACIOPHY_BIST_CFG1_USB_EN), 10, 10000); - if (ret) { - if (!error) - error = ret; - dev_dbg(atcphy->dev, "BIST USB_EN phase=3 value=%#x mode=%u pipe=%u\n", - reg, atcphy->mode, atcphy->pipe_state); + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_BIST_CFG1_USB_EN 3rd toggle\n"); - } core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_USB_EN); atcphy_usb3_bist_step_dbg(atcphy, 3); core_clear32(atcphy, ACIOPHY_BIST_CFG1, ACIOPHY_BIST_CFG1_CLOCK_EN); @@ -1340,12 +1328,11 @@ static int atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *atc core_set32(atcphy, ACIOPHY_TOP_BIST_OV_CFG, ACIOPHY_TOP_BIST_OV_CFG_LN0_RESET_N_OV); ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); - if (ret) { - if (!error) - error = ret; + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "Timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); - } core_set32(atcphy, ACIOPHY_TOP_BIST_READ_CTRL, ACIOPHY_TOP_BIST_READ_CTRL_LN0_PHY_STATUS_RE); @@ -1366,21 +1353,19 @@ static int atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *atc ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, (reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK0), 10, 10000); - if (ret) { - if (!error) - error = ret; + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK0\n"); - } ret = readl_poll_timeout(atcphy->regs.core + ACIOPHY_TOP_PHY_STAT, reg, !(reg & ACIOPHY_TOP_PHY_STAT_LN0_UNK23), 10, 10000); - if (ret) { - if (!error) - error = ret; + if (ret && !error) + error = ret; + if (ret) dev_warn(atcphy->dev, "timed out waiting for ACIOPHY_TOP_PHY_STAT_LN0_UNK23\n"); - } atcphy_clear_nonselected_phy_reset(atcphy); @@ -1425,7 +1410,13 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h ret = atcphy_configure_pipehandler_usb3_host_t8103(atcphy); else ret = atcphy_configure_pipehandler_usb3_host_t8122(atcphy); - if (ret) { + if (atcphy->hw->dp_t8122) + dev_dbg(atcphy->dev, "USB3 host BIST done: status2=%08x ov1=%08x ov2=%08x\n", + readl(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2), + readl(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_CFG_OV1), + readl(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_CFG_OV2)); + /* T8140 retries a failed BIST; other PHYs carry on as before. */ + if (ret && atcphy->hw->has_t8140_dp) { int cleanup; clear32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, @@ -1437,12 +1428,6 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h atcphy->mode_dirty = true; return ret; } - atcphy_configure_pipehandler_usb3_host_t8122(atcphy); - if (atcphy->hw->dp_t8122) - dev_dbg(atcphy->dev, "USB3 host BIST done: status2=%08x ov1=%08x ov2=%08x\n", - readl(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_STATUS2), - readl(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_CFG_OV1), - readl(atcphy->regs.core + ACIOPHY_BIST_MAC_USBPHY_CFG_OV2)); } /* Configure PIPE mux to USB3 PHY */ @@ -1460,8 +1445,10 @@ static int atcphy_configure_pipehandler_usb3(struct apple_atcphy *atcphy, bool h ret = atcphy_pipehandler_unlock(atcphy); if (ret) { dev_warn(atcphy->dev, "Failed to unlock pipehandler"); - atcphy->mode_dirty = true; - return ret; + if (atcphy->hw->has_t8140_dp) { + atcphy->mode_dirty = true; + return ret; + } } } @@ -1534,6 +1521,7 @@ static bool atcphy_needs_dummy_phy(struct apple_atcphy *atcphy) static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy, bool lock) { + /* pipe_pending is only ever set on T8140, see atcphy_configure_pipehandler(). */ bool live_usb3 = atcphy->pipe_pending || (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->host_active && atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3); @@ -1553,38 +1541,37 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy, bool set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXVALID); set32(atcphy->regs.pipehandler + PIPEHANDLER_OVERRIDE, PIPEHANDLER_OVERRIDE_RXDETECT); - ret = atcphy_pipehandler_lock(atcphy); - if (ret) { - /* A live controller must acknowledge before its backend changes. */ - if (live_usb3) { - if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) - atcphy->mode_dirty = true; - return ret; - } - dev_warn(atcphy->dev, "Failed to lock pipehandler"); if (lock) { ret = atcphy_pipehandler_lock(atcphy); - if (ret) + if (ret) { + /* A live controller must acknowledge before its backend changes. */ + if (live_usb3) { + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; + return ret; + } dev_warn(atcphy->dev, "Failed to lock pipehandler"); + } } /* Switch to dummy PHY */ atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); - ret = atcphy_pipehandler_unlock(atcphy); - if (ret) { - /* The mux changed, but the handoff has not completed. Retain ownership. */ - if (live_usb3) { - if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) - atcphy->mode_dirty = true; - return ret; - } - dev_warn(atcphy->dev, "Failed to unlock pipehandler"); if (lock) { ret = atcphy_pipehandler_unlock(atcphy); - if (ret) + if (ret) { + /* + * The mux changed, but the handoff has not completed. + * Retain ownership. + */ + if (live_usb3) { + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY) + atcphy->mode_dirty = true; + return ret; + } dev_warn(atcphy->dev, "Failed to unlock pipehandler"); + } } mask32(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE, @@ -1611,6 +1598,8 @@ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) switch (state) { case ATCPHY_PIPEHANDLER_STATE_USB3: + if (atcphy->hw->has_t8140_dp) + atcphy->pipe_pending = true; ret = atcphy_configure_pipehandler_usb3(atcphy, host); break; case ATCPHY_PIPEHANDLER_STATE_USB4: @@ -1632,8 +1621,8 @@ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) readl(atcphy->regs.pipehandler + PIPEHANDLER_LOCK_ACK), readl(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN), readl(atcphy->regs.pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE)); - atcphy->pipe_state = state; - if (!ret) { + /* T8140 keeps a failed attempt retryable; other PHYs configure once per mode. */ + if (!ret || !atcphy->hw->has_t8140_dp) { atcphy->pipe_state = state; atcphy->pipe_pending = false; } @@ -1649,27 +1638,24 @@ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) * controller no longer runs, so the request times out. On PHYs with * park_pipe_unlocked the mux is then switched without it. */ -static void atcphy_park_pipehandler(struct apple_atcphy *atcphy, bool stopped) +static int atcphy_park_pipehandler(struct apple_atcphy *atcphy, bool stopped) { bool lock = !(stopped && atcphy->hw->park_pipe_unlocked); int ret; lockdep_assert_held(&atcphy->lock); - dev_dbg(atcphy->dev, "PIPE park: mode=%u pipe=%u host=%u dirty=%u owned=%u idle=%u\n", - atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty, - atcphy->dp_owned_pmas, atcphy->dp_idle_pmas); if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !atcphy->pipe_pending) return 0; - ret = atcphy_configure_pipehandler_dummy(atcphy); - if (ret) { if (!lock) dev_dbg(atcphy->dev, "Parking the PIPE of a stopped controller\n"); ret = atcphy_configure_pipehandler_dummy(atcphy, lock); - if (ret) + if (ret) { dev_warn(atcphy->dev, "Failed to switch PIPE to dummy: %d\n", ret); - return ret; + /* T8140 retains a failed handoff; other PHYs park regardless. */ + if (atcphy->hw->has_t8140_dp) + return ret; } atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; atcphy->pipe_pending = false; @@ -1683,9 +1669,8 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, PIPEHANDLER_MUX_CTRL_CLK_DUMMY); atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; - if (atcphy_needs_dummy_phy(atcphy)) atcphy->pipe_pending = false; - if (!atcphy->hw->has_usb4) + if (atcphy_needs_dummy_phy(atcphy)) atcphy_enable_dummy_phy(atcphy); } @@ -1766,8 +1751,6 @@ static int atcphy_configure_lanes(struct apple_atcphy *atcphy, enum atcphy_mode else core_clear32(atcphy, atcphy->hw->aciophy_crossbar, ACIOPHY_CROSSBAR_DP_BOTH_PMA); - - /* unclear if we need the remainder for t8122 */ /* * The PMA FSM override below is for the T8103 generation. T8122 DP lanes * are started by the link rate sequence, see atcphy_dp_set_rate_t8122(). @@ -1858,7 +1841,6 @@ static int atcphy_enable_dp_aux(struct apple_atcphy *atcphy) if (atcphy->hw->has_t8140_dp) return j700_aux_enable(atcphy); - /* FIXME */ /* T8122 PHYs with DP support enable AUX in atcphy_dp_aux_on_t8122() */ if (atcphy->hw->gen == ATCPHY_GENERATION_T8122) return 0; @@ -1922,7 +1904,6 @@ static int atcphy_disable_dp_aux(struct apple_atcphy *atcphy) return ret ? ret : j700_aux_disable(atcphy); } - /* FIXME */ /* T8122 PHYs with DP support stop AUX in atcphy_dp_stop_t8122() */ if (atcphy->hw->gen != ATCPHY_GENERATION_T8122) { set32(atcphy->regs.lpdptx + LPDPTX_AUX_CONTROL, LPDPTX_AUX_PWN_DOWN); @@ -2183,9 +2164,6 @@ static int atcphy_dp_configure(struct apple_atcphy *atcphy, enum atcphy_dp_link_ u32 reg; guard(mutex)(&atcphy->lock); - /* A failed transition may have asserted the PHY resets already. */ - if (atcphy->mode_dirty || atcphy->pipe_pending) - return -EBUSY; /* the AUSPLL is in use as the Thunderbolt DP tunnel pixel clock */ if (atcphy->tunnel_clock_on || atcphy->tunnel_saved) return -EBUSY; @@ -2655,8 +2633,7 @@ static void atc_tunnel_wake_t8103(struct apple_atcphy *atcphy) udelay(2); } -/* The pixel clock selector for a DP link rate code. */ -static int atc_tunnel_rate_selector(u8 rate) +static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) { u32 selector, value; int ret; @@ -2664,28 +2641,20 @@ static int atc_tunnel_rate_selector(u8 rate) lockdep_assert_held(&atcphy->lock); switch (rate) { case 0x06: /* RBR */ - return 4; + selector = 4; + break; case 0x0a: /* HBR */ - return 3; + selector = 3; + break; case 0x14: /* HBR2 */ - return 1; + selector = 1; + break; case 0x1e: /* HBR3 */ selector = 0; break; default: return -EINVAL; } -} - -static int atc_tunnel_start_t8103(struct apple_atcphy *atcphy, u8 rate) -{ - int selector, ret; - u32 value; - - lockdep_assert_held(&atcphy->lock); - selector = atc_tunnel_rate_selector(rate); - if (selector < 0) - return selector; if (atcphy->tunnel_clock_on) { if (atcphy->tunnel_rate == rate) return 0; @@ -2914,24 +2883,25 @@ static int atc_tunnel_start_t602x(struct apple_atcphy *atcphy) core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_CLAMP_OV); udelay(2); - /* same for the TX lanes */ - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_SMALL_OV); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x30); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc0); udelay(2); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_BIG_OV); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x03); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0x0c); udelay(2); - core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP); - core_set32(atcphy, ACIOPHY_SLEEP_CTRL, ACIOPHY_SLEEP_CTRL_TX_CLAMP_OV); + core_clear32(atcphy, ACIOPHY_SLEEP_CTRL, 0x300); + core_set32(atcphy, ACIOPHY_SLEEP_CTRL, 0xc00); udelay(2); - /* and for the RX lanes */ - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG); + /* Additional override bit alongside RX_BIG_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc00); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_BIG_OV); udelay(2); - core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL); + /* Additional override bit alongside RX_SMALL_OV; exact meaning not independently confirmed. */ + core_set32(atcphy, ACIOPHY_CFG0, 0xc0); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_SMALL_OV); udelay(2); - core_clear32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP); + /* Additional override bit alongside RX_CLAMP_OV; exact meaning not independently confirmed. */ + core_clear32(atcphy, ACIOPHY_CFG0, 0xc000); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_RX_CLAMP_OV); udelay(2); core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, BIT(2)); @@ -3291,6 +3261,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) u32 reg; lockdep_assert_held(&atcphy->lock); + /* The T8140 transition state is never set on other PHYs. */ if (atcphy->pipe_pending) return -EBUSY; if (atcphy->usb_off_pending && mode != APPLE_ATCPHY_MODE_OFF) @@ -3311,7 +3282,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) return ret; } if (atcphy->hw->has_t8140_dp && atcphy_modes[mode].enable_dp_aux && - !atcphy->dp_tunables) + !atcphy->t8140_dp_tunables) return -EOPNOTSUPP; if (atcphy->hw->has_t8140_dp && (!atcphy_modes[mode].enable_dp_aux || atcphy->mode_dirty) && @@ -3321,7 +3292,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) if (ret) return ret; } - /* Keep failed transitions retryable even when the requested mode is unchanged. */ + /* T8140 keeps failed transitions retryable even when the mode is unchanged. */ if (atcphy->hw->has_t8140_dp) atcphy->mode_dirty = true; /* the mode change below may power the PHY down: stop the tunnel clock first */ @@ -3348,7 +3319,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) } } ret = atcphy_power_off(atcphy); - if (ret) + if (ret && atcphy->hw->has_t8140_dp) return ret; /* Recovery data was discharged before OFF, not discarded while dirty. */ atcphy->dp_usb_context_valid = 0; @@ -3357,7 +3328,7 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) atcphy->usb_off_drained = false; atcphy->mode = mode; atcphy->mode_dirty = false; - return 0; + return ret; } /* @@ -3585,24 +3556,27 @@ static int atcphy_usb2_init(struct phy *phy) { struct apple_atcphy *atcphy = phy_get_drvdata(phy); + if (!atcphy->fixed_usb2 && !atcphy->hw->has_t8140_dp) + return 0; + guard(mutex)(&atcphy->lock); + atcphy_restore_after_pd_off(atcphy); if (atcphy->usb_off_pending) return -EBUSY; if (atcphy->pipe_pending && (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty)) { - int ret = atcphy_park_pipehandler(atcphy); + int ret = atcphy_park_pipehandler(atcphy, false); if (ret) return ret; } - /* A delayed shutdown may have completed after the last Type-C event. */ + /* T8140: a delayed shutdown may have completed after the last Type-C event. */ if (!atcphy->fixed_usb2) { if (atcphy->mode == APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) return atcphy_configure(atcphy, atcphy->typec_mode); return 0; } - atcphy_restore_after_pd_off(atcphy); /* * dwc3 initialises its PHYs after releasing its reset, and asserting that @@ -3635,15 +3609,10 @@ static int atcphy_usb3_power_off(struct phy *phy) guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); - ret = atcphy_park_pipehandler(atcphy); + /* Only T8140 reports a failed park, and then keeps the controller's PIPE. */ + ret = atcphy_park_pipehandler(atcphy, true); if (ret) return ret; - if (atcphy->mode != APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) { - ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); - if (ret) - return ret; - } - atcphy_park_pipehandler(atcphy, true); atcphy->host_active = false; /* @@ -3651,13 +3620,17 @@ static int atcphy_usb3_power_off(struct phy *phy) * e.g. when a monitor's built-in hub re-enumerates after a hot-plug. * Powering the PHY off then cuts the DP lanes under DCP, which reports a * DPTX FIFO error and drops the display. The Type-C mux turns the PHY off - * when the alt mode actually ends, so leave a DP mode alone here. + * when the alt mode actually ends, so leave a DP mode alone here, unless + * a failed T8140 transition left it to be cleaned up. */ - if (atcphy_modes[atcphy->mode].enable_dp_aux) + if (atcphy_modes[atcphy->mode].enable_dp_aux && !atcphy->mode_dirty) return 0; - if (atcphy->mode != APPLE_ATCPHY_MODE_OFF) - atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); + if (atcphy->mode != APPLE_ATCPHY_MODE_OFF || atcphy->mode_dirty) { + ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_OFF); + if (ret && atcphy->hw->has_t8140_dp) + return ret; + } return 0; } @@ -3688,18 +3661,19 @@ static int atcphy_usb3_set_mode(struct phy *phy, enum phy_mode mode, int submode * driver and then another one from the generic xhci code) but must only configure the * PIPE handler once. Nothing needs to be done either when the PIPE is already routed to * the backend the current mode uses, which includes the dummy backend of the USB2 modes. + * + * T8140: a failed lane/domain transition must be restored, and a failed + * handoff parked, before the PIPE is routed again. */ - /* A failed lane/domain transition must be restored before routing PIPE. */ if (atcphy->mode_dirty) return -EBUSY; if (atcphy->pipe_pending) { - int ret = atcphy_park_pipehandler(atcphy); + int ret = atcphy_park_pipehandler(atcphy, false); if (ret) return ret; } - if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state && - !atcphy->pipe_pending) + if (atcphy->pipe_state == atcphy_modes[atcphy->mode].pipehandler_state) return 0; return atcphy_configure_pipehandler(atcphy, mode == PHY_MODE_USB_HOST); @@ -3770,10 +3744,10 @@ static int atcphy_dpphy_configure(struct phy *phy, union phy_configure_opts *opt struct apple_atcphy *atcphy = phy_get_drvdata(phy); enum atcphy_dp_link_rate link_rate; - if (atcphy->hw->dp_t8122) - return atcphy_dpphy_configure_t8122(atcphy, opts); if (atcphy->hw->has_t8140_dp) return j700_dp_configure(atcphy, opts); + if (atcphy->hw->dp_t8122) + return atcphy_dpphy_configure_t8122(atcphy, opts); if (opts->set_voltages) return -EINVAL; @@ -3840,11 +3814,6 @@ int apple_atc_dp_tunnel_rate_source(struct phy *phy, unsigned int source, if (!apple_atc_tunnel_is_t6030(atcphy) || apple_dp_tunnel_t6030_pclk(source, dpin) < 0) return -EOPNOTSUPP; - if (of_machine_is_compatible("apple,j416s") && - (!of_device_is_compatible(atcphy->np, "apple,t6020-atcphy") || - !apple_atc_is_typec_core(atcphy->res.core->start) || - resource_size(atcphy->res.core) < 0x7048)) - return -EINVAL; guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); /* A stale source must not retune or stop another source's clock. */ @@ -4004,10 +3973,12 @@ static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); - /* Do not reset the controller or USB2 PHY if PIPE parking fails. */ - ret = atcphy_park_pipehandler(atcphy); - if (ret) - return ret; + /* T8140: do not reset the controller or USB2 PHY if PIPE parking fails. */ + if (atcphy->hw->has_t8140_dp) { + ret = atcphy_park_pipehandler(atcphy, true); + if (ret) + return ret; + } _atcphy_dwc3_reset_assert(atcphy); atcphy_park_pipehandler(atcphy, true); @@ -4057,26 +4028,23 @@ static int atcphy_probe_rcdev(struct apple_atcphy *atcphy) static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation orientation) { struct apple_atcphy *atcphy = typec_switch_get_drvdata(sw); - bool swap_lanes; guard(mutex)(&atcphy->lock); switch (orientation) { case TYPEC_ORIENTATION_NONE: - return 0; + break; case TYPEC_ORIENTATION_NORMAL: + /* The mode alone cannot describe a T8140 route after its orientation changes. */ + if (atcphy->hw->has_t8140_dp && atcphy->swap_lanes) + atcphy->mode_dirty = true; atcphy->swap_lanes = false; break; case TYPEC_ORIENTATION_REVERSE: + if (atcphy->hw->has_t8140_dp && !atcphy->swap_lanes) + atcphy->mode_dirty = true; atcphy->swap_lanes = true; break; - default: - return -EINVAL; - } - /* The mode alone cannot describe a route after its orientation changes. */ - if (atcphy->swap_lanes != swap_lanes) { - atcphy->swap_lanes = swap_lanes; - atcphy->mode_dirty = true; } dev_dbg(atcphy->dev, "orientation %d, lanes swapped %d\n", orientation, atcphy->swap_lanes); @@ -4379,15 +4347,13 @@ static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_sta target_mode != APPLE_ATCPHY_MODE_DP) return -EOPNOTSUPP; - if (atcphy->mode == target_mode) - dev_dbg(atcphy->dev, "mux target=%u\n", target_mode); if (atcphy->mode == target_mode && !atcphy->mode_dirty) { - /* Retry a failed PIPE setup without disturbing the configured lanes. */ - if (atcphy->fixed_usb2 && atcphy->host_active && + /* T8140: retry a failed PIPE setup without disturbing the configured lanes. */ + if (atcphy->hw->has_t8140_dp && atcphy->fixed_usb2 && atcphy->host_active && (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || atcphy->pipe_pending)) { if (atcphy->pipe_pending) { - ret = atcphy_park_pipehandler(atcphy); + ret = atcphy_park_pipehandler(atcphy, false); if (ret) return ret; } @@ -4406,10 +4372,8 @@ static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_sta case APPLE_ATCPHY_MODE_USB3_DP: case APPLE_ATCPHY_MODE_DP: if (atcphy->hw->has_t8140_dp && atcphy_modes[target_mode].enable_dp_aux && - !atcphy->dp_tunables) + !atcphy->t8140_dp_tunables) return -EOPNOTSUPP; - /* Without the lane tunables the SuperSpeed lanes are not calibrated */ - if (!atcphy->ss_tunables) /* * Without the lane tunables the SuperSpeed lanes are not calibrated. * T8122 four-lane DP programs its lanes itself and needs only the @@ -4436,24 +4400,29 @@ static int atcphy_mux_set_locked(struct typec_mux_dev *mux, struct typec_mux_sta * mode without a USB3 backend, and route it to the new backend * afterwards while dwc3 is up in host mode. */ - ret = atcphy_park_pipehandler(atcphy); - if (ret) - return ret; + /* + * Even another USB3 mode can move a T8140 backend to the other PMA + * when orientation changes, so T8140 parks before every rebuild. + */ + if (pipe_state != ATCPHY_PIPEHANDLER_STATE_USB3 || atcphy->hw->has_t8140_dp) { + ret = atcphy_park_pipehandler(atcphy, false); + if (ret) + return ret; + } ret = atcphy_configure(atcphy, target_mode); if (ret) return ret; - if (atcphy->host_active && atcphy->pipe_state != pipe_state) ret = atcphy_configure_pipehandler(atcphy, true); return ret; } - /* A failed USB3 handoff must finish parking before ordinary-port lane work. */ + /* A failed T8140 USB3 handoff must finish parking before ordinary-port lane work. */ if (atcphy->pipe_pending) { - ret = atcphy_park_pipehandler(atcphy); + ret = atcphy_park_pipehandler(atcphy, false); if (ret) return ret; } @@ -4549,7 +4518,7 @@ static int atcphy_load_t8140_dp(struct apple_atcphy *atcphy) atcphy->dp_presets_valid[i] = true; } - atcphy->dp_tunables = true; + atcphy->t8140_dp_tunables = true; return 0; } diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index de7087cd8bac6f..a099e5a8101509 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -125,6 +125,14 @@ static const struct apple_cio_hw apple_cio_t6030_hw = { .dp = &apple_dpin_m3, }; +/* + * T8142 (M5) Type 7 ACIO: ready is checked by acio->type7. Thunderbolt DP + * tunnels are not driven on this SoC yet. + */ +static const struct apple_cio_hw apple_cio_t8142_hw = { + .dp = &apple_dpin_disabled, +}; + #define APPLE_CIO_NHI_HOP_COUNT 0x0 #define APPLE_CIO_NHI_HOP_COUNT_MASK GENMASK(9, 0) @@ -2551,6 +2559,12 @@ static int apple_cio_start(struct apple_cio *acio) FIELD_GET(APPLE_CIO_M3_STAT_STATE, state) == APPLE_CIO_M3_STAT_STATE_READY, 100, 500000); + else if (acio->type7) + /* Type 7 reports ready as state 1; other nonzero states are not ready. */ + ret = readl_poll_timeout(acio->rc_base + APPLE_CIO_M3_STAT, state, + FIELD_GET(APPLE_CIO_M3_STAT_STATE, state) == + APPLE_CIO_M3_STAT_STATE_READY, + 100, 500000); else ret = readl_poll_timeout(acio->rc_base + APPLE_CIO_M3_STAT, state, state & APPLE_CIO_M3_STAT_STATE, 100, 500000); @@ -3202,9 +3216,6 @@ static void apple_cio_remove(struct platform_device *pdev) /* A set may hold the switch read lock while waiting for acio->lock. */ typec_thunderbolt_switch_unregister(acio->tbt_switch); guard(mutex)(&acio->lock); - /* The worker can notify this switch; stop current and future enqueues. */ - disable_delayed_work_sync(&acio->pcie_tunnel_work); - typec_thunderbolt_switch_unregister(acio->tbt_switch); if (acio->current_cable_info) apple_cio_stop(acio); } @@ -3212,6 +3223,9 @@ static void apple_cio_remove(struct platform_device *pdev) static const struct of_device_id apple_acio_match[] = { { .compatible = "apple,t8142-usb4-acio", + .data = &apple_cio_t8142_hw, + }, + { .compatible = "apple,t6030-usb4-acio", .data = &apple_cio_t6030_hw, }, diff --git a/drivers/usb/dwc3/dwc3-apple.c b/drivers/usb/dwc3/dwc3-apple.c index 248ccbf90027c1..c8fc03421051f0 100644 --- a/drivers/usb/dwc3/dwc3-apple.c +++ b/drivers/usb/dwc3/dwc3-apple.c @@ -254,52 +254,6 @@ static int dwc3_apple_core_init(struct dwc3_apple *appledwc) return ret; } -/* Keep failed releases owned by the glue, without repeating completed teardown. */ -static int dwc3_apple_core_stop(struct dwc3_apple *appledwc) -{ - struct dwc3 *dwc = &appledwc->dwc; - int i, ret; - - lockdep_assert_held(&appledwc->lock); - if (appledwc->core_active) { - if (!appledwc->core_exit_pending) { - dwc3_enable_susphy(dwc, true); - dwc3_event_buffers_cleanup(dwc); - appledwc->core_exit_pending = true; - } - for (i = 0; i < dwc->num_usb3_ports; i++) { - if (!(appledwc->usb3_power_off_pending & BIT(i))) - continue; - ret = phy_power_off(dwc->usb3_generic_phy[i]); - if (ret) - return ret; - appledwc->usb3_power_off_pending &= ~BIT(i); - } - for (i = 0; i < dwc->num_usb2_ports; i++) { - if (!(appledwc->usb2_power_off_pending & BIT(i))) - continue; - ret = phy_power_off(dwc->usb2_generic_phy[i]); - if (ret) - return ret; - appledwc->usb2_power_off_pending &= ~BIT(i); - } - usb_phy_set_suspend(dwc->usb3_phy, 1); - usb_phy_set_suspend(dwc->usb2_phy, 1); - dwc3_core_exit_after_phy_power_off(dwc); - appledwc->core_active = false; - appledwc->core_exit_pending = false; - } - if (appledwc->reset_pending) { - ret = reset_control_assert(appledwc->reset); - if (ret) - return ret; - appledwc->reset_pending = false; - } - dev_pm_syscore_device(appledwc->dev, false); - - return 0; -} - /* * Bring the core up for the given role: configure the USB2 PHY, release the reset and probe * (on the first call) or initialise the core. @@ -311,11 +265,6 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st lockdep_assert_held(&appledwc->lock); - /* A failed exit must finish before any new PHY reference is acquired. */ - ret = dwc3_apple_core_stop(appledwc); - if (ret) - return ret; - /* * The USB2 PHY on this platform must be configured for device mode before dwc3 tries to * access it. Otherwise, the new configuration will sometimes only take affect after the @@ -348,9 +297,6 @@ static int dwc3_apple_core_start(struct dwc3_apple *appledwc, enum dwc3_apple_st return ret; } - appledwc->reset_pending = true; - /* Keep register access powered until both PHY shutdown and reset finish. */ - dev_pm_syscore_device(appledwc->dev, true); ret = dwc3_apple_core_init(appledwc); if (ret) { ret_reset = reset_control_assert(appledwc->reset); @@ -650,9 +596,10 @@ static int dwc3_apple_probe(struct platform_device *pdev) scoped_guard(mutex, &appledwc->lock) ret = dwc3_apple_init(appledwc, DWC3_APPLE_HOST); if (ret) { - /* A failed cleanup remains visible even when probe cannot be retried. */ - if (appledwc->core_active || appledwc->reset_pending) - dev_err(dev, "Host startup left pending PHY/reset cleanup\n"); + /* + * dwc3_apple_init() has already undone the core initialisation: + * release the core's resources without a second shutdown. + */ if (appledwc->state != DWC3_APPLE_PROBE_PENDING) dwc3_core_remove_no_shutdown(&appledwc->dwc); usb_role_switch_unregister(appledwc->role_sw); @@ -677,8 +624,16 @@ static void dwc3_apple_remove(struct platform_device *pdev) * dwc3_apple_exit can be called unconditionally because it checks the state itself. */ dwc3_apple_exit(appledwc); - if (appledwc->state != DWC3_APPLE_PROBE_PENDING) + if (appledwc->state == DWC3_APPLE_PROBE_PENDING) + return; + /* + * dwc3_apple_exit() has stopped the host and the core of a fixed-hub + * controller: release its resources without running either exit twice. + */ + if (appledwc->fixed_hub) dwc3_core_remove_no_shutdown(&appledwc->dwc); + else + dwc3_core_remove(&appledwc->dwc); } static const struct of_device_id dwc3_apple_of_match[] = { From 54a6ea25d48d4fd3e6899b62e4e78610a8d45951 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Wed, 14 Oct 2026 20:02:50 +0200 Subject: [PATCH 558/631] drm: apple: reset the Type-C relink budget on every DPMS wake A USB-C monitor with a built-in hub drops the DP link right after the first modeset of every DPMS wake, which dcp_typec_relink() recovers by replaying an unplug and replug. Its retry counter is only reset when the Type-C generation changes, i.e. on a real cable replug, so all wakes of one connection share DPTX_RELINK_RETRIES tries: after three wakes the fourth drop is handled as an unplug and the display stays dark until the cable is replugged. The limit exists to stop relink loops within one link bring-up. Reset it in dcp_poweron() when the CRTC comes back from DPMS off. No relink can run while the CRTC is off, dcp_typec_relink_start() refuses then. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/dcp.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 6ef18230c86967..690b488cb1fbeb 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -1144,7 +1144,8 @@ static void dcp_typec_reconnect_work(struct work_struct *work) * the sink, which retrains the link once the hub has settled, as a cable * replug does. * - * The retries cover one connection: the replay itself does not reset them. + * The retries cover one connection or one DPMS wake: the replay itself + * does not reset them, dcp_poweron() does. */ static bool dcp_typec_relink_start(struct apple_dcp *dcp, const char *why) { @@ -1756,6 +1757,12 @@ void dcp_poweron(struct platform_device *pdev) READ_ONCE(dcp->typec_cable_connected); WRITE_ONCE(dcp->typec_crtc_off, false); WRITE_ONCE(dcp->typec_follow_start, false); + /* + * A DPMS wake retrains the link like a replug: give it a fresh + * relink budget. No relink runs while the CRTC was off. + */ + if (wait_for_typec_hpd) + dcp->relink_tries = 0; /* * A Type-C CRTC disable releases its DPTX session. Re-establish it From 55f589a59835572cfabfa9f79afeb5b62f34c4c7 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:02 -0400 Subject: [PATCH 559/631] pmdomain: apple: serve T8140 from the shared PMP report and PMGR drivers Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. The T8140 PMP report support from aurora-wip moves into the shared pmp-report driver: the report offsets, the fast-die effort for cpufreq, and the entry lifetime. It replaces integration's separate T8140 copy. The T8140 report entry keeps its t6000 fallback compatible. On T8140 and T8142, PMGR power-state transitions that time out are undone outside the system-sleep window again, as in aurora-wip: J700 hits this on every boot with venc0_sys. On T8140 the PMP endpoint leaves integration's UPDATE_IOREG confirmation and power-reply interception out, as aurora-wip did. Assisted-by: LLM Signed-off-by: Diljit Singh --- arch/arm64/boot/dts/apple/t8140.dtsi | 212 +-------- drivers/cpufreq/apple-soc-cpufreq.c | 357 +-------------- drivers/pmdomain/apple/Kconfig | 42 +- drivers/pmdomain/apple/Makefile | 1 - drivers/pmdomain/apple/pmgr-pwrstate.c | 43 +- drivers/pmdomain/apple/pmp-report-neo.c | 529 ----------------------- drivers/pmdomain/apple/pmp-report.c | 268 ++++++++---- drivers/soc/apple/pmp.rs | 16 +- include/linux/soc/apple/pmp-report-neo.h | 26 -- 9 files changed, 265 insertions(+), 1229 deletions(-) delete mode 100644 drivers/pmdomain/apple/pmp-report-neo.c delete mode 100644 include/linux/soc/apple/pmp-report-neo.h diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 00640dc340f360..89a5a609817e42 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -372,80 +372,6 @@ }; }; - gpu_opp: opp-table-gpu { - compatible = "operating-points-v2"; - - /* - * NOTE: The operating points are device-specific and must be - * filled in by the bootloader. Entries keep the order of the - * GPU performance states, which is not sorted by frequency. - */ - opp00 { - opp-hz = /bits/ 64 <0>; - opp-microvolt = <125000>; - }; - opp01 { - opp-hz = /bits/ 64 <338000000>; - opp-microvolt = <605000>; - }; - opp02 { - opp-hz = /bits/ 64 <492000000>; - opp-microvolt = <630000>; - }; - opp03 { - opp-hz = /bits/ 64 <618000000>; - opp-microvolt = <650000>; - }; - opp04 { - opp-hz = /bits/ 64 <796000000>; - opp-microvolt = <695000>; - }; - opp05 { - opp-hz = /bits/ 64 <928000000>; - opp-microvolt = <745000>; - }; - opp06 { - opp-hz = /bits/ 64 <952000000>; - opp-microvolt = <800000>; - }; - opp07 { - opp-hz = /bits/ 64 <1056000000>; - opp-microvolt = <800000>; - }; - opp08 { - opp-hz = /bits/ 64 <1053000000>; - opp-microvolt = <845000>; - }; - opp09 { - opp-hz = /bits/ 64 <1170000000>; - opp-microvolt = <845000>; - }; - opp10 { - opp-hz = /bits/ 64 <1152000000>; - opp-microvolt = <885000>; - }; - opp11 { - opp-hz = /bits/ 64 <1278000000>; - opp-microvolt = <885000>; - }; - opp12 { - opp-hz = /bits/ 64 <1204000000>; - opp-microvolt = <905000>; - }; - opp13 { - opp-hz = /bits/ 64 <1338000000>; - opp-microvolt = <905000>; - }; - opp14 { - opp-hz = /bits/ 64 <1326000000>; - opp-microvolt = <975000>; - }; - opp15 { - opp-hz = /bits/ 64 <1470000000>; - opp-microvolt = <975000>; - }; - }; - timer { compatible = "arm,armv8-timer"; interrupt-parent = <&aic>; @@ -571,11 +497,6 @@ label = "pmp-gfx"; #power-domain-cells = <0>; power-domains = <&ps_gfx>; - compatible = "apple,t8140-pmp-v2-report-entry"; - reg = <0x9>; - label = "pmp-gfx"; - #power-domain-cells = <0>; - power-domains = <&ps_gfx>; apple,always-on; }; }; @@ -753,6 +674,7 @@ /* child nodes are added in t8140-pmgr.dtsi */ }; + aic: interrupt-controller@301000000 { compatible = "apple,t8140-aic3", "apple,t8122-aic3"; #interrupt-cells = <3>; @@ -1241,44 +1163,6 @@ status = "disabled"; }; - gfx1_mbox: mailbox@482e00000 { - compatible = "apple,t8140-ascwrap-v6"; - reg = <0x4 0x82e00000 0x0 0x88000>, - <0x4 0x82850000 0x0 0x8>; - reg-names = "wrapper", "iop-vbar"; - interrupt-parent = <&aic>; - interrupts = , - ; - interrupt-names = "send-empty", "recv-not-empty"; - apple,firmware-role = "GFX1"; - reg = <0x4 0x82600000 0x0 0x4000>, - <0x4 0x82e00000 0x0 0x4000>, - <0x4 0x80000000 0x0 0x4000000>; - reg-names = "asc", "sgx"; - mboxes = <&gfx_mbox>, <&gfx1_mbox>; - power-domains = <&ps_gfx>; - memory-region = <&uat_ttbs>, <&uat_pagetables>, - <&uat_handoff>; - memory-region-names = "ttbs", "pagetables", "handoff"; - operating-points-v2 = <&gpu_opp>; - status = "disabled"; - }; - - gfx_mbox: mailbox@482600000 { - compatible = "apple,t8140-ascwrap-v6"; - reg = <0x4 0x82600000 0x0 0x88000>, - <0x4 0x82050000 0x0 0x8>; - reg-names = "wrapper", "iop-vbar"; - interrupt-parent = <&aic>; - interrupts = , - ; - interrupt-names = "send-empty", "recv-not-empty"; - apple,firmware-role = "GFX"; - power-domains = <&ps_gfx>; - #mbox-cells = <0>; - status = "disabled"; - }; - gfx1_mbox: mailbox@482e00000 { compatible = "apple,t8140-ascwrap-v6"; reg = <0x4 0x82e00000 0x0 0x88000>, @@ -1293,6 +1177,8 @@ #mbox-cells = <0>; status = "disabled"; }; + }; +}; &soc { disp0_dart: iommu@412300000 { @@ -1341,96 +1227,6 @@ status = "disabled"; }; - /* - * Perf-state floors of voter agents 0-3, and the rail state and - * interface enable words, in the PMC window at 0x302800000. - */ - pmc: power-management@302800000 { - compatible = "apple,t8140-pmc"; - reg = <0x3 0x02800000 0x0 0x10000>, - <0x3 0x028aa000 0x0 0x3000>; - reg-names = "voters", "rails"; - power-domains = <&ps_pmc>; - #address-cells = <1>; - #size-cells = <0>; - /* PMP, CLPC and AP agents */ - apple,static-voters = <0 1 3>; - status = "disabled"; - - clpc: performance-controller@1 { - compatible = "apple,t8140-clpc"; - reg = <1>; - apple,gpu-power-state = <&ps_gfx>; - }; - }; - - /* - * Bandwidth scratch and request registers, 0x20000 and 0x68000 - * into the display clock window at 0x302800000 (the sixth disp0 - * register range in the Apple Device Tree). - */ - pmgr_dcp: power-management@302820000 { - reg = <0x3 0x02820000 0x0 0x8>, - <0x3 0x02868000 0x0 0x4>; - reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; - #apple,bw-scratch-cells = <3>; - #apple,bw-doorbell-cells = <2>; - }; - - dcp: dcp@412e00000 { - compatible = "apple,t8140-dcp", "apple,dcp"; - reg = <0x4 0x12e00000 0x0 0x4000>, - <0x4 0x10000000 0x0 0xafc000>, - <0x4 0x12240000 0x0 0x10000>, - <0x4 0x123a8000 0x0 0x4000>, - <0x4 0x123ac000 0x0 0x4000>, - <0x4 0x12800000 0x0 0x800000>, - <0x4 0x12850000 0x0 0x4000>; - reg-names = "coproc", "disp-0", "disp-1", "disp-2", - "disp-3", "disp-4", "iop-vbar"; - mboxes = <&dcp_mbox>; - mbox-names = "mbox"; - iommus = <&dcp_dart 23>; - power-domains = <&ps_disp_cpu>; - clocks = <&clk_disp0_154>, <&clk_disp0_194>; - clock-names = "clock-154", "clock-194"; - apple,bw-scratch = <&pmgr_dcp 0 5 0>; - apple,bw-doorbell = <&pmgr_dcp 1 6>; - /* - * A single zero selects surface 0; J700 kernels that - * read this as a surface list take it the same way. - */ - apple,iomfb-surfaces = <0>; - /* generate a phandle for the loader */ - phandle = <&dcp>; - /* bus properties for the piodma child device */ - #address-cells = <2>; - #size-cells = <2>; - ranges; - /* the loader enables it when it hands the display over */ - status = "disabled"; - - disp0_piodma: piodma { - iommus = <&disp0_dart 16>; - phandle = <&disp0_piodma>; - }; - }; - - dcp_mbox: mailbox@412e08000 { - compatible = "apple,t8140-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x4 0x12e08000 0x0 0x4000>; - interrupt-parent = <&aic>; - interrupts = , - , - , - ; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; - #mbox-cells = <0>; - power-domains = <&ps_disp_cpu>; - status = "disabled"; - }; - /* * Perf-state floors of voter agents 0-3, and the floor the PMC * honours for each agent and the interface enable word of each @@ -1542,6 +1338,7 @@ pinmux = , ; }; + }; avd: avd@381080000 { @@ -1994,7 +1791,6 @@ power-domains = <&ps_ane_sys>; status = "disabled"; }; - }; }; #include "t8140-pmgr.dtsi" diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index 5cbf5d4a8d6b37..86273ba1d64d80 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -7,31 +7,32 @@ * Based on scpi-cpufreq.c */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include #include -#include #include #include #include -#include #include #include -#include -#include -#include #include #include #include #include -#include #include -#include -#include -#include -#include -#include #define APPLE_DVFS_CMD 0x20 #define APPLE_DVFS_CMD_BUSY BIT(31) @@ -516,330 +517,6 @@ static void apple_fast_die_arm(struct apple_fast_die *fd) HRTIMER_MODE_REL_SOFT); } -/* Take a sample and queue an update if the limit changes. Call with @lock held. */ -static void apple_fast_die_sample(struct apple_fast_die *fd) -{ - struct apple_cpu_priv *priv = fd->policy->driver_data; - unsigned int limit; - u8 effort; - int ret; - - ret = apple_neo_pmp_report_fast_die_effort(fd->report, fd->lane, &effort); - if (!ret) { - limit = apple_fast_die_limit(fd, effort); - fd->waiting = false; - } else { - limit = fd->fallback; - fd->waiting = true; - if (!fd->warned && - time_after(jiffies, fd->start + APPLE_FAST_DIE_WARN_S * HZ)) { - fd->warned = true; - dev_warn(priv->cpu_dev, - "PMP not running after %d s (%d): fast die-temperature limit unavailable, CPUs %*pbl stay limited to %u kHz\n", - APPLE_FAST_DIE_WARN_S, ret, - cpumask_pr_args(fd->cpus), limit); - } - } - - if (limit != fd->limit) { - WRITE_ONCE(fd->limit, limit); - schedule_work(&fd->work); - } -} - -static void apple_fast_die_work(struct work_struct *work) -{ - struct apple_fast_die *fd = container_of(work, struct apple_fast_die, work); - struct apple_cpu_priv *priv = fd->policy->driver_data; - unsigned int limit = READ_ONCE(fd->limit); - int ret; - - ret = freq_qos_update_request(&fd->req, limit); - if (ret >= 0) - return; - - dev_warn_ratelimited(priv->cpu_dev, - "failed to update the fast die-temperature limit: %d\n", ret); - /* Make the next sample queue the update again, and make sure it comes. */ - spin_lock_bh(&fd->lock); - if (fd->limit == limit) { - fd->limit = 0; - apple_fast_die_arm(fd); - } - spin_unlock_bh(&fd->lock); -} - -static enum hrtimer_restart apple_fast_die_poll(struct hrtimer *t) -{ - struct apple_fast_die *fd = container_of(t, struct apple_fast_die, poll); - enum hrtimer_restart restart = HRTIMER_NORESTART; - - spin_lock(&fd->lock); - apple_fast_die_sample(fd); - if (!fd->stopping && (!fd->per_cpu || fd->limit < fd->max)) { - hrtimer_forward_now(t, apple_fast_die_interval(fd)); - restart = HRTIMER_RESTART; - } else { - fd->polling = false; - } - spin_unlock(&fd->lock); - - return restart; -} - -static void apple_fast_die_tick(struct timer_list *t) -{ - struct apple_fast_die_timer *ft = timer_container_of(ft, t, timer); - struct apple_fast_die *fd = ft->fd; - unsigned long now = jiffies; - - /* - * The first busy CPU of the cluster to run in a tick takes the sample, - * unless the non-deferrable timer is sampling. - */ - if (xchg(&fd->last_poll, now) != now && spin_trylock(&fd->lock)) { - if (!fd->polling) { - apple_fast_die_sample(fd); - apple_fast_die_arm(fd); - } - spin_unlock(&fd->lock); - } - mod_timer(t, now + 1); -} - -static int apple_fast_die_cpu_online(unsigned int cpu) -{ - struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); - - if (!ft->fd) - return 0; - - /* Wait for a running sample, which rearms the timer, before arming it. */ - timer_delete_sync(&ft->timer); - ft->timer.expires = jiffies + 1; - add_timer_on(&ft->timer, cpu); - - return 0; -} - -static int apple_fast_die_cpu_offline(unsigned int cpu) -{ - struct apple_fast_die_timer *ft = per_cpu_ptr(&apple_fast_die_timers, cpu); - - if (ft->fd) - timer_delete_sync(&ft->timer); - - return 0; -} - -/* - * Called at the end of ->init(), with the CPU hotplug lock held. The core - * fills policy->related_cpus only after ->init() returns, and may call ->exit() - * before it does, so the limit keeps its own copy of the cluster's CPUs. - */ -static int apple_fast_die_init(struct cpufreq_policy *policy, - struct device_node *cluster) -{ - struct apple_cpu_priv *priv = policy->driver_data; - struct of_phandle_args args; - struct apple_fast_die *fd; - unsigned int cpu, i, n = 0; - bool have_power = true; - int parsed, ret; - - parsed = of_parse_phandle_with_fixed_args(cluster, "apple,fast-die-effort", - 1, 0, &args); - /* Required thermal admission must fail closed with an older DT. */ - if (parsed == -ENOENT && !priv->info->needs_thermal_policy) - return 0; - - while (policy->freq_table[n].frequency != CPUFREQ_TABLE_END) - n++; - - fd = kzalloc_flex(*fd, power, n); - if (!fd) { - ret = -ENOMEM; - goto err_put; - } - fd->nr_states = n; - if (!zalloc_cpumask_var(&fd->cpus, GFP_KERNEL)) { - ret = -ENOMEM; - goto err_free; - } - - for (i = 0; i < n; i++) { - unsigned long rate = policy->freq_table[i].frequency * 1000UL + 999; - struct dev_pm_opp *opp = dev_pm_opp_find_freq_floor(priv->cpu_dev, &rate); - - if (IS_ERR(opp)) { - ret = PTR_ERR(opp); - goto err_free; - } - fd->power[i] = dev_pm_opp_get_power(opp); - dev_pm_opp_put(opp); - if (!fd->power[i]) - have_power = false; - } - /* Without an energy model, frequency stands in for power. */ - if (!have_power) - for (i = 0; i < n; i++) - fd->power[i] = policy->freq_table[i].frequency; - - fd->policy = policy; - spin_lock_init(&fd->lock); - INIT_WORK(&fd->work, apple_fast_die_work); - hrtimer_setup(&fd->poll, apple_fast_die_poll, CLOCK_MONOTONIC, - HRTIMER_MODE_REL_SOFT); - cpumask_copy(fd->cpus, policy->cpus); - fd->per_cpu = apple_fast_die_hp_state > 0; - for_each_cpu(cpu, fd->cpus) - if (tick_nohz_full_cpu(cpu)) - fd->per_cpu = false; - fd->max = policy->freq_table[n - 1].frequency; - fd->fallback = apple_fast_die_fit(fd, div_u64(fd->power[n - 1], 3)); - fd->start = jiffies; - - if (parsed) { - /* The limit is meant to be there, so keep the fallback. */ - dev_warn(priv->cpu_dev, - "invalid apple,fast-die-effort (%d): CPUs %*pbl stay limited to %u kHz\n", - parsed, cpumask_pr_args(fd->cpus), fd->fallback); - fd->warned = true; - } else { - fd->report = args.np; - fd->lane = args.args[0]; - } - - /* Hold the fallback limit until a sample finds the PMP running. */ - fd->limit = fd->fallback; - ret = freq_qos_add_request(&policy->constraints, &fd->req, FREQ_QOS_MAX, - fd->limit); - if (ret < 0) - goto err_free; - - priv->fast_die = fd; - - if (fd->per_cpu) { - for_each_cpu(cpu, fd->cpus) { - struct apple_fast_die_timer *ft = - per_cpu_ptr(&apple_fast_die_timers, cpu); - - timer_setup(&ft->timer, apple_fast_die_tick, - TIMER_DEFERRABLE | TIMER_PINNED); - ft->fd = fd; - if (cpu_online(cpu)) - apple_fast_die_cpu_online(cpu); - } - } - - spin_lock_bh(&fd->lock); - apple_fast_die_arm(fd); - spin_unlock_bh(&fd->lock); - - dev_dbg(priv->cpu_dev, "fast die-temperature limit set up, fallback %u kHz\n", - fd->fallback); - - return 0; - -err_free: - free_cpumask_var(fd->cpus); - kfree(fd); -err_put: - if (!parsed) - of_node_put(args.np); - return ret; -} - -static void apple_fast_die_exit(struct apple_cpu_priv *priv) -{ - struct apple_fast_die *fd = priv->fast_die; - unsigned int cpu; - - if (!fd) - return; - - /* - * Keep the update work from arming the hrtimer again, stop the per-CPU - * timers, which can arm it too, then the hrtimer, and then drain the - * update any of them may have queued. - */ - spin_lock_bh(&fd->lock); - fd->stopping = true; - spin_unlock_bh(&fd->lock); - if (fd->per_cpu) { - for_each_cpu(cpu, fd->cpus) { - struct apple_fast_die_timer *ft = - per_cpu_ptr(&apple_fast_die_timers, cpu); - - timer_shutdown_sync(&ft->timer); - ft->fd = NULL; - } - } - hrtimer_cancel(&fd->poll); - cancel_work_sync(&fd->work); - freq_qos_remove_request(&fd->req); - - priv->fast_die = NULL; - of_node_put(fd->report); - free_cpumask_var(fd->cpus); - kfree(fd); -} - -/* The fastest state whose power is at most @budget, or the slowest. */ -static unsigned int apple_fast_die_fit(const struct apple_fast_die *fd, - u64 budget) -{ - unsigned int i, idx = 0; - - for (i = 0; i < fd->nr_states; i++) - if (fd->power[i] <= budget) - idx = i; - - return fd->policy->freq_table[idx].frequency; -} - -/* - * Map an effort to a frequency limit, in kHz: the fastest state whose power - * fits a budget of (255 - effort) / 255 of the fastest state's power. Any - * mapping that lowers the limit monotonically as the effort rises would do, - * since the firmware's loop is closed on the die temperature and corrects - * the error; power is the natural axis for the output of a temperature - * controller. - */ -static unsigned int apple_fast_die_limit(const struct apple_fast_die *fd, - unsigned int effort) -{ - /* - * Multiply first: dividing the fastest state's power by 255 first - * rounds it down, which would limit an unlimited cluster one state low. - */ - return apple_fast_die_fit(fd, - div_u64((u64)fd->power[fd->nr_states - 1] * - (APPLE_FAST_DIE_EFFORT_MAX - effort), - APPLE_FAST_DIE_EFFORT_MAX)); -} - -static ktime_t apple_fast_die_interval(const struct apple_fast_die *fd) -{ - if (!fd->waiting) - return ms_to_ktime(APPLE_FAST_DIE_POLL_MS); - if (!fd->warned) - return ms_to_ktime(APPLE_FAST_DIE_WAIT_MS); - return ms_to_ktime(APPLE_FAST_DIE_GAVE_UP_MS); -} - -/* Arm the non-deferrable timer if the limit needs it. Call with @lock held. */ -static void apple_fast_die_arm(struct apple_fast_die *fd) -{ - if (fd->stopping || fd->polling || - (fd->per_cpu && fd->limit >= fd->max)) - return; - - fd->polling = true; - hrtimer_start(&fd->poll, apple_fast_die_interval(fd), - HRTIMER_MODE_REL_SOFT); -} - /* Take a sample and queue an update if the limit changes. Call with @lock held. */ static void apple_fast_die_sample(struct apple_fast_die *fd) { @@ -1232,16 +909,6 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) policy->dvfs_possible_from_any_cpu = true; policy->fast_switch_possible = !info->verify_transition; policy->suspend_freq = freq_table[0].frequency; - /* Unqualified SoCs still require protection above their safe ceiling. */ - if (info->needs_thermal_policy) { - cpufreq_for_each_valid_entry(p, policy->freq_table) { - if (p->driver_data <= info->max_unmanaged_pstate) - continue; - dev_err(cpu_dev, "higher P-states require a qualified thermal policy\n"); - ret = -ENODEV; - goto out_free_cpufreq_table; - } - } return 0; diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index 74907bf6e55eb9..0e8f833861880d 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -20,31 +20,6 @@ config APPLE_PMP_REPORT depends on PM select PM_GENERIC_DOMAINS -config APPLE_PMC - bool "Apple PMC voter interface" - depends on OF - depends on PM - select PM_GENERIC_DOMAINS - help - The PMC in some Apple SoCs arbitrates the performance states of - shared rails, such as the DRAM controller and the SoC fabric, - between voter agents. This driver enables the static agents and - provides power domains that enable a dynamic agent, such as the - GPU, while its device is powered on. - -endif - -config APPLE_PMP_REPORT_KUNIT_TEST - tristate "Apple PMP report validation tests" if !KUNIT_ALL_TESTS - depends on KUNIT && PM - select PM_GENERIC_DOMAINS - default KUNIT_ALL_TESTS - help - Tests the PMP report layout, request acknowledgment flags and boot - argument validation without accessing hardware. - Includes malformed tables, address arithmetic and device ordering. - The PMP report driver does not need to be enabled to run this suite. - config APPLE_PMC bool "Apple PMC voter interface" depends on OF @@ -59,12 +34,6 @@ config APPLE_PMC GPU, while its device is powered on. It can also limit the DRAM controller rail as a thermal cooling device. - -config APPLE_PMP_REPORT_NEO - bool "Apple T8140 PMP report and fast die-temperature limits" - depends on ARCH_APPLE && OF && PM - select PM_GENERIC_DOMAINS - default ARCH_APPLE config APPLE_PMC_KUNIT_TEST bool "KUnit tests for the Apple PMC voter interface" if !KUNIT_ALL_TESTS depends on APPLE_PMC && KUNIT=y @@ -76,3 +45,14 @@ config APPLE_PMC_KUNIT_TEST registers are simulated in memory. Only useful for kernel developers. endif + +config APPLE_PMP_REPORT_KUNIT_TEST + tristate "Apple PMP report validation tests" if !KUNIT_ALL_TESTS + depends on KUNIT && PM + select PM_GENERIC_DOMAINS + default KUNIT_ALL_TESTS + help + Tests the PMP report layout, request acknowledgment flags and boot + argument validation without accessing hardware. + Includes malformed tables, address arithmetic and device ordering. + The PMP report driver does not need to be enabled to run this suite. diff --git a/drivers/pmdomain/apple/Makefile b/drivers/pmdomain/apple/Makefile index 99f2c3dcef9bd9..fdd5e4f1987d74 100644 --- a/drivers/pmdomain/apple/Makefile +++ b/drivers/pmdomain/apple/Makefile @@ -5,4 +5,3 @@ obj-$(CONFIG_APPLE_PMP_REPORT) += pmp-report.o obj-$(CONFIG_APPLE_PMP_REPORT_KUNIT_TEST) += pmp-report-test.o obj-$(CONFIG_APPLE_PMC) += pmc.o -obj-$(CONFIG_APPLE_PMP_REPORT_NEO) += pmp-report-neo.o diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index f66b9837c01e86..fcafa198656bce 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -49,8 +49,10 @@ struct apple_pmgr_ps { bool force_reset; bool externally_clocked; bool retained_25g83; - bool system_sleep; bool firmware_owned; + /* T8140: follow a transition that times out with the opposite one. */ + bool undo_timeout; + bool system_sleep; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) @@ -150,9 +152,10 @@ static bool apple_pmgr_ps_is_active(struct apple_pmgr_ps *ps) } /* - * genpd keeps a domain off when ->power_on() fails and on when ->power_off() - * fails. Follow a failed transition with the opposite one, so the domain is - * not left requesting a state genpd does not know it is in. + * With undo_timeout (T8140, T8142): genpd keeps a domain off when ->power_on() fails + * and on when ->power_off() fails. Follow a failed transition with the + * opposite one, so the domain is not left requesting a state genpd does not + * know it is in. * * The exception is a failed power-on in the noirq and syscore phases of a * system sleep transition: genpd ignores the error there and records the @@ -170,7 +173,7 @@ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) return 0; ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); - if (ret && !ps->system_sleep) + if (ret && ps->undo_timeout && !ps->system_sleep) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); return ret; @@ -178,10 +181,14 @@ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) { + struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); int ret; + if (ps->firmware_owned) + return 0; + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); - if (ret) + if (ret && ps->undo_timeout) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); return ret; @@ -192,11 +199,26 @@ static void apple_pmgr_ps_set_system_sleep(struct device *dev, bool sleep) struct apple_pmgr_ps *ps = dev_get_drvdata(dev); unsigned long flags; + /* Only read by domains that undo a failed transition. */ + if (!ps->undo_timeout) + return; + spin_lock_irqsave(&ps->genpd.slock, flags); ps->system_sleep = sleep; spin_unlock_irqrestore(&ps->genpd.slock, flags); } +/* + * Domains that do not undo a failed transition need no sleep window: let the + * PM core treat them as it treats a device without PM callbacks. + */ +static int apple_pmgr_ps_prepare(struct device *dev) +{ + struct apple_pmgr_ps *ps = dev_get_drvdata(dev); + + return !ps->undo_timeout; +} + static int apple_pmgr_ps_suspend_late(struct device *dev) { apple_pmgr_ps_set_system_sleep(dev, true); @@ -358,6 +380,8 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } ps->retained_25g83 = apple_pmgr_retained_25g83(node, ps->offset); + ps->undo_timeout = of_device_is_compatible(node, "apple,t8140-pmgr-pwrstate") || + of_device_is_compatible(node, "apple,t8142-pmgr-pwrstate"); ps->genpd.flags |= GENPD_FLAG_IRQ_SAFE; ps->genpd.name = name; ps->genpd.power_on = apple_pmgr_ps_power_on; @@ -371,7 +395,8 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); - if (!ps->retained_25g83 && ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE) + if (!ps->retained_25g83 && ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && + !ps->firmware_owned) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); @@ -390,7 +415,6 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) return dev_err_probe(dev, -EBUSY, "inherited domain %s is not active\n", name); ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; } - if (of_property_read_bool(node, "apple,always-on")) { if (ps->firmware_owned) { /* Consumers (the boot framebuffer) need it; never try to turn it on. */ if (!active) @@ -414,7 +438,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) } /* Turn on auto-PM if the domain is already on */ - if (active && !ps->retained_25g83) + if (active && !ps->retained_25g83 && !ps->firmware_owned) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, APPLE_PMGR_AUTO_ENABLE); @@ -483,6 +507,7 @@ static const struct of_device_id apple_pmgr_ps_of_match[] = { MODULE_DEVICE_TABLE(of, apple_pmgr_ps_of_match); static const struct dev_pm_ops apple_pmgr_ps_pm_ops = { + .prepare = pm_sleep_ptr(apple_pmgr_ps_prepare), LATE_SYSTEM_SLEEP_PM_OPS(apple_pmgr_ps_suspend_late, apple_pmgr_ps_resume_early) }; diff --git a/drivers/pmdomain/apple/pmp-report-neo.c b/drivers/pmdomain/apple/pmp-report-neo.c deleted file mode 100644 index d0c27cb8250789..00000000000000 --- a/drivers/pmdomain/apple/pmp-report-neo.c +++ /dev/null @@ -1,529 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Apple SoC PMP power state reporting driver - * - * Copyright The Asahi Linux Contributors - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define PMP_REPORT_READY 0x1 - -/** - * struct apple_neo_pmp_report_offsets - per-SoC layout of the report region - * @tgt_read: power state request, read side - * @tgt_write: power state request, write side - * @actual: power state acknowledgment - * @status: PMP status; bit 0 is set while the PMP is running - * @fast_die: 32-bit word holding the fast die-temperature effort of each - * CPU cluster, one byte per lane; 0 if the SoC has none - * @fast_die_lanes: number of lanes at @fast_die, at most 4 - */ -struct apple_neo_pmp_report_offsets { - u32 tgt_read; - u32 tgt_write; - u32 actual; - u32 status; - u32 fast_die; - u32 fast_die_lanes; -}; - -struct apple_neo_pmp_report; - -static int apple_neo_pmp_dvfs_create(struct apple_neo_pmp_report *report); - -struct apple_neo_pmp_report { - struct device *dev; - const struct apple_neo_pmp_report_offsets *offsets; - void __iomem *base; - spinlock_t lock; - struct list_head list; - struct dentry *debugfs; -}; - -/* Bound report regions, protected by RCU until managed teardown drains readers. */ -static LIST_HEAD(apple_neo_pmp_reports); -static DEFINE_MUTEX(apple_neo_pmp_reports_lock); -static struct dentry *apple_neo_pmp_report_debugfs; - -/** - * apple_neo_pmp_report_fast_die_effort() - read the fast die-temperature effort - * of a CPU cluster - * @np: device tree node of the report region - * @lane: lane of the cluster in the fast die-temperature report - * @effort: set to the effort on success - * - * The PMP of some SoCs runs a control loop on its own die-temperature sensors - * and publishes an 8-bit effort for each CPU cluster: 0 requests no limit, - * and the effort rises towards 255 as the die exceeds the firmware's target. - * The effort is only valid while the PMP reports that it is running; a - * stopped PMP reads 0. - * - * Context: Any context. - * Return: 0 on success, -EPROBE_DEFER if the region is not bound (yet), - * -EOPNOTSUPP if the SoC's PMP publishes no such effort, -EINVAL if @lane is - * out of range, or -EAGAIN if the PMP does not report that it is running. - */ -int apple_neo_pmp_report_fast_die_effort(const struct device_node *np, - unsigned int lane, u8 *effort) -{ - const struct apple_neo_pmp_report_offsets *offs; - const struct apple_neo_pmp_report *rep; - int ret = -EPROBE_DEFER; - - rcu_read_lock(); - list_for_each_entry_rcu(rep, &apple_neo_pmp_reports, list) { - if (rep->dev->of_node != np) - continue; - - offs = rep->offsets; - if (!offs->fast_die) { - ret = -EOPNOTSUPP; - } else if (lane >= offs->fast_die_lanes) { - ret = -EINVAL; - } else if (!(readl(rep->base + offs->status) & PMP_REPORT_READY)) { - ret = -EAGAIN; - } else { - *effort = readl(rep->base + offs->fast_die) >> - (lane * BITS_PER_BYTE); - ret = 0; - } - break; - } - rcu_read_unlock(); - - return ret; -} -EXPORT_SYMBOL_GPL(apple_neo_pmp_report_fast_die_effort); - -/* Read only the documented status and fast-die entries, never the aperture. */ -static int apple_neo_pmp_report_status_show(struct seq_file *s, void *unused) -{ - struct apple_neo_pmp_report *rep = s->private; - const struct apple_neo_pmp_report_offsets *offs = rep->offsets; - u32 status = readl(rep->base + offs->status); - unsigned int lane; - u32 effort; - - seq_printf(s, "running: %u\n", !!(status & PMP_REPORT_READY)); - if (!(status & PMP_REPORT_READY) || !offs->fast_die) - return 0; - - effort = readl(rep->base + offs->fast_die); - for (lane = 0; lane < offs->fast_die_lanes; lane++) - seq_printf(s, "lane%u: %u\n", lane, - (effort >> (lane * BITS_PER_BYTE)) & 0xff); - - return 0; -} -DEFINE_SHOW_ATTRIBUTE(apple_neo_pmp_report_status); - -static void apple_neo_pmp_report_release(void *data) -{ - struct apple_neo_pmp_report *rep = data; - - /* Withdraw child providers while their power callbacks can still use MMIO. */ - of_platform_depopulate(rep->dev); - mutex_lock(&apple_neo_pmp_reports_lock); - if (!list_empty(&rep->list)) - list_del_rcu(&rep->list); - mutex_unlock(&apple_neo_pmp_reports_lock); - synchronize_rcu(); - /* The debugfs proxy drains active reads and rejects subsequent accesses. */ - debugfs_remove(rep->debugfs); -} - -static int apple_neo_pmp_report_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; - struct apple_neo_pmp_report *rep; - int ret; - - rep = devm_kzalloc(dev, sizeof(*rep), GFP_KERNEL); - if (!rep) - return -ENOMEM; - - rep->dev = dev; - spin_lock_init(&rep->lock); - rep->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(rep->base)) - return PTR_ERR(rep->base); - rep->offsets = of_device_get_match_data(dev); - dev_set_drvdata(dev, rep); - ret = apple_neo_pmp_dvfs_create(rep); - if (ret) - return ret; - - INIT_LIST_HEAD(&rep->list); - ret = devm_add_action_or_reset(dev, apple_neo_pmp_report_release, rep); - if (ret) - return ret; - - mutex_lock(&apple_neo_pmp_reports_lock); - list_add_tail_rcu(&rep->list, &apple_neo_pmp_reports); - mutex_unlock(&apple_neo_pmp_reports_lock); - - if (rep->offsets->fast_die) - rep->debugfs = debugfs_create_file(dev_name(dev), 0400, - apple_neo_pmp_report_debugfs, rep, - &apple_neo_pmp_report_status_fops); - - /* Publish the region before child power callbacks start looking it up. */ - ret = of_platform_populate(np, NULL, NULL, dev); - if (ret) - return dev_err_probe(dev, ret, "failed to create child devices\n"); - - return 0; -} - -static const struct apple_neo_pmp_report_offsets apple_neo_pmp_offsets_t600x = { - .tgt_read = 0xf80, - .tgt_write = 0x107c0, - .actual = 0x1000, - .status = 0x10, -}; - -static const struct apple_neo_pmp_report_offsets apple_neo_pmp_offsets_t602x = { - .tgt_read = 0x2000, - .tgt_write = 0x11000, - .actual = 0x2080, - .status = 0x10, -}; - -static const struct apple_neo_pmp_report_offsets apple_neo_pmp_offsets_t8112 = { - .tgt_read = 0xa00, - .tgt_write = 0x10500, - .actual = 0xa40, - .status = 0x10, -}; - -/* The fast die-temperature report is the low two bytes of entry 0x1c1. */ -static const struct apple_neo_pmp_report_offsets apple_neo_pmp_offsets_t8140 = { - .tgt_read = 0x1880, - .tgt_write = 0x10c40, - .actual = 0x18c0, - .status = 0x10, - .fast_die = 0x1c10, - .fast_die_lanes = 2, -}; - -static const struct of_device_id apple_neo_pmp_report_of_match[] = { - { .compatible = "apple,t8140-pmp-v2-report", .data = &apple_neo_pmp_offsets_t8140 }, - {} -}; - -static struct platform_driver apple_neo_pmp_report_driver = { - .probe = apple_neo_pmp_report_probe, - .driver = { - .name = "apple-pmp-report-neo", - .of_match_table = apple_neo_pmp_report_of_match, - /* Power-domain providers are not intended for manual unbinding. */ - .suppress_bind_attrs = true, - }, -}; - -struct apple_neo_pmp_report_entry { - struct device_node *node; - struct device_node *report_node; - struct generic_pm_domain genpd; - u32 id; - bool removed; -}; - -#define genpd_to_apple_neo_pmp_report_entry(_genpd) \ - container_of(_genpd, struct apple_neo_pmp_report_entry, genpd) - -static int apple_neo_pmp_report_set_state(struct apple_neo_pmp_report *rep, u32 id, - bool enable) -{ - u64 bit_val = BIT_ULL(id); - u64 val; - unsigned long flags; - - spin_lock_irqsave(&rep->lock, flags); - val = readq(rep->base + rep->offsets->tgt_read); - val &= ~bit_val; - if (enable) - val |= bit_val; - writeq(val, rep->base + rep->offsets->tgt_write); - spin_unlock_irqrestore(&rep->lock, flags); - val = readq(rep->base + rep->offsets->status); - if ((val & PMP_REPORT_READY) == 0) - return 0; - return readq_poll_timeout_atomic( - rep->base + rep->offsets->actual, - val, - !!(val & bit_val) == !!enable, - 100, - 50000); -} - -static int apple_neo_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) -{ - struct apple_neo_pmp_report_entry *ent = genpd_to_apple_neo_pmp_report_entry(genpd); - struct apple_neo_pmp_report *rep; - int ret = -ENODEV; - - if (READ_ONCE(ent->removed)) - return ret; - - rcu_read_lock(); - list_for_each_entry_rcu(rep, &apple_neo_pmp_reports, list) { - if (rep->dev->of_node == ent->report_node) { - ret = apple_neo_pmp_report_set_state(rep, ent->id, enable); - break; - } - } - rcu_read_unlock(); - return ret; -} - -static int apple_neo_pmp_report_entry_power_on(struct generic_pm_domain *genpd) -{ - return apple_neo_pmp_report_entry_set_state(genpd, true); -} - -static int apple_neo_pmp_report_entry_power_off(struct generic_pm_domain *genpd) -{ - return apple_neo_pmp_report_entry_set_state(genpd, false); -} - -/* ADT DVFS-STATE: four 16-byte entries beginning at entry 8. */ -#define APPLE_PMP_DVFS_OFFSET 0x80 -#define APPLE_PMP_DVFS_STRIDE 16 -#define APPLE_PMP_DVFS_COUNT 4 - -static int apple_neo_pmp_dvfs_show(struct seq_file *seq, void *unused) -{ - struct apple_neo_pmp_report *report = seq->private; - u64 values[APPLE_PMP_DVFS_COUNT][2]; - unsigned int index; - - if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) - return -EAGAIN; - - for (index = 0; index < ARRAY_SIZE(values); index++) { - void __iomem *entry = report->base + APPLE_PMP_DVFS_OFFSET + - index * APPLE_PMP_DVFS_STRIDE; - - values[index][0] = readq(entry); - values[index][1] = readq(entry + sizeof(values[index][0])); - } - - if (!(readl(report->base + report->offsets->status) & PMP_REPORT_READY)) - return -EAGAIN; - - for (index = 0; index < ARRAY_SIZE(values); index++) - seq_printf(seq, "entry%u: 0x%016llx 0x%016llx\n", index + 8, - values[index][0], values[index][1]); - - return 0; -} -DEFINE_SHOW_ATTRIBUTE(apple_neo_pmp_dvfs); - -static void apple_neo_pmp_dvfs_remove(void *directory) -{ - debugfs_remove_recursive(directory); -} - -static int apple_neo_pmp_dvfs_create(struct apple_neo_pmp_report *report) -{ - struct device *dev = report->dev; - struct resource *resource; - struct dentry *directory; - char *name; - int ret; - - if (!of_device_is_compatible(dev->of_node, "apple,t8140-pmp-v2-report")) - return 0; - - resource = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 0); - if (resource_size(resource) < APPLE_PMP_DVFS_OFFSET + - APPLE_PMP_DVFS_COUNT * APPLE_PMP_DVFS_STRIDE) - return dev_err_probe(dev, -EINVAL, "DVFS report exceeds resource\n"); - - name = devm_kasprintf(dev, GFP_KERNEL, "apple-pmp-%s", dev_name(dev)); - if (!name) - return -ENOMEM; - - directory = debugfs_create_dir(name, NULL); - if (IS_ERR(directory)) - return 0; - - ret = devm_add_action_or_reset(dev, apple_neo_pmp_dvfs_remove, directory); - if (ret) - return ret; - - debugfs_create_file("dvfs", 0400, directory, report, &apple_neo_pmp_dvfs_fops); - return 0; -} - -static void apple_neo_pmp_report_entry_free(struct apple_neo_pmp_report_entry *ent) -{ - of_node_put(ent->report_node); - of_node_put(ent->node); - kfree(ent->genpd.name); - kfree(ent); -} - -static void apple_neo_pmp_report_entry_remove(struct platform_device *pdev) -{ - struct apple_neo_pmp_report_entry *ent = platform_get_drvdata(pdev); - - WRITE_ONCE(ent->removed, true); - of_genpd_del_provider(ent->node); - if (pm_genpd_remove(&ent->genpd)) { - /* The core still owns this domain. Keep its now inert storage alive. */ - dev_warn(&pdev->dev, "retaining busy power domain %s\n", ent->genpd.name); - return; - } - apple_neo_pmp_report_entry_free(ent); -} - -static int apple_neo_pmp_report_entry_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct device_node *node = dev->of_node; - struct apple_neo_pmp_report_entry *ent; - int ret; - const char *name; - struct of_phandle_iterator it; - - ent = kzalloc_obj(*ent); - if (!ent) - return -ENOMEM; - - ent->node = of_node_get(node); - ent->report_node = of_node_get(dev->parent->of_node); - - ret = of_property_read_u32(node, "reg", &ent->id); - if (ret) { - ret = dev_err_probe(dev, ret, "missing reg property\n"); - goto err_free; - } - if (ent->id >= 64) { - ret = -EINVAL; - goto err_free; - } - - ret = of_property_read_string(node, "label", &name); - if (ret < 0) { - ret = dev_err_probe(dev, ret, "missing label property\n"); - goto err_free; - } - ent->genpd.name = kstrdup(name, GFP_KERNEL); - if (!ent->genpd.name) { - ret = -ENOMEM; - goto err_free; - } - - if (of_property_read_bool(node, "apple,always-on")) { - ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; - apple_neo_pmp_report_entry_set_state(&ent->genpd, true); - } - - ent->genpd.power_on = apple_neo_pmp_report_entry_power_on; - ent->genpd.power_off = apple_neo_pmp_report_entry_power_off; - - ret = pm_genpd_init(&ent->genpd, NULL, true); - if (ret) { - ret = dev_err_probe(dev, ret, "pm_genpd_init failed\n"); - goto err_free; - } - - ret = of_genpd_add_provider_simple(node, &ent->genpd); - if (ret) { - ret = dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); - goto err_remove_domain; - } - - of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { - struct of_phandle_args parent, child; - - parent.np = it.node; - parent.args_count = of_phandle_iterator_args(&it, parent.args, MAX_PHANDLE_ARGS); - child.np = node; - child.args_count = 0; - ret = of_genpd_add_subdomain(&parent, &child); - - if (ret == -EPROBE_DEFER) { - of_node_put(parent.np); - goto err_remove; - } else if (ret < 0) { - dev_err(dev, "failed to add to parent domain: %d (%s -> %s)\n", - ret, it.node->name, node->name); - of_node_put(parent.np); - goto err_remove; - } - } - - pm_genpd_remove_device(dev); - platform_set_drvdata(pdev, ent); - - return 0; -err_remove: - of_genpd_del_provider(node); -err_remove_domain: - WRITE_ONCE(ent->removed, true); - if (pm_genpd_remove(&ent->genpd)) { - dev_warn(dev, "retaining busy power domain %s\n", ent->genpd.name); - return ret; - } -err_free: - apple_neo_pmp_report_entry_free(ent); - return ret; -} - -static const struct of_device_id apple_neo_pmp_report_entry_of_match[] = { - { .compatible = "apple,t8140-pmp-v2-report-entry" }, - {} -}; - -static struct platform_driver apple_neo_pmp_report_entry_driver = { - .probe = apple_neo_pmp_report_entry_probe, - .remove = apple_neo_pmp_report_entry_remove, - .driver = { - .name = "apple-pmp-report-neo-entry", - .of_match_table = apple_neo_pmp_report_entry_of_match, - /* Power-domain providers are not intended for manual unbinding. */ - .suppress_bind_attrs = true, - }, -}; - -MODULE_DEVICE_TABLE(of, apple_neo_pmp_report_of_match); -MODULE_DEVICE_TABLE(of, apple_neo_pmp_report_entry_of_match); - -static int __init apple_neo_pmp_report_init(void) -{ - apple_neo_pmp_report_debugfs = debugfs_create_dir("apple-pmp-report-neo", NULL); - platform_driver_register(&apple_neo_pmp_report_entry_driver); - platform_driver_register(&apple_neo_pmp_report_driver); - return 0; -} - -static void __exit apple_neo_pmp_report_exit(void) -{ - debugfs_remove_recursive(apple_neo_pmp_report_debugfs); - platform_driver_unregister(&apple_neo_pmp_report_entry_driver); - platform_driver_unregister(&apple_neo_pmp_report_driver); -} - -module_init(apple_neo_pmp_report_init); -module_exit(apple_neo_pmp_report_exit); - -MODULE_DESCRIPTION("PMP power state reporting driver for Apple SoCs"); -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index d6da36bc402066..9d585f5a0f712d 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -7,10 +7,10 @@ #include #include +#include #include #include #include -#include #include #include #include @@ -22,10 +22,8 @@ #include #include #include -#include -#include -#include #include +#include #include #include #include @@ -39,16 +37,6 @@ #define PMP_REPORT_READY 0x1 -/** - * struct apple_pmp_report_offsets - per-SoC layout of the report region - * @tgt_read: power state request, read side - * @tgt_write: power state request, write side - * @actual: power state acknowledgment - * @status: PMP status; bit 0 is set while the PMP is running - * @fast_die: 32-bit word holding the fast die-temperature effort of each - * CPU cluster, one byte per lane; 0 if the SoC has none - * @fast_die_lanes: number of lanes at @fast_die, at most 4 - */ struct apple_pmp_report_offsets { u32 tgt_read; u32 tgt_write; @@ -63,7 +51,18 @@ struct apple_pmp_report_offsets { bool temps; /* With starts_pmp: soc-device record i holds device id i + 1. */ bool devices_ordered; + /* + * The report is published in apple_pmp_reports, where the fast + * die-temperature reader and the power callbacks of its entries look + * it up by node, and its teardown withdraws the entries first (T8140). + */ + bool published; + /* + * With published: 32-bit word holding the fast die-temperature effort + * of each CPU cluster, one byte per lane; 0 if the SoC has none. + */ u32 fast_die; + /* With fast_die: number of lanes at fast_die, at most 4. */ u32 fast_die_lanes; }; @@ -74,6 +73,10 @@ struct apple_pmp_report { void __iomem *temp_sram; spinlock_t lock; + /* Only with offsets->published */ + struct list_head list; + struct dentry *debugfs; + /* Only with offsets->starts_pmp */ struct device_node *pmp; u64 seed; @@ -618,8 +621,6 @@ static const struct { { "tp000m", "Tp000m", 0x6937c }, { "tp001m", "Tp001m", 0x69420 }, { "tp002m", "Tp002m", 0x694c4 }, - struct list_head list; - struct dentry *debugfs; }; #define T6030_PMP_TEMP_CHANNELS (1 + ARRAY_SIZE(apple_pmp_t6030_temps)) @@ -1104,10 +1105,8 @@ int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int t return ret; } EXPORT_SYMBOL_GPL(apple_pmp_report_wait_supplier_ready); -/* - * Bound report regions, for apple_pmp_report_fast_die_effort(). Regions are - * added once they are set up and never removed: the driver cannot be unbound. - */ + +/* Bound published report regions, protected by RCU until managed teardown drains readers. */ static LIST_HEAD(apple_pmp_reports); static DEFINE_MUTEX(apple_pmp_reports_lock); static struct dentry *apple_pmp_report_debugfs; @@ -1126,9 +1125,10 @@ static struct dentry *apple_pmp_report_debugfs; * stopped PMP reads 0. * * Context: Any context. - * Return: 0 on success, -EPROBE_DEFER if the region is not bound (yet), - * -EOPNOTSUPP if the SoC's PMP publishes no such effort, -EINVAL if @lane is - * out of range, or -EAGAIN if the PMP does not report that it is running. + * Return: 0 on success, -EPROBE_DEFER if the region is not bound (yet) or + * is not a published region (only T8140 regions are), -EOPNOTSUPP if the + * SoC's PMP publishes no such effort, -EINVAL if @lane is out of range, or + * -EAGAIN if the PMP does not report that it is running. */ int apple_pmp_report_fast_die_effort(const struct device_node *np, unsigned int lane, u8 *effort) @@ -1199,6 +1199,35 @@ static void apple_pmp_report_release(void *data) debugfs_remove(rep->debugfs); } +/* + * With offsets->published: publish the region before its children are + * populated, and withdraw it again on teardown, children first. + */ +static int apple_pmp_report_publish(struct apple_pmp_report *rep) +{ + struct device *dev = rep->dev; + int ret; + + INIT_LIST_HEAD(&rep->list); + ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); + if (ret) + return ret; + + mutex_lock(&apple_pmp_reports_lock); + list_add_tail_rcu(&rep->list, &apple_pmp_reports); + /* Only SoCs with a fast die-temperature report have the directory. */ + if (rep->offsets->fast_die && !apple_pmp_report_debugfs) + apple_pmp_report_debugfs = debugfs_create_dir("apple-pmp-report", NULL); + mutex_unlock(&apple_pmp_reports_lock); + + if (rep->offsets->fast_die) + rep->debugfs = debugfs_create_file(dev_name(dev), 0400, + apple_pmp_report_debugfs, rep, + &apple_pmp_report_status_fops); + + return 0; +} + static int apple_pmp_report_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1248,34 +1277,16 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret) return ret; } - dev_set_drvdata(dev, rep); - ret = apple_pmp_dvfs_create(rep); - if (ret) - return ret; - - INIT_LIST_HEAD(&rep->list); - ret = devm_add_action_or_reset(dev, apple_pmp_report_release, rep); - if (ret) - return ret; + if (rep->offsets->published) { + /* Publish the region before child power callbacks start looking it up. */ + ret = apple_pmp_report_publish(rep); + if (ret) + return ret; + } ret = of_platform_populate(np, NULL, NULL, dev); if (ret && !rep->offsets->starts_pmp) return dev_err_probe(dev, ret, "failed to create child devices\n"); - mutex_lock(&apple_pmp_reports_lock); - list_add_tail_rcu(&rep->list, &apple_pmp_reports); - mutex_unlock(&apple_pmp_reports_lock); - - if (rep->offsets->fast_die) - rep->debugfs = debugfs_create_file(dev_name(dev), 0400, - apple_pmp_report_debugfs, rep, - &apple_pmp_report_status_fops); - - /* Publish the region before child power callbacks start looking it up. */ - ret = of_platform_populate(np, NULL, NULL, dev); - if (ret) - return dev_err_probe(dev, ret, "failed to create child devices\n"); - debugfs_create_file(dev_name(dev), 0400, apple_pmp_report_debugfs, - rep, &apple_pmp_report_status_fops); if (rep->offsets->starts_pmp) { /* 4: the always-on entries have seeded their requests */ request = readq(rep->base + rep->offsets->tgt_read); @@ -1371,22 +1382,25 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t6031 = { .devices_ordered = true, }; -/* Same PTD apertures; startup belongs to the native PMP supplier. */ -static const struct apple_pmp_report_offsets apple_pmp_offsets_j613_25g83 = { - .tgt_read = 0x1000, - .tgt_write = 0x10800, - .actual = 0x1080, - .status = 0x10, /* The fast die-temperature report is the low two bytes of entry 0x1c1. */ static const struct apple_pmp_report_offsets apple_pmp_offsets_t8140 = { .tgt_read = 0x1880, .tgt_write = 0x10c40, .actual = 0x18c0, .status = 0x10, + .published = true, .fast_die = 0x1c10, .fast_die_lanes = 2, }; +/* Same PTD apertures; startup belongs to the native PMP supplier. */ +static const struct apple_pmp_report_offsets apple_pmp_offsets_j613_25g83 = { + .tgt_read = 0x1000, + .tgt_write = 0x10800, + .actual = 0x1080, + .status = 0x10, +}; + static const struct of_device_id apple_pmp_report_of_match[] = { { .compatible = "apple,j613-25g83-pmp-report", .data = &apple_pmp_offsets_j613_25g83 }, { .compatible = "apple,t6000-pmp-v2-report", .data = &apple_pmp_offsets_t600x }, @@ -1410,19 +1424,26 @@ static struct platform_driver apple_pmp_report_driver = { }; struct apple_pmp_report_entry { - struct device_node *node; - struct device_node *report_node; + struct device *dev; struct generic_pm_domain genpd; u32 id; /* The PMP does not acknowledge this request. */ bool no_ack; + + /* + * Only for an entry of a published report (T8140), which owns this + * storage and looks its report up by node: see apple_pmp_t8140_entry_probe(). + */ + struct device_node *node; + struct device_node *report_node; bool removed; }; #define genpd_to_apple_pmp_report_entry(_genpd) \ container_of(_genpd, struct apple_pmp_report_entry, genpd) -static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) +static int apple_pmp_report_request(struct apple_pmp_report *rep, u32 id, + bool enable, bool no_ack) { u64 bit_val = BIT_ULL(id); u64 val; @@ -1436,7 +1457,7 @@ static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enab writeq(val, rep->base + rep->offsets->tgt_write); spin_unlock_irqrestore(&rep->lock, flags); val = readq(rep->base + rep->offsets->status); - if ((val & PMP_REPORT_READY) == 0 || ent->no_ack) + if ((val & PMP_REPORT_READY) == 0 || no_ack) return 0; return readq_poll_timeout_atomic( rep->base + rep->offsets->actual, @@ -1446,7 +1467,25 @@ static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enab 50000); } -static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, bool enable) +static int apple_pmp_report_set_state(struct generic_pm_domain *genpd, bool enable) +{ + struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); + struct apple_pmp_report *rep = dev_get_drvdata(ent->dev->parent); + + return apple_pmp_report_request(rep, ent->id, enable, ent->no_ack); +} + +static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, true); +} + +static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) +{ + return apple_pmp_report_set_state(genpd, false); +} + +static int apple_pmp_t8140_entry_set_state(struct generic_pm_domain *genpd, bool enable) { struct apple_pmp_report_entry *ent = genpd_to_apple_pmp_report_entry(genpd); struct apple_pmp_report *rep; @@ -1458,7 +1497,7 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo rcu_read_lock(); list_for_each_entry_rcu(rep, &apple_pmp_reports, list) { if (rep->dev->of_node == ent->report_node) { - ret = apple_pmp_report_set_state(rep, ent->id, enable); + ret = apple_pmp_report_request(rep, ent->id, enable, false); break; } } @@ -1466,14 +1505,14 @@ static int apple_pmp_report_entry_set_state(struct generic_pm_domain *genpd, boo return ret; } -static int apple_pmp_report_entry_power_on(struct generic_pm_domain *genpd) +static int apple_pmp_t8140_entry_power_on(struct generic_pm_domain *genpd) { - return apple_pmp_report_set_state(genpd, true); + return apple_pmp_t8140_entry_set_state(genpd, true); } -static int apple_pmp_report_entry_power_off(struct generic_pm_domain *genpd) +static int apple_pmp_t8140_entry_power_off(struct generic_pm_domain *genpd) { - return apple_pmp_report_set_state(genpd, false); + return apple_pmp_t8140_entry_set_state(genpd, false); } static void apple_pmp_report_entry_free(struct apple_pmp_report_entry *ent) @@ -1488,6 +1527,10 @@ static void apple_pmp_report_entry_remove(struct platform_device *pdev) { struct apple_pmp_report_entry *ent = platform_get_drvdata(pdev); + /* Only entries of a published report (T8140) are withdrawn. */ + if (!ent) + return; + WRITE_ONCE(ent->removed, true); of_genpd_del_provider(ent->node); if (pm_genpd_remove(&ent->genpd)) { @@ -1498,13 +1541,17 @@ static void apple_pmp_report_entry_remove(struct platform_device *pdev) apple_pmp_report_entry_free(ent); } -static int apple_pmp_report_entry_probe(struct platform_device *pdev) +/* + * An entry of a published report (T8140). Its storage, name and node + * references are its own, not the device's: if genpd refuses to remove a + * busy domain, they stay alive with callbacks that return -ENODEV. + */ +static int apple_pmp_t8140_entry_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *node = dev->of_node; struct apple_pmp_report_entry *ent; int ret; - bool record; const char *name; struct of_phandle_iterator it; @@ -1524,9 +1571,6 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) ret = -EINVAL; goto err_free; } - ret = apple_pmp_report_bit(dev_get_drvdata(dev->parent), node, &ent->id, &record); - if (ret) - return dev_err_probe(dev, ret, "no request bit (reg, soc-device record)\n"); ret = of_property_read_string(node, "label", &name); if (ret < 0) { @@ -1539,16 +1583,13 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) goto err_free; } - ent->no_ack = ((struct apple_pmp_report *)dev_get_drvdata(dev->parent))->offsets->starts_pmp && - of_property_read_bool(node, "apple,no-ack"); if (of_property_read_bool(node, "apple,always-on")) { ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; - apple_pmp_report_set_state(&ent->genpd, true); + apple_pmp_t8140_entry_set_state(&ent->genpd, true); } - ent->genpd.name = name; - ent->genpd.power_on = apple_pmp_report_entry_power_on; - ent->genpd.power_off = apple_pmp_report_entry_power_off; + ent->genpd.power_on = apple_pmp_t8140_entry_power_on; + ent->genpd.power_off = apple_pmp_t8140_entry_power_off; ret = pm_genpd_init(&ent->genpd, NULL, true); if (ret) { @@ -1599,6 +1640,83 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) return ret; } +static int apple_pmp_report_entry_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + const struct apple_pmp_report *rep = dev_get_drvdata(dev->parent); + struct apple_pmp_report_entry *ent; + int ret; + bool record; + const char *name; + struct of_phandle_iterator it; + + /* Entries of a published report (T8140) look it up by node. */ + if (rep->offsets->published) + return apple_pmp_t8140_entry_probe(pdev); + + ent = devm_kzalloc(dev, sizeof(*ent), GFP_KERNEL); + if (!ent) + return -ENOMEM; + + ent->dev = dev; + + ret = apple_pmp_report_bit(dev_get_drvdata(dev->parent), node, &ent->id, &record); + if (ret) + return dev_err_probe(dev, ret, "no request bit (reg, soc-device record)\n"); + + ret = of_property_read_string(node, "label", &name); + if (ret < 0) + return dev_err_probe(dev, ret, "missing label property\n"); + + ent->no_ack = ((struct apple_pmp_report *)dev_get_drvdata(dev->parent))->offsets->starts_pmp && + of_property_read_bool(node, "apple,no-ack"); + if (of_property_read_bool(node, "apple,always-on")) { + ent->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP; + apple_pmp_report_set_state(&ent->genpd, true); + } + + ent->genpd.name = name; + ent->genpd.power_on = apple_pmp_report_entry_power_on; + ent->genpd.power_off = apple_pmp_report_entry_power_off; + + ret = pm_genpd_init(&ent->genpd, NULL, true); + if (ret) + return dev_err_probe(dev, ret, "pm_genpd_init failed\n"); + + ret = of_genpd_add_provider_simple(node, &ent->genpd); + if (ret) + return dev_err_probe(dev, ret, "of_genpd_add_provider_simple failed\n"); + + of_for_each_phandle(&it, ret, node, "power-domains", "#power-domain-cells", -1) { + struct of_phandle_args parent, child; + + parent.np = it.node; + parent.args_count = of_phandle_iterator_args(&it, parent.args, MAX_PHANDLE_ARGS); + child.np = node; + child.args_count = 0; + ret = of_genpd_add_subdomain(&parent, &child); + + if (ret == -EPROBE_DEFER) { + of_node_put(parent.np); + goto err_remove; + } else if (ret < 0) { + dev_err(dev, "failed to add to parent domain: %d (%s -> %s)\n", + ret, it.node->name, node->name); + of_node_put(parent.np); + goto err_remove; + } + } + + pm_genpd_remove_device(dev); + + return 0; +err_remove: + of_genpd_del_provider(node); + pm_genpd_remove(&ent->genpd); + return ret; +} + static const struct of_device_id apple_pmp_report_entry_of_match[] = { { .compatible = "apple,t6000-pmp-v2-report-entry" }, {} @@ -1625,9 +1743,9 @@ static int __init apple_pmp_report_init(void) static void __exit apple_pmp_report_exit(void) { - debugfs_remove_recursive(apple_pmp_report_debugfs); platform_driver_unregister(&apple_pmp_report_entry_driver); platform_driver_unregister(&apple_pmp_report_driver); + debugfs_remove_recursive(apple_pmp_report_debugfs); } module_init(apple_pmp_report_init); diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 07baa134460b6c..0533d7a3b5acfe 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -130,6 +130,9 @@ impl PmpState { #[pin_data] struct PmpData { dev: ARef, + /// T8140 firmware is not sent the UPDATE_IOREG confirmation, and its + /// messages are not taken as device-power replies. + t8140: bool, pmp_mmio: Pin>>>, asc_mmio: Pin>>>, #[pin] @@ -143,7 +146,7 @@ struct PmpData { } impl PmpData { - fn new(dev: &platform::Device) -> Result> { + fn new(dev: &platform::Device, t8140: bool) -> Result> { let pmp_req = dev.io_request_by_name(c"pmp").ok_or(EINVAL)?; let pmp_mmio = KBox::pin_init(pmp_req.iomap_sized::(), GFP_KERNEL)?; let asc_req = dev.io_request_by_name(c"asc").ok_or(EINVAL)?; @@ -152,6 +155,7 @@ impl PmpData { try_pin_init!( PmpData { dev: dev.as_ref().into(), + t8140, pmp_mmio, asc_mmio, rtkit <- new_mutex!(None), @@ -361,7 +365,8 @@ impl PmpData { } fn recv_message(&self, msg: u64) -> Result<()> { let pm_command = (msg >> PM_COMMAND_SHIFT) & 0xff; - if msg & PM_COMMAND_MARKER != 0 && (pm_command == 0x0f || pm_command == 0x10) { + if !self.t8140 && msg & PM_COMMAND_MARKER != 0 && (pm_command == 0x0f || pm_command == 0x10) + { let mut state = self.power_command.lock(); state.reply(msg); if *state == Command::Failed { @@ -380,7 +385,7 @@ impl PmpData { OPC_SET_IOREG => self.set_ioreg(msg & SET_IOREG_INDEX_MASK)?, // Firmware waits for this confirmation before publishing reports. // Do not echo the registry index/payload or add a generic ACK bit. - OPC_UPDATE_IOREG => OPC_CONFIRM_IOREG << OPC_SHIFT, + OPC_UPDATE_IOREG if !self.t8140 => OPC_CONFIRM_IOREG << OPC_SHIFT, _ => { dev_err!(self.dev, "Got unknown message {}", msg); return Err(EIO); @@ -510,11 +515,12 @@ impl platform::Driver for PmpDriver { fn probe(pdev: &platform::Device, info: Option<&bool>) -> impl PinInit { let dev: ARef = pdev.as_ref().into(); - if info.copied().unwrap_or(false) { + let t8140 = info.copied().unwrap_or(false); + if t8140 { // SAFETY: The T8140 DART accepts 42-bit IOVAs; no DMA allocations exist yet. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; } - let data = PmpData::new(pdev)?; + let data = PmpData::new(pdev, t8140)?; let node = dev.fwnode().ok_or(EIO)?; let dvid = node .property_read(c"apple,dram-vendor-id") diff --git a/include/linux/soc/apple/pmp-report-neo.h b/include/linux/soc/apple/pmp-report-neo.h deleted file mode 100644 index 1d0db98281195c..00000000000000 --- a/include/linux/soc/apple/pmp-report-neo.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* - * Apple SoC PMP report region - * Copyright (C) The Asahi Linux Contributors - */ - -#ifndef _LINUX_SOC_APPLE_PMP_REPORT_NEO_H_ -#define _LINUX_SOC_APPLE_PMP_REPORT_NEO_H_ - -#include -#include - -struct device_node; - -#if IS_ENABLED(CONFIG_APPLE_PMP_REPORT_NEO) -int apple_neo_pmp_report_fast_die_effort(const struct device_node *np, - unsigned int lane, u8 *effort); -#else -static inline int apple_neo_pmp_report_fast_die_effort(const struct device_node *np, - unsigned int lane, u8 *effort) -{ - return -ENODEV; -} -#endif - -#endif /* _LINUX_SOC_APPLE_PMP_REPORT_NEO_H_ */ From fdec7c503f548aa5c109c01c5f40ad5856eab308 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Wed, 14 Oct 2026 21:41:25 +0200 Subject: [PATCH 560/631] drm: apple: do not report a bad link for a powered-off Type-C CRTC Since commit 57f9f3623299 ("drm/apple: fence EDID publication by service lifetime and unblock RX"), the teardown of DCP's AV service schedules the connector hotplug worker to invalidate the EDID. DCP tears that service down whenever a Type-C CRTC is powered off with the display attached (DPMS off). dcp_hotplug() then finds the connector connected without a valid mode and reports the link as bad. wlroots treats a bad link as a disconnect and replaces the output on the next wake, which powers the CRTC off and on again, and that power off does the same: the display never comes back from the first DPMS off (Mac mini, EIZO EV2780 on USB-C). There is no link while the CRTC is off, and dcp_poweron() retrains one on wake. Leave the link status and the CRTC alone then; the EDID is still invalidated. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/iomfb.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index e0aa8b2e8d29c5..f2673d637279ca 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -960,8 +960,16 @@ void dcp_hotplug(struct work_struct *work) * display modes from atomic_flush, so userspace needs to trigger a * flush, or the CRTC gets no signal. */ + /* + * Not for a Type-C CRTC powered off with the display attached: it has + * no link, and dcp_poweron() retrains one on wake. DCP tears its AV + * service down at power off, which lands here; a link reported bad + * then makes wlroots replace the output on wake, which powers the + * CRTC off and on again, and that wake does the same, forever. + */ if (!notify_only && connector->base.state && !READ_ONCE(dcp->mode_state.valid) && connector->connected && + !READ_ONCE(dcp->typec_crtc_off) && !(dcp_uses_t6020_tunnel_flow(dcp))) { drm_connector_set_link_status_property(&connector->base, DRM_MODE_LINK_STATUS_BAD); From 0ad5362d468eb89129c906eec0cbf6f58e4d0451 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:02 -0400 Subject: [PATCH 561/631] soc: apple: finish the merge of DART, QSPI and SMC hwmon Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. apple-dart: keep integration's reserved-table firmware handoff, and use the aurora-wip mirroring of firmware-owned slots only on DARTs that mirror firmware roots without that handoff (T8140 and T8142). spi-apple-qspi: keep integration's read-only default and its bounded J613 NVRAM writes on T8122 and T8132. The T8140 controller keeps the aurora-wip read and write paths. macsmc-hwmon: integration's driver, with the J700 and J813 sensors from aurora-wip. Assisted-by: LLM Signed-off-by: Diljit Singh --- .../bindings/hwmon/apple,smc-hwmon.yaml | 5 - drivers/hwmon/macsmc-hwmon.c | 152 ++++++++-------- drivers/iommu/apple-dart.c | 146 ++++++++++++--- drivers/nvme/host/apple.c | 2 - drivers/rtc/rtc-apple-pmu.c | 7 - drivers/soc/apple/mailbox.c | 3 - drivers/spi/Kconfig | 7 +- drivers/spi/spi-apple-qspi.c | 172 +++++++++++++++--- 8 files changed, 350 insertions(+), 144 deletions(-) diff --git a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml index 0c585686f2e9b9..df155f9c34a65b 100644 --- a/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml +++ b/Documentation/devicetree/bindings/hwmon/apple,smc-hwmon.yaml @@ -50,8 +50,6 @@ properties: Channels are reporting-only; their indices depend on the firmware and must not be used by a static thermal policy. - apple,cpu-thermal-policy: - type: boolean "#thermal-sensor-cells": const: 1 description: @@ -88,7 +86,6 @@ patternProperties: pattern: "^[A-Za-z0-9]{4}$" description: Writeable key to enable/disable manual fan control - "^power-[A-Za-z0-9]{4}$": $ref: "#/$defs/sensor" unevaluatedProperties: false @@ -106,8 +103,6 @@ allOf: required: - apple,discover-temperatures then: - properties: - apple,cpu-thermal-policy: false patternProperties: "^temperature-": false diff --git a/drivers/hwmon/macsmc-hwmon.c b/drivers/hwmon/macsmc-hwmon.c index 94358065a44f16..06d2c6e2d0ac38 100644 --- a/drivers/hwmon/macsmc-hwmon.c +++ b/drivers/hwmon/macsmc-hwmon.c @@ -689,6 +689,74 @@ static int macsmc_hwmon_create_sensor(struct device *dev, struct apple_smc *smc, return 0; } +/* + * A SoC die temperature sensor: a readable float temperature key, labelled + * "SoC die ". + */ +static int macsmc_hwmon_create_die_sensor(struct macsmc_hwmon *hwmon, const char *key, + struct macsmc_hwmon_sensor *sensor) +{ + int ret; + + ret = macsmc_hwmon_parse_key(hwmon->dev, hwmon->smc, sensor, key); + if (ret) + return ret; + if (!(sensor->info.flags & APPLE_SMC_READABLE) || + sensor->info.type_code != __SMC_KEY('f', 'l', 't', ' ') || + sensor->info.size != sizeof(u32)) { + dev_info(hwmon->dev, "SoC die key %s is not a readable float; ignored\n", key); + return -EINVAL; + } + snprintf(sensor->label, sizeof(sensor->label), "SoC die %s", key); + sensor->attrs = HWMON_T_INPUT | HWMON_T_LABEL; + sensor->die = true; + return 0; +} + +static int macsmc_hwmon_create_pressure(struct macsmc_hwmon *hwmon, + struct device_node *hwmon_node) +{ + struct macsmc_hwmon_sensor pressure = {}; + const char *key; + s32 val; + int ret; + + ret = of_property_read_string(hwmon_node, "apple,thermal-pressure-key", + &key); + if (ret == -EINVAL) { + const struct macsmc_hwmon_board *board = macsmc_hwmon_board(); + + if (!board || !board->pressure_key) + return 0; + key = board->pressure_key; + } else if (ret) { + return ret; + } + + ret = macsmc_hwmon_parse_key(hwmon->dev, hwmon->smc, &pressure, key); + if (ret) + goto invalid; + + if (pressure.info.size != sizeof(s32) || + pressure.info.type_code != __SMC_KEY('s', 'i', '3', '2') || + !(pressure.info.flags & APPLE_SMC_READABLE)) { + ret = -EINVAL; + goto invalid; + } + + ret = apple_smc_read_s32(hwmon->smc, pressure.macsmc_key, &val); + if (ret) + goto invalid; + + hwmon->pressure = pressure; + return 0; + +invalid: + dev_warn(hwmon->dev, "Ignoring invalid thermal pressure key %s (%d)\n", + key, ret); + return 0; +} + /* * Account for the temperature sensor just set up at the end of @temp. * @@ -772,74 +840,6 @@ static int macsmc_hwmon_discover_temperatures(struct macsmc_hwmon *hwmon) return 0; } -/* - * A SoC die temperature sensor: a readable float temperature key, labelled - * "SoC die ". - */ -static int macsmc_hwmon_create_die_sensor(struct macsmc_hwmon *hwmon, const char *key, - struct macsmc_hwmon_sensor *sensor) -{ - int ret; - - ret = macsmc_hwmon_parse_key(hwmon->dev, hwmon->smc, sensor, key); - if (ret) - return ret; - if (!(sensor->info.flags & APPLE_SMC_READABLE) || - sensor->info.type_code != __SMC_KEY('f', 'l', 't', ' ') || - sensor->info.size != sizeof(u32)) { - dev_info(hwmon->dev, "SoC die key %s is not a readable float; ignored\n", key); - return -EINVAL; - } - snprintf(sensor->label, sizeof(sensor->label), "SoC die %s", key); - sensor->attrs = HWMON_T_INPUT | HWMON_T_LABEL; - sensor->die = true; - return 0; -} - -static int macsmc_hwmon_create_pressure(struct macsmc_hwmon *hwmon, - struct device_node *hwmon_node) -{ - struct macsmc_hwmon_sensor pressure = {}; - const char *key; - s32 val; - int ret; - - ret = of_property_read_string(hwmon_node, "apple,thermal-pressure-key", - &key); - if (ret == -EINVAL) { - const struct macsmc_hwmon_board *board = macsmc_hwmon_board(); - - if (!board || !board->pressure_key) - return 0; - key = board->pressure_key; - } else if (ret) { - return ret; - } - - ret = macsmc_hwmon_parse_key(hwmon->dev, hwmon->smc, &pressure, key); - if (ret) - goto invalid; - - if (pressure.info.size != sizeof(s32) || - pressure.info.type_code != __SMC_KEY('s', 'i', '3', '2') || - !(pressure.info.flags & APPLE_SMC_READABLE)) { - ret = -EINVAL; - goto invalid; - } - - ret = apple_smc_read_s32(hwmon->smc, pressure.macsmc_key, &val); - if (ret) - goto invalid; - - hwmon->pressure = pressure; - return 0; - -invalid: - dev_warn(hwmon->dev, "Ignoring invalid thermal pressure key %s (%d)\n", - key, ret); - return 0; -} - /* * Fan data is exposed by the SMC as multiple sensors. * @@ -919,6 +919,7 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, u32 n_current = 0, n_fan = 0, n_power = 0, n_temperature = 0, n_voltage = 0; const char *die_keys = macsmc_hwmon_die_key_list(), *list; char key[5]; + bool indexed; int ret; for_each_child_of_node_with_prefix(hwmon_node, key_node, "current-") { @@ -996,10 +997,18 @@ static int macsmc_hwmon_populate_sensors(struct macsmc_hwmon *hwmon, if (!hwmon->temp.sensors) return -ENOMEM; + /* + * Only a node that is a thermal sensor provider (T8140) has zones + * referring to its temperature channels by index: there a node + * that fails to set up keeps its channel. Elsewhere it is skipped. + */ + indexed = of_property_present(hwmon_node, "#thermal-sensor-cells"); for_each_child_of_node_with_prefix(hwmon_node, key_node, "temperature-") { sensor = &hwmon->temp.sensors[hwmon->temp.count]; ret = macsmc_hwmon_create_sensor(hwmon->dev, hwmon->smc, key_node, sensor); + if (ret && !indexed) + continue; if (ret) dev_warn(hwmon->dev, "%pOFn unavailable (%d), temp%u hidden\n", key_node, ret, hwmon->temp.count + 1); @@ -1341,15 +1350,16 @@ static int macsmc_hwmon_probe(struct platform_device *pdev) return ret; } - ret = macsmc_hwmon_discover_temperatures(hwmon); - if (ret) - return dev_err_probe(hwmon->dev, ret, "Temperature discovery failed\n"); if (macsmc_hwmon_is_m3()) { ret = macsmc_hwmon_create_pressure(hwmon, hwmon->dev->of_node); if (ret) return ret; } + ret = macsmc_hwmon_discover_temperatures(hwmon); + if (ret) + return dev_err_probe(hwmon->dev, ret, "Temperature discovery failed\n"); + if (!hwmon->curr.count && !hwmon->fan.count && !hwmon->power.count && !hwmon->temp.count && !hwmon->volt.count && !hwmon->pressure.macsmc_key) { diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 0ee29b60992909..db90c8d3877150 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -235,6 +235,8 @@ struct apple_dart_hw { struct apple_dart_fw_root { u64 *entry; u64 **leaf; + /* Linux root entry already verified against each firmware slot. */ + u64 *accepted; }; /* @@ -635,7 +637,10 @@ static bool apple_dart_fw_slot_mirrors(const struct apple_dart *dart, * A locked root belongs to firmware. Publish only into vacant slots and * remember exactly what we installed, so teardown cannot erase firmware * mappings added after Linux attached. A slot firmware already holds is - * never written; its descendants must still mirror Linux's mappings, and + * never written. It is accepted once if it mirrors Linux's mappings, and + * apple_dart_check_fw_map() checks later mappings into it page by page. + * With the reserved-table handoff (fw_handoff) its descendants must still + * mirror Linux's mappings at every publication, and * apple_dart_check_fw_map() rejects new mappings anywhere in that slot. */ static int apple_dart_publish_root(const struct apple_dart *dart, @@ -655,7 +660,8 @@ static int apple_dart_publish_root(const struct apple_dart *dart, continue; if (previous || !apple_dart_fw_slot_owned(fw, live, i)) return -EBUSY; - if (!apple_dart_fw_slot_mirrors(dart, fw, next, i, entries)) + if ((dart->fw_handoff || next != fw->accepted[i]) && + !apple_dart_fw_slot_mirrors(dart, fw, next, i, entries)) return -EBUSY; } @@ -667,6 +673,7 @@ static int apple_dart_publish_root(const struct apple_dart *dart, if (next == previous) continue; if (!previous && apple_dart_fw_slot_owned(fw, live, i)) { + fw->accepted[i] = next; continue; } if (cmpxchg(&live[i], previous, next) != previous) @@ -703,6 +710,7 @@ static void apple_dart_free_fw_root(struct apple_dart *dart, for (i = 0; i < dart->pgsize / sizeof(u64); i++) if (fw->leaf[i]) memunmap(fw->leaf[i]); + kfree(fw->accepted); kfree(fw->leaf); kfree(fw->entry); kfree(fw); @@ -711,17 +719,23 @@ static void apple_dart_free_fw_root(struct apple_dart *dart, /* * Record the leaf tables behind the slots firmware holds in a three-level * T8110-format locked root, on DARTs whose loader re-exports firmware - * translations (fw_handoff). Other roots are not snapshotted; their firmware + * translations (fw_mirror). Other roots are not snapshotted; their firmware * slots keep rejecting Linux mappings as before. + * + * With the reserved-table handoff (fw_handoff) every table page must be + * reserved RAM that does not change while it is recorded, and any failure + * fails the attach. Without it, a leaf table that cannot be mapped is left + * out and a failed allocation leaves the root unrecorded. */ static struct apple_dart_fw_root * apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) { size_t i, entries = dart->pgsize / sizeof(u64); struct apple_dart_fw_root *fw; + struct apple_dart_fw_root *nomem = dart->fw_handoff ? ERR_PTR(-ENOMEM) : NULL; /* Only the T8110-style table layout this walker decodes. */ - if (!dart->fw_handoff || dart->hw->fmt != APPLE_DART2) + if (!dart->fw_mirror || dart->hw->fmt != APPLE_DART2) return NULL; if (dart->hw->tcr_4level && (apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level)) @@ -729,12 +743,13 @@ apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) fw = kzalloc(sizeof(*fw), GFP_KERNEL); if (!fw) - return ERR_PTR(-ENOMEM); + return nomem; fw->entry = kcalloc(entries, sizeof(*fw->entry), GFP_KERNEL); fw->leaf = kcalloc(entries, sizeof(*fw->leaf), GFP_KERNEL); - if (!fw->entry || !fw->leaf) { + fw->accepted = kcalloc(entries, sizeof(*fw->accepted), GFP_KERNEL); + if (!fw->entry || !fw->leaf || !fw->accepted) { apple_dart_free_fw_root(dart, fw); - return ERR_PTR(-ENOMEM); + return nomem; } for (i = 0; i < entries; i++) { @@ -743,6 +758,14 @@ apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) if (!(pte & APPLE_DART_PTE_VALID)) continue; + if (!dart->fw_handoff) { + fw->leaf[i] = memremap(apple_dart_pte_to_paddr(dart, pte), + dart->pgsize, MEMREMAP_WB); + if (!fw->leaf[i]) + continue; + fw->entry[i] = pte; + continue; + } if (!apple_dart_fw_table_decode(dart, pte, &phys)) { apple_dart_free_fw_root(dart, fw); return ERR_PTR(-EINVAL); @@ -908,7 +931,8 @@ apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg, * before attachment, so replace the old root entries. * T8110 v2.2+ instead retains firmware-owned entries. */ - if (dart->hw->type != DART_T8110 || dart->version < 0x0202) { + if (!(dart->fw_mirror && !dart->fw_handoff) && + (dart->hw->type != DART_T8110 || dart->version < 0x0202)) { size_t entry; dma_wmb(); @@ -1041,7 +1065,7 @@ static bool apple_dart_fw_slot_foreign(const u64 *live, const u64 *owned, } /* - * On T8140 DARTs that keep firmware roots, the hardware keeps + * On DARTs that keep firmware roots (T8140 and T8142), the hardware keeps * using the translations behind the firmware-owned root slots, e.g. the * display controller scanning out the boot framebuffer while Linux attaches, * or the AOP firmware's own DMA on the AOP DART streams that Linux shares. @@ -1069,6 +1093,7 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, return 0; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + const struct apple_dart_fw_root *fw; const u64 *live, *owned; u64 base, span, start_dva, end_dva; bool four_level; @@ -1107,8 +1132,16 @@ static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, spin_lock_irqsave(&dart->lock, flags); live = dart->locked_ttbr[sid][0]; owned = dart->locked_owned[sid][0]; + fw = dart->locked_fw[sid][0]; + /* + * Four-level roots are not snapshotted, and with the reserved-table + * handoff firmware may replace its slots' descendants: there every + * valid slot Linux did not publish is firmware's. + */ for (i = first; live && i <= last; i++) - if (apple_dart_fw_slot_foreign(live, owned, i)) + if (four_level || dart->fw_handoff ? + apple_dart_fw_slot_foreign(live, owned, i) : + apple_dart_fw_slot_owned(fw, live, i)) __set_bit(i, fw_slots); spin_unlock_irqrestore(&dart->lock, flags); if (!live) @@ -1389,8 +1422,48 @@ static bool apple_dart_domain_uses_fw(struct apple_dart_domain *domain, int i, } /* - * Hardware continues to use every firmware-owned slot. Its entire IOVA - * span is excluded from Linux allocation, including currently unmapped holes. + * With the reserved-table handoff (fw_handoff), hardware continues to use + * every firmware-owned slot. Its entire IOVA span is excluded from Linux + * allocation, including currently unmapped holes. Called with dart->lock held. + */ +static bool apple_dart_fw_handoff_page_busy(struct apple_dart *dart, + struct apple_dart_domain *domain, + int i, + struct apple_dart_atomic_stream_map *map, + u64 addr) +{ + u32 entries = dart->pgsize / sizeof(u64); + u32 pgshift = ilog2(dart->pgsize), bits = ilog2(entries); + int sid; + + for (sid = 0; sid < dart->num_streams; sid++) { + bool four; + u64 idx, slot; + const u64 *live, *owned; + + if (!apple_dart_domain_uses_fw(domain, i, map, sid)) + continue; + four = apple_dart_readl(dart, DART_TCR(dart, sid)) & + dart->hw->tcr_4level; + idx = four ? 0 : addr >> (pgshift + 2 * bits); + slot = (addr >> (pgshift + (four ? 2 : 1) * bits)) & + (entries - 1); + if (idx >= DART_MAX_TTBR) + continue; + live = dart->locked_ttbr[sid][idx]; + owned = dart->locked_owned[sid][idx]; + /* All pages of an inherited slot remain firmware's. */ + if (live && apple_dart_fw_slot_foreign(live, owned, slot)) + return true; + } + return false; +} + +/* + * Hardware continues to use every firmware-owned slot. Accept a private + * software mapping only when every stream using such a slot agrees on the + * full-page translation and its protection/cache attributes. With the + * reserved-table handoff, refuse any mapping into such a slot instead. */ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, phys_addr_t paddr, size_t size, int prot) @@ -1405,30 +1478,43 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, unsigned long flags; size_t off; - if (!dart->fw_handoff) + if (!dart->fw_mirror) continue; spin_lock_irqsave(&dart->lock, flags); for (off = 0; off < size; off += dart->pgsize) { u64 addr = dva + off; + u64 leaf = (addr >> pgshift) & (entries - 1); + u64 slot = (addr >> (pgshift + bits)) & (entries - 1); + u64 idx = addr >> (pgshift + 2 * bits); + if (dart->fw_handoff) { + if (apple_dart_fw_handoff_page_busy(dart, domain, i, + map, addr)) { + ret = -EBUSY; + goto unlock; + } + continue; + } + if (idx >= DART_MAX_TTBR) + continue; for (sid = 0; sid < dart->num_streams; sid++) { - bool four; - u64 idx, slot; - const u64 *live, *owned; + const struct apple_dart_fw_root *fw; + phys_addr_t phys; + u64 pte; + int fw_prot; if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; - four = apple_dart_readl(dart, DART_TCR(dart, sid)) & - dart->hw->tcr_4level; - idx = four ? 0 : addr >> (pgshift + 2 * bits); - slot = (addr >> (pgshift + (four ? 2 : 1) * bits)) & - (entries - 1); - if (idx >= DART_MAX_TTBR) + fw = dart->locked_fw[sid][idx]; + if (!dart->locked_ttbr[sid][idx] || + !apple_dart_fw_slot_owned(fw, + dart->locked_ttbr[sid][idx], slot)) continue; - live = dart->locked_ttbr[sid][idx]; - owned = dart->locked_owned[sid][idx]; - /* All pages of an inherited slot remain firmware's. */ - if (live && apple_dart_fw_slot_foreign(live, owned, slot)) { + pte = READ_ONCE(fw->leaf[slot][leaf]); + if (!apple_dart_fw_leaf_decode(pte, addr, + &phys, &fw_prot) || + !apple_dart_fw_mapping_matches(phys, fw_prot, + paddr + off, prot)) { ret = -EBUSY; goto unlock; } @@ -1456,7 +1542,7 @@ apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) size_t page = (address / dart->pgsize) % entries; size_t slot = address / dart->pgsize / entries; - if (!dart->locked || !dart->fw_handoff || slot >= entries) + if (!dart->locked || !dart->fw_mirror || slot >= entries) return 0; spin_lock_irqsave(&dart->lock, flags); for (sid = 0; sid < dart->num_streams; sid++) { @@ -1726,7 +1812,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, /* Make inherited translations visible before core reservation setup. */ for_each_stream_map(i, cfg, stream) { - if (!stream->dart->locked || !stream->dart->fw_handoff) + if (!stream->dart->locked || !stream->dart->fw_mirror) continue; apple_dart_set_locked_window(dart_domain, stream); for (index = 0; index < pgtbl_cfg.apple_dart_cfg.n_ttbrs; index++) { @@ -2846,8 +2932,10 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = false; else dart->locked = apple_dart_is_locked(dart); + /* These loaders re-export the firmware's own display translations. */ dart->fw_mirror = dart->locked && - of_device_is_compatible(dev->of_node, "apple,t8140-dart"); + (of_device_is_compatible(pdev->dev.of_node, "apple,t8140-dart") || + of_device_is_compatible(pdev->dev.of_node, "apple,t8142-dart")); /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); if (!dart->locked) { diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 0b71973f33622f..12d88a8d75672e 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -2415,8 +2415,6 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) /* Released by apple_nvme_free_ctrl(), including asynchronous users. */ kref_get(&anv->ref); - if (anv->hw->has_lsq_nvmmu) - anv->ctrl.crypto_profile = &anv->crypto_profile; if (anv->hw->has_lsq_nvmmu) anv->ctrl.crypto_profile = &anv->crypto_profile; diff --git a/drivers/rtc/rtc-apple-pmu.c b/drivers/rtc/rtc-apple-pmu.c index 8c7e4156b015f0..2e3161a914c8a2 100644 --- a/drivers/rtc/rtc-apple-pmu.c +++ b/drivers/rtc/rtc-apple-pmu.c @@ -24,10 +24,6 @@ #define BAKU_RTC_HZ_SHIFT 16 #define BAKU_RTC_MAX_DELAY 3600 -static bool experimental; -module_param(experimental, bool, 0444); -MODULE_PARM_DESC(experimental, "Enable alarms with retained SPMI/AIC power in s2idle"); - struct baku_rtc { struct regmap *map; struct nvmem_cell *offset; @@ -312,9 +308,6 @@ static int baku_rtc_probe(struct platform_device *pdev) struct baku_rtc *baku; int error; - /* Retention has only been measured on J700 with the explicit opt-in. */ - if (!experimental) - return -ENODEV; baku = devm_kzalloc(dev, sizeof(*baku), GFP_KERNEL); if (!baku) return -ENOMEM; diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 53be121ac970a3..7804cf7922820c 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -773,9 +773,6 @@ static int apple_mbox_probe(struct platform_device *pdev) if (ret) return ret; ret = devm_add_action_or_reset(dev, apple_mbox_quiesce, mbox); - if (ret) - return ret; - ret = devm_add_action_or_reset(dev, apple_mbox_quiesce, mbox); if (ret) return ret; diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 67df03c7952975..f700acbcf8a168 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -137,9 +137,10 @@ config SPI_APPLE_QSPI This enables support for Apple's command-FIFO QSPIMC controller. Unlike the ordinary Apple SPIMC block, QSPIMC accepts packed SPI memory descriptors and is used for the boot NOR (and its NVRAM) - on T8122 (J613 25G83), T8132 (M4) and T8140 systems. The driver - defaults read-only. Only the pinned J613 NVRAM layout can enable - bounded writes through an explicit root runtime opt-in. + on T8122 (J613 25G83), T8132 (M4) and T8140 systems. On T8122 and + T8132 the driver defaults read-only, and only the pinned J613 NVRAM + layout can enable bounded writes through an explicit root runtime + opt-in. T8140 is not restricted. config SPI_AR934X tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver" diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 2c563fc226c866..0367b63ae8aaee 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -7,11 +7,15 @@ * command FIFO. This driver intentionally implements the PIO spi-mem path * only; that is sufficient for SPI NOR discovery and access. * - * The boot NOR also holds iBoot and the system configuration, so the driver - * starts read-only: only identification, status and read opcodes reach the - * controller unless root explicitly clears read_only at runtime. Even then, only - * bounded J613 NVRAM page programs and 4 KiB erases are admitted. - * A power loss can interrupt a NOR update within the admitted range. + * On T8122 and T8132 the boot NOR also holds iBoot and the system + * configuration, so those controllers start read-only: only identification, + * status and read opcodes reach the controller unless root explicitly clears + * read_only at runtime. Even then, only bounded J613 NVRAM page programs and + * 4 KiB erases are admitted. A power loss can interrupt a NOR update within + * the admitted range. + * + * T8140 keeps unrestricted spi-mem access with dual and quad transfers; the + * read_only policy does not apply to it. */ #include @@ -25,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -128,7 +133,7 @@ static const struct kernel_param_ops apple_qspi_read_only_ops = { .get = apple_qspi_get_read_only, }; module_param_cb(read_only, &apple_qspi_read_only_ops, &read_only, 0644); -MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded J613 NVRAM writes"); +MODULE_PARM_DESC(read_only, "Default true; root runtime opt-in for bounded J613 NVRAM writes (T8122/T8132 only)"); static bool apple_qspi_opcode_is_read(u8 opcode) { @@ -151,7 +156,47 @@ static bool apple_qspi_opcode_is_read(u8 opcode) } } +/* Per-compatible controller behaviour. */ +struct apple_qspi_hw { + /* + * Gate requests through the read_only/J613 NVRAM write policy. Without + * it, every supported spi-mem operation reaches the controller. + */ + bool nvram_policy; + /* + * Send TX-only operations as one payload stream: zero header counts and + * a length word that covers opcode, address and data. Otherwise the + * length word carries only the data length and the descriptor counts the + * opcode and address bytes. + */ + bool tx_single_stream; + /* Advertise dual and quad transfers to spi-mem. */ + bool multi_io; + /* + * Service the FIFOs by microsecond polling instead of once per jiffy + * while waiting for a transfer to complete. + */ + bool us_poll; + /* Refuse a requested clock below the divider range instead of clamping. */ + bool exact_clock; + /* Report the controller at probe at debug level only. */ + bool quiet_probe; +}; + +static const struct apple_qspi_hw apple_qspi_t8122_hw = { + .nvram_policy = true, + .tx_single_stream = true, +}; + +static const struct apple_qspi_hw apple_qspi_t8140_hw = { + .multi_io = true, + .us_poll = true, + .exact_clock = true, + .quiet_probe = true, +}; + struct apple_qspi { + const struct apple_qspi_hw *hw; /* Only the pinned J613 controller/partition may opt in. */ bool nvram_write_candidate; bool write_fault; @@ -424,10 +469,14 @@ static int apple_qspi_set_frequency(struct apple_qspi *qspi, return -EINVAL; divisor = DIV_ROUND_UP(input, requested); - /* Never exceed the requested clock when it is below our range. */ - if (divisor > 256) - return -EINVAL; - divisor = max_t(u32, divisor, 2); + if (qspi->hw->exact_clock) { + /* Never exceed the requested clock when it is below our range. */ + if (divisor > 256) + return -EINVAL; + divisor = max_t(u32, divisor, 2); + } else { + divisor = clamp_t(u32, divisor, 2, 256); + } apple_qspi_c_write(qspi, APPLE_QSPI_C_CLKDIV, divisor - 1); return 0; } @@ -452,10 +501,56 @@ static bool apple_qspi_poll_complete(struct apple_qspi *qspi) try_wait_for_completion(&qspi->done); } +static bool apple_qspi_wait_complete_tick(struct apple_qspi *qspi) +{ + unsigned long deadline = jiffies + + msecs_to_jiffies(APPLE_QSPI_TIMEOUT_MS); + unsigned long flags; + u32 p_status, c_status; + + do { + if (try_wait_for_completion(&qspi->done)) + return true; + + /* + * TX-only commands do not reliably raise the wired interrupt on + * T8140, although the completion bit is latched in the controller. + * Poll both status banks as a fallback and service the FIFOs so a + * page program larger than the TX FIFO can continue making progress. + */ + p_status = apple_qspi_p_read(qspi, APPLE_QSPI_P_IRQ_STATUS); + c_status = apple_qspi_c_read(qspi, APPLE_QSPI_C_IRQ_STATUS); + if (p_status) + apple_qspi_p_write(qspi, APPLE_QSPI_P_IRQ_STATUS, p_status); + if (c_status) + apple_qspi_c_write(qspi, APPLE_QSPI_C_IRQ_STATUS, c_status); + + spin_lock_irqsave(&qspi->lock, flags); + if (qspi->active) { + apple_qspi_drain_rx(qspi); + apple_qspi_fill_tx(qspi); + } + spin_unlock_irqrestore(&qspi->lock, flags); + + if (c_status & APPLE_QSPI_C_IRQ_COMPLETE) + return true; + if (time_after_eq(jiffies, deadline)) + break; + + if (wait_for_completion_timeout(&qspi->done, 1)) + return true; + } while (time_before(jiffies, deadline)); + + return false; +} + static bool apple_qspi_wait_complete(struct apple_qspi *qspi) { bool finished; + if (!qspi->hw->us_poll) + return apple_qspi_wait_complete_tick(qspi); + /* * T8140 can latch completion without delivering the wired IRQ, and * the RX threshold IRQ does not reliably drain the FIFO. A jiffy @@ -480,7 +575,7 @@ static int apple_qspi_transfer_op(struct spi_mem *mem, u32 descriptor, mode; u32 p_status, c_status; int ret, retries; - bool complete; + bool complete, tx_stream; ret = apple_qspi_shape_op(op, &shape); if (ret) @@ -517,10 +612,13 @@ static int apple_qspi_transfer_op(struct spi_mem *mem, mode = apple_qspi_width_mode(shape.width); if (op->data.dir == SPI_MEM_DATA_IN) mode |= APPLE_QSPI_CMD_RX; - /* TX-only transfers contain the opcode, address and payload in one - * byte stream. RX transfers keep the read header separate from data. + /* On tx_single_stream controllers, TX-only transfers contain the + * opcode, address and payload in one byte stream. RX transfers keep the + * read header separate from data. */ - if (op->data.dir != SPI_MEM_DATA_IN) { + tx_stream = qspi->hw->tx_single_stream && + op->data.dir != SPI_MEM_DATA_IN; + if (tx_stream) { shape.shared_count = 0; shape.multi_count = 0; } @@ -548,7 +646,7 @@ static int apple_qspi_transfer_op(struct spi_mem *mem, qspi->active = true; apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, descriptor); apple_qspi_p_write(qspi, APPLE_QSPI_P_TXFIFO, - op->data.dir == SPI_MEM_DATA_IN ? op->data.nbytes : tx_len); + tx_stream ? tx_len : op->data.nbytes); apple_qspi_fill_tx(qspi); spin_unlock_irqrestore(&qspi->lock, flags); @@ -802,10 +900,20 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) return ret; } +/* Controllers without the NVRAM policy pass every supported request through. */ +static int apple_qspi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) +{ + struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); + + if (!qspi->hw->nvram_policy) + return apple_qspi_transfer_op(mem, op); + return apple_qspi_exec_op(mem, op); +} + static const struct spi_controller_mem_ops apple_qspi_mem_ops = { .adjust_op_size = apple_qspi_adjust_op_size, .supports_op = apple_qspi_supports_op, - .exec_op = apple_qspi_exec_op, + .exec_op = apple_qspi_mem_exec_op, }; /* Require the pinned partition, aperture and single physical chip select. */ @@ -858,6 +966,9 @@ static int apple_qspi_probe(struct platform_device *pdev) return -ENOMEM; qspi = spi_controller_get_devdata(ctlr); + qspi->hw = of_device_get_match_data(dev); + if (!qspi->hw) + return -ENODEV; qspi->nvram_write_candidate = of_device_is_compatible(dev->of_node, "apple,t8122-qspi"); if (qspi->nvram_write_candidate && !of_machine_is_compatible("apple,j613")) return -ENODEV; @@ -871,7 +982,9 @@ static int apple_qspi_probe(struct platform_device *pdev) qspi->nvram_write_candidate &= res->start == 0x2a1118000ULL && apple_qspi_nvram_layout(dev->of_node); - dev_info(dev, "NVRAM layout candidate: %u\n", qspi->nvram_write_candidate); + if (qspi->hw->nvram_policy) + dev_info(dev, "NVRAM layout candidate: %u\n", + qspi->nvram_write_candidate); qspi->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(qspi->base)) return PTR_ERR(qspi->base); @@ -905,10 +1018,16 @@ static int apple_qspi_probe(struct platform_device *pdev) qspi->tx_depth, qspi->rx_depth); apple_qspi_init_config(qspi); - /* Keep QE and every other flash configuration bit untouched. */ + /* + * Without multi_io, single-wire modes keep QE and every other flash + * configuration bit untouched. + */ ctlr->num_chipselect = 1; ctlr->max_native_cs = 1; ctlr->mode_bits = SPI_CPHA | SPI_CPOL; + if (qspi->hw->multi_io) + ctlr->mode_bits |= SPI_RX_DUAL | SPI_RX_QUAD | + SPI_TX_DUAL | SPI_TX_QUAD; ctlr->bits_per_word_mask = SPI_BPW_MASK(8); ctlr->mem_ops = &apple_qspi_mem_ops; @@ -922,9 +1041,14 @@ static int apple_qspi_probe(struct platform_device *pdev) runtime_ready = true; mutex_unlock(&apple_qspi_policy_lock); version = apple_qspi_p_read(qspi, APPLE_QSPI_P_VERSION); - dev_info(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u%s\n", - version, clk_get_rate(qspi->clk), qspi->tx_depth, - qspi->rx_depth, read_only ? ", read-only" : ""); + if (qspi->hw->quiet_probe) + dev_dbg(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u\n", + version, clk_get_rate(qspi->clk), qspi->tx_depth, + qspi->rx_depth); + else + dev_info(dev, "QSPIMC version %#x, %lu Hz clock, txq=%u rxq=%u%s\n", + version, clk_get_rate(qspi->clk), qspi->tx_depth, + qspi->rx_depth, read_only ? ", read-only" : ""); return 0; } @@ -984,9 +1108,9 @@ static struct attribute *apple_qspi_attrs[] = { ATTRIBUTE_GROUPS(apple_qspi); static const struct of_device_id apple_qspi_of_match[] = { - { .compatible = "apple,t8122-qspi" }, - { .compatible = "apple,t8132-qspi" }, - { .compatible = "apple,t8140-qspi" }, + { .compatible = "apple,t8122-qspi", .data = &apple_qspi_t8122_hw }, + { .compatible = "apple,t8132-qspi", .data = &apple_qspi_t8122_hw }, + { .compatible = "apple,t8140-qspi", .data = &apple_qspi_t8140_hw }, {} }; MODULE_DEVICE_TABLE(of, apple_qspi_of_match); From 57da8f5abc8524f5421e693fb5e972f0b02f5b23 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Sun, 11 Oct 2026 21:00:37 +0200 Subject: [PATCH 562/631] drm: apple: audio: limit the number of periods to what SIO can serve SIO has 64 descriptor slots, shared by all its channels, and its cyclic prep takes one slot per period. The PCM allowed any number of periods, so PipeWire's default of 256-frame periods in a 32768-frame buffer asked for 128 of them. sio_prep_dma_cyclic() then ran out of slots, snd_dmaengine_pcm_trigger() returned -ENOMEM, ALSA undid the START with a STOP (a stop_link round trip to DCP), and PipeWire retried about 100 times a second with no sound. Up to three DP audio outputs share one SIO on t600x and t602x, so allow at most 16 periods per stream, 48 slots for all three. Fixes: cb24343242a7 ("gpu: drm: apple: Add DCP audio driver") Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/audio.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index b2706c7ac3f823..cfdfd606cf129e 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -34,6 +34,13 @@ #define DCPAUD_ELEMENTS_MAXSIZE 16384 #define DCPAUD_PRODUCTATTRS_MAXSIZE 1024 +/* + * SIO takes one of its 64 descriptor slots per period, shared by all its + * channels, and up to three DP audio outputs use one SIO. A stream with + * more periods fails START with -ENOMEM. + */ +#define DCPAUD_PERIODS_MAX 16 + struct dcp_audio { struct device *dev; struct device *dcp_dev; @@ -78,7 +85,7 @@ static const struct snd_pcm_hardware dcp_pcm_hw = { .period_bytes_min = 4096, /* TODO */ .period_bytes_max = SIZE_MAX, .periods_min = 2, - .periods_max = UINT_MAX, + .periods_max = DCPAUD_PERIODS_MAX, }; /* @@ -104,7 +111,7 @@ static const struct snd_pcm_hardware dcp_pcm_hw_unplugged = { .period_bytes_min = 4096, .period_bytes_max = SIZE_MAX, .periods_min = 2, - .periods_max = UINT_MAX, + .periods_max = DCPAUD_PERIODS_MAX, }; static bool dcpaud_h17p(struct dcp_audio *dcpaud) @@ -329,7 +336,7 @@ static int dcp_pcm_open(struct snd_pcm_substream *substream) hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED; hw.periods_min = 2; - hw.periods_max = UINT_MAX; + hw.periods_max = DCPAUD_PERIODS_MAX; hw.period_bytes_min = 256; hw.period_bytes_max = SIZE_MAX; // TODO dma_get_max_seg_size(dma_dev); hw.buffer_bytes_max = SIZE_MAX; From bad1bc31bc1a5149d923b2bb83be43817a25db24 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:02 -0400 Subject: [PATCH 563/631] wifi, Bluetooth: mt7932: finish the merge of the J700 radio Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. The J700 radio follows aurora-wip, together with integration's changes that do not conflict with it. Remove the duplicate channel-policy and transport code that the replay left behind. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/bluetooth/mt7932_bt_pcie.c | 23 +-------- drivers/net/wireless/mediatek/mt7932/mt7932.h | 1 - .../net/wireless/mediatek/mt7932/startup.c | 51 ------------------- 3 files changed, 2 insertions(+), 73 deletions(-) diff --git a/drivers/bluetooth/mt7932_bt_pcie.c b/drivers/bluetooth/mt7932_bt_pcie.c index 8adff8e60dd020..f4a7e39232d7b6 100644 --- a/drivers/bluetooth/mt7932_bt_pcie.c +++ b/drivers/bluetooth/mt7932_bt_pcie.c @@ -1090,8 +1090,8 @@ static void bt7932_cleanup(struct bt7932 *bt, bool release_dma) } /* - * Bluetooth starts after the Wi-Fi function has initialized its firmware. - * Wait until mt7932-fullmac is bound to function 0, then link the + * Admission was only qualified with the Wi-Fi function already running its + * firmware. Wait until mt7932-fullmac is bound to function 0, then link the * two functions so that the driver core unbinds this one first. */ static int bt7932_wait_for_wifi(struct pci_dev *pdev) @@ -1109,21 +1109,6 @@ static int bt7932_wait_for_wifi(struct pci_dev *pdev) else if (!mt7932_fullmac_ready(wifi)) ret = dev_err_probe(&pdev->dev, -ENODEV, "the Wi-Fi function failed to initialize\n"); - * Admission was only qualified with the Wi-Fi function already running its - * firmware. Wait until mt7932-fullmac is bound to function 0, then link the - * two functions so that the driver core unbinds this one first. - */ -static int bt7932_wait_for_wifi(struct pci_dev *pdev) -{ - struct pci_dev *wifi = pci_get_slot(pdev->bus, PCI_DEVFN(0, 0)); - int ret = 0; - - if (!wifi) - return -ENODEV; - if (!device_is_bound(&wifi->dev) || - strcmp(dev_driver_string(&wifi->dev), "mt7932-fullmac")) - ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, - "waiting for the Wi-Fi function\n"); else if (!device_link_add(&pdev->dev, &wifi->dev, DL_FLAG_AUTOREMOVE_CONSUMER)) ret = -EINVAL; pci_dev_put(wifi); @@ -1150,10 +1135,6 @@ static int bt7932_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = bt7932_wait_for_wifi(pdev); if (ret) return ret; - /* The radio transport requires ASPM and clock PM off on this link. */ - ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); - if (ret) - dev_warn(&pdev->dev, "link power management left as found: %d\n", ret); /* Admission was qualified with ASPM and clock PM off on this link. */ ret = pci_disable_link_state(pdev, PCIE_LINK_STATE_ALL); if (ret) diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index 11efdce6b7a5b4..aae173b97cc134 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -111,7 +111,6 @@ struct mt7932 { struct work_struct startup_work; struct mutex command_mutex; bool startup_started, rf_ready, reg_pending, policy_failed, reg_retryable; - bool ready; bool interface_registered, interface_up; u32 reg_generation, reg_attempted; struct mt7932_reg_snapshot reg_desired; diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index fda897e941cbc2..679c9193f85bc5 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -263,44 +263,6 @@ static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *poli return; } wiphy_lock(m->wiphy); - for (i = 0; i < 17; i++) { - struct ieee80211_channel *channel = mt_channel(m, i); - bool forbidden = !mt7932_policy_permits(policy, channel->hw_value); - - __assign_bit(i, m->policy_disabled, forbidden); - if (forbidden) - channel->flags |= IEEE80211_CHAN_DISABLED; - } - wiphy_unlock(m->wiphy); - rtnl_unlock(); -} - -/* Retry only a completed file lookup that failed before any policy SET. - * Consume the flag under the same lock as the worker publication, so repeated - * userspace requests cannot continually invalidate an in-flight attempt. - */ -void mt_retry_missing_policy(struct mt7932 *m) -{ - unsigned long flags; - - spin_lock_irqsave(&m->response_lock, flags); - if (m->reg_retryable && m->reg_pending && m->interface_registered && - !m->stopping && !m->policy_failed && !m->link_failed && !m->cal_state.error) { - m->reg_retryable = false; - m->reg_generation++; - schedule_work(&m->startup_work); - } - spin_unlock_irqrestore(&m->response_lock, flags); -/* Keep cfg80211's channel list in step with the channels the country - * package leaves out of the firmware domain. cfg80211 recomputes the flags - * on its next regulatory change, which runs this again. - */ -static void mt_policy_disable(struct mt7932 *m, const struct mt7932_policy *policy) -{ - unsigned int i; - - rtnl_lock(); - wiphy_lock(m->wiphy); for (i = 0; i < MT7932_CHANNELS; i++) { struct ieee80211_channel *channel = mt_channel(m, i); bool forbidden = !mt7932_policy_permits(policy, channel->hw_value); @@ -401,16 +363,6 @@ static void mt_startup_work(struct work_struct *work) ret = mt_request_input(m, &file, path); } if (!ret) { - ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); - if (ret) - dev_err_ratelimited(&m->pdev->dev, "local input %s has invalid policy: %d\n", path, ret); - } - if (!ret) { - mt7932_policy_filter(®, &policy); - mt_policy_disable(m, &policy, generation); - if (!ret) { - mt7932_policy_filter(®, &policy); - mt_policy_disable(m, &policy); ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); if (!ret) { mt7932_policy_filter(®, &policy); @@ -486,9 +438,6 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request reg.domain[4] = 1; /* 2 GHz remains 20 MHz. */ reg.domain[5] = mt7932_domain_5g_bw(); for (i = 0; i < MT7932_CHANNELS; i++) { - struct ieee80211_channel *channel = i < MT7932_CHANNELS_2G ? - &m->channels[i] : &m->channels5[i - MT7932_CHANNELS_2G]; - for (i = 0; i < 17; i++) { struct ieee80211_channel *channel = mt_channel(m, i); u32 flags = channel->flags; From 29666f0aa13ab5d2eb0a66be4eac8036c489b507 Mon Sep 17 00:00:00 2001 From: Thomas Glanzmann Date: Mon, 12 Oct 2026 12:45:13 +0200 Subject: [PATCH 564/631] media: apple: avd: allow more than 32 capture buffers per context The capture queue never sets max_num_buffers, so vb2 limits every context to VB2_MAX_FRAME (32) buffers. Nothing in the driver needs that limit: reference frames are looked up by timestamp (vb2_find_buffer()) and handed to the hardware by DMA address, never by buffer index. 32 buffers leave too little room for userspace to decode ahead. With a 16-frame DPB, four frame threads and the frames held for display, mpv's decoder queue (--vd-queue) gets only a few frames before VIDIOC_CREATE_BUFS fails with ENOBUFS, and the FFmpeg V4L2 request hwaccel then falls back to software decoding. Decoding ahead is needed because the t8103 AVD decodes 4K59.94 H.264 at only ~41 fps during a 125-156 Mbit/s burst, so playback drops frames unless the decoder has built up a lead of 60-90 frames beforehand. Add a max_capture_buffers module parameter, default 128. Buffers are still allocated on demand by userspace, so this only raises the limit. Signed-off-by: Thomas Glanzmann --- drivers/media/platform/apple/avd/avd-drv.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 8f7a9d99322b5f..270ffb69965960 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -429,6 +429,17 @@ static void avd_device_run(void *priv) avd_job_finish(ctx, result); } +/* + * Capture buffers are found by timestamp and handed to the hardware by DMA + * address, so nothing limits them to VB2_MAX_FRAME. A larger pool lets + * userspace decode ahead to ride out bitrate spikes the hardware can't + * decode in real time (e.g. 150+ Mbit/s 4K60 H.264). + */ +static unsigned int max_capture_buffers = 128; +module_param(max_capture_buffers, uint, 0444); +MODULE_PARM_DESC(max_capture_buffers, + "Maximum number of capture buffers per context (default 128)"); + static int avd_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) { @@ -466,6 +477,7 @@ static int avd_queue_init(void *priv, struct vb2_queue *src_vq, dst_vq->lock = &ctx->dev->vdev_lock; dst_vq->dev = ctx->dev->v4l2_dev.dev; dst_vq->allow_cache_hints = true; + dst_vq->max_num_buffers = max(max_capture_buffers, VB2_MAX_FRAME); return vb2_queue_init(dst_vq); } From ec942916c930db441534f94d09c43494d263333c Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:02 -0400 Subject: [PATCH 565/631] drm: drop the isolated T8140 graphics copies The main asahi and apple drivers now serve T8140 and T8142 themselves, so the separate asahi-neo, apple-neo and drm_neo copies introduced for the isolated T8140 profile are no longer used. Remove them, together with their Kconfig, Makefile, Rust module and UAPI header entries, and the test of the isolated display registration. Assisted-by: LLM Signed-off-by: Diljit Singh --- drivers/gpu/drm/Kconfig | 2 - drivers/gpu/drm/Makefile | 2 - drivers/gpu/drm/apple-neo/Kconfig | 43 - drivers/gpu/drm/apple-neo/Makefile | 23 - drivers/gpu/drm/apple-neo/afk-service-test.c | 142 - drivers/gpu/drm/apple-neo/afk.c | 1398 ------ drivers/gpu/drm/apple-neo/afk.h | 347 -- drivers/gpu/drm/apple-neo/afk_test.c | 44 - drivers/gpu/drm/apple-neo/apple_drv.c | 988 ---- drivers/gpu/drm/apple-neo/audio.c | 815 ---- drivers/gpu/drm/apple-neo/audio.h | 20 - drivers/gpu/drm/apple-neo/av.c | 434 -- drivers/gpu/drm/apple-neo/av.h | 17 - drivers/gpu/drm/apple-neo/connector.c | 156 - drivers/gpu/drm/apple-neo/connector.h | 64 - drivers/gpu/drm/apple-neo/dcp-internal.h | 475 -- drivers/gpu/drm/apple-neo/dcp-link.h | 95 - drivers/gpu/drm/apple-neo/dcp.c | 3986 ----------------- drivers/gpu/drm/apple-neo/dcp.h | 108 - drivers/gpu/drm/apple-neo/dcp_backlight.c | 476 -- drivers/gpu/drm/apple-neo/dcp_backlight.h | 233 - .../gpu/drm/apple-neo/dcp_backlight_test.c | 379 -- drivers/gpu/drm/apple-neo/dcp_trace.c | 3 - drivers/gpu/drm/apple-neo/dptxep.c | 878 ---- drivers/gpu/drm/apple-neo/dptxep.h | 81 - drivers/gpu/drm/apple-neo/epic/dpavservep.c | 302 -- drivers/gpu/drm/apple-neo/epic/dpavservep.h | 26 - drivers/gpu/drm/apple-neo/hdmi-codec-chmap.h | 123 - drivers/gpu/drm/apple-neo/ibootep.c | 29 - drivers/gpu/drm/apple-neo/iomfb-state-test.c | 127 - drivers/gpu/drm/apple-neo/iomfb-state.h | 110 - drivers/gpu/drm/apple-neo/iomfb.c | 1588 ------- drivers/gpu/drm/apple-neo/iomfb.h | 408 -- drivers/gpu/drm/apple-neo/iomfb_h17p.c | 740 --- drivers/gpu/drm/apple-neo/iomfb_h17p.h | 132 - drivers/gpu/drm/apple-neo/iomfb_h17p_test.c | 154 - drivers/gpu/drm/apple-neo/iomfb_internal.h | 114 - drivers/gpu/drm/apple-neo/iomfb_plane.h | 109 - drivers/gpu/drm/apple-neo/iomfb_template.c | 2605 ----------- drivers/gpu/drm/apple-neo/iomfb_template.h | 285 -- drivers/gpu/drm/apple-neo/iomfb_v12_3.c | 109 - drivers/gpu/drm/apple-neo/iomfb_v12_3.h | 17 - drivers/gpu/drm/apple-neo/iomfb_v13_3.c | 114 - drivers/gpu/drm/apple-neo/iomfb_v13_3.h | 17 - drivers/gpu/drm/apple-neo/parser.c | 1203 ----- drivers/gpu/drm/apple-neo/parser.h | 142 - drivers/gpu/drm/apple-neo/plane.c | 561 --- drivers/gpu/drm/apple-neo/plane.h | 42 - drivers/gpu/drm/apple-neo/systemep.c | 150 - drivers/gpu/drm/apple-neo/trace.c | 9 - drivers/gpu/drm/apple-neo/trace.h | 629 --- drivers/gpu/drm/apple-neo/version_utils.h | 15 - drivers/gpu/drm/asahi-neo/Kconfig | 14 - drivers/gpu/drm/asahi-neo/Makefile | 3 - drivers/gpu/drm/asahi-neo/alloc.rs | 1087 ----- drivers/gpu/drm/asahi-neo/asahi_core.rs | 93 - drivers/gpu/drm/asahi-neo/buffer.rs | 809 ---- drivers/gpu/drm/asahi-neo/channel.rs | 720 --- drivers/gpu/drm/asahi-neo/cleanup.rs | 118 - drivers/gpu/drm/asahi-neo/crashdump.rs | 307 -- drivers/gpu/drm/asahi-neo/debug.rs | 142 - drivers/gpu/drm/asahi-neo/driver.rs | 289 -- drivers/gpu/drm/asahi-neo/event.rs | 257 -- drivers/gpu/drm/asahi-neo/file.rs | 1332 ------ drivers/gpu/drm/asahi-neo/float.rs | 388 -- drivers/gpu/drm/asahi-neo/fw/buffer.rs | 184 - drivers/gpu/drm/asahi-neo/fw/channels.rs | 443 -- drivers/gpu/drm/asahi-neo/fw/compute.rs | 120 - drivers/gpu/drm/asahi-neo/fw/event.rs | 104 - drivers/gpu/drm/asahi-neo/fw/fragment.rs | 296 -- drivers/gpu/drm/asahi-neo/fw/initdata.rs | 1352 ------ drivers/gpu/drm/asahi-neo/fw/job.rs | 160 - drivers/gpu/drm/asahi-neo/fw/microseq.rs | 412 -- drivers/gpu/drm/asahi-neo/fw/mod.rs | 15 - drivers/gpu/drm/asahi-neo/fw/types.rs | 224 - drivers/gpu/drm/asahi-neo/fw/vertex.rs | 193 - drivers/gpu/drm/asahi-neo/fw/workqueue.rs | 187 - drivers/gpu/drm/asahi-neo/g17/buffer.rs | 885 ---- .../gpu/drm/asahi-neo/g17/buffer/target.rs | 297 -- drivers/gpu/drm/asahi-neo/g17/channel.rs | 219 - drivers/gpu/drm/asahi-neo/g17/command.rs | 447 -- drivers/gpu/drm/asahi-neo/g17/completion.rs | 520 --- .../asahi-neo/g17/completion/publication.rs | 209 - drivers/gpu/drm/asahi-neo/g17/context.rs | 398 -- .../gpu/drm/asahi-neo/g17/context/identity.rs | 71 - drivers/gpu/drm/asahi-neo/g17/dependency.rs | 83 - drivers/gpu/drm/asahi-neo/g17/event.rs | 121 - drivers/gpu/drm/asahi-neo/g17/feed.rs | 489 -- drivers/gpu/drm/asahi-neo/g17/fence.rs | 257 -- drivers/gpu/drm/asahi-neo/g17/freelist.rs | 727 --- drivers/gpu/drm/asahi-neo/g17/fw/buffer.rs | 254 -- drivers/gpu/drm/asahi-neo/g17/fw/channels.rs | 527 --- drivers/gpu/drm/asahi-neo/g17/fw/compute.rs | 442 -- drivers/gpu/drm/asahi-neo/g17/fw/context.rs | 72 - drivers/gpu/drm/asahi-neo/g17/fw/freelist.rs | 124 - drivers/gpu/drm/asahi-neo/g17/fw/initdata.rs | 2369 ---------- drivers/gpu/drm/asahi-neo/g17/fw/kick.rs | 549 --- drivers/gpu/drm/asahi-neo/g17/fw/mod.rs | 109 - drivers/gpu/drm/asahi-neo/g17/fw/qos.rs | 34 - drivers/gpu/drm/asahi-neo/g17/fw/queue.rs | 611 --- drivers/gpu/drm/asahi-neo/g17/fw/render.rs | 1329 ------ .../drm/asahi-neo/g17/fw/render/storage.rs | 178 - drivers/gpu/drm/asahi-neo/g17/hazard.rs | 120 - drivers/gpu/drm/asahi-neo/g17/ids.rs | 60 - drivers/gpu/drm/asahi-neo/g17/initdata.rs | 1451 ------ drivers/gpu/drm/asahi-neo/g17/job.rs | 396 -- drivers/gpu/drm/asahi-neo/g17/kick.rs | 355 -- drivers/gpu/drm/asahi-neo/g17/mod.rs | 1233 ----- drivers/gpu/drm/asahi-neo/g17/object.rs | 579 --- drivers/gpu/drm/asahi-neo/g17/power.rs | 139 - drivers/gpu/drm/asahi-neo/g17/preparation.rs | 186 - drivers/gpu/drm/asahi-neo/g17/qos.rs | 249 - .../gpu/drm/asahi-neo/g17/queue/admission.rs | 541 --- .../gpu/drm/asahi-neo/g17/queue/compute.rs | 1923 -------- .../drm/asahi-neo/g17/queue/compute/exit.rs | 212 - .../asahi-neo/g17/queue/compute/retirement.rs | 242 - drivers/gpu/drm/asahi-neo/g17/queue/mod.rs | 407 -- drivers/gpu/drm/asahi-neo/g17/queue/render.rs | 1595 ------- .../g17/queue/render/dependencies.rs | 109 - .../drm/asahi-neo/g17/queue/render/memory.rs | 723 --- .../asahi-neo/g17/queue/render/observation.rs | 85 - .../drm/asahi-neo/g17/queue/render/parked.rs | 243 - .../asahi-neo/g17/queue/render/publication.rs | 165 - .../asahi-neo/g17/queue/render/retirement.rs | 348 -- .../drm/asahi-neo/g17/queue/render/warm.rs | 49 - drivers/gpu/drm/asahi-neo/g17/recovery.rs | 301 -- .../drm/asahi-neo/g17/recovery/attribution.rs | 110 - .../gpu/drm/asahi-neo/g17/recovery/slot.rs | 74 - drivers/gpu/drm/asahi-neo/g17/runtime.rs | 1181 ----- .../gpu/drm/asahi-neo/g17/runtime/backend.rs | 190 - .../asahi-neo/g17/runtime/compute_control.rs | 96 - .../drm/asahi-neo/g17/runtime/compute_exit.rs | 177 - .../asahi-neo/g17/runtime/compute_replay.rs | 115 - .../gpu/drm/asahi-neo/g17/runtime/events.rs | 475 -- drivers/gpu/drm/asahi-neo/g17/runtime/feed.rs | 33 - .../gpu/drm/asahi-neo/g17/runtime/render.rs | 744 --- .../g17/runtime/render/dependencies.rs | 34 - .../drm/asahi-neo/g17/runtime/render/exit.rs | 39 - .../drm/asahi-neo/g17/runtime/render/grow.rs | 323 -- .../asahi-neo/g17/runtime/render/observe.rs | 235 - .../drm/asahi-neo/g17/runtime/render/pools.rs | 241 - .../asahi-neo/g17/runtime/render/prepare.rs | 717 --- .../asahi-neo/g17/runtime/render/recovery.rs | 37 - .../asahi-neo/g17/runtime/render/settle.rs | 235 - .../asahi-neo/g17/runtime/render/submit.rs | 324 -- .../asahi-neo/g17/runtime/render/teardown.rs | 164 - .../gpu/drm/asahi-neo/g17/runtime/teardown.rs | 199 - drivers/gpu/drm/asahi-neo/g17/status.rs | 139 - drivers/gpu/drm/asahi-neo/g17/teardown.rs | 229 - drivers/gpu/drm/asahi-neo/g17/timeout.rs | 60 - drivers/gpu/drm/asahi-neo/g17/validation.rs | 145 - drivers/gpu/drm/asahi-neo/gem.rs | 315 -- drivers/gpu/drm/asahi-neo/gpu.rs | 1733 ------- drivers/gpu/drm/asahi-neo/hw/mod.rs | 660 --- drivers/gpu/drm/asahi-neo/hw/t600x.rs | 162 - drivers/gpu/drm/asahi-neo/hw/t602x.rs | 180 - drivers/gpu/drm/asahi-neo/hw/t8103.rs | 93 - drivers/gpu/drm/asahi-neo/hw/t8112.rs | 106 - drivers/gpu/drm/asahi-neo/hw/t8140.rs | 403 -- drivers/gpu/drm/asahi-neo/initdata.rs | 1025 ----- drivers/gpu/drm/asahi-neo/mem.rs | 211 - drivers/gpu/drm/asahi-neo/microseq.rs | 63 - drivers/gpu/drm/asahi-neo/mmu.rs | 2675 ----------- drivers/gpu/drm/asahi-neo/mmu/bind.rs | 224 - drivers/gpu/drm/asahi-neo/mmu/context.rs | 287 -- drivers/gpu/drm/asahi-neo/mmu/lifetime.rs | 634 --- drivers/gpu/drm/asahi-neo/mmu/residency.rs | 647 --- drivers/gpu/drm/asahi-neo/mmu/shared.rs | 134 - drivers/gpu/drm/asahi-neo/object.rs | 733 --- drivers/gpu/drm/asahi-neo/pgtable.rs | 892 ---- drivers/gpu/drm/asahi-neo/queue/common.rs | 42 - drivers/gpu/drm/asahi-neo/queue/compute.rs | 385 -- drivers/gpu/drm/asahi-neo/queue/mod.rs | 958 ---- drivers/gpu/drm/asahi-neo/queue/render.rs | 1400 ------ drivers/gpu/drm/asahi-neo/regs.rs | 742 --- drivers/gpu/drm/asahi-neo/slotalloc.rs | 323 -- drivers/gpu/drm/asahi-neo/stats.rs | 78 - drivers/gpu/drm/asahi-neo/sysfs.c | 131 - drivers/gpu/drm/asahi-neo/sysfs.h | 12 - drivers/gpu/drm/asahi-neo/sysfs_exports.rs | 37 - drivers/gpu/drm/asahi-neo/util.rs | 142 - drivers/gpu/drm/asahi-neo/vm/mod.rs | 5 - drivers/gpu/drm/asahi-neo/vm/range.rs | 125 - drivers/gpu/drm/asahi-neo/workqueue.rs | 1019 ----- include/uapi/drm/asahi_neo_drm.h | 1524 ------- rust/kernel/drm_neo/device.rs | 338 -- rust/kernel/drm_neo/driver.rs | 188 - rust/kernel/drm_neo/file.rs | 114 - rust/kernel/drm_neo/gem/mod.rs | 497 -- rust/kernel/drm_neo/gem/shmem.rs | 584 --- rust/kernel/drm_neo/gpuvm.rs | 1375 ------ rust/kernel/drm_neo/ioctl.rs | 167 - rust/kernel/drm_neo/mm.rs | 344 -- rust/kernel/drm_neo/mod.rs | 24 - rust/kernel/drm_neo/sched.rs | 549 --- rust/kernel/drm_neo/syncobj.rs | 83 - rust/kernel/lib.rs | 2 - rust/uapi/uapi_helper.h | 1 - tools/asahi/neo-registration-tests.py | 121 - 199 files changed, 82153 deletions(-) delete mode 100644 drivers/gpu/drm/apple-neo/Kconfig delete mode 100644 drivers/gpu/drm/apple-neo/Makefile delete mode 100644 drivers/gpu/drm/apple-neo/afk-service-test.c delete mode 100644 drivers/gpu/drm/apple-neo/afk.c delete mode 100644 drivers/gpu/drm/apple-neo/afk.h delete mode 100644 drivers/gpu/drm/apple-neo/afk_test.c delete mode 100644 drivers/gpu/drm/apple-neo/apple_drv.c delete mode 100644 drivers/gpu/drm/apple-neo/audio.c delete mode 100644 drivers/gpu/drm/apple-neo/audio.h delete mode 100644 drivers/gpu/drm/apple-neo/av.c delete mode 100644 drivers/gpu/drm/apple-neo/av.h delete mode 100644 drivers/gpu/drm/apple-neo/connector.c delete mode 100644 drivers/gpu/drm/apple-neo/connector.h delete mode 100644 drivers/gpu/drm/apple-neo/dcp-internal.h delete mode 100644 drivers/gpu/drm/apple-neo/dcp-link.h delete mode 100644 drivers/gpu/drm/apple-neo/dcp.c delete mode 100644 drivers/gpu/drm/apple-neo/dcp.h delete mode 100644 drivers/gpu/drm/apple-neo/dcp_backlight.c delete mode 100644 drivers/gpu/drm/apple-neo/dcp_backlight.h delete mode 100644 drivers/gpu/drm/apple-neo/dcp_backlight_test.c delete mode 100644 drivers/gpu/drm/apple-neo/dcp_trace.c delete mode 100644 drivers/gpu/drm/apple-neo/dptxep.c delete mode 100644 drivers/gpu/drm/apple-neo/dptxep.h delete mode 100644 drivers/gpu/drm/apple-neo/epic/dpavservep.c delete mode 100644 drivers/gpu/drm/apple-neo/epic/dpavservep.h delete mode 100644 drivers/gpu/drm/apple-neo/hdmi-codec-chmap.h delete mode 100644 drivers/gpu/drm/apple-neo/ibootep.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb-state-test.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb-state.h delete mode 100644 drivers/gpu/drm/apple-neo/iomfb.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb.h delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_h17p.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_h17p.h delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_h17p_test.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_internal.h delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_plane.h delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_template.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_template.h delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_v12_3.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_v12_3.h delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_v13_3.c delete mode 100644 drivers/gpu/drm/apple-neo/iomfb_v13_3.h delete mode 100644 drivers/gpu/drm/apple-neo/parser.c delete mode 100644 drivers/gpu/drm/apple-neo/parser.h delete mode 100644 drivers/gpu/drm/apple-neo/plane.c delete mode 100644 drivers/gpu/drm/apple-neo/plane.h delete mode 100644 drivers/gpu/drm/apple-neo/systemep.c delete mode 100644 drivers/gpu/drm/apple-neo/trace.c delete mode 100644 drivers/gpu/drm/apple-neo/trace.h delete mode 100644 drivers/gpu/drm/apple-neo/version_utils.h delete mode 100644 drivers/gpu/drm/asahi-neo/Kconfig delete mode 100644 drivers/gpu/drm/asahi-neo/Makefile delete mode 100644 drivers/gpu/drm/asahi-neo/alloc.rs delete mode 100644 drivers/gpu/drm/asahi-neo/asahi_core.rs delete mode 100644 drivers/gpu/drm/asahi-neo/buffer.rs delete mode 100644 drivers/gpu/drm/asahi-neo/channel.rs delete mode 100644 drivers/gpu/drm/asahi-neo/cleanup.rs delete mode 100644 drivers/gpu/drm/asahi-neo/crashdump.rs delete mode 100644 drivers/gpu/drm/asahi-neo/debug.rs delete mode 100644 drivers/gpu/drm/asahi-neo/driver.rs delete mode 100644 drivers/gpu/drm/asahi-neo/event.rs delete mode 100644 drivers/gpu/drm/asahi-neo/file.rs delete mode 100644 drivers/gpu/drm/asahi-neo/float.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/buffer.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/channels.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/compute.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/event.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/fragment.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/initdata.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/job.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/microseq.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/mod.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/types.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/vertex.rs delete mode 100644 drivers/gpu/drm/asahi-neo/fw/workqueue.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/buffer.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/buffer/target.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/channel.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/command.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/completion.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/completion/publication.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/context.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/context/identity.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/dependency.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/event.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/feed.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fence.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/freelist.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/buffer.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/channels.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/compute.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/context.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/freelist.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/initdata.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/kick.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/mod.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/qos.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/queue.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/render.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/fw/render/storage.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/hazard.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/ids.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/initdata.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/job.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/kick.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/mod.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/object.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/power.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/preparation.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/qos.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/admission.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/compute.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/compute/exit.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/compute/retirement.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/mod.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render/dependencies.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render/memory.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render/observation.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render/parked.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render/publication.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render/retirement.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/queue/render/warm.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/recovery.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/recovery/attribution.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/recovery/slot.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/backend.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/compute_control.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/compute_exit.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/compute_replay.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/events.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/feed.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/dependencies.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/exit.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/grow.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/observe.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/pools.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/prepare.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/recovery.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/settle.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/submit.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/render/teardown.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/runtime/teardown.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/status.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/teardown.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/timeout.rs delete mode 100644 drivers/gpu/drm/asahi-neo/g17/validation.rs delete mode 100644 drivers/gpu/drm/asahi-neo/gem.rs delete mode 100644 drivers/gpu/drm/asahi-neo/gpu.rs delete mode 100644 drivers/gpu/drm/asahi-neo/hw/mod.rs delete mode 100644 drivers/gpu/drm/asahi-neo/hw/t600x.rs delete mode 100644 drivers/gpu/drm/asahi-neo/hw/t602x.rs delete mode 100644 drivers/gpu/drm/asahi-neo/hw/t8103.rs delete mode 100644 drivers/gpu/drm/asahi-neo/hw/t8112.rs delete mode 100644 drivers/gpu/drm/asahi-neo/hw/t8140.rs delete mode 100644 drivers/gpu/drm/asahi-neo/initdata.rs delete mode 100644 drivers/gpu/drm/asahi-neo/mem.rs delete mode 100644 drivers/gpu/drm/asahi-neo/microseq.rs delete mode 100644 drivers/gpu/drm/asahi-neo/mmu.rs delete mode 100644 drivers/gpu/drm/asahi-neo/mmu/bind.rs delete mode 100644 drivers/gpu/drm/asahi-neo/mmu/context.rs delete mode 100644 drivers/gpu/drm/asahi-neo/mmu/lifetime.rs delete mode 100644 drivers/gpu/drm/asahi-neo/mmu/residency.rs delete mode 100644 drivers/gpu/drm/asahi-neo/mmu/shared.rs delete mode 100644 drivers/gpu/drm/asahi-neo/object.rs delete mode 100644 drivers/gpu/drm/asahi-neo/pgtable.rs delete mode 100644 drivers/gpu/drm/asahi-neo/queue/common.rs delete mode 100644 drivers/gpu/drm/asahi-neo/queue/compute.rs delete mode 100644 drivers/gpu/drm/asahi-neo/queue/mod.rs delete mode 100644 drivers/gpu/drm/asahi-neo/queue/render.rs delete mode 100644 drivers/gpu/drm/asahi-neo/regs.rs delete mode 100644 drivers/gpu/drm/asahi-neo/slotalloc.rs delete mode 100644 drivers/gpu/drm/asahi-neo/stats.rs delete mode 100644 drivers/gpu/drm/asahi-neo/sysfs.c delete mode 100644 drivers/gpu/drm/asahi-neo/sysfs.h delete mode 100644 drivers/gpu/drm/asahi-neo/sysfs_exports.rs delete mode 100644 drivers/gpu/drm/asahi-neo/util.rs delete mode 100644 drivers/gpu/drm/asahi-neo/vm/mod.rs delete mode 100644 drivers/gpu/drm/asahi-neo/vm/range.rs delete mode 100644 drivers/gpu/drm/asahi-neo/workqueue.rs delete mode 100644 include/uapi/drm/asahi_neo_drm.h delete mode 100644 rust/kernel/drm_neo/device.rs delete mode 100644 rust/kernel/drm_neo/driver.rs delete mode 100644 rust/kernel/drm_neo/file.rs delete mode 100644 rust/kernel/drm_neo/gem/mod.rs delete mode 100644 rust/kernel/drm_neo/gem/shmem.rs delete mode 100644 rust/kernel/drm_neo/gpuvm.rs delete mode 100644 rust/kernel/drm_neo/ioctl.rs delete mode 100644 rust/kernel/drm_neo/mm.rs delete mode 100644 rust/kernel/drm_neo/mod.rs delete mode 100644 rust/kernel/drm_neo/sched.rs delete mode 100644 rust/kernel/drm_neo/syncobj.rs delete mode 100644 tools/asahi/neo-registration-tests.py diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 781765bbdcab3f..325223d75e5c80 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -368,6 +368,4 @@ source "drivers/gpu/drm/apple/Kconfig.tests" config DRM_PANEL_ORIENTATION_QUIRKS tristate -source "drivers/gpu/drm/asahi-neo/Kconfig" -source "drivers/gpu/drm/apple-neo/Kconfig" diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index d2c2eef1a276b3..846f2da7c81a08 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -263,6 +263,4 @@ quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) $(obj)/%.hdrtest: $(src)/%.h FORCE $(call if_changed_dep,hdrtest) -obj-$(CONFIG_DRM_ASAHI_NEO) += asahi-neo/ -obj-$(CONFIG_DRM_APPLE_NEO) += apple-neo/ diff --git a/drivers/gpu/drm/apple-neo/Kconfig b/drivers/gpu/drm/apple-neo/Kconfig deleted file mode 100644 index e24927ca4648c4..00000000000000 --- a/drivers/gpu/drm/apple-neo/Kconfig +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only OR MIT -config DRM_APPLE_NEO - tristate "DRM Support for the Apple T8140 display controller" - depends on DRM && OF && ARM64 - depends on ARCH_APPLE || COMPILE_TEST - depends on APPLE_RTKIT - depends on TYPEC - depends on OF_ADDRESS - select APPLE_PMP_REPORT - select DRM_CLIENT_SELECTION - select DRM_KMS_HELPER - select DRM_KMS_DMA_HELPER - select DRM_GEM_DMA_HELPER - select VIDEOMODE_HELPERS - depends on BACKLIGHT_CLASS_DEVICE - select MULTIPLEXER - help - Say Y for native display support on the MacBook Neo. - -config DRM_APPLE_NEO_AUDIO - bool "DisplayPort/HDMI Audio support" - default y - depends on DRM_APPLE_NEO - depends on SND - select SND_PCM - select SND_DMAENGINE_PCM - -config DRM_APPLE_NEO_DEBUG - bool "Enable additional driver debugging" - depends on DRM_APPLE_NEO - depends on EXPERT # only for developers - -config DRM_APPLE_NEO_KUNIT_TEST - tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS - depends on KUNIT - default KUNIT_ALL_TESTS - help - Build tests for the Apple display driver's AFK ring parser and - backlight policy (takeover, DPMS, suspend and present completion). - The tests cover the header strides used by older firmware and - larger rings, as well as malformed ring size combinations. - This option is intended for developer test kernels. - Also test the Type-C modeset, hotplug and service ownership state. diff --git a/drivers/gpu/drm/apple-neo/Makefile b/drivers/gpu/drm/apple-neo/Makefile deleted file mode 100644 index 89e62139560007..00000000000000 --- a/drivers/gpu/drm/apple-neo/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only OR MIT - -CFLAGS_trace.o = -I$(src) - -appledrm_neo-y := apple_drv.o -appledrm_neo-y += afk.o dcp.o dcp_backlight.o dptxep.o iomfb.o parser.o systemep.o -appledrm_neo-$(CONFIG_DRM_APPLE_NEO_AUDIO) += audio.o -appledrm_neo-$(CONFIG_DRM_APPLE_NEO_AUDIO) += av.o -appledrm_neo-y += connector.o -appledrm_neo-y += ibootep.o -appledrm_neo-y += iomfb_v12_3.o -appledrm_neo-y += iomfb_v13_3.o -appledrm_neo-y += iomfb_h17p.o -appledrm_neo-$(CONFIG_DRM_APPLE_NEO_KUNIT_TEST) += iomfb_h17p_test.o -appledrm_neo-y += epic/dpavservep.o -appledrm_neo-y += plane.o - - -appledrm_neo-$(CONFIG_TRACING) += trace.o - -obj-$(CONFIG_DRM_APPLE_NEO) += appledrm_neo.o -obj-$(CONFIG_DRM_APPLE_NEO_KUNIT_TEST) += afk_test.o -obj-$(CONFIG_DRM_APPLE_NEO_KUNIT_TEST) += dcp_backlight_test.o diff --git a/drivers/gpu/drm/apple-neo/afk-service-test.c b/drivers/gpu/drm/apple-neo/afk-service-test.c deleted file mode 100644 index 65345b991d2e4f..00000000000000 --- a/drivers/gpu/drm/apple-neo/afk-service-test.c +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#include -#include - -#include "afk.h" - -static const struct neo_apple_epic_service_ops reusable_ops = { - .name = "reusable", - .reusable = true, -}; - -static const struct neo_apple_epic_service_ops permanent_ops = { - .name = "permanent", -}; - -static int neo_afk_service_test_init(struct kunit *test) -{ - struct neo_apple_epic_service *service; - - service = kunit_kzalloc(test, sizeof(*service), GFP_KERNEL); - if (!service) - return -ENOMEM; - spin_lock_init(&service->lock); - test->priv = service; - return 0; -} - -static void neo_afk_test_retire(struct neo_apple_epic_service *service) -{ - unsigned long flags; - - spin_lock_irqsave(&service->lock, flags); - service->enabled = false; - service->torndown = true; - spin_unlock_irqrestore(&service->lock, flags); -} - -static void neo_afk_service_many_generations_test(struct kunit *test) -{ - struct neo_apple_epic_service *service = test->priv; - unsigned int i; - - /* Repeated unplug/replug must not consume an unbounded slot budget. */ - for (i = 0; i < 4 * AFK_MAX_CHANNEL; i++) { - KUNIT_ASSERT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, i)); - KUNIT_EXPECT_EQ(test, service->channel, i); - KUNIT_EXPECT_TRUE(test, neo_afk_service_matches(service, i)); - neo_afk_test_retire(service); - } -} - -static void neo_afk_service_reader_pin_test(struct kunit *test) -{ - struct neo_apple_epic_service *service = test->priv; - - KUNIT_ASSERT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 1)); - KUNIT_ASSERT_PTR_EQ(test, neo_afk_service_get(service), service); /* owner */ - KUNIT_ASSERT_PTR_EQ(test, neo_afk_service_get(service), service); /* reader */ - neo_afk_test_retire(service); - neo_afk_service_put(service); /* owner detaches while EDID is being read */ - KUNIT_EXPECT_FALSE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); - KUNIT_EXPECT_EQ(test, service->channel, 1U); - KUNIT_EXPECT_PTR_EQ(test, neo_afk_service_get(service), NULL); - neo_afk_service_put(service); - KUNIT_EXPECT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); -} - -static void neo_afk_service_pending_reply_test(struct kunit *test) -{ - struct neo_apple_epic_service *service = test->priv; - - KUNIT_ASSERT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 1)); - set_bit(3, service->cmd_map); - neo_afk_test_retire(service); - KUNIT_EXPECT_FALSE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); - KUNIT_EXPECT_TRUE(test, neo_afk_service_matches(service, 1)); - KUNIT_EXPECT_FALSE(test, neo_afk_service_matches(service, 2)); - /* A timed-out command retains its DMA and slot until the late reply. */ - service->cmds[3].free_on_ack = true; - KUNIT_EXPECT_FALSE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); - clear_bit(3, service->cmd_map); - KUNIT_EXPECT_FALSE(test, neo_afk_service_matches(service, 1)); - KUNIT_EXPECT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); -} - -static void neo_afk_service_completed_reader_test(struct kunit *test) -{ - struct neo_apple_epic_service *service = test->priv; - - KUNIT_ASSERT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 1)); - KUNIT_ASSERT_PTR_EQ(test, neo_afk_service_get(service), service); - set_bit(0, service->cmd_map); - neo_afk_test_retire(service); - /* Reply arrived, but the waiting thread has not consumed it yet. */ - service->cmds[0].done = true; - KUNIT_EXPECT_FALSE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); - clear_bit(0, service->cmd_map); - KUNIT_EXPECT_FALSE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); - neo_afk_service_put(service); - KUNIT_EXPECT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); -} - -static void neo_afk_service_tag_sequence_test(struct kunit *test) -{ - struct neo_apple_epic_service *service = test->priv; - - KUNIT_ASSERT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 1)); - service->cmd_tag = 0xa9; - service->cookie = test; - neo_afk_test_retire(service); - KUNIT_EXPECT_TRUE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); - KUNIT_EXPECT_EQ(test, service->cmd_tag, (u8)0xa9); - KUNIT_EXPECT_PTR_EQ(test, service->cookie, NULL); -} - -static void neo_afk_service_unconverted_owner_test(struct kunit *test) -{ - struct neo_apple_epic_service *service = test->priv; - - KUNIT_ASSERT_TRUE(test, neo_afk_service_reinit(service, NULL, &permanent_ops, 1)); - neo_afk_test_retire(service); - KUNIT_EXPECT_FALSE(test, neo_afk_service_reinit(service, NULL, &reusable_ops, 2)); -} - -static struct kunit_case neo_afk_service_cases[] = { - KUNIT_CASE(neo_afk_service_many_generations_test), - KUNIT_CASE(neo_afk_service_reader_pin_test), - KUNIT_CASE(neo_afk_service_pending_reply_test), - KUNIT_CASE(neo_afk_service_completed_reader_test), - KUNIT_CASE(neo_afk_service_tag_sequence_test), - KUNIT_CASE(neo_afk_service_unconverted_owner_test), - { } -}; - -static struct kunit_suite neo_afk_service_suite = { - .name = "apple-afk-service-lifetime", - .init = neo_afk_service_test_init, - .test_cases = neo_afk_service_cases, -}; - -kunit_test_suite(neo_afk_service_suite); -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple-neo/afk.c b/drivers/gpu/drm/apple-neo/afk.c deleted file mode 100644 index 935fcbf483b1a8..00000000000000 --- a/drivers/gpu/drm/apple-neo/afk.c +++ /dev/null @@ -1,1398 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2022 Sven Peter */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "afk.h" -#include "trace.h" - -struct neo_afk_receive_message_work { - struct neo_apple_dcp_afkep *ep; - u64 message; - struct work_struct work; -}; - -#define RBEP_TYPE GENMASK(63, 48) - -enum rbep_msg_type { - RBEP_INIT = 0x80, - RBEP_INIT_ACK = 0xa0, - RBEP_GETBUF = 0x89, - RBEP_GETBUF_ACK = 0xa1, - RBEP_INIT_TX = 0x8a, - RBEP_INIT_RX = 0x8b, - RBEP_START = 0xa3, - RBEP_START_ACK = 0x86, - RBEP_SEND = 0xa2, - RBEP_RECV = 0x85, - RBEP_SHUTDOWN = 0xc0, - RBEP_SHUTDOWN_ACK = 0xc1, -}; - -#define BLOCK_SHIFT 6 - -#define GETBUF_SIZE GENMASK(31, 16) -#define GETBUF_TAG GENMASK(15, 0) -#define GETBUF_ACK_DVA GENMASK(47, 0) - -#define INITRB_OFFSET GENMASK(47, 32) -#define INITRB_SIZE GENMASK(31, 16) -#define INITRB_TAG GENMASK(15, 0) - -#define SEND_WPTR GENMASK(31, 0) - -static void neo_afk_send(struct neo_apple_dcp_afkep *ep, u64 message) -{ - neo_dcp_send_message(ep->neo_dcp, ep->endpoint, message); -} - -static void neo_afk_release_context(void *data) -{ - struct neo_apple_dcp_afkep *afkep = data; - - neo_afk_quiesce(afkep); - if (!afkep->neo_dcp->retain_dma) - kfree(afkep); -} - -struct neo_apple_dcp_afkep *neo_afk_init(struct neo_apple_dcp *neo_dcp, u32 endpoint, - const struct neo_apple_epic_service_ops *ops) -{ - struct neo_apple_dcp_afkep *afkep; - int ret, i; - - afkep = kzalloc_obj(*afkep); - if (!afkep) - return ERR_PTR(-ENOMEM); - - afkep->ops = ops; - afkep->neo_dcp = neo_dcp; - afkep->endpoint = endpoint; - afkep->wq = alloc_ordered_workqueue("apple-dcp-afkep%02x", - WQ_MEM_RECLAIM, endpoint); - if (!afkep->wq) { - ret = -ENOMEM; - goto out_free_afkep; - } - - init_completion(&afkep->started); - init_completion(&afkep->stopped); - spin_lock_init(&afkep->lock); - for (i = 0; i < AFK_MAX_CHANNEL; i++) - spin_lock_init(&afkep->services[i].lock); - ret = devm_add_action_or_reset(neo_dcp->dev, neo_afk_release_context, afkep); - if (ret) - return ERR_PTR(ret); - - return afkep; - -out_free_afkep: - kfree(afkep); - return ERR_PTR(ret); -} - -void neo_afk_quiesce(struct neo_apple_dcp_afkep *afkep) -{ - if (IS_ERR_OR_NULL(afkep)) - return; - if (afkep->wq) { - destroy_workqueue(afkep->wq); - afkep->wq = NULL; - } - debugfs_remove_recursive(afkep->debugfs_entry); - afkep->debugfs_entry = NULL; -} - -void neo_afk_shutdown(struct neo_apple_dcp_afkep *afkep) -{ - unsigned long flags; - - neo_afk_send(afkep, FIELD_PREP(RBEP_TYPE, RBEP_SHUTDOWN)); - int ret; - - ret = wait_for_completion_timeout(&afkep->stopped, msecs_to_jiffies(1000)); - if (ret <= 0) { - dev_err(afkep->neo_dcp->dev, "Timed out shutting down AFK endpoint %02x", afkep->endpoint); - } - - spin_lock_irqsave(&afkep->lock, flags); - afkep->stopping = true; - spin_unlock_irqrestore(&afkep->lock, flags); - neo_afk_quiesce(afkep); - if (afkep->rx_scratch) { - devm_kfree(afkep->neo_dcp->dev, afkep->rx_scratch); - afkep->rx_scratch = NULL; - } -} - -int neo_afk_start(struct neo_apple_dcp_afkep *ep) -{ - int ret; - - reinit_completion(&ep->started); - ret = apple_rtkit_start_ep(ep->neo_dcp->rtk, ep->endpoint); - if (ret) - return ret; - neo_afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_INIT)); - - ret = wait_for_completion_timeout(&ep->started, msecs_to_jiffies(1000)); - if (ret <= 0) - return -ETIMEDOUT; - else - return 0; -} - -static void neo_afk_getbuf(struct neo_apple_dcp_afkep *ep, u64 message) -{ - u32 size = FIELD_GET(GETBUF_SIZE, message) << BLOCK_SHIFT; - u16 tag = FIELD_GET(GETBUF_TAG, message); - u64 reply; - - trace_neo_afk_getbuf(ep, size, tag); - - if (ep->bfr) { - dev_err(ep->neo_dcp->dev, - "Got GETBUF message but buffer already exists\n"); - return; - } - - /* An unstopped adopted session owns this ring until the machine resets. */ - if (READ_ONCE(ep->neo_dcp->retain_dma)) - ep->bfr = dma_alloc_coherent(ep->neo_dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); - else - ep->bfr = dmam_alloc_coherent(ep->neo_dcp->dev, size, &ep->bfr_dma, GFP_KERNEL); - if (!ep->bfr) { - dev_err(ep->neo_dcp->dev, "Failed to allocate %d bytes buffer\n", - size); - return; - } - - ep->bfr_size = size; - ep->bfr_tag = tag; - - reply = FIELD_PREP(RBEP_TYPE, RBEP_GETBUF_ACK); - reply |= FIELD_PREP(GETBUF_ACK_DVA, ep->bfr_dma); - neo_afk_send(ep, reply); -} - -static void neo_afk_init_rxtx(struct neo_apple_dcp_afkep *ep, u64 message, - struct neo_afk_ringbuffer *bfr) -{ - u32 base = FIELD_GET(INITRB_OFFSET, message) << BLOCK_SHIFT; - u32 size = FIELD_GET(INITRB_SIZE, message) << BLOCK_SHIFT; - u16 tag = FIELD_GET(INITRB_TAG, message); - u32 bufsz, end, stride; - u8 *hdr; - - if (tag != ep->bfr_tag) { - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: expected tag 0x%x but got 0x%x\n", - ep->endpoint, ep->bfr_tag, tag); - return; - } - - if (bfr->ready) { - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: buffer is already initialized\n", - ep->endpoint); - return; - } - - if (!ep->bfr || base >= ep->bfr_size || size < sizeof(u32)) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: invalid ring base 0x%x or size 0x%x\n", - ep->endpoint, base, size); - return; - } - - end = base + size; - if (end > ep->bfr_size) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: requested end 0x%x > max size 0x%lx\n", - ep->endpoint, end, ep->bfr_size); - return; - } - - hdr = (u8 *)ep->bfr + base; - dma_rmb(); - bufsz = le32_to_cpu(READ_ONCE(*(__le32 *)hdr)); - if (neo_afk_ring_stride(size, bufsz, &stride)) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: invalid ring payload size 0x%x of 0x%x\n", - ep->endpoint, bufsz, size); - return; - } - if (bufsz < BIT(BLOCK_SHIFT)) { - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: ring cannot hold an entry\n", - ep->endpoint); - return; - } - if (bfr == &ep->rxbfr) { - /* Receive dispatch cannot recurse on the ordered endpoint queue. */ - ep->rx_scratch = devm_kmalloc(ep->neo_dcp->dev, bufsz, GFP_KERNEL); - if (!ep->rx_scratch) - return; - } - - bfr->rptr = (__le32 *)(hdr + stride); - bfr->wptr = (__le32 *)(hdr + 2 * stride); - bfr->buf = hdr + 3 * stride; - bfr->bufsz = bufsz; - bfr->stride = stride; - bfr->ready = true; - - if (ep->rxbfr.ready && ep->txbfr.ready) - neo_afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_START)); -} - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_DEBUG) -static void neo_afk_populate_service_debugfs(struct neo_apple_epic_service *srv); -static void neo_afk_remove_service_debugfs(struct neo_apple_epic_service *srv); -#else -static void neo_afk_populate_service_debugfs(struct neo_apple_epic_service *srv) -{ -} -static void neo_afk_remove_service_debugfs(struct neo_apple_epic_service *srv) -{ -} -#endif - -static const struct neo_apple_epic_service_ops * -neo_afk_match_service(struct neo_apple_dcp_afkep *ep, const char *name) -{ - const struct neo_apple_epic_service_ops *ops; - - if (!name || !name[0]) - return NULL; - if (!ep->ops) - return NULL; - - for (ops = ep->ops; ops->name[0]; ops++) { - if (strcmp(ops->name, name)) - continue; - - return ops; - } - - return NULL; -} - -static struct neo_apple_epic_service *neo_afk_epic_find_service(struct neo_apple_dcp_afkep *ep, - u32 channel) -{ - struct neo_apple_epic_service *service; - u32 i; - - for (i = 0; i < ep->num_channels; i++) { - service = &ep->services[i]; - if (neo_afk_service_matches(service, channel)) - return service; - } - - return NULL; -} - -static void neo_afk_recv_handle_init(struct neo_apple_dcp_afkep *ep, u32 channel, - u8 *payload, size_t payload_size) -{ - char name[32]; - s64 neo_epic_unit = -1; - struct neo_apple_epic_service *service = NULL; - u32 ch_idx; - const char *service_name = name; - const char *neo_epic_name = NULL, *neo_epic_class = NULL; - const struct neo_apple_epic_service_ops *ops; - struct neo_dcp_parse_ctx ctx; - u8 *props = payload + sizeof(name); - size_t props_size = payload_size - sizeof(name); - - if (neo_afk_epic_find_service(ep, channel)) - return; - - if (payload_size < sizeof(name)) { - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", - ep->endpoint, payload_size); - return; - } - - strscpy(name, payload, sizeof(name)); - - /* - * in DCP firmware 13.2 DCP reports interface-name as name which starts - * with "dispext%d" using -1 s ID for "dcp". In the 12.3 firmware - * EPICProviderClass was used. If the init call has props parse them and - * use EPICProviderClass to match the service. - */ - if (props_size > 36) { - int ret = neo_parse(props, props_size, &ctx); - if (ret) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: Failed to parse service init props for %s\n", - ep->endpoint, name); - return; - } - ret = neo_parse_epic_service_init(&ctx, &neo_epic_name, &neo_epic_class, &neo_epic_unit, - ep->neo_dcp->fw_compat == DCP_FIRMWARE_H17P); - if (ret) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: failed to extract init props: %d\n", - ep->endpoint, ret); - return; - } - service_name = neo_epic_class; - } else { - service_name = name; - } - - if (ep->match_epic_name) - ops = neo_afk_match_service(ep, neo_epic_name); - else - ops = neo_afk_match_service(ep, service_name); - - if (!ops) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: unable to match service %s on channel %d\n", - ep->endpoint, service_name, channel); - goto free; - } - - for (ch_idx = 0; ch_idx < AFK_MAX_CHANNEL; ch_idx++) { - struct neo_apple_epic_service *candidate = &ep->services[ch_idx]; - - if (neo_afk_service_reinit(candidate, ep, ops, channel)) { - service = candidate; - break; - } - } - if (!service) { - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: no reusable service slots\n", - ep->endpoint); - goto free; - } - ep->num_channels = max(ep->num_channels, ch_idx + 1); - ops->init(service, neo_epic_name, neo_epic_class, neo_epic_unit); - dev_info(ep->neo_dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", - ep->endpoint, service_name, channel); - - neo_afk_populate_service_debugfs(service); - -free: - kfree(neo_epic_name); - kfree(neo_epic_class); -} - -static void neo_afk_recv_handle_teardown(struct neo_apple_dcp_afkep *ep, u32 channel) -{ - struct neo_apple_epic_service *service; - const struct neo_apple_epic_service_ops *ops; - unsigned long flags; - - service = neo_afk_epic_find_service(ep, channel); - if (!service) { - dev_warn(ep->neo_dcp->dev, "AFK[ep:%02x]: teardown for disabled channel %u\n", - ep->endpoint, channel); - return; - } - - neo_afk_remove_service_debugfs(service); - - spin_lock_irqsave(&service->lock, flags); - if (service->torndown) { - spin_unlock_irqrestore(&service->lock, flags); - return; - } - /* Outstanding commands remain discoverable until their replies arrive. */ - service->torndown = true; - service->enabled = false; - ops = service->ops; - spin_unlock_irqrestore(&service->lock, flags); - - if (ops->teardown) - ops->teardown(service); -} - -static void neo_afk_recv_handle_reply(struct neo_apple_dcp_afkep *ep, u32 channel, - u16 tag, void *payload, size_t payload_size) -{ - struct neo_epic_cmd *cmd = payload; - struct neo_apple_epic_service *service; - unsigned long flags; - u8 idx = tag & 0xff; - void *rxbuf, *txbuf; - dma_addr_t rxbuf_dma, txbuf_dma; - size_t rxlen, txlen; - - service = neo_afk_epic_find_service(ep, channel); - if (!service) { - dev_warn(ep->neo_dcp->dev, "AFK[ep:%02x]: command reply on disabled channel %u\n", - ep->endpoint, channel); - return; - } - - if (payload_size < sizeof(*cmd)) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: command reply on channel %d too small: %ld\n", - ep->endpoint, channel, payload_size); - return; - } - - if (idx >= MAX_PENDING_CMDS) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: command reply on channel %d out of range: %d\n", - ep->endpoint, channel, idx); - return; - } - - spin_lock_irqsave(&service->lock, flags); - if (!test_bit(idx, service->cmd_map) || service->cmds[idx].done) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: command reply on channel %d already handled\n", - ep->endpoint, channel); - spin_unlock_irqrestore(&service->lock, flags); - return; - } - - if (tag != service->cmds[idx].tag) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: command reply on channel %d has invalid tag: expected 0x%04x != 0x%04x\n", - ep->endpoint, channel, tag, service->cmds[idx].tag); - spin_unlock_irqrestore(&service->lock, flags); - return; - } - - service->cmds[idx].done = true; - service->cmds[idx].retcode = le32_to_cpu(cmd->retcode); - if (service->cmds[idx].free_on_ack) { - /* defer freeing until we're no longer in atomic context */ - rxbuf = service->cmds[idx].rxbuf; - txbuf = service->cmds[idx].txbuf; - rxlen = service->cmds[idx].rxlen; - txlen = service->cmds[idx].txlen; - rxbuf_dma = service->cmds[idx].rxbuf_dma; - txbuf_dma = service->cmds[idx].txbuf_dma; - bitmap_release_region(service->cmd_map, idx, 0); - } else { - rxbuf = txbuf = NULL; - rxlen = txlen = 0; - } - if (service->cmds[idx].completion) - complete(service->cmds[idx].completion); - - spin_unlock_irqrestore(&service->lock, flags); - - if (rxbuf && rxlen) - dma_free_coherent(ep->neo_dcp->dev, rxlen, rxbuf, rxbuf_dma); - if (txbuf && txlen) - dma_free_coherent(ep->neo_dcp->dev, txlen, txbuf, txbuf_dma); -} - -struct neo_epic_std_service_ap_call { - __le32 unk0; - __le32 unk1; - __le32 type; - __le32 len; - __le32 magic; - u8 _unk[48]; -} __attribute__((packed)); - -static void neo_afk_recv_handle_std_service(struct neo_apple_dcp_afkep *ep, u32 channel, - u32 type, struct neo_epic_hdr *ehdr, - struct neo_epic_sub_hdr *eshdr, - void *payload, size_t payload_size) -{ - struct neo_apple_epic_service *service = neo_afk_epic_find_service(ep, channel); - - if (!service) { - dev_warn(ep->neo_dcp->dev, - "AFK[ep:%02x]: std service notify on disabled channel %u\n", - ep->endpoint, channel); - return; - } - if (service->torndown) { - dev_warn(ep->neo_dcp->dev, - "AFK[ep:%02x]: std service notify on torn down service " - "(chan:%u)\n", ep->endpoint, channel); - return; - } - - if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_NOTIFY) { - struct neo_epic_std_service_ap_call *call = payload; - size_t call_size; - void *reply; - int ret; - - if (payload_size < sizeof(*call)) - return; - - call_size = le32_to_cpu(call->len); - if (payload_size < sizeof(*call) + call_size) - return; - - if (!service->ops->call) - return; - reply = kzalloc(payload_size, GFP_KERNEL); - if (!reply) - return; - - ret = service->ops->call(service, le32_to_cpu(call->type), - payload + sizeof(*call), call_size, - reply + sizeof(*call), call_size); - if (ret) { - dev_warn(ep->neo_dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) apcall %u handler returned %d, sending NO reply\n", - ep->endpoint, service->ops->name, channel, - le32_to_cpu(call->type), ret); - kfree(reply); - return; - } - - memcpy(reply, call, sizeof(*call)); - neo_afk_send_epic(ep, channel, le16_to_cpu(eshdr->tag), - EPIC_TYPE_NOTIFY_ACK, EPIC_CAT_REPLY, - EPIC_SUBTYPE_STD_SERVICE, reply, payload_size); - kfree(reply); - - return; - } - - if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_REPORT) { - if (service->ops->report) - service->ops->report(service, le16_to_cpu(eshdr->type), - payload, payload_size); - return; - } - - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: channel %d received unhandled standard service message: %x / %x\n", - ep->endpoint, channel, type, eshdr->category); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); -} - -static bool neo_afk_validate_h17p_header(const u8 *data, size_t size) -{ - /* The measured length counts bytes after the first eight header bytes. */ - return size >= sizeof(struct neo_epic_hdr) + - sizeof(struct neo_epic_sub_hdr_h17p) && - get_unaligned_le32(data + 4) == size - 8; -} - -static void neo_afk_recv_handle(struct neo_apple_dcp_afkep *ep, u32 channel, u32 type, - u8 *data, size_t data_size) -{ - struct neo_apple_epic_service *service; - struct neo_epic_hdr *ehdr = (struct neo_epic_hdr *)data; - struct neo_epic_sub_hdr *eshdr = - (struct neo_epic_sub_hdr *)(data + sizeof(*ehdr)); - struct neo_epic_sub_hdr h17p_eshdr; - size_t hdr_len = sizeof(*ehdr) + sizeof(*eshdr); - u16 subtype; - u8 *payload; - size_t payload_size; - - if (ep->neo_dcp->fw_compat == DCP_FIRMWARE_H17P) { - const struct neo_epic_sub_hdr_h17p *c = - (const struct neo_epic_sub_hdr_h17p *)(data + sizeof(*ehdr)); - - hdr_len = sizeof(*ehdr) + sizeof(*c); - if (data_size < hdr_len) - goto too_small; - if (ep->neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !neo_afk_validate_h17p_header(data, data_size)) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: invalid compact message length\n", - ep->endpoint); - return; - } - - /* - * H17P multiplexes every service of an endpoint onto queue - * channel 0 and carries the real channel in the EPIC header - * instead: byte 0 is a sequence counter, byte 1 a flags byte - * and bytes 2-3 the channel. ep:20 announces "system" as - * channel 1 and "powerlog-service" as channel 3, then sends the - * mNits/uAmps/iDAC backlight reports on channel 3. Taking the - * queue channel at face value binds "system" to 0 and makes - * every later announce and report look like a duplicate. - */ - channel = le16_to_cpup((__le16 *)(data + 2)); - - /* - * Normalise into the wide sub-header the rest of this file - * expects, mapping H17P's announce subtype onto the canonical - * one so no downstream check has to know the difference. - */ - memset(&h17p_eshdr, 0, sizeof(h17p_eshdr)); - h17p_eshdr.category = c->category; - h17p_eshdr.type = cpu_to_le16(c->type == EPIC_SUBTYPE_ANNOUNCE_H17P ? - EPIC_SUBTYPE_ANNOUNCE : c->type); - h17p_eshdr.tag = cpu_to_le16(le32_to_cpu(c->tag)); - eshdr = &h17p_eshdr; - } - - if (data_size < hdr_len) - goto too_small; - - subtype = le16_to_cpu(eshdr->type); - payload = data + hdr_len; - payload_size = data_size - hdr_len; - - trace_neo_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); - - service = neo_afk_epic_find_service(ep, channel); - - if (!service) { - if (type != EPIC_TYPE_NOTIFY && type != EPIC_TYPE_REPLY) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: expected notify but got 0x%x on channel %d\n", - ep->endpoint, type, channel); - return; - } - if (eshdr->category != EPIC_CAT_REPORT) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: expected report but got 0x%x on channel %d\n", - ep->endpoint, eshdr->category, channel); - return; - } - if (subtype == EPIC_SUBTYPE_TEARDOWN) { - dev_dbg(ep->neo_dcp->dev, - "AFK[ep:%02x]: teardown without service on channel %d\n", - ep->endpoint, channel); - return; - } - if (subtype != EPIC_SUBTYPE_ANNOUNCE) { - dev_err(ep->neo_dcp->dev, - "AFK[ep:%02x]: expected announce but got 0x%x on channel %d\n", - ep->endpoint, subtype, channel); - return; - } - - return neo_afk_recv_handle_init(ep, channel, payload, payload_size); - } - - if (!service) { - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: channel %d has no service\n", - ep->endpoint, channel); - return; - } - - if (type == EPIC_TYPE_NOTIFY && eshdr->category == EPIC_CAT_REPORT && - subtype == EPIC_SUBTYPE_TEARDOWN) - return neo_afk_recv_handle_teardown(ep, channel); - - if (type == EPIC_TYPE_REPLY && eshdr->category == EPIC_CAT_REPLY) - return neo_afk_recv_handle_reply(ep, channel, - le16_to_cpu(eshdr->tag), payload, - payload_size); - - if (subtype == EPIC_SUBTYPE_STD_SERVICE) - return neo_afk_recv_handle_std_service( - ep, channel, type, ehdr, eshdr, payload, payload_size); - - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: channel %d received unhandled message " - "(type %x subtype %x)\n", ep->endpoint, channel, type, subtype); - print_hex_dump(KERN_INFO, "AFK: ", DUMP_PREFIX_NONE, 16, 1, payload, - payload_size, true); - return; - -too_small: - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: payload too small: %lx\n", - ep->endpoint, data_size); -} - -static bool neo_afk_recv(struct neo_apple_dcp_afkep *ep) -{ - struct neo_afk_qe *hdr; - u32 rptr, wptr; - u32 magic, size, channel, type; - - if (!ep->rxbfr.ready) { - dev_err(ep->neo_dcp->dev, "AFK[ep:%02x]: got RECV but not ready\n", - ep->endpoint); - return false; - } - - rptr = le32_to_cpu(*ep->rxbfr.rptr); - wptr = le32_to_cpu(*ep->rxbfr.wptr); - trace_neo_afk_recv_rwptr_pre(ep, rptr, wptr); - - if (rptr == wptr) - return false; - - if (rptr > (ep->rxbfr.bufsz - sizeof(*hdr))) { - dev_warn(ep->neo_dcp->dev, - "AFK[ep:%02x]: rptr out of bounds: 0x%x > 0x%lx\n", - ep->endpoint, rptr, ep->rxbfr.bufsz - sizeof(*hdr)); - return false; - } - - dma_rmb(); - - hdr = ep->rxbfr.buf + rptr; - magic = le32_to_cpu(hdr->magic); - size = le32_to_cpu(hdr->size); - trace_neo_afk_recv_qe(ep, rptr, magic, size); - - if (magic != QE_MAGIC) { - dev_warn(ep->neo_dcp->dev, "AFK[ep:%02x]: invalid queue entry magic: 0x%x\n", - ep->endpoint, magic); - return false; - } - - /* - * If there's not enough space for the payload the co-processor inserted - * the current dummy queue entry and we have to advance to the next one - * which will contain the real data. - */ - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { - rptr = 0; - hdr = ep->rxbfr.buf + rptr; - magic = le32_to_cpu(hdr->magic); - size = le32_to_cpu(hdr->size); - trace_neo_afk_recv_qe(ep, rptr, magic, size); - - if (magic != QE_MAGIC) { - dev_warn(ep->neo_dcp->dev, - "AFK[ep:%02x]: invalid next queue entry magic: 0x%x\n", - ep->endpoint, magic); - return false; - } - } - - if (rptr + size + sizeof(*hdr) > ep->rxbfr.bufsz) { - dev_warn(ep->neo_dcp->dev, - "AFK[ep:%02x]: queue entry out of bounds: 0x%lx > 0x%lx\n", - ep->endpoint, rptr + size + sizeof(*hdr), ep->rxbfr.bufsz); - return false; - } - - channel = le32_to_cpu(hdr->channel); - type = le32_to_cpu(hdr->type); - /* - * Publishing rptr releases this entry to the firmware. Handlers can - * sleep and send replies, so retain a private copy before allowing - * the producer to reuse the ring storage. - */ - memcpy(ep->rx_scratch, hdr->data, size); - - rptr = neo_afk_ring_advance(rptr, sizeof(*hdr) + size, - ep->rxbfr.bufsz, ep->rxbfr.stride); - - dma_mb(); - - *ep->rxbfr.rptr = cpu_to_le32(rptr); - trace_neo_afk_recv_rwptr_post(ep, rptr, wptr); - - neo_afk_recv_handle(ep, channel, type, ep->rx_scratch, size); - - return true; -} - -static void neo_afk_receive_message_worker(struct work_struct *work_) -{ - struct neo_afk_receive_message_work *work; - u16 type; - - work = container_of(work_, struct neo_afk_receive_message_work, work); - - type = FIELD_GET(RBEP_TYPE, work->message); - switch (type) { - case RBEP_INIT_ACK: - break; - - case RBEP_START_ACK: - complete_all(&work->ep->started); - break; - - case RBEP_SHUTDOWN_ACK: - complete_all(&work->ep->stopped); - break; - - case RBEP_GETBUF: - neo_afk_getbuf(work->ep, work->message); - break; - - case RBEP_INIT_TX: - neo_afk_init_rxtx(work->ep, work->message, &work->ep->txbfr); - break; - - case RBEP_INIT_RX: - neo_afk_init_rxtx(work->ep, work->message, &work->ep->rxbfr); - break; - - case RBEP_RECV: - while (neo_afk_recv(work->ep)) - ; - break; - - default: - dev_err(work->ep->neo_dcp->dev, - "Received unknown AFK message type: 0x%x\n", type); - } - - kfree(work); -} - -int neo_afk_receive_message(struct neo_apple_dcp_afkep *ep, u64 message) -{ - struct neo_afk_receive_message_work *work; - unsigned long flags; - - if (!ep) - return -ENODEV; - - // TODO: comment why decoupling from rtkit thread is required here - work = kzalloc(sizeof(*work), GFP_KERNEL); - if (!work) - return -ENOMEM; - - work->ep = ep; - work->message = message; - INIT_WORK(&work->work, neo_afk_receive_message_worker); - spin_lock_irqsave(&ep->lock, flags); - if (ep->stopping) { - spin_unlock_irqrestore(&ep->lock, flags); - kfree(work); - return -ESHUTDOWN; - } - queue_work(ep->wq, &work->work); - spin_unlock_irqrestore(&ep->lock, flags); - - return 0; -} - -int neo_afk_send_epic(struct neo_apple_dcp_afkep *ep, u32 channel, u16 tag, - enum neo_epic_type etype, enum neo_epic_category ecat, u8 stype, - const void *payload, size_t payload_len) -{ - u32 rptr, wptr; - struct neo_afk_qe *hdr, *hdr2; - struct neo_epic_hdr *ehdr; - struct neo_epic_sub_hdr *eshdr; - unsigned long flags; - size_t total_epic_size, total_size; - int ret; - - /* - * H17P frames EPIC messages differently (see afk_recv_handle()): an - * 8-byte sub-header, with the service channel carried in the EPIC - * header on queue channel 0. Only the receive side of that format is - * known, and the 24-byte framing below would be misparsed, so send - * nothing. The display path needs no AP-initiated EPIC traffic: the - * callers are optional (verbose firmware logging, EDID copy, audio, - * DPTX, debugfs) or replies to calls only DPTX services make. - */ - if (ep->neo_dcp->fw_compat == DCP_FIRMWARE_H17P) { - dev_dbg_once(ep->neo_dcp->dev, - "AFK[ep:%02x]: not sending EPIC message on H17P\n", - ep->endpoint); - return -EOPNOTSUPP; - } - - spin_lock_irqsave(&ep->lock, flags); - if (ep->stopping) { - ret = -ESHUTDOWN; - goto out; - } - - dma_rmb(); - rptr = le32_to_cpu(*ep->txbfr.rptr); - wptr = le32_to_cpu(*ep->txbfr.wptr); - trace_neo_afk_send_rwptr_pre(ep, rptr, wptr); - total_epic_size = sizeof(*ehdr) + sizeof(*eshdr) + payload_len; - total_size = sizeof(*hdr) + total_epic_size; - - hdr = hdr2 = NULL; - - /* - * We need to figure out how to place the entire headers and payload - * into the ring buffer: - * - If the write pointer is in front of the read pointer we just need - * enough space inbetween to store everything. - * - If the read pointer has already wrapper around the end of the - * buffer we can - * a) either store the entire payload at the writer pointer if - * there's enough space until the end, - * b) or just store the queue entry at the write pointer to indicate - * that we need to wrap to the start and then store the headers - * and the payload at the beginning of the buffer. The queue - * header has to be store twice in this case. - * In either case we have to ensure that there's always enough space - * so that we don't accidentally overwrite other buffers. - */ - if (wptr < rptr) { - /* - * If wptr < rptr we can't wrap around and only have to make - * sure that there's enough space for the entire payload. - */ - if (wptr + total_size > rptr) { - ret = -ENOMEM; - goto out; - } - - hdr = ep->txbfr.buf + wptr; - wptr += sizeof(*hdr); - } else { - /* We need enough space to place at least a queue entry */ - if (wptr + sizeof(*hdr) > ep->txbfr.bufsz) { - ret = -ENOMEM; - goto out; - } - - /* - * If we can place a single queue entry but not the full payload - * we need to place one queue entry at the end of the ring - * buffer and then another one together with the entire - * payload at the beginning. - */ - if (wptr + total_size > ep->txbfr.bufsz) { - /* - * Ensure there's space for the queue entry at the - * beginning - */ - if (sizeof(*hdr) > rptr) { - ret = -ENOMEM; - goto out; - } - - /* - * Place two queue entries to indicate we want to wrap - * around to the firmware. - */ - hdr = ep->txbfr.buf + wptr; - hdr2 = ep->txbfr.buf; - wptr = sizeof(*hdr); - - /* Ensure there's enough space for the entire payload */ - if (wptr + total_epic_size > rptr) { - ret = -ENOMEM; - goto out; - } - } else { - /* We have enough space to place the entire payload */ - hdr = ep->txbfr.buf + wptr; - wptr += sizeof(*hdr); - } - } - /* - * At this point we're guaranteed that hdr (and possibly hdr2) point - * to a buffer large enough to fit the queue entry and that we have - * enough space at wptr to store the payload. - */ - - hdr->magic = cpu_to_le32(QE_MAGIC); - hdr->size = cpu_to_le32(total_epic_size); - hdr->channel = cpu_to_le32(channel); - hdr->type = cpu_to_le32(etype); - if (hdr2) - memcpy(hdr2, hdr, sizeof(*hdr)); - - ehdr = ep->txbfr.buf + wptr; - memset(ehdr, 0, sizeof(*ehdr)); - ehdr->version = 2; - ehdr->seq = cpu_to_le16(ep->qe_seq++); - ehdr->timestamp = cpu_to_le64(0); - wptr += sizeof(*ehdr); - - eshdr = ep->txbfr.buf + wptr; - memset(eshdr, 0, sizeof(*eshdr)); - eshdr->length = cpu_to_le32(payload_len); - eshdr->version = 4; - eshdr->category = ecat; - eshdr->type = cpu_to_le16(stype); - eshdr->timestamp = cpu_to_le64(0); - eshdr->tag = cpu_to_le16(tag); - if (ecat == EPIC_CAT_REPLY) - eshdr->inline_len = cpu_to_le32(payload_len - 4); - else - eshdr->inline_len = cpu_to_le32(0); - wptr += sizeof(*eshdr); - - memcpy(ep->txbfr.buf + wptr, payload, payload_len); - wptr += payload_len; - wptr = neo_afk_ring_advance(wptr, 0, ep->txbfr.bufsz, - ep->txbfr.stride); - trace_neo_afk_send_rwptr_post(ep, rptr, wptr); - - /* Publish the completed entry before allowing the consumer to read it. */ - dma_wmb(); - *ep->txbfr.wptr = cpu_to_le32(wptr); - neo_afk_send(ep, FIELD_PREP(RBEP_TYPE, RBEP_SEND) | - FIELD_PREP(SEND_WPTR, wptr)); - ret = 0; - -out: - spin_unlock_irqrestore(&ep->lock, flags); - return ret; -} - -int neo_afk_send_command(struct neo_apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, void *output, - size_t output_len, u32 *retcode) -{ - return neo_afk_send_command_timeout(service, type, payload, payload_len, - output, output_len, retcode, - MSEC_PER_SEC); -} - -int neo_afk_send_command_timeout(struct neo_apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, - void *output, size_t output_len, u32 *retcode, - unsigned int timeout_ms) -{ - struct neo_epic_cmd cmd; - void *rxbuf, *txbuf; - dma_addr_t rxbuf_dma, txbuf_dma; - unsigned long flags; - int ret, idx; - u16 tag; - struct neo_apple_dcp_afkep *ep; - DECLARE_COMPLETION_ONSTACK(completion); - - service = neo_afk_service_get(service); - if (!service) - return -ENODEV; - ep = service->ep; - if (READ_ONCE(ep->neo_dcp->quiescing)) { - ret = -ENODEV; - goto err_put_service; - } - - rxbuf = dma_alloc_coherent(ep->neo_dcp->dev, output_len, &rxbuf_dma, - GFP_KERNEL); - if (!rxbuf) { - ret = -ENOMEM; - goto err_put_service; - } - txbuf = dma_alloc_coherent(ep->neo_dcp->dev, payload_len, &txbuf_dma, - GFP_KERNEL); - if (!txbuf) { - ret = -ENOMEM; - goto err_free_rxbuf; - } - - memcpy(txbuf, payload, payload_len); - - memset(&cmd, 0, sizeof(cmd)); - cmd.retcode = cpu_to_le32(0); - cmd.rxbuf = cpu_to_le64(rxbuf_dma); - cmd.rxlen = cpu_to_le32(output_len); - cmd.txbuf = cpu_to_le64(txbuf_dma); - cmd.txlen = cpu_to_le32(payload_len); - - spin_lock_irqsave(&service->lock, flags); - if (service->torndown || !service->enabled) { - ret = -ENODEV; - goto err_unlock; - } - idx = bitmap_find_free_region(service->cmd_map, MAX_PENDING_CMDS, 0); - if (idx < 0) { - ret = -ENOSPC; - goto err_unlock; - } - - tag = (service->cmd_tag & 0xff) << 8; - tag |= idx & 0xff; - service->cmd_tag++; - - service->cmds[idx].tag = tag; - service->cmds[idx].rxbuf = rxbuf; - service->cmds[idx].txbuf = txbuf; - service->cmds[idx].rxbuf_dma = rxbuf_dma; - service->cmds[idx].txbuf_dma = txbuf_dma; - service->cmds[idx].rxlen = output_len; - service->cmds[idx].txlen = payload_len; - service->cmds[idx].free_on_ack = false; - service->cmds[idx].done = false; - service->cmds[idx].completion = &completion; - init_completion(&completion); - - ret = neo_afk_send_epic(service->ep, service->channel, tag, - EPIC_TYPE_COMMAND, EPIC_CAT_COMMAND, type, &cmd, - sizeof(cmd)); - spin_unlock_irqrestore(&service->lock, flags); - if (ret) - goto err_free_cmd; - - ret = wait_for_completion_timeout(&completion, - msecs_to_jiffies(timeout_ms)); - - if (ret <= 0) { - spin_lock_irqsave(&service->lock, flags); - /* - * Check again while we're inside the lock to make sure - * the command wasn't completed just after - * wait_for_completion_timeout returned. - */ - if (!service->cmds[idx].done) { - service->cmds[idx].completion = NULL; - service->cmds[idx].free_on_ack = true; - spin_unlock_irqrestore(&service->lock, flags); - dev_warn(ep->neo_dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) command type 0x%x tag 0x%04x timed out after %u ms\n", - ep->endpoint, service->ops->name, service->channel, - type, tag, timeout_ms); - neo_afk_service_put(service); - return -ETIMEDOUT; - } - spin_unlock_irqrestore(&service->lock, flags); - } - - ret = 0; - if (retcode) - *retcode = service->cmds[idx].retcode; - if (output && output_len) - memcpy(output, rxbuf, output_len); - -err_free_cmd: - spin_lock_irqsave(&service->lock, flags); - service->cmds[idx].completion = NULL; - bitmap_release_region(service->cmd_map, idx, 0); -err_unlock: - spin_unlock_irqrestore(&service->lock, flags); - dma_free_coherent(ep->neo_dcp->dev, payload_len, txbuf, txbuf_dma); -err_free_rxbuf: - dma_free_coherent(ep->neo_dcp->dev, output_len, rxbuf, rxbuf_dma); -err_put_service: - neo_afk_service_put(service); - return ret; -} - -int neo_afk_service_call(struct neo_apple_epic_service *service, u16 group, u32 command, - const void *data, size_t data_len, size_t data_pad, - void *output, size_t output_len, size_t output_pad) -{ - return neo_afk_service_call_timeout(service, group, command, data, data_len, - data_pad, output, output_len, - output_pad, MSEC_PER_SEC); -} - -int neo_afk_service_call_timeout(struct neo_apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms) -{ - struct neo_epic_service_call *call; - void *bfr; - size_t bfr_len = max(data_len + data_pad, output_len + output_pad) + - sizeof(*call); - int ret; - u32 retcode; - u32 retlen; - - if (!service) - return -ENODEV; - - bfr = kzalloc(bfr_len, GFP_KERNEL); - if (!bfr) - return -ENOMEM; - - call = bfr; - - memset(call, 0, sizeof(*call)); - call->group = cpu_to_le16(group); - call->command = cpu_to_le32(command); - call->data_len = cpu_to_le32(data_len + data_pad); - call->magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC); - - memcpy(bfr + sizeof(*call), data, data_len); - - ret = neo_afk_send_command_timeout(service, EPIC_SUBTYPE_STD_SERVICE, bfr, - bfr_len, bfr, bfr_len, &retcode, - timeout_ms); - if (ret) { - dev_warn(service->ep->neo_dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u failed to complete: %d\n", - service->ep->endpoint, service->ops->name, - service->channel, group, command, ret); - goto out; - } - if (retcode) { - dev_warn(service->ep->neo_dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u returned retcode 0x%08x (reply data_len %u)\n", - service->ep->endpoint, service->ops->name, - service->channel, group, command, retcode, - le32_to_cpu(call->data_len)); - ret = -EINVAL; - goto out; - } - if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || - le16_to_cpu(call->group) != group || - le32_to_cpu(call->command) != command) { - dev_warn(service->ep->neo_dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) service call reply mismatch: sent magic 0x%08x group %u cmd %u, got magic 0x%08x group %u cmd %u\n", - service->ep->endpoint, service->ops->name, - service->channel, EPIC_SERVICE_CALL_MAGIC, group, - command, le32_to_cpu(call->magic), - le16_to_cpu(call->group), le32_to_cpu(call->command)); - ret = -EINVAL; - goto out; - } - dev_dbg(service->ep->neo_dcp->dev, - "AFK[ep:%02x]: %s (chan:%u) service call group %u cmd %u ok (reply data_len %u)\n", - service->ep->endpoint, service->ops->name, service->channel, - group, command, le32_to_cpu(call->data_len)); - - retlen = le32_to_cpu(call->data_len); - if (output_len < retlen) - retlen = output_len; - if (output && output_len) { - memset(output, 0, output_len); - memcpy(output, bfr + sizeof(*call), retlen); - } - -out: - kfree(bfr); - return ret; -} - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_DEBUG) - -#define AFK_DEBUGFS_MAX_REPLY 8192 - -static ssize_t service_call_write_file(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct neo_apple_epic_service *srv = file->private_data; - void *buf; - int ret; - struct { - u32 group; - u32 command; - } call_info; - - if (count < sizeof(call_info)) - return -EINVAL; - if (!srv->debugfs.scratch) { - srv->debugfs.scratch = \ - devm_kzalloc(srv->ep->neo_dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); - if (!srv->debugfs.scratch) - return -ENOMEM; - } - - if (copy_from_user(&call_info, user_buf, sizeof(call_info))) - return -EFAULT; - user_buf += sizeof(call_info); - count -= sizeof(call_info); - - buf = kmalloc(count, GFP_KERNEL); - if (!buf) - return -ENOMEM; - if (copy_from_user(buf, user_buf, count)) { - kfree(buf); - return -EFAULT; - } - - memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); - dma_mb(); - - ret = neo_afk_service_call(srv, call_info.group, call_info.command, buf, count, 0, - srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY, 0); - kfree(buf); - - if (ret < 0) - return ret; - - return count + sizeof(call_info); -} - -static ssize_t service_call_read_file(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct neo_apple_epic_service *srv = file->private_data; - - if (!srv->debugfs.scratch) - return -EINVAL; - - return simple_read_from_buffer(user_buf, count, ppos, - srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY); -} - -static const struct file_operations service_call_fops = { - .open = simple_open, - .write = service_call_write_file, - .read = service_call_read_file, -}; - -static ssize_t service_raw_call_write_file(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct neo_apple_epic_service *srv = file->private_data; - u32 retcode; - int ret; - - if (!count) - return 0; - if (count > AFK_DEBUGFS_MAX_REPLY) - return -E2BIG; - - if (!srv->debugfs.scratch) { - srv->debugfs.scratch = \ - devm_kzalloc(srv->ep->neo_dcp->dev, AFK_DEBUGFS_MAX_REPLY, GFP_KERNEL); - if (!srv->debugfs.scratch) - return -ENOMEM; - } - - memset(srv->debugfs.scratch, 0, AFK_DEBUGFS_MAX_REPLY); - if (copy_from_user(srv->debugfs.scratch, user_buf, count)) - return -EFAULT; - - ret = neo_afk_send_command(srv, EPIC_SUBTYPE_STD_SERVICE, srv->debugfs.scratch, count, - srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY, &retcode); - if (ret < 0) - return ret; - if (retcode) - return -EINVAL; - - return count; -} - -static ssize_t service_raw_call_read_file(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct neo_apple_epic_service *srv = file->private_data; - - if (!srv->debugfs.scratch) - return -EINVAL; - - return simple_read_from_buffer(user_buf, count, ppos, - srv->debugfs.scratch, AFK_DEBUGFS_MAX_REPLY); -} - -static const struct file_operations service_raw_call_fops = { - .open = simple_open, - .write = service_raw_call_write_file, - .read = service_raw_call_read_file, -}; - -static void neo_afk_populate_service_debugfs(struct neo_apple_epic_service *srv) -{ - if (!srv->ep->debugfs_entry || !srv->ops) - return; - - if (strcmp(srv->ops->name, "DCPAVAudioInterface") == 0) { - srv->debugfs.entry = debugfs_create_dir(srv->ops->name, - srv->ep->debugfs_entry); - debugfs_create_file("call", 0600, srv->debugfs.entry, srv, - &service_call_fops); - debugfs_create_file("raw_call", 0600, srv->debugfs.entry, srv, - &service_raw_call_fops); - } -} - -static void neo_afk_remove_service_debugfs(struct neo_apple_epic_service *srv) -{ - if (srv->debugfs.entry) { - debugfs_remove_recursive(srv->debugfs.entry); - srv->debugfs.entry = NULL; - } -} - -#endif diff --git a/drivers/gpu/drm/apple-neo/afk.h b/drivers/gpu/drm/apple-neo/afk.h deleted file mode 100644 index 3ec1f0cddcf37b..00000000000000 --- a/drivers/gpu/drm/apple-neo/afk.h +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * AFK (Apple Firmware Kit) EPIC (EndPoint Interface Client) support - */ -/* Copyright 2022 Sven Peter */ - -#ifndef _DRM_APPLE_DCP_AFK_H -#define _DRM_APPLE_DCP_AFK_H - -#include -#include -#include -#include -#include -#include -#include - -#include "dcp.h" - -#define AFK_MAX_CHANNEL 16 -#define MAX_PENDING_CMDS 16 - -struct neo_apple_epic_service_ops; -struct neo_apple_dcp_afkep; - -struct neo_epic_cmd_info { - u16 tag; - - void *rxbuf; - void *txbuf; - dma_addr_t rxbuf_dma; - dma_addr_t txbuf_dma; - size_t rxlen; - size_t txlen; - - u32 retcode; - bool done; - bool free_on_ack; - struct completion *completion; -}; - -struct neo_apple_epic_service { - const struct neo_apple_epic_service_ops *ops; - struct neo_apple_dcp_afkep *ep; - - struct neo_epic_cmd_info cmds[MAX_PENDING_CMDS]; - DECLARE_BITMAP(cmd_map, MAX_PENDING_CMDS); - u8 cmd_tag; - spinlock_t lock; - /* Pins held by owners and synchronous callers; protected by lock. */ - unsigned int users; - - u32 channel; - bool enabled; - bool torndown; - - void *cookie; - - struct { - struct dentry *entry; - u8 *scratch; - } debugfs; -}; - -enum neo_epic_subtype; - -struct neo_apple_epic_service_ops { - const char name[32]; - /* All retained pointers must be pinned before enabling slot reuse. */ - bool reusable; - - void (*init)(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit); - int (*call)(struct neo_apple_epic_service *service, u32 idx, - const void *data, size_t data_size, void *reply, - size_t reply_size); - int (*report)(struct neo_apple_epic_service *service, enum neo_epic_subtype type, - const void *data, size_t data_size); - void (*teardown)(struct neo_apple_epic_service *service); -}; - -static inline int neo_afk_ring_stride(u32 total, u32 bufsz, u32 *stride) -{ - u32 header, block; - - if (bufsz >= total) - return -EINVAL; - header = total - bufsz; - block = header / 3; - if (header % 3 || block < 0x40 || block % 0x40 || - bufsz < block || bufsz % block) - return -EINVAL; - - *stride = block; - return 0; -} - -static inline u32 neo_afk_ring_advance(u32 ptr, u32 bytes, u32 bufsz, - u32 stride) -{ - u32 next = roundup(ptr + bytes, stride); - - return next == bufsz ? 0 : next; -} -static inline struct neo_apple_epic_service *neo_afk_service_get(struct neo_apple_epic_service *service) -{ - unsigned long flags; - bool available; - - if (!service) - return NULL; - - spin_lock_irqsave(&service->lock, flags); - available = service->enabled && !service->torndown; - if (available) - service->users++; - spin_unlock_irqrestore(&service->lock, flags); - - return available ? service : NULL; -} - -static inline void neo_afk_service_put(struct neo_apple_epic_service *service) -{ - unsigned long flags; - - spin_lock_irqsave(&service->lock, flags); - if (!WARN_ON(!service->users)) - service->users--; - spin_unlock_irqrestore(&service->lock, flags); -} - -static inline void neo_afk_service_disable(struct neo_apple_epic_service *service) -{ - unsigned long flags; - - spin_lock_irqsave(&service->lock, flags); - service->enabled = false; - spin_unlock_irqrestore(&service->lock, flags); -} - -static inline bool neo_afk_service_matches(struct neo_apple_epic_service *service, - u32 channel) -{ - unsigned long flags; - bool found; - - spin_lock_irqsave(&service->lock, flags); - /* Teardown does not cancel commands or relinquish their DMA. */ - found = service->channel == channel && - (service->enabled || - !bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)); - spin_unlock_irqrestore(&service->lock, flags); - return found; -} - -/* The caller holds service->lock. */ -static inline bool neo_afk_service_reclaimable(struct neo_apple_epic_service *service) -{ - return !service->ops || - (service->ops->reusable && service->torndown && - !service->enabled && !service->users && - bitmap_empty(service->cmd_map, MAX_PENDING_CMDS)); -} - -static inline bool -neo_afk_service_reinit(struct neo_apple_epic_service *service, - struct neo_apple_dcp_afkep *ep, - const struct neo_apple_epic_service_ops *ops, u32 channel) -{ - unsigned long flags; - bool reusable; - - spin_lock_irqsave(&service->lock, flags); - reusable = neo_afk_service_reclaimable(service); - if (reusable) { - service->enabled = true; - service->torndown = false; - service->ops = ops; - service->ep = ep; - service->channel = channel; - service->cookie = NULL; - /* Keep the command-tag sequence across service generations. */ - } - spin_unlock_irqrestore(&service->lock, flags); - return reusable; -} - -struct neo_afk_ringbuffer_header { - __le32 bufsz; - u32 unk; - u32 _pad1[14]; - __le32 rptr; - u32 _pad2[15]; - __le32 wptr; - u32 _pad3[15]; -}; - -struct neo_afk_qe { -#define QE_MAGIC 0x20504f49 // ' POI' - __le32 magic; - __le32 size; - __le32 channel; - __le32 type; - u8 data[]; -}; - -struct neo_epic_hdr { - u8 version; - __le16 seq; - u8 _pad; - __le32 unk; - __le64 timestamp; -} __attribute__((packed)); - -struct neo_epic_sub_hdr { - __le32 length; - u8 version; - u8 category; - __le16 type; - __le64 timestamp; - __le16 tag; - __le16 unk; - __le32 inline_len; -} __attribute__((packed)); - -/* - * H17P compresses the EPIC sub-header from 24 bytes to 8 and drops the - * separate length/timestamp: the announce for the "system" service puts its - * name at data + 0x18, not data + 0x28. An announce carries type 0x11 and a - * standard-service report carries the usual 0xc0. - */ -struct neo_epic_sub_hdr_h17p { - u8 type; - u8 category; - __le16 flags; - __le32 tag; -} __packed; - -#define EPIC_SUBTYPE_ANNOUNCE_H17P 0x11 - -struct neo_epic_cmd { - __le32 retcode; - __le64 rxbuf; - __le64 txbuf; - __le32 rxlen; - __le32 txlen; - u8 rxcookie; - u8 txcookie; -} __attribute__((packed)); - -struct neo_epic_service_call { - u8 _pad0[2]; - __le16 group; - __le32 command; - __le32 data_len; -#define EPIC_SERVICE_CALL_MAGIC 0x69706378 - __le32 magic; - u8 _pad1[48]; -} __attribute__((packed)); -static_assert(sizeof(struct neo_epic_service_call) == 64); - -enum neo_epic_type { - EPIC_TYPE_NOTIFY = 0, - EPIC_TYPE_COMMAND = 3, - EPIC_TYPE_REPLY = 4, - EPIC_TYPE_NOTIFY_ACK = 8, -}; - -enum neo_epic_category { - EPIC_CAT_REPORT = 0x00, - EPIC_CAT_NOTIFY = 0x10, - EPIC_CAT_REPLY = 0x20, - EPIC_CAT_COMMAND = 0x30, -}; - -enum neo_epic_subtype { - EPIC_SUBTYPE_ANNOUNCE = 0x30, - EPIC_SUBTYPE_TEARDOWN = 0x32, - EPIC_SUBTYPE_STD_SERVICE = 0xc0, -}; - -struct neo_afk_ringbuffer { - bool ready; - __le32 *rptr; - __le32 *wptr; - void *buf; - size_t bufsz; - u32 stride; -}; - - -struct neo_apple_dcp_afkep { - struct neo_apple_dcp *neo_dcp; - - u32 endpoint; - struct workqueue_struct *wq; - - struct completion started; - struct completion stopped; - - void *bfr; - u16 bfr_tag; - size_t bfr_size; - dma_addr_t bfr_dma; - - struct neo_afk_ringbuffer txbfr; - struct neo_afk_ringbuffer rxbfr; - /* Private receive entry, owned by the ordered endpoint worker. */ - void *rx_scratch; - - spinlock_t lock; - u16 qe_seq; - bool stopping; /* lock: no new receive work after shutdown. */ - - const struct neo_apple_epic_service_ops *ops; - struct neo_apple_epic_service services[AFK_MAX_CHANNEL]; - u32 num_channels; - - struct dentry *debugfs_entry; - - bool match_epic_name; -}; - -struct neo_apple_dcp_afkep *neo_afk_init(struct neo_apple_dcp *neo_dcp, u32 endpoint, - const struct neo_apple_epic_service_ops *ops); -int neo_afk_start(struct neo_apple_dcp_afkep *ep); -void neo_afk_shutdown(struct neo_apple_dcp_afkep *ep); -void neo_afk_quiesce(struct neo_apple_dcp_afkep *ep); -int neo_afk_receive_message(struct neo_apple_dcp_afkep *ep, u64 message); -int neo_afk_send_epic(struct neo_apple_dcp_afkep *ep, u32 channel, u16 tag, - enum neo_epic_type etype, enum neo_epic_category ecat, u8 stype, - const void *payload, size_t payload_len); -int neo_afk_send_command(struct neo_apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, void *output, - size_t output_len, u32 *retcode); -int neo_afk_send_command_timeout(struct neo_apple_epic_service *service, u8 type, - const void *payload, size_t payload_len, - void *output, size_t output_len, u32 *retcode, - unsigned int timeout_ms); -int neo_afk_service_call(struct neo_apple_epic_service *service, u16 group, u32 command, - const void *data, size_t data_len, size_t data_pad, - void *output, size_t output_len, size_t output_pad); -int neo_afk_service_call_timeout(struct neo_apple_epic_service *service, u16 group, - u32 command, const void *data, size_t data_len, - size_t data_pad, void *output, size_t output_len, - size_t output_pad, unsigned int timeout_ms); -#endif diff --git a/drivers/gpu/drm/apple-neo/afk_test.c b/drivers/gpu/drm/apple-neo/afk_test.c deleted file mode 100644 index ba71901920cc1f..00000000000000 --- a/drivers/gpu/drm/apple-neo/afk_test.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2026 Ryan Murray */ - -#include -#include - -#include "afk.h" - -static void neo_afk_ring_stride_test(struct kunit *test) -{ - u32 stride; - - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x10c0, 0x1000, &stride), 0); - KUNIT_EXPECT_EQ(test, stride, 0x40U); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x1180, 0x1000, &stride), 0); - KUNIT_EXPECT_EQ(test, stride, 0x80U); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x1000, 0x1000, &stride), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x10bd, 0x1000, &stride), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x10c1, 0x1000, &stride), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0xc0, 0, &stride), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x10c3, 0x1000, &stride), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x10c0, 0x1001, &stride), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x11c0, 0x1040, &stride), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x1180, 0x1000, &stride), 0); - KUNIT_EXPECT_EQ(test, neo_afk_ring_advance(0xf80, 0x80, 0x1000, stride), 0U); - KUNIT_EXPECT_EQ(test, neo_afk_ring_advance(0xe80, 0x80, 0x1000, stride), - 0xf00U); - KUNIT_EXPECT_EQ(test, neo_afk_ring_stride(0x10c0, 0x1000, &stride), 0); - KUNIT_EXPECT_EQ(test, neo_afk_ring_advance(0xfc0, 0x40, 0x1000, stride), 0U); -} - -static struct kunit_case neo_afk_ring_cases[] = { - KUNIT_CASE(neo_afk_ring_stride_test), - {} -}; - -static struct kunit_suite neo_afk_ring_suite = { - .name = "apple-afk-ring", - .test_cases = neo_afk_ring_cases, -}; - -kunit_test_suite(neo_afk_ring_suite); - -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple-neo/apple_drv.c b/drivers/gpu/drm/apple-neo/apple_drv.c deleted file mode 100644 index 6d665800061af8..00000000000000 --- a/drivers/gpu/drm/apple-neo/apple_drv.c +++ /dev/null @@ -1,988 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ -/* Based on meson driver which is - * Copyright (C) 2016 BayLibre, SAS - * Author: Neil Armstrong - * Copyright (C) 2015 Amlogic, Inc. All rights reserved. - * Copyright (C) 2014 Endless Mobile - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dcp.h" -#include "plane.h" - -#define DRIVER_NAME "apple" -#define DRIVER_DESC "Apple display controller DRM driver" - -#define MAX_COPROCESSORS 8 - -struct neo_apple_drm_private { - struct drm_device drm; -}; - -DEFINE_DRM_GEM_DMA_FOPS(apple_fops); - -#define DART_PAGE_SIZE 16384 - -static int neo_apple_drm_gem_dumb_create(struct drm_file *file_priv, - struct drm_device *drm, - struct drm_mode_create_dumb *args) -{ - args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 64); - args->size = round_up(args->pitch * args->height, DART_PAGE_SIZE); - - return drm_gem_dma_dumb_create_internal(file_priv, drm, args); -} - -#ifdef CONFIG_DRM_FBDEV_EMULATION -static int neo_apple_drm_fbdev_probe(struct drm_fb_helper *helper, - struct drm_fb_helper_surface_size *sizes) -{ - struct drm_crtc *crtc; - - drm_for_each_crtc(crtc, helper->dev) { - struct neo_apple_dcp *neo_dcp; - - neo_dcp = platform_get_drvdata(to_apple_crtc(crtc)->neo_dcp); - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - /* H17P applies the stored fourth byte as straight alpha. */ - sizes->surface_bpp = 32; - sizes->surface_depth = 32; - break; - } - } - - return drm_fbdev_dma_driver_fbdev_probe(helper, sizes); -} -#endif - -/* - * A compositor or boot splash is closing the device. drm_release() has - * already taken its file off the file list, so the Type-C fabric can tell - * whether any other one still has the device open. - */ -static void neo_apple_drm_postclose(struct drm_device *drm, struct drm_file *file) -{ - if (file->was_master) - neo_dcp_typec_reorder(); -} - -static const struct drm_driver neo_apple_drm_driver = { - DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(neo_apple_drm_gem_dumb_create), -#ifdef CONFIG_DRM_FBDEV_EMULATION - .fbdev_probe = neo_apple_drm_fbdev_probe, -#else - DRM_FBDEV_DMA_DRIVER_OPS, -#endif - .postclose = neo_apple_drm_postclose, - .name = DRIVER_NAME, - .desc = DRIVER_DESC, - .major = 1, - .minor = 0, - .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC | DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, - .fops = &apple_fops, -}; - -static enum drm_connector_status -neo_apple_connector_detect(struct drm_connector *connector, bool force) -{ - struct neo_apple_connector *neo_apple_connector = to_apple_connector(connector); - - return neo_apple_connector->connected ? connector_status_connected : - connector_status_disconnected; -} - -static void neo_apple_connector_oob_hotplug(struct drm_connector *connector, - enum drm_connector_status status) -{ - struct neo_apple_connector *neo_apple_connector = to_apple_connector(connector); - - if (status == connector_status_connected) - neo_dcp_dptx_connect_oob(neo_apple_connector->neo_dcp, 0); - else if (status == connector_status_disconnected) - neo_dcp_dptx_disconnect_oob(neo_apple_connector->neo_dcp, 0); - else if (status == connector_status_unknown) - neo_dcp_retrain_oob(neo_apple_connector); - else - dev_err(&neo_apple_connector->neo_dcp->dev, "unexpected connector status" - ":0x%x in oob_hotplug event\n", (u32)status); -} - -static void neo_apple_crtc_atomic_enable(struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - struct drm_crtc_state *crtc_state; - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - - if (crtc_state->active_changed && crtc_state->active) { - struct neo_apple_crtc *neo_apple_crtc = to_apple_crtc(crtc); - neo_dcp_poweron(neo_apple_crtc->neo_dcp); - /* Force the CTM to be set on first swap */ - crtc_state->color_mgmt_changed = true; - } - - if (crtc_state->active) - neo_dcp_crtc_atomic_modeset(crtc, state); -} - -static void neo_apple_crtc_atomic_disable(struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - struct drm_crtc_state *crtc_state; - struct neo_apple_crtc *neo_apple_crtc = to_apple_crtc(crtc); - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - - if (crtc_state->active_changed && !crtc_state->active) { - neo_dcp_poweroff(neo_apple_crtc->neo_dcp); - } - - if (crtc->state->event && !crtc->state->active) { - spin_lock_irq(&crtc->dev->event_lock); - drm_crtc_send_vblank_event(crtc, crtc->state->event); - spin_unlock_irq(&crtc->dev->event_lock); - - crtc->state->event = NULL; - } -} - -static void neo_apple_crtc_atomic_begin(struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - struct neo_apple_crtc *neo_apple_crtc = to_apple_crtc(crtc); - unsigned long flags; - - if (crtc->state->event) { - spin_lock_irqsave(&crtc->dev->event_lock, flags); - neo_apple_crtc->event = crtc->state->event; - spin_unlock_irqrestore(&crtc->dev->event_lock, flags); - crtc->state->event = NULL; - } -} - -static void neo_apple_crtc_cleanup(struct drm_crtc *crtc) -{ - drm_crtc_cleanup(crtc); - kfree(to_apple_crtc(crtc)); -} - -static int neo_apple_crtc_parse_crc_source(const char *source, bool *enabled) -{ - int ret = 0; - - if (!source) { - *enabled = false; - } else if (strcmp(source, "auto") == 0) { - *enabled = true; - } else { - *enabled = false; - ret = -EINVAL; - } - - return ret; -} - -static int neo_apple_crtc_set_crc_source(struct drm_crtc *crtc, const char *source) -{ - bool enabled = false; - - int ret = neo_apple_crtc_parse_crc_source(source, &enabled); - - if (!ret) - neo_dcp_set_crc(crtc, enabled); - - return ret; -} - -static int neo_apple_crtc_verify_crc_source(struct drm_crtc *crtc, - const char *source, - size_t *values_cnt) -{ - bool enabled; - - if (neo_apple_crtc_parse_crc_source(source, &enabled) < 0) { - pr_warn("dcp: Invalid CRC source name %s\n", source); - return -EINVAL; - } - - *values_cnt = 1; - - return 0; -} - -static const char * const neo_apple_crtc_crc_sources[] = {"auto"}; - -static const char *const * neo_apple_crtc_get_crc_sources(struct drm_crtc *crtc, - size_t *count) -{ - *count = ARRAY_SIZE(neo_apple_crtc_crc_sources); - return neo_apple_crtc_crc_sources; -} - -static const struct drm_crtc_funcs neo_apple_crtc_funcs = { - .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state, - .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state, - .destroy = neo_apple_crtc_cleanup, - .page_flip = drm_atomic_helper_page_flip, - .reset = drm_atomic_helper_crtc_reset, - .set_config = drm_atomic_helper_set_config, - .set_crc_source = neo_apple_crtc_set_crc_source, - .verify_crc_source = neo_apple_crtc_verify_crc_source, - .get_crc_sources = neo_apple_crtc_get_crc_sources, - -}; - -static const struct drm_mode_config_funcs apple_mode_config_funcs = { - .atomic_check = drm_atomic_helper_check, - .atomic_commit = drm_atomic_helper_commit, - .fb_create = drm_gem_fb_create, -}; - -static void apple_atomic_commit_tail(struct drm_atomic_state *state) -{ - struct drm_plane *plane; - struct drm_plane_state *old, *new; - int i; - - /* Retain displaced scanout even when the CRTC stays inactive. */ - for_each_oldnew_plane_in_state(state, plane, old, new, i) { - struct neo_apple_plane_state *apple_state = to_apple_plane_state(new); - struct neo_dcp_fb_reference *entry = apple_state->retirement; - - if (!entry) - continue; - - apple_state->retirement = NULL; - neo_dcp_retain_framebuffer(to_apple_crtc(old->crtc)->neo_dcp, entry); - } - - drm_atomic_helper_commit_tail_rpm(state); -} - -static const struct drm_mode_config_helper_funcs apple_mode_config_helpers = { - .atomic_commit_tail = apple_atomic_commit_tail, -}; - -static void appledrm_neo_connector_cleanup(struct drm_connector *connector) -{ - struct neo_apple_connector *neo_apple_connector = to_apple_connector(connector); - - drm_connector_cleanup(connector); - kfree(neo_apple_connector->color_elements.data); - kfree(neo_apple_connector->timing_elements.data); - kfree(neo_apple_connector->display_attributes.data); - kfree(neo_apple_connector->transport.data); - kfree(neo_apple_connector); -} - -static const struct drm_connector_funcs neo_apple_connector_funcs = { - .fill_modes = drm_helper_probe_single_connector_modes, - .destroy = appledrm_neo_connector_cleanup, - .reset = drm_atomic_helper_connector_reset, - .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, - .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, - .detect = neo_apple_connector_detect, - .debugfs_init = neo_apple_connector_debugfs_init, - .oob_hotplug_event = neo_apple_connector_oob_hotplug, -}; - -/* - * A Type-C connector is attached to every pipeline that can drive its port, so - * the atomic helper cannot pick one on its own. Follow the fabric's choice. - */ -static struct drm_encoder * -neo_apple_connector_atomic_best_encoder(struct drm_connector *conn, - struct drm_atomic_state *state) -{ - struct neo_apple_connector *neo_apple_connector = to_apple_connector(conn); - struct drm_connector_state *conn_state; - struct drm_encoder *encoder; - - /* - * Type-C tunnels are routed before the modeset and cannot move here. - * Reject stale topology and explicit assignments to another pipeline. - */ - if (neo_apple_connector->port_encoder) { - conn_state = drm_atomic_get_new_connector_state(state, conn); - if (!conn_state || !conn_state->crtc || - READ_ONCE(neo_apple_connector->neo_dcp) != - to_apple_crtc(conn_state->crtc)->neo_dcp) - return NULL; - return neo_apple_connector->port_encoder; - } - - drm_connector_for_each_possible_encoder(conn, encoder) - return encoder; - - return NULL; -} - -static const struct drm_connector_helper_funcs neo_apple_connector_helper_funcs = { - .get_modes = neo_dcp_get_modes, - .mode_valid = neo_dcp_mode_valid, - .atomic_best_encoder = neo_apple_connector_atomic_best_encoder, -}; - -static const struct drm_crtc_helper_funcs neo_apple_crtc_helper_funcs = { - .atomic_begin = neo_apple_crtc_atomic_begin, - .atomic_check = neo_dcp_crtc_atomic_check, - .atomic_flush = neo_dcp_flush, - .atomic_enable = neo_apple_crtc_atomic_enable, - .atomic_disable = neo_apple_crtc_atomic_disable, - .mode_fixup = neo_dcp_crtc_mode_fixup, -}; - -static int neo_apple_connector_create(struct drm_device *drm, - struct platform_device *neo_dcp, - struct neo_apple_crtc *crtc, - struct apple_encoder *encoder, - int connector_type, bool attach_fwnode) -{ - struct neo_apple_connector *connector; - int ret; - - connector = kzalloc_obj(*connector); - if (!connector) - return -ENOMEM; - - mutex_init(&connector->chunk_lock); - drm_connector_helper_add(&connector->base, - &neo_apple_connector_helper_funcs); - if (attach_fwnode) - connector->base.fwnode = fwnode_handle_get(neo_dcp->dev.fwnode); - - ret = drm_connector_init(drm, &connector->base, &neo_apple_connector_funcs, - connector_type); - if (ret) - goto err_free; - ret = drm_connector_attach_vrr_capable_property(&connector->base); - if (ret) - goto err_cleanup; - - connector->base.polled = DRM_CONNECTOR_POLL_HPD; - connector->connected = false; - connector->neo_dcp = neo_dcp; - INIT_WORK(&connector->hotplug_wq, neo_dcp_hotplug); - - ret = drm_connector_attach_encoder(&connector->base, &encoder->base); - if (ret) - goto err_cleanup; - neo_dcp_link(neo_dcp, crtc, connector); - - return 0; - -err_cleanup: - drm_connector_cleanup(&connector->base); - kfree(connector); - return ret; -err_free: - fwnode_handle_put(connector->base.fwnode); - kfree(connector); - return ret; -} - -static int apple_probe_per_dcp(struct device *dev, - struct drm_device *drm, - struct platform_device *neo_dcp, - int num, bool neo_dcp_ext, - struct drm_encoder **encoder, - u32 *crtc_mask) -{ - struct neo_apple_crtc *crtc; - struct apple_encoder *enc; - struct drm_plane *planes[DCP_MAX_PLANES]; - unsigned long *neo_iomfb_surfaces = neo_dcp_get_iomfb_surfaces(neo_dcp); - int ret; - int connector_type; - u32 surf; - int zpos = 0; - bool supports_l10r = !neo_dcp_fw_compat_is_12_x(neo_dcp); - struct neo_apple_dcp *neo_dcp_data = platform_get_drvdata(neo_dcp); - bool supports_xrgb2101010; - enum drm_plane_type plane_type; - - supports_xrgb2101010 = neo_dcp_data->fw_compat != DCP_FIRMWARE_H17P || - neo_dcp_data->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G; - if (bitmap_empty(neo_iomfb_surfaces, DCP_MAX_PLANES)) - return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); - - for_each_set_bit(surf, neo_iomfb_surfaces, DCP_MAX_PLANES) { - plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; - planes[zpos] = neo_apple_plane_init(drm, 1U << num, surf, - supports_l10r, - supports_xrgb2101010, - plane_type); - if (IS_ERR(planes[zpos])) - return PTR_ERR(planes[zpos]); - - ret = drm_plane_create_zpos_immutable_property(planes[zpos], zpos); - if (ret) - return ret; - - zpos++; - } - - /* - * Even though we have an overlay plane, we cannot expose it to legacy - * userspace for cursors as we cannot make the same guarantees as ye olde - * hardware cursor planes such userspace would expect us to. Modern userspace - * knows what to do with overlays. - */ - crtc = kzalloc(sizeof(*crtc), GFP_KERNEL); - if (!crtc) - return -ENOMEM; - ret = drm_crtc_init_with_planes(drm, &crtc->base, planes[0], NULL, - &neo_apple_crtc_funcs, NULL); - if (ret) - return ret; - - drm_crtc_helper_add(&crtc->base, &neo_apple_crtc_helper_funcs); - drm_crtc_enable_color_mgmt(&crtc->base, 0, true, 0); - - crtc->neo_dcp = neo_dcp; - *crtc_mask = drm_crtc_mask(&crtc->base); - connector_type = neo_dcp_get_connector_type(neo_dcp); - - /* - * Type-C outputs are physical ports, not pipelines: several DCPs may be - * able to drive one port, and which one does can change at runtime. - * Those connectors are created per port once every pipeline is known, - * so a pipeline whose only output is Type-C gets no connector here -- - * and no encoder either, because the port's encoder spans every - * pipeline that can drive it. - */ - if (connector_type == DRM_MODE_CONNECTOR_USB) { - neo_dcp_link(neo_dcp, crtc, NULL); - *encoder = NULL; - return 0; - } - - enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, - DRM_MODE_ENCODER_TMDS); - if (IS_ERR(enc)) - return PTR_ERR(enc); - enc->base.possible_crtcs = *crtc_mask; - *encoder = &enc->base; - - return neo_apple_connector_create(drm, neo_dcp, crtc, enc, connector_type, - neo_dcp_ext); -} - -/* - * Create a connector per physical Type-C port and attach it to every - * pipeline that can drive it. The M2 Pro/Max laptops append a second - * connector per port for a dock's second DP tunnel, preserving primary - * connector names. - */ -static int apple_probe_typec_ports(struct drm_device *drm, - struct platform_device **neo_dcp, - u32 *crtc_mask, int num_dcp) -{ - unsigned int idx, nr_ports = neo_dcp_typec_nr_ports(); - int i, ret; - - /* Keep the existing physical-port connector numbers stable. On the - * M2 Pro/Max laptops, append one more connector per port for a second - * USB4 DP tunnel. - */ - for (idx = 0; idx < nr_ports * (neo_dcp_typec_dual_stream() ? 2 : 1); - idx++) { - struct neo_apple_connector *connector; - struct apple_encoder *enc; - unsigned int port_idx = idx % nr_ports; - bool secondary = idx >= nr_ports; - u32 mask = 0; - - connector = kzalloc_obj(*connector); - if (!connector) - return -ENOMEM; - - mutex_init(&connector->chunk_lock); - drm_connector_helper_add(&connector->base, - &neo_apple_connector_helper_funcs); - - ret = drm_connector_init(drm, &connector->base, - &neo_apple_connector_funcs, - DRM_MODE_CONNECTOR_DisplayPort); - if (ret) { - kfree(connector); - return ret; - } - - ret = drm_connector_attach_vrr_capable_property(&connector->base); - if (ret) - return ret; - - connector->base.polled = DRM_CONNECTOR_POLL_HPD; - connector->connected = false; - connector->neo_dcp = NULL; - INIT_WORK(&connector->hotplug_wq, neo_dcp_hotplug); - - for (i = 0; i < num_dcp; i++) { - if (!neo_dcp_typec_port_has_candidate(port_idx, neo_dcp[i])) - continue; - /* - * A dock's second stream (DPIN1) always runs on the - * lowest-indexed Type-C-only pipeline (dcpext1). Fix the - * connector to it: compositors read possible_crtcs once - * and would otherwise pair it with a pipeline the stream - * can never be routed to. - */ - if (secondary) { - if (neo_dcp_is_typec_only(neo_dcp[i]) && - (!mask || crtc_mask[i] < mask)) - mask = crtc_mask[i]; - continue; - } - mask |= crtc_mask[i]; - } - - if (!mask) { - drm_warn(drm, "Type-C port %u has no display pipeline\n", - port_idx); - return -ENODEV; - } - - /* - * One encoder for the port, spanning every pipeline that can - * drive it. Attaching one encoder per pipeline would describe - * the same hardware, but userspace takes the CRTCs a connector - * can use to be what all of its encoders have in common, and - * single-pipeline encoders have nothing in common -- the port - * ends up with no usable CRTC and the monitor is left dark. - */ - enc = drmm_simple_encoder_alloc(drm, struct apple_encoder, base, - DRM_MODE_ENCODER_TMDS); - if (IS_ERR(enc)) - return PTR_ERR(enc); - enc->base.possible_crtcs = mask; - - ret = drm_connector_attach_encoder(&connector->base, &enc->base); - if (ret) - return ret; - connector->port_encoder = &enc->base; - connector->candidate_crtcs = mask; - - neo_dcp_typec_port_set_connector(port_idx, secondary, connector); - } - - return 0; -} - -static int apple_get_fb_resource(struct device *dev, const char *name, - struct resource *fb_r) -{ - int idx, ret = -ENODEV; - struct device_node *node; - - idx = of_property_match_string(dev->of_node, "memory-region-names", name); - - node = of_parse_phandle(dev->of_node, "memory-region", idx); - if (!node) { - dev_err(dev, "reserved-memory node '%s' not found\n", name); - return -ENODEV; - } - - if (!of_device_is_available(node)) { - dev_err(dev, "reserved-memory node '%s' is unavailable\n", name); - goto err; - } - - if (!of_device_is_compatible(node, "framebuffer")) { - dev_err(dev, "reserved-memory node '%s' is incompatible\n", - node->full_name); - goto err; - } - - ret = of_address_to_resource(node, 0, fb_r); - -err: - of_node_put(node); - return ret; -} - -static const struct of_device_id neo_apple_dcp_id_tbl[] = { - { .compatible = "apple,dcp" }, - { .compatible = "apple,dcpext" }, - {}, -}; - -/* - * The T8140 external display coprocessor needs firmware support this driver - * does not have yet. Leave it out of the display subsystem, so that an - * enabled node cannot hold back the internal panel's component master. - */ -static bool neo_apple_dcp_usable(struct device_node *np) -{ - return of_device_is_available(np) && - !of_device_is_compatible(np, "apple,t8140-dcpext"); -} - -static int neo_apple_drm_init_dcp(struct device *dev) -{ - struct neo_apple_drm_private *apple = dev_get_drvdata(dev); - struct platform_device *neo_dcp[MAX_COPROCESSORS]; - struct drm_encoder *encoder[MAX_COPROCESSORS]; - u32 crtc_mask[MAX_COPROCESSORS] = {}; - struct device_node *np; - u64 timeout; - int i, ret, num_dcp = 0; - - for_each_matching_node(np, neo_apple_dcp_id_tbl) { - bool neo_dcp_ext; - if (!neo_apple_dcp_usable(np)) - continue; - neo_dcp_ext = of_device_is_compatible(np, "apple,dcpext") || - of_property_present(np, "phys"); - - neo_dcp[num_dcp] = of_find_device_by_node(np); - if (!neo_dcp[num_dcp]) - continue; - - device_link_add(dev, &neo_dcp[num_dcp]->dev, DL_FLAG_AUTOREMOVE_SUPPLIER); - - ret = apple_probe_per_dcp(dev, &apple->drm, neo_dcp[num_dcp], - num_dcp, neo_dcp_ext, &encoder[num_dcp], - &crtc_mask[num_dcp]); - if (ret) - continue; - - num_dcp++; - } - - if (num_dcp < 1) - return -ENODEV; - - /* - * Build the Type-C port connectors before starting any pipeline: a - * pipeline can be handed a route as soon as the Type-C mux notifies, - * and it needs somewhere to report the display. - */ - ret = apple_probe_typec_ports(&apple->drm, neo_dcp, crtc_mask, num_dcp); - if (ret) - return ret; - - for (i = 0; i < num_dcp; i++) { - ret = neo_dcp_start(neo_dcp[i]); - if (ret) - dev_warn(dev, "DCP[%d] failed to start: %d\n", i, ret); - } - - /* - * Starting DPTX might take some time. - */ - timeout = get_jiffies_64() + msecs_to_jiffies(3000); - - for (i = 0; i < num_dcp; ++i) { - u64 jiffies = get_jiffies_64(); - u64 wait = time_after_eq64(jiffies, timeout) ? - 0 : - timeout - jiffies; - ret = neo_dcp_wait_ready(neo_dcp[i], wait); - /* There is nothing we can do if a dcp/dcpext does not boot - * (successfully). Ignoring it should not do any harm now. - * Needs to reevaluated when adding dcpext support. - */ - if (ret) - dev_warn(dev, "DCP[%d] not ready: %d\n", i, ret); - } - /* HACK: Wait for dcp* to settle before a modeset */ - msleep(100); - - return 0; -} - -static int neo_apple_drm_init(struct device *dev) -{ - struct neo_apple_drm_private *apple; - struct resource fb_r; - resource_size_t fb_size; - int ret; - - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); - if (ret) - return ret; - - ret = apple_get_fb_resource(dev, "framebuffer", &fb_r); - if (ret) - return ret; - - apple = devm_drm_dev_alloc(dev, &neo_apple_drm_driver, - struct neo_apple_drm_private, drm); - if (IS_ERR(apple)) - return PTR_ERR(apple); - - dev_set_drvdata(dev, apple); - - ret = component_bind_all(dev, apple); - if (ret) - return ret; - - ret = drmm_mode_config_init(&apple->drm); - if (ret) - goto err_unbind; - - /* - * IOMFB::UPPipeDCP_H13P::verify_surfaces produces the error "plane - * requires a minimum of 32x32 for the source buffer" if smaller - */ - apple->drm.mode_config.min_width = 32; - apple->drm.mode_config.min_height = 32; - - /* - * TODO: this is the max framebuffer size not the maximal supported - * output resolution. DCP reports the maximal framebuffer size take it - * from there. - * Hardcode it for now to the M1 Max DCP reported 'MaxSrcBufferWidth' - * and 'MaxSrcBufferHeight' of 16384. - */ - apple->drm.mode_config.max_width = 16384; - apple->drm.mode_config.max_height = 16384; - - apple->drm.mode_config.funcs = &apple_mode_config_funcs; - apple->drm.mode_config.helper_private = &apple_mode_config_helpers; - - ret = neo_apple_drm_init_dcp(dev); - if (ret) - goto err_unbind; - - drm_mode_config_reset(&apple->drm); - - fb_size = fb_r.end - fb_r.start + 1; - ret = aperture_remove_conflicting_devices(fb_r.start, fb_size, - neo_apple_drm_driver.name); - if (ret) { - dev_err(dev, "Failed remove fb: %d\n", ret); - goto err_unbind; - } - - ret = drm_dev_register(&apple->drm, 0); - if (ret) - goto err_unbind; - - /* the fabric keeps order only once registered: catch up on probe */ - neo_dcp_typec_reorder(); - - drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); - - return 0; - -err_unbind: - component_unbind_all(dev, NULL); - return ret; -} - -static void neo_apple_drm_uninit(struct device *dev) -{ - struct neo_apple_drm_private *apple = dev_get_drvdata(dev); - - drm_dev_unplug(&apple->drm); - drm_atomic_helper_shutdown(&apple->drm); - - component_unbind_all(dev, NULL); - - dev_set_drvdata(dev, NULL); -} - -static int neo_apple_drm_bind(struct device *dev) -{ - return neo_apple_drm_init(dev); -} - -static void neo_apple_drm_unbind(struct device *dev) -{ - neo_apple_drm_uninit(dev); -} - -const struct component_master_ops neo_apple_drm_ops = { - .bind = neo_apple_drm_bind, - .unbind = neo_apple_drm_unbind, -}; - -static int add_dcp_components(struct device *dev, - struct component_match **matchptr) -{ - struct device_node *np, *endpoint, *port; - int num = 0; - - for_each_matching_node(np, neo_apple_dcp_id_tbl) { - if (neo_apple_dcp_usable(np)) { - drm_of_component_match_add(dev, matchptr, - component_compare_of, np); - num++; - for_each_endpoint_of_node(np, endpoint) { - port = of_graph_get_remote_port_parent(endpoint); - if (!port) - continue; - -#if !IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) - if (of_device_is_compatible(port, "apple,dpaudio")) { - of_node_put(port); - continue; - } -#endif - - /* - * The ATC phy driver is not part of the component - * collection for the Apple display-subsystem so - * ignore it here. - */ - if (of_device_is_compatible(port, "apple,t8103-atcphy")) { - of_node_put(port); - continue; - } - - if (of_device_is_available(port)) - drm_of_component_match_add(dev, matchptr, - component_compare_of, - port); - of_node_put(port); - } - } - } - - return num; -} - -static int apple_platform_probe(struct platform_device *pdev) -{ - struct device *mdev = &pdev->dev; - struct component_match *match = NULL; - int num_dcp; - - /* add DCP components, handle less than 1 as probe error */ - num_dcp = add_dcp_components(mdev, &match); - if (num_dcp < 1) - return -ENODEV; - - return component_master_add_with_match(mdev, &neo_apple_drm_ops, match); -} - -static void apple_platform_remove(struct platform_device *pdev) -{ - component_master_del(&pdev->dev, &neo_apple_drm_ops); -} - -static const struct of_device_id of_match[] = { - { .compatible = "apple,t8140-display-subsystem" }, - {} -}; -MODULE_DEVICE_TABLE(of, of_match); - -#ifdef CONFIG_PM_SLEEP -static int apple_platform_suspend(struct device *dev) -{ - struct neo_apple_drm_private *apple = dev_get_drvdata(dev); - - if (apple) - return drm_mode_config_helper_suspend(&apple->drm); - - return 0; -} - -static int apple_platform_resume(struct device *dev) -{ - struct neo_apple_drm_private *apple = dev_get_drvdata(dev); - - if (apple) - drm_mode_config_helper_resume(&apple->drm); - - return 0; -} - -static const struct dev_pm_ops apple_platform_pm_ops = { - .suspend = apple_platform_suspend, - .resume = apple_platform_resume, -}; -#endif - -static struct platform_driver apple_platform_driver = { - .driver = { - .name = "apple-drm-neo", - .of_match_table = of_match, -#ifdef CONFIG_PM_SLEEP - .pm = &apple_platform_pm_ops, -#endif - }, - .probe = apple_platform_probe, - .remove = apple_platform_remove, -}; - - - -static int __init appledrm_neo_register(void) -{ - int ret; - - /* The generic audio compatible belongs to the legacy driver on other SoCs. */ - if (!of_machine_is_compatible("apple,t8140") || drm_firmware_drivers_only()) - return -ENODEV; - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) - ret = neo_dcp_audio_register(); - if (ret) - return ret; -#endif - ret = neo_dcp_register(); - if (ret) - goto unregister_audio; - ret = platform_driver_register(&apple_platform_driver); - if (!ret) - return 0; - - neo_dcp_unregister(); -unregister_audio: -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) - neo_dcp_audio_unregister(); -#endif - return ret; -} - -static void __exit appledrm_neo_unregister(void) -{ -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) - neo_dcp_audio_unregister(); -#endif - neo_dcp_unregister(); - platform_driver_unregister(&apple_platform_driver); -} - -module_init(appledrm_neo_register); -module_exit(appledrm_neo_unregister); - -MODULE_AUTHOR("Asahi Linux contributors"); -MODULE_DESCRIPTION(DRIVER_DESC); -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple-neo/audio.c b/drivers/gpu/drm/apple-neo/audio.c deleted file mode 100644 index f9eea5e6fa7b07..00000000000000 --- a/drivers/gpu/drm/apple-neo/audio.c +++ /dev/null @@ -1,815 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * DCP Audio Bits - * - * Copyright (C) The Asahi Linux Contributors - * - * TODO: - * - figure some nice identification of the sound card (in case - * there's many DCP instances) - */ - -#define DEBUG - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "av.h" -#include "dcp.h" -#include "audio.h" -#include "parser.h" - -#define DCPAUD_ELEMENTS_MAXSIZE 16384 -#define DCPAUD_PRODUCTATTRS_MAXSIZE 1024 - -struct neo_dcp_audio { - struct device *dev; - struct device *neo_dcp_dev; - struct device *dma_dev; - struct device_link *dma_link; - struct dma_chan *chan; - struct snd_card *card; - struct snd_jack *jack; - struct snd_pcm_substream *substream; - unsigned int open_cookie; - bool open_unplugged; - - struct mutex data_lock; - bool neo_dcp_connected; /// dcp status keep for delayed initialization - bool connected; - unsigned int connection_cookie; - - struct snd_pcm_chmap_elem selected_chmap; - struct neo_dcp_sound_cookie selected_cookie; - void *elements; - void *productattrs; - - struct snd_pcm_chmap *chmap_info; -}; - -static const struct snd_pcm_hardware neo_dcp_pcm_hw = { - .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED, - .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_LE | - SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE, - .rates = SNDRV_PCM_RATE_CONTINUOUS, - .rate_min = 0, - .rate_max = UINT_MAX, - .channels_min = 1, - .channels_max = 16, - .buffer_bytes_max = SIZE_MAX, - .period_bytes_min = 4096, /* TODO */ - .period_bytes_max = SIZE_MAX, - .periods_min = 2, - .periods_max = UINT_MAX, -}; - -/* - * Without a sink the DCP has no audio elements to derive constraints from. - * Offer a plain stereo format so that the PCM can still be opened and - * configured, as HDMI PCMs on other platforms can. Userspace (PipeWire, - * PulseAudio) probes the PCM once when the card appears and drops the - * output for good if that fails, so a monitor attached later would never - * show up. Starting the stream still fails until a sink is connected. - */ -static const struct snd_pcm_hardware neo_dcp_pcm_hw_unplugged = { - .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED, - .formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE | - SNDRV_PCM_FMTBIT_S32_LE, - .rates = SNDRV_PCM_RATE_32000 | SNDRV_PCM_RATE_44100 | - SNDRV_PCM_RATE_48000, - .rate_min = 32000, - .rate_max = 48000, - .channels_min = 2, - .channels_max = 2, - .buffer_bytes_max = SIZE_MAX, - .period_bytes_min = 4096, - .period_bytes_max = SIZE_MAX, - .periods_min = 2, - .periods_max = UINT_MAX, -}; - -static int neo_dcpaud_read_remote_info(struct neo_dcp_audio *neo_dcpaud) -{ - int ret; - - ret = neo_dcp_audiosrv_get_elements(neo_dcpaud->neo_dcp_dev, neo_dcpaud->elements, - DCPAUD_ELEMENTS_MAXSIZE); - if (ret < 0) - return ret; - - ret = neo_dcp_audiosrv_get_product_attrs(neo_dcpaud->neo_dcp_dev, neo_dcpaud->productattrs, - DCPAUD_PRODUCTATTRS_MAXSIZE); - if (ret < 0) - return ret; - - return 0; -} - -static int neo_dcpaud_interval_bitmask(struct snd_interval *i, - unsigned int mask) -{ - struct snd_interval range; - if (!mask) - return -EINVAL; - - snd_interval_any(&range); - range.min = __ffs(mask); - range.max = __fls(mask); - return snd_interval_refine(i, &range); -} - -extern const struct snd_pcm_hw_constraint_list snd_pcm_known_rates; - -static void neo_dcpaud_fill_fmt_sieve(struct snd_pcm_hw_params *params, - struct neo_dcp_sound_format_mask *sieve) -{ - struct snd_interval *c = hw_param_interval(params, - SNDRV_PCM_HW_PARAM_CHANNELS); - struct snd_interval *r = hw_param_interval(params, - SNDRV_PCM_HW_PARAM_RATE); - struct snd_mask *f = hw_param_mask(params, - SNDRV_PCM_HW_PARAM_FORMAT); - int i; - - sieve->nchans = GENMASK(c->max, c->min); - sieve->formats = f->bits[0] | ((u64) f->bits[1]) << 32; /* TODO: don't open-code */ - - for (i = 0; i < snd_pcm_known_rates.count; i++) { - unsigned int rate = snd_pcm_known_rates.list[i]; - - if (snd_interval_test(r, rate)) - sieve->rates |= 1u << i; - } -} - -static void neo_dcpaud_consult_elements(struct neo_dcp_audio *neo_dcpaud, - struct snd_pcm_hw_params *params, - struct neo_dcp_sound_format_mask *hits) -{ - struct neo_dcp_sound_format_mask sieve; - struct neo_dcp_parse_ctx elements = { - .neo_dcp = dev_get_drvdata(neo_dcpaud->neo_dcp_dev), - .blob = neo_dcpaud->elements + 4, - .len = DCPAUD_ELEMENTS_MAXSIZE - 4, - .pos = 0, - }; - - neo_dcpaud_fill_fmt_sieve(params, &sieve); - dev_dbg(neo_dcpaud->dev, "elements in: %llx %x %x\n", sieve.formats, sieve.nchans, sieve.rates); - neo_parse_sound_constraints(&elements, &sieve, hits); - dev_dbg(neo_dcpaud->dev, "elements out: %llx %x %x\n", hits->formats, hits->nchans, hits->rates); -} - -static int neo_dcpaud_select_cookie(struct neo_dcp_audio *neo_dcpaud, - struct snd_pcm_hw_params *params) -{ - struct neo_dcp_sound_format_mask sieve; - struct neo_dcp_parse_ctx elements = { - .neo_dcp = dev_get_drvdata(neo_dcpaud->neo_dcp_dev), - .blob = neo_dcpaud->elements + 4, - .len = DCPAUD_ELEMENTS_MAXSIZE - 4, - .pos = 0, - }; - - neo_dcpaud_fill_fmt_sieve(params, &sieve); - return neo_parse_sound_mode(&elements, &sieve, &neo_dcpaud->selected_chmap, - &neo_dcpaud->selected_cookie); -} - -static int neo_dcpaud_rule_channels(struct snd_pcm_hw_params *params, - struct snd_pcm_hw_rule *rule) -{ - struct neo_dcp_audio *neo_dcpaud = rule->private; - struct snd_interval *c = hw_param_interval(params, - SNDRV_PCM_HW_PARAM_CHANNELS); - struct neo_dcp_sound_format_mask hits = {0, 0, 0}; - - neo_dcpaud_consult_elements(neo_dcpaud, params, &hits); - - return neo_dcpaud_interval_bitmask(c, hits.nchans); -} - -static int neo_dcpaud_refine_fmt_mask(struct snd_mask *m, u64 mask) -{ - struct snd_mask mask_mask; - - if (!mask) - return -EINVAL; - mask_mask.bits[0] = mask; - mask_mask.bits[1] = mask >> 32; - - return snd_mask_refine(m, &mask_mask); -} - -static int neo_dcpaud_rule_format(struct snd_pcm_hw_params *params, - struct snd_pcm_hw_rule *rule) -{ - struct neo_dcp_audio *neo_dcpaud = rule->private; - struct snd_mask *f = hw_param_mask(params, - SNDRV_PCM_HW_PARAM_FORMAT); - struct neo_dcp_sound_format_mask hits; - - neo_dcpaud_consult_elements(neo_dcpaud, params, &hits); - - return neo_dcpaud_refine_fmt_mask(f, hits.formats); -} - -static int neo_dcpaud_rule_rate(struct snd_pcm_hw_params *params, - struct snd_pcm_hw_rule *rule) -{ - struct neo_dcp_audio *neo_dcpaud = rule->private; - struct snd_interval *r = hw_param_interval(params, - SNDRV_PCM_HW_PARAM_RATE); - struct neo_dcp_sound_format_mask hits; - - neo_dcpaud_consult_elements(neo_dcpaud, params, &hits); - - return snd_interval_rate_bits(r, hits.rates); -} - -static int neo_dcpaud_init_dma(struct neo_dcp_audio *neo_dcpaud) -{ - struct dma_chan *chan; - if (neo_dcpaud->chan) - return 0; - - chan = of_dma_request_slave_channel(neo_dcpaud->dev->of_node, "tx"); - /* squelch dma channel request errors, the driver will try again alter */ - if (!chan) { - dev_warn(neo_dcpaud->dev, "audio TX DMA channel request failed\n"); - return -ENXIO; - } else if (chan == ERR_PTR(-EPROBE_DEFER)) { - dev_info(neo_dcpaud->dev, "audio TX DMA channel is not ready yet\n"); - return -ENXIO; - } else if (IS_ERR(chan)) { - dev_warn(neo_dcpaud->dev, "audio TX DMA channel request failed: %ld\n", PTR_ERR(chan)); - return PTR_ERR(chan); - } - neo_dcpaud->chan = chan; - - snd_pcm_set_managed_buffer(neo_dcpaud->substream, SNDRV_DMA_TYPE_DEV_IRAM, - neo_dcpaud->chan->device->dev, 1024 * 1024, - SIZE_MAX); - - return 0; -} - -static int neo_dcp_pcm_open(struct snd_pcm_substream *substream) -{ - struct neo_dcp_audio *neo_dcpaud = substream->pcm->private_data; - struct snd_dmaengine_dai_dma_data dma_data = { - .flags = SND_DMAENGINE_PCM_DAI_FLAG_PACK, - }; - struct snd_pcm_hardware hw; - int ret; - - mutex_lock(&neo_dcpaud->data_lock); - ret = neo_dcpaud_init_dma(neo_dcpaud); - if (ret < 0) { - mutex_unlock(&neo_dcpaud->data_lock); - return ret; - } - - neo_dcpaud->open_unplugged = !neo_dcpaud->connected; - neo_dcpaud->open_cookie = neo_dcpaud->connection_cookie; - mutex_unlock(&neo_dcpaud->data_lock); - - if (neo_dcpaud->open_unplugged) { - hw = neo_dcp_pcm_hw_unplugged; - goto refine; - } - - ret = neo_dcpaud_read_remote_info(neo_dcpaud); - if (ret < 0) - return ret; - - snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_FORMAT, - neo_dcpaud_rule_format, neo_dcpaud, - SNDRV_PCM_HW_PARAM_CHANNELS, SNDRV_PCM_HW_PARAM_RATE, -1); - snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS, - neo_dcpaud_rule_channels, neo_dcpaud, - SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_RATE, -1); - snd_pcm_hw_rule_add(substream->runtime, 0, SNDRV_PCM_HW_PARAM_RATE, - neo_dcpaud_rule_rate, neo_dcpaud, - SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, -1); - - hw = neo_dcp_pcm_hw; -refine: - hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED; - hw.periods_min = 2; - hw.periods_max = UINT_MAX; - hw.period_bytes_min = 256; - hw.period_bytes_max = SIZE_MAX; // TODO dma_get_max_seg_size(dma_dev); - hw.buffer_bytes_max = SIZE_MAX; - hw.fifo_size = 16; - ret = snd_dmaengine_pcm_refine_runtime_hwparams(substream, &dma_data, - &hw, neo_dcpaud->chan); - if (ret) - return ret; - substream->runtime->hw = hw; - - return snd_dmaengine_pcm_open(substream, neo_dcpaud->chan); -} - -static int neo_dcp_pcm_close(struct snd_pcm_substream *substream) -{ - struct neo_dcp_audio *neo_dcpaud = substream->pcm->private_data; - neo_dcpaud->selected_chmap.channels = 0; - neo_dcpaud->open_unplugged = false; - - return snd_dmaengine_pcm_close(substream); -} - -static int neo_dcpaud_connection_up(struct neo_dcp_audio *neo_dcpaud) -{ - bool ret; - mutex_lock(&neo_dcpaud->data_lock); - ret = neo_dcpaud->connected && - neo_dcpaud->open_cookie == neo_dcpaud->connection_cookie; - mutex_unlock(&neo_dcpaud->data_lock); - return ret; -} - -static int neo_dcp_pcm_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params) -{ - struct neo_dcp_audio *neo_dcpaud = substream->pcm->private_data; - struct dma_slave_config slave_config; - struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream); - int ret; - - if (neo_dcpaud->open_unplugged) - return 0; - - if (!neo_dcpaud_connection_up(neo_dcpaud)) - return -ENXIO; - - ret = neo_dcpaud_select_cookie(neo_dcpaud, params); - if (ret < 0) - return ret; - if (!ret) - return -EINVAL; - - memset(&slave_config, 0, sizeof(slave_config)); - ret = snd_hwparams_to_dma_slave_config(substream, params, &slave_config); - dev_info(neo_dcpaud->dev, "snd_hwparams_to_dma_slave_config: %d\n", ret); - if (ret < 0) - return ret; - - slave_config.direction = DMA_MEM_TO_DEV; - /* - * The data entry from the DMA controller to the DPA peripheral - * is 32-bit wide no matter the actual sample size. - */ - slave_config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - - ret = dmaengine_slave_config(chan, &slave_config); - dev_info(neo_dcpaud->dev, "dmaengine_slave_config: %d\n", ret); - return ret; -} - -static int neo_dcp_pcm_hw_free(struct snd_pcm_substream *substream) -{ - struct neo_dcp_audio *neo_dcpaud = substream->pcm->private_data; - - if (!neo_dcpaud_connection_up(neo_dcpaud)) - return 0; - - return neo_dcp_audiosrv_unprepare(neo_dcpaud->neo_dcp_dev); -} - -static int neo_dcp_pcm_prepare(struct snd_pcm_substream *substream) -{ - struct neo_dcp_audio *neo_dcpaud = substream->pcm->private_data; - - /* alsa-lib prepares right after hw_params; see dcp_pcm_hw_unplugged */ - if (neo_dcpaud->open_unplugged) - return 0; - - if (!neo_dcpaud_connection_up(neo_dcpaud)) - return -ENXIO; - - return neo_dcp_audiosrv_prepare(neo_dcpaud->neo_dcp_dev, - &neo_dcpaud->selected_cookie); -} - -static int neo_dcp_pcm_trigger(struct snd_pcm_substream *substream, int cmd) -{ - struct neo_dcp_audio *neo_dcpaud = substream->pcm->private_data; - int ret; - - switch (cmd) { - case SNDRV_PCM_TRIGGER_START: - case SNDRV_PCM_TRIGGER_RESUME: - if (!neo_dcpaud_connection_up(neo_dcpaud)) - return -ENXIO; - - WARN_ON(pm_runtime_get_sync(neo_dcpaud->dev) < 0); - ret = neo_dcp_audiosrv_startlink(neo_dcpaud->neo_dcp_dev, - &neo_dcpaud->selected_cookie); - if (ret < 0) - return ret; - break; - - case SNDRV_PCM_TRIGGER_STOP: - case SNDRV_PCM_TRIGGER_SUSPEND: - break; - - default: - return -EINVAL; - } - - ret = snd_dmaengine_pcm_trigger(substream, cmd); - if (ret < 0) - return ret; - - switch (cmd) { - case SNDRV_PCM_TRIGGER_START: - case SNDRV_PCM_TRIGGER_RESUME: - break; - - case SNDRV_PCM_TRIGGER_STOP: - case SNDRV_PCM_TRIGGER_SUSPEND: - ret = neo_dcp_audiosrv_stoplink(neo_dcpaud->neo_dcp_dev); - pm_runtime_mark_last_busy(neo_dcpaud->dev); - __pm_runtime_put_autosuspend(neo_dcpaud->dev); - if (ret < 0) - return ret; - break; - } - - return 0; -} - -struct snd_pcm_ops neo_dcp_playback_ops = { - .open = neo_dcp_pcm_open, - .close = neo_dcp_pcm_close, - .hw_params = neo_dcp_pcm_hw_params, - .hw_free = neo_dcp_pcm_hw_free, - .prepare = neo_dcp_pcm_prepare, - .trigger = neo_dcp_pcm_trigger, - .pointer = snd_dmaengine_pcm_pointer, -}; - -// Transitional workaround: for the chmap control TLV, advertise options -// copied from hdmi-codec.c -#include "hdmi-codec-chmap.h" - -static int neo_dcpaud_chmap_ctl_get(struct snd_kcontrol *kcontrol, - struct snd_ctl_elem_value *ucontrol) -{ - struct snd_pcm_chmap *info = snd_kcontrol_chip(kcontrol); - struct neo_dcp_audio *neo_dcpaud = info->private_data; - unsigned int i; - - for (i = 0; i < info->max_channels; i++) - ucontrol->value.integer.value[i] = \ - (i < neo_dcpaud->selected_chmap.channels) ? - neo_dcpaud->selected_chmap.map[i] : SNDRV_CHMAP_UNKNOWN; - - return 0; -} - - -static int neo_dcpaud_create_chmap_ctl(struct neo_dcp_audio *neo_dcpaud) -{ - struct snd_pcm *pcm = neo_dcpaud->substream->pcm; - struct snd_pcm_chmap *chmap_info; - int ret; - - ret = snd_pcm_add_chmap_ctls(pcm, SNDRV_PCM_STREAM_PLAYBACK, NULL, - neo_dcp_pcm_hw.channels_max, 0, &chmap_info); - if (ret < 0) - return ret; - - chmap_info->kctl->get = neo_dcpaud_chmap_ctl_get; - chmap_info->chmap = hdmi_codec_8ch_chmaps; - chmap_info->private_data = neo_dcpaud; - - return 0; -} - -static int neo_dcpaud_create_pcm(struct neo_dcp_audio *neo_dcpaud) -{ - struct snd_card *card = neo_dcpaud->card; - struct snd_pcm *pcm; - int ret; - -#define NUM_PLAYBACK 1 -#define NUM_CAPTURE 0 - - ret = snd_pcm_new(card, card->shortname, 0, NUM_PLAYBACK, NUM_CAPTURE, &pcm); - if (ret) - return ret; - - snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &neo_dcp_playback_ops); - neo_dcpaud->substream = pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; - pcm->nonatomic = true; - pcm->private_data = neo_dcpaud; - strscpy(pcm->name, card->shortname, sizeof(pcm->name)); - - return 0; -} - -/* expects to be called with data_lock locked and unlocks it */ -static void neo_dcpaud_report_hotplug(struct neo_dcp_audio *neo_dcpaud, bool connected) -{ - struct snd_pcm_substream *substream = neo_dcpaud->substream; - - if (!neo_dcpaud->card || neo_dcpaud->connected == connected) { - mutex_unlock(&neo_dcpaud->data_lock); - return; - } - - neo_dcpaud->connected = connected; - if (connected) - neo_dcpaud->connection_cookie++; - mutex_unlock(&neo_dcpaud->data_lock); - - snd_jack_report(neo_dcpaud->jack, connected ? SND_JACK_AVOUT : 0); - - if (!connected) { - snd_pcm_stream_lock(substream); - if (substream->runtime) - snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED); - snd_pcm_stream_unlock(substream); - } -} - -static int neo_dcpaud_create_jack(struct neo_dcp_audio *neo_dcpaud) -{ - struct snd_card *card = neo_dcpaud->card; - - return snd_jack_new(card, "HDMI/DP", SND_JACK_AVOUT, - &neo_dcpaud->jack, true, false); -} - -static void neo_dcpaud_set_card_names(struct neo_dcp_audio *neo_dcpaud) -{ - struct snd_card *card = neo_dcpaud->card; - - strscpy(card->driver, "apple_dcp", sizeof(card->driver)); - strscpy(card->longname, "Apple DisplayPort", sizeof(card->longname)); - strscpy(card->shortname, "Apple DisplayPort", sizeof(card->shortname)); -} - -#ifdef CONFIG_SND_DEBUG -static void neo_dcpaud_expose_debugfs_blob(struct neo_dcp_audio *neo_dcpaud, const char *name, void *base, size_t size) -{ - struct debugfs_blob_wrapper *wrapper; - wrapper = devm_kzalloc(neo_dcpaud->dev, sizeof(*wrapper), GFP_KERNEL); - if (!wrapper) - return; - wrapper->data = base; - wrapper->size = size; - debugfs_create_blob(name, 0600, neo_dcpaud->card->debugfs_root, wrapper); -} -#else -static void neo_dcpaud_expose_debugfs_blob(struct neo_dcp_audio *neo_dcpaud, const char *name, void *base, size_t size) {} -#endif - -extern bool neo_hdmi_audio; - -static int neo_dcpaud_init_snd_card(struct neo_dcp_audio *neo_dcpaud) -{ - int ret; - if (!neo_hdmi_audio) - return -ENODEV; - - - ret = snd_card_new(neo_dcpaud->dev, SNDRV_DEFAULT_IDX1, SNDRV_DEFAULT_STR1, - THIS_MODULE, 0, &neo_dcpaud->card); - if (ret) - return ret; - - neo_dcpaud_set_card_names(neo_dcpaud); - - ret = neo_dcpaud_create_pcm(neo_dcpaud); - if (ret) - goto err_free_card; - - ret = neo_dcpaud_create_chmap_ctl(neo_dcpaud); - if (ret) - goto err_free_card; - - ret = neo_dcpaud_create_jack(neo_dcpaud); - if (ret) - goto err_free_card; - - ret = snd_card_register(neo_dcpaud->card); - if (ret) - goto err_free_card; - - return 0; -err_free_card: - dev_warn(neo_dcpaud->dev, "Failed to initialize sound card: %d\n", ret); - snd_card_free(neo_dcpaud->card); - neo_dcpaud->card = NULL; - return ret; -} - -void neo_dcpaud_connect(struct platform_device *pdev, bool connected) -{ - struct neo_dcp_audio *neo_dcpaud = platform_get_drvdata(pdev); - - mutex_lock(&neo_dcpaud->data_lock); - - neo_dcpaud_report_hotplug(neo_dcpaud, connected); -} - -void neo_dcpaud_disconnect(struct platform_device *pdev) -{ - struct neo_dcp_audio *neo_dcpaud = platform_get_drvdata(pdev); - - mutex_lock(&neo_dcpaud->data_lock); - - neo_dcpaud_report_hotplug(neo_dcpaud, false); -} - -static int neo_dcpaud_comp_bind(struct device *dev, struct device *main, void *data) -{ - struct neo_dcp_audio *neo_dcpaud = dev_get_drvdata(dev); - struct device_node *endpoint, *neo_dcp_node = NULL; - struct platform_device *neo_dcp_pdev, *dma_pdev; - struct of_phandle_args dma_spec; - int index; - int ret; - - pm_runtime_get_noresume(dev); - pm_runtime_set_active(dev); - - ret = devm_pm_runtime_enable(dev); - if (ret) - return dev_err_probe(dev, ret, "Failed to enable runtime PM: %d\n", ret); - - /* find linked DCP instance */ - endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); - if (endpoint) { - neo_dcp_node = of_graph_get_remote_port_parent(endpoint); - of_node_put(endpoint); - } - if (!neo_dcp_node || !of_device_is_available(neo_dcp_node)) { - of_node_put(neo_dcp_node); - dev_info(dev, "No audio support\n"); - goto rpm_put; - } - - index = of_property_match_string(dev->of_node, "dma-names", "tx"); - if (index < 0) { - dev_err(dev, "No dma-names property\n"); - goto rpm_put; - } - - if (of_parse_phandle_with_args(dev->of_node, "dmas", "#dma-cells", index, - &dma_spec) || !dma_spec.np) { - dev_err(dev, "Failed to parse dmas property\n"); - goto rpm_put; - } - - neo_dcp_pdev = of_find_device_by_node(neo_dcp_node); - of_node_put(neo_dcp_node); - if (!neo_dcp_pdev) { - dev_info(dev, "No DP/HDMI audio device, dcp not ready\n"); - goto rpm_put; - } - neo_dcpaud->neo_dcp_dev = &neo_dcp_pdev->dev; - - dma_pdev = of_find_device_by_node(dma_spec.np); - of_node_put(dma_spec.np); - if (!dma_pdev) { - dev_info(dev, "No DMA device\n"); - goto rpm_put; - } - neo_dcpaud->dma_dev = &dma_pdev->dev; - - neo_dcpaud->dma_link = device_link_add(dev, neo_dcpaud->dma_dev, - DL_FLAG_PM_RUNTIME | - DL_FLAG_RPM_ACTIVE | - DL_FLAG_STATELESS); - - /* ignore errors to prevent audio issues affecting the display side */ - ret = neo_dcpaud_init_snd_card(neo_dcpaud); - - if (!ret) { - neo_dcpaud_expose_debugfs_blob(neo_dcpaud, "selected_cookie", &neo_dcpaud->selected_cookie, - sizeof(neo_dcpaud->selected_cookie)); - neo_dcpaud_expose_debugfs_blob(neo_dcpaud, "elements", neo_dcpaud->elements, - DCPAUD_ELEMENTS_MAXSIZE); - neo_dcpaud_expose_debugfs_blob(neo_dcpaud, "product_attrs", neo_dcpaud->productattrs, - DCPAUD_PRODUCTATTRS_MAXSIZE); - } - -rpm_put: - pm_runtime_put(dev); - - return 0; -} - -static void neo_dcpaud_comp_unbind(struct device *dev, struct device *main, - void *data) -{ - struct neo_dcp_audio *neo_dcpaud = dev_get_drvdata(dev); - - /* snd_card_free_when_closed() checks for NULL */ - snd_card_free_when_closed(neo_dcpaud->card); - - if (neo_dcpaud->dma_link) - device_link_del(neo_dcpaud->dma_link); -} - -static const struct component_ops neo_dcpaud_comp_ops = { - .bind = neo_dcpaud_comp_bind, - .unbind = neo_dcpaud_comp_unbind, -}; - -static int neo_dcpaud_probe(struct platform_device *pdev) -{ - struct neo_dcp_audio *neo_dcpaud; - - neo_dcpaud = devm_kzalloc(&pdev->dev, sizeof(*neo_dcpaud), GFP_KERNEL); - if (!neo_dcpaud) - return -ENOMEM; - - neo_dcpaud->elements = devm_kzalloc(&pdev->dev, DCPAUD_ELEMENTS_MAXSIZE, - GFP_KERNEL); - if (!neo_dcpaud->elements) - return -ENOMEM; - - neo_dcpaud->productattrs = devm_kzalloc(&pdev->dev, DCPAUD_PRODUCTATTRS_MAXSIZE, - GFP_KERNEL); - if (!neo_dcpaud->productattrs) - return -ENOMEM; - - neo_dcpaud->dev = &pdev->dev; - mutex_init(&neo_dcpaud->data_lock); - platform_set_drvdata(pdev, neo_dcpaud); - - return component_add(&pdev->dev, &neo_dcpaud_comp_ops); -} - -static void neo_dcpaud_remove(struct platform_device *pdev) -{ - component_del(&pdev->dev, &neo_dcpaud_comp_ops); -} - -static void neo_dcpaud_shutdown(struct platform_device *pdev) -{ - component_del(&pdev->dev, &neo_dcpaud_comp_ops); -} - -static __maybe_unused int neo_dcpaud_suspend(struct device *dev) -{ - /* - * Using snd_power_change_state() does not work since the sound card - * is what resumes runtime PM. - */ - - return 0; -} - -static __maybe_unused int neo_dcpaud_resume(struct device *dev) -{ - return 0; -} - -static DEFINE_RUNTIME_DEV_PM_OPS(neo_dcpaud_pm_ops, neo_dcpaud_suspend, neo_dcpaud_resume, NULL); - -static const struct of_device_id neo_dcpaud_of_match[] = { - { .compatible = "apple,dpaudio" }, - {} -}; - -static struct platform_driver neo_dcpaud_driver = { - .driver = { - .name = "dcp-dp-audio-neo", - .of_match_table = neo_dcpaud_of_match, - .pm = pm_ptr(&neo_dcpaud_pm_ops), - }, - .probe = neo_dcpaud_probe, - .remove = neo_dcpaud_remove, - .shutdown = neo_dcpaud_shutdown, -}; - -int __init neo_dcp_audio_register(void) -{ - return platform_driver_register(&neo_dcpaud_driver); -} - -void neo_dcp_audio_unregister(void) -{ - platform_driver_unregister(&neo_dcpaud_driver); -} diff --git a/drivers/gpu/drm/apple-neo/audio.h b/drivers/gpu/drm/apple-neo/audio.h deleted file mode 100644 index a57bb7ba7e9f02..00000000000000 --- a/drivers/gpu/drm/apple-neo/audio.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef __AUDIO_H__ -#define __AUDIO_H__ - -#include - -struct device; -struct platform_device; -struct neo_dcp_sound_cookie; - -int neo_dcp_audiosrv_prepare(struct device *dev, struct neo_dcp_sound_cookie *cookie); -int neo_dcp_audiosrv_startlink(struct device *dev, struct neo_dcp_sound_cookie *cookie); -int neo_dcp_audiosrv_stoplink(struct device *dev); -int neo_dcp_audiosrv_unprepare(struct device *dev); -int neo_dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize); -int neo_dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsize); - -void neo_dcpaud_connect(struct platform_device *pdev, bool connected); -void neo_dcpaud_disconnect(struct platform_device *pdev); - -#endif /* __AUDIO_H__ */ diff --git a/drivers/gpu/drm/apple-neo/av.c b/drivers/gpu/drm/apple-neo/av.c deleted file mode 100644 index de16792c95f2ec..00000000000000 --- a/drivers/gpu/drm/apple-neo/av.c +++ /dev/null @@ -1,434 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2023 Martin PoviÅ¡er */ - -// #define DEBUG - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "audio.h" -#include "afk.h" -#include "av.h" -#include "dcp.h" -#include "dcp-internal.h" - -struct neo_dcp_av_audio_cmds { - /* commands in group 0*/ - u32 open; - u32 close; - u32 prepare; - u32 start_link; - u32 stop_link; - u32 unprepare; - /* commands in group 1*/ - u32 get_elements; - u32 get_product_attrs; -}; - -static const struct neo_dcp_av_audio_cmds neo_dcp_av_audio_cmds_v12_3 = { - .open = 6, - .close = 7, - .prepare = 8, - .start_link = 9, - .stop_link = 12, - .unprepare = 13, - .get_elements = 18, - .get_product_attrs = 20, -}; - -static const struct neo_dcp_av_audio_cmds neo_dcp_av_audio_cmds_v13_5 = { - .open = 4, - .close = 5, - .prepare = 6, - .start_link = 7, - .stop_link = 10, - .unprepare = 11, - .get_elements = 16, - .get_product_attrs = 18, -}; - -struct audiosrv_data { - struct platform_device *neo_audio_dev; - bool plugged; - struct mutex plug_lock; - - struct neo_apple_epic_service *srv; - struct rw_semaphore srv_rwsem; - /* Workqueue for starting the audio service */ - struct work_struct start_av_service_wq; - - struct neo_dcp_av_audio_cmds cmds; - - bool warned_get_elements; - bool warned_get_product_attrs; - bool is_open; -}; - -static void neo_av_interface_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ -} - -static void neo_av_interface_teardown(struct neo_apple_epic_service *service) -{ - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - struct audiosrv_data *asrv = neo_dcp->audiosrv; - - neo_afk_service_disable(service); - - mutex_lock(&asrv->plug_lock); - - asrv->plugged = false; - if (asrv->neo_audio_dev) - neo_dcpaud_disconnect(asrv->neo_audio_dev); - - mutex_unlock(&asrv->plug_lock); -} - -static void neo_av_audiosrv_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - struct audiosrv_data *asrv = neo_dcp->audiosrv; - - mutex_lock(&asrv->plug_lock); - - down_write(&asrv->srv_rwsem); - asrv->srv = service; - up_write(&asrv->srv_rwsem); - - asrv->plugged = true; - mutex_unlock(&asrv->plug_lock); - schedule_work(&asrv->start_av_service_wq); -} - -static void neo_av_audiosrv_teardown(struct neo_apple_epic_service *service) -{ - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - struct audiosrv_data *asrv = neo_dcp->audiosrv; - - mutex_lock(&asrv->plug_lock); - - down_write(&asrv->srv_rwsem); - asrv->srv = NULL; - up_write(&asrv->srv_rwsem); - - asrv->plugged = false; - if (asrv->neo_audio_dev) - neo_dcpaud_disconnect(asrv->neo_audio_dev); - - mutex_unlock(&asrv->plug_lock); -} - -int neo_dcp_audiosrv_prepare(struct device *dev, struct neo_dcp_sound_cookie *cookie) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - struct audiosrv_data *asrv = neo_dcp->audiosrv; - int ret; - - down_write(&asrv->srv_rwsem); - ret = neo_afk_service_call(asrv->srv, 0, asrv->cmds.prepare, cookie, - sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, - 64); - up_write(&asrv->srv_rwsem); - - return ret; -} - -int neo_dcp_audiosrv_startlink(struct device *dev, struct neo_dcp_sound_cookie *cookie) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - struct audiosrv_data *asrv = neo_dcp->audiosrv; - int ret; - - down_write(&asrv->srv_rwsem); - ret = neo_afk_service_call(asrv->srv, 0, asrv->cmds.start_link, cookie, - sizeof(*cookie), 64 - sizeof(*cookie), NULL, 0, - 64); - up_write(&asrv->srv_rwsem); - - return ret; -} - -int neo_dcp_audiosrv_stoplink(struct device *dev) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - struct audiosrv_data *asrv = neo_dcp->audiosrv; - int ret; - - down_write(&asrv->srv_rwsem); - ret = neo_afk_service_call(asrv->srv, 0, asrv->cmds.stop_link, NULL, 0, 64, - NULL, 0, 64); - up_write(&asrv->srv_rwsem); - - return ret; -} - -int neo_dcp_audiosrv_unprepare(struct device *dev) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - struct audiosrv_data *asrv = neo_dcp->audiosrv; - int ret; - - down_write(&asrv->srv_rwsem); - ret = neo_afk_service_call(asrv->srv, 0, asrv->cmds.unprepare, NULL, 0, 64, - NULL, 0, 64); - up_write(&asrv->srv_rwsem); - - return ret; -} - -static int -neo_dcp_audiosrv_osobject_call(struct neo_apple_epic_service *service, u16 group, - u32 command, void *output, size_t output_maxsize, - size_t *output_size) -{ - struct { - __le64 max_size; - u8 _pad1[24]; - __le64 used_size; - u8 _pad2[8]; - } __attribute__((packed)) *hdr; - static_assert(sizeof(*hdr) == 48); - size_t bfr_len = output_maxsize + sizeof(*hdr); - void *bfr __free(kfree) = NULL; - int ret; - - bfr = kzalloc(bfr_len, GFP_KERNEL); - if (!bfr) - return -ENOMEM; - - hdr = bfr; - hdr->max_size = cpu_to_le64(output_maxsize); - ret = neo_afk_service_call(service, group, command, hdr, sizeof(*hdr), output_maxsize, - bfr, sizeof(*hdr) + output_maxsize, 0); - if (ret) - return ret; - - if (output) - memcpy(output, bfr + sizeof(*hdr), output_maxsize); - - if (output_size) - *output_size = le64_to_cpu(hdr->used_size); - - return 0; -} - -int neo_dcp_audiosrv_get_elements(struct device *dev, void *elements, size_t maxsize) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - struct audiosrv_data *asrv = neo_dcp->audiosrv; - size_t size; - int ret; - - down_write(&asrv->srv_rwsem); - ret = neo_dcp_audiosrv_osobject_call(asrv->srv, 1, asrv->cmds.get_elements, - elements, maxsize, &size); - up_write(&asrv->srv_rwsem); - - if (ret && !asrv->warned_get_elements) { - dev_err(dev, "audiosrv: error getting elements: %d\n", ret); - asrv->warned_get_elements = true; - } else if (!ret) { - dev_dbg(dev, "audiosrv: got %zd bytes worth of elements\n", size); - } - - return ret; -} - -int neo_dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsize) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - struct audiosrv_data *asrv = neo_dcp->audiosrv; - size_t size; - int ret; - - down_write(&asrv->srv_rwsem); - ret = neo_dcp_audiosrv_osobject_call(asrv->srv, 1, - asrv->cmds.get_product_attrs, attrs, - maxsize, &size); - up_write(&asrv->srv_rwsem); - - if (ret && !asrv->warned_get_product_attrs) { - dev_err(dev, "audiosrv: error getting product attributes: %d\n", ret); - asrv->warned_get_product_attrs = true; - } else if (!ret) { - dev_dbg(dev, "audiosrv: got %zd bytes worth of product attributes\n", size); - } - - return ret; -} - -static int neo_av_audiosrv_report(struct neo_apple_epic_service *service, u32 idx, - const void *data, size_t data_size) -{ - dev_dbg(service->ep->neo_dcp->dev, "got audio report %d size %zx\n", idx, data_size); -#ifdef DEBUG - print_hex_dump(KERN_DEBUG, "audio report: ", DUMP_PREFIX_NONE, 16, 1, data, data_size, true); -#endif - - return 0; -} - -static const struct neo_apple_epic_service_ops neo_avep_ops[] = { - { - .name = "DCPAVSimpleVideoInterface", - .init = neo_av_interface_init, - .teardown = neo_av_interface_teardown, - }, - { - .name = "DCPAVAudioInterface", - .init = neo_av_audiosrv_init, - .report = neo_av_audiosrv_report, - .teardown = neo_av_audiosrv_teardown, - }, - {} -}; - -void neo_av_service_connect(struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_epic_service *service; - struct audiosrv_data *asrv = neo_dcp->audiosrv; - int ret; - - scoped_guard(rwsem_write, &asrv->srv_rwsem) { - if (!asrv->srv) - return; - service = asrv->srv; - } - - /* open AV audio service */ - dev_info(neo_dcp->dev, "%s: starting audio service, plugged:%d\n", __func__, asrv->plugged); - if (asrv->is_open) - return; - - ret = neo_afk_service_call(service, 0, asrv->cmds.open, NULL, 0, 32, - NULL, 0, 32); - if (ret) { - dev_err(neo_dcp->dev, "error opening audio service: %d\n", ret); - return; - } - mutex_lock(&asrv->plug_lock); - asrv->is_open = true; - - if (asrv->neo_audio_dev) - neo_dcpaud_connect(asrv->neo_audio_dev, asrv->plugged); - mutex_unlock(&asrv->plug_lock); -} - -void neo_av_service_disconnect(struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_epic_service *service; - struct audiosrv_data *asrv = neo_dcp->audiosrv; - int ret; - - scoped_guard(rwsem_write, &asrv->srv_rwsem) { - if (!asrv->srv) - return; - service = asrv->srv; - } - - /* close AV audio service */ - dev_info(neo_dcp->dev, "%s: stopping audio service\n", __func__); - if (!asrv->is_open) - return; - - mutex_lock(&asrv->plug_lock); - - if (asrv->neo_audio_dev) - neo_dcpaud_disconnect(asrv->neo_audio_dev); - - mutex_unlock(&asrv->plug_lock); - - ret = neo_afk_service_call(service, 0, asrv->cmds.close, NULL, 0, 16, - NULL, 0, 16); - if (ret) { - dev_err(neo_dcp->dev, "error closing audio service: %d\n", ret); - } - if (service->torndown) - neo_afk_service_disable(service); - asrv->is_open = false; -} - -static void neo_av_work_service_start(struct work_struct *work) -{ - struct audiosrv_data *audiosrv_data; - struct neo_apple_dcp *neo_dcp; - - audiosrv_data = container_of(work, struct audiosrv_data, start_av_service_wq); - - scoped_guard(rwsem_read, &audiosrv_data->srv_rwsem) { - if (!audiosrv_data->srv || - !audiosrv_data->srv->ep || - !audiosrv_data->srv->ep->neo_dcp) { - pr_err("%s: dcp: av: NULL ptr during startup\n", __func__); - return; - } - neo_dcp = audiosrv_data->srv->ep->neo_dcp; - } - - neo_av_service_connect(neo_dcp); -} - -int neo_avep_init(struct neo_apple_dcp *neo_dcp) -{ - struct audiosrv_data *audiosrv_data; - struct platform_device *neo_audio_pdev; - struct device *dev = neo_dcp->dev; - struct device_node *endpoint, *neo_audio_node = NULL; - - audiosrv_data = devm_kzalloc(neo_dcp->dev, sizeof(*audiosrv_data), GFP_KERNEL); - if (!audiosrv_data) - return -ENOMEM; - init_rwsem(&audiosrv_data->srv_rwsem); - mutex_init(&audiosrv_data->plug_lock); - - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - audiosrv_data->cmds = neo_dcp_av_audio_cmds_v12_3; - break; - case DCP_FIRMWARE_V_13_5: - audiosrv_data->cmds = neo_dcp_av_audio_cmds_v13_5; - break; - default: - dev_err(neo_dcp->dev, "Audio not supported for firmware\n"); - return -ENODEV; - } - - neo_dcp->audiosrv = audiosrv_data; - INIT_WORK(&audiosrv_data->start_av_service_wq, neo_av_work_service_start); - - endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); - if (endpoint) { - neo_audio_node = of_graph_get_remote_port_parent(endpoint); - of_node_put(endpoint); - } - if (!neo_audio_node || !of_device_is_available(neo_audio_node)) { - of_node_put(neo_audio_node); - dev_info(dev, "No audio support\n"); - return 0; - } - - neo_audio_pdev = of_find_device_by_node(neo_audio_node); - of_node_put(neo_audio_node); - if (!neo_audio_pdev) { - dev_info(dev, "No DP/HDMI audio device not ready\n"); - return 0; - } - neo_dcp->audiosrv->neo_audio_dev = neo_audio_pdev; - - device_link_add(&neo_audio_pdev->dev, dev, - DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); - - neo_dcp->neo_avep = neo_afk_init(neo_dcp, AV_ENDPOINT, neo_avep_ops); - if (IS_ERR(neo_dcp->neo_avep)) - return PTR_ERR(neo_dcp->neo_avep); - neo_dcp->neo_avep->debugfs_entry = neo_dcp->ep_debugfs[AV_ENDPOINT - 0x20]; - return neo_afk_start(neo_dcp->neo_avep); -} diff --git a/drivers/gpu/drm/apple-neo/av.h b/drivers/gpu/drm/apple-neo/av.h deleted file mode 100644 index 83d5c76f2a486d..00000000000000 --- a/drivers/gpu/drm/apple-neo/av.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef __AV_H__ -#define __AV_H__ - -#include "parser.h" - -//int avep_audiosrv_startlink(struct apple_dcp *dcp, struct dcp_sound_cookie *cookie); -//int avep_audiosrv_stoplink(struct apple_dcp *dcp); - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) -void neo_av_service_connect(struct neo_apple_dcp *neo_dcp); -void neo_av_service_disconnect(struct neo_apple_dcp *neo_dcp); -#else -static inline void neo_av_service_connect(struct neo_apple_dcp *neo_dcp) { } -static inline void neo_av_service_disconnect(struct neo_apple_dcp *neo_dcp) { } -#endif - -#endif /* __AV_H__ */ diff --git a/drivers/gpu/drm/apple-neo/connector.c b/drivers/gpu/drm/apple-neo/connector.c deleted file mode 100644 index 713b3f4927151b..00000000000000 --- a/drivers/gpu/drm/apple-neo/connector.c +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ OR MIT -/* - * Copyright (C) The Asahi Linux Contributors - */ - -#include "connector.h" - -#include "linux/err.h" -#include -#include -#include -#include -#include - -#include - -#include "dcp-internal.h" - -enum neo_dcp_chunk_type { - DCP_CHUNK_COLOR_ELEMENTS, - DCP_CHUNK_TIMING_ELELMENTS, - DCP_CHUNK_DISPLAY_ATTRIBUTES, - DCP_CHUNK_TRANSPORT, - DCP_CHUNK_NUM_TYPES, -}; - -static int chunk_show(struct seq_file *m, - enum neo_dcp_chunk_type chunk_type) -{ - struct neo_apple_connector *apple_con = m->private; - struct neo_dcp_chunks *chunk = NULL; - - mutex_lock(&apple_con->chunk_lock); - - switch (chunk_type) { - case DCP_CHUNK_COLOR_ELEMENTS: - chunk = &apple_con->color_elements; - break; - case DCP_CHUNK_TIMING_ELELMENTS: - chunk = &apple_con->timing_elements; - break; - case DCP_CHUNK_DISPLAY_ATTRIBUTES: - chunk = &apple_con->display_attributes; - break; - case DCP_CHUNK_TRANSPORT: - chunk = &apple_con->transport; - break; - default: - break; - } - - if (chunk) - seq_write(m, chunk->data, chunk->length); - - mutex_unlock(&apple_con->chunk_lock); - - return 0; -} - -#define CONNECTOR_DEBUGFS_ENTRY(name, type) \ -static int chunk_ ## name ## _show(struct seq_file *m, void *data) \ -{ \ - return chunk_show(m, type); \ -} \ -static int chunk_ ## name ## _open(struct inode *inode, struct file *file) \ -{ \ - return single_open(file, chunk_ ## name ## _show, inode->i_private); \ -} \ -static const struct file_operations chunk_ ## name ## _fops = { \ - .owner = THIS_MODULE, \ - .open = chunk_ ## name ## _open, \ - .read = seq_read, \ - .llseek = seq_lseek, \ - .release = single_release, \ -} - -CONNECTOR_DEBUGFS_ENTRY(color, DCP_CHUNK_COLOR_ELEMENTS); -CONNECTOR_DEBUGFS_ENTRY(timing, DCP_CHUNK_TIMING_ELELMENTS); -CONNECTOR_DEBUGFS_ENTRY(display_attribs, DCP_CHUNK_DISPLAY_ATTRIBUTES); -CONNECTOR_DEBUGFS_ENTRY(transport, DCP_CHUNK_TRANSPORT); - -static void neo_dcp_afk_debugfs_root(struct platform_device *pdev, int ep, struct dentry *root) -{ -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_DEBUG) - struct dentry *entry = NULL; - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - switch (ep) { - case AV_ENDPOINT: - entry = debugfs_create_dir("avep", root); - break; - default: - break; - } - - if (!IS_ERR_OR_NULL(entry)) - neo_dcp->ep_debugfs[ep - 0x20] = entry; -#endif -} - -void neo_apple_connector_debugfs_init(struct drm_connector *connector, struct dentry *root) -{ - struct neo_apple_connector *apple_con = to_apple_connector(connector); - - debugfs_create_file("ColorElements", 0444, root, apple_con, - &chunk_color_fops); - debugfs_create_file("TimingElements", 0444, root, apple_con, - &chunk_timing_fops); - debugfs_create_file("DisplayAttributes", 0444, root, apple_con, - &chunk_display_attribs_fops); - debugfs_create_file("Transport", 0444, root, apple_con, - &chunk_transport_fops); - - switch (connector->connector_type) { - case DRM_MODE_CONNECTOR_DisplayPort: - case DRM_MODE_CONNECTOR_HDMIA: - neo_dcp_afk_debugfs_root(apple_con->neo_dcp, AV_ENDPOINT, root); - break; - default: - break; - } -} - -static void neo_dcp_connector_set_dict(struct neo_apple_connector *connector, - struct neo_dcp_chunks *dict, - struct neo_dcp_chunks *chunks) -{ - /* - * Not devm: the buffer is allocated by whichever DCP received the - * chunks, but a Type-C port can move to another pipeline before it is - * replaced, and freeing it against the connector's current DCP then - * fails devres_destroy() -- one WARN and one leak per dictionary. - */ - kfree(dict->data); - - *dict = *chunks; -} - -void neo_dcp_connector_update_dict(struct neo_apple_connector *connector, const char *key, - struct neo_dcp_chunks *chunks) -{ - mutex_lock(&connector->chunk_lock); - if (!strcmp(key, "ColorElements")) - neo_dcp_connector_set_dict(connector, &connector->color_elements, chunks); - else if (!strcmp(key, "TimingElements")) - neo_dcp_connector_set_dict(connector, &connector->timing_elements, chunks); - else if (!strcmp(key, "DisplayAttributes")) - neo_dcp_connector_set_dict(connector, &connector->display_attributes, chunks); - else if (!strcmp(key, "Transport")) - neo_dcp_connector_set_dict(connector, &connector->transport, chunks); - - chunks->data = NULL; - chunks->length = 0; - - mutex_unlock(&connector->chunk_lock); -} diff --git a/drivers/gpu/drm/apple-neo/connector.h b/drivers/gpu/drm/apple-neo/connector.h deleted file mode 100644 index 4d91fc2639eac6..00000000000000 --- a/drivers/gpu/drm/apple-neo/connector.h +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* "Copyright" 2021 Alyssa Rosenzweig */ - -#ifndef __APPLE_CONNECTOR_H__ -#define __APPLE_CONNECTOR_H__ - -#include - -#include -#include "drm/drm_connector.h" -#include "drm/drm_edid.h" - -struct neo_apple_connector; - -#include "dcp-internal.h" - -void neo_dcp_hotplug(struct work_struct *work); -void neo_dcp_retrain_oob(struct neo_apple_connector *connector); - -/* - * How many DCP pipelines may offer a route to one Type-C port. The Type-C mux - * class caps the mode-switch providers per connector (TYPEC_MUX_MAX_DEVS) and - * the ATC PHY already claims one of those slots, so this is generous. - */ -struct neo_apple_connector { - struct drm_connector base; - bool connected; - - /* The pipeline currently driving this connector, NULL if unrouted. */ - struct platform_device *neo_dcp; - - /* - * A Type-C connector is a physical port rather than a fixed pipeline, - * and it has one encoder covering every pipeline that can drive it. - * One encoder per pipeline would describe the same hardware, but - * userspace takes the CRTCs a connector can use to be what all of its - * encoders have in common, and single-pipeline encoders have nothing - * in common. - */ - struct drm_encoder *port_encoder; - - /* the CRTCs of every pipeline that can drive the port */ - u32 candidate_crtcs; - - const struct drm_edid *drm_edid; - - /* Workqueue for sending hotplug events to the associated device */ - struct work_struct hotplug_wq; - - struct mutex chunk_lock; - - struct neo_dcp_chunks color_elements; - struct neo_dcp_chunks timing_elements; - struct neo_dcp_chunks display_attributes; - struct neo_dcp_chunks transport; -}; - -#define to_apple_connector(x) container_of(x, struct neo_apple_connector, base) - -void neo_apple_connector_debugfs_init(struct drm_connector *connector, struct dentry *root); - -void neo_dcp_connector_update_dict(struct neo_apple_connector *connector, const char *key, - struct neo_dcp_chunks *chunks); -#endif diff --git a/drivers/gpu/drm/apple-neo/dcp-internal.h b/drivers/gpu/drm/apple-neo/dcp-internal.h deleted file mode 100644 index e698052013ad79..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp-internal.h +++ /dev/null @@ -1,475 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -#ifndef __APPLE_DCP_INTERNAL_H__ -#define __APPLE_DCP_INTERNAL_H__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dptxep.h" -#include "dcp_backlight.h" -#include "iomfb.h" -#include "iomfb_h17p.h" -#include "iomfb-state.h" -#include "iomfb_v12_3.h" -#include "iomfb_v13_3.h" -#include "epic/dpavservep.h" - -#define DCP_MAX_PLANES 4 -#define DCP_MAX_TYPEC_ROUTES 4 - -struct neo_apple_dcp; -struct neo_apple_dcp_afkep; - -/* Snapshot of a completed present, independent of the latest DRM state. */ -struct neo_iomfb_scanout_h17p { - struct neo_dcp_swap_submit_req_h17p request; - struct drm_framebuffer *fb[SWAP_SURFACES]; -}; - -/* One operation, including all of its nested replies and present completion. */ -struct neo_iomfb_transaction { - struct list_head link; - void (*start)(struct neo_apple_dcp *neo_dcp, struct neo_iomfb_transaction *transaction); - void (*release)(struct neo_iomfb_transaction *transaction); - struct neo_dcp_backlight_present backlight; - bool backlight_reserved; - bool backlight_failed; - bool brightness_only; - bool completed; -}; -struct neo_apple_dcp_typec_port; - -struct neo_apple_dcp_typec_route { - struct neo_apple_dcp *neo_dcp; - struct neo_apple_dcp_typec_port *port; - struct list_head port_link; - struct phy *phy; - struct mux_control *xbar; - struct typec_mux_dev *typec_mux; - u32 neo_dptx_phy; - u32 mux_index; - bool selected; - /* crossbar output actually selected: xbar (dpphy) or a Thunderbolt dpin */ - unsigned int tunnel_dpin; - struct mux_control *active_xbar; - bool tunnel; - /* tunnel: crossbar brought up (at DidChangeLinkConfiguration) */ - bool xbar_up; -}; - -bool neo_dcp_is_typec_output(struct neo_apple_dcp *neo_dcp); -void neo_dcp_swap_watchdog_arm(struct neo_apple_dcp *neo_dcp); -void neo_dcp_swap_watchdog_complete(struct neo_apple_dcp *neo_dcp); -bool neo_dcp_is_usb4_output(struct neo_apple_dcp *neo_dcp); -void neo_dcp_retry_placeholder_edid(struct neo_apple_dcp *neo_dcp, - const struct drm_edid *drm_edid); - -struct dcpav_service_epic; - -enum neo_dcp_firmware_version { - DCP_FIRMWARE_UNKNOWN, - DCP_FIRMWARE_V_12_3, - DCP_FIRMWARE_V_13_5, - DCP_FIRMWARE_H17P, -}; - -enum { - SYSTEM_ENDPOINT = 0x20, - TEST_ENDPOINT = 0x21, - DCP_EXPERT_ENDPOINT = 0x22, - DISP0_ENDPOINT = 0x23, - DPAVSERV_ENDPOINT = 0x28, - AV_ENDPOINT = 0x29, - DPTX_ENDPOINT = 0x2a, - HDCP_ENDPOINT = 0x2b, - REMOTE_ALLOC_ENDPOINT = 0x2d, - IOMFB_ENDPOINT = 0x37, -}; - -/* Temporary backing for a chunked transfer via setDCPAVPropStart/Chunk/End */ -struct neo_dcp_chunks { - size_t length; - void *data; -}; - -#define DCP_MAX_MAPPINGS (128) /* should be enough */ -#define MAX_DISP_REGISTERS (7) - -struct neo_dcp_mem_descriptor { - size_t size; - void *buf; - dma_addr_t dva; - struct sg_table map; - u64 reg; - /* live iommu mappings installed by dcpep_cb_map_piodma() */ - bool piodma_mapped; -}; - -/* Limit on call stack depth (arbitrary). Some nesting is required */ -#define DCP_MAX_CALL_DEPTH 8 - -typedef void (*neo_dcp_callback_t)(struct neo_apple_dcp *, void *, void *); - -struct neo_dcp_channel { - neo_dcp_callback_t callbacks[DCP_MAX_CALL_DEPTH]; - void *cookies[DCP_MAX_CALL_DEPTH]; - void *output[DCP_MAX_CALL_DEPTH]; - /* Header of each pending AP command record, as dcp_push() wrote it */ - struct neo_dcp_packet_header header[DCP_MAX_CALL_DEPTH]; - u16 end[DCP_MAX_CALL_DEPTH]; - - /* Current depth of the call stack. Less than DCP_MAX_CALL_DEPTH */ - u8 depth; - /* Already warned about busy channel */ - bool warned_busy; -}; - -struct neo_dcp_fb_reference { - struct list_head head; - struct drm_framebuffer *fb; - /* - * Id of the swap that unbinds @fb, i.e. the one that puts its - * replacement on screen. Only meaningful once @armed is set: the - * firmware assigns the id in the swap_start reply, which happens after - * the atomic commit that displaced @fb has already queued this entry. - */ - u32 swap_id; - bool armed; -}; - -#define MAX_NOTCH_HEIGHT 160 - -struct neo_dcp_brightness { - struct backlight_device *bl_dev; - u32 maximum; - u32 dac; - int nits; - int scale; - bool update; -}; - -struct audiosrv_data; - -/** laptop/AiO integrated panel parameters from DT */ -struct neo_dcp_panel { - /// panel width in millimeter - int width_mm; - /// panel height in millimeter - int height_mm; - /// panel has a mini-LED backlight - bool has_mini_led; -}; - -enum neo_dcp_iomfb_method_profile { - DCP_IOMFB_METHODS_DEFAULT, - DCP_IOMFB_METHODS_H17G, -}; - -struct neo_apple_dcp_hw_data { - u32 num_dptx_ports; - enum neo_dcp_iomfb_method_profile neo_iomfb_method_profile; - /* - * The bootloader leaves the coprocessor running. Attach to it with a - * standard RTKit wake instead of starting or restarting the ASC. - */ - bool adopt_live_session; - enum neo_dcp_firmware_version firmware_compat; - /* Firmware-visible clock aperture, independent of the AP PMGR window. */ - const struct resource *firmware_clock; - u32 firmware_scratch; - u32 firmware_request; -}; - -/* TODO: move IOMFB members to its own struct */ -struct neo_apple_dcp { - /* An adopted session has no confirmed firmware DMA stop boundary. */ - bool retain_dma; - bool quiescing; - bool drm_retained; - struct device *dev; - struct platform_device *piodma; - bool piodma_created; - struct iommu_domain *iommu_dom; - /* which of the nine cumulative A031 notify-client states to send next */ - unsigned int a031_step; - struct apple_rtkit *rtk; - struct neo_apple_crtc *crtc; - struct neo_apple_connector *connector; - - struct neo_apple_dcp_hw_data hw; - - /* firmware version and compatible firmware version */ - enum neo_dcp_firmware_version fw_compat; - - /* Coprocessor control register */ - void __iomem *coproc_reg; - - /* DCP has crashed */ - bool crashed; - - DECLARE_BITMAP(neo_iomfb_surfaces, DCP_MAX_PLANES); - - /************* IOMFB ************************************************** - * everything below is mostly used inside IOMFB but it could make * - * sense to keep some of the members in apple_dcp. * - **********************************************************************/ - - /* clock rate request by dcp in */ - struct clk *clk; - struct clk *clk_194; - - /* DCP shared memory */ - void *shmem; - - /* Display registers mappable to the DCP */ - struct resource *disp_registers[MAX_DISP_REGISTERS]; - unsigned int nr_disp_registers; - - struct resource disp_bw_scratch_res; - struct resource disp_bw_doorbell_res; - u32 disp_bw_scratch_index; - u32 disp_bw_scratch_offset; - u32 disp_bw_doorbell_index; - u32 disp_bw_doorbell_offset; - - u32 index; - - /* Bitmap of memory descriptors used for mappings made by the DCP */ - DECLARE_BITMAP(memdesc_map, DCP_MAX_MAPPINGS); - - /* Indexed table of memory descriptors */ - struct neo_dcp_mem_descriptor memdesc[DCP_MAX_MAPPINGS]; - - struct neo_dcp_channel ch_cmd, ch_oobcmd; - struct neo_dcp_channel ch_cb, ch_oobcb, ch_async, ch_oobasync; - - /* iomfb EP callback handlers */ - const neo_iomfb_cb_handler *cb_handlers; - - /* Active chunked transfer. There can only be one at a time. */ - struct neo_dcp_chunks chunks; - - /* Queued swap. Owned by the DCP to avoid per-swap memory allocation */ - union { - struct neo_dcp_swap_submit_req_v12_3 v12_3; - struct neo_dcp_swap_submit_req_v13_3 v13_3; - struct neo_dcp_swap_submit_req_h17p h17p; - } swap; - - /* swap id of the last completed swap */ - u32 last_swap_id; - /* last_swap_id is only meaningful after the first swap completes */ - bool have_swap_complete; - ktime_t swap_start; - u64 swap_submit_timestamp; - - /* Current display mode */ - struct neo_dcp_mode_state mode_state; - /* One HPD pulse after a placeholder EDID, per Type-C connection. */ - bool placeholder_retried; - u64 typec_generation; /* hpd_mutex: identifies the current connection */ - u64 placeholder_generation; - struct delayed_work placeholder_edid_wq; - bool use_timestamps; - bool vrr_enabled; - struct neo_dcp_set_digital_out_mode_req mode; - - /* completion for active turning true */ - struct completion start_done; - - /* Is the DCP booted? */ - bool active; - /* A successful initial enable keeps the H17P pipe running across DPMS. */ - bool pipe_enabled_h17p; - - /* eDP display without DP-HDMI conversion */ - bool main_display; - - /* clear all surfaces on init */ - bool surfaces_cleared; - - /* enable CRC calculation */ - bool crc_enabled; - - /* Modes valid for the connected display */ - struct neo_dcp_display_mode *modes; - unsigned int nr_modes; - - /* Attributes of the connector */ - int connector_type; - int fixed_connector_type; - - /* Attributes of the connected display */ - int width_mm, height_mm; - - unsigned notch_height; - - /* Workqueue for sending vblank events when a dcp swap is not possible */ - struct work_struct vblank_wq; - - /* Completes a Type-C swap that DCP dropped, and recovers the pipe. */ - struct delayed_work swap_watchdog_wq; - unsigned int swap_watchdog_retrains; - - /* List of referenced framebuffers pending a completed replacement swap. - * The commit and RTKit work queues share it under this lock. - */ - struct list_head swapped_out_fbs; - /* Protects swapped_out_fbs. */ - struct mutex swapped_out_fbs_lock; - - struct neo_dcp_brightness brightness; - /* Workqueue for updating the initial brightness */ - struct work_struct bl_register_wq; - struct mutex bl_register_mutex; - /* Workqueue for updating the brightness */ - struct work_struct bl_update_wq; - /* Registers the H17P backlight when the loader reports no level. */ - struct delayed_work bl_fallback_wq; - - /* integrated panel if present */ - struct neo_dcp_panel panel; - - struct neo_apple_dcp_afkep *neo_systemep; - struct completion neo_systemep_done; - - struct neo_apple_dcp_afkep *neo_ibootep; - struct neo_apple_dcp_afkep *dcpavservep; - struct neo_dcpavserv neo_dcpavserv; - - struct neo_apple_dcp_afkep *neo_avep; - struct audiosrv_data *audiosrv; - - struct neo_apple_dcp_afkep *neo_dptxep; - - struct neo_dptx_port neo_dptxport[2]; - - /* debugfs entries */ - struct dentry *ep_debugfs[0x20]; - - /* these fields are output port specific */ - struct phy *phy; - struct phy *fixed_phy; - struct mux_control *xbar; - struct typec_mux *typec_mux; - struct neo_apple_dcp_typec_route typec_routes[DCP_MAX_TYPEC_ROUTES]; - struct neo_apple_dcp_typec_route *active_typec_route; - u32 nr_typec_routes; - bool phy_managed_by_typec; - bool typec_cable_connected; - /* DPTX feeds a Thunderbolt DP IN adapter, not the Type-C PHY lanes */ - bool neo_dptx_tunnel; - /* DFP port in the DPTX target: 0 = dpphy, 1 = dpin0, 2 = dpin1 */ - u8 neo_dptx_dfp_port; - /* wakes/sleeps the Thunderbolt DP IN adapter from DCP Activate/Deactivate */ - int (*tb_dpin_set_active)(void *ctx, bool active); - void *tb_dpin_ctx; - /* - * Serializes the Thunderbolt DP IN callback and tunnel crossbar state - * between DCP apcalls and tunnel teardown; never held while waiting - * for DCP. - */ - struct mutex tb_lock; - bool tb_clock_ok; - /* CRTC powered off while the Type-C cable stays attached */ - bool typec_crtc_off; - /* IOMFB reports its video interface ready after DPTX link training. */ - struct completion typec_iomfb_hpd_ready; - struct delayed_work typec_reconnect_wq; - struct delayed_work typec_fabric_retrain_wq; - u32 typec_reconnect_tries; - - struct gpio_desc *hdmi_hpd; - struct gpio_desc *hdmi_pwren; - struct gpio_desc *dp2hdmi_pwren; - - struct mutex hpd_mutex; - - u32 neo_dptx_phy; - u32 neo_dptx_die; - u32 fixed_dptx_phy; - u32 fixed_mux_index; - bool fixed_route_selected; - struct neo_apple_connector *fixed_connector; - struct neo_apple_connector *typec_connector; - int hdmi_hpd_irq; - - /* H17P policy shared by backlight, commit, and reply workers. */ - struct { - /* Protects state and callback installation. */ - spinlock_t lock; - struct neo_dcp_backlight_state state; - void (*kick)(struct neo_apple_dcp *neo_dcp); - /* Rate limit for firmware re-send requests. */ - unsigned long resent_at; - bool resent; - } backlight; - - /* Staging wire record; serialization never mutates the atomic inputs. */ - struct neo_dcp_present_h17p present_h17p; - struct neo_dcp_present_state_h17p present_state_h17p; - DECLARE_BITMAP(unknown_callbacks, IOMFB_MAX_CB); - /* Runtime callbacks whose record size has been reported once. */ - DECLARE_BITMAP(sized_callbacks, IOMFB_MAX_CB); - - /* Serializes H17P transmit preparation with RTKit receive callbacks. */ - struct { - /* Protects queue, channel stacks, and the current owner. */ - struct mutex lock; - struct list_head pending; - struct neo_iomfb_transaction *active; - struct task_struct *owner; - struct work_struct work; - struct delayed_work timeout; - struct delayed_work backlight_retry; - unsigned long deadline; - unsigned int queued; - bool stopped; - bool backlight_queued; - atomic_t opaque_x_state; - struct neo_iomfb_scanout_h17p *scanout; - struct neo_iomfb_scanout_h17p *next_scanout; - } neo_iomfb; -}; - -void neo_iomfb_scanout_complete_h17p(struct neo_apple_dcp *neo_dcp); -void neo_iomfb_present_backlight_h17p(struct neo_apple_dcp *neo_dcp); -bool neo_iomfb_present_brightness_only_h17p(struct neo_apple_dcp *neo_dcp); -bool neo_iomfb_present_complete_h17p(struct neo_apple_dcp *neo_dcp); -void neo_iomfb_present_failed_h17p(struct neo_apple_dcp *neo_dcp); -bool neo_iomfb_apply_backlight_h17p(struct neo_apple_dcp *neo_dcp, - const struct neo_dcp_swap_submit_req_h17p *request, - struct neo_dcp_present_h17p *wire); -void neo_iomfb_apply_opaque_x_h17p(struct neo_apple_dcp *neo_dcp, neo_dcp_callback_t callback, - void *cookie); -void neo_iomfb_opaque_x_reset_h17p(struct neo_apple_dcp *neo_dcp); -int neo_iomfb_configure_backlight_h17p(struct neo_apple_dcp *neo_dcp, u32 maximum, - bool inherited_valid, u32 inherited, - bool default_valid, u32 default_nits); -void neo_iomfb_queue_init(struct neo_apple_dcp *neo_dcp); -void neo_iomfb_queue_stop(struct neo_apple_dcp *neo_dcp); -bool neo_iomfb_queue_drain(struct neo_apple_dcp *neo_dcp, unsigned long timeout); -int neo_iomfb_queue(struct neo_apple_dcp *neo_dcp, struct neo_iomfb_transaction *transaction); -void neo_iomfb_queue_crc_h17p(struct neo_apple_dcp *neo_dcp, u32 swap_id); - -void neo_dcp_drm_crtc_page_flip(struct neo_apple_dcp *neo_dcp, ktime_t now); -void neo_dcp_handle_hotplug_actions(struct neo_apple_dcp *neo_dcp, unsigned int action); - -int neo_dcp_backlight_register(struct neo_apple_dcp *neo_dcp); -int neo_dcp_backlight_update(struct neo_apple_dcp *neo_dcp); -bool neo_dcp_has_panel(struct neo_apple_dcp *neo_dcp); - -#define DCP_AUDIO_MAX_CHANS 15 - -#endif /* __APPLE_DCP_INTERNAL_H__ */ diff --git a/drivers/gpu/drm/apple-neo/dcp-link.h b/drivers/gpu/drm/apple-neo/dcp-link.h deleted file mode 100644 index 400b58ef5660de..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp-link.h +++ /dev/null @@ -1,95 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Copyright The Asahi Linux Contributors */ - -#ifndef __APPLE_DCP_LINK_H__ -#define __APPLE_DCP_LINK_H__ - -#include -#include -#include - -#include - -/* - * H17-generation DCP firmware uses the ordinary IOMFB transport once the - * INIT/SetShmem handshake is done: the same six fixed channels (CMD 0x00000, - * OOBCMD 0x08000, ASYNC 0x40000, OOBASYNC 0x48000, CB 0x60000, - * OOBCB 0x68000), selected by the context field of the message. Records are - * allocated on each channel's stack at - * off_next = off_cur + align_up(12 + in_len + out_len, 0x40) - * which is the rule dcp_packet_start() and DCP_PACKET_ALIGNMENT implement in - * iomfb.c. - * - * This header only describes the callback payloads whose layout differs from - * older firmware. - */ - -#define APPLE_DCP_H17P_CLOCK_ID_154 0x154 -#define APPLE_DCP_H17P_CLOCK_ID_194 0x194 - -struct neo_apple_dcp_h17p_clock_request { - __le32 service; - __le32 clock_id; -} __packed; - -struct neo_apple_dcp_h17p_rt_bw_config { - __le64 scratch; - __le64 clock_request; -} __packed; - -struct neo_apple_dcp_h17p_rt_bw_request { - __le32 config_null; -} __packed; - -struct neo_apple_dcp_h17p_rt_bw_reply { - struct neo_apple_dcp_h17p_rt_bw_config config; - __le32 status; -} __packed; - -static_assert(sizeof(struct neo_apple_dcp_h17p_clock_request) == 0x08); -static_assert(sizeof(struct neo_apple_dcp_h17p_rt_bw_config) == 0x10); -static_assert(sizeof(struct neo_apple_dcp_h17p_rt_bw_request) == 0x04); -static_assert(sizeof(struct neo_apple_dcp_h17p_rt_bw_reply) == 0x14); - -static inline int -neo_apple_dcp_h17p_clock_rate(const struct neo_apple_dcp_h17p_clock_request *request, - u64 rate_154, u64 rate_194, u64 *rate) -{ - if (!request || !rate) - return -EINVAL; - - switch (le32_to_cpu(request->clock_id)) { - case APPLE_DCP_H17P_CLOCK_ID_154: - *rate = rate_154; - return 0; - case APPLE_DCP_H17P_CLOCK_ID_194: - *rate = rate_194; - return 0; - default: - return -ENOENT; - } -} - -static inline void -neo_apple_dcp_h17p_rt_bw_encode(struct neo_apple_dcp_h17p_rt_bw_config *config, - u64 scratch, u64 clock_request) -{ - config->scratch = cpu_to_le64(scratch); - config->clock_request = cpu_to_le64(clock_request); -} - -/* D003: fill in the bandwidth config unless the firmware passed it as null */ -static inline int -neo_apple_dcp_h17p_rt_bw_encode_reply(const struct neo_apple_dcp_h17p_rt_bw_request *request, - struct neo_apple_dcp_h17p_rt_bw_reply *reply, - u64 scratch, u64 clock_request) -{ - if (le32_to_cpu(request->config_null) & 1) - return -EINVAL; - - neo_apple_dcp_h17p_rt_bw_encode(&reply->config, scratch, clock_request); - reply->status = cpu_to_le32(0); - return 0; -} - -#endif /* __APPLE_DCP_LINK_H__ */ diff --git a/drivers/gpu/drm/apple-neo/dcp.c b/drivers/gpu/drm/apple-neo/dcp.c deleted file mode 100644 index d44d7a52fd096b..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp.c +++ /dev/null @@ -1,3986 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "afk.h" -#include "av.h" -#include "dcp.h" -#include "dcp-internal.h" -#include "iomfb.h" -#include "parser.h" -#include "trace.h" - -#define APPLE_DCP_COPROC_CPU_CONTROL 0x44 -#define APPLE_DCP_COPROC_CPU_CONTROL_RUN BIT(4) - -struct neo_dcp_session_ref { - struct list_head link; - struct device *dev; - struct iommu_group *group; -}; - -static LIST_HEAD(neo_dcp_retained_sessions); -static DEFINE_MUTEX(neo_dcp_retained_sessions_lock); - -static bool neo_dcp_session_retained(struct device *dev) -{ - struct neo_dcp_session_ref *session; - bool retained = false; - - mutex_lock(&neo_dcp_retained_sessions_lock); - list_for_each_entry(session, &neo_dcp_retained_sessions, link) { - if (dev_fwnode(session->dev) == dev_fwnode(dev)) { - retained = true; - break; - } - } - mutex_unlock(&neo_dcp_retained_sessions_lock); - return retained; -} - -static int neo_dcp_pin_live_session(struct neo_apple_dcp *neo_dcp) -{ - struct neo_dcp_session_ref *session; - - if (!neo_dcp->hw.adopt_live_session || neo_dcp->fw_compat != DCP_FIRMWARE_H17P || - neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G || neo_dcp->retain_dma) - return 0; - session = kzalloc_obj(*session); - if (!session) - return -ENOMEM; - - /* Keep the DMA device, domain usage and driver code valid until reset. */ - session->dev = get_device(neo_dcp->dev); - session->group = iommu_group_get(neo_dcp->dev); - __module_get(THIS_MODULE); - WRITE_ONCE(neo_dcp->retain_dma, true); - mutex_lock(&neo_dcp_retained_sessions_lock); - list_add_tail(&session->link, &neo_dcp_retained_sessions); - mutex_unlock(&neo_dcp_retained_sessions_lock); - dev_info(neo_dcp->dev, "live H17P DMA resources retained until reboot; rebind unsupported\n"); - return 0; -} - -static void neo_dcp_release_dma_domain(void *data) -{ - struct neo_apple_dcp *neo_dcp = data; - - if (!neo_dcp->retain_dma) - iommu_device_unuse_default_domain(neo_dcp->dev); -} - -static void neo_dcp_release_context(void *data) -{ - struct neo_apple_dcp *neo_dcp = data; - - if (!neo_dcp->retain_dma) - kfree(neo_dcp); -} - -static void neo_dcp_release_rtkit(void *data) -{ - struct neo_apple_dcp *neo_dcp = data; - - if (neo_dcp->retain_dma) { - WRITE_ONCE(neo_dcp->quiescing, true); - apple_rtkit_free_retaining_buffers(neo_dcp->rtk); - } else { - apple_rtkit_free(neo_dcp->rtk); - } - neo_dcp->rtk = NULL; -} - -#define DCP_BOOT_TIMEOUT msecs_to_jiffies(1000) - -static bool show_notch; -module_param(show_notch, bool, 0644); -MODULE_PARM_DESC(show_notch, "Use the full display height and shows the notch"); - -bool neo_hdmi_audio; -module_param(neo_hdmi_audio, bool, 0644); -MODULE_PARM_DESC(neo_hdmi_audio, "Enable unstable HDMI audio support"); - -static bool unstable_edid = true; -module_param(unstable_edid, bool, 0644); -MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); - -struct neo_apple_dcp_typec_port { - struct list_head link; - struct list_head routes; - struct device_node *connector_np; - struct neo_apple_dcp_typec_route *owner; - struct neo_apple_dcp_typec_route *secondary_owner; - /* Keep a port on its last DCP while that pipeline remains free. */ - struct neo_apple_dcp_typec_route *preferred_route; - /* Ignore the USB4 fallback immediately following this port's DP teardown. */ - unsigned long dp_release_deadline; - /* DRM connector for this physical port, driven by whichever DCP owns it */ - struct neo_apple_connector *connector; - /* A second logical stream through this port's USB4 dock. */ - struct neo_apple_connector *secondary_connector; - /* last mux state acted on, to collapse the per-candidate notifications */ - struct typec_altmode *applied_alt; - unsigned long applied_mode; - u32 applied_status; - u32 applied_conf; - bool applied_valid; - bool hpd; - /* Direct DP-alt: the port is in DP mode, routed or not, and its HPD. */ - bool dp_wanted; - bool dp_hpd; - /* dcp_typec_rebalance_locked()'s plan for the port's two streams */ - struct neo_apple_dcp_typec_route *target; - struct neo_apple_dcp_typec_route *secondary_target; - /* left out of the plan: its planned pipeline is held by a tunnel */ - bool plan_dark; -}; - -static DEFINE_MUTEX(neo_dcp_typec_fabric_lock); -static LIST_HEAD(neo_dcp_typec_ports); - -static int neo_dcp_dptx_connect(struct neo_apple_dcp *neo_dcp, u32 port); -static void disconnected_hpd_event(struct neo_apple_connector *connector); - -bool neo_dcp_is_typec_output(struct neo_apple_dcp *neo_dcp) -{ - return neo_dcp->active_typec_route || - neo_dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB; -} - -bool neo_dcp_is_usb4_output(struct neo_apple_dcp *neo_dcp) -{ - return neo_dcp->active_typec_route && neo_dcp->active_typec_route->tunnel; -} - -static bool neo_dcp_typec_route_is_dp(const struct typec_mux_state *state) -{ - return state->alt && state->alt->svid == USB_TYPEC_DP_SID && - state->mode >= TYPEC_DP_STATE_A && - state->mode <= TYPEC_DP_STATE_F; -} - -/* Keep a live fixed output on its own pipeline. */ -static bool neo_dcp_typec_route_fixed_output_busy(struct neo_apple_dcp_typec_route *route) -{ - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - - if (neo_dcp->fixed_connector_type == DRM_MODE_CONNECTOR_USB) - return false; - if (neo_dcp->fixed_connector && neo_dcp->fixed_connector->connected) - return true; - if (neo_dcp->hdmi_hpd && gpiod_get_value_cansleep(neo_dcp->hdmi_hpd)) - return true; - - return false; -} - -static bool neo_dcp_typec_route_available(struct neo_apple_dcp_typec_route *route) -{ - return !route->neo_dcp->active_typec_route && - !neo_dcp_typec_route_fixed_output_busy(route); -} - -/* - * Machines whose USB4 docks carry two independent DP streams through one - * port: DPIN0 drives the hybrid dcpext0, DPIN1 the Type-C-only dcpext1, and - * each port has a second connector for the DPIN1 stream. - */ -bool neo_dcp_typec_dual_stream(void) -{ - return false; -} - -bool neo_dcp_is_typec_only(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - return !neo_dcp->fixed_phy; -} - -/* - * Can this route feed the given connector without changing its - * possible_crtcs? On dual-stream machines those are fixed at probe, since - * compositors read them once when the connector appears and pair the - * connector with a CRTC before this fabric has routed it. - */ -static bool neo_dcp_typec_route_fits(struct neo_apple_dcp_typec_route *route, - struct neo_apple_connector *connector) -{ - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - - if (!neo_dcp_typec_dual_stream() || !connector || !neo_dcp->crtc) - return true; - return connector->candidate_crtcs & drm_crtc_mask(&neo_dcp->crtc->base); -} - -static int neo_dcp_typec_route_activate(struct neo_apple_dcp_typec_route *route, - struct mux_control *xbar); -static int neo_dcp_typec_route_deactivate(struct neo_apple_dcp_typec_route *route); -static int neo_dcp_dpxbar_preselect(struct mux_control *mux, int state); -static int neo_dcp_dptx_disconnect(struct neo_apple_dcp *neo_dcp, u32 port); - -static int neo_dcp_dpxbar_tunnel_select_source(struct mux_control *mux, int state) -{ - typeof(&apple_dpxbar_tunnel_select_source) select = - symbol_get(apple_dpxbar_tunnel_select_source); - int ret; - - if (!select) - return -ENOENT; - ret = select(mux, state); - symbol_put(apple_dpxbar_tunnel_select_source); - return ret; -} - -/* - * For a port without a prior owner, rank pipelines by CRTC index. A pipeline - * whose fixed output is live is not a candidate at all, so a hybrid is only - * ever ranked here when it is genuinely free. - */ -static unsigned int neo_dcp_typec_route_score(struct neo_apple_dcp_typec_route *route) -{ - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - - if (!neo_dcp->crtc) - return UINT_MAX - 1; - - return drm_crtc_index(&neo_dcp->crtc->base); -} - -static int neo_dcp_typec_route_activate(struct neo_apple_dcp_typec_route *route, - struct mux_control *xbar) -{ - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - struct neo_apple_connector *connector = - xbar != route->xbar && route->tunnel_dpin == 1 ? - route->port->secondary_connector : route->port->connector; - int ret; - - /* - * The fixed output's HPD handler leaves disconnects to DCP, so the port - * can still be marked connected to a display that is gone. Release it - * (a no-op otherwise), or connecting the borrowed route returns early. - */ - neo_dcp_dptx_disconnect(neo_dcp, 0); - - if (neo_dcp->fixed_route_selected) { - ret = mux_control_deselect(neo_dcp->xbar); - if (ret) - return ret; - neo_dcp->fixed_route_selected = false; - } - - /* - * Thunderbolt DP IN: the crossbar connection may only be brought up - * once DCP has configured the link and the tunnel pixel clock runs, - * see dcp_tunnel_crossbar_up(). Just remember the output here. - */ - if (xbar != route->xbar && route->mux_index) { - /* - * The T602X crossbar must point a DP IN at its pipeline before - * DCP probes AUX; other crossbars select it at link-up and - * report -EOPNOTSUPP here. - */ - ret = neo_dcp_dpxbar_tunnel_select_source(xbar, route->mux_index); - if (ret == -EOPNOTSUPP) - ret = 0; - } else { - ret = xbar == route->xbar ? - mux_control_select(xbar, route->mux_index) : 0; - } - if (ret) { - if (neo_dcp->xbar) { - int restore_ret; - - restore_ret = mux_control_select(neo_dcp->xbar, - neo_dcp->fixed_mux_index); - if (!restore_ret) - neo_dcp->fixed_route_selected = true; - else - dev_err(neo_dcp->dev, - "failed to restore fixed display route: %d\n", - restore_ret); - } - return ret; - } - - neo_dcp->phy = route->phy; - neo_dcp->neo_dptx_phy = route->neo_dptx_phy; - neo_dcp->connector_type = DRM_MODE_CONNECTOR_USB; - if (connector) { - WRITE_ONCE(connector->neo_dcp, to_platform_device(neo_dcp->dev)); - neo_dcp->typec_connector = connector; - neo_dcp->connector = connector; - - /* - * Narrow the port to the pipeline now driving it. The encoder - * spans every pipeline that could, which is what lets userspace - * see the port as usable at all -- but only one of them is - * routed to the display, and userspace has no way to tell which. - * Offering it the choice makes it pair the port with a pipeline - * holding a different monitor's mode list, and the modeset is - * rejected with no way for it to recover. The hotplug that - * follows publishes the actual binding on dual-stream machines - * too. Clients must re-read encoder topology after a hotplug. - */ - if (connector->port_encoder && neo_dcp->crtc) - connector->port_encoder->possible_crtcs = - drm_crtc_mask(&neo_dcp->crtc->base); - } - neo_dcp->active_typec_route = route; - scoped_guard(mutex, &neo_dcp->tb_lock) { - route->active_xbar = xbar; - route->tunnel = xbar != route->xbar; - route->xbar_up = !route->tunnel; - neo_dcp->neo_dptx_tunnel = route->tunnel; - /* crossbar controls are dpphy, dpin0, dpin1: same order as the DFP port */ - neo_dcp->neo_dptx_dfp_port = route->tunnel ? xbar - &route->xbar->chip->mux[0] : 0; - neo_dcp->tb_clock_ok = false; - } - route->selected = true; - - dev_info(neo_dcp->dev, "allocated Type-C DPTX PHY %u\n", route->neo_dptx_phy); - return 0; -} - -static int neo_dcp_typec_route_deactivate(struct neo_apple_dcp_typec_route *route) -{ - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - struct neo_apple_connector *connector = neo_dcp->typec_connector; - struct mux_control *active_xbar = route->active_xbar; - bool was_tunnel = route->tunnel; - int ret = 0; - - /* - * Under tb_lock so a DCP link (re)configuration can neither select the - * crossbar nor restart the tunnel pixel clock behind our back. - */ - scoped_guard(mutex, &neo_dcp->tb_lock) { - if (!route->tunnel || route->xbar_up) - ret = mux_control_deselect(route->active_xbar ?: route->xbar); - else - neo_dcp_dpxbar_preselect(route->active_xbar, MUX_IDLE_DISCONNECT); - if (ret) - return ret; - route->xbar_up = false; - - if (route->tunnel && neo_dcp->phy) { - /* the tunnel pixel clock must not outlive the tunnel */ - typeof(&apple_atc_dp_tunnel_rate) stop = - symbol_get(apple_atc_dp_tunnel_rate); - - if (stop) { - stop(neo_dcp->phy, route->tunnel_dpin, 0); - symbol_put(apple_atc_dp_tunnel_rate); - } - } - route->active_xbar = NULL; - route->tunnel = false; - neo_dcp->neo_dptx_tunnel = false; - neo_dcp->neo_dptx_dfp_port = 0; - neo_dcp->tb_dpin_set_active = NULL; - neo_dcp->tb_dpin_ctx = NULL; - neo_dcp->tb_clock_ok = false; - } - if (was_tunnel && route->mux_index) { - int sel = neo_dcp_dpxbar_tunnel_select_source(active_xbar, -1); - - if (sel && sel != -EOPNOTSUPP) - dev_warn(neo_dcp->dev, "DP tunnel source reset failed: %d\n", sel); - } - route->selected = false; - if (neo_dcp->active_typec_route == route) - neo_dcp->active_typec_route = NULL; - - if (connector && connector->neo_dcp == to_platform_device(neo_dcp->dev)) { - /* - * Until the port is activated again it has no pipeline behind - * it, and nothing can read modes or EDID from it. Report it - * disconnected for that window: leaving a connected connector - * whose ->dcp is NULL lets anything probing it in between -- - * a compositor starting up while the fabric is still settling - * -- see an output it cannot get a mode for, and give up on - * it. The new pipeline marks it connected again once the - * display has come back up on it. - */ - WRITE_ONCE(connector->connected, false); - /* hotplug work queued before this checks for it */ - WRITE_ONCE(connector->neo_dcp, NULL); - - /* Unrouted: the port could go to any of its pipelines again. */ - if (connector->port_encoder) - connector->port_encoder->possible_crtcs = - connector->candidate_crtcs; - } - neo_dcp->typec_connector = NULL; - neo_dcp->connector = neo_dcp->fixed_connector; - - if (neo_dcp->fixed_connector_type != DRM_MODE_CONNECTOR_USB) { - neo_dcp->connector_type = neo_dcp->fixed_connector_type; - - /* - * Only hand the pipeline back to its fixed output if that output - * is live. Re-targeting the DPTX endpoint at the fixed PHY while - * nothing is attached there leaves DCP unable to train a link on a - * later Type-C target: it answers DEVICE_NOT_STARTED and every - * following DPTX call times out. Park on a Type-C PHY instead, - * for the same reason the USB-C-only case does below. - */ - if (neo_dcp->hdmi_hpd && gpiod_get_value_cansleep(neo_dcp->hdmi_hpd)) { - neo_dcp->phy = neo_dcp->fixed_phy; - neo_dcp->neo_dptx_phy = neo_dcp->fixed_dptx_phy; - - if (neo_dcp->xbar) { - ret = mux_control_select(neo_dcp->xbar, - neo_dcp->fixed_mux_index); - if (ret) - return ret; - neo_dcp->fixed_route_selected = true; - } - } else { - neo_dcp->phy = neo_dcp->typec_routes[0].phy; - neo_dcp->neo_dptx_phy = neo_dcp->typec_routes[0].neo_dptx_phy; - } - } else { - /* Keep DPTX endpoint discovery working before a cable is attached. */ - neo_dcp->phy = neo_dcp->typec_routes[0].phy; - neo_dcp->neo_dptx_phy = neo_dcp->typec_routes[0].neo_dptx_phy; - neo_dcp->connector_type = DRM_MODE_CONNECTOR_USB; - } - - return 0; -} - -static void neo_dcp_typec_retrain_work(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = - container_of(to_delayed_work(work), struct neo_apple_dcp, - typec_fabric_retrain_wq); - - struct neo_apple_connector *connector = READ_ONCE(neo_dcp->typec_connector); - - if (READ_ONCE(neo_dcp->active_typec_route) && connector) - neo_dcp_retrain_oob(connector); -} - -static void neo_dcp_typec_retrain_active_routes(void) -{ - struct neo_apple_dcp_typec_port *port; - - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - if (port->owner) - mod_delayed_work(system_freezable_wq, - &port->owner->neo_dcp->typec_fabric_retrain_wq, - msecs_to_jiffies(200)); - if (port->secondary_owner) - mod_delayed_work(system_freezable_wq, - &port->secondary_owner->neo_dcp->typec_fabric_retrain_wq, - msecs_to_jiffies(200)); - } -} - -static struct neo_apple_dcp_typec_route * -neo_dcp_typec_port_route(struct neo_apple_dcp_typec_port *port, struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_dcp_typec_route *route; - - list_for_each_entry(route, &port->routes, port_link) - if (route->neo_dcp == neo_dcp) - return route; - - return NULL; -} - -/* - * The DRM device once it has bound, that is once every pipeline has its - * CRTC and every port its connectors; NULL before. - */ -static struct drm_device *neo_dcp_typec_drm(void) -{ - struct neo_apple_dcp_typec_port *port; - struct neo_apple_dcp_typec_route *route; - struct drm_device *drm = NULL; - - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - if (!port->connector || !port->secondary_connector) - return NULL; - list_for_each_entry(route, &port->routes, port_link) { - if (!route->neo_dcp->crtc) - return NULL; - drm = route->neo_dcp->crtc->base.dev; - } - } - - return drm; -} - -/* - * Has a compositor (or boot splash) taken the display? It paired its - * connectors with CRTCs when it started and keeps that pairing, also - * across a VT switch, where it drops DRM master but keeps the device - * open; moving routes under it would hand connectors to pipelines driving - * other displays. So the routes are frozen while any open file has ever - * been master, and thaw when the last such file is closed. - * - * Taken under dcp_typec_fabric_lock, next to the routing it decides, so - * the order is the fabric lock, then filelist_mutex. Nothing nests them - * the other way: filelist_mutex only covers list edits and walks, and - * drm_release() drops it before drm_file_free() calls postclose, which is - * where a close takes the fabric lock. drm_open() makes a file master - * before adding it here, but its owner cannot have paired anything before - * open() returns, and it re-probes on the hotplugs the moves send. - */ -static bool neo_dcp_typec_frozen(struct drm_device *drm) -{ - struct drm_file *file; - - guard(mutex)(&drm->filelist_mutex); - list_for_each_entry(file, &drm->filelist, lhead) { - /* set under master_mutex, and only ever from false to true */ - if (READ_ONCE(file->was_master)) - return true; - } - - return false; -} - -/* Are the routes kept in compositor pairing order right now? */ -static bool neo_dcp_typec_keep_order(void) -{ - struct drm_device *drm; - - if (!neo_dcp_typec_dual_stream()) - return false; - drm = neo_dcp_typec_drm(); - - return drm && READ_ONCE(drm->registered) && !neo_dcp_typec_frozen(drm); -} - -/* A Thunderbolt tunnel holds @dcp's pipeline: it never moves. */ -static bool neo_dcp_typec_tunnel_held(struct neo_apple_dcp *neo_dcp) -{ - return neo_dcp->active_typec_route && neo_dcp->active_typec_route->tunnel; -} - -/* - * Give @dcp's pipeline to the first stream, in connector order, that - * wants one, has none yet and has it in its possible_crtcs. A tunnel - * stream wants one once it is set up, or as @arriving/@dpin. A direct - * DP-alt stream wants one once its sink asserts HPD: only then can its - * connector read connected, and a compositor pairs only those. - */ -static void neo_dcp_typec_plan_pipeline(struct neo_apple_dcp *neo_dcp, struct drm_crtc *crtc, - struct neo_apple_dcp_typec_port *arriving, - unsigned int dpin) -{ - struct neo_apple_dcp_typec_port *port; - bool secondary = false; - - /* primary connectors in port order, then the DPIN1 ones */ - do { - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - struct neo_apple_dcp_typec_route **slot = secondary ? - &port->secondary_target : &port->target; - struct neo_apple_connector *connector = secondary ? - port->secondary_connector : port->connector; - struct neo_apple_dcp_typec_route *route = - neo_dcp_typec_port_route(port, neo_dcp); - bool wants; - - if (secondary) - wants = port->secondary_owner || - (port == arriving && dpin == 1); - else - wants = (port->owner && port->owner->tunnel) || - (port->dp_wanted && port->dp_hpd && - !port->plan_dark) || - (port == arriving && dpin == 0); - - if (!wants || *slot || !route || - !(connector->candidate_crtcs & drm_crtc_mask(crtc))) - continue; - - *slot = route; - return; - } - secondary = !secondary; - } while (secondary); -} - -/* - * The pairing a compositor starting now would make, wherever the streams - * sit at the moment: pipelines in CRTC index order, each to the first - * stream that wants it. A pipeline whose fixed output is live is left to - * that output, unless a tunnel holds it: the output cannot have it back - * then, and its connector reads disconnected. - * - * Tunnels are planned in connector order like any other stream, not - * seated on the pipelines they hold: the compositor pairs their connectors - * by order too, and seating them would plan the direct streams around - * pairings it never makes. A direct stream planned onto a pipeline a - * tunnel holds cannot have it and stays dark, so it is left out and the - * pass re-run, or every stream after it would be planned one pipeline - * off. Streams only ever drop out, so that settles within one pass per - * port. It also brings a tunnel's plan onto the pipeline it holds - * wherever direct streams ahead of it were in the way. A tunnel still - * planned elsewhere cannot be brought there by any direct stream dropping - * out, and the compositor pairs its connector wrongly whatever they do. - */ -static void neo_dcp_typec_plan(struct drm_device *drm, - struct neo_apple_dcp_typec_port *arriving, unsigned int dpin) -{ - struct neo_apple_dcp_typec_port *port; - struct drm_crtc *crtc; - bool again; - - list_for_each_entry(port, &neo_dcp_typec_ports, link) - port->plan_dark = false; - - do { - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - port->target = NULL; - port->secondary_target = NULL; - } - - /* CRTCs are listed in index order */ - drm_for_each_crtc(crtc, drm) { - struct neo_apple_dcp *neo_dcp = - platform_get_drvdata(to_apple_crtc(crtc)->neo_dcp); - - if (neo_dcp->nr_typec_routes && - (neo_dcp_typec_tunnel_held(neo_dcp) || - !neo_dcp_typec_route_fixed_output_busy(&neo_dcp->typec_routes[0]))) - neo_dcp_typec_plan_pipeline(neo_dcp, crtc, arriving, dpin); - } - - again = false; - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - if (port == arriving || !port->target || - (port->owner && port->owner->tunnel) || - !neo_dcp_typec_tunnel_held(port->target->neo_dcp)) - continue; - port->plan_dark = true; - again = true; - } - } while (again); -} - -/* Replay HPD to the pipeline a direct DP-alt port has just been given. */ -static void neo_dcp_typec_port_attach(struct neo_apple_dcp_typec_port *port) -{ - struct neo_apple_dcp *neo_dcp = port->owner->neo_dcp; - - port->hpd = port->dp_hpd; - if (!port->hpd) - return; - - WRITE_ONCE(neo_dcp->typec_cable_connected, true); - if (neo_dcp->typec_connector) - neo_dcp_dptx_connect_oob(to_platform_device(neo_dcp->dev), 0); -} - -static struct neo_apple_dcp_typec_route * -neo_dcp_typec_lowest_free(struct neo_apple_dcp_typec_port *port) -{ - struct neo_apple_dcp_typec_route *candidate, *best = NULL; - unsigned int best_score = UINT_MAX; - - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (!neo_dcp_typec_route_available(candidate)) - continue; - /* - * Prefer the lowest free CRTC index for a stable allocation order. - */ - score = neo_dcp_typec_route_score(candidate); - if (score < best_score) { - best = candidate; - best_score = score; - } - } - - return best; -} - -/* - * Route the direct DP-alt ports left waiting for a pipeline. Their DP - * state is not reported again while it stays the same, so nothing else - * would. Each goes back to the pipeline it last had if that is free, as - * a compositor keeps a reconnected connector's CRTC, and otherwise takes - * the lowest free one, the CRTC a compositor gives a new connector. - */ -static void neo_dcp_typec_route_waiting(void) -{ - struct neo_apple_dcp_typec_port *port; - struct neo_apple_dcp_typec_route *route; - - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - if (port->owner || !port->dp_wanted || !port->dp_hpd) - continue; - - route = port->preferred_route; - if (!route || !neo_dcp_typec_route_available(route)) - route = neo_dcp_typec_lowest_free(port); - if (!route || neo_dcp_typec_route_activate(route, route->xbar)) - continue; - port->owner = route; - port->dp_release_deadline = 0; - neo_dcp_typec_port_attach(port); - } -} - -/* - * Prefer connector order when placing Type-C streams before a DRM master - * starts. This is an allocation policy, not a promise about the CRTC that - * userspace must choose: each connected encoder advertises only its actual - * route, and atomic encoder selection checks that ownership. Route changes - * publish the new mask through hotplug; the plan uses candidate_crtcs so - * narrowing the advertised mask does not remove hardware routing choices. - * - * Only direct DP-alt routes move: each goes to exactly its planned - * pipeline. A Thunderbolt tunnel never moves once set up, so where one - * sits on a pipeline planned for a direct stream, or the plan has nothing - * for it, that stream stays unrouted and its connector disconnected. - * Placing it on some other pipeline would have the compositor cross both - * displays; dark is the better failure. @arriving/@dpin is a tunnel - * stream asking for a pipeline, planned like any other; its route is - * returned for apple_dcp_tb_dp_tunnel() to set up, or NULL if the plan - * has none for it or another tunnel holds that one. - * - * A move is an unplug and replug. Every route that moves is taken down - * first, so that two never share a pipeline midway, then each goes up on - * its new pipeline with its HPD replayed; the hotplugs this sends make - * fbdev and userspace re-probe. - */ -static struct neo_apple_dcp_typec_route * -neo_dcp_typec_rebalance_locked(struct neo_apple_dcp_typec_port *arriving, - unsigned int dpin) -{ - struct drm_device *drm = neo_dcp_typec_drm(); - struct neo_apple_dcp_typec_route *planned = NULL; - struct neo_apple_dcp_typec_port *port; - - lockdep_assert_held(&neo_dcp_typec_fabric_lock); - - neo_dcp_typec_plan(drm, arriving, dpin); - if (arriving) { - planned = dpin ? arriving->secondary_target : arriving->target; - /* it will be refused: plan as if it had not asked */ - if (planned && neo_dcp_typec_tunnel_held(planned->neo_dcp)) { - planned = NULL; - neo_dcp_typec_plan(drm, NULL, 0); - } - } - - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - struct neo_apple_dcp_typec_route *owner = port->owner; - struct neo_apple_dcp *neo_dcp; - - if (!owner || owner->tunnel || owner == port->target) - continue; - - neo_dcp = owner->neo_dcp; - dev_info(neo_dcp->dev, "re-routing %pOF from %s to %s for Type-C connector order\n", - port->connector_np, dev_name(neo_dcp->dev), - port->target ? dev_name(port->target->neo_dcp->dev) : "none"); - if (port->hpd || neo_dcp->typec_cable_connected || - (neo_dcp->typec_connector && neo_dcp->typec_connector->connected)) - neo_dcp_dptx_disconnect_oob(to_platform_device(neo_dcp->dev), 0); - port->hpd = false; - if (neo_dcp_typec_route_deactivate(owner) && owner->selected) { - /* still routed: leave the display where it is */ - neo_dcp_typec_port_attach(port); - continue; - } - port->owner = NULL; - - /* as after a DP exit, hand the hybrid back to a live HDMI */ - if (neo_dcp->hdmi_hpd && neo_dcp->active && - gpiod_get_value_cansleep(neo_dcp->hdmi_hpd)) - neo_dcp_dptx_connect(neo_dcp, 0); - } - - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - struct neo_apple_dcp_typec_route *route = port->target; - - if (port->owner || !port->dp_wanted || !port->dp_hpd || - port == arriving) - continue; - - if (!route || !neo_dcp_typec_route_available(route) || - neo_dcp_typec_route_activate(route, route->xbar)) { - /* - * Dark for now. Its unchanged DP state is not reported - * again, so it is placed when the plan next runs, or - * once a compositor owns the display, when a pipeline - * is freed (dcp_typec_route_waiting()). - */ - port->applied_valid = false; - continue; - } - port->owner = route; - port->preferred_route = route; - port->dp_release_deadline = 0; - neo_dcp_typec_port_attach(port); - } - - return planned; -} - -/* - * A route has let its pipeline go. Until a compositor owns the display - * the plan places everything anew; after that the pipeline goes to a port - * left waiting for one. - */ -static void neo_dcp_typec_pipeline_freed(void) -{ - if (!neo_dcp_typec_dual_stream()) - return; - - if (neo_dcp_typec_keep_order()) - neo_dcp_typec_rebalance_locked(NULL, 0); - else - neo_dcp_typec_route_waiting(); -} - -/* - * Re-run the pairing pass from outside the fabric: once DRM is registered, - * as ports routed before that could not follow it, and when the last - * compositor or boot splash has closed the device, as the next one pairs - * the connectors from scratch. - */ -void neo_dcp_typec_reorder(void) -{ - if (!neo_dcp_typec_dual_stream()) - return; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - if (neo_dcp_typec_keep_order()) - neo_dcp_typec_rebalance_locked(NULL, 0); -} - -static int neo_dcp_typec_route_set(struct typec_mux_dev *mux, - struct typec_mux_state *state) -{ - struct neo_apple_dcp_typec_route *route = typec_mux_get_drvdata(mux); - struct neo_apple_dcp_typec_port *port = route->port; - struct neo_apple_dcp_typec_route *best = NULL; - bool is_dp = neo_dcp_typec_route_is_dp(state); - struct typec_displayport_data *dp_data = is_dp ? state->data : NULL; - u32 dp_status = dp_data ? dp_data->status : 0; - u32 dp_conf = dp_data ? dp_data->conf : 0; - bool hpd, was_counted; - int ret = 0; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - /* - * Every candidate route for this port is notified with the same state, - * so only the first to arrive does the work; the others return early. - * - * Deliberately not a nominated coordinator: fwnode_typec_mux_get() - * caps the providers one connector may have and drops the remainder - * without a word, so a nominated route might never be called at all -- - * and the port would then never be routed. - */ - if (port->applied_valid && port->applied_alt == state->alt && - port->applied_mode == state->mode && - port->applied_status == dp_status && port->applied_conf == dp_conf) - return 0; - - port->applied_alt = state->alt; - port->applied_mode = state->mode; - port->applied_status = dp_status; - port->applied_conf = dp_conf; - /* Failed route acquisition must remain retryable on the next update. */ - port->applied_valid = false; - - /* did the pairing pass count this port's direct stream so far? */ - was_counted = port->dp_wanted && port->dp_hpd; - - if (!is_dp) { - port->dp_wanted = false; - port->dp_hpd = false; - - /* a Thunderbolt/USB4 DP tunnel is torn down by its own path */ - if (port->owner && port->owner->tunnel) { - port->applied_valid = true; - return 0; - } - if (port->owner) { - struct neo_apple_dcp *neo_dcp = port->owner->neo_dcp; - - port->preferred_route = port->owner; - if (port->hpd || neo_dcp->typec_cable_connected || - (neo_dcp->typec_connector && - neo_dcp->typec_connector->connected)) - neo_dcp_dptx_disconnect_oob(to_platform_device(neo_dcp->dev), 0); - port->hpd = false; - ret = neo_dcp_typec_route_deactivate(port->owner); - if (ret) - return ret; - port->owner = NULL; - port->dp_release_deadline = jiffies + msecs_to_jiffies(10000); - if (neo_dcp->hdmi_hpd && neo_dcp->active && - gpiod_get_value_cansleep(neo_dcp->hdmi_hpd)) - neo_dcp_dptx_connect(neo_dcp, 0); - neo_dcp_typec_pipeline_freed(); - } else if (was_counted && neo_dcp_typec_keep_order()) { - /* a stream the plan left dark is gone: plan the rest */ - neo_dcp_typec_rebalance_locked(NULL, 0); - } - - /* - * A port leaving DP can report SAFE/NONE before falling back to USB4. - * Resetting every other live CRTC for that same cable removal blanks - * unaffected displays. Keep the guard across the Type-C state sequence; - * a later, independent USB4 attach still gets recovery. - */ - if (state->mode == TYPEC_MODE_USB4) { - if (!port->dp_release_deadline || - time_after_eq(jiffies, port->dp_release_deadline)) - neo_dcp_typec_retrain_active_routes(); - port->dp_release_deadline = 0; - } - port->applied_valid = true; - return 0; - } - - /* - * A Thunderbolt DP tunnel still owns the port (its teardown is on the - * way): don't act on or remember this state; the tunnel teardown drops - * what was recorded so the next update is applied. - */ - if (port->owner && port->owner->tunnel) { - port->applied_valid = false; - return 0; - } - - hpd = dp_data && (dp_data->status & DP_STATUS_HPD_STATE); - port->dp_wanted = true; - port->dp_hpd = hpd; - - /* - * Until a compositor owns the display, a direct DP-alt stream has a - * pipeline only while its sink asserts HPD, and the pairing pass - * places it: HPD coming or going is the stream connecting or - * disconnecting as far as a compositor can tell. Otherwise the port - * takes the lowest free pipeline on DP entry, as a compositor does. - */ - if (neo_dcp_typec_keep_order()) { - struct neo_apple_dcp_typec_route *owner = port->owner; - - /* it connects or disconnects, or is not routed as it should be */ - if (hpd != was_counted || !owner != !hpd) - neo_dcp_typec_rebalance_locked(NULL, 0); - if (!port->owner) { - if (hpd) - return -EBUSY; - port->applied_valid = true; - return 0; - } - /* just attached, its HPD replayed: nothing left to apply */ - if (port->owner != owner) { - port->applied_valid = true; - return 0; - } - } - - if (!port->owner) { - if (port->preferred_route && - neo_dcp_typec_route_available(port->preferred_route)) - best = port->preferred_route; - - if (!best) - best = neo_dcp_typec_lowest_free(port); - - if (!best) - return -EBUSY; - - ret = neo_dcp_typec_route_activate(best, best->xbar); - if (ret) - return ret; - port->owner = best; - port->dp_release_deadline = 0; - } - - - if (!hpd && port->hpd) { - neo_dcp_dptx_disconnect_oob(to_platform_device(port->owner->neo_dcp->dev), 0); - } else if (hpd && !port->hpd) { - struct neo_apple_dcp *neo_dcp = port->owner->neo_dcp; - - WRITE_ONCE(neo_dcp->typec_cable_connected, true); - if (neo_dcp->typec_connector) - neo_dcp_dptx_connect_oob(to_platform_device(neo_dcp->dev), 0); - } else if (hpd && dp_data && (dp_data->status & DP_STATUS_IRQ_HPD)) { - struct neo_apple_dcp *neo_dcp = port->owner->neo_dcp; - - if (neo_dcp->typec_connector) - neo_dcp_retrain_oob(neo_dcp->typec_connector); - } - port->hpd = hpd; - port->applied_valid = true; - - return 0; -} - -/* - * Crossbar connection up/down, looked up at runtime so appledrm_neo does not - * require the crossbar driver to be built. - */ -static int neo_dcp_dpxbar_link(struct mux_control *mux, bool up) -{ - typeof(&apple_dpxbar_link_up) fn; - int ret; - - fn = up ? symbol_get(apple_dpxbar_link_up) : - symbol_get(apple_dpxbar_link_down); - if (!fn) - return -ENOENT; - ret = fn(mux); - if (up) - symbol_put(apple_dpxbar_link_up); - else - symbol_put(apple_dpxbar_link_down); - return ret; -} - -static int neo_dcp_dpxbar_preselect(struct mux_control *mux, int state) -{ - typeof(&apple_dpxbar_preselect) fn = symbol_get(apple_dpxbar_preselect); - int ret; - - if (!fn) - return -ENOENT; - ret = fn(mux, state); - symbol_put(apple_dpxbar_preselect); - return ret; -} - -/* - * Thunderbolt DP IN, before DCP hears about the display: point the DP IN - * output at this pipeline and wake the ATC's DP clock path. The crossbar - * connection itself still only comes up in dcp_tunnel_crossbar_up(), but an - * output left at its idle source (dispext0) only suits the pipeline that is - * dispext0: any other one activates the link, gets no answer from the sink - * and gives up with DEVICE_NOT_RESPONDING before it ever sets a link rate. - */ -static void neo_dcp_tunnel_prepare(struct neo_apple_dcp_typec_route *route, - struct mux_control *xbar) -{ - typeof(&apple_atc_dp_tunnel_open) open; - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - int ret; - - ret = neo_dcp_dpxbar_preselect(xbar, route->mux_index); - if (ret && ret != -EOPNOTSUPP) - dev_warn(neo_dcp->dev, "DP tunnel crossbar preselect failed: %d\n", ret); - - open = symbol_get(apple_atc_dp_tunnel_open); - if (!open) - return; - ret = open(neo_dcp->phy); - symbol_put(apple_atc_dp_tunnel_open); - if (ret && ret != -EOPNOTSUPP) - dev_warn(neo_dcp->dev, "DP tunnel PHY open failed: %d\n", ret); -} - -/* DP IN adapter handshake through the thunderbolt glue; tb_lock held */ -static int neo_dcp_tunnel_dpin_locked(struct neo_apple_dcp *neo_dcp, bool active) -{ - lockdep_assert_held(&neo_dcp->tb_lock); - if (!neo_dcp->tb_dpin_set_active) - return -ENODEV; - return neo_dcp->tb_dpin_set_active(neo_dcp->tb_dpin_ctx, active); -} - -/* - * Thunderbolt DP IN, from DCP's DidChangeLinkConfiguration once a link rate - * is set: bring the crossbar connection up (FIFO/PCLK/ATC enables) now that - * the tunnel pixel clock runs, and re-assert the DP IN adapter's - * DPTX_INACTIVE=0 afterwards. The first time this is the mux selection; after - * a re-link only the clocks are brought back (the mux selection and the ATC - * output enable are kept, see dcp_tunnel_crossbar_down()). - * Runs inside a DCP apcall: only tb_lock, which nobody holds across a DCP call. - */ -int neo_dcp_tunnel_crossbar_up(struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_dcp_typec_route *route; - int ret; - - guard(mutex)(&neo_dcp->tb_lock); - route = neo_dcp->active_typec_route; - if (!route || !route->tunnel || !route->active_xbar) - return -ENODEV; - if (!neo_dcp->tb_clock_ok) { - dev_warn(neo_dcp->dev, "no DP tunnel pixel clock, crossbar left down\n"); - return -EIO; - } - if (!route->xbar_up) { - /* never block a DCP call on the mux semaphore */ - ret = mux_control_try_select(route->active_xbar, route->mux_index); - if (!ret) - route->xbar_up = true; - } else { - ret = neo_dcp_dpxbar_link(route->active_xbar, true); - } - if (ret) { - dev_warn(neo_dcp->dev, "DP tunnel crossbar up failed: %d\n", ret); - return ret; - } - return neo_dcp_tunnel_dpin_locked(neo_dcp, true); -} - -/* - * Thunderbolt DP IN, from DCP's WillChangeLinkConfiguration on an established - * link: DP IN inactive, crossbar clocks down (mux selection and ATC output - * enable kept). DP IN goes active again in dcp_tunnel_crossbar_up(). - */ -int neo_dcp_tunnel_crossbar_down(struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_dcp_typec_route *route; - - guard(mutex)(&neo_dcp->tb_lock); - route = neo_dcp->active_typec_route; - if (!route || !route->tunnel || !route->xbar_up) - return 0; - neo_dcp_tunnel_dpin_locked(neo_dcp, false); - return neo_dcp_dpxbar_link(route->active_xbar, false); -} - -/* - * Thunderbolt DP IN, from DCP's SetLinkRate: start (rate != 0) or stop the - * tunnel pixel clock. A stopped clock also takes the crossbar connection down. - */ -int neo_dcp_tunnel_set_rate(struct neo_apple_dcp *neo_dcp, struct phy *phy, u32 link_rate) -{ - typeof(&apple_atc_dp_tunnel_rate) fn; - struct neo_apple_dcp_typec_route *route; - int ret; - - guard(mutex)(&neo_dcp->tb_lock); - if (!neo_dcp->neo_dptx_tunnel) - return -ENODEV; - fn = symbol_get(apple_atc_dp_tunnel_rate); - if (!fn) { - dev_err(neo_dcp->dev, "phy-apple-atc not loaded, no DP tunnel clock\n"); - return -ENOENT; - } - route = neo_dcp->active_typec_route; - if (!route) { - symbol_put(apple_atc_dp_tunnel_rate); - return -ENODEV; - } - if (!link_rate && route->xbar_up) - neo_dcp_dpxbar_link(route->active_xbar, false); - ret = fn(phy, route->tunnel_dpin, link_rate); - symbol_put(apple_atc_dp_tunnel_rate); - neo_dcp->tb_clock_ok = !ret && link_rate; - if (ret) - dev_warn(neo_dcp->dev, "DP tunnel pixel clock (rate 0x%x) failed: %d\n", - link_rate, ret); - return ret; -} - -/* Thunderbolt DP IN: DCP Activate/Deactivate */ -int neo_dcp_tunnel_dpin_activate(struct neo_apple_dcp *neo_dcp, bool active) -{ - guard(mutex)(&neo_dcp->tb_lock); - if (!neo_dcp->neo_dptx_tunnel) - return 0; - return neo_dcp_tunnel_dpin_locked(neo_dcp, active); -} - -/* - * Thunderbolt DP tunnels: the host router's DP IN adapters sit behind the - * crossbar's dpin0/dpin1 outputs of the port's ATC. When the Thunderbolt - * connection manager has set up a tunnel from one of them, route a free - * display pipeline there and tell DCP a display is attached, so it trains - * the link (and completes DPRX) through the tunnel. - */ -int neo_apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, - bool active, int (*set_active)(void *ctx, bool active), - void *ctx) -{ - struct neo_apple_dcp_typec_port *port = NULL, *pos; - struct neo_apple_dcp_typec_route *candidate, *best = NULL, *planned = NULL; - struct neo_apple_dcp_typec_route **slot; - unsigned int best_score = UINT_MAX; - struct mux_control *ctl; - struct neo_apple_dcp *neo_dcp; - bool ordered; - int ret; - - if (!connector_np || dpin > 1) - return -EINVAL; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - list_for_each_entry(pos, &neo_dcp_typec_ports, link) { - if (pos->connector_np == connector_np) { - port = pos; - break; - } - } - if (!port) - return -ENODEV; - slot = dpin ? &port->secondary_owner : &port->owner; - - if (!active) { - if (!*slot || !(*slot)->tunnel || - (*slot)->tunnel_dpin != dpin) - return 0; - neo_dcp = (*slot)->neo_dcp; - if (port->hpd || neo_dcp->typec_cable_connected) - neo_dcp_dptx_disconnect_oob(to_platform_device(neo_dcp->dev), 0); - ret = neo_dcp_typec_route_deactivate(*slot); - if (ret) { - /* the caller's context is going away regardless */ - scoped_guard(mutex, &neo_dcp->tb_lock) { - neo_dcp->neo_dptx_tunnel = false; - neo_dcp->tb_dpin_set_active = NULL; - neo_dcp->tb_dpin_ctx = NULL; - } - return ret; - } - *slot = NULL; - port->hpd = !!(port->owner || port->secondary_owner); - /* re-apply the next Type-C mux state in full */ - port->applied_valid = false; - if (neo_dcp->hdmi_hpd && neo_dcp->active && - gpiod_get_value_cansleep(neo_dcp->hdmi_hpd)) - neo_dcp_dptx_connect(neo_dcp, 0); - neo_dcp_typec_pipeline_freed(); - return 0; - } - - if (*slot) { - if ((*slot)->tunnel && (*slot)->tunnel_dpin == dpin) - return 0; - dev_warn((*slot)->neo_dcp->dev, - "port already routed, not taking DP tunnel dpin%u\n", dpin); - return -EBUSY; - } - if (port->owner && !port->owner->tunnel) - return -EBUSY; - - /* - * Until a compositor owns the display, the pairing pass decides where - * the stream goes, moving direct DP-alt routes out of its way, and - * whether it gets a pipeline at all. - */ - ordered = neo_dcp_typec_keep_order(); - if (ordered) - planned = neo_dcp_typec_rebalance_locked(port, dpin); - - list_for_each_entry(candidate, &port->routes, port_link) { - unsigned int score; - - if (ordered && candidate != planned) - continue; - if (!neo_dcp_typec_route_available(candidate)) - continue; - if (!neo_dcp_typec_route_fits(candidate, dpin ? - port->secondary_connector : - port->connector)) - continue; - score = neo_dcp_typec_route_score(candidate); - /* - * DPIN0 prefers the hybrid dcpext0, which completes tunneled - * link training. On dual-stream machines DPIN1 is confined to - * dcpext1 by its connector's possible_crtcs, so both pipelines - * drive independent streams through one dock. - */ - if (neo_dcp_typec_dual_stream() && dpin == 0 && - !candidate->neo_dcp->fixed_phy && score < UINT_MAX - 100) - score += 100; - if (score < best_score) { - best = candidate; - best_score = score; - } - } - if (!best) { - ret = -EBUSY; - goto err_reorder; - } - - /* The route's crossbar control is dpphy (0); dpin0/dpin1 are 1/2. */ - if (best->xbar != &best->xbar->chip->mux[0] || - best->xbar->chip->controllers < 3) { - ret = -EOPNOTSUPP; - goto err_reorder; - } - ctl = &best->xbar->chip->mux[1 + dpin]; - - neo_dcp = best->neo_dcp; - scoped_guard(mutex, &neo_dcp->tb_lock) { - neo_dcp->tb_dpin_set_active = set_active; - neo_dcp->tb_dpin_ctx = ctx; - } - best->tunnel_dpin = dpin; - ret = neo_dcp_typec_route_activate(best, ctl); - if (ret) { - scoped_guard(mutex, &neo_dcp->tb_lock) { - neo_dcp->tb_dpin_set_active = NULL; - neo_dcp->tb_dpin_ctx = NULL; - } - goto err_reorder; - } - *slot = best; - /* the port is in USB4 mode, not DP-alt */ - port->dp_wanted = false; - neo_dcp_tunnel_prepare(best, ctl); - - dev_info(neo_dcp->dev, "display routed to Thunderbolt DP tunnel dpin%u\n", dpin); - - /* - * The DP IN adapter may only be woken (DPTX_INACTIVE=0) while DCP - * drives the DPTX, i.e. from DCP's Activate call; waking it earlier - * hangs the machine. dptxep calls set_active back from Activate and - * Deactivate (set above, before the route became a tunnel). - */ - if (!neo_dcp->typec_connector) - dev_warn(neo_dcp->dev, "no Type-C connector for the DP tunnel\n"); - WRITE_ONCE(neo_dcp->typec_cable_connected, true); - port->hpd = true; - if (neo_dcp->typec_connector) - neo_dcp_dptx_connect_oob(to_platform_device(neo_dcp->dev), 0); - - return 0; - -err_reorder: - /* the pass kept a pipeline for this stream: give it to the others */ - if (planned) - neo_dcp_typec_rebalance_locked(NULL, 0); - return ret; -} -EXPORT_SYMBOL_GPL(neo_apple_dcp_tb_dp_tunnel); - -static struct neo_apple_dcp_typec_port * -neo_dcp_typec_port_get(struct device_node *connector_np) -{ - struct neo_apple_dcp_typec_port *port, *pos; - - lockdep_assert_held(&neo_dcp_typec_fabric_lock); - - list_for_each_entry(port, &neo_dcp_typec_ports, link) { - if (port->connector_np == connector_np) { - of_node_put(connector_np); - return port; - } - } - - port = kzalloc_obj(*port); - if (!port) { - of_node_put(connector_np); - return NULL; - } - - INIT_LIST_HEAD(&port->routes); - port->connector_np = connector_np; - - /* - * Insert in device-tree order rather than DCP probe order. The list - * index becomes the DRM connector index, and userspace keys its - * per-monitor configuration (scale, rotation, layout) on the connector - * name -- so it has to mean the same physical port on every boot. - */ - list_for_each_entry(pos, &neo_dcp_typec_ports, link) - if (strcmp(of_node_full_name(connector_np), - of_node_full_name(pos->connector_np)) < 0) - break; - list_add_tail(&port->link, &pos->link); - return port; -} - -static struct neo_apple_dcp_typec_port *neo_dcp_typec_port_by_index(unsigned int idx) -{ - struct neo_apple_dcp_typec_port *port; - unsigned int i = 0; - - lockdep_assert_held(&neo_dcp_typec_fabric_lock); - - list_for_each_entry(port, &neo_dcp_typec_ports, link) - if (i++ == idx) - return port; - - return NULL; -} - -unsigned int neo_dcp_typec_nr_ports(void) -{ - struct neo_apple_dcp_typec_port *port; - unsigned int n = 0; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - list_for_each_entry(port, &neo_dcp_typec_ports, link) - n++; - - return n; -} - -struct device_node *neo_dcp_typec_port_of_node(unsigned int idx) -{ - struct neo_apple_dcp_typec_port *port; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - port = neo_dcp_typec_port_by_index(idx); - - return port ? port->connector_np : NULL; -} - -bool neo_dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev) -{ - struct neo_apple_dcp_typec_port *port; - struct neo_apple_dcp_typec_route *route; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - port = neo_dcp_typec_port_by_index(idx); - if (!port) - return false; - - list_for_each_entry(route, &port->routes, port_link) - if (route->neo_dcp->dev == &pdev->dev) - return true; - - return false; -} - -void neo_dcp_typec_port_set_connector(unsigned int idx, bool secondary, - struct neo_apple_connector *connector) -{ - struct neo_apple_dcp_typec_port *port; - struct neo_apple_dcp_typec_route *owner; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - port = neo_dcp_typec_port_by_index(idx); - if (!port) - return; - - if (secondary) { - port->secondary_connector = connector; - owner = port->secondary_owner; - } else { - port->connector = connector; - owner = port->owner; - } - - /* - * The port may already have been routed, either before DRM bound or - * while these connectors were being created. Adopt that owner now, - * otherwise its display would be reported on the pipeline's fixed - * connector instead of the port it is actually plugged into. - */ - if (owner) { - struct neo_apple_dcp *neo_dcp = owner->neo_dcp; - - if (neo_dcp->crtc && connector->port_encoder) - connector->port_encoder->possible_crtcs = - drm_crtc_mask(&neo_dcp->crtc->base); - - connector->neo_dcp = to_platform_device(neo_dcp->dev); - neo_dcp->typec_connector = connector; - neo_dcp->connector = connector; - neo_dcp->connector_type = DRM_MODE_CONNECTOR_USB; - } -} - -static void neo_dcp_typec_route_unregister(void *data) -{ - struct neo_apple_dcp_typec_route *route = data; - struct neo_apple_dcp_typec_port *port = route->port; - - typec_mux_unregister(route->typec_mux); - - guard(mutex)(&neo_dcp_typec_fabric_lock); - if (port->preferred_route == route) - port->preferred_route = NULL; - if (port->owner == route) { - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - - if (port->hpd || neo_dcp->typec_cable_connected) - neo_dcp_dptx_disconnect_oob(to_platform_device(neo_dcp->dev), 0); - port->hpd = false; - neo_dcp_typec_route_deactivate(route); - port->owner = NULL; - } - if (port->secondary_owner == route) { - struct neo_apple_dcp *neo_dcp = route->neo_dcp; - - if (port->hpd || neo_dcp->typec_cable_connected) - neo_dcp_dptx_disconnect_oob(to_platform_device(neo_dcp->dev), 0); - neo_dcp_typec_route_deactivate(route); - port->secondary_owner = NULL; - } - port->hpd = !!(port->owner || port->secondary_owner); - list_del(&route->port_link); - if (list_empty(&port->routes)) { - list_del(&port->link); - of_node_put(port->connector_np); - kfree(port); - } -} - -static int neo_dcp_register_typec_routes(struct neo_apple_dcp *neo_dcp) -{ - struct device_node *routes __free(device_node) = - of_get_child_by_name(neo_dcp->dev->of_node, "typec-routes"); - struct device *dev = neo_dcp->dev; - u32 route_index; - int ret; - - if (!routes) - return 0; - - for_each_available_child_of_node_scoped(routes, route_np) { - struct neo_apple_dcp_typec_route *route; - struct device_node *endpoint __free(device_node) = NULL; - struct device_node *connector_np; - struct typec_mux_desc desc = {}; - const char *name, *mux_name; - - if (neo_dcp->nr_typec_routes == DCP_MAX_TYPEC_ROUTES) - return dev_err_probe(dev, -E2BIG, "Too many Type-C display routes\n"); - - ret = of_property_read_u32(route_np, "reg", &route_index); - if (ret) - return dev_err_probe(dev, ret, "%pOF: missing route index\n", route_np); - if (route_index >= DCP_MAX_TYPEC_ROUTES) - return dev_err_probe(dev, -EINVAL, "%pOF: invalid route index %u\n", - route_np, route_index); - - name = devm_kasprintf(dev, GFP_KERNEL, "typec%u", route_index); - if (!name) - return -ENOMEM; - - /* - * The DT lookups above are per-DCP, but the typec_mux class is - * global. Several DCPs can offer a route to the same Type-C port, - * so the registered mux needs a name unique across all of them. - */ - mux_name = devm_kasprintf(dev, GFP_KERNEL, "%s-typec%u", - dev_name(dev), route_index); - if (!mux_name) - return -ENOMEM; - - route = &neo_dcp->typec_routes[neo_dcp->nr_typec_routes]; - route->neo_dcp = neo_dcp; - INIT_LIST_HEAD(&route->port_link); - route->phy = devm_phy_get(dev, name); - if (IS_ERR(route->phy)) - return dev_err_probe(dev, PTR_ERR(route->phy), - "%pOF: failed to get DP PHY\n", route_np); - - route->xbar = devm_mux_control_get(dev, name); - if (IS_ERR(route->xbar)) - return dev_err_probe(dev, PTR_ERR(route->xbar), - "%pOF: failed to get display crossbar\n", route_np); - - ret = of_property_read_u32_index(dev->of_node, "apple,typec-mux-indices", - route_index, &route->mux_index); - if (ret) - return dev_err_probe(dev, ret, "%pOF: missing crossbar state\n", - route_np); - - ret = of_property_read_u32_index(dev->of_node, "apple,typec-dptx-phys", - route_index, &route->neo_dptx_phy); - if (ret) - return dev_err_probe(dev, ret, "%pOF: missing DPTX PHY index\n", - route_np); - - endpoint = of_graph_get_next_endpoint(route_np, NULL); - if (!endpoint) - return dev_err_probe(dev, -EINVAL, - "%pOF: missing Type-C graph endpoint\n", - route_np); - connector_np = of_graph_get_remote_port_parent(endpoint); - if (!connector_np) - return dev_err_probe(dev, -EINVAL, - "%pOF: missing Type-C connector\n", - route_np); - - mutex_lock(&neo_dcp_typec_fabric_lock); - route->port = neo_dcp_typec_port_get(connector_np); - if (route->port) - list_add_tail(&route->port_link, &route->port->routes); - mutex_unlock(&neo_dcp_typec_fabric_lock); - if (!route->port) - return -ENOMEM; - - desc.fwnode = of_fwnode_handle(route_np); - desc.set = neo_dcp_typec_route_set; - desc.name = mux_name; - desc.drvdata = route; - route->typec_mux = typec_mux_register(dev, &desc); - if (IS_ERR(route->typec_mux)) { - mutex_lock(&neo_dcp_typec_fabric_lock); - list_del(&route->port_link); - if (list_empty(&route->port->routes)) { - list_del(&route->port->link); - of_node_put(route->port->connector_np); - kfree(route->port); - } - mutex_unlock(&neo_dcp_typec_fabric_lock); - return dev_err_probe(dev, PTR_ERR(route->typec_mux), - "%pOF: failed to register Type-C route\n", route_np); - } - - ret = devm_add_action_or_reset(dev, neo_dcp_typec_route_unregister, route); - if (ret) - return ret; - - if (!neo_dcp->phy) - neo_dcp->phy = route->phy; - neo_dcp->nr_typec_routes++; - } - - if (!neo_dcp->nr_typec_routes) - return dev_err_probe(dev, -EINVAL, "Type-C route container is empty\n"); - - neo_dcp->phy_managed_by_typec = true; - return 0; -} - - -/* copied and simplified from drm_vblank.c */ -static void send_vblank_event(struct drm_device *dev, - struct drm_pending_vblank_event *e, - u64 seq, ktime_t now) -{ - struct timespec64 tv; - - if (e->event.base.type != DRM_EVENT_FLIP_COMPLETE) - return; - - tv = ktime_to_timespec64(now); - e->event.vbl.sequence = seq; - /* - * e->event is a user space structure, with hardcoded unsigned - * 32-bit seconds/microseconds. This is safe as we always use - * monotonic timestamps since linux-4.15 - */ - e->event.vbl.tv_sec = tv.tv_sec; - e->event.vbl.tv_usec = tv.tv_nsec / 1000; - - /* - * Use the same timestamp for any associated fence signal to avoid - * mismatch in timestamps for vsync & fence events triggered by the - * same HW event. Frameworks like SurfaceFlinger in Android expects the - * retire-fence timestamp to match exactly with HW vsync as it uses it - * for its software vsync modeling. - */ - drm_send_event_timestamp_locked(dev, &e->base, now); -} - -/** - * dcp_crtc_send_page_flip_event - helper to send vblank event after pageflip - * - * Compensate for unknown slack between page flip and arrival of the - * swap_complete callback. Minimal observed duration on DCP with HDMI output - * was around 2.3 ms. If the fb swap was submitted closer to the expected - * swap_complete it gets a penalty of one frame duration. This is on the border - * of unreasonable considering that Apple advertises support for 240 Hz (frame - * duration of 4.167 ms). - * It is unreasonable considering kwin's kms commit scheduling. Kwin commits - * 1.5 ms + the mode's vblank time before the expected next page flip - * completion. This results in presenting at half the display's rate for HDMI - * outputs. - * This might be a difference between dcp and dcpext. - */ -static void neo_dcp_crtc_send_page_flip_event(struct neo_apple_crtc *crtc, - struct drm_pending_vblank_event *e, - ktime_t now, ktime_t start) -{ - struct drm_device *dev = crtc->base.dev; - u64 seq; - unsigned int pipe = drm_crtc_index(&crtc->base); - ktime_t flip; - - seq = 0; - if (start != KTIME_MIN) { - s64 delta = ktime_us_delta(now, start); - if (delta <= 500) - flip = now; - else if (delta >= 2500) - flip = ktime_sub_us(now, 1000); - else - flip = ktime_sub_us(now, (delta - 500) / 2); - } else { - flip = now; - } - e->pipe = pipe; - send_vblank_event(dev, e, seq, flip); -} - -/* HACK: moved here to avoid circular dependency between apple_drv and dcp */ -void neo_dcp_drm_crtc_vblank(struct neo_apple_crtc *crtc) -{ - unsigned long flags; - - spin_lock_irqsave(&crtc->base.dev->event_lock, flags); - if (crtc->event) { - drm_crtc_send_vblank_event(&crtc->base, crtc->event); - crtc->event = NULL; - } - spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); -} - -void neo_dcp_drm_crtc_page_flip(struct neo_apple_dcp *neo_dcp, ktime_t now) -{ - unsigned long flags; - struct neo_apple_crtc *crtc = neo_dcp->crtc; - - spin_lock_irqsave(&crtc->base.dev->event_lock, flags); - if (crtc->event) { - if (crtc->event->event.base.type == DRM_EVENT_FLIP_COMPLETE) - neo_dcp_crtc_send_page_flip_event(crtc, crtc->event, now, neo_dcp->swap_start); - else - drm_crtc_send_vblank_event(&crtc->base, crtc->event); - crtc->event = NULL; - neo_dcp->swap_start = KTIME_MIN; - } - spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); -} - -void neo_dcp_set_dimensions(struct neo_apple_dcp *neo_dcp) -{ - int i; - int width_mm = neo_dcp->width_mm; - int height_mm = neo_dcp->height_mm; - - if (width_mm == 0 || height_mm == 0) { - width_mm = neo_dcp->panel.width_mm; - height_mm = neo_dcp->panel.height_mm; - } - - /* Set the connector info */ - if (neo_dcp->connector) { - struct drm_connector *connector = &neo_dcp->connector->base; - - mutex_lock(&connector->dev->mode_config.mutex); - connector->display_info.width_mm = width_mm; - connector->display_info.height_mm = height_mm; - mutex_unlock(&connector->dev->mode_config.mutex); - } - - /* - * Fix up any probed modes. Modes are created when parsing - * TimingElements, dimensions are calculated when parsing - * DisplayAttributes, and TimingElements may be sent first - */ - for (i = 0; i < neo_dcp->nr_modes; ++i) { - neo_dcp->modes[i].mode.width_mm = width_mm; - neo_dcp->modes[i].mode.height_mm = height_mm; - } -} - -bool neo_dcp_has_panel(struct neo_apple_dcp *neo_dcp) -{ - return neo_dcp->panel.width_mm > 0; -} - -int neo_dcp_set_crc(struct drm_crtc *crtc, bool enabled) -{ - struct neo_apple_crtc *ac = to_apple_crtc(crtc); - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(ac->neo_dcp); - - WRITE_ONCE(neo_dcp->crc_enabled, enabled); - - return 0; -} - -/* - * Helper to send a DRM vblank event. We do not know how call swap_submit_dcp - * without surfaces. To avoid timeouts in drm_atomic_helper_wait_for_vblanks - * send a vblank event via a workqueue. - */ -static void neo_dcp_delayed_vblank(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp; - - struct neo_apple_crtc *crtc; - - neo_dcp = container_of(work, struct neo_apple_dcp, vblank_wq); - /* RTKit can report a crash before the pipeline has a CRTC. */ - crtc = READ_ONCE(neo_dcp->crtc); - if (!crtc) - return; - mdelay(5); - neo_dcp_drm_crtc_vblank(crtc); -} - -static struct neo_apple_dcp_afkep *neo_dcp_afkep(struct neo_apple_dcp *neo_dcp, u8 endpoint) -{ - switch (endpoint) { - case AV_ENDPOINT: - return neo_dcp->neo_avep; - case SYSTEM_ENDPOINT: - return neo_dcp->neo_systemep; - case DISP0_ENDPOINT: - return neo_dcp->neo_ibootep; - case DPAVSERV_ENDPOINT: - return neo_dcp->dcpavservep; - case DPTX_ENDPOINT: - return neo_dcp->neo_dptxep; - default: - return NULL; - } -} - -#define DCP_SWAP_WATCHDOG_MS 1000 -#define DCP_SWAP_WATCHDOG_RETRAINS 5 - -/* - * DCP drops swaps without completing them while an external pipe is not - * enabled, for instance after a modeset that raced a Type-C sink which had - * not asserted HPD yet. Userspace would then wait for the flip until the - * commit times out, stalling every output it drives. Complete the flip, - * mark the mode invalid so later commits do not wait for DCP, and let the - * hotplug worker re-apply the active CRTC. - */ -static void neo_dcp_swap_watchdog(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = - container_of(to_delayed_work(work), struct neo_apple_dcp, - swap_watchdog_wq); - - dev_warn(neo_dcp->dev, "swap not completed, retraining the display\n"); - neo_dcp_mode_invalidate(&neo_dcp->mode_state); - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - if (neo_dcp->connector && - neo_dcp->swap_watchdog_retrains++ < DCP_SWAP_WATCHDOG_RETRAINS) - schedule_work(&neo_dcp->connector->hotplug_wq); -} - -void neo_dcp_swap_watchdog_arm(struct neo_apple_dcp *neo_dcp) -{ - if (neo_dcp_is_typec_output(neo_dcp)) - mod_delayed_work(system_wq, &neo_dcp->swap_watchdog_wq, - msecs_to_jiffies(DCP_SWAP_WATCHDOG_MS)); -} - -void neo_dcp_swap_watchdog_complete(struct neo_apple_dcp *neo_dcp) -{ - cancel_delayed_work(&neo_dcp->swap_watchdog_wq); - neo_dcp->swap_watchdog_retrains = 0; -} - -static void neo_dcp_recv_msg(void *cookie, u8 endpoint, u64 message) -{ - struct neo_apple_dcp *neo_dcp = cookie; - struct neo_apple_dcp_afkep *ep; - - if (READ_ONCE(neo_dcp->quiescing)) - return; - - trace_neo_dcp_recv_msg(neo_dcp, endpoint, message); - - /* - * H17P starts endpoints that have no AFK instance behind them (the - * remote allocator endpoint, for one). Log their messages rather than - * dereferencing a NULL afkep or warning about an unknown endpoint. - */ - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && endpoint != IOMFB_ENDPOINT && - !neo_dcp_afkep(neo_dcp, endpoint)) { - dev_dbg_ratelimited(neo_dcp->dev, "ep %#04x: %#llx\n", endpoint, - message); - return; - } - - switch (endpoint) { - case IOMFB_ENDPOINT: - return neo_iomfb_recv_msg(neo_dcp, message); - case AV_ENDPOINT: - ep = neo_dcp->neo_avep; - break; - case SYSTEM_ENDPOINT: - ep = neo_dcp->neo_systemep; - break; - case DISP0_ENDPOINT: - ep = neo_dcp->neo_ibootep; - break; - case DPAVSERV_ENDPOINT: - ep = neo_dcp->dcpavservep; - break; - case DPTX_ENDPOINT: - ep = neo_dcp->neo_dptxep; - break; - default: - ep = NULL; - break; - } - - if (!ep) { - dev_warn_ratelimited(neo_dcp->dev, - "dropping message for unhandled endpoint %#x\n", - endpoint); - return; - } - - neo_afk_receive_message(ep, message); -} - -static void neo_dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_size) -{ - struct neo_apple_dcp *neo_dcp = cookie; - - if (READ_ONCE(neo_dcp->quiescing)) - return; - - neo_dcp->crashed = true; - dev_err(neo_dcp->dev, "DCP has crashed\n"); - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - /* No in-place restart exists; the last scanout stays retained. */ - dev_err(neo_dcp->dev, "a reboot is required to restore the display\n"); - /* - * Nothing will complete a present that was in flight. Before - * the pipeline is linked there is no event to complete. - */ - if (READ_ONCE(neo_dcp->crtc)) - schedule_work(&neo_dcp->vblank_wq); - } - if (neo_dcp->connector) { - neo_dcp->connector->connected = 0; - drm_edid_free(neo_dcp->connector->drm_edid); - neo_dcp->connector->drm_edid = NULL; - schedule_work(&neo_dcp->connector->hotplug_wq); - } - complete(&neo_dcp->start_done); -} - -/* - * Check that every IOMMU page of [iova, iova + size) translates to the same - * offset from @phys as it has from @iova. - */ -static int neo_dcp_check_iova_contiguous(struct iommu_domain *domain, - dma_addr_t iova, size_t size, - phys_addr_t phys) -{ - unsigned long pgsize = domain->pgsize_bitmap ? - 1UL << __ffs(domain->pgsize_bitmap) : PAGE_SIZE; - dma_addr_t end = iova + size; - dma_addr_t addr = iova; - - while (addr < end) { - if (iommu_iova_to_phys(domain, addr) != phys + (addr - iova)) - return -EINVAL; - addr = ALIGN_DOWN(addr, pgsize) + pgsize; - } - - return 0; -} - -static int neo_dcp_rtk_shmem_setup(void *cookie, struct apple_rtkit_shmem *bfr) -{ - struct neo_apple_dcp *neo_dcp = cookie; - int ret; - - if (bfr->iova) { - struct iommu_domain *domain = - iommu_get_domain_for_dev(neo_dcp->dev); - phys_addr_t phy_addr; - - if (!domain) - return -ENOMEM; - - // TODO: get map from device-tree - phy_addr = iommu_iova_to_phys(domain, bfr->iova); - if (!phy_addr) - return -ENOMEM; - - /* memremap() below needs the whole buffer physically contiguous */ - ret = neo_dcp_check_iova_contiguous(domain, bfr->iova, bfr->size, - phy_addr); - if (ret) { - dev_err(neo_dcp->dev, - "shmem_setup: iova %pad (%#zx bytes) is not physically contiguous\n", - &bfr->iova, bfr->size); - return ret; - } - - /* Firmware writes without CPU cache maintenance. */ - bfr->buffer = memremap(phy_addr, bfr->size, MEMREMAP_WC); - if (!bfr->buffer) - return -ENOMEM; - - bfr->is_mapped = true; - dev_info(neo_dcp->dev, - "shmem_setup: iova: %lx -> pa: %lx -> iomem: %lx\n", - (uintptr_t)bfr->iova, (uintptr_t)phy_addr, - (uintptr_t)bfr->buffer); - } else { - bfr->buffer = dma_alloc_coherent(neo_dcp->dev, bfr->size, - &bfr->iova, GFP_KERNEL); - if (!bfr->buffer) - return -ENOMEM; - - dev_info(neo_dcp->dev, "shmem_setup: iova: %lx, buffer: %lx\n", - (uintptr_t)bfr->iova, (uintptr_t)bfr->buffer); - } - - return 0; -} - -static void neo_dcp_rtk_shmem_destroy(void *cookie, struct apple_rtkit_shmem *bfr) -{ - struct neo_apple_dcp *neo_dcp = cookie; - - if (bfr->is_mapped) - memunmap(bfr->buffer); - else - dma_free_coherent(neo_dcp->dev, bfr->size, bfr->buffer, bfr->iova); -} - -static struct apple_rtkit_ops rtkit_ops = { - .crashed = neo_dcp_rtk_crashed, - .recv_message = neo_dcp_recv_msg, - .shmem_setup = neo_dcp_rtk_shmem_setup, - .shmem_destroy = neo_dcp_rtk_shmem_destroy, -}; - -void neo_dcp_send_message(struct neo_apple_dcp *neo_dcp, u8 endpoint, u64 message) -{ - if (READ_ONCE(neo_dcp->quiescing)) - return; - trace_neo_dcp_send_msg(neo_dcp, endpoint, message); - apple_rtkit_send_message(neo_dcp->rtk, endpoint, message, NULL, - true); -} - -int neo_dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - struct platform_device *pdev = to_apple_crtc(crtc)->neo_dcp; - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - struct drm_crtc_state *crtc_state; - bool needs_modeset; - - if (neo_dcp->crashed) - return -EINVAL; - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - - needs_modeset = drm_atomic_crtc_needs_modeset(crtc_state) || - !READ_ONCE(neo_dcp->mode_state.valid); - if (!needs_modeset && (!neo_dcp->connector || !neo_dcp->connector->connected)) { - /* - * Resume restores the mode before the firmware reports the - * display back, so a plane-only commit lands here while the - * connector is still marked disconnected. Rejecting it makes - * the compositor fail every flip and give up on the output; - * dcp_flush() defers the commit until the link returns. - */ - dev_dbg(neo_dcp->dev, - "crtc_atomic_check: deferring commit, link still down\n"); - } - - return 0; -} - -int neo_dcp_get_connector_type(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - return neo_dcp->fixed_connector_type; -} - -bool neo_dcp_has_typec_routes(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - return neo_dcp->nr_typec_routes; -} - -#define DPTX_CONNECT_TIMEOUT msecs_to_jiffies(2000) -#define DPTX_TUNNEL_CONNECT_TIMEOUT msecs_to_jiffies(8000) -#define DPTX_RECONNECT_DELAY msecs_to_jiffies(1000) -#define DPTX_RECONNECT_RETRIES 5 - -static int neo_dcp_dptx_connect(struct neo_apple_dcp *neo_dcp, u32 port) -{ - unsigned long timeout; - int ret = 0; - - if (!neo_dcp->phy) { - dev_warn(neo_dcp->dev, "dcp_dptx_connect: missing phy\n"); - return -ENODEV; - } - dev_info(neo_dcp->dev, - "%s(port=%d) target=%u:%u typec=%d route=%s conn_type=%d connected=%d\n", - __func__, port, neo_dcp->neo_dptx_die, neo_dcp->neo_dptx_phy, - neo_dcp_is_typec_output(neo_dcp), - neo_dcp->active_typec_route ? "borrowed" : "fixed", - neo_dcp->connector_type, neo_dcp->neo_dptxport[port].connected); - - mutex_lock(&neo_dcp->hpd_mutex); - if (!neo_dcp->neo_dptxport[port].enabled) { - dev_warn(neo_dcp->dev, "dcp_dptx_connect: dptx service for port %d not enabled\n", port); - ret = -ENODEV; - goto out_unlock; - } - - if (neo_dcp->neo_dptxport[port].connected) - goto out_unlock; - - reinit_completion(&neo_dcp->neo_dptxport[port].linkcfg_completion); - neo_dcp->neo_dptxport[port].atcphy = neo_dcp->phy; - ret = neo_dptxport_validate_connection(neo_dcp->neo_dptxport[port].service, - neo_dcp->neo_dptx_dfp_port, - neo_dcp->neo_dptx_phy, neo_dcp->neo_dptx_die); - if (ret) { - dev_err(neo_dcp->dev, - "dcp_dptx_connect: failed to validate DPTX target %u:%u: %d\n", - neo_dcp->neo_dptx_die, neo_dcp->neo_dptx_phy, ret); - goto out_unlock; - } - - ret = neo_dptxport_connect(neo_dcp->neo_dptxport[port].service, - neo_dcp->neo_dptx_dfp_port, - neo_dcp->neo_dptx_phy, neo_dcp->neo_dptx_die, - neo_dcp_is_typec_output(neo_dcp)); - if (ret) { - dev_err(neo_dcp->dev, - "dcp_dptx_connect: failed to connect DPTX target %u:%u: %d\n", - neo_dcp->neo_dptx_die, neo_dcp->neo_dptx_phy, ret); - goto out_unlock; - } - - ret = neo_dptxport_request_display(neo_dcp->neo_dptxport[port].service); - if (ret) { - dev_err(neo_dcp->dev, - "dcp_dptx_connect: failed to request display: %d\n", - ret); - goto out_release; - } - neo_dcp->neo_dptxport[port].connected = true; - if (neo_dcp_is_typec_output(neo_dcp)) { - if (neo_dcp_is_usb4_output(neo_dcp) && false) - ret = neo_dptxport_set_hpd_timeout(neo_dcp->neo_dptxport[port].service, - true, 8000); - else - ret = neo_dptxport_set_hpd(neo_dcp->neo_dptxport[port].service, true); - if (ret) { - dev_err(neo_dcp->dev, - "dcp_dptx_connect: failed to assert Type-C HPD: %d\n", - ret); - neo_dcp->neo_dptxport[port].connected = false; - goto out_release; - } - } - - mutex_unlock(&neo_dcp->hpd_mutex); - timeout = neo_dcp_is_usb4_output(neo_dcp) && false ? - DPTX_TUNNEL_CONNECT_TIMEOUT : DPTX_CONNECT_TIMEOUT; - ret = wait_for_completion_timeout(&neo_dcp->neo_dptxport[port].linkcfg_completion, - timeout); - if (!ret) { - dev_err(neo_dcp->dev, - "dcp_dptx_connect: timed out waiting for port %u link configuration\n", - port); - ret = -ETIMEDOUT; - goto out_disconnect; - } - - dev_dbg(neo_dcp->dev, "dcp_dptx_connect: waited %d ms for link\n", - jiffies_to_msecs(timeout - ret)); - - usleep_range(5, 10); - - if (neo_dcp->connector_type == DRM_MODE_CONNECTOR_DisplayPort) - neo_dptxport_set_hpd(neo_dcp->neo_dptxport[port].service, true); - - if (neo_dcp->neo_avep) - neo_av_service_connect(neo_dcp); - - return 0; - -out_disconnect: - mutex_lock(&neo_dcp->hpd_mutex); - neo_dcp->neo_dptxport[port].connected = false; -out_release: - neo_dptxport_release_display(neo_dcp->neo_dptxport[port].service); - -out_unlock: - mutex_unlock(&neo_dcp->hpd_mutex); - return ret; -} - -static bool neo_dcp_edid_is_placeholder(const struct drm_edid *drm_edid) -{ - const u8 *raw = (const u8 *)drm_edid_raw(drm_edid); - static const u8 name[] = "Non-PnP"; - unsigned int i; - - if (!raw) - return false; - - /* EDID contains interior NUL bytes, so this cannot use strnstr(). */ - for (i = 0; i + sizeof(name) - 1 <= sizeof(struct edid); i++) { - if (!memcmp(raw + i, name, sizeof(name) - 1)) - return true; - } - - return false; -} - -void neo_dcp_retry_placeholder_edid(struct neo_apple_dcp *neo_dcp, - const struct drm_edid *drm_edid) -{ - guard(mutex)(&neo_dcp->hpd_mutex); - if (!neo_dcp_is_typec_output(neo_dcp) || !neo_dcp->typec_cable_connected || - neo_dcp->placeholder_retried) - return; - if (!neo_dcp_edid_is_placeholder(drm_edid)) - return; - - neo_dcp->placeholder_retried = true; - neo_dcp->placeholder_generation = neo_dcp->typec_generation; - schedule_delayed_work(&neo_dcp->placeholder_edid_wq, msecs_to_jiffies(300)); -} - -static void neo_dcp_placeholder_edid_work(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = - container_of(to_delayed_work(work), struct neo_apple_dcp, - placeholder_edid_wq); - struct neo_apple_epic_service *service; - u64 generation; - int ret; - - mutex_lock(&neo_dcp->hpd_mutex); - generation = neo_dcp->placeholder_generation; - if (!neo_dcp->typec_cable_connected || !neo_dcp->neo_dptxport[0].connected || - !neo_dcp->neo_dptxport[0].enabled || generation != neo_dcp->typec_generation) - goto out_unlock; - service = neo_dcp->neo_dptxport[0].service; - - /* - * Some adapters answer the first connection with a 1024x768 - * placeholder and publish the panel EDID only after HPD drops - * and returns. One pulse; a second placeholder is left alone. - */ - ret = neo_dptxport_set_hpd(service, false); - if (ret) { - dev_info(neo_dcp->dev, "placeholder EDID: HPD drop failed: %d\n", - ret); - goto out_unlock; - } - mutex_unlock(&neo_dcp->hpd_mutex); - - msleep(1000); - - mutex_lock(&neo_dcp->hpd_mutex); - if (!neo_dcp->typec_cable_connected || !neo_dcp->neo_dptxport[0].connected || - generation != neo_dcp->typec_generation) - goto out_unlock; - - ret = neo_dptxport_set_hpd(service, true); - if (ret) - dev_info(neo_dcp->dev, "placeholder EDID: HPD assert failed: %d\n", - ret); -out_unlock: - mutex_unlock(&neo_dcp->hpd_mutex); -} - -static void neo_dcp_typec_reconnect_work(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = - container_of(to_delayed_work(work), struct neo_apple_dcp, - typec_reconnect_wq); - int ret; - - if (!READ_ONCE(neo_dcp->typec_cable_connected)) - return; - - ret = neo_dcp_dptx_connect(neo_dcp, 0); - if (!ret) { - neo_dcp->typec_reconnect_tries = 0; - return; - } - - if (++neo_dcp->typec_reconnect_tries < - (neo_dcp_is_usb4_output(neo_dcp) && false ? - 1 : DPTX_RECONNECT_RETRIES)) { - mod_delayed_work(system_freezable_wq, &neo_dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - return; - } - - dev_err(neo_dcp->dev, "Type-C DPTX reconnect failed after %u retries: %d\n", - neo_dcp->typec_reconnect_tries, ret); -} - -static void disconnected_hpd_event(struct neo_apple_connector *con) -{ - if (con && con->connected) { - con->connected = 0; - drm_edid_free(con->drm_edid); - con->drm_edid = NULL; - drm_kms_helper_connector_hotplug_event(&con->base); - } -} - -static int neo_dcp_dptx_disconnect(struct neo_apple_dcp *neo_dcp, u32 port) -{ - dev_info(neo_dcp->dev, "%s(port=%d)\n", __func__, port); - - mutex_lock(&neo_dcp->hpd_mutex); - if (neo_dcp->neo_dptxport[port].enabled && neo_dcp->neo_dptxport[port].connected) { - neo_dptxport_release_display(neo_dcp->neo_dptxport[port].service); - neo_dcp->neo_dptxport[port].connected = false; - } - mutex_unlock(&neo_dcp->hpd_mutex); - - return 0; -} - -int neo_dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - int ret; - - if (neo_dcp_is_typec_output(neo_dcp)) { - cancel_delayed_work_sync(&neo_dcp->placeholder_edid_wq); - guard(mutex)(&neo_dcp->hpd_mutex); - neo_dcp->typec_generation++; - WRITE_ONCE(neo_dcp->typec_cable_connected, true); - neo_dcp->typec_reconnect_tries = 0; - neo_dcp->placeholder_retried = false; - cancel_delayed_work(&neo_dcp->typec_reconnect_wq); - } - - ret = neo_dcp_dptx_connect(neo_dcp, port); - if (ret && neo_dcp_is_typec_output(neo_dcp)) - mod_delayed_work(system_freezable_wq, &neo_dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - - return ret; -} - -int neo_dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - if (neo_dcp_is_typec_output(neo_dcp)) { - scoped_guard(mutex, &neo_dcp->hpd_mutex) { - WRITE_ONCE(neo_dcp->typec_cable_connected, false); - neo_dcp->typec_generation++; - } - WRITE_ONCE(neo_dcp->typec_crtc_off, false); - reinit_completion(&neo_dcp->typec_iomfb_hpd_ready); - cancel_delayed_work_sync(&neo_dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&neo_dcp->placeholder_edid_wq); - } - - disconnected_hpd_event(neo_dcp->connector); - - if (neo_dcp->neo_avep) - neo_av_service_disconnect(neo_dcp); - - if (neo_dcp->neo_dptxport[port].enabled) - neo_dptxport_set_hpd(neo_dcp->neo_dptxport[port].service, false); - - return neo_dcp_dptx_disconnect(neo_dcp, port); -} - -static irqreturn_t neo_dcp_dp2hdmi_hpd(int irq, void *data) -{ - struct neo_apple_dcp *neo_dcp = data; - bool connected; - - guard(mutex)(&neo_dcp_typec_fabric_lock); - - if (READ_ONCE(neo_dcp->active_typec_route)) { - /* - * Until a compositor owns the display, a live HDMI output - * takes its pipeline back from a direct DP-alt route: the - * compositor pairs the HDMI connector with it first. - */ - if (neo_dcp_typec_keep_order() && - gpiod_get_value_cansleep(neo_dcp->hdmi_hpd)) { - msleep(500); - if (gpiod_get_value_cansleep(neo_dcp->hdmi_hpd)) - neo_dcp_typec_rebalance_locked(NULL, 0); - } - return IRQ_HANDLED; - } - connected = gpiod_get_value_cansleep(neo_dcp->hdmi_hpd); - - /* do nothing on disconnect and trust that dcp detects it itself. - * Parallel disconnect HPDs result drm disabling the CRTC even when it - * should not. - * The interrupt should be changed to rising but for now the disconnect - * IRQs might be helpful for debugging. - */ - dev_info(neo_dcp->dev, "DP2HDMI HPD irq, connected:%d\n", connected); - - if (connected) { - msleep(500); - connected = gpiod_get_value_cansleep(neo_dcp->hdmi_hpd); - dev_info(neo_dcp->dev, "DP2HDMI HPD irq, 500ms debounce: connected:%d\n", connected); - } - - if (connected) - neo_dcp_dptx_connect(neo_dcp, 0); - - return IRQ_HANDLED; -} - -void neo_dcp_link(struct platform_device *pdev, struct neo_apple_crtc *crtc, - struct neo_apple_connector *connector) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - WRITE_ONCE(neo_dcp->crtc, crtc); - - /* - * Type-C connectors belong to physical ports and are bound by the - * display fabric when a pipeline takes a route, so a pipeline with no - * fixed output simply has no connector until then. - */ - if (!connector) - return; - - neo_dcp->fixed_connector = connector; - if (!neo_dcp->active_typec_route) { - neo_dcp->connector = connector; - neo_dcp->connector_type = neo_dcp->fixed_connector_type; - } -} - - -bool neo_dcp_fw_compat_is_12_x(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - return neo_dcp->fw_compat == DCP_FIRMWARE_V_12_3; -} - -unsigned long* neo_dcp_get_iomfb_surfaces(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - return neo_dcp->neo_iomfb_surfaces; -} - -int neo_dcp_start(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - int ret; - - if (!neo_dcp->rtk) - return -ENODEV; - - init_completion(&neo_dcp->start_done); - - /* - * The coprocessor start can fail (or be recovered from) without tearing - * the DRM side down, so this can be reached with no RTKit instance at - * all. Starting an endpoint then dereferences NULL inside - * apple_rtkit_start_ep() and takes the machine with it. - */ - if (!neo_dcp->rtk || IS_ERR(neo_dcp->rtk)) { - dev_err(neo_dcp->dev, - "cannot start endpoints: RTKit is not initialised\n"); - return -ENODEV; - } - - /* start RTKit endpoints */ - ret = neo_systemep_init(neo_dcp); - if (ret) - dev_warn(neo_dcp->dev, "Failed to start system endpoint: %d\n", ret); - - if (unstable_edid && !neo_dcp_has_panel(neo_dcp)) { - ret = dpavservep_init(neo_dcp); - if (ret) - dev_warn(neo_dcp->dev, "Failed to start DPAVSERV endpoint: %d", - ret); - } - - if (neo_dcp->phy && neo_dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { - ret = neo_ibootep_init(neo_dcp); - if (ret) - dev_warn(neo_dcp->dev, "Failed to start IBOOT endpoint: %d\n", - ret); - - ret = neo_dptxep_init(neo_dcp); - if (ret) { - dev_warn(neo_dcp->dev, "Failed to start DPTX endpoint: %d\n", - ret); -#ifdef DCP_DPTX_DISCONNECT_ON_INIT - /* - * This disconnect / connect cycle on init is only necessary - * when using dcp0 on j473, j474s and presumedly j475c. - * Since dcp0 is not used at the moment let's avoid this - * since it is possibly the cause for startup issues. - */ - } else if (neo_dcp->neo_dptxport[0].enabled) { - bool connected; - /* force disconnect on start - necessary if the display - * is already up from m1n1 - */ - neo_dptxport_set_hpd(neo_dcp->neo_dptxport[0].service, false); - neo_dptxport_release_display(neo_dcp->neo_dptxport[0].service); - usleep_range(10 * USEC_PER_MSEC, 25 * USEC_PER_MSEC); - - connected = gpiod_get_value_cansleep(neo_dcp->hdmi_hpd); - dev_info(neo_dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - - // necessary on j473/j474 but not on j314c - if (connected) - neo_dcp_dptx_connect(neo_dcp, 0); -#endif - } - } else if (neo_dcp->phy) { - dev_warn(neo_dcp->dev, "OS firmware incompatible with dptxport EP\n"); - } - ret = neo_iomfb_start_rtkit(neo_dcp); - if (ret) - dev_err(neo_dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) - if (neo_hdmi_audio) { - ret = neo_avep_init(neo_dcp); - if (ret) - dev_warn(neo_dcp->dev, "Failed to start AV endpoint: %d", ret); - ret = 0; - } -#endif - - return ret; -} - -static void _dcp_poweroff(struct neo_apple_dcp *neo_dcp) -{ - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - neo_iomfb_poweroff_v12_3(neo_dcp); - break; - case DCP_FIRMWARE_V_13_5: - neo_iomfb_poweroff_v13_3(neo_dcp); - break; - case DCP_FIRMWARE_H17P: - neo_iomfb_poweroff_h17p(neo_dcp); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", neo_dcp->fw_compat); - break; - } -} - -static int neo_dcp_enable_dp2hdmi_hpd(struct neo_apple_dcp *neo_dcp) -{ - if (neo_dcp_is_typec_output(neo_dcp)) { - if (READ_ONCE(neo_dcp->typec_cable_connected)) - neo_dcp_dptx_connect(neo_dcp, 0); - } else if (neo_dcp->hdmi_hpd) { - /* Check HPD before enabling the edge-triggered IRQ. */ - bool connected = gpiod_get_value_cansleep(neo_dcp->hdmi_hpd); - dev_info(neo_dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - - if (connected) - neo_dcp_dptx_connect(neo_dcp, 0); - else - _dcp_poweroff(neo_dcp); - } - - if (neo_dcp->hdmi_hpd_irq) - enable_irq(neo_dcp->hdmi_hpd_irq); - - return 0; -} - -int neo_dcp_wait_ready(struct platform_device *pdev, u64 timeout) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - int ret; - - if (neo_dcp->crashed) - return -ENODEV; - if (neo_dcp->active) - return neo_dcp_enable_dp2hdmi_hpd(neo_dcp); - if (timeout <= 0) - return -ETIMEDOUT; - - ret = wait_for_completion_timeout(&neo_dcp->start_done, timeout); - if (ret < 0) - return ret; - - if (neo_dcp->crashed) - return -ENODEV; - - if (neo_dcp->active) - neo_dcp_enable_dp2hdmi_hpd(neo_dcp); - - return neo_dcp->active ? 0 : -ETIMEDOUT; -} - -static void __maybe_unused neo_dcp_sleep(struct neo_apple_dcp *neo_dcp) -{ - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - neo_iomfb_sleep_v12_3(neo_dcp); - break; - case DCP_FIRMWARE_V_13_5: - neo_iomfb_sleep_v13_3(neo_dcp); - break; - case DCP_FIRMWARE_H17P: - neo_iomfb_sleep_h17p(neo_dcp); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", neo_dcp->fw_compat); - break; - } -} - -#define DCP_BL_FALLBACK_DELAY msecs_to_jiffies(2000) - -static bool neo_dcp_uses_soft_dpms(struct neo_apple_dcp *neo_dcp) -{ - return (neo_dcp_backlight_active(neo_dcp) && neo_dcp_has_panel(neo_dcp)) || - (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - neo_dcp->connector_type == DRM_MODE_CONNECTOR_eDP); -} - -void neo_dcp_poweron(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - bool wait_for_typec_hpd = false; - unsigned long remaining; - int ret; - - if (neo_dcp_uses_soft_dpms(neo_dcp)) { - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !READ_ONCE(neo_dcp->pipe_enabled_h17p)) { - neo_iomfb_poweron_h17p(neo_dcp); - if (!READ_ONCE(neo_dcp->pipe_enabled_h17p)) { - dev_err(neo_dcp->dev, "initial display pipe enable failed\n"); - WRITE_ONCE(neo_dcp->crashed, true); - return; - } - /* The powerlog takeover report normally precedes this. */ - if (neo_dcp_has_panel(neo_dcp)) - queue_delayed_work(system_freezable_wq, - &neo_dcp->bl_fallback_wq, - DCP_BL_FALLBACK_DELAY); - } - ret = neo_dcp_backlight_dpms(neo_dcp, true); - if (ret) - dev_warn(neo_dcp->dev, "backlight restore unavailable: %d\n", ret); - return; - } - - if (neo_dcp_is_typec_output(neo_dcp)) { - wait_for_typec_hpd = READ_ONCE(neo_dcp->typec_crtc_off) && - READ_ONCE(neo_dcp->typec_cable_connected); - WRITE_ONCE(neo_dcp->typec_crtc_off, false); - - /* - * A Type-C CRTC disable releases its DPTX session. Re-establish it - * synchronously before IOMFB is powered back on. - */ - if (READ_ONCE(neo_dcp->typec_cable_connected)) { - cancel_delayed_work(&neo_dcp->typec_reconnect_wq); - neo_dcp->typec_reconnect_tries = 0; - ret = neo_dcp_dptx_connect(neo_dcp, 0); - if (ret) - mod_delayed_work(system_freezable_wq, - &neo_dcp->typec_reconnect_wq, - DPTX_RECONNECT_DELAY); - else if (wait_for_typec_hpd) { - remaining = wait_for_completion_timeout( - &neo_dcp->typec_iomfb_hpd_ready, - msecs_to_jiffies(3000)); - if (!remaining) - dev_warn(neo_dcp->dev, - "Type-C IOMFB hotplug not ready on wake\n"); - } - } - } else if (neo_dcp->hdmi_hpd) { - bool connected = gpiod_get_value_cansleep(neo_dcp->hdmi_hpd); - dev_info(neo_dcp->dev, "%s: DP2HDMI HPD connected:%d\n", __func__, connected); - - if (connected) - neo_dcp_dptx_connect(neo_dcp, 0); - } - - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - neo_iomfb_poweron_v12_3(neo_dcp); - break; - case DCP_FIRMWARE_V_13_5: - neo_iomfb_poweron_v13_3(neo_dcp); - break; - case DCP_FIRMWARE_H17P: - neo_iomfb_poweron_h17p(neo_dcp); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", neo_dcp->fw_compat); - break; - } - if (neo_dcp->neo_avep) - neo_av_service_connect(neo_dcp); -} - -void neo_dcp_poweroff(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - int ret; - - cancel_delayed_work(&neo_dcp->swap_watchdog_wq); - - /* Internal H17P DPMS is a brightness present, never a pipe stop. */ - if (neo_dcp_uses_soft_dpms(neo_dcp)) { - ret = neo_dcp_backlight_dpms(neo_dcp, false); - if (ret) - dev_warn(neo_dcp->dev, "backlight blank unavailable: %d\n", ret); - return; - } - - if (neo_dcp->neo_avep) - neo_av_service_disconnect(neo_dcp); - - /* - * Powering a Type-C CRTC off drops DCP's synthetic HPD, and the firmware - * reports that as an unplug. The display is still attached: keep the - * connector connected (see dcpep_cb_hotplug()) and let dcp_poweron() - * re-establish the DPTX session. Recreating it here instead makes the - * display vanish and come back, and compositors light a returning - * display, so DPMS off never sticks. Cable removal is reported through - * the Type-C mux. - */ - if (neo_dcp_is_typec_output(neo_dcp)) { - reinit_completion(&neo_dcp->typec_iomfb_hpd_ready); - if (READ_ONCE(neo_dcp->typec_cable_connected)) - WRITE_ONCE(neo_dcp->typec_crtc_off, true); - /* dcp_poweron() reconnects the link on DPMS wake. */ - cancel_delayed_work(&neo_dcp->typec_reconnect_wq); - } - - _dcp_poweroff(neo_dcp); - - if (neo_dcp_is_typec_output(neo_dcp)) { - /* DCP owns a synthetic HPD for Type-C. Release it with the CRTC. */ - if (neo_dcp->neo_dptxport[0].enabled && neo_dcp->neo_dptxport[0].connected) { - ret = neo_dptxport_set_hpd(neo_dcp->neo_dptxport[0].service, false); - if (ret) - dev_warn(neo_dcp->dev, - "failed to deassert Type-C DPTX HPD: %d\n", ret); - neo_dcp_dptx_disconnect(neo_dcp, 0); - } - } else if (neo_dcp->hdmi_hpd) { - bool connected = gpiod_get_value_cansleep(neo_dcp->hdmi_hpd); - if (!connected) { - disconnected_hpd_event(neo_dcp->connector); - neo_dcp_dptx_disconnect(neo_dcp, 0); - } - } -} - -static void neo_dcp_work_register_backlight(struct work_struct *work) -{ - int ret; - struct neo_apple_dcp *neo_dcp; - - neo_dcp = container_of(work, struct neo_apple_dcp, bl_register_wq); - - mutex_lock(&neo_dcp->bl_register_mutex); - if (neo_dcp->brightness.bl_dev) - goto out_unlock; - - /* try to register backlight device, */ - ret = neo_dcp_backlight_register(neo_dcp); - if (ret == -ENODATA) - goto out_unlock; - if (ret) { - dev_err(neo_dcp->dev, "Unable to register backlight device: %d\n", ret); - /* - * The H17P policy encodes the panel ceiling into every present - * and keeps working without a class device; only older - * firmware stops sending brightness here. - */ - if (!neo_dcp_backlight_active(neo_dcp)) - neo_dcp->brightness.maximum = 0; - } - -out_unlock: - mutex_unlock(&neo_dcp->bl_register_mutex); -} - -static void neo_dcp_work_update_backlight(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp; - - neo_dcp = container_of(work, struct neo_apple_dcp, bl_update_wq); - - neo_dcp_backlight_update(neo_dcp); -} - -/* - * H17P never publishes the panel level, and the loader's level arrives only - * as an optional powerlog report. Without one, register the backlight at - * the middle of the panel range. That value is only reported: no level is - * presented until userspace writes one, so the panel keeps the loader's - * level. - */ -static void neo_dcp_work_backlight_fallback(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = container_of(to_delayed_work(work), - struct neo_apple_dcp, bl_fallback_wq); - u32 nits = neo_dcp->brightness.maximum / 2; - int ret; - - if (READ_ONCE(neo_dcp->quiescing) || READ_ONCE(neo_dcp->crashed) || - neo_dcp_backlight_active(neo_dcp)) - return; - - ret = neo_iomfb_configure_backlight_h17p(neo_dcp, neo_dcp->brightness.maximum, - false, 0, true, nits); - if (!ret) - dev_info(neo_dcp->dev, - "no loader brightness reported; backlight starts at %u nits without a panel change\n", - nits); - else if (ret != -EBUSY) - dev_warn(neo_dcp->dev, "backlight registration failed: %d\n", ret); -} - -static void neo_dcp_release_piodma_iommu_dev(struct neo_apple_dcp *neo_dcp) -{ - if (neo_dcp->retain_dma) { - WRITE_ONCE(neo_dcp->quiescing, true); - return; - } - if (neo_dcp->piodma) { - if (neo_dcp->piodma_created) - of_platform_device_destroy(&neo_dcp->piodma->dev, NULL); - else - put_device(&neo_dcp->piodma->dev); - } - neo_dcp->piodma = NULL; - neo_dcp->iommu_dom = NULL; - neo_dcp->piodma_created = false; -} - -void neo_dcp_retain_framebuffer(struct platform_device *pdev, - struct neo_dcp_fb_reference *entry) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - mutex_lock(&neo_dcp->swapped_out_fbs_lock); - list_add_tail(&entry->head, &neo_dcp->swapped_out_fbs); - mutex_unlock(&neo_dcp->swapped_out_fbs_lock); -} - -void neo_dcp_arm_retained_framebuffers(struct neo_apple_dcp *neo_dcp, u32 swap_id) -{ - struct neo_dcp_fb_reference *entry; - - mutex_lock(&neo_dcp->swapped_out_fbs_lock); - list_for_each_entry(entry, &neo_dcp->swapped_out_fbs, head) { - if (entry->armed) - continue; - entry->swap_id = swap_id; - entry->armed = true; - } - mutex_unlock(&neo_dcp->swapped_out_fbs_lock); -} - -void neo_dcp_unarm_retained_framebuffers(struct neo_apple_dcp *neo_dcp, u32 swap_id) -{ - struct neo_dcp_fb_reference *entry; - - mutex_lock(&neo_dcp->swapped_out_fbs_lock); - list_for_each_entry(entry, &neo_dcp->swapped_out_fbs, head) - if (entry->armed && entry->swap_id == swap_id) - entry->armed = false; - mutex_unlock(&neo_dcp->swapped_out_fbs_lock); -} - -void neo_dcp_release_retained_framebuffers(struct neo_apple_dcp *neo_dcp, u32 swap_id) -{ - struct neo_dcp_fb_reference *entry, *tmp; - LIST_HEAD(completed); - - mutex_lock(&neo_dcp->swapped_out_fbs_lock); - list_for_each_entry_safe(entry, tmp, &neo_dcp->swapped_out_fbs, head) { - if (!entry->armed || entry->swap_id != swap_id) - continue; - list_move_tail(&entry->head, &completed); - } - mutex_unlock(&neo_dcp->swapped_out_fbs_lock); - - list_for_each_entry_safe(entry, tmp, &completed, head) { - list_del(&entry->head); - drm_framebuffer_put(entry->fb); - kfree(entry); - } -} - -void neo_dcp_release_all_retained_framebuffers(struct neo_apple_dcp *neo_dcp) -{ - struct neo_dcp_fb_reference *entry, *tmp; - LIST_HEAD(completed); - - mutex_lock(&neo_dcp->swapped_out_fbs_lock); - list_splice_init(&neo_dcp->swapped_out_fbs, &completed); - mutex_unlock(&neo_dcp->swapped_out_fbs_lock); - - list_for_each_entry_safe(entry, tmp, &completed, head) { - list_del(&entry->head); - drm_framebuffer_put(entry->fb); - kfree(entry); - } -} - -static int neo_dcp_create_piodma_iommu_dev(struct neo_apple_dcp *neo_dcp) -{ - int ret; - struct device_node *node __free(device_node) = of_get_child_by_name(neo_dcp->dev->of_node, "piodma"); - - if (!node) - return dev_err_probe(neo_dcp->dev, -ENODEV, - "Failed to get piodma child DT node\n"); - - neo_dcp->piodma = of_find_device_by_node(node); - neo_dcp->piodma_created = !neo_dcp->piodma; - if (neo_dcp->piodma_created) - neo_dcp->piodma = of_platform_device_create(node, NULL, neo_dcp->dev); - if (!neo_dcp->piodma) - return dev_err_probe(neo_dcp->dev, -ENODEV, - "Failed to create piodma pdev for %pOF\n", node); - - ret = dma_set_mask_and_coherent(&neo_dcp->piodma->dev, DMA_BIT_MASK(42)); - if (ret) - goto err_destroy_pdev; - - ret = of_dma_configure(&neo_dcp->piodma->dev, node, true); - if (ret) { - ret = dev_err_probe(neo_dcp->dev, ret, - "Failed to configure IOMMU child DMA\n"); - goto err_destroy_pdev; - } - - neo_dcp->iommu_dom = iommu_get_domain_for_dev(&neo_dcp->piodma->dev); - if (IS_ERR(neo_dcp->iommu_dom)) { - ret = dev_err_probe(neo_dcp->dev, PTR_ERR(neo_dcp->iommu_dom), - "Failed to get default iommu domain for " - "piodma device\n"); - neo_dcp->iommu_dom = NULL; - goto err_destroy_pdev; - } - - return 0; -err_destroy_pdev: - neo_dcp_release_piodma_iommu_dev(neo_dcp); - return ret; -} - -static int neo_dcp_get_bw_scratch_reg(struct neo_apple_dcp *neo_dcp, u32 expected) -{ - struct of_phandle_args ph_args; - u32 addr_idx, disp_idx, offset; - int ret; - - ret = of_parse_phandle_with_args(neo_dcp->dev->of_node, "apple,bw-scratch", - "#apple,bw-scratch-cells", 0, &ph_args); - if (ret < 0) { - dev_err(neo_dcp->dev, "Failed to read 'apple,bw-scratch': %d\n", ret); - return ret; - } - - if (ph_args.args_count != 3) { - dev_err(neo_dcp->dev, "Unexpected 'apple,bw-scratch' arg count %d\n", - ph_args.args_count); - ret = -EINVAL; - goto err_of_node_put; - } - - addr_idx = ph_args.args[0]; - disp_idx = ph_args.args[1]; - offset = ph_args.args[2]; - - if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { - dev_err(neo_dcp->dev, "Unexpected disp_reg value in 'apple,bw-scratch': %d\n", - disp_idx); - ret = -EINVAL; - goto err_of_node_put; - } - - ret = of_address_to_resource(ph_args.np, addr_idx, &neo_dcp->disp_bw_scratch_res); - if (ret < 0) { - dev_err(neo_dcp->dev, "Failed to get 'apple,bw-scratch' resource %d from %pOF\n", - addr_idx, ph_args.np); - goto err_of_node_put; - } - if (offset > resource_size(&neo_dcp->disp_bw_scratch_res) - 4) { - ret = -EINVAL; - goto err_of_node_put; - } - - neo_dcp->disp_registers[disp_idx] = &neo_dcp->disp_bw_scratch_res; - neo_dcp->disp_bw_scratch_index = disp_idx; - neo_dcp->disp_bw_scratch_offset = offset; - ret = 0; - -err_of_node_put: - of_node_put(ph_args.np); - return ret; -} - -static int neo_dcp_get_bw_doorbell_reg(struct neo_apple_dcp *neo_dcp, u32 expected) -{ - struct of_phandle_args ph_args; - u32 addr_idx, disp_idx; - int ret; - - ret = of_parse_phandle_with_args(neo_dcp->dev->of_node, "apple,bw-doorbell", - "#apple,bw-doorbell-cells", 0, &ph_args); - if (ret < 0) { - dev_err(neo_dcp->dev, "Failed to read 'apple,bw-doorbell': %d\n", ret); - return ret; - } - - if (ph_args.args_count != 2) { - dev_err(neo_dcp->dev, "Unexpected 'apple,bw-doorbell' arg count %d\n", - ph_args.args_count); - ret = -EINVAL; - goto err_of_node_put; - } - - addr_idx = ph_args.args[0]; - disp_idx = ph_args.args[1]; - - if (disp_idx != expected || disp_idx >= MAX_DISP_REGISTERS) { - dev_err(neo_dcp->dev, "Unexpected disp_reg value in 'apple,bw-doorbell': %d\n", - disp_idx); - ret = -EINVAL; - goto err_of_node_put; - } - - ret = of_address_to_resource(ph_args.np, addr_idx, &neo_dcp->disp_bw_doorbell_res); - if (ret < 0) { - dev_err(neo_dcp->dev, "Failed to get 'apple,bw-doorbell' resource %d from %pOF\n", - addr_idx, ph_args.np); - goto err_of_node_put; - } - neo_dcp->disp_bw_doorbell_index = disp_idx; - neo_dcp->disp_registers[disp_idx] = &neo_dcp->disp_bw_doorbell_res; - ret = 0; - -err_of_node_put: - of_node_put(ph_args.np); - return ret; -} - -static int neo_dcp_get_disp_regs(struct neo_apple_dcp *neo_dcp) -{ - struct platform_device *pdev = to_platform_device(neo_dcp->dev); - int count = 0; - int i, ret; - char name[16]; - const char *reg_name; - struct resource *res; - - if (of_property_present(neo_dcp->dev->of_node, "reg-names")) { - ret = of_property_count_strings(neo_dcp->dev->of_node, "reg-names"); - if (ret < 0) - return ret; - - for (i = 0; i < ret; i++) { - if (of_property_read_string_index(neo_dcp->dev->of_node, - "reg-names", i, ®_name)) - return -EINVAL; - if (!strncmp(reg_name, "disp-", 5)) - count++; - } - } else { - count = pdev->num_resources - 1; - } - - if (count <= 0 || count > MAX_DISP_REGISTERS) - return -EINVAL; - - for (i = 0; i < count; ++i) { - if (of_property_present(neo_dcp->dev->of_node, "reg-names")) { - snprintf(name, sizeof(name), "disp-%d", i); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - name); - } else { - res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); - } - if (!res) - return -EINVAL; - neo_dcp->disp_registers[i] = res; - } - - if (neo_dcp->hw.firmware_clock) { - /* The internal firmware's last aperture follows five display windows. */ - if (count != 5 || resource_size(neo_dcp->hw.firmware_clock) < 4 || - neo_dcp->hw.firmware_scratch > resource_size(neo_dcp->hw.firmware_clock) - 4 || - neo_dcp->hw.firmware_request > resource_size(neo_dcp->hw.firmware_clock) - 4) - return -EINVAL; - - neo_dcp->disp_bw_scratch_res = *neo_dcp->hw.firmware_clock; - neo_dcp->disp_bw_scratch_index = count; - neo_dcp->disp_bw_scratch_offset = neo_dcp->hw.firmware_scratch; - neo_dcp->disp_bw_doorbell_res = neo_dcp->disp_bw_scratch_res; - neo_dcp->disp_bw_doorbell_res.start += neo_dcp->hw.firmware_request; - neo_dcp->disp_bw_doorbell_res.end = neo_dcp->disp_bw_doorbell_res.start + 3; - neo_dcp->disp_registers[count] = &neo_dcp->disp_bw_scratch_res; - neo_dcp->nr_disp_registers = count + 1; - return 0; - } - - /* load pmgr bandwidth scratch resource and offset */ - ret = neo_dcp_get_bw_scratch_reg(neo_dcp, count); - if (ret < 0) - return ret; - count += 1; - - /* load pmgr bandwidth doorbell resource if present (only on t8103) */ - if (of_property_present(neo_dcp->dev->of_node, "apple,bw-doorbell")) { - ret = neo_dcp_get_bw_doorbell_reg(neo_dcp, count); - if (ret < 0) - return ret; - count += 1; - } - - neo_dcp->nr_disp_registers = count; - return 0; -} - -#define DCP_FW_VERSION_MIN_LEN 3 -#define DCP_FW_VERSION_MAX_LEN 5 -#define DCP_FW_VERSION_STR_LEN (DCP_FW_VERSION_MAX_LEN * 4) - -static int neo_dcp_read_fw_version(struct device *dev, const char *name, - char *version_str) -{ - u32 ver[DCP_FW_VERSION_MAX_LEN]; - int len_str; - int len; - - len = of_property_read_variable_u32_array(dev->of_node, name, ver, - DCP_FW_VERSION_MIN_LEN, - DCP_FW_VERSION_MAX_LEN); - - switch (len) { - case 3: - len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, - "%d.%d.%d", ver[0], ver[1], ver[2]); - break; - case 4: - len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, - "%d.%d.%d.%d", ver[0], ver[1], ver[2], - ver[3]); - break; - case 5: - len_str = scnprintf(version_str, DCP_FW_VERSION_STR_LEN, - "%d.%d.%d.%d.%d", ver[0], ver[1], ver[2], - ver[3], ver[4]); - break; - default: - len_str = strscpy(version_str, "UNKNOWN", - DCP_FW_VERSION_STR_LEN); - if (len >= 0) - len = -EOVERFLOW; - break; - } - - if (len_str >= DCP_FW_VERSION_STR_LEN) - dev_warn(dev, "'%s' truncated: '%s'\n", name, version_str); - - return len; -} - -static enum neo_dcp_firmware_version neo_dcp_check_firmware_version(struct device *dev) -{ - const struct neo_apple_dcp_hw_data *hw = of_device_get_match_data(dev); - char compat_str[DCP_FW_VERSION_STR_LEN]; - char fw_str[DCP_FW_VERSION_STR_LEN]; - int ret; - - /* - * SoCs introduced with H17-generation firmware pin its interface; the - * loader may not recognise their boot firmware version. - */ - if (hw->firmware_compat != DCP_FIRMWARE_UNKNOWN) - return hw->firmware_compat; - - /* firmware version is just informative */ - neo_dcp_read_fw_version(dev, "apple,firmware-version", fw_str); - - ret = neo_dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); - if (ret < 0) { - dev_err(dev, "Could not read 'apple,firmware-compat': %d\n", ret); - return DCP_FIRMWARE_UNKNOWN; - } - - if (strncmp(compat_str, "12.3.0", sizeof(compat_str)) == 0) - return DCP_FIRMWARE_V_12_3; - /* - * m1n1 reports firmware version 13.5 as compatible with 13.3. This is - * only true for the iomfb endpoint. The interface for the dptx-port - * endpoint changed between 13.3 and 13.5. The driver will only support - * firmware 13.5. Check the actual firmware version for compat version - * 13.3 until m1n1 reports 13.5 as "firmware-compat". - */ - else if ((strncmp(compat_str, "13.3.0", sizeof(compat_str)) == 0) && - (strncmp(fw_str, "13.5.0", sizeof(compat_str)) == 0)) - return DCP_FIRMWARE_V_13_5; - else if (strncmp(compat_str, "13.5.0", sizeof(compat_str)) == 0) - return DCP_FIRMWARE_V_13_5; - - dev_err(dev, "DCP firmware-compat %s (FW: %s) is not supported\n", - compat_str, fw_str); - - return DCP_FIRMWARE_UNKNOWN; -} - -static int neo_dcp_connector_type_from_dt(struct device_node *np) -{ - if (of_property_match_string(np, "apple,connector-type", "HDMI-A") >= 0) - return DRM_MODE_CONNECTOR_HDMIA; - if (of_property_match_string(np, "apple,connector-type", "DP") >= 0) - return DRM_MODE_CONNECTOR_DisplayPort; - if (of_property_match_string(np, "apple,connector-type", "USB-C") >= 0) - return DRM_MODE_CONNECTOR_USB; - - return DRM_MODE_CONNECTOR_Unknown; -} - -static void neo_dcp_disable_typec_work(struct neo_apple_dcp *neo_dcp, bool release_cable) -{ - scoped_guard(mutex, &neo_dcp->hpd_mutex) { - if (release_cable) - WRITE_ONCE(neo_dcp->typec_cable_connected, false); - neo_dcp->typec_generation++; - } - /* Block new enqueues as well as draining users of the AFK endpoints. */ - disable_delayed_work_sync(&neo_dcp->typec_reconnect_wq); - disable_delayed_work_sync(&neo_dcp->placeholder_edid_wq); - disable_delayed_work_sync(&neo_dcp->typec_fabric_retrain_wq); -} - -static void neo_dcp_enable_typec_work(struct neo_apple_dcp *neo_dcp) -{ - enable_delayed_work(&neo_dcp->typec_reconnect_wq); - enable_delayed_work(&neo_dcp->placeholder_edid_wq); - enable_delayed_work(&neo_dcp->typec_fabric_retrain_wq); - /* A cable can be routed before the DRM component binds. */ - if (READ_ONCE(neo_dcp->typec_cable_connected)) - mod_delayed_work(system_freezable_wq, &neo_dcp->typec_reconnect_wq, 0); -} - -static int neo_dcp_comp_bind(struct device *dev, struct device *main, void *data) -{ - struct device_node *panel_np; - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - u32 cpu_ctrl; - int ret; - - if (READ_ONCE(neo_dcp->quiescing)) - return dev_err_probe(dev, -EBUSY, - "Live DCP session cannot be rebound; reboot required\n"); - - /* A timed-out prior session may still scan these mappings. */ - mutex_lock(&neo_dcp->swapped_out_fbs_lock); - ret = list_empty(&neo_dcp->swapped_out_fbs) ? 0 : -EBUSY; - mutex_unlock(&neo_dcp->swapped_out_fbs_lock); - if (ret) - return dev_err_probe(dev, ret, - "Previous scanout has not been stopped\n"); - - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); - if (ret) - return ret; - - neo_dcp->coproc_reg = devm_platform_ioremap_resource_byname(to_platform_device(dev), "coproc"); - if (IS_ERR(neo_dcp->coproc_reg)) - return PTR_ERR(neo_dcp->coproc_reg); - - /* - * Display coprocessors running H17-generation firmware are left running - * by the bootloader with their firmware mapped through a locked DART. - * Linux attaches to that session with a standard RTKit INIT/HELLO - * exchange and must not stop or restart the ASC. - */ - if (neo_dcp->hw.adopt_live_session) { - cpu_ctrl = readl_relaxed(neo_dcp->coproc_reg + - APPLE_DCP_COPROC_CPU_CONTROL); - if (!(cpu_ctrl & APPLE_DCP_COPROC_CPU_CONTROL_RUN)) - return dev_err_probe(dev, -EOPNOTSUPP, - "DCP was not left running by the bootloader\n"); - } - - if (neo_dcp->index || neo_dcp->neo_dptx_phy || neo_dcp->neo_dptx_die) - dev_info(dev, "DCP index:%u dptx target phy: %u dptx die: %u\n", - neo_dcp->index, neo_dcp->neo_dptx_phy, neo_dcp->neo_dptx_die); - - if (!show_notch) - ret = of_property_read_u32(dev->of_node, "apple,notch-height", - &neo_dcp->notch_height); - - if (neo_dcp->notch_height > MAX_NOTCH_HEIGHT) - neo_dcp->notch_height = MAX_NOTCH_HEIGHT; - if (neo_dcp->notch_height > 0) - dev_info(dev, "Detected display with notch of %u pixel\n", neo_dcp->notch_height); - - /* initialize brightness scale to a sensible default to avoid divide by 0*/ - neo_dcp->brightness.scale = 65536; - panel_np = of_get_compatible_child(dev->of_node, "apple,panel-mini-led"); - if (panel_np) - neo_dcp->panel.has_mini_led = true; - else - panel_np = of_get_compatible_child(dev->of_node, "apple,panel"); - - if (panel_np) { - const char height_prop[2][16] = { "adj-height-mm", "height-mm" }; - - if (of_device_is_available(panel_np)) { - ret = of_property_read_u32(panel_np, "apple,max-brightness", - &neo_dcp->brightness.maximum); - if (ret) - dev_err(dev, "Missing property 'apple,max-brightness'\n"); - } - - of_property_read_u32(panel_np, "width-mm", &neo_dcp->panel.width_mm); - /* use adjusted height as long as the notch is hidden */ - of_property_read_u32(panel_np, height_prop[!neo_dcp->notch_height], - &neo_dcp->panel.height_mm); - - of_node_put(panel_np); - neo_dcp->fixed_connector_type = DRM_MODE_CONNECTOR_eDP; - neo_dcp->connector_type = DRM_MODE_CONNECTOR_eDP; - INIT_WORK(&neo_dcp->bl_register_wq, neo_dcp_work_register_backlight); - mutex_init(&neo_dcp->bl_register_mutex); - INIT_WORK(&neo_dcp->bl_update_wq, neo_dcp_work_update_backlight); - } - - ret = neo_dcp_create_piodma_iommu_dev(neo_dcp); - if (ret) - return dev_err_probe(dev, ret, - "Failed to create PIODMA iommu child device\n"); - if (!neo_dcp->iommu_dom) { - ret = dev_err_probe(dev, -EPROBE_DEFER, - "PIODMA iommu domain is unavailable\n"); - goto err_piodma; - } - - ret = neo_dcp_get_disp_regs(neo_dcp); - if (ret) { - dev_err(dev, "failed to find display registers\n"); - goto err_piodma; - } - - neo_dcp->clk = devm_clk_get(dev, NULL); - if (IS_ERR(neo_dcp->clk)) { - ret = dev_err_probe(dev, PTR_ERR(neo_dcp->clk), - "Unable to find clock\n"); - goto err_piodma; - } - neo_dcp->clk_194 = devm_clk_get_optional(dev, "clock-194"); - if (IS_ERR(neo_dcp->clk_194)) { - ret = dev_err_probe(dev, PTR_ERR(neo_dcp->clk_194), - "Unable to find clock 0x194\n"); - goto err_piodma; - } - - bitmap_zero(neo_dcp->memdesc_map, DCP_MAX_MAPPINGS); - // TDOD: mem_desc IDs start at 1, for simplicity just skip '0' entry - set_bit(0, neo_dcp->memdesc_map); - - INIT_WORK(&neo_dcp->vblank_wq, neo_dcp_delayed_vblank); - INIT_DELAYED_WORK(&neo_dcp->swap_watchdog_wq, neo_dcp_swap_watchdog); - - if (!neo_dcp->hw.adopt_live_session) { - cpu_ctrl = - readl_relaxed(neo_dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - writel_relaxed(cpu_ctrl | APPLE_DCP_COPROC_CPU_CONTROL_RUN, - neo_dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); - } - - /* Registering the mailbox can already expose RTKit buffer requests. */ - ret = neo_dcp_pin_live_session(neo_dcp); - if (ret) - goto err_piodma; - neo_dcp->rtk = apple_rtkit_init(dev, neo_dcp, "mbox", 0, &rtkit_ops); - if (IS_ERR(neo_dcp->rtk)) { - ret = dev_err_probe(dev, PTR_ERR(neo_dcp->rtk), - "Failed to initialize RTKit\n"); - goto err_piodma; - } - ret = devm_add_action_or_reset(dev, neo_dcp_release_rtkit, neo_dcp); - if (ret) - goto err_piodma; - - if (neo_dcp->hw.adopt_live_session) - dev_info(dev, "negotiating RTKit with the running DCP\n"); - - ret = apple_rtkit_wake(neo_dcp->rtk); - if (ret) { - ret = dev_err_probe(dev, ret, "Failed to boot RTKit: %d\n", ret); - goto err_piodma; - } - neo_dcp_enable_typec_work(neo_dcp); - return 0; - -err_piodma: - neo_dcp_release_piodma_iommu_dev(neo_dcp); - return ret; -} - -/* - * We need to shutdown DCP before tearing down the display subsystem. Otherwise - * the DCP will crash and briefly flash a green screen of death. - */ -static void neo_dcp_comp_unbind(struct device *dev, struct device *main, void *data) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - - if (!neo_dcp) - return; - - if (neo_dcp->retain_dma) { - /* No firmware stop is acknowledged: detach host users, never DMA. */ - WRITE_ONCE(neo_dcp->quiescing, true); - if (neo_dcp->crtc && !neo_dcp->drm_retained) { - drm_dev_get(neo_dcp->crtc->base.dev); - neo_dcp->drm_retained = true; - } - neo_iomfb_queue_stop(neo_dcp); - neo_afk_quiesce(neo_dcp->neo_avep); - neo_afk_quiesce(neo_dcp->neo_dptxep); - neo_afk_quiesce(neo_dcp->neo_ibootep); - neo_afk_quiesce(neo_dcp->neo_systemep); - neo_afk_quiesce(neo_dcp->dcpavservep); - cancel_delayed_work_sync(&neo_dcp->bl_fallback_wq); - if (neo_dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { - cancel_work_sync(&neo_dcp->bl_register_wq); - cancel_work_sync(&neo_dcp->bl_update_wq); - } - cancel_delayed_work_sync(&neo_dcp->typec_reconnect_wq); - cancel_delayed_work_sync(&neo_dcp->typec_fabric_retrain_wq); - cancel_work_sync(&neo_dcp->vblank_wq); - if (neo_dcp->connector) - cancel_work_sync(&neo_dcp->connector->hotplug_wq); - if (neo_dcp->typec_connector && neo_dcp->typec_connector != neo_dcp->connector) - cancel_work_sync(&neo_dcp->typec_connector->hotplug_wq); - dev_warn(dev, "retaining live DCP DMA resources after unbind; reboot required\n"); - return; - } - - if (neo_dcp->hdmi_hpd_irq) - disable_irq(neo_dcp->hdmi_hpd_irq); - - neo_dcp_disable_typec_work(neo_dcp, true); - typec_mux_put(neo_dcp->typec_mux); - - if (neo_dcp->neo_avep) { - neo_av_service_disconnect(neo_dcp); - neo_afk_shutdown(neo_dcp->neo_avep); - neo_dcp->neo_avep = NULL; - } - - if (neo_dcp->neo_dptxep) { - neo_afk_shutdown(neo_dcp->neo_dptxep); - neo_dcp->neo_dptxep = NULL; - } - - if (neo_dcp->neo_ibootep) { - neo_afk_shutdown(neo_dcp->neo_ibootep); - neo_dcp->neo_ibootep = NULL; - } - - if (neo_dcp->neo_systemep) { - neo_afk_shutdown(neo_dcp->neo_systemep); - neo_dcp->neo_systemep = NULL; - } - - if (neo_dcp->dcpavservep) { - neo_afk_shutdown(neo_dcp->dcpavservep); - dpavservep_detach(neo_dcp); - neo_dcp->dcpavservep = NULL; - } - - if (neo_dcp->shmem) - neo_iomfb_shutdown(neo_dcp); - - neo_iomfb_queue_stop(neo_dcp); - neo_dcp_release_piodma_iommu_dev(neo_dcp); - - cancel_delayed_work_sync(&neo_dcp->bl_fallback_wq); - if (neo_dcp->connector_type == DRM_MODE_CONNECTOR_eDP) { - cancel_work_sync(&neo_dcp->bl_register_wq); - cancel_work_sync(&neo_dcp->bl_update_wq); - } - cancel_delayed_work_sync(&neo_dcp->swap_watchdog_wq); - cancel_work_sync(&neo_dcp->vblank_wq); - - devm_clk_put(dev, neo_dcp->clk); - neo_dcp->clk = NULL; - /* optional and possibly NULL, released with the bind's devres group */ - neo_dcp->clk_194 = NULL; -} - -static const struct component_ops neo_dcp_comp_ops = { - .bind = neo_dcp_comp_bind, - .unbind = neo_dcp_comp_unbind, -}; - -static int neo_dcp_platform_probe(struct platform_device *pdev) -{ - const struct neo_apple_dcp_hw_data *hw = of_device_get_match_data(&pdev->dev); - enum neo_dcp_firmware_version fw_compat; - struct device *dev = &pdev->dev; - struct neo_apple_dcp *neo_dcp; - int ret, surf, num_surfs; - u32 surf_en; - u32 mux_index; - - if (neo_dcp_session_retained(dev)) - return dev_err_probe(dev, -EBUSY, "Previous live DCP session requires a reboot\n"); - - /* Not part of the display subsystem yet; see apple_dcp_usable(). */ - if (of_device_is_compatible(dev->of_node, "apple,t8140-dcpext")) { - dev_info(dev, "external display coprocessor not supported yet\n"); - return -ENODEV; - } - - fw_compat = neo_dcp_check_firmware_version(dev); - if (fw_compat == DCP_FIRMWARE_UNKNOWN) - return -ENODEV; - - /* Check for "apple,bw-scratch" to avoid probing appledrm_neo with outdated - * device trees. This prevents replacing simpledrm and ending up without - * display. - */ - if (!of_property_present(dev->of_node, "apple,bw-scratch") && - !hw->firmware_clock) - return dev_err_probe(dev, -ENODEV, "Incompatible devicetree! " - "Use devicetree matching this kernel.\n"); - - neo_dcp = kzalloc_obj(*neo_dcp); - if (!neo_dcp) - return -ENOMEM; - ret = devm_add_action_or_reset(dev, neo_dcp_release_context, neo_dcp); - if (ret) - return ret; - - INIT_LIST_HEAD(&neo_dcp->swapped_out_fbs); - mutex_init(&neo_dcp->swapped_out_fbs_lock); - spin_lock_init(&neo_dcp->backlight.lock); - neo_iomfb_queue_init(neo_dcp); - - neo_dcp->fw_compat = fw_compat; - neo_dcp->dev = dev; - /* - * Type-C and Thunderbolt routes can be activated as soon as they are - * registered below, before the DRM device binds. - */ - mutex_init(&neo_dcp->hpd_mutex); - mutex_init(&neo_dcp->tb_lock); - spin_lock_init(&neo_dcp->mode_state.lock); - spin_lock_init(&neo_dcp->neo_dcpavserv.lock); - neo_dcp->hw = *(struct neo_apple_dcp_hw_data *)of_device_get_match_data(dev); - - /* The domain use must survive driver-core DMA cleanup for a live session. */ - ret = iommu_device_use_default_domain(dev); - if (ret) - return ret; - ret = devm_add_action_or_reset(dev, neo_dcp_release_dma_domain, neo_dcp); - if (ret) - return ret; - neo_dcp->fixed_connector_type = neo_dcp_connector_type_from_dt(dev->of_node); - neo_dcp->connector_type = neo_dcp->fixed_connector_type; - of_property_read_u32(dev->of_node, "apple,dcp-index", &neo_dcp->index); - of_property_read_u32(dev->of_node, "apple,dptx-phy", &neo_dcp->neo_dptx_phy); - of_property_read_u32(dev->of_node, "apple,dptx-die", &neo_dcp->neo_dptx_die); - neo_dcp->fixed_dptx_phy = neo_dcp->neo_dptx_phy; - init_completion(&neo_dcp->typec_iomfb_hpd_ready); - INIT_DELAYED_WORK(&neo_dcp->typec_reconnect_wq, - neo_dcp_typec_reconnect_work); - INIT_DELAYED_WORK(&neo_dcp->placeholder_edid_wq, - neo_dcp_placeholder_edid_work); - INIT_DELAYED_WORK(&neo_dcp->typec_fabric_retrain_wq, - neo_dcp_typec_retrain_work); - INIT_DELAYED_WORK(&neo_dcp->bl_fallback_wq, neo_dcp_work_backlight_fallback); - /* Balanced by enable at successful component bind. */ - disable_delayed_work(&neo_dcp->typec_reconnect_wq); - disable_delayed_work(&neo_dcp->placeholder_edid_wq); - disable_delayed_work(&neo_dcp->typec_fabric_retrain_wq); - - platform_set_drvdata(pdev, neo_dcp); - - neo_dcp->phy = devm_phy_optional_get(dev, "dp-phy"); - if (IS_ERR(neo_dcp->phy)) { - dev_err(dev, "Failed to get dp-phy: %ld\n", PTR_ERR(neo_dcp->phy)); - return PTR_ERR(neo_dcp->phy); - } - neo_dcp->fixed_phy = neo_dcp->phy; - - bitmap_zero(neo_dcp->neo_iomfb_surfaces, DCP_MAX_PLANES); - if (!of_property_present(dev->of_node, "apple,iomfb-surfaces")) - num_surfs = 0; - else - num_surfs = of_property_count_elems_of_size(dev->of_node, - "apple,iomfb-surfaces", - sizeof(u32)); - - if (num_surfs == 0 || num_surfs == -ENODATA) { - set_bit(0, neo_dcp->neo_iomfb_surfaces); - set_bit(1, neo_dcp->neo_iomfb_surfaces); - } else if (num_surfs < 0) { - return num_surfs; - } else if (num_surfs > DCP_MAX_PLANES) { - dev_err(dev, "Number of iomfb-surfaces (%d) exceeds DCP_MAX_PLANES\n", - num_surfs); - return -EINVAL; - } - - surf = 0; - of_property_for_each_u32(dev->of_node, "apple,iomfb-surfaces", surf_en) { - if (surf_en) - set_bit(surf, neo_dcp->neo_iomfb_surfaces); - surf++; - } - /* The qualified J700 loader names surface zero rather than enabling it. */ - if (of_machine_is_compatible("apple,j700") && - of_device_is_compatible(dev->of_node, "apple,t8140-dcp") && - num_surfs == 1 && - !of_property_read_u32(dev->of_node, "apple,iomfb-surfaces", &surf_en) && - surf_en == 0) - set_bit(0, neo_dcp->neo_iomfb_surfaces); - - if (neo_dcp->phy) { - int ret; - /* - * Request DP2HDMI related GPIOs as optional for DP-altmode - * compatibility. J180D misses a dp2hdmi-pwren GPIO in the - * template ADT. TODO: check device ADT - */ - neo_dcp->hdmi_hpd = devm_gpiod_get_optional(dev, "hdmi-hpd", GPIOD_IN); - if (IS_ERR(neo_dcp->hdmi_hpd)) - return PTR_ERR(neo_dcp->hdmi_hpd); - if (neo_dcp->hdmi_hpd) { - int irq = gpiod_to_irq(neo_dcp->hdmi_hpd); - if (irq < 0) { - dev_err(dev, "failed to translate HDMI hpd GPIO to IRQ\n"); - return irq; - } - neo_dcp->hdmi_hpd_irq = irq; - - ret = devm_request_threaded_irq(dev, neo_dcp->hdmi_hpd_irq, - NULL, neo_dcp_dp2hdmi_hpd, - IRQF_ONESHOT | IRQF_NO_AUTOEN | - IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING, - "dp2hdmi-hpd-irq", neo_dcp); - if (ret < 0) { - dev_err(dev, "failed to request HDMI hpd irq %d: %d\n", - irq, ret); - return ret; - } - } - - /* - * Power DP2HDMI on as it is required for the HPD irq. - * TODO: check if one is sufficient for the hpd to save power - * on battery powered Macbooks. - */ - neo_dcp->hdmi_pwren = devm_gpiod_get_optional(dev, "hdmi-pwren", GPIOD_OUT_HIGH); - if (IS_ERR(neo_dcp->hdmi_pwren)) - return PTR_ERR(neo_dcp->hdmi_pwren); - - neo_dcp->dp2hdmi_pwren = devm_gpiod_get_optional(dev, "dp2hdmi-pwren", GPIOD_OUT_HIGH); - if (IS_ERR(neo_dcp->dp2hdmi_pwren)) - return PTR_ERR(neo_dcp->dp2hdmi_pwren); - - /* - * A DCP may have both a fixed HDMI/DP route and allocatable Type-C - * routes. Keep the fixed route selected until the allocator borrows - * this otherwise-idle pipeline for a Type-C display. - */ - ret = neo_dcp->fixed_phy ? - of_property_read_u32(dev->of_node, "mux-index", &mux_index) : - -ENODATA; - if (!ret) { - neo_dcp->fixed_mux_index = mux_index; - neo_dcp->xbar = devm_mux_control_get(dev, "dp-xbar"); - if (IS_ERR(neo_dcp->xbar)) { - dev_err(dev, "Failed to get dp-xbar: %ld\n", PTR_ERR(neo_dcp->xbar)); - return PTR_ERR(neo_dcp->xbar); - } - ret = mux_control_select(neo_dcp->xbar, mux_index); - if (ret) - dev_warn(dev, "mux_control_select failed: %d\n", ret); - else - neo_dcp->fixed_route_selected = true; - - /* - * Switch atcphy to DP-only. should move to a Macbook Pro - * 14-/16-inch specific DP-to-HDMI drm_bridge. - */ - neo_dcp->typec_mux = fwnode_typec_mux_get(dev_fwnode(neo_dcp->dev)); - if (!IS_ERR_OR_NULL(neo_dcp->typec_mux)) { - struct typec_altmode alt = { - .svid = USB_TYPEC_DP_SID, - }; - struct typec_mux_state state = { - .alt = &alt, - .mode = TYPEC_DP_STATE_C, - }; - int ret = typec_mux_set(neo_dcp->typec_mux, &state); - dev_info(dev, "typec_mux_set() returned: %d\n", ret); - if (!ret) - neo_dcp->phy_managed_by_typec = true; - } else { - dev_info(dev, "fwnode_typec_mux_get() returned: %ld\n", - IS_ERR(neo_dcp->typec_mux) ? PTR_ERR(neo_dcp->typec_mux) : 0); - neo_dcp->typec_mux = NULL; - } - } - } - - ret = neo_dcp_register_typec_routes(neo_dcp); - if (ret) - return ret; - - return component_add(&pdev->dev, &neo_dcp_comp_ops); -} - -static void neo_dcp_platform_remove(struct platform_device *pdev) -{ - component_del(&pdev->dev, &neo_dcp_comp_ops); -} - -static void neo_dcp_platform_shutdown(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - if (neo_dcp && neo_dcp->hw.adopt_live_session && - neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - /* - * This firmware keeps scanning across soft DPMS and has no - * qualified stop sequence. Component unbind releases the RTKit - * devres group and the piodma domain while firmware may still - * access them. Stop host submissions but keep the component and - * its DMA resources alive until the system resets. - */ - neo_iomfb_queue_stop(neo_dcp); - cancel_delayed_work_sync(&neo_dcp->bl_fallback_wq); - cancel_work_sync(&neo_dcp->bl_register_wq); - cancel_work_sync(&neo_dcp->bl_update_wq); - return; - } - - component_del(&pdev->dev, &neo_dcp_comp_ops); -} - -/* - * The display stays powered across system sleep, and the blanking present - * queued by the DRM and backlight suspend handlers must reach the firmware - * before its mailbox interrupt is suspended. - */ -static void neo_dcp_drain_for_sleep(struct neo_apple_dcp *neo_dcp) -{ - int pass; - - if (!neo_dcp->hw.adopt_live_session || neo_dcp->fw_compat != DCP_FIRMWARE_H17P || - neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return; - - /* - * A completed present can schedule one more brightness present, and a - * rejected one a delayed retry; run a pending retry now instead of - * letting it fire after the mailbox has suspended. Four passes cover - * the first attempt and the policy's three retries. - */ - for (pass = 0; pass < 4; pass++) { - flush_delayed_work(&neo_dcp->neo_iomfb.backlight_retry); - if (neo_dcp_has_panel(neo_dcp)) - flush_work(&neo_dcp->bl_update_wq); - if (!neo_iomfb_queue_drain(neo_dcp, msecs_to_jiffies(500))) { - dev_warn(neo_dcp->dev, "display updates still pending at suspend\n"); - return; - } - /* - * A crashed or stopped DCP takes no more presents, so a level - * still pending there can never be sent and is not waited for. - * Checking after the drain, which returns at once for such a - * DCP, also covers a crash during the pass. - */ - if (READ_ONCE(neo_dcp->crashed) || READ_ONCE(neo_dcp->neo_iomfb.stopped)) - return; - if (!neo_dcp_backlight_active(neo_dcp) || !neo_dcp_backlight_pending(neo_dcp)) - return; - } - dev_warn(neo_dcp->dev, "brightness change still pending at suspend\n"); -} - -static int neo_dcp_platform_suspend(struct device *dev) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - - neo_dcp_drain_for_sleep(neo_dcp); - /* - * The Type-C route reports cable removal through - * dcp_dptx_disconnect_oob(). A DP tunnel kept through the sleep stays - * connected, and resume powers the CRTC back up through dcp_poweron() - * as after DPMS off. - */ - neo_dcp_disable_typec_work(neo_dcp, false); - cancel_delayed_work_sync(&neo_dcp->swap_watchdog_wq); - - if (neo_dcp->neo_avep) - neo_av_service_disconnect(neo_dcp); - - if (neo_dcp->hdmi_hpd_irq) { - disable_irq(neo_dcp->hdmi_hpd_irq); - if (!neo_dcp->active_typec_route) { - disconnected_hpd_event(neo_dcp->connector); - neo_dcp_dptx_disconnect(neo_dcp, 0); - } - } - /* - * Set the device as a wakeup device, which forces its power - * domains to stay on. We need this as we do not support full - * shutdown properly yet. - */ - device_set_wakeup_path(dev); - - return 0; -} - -static int neo_dcp_platform_resume(struct device *dev) -{ - struct neo_apple_dcp *neo_dcp = dev_get_drvdata(dev); - - neo_dcp_enable_typec_work(neo_dcp); - if (neo_dcp->hdmi_hpd_irq) - enable_irq(neo_dcp->hdmi_hpd_irq); - - if (neo_dcp->neo_avep) - neo_av_service_connect(neo_dcp); - - return 0; -} - -static DEFINE_SIMPLE_DEV_PM_OPS(neo_dcp_platform_pm_ops, - neo_dcp_platform_suspend, neo_dcp_platform_resume); - - -static const struct neo_apple_dcp_hw_data neo_apple_dcp_hw_t6020 = { - .num_dptx_ports = 1, -}; - -static const struct neo_apple_dcp_hw_data neo_apple_dcp_hw_t8112 = { - .num_dptx_ports = 2, -}; - -/* The internal T8140 endpoint uses the measured H17P method profile. */ -static const struct resource t8140_firmware_clock = - DEFINE_RES_MEM(0x302800000, 0xbc000); - -static const struct neo_apple_dcp_hw_data neo_apple_dcp_hw_t8140 = { - .num_dptx_ports = 0, - .adopt_live_session = true, - .firmware_compat = DCP_FIRMWARE_H17P, - .firmware_clock = &t8140_firmware_clock, - .firmware_scratch = 0x20000, - .firmware_request = 0x68000, -}; - -/* - * M5 (T8142) runs H17-generation DCP firmware with the H17G method numbering - * and is left running by the bootloader. - */ -static const struct neo_apple_dcp_hw_data neo_apple_dcp_hw_t8142 = { - .num_dptx_ports = 0, - .neo_iomfb_method_profile = DCP_IOMFB_METHODS_H17G, - .adopt_live_session = true, - .firmware_compat = DCP_FIRMWARE_H17P, -}; - -static const struct neo_apple_dcp_hw_data neo_apple_dcp_hw_dcp = { - .num_dptx_ports = 0, -}; - -static const struct neo_apple_dcp_hw_data neo_apple_dcp_hw_dcpext = { - .num_dptx_ports = 2, -}; - -static const struct of_device_id of_match[] = { - { .compatible = "apple,t8140-dcp", .data = &neo_apple_dcp_hw_t8140, }, - {} -}; -MODULE_DEVICE_TABLE(of, of_match); - -static struct platform_driver apple_platform_driver = { - .driver_managed_dma = true, - .probe = neo_dcp_platform_probe, - .remove = neo_dcp_platform_remove, - .shutdown = neo_dcp_platform_shutdown, - .driver = { - .name = "apple-dcp-neo", - .of_match_table = of_match, - .pm = pm_sleep_ptr(&neo_dcp_platform_pm_ops), - }, -}; - -int __init neo_dcp_register(void) -{ - return platform_driver_register(&apple_platform_driver); -} - -void neo_dcp_unregister(void) -{ - platform_driver_unregister(&apple_platform_driver); -} diff --git a/drivers/gpu/drm/apple-neo/dcp.h b/drivers/gpu/drm/apple-neo/dcp.h deleted file mode 100644 index 7acf095f7b5022..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp.h +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -#ifndef __APPLE_DCP_H__ -#define __APPLE_DCP_H__ - -#include -#include -#include - -#include "connector.h" -#include "dcp-internal.h" -#include "parser.h" - -struct neo_apple_crtc { - struct drm_crtc base; - struct drm_pending_vblank_event *event; - bool vsync_disabled; - - /* Reference to the DCP device owning this CRTC */ - struct platform_device *neo_dcp; -}; - -#define to_apple_crtc(x) container_of(x, struct neo_apple_crtc, base) - -struct apple_encoder { - struct drm_encoder base; -}; - -#define to_apple_encoder(x) container_of(x, struct apple_encoder, base) - -void neo_dcp_poweroff(struct platform_device *pdev); -void neo_dcp_poweron(struct platform_device *pdev); -int neo_dcp_set_crc(struct drm_crtc *crtc, bool enabled); -int neo_dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state); -int neo_dcp_get_connector_type(struct platform_device *pdev); -bool neo_dcp_has_typec_routes(struct platform_device *pdev); - -/* - * The Type-C display fabric. Ports are enumerated in device-tree order, not - * DCP probe order, so a given physical port keeps the same DRM connector index - * across boots -- userspace keys its per-monitor configuration on that name. - */ -unsigned int neo_dcp_typec_nr_ports(void); -struct device_node *neo_dcp_typec_port_of_node(unsigned int idx); -bool neo_dcp_typec_port_has_candidate(unsigned int idx, struct platform_device *pdev); -void neo_dcp_typec_port_set_connector(unsigned int idx, bool secondary, - struct neo_apple_connector *connector); -bool neo_dcp_typec_dual_stream(void); -void neo_dcp_typec_reorder(void); -bool neo_dcp_is_typec_only(struct platform_device *pdev); -bool neo_dcp_fw_compat_is_12_x(struct platform_device *pdev); -unsigned long* neo_dcp_get_iomfb_surfaces(struct platform_device *pdev); -void neo_dcp_link(struct platform_device *pdev, struct neo_apple_crtc *apple, - struct neo_apple_connector *connector); -int neo_dcp_start(struct platform_device *pdev); -int neo_dcp_wait_ready(struct platform_device *pdev, u64 timeout); -void neo_dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state); -void neo_dcp_retain_framebuffer(struct platform_device *pdev, - struct neo_dcp_fb_reference *entry); -void neo_dcp_arm_retained_framebuffers(struct neo_apple_dcp *neo_dcp, u32 swap_id); -void neo_dcp_unarm_retained_framebuffers(struct neo_apple_dcp *neo_dcp, u32 swap_id); -void neo_dcp_release_retained_framebuffers(struct neo_apple_dcp *neo_dcp, u32 swap_id); -void neo_dcp_release_all_retained_framebuffers(struct neo_apple_dcp *neo_dcp); -bool neo_dcp_is_initialized(struct platform_device *pdev); -void neo_apple_crtc_vblank(struct neo_apple_crtc *apple); -void neo_dcp_drm_crtc_vblank(struct neo_apple_crtc *crtc); -int neo_dcp_get_modes(struct drm_connector *connector); -enum drm_mode_status neo_dcp_mode_valid(struct drm_connector *connector, - const struct drm_display_mode *mode); -int neo_dcp_crtc_atomic_modeset(struct drm_crtc *crtc, - struct drm_atomic_state *state); -bool neo_dcp_crtc_mode_fixup(struct drm_crtc *crtc, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode); -void neo_dcp_set_dimensions(struct neo_apple_dcp *neo_dcp); -void neo_dcp_send_message(struct neo_apple_dcp *neo_dcp, u8 endpoint, u64 message); - -int neo_dcp_dptx_connect_oob(struct platform_device *pdev, u32 port); -int neo_dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port); - -struct neo_apple_dcp; -struct phy; -/* Thunderbolt DP tunnels, called from the DPTX endpoint */ -int neo_dcp_tunnel_crossbar_up(struct neo_apple_dcp *neo_dcp); -int neo_dcp_tunnel_crossbar_down(struct neo_apple_dcp *neo_dcp); -int neo_dcp_tunnel_set_rate(struct neo_apple_dcp *neo_dcp, struct phy *phy, u32 link_rate); -int neo_dcp_tunnel_dpin_activate(struct neo_apple_dcp *neo_dcp, bool active); - -int neo_iomfb_start_rtkit(struct neo_apple_dcp *neo_dcp); -void neo_iomfb_shutdown(struct neo_apple_dcp *neo_dcp); -/* rtkit message handler for IOMFB messages */ -void neo_iomfb_recv_msg(struct neo_apple_dcp *neo_dcp, u64 message); - -int neo_systemep_init(struct neo_apple_dcp *neo_dcp); -int neo_dptxep_init(struct neo_apple_dcp *neo_dcp); -int neo_ibootep_init(struct neo_apple_dcp *neo_dcp); -int dpavservep_init(struct neo_apple_dcp *neo_dcp); -int neo_avep_init(struct neo_apple_dcp *neo_dcp); - - -int __init neo_dcp_register(void); -void neo_dcp_unregister(void); - -int __init neo_dcp_audio_register(void); -void neo_dcp_audio_unregister(void); - -#endif diff --git a/drivers/gpu/drm/apple-neo/dcp_backlight.c b/drivers/gpu/drm/apple-neo/dcp_backlight.c deleted file mode 100644 index f9fad8ca2af323..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp_backlight.c +++ /dev/null @@ -1,476 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright (C) The Asahi Linux Contributors */ - -#include -#include -#include -#include - -#include -#include -#include -#include "linux/jiffies.h" - -#include "dcp.h" -#include "dcp-internal.h" - -#define MIN_BRIGHTNESS_PART1 2U -#define MAX_BRIGHTNESS_PART1 99U -#define MIN_BRIGHTNESS_PART2 103U -#define MAX_BRIGHTNESS_PART2 510U - -/* - * lookup for display brightness 2 to 99 nits - * */ -static u32 brightness_part1[] = { - 0x0000000, 0x0810038, 0x0f000bd, 0x143011c, - 0x1850165, 0x1bc01a1, 0x1eb01d4, 0x2140200, - 0x2380227, 0x2590249, 0x2770269, 0x2930285, - 0x2ac02a0, 0x2c402b8, 0x2d902cf, 0x2ee02e4, - 0x30102f8, 0x314030b, 0x325031c, 0x335032d, - 0x345033d, 0x354034d, 0x362035b, 0x3700369, - 0x37d0377, 0x38a0384, 0x3960390, 0x3a2039c, - 0x3ad03a7, 0x3b803b3, 0x3c303bd, 0x3cd03c8, - 0x3d703d2, 0x3e103dc, 0x3ea03e5, 0x3f303ef, - 0x3fc03f8, 0x4050400, 0x40d0409, 0x4150411, - 0x41d0419, 0x4250421, 0x42d0429, 0x4340431, - 0x43c0438, 0x443043f, 0x44a0446, 0x451044d, - 0x4570454, 0x45e045b, 0x4640461, 0x46b0468, - 0x471046e, 0x4770474, 0x47d047a, 0x4830480, - 0x4890486, 0x48e048b, 0x4940491, 0x4990497, - 0x49f049c, 0x4a404a1, 0x4a904a7, 0x4ae04ac, - 0x4b304b1, 0x4b804b6, 0x4bd04bb, 0x4c204c0, - 0x4c704c5, 0x4cc04c9, 0x4d004ce, 0x4d504d3, - 0x4d904d7, 0x4de04dc, 0x4e204e0, 0x4e704e4, - 0x4eb04e9, 0x4ef04ed, 0x4f304f1, 0x4f704f5, - 0x4fb04f9, 0x4ff04fd, 0x5030501, 0x5070505, - 0x50b0509, 0x50f050d, 0x5130511, 0x5160515, - 0x51a0518, 0x51e051c, 0x5210520, 0x5250523, - 0x5290527, 0x52c052a, 0x52f052e, 0x5330531, - 0x5360535, 0x53a0538, 0x53d053b, 0x540053f, - 0x5440542, 0x5470545, 0x54a0548, 0x54d054c, - 0x550054f, 0x5530552, 0x5560555, 0x5590558, - 0x55c055b, 0x55f055e, 0x5620561, 0x5650564, - 0x5680567, 0x56b056a, 0x56e056d, 0x571056f, - 0x5740572, 0x5760575, 0x5790578, 0x57c057b, - 0x57f057d, 0x5810580, 0x5840583, 0x5870585, - 0x5890588, 0x58c058b, 0x58f058d -}; - -static u32 brightness_part12[] = { 0x58f058d, 0x59d058f }; - -/* - * lookup table for display brightness 103.3 to 510 nits - * */ -static u32 brightness_part2[] = { - 0x59d058f, 0x5b805ab, 0x5d105c5, 0x5e805dd, - 0x5fe05f3, 0x6120608, 0x625061c, 0x637062e, - 0x6480640, 0x6580650, 0x6680660, 0x677066f, - 0x685067e, 0x693068c, 0x6a00699, 0x6ac06a6, - 0x6b806b2, 0x6c406be, 0x6cf06ca, 0x6da06d5, - 0x6e506df, 0x6ef06ea, 0x6f906f4, 0x70206fe, - 0x70c0707, 0x7150710, 0x71e0719, 0x7260722, - 0x72f072a, 0x7370733, 0x73f073b, 0x7470743, - 0x74e074a, 0x7560752, 0x75d0759, 0x7640760, - 0x76b0768, 0x772076e, 0x7780775, 0x77f077c, - 0x7850782, 0x78c0789, 0x792078f, 0x7980795, - 0x79e079b, 0x7a407a1, 0x7aa07a7, 0x7af07ac, - 0x7b507b2, 0x7ba07b8, 0x7c007bd, 0x7c507c2, - 0x7ca07c8, 0x7cf07cd, 0x7d407d2, 0x7d907d7, - 0x7de07dc, 0x7e307e1, 0x7e807e5, 0x7ec07ea, - 0x7f107ef, 0x7f607f3, 0x7fa07f8, 0x7fe07fc -}; - - -bool neo_dcp_backlight_active(struct neo_apple_dcp *neo_dcp) -{ - return neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - READ_ONCE(neo_dcp->backlight.state.ready); -} - -int neo_dcp_backlight_takeover(struct neo_apple_dcp *neo_dcp, u32 millinits) -{ - u32 maximum, nits; - int ret; - - if (neo_dcp_backlight_active(neo_dcp)) - return 0; - /* H17P takes nits up to the panel ceiling, not the older DAC table's. */ - maximum = neo_dcp->brightness.maximum; - ret = neo_dcp_bl_takeover_nits(maximum, millinits, &nits); - if (ret) - return ret; - - /* Keep the first bounded hint. Later reports are not user requests. */ - return neo_iomfb_configure_backlight_h17p(neo_dcp, maximum, true, nits, false, 0); -} - -static int neo_dcp_get_brightness(struct backlight_device *bd) -{ - struct neo_apple_dcp *neo_dcp = bl_get_data(bd); - unsigned long flags; - u32 actual; - - if (READ_ONCE(neo_dcp->quiescing)) - return -ENODEV; - - if (neo_dcp_backlight_active(neo_dcp)) { - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - actual = neo_dcp->backlight.state.actual; - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - return actual; - } - - return neo_dcp->brightness.nits; -} - -int neo_dcp_backlight_configure(struct neo_apple_dcp *neo_dcp, u32 maximum, - bool inherited_valid, u32 inherited, - bool default_valid, u32 default_nits, - void (*kick)(struct neo_apple_dcp *neo_dcp)) -{ - unsigned long flags; - int ret; - - if (neo_dcp->fw_compat != DCP_FIRMWARE_H17P || !neo_dcp_has_panel(neo_dcp) || - !neo_dcp->crtc || !kick) - return -EINVAL; - - mutex_lock(&neo_dcp->bl_register_mutex); - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - if (neo_dcp->backlight.state.ready || neo_dcp->brightness.bl_dev) { - ret = -EBUSY; - } else { - ret = neo_dcp_bl_init(&neo_dcp->backlight.state, maximum, - inherited_valid, inherited, - default_valid, default_nits); - if (!ret) - neo_dcp->backlight.kick = kick; - } - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - mutex_unlock(&neo_dcp->bl_register_mutex); - if (!ret) - schedule_work(&neo_dcp->bl_register_wq); - return ret; -} - -bool neo_dcp_backlight_seed(struct neo_apple_dcp *neo_dcp, u32 nits) -{ - unsigned long flags; - bool seeded; - - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - seeded = neo_dcp_bl_seed(&neo_dcp->backlight.state, nits); - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - return seeded; -} - -static void neo_dcp_backlight_kick(struct neo_apple_dcp *neo_dcp) -{ - void (*kick)(struct neo_apple_dcp *neo_dcp) = READ_ONCE(neo_dcp->backlight.kick); - - if (kick) - kick(neo_dcp); -} - -int neo_dcp_backlight_dpms(struct neo_apple_dcp *neo_dcp, bool on) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - ret = neo_dcp_bl_dpms(&neo_dcp->backlight.state, on); - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - if (!ret) - neo_dcp_backlight_kick(neo_dcp); - return ret; -} - -int neo_dcp_backlight_prepare(struct neo_apple_dcp *neo_dcp, bool have_surface, - struct neo_dcp_backlight_present *present) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - ret = neo_dcp_bl_prepare(&neo_dcp->backlight.state, have_surface, present); - /* No reservation: carry the last completed level on an ordinary present. */ - if (ret == -EALREADY) - present->nits = neo_dcp->backlight.state.actual; - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - return ret; -} - -bool neo_dcp_backlight_complete(struct neo_apple_dcp *neo_dcp, u64 sequence, bool accepted) -{ - unsigned long flags; - bool completed; - - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - completed = neo_dcp_bl_complete(&neo_dcp->backlight.state, sequence, accepted); - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - /* Queue retries after the central queue has finished this transaction. */ - return completed; -} - -bool neo_dcp_backlight_pending(struct neo_apple_dcp *neo_dcp) -{ - unsigned long flags; - bool pending; - - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - pending = neo_dcp->backlight.state.dirty && !neo_dcp->backlight.state.in_flight; - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - return pending; -} - -/* - * Called from the receive path, which must not queue a transaction itself; - * the caller schedules the update worker when this returns true. Requests - * are honoured at most once per second, so a firmware that repeats the - * request after every brightness present cannot keep the queue busy. - */ -bool neo_dcp_backlight_resend(struct neo_apple_dcp *neo_dcp) -{ - unsigned long flags; - bool resend = false; - - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - if (!neo_dcp->backlight.resent || - time_after(jiffies, neo_dcp->backlight.resent_at + HZ)) { - resend = neo_dcp_bl_resend(&neo_dcp->backlight.state); - if (resend) { - neo_dcp->backlight.resent = true; - neo_dcp->backlight.resent_at = jiffies; - } - } - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - return resend; -} - -unsigned int neo_dcp_backlight_retry_delay(struct neo_apple_dcp *neo_dcp) -{ - unsigned long flags; - unsigned int delay; - - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - delay = neo_dcp_bl_retry_delay(&neo_dcp->backlight.state); - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - return delay; -} - -#define SCALE_FACTOR (1 << 10) - -static u32 interpolate(int val, int min, int max, u32 *tbl, size_t tbl_size) -{ - u32 frac; - u64 low, high; - u32 interpolated = (tbl_size - 1) * ((val - min) * SCALE_FACTOR) / (max - min); - - size_t index = interpolated / SCALE_FACTOR; - - if (WARN(index + 1 >= tbl_size, "invalid index %zu for brightness %u\n", index, val)) - return tbl[tbl_size / 2]; - - frac = interpolated & (SCALE_FACTOR - 1); - low = tbl[index]; - high = tbl[index + 1]; - - return ((frac * high) + ((SCALE_FACTOR - frac) * low)) / SCALE_FACTOR; -} - -static u32 calculate_dac(struct neo_apple_dcp *neo_dcp, int val) -{ - u32 dac; - - if (val <= MIN_BRIGHTNESS_PART1) - return 16 * brightness_part1[0]; - else if (val == MAX_BRIGHTNESS_PART1) - return 16 * brightness_part1[ARRAY_SIZE(brightness_part1) - 1]; - else if (val == MIN_BRIGHTNESS_PART2) - return 16 * brightness_part2[0]; - else if (val >= MAX_BRIGHTNESS_PART2) - return brightness_part2[ARRAY_SIZE(brightness_part2) - 1]; - - if (val < MAX_BRIGHTNESS_PART1) { - dac = interpolate(val, MIN_BRIGHTNESS_PART1, MAX_BRIGHTNESS_PART1, - brightness_part1, ARRAY_SIZE(brightness_part1)); - } else if (val > MIN_BRIGHTNESS_PART2) { - dac = interpolate(val, MIN_BRIGHTNESS_PART2, MAX_BRIGHTNESS_PART2, - brightness_part2, ARRAY_SIZE(brightness_part2)); - } else { - dac = interpolate(val, MAX_BRIGHTNESS_PART1, MIN_BRIGHTNESS_PART2, - brightness_part12, ARRAY_SIZE(brightness_part12)); - } - - return 16 * dac; -} - -static int drm_crtc_set_brightness(struct neo_apple_dcp *neo_dcp) -{ - struct drm_atomic_state *state; - struct drm_crtc_state *crtc_state; - struct drm_modeset_acquire_ctx ctx; - struct drm_crtc *crtc = &neo_dcp->crtc->base; - int ret = 0; - - state = drm_atomic_state_alloc(crtc->dev); - if (!state) - return -ENOMEM; - - drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE); - state->acquire_ctx = &ctx; - -retry: - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) { - ret = PTR_ERR(crtc_state); - goto out; - } - - if (!neo_dcp->brightness.update) - goto out; - - /* Re-present the current scanout when brightness is the only change. */ - ret = drm_atomic_add_affected_planes(state, crtc); - if (ret) - goto out; - - crtc_state->color_mgmt_changed |= true; - - ret = drm_atomic_commit(state); - -out: - if (ret == -EDEADLK) { - drm_atomic_state_clear(state); - ret = drm_modeset_backoff(&ctx); - if (!ret) - goto retry; - } - - drm_atomic_state_put(state); - drm_modeset_drop_locks(&ctx); - drm_modeset_acquire_fini(&ctx); - - return ret; -} - -int neo_dcp_backlight_update(struct neo_apple_dcp *neo_dcp) -{ - if (READ_ONCE(neo_dcp->quiescing)) - return -ENODEV; - if (neo_dcp_backlight_active(neo_dcp)) { - neo_dcp_backlight_kick(neo_dcp); - return 0; - } - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return -ENODATA; - - /* - * Do not actively try to change brightness if no mode is set. - * TODO: should this be reflected the in backlight's power property? - * defer this hopefully until it becomes irrelevant due to proper - * drm integrated backlight handling - */ - if (!READ_ONCE(neo_dcp->mode_state.valid)) - return 0; - - /* Wait 1 vblank cycle in the hope an atomic swap has already updated - * the brightness */ - msleep((1001 + 23) / 24); // 42ms for 23.976 fps - - return drm_crtc_set_brightness(neo_dcp); -} - -static int neo_dcp_set_brightness(struct backlight_device *bd) -{ - int ret = 0; - struct neo_apple_dcp *neo_dcp = bl_get_data(bd); - int brightness = backlight_get_brightness(bd); - unsigned long flags; - - if (READ_ONCE(neo_dcp->quiescing)) - return -ENODEV; - - if (neo_dcp_backlight_active(neo_dcp)) { - /* Preserve the requested level even while the core forces zero. */ - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - ret = neo_dcp_bl_request(&neo_dcp->backlight.state, - bd->props.brightness, - backlight_is_blank(bd), - bd->props.state & BL_CORE_SUSPENDED); - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - if (!ret) - neo_dcp_backlight_kick(neo_dcp); - return ret; - } - - ret = drm_modeset_lock_single_interruptible(&neo_dcp->crtc->base.mutex); - if (ret) - return ret; - - neo_dcp->brightness.dac = calculate_dac(neo_dcp, brightness); - neo_dcp->brightness.update = true; - - drm_modeset_unlock(&neo_dcp->crtc->base.mutex); - - return neo_dcp_backlight_update(neo_dcp); -} - -static bool neo_dcp_backlight_controls_device(struct backlight_device *bd, - struct device *display_dev) -{ - struct neo_apple_dcp *neo_dcp = bl_get_data(bd); - - return !display_dev || display_dev == neo_dcp->dev || - (neo_dcp->crtc && display_dev == neo_dcp->crtc->base.dev->dev); -} - -static const struct backlight_ops neo_dcp_backlight_ops = { - .options = BL_CORE_SUSPENDRESUME, - .get_brightness = neo_dcp_get_brightness, - .update_status = neo_dcp_set_brightness, - .controls_device = neo_dcp_backlight_controls_device, -}; - -int neo_dcp_backlight_register(struct neo_apple_dcp *neo_dcp) -{ - struct device *dev = neo_dcp->dev; - struct backlight_device *bl_dev; - unsigned long flags; - struct backlight_properties props = { - .type = BACKLIGHT_PLATFORM, - .brightness = neo_dcp->brightness.nits, - .scale = BACKLIGHT_SCALE_LINEAR, - }; - props.max_brightness = min(neo_dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !neo_dcp_backlight_active(neo_dcp)) - return -ENODATA; - if (neo_dcp_backlight_active(neo_dcp)) { - spin_lock_irqsave(&neo_dcp->backlight.lock, flags); - if (!neo_dcp->backlight.state.ready) { - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - return -ENODATA; - } - /* Freeze takeover before userspace can discover the device. */ - neo_dcp->backlight.state.controlled = true; - props.brightness = neo_dcp->backlight.state.target; - props.max_brightness = neo_dcp->backlight.state.maximum; - spin_unlock_irqrestore(&neo_dcp->backlight.lock, flags); - } - - bl_dev = devm_backlight_device_register(dev, "apple-panel-bl", dev, neo_dcp, - &neo_dcp_backlight_ops, &props); - if (IS_ERR(bl_dev)) - return PTR_ERR(bl_dev); - - neo_dcp->brightness.bl_dev = bl_dev; - if (!neo_dcp_backlight_active(neo_dcp)) - neo_dcp->brightness.dac = calculate_dac(neo_dcp, neo_dcp->brightness.nits); - - return 0; -} diff --git a/drivers/gpu/drm/apple-neo/dcp_backlight.h b/drivers/gpu/drm/apple-neo/dcp_backlight.h deleted file mode 100644 index 4b73a54fd29866..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp_backlight.h +++ /dev/null @@ -1,233 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Copyright 2026 Ryan Murray */ - -#ifndef __APPLE_DCP_BACKLIGHT_H__ -#define __APPLE_DCP_BACKLIGHT_H__ - -#include -#include -#include -#include - -/* Commanded nits, independent of firmware encoding. No fade policy. */ -struct neo_dcp_backlight_state { - u32 maximum; - u32 target; - u32 actual; - u32 sent_nits; - u64 sequence; - u8 retries; - bool ready; - bool controlled; - bool dpms_off; - bool core_blank; - bool suspended; - bool dirty; - bool in_flight; - /* - * The panel shows a level Linux knows: the loader's reported level, or - * one a completed present carried. A default used for registration - * without a loader report is not known until a present carries a level. - */ - bool level_known; -}; - -struct neo_dcp_backlight_present { - u64 sequence; - u32 nits; -}; - -/* - * Public powerlog reports use millinits. A report slightly above the panel - * ceiling describes the brightest level the panel can show, so clamp it; one - * more than twice the ceiling is not a credible panel level. - */ -static inline int neo_dcp_bl_takeover_nits(u32 maximum, u32 millinits, u32 *nits) -{ - if (!maximum || maximum > INT_MAX / 2000) - return -EINVAL; - if (millinits / 1000 > 2 * maximum) - return -ERANGE; - *nits = min(millinits / 1000, maximum); - return 0; -} - -/* - * An inherited level comes from a loader report. A default level is only - * reported to userspace; it is never presented on its own (see level_known). - */ -static inline int neo_dcp_bl_init(struct neo_dcp_backlight_state *state, u32 maximum, - bool inherited_valid, u32 inherited, - bool default_valid, u32 default_nits) -{ - u32 nits; - - if (!maximum || maximum > INT_MAX) - return -EINVAL; - if (!inherited_valid && !default_valid) - return -ENODATA; - nits = inherited_valid ? inherited : default_nits; - if (nits > maximum) - return -ERANGE; - - *state = (struct neo_dcp_backlight_state) { - .maximum = maximum, - .target = nits, - .actual = nits, - .ready = true, - .level_known = inherited_valid, - }; - return 0; -} - -/* - * A decoded powerlog sample can seed takeover only before Linux control. - * It reports the level the panel shows, so the level is then known. - */ -static inline bool neo_dcp_bl_seed(struct neo_dcp_backlight_state *state, u32 nits) -{ - if (!state->ready || state->controlled || nits > state->maximum) - return false; - state->target = nits; - state->actual = nits; - state->level_known = true; - return true; -} - -static inline u32 neo_dcp_bl_effective(const struct neo_dcp_backlight_state *state) -{ - if (state->dpms_off || state->core_blank || state->suspended) - return 0; - return state->target; -} - -static inline int neo_dcp_bl_request(struct neo_dcp_backlight_state *state, u32 nits, - bool core_blank, bool suspended) -{ - if (!state->ready) - return -ENODATA; - if (nits > state->maximum) - return -ERANGE; - state->controlled = true; - state->retries = 0; - /* - * Until a present carries a level, the panel does not show the target, - * so even a request for the reported default has to be presented. - */ - if (state->target != nits || state->core_blank != core_blank || - state->suspended != suspended || !state->level_known) { - state->target = nits; - state->core_blank = core_blank; - state->suspended = suspended; - state->dirty = true; - } - return 0; -} - -static inline int neo_dcp_bl_dpms(struct neo_dcp_backlight_state *state, bool on) -{ - if (!state->ready) - return -ENODATA; - state->controlled = true; - state->retries = 0; - if (state->dpms_off != !on) { - state->dpms_off = !on; - state->dirty = true; - } - return 0; -} - -/* - * The firmware asked for the level again. Re-send the effective level once - * Linux controls it and knows it, unless a present already carries or will - * carry it. A default level is never sent this way. - */ -static inline bool neo_dcp_bl_resend(struct neo_dcp_backlight_state *state) -{ - if (!state->ready || !state->controlled || !state->level_known || - state->dirty || state->in_flight) - return false; - state->dirty = true; - return true; -} - -/* - * Called by the single outbound queue, with a pinned, accepted scanout or a - * prepared replacement. Soft-off never releases it. A brightness-only present - * must use the last accepted surface, not a rejected replacement's DRM state. - */ -static inline int neo_dcp_bl_prepare(struct neo_dcp_backlight_state *state, - bool have_surface, - struct neo_dcp_backlight_present *present) -{ - if (!state->ready || !have_surface) - return -ENODATA; - if (state->in_flight) - return -EBUSY; - if (!state->dirty) - return -EALREADY; - - state->controlled = true; - state->in_flight = true; - state->dirty = false; - state->sent_nits = neo_dcp_bl_effective(state); - present->nits = state->sent_nits; - present->sequence = ++state->sequence; - return 0; -} - -/* - * accepted means both accepted submit and its matching completed present. - * Rejection/serialization failure cancels the reservation without changing - * actual. This result is NEVER a framebuffer retirement authorization. - */ -static inline bool neo_dcp_bl_complete(struct neo_dcp_backlight_state *state, - u64 sequence, bool accepted) -{ - if (!state->in_flight || sequence != state->sequence) - return false; - state->in_flight = false; - if (accepted) { - state->actual = state->sent_nits; - state->retries = 0; - state->level_known = true; - } else { - state->dirty = true; - if (state->retries < 4) - state->retries++; - } - return true; -} - -/* Three deferred retries per user request; a permanent rejection stays dirty. */ -static inline unsigned int neo_dcp_bl_retry_delay(const struct neo_dcp_backlight_state *state) -{ - if (!state->ready || !state->dirty || state->in_flight || - !state->retries || state->retries > 3) - return 0; - return 100U << (state->retries - 1); -} - -struct neo_apple_dcp; - -/* - * kick only queues work; it must not submit a top-level RPC from the caller. - * The central IOMFB queue owns surface references, retries, and wire encoding. - * Configure before registration and keep the callback valid until teardown. - */ -bool neo_dcp_backlight_active(struct neo_apple_dcp *neo_dcp); -int neo_dcp_backlight_takeover(struct neo_apple_dcp *neo_dcp, u32 millinits); -int neo_dcp_backlight_configure(struct neo_apple_dcp *neo_dcp, u32 maximum, - bool inherited_valid, u32 inherited, - bool default_valid, u32 default_nits, - void (*kick)(struct neo_apple_dcp *neo_dcp)); -bool neo_dcp_backlight_seed(struct neo_apple_dcp *neo_dcp, u32 nits); -int neo_dcp_backlight_dpms(struct neo_apple_dcp *neo_dcp, bool on); -int neo_dcp_backlight_prepare(struct neo_apple_dcp *neo_dcp, bool have_surface, - struct neo_dcp_backlight_present *present); -bool neo_dcp_backlight_complete(struct neo_apple_dcp *neo_dcp, u64 sequence, bool accepted); -bool neo_dcp_backlight_pending(struct neo_apple_dcp *neo_dcp); -bool neo_dcp_backlight_resend(struct neo_apple_dcp *neo_dcp); -unsigned int neo_dcp_backlight_retry_delay(struct neo_apple_dcp *neo_dcp); - -#endif /* __APPLE_DCP_BACKLIGHT_H__ */ diff --git a/drivers/gpu/drm/apple-neo/dcp_backlight_test.c b/drivers/gpu/drm/apple-neo/dcp_backlight_test.c deleted file mode 100644 index 0c2123db25287c..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp_backlight_test.c +++ /dev/null @@ -1,379 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2026 Ryan Murray */ - -#include -#include - -#include "dcp_backlight.h" - -static void backlight_millinits_takeover_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - u32 nits = 123; - - KUNIT_EXPECT_EQ(test, neo_dcp_bl_takeover_nits(0, 100000, &nits), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_takeover_nits(U32_MAX, 100000, &nits), -EINVAL); - KUNIT_EXPECT_EQ(test, nits, 123U); - /* Reports above the panel ceiling clamp to it. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_takeover_nits(525, 525999, &nits), 0); - KUNIT_EXPECT_EQ(test, nits, 525U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_takeover_nits(509, 509001, &nits), 0); - KUNIT_EXPECT_EQ(test, nits, 509U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_takeover_nits(509, 1018999, &nits), 0); - KUNIT_EXPECT_EQ(test, nits, 509U); - /* More than twice the ceiling is not a panel level. */ - nits = 123; - KUNIT_EXPECT_EQ(test, neo_dcp_bl_takeover_nits(509, 1019000, &nits), -ERANGE); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_takeover_nits(509, U32_MAX, &nits), -ERANGE); - KUNIT_EXPECT_EQ(test, nits, 123U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_takeover_nits(509, 100999, &nits), 0); - KUNIT_EXPECT_EQ(test, nits, 100U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 509, true, nits, false, 0), 0); - KUNIT_EXPECT_EQ(test, state.actual, 100U); - KUNIT_EXPECT_FALSE(test, state.dirty); - /* Registration freezes the hint before exposing the controls. */ - state.controlled = true; - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_seed(&state, 400)); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_seed(&state, 0)); - KUNIT_EXPECT_EQ(test, state.target, 100U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_takeover_nits(509, 0, &nits), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 509, true, nits, false, 0), 0); - KUNIT_EXPECT_EQ(test, state.target, 0U); - KUNIT_EXPECT_FALSE(test, state.dirty); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_takeover_nits(509, 509000, &nits), 0); - KUNIT_EXPECT_EQ(test, nits, 509U); -} - -static void backlight_takeover_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state = {}; - - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 140, true, 36), 0); - KUNIT_EXPECT_EQ(test, state.target, 140U); - KUNIT_EXPECT_EQ(test, state.actual, 140U); - KUNIT_EXPECT_FALSE(test, state.dirty); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_seed(&state, 400)); - KUNIT_EXPECT_EQ(test, state.target, 400U); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_seed(&state, 601)); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 400, false, false), 0); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_seed(&state, 0)); - KUNIT_EXPECT_EQ(test, state.target, 400U); - KUNIT_EXPECT_EQ(test, state.actual, 400U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 0, true, 36), 0); - KUNIT_EXPECT_EQ(test, state.target, 0U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, false, 0, true, 36), 0); - KUNIT_EXPECT_EQ(test, state.target, 36U); -} - -static void backlight_range_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state = {}; - - KUNIT_EXPECT_EQ(test, neo_dcp_bl_init(&state, 600, false, 0, false, 0), -ENODATA); - KUNIT_EXPECT_FALSE(test, state.ready); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_request(&state, 140, false, false), -ENODATA); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_dpms(&state, false), -ENODATA); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_init(&state, 0, true, 0, false, 0), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_init(&state, U32_MAX, true, 0, false, 0), -EINVAL); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_init(&state, 600, true, 601, false, 0), -ERANGE); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 140, false, 0), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_request(&state, 601, false, false), -ERANGE); - KUNIT_EXPECT_EQ(test, state.target, 140U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 600, false, false), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 600U); -} - -static void backlight_dpms_cycles_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - int i; - - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 400, false, 0), 0); - for (i = 0; i < 20; i++) { - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 0U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_EQ(test, state.actual, 0U); - KUNIT_EXPECT_EQ(test, state.target, 400U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, true), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 400U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_EQ(test, state.actual, 400U); - } - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 0U); - KUNIT_EXPECT_EQ(test, state.target, 400U); -} - -static void backlight_blanked_target_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 400, false, 0), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 140, false, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 0U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_EQ(test, state.target, 140U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, true), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 140U); - KUNIT_EXPECT_EQ(test, state.actual, 0U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_EQ(test, state.actual, 140U); -} - -static void backlight_suspend_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - int off_first; - - for (off_first = 0; off_first < 2; off_first++) { - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 140, false, 0), 0); - if (off_first) - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 140, true, true), 0); - if (!off_first) - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 36, true, true), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 0U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, true), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 0U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 36, false, false), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 0U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, true), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 36U); - KUNIT_EXPECT_EQ(test, state.target, 36U); - } - /* A core blank also keeps the panel dark independently of DRM DPMS. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 400, true, false), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 0U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 400, false, false), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_effective(&state), 400U); -} - -static void backlight_surface_and_busy_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present first, second; - - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 140, false, 0), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_prepare(&state, true, &first), -EALREADY); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 36, false, false), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_prepare(&state, false, &first), -ENODATA); - KUNIT_EXPECT_TRUE(test, state.dirty); - KUNIT_EXPECT_FALSE(test, state.in_flight); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &first), 0); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_prepare(&state, true, &second), -EBUSY); - KUNIT_EXPECT_EQ(test, state.actual, 140U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 400, false, false), 0); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, first.sequence, true)); - KUNIT_EXPECT_EQ(test, state.actual, 36U); - KUNIT_EXPECT_EQ(test, state.target, 400U); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &second), 0); - KUNIT_EXPECT_EQ(test, second.nits, 400U); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_complete(&state, first.sequence, true)); - KUNIT_EXPECT_TRUE(test, state.in_flight); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, second.sequence, true)); - KUNIT_EXPECT_EQ(test, state.actual, 400U); -} - -static void backlight_reject_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 140, false, 0), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 400, false, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, false)); - KUNIT_EXPECT_EQ(test, state.actual, 140U); - KUNIT_EXPECT_TRUE(test, state.dirty); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 0U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_EQ(test, state.target, 400U); - KUNIT_EXPECT_EQ(test, state.actual, 0U); -} - -static void backlight_inflight_dpms_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 600, true, 140, false, 0), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, true), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_TRUE(test, state.dirty); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 0U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, true), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 140U); -} - -static void backlight_resend_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - - /* Nothing is re-sent before Linux controls the level. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, true, 140, false, 0), 0); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_resend(&state)); - KUNIT_EXPECT_FALSE(test, state.dirty); - - state.controlled = true; - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_resend(&state)); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_resend(&state)); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 140U); - /* The present in flight already carries the level. */ - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_resend(&state)); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_FALSE(test, state.dirty); - - /* A blanked panel is re-sent dark, keeping the restore target. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_resend(&state)); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 0U); - KUNIT_EXPECT_EQ(test, state.target, 140U); -} - -static void backlight_fallback_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - - /* A registration default is reported, never presented on its own. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, false, 0, true, 262), 0); - KUNIT_EXPECT_FALSE(test, state.level_known); - state.controlled = true; - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_resend(&state)); - KUNIT_EXPECT_FALSE(test, state.dirty); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), -EALREADY); - - /* A rejected userspace level does not become known. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 300, false, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 300U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, false)); - KUNIT_EXPECT_FALSE(test, state.level_known); - - /* Once a present carrying it completes, the level is re-sendable. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_TRUE(test, state.level_known); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_resend(&state)); - - /* DPMS has to light the panel at some level: wake uses the default. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, false, 0, true, 262), 0); - state.controlled = true; - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 0U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_dpms(&state, true), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 262U); -} - -static void backlight_fallback_same_level_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - - /* A userspace write of the reported default is still presented. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, false, 0, true, 262), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 262, false, false), 0); - KUNIT_EXPECT_TRUE(test, state.dirty); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 262U); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, true)); - KUNIT_EXPECT_TRUE(test, state.level_known); - - /* Once the level is known, repeating it presents nothing. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 262, false, false), 0); - KUNIT_EXPECT_FALSE(test, state.dirty); - KUNIT_EXPECT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), -EALREADY); - - /* A rejected present leaves it unknown, so the same write is retried. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, false, 0, true, 262), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 262, false, false), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_complete(&state, present.sequence, false)); - KUNIT_EXPECT_FALSE(test, state.level_known); - state.dirty = false; - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 262, false, false), 0); - KUNIT_EXPECT_TRUE(test, state.dirty); - - /* An inherited level is known from the start. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, true, 180, false, 0), 0); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_request(&state, 180, false, false), 0); - KUNIT_EXPECT_FALSE(test, state.dirty); -} - -static void backlight_seed_known_test(struct kunit *test) -{ - struct neo_dcp_backlight_state state; - struct neo_dcp_backlight_present present; - - /* A loader sample seeded over a registration default is known. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, false, 0, true, 262), 0); - KUNIT_EXPECT_FALSE(test, state.level_known); - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_seed(&state, 180)); - KUNIT_EXPECT_TRUE(test, state.level_known); - state.controlled = true; - KUNIT_EXPECT_TRUE(test, neo_dcp_bl_resend(&state)); - KUNIT_ASSERT_EQ(test, neo_dcp_bl_prepare(&state, true, &present), 0); - KUNIT_EXPECT_EQ(test, present.nits, 180U); - - /* A refused sample leaves the default unknown. */ - KUNIT_ASSERT_EQ(test, neo_dcp_bl_init(&state, 525, false, 0, true, 262), 0); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_seed(&state, 526)); - KUNIT_EXPECT_FALSE(test, state.level_known); - state.controlled = true; - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_seed(&state, 180)); - KUNIT_EXPECT_FALSE(test, state.level_known); - KUNIT_EXPECT_FALSE(test, neo_dcp_bl_resend(&state)); -} - -static struct kunit_case backlight_cases[] = { - KUNIT_CASE(backlight_millinits_takeover_test), - KUNIT_CASE(backlight_takeover_test), - KUNIT_CASE(backlight_range_test), - KUNIT_CASE(backlight_dpms_cycles_test), - KUNIT_CASE(backlight_blanked_target_test), - KUNIT_CASE(backlight_suspend_test), - KUNIT_CASE(backlight_surface_and_busy_test), - KUNIT_CASE(backlight_reject_test), - KUNIT_CASE(backlight_inflight_dpms_test), - KUNIT_CASE(backlight_resend_test), - KUNIT_CASE(backlight_fallback_test), - KUNIT_CASE(backlight_fallback_same_level_test), - KUNIT_CASE(backlight_seed_known_test), - {} -}; - -static struct kunit_suite backlight_suite = { - .name = "apple-dcp-backlight", - .test_cases = backlight_cases, -}; - -kunit_test_suite(backlight_suite); - -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple-neo/dcp_trace.c b/drivers/gpu/drm/apple-neo/dcp_trace.c deleted file mode 100644 index d18e71af73a74d..00000000000000 --- a/drivers/gpu/drm/apple-neo/dcp_trace.c +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#define CREATE_TRACE_POINTS -#include "dcp_trace.h" \ No newline at end of file diff --git a/drivers/gpu/drm/apple-neo/dptxep.c b/drivers/gpu/drm/apple-neo/dptxep.c deleted file mode 100644 index 486195d97fe082..00000000000000 --- a/drivers/gpu/drm/apple-neo/dptxep.c +++ /dev/null @@ -1,878 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2022 Sven Peter */ - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "afk.h" -#include "dcp.h" -#include "dptxep.h" -#include "parser.h" -#include "trace.h" - -struct dcpdptx_connection_cmd { - __le32 unk; - __le32 target; -} __attribute__((packed)); - -struct dcpdptx_hotplug_cmd { - u8 _pad0[16]; - __le32 unk; -} __attribute__((packed)); - -struct neo_dptxport_apcall_link_rate { - __le32 retcode; - u8 _unk0[12]; - __le32 link_rate; - u8 _unk1[12]; -} __attribute__((packed)); - -struct neo_dptxport_apcall_lane_count { - __le32 retcode; - u8 _unk0[12]; - __le64 lane_count; - u8 _unk1[8]; -} __attribute__((packed)); - -struct neo_dptxport_apcall_set_active_lane_count { - __le32 retcode; - u8 _unk0[12]; - __le64 lane_count; - u8 _unk1[8]; -} __packed; - -struct neo_dptxport_apcall_get_support { - __le32 retcode; - u8 _unk0[12]; - __le32 supported; - u8 _unk1[12]; -} __attribute__((packed)); - -struct neo_dptxport_apcall_max_drive_settings { - __le32 retcode; - u8 _unk0[12]; - __le32 max_drive_settings[2]; - u8 _unk1[8]; -}; - -struct neo_dptxport_apcall_drive_settings { - __le32 retcode; - u8 _unk0[12]; - __le32 unk1; - __le32 unk2; - __le32 unk3; - __le32 unk4; - __le32 unk5; - __le32 unk6; - __le32 unk7; -}; - -struct neo_dptxport_apcall_set_tiled { - __le32 retcode; -}; - -/* - * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel uses the same - * plain CORE|ATC|DIE|CONNECTED target as a direct alt-mode PHY. CORE is the - * DFP port (0 = dpphy, 1/2 = dpin0/dpin1, dcp->dptx_dfp_port), ATC is the - * route's own ATC index -- no separate DPIN field. - */ -static u32 neo_dptxport_remote_target(struct neo_apple_dcp *neo_dcp, u8 core, u8 atc, - u8 die) -{ - return FIELD_PREP(DCPDPTX_REMOTE_PORT_CORE, core) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_ATC, atc) | - FIELD_PREP(DCPDPTX_REMOTE_PORT_DIE, die) | - DCPDPTX_REMOTE_PORT_CONNECTED; -} - -int neo_dptxport_validate_connection(struct neo_apple_epic_service *service, u8 core, - u8 atc, u8 die) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - struct dcpdptx_connection_cmd cmd, resp; - int ret; - u32 target = neo_dptxport_remote_target(service->ep->neo_dcp, core, atc, die); - - /* - * attributes: role (0 = direct PHY, 1 = Thunderbolt/USB4 DP IN) | - * supportsHPD << 8. A real, hardware-validated reference - * implementation of the equivalent Thunderbolt DP tunnel mechanism - * on a different SoC (aurora-silicon/linux#8) adds this exact role - * bit for its tunnel routes; our own analog-DPIN path has never set - * it, always sending a plain direct-PHY attributes value even - * though this is a genuinely USB4-tunneled connection. DCP firmware - * appears to treat a failed AUX probe as fatal for role=0 but keeps - * training regardless for role=1 (a Thunderbolt tunnel's AUX - * proxying may not respond as fast as a direct PHY's): repeated - * test runs with substantially different Activate/crossbar ordering - * all reached the identical INACTIVE_SINK_DETECTED/DPRX-timeout - * outcome, which points at a signal we are not sending at all - * rather than a timing issue fixable by reordering. - */ - u32 attrs = 0x100 | (neo_dcp_is_usb4_output(service->ep->neo_dcp) ? 1 : 0); - - trace_neo_dptxport_validate_connection(neo_dptx, core, atc, die); - neo_dptx->validate_calls++; - dev_dbg(service->ep->neo_dcp->dev, - "DPTX validate: call #%u this boot target=0x%x core=%u atc=%u die=%u attrs=0x%x caller=%pS\n", - neo_dptx->validate_calls, target, core, atc, die, - attrs, __builtin_return_address(0)); - - cmd.target = cpu_to_le32(target); - cmd.unk = cpu_to_le32(attrs); - ret = neo_afk_service_call(service, 0, 12, &cmd, sizeof(cmd), 40, &resp, - sizeof(resp), 40); - if (ret) - return ret; - - if (le32_to_cpu(resp.target) != target) { - dev_warn(service->ep->neo_dcp->dev, - "validate_connection: target reply 0x%x (sent 0x%x)\n", - le32_to_cpu(resp.target), target); - return -EINVAL; - } - if (le32_to_cpu(resp.unk) != attrs) { - /* only the Thunderbolt DP IN role is allowed to differ */ - if (!neo_dcp_is_usb4_output(service->ep->neo_dcp)) { - dev_warn(service->ep->neo_dcp->dev, - "validate_connection: attrs reply 0x%x (sent 0x%x), rejecting\n", - le32_to_cpu(resp.unk), attrs); - return -EINVAL; - } - dev_info(service->ep->neo_dcp->dev, - "validate_connection: attrs reply 0x%x (sent 0x%x)\n", - le32_to_cpu(resp.unk), attrs); - } - - return 0; -} - -int neo_dptxport_connect(struct neo_apple_epic_service *service, u8 core, u8 atc, - u8 die, bool supports_hpd) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - struct dcpdptx_connection_cmd cmd, resp; - /* same role bit as dptxport_validate_connection() above */ - u32 unk_field = (supports_hpd ? DCPDPTX_REMOTE_PORT_SUPPORTS_HPD : 0) | - (neo_dcp_is_usb4_output(service->ep->neo_dcp) ? 1 : 0); - int ret; - u32 target = neo_dptxport_remote_target(service->ep->neo_dcp, core, atc, die); - - trace_neo_dptxport_connect(neo_dptx, core, atc, die); - neo_dptx->connect_calls++; - dev_dbg(service->ep->neo_dcp->dev, - "DPTX connect: call #%u this boot target=0x%x unk=0x%x caller=%pS\n", - neo_dptx->connect_calls, target, unk_field, - __builtin_return_address(0)); - - cmd.target = cpu_to_le32(target); - cmd.unk = cpu_to_le32(unk_field); - ret = neo_afk_service_call(service, 0, 11, &cmd, sizeof(cmd), 24, &resp, - sizeof(resp), 24); - if (ret) - return ret; - - if (le32_to_cpu(resp.target) != target) { - dev_warn(service->ep->neo_dcp->dev, - "connect: target reply 0x%x (sent 0x%x)\n", - le32_to_cpu(resp.target), target); - return -EINVAL; - } - if (le32_to_cpu(resp.unk) != unk_field) - dev_notice(service->ep->neo_dcp->dev, "unexpected unk field in reply: 0x%x (0x%x)\n", - le32_to_cpu(resp.unk), unk_field); - - return 0; -} - -int neo_dptxport_request_display(struct neo_apple_epic_service *service) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - int ret; - - neo_dptx->request_calls++; - dev_dbg(service->ep->neo_dcp->dev, - "DPTX request_display: call #%u this boot caller=%pS\n", - neo_dptx->request_calls, __builtin_return_address(0)); - ret = neo_afk_service_call(service, 0, 6, NULL, 0, 16, NULL, 0, 16); - dev_dbg(service->ep->neo_dcp->dev, - "DPTX request_display: call #%u result=%d\n", - neo_dptx->request_calls, ret); - return ret; -} - -int neo_dptxport_release_display(struct neo_apple_epic_service *service) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - int ret; - - neo_dptx->release_calls++; - dev_dbg(service->ep->neo_dcp->dev, - "DPTX release_display: call #%u this boot caller=%pS\n", - neo_dptx->release_calls, __builtin_return_address(0)); - ret = neo_afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); - dev_dbg(service->ep->neo_dcp->dev, - "DPTX release_display: call #%u result=%d\n", - neo_dptx->release_calls, ret); - return ret; -} - -int neo_dptxport_set_hpd_timeout(struct neo_apple_epic_service *service, bool hpd, - unsigned int timeout_ms) -{ - struct dcpdptx_hotplug_cmd cmd, resp; - int ret; - - memset(&cmd, 0, sizeof(cmd)); - - if (hpd) - cmd.unk = cpu_to_le32(1); - - ret = neo_afk_service_call_timeout(service, 8, 8, &cmd, sizeof(cmd), 12, - &resp, sizeof(resp), 12, timeout_ms); - if (ret) - return ret; - if (le32_to_cpu(resp.unk) != hpd) { - dev_warn(service->ep->neo_dcp->dev, - "set_hpd: unk reply 0x%x (sent hpd=%d)\n", - le32_to_cpu(resp.unk), hpd); - return -EINVAL; - } - return 0; -} - -int neo_dptxport_set_hpd(struct neo_apple_epic_service *service, bool hpd) -{ - return neo_dptxport_set_hpd_timeout(service, hpd, MSEC_PER_SEC); -} - -static int -neo_dptxport_call_get_max_drive_settings(struct neo_apple_epic_service *service, - void *reply_, size_t reply_size) -{ - struct neo_dptxport_apcall_max_drive_settings *reply = reply_; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - - reply->retcode = cpu_to_le32(0); - reply->max_drive_settings[0] = cpu_to_le32(0x3); - reply->max_drive_settings[1] = cpu_to_le32(0x3); - - return 0; -} - -static int -neo_dptxport_call_get_drive_settings(struct neo_apple_epic_service *service, - const void *request_, size_t request_size, - void *reply_, size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - const struct neo_dptxport_apcall_drive_settings *request = request_; - struct neo_dptxport_apcall_drive_settings *reply = reply_; - - if (reply_size < sizeof(*reply) || request_size < sizeof(*request)) - return -EINVAL; - - *reply = *request; - - /* Clear the rest of the buffer */ - memset(reply_ + sizeof(*reply), 0, reply_size - sizeof(*reply)); - - /* - * retcode appears to be lane count, seeing 2 for USB-C dp alt mode - * with lanes splitted for DP/USB3. - */ - if (le32_to_cpu(reply->retcode) != neo_dptx->lane_count) - dev_err(service->ep->neo_dcp->dev, - "get_drive_settings: unexpected retcode %d\n", - reply->retcode); - - reply->retcode = cpu_to_le32(neo_dptx->lane_count); - reply->unk5 = cpu_to_le32(neo_dptx->drive_settings[0]); - reply->unk6 = cpu_to_le32(0); - reply->unk7 = cpu_to_le32(neo_dptx->drive_settings[1]); - - return 0; -} - -static int -neo_dptxport_call_set_drive_settings(struct neo_apple_epic_service *service, - const void *request_, size_t request_size, - void *reply_, size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - const struct neo_dptxport_apcall_drive_settings *request = request_; - struct neo_dptxport_apcall_drive_settings *reply = reply_; - - if (reply_size < sizeof(*reply) || request_size < sizeof(*request)) - return -EINVAL; - - *reply = *request; - reply->retcode = cpu_to_le32(0); - - dev_info(service->ep->neo_dcp->dev, "set_drive_settings: %d:%d:%d:%d:%d:%d:%d\n", - request->unk1, request->unk2, request->unk3, request->unk4, - request->unk5, request->unk6, request->unk7); - - neo_dptx->drive_settings[0] = le32_to_cpu(reply->unk5); - neo_dptx->drive_settings[1] = le32_to_cpu(reply->unk7); - - return 0; -} - -static int neo_dptxport_call_get_max_link_rate(struct neo_apple_epic_service *service, - void *reply_, size_t reply_size) -{ - struct neo_dptxport_apcall_link_rate *reply = reply_; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - - reply->retcode = cpu_to_le32(0); - reply->link_rate = cpu_to_le32(LINK_RATE_HBR3); - - return 0; -} - -static int neo_dptxport_call_get_max_lane_count(struct neo_apple_epic_service *service, - void *reply_, size_t reply_size) -{ - struct neo_dptxport_apcall_lane_count *reply = reply_; - struct neo_dptx_port *neo_dptx = service->cookie; - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - union phy_configure_opts phy_ops; - int ret; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - - if (!neo_dptx->atcphy) { - /* USB4 DP IN: no ATC DP PHY to validate. */ - neo_dptx->lane_count = 4; - reply->retcode = cpu_to_le32(0); - reply->lane_count = cpu_to_le64(4); - dev_info(neo_dcp->dev, "get_max_lane_count: USB4 DP IN, 4 lanes\n"); - return 0; - } - - ret = phy_validate(neo_dptx->atcphy, PHY_MODE_DP, 0, &phy_ops); - if (ret < 0) { - dev_err(neo_dcp->dev, "phy_validate failed: %d\n", ret); - reply->retcode = cpu_to_le32(1); - reply->lane_count = cpu_to_le64(0); - } else { - if (phy_ops.dp.lanes < 2) { - // phy_validate might return 0 lanes if atc phy is not - // yet switched to DP mode - dev_dbg(neo_dcp->dev, "get_max_lane_count: phy lanes: %d\n", - phy_ops.dp.lanes); - // default to 4 lanes - neo_dptx->lane_count = 4; - } else { - neo_dptx->lane_count = phy_ops.dp.lanes; - } - reply->retcode = cpu_to_le32(0); - reply->lane_count = cpu_to_le64(neo_dptx->lane_count); - } - - return 0; -} - -static int neo_dptxport_call_set_active_lane_count(struct neo_apple_epic_service *service, - const void *data, size_t data_size, - void *reply_, size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - const struct neo_dptxport_apcall_set_active_lane_count *request = data; - struct neo_dptxport_apcall_set_active_lane_count *reply = reply_; - int ret = 0; - int retcode = 0; - - if (reply_size < sizeof(*reply)) - return -1; - if (data_size < sizeof(*request)) - return -1; - - u64 lane_count = le64_to_cpu(request->lane_count); - - if (neo_dptx->lane_count < lane_count) - dev_err(neo_dcp->dev, "set_active_lane_count: unexpected lane " - "count:%llu phy: %d\n", lane_count, neo_dptx->lane_count); - - switch (lane_count) { - case 0 ... 2: - case 4: - neo_dptx->phy_ops.dp.lanes = lane_count; - /* USB4 DP IN has no ATC PHY; still accept the lane count. */ - neo_dptx->phy_ops.dp.set_lanes = - neo_dcp_is_usb4_output(neo_dcp) || (neo_dcp->neo_dptx_phy > 3); - break; - default: - dev_err(neo_dcp->dev, "set_active_lane_count: invalid lane count:%llu\n", lane_count); - retcode = 1; - lane_count = 0; - break; - } - - if (neo_dptx->phy_ops.dp.set_lanes) { - if (neo_dptx->atcphy && !neo_dcp_is_usb4_output(neo_dcp)) { - ret = phy_configure(neo_dptx->atcphy, &neo_dptx->phy_ops); - if (ret) - return ret; - } - neo_dptx->phy_ops.dp.set_lanes = 0; - neo_dptx->lane_count = lane_count; - } - - reply->retcode = cpu_to_le32(retcode); - reply->lane_count = cpu_to_le64(lane_count); - - if (lane_count > 0) { - dev_info(neo_dcp->dev, "USB4/DPTX: SET_ACTIVE_LANE_COUNT %llu\n", - lane_count); - /* - * dcp_usb4_drm_allowed() (usb4_force_dptx) used to gate this - * for the old manual-training sysfs knob (module_param_cb - * usb4_dptx_train), removed wholesale in 0dc9f50 when - * apple_dcp_tb_dp_tunnel() replaced it with automatic tunnel - * detection -- but usb4_force_dptx itself, and this gate, - * were left behind with no way left to ever set it true. - * dcp_dptx_connect()'s single wait_for_completion_timeout() - * on linkcfg_completion is now shared by both the alt-mode - * and USB4-tunnel paths (also part of 0dc9f50's unification), - * so gating it here meant every tunneled connect timed out - * after a fully successful DPRX/AUX handshake and lane - * negotiation -- firmware does not send FORCE_HOTPLUG_DETECT - * (the only other completion site) for a tunnel. The - * reference (aurora-silicon/linux#8) completes - * linkcfg_completion here unconditionally; usb4_lane_completion - * has no consumer anywhere in this tree, so this drops both the - * dead gate and the dead completion and matches the reference. - */ - complete(&neo_dptx->linkcfg_completion); - } - - return ret; -} - -static int neo_dptxport_call_get_link_rate(struct neo_apple_epic_service *service, - void *reply_, size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - struct neo_dptxport_apcall_link_rate *reply = reply_; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - - reply->retcode = cpu_to_le32(0); - reply->link_rate = cpu_to_le32(neo_dptx->link_rate); - - return 0; -} - -static int -neo_dptxport_call_will_change_link_config(struct neo_apple_epic_service *service) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - - neo_dptx->phy_ops.dp.set_lanes = 0; - neo_dptx->phy_ops.dp.set_rate = 0; - neo_dptx->phy_ops.dp.set_voltages = 0; - - return 0; -} - -static int -neo_dptxport_call_did_change_link_config(struct neo_apple_epic_service *service) -{ - /* - * Ported from aurora-silicon/linux#8: bringing the tunnel crossbar - * connection up (dcp_tunnel_crossbar_up()) now happens in - * dptxport_call(), after this succeeds, gated on - * dptx_tunnel && link_rate -- not inside this handler, and its - * result is never propagated back to DCP as a call failure. - */ - - /* assume the link config did change and wait a little bit */ - mdelay(10); - - return 0; -} - -static int neo_dptxport_call_set_link_rate(struct neo_apple_epic_service *service, - const void *data, size_t data_size, - void *reply_, size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - const struct neo_dptxport_apcall_link_rate *request = data; - struct neo_dptxport_apcall_link_rate *reply = reply_; - u32 link_rate, phy_link_rate; - bool phy_set_rate = false; - int ret; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - if (data_size < sizeof(*request)) - return -EINVAL; - - link_rate = le32_to_cpu(request->link_rate); - trace_neo_dptxport_call_set_link_rate(neo_dptx, link_rate); - dev_info(service->ep->neo_dcp->dev, "DPTXPort: SET_LINK_RATE 0x%x\n", - link_rate); - - switch (link_rate) { - case LINK_RATE_RBR: - phy_link_rate = 1620; - phy_set_rate = true; - break; - case LINK_RATE_HBR: - phy_link_rate = 2700; - phy_set_rate = true; - break; - case LINK_RATE_HBR2: - phy_link_rate = 5400; - phy_set_rate = true; - break; - case LINK_RATE_HBR3: - phy_link_rate = 8100; - phy_set_rate = true; - break; - case 0: - phy_link_rate = 0; - phy_set_rate = true; - break; - default: - dev_err(service->ep->neo_dcp->dev, - "DPTXPort: Unsupported link rate 0x%x requested\n", - link_rate); - link_rate = 0; - phy_set_rate = false; - break; - } - - if (phy_set_rate) { - /* - * Ported from aurora-silicon/linux#8: a Thunderbolt DP tunnel - * starts/stops its own pixel clock instead of configuring the - * PHY directly, and never fails the apcall over it -- a - * failed clock just keeps the crossbar connection down - * (dcp->tb_clock_ok), which dcp_tunnel_crossbar_up() checks. - */ - if (neo_dptx->atcphy && service->ep->neo_dcp->neo_dptx_tunnel) { - neo_dcp_tunnel_set_rate(service->ep->neo_dcp, neo_dptx->atcphy, - link_rate); - } else if (neo_dptx->atcphy) { - neo_dptx->phy_ops.dp.link_rate = phy_link_rate; - neo_dptx->phy_ops.dp.set_rate = 1; - ret = phy_configure(neo_dptx->atcphy, &neo_dptx->phy_ops); - if (ret) - return ret; - } - - neo_dptx->link_rate = neo_dptx->pending_link_rate = link_rate; - } - - //dptx->pending_link_rate = link_rate; - reply->retcode = cpu_to_le32(0); - reply->link_rate = cpu_to_le32(link_rate); - - return 0; -} - -static int neo_dptxport_call_get_supports_hpd(struct neo_apple_epic_service *service, - void *reply_, size_t reply_size) -{ - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - struct neo_dptxport_apcall_get_support *reply = reply_; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - - reply->retcode = cpu_to_le32(0); - /* - * Analog DPIN CORE=1 already uses AFK set_hpd (returns 0). - * Denying HPD here made request_display ACTIVATE then 22/24 - * DEVICE_NOT_STARTED (~5.5s). Advertise HPD so firmware uses - * that path instead of waiting for a PHY start. - */ - reply->supported = cpu_to_le32(neo_dcp_is_typec_output(neo_dcp) ? 1 : 0); - dev_info(neo_dcp->dev, "DPTXPort: GET_SUPPORTS_HPD %u usb4=%d\n", - le32_to_cpu(reply->supported), neo_dcp_is_usb4_output(neo_dcp)); - return 0; -} - -static int -neo_dptxport_call_get_supports_downspread(struct neo_apple_epic_service *service, - void *reply_, size_t reply_size) -{ - struct neo_dptxport_apcall_get_support *reply = reply_; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - - reply->retcode = cpu_to_le32(0); - reply->supported = cpu_to_le32(0); - return 0; -} - -static int neo_dptxport_call_set_tiled_display_hint(void *reply_, - size_t reply_size) -{ - struct neo_dptxport_apcall_set_tiled *reply = reply_; - - if (reply_size < sizeof(*reply)) - return -EINVAL; - - reply->retcode = cpu_to_le32(1); - return 0; -} - -static int -neo_dptxport_call_activate(struct neo_apple_epic_service *service, - const void *data, size_t data_size, - void *reply, size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - - /* - * Ported from aurora-silicon/linux#8: no crossbar, and no PHY mode - * change for a tunnel (dptx->atcphy must stay in USB4/TBT mode the - * whole connection, see dcp_tunnel_set_rate()). The DP IN adapter is - * only woken (DPTX_INACTIVE=0) here, via dcp_tunnel_dpin_activate(); - * waking it earlier hangs the machine. Activate always replies - * success to DCP. - */ - if (neo_dptx->atcphy && !neo_dcp->phy_managed_by_typec) - phy_set_mode_ext(neo_dptx->atcphy, PHY_MODE_DP, neo_dcp->index); - if (neo_dcp->neo_dptx_tunnel) - neo_dcp_tunnel_dpin_activate(neo_dcp, true); - - memcpy(reply, data, min(reply_size, data_size)); - if (reply_size >= 4) - memset(reply, 0, 4); - - return 0; -} - -static int -neo_dptxport_call_deactivate(struct neo_apple_epic_service *service, - const void *data, size_t data_size, - void *reply, size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - - dev_info(neo_dcp->dev, "DPTXPort: DEACTIVATE\n"); - if (neo_dcp->neo_dptx_tunnel) - neo_dcp_tunnel_dpin_activate(neo_dcp, false); - if (neo_dptx->atcphy && !neo_dcp->phy_managed_by_typec) - phy_set_mode_ext(neo_dptx->atcphy, PHY_MODE_INVALID, 0); - - memcpy(reply, data, min(reply_size, data_size)); - if (reply_size >= 4) - memset(reply, 0, 4); - - return 0; -} - -static int neo_dptxport_call(struct neo_apple_epic_service *service, u32 idx, - const void *data, size_t data_size, void *reply, - size_t reply_size) -{ - struct neo_dptx_port *neo_dptx = service->cookie; - trace_neo_dptxport_apcall(neo_dptx, idx, data_size); - dev_dbg(service->ep->neo_dcp->dev, "DPTXPort: APCALL %u (%zu bytes)\n", - idx, data_size); - if (data_size) - print_hex_dump_debug("DPTXPort: apcall data: ", - DUMP_PREFIX_OFFSET, 16, 1, data, - min(data_size, (size_t)64), true); - - switch (idx) { - case DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG: - /* - * Ported from aurora-silicon/linux#8: a re-link on an - * established tunnel takes the crossbar connection down - * first. - */ - if (service->ep->neo_dcp->neo_dptx_tunnel && neo_dptx->link_rate) - neo_dcp_tunnel_crossbar_down(service->ep->neo_dcp); - return neo_dptxport_call_will_change_link_config(service); - case DPTX_APCALL_DID_CHANGE_LINK_CONFIG: { - int ret = neo_dptxport_call_did_change_link_config(service); - - if (!ret && service->ep->neo_dcp->neo_dptx_tunnel && neo_dptx->link_rate) - neo_dcp_tunnel_crossbar_up(service->ep->neo_dcp); - return ret; - } - case DPTX_APCALL_GET_MAX_LINK_RATE: - return neo_dptxport_call_get_max_link_rate(service, reply, - reply_size); - case DPTX_APCALL_GET_LINK_RATE: - return neo_dptxport_call_get_link_rate(service, reply, reply_size); - case DPTX_APCALL_SET_LINK_RATE: - return neo_dptxport_call_set_link_rate(service, data, data_size, - reply, reply_size); - case DPTX_APCALL_GET_MAX_LANE_COUNT: - return neo_dptxport_call_get_max_lane_count(service, reply, reply_size); - case DPTX_APCALL_GET_ACTIVE_LANE_COUNT: { - struct neo_dptxport_apcall_lane_count *lc = reply; - - if (reply_size < sizeof(*lc)) - return -EINVAL; - lc->retcode = cpu_to_le32(0); - lc->lane_count = cpu_to_le64(neo_dptx->lane_count); - return 0; - } - case DPTX_APCALL_SET_ACTIVE_LANE_COUNT: - return neo_dptxport_call_set_active_lane_count(service, data, data_size, - reply, reply_size); - case DPTX_APCALL_GET_SUPPORTS_HPD: - return neo_dptxport_call_get_supports_hpd(service, reply, - reply_size); - case DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD: - return neo_dptxport_call_get_supports_downspread(service, reply, - reply_size); - case DPTX_APCALL_GET_MAX_DRIVE_SETTINGS: - return neo_dptxport_call_get_max_drive_settings(service, reply, - reply_size); - case DPTX_APCALL_DEVICE_NOT_RESPONDING: - case DPTX_APCALL_DEVICE_BUSY_TIMEOUT: - case DPTX_APCALL_DEVICE_NOT_STARTED: - dev_warn(service->ep->neo_dcp->dev, - "DPTXPort: firmware reports link fault %u on target %u:%u\n", - idx, service->ep->neo_dcp->neo_dptx_die, - service->ep->neo_dcp->neo_dptx_phy); - memcpy(reply, data, min(reply_size, data_size)); - return 0; - case DPTX_APCALL_SET_TILED_DISPLAY_HINTS: - memcpy(reply, data, min(reply_size, data_size)); - return neo_dptxport_call_set_tiled_display_hint(reply, reply_size); - case DPTX_APCALL_GET_DRIVE_SETTINGS: - return neo_dptxport_call_get_drive_settings(service, data, data_size, - reply, reply_size); - case DPTX_APCALL_SET_DRIVE_SETTINGS: - return neo_dptxport_call_set_drive_settings(service, data, data_size, - reply, reply_size); - case DPTX_APCALL_ACTIVATE: - return neo_dptxport_call_activate(service, data, data_size, - reply, reply_size); - case DPTX_APCALL_DEACTIVATE: - return neo_dptxport_call_deactivate(service, data, data_size, - reply, reply_size); - case DPTX_APCALL_FORCE_HOTPLUG_DETECT: - dev_info(service->ep->neo_dcp->dev, - "DPTXPort: FORCE_HOTPLUG_DETECT\n"); - complete(&neo_dptx->linkcfg_completion); - memcpy(reply, data, min(reply_size, data_size)); - if (reply_size >= 4) - memset(reply, 0, 4); - return 0; - case DPTX_APCALL_INACTIVE_SINK_DETECTED: - /* - * Normal prelude to link training on USB4. Ack and wait for - * SET_ACTIVE_LANE_COUNT. Treating this as failure made - * firmware DEACTIVATE after it had already set lanes/rate. - */ - dev_info(service->ep->neo_dcp->dev, - "DPTXPort: INACTIVE_SINK_DETECTED (keep waiting for lanes)\n"); - memcpy(reply, data, min(reply_size, data_size)); - if (reply_size >= 4) - memset(reply, 0, 4); - return 0; - default: - /* just try to ACK and hope for the best... */ - dev_info(service->ep->neo_dcp->dev, "DPTXPort: acking unhandled call %u\n", - idx); - fallthrough; - case DPTX_APCALL_GET_DOWN_SPREAD: - case DPTX_APCALL_SET_DOWN_SPREAD: - memcpy(reply, data, min(reply_size, data_size)); - if (reply_size >= 4) - memset(reply, 0, 4); - return 0; - } -} - -static void neo_dptxport_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ - - if (strcmp(name, "dcpdptx-port-epic")) - return; - if (strcmp(class, "AppleDCPDPTXRemotePort")) - return; - - trace_neo_dptxport_init(service->ep->neo_dcp, unit); - - switch (unit) { - case 0: - case 1: - if (service->ep->neo_dcp->neo_dptxport[unit].enabled) { - dev_err(service->ep->neo_dcp->dev, - "DPTXPort: unit %lld already exists\n", unit); - return; - } - service->ep->neo_dcp->neo_dptxport[unit].unit = unit; - service->ep->neo_dcp->neo_dptxport[unit].service = service; - service->ep->neo_dcp->neo_dptxport[unit].enabled = true; - service->cookie = (void *)&service->ep->neo_dcp->neo_dptxport[unit]; - complete(&service->ep->neo_dcp->neo_dptxport[unit].enable_completion); - break; - default: - dev_err(service->ep->neo_dcp->dev, "DPTXPort: invalid unit %lld\n", - unit); - } -} - -static const struct neo_apple_epic_service_ops neo_dptxep_ops[] = { - { - .name = "AppleDCPDPTXRemotePort", - .init = neo_dptxport_init, - .call = neo_dptxport_call, - }, - {} -}; - -int neo_dptxep_init(struct neo_apple_dcp *neo_dcp) -{ - int ret; - u32 port; - unsigned long timeout = msecs_to_jiffies(1000); - - init_completion(&neo_dcp->neo_dptxport[0].enable_completion); - init_completion(&neo_dcp->neo_dptxport[1].enable_completion); - init_completion(&neo_dcp->neo_dptxport[0].linkcfg_completion); - init_completion(&neo_dcp->neo_dptxport[1].linkcfg_completion); - - neo_dcp->neo_dptxep = neo_afk_init(neo_dcp, DPTX_ENDPOINT, neo_dptxep_ops); - if (IS_ERR(neo_dcp->neo_dptxep)) - return PTR_ERR(neo_dcp->neo_dptxep); - - ret = neo_afk_start(neo_dcp->neo_dptxep); - if (ret) - return ret; - - for (port = 0; port < neo_dcp->hw.num_dptx_ports; port++) { - ret = wait_for_completion_timeout(&neo_dcp->neo_dptxport[port].enable_completion, - timeout); - if (!ret) - return -ETIMEDOUT; - else if (ret < 0) - return ret; - timeout = ret; - } - - return 0; -} diff --git a/drivers/gpu/drm/apple-neo/dptxep.h b/drivers/gpu/drm/apple-neo/dptxep.h deleted file mode 100644 index e39e73ef409361..00000000000000 --- a/drivers/gpu/drm/apple-neo/dptxep.h +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef __APPLE_DCP_DPTXEP_H__ -#define __APPLE_DCP_DPTXEP_H__ - -#include -#include - -enum neo_dptx_apcall { - DPTX_APCALL_ACTIVATE = 0, - DPTX_APCALL_DEACTIVATE = 1, - DPTX_APCALL_GET_MAX_DRIVE_SETTINGS = 2, - DPTX_APCALL_SET_DRIVE_SETTINGS = 3, - DPTX_APCALL_GET_DRIVE_SETTINGS = 4, - DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG = 5, - DPTX_APCALL_DID_CHANGE_LINK_CONFIG = 6, - DPTX_APCALL_GET_MAX_LINK_RATE = 7, - DPTX_APCALL_GET_LINK_RATE = 8, - DPTX_APCALL_SET_LINK_RATE = 9, - DPTX_APCALL_GET_MAX_LANE_COUNT = 10, - DPTX_APCALL_GET_ACTIVE_LANE_COUNT = 11, - DPTX_APCALL_SET_ACTIVE_LANE_COUNT = 12, - DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD = 13, - DPTX_APCALL_GET_DOWN_SPREAD = 14, - DPTX_APCALL_SET_DOWN_SPREAD = 15, - DPTX_APCALL_GET_SUPPORTS_LANE_MAPPING = 16, - DPTX_APCALL_SET_LANE_MAP = 17, - DPTX_APCALL_GET_SUPPORTS_HPD = 18, - DPTX_APCALL_FORCE_HOTPLUG_DETECT = 19, - DPTX_APCALL_INACTIVE_SINK_DETECTED = 20, - DPTX_APCALL_SET_TILED_DISPLAY_HINTS = 21, - DPTX_APCALL_DEVICE_NOT_RESPONDING = 22, - DPTX_APCALL_DEVICE_BUSY_TIMEOUT = 23, - DPTX_APCALL_DEVICE_NOT_STARTED = 24, -}; - -#define DCPDPTX_REMOTE_PORT_CORE GENMASK(3, 0) -#define DCPDPTX_REMOTE_PORT_ATC GENMASK(7, 4) -#define DCPDPTX_REMOTE_PORT_DIE GENMASK(11, 8) -#define DCPDPTX_REMOTE_PORT_CONNECTED BIT(15) -#define DCPDPTX_REMOTE_PORT_SUPPORTS_HPD BIT(8) - -enum neo_dptx_link_rate { - LINK_RATE_RBR = 0x06, - LINK_RATE_HBR = 0x0a, - LINK_RATE_HBR2 = 0x14, - LINK_RATE_HBR3 = 0x1e, -}; - -struct neo_apple_epic_service; - -struct neo_dptx_port { - bool enabled, connected; - struct completion enable_completion; - struct completion linkcfg_completion; - u32 unit; - struct neo_apple_epic_service *service; - union phy_configure_opts phy_ops; - struct phy *atcphy; - struct mux_control *mux; - u32 lane_count; - u32 link_rate, pending_link_rate; - u32 drive_settings[2]; - /* - * Per-boot invocation counters for the validate/connect/request/ - * release AFK calls, used together with the caller return address - * logged alongside them to correlate call ordering when diagnosing - * link-training sequencing issues. - */ - u32 validate_calls, connect_calls, request_calls, release_calls; -}; - -int neo_dptxport_validate_connection(struct neo_apple_epic_service *service, u8 core, - u8 atc, u8 die); -int neo_dptxport_connect(struct neo_apple_epic_service *service, u8 core, u8 atc, - u8 die, bool supports_hpd); -int neo_dptxport_request_display(struct neo_apple_epic_service *service); -int neo_dptxport_release_display(struct neo_apple_epic_service *service); -int neo_dptxport_set_hpd(struct neo_apple_epic_service *service, bool hpd); -int neo_dptxport_set_hpd_timeout(struct neo_apple_epic_service *service, bool hpd, - unsigned int timeout_ms); - -#endif diff --git a/drivers/gpu/drm/apple-neo/epic/dpavservep.c b/drivers/gpu/drm/apple-neo/epic/dpavservep.c deleted file mode 100644 index 8ec6b858822c93..00000000000000 --- a/drivers/gpu/drm/apple-neo/epic/dpavservep.c +++ /dev/null @@ -1,302 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#include "dpavservep.h" - -#include - -#include -#include -#include - -#include "../afk.h" -#include "../dcp.h" -#include "../dcp-internal.h" -#include "../trace.h" - -static void neo_dcpavserv_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - unsigned long flags; - - trace_neo_dcpavserv_init(neo_dcp, unit); - - if (unit == 0 && name && !strcmp(name, "dcpav-service-epic")) { - spin_lock_irqsave(&neo_dcp->neo_dcpavserv.lock, flags); - if (neo_dcp->neo_dcpavserv.enabled) { - spin_unlock_irqrestore(&neo_dcp->neo_dcpavserv.lock, flags); - dev_err(neo_dcp->dev, - "DCPAVSERV: unit %lld already exists\n", unit); - return; - } - neo_dcp->neo_dcpavserv.service = neo_afk_service_get(service); - neo_dcp->neo_dcpavserv.enabled = true; - service->cookie = &neo_dcp->neo_dcpavserv; - complete(&neo_dcp->neo_dcpavserv.enable_completion); - spin_unlock_irqrestore(&neo_dcp->neo_dcpavserv.lock, flags); - } -} - -static void neo_dcpavserv_teardown(struct neo_apple_epic_service *service) -{ - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - unsigned long flags; - bool owned = false; - - spin_lock_irqsave(&neo_dcp->neo_dcpavserv.lock, flags); - if (neo_dcp->neo_dcpavserv.service == service) { - neo_dcp->neo_dcpavserv.enabled = false; - neo_dcp->neo_dcpavserv.service = NULL; - reinit_completion(&neo_dcp->neo_dcpavserv.enable_completion); - owned = true; - } - spin_unlock_irqrestore(&neo_dcp->neo_dcpavserv.lock, flags); - if (owned) - neo_afk_service_put(service); -} - -void dpavservep_detach(struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_epic_service *service; - unsigned long flags; - - spin_lock_irqsave(&neo_dcp->neo_dcpavserv.lock, flags); - service = neo_dcp->neo_dcpavserv.service; - neo_dcp->neo_dcpavserv.service = NULL; - neo_dcp->neo_dcpavserv.enabled = false; - spin_unlock_irqrestore(&neo_dcp->neo_dcpavserv.lock, flags); - if (service) { - neo_afk_service_disable(service); - neo_afk_service_put(service); - } -} - -static void dcpdpserv_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ -} - -static void dcpdpserv_teardown(struct neo_apple_epic_service *service) -{ - neo_afk_service_disable(service); -} - -struct neo_dcpavserv_status_report { - u32 unk00[4]; - u8 flag0; - u8 flag1; - u8 flag2; - u8 flag3; - u32 unk14[3]; - u32 status; - u32 unk24[3]; -} __packed; - -struct neo_dpavserv_copy_edid_cmd { - __le64 max_size; - u8 _pad1[24]; - __le64 used_size; - u8 _pad2[8]; -} __packed; - -#define EDID_LEADING_DATA_SIZE 8 -#define EDID_BLOCK_SIZE 128 -#define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E -#define EDID_MAX_SIZE SZ_32K -#define EDID_BUF_SIZE (EDID_LEADING_DATA_SIZE + EDID_MAX_SIZE) - -struct neo_dpavserv_copy_edid_resp { - __le64 max_size; - u8 _pad1[24]; - __le64 used_size; - u8 _pad2[8]; - u8 data[]; -} __packed; -static_assert(sizeof(struct neo_dpavserv_copy_edid_resp) == 48); - -static int neo_parse_report(struct neo_apple_epic_service *service, enum neo_epic_subtype type, - const void *data, size_t data_size) -{ -#if defined(DEBUG) - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - const struct neo_epic_service_call *call; - const void *payload; - size_t payload_size; - - dev_dbg(neo_dcp->dev, "dcpavserv[ch:%u]: report type:%02x len:%zu\n", - service->channel, type, data_size); - - if (type != EPIC_SUBTYPE_STD_SERVICE) - return 0; - - if (data_size < sizeof(*call)) - return 0; - - call = data; - - if (le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC) { - dev_warn(neo_dcp->dev, "dcpavserv[ch:%u]: report magic 0x%08x != 0x%08x\n", - service->channel, le32_to_cpu(call->magic), EPIC_SERVICE_CALL_MAGIC); - return 0; - } - - payload_size = data_size - sizeof(*call); - if (payload_size < le32_to_cpu(call->data_len)) { - dev_warn(neo_dcp->dev, "dcpavserv[ch:%u]: report payload size %zu call len %u\n", - service->channel, payload_size, le32_to_cpu(call->data_len)); - return 0; - } - payload_size = le32_to_cpu(call->data_len); - payload = data + sizeof(*call); - - if (le16_to_cpu(call->group) == 2 && le16_to_cpu(call->command) == 0) { - if (payload_size == sizeof(struct neo_dcpavserv_status_report)) { - const struct neo_dcpavserv_status_report *stat = payload; - dev_info(neo_dcp->dev, "dcpavserv[ch:%u]: flags: 0x%02x,0x%02x,0x%02x,0x%02x status:%u\n", - service->channel, stat->flag0, stat->flag1, - stat->flag2, stat->flag3, stat->status); - } else { - dev_dbg(neo_dcp->dev, "dcpavserv[ch:%u]: report payload size %zu\n", service->channel, payload_size); - } - } else { - print_hex_dump(KERN_DEBUG, "dcpavserv report: ", DUMP_PREFIX_NONE, - 16, 1, payload, payload_size, true); - } -#endif - - return 0; -} - -static int neo_dcpavserv_report(struct neo_apple_epic_service *service, - enum neo_epic_subtype type, const void *data, - size_t data_size) -{ - return neo_parse_report(service, type, data, data_size); -} - -static int dcpdpserv_report(struct neo_apple_epic_service *service, - enum neo_epic_subtype type, const void *data, - size_t data_size) -{ - return neo_parse_report(service, type, data, data_size); -} - -static const struct drm_edid *neo_dcpavserv_read_edid(struct neo_apple_epic_service *service) -{ - struct neo_dpavserv_copy_edid_cmd cmd; - struct neo_dpavserv_copy_edid_resp *resp __free(kfree) = NULL; - size_t resp_size = sizeof(*resp) + EDID_BUF_SIZE; - int num_blocks; - u64 data_size; - int ret; - - memset(&cmd, 0, sizeof(cmd)); - cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); - resp = kzalloc(resp_size, GFP_KERNEL); - if (!resp) - return ERR_PTR(-ENOMEM); - - ret = neo_afk_service_call(service, 1, 7, &cmd, sizeof(cmd), EDID_BUF_SIZE, resp, - resp_size, 0); - if (ret < 0) - return ERR_PTR(ret); - - if (le64_to_cpu(resp->max_size) != EDID_BUF_SIZE) - return ERR_PTR(-EIO); - - // print_hex_dump(KERN_DEBUG, "dpavserv EDID cmd: ", DUMP_PREFIX_NONE, - // 16, 1, resp, 192, true); - - data_size = le64_to_cpu(resp->used_size); - if (data_size < EDID_LEADING_DATA_SIZE + EDID_BLOCK_SIZE || - data_size > EDID_BUF_SIZE) - return ERR_PTR(-EIO); - - /* - * An HDMI 2.x sink using the HF-EEODB announces fewer extension blocks - * in the base block than it sends, so the payload may be longer than - * the base block says. Accept any whole number of blocks at least that - * long and leave the block count to drm_edid, which knows about - * HF-EEODB. - */ - num_blocks = resp->data[EDID_LEADING_DATA_SIZE + EDID_EXT_BLOCK_COUNT_OFFSET]; - if ((1 + num_blocks) * EDID_BLOCK_SIZE > data_size - EDID_LEADING_DATA_SIZE || - (data_size - EDID_LEADING_DATA_SIZE) % EDID_BLOCK_SIZE) - return ERR_PTR(-EIO); - - return drm_edid_alloc(resp->data + EDID_LEADING_DATA_SIZE, - data_size - EDID_LEADING_DATA_SIZE); -} - -const struct drm_edid *neo_dcpavserv_copy_edid(struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_epic_service *service; - const struct drm_edid *edid; - unsigned long flags; - - spin_lock_irqsave(&neo_dcp->neo_dcpavserv.lock, flags); - service = neo_afk_service_get(neo_dcp->neo_dcpavserv.service); - spin_unlock_irqrestore(&neo_dcp->neo_dcpavserv.lock, flags); - if (!service) - return ERR_PTR(-ENODEV); - - edid = neo_dcpavserv_read_edid(service); - spin_lock_irqsave(&neo_dcp->neo_dcpavserv.lock, flags); - if (neo_dcp->neo_dcpavserv.service != service) { - if (!IS_ERR(edid)) - drm_edid_free(edid); - edid = ERR_PTR(-ENODEV); - } - spin_unlock_irqrestore(&neo_dcp->neo_dcpavserv.lock, flags); - neo_afk_service_put(service); - return edid; -} - -static const struct neo_apple_epic_service_ops dpavservep_ops[] = { - { - .name = "dcpav-service-epic", - .reusable = true, - .init = neo_dcpavserv_init, - .teardown = neo_dcpavserv_teardown, - .report = neo_dcpavserv_report, - }, - { - .name = "dcpdp-service-epic", - .reusable = true, - .init = dcpdpserv_init, - .teardown = dcpdpserv_teardown, - .report = dcpdpserv_report, - }, - {}, -}; - -int dpavservep_init(struct neo_apple_dcp *neo_dcp) -{ - struct neo_apple_dcp_afkep *ep; - int ret; - - init_completion(&neo_dcp->neo_dcpavserv.enable_completion); - - ep = neo_afk_init(neo_dcp, DPAVSERV_ENDPOINT, dpavservep_ops); - if (IS_ERR(ep)) - return PTR_ERR(ep); - neo_dcp->dcpavservep = ep; - - neo_dcp->dcpavservep->match_epic_name = true; - - ret = neo_afk_start(neo_dcp->dcpavservep); - if (ret) { - neo_afk_shutdown(ep); - dpavservep_detach(neo_dcp); - neo_dcp->dcpavservep = NULL; - return ret; - } - - ret = wait_for_completion_timeout(&neo_dcp->neo_dcpavserv.enable_completion, - msecs_to_jiffies(1000)); - if (ret >= 0) - return 0; - - return ret; -} diff --git a/drivers/gpu/drm/apple-neo/epic/dpavservep.h b/drivers/gpu/drm/apple-neo/epic/dpavservep.h deleted file mode 100644 index f8caa5262a523a..00000000000000 --- a/drivers/gpu/drm/apple-neo/epic/dpavservep.h +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#ifndef _DRM_APPLE_EPIC_DPAVSERV_H -#define _DRM_APPLE_EPIC_DPAVSERV_H - -#include -#include -#include - -struct drm_edid; -struct neo_apple_epic_service; -struct neo_apple_dcp; - -struct neo_dcpavserv { - spinlock_t lock; /* Protects the service pointer and its owner pin. */ - bool enabled; - struct completion enable_completion; - u32 unit; - struct neo_apple_epic_service *service; -}; - -const struct drm_edid *neo_dcpavserv_copy_edid(struct neo_apple_dcp *neo_dcp); -void dpavservep_detach(struct neo_apple_dcp *neo_dcp); - -#endif /* _DRM_APPLE_EPIC_DPAVSERV_H */ diff --git a/drivers/gpu/drm/apple-neo/hdmi-codec-chmap.h b/drivers/gpu/drm/apple-neo/hdmi-codec-chmap.h deleted file mode 100644 index f98e1e86b89602..00000000000000 --- a/drivers/gpu/drm/apple-neo/hdmi-codec-chmap.h +++ /dev/null @@ -1,123 +0,0 @@ -// copied from sound/soc/codecs/hdmi-codec.c - -#include - -/* Channel maps for multi-channel playbacks, up to 8 n_ch */ -static const struct snd_pcm_chmap_elem hdmi_codec_8ch_chmaps[] = { - { .channels = 2, /* CA_ID 0x00 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR } }, - { .channels = 4, /* CA_ID 0x01 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA } }, - { .channels = 4, /* CA_ID 0x02 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC } }, - { .channels = 4, /* CA_ID 0x03 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC } }, - { .channels = 6, /* CA_ID 0x04 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 6, /* CA_ID 0x05 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 6, /* CA_ID 0x06 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 6, /* CA_ID 0x07 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 6, /* CA_ID 0x08 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, - { .channels = 6, /* CA_ID 0x09 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, - { .channels = 6, /* CA_ID 0x0A */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, - { .channels = 6, /* CA_ID 0x0B */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR } }, - { .channels = 8, /* CA_ID 0x0C */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 8, /* CA_ID 0x0D */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 8, /* CA_ID 0x0E */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 8, /* CA_ID 0x0F */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RC, SNDRV_CHMAP_NA } }, - { .channels = 8, /* CA_ID 0x10 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, - { .channels = 8, /* CA_ID 0x11 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, - { .channels = 8, /* CA_ID 0x12 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, - { .channels = 8, /* CA_ID 0x13 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC, SNDRV_CHMAP_RL, SNDRV_CHMAP_RR, - SNDRV_CHMAP_RLC, SNDRV_CHMAP_RRC } }, - { .channels = 8, /* CA_ID 0x14 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x15 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x16 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x17 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x18 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x19 */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x1A */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x1B */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x1C */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x1D */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x1E */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { .channels = 8, /* CA_ID 0x1F */ - .map = { SNDRV_CHMAP_FL, SNDRV_CHMAP_FR, SNDRV_CHMAP_LFE, - SNDRV_CHMAP_FC, SNDRV_CHMAP_NA, SNDRV_CHMAP_NA, - SNDRV_CHMAP_FLC, SNDRV_CHMAP_FRC } }, - { } -}; diff --git a/drivers/gpu/drm/apple-neo/ibootep.c b/drivers/gpu/drm/apple-neo/ibootep.c deleted file mode 100644 index a84b527ca10031..00000000000000 --- a/drivers/gpu/drm/apple-neo/ibootep.c +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2023 */ - -#include - -#include "afk.h" -#include "dcp.h" - -static void disp_service_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ -} - - -static const struct neo_apple_epic_service_ops neo_ibootep_ops[] = { - { - .name = "disp0-service", - .init = disp_service_init, - }, - {} -}; - -int neo_ibootep_init(struct neo_apple_dcp *neo_dcp) -{ - neo_dcp->neo_ibootep = neo_afk_init(neo_dcp, DISP0_ENDPOINT, neo_ibootep_ops); - neo_afk_start(neo_dcp->neo_ibootep); - - return 0; -} diff --git a/drivers/gpu/drm/apple-neo/iomfb-state-test.c b/drivers/gpu/drm/apple-neo/iomfb-state-test.c deleted file mode 100644 index 5b597580e36069..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb-state-test.c +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#include -#include - -#include "iomfb-state.h" - -static void neo_dcp_mode_success_test(struct kunit *test) -{ - struct neo_dcp_mode_state state = { }; - bool connected = true; - - spin_lock_init(&state.lock); - neo_dcp_mode_begin(&state); - KUNIT_EXPECT_FALSE(test, state.valid); - KUNIT_EXPECT_EQ(test, neo_dcp_mode_finish(&state, true, &connected), 0U); - KUNIT_EXPECT_TRUE(test, state.valid); - KUNIT_EXPECT_FALSE(test, state.changing); -} - -static void neo_dcp_mode_failure_test(struct kunit *test) -{ - struct neo_dcp_mode_state state = { .valid = true }; - bool connected = true; - - spin_lock_init(&state.lock); - neo_dcp_mode_begin(&state); - neo_dcp_mode_finish(&state, false, &connected); - KUNIT_EXPECT_FALSE(test, state.valid); - KUNIT_EXPECT_FALSE(test, state.changing); -} - -static void neo_dcp_mode_deferred_disconnect_test(struct kunit *test) -{ - struct neo_dcp_mode_state state = { }; - bool connected = true; - - spin_lock_init(&state.lock); - neo_dcp_mode_begin(&state); - KUNIT_EXPECT_EQ(test, neo_dcp_mode_hotplug(&state, false, &connected), 0U); - KUNIT_EXPECT_TRUE(test, connected); - KUNIT_EXPECT_EQ(test, neo_dcp_mode_finish(&state, true, &connected), - (unsigned int)(DCP_HOTPLUG_VBLANK | DCP_HOTPLUG_NOTIFY)); - KUNIT_EXPECT_FALSE(test, connected); - KUNIT_EXPECT_FALSE(test, state.valid); - KUNIT_EXPECT_FALSE(test, state.hotplug_pending); -} - -static void neo_dcp_mode_deferred_connect_test(struct kunit *test) -{ - struct neo_dcp_mode_state state = { }; - bool connected = false; - - spin_lock_init(&state.lock); - neo_dcp_mode_begin(&state); - neo_dcp_mode_hotplug(&state, true, &connected); - KUNIT_EXPECT_EQ(test, neo_dcp_mode_finish(&state, true, &connected), - (unsigned int)DCP_HOTPLUG_NOTIFY); - KUNIT_EXPECT_TRUE(test, connected); - KUNIT_EXPECT_FALSE(test, state.valid); - /* A recovery modeset without another link change can now succeed. */ - neo_dcp_mode_begin(&state); - neo_dcp_mode_finish(&state, true, &connected); - KUNIT_EXPECT_TRUE(test, state.valid); -} - -static void neo_dcp_mode_transient_disconnect_test(struct kunit *test) -{ - struct neo_dcp_mode_state state = { }; - bool connected = true; - - spin_lock_init(&state.lock); - neo_dcp_mode_begin(&state); - neo_dcp_mode_hotplug(&state, false, &connected); - neo_dcp_mode_hotplug(&state, true, &connected); - KUNIT_EXPECT_EQ(test, neo_dcp_mode_finish(&state, true, &connected), - (unsigned int)DCP_HOTPLUG_NOTIFY); - KUNIT_EXPECT_TRUE(test, connected); - KUNIT_EXPECT_FALSE(test, state.valid); -} - -static void neo_dcp_mode_invalidation_order_test(struct kunit *test) -{ - struct neo_dcp_mode_state state = { }; - bool connected = true; - - spin_lock_init(&state.lock); - /* Invalidation just before completion must win. */ - neo_dcp_mode_begin(&state); - neo_dcp_mode_invalidate(&state); - neo_dcp_mode_finish(&state, true, &connected); - KUNIT_EXPECT_FALSE(test, state.valid); - /* So must invalidation just after completion. */ - neo_dcp_mode_begin(&state); - neo_dcp_mode_finish(&state, true, &connected); - neo_dcp_mode_invalidate(&state); - KUNIT_EXPECT_FALSE(test, state.valid); -} - -static void neo_dcp_mode_disconnected_test(struct kunit *test) -{ - struct neo_dcp_mode_state state = { }; - bool connected = false; - - spin_lock_init(&state.lock); - neo_dcp_mode_begin(&state); - neo_dcp_mode_finish(&state, true, &connected); - KUNIT_EXPECT_FALSE(test, state.valid); -} - -static struct kunit_case neo_dcp_mode_cases[] = { - KUNIT_CASE(neo_dcp_mode_success_test), - KUNIT_CASE(neo_dcp_mode_failure_test), - KUNIT_CASE(neo_dcp_mode_deferred_disconnect_test), - KUNIT_CASE(neo_dcp_mode_deferred_connect_test), - KUNIT_CASE(neo_dcp_mode_transient_disconnect_test), - KUNIT_CASE(neo_dcp_mode_invalidation_order_test), - KUNIT_CASE(neo_dcp_mode_disconnected_test), - { } -}; - -static struct kunit_suite neo_dcp_mode_suite = { - .name = "apple-iomfb-mode-state", - .test_cases = neo_dcp_mode_cases, -}; - -kunit_test_suite(neo_dcp_mode_suite); -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple-neo/iomfb-state.h b/drivers/gpu/drm/apple-neo/iomfb-state.h deleted file mode 100644 index 800146a8c236f3..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb-state.h +++ /dev/null @@ -1,110 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -#ifndef __APPLE_IOMFB_STATE_H__ -#define __APPLE_IOMFB_STATE_H__ - -#include - -/* Shared by the atomic commit worker and the firmware callback thread. */ -struct neo_dcp_mode_state { - spinlock_t lock; /* Protects all state and hotplug connector updates. */ - bool valid; - bool changing; - bool invalidated; - bool hotplug_pending; - bool connected_pending; -}; - -enum neo_dcp_hotplug_action { - DCP_HOTPLUG_NONE = 0, - DCP_HOTPLUG_VBLANK = BIT(0), - DCP_HOTPLUG_NOTIFY = BIT(1), -}; - -static inline void neo_dcp_mode_begin(struct neo_dcp_mode_state *state) -{ - unsigned long flags; - - spin_lock_irqsave(&state->lock, flags); - state->valid = false; - state->changing = true; - state->invalidated = false; - state->hotplug_pending = false; - spin_unlock_irqrestore(&state->lock, flags); -} - -static inline void neo_dcp_mode_invalidate(struct neo_dcp_mode_state *state) -{ - unsigned long flags; - - spin_lock_irqsave(&state->lock, flags); - state->valid = false; - state->invalidated = true; - spin_unlock_irqrestore(&state->lock, flags); -} - -/* The caller holds state->lock. */ -static inline unsigned int -neo_dcp_mode_apply_hotplug(struct neo_dcp_mode_state *state, bool connected, - bool *connector_connected) -{ - unsigned int action = DCP_HOTPLUG_NONE; - - if (!connected) { - state->valid = false; - action |= DCP_HOTPLUG_VBLANK; - } - if (connector_connected && *connector_connected != connected) { - *connector_connected = connected; - state->valid = false; - action |= DCP_HOTPLUG_NOTIFY; - } - return action; -} - -static inline unsigned int -neo_dcp_mode_hotplug(struct neo_dcp_mode_state *state, bool connected, - bool *connector_connected) -{ - unsigned long flags; - unsigned int action = DCP_HOTPLUG_NONE; - - spin_lock_irqsave(&state->lock, flags); - if (state->changing) { - state->hotplug_pending = true; - state->connected_pending = connected; - /* A later connect must not erase a link loss during this call. */ - if (!connected) - state->invalidated = true; - } else { - action = neo_dcp_mode_apply_hotplug(state, connected, - connector_connected); - } - spin_unlock_irqrestore(&state->lock, flags); - return action; -} - -static inline unsigned int -neo_dcp_mode_finish(struct neo_dcp_mode_state *state, bool completed, - bool *connector_connected) -{ - unsigned long flags; - unsigned int action = DCP_HOTPLUG_NONE; - - spin_lock_irqsave(&state->lock, flags); - state->valid = completed && !state->invalidated; - state->changing = false; - if (state->hotplug_pending) { - state->hotplug_pending = false; - action = neo_dcp_mode_apply_hotplug(state, state->connected_pending, - connector_connected); - } - if (connector_connected && !*connector_connected) - state->valid = false; - /* A transient unplug also needs recovery if the final state is up. */ - if (state->invalidated && connector_connected && *connector_connected) - action |= DCP_HOTPLUG_NOTIFY; - spin_unlock_irqrestore(&state->lock, flags); - return action; -} - -#endif diff --git a/drivers/gpu/drm/apple-neo/iomfb.c b/drivers/gpu/drm/apple-neo/iomfb.c deleted file mode 100644 index 7e8fdc055eff38..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb.c +++ /dev/null @@ -1,1588 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dcp.h" -#include "dcp-internal.h" -#include "iomfb.h" -#include "iomfb_internal.h" -#include "parser.h" -#include "trace.h" - -static int neo_dcp_tx_offset(enum neo_dcp_context_id id) -{ - switch (id) { - case DCP_CONTEXT_CB: - case DCP_CONTEXT_CMD: - return 0x00000; - case DCP_CONTEXT_OOBCB: - case DCP_CONTEXT_OOBCMD: - return 0x08000; - default: - return -EINVAL; - } -} - -static int neo_dcp_channel_offset(enum neo_dcp_context_id id) -{ - switch (id) { - case DCP_CONTEXT_ASYNC: - return 0x40000; - case DCP_CONTEXT_OOBASYNC: - return 0x48000; - case DCP_CONTEXT_CB: - return 0x60000; - case DCP_CONTEXT_OOBCB: - return 0x68000; - default: - return neo_dcp_tx_offset(id); - } -} - -static inline u64 dcpep_set_shmem(u64 dart_va) -{ - return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_SET_SHMEM) | - FIELD_PREP(IOMFB_SHMEM_FLAG, IOMFB_SHMEM_FLAG_VALUE) | - FIELD_PREP(IOMFB_SHMEM_DVA, dart_va); -} - -static inline u64 dcpep_msg(enum neo_dcp_context_id id, u32 length, u16 offset) -{ - return FIELD_PREP(IOMFB_MESSAGE_TYPE, IOMFB_MESSAGE_TYPE_MSG) | - FIELD_PREP(IOMFB_MSG_CONTEXT, id) | - FIELD_PREP(IOMFB_MSG_OFFSET, offset) | - FIELD_PREP(IOMFB_MSG_LENGTH, length); -} - -static inline u64 dcpep_ack(enum neo_dcp_context_id id) -{ - return dcpep_msg(id, 0, 0) | IOMFB_MSG_ACK; -} - -/* - * A channel is busy if we have sent a message that has yet to be - * acked. The driver must not sent a message to a busy channel. - */ -static bool neo_dcp_channel_busy(struct neo_dcp_channel *ch) -{ - return (ch->depth != 0); -} - -/* - * Get the context ID passed to the DCP for a command we push. The rule is - * simple: callback contexts are used when replying to the DCP, command - * contexts are used otherwise. That corresponds to a non/zero call stack - * depth. This rule frees the caller from tracking the call context manually. - */ -static enum neo_dcp_context_id neo_dcp_call_context(struct neo_apple_dcp *neo_dcp, bool oob) -{ - u8 depth = oob ? neo_dcp->ch_oobcmd.depth : neo_dcp->ch_cmd.depth; - - if (depth) - return oob ? DCP_CONTEXT_OOBCB : DCP_CONTEXT_CB; - else - return oob ? DCP_CONTEXT_OOBCMD : DCP_CONTEXT_CMD; -} - -/* Get a channel for a context */ -static struct neo_dcp_channel *neo_dcp_get_channel(struct neo_apple_dcp *neo_dcp, - enum neo_dcp_context_id context) -{ - switch (context) { - case DCP_CONTEXT_CB: - return &neo_dcp->ch_cb; - case DCP_CONTEXT_CMD: - return &neo_dcp->ch_cmd; - case DCP_CONTEXT_OOBCB: - return &neo_dcp->ch_oobcb; - case DCP_CONTEXT_OOBCMD: - return &neo_dcp->ch_oobcmd; - case DCP_CONTEXT_ASYNC: - return &neo_dcp->ch_async; - case DCP_CONTEXT_OOBASYNC: - return &neo_dcp->ch_oobasync; - default: - return NULL; - } -} - -/* Get the start of a packet: after the end of the previous packet */ -static u16 neo_dcp_packet_start(struct neo_dcp_channel *ch, u8 depth) -{ - if (depth > 0) - return ch->end[depth - 1]; - else - return 0; -} - -/* Pushes and pops the depth of the call stack with safety checks */ -static u8 neo_dcp_push_depth(u8 *depth) -{ - u8 ret = (*depth)++; - - WARN_ON(ret >= DCP_MAX_CALL_DEPTH); - return ret; -} - -static u8 neo_dcp_pop_depth(u8 *depth) -{ - WARN_ON((*depth) == 0); - - return --(*depth); -} - -/* Older firmware and the H17G method profile retain their existing transport. */ -static bool neo_iomfb_uses_queue(struct neo_apple_dcp *neo_dcp) -{ - return neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G; -} - -static bool neo_iomfb_channels_idle(struct neo_apple_dcp *neo_dcp) -{ - return !neo_dcp->ch_cmd.depth && !neo_dcp->ch_cb.depth && - !neo_dcp->ch_oobcmd.depth && !neo_dcp->ch_oobcb.depth && - !neo_dcp->ch_async.depth && !neo_dcp->ch_oobasync.depth; -} - -enum neo_iomfb_opaque_x_state { - IOMFB_OPAQUE_X_WAITING, - IOMFB_OPAQUE_X_NEEDED, - IOMFB_OPAQUE_X_QUEUED, - IOMFB_OPAQUE_X_READY, -}; - -struct neo_iomfb_opaque_x_transaction { - struct neo_iomfb_transaction transaction; -}; - -static void neo_iomfb_opaque_x_start(struct neo_apple_dcp *neo_dcp, - struct neo_iomfb_transaction *transaction); -static void neo_iomfb_opaque_x_release(struct neo_iomfb_transaction *transaction); - -static int neo_iomfb_enqueue_opaque_x(struct neo_apple_dcp *neo_dcp) -{ - struct neo_iomfb_opaque_x_transaction *opaque; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - if (atomic_read(&neo_dcp->neo_iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) - return 0; - if (neo_dcp->neo_iomfb.queued >= 32) - return -EBUSY; - opaque = kzalloc_obj(*opaque); - if (!opaque) - return -ENOMEM; - if (atomic_cmpxchg(&neo_dcp->neo_iomfb.opaque_x_state, IOMFB_OPAQUE_X_NEEDED, - IOMFB_OPAQUE_X_QUEUED) != IOMFB_OPAQUE_X_NEEDED) { - kfree(opaque); - return 0; - } - - opaque->transaction.start = neo_iomfb_opaque_x_start; - opaque->transaction.release = neo_iomfb_opaque_x_release; - neo_dcp->neo_iomfb.queued++; - list_add(&opaque->transaction.link, &neo_dcp->neo_iomfb.pending); - return 0; -} - -static void neo_iomfb_discard_pending(struct neo_apple_dcp *neo_dcp) -{ - struct neo_iomfb_transaction *transaction, *next; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - list_for_each_entry_safe(transaction, next, &neo_dcp->neo_iomfb.pending, link) { - list_del(&transaction->link); - neo_dcp->neo_iomfb.queued--; - if (transaction->brightness_only) - neo_dcp->neo_iomfb.backlight_queued = false; - transaction->release(transaction); - } -} - -static void neo_iomfb_queue_advance(struct neo_apple_dcp *neo_dcp) -{ - struct neo_iomfb_transaction *transaction = neo_dcp->neo_iomfb.active; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - if (READ_ONCE(neo_dcp->crashed) || neo_dcp->neo_iomfb.stopped) { - neo_iomfb_discard_pending(neo_dcp); - /* The active operation may still be visible to firmware. */ - return; - } - if (!neo_iomfb_channels_idle(neo_dcp) || neo_dcp->present_state_h17p.pending) - return; - - if (transaction) { - neo_dcp->neo_iomfb.active = NULL; - cancel_delayed_work(&neo_dcp->neo_iomfb.timeout); - if (transaction->brightness_only) - neo_dcp->neo_iomfb.backlight_queued = false; - /* A new level requested during a successful present stays pending. */ - if (neo_dcp_backlight_pending(neo_dcp)) { - if (transaction->completed) { - schedule_work(&neo_dcp->bl_update_wq); - } else if (transaction->backlight_failed) { - unsigned int delay = neo_dcp_backlight_retry_delay(neo_dcp); - - if (delay) - mod_delayed_work(system_wq, &neo_dcp->neo_iomfb.backlight_retry, - msecs_to_jiffies(delay)); - else - dev_warn_ratelimited(neo_dcp->dev, - "backlight retry limit reached\n"); - } - } - transaction->release(transaction); - } - if (neo_iomfb_enqueue_opaque_x(neo_dcp)) { - WRITE_ONCE(neo_dcp->crashed, true); - neo_iomfb_discard_pending(neo_dcp); - schedule_work(&neo_dcp->vblank_wq); - return; - } - if (!list_empty(&neo_dcp->neo_iomfb.pending)) - schedule_work(&neo_dcp->neo_iomfb.work); -} - -static void neo_iomfb_queue_work(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = container_of(work, struct neo_apple_dcp, neo_iomfb.work); - struct neo_iomfb_transaction *transaction; - - mutex_lock(&neo_dcp->neo_iomfb.lock); - if (READ_ONCE(neo_dcp->crashed) || neo_dcp->neo_iomfb.stopped) { - neo_iomfb_discard_pending(neo_dcp); - goto unlock; - } - if (neo_dcp->neo_iomfb.active || !neo_iomfb_channels_idle(neo_dcp) || - neo_dcp->present_state_h17p.pending || list_empty(&neo_dcp->neo_iomfb.pending)) - goto unlock; - - transaction = list_first_entry(&neo_dcp->neo_iomfb.pending, - struct neo_iomfb_transaction, link); - list_del(&transaction->link); - neo_dcp->neo_iomfb.queued--; - neo_dcp->neo_iomfb.active = transaction; - WRITE_ONCE(neo_dcp->neo_iomfb.owner, current); - neo_dcp->neo_iomfb.deadline = jiffies + msecs_to_jiffies(10000); - mod_delayed_work(system_wq, &neo_dcp->neo_iomfb.timeout, msecs_to_jiffies(10000)); - transaction->start(neo_dcp, transaction); - WRITE_ONCE(neo_dcp->neo_iomfb.owner, NULL); - neo_iomfb_queue_advance(neo_dcp); -unlock: - mutex_unlock(&neo_dcp->neo_iomfb.lock); -} - -static void neo_iomfb_queue_timeout(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = container_of(to_delayed_work(work), - struct neo_apple_dcp, neo_iomfb.timeout); - - mutex_lock(&neo_dcp->neo_iomfb.lock); - if (neo_dcp->neo_iomfb.active && time_before(jiffies, neo_dcp->neo_iomfb.deadline)) { - mod_delayed_work(system_wq, &neo_dcp->neo_iomfb.timeout, - neo_dcp->neo_iomfb.deadline - jiffies); - } else if (neo_dcp->neo_iomfb.active) { - WRITE_ONCE(neo_dcp->crashed, true); - dev_err(neo_dcp->dev, "IOMFB transaction timed out\n"); - neo_iomfb_discard_pending(neo_dcp); - schedule_work(&neo_dcp->vblank_wq); - } - mutex_unlock(&neo_dcp->neo_iomfb.lock); -} - -static void neo_iomfb_opaque_x_complete(struct neo_apple_dcp *neo_dcp, void *out, - void *cookie) -{ - u32 status = out ? *(u32 *)out : ~0U; - - (void)cookie; - - if (status) { - dev_err(neo_dcp->dev, "opaque X property failed: %u\n", status); - WRITE_ONCE(neo_dcp->crashed, true); - schedule_work(&neo_dcp->vblank_wq); - return; - } - - atomic_set(&neo_dcp->neo_iomfb.opaque_x_state, IOMFB_OPAQUE_X_READY); -} - -static void neo_iomfb_opaque_x_start(struct neo_apple_dcp *neo_dcp, - struct neo_iomfb_transaction *transaction) -{ - (void)transaction; - neo_iomfb_apply_opaque_x_h17p(neo_dcp, neo_iomfb_opaque_x_complete, NULL); -} - -static void neo_iomfb_opaque_x_release(struct neo_iomfb_transaction *transaction) -{ - kfree(container_of(transaction, struct neo_iomfb_opaque_x_transaction, - transaction)); -} - -void neo_iomfb_opaque_x_reset_h17p(struct neo_apple_dcp *neo_dcp) -{ - atomic_set(&neo_dcp->neo_iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); -} - -static void neo_iomfb_backlight_retry(struct work_struct *work) -{ - struct neo_apple_dcp *neo_dcp = container_of(to_delayed_work(work), struct neo_apple_dcp, - neo_iomfb.backlight_retry); - - if (!READ_ONCE(neo_dcp->crashed) && !READ_ONCE(neo_dcp->neo_iomfb.stopped) && - neo_dcp_backlight_pending(neo_dcp)) - schedule_work(&neo_dcp->bl_update_wq); -} - -void neo_iomfb_queue_init(struct neo_apple_dcp *neo_dcp) -{ - mutex_init(&neo_dcp->neo_iomfb.lock); - INIT_LIST_HEAD(&neo_dcp->neo_iomfb.pending); - INIT_WORK(&neo_dcp->neo_iomfb.work, neo_iomfb_queue_work); - INIT_DELAYED_WORK(&neo_dcp->neo_iomfb.timeout, neo_iomfb_queue_timeout); - INIT_DELAYED_WORK(&neo_dcp->neo_iomfb.backlight_retry, neo_iomfb_backlight_retry); - atomic_set(&neo_dcp->neo_iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); -} - -void neo_iomfb_queue_stop(struct neo_apple_dcp *neo_dcp) -{ - if (!neo_iomfb_uses_queue(neo_dcp)) - return; - - mutex_lock(&neo_dcp->neo_iomfb.lock); - neo_dcp->neo_iomfb.stopped = true; - neo_iomfb_discard_pending(neo_dcp); - mutex_unlock(&neo_dcp->neo_iomfb.lock); - cancel_work_sync(&neo_dcp->neo_iomfb.work); - cancel_delayed_work_sync(&neo_dcp->neo_iomfb.timeout); - cancel_delayed_work_sync(&neo_dcp->neo_iomfb.backlight_retry); -} - -/* - * Wait until every queued transaction, including a present's completion, has - * finished. A stopped or crashed queue has nothing left to deliver. - */ -bool neo_iomfb_queue_drain(struct neo_apple_dcp *neo_dcp, unsigned long timeout) -{ - unsigned long deadline = jiffies + timeout; - bool idle; - - if (!neo_iomfb_uses_queue(neo_dcp)) - return true; - - for (;;) { - mutex_lock(&neo_dcp->neo_iomfb.lock); - idle = READ_ONCE(neo_dcp->crashed) || neo_dcp->neo_iomfb.stopped || - (!neo_dcp->neo_iomfb.active && list_empty(&neo_dcp->neo_iomfb.pending) && - !neo_dcp->present_state_h17p.pending); - mutex_unlock(&neo_dcp->neo_iomfb.lock); - if (idle) - return true; - if (time_after(jiffies, deadline)) - return false; - usleep_range(2000, 4000); - } -} - -int neo_iomfb_queue(struct neo_apple_dcp *neo_dcp, struct neo_iomfb_transaction *transaction) -{ - int ret = 0; - - mutex_lock(&neo_dcp->neo_iomfb.lock); - if (READ_ONCE(neo_dcp->crashed) || neo_dcp->neo_iomfb.stopped) { - ret = -EIO; - } else if (transaction->brightness_only && neo_dcp->neo_iomfb.backlight_queued) { - ret = -EALREADY; - } else if (neo_dcp->neo_iomfb.queued >= 32) { - ret = -EBUSY; - } else { - if (transaction->brightness_only) - neo_dcp->neo_iomfb.backlight_queued = true; - neo_dcp->neo_iomfb.queued++; - list_add_tail(&transaction->link, &neo_dcp->neo_iomfb.pending); - schedule_work(&neo_dcp->neo_iomfb.work); - } - mutex_unlock(&neo_dcp->neo_iomfb.lock); - return ret; -} - -struct neo_iomfb_crc_transaction { - struct neo_iomfb_transaction transaction; - u32 swap_id; -}; - -static void neo_iomfb_crc_complete(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_iomfb_crc_transaction *crc = cookie; - struct neo_dcp_packet_header *header; - u32 value; - - /* A106 has no input and a single little-endian blend-output CRC. */ - if (!out) - goto invalid; - header = out - sizeof(*header); - if (header->in_len || header->out_len != sizeof(__le32)) - goto invalid; - value = get_unaligned_le32(out); - if (READ_ONCE(neo_dcp->crc_enabled)) - drm_crtc_add_crc_entry(&neo_dcp->crtc->base, true, crc->swap_id, &value); - return; - -invalid: - dev_err(neo_dcp->dev, "invalid blend CRC response\n"); - WRITE_ONCE(neo_dcp->crashed, true); -} - -static void neo_iomfb_crc_start(struct neo_apple_dcp *neo_dcp, - struct neo_iomfb_transaction *transaction) -{ - static const struct neo_dcp_method_entry method = { - .tag = { 'A', '1', '0', '6' }, - .name = "read_blend_crc", - }; - struct neo_iomfb_crc_transaction *crc = container_of(transaction, - struct neo_iomfb_crc_transaction, - transaction); - - if (READ_ONCE(neo_dcp->crc_enabled)) - neo_dcp_push(neo_dcp, false, &method, 0, sizeof(__le32), NULL, - neo_iomfb_crc_complete, crc); -} - -static void neo_iomfb_crc_release(struct neo_iomfb_transaction *transaction) -{ - kfree(container_of(transaction, struct neo_iomfb_crc_transaction, transaction)); -} - -void neo_iomfb_queue_crc_h17p(struct neo_apple_dcp *neo_dcp, u32 swap_id) -{ - struct neo_iomfb_crc_transaction *crc; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - if (!READ_ONCE(neo_dcp->crc_enabled) || READ_ONCE(neo_dcp->crashed) || - neo_dcp->neo_iomfb.stopped || neo_dcp->neo_iomfb.queued >= 32) - return; - crc = kzalloc_obj(*crc); - if (!crc) - return; - crc->swap_id = swap_id; - crc->transaction.start = neo_iomfb_crc_start; - crc->transaction.release = neo_iomfb_crc_release; - /* Read the completed frame before another queued present changes it. */ - list_add(&crc->transaction.link, &neo_dcp->neo_iomfb.pending); - neo_dcp->neo_iomfb.queued++; - /* The receiver advances the queue only after acknowledging completion. */ -} - -struct neo_iomfb_command { - struct neo_iomfb_transaction transaction; - struct neo_dcp_method_entry method; - neo_dcp_callback_t callback; - void *cookie; - u32 in_len; - u32 out_len; - bool oob; - u8 data[]; -}; - -static void neo_iomfb_command_start(struct neo_apple_dcp *neo_dcp, - struct neo_iomfb_transaction *transaction) -{ - struct neo_iomfb_command *command = container_of(transaction, - struct neo_iomfb_command, transaction); - - neo_dcp_push(neo_dcp, command->oob, &command->method, command->in_len, - command->out_len, command->data, command->callback, command->cookie); -} - -static void neo_iomfb_command_release(struct neo_iomfb_transaction *transaction) -{ - kfree(container_of(transaction, struct neo_iomfb_command, transaction)); -} - -static int neo_iomfb_queue_command(struct neo_apple_dcp *neo_dcp, bool oob, - const struct neo_dcp_method_entry *method, - u32 in_len, u32 out_len, void *data, - neo_dcp_callback_t callback, void *cookie) -{ - struct neo_iomfb_command *command; - int ret; - - if ((u64)sizeof(struct neo_dcp_packet_header) + in_len + out_len > 0x8000) - return -EMSGSIZE; - command = kzalloc(struct_size(command, data, in_len), GFP_KERNEL); - if (!command) - return -ENOMEM; - command->transaction.start = neo_iomfb_command_start; - command->transaction.release = neo_iomfb_command_release; - command->method = *method; - command->callback = callback; - command->cookie = cookie; - command->in_len = in_len; - command->out_len = out_len; - command->oob = oob; - if (in_len) - memcpy(command->data, data, in_len); - ret = neo_iomfb_queue(neo_dcp, &command->transaction); - if (ret) - neo_iomfb_command_release(&command->transaction); - return ret; -} - -/* Call a DCP function given by a tag */ -void neo_dcp_push(struct neo_apple_dcp *neo_dcp, bool oob, const struct neo_dcp_method_entry *call, - u32 in_len, u32 out_len, void *data, neo_dcp_callback_t cb, - void *cookie) -{ - struct neo_dcp_method_entry resolved = *call; - - /* Reply chains stay on the serialized receiver; callers enqueue copies. */ - if (neo_iomfb_uses_queue(neo_dcp) && READ_ONCE(neo_dcp->neo_iomfb.owner) != current) { - if (neo_iomfb_queue_command(neo_dcp, oob, call, in_len, out_len, - data, cb, cookie)) - WRITE_ONCE(neo_dcp->crashed, true); - return; - } - - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(neo_dcp->crashed)) - return; - - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G && - call->tag_h17g[0]) - memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); - call = &resolved; - - enum neo_dcp_context_id context = neo_dcp_call_context(neo_dcp, oob); - struct neo_dcp_channel *ch = neo_dcp_get_channel(neo_dcp, context); - - struct neo_dcp_packet_header header = { - .in_len = in_len, - .out_len = out_len, - - /* Tag is reversed due to endianness of the fourcc */ - .tag[0] = call->tag[3], - .tag[1] = call->tag[2], - .tag[2] = call->tag[1], - .tag[3] = call->tag[0], - }; - - u8 depth; - u16 offset; - - if (neo_iomfb_uses_queue(neo_dcp) && - (!ch || ch->depth >= DCP_MAX_CALL_DEPTH || - (u64)sizeof(header) + in_len + out_len > 0x8000 || - neo_dcp_packet_start(ch, ch->depth) > - 0x8000 - ALIGN(sizeof(header) + in_len + out_len, - DCP_PACKET_ALIGNMENT))) { - dev_err(neo_dcp->dev, "invalid IOMFB command envelope\n"); - WRITE_ONCE(neo_dcp->crashed, true); - return; - } - depth = neo_dcp_push_depth(&ch->depth); - offset = neo_dcp_packet_start(ch, depth); - - void *out = neo_dcp->shmem + neo_dcp_tx_offset(context) + offset; - void *out_data = out + sizeof(header); - size_t data_len = sizeof(header) + in_len + out_len; - - memcpy(out, &header, sizeof(header)); - - if (in_len > 0) - memcpy(out_data, data, in_len); - /* An unwritten status must not look like a successful response. */ - if (out_len) - memset(out_data + in_len, 0xff, out_len); - - trace_neo_iomfb_push(neo_dcp, call, context, offset, depth); - - ch->callbacks[depth] = cb; - ch->cookies[depth] = cookie; - ch->output[depth] = out + sizeof(header) + in_len; - ch->header[depth] = header; - ch->end[depth] = offset + ALIGN(data_len, DCP_PACKET_ALIGNMENT); - - neo_dcp_send_message(neo_dcp, IOMFB_ENDPOINT, - dcpep_msg(context, data_len, offset)); -} - -/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ -int neo_dcp_parse_tag(char tag[4]) -{ - u32 d[3]; - int i; - - if (tag[3] != 'D') - return -EINVAL; - - for (i = 0; i < 3; ++i) { - d[i] = (u32)(tag[i] - '0'); - - if (d[i] > 9) - return -EINVAL; - } - - return d[0] + (d[1] * 10) + (d[2] * 100); -} - -/* Ack a callback from the DCP */ -void neo_dcp_ack(struct neo_apple_dcp *neo_dcp, enum neo_dcp_context_id context) -{ - struct neo_dcp_channel *ch = neo_dcp_get_channel(neo_dcp, context); - - neo_dcp_pop_depth(&ch->depth); - neo_dcp_send_message(neo_dcp, IOMFB_ENDPOINT, - dcpep_ack(context)); -} - -/* - * Helper to send a DRM hotplug event. The DCP is accessed from a single - * (RTKit) thread. To handle hotplug callbacks, we need to call - * drm_kms_helper_hotplug_event, which does an atomic commit (via DCP) and - * waits for vblank (a DCP callback). That means we deadlock if we call from - * the RTKit thread! Instead, move the call to another thread via a workqueue. - */ -static int neo_dcp_retrain_active_crtc(struct neo_apple_connector *connector) -{ - struct drm_device *dev = connector->base.dev; - struct drm_modeset_acquire_ctx ctx; - struct drm_crtc *crtc; - int ret; - - DRM_MODESET_LOCK_ALL_BEGIN(dev, ctx, 0, ret); - - crtc = connector->base.state ? connector->base.state->crtc : NULL; - if (crtc && crtc->state && crtc->state->active) - ret = drm_atomic_helper_reset_crtc(crtc, &ctx); - else - ret = 0; - - DRM_MODESET_LOCK_ALL_END(dev, ctx, ret); - - return ret; -} - -void neo_dcp_handle_hotplug_actions(struct neo_apple_dcp *neo_dcp, unsigned int action) -{ - if (action & DCP_HOTPLUG_VBLANK) - schedule_work(&neo_dcp->vblank_wq); - if ((action & DCP_HOTPLUG_NOTIFY) && neo_dcp->connector) - schedule_work(&neo_dcp->connector->hotplug_wq); -} - -void neo_dcp_retrain_oob(struct neo_apple_connector *connector) -{ - struct platform_device *pdev = READ_ONCE(connector->neo_dcp); - struct neo_apple_dcp *neo_dcp; - - /* a Type-C port with no pipeline behind it has nothing to retrain */ - if (!pdev || !READ_ONCE(connector->connected)) - return; - neo_dcp = platform_get_drvdata(pdev); - - /* - * Bringing up another high-speed Type-C route can disturb an active DPTX - * stream without changing its HPD state. Invalidate the IOMFB mode and - * use the normal hotplug worker to replay the active CRTC from process - * context; sending a synthetic disconnect would tear down the connector. - */ - neo_dcp_mode_invalidate(&neo_dcp->mode_state); - schedule_work(&connector->hotplug_wq); -} - -void neo_dcp_hotplug(struct work_struct *work) -{ - struct neo_apple_connector *connector; - struct platform_device *pdev; - struct neo_apple_dcp *neo_dcp; - int ret; - - connector = container_of(work, struct neo_apple_connector, hotplug_wq); - - pdev = READ_ONCE(connector->neo_dcp); - if (!pdev) { /* a Type-C port unrouted after this was queued */ - drm_kms_helper_connector_hotplug_event(&connector->base); - return; - } - neo_dcp = platform_get_drvdata(pdev); - dev_info(neo_dcp->dev, "%s() connected:%d valid_mode:%d nr_modes:%u\n", __func__, - connector->connected, READ_ONCE(neo_dcp->mode_state.valid), neo_dcp->nr_modes); - - if (!connector->connected) { - drm_edid_free(connector->drm_edid); - connector->drm_edid = NULL; - } - - /* - * DCP defers link training until we set a display mode. But we set - * display modes from atomic_flush, so userspace needs to trigger a - * flush, or the CRTC gets no signal. - */ - if (connector->base.state && !READ_ONCE(neo_dcp->mode_state.valid) && connector->connected && - !(neo_dcp_is_usb4_output(neo_dcp) && false)) { - drm_connector_set_link_status_property(&connector->base, - DRM_MODE_LINK_STATUS_BAD); - - /* - * A short Type-C route interruption can leave the DRM CRTC active - * while DCP has discarded its display mode. Userspace is then free - * to keep submitting plane-only commits, none of which retrains the - * link. Re-apply the active CRTC state once the DPTX link-config - * callback has completed. - */ - ret = neo_dcp_retrain_active_crtc(connector); - if (ret) - dev_warn(neo_dcp->dev, - "failed to retrain active CRTC after hotplug: %d\n", - ret); - } - - drm_kms_helper_connector_hotplug_event(&connector->base); -} - -static void dcpep_handle_cb(struct neo_apple_dcp *neo_dcp, enum neo_dcp_context_id context, - void *data, u32 length, u16 offset) -{ - struct device *dev = neo_dcp->dev; - struct neo_dcp_packet_header *hdr = data; - void *in, *out; - int tag; - struct neo_dcp_channel *ch = neo_dcp_get_channel(neo_dcp, context); - u8 depth; - bool handled; - - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || - offset >= 0x8000 || length > 0x8000 - offset)) { - dev_err(dev, "invalid IOMFB callback envelope\n"); - neo_dcp->crashed = true; - return; - } - - tag = neo_dcp_parse_tag(hdr->tag); - handled = tag >= 0 && tag < IOMFB_MAX_CB && neo_dcp->cb_handlers && - neo_dcp->cb_handlers[tag]; - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - (tag < 0 || tag >= IOMFB_MAX_CB || - (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || - (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - handled && - !neo_iomfb_check_callback_h17p(neo_dcp, tag, hdr->in_len, hdr->out_len)))) { - dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", - tag, hdr->in_len, hdr->out_len); - neo_dcp->crashed = true; - return; - } - - in = data + sizeof(*hdr); - out = in + hdr->in_len; - - if (!handled) { - if (!neo_iomfb_uses_queue(neo_dcp) || - !test_and_set_bit(tag, neo_dcp->unknown_callbacks)) - dev_warn(dev, "received unknown callback %c%c%c%c\n", - hdr->tag[3], hdr->tag[2], hdr->tag[1], hdr->tag[0]); - /* - * Leaving a callback unanswered wedges the coprocessor: it - * waits for the ack forever and the outer call never returns. - * On H17P answer it with a zeroed output instead. - */ - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P) { - if (hdr->out_len) - memset(out, 0, hdr->out_len); - depth = neo_dcp_push_depth(&ch->depth); - ch->output[depth] = out; - ch->end[depth] = offset + - ALIGN(length, DCP_PACKET_ALIGNMENT); - neo_dcp_ack(neo_dcp, context); - } - return; - } - - // TODO: verify that in_len and out_len match our prototypes - // for now just clear the out data to have at least consistent results - if (hdr->out_len) - memset(out, 0, hdr->out_len); - - depth = neo_dcp_push_depth(&ch->depth); - ch->output[depth] = out; - ch->end[depth] = offset + ALIGN(length, DCP_PACKET_ALIGNMENT); - - if (neo_dcp->cb_handlers[tag](neo_dcp, tag, out, in)) - neo_dcp_ack(neo_dcp, context); -} - -static void dcpep_handle_ack(struct neo_apple_dcp *neo_dcp, enum neo_dcp_context_id context) -{ - struct neo_dcp_channel *ch = neo_dcp_get_channel(neo_dcp, context); - const struct neo_dcp_packet_header *record, *sent; - void *cookie, *out; - neo_dcp_callback_t cb; - - if (!ch) { - dev_warn(neo_dcp->dev, "ignoring ack on context %X\n", context); - return; - } - - if (!ch->depth) { - dev_warn(neo_dcp->dev, "ignoring ack on idle context %X\n", context); - return; - } - - neo_dcp_pop_depth(&ch->depth); - - cb = ch->callbacks[ch->depth]; - cookie = ch->cookies[ch->depth]; - out = ch->output[ch->depth]; - sent = &ch->header[ch->depth]; - - ch->callbacks[ch->depth] = NULL; - ch->cookies[ch->depth] = NULL; - ch->output[ch->depth] = NULL; - - /* - * An ack completes the innermost pending command on its context. The - * firmware writes the reply in place into that AP command record, whose - * address dcp_push() saved; the ack itself carries neither offset nor - * length on any firmware. A command sent from within a callback is - * acked on the callback context, whose receive area holds the callback - * record rather than the reply, so the ack cannot be used to find it. - * - * On older firmware the lengths in the record header must still be - * the ones that were sent; otherwise the record was overwritten and - * its output cannot be trusted. The tag is left out of the check: it - * has only been seen preserved on 13.5 firmware. - */ - if (neo_dcp->fw_compat != DCP_FIRMWARE_H17P) { - record = out - sent->in_len - sizeof(*record); - if (record->in_len != sent->in_len || - record->out_len != sent->out_len) { - dev_err(neo_dcp->dev, - "corrupt %c%c%c%c command record on context %X\n", - sent->tag[3], sent->tag[2], sent->tag[1], - sent->tag[0], context); - WRITE_ONCE(neo_dcp->crashed, true); - /* - * Still complete the command, without an output, so - * that its callback drops the cookie reference and - * wakes the waiter instead of leaving it to time out. - */ - out = NULL; - } - } - - if (cb) - cb(neo_dcp, out, cookie); -} - -static void dcpep_got_msg(struct neo_apple_dcp *neo_dcp, u64 message) -{ - enum neo_dcp_context_id ctx_id; - u16 offset; - u32 length; - int channel_offset; - void *data; - - ctx_id = FIELD_GET(IOMFB_MSG_CONTEXT, message); - offset = FIELD_GET(IOMFB_MSG_OFFSET, message); - length = FIELD_GET(IOMFB_MSG_LENGTH, message); - - channel_offset = neo_dcp_channel_offset(ctx_id); - - if (channel_offset < 0) { - dev_warn(neo_dcp->dev, "invalid context received %u\n", ctx_id); - return; - } - - data = neo_dcp->shmem + channel_offset + offset; - - if (FIELD_GET(IOMFB_MSG_ACK, message)) - dcpep_handle_ack(neo_dcp, ctx_id); - else - dcpep_handle_cb(neo_dcp, ctx_id, data, length, offset); -} - -int neo_dcp_get_modes(struct drm_connector *connector) -{ - struct neo_apple_connector *neo_apple_connector = to_apple_connector(connector); - struct platform_device *pdev = READ_ONCE(neo_apple_connector->neo_dcp); - struct neo_apple_dcp *neo_dcp; - - struct drm_device *dev = connector->dev; - struct drm_display_mode *mode; - u16 min_vfreq = 0, max_vfreq = 0; - bool vrr_capable = false; - int i; - - /* A Type-C port has no pipeline while the fabric is moving it. */ - if (!pdev) - return 0; - neo_dcp = platform_get_drvdata(pdev); - - for (i = 0; i < neo_dcp->nr_modes; ++i) { - if (neo_dcp->modes[i].vrr) { - u16 lo = neo_dcp->modes[i].min_vrr >> 16; - u16 hi = neo_dcp->modes[i].max_vrr >> 16; - - if (!min_vfreq || lo < min_vfreq) - min_vfreq = lo; - if (hi > max_vfreq) - max_vfreq = hi; - } - vrr_capable |= neo_dcp->modes[i].vrr; - mode = drm_mode_duplicate(dev, &neo_dcp->modes[i].mode); - - if (!mode) { - dev_err(dev->dev, "Failed to duplicate display mode\n"); - return 0; - } - - drm_mode_probed_add(connector, mode); - } - drm_connector_set_vrr_capable_property(connector, vrr_capable); - - /* H17P sends no EPIC commands; see afk_send_epic(). */ - if (neo_dcp->nr_modes && neo_dcp->fw_compat != DCP_FIRMWARE_H17P && - !neo_apple_connector->drm_edid) { - const struct drm_edid *edid; - edid = neo_dcpavserv_copy_edid(neo_dcp); - if (IS_ERR_OR_NULL(edid)) { - /* - * An internal panel has no AV service and so no EDID; - * do not report that on every mode probe. - */ - if (neo_dcp_has_panel(neo_dcp) && PTR_ERR(edid) == -ENODEV) - dev_dbg(neo_dcp->dev, "no EDID source for the panel\n"); - else - dev_info(neo_dcp->dev, "copy_edid failed: %pe\n", edid); - } else { - drm_edid_free(neo_apple_connector->drm_edid); - neo_apple_connector->drm_edid = edid; - } - } - if (neo_dcp->nr_modes && neo_apple_connector->drm_edid) { - drm_edid_connector_update(connector, neo_apple_connector->drm_edid); - neo_dcp_retry_placeholder_edid(neo_dcp, neo_apple_connector->drm_edid); - } - - /* - * An internal panel has no EDID, so nothing fills in the refresh range - * that userspace needs before it will drive VRR. Supply the range DCP - * reported for the mode. This has to follow the EDID update, which - * resets display_info. - */ - if (vrr_capable && max_vfreq && - !connector->display_info.monitor_range.max_vfreq) { - connector->display_info.monitor_range.min_vfreq = min_vfreq; - connector->display_info.monitor_range.max_vfreq = max_vfreq; - } - - return neo_dcp->nr_modes; -} - -/* The user may own drm_display_mode, so we need to search for our copy */ -struct neo_dcp_display_mode *neo_lookup_mode(struct neo_apple_dcp *neo_dcp, - const struct drm_display_mode *mode) -{ - int i; - - for (i = 0; i < neo_dcp->nr_modes; ++i) { - if (drm_mode_match(mode, &neo_dcp->modes[i].mode, - DRM_MODE_MATCH_TIMINGS | - DRM_MODE_MATCH_CLOCK)) - return &neo_dcp->modes[i]; - } - - return NULL; -} - -/* - * H17P keeps the mode the bootloader programmed and never sends - * set_digital_out_mode (see DCP_INHERIT_BOOT_MODE in iomfb_template.c), so a - * modeset to any other mode would silently not be applied. The bootloader - * brings the panel up in its native timing, which is the mode the firmware - * scores highest and enumerate_modes() marks preferred; offer only that one. - */ -static bool neo_dcp_mode_settable(struct neo_apple_dcp *neo_dcp, - const struct drm_display_mode *mode) -{ - struct neo_dcp_display_mode *neo_dcp_mode = neo_lookup_mode(neo_dcp, mode); - - if (!neo_dcp_mode) - return false; - - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P) - return neo_dcp_mode->mode.type & DRM_MODE_TYPE_PREFERRED; - - return true; -} - -enum drm_mode_status neo_dcp_mode_valid(struct drm_connector *connector, - const struct drm_display_mode *mode) -{ - struct neo_apple_connector *neo_apple_connector = to_apple_connector(connector); - struct platform_device *pdev = READ_ONCE(neo_apple_connector->neo_dcp); - struct neo_apple_dcp *neo_dcp; - - if (!pdev) - return MODE_ERROR; - neo_dcp = platform_get_drvdata(pdev); - - return neo_dcp_mode_settable(neo_dcp, mode) ? MODE_OK : MODE_BAD; -} - -int neo_dcp_crtc_atomic_modeset(struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - struct neo_apple_crtc *neo_apple_crtc = to_apple_crtc(crtc); - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(neo_apple_crtc->neo_dcp); - struct drm_crtc_state *crtc_state; - unsigned int action; - int ret = -EIO; - bool modeset; - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - if (!crtc_state) - return 0; - - modeset = drm_atomic_crtc_needs_modeset(crtc_state) || - !READ_ONCE(neo_dcp->mode_state.valid); - - if (!modeset) - return 0; - - /* ignore no mode, poweroff is handled elsewhere */ - if (crtc_state->mode.hdisplay == 0 && crtc_state->mode.vdisplay == 0) - return 0; - - neo_dcp_mode_begin(&neo_dcp->mode_state); - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - ret = neo_iomfb_modeset_v12_3(neo_dcp, crtc_state); - break; - case DCP_FIRMWARE_V_13_5: - ret = neo_iomfb_modeset_v13_3(neo_dcp, crtc_state); - break; - case DCP_FIRMWARE_H17P: - ret = neo_iomfb_modeset_h17p(neo_dcp, crtc_state); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", - neo_dcp->fw_compat); - break; - } - - action = neo_dcp_mode_finish(&neo_dcp->mode_state, !ret, neo_dcp->connector ? - &neo_dcp->connector->connected : NULL); - neo_dcp_handle_hotplug_actions(neo_dcp, action); - - return ret; -} - -bool neo_dcp_crtc_mode_fixup(struct drm_crtc *crtc, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) -{ - struct neo_apple_crtc *neo_apple_crtc = to_apple_crtc(crtc); - struct platform_device *pdev = neo_apple_crtc->neo_dcp; - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - /* TODO: support synthesized modes through scaling */ - return neo_dcp_mode_settable(neo_dcp, mode); -} - - -static void neo_iomfb_scanout_release_h17p(struct neo_iomfb_scanout_h17p *scanout) -{ - unsigned int i; - - if (!scanout) - return; - for (i = 0; i < SWAP_SURFACES; i++) - if (scanout->fb[i]) - drm_framebuffer_put(scanout->fb[i]); - kfree(scanout); -} - -static struct neo_iomfb_scanout_h17p * -neo_iomfb_scanout_prepare_h17p(struct neo_apple_dcp *neo_dcp, struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - struct neo_iomfb_scanout_h17p *scanout; - struct drm_plane *plane; - struct drm_plane_state *old_state, *new_state; - unsigned int i, slot; - int index; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - scanout = kzalloc_obj(*scanout); - if (!scanout) - return NULL; - if (neo_dcp->neo_iomfb.scanout) { - scanout->request = neo_dcp->neo_iomfb.scanout->request; - for (i = 0; i < SWAP_SURFACES; i++) { - scanout->fb[i] = neo_dcp->neo_iomfb.scanout->fb[i]; - if (scanout->fb[i]) - drm_framebuffer_get(scanout->fb[i]); - } - } - - for_each_oldnew_plane_in_state(state, plane, old_state, new_state, index) { - if (old_state->crtc != crtc && new_state->crtc != crtc) - continue; - slot = to_apple_plane(plane)->neo_iomfb_surf; - if (slot >= SWAP_SURFACES) { - neo_iomfb_scanout_release_h17p(scanout); - return NULL; - } - if (scanout->fb[slot]) - drm_framebuffer_put(scanout->fb[slot]); - scanout->fb[slot] = NULL; - if (new_state->crtc == crtc && new_state->visible && new_state->fb) { - scanout->fb[slot] = new_state->fb; - drm_framebuffer_get(new_state->fb); - } - } - return scanout; -} - -void neo_iomfb_scanout_complete_h17p(struct neo_apple_dcp *neo_dcp) -{ - struct neo_iomfb_scanout_h17p *scanout = neo_dcp->neo_iomfb.next_scanout; - struct neo_iomfb_scanout_h17p *previous = neo_dcp->neo_iomfb.scanout; - struct neo_dcp_swap_submit_req_h17p *request = &neo_dcp->swap.h17p; - unsigned int i; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - if (!scanout) - return; - - /* Keep common present arguments, including the null output pointers. */ - scanout->request = *request; - /* Preserve unchanged planes when only a subset was presented. */ - for (i = 0; i < SWAP_SURFACES; i++) { - if (!(request->swap.swap_enabled & BIT(i)) && previous) { - scanout->request.surf[i] = previous->request.surf[i]; - scanout->request.surf_iova[i] = previous->request.surf_iova[i]; - scanout->request.swap.src_rect[i] = previous->request.swap.src_rect[i]; - scanout->request.swap.dst_rect[i] = previous->request.swap.dst_rect[i]; - scanout->request.swap.surf_ids[i] = previous->request.swap.surf_ids[i]; - scanout->request.swap.surf_flags[i] = previous->request.swap.surf_flags[i]; - scanout->request.swap.surf_unk[i] = previous->request.swap.surf_unk[i]; - } - scanout->request.surf_null[i] = !scanout->fb[i]; - } - /* Brightness re-presents all pinned surfaces with a fresh swap ID. */ - /* Replay only a background established by a completed present. */ - scanout->request.swap.swap_enabled = - request->swap.swap_enabled & IOMFB_SET_BACKGROUND; - if (previous) - scanout->request.swap.swap_enabled |= - previous->request.swap.swap_enabled & IOMFB_SET_BACKGROUND; - for (i = 0; i < SWAP_SURFACES; i++) - if (scanout->fb[i]) - scanout->request.swap.swap_enabled |= BIT(i); - scanout->request.swap.swap_completed = scanout->request.swap.swap_enabled; - if (!(request->swap.swap_enabled & IOMFB_SET_BACKGROUND) && previous) - scanout->request.swap.bg_color = previous->request.swap.bg_color; - neo_dcp->neo_iomfb.scanout = scanout; - neo_dcp->neo_iomfb.next_scanout = NULL; - neo_iomfb_scanout_release_h17p(previous); -} - -bool neo_iomfb_present_brightness_only_h17p(struct neo_apple_dcp *neo_dcp) -{ - return neo_dcp->neo_iomfb.active && neo_dcp->neo_iomfb.active->brightness_only; -} - -bool neo_iomfb_apply_backlight_h17p(struct neo_apple_dcp *neo_dcp, - const struct neo_dcp_swap_submit_req_h17p *request, - struct neo_dcp_present_h17p *wire) -{ - struct neo_iomfb_transaction *transaction = neo_dcp->neo_iomfb.active; - struct neo_dcp_backlight_present present; - bool have_surface = false; - unsigned int i; - int ret; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - if (!neo_dcp_backlight_active(neo_dcp)) { - if (!neo_dcp_has_panel(neo_dcp)) - return true; - if (!neo_dcp->brightness.maximum) - return false; - neo_iomfb_encode_backlight_h17p(wire, 0, - neo_dcp->brightness.maximum, false); - return true; - } - if (!transaction || !neo_dcp->brightness.maximum) - return false; - for (i = 0; i < SWAP_SURFACES; i++) - have_surface |= !request->surf_null[i]; - ret = neo_dcp_backlight_prepare(neo_dcp, have_surface, &present); - if (ret && ret != -EALREADY) - return false; - if (!ret) { - transaction->backlight = present; - transaction->backlight_reserved = true; - } - neo_iomfb_encode_backlight_h17p(wire, present.nits, - neo_dcp->brightness.maximum, !ret); - return true; -} - -void neo_iomfb_present_failed_h17p(struct neo_apple_dcp *neo_dcp) -{ - struct neo_iomfb_transaction *transaction = neo_dcp->neo_iomfb.active; - - if (transaction && transaction->backlight_reserved) { - neo_dcp_backlight_complete(neo_dcp, transaction->backlight.sequence, false); - transaction->backlight_reserved = false; - transaction->backlight_failed = true; - } -} - -bool neo_iomfb_present_complete_h17p(struct neo_apple_dcp *neo_dcp) -{ - struct neo_iomfb_transaction *transaction = neo_dcp->neo_iomfb.active; - - lockdep_assert_held(&neo_dcp->neo_iomfb.lock); - if (transaction) { - transaction->completed = true; - if (transaction->backlight_reserved) { - neo_dcp_backlight_complete(neo_dcp, transaction->backlight.sequence, true); - transaction->backlight_reserved = false; - } - if (transaction->brightness_only) - return false; - } - atomic_cmpxchg(&neo_dcp->neo_iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, - IOMFB_OPAQUE_X_NEEDED); - neo_iomfb_scanout_complete_h17p(neo_dcp); - return true; -} - -static void neo_iomfb_backlight_start(struct neo_apple_dcp *neo_dcp, - struct neo_iomfb_transaction *transaction) -{ - struct neo_iomfb_scanout_h17p *scanout = neo_dcp->neo_iomfb.scanout; - bool have_surface = false; - unsigned int i; - - if (!neo_dcp_backlight_pending(neo_dcp) || !READ_ONCE(neo_dcp->mode_state.valid) || !scanout || - !neo_dcp->connector || !neo_dcp->connector->connected) - return; - for (i = 0; i < SWAP_SURFACES; i++) - have_surface |= !!scanout->fb[i]; - if (!have_surface) - return; - - neo_dcp->swap.h17p = scanout->request; - neo_iomfb_present_backlight_h17p(neo_dcp); -} - -static void neo_iomfb_backlight_release(struct neo_iomfb_transaction *transaction) -{ - kfree(transaction); -} - -static void neo_iomfb_backlight_kick(struct neo_apple_dcp *neo_dcp) -{ - struct neo_iomfb_transaction *transaction; - - transaction = kzalloc_obj(*transaction); - if (!transaction) - return; - transaction->start = neo_iomfb_backlight_start; - transaction->release = neo_iomfb_backlight_release; - transaction->brightness_only = true; - if (neo_iomfb_queue(neo_dcp, transaction)) - kfree(transaction); -} - -int neo_iomfb_configure_backlight_h17p(struct neo_apple_dcp *neo_dcp, u32 maximum, - bool inherited_valid, u32 inherited, - bool default_valid, u32 default_nits) -{ - if (!neo_iomfb_uses_queue(neo_dcp)) - return -EINVAL; - return neo_dcp_backlight_configure(neo_dcp, maximum, inherited_valid, inherited, - default_valid, default_nits, neo_iomfb_backlight_kick); -} - -struct neo_iomfb_atomic_transaction { - struct neo_iomfb_transaction transaction; - struct drm_atomic_state *state; - struct drm_crtc *crtc; - struct neo_apple_dcp *neo_dcp; - struct neo_iomfb_scanout_h17p *scanout; -}; - -static void neo_iomfb_atomic_start(struct neo_apple_dcp *neo_dcp, - struct neo_iomfb_transaction *transaction) -{ - struct neo_iomfb_atomic_transaction *atomic = container_of(transaction, - struct neo_iomfb_atomic_transaction, transaction); - - if (!READ_ONCE(neo_dcp->mode_state.valid) || !neo_dcp->connector || !neo_dcp->connector->connected) { - schedule_work(&neo_dcp->vblank_wq); - return; - } - atomic->scanout = neo_iomfb_scanout_prepare_h17p(neo_dcp, atomic->crtc, atomic->state); - if (!atomic->scanout) { - WRITE_ONCE(neo_dcp->crashed, true); - schedule_work(&neo_dcp->vblank_wq); - return; - } - neo_dcp->neo_iomfb.next_scanout = atomic->scanout; - neo_iomfb_flush_h17p(neo_dcp, atomic->crtc, atomic->state); -} - -static void neo_iomfb_atomic_release(struct neo_iomfb_transaction *transaction) -{ - struct neo_iomfb_atomic_transaction *atomic = container_of(transaction, - struct neo_iomfb_atomic_transaction, transaction); - - if (atomic->scanout && atomic->neo_dcp->neo_iomfb.scanout != atomic->scanout) { - if (atomic->neo_dcp->neo_iomfb.next_scanout == atomic->scanout) - atomic->neo_dcp->neo_iomfb.next_scanout = NULL; - neo_iomfb_scanout_release_h17p(atomic->scanout); - } - drm_atomic_state_put(atomic->state); - kfree(atomic); -} - -static void neo_iomfb_queue_atomic(struct neo_apple_dcp *neo_dcp, struct drm_crtc *crtc, - struct drm_atomic_state *state) -{ - struct neo_iomfb_atomic_transaction *atomic; - - atomic = kzalloc_obj(*atomic); - if (!atomic) - goto failed; - atomic->transaction.start = neo_iomfb_atomic_start; - atomic->transaction.release = neo_iomfb_atomic_release; - atomic->state = drm_atomic_state_get(state); - atomic->crtc = crtc; - atomic->neo_dcp = neo_dcp; - if (!neo_iomfb_queue(neo_dcp, &atomic->transaction)) - return; - neo_iomfb_atomic_release(&atomic->transaction); -failed: - WRITE_ONCE(neo_dcp->crashed, true); - schedule_work(&neo_dcp->vblank_wq); -} - -void neo_dcp_flush(struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - struct platform_device *pdev = to_apple_crtc(crtc)->neo_dcp; - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - if (neo_iomfb_uses_queue(neo_dcp)) { - neo_iomfb_queue_atomic(neo_dcp, crtc, state); - return; - } - - /* - * DCP does not complete swaps after a link loss. A plane-only commit - * arriving between the reconnect callback and the required modeset - * would otherwise leave an unsignalled flip event in front of the - * recovery commit. Modesets set valid_mode before reaching flush. - * - * The same applies while a connector is still disconnected: resume - * re-runs the modeset, which marks the mode valid again, before the - * firmware has reported the display back. - */ - if (!READ_ONCE(neo_dcp->mode_state.valid) || !neo_dcp->connector || !neo_dcp->connector->connected) { - schedule_work(&neo_dcp->vblank_wq); - return; - } - - if (neo_dcp_channel_busy(&neo_dcp->ch_cmd)) - { - if (!neo_dcp->ch_cmd.warned_busy) { - dev_err(neo_dcp->dev, "unexpected busy command channel\n"); - neo_dcp->ch_cmd.warned_busy = true; - } - /* HACK: issue a delayed vblank event to avoid timeouts in - * drm_atomic_helper_wait_for_vblanks(). - */ - schedule_work(&neo_dcp->vblank_wq); - return; - } else if (neo_dcp->ch_cmd.warned_busy) { - neo_dcp->ch_cmd.warned_busy = false; - } - - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - neo_iomfb_flush_v12_3(neo_dcp, crtc, state); - break; - case DCP_FIRMWARE_V_13_5: - neo_iomfb_flush_v13_3(neo_dcp, crtc, state); - break; - case DCP_FIRMWARE_H17P: - neo_iomfb_flush_h17p(neo_dcp, crtc, state); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", neo_dcp->fw_compat); - break; - } -} - -static void neo_iomfb_start(struct neo_apple_dcp *neo_dcp) -{ - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - neo_iomfb_start_v12_3(neo_dcp); - break; - case DCP_FIRMWARE_V_13_5: - neo_iomfb_start_v13_3(neo_dcp); - break; - case DCP_FIRMWARE_H17P: - neo_iomfb_start_h17p(neo_dcp); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", neo_dcp->fw_compat); - break; - } -} - -bool neo_dcp_is_initialized(struct platform_device *pdev) -{ - struct neo_apple_dcp *neo_dcp = platform_get_drvdata(pdev); - - return neo_dcp->active; -} - -static void neo_iomfb_recv_message(struct neo_apple_dcp *neo_dcp, u64 message) -{ - enum dcpep_type type = FIELD_GET(IOMFB_MESSAGE_TYPE, message); - - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(neo_dcp->crashed)) - return; - - if (type == IOMFB_MESSAGE_TYPE_INITIALIZED) { - /* - * H17P reports its interface version (bits 63:48) and a - * firmware hash in the high half of the InitComplete word. - * The transport below it is unchanged. - */ - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P) - dev_info(neo_dcp->dev, - "IOMFB: init complete %#llx (version %llu)\n", - message, - FIELD_GET(GENMASK_ULL(63, 48), message)); - neo_iomfb_start(neo_dcp); - } else if (type == IOMFB_MESSAGE_TYPE_MSG) - dcpep_got_msg(neo_dcp, message); - else - dev_warn(neo_dcp->dev, "Ignoring unknown message %llx\n", message); -} - -void neo_iomfb_recv_msg(struct neo_apple_dcp *neo_dcp, u64 message) -{ - if (!neo_iomfb_uses_queue(neo_dcp)) { - neo_iomfb_recv_message(neo_dcp, message); - return; - } - - mutex_lock(&neo_dcp->neo_iomfb.lock); - WRITE_ONCE(neo_dcp->neo_iomfb.owner, current); - if (!neo_dcp->neo_iomfb.stopped) - neo_iomfb_recv_message(neo_dcp, message); - WRITE_ONCE(neo_dcp->neo_iomfb.owner, NULL); - neo_iomfb_queue_advance(neo_dcp); - mutex_unlock(&neo_dcp->neo_iomfb.lock); -} - -int neo_iomfb_start_rtkit(struct neo_apple_dcp *neo_dcp) -{ - dma_addr_t shmem_iova; - int ret; - - if (neo_iomfb_uses_queue(neo_dcp)) { - mutex_lock(&neo_dcp->neo_iomfb.lock); - if (neo_dcp->neo_iomfb.active || neo_dcp->neo_iomfb.scanout) { - mutex_unlock(&neo_dcp->neo_iomfb.lock); - return -EBUSY; - } - neo_dcp->neo_iomfb.stopped = false; - mutex_unlock(&neo_dcp->neo_iomfb.lock); - } - - /* - * H17P firmware expects the remote allocator endpoint to be started - * before IOMFB. A firmware subsystem whose endpoint was never started - * has nothing answering it on the AP side, and without this one the - * display DART faults. - */ - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P) { - ret = apple_rtkit_start_ep(neo_dcp->rtk, REMOTE_ALLOC_ENDPOINT); - if (ret) - return ret; - } - ret = apple_rtkit_start_ep(neo_dcp->rtk, IOMFB_ENDPOINT); - if (ret) - return ret; - - neo_dcp->shmem = dma_alloc_coherent(neo_dcp->dev, DCP_SHMEM_SIZE, &shmem_iova, - GFP_KERNEL); - if (!neo_dcp->shmem) - return -ENOMEM; - - neo_dcp_send_message(neo_dcp, IOMFB_ENDPOINT, dcpep_set_shmem(shmem_iova)); - - return 0; -} - -void neo_iomfb_shutdown(struct neo_apple_dcp *neo_dcp) -{ - /* We're going down */ - neo_dcp->active = false; - neo_dcp_mode_invalidate(&neo_dcp->mode_state); - - switch (neo_dcp->fw_compat) { - case DCP_FIRMWARE_V_12_3: - neo_iomfb_shutdown_v12_3(neo_dcp); - break; - case DCP_FIRMWARE_V_13_5: - neo_iomfb_shutdown_v13_3(neo_dcp); - break; - case DCP_FIRMWARE_H17P: - neo_iomfb_shutdown_h17p(neo_dcp); - break; - default: - WARN_ONCE(true, "Unexpected firmware version: %u\n", neo_dcp->fw_compat); - break; - } -} diff --git a/drivers/gpu/drm/apple-neo/iomfb.h b/drivers/gpu/drm/apple-neo/iomfb.h deleted file mode 100644 index ef432d78cac774..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb.h +++ /dev/null @@ -1,408 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -#ifndef __APPLE_DCPEP_H__ -#define __APPLE_DCPEP_H__ - -#include - -#include "version_utils.h" - -/* Fixed size of shared memory between DCP and AP */ -#define DCP_SHMEM_SIZE 0x100000 - -/* DCP message contexts */ -enum neo_dcp_context_id { - /* Callback */ - DCP_CONTEXT_CB = 0, - - /* Command */ - DCP_CONTEXT_CMD = 2, - - /* Asynchronous */ - DCP_CONTEXT_ASYNC = 3, - - /* Out-of-band callback */ - DCP_CONTEXT_OOBCB = 4, - - /* Out-of-band command */ - DCP_CONTEXT_OOBCMD = 6, - - /* Out-of-band Asynchronous */ - DCP_CONTEXT_OOBASYNC = 7, - - DCP_NUM_CONTEXTS -}; - -/* RTKit endpoint message types */ -enum dcpep_type { - /* Set shared memory */ - IOMFB_MESSAGE_TYPE_SET_SHMEM = 0, - - /* DCP is initialized */ - IOMFB_MESSAGE_TYPE_INITIALIZED = 1, - - /* Remote procedure call */ - IOMFB_MESSAGE_TYPE_MSG = 2, -}; - -/* - * IOMFB supports the setting of a number of parameters - * that alter various aspects of the connected sink's - * behaviour at runtime. - */ -enum neo_iomfb_parameter { - IOMFBPARAM_ADAPTIVE_SYNC = 14, -}; - -#define IOMFB_MESSAGE_TYPE GENMASK_ULL( 3, 0) - -/* Message */ -#define IOMFB_MSG_LENGTH GENMASK_ULL(63, 32) -#define IOMFB_MSG_OFFSET GENMASK_ULL(31, 16) -#define IOMFB_MSG_CONTEXT GENMASK_ULL(11, 8) -#define IOMFB_MSG_ACK BIT_ULL(6) - -/* Set shmem */ -#define IOMFB_SHMEM_DVA GENMASK_ULL(63, 16) -#define IOMFB_SHMEM_FLAG GENMASK_ULL( 7, 4) -#define IOMFB_SHMEM_FLAG_VALUE 4 - -struct neo_dcp_packet_header { - char tag[4]; - u32 in_len; - u32 out_len; -} __packed; - -#define DCP_IS_NULL(ptr) ((ptr) ? 1 : 0) -#define DCP_PACKET_ALIGNMENT (0x40) - -enum neo_iomfb_property_id { - IOMFB_PROPERTY_NITS = 15, // divide by Brightness_Scale -}; - -#define IOMFB_BRIGHTNESS_MIN 0x10000000 - -/* Structures used in v12.0 firmware */ - -#define SWAP_SURFACES 4 - -struct neo_dcp_iouserclient { - /* Handle for the IOUserClient. macOS sets this to a kernel VA. */ - u64 handle; - u32 unk; - u8 flag1; - u8 flag2; - u8 padding[2]; -} __packed; - -/* - * Update background color to struct dcp_swap.bg_color - */ -#define IOMFB_SET_BACKGROUND BIT(31) - -struct neo_dcp_allocate_bandwidth_req { - u64 unk1; - u64 unk2; - u64 unk3; - u8 unk1_null; - u8 unk2_null; - u8 padding[8]; -} __packed; - -struct neo_dcp_allocate_bandwidth_resp { - u64 unk1; - u64 unk2; - u32 ret; -} __packed; - -struct neo_dcp_rt_bandwidth { - u64 unk1; - u64 reg_scratch; - u64 reg_doorbell; - u32 unk2; - u32 doorbell_bit; - u32 padding[7]; -} __packed; - -struct frame_sync_props { - u8 unk[28]; -}; - -struct neo_dcp_set_frame_sync_props_req { - struct frame_sync_props props; - u8 frame_sync_props_null; - u8 padding[3]; -} __packed; - -struct neo_dcp_set_frame_sync_props_resp { - struct frame_sync_props props; -} __packed; - -/* Method calls */ - -enum dcpep_method { - dcpep_late_init_signal, - dcpep_setup_video_limits, - dcpep_set_create_dfb, - dcpep_start_signal, - dcpep_swap_start, - dcpep_swap_submit, - dcpep_set_display_device, - dcpep_set_digital_out_mode, - dcpep_create_default_fb, - dcpep_set_display_refresh_properties, - dcpep_flush_supports_power, - dcpep_set_power_state, - dcpep_first_client_open, - dcpep_register_dfb_surface, - dcpep_set_parameter_dcp, - dcpep_enable_disable_video_power_savings, - dcpep_is_main_display, - iomfbep_a131_pmu_service_matched, - iomfbep_a132_backlight_service_matched, - iomfbep_a358_vi_set_temperature_hint, - iomfbep_get_color_remap_mode, - iomfbep_last_client_close, - iomfbep_abort_swaps_dcp, - iomfbep_set_matrix, - /* - * H17P default-framebuffer queries, issued from inside - * start_hardware_boot (D121) between create_default_fb and - * setup_video_limits. - */ - dcpep_get_dfb_compression_info, - dcpep_get_dfb_info, - dcpep_get_dfb_layout, - dcpep_get_dfb_state, - dcpep_set_dfb_dimensions, - dcpep_commit_dfb_info, - dcpep_dfb_query, - dcpep_dfb_ready_query, - /* - * H17P pipe-configuration calls made before the first swap. Most - * names are unknown; they are identified by method number. - */ - dcpep_pipe_cfg_415, - dcpep_pipe_cfg_031, - dcpep_pipe_cfg_414, - dcpep_pipe_query_478, - dcpep_pipe_query_474, - dcpep_pipe_set_476, - dcpep_pipe_set_428, - dcpep_num_methods -}; - -#define IOMFB_METHOD(tag, name) [name] = { #name, { tag[0], tag[1], tag[2], tag[3] } } -#define IOMFB_METHOD_H17(tag, h17g, name) [name] = { #name, \ - { tag[0], tag[1], tag[2], tag[3] }, \ - { h17g[0], h17g[1], h17g[2], h17g[3] } } - -struct neo_dcp_method_entry { - const char *name; - char tag[4]; - /* H17G shares the H17P transport but changes selected method IDs. */ - char tag_h17g[4]; -}; - -#define IOMFB_MAX_CB (1000) - -bool neo_iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len); -struct neo_apple_dcp; -bool neo_iomfb_check_callback_h17p(struct neo_apple_dcp *neo_dcp, int tag, u32 in_len, - u32 out_len); - -typedef bool (*neo_iomfb_cb_handler)(struct neo_apple_dcp *, int, void *, void *); - -/* Prototypes */ - -struct neo_dcp_set_digital_out_mode_req { - u32 color_mode_id; - u32 timing_mode_id; -} __packed; - -struct neo_dcp_map_buf_req { - u64 buffer; - u8 unk; - u8 buf_null; - u8 vaddr_null; - u8 dva_null; -} __packed; - -struct neo_dcp_unmap_buf_resp { - u64 buffer; - u64 vaddr; - u64 dva; - u8 unk; - u8 buf_null; -} __packed; - -struct neo_dcp_allocate_buffer_req { - u32 unk0; - u64 size; - u32 unk2; - u8 paddr_null; - u8 dva_null; - u8 dva_size_null; - u8 padding; -} __packed; - -struct neo_dcp_allocate_buffer_resp { - u64 paddr; - u64 dva; - u64 dva_size; - u32 mem_desc_id; -} __packed; - -struct neo_dcp_map_physical_req { - u64 paddr; - u64 size; - u32 flags; - u8 dva_null; - u8 dva_size_null; - u8 padding[2]; -} __packed; - -struct neo_dcp_map_physical_resp { - u64 dva; - u64 dva_size; - u32 mem_desc_id; -} __packed; - -struct neo_dcp_get_uint_prop_req { - char obj[4]; - char key[0x40]; - u64 value; - u8 value_null; - u8 padding[3]; -} __packed; - -struct neo_dcp_get_uint_prop_resp { - u64 value; - u8 ret; - u8 padding[3]; -} __packed; - -struct neo_iomfb_sr_set_property_int_req { - char obj[4]; - char key[0x40]; - u64 value; - u8 value_null; - u8 padding[3]; -} __packed; - -struct neo_iomfb_set_fx_prop_req { - char obj[4]; - char key[0x40]; - u32 value; -} __packed; - -struct neo_dcp_set_power_state_resp { - u32 unkint; - u32 ret; -} __packed; - -struct neo_dcp_set_dcpav_prop_chunk_req { - char data[0x1000]; - u32 offset; - u32 length; -} __packed; - -struct neo_dcp_set_dcpav_prop_end_req { - char key[0x40]; -} __packed; - -struct neo_dcp_set_parameter_dcp { - u32 param; - u32 value[8]; - u32 count; -} __packed; - -struct neo_dcp_swap_complete_intent_gated { - u32 swap_id; - u8 unkBool; - u32 unkInt; - u32 width; - u32 height; -} __packed; - -struct neo_dcp_read_edt_data_req { - char key[0x40]; - u32 count; - u32 value[8]; -} __packed; - -struct neo_dcp_read_edt_data_resp { - u32 value[8]; - u8 ret; -} __packed; - -struct neo_iomfb_property { - u32 id; - u32 value; -} __packed; - -struct neo_iomfb_get_color_remap_mode_req { - u32 mode; - u8 mode_null; - u8 padding[3]; -} __packed; - -struct neo_iomfb_get_color_remap_mode_resp { - u32 mode; - u32 ret; -} __packed; - -struct neo_iomfb_last_client_close_req { - u8 unkint_null; - u8 padding[3]; -} __packed; - -struct neo_iomfb_last_client_close_resp { - u32 unkint; -} __packed; - -struct io_user_client { - u64 addr; - u32 unk; - u8 flag1; - u8 flag2; - u8 pad[2]; -} __packed; - -struct neo_iomfb_abort_swaps_dcp_req { - struct io_user_client client; - u8 client_null; - u8 pad[3]; -} __packed; - -struct neo_iomfb_abort_swaps_dcp_resp { - struct io_user_client client; - u32 ret; -} __packed; - -struct neo_iomfb_set_matrix_req { - u32 location; - u64 matrix[9]; - u8 matrix_null; - u8 padding[3]; -} __packed; - -struct neo_iomfb_set_matrix_resp { - u32 ret; -} __packed; - -struct dcpep_get_tiling_state_req { - u32 event; - u32 param; - u32 value; - u8 value_null; - u8 padding[3]; -} __packed; - -struct dcpep_get_tiling_state_resp { - u32 value; - u32 ret; -} __packed; - -#endif diff --git a/drivers/gpu/drm/apple-neo/iomfb_h17p.c b/drivers/gpu/drm/apple-neo/iomfb_h17p.c deleted file mode 100644 index a27b79d96ed6cf..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_h17p.c +++ /dev/null @@ -1,740 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#include -#include - -#include "dcp-link.h" -#include "iomfb_h17p.h" -#include "iomfb_v12_3.h" -#include "iomfb_v13_3.h" -#include "version_utils.h" - -struct neo_dcp_callback_size { - u32 in_len; - u32 out_len; - bool valid; - /* Lengths bound what the handler reads and writes; no capture exists. */ - bool minimum; -}; - -/* - * Callbacks that only arrive after startup: present notifications, swap - * aborts, the backlight re-send request, wall-clock and clock-rate queries. - * Their record lengths are not in the startup observations, so accept any - * record that holds what the handler reads and the reply it writes. The - * dispatcher zeroes the rest of a longer reply. - */ -#define DCP_CB_AT_LEAST(in, out) \ - { .in_len = (in), .out_len = (out), .valid = true, .minimum = true } - -/* R-IOMFB-startup-run21 and R-ANALYTICS Linux boundary observations. */ -static const struct neo_dcp_callback_size callback_sizes[IOMFB_MAX_CB] = { - [0] = { 0x0, 0x4, true }, - [1] = { 0x0, 0x4, true }, - [2] = DCP_CB_AT_LEAST(0x0, 0x0), - [3] = { 0x4, 0x14, true }, - [6] = { 0x54, 0x50, true }, - [100] = { 0x0, 0x0, true }, - [101] = { 0x0, 0x4, true }, - [102] = { 0x44, 0x0, true }, - [104] = { 0x44, 0x0, true }, - [107] = DCP_CB_AT_LEAST(0x0, 0x0), - [108] = { 0x0, 0x4, true }, - [109] = { 0x0, 0x4, true }, - [110] = { 0x0, 0x4, true }, - [111] = { 0x0, 0x4, true }, - [112] = { 0x0, 0x4, true }, - [113] = { 0x0, 0x4, true }, - [114] = { 0x1044, 0x1004, true }, - [117] = DCP_CB_AT_LEAST(0x0, 0x0), - [118] = DCP_CB_AT_LEAST(0x0, 0x4), - [119] = DCP_CB_AT_LEAST(0x0, 0x4), - [120] = DCP_CB_AT_LEAST(0x0, 0x4), - [121] = { 0x0, 0x4, true }, - [122] = DCP_CB_AT_LEAST(0x0, 0x1), - [123] = { 0x0, 0x4, true }, - [124] = DCP_CB_AT_LEAST(0x0, 0x4), - [125] = { 0x64, 0x24, true }, - [127] = { 0x4, 0x4, true }, - [128] = { 0x1008, 0x4, true }, - [129] = { 0x40, 0x4, true }, - [130] = DCP_CB_AT_LEAST(0x22, 0x14), - [201] = { 0xc, 0x10, true }, - [202] = { 0x1a, 0x0, true }, - [206] = { 0x0, 0x4, true }, - [207] = { 0x0, 0x4, true }, - [208] = DCP_CB_AT_LEAST(0x0, 0x0), - [209] = DCP_CB_AT_LEAST(0x0, 0x8), - [300] = { 0x10, 0x0, true }, - [400] = { 0x4c, 0xc04, true }, - [401] = { 0x50, 0xc, true }, - [404] = DCP_CB_AT_LEAST(0x0, 0x0), - [406] = { 0x48, 0x0, true }, - [408] = DCP_CB_AT_LEAST(0x8, 0x8), - [411] = { 0x10, 0x1c, true }, - [413] = { 0x1048, 0x4, true }, - [414] = { 0x50, 0x4, true }, - [415] = { 0x4c, 0x4, true }, - [451] = { 0x14, 0x1c, true }, - [452] = DCP_CB_AT_LEAST(0x18, 0x14), - [454] = { 0x4, 0x1, true }, - [552] = { 0x1044, 0x4, true }, - [561] = { 0x1044, 0x4, true }, - [563] = { 0x4c, 0x4, true }, - [565] = { 0x48, 0x4, true }, - [567] = DCP_CB_AT_LEAST(0x0, 0x1), - [572] = DCP_CB_AT_LEAST(0x0, 0x4), - [574] = { 0x4, 0x8, true }, - [575] = { 0x58, 0x4c, true }, - [576] = DCP_CB_AT_LEAST(0x0, 0x0), - [582] = { 0x8, 0x4, true }, - [583] = DCP_CB_AT_LEAST(0x0, 0x0), - [584] = DCP_CB_AT_LEAST(0x0, 0x0), - [585] = DCP_CB_AT_LEAST(0xe4, 0xe0), - [590] = { 0x730, 0x0, true }, - [592] = DCP_CB_AT_LEAST(0x11, 0x0), - [593] = DCP_CB_AT_LEAST(0x4, 0x0), - [594] = DCP_CB_AT_LEAST(0x1, 0x0), - [595] = DCP_CB_AT_LEAST(0x0, 0x0), - [597] = DCP_CB_AT_LEAST(0x0, 0x1), - [598] = DCP_CB_AT_LEAST(0x0, 0x1), - [599] = { 0x0, 0x0, true }, -}; - -bool neo_iomfb_validate_callback_h17p(int tag, u32 in_len, u32 out_len) -{ - const struct neo_dcp_callback_size *size; - - if (tag < 0 || tag >= ARRAY_SIZE(callback_sizes)) - return false; - size = &callback_sizes[tag]; - if (!size->valid) - return false; - if (size->minimum) - return in_len >= size->in_len && out_len >= size->out_len; - return size->in_len == in_len && size->out_len == out_len; -} - -struct neo_dcp_h17p_hotplug_request { - __le64 connected; - u8 tiled_display[0x4c]; - u8 tiled_display_null; - u8 padding[3]; -} __packed; - -struct neo_dcp_h17p_swap_info_request { - u8 swap_info[0xe0]; - u8 swap_info_null; - u8 padding[3]; -} __packed; - -static_assert(sizeof(struct neo_dcp_h17p_hotplug_request) == 0x58); -static_assert(sizeof(struct neo_dcp_h17p_swap_info_request) == 0xe4); -/* D590 swap_complete_ap_gated is 0x730 bytes on the wire. */ -static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); -static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); - -void neo_iomfb_serialize_present_h17p(struct neo_dcp_present_h17p *wire, - const struct neo_dcp_swap_submit_req_h17p *request) -{ - size_t tail_offset = offsetof(struct neo_dcp_swap_submit_req_h17p, surf_iova); - size_t tail_size = sizeof(*request) - tail_offset; - - static_assert(sizeof(request->swap) == sizeof(wire->swap) + 2); - static_assert(sizeof(request->surf) == sizeof(wire->surf)); - static_assert(sizeof(*request) - - offsetof(struct neo_dcp_swap_submit_req_h17p, surf_iova) == - sizeof(wire->tail) - 2); - - memcpy(wire->swap, &request->swap, sizeof(wire->swap)); - memcpy(wire->surf, request->surf, sizeof(wire->surf)); - memcpy(wire->tail, (const u8 *)request + tail_offset, tail_size); - memset(wire->tail + tail_size, 0, sizeof(wire->tail) - tail_size); -} - -/* Integer commanded nits are exact binary64 values; no kernel FP is needed. */ -static u64 neo_iomfb_nits_binary64_h17p(u32 nits) -{ - u64 value = 0; - unsigned int exponent; - - if (nits) { - exponent = fls(nits) - 1; - value = (u64)(exponent + 1023) << 52; - value |= ((u64)nits << (52 - exponent)) & GENMASK_ULL(51, 0); - } - return value; -} - -void neo_iomfb_encode_backlight_h17p(struct neo_dcp_present_h17p *wire, u32 nits, - u32 maximum, bool update) -{ - u64 ceiling = neo_iomfb_nits_binary64_h17p(maximum); - u64 unity = neo_iomfb_nits_binary64_h17p(1); - - /* This profile does not use the older DAC/power words. */ - memset(wire->swap + 0x32f, 0, 0xd); - memset(wire->swap + 0x354, 0, 0x9a); - /* The ceiling pair must remain valid even on a brightness-idle frame. */ - put_unaligned_le64(ceiling, wire->swap + 0x36e); - put_unaligned_le64(ceiling, wire->swap + 0x376); - if (!update) - return; - - /* Linux brightness changes and soft DPMS use the same activation bytes. */ - put_unaligned_le32(1, wire->swap + 0x354); - wire->swap[0x358] = 1; - wire->swap[0x359] = 1; - wire->swap[0x35a] = 1; - put_unaligned_le16(1, wire->swap + 0x35c); - put_unaligned_le64(neo_iomfb_nits_binary64_h17p(nits), wire->swap + 0x35e); - put_unaligned_le64(unity, wire->swap + 0x37e); - put_unaligned_le64(unity, wire->swap + 0x3e6); -} - -static const struct neo_dcp_method_entry neo_dcp_methods[dcpep_num_methods] = { - IOMFB_METHOD("A000", dcpep_late_init_signal), - IOMFB_METHOD_H17("A025", "A029", dcpep_setup_video_limits), /* (0,0) */ - IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), /* nested in D206 */ - IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), /* nested in D207 */ - IOMFB_METHOD("A385", dcpep_set_create_dfb), /* first call inside D121; (0,0) */ - IOMFB_METHOD("A386", iomfbep_a358_vi_set_temperature_hint), /* nested in D100 */ - IOMFB_METHOD("A401", dcpep_start_signal), - IOMFB_METHOD("A406", dcpep_swap_start), - /* - * swap_start moved A407 -> A406 on H17P, but swap_submit did not move - * with it: every swap is A406 followed by A408, A407 is unused, and the - * A408 response is 0x0c bytes (dcp_swap_submit_resp_h17p). - */ - IOMFB_METHOD("A408", dcpep_swap_submit), - IOMFB_METHOD("A411", dcpep_set_display_device), - IOMFB_METHOD("A412", dcpep_is_main_display), - IOMFB_METHOD("A413", dcpep_set_digital_out_mode), - IOMFB_METHOD("A423", iomfbep_set_matrix), - IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), - IOMFB_METHOD("A442", dcpep_set_parameter_dcp), /* (0x28, 0x4) */ - IOMFB_METHOD("A446", dcpep_create_default_fb), /* follows A385 in D121 */ - /* - * The default-framebuffer block the firmware expects inside D121, - * straight after A446 and before A025. Sizes are (in, out). - */ - IOMFB_METHOD_H17("A035", "A039", dcpep_get_dfb_compression_info), /* (0x8, 0x8) */ - IOMFB_METHOD_H17("A034", "A038", dcpep_get_dfb_info), /* (0x2c, 0x2c) */ - IOMFB_METHOD("A103", dcpep_get_dfb_layout), /* (0xc, 0x8) */ - IOMFB_METHOD("A445", dcpep_get_dfb_state), /* (0x4, 0x8) */ - IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), /* (0x8, 0) w,h */ - IOMFB_METHOD("A105", dcpep_commit_dfb_info), /* (0, 0) */ - IOMFB_METHOD_H17("A033", "A037", dcpep_dfb_query), /* (0, 0x4) */ - IOMFB_METHOD("A380", dcpep_dfb_ready_query), /* (0, 0x4) */ - IOMFB_METHOD("A415", dcpep_pipe_cfg_415), /* (0xc, 0xc) zeros */ - /* update_notify_clients_dcp */ - IOMFB_METHOD_H17("A031", "A035", dcpep_pipe_cfg_031), /* (0x6c, 0) */ - IOMFB_METHOD("A414", dcpep_pipe_cfg_414), /* (0x8, 0x8) zeros */ - IOMFB_METHOD("A478", dcpep_pipe_query_478), /* (0, 0x4) */ - IOMFB_METHOD("A474", dcpep_pipe_query_474), /* (0, 0x4) */ - IOMFB_METHOD("A476", dcpep_pipe_set_476), /* (0x4, 0) in 1 */ - IOMFB_METHOD("A428", dcpep_pipe_set_428), /* (0x4, 0x4) 0x10000 */ - IOMFB_METHOD("A450", dcpep_enable_disable_video_power_savings), /* in = u32 0 */ - IOMFB_METHOD("A457", dcpep_first_client_open), /* (0, 0) */ - /* D561 (displayMinRefreshInterval) arrives while A465 is in flight */ - IOMFB_METHOD("A465", dcpep_set_display_refresh_properties), - IOMFB_METHOD("A468", dcpep_flush_supports_power), /* follows A025 in D121, in = 1 */ - /* - * iomfbep_last_client_close and iomfbep_abort_swaps_dcp are deliberately - * absent: their H17P numbers are unverified, so the teardown paths skip - * them (DCP_HAS_CLIENT_TEARDOWN in iomfb_template.c). - */ - IOMFB_METHOD("A473", dcpep_set_power_state), /* out-of-band; out 0x8 */ - IOMFB_METHOD("A472", dcpep_register_dfb_surface), /* default FB surface, nested in D582 */ -}; - -#define DCP_FW h17p -#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) - -#include "iomfb_template.c" - -void neo_iomfb_apply_opaque_x_h17p(struct neo_apple_dcp *neo_dcp, neo_dcp_callback_t callback, - void *cookie) -{ - static const struct neo_dcp_method_entry method = { - .name = "apply_property", - .tag = { 'A', '3', '5', '2' }, - }; - struct neo_dcp_apply_property_h17p property = neo_dcp_opaque_x_property_h17p(); - - neo_dcp_push(neo_dcp, false, &method, sizeof(property), sizeof(u32), &property, - callback, cookie); -} - -static bool trampoline_rt_bandwidth_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - /* - * The bandwidth scratch and doorbell come from the DCP's DT node; the - * apple,bw-doorbell binding has no offset cell. - */ - u64 scratch = neo_dcp->disp_bw_scratch_res.start + - neo_dcp->disp_bw_scratch_offset; - u64 clock_request = neo_dcp->disp_bw_doorbell_res.start; - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - if (neo_apple_dcp_h17p_rt_bw_encode_reply(in, out, scratch, clock_request)) - dev_warn(neo_dcp->dev, "D003 bandwidth request is malformed\n"); - return true; -} - -static bool trampoline_get_frequency_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_apple_dcp_h17p_clock_request *request = in; - u64 rate; - int ret; - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - ret = neo_apple_dcp_h17p_clock_rate(request, clk_get_rate(neo_dcp->clk), - clk_get_rate(neo_dcp->clk_194), &rate); - if (ret) { - dev_warn(neo_dcp->dev, "unknown display clock ID %#x\n", - le32_to_cpu(request->clock_id)); - rate = 0; - } - - *(__le64 *)out = cpu_to_le64(rate); - return true; -} - -static bool trampoline_analytics_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_zero(neo_dcp, tag, out, in); - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - /* The dispatcher has validated the measured D114 reply size. */ - memset(out, 0, 0x1004); - *(u8 *)out = 'd'; - return true; -} - -static bool trampoline_hotplug_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_dcp_h17p_hotplug_request *request = in; - - if (!(request->tiled_display_null & 1)) - memcpy(out, request->tiled_display, - sizeof(request->tiled_display)); - return trampoline_hotplug(neo_dcp, tag, out, in); -} - -static bool trampoline_swap_info_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_dcp_h17p_swap_info_request *request = in; - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - if (!(request->swap_info_null & 1)) - memcpy(out, request->swap_info, sizeof(request->swap_info)); - return true; -} - -struct neo_dcp_h17p_provider_request { - u8 scope[4]; - char name[64]; - __le32 capacity; - u8 reserved[4]; -} __packed; - -struct neo_dcp_h17p_provider_reply { - u8 data[0xc00]; - __le32 length; -} __packed; - -struct neo_dcp_h17p_provider_property { - const char *name; - const char *dt_name; - u32 length; - const u8 *legacy_data; -}; - -static_assert(sizeof(struct neo_dcp_h17p_provider_request) == 0x4c); -static_assert(sizeof(struct neo_dcp_h17p_provider_reply) == 0xc04); - -/* Compatibility data matches the independently decoded J700 disp0 ADT. */ -static const u8 j700_provider_data_0[] = { - 0x0d, 0x00, 0x00, 0x00 -}; - -static const u8 j700_provider_data_1[] = { - 0x01, 0x00, 0x00, 0x00 -}; - -static const u8 j700_provider_data_2[] = { - 0x02, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, - 0x64, 0x00, 0x00, 0x00, 0x96, 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, - 0xfa, 0x00, 0x00, 0x00, 0x2c, 0x01, 0x00, 0x00, 0x5e, 0x01, 0x00, 0x00, - 0x90, 0x01, 0x00, 0x00, 0xc2, 0x01, 0x00, 0x00, 0xf4, 0x01, 0x00, 0x00, - 0x0d, 0x02, 0x00, 0x00 -}; - -static const u8 j700_provider_data_3[] = { - 0x64, 0x00, 0x00, 0x00 -}; - -static const u8 j700_provider_data_4[] = { - 0x34, 0x03, 0x00, 0x00, 0x84, 0x03, 0x00, 0x00, 0x24, 0x05, 0x00, 0x00, - 0x52, 0x07, 0x00, 0x00, 0x11, 0x09, 0x00, 0x00, 0xde, 0x0a, 0x00, 0x00, - 0xd2, 0x0c, 0x00, 0x00, 0xe2, 0x0e, 0x00, 0x00, 0xfb, 0x10, 0x00, 0x00, - 0x64, 0x13, 0x00, 0x00, 0xd5, 0x15, 0x00, 0x00, 0x68, 0x18, 0x00, 0x00, - 0xa2, 0x19, 0x00, 0x00 -}; - -/* Only these provider transfers have been measured against the disp0 ADT. */ -static const struct neo_dcp_h17p_provider_property provider_properties[] = { - { "power-lut-data-x", "apple,power-lut-data-x", 4, j700_provider_data_0 }, - { "power-lut-data-y", "apple,power-lut-data-y", 4, j700_provider_data_1 }, - { "power-lut-data-xindex", "apple,power-lut-data-xindex", 52, j700_provider_data_2 }, - { "power-lut-data-yindex", "apple,power-lut-data-yindex", 4, j700_provider_data_3 }, - { "power-lut-data-lut", "apple,power-lut-data-lut", 52, j700_provider_data_4 }, - { "power-lut-vbatt-cur-nominal", "apple,power-lut-vbatt-cur-nominal", 0, NULL }, -}; - -static const struct neo_dcp_h17p_provider_property * -neo_dcp_provider_property_h17p(const struct neo_dcp_h17p_provider_request *request) -{ - unsigned int i; - - if (memcmp(request->scope, "VORP", sizeof(request->scope)) || - le32_to_cpu(request->capacity) != - sizeof_field(struct neo_dcp_h17p_provider_reply, data) || - memchr_inv(request->reserved, 0, sizeof(request->reserved)) || - strnlen(request->name, sizeof(request->name)) == sizeof(request->name)) - return NULL; - - /* Bytes following the first NUL were not initialized by the firmware. */ - for (i = 0; i < ARRAY_SIZE(provider_properties); i++) - if (!strcmp(request->name, provider_properties[i].name)) - return &provider_properties[i]; - return NULL; -} - -static bool trampoline_provider_property_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_dcp_h17p_provider_property *property; - struct neo_dcp_h17p_provider_reply *reply = out; - struct device_node *node = neo_dcp->dev->of_node; - int length, ret; - - /* The dispatcher already cleared the exact H17G output length. */ - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return true; - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - property = neo_dcp_provider_property_h17p(in); - if (!property) { - dev_err(neo_dcp->dev, "unqualified D400 provider property request\n"); - goto fail; - } - - length = of_property_count_u8_elems(node, property->dt_name); - if (!property->length) { - /* Only this absent property has a measured empty reply. */ - if (of_property_present(node, property->dt_name)) - goto fail; - } else if (!of_property_present(node, property->dt_name) && - of_machine_is_compatible("apple,j700") && - of_device_is_compatible(node, "apple,t8140-dcp")) { - /* Qualified older loaders lack the newly described table properties. */ - memset(reply, 0, sizeof(*reply)); - memcpy(reply->data, property->legacy_data, property->length); - reply->length = cpu_to_le32(property->length); - dev_warn_once(neo_dcp->dev, "using J700 provider data for an older loader\n"); - return true; - } else if (length != (int)property->length) { - dev_err(neo_dcp->dev, "invalid provider property %s: %d bytes\n", - property->dt_name, length); - goto fail; - } - - memset(reply, 0, sizeof(*reply)); - if (property->length) { - ret = of_property_read_u8_array(node, property->dt_name, - reply->data, property->length); - if (ret) - goto fail; - } - reply->length = cpu_to_le32(property->length); - return true; - -fail: - WRITE_ONCE(neo_dcp->crashed, true); - return false; -} - -static bool trampoline_luma_span_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - /* Preserve the older profile's already-cleared unknown-callback reply. */ - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return true; - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - if (get_unaligned_le32(in)) { - dev_err(neo_dcp->dev, "unqualified D574 range selector\n"); - WRITE_ONCE(neo_dcp->crashed, true); - return false; - } - - /* Selector zero receives binary32 219.0 followed by a zero word. */ - put_unaligned_le64(0x435b0000, out); - return true; -} - -static bool trampoline_get_uint_prop_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_dcp_get_uint_prop_req *request = in; - - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_get_uint_prop(neo_dcp, tag, out, in); - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - /* The observed startup query uses Asahi's existing Temperature reply. */ - if (memcmp(request->obj, "SUMP", sizeof(request->obj)) || - strnlen(request->key, sizeof(request->key)) == sizeof(request->key) || - strcmp(request->key, "Temperature") || request->value_null || - memchr_inv(request->padding, 0, sizeof(request->padding))) { - dev_err(neo_dcp->dev, "unqualified D401 integer property query\n"); - WRITE_ONCE(neo_dcp->crashed, true); - return false; - } - - put_unaligned_le64(3029, out); - *((u8 *)out + offsetof(struct neo_dcp_get_uint_prop_resp, ret)) = true; - return true; -} - -static bool trampoline_allocate_buffer_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_dcp_allocate_buffer_req *wire = in; - struct neo_dcp_allocate_buffer_req request = { 0 }; - struct neo_dcp_allocate_buffer_resp reply; - - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_allocate_buffer(neo_dcp, tag, out, in); - - static_assert(sizeof(*wire) == 0x14); - static_assert(sizeof(reply) == 0x1c); - static_assert(offsetof(struct neo_dcp_allocate_buffer_req, size) == 4); - static_assert(offsetof(struct neo_dcp_allocate_buffer_req, unk2) == 12); - static_assert(offsetof(struct neo_dcp_allocate_buffer_req, paddr_null) == 16); - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - request.size = get_unaligned_le64(&wire->size); - reply = dcpep_cb_allocate_buffer(neo_dcp, &request); - if (!reply.dva_size) { - /* Exhaustion and allocation errors use the all-zero reply. */ - memset(out, 0, sizeof(reply)); - return true; - } - - put_unaligned_le64(reply.paddr, out); - put_unaligned_le64(reply.dva, (u8 *)out + 8); - put_unaligned_le64(reply.dva_size, (u8 *)out + 16); - put_unaligned_le32(reply.mem_desc_id, (u8 *)out + 24); - return true; -} - -static bool trampoline_map_piodma_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_dcp_map_buf_req *wire = in; - struct neo_dcp_map_buf_req request = { 0 }; - struct neo_dcp_map_buf_resp_h17p reply = { 0 }; - u64 buffer; - - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_map_piodma(neo_dcp, tag, out, in); - - static_assert(sizeof(*wire) == 0xc); - static_assert(sizeof(reply) == 0x10); - static_assert(offsetof(struct neo_dcp_map_buf_req, unk) == 8); - static_assert(offsetof(struct neo_dcp_map_buf_req, dva_null) == 11); - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - buffer = get_unaligned_le64(&wire->buffer); - request.buffer = buffer; - if (!dcpep_map_piodma(neo_dcp, &request, &reply)) { - /* A rejected mapping echoes the descriptor with a zero DVA. */ - put_unaligned_le32(buffer, out); - put_unaligned_le64(0, (u8 *)out + 4); - put_unaligned_le32(0, (u8 *)out + 12); - return true; - } - - put_unaligned_le32(reply.buffer, out); - put_unaligned_le64(reply.dva, (u8 *)out + 4); - put_unaligned_le32(reply.unk, (u8 *)out + 12); - return true; -} - -static bool trampoline_unmap_piodma_h17p(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - const struct neo_dcp_unmap_buf_resp *wire = in; - struct neo_dcp_unmap_buf_resp request = { 0 }; - - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_unmap_piodma(neo_dcp, tag, out, in); - - static_assert(sizeof(*wire) == 0x1a); - static_assert(offsetof(struct neo_dcp_unmap_buf_resp, vaddr) == 8); - static_assert(offsetof(struct neo_dcp_unmap_buf_resp, dva) == 16); - static_assert(offsetof(struct neo_dcp_unmap_buf_resp, buf_null) == 25); - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - request.buffer = get_unaligned_le64(&wire->buffer); - request.vaddr = get_unaligned_le64(&wire->vaddr); - request.dva = get_unaligned_le64(&wire->dva); - request.unk = wire->unk; - request.buf_null = wire->buf_null; - dcpep_unmap_piodma(neo_dcp, &request); - return true; -} - -static bool -trampoline_release_mem_desc_h17p(struct neo_apple_dcp *neo_dcp, int tag, void *out, - void *in) -{ - u32 id; - - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - return trampoline_release_mem_desc(neo_dcp, tag, out, in); - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - id = get_unaligned_le32(in); - *(u8 *)out = dcpep_cb_release_mem_desc(neo_dcp, &id); - return true; -} - -/* H17P callback numbering is not a uniform shift of the v13.5 table. */ -static const neo_iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { - [0] = dcpep_cb_d000_h17p, /* acked after a nested A033 */ - [1] = trampoline_true, - [2] = trampoline_nop, - [3] = trampoline_rt_bandwidth_h17p, - [6] = trampoline_set_frame_sync_props_h17p, /* 0x54/0x50, identity scale */ - [100] = iomfbep_cb_match_pmu_service, /* match_pmu_service */ - [101] = trampoline_zero, - [102] = trampoline_nop, /* set_number_property */ - [104] = trampoline_nop, /* set_boolean_property */ - [107] = trampoline_nop, - [108] = trampoline_true, /* create_provider_service */ - [109] = trampoline_true, /* create_product_service */ - [110] = trampoline_true, /* create_PMU_service */ - [111] = trampoline_true, /* create_iomfb_service */ - [112] = trampoline_create_backlight_service, /* create_backlight_service */ - [113] = trampoline_true, /* create_nvram_service */ - [114] = trampoline_analytics_h17p, /* CoreAnalyticsSendEvent */ - [117] = trampoline_nop, /* set_idle_caching_state_ap */ - [118] = trampoline_zero, /* upload_trace_start */ - [119] = trampoline_zero, /* upload_trace_chunk */ - [120] = trampoline_zero, /* upload_trace_end */ - [121] = dcpep_cb_boot_1, /* start_hardware_boot */ - [122] = trampoline_false, /* is_dark_boot */ - [123] = trampoline_false, /* is_waking_from_hibernate */ - [124] = trampoline_zero, /* detect_fastsim */ - [125] = trampoline_read_edt_data, /* read_edt_data */ - [127] = trampoline_prop_start, /* setDCPAVPropStart */ - [128] = trampoline_prop_chunk, /* setDCPAVPropChunk */ - [129] = trampoline_prop_end, /* setDCPAVPropEnd */ - [130] = trampoline_allocate_bandwidth, /* allocate_bandwidth */ - [201] = trampoline_map_piodma_h17p, /* map_buf */ - [202] = trampoline_unmap_piodma_h17p, /* unmap_buf */ - [206] = iomfbep_cb_match_pmu_service_2, /* match_pmu_service */ - [207] = iomfbep_cb_match_backlight_service, /* match_backlight_service */ - [208] = trampoline_nop, /* update_backlight_factor_prop */ - [209] = trampoline_get_time, /* get_calendar_time_ms */ - [300] = trampoline_pr_publish, - [400] = trampoline_provider_property_h17p, - [401] = trampoline_get_uint_prop_h17p, /* get_uint_prop */ - [404] = trampoline_nop, /* set_uint_prop */ - [406] = trampoline_set_fx_prop, /* set_fx_prop */ - [408] = trampoline_get_frequency_h17p, /* getClockFrequency */ - [411] = trampoline_map_reg, /* mapDeviceMemoryWithIndex */ - [413] = trampoline_true, /* setProperty */ - [414] = trampoline_sr_set_property_int, /* setProperty */ - [415] = trampoline_true, /* setProperty */ - [451] = trampoline_allocate_buffer_h17p, /* allocate_buffer */ - [452] = trampoline_map_physical, /* prepare */ - [454] = trampoline_release_mem_desc_h17p, /* release_descriptor */ - [552] = trampoline_true, - [561] = trampoline_true, - [563] = trampoline_true, - [565] = trampoline_true, - [567] = trampoline_true, - [572] = trampoline_zero, /* powerUpDART */ - [574] = trampoline_luma_span_h17p, - [575] = trampoline_hotplug_h17p, - [576] = trampoline_nop, - [582] = iomfbep_cb_create_dfb_surface, /* create_default_fb_surface */ - [583] = trampoline_nop, /* clear_default_surface */ - [584] = trampoline_nop, /* swap_notify_gated */ - [585] = trampoline_swap_info_h17p, /* swap_info_notify_dispatch */ - [590] = trampoline_swap_complete, /* swap_complete_ap_gated */ - [592] = trampoline_swap_complete_intent_gated, /* swap_complete_intent_gated */ - [593] = trampoline_abort_swap_ap_gated, /* abort_swap_ap_gated */ - [594] = trampoline_enable_backlight_message_ap_gated, - [595] = trampoline_nop, /* setSystemConsoleMode */ - [597] = trampoline_false, /* isDFBAllocated */ - [598] = trampoline_false, - [599] = trampoline_nop, /* find_swap_function_gated */ -}; - -/* - * Validate a callback record and, the first time each runtime callback whose - * length is only bounded arrives with a different length, report the length - * actually seen. Those reports are the first capture of these records. - */ -bool neo_iomfb_check_callback_h17p(struct neo_apple_dcp *neo_dcp, int tag, u32 in_len, - u32 out_len) -{ - const struct neo_dcp_callback_size *size; - - if (!neo_iomfb_validate_callback_h17p(tag, in_len, out_len)) - return false; - size = &callback_sizes[tag]; - if (size->minimum && - (in_len != size->in_len || out_len != size->out_len) && - !test_and_set_bit(tag, neo_dcp->sized_callbacks)) - dev_info(neo_dcp->dev, - "callback D%03d record is %#x/%#x bytes; its handler uses %#x/%#x\n", - tag, in_len, out_len, size->in_len, size->out_len); - return true; -} - -void neo_iomfb_start_h17p(struct neo_apple_dcp *neo_dcp) -{ - WRITE_ONCE(neo_dcp->pipe_enabled_h17p, false); - neo_dcp->cb_handlers = cb_handlers; - neo_dcp_start_signal(neo_dcp, false, neo_dcp_started, NULL); -} - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_KUNIT_TEST) -/* Returns the first handled tag without reply bounds, or -1. */ -int neo_iomfb_h17p_first_unbounded_callback(void) -{ - int tag; - - for (tag = 0; tag < IOMFB_MAX_CB; tag++) - if (cb_handlers[tag] && !callback_sizes[tag].valid) - return tag; - return -1; -} -#endif - -#undef DCP_FW_VER -#undef DCP_FW diff --git a/drivers/gpu/drm/apple-neo/iomfb_h17p.h b/drivers/gpu/drm/apple-neo/iomfb_h17p.h deleted file mode 100644 index 3f9c26c84a99ee..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_h17p.h +++ /dev/null @@ -1,132 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ -/* Copyright The Asahi Linux Contributors */ - -#ifndef __APPLE_IOMFB_H17P_H__ -#define __APPLE_IOMFB_H17P_H__ - -#include "version_utils.h" - -#define DCP_FW h17p -#define DCP_FW_VER DCP_FW_VERSION(26, 6, 0) - -#include "iomfb_template.h" - -static_assert(sizeof(struct neo_dcp_swap_h17p) == 0x58a); -static_assert(sizeof(struct neo_dcp_surface_h17p) == 0x22c); -static_assert(sizeof(struct neo_dcp_swap_submit_req_h17p) == 0xe9c); -static_assert(sizeof(struct neo_dcp_swap_submit_resp_h17p) == 0x0c); - -/* Owned by the RTKit receive thread, independently of DRM state lifetime. */ -struct neo_dcp_present_state_h17p { - u32 swap_id; - bool pending; - bool accepted; - /* The firmware aborted the present before its submit reply. */ - bool aborted; - /* The most recent aborted present, whose late completion is ignored. */ - bool has_last_aborted; - u32 last_aborted; -}; - -static inline bool -neo_dcp_present_begin_h17p(struct neo_dcp_present_state_h17p *state, u32 swap_id) -{ - if (state->pending) - return false; - state->swap_id = swap_id; - state->pending = true; - state->accepted = false; - state->aborted = false; - return true; -} - -static inline bool -neo_dcp_present_submit_h17p(struct neo_dcp_present_state_h17p *state, u32 swap_id, - bool accepted) -{ - if (!state->pending || state->accepted || state->swap_id != swap_id) - return false; - state->accepted = accepted; - if (!accepted || state->aborted) - state->pending = false; - return true; -} - -/* - * An aborted present never completes. Before its submit reply, the abort is - * recorded and the reply ends the present instead. - */ -static inline bool -neo_dcp_present_abort_h17p(struct neo_dcp_present_state_h17p *state, u32 swap_id) -{ - if (!state->pending || state->swap_id != swap_id) - return false; - if (state->accepted) { - state->pending = false; - state->accepted = false; - } else { - state->aborted = true; - } - state->has_last_aborted = true; - state->last_aborted = swap_id; - return true; -} - -/* Whether the firmware aborted @swap_id; a completion for it is stale. */ -static inline bool -neo_dcp_present_was_aborted_h17p(const struct neo_dcp_present_state_h17p *state, - u32 swap_id) -{ - return state->has_last_aborted && state->last_aborted == swap_id; -} - -static inline bool -neo_dcp_present_complete_h17p(struct neo_dcp_present_state_h17p *state, u32 swap_id) -{ - if (!state->pending || !state->accepted || state->swap_id != swap_id) - return false; - state->pending = false; - state->accepted = false; - return true; -} - -/* The H17P swap boundary is independent of the template's native layout. */ -struct neo_dcp_present_h17p { - u8 swap[0x588]; - struct neo_dcp_surface_h17p surf[SWAP_SURFACES]; - u8 tail[0x64]; -} __packed; - -static_assert(sizeof(struct neo_dcp_present_h17p) == 0xe9c); -static_assert(offsetof(struct neo_dcp_present_h17p, surf) == 0x588); -static_assert(offsetof(struct neo_dcp_present_h17p, tail) == 0xe38); - -struct neo_dcp_apply_property_h17p { - u32 property; - u32 value; -} __packed; - -static_assert(sizeof(struct neo_dcp_apply_property_h17p) == 0x8); - -static inline struct neo_dcp_apply_property_h17p neo_dcp_opaque_x_property_h17p(void) -{ - return (struct neo_dcp_apply_property_h17p) { - .property = 0x49, - .value = 0, - }; -} - -void neo_iomfb_encode_backlight_h17p(struct neo_dcp_present_h17p *wire, u32 nits, - u32 maximum, bool update); - -void neo_iomfb_serialize_present_h17p(struct neo_dcp_present_h17p *wire, - const struct neo_dcp_swap_submit_req_h17p *request); - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_KUNIT_TEST) -int neo_iomfb_h17p_first_unbounded_callback(void); -#endif - -#undef DCP_FW_VER -#undef DCP_FW - -#endif /* __APPLE_IOMFB_H17P_H__ */ diff --git a/drivers/gpu/drm/apple-neo/iomfb_h17p_test.c b/drivers/gpu/drm/apple-neo/iomfb_h17p_test.c deleted file mode 100644 index b506d133ed7b3e..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_h17p_test.c +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2026 Ryan Murray */ - -#include -#include -#include - -#include "iomfb_h17p.h" - -static void h17p_surface_wire_test(struct kunit *test) -{ - struct neo_dcp_swap_submit_req_h17p *request; - struct neo_dcp_present_h17p *wire; - struct neo_dcp_surface *surface; - u64 iova = 0x123456789aULL; - - request = kunit_kzalloc(test, sizeof(*request), GFP_KERNEL); - wire = kunit_kmalloc(test, sizeof(*wire), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, request); - KUNIT_ASSERT_NOT_NULL(test, wire); - surface = &request->surf[0].base; - memset(wire, 0xa5, sizeof(*wire)); - surface->is_premultiplied = 1; - surface->plane_cnt = 1; - surface->plane_cnt2 = 1; - surface->format = DCP_FORMAT_BGRA; - surface->xfer_func = DCP_XFER_FUNC_SDR; - surface->colorspace = DCP_COLORSPACE_NATIVE; - surface->stride = 10176; - surface->width = 2408; - surface->height = 1506; - surface->buf_size = 10176 * 1506; - surface->planes[0].width = 2408; - surface->planes[0].height = 1506; - surface->planes[0].stride = 10176; - surface->planes[0].size = 10176 * 1506; - request->surf_iova[0] = iova; - request->surf_null[0] = false; - request->surf_null[1] = true; - request->surf_null[2] = true; - request->surf_null[3] = true; - - neo_iomfb_serialize_present_h17p(wire, request); - - KUNIT_EXPECT_MEMEQ(test, wire->swap, &request->swap, sizeof(wire->swap)); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x593), - DCP_FORMAT_BGRA); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x59d), 10176U); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5a9), 2408U); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5ad), 1506U); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5b1), - 10176U * 1506U); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e1), 2408U); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5e5), 1506U); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f1), 10176U); - KUNIT_EXPECT_EQ(test, get_unaligned_le32((u8 *)wire + 0x5f5), - 10176U * 1506U); - KUNIT_EXPECT_EQ(test, get_unaligned_le64((u8 *)wire + 0xe38), iova); - KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 2], 0U); - KUNIT_EXPECT_EQ(test, wire->tail[sizeof(wire->tail) - 1], 0U); -} - -static void h17p_opaque_x_parameter_test(struct kunit *test) -{ - struct neo_dcp_apply_property_h17p property = neo_dcp_opaque_x_property_h17p(); - - KUNIT_EXPECT_EQ(test, property.property, 0x49U); - KUNIT_EXPECT_EQ(test, property.value, 0U); -} - -static void h17p_callback_bounds_test(struct kunit *test) -{ - /* Every installed handler is reachable without failing the session. */ - KUNIT_EXPECT_EQ(test, neo_iomfb_h17p_first_unbounded_callback(), -1); - - /* Startup observations stay exact. */ - KUNIT_EXPECT_TRUE(test, neo_iomfb_validate_callback_h17p(114, 0x1044, 0x1004)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(114, 0x1044, 0x1008)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(590, 0x734, 0x0)); - - /* Runtime callbacks must hold what their handlers read and write. */ - KUNIT_EXPECT_TRUE(test, neo_iomfb_validate_callback_h17p(209, 0x0, 0x8)); - KUNIT_EXPECT_TRUE(test, neo_iomfb_validate_callback_h17p(209, 0x4, 0xc)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(209, 0x0, 0x4)); - KUNIT_EXPECT_TRUE(test, neo_iomfb_validate_callback_h17p(593, 0x4, 0x0)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(593, 0x2, 0x0)); - KUNIT_EXPECT_TRUE(test, neo_iomfb_validate_callback_h17p(594, 0x4, 0x0)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(594, 0x0, 0x0)); - KUNIT_EXPECT_TRUE(test, neo_iomfb_validate_callback_h17p(585, 0xe4, 0xe0)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(585, 0xe0, 0xe0)); - - /* Tags without a handler or out of range never validate. */ - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(5, 0x0, 0x0)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(-1, 0x0, 0x0)); - KUNIT_EXPECT_FALSE(test, neo_iomfb_validate_callback_h17p(IOMFB_MAX_CB, 0x0, 0x0)); -} - -static void h17p_present_abort_test(struct kunit *test) -{ - struct neo_dcp_present_state_h17p state = {}; - - /* An abort after acceptance ends the present without a completion. */ - KUNIT_ASSERT_TRUE(test, neo_dcp_present_begin_h17p(&state, 7)); - KUNIT_ASSERT_TRUE(test, neo_dcp_present_submit_h17p(&state, 7, true)); - KUNIT_EXPECT_FALSE(test, neo_dcp_present_abort_h17p(&state, 6)); - KUNIT_EXPECT_TRUE(test, state.pending); - KUNIT_EXPECT_FALSE(test, neo_dcp_present_was_aborted_h17p(&state, 7)); - KUNIT_EXPECT_TRUE(test, neo_dcp_present_abort_h17p(&state, 7)); - KUNIT_EXPECT_FALSE(test, state.pending); - KUNIT_EXPECT_FALSE(test, neo_dcp_present_complete_h17p(&state, 7)); - /* A late completion for it is recognisable as stale, not unexpected. */ - KUNIT_EXPECT_TRUE(test, neo_dcp_present_was_aborted_h17p(&state, 7)); - KUNIT_EXPECT_FALSE(test, neo_dcp_present_was_aborted_h17p(&state, 6)); - KUNIT_EXPECT_FALSE(test, neo_dcp_present_abort_h17p(&state, 7)); - - /* An abort before the submit reply leaves the reply to finish it. */ - KUNIT_ASSERT_TRUE(test, neo_dcp_present_begin_h17p(&state, 8)); - KUNIT_EXPECT_TRUE(test, neo_dcp_present_abort_h17p(&state, 8)); - KUNIT_EXPECT_TRUE(test, state.pending); - KUNIT_EXPECT_TRUE(test, state.aborted); - KUNIT_ASSERT_TRUE(test, neo_dcp_present_submit_h17p(&state, 8, true)); - KUNIT_EXPECT_FALSE(test, state.pending); - KUNIT_EXPECT_FALSE(test, neo_dcp_present_complete_h17p(&state, 8)); - - /* The next present starts clean and completes normally. */ - KUNIT_ASSERT_TRUE(test, neo_dcp_present_begin_h17p(&state, 9)); - KUNIT_EXPECT_FALSE(test, state.aborted); - KUNIT_ASSERT_TRUE(test, neo_dcp_present_submit_h17p(&state, 9, true)); - KUNIT_EXPECT_TRUE(test, state.pending); - /* A stale completion for 8 neither matches nor disturbs present 9. */ - KUNIT_EXPECT_FALSE(test, neo_dcp_present_complete_h17p(&state, 8)); - KUNIT_EXPECT_TRUE(test, neo_dcp_present_was_aborted_h17p(&state, 8)); - KUNIT_EXPECT_TRUE(test, state.pending); - KUNIT_EXPECT_TRUE(test, neo_dcp_present_complete_h17p(&state, 9)); - KUNIT_EXPECT_FALSE(test, state.pending); - KUNIT_EXPECT_FALSE(test, neo_dcp_present_was_aborted_h17p(&state, 9)); -} - -static struct kunit_case h17p_present_cases[] = { - KUNIT_CASE(h17p_surface_wire_test), - KUNIT_CASE(h17p_opaque_x_parameter_test), - KUNIT_CASE(h17p_callback_bounds_test), - KUNIT_CASE(h17p_present_abort_test), - {} -}; - -static struct kunit_suite h17p_present_suite = { - .name = "apple-dcp-h17p-present", - .test_cases = h17p_present_cases, -}; - -kunit_test_suite(h17p_present_suite); - -MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/apple-neo/iomfb_internal.h b/drivers/gpu/drm/apple-neo/iomfb_internal.h deleted file mode 100644 index 65a564f3a32b8e..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_internal.h +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#include -#include - -#include "dcp-internal.h" - -struct neo_apple_dcp; - -typedef void (*neo_dcp_callback_t)(struct neo_apple_dcp *, void *, void *); - - -#define DCP_THUNK_VOID(func, handle) \ - static void func(struct neo_apple_dcp *neo_dcp, bool oob, neo_dcp_callback_t cb, \ - void *cookie) \ - { \ - neo_dcp_push(neo_dcp, oob, &neo_dcp_methods[handle], 0, 0, NULL, cb, cookie); \ - } - -#define DCP_THUNK_OUT(func, handle, T) \ - static void func(struct neo_apple_dcp *neo_dcp, bool oob, neo_dcp_callback_t cb, \ - void *cookie) \ - { \ - neo_dcp_push(neo_dcp, oob, &neo_dcp_methods[handle], 0, sizeof(T), NULL, cb, cookie); \ - } - -#define DCP_THUNK_IN(func, handle, T) \ - static void func(struct neo_apple_dcp *neo_dcp, bool oob, T *data, \ - neo_dcp_callback_t cb, void *cookie) \ - { \ - neo_dcp_push(neo_dcp, oob, &neo_dcp_methods[handle], sizeof(T), 0, data, cb, cookie); \ - } - -#define DCP_THUNK_INOUT(func, handle, T_in, T_out) \ - static void func(struct neo_apple_dcp *neo_dcp, bool oob, T_in *data, \ - neo_dcp_callback_t cb, void *cookie) \ - { \ - neo_dcp_push(neo_dcp, oob, &neo_dcp_methods[handle], sizeof(T_in), sizeof(T_out), data, \ - cb, cookie); \ - } - -#define IOMFB_THUNK_INOUT(name) \ - static void neo_iomfb_ ## name(struct neo_apple_dcp *neo_dcp, bool oob, \ - struct neo_iomfb_ ## name ## _req *data, \ - neo_dcp_callback_t cb, void *cookie) \ - { \ - neo_dcp_push(neo_dcp, oob, &neo_dcp_methods[iomfbep_ ## name], \ - sizeof(struct neo_iomfb_ ## name ## _req), \ - sizeof(struct neo_iomfb_ ## name ## _resp), \ - data, cb, cookie); \ - } - -/* - * Define type-safe trampolines. Define typedefs to enforce type-safety on the - * input data (so if the types don't match, gcc errors out). - */ - -#define TRAMPOLINE_VOID(func, handler) \ - static bool __maybe_unused func(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) \ - { \ - trace_neo_iomfb_callback(neo_dcp, tag, #handler); \ - handler(neo_dcp); \ - return true; \ - } - -#define TRAMPOLINE_IN(func, handler, T_in) \ - typedef void (*callback_##handler)(struct neo_apple_dcp *, T_in *); \ - \ - static bool __maybe_unused func(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) \ - { \ - callback_##handler cb = handler; \ - \ - trace_neo_iomfb_callback(neo_dcp, tag, #handler); \ - cb(neo_dcp, in); \ - return true; \ - } - -#define TRAMPOLINE_INOUT(func, handler, T_in, T_out) \ - typedef T_out (*callback_##handler)(struct neo_apple_dcp *, T_in *); \ - \ - static bool __maybe_unused func(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) \ - { \ - T_out *typed_out = out; \ - callback_##handler cb = handler; \ - \ - trace_neo_iomfb_callback(neo_dcp, tag, #handler); \ - *typed_out = cb(neo_dcp, in); \ - return true; \ - } - -#define TRAMPOLINE_OUT(func, handler, T_out) \ - static bool __maybe_unused func(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) \ - { \ - T_out *typed_out = out; \ - \ - trace_neo_iomfb_callback(neo_dcp, tag, #handler); \ - *typed_out = handler(neo_dcp); \ - return true; \ - } - -/* Call a DCP function given by a tag */ -void neo_dcp_push(struct neo_apple_dcp *neo_dcp, bool oob, const struct neo_dcp_method_entry *call, - u32 in_len, u32 out_len, void *data, neo_dcp_callback_t cb, - void *cookie); - -/* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ -int neo_dcp_parse_tag(char tag[4]); - -void neo_dcp_ack(struct neo_apple_dcp *neo_dcp, enum neo_dcp_context_id context); - -/* The user may own drm_display_mode, so we need to search for our copy */ -struct neo_dcp_display_mode *neo_lookup_mode(struct neo_apple_dcp *neo_dcp, - const struct drm_display_mode *mode); diff --git a/drivers/gpu/drm/apple-neo/iomfb_plane.h b/drivers/gpu/drm/apple-neo/iomfb_plane.h deleted file mode 100644 index f2a191de45bdbc..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_plane.h +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Copyright (C) The Asahi Linux Contributors - */ - - -#ifndef __APPLE_IOMFB_PLANE_H__ -#define __APPLE_IOMFB_PLANE_H__ - -#include - -#include - -#define DCP_SURF_MAX_PLANES 3 - -#define DCP_FORMAT_BGRA fourcc_code('A', 'R', 'G', 'B') -#define DCP_FORMAT_RGBA fourcc_code('A', 'B', 'G', 'R') - -#define DCP_FORMAT_W30R fourcc_code('r', '0', '3', 'w') // wide gamut packed 10-bit RGB without alpha -#define DCP_FORMAT_L10R fourcc_code('r', '0', '1', 'l') // full range packed 10-bit RGB with alpha - -#define DCP_FORMAT_420V fourcc_code('v', '0', '2', '4') // NV12 video range 2 plane 8-bit YCbCr -#define DCP_FORMAT_420F fourcc_code('f', '0', '2', '4') // NV12 full range 2 plane 8-bit YCbCr -#define DCP_FORMAT_422V fourcc_code('v', '2', '2', '4') // NV16 video range 2 plane 8-bit YCbCr -#define DCP_FORMAT_422F fourcc_code('f', '2', '2', '4') // NV16 full range 2 plane 8-bit YCbCr -#define DCP_FORMAT_444V fourcc_code('v', '4', '4', '4') // NV24 video range 2 plane 8-bit YCbCr -#define DCP_FORMAT_444F fourcc_code('f', '4', '4', '4') // NV24 full range 2 plane 8-bit YCbCr - -#define DCP_FORMAT_X420 fourcc_code('0', '2', '4', 'x') // P010 video range 2 plane 10-bit YCbCR -#define DCP_FORMAT_X422 fourcc_code('2', '2', '4', 'x') // P210 video range 2 plane 10-bit YCbCR -#define DCP_FORMAT_X444 fourcc_code('4', '4', '4', 'x') // P410 video range 2 plane 10-bit YCbCR - -#define DCP_FORMAT_XF20 fourcc_code('0', '2', 'f', 'x') // P010 full range 2 plane 10-bit YCbCR -#define DCP_FORMAT_XF22 fourcc_code('2', '2', 'f', 'x') // P210 full range 2 plane 10-bit YCbCR -#define DCP_FORMAT_XF44 fourcc_code('4', '4', 'f', 'x') // P410 full range 2 plane 10-bit YCbCR - -enum neo_dcp_colorspace { - DCP_COLORSPACE_BG_SRGB = 0, - DCP_COLORSPACE_BT601 = 1, - DCP_COLORSPACE_BT709 = 2, - DCP_COLORSPACE_BG_BT2020 = 9, - DCP_COLORSPACE_NATIVE = 12, -}; - -enum neo_dcp_xfer_func { - DCP_XFER_FUNC_BT601 = 1, - DCP_XFER_FUNC_BT1886 = 2, - DCP_XFER_FUNC_SDR = 13, - DCP_XFER_FUNC_HDR = 16, -}; - -struct neo_dcp_rect { - u32 x; - u32 y; - u32 w; - u32 h; -} __packed; - -/* Information describing a plane of a planar compressed surface */ -struct neo_dcp_plane_info { - u32 width; - u32 height; - u32 base; - u32 offset; - u32 stride; - u32 size; - u16 tile_size; - u8 tile_w; - u8 tile_h; - u32 unk[13]; -} __packed; - -struct neo_dcp_component_types { - u8 count; - u8 types[7]; -} __packed; - -/* Information describing a surface */ -struct neo_dcp_surface { - u8 is_tiled; - u8 is_tearing_allowed; - u8 is_premultiplied; - u32 plane_cnt; - u32 plane_cnt2; - u32 format; /* DCP fourcc */ - u32 ycbcr_matrix; - u8 xfer_func; - u8 colorspace; - u32 stride; - u16 pix_size; - u8 pel_w; - u8 pel_h; - u32 offset; - u32 width; - u32 height; - u32 buf_size; - u64 protection_opts; - u32 surface_id; - struct neo_dcp_component_types comp_types[DCP_SURF_MAX_PLANES]; - u64 has_comp; - struct neo_dcp_plane_info planes[DCP_SURF_MAX_PLANES]; - u64 has_planes; - u32 compression_info[DCP_SURF_MAX_PLANES][13]; - u64 has_compr_info; - u32 unk_num; - u32 unk_denom; -} __packed; - -#endif /* __APPLE_IOMFB_PLANE_H__ */ diff --git a/drivers/gpu/drm/apple-neo/iomfb_template.c b/drivers/gpu/drm/apple-neo/iomfb_template.c deleted file mode 100644 index def92694ff5726..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_template.c +++ /dev/null @@ -1,2605 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Copyright 2021 Alyssa Rosenzweig - * Copyright The Asahi Linux Contributors - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "dcp.h" -#include "dcp-internal.h" -#include "iomfb.h" -#include "iomfb_internal.h" -#include "parser.h" -#include "trace.h" -#include "version_utils.h" - -/* Register defines used in bandwidth setup structure */ -#define REG_DOORBELL_BIT(idx) (2 + (idx)) - -static_assert(offsetof(struct DCP_FW_NAME(neo_dcp_swap), timestamp[6]) == 0x30); -static_assert(offsetof(struct DCP_FW_NAME(neo_dcp_swap), flags1) == 0x40); - -struct neo_dcp_wait_cookie { - struct kref refcount; - struct completion done; - u32 status; -}; - -static void release_wait_cookie(struct kref *ref) -{ - struct neo_dcp_wait_cookie *cookie; - cookie = container_of(ref, struct neo_dcp_wait_cookie, refcount); - - kfree(cookie); -} - -DCP_THUNK_OUT(neo_iomfb_a131_pmu_service_matched, iomfbep_a131_pmu_service_matched, u32); -DCP_THUNK_OUT(neo_iomfb_a132_backlight_service_matched, iomfbep_a132_backlight_service_matched, u32); -DCP_THUNK_OUT(neo_iomfb_a358_vi_set_temperature_hint, iomfbep_a358_vi_set_temperature_hint, u32); - -IOMFB_THUNK_INOUT(set_matrix); -IOMFB_THUNK_INOUT(get_color_remap_mode); -IOMFB_THUNK_INOUT(last_client_close); -IOMFB_THUNK_INOUT(abort_swaps_dcp); - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER -static void neo_dcp_swap_submit(struct neo_apple_dcp *neo_dcp, bool oob, - struct neo_dcp_swap_submit_req_h17p *request, - neo_dcp_callback_t cb, void *cookie) -{ - const void *data = request; - - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - neo_iomfb_serialize_present_h17p(&neo_dcp->present_h17p, request); - if (!neo_iomfb_apply_backlight_h17p(neo_dcp, request, &neo_dcp->present_h17p)) { - if (cb) - cb(neo_dcp, NULL, cookie); - return; - } - data = &neo_dcp->present_h17p; - } - - neo_dcp_push(neo_dcp, oob, &neo_dcp_methods[dcpep_swap_submit], sizeof(*request), - sizeof(struct neo_dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); -} -#else -DCP_THUNK_INOUT(neo_dcp_swap_submit, dcpep_swap_submit, - struct DCP_FW_NAME(neo_dcp_swap_submit_req), - struct DCP_FW_NAME(neo_dcp_swap_submit_resp)); -#endif - -DCP_THUNK_INOUT(neo_dcp_swap_start, dcpep_swap_start, struct DCP_FW_NAME(neo_dcp_swap_start_req), - struct DCP_FW_NAME(neo_dcp_swap_start_resp)); - -DCP_THUNK_INOUT(neo_dcp_set_power_state, dcpep_set_power_state, - struct DCP_FW_NAME(neo_dcp_set_power_state_req), - struct neo_dcp_set_power_state_resp); - -/* - * H17P firmware expects set_power_state (A473) on the out-of-band command - * channel. Sent on the ordinary command channel it is never answered, which - * strands the power transition and every later call behind it. - */ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER -#define DCP_POWER_OOB true -#else -#define DCP_POWER_OOB false -#endif - -/* - * H17P firmware is attached to a display the bootloader has already brought - * up, and it does not accept set_digital_out_mode (A413) for that display: - * the call fails (0x8000000b) and the timing generator never starts, so there - * is no vblank and no swap completion. Keep the mode the bootloader - * programmed, which is the panel's native mode, instead of requesting one. - */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_INHERIT_BOOT_MODE true -#else -#define DCP_INHERIT_BOOT_MODE false -#endif - -/* - * abort_swaps_dcp and last_client_close have no verified H17P method numbers, - * and the H17P table leaves them out. Rather than send the firmware a record - * it would decode as some other call, power-off and sleep skip both and go - * straight to set_power_state, whose number and sizes are known. Power-off - * still sends its clear swap first. - */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_HAS_CLIENT_TEARDOWN false -#else -#define DCP_HAS_CLIENT_TEARDOWN true -#endif - -DCP_THUNK_INOUT(neo_dcp_set_digital_out_mode, dcpep_set_digital_out_mode, - struct neo_dcp_set_digital_out_mode_req, u32); - -DCP_THUNK_INOUT(neo_dcp_set_display_device, dcpep_set_display_device, u32, u32); - -DCP_THUNK_OUT(neo_dcp_set_display_refresh_properties, - dcpep_set_display_refresh_properties, u32); - -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -DCP_THUNK_INOUT(neo_dcp_late_init_signal, dcpep_late_init_signal, u32, u32); -#else -DCP_THUNK_OUT(neo_dcp_late_init_signal, dcpep_late_init_signal, u32); -#endif -DCP_THUNK_IN(neo_dcp_flush_supports_power, dcpep_flush_supports_power, u32); -DCP_THUNK_OUT(neo_dcp_create_default_fb, dcpep_create_default_fb, u32); -DCP_THUNK_OUT(neo_dcp_start_signal, dcpep_start_signal, u32); -DCP_THUNK_VOID(neo_dcp_setup_video_limits, dcpep_setup_video_limits); -DCP_THUNK_VOID(neo_dcp_set_create_dfb, dcpep_set_create_dfb); - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * Default-framebuffer queries issued from inside start_hardware_boot (D121). - * The getters take a zeroed input of the expected size; the firmware only - * checks the size. - */ -struct neo_dcp_dfb_info_h17p { - u8 data[0x2c]; -} __packed; - -struct neo_dcp_dfb_layout_req_h17p { - u8 data[0xc]; -} __packed; - -struct neo_dcp_dfb_dimensions_h17p { - u32 width; - u32 height; -} __packed; - -DCP_THUNK_INOUT(neo_dcp_get_dfb_compression_info, dcpep_get_dfb_compression_info, - u64, u64); -DCP_THUNK_INOUT(neo_dcp_get_dfb_info, dcpep_get_dfb_info, - struct neo_dcp_dfb_info_h17p, struct neo_dcp_dfb_info_h17p); -DCP_THUNK_INOUT(neo_dcp_get_dfb_layout, dcpep_get_dfb_layout, - struct neo_dcp_dfb_layout_req_h17p, u64); -DCP_THUNK_INOUT(neo_dcp_get_dfb_state, dcpep_get_dfb_state, u32, u64); -DCP_THUNK_IN(neo_dcp_set_dfb_dimensions, dcpep_set_dfb_dimensions, - struct neo_dcp_dfb_dimensions_h17p); -DCP_THUNK_VOID(neo_dcp_commit_dfb_info, dcpep_commit_dfb_info); -DCP_THUNK_OUT(neo_dcp_dfb_query, dcpep_dfb_query, u32); -DCP_THUNK_OUT(neo_dcp_dfb_ready_query, dcpep_dfb_ready_query, u32); - -/* - * Pipe-configuration calls the firmware expects between the boot chain and - * the first swap. Most of them have no known name; they are identified by - * method number and wire size. - */ -struct neo_dcp_pipe_cfg_415 { - u8 data[0xc]; -} __packed; - -/* - * A031 is update_notify_clients_dcp(u32[27]): an array of the DCP - * notification clients the AP wants delivered. Clients that are not - * registered here never send the callbacks they gate. - */ -struct neo_dcp_pipe_cfg_031 { - __le32 clients[27]; -} __packed; -static_assert(sizeof(struct neo_dcp_pipe_cfg_031) == 0x6c); - -DCP_THUNK_INOUT(neo_dcp_pipe_cfg_415, dcpep_pipe_cfg_415, - struct neo_dcp_pipe_cfg_415, struct neo_dcp_pipe_cfg_415); -DCP_THUNK_IN(neo_dcp_pipe_cfg_031, dcpep_pipe_cfg_031, struct neo_dcp_pipe_cfg_031); -DCP_THUNK_INOUT(neo_dcp_pipe_cfg_414, dcpep_pipe_cfg_414, u64, u64); -DCP_THUNK_OUT(neo_dcp_pipe_query_478, dcpep_pipe_query_478, u32); -DCP_THUNK_OUT(neo_dcp_pipe_query_474, dcpep_pipe_query_474, u32); -DCP_THUNK_IN(neo_dcp_pipe_set_476, dcpep_pipe_set_476, u32); -DCP_THUNK_INOUT(neo_dcp_pipe_set_428, dcpep_pipe_set_428, u32, u32); -#endif - -DCP_THUNK_VOID(neo_dcp_first_client_open, dcpep_first_client_open); - -DCP_THUNK_INOUT(neo_dcp_set_parameter_dcp, dcpep_set_parameter_dcp, - struct neo_dcp_set_parameter_dcp, u32); - -DCP_THUNK_INOUT(neo_dcp_enable_disable_video_power_savings, - dcpep_enable_disable_video_power_savings, u32, int); - -DCP_THUNK_OUT(neo_dcp_is_main_display, dcpep_is_main_display, u32); - -/* DCP callback handlers */ -static void dcpep_cb_nop(struct neo_apple_dcp *neo_dcp) -{ - /* No operation */ -} - -static u8 dcpep_cb_true(struct neo_apple_dcp *neo_dcp) -{ - return true; -} - -static u8 dcpep_cb_false(struct neo_apple_dcp *neo_dcp) -{ - return false; -} - -static u32 dcpep_cb_zero(struct neo_apple_dcp *neo_dcp) -{ - return 0; -} - -static void dcpep_cb_swap_complete(struct neo_apple_dcp *neo_dcp, - struct DCP_FW_NAME(dc_swap_complete_resp) *resp) -{ - ktime_t now = ktime_get(); - bool retire = true; - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !neo_dcp_present_complete_h17p(&neo_dcp->present_state_h17p, resp->swap_id)) { - /* - * Abort semantics are not captured on H17P. If an aborted - * present completes after all, its framebuffers were already - * unarmed and its event signalled; record it and carry on. - */ - if (neo_dcp_present_was_aborted_h17p(&neo_dcp->present_state_h17p, - resp->swap_id)) { - dev_warn_ratelimited(neo_dcp->dev, - "completion for aborted present %u ignored\n", - resp->swap_id); - return; - } - dev_err(neo_dcp->dev, "unexpected present completion %u\n", resp->swap_id); - neo_dcp->crashed = true; - return; - } - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - retire = neo_iomfb_present_complete_h17p(neo_dcp); -#endif - trace_neo_iomfb_swap_complete(neo_dcp, resp->swap_id); - neo_dcp->last_swap_id = resp->swap_id; - neo_dcp_swap_watchdog_complete(neo_dcp); - if (retire) { - neo_dcp_release_retained_framebuffers(neo_dcp, resp->swap_id); - neo_dcp_drm_crtc_page_flip(neo_dcp, now); - } - if (READ_ONCE(neo_dcp->crc_enabled)) { -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - neo_iomfb_queue_crc_h17p(neo_dcp, resp->swap_id); - return; - } -#endif - u32 crc32 = 0; - drm_crtc_add_crc_entry(&neo_dcp->crtc->base, true, resp->swap_id, &crc32); - } -} - -/* special */ -static void complete_vi_set_temperature_hint(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - // ack D100 cb_match_pmu_service - neo_dcp_ack(neo_dcp, DCP_CONTEXT_CB); -} - -static bool iomfbep_cb_match_pmu_service(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) -{ - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - neo_iomfb_a358_vi_set_temperature_hint(neo_dcp, false, - complete_vi_set_temperature_hint, - NULL); - - // return false for deferred ACK - return false; -} - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P asks the AP to make the default framebuffer surface (D582). The - * firmware expects the AP to register a surface from inside that callback, - * with a nested A472 carrying a 0x22c-byte surface descriptor, and only then - * to report success. Without it the firmware has no surface, programs the - * display engine from a null base and the piodma DART faults. - */ -struct neo_dcp_h17p_dfb_surface_req { - u8 surface[0x22c]; - u8 surface_null; - u8 padding[3]; -} __packed; -static_assert(sizeof(struct neo_dcp_h17p_dfb_surface_req) == 0x230); - -/* - * A472 has a 4-byte output; with out_len 0 the firmware has nowhere to write - * its result and stops answering. - */ -DCP_THUNK_INOUT(neo_dcp_register_dfb_surface, dcpep_register_dfb_surface, - struct neo_dcp_h17p_dfb_surface_req, u32); - -static void neo_dcp_ack_dfb_surface(struct neo_apple_dcp *neo_dcp) -{ - struct neo_dcp_channel *ch = &neo_dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - neo_dcp_ack(neo_dcp, DCP_CONTEXT_CB); -} - -/* - * Fill the default surface descriptor. Fields not set here are zero. The - * format at +0x0b is the fourcc 'b3a8', stored byte-reversed; +0x35 is the - * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose - * meaning is not known. - */ -static void neo_dcp_fill_dfb_surface(struct neo_apple_dcp *neo_dcp, u8 *s) -{ - s[0x002] = 0x01; - memcpy(s + 0x00b, "8a3b", 4); - s[0x019] = 0x01; - s[0x01b] = 0x01; - s[0x01c] = 0x01; - s[0x02b] = 0x4d; - s[0x02c] = 0x01; - s[0x035] = 0x02; - s[0x051] = 0x01; - s[0x149] = 0x01; - - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - /* Internal H17P default registration differs from the older profile. */ - memcpy(s + 11, "ARGB", 4); - put_unaligned_le32(0x00e44000, s + 41); - s[53] = BIT(2); - } -} - -/* - * A471 (update_dfb) is never sent: on H17P it makes the coprocessor take an - * AXI read error. Scanout does not need it, since every swap carries its own - * surface. Just acknowledge the callback. - */ -static void complete_dfb_surface(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - neo_dcp_ack_dfb_surface(neo_dcp); -} - -static bool iomfbep_cb_create_dfb_surface(struct neo_apple_dcp *neo_dcp, int tag, - void *out, void *in) -{ - struct neo_dcp_h17p_dfb_surface_req *req; - - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - - /* - * The request is too large for the stack this deep in the RTKit - * receive work (rx work -> iomfb_recv_msg -> dcpep_handle_cb -> here -> - * dcp_push). dcp_push() copies the payload into shared memory before - * returning, so a short-lived allocation is enough. - */ - req = kzalloc_obj(*req); - if (!req) - return true; - - neo_dcp_fill_dfb_surface(neo_dcp, req->surface); - - neo_dcp_register_dfb_surface(neo_dcp, false, req, complete_dfb_surface, NULL); - kfree(req); - - /* deferred ACK: the A472 completion reports success for us */ - return false; -} -#endif - -static void complete_pmu_service_matched(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_channel *ch = &neo_dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - - // ack D206 cb_match_pmu_service_2 - neo_dcp_ack(neo_dcp, DCP_CONTEXT_CB); -} - -static bool iomfbep_cb_match_pmu_service_2(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) -{ - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - - neo_iomfb_a131_pmu_service_matched(neo_dcp, false, complete_pmu_service_matched, - out); - - // return false for deferred ACK - return false; -} - -static void complete_backlight_service_matched(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_channel *ch = &neo_dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - - // ack D206 cb_match_backlight_service - neo_dcp_ack(neo_dcp, DCP_CONTEXT_CB); -} - -static bool iomfbep_cb_match_backlight_service(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) -{ - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - - if (!neo_dcp_has_panel(neo_dcp)) { - u8 *succ = out; - *succ = true; - return true; - } - - neo_iomfb_a132_backlight_service_matched(neo_dcp, false, complete_backlight_service_matched, out); - - // return false for deferred ACK - return false; -} - -static void neo_iomfb_cb_pr_publish(struct neo_apple_dcp *neo_dcp, struct neo_iomfb_property *prop) -{ - switch (prop->id) { - case IOMFB_PROPERTY_NITS: - { -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* Measured H17P takeover comes from the powerlog hint interface. */ - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - break; -#endif - if (neo_dcp_has_panel(neo_dcp)) { - neo_dcp->brightness.nits = prop->value / neo_dcp->brightness.scale; - /* notify backlight device of the initial brightness */ - if (!neo_dcp->brightness.bl_dev && neo_dcp->brightness.maximum > 0) - schedule_work(&neo_dcp->bl_register_wq); - trace_neo_iomfb_brightness(neo_dcp, prop->value); - } - break; - } - default: - dev_dbg(neo_dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); - } -} - -static struct neo_dcp_get_uint_prop_resp -dcpep_cb_get_uint_prop(struct neo_apple_dcp *neo_dcp, struct neo_dcp_get_uint_prop_req *req) -{ - struct neo_dcp_get_uint_prop_resp resp = (struct neo_dcp_get_uint_prop_resp){ - .value = 0 - }; - - if (neo_dcp->panel.has_mini_led && - memcmp(req->obj, "SUMP", sizeof(req->obj)) == 0) { /* "PMUS */ - if (strncmp(req->key, "Temperature", sizeof(req->key)) == 0) { - /* - * TODO: value from j314c, find out if it is temperature in - * centigrade C and which temperature sensor reports it - */ - resp.value = 3029; - resp.ret = true; - } - } - - return resp; -} - -static u8 iomfbep_cb_sr_set_property_int(struct neo_apple_dcp *neo_dcp, - struct neo_iomfb_sr_set_property_int_req *req) -{ - if (memcmp(req->obj, "FMOI", sizeof(req->obj)) == 0) { /* "IOMF */ - if (strncmp(req->key, "Brightness_Scale", sizeof(req->key)) == 0) { - if (!req->value_null) - neo_dcp->brightness.scale = req->value; - } - } - - return 1; -} - -static void iomfbep_cb_set_fx_prop(struct neo_apple_dcp *neo_dcp, struct neo_iomfb_set_fx_prop_req *req) -{ - // TODO: trace this, see if there properties which needs to used later -} - -/* - * Callback to map a buffer allocated with allocate_buf for PIODMA usage. - * PIODMA is separate from the main DCP and uses own IOVA space on a dedicated - * stream of the display DART, rather than the expected DCP DART. - */ -static bool dcpep_map_piodma(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_map_buf_req *req, - struct DCP_FW_NAME(neo_dcp_map_buf_resp) * resp) -{ - struct neo_dcp_mem_descriptor *memdesc; - struct sg_table *map; - size_t size; - ssize_t ret; - - if (req->buffer >= ARRAY_SIZE(neo_dcp->memdesc)) - goto reject; - - memdesc = &neo_dcp->memdesc[req->buffer]; - map = &memdesc->map; - size = ALIGN(memdesc->size, SZ_16K); - - if (!test_bit(req->buffer, neo_dcp->memdesc_map) || !map->sgl || - memdesc->piodma_mapped || !neo_dcp->iommu_dom) - goto reject; - - /* use the piodma iommu domain to map against the right IOMMU */ - ret = iommu_map_sgtable(neo_dcp->iommu_dom, memdesc->dva, map, - IOMMU_READ | IOMMU_WRITE); - - if (ret != (ssize_t)size) { - dev_err(neo_dcp->dev, - "iommu_map_sgtable() returned %zd instead of %zu\n", - ret, size); - if (ret > 0) - iommu_unmap(neo_dcp->iommu_dom, memdesc->dva, ret); - goto reject; - } - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - memdesc->piodma_mapped = true; - - /* the H17P reply echoes the buffer index */ - *resp = (struct DCP_FW_NAME(neo_dcp_map_buf_resp)) { - .buffer = req->buffer, - .dva = memdesc->dva, - }; -#else - *resp = (struct DCP_FW_NAME(neo_dcp_map_buf_resp)) { .dva = memdesc->dva }; -#endif - return true; - -reject: - dev_err(neo_dcp->dev, "denying map of invalid buffer %llx for piodma\n", - req->buffer); - return false; -} - -static struct DCP_FW_NAME(neo_dcp_map_buf_resp) dcpep_cb_map_piodma(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_map_buf_req *req) -{ - struct DCP_FW_NAME(neo_dcp_map_buf_resp) resp = { 0 }; - - if (dcpep_map_piodma(neo_dcp, req, &resp)) - return resp; - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - resp.buffer = req->buffer; -#else - resp.ret = EINVAL; -#endif - return resp; -} - -static bool dcpep_unmap_piodma(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_unmap_buf_resp *request) -{ - struct neo_dcp_mem_descriptor *memdesc; - size_t size, unmapped; - - if (request->buffer >= ARRAY_SIZE(neo_dcp->memdesc)) { - dev_warn(neo_dcp->dev, "unmap request for out of range buffer %llu\n", - request->buffer); - return false; - } - - memdesc = &neo_dcp->memdesc[request->buffer]; - - if (!test_bit(request->buffer, neo_dcp->memdesc_map) || !memdesc->buf || - !neo_dcp->iommu_dom) - goto not_mapped; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (!memdesc->piodma_mapped) - goto not_mapped; -#endif - - if (memdesc->dva != request->dva) { - dev_warn(neo_dcp->dev, - "unmap buffer %llu address mismatch memdesc.dva:%llx dva:%llx\n", - request->buffer, - memdesc->dva, request->dva); - return false; - } - - /* use the piodma iommu domain to unmap from the right IOMMU */ - size = ALIGN(memdesc->size, SZ_16K); - unmapped = iommu_unmap(neo_dcp->iommu_dom, memdesc->dva, size); - if (unmapped != size) { - dev_err(neo_dcp->dev, "iommu_unmap() returned %zu instead of %zu\n", - unmapped, size); - return false; - } - memdesc->piodma_mapped = false; - return true; - -not_mapped: - dev_warn(neo_dcp->dev, - "unmap for non-mapped buffer %llu iova:0x%08llx\n", - request->buffer, request->dva); - return false; -} - -static void dcpep_cb_unmap_piodma(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_unmap_buf_resp *request) -{ - dcpep_unmap_piodma(neo_dcp, request); -} - -/* - * Allocate an IOVA contiguous buffer mapped to the DCP. The buffer need not be - * physically contiguous, however we should save the sgtable in case the - * buffer needs to be later mapped for PIODMA. - */ -static struct neo_dcp_allocate_buffer_resp -dcpep_cb_allocate_buffer(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_allocate_buffer_req *req) -{ - struct neo_dcp_allocate_buffer_resp resp = { 0 }; - struct neo_dcp_mem_descriptor allocated = { 0 }; - size_t size, rounded; - u32 id; - int ret; - - /* Validate both wire-size and IOMMU-page rounding before allocating. */ - if (!req->size || req->size > SIZE_MAX || - check_add_overflow((size_t)req->size, (size_t)4095, &rounded)) - return resp; - allocated.size = round_down(rounded, 4096); - if (check_add_overflow(allocated.size, (size_t)SZ_16K - 1, &rounded)) - return resp; - size = round_down(rounded, SZ_16K); - - id = find_first_zero_bit(neo_dcp->memdesc_map, DCP_MAX_MAPPINGS); - - if (id >= DCP_MAX_MAPPINGS) { - dev_warn(neo_dcp->dev, "DCP overflowed mapping table, ignoring\n"); - return resp; - } - - allocated.buf = dma_alloc_coherent(neo_dcp->dev, size, &allocated.dva, - GFP_KERNEL); - if (!allocated.buf) - return resp; - - ret = dma_get_sgtable(neo_dcp->dev, &allocated.map, allocated.buf, - allocated.dva, size); - if (ret) { - dma_free_coherent(neo_dcp->dev, size, allocated.buf, allocated.dva); - return resp; - } - - /* Callbacks are serialized; publish only a fully initialized descriptor. */ - neo_dcp->memdesc[id] = allocated; - set_bit(id, neo_dcp->memdesc_map); - resp.mem_desc_id = id; - resp.dva_size = allocated.size; - resp.dva = allocated.dva; - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * Linux allocation traces correlate the first H17P reply address with - * the first SG page. The remaining pages need not be contiguous. - */ - if (allocated.map.sgl) - resp.paddr = sg_phys(allocated.map.sgl); -#endif - - return resp; -} - -static u8 dcpep_cb_release_mem_desc(struct neo_apple_dcp *neo_dcp, u32 *mem_desc_id) -{ - struct neo_dcp_mem_descriptor *memdesc; - size_t size, unmapped; - u32 id = *mem_desc_id; - - if (id >= DCP_MAX_MAPPINGS) { - dev_warn(neo_dcp->dev, - "unmap request for out of range mem_desc_id %u", id); - return 0; - } - - if (!test_bit(id, neo_dcp->memdesc_map)) { - dev_warn(neo_dcp->dev, "unmap request for unused mem_desc_id %u\n", - id); - return 0; - } - - memdesc = &neo_dcp->memdesc[id]; - size = ALIGN(memdesc->size, SZ_16K); - - /* - * Tear down any translation still standing before the pages go back to - * the allocator. The firmware is expected to unmap piodma first, but - * if it releases a descriptor without doing so we would hand the pages - * back while the DART still resolves them -- the display engine would - * then read whoever owns them next. Only the H17P map_piodma handler - * sets piodma_mapped, so older firmware is unaffected. - */ - if (memdesc->piodma_mapped) { - dev_warn(neo_dcp->dev, - "releasing buffer %u while still mapped for piodma; unmapping first\n", - id); - if (!neo_dcp->iommu_dom) - return 0; - unmapped = iommu_unmap(neo_dcp->iommu_dom, memdesc->dva, size); - if (unmapped != size) { - dev_err(neo_dcp->dev, - "failed to unmap buffer %u before release: %zu of %zu\n", - id, unmapped, size); - return 0; - } - memdesc->piodma_mapped = false; - } - clear_bit(id, neo_dcp->memdesc_map); - - if (memdesc->buf) { - sg_free_table(&memdesc->map); - dma_free_coherent(neo_dcp->dev, size, memdesc->buf, memdesc->dva); - memdesc->buf = NULL; - memset(&memdesc->map, 0, sizeof(memdesc->map)); - } else if (memdesc->size) { - /* A register range mapped by map_physical */ - dma_unmap_resource(neo_dcp->dev, memdesc->dva, memdesc->size, - DMA_BIDIRECTIONAL, 0); - memdesc->reg = 0; - } - memdesc->dva = 0; - - memdesc->size = 0; - - return 1; -} - -/* Validate that the specified region is a display register */ -static bool is_disp_register(struct neo_apple_dcp *neo_dcp, u64 start, u64 end) -{ - int i; - - for (i = 0; i < neo_dcp->nr_disp_registers; ++i) { - struct resource *r = neo_dcp->disp_registers[i]; - - if ((start >= r->start) && (end <= r->end)) - return true; - } - - return false; -} - -/* - * Map contiguous physical memory into the DCP's address space. The firmware - * uses this to map the display registers we advertise in - * sr_map_device_memory_with_index, so we bounds check against that to guard - * safe against malicious coprocessors. - */ -static struct neo_dcp_map_physical_resp -dcpep_cb_map_physical(struct neo_apple_dcp *neo_dcp, struct neo_dcp_map_physical_req *req) -{ - u64 size, end; - dma_addr_t dva; - u32 id; - - /* Both values come from the coprocessor; reject wrapping ranges. */ - if (!req->size || check_add_overflow(req->size, 4095ULL, &size) || - check_add_overflow(req->paddr, round_down(size, 4096) - 1, &end) || - !is_disp_register(neo_dcp, req->paddr, end)) { - dev_err(neo_dcp->dev, "refusing to map phys address %llx size %llx\n", - req->paddr, req->size); - return (struct neo_dcp_map_physical_resp){}; - } - size = round_down(size, 4096); - - id = find_first_zero_bit(neo_dcp->memdesc_map, DCP_MAX_MAPPINGS); - if (id >= DCP_MAX_MAPPINGS) { - dev_warn(neo_dcp->dev, "DCP overflowed mapping table, ignoring\n"); - return (struct neo_dcp_map_physical_resp){}; - } - - dva = dma_map_resource(neo_dcp->dev, req->paddr, size, DMA_BIDIRECTIONAL, 0); - if (dma_mapping_error(neo_dcp->dev, dva)) { - dev_err(neo_dcp->dev, "failed to map phys address %llx size %llx\n", - req->paddr, size); - return (struct neo_dcp_map_physical_resp){}; - } - - set_bit(id, neo_dcp->memdesc_map); - neo_dcp->memdesc[id].size = size; - neo_dcp->memdesc[id].reg = req->paddr; - neo_dcp->memdesc[id].dva = dva; - - return (struct neo_dcp_map_physical_resp){ - .dva_size = size, - .mem_desc_id = id, - .dva = dva, - }; -} - -static u64 dcpep_cb_get_frequency(struct neo_apple_dcp *neo_dcp) -{ - return clk_get_rate(neo_dcp->clk); -} - -static struct DCP_FW_NAME(neo_dcp_map_reg_resp) dcpep_cb_map_reg(struct neo_apple_dcp *neo_dcp, - struct DCP_FW_NAME(neo_dcp_map_reg_req) *req) -{ - if (req->index >= neo_dcp->nr_disp_registers) { - dev_warn(neo_dcp->dev, "attempted to read invalid reg index %u\n", - req->index); - - return (struct DCP_FW_NAME(neo_dcp_map_reg_resp)){ .ret = 1 }; - } else { - struct resource *rsrc = neo_dcp->disp_registers[req->index]; -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P reaches these apertures physically, not through its - * DART, and expects dva == addr in the reply. Trying to - * dma_map_resource() them instead fails and hands the firmware - * DMA_MAPPING_ERROR as a device address. - */ - dma_addr_t dva = rsrc->start; -#elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - dma_addr_t dva = dma_map_resource(neo_dcp->dev, rsrc->start, resource_size(rsrc), - DMA_BIDIRECTIONAL, 0); - WARN_ON(dva == DMA_MAPPING_ERROR); -#endif - - return (struct DCP_FW_NAME(neo_dcp_map_reg_resp)){ - .addr = rsrc->start, - .length = resource_size(rsrc), -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - .dva = dva, -#endif - }; - } -} - -static struct neo_dcp_read_edt_data_resp -dcpep_cb_read_edt_data(struct neo_apple_dcp *neo_dcp, struct neo_dcp_read_edt_data_req *req) -{ - /* Observe boot-property requests without inventing firmware timings. */ - if (neo_dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(neo_dcp->dev, - "read_edt_data key=%.*s count=%u default0=%#x ret=0\n", - (int)sizeof(req->key), req->key, req->count, - req->value[0]); - - return (struct neo_dcp_read_edt_data_resp){ - .value[0] = req->value[0], - .ret = 0, - }; -} - -static void iomfbep_cb_enable_backlight_message_ap_gated(struct neo_apple_dcp *neo_dcp, - u8 *enabled) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* H17P carries the level only in presents; ask for one more. */ - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - if (neo_dcp_has_panel(neo_dcp) && neo_dcp_backlight_active(neo_dcp) && - neo_dcp_backlight_resend(neo_dcp)) - schedule_work(&neo_dcp->bl_update_wq); - return; - } -#endif - /* - * update backlight brightness on next swap, on non mini-LED displays - * DCP seems to set an invalid iDAC value after coming out of DPMS. - * syslog: "[BrightnessLCD.cpp:743][AFK]nitsToDBV: iDAC out of range" - */ - neo_dcp->brightness.update = true; - /* Backlight handling is only set up for panels described in the DT. */ - if (neo_dcp_has_panel(neo_dcp)) - schedule_work(&neo_dcp->bl_update_wq); -} - -/* Chunked data transfer for property dictionaries */ -static u8 dcpep_cb_prop_start(struct neo_apple_dcp *neo_dcp, u32 *length) -{ - if (neo_dcp->chunks.data != NULL) { - dev_warn(neo_dcp->dev, "ignoring spurious transfer start\n"); - return false; - } - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * This length comes straight off the wire. A bogus value would turn - * the allocation below into a multi-gigabyte GFP_KERNEL request. - * Property blobs are a few KiB, so bound it. - */ - if (*length == 0 || *length > SZ_1M) { - dev_warn(neo_dcp->dev, "rejecting %u byte property transfer\n", - *length); - return false; - } -#endif - - neo_dcp->chunks.length = *length; - neo_dcp->chunks.data = kzalloc(*length, GFP_KERNEL); - - if (!neo_dcp->chunks.data) { - dev_warn(neo_dcp->dev, "failed to allocate chunks\n"); - return false; - } - - return true; -} - -static u8 dcpep_cb_prop_chunk(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_set_dcpav_prop_chunk_req *req) -{ - if (!neo_dcp->chunks.data) { - dev_warn(neo_dcp->dev, "ignoring spurious chunk\n"); - return false; - } - - if (req->offset + req->length > neo_dcp->chunks.length) { - dev_warn(neo_dcp->dev, "ignoring overflowing chunk\n"); - return false; - } - - memcpy(neo_dcp->chunks.data + req->offset, req->data, req->length); - return true; -} - -static bool dcpep_process_chunks(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_set_dcpav_prop_end_req *req) -{ - struct neo_dcp_parse_ctx ctx; - int ret; - - if (!neo_dcp->chunks.data) { - dev_warn(neo_dcp->dev, "ignoring spurious end\n"); - return false; - } - - /* used just as opaque pointer for tracing */ - ctx.neo_dcp = neo_dcp; - - ret = neo_parse(neo_dcp->chunks.data, neo_dcp->chunks.length, &ctx); - - if (ret) { - dev_warn(neo_dcp->dev, "bad header on dcpav props\n"); - return false; - } - - if (!strcmp(req->key, "TimingElements")) { - neo_dcp->modes = neo_enumerate_modes(&ctx, &neo_dcp->nr_modes, - neo_dcp->width_mm, neo_dcp->height_mm, - neo_dcp->notch_height, - neo_dcp->fixed_connector_type == - DRM_MODE_CONNECTOR_eDP); - - if (IS_ERR(neo_dcp->modes)) { - dev_warn(neo_dcp->dev, "failed to parse modes\n"); - neo_dcp->modes = NULL; - neo_dcp->nr_modes = 0; - return false; - } - if (neo_dcp->nr_modes == 0) - dev_warn(neo_dcp->dev, "TimingElements without valid modes!\n"); - } else if (!strcmp(req->key, "DisplayAttributes")) { - ret = neo_parse_display_attributes(&ctx, &neo_dcp->width_mm, - &neo_dcp->height_mm); - - if (ret) { - dev_warn(neo_dcp->dev, "failed to parse display attribs\n"); - return false; - } - - neo_dcp_set_dimensions(neo_dcp); - } - - return true; -} - -static u8 dcpep_cb_prop_end(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_set_dcpav_prop_end_req *req) -{ - u8 resp = dcpep_process_chunks(neo_dcp, req); - - if (neo_dcp->fixed_connector_type != DRM_MODE_CONNECTOR_eDP) - dev_info(neo_dcp->dev, - "DCP property key=%.*s bytes=%zu accepted=%u nr_modes=%u\n", - (int)sizeof(req->key), req->key, neo_dcp->chunks.length, - resp, neo_dcp->nr_modes); - - /* move chunked data to connector to provide it via debugfs */ - neo_dcp_connector_update_dict(neo_dcp->connector, req->key, &neo_dcp->chunks); - neo_dcp->chunks.data = NULL; - neo_dcp->chunks.length = 0; - - return resp; -} - -/* Boot sequence */ -static void boot_done(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_channel *ch = &neo_dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - dev_dbg(neo_dcp->dev, "boot done\n"); - - *succ = true; - neo_dcp_ack(neo_dcp, DCP_CONTEXT_CB); -} - -static void boot_5b(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - neo_dcp_set_display_refresh_properties(neo_dcp, false, boot_done, NULL); -} - -static void boot_5(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* H17P expects A380 right before set_display_refresh_properties */ - neo_dcp_dfb_ready_query(neo_dcp, false, boot_5b, NULL); -#else - boot_5b(neo_dcp, out, cookie); -#endif -} - -static void boot_4b(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - u32 value = neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G; -#else - u32 value = 1; -#endif - neo_dcp_late_init_signal(neo_dcp, false, &value, boot_5, NULL); -#else - neo_dcp_late_init_signal(neo_dcp, false, boot_5, NULL); -#endif -} - -static void boot_4(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* H17P expects A033 between flush_supports_power and late_init_signal */ - neo_dcp_dfb_query(neo_dcp, false, boot_4b, NULL); -#else - boot_4b(neo_dcp, out, cookie); -#endif -} - -static void boot_3(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - u32 v_true = true; - - neo_dcp_flush_supports_power(neo_dcp, false, &v_true, boot_4, NULL); -} - -static void boot_2(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - neo_dcp_setup_video_limits(neo_dcp, false, boot_3, NULL); -} - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P expects the default framebuffer to be described between - * create_default_fb and setup_video_limits: - * get_dfb_compression_info -> get_dfb_info -> get_dfb_layout -> - * get_dfb_state -> set_dfb_dimensions(w, h) -> commit_dfb_info - * Without it the firmware never finishes its own default framebuffer setup. - */ -static void boot_1_11(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - neo_dcp_commit_dfb_info(neo_dcp, false, boot_2, NULL); -} - -static void boot_1_10(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_dfb_dimensions_h17p dim; - - /* - * get_dfb_state returns the default framebuffer dimensions, which are - * passed back unchanged to set_dfb_dimensions. - */ - memcpy(&dim, out, sizeof(dim)); - if (!dim.width || !dim.height || dim.width > 16384 || dim.height > 16384) { - struct neo_dcp_channel *ch = &neo_dcp->ch_cb; - - dev_err(neo_dcp->dev, "invalid default framebuffer dimensions %ux%u\n", - dim.width, dim.height); - *(u8 *)ch->output[ch->depth - 1] = false; - neo_dcp_ack(neo_dcp, DCP_CONTEXT_CB); - return; - } - dev_dbg(neo_dcp->dev, "default framebuffer dimensions %ux%u\n", - dim.width, dim.height); - - neo_dcp_set_dfb_dimensions(neo_dcp, false, &dim, boot_1_11, NULL); -} - -static void boot_1_9(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - u32 zero = 0; - - neo_dcp_get_dfb_state(neo_dcp, false, &zero, boot_1_10, NULL); -} - -static void boot_1_8(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_dfb_layout_req_h17p req = {}; - - neo_dcp_get_dfb_layout(neo_dcp, false, &req, boot_1_9, NULL); -} - -static void boot_1_7(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_dfb_info_h17p info = {}; - - neo_dcp_get_dfb_info(neo_dcp, false, &info, boot_1_8, NULL); -} - -static void boot_1_6(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - u64 zero = 0; - - neo_dcp_get_dfb_compression_info(neo_dcp, false, &zero, boot_1_7, NULL); -} -#endif - -static void boot_1_5(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - neo_dcp_create_default_fb(neo_dcp, false, boot_1_6, NULL); -#else - neo_dcp_create_default_fb(neo_dcp, false, boot_2, NULL); -#endif -} - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -static void d000_done(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_channel *ch = &neo_dcp->ch_cb; - u8 *succ = ch->output[ch->depth - 1]; - - *succ = true; - neo_dcp_ack(neo_dcp, DCP_CONTEXT_CB); -} - -/* - * The firmware expects D000 to be answered only after a nested A033 - * (dfb_query), and then with 1. Defer the ACK until that call completes. - */ -static bool dcpep_cb_d000_h17p(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) -{ - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - neo_dcp_dfb_query(neo_dcp, false, d000_done, NULL); - return false; -} -#endif - -/* Use special function signature to defer the ACK */ -static bool dcpep_cb_boot_1(struct neo_apple_dcp *neo_dcp, int tag, void *out, void *in) -{ - trace_neo_iomfb_callback(neo_dcp, tag, __func__); - neo_dcp_set_create_dfb(neo_dcp, false, boot_1_5, NULL); - return false; -} - -static struct neo_dcp_allocate_bandwidth_resp dcpep_cb_allocate_bandwidth(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_allocate_bandwidth_req *req) -{ - return (struct neo_dcp_allocate_bandwidth_resp){ - .unk1 = req->unk1, - .unk2 = req->unk2, - .ret = 1, - }; -} - -static struct neo_dcp_rt_bandwidth dcpep_cb_rt_bandwidth(struct neo_apple_dcp *neo_dcp) -{ - struct neo_dcp_rt_bandwidth rt_bw = (struct neo_dcp_rt_bandwidth){ - .reg_scratch = 0, - .reg_doorbell = 0, - .doorbell_bit = 0, - }; - - if (neo_dcp->disp_bw_scratch_index) { - u32 offset = neo_dcp->disp_bw_scratch_offset; - u32 index = neo_dcp->disp_bw_scratch_index; - rt_bw.reg_scratch = neo_dcp->disp_registers[index]->start + offset; - } - - if (neo_dcp->disp_bw_doorbell_index) { - u32 index = neo_dcp->disp_bw_doorbell_index; - rt_bw.reg_doorbell = neo_dcp->disp_registers[index]->start; - rt_bw.doorbell_bit = REG_DOORBELL_BIT(neo_dcp->index); - /* - * This is most certainly not padding. t8103-dcp crashes without - * setting this immediately during modeset on 12.3 and 13.5 - * firmware. - */ - rt_bw.padding[3] = 0x4; - } - - return rt_bw; -} - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P's D006 is 0x54 in / 0x50 out, far wider than the 28-byte - * frame_sync_props of older firmware, so the fields that matter sit past - * anything the old struct could describe. - * - * The firmware expects zeros except for three u32 at 0x24, 0x28 and 0x2c, - * each 0x10000 -- 1.0 in 16.16 fixed point, the identity scale. An all-zero - * reply hands the pipe a zero scale, and it then programs a null surface - * base: the display DART faults at DVA 0 and the coprocessor takes an AXI - * error. - */ -#define DCP_FRAME_SYNC_IDENTITY 0x10000 - -struct neo_dcp_set_frame_sync_props_req_h17p { - u8 unk[0x54]; -} __packed; - -struct neo_dcp_set_frame_sync_props_resp_h17p { - u8 unk[0x50]; -} __packed; - -static struct neo_dcp_set_frame_sync_props_resp_h17p -dcpep_cb_set_frame_sync_props_h17p(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_set_frame_sync_props_req_h17p *req) -{ - struct neo_dcp_set_frame_sync_props_resp_h17p resp = {}; - static const u16 identity_at[] = { 0x24, 0x28, 0x2c }; - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(identity_at); i++) { - u32 v = DCP_FRAME_SYNC_IDENTITY; - - memcpy(&resp.unk[identity_at[i]], &v, sizeof(v)); - } - - return resp; -} -#endif - -static struct neo_dcp_set_frame_sync_props_resp -dcpep_cb_set_frame_sync_props(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_set_frame_sync_props_req *req) -{ - return (struct neo_dcp_set_frame_sync_props_resp){}; -} - -/* Callback to get the current time as milliseconds since the UNIX epoch */ -static u64 dcpep_cb_get_time(struct neo_apple_dcp *neo_dcp) -{ - return ktime_to_ms(ktime_get_real()); -} - -struct neo_dcp_swap_cookie { - struct kref refcount; - struct completion done; - u32 status; - u32 swap_id; -}; - -static void release_swap_cookie(struct kref *ref) -{ - struct neo_dcp_swap_cookie *cookie; - cookie = container_of(ref, struct neo_dcp_swap_cookie, refcount); - - kfree(cookie); -} - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_SWAP_FLAGS1 0x34865202ULL -#define DCP_SWAP_FLAGS2 0x4ULL - -/* - * H17P's swap surface has no is_tearing_allowed and no is_premultiplied: - * plane_cnt sits at +1, plane_cnt2 at +5 and format at +9, two bytes earlier - * than on every other firmware. With the older layout the firmware reads the - * format two bytes late and its surface check rejects the swap as an - * unsupported format. - * - * H17G retains this public packing path. The measured H17P profile uses - * a separate wire record with the SPEC's 0x588 swap boundary. - */ -static void neo_dcp_h17p_prepare_swap(struct neo_apple_dcp *neo_dcp) -{ - struct DCP_FW_NAME(neo_dcp_swap_submit_req) *req = &DCP_FW_UNION(neo_dcp->swap); - unsigned int i; - - /* - * The firmware expects flags1 = 0x34865202 and flags2 = 4 (at +0x40 - * and +0x48) in every swap. - */ - req->swap.flags1 = DCP_SWAP_FLAGS1; - req->swap.flags2 = DCP_SWAP_FLAGS2; - - /* - * The H17P gap before swap_id carries two small counters the firmware - * expects to be 2 (at +0x50) and 3 (at +0x78). The gap starts at - * +0x50, so these are indices 0 and 0x28 of the blob. - */ - req->swap.h17p_pre_swap_id[0x00] = 2; - req->swap.h17p_pre_swap_id[0x28] = 3; - - /* The measured H17P layout is serialized without shifting its surfaces. */ - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return; - - for (i = 0; i < SWAP_SURFACES; i++) { - u8 *p = (u8 *)&req->surf[i]; - size_t n = sizeof(req->surf[i]); - - memmove(p + 1, p + 3, n - 3); - memset(p + n - 2, 0, 2); - } - - /* - * Everything from surf_iova onward also sits two bytes earlier on - * H17P: surf_iova[0] is at +0xe38 of the 0xe9c-byte record, not - * +0xe3a. Left in place, the firmware reads every IOVA two bytes off - * and the trailing null-flag block misaligned, so it faults on a bogus - * address or verifies layers that are not present. Shift the whole - * block down. - */ - { - u8 *r = (u8 *)req; - size_t n = sizeof(*req); - size_t off = offsetof(typeof(*req), surf_iova); - - static_assert(offsetof(typeof(*req), surf_iova) == 0xe3a, - "H17P source IOVA offset"); - static_assert(sizeof(*req) == 0xe9c, "H17P request size"); - memmove(r + off - 2, r + off, n - off); - memset(r + n - 2, 0, 2); - } -} -#else -static void neo_dcp_h17p_prepare_swap(struct neo_apple_dcp *neo_dcp) { } -#endif - -static bool neo_dcp_present_retires(struct neo_apple_dcp *neo_dcp) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return !neo_iomfb_present_brightness_only_h17p(neo_dcp); -#endif - return true; -} - -static void neo_dcp_present_failed(struct neo_apple_dcp *neo_dcp) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - neo_iomfb_present_failed_h17p(neo_dcp); -#endif -} - -static bool neo_dcp_present_begin(struct neo_apple_dcp *neo_dcp, u32 swap_id) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !neo_dcp_present_begin_h17p(&neo_dcp->present_state_h17p, swap_id)) { - dev_err(neo_dcp->dev, "overlapping present %u\n", swap_id); - neo_dcp->crashed = true; - return false; - } -#endif - return true; -} - -/* - * Returns 0 if the submitted present stands, -ECANCELED if the firmware - * aborted it before replying (already reported), or -EPROTO if no present - * was expecting this reply. - */ -static int neo_dcp_present_submit(struct neo_apple_dcp *neo_dcp, u32 swap_id, bool accepted) -{ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !neo_dcp_present_submit_h17p(&neo_dcp->present_state_h17p, swap_id, accepted)) { - dev_err(neo_dcp->dev, "unexpected present submission %u\n", swap_id); - neo_dcp->crashed = true; - return -EPROTO; - } - /* An abort received before this reply turns acceptance into failure. */ - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - accepted && neo_dcp->present_state_h17p.aborted) { - dev_warn_ratelimited(neo_dcp->dev, "firmware aborted present %u\n", - swap_id); - return -ECANCELED; - } -#endif - return 0; -} - -static void neo_dcp_prepare_clear_swap(struct neo_apple_dcp *neo_dcp, void *request) -{ - typeof(DCP_FW_UNION(neo_dcp->swap)) *swap = request; - - /* Clear surfaces. */ - memset(swap, 0, sizeof(*swap)); - - swap->swap.swap_enabled = - swap->swap.swap_completed = IOMFB_SET_BACKGROUND | 0x7; - swap->swap.bg_color = 0xFF000000; - - /* - * Turn off the backlight. This matters because the DCP's idea of - * backlight brightness gets desynced after a power change, and it - * needs to be told it's going to turn off so it will consider the - * subsequent update on poweron an actual change and restore the - * brightness. - */ - if (neo_dcp_has_panel(neo_dcp)) { - swap->swap.bl_unk = 1; - swap->swap.bl_value = 0; - swap->swap.bl_power = 0; - } - - /* Null all surfaces */ - for (int l = 0; l < SWAP_SURFACES; l++) - swap->surf_null[l] = true; -#if DCP_FW_VERSION(13, 2, 0) <= DCP_FW_VER -#if DCP_FW_VERSION(26, 0, 0) > DCP_FW_VER - for (int l = 0; l < 5; l++) - swap->surf2_null[l] = true; -#endif - swap->unknown_pointer_null = true; - swap->unknown_output_null = true; -#endif -} - -static void neo_dcp_swap_cleared(struct neo_apple_dcp *neo_dcp, void *data, void *cookie) -{ - struct DCP_FW_NAME(neo_dcp_swap_submit_resp) *resp = data; - struct neo_dcp_swap_cookie *info = cookie; - u32 swap_id = DCP_FW_UNION(neo_dcp->swap).swap.swap_id; - u32 status = resp ? resp->ret : ~0U; - int ret; - - ret = neo_dcp_present_submit(neo_dcp, swap_id, !status); - if (ret) - status = ~0U; - - if (status) { - /* An aborted present has already been reported. */ - if (ret != -ECANCELED) - dev_err(neo_dcp->dev, "swap_clear failed! status %u\n", status); - neo_dcp_present_failed(neo_dcp); - if (neo_dcp_present_retires(neo_dcp)) { - neo_dcp_unarm_retained_framebuffers(neo_dcp, swap_id); - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - } - } - if (info) { - WRITE_ONCE(info->status, status); - complete(&info->done); - kref_put(&info->refcount, release_swap_cookie); - } -} - -static void neo_dcp_swap_clear_started(struct neo_apple_dcp *neo_dcp, void *data, - void *cookie) -{ - struct DCP_FW_NAME(neo_dcp_swap_start_resp) *resp = data; - struct neo_dcp_swap_cookie *info = cookie; - - if (!resp || resp->ret) { - if (info) { - WRITE_ONCE(info->status, resp ? resp->ret : ~0U); - complete(&info->done); - kref_put(&info->refcount, release_swap_cookie); - } - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - return; - } - if (!neo_dcp_present_begin(neo_dcp, resp->swap_id)) { - if (info) { - WRITE_ONCE(info->status, ~0U); - complete(&info->done); - kref_put(&info->refcount, release_swap_cookie); - } - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - return; - } -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - neo_dcp_prepare_clear_swap(neo_dcp, &DCP_FW_UNION(neo_dcp->swap)); -#endif - DCP_FW_UNION(neo_dcp->swap).swap.swap_id = resp->swap_id; - if (neo_dcp_present_retires(neo_dcp)) - neo_dcp_arm_retained_framebuffers(neo_dcp, resp->swap_id); - - if (info) - info->swap_id = resp->swap_id; - - neo_dcp_h17p_prepare_swap(neo_dcp); - neo_dcp_swap_submit(neo_dcp, false, &DCP_FW_UNION(neo_dcp->swap), neo_dcp_swap_cleared, cookie); -} - -static void neo_dcp_on_final(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_wait_cookie *wait = cookie; - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - out && get_unaligned_le64(out) == 1) - WRITE_ONCE(neo_dcp->pipe_enabled_h17p, true); -#endif - - if (wait) { - complete(&wait->done); - kref_put(&wait->refcount, release_wait_cookie); - } -} - -static void neo_dcp_on_set_power_state(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct DCP_FW_NAME(neo_dcp_set_power_state_req) req = { - .unklong = 1, - }; - - neo_dcp_set_power_state(neo_dcp, DCP_POWER_OOB, &req, neo_dcp_on_final, cookie); -} - -static void neo_dcp_on_set_parameter(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, - .value = { 0 }, -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - .count = 3, -#else - .count = 1, -#endif - }; - - neo_dcp_set_parameter_dcp(neo_dcp, false, ¶m, neo_dcp_on_set_power_state, cookie); -} - -void DCP_FW_NAME(neo_iomfb_poweron)(struct neo_apple_dcp *neo_dcp) -{ - struct neo_dcp_wait_cookie *cookie; - int ret; - u32 handle; - dev_info(neo_dcp->dev, "dcp_poweron() starting\n"); - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - neo_iomfb_opaque_x_reset_h17p(neo_dcp); -#endif - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) - return; - - cookie->status = ~0U; - init_completion(&cookie->done); - kref_init(&cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&cookie->refcount); - - if (neo_dcp->main_display) { -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P takes set_display_device (A411) once, in the - * client-open sequence. Repeating it after the client is open - * is a call the firmware stops answering, which wedges the - * command channel and with it every later swap. - */ - neo_dcp_on_set_power_state(neo_dcp, NULL, cookie); -#else - handle = 0; - neo_dcp_set_display_device(neo_dcp, false, &handle, neo_dcp_on_set_power_state, - cookie); -#endif - } else { - handle = 2; - neo_dcp_set_display_device(neo_dcp, false, &handle, - neo_dcp_on_set_parameter, cookie); - } - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(10000)); - - if (ret == 0) { - dev_warn(neo_dcp->dev, "wait for power timed out, connector will be broken\n"); - } else if (ret > 0) { - int msecs = jiffies_to_msecs(ret); - if (msecs > 6000) - dev_info(neo_dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); - else - dev_warn(neo_dcp->dev, "dcp_set_power_state_req returned, %d ms remaining\n", msecs); - } else { - drm_connector_set_link_status_property(&neo_dcp->connector->base, - DRM_MODE_LINK_STATUS_BAD); - dev_warn(neo_dcp->dev, "wait for completion error: %d\n", ret); - } - - kref_put(&cookie->refcount, release_wait_cookie);; - - /* Force a brightness update after poweron, to restore the brightness */ - neo_dcp->brightness.update = true; -} - -static void complete_set_powerstate(struct neo_apple_dcp *neo_dcp, void *out, - void *cookie) -{ - struct neo_dcp_set_power_state_resp *resp = out; - struct neo_dcp_wait_cookie *wait = cookie; - - if (wait) { - WRITE_ONCE(wait->status, resp ? resp->ret : ~0U); - complete(&wait->done); - kref_put(&wait->refcount, release_wait_cookie); - } -} - -static bool neo_dcp_power_stop_confirmed(struct neo_dcp_wait_cookie *cookie) -{ - /* H17P stop semantics still require hardware qualification. */ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - return false; -#else - return READ_ONCE(cookie->status) == 0; -#endif -} - -static void last_client_closed_poff(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct DCP_FW_NAME(neo_dcp_set_power_state_req) power_req = { - .unklong = 0, - }; - neo_dcp_set_power_state(neo_dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, - cookie); -} - -static void aborted_swaps_dcp_poff(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_iomfb_last_client_close_req last_client_req = {}; - neo_iomfb_last_client_close(neo_dcp, false, &last_client_req, - last_client_closed_poff, cookie); -} - -void DCP_FW_NAME(neo_iomfb_poweroff)(struct neo_apple_dcp *neo_dcp) -{ - int ret, swap_id; - struct neo_iomfb_abort_swaps_dcp_req abort_req = { - .client = { - .flag2 = 1, - }, - }; - struct neo_dcp_swap_cookie *cookie; - struct neo_dcp_wait_cookie *poff_cookie; - struct DCP_FW_NAME(neo_dcp_swap_start_req) swap_req = { 0 }; - typeof(DCP_FW_UNION(neo_dcp->swap)) *swap = &DCP_FW_UNION(neo_dcp->swap); - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - /* The measured profile has no safe pipe-off or surface-free transition. */ - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return; -#endif - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) - return; - cookie->status = ~0U; - init_completion(&cookie->done); - kref_init(&cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&cookie->refcount); - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G) - neo_dcp_prepare_clear_swap(neo_dcp, swap); -#else - neo_dcp_prepare_clear_swap(neo_dcp, swap); -#endif - - neo_dcp_swap_start(neo_dcp, false, &swap_req, neo_dcp_swap_clear_started, cookie); - - /* - * On unplug the firmware powers the external pipe down on its own and - * can take tens of milliseconds before it answers (and swallows) the - * clear swap. That is not a crash: a real one is reported through the - * RTKit crash callback. Wait longer and carry on with the power-off - * either way, otherwise every later modeset fails with -EINVAL. - */ - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(500)); - swap_id = cookie->swap_id; - if (READ_ONCE(cookie->status)) - ret = 0; - kref_put(&cookie->refcount, release_swap_cookie); - if (ret <= 0) - dev_warn(neo_dcp->dev, "%s: clear swap timed out\n", __func__); - else - dev_dbg(neo_dcp->dev, "%s: clear swap submitted: %u\n", __func__, - swap_id); - - poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); - if (!poff_cookie) - return; - poff_cookie->status = ~0U; - init_completion(&poff_cookie->done); - kref_init(&poff_cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&poff_cookie->refcount); - - if (DCP_HAS_CLIENT_TEARDOWN) { - neo_iomfb_abort_swaps_dcp(neo_dcp, false, &abort_req, - aborted_swaps_dcp_poff, poff_cookie); - } else { - dev_dbg_once(neo_dcp->dev, - "power-off: skipping abort_swaps/last_client_close\n"); - last_client_closed_poff(neo_dcp, NULL, poff_cookie); - } - ret = wait_for_completion_timeout(&poff_cookie->done, - msecs_to_jiffies(1000)); - - if (ret == 0) - dev_warn(neo_dcp->dev, "setPowerState(0) timeout %u ms\n", 1000); - else if (ret > 0) { - dev_dbg(neo_dcp->dev, - "setPowerState(0) finished with %d ms to spare", - jiffies_to_msecs(ret)); - if (neo_dcp_power_stop_confirmed(poff_cookie)) - neo_dcp_release_all_retained_framebuffers(neo_dcp); - else - dev_warn(neo_dcp->dev, "scanout stop was not confirmed\n"); - } - - kref_put(&poff_cookie->refcount, release_wait_cookie); - - dev_info(neo_dcp->dev, "dcp_poweroff() done\n"); -} - -static void last_client_closed_sleep(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct DCP_FW_NAME(neo_dcp_set_power_state_req) power_req = { - .unklong = 0, - }; - neo_dcp_set_power_state(neo_dcp, DCP_POWER_OOB, &power_req, complete_set_powerstate, cookie); -} - -static void aborted_swaps_dcp_sleep(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_iomfb_last_client_close_req req = { 0 }; - neo_iomfb_last_client_close(neo_dcp, false, &req, last_client_closed_sleep, cookie); -} - -void DCP_FW_NAME(neo_iomfb_sleep)(struct neo_apple_dcp *neo_dcp) -{ - int ret; - struct neo_iomfb_abort_swaps_dcp_req req = { - .client = { - .flag2 = 1, - }, - }; - - struct neo_dcp_wait_cookie *cookie; - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - return; -#endif - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) - return; - cookie->status = ~0U; - init_completion(&cookie->done); - kref_init(&cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&cookie->refcount); - - if (DCP_HAS_CLIENT_TEARDOWN) { - neo_iomfb_abort_swaps_dcp(neo_dcp, false, &req, aborted_swaps_dcp_sleep, - cookie); - } else { - dev_dbg_once(neo_dcp->dev, - "sleep: skipping abort_swaps/last_client_close\n"); - last_client_closed_sleep(neo_dcp, NULL, cookie); - } - ret = wait_for_completion_timeout(&cookie->done, - msecs_to_jiffies(1000)); - - if (ret == 0) - dev_warn(neo_dcp->dev, "setDCPPower(0) timeout %u ms\n", 1000); - - kref_put(&cookie->refcount, release_wait_cookie); - dev_info(neo_dcp->dev, "dcp_sleep() done\n"); -} - -static void dcpep_cb_hotplug(struct neo_apple_dcp *neo_dcp, u64 *connected) -{ - struct neo_apple_connector *connector = neo_dcp->connector; - unsigned int action; - - /* DCP issues hotplug_gated callbacks after SetPowerState() calls on - * devices with display (macbooks, imacs). This must not result in - * connector state changes on DRM side. Some applications won't enable - * a CRTC with a connector in disconnected state. Weston after DPMS off - * is one example. dcp_is_main_display() returns true on devices with - * integrated display. Ignore the hotplug_gated() callbacks there. - */ - if (neo_dcp->main_display) - return; - /* - * Report firmware hotplug independently of the USB4 PHY experiment. - * Reassigning lpdptxphy blanked eDP even with these callbacks ignored; - * suppressing connector notifications does not protect the panel. - * Mode probing still uses this DCP's firmware modes, and mode_valid - * rejects modes absent from that list. Do not synthesize a mode here. - */ - - /* - * Same for the unplug a Type-C output reports after its CRTC was - * powered off with the cable still attached (see dcp_poweroff()). - */ - if (!(*connected) && READ_ONCE(neo_dcp->typec_crtc_off) && - READ_ONCE(neo_dcp->typec_cable_connected)) { - dev_dbg(neo_dcp->dev, "cb_hotplug() ignoring unplug of powered-off Type-C output\n"); - neo_dcp_mode_invalidate(&neo_dcp->mode_state); - schedule_work(&neo_dcp->vblank_wq); - return; - } - if (neo_dcp_is_typec_output(neo_dcp) && *connected && neo_dcp->nr_modes) - complete_all(&neo_dcp->typec_iomfb_hpd_ready); - - action = neo_dcp_mode_hotplug(&neo_dcp->mode_state, !!(*connected), - connector ? &connector->connected : NULL); - /* - * A Type-C sink can assert HPD only after a modeset has already failed, - * as a TV behind a DP-to-HDMI converter does when it wakes from standby. - * The connector state does not change then; re-apply the mode anyway. - */ - if (*connected && neo_dcp_is_typec_output(neo_dcp) && - !READ_ONCE(neo_dcp->mode_state.valid) && - !READ_ONCE(neo_dcp->mode_state.changing)) { - neo_dcp->swap_watchdog_retrains = 0; - action |= DCP_HOTPLUG_NOTIFY; - } - neo_dcp_handle_hotplug_actions(neo_dcp, action); -} - -static void -dcpep_cb_swap_complete_intent_gated(struct neo_apple_dcp *neo_dcp, - struct neo_dcp_swap_complete_intent_gated *info) -{ - trace_neo_iomfb_swap_complete_intent_gated(neo_dcp, info->swap_id, - info->width, info->height); -} - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER -/* - * An aborted H17P present never completes. Finish it like a rejected one: - * keep the displaced framebuffers for the next completed present and signal - * the DRM event, so neither the queue nor userspace waits for a completion. - */ -static void neo_dcp_present_aborted(struct neo_apple_dcp *neo_dcp, u32 swap_id) -{ - if (!neo_dcp_present_abort_h17p(&neo_dcp->present_state_h17p, swap_id)) { - dev_warn_ratelimited(neo_dcp->dev, - "abort for present %u, which is not in flight\n", - swap_id); - return; - } - /* The submit reply has not arrived; it finishes the present. */ - if (neo_dcp->present_state_h17p.pending) - return; - - dev_warn_ratelimited(neo_dcp->dev, "firmware aborted present %u\n", swap_id); - /* No completion follows, so the swap watchdog must not wait for one. */ - neo_dcp_swap_watchdog_complete(neo_dcp); - neo_dcp_present_failed(neo_dcp); - if (neo_dcp_present_retires(neo_dcp)) { - neo_dcp_unarm_retained_framebuffers(neo_dcp, swap_id); - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - } -} -#endif - -static void -dcpep_cb_abort_swap_ap_gated(struct neo_apple_dcp *neo_dcp, u32 *swap_id) -{ - trace_neo_iomfb_abort_swap_ap_gated(neo_dcp, *swap_id); -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) - neo_dcp_present_aborted(neo_dcp, *swap_id); -#endif -} - -static struct dcpep_get_tiling_state_resp -dcpep_cb_get_tiling_state(struct neo_apple_dcp *neo_dcp, - struct dcpep_get_tiling_state_req *req) -{ - return (struct dcpep_get_tiling_state_resp){ - .value = 0, - .ret = 1, - }; -} - -static u8 dcpep_cb_create_backlight_service(struct neo_apple_dcp *neo_dcp) -{ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P asks during start_ios_gated, before any mode is known, but - * dcp_has_panel() only becomes true once the panel dimensions have been - * parsed. An internal panel always has a backlight, so answer for the - * connector; answering false makes the firmware report that it could - * not find the backlight service. - */ - return neo_dcp_has_panel(neo_dcp) || - neo_dcp->connector_type == DRM_MODE_CONNECTOR_eDP; -#else - return neo_dcp_has_panel(neo_dcp); -#endif -} - -TRAMPOLINE_VOID(trampoline_nop, dcpep_cb_nop); -TRAMPOLINE_OUT(trampoline_true, dcpep_cb_true, u8); -TRAMPOLINE_OUT(trampoline_false, dcpep_cb_false, u8); -TRAMPOLINE_OUT(trampoline_zero, dcpep_cb_zero, u32); -TRAMPOLINE_IN(trampoline_swap_complete, dcpep_cb_swap_complete, - struct DCP_FW_NAME(dc_swap_complete_resp)); -TRAMPOLINE_INOUT(trampoline_get_uint_prop, dcpep_cb_get_uint_prop, - struct neo_dcp_get_uint_prop_req, struct neo_dcp_get_uint_prop_resp); -TRAMPOLINE_IN(trampoline_set_fx_prop, iomfbep_cb_set_fx_prop, - struct neo_iomfb_set_fx_prop_req) -TRAMPOLINE_INOUT(trampoline_map_piodma, dcpep_cb_map_piodma, - struct neo_dcp_map_buf_req, struct DCP_FW_NAME(neo_dcp_map_buf_resp)); -TRAMPOLINE_IN(trampoline_unmap_piodma, dcpep_cb_unmap_piodma, - struct neo_dcp_unmap_buf_resp); -TRAMPOLINE_INOUT(trampoline_sr_set_property_int, iomfbep_cb_sr_set_property_int, - struct neo_iomfb_sr_set_property_int_req, u8); -TRAMPOLINE_INOUT(trampoline_allocate_buffer, dcpep_cb_allocate_buffer, - struct neo_dcp_allocate_buffer_req, - struct neo_dcp_allocate_buffer_resp); -TRAMPOLINE_INOUT(trampoline_map_physical, dcpep_cb_map_physical, - struct neo_dcp_map_physical_req, struct neo_dcp_map_physical_resp); -TRAMPOLINE_INOUT(trampoline_release_mem_desc, dcpep_cb_release_mem_desc, u32, - u8); -TRAMPOLINE_INOUT(trampoline_map_reg, dcpep_cb_map_reg, - struct DCP_FW_NAME(neo_dcp_map_reg_req), - struct DCP_FW_NAME(neo_dcp_map_reg_resp)); -TRAMPOLINE_INOUT(trampoline_read_edt_data, dcpep_cb_read_edt_data, - struct neo_dcp_read_edt_data_req, struct neo_dcp_read_edt_data_resp); -TRAMPOLINE_INOUT(trampoline_prop_start, dcpep_cb_prop_start, u32, u8); -TRAMPOLINE_INOUT(trampoline_prop_chunk, dcpep_cb_prop_chunk, - struct neo_dcp_set_dcpav_prop_chunk_req, u8); -TRAMPOLINE_INOUT(trampoline_prop_end, dcpep_cb_prop_end, - struct neo_dcp_set_dcpav_prop_end_req, u8); -TRAMPOLINE_INOUT(trampoline_allocate_bandwidth, dcpep_cb_allocate_bandwidth, - struct neo_dcp_allocate_bandwidth_req, struct neo_dcp_allocate_bandwidth_resp); -TRAMPOLINE_OUT(trampoline_rt_bandwidth, dcpep_cb_rt_bandwidth, - struct neo_dcp_rt_bandwidth); -TRAMPOLINE_INOUT(trampoline_set_frame_sync_props, dcpep_cb_set_frame_sync_props, - struct neo_dcp_set_frame_sync_props_req, - struct neo_dcp_set_frame_sync_props_resp); -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -TRAMPOLINE_INOUT(trampoline_set_frame_sync_props_h17p, - dcpep_cb_set_frame_sync_props_h17p, - struct neo_dcp_set_frame_sync_props_req_h17p, - struct neo_dcp_set_frame_sync_props_resp_h17p); -#endif -TRAMPOLINE_OUT(trampoline_get_frequency, dcpep_cb_get_frequency, u64); -TRAMPOLINE_OUT(trampoline_get_time, dcpep_cb_get_time, u64); -TRAMPOLINE_IN(trampoline_hotplug, dcpep_cb_hotplug, u64); -TRAMPOLINE_IN(trampoline_swap_complete_intent_gated, - dcpep_cb_swap_complete_intent_gated, - struct neo_dcp_swap_complete_intent_gated); -TRAMPOLINE_IN(trampoline_abort_swap_ap_gated, dcpep_cb_abort_swap_ap_gated, u32); -TRAMPOLINE_IN(trampoline_enable_backlight_message_ap_gated, - iomfbep_cb_enable_backlight_message_ap_gated, u8); -TRAMPOLINE_IN(trampoline_pr_publish, neo_iomfb_cb_pr_publish, - struct neo_iomfb_property); -TRAMPOLINE_INOUT(trampoline_get_tiling_state, dcpep_cb_get_tiling_state, - struct dcpep_get_tiling_state_req, struct dcpep_get_tiling_state_resp); -TRAMPOLINE_OUT(trampoline_create_backlight_service, dcpep_cb_create_backlight_service, u8); - -/* - * Callback for swap requests. If a swap failed, we'll never get a swap - * complete event so we need to fake a vblank event early to avoid a hang. - */ - -static void neo_dcp_swapped(struct neo_apple_dcp *neo_dcp, void *data, void *cookie) -{ - int ret; - struct DCP_FW_NAME(neo_dcp_swap_submit_resp) *resp = data; - u32 swap_id = DCP_FW_UNION(neo_dcp->swap).swap.swap_id; - u32 status = resp ? resp->ret : ~0U; - - ret = neo_dcp_present_submit(neo_dcp, swap_id, !status); - if (ret) - status = ~0U; - - if (status) { - /* An aborted present has already been reported. */ - if (ret != -ECANCELED) - dev_err(neo_dcp->dev, "swap failed! status %u\n", status); - neo_dcp_present_failed(neo_dcp); - if (neo_dcp_present_retires(neo_dcp)) { - neo_dcp_unarm_retained_framebuffers(neo_dcp, swap_id); - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - } - return; - } - neo_dcp->swap_start = ktime_get(); - neo_dcp->swap_submit_timestamp = arch_timer_read_counter(); - neo_dcp_swap_watchdog_arm(neo_dcp); -} - -static void neo_dcp_swap_started(struct neo_apple_dcp *neo_dcp, void *data, void *cookie) -{ - struct DCP_FW_NAME(neo_dcp_swap_start_resp) *resp = data; - - if (!resp || resp->ret) { - dev_err(neo_dcp->dev, "swap_start was rejected\n"); - if (neo_dcp_present_retires(neo_dcp)) - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - return; - } - if (!neo_dcp_present_begin(neo_dcp, resp->swap_id)) { - if (neo_dcp_present_retires(neo_dcp)) - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); - return; - } - DCP_FW_UNION(neo_dcp->swap).swap.swap_id = resp->swap_id; - if (neo_dcp_present_retires(neo_dcp)) - neo_dcp_arm_retained_framebuffers(neo_dcp, resp->swap_id); - - trace_neo_iomfb_swap_submit(neo_dcp, resp->swap_id); - neo_dcp_h17p_prepare_swap(neo_dcp); - neo_dcp_swap_submit(neo_dcp, false, &DCP_FW_UNION(neo_dcp->swap), neo_dcp_swapped, NULL); -} - -/* Helpers to modeset and swap, used to flush */ -static void do_swap(struct neo_apple_dcp *neo_dcp, void *data, void *cookie) -{ - struct DCP_FW_NAME(neo_dcp_swap_start_req) start_req = { 0 }; - - if (READ_ONCE(neo_dcp->mode_state.valid) && neo_dcp->connector && - READ_ONCE(neo_dcp->connector->connected)) - neo_dcp_swap_start(neo_dcp, false, &start_req, neo_dcp_swap_started, NULL); - else - neo_dcp_drm_crtc_vblank(neo_dcp->crtc); -} - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * H17P expects A428 between set_matrix and swap_start, with 0x10000 (1.0 in - * 16.16 fixed point), the same identity value the D006 reply carries. - */ -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER -void neo_iomfb_present_backlight_h17p(struct neo_apple_dcp *neo_dcp) -{ - do_swap(neo_dcp, NULL, NULL); -} -#endif - -static void neo_dcp_pre_swap_a428(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - u32 identity = 0x10000; - - neo_dcp_pipe_set_428(neo_dcp, false, &identity, do_swap, cookie); -} -#endif - -static void complete_set_digital_out_mode(struct neo_apple_dcp *neo_dcp, void *data, - void *cookie) -{ - struct neo_dcp_wait_cookie *wait = cookie; - - if (wait) { - complete(&wait->done); - kref_put(&wait->refcount, release_wait_cookie); - } -} - -/* DCP applies Adaptive Sync changes when the display mode is reselected. */ -static void neo_dcp_on_set_adaptive_sync(struct neo_apple_dcp *neo_dcp, void *out, - void *cookie) -{ - neo_dcp_set_digital_out_mode(neo_dcp, false, &neo_dcp->mode, - complete_set_digital_out_mode, cookie); -} - -static void neo_dcp_set_adaptive_sync(struct neo_apple_dcp *neo_dcp, u32 min_vrr, - void *cookie) -{ - struct neo_dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, - .value = { - min_vrr, /* minRR, 16.16 fixed-point Hz */ - 0, /* mediaTargetRate */ - 0, /* fractional rate */ - }, -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - .count = 3, -#else - .count = 1, -#endif - }; - - neo_dcp_set_parameter_dcp(neo_dcp, false, ¶m, neo_dcp_on_set_adaptive_sync, - cookie); -} - -int DCP_FW_NAME(neo_iomfb_modeset)(struct neo_apple_dcp *neo_dcp, - struct drm_crtc_state *crtc_state) -{ - struct neo_dcp_display_mode *mode; - struct neo_dcp_wait_cookie *cookie; - struct neo_dcp_color_mode *cmode = NULL; - int ret; - - mode = neo_lookup_mode(neo_dcp, &crtc_state->mode); - if (!mode) { - dev_err(neo_dcp->dev, "no match for " DRM_MODE_FMT "\n", - DRM_MODE_ARG(&crtc_state->mode)); - return -EIO; - } - - dev_info(neo_dcp->dev, - "set_digital_out_mode(color:%d timing:%d) " DRM_MODE_FMT "\n", - mode->color_mode_id, mode->timing_mode_id, - DRM_MODE_ARG(&crtc_state->mode)); - if (mode->color_mode_id == mode->sdr_rgb.id) - cmode = &mode->sdr_rgb; - else if (mode->color_mode_id == mode->sdr_444.id) - cmode = &mode->sdr_444; - else if (mode->color_mode_id == mode->sdr.id) - cmode = &mode->sdr; - else if (mode->color_mode_id == mode->best.id) - cmode = &mode->best; - if (cmode) - dev_info(neo_dcp->dev, - "set_digital_out_mode() color mode depth:%hhu format:%u " - "colorimetry:%u eotf:%u range:%u vrr:%u\n", cmode->depth, - cmode->format, cmode->colorimetry, cmode->eotf, - cmode->range, mode->vrr); - - neo_dcp->mode = (struct neo_dcp_set_digital_out_mode_req){ - .color_mode_id = mode->color_mode_id, - .timing_mode_id = mode->timing_mode_id - }; - - /* Built-in ProMotion panels require timestamps even in fixed-120 mode. */ - neo_dcp->use_timestamps = mode->vrr && neo_dcp->main_display; - - cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); - if (!cookie) { - return -ENOMEM; - } - - cookie->status = ~0U; - init_completion(&cookie->done); - kref_init(&cookie->refcount); - /* increase refcount to ensure the receiver has a reference */ - kref_get(&cookie->refcount); - - neo_dcp->swap_submit_timestamp = 0; - - if (DCP_INHERIT_BOOT_MODE) { - dev_dbg(neo_dcp->dev, "inheriting the bootloader's display mode\n"); - complete_set_digital_out_mode(neo_dcp, NULL, cookie); - } else if (mode->vrr) - neo_dcp_set_adaptive_sync(neo_dcp, - crtc_state->vrr_enabled ? mode->min_vrr : 0, - cookie); - else - neo_dcp_set_digital_out_mode(neo_dcp, false, &neo_dcp->mode, - complete_set_digital_out_mode, cookie); - - /* - * The DCP firmware has an internal timeout of ~8 seconds for - * modesets. Add an extra 500ms to safe side that the modeset - * call has returned. - */ - ret = wait_for_completion_timeout(&cookie->done, - msecs_to_jiffies(8500)); - - dev_info(neo_dcp->dev, "set_digital_out_mode finished:%d\n", ret); - - if (ret == 0) { - dev_info(neo_dcp->dev, "set_digital_out_mode timed out\n"); - kref_put(&cookie->refcount, release_wait_cookie); - return -EIO; - } else if (ret < 0) { - dev_info(neo_dcp->dev, - "waiting on set_digital_out_mode failed:%d\n", ret); - kref_put(&cookie->refcount, release_wait_cookie); - return -EIO; - } else { - dev_dbg(neo_dcp->dev, - "set_digital_out_mode finished with %d to spare\n", - jiffies_to_msecs(ret)); - } - kref_put(&cookie->refcount, release_wait_cookie); - neo_dcp->vrr_enabled = mode->vrr && crtc_state->vrr_enabled; - - return 0; -} - -void DCP_FW_NAME(neo_iomfb_flush)(struct neo_apple_dcp *neo_dcp, struct drm_crtc *crtc, struct drm_atomic_state *state) -{ - struct drm_plane *plane; - struct drm_plane_state *new_state, *old_state; - struct drm_crtc_state *crtc_state; - struct DCP_FW_NAME(neo_dcp_swap_submit_req) *req = &DCP_FW_UNION(neo_dcp->swap); - int plane_idx, l; - int has_surface = 0; - bool update_brightness; - - crtc_state = drm_atomic_get_new_crtc_state(state, crtc); - - /* Reset all surfaces to defaults */ - memset(req, 0, sizeof(*req)); - for (l = 0; l < SWAP_SURFACES; l++) - req->surf_null[l] = true; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) - for (l = 0; l < 5; l++) - req->surf2_null[l] = true; -#endif - req->unknown_pointer_null = true; - req->unknown_output_null = true; -#endif - - /* - * Clear all surfaces on startup. The boot framebuffer in surface 0 - * sticks around. - */ - if (!neo_dcp->surfaces_cleared) { - req->swap.swap_enabled = IOMFB_SET_BACKGROUND | 0x7; - req->swap.bg_color = 0xFF000000; - neo_dcp->surfaces_cleared = true; - } - - for_each_oldnew_plane_in_state(state, plane, old_state, new_state, plane_idx) { - struct neo_apple_plane_state *apple_state = to_apple_plane_state(new_state); - struct neo_apple_plane *apl_plane = to_apple_plane(plane); - - /* skip planes not for this crtc */ - if (old_state->crtc != crtc && new_state->crtc != crtc) - continue; - - l = apl_plane->neo_iomfb_surf; - req->swap.swap_enabled |= BIT(l); - - if (!new_state->fb || !new_state->visible) { - continue; - } - req->surf_null[l] = false; - has_surface = 1; - - req->swap.src_rect[l] = apple_state->src_rect; - req->swap.dst_rect[l] = apple_state->dst_rect; - - if (neo_dcp->notch_height > 0) - req->swap.dst_rect[l].y += neo_dcp->notch_height; - - req->surf_iova[l] = apple_state->iova; - req->surf[l].base = apple_state->surf; - - /* Use sRGB colorspace only for internal panels. External - * displays are expected to have EDID and user space can use - * the contained colorimetry information to provide native - * colors. - */ - if (neo_dcp->connector_type == DRM_MODE_CONNECTOR_eDP && - req->surf[l].base.colorspace == DCP_COLORSPACE_BG_SRGB) - req->surf[l].base.colorspace = DCP_COLORSPACE_NATIVE; - } - - if (!has_surface && !crtc_state->color_mgmt_changed) { - if (crtc_state->enable && crtc_state->active && - !crtc_state->planes_changed) { - schedule_work(&neo_dcp->vblank_wq); - return; - } - - /* Set black background */ - req->swap.swap_enabled |= IOMFB_SET_BACKGROUND; - req->swap.bg_color = 0xFF000000; - req->clear = 1; - } - - if (has_surface && (neo_dcp->use_timestamps || neo_dcp->vrr_enabled)) { - u64 submit_timestamp = neo_dcp->swap_submit_timestamp; - u64 timestamp = arch_timer_read_counter(); - - /* - * IOMobileFramebuffer uses Mach continuous-time values here. On - * Apple Silicon that is the ARM architectural counter. Empirical - * testing shows that using the current submission time together - * with the previous accepted swap makes DCP follow swap pacing. - */ - if (!submit_timestamp) - submit_timestamp = timestamp; - - /* Firmware 12.x/13.x requires timestamp types 1, 2, and 7. */ - req->swap.timestamp[0] = timestamp; - req->swap.timestamp[1] = submit_timestamp; - req->swap.timestamp[6] = timestamp; - - trace_neo_iomfb_vrr_timestamps(neo_dcp, neo_dcp->vrr_enabled, timestamp, - submit_timestamp); - } - - /* These fields should be set together */ - req->swap.swap_completed = req->swap.swap_enabled; - - /* update brightness if changed */ - update_brightness = neo_dcp_has_panel(neo_dcp) && neo_dcp->brightness.update; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - update_brightness = update_brightness && - neo_dcp->hw.neo_iomfb_method_profile == DCP_IOMFB_METHODS_H17G; -#endif - if (update_brightness) { - req->swap.bl_unk = 1; - req->swap.bl_value = neo_dcp->brightness.dac; - req->swap.bl_power = 0x40; - neo_dcp->brightness.update = false; - } - - if (crtc_state->color_mgmt_changed) { - struct neo_iomfb_set_matrix_req mat = { - .location = 9, - }; - - if (crtc_state->ctm) { - struct drm_color_ctm *ctm = (struct drm_color_ctm *)crtc_state->ctm->data; - memcpy(mat.matrix, ctm->matrix, sizeof(mat.matrix)); - } else { - mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; - } - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - neo_iomfb_set_matrix(neo_dcp, false, &mat, neo_dcp_pre_swap_a428, NULL); -#else - neo_iomfb_set_matrix(neo_dcp, false, &mat, do_swap, NULL); -#endif - } else - do_swap(neo_dcp, NULL, NULL); -} - -static void res_is_main_display(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - struct neo_apple_connector *connector; - bool result; - - /* No reply: the DCP is marked crashed, so only end the start-up wait. */ - if (!out) { - complete(&neo_dcp->start_done); - return; - } - - /* - * The reply is a one-byte boolean. The firmware leaves the rest of - * the 32-bit slot alone, and dcp_push() fills that with 0xff. - */ - result = *(u8 *)out; - - dev_info(neo_dcp->dev, "DCP is_main_display: %d\n", result); - - neo_dcp->main_display = result; - - connector = neo_dcp->connector; - if (connector) { - connector->connected = neo_dcp->nr_modes > 0; - schedule_work(&connector->hotplug_wq); - } - - neo_dcp->active = true; - complete(&neo_dcp->start_done); -} - -static void init_3(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - neo_dcp_is_main_display(neo_dcp, false, res_is_main_display, NULL); -} - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -/* - * The client-open sequence H17P firmware expects before the first swap, in - * H17P method numbers (dcp_push() applies the H17G aliases): - * - * A445 A411 A445 A415 A450 A031 x9 A414 A478 A457 A474 A478 A476 A478 A412 - * - * A412 is not in the table: its result feeds res_is_main_display(), so it - * stays on the existing path. - */ -enum neo_dcp_client_step_kind { - DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A415, DCP_STEP_A450, - DCP_STEP_A031, DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, - DCP_STEP_A474, DCP_STEP_A476, -}; - -static const u8 neo_dcp_client_open_seq[] = { - DCP_STEP_A445, DCP_STEP_A411, DCP_STEP_A445, DCP_STEP_A415, - DCP_STEP_A450, - DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, - DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, DCP_STEP_A031, - DCP_STEP_A031, - DCP_STEP_A414, DCP_STEP_A478, DCP_STEP_A457, DCP_STEP_A474, - DCP_STEP_A478, DCP_STEP_A476, DCP_STEP_A478, -}; - -static void neo_dcp_client_open_run(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - unsigned long idx = (unsigned long)cookie; - neo_dcp_callback_t next = neo_dcp_client_open_run; - void *ncookie = (void *)(idx + 1); - - if (idx >= ARRAY_SIZE(neo_dcp_client_open_seq)) { - init_3(neo_dcp, NULL, NULL); - return; - } - - switch (neo_dcp_client_open_seq[idx]) { - case DCP_STEP_A445: { - u32 zero = 0; - - neo_dcp_get_dfb_state(neo_dcp, false, &zero, next, ncookie); - break; - } - case DCP_STEP_A411: { - u32 handle = 0; - - neo_dcp_set_display_device(neo_dcp, false, &handle, next, ncookie); - break; - } - case DCP_STEP_A415: { - struct neo_dcp_pipe_cfg_415 cfg = {}; - - neo_dcp_pipe_cfg_415(neo_dcp, false, &cfg, next, ncookie); - break; - } - case DCP_STEP_A450: { - u32 val = 0; - - neo_dcp_enable_disable_video_power_savings(neo_dcp, false, &val, next, - ncookie); - break; - } - case DCP_STEP_A031: { - /* - * Nine cumulative states: each call repeats the previous set - * of notification clients and adds one, so state n lists n - * clients. - */ - static const u8 notify_clients[9][8] = { - {}, - { 6 }, - { 6, 7 }, - { 6, 7, 8 }, - { 6, 7, 8, 10 }, - { 6, 7, 8, 10, 11 }, - { 6, 7, 8, 10, 11, 12 }, - { 6, 7, 8, 10, 11, 12, 25 }, - { 6, 7, 8, 10, 11, 12, 25, 26 }, - }; - struct neo_dcp_pipe_cfg_031 cfg = {}; - unsigned int step = neo_dcp->a031_step; - unsigned int i; - - if (step >= ARRAY_SIZE(notify_clients)) - step = ARRAY_SIZE(notify_clients) - 1; - for (i = 0; i < step; i++) - cfg.clients[i] = cpu_to_le32(notify_clients[step][i]); - neo_dcp->a031_step++; - - neo_dcp_pipe_cfg_031(neo_dcp, false, &cfg, next, ncookie); - break; - } - case DCP_STEP_A414: { - u64 val = 0; - - neo_dcp_pipe_cfg_414(neo_dcp, false, &val, next, ncookie); - break; - } - case DCP_STEP_A478: - neo_dcp_pipe_query_478(neo_dcp, false, next, ncookie); - break; - case DCP_STEP_A457: - neo_dcp_first_client_open(neo_dcp, false, next, ncookie); - break; - case DCP_STEP_A474: - neo_dcp_pipe_query_474(neo_dcp, false, next, ncookie); - break; - case DCP_STEP_A476: { - /* - * A476 takes a single u32, which the firmware expects to be 1. - * Its name is not known for certain: the shape (in 4, out 0) - * fits both set_dcp_clamshellstate() and - * set_has_frame_swap_function(). - */ - u32 state = 1; - - neo_dcp_pipe_set_476(neo_dcp, false, &state, next, ncookie); - break; - } - default: - dev_warn(neo_dcp->dev, "unknown client-open step %u\n", - neo_dcp_client_open_seq[idx]); - init_3(neo_dcp, NULL, NULL); - break; - } -} - -static void init_1(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - /* every client-open sequence starts from the empty A031 state */ - neo_dcp->a031_step = 0; - neo_dcp_client_open_run(neo_dcp, NULL, (void *)0UL); -} -#else -static void init_2(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - neo_dcp_first_client_open(neo_dcp, false, init_3, NULL); -} - -static void init_1(struct neo_apple_dcp *neo_dcp, void *out, void *cookie) -{ - u32 val = 0; - neo_dcp_enable_disable_video_power_savings(neo_dcp, false, &val, init_2, NULL); -} -#endif - -static void neo_dcp_started(struct neo_apple_dcp *neo_dcp, void *data, void *cookie) -{ - struct neo_iomfb_get_color_remap_mode_req color_remap = - (struct neo_iomfb_get_color_remap_mode_req){ - .mode = 6, - }; - - dev_info(neo_dcp->dev, "DCP booted\n"); - - neo_iomfb_get_color_remap_mode(neo_dcp, false, &color_remap, init_1, cookie); -} - -void DCP_FW_NAME(neo_iomfb_shutdown)(struct neo_apple_dcp *neo_dcp) -{ - struct DCP_FW_NAME(neo_dcp_set_power_state_req) req = { - /* defaults are ok */ - }; - struct neo_dcp_wait_cookie *cookie; - int ret; - -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER - if (neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - neo_iomfb_queue_stop(neo_dcp); - return; - } -#endif - - cookie = kzalloc_obj(*cookie); - if (!cookie) - return; - cookie->status = ~0U; - init_completion(&cookie->done); - kref_init(&cookie->refcount); - kref_get(&cookie->refcount); - neo_dcp_set_power_state(neo_dcp, DCP_POWER_OOB, &req, complete_set_powerstate, cookie); - ret = wait_for_completion_timeout(&cookie->done, msecs_to_jiffies(1000)); - if (ret > 0 && neo_dcp_power_stop_confirmed(cookie)) - neo_dcp_release_all_retained_framebuffers(neo_dcp); - else - dev_warn(neo_dcp->dev, "shutdown scanout stop was not confirmed\n"); - kref_put(&cookie->refcount, release_wait_cookie); -} diff --git a/drivers/gpu/drm/apple-neo/iomfb_template.h b/drivers/gpu/drm/apple-neo/iomfb_template.h deleted file mode 100644 index 1a2a7b7ff5cb95..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_template.h +++ /dev/null @@ -1,285 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -/* - * This file is intended to be included multiple times with IOMFB_VER - * defined to declare DCP firmware version dependent structs. - */ - -#ifdef DCP_FW_VER - -#include - -#include - -#include "iomfb.h" -#include "iomfb_plane.h" -#include "plane.h" -#include "version_utils.h" - -struct DCP_FW_NAME(neo_dcp_swap) { - /* IOMobileFramebuffer timestamp types 1 through 7, in wire order. */ - u64 timestamp[7]; - u64 unk_38; - - u64 flags1; - u64 flags2; - -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P inserts 0x48 bytes between flags2 and the swap ID, which moves - * to +0x98. Every following field shifts with it: surf_ids +0x9c, - * src_rect +0xac, surf_flags +0xec, dst_rect +0x10c, - * swap_enabled/completed +0x14c/+0x150, bg_color +0x154. The gap holds - * two small u32 counters (at +0x50 and +0x78) that are not modelled. - */ - u8 h17p_pre_swap_id[0x48]; -#endif - - u32 swap_id; - - u32 surf_ids[SWAP_SURFACES]; - struct neo_dcp_rect src_rect[SWAP_SURFACES]; - u32 surf_flags[SWAP_SURFACES]; - u32 surf_unk[SWAP_SURFACES]; - struct neo_dcp_rect dst_rect[SWAP_SURFACES]; - u32 swap_enabled; - u32 swap_completed; - - u32 bg_color; - u8 unk_110[0x1b8]; - u32 unk_2c8; - u8 unk_2cc[0x14]; - u32 unk_2e0; -#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) - u16 unk_2e2; -#else - u8 unk_2e2[3]; -#endif - u64 bl_unk; - u32 bl_value; // min value is 0x10000000 - u8 bl_power; // constant 0x40 for on - u8 unk_2f3[0x2d]; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 unk_320[0x13f]; - u64 unk_1; -#endif -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * The H17P A408 request is 0xe9c bytes; with a 0x22c-byte surface, - * SWAP_SURFACES = 4 and no surf2 block, that leaves this tail. - */ - u8 h17p_tail[0xda]; -#endif -} __packed; - -/* Information describing a surface */ -struct DCP_FW_NAME(neo_dcp_surface) { - struct neo_dcp_surface base; -#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) - u8 padding[7]; -#else - u8 padding[47]; -#endif -} __packed; - -/* Prototypes */ - -struct DCP_FW_NAME(neo_dcp_swap_submit_req) { - struct DCP_FW_NAME(neo_dcp_swap) swap; - struct DCP_FW_NAME(neo_dcp_surface) surf[SWAP_SURFACES]; - u64 surf_iova[SWAP_SURFACES]; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u64 unk_u64_a[SWAP_SURFACES]; - /* - * H17P carries no surf2 block: its trailing u8 null-flag run is 8 bytes - * (swap_null + surf_null[4] + three tail bools). - */ -#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) - struct DCP_FW_NAME(neo_dcp_surface) surf2[5]; - u64 surf2_iova[5]; -#endif -#endif - u8 unkbool; - u64 unkdouble; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u64 unkU64; - u8 unkbool2; -#endif - u32 clear; // or maybe switch to default fb? -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u32 unkU32Ptr; -#endif - u8 swap_null; - u8 surf_null[SWAP_SURFACES]; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) -#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) - u8 surf2_null[5]; -#endif -#endif - u8 unkoutbool_null; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 unknown_pointer_null; - u8 unknown_output_null; -#endif -#if DCP_FW_VER < DCP_FW_VERSION(26, 0, 0) - u8 padding[1]; -#endif -} __packed; - -struct DCP_FW_NAME(neo_dcp_swap_submit_resp) { - u8 unkoutbool; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u32 unkU32out; -#endif - u32 ret; - u8 padding[3]; -} __packed; - -/* - * A406 swap_start / A473 set_power_state. H17P shrank both requests: the - * 0x10-byte struct dcp_iouserclient became a bare u64 handle and the - * swap_start response no longer echoes it. swap_start is 0x10 in / 0x8 out, - * set_power_state 0x10 in. - */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -struct DCP_FW_NAME(neo_dcp_swap_start_req) { - u32 swap_id; - u64 client; /* IOUserClient kernel VA */ - u8 client_null; - u8 padding[3]; -} __packed; - -struct DCP_FW_NAME(neo_dcp_swap_start_resp) { - u32 swap_id; - u32 ret; -} __packed; - -struct DCP_FW_NAME(neo_dcp_set_power_state_req) { - u64 unklong; - u32 unkint; /* widened from u8 on H17P */ - u8 unkint_null; - u8 padding[3]; -} __packed; - -static_assert(sizeof(struct DCP_FW_NAME(neo_dcp_swap_start_req)) == 0x10); -static_assert(sizeof(struct DCP_FW_NAME(neo_dcp_swap_start_resp)) == 0x8); -static_assert(sizeof(struct DCP_FW_NAME(neo_dcp_set_power_state_req)) == 0x10); -#else -struct DCP_FW_NAME(neo_dcp_swap_start_req) { - u32 swap_id; - struct neo_dcp_iouserclient client; - u8 swap_id_null; - u8 client_null; - u8 padding[2]; -} __packed; - -struct DCP_FW_NAME(neo_dcp_swap_start_resp) { - u32 swap_id; - struct neo_dcp_iouserclient client; - u32 ret; -} __packed; - -struct DCP_FW_NAME(neo_dcp_set_power_state_req) { - u64 unklong; - u8 unkbool; - u8 unkint_null; - u8 padding[2]; -} __packed; -#endif - -/* - * D201 map_piodma reply. H17P declares a 16-byte output, not 20, laid out as - * { u32 buffer index, u64 dva, u32 pad } rather than { u64 vaddr, u64 dva, - * u32 ret }. With the old layout the DVA lands four bytes late and the - * display engine DMAs to a bogus address. - */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -struct DCP_FW_NAME(neo_dcp_map_buf_resp) { - u32 buffer; - u64 dva; - u32 unk; -} __packed; -static_assert(sizeof(struct DCP_FW_NAME(neo_dcp_map_buf_resp)) == 0x10); -#else -struct DCP_FW_NAME(neo_dcp_map_buf_resp) { - u64 vaddr; - u64 dva; - u32 ret; -} __packed; -#endif - -struct DCP_FW_NAME(dc_swap_complete_resp) { - u32 swap_id; /* +0x000 */ - u8 unkbool; /* +0x004 */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) - /* - * H17P: swap_complete_ap_gated(u32, bool, SwapCompleteData const *, - * SwapInfoBlob const *, u32, bool) is 0x730 bytes on the wire, 89 more - * than the pre-H17P record: the SwapCompleteData grew from 8 to 0x22, - * swap_info became eight 0xe0-byte SwapInfoBlobs rather than one - * 0x6c5 blob, and the tail gained a flag and three pad bytes. Only - * swap_info_count of those blobs are populated; the rest are - * uninitialised DCP buffer memory. - */ - u32 hol_swap_ids[4]; /* +0x005 */ - u32 hol_arg; /* +0x015 */ - u32 hol_count; /* +0x019 */ - u8 hol_valid; /* +0x01d */ - u64 displayed_data; /* +0x01e, unaligned */ - u8 notify_displayed; /* +0x026 */ - u8 swap_info[8][0xe0]; /* +0x027, blob[0]+0x38 == swap_id */ - u32 swap_info_count; /* +0x727, max 8 */ - u8 completion_flag; /* +0x72b */ - u8 swap_data_null; /* +0x72c, 0 => data at +0x005 valid */ - u8 padding[3]; /* +0x72d, to sizeof == 0x730 */ -#else - u64 swap_data; -#if DCP_FW_VER < DCP_FW_VERSION(13, 2, 0) - u8 swap_info[0x6c4]; -#else - u8 swap_info[0x6c5]; -#endif - u32 unkint; - u8 swap_info_null; -#endif -} __packed; - -struct DCP_FW_NAME(neo_dcp_map_reg_req) { - char obj[4]; - u32 index; - u32 flags; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 unk_u64_null; -#endif - u8 addr_null; - u8 length_null; -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u8 padding[1]; -#else - u8 padding[2]; -#endif -} __packed; - -struct DCP_FW_NAME(neo_dcp_map_reg_resp) { -#if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) - u64 dva; -#endif - u64 addr; - u64 length; - u32 ret; -} __packed; - - -struct neo_apple_dcp; - -int DCP_FW_NAME(neo_iomfb_modeset)(struct neo_apple_dcp *neo_dcp, - struct drm_crtc_state *crtc_state); -void DCP_FW_NAME(neo_iomfb_flush)(struct neo_apple_dcp *neo_dcp, struct drm_crtc *crtc, struct drm_atomic_state *state); -void DCP_FW_NAME(neo_iomfb_poweron)(struct neo_apple_dcp *neo_dcp); -void DCP_FW_NAME(neo_iomfb_poweroff)(struct neo_apple_dcp *neo_dcp); -void DCP_FW_NAME(neo_iomfb_sleep)(struct neo_apple_dcp *neo_dcp); -void DCP_FW_NAME(neo_iomfb_start)(struct neo_apple_dcp *neo_dcp); -void DCP_FW_NAME(neo_iomfb_shutdown)(struct neo_apple_dcp *neo_dcp); - -#endif diff --git a/drivers/gpu/drm/apple-neo/iomfb_v12_3.c b/drivers/gpu/drm/apple-neo/iomfb_v12_3.c deleted file mode 100644 index 52036ead11de3c..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_v12_3.c +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#include "iomfb_v12_3.h" -#include "iomfb_v13_3.h" -#include "iomfb_h17p.h" -#include "version_utils.h" - -static const struct neo_dcp_method_entry neo_dcp_methods[dcpep_num_methods] = { - IOMFB_METHOD("A000", dcpep_late_init_signal), - IOMFB_METHOD("A029", dcpep_setup_video_limits), - IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), - IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), - IOMFB_METHOD("A357", dcpep_set_create_dfb), - IOMFB_METHOD("A358", iomfbep_a358_vi_set_temperature_hint), - IOMFB_METHOD("A401", dcpep_start_signal), - IOMFB_METHOD("A407", dcpep_swap_start), - IOMFB_METHOD("A408", dcpep_swap_submit), - IOMFB_METHOD("A410", dcpep_set_display_device), - IOMFB_METHOD("A411", dcpep_is_main_display), - IOMFB_METHOD("A412", dcpep_set_digital_out_mode), - IOMFB_METHOD("A422", iomfbep_set_matrix), - IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), - IOMFB_METHOD("A439", dcpep_set_parameter_dcp), - IOMFB_METHOD("A443", dcpep_create_default_fb), - IOMFB_METHOD("A447", dcpep_enable_disable_video_power_savings), - IOMFB_METHOD("A454", dcpep_first_client_open), - IOMFB_METHOD("A455", iomfbep_last_client_close), - IOMFB_METHOD("A460", dcpep_set_display_refresh_properties), - IOMFB_METHOD("A463", dcpep_flush_supports_power), - IOMFB_METHOD("A464", iomfbep_abort_swaps_dcp), - IOMFB_METHOD("A468", dcpep_set_power_state), -}; - -#define DCP_FW v12_3 -#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) - -#include "iomfb_template.c" - -static const neo_iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { - [0] = trampoline_true, /* did_boot_signal */ - [1] = trampoline_true, /* did_power_on_signal */ - [2] = trampoline_nop, /* will_power_off_signal */ - [3] = trampoline_rt_bandwidth, - [100] = iomfbep_cb_match_pmu_service, - [101] = trampoline_zero, /* get_display_default_stride */ - [102] = trampoline_nop, /* set_number_property */ - [103] = trampoline_nop, /* set_boolean_property */ - [106] = trampoline_nop, /* remove_property */ - [107] = trampoline_true, /* create_provider_service */ - [108] = trampoline_true, /* create_product_service */ - [109] = trampoline_true, /* create_pmu_service */ - [110] = trampoline_true, /* create_iomfb_service */ - [111] = trampoline_create_backlight_service, - [116] = dcpep_cb_boot_1, - [117] = trampoline_false, /* is_dark_boot */ - [118] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ - [120] = trampoline_read_edt_data, - [122] = trampoline_prop_start, - [123] = trampoline_prop_chunk, - [124] = trampoline_prop_end, - [201] = trampoline_map_piodma, - [202] = trampoline_unmap_piodma, - [206] = iomfbep_cb_match_pmu_service_2, - [207] = iomfbep_cb_match_backlight_service, - [208] = trampoline_get_time, - [211] = trampoline_nop, /* update_backlight_factor_prop */ - [300] = trampoline_pr_publish, - [401] = trampoline_get_uint_prop, - [404] = trampoline_nop, /* sr_set_uint_prop */ - [406] = trampoline_set_fx_prop, - [408] = trampoline_get_frequency, - [411] = trampoline_map_reg, - [413] = trampoline_true, /* sr_set_property_dict */ - [414] = trampoline_sr_set_property_int, - [415] = trampoline_true, /* sr_set_property_bool */ - [451] = trampoline_allocate_buffer, - [452] = trampoline_map_physical, - [456] = trampoline_release_mem_desc, - [552] = trampoline_true, /* set_property_dict_0 */ - [561] = trampoline_true, /* set_property_dict */ - [563] = trampoline_true, /* set_property_int */ - [565] = trampoline_true, /* set_property_bool */ - [567] = trampoline_true, /* set_property_str */ - [574] = trampoline_zero, /* power_up_dart */ - [576] = trampoline_hotplug, - [577] = trampoline_nop, /* powerstate_notify */ - [582] = trampoline_true, /* create_default_fb_surface */ - [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ - [588] = trampoline_nop, /* resize_default_fb_surface_gated */ - [589] = trampoline_swap_complete, - [591] = trampoline_swap_complete_intent_gated, - [592] = trampoline_abort_swap_ap_gated, - [593] = trampoline_enable_backlight_message_ap_gated, - [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ - [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ - [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ - [598] = trampoline_nop, /* find_swap_function_gated */ -}; - -void DCP_FW_NAME(neo_iomfb_start)(struct neo_apple_dcp *neo_dcp) -{ - neo_dcp->cb_handlers = cb_handlers; - - neo_dcp_start_signal(neo_dcp, false, neo_dcp_started, NULL); -} - -#undef DCP_FW_VER -#undef DCP_FW diff --git a/drivers/gpu/drm/apple-neo/iomfb_v12_3.h b/drivers/gpu/drm/apple-neo/iomfb_v12_3.h deleted file mode 100644 index 7359685d981fe5..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_v12_3.h +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#ifndef __APPLE_IOMFB_V12_3_H__ -#define __APPLE_IOMFB_V12_3_H__ - -#include "version_utils.h" - -#define DCP_FW v12_3 -#define DCP_FW_VER DCP_FW_VERSION(12, 3, 0) - -#include "iomfb_template.h" - -#undef DCP_FW_VER -#undef DCP_FW - -#endif /* __APPLE_IOMFB_V12_3_H__ */ diff --git a/drivers/gpu/drm/apple-neo/iomfb_v13_3.c b/drivers/gpu/drm/apple-neo/iomfb_v13_3.c deleted file mode 100644 index 0ae62cd345561a..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_v13_3.c +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#include "iomfb_v12_3.h" -#include "iomfb_v13_3.h" -#include "iomfb_h17p.h" -#include "version_utils.h" - -static const struct neo_dcp_method_entry neo_dcp_methods[dcpep_num_methods] = { - IOMFB_METHOD("A000", dcpep_late_init_signal), - IOMFB_METHOD("A029", dcpep_setup_video_limits), - IOMFB_METHOD("A131", iomfbep_a131_pmu_service_matched), - IOMFB_METHOD("A132", iomfbep_a132_backlight_service_matched), - IOMFB_METHOD("A373", dcpep_set_create_dfb), - IOMFB_METHOD("A374", iomfbep_a358_vi_set_temperature_hint), - IOMFB_METHOD("A401", dcpep_start_signal), - IOMFB_METHOD("A407", dcpep_swap_start), - IOMFB_METHOD("A408", dcpep_swap_submit), - IOMFB_METHOD("A410", dcpep_set_display_device), - IOMFB_METHOD("A411", dcpep_is_main_display), - IOMFB_METHOD("A412", dcpep_set_digital_out_mode), - IOMFB_METHOD("A422", iomfbep_set_matrix), - IOMFB_METHOD("A426", iomfbep_get_color_remap_mode), - IOMFB_METHOD("A441", dcpep_set_parameter_dcp), - IOMFB_METHOD("A445", dcpep_create_default_fb), - IOMFB_METHOD("A449", dcpep_enable_disable_video_power_savings), - IOMFB_METHOD("A456", dcpep_first_client_open), - IOMFB_METHOD("A457", iomfbep_last_client_close), - IOMFB_METHOD("A463", dcpep_set_display_refresh_properties), - IOMFB_METHOD("A466", dcpep_flush_supports_power), - IOMFB_METHOD("A467", iomfbep_abort_swaps_dcp), - IOMFB_METHOD("A472", dcpep_set_power_state), -}; - -#define DCP_FW v13_3 -#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) - -#include "iomfb_template.c" - -static const neo_iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { - [0] = trampoline_true, /* did_boot_signal */ - [1] = trampoline_true, /* did_power_on_signal */ - [2] = trampoline_nop, /* will_power_off_signal */ - [3] = trampoline_rt_bandwidth, - [6] = trampoline_set_frame_sync_props, - [100] = iomfbep_cb_match_pmu_service, - [101] = trampoline_zero, /* get_display_default_stride */ - [102] = trampoline_nop, /* set_number_property */ - [103] = trampoline_nop, /* trigger_user_cal_loader */ - [104] = trampoline_nop, /* set_boolean_property */ - [107] = trampoline_nop, /* remove_property */ - [108] = trampoline_true, /* create_provider_service */ - [109] = trampoline_true, /* create_product_service */ - [110] = trampoline_true, /* create_pmu_service */ - [111] = trampoline_true, /* create_iomfb_service */ - [112] = trampoline_create_backlight_service, - [113] = trampoline_true, /* create_nvram_service? */ - [114] = trampoline_get_tiling_state, - [115] = trampoline_false, /* set_tiling_state */ - [120] = dcpep_cb_boot_1, - [121] = trampoline_false, /* is_dark_boot */ - [122] = trampoline_false, /* is_dark_boot / is_waking_from_hibernate*/ - [124] = trampoline_read_edt_data, - [126] = trampoline_prop_start, - [127] = trampoline_prop_chunk, - [128] = trampoline_prop_end, - [129] = trampoline_allocate_bandwidth, - [201] = trampoline_map_piodma, - [202] = trampoline_unmap_piodma, - [206] = iomfbep_cb_match_pmu_service_2, - [207] = iomfbep_cb_match_backlight_service, - [208] = trampoline_nop, /* update_backlight_factor_prop */ - [209] = trampoline_get_time, - [300] = trampoline_pr_publish, - [401] = trampoline_get_uint_prop, - [404] = trampoline_nop, /* sr_set_uint_prop */ - [406] = trampoline_set_fx_prop, - [408] = trampoline_get_frequency, - [411] = trampoline_map_reg, - [413] = trampoline_true, /* sr_set_property_dict */ - [414] = trampoline_sr_set_property_int, - [415] = trampoline_true, /* sr_set_property_bool */ - [451] = trampoline_allocate_buffer, - [452] = trampoline_map_physical, - [454] = trampoline_release_mem_desc, - [552] = trampoline_true, /* set_property_dict_0 */ - [561] = trampoline_true, /* set_property_dict */ - [563] = trampoline_true, /* set_property_int */ - [565] = trampoline_true, /* set_property_bool */ - [567] = trampoline_true, /* set_property_str */ - [574] = trampoline_zero, /* power_up_dart */ - [576] = trampoline_hotplug, - [577] = trampoline_nop, /* powerstate_notify */ - [582] = trampoline_true, /* create_default_fb_surface */ - [584] = trampoline_nop, /* IOMobileFramebufferAP::clear_default_surface */ - [588] = trampoline_nop, /* resize_default_fb_surface_gated */ - [589] = trampoline_swap_complete, - [591] = trampoline_swap_complete_intent_gated, - [592] = trampoline_abort_swap_ap_gated, - [593] = trampoline_enable_backlight_message_ap_gated, - [594] = trampoline_nop, /* IOMobileFramebufferAP::setSystemConsoleMode */ - [596] = trampoline_false, /* IOMobileFramebufferAP::isDFBAllocated */ - [597] = trampoline_false, /* IOMobileFramebufferAP::preserveContents */ - [598] = trampoline_nop, /* find_swap_function_gated */ -}; -void DCP_FW_NAME(neo_iomfb_start)(struct neo_apple_dcp *neo_dcp) -{ - neo_dcp->cb_handlers = cb_handlers; - - neo_dcp_start_signal(neo_dcp, false, neo_dcp_started, NULL); -} - -#undef DCP_FW_VER -#undef DCP_FW diff --git a/drivers/gpu/drm/apple-neo/iomfb_v13_3.h b/drivers/gpu/drm/apple-neo/iomfb_v13_3.h deleted file mode 100644 index bbb3156b40f893..00000000000000 --- a/drivers/gpu/drm/apple-neo/iomfb_v13_3.h +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#ifndef __APPLE_IOMFB_V13_3_H__ -#define __APPLE_IOMFB_V13_3_H__ - -#include "version_utils.h" - -#define DCP_FW v13_3 -#define DCP_FW_VER DCP_FW_VERSION(13, 3, 0) - -#include "iomfb_template.h" - -#undef DCP_FW_VER -#undef DCP_FW - -#endif /* __APPLE_IOMFB_V13_3_H__ */ diff --git a/drivers/gpu/drm/apple-neo/parser.c b/drivers/gpu/drm/apple-neo/parser.c deleted file mode 100644 index 6b7bee8c1a00e7..00000000000000 --- a/drivers/gpu/drm/apple-neo/parser.c +++ /dev/null @@ -1,1203 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -#include -#include -#include -#include -#include -#include - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) -#include // for sound format masks -#endif - -#include "parser.h" -#include "trace.h" - -#define DCP_PARSE_HEADER 0xd3 - -enum neo_dcp_parse_type { - DCP_TYPE_DICTIONARY = 1, - DCP_TYPE_ARRAY = 2, - DCP_TYPE_INT64 = 4, - DCP_TYPE_STRING = 9, - DCP_TYPE_BLOB = 10, - DCP_TYPE_BOOL = 11 -}; - -struct neo_dcp_parse_tag { - unsigned int size : 24; - enum neo_dcp_parse_type type : 5; - unsigned int padding : 2; - bool last : 1; -} __packed; - -static const void *neo_parse_bytes(struct neo_dcp_parse_ctx *ctx, size_t count) -{ - const void *ptr; - - if (ctx->pos > ctx->len || count > ctx->len - ctx->pos) - return ERR_PTR(-EINVAL); - - ptr = ctx->blob + ctx->pos; - ctx->pos += count; - return ptr; -} - -static const u32 *neo_parse_u32(struct neo_dcp_parse_ctx *ctx) -{ - return neo_parse_bytes(ctx, sizeof(u32)); -} - -static const struct neo_dcp_parse_tag *neo_parse_tag(struct neo_dcp_parse_ctx *ctx) -{ - const struct neo_dcp_parse_tag *tag; - - /* Align to 32-bits */ - if (ctx->pos > U32_MAX - 3) - return ERR_PTR(-EINVAL); - ctx->pos = round_up(ctx->pos, 4); - - tag = neo_parse_bytes(ctx, sizeof(struct neo_dcp_parse_tag)); - - if (IS_ERR(tag)) - return tag; - - if (tag->padding) - return ERR_PTR(-EINVAL); - - return tag; -} - -static const struct neo_dcp_parse_tag *neo_parse_tag_of_type(struct neo_dcp_parse_ctx *ctx, - enum neo_dcp_parse_type type) -{ - const struct neo_dcp_parse_tag *tag = neo_parse_tag(ctx); - - if (IS_ERR(tag)) - return tag; - - if (tag->type != type) - return ERR_PTR(-EINVAL); - - return tag; -} - -/* Bound stack use when ignoring nested firmware dictionaries. */ -#define DCP_PARSE_MAX_DEPTH 32 - -static int skip_value(struct neo_dcp_parse_ctx *handle, unsigned int depth) -{ - const struct neo_dcp_parse_tag *tag; - const void *data; - unsigned int count, i; - int ret; - - if (depth >= DCP_PARSE_MAX_DEPTH) - return -EINVAL; - - tag = neo_parse_tag(handle); - if (IS_ERR(tag)) - return PTR_ERR(tag); - - switch (tag->type) { - case DCP_TYPE_DICTIONARY: - case DCP_TYPE_ARRAY: - count = tag->size; - if (tag->type == DCP_TYPE_DICTIONARY) - count *= 2; - - for (i = 0; i < count; ++i) { - ret = skip_value(handle, depth + 1); - if (ret) - return ret; - } - return 0; - - case DCP_TYPE_INT64: - data = neo_parse_bytes(handle, sizeof(s64)); - break; - - case DCP_TYPE_STRING: - case DCP_TYPE_BLOB: - data = neo_parse_bytes(handle, tag->size); - break; - - case DCP_TYPE_BOOL: - return 0; - - default: - return -EINVAL; - } - - return IS_ERR(data) ? PTR_ERR(data) : 0; -} - -static int skip(struct neo_dcp_parse_ctx *handle) -{ - return skip_value(handle, 0); -} - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) -static int skip_pair(struct neo_dcp_parse_ctx *handle) -{ - int ret; - - ret = skip(handle); - if (ret) - return ret; - - return skip(handle); -} - -static bool consume_string(struct neo_dcp_parse_ctx *ctx, const char *specimen) -{ - struct neo_dcp_parse_ctx probe = *ctx; - const struct neo_dcp_parse_tag *tag; - const char *key; - - tag = neo_parse_tag_of_type(&probe, DCP_TYPE_STRING); - if (IS_ERR(tag) || tag->size != strlen(specimen)) - return false; - - key = neo_parse_bytes(&probe, tag->size); - if (IS_ERR(key) || memcmp(key, specimen, tag->size)) - return false; - - *ctx = probe; - return true; -} -#endif - -/* Caller must free the result */ -static char *neo_parse_string(struct neo_dcp_parse_ctx *handle) -{ - const struct neo_dcp_parse_tag *tag = neo_parse_tag_of_type(handle, DCP_TYPE_STRING); - const char *in; - char *out; - - if (IS_ERR(tag)) - return (void *)tag; - - in = neo_parse_bytes(handle, tag->size); - if (IS_ERR(in)) - return (void *)in; - - out = kmalloc(tag->size + 1, GFP_KERNEL); - if (!out) - return ERR_PTR(-ENOMEM); - - memcpy(out, in, tag->size); - out[tag->size] = '\0'; - return out; -} - -static int neo_parse_int_bound(struct neo_dcp_parse_ctx *handle, s64 *value, s64 min, s64 max) -{ - const void *tag = neo_parse_tag_of_type(handle, DCP_TYPE_INT64); - const void *in; - - if (IS_ERR(tag)) - return PTR_ERR(tag); - - in = neo_parse_bytes(handle, sizeof(s64)); - - if (IS_ERR(in)) - return PTR_ERR(in); - - memcpy(value, in, sizeof(*value)); - - if (*value < min || *value > max) - return -EINVAL; - - return 0; -} - -static int neo_parse_int(struct neo_dcp_parse_ctx *handle, s64 *value) -{ - return neo_parse_int_bound(handle, value, S64_MIN, S64_MAX); -} - -static int neo_parse_bool(struct neo_dcp_parse_ctx *handle, bool *b) -{ - const struct neo_dcp_parse_tag *tag = neo_parse_tag_of_type(handle, DCP_TYPE_BOOL); - - if (IS_ERR(tag)) - return PTR_ERR(tag); - - *b = !!tag->size; - return 0; -} - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) -static int neo_parse_blob(struct neo_dcp_parse_ctx *handle, size_t size, u8 const **blob) -{ - const struct neo_dcp_parse_tag *tag = neo_parse_tag_of_type(handle, DCP_TYPE_BLOB); - const u8 *out; - - if (IS_ERR(tag)) - return PTR_ERR(tag); - - if (tag->size < size) - return -EINVAL; - - out = neo_parse_bytes(handle, tag->size); - - if (IS_ERR(out)) - return PTR_ERR(out); - - *blob = out; - return 0; -} -#endif - -struct iterator { - struct neo_dcp_parse_ctx *handle; - u32 idx, len; -}; - -static int iterator_begin(struct neo_dcp_parse_ctx *handle, struct iterator *it, - bool dict) -{ - const struct neo_dcp_parse_tag *tag; - enum neo_dcp_parse_type type = dict ? DCP_TYPE_DICTIONARY : DCP_TYPE_ARRAY; - - *it = (struct iterator) { - .handle = handle, - .idx = 0 - }; - - tag = neo_parse_tag_of_type(it->handle, type); - if (IS_ERR(tag)) - return PTR_ERR(tag); - - it->len = tag->size; - return 0; -} - -int neo_parse(const void *blob, size_t size, struct neo_dcp_parse_ctx *ctx) -{ - const u32 *header; - - if (size > U32_MAX) - return -EINVAL; - - *ctx = (struct neo_dcp_parse_ctx) { - .blob = blob, - .len = size, - .pos = 0, - }; - - header = neo_parse_u32(ctx); - if (IS_ERR(header)) - return PTR_ERR(header); - - if (*header != DCP_PARSE_HEADER) - return -EINVAL; - - return 0; -} - -static int neo_parse_dimension(struct neo_dcp_parse_ctx *handle, struct dimension *dim) -{ - struct iterator it; - int ret = 0; - - *dim = (struct dimension) { - .active = -1, - .total = -1, - .front_porch = -1, - .sync_width = -1, - .precise_sync_rate = -1, - }; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - char *key = neo_parse_string(it.handle); - - if (IS_ERR(key)) - ret = PTR_ERR(key); - else if (!strcmp(key, "Active")) - ret = neo_parse_int_bound(it.handle, &dim->active, 0, U16_MAX); - else if (!strcmp(key, "Total")) - ret = neo_parse_int_bound(it.handle, &dim->total, 0, U16_MAX); - else if (!strcmp(key, "FrontPorch")) - ret = neo_parse_int_bound(it.handle, &dim->front_porch, 0, U16_MAX); - else if (!strcmp(key, "SyncWidth")) - ret = neo_parse_int_bound(it.handle, &dim->sync_width, 0, U16_MAX); - else if (!strcmp(key, "PreciseSyncRate")) - ret = neo_parse_int_bound(it.handle, &dim->precise_sync_rate, 1, U32_MAX); - else - ret = skip(it.handle); - - if (!IS_ERR_OR_NULL(key)) - kfree(key); - - if (ret) - return ret; - } - - if (dim->active <= 0 || dim->total < dim->active || - dim->front_porch < 0 || dim->sync_width < 0 || - dim->front_porch + dim->sync_width > dim->total - dim->active) - return -EINVAL; - - return 0; -} - -struct color_mode { - s64 colorimetry; - s64 depth; - s64 dynamic_range; - s64 eotf; - s64 id; - s64 pixel_encoding; - s64 score; -}; - -static int fill_color_mode(struct neo_dcp_color_mode *color, - struct color_mode *cmode) -{ - if (cmode->id < 0 || cmode->id > U32_MAX) - return -EINVAL; - - if (color->score >= cmode->score) - return 0; - - if (cmode->colorimetry < 0 || cmode->colorimetry >= DCP_COLORIMETRY_COUNT) - return -EINVAL; - if (cmode->depth < 8 || cmode->depth > 12) - return -EINVAL; - if (cmode->dynamic_range < 0 || cmode->dynamic_range >= DCP_COLOR_YCBCR_RANGE_COUNT) - return -EINVAL; - if (cmode->eotf < 0 || cmode->eotf >= DCP_EOTF_COUNT) - return -EINVAL; - if (cmode->pixel_encoding < 0 || cmode->pixel_encoding >= DCP_COLOR_FORMAT_COUNT) - return -EINVAL; - - color->score = cmode->score; - color->id = cmode->id; - color->eotf = cmode->eotf; - color->format = cmode->pixel_encoding; - color->colorimetry = cmode->colorimetry; - color->range = cmode->dynamic_range; - color->depth = cmode->depth; - - return 0; -} - -static int neo_parse_color_modes(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_display_mode *out) -{ - struct iterator outer_it; - int ret = 0; - out->sdr_444.score = -1; - out->sdr_rgb.score = -1; - out->sdr.score = -1; - out->best.score = -1; - - ret = iterator_begin(handle, &outer_it, false); - if (ret) - return ret; - - for (; outer_it.idx < outer_it.len; ++outer_it.idx) { - struct iterator it; - bool is_virtual = true; - struct color_mode cmode = { - .colorimetry = -1, - .depth = -1, - .dynamic_range = -1, - .eotf = -1, - .id = -1, - .pixel_encoding = -1, - .score = -1, - }; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - char *key = neo_parse_string(it.handle); - - if (IS_ERR(key)) - ret = PTR_ERR(key); - else if (!strcmp(key, "Colorimetry")) - ret = neo_parse_int(it.handle, &cmode.colorimetry); - else if (!strcmp(key, "Depth")) - ret = neo_parse_int(it.handle, &cmode.depth); - else if (!strcmp(key, "DynamicRange")) - ret = neo_parse_int(it.handle, &cmode.dynamic_range); - else if (!strcmp(key, "EOTF")) - ret = neo_parse_int(it.handle, &cmode.eotf); - else if (!strcmp(key, "ID")) - ret = neo_parse_int(it.handle, &cmode.id); - else if (!strcmp(key, "IsVirtual")) - ret = neo_parse_bool(it.handle, &is_virtual); - else if (!strcmp(key, "PixelEncoding")) - ret = neo_parse_int(it.handle, &cmode.pixel_encoding); - else if (!strcmp(key, "Score")) - ret = neo_parse_int(it.handle, &cmode.score); - else - ret = skip(it.handle); - - if (!IS_ERR_OR_NULL(key)) - kfree(key); - - if (ret) - return ret; - } - - /* Skip virtual or partial entries */ - if (is_virtual || cmode.score < 0 || cmode.id < 0) - continue; - - trace_neo_iomfb_color_mode(handle->neo_dcp, cmode.id, cmode.score, - cmode.depth, cmode.colorimetry, - cmode.eotf, cmode.dynamic_range, - cmode.pixel_encoding); - - if (cmode.eotf == DCP_EOTF_SDR_GAMMA) { - if (cmode.pixel_encoding == DCP_COLOR_FORMAT_RGB && - cmode.depth <= 10) - fill_color_mode(&out->sdr_rgb, &cmode); - else if (cmode.pixel_encoding == DCP_COLOR_FORMAT_YCBCR444 && - cmode.depth <= 10) - fill_color_mode(&out->sdr_444, &cmode); - fill_color_mode(&out->sdr, &cmode); - } - fill_color_mode(&out->best, &cmode); - } - - return 0; -} - -/* - * Calculate the pixel clock for a mode given the 16:16 fixed-point refresh - * rate. The pixel clock is the refresh rate times the pixel count. DRM - * specifies the clock in kHz. The intermediate result may overflow a u32, so - * use a u64 where required. - */ -static u64 calculate_clock(struct dimension *horiz, struct dimension *vert) -{ - u32 pixels = horiz->total * vert->total; - u64 clock = mul_u32_u32(pixels, vert->precise_sync_rate); - - return DIV_ROUND_CLOSEST_ULL(clock >> 16, 1000); -} - -static int neo_parse_mode(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_display_mode *out, s64 *score, int width_mm, - int height_mm, unsigned notch_height, bool internal) -{ - int ret = 0; - struct iterator it; - struct dimension horiz = { .active = -1 }, vert = { .active = -1 }; - s64 min_vrr = 0, max_vrr = 0; - u64 clock; - s64 id = -1; - s64 best_color_mode = -1; - bool is_virtual = false; - struct drm_display_mode *mode = &out->mode; - - *out = (struct neo_dcp_display_mode) { - .sdr_rgb.score = -1, - .sdr_444.score = -1, - .sdr.score = -1, - .best.score = -1, - }; - *score = -1; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - char *key = neo_parse_string(it.handle); - - if (IS_ERR(key)) - ret = PTR_ERR(key); - else if (is_virtual) - ret = skip(it.handle); - else if (!strcmp(key, "HorizontalAttributes")) - ret = neo_parse_dimension(it.handle, &horiz); - else if (!strcmp(key, "VerticalAttributes")) - ret = neo_parse_dimension(it.handle, &vert); - else if (!strcmp(key, "MinimumVariableRefreshRate")) - ret = neo_parse_int_bound(it.handle, &min_vrr, 0, U32_MAX); - else if (!strcmp(key, "MaximumVariableRefreshRate")) - ret = neo_parse_int_bound(it.handle, &max_vrr, 0, U32_MAX); - else if (!strcmp(key, "ColorModes")) - ret = neo_parse_color_modes(it.handle, out); - else if (!strcmp(key, "ID")) - ret = neo_parse_int_bound(it.handle, &id, 0, U32_MAX); - else if (!strcmp(key, "IsVirtual")) - ret = neo_parse_bool(it.handle, &is_virtual); - else if (!strcmp(key, "Score")) - ret = neo_parse_int_bound(it.handle, score, 0, S64_MAX); - else - ret = skip(it.handle); - - if (!IS_ERR_OR_NULL(key)) - kfree(key); - - if (ret) { - trace_neo_iomfb_parse_mode_fail(id, &horiz, &vert, best_color_mode, is_virtual, *score); - return ret; - } - } - if (out->sdr_rgb.score >= 0) - best_color_mode = out->sdr_rgb.id; - else if (out->sdr_444.score >= 0) - best_color_mode = out->sdr_444.id; - else if (out->sdr.score >= 0) - best_color_mode = out->sdr.id; - else if (out->best.score >= 0) - best_color_mode = out->best.id; - - trace_neo_iomfb_parse_mode_success(id, &horiz, &vert, best_color_mode, - is_virtual, *score); - - /* - * Reject modes without valid color mode. - */ - if (best_color_mode < 0 || id < 0 || *score < 0 || - horiz.active <= 0 || vert.active <= notch_height || - vert.precise_sync_rate <= 0) - return -EINVAL; - - /* - * We need to skip virtual modes. In some cases, virtual modes are "too - * big" for the monitor and can cause breakage. It is unclear why the - * DCP reports these modes at all. Treat as a recoverable error. - */ - if (is_virtual) - return -EINVAL; - - /* - * An internal ProMotion panel carries no EDID or DisplayID, so DCP - * reports no adaptive-sync range for it. Assume the ProMotion floor of - * 24 Hz up to whatever rate the mode itself advertises. - */ - if (internal && vert.precise_sync_rate >> 16 == 120) { - out->min_vrr = 24 << 16; - out->max_vrr = vert.precise_sync_rate; - out->vrr = true; - } - - /* Refresh rates are reported by DCP as 16.16 fixed-point Hz. */ - if (min_vrr && max_vrr > min_vrr) { - out->min_vrr = min_vrr; - out->max_vrr = max_vrr; - out->vrr = true; - } - - clock = calculate_clock(&horiz, &vert); - if (!clock || clock > INT_MAX) - return -EINVAL; - - vert.active -= notch_height; - vert.sync_width += notch_height; - - /* From here we must succeed. Start filling out the mode. */ - *mode = (struct drm_display_mode) { - .type = DRM_MODE_TYPE_DRIVER, - .clock = clock, - - .vdisplay = vert.active, - .vsync_start = vert.active + vert.front_porch, - .vsync_end = vert.active + vert.front_porch + vert.sync_width, - .vtotal = vert.total, - - .hdisplay = horiz.active, - .hsync_start = horiz.active + horiz.front_porch, - .hsync_end = horiz.active + horiz.front_porch + - horiz.sync_width, - .htotal = horiz.total, - - .width_mm = width_mm, - .height_mm = height_mm, - }; - - drm_mode_set_name(mode); - - out->timing_mode_id = id; - out->color_mode_id = best_color_mode; - - trace_neo_iomfb_timing_mode(handle->neo_dcp, id, *score, horiz.active, - vert.active, vert.precise_sync_rate, - best_color_mode); - - return 0; -} - -struct neo_dcp_display_mode *neo_enumerate_modes(struct neo_dcp_parse_ctx *handle, - unsigned int *count, int width_mm, - int height_mm, unsigned notch_height, - bool internal) -{ - struct iterator it; - int ret; - struct neo_dcp_display_mode *mode, *modes; - struct neo_dcp_display_mode *best_mode = NULL; - s64 score, best_score = -1; - - ret = iterator_begin(handle, &it, false); - - if (ret) - return ERR_PTR(ret); - - /* Start with a worst case allocation. */ - modes = kcalloc(it.len, sizeof(*modes), GFP_KERNEL); - *count = 0; - - if (!modes) - return ERR_PTR(-ENOMEM); - - for (; it.idx < it.len; ++it.idx) { - struct neo_dcp_parse_ctx entry = *it.handle; - - mode = &modes[*count]; - ret = neo_parse_mode(it.handle, mode, &score, width_mm, height_mm, - notch_height, internal); - - /* Errors for a single mode are recoverable -- just skip it. */ - if (ret) { - /* A failed parse can stop in the middle of the dictionary. */ - *it.handle = entry; - ret = skip(it.handle); - if (ret) { - kfree(modes); - return ERR_PTR(ret); - } - continue; - } - - /* Process a successful mode */ - (*count)++; - - if (score > best_score) { - best_score = score; - best_mode = mode; - } - } - - if (best_mode != NULL) - best_mode->mode.type |= DRM_MODE_TYPE_PREFERRED; - - return modes; -} - -int neo_parse_display_attributes(struct neo_dcp_parse_ctx *handle, int *width_mm, - int *height_mm) -{ - int ret = 0; - struct iterator it; - s64 width_cm = 0, height_cm = 0; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - char *key = neo_parse_string(it.handle); - - if (IS_ERR(key)) - ret = PTR_ERR(key); - else if (!strcmp(key, "MaxHorizontalImageSize")) - ret = neo_parse_int(it.handle, &width_cm); - else if (!strcmp(key, "MaxVerticalImageSize")) - ret = neo_parse_int(it.handle, &height_cm); - else - ret = skip(it.handle); - - if (!IS_ERR_OR_NULL(key)) - kfree(key); - - if (ret) - return ret; - } - - /* 1cm = 10mm */ - *width_mm = 10 * width_cm; - *height_mm = 10 * height_cm; - - return 0; -} - -int neo_parse_epic_service_init(struct neo_dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit, bool h17p_keys) -{ - int ret = 0; - struct iterator it; - bool parsed_unit = false; - bool parsed_name = false; - bool parsed_class = false; - - *name = ERR_PTR(-ENOENT); - *class = ERR_PTR(-ENOENT); - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - char *key = neo_parse_string(it.handle); - - if (IS_ERR(key)) { - ret = PTR_ERR(key); - break; - } - - /* - * H17P advertises its system service with lower-case keys -- - * "compartment", "name", "interface-id" -- where earlier - * firmware uses EPICName/EPICProviderClass/EPICUnit. Without - * them the service never registers, the firmware has no system - * service to ask and idles the panel, so no vblank and no swap - * ever completes. Only H17P callers accept them. If a - * dictionary carries both spellings, the later value wins and - * the earlier string is freed. - */ - if (!strcmp(key, "EPICName") || - (h17p_keys && !strcmp(key, "compartment"))) { - if (parsed_name) - kfree(*name); - *name = neo_parse_string(it.handle); - if (IS_ERR(*name)) - ret = PTR_ERR(*name); - else - parsed_name = true; - } else if (!strcmp(key, "EPICProviderClass") || - (h17p_keys && !strcmp(key, "name"))) { - if (parsed_class) - kfree(*class); - *class = neo_parse_string(it.handle); - if (IS_ERR(*class)) - ret = PTR_ERR(*class); - else - parsed_class = true; - } else if (!strcmp(key, "EPICUnit") || - (h17p_keys && !strcmp(key, "interface-id"))) { - ret = neo_parse_int(it.handle, unit); - if (!ret) - parsed_unit = true; - } else { - ret = skip(it.handle); - } - - kfree(key); - if (ret) - break; - } - - if (!parsed_unit || !parsed_name || !parsed_class) - ret = -ENOENT; - - if (ret) { - if (!IS_ERR(*name)) { - kfree(*name); - *name = ERR_PTR(ret); - } - if (!IS_ERR(*class)) { - kfree(*class); - *class = ERR_PTR(ret); - } - } - - return ret; -} - -#if IS_ENABLED(CONFIG_DRM_APPLE_NEO_AUDIO) -static int neo_parse_sample_rate_bit(struct neo_dcp_parse_ctx *handle, unsigned int *ratebit) -{ - s64 rate; - int ret = neo_parse_int(handle, &rate); - - if (ret) - return ret; - - *ratebit = snd_pcm_rate_to_rate_bit(rate); - if (*ratebit == SNDRV_PCM_RATE_KNOT) { - /* - * The rate wasn't recognized, and unless we supply - * a supplementary constraint, the SNDRV_PCM_RATE_KNOT bit - * will allow any rate. So clear it. - */ - *ratebit = 0; - } - - return 0; -} - -static int neo_parse_sample_fmtbit(struct neo_dcp_parse_ctx *handle, u64 *fmtbit) -{ - s64 sample_size; - int ret = neo_parse_int(handle, &sample_size); - - if (ret) - return ret; - - switch (sample_size) { - case 16: - *fmtbit = SNDRV_PCM_FMTBIT_S16; - break; - case 20: - *fmtbit = SNDRV_PCM_FMTBIT_S20; - break; - case 24: - *fmtbit = SNDRV_PCM_FMTBIT_S24; - break; - case 32: - *fmtbit = SNDRV_PCM_FMTBIT_S32; - break; - default: - *fmtbit = 0; - break; - } - - return 0; -} - -static struct { - const char *label; - u8 type; -} chan_position_names[] = { - { "Front Left", SNDRV_CHMAP_FL }, - { "Front Right", SNDRV_CHMAP_FR }, - { "Rear Left", SNDRV_CHMAP_RL }, - { "Rear Right", SNDRV_CHMAP_RR }, - { "Front Center", SNDRV_CHMAP_FC }, - { "Low Frequency Effects", SNDRV_CHMAP_LFE }, - { "Rear Center", SNDRV_CHMAP_RC }, - { "Front Left Center", SNDRV_CHMAP_FLC }, - { "Front Right Center", SNDRV_CHMAP_FRC }, - { "Rear Left Center", SNDRV_CHMAP_RLC }, - { "Rear Right Center", SNDRV_CHMAP_RRC }, - { "Front Left Wide", SNDRV_CHMAP_FLW }, - { "Front Right Wide", SNDRV_CHMAP_FRW }, - { "Front Left High", SNDRV_CHMAP_FLH }, - { "Front Center High", SNDRV_CHMAP_FCH }, - { "Front Right High", SNDRV_CHMAP_FRH }, - { "Top Center", SNDRV_CHMAP_TC }, -}; - -static void append_chmap(struct snd_pcm_chmap_elem *chmap, u8 type) -{ - if (!chmap || chmap->channels >= ARRAY_SIZE(chmap->map)) - return; - - chmap->map[chmap->channels] = type; - chmap->channels++; -} - -static int neo_parse_chmap(struct neo_dcp_parse_ctx *handle, struct snd_pcm_chmap_elem *chmap) -{ - struct iterator it; - int i, ret; - - if (!chmap) - return skip(handle); - - chmap->channels = 0; - - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - for (i = 0; i < ARRAY_SIZE(chan_position_names); i++) - if (consume_string(it.handle, chan_position_names[i].label)) - break; - - if (i == ARRAY_SIZE(chan_position_names)) { - ret = skip(it.handle); - if (ret) - return ret; - - append_chmap(chmap, SNDRV_CHMAP_UNKNOWN); - continue; - } - - append_chmap(chmap, chan_position_names[i].type); - } - - return 0; -} - -static int neo_parse_chan_layout_element(struct neo_dcp_parse_ctx *handle, - unsigned int *nchans_out, - struct snd_pcm_chmap_elem *chmap) -{ - struct iterator it; - int ret; - s64 nchans = 0; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - if (consume_string(it.handle, "ActiveChannelCount")) - ret = neo_parse_int(it.handle, &nchans); - else if (consume_string(it.handle, "ChannelLayout")) - ret = neo_parse_chmap(it.handle, chmap); - else - ret = skip_pair(it.handle); - - if (ret) - return ret; - } - - if (nchans_out) - *nchans_out = nchans; - - return 0; -} - -static int neo_parse_nchans_mask(struct neo_dcp_parse_ctx *handle, unsigned int *mask) -{ - struct iterator it; - int ret; - - *mask = 0; - - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - int nchans; - - ret = neo_parse_chan_layout_element(it.handle, &nchans, NULL); - if (ret) - return ret; - *mask |= 1 << nchans; - } - - return 0; -} - -static int neo_parse_avep_element(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_sound_format_mask *sieve, - struct neo_dcp_sound_format_mask *hits) -{ - struct neo_dcp_sound_format_mask mask = {0, 0, 0}; - struct iterator it; - int ret; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - if (consume_string(handle, "StreamSampleRate")) - ret = neo_parse_sample_rate_bit(it.handle, &mask.rates); - else if (consume_string(handle, "SampleSize")) - ret = neo_parse_sample_fmtbit(it.handle, &mask.formats); - else if (consume_string(handle, "AudioChannelLayoutElements")) - ret = neo_parse_nchans_mask(it.handle, &mask.nchans); - else - ret = skip_pair(it.handle); - - if (ret) - return ret; - } - - trace_neo_avep_sound_mode(handle->neo_dcp, mask.rates, mask.formats, mask.nchans); - - if (!(mask.rates & sieve->rates) || !(mask.formats & sieve->formats) || - !(mask.nchans & sieve->nchans)) - return 0; - - if (hits) { - hits->rates |= mask.rates; - hits->formats |= mask.formats; - hits->nchans |= mask.nchans; - } - - return 1; -} - -static int neo_parse_mode_in_avep_element(struct neo_dcp_parse_ctx *handle, - unsigned int selected_nchans, - struct snd_pcm_chmap_elem *chmap, - struct neo_dcp_sound_cookie *cookie) -{ - struct iterator it; - struct neo_dcp_parse_ctx save_handle; - int ret; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - if (consume_string(it.handle, "AudioChannelLayoutElements")) { - struct iterator inner_it; - int nchans; - - ret = iterator_begin(it.handle, &inner_it, false); - if (ret) - return ret; - - for (; inner_it.idx < inner_it.len; ++inner_it.idx) { - save_handle = *it.handle; - ret = neo_parse_chan_layout_element(inner_it.handle, - &nchans, NULL); - if (ret) - return ret; - - if (nchans != selected_nchans) - continue; - - /* - * Now that we know this layout matches the - * selected channel number, reread the element - * and fill in the channel map. - */ - *inner_it.handle = save_handle; - ret = neo_parse_chan_layout_element(inner_it.handle, - NULL, chmap); - if (ret) - return ret; - } - } else if (consume_string(it.handle, "ElementData")) { - const u8 *blob; - - ret = neo_parse_blob(it.handle, sizeof(*cookie), &blob); - if (ret) - return ret; - - if (cookie) - memcpy(cookie, blob, sizeof(*cookie)); - } else { - ret = skip_pair(it.handle); - if (ret) - return ret; - } - } - - return 0; -} - -int neo_parse_sound_constraints(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_sound_format_mask *sieve, - struct neo_dcp_sound_format_mask *hits) -{ - int ret; - struct iterator it; - - if (hits) { - hits->rates = 0; - hits->formats = 0; - hits->nchans = 0; - } - - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - ret = neo_parse_avep_element(it.handle, sieve, hits); - - if (ret < 0) - return ret; - } - - return 0; -} - -int neo_parse_sound_mode(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_sound_format_mask *sieve, - struct snd_pcm_chmap_elem *chmap, - struct neo_dcp_sound_cookie *cookie) -{ - struct neo_dcp_parse_ctx save_handle; - struct iterator it; - int ret; - - ret = iterator_begin(handle, &it, false); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - save_handle = *it.handle; - ret = neo_parse_avep_element(it.handle, sieve, NULL); - - if (!ret) - continue; - - if (ret < 0) - return ret; - - ret = neo_parse_mode_in_avep_element(&save_handle, __ffs(sieve->nchans), - chmap, cookie); - if (ret < 0) - return ret; - return 1; - } - - return 0; -} -#endif - -int neo_parse_system_log_mnits(struct neo_dcp_parse_ctx *handle, struct neo_dcp_system_ev_mnits *entry) -{ - struct iterator it; - int ret; - s64 mnits = -1; - s64 idac = -1; - s64 timestamp = -1; - bool type_match = false; - - ret = iterator_begin(handle, &it, true); - if (ret) - return ret; - - for (; it.idx < it.len; ++it.idx) { - char *key = neo_parse_string(it.handle); - if (IS_ERR(key)) { - ret = PTR_ERR(key); - } else if (!strcmp(key, "mNits")) { - ret = neo_parse_int(it.handle, &mnits); - } else if (!strcmp(key, "iDAC")) { - ret = neo_parse_int(it.handle, &idac); - } else if (!strcmp(key, "logEvent")) { - const char *value = neo_parse_string(it.handle); - - ret = IS_ERR(value) ? PTR_ERR(value) : 0; - if (!ret) { - type_match = strcmp(value, "Display (Event Forward)") == 0; - kfree(value); - } - } else if (!strcmp(key, "timestamp")) { - ret = neo_parse_int(it.handle, ×tamp); - } else { - ret = skip(it.handle); - } - - if (!IS_ERR_OR_NULL(key)) - kfree(key); - - if (ret) { - pr_err("dcp parser: failed to parse mNits sys event\n"); - return ret; - } - } - - if (!type_match || mnits < 0 || idac < 0 || timestamp < 0) - return -EINVAL; - if (mnits > U32_MAX) - return -ERANGE; - - entry->millinits = mnits; - entry->idac = idac; - entry->timestamp = timestamp; - - return 0; -} diff --git a/drivers/gpu/drm/apple-neo/parser.h b/drivers/gpu/drm/apple-neo/parser.h deleted file mode 100644 index d62c33c3234166..00000000000000 --- a/drivers/gpu/drm/apple-neo/parser.h +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2021 Alyssa Rosenzweig */ - -#ifndef __APPLE_DCP_PARSER_H__ -#define __APPLE_DCP_PARSER_H__ - -/* For mode parsing */ -#include - -struct neo_apple_dcp; - -struct neo_dcp_parse_ctx { - struct neo_apple_dcp *neo_dcp; - const void *blob; - u32 pos, len; -}; - -enum neo_dcp_color_eotf { - DCP_EOTF_SDR_GAMMA = 0, // "SDR gamma" - DCP_EOTF_HDR_GAMMA = 1, // "HDR gamma" - DCP_EOTF_ST_2084 = 2, // "ST 2084 (PQ)" - DCP_EOTF_BT_2100 = 3, // "BT.2100 (HLG)" - DCP_EOTF_COUNT -}; - -enum neo_dcp_color_format { - DCP_COLOR_FORMAT_RGB = 0, // "RGB" - DCP_COLOR_FORMAT_YCBCR420 = 1, // "YUV 4:2:0" - DCP_COLOR_FORMAT_YCBCR422 = 3, // "YUV 4:2:2" - DCP_COLOR_FORMAT_YCBCR444 = 2, // "YUV 4:4:4" - DCP_COLOR_FORMAT_DV_NATIVE = 4, // "DolbyVision (native)" - DCP_COLOR_FORMAT_DV_HDMI = 5, // "DolbyVision (HDMI)" - DCP_COLOR_FORMAT_YCBCR422_DP = 6, // "YCbCr 4:2:2 (DP tunnel)" - DCP_COLOR_FORMAT_YCBCR422_HDMI = 7, // "YCbCr 4:2:2 (HDMI tunnel)" - DCP_COLOR_FORMAT_DV_LL_YCBCR422 = 8, // "DolbyVision LL YCbCr 4:2:2" - DCP_COLOR_FORMAT_DV_LL_YCBCR422_DP = 9, // "DolbyVision LL YCbCr 4:2:2 (DP)" - DCP_COLOR_FORMAT_DV_LL_YCBCR422_HDMI = 10, // "DolbyVision LL YCbCr 4:2:2 (HDMI)" - DCP_COLOR_FORMAT_DV_LL_YCBCR444 = 11, // "DolbyVision LL YCbCr 4:4:4" - DCP_COLOR_FORMAT_DV_LL_RGB422 = 12, // "DolbyVision LL RGB 4:2:2" - DCP_COLOR_FORMAT_GRGB_BLUE_422 = 13, // "GRGB as YCbCr422 (Even line blue)" - DCP_COLOR_FORMAT_GRGB_RED_422 = 14, // "GRGB as YCbCr422 (Even line red)" - DCP_COLOR_FORMAT_COUNT -}; - -enum neo_dcp_colorimetry { - DCP_COLORIMETRY_BT601 = 0, // "SMPTE 170M/BT.601" - DCP_COLORIMETRY_BT709 = 1, // "BT.701" - DCP_COLORIMETRY_XVYCC_601 = 2, // "xvYCC601" - DCP_COLORIMETRY_XVYCC_709 = 3, // "xvYCC709" - DCP_COLORIMETRY_SYCC_601 = 4, // "sYCC601" - DCP_COLORIMETRY_ADOBE_YCC_601 = 5, // "AdobeYCC601" - DCP_COLORIMETRY_BT2020_CYCC = 6, // "BT.2020 (c)" - DCP_COLORIMETRY_BT2020_YCC = 7, // "BT.2020 (nc)" - DCP_COLORIMETRY_VSVDB = 8, // "DolbyVision VSVDB" - DCP_COLORIMETRY_BT2020_RGB = 9, // "BT.2020 (RGB)" - DCP_COLORIMETRY_SRGB = 10, // "sRGB" - DCP_COLORIMETRY_SCRGB = 11, // "scRGB" - DCP_COLORIMETRY_SCRGB_FIXED = 12, // "scRGBfixed" - DCP_COLORIMETRY_ADOBE_RGB = 13, // "AdobeRGB" - DCP_COLORIMETRY_DCI_P3_RGB_D65 = 14, // "DCI-P3 (D65)" - DCP_COLORIMETRY_DCI_P3_RGB_THEATER = 15, // "DCI-P3 (Theater)" - DCP_COLORIMETRY_RGB = 16, // "Default RGB" - DCP_COLORIMETRY_COUNT -}; - -enum neo_dcp_color_range { - DCP_COLOR_YCBCR_RANGE_FULL = 0, - DCP_COLOR_YCBCR_RANGE_LIMITED = 1, - DCP_COLOR_YCBCR_RANGE_COUNT -}; - -struct neo_dcp_color_mode { - s64 score; - u32 id; - enum neo_dcp_color_eotf eotf; - enum neo_dcp_color_format format; - enum neo_dcp_colorimetry colorimetry; - enum neo_dcp_color_range range; - u8 depth; -}; - -/* - * Represents a single display mode. These mode objects are populated at - * runtime based on the TimingElements dictionary sent by the DCP. - */ -struct neo_dcp_display_mode { - struct drm_display_mode mode; - u32 color_mode_id; - u32 timing_mode_id; - u32 min_vrr; - u32 max_vrr; - struct neo_dcp_color_mode sdr_rgb; - struct neo_dcp_color_mode sdr_444; - struct neo_dcp_color_mode sdr; - struct neo_dcp_color_mode best; - bool vrr; -}; - -struct dimension { - s64 total, front_porch, sync_width, active; - s64 precise_sync_rate; -}; - -int neo_parse(const void *blob, size_t size, struct neo_dcp_parse_ctx *ctx); -struct neo_dcp_display_mode *neo_enumerate_modes(struct neo_dcp_parse_ctx *handle, - unsigned int *count, int width_mm, - int height_mm, unsigned notch_height, - bool internal); -int neo_parse_display_attributes(struct neo_dcp_parse_ctx *handle, int *width_mm, - int *height_mm); -int neo_parse_epic_service_init(struct neo_dcp_parse_ctx *handle, const char **name, - const char **class, s64 *unit, bool h17p_keys); - -struct neo_dcp_sound_format_mask { - u64 formats; /* SNDRV_PCM_FMTBIT_* */ - unsigned int rates; /* SNDRV_PCM_RATE_* */ - unsigned int nchans; -}; - -struct neo_dcp_sound_cookie { - u8 data[24]; -}; - -struct snd_pcm_chmap_elem; -int neo_parse_sound_constraints(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_sound_format_mask *sieve, - struct neo_dcp_sound_format_mask *hits); -int neo_parse_sound_mode(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_sound_format_mask *sieve, - struct snd_pcm_chmap_elem *chmap, - struct neo_dcp_sound_cookie *cookie); - -struct neo_dcp_system_ev_mnits { - u32 timestamp; - u32 millinits; - u32 idac; -}; - -int neo_parse_system_log_mnits(struct neo_dcp_parse_ctx *handle, - struct neo_dcp_system_ev_mnits *entry); - -#endif diff --git a/drivers/gpu/drm/apple-neo/plane.c b/drivers/gpu/drm/apple-neo/plane.c deleted file mode 100644 index 41e7215578f50c..00000000000000 --- a/drivers/gpu/drm/apple-neo/plane.c +++ /dev/null @@ -1,561 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Copyright (C) The Asahi Linux Contributors - */ - -#include "plane.h" - -#include "dcp-internal.h" -#include "iomfb_internal.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define FRAC_16_16(mult, div) (((mult) << 16) / (div)) - -static int neo_apple_plane_atomic_check(struct drm_plane *plane, - struct drm_atomic_state *state) -{ - struct drm_plane_state *new_plane_state; - struct drm_crtc_state *crtc_state; - struct drm_rect *dst; - int ret; - - new_plane_state = drm_atomic_get_new_plane_state(state, plane); - - if (!new_plane_state->crtc) - return 0; - - crtc_state = drm_atomic_get_crtc_state(state, new_plane_state->crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); - - /* - * DCP limits downscaling to 2x and upscaling to 4x. Attempting to - * scale outside these bounds errors out when swapping. - * - * This function also takes care of clipping the src/dest rectangles, - * which is required for correct operation. Partially off-screen - * surfaces may appear corrupted. - * - * DCP does not distinguish plane types in the hardware, so we set - * can_position. If the primary plane does not fill the screen, the - * hardware will fill in zeroes (black). - */ - ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state, - FRAC_16_16(1, 2), - FRAC_16_16(4, 1), - true, true); - if (ret < 0) - return ret; - - if (!new_plane_state->visible) - return 0; - - /* - * DCP does not allow a surface to clip off the screen, and will crash - * if any blended surface is smaller than 32x32. Reject the atomic op - * if the plane will crash DCP. - * - * This is most pertinent to cursors. Userspace should fall back to - * software cursors if the plane check is rejected. - */ - dst = &new_plane_state->dst; - if (drm_rect_width(dst) < 32 || drm_rect_height(dst) < 32) { - dev_err_once(state->dev->dev, - "Plane operation would have crashed DCP! Rejected!\n\ - DCP requires 32x32 of every plane to be within screen space.\n\ - Your compositor asked to overlay [%dx%d, %dx%d] on %dx%d.\n\ - This is not supported, and your compositor should have\n\ - switched to software compositing when this operation failed.\n\ - You should not have noticed this at all. If your screen\n\ - froze/hitched, or your compositor crashed, please report\n\ - this to the your compositor's developers. We will not\n\ - throw this error again until you next reboot.\n", - dst->x1, dst->y1, dst->x2, dst->y2, - crtc_state->mode.hdisplay, crtc_state->mode.vdisplay); - return -EINVAL; - } - - /* - * Pitches have to be 64-byte aligned. - */ - for (u32 i = 0; i < new_plane_state->fb->format->num_planes; i++) - if (new_plane_state->fb->pitches[i] & 63) - return -EINVAL; - - /* - * FIXME: dcp can currently only use multi-planar buffers using the same - * object for all planes. It has a mandatory iommu so it should - * be no problem to map multiple objects "linearly" into DCP - * virtual address space and calculate the offsets accordingly. - * Or maybe it can accept multiple BOs via the per plane field - * `base`. - */ - if (new_plane_state->fb->format->num_planes > 1) { - const struct drm_gem_object *first = new_plane_state->fb->obj[0]; - for (u32 i = 1; i < new_plane_state->fb->format->num_planes; i++) - if (new_plane_state->fb->obj[i] != NULL && - new_plane_state->fb->obj[i] != first) - return -EINVAL; - } - - return 0; -} - -/* - * DRM specifies rectangles as start and end coordinates. DCP specifies - * rectangles as a start coordinate and a width/height. Convert a DRM rectangle - * to a DCP rectangle. - */ -static struct neo_dcp_rect drm_to_dcp_rect(const struct drm_rect *rect) -{ - return (struct neo_dcp_rect){ .x = rect->x1, - .y = rect->y1, - .w = drm_rect_width(rect), - .h = drm_rect_height(rect), - }; -} - -static struct neo_dcp_rect drm_to_dcp_rect_fp(const struct drm_rect *fp_rect) -{ - struct drm_rect rect; - drm_rect_fp_to_int(&rect, fp_rect); - return drm_to_dcp_rect(&rect); -} - -static u32 drm_format_to_dcp(u32 drm, enum drm_color_range range) -{ - bool fr = range == DRM_COLOR_YCBCR_FULL_RANGE; - switch (drm) { - case DRM_FORMAT_XRGB8888: - case DRM_FORMAT_ARGB8888: - return DCP_FORMAT_BGRA; - - case DRM_FORMAT_XBGR8888: - case DRM_FORMAT_ABGR8888: - return DCP_FORMAT_RGBA; - - case DRM_FORMAT_XRGB2101010: - case DRM_FORMAT_ARGB2101010: - return DCP_FORMAT_L10R; - - /* semi planar YCbCr formats, limited and full range */ - case DRM_FORMAT_NV12: - return fr ? DCP_FORMAT_420F : DCP_FORMAT_420V; - case DRM_FORMAT_NV16: - return fr ? DCP_FORMAT_422F : DCP_FORMAT_422V; - case DRM_FORMAT_NV24: - return fr ? DCP_FORMAT_444F : DCP_FORMAT_444V; - - /* semi planar 10-bit YCbCr formats, limited and full range */ - case DRM_FORMAT_P010: - return fr ? DCP_FORMAT_XF20 : DCP_FORMAT_X420; - case DRM_FORMAT_P210: - return fr ? DCP_FORMAT_XF22 : DCP_FORMAT_X422; - /* - * TODO: missing DRM fourcc for P410 - */ -#if defined(DRM_FORMAT_P410) - case DRM_FORMAT_P410: - return fr ? DCP_FORMAT_XF44 : DCP_FORMAT_X444; -#endif - } - - pr_warn("DRM format %X not supported in DCP\n", drm); - return 0; -} - -static enum neo_dcp_xfer_func get_xfer_func(bool is_yuv, enum drm_color_encoding enc) -{ - if (!is_yuv) - return DCP_XFER_FUNC_SDR; - - switch (enc) { - case DRM_COLOR_YCBCR_BT601: - return DCP_XFER_FUNC_BT601; - case DRM_COLOR_YCBCR_BT709: - case DRM_COLOR_YCBCR_BT2020: - return DCP_XFER_FUNC_BT1886; - default: - return DCP_XFER_FUNC_SDR; - } -} - -static enum neo_dcp_colorspace get_colorspace(bool is_yuv, - enum drm_color_encoding enc) -{ - if (!is_yuv) - return DCP_COLORSPACE_NATIVE; - - switch (enc) { - case DRM_COLOR_YCBCR_BT601: - return DCP_COLORSPACE_BT601; - case DRM_COLOR_YCBCR_BT709: - return DCP_COLORSPACE_BT709; - case DRM_COLOR_YCBCR_BT2020: - return DCP_COLORSPACE_BG_BT2020; - default: - return DCP_COLORSPACE_NATIVE; - } -} - -static void neo_apple_plane_atomic_update(struct drm_plane *plane, - struct drm_atomic_state *state) -{ - struct drm_plane_state *base = drm_atomic_get_new_plane_state(state, plane); - struct neo_apple_plane_state *new_state; - struct drm_gem_dma_object *obj; - bool is_premultiplied = false; - - if (!base) - return; - - new_state = to_apple_plane_state(base); - - if (!base->fb) { - memset(&new_state->surf, 0, sizeof(new_state->surf)); - return; - } - - struct drm_framebuffer *fb = base->fb; - const struct drm_format_info *fmt = fb->format; - /* - * DCP doesn't support XBGR8 / XRGB8 / XBGR2101010 natively. Blending as - * pre-multiplied alpha with a black background can be used as - * workaround for the bottommost plane. - */ - if (fmt->format == DRM_FORMAT_XRGB8888 || - fmt->format == DRM_FORMAT_XBGR8888 || - fmt->format == DRM_FORMAT_XBGR2101010) - is_premultiplied = true; - - new_state->src_rect = drm_to_dcp_rect_fp(&base->src); - new_state->dst_rect = drm_to_dcp_rect(&base->dst); - - new_state->surf = (struct neo_dcp_surface){ - .is_premultiplied = is_premultiplied, - .plane_cnt = fb->format->num_planes, - .plane_cnt2 = fb->format->num_planes, - .format = drm_format_to_dcp(fmt->format, base->color_range), - .xfer_func = get_xfer_func(fmt->is_yuv, base->color_encoding), - .colorspace = get_colorspace(fmt->is_yuv, base->color_encoding), - .stride = fb->pitches[0], - .width = fb->width, - .height = fb->height, - .buf_size = fb->height * fb->pitches[0], - // .surface_id = req->swap.surf_ids[l], - - /* Only used for compressed or multiplanar surfaces */ - .pix_size = 1, - .pel_w = 1, - .pel_h = 1, - .has_comp = 1, - .has_planes = 1, - }; - - /* Populate plane information for planar formats */ - struct neo_dcp_surface *surf = &new_state->surf; - for (int i = 0; fb->format->num_planes && i < fb->format->num_planes; i++) { - u32 width = drm_format_info_plane_width(fb->format, fb->width, i); - u32 height = drm_format_info_plane_height(fb->format, fb->height, i); - u32 bh = drm_format_info_block_height(fb->format, i); - u32 bw = drm_format_info_block_width(fb->format, i); - - surf->planes[i] = (struct neo_dcp_plane_info){ - .width = width, - .height = height, - .base = fb->offsets[i] - fb->offsets[0], - .offset = fb->offsets[i] - fb->offsets[0], - .stride = fb->pitches[i], - .size = height * fb->pitches[i], - .tile_size = bw * bh, - .tile_w = bw, - .tile_h = bh, - }; - - if (i > 0) - surf->buf_size += surf->planes[i].size; - } - - /* the obvious helper call drm_fb_dma_get_gem_addr() adjusts - * the address for source x/y offsets. Since IOMFB has a direct - * support source position prefer that. - */ - obj = drm_fb_dma_get_gem_obj(base->fb, 0); - if (obj) - new_state->iova = obj->dma_addr + base->fb->offsets[0]; -} - -static void neo_apple_plane_cleanup_fb(struct drm_plane *plane, - struct drm_plane_state *state) -{ - struct neo_apple_plane_state *apple_state = to_apple_plane_state(state); - struct neo_dcp_fb_reference *entry = apple_state->retirement; - - if (!entry) - return; - - apple_state->retirement = NULL; - drm_framebuffer_put(entry->fb); - kfree(entry); -} - -static int neo_apple_plane_prepare_fb(struct drm_plane *plane, - struct drm_plane_state *state) -{ - struct drm_plane_state *old; - struct neo_dcp_fb_reference *entry; - int ret; - - ret = drm_gem_plane_helper_prepare_fb(plane, state); - if (ret) - return ret; - - old = drm_atomic_get_old_plane_state(state->state, plane); - if (!old || !old->crtc || !old->fb || old->fb == state->fb) - return 0; - - entry = kzalloc_obj(*entry); - if (!entry) - return -ENOMEM; - - drm_framebuffer_get(old->fb); - entry->fb = old->fb; - to_apple_plane_state(state)->retirement = entry; - return 0; -} - -static const struct drm_plane_helper_funcs apple_primary_plane_helper_funcs = { - .prepare_fb = neo_apple_plane_prepare_fb, - .cleanup_fb = neo_apple_plane_cleanup_fb, - .atomic_check = neo_apple_plane_atomic_check, - .atomic_update = neo_apple_plane_atomic_update, - .get_scanout_buffer = drm_fb_dma_get_scanout_buffer, -}; - -static const struct drm_plane_helper_funcs neo_apple_plane_helper_funcs = { - .prepare_fb = neo_apple_plane_prepare_fb, - .cleanup_fb = neo_apple_plane_cleanup_fb, - .atomic_check = neo_apple_plane_atomic_check, - .atomic_update = neo_apple_plane_atomic_update, -}; - -// Duplicate drm_atomic_helper_plane_reset but allocate struct apple_plane_state -static void neo_apple_plane_reset(struct drm_plane *plane) -{ - struct neo_apple_plane_state *state = to_apple_plane_state(plane->state); - if (state) { - neo_apple_plane_cleanup_fb(plane, &state->base); - __drm_atomic_helper_plane_destroy_state(&state->base); - } - - kfree(state); - plane->state = NULL; - state = kzalloc(sizeof(*state), GFP_KERNEL); - if (state) - __drm_atomic_helper_plane_reset(plane, &state->base); -} - -static struct drm_plane_state * -neo_apple_plane_duplicate_state(struct drm_plane *plane) -{ - struct neo_apple_plane_state *neo_apple_plane_state, *old_apple_plane_state; - - if (!plane->state) - return NULL; - - old_apple_plane_state = to_apple_plane_state(plane->state); - neo_apple_plane_state = kzalloc(sizeof(*neo_apple_plane_state), GFP_KERNEL); - if (!neo_apple_plane_state) - return NULL; - - __drm_atomic_helper_plane_duplicate_state(plane, &neo_apple_plane_state->base); - - neo_apple_plane_state->surf = old_apple_plane_state->surf; - neo_apple_plane_state->src_rect = old_apple_plane_state->src_rect; - neo_apple_plane_state->dst_rect = old_apple_plane_state->dst_rect; - neo_apple_plane_state->iova = old_apple_plane_state->iova; - - return &neo_apple_plane_state->base; -} - -static void neo_apple_plane_destroy_state(struct drm_plane *plane, - struct drm_plane_state *state) -{ - neo_apple_plane_cleanup_fb(plane, state); - drm_atomic_helper_plane_destroy_state(plane, state); -} - -static const struct drm_plane_funcs neo_apple_plane_funcs = { - .update_plane = drm_atomic_helper_update_plane, - .disable_plane = drm_atomic_helper_disable_plane, - .reset = neo_apple_plane_reset, - .atomic_duplicate_state = neo_apple_plane_duplicate_state, - .atomic_destroy_state = neo_apple_plane_destroy_state, -}; - -/* - * Table of supported formats, mapping from DRM fourccs to DCP fourccs. - * - * For future work, DCP supports more formats not listed, including YUV - * formats, an extra RGBA format, and a biplanar RGB10_A8 format (fourcc b3a8) - * used for HDR. - * - * Note: we don't have non-alpha formats but userspace breaks without XRGB. It - * doesn't matter for the primary plane, but cursors/overlays must not - * advertise formats without alpha. - */ -static const u32 neo_dcp_primary_formats[] = { - DRM_FORMAT_XRGB2101010, - DRM_FORMAT_ARGB2101010, - DRM_FORMAT_XRGB8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_XBGR8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_NV12, - DRM_FORMAT_NV16, - DRM_FORMAT_NV24, - DRM_FORMAT_P010, - DRM_FORMAT_P210, -#if defined(DRM_FORMAT_P410) - DRM_FORMAT_P410, -#endif -}; - -/* H17P 10-bit XRGB alpha semantics have not been qualified. */ -static const u32 neo_dcp_primary_formats_h17p[] = { - DRM_FORMAT_ARGB2101010, - DRM_FORMAT_XRGB8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_XBGR8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_NV12, - DRM_FORMAT_NV16, - DRM_FORMAT_NV24, - DRM_FORMAT_P010, - DRM_FORMAT_P210, -#if defined(DRM_FORMAT_P410) - DRM_FORMAT_P410, -#endif -}; - -static const u32 neo_dcp_overlay_formats[] = { - DRM_FORMAT_ARGB2101010, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_NV12, - DRM_FORMAT_NV16, - DRM_FORMAT_NV24, - DRM_FORMAT_P010, - DRM_FORMAT_P210, -#if defined(DRM_FORMAT_P410) - DRM_FORMAT_P410, -#endif -}; - -/* - * Formats for the 12.x firmware which does not support "l10r" / ARGB2101010 - */ -static const u32 neo_dcp_primary_formats_12_x[] = { - DRM_FORMAT_XRGB8888, - DRM_FORMAT_ARGB8888, - DRM_FORMAT_XBGR8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_NV12, - DRM_FORMAT_NV16, - DRM_FORMAT_NV24, - DRM_FORMAT_P010, - DRM_FORMAT_P210, -#if defined(DRM_FORMAT_P410) - DRM_FORMAT_P410, -#endif -}; - -static const u32 neo_dcp_overlay_formats_12_x[] = { - DRM_FORMAT_ARGB8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_NV12, - DRM_FORMAT_NV16, - DRM_FORMAT_NV24, - DRM_FORMAT_P010, - DRM_FORMAT_P210, -#if defined(DRM_FORMAT_P410) - DRM_FORMAT_P410, -#endif -}; - -u64 neo_apple_format_modifiers[] = { - DRM_FORMAT_MOD_LINEAR, - DRM_FORMAT_MOD_INVALID -}; - -struct drm_plane *neo_apple_plane_init(struct drm_device *dev, - unsigned long possible_crtcs, - u32 neo_iomfb_surf, - bool supports_l10r, - bool supports_xrgb2101010, - enum drm_plane_type type) -{ - struct neo_apple_plane *plane; - const u32 *fmts; - u32 num_fmts; - - switch (type) { - case DRM_PLANE_TYPE_PRIMARY: - if (supports_l10r && supports_xrgb2101010) { - fmts = neo_dcp_primary_formats; - num_fmts = ARRAY_SIZE(neo_dcp_primary_formats); - } else if (supports_l10r) { - fmts = neo_dcp_primary_formats_h17p; - num_fmts = ARRAY_SIZE(neo_dcp_primary_formats_h17p); - } else { - fmts = neo_dcp_primary_formats_12_x; - num_fmts = ARRAY_SIZE(neo_dcp_primary_formats_12_x); - } - plane = drmm_universal_plane_alloc(dev, struct neo_apple_plane, base, possible_crtcs, - &neo_apple_plane_funcs, fmts, num_fmts, - neo_apple_format_modifiers, type, NULL); - break; - case DRM_PLANE_TYPE_OVERLAY: - case DRM_PLANE_TYPE_CURSOR: - if (supports_l10r) { - fmts = neo_dcp_overlay_formats; - num_fmts = ARRAY_SIZE(neo_dcp_overlay_formats); - } else { - fmts = neo_dcp_overlay_formats_12_x; - num_fmts = ARRAY_SIZE(neo_dcp_overlay_formats_12_x); - } - plane = drmm_universal_plane_alloc(dev, struct neo_apple_plane, base, possible_crtcs, - &neo_apple_plane_funcs, fmts, num_fmts, - neo_apple_format_modifiers, type, NULL); - break; - default: - return ERR_PTR(-EINVAL); - } - - if (IS_ERR(plane)) - return ERR_PTR(PTR_ERR(plane)); - - drm_plane_create_color_properties(&plane->base, - (1 << DRM_COLOR_ENCODING_MAX) - 1, - (1 << DRM_COLOR_RANGE_MAX) - 1, - DRM_COLOR_YCBCR_BT709, - DRM_COLOR_YCBCR_LIMITED_RANGE); - - if (type == DRM_PLANE_TYPE_PRIMARY) - drm_plane_helper_add(&plane->base, &apple_primary_plane_helper_funcs); - else - drm_plane_helper_add(&plane->base, &neo_apple_plane_helper_funcs); - - plane->neo_iomfb_surf = neo_iomfb_surf; - - return &plane->base; -} diff --git a/drivers/gpu/drm/apple-neo/plane.h b/drivers/gpu/drm/apple-neo/plane.h deleted file mode 100644 index ed8080a29915cc..00000000000000 --- a/drivers/gpu/drm/apple-neo/plane.h +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* - * Copyright (C) The Asahi Linux Contributors - */ - -#ifndef __APPLE_PLANE_H__ -#define __APPLE_PLANE_H__ - -#include - -#include - -#include "iomfb_plane.h" - -struct neo_apple_plane { - struct drm_plane base; - u32 neo_iomfb_surf; -}; - -#define to_apple_plane(x) container_of(x, struct neo_apple_plane, base) - -struct neo_dcp_fb_reference; - -struct neo_apple_plane_state { - struct drm_plane_state base; - struct neo_dcp_surface surf; - struct neo_dcp_rect src_rect; - struct neo_dcp_rect dst_rect; - u64 iova; - struct neo_dcp_fb_reference *retirement; -}; - -#define to_apple_plane_state(x) container_of(x, struct neo_apple_plane_state, base) - -struct drm_plane *neo_apple_plane_init(struct drm_device *dev, - unsigned long possible_crtcs, - u32 neo_iomfb_surf, - bool supports_l10r, - bool supports_xrgb2101010, - enum drm_plane_type type); - -#endif /* __APPLE_PLANE_H__ */ diff --git a/drivers/gpu/drm/apple-neo/systemep.c b/drivers/gpu/drm/apple-neo/systemep.c deleted file mode 100644 index 38a8f28b9c4537..00000000000000 --- a/drivers/gpu/drm/apple-neo/systemep.c +++ /dev/null @@ -1,150 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright 2022 Sven Peter */ - -#include - -#include "afk.h" -#include "dcp.h" -#include "parser.h" - -static bool enable_verbose_logging; -module_param(enable_verbose_logging, bool, 0644); -MODULE_PARM_DESC(enable_verbose_logging, "Enable DCP firmware verbose logging"); - -/* - * Serialized setProperty("gAFKConfigLogMask", 0xffff) IPC call which - * will set the DCP firmware log level to the most verbose setting - */ -#define SYSTEM_SET_PROPERTY 0x43 -static const u8 setprop_gAFKConfigLogMask_ffff[] = { - 0x14, 0x00, 0x00, 0x00, 0x67, 0x41, 0x46, 0x4b, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x4c, 0x6f, 0x67, 0x4d, 0x61, 0x73, - 0x6b, 0x00, 0x00, 0x00, 0xd3, 0x00, 0x00, 0x00, 0x40, 0x00, - 0x00, 0x84, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; - -struct neo_systemep_work { - struct neo_apple_epic_service *service; - struct work_struct work; -}; - -static void system_log_work(struct work_struct *work_) -{ - struct neo_systemep_work *work = - container_of(work_, struct neo_systemep_work, work); - - neo_afk_send_command(work->service, SYSTEM_SET_PROPERTY, - setprop_gAFKConfigLogMask_ffff, - sizeof(setprop_gAFKConfigLogMask_ffff), NULL, - sizeof(setprop_gAFKConfigLogMask_ffff), NULL); - complete(&work->service->ep->neo_dcp->neo_systemep_done); - kfree(work); -} - -static void system_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ - struct neo_systemep_work *work; - - if (!enable_verbose_logging) - return; - - /* - * We're called from the service message handler thread and can't - * dispatch blocking message from there. - */ - work = kzalloc(sizeof(*work), GFP_KERNEL); - if (!work) - return; - - work->service = service; - INIT_WORK(&work->work, system_log_work); - schedule_work(&work->work); -} - -static void powerlog_init(struct neo_apple_epic_service *service, const char *name, - const char *class, s64 unit) -{ -} - -static int powerlog_report(struct neo_apple_epic_service *service, enum neo_epic_subtype type, - const void *data, size_t data_size) -{ - struct neo_dcp_system_ev_mnits mnits; - struct neo_dcp_parse_ctx neo_parse_ctx; - struct neo_apple_dcp *neo_dcp = service->ep->neo_dcp; - int ret; - - /* Later reports must not overwrite a latched hint or Linux's target. */ - if (neo_dcp_backlight_active(neo_dcp)) - return 0; - - dev_dbg(neo_dcp->dev, "systemep[ch:%u]: report type:%02x len:%zu\n", - service->channel, type, data_size); - - if (type != EPIC_SUBTYPE_STD_SERVICE) - return 0; - - ret = neo_parse(data, data_size, &neo_parse_ctx); - if (ret) { - dev_warn(service->ep->neo_dcp->dev, "systemep: failed to parse report: %d\n", ret); - return ret; - } - - ret = neo_parse_system_log_mnits(&neo_parse_ctx, &mnits); - if (ret) { - /* ignore parse errors in the case dcp sends unknown log events */ - dev_dbg(neo_dcp->dev, "systemep: failed to parse mNits event: %d\n", ret); - return 0; - } - - dev_dbg(neo_dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", - mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); - if (neo_dcp->fw_compat == DCP_FIRMWARE_H17P && - neo_dcp->hw.neo_iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - if (!neo_dcp_has_panel(neo_dcp)) - return 0; - ret = neo_dcp_backlight_takeover(neo_dcp, mnits.millinits); - if (ret && ret != -EBUSY) - dev_dbg(neo_dcp->dev, "backlight takeover hint unavailable: %d\n", ret); - return 0; - } - - neo_dcp->brightness.nits = mnits.millinits / 1000; - - return 0; -} - -static const struct neo_apple_epic_service_ops neo_systemep_ops[] = { - { - .name = "system", - .init = system_init, - }, - { - .name = "powerlog-service", - .init = powerlog_init, - .report = powerlog_report, - }, - {} -}; - -int neo_systemep_init(struct neo_apple_dcp *neo_dcp) -{ - init_completion(&neo_dcp->neo_systemep_done); - - neo_dcp->neo_systemep = neo_afk_init(neo_dcp, SYSTEM_ENDPOINT, neo_systemep_ops); - neo_afk_start(neo_dcp->neo_systemep); - - if (!enable_verbose_logging) - return 0; - - /* - * Timeouts aren't really fatal here: in the worst case we just weren't - * able to enable additional debug prints inside DCP - */ - if (!wait_for_completion_timeout(&neo_dcp->neo_systemep_done, - msecs_to_jiffies(MSEC_PER_SEC))) - dev_err(neo_dcp->dev, "systemep: couldn't enable verbose logs\n"); - - return 0; -} diff --git a/drivers/gpu/drm/apple-neo/trace.c b/drivers/gpu/drm/apple-neo/trace.c deleted file mode 100644 index 6f40d5a583df01..00000000000000 --- a/drivers/gpu/drm/apple-neo/trace.c +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ OR MIT -/* - * Tracepoints for Apple DCP driver - * - * Copyright (C) The Asahi Linux Contributors - */ - -#define CREATE_TRACE_POINTS -#include "trace.h" diff --git a/drivers/gpu/drm/apple-neo/trace.h b/drivers/gpu/drm/apple-neo/trace.h deleted file mode 100644 index 1cfeeaaebe7ae5..00000000000000 --- a/drivers/gpu/drm/apple-neo/trace.h +++ /dev/null @@ -1,629 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright (C) The Asahi Linux Contributors */ - -#undef TRACE_SYSTEM -#define TRACE_SYSTEM neo_dcp - -#if !defined(_TRACE_DCP_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_DCP_H - -#include "afk.h" -#include "dptxep.h" -#include "dcp-internal.h" -#include "parser.h" - -#include -#include -#include - -#define show_dcp_endpoint(ep) \ - __print_symbolic(ep, { SYSTEM_ENDPOINT, "system" }, \ - { TEST_ENDPOINT, "test" }, \ - { DCP_EXPERT_ENDPOINT, "dcpexpert" }, \ - { DISP0_ENDPOINT, "disp0" }, \ - { DPTX_ENDPOINT, "dptxport" }, \ - { HDCP_ENDPOINT, "hdcp" }, \ - { REMOTE_ALLOC_ENDPOINT, "remotealloc" }, \ - { IOMFB_ENDPOINT, "iomfb" }) -#define print_epic_type(etype) \ - __print_symbolic(etype, { EPIC_TYPE_NOTIFY, "notify" }, \ - { EPIC_TYPE_COMMAND, "command" }, \ - { EPIC_TYPE_REPLY, "reply" }, \ - { EPIC_TYPE_NOTIFY_ACK, "notify-ack" }) - -#define print_epic_category(ecat) \ - __print_symbolic(ecat, { EPIC_CAT_REPORT, "report" }, \ - { EPIC_CAT_NOTIFY, "notify" }, \ - { EPIC_CAT_REPLY, "reply" }, \ - { EPIC_CAT_COMMAND, "command" }) - -#define show_dptxport_apcall(idx) \ - __print_symbolic( \ - idx, { DPTX_APCALL_ACTIVATE, "activate" }, \ - { DPTX_APCALL_DEACTIVATE, "deactivate" }, \ - { DPTX_APCALL_GET_MAX_DRIVE_SETTINGS, \ - "get_max_drive_settings" }, \ - { DPTX_APCALL_SET_DRIVE_SETTINGS, "set_drive_settings" }, \ - { DPTX_APCALL_GET_DRIVE_SETTINGS, "get_drive_settings" }, \ - { DPTX_APCALL_WILL_CHANGE_LINKG_CONFIG, \ - "will_change_link_config" }, \ - { DPTX_APCALL_DID_CHANGE_LINK_CONFIG, \ - "did_change_link_config" }, \ - { DPTX_APCALL_GET_MAX_LINK_RATE, "get_max_link_rate" }, \ - { DPTX_APCALL_GET_LINK_RATE, "get_link_rate" }, \ - { DPTX_APCALL_SET_LINK_RATE, "set_link_rate" }, \ - { DPTX_APCALL_GET_MAX_LANE_COUNT, \ - "get_max_lane_count" }, \ - { DPTX_APCALL_GET_ACTIVE_LANE_COUNT, \ - "get_active_lane_count" }, \ - { DPTX_APCALL_SET_ACTIVE_LANE_COUNT, \ - "set_active_lane_count" }, \ - { DPTX_APCALL_GET_SUPPORTS_DOWN_SPREAD, \ - "get_supports_downspread" }, \ - { DPTX_APCALL_GET_DOWN_SPREAD, "get_downspread" }, \ - { DPTX_APCALL_SET_DOWN_SPREAD, "set_downspread" }, \ - { DPTX_APCALL_GET_SUPPORTS_LANE_MAPPING, \ - "get_supports_lane_mapping" }, \ - { DPTX_APCALL_SET_LANE_MAP, "set_lane_map" }, \ - { DPTX_APCALL_GET_SUPPORTS_HPD, "get_supports_hpd" }, \ - { DPTX_APCALL_FORCE_HOTPLUG_DETECT, "force_hotplug_detect" }, \ - { DPTX_APCALL_INACTIVE_SINK_DETECTED, \ - "inactive_sink_detected" }, \ - { DPTX_APCALL_SET_TILED_DISPLAY_HINTS, \ - "set_tiled_display_hints" }, \ - { DPTX_APCALL_DEVICE_NOT_RESPONDING, \ - "device_not_responding" }, \ - { DPTX_APCALL_DEVICE_BUSY_TIMEOUT, "device_busy_timeout" }, \ - { DPTX_APCALL_DEVICE_NOT_STARTED, "device_not_started" }) - -TRACE_EVENT(neo_dcp_recv_msg, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u8 endpoint, u64 message), - TP_ARGS(neo_dcp, endpoint, message), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dcp->dev)) - __field(u8, endpoint) - __field(u64, message)), - - TP_fast_assign(__assign_str(devname); - __entry->endpoint = endpoint; - __entry->message = message;), - - TP_printk("%s: endpoint 0x%x (%s): received message 0x%016llx", - __get_str(devname), __entry->endpoint, - show_dcp_endpoint(__entry->endpoint), __entry->message)); - -TRACE_EVENT(neo_dcp_send_msg, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u8 endpoint, u64 message), - TP_ARGS(neo_dcp, endpoint, message), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dcp->dev)) - __field(u8, endpoint) - __field(u64, message)), - - TP_fast_assign(__assign_str(devname); - __entry->endpoint = endpoint; - __entry->message = message;), - - TP_printk("%s: endpoint 0x%x (%s): will send message 0x%016llx", - __get_str(devname), __entry->endpoint, - show_dcp_endpoint(__entry->endpoint), __entry->message)); - -TRACE_EVENT( - neo_afk_getbuf, TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 size, u16 tag), - TP_ARGS(ep, size, tag), - - TP_STRUCT__entry(__string(devname, dev_name(ep->neo_dcp->dev)) - __field(u8, endpoint) __field(u32, size) - __field(u16, tag)), - - TP_fast_assign(__assign_str(devname); - __entry->endpoint = ep->endpoint; __entry->size = size; - __entry->tag = tag;), - - TP_printk( - "%s: endpoint 0x%x (%s): get buffer with size 0x%x and tag 0x%x", - __get_str(devname), __entry->endpoint, - show_dcp_endpoint(__entry->endpoint), __entry->size, - __entry->tag)); - -DECLARE_EVENT_CLASS(neo_afk_rwptr_template, - TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 rptr, u32 wptr), - TP_ARGS(ep, rptr, wptr), - - TP_STRUCT__entry(__string(devname, dev_name(ep->neo_dcp->dev)) - __field(u8, endpoint) __field(u32, rptr) - __field(u32, wptr)), - - TP_fast_assign(__assign_str(devname); - __entry->endpoint = ep->endpoint; - __entry->rptr = rptr; __entry->wptr = wptr;), - - TP_printk("%s: endpoint 0x%x (%s): rptr 0x%x, wptr 0x%x", - __get_str(devname), __entry->endpoint, - show_dcp_endpoint(__entry->endpoint), __entry->rptr, - __entry->wptr)); - -DEFINE_EVENT(neo_afk_rwptr_template, neo_afk_recv_rwptr_pre, - TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 rptr, u32 wptr), - TP_ARGS(ep, rptr, wptr)); -DEFINE_EVENT(neo_afk_rwptr_template, neo_afk_recv_rwptr_post, - TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 rptr, u32 wptr), - TP_ARGS(ep, rptr, wptr)); -DEFINE_EVENT(neo_afk_rwptr_template, neo_afk_send_rwptr_pre, - TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 rptr, u32 wptr), - TP_ARGS(ep, rptr, wptr)); -DEFINE_EVENT(neo_afk_rwptr_template, neo_afk_send_rwptr_post, - TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 rptr, u32 wptr), - TP_ARGS(ep, rptr, wptr)); - -TRACE_EVENT( - neo_afk_recv_qe, - TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 rptr, u32 magic, u32 size), - TP_ARGS(ep, rptr, magic, size), - - TP_STRUCT__entry(__string(devname, dev_name(ep->neo_dcp->dev)) - __field(u8, endpoint) __field(u32, rptr) - __field(u32, magic) - __field(u32, size)), - - TP_fast_assign(__assign_str(devname); - __entry->endpoint = ep->endpoint; __entry->rptr = rptr; - __entry->magic = magic; __entry->size = size;), - - TP_printk("%s: endpoint 0x%x (%s): QE rptr 0x%x, magic 0x%x, size 0x%x", - __get_str(devname), __entry->endpoint, - show_dcp_endpoint(__entry->endpoint), __entry->rptr, - __entry->magic, __entry->size)); - -TRACE_EVENT( - neo_afk_recv_handle, - TP_PROTO(struct neo_apple_dcp_afkep *ep, u32 channel, u32 type, - u32 data_size, struct neo_epic_hdr *ehdr, - struct neo_epic_sub_hdr *eshdr), - TP_ARGS(ep, channel, type, data_size, ehdr, eshdr), - - TP_STRUCT__entry(__string(devname, dev_name(ep->neo_dcp->dev)) __field( - u8, endpoint) __field(u32, channel) __field(u32, type) - __field(u32, data_size) __field(u8, category) - __field(u16, subtype) - __field(u16, tag)), - - TP_fast_assign(__assign_str(devname); - __entry->endpoint = ep->endpoint; - __entry->channel = channel; __entry->type = type; - __entry->data_size = data_size; - __entry->category = eshdr->category, - __entry->subtype = le16_to_cpu(eshdr->type), - __entry->tag = le16_to_cpu(eshdr->tag)), - - TP_printk( - "%s: endpoint 0x%x (%s): channel 0x%x, type 0x%x (%s), data_size 0x%x, category: 0x%x (%s), subtype: 0x%x, seq: 0x%x", - __get_str(devname), __entry->endpoint, - show_dcp_endpoint(__entry->endpoint), __entry->channel, - __entry->type, print_epic_type(__entry->type), - __entry->data_size, __entry->category, - print_epic_category(__entry->category), __entry->subtype, - __entry->tag)); - -TRACE_EVENT(neo_iomfb_callback, - TP_PROTO(struct neo_apple_dcp *neo_dcp, int tag, const char *name), - TP_ARGS(neo_dcp, tag, name), - - TP_STRUCT__entry( - __string(devname, dev_name(neo_dcp->dev)) - __field(int, tag) - __field(const char *, name) - ), - - TP_fast_assign( - __assign_str(devname); - __entry->tag = tag; __entry->name = name; - ), - - TP_printk("%s: Callback D%03d %s", __get_str(devname), __entry->tag, - __entry->name)); - -TRACE_EVENT(neo_iomfb_push, - TP_PROTO(struct neo_apple_dcp *neo_dcp, - const struct neo_dcp_method_entry *method, int context, - int offset, int depth), - TP_ARGS(neo_dcp, method, context, offset, depth), - - TP_STRUCT__entry( - __string(devname, dev_name(neo_dcp->dev)) - __string(name, method->name) - __field(int, context) - __field(int, offset) - __field(int, depth)), - - TP_fast_assign( - __assign_str(devname); - __assign_str(name); - __entry->context = context; __entry->offset = offset; - __entry->depth = depth; - ), - - TP_printk("%s: Method %s: context %u, offset %u, depth %u", - __get_str(devname), __get_str(name), __entry->context, - __entry->offset, __entry->depth)); - -TRACE_EVENT(neo_iomfb_swap_submit, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 swap_id), - TP_ARGS(neo_dcp, swap_id), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, swap_id) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->swap_id = swap_id; - ), - TP_printk("dcp=%llx, swap_id=%d", - __entry->neo_dcp, - __entry->swap_id) -); - -TRACE_EVENT(neo_iomfb_vrr_timestamps, - TP_PROTO(struct neo_apple_dcp *neo_dcp, bool enabled, u64 timestamp, - u64 submit_timestamp), - TP_ARGS(neo_dcp, enabled, timestamp, submit_timestamp), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(bool, enabled) - __field(u64, timestamp) - __field(u64, submit_timestamp) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->enabled = enabled; - __entry->timestamp = timestamp; - __entry->submit_timestamp = submit_timestamp; - ), - TP_printk("dcp=%llx, enabled=%u, timestamp=%llu, previous=%llu", - __entry->neo_dcp, __entry->enabled, __entry->timestamp, - __entry->submit_timestamp) -); - -TRACE_EVENT(neo_iomfb_swap_complete, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 swap_id), - TP_ARGS(neo_dcp, swap_id), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, swap_id) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->swap_id = swap_id; - ), - TP_printk("dcp=%llx, swap_id=%d", - __entry->neo_dcp, - __entry->swap_id - ) -); - -TRACE_EVENT(neo_iomfb_swap_complete_intent_gated, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 swap_id, u32 width, u32 height), - TP_ARGS(neo_dcp, swap_id, width, height), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, swap_id) - __field(u32, width) - __field(u32, height) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->swap_id = swap_id; - __entry->height = height; - __entry->width = width; - ), - TP_printk("dcp=%llx, swap_id=%u %ux%u", - __entry->neo_dcp, - __entry->swap_id, - __entry->width, - __entry->height - ) -); - -TRACE_EVENT(neo_iomfb_abort_swap_ap_gated, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 swap_id), - TP_ARGS(neo_dcp, swap_id), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, swap_id) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->swap_id = swap_id; - ), - TP_printk("dcp=%llx, swap_id=%u", - __entry->neo_dcp, - __entry->swap_id - ) -); - -DECLARE_EVENT_CLASS(neo_iomfb_parse_mode_template, - TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), - TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score), - - TP_STRUCT__entry(__field(s64, id) - __field_struct(struct dimension, horiz) - __field_struct(struct dimension, vert) - __field(s64, best_color_mode) - __field(bool, is_virtual) - __field(s64, score)), - - TP_fast_assign(__entry->id = id; - __entry->horiz = *horiz; - __entry->vert = *vert; - __entry->best_color_mode = best_color_mode; - __entry->is_virtual = is_virtual; - __entry->score = score;), - - TP_printk("id: %lld, best_color_mode: %lld, resolution:%lldx%lld virtual: %d, score: %lld", - __entry->id, __entry->best_color_mode, - __entry->horiz.active, __entry->vert.active, - __entry->is_virtual, __entry->score)); - -DEFINE_EVENT(neo_iomfb_parse_mode_template, neo_iomfb_parse_mode_success, - TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), - TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score)); - -DEFINE_EVENT(neo_iomfb_parse_mode_template, neo_iomfb_parse_mode_fail, - TP_PROTO(s64 id, struct dimension *horiz, struct dimension *vert, s64 best_color_mode, bool is_virtual, s64 score), - TP_ARGS(id, horiz, vert, best_color_mode, is_virtual, score)); - -TRACE_EVENT(neo_dcpavserv_init, TP_PROTO(struct neo_apple_dcp *neo_dcp, u64 unit), - TP_ARGS(neo_dcp, unit), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dcp->dev)) - __field(u64, unit)), - - TP_fast_assign(__assign_str(devname); - __entry->unit = unit;), - - TP_printk("%s: dcpav-service unit %lld initialized", __get_str(devname), - __entry->unit)); - -TRACE_EVENT(neo_dptxport_init, TP_PROTO(struct neo_apple_dcp *neo_dcp, u64 unit), - TP_ARGS(neo_dcp, unit), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dcp->dev)) - __field(u64, unit)), - - TP_fast_assign(__assign_str(devname); - __entry->unit = unit;), - - TP_printk("%s: dptxport unit %lld initialized", __get_str(devname), - __entry->unit)); - -TRACE_EVENT( - neo_dptxport_apcall, - TP_PROTO(struct neo_dptx_port *neo_dptx, int idx, size_t len), - TP_ARGS(neo_dptx, idx, len), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dptx->service->ep->neo_dcp->dev)) - __field(u32, unit) __field(int, idx) __field(size_t, len)), - - TP_fast_assign(__assign_str(devname); - __entry->unit = neo_dptx->unit; __entry->idx = idx; __entry->len = len;), - - TP_printk("%s: dptx%d: AP Call %d (%s) with len %lu", __get_str(devname), - __entry->unit, - __entry->idx, show_dptxport_apcall(__entry->idx), __entry->len)); - -TRACE_EVENT( - neo_dptxport_validate_connection, - TP_PROTO(struct neo_dptx_port *neo_dptx, u8 core, u8 atc, u8 die), - TP_ARGS(neo_dptx, core, atc, die), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dptx->service->ep->neo_dcp->dev)) - __field(u32, unit) __field(u8, core) __field(u8, atc) __field(u8, die)), - - TP_fast_assign(__assign_str(devname); - __entry->unit = neo_dptx->unit; __entry->core = core; __entry->atc = atc; __entry->die = die;), - - TP_printk("%s: dptx%d: core %d, atc %d, die %d", __get_str(devname), - __entry->unit, __entry->core, __entry->atc, __entry->die)); - -TRACE_EVENT( - neo_dptxport_connect, - TP_PROTO(struct neo_dptx_port *neo_dptx, u8 core, u8 atc, u8 die), - TP_ARGS(neo_dptx, core, atc, die), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dptx->service->ep->neo_dcp->dev)) - __field(u32, unit) __field(u8, core) __field(u8, atc) __field(u8, die)), - - TP_fast_assign(__assign_str(devname); - __entry->unit = neo_dptx->unit; __entry->core = core; __entry->atc = atc; __entry->die = die;), - - TP_printk("%s: dptx%d: core %d, atc %d, die %d", __get_str(devname), - __entry->unit, __entry->core, __entry->atc, __entry->die)); - -TRACE_EVENT( - neo_dptxport_call_set_link_rate, - TP_PROTO(struct neo_dptx_port *neo_dptx, u32 link_rate), - TP_ARGS(neo_dptx, link_rate), - - TP_STRUCT__entry(__string(devname, dev_name(neo_dptx->service->ep->neo_dcp->dev)) - __field(u32, unit) - __field(u32, link_rate)), - - TP_fast_assign(__assign_str(devname); - __entry->unit = neo_dptx->unit; - __entry->link_rate = link_rate;), - - TP_printk("%s: dptx%d: link rate 0x%x", __get_str(devname), __entry->unit, - __entry->link_rate)); - -TRACE_EVENT(neo_iomfb_brightness, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 nits), - TP_ARGS(neo_dcp, nits), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, nits) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->nits = nits; - ), - TP_printk("dcp=%llx, nits=%u (raw=0x%05x)", - __entry->neo_dcp, - __entry->nits >> 16, - __entry->nits - ) -); - -#define show_eotf(eotf) \ - __print_symbolic(eotf, { 0, "SDR gamma"}, \ - { 1, "HDR gamma"}, \ - { 2, "ST 2084 (PQ)"}, \ - { 3, "BT.2100 (HLG)"}, \ - { 4, "unexpected"}) - -#define show_encoding(enc) \ - __print_symbolic(enc, { 0, "RGB"}, \ - { 1, "YUV 4:2:0"}, \ - { 3, "YUV 4:2:2"}, \ - { 2, "YUV 4:4:4"}, \ - { 4, "DolbyVision (native)"}, \ - { 5, "DolbyVision (HDMI)"}, \ - { 6, "YCbCr 4:2:2 (DP tunnel)"}, \ - { 7, "YCbCr 4:2:2 (HDMI tunnel)"}, \ - { 8, "DolbyVision LL YCbCr 4:2:2"}, \ - { 9, "DolbyVision LL YCbCr 4:2:2 (DP)"}, \ - {10, "DolbyVision LL YCbCr 4:2:2 (HDMI)"}, \ - {11, "DolbyVision LL YCbCr 4:4:4"}, \ - {12, "DolbyVision LL RGB 4:2:2"}, \ - {13, "GRGB as YCbCr422 (Even line blue)"}, \ - {14, "GRGB as YCbCr422 (Even line red)"}, \ - {15, "unexpected"}) - -#define show_colorimetry(col) \ - __print_symbolic(col, { 0, "SMPTE 170M/BT.601"}, \ - { 1, "BT.701"}, \ - { 2, "xvYCC601"}, \ - { 3, "xvYCC709"}, \ - { 4, "sYCC601"}, \ - { 5, "AdobeYCC601"}, \ - { 6, "BT.2020 (c)"}, \ - { 7, "BT.2020 (nc)"}, \ - { 8, "DolbyVision VSVDB"}, \ - { 9, "BT.2020 (RGB)"}, \ - {10, "sRGB"}, \ - {11, "scRGB"}, \ - {12, "scRGBfixed"}, \ - {13, "AdobeRGB"}, \ - {14, "DCI-P3 (D65)"}, \ - {15, "DCI-P3 (Theater)"}, \ - {16, "Default RGB"}, \ - {17, "unexpected"}) - -#define show_range(range) \ - __print_symbolic(range, { 0, "Full"}, \ - { 1, "Limited"}, \ - { 2, "unexpected"}) - -TRACE_EVENT(neo_iomfb_color_mode, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 id, u32 score, u32 depth, - u32 colorimetry, u32 eotf, u32 range, u32 pixel_enc), - TP_ARGS(neo_dcp, id, score, depth, colorimetry, eotf, range, pixel_enc), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, id) - __field(u32, score) - __field(u32, depth) - __field(u32, colorimetry) - __field(u32, eotf) - __field(u32, range) - __field(u32, pixel_enc) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->id = id; - __entry->score = score; - __entry->depth = depth; - __entry->colorimetry = min_t(u32, colorimetry, 17U); - __entry->eotf = min_t(u32, eotf, 4U); - __entry->range = min_t(u32, range, 2U); - __entry->pixel_enc = min_t(u32, pixel_enc, 15U); - ), - TP_printk("dcp=%llx, id=%u, score=%u, depth=%u, colorimetry=%s, eotf=%s, range=%s, pixel_enc=%s", - __entry->neo_dcp, - __entry->id, - __entry->score, - __entry->depth, - show_colorimetry(__entry->colorimetry), - show_eotf(__entry->eotf), - show_range(__entry->range), - show_encoding(__entry->pixel_enc) - ) -); - -TRACE_EVENT(neo_iomfb_timing_mode, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 id, u32 score, u32 width, - u32 height, u32 clock, u32 color_mode), - TP_ARGS(neo_dcp, id, score, width, height, clock, color_mode), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, id) - __field(u32, score) - __field(u32, width) - __field(u32, height) - __field(u32, clock) - __field(u32, color_mode) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->id = id; - __entry->score = score; - __entry->width = width; - __entry->height = height; - __entry->clock = clock; - __entry->color_mode = color_mode; - ), - TP_printk("dcp=%llx, id=%u, score=%u, %ux%u@%u.%u, color_mode=%u", - __entry->neo_dcp, - __entry->id, - __entry->score, - __entry->width, - __entry->height, - __entry->clock >> 16, - ((__entry->clock & 0xffff) * 1000) >> 16, - __entry->color_mode - ) -); - -TRACE_EVENT(neo_avep_sound_mode, - TP_PROTO(struct neo_apple_dcp *neo_dcp, u32 rates, u64 formats, unsigned int nchans), - TP_ARGS(neo_dcp, rates, formats, nchans), - TP_STRUCT__entry( - __field(u64, neo_dcp) - __field(u32, rates) - __field(u64, formats) - __field(unsigned int, nchans) - ), - TP_fast_assign( - __entry->neo_dcp = (u64)neo_dcp; - __entry->rates = rates; - __entry->formats = formats; - __entry->nchans = nchans; - ), - TP_printk("dcp=%llx, rates=%#x, formats=%#llx, nchans=%#x", - __entry->neo_dcp, - __entry->rates, - __entry->formats, - __entry->nchans - ) -); - -#endif /* _TRACE_DCP_H */ - -/* This part must be outside protection */ - -#undef TRACE_INCLUDE_FILE -#define TRACE_INCLUDE_FILE trace - -#undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH . - -#include diff --git a/drivers/gpu/drm/apple-neo/version_utils.h b/drivers/gpu/drm/apple-neo/version_utils.h deleted file mode 100644 index 5a33ce1db61c47..00000000000000 --- a/drivers/gpu/drm/apple-neo/version_utils.h +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Copyright The Asahi Linux Contributors */ - -#ifndef __APPLE_VERSION_UTILS_H__ -#define __APPLE_VERSION_UTILS_H__ - -#include -#include - -#define DCP_FW_UNION(u) (u).DCP_FW -#define DCP_FW_SUFFIX CONCATENATE(_, DCP_FW) -#define DCP_FW_NAME(name) CONCATENATE(name, DCP_FW_SUFFIX) -#define DCP_FW_VERSION(x, y, z) ( ((x) << 16) | ((y) << 8) | (z) ) - -#endif /*__APPLE_VERSION_UTILS_H__*/ diff --git a/drivers/gpu/drm/asahi-neo/Kconfig b/drivers/gpu/drm/asahi-neo/Kconfig deleted file mode 100644 index 14005176f72bf7..00000000000000 --- a/drivers/gpu/drm/asahi-neo/Kconfig +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -config DRM_ASAHI_NEO - tristate "Asahi Neo (Apple T8140 G17 GPU)" - depends on RUST && DRM=y - depends on ARM64 && ARCH_APPLE && MMU && IOMMU_SUPPORT - depends on PAGE_SIZE_16KB - select RUST_DRM_SCHED - select RUST_DRM_GEM_SHMEM_HELPER - select RUST_DRM_GPUVM - select RUST_APPLE_RTKIT - select WANT_DEV_COREDUMP - help - Native graphics support for the T8140 G17P GPU in the MacBook Neo. - Uses an independent command interface from the M1/M2/M3 driver. diff --git a/drivers/gpu/drm/asahi-neo/Makefile b/drivers/gpu/drm/asahi-neo/Makefile deleted file mode 100644 index 055d0b797b254d..00000000000000 --- a/drivers/gpu/drm/asahi-neo/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_DRM_ASAHI_NEO) += asahi_neo.o -asahi_neo-y := asahi_core.o sysfs.o diff --git a/drivers/gpu/drm/asahi-neo/alloc.rs b/drivers/gpu/drm/asahi-neo/alloc.rs deleted file mode 100644 index 5e55ce5ad975a6..00000000000000 --- a/drivers/gpu/drm/asahi-neo/alloc.rs +++ /dev/null @@ -1,1087 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU kernel object allocator. -//! -//! This kernel driver needs to manage a large number of GPU objects, in both firmware/kernel -//! address space and user address space. This module implements a simple grow-only heap allocator -//! based on the DRM MM range allocator, and a debug allocator that allocates each object as a -//! separate GEM object. -//! -//! Allocations may optionally have debugging enabled, which adds preambles that store metadata -//! about the allocation. This is useful for live debugging using the hypervisor or postmortem -//! debugging with a GPU memory snapshot, since it makes it easier to identify use-after-free and -//! caching issues. - -use kernel::{ - drm_neo::mm, - error::Result, - prelude::*, - str::CString, // -}; - -use crate::debug::*; -use crate::driver::{ - AsahiDevRef, - AsahiDevice, // -}; -use crate::fw::types::Zeroable; -use crate::mmu; -use crate::object::{ - GpuArray, - GpuObject, - GpuOnlyArray, - GpuStruct, - GpuWeakPointer, // -}; -use crate::util::RangeExt; - -use core::cmp::Ordering; -use core::fmt; -use core::fmt::{ - Debug, - Formatter, // -}; -use core::marker::PhantomData; -use core::mem; -use core::ops::Range; -use core::ptr::NonNull; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Alloc; - -#[cfg(not(CONFIG_DRM_ASAHI_NEO_DEBUG_ALLOCATOR))] -/// The driver-global allocator type -pub(crate) type DefaultAllocator = HeapAllocator; - -#[cfg(not(CONFIG_DRM_ASAHI_NEO_DEBUG_ALLOCATOR))] -/// The driver-global allocation type -pub(crate) type DefaultAllocation = HeapAllocation; - -#[cfg(CONFIG_DRM_ASAHI_NEO_DEBUG_ALLOCATOR)] -/// The driver-global allocator type -pub(crate) type DefaultAllocator = SimpleAllocator; - -#[cfg(CONFIG_DRM_ASAHI_NEO_DEBUG_ALLOCATOR)] -/// The driver-global allocation type -pub(crate) type DefaultAllocation = SimpleAllocation; - -/// Represents a raw allocation (without any type information). -pub(crate) trait RawAllocation { - /// Returns the CPU-side pointer (if CPU mapping is enabled) as a byte non-null pointer. - fn ptr(&self) -> Option>; - /// Returns the GPU VA pointer as a u64. - fn gpu_ptr(&self) -> u64; - /// Returns the AsahiDevice that owns this allocation. - fn device(&self) -> &AsahiDevice; -} - -/// Represents a typed allocation. -pub(crate) trait Allocation: Debug { - /// Returns the typed CPU-side pointer (if CPU mapping is enabled). - fn ptr(&self) -> Option>; - /// Returns the GPU VA pointer as a u64. - fn gpu_ptr(&self) -> u64; - /// Returns the size of the allocation in bytes. - fn size(&self) -> usize; - /// Returns the AsahiDevice that owns this allocation. - fn device(&self) -> &AsahiDevice; -} - -/// A generic typed allocation wrapping a RawAllocation. -/// -/// This is currently the only Allocation implementation, since it is shared by all allocators. -/// -/// # Invariants -/// The alloaction at `alloc` must have a size equal or greater than `alloc_size` plus `debug_offset` plus `padding`. -pub(crate) struct GenericAlloc { - alloc: U, - alloc_size: usize, - debug_offset: usize, - padding: usize, - tag: u32, - pad_word: u32, - _p: PhantomData, -} - -impl Allocation for GenericAlloc { - /// Returns a pointer to the inner (usable) part of the allocation. - fn ptr(&self) -> Option> { - // SAFETY: self.debug_offset is always within the allocation per the invariant, so is safe to add - // to the base pointer. - unsafe { self.alloc.ptr().map(|p| p.add(self.debug_offset).cast()) } - } - /// Returns the GPU pointer to the inner (usable) part of the allocation. - fn gpu_ptr(&self) -> u64 { - self.alloc.gpu_ptr() + self.debug_offset as u64 - } - /// Returns the size of the inner (usable) part of the allocation. - fn size(&self) -> usize { - self.alloc_size - } - fn device(&self) -> &AsahiDevice { - self.alloc.device() - } -} - -impl Debug for GenericAlloc { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct(core::any::type_name::>()) - .field("ptr", &format_args!("{:?}", self.ptr())) - .field("gpu_ptr", &format_args!("{:#X?}", self.gpu_ptr())) - .field("size", &format_args!("{:#X?}", self.size())) - .finish() - } -} - -/// Debugging data associated with an allocation, when debugging is enabled. -#[repr(C)] -struct AllocDebugData { - state: u32, - tag: u32, - size: u64, - base_gpuva: u64, - obj_gpuva: u64, - name: [u8; 0x20], -} - -/// Magic flag indicating a live allocation. -const STATE_LIVE: u32 = u32::from_le_bytes(*b"LIVE"); -/// Magic flag indicating a freed allocation. -const STATE_DEAD: u32 = u32::from_le_bytes(*b"DEAD"); - -/// Marker byte to identify when firmware/GPU write beyond the end of an allocation. -const GUARD_MARKER: u32 = 0x93939393; - -impl Drop for GenericAlloc { - fn drop(&mut self) { - let debug_len = mem::size_of::(); - if self.debug_offset >= debug_len { - if let Some(p) = self.alloc.ptr() { - // SAFETY: self.debug_offset is always greater than the alloc size per - // the invariant, and greater than debug_len as checked above. - unsafe { - let p = p.as_ptr().add(self.debug_offset - debug_len); - (p as *mut u32).write(STATE_DEAD); - } - } - } - if debug_enabled(DebugFlags::FillAllocations) { - if let Some(p) = self.ptr() { - // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xde, self.size()) }; - } - } - if self.padding != 0 { - if let Some(p) = self.ptr() { - // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing - // the inner base pointer, after `size()` bytes. - let guard = unsafe { - core::slice::from_raw_parts( - (p.as_ptr() as *mut u8 as *const u8).add(self.size()), - self.padding, - ) - }; - let mut first_err = None; - let mut last_err = 0; - for (i, p) in guard.iter().enumerate() { - if *p != (self.pad_word >> (8 * (i & 3))) as u8 { - if first_err.is_none() { - first_err = Some(i); - } - last_err = i; - } - } - if let Some(start) = first_err { - dev_warn!( - self.device().as_ref(), - "Allocator: Corruption after object of type {}/{:#x} at {:#x}:{:#x} + {:#x}..={:#x}\n", - core::any::type_name::(), - self.tag, - self.gpu_ptr(), - self.size(), - start, - last_err, - ); - } - } - } - } -} - -static_assert!(mem::size_of::() == 0x40); - -/// A trait representing an allocator. -pub(crate) trait Allocator { - /// The raw allocation type used by this allocator. - type Raw: RawAllocation; - // TODO: Needs associated_type_defaults - // type Allocation = GenericAlloc; - - /// Returns whether CPU-side mapping is enabled. - fn cpu_maps(&self) -> bool; - /// Returns the minimum alignment for allocations. - fn min_align(&self) -> usize; - /// Allocate an object of the given size in bytes with the given alignment. - fn alloc(&mut self, size: usize, align: usize) -> Result; - - /// Returns a tuple of (count, size) of how much garbage (freed but not yet reusable objects) - /// exists in this allocator. Optional. - fn garbage(&self) -> (usize, usize) { - (0, 0) - } - /// Collect garbage for this allocator, up to the given object count. Optional. - fn collect_garbage(&mut self, _count: usize) {} - - /// Allocate a new GpuStruct object. See [`GpuObject::new`]. - #[inline(never)] - fn new_object( - &mut self, - inner: T, - callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, - ) -> Result>> { - GpuObject::>::new(self.alloc_object()?, inner, callback) - } - - /// Allocate a new GpuStruct object. See [`GpuObject::new_default`]. - #[inline(never)] - fn new_default( - &mut self, - ) -> Result>> - where - for<'a> ::Raw<'a>: Default + Zeroable, - { - GpuObject::>::new_default(self.alloc_object()?) - } - - /// Allocate a new GpuStruct object. See [`GpuObject::new_init`]. - #[inline(never)] - fn new_init<'a, T: GpuStruct, R: PinInit, F>, E, F>( - &mut self, - inner_init: impl Init, - raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, - ) -> Result>> - where - kernel::error::Error: core::convert::From, - kernel::error::Error: core::convert::From, - { - GpuObject::>::new_init_prealloc( - self.alloc_object()?, - |_p| inner_init, - raw_init, - ) - } - - /// Allocate a generic buffer of the given size and alignment, applying the debug features if - /// enabled to tag it and detect overflows. - fn alloc_generic( - &mut self, - size: usize, - align: usize, - tag: Option, - ) -> Result> { - let padding = if debug_enabled(DebugFlags::DetectOverflows) { - size - } else { - 0 - }; - - let ret: GenericAlloc = - if self.cpu_maps() && debug_enabled(debug::DebugFlags::DebugAllocations) { - let debug_align = self.min_align().max(align); - let debug_len = mem::size_of::(); - let debug_offset = (debug_len * 2 + debug_align - 1) & !(debug_align - 1); - - let alloc = self.alloc(size + debug_offset + padding, align)?; - - let mut debug = AllocDebugData { - state: STATE_LIVE, - tag: tag.unwrap_or(0), - size: size as u64, - base_gpuva: alloc.gpu_ptr(), - obj_gpuva: alloc.gpu_ptr() + debug_offset as u64, - name: [0; 0x20], - }; - - let name = core::any::type_name::().as_bytes(); - let len = name.len().min(debug.name.len() - 1); - debug.name[..len].copy_from_slice(&name[..len]); - - if let Some(p) = alloc.ptr() { - // SAFETY: Per the size calculations above, this pointer math and the - // writes never exceed the allocation size. - unsafe { - let p = p.as_ptr(); - p.write_bytes(0x42, debug_offset - 2 * debug_len); - let cur = p.add(debug_offset - debug_len) as *mut AllocDebugData; - let prev = p.add(debug_offset - 2 * debug_len) as *mut AllocDebugData; - prev.copy_from(cur, 1); - cur.copy_from(&debug, 1); - }; - } - - GenericAlloc { - alloc, - alloc_size: size, - debug_offset, - tag: tag.unwrap_or(0), - pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, - padding, - _p: PhantomData, - } - } else { - GenericAlloc { - alloc: self.alloc(size + padding, align)?, - alloc_size: size, - debug_offset: 0, - tag: tag.unwrap_or(0), - pad_word: tag.unwrap_or(GUARD_MARKER) | 0x81818181, - padding, - _p: PhantomData, - } - }; - - if debug_enabled(DebugFlags::FillAllocations) { - if let Some(p) = ret.ptr() { - // SAFETY: Writing to our inner base pointer with our known inner size is safe. - unsafe { (p.as_ptr() as *mut u8).write_bytes(0xaa, ret.size()) }; - } - } - - if padding != 0 { - if let Some(p) = ret.ptr() { - // SAFETY: Per the invariant, we have at least `self.padding` bytes trailing - // the inner base pointer, after `size()` bytes. - let guard = unsafe { - core::slice::from_raw_parts_mut( - (p.as_ptr() as *mut u8).add(ret.size()), - padding, - ) - }; - for (i, p) in guard.iter_mut().enumerate() { - *p = (ret.pad_word >> (8 * (i & 3))) as u8; - } - } - } - - Ok(ret) - } - - /// Allocate an object of a given type, without actually initializing the allocation. - /// - /// This is useful to directly call [`GpuObject::new_*`], without borrowing a reference to the - /// allocator for the entire duration (e.g. if further allocations need to happen inside the - /// callbacks). - fn alloc_object(&mut self) -> Result> { - let size = mem::size_of::>(); - let align = mem::align_of::>(); - - self.alloc_generic(size, align, None) - } - - /// Allocate an empty `GpuArray` of a given type and length. - fn array_empty( - &mut self, - count: usize, - ) -> Result>> { - let size = mem::size_of::() * count; - let align = mem::align_of::(); - - let alloc = self.alloc_generic(size, align, None)?; - GpuArray::>::empty(alloc, count) - } - - /// Allocate an empty `GpuArray` of a given type and length. - fn array_empty_tagged( - &mut self, - count: usize, - tag: &[u8; 4], - ) -> Result>> { - let size = mem::size_of::() * count; - let align = mem::align_of::(); - - let alloc = self.alloc_generic(size, align, Some(u32::from_le_bytes(*tag)))?; - GpuArray::>::empty(alloc, count) - } - - /// Allocate an empty `GpuOnlyArray` of a given type and length. - fn array_gpuonly( - &mut self, - count: usize, - ) -> Result>> { - let size = mem::size_of::() * count; - let align = mem::align_of::(); - - let alloc = self.alloc_generic(size, align, None)?; - GpuOnlyArray::>::new(alloc, count) - } -} - -/// A simple allocation backed by a separate GEM object. -/// -/// # Invariants -/// `ptr` is either None or a valid, non-null pointer to the CPU view of the object. -/// `gpu_ptr` is the GPU-side VA of the object. -pub(crate) struct SimpleAllocation { - dev: AsahiDevRef, - ptr: Option>, - gpu_ptr: u64, - _mapping: mmu::KernelMapping, - obj: crate::gem::ObjectRef, -} - -/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to send across threads. -unsafe impl Send for SimpleAllocation {} -/// SAFETY: `SimpleAllocation` just points to raw memory and should be safe to share across threads. -unsafe impl Sync for SimpleAllocation {} - -impl Drop for SimpleAllocation { - fn drop(&mut self) { - mod_dev_dbg!( - self.device(), - "SimpleAllocator: drop object @ {:#x}\n", - self.gpu_ptr() - ); - if debug_enabled(DebugFlags::FillAllocations) { - if let Ok(mut vmap) = self.obj.vmap() { - vmap.memset(0x42); - } - } - } -} - -impl RawAllocation for SimpleAllocation { - fn ptr(&self) -> Option> { - self.ptr - } - fn gpu_ptr(&self) -> u64 { - self.gpu_ptr - } - fn device(&self) -> &AsahiDevice { - &self.dev - } -} - -/// A simple allocator that allocates each object as its own GEM object, aligned to the end of a -/// page. -/// -/// This is very slow, but it has the advantage that over-reads by the firmware or GPU will fault on -/// the guard page after the allocation, which can be useful to validate that the firmware's or -/// GPU's idea of object size what we expect. -pub(crate) struct SimpleAllocator { - dev: AsahiDevRef, - range: Range, - prot: mmu::Prot, - vm: mmu::Vm, - min_align: usize, - cpu_maps: bool, -} - -impl SimpleAllocator { - /// Create a new `SimpleAllocator` for a given address range and `Vm`. - #[allow(dead_code)] - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - dev: &AsahiDevice, - vm: &mmu::Vm, - range: Range, - min_align: usize, - prot: mmu::Prot, - _block_size: usize, - mut cpu_maps: bool, - _name: fmt::Arguments<'_>, - _keep_garbage: bool, - ) -> Result { - if debug_enabled(DebugFlags::ForceCPUMaps) { - cpu_maps = true; - } - Ok(SimpleAllocator { - dev: dev.into(), - vm: vm.clone(), - range, - prot, - min_align, - cpu_maps, - }) - } -} - -impl Allocator for SimpleAllocator { - type Raw = SimpleAllocation; - - fn cpu_maps(&self) -> bool { - self.cpu_maps - } - - fn min_align(&self) -> usize { - self.min_align - } - - #[inline(never)] - fn alloc(&mut self, size: usize, align: usize) -> Result { - let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; - let align = self.min_align.max(align); - let offset = (size_aligned - size) & !(align - 1); - - mod_dev_dbg!( - &self.dev, - "SimpleAllocator::new: size={:#x} size_al={:#x} al={:#x} off={:#x}\n", - size, - size_aligned, - align, - offset - ); - - let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; - let p = obj.vmap()?.as_mut_ptr() as *mut u8; - if debug_enabled(DebugFlags::FillAllocations) { - obj.vmap()?.memset(0xde); - } - let mapping = obj.map_into_range( - &self.vm, - self.range.clone(), - self.min_align.max(mmu::UAT_PGSZ) as u64, - self.prot, - true, - )?; - - let iova = mapping.iova(); - - // SAFETY: Per the math above to calculate `size_aligned`, this can never overflow. - let ptr = unsafe { p.add(offset) }; - let gpu_ptr = iova + offset as u64; - - mod_dev_dbg!( - &self.dev, - "SimpleAllocator::new -> {:#?} / {:#?} | {:#x} / {:#x}\n", - p, - ptr, - iova, - gpu_ptr - ); - - Ok(SimpleAllocation { - dev: self.dev.clone(), - ptr: NonNull::new(ptr), - gpu_ptr, - _mapping: mapping, - obj, - }) - } -} - -/// Inner data for an allocation from the heap allocator. -/// -/// This is wrapped in an `mm::Node`. -pub(crate) struct HeapAllocationInner { - dev: AsahiDevRef, - ptr: Option>, - real_size: usize, -} - -/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to send across threads. -unsafe impl Send for HeapAllocationInner {} -/// SAFETY: `HeapAllocationInner` just points to raw memory and should be safe to share between threads. -unsafe impl Sync for HeapAllocationInner {} - -/// Outer view of a heap allocation. -/// -/// This uses an Option<> so we can move the internal `Node` into the garbage pool when it gets -/// dropped. -/// -/// # Invariants -/// The `Option` must always be `Some(...)` while this object is alive. -pub(crate) struct HeapAllocation(Option>); - -impl Drop for HeapAllocation { - fn drop(&mut self) { - let node = self.0.take().unwrap(); - let size = node.size(); - let alloc = node.alloc_ref(); - - alloc.with(|a| { - if let Some(garbage) = a.garbage.as_mut() { - if garbage.push(node, GFP_KERNEL).is_err() { - dev_err!( - &a.dev.as_ref(), - "HeapAllocation[{}]::drop: Failed to keep garbage\n", - &*a.name, - ); - } - a.total_garbage += size as usize; - None - } else { - // We need to ensure node survives this scope, since dropping it - // will try to take the mm lock and deadlock us - Some(node) - } - }); - } -} - -impl mm::AllocInner for HeapAllocatorInner { - fn drop_object( - &mut self, - start: u64, - _size: u64, - _color: usize, - obj: &mut HeapAllocationInner, - ) { - /* real_size == 0 means it's a guard node */ - if obj.real_size > 0 { - mod_dev_dbg!( - obj.dev, - "HeapAllocator[{}]: drop object @ {:#x} ({} bytes)\n", - &*self.name, - start, - obj.real_size, - ); - self.allocated -= obj.real_size; - } - } -} - -impl RawAllocation for HeapAllocation { - // SAFETY: This function must always return a valid pointer. - // Since the HeapAllocation contains a reference to the - // backing_objects array that contains the object backing this pointer, - // and objects are only ever added to it, this pointer is guaranteed to - // remain valid for the lifetime of the HeapAllocation. - fn ptr(&self) -> Option> { - self.0.as_ref().unwrap().ptr - } - // SAFETY: This function must always return a valid GPU pointer. - // See the explanation in ptr(). - fn gpu_ptr(&self) -> u64 { - self.0.as_ref().unwrap().start() - } - fn device(&self) -> &AsahiDevice { - &self.0.as_ref().unwrap().dev - } -} - -/// Inner data for a heap allocator which uses the DRM MM range allocator to manage the heap. -/// -/// This is wrapped by an `mm::Allocator`. -struct HeapAllocatorInner { - dev: AsahiDevRef, - allocated: usize, - backing_objects: KVec<(crate::gem::ObjectRef, mmu::KernelMapping, u64)>, - garbage: Option>>, - total_garbage: usize, - name: CString, -} - -/// A heap allocator which uses the DRM MM range allocator to manage its objects. -/// -/// The heap is composed of a series of GEM objects. This implementation only ever grows the heap, -/// never shrinks it. -pub(crate) struct HeapAllocator { - dev: AsahiDevRef, - range: Range, - top: u64, - prot: mmu::Prot, - vm: mmu::Vm, - min_align: usize, - block_size: usize, - cpu_maps: bool, - guard_nodes: KVec>, - mm: mm::Allocator, - name: CString, - garbage: Option>>, -} - -impl HeapAllocator { - /// Create a new HeapAllocator for a given `Vm` and address range. - #[allow(dead_code)] - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - dev: &AsahiDevice, - vm: &mmu::Vm, - range: Range, - min_align: usize, - prot: mmu::Prot, - block_size: usize, - mut cpu_maps: bool, - name: fmt::Arguments<'_>, - keep_garbage: bool, - ) -> Result { - if !min_align.is_power_of_two() { - return Err(EINVAL); - } - if debug_enabled(DebugFlags::ForceCPUMaps) { - cpu_maps = true; - } - - let name = CString::try_from_fmt(name)?; - - let inner = HeapAllocatorInner { - dev: dev.into(), - allocated: 0, - backing_objects: KVec::new(), - // TODO: This clearly needs a try_clone() or similar - name: CString::try_from_fmt(fmt!("{}", &*name))?, - garbage: if keep_garbage { - Some(KVec::new()) - } else { - None - }, - total_garbage: 0, - }; - - let mm = mm::Allocator::new(range.start, range.range(), inner)?; - - Ok(HeapAllocator { - dev: dev.into(), - vm: vm.clone(), - top: range.start, - range, - prot, - min_align, - block_size: block_size.max(min_align), - cpu_maps, - guard_nodes: KVec::new(), - mm, - name, - garbage: if keep_garbage { - Some({ - let mut v = KVec::new(); - v.reserve(128, GFP_KERNEL)?; - v - }) - } else { - None - }, - }) - } - - /// Add a new backing block of the given size to this heap. - /// - /// If CPU mapping is enabled, this also adds a guard node to the range allocator to ensure that - /// objects cannot straddle backing block boundaries, since we cannot easily create a contiguous - /// CPU VA mapping for them. This can create some fragmentation. If CPU mapping is disabled, we - /// skip the guard blocks, since the GPU view of the heap is always contiguous. - #[inline(never)] - fn add_block(&mut self, size: usize) -> Result { - let size_aligned = (size + mmu::UAT_PGSZ - 1) & !mmu::UAT_PGMSK; - - mod_dev_dbg!( - &self.dev, - "HeapAllocator[{}]::add_block: size={:#x} size_al={:#x}\n", - &*self.name, - size, - size_aligned, - ); - - if self.top.saturating_add(size_aligned as u64) > self.range.end { - dev_err!( - self.dev.as_ref(), - "HeapAllocator[{}]::add_block: Exhausted VA space\n", - &*self.name, - ); - } - - let mut obj = crate::gem::new_kernel_object(&self.dev, size_aligned)?; - if self.cpu_maps && debug_enabled(DebugFlags::FillAllocations) { - obj.vmap()?.memset(0xde); - } - - let gpu_ptr = self.top; - let mapping = obj - .map_at(&self.vm, gpu_ptr, self.prot, self.cpu_maps) - .inspect_err(|err| { - dev_err!( - self.dev.as_ref(), - "HeapAllocator[{}]::add_block: Failed to map at {:#x} ({:?})\n", - &*self.name, - gpu_ptr, - err - ); - })?; - - if self.cpu_maps { - // Create virtual mapping here ahead of time so that the vmap() in - // alloc_inner() does not take the the object's dma_resv lock while - // the mm lock is locked. mmu::Vm requires the opposite lock order. - obj.vmap()?; - } - - self.mm - .with_inner(|inner| inner.backing_objects.reserve(1, GFP_KERNEL))?; - - let mut new_top = self.top + size_aligned as u64; - if self.cpu_maps { - let guard = self.min_align.max(mmu::UAT_PGSZ); - mod_dev_dbg!( - self.dev, - "HeapAllocator[{}]::add_block: Adding guard node {:#x}:{:#x}\n", - &*self.name, - new_top, - guard - ); - - let inner = HeapAllocationInner { - dev: self.dev.clone(), - ptr: None, - real_size: 0, - }; - - let node = match self.mm.reserve_node(inner, new_top, guard as u64, 0) { - Ok(a) => a, - Err(a) => { - dev_err!( - self.dev.as_ref(), - "HeapAllocator[{}]::add_block: Failed to reserve guard node {:#x}:{:#x}: {:?}\n", - &*self.name, - guard, - new_top, - a - ); - return Err(EIO); - } - }; - - self.guard_nodes.push(node, GFP_KERNEL)?; - - new_top += guard as u64; - } - mod_dev_dbg!( - &self.dev, - "HeapAllocator[{}]::add_block: top={:#x}\n", - &*self.name, - new_top - ); - - self.mm.with_inner(|inner| { - inner - .backing_objects - .push((obj, mapping, gpu_ptr), GFP_KERNEL) - })?; - - self.top = new_top; - - cls_dev_dbg!( - MemStats, - &self.dev, - "{} Heap: grow to {} bytes\n", - &*self.name, - self.top - self.range.start - ); - - Ok(()) - } - - /// Find the backing object index that backs a given GPU address. - fn find_obj(&mut self, addr: u64) -> Result { - self.mm.with_inner(|inner| { - inner - .backing_objects - .binary_search_by(|obj| { - let start = obj.2; - let end = obj.2 + obj.0.size() as u64; - if start > addr { - Ordering::Greater - } else if end <= addr { - Ordering::Less - } else { - Ordering::Equal - } - }) - .or(Err(ENOENT)) - }) - } - - fn alloc_inner(&mut self, size: usize, align: usize) -> Result { - if align != 0 && !align.is_power_of_two() { - return Err(EINVAL); - } - let align = self.min_align.max(align); - let size_aligned = (size + align - 1) & !(align - 1); - - mod_dev_dbg!( - &self.dev, - "HeapAllocator[{}]::new: size={:#x} size_al={:#x}\n", - &*self.name, - size, - size_aligned, - ); - - let inner = HeapAllocationInner { - dev: self.dev.clone(), - ptr: None, - real_size: size, - }; - - let mut node = match self.mm.insert_node_generic( - inner, - size_aligned as u64, - align as u64, - 0, - mm::InsertMode::Best, - ) { - Ok(a) => a, - Err(a) => { - dev_err!( - &self.dev.as_ref(), - "HeapAllocator[{}]::new: Failed to insert node of size {:#x} / align {:#x}: {:?}\n", - &*self.name, size_aligned, align, a - ); - return Err(a); - } - }; - - self.mm.with_inner(|inner| inner.allocated += size); - - let mut new_object = false; - let start = node.start(); - let end = start + node.size(); - if end > self.top { - if start > self.top { - dev_warn!( - self.dev.as_ref(), - "HeapAllocator[{}]::alloc: top={:#x}, start={:#x}\n", - &*self.name, - self.top, - start - ); - } - let block_size = self.block_size.max((end - self.top) as usize); - self.add_block(block_size)?; - new_object = true; - } - assert!(end <= self.top); - - if self.cpu_maps { - mod_dev_dbg!( - self.dev, - "HeapAllocator[{}]::alloc: mapping to CPU\n", - &*self.name - ); - - let idx = if new_object { - None - } else { - Some(match self.find_obj(start) { - Ok(a) => a, - Err(_) => { - dev_warn!( - self.dev.as_ref(), - "HeapAllocator[{}]::alloc: Failed to find object at {:#x}\n", - &*self.name, - start - ); - return Err(EIO); - } - }) - }; - let (obj_start, obj_size, p) = self.mm.with_inner(|inner| -> Result<_> { - let idx = idx.unwrap_or(inner.backing_objects.len() - 1); - let obj = &mut inner.backing_objects[idx]; - let p = obj.0.vmap()?.as_mut_ptr() as *mut u8; - Ok((obj.2, obj.0.size(), p)) - })?; - assert!(obj_start <= start); - assert!(obj_start + obj_size as u64 >= end); - node.as_mut().inner_mut().ptr = - // SAFETY: Per the asserts above, this offset is always within the allocation. - NonNull::new(unsafe { p.add((start - obj_start) as usize) }); - mod_dev_dbg!( - self.dev, - "HeapAllocator[{}]::alloc: CPU pointer = {:?}\n", - &*self.name, - node.ptr - ); - } - - mod_dev_dbg!( - self.dev, - "HeapAllocator[{}]::alloc: Allocated {:#x} bytes @ {:#x}\n", - &*self.name, - end - start, - start - ); - - Ok(HeapAllocation(Some(node))) - } -} - -impl Allocator for HeapAllocator { - type Raw = HeapAllocation; - - fn cpu_maps(&self) -> bool { - self.cpu_maps - } - - fn min_align(&self) -> usize { - self.min_align - } - - fn alloc(&mut self, size: usize, align: usize) -> Result { - let ret = self.alloc_inner(size, align); - - if ret.is_err() { - dev_warn!( - self.dev.as_ref(), - "HeapAllocator[{}]::alloc: Allocation of {:#x}({:#x}) size object failed\n", - &*self.name, - size, - align - ); - } - ret - } - - fn garbage(&self) -> (usize, usize) { - self.mm.with_inner(|inner| { - if let Some(g) = inner.garbage.as_ref() { - (g.len(), inner.total_garbage) - } else { - (0, 0) - } - }) - } - - fn collect_garbage(&mut self, mut count: usize) { - if let Some(garbage) = self.garbage.as_mut() { - garbage.clear(); - - while count > 0 { - let block = count.min(garbage.capacity()); - assert!(block > 0); - - // Take the garbage out of the inner block, so we can safely drop it without deadlocking - self.mm.with_inner(|inner| { - if let Some(g) = inner.garbage.as_mut() { - for node in g.drain(0..block) { - inner.total_garbage -= node.size() as usize; - garbage - .push(node, GFP_KERNEL) - .expect("push() failed after reserve()"); - } - } - }); - - count -= block; - // Now drop it - garbage.clear(); - } - } - } -} - -impl Drop for HeapAllocatorInner { - fn drop(&mut self) { - mod_dev_dbg!( - self.dev, - "HeapAllocator[{}]: dropping allocator\n", - &*self.name - ); - if self.allocated > 0 { - // This should never happen - dev_crit!( - self.dev.as_ref(), - "HeapAllocator[{}]: dropping with {} bytes allocated\n", - &*self.name, - self.allocated - ); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/asahi_core.rs b/drivers/gpu/drm/asahi-neo/asahi_core.rs deleted file mode 100644 index 483d1ea2b17d09..00000000000000 --- a/drivers/gpu/drm/asahi-neo/asahi_core.rs +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#![recursion_limit = "2048"] - -//! Driver for the Apple AGX GPUs found in Apple Silicon SoCs. - -mod alloc; -mod buffer; -mod channel; -mod cleanup; -#[cfg(CONFIG_DEV_COREDUMP)] -mod crashdump; -mod debug; -mod driver; -mod event; -mod file; -mod float; -mod fw; -mod gem; -mod g17; -mod gpu; -mod hw; -mod initdata; -mod mem; -mod microseq; -mod mmu; -mod object; -mod pgtable; -mod queue; -mod regs; -mod slotalloc; -mod stats; -mod sysfs_exports; -mod util; -#[cfg(CONFIG_DRM_ASAHI_NEO_MAPLE_TREE)] -mod vm; -mod workqueue; - -use kernel::prelude::*; - -#[pin_data] -struct DriverModule { - // Field drop order unregisters/joins driver producers before queue teardown. - #[pin] - _driver: kernel::driver::Registration>, - _cleanup: cleanup::QueueOwner, -} - -impl kernel::InPlaceModule for DriverModule { - fn init(module: &'static kernel::ThisModule) -> impl PinInit { - try_pin_init!(Self { - // Initialize the queue first, before registration can invoke probe. - _cleanup: cleanup::QueueOwner::new()?, - _driver <- kernel::driver::Registration::new( - ::NAME, - module, - ), - }) - } -} - -module! { - type: DriverModule, - name: "asahi_neo", - description: "G17 GPU driver for Apple T8140", - license: "Dual MIT/GPL", - params: { - debug_flags: u64 { - default: 0, - // permissions: 0o644, - description: "Debug flags", - }, - fault_control: u32 { - default: 0xb, - // permissions: 0, - description: "Fault control (0x0: hard faults, 0xb: macOS default)", - }, - initial_tvb_size: usize { - default: 0x8, - // permissions: 0o644, - description: "Initial TVB size in blocks", - }, - robust_isolation: u32 { - default: 0, - // permissions: 0o644, - description: "Fully isolate GPU contexts (limits performance)", - }, - stats_export: u8 { - default: 1, - // permissions: 0, - description: "Export AGX firmware stats to sysfs (1 = on, 0 = off)", - }, - }, -} diff --git a/drivers/gpu/drm/asahi-neo/buffer.rs b/drivers/gpu/drm/asahi-neo/buffer.rs deleted file mode 100644 index ba3823098af645..00000000000000 --- a/drivers/gpu/drm/asahi-neo/buffer.rs +++ /dev/null @@ -1,809 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Tiled Vertex Buffer management -//! -//! This module manages the Tiled Vertex Buffer, also known as the Parameter Buffer (in imgtec -//! parlance) or the tiler heap (on other architectures). This buffer holds transformed primitive -//! data between the vertex/tiling stage and the fragment stage. -//! -//! On AGX, the buffer is a heap of 128K blocks split into 32K pages (which must be aligned to a -//! multiple of 32K in VA space). The buffer can be shared between multiple render jobs, and each -//! will allocate pages from it during vertex processing and return them during fragment processing. -//! -//! If the buffer runs out of free pages, the vertex pass stops and a partial fragment pass occurs, -//! spilling the intermediate render target state to RAM (a partial render). This is all managed -//! transparently by the firmware. Since partial renders are less efficient, the kernel must grow -//! the heap in response to feedback from the firmware to avoid partial renders in the future. -//! Currently, we only ever grow the heap, and never shrink it. -//! -//! AGX also supports memoryless render targets, which can be used for intermediate results within -//! a render pass. To support partial renders, it seems the GPU/firmware has the ability to borrow -//! pages from the TVB buffer as a temporary render target buffer. Since this happens during a -//! partial render itself, if the buffer runs out of space, it requires synchronous growth in -//! response to a firmware interrupt. This is not currently supported, but may be in the future, -//! though it is unclear whether it is worth the effort. -//! -//! This module is also in charge of managing the temporary objects associated with a single render -//! pass, which includes the top-level tile array, the tail pointer cache, preemption buffers, and -//! other miscellaneous structures collectively managed as a "scene". -//! -//! To avoid runaway memory usage, there is a maximum size for buffers (at that point it's unlikely -//! that partial renders will incur much overhead over the buffer data access itself). This is -//! different depending on whether memoryless render targets are in use, and is currently hardcoded. -//! to the most common value used by macOS. - -use crate::debug::*; -use crate::fw::buffer; -use crate::fw::types::*; -use crate::util::*; -use crate::{ - alloc, - fw, - gpu, - hw, - mmu, - slotalloc, // -}; -use core::sync::atomic::Ordering; -use kernel::new_mutex; -use kernel::prelude::*; -use kernel::sync::{ - Arc, - Mutex, // -}; -use kernel::{ - c_str, - static_lock_class, // -}; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Buffer; - -/// There are 127 GPU/firmware-side buffer manager slots (yes, 127, not 128). -const NUM_BUFFERS: u32 = 127; - -/// Page size bits for buffer pages (32K). VAs must be aligned to this size. -pub(crate) const PAGE_SHIFT: usize = 15; -/// Page size for buffer pages. -pub(crate) const PAGE_SIZE: usize = 1 << PAGE_SHIFT; -/// Number of pages in a buffer block, which should be contiguous in VA space. -pub(crate) const PAGES_PER_BLOCK: usize = 4; -/// Size of a buffer block. -pub(crate) const BLOCK_SIZE: usize = PAGE_SIZE * PAGES_PER_BLOCK; - -/// Metadata about the tiling configuration for a scene. This is computed in the `render` module. -/// based on dimensions, tile size, and other info. -pub(crate) struct TileInfo { - /// Tile count in the X dimension. Tiles are always 32x32. - pub(crate) tiles_x: u32, - /// Tile count in the Y dimension. Tiles are always 32x32. - pub(crate) tiles_y: u32, - /// Total tile count. - pub(crate) tiles: u32, - /// Micro-tile width (16 or 32). - pub(crate) utile_width: u32, - /// Micro-tile height (16 or 32). - pub(crate) utile_height: u32, - // Macro-tiles in the X dimension. Always 4. - //pub(crate) mtiles_x: u32, - // Macro-tiles in the Y dimension. Always 4. - //pub(crate) mtiles_y: u32, - /// Tiles per macro-tile in the X dimension. - pub(crate) tiles_per_mtile_x: u32, - /// Tiles per macro-tile in the Y dimension. - pub(crate) tiles_per_mtile_y: u32, - // Total tiles per macro-tile. - //pub(crate) tiles_per_mtile: u32, - /// Micro-tiles per macro-tile in the X dimension. - pub(crate) utiles_per_mtile_x: u32, - /// Micro-tiles per macro-tile in the Y dimension. - pub(crate) utiles_per_mtile_y: u32, - // Total micro-tiles per macro-tile. - //pub(crate) utiles_per_mtile: u32, - /// Size of the top-level tilemap, in bytes (for all layers, one cluster). - pub(crate) tilemap_size: usize, - /// Size of the Tail Pointer Cache, in bytes (for all layers * clusters). - pub(crate) tpc_size: usize, - /// Number of blocks in the clustering meta buffer (for clustering) per layer. - pub(crate) meta1_layer_stride: u32, - /// Number of blocks in the clustering meta buffer (for clustering). - pub(crate) meta1_blocks: u32, - /// Layering metadata size. - pub(crate) layermeta_size: usize, - /// Minimum number of TVB blocks for this render. - pub(crate) min_tvb_blocks: usize, - /// Tiling parameter structure passed to firmware. - pub(crate) params: fw::vertex::raw::TilingParameters, -} - -/// A single scene, representing a render pass and its required buffers. -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct Scene { - object: GpuObject, - slot: u32, - rebind: bool, - preempt2_off: usize, - preempt3_off: usize, - // Note: these are dead code only on some version variants. - // It's easier to do this than to propagate the version conditionals everywhere. - #[allow(dead_code)] - meta1_off: usize, - #[allow(dead_code)] - meta2_off: usize, - #[allow(dead_code)] - meta3_off: usize, - #[allow(dead_code)] - meta4_off: usize, -} - -#[versions(AGX_NEO)] -impl Scene::ver { - /// Returns true if the buffer was bound to a fresh manager slot, and therefore needs an init - /// command before a render. - pub(crate) fn rebind(&self) -> bool { - self.rebind - } - - /// Returns the buffer manager slot this scene's buffer was bound to. - pub(crate) fn slot(&self) -> u32 { - self.slot - } - - /// Returns the GPU pointer to the [`buffer::Scene::ver`]. - pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, buffer::Scene::ver> { - self.object.gpu_pointer() - } - - /// Returns the GPU weak pointer to the [`buffer::Scene::ver`]. - pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { - self.object.weak_pointer() - } - - /// Returns the GPU weak pointer to the kernel-side temp buffer. - /// (purpose unknown...) - pub(crate) fn kernel_buffer_pointer(&self) -> GpuWeakPointer<[u8]> { - self.object.buffer.inner.lock().kernel_buffer.weak_pointer() - } - - /// Returns the GPU pointer to the `buffer::Info::ver` object associated with this Scene. - pub(crate) fn buffer_pointer(&self) -> GpuPointer<'_, buffer::Info::ver> { - // SAFETY: We can't return the strong pointer directly since its lifetime crosses a lock, - // but we know its lifetime will be valid as long as &self since we hold a reference to the - // buffer, so just construct the strong pointer with the right lifetime here. - unsafe { self.weak_buffer_pointer().upgrade() } - } - - /// Returns the GPU weak pointer to the `buffer::Info::ver` object associated with this Scene. - pub(crate) fn weak_buffer_pointer(&self) -> GpuWeakPointer { - self.object.buffer.inner.lock().info.weak_pointer() - } - - /// Returns the GPU pointer to the TVB heap metadata buffer. - pub(crate) fn tvb_heapmeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { - self.object.tvb_heapmeta.gpu_pointer() - } - - /// Returns the GPU pointer to the layer metadata buffer. - pub(crate) fn tvb_layermeta_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { - self.object.tvb_heapmeta.gpu_offset_pointer(0x200) - } - - /// Returns the GPU pointer to the top-level TVB tilemap buffer. - pub(crate) fn tvb_tilemap_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { - self.object.tvb_tilemap.gpu_pointer() - } - - /// Returns the GPU pointer to the Tail Pointer Cache buffer. - pub(crate) fn tpc_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { - self.object.tpc.gpu_pointer() - } - - /// Returns the GPU pointer to the first preemption scratch buffer. - pub(crate) fn preempt_buf_1_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { - self.object.preempt_buf.gpu_pointer() - } - - /// Returns the GPU pointer to the second preemption scratch buffer. - pub(crate) fn preempt_buf_2_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { - self.object - .preempt_buf - .gpu_offset_pointer(self.preempt2_off) - } - - /// Returns the GPU pointer to the third preemption scratch buffer. - pub(crate) fn preempt_buf_3_pointer(&self) -> GpuPointer<'_, &'_ [u8]> { - self.object - .preempt_buf - .gpu_offset_pointer(self.preempt3_off) - } - - /// Returns the GPU pointer to the per-cluster tilemap buffer, if clustering is enabled. - #[allow(dead_code)] - pub(crate) fn cluster_tilemaps_pointer(&self) -> Option> { - self.object - .clustering - .as_ref() - .map(|c| c.tilemaps.gpu_pointer()) - } - - /// Returns the GPU pointer to the clustering layer metadata buffer, if clustering is enabled. - #[allow(dead_code)] - pub(crate) fn tvb_cluster_layermeta_pointer(&self) -> Option> { - self.object - .clustering - .as_ref() - .map(|c| c.meta.gpu_pointer()) - } - - /// Returns the GPU pointer to the clustering metadata 1 buffer, if clustering is enabled. - #[allow(dead_code)] - pub(crate) fn meta_1_pointer(&self) -> Option> { - self.object - .clustering - .as_ref() - .map(|c| c.meta.gpu_offset_pointer(self.meta1_off)) - } - - /// Returns the GPU pointer to the clustering metadata 2 buffer, if clustering is enabled. - #[allow(dead_code)] - pub(crate) fn meta_2_pointer(&self) -> Option> { - self.object - .clustering - .as_ref() - .map(|c| c.meta.gpu_offset_pointer(self.meta2_off)) - } - - /// Returns the GPU pointer to the clustering metadata 3 buffer, if clustering is enabled. - #[allow(dead_code)] - pub(crate) fn meta_3_pointer(&self) -> Option> { - self.object - .clustering - .as_ref() - .map(|c| c.meta.gpu_offset_pointer(self.meta3_off)) - } - - /// Returns the GPU pointer to the clustering metadata 4 buffer, if clustering is enabled. - #[allow(dead_code)] - pub(crate) fn meta_4_pointer(&self) -> Option> { - self.object - .clustering - .as_ref() - .map(|c| c.meta.gpu_offset_pointer(self.meta4_off)) - } -} - -#[versions(AGX_NEO)] -impl Drop for Scene::ver { - fn drop(&mut self) { - let mut inner = self.object.buffer.inner.lock(); - assert_ne!(inner.active_scenes, 0); - inner.active_scenes -= 1; - - if inner.active_scenes == 0 { - mod_pr_debug!( - "Buffer: no scenes left, dropping slot {}", - inner.active_slot.take().unwrap().slot() - ); - inner.active_slot = None; - } - } -} - -/// Inner data for a single TVB buffer object. -#[versions(AGX_NEO)] -struct BufferInner { - info: GpuObject, - ualloc: Arc>, - ualloc_priv: Arc>, - blocks: KVec>, - max_blocks: usize, - max_blocks_nomemless: usize, - mgr: BufferManager::ver, - active_scenes: usize, - active_slot: Option>, - last_token: Option, - tpc: Option>>, - kernel_buffer: GpuArray, - stats: GpuObject, - cfg: &'static hw::HwConfig, - preempt1_size: usize, - preempt2_size: usize, - preempt3_size: usize, - num_clusters: usize, -} - -/// Locked and reference counted TVB buffer. -#[versions(AGX_NEO)] -pub(crate) struct Buffer { - inner: Arc>, -} - -#[versions(AGX_NEO)] -impl Buffer::ver { - /// Create a new Buffer for a given VM, given the per-VM allocators. - pub(crate) fn new( - gpu: &dyn gpu::GpuManager, - alloc: &mut gpu::KernelAllocators, - ualloc: Arc>, - ualloc_priv: Arc>, - mgr: &BufferManager::ver, - ) -> Result { - // These are the typical max numbers on macOS. - // 8GB machines have this halved. - let max_size: usize = 862_322_688; // bytes - let max_size_nomemless = max_size / 3; - - let max_blocks = max_size / BLOCK_SIZE; - let max_blocks_nomemless = max_size_nomemless / BLOCK_SIZE; - let max_pages = max_blocks * PAGES_PER_BLOCK; - let max_pages_nomemless = max_blocks_nomemless * PAGES_PER_BLOCK; - - let num_clusters = gpu.get_dyncfg().id.num_clusters as usize; - let num_clusters_adj = if num_clusters > 1 { - num_clusters + 1 - } else { - 1 - }; - - let preempt1_size = num_clusters_adj * gpu.get_cfg().preempt1_size; - let preempt2_size = num_clusters_adj * gpu.get_cfg().preempt2_size; - let preempt3_size = num_clusters_adj * gpu.get_cfg().preempt3_size; - - let shared = &mut alloc.shared; - let info = alloc.private.new_init( - { - let ualloc_priv = &ualloc_priv; - try_init!(buffer::Info::ver { - block_ctl: shared.new_default::()?, - counter: shared.new_default::()?, - page_list: ualloc_priv.lock().array_empty_tagged(max_pages, b"PLST")?, - block_list: ualloc_priv - .lock() - .array_empty_tagged(max_blocks * 2, b"BLST")?, - }) - }, - |inner, _p| { - try_init!(buffer::raw::Info::ver { - gpu_counter: 0x0, - unk_4: 0, - last_id: 0x0, - cur_id: -1, - unk_10: 0x0, - gpu_counter2: 0x0, - unk_18: 0x0, - #[ver(V < V13_0B4 || G >= G14X)] - unk_1c: 0x0, - page_list: inner.page_list.gpu_pointer(), - page_list_size: (4 * max_pages).try_into()?, - page_count: AtomicU32::new(0), - max_blocks: max_blocks.try_into()?, - block_count: AtomicU32::new(0), - unk_38: 0x0, - block_list: inner.block_list.gpu_pointer(), - block_ctl: inner.block_ctl.gpu_pointer(), - last_page: AtomicU32::new(0), - gpu_page_ptr1: 0x0, - gpu_page_ptr2: 0x0, - unk_58: 0x0, - block_size: BLOCK_SIZE as u32, - unk_60: U64(0x0), - counter: inner.counter.gpu_pointer(), - unk_70: 0x0, - unk_74: 0x0, - unk_78: 0x0, - unk_7c: 0x0, - unk_80: 0x1, - max_pages: max_pages.try_into()?, - max_pages_nomemless: max_pages_nomemless.try_into()?, - unk_8c: 0x0, - unk_90: Default::default(), - }) - }, - )?; - - // Technically similar to Scene below, let's play it safe. - let kernel_buffer = alloc.shared.array_empty_tagged(0x40, b"KBUF")?; - let stats = alloc - .shared - .new_object(Default::default(), |_inner| buffer::raw::Stats { - reset: AtomicU32::from(1), - ..Default::default() - })?; - - Ok(Buffer::ver { - inner: Arc::pin_init( - new_mutex!(BufferInner::ver { - info, - ualloc, - ualloc_priv, - blocks: KVec::new(), - max_blocks, - max_blocks_nomemless, - mgr: mgr.clone(), - active_scenes: 0, - active_slot: None, - last_token: None, - tpc: None, - kernel_buffer, - stats, - cfg: gpu.get_cfg(), - preempt1_size, - preempt2_size, - preempt3_size, - num_clusters, - }), - GFP_KERNEL, - )?, - }) - } - - /// Returns the total block count allocated to this Buffer. - pub(crate) fn block_count(&self) -> u32 { - self.inner.lock().blocks.len() as u32 - } - - /// Automatically grow the Buffer based on feedback from the statistics. - pub(crate) fn auto_grow(&self) -> Result { - let inner = self.inner.lock(); - - let used_pages = inner.stats.with(|raw, _inner| { - let used = raw.max_pages.load(Ordering::Relaxed); - raw.reset.store(1, Ordering::Release); - used as usize - }); - - let need_blocks = (used_pages * 2) - .div_ceil(PAGES_PER_BLOCK) - .min(inner.max_blocks_nomemless); - let want_blocks = (used_pages * 3) - .div_ceil(PAGES_PER_BLOCK) - .min(inner.max_blocks_nomemless); - - let cur_count = inner.blocks.len(); - - if need_blocks <= cur_count { - Ok(false) - } else { - // Grow to 3x requested size (same logic as macOS) - core::mem::drop(inner); - self.ensure_blocks(want_blocks)?; - Ok(true) - } - } - - /// Synchronously grow the Buffer. - pub(crate) fn sync_grow(&self) { - let inner = self.inner.lock(); - - let cur_count = inner.blocks.len(); - core::mem::drop(inner); - if self.ensure_blocks(cur_count + 10).is_err() { - pr_err!("BufferManager: Failed to grow buffer synchronously\n"); - } - } - - /// Ensure that the buffer has at least a certain minimum size in blocks. - pub(crate) fn ensure_blocks(&self, min_blocks: usize) -> Result { - let mut inner = self.inner.lock(); - - let cur_count = inner.blocks.len(); - if cur_count >= min_blocks { - return Ok(false); - } - if min_blocks > inner.max_blocks { - return Err(ENOMEM); - } - - let add_blocks = min_blocks - cur_count; - let new_count = min_blocks; - - let mut new_blocks: KVec> = KVec::new(); - - // Allocate the new blocks first, so if it fails they will be dropped - let mut ualloc = inner.ualloc.lock(); - for _i in 0..add_blocks { - new_blocks.push(ualloc.array_gpuonly(BLOCK_SIZE)?, GFP_KERNEL)?; - } - core::mem::drop(ualloc); - - // Then actually commit them - inner.blocks.reserve(add_blocks, GFP_KERNEL)?; - - for (i, block) in new_blocks.into_iter().enumerate() { - let page_num = (block.gpu_va().get() >> PAGE_SHIFT) as u32; - - inner - .blocks - .push(block, GFP_KERNEL) - .expect("push() failed after reserve()"); - inner.info.block_list[2 * (cur_count + i)] = page_num; - for j in 0..PAGES_PER_BLOCK { - inner.info.page_list[(cur_count + i) * PAGES_PER_BLOCK + j] = page_num + j as u32; - } - } - - inner.info.block_ctl.with(|raw, _inner| { - raw.total.store(new_count as u32, Ordering::SeqCst); - raw.wptr.store(new_count as u32, Ordering::SeqCst); - }); - - /* Only do this update if the buffer manager is idle (which means we own it) */ - if inner.active_scenes == 0 { - let page_count = (new_count * PAGES_PER_BLOCK) as u32; - inner.info.with(|raw, _inner| { - raw.page_count.store(page_count, Ordering::Relaxed); - raw.block_count.store(new_count as u32, Ordering::Relaxed); - raw.last_page.store(page_count - 1, Ordering::Relaxed); - }); - } - - Ok(true) - } - - /// Create a new [`Scene::ver`] (render pass) using this buffer. - pub(crate) fn new_scene( - &self, - alloc: &mut gpu::KernelAllocators, - tile_info: &TileInfo, - ) -> Result { - let mut inner = self.inner.lock(); - - let tilemap_size = tile_info.tilemap_size; - let tpc_size = tile_info.tpc_size; - - // TODO: what is this exactly? - mod_pr_debug!("Buffer: Allocating TVB buffers\n"); - - // This seems to be a list, with 4x2 bytes of headers and 8 bytes per entry. - // On single-cluster devices, the used length always seems to be 1. - // On M1 Ultra, it can grow and usually doesn't exceed 64 entries. - // macOS allocates a whole 64K * 0x80 for this, so let's go with - // that to be safe... - let user_buffer = inner.ualloc.lock().array_empty_tagged( - if inner.num_clusters > 1 { - 0x10080 - } else { - 0x80 - }, - b"UBUF", - )?; - - let tvb_heapmeta = inner - .ualloc - .lock() - .array_empty_tagged(0x200 + tile_info.layermeta_size, b"HMTA")?; - let tvb_tilemap = inner - .ualloc - .lock() - .array_empty_tagged(tilemap_size, b"TMAP")?; - - mod_pr_debug!("Buffer: Allocating misc buffers\n"); - let preempt_buf = inner.ualloc.lock().array_empty_tagged( - inner.preempt1_size + inner.preempt2_size + inner.preempt3_size, - b"PRMT", - )?; - - let tpc = match inner.tpc.as_ref() { - Some(buf) if buf.len() >= tpc_size => buf.clone(), - _ => { - // MacOS allocates this as shared GPU+FW, but - // priv seems to work and might be faster? - // Needs to be FW-writable anyway, so ualloc - // won't work. - let buf = Arc::new( - inner.ualloc_priv.lock().array_empty_tagged( - (tpc_size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK, - b"TPC ", - )?, - GFP_KERNEL, - )?; - inner.tpc = Some(buf.clone()); - buf - } - }; - - let mut clmeta_size = 0; - let mut meta1_size = 0; - let mut meta2_size = 0; - let mut meta3_size = 0; - - let clustering = if inner.num_clusters > 1 { - let cfg = inner.cfg.clustering.as_ref().unwrap(); - - clmeta_size = tile_info.layermeta_size * cfg.max_splits; - // Maybe: (4x4 macro tiles + 1 global page)*n, 32bit each (17*4*n) - // Unused on t602x? - meta1_size = align(tile_info.meta1_blocks as usize * cfg.meta1_blocksize, 0x80); - meta2_size = align(cfg.meta2_size, 0x80); - meta3_size = align(cfg.meta3_size, 0x80); - let meta4_size = cfg.meta4_size; - - let meta_size = clmeta_size + meta1_size + meta2_size + meta3_size + meta4_size; - - mod_pr_debug!("Buffer: Allocating clustering buffers\n"); - let tilemaps = inner - .ualloc - .lock() - .array_empty_tagged(cfg.max_splits * tilemap_size, b"CTMP")?; - let meta = inner.ualloc.lock().array_empty_tagged(meta_size, b"CMTA")?; - Some(buffer::ClusterBuffers { tilemaps, meta }) - } else { - None - }; - - // Could be made strong, but we wind up with a deadlock if we try to grab the - // pointer through the inner.buffer path inside the closure. - let stats_pointer = inner.stats.weak_pointer(); - - let _gpu = &mut alloc.gpu; - - // macOS allocates this as private. However, the firmware does not - // DC CIVAC this before reading it (like it does most other things), - // which causes odd cache incoherency bugs when combined with - // speculation on the firmware side (maybe). This doesn't happen - // on macOS because these structs are a circular pool that is mapped - // already initialized. Just mark this shared for now. - let scene = alloc.shared.new_init( - try_init!(buffer::Scene::ver { - user_buffer: user_buffer, - buffer: self.clone(), - tvb_heapmeta: tvb_heapmeta, - tvb_tilemap: tvb_tilemap, - tpc: tpc, - clustering: clustering, - preempt_buf: preempt_buf, - #[ver(G >= G14X)] - control_word: _gpu.array_empty_tagged(1, b"CWRD")?, - }), - |inner, _p| { - try_init!(buffer::raw::Scene::ver { - #[ver(G >= G14X)] - control_word: inner.control_word.gpu_pointer(), - #[ver(G >= G14X)] - control_word2: inner.control_word.gpu_pointer(), - pass_page_count: AtomicU32::new(0), - unk_4: 0, - unk_8: U64(0), - unk_10: U64(0), - user_buffer: inner.user_buffer.gpu_pointer(), - unk_20: 0, - #[ver(V >= V13_3)] - unk_28: U64(0), - stats: stats_pointer, - total_page_count: AtomicU32::new(0), - #[ver(G < G14X)] - unk_30: U64(0), - #[ver(G < G14X)] - unk_38: U64(0), - }) - }, - )?; - - let mut rebind = false; - - if inner.active_slot.is_none() { - assert_eq!(inner.active_scenes, 0); - - let slot = inner.mgr.0.get_inner(inner.last_token, |inner, mgr| { - inner.owners[mgr.slot() as usize] = Some(self.clone()); - Ok(()) - })?; - rebind = slot.changed(); - - mod_pr_debug!("Buffer: assigning slot {} (rebind={})", slot.slot(), rebind); - - inner.last_token = Some(slot.token()); - inner.active_slot = Some(slot); - } - - inner.active_scenes += 1; - - Ok(Scene::ver { - object: scene, - slot: inner.active_slot.as_ref().unwrap().slot(), - rebind, - preempt2_off: inner.preempt1_size, - preempt3_off: inner.preempt1_size + inner.preempt2_size, - meta1_off: clmeta_size, - meta2_off: clmeta_size + meta1_size, - meta3_off: clmeta_size + meta1_size + meta2_size, - meta4_off: clmeta_size + meta1_size + meta2_size + meta3_size, - }) - } - - /// Increment the buffer manager usage count. Should we done once we know the Scene is ready - /// to be committed and used in commands submitted to the GPU. - pub(crate) fn increment(&self) { - let inner = self.inner.lock(); - inner.info.counter.with(|raw, _inner| { - // We could use fetch_add, but the non-LSE atomic - // sequence Rust produces confuses the hypervisor. - // We have inner locked anyway, so this is not racy. - let v = raw.count.load(Ordering::Relaxed); - raw.count.store(v + 1, Ordering::Relaxed); - }); - } - - pub(crate) fn any_ref(&self) -> Arc { - self.inner.clone() - } -} - -#[versions(AGX_NEO)] -impl Clone for Buffer::ver { - fn clone(&self) -> Self { - Buffer::ver { - inner: self.inner.clone(), - } - } -} - -#[versions(AGX_NEO)] -struct BufferSlotInner(); - -#[versions(AGX_NEO)] -impl slotalloc::SlotItem for BufferSlotInner::ver { - type Data = BufferManagerInner::ver; - - fn release(&mut self, data: &mut Self::Data, slot: u32) { - mod_pr_debug!("BufferManager: Released slot {}\n", slot); - data.owners[slot as usize] = None; - } -} - -/// Inner data for the buffer manager, to be protected by the SlotAllocator lock. -#[versions(AGX_NEO)] -pub(crate) struct BufferManagerInner { - owners: KVec>, -} - -/// The GPU-global buffer manager, used to allocate and release buffer slots from the pool. -#[versions(AGX_NEO)] -pub(crate) struct BufferManager(slotalloc::SlotAllocator); - -#[versions(AGX_NEO)] -impl BufferManager::ver { - pub(crate) fn new() -> Result { - let mut owners = KVec::new(); - for _i in 0..(NUM_BUFFERS as usize) { - owners.push(None, GFP_KERNEL)?; - } - Ok(BufferManager::ver(slotalloc::SlotAllocator::new( - NUM_BUFFERS, - BufferManagerInner::ver { owners }, - |_inner, _slot| Some(BufferSlotInner::ver()), - c_str!("BufferManager::SlotAllocator"), - static_lock_class!(), - static_lock_class!(), - )?)) - } - - /// Signals a Buffer to synchronously grow. - pub(crate) fn grow(&self, slot: u32) { - match self - .0 - .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) - { - Some(owner) => { - pr_err!( - "BufferManager: Unexpected grow request for slot {}. This might deadlock. Please report this bug.\n", - slot - ); - owner.sync_grow(); - } - None => { - pr_err!( - "BufferManager: Received grow request for empty slot {}\n", - slot - ); - } - } - } -} - -#[versions(AGX_NEO)] -impl Clone for BufferManager::ver { - fn clone(&self) -> Self { - BufferManager::ver(self.0.clone()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/channel.rs b/drivers/gpu/drm/asahi-neo/channel.rs deleted file mode 100644 index 590d2e5fa3aae9..00000000000000 --- a/drivers/gpu/drm/asahi-neo/channel.rs +++ /dev/null @@ -1,720 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU ring buffer channels -//! -//! The GPU firmware use a set of ring buffer channels to receive commands from the driver and send -//! it notifications and status messages. -//! -//! These ring buffers mostly follow uniform conventions, so they share the same base -//! implementation. - -use crate::debug::*; -use crate::driver::{ - AsahiDevRef, - AsahiDevice, // -}; -use crate::fw::channels::*; -use crate::fw::initdata::{ - raw, - ChannelRing, // -}; -use crate::fw::types::*; -use crate::{ - buffer, - event, - gpu, - mem, - module_parameters, // -}; -use core::sync::atomic::Ordering; -use kernel::{ - c_str, - prelude::*, - sync::Arc, - time::{ - delay::fsleep, - Delta, - Instant, - Monotonic, // - }, -}; - -pub(crate) use crate::fw::channels::PipeType; - -/// A receive (FW->driver) channel. -pub(crate) struct RxChannel -where - for<'a> ::Raw<'a>: Debug + Default + Zeroable, -{ - ring: ChannelRing, - // FIXME: needs feature(generic_const_exprs) - //rptr: [u32; T::SUB_CHANNELS], - rptr: [u32; 6], - count: u32, -} - -impl RxChannel -where - for<'a> ::Raw<'a>: Debug + Default + Zeroable, -{ - /// Allocates a new receive channel with a given message count. - pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { - Ok(RxChannel { - ring: ChannelRing { - state: alloc.shared.new_default()?, - ring: alloc.shared.array_empty(T::SUB_CHANNELS * count)?, - }, - rptr: Default::default(), - count: count as u32, - }) - } - - /// Receives a message on the specified sub-channel index, optionally leaving in the ring - /// buffer. - /// - /// Returns None if the channel is empty. - fn get_or_peek(&mut self, index: usize, peek: bool) -> Option { - self.ring.state.with(|raw, _inner| { - let wptr = T::wptr(raw, index); - let rptr = &mut self.rptr[index]; - if wptr == *rptr { - None - } else { - let off = self.count as usize * index; - let msg = self.ring.ring[off + *rptr as usize]; - if !peek { - *rptr = (*rptr + 1) % self.count; - T::set_rptr(raw, index, *rptr); - } - Some(msg) - } - }) - } - - /// Receives a message on the specified sub-channel index, and dequeues it from the ring buffer. - /// - /// Returns None if the channel is empty. - pub(crate) fn get(&mut self, index: usize) -> Option { - self.get_or_peek(index, false) - } - - /// Peeks a message on the specified sub-channel index, leaving it in the ring buffer. - /// - /// Returns None if the channel is empty. - pub(crate) fn peek(&mut self, index: usize) -> Option { - self.get_or_peek(index, true) - } -} - -/// A transmit (driver->FW) channel. -pub(crate) struct TxChannel -where - for<'a> ::Raw<'a>: Debug + Default + Zeroable, -{ - ring: ChannelRing, - wptr: u32, - count: u32, -} - -impl TxChannel -where - for<'a> ::Raw<'a>: Debug + Default + Zeroable, -{ - /// Allocates a new cached transmit channel with a given message count. - pub(crate) fn new(alloc: &mut gpu::KernelAllocators, count: usize) -> Result> { - Ok(TxChannel { - ring: ChannelRing { - state: alloc.shared.new_default()?, - ring: alloc.private.array_empty(count)?, - }, - wptr: 0, - count: count as u32, - }) - } - - /// Allocates a new uncached transmit channel with a given message count. - pub(crate) fn new_uncached( - alloc: &mut gpu::KernelAllocators, - count: usize, - ) -> Result> { - Ok(TxChannel { - ring: ChannelRing { - state: alloc.shared.new_default()?, - ring: alloc.shared.array_empty(count)?, - }, - wptr: 0, - count: count as u32, - }) - } - - /// Send a message to the ring, returning a cookie with the ring buffer position. - /// - /// This will poll/block if the ring is full, which we don't really expect to happen. - pub(crate) fn put(&mut self, msg: &U) -> u32 { - self.ring.state.with(|raw, _inner| { - let next_wptr = (self.wptr + 1) % self.count; - let mut rptr = T::rptr(raw); - if next_wptr == rptr { - pr_err!( - "TX ring buffer is full! Waiting... ({}, {})\n", - next_wptr, - rptr - ); - // TODO: block properly on incoming messages? - while next_wptr == rptr { - fsleep(Delta::from_millis(8)); - rptr = T::rptr(raw); - } - } - self.ring.ring[self.wptr as usize] = *msg; - mem::sync(); - T::set_wptr(raw, next_wptr); - self.wptr = next_wptr; - }); - self.wptr - } - - /// Wait for a previously submitted message to be popped off of the ring by the GPU firmware. - /// - /// This busy-loops, and is intended to be used for rare cases when we need to block for - /// completion of a cache management or invalidation operation synchronously (which - /// the firmware normally completes fast enough not to be worth sleeping for). - /// If the poll takes longer than 10ms, this switches to sleeping between polls. - pub(crate) fn wait_for(&mut self, wptr: u32, timeout_ms: i64) -> Result { - const MAX_FAST_POLL: i64 = 10; - let start = Instant::::now(); - let timeout_ms = timeout_ms.max(1); - let timeout_fast = Delta::from_millis(timeout_ms.min(MAX_FAST_POLL)); - let timeout_slow = Delta::from_millis(timeout_ms); - self.ring.state.with(|raw, _inner| { - while start.elapsed() < timeout_fast { - if T::rptr(raw) == wptr { - return Ok(()); - } - mem::sync(); - } - while start.elapsed() < timeout_slow { - if T::rptr(raw) == wptr { - return Ok(()); - } - fsleep(Delta::from_millis(5)); - mem::sync(); - } - Err(ETIMEDOUT) - }) - } -} - -/// Device Control channel for global device management commands. -#[versions(AGX_NEO)] -pub(crate) struct DeviceControlChannel { - dev: AsahiDevRef, - ch: TxChannel, -} - -#[versions(AGX_NEO)] -impl DeviceControlChannel::ver { - const COMMAND_TIMEOUT_MS: i64 = 1000; - - /// Allocate a new Device Control channel. - pub(crate) fn new( - dev: &AsahiDevice, - alloc: &mut gpu::KernelAllocators, - ) -> Result { - Ok(DeviceControlChannel::ver { - dev: dev.into(), - ch: TxChannel::::new(alloc, 0x100)?, - }) - } - - /// Returns the raw `ChannelRing` structure to pass to firmware. - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - self.ch.ring.to_raw() - } - - /// Submits a Device Control command. - pub(crate) fn send(&mut self, msg: &DeviceControlMsg::ver) -> u32 { - cls_dev_dbg!(DeviceControlCh, self.dev, "DeviceControl: {:?}\n", msg); - self.ch.put(msg) - } - - /// Waits for a previously submitted Device Control command to complete. - pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { - self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) - } -} - -/// Pipe channel to submit WorkQueue execution requests. -#[versions(AGX_NEO)] -pub(crate) struct PipeChannel { - dev: AsahiDevRef, - ch: TxChannel, -} - -#[versions(AGX_NEO)] -impl PipeChannel::ver { - /// Allocate a new Pipe submission channel. - pub(crate) fn new( - dev: &AsahiDevice, - alloc: &mut gpu::KernelAllocators, - ) -> Result { - Ok(PipeChannel::ver { - dev: dev.into(), - ch: TxChannel::::new(alloc, 0x100)?, - }) - } - - /// Returns the raw `ChannelRing` structure to pass to firmware. - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - self.ch.ring.to_raw() - } - - /// Submits a Pipe kick command to the firmware. - pub(crate) fn send(&mut self, msg: &PipeMsg::ver) { - cls_dev_dbg!(PipeCh, self.dev, "Pipe: {:?}\n", msg); - self.ch.put(msg); - } -} - -/// Firmware Control channel, used for secure cache flush requests. -pub(crate) struct FwCtlChannel { - dev: AsahiDevRef, - ch: TxChannel, -} - -impl FwCtlChannel { - const COMMAND_TIMEOUT_MS: i64 = 1000; - - /// Allocate a new Firmware Control channel. - pub(crate) fn new( - dev: &AsahiDevice, - alloc: &mut gpu::KernelAllocators, - ) -> Result { - Ok(FwCtlChannel { - dev: dev.into(), - ch: TxChannel::::new_uncached(alloc, 0x100)?, - }) - } - - /// Returns the raw `ChannelRing` structure to pass to firmware. - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - self.ch.ring.to_raw() - } - - /// Submits a Firmware Control command to the firmware. - pub(crate) fn send(&mut self, msg: &FwCtlMsg) -> u32 { - cls_dev_dbg!(FwCtlCh, self.dev, "FwCtl: {:?}\n", msg); - self.ch.put(msg) - } - - /// Waits for a previously submitted Firmware Control command to complete. - pub(crate) fn wait_for(&mut self, wptr: u32) -> Result { - self.ch.wait_for(wptr, Self::COMMAND_TIMEOUT_MS) - } -} - -/// Event channel, used to notify the driver of command completions, GPU faults and errors, and -/// other events. -#[versions(AGX_NEO)] -pub(crate) struct EventChannel { - dev: AsahiDevRef, - ch: RxChannel, - ev_mgr: Arc, - buf_mgr: buffer::BufferManager::ver, - gpu: Option>, -} - -#[versions(AGX_NEO)] -impl EventChannel::ver { - /// Allocate a new Event channel. - pub(crate) fn new( - dev: &AsahiDevice, - alloc: &mut gpu::KernelAllocators, - ev_mgr: Arc, - buf_mgr: buffer::BufferManager::ver, - ) -> Result { - Ok(EventChannel::ver { - dev: dev.into(), - ch: RxChannel::::new(alloc, 0x100)?, - ev_mgr, - buf_mgr, - gpu: None, - }) - } - - /// Registers the managing `Gpu` instance that will handle events on this channel. - pub(crate) fn set_manager(&mut self, gpu: Arc) { - self.gpu = Some(gpu); - } - - /// Returns the raw `ChannelRing` structure to pass to firmware. - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - self.ch.ring.to_raw() - } - - /// Polls for new Event messages on this ring. - pub(crate) fn poll(&mut self) { - while let Some(msg) = self.ch.get(0) { - // SAFETY: The raw view is always valid for all bit patterns. - let tag = unsafe { msg.raw.0 }; - match tag { - 0..=EVENT_MAX => { - // SAFETY: Since we have checked the tag to be in range, - // accessing the enum view is valid. - let msg = unsafe { msg.msg }; - - cls_dev_dbg!(EventCh, self.dev, "Event: {:?}\n", msg); - match msg { - EventMsg::Fault => match self.gpu.as_ref() { - Some(gpu) => gpu.handle_fault(), - None => { - dev_crit!( - self.dev.as_ref(), - "EventChannel: No GPU manager available!\n" - ) - } - }, - EventMsg::Timeout { - counter, - unk_8, - event_slot, - } => match self.gpu.as_ref() { - Some(gpu) => gpu.handle_timeout(counter, event_slot, unk_8), - None => { - dev_crit!( - self.dev.as_ref(), - "EventChannel: No GPU manager available!\n" - ) - } - }, - EventMsg::Flag { firing, .. } => { - for (i, flags) in firing.iter().enumerate() { - for j in 0..32 { - if flags & (1u32 << j) != 0 { - self.ev_mgr.signal((i * 32 + j) as u32); - } - } - } - } - EventMsg::GrowTVB { - vm_slot, - buffer_slot, - counter, - } => match self.gpu.as_ref() { - Some(gpu) => { - self.buf_mgr.grow(buffer_slot); - gpu.ack_grow(buffer_slot, vm_slot, counter); - } - None => { - dev_crit!( - self.dev.as_ref(), - "EventChannel: No GPU manager available!\n" - ) - } - }, - EventMsg::ChannelError { - error_type, - pipe_type, - event_slot, - event_value, - } => match self.gpu.as_ref() { - Some(gpu) => { - let error_type = match error_type { - 0 => ChannelErrorType::MemoryError, - 1 => ChannelErrorType::DMKill, - 2 => ChannelErrorType::Aborted, - 3 => ChannelErrorType::Unk3, - a => ChannelErrorType::Unknown(a), - }; - gpu.handle_channel_error( - error_type, - pipe_type, - event_slot, - event_value, - ); - } - None => { - dev_crit!( - self.dev.as_ref(), - "EventChannel: No GPU manager available!\n" - ) - } - }, - msg => { - dev_crit!(self.dev.as_ref(), "Unknown event message: {:?}\n", msg); - } - } - } - _ => { - // SAFETY: The raw view is always valid for all bit patterns. - dev_warn!(self.dev.as_ref(), "Unknown event message: {:?}\n", unsafe { - msg.raw - }); - } - } - } - } -} - -/// Firmware Log channel. This one is pretty special, since it has 6 sub-channels (for different log -/// levels), and it also uses a side buffer to actually hold the log messages, only passing around -/// pointers in the main buffer. -pub(crate) struct FwLogChannel { - dev: AsahiDevRef, - ch: RxChannel, - payload_buf: GpuArray, -} - -impl FwLogChannel { - const RING_SIZE: usize = 0x100; - const BUF_SIZE: usize = 0x100; - - /// Allocate a new Firmware Log channel. - pub(crate) fn new( - dev: &AsahiDevice, - alloc: &mut gpu::KernelAllocators, - ) -> Result { - Ok(FwLogChannel { - dev: dev.into(), - ch: RxChannel::::new(alloc, Self::RING_SIZE)?, - payload_buf: alloc - .shared - .array_empty(Self::BUF_SIZE * FwLogChannelState::SUB_CHANNELS)?, - }) - } - - /// Returns the raw `ChannelRing` structure to pass to firmware. - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - self.ch.ring.to_raw() - } - - /// Returns the GPU pointers to the firmware log payload buffer. - pub(crate) fn get_buf(&self) -> GpuWeakPointer<[RawFwLogPayloadMsg]> { - self.payload_buf.weak_pointer() - } - - /// Polls for new log messages on all sub-rings. - pub(crate) fn poll(&mut self) { - for i in 0..=FwLogChannelState::SUB_CHANNELS - 1 { - while let Some(msg) = self.ch.peek(i) { - cls_dev_dbg!(FwLogCh, self.dev, "FwLog{}: {:?}\n", i, msg); - if msg.msg_type != 2 { - dev_warn!(self.dev.as_ref(), "Unknown FWLog{} message: {:?}\n", i, msg); - self.ch.get(i); - continue; - } - if msg.msg_index.0 as usize >= Self::BUF_SIZE { - dev_warn!( - self.dev.as_ref(), - "FWLog{} message index out of bounds: {:?}\n", - i, - msg - ); - self.ch.get(i); - continue; - } - let index = Self::BUF_SIZE * i + msg.msg_index.0 as usize; - let payload = &self.payload_buf.as_slice()[index]; - if payload.msg_type != 3 { - dev_warn!( - self.dev.as_ref(), - "Unknown FWLog{} payload: {:?}\n", - i, - payload - ); - self.ch.get(i); - continue; - } - let msg = if let Some(end) = payload.msg.iter().position(|&r| r == 0) { - CStr::from_bytes_with_nul(&(*payload.msg)[..end + 1]) - .unwrap_or(c_str!("cstr_err")) - } else { - dev_warn!( - self.dev.as_ref(), - "FWLog{} payload not NUL-terminated: {:?}\n", - i, - payload - ); - self.ch.get(i); - continue; - }; - match i { - 0 => dev_dbg!(self.dev.as_ref(), "FWLog: {}\n", msg), - 1 => dev_info!(self.dev.as_ref(), "FWLog: {}\n", msg), - 2 => dev_notice!(self.dev.as_ref(), "FWLog: {}\n", msg), - 3 => dev_warn!(self.dev.as_ref(), "FWLog: {}\n", msg), - 4 => dev_err!(self.dev.as_ref(), "FWLog: {}\n", msg), - 5 => dev_crit!(self.dev.as_ref(), "FWLog: {}\n", msg), - _ => (), - }; - self.ch.get(i); - } - } - } -} - -pub(crate) struct KTraceChannel { - dev: AsahiDevRef, - ch: RxChannel, -} - -/// KTrace channel, used to receive detailed execution trace markers from the firmware. -/// We currently disable this in initdata, so no messages are expected here at this time. -impl KTraceChannel { - /// Allocate a new KTrace channel. - pub(crate) fn new( - dev: &AsahiDevice, - alloc: &mut gpu::KernelAllocators, - ) -> Result { - Ok(KTraceChannel { - dev: dev.into(), - ch: RxChannel::::new(alloc, 0x200)?, - }) - } - - /// Returns the raw `ChannelRing` structure to pass to firmware. - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - self.ch.ring.to_raw() - } - - /// Polls for new KTrace messages on this ring. - pub(crate) fn poll(&mut self) { - while let Some(msg) = self.ch.get(0) { - cls_dev_dbg!(KTraceCh, self.dev, "KTrace: {:?}\n", msg); - } - } -} - -/// Statistics channel, reporting power-related statistics to the driver. -/// Debug-logs the messages and, when the `stats_export` module parameter -/// is enabled, also updates the shared `StatsSnapshot` exported via -/// `/sys/class/drm_neo/card*/device/agx_stats`. -#[versions(AGX_NEO)] -pub(crate) struct StatsChannel { - dev: AsahiDevRef, - ch: RxChannel, - snap: Arc, - /// Rate of firmware stats-message timestamps (Hz). All stats timestamps - /// run on the chip base clock (`HwConfig::base_clock_hz`, 24 MHz on every - /// supported SoC), not nanoseconds. - ts_hz: u64, - /// Last `Utilization` message timestamp (raw base-clock ticks), for - /// utilization-weighted `busy_ns` integration. - last_util_ts: u64, -} - -#[versions(AGX_NEO)] -impl StatsChannel::ver { - /// Allocate a new Statistics channel. - pub(crate) fn new( - dev: &AsahiDevice, - alloc: &mut gpu::KernelAllocators, - snap: Arc, - ts_hz: u64, - ) -> Result { - Ok(StatsChannel::ver { - dev: dev.into(), - ch: RxChannel::::new(alloc, 0x100)?, - snap, - ts_hz, - last_util_ts: 0, - }) - } - - /// Returns the raw `ChannelRing` structure to pass to firmware. - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - self.ch.ring.to_raw() - } - - /// Polls for new statistics messages on this ring. - pub(crate) fn poll(&mut self) { - while let Some(msg) = self.ch.get(0) { - // SAFETY: The raw view is always valid for all bit patterns. - let tag = unsafe { msg.raw.0 }; - match tag { - 0..=STATS_MAX::ver => { - // SAFETY: Since we have checked the tag to be in range, - // accessing the enum view is valid. - let msg = unsafe { msg.msg }; - cls_dev_dbg!(StatsCh, self.dev, "Stats: {:?}\n", msg); - if *module_parameters::stats_export.value() != 0 { - // Retain the latest values for /sys/class/drm_neo/card*/ - // device/agx_stats. Read-only with respect to - // scheduling or power behaviour: the firmware already - // sent the message, we only keep what it said. - match &msg { - StatsMsg::ver::Utilization { - timestamp, - util1, - util2, - util3, - util4, - } => { - self.snap.util1.store(*util1, Ordering::Relaxed); - self.snap.util2.store(*util2, Ordering::Relaxed); - self.snap.util3.store(*util3, Ordering::Relaxed); - self.snap.util4.store(*util4, Ordering::Relaxed); - // busy_ns: utilization-weighted busy time in - // nanoseconds. Each Utilization window - // contributes its duration scaled by the - // busiest subqueue's percentage. MEASURED on - // T6021 (round-3 window): util1..4 read 100 - // across a saturated 30 s matmul and 0-2 on - // an idle desktop, while FwBusy timestamp - // deltas (the prior producer) covered 0.17% - // of that matmul. The timestamps are base- - // clock ticks (24 MHz), so convert to ns - // once here. The kernel runtime has no - // 128-bit division, so split whole seconds - // from the tick remainder: whole seconds - // contribute util * 1e7 ns each (1e9 / 100 - // exactly), the remainder contributes - // rem * util * 1e9 / (100 * ts_hz), which - // cannot overflow u64 (rem < ts_hz <= 24e6 - // on supported SoCs). - let util = [*util1, *util2, *util3, *util4] - .into_iter() - .max() - .unwrap_or(0) - .min(100) as u64; - let ts = timestamp.0; - if self.last_util_ts != 0 && ts > self.last_util_ts { - let delta = ts - self.last_util_ts; - let whole_s = delta / self.ts_hz; - let rem = delta % self.ts_hz; - let busy = whole_s - .saturating_mul(util) - .saturating_mul(10_000_000) - .saturating_add( - rem * util * 1_000_000_000 / (100 * self.ts_hz), - ); - self.snap.busy_ns.fetch_add(busy, Ordering::Relaxed); - } - self.last_util_ts = ts; - } - StatsMsg::ver::PowerState { pstate, .. } => { - self.snap.pstate.store(*pstate, Ordering::Relaxed); - } - StatsMsg::ver::AvgPower { avg_power, .. } => { - self.snap.avg_power_mw.store(*avg_power, Ordering::Relaxed); - } - StatsMsg::ver::Temperature { - raw_value, scale, .. - } => { - self.snap - .temperature_raw - .store(*raw_value, Ordering::Relaxed); - self.snap.temperature_scale.store(*scale, Ordering::Relaxed); - } - _ => {} - } - } - } - _ => { - // SAFETY: The raw view is always valid for all bit patterns. - pr_warn!("Unknown stats message: {:?}\n", unsafe { msg.raw }); - } - } - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/cleanup.rs b/drivers/gpu/drm/asahi-neo/cleanup.rs deleted file mode 100644 index e3dd5ad95dc0ce..00000000000000 --- a/drivers/gpu/drm/asahi-neo/cleanup.rs +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Reservation-taking timestamp destruction must not block fence producers. -//! -//! The module owns this queue outside its platform registration. Files and -//! exported buffers retain the module; detached scheduler cleanup retains it -//! until its driver callback returns. Unregister joins the firmware/event work. -//! Only after those producer boundaries close may the module destroy this queue. - -use core::{ - ops::Deref, - ptr::{self, NonNull}, - sync::atomic::{AtomicPtr, Ordering}, -}; -use kernel::{ - bindings, c_str, impl_has_work, new_work, - prelude::*, - workqueue::{Queue, Work, WorkItem}, -}; - -static QUEUE: AtomicPtr = AtomicPtr::new(ptr::null_mut()); - -/// Destroyed by module exit after platform-driver unregister, never by a worker. -pub(crate) struct QueueOwner(NonNull); - -// SAFETY: QueueOwner only transfers ownership of a thread-safe kernel queue. -unsafe impl Send for QueueOwner {} -// SAFETY: The queue API synchronizes all accesses; destruction requires ownership. -unsafe impl Sync for QueueOwner {} - -impl QueueOwner { - pub(crate) fn new() -> Result { - // SAFETY: The name is NUL terminated and contains no format specifiers. - let raw = unsafe { - bindings::alloc_workqueue_noprof( - c_str!("asahi-ts-cleanup").as_char_ptr(), - bindings::wq_flags_WQ_UNBOUND | bindings::wq_flags_WQ_MEM_RECLAIM, - 0, - ) - }; - let raw = NonNull::new(raw).ok_or(ENOMEM)?; - if QUEUE - .compare_exchange(ptr::null_mut(), raw.as_ptr(), Ordering::AcqRel, Ordering::Acquire) - .is_err() - { - // SAFETY: This fresh queue has never been published or used. - unsafe { bindings::destroy_workqueue(raw.as_ptr()) }; - return Err(EBUSY); - } - Ok(Self(raw)) - } -} - -impl Drop for QueueOwner { - fn drop(&mut self) { - // No external producers remain: module registration was dropped first. - // Pending cleanup owns no queue reference and cannot enqueue more work. - // SAFETY: The module's unique owner retains the queue through its drain. - unsafe { bindings::destroy_workqueue(self.0.as_ptr()) }; - QUEUE.store(ptr::null_mut(), Ordering::Release); - } -} - -#[pin_data] -struct Item { - value: T, - #[pin] - work: Work, -} - -impl_has_work! { - impl{T: Send + 'static} HasWork> for Item { self.work } -} - -impl WorkItem for Item { - type Pointer = Pin>; - - fn run(this: Self::Pointer) { - drop(this); - } -} - -/// Allocate the destruction work before publication; final handoff cannot fail. -pub(crate) struct Deferred(Option>>>); - -impl Deferred { - pub(crate) fn new(value: T) -> Result { - let item = KBox::pin_init( - pin_init!(Item { - value, - work <- new_work!("Asahi timestamp cleanup"), - }), - GFP_KERNEL, - )?; - Ok(Self(Some(item))) - } -} - -impl Deref for Deferred { - type Target = T; - - fn deref(&self) -> &T { - &self.0.as_ref().unwrap().value - } -} - -impl Drop for Deferred { - fn drop(&mut self) { - if let Some(item) = self.0.take() { - let raw = QUEUE.load(Ordering::Acquire); - assert!(!raw.is_null()); - // SAFETY: Module ownership and producer joins retain the queue until - // this handoff has returned. The unique pinned box is never queued - // twice; enqueue transfers it without allocation or a failure path. - unsafe { Queue::from_raw(raw) }.enqueue(item); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/crashdump.rs b/drivers/gpu/drm/asahi-neo/crashdump.rs deleted file mode 100644 index 148cae7aa9b460..00000000000000 --- a/drivers/gpu/drm/asahi-neo/crashdump.rs +++ /dev/null @@ -1,307 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU crash dump formatter -//! -//! Takes a raw dump of firmware/kernel mapped pages from `pgtable` and formats it into -//! an ELF core dump suitable for dumping into userspace. - -use core::mem::size_of; - -use kernel::{ - devcoredump::DevCoreDump, - error::Result, - page::{Page, PAGE_MASK, PAGE_SHIFT, PAGE_SIZE}, - prelude::*, - types::Owned, - uapi, -}; - -use crate::hw; -use crate::pgtable::{self, DumpedPage, Prot, UAT_PGSZ}; -use crate::util::align; - -pub(crate) struct CrashDump { - headers: KVVec, - pages: KVVec>, -} - -const NOTE_NAME_AGX: &str = "AGX"; -const NOTE_AGX_DUMP_INFO: u32 = 1; - -const NOTE_NAME_RTKIT: &str = "RTKIT"; -const NOTE_RTKIT_CRASHLOG: u32 = 1; - -#[repr(C)] -pub(crate) struct AGXDumpInfo { - initdata_address: u64, - chip_id: u32, - gpu_gen: hw::GpuGen, - gpu_variant: hw::GpuVariant, - gpu_rev: hw::GpuRevision, - total_active_cores: u32, - firmware_version: [u32; 6], - reserved: u32, -} - -struct ELFNote { - name: &'static str, - ty: u32, - data: KVVec, -} - -pub(crate) struct CrashDumpBuilder { - page_dump: KVVec, - notes: KVec, -} - -/// Helper to convert ELF headers into byte slices -/// TODO: Hook this up into kernel::AsBytes somehow -/// -/// # Safety -/// -/// Types implementing this trait must have no padding bytes. -unsafe trait AsBytes: Sized { - fn as_bytes(&self) -> &[u8] { - // SAFETY: This trait is only implemented for types with no padding bytes - unsafe { core::slice::from_raw_parts(self as *const _ as *const u8, size_of::()) } - } - fn slice_as_bytes(slice: &[Self]) -> &[u8] { - // SAFETY: This trait is only implemented for types with no padding bytes - unsafe { - core::slice::from_raw_parts(slice.as_ptr() as *const u8, core::mem::size_of_val(slice)) - } - } -} - -// SAFETY: This type has no padding -unsafe impl AsBytes for uapi::Elf64_Ehdr {} -// SAFETY: This type has no padding -unsafe impl AsBytes for uapi::Elf64_Phdr {} -// SAFETY: This type has no padding -unsafe impl AsBytes for uapi::Elf64_Nhdr {} -// SAFETY: This type has no padding -unsafe impl AsBytes for AGXDumpInfo {} - -const FIRMWARE_ENTRYPOINT: u64 = 0xFFFFFF8000000000u64; - -impl CrashDumpBuilder { - pub(crate) fn new(page_dump: KVVec) -> Result { - Ok(CrashDumpBuilder { - page_dump, - notes: KVec::new(), - }) - } - - pub(crate) fn add_agx_info( - &mut self, - cfg: &hw::HwConfig, - dyncfg: &hw::DynConfig, - initdata_address: u64, - ) -> Result { - let mut info = AGXDumpInfo { - chip_id: cfg.chip_id, - gpu_gen: dyncfg.id.gpu_gen, - gpu_variant: dyncfg.id.gpu_variant, - gpu_rev: dyncfg.id.gpu_rev, - total_active_cores: dyncfg.id.total_active_cores, - firmware_version: [0; 6], - reserved: 0, - initdata_address, - }; - info.firmware_version[..dyncfg.firmware_version.len().min(6)] - .copy_from_slice(&dyncfg.firmware_version); - - let mut data = KVVec::new(); - data.extend_from_slice(info.as_bytes(), GFP_KERNEL)?; - - self.notes.push( - ELFNote { - name: NOTE_NAME_AGX, - ty: NOTE_AGX_DUMP_INFO, - data, - }, - GFP_KERNEL, - )?; - Ok(()) - } - - pub(crate) fn add_crashlog(&mut self, crashlog: &[u8]) -> Result { - let mut data = KVVec::new(); - data.extend_from_slice(crashlog, GFP_KERNEL)?; - - self.notes.push( - ELFNote { - name: NOTE_NAME_RTKIT, - ty: NOTE_RTKIT_CRASHLOG, - data, - }, - GFP_KERNEL, - )?; - - Ok(()) - } - - pub(crate) fn finalize(self) -> Result { - let CrashDumpBuilder { page_dump, notes } = self; - - let mut ehdr: uapi::Elf64_Ehdr = Default::default(); - - ehdr.e_ident[uapi::EI_MAG0 as usize..=uapi::EI_MAG3 as usize].copy_from_slice(b"\x7fELF"); - ehdr.e_ident[uapi::EI_CLASS as usize] = uapi::ELFCLASS64 as u8; - ehdr.e_ident[uapi::EI_DATA as usize] = uapi::ELFDATA2LSB as u8; - ehdr.e_ident[uapi::EI_VERSION as usize] = uapi::EV_CURRENT as u8; - ehdr.e_type = uapi::ET_CORE as u16; - ehdr.e_machine = uapi::EM_AARCH64 as u16; - ehdr.e_version = uapi::EV_CURRENT; - ehdr.e_entry = FIRMWARE_ENTRYPOINT; - ehdr.e_ehsize = core::mem::size_of::() as u16; - ehdr.e_phentsize = core::mem::size_of::() as u16; - - let phdr_offset = core::mem::size_of::(); - - // PHDRs come after the ELF header - ehdr.e_phoff = phdr_offset as u64; - - let mut phdrs = KVVec::new(); - - // First PHDR is the NOTE section - phdrs.push( - uapi::Elf64_Phdr { - p_type: uapi::PT_NOTE, - p_flags: uapi::PF_R, - p_align: 1, - ..Default::default() - }, - GFP_KERNEL, - )?; - - // Generate the page phdrs. The offset will be fixed up later. - let mut off: usize = 0; - let mut next = None; - let mut pages: KVVec> = KVVec::new(); - - for mut page in page_dump { - let vaddr = page.iova; - let paddr = page.pte & pgtable::PTE_ADDR_BITS; - let flags = Prot::from_pte(page.pte).elf_flags(); - let valid = page.data.is_some(); - let cur = (vaddr, paddr, flags, valid); - if Some(cur) != next { - phdrs.push( - uapi::Elf64_Phdr { - p_type: uapi::PT_LOAD, - p_offset: if valid { off as u64 } else { 0 }, - p_vaddr: vaddr, - p_paddr: paddr, - p_filesz: if valid { UAT_PGSZ as u64 } else { 0 }, - p_memsz: UAT_PGSZ as u64, - p_flags: flags, - p_align: UAT_PGSZ as u64, - }, - GFP_KERNEL, - )?; - if valid { - off += UAT_PGSZ; - } - } else { - let ph = phdrs.last_mut().unwrap(); - ph.p_memsz += UAT_PGSZ as u64; - if valid { - ph.p_filesz += UAT_PGSZ as u64; - off += UAT_PGSZ; - } - } - if let Some(data_page) = page.data.take() { - pages.push(data_page, GFP_KERNEL)?; - } - next = Some(( - vaddr + UAT_PGSZ as u64, - paddr + UAT_PGSZ as u64, - flags, - valid, - )); - } - - ehdr.e_phnum = phdrs.len() as u16; - - let note_offset = phdr_offset + size_of::() * phdrs.len(); - - let mut note_data: KVVec = KVVec::new(); - - for note in notes { - let hdr = uapi::Elf64_Nhdr { - n_namesz: note.name.len() as u32 + 1, - n_descsz: note.data.len() as u32, - n_type: note.ty, - }; - note_data.extend_from_slice(hdr.as_bytes(), GFP_KERNEL)?; - note_data.extend_from_slice(note.name.as_bytes(), GFP_KERNEL)?; - note_data.push(0, GFP_KERNEL)?; - while note_data.len() & 3 != 0 { - note_data.push(0, GFP_KERNEL)?; - } - note_data.extend_from_slice(¬e.data, GFP_KERNEL)?; - while note_data.len() & 3 != 0 { - note_data.push(0, GFP_KERNEL)?; - } - } - - // NOTE section comes after the PHDRs - phdrs[0].p_offset = note_offset as u64; - phdrs[0].p_filesz = note_data.len() as u64; - - // Align data section to the page size - let data_offset = align(note_offset + note_data.len(), UAT_PGSZ); - - // Fix up data PHDR offsets - for phdr in &mut phdrs[1..] { - phdr.p_offset += data_offset as u64; - } - - // Build ELF header buffer - let mut headers: KVVec = KVVec::from_elem(0, data_offset, GFP_KERNEL)?; - - headers[0..size_of::()].copy_from_slice(ehdr.as_bytes()); - headers[phdr_offset..phdr_offset + phdrs.len() * size_of::()] - .copy_from_slice(AsBytes::slice_as_bytes(&phdrs)); - headers[note_offset..note_offset + note_data.len()].copy_from_slice(¬e_data); - - Ok(CrashDump { headers, pages }) - } -} - -impl DevCoreDump for CrashDump { - fn read(&self, buf: &mut [u8], mut offset: usize) -> Result { - let mut read = 0; - let mut left = buf.len(); - if offset < self.headers.len() { - let block = left.min(self.headers.len() - offset); - buf[..block].copy_from_slice(&self.headers[offset..offset + block]); - read += block; - offset += block; - left -= block; - } - if left == 0 { - return Ok(read); - } - offset -= self.headers.len(); // Offset from the page area - - while left > 0 { - let page_index = offset >> PAGE_SHIFT; - let page_offset = offset & !PAGE_MASK; - let block = left.min(PAGE_SIZE - page_offset); - let Some(page) = self.pages.get(page_index) else { - break; - }; - let slice = &mut buf[read..read + block]; - // SAFETY: We own the page, and the slice guarantees the - // dst length is sufficient. - unsafe { page.read_raw(slice.as_mut_ptr(), page_offset, slice.len())? }; - read += block; - offset += block; - left -= block; - } - - Ok(read) - } -} diff --git a/drivers/gpu/drm/asahi-neo/debug.rs b/drivers/gpu/drm/asahi-neo/debug.rs deleted file mode 100644 index 50628ade5ab8e4..00000000000000 --- a/drivers/gpu/drm/asahi-neo/debug.rs +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#![allow(dead_code)] - -//! Debug enable/disable flags and convenience macros - -#[allow(unused_imports)] -pub(crate) use super::{ - cls_dev_dbg, - cls_pr_debug, - debug, - mod_dev_dbg, - mod_pr_debug, // -}; -use crate::module_parameters; -use core::sync::atomic::{ - AtomicU64, - Ordering, // -}; - -static DEBUG_FLAGS: AtomicU64 = AtomicU64::new(0); - -/// Debug flag bit indices -pub(crate) enum DebugFlags { - // 0-4: Memory-related debug - Mmu = 0, - PgTable = 1, - Alloc = 2, - Gem = 3, - Object = 4, - - // 5-7: Firmware objects and resources - Event = 5, - Buffer = 6, - WorkQueue = 7, - - // 8-13: DRM interface, rendering, compute, GPU globals - Gpu = 8, - File = 9, - Queue = 10, - Render = 11, - Compute = 12, - Errors = 13, - - // 14-15: Misc stats - MemStats = 14, - TVBStats = 15, - - // 16-22: Channels - FwLogCh = 16, - KTraceCh = 17, - StatsCh = 18, - EventCh = 19, - PipeCh = 20, - DeviceControlCh = 21, - FwCtlCh = 22, - - // 32-35: Allocator debugging - FillAllocations = 32, - DebugAllocations = 33, - DetectOverflows = 34, - ForceCPUMaps = 35, - - // 36-: Behavior flags - ConservativeTlbi = 36, - KeepGpuPowered = 37, - WaitForPowerOff = 38, - NoGpuRecovery = 39, - DisableClustering = 40, - - // 48-: Misc - Debug0 = 48, - Debug1 = 49, - Debug2 = 50, - Debug3 = 51, - Debug4 = 52, - Debug5 = 53, - Debug6 = 54, - Debug7 = 55, - - VerboseFaults = 61, - AllowUnknownOverrides = 62, - OopsOnGpuCrash = 63, -} - -/// Update the cached global debug flags from the module parameter -pub(crate) fn update_debug_flags() { - let flags = *module_parameters::debug_flags.value(); - - DEBUG_FLAGS.store(flags, Ordering::Relaxed); -} - -/// Check whether debug is enabled for a given flag -#[inline(always)] -pub(crate) fn debug_enabled(flag: DebugFlags) -> bool { - DEBUG_FLAGS.load(Ordering::Relaxed) & 1 << (flag as usize) != 0 -} - -/// Run some code only if debug is enabled for the calling module -#[macro_export] -macro_rules! debug { - ($($arg:tt)*) => { - if $crate::debug::debug_enabled(DEBUG_CLASS) { - $($arg)* - } - }; -} - -/// pr_info!() if debug is enabled for the calling module -#[macro_export] -macro_rules! mod_pr_debug ( - ($($arg:tt)*) => ( - $crate::debug! { ::kernel::pr_info! ( $($arg)* ); } - ) -); - -/// dev_info!() if debug is enabled for the calling module -#[macro_export] -macro_rules! mod_dev_dbg ( - ($dev:expr, $($arg:tt)*) => ( - $crate::debug! { ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); } - ) -); - -/// pr_info!() if debug is enabled for a specific module -#[macro_export] -macro_rules! cls_pr_debug ( - ($cls:ident, $($arg:tt)*) => ( - if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { - ::kernel::pr_info! ( $($arg)* ); - } - ) -); - -/// dev_info!() if debug is enabled for a specific module -#[macro_export] -macro_rules! cls_dev_dbg ( - ($cls:ident, $dev:expr, $($arg:tt)*) => ( - if $crate::debug::debug_enabled($crate::debug::DebugFlags::$cls) { - ::kernel::dev_info! ( $dev.as_ref(), $($arg)* ); - } - ) -); diff --git a/drivers/gpu/drm/asahi-neo/driver.rs b/drivers/gpu/drm/asahi-neo/driver.rs deleted file mode 100644 index 0c502a21d312e7..00000000000000 --- a/drivers/gpu/drm/asahi-neo/driver.rs +++ /dev/null @@ -1,289 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Top-level GPU driver implementation. - -use crate::module_parameters; -use kernel::bindings; -use kernel::{ - c_str, - device::Core, - dma::{ - Device, - DmaMask, // - }, - drm_neo, - drm_neo::ioctl, - of, - platform, - prelude::*, - sync::{ - aref::ARef, - Arc, // - }, // -}; - -use crate::{ - debug, - file, - g17, - gem::AsahiObject, - gpu, - hw, - regs, // -}; - -use kernel::macros::vtable; - -/// Holds a reference to the top-level GPU object. -#[pin_data] -pub(crate) struct AsahiData { - #[pin] - pub(crate) gpu: Arc, - pub(crate) pdev: ARef, - pub(crate) resources: regs::Resources, -} - -unsafe impl Send for AsahiData {} -unsafe impl Sync for AsahiData {} - -pub(crate) struct AsahiDriver { - drm_neo: ARef>, - /// Raw `struct device *` of the platform device. Stored once at probe - /// time so the `Drop` impl can find the sysfs file on unregister. - raw_dev: *mut bindings::device, - raw_stats: *mut core::ffi::c_void, - // Keep the allocation alive until sysfs readers have drained. - stats: Option>, -} - -unsafe impl Send for AsahiDriver {} -unsafe impl Sync for AsahiDriver {} - -/// Convenience type alias for the DRM device type for this driver. -pub(crate) type AsahiDevice = drm_neo::device::Device; -pub(crate) type AsahiDevRef = ARef; - -/// DRM Driver metadata -const INFO: drm_neo::driver::DriverInfo = drm_neo::driver::DriverInfo { - major: 0, - minor: 0, - patchlevel: 0, - name: c_str!("asahi"), - desc: c_str!("Apple AGX Graphics"), -}; - -/// DRM Driver implementation for `AsahiDriver`. -#[vtable] -impl drm_neo::driver::Driver for AsahiDriver { - /// Our `DeviceData` type, reference-counted - type Data = AsahiData; - /// Our `File` type. - type File = file::File; - /// Our `Object` type. - type Object = drm_neo::gem::shmem::Object; - - const INFO: drm_neo::driver::DriverInfo = INFO; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - const FEATURES: u32 = drm_neo::driver::FEAT_GEM - | drm_neo::driver::FEAT_RENDER - | drm_neo::driver::FEAT_SYNCOBJ - | drm_neo::driver::FEAT_SYNCOBJ_TIMELINE - | drm_neo::driver::FEAT_GEM_GPUVA; - - kernel::declare_drm_neo_ioctls! { - (ASAHI_NEO_GET_PARAMS, drm_asahi_neo_get_params, - ioctl::RENDER_ALLOW, crate::file::File::get_params), - (ASAHI_NEO_GET_TIME, drm_asahi_neo_get_time, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::get_time), - (ASAHI_NEO_VM_CREATE, drm_asahi_neo_vm_create, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_create), - (ASAHI_NEO_VM_DESTROY, drm_asahi_neo_vm_destroy, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_destroy), - (ASAHI_NEO_VM_BIND, drm_asahi_neo_vm_bind, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::vm_bind), - (ASAHI_NEO_GEM_CREATE, drm_asahi_neo_gem_create, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_create), - (ASAHI_NEO_GEM_MMAP_OFFSET, drm_asahi_neo_gem_mmap_offset, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_mmap_offset), - (ASAHI_NEO_GEM_BIND_OBJECT, drm_asahi_neo_gem_bind_object, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_bind_object), - (ASAHI_NEO_QUEUE_CREATE, drm_asahi_neo_queue_create, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_create), - (ASAHI_NEO_QUEUE_DESTROY, drm_asahi_neo_queue_destroy, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::queue_destroy), - (ASAHI_NEO_SUBMIT, drm_asahi_neo_submit, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::submit), - (ASAHI_NEO_GEM_MADVISE, drm_asahi_neo_gem_madvise, - ioctl::AUTH | ioctl::RENDER_ALLOW, crate::file::File::gem_madvise), - } -} - -/// Firmware interface selected by the hardware compatible. -pub(crate) enum ProbeConfig { - Legacy(&'static hw::HwConfig), - G17(&'static g17::Config), -} - -// OF Device ID table. -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - ::IdInfo, - [ - ( - of::DeviceId::new(c_str!("apple,agx-t8140")), - ProbeConfig::G17(&hw::t8140::CONFIG) - ), - ] -); - -/// Platform Driver implementation for `AsahiDriver`. -impl platform::Driver for AsahiDriver { - type IdInfo = ProbeConfig; - const OF_ID_TABLE: Option> = Some(&OF_TABLE); - - fn unbind(_pdev: &platform::Device, this: Pin<&Self>) { - if let Some(gpu) = (*this.drm_neo).gpu.as_any().downcast_ref::() { - // Driver data is dropped after devres, so CPU control must happen in unbind. - gpu.shutdown(); - } - } - - /// Device probe function. - fn probe( - pdev: &platform::Device, - info: Option<&Self::IdInfo>, - ) -> impl PinInit { - debug::update_debug_flags(); - - dev_info!(pdev.as_ref(), "Probing...\n"); - - let cfg = match info.ok_or(ENODEV)? { - ProbeConfig::Legacy(cfg) => *cfg, - ProbeConfig::G17(cfg) => return Self::probe_g17(pdev, cfg), - }; - - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; - - let res = regs::Resources::new(pdev)?; - - // Initialize misc MMIO - res.init_mmio()?; - - // Start the coprocessor CPU, so UAT can initialize the handoff - regs::Resources::start_cpu(pdev)?; - - let fwnode = pdev.as_ref().fwnode().ok_or(EIO)?; - let compat: KVec = fwnode - .property_read_array_vec(c_str!("apple,firmware-compat"), 3)? - .required_by(pdev.as_ref())?; - - // SAFETY: GPU construction uses the DRM allocation but not its driver data. - // No userspace or driver-data work is published before init_data() succeeds. - let drm_neo: ARef = unsafe { drm_neo::device::Device::new_uninit(pdev.as_ref())? }; - - let gpu = match (cfg.gpu_gen, cfg.gpu_variant, compat.as_slice()) { - (hw::GpuGen::G13, _, &[12, 3, 0]) => { - gpu::GpuManagerG13V12_3::new(&drm_neo.clone(), &res, cfg)? as Arc - } - (hw::GpuGen::G14, hw::GpuVariant::G, &[12, 4, 0]) => { - gpu::GpuManagerG14V12_4::new(&drm_neo.clone(), &res, cfg)? as Arc - } - (hw::GpuGen::G13, _, &[13, 5, 0]) => { - gpu::GpuManagerG13V13_5::new(&drm_neo.clone(), &res, cfg)? as Arc - } - (hw::GpuGen::G14, hw::GpuVariant::G, &[13, 5, 0]) => { - gpu::GpuManagerG14V13_5::new(&drm_neo.clone(), &res, cfg)? as Arc - } - (hw::GpuGen::G14, _, &[13, 5, 0]) => { - gpu::GpuManagerG14XV13_5::new(&drm_neo.clone(), &res, cfg)? as Arc - } - _ => { - dev_info!( - pdev.as_ref(), - "Unsupported GPU/firmware combination ({:?}, {:?}, {:?})\n", - cfg.gpu_gen, - cfg.gpu_variant, - compat - ); - return Err(ENODEV); - } - }; - - let data = try_pin_init!(AsahiData { - gpu, - pdev: pdev.into(), - resources: res, - }); - - // SAFETY: This is the sole initializer; driver data has not been accessed yet. - unsafe { drm_neo.init_data(data)? }; - - (*drm_neo).gpu.init()?; - - drm_neo::driver::Registration::new_foreign_owned(&drm_neo, pdev.as_ref(), 0)?; - - // Register the sysfs file on the platform device. Must happen after - // the DRM device is registered so the device is fully bound. - let raw_dev = pdev.as_ref().as_raw(); - let stats = (*drm_neo).gpu.stats_snapshot(); - let snapshot = stats.as_ref().map_or(core::ptr::null(), Arc::as_ptr); - let raw_stats = crate::sysfs_exports::register( - raw_dev, - snapshot, - stats.is_some() && *module_parameters::stats_export.value() != 0, - )?; - - Ok(Self { drm_neo, raw_dev, raw_stats, stats }) - } -} - -impl AsahiDriver { - fn probe_g17(pdev: &platform::Device, cfg: &'static g17::Config) -> Result { - // SAFETY: The GPU performs DMA through a UAT whose output width is part of the SoC config. - unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; - let res = regs::Resources::new(pdev)?; - // SAFETY: GPU construction cannot access driver data before init_data(). - // Failed construction releases the allocation without dropping uninitialized data. - let drm_neo: ARef = unsafe { drm_neo::device::Device::new_uninit(pdev.as_ref())? }; - let gpu = g17::Gpu::new(pdev, &drm_neo, cfg, res.clone())?; - let data_gpu = gpu.clone() as Arc; - let data = try_pin_init!(AsahiData { - gpu: data_gpu, - pdev: pdev.into(), - resources: res, - }); - // SAFETY: This is the sole initializer; no driver-data users have been published. - if let Err(error) = unsafe { drm_neo.init_data(data) } { - gpu.shutdown(); - return Err(error); - } - if let Err(error) = drm_neo::driver::Registration::new_foreign_owned(&drm_neo, pdev.as_ref(), 0) { - gpu.shutdown(); - return Err(error); - } - // G17's firmware channel format has no supported stats decoder yet. - let raw_dev = pdev.as_ref().as_raw(); - let raw_stats = match crate::sysfs_exports::register( - raw_dev, core::ptr::null(), false, - ) { - Ok(handle) => handle, - Err(error) => { - gpu.shutdown(); - return Err(error); - } - }; - Ok(Self { drm_neo, raw_dev, raw_stats, stats: None }) - } -} - -impl Drop for AsahiDriver { - fn drop(&mut self) { - crate::sysfs_exports::unregister(self.raw_dev, self.raw_stats); - drop(self.stats.take()); - if let Some(gpu) = (*self.drm_neo).gpu.as_any().downcast_ref::() { - // Also covers failure to allocate platform driver data after the probe body returned. - gpu.shutdown(); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/event.rs b/drivers/gpu/drm/asahi-neo/event.rs deleted file mode 100644 index edd7d701e665cd..00000000000000 --- a/drivers/gpu/drm/asahi-neo/event.rs +++ /dev/null @@ -1,257 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU event manager -//! -//! The GPU firmware manages work completion by using event objects (Apple calls them "stamps"), -//! which are monotonically incrementing counters. There are a fixed number of objects, and -//! they are managed with a `SlotAllocator`. -//! -//! This module manages the set of available events and lets users compute expected values. -//! It also manages signaling owners when the GPU firmware reports that an event fired. - -use crate::debug::*; -use crate::fw::types::*; -use crate::{ - gpu, - slotalloc, - workqueue, // -}; -use core::cmp; -use core::sync::atomic::Ordering; -use kernel::prelude::*; -use kernel::sync::Arc; -use kernel::{ - c_str, - static_lock_class, // -}; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Event; - -/// Number of events managed by the firmware. -const NUM_EVENTS: u32 = 128; - -/// Inner data associated with a given event slot. -pub(crate) struct EventInner { - /// CPU pointer to the driver notification event stamp - stamp: *const AtomicU32, - /// GPU pointer to the driver notification event stamp - gpu_stamp: GpuWeakPointer, - /// GPU pointer to the firmware-internal event stamp - gpu_fw_stamp: GpuWeakPointer, -} - -/// SAFETY: The event slots are safe to send across threads. -unsafe impl Send for EventInner {} - -/// Alias for an event token, which allows requesting the same event. -pub(crate) type Token = slotalloc::SlotToken; -/// Alias for an allocated `Event` that has a slot. -pub(crate) type Event = slotalloc::Guard; - -/// Represents a given stamp value for an event. -#[derive(Eq, PartialEq, Copy, Clone, Debug)] -#[repr(transparent)] -pub(crate) struct EventValue(u32); - -impl EventValue { - /// Returns the `EventValue` that succeeds this one. - pub(crate) fn next(&self) -> EventValue { - EventValue(self.0.wrapping_add(0x100)) - } - - /// Increments this `EventValue` in place. - pub(crate) fn increment(&mut self) { - self.0 = self.0.wrapping_add(0x100); - } - - /* Not used - /// Increments this `EventValue` in place by a certain count. - pub(crate) fn add(&mut self, val: u32) { - self.0 = self - .0 - .wrapping_add(val.checked_mul(0x100).expect("Adding too many events")); - } - */ - - /// Increments this `EventValue` in place by a certain count. - pub(crate) fn sub(&mut self, val: u32) { - self.0 = self - .0 - .wrapping_sub(val.checked_mul(0x100).expect("Subtracting too many events")); - } - - /// Computes the delta between this event and another event. - pub(crate) fn delta(&self, other: &EventValue) -> i32 { - (self.0.wrapping_sub(other.0) as i32) >> 8 - } -} - -impl PartialOrd for EventValue { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for EventValue { - fn cmp(&self, other: &Self) -> cmp::Ordering { - self.delta(other).cmp(&0) - } -} - -impl EventInner { - /// Returns the GPU pointer to the driver notification stamp - pub(crate) fn stamp_pointer(&self) -> GpuWeakPointer { - self.gpu_stamp - } - - /// Returns the GPU pointer to the firmware internal stamp - pub(crate) fn fw_stamp_pointer(&self) -> GpuWeakPointer { - self.gpu_fw_stamp - } - - /// Fetches the current event value from shared memory - pub(crate) fn current(&self) -> EventValue { - // SAFETY: The pointer is always valid as constructed in - // EventManager below, and outside users cannot construct - // new EventInners, nor move or copy them, and Guards as - // returned by the SlotAllocator hold a reference to the - // SlotAllocator containing the EventManagerInner, which - // keeps the GpuObject the stamp is contained within alive. - EventValue(unsafe { &*self.stamp }.load(Ordering::Acquire)) - } -} - -impl slotalloc::SlotItem for EventInner { - type Data = EventManagerInner; - - fn release(&mut self, data: &mut Self::Data, slot: u32) { - mod_pr_debug!("EventManager: Released slot {}\n", slot); - data.owners[slot as usize] = None; - } -} - -/// Inner data for the event manager, to be protected by the SlotAllocator lock. -pub(crate) struct EventManagerInner { - stamps: GpuArray, - fw_stamps: GpuArray, - // Note: Use dyn to avoid having to version this entire module. - owners: KVec>>, -} - -/// Top-level EventManager object. -pub(crate) struct EventManager { - alloc: slotalloc::SlotAllocator, -} - -impl EventManager { - /// Create a new EventManager. - #[inline(never)] - pub(crate) fn new(alloc: &mut gpu::KernelAllocators) -> Result { - let mut owners = KVec::new(); - for _i in 0..(NUM_EVENTS as usize) { - owners.push(None, GFP_KERNEL)?; - } - let inner = EventManagerInner { - stamps: alloc.shared.array_empty(NUM_EVENTS as usize)?, - fw_stamps: alloc.private.array_empty(NUM_EVENTS as usize)?, - owners, - }; - - for slot in 0..NUM_EVENTS { - inner.stamps[slot as usize] - .0 - .store(slot << 24, Ordering::Relaxed); - } - - Ok(EventManager { - alloc: slotalloc::SlotAllocator::new( - NUM_EVENTS, - inner, - |inner: &mut EventManagerInner, slot| { - Some(EventInner { - stamp: &inner.stamps[slot as usize].0, - gpu_stamp: inner.stamps.weak_item_pointer(slot as usize), - gpu_fw_stamp: inner.fw_stamps.weak_item_pointer(slot as usize), - }) - }, - c_str!("EventManager::SlotAllocator"), - static_lock_class!(), - static_lock_class!(), - )?, - }) - } - - /// Gets a free `Event`, optionally trying to reuse the last one allocated by this caller. - pub(crate) fn get( - &self, - token: Option, - owner: Arc, - ) -> Result { - let ev = self.alloc.get_inner(token, |inner, ev| { - mod_pr_debug!( - "EventManager: Registered owner {:p} on slot {}\n", - &*owner, - ev.slot() - ); - inner.owners[ev.slot() as usize] = Some(owner); - Ok(()) - })?; - Ok(ev) - } - - /// Signals an event by slot, indicating completion (of one or more commands). - pub(crate) fn signal(&self, slot: u32) { - match self - .alloc - .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) - { - Some(owner) => { - owner.signal(); - } - None => { - mod_pr_debug!("EventManager: Received event for empty slot {}\n", slot); - } - } - } - - /// Marks the owner of an event as having lost its work due to a GPU error. - pub(crate) fn mark_error(&self, slot: u32, wait_value: u32, error: workqueue::WorkError) { - match self - .alloc - .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) - { - Some(owner) => { - owner.mark_error(EventValue(wait_value), error); - } - None => { - pr_err!("Received error for empty slot {}\n", slot); - } - } - } - - /// Returns a reference to the workqueue owning an event. - pub(crate) fn get_owner( - &self, - slot: u32, - ) -> Option> { - self.alloc - .with_inner(|inner| inner.owners[slot as usize].as_ref().cloned()) - } - - /// Fail all commands, used when the GPU crashes. - pub(crate) fn fail_all(&self, error: workqueue::WorkError) { - let mut owners: KVec> = KVec::new(); - - self.alloc.with_inner(|inner| { - for wq in inner.owners.iter().filter_map(|o| o.as_ref()).cloned() { - if owners.push(wq, GFP_KERNEL).is_err() { - pr_err!("Failed to signal failure to WorkQueue\n"); - } - } - }); - - for wq in owners { - wq.fail_all(error); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/file.rs b/drivers/gpu/drm/asahi-neo/file.rs deleted file mode 100644 index 2634e9535d9e19..00000000000000 --- a/drivers/gpu/drm/asahi-neo/file.rs +++ /dev/null @@ -1,1332 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#![allow(clippy::unusual_byte_groupings)] - -//! File implementation, which represents a single DRM client. -//! -//! This is in charge of managing the resources associated with one GPU client, including an -//! arbitrary number of submission queues and Vm objects, and reporting hardware/driver -//! information to userspace and accepting submissions. - -use crate::debug::*; -use crate::driver::AsahiDevice; -use crate::{ - alloc, - buffer, - driver, - gem, - mmu, - module_parameters, - queue, - util::{ - align, - align_down, - gcd, - AnyBitPattern, - RangeExt, - Reader, // - }, // -}; -use core::mem::MaybeUninit; -use core::ops::Deref; -use core::ops::Range; -use core::ptr::addr_of_mut; -use kernel::bindings; -use kernel::dma_fence::RawDmaFence; -use kernel::drm_neo::gem::BaseObject; -use kernel::error::code::*; -use kernel::new_mutex; -use kernel::prelude::*; -use kernel::sync::{ - Arc, - Mutex, // -}; -use kernel::time::NSEC_PER_SEC; -use kernel::uaccess::{ - UserPtr, - UserSlice, // -}; -use kernel::{ - dma_fence, - drm_neo, - uapi, - xarray, // -}; - -const DEBUG_CLASS: DebugFlags = DebugFlags::File; - -pub(crate) const MAX_COMMANDS_PER_SUBMISSION: u32 = 64; - -/// A client instance of an `mmu::Vm` address space. -struct Vm { - shrinker: Option>, - ualloc: Arc>, - ualloc_priv: Arc>, - vm: mmu::Vm, - user_range: Range, - kernel_range: Range, - _dummy_mapping: mmu::KernelMapping, -} - -impl Drop for Vm { - fn drop(&mut self) { - self.shrinker = None; - self.vm.close_idle_reclaim(); - if let Some(status) = self.vm.status() { - // Retained graphs may keep this status alive; the client's mirror - // object must not outlive the client's VM handle. - status.release_mirror(); - } - if self.vm.status().is_some() { - // Tracked jobs can outlive their file's VM handle. Their last guard performs - // these unmaps once no accepted work can still dereference the mappings. - if let Err(error) = self - .vm - .unmap_user_ranges(self.user_range.clone(), self.kernel_range.clone()) - { - pr_err!("Vm::Drop: deferred user unmap failed: {:?}\n", error); - } - self.vm.bo_deferred_cleanup(); - return; - } - // When the user Vm is dropped, unmap everything in the user range - let left_range = self.user_range.start..self.kernel_range.start; - let right_range = self.kernel_range.end..self.user_range.end; - - if !left_range.is_empty() - && self - .vm - .unmap_range(left_range.start, left_range.range()) - .is_err() - { - pr_err!("Vm::Drop: vm.unmap_range() failed\n"); - } - if !right_range.is_empty() - && self - .vm - .unmap_range(right_range.start, right_range.range()) - .is_err() - { - pr_err!("Vm::Drop: vm.unmap_range() failed\n"); - } - - self.vm.bo_deferred_cleanup(); - } -} - -/// Sync object from userspace. -pub(crate) struct SyncItem { - pub(crate) syncobj: drm_neo::syncobj::SyncObj, - pub(crate) fence: Option, - pub(crate) chain_fence: Option, - pub(crate) timeline_value: u64, -} - -impl SyncItem { - fn parse_one(file: &DrmFile, data: uapi::drm_asahi_neo_sync, out: bool) -> Result { - match data.sync_type { - uapi::drm_asahi_neo_sync_type_DRM_ASAHI_NEO_SYNC_SYNCOBJ => { - if data.timeline_value != 0 { - cls_pr_debug!(Errors, "Non-timeline sync object with a nonzero value\n"); - return Err(EINVAL); - } - let syncobj = drm_neo::syncobj::SyncObj::lookup_handle(file, data.handle)?; - - Ok(SyncItem { - fence: if out { - None - } else { - Some(syncobj.fence_get().ok_or_else(|| { - cls_pr_debug!(Errors, "Failed to get fence from sync object\n"); - EINVAL - })?) - }, - syncobj, - chain_fence: None, - timeline_value: data.timeline_value, - }) - } - uapi::drm_asahi_neo_sync_type_DRM_ASAHI_NEO_SYNC_TIMELINE_SYNCOBJ => { - let syncobj = drm_neo::syncobj::SyncObj::lookup_handle(file, data.handle)?; - let fence = if out { - None - } else { - syncobj - .fence_get() - .ok_or_else(|| { - cls_pr_debug!( - Errors, - "Failed to get fence from timeline sync object\n" - ); - EINVAL - })? - .chain_find_seqno(data.timeline_value)? - }; - - Ok(SyncItem { - fence, - syncobj, - chain_fence: if out { - Some(dma_fence::FenceChain::new()?) - } else { - None - }, - timeline_value: data.timeline_value, - }) - } - _ => { - cls_pr_debug!(Errors, "Invalid sync type {}\n", data.sync_type); - Err(EINVAL) - } - } - } - - fn parse_array( - file: &DrmFile, - ptr: u64, - in_count: u32, - out_count: u32, - ) -> Result> { - let count = in_count.checked_add(out_count).ok_or(EOVERFLOW)?; - - const STRIDE: usize = core::mem::size_of::(); - let size = STRIDE.checked_mul(count as usize).ok_or(EOVERFLOW)?; - let mut vec = KVec::with_capacity(count as usize, GFP_KERNEL)?; - - // SAFETY: We only read this once, so there are no TOCTOU issues. - let mut reader = UserSlice::new(UserPtr::from_addr(ptr as _), size).reader(); - - for i in 0..count { - let mut sync: MaybeUninit = MaybeUninit::uninit(); - - // SAFETY: The size of `sync` is STRIDE - reader.read_raw(unsafe { - core::slice::from_raw_parts_mut(sync.as_mut_ptr() as *mut MaybeUninit, STRIDE) - })?; - - // SAFETY: All bit patterns in the struct are valid - let sync = unsafe { sync.assume_init() }; - - vec.push(SyncItem::parse_one(file, sync, i >= in_count)?, GFP_KERNEL)?; - } - - Ok(vec) - } -} - -#[derive(Clone)] -pub(crate) enum Object { - TimestampBuffer(Arc), -} - -/// State associated with a client. -// #[pin_data] -pub(crate) struct File { - id: u64, - /// Available VM range for the user - user_range: Range, - // #[pin] - vms: xarray::XArray>, - // #[pin] - queues: xarray::XArray>>>, - // #[pin] - objects: xarray::XArray>, -} - -/// Convenience type alias for our DRM `File` type. -pub(crate) type DrmFile = drm_neo::File; - -/// Minimum reserved AS for kernel mappings -const VM_KERNEL_MIN_SIZE: u64 = 0x20000000; - -impl drm_neo::file::DriverFile for File { - type Driver = driver::AsahiDriver; - - /// Create a new `File` instance for a fresh client. - fn open(device: &AsahiDevice) -> Result>> { - debug::update_debug_flags(); - - let gpu = &device.gpu; - let id = gpu.ids().file.next(); - let user_range = gpu.uat_geometry().user_usable_range(); - - mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); - Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) - } - - fn as_raw(&self) -> *mut bindings::drm_file { - todo!() - } -} - -// SAFETY: All bit patterns are valid by construction. -unsafe impl AnyBitPattern for uapi::drm_asahi_neo_gem_bind_op {} - -impl File { - fn new(id: u64, user_range: Range) -> impl PinInit { - unsafe { - pin_init::pin_init_from_closure(move |slot: *mut Self| { - let raw_vms = addr_of_mut!((*slot).vms); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_vms)?; - - let raw_queues = addr_of_mut!((*slot).queues); - xarray::XArray::>>>::new( - xarray::AllocKind::Alloc1, - ) - .__pinned_init(raw_queues)?; - - let raw_objects = addr_of_mut!((*slot).objects); - xarray::XArray::>::new(xarray::AllocKind::Alloc1) - .__pinned_init(raw_objects)?; - - (*slot).id = id; - (*slot).user_range = user_range; - Ok(()) - }) - } - } - - fn vms(self: Pin<&Self>) -> Pin<&xarray::XArray>> { - // SAFETY: Structural pinned projection for vms. - // We never move out of this field. - unsafe { self.map_unchecked(|s| &s.vms) } - } - - #[allow(clippy::type_complexity)] - fn queues(self: Pin<&Self>) -> Pin<&xarray::XArray>>>> { - // SAFETY: Structural pinned projection for queues. - // We never move out of this field. - unsafe { self.map_unchecked(|s| &s.queues) } - } - - fn objects(self: Pin<&Self>) -> Pin<&xarray::XArray>> { - // SAFETY: Structural pinned projection for objects. - // We never move out of this field. - unsafe { self.map_unchecked(|s| &s.objects) } - } - - /// IOCTL: get_param: Get a driver parameter value. - pub(crate) fn get_params( - device: &AsahiDevice, - data: &uapi::drm_asahi_neo_get_params, - file: &DrmFile, - ) -> Result { - mod_dev_dbg!(device, "[File {}]: IOCTL: get_params\n", file.inner().id); - - let gpu = &device.gpu; - - match data.param_group { - uapi::DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS if gpu.supports_vm_status() => { - return Self::get_vm_status(data, file); - } - uapi::DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS_MIRROR if gpu.supports_vm_status() => { - return Self::register_vm_status_mirror(data, file); - } - uapi::DRM_ASAHI_NEO_PARAM_GROUP_QUEUE_LIMITS => { - if data.pad != 0 - || data.size < core::mem::size_of::() as u64 - { - return Err(EINVAL); - } - let limits = gpu.queue_limits().ok_or(EINVAL)?; - let words = [limits.max_queues, limits.max_in_flight_per_queue, 0, 0]; - UserSlice::new( - UserPtr::from_addr(data.pointer as _), - core::mem::size_of_val(&words), - ) - .writer() - .write(&words)?; - return Ok(0); - } - _ => {} - } - - if data.param_group != 0 || data.pad != 0 { - cls_pr_debug!(Errors, "get_params: Invalid arguments\n"); - return Err(EINVAL); - } - - if gpu.is_crashed() { - return Err(ENODEV); - } - - let mut params = uapi::drm_asahi_neo_params_global { - features: 0, - - vm_start: file.inner().user_range.start, - vm_end: file.inner().user_range.end, - vm_kernel_min_size: VM_KERNEL_MIN_SIZE, - - max_commands_per_submission: MAX_COMMANDS_PER_SUBMISSION, - max_attachments: crate::microseq::MAX_ATTACHMENTS as u32, - - command_timestamp_frequency_hz: 1_000_000_000, // User timestamps always in nanoseconds - - ..pin_init::zeroed() - }; - - gpu.get_params(&mut params)?; - - if gpu.supports_vm_status() { - params.features |= uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_VM_STATUS as u64 - | uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_VM_STATUS_MIRROR as u64; - } - if gpu.queue_limits().is_some() { - params.features |= uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_INDEPENDENT_QUEUES as u64; - } - - if *module_parameters::fault_control.value() == 0xb { - params.features |= uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_SOFT_FAULTS as u64; - } - - let size = core::mem::size_of::().min(data.size.try_into()?); - - // SAFETY: We only write to this userptr once, so there are no TOCTOU issues. - let mut params_writer = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size).writer(); - - // SAFETY: `size` is at most the sizeof of `params` - params_writer.write_slice(unsafe { - core::slice::from_raw_parts(¶ms as *const _ as *const u8, size) - })?; - - Ok(0) - } - - /// Query only file-owned VM state; this does not start recovery or wait for work. - fn get_vm_status(data: &uapi::drm_asahi_neo_get_params, file: &DrmFile) -> Result { - let size = core::mem::size_of::(); - if data.pad != 0 || data.size < size as u64 { - return Err(EINVAL); - } - let user = UserPtr::from_addr(data.pointer as _); - let [vm_id, flags, _, pad] = UserSlice::new(user, size).reader().read::<[u32; 4]>()?; - if flags != 0 || pad != 0 { - return Err(EINVAL); - } - let status = Self::vm_status(file, vm_id)?; - UserSlice::new(user, size) - .writer() - .write(&[vm_id, 0, status.reported_error() as u32, 0])?; - Ok(0) - } - - /// Register a file-owned GEM word as the mirror of a file-owned VM's status. - fn register_vm_status_mirror(data: &uapi::drm_asahi_neo_get_params, file: &DrmFile) -> Result { - let size = core::mem::size_of::(); - if data.pad != 0 || data.size < size as u64 { - return Err(EINVAL); - } - let [vm_id, handle, offset, flags] = - UserSlice::new(UserPtr::from_addr(data.pointer as _), size) - .reader() - .read::<[u32; 4]>()?; - if flags != 0 || handle == 0 || offset % core::mem::align_of::() as u32 != 0 { - return Err(EINVAL); - } - let status = Self::vm_status(file, vm_id)?; - let owner = gem::ObjectRef::new(gem::Object::lookup_handle(file, handle)?); - status.attach_mirror(owner, offset as usize)?; - Ok(0) - } - - fn vm_status(file: &DrmFile, vm_id: u32) -> Result> { - let vms = file.inner().vms(); - let guard = vms.lock(); - Ok(guard - .get(vm_id as usize) - .ok_or(ENOENT)? - .vm - .status() - .ok_or(EINVAL)? - .clone()) - } - - /// IOCTL: vm_create: Create a new `Vm`. - pub(crate) fn vm_create( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_vm_create, - file: &DrmFile, - ) -> Result { - let kernel_range = data.kernel_start..data.kernel_end; - let user_range = file.inner().user_range.clone(); - - // Validate requested kernel range - if !user_range.is_superset(kernel_range.clone()) - || kernel_range.range() < VM_KERNEL_MIN_SIZE - || kernel_range.start & (mmu::UAT_PGMSK as u64) != 0 - || kernel_range.end & (mmu::UAT_PGMSK as u64) != 0 - { - cls_pr_debug!(Errors, "vm_create: Invalid kernel range\n"); - return Err(EINVAL); - } - - // Align to buffer::PAGE_SIZE so the allocators are happy - let kernel_range = align(kernel_range.start, buffer::PAGE_SIZE as u64) - ..align_down(kernel_range.end, buffer::PAGE_SIZE as u64); - - let kernel_half_size = align_down(kernel_range.range() >> 1, buffer::PAGE_SIZE as u64); - let kernel_gpu_range = kernel_range.start..(kernel_range.start + kernel_half_size); - let kernel_gpufw_range = kernel_gpu_range.end..kernel_range.end; - - let gpu = &device.gpu; - let file_id = file.inner().id; - let vm = gpu.new_vm(kernel_range.clone())?; - - let vm_xa = file.inner().vms(); - let resv = vm_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; - let id: u32 = resv.index().try_into()?; - - mod_dev_dbg!(device, "[File {} VM {}]: VM Create\n", file_id, id); - mod_dev_dbg!( - device, - "[File {} VM {}]: Creating allocators\n", - file_id, - id - ); - let ualloc = Arc::pin_init( - new_mutex!(alloc::DefaultAllocator::new( - device, - &vm, - kernel_gpu_range, - buffer::PAGE_SIZE, - mmu::PROT_GPU_SHARED_RW, - 512 * 1024, - true, - fmt!("File {} VM {} GPU Shared", file_id, id), - false, - )?), - GFP_KERNEL, - )?; - let ualloc_priv = Arc::pin_init( - new_mutex!(alloc::DefaultAllocator::new( - device, - &vm, - kernel_gpufw_range, - buffer::PAGE_SIZE, - mmu::PROT_GPU_FW_PRIV_RW, - 64 * 1024, - true, - fmt!("File {} VM {} GPU FW Private", file_id, id), - false, - )?), - GFP_KERNEL, - )?; - - mod_dev_dbg!( - device, - "[File {} VM {}]: Creating dummy object\n", - file_id, - id - ); - let mut dummy_obj = gem::new_kernel_object(device, 0x4000)?; - dummy_obj.vmap()?.memset(0); - let dummy_mapping = dummy_obj.map_at( - &vm, - gpu.uat_geometry().unk_page(), - mmu::PROT_GPU_SHARED_RW, - true, - )?; - - mod_dev_dbg!(device, "[File {} VM {}]: VM created\n", file_id, id); - resv.fill(KBox::new( - Vm { - shrinker: if vm.residency_reclaim_enabled() { - Some(mmu::VmShrinker::new(&vm)?) - } else { - None - }, - ualloc, - ualloc_priv, - vm, - user_range, - kernel_range, - _dummy_mapping: dummy_mapping, - }, - GFP_KERNEL, - )?)?; - - data.vm_id = id; - - Ok(0) - } - - /// IOCTL: vm_destroy: Destroy a `Vm`. - pub(crate) fn vm_destroy( - _device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_vm_destroy, - file: &DrmFile, - ) -> Result { - let vm = file.inner().vms().remove(data.vm_id as usize); - if vm.is_none() { - Err(ENOENT) - } else { - Ok(0) - } - } - - /// IOCTL: gem_create: Create a new GEM object. - pub(crate) fn gem_create( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_gem_create, - file: &DrmFile, - ) -> Result { - mod_dev_dbg!( - device, - "[File {}]: IOCTL: gem_create size={:#x?}\n", - file.inner().id, - data.size - ); - - if (data.flags - & !(uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_WRITEBACK - | uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_VM_PRIVATE)) - != 0 - || (data.flags & uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_VM_PRIVATE == 0 - && data.vm_id != 0) - { - cls_pr_debug!(Errors, "gem_create: Invalid arguments\n"); - return Err(EINVAL); - } - - let resv_gem; - let mut reclaim_cpu = false; - let resv_obj = if data.flags & uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_VM_PRIVATE != 0 { - let vms = file.inner().vms(); - let vms = vms.lock(); - let vm = &vms.get(data.vm_id.try_into()?).ok_or(ENOENT)?.vm; - reclaim_cpu = vm.residency_reclaim_enabled(); - resv_gem = vm.get_resv_obj(); - Some(resv_gem.deref()) - } else { - None - }; - - let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; - - let handle = gem.create_handle(file)?; - data.handle = handle; - - mod_dev_dbg!( - device, - "[File {}]: IOCTL: gem_create size={:#x} handle={:#x?}\n", - file.inner().id, - data.size, - data.handle - ); - - Ok(0) - } - - /// IOCTL: gem_mmap_offset: Assign an mmap offset to a GEM object. - pub(crate) fn gem_mmap_offset( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_gem_mmap_offset, - file: &DrmFile, - ) -> Result { - mod_dev_dbg!( - device, - "[File {}]: IOCTL: gem_mmap_offset handle={:#x?}\n", - file.inner().id, - data.handle - ); - - if data.flags != 0 { - cls_pr_debug!(Errors, "gem_mmap_offset: Unexpected flags\n"); - return Err(EINVAL); - } - - let gem = gem::Object::lookup_handle(file, data.handle)?; - data.offset = gem.create_mmap_offset()?; - Ok(0) - } - - /// IOCTL: gem_madvise: Mark a VM-private object purgeable or needed. - pub(crate) fn gem_madvise( - _device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_gem_madvise, - file: &DrmFile, - ) -> Result { - if data.pad != 0 || data.madv > uapi::drm_asahi_neo_gem_madv_DRM_ASAHI_NEO_GEM_MADV_DONTNEED { - return Err(EINVAL); - } - let gem = gem::Object::lookup_handle(file, data.handle)?; - data.retained = gem::madvise( - &gem, - data.madv == uapi::drm_asahi_neo_gem_madv_DRM_ASAHI_NEO_GEM_MADV_DONTNEED, - )?; - Ok(0) - } - - /// IOCTL: vm_bind: Map or unmap memory into a Vm. - pub(crate) fn vm_bind( - device: &AsahiDevice, - data: &uapi::drm_asahi_neo_vm_bind, - file: &DrmFile, - ) -> Result { - mod_dev_dbg!( - device, - "[File {} VM {}]: IOCTL: vm_bind\n", - file.inner().id, - data.vm_id, - ); - - if data.stride == 0 || data.pad != 0 { - cls_pr_debug!(Errors, "vm_bind: Unexpected headers\n"); - return Err(EINVAL); - } - - if device.gpu.as_any().is::() { - return Self::vm_bind_g17(data, file); - } - - let vm_id = data.vm_id.try_into()?; - - let mut vec = KVec::new(); - let size = usize::try_from(data.stride)? - .checked_mul(usize::try_from(data.num_binds)?) - .ok_or(EOVERFLOW)?; - let reader = UserSlice::new(UserPtr::from_addr(data.userptr as _), size).reader(); - reader.read_all(&mut vec, GFP_KERNEL)?; - let mut reader = Reader::new(&vec); - - for _i in 0..data.num_binds { - let bind: uapi::drm_asahi_neo_gem_bind_op = reader.read_up_to(data.stride as usize)?; - Self::do_gem_bind_unbind(vm_id, &bind, file)?; - } - - Ok(0) - } - - /// Validate all operations and retain their objects before changing a tracked VM. - fn vm_bind_g17(data: &uapi::drm_asahi_neo_vm_bind, file: &DrmFile) -> Result { - let count = usize::try_from(data.num_binds)?; - let stride = usize::try_from(data.stride)?; - let size = stride.checked_mul(count).ok_or(EOVERFLOW)?; - let mut bytes = KVec::new(); - UserSlice::new(UserPtr::from_addr(data.userptr as _), size) - .reader() - .read_all(&mut bytes, GFP_KERNEL)?; - let (vm, user_range, kernel_range) = { - let vms = file.inner().vms(); - let guard = vms.lock(); - let entry = guard.get(data.vm_id as usize).ok_or(ENOENT)?; - ( - entry.vm.clone(), - entry.user_range.clone(), - entry.kernel_range.clone(), - ) - }; - let mut batch = mmu::PreparedUserBindBatch::new(count)?; - let mut unmaps = 0; - // A fresh reader per declared stride also skips extension bytes in longer records. - for bytes in bytes.chunks_exact(stride) { - let op: uapi::drm_asahi_neo_gem_bind_op = Reader::new(bytes).read_up_to(stride)?; - if op.range == 0 || (op.addr | op.range | op.offset) & mmu::UAT_PGMSK as u64 != 0 { - return Err(EINVAL); - } - let range = op.addr..op.addr.checked_add(op.range).ok_or(EINVAL)?; - if !user_range.is_superset(range.clone()) - || kernel_range.overlaps(range.clone()) - || vm.driver_range_overlaps(range.clone()) - { - return Err(EINVAL); - } - if op.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_UNBIND != 0 { - if op.offset != 0 - || op.flags != uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_UNBIND - || op.handle != 0 - { - return Err(EINVAL); - } - batch.push_unmap(vm.prepare_user_unmap(op.addr, op.range)?)?; - unmaps += 1; - continue; - } - vm.wait_for_user_map_admission()?; - if op.flags - & !(uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_READ - | uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_WRITE - | uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_SINGLE_PAGE) - != 0 - { - return Err(EINVAL); - } - let read = op.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_READ != 0; - let write = op.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_WRITE != 0; - let prot = match (read, write) { - (true, true) => mmu::PROT_GPU_SHARED_RW, - (true, false) => mmu::PROT_GPU_SHARED_RO, - (false, true) => mmu::PROT_GPU_SHARED_WO, - (false, false) => return Err(EINVAL), - }; - let single_page = op.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_SINGLE_PAGE != 0; - let object = gem::Object::lookup_handle(file, op.handle)?; - let accessed = if single_page { - mmu::UAT_PGSZ as u64 - } else { - op.range - }; - if op.offset.checked_add(accessed).ok_or(EINVAL)? > u64::try_from(object.size())? { - return Err(EINVAL); - } - vm.validate_context_binding(op.addr, op.range, single_page)?; - batch.push_map(vm.prepare_bind_object( - &object, - op.addr, - op.range, - op.offset, - prot, - single_page, - )?)?; - } - vm.reserve_deferred_user_unmaps(unmaps)?; - vm.commit_prepared_user_bind_batch(&mut batch)?; - vm.bo_deferred_cleanup(); - Ok(0) - } - - pub(crate) fn do_gem_bind_unbind( - vm_id: usize, - data: &uapi::drm_asahi_neo_gem_bind_op, - file: &DrmFile, - ) -> Result { - if data.range == 0 { - return Err(EINVAL); - } - if (data.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_UNBIND) != 0 { - Self::do_gem_unbind(vm_id, data, file) - } else { - Self::do_gem_bind(vm_id, data, file) - } - } - - pub(crate) fn do_gem_bind( - vm_id: usize, - data: &uapi::drm_asahi_neo_gem_bind_op, - file: &DrmFile, - ) -> Result { - if (data.addr | data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { - cls_pr_debug!( - Errors, - "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", - data.addr, - data.range - ); - return Err(EINVAL); // Must be page aligned - } - - if (data.flags - & !(uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_READ - | uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_WRITE - | uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_SINGLE_PAGE)) - != 0 - { - cls_pr_debug!(Errors, "gem_bind: Invalid flags {:#x}\n", data.flags); - return Err(EINVAL); - } - - let single_page = data.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_SINGLE_PAGE != 0; - - let bo = gem::Object::lookup_handle(file, data.handle)?; - - let start = data.addr; - let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; - let range = start..end; - - let bo_accessed_size = if single_page { - mmu::UAT_PGMSK as u64 - } else { - data.range - }; - let end_off = data.offset.checked_add(bo_accessed_size).ok_or(EINVAL)?; - if end_off as usize > bo.size() { - return Err(EINVAL); - } - - if !file.inner().user_range.is_superset(range.clone()) { - cls_pr_debug!( - Errors, - "gem_bind: Invalid map range {:#x}..{:#x} (not contained in user range)\n", - start, - end - ); - return Err(EINVAL); // Invalid map range - } - - let prot = if data.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_READ != 0 { - if data.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_WRITE != 0 { - mmu::PROT_GPU_SHARED_RW - } else { - mmu::PROT_GPU_SHARED_RO - } - } else if data.flags & uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_WRITE != 0 { - mmu::PROT_GPU_SHARED_WO - } else { - cls_pr_debug!( - Errors, - "gem_bind: Must specify read or write (flags: {:#x})\n", - data.flags - ); - return Err(EINVAL); // Must specify one of DRM_ASAHI_NEO_BIND_{READ,WRITE} - }; - - let vms_xa = file.inner().vms(); - let guard = vms_xa.lock(); - let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; - - // Clone it immediately so we aren't holding the XArray lock - let vm = guarded_vm.vm.clone(); - let kernel_range = guarded_vm.kernel_range.clone(); - let _ = guarded_vm; - core::mem::drop(guard); - - if kernel_range.overlaps(range) { - cls_pr_debug!( - Errors, - "gem_bind: Invalid map range {:#x}..{:#x} (intrudes in kernel range)\n", - start, - end - ); - return Err(EINVAL); - } - - vm.bind_object(&bo, data.addr, data.range, data.offset, prot, single_page)?; - - vm.bo_deferred_cleanup(); - - Ok(0) - } - - pub(crate) fn do_gem_unbind( - vm_id: usize, - data: &uapi::drm_asahi_neo_gem_bind_op, - file: &DrmFile, - ) -> Result { - if data.offset != 0 - || data.flags != uapi::drm_asahi_neo_bind_flags_DRM_ASAHI_NEO_BIND_UNBIND - || data.handle != 0 - { - cls_pr_debug!(Errors, "gem_unbind: offset/flags/handle not zero\n"); - return Err(EINVAL); - } - - if (data.addr | data.range) as usize & mmu::UAT_PGMSK != 0 { - cls_pr_debug!( - Errors, - "gem_bind: Addr/range/offset not page aligned: {:#x} {:#x}\n", - data.addr, - data.range - ); - return Err(EINVAL); // Must be page aligned - } - - let start = data.addr; - let end = data.addr.checked_add(data.range).ok_or(EINVAL)?; - let range = start..end; - - if !file.inner().user_range.is_superset(range.clone()) { - cls_pr_debug!( - Errors, - "gem_bind: Invalid unmap range {:#x}..{:#x} (not contained in user range)\n", - start, - end - ); - return Err(EINVAL); // Invalid map range - } - - let vms_xa = file.inner().vms(); - let guard = vms_xa.lock(); - let guarded_vm = guard.get(vm_id).ok_or(ENOENT)?; - - // Clone it immediately so we aren't holding the XArray lock - let vm = guarded_vm.vm.clone(); - let kernel_range = guarded_vm.kernel_range.clone(); - let _ = guarded_vm; - core::mem::drop(guard); - - if kernel_range.overlaps(range.clone()) { - cls_pr_debug!( - Errors, - "gem_bind: Invalid unmap range {:#x}..{:#x} (intrudes in kernel range)\n", - start, - end - ); - return Err(EINVAL); - } - - vm.unmap_range(range.start, range.range())?; - - vm.bo_deferred_cleanup(); - - Ok(0) - } - - pub(crate) fn unbind_gem_object(file: &DrmFile, bo: &gem::Object) -> Result { - // TODO: use iter() - let mut index = 0; - loop { - let vms = file.inner().vms(); - let item = vms.find(index, usize::MAX); - match item { - Some((idx, file_vm)) => { - // Clone since we can't hold the xarray spinlock while - // calling drop_mappings() - let vm = file_vm.borrow().vm.clone(); - core::mem::drop(file_vm); - vm.drop_mappings(bo)?; - if idx == usize::MAX { - break; - } - index = idx + 1; - } - None => break, - } - } - Ok(()) - } - - /// IOCTL: gem_bind_object: Map or unmap a GEM object as a special object. - pub(crate) fn gem_bind_object( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_gem_bind_object, - file: &DrmFile, - ) -> Result { - mod_dev_dbg!( - device, - "[File {} VM {}]: IOCTL: gem_bind_object op={:?} handle={:#x?} flags={:#x?} {:#x?}:{:#x?} object_handle={:#x?}\n", - file.inner().id, - data.vm_id, - data.op, - data.handle, - data.flags, - data.offset, - data.range, - data.object_handle - ); - - if data.pad != 0 { - cls_pr_debug!(Errors, "gem_bind_object: Unexpected pad\n"); - return Err(EINVAL); - } - - if data.vm_id != 0 { - cls_pr_debug!(Errors, "gem_bind_object: Unexpected vm_id\n"); - return Err(EINVAL); - } - - match data.op { - uapi::drm_asahi_neo_bind_object_op_DRM_ASAHI_NEO_BIND_OBJECT_OP_BIND => { - Self::do_gem_bind_object(device, data, file) - } - uapi::drm_asahi_neo_bind_object_op_DRM_ASAHI_NEO_BIND_OBJECT_OP_UNBIND => { - Self::do_gem_unbind_object(device, data, file) - } - _ => { - cls_pr_debug!(Errors, "gem_bind_object: Invalid op {}\n", data.op); - Err(EINVAL) - } - } - } - - pub(crate) fn do_gem_bind_object( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_gem_bind_object, - file: &DrmFile, - ) -> Result { - if (data.range | data.offset) as usize & mmu::UAT_PGMSK != 0 { - cls_pr_debug!( - Errors, - "gem_bind_object: Range/offset not page aligned: {:#x} {:#x}\n", - data.range, - data.offset - ); - return Err(EINVAL); // Must be page aligned - } - - if data.flags != uapi::drm_asahi_neo_bind_object_flags_DRM_ASAHI_NEO_BIND_OBJECT_USAGE_TIMESTAMPS { - cls_pr_debug!(Errors, "gem_bind_object: Invalid flags {:#x}\n", data.flags); - return Err(EINVAL); - } - - let offset = data.offset.try_into()?; - let end_offset = data - .offset - .checked_add(data.range) - .ok_or(EINVAL)? - .try_into()?; - let bo = gem::ObjectRef::new(gem::Object::lookup_handle(file, data.handle)?); - - let mapping = Arc::new( - device.gpu.map_timestamp_buffer(bo, offset..end_offset)?, - GFP_KERNEL, - )?; - let obj = KBox::new(Object::TimestampBuffer(mapping), GFP_KERNEL)?; - let handle = file - .inner() - .objects() - .lock() - .insert_limit(1..=u32::MAX, obj, GFP_KERNEL); - // StoreError owns the rejected object. Convert it only after the - // temporary XArray guard is gone: dropping a mapping may sleep. - let handle = handle? as u64; - - data.object_handle = handle as u32; - Ok(0) - } - - pub(crate) fn do_gem_unbind_object( - _device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_gem_bind_object, - file: &DrmFile, - ) -> Result { - if data.range != 0 || data.offset != 0 { - cls_pr_debug!( - Errors, - "gem_unbind_object: Range/offset not zero: {:#x} {:#x}\n", - data.range, - data.offset - ); - return Err(EINVAL); - } - - if data.flags != 0 { - cls_pr_debug!( - Errors, - "gem_unbind_object: Invalid flags {:#x}\n", - data.flags - ); - return Err(EINVAL); - } - - if data.handle != 0 { - cls_pr_debug!( - Errors, - "gem_unbind_object: Invalid handle {}\n", - data.handle - ); - return Err(EINVAL); - } - - let object = file.inner().objects().remove(data.object_handle as usize); - if object.is_none() { - Err(ENOENT) - } else { - Ok(0) - } - } - - /// IOCTL: queue_create: Create a new command submission queue of a given type. - pub(crate) fn queue_create( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_queue_create, - file: &DrmFile, - ) -> Result { - let file_id = file.inner().id; - - mod_dev_dbg!( - device, - "[File {} VM {}]: Creating queue prio={:?} flags={:#x?}\n", - file_id, - data.vm_id, - data.priority, - data.flags, - ); - - if data.flags != 0 || data.priority > uapi::drm_asahi_neo_priority_DRM_ASAHI_NEO_PRIORITY_REALTIME { - cls_pr_debug!(Errors, "queue_create: Invalid arguments\n"); - return Err(EINVAL); - } - - // TODO: Allow with CAP_SYS_NICE - if data.priority >= uapi::drm_asahi_neo_priority_DRM_ASAHI_NEO_PRIORITY_HIGH { - cls_pr_debug!(Errors, "queue_create: Invalid priority\n"); - return Err(EINVAL); - } - - let queues_xa = file.inner().queues(); - let resv = queues_xa.lock().reserve_limit(1..=u32::MAX, GFP_KERNEL)?; - let vms_xa = file.inner().vms(); - let guard = vms_xa.lock(); - let file_vm = guard.get(data.vm_id.try_into()?).ok_or(ENOENT)?; - let vm = file_vm.vm.clone(); - let ualloc = file_vm.ualloc.clone(); - let ualloc_priv = file_vm.ualloc_priv.clone(); - // Drop the vms lock eagerly - let _ = file_vm; - core::mem::drop(guard); - - let queue = device.gpu.new_queue( - vm, - ualloc, - ualloc_priv, - // TODO: Plumb deeper the enum - uapi::drm_asahi_neo_priority_DRM_ASAHI_NEO_PRIORITY_REALTIME - data.priority, - data.usc_exec_base, - )?; - - data.queue_id = resv.index().try_into()?; - resv.fill(Arc::pin_init(new_mutex!(queue), GFP_KERNEL)?)?; - - Ok(0) - } - - /// IOCTL: queue_destroy: Destroy a command submission queue. - pub(crate) fn queue_destroy( - _device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_queue_destroy, - file: &DrmFile, - ) -> Result { - // grab the queue so the xarray spinlock is dropped first - let queue = file.inner().queues().remove(data.queue_id as usize); - if queue.is_none() { - Err(ENOENT) - } else { - Ok(0) - } - } - - /// IOCTL: submit: Submit GPU work to a command submission queue. - pub(crate) fn submit( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_submit, - file: &DrmFile, - ) -> Result { - debug::update_debug_flags(); - - if data.flags != 0 || data.pad != 0 { - cls_pr_debug!(Errors, "submit: Invalid arguments\n"); - return Err(EINVAL); - } - - let gpu = &device.gpu; - if let Some(manager) = gpu.manager() { - manager.update_globals(); - } - - // Upgrade to Arc to drop the XArray lock early - let queue: Arc>> = file - .inner() - .queues() - .lock() - .get(data.queue_id.try_into()?) - .ok_or(ENOENT)? - .into(); - - let id = gpu.ids().submission.next(); - mod_dev_dbg!( - device, - "[File {} Queue {}]: IOCTL: submit (submission ID: {})\n", - file.inner().id, - data.queue_id, - id - ); - - mod_dev_dbg!( - device, - "[File {} Queue {}]: IOCTL: submit({}): Parsing syncs\n", - file.inner().id, - data.queue_id, - id - ); - let mut vec = KVec::new(); - let copy_commands = |bytes: &mut KVec| { - UserSlice::new( - UserPtr::from_addr(data.cmdbuf as _), - data.cmdbuf_size as usize, - ) - .reader() - .read_all(bytes, GFP_KERNEL) - }; - let guarded = gpu.as_any().is::(); - // Once G17 output syncobjs are resolved, its queue must install a fence - // on every failure. Take the fallible userspace snapshot before that point. - if guarded { - copy_commands(&mut vec)?; - } - let syncs = - SyncItem::parse_array(file, data.syncs, data.in_sync_count, data.out_sync_count)?; - - mod_dev_dbg!( - device, - "[File {} Queue {}]: IOCTL: submit({}): Parsing commands\n", - file.inner().id, - data.queue_id, - id - ); - - // Copy the command buffer into the kernel. Because we need to iterate - // the command buffer twice, we do this in one big copy_from_user to - // avoid TOCTOU issues. - if !guarded { - copy_commands(&mut vec)?; - } - - let objects = file.inner().objects(); - let ret = queue - .lock() - .submit(id, syncs, data.in_sync_count as usize, &vec, objects); - - if ret.is_ok() { - if let Some((util_min, duration_ms)) = gpu.syncobj_wait_hint() { - file.set_syncobj_wait_hint(util_min, duration_ms); - } - } - - match ret { - Err(ERESTARTSYS) => Err(ERESTARTSYS), - Err(e) => { - dev_info!( - device.as_ref(), - "[File {} Queue {}]: IOCTL: submit failed! (submission ID: {} err: {:?})\n", - file.inner().id, - data.queue_id, - id, - e - ); - Err(e) - } - Ok(()) => Ok(0), - } - } - - /// IOCTL: get_time: Get the current GPU timer value. - pub(crate) fn get_time( - device: &AsahiDevice, - data: &mut uapi::drm_asahi_neo_get_time, - _file: &DrmFile, - ) -> Result { - if data.flags != 0 { - cls_pr_debug!(Errors, "get_time: Unexpected flags\n"); - return Err(EINVAL); - } - - // TODO: Do this on device-init for perf. - let gpu = &device.gpu; - let frequency_hz = gpu.base_clock_hz() as u64; - let ts_gcd = gcd(frequency_hz, NSEC_PER_SEC as u64); - - let num = (NSEC_PER_SEC as u64) / ts_gcd; - let den = frequency_hz / ts_gcd; - - let raw: u64; - - // SAFETY: Assembly only loads the timer - unsafe { - core::arch::asm!( - "mrs {x}, CNTPCT_EL0", - x = out(reg) raw - ); - } - - data.gpu_timestamp = (raw * num) / den; - - Ok(0) - } -} - -impl Drop for File { - fn drop(&mut self) { - mod_pr_debug!("[File {}]: Closing...\n", self.id); - } -} diff --git a/drivers/gpu/drm/asahi-neo/float.rs b/drivers/gpu/drm/asahi-neo/float.rs deleted file mode 100644 index 46c05699d77225..00000000000000 --- a/drivers/gpu/drm/asahi-neo/float.rs +++ /dev/null @@ -1,388 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Basic soft floating-point support -//! -//! The GPU firmware requires a large number of power-related configuration values, many of which -//! are IEEE 754 32-bit floating point values. These values change not only between GPU/SoC -//! variants, but also between specific hardware platforms using these SoCs, so they must be -//! derived from device tree properties. There are many redundant values computed from the same -//! inputs with simple add/sub/mul/div calculations, plus a few values that are actually specific -//! to each individual device depending on its binning and fused voltage configuration, so it -//! doesn't make sense to store the final values to be passed to the firmware in the device tree. -//! -//! Therefore, we need a way to perform floating-point calculations in the kernel. -//! -//! Using the actual FPU from kernel mode is asking for trouble, since there is no way to bound -//! the execution of FPU instructions to a controlled section of code without outright putting it -//! in its own compilation unit, which is quite painful for Rust. Since these calculations only -//! have to happen at initialization time and there is no need for performance, let's use a simple -//! software float implementation instead. -//! -//! This implementation makes no attempt to be fully IEEE754 compliant, but it's good enough and -//! gives bit-identical results to macOS in the vast majority of cases, with one or two exceptions -//! related to slightly non-compliant rounding. - -use core::ops; -use kernel::{ - of, - prelude::*, // -}; - -/// An IEEE754-compatible floating point number implemented in software. -#[derive(Default, Debug, Copy, Clone)] -#[repr(transparent)] -pub(crate) struct F32(u32); - -// SAFETY: F32 is a transparent repr of `u32` and therefore zeroable -unsafe impl Zeroable for F32 {} - -#[derive(Default, Debug, Copy, Clone)] -struct F32U { - sign: bool, - exp: i32, - frac: i64, -} - -impl F32 { - /// Convert a raw 32-bit representation into an F32 - pub(crate) const fn from_bits(u: u32) -> F32 { - F32(u) - } - - // Convert a `f32` value into an F32 - // - // This must ONLY be used in const context. Use the `f32!{}` macro to do it safely. - #[doc(hidden)] - pub(crate) const fn from_f32(v: f32) -> F32 { - F32(v.to_bits()) - } - - // Convert an F32 into a `f32` value - // - // For testing only. - #[doc(hidden)] - #[cfg(test)] - pub(crate) fn to_f32(self) -> f32 { - f32::from_bits(self.0) - } - - const fn unpack(&self) -> F32U { - F32U { - sign: self.0 & (1 << 31) != 0, - exp: ((self.0 >> 23) & 0xff) as i32 - 127, - frac: (((self.0 & 0x7fffff) | 0x800000) as i64) << 9, - } - .norm() - } -} - -/// Safely construct an `F32` out of a constant floating-point value. -/// -/// This ensures that the conversion happens in const context, so no floating point operations are -/// emitted. -#[macro_export] -macro_rules! f32 { - ([$($val:expr),*]) => {{ - [$(f32!($val)),*] - }}; - ($val:expr) => {{ - const _K: $crate::float::F32 = $crate::float::F32::from_f32($val); - _K - }}; -} - -impl ops::Neg for F32 { - type Output = F32; - - fn neg(self) -> F32 { - F32(self.0 ^ (1 << 31)) - } -} - -impl ops::Add for F32 { - type Output = F32; - - fn add(self, rhs: F32) -> F32 { - self.unpack().add(rhs.unpack()).pack() - } -} - -impl ops::Sub for F32 { - type Output = F32; - - fn sub(self, rhs: F32) -> F32 { - self.unpack().add((-rhs).unpack()).pack() - } -} - -impl ops::Mul for F32 { - type Output = F32; - - fn mul(self, rhs: F32) -> F32 { - self.unpack().mul(rhs.unpack()).pack() - } -} - -impl ops::Div for F32 { - type Output = F32; - - fn div(self, rhs: F32) -> F32 { - self.unpack().div(rhs.unpack()).pack() - } -} - -macro_rules! from_ints { - ($u:ty, $i:ty) => { - impl From<$i> for F32 { - fn from(v: $i) -> F32 { - F32U::from_i64(v as i64).pack() - } - } - impl From<$u> for F32 { - fn from(v: $u) -> F32 { - F32U::from_u64(v as u64).pack() - } - } - }; -} - -from_ints!(u8, i8); -from_ints!(u16, i16); -from_ints!(u32, i32); -from_ints!(u64, i64); - -impl F32U { - const INFINITY: F32U = f32!(f32::INFINITY).unpack(); - const NEG_INFINITY: F32U = f32!(f32::NEG_INFINITY).unpack(); - - fn from_i64(v: i64) -> F32U { - F32U { - sign: v < 0, - exp: 32, - frac: v.abs(), - } - .norm() - } - - fn from_u64(mut v: u64) -> F32U { - let mut exp = 32; - if v >= (1 << 63) { - exp = 31; - v >>= 1; - } - F32U { - sign: false, - exp, - frac: v as i64, - } - .norm() - } - - fn shr(&mut self, shift: i32) { - if shift > 63 { - self.exp = 0; - self.frac = 0; - } else { - self.frac >>= shift; - } - } - - fn align(a: &mut F32U, b: &mut F32U) { - if a.exp > b.exp { - b.shr(a.exp - b.exp); - b.exp = a.exp; - } else { - a.shr(b.exp - a.exp); - a.exp = b.exp; - } - } - - fn mul(self, other: F32U) -> F32U { - F32U { - sign: self.sign != other.sign, - exp: self.exp + other.exp, - frac: ((self.frac >> 8) * (other.frac >> 8)) >> 16, - } - } - - fn div(self, other: F32U) -> F32U { - if other.frac == 0 || self.is_inf() { - if self.sign { - F32U::NEG_INFINITY - } else { - F32U::INFINITY - } - } else { - F32U { - sign: self.sign != other.sign, - exp: self.exp - other.exp, - frac: ((self.frac << 24) / (other.frac >> 8)), - } - } - } - - fn add(mut self, mut other: F32U) -> F32U { - F32U::align(&mut self, &mut other); - if self.sign == other.sign { - self.frac += other.frac; - } else { - self.frac -= other.frac; - } - if self.frac < 0 { - self.sign = !self.sign; - self.frac = -self.frac; - } - self - } - - const fn norm(mut self) -> F32U { - let lz = self.frac.leading_zeros() as i32; - if lz > 31 { - self.frac <<= lz - 31; - self.exp -= lz - 31; - } else if lz < 31 { - self.frac >>= 31 - lz; - self.exp += 31 - lz; - } - - if self.is_zero() { - return F32U { - sign: self.sign, - frac: 0, - exp: 0, - }; - } - self - } - - const fn is_zero(&self) -> bool { - self.frac == 0 || self.exp < -126 - } - - const fn is_inf(&self) -> bool { - self.exp > 127 - } - - const fn pack(mut self) -> F32 { - self = self.norm(); - if !self.is_zero() { - self.frac += 0x100; - self = self.norm(); - } - - if self.is_inf() { - if self.sign { - return f32!(f32::NEG_INFINITY); - } else { - return f32!(f32::INFINITY); - } - } else if self.is_zero() { - if self.sign { - return f32!(-0.0); - } else { - return f32!(0.0); - } - } - - F32(if self.sign { 1u32 << 31 } else { 0u32 } - | ((self.exp + 127) as u32) << 23 - | ((self.frac >> 9) & 0x7fffff) as u32) - } -} - -impl<'a> TryFrom> for F32 { - type Error = Error; - - fn try_from(p: of::Property<'_>) -> core::result::Result { - let bits: u32 = p.try_into()?; - Ok(F32::from_bits(bits)) - } -} - -impl of::PropertyUnit for F32 { - const UNIT_SIZE: usize = 4; - - fn from_bytes(data: &[u8]) -> Result { - Ok(F32::from_bits(::from_bytes(data)?)) - } -} - -// TODO: Make this an actual test and figure out how to make it run. -#[cfg(test)] -mod tests { - #[test] - fn test_all() { - fn add(a: f32, b: f32) { - println!( - "{} + {} = {} {}", - a, - b, - (F32::from_f32(a) + F32::from_f32(b)).to_f32(), - a + b - ); - } - fn sub(a: f32, b: f32) { - println!( - "{} - {} = {} {}", - a, - b, - (F32::from_f32(a) - F32::from_f32(b)).to_f32(), - a - b - ); - } - fn mul(a: f32, b: f32) { - println!( - "{} * {} = {} {}", - a, - b, - (F32::from_f32(a) * F32::from_f32(b)).to_f32(), - a * b - ); - } - fn div(a: f32, b: f32) { - println!( - "{} / {} = {} {}", - a, - b, - (F32::from_f32(a) / F32::from_f32(b)).to_f32(), - a / b - ); - } - - fn test(a: f32, b: f32) { - add(a, b); - sub(a, b); - mul(a, b); - div(a, b); - } - - test(1.123, 7.567); - test(1.123, 1.456); - test(7.567, 1.123); - test(1.123, -7.567); - test(1.123, -1.456); - test(7.567, -1.123); - test(-1.123, -7.567); - test(-1.123, -1.456); - test(-7.567, -1.123); - test(1000.123, 0.001); - test(1000.123, 0.0000001); - test(0.0012, 1000.123); - test(0.0000001, 1000.123); - test(0., 0.); - test(0., 1.); - test(1., 0.); - test(1., 1.); - test(2., f32::INFINITY); - test(2., f32::NEG_INFINITY); - test(f32::INFINITY, 2.); - test(f32::NEG_INFINITY, 2.); - test(f32::NEG_INFINITY, 2.); - test(f32::MAX, 2.); - test(f32::MIN, 2.); - test(f32::MIN_POSITIVE, 2.); - test(2., f32::MAX); - test(2., f32::MIN); - test(2., f32::MIN_POSITIVE); - } -} diff --git a/drivers/gpu/drm/asahi-neo/fw/buffer.rs b/drivers/gpu/drm/asahi-neo/fw/buffer.rs deleted file mode 100644 index 04892ddd06fc0b..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/buffer.rs +++ /dev/null @@ -1,184 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU tiled vertex buffer control firmware structures - -use super::types::*; -use super::workqueue; -use crate::{ - default_zeroed, - no_debug, - trivial_gpustruct, // -}; -use kernel::sync::Arc; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct BlockControl { - pub(crate) total: AtomicU32, - pub(crate) wptr: AtomicU32, - pub(crate) unk: AtomicU32, - pub(crate) pad: Pad<0x34>, - } - default_zeroed!(BlockControl); - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct Counter { - pub(crate) count: AtomicU32, - __pad: Pad<0x3c>, - } - default_zeroed!(Counter); - - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct Stats { - pub(crate) max_pages: AtomicU32, - pub(crate) max_b: AtomicU32, - pub(crate) overflow_count: AtomicU32, - pub(crate) gpu_c: AtomicU32, - pub(crate) __pad0: Pad<0x10>, - pub(crate) reset: AtomicU32, - pub(crate) __pad1: Pad<0x1c>, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct Info<'a> { - pub(crate) gpu_counter: u32, - pub(crate) unk_4: u32, - pub(crate) last_id: i32, - pub(crate) cur_id: i32, - pub(crate) unk_10: u32, - pub(crate) gpu_counter2: u32, - pub(crate) unk_18: u32, - - #[ver(V < V13_0B4 || G >= G14X)] - pub(crate) unk_1c: u32, - - pub(crate) page_list: GpuPointer<'a, &'a [u32]>, - pub(crate) page_list_size: u32, - pub(crate) page_count: AtomicU32, - pub(crate) max_blocks: u32, - pub(crate) block_count: AtomicU32, - pub(crate) unk_38: u32, - pub(crate) block_list: GpuPointer<'a, &'a [u32]>, - pub(crate) block_ctl: GpuPointer<'a, super::BlockControl>, - pub(crate) last_page: AtomicU32, - pub(crate) gpu_page_ptr1: u32, - pub(crate) gpu_page_ptr2: u32, - pub(crate) unk_58: u32, - pub(crate) block_size: u32, - pub(crate) unk_60: U64, - pub(crate) counter: GpuPointer<'a, super::Counter>, - pub(crate) unk_70: u32, - pub(crate) unk_74: u32, - pub(crate) unk_78: u32, - pub(crate) unk_7c: u32, - pub(crate) unk_80: u32, - pub(crate) max_pages: u32, - pub(crate) max_pages_nomemless: u32, - pub(crate) unk_8c: u32, - pub(crate) unk_90: Array<0x30, u8>, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct Scene<'a> { - #[ver(G >= G14X)] - pub(crate) control_word: GpuPointer<'a, &'a [u32]>, - #[ver(G >= G14X)] - pub(crate) control_word2: GpuPointer<'a, &'a [u32]>, - pub(crate) pass_page_count: AtomicU32, - pub(crate) unk_4: u32, - pub(crate) unk_8: U64, - pub(crate) unk_10: U64, - pub(crate) user_buffer: GpuPointer<'a, &'a [u8]>, - pub(crate) unk_20: u32, - #[ver(V >= V13_3)] - pub(crate) unk_28: U64, - pub(crate) stats: GpuWeakPointer, - pub(crate) total_page_count: AtomicU32, - #[ver(G < G14X)] - pub(crate) unk_30: U64, // pad - #[ver(G < G14X)] - pub(crate) unk_38: U64, // pad - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct InitBuffer<'a> { - pub(crate) tag: workqueue::CommandType, - pub(crate) vm_slot: u32, - pub(crate) buffer_slot: u32, - pub(crate) unk_c: u32, - pub(crate) block_count: u32, - pub(crate) buffer: GpuPointer<'a, super::Info::ver>, - pub(crate) stamp_value: EventValue, - } -} - -trivial_gpustruct!(BlockControl); -trivial_gpustruct!(Counter); -trivial_gpustruct!(Stats); - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct Info { - pub(crate) block_ctl: GpuObject, - pub(crate) counter: GpuObject, - pub(crate) page_list: GpuArray, - pub(crate) block_list: GpuArray, -} - -#[versions(AGX_NEO)] -impl GpuStruct for Info::ver { - type Raw<'a> = raw::Info::ver<'a>; -} - -pub(crate) struct ClusterBuffers { - pub(crate) tilemaps: GpuArray, - pub(crate) meta: GpuArray, -} - -#[versions(AGX_NEO)] -pub(crate) struct Scene { - pub(crate) user_buffer: GpuArray, - pub(crate) buffer: crate::buffer::Buffer::ver, - pub(crate) tvb_heapmeta: GpuArray, - pub(crate) tvb_tilemap: GpuArray, - pub(crate) tpc: Arc>, - pub(crate) clustering: Option, - pub(crate) preempt_buf: GpuArray, - #[ver(G >= G14X)] - pub(crate) control_word: GpuArray, -} - -#[versions(AGX_NEO)] -no_debug!(Scene::ver); - -#[versions(AGX_NEO)] -impl GpuStruct for Scene::ver { - type Raw<'a> = raw::Scene::ver<'a>; -} - -#[versions(AGX_NEO)] -pub(crate) struct InitBuffer { - pub(crate) scene: Arc, -} - -#[versions(AGX_NEO)] -no_debug!(InitBuffer::ver); - -#[versions(AGX_NEO)] -impl workqueue::Command for InitBuffer::ver {} - -#[versions(AGX_NEO)] -impl GpuStruct for InitBuffer::ver { - type Raw<'a> = raw::InitBuffer::ver<'a>; -} diff --git a/drivers/gpu/drm/asahi-neo/fw/channels.rs b/drivers/gpu/drm/asahi-neo/fw/channels.rs deleted file mode 100644 index 65d7c6367bedd8..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/channels.rs +++ /dev/null @@ -1,443 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU communication channel firmware structures (ring buffers) - -use super::types::*; -use crate::default_zeroed; -use core::sync::atomic::Ordering; -use kernel::static_assert; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct ChannelState<'a> { - pub(crate) read_ptr: AtomicU32, - __pad0: Pad<0x1c>, - pub(crate) write_ptr: AtomicU32, - __pad1: Pad<0xc>, - _p: PhantomData<&'a ()>, - } - default_zeroed!(<'a>, ChannelState<'a>); - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct FwCtlChannelState<'a> { - pub(crate) read_ptr: AtomicU32, - __pad0: Pad<0xc>, - pub(crate) write_ptr: AtomicU32, - __pad1: Pad<0xc>, - _p: PhantomData<&'a ()>, - } - default_zeroed!(<'a>, FwCtlChannelState<'a>); -} - -pub(crate) trait RxChannelState: GpuStruct + Debug + Default -where - for<'a> ::Raw<'a>: Default + Zeroable, -{ - const SUB_CHANNELS: usize; - - fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32; - fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32); -} - -#[derive(Debug, Default)] -pub(crate) struct ChannelState {} - -impl GpuStruct for ChannelState { - type Raw<'a> = raw::ChannelState<'a>; -} - -impl RxChannelState for ChannelState { - const SUB_CHANNELS: usize = 1; - - fn wptr(raw: &Self::Raw<'_>, _index: usize) -> u32 { - raw.write_ptr.load(Ordering::Acquire) - } - - fn set_rptr(raw: &Self::Raw<'_>, _index: usize, rptr: u32) { - raw.read_ptr.store(rptr, Ordering::Release); - } -} - -#[derive(Debug, Default)] -pub(crate) struct FwLogChannelState {} - -impl GpuStruct for FwLogChannelState { - type Raw<'a> = Array<6, raw::ChannelState<'a>>; -} - -impl RxChannelState for FwLogChannelState { - const SUB_CHANNELS: usize = 6; - - fn wptr(raw: &Self::Raw<'_>, index: usize) -> u32 { - raw[index].write_ptr.load(Ordering::Acquire) - } - - fn set_rptr(raw: &Self::Raw<'_>, index: usize, rptr: u32) { - raw[index].read_ptr.store(rptr, Ordering::Release); - } -} - -#[derive(Debug, Default)] -pub(crate) struct FwCtlChannelState {} - -impl GpuStruct for FwCtlChannelState { - type Raw<'a> = raw::FwCtlChannelState<'a>; -} - -pub(crate) trait TxChannelState: GpuStruct + Debug + Default { - fn rptr(raw: &Self::Raw<'_>) -> u32; - fn set_wptr(raw: &Self::Raw<'_>, wptr: u32); -} - -impl TxChannelState for ChannelState { - fn rptr(raw: &Self::Raw<'_>) -> u32 { - raw.read_ptr.load(Ordering::Acquire) - } - - fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { - raw.write_ptr.store(wptr, Ordering::Release); - } -} - -impl TxChannelState for FwCtlChannelState { - fn rptr(raw: &Self::Raw<'_>) -> u32 { - raw.read_ptr.load(Ordering::Acquire) - } - - fn set_wptr(raw: &Self::Raw<'_>, wptr: u32) { - raw.write_ptr.store(wptr, Ordering::Release); - } -} - -#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum PipeType { - #[default] - Vertex = 0, - Fragment = 1, - Compute = 2, -} - -#[versions(AGX_NEO)] -#[derive(Debug, Copy, Clone, Default)] -#[repr(C)] -pub(crate) struct RunWorkQueueMsg { - pub(crate) pipe_type: PipeType, - pub(crate) work_queue: Option>, - pub(crate) wptr: u32, - pub(crate) event_slot: u32, - pub(crate) is_new: bool, - #[ver(V >= V13_2 && G == G14)] - pub(crate) __pad: Pad<0x2b>, - #[ver(V < V13_2 || G != G14)] - pub(crate) __pad: Pad<0x1b>, -} - -#[versions(AGX_NEO)] -pub(crate) type PipeMsg = RunWorkQueueMsg::ver; - -#[versions(AGX_NEO)] -pub(crate) const DEVICECONTROL_SZ: usize = { - #[ver(V < V13_2 || G != G14)] - { - 0x2c - } - #[ver(V >= V13_2 && G == G14)] - { - 0x3c - } -}; - -// TODO: clean up when arbitrary_enum_discriminant is stable -// https://github.com/rust-lang/rust/issues/60553 - -#[versions(AGX_NEO)] -#[derive(Debug, Copy, Clone)] -#[repr(C, u32)] -#[allow(dead_code)] -pub(crate) enum DeviceControlMsg { - Unk00(Array), - Unk01(Array), - Unk02(Array), - Unk03(Array), - Unk04(Array), - Unk05(Array), - Unk06(Array), - Unk07(Array), - Unk08(Array), - Unk09(Array), - Unk0a(Array), - Unk0b(Array), - Unk0c(Array), - #[ver(V >= V13_3)] - Unk0d(Array), - GrowTVBAck { - unk_4: u32, - buffer_slot: u32, - vm_slot: u32, - counter: u32, - subpipe: u32, - halt_count: U64, - __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x1c }>, - }, - RecoverChannel { - pipe_type: u32, - work_queue: GpuWeakPointer, - event_value: u32, - __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x10 }>, - }, - IdlePowerOff { - val: u32, - __pad: Pad<{ DEVICECONTROL_SZ::ver - 0x4 }>, - }, - Unk10(Array), - Unk11(Array), - Unk12(Array), - Unk13(Array), - Unk14(Array), // Init? - Unk15(Array), // Enable something - Unk16(Array), // Disable something - DestroyContext { - unk_4: u32, - ctx_23: u8, - #[ver(V < V13_3)] - __pad0: Pad<3>, - unk_c: U32, - unk_10: U32, - ctx_0: u8, - ctx_1: u8, - ctx_4: u8, - #[ver(V < V13_3)] - __pad1: Pad<1>, - #[ver(V < V13_3)] - unk_18: u32, - gpu_context: Option>, - #[ver(V < V13_3)] - __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x20 }>, - #[ver(V >= V13_3)] - __pad2: Pad<{ DEVICECONTROL_SZ::ver - 0x18 }>, - }, - Unk18(Array), - Initialize(Pad), // Update RegionC -} - -#[versions(AGX_NEO)] -static_assert!(core::mem::size_of::() == 4 + DEVICECONTROL_SZ::ver); - -#[versions(AGX_NEO)] -default_zeroed!(DeviceControlMsg::ver); - -#[derive(Copy, Clone, Default, Debug)] -#[repr(C)] -#[allow(dead_code)] -pub(crate) struct FwCtlMsg { - pub(crate) addr: U64, - pub(crate) unk_8: u32, - pub(crate) slot: u32, - pub(crate) page_count: u16, - pub(crate) unk_12: u16, -} - -pub(crate) const EVENT_SZ: usize = 0x34; - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(C, u32)] -#[allow(dead_code)] -pub(crate) enum ChannelErrorType { - MemoryError, - DMKill, - Aborted, - Unk3, - Unknown(u32), -} - -#[derive(Debug, Copy, Clone)] -#[repr(C, u32)] -#[allow(dead_code)] -pub(crate) enum EventMsg { - Fault, - Flag { - firing: [u32; 4], - unk_14: u16, - }, - Unk2(Array), - Unk3(Array), - Timeout { - counter: u32, - unk_8: u32, - event_slot: i32, - }, - Unk5(Array), - Unk6(Array), - GrowTVB { - vm_slot: u32, - buffer_slot: u32, - counter: u32, - }, - ChannelError { - error_type: u32, - pipe_type: u32, - event_slot: u32, - event_value: u32, - }, - // Max discriminant: 0x8 -} - -static_assert!(core::mem::size_of::() == 4 + EVENT_SZ); - -pub(crate) const EVENT_MAX: u32 = 0x8; - -#[derive(Copy, Clone)] -#[repr(C)] -pub(crate) union RawEventMsg { - pub(crate) raw: (u32, Array), - pub(crate) msg: EventMsg, -} - -default_zeroed!(RawEventMsg); - -#[derive(Debug, Copy, Clone, Default)] -#[repr(C)] -pub(crate) struct RawFwLogMsg { - pub(crate) msg_type: u32, - __pad0: u32, - pub(crate) msg_index: U64, - __pad1: Pad<0x28>, -} - -#[derive(Debug, Copy, Clone, Default)] -#[repr(C)] -pub(crate) struct RawFwLogPayloadMsg { - pub(crate) msg_type: u32, - pub(crate) seq_no: u32, - pub(crate) timestamp: U64, - pub(crate) msg: Array<0xc8, u8>, -} - -#[derive(Debug, Copy, Clone, Default)] -#[repr(C)] -pub(crate) struct RawKTraceMsg { - pub(crate) msg_type: u32, - pub(crate) timestamp: U64, - pub(crate) args: Array<4, U64>, - pub(crate) code: u8, - pub(crate) channel: u8, - __pad: Pad<1>, - pub(crate) thread: u8, - pub(crate) unk_flag: U64, -} - -#[versions(AGX_NEO)] -pub(crate) const STATS_SZ: usize = { - #[ver(V < V13_0B4)] - { - 0x2c - } - #[ver(V >= V13_0B4)] - { - 0x3c - } -}; - -#[versions(AGX_NEO)] -#[derive(Debug, Copy, Clone)] -#[repr(C, u32)] -#[allow(dead_code)] -pub(crate) enum StatsMsg { - Power { - // 0x00 - __pad: Pad<0x18>, - power: U64, - }, - Unk1(Array<{ STATS_SZ::ver }, u8>), - PowerOn { - // 0x02 - off_time: U64, - }, - PowerOff { - // 0x03 - on_time: U64, - }, - Utilization { - // 0x04 - timestamp: U64, - util1: u32, - util2: u32, - util3: u32, - util4: u32, - }, - Unk5(Array<{ STATS_SZ::ver }, u8>), - Unk6(Array<{ STATS_SZ::ver }, u8>), - Unk7(Array<{ STATS_SZ::ver }, u8>), - Unk8(Array<{ STATS_SZ::ver }, u8>), - AvgPower { - // 0x09 - active_cs: U64, - unk2: u32, - unk3: u32, - unk4: u32, - avg_power: u32, - }, - Temperature { - // 0x0a - __pad: Pad<0x8>, - raw_value: u32, - scale: u32, - tmin: u32, - tmax: u32, - }, - PowerState { - // 0x0b - timestamp: U64, - last_busy_ts: U64, - active: u32, - poweroff: u32, - unk1: u32, - pstate: u32, - unk2: u32, - unk3: u32, - }, - FwBusy { - // 0x0c - timestamp: U64, - busy: u32, - }, - PState { - // 0x0d - __pad: Pad<0x8>, - ps_min: u32, - unk1: u32, - ps_max: u32, - unk2: u32, - }, - TempSensor { - // 0x0e - __pad: Pad<0x4>, - sensor_id: u32, - raw_value: u32, - scale: u32, - tmin: u32, - tmax: u32, - }, // Max discriminant: 0xe -} - -#[versions(AGX_NEO)] -static_assert!(core::mem::size_of::() == 4 + STATS_SZ::ver); - -#[versions(AGX_NEO)] -pub(crate) const STATS_MAX: u32 = 0xe; - -#[versions(AGX_NEO)] -#[derive(Copy, Clone)] -#[repr(C)] -pub(crate) union RawStatsMsg { - pub(crate) raw: (u32, Array<{ STATS_SZ::ver }, u8>), - pub(crate) msg: StatsMsg::ver, -} - -#[versions(AGX_NEO)] -default_zeroed!(RawStatsMsg::ver); diff --git a/drivers/gpu/drm/asahi-neo/fw/compute.rs b/drivers/gpu/drm/asahi-neo/fw/compute.rs deleted file mode 100644 index 916356298418a0..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/compute.rs +++ /dev/null @@ -1,120 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU compute job firmware structures - -use super::types::*; -use super::{ - event, - job, - workqueue, // -}; -use crate::{ - microseq, - mmu, // -}; -use kernel::sync::Arc; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobParameters1<'a> { - pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, - pub(crate) cdm_ctrl_stream_base: U64, - pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, - pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, - pub(crate) preempt_buf4: GpuPointer<'a, &'a [u8]>, - pub(crate) preempt_buf5: GpuPointer<'a, &'a [u8]>, - pub(crate) usc_exec_base_cp: U64, - pub(crate) unk_38: U64, - pub(crate) helper_program: u32, - pub(crate) unk_44: u32, - pub(crate) helper_arg: U64, - pub(crate) helper_cfg: u32, - pub(crate) unk_54: u32, - pub(crate) unk_58: u32, - pub(crate) unk_5c: u32, - pub(crate) iogpu_unk_40: u32, - pub(crate) __pad: Pad<0xfc>, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobParameters2<'a> { - #[ver(V >= V13_0B4)] - pub(crate) unk_0_0: u32, - pub(crate) unk_0: Array<0x24, u8>, - pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, - pub(crate) cdm_ctrl_stream_end: U64, - pub(crate) unk_34: Array<0x20, u8>, - pub(crate) unk_g14x: u32, - pub(crate) unk_58: u32, - #[ver(V < V13_0B4)] - pub(crate) unk_5c: u32, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct RunCompute<'a> { - pub(crate) tag: workqueue::CommandType, - - #[ver(V >= V13_0B4)] - pub(crate) counter: U64, - - pub(crate) unk_4: u32, - pub(crate) vm_slot: u32, - pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, - pub(crate) unk_pointee: u32, - #[ver(G < G14X)] - pub(crate) __pad0: Array<0x50, u8>, - #[ver(G < G14X)] - pub(crate) job_params1: JobParameters1<'a>, - #[ver(G >= G14X)] - pub(crate) registers: job::raw::RegisterArray, - pub(crate) __pad1: Array<0x20, u8>, - pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, - pub(crate) microsequence_size: u32, - pub(crate) job_params2: JobParameters2::ver<'a>, - pub(crate) encoder_params: job::raw::EncoderParams, - pub(crate) meta: job::raw::JobMeta, - pub(crate) command_time: U64, - pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, - pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, - pub(crate) client_sequence: u8, - pub(crate) pad_2d1: Array<3, u8>, - pub(crate) unk_2d4: u32, - pub(crate) unk_2d8: u8, - #[ver(V >= V13_0B4)] - pub(crate) context_store_req: U64, - #[ver(V >= V13_0B4)] - pub(crate) context_store_compl: U64, - #[ver(V >= V13_0B4)] - pub(crate) unk_2e9: Array<0x14, u8>, - #[ver(V >= V13_0B4)] - pub(crate) unk_flag: U32, - #[ver(V >= V13_0B4)] - pub(crate) unk_pad: Array<0x10, u8>, - } -} - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct RunCompute { - pub(crate) notifier: Arc>, - pub(crate) preempt_buf: GpuArray, - pub(crate) micro_seq: microseq::MicroSequence, - pub(crate) vm_bind: mmu::VmBind, - pub(crate) timestamps: Arc>, - pub(crate) user_timestamps: job::UserTimestamps, -} - -#[versions(AGX_NEO)] -impl GpuStruct for RunCompute::ver { - type Raw<'a> = raw::RunCompute::ver<'a>; -} - -#[versions(AGX_NEO)] -impl workqueue::Command for RunCompute::ver {} diff --git a/drivers/gpu/drm/asahi-neo/fw/event.rs b/drivers/gpu/drm/asahi-neo/fw/event.rs deleted file mode 100644 index caeb7b5317f379..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/event.rs +++ /dev/null @@ -1,104 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU events control structures & stamps - -use super::types::*; -use crate::{ - default_zeroed, - trivial_gpustruct, // -}; -use core::sync::atomic::Ordering; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug, Clone, Copy, Default)] - #[repr(C)] - pub(crate) struct LinkedListHead { - pub(crate) prev: Option>, - pub(crate) next: Option>, - } - - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct NotifierList { - pub(crate) list_head: LinkedListHead, - pub(crate) unkptr_10: U64, - } - default_zeroed!(NotifierList); - - #[versions(AGX_NEO)] - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct NotifierState { - unk_14: u32, - unk_18: U64, - unk_20: u32, - vm_slot: u32, - has_vtx: u32, - pstamp_vtx: Array<4, U64>, - has_frag: u32, - pstamp_frag: Array<4, U64>, - has_comp: u32, - pstamp_comp: Array<4, U64>, - #[ver(G >= G14 && V < V13_0B4)] - unk_98_g14_0: Array<0x14, u8>, - in_list: u32, - list_head: LinkedListHead, - #[ver(G >= G14 && V < V13_0B4)] - unk_a8_g14_0: Pad<4>, - #[ver(V >= V13_0B4)] - pub(crate) unk_buf: Array<0x8, u8>, // Init to all-ff - } - - #[versions(AGX_NEO)] - impl Default for NotifierState::ver { - fn default() -> Self { - #[allow(unused_mut)] - // SAFETY: All bit patterns are valid for this type. - let mut s: Self = unsafe { core::mem::zeroed() }; - #[ver(V >= V13_0B4)] - s.unk_buf = Array::new([0xff; 0x8]); - s - } - } - - #[derive(Debug)] - #[repr(transparent)] - pub(crate) struct Threshold(AtomicU64); - default_zeroed!(Threshold); - - impl Threshold { - pub(crate) fn increase(&self, amount: u32) { - // We could use fetch_add, but the non-LSE atomic - // sequence Rust produces confuses the hypervisor. - let v = self.0.load(Ordering::Relaxed); - self.0.store(v + (amount as u64), Ordering::Relaxed); - } - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct Notifier<'a> { - pub(crate) threshold: GpuPointer<'a, super::Threshold>, - pub(crate) generation: AtomicU32, - pub(crate) cur_count: AtomicU32, - pub(crate) unk_10: AtomicU32, - pub(crate) state: NotifierState::ver, - } -} - -trivial_gpustruct!(Threshold); -trivial_gpustruct!(NotifierList); - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct Notifier { - pub(crate) threshold: GpuObject, -} - -#[versions(AGX_NEO)] -impl GpuStruct for Notifier::ver { - type Raw<'a> = raw::Notifier::ver<'a>; -} diff --git a/drivers/gpu/drm/asahi-neo/fw/fragment.rs b/drivers/gpu/drm/asahi-neo/fw/fragment.rs deleted file mode 100644 index 8c38be4679ed24..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/fragment.rs +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU fragment job firmware structures - -use super::types::*; -use super::{ - event, - job, - workqueue, // -}; -use crate::{ - buffer, - fw, - microseq, - mmu, // -}; -use kernel::sync::Arc; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct BackgroundProgram { - pub(crate) rsrc_spec: U64, - pub(crate) address: U64, - } - - #[derive(Debug, Clone, Copy, Default)] - #[repr(C)] - pub(crate) struct EotProgram { - pub(crate) unk_0: U64, - pub(crate) unk_8: u32, - pub(crate) rsrc_spec: u32, - pub(crate) unk_10: u32, - pub(crate) address: u32, - pub(crate) unk_18: u32, - pub(crate) unk_1c_padding: u32, - } - - impl EotProgram { - pub(crate) fn new(rsrc_spec: u32, address: u32) -> EotProgram { - EotProgram { - rsrc_spec, - address, - ..Default::default() - } - } - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct ArrayAddr { - pub(crate) ptr: U64, - pub(crate) unk_padding: U64, - } - - #[versions(AGX_NEO)] - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct AuxFBInfo { - pub(crate) isp_ctl: u32, - pub(crate) unk2: u32, - pub(crate) width: u32, - pub(crate) height: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk3: U64, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobParameters1<'a> { - pub(crate) utile_config: u32, - pub(crate) unk_4: u32, - pub(crate) bg: BackgroundProgram, - pub(crate) ppp_multisamplectl: U64, - pub(crate) isp_scissor_base: U64, - pub(crate) isp_dbias_base: U64, - pub(crate) aux_fb_info: AuxFBInfo::ver, - pub(crate) isp_zls_pixels: U64, - pub(crate) isp_oclqry_base: U64, - pub(crate) zls_ctrl: U64, - - #[ver(G >= G14)] - pub(crate) unk_58_g14_0: U64, - #[ver(G >= G14)] - pub(crate) unk_58_g14_8: U64, - - pub(crate) z_load: U64, - pub(crate) z_store: U64, - pub(crate) s_load: U64, - pub(crate) s_store: U64, - - #[ver(G >= G14)] - pub(crate) unk_68_g14_0: Array<0x20, u8>, - - pub(crate) z_load_stride: U64, - pub(crate) z_store_stride: U64, - pub(crate) s_load_stride: U64, - pub(crate) s_store_stride: U64, - pub(crate) z_load_comp: U64, - pub(crate) z_load_comp_stride: U64, - pub(crate) z_store_comp: U64, - pub(crate) z_store_comp_stride: U64, - pub(crate) s_load_comp: U64, - pub(crate) s_load_comp_stride: U64, - pub(crate) s_store_comp: U64, - pub(crate) s_store_comp_stride: U64, - pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, - pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, - pub(crate) mtile_stride_dwords: U64, - pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, - pub(crate) tile_config: U64, - pub(crate) aux_fb: GpuPointer<'a, &'a [u8]>, - pub(crate) unk_108: Array<0x6, U64>, - pub(crate) usc_exec_base_isp: U64, - pub(crate) unk_140: U64, - pub(crate) helper_program: u32, - pub(crate) unk_14c: u32, - pub(crate) helper_arg: U64, - pub(crate) unk_158: U64, - pub(crate) unk_160: U64, - - #[ver(G < G14)] - pub(crate) __pad: Pad<0x1d8>, - #[ver(G >= G14)] - pub(crate) __pad: Pad<0x1a8>, - #[ver(V < V13_0B4)] - pub(crate) __pad1: Pad<0x8>, - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobParameters2 { - pub(crate) eot_rsrc_spec: u32, - pub(crate) eot_usc: u32, - pub(crate) unk_8: u32, - pub(crate) unk_c: u32, - pub(crate) isp_merge_upper_x: F32, - pub(crate) isp_merge_upper_y: F32, - pub(crate) unk_18: U64, - pub(crate) utiles_per_mtile_y: u16, - pub(crate) utiles_per_mtile_x: u16, - pub(crate) unk_24: u32, - pub(crate) tile_counts: u32, - pub(crate) tib_blocks: u32, - pub(crate) isp_bgobjdepth: u32, - pub(crate) isp_bgobjvals: u32, - pub(crate) unk_38: u32, - pub(crate) unk_3c: u32, - pub(crate) helper_cfg: u32, - pub(crate) __pad: Pad<0xac>, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobParameters3 { - pub(crate) isp_dbias_base: ArrayAddr, - pub(crate) isp_scissor_base: ArrayAddr, - pub(crate) isp_oclqry_base: U64, - pub(crate) unk_118: U64, - pub(crate) unk_120: Array<0x25, U64>, - pub(crate) unk_partial_bg: BackgroundProgram, - pub(crate) unk_258: U64, - pub(crate) unk_260: U64, - pub(crate) unk_268: U64, - pub(crate) unk_270: U64, - pub(crate) partial_bg: BackgroundProgram, - pub(crate) zls_ctrl: U64, - pub(crate) unk_290: U64, - pub(crate) z_load: U64, - pub(crate) z_partial_stride: U64, - pub(crate) z_partial_comp_stride: U64, - pub(crate) z_store: U64, - pub(crate) z_partial: U64, - pub(crate) z_partial_comp: U64, - pub(crate) s_load: U64, - pub(crate) s_partial_stride: U64, - pub(crate) s_partial_comp_stride: U64, - pub(crate) s_store: U64, - pub(crate) s_partial: U64, - pub(crate) s_partial_comp: U64, - pub(crate) unk_2f8: Array<2, U64>, - pub(crate) tib_blocks: u32, - pub(crate) unk_30c: u32, - pub(crate) aux_fb_info: AuxFBInfo::ver, - pub(crate) tile_config: U64, - pub(crate) unk_328_padding: Array<0x8, u8>, - pub(crate) unk_partial_eot: EotProgram, - pub(crate) partial_eot: EotProgram, - pub(crate) isp_bgobjdepth: u32, - pub(crate) isp_bgobjvals: u32, - pub(crate) sample_size: u32, - pub(crate) unk_37c: u32, - pub(crate) unk_380: U64, - pub(crate) unk_388: U64, - - #[ver(V >= V13_0B4)] - pub(crate) unk_390_0: U64, - - pub(crate) isp_zls_pixels: U64, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct RunFragment<'a> { - pub(crate) tag: workqueue::CommandType, - - #[ver(V >= V13_0B4)] - pub(crate) counter: U64, - - pub(crate) vm_slot: u32, - pub(crate) unk_8: u32, - pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, - pub(crate) microsequence_size: u32, - pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, - pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, - pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, - pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, - pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, - pub(crate) ppp_multisamplectl: U64, - pub(crate) samples: u32, - pub(crate) tiles_per_mtile_y: u16, - pub(crate) tiles_per_mtile_x: u16, - pub(crate) unk_50: U64, - pub(crate) unk_58: U64, - pub(crate) isp_merge_upper_x: F32, - pub(crate) isp_merge_upper_y: F32, - pub(crate) unk_68: U64, - pub(crate) tile_count: U64, - - #[ver(G < G14X)] - pub(crate) job_params1: JobParameters1::ver<'a>, - #[ver(G < G14X)] - pub(crate) job_params2: JobParameters2, - #[ver(G >= G14X)] - pub(crate) registers: job::raw::RegisterArray, - - pub(crate) job_params3: JobParameters3::ver, - pub(crate) unk_758_flag: u32, - pub(crate) unk_75c_flag: u32, - pub(crate) unk_buf: Array<0x110, u8>, - pub(crate) busy_flag: u32, - pub(crate) tvb_overflow_count: u32, - pub(crate) unk_878: u32, - pub(crate) encoder_params: job::raw::EncoderParams, - pub(crate) process_empty_tiles: u32, - pub(crate) no_clear_pipeline_textures: u32, - pub(crate) msaa_zs: u32, - pub(crate) unk_pointee: u32, - #[ver(V >= V13_3)] - pub(crate) unk_v13_3: u32, - pub(crate) meta: job::raw::JobMeta, - pub(crate) unk_after_meta: u32, - pub(crate) unk_buf_0: U64, - pub(crate) unk_buf_8: U64, - pub(crate) unk_buf_10: U64, - pub(crate) command_time: U64, - pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, - pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, - pub(crate) client_sequence: u8, - pub(crate) pad_925: Array<3, u8>, - pub(crate) unk_928: u32, - pub(crate) unk_92c: u8, - - #[ver(V >= V13_0B4)] - pub(crate) unk_ts: U64, - - #[ver(V >= V13_0B4)] - pub(crate) unk_92d_8: Array<0x1b, u8>, - } -} - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct RunFragment { - pub(crate) notifier: Arc>, - pub(crate) scene: Arc, - pub(crate) micro_seq: microseq::MicroSequence, - pub(crate) vm_bind: mmu::VmBind, - pub(crate) aux_fb: GpuArray, - pub(crate) timestamps: Arc>, - pub(crate) user_timestamps: job::UserTimestamps, -} - -#[versions(AGX_NEO)] -impl GpuStruct for RunFragment::ver { - type Raw<'a> = raw::RunFragment::ver<'a>; -} - -#[versions(AGX_NEO)] -impl workqueue::Command for RunFragment::ver {} diff --git a/drivers/gpu/drm/asahi-neo/fw/initdata.rs b/drivers/gpu/drm/asahi-neo/fw/initdata.rs deleted file mode 100644 index 09943b57d68152..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/initdata.rs +++ /dev/null @@ -1,1352 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU initialization / global structures - -use super::channels; -use super::types::*; -use crate::{ - default_zeroed, - gem, - mmu, - no_debug, - trivial_gpustruct, // -}; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct ChannelRing { - pub(crate) state: Option>, - pub(crate) ring: Option>, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct PipeChannels { - pub(crate) vtx: ChannelRing, - pub(crate) frag: ChannelRing, - pub(crate) comp: ChannelRing, - } - #[versions(AGX_NEO)] - default_zeroed!(PipeChannels::ver); - - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct FwStatusFlags { - pub(crate) halt_count: AtomicU64, - __pad0: Pad<0x8>, - pub(crate) halted: AtomicU32, - __pad1: Pad<0xc>, - pub(crate) resume: AtomicU32, - __pad2: Pad<0xc>, - pub(crate) unk_40: u32, - __pad3: Pad<0xc>, - pub(crate) unk_ctr: u32, - __pad4: Pad<0xc>, - pub(crate) unk_60: u32, - __pad5: Pad<0xc>, - pub(crate) unk_70: u32, - __pad6: Pad<0xc>, - } - - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct FwStatus { - pub(crate) fwctl_channel: ChannelRing, - pub(crate) flags: FwStatusFlags, - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct HwDataShared1 { - pub(crate) table: Array<16, i32>, - pub(crate) unk_44: Array<0x60, u8>, - pub(crate) unk_a4: u32, - pub(crate) unk_a8: u32, - } - default_zeroed!(HwDataShared1); - - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct HwDataShared2Curve { - pub(crate) unk_0: u32, - pub(crate) unk_4: u32, - pub(crate) t1: Array<16, u16>, - pub(crate) t2: Array<16, i16>, - pub(crate) t3: Array<8, Array<16, i32>>, - } - - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct HwDataShared2G14 { - pub(crate) unk_0: Array<5, u32>, - pub(crate) unk_14: u32, - pub(crate) unk_18: Array<8, u32>, - pub(crate) curve1: HwDataShared2Curve, - pub(crate) curve2: HwDataShared2Curve, - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct HwDataShared2 { - pub(crate) table: Array<10, i32>, - pub(crate) unk_28: Array<0x10, u8>, - pub(crate) g14: HwDataShared2G14, - pub(crate) unk_500: u32, - pub(crate) unk_504: u32, - pub(crate) unk_508: u32, - pub(crate) unk_50c: u32, - } - default_zeroed!(HwDataShared2); - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct HwDataShared3 { - pub(crate) unk_0: u32, - pub(crate) unk_4: u32, - pub(crate) unk_8: u32, - pub(crate) table: Array<16, u32>, - pub(crate) unk_4c: u32, - } - default_zeroed!(HwDataShared3); - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct HwDataA130Extra { - pub(crate) unk_0: Array<0x38, u8>, - pub(crate) unk_38: u32, - pub(crate) unk_3c: u32, - pub(crate) gpu_se_inactive_threshold: u32, - pub(crate) unk_44: u32, - pub(crate) gpu_se_engagement_criteria: i32, - pub(crate) gpu_se_reset_criteria: u32, - pub(crate) unk_50: u32, - pub(crate) unk_54: u32, - pub(crate) unk_58: u32, - pub(crate) unk_5c: u32, - pub(crate) gpu_se_filter_a_neg: F32, - pub(crate) gpu_se_filter_1_a_neg: F32, - pub(crate) gpu_se_filter_a: F32, - pub(crate) gpu_se_filter_1_a: F32, - pub(crate) gpu_se_ki_dt: F32, - pub(crate) gpu_se_ki_1_dt: F32, - pub(crate) unk_78: F32, - pub(crate) unk_7c: F32, - pub(crate) gpu_se_kp: F32, - pub(crate) gpu_se_kp_1: F32, - pub(crate) unk_88: u32, - pub(crate) unk_8c: u32, - pub(crate) max_pstate_scaled_1: u32, - pub(crate) unk_94: u32, - pub(crate) unk_98: u32, - pub(crate) unk_9c: F32, - pub(crate) unk_a0: u32, - pub(crate) unk_a4: u32, - pub(crate) gpu_se_filter_time_constant_ms: u32, - pub(crate) gpu_se_filter_time_constant_1_ms: u32, - pub(crate) gpu_se_filter_time_constant_clks: U64, - pub(crate) gpu_se_filter_time_constant_1_clks: U64, - pub(crate) unk_c0: u32, - pub(crate) unk_c4: F32, - pub(crate) unk_c8: Array<0x4c, u8>, - pub(crate) unk_114: F32, - pub(crate) unk_118: u32, - pub(crate) unk_11c: u32, - pub(crate) unk_120: u32, - pub(crate) unk_124: u32, - pub(crate) max_pstate_scaled_2: u32, - pub(crate) unk_12c: Array<0x8c, u8>, - } - default_zeroed!(HwDataA130Extra); - - #[repr(C)] - pub(crate) struct T81xxData { - pub(crate) unk_d8c: u32, - pub(crate) unk_d90: u32, - pub(crate) unk_d94: u32, - pub(crate) unk_d98: u32, - pub(crate) unk_d9c: F32, - pub(crate) unk_da0: u32, - pub(crate) unk_da4: F32, - pub(crate) unk_da8: u32, - pub(crate) unk_dac: F32, - pub(crate) unk_db0: u32, - pub(crate) unk_db4: u32, - pub(crate) unk_db8: F32, - pub(crate) unk_dbc: F32, - pub(crate) unk_dc0: u32, - pub(crate) unk_dc4: u32, - pub(crate) unk_dc8: u32, - pub(crate) max_pstate_scaled: u32, - } - default_zeroed!(T81xxData); - - #[versions(AGX_NEO)] - #[derive(Default, Copy, Clone)] - #[repr(C)] - pub(crate) struct PowerZone { - pub(crate) val: F32, - pub(crate) target: u32, - pub(crate) target_off: u32, - pub(crate) filter_tc_x4: u32, - pub(crate) filter_tc_xperiod: u32, - #[ver(V >= V13_0B4)] - pub(crate) unk_10: u32, - #[ver(V >= V13_0B4)] - pub(crate) unk_14: u32, - pub(crate) filter_a_neg: F32, - pub(crate) filter_a: F32, - pub(crate) pad: u32, - } - - #[versions(AGX_NEO)] - const MAX_CORES_PER_CLUSTER: usize = { - #[ver(G >= G14X)] - { - 16 - } - #[ver(G < G14X)] - { - 8 - } - }; - - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct AuxLeakCoef { - pub(crate) afr_1: Array<2, F32>, - pub(crate) cs_1: Array<2, F32>, - pub(crate) afr_2: Array<2, F32>, - pub(crate) cs_2: Array<2, F32>, - } - - #[versions(AGX_NEO)] - #[repr(C)] - pub(crate) struct HwDataA { - pub(crate) unk_0: u32, - pub(crate) clocks_per_period: u32, - - #[ver(V >= V13_0B4)] - pub(crate) clocks_per_period_2: u32, - - pub(crate) unk_8: u32, - pub(crate) pwr_status: AtomicU32, - pub(crate) unk_10: F32, - pub(crate) unk_14: u32, - pub(crate) unk_18: u32, - pub(crate) unk_1c: u32, - pub(crate) unk_20: u32, - pub(crate) unk_24: u32, - pub(crate) actual_pstate: u32, - pub(crate) tgt_pstate: u32, - pub(crate) unk_30: u32, - pub(crate) cur_pstate: u32, - pub(crate) unk_38: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_3c_0: u32, - - pub(crate) base_pstate_scaled: u32, - pub(crate) unk_40: u32, - pub(crate) max_pstate_scaled: u32, - pub(crate) unk_48: u32, - pub(crate) min_pstate_scaled: u32, - pub(crate) freq_mhz: F32, - pub(crate) unk_54: Array<0x20, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_74_0: u32, - - pub(crate) sram_k: Array<0x10, F32>, - pub(crate) unk_b4: Array<0x100, u8>, - pub(crate) unk_1b4: u32, - pub(crate) temp_c: u32, - pub(crate) avg_power_mw: u32, - pub(crate) update_ts: U64, - pub(crate) unk_1c8: u32, - pub(crate) unk_1cc: Array<0x478, u8>, - pub(crate) pad_644: Pad<0x8>, - pub(crate) unk_64c: u32, - pub(crate) unk_650: u32, - pub(crate) pad_654: u32, - pub(crate) pwr_filter_a_neg: F32, - pub(crate) pad_65c: u32, - pub(crate) pwr_filter_a: F32, - pub(crate) pad_664: u32, - pub(crate) pwr_integral_gain: F32, - pub(crate) pad_66c: u32, - pub(crate) pwr_integral_min_clamp: F32, - pub(crate) max_power_1: F32, - pub(crate) pwr_proportional_gain: F32, - pub(crate) pad_67c: u32, - pub(crate) pwr_pstate_related_k: F32, - pub(crate) pwr_pstate_max_dc_offset: i32, - pub(crate) unk_688: u32, - pub(crate) max_pstate_scaled_2: u32, - pub(crate) pad_690: u32, - pub(crate) unk_694: u32, - pub(crate) max_power_2: u32, - pub(crate) pad_69c: Pad<0x18>, - pub(crate) unk_6b4: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_6b8_0: Array<0x10, u8>, - - pub(crate) max_pstate_scaled_3: u32, - pub(crate) unk_6bc: u32, - pub(crate) pad_6c0: Pad<0x14>, - pub(crate) ppm_filter_tc_periods_x4: u32, - pub(crate) unk_6d8: u32, - pub(crate) pad_6dc: u32, - pub(crate) ppm_filter_a_neg: F32, - pub(crate) pad_6e4: u32, - pub(crate) ppm_filter_a: F32, - pub(crate) pad_6ec: u32, - pub(crate) ppm_ki_dt: F32, - pub(crate) pad_6f4: u32, - pub(crate) pwr_integral_min_clamp_2: u32, - pub(crate) unk_6fc: F32, - pub(crate) ppm_kp: F32, - pub(crate) pad_704: u32, - pub(crate) unk_708: u32, - pub(crate) pwr_min_duty_cycle: u32, - pub(crate) max_pstate_scaled_4: u32, - pub(crate) unk_714: u32, - pub(crate) pad_718: u32, - pub(crate) unk_71c: F32, - pub(crate) max_power_3: u32, - pub(crate) cur_power_mw_2: u32, - pub(crate) ppm_filter_tc_ms: u32, - pub(crate) unk_72c: u32, - - #[ver(V >= V13_0B4)] - pub(crate) ppm_filter_tc_clks: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_730_4: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_730_8: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_730_c: u32, - - pub(crate) unk_730: F32, - pub(crate) unk_734: u32, - pub(crate) unk_738: u32, - pub(crate) unk_73c: u32, - pub(crate) unk_740: u32, - pub(crate) unk_744: u32, - pub(crate) unk_748: Array<0x4, F32>, - pub(crate) unk_758: u32, - pub(crate) perf_tgt_utilization: u32, - pub(crate) pad_760: u32, - pub(crate) perf_boost_min_util: u32, - pub(crate) perf_boost_ce_step: u32, - pub(crate) perf_reset_iters: u32, - pub(crate) pad_770: u32, - pub(crate) unk_774: u32, - pub(crate) unk_778: u32, - pub(crate) perf_filter_drop_threshold: u32, - pub(crate) perf_filter_a_neg: F32, - pub(crate) perf_filter_a2_neg: F32, - pub(crate) perf_filter_a: F32, - pub(crate) perf_filter_a2: F32, - pub(crate) perf_ki: F32, - pub(crate) perf_ki2: F32, - pub(crate) perf_integral_min_clamp: F32, - pub(crate) unk_79c: F32, - pub(crate) perf_kp: F32, - pub(crate) perf_kp2: F32, - pub(crate) boost_state_unk_k: F32, - pub(crate) base_pstate_scaled_2: u32, - pub(crate) max_pstate_scaled_5: u32, - pub(crate) base_pstate_scaled_3: u32, - pub(crate) pad_7b8: u32, - pub(crate) perf_cur_utilization: F32, - pub(crate) perf_tgt_utilization_2: u32, - pub(crate) pad_7c4: Pad<0x18>, - pub(crate) unk_7dc: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_7e0_0: Array<0x10, u8>, - - pub(crate) base_pstate_scaled_4: u32, - pub(crate) pad_7e4: u32, - pub(crate) unk_7e8: Array<0x14, u8>, - pub(crate) unk_7fc: F32, - pub(crate) pwr_min_duty_cycle_2: F32, - pub(crate) max_pstate_scaled_6: F32, - pub(crate) max_freq_mhz: u32, - pub(crate) pad_80c: u32, - pub(crate) unk_810: u32, - pub(crate) pad_814: u32, - pub(crate) pwr_min_duty_cycle_3: u32, - pub(crate) unk_81c: u32, - pub(crate) pad_820: u32, - pub(crate) min_pstate_scaled_4: F32, - pub(crate) max_pstate_scaled_7: u32, - pub(crate) unk_82c: u32, - pub(crate) unk_alpha_neg: F32, - pub(crate) unk_alpha: F32, - pub(crate) unk_838: u32, - pub(crate) unk_83c: u32, - pub(crate) pad_840: Pad<0x2c>, - pub(crate) unk_86c: u32, - pub(crate) fast_die0_sensor_mask: U64, - #[ver(G >= G14X)] - pub(crate) fast_die1_sensor_mask: U64, - pub(crate) fast_die0_release_temp_cc: u32, - pub(crate) unk_87c: i32, - pub(crate) unk_880: u32, - pub(crate) unk_884: u32, - pub(crate) pad_888: u32, - pub(crate) unk_88c: u32, - pub(crate) pad_890: u32, - pub(crate) unk_894: F32, - pub(crate) pad_898: u32, - pub(crate) fast_die0_ki_dt: F32, - pub(crate) pad_8a0: u32, - pub(crate) unk_8a4: u32, - pub(crate) unk_8a8: F32, - pub(crate) fast_die0_kp: F32, - pub(crate) pad_8b0: u32, - pub(crate) unk_8b4: u32, - pub(crate) pwr_min_duty_cycle_4: u32, - pub(crate) max_pstate_scaled_8: u32, - pub(crate) max_pstate_scaled_9: u32, - pub(crate) fast_die0_prop_tgt_delta: u32, - pub(crate) unk_8c8: u32, - pub(crate) unk_8cc: u32, - pub(crate) pad_8d0: Pad<0x14>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_8e4_0: Array<0x10, u8>, - - pub(crate) unk_8e4: u32, - pub(crate) unk_8e8: u32, - pub(crate) max_pstate_scaled_10: u32, - pub(crate) unk_8f0: u32, - pub(crate) unk_8f4: u32, - pub(crate) pad_8f8: u32, - pub(crate) pad_8fc: u32, - pub(crate) unk_900: Array<0x24, u8>, - - pub(crate) unk_coef_a1: Array<8, Array>, - pub(crate) unk_coef_a2: Array<8, Array>, - - pub(crate) pad_b24: Pad<0x70>, - pub(crate) max_pstate_scaled_11: u32, - pub(crate) freq_with_off: u32, - pub(crate) unk_b9c: u32, - pub(crate) unk_ba0: U64, - pub(crate) unk_ba8: U64, - pub(crate) unk_bb0: u32, - pub(crate) unk_bb4: u32, - - #[ver(V >= V13_3)] - pub(crate) pad_bb8_0: Pad<0x200>, - #[ver(V >= V13_5)] - pub(crate) pad_bb8_200: Pad<0x8>, - - pub(crate) pad_bb8: Pad<0x74>, - pub(crate) unk_c2c: u32, - pub(crate) power_zone_count: u32, - pub(crate) max_power_4: u32, - pub(crate) max_power_5: u32, - pub(crate) max_power_6: u32, - pub(crate) unk_c40: u32, - pub(crate) unk_c44: F32, - pub(crate) avg_power_target_filter_a_neg: F32, - pub(crate) avg_power_target_filter_a: F32, - pub(crate) avg_power_target_filter_tc_x4: u32, - pub(crate) avg_power_target_filter_tc_xperiod: u32, - - #[ver(V >= V13_0B4)] - pub(crate) avg_power_target_filter_tc_clks: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_c58_4: u32, - - pub(crate) power_zones: Array<5, PowerZone::ver>, - pub(crate) avg_power_filter_tc_periods_x4: u32, - pub(crate) unk_cfc: u32, - pub(crate) unk_d00: u32, - pub(crate) avg_power_filter_a_neg: F32, - pub(crate) unk_d08: u32, - pub(crate) avg_power_filter_a: F32, - pub(crate) unk_d10: u32, - pub(crate) avg_power_ki_dt: F32, - pub(crate) unk_d18: u32, - pub(crate) unk_d1c: u32, - pub(crate) unk_d20: F32, - pub(crate) avg_power_kp: F32, - pub(crate) unk_d28: u32, - pub(crate) unk_d2c: u32, - pub(crate) avg_power_min_duty_cycle: u32, - pub(crate) max_pstate_scaled_12: u32, - pub(crate) max_pstate_scaled_13: u32, - pub(crate) unk_d3c: u32, - pub(crate) max_power_7: F32, - pub(crate) max_power_8: u32, - pub(crate) unk_d48: u32, - pub(crate) avg_power_filter_tc_ms: u32, - pub(crate) unk_d50: u32, - - #[ver(V >= V13_0B4)] - pub(crate) avg_power_filter_tc_clks: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_d54_4: Array<0xc, u8>, - - pub(crate) unk_d54: Array<0x10, u8>, - pub(crate) max_pstate_scaled_14: u32, - pub(crate) unk_d68: Array<0x24, u8>, - - pub(crate) t81xx_data: T81xxData, - - pub(crate) unk_dd0: Array<0x40, u8>, - - #[ver(V >= V13_2)] - pub(crate) unk_e10_pad: Array<0x10, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_e10_0: HwDataA130Extra, - - pub(crate) unk_e10: Array<0xc, u8>, - - pub(crate) fast_die0_sensor_mask_2: U64, - #[ver(G >= G14X)] - pub(crate) fast_die1_sensor_mask_2: U64, - - pub(crate) unk_e24: u32, - pub(crate) unk_e28: u32, - pub(crate) unk_e2c: Pad<0x1c>, - pub(crate) unk_coef_b1: Array<8, Array>, - pub(crate) unk_coef_b2: Array<8, Array>, - - #[ver(G >= G14X)] - pub(crate) pad_1048_0: Pad<0x600>, - - pub(crate) pad_1048: Pad<0x5e4>, - - pub(crate) fast_die0_sensor_mask_alt: U64, - #[ver(G >= G14X)] - pub(crate) fast_die1_sensor_mask_alt: U64, - #[ver(V < V13_0B4)] - pub(crate) fast_die0_sensor_present: U64, - - pub(crate) unk_163c: u32, - - pub(crate) unk_1640: Array<0x2000, u8>, - - #[ver(G >= G14X)] - pub(crate) unk_3640_0: Array<0x2000, u8>, - - pub(crate) unk_3640: u32, - pub(crate) unk_3644: u32, - pub(crate) hws1: HwDataShared1, - - #[ver(V >= V13_0B4)] - pub(crate) unk_hws2: Array<16, u16>, - - pub(crate) hws2: HwDataShared2, - pub(crate) unk_3c00: u32, - pub(crate) unk_3c04: u32, - pub(crate) hws3: HwDataShared3, - pub(crate) unk_3c58: Array<0x3c, u8>, - pub(crate) unk_3c94: u32, - pub(crate) unk_3c98: U64, - pub(crate) unk_3ca0: U64, - pub(crate) unk_3ca8: U64, - pub(crate) unk_3cb0: U64, - pub(crate) ts_last_idle: U64, - pub(crate) ts_last_poweron: U64, - pub(crate) ts_last_poweroff: U64, - pub(crate) unk_3cd0: U64, - pub(crate) unk_3cd8: U64, - - #[ver(V >= V13_0B4)] - pub(crate) unk_3ce0_0: u32, - - pub(crate) unk_3ce0: u32, - pub(crate) unk_3ce4: u32, - pub(crate) unk_3ce8: u32, - pub(crate) unk_3cec: u32, - pub(crate) unk_3cf0: u32, - pub(crate) core_leak_coef: Array<8, F32>, - pub(crate) sram_leak_coef: Array<8, F32>, - - #[ver(V >= V13_0B4)] - pub(crate) aux_leak_coef: AuxLeakCoef, - #[ver(V >= V13_0B4)] - pub(crate) unk_3d34_0: Array<0x18, u8>, - - pub(crate) unk_3d34: Array<0x38, u8>, - } - #[versions(AGX_NEO)] - default_zeroed!(HwDataA::ver); - #[versions(AGX_NEO)] - no_debug!(HwDataA::ver); - - #[derive(Debug, Default, Clone, Copy)] - #[repr(C)] - pub(crate) struct IOMapping { - pub(crate) phys_addr: U64, - pub(crate) virt_addr: U64, - pub(crate) total_size: u32, - pub(crate) element_size: u32, - pub(crate) readwrite: U64, - } - - #[versions(AGX_NEO)] - const IO_MAPPING_COUNT: usize = { - #[ver(V < V13_0B4)] - { - 0x14 - } - #[ver(V >= V13_0B4 && V < V13_3)] - { - 0x17 - } - #[ver(V >= V13_3 && V < V13_5)] - { - 0x18 - } - #[ver(V >= V13_5)] - { - 0x19 - } - }; - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct HwDataBAuxPStates { - pub(crate) cs_max_pstate: u32, - pub(crate) cs_frequencies: Array<0x10, u32>, - pub(crate) cs_voltages: Array<0x10, Array<0x2, u32>>, - pub(crate) cs_voltages_sram: Array<0x10, Array<0x2, u32>>, - pub(crate) cs_unkpad: u32, - pub(crate) afr_max_pstate: u32, - pub(crate) afr_frequencies: Array<0x8, u32>, - pub(crate) afr_voltages: Array<0x8, Array<0x2, u32>>, - pub(crate) afr_voltages_sram: Array<0x8, Array<0x2, u32>>, - pub(crate) afr_unkpad: u32, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct HwDataB { - #[ver(V < V13_0B4)] - pub(crate) unk_0: U64, - - pub(crate) unk_8: U64, - - #[ver(V < V13_0B4)] - pub(crate) unk_10: U64, - - pub(crate) unk_18: U64, - pub(crate) unk_20: U64, - pub(crate) unk_28: U64, - pub(crate) unk_30: U64, - pub(crate) timestamp_area_base: U64, - pub(crate) pad_40: Pad<0x20>, - - #[ver(V < V13_0B4)] - pub(crate) yuv_matrices: Array<0xf, Array<3, Array<4, i16>>>, - - #[ver(V >= V13_0B4)] - pub(crate) yuv_matrices: Array<0x3f, Array<3, Array<4, i16>>>, - - pub(crate) pad_1c8: Pad<0x8>, - pub(crate) io_mappings: Array, - - #[ver(V >= V13_0B4)] - pub(crate) sgx_sram_ptr: U64, - - pub(crate) chip_id: u32, - pub(crate) unk_454: u32, - pub(crate) unk_458: u32, - pub(crate) unk_45c: u32, - pub(crate) unk_460: u32, - pub(crate) unk_464: u32, - pub(crate) unk_468: u32, - pub(crate) unk_46c: u32, - pub(crate) unk_470: u32, - pub(crate) unk_474: u32, - pub(crate) unk_478: u32, - pub(crate) unk_47c: u32, - pub(crate) unk_480: u32, - pub(crate) unk_484: u32, - pub(crate) unk_488: u32, - pub(crate) unk_48c: u32, - pub(crate) base_clock_khz: u32, - pub(crate) power_sample_period: u32, - pub(crate) pad_498: Pad<0x4>, - pub(crate) unk_49c: u32, - pub(crate) unk_4a0: u32, - pub(crate) unk_4a4: u32, - pub(crate) pad_4a8: Pad<0x4>, - pub(crate) unk_4ac: u32, - pub(crate) pad_4b0: Pad<0x8>, - pub(crate) unk_4b8: u32, - pub(crate) unk_4bc: Array<0x4, u8>, - pub(crate) unk_4c0: u32, - pub(crate) unk_4c4: u32, - pub(crate) unk_4c8: u32, - pub(crate) unk_4cc: u32, - pub(crate) unk_4d0: u32, - pub(crate) unk_4d4: u32, - pub(crate) unk_4d8: Array<0x4, u8>, - pub(crate) unk_4dc: u32, - pub(crate) unk_4e0: U64, - pub(crate) unk_4e8: u32, - pub(crate) unk_4ec: u32, - pub(crate) unk_4f0: u32, - pub(crate) unk_4f4: u32, - pub(crate) unk_4f8: u32, - pub(crate) unk_4fc: u32, - pub(crate) unk_500: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_504_0: u32, - - pub(crate) unk_504: u32, - pub(crate) unk_508: u32, - pub(crate) unk_50c: u32, - pub(crate) unk_510: u32, - pub(crate) unk_514: u32, - pub(crate) unk_518: u32, - pub(crate) unk_51c: u32, - pub(crate) unk_520: u32, - pub(crate) unk_524: u32, - pub(crate) unk_528: u32, - pub(crate) unk_52c: u32, - pub(crate) unk_530: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_534_0: u32, - - pub(crate) unk_534: u32, - pub(crate) unk_538: u32, - - pub(crate) num_frags: u32, - pub(crate) unk_540: u32, - pub(crate) unk_544: u32, - pub(crate) unk_548: u32, - pub(crate) unk_54c: u32, - pub(crate) unk_550: u32, - pub(crate) unk_554: u32, - pub(crate) uat_ttb_base: U64, - pub(crate) gpu_core_id: u32, - pub(crate) gpu_rev_id: u32, - pub(crate) num_cores: u32, - pub(crate) max_pstate: u32, - - #[ver(V < V13_0B4)] - pub(crate) num_pstates: u32, - - pub(crate) frequencies: Array<0x10, u32>, - pub(crate) voltages: Array<0x10, [u32; 0x8]>, - pub(crate) voltages_sram: Array<0x10, [u32; 0x8]>, - - #[ver(V >= V13_3)] - pub(crate) unk_9f4_0: Pad<64>, - - pub(crate) sram_k: Array<0x10, F32>, - pub(crate) unk_9f4: Array<0x10, u32>, - pub(crate) rel_max_powers: Array<0x10, u32>, - pub(crate) rel_boost_freqs: Array<0x10, u32>, - - #[ver(V >= V13_3)] - pub(crate) unk_arr_0: Array<32, u32>, - - #[ver(V < V13_0B4)] - pub(crate) min_sram_volt: u32, - - #[ver(V < V13_0B4)] - pub(crate) unk_ab8: u32, - - #[ver(V < V13_0B4)] - pub(crate) unk_abc: u32, - - #[ver(V < V13_0B4)] - pub(crate) unk_ac0: u32, - - #[ver(V >= V13_0B4)] - pub(crate) aux_ps: HwDataBAuxPStates, - - #[ver(V >= V13_3)] - pub(crate) pad_ac4_0: Array<0x44c, u8>, - - pub(crate) pad_ac4: Pad<0x8>, - pub(crate) unk_acc: u32, - pub(crate) unk_ad0: u32, - pub(crate) pad_ad4: Pad<0x10>, - pub(crate) unk_ae4: Array<0x4, u32>, - pub(crate) pad_af4: Pad<0x4>, - pub(crate) unk_af8: u32, - pub(crate) pad_afc: Pad<0x8>, - pub(crate) unk_b04: u32, - pub(crate) unk_b08: u32, - pub(crate) unk_b0c: u32, - - #[ver(G >= G14X)] - pub(crate) pad_b10_0: Array<0x8, u8>, - - pub(crate) unk_b10: u32, - pub(crate) timer_offset: U64, - pub(crate) unk_b1c: u32, - pub(crate) unk_b20: u32, - pub(crate) unk_b24: u32, - pub(crate) unk_b28: u32, - pub(crate) unk_b2c: u32, - pub(crate) unk_b30: u32, - pub(crate) unk_b34: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_b38_0: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_b38_4: u32, - - #[ver(V >= V13_3)] - pub(crate) unk_b38_8: u32, - - pub(crate) unk_b38: Array<0xc, u32>, - pub(crate) unk_b68: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_b6c: Array<0xd0, u8>, - - #[ver(G >= G14X)] - pub(crate) unk_c3c_0: Array<0x8, u8>, - - #[ver(G < G14X && V >= V13_5)] - pub(crate) unk_c3c_8: Array<0x10, u8>, - - #[ver(V >= V13_5)] - pub(crate) unk_c3c_18: Array<0x20, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_c3c: u32, - } - #[versions(AGX_NEO)] - default_zeroed!(HwDataB::ver); - - #[derive(Debug)] - #[repr(C, packed)] - pub(crate) struct GpuStatsVtx { - // This changes all the time and we don't use it, let's just make it a big buffer - pub(crate) opaque: Array<0x3000, u8>, - } - default_zeroed!(GpuStatsVtx); - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct GpuStatsFrag { - // This changes all the time and we don't use it, let's just make it a big buffer - // except for these two fields which may need init. - #[ver(G >= G14X)] - pub(crate) unk1_0: Array<0x910, u8>, - pub(crate) unk1: Array<0x100, u8>, - pub(crate) cur_stamp_id: i32, - pub(crate) unk2: Array<0x14, u8>, - pub(crate) unk_id: i32, - pub(crate) unk3: Array<0x1000, u8>, - } - - #[versions(AGX_NEO)] - impl Default for GpuStatsFrag::ver { - fn default() -> Self { - Self { - #[ver(G >= G14X)] - unk1_0: Default::default(), - unk1: Default::default(), - cur_stamp_id: -1, - unk2: Default::default(), - unk_id: -1, - unk3: Default::default(), - } - } - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct GpuGlobalStatsVtx { - pub(crate) total_cmds: u32, - pub(crate) stats: GpuStatsVtx, - } - default_zeroed!(GpuGlobalStatsVtx); - - #[versions(AGX_NEO)] - #[derive(Debug, Default)] - #[repr(C)] - pub(crate) struct GpuGlobalStatsFrag { - pub(crate) total_cmds: u32, - pub(crate) unk_4: u32, - pub(crate) stats: GpuStatsFrag::ver, - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct GpuStatsComp { - // This changes all the time and we don't use it, let's just make it a big buffer - pub(crate) opaque: Array<0x3000, u8>, - } - default_zeroed!(GpuStatsComp); - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct RuntimeScratch { - pub(crate) unk_280: Array<0x6800, u8>, - pub(crate) unk_6a80: u32, - pub(crate) gpu_idle: u32, - pub(crate) unkpad_6a88: Pad<0x14>, - pub(crate) unk_6a9c: u32, - pub(crate) unk_ctr0: u32, - pub(crate) unk_ctr1: u32, - pub(crate) unk_6aa8: u32, - pub(crate) unk_6aac: u32, - pub(crate) unk_ctr2: u32, - pub(crate) unk_6ab4: u32, - pub(crate) unk_6ab8: u32, - pub(crate) unk_6abc: u32, - pub(crate) unk_6ac0: u32, - pub(crate) unk_6ac4: u32, - pub(crate) unk_ctr3: u32, - pub(crate) unk_6acc: u32, - pub(crate) unk_6ad0: u32, - pub(crate) unk_6ad4: u32, - pub(crate) unk_6ad8: u32, - pub(crate) unk_6adc: u32, - pub(crate) unk_6ae0: u32, - pub(crate) unk_6ae4: u32, - pub(crate) unk_6ae8: u32, - pub(crate) unk_6aec: u32, - pub(crate) unk_6af0: u32, - pub(crate) unk_ctr4: u32, - pub(crate) unk_ctr5: u32, - pub(crate) unk_6afc: u32, - pub(crate) pad_6b00: Pad<0x38>, - - #[ver(G >= G14X)] - pub(crate) pad_6b00_extra: Array<0x4800, u8>, - - pub(crate) unk_6b38: u32, - pub(crate) pad_6b3c: Pad<0x84>, - } - #[versions(AGX_NEO)] - default_zeroed!(RuntimeScratch::ver); - - #[versions(AGX_NEO)] - #[repr(C)] - pub(crate) struct RuntimePointers<'a> { - pub(crate) pipes: Array<4, PipeChannels::ver>, - - pub(crate) device_control: - ChannelRing, - pub(crate) event: ChannelRing, - pub(crate) fw_log: ChannelRing, - pub(crate) ktrace: ChannelRing, - pub(crate) stats: ChannelRing, - - pub(crate) __pad0: Pad<0x50>, - pub(crate) unk_160: U64, - pub(crate) unk_168: U64, - pub(crate) stats_vtx: GpuPointer<'a, super::GpuGlobalStatsVtx>, - pub(crate) stats_frag: GpuPointer<'a, super::GpuGlobalStatsFrag::ver>, - pub(crate) stats_comp: GpuPointer<'a, super::GpuStatsComp>, - pub(crate) hwdata_a: GpuPointer<'a, super::HwDataA::ver>, - pub(crate) unkptr_190: GpuPointer<'a, &'a [u8]>, - pub(crate) unkptr_198: GpuPointer<'a, &'a [u8]>, - pub(crate) hwdata_b: GpuPointer<'a, super::HwDataB::ver>, - pub(crate) hwdata_b_2: GpuPointer<'a, super::HwDataB::ver>, - pub(crate) fwlog_buf: Option>, - pub(crate) unkptr_1b8: GpuPointer<'a, &'a [u8]>, - - #[ver(G < G14X)] - pub(crate) unkptr_1c0: GpuPointer<'a, &'a [u8]>, - #[ver(G < G14X)] - pub(crate) unkptr_1c8: GpuPointer<'a, &'a [u8]>, - - pub(crate) unk_1d0: u32, - pub(crate) unk_1d4: u32, - pub(crate) unk_1d8: Array<0x3c, u8>, - pub(crate) buffer_mgr_ctl_gpu_addr: U64, - pub(crate) buffer_mgr_ctl_fw_addr: U64, - pub(crate) __pad1: Pad<0x5c>, - pub(crate) gpu_scratch: RuntimeScratch::ver, - } - #[versions(AGX_NEO)] - no_debug!(RuntimePointers::ver<'_>); - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct PendingStamp { - pub(crate) info: AtomicU32, - pub(crate) wait_value: AtomicU32, - } - default_zeroed!(PendingStamp); - - #[derive(Debug, Clone, Copy)] - #[repr(C, packed)] - pub(crate) struct FaultInfo { - pub(crate) unk_0: u32, - pub(crate) unk_4: u32, - pub(crate) queue_uuid: u32, - pub(crate) unk_c: u32, - pub(crate) unk_10: u32, - pub(crate) unk_14: u32, - } - default_zeroed!(FaultInfo); - - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct PowerZoneGlobal { - pub(crate) target: u32, - pub(crate) target_off: u32, - pub(crate) filter_tc: u32, - } - default_zeroed!(PowerZoneGlobal); - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct Globals { - pub(crate) ktrace_enable: u32, - pub(crate) unk_4: Array<0x20, u8>, - - #[ver(V >= V13_2)] - pub(crate) unk_24_0: u32, - - pub(crate) unk_24: u32, - - #[ver(V >= V13_0B4)] - pub(crate) debug: u32, - - #[ver(V >= V13_3)] - pub(crate) unk_28_4: u32, - - pub(crate) unk_28: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_2c_0: u32, - - pub(crate) unk_2c: u32, - pub(crate) unk_30: u32, - pub(crate) unk_34: u32, - pub(crate) unk_38: Array<0x1c, u8>, - - // pub(crate) sub: GlobalsSub::ver, - pub(crate) unk_54: u16, - pub(crate) unk_56: u16, - pub(crate) unk_58: u16, - pub(crate) unk_5a: U32, - pub(crate) unk_5e: U32, - pub(crate) unk_62: U32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_66_0: Array<0xc, u8>, - - pub(crate) unk_66: U32, - pub(crate) unk_6a: Array<0x16, u8>, - // end GlobalsSub::ver - - pub(crate) unk_80: Array<0xf80, u8>, - pub(crate) unk_1000: Array<0x7000, u8>, - pub(crate) unk_8000: Array<0x900, u8>, - - #[ver(G >= G14X)] - pub(crate) unk_8900_pad: Array<0x484c, u8>, - - #[ver(V >= V13_3)] - pub(crate) unk_8900_pad2: Array<0x54, u8>, - - pub(crate) unk_8900: u32, - pub(crate) pending_submissions: AtomicU32, - pub(crate) max_power: u32, - pub(crate) max_pstate_scaled: u32, - pub(crate) max_pstate_scaled_2: u32, - pub(crate) unk_8914: u32, - pub(crate) unk_8918: u32, - pub(crate) max_pstate_scaled_3: u32, - pub(crate) unk_8920: u32, - pub(crate) power_zone_count: u32, - pub(crate) avg_power_filter_tc_periods: u32, - pub(crate) avg_power_ki_dt: F32, - pub(crate) avg_power_kp: F32, - pub(crate) avg_power_min_duty_cycle: u32, - pub(crate) avg_power_target_filter_tc: u32, - pub(crate) power_zones: Array<5, PowerZoneGlobal>, - pub(crate) unk_8978: Array<0x44, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_89bc_0: Array<0x3c, u8>, - - pub(crate) unk_89bc: u32, - pub(crate) fast_die0_release_temp: u32, - pub(crate) unk_89c4: i32, - pub(crate) fast_die0_prop_tgt_delta: u32, - pub(crate) fast_die0_kp: F32, - pub(crate) fast_die0_ki_dt: F32, - pub(crate) unk_89d4: Array<0xc, u8>, - pub(crate) unk_89e0: u32, - pub(crate) max_power_2: u32, - pub(crate) ppm_kp: F32, - pub(crate) ppm_ki_dt: F32, - pub(crate) unk_89f0: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_89f4_0: Array<0x8, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_89f4_8: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_89f4_c: Array<0x50, u8>, - - #[ver(V >= V13_3)] - pub(crate) unk_89f4_5c: Array<0xc, u8>, - - pub(crate) unk_89f4: u32, - pub(crate) hws1: HwDataShared1, - pub(crate) hws2: HwDataShared2, - - #[ver(V >= V13_0B4)] - pub(crate) idle_off_standby_timer: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_hws2_4: Array<0x8, F32>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_hws2_24: u32, - - pub(crate) unk_hws2_28: u32, - - pub(crate) hws3: HwDataShared3, - pub(crate) unk_9004: Array<8, u8>, - pub(crate) unk_900c: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_9010_0: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_9010_4: Array<0x14, u8>, - - pub(crate) unk_9010: Array<0x2c, u8>, - pub(crate) unk_903c: u32, - pub(crate) unk_9040: Array<0xc0, u8>, - pub(crate) unk_9100: Array<0x6f00, u8>, - pub(crate) unk_10000: Array<0xe50, u8>, - pub(crate) unk_10e50: u32, - pub(crate) unk_10e54: Array<0x2c, u8>, - - #[ver((G >= G14X && V < V13_3) || (G <= G14 && V >= V13_3))] - pub(crate) unk_x_pad: Array<0x4, u8>, - - // bit 0: sets sgx_reg 0x17620 - // bit 1: sets sgx_reg 0x17630 - pub(crate) fault_control: u32, - pub(crate) do_init: u32, - pub(crate) unk_10e88: Array<0x188, u8>, - pub(crate) idle_ts: U64, - pub(crate) idle_unk: U64, - pub(crate) progress_check_interval_3d: u32, - pub(crate) progress_check_interval_ta: u32, - pub(crate) progress_check_interval_cl: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_1102c_0: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_1102c_4: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_1102c_8: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_1102c_c: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_1102c_10: u32, - - pub(crate) unk_1102c: u32, - pub(crate) idle_off_delay_ms: AtomicU32, - pub(crate) fender_idle_off_delay_ms: u32, - pub(crate) fw_early_wake_timeout_ms: u32, - #[ver(V == V13_3)] - pub(crate) ps_pad_0: Pad<0x8>, - pub(crate) pending_stamps: Array<0x100, PendingStamp>, - #[ver(V != V13_3)] - pub(crate) ps_pad_0: Pad<0x8>, - pub(crate) unkpad_ps: Pad<0x78>, - pub(crate) unk_117bc: u32, - pub(crate) fault_info: FaultInfo, - pub(crate) counter: u32, - pub(crate) unk_118dc: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_118e0_0: Array<0x9c, u8>, - - #[ver(G >= G14X)] - pub(crate) unk_118e0_9c: Array<0x580, u8>, - - #[ver(V >= V13_3)] - pub(crate) unk_118e0_9c_x: Array<0x8, u8>, - - pub(crate) cl_context_switch_timeout_ms: u32, - - #[ver(V >= V13_0B4)] - pub(crate) cl_kill_timeout_ms: u32, - - pub(crate) cdm_context_store_latency_threshold: u32, - pub(crate) unk_118e8: u32, - pub(crate) unk_118ec: Array<0x400, u8>, - pub(crate) unk_11cec: Array<0x54, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_11d40: Array<0x19c, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_11edc: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_11ee0: Array<0x1c, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_11efc: u32, - - #[ver(V >= V13_3)] - pub(crate) unk_11f00: Array<0x280, u8>, - } - #[versions(AGX_NEO)] - default_zeroed!(Globals::ver); - - #[derive(Debug, Default, Clone, Copy)] - #[repr(C, packed)] - pub(crate) struct UatLevelInfo { - pub(crate) unk_3: u8, - pub(crate) unk_1: u8, - pub(crate) unk_2: u8, - pub(crate) index_shift: u8, - pub(crate) num_entries: u16, - pub(crate) unk_4: u16, - pub(crate) unk_8: U64, - pub(crate) unk_10: U64, - pub(crate) index_mask: U64, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct InitData<'a> { - #[ver(V >= V13_0B4)] - pub(crate) ver_info: Array<0x4, u16>, - - pub(crate) unk_buf: GpuPointer<'a, &'a [u8]>, - pub(crate) unk_8: u32, - pub(crate) unk_c: u32, - pub(crate) runtime_pointers: GpuPointer<'a, super::RuntimePointers::ver>, - pub(crate) globals: GpuPointer<'a, super::Globals::ver>, - pub(crate) fw_status: GpuPointer<'a, super::FwStatus>, - pub(crate) uat_page_size: u16, - pub(crate) uat_page_bits: u8, - pub(crate) uat_num_levels: u8, - pub(crate) uat_level_info: Array<0x3, UatLevelInfo>, - pub(crate) __pad0: Pad<0x14>, - pub(crate) host_mapped_fw_allocations: u32, - pub(crate) unk_ac: u32, - pub(crate) unk_b0: u32, - pub(crate) unk_b4: u32, - pub(crate) unk_b8: u32, - } -} - -#[derive(Debug)] -pub(crate) struct ChannelRing -where - for<'a> ::Raw<'a>: Debug, -{ - pub(crate) state: GpuObject, - pub(crate) ring: GpuArray, -} - -impl ChannelRing -where - for<'a> ::Raw<'a>: Debug, -{ - pub(crate) fn to_raw(&self) -> raw::ChannelRing { - raw::ChannelRing { - state: Some(self.state.weak_pointer()), - ring: Some(self.ring.weak_pointer()), - } - } -} - -trivial_gpustruct!(FwStatus); -trivial_gpustruct!(GpuGlobalStatsVtx); -#[versions(AGX_NEO)] -trivial_gpustruct!(GpuGlobalStatsFrag::ver); -trivial_gpustruct!(GpuStatsComp); - -#[versions(AGX_NEO)] -trivial_gpustruct!(HwDataA::ver); - -#[versions(AGX_NEO)] -trivial_gpustruct!(HwDataB::ver); - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct Stats { - pub(crate) vtx: GpuObject, - pub(crate) frag: GpuObject, - pub(crate) comp: GpuObject, -} - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct RuntimePointers { - pub(crate) stats: Stats::ver, - - pub(crate) hwdata_a: GpuObject, - pub(crate) unkptr_190: GpuArray, - pub(crate) unkptr_198: GpuArray, - pub(crate) hwdata_b: GpuObject, - - pub(crate) unkptr_1b8: GpuArray, - pub(crate) unkptr_1c0: GpuArray, - pub(crate) unkptr_1c8: GpuArray, - - pub(crate) buffer_mgr_ctl: gem::ObjectRef, - pub(crate) buffer_mgr_ctl_low_mapping: Option, - pub(crate) buffer_mgr_ctl_high_mapping: Option, -} - -#[versions(AGX_NEO)] -impl GpuStruct for RuntimePointers::ver { - type Raw<'a> = raw::RuntimePointers::ver<'a>; -} - -#[versions(AGX_NEO)] -trivial_gpustruct!(Globals::ver); - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct InitData { - pub(crate) unk_buf: GpuArray, - pub(crate) runtime_pointers: GpuObject, - pub(crate) globals: GpuObject, - pub(crate) fw_status: GpuObject, -} - -#[versions(AGX_NEO)] -impl GpuStruct for InitData::ver { - type Raw<'a> = raw::InitData::ver<'a>; -} diff --git a/drivers/gpu/drm/asahi-neo/fw/job.rs b/drivers/gpu/drm/asahi-neo/fw/job.rs deleted file mode 100644 index e4f2f9225ea050..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/job.rs +++ /dev/null @@ -1,160 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Common GPU job firmware structures - -use super::types::*; -use crate::{ - default_zeroed, - mmu, - trivial_gpustruct, // -}; -use kernel::prelude::Result; -use kernel::sync::Arc; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct JobMeta { - pub(crate) unk_0: u16, - pub(crate) unk_2: u8, - pub(crate) no_preemption: u8, - pub(crate) stamp: GpuWeakPointer, - pub(crate) fw_stamp: GpuWeakPointer, - pub(crate) stamp_value: EventValue, - pub(crate) stamp_slot: u32, - pub(crate) evctl_index: u32, - pub(crate) flush_stamps: u32, - pub(crate) uuid: u32, - pub(crate) event_seq: u32, - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct EncoderParams { - pub(crate) unk_8: u32, - pub(crate) sync_grow: u32, - pub(crate) unk_10: u32, - pub(crate) encoder_id: u32, - pub(crate) unk_18: u32, - pub(crate) unk_mask: u32, - pub(crate) sampler_array: U64, - pub(crate) sampler_count: u32, - pub(crate) sampler_max: u32, - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobTimestamps { - pub(crate) start: AtomicU64, - pub(crate) end: AtomicU64, - } - default_zeroed!(JobTimestamps); - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct RenderTimestamps { - pub(crate) vtx: JobTimestamps, - pub(crate) frag: JobTimestamps, - } - default_zeroed!(RenderTimestamps); - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct Register { - pub(crate) number: u32, - pub(crate) value: U64, - } - default_zeroed!(Register); - - impl Register { - fn new(number: u32, value: u64) -> Register { - Register { - number, - value: U64(value), - } - } - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct RegisterArray { - pub(crate) registers: Array<128, Register>, - pub(crate) pad: Array<0x100, u8>, - - pub(crate) addr: GpuWeakPointer>, - pub(crate) count: u16, - pub(crate) length: u16, - pub(crate) unk_pad: u32, - } - - impl RegisterArray { - pub(crate) fn new( - self_ptr: GpuWeakPointer>, - cb: impl FnOnce(&mut RegisterArray), - ) -> RegisterArray { - let mut array = RegisterArray { - registers: Default::default(), - pad: Default::default(), - addr: self_ptr, - count: 0, - length: 0, - unk_pad: 0, - }; - - cb(&mut array); - - array - } - - pub(crate) fn add(&mut self, number: u32, value: u64) { - self.registers[self.count as usize] = Register::new(number, value); - self.count += 1; - self.length += core::mem::size_of::() as u16; - } - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct TimestampPointers<'a> { - pub(crate) start_addr: Option>, - pub(crate) end_addr: Option>, - } -} - -trivial_gpustruct!(JobTimestamps); -trivial_gpustruct!(RenderTimestamps); - -#[derive(Debug)] -pub(crate) struct UserTimestamp { - pub(crate) mapping: Arc, - pub(crate) offset: usize, -} - -#[derive(Debug, Default)] -pub(crate) struct UserTimestamps { - pub(crate) start: Option, - pub(crate) end: Option, -} - -impl UserTimestamps { - pub(crate) fn any(&self) -> bool { - self.start.is_some() || self.end.is_some() - } - - pub(crate) fn pointers(&self) -> Result> { - Ok(raw::TimestampPointers { - start_addr: self - .start - .as_ref() - .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) - .transpose()?, - end_addr: self - .end - .as_ref() - .map(|a| GpuPointer::from_mapping(&a.mapping, a.offset)) - .transpose()?, - }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/fw/microseq.rs b/drivers/gpu/drm/asahi-neo/fw/microseq.rs deleted file mode 100644 index af46b1073af7d9..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/microseq.rs +++ /dev/null @@ -1,412 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU firmware microsequence operations - -use super::types::*; -use super::{ - buffer, - compute, - fragment, - initdata, - job, - vertex, - workqueue, // -}; -use crate::default_zeroed; - -pub(crate) trait Operation {} - -#[derive(Debug, Copy, Clone)] -#[repr(u32)] -enum OpCode { - WaitForIdle = 0x01, - WaitForIdle2 = 0x02, - RetireStamp = 0x18, - #[allow(dead_code)] - Timestamp = 0x19, - StartVertex = 0x22, - FinalizeVertex = 0x23, - StartFragment = 0x24, - FinalizeFragment = 0x25, - StartCompute = 0x29, - FinalizeCompute = 0x2a, -} - -#[derive(Debug, Copy, Clone)] -#[repr(u32)] -pub(crate) enum Pipe { - Vertex = 1 << 0, - Fragment = 1 << 8, - Compute = 1 << 15, -} - -pub(crate) const MAX_ATTACHMENTS: usize = 16; - -#[derive(Debug, Clone, Copy)] -#[repr(C)] -pub(crate) struct Attachment { - pub(crate) address: U64, - pub(crate) size: u32, - pub(crate) unk_c: u16, - pub(crate) unk_e: u16, -} -default_zeroed!(Attachment); - -#[derive(Debug, Clone, Copy, Default)] -#[repr(C)] -pub(crate) struct Attachments { - pub(crate) list: Array, - pub(crate) count: u32, -} - -#[derive(Debug, Copy, Clone)] -#[repr(transparent)] -pub(crate) struct OpHeader(u32); - -impl OpHeader { - const fn new(opcode: OpCode) -> OpHeader { - OpHeader(opcode as u32) - } - const fn with_args(opcode: OpCode, args: u32) -> OpHeader { - OpHeader(opcode as u32 | args) - } -} - -macro_rules! simple_op { - ($name:ident) => { - #[allow(dead_code)] - #[derive(Debug, Copy, Clone)] - pub(crate) struct $name(OpHeader); - - impl $name { - pub(crate) const HEADER: $name = $name(OpHeader::new(OpCode::$name)); - } - }; -} - -pub(crate) mod op { - use super::*; - - simple_op!(StartVertex); - simple_op!(FinalizeVertex); - simple_op!(StartFragment); - simple_op!(FinalizeFragment); - simple_op!(StartCompute); - simple_op!(FinalizeCompute); - simple_op!(WaitForIdle2); - - #[allow(dead_code)] - #[derive(Debug, Copy, Clone)] - pub(crate) struct RetireStamp(OpHeader); - impl RetireStamp { - pub(crate) const HEADER: RetireStamp = - RetireStamp(OpHeader::with_args(OpCode::RetireStamp, 0x40000000)); - } - - #[allow(dead_code)] - #[derive(Debug, Copy, Clone)] - pub(crate) struct WaitForIdle(OpHeader); - impl WaitForIdle { - pub(crate) const fn new(pipe: Pipe) -> WaitForIdle { - WaitForIdle(OpHeader::with_args(OpCode::WaitForIdle, (pipe as u32) << 8)) - } - } - - #[allow(dead_code)] - #[derive(Debug, Copy, Clone)] - pub(crate) struct Timestamp(OpHeader); - impl Timestamp { - #[allow(dead_code)] - pub(crate) const fn new(flag: bool) -> Timestamp { - Timestamp(OpHeader::with_args(OpCode::Timestamp, (flag as u32) << 31)) - } - } -} - -#[derive(Debug)] -#[repr(C)] -pub(crate) struct WaitForIdle { - pub(crate) header: op::WaitForIdle, -} - -impl Operation for WaitForIdle {} - -#[derive(Debug)] -#[repr(C)] -pub(crate) struct WaitForIdle2 { - pub(crate) header: op::WaitForIdle2, -} - -impl Operation for WaitForIdle2 {} - -#[derive(Debug)] -#[repr(C)] -pub(crate) struct RetireStamp { - pub(crate) header: op::RetireStamp, -} - -impl Operation for RetireStamp {} - -#[versions(AGX_NEO)] -#[derive(Debug)] -#[repr(C)] -pub(crate) struct Timestamp<'a> { - pub(crate) header: op::Timestamp, - pub(crate) command_time: GpuWeakPointer, - pub(crate) ts_pointers: GpuWeakPointer>, - // Unused? - pub(crate) update_ts: GpuWeakPointer>>, - pub(crate) work_queue: GpuWeakPointer, - pub(crate) user_ts_pointers: GpuWeakPointer>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_ts: GpuWeakPointer, - - pub(crate) uuid: u32, - pub(crate) unk_30_padding: u32, -} - -#[versions(AGX_NEO)] -impl<'a> Operation for Timestamp::ver<'a> {} - -#[versions(AGX_NEO)] -#[derive(Debug)] -#[repr(C)] -pub(crate) struct StartVertex<'a> { - pub(crate) header: op::StartVertex, - pub(crate) tiling_params: Option>, - pub(crate) job_params1: Option>>, - #[ver(G >= G14X)] - pub(crate) registers: GpuWeakPointer, - pub(crate) buffer: GpuWeakPointer, - pub(crate) scene: GpuWeakPointer, - pub(crate) stats: GpuWeakPointer, - pub(crate) work_queue: GpuWeakPointer, - pub(crate) vm_slot: u32, - pub(crate) unk_38: u32, - pub(crate) event_generation: u32, - pub(crate) buffer_slot: u32, - pub(crate) unk_44: u32, - pub(crate) event_seq: U64, - pub(crate) unk_50: u32, - pub(crate) unk_pointer: GpuWeakPointer, - pub(crate) unk_job_buf: GpuWeakPointer, - pub(crate) unk_64: u32, - pub(crate) unk_68: u32, - pub(crate) uuid: u32, - pub(crate) attachments: Attachments, - pub(crate) padding: u32, - - #[ver(V >= V13_0B4)] - pub(crate) counter: U64, - - #[ver(V >= V13_0B4)] - pub(crate) notifier_buf: GpuWeakPointer>, - - pub(crate) unk_178: u32, -} - -#[versions(AGX_NEO)] -impl<'a> Operation for StartVertex::ver<'a> {} - -#[versions(AGX_NEO)] -#[derive(Debug)] -#[repr(C)] -pub(crate) struct FinalizeVertex { - pub(crate) header: op::FinalizeVertex, - pub(crate) scene: GpuWeakPointer, - pub(crate) buffer: GpuWeakPointer, - pub(crate) stats: GpuWeakPointer, - pub(crate) work_queue: GpuWeakPointer, - pub(crate) vm_slot: u32, - pub(crate) unk_28: u32, - pub(crate) unk_pointer: GpuWeakPointer, - pub(crate) unk_34: u32, - pub(crate) uuid: u32, - pub(crate) fw_stamp: GpuWeakPointer, - pub(crate) stamp_value: EventValue, - pub(crate) unk_48: U64, - pub(crate) unk_50: u32, - pub(crate) unk_54: u32, - pub(crate) unk_58: U64, - pub(crate) unk_60: u32, - pub(crate) unk_64: u32, - pub(crate) unk_68: u32, - - #[ver(G >= G14 && V < V13_0B4)] - pub(crate) unk_68_g14: U64, - - pub(crate) restart_branch_offset: i32, - pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic - - #[ver(V >= V13_0B4)] - pub(crate) unk_74: Array<0x10, u8>, -} - -#[versions(AGX_NEO)] -impl Operation for FinalizeVertex::ver {} - -#[versions(AGX_NEO)] -#[derive(Debug)] -#[repr(C)] -pub(crate) struct StartFragment<'a> { - pub(crate) header: op::StartFragment, - pub(crate) job_params2: Option>, - pub(crate) job_params1: Option>>, - #[ver(G >= G14X)] - pub(crate) registers: GpuWeakPointer, - pub(crate) scene: GpuPointer<'a, buffer::Scene::ver>, - pub(crate) stats: GpuWeakPointer, - pub(crate) busy_flag: GpuWeakPointer, - pub(crate) tvb_overflow_count: GpuWeakPointer, - pub(crate) unk_pointer: GpuWeakPointer, - pub(crate) work_queue: GpuWeakPointer, - pub(crate) work_item: GpuWeakPointer, - pub(crate) vm_slot: u32, - pub(crate) unk_50: u32, - pub(crate) event_generation: u32, - pub(crate) buffer_slot: u32, - pub(crate) sync_grow: u32, - pub(crate) event_seq: U64, - pub(crate) unk_68: u32, - pub(crate) unk_758_flag: GpuWeakPointer, - pub(crate) unk_job_buf: GpuWeakPointer, - #[ver(V >= V13_3)] - pub(crate) unk_7c_0: U64, - pub(crate) unk_7c: u32, - pub(crate) unk_80: u32, - pub(crate) unk_84: u32, - pub(crate) uuid: u32, - pub(crate) attachments: Attachments, - pub(crate) padding: u32, - - #[ver(V >= V13_0B4)] - pub(crate) counter: U64, - - #[ver(V >= V13_0B4)] - pub(crate) notifier_buf: GpuWeakPointer>, -} - -#[versions(AGX_NEO)] -impl<'a> Operation for StartFragment::ver<'a> {} - -#[versions(AGX_NEO)] -#[derive(Debug)] -#[repr(C)] -pub(crate) struct FinalizeFragment { - pub(crate) header: op::FinalizeFragment, - pub(crate) uuid: u32, - pub(crate) unk_8: u32, - pub(crate) fw_stamp: GpuWeakPointer, - pub(crate) stamp_value: EventValue, - pub(crate) unk_18: u32, - pub(crate) scene: GpuWeakPointer, - pub(crate) buffer: GpuWeakPointer, - pub(crate) unk_2c: U64, - pub(crate) stats: GpuWeakPointer, - pub(crate) unk_pointer: GpuWeakPointer, - pub(crate) busy_flag: GpuWeakPointer, - pub(crate) work_queue: GpuWeakPointer, - pub(crate) work_item: GpuWeakPointer, - pub(crate) vm_slot: u32, - pub(crate) unk_60: u32, - pub(crate) unk_758_flag: GpuWeakPointer, - #[ver(V >= V13_3)] - pub(crate) unk_6c_0: U64, - pub(crate) unk_6c: U64, - pub(crate) unk_74: U64, - pub(crate) unk_7c: U64, - pub(crate) unk_84: U64, - pub(crate) unk_8c: U64, - - #[ver(G == G14 && V < V13_0B4)] - pub(crate) unk_8c_g14: U64, - - pub(crate) restart_branch_offset: i32, - pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic - - #[ver(V >= V13_0B4)] - pub(crate) unk_9c: Array<0x10, u8>, -} - -#[versions(AGX_NEO)] -impl Operation for FinalizeFragment::ver {} - -#[versions(AGX_NEO)] -#[derive(Debug)] -#[repr(C)] -pub(crate) struct StartCompute<'a> { - pub(crate) header: op::StartCompute, - pub(crate) unk_pointer: GpuWeakPointer, - pub(crate) job_params1: Option>>, - #[ver(G >= G14X)] - pub(crate) registers: GpuWeakPointer, - pub(crate) stats: GpuWeakPointer, - pub(crate) work_queue: GpuWeakPointer, - pub(crate) vm_slot: u32, - pub(crate) unk_28: u32, - pub(crate) event_generation: u32, - pub(crate) event_seq: U64, - pub(crate) unk_38: u32, - pub(crate) job_params2: GpuWeakPointer>, - pub(crate) unk_44: u32, - pub(crate) uuid: u32, - pub(crate) attachments: Attachments, - pub(crate) padding: u32, - - #[ver(V >= V13_0B4)] - pub(crate) unk_flag: GpuWeakPointer, - - #[ver(V >= V13_0B4)] - pub(crate) counter: U64, - - #[ver(V >= V13_0B4)] - pub(crate) notifier_buf: GpuWeakPointer>, -} - -#[versions(AGX_NEO)] -impl<'a> Operation for StartCompute::ver<'a> {} - -#[versions(AGX_NEO)] -#[derive(Debug)] -#[repr(C)] -pub(crate) struct FinalizeCompute<'a> { - pub(crate) header: op::FinalizeCompute, - pub(crate) stats: GpuWeakPointer, - pub(crate) work_queue: GpuWeakPointer, - pub(crate) vm_slot: u32, - #[ver(V < V13_0B4)] - pub(crate) unk_18: u32, - pub(crate) job_params2: GpuWeakPointer>, - pub(crate) unk_24: u32, - pub(crate) uuid: u32, - pub(crate) fw_stamp: GpuWeakPointer, - pub(crate) stamp_value: EventValue, - pub(crate) unk_38: u32, - pub(crate) unk_3c: u32, - pub(crate) unk_40: u32, - pub(crate) unk_44: u32, - pub(crate) unk_48: u32, - pub(crate) unk_4c: u32, - pub(crate) unk_50: u32, - pub(crate) unk_54: u32, - pub(crate) unk_58: u32, - - #[ver(G == G14 && V < V13_0B4)] - pub(crate) unk_5c_g14: U64, - - pub(crate) restart_branch_offset: i32, - pub(crate) has_attachments: u32, // Check DCMP errors bits 2,3 1=ktrace 2=log 3=panic - - #[ver(V >= V13_0B4)] - pub(crate) unk_64: Array<0xd, u8>, - - #[ver(V >= V13_0B4)] - pub(crate) unk_flag: GpuWeakPointer, - - #[ver(V >= V13_0B4)] - pub(crate) unk_79: Array<0x7, u8>, -} - -#[versions(AGX_NEO)] -impl<'a> Operation for FinalizeCompute::ver<'a> {} diff --git a/drivers/gpu/drm/asahi-neo/fw/mod.rs b/drivers/gpu/drm/asahi-neo/fw/mod.rs deleted file mode 100644 index a5649aa20d3a8e..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Firmware structures for Apple AGX GPUs - -pub(crate) mod buffer; -pub(crate) mod channels; -pub(crate) mod compute; -pub(crate) mod event; -pub(crate) mod fragment; -pub(crate) mod initdata; -pub(crate) mod job; -pub(crate) mod microseq; -pub(crate) mod types; -pub(crate) mod vertex; -pub(crate) mod workqueue; diff --git a/drivers/gpu/drm/asahi-neo/fw/types.rs b/drivers/gpu/drm/asahi-neo/fw/types.rs deleted file mode 100644 index f55d6cec6b8ca3..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/types.rs +++ /dev/null @@ -1,224 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Common types for firmware structure definitions - -use crate::{alloc, object}; -use core::fmt; -use core::ops::{Deref, DerefMut, Index, IndexMut}; - -pub(crate) use crate::event::EventValue; -pub(crate) use crate::object::{ - GpuPointer, - GpuStruct, - GpuWeakPointer, // -}; -pub(crate) use crate::{ - f32, - float::F32, // -}; - -pub(crate) use core::fmt::Debug; -pub(crate) use core::marker::PhantomData; -pub(crate) use core::sync::atomic::{ - AtomicI32, - AtomicU32, - AtomicU64, // -}; -pub(crate) use kernel::macros::versions; -pub(crate) use kernel::prelude::Zeroable; - -// Make the trait visible -pub(crate) use crate::alloc::Allocator as _Allocator; - -/// General allocator type used for the driver -pub(crate) type Allocator = alloc::DefaultAllocator; - -/// General GpuObject type used for the driver -pub(crate) type GpuObject = - object::GpuObject>; - -/// General GpuArray type used for the driver -pub(crate) type GpuArray = object::GpuArray>; - -/// General GpuOnlyArray type used for the driver -pub(crate) type GpuOnlyArray = - object::GpuOnlyArray>; - -/// A stamp slot that is shared between firmware and the driver. -#[derive(Debug, Default)] -#[repr(transparent)] -pub(crate) struct Stamp(pub(crate) AtomicU32); - -/// A stamp slot that is for private firmware use. -/// -/// This is a separate type to guard against pointer type confusion. -#[derive(Debug, Default)] -#[repr(transparent)] -pub(crate) struct FwStamp(pub(crate) AtomicU32); - -/// An unaligned u64 type. -/// -/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible -/// with `#[derive(Debug)]` and atomics. -#[derive(Copy, Clone, Default)] -#[repr(C, packed(1))] -pub(crate) struct U64(pub(crate) u64); - -// SAFETY: U64 is zeroable just like u64 -unsafe impl Zeroable for U64 {} - -impl fmt::Debug for U64 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let v = self.0; - f.write_fmt(format_args!("{:#x}", v)) - } -} - -/// An unaligned u32 type. -/// -/// This is useful to avoid having to pack firmware structures entirely, since that is incompatible -/// with `#[derive(Debug)]` and atomics. -#[derive(Copy, Clone, Default)] -#[repr(C, packed(1))] -pub(crate) struct U32(pub(crate) u32); - -// SAFETY: U32 is zeroable just like u32 -unsafe impl Zeroable for U32 {} - -impl fmt::Debug for U32 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let v = self.0; - f.write_fmt(format_args!("{:#x}", v)) - } -} - -/// Create a dummy `Debug` implementation, for when we need it but it's too painful to write by -/// hand or not very useful. -#[macro_export] -macro_rules! no_debug { - ($type:ty) => { - impl ::core::fmt::Debug for $type { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "...") - } - } - }; -} - -/// Implement Zeroable for a given type (and Default along with it). -/// -/// # Safety -/// -/// This macro must only be used if a type only contains primitive types which can be -/// zero-initialized, FFI structs intended to be zero-initialized, or other types which -/// impl Zeroable. -#[macro_export] -macro_rules! default_zeroed { - (<$($lt:lifetime),*>, $type:ty) => { - impl<$($lt),*> Default for $type { - fn default() -> $type { - unsafe { core::mem::zeroed() } - } - } - // SAFETY: The user is responsible for ensuring this is safe. - unsafe impl<$($lt),*> ::pin_init::Zeroable for $type {} - }; - ($type:ty) => { - impl Default for $type { - fn default() -> $type { - unsafe { core::mem::zeroed() } - } - } - // SAFETY: The user is responsible for ensuring this is safe. - unsafe impl ::pin_init::Zeroable for $type {} - }; -} - -/// A convenience type for a number of padding bytes. Hidden from Debug formatting. -#[derive(Copy, Clone)] -#[repr(C, packed)] -pub(crate) struct Pad([u8; N]); - -/// SAFETY: Primitive type, safe to zero-init. -unsafe impl Zeroable for Pad {} - -impl Default for Pad { - fn default() -> Self { - unsafe { core::mem::zeroed() } - } -} - -impl fmt::Debug for Pad { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_fmt(format_args!("")) - } -} - -/// A convenience type for a fixed-sized array with Default/Zeroable impls. -#[derive(Copy, Clone)] -#[repr(C)] -pub(crate) struct Array([T; N]); - -impl Array { - pub(crate) fn new(data: [T; N]) -> Self { - Self(data) - } -} - -// SAFETY: Arrays of Zeroable values can be safely Zeroable. -unsafe impl Zeroable for Array {} - -impl Default for Array { - fn default() -> Self { - unsafe { core::mem::zeroed() } - } -} - -impl Index for Array { - type Output = T; - - fn index(&self, index: usize) -> &Self::Output { - &self.0[index] - } -} - -impl IndexMut for Array { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.0[index] - } -} - -impl Deref for Array { - type Target = [T; N]; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for Array { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl fmt::Debug for Array { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.0.fmt(f) - } -} - -/// Convenience macro to define an identically-named trivial GpuStruct with no inner fields for a -/// given raw type name. -#[macro_export] -macro_rules! trivial_gpustruct { - ($type:ident) => { - #[derive(Debug)] - pub(crate) struct $type {} - - impl GpuStruct for $type { - type Raw<'a> = raw::$type; - } - $crate::default_zeroed!($type); - }; -} diff --git a/drivers/gpu/drm/asahi-neo/fw/vertex.rs b/drivers/gpu/drm/asahi-neo/fw/vertex.rs deleted file mode 100644 index 9f5aedc7875b04..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/vertex.rs +++ /dev/null @@ -1,193 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU vertex job firmware structures - -use super::types::*; -use super::{ - event, - job, - workqueue, // -}; -use crate::{ - buffer, - fw, - microseq, - mmu, // -}; -use kernel::sync::Arc; - -pub(crate) mod raw { - use super::*; - - #[derive(Debug, Default, Copy, Clone)] - #[repr(C)] - pub(crate) struct TilingParameters { - pub(crate) rgn_size: u32, - pub(crate) unk_4: u32, - pub(crate) ppp_ctrl: u32, - pub(crate) x_max: u16, - pub(crate) y_max: u16, - pub(crate) te_screen: u32, - pub(crate) te_mtile1: u32, - pub(crate) te_mtile2: u32, - pub(crate) tiles_per_mtile: u32, - pub(crate) tpc_stride: u32, - pub(crate) unk_24: u32, - pub(crate) unk_28: u32, - pub(crate) helper_cfg: u32, - pub(crate) __pad: Pad<0x70>, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobParameters1<'a> { - pub(crate) unk_0: U64, - pub(crate) unk_8: F32, - pub(crate) unk_c: F32, - pub(crate) tvb_tilemap: GpuPointer<'a, &'a [u8]>, - #[ver(G < G14)] - pub(crate) tvb_cluster_tilemaps: Option>, - pub(crate) tpc: GpuPointer<'a, &'a [u8]>, - pub(crate) tvb_heapmeta: GpuPointer<'a, &'a [u8]>, - pub(crate) iogpu_unk_54: U64, - pub(crate) iogpu_unk_56: U64, - #[ver(G < G14)] - pub(crate) tvb_cluster_meta1: Option>, - pub(crate) utile_config: u32, - pub(crate) unk_4c: u32, - pub(crate) ppp_multisamplectl: U64, - pub(crate) tvb_layermeta: GpuPointer<'a, &'a [u8]>, - #[ver(G < G14)] - pub(crate) tvb_cluster_layermeta: Option>, - #[ver(G < G14)] - pub(crate) core_mask: Array<2, u32>, - pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, - pub(crate) preempt_buf2: GpuPointer<'a, &'a [u8]>, - pub(crate) unk_80: U64, - pub(crate) preempt_buf3: GpuPointer<'a, &'a [u8]>, - pub(crate) vdm_ctrl_stream_base: U64, - #[ver(G < G14)] - pub(crate) tvb_cluster_meta2: Option>, - #[ver(G < G14)] - pub(crate) tvb_cluster_meta3: Option>, - #[ver(G < G14)] - pub(crate) tiling_control: u32, - #[ver(G < G14)] - pub(crate) unk_ac: u32, - pub(crate) unk_b0: Array<6, U64>, - pub(crate) usc_exec_base_ta: U64, - #[ver(G < G14)] - pub(crate) tvb_cluster_meta4: Option>, - #[ver(G < G14)] - pub(crate) unk_f0: U64, - pub(crate) unk_f8: U64, - pub(crate) helper_program: u32, - pub(crate) unk_104: u32, - pub(crate) helper_arg: U64, - pub(crate) unk_110: U64, - pub(crate) unk_118: u32, - #[ver(G >= G14)] - pub(crate) __pad: Pad<{ 8 * 9 + 0x268 }>, - #[ver(G < G14)] - pub(crate) __pad: Pad<0x268>, - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct JobParameters2<'a> { - pub(crate) unk_480: Array<4, u32>, - pub(crate) unk_498: U64, - pub(crate) unk_4a0: u32, - pub(crate) preempt_buf1: GpuPointer<'a, &'a [u8]>, - pub(crate) unk_4ac: u32, - pub(crate) unk_4b0: U64, - pub(crate) unk_4b8: u32, - pub(crate) unk_4bc: U64, - pub(crate) unk_4c4_padding: Array<0x48, u8>, - pub(crate) unk_50c: u32, - pub(crate) unk_510: U64, - pub(crate) unk_518: U64, - pub(crate) unk_520: U64, - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct RunVertex<'a> { - pub(crate) tag: workqueue::CommandType, - - #[ver(V >= V13_0B4)] - pub(crate) counter: U64, - - pub(crate) vm_slot: u32, - pub(crate) unk_8: u32, - pub(crate) notifier: GpuPointer<'a, event::Notifier::ver>, - pub(crate) buffer_slot: u32, - pub(crate) unk_1c: u32, - pub(crate) buffer: GpuPointer<'a, fw::buffer::Info::ver>, - pub(crate) scene: GpuPointer<'a, fw::buffer::Scene::ver>, - pub(crate) unk_buffer_buf: GpuWeakPointer<[u8]>, - pub(crate) unk_34: u32, - - #[ver(G < G14X)] - pub(crate) job_params1: JobParameters1::ver<'a>, - #[ver(G < G14X)] - pub(crate) tiling_params: TilingParameters, - #[ver(G >= G14X)] - pub(crate) registers: job::raw::RegisterArray, - - pub(crate) tpc: GpuPointer<'a, &'a [u8]>, - pub(crate) tpc_size: U64, - pub(crate) microsequence: GpuPointer<'a, &'a [u8]>, - pub(crate) microsequence_size: u32, - pub(crate) fragment_stamp_slot: u32, - pub(crate) fragment_stamp_value: EventValue, - pub(crate) unk_pointee: u32, - pub(crate) unk_pad: u32, - pub(crate) job_params2: JobParameters2<'a>, - pub(crate) encoder_params: job::raw::EncoderParams, - pub(crate) unk_55c: u32, - pub(crate) unk_560: u32, - pub(crate) sync_grow: u32, - pub(crate) unk_568: u32, - pub(crate) uses_scratch: u32, - pub(crate) meta: job::raw::JobMeta, - pub(crate) unk_after_meta: u32, - pub(crate) unk_buf_0: U64, - pub(crate) unk_buf_8: U64, - pub(crate) unk_buf_10: U64, - pub(crate) command_time: U64, - pub(crate) timestamp_pointers: job::raw::TimestampPointers<'a>, - pub(crate) user_timestamp_pointers: job::raw::TimestampPointers<'a>, - pub(crate) client_sequence: u8, - pub(crate) pad_5d5: Array<3, u8>, - pub(crate) unk_5d8: u32, - pub(crate) unk_5dc: u8, - - #[ver(V >= V13_0B4)] - pub(crate) unk_ts: U64, - - #[ver(V >= V13_0B4)] - pub(crate) unk_5dd_8: Array<0x1b, u8>, - } -} - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct RunVertex { - pub(crate) notifier: Arc>, - pub(crate) scene: Arc, - pub(crate) micro_seq: microseq::MicroSequence, - pub(crate) vm_bind: mmu::VmBind, - pub(crate) timestamps: Arc>, - pub(crate) user_timestamps: job::UserTimestamps, -} - -#[versions(AGX_NEO)] -impl GpuStruct for RunVertex::ver { - type Raw<'a> = raw::RunVertex::ver<'a>; -} - -#[versions(AGX_NEO)] -impl workqueue::Command for RunVertex::ver {} diff --git a/drivers/gpu/drm/asahi-neo/fw/workqueue.rs b/drivers/gpu/drm/asahi-neo/fw/workqueue.rs deleted file mode 100644 index 414c3c4e1a6887..00000000000000 --- a/drivers/gpu/drm/asahi-neo/fw/workqueue.rs +++ /dev/null @@ -1,187 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU work queue firmware structes - -use super::event; -use super::types::*; -use crate::event::EventValue; -use crate::{ - default_zeroed, - trivial_gpustruct, // -}; -use kernel::sync::Arc; - -#[derive(Debug)] -#[repr(u32)] -pub(crate) enum CommandType { - RunVertex = 0, - RunFragment = 1, - #[allow(dead_code)] - RunBlitter = 2, - RunCompute = 3, - Barrier = 4, - InitBuffer = 6, -} - -pub(crate) trait Command: GpuStruct + Send + Sync {} - -pub(crate) mod raw { - use super::*; - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct Barrier { - pub(crate) tag: CommandType, - pub(crate) wait_stamp: GpuWeakPointer, - pub(crate) wait_value: EventValue, - pub(crate) wait_slot: u32, - pub(crate) stamp_self: EventValue, - pub(crate) uuid: u32, - pub(crate) external_barrier: u32, - // G14X addition - pub(crate) internal_barrier_type: u32, - pub(crate) padding: Pad<0x1c>, - } - - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct GpuContextData { - pub(crate) unk_0: u8, - pub(crate) unk_1: u8, - unk_2: Array<0x2, u8>, - pub(crate) unk_4: u8, - pub(crate) unk_5: u8, - unk_6: Array<0x18, u8>, - pub(crate) unk_1e: u8, - pub(crate) unk_1f: u8, - unk_20: Array<0x3, u8>, - pub(crate) unk_23: u8, - unk_24: Array<0x1c, u8>, - } - - impl Default for GpuContextData { - fn default() -> Self { - Self { - unk_0: 0xff, - unk_1: 0xff, - unk_2: Default::default(), - unk_4: 0, - unk_5: 1, - unk_6: Default::default(), - unk_1e: 0xff, - unk_1f: 0, - unk_20: Default::default(), - unk_23: 2, - unk_24: Default::default(), - } - } - } - - #[derive(Debug)] - #[repr(C)] - pub(crate) struct RingState { - pub(crate) gpu_doneptr: AtomicU32, - __pad0: Pad<0xc>, - pub(crate) unk_10: AtomicU32, - __pad1: Pad<0xc>, - pub(crate) unk_20: AtomicU32, - __pad2: Pad<0xc>, - pub(crate) gpu_rptr: AtomicU32, - __pad3: Pad<0xc>, - pub(crate) cpu_wptr: AtomicU32, - __pad4: Pad<0xc>, - pub(crate) rb_size: u32, - __pad5: Pad<0xc>, - // This isn't part of the structure, but it's here as a - // debugging hack so we can inspect what ring position - // the driver considered complete and freeable. - pub(crate) cpu_freeptr: AtomicU32, - __pad6: Pad<0xc>, - } - default_zeroed!(RingState); - - #[derive(Debug, Clone, Copy)] - #[repr(C)] - pub(crate) struct Priority( - pub(crate) u32, - pub(crate) u32, - pub(crate) U64, - pub(crate) u32, - pub(crate) u32, - pub(crate) u32, - ); - - pub(crate) const PRIORITY: [Priority; 4] = [ - Priority(0, 0, U64(0xffff_ffff_ffff_0000), 1, 0, 1), - Priority(1, 1, U64(0xffff_ffff_0000_0000), 0, 0, 0), - Priority(2, 2, U64(0xffff_0000_0000_0000), 0, 0, 2), - Priority(3, 3, U64(0x0000_0000_0000_0000), 0, 0, 3), - ]; - - impl Default for Priority { - fn default() -> Priority { - PRIORITY[2] - } - } - - #[versions(AGX_NEO)] - #[derive(Debug)] - #[repr(C)] - pub(crate) struct QueueInfo<'a> { - pub(crate) state: GpuPointer<'a, super::RingState>, - pub(crate) ring: GpuPointer<'a, &'a [u64]>, - pub(crate) notifier_list: GpuPointer<'a, event::NotifierList>, - pub(crate) gpu_buf: GpuPointer<'a, &'a [u8]>, - pub(crate) gpu_rptr1: AtomicU32, - pub(crate) gpu_rptr2: AtomicU32, - pub(crate) gpu_rptr3: AtomicU32, - pub(crate) event_id: AtomicI32, - pub(crate) priority: Priority, - pub(crate) unk_4c: i32, - pub(crate) uuid: u32, - pub(crate) unk_54: i32, - pub(crate) unk_58: U64, - pub(crate) busy: AtomicU32, - pub(crate) __pad: Pad<0x20>, - #[ver(V >= V13_2 && G < G14X)] - pub(crate) unk_84_0: u32, - pub(crate) unk_84_state: AtomicU32, - pub(crate) error_count: AtomicU32, - pub(crate) unk_8c: u32, - pub(crate) unk_90: u32, - pub(crate) unk_94: u32, - pub(crate) pending: AtomicU32, - pub(crate) unk_9c: u32, - pub(crate) gpu_context: GpuPointer<'a, super::GpuContextData>, - pub(crate) unk_a8: U64, - #[ver(V >= V13_2 && G < G14X)] - pub(crate) unk_b0: u32, - } -} - -trivial_gpustruct!(Barrier); -trivial_gpustruct!(RingState); - -impl Command for Barrier {} - -pub(crate) struct GpuContextData { - pub(crate) _buffer: Arc, -} -impl GpuStruct for GpuContextData { - type Raw<'a> = raw::GpuContextData; -} - -#[versions(AGX_NEO)] -#[derive(Debug)] -pub(crate) struct QueueInfo { - pub(crate) state: GpuObject, - pub(crate) ring: GpuArray, - pub(crate) gpu_buf: GpuArray, - pub(crate) notifier_list: Arc>, - pub(crate) gpu_context: Arc, -} - -#[versions(AGX_NEO)] -impl GpuStruct for QueueInfo::ver { - type Raw<'a> = raw::QueueInfo::ver<'a>; -} diff --git a/drivers/gpu/drm/asahi-neo/g17/buffer.rs b/drivers/gpu/drm/asahi-neo/g17/buffer.rs deleted file mode 100644 index 90ab8ae21aa14f..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/buffer.rs +++ /dev/null @@ -1,885 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Parameter-buffer IDs, scene ownership and sparse tiler heap growth. -//! -//! The device mutex serializes the shared ID allocator. A render graph owns -//! its buffer ID independently of the submission references using that graph; -//! a published command keeps its reference until firmware retirement is witnessed. - -use super::fw::buffer as fw; -use crate::hw::t8140::resources as cfg; -pub(crate) mod target; - -use kernel::prelude::*; - -const BUFFER_IDS: usize = 127; - -#[derive(Copy, Clone)] -struct BufferId { - references: u8, - owner: bool, - token: u64, -} -impl BufferId { - fn free(self) -> bool { - self.references == 0 && !self.owner - } -} - -/// Device-wide identities for retained render parameter buffers. -pub(crate) struct BufferIds { - entries: [BufferId; BUFFER_IDS], - returned: [u8; BUFFER_IDS], - returned_count: usize, - next_token: u64, -} -impl BufferIds { - pub(crate) fn new() -> impl Init { - kernel::try_init!(Self { - entries <- pin_init::init_array_from_fn(|_| BufferId { - references: 0, - owner: false, - token: 0, - }), - returned <- pin_init::init_array_from_fn(|index| (BUFFER_IDS - index - 1) as u8), - returned_count: BUFFER_IDS, - next_token: 0, - }) - } - fn remove_returned(&mut self, id: u8) { - if let Some(at) = self.returned[..self.returned_count] - .iter() - .position(|entry| *entry == id) - { - self.returned.copy_within(at + 1..self.returned_count, at); - self.returned_count -= 1; - } - } - fn return_id(&mut self, id: u8) { - if !self.returned[..self.returned_count].contains(&id) && self.returned_count < BUFFER_IDS { - self.returned[self.returned_count] = id; - self.returned_count += 1; - } - } - fn fresh_token(&mut self) -> u64 { - let token = if self.next_token == u64::MAX { - 0 - } else { - self.next_token - }; - self.next_token = token + 1; - token - } - pub(crate) fn allocate_render(&mut self) -> Result<(u8, u64)> { - let id = if self.returned_count != 0 { - self.returned_count -= 1; - self.returned[self.returned_count] - } else { - self.entries - .iter() - .position(|entry| entry.free()) - .ok_or(ENOSPC)? as u8 - }; - if !self.entries[usize::from(id)].free() { - return Err(EIO); - } - let token = self.fresh_token(); - self.entries[usize::from(id)] = BufferId { - references: 1, - owner: false, - token, - }; - Ok((id, token)) - } - - /// Retains the existing identity for a graph independently of its jobs. - pub(crate) fn reserve_owner(&mut self, id: u8) -> Result { - let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; - if entry.owner { - return Ok(()); - } - if entry.free() { - let token = self.fresh_token(); - self.entries[usize::from(id)].token = token; - } - self.entries[usize::from(id)].owner = true; - self.remove_returned(id); - Ok(()) - } - pub(crate) fn acquire_owner(&mut self, id: u8) -> Result<(u8, u64)> { - let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; - if !entry.owner { - return Err(EINVAL); - } - entry.references = entry.references.checked_add(1).ok_or(ENOSPC)?; - Ok((id, entry.token)) - } - /// Preflight shared completion bookkeeping before any of its counters change. - pub(crate) fn can_release(&self, id: u8) -> Result { - let entry = self.entries.get(usize::from(id)).ok_or(EINVAL)?; - if entry.references == 0 { - return Err(EINVAL); - } - Ok(()) - } - pub(crate) fn release(&mut self, id: u8) -> Result { - let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; - entry.references = entry.references.checked_sub(1).ok_or(EINVAL)?; - if entry.free() { - self.return_id(id); - } - Ok(()) - } - pub(crate) fn unreserve_owner(&mut self, id: u8) -> Result { - let entry = self.entries.get_mut(usize::from(id)).ok_or(EINVAL)?; - if !entry.owner { - return Ok(()); - } - entry.owner = false; - if entry.free() { - self.return_id(id); - } - Ok(()) - } -} - -/// Each scene and its shorter scratch slot are one lifetime. A completed -/// paired render releases both; merely publishing a later scene does not. -pub(crate) struct Scenes { - owners: [Option<(u64, u8)>; fw::SCRATCH_SLOTS], - next: usize, -} -impl Scenes { - pub(crate) const fn new() -> Self { - Self { - owners: [None; fw::SCRATCH_SLOTS], - next: fw::FIRST_SCENE, - } - } - pub(crate) fn select(&self, ordinal: u64) -> Result { - if let Some((_, scene)) = self - .owners - .iter() - .flatten() - .find(|(owner, _)| *owner == ordinal) - { - return Ok(usize::from(*scene)); - } - (0..fw::SCENES) - .map(|offset| (self.next + offset) % fw::SCENES) - .find(|scene| self.owners[scene % fw::SCRATCH_SLOTS].is_none()) - .ok_or(ENOSPC) - } - pub(crate) fn commit(&mut self, ordinal: u64, scene: usize) -> Result { - if self.select(ordinal)? != scene { - return Err(EINVAL); - } - let slot = &mut self.owners[scene % fw::SCRATCH_SLOTS]; - if slot.is_none() { - *slot = Some((ordinal, scene as u8)); - self.next = (scene + 1) % fw::SCENES; - } - Ok(()) - } - pub(crate) fn release(&mut self, ordinal: u64) -> Result { - let slot = self - .owners - .iter_mut() - .find(|entry| entry.is_some_and(|(owner, _)| owner == ordinal)) - .ok_or(EINVAL)?; - *slot = None; - Ok(()) - } -} - -/// A snapshot of the block ring; all values are modulo its fixed capacity. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct Cursors { - pub(crate) total: u32, - pub(crate) committed: u32, - pub(crate) read: u32, -} -impl Cursors { - fn valid(self) -> bool { - [self.total, self.committed, self.read] - .into_iter() - .all(|v| (v as usize) < cfg::TVB_BLOCK_SLOTS) - } - /// Firmware-paused growth permits consumption of the already committed - /// prefix. Retired-host growth requires the complete snapshot unchanged. - pub(crate) fn permits(self, after: Self, firmware_request: bool) -> bool { - if !self.valid() - || !after.valid() - || self.total != after.total - || self.committed != after.committed - { - return false; - } - if !firmware_request { - return self == after; - } - let capacity = cfg::TVB_BLOCK_SLOTS as u32; - (after.read + capacity - self.read) % capacity - <= (self.committed + capacity - self.read) % capacity - } -} - -/// Checked block-table indices for one bulk allocation. One ring slot stays -/// empty even when all physical backing has been retained by the owner. -pub(crate) struct Growth { - before: Cursors, - added: usize, - retained: usize, - target: usize, -} -impl Growth { - pub(crate) fn new(before: Cursors, added: usize) -> Result { - if !before.valid() || before.total != before.committed || added == 0 { - return Err(EINVAL); - } - let retained = (before.total as usize + cfg::TVB_BLOCK_SLOTS - before.read as usize) - % cfg::TVB_BLOCK_SLOTS; - if retained.checked_add(added).ok_or(EOVERFLOW)? >= cfg::TVB_BLOCK_SLOTS { - return Err(ENOSPC); - } - let target = (before.total as usize + added) % cfg::TVB_BLOCK_SLOTS; - Ok(Self { - before, - added, - retained, - target, - }) - } - pub(crate) fn target(&self) -> u32 { - self.target as u32 - } - pub(crate) fn retained(&self) -> usize { - self.retained - } - pub(crate) fn available(&self) -> usize { - self.retained + self.added - } - pub(crate) fn added_slot(&self, index: usize) -> Result { - if index >= self.added { - return Err(EINVAL); - } - Ok((self.before.total as usize + index) % cfg::TVB_BLOCK_SLOTS) - } - pub(crate) fn available_slot(&self, index: usize) -> Result { - if index >= self.available() { - return Err(EINVAL); - } - Ok((self.before.read as usize + index) % cfg::TVB_BLOCK_SLOTS) - } -} - -pub(crate) fn firmware_growth_target(current: usize) -> usize { - current - .saturating_add(fw::GROW_BLOCKS) - .min(cfg::TVB_MAX_BLOCKS) -} - -/// The high-water sample estimates capacity, not current occupancy. A growth -/// reserves three times the demand only once twice that demand no longer fits. -pub(crate) fn automatic_growth_target(used_pages: usize, current: usize) -> Option { - let maximum = fw::NORMAL_MAX_BLOCKS; - if current >= maximum { - return None; - } - let need = used_pages - .saturating_mul(2) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) - .min(maximum); - if need <= current { - return None; - } - Some( - used_pages - .saturating_mul(3) - .div_ceil(cfg::TVB_PAGES_PER_BLOCK) - .min(maximum), - ) -} - -use super::object::{CpuMap, KernelObject}; -use crate::{driver::AsahiDevice, mmu}; -use core::ops::Range; - -pub(crate) fn compact_alias( - object: &KernelObject, - vm: &mmu::Vm, - source: Range, - align: u64, - prot: mmu::Prot, - guard: usize, -) -> Result { - for range in cfg::compact_ranges() { - match object.map_range(vm, source.clone(), range, align, prot, guard) { - Ok(mapping) => return Ok(mapping), - Err(error) if error == ENOSPC => {} - Err(error) => return Err(error), - } - } - Err(ENOSPC) -} - -/// Allocated aliases remain private until the owner installs them in its VM -/// cache. On rollback mappings are destroyed before backing memory. -pub(crate) struct TvbGrowth { - mappings: KVec, - addresses: KVec, - extension: KernelObject, -} - -/// Sparse 128-KiB data blocks at 160-KiB backing strides. Firmware owns their -/// contents; CPU zeroing occurs only on fresh backing, never on pool reuse. -pub(crate) struct Tvb { - addresses: KVec, - // Retain sparse-block backing until the pool and its aliases retire. - _primary: KernelObject, - extensions: KVec, -} -impl Tvb { - fn bytes(blocks: usize) -> Result { - blocks - .checked_sub(1) - .and_then(|v| v.checked_mul(cfg::TVB_BLOCK_STRIDE as usize)) - .and_then(|v| v.checked_add(cfg::TVB_BLOCK_SIZE)) - .ok_or(EOVERFLOW) - } - fn map_block( - object: &KernelObject, - vm: &mmu::Vm, - relative: usize, - ) -> Result { - let start = relative - .checked_mul(cfg::TVB_BLOCK_STRIDE as usize) - .ok_or(EOVERFLOW)?; - let source = start..start.checked_add(cfg::TVB_BLOCK_SIZE).ok_or(EOVERFLOW)?; - let mapping = compact_alias( - object, - vm, - source, - cfg::TVB_PAGE, - mmu::PROT_GPU_SHARED_RW, - cfg::TVB_BLOCK_STRIDE as usize - cfg::TVB_BLOCK_SIZE, - )?; - fw::compact_page(mapping.iova())?; - Ok(mapping) - } - pub(crate) fn new( - dev: &AsahiDevice, - vm: &mmu::Vm, - blocks: usize, - ) -> Result<(Self, KVec)> { - if blocks == 0 || blocks > cfg::TVB_MAX_BLOCKS { - return Err(ERANGE); - } - let backing = KernelObject::backing(dev, Self::bytes(blocks)?, CpuMap::WriteCombined)?; - let mut mappings = KVec::with_capacity(blocks, GFP_KERNEL)?; - let mut addresses = KVec::with_capacity(blocks, GFP_KERNEL)?; - for index in 0..blocks { - let mapping = Self::map_block(&backing, vm, index)?; - addresses.push(mapping.iova(), GFP_KERNEL)?; - mappings.push(mapping, GFP_KERNEL)?; - } - Ok(( - Self { - addresses, - _primary: backing, - extensions: KVec::new(), - }, - mappings, - )) - } - pub(crate) fn blocks(&self) -> usize { - self.addresses.len() - } - pub(crate) fn addresses(&self) -> &[u64] { - &self.addresses - } - - pub(crate) fn prepare_growth( - &mut self, - dev: &AsahiDevice, - vm: &mmu::Vm, - target: usize, - ) -> Result> { - if target <= self.blocks() { - return Ok(None); - } - if target > cfg::TVB_MAX_BLOCKS { - return Err(ERANGE); - } - let added = target - self.blocks(); - self.extensions.reserve(1, GFP_KERNEL)?; - self.addresses.reserve(added, GFP_KERNEL)?; - let backing = KernelObject::backing(dev, Self::bytes(added)?, CpuMap::WriteCombined)?; - let mut mappings = KVec::with_capacity(added, GFP_KERNEL)?; - let mut addresses = KVec::with_capacity(added, GFP_KERNEL)?; - for relative in 0..added { - let mapping = Self::map_block(&backing, vm, relative)?; - addresses.push(mapping.iova(), GFP_KERNEL)?; - mappings.push(mapping, GFP_KERNEL)?; - } - Ok(Some(TvbGrowth { - mappings, - addresses, - extension: backing, - })) - } -} - -use super::object::Allocator; -use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; -use kernel::sync::Arc; - -/// First-free metrics entries are shared by all parameter-buffer graphs. -/// An entry returns only when the graph and its firmware references retire. -pub(crate) struct MetricsIds { - used: [AtomicU64; 2], -} -impl MetricsIds { - pub(crate) fn new() -> Self { - Self { - used: [ - AtomicU64::new(0), - AtomicU64::new(!((1u64 << (fw::METRICS_SLOTS - 64)) - 1)), - ], - } - } -} - -pub(crate) struct MetricsLease { - _backing: Arc, - allocator: Arc, - slot: usize, - addresses: fw::MetricsAddresses, -} -impl MetricsLease { - /// Clears only the selected retired metrics entry before reusing it. - pub(crate) fn new( - ids: &Arc, - object: &Arc, - bases: fw::MetricsAddresses, - ) -> Result { - let mut selected = None; - for (bank, used) in ids.used.iter().enumerate() { - let mut value = used.load(Ordering::Acquire); - loop { - let bit = value.trailing_ones(); - if bit == 64 { - break; - } - match used.compare_exchange_weak( - value, - value | (1u64 << bit), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - selected = Some(bank * 64 + bit as usize); - break; - } - Err(current) => value = current, - } - } - if selected.is_some() { - break; - } - } - let slot = selected.ok_or(ENOSPC)?; - // Construct the lease before any fallible address/view validation so - // every constructor failure returns the selected bit. - let mut lease = Self { - _backing: object.clone(), - allocator: ids.clone(), - slot, - addresses: fw::MetricsAddresses::default(), - }; - let offset = slot * fw::METRICS_STRIDE; - lease.addresses = fw::MetricsAddresses { - firmware: bases.firmware.checked_add(offset as u64).ok_or(EOVERFLOW)?, - client: bases.client.checked_add(offset as u64).ok_or(EOVERFLOW)?, - }; - let words = object.pointer(offset, fw::METRICS_STRIDE)?.cast::(); - object.word(offset)?; - for index in 0..fw::SCENES { - // SAFETY: The complete naturally aligned metrics entry is inside - // the retained object; the exclusive lease excludes other owners. - unsafe { words.add(index).write_volatile(0) }; - } - Ok(lease) - } -} -impl Drop for MetricsLease { - fn drop(&mut self) { - self.allocator.used[self.slot / 64] - .fetch_and(!(1u64 << (self.slot % 64)), Ordering::Release); - } -} - -pub(crate) struct ManagerArgs { - pub(crate) scratch: u64, - pub(crate) discard: u64, - pub(crate) buffer_id: u8, - pub(crate) generation: u64, - pub(crate) first_ordinal: u64, -} - -/// A retained parameter-buffer graph and all of its sparse backing. Client -/// mappings live in the VM's owner cache; graph replacement never returns a -/// published backing early merely because an error fence was signalled. -pub(crate) struct Manager { - _page_list: mmu::KernelMapping, - graph: KernelObject, - tvb: Tvb, - metrics: MetricsLease, - scenes: Scenes, - page_list_client: u64, - scratch: u64, - buffer_id: u8, - generation: u64, -} - -pub(crate) struct Submitted<'a> { - word: &'a AtomicU32, - next: u32, -} -impl Submitted<'_> { - pub(crate) fn commit(self) { - self.word.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - } -} - -impl Manager { - pub(crate) fn new( - alloc: &Allocator<'_>, - vm: &mmu::Vm, - tvb: Tvb, - metrics: MetricsLease, - args: ManagerArgs, - ) -> Result { - let mut graph = alloc.kernel( - fw::SIZE, - cfg::TVB_PAGE, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let source = fw::PAGE_LIST..fw::PAGE_LIST + fw::PAGE_LIST_BYTES; - let page_list = compact_alias( - &graph, - vm, - source, - cfg::TVB_PAGE, - mmu::PROT_GPU_FW_SHARED_RW, - mmu::UAT_PGSZ, - )?; - let address = graph.gpu_va(); - graph.initialize::(0, |view| { - view.init(&fw::GraphArgs { - fw_va: address, - page_list_client: page_list.iova(), - metrics: metrics.addresses, - blocks: tvb.addresses(), - scratch: args.scratch, - discard: args.discard, - buffer_id: args.buffer_id.into(), - generation: args.generation, - }) - })?; - fence(Ordering::SeqCst); - let mut scenes = Scenes::new(); - scenes.commit(args.first_ordinal, fw::FIRST_SCENE)?; - let page_list_client = page_list.iova(); - Ok(Self { - _page_list: page_list, - graph, - tvb, - metrics, - scenes, - page_list_client, - scratch: args.scratch, - buffer_id: args.buffer_id, - generation: args.generation, - }) - } - pub(crate) fn buffer_id(&self) -> u8 { - self.buffer_id - } - pub(crate) fn blocks(&self) -> usize { - self.tvb.blocks() - } - pub(crate) fn state_va(&self) -> u64 { - self.graph.gpu_va() - } - pub(crate) fn page_list_va(&self) -> u64 { - self.page_list_client - } - pub(crate) fn trailer_va(&self) -> u64 { - self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (fw::SCENES * size_of::()) as u64 - } - pub(crate) fn scene_va(&self, index: usize) -> Result { - if index >= fw::SCENES { - return Err(EINVAL); - } - Ok(self.graph.gpu_va() + fw::SCENE_TABLE as u64 + (index * size_of::()) as u64) - } - pub(crate) fn scene_registers(&self, index: usize) -> Result<(u64, u64)> { - fw::scene_registers(index, self.scratch, self.metrics.addresses.client) - } - pub(crate) fn prepare_scene(&mut self, ordinal: u64) -> Result { - let scene = self.scenes.select(ordinal)?; - let offset = fw::SCENE_TABLE + scene * size_of::(); - let used = self - .graph - .word(offset + core::mem::offset_of!(fw::Scene, used))?; - let generation = self - .graph - .pointer(offset + core::mem::offset_of!(fw::Scene, generation), 8)?; - let id = self - .graph - .word(core::mem::offset_of!(fw::State, buffer_id))?; - used.store(0, Ordering::Relaxed); - // SAFETY: The complete packed field was checked before any write. - // This scene is exclusively retained or still unclaimed. - unsafe { generation.cast::().write_unaligned(self.generation) }; - id.store(u32::from(self.buffer_id), Ordering::Relaxed); - fence(Ordering::SeqCst); - Ok(scene) - } - pub(crate) fn claim_scene(&mut self, ordinal: u64, scene: usize) -> Result { - self.scenes.commit(ordinal, scene) - } - pub(crate) fn retire_scene(&mut self, ordinal: u64) -> Result { - self.scenes.release(ordinal) - } - pub(crate) fn prepare_submission(&mut self) -> Result> { - let word = self.graph.word(fw::COUNTER)?; - Ok(Submitted { - word, - next: word.load(Ordering::Relaxed).wrapping_add(1), - }) - } - pub(crate) fn automatic_target(&self) -> Result> { - let max_pages = self - .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, max_pages))?; - let reset = self - .graph - .word(fw::CONTROL + core::mem::offset_of!(fw::Control, reset))?; - fence(Ordering::Acquire); - let pages = max_pages.load(Ordering::Relaxed); - fence(Ordering::Release); - reset.store(1, Ordering::Relaxed); - Ok(automatic_growth_target(pages as usize, self.blocks())) - } - fn cursors(&self) -> Result { - Ok(Cursors { - total: self.graph.word(fw::CONTROL)?.load(Ordering::Relaxed), - committed: self.graph.word(fw::CONTROL + 4)?.load(Ordering::Relaxed), - read: self.graph.word(fw::CONTROL + 8)?.load(Ordering::Relaxed), - }) - } -} - -/// All allocations and inventory words prepared without publishing shared state. -/// The pair remains exclusively borrowed until commit or discard. -pub(crate) struct PreparedGrowth { - owner: u64, - previous: usize, - target: usize, - before: Cursors, - plan: Growth, - growth: TvbGrowth, - added: KVec<(usize, u32)>, - rebuilt: KVec, - pages: u32, - descriptor: Option<[u32; 4]>, -} -impl Manager { - /// Validate every retained sparse block and its reserved trailing guard in - /// one VM snapshot before constructing more aliases. - pub(crate) fn validate_vm(&self, vm: &mmu::Vm) -> Result { - if self.blocks() < cfg::TVB_INITIAL_PAGES.len() - || self.blocks() > cfg::TVB_MAX_BLOCKS - || !vm.covers_sparse_blocks_with_guards( - self.tvb.addresses(), - cfg::TVB_BLOCK_SIZE as u64, - cfg::TVB_BLOCK_STRIDE, - ) - { - return Err(EFAULT); - } - Ok(()) - } - - /// Allocate growth and precompute its words while the pair is exclusively - /// borrowed. A descriptor snapshot requests idle-host growth; None leaves - /// the firmware-owned page list and PB state untouched. - pub(crate) fn prepare_growth( - &mut self, - dev: &AsahiDevice, - vm: &mmu::Vm, - target: usize, - descriptor: Option<[u32; 4]>, - ) -> Result> { - let previous = self.blocks(); - if target <= previous { - return Ok(None); - } - self.validate_vm(vm)?; - let Some(growth) = self.tvb.prepare_growth(dev, vm, target)? else { - return Ok(None); - }; - let before = self.cursors()?; - let plan = Growth::new(before, target - previous)?; - let firmware_request = descriptor.is_none(); - let pages = target - .checked_mul(cfg::TVB_PAGES_PER_BLOCK) - .ok_or(EOVERFLOW)? as u32; - let mut added = KVec::with_capacity(growth.addresses.len(), GFP_KERNEL)?; - for (index, &address) in growth.addresses.iter().enumerate() { - let first = fw::compact_page(address)?; - first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) - .ok_or(EOVERFLOW)?; - added.push((plan.added_slot(index)?, first), GFP_KERNEL)?; - } - let rebuilt_count = if firmware_request { - 0 - } else { - plan.available() * cfg::TVB_PAGES_PER_BLOCK - }; - let mut rebuilt = KVec::with_capacity(rebuilt_count, GFP_KERNEL)?; - if !firmware_request { - if self.cursors()? != before { - return Err(EAGAIN); - } - for ordinal in 0..plan.available() { - let first = if ordinal < plan.retained() { - let slot = plan.available_slot(ordinal)?; - self.graph - .word(fw::BLOCK_TABLE + slot * size_of::())? - .load(Ordering::Relaxed) - } else { - added[ordinal - plan.retained()].1 - }; - first - .checked_add(cfg::TVB_PAGES_PER_BLOCK as u32 - 1) - .ok_or(EOVERFLOW)?; - for page in 0..cfg::TVB_PAGES_PER_BLOCK { - rebuilt.push(first + page as u32, GFP_KERNEL)?; - } - } - } - let descriptor = - descriptor.map(|old| fw::reload_descriptor(old, self.page_list_client, pages)); - Ok(Some(PreparedGrowth { - owner: self.graph.gpu_va(), - previous, - target, - before, - plan, - growth, - added, - rebuilt, - pages, - descriptor, - })) - } - - /// The caller holds the device mutex after revalidating its preparation - /// epoch. The final alias installation and all firmware writes occur under - /// that same exclusion. The returned PB row is committed there as well. - pub(crate) fn commit_growth( - &mut self, - prepared: &mut Option, - install: impl FnOnce(usize, usize, &mut KVec) -> Result, - ) -> Result> { - let candidate = prepared.as_mut().ok_or(EINVAL)?; - if self.graph.gpu_va() != candidate.owner || self.blocks() != candidate.previous { - return Err(EBUSY); - } - let firmware_request = candidate.descriptor.is_none(); - let base = self.graph.pointer(0, fw::SIZE)?.cast::(); - // Bounds, actual alignment and every reserved host entry are checked - // before transferring aliases or consuming the caller-owned plan. - self.graph.word(0)?; - if self.tvb.extensions.len() == self.tvb.extensions.capacity() - || candidate.growth.addresses.len() - > self.tvb.addresses.capacity() - self.tvb.addresses.len() - { - return Err(EIO); - } - if !candidate.before.permits(self.cursors()?, firmware_request) { - return Err(EAGAIN); - } - // The callback retains its input on error. The caller can therefore - // drop every rejected mapping/backing after releasing the device lock. - install( - candidate.previous, - candidate.target, - &mut candidate.growth.mappings, - )?; - if !candidate.growth.mappings.is_empty() { - return Err(EIO); - } - let PreparedGrowth { - before, - plan, - growth, - added, - rebuilt, - pages, - descriptor, - .. - } = prepared.take().ok_or(EIO)?; - let TvbGrowth { - addresses, - extension, - .. - } = growth; - - // From here all vector capacity, indices, addresses and CPU views have - // been checked. Retain backing before publishing its block entries. - for address in addresses.into_iter() { - append_reserved(&mut self.tvb.addresses, address); - } - let write = |offset: usize, value: u32| { - // SAFETY: The full graph and base alignment were checked above. - // Every offset is an aligned field or a checked array index of - // Graph. No mutable reference aliases firmware-owned fields. - unsafe { AtomicU32::from_ptr(base.add(offset / 4)).store(value, Ordering::Relaxed) }; - }; - for (slot, first) in added.into_iter() { - let at = fw::BLOCK_TABLE + slot * size_of::(); - write(at, first); - write(at + 4, 0); - } - if descriptor.is_some() { - for (entry, value) in rebuilt.into_iter().enumerate() { - write(fw::PAGE_LIST + entry * 4, value); - } - write(core::mem::offset_of!(fw::State, pages), pages); - write(core::mem::offset_of!(fw::State, producer), plan.target()); - write(core::mem::offset_of!(fw::State, consumer), before.read); - write(core::mem::offset_of!(fw::State, last_page), pages - 1); - } - append_reserved(&mut self.tvb.extensions, extension); - write(fw::CONTROL, plan.target()); - fence(Ordering::SeqCst); - write(fw::CONTROL + 4, plan.target()); - fence(Ordering::SeqCst); - Ok(descriptor) - } -} - -/// Caller reserves all needed entries before any firmware-visible mutation. -fn append_reserved(values: &mut KVec, value: T) { - let length = values.len(); - values.spare_capacity_mut()[0].write(value); - // SAFETY: All callers preflighted this spare entry and initialized it once. - unsafe { values.set_len(length + 1) }; -} diff --git a/drivers/gpu/drm/asahi-neo/g17/buffer/target.rs b/drivers/gpu/drm/asahi-neo/g17/buffer/target.rs deleted file mode 100644 index 07ead6056329e5..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/buffer/target.rs +++ /dev/null @@ -1,297 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Rotating tile-map/heap metadata and the retained tail-pointer cache. -//! Larger targets replace only the objects that need more capacity. Every old -//! backing keeps the exact commands that still reference it until retirement. - -use super::compact_alias; -use crate::g17::{ - fw::render::RenderPass, - object::{CpuMap, KernelObject}, -}; -use crate::{ - driver::AsahiDevice, - hw::t8140::{queues, resources as cfg}, - mmu, -}; -use kernel::prelude::*; - -const DEPTH: usize = queues::RENDER_DEPTH as usize; - -fn aligned(value: usize, alignment: usize) -> Result { - value - .checked_add(alignment - 1) - .map(|v| v & !(alignment - 1)) - .ok_or(EOVERFLOW) -} - -/// Per-command geometry, independent of the capacity already allocated. -#[derive(Copy, Clone, Debug)] -pub(crate) struct Layout { - pub(crate) layer_offset: usize, - pub(crate) heap_offset: usize, - pub(crate) used: usize, - pub(crate) tpc_size: usize, - pub(crate) tvb_blocks: usize, -} -impl Layout { - pub(crate) fn new(pass: &RenderPass, clusters: u32) -> Result { - let width = pass.width as usize; - let height = pass.height as usize; - let layers = pass.layers as usize; - let (uw, uh) = (pass.utile_width as usize, pass.utile_height as usize); - if width == 0 - || height == 0 - || layers == 0 - || clusters == 0 - || uw == 0 - || uh == 0 - || 32 % uw != 0 - || 32 % uh != 0 - { - return Err(EINVAL); - } - let utiles = (32 / uw) * (32 / uh); - let tiles_x = width.div_ceil(32); - let tiles_y = height.div_ceil(32); - let macro_x = aligned(tiles_x.div_ceil(4), 4)?; - let macro_y = aligned(tiles_y.div_ceil(4), 4)?; - let macro_tiles = macro_x.checked_mul(macro_y).ok_or(EOVERFLOW)?; - let tilemap_size = aligned( - 5usize - .checked_mul(macro_tiles) - .and_then(|v| v.checked_mul(utiles)) - .ok_or(EOVERFLOW)?, - 4, - )? - .checked_mul(16) - .and_then(|v| v.checked_mul(layers)) - .ok_or(EOVERFLOW)?; - let tpc_size = 8usize - .checked_mul(utiles) - .and_then(|v| v.checked_mul(macro_tiles)) - .map(|v| v / 4 * 4) - .and_then(|v| v.checked_mul(16)) - .and_then(|v| v.checked_mul(layers)) - .and_then(|v| v.checked_mul(clusters as usize)) - .ok_or(EOVERFLOW)?; - let layer_offset = aligned(tilemap_size, 0x1000)?; - let heap_offset = layer_offset - .checked_add(if layers > 1 { 0x100 } else { 0 }) - .ok_or(EOVERFLOW)?; - let used = heap_offset.checked_add(0x200).ok_or(EOVERFLOW)?; - let tiles = tiles_x.checked_mul(tiles_y).ok_or(EOVERFLOW)?; - let mut tvb_blocks = aligned(tiles.div_ceil(128), 8)?.max(8); - if clusters > 1 { - tvb_blocks = tvb_blocks.max( - 7usize - .checked_add(2usize.checked_mul(layers).ok_or(EOVERFLOW)?) - .ok_or(EOVERFLOW)?, - ); - } - tvb_blocks = tvb_blocks.max(cfg::TVB_INITIAL_PAGES.len()); - Ok(Self { - layer_offset, - heap_offset, - used, - tpc_size, - tvb_blocks, - }) - } - pub(crate) fn tpc_capacity(&self) -> Result { - Ok(aligned(self.tpc_size, mmu::UAT_PGSZ)?.max(cfg::TPC_MIN)) - } - pub(crate) fn record_stride(&self) -> Result { - Ok(aligned(self.used, mmu::UAT_PGSZ)?.max(cfg::TARGET_RECORD_MIN)) - } -} - -struct Object { - address: u64, - _mapping: mmu::KernelMapping, - backing: KernelObject, -} -impl Object { - fn new(dev: &AsahiDevice, vm: &mmu::Vm, size: usize) -> Result { - let backing = KernelObject::backing(dev, size, CpuMap::WriteCombined)?; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let mapping = compact_alias(&backing, vm, 0..size, cfg::TVB_PAGE, prot, mmu::UAT_PGSZ)?; - Ok(Self { - address: mapping.iova(), - _mapping: mapping, - backing, - }) - } -} -struct Retired { - _object: Object, - owners: [Option; DEPTH], -} - -#[derive(Copy, Clone, Debug)] -pub(crate) struct Addresses { - pub(crate) tilemap: u64, - pub(crate) layer: u64, - pub(crate) heap: u64, -} - -pub(crate) struct Target { - records: Object, - tpc: Object, - stride: usize, - slots: [Option; DEPTH], - retired: KVec, -} -impl Target { - /// Keep this allocation frame separate from the rest of the render graph. - #[inline(never)] - pub(crate) fn new(dev: &AsahiDevice, vm: &mmu::Vm, layout: &Layout) -> Result> { - let tpc = layout.tpc_capacity()?; - let stride = layout.record_stride()?; - if tpc > cfg::TPC_MAX || stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } - let records = Object::new(dev, vm, stride * DEPTH)?; - let tpc = Object::new(dev, vm, tpc)?; - KBox::new( - Self { - records, - tpc, - stride, - slots: [None; DEPTH], - retired: KVec::new(), - }, - GFP_KERNEL, - ) - .map_err(Into::into) - } - pub(crate) fn tpc_va(&self) -> u64 { - self.tpc.address - } - - /// Only the span consumed by this render is cleared. A small render does - /// not clear a previous large target's complete retained record capacity. - pub(crate) fn prepare(&mut self, ordinal: u64, layout: &Layout) -> Result { - let slot = ordinal as usize % DEPTH; - if self.slots[slot].is_some_and(|owner| owner != ordinal) { - return Err(EBUSY); - } - if layout.used > self.stride { - return Err(ERANGE); - } - let offset = slot * self.stride; - let span = aligned(layout.used, mmu::UAT_PGSZ)?.min(self.stride); - self.records - .backing - .bytes_mut() - .get_mut(offset..offset + span) - .ok_or(ERANGE)? - .fill(0); - self.slots[slot] = Some(ordinal); - let tilemap = self - .records - .address - .checked_add(offset as u64) - .ok_or(EOVERFLOW)?; - Ok(Addresses { - tilemap, - layer: tilemap - .checked_add(layout.layer_offset as u64) - .ok_or(EOVERFLOW)?, - heap: tilemap - .checked_add(layout.heap_offset as u64) - .ok_or(EOVERFLOW)?, - }) - } - - /// Growth never shrinks a live allocation. A successfully replaced TPC - /// stays enlarged even if allocating the following record object fails. - pub(crate) fn ensure_capacity( - &mut self, - dev: &AsahiDevice, - vm: &mmu::Vm, - layout: &Layout, - owners: &[Option; DEPTH], - ) -> Result { - let tpc = layout.tpc_capacity()?; - let stride = layout.record_stride()?; - let mut grew = false; - if tpc > self.tpc.backing.size() { - if tpc > cfg::TPC_MAX { - return Err(ERANGE); - } - Self::replace(&mut self.retired, &mut self.tpc, dev, vm, tpc, owners)?; - grew = true; - } - if stride > self.stride { - if stride > cfg::TARGET_RECORD_MAX { - return Err(ERANGE); - } - Self::replace( - &mut self.retired, - &mut self.records, - dev, - vm, - stride * DEPTH, - owners, - )?; - self.stride = stride; - self.slots = [None; DEPTH]; - grew = true; - } - Ok(grew) - } - fn replace( - retired: &mut KVec, - old: &mut Object, - dev: &AsahiDevice, - vm: &mmu::Vm, - size: usize, - owners: &[Option; DEPTH], - ) -> Result { - let retain = owners.iter().any(Option::is_some); - if retain { - retired.reserve(1, GFP_KERNEL)?; - } - let new = Object::new(dev, vm, size)?; - let previous = core::mem::replace(old, new); - if retain { - super::append_reserved( - retired, - Retired { - _object: previous, - owners: *owners, - }, - ); - } - Ok(()) - } - pub(crate) fn retire(&mut self, ordinal: u64) -> Result { - let slot = &mut self.slots[ordinal as usize % DEPTH]; - if *slot == Some(ordinal) { - *slot = None; - } - let mut index = 0; - while index < self.retired.len() { - for owner in &mut self.retired[index].owners { - if *owner == Some(ordinal) { - *owner = None; - } - } - if self.retired[index].owners.iter().all(Option::is_none) { - drop(self.retired.remove(index)?); - } else { - index += 1; - } - } - Ok(()) - } - /// Cancels only an unpublished record; older referenced backing remains - /// owned until normal paired retirement or the firmware-stop boundary. - pub(crate) fn cancel(&mut self, ordinal: u64) { - let slot = &mut self.slots[ordinal as usize % DEPTH]; - if *slot == Some(ordinal) { - *slot = None; - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/channel.rs b/drivers/gpu/drm/asahi-neo/g17/channel.rs deleted file mode 100644 index 8e4740b76a1661..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/channel.rs +++ /dev/null @@ -1,219 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Firmware-produced rings and the device-control producer. -//! -//! The device mutex serializes every caller. Entries are copied before a -//! single aligned consumer store returns their slots to the firmware. Trace -//! and statistics transport errors quarantine only that ring; event errors -//! propagate to the device owner. Device-control publication reserves one -//! empty slot and orders the record, power tally and producer together. - -use core::sync::atomic::{fence, Ordering}; -use kernel::{device::Device, prelude::*}; - -use super::fw::{channels::*, initdata::status}; -use super::initdata::InitData; -use super::Role; - -/// Firmware rings whose contents do not affect host scheduling. -#[derive(Copy, Clone, Debug)] -enum ReportRing { - Trace = 0, - Statistics = 1, -} - -impl ReportRing { - fn geometry(self) -> (usize, usize, u32, u32) { - match self { - Self::Trace => ( - status::TRACE_STATE as usize, - status::TRACE_RING as usize, - TRACE_SLOTS, - TRACE_TYPES, - ), - Self::Statistics => ( - status::STATS_STATE as usize, - status::STATS_RING as usize, - STATISTICS_SLOTS, - STATISTICS_TYPES, - ), - } - } -} - -/// Host-side receive-ring state. A quarantined ring stays stopped until the -/// device is destroyed; firmware restart does not clear it. -#[derive(Debug)] -pub(crate) struct Rings { - quarantined: [[bool; 2]; 2], -} - -impl Rings { - pub(crate) const fn new() -> Self { - Self { - quarantined: [[false; 2]; 2], - } - } - - pub(crate) fn drain_trace(&mut self, dev: &Device, init: &InitData, role: Role) { - self.drain_report(dev, init, role, ReportRing::Trace); - } - - pub(crate) fn drain_statistics(&mut self, dev: &Device, init: &InitData, role: Role) { - self.drain_report(dev, init, role, ReportRing::Statistics); - } - - fn drain_report(&mut self, dev: &Device, init: &InitData, role: Role, ring: ReportRing) { - let quarantined = &mut self.quarantined[role as usize][ring as usize]; - if *quarantined { - return; - } - if let Err(error) = Self::consume_report(init, role, ring) { - *quarantined = true; - dev_err!( - dev.as_ref(), - "G17 {:?} {:?} ring stopped: {:?}\n", - role, - ring, - error - ); - } - } - - fn consume_report(init: &InitData, role: Role, ring: ReportRing) -> Result { - let (state, entries, capacity, allowed_types) = ring.geometry(); - let consumer_word = init.status_word(role, state)?; - let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; - fence(Ordering::Acquire); - let mut consumer = consumer_word.load(Ordering::Relaxed); - let producer = producer_word.load(Ordering::Relaxed); - // The producer makes the entries it covers available to the host. - fence(Ordering::Acquire); - if consumer >= capacity || producer >= capacity { - return Err(EIO); - } - while consumer != producer { - let offset = entries + consumer as usize * EVENT_RECORD_SIZE; - let tag = match ring { - // Copy the entire trace entry before returning its slot. - ReportRing::Trace => init.status_read_words::(role, offset)?[0], - ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], - }; - if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { - return Err(EIO); - } - consumer = (consumer + 1) & (capacity - 1); - fence(Ordering::SeqCst); - consumer_word.store(consumer, Ordering::Relaxed); - if matches!(ring, ReportRing::Trace) { - fence(Ordering::SeqCst); - } - } - if matches!(ring, ReportRing::Statistics) { - fence(Ordering::SeqCst); - } - Ok(()) - } - - /// Publishes one primary device-control record. On EAGAIN the caller - /// retains the request and retries before consuming more dependent events. - /// The returned cursor is the value the consumer must reach to retire it. - /// The caller announces the publication through the coprocessor mailbox. - pub(crate) fn publish_control(init: &InitData, record: &ControlRecord) -> Result { - fence(Ordering::Acquire); - let consumer = init.control_consumer()?.load(Ordering::Relaxed); - let producer = init.control_producer()?.load(Ordering::Relaxed); - if consumer >= DEVICE_CONTROL_SLOTS || producer >= DEVICE_CONTROL_SLOTS { - return Err(EIO); - } - let next = (producer + 1) & (DEVICE_CONTROL_SLOTS - 1); - if next == consumer { - return Err(EAGAIN); - } - // InitData validates every mapping before the first write, then writes - // record, packed tally, barrier, producer, barrier without failure. - init.publish_control( - producer, - record.bytes(), - record.opcode().asserts_gpu_power(), - next, - )?; - Ok(next) - } -} - -/// One bounded event-ring pass. No shared-memory borrow survives `next`, so -/// the caller can service the copied event using the complete device owner. -pub(crate) struct Events { - role: Role, - remaining: u32, - unknown_reported: bool, -} - -impl Events { - /// A bounded pass can leave records written while its earlier slots were - /// consumed. Check both cursors again before the worker goes idle. - pub(crate) fn pending(init: &InitData, role: Role) -> Result { - let state = status::EVENT_STATE as usize; - fence(Ordering::Acquire); - let consumer = init.status_word(role, state)?.load(Ordering::Relaxed); - let producer = init - .status_word(role, state + RX_PRODUCER_OFFSET)? - .load(Ordering::Relaxed); - if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { - return Err(EIO); - } - Ok(consumer != producer) - } - - pub(crate) fn new(role: Role) -> Self { - Self { - role, - remaining: EVENT_SLOTS, - unknown_reported: false, - } - } - - pub(crate) fn next(&mut self, dev: &Device, init: &InitData) -> Result> { - let role = self.role; - let state = status::EVENT_STATE as usize; - let consumer_word = init.status_word(role, state)?; - let producer_word = init.status_word(role, state + RX_PRODUCER_OFFSET)?; - while self.remaining != 0 { - init.validate_event_ring(role)?; - self.remaining -= 1; - let consumer = consumer_word.load(Ordering::Relaxed); - let producer = producer_word.load(Ordering::Relaxed); - if consumer >= EVENT_SLOTS || producer >= EVENT_SLOTS { - return Err(EIO); - } - if consumer == producer { - self.remaining = 0; - break; - } - // Observe the producer before reading the record it publishes. - fence(Ordering::Acquire); - let offset = status::EVENT_RING as usize + consumer as usize * EVENT_RECORD_SIZE; - let record = EventRecord::from_words(init.status_read_words(role, offset)?); - let event = record.decode(); - fence(Ordering::SeqCst); - consumer_word.store((consumer + 1) & (EVENT_SLOTS - 1), Ordering::Relaxed); - fence(Ordering::SeqCst); - match event? { - Event::Unknown(tag) => { - if !self.unknown_reported { - dev_warn!( - dev.as_ref(), - "G17 {:?} unknown firmware event {}\n", - role, - tag - ); - self.unknown_reported = true; - } - } - event => return Ok(Some(event)), - } - } - Ok(None) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/command.rs b/drivers/gpu/drm/asahi-neo/g17/command.rs deleted file mode 100644 index aa7d140d177af2..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/command.rs +++ /dev/null @@ -1,447 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Decode a submission and validate its user resources before publishing any work. - -use super::fw::{ - compute::ScratchRequest, - kick::WriteRange, - render::{RenderPass, UscProgram, ZlsSurface}, -}; -use crate::util::{AnyBitPattern, Reader}; -use core::ops::Range; -use kernel::{prelude::*, uapi}; - -const MAX_ATTACHMENTS: usize = crate::microseq::MAX_ATTACHMENTS; -const USC_WINDOW_SIZE: u64 = 1 << 32; -const MAX_SAMPLERS: u32 = 1024; -const SAMPLER_SIZE: u64 = 8; -const PAGE_SIZE: u64 = 0x4000; -const ZLS_METADATA_STRIDE: u64 = 0x80; -const MAX_DIMENSION: u16 = 16384; -const MAX_LAYERS: u16 = 2048; -const MAX_TILE_BUFFER: u64 = 32768; -const PROCESS_EMPTY: u32 = uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_PROCESS_EMPTY_TILES; -const RSRC_SPEC_HI: u32 = uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_RSRC_SPEC_HI; -const DBIAS_INT: u32 = uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_DBIAS_IS_INT; -const FEW_PRIMITIVES: u32 = uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_FEW_PRIMITIVES; -const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_FRAGMENT_BARRIERS; -const RENDER_FLAGS: u32 = - PROCESS_EMPTY | RSRC_SPEC_HI | DBIAS_INT | FEW_PRIMITIVES | FRAGMENT_BARRIERS; - -/// Attachment settings persist until replaced, including by an empty list. -#[derive(Clone, Copy)] -pub(crate) struct Attachments { - ranges: [WriteRange; MAX_ATTACHMENTS], - count: usize, -} - -impl Attachments { - const EMPTY: Self = Self { - ranges: [WriteRange { va: 0, size: 0 }; MAX_ATTACHMENTS], - count: 0, - }; - - pub(crate) fn as_slice(&self) -> &[WriteRange] { - &self.ranges[..self.count] - } - - fn parse(bytes: &[u8]) -> Result { - let size = size_of::(); - if bytes.len() % size != 0 || bytes.len() / size > MAX_ATTACHMENTS { - return Err(EINVAL); - } - let mut list = Self::EMPTY; - let mut reader = Reader::new(bytes); - for range in &mut list.ranges[..bytes.len() / size] { - let item: uapi::drm_asahi_neo_attachment = reader.read()?; - if item.flags != 0 || item.pad != 0 { - return Err(EINVAL); - } - *range = WriteRange { - va: item.pointer, - size: item.size, - }; - list.count += 1; - } - Ok(list) - } - - fn validate(&self, space: &impl AddressSpace) -> Result { - for range in self.as_slice() { - require_range(space, range.va, range.size, Access::Write)?; - } - Ok(()) - } -} - -// Commands stay inline in the submission plan, avoiding an allocation per render pass. -#[allow(clippy::large_enum_variant)] -pub(crate) enum Payload { - Render(uapi::drm_asahi_neo_cmd_render, [Attachments; 2]), - Compute(uapi::drm_asahi_neo_cmd_compute, Attachments), -} - -pub(crate) struct Command { - /// Render and compute prefix indices, before this command. - pub(crate) barriers: [u16; 2], - pub(crate) payload: Payload, -} - -/// Parser over the single userspace snapshot taken by the submit ioctl. -/// Call `finish` after consuming it to reject submissions with no hardware work. -pub(crate) struct Parser<'a> { - bytes: &'a [u8], - counts: [u16; 2], - attachments: [Attachments; 3], -} - -impl<'a> Parser<'a> { - pub(crate) fn new(bytes: &'a [u8]) -> Self { - Self { - bytes, - counts: [0; 2], - attachments: [Attachments::EMPTY; 3], - } - } - - pub(crate) fn next(&mut self) -> Result> { - while !self.bytes.is_empty() { - let mut reader = Reader::new(self.bytes); - let header: uapi::drm_asahi_neo_cmd_header = reader.read()?; - let start = size_of::(); - let end = start + header.size as usize; - let bytes = self.bytes.get(start..end).ok_or(EINVAL)?; - self.bytes = &self.bytes[end..]; - let barriers = [header.vdm_barrier, header.cdm_barrier]; - match header.cmd_type as u32 { - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_RENDER - | uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_COMPUTE => { - if barriers.iter().zip(self.counts).any(|(&barrier, count)| { - barrier != uapi::DRM_ASAHI_NEO_BARRIER_NONE as u16 && barrier > count - }) || u32::from(self.counts[0]) + u32::from(self.counts[1]) - >= crate::file::MAX_COMMANDS_PER_SUBMISSION - { - return Err(EINVAL); - } - let payload = if header.cmd_type as u32 - == uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_RENDER - { - self.counts[0] += 1; - Payload::Render(payload(bytes)?, [self.attachments[0], self.attachments[1]]) - } else { - self.counts[1] += 1; - Payload::Compute(payload(bytes)?, self.attachments[2]) - }; - return Ok(Some(Command { barriers, payload })); - } - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_SET_VERTEX_ATTACHMENTS - | uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_SET_FRAGMENT_ATTACHMENTS - | uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_SET_COMPUTE_ATTACHMENTS => { - if barriers != [uapi::DRM_ASAHI_NEO_BARRIER_NONE as u16; 2] { - return Err(EINVAL); - } - let slot = header.cmd_type as usize - - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_SET_VERTEX_ATTACHMENTS as usize; - self.attachments[slot] = Attachments::parse(bytes)?; - } - _ => return Err(EINVAL), - } - } - Ok(None) - } - - pub(crate) fn finish(&self) -> Result { - if !self.bytes.is_empty() || self.counts == [0; 2] { - return Err(EINVAL); - } - Ok(()) - } -} - -fn payload(bytes: &[u8]) -> Result { - if bytes - .get(size_of::()..) - .is_some_and(|tail| tail.iter().any(|&b| b != 0)) - { - return Err(EINVAL); - } - Reader::new(bytes).read_up_to(bytes.len()) -} - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum Access { - Read, - Write, - ReadWrite, -} - -/// The caller retains the VM mappings while validating and executing the command. -pub(crate) trait AddressSpace { - fn covers(&self, address: u64, size: u64, access: Access) -> bool; -} - -fn require_range(space: &impl AddressSpace, address: u64, size: u64, access: Access) -> Result { - if size == 0 || address.checked_add(size).is_none() || !space.covers(address, size, access) { - return Err(EINVAL); - } - Ok(()) -} - -/// A validated four-GiB USC window owned by one queue. -pub(crate) struct UscWindow { - base: u64, -} -impl UscWindow { - pub(crate) fn new(base: u64, range: Range) -> Result { - if base & (USC_WINDOW_SIZE - 1) != 0 - || base < range.start - || base - .checked_add(USC_WINDOW_SIZE) - .is_none_or(|end| end > range.end) - { - return Err(EINVAL); - } - Ok(Self { base }) - } - - fn program(&self, space: &impl AddressSpace, tagged_offset: u32) -> Result { - if tagged_offset == 0 { - return Err(EINVAL); - } - require_range( - space, - self.base + (tagged_offset & !7) as u64, - 4, - Access::Read, - ) - } -} - -fn sampler(space: &impl AddressSpace, heap: u64, count: u32) -> Result { - if count > MAX_SAMPLERS || (heap == 0) != (count == 0) || heap & (SAMPLER_SIZE - 1) != 0 { - return Err(EINVAL); - } - if count != 0 { - require_range(space, heap, u64::from(count) * SAMPLER_SIZE, Access::Read)?; - } - Ok(()) -} - -fn helper_is_empty(helper: &uapi::drm_asahi_neo_helper_program) -> bool { - helper.binary == 0 && helper.cfg == 0 && helper.data == 0 -} - -fn surface( - space: &impl AddressSpace, - zls: &uapi::drm_asahi_neo_zls_buffer, - layers: u16, -) -> Result { - if (zls.base == 0 && (zls.comp_base != 0 || zls.stride != 0 || zls.comp_stride != 0)) - || (zls.comp_base == 0 && zls.comp_stride != 0) - || (layers > 1 && zls.base != 0 && zls.stride == 0) - || (zls.stride != 0 && u64::from(zls.stride) & (PAGE_SIZE - 1) != 1) - || u64::from(zls.comp_stride) & (PAGE_SIZE - 1) != 0 - { - return Err(EINVAL); - } - if zls.base != 0 { - let stride = if zls.stride == 0 { - 0 - } else { - (u64::from(zls.stride) / PAGE_SIZE + 1) * PAGE_SIZE - }; - require_range( - space, - zls.base, - stride * u64::from(layers - 1) + 1, - Access::ReadWrite, - )?; - } - if zls.comp_base != 0 { - let stride = (u64::from(zls.comp_stride) / PAGE_SIZE + 1) * ZLS_METADATA_STRIDE; - require_range( - space, - zls.comp_base, - stride * u64::from(layers - 1) + 1, - Access::ReadWrite, - )?; - } - Ok(ZlsSurface { - va: zls.base, - compression_va: zls.comp_base, - stride: zls.stride.into(), - compression_stride: zls.comp_stride.into(), - }) -} - -// As with Payload, retain the bounded attachment lists inside the submission plan. -#[allow(clippy::large_enum_variant)] -pub(crate) enum Validated { - Render { - pass: RenderPass, - attachments: [Attachments; 2], - timestamps: [uapi::drm_asahi_neo_timestamps; 2], - }, - Compute { - scratch: ScratchRequest, - usc_base: u64, - cdm_va: u64, - cdm_end_va: u64, - sampler_heap_va: u64, - sampler_count: u32, - timestamps: uapi::drm_asahi_neo_timestamps, - }, -} - -impl Payload { - pub(crate) fn prior(&self) -> [u64; 2] { - match self { - Self::Render(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], - Self::Compute(cmd, _) => [cmd.prior_vdm, cmd.prior_cdm], - } - } - - pub(crate) fn validate( - &self, - window: &UscWindow, - space: &impl AddressSpace, - ) -> Result { - match self { - Self::Render(cmd, attachments) => { - if cmd.flags & !RENDER_FLAGS != 0 - || !(1..=MAX_DIMENSION).contains(&cmd.width_px) - || !(1..=MAX_DIMENSION).contains(&cmd.height_px) - || !(1..=MAX_LAYERS).contains(&cmd.layers) - || !matches!( - (cmd.utile_width_px, cmd.utile_height_px), - (16, 16) | (32, 16) | (32, 32) - ) - { - return Err(EINVAL); - } - let samples_log2 = match cmd.samples { - 1 => 0, - 2 => 1, - 4 => 2, - _ => return Err(EINVAL), - }; - if u64::from(cmd.sample_size_B) - * u64::from(cmd.utile_width_px) - * u64::from(cmd.utile_height_px) - * u64::from(cmd.samples) - > MAX_TILE_BUFFER - || cmd.vdm_ctrl_stream_base == 0 - || cmd.vdm_ctrl_stream_base & 3 != 0 - || cmd.isp_scissor_base == 0 - || cmd.isp_scissor_base & 7 != 0 - { - return Err(EINVAL); - } - require_range(space, cmd.vdm_ctrl_stream_base, 4, Access::Read)?; - require_range(space, cmd.isp_scissor_base, 8, Access::Read)?; - for (va, access) in [ - (cmd.isp_dbias_base, Access::Read), - (cmd.isp_oclqry_base, Access::Write), - ] { - if va != 0 { - if va & 7 != 0 { - return Err(EINVAL); - } - require_range(space, va, 8, access)?; - } - } - let depth = surface(space, &cmd.depth, cmd.layers)?; - let stencil = surface(space, &cmd.stencil, cmd.layers)?; - sampler(space, cmd.sampler_heap, cmd.sampler_count.into())?; - if !helper_is_empty(&cmd.vertex_helper) || !helper_is_empty(&cmd.fragment_helper) { - return Err(EINVAL); - } - for program in [&cmd.bg, &cmd.eot, &cmd.partial_bg, &cmd.partial_eot] { - window.program(space, program.usc)?; - } - for list in attachments { - list.validate(space)?; - } - let pass = RenderPass { - width: cmd.width_px.into(), - height: cmd.height_px.into(), - layers: cmd.layers, - utile_width: cmd.utile_width_px, - utile_height: cmd.utile_height_px, - samples_log2, - sample_size: cmd.sample_size_B, - process_empty_tiles: cmd.flags & PROCESS_EMPTY != 0, - msaa_control: cmd.ppp_multisamplectl, - ppp_control: cmd.ppp_ctrl.into(), - vdm_va: cmd.vdm_ctrl_stream_base, - scissor_va: cmd.isp_scissor_base, - depth_bias_va: cmd.isp_dbias_base, - depth_bias_is_int: cmd.flags & DBIAS_INT != 0, - few_primitives: cmd.flags & FEW_PRIMITIVES != 0, - fragment_barriers: cmd.flags & FRAGMENT_BARRIERS != 0, - occlusion_query_va: cmd.isp_oclqry_base, - sampler_heap_va: cmd.sampler_heap, - sampler_count: cmd.sampler_count.into(), - zls_control: cmd.zls_ctrl, - depth_dimensions: cmd.isp_zls_pixels.into(), - depth, - stencil, - depth_clear: cmd.isp_bgobjdepth, - stencil_clear: cmd.isp_bgobjvals, - merge_upper_x: cmd.isp_merge_upper_x, - merge_upper_y: cmd.isp_merge_upper_y, - usc_base: window.base, - bg: UscProgram { - offset: cmd.bg.usc, - rsrc_spec: u64::from(cmd.bg.rsrc_spec) - | (u64::from(cmd.bg_rsrc_spec_hi) << 32), - }, - partial_bg: UscProgram { - offset: cmd.partial_bg.usc, - rsrc_spec: u64::from(cmd.partial_bg.rsrc_spec) - | (u64::from(cmd.bg_partial_rsrc_spec_hi) << 32), - }, - eot_offset: cmd.eot.usc, - eot_rsrc_spec: u64::from(cmd.eot.rsrc_spec) - | (u64::from(cmd.bg_eot_rsrc_spec_hi) << 32), - partial_eot_offset: cmd.partial_eot.usc, - partial_eot_rsrc_spec: u64::from(cmd.partial_eot.rsrc_spec) - | (u64::from(cmd.bg_eot_partial_rsrc_spec_hi) << 32), - rsrc_spec_hi: cmd.flags & RSRC_SPEC_HI != 0, - }; - Ok(Validated::Render { - pass, - attachments: *attachments, - timestamps: [cmd.ts_vtx, cmd.ts_frag], - }) - } - Self::Compute(cmd, attachments) => { - if cmd.flags != 0 - || cmd.cdm_ctrl_stream_base & 3 != 0 - || cmd.cdm_ctrl_stream_end & 3 != 0 - || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base - || !helper_is_empty(&cmd.helper) - { - return Err(EINVAL); - } - require_range( - space, - cmd.cdm_ctrl_stream_base, - cmd.cdm_ctrl_stream_end - cmd.cdm_ctrl_stream_base, - Access::Read, - )?; - sampler(space, cmd.sampler_heap, cmd.sampler_count)?; - attachments.validate(space)?; - Ok(Validated::Compute { - scratch: ScratchRequest::new(cmd.scs_layout)?, - usc_base: window.base, - cdm_va: cmd.cdm_ctrl_stream_base, - cdm_end_va: cmd.cdm_ctrl_stream_end - 4, - sampler_heap_va: cmd.sampler_heap, - sampler_count: cmd.sampler_count, - timestamps: cmd.ts, - }) - } - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/completion.rs b/drivers/gpu/drm/asahi-neo/g17/completion.rs deleted file mode 100644 index 570da5d23af0ca..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/completion.rs +++ /dev/null @@ -1,520 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Per-command fences, timestamp destinations and VM pins retained through retirement. -//! -//! Runtime queues own completions before making work visible. They defer fence signalling -//! until after the device mutex is released, and retain failed visible work until a retirement -//! witness or processor stop. An error fence never substitutes for that ownership boundary. - -use super::{ - context::{Context, WorkStateLease}, - fence::{self, CompletionFence, Member}, - object::{CpuMap, KernelObject}, - recovery::Class, - status::VmStatus, -}; -use crate::{driver::AsahiDevice, file, hw::t8140::{dynamic, resources as cfg}, mmu}; -use core::sync::atomic::{fence as barrier, AtomicBool, AtomicI32, AtomicU8, Ordering}; -use kernel::{ - dma_fence::{Fence, FenceContexts, RawDmaFence, UserFence}, - new_mutex, - prelude::*, - sync::{Arc, Mutex}, - uapi, xarray, -}; - -mod publication; -pub(crate) use publication::{Prepared as PreparedPublication, Publication}; - -struct Destination { - // Keep the existing GPU alias alive along with its prepared CPU view. - _mapping: Arc, - cpu: mmu::TimestampMapping, -} - -impl Destination { - fn resolve( - objects: Pin<&xarray::XArray>>, - timestamp: uapi::drm_asahi_neo_timestamp, - ) -> Result> { - if timestamp.handle == 0 { - return Ok(None); - } - let object = objects - .lock() - .get(timestamp.handle.try_into()?) - .ok_or(ENOENT)? - .clone(); - let file::Object::TimestampBuffer(mapping) = object; - let offset: usize = timestamp.offset.try_into()?; - if offset.checked_add(size_of::()).ok_or(EINVAL)? > mapping.size() { - return Err(ERANGE); - } - let cpu = mapping.timestamp_mapping(offset)?; - Ok(Some(Self { _mapping: mapping, cpu })) - } - - fn write(&self, ticks: u64) { - const NSEC_PER_SEC: u64 = 1_000_000_000; - let ns = (ticks / cfg::COMMAND_TIMESTAMP_HZ) - .wrapping_mul(NSEC_PER_SEC) - .wrapping_add( - ticks % cfg::COMMAND_TIMESTAMP_HZ * NSEC_PER_SEC / cfg::COMMAND_TIMESTAMP_HZ, - ); - self.cpu.write(ns); - barrier(Ordering::Release); - } -} - -/// Timestamp mappings resolved and retained before any command is enqueued. -pub(crate) struct Destinations(Option; 4]>>); - -impl Destinations { - pub(crate) fn resolve( - objects: Pin<&xarray::XArray>>, - timestamps: &[uapi::drm_asahi_neo_timestamps], - ) -> Result { - if timestamps.is_empty() || timestamps.len() > 2 { - return Err(EINVAL); - } - let mut destinations = core::array::from_fn(|_| None); - for (index, timestamp) in timestamps.iter().enumerate() { - destinations[2 * index] = Destination::resolve(objects, timestamp.start)?; - destinations[2 * index + 1] = Destination::resolve(objects, timestamp.end)?; - } - // Empty submissions need no cleanup allocation. Timestamped commands - // prepare the unique work item before any output fence is published. - let retained = if destinations.iter().any(Option::is_some) { - Some(crate::cleanup::Deferred::new(destinations)?) - } else { - None - }; - Ok(Self(retained)) - } - - fn iter(&self) -> core::slice::Iter<'_, Option> { - let destinations: &[Option] = - self.0.as_deref().map_or(&[], |destinations| destinations.as_slice()); - destinations.iter() - } -} - -/// Public outputs do not own physical command resources or a scheduler job. -struct Output { - fence: UserFence, - member: Member, - destinations: Destinations, -} - -impl Output { - fn publish(&self, result: Result<[u64; 4]>) { - match result { - Ok(ticks) => { - for (destination, ticks) in self.destinations.iter().zip(ticks) { - if let Some(destination) = destination { - destination.write(ticks); - } - } - } - Err(error) => self.fence.set_error(error), - } - self.fence.signal(); - self.member.complete(result.map(|_| ())); - } -} - -enum HostOutput { - Immediate(Output), - Ordered(Arc), -} - -impl HostOutput { - fn output(&self) -> &Output { - match self { - Self::Immediate(output) => output, - Self::Ordered(publication) => publication.output(), - } - } - - fn finish(&self, result: Result<[u64; 4]>) { - match self { - Self::Immediate(output) => output.publish(result), - Self::Ordered(publication) => publication.finish(result), - } - } -} - -/// Firmware writes two aligned words in a fresh write-combined page per compute command. -struct ComputeTimestamps { - // Remove the GPU alias before releasing its backing. - mapping: mmu::KernelMapping, - object: KernelObject, -} - -impl ComputeTimestamps { - fn new(dev: &AsahiDevice, vm: &mmu::Vm) -> Result { - let object = KernelObject::backing(dev, mmu::UAT_PGSZ, CpuMap::WriteCombined)?; - let mapping = object.alias_in( - vm, - // Canonical shared queue aliases may be installed in this VM later. - // Private timestamp pages must never occupy their retained addresses. - dynamic::CLIENT_LOWER, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - )?; - Ok(Self { mapping, object }) - } -} - -#[pin_data] -pub(crate) struct Completion { - status: Arc, - output: HostOutput, - // Present only when this command's public result may wait for a render - // predecessor's host writes. DRM credits must follow the terminal result. - terminal: Option>, - compute: Option, - feed: Option>, - #[pin] - vm_job: Mutex>, - #[pin] - render: Mutex>, - signalled: AtomicBool, - retirement_queued: AtomicBool, - accepted: AtomicBool, - timeout_error: AtomicI32, - replays: AtomicU8, - replay_timeout: AtomicBool, - firmware_consumer: AtomicBool, - class: AtomicU8, -} - -impl Completion { - pub(crate) fn new( - dev: &AsahiDevice, - contexts: &FenceContexts, - context: &Arc, - member: Member, - vm_job: mmu::VmJobGuard, - destinations: Destinations, - compute: bool, - host_predecessors: bool, - feed: Option>, - ) -> Result> { - let fence = fence::independent(contexts)?; - let ordered = !compute - && (host_predecessors || destinations.iter().any(Option::is_some)); - let terminal = if host_predecessors { - Some(fence::independent(contexts)?) - } else { - None - }; - let output = Output { fence, member, destinations }; - let output = if ordered { - HostOutput::Ordered(Publication::new(output, context.status().clone())?) - } else { - HostOutput::Immediate(output) - }; - let compute = if compute { - Some(ComputeTimestamps::new(dev, context.vm())?) - } else { - None - }; - Arc::pin_init( - pin_init!(Self { - status: context.status().clone(), - output, - terminal, - compute, - feed, - vm_job <- new_mutex!(Some(vm_job), "G17 command VM pin"), - render <- new_mutex!(None, "G17 command render lease"), - signalled: AtomicBool::new(false), - retirement_queued: AtomicBool::new(false), - accepted: AtomicBool::new(false), - timeout_error: AtomicI32::new(0), - replays: AtomicU8::new(0), - replay_timeout: AtomicBool::new(false), - firmware_consumer: AtomicBool::new(false), - class: AtomicU8::new(0), - }), - GFP_KERNEL, - ) - } - - pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.output.output().fence) - } - pub(crate) fn scheduler_fence(&self) -> Fence { - self.terminal.as_ref().map_or_else(|| self.fence(), |fence| Fence::from_fence(fence)) - } - pub(crate) fn publication(&self) -> Option> { - match &self.output { - HostOutput::Ordered(publication) => Some(publication.clone()), - HostOutput::Immediate(_) => None, - } - } - fn signal_terminal(&self, result: Result<[u64; 4]>) { - if let Some(fence) = &self.terminal { - if let Err(error) = result { - fence.set_error(error); - } - fence.signal(); - } - } - /// All fallible scheduler setup has succeeded. Failures from this point - /// must be visible through VM_STATUS before the accepted job's fence. - pub(crate) fn mark_accepted(&self) { - self.accepted.store(true, Ordering::Release); - } - pub(crate) fn status(&self) -> &Arc { - &self.status - } - pub(crate) fn work_state(&self) -> Result> { - self.output.output().member.work_state() - } - pub(crate) fn has_timestamps(&self) -> bool { - self.output.output().destinations.iter().any(Option::is_some) - } - - pub(crate) fn compute_timestamp_va(&self) -> Result { - Ok(self.compute.as_ref().ok_or(EINVAL)?.mapping.iova()) - } - - /// The caller has proved that firmware retired this command, or has - /// received its scheduler kill acknowledgement with recovery closed. - /// These are the command's public timestamp words, not queue-private state. - pub(crate) fn compute_words(&self) -> Result<[u64; 2]> { - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; - let words = [ - object.dword(0)?.load(Ordering::Relaxed), - object.dword(8)?.load(Ordering::Relaxed), - ]; - barrier(Ordering::Acquire); - Ok(words) - } - - pub(crate) fn replay_witness_words(&self) -> Option<[u64; 2]> { - if self.signalled.load(Ordering::Acquire) { - return None; - } - self.compute_words().ok() - } - - pub(crate) fn begin_replay(&self) -> bool { - let accepted = self - .replays - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { - (count < 3).then_some(count + 1) - }) - .is_ok(); - if accepted { - self.replay_timeout.store(true, Ordering::Release); - } - accepted - } - - pub(crate) fn replay_count(&self) -> u8 { - self.replays.load(Ordering::Acquire) - } - - pub(crate) fn note_firmware_consumer(&self) { - self.firmware_consumer.store(true, Ordering::Release); - } - - pub(crate) fn has_firmware_consumer(&self) -> bool { - self.firmware_consumer.load(Ordering::Acquire) - } - - pub(crate) fn take_replay_timeout(&self) -> bool { - !self.signalled.load(Ordering::Acquire) && self.replay_timeout.swap(false, Ordering::AcqRel) - } - - /// Only a witnessed, retained command can be reset before republishing it. - pub(crate) fn reset_for_replay(&self) -> Result { - if self.signalled.load(Ordering::Acquire) { - return Err(ECANCELED); - } - let object = &self.compute.as_ref().ok_or(EINVAL)?.object; - let start = object.dword(0)?; - let end = object.dword(8)?; - start.store(0, Ordering::Relaxed); - end.store(0, Ordering::Relaxed); - barrier(Ordering::SeqCst); - Ok(()) - } - - /// A retirement witness has been accepted by an off-lock completion batch. - /// This is separate from fence signalling and from a failure still owned by firmware. - pub(crate) fn note_retirement_queued(&self) { - self.retirement_queued.store(true, Ordering::Release); - } - - pub(crate) fn retirement_queued(&self) -> bool { - self.retirement_queued.load(Ordering::Acquire) - } - - /// The first timeout request retains its errno while the physical graph is - /// borrowed. Duplicate watchdog or cancellation callbacks cannot replace it. - pub(crate) fn defer_timeout(&self, error: Error) { - let _ = self.timeout_error.compare_exchange( - 0, - error.to_errno(), - Ordering::AcqRel, - Ordering::Acquire, - ); - } - - pub(crate) fn take_deferred_timeout(&self) -> Option { - let error = self.timeout_error.swap(0, Ordering::AcqRel); - (error != 0).then(|| Error::from_errno(error)) - } - - pub(crate) fn classify(&self, class: Class) { - self.class.store( - match class { - Class::Spared => 1, - Class::Guilty => 2, - }, - Ordering::Release, - ); - } - - pub(crate) fn spared(&self) -> bool { - self.class.load(Ordering::Acquire) == 1 - } - - /// Installed before the first firmware-visible prefix. Cancellation takes - /// the same lock after claiming the signal, so it cannot miss a late lease. - pub(crate) fn attach_render_lease( - &self, - lease: super::queue::admission::RenderLease, - ) -> Result { - let mut render = self.render.lock(); - if self.signalled.load(Ordering::Acquire) { - return Err(ECANCELED); - } - if render.is_some() { - return Err(EIO); - } - *render = Some(lease); - Ok(()) - } - - pub(crate) fn render_slot(&self) -> Option { - self.render - .lock() - .as_ref() - .map(super::queue::admission::RenderLease::slot) - } - - /// Called under the device mutex before deferring the error signal. - pub(crate) fn quarantine_render(&self) { - if let Some(lease) = self.render.lock().as_ref() { - lease.quarantine(); - } - } - - /// An exact whole-pair drain or processor stop allows rehabilitation. The - /// VM pin goes first; the still-held lease excludes reuse until both settle. - pub(crate) fn release_after_retirement(&self) { - drop(self.vm_job.lock().take()); - let lease = self.render.lock().take(); - if let Some(lease) = lease.as_ref() { - lease.rehabilitate_after_firmware_drain(); - } - drop(lease); - } - - fn release_healthy(&self) { - drop(self.render.lock().take()); - drop(self.vm_job.lock().take()); - } - - /// Exact retirement permits the physical lease and VM pin to be released - /// before waking userspace. A failure fence alone never establishes this. - #[track_caller] - pub(crate) fn complete(&self, result: Result<[u64; 4]>) { - if self.signalled.swap(true, Ordering::AcqRel) { - return; - } - if let (Some(feed), Ok(ticks)) = (&self.feed, &result) { - feed.note_render_pass(ticks[2], ticks[3]); - } - if let Err(error) = result { - if self.spared() { - self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); - } - } - self.release_healthy(); - self.signal_terminal(result); - self.output.finish(result); - } - - /// A rejected, never-published command cannot poison the VM for contention. - #[track_caller] - pub(crate) fn fail_unpublished(&self, error: Error) { - if self.signalled.swap(true, Ordering::AcqRel) { - return; - } - self.status.record_if_device_loss(error); - if self.accepted.load(Ordering::Acquire) { - self.status.report_failure(error); - } - self.release_healthy(); - self.signal_terminal(Err(error)); - self.output.finish(Err(error)); - } - - /// The physical queue must already be quarantined. Keep mappings pinned while - /// waking waiters, even if queue destruction requested cancellation. - #[track_caller] - pub(crate) fn fail_while_owned(&self, error: Error) { - if self.signalled.swap(true, Ordering::AcqRel) { - return; - } - if let Some(guard) = self.vm_job.lock().as_ref() { - guard.quarantine(); - } - if self.spared() { - self.status.report_failure(error); - } else if error != ECANCELED { - self.status.record(error); - } - self.signal_terminal(Err(error)); - self.output.finish(Err(error)); - } -} - -/// Collected under the device mutex and executed only after releasing it. -/// Runtime queues establish the ownership witness before constructing `Retired` -/// or `Release`; `FailedOwned` leaves their physical and VM ownership intact. -pub(crate) enum Deferred { - Retired(Arc, Result<[u64; 4]>), - Drained(Arc, Result<[u64; 4]>), - FailedOwned(Arc, Error), - Unpublished(Arc, Error), - Release(Arc), -} - -impl Deferred { - pub(crate) fn finish(self) { - match self { - Self::Retired(completion, result) => { - completion.complete(result); - // An earlier failure may already have signalled this fence while - // retaining the pin. Retirement is a separate release witness. - completion.release_after_retirement(); - } - Self::Drained(completion, result) => { - completion.release_after_retirement(); - completion.complete(result); - } - Self::FailedOwned(completion, error) => completion.fail_while_owned(error), - Self::Unpublished(completion, error) => completion.fail_unpublished(error), - Self::Release(completion) => completion.release_after_retirement(), - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/completion/publication.rs b/drivers/gpu/drm/asahi-neo/g17/completion/publication.rs deleted file mode 100644 index a1e05a9408552c..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/completion/publication.rs +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Host-only ordering of timestamp writes, independent of scheduler credits. -//! -//! Links point from earlier logical render ordinals to later ones. They retain -//! only public output state, never a packet, physical render lease or VM job pin. -//! All link storage is allocated before acceptance. Completion neither allocates -//! nor waits for another command; the last prerequisite publishes a ready result. - -use super::Output; -use crate::g17::{dependency::inherited_error, status::VmStatus}; -use kernel::{new_mutex, prelude::*, sync::{Arc, Mutex}}; - -type Links = Option>; - -struct Link { - target: Arc, - result: Result, - next: Links, - // Reuse an already-consumed link as a traversal frame instead of recursing - // or allocating when publication unlocks a long chain of ready successors. - resume: Links, -} - -struct State { - remaining: usize, - result: Option>, - predecessor_error: Option, - claimed: bool, - published: Option, - successors: Links, -} - -impl State { - fn claim(&mut self) -> Option> { - if self.remaining != 0 || self.claimed { - return None; - } - let result = self.result.take()?; - self.claimed = true; - Some(match (result, self.predecessor_error) { - (Ok(_), Some(error)) => Err(error), - (result, _) => result, - }) - } -} - -#[pin_data] -pub(crate) struct Publication { - output: Output, - status: Arc, - #[pin] - state: Mutex, -} - -impl Publication { - pub(super) fn new(output: Output, status: Arc) -> Result> { - Arc::pin_init( - pin_init!(Self { - output, - status, - state <- new_mutex!(State { - // Acceptance installs all links before releasing this sentinel. - remaining: 1, - result: None, - predecessor_error: None, - claimed: false, - published: None, - successors: None, - }, "G17 host timestamp publication"), - }), - GFP_KERNEL, - ) - } - - pub(super) fn output(&self) -> &Output { - &self.output - } - - /// One terminal result, offered separately from its physical-retirement proof. - pub(super) fn finish(&self, result: Result<[u64; 4]>) { - let ready = { - let mut state = self.state.lock(); - if state.claimed || state.result.is_some() { - return; - } - state.result = Some(result); - state.claim() - }; - if let Some(result) = ready { - Self::drain(self.publish(result)); - } - } - - fn publish(&self, result: Result<[u64; 4]>) -> Links { - if let Err(error) = result { - // Inherited failure is visible to the client without inventing a - // permanent admission failure in this command's VM. - self.status.report_failure(error); - } - self.output.publish(result); - let result = result.map(|_| ()); - let mut links = { - let mut state = self.state.lock(); - // A late link is acknowledged only after timestamp writes, the public - // fence and aggregate membership have all been published off this lock. - state.published = Some(result); - state.successors.take() - }; - let mut next = links.as_mut(); - while let Some(link) = next { - link.result = result; - next = link.next.as_mut(); - } - links - } - - fn predecessor_done(&self, result: Result) -> Links { - let ready = { - let mut state = self.state.lock(); - debug_assert!(state.remaining != 0); - if let Err(error) = result { - if state.predecessor_error.is_none() { - state.predecessor_error = Some(inherited_error(error)); - } - } - state.remaining -= 1; - state.claim() - }; - ready.and_then(|result| self.publish(result)) - } - - fn add_successor(&self, mut link: KBox) { - let published = { - let mut state = self.state.lock(); - match state.published { - Some(result) => result, - None => { - link.next = state.successors.take(); - state.successors = Some(link); - return; - } - } - }; - Self::drain(link.target.predecessor_done(published)); - } - - /// Iterative depth-first notification, O(nodes + links), with no stack growth. - fn drain(mut pending: Links) { - let mut frames: Links = None; - loop { - if let Some(mut link) = pending.take() { - pending = link.next.take(); - let children = link.target.predecessor_done(link.result); - if children.is_some() { - link.resume = pending.take(); - link.next = frames.take(); - frames = Some(link); - pending = children; - } - } else if let Some(mut frame) = frames.take() { - frames = frame.next.take(); - pending = frame.resume.take(); - } else { - break; - } - } - } -} - -/// Temporary backwards references exist only during fallible setup. Installed -/// links retain the successor only, so closing a queue cannot form an Arc cycle. -pub(crate) struct Prepared { - target: Arc, - links: KVec<(Arc, KBox)>, -} - -impl Prepared { - pub(crate) fn new<'a>( - target: Arc, - predecessors: impl Iterator> + Clone, - ) -> Result { - let mut links = KVec::with_capacity(predecessors.clone().count(), GFP_KERNEL)?; - for predecessor in predecessors { - let link = KBox::new(Link { - target: target.clone(), - result: Ok(()), - next: None, - resume: None, - }, GFP_KERNEL)?; - links.push((predecessor.clone(), link), GFP_KERNEL)?; - } - Ok(Self { target, links }) - } - - /// Called after all fallible scheduler setup and before any GPU publication. - pub(crate) fn activate(self) { - { - let mut state = self.target.state.lock(); - debug_assert!(state.remaining == 1 && !state.claimed); - // The fully allocated link vector bounds this count below usize::MAX. - state.remaining += self.links.len(); - } - for (predecessor, link) in self.links { - predecessor.add_successor(link); - } - Publication::drain(self.target.predecessor_done(Ok(()))); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/context.rs b/drivers/gpu/drm/asahi-neo/g17/context.rs deleted file mode 100644 index afd0d1d22f5bbb..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/context.rs +++ /dev/null @@ -1,398 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Logical queue ownership of scheduler, QoS and work-state identities. -//! -//! Installed bindings retain this context until replacement is acknowledged. -//! A job keeps its work-state lease through exact retirement, even if its error -//! fence has already been signalled. Context-local accounting never waits for GPU work. - -mod identity; -use super::{ - freelist::{RenderIds, RenderPool}, - fw::{context as fw, queue::Policy}, - object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, - status::VmStatus, -}; -use crate::mmu::{self, ExecutionContext, Vm, UAT_PGSZ}; -use core::sync::atomic::{fence, AtomicBool, AtomicU32, Ordering}; -use identity::{RenderSlots, WorkStates, FREE, NODE_STRIDE, STATE_COUNT}; -use kernel::{ - new_mutex, - prelude::*, - sync::{Arc, Mutex}, -}; - -/// QoS IDs belong to one device; the two lowest values are reserved. -#[pin_data] -pub(crate) struct QosIds { - #[pin] - used: Mutex, -} -impl QosIds { - pub(crate) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { used <- new_mutex!(3, "G17 QoS IDs") }), - GFP_KERNEL, - ) - } - fn acquire(self: &Arc) -> Result { - let mut used = self.used.lock(); - let id = used.trailing_ones(); - if id == 128 { - return Err(ENOSPC); - } - *used |= 1u128 << id; - Ok(QosLease { - ids: self.clone(), - id: id as u8, - }) - } -} -struct QosLease { - ids: Arc, - id: u8, -} -impl Drop for QosLease { - fn drop(&mut self) { - *self.ids.used.lock() &= !(1u128 << self.id); - } -} - -struct WorkStorage { - object: PooledObject, - words: [Option; STATE_COUNT], - counts: [u32; STATE_COUNT], - render: RenderSlots, -} -impl WorkStorage { - fn initialize(&mut self, word: u32) -> Result { - let slot = (word & 63) as usize; - if slot >= STATE_COUNT { - return Err(EINVAL); - } - let counter = UAT_PGSZ + slot * 4; - let node = fw::WorkNode::new(self.object.gpu_va() + counter as u64, word).ok_or(EINVAL)?; - // The exact work-state lease excludes a previous firmware owner. Only - // the node's 0xc8 initialized bytes are replaced; the stride tail is retained. - self.object.write(slot * NODE_STRIDE, node)?; - self.object.word(counter)?.store(0, Ordering::Relaxed); - self.words[slot] = Some(word); - self.counts[slot] = 0; - fence(Ordering::Release); - Ok(()) - } - fn change_count(&mut self, word: u32, kicks: u32, add: bool) -> Result { - let slot = (word & 63) as usize; - if self.words.get(slot) != Some(&Some(word)) { - return Err(EINVAL); - } - let value = if add { - self.counts[slot].checked_add(kicks) - } else { - self.counts[slot].checked_sub(kicks) - } - .ok_or(EOVERFLOW)?; - self.object - .word(UAT_PGSZ + slot * 4)? - .store(value, Ordering::Relaxed); - self.counts[slot] = value; - fence(Ordering::SeqCst); - Ok(value) - } -} - -/// Firmware-visible pages of a context, retained by an installed queue whose -/// last owner has exited (see [`Context::retain_firmware_pages`]). -pub(crate) struct FirmwarePages { - _scheduler: Arc, - _work: Arc>, -} - -pub(crate) struct Context { - execution: Arc, - status: Arc, - scheduler: Arc, - scheduler_generation: u32, - published: AtomicBool, - qos: QosLease, - owner_pid: u32, - states: WorkStates, - work: Arc>, - work_base: u64, - render: Arc>>>, - /// Fixed for the context's life. Physical queues bound to it carry this - /// profile in every record, ring, doorbell and registration. - policy: Policy, -} -impl Context { - pub(crate) fn new( - alloc: &Allocator<'_>, - pool: &Arc, - qos: &Arc, - execution: Arc, - owner_pid: u32, - policy: Policy, - ) -> Result { - let status = execution.vm().status().ok_or(EINVAL)?.clone(); - let qos = qos.acquire()?; - let mut scheduler = alloc.kernel( - UAT_PGSZ, - UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - scheduler.initialize::(0, |value| { - value.init(); - Ok(()) - })?; - fence(Ordering::SeqCst); - let scheduler_generation = super::ids::OBJECTS.object()?; - let scheduler = Arc::new(scheduler, GFP_KERNEL)?; - let mut object = pool.allocate( - alloc, - Family::WorkState, - 2 * UAT_PGSZ, - UAT_PGSZ, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let work_base = object.gpu_va(); - object.write( - UAT_PGSZ + 0x400, - fw::WorkHead { - next: 0, - previous: work_base + (UAT_PGSZ + 0x400) as u64, - unk10: 0, - unk14: 0, - }, - )?; - fence(Ordering::Release); - let work = Arc::pin_init( - new_mutex!( - WorkStorage { - object, - words: [None; STATE_COUNT], - counts: [0; STATE_COUNT], - render: RenderSlots::new() - }, - "G17 work states" - ), - GFP_KERNEL, - )?; - let render = Arc::pin_init(new_mutex!(None, "G17 context private memory"), GFP_KERNEL)?; - Ok(Self { - execution, - status, - scheduler, - scheduler_generation, - published: AtomicBool::new(false), - qos, - owner_pid, - states: WorkStates::new(), - work, - work_base, - render, - policy, - }) - } - pub(crate) fn vm(&self) -> &Vm { - self.execution.vm() - } - pub(crate) fn status(&self) -> &Arc { - &self.status - } - pub(crate) fn id(&self) -> u32 { - self.execution.id() - } - - pub(crate) fn generation(&self) -> u8 { - self.execution.generation() - } - pub(crate) fn is_current(&self) -> bool { - self.execution.is_current() - } - pub(crate) fn owner_pid(&self) -> u32 { - self.owner_pid - } - pub(crate) fn policy(&self) -> Policy { - self.policy - } - pub(crate) fn qos_id(&self) -> u8 { - self.qos.id - } - pub(crate) fn scheduler_va(&self) -> u64 { - self.scheduler.gpu_va() - } - /// Installed physical channels retain this page after the logical execution - /// context and its VM lease have completed their scheduler release. - pub(crate) fn scheduler_owner(&self) -> Arc { - self.scheduler.clone() - } - /// Scheduler release and command retirement have both been witnessed. - /// Keep installed storage alive while returning the finite TTBAT identity. - pub(crate) fn release_execution(&self) { - self.execution.release(); - } - pub(crate) fn scheduler_generation(&self) -> u32 { - self.scheduler_generation - } - /// The pages an installed queue record of this context still names: the - /// scheduler page and the work storage holding the job-list head. A retained - /// physical queue keeps these after dropping the context itself. - pub(crate) fn retain_firmware_pages(&self) -> FirmwarePages { - FirmwarePages { - _scheduler: self.scheduler.clone(), - _work: self.work.clone(), - } - } - pub(crate) fn work_head_va(&self) -> u64 { - self.work_base + (UAT_PGSZ + 0x400) as u64 - } - /// Shared scheduler list head, sampled under the context's storage lock. - pub(crate) fn work_head(&self) -> Result<[u64; 2]> { - let work = self.work.lock(); - let head = [ - work.object.dword(UAT_PGSZ + 0x400)?.load(Ordering::Relaxed), - work.object.dword(UAT_PGSZ + 0x408)?.load(Ordering::Relaxed), - ]; - fence(Ordering::Acquire); - Ok(head) - } - - pub(crate) fn is_published(&self) -> bool { - self.published.load(Ordering::Acquire) - } - pub(crate) fn mark_published(&self) { - self.published.store(true, Ordering::Release); - } - - pub(crate) fn release_identity(&self) -> Result<[u8; 4]> { - let mut result = [0; 4]; - fence(Ordering::Acquire); - for (value, offset) in result.iter_mut().zip([0, 1, 4, 0x26]) { - let pointer = self.scheduler.pointer(offset, 1)?; - // SAFETY: pointer checked against retained scheduler allocation; - // a byte read observes firmware state without borrowing its fields. - *value = unsafe { core::ptr::read_volatile(pointer) }; - } - Ok(result) - } - pub(crate) fn render_pool( - &self, - alloc: &Allocator<'_>, - ids: &Arc, - render_global: &Vm, - ) -> Result> { - if !self.is_current() { - return Err(EFAULT); - } - let mut render = self.render.lock(); - if let Some(pool) = &*render { - // A pool vacated under memory pressure is re-armed before reuse. - pool.ensure_backing(alloc, render_global, self.vm())?; - return Ok(pool.clone()); - } - let pool = Arc::new( - RenderPool::new(alloc, ids, render_global, self.vm())?, - GFP_KERNEL, - )?; - *render = Some(pool.clone()); - Ok(pool) - } - /// Drops the pool once its free-list release is consumed and the closed - /// scheduler is detached. Installed compute bindings may retain the - /// context itself until replacement; they never use the render pool. - pub(crate) fn take_render_pool(&self) -> Option> { - self.render.lock().take() - } - pub(crate) fn render_pools_released(&self, released: impl Fn(u32, u64) -> bool) -> bool { - self.render - .lock() - .as_ref() - .is_none_or(|pool| released(u32::from(pool.id()), pool.control_va())) - } -} - -/// Shared by all engines of one ioctl. Queued work initially consumes no node. -pub(crate) struct WorkStateLease { - context: Arc, - word: AtomicU32, -} -impl WorkStateLease { - pub(crate) fn deferred(context: Arc) -> Self { - Self { - context, - word: AtomicU32::new(FREE), - } - } - pub(crate) fn acquire(&self) -> Result { - let current = self.word(); - if current != FREE { - return Ok(current); - } - let Some(candidate) = self.context.states.allocate() else { - let current = self.word(); - return if current != FREE { - Ok(current) - } else { - Err(EBUSY) - }; - }; - if let Err(error) = self.context.work.lock().initialize(candidate) { - self.context.states.release(candidate); - return Err(error); - } - match self - .word - .compare_exchange(FREE, candidate, Ordering::AcqRel, Ordering::Acquire) - { - Ok(_) => Ok(candidate), - Err(installed) => { - self.context.states.release(candidate); - Ok(installed) - } - } - } - pub(crate) fn word(&self) -> u32 { - self.word.load(Ordering::Acquire) - } - pub(crate) fn node_va(&self) -> Result { - let word = self.word(); - if word == FREE { - return Err(EINVAL); - } - Ok(self.context.work_base + u64::from(word & 63) * NODE_STRIDE as u64) - } - pub(crate) fn add_submitted_kicks(&self, kicks: u32) -> Result { - self.context - .work - .lock() - .change_count(self.word(), kicks, true) - } - pub(crate) fn cancel_unpublished_kicks(&self, kicks: u32) -> Result { - self.context - .work - .lock() - .change_count(self.word(), kicks, false) - } - - pub(crate) fn render_slot(&self, pair: u8) -> Result { - self.context.work.lock().render.get(pair).ok_or(ENOSPC) - } - pub(crate) fn channel_payload(&self, pair: u8, ordinal: u64) -> Result { - self.context - .work - .lock() - .render - .payload(pair, ordinal) - .ok_or(EINVAL) - } -} -impl Drop for WorkStateLease { - fn drop(&mut self) { - let word = self.word(); - if word != FREE { - self.context.states.release(word); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/context/identity.rs b/drivers/gpu/drm/asahi-neo/g17/context/identity.rs deleted file mode 100644 index 6c48367e2d6864..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/context/identity.rs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Context-local work-state identities. The low six bits address a node; -//! the complete sequence distinguishes its owner until exact retirement. - -use core::sync::atomic::{AtomicU32, Ordering}; - -pub(super) const FREE: u32 = u32::MAX; -pub(super) const STATE_COUNT: usize = 63; -pub(super) const NODE_STRIDE: usize = 0x100; - -pub(super) struct WorkStates { - next: AtomicU32, - live: [AtomicU32; 64], -} -impl WorkStates { - pub(super) const fn new() -> Self { - Self { - next: AtomicU32::new(0), - live: [const { AtomicU32::new(FREE) }; 64], - } - } - pub(super) fn allocate(&self) -> Option { - for _ in 0..64 { - let word = self.next.fetch_add(1, Ordering::Relaxed); - let index = (word & 63) as usize; - if index != STATE_COUNT - && self.live[index] - .compare_exchange(FREE, word, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - { - return Some(word); - } - } - None - } - pub(super) fn release(&self, word: u32) -> bool { - let index = (word & 63) as usize; - index != STATE_COUNT - && self.live[index] - .compare_exchange(word, FREE, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } -} - -/// Two stable logical ordinals, independent of physical pair numbering. -pub(super) struct RenderSlots { - pairs: [Option; 2], - origins: [Option; 2], -} -impl RenderSlots { - pub(super) const fn new() -> Self { - Self { - pairs: [None; 2], - origins: [None; 2], - } - } - pub(super) fn get(&mut self, pair: u8) -> Option { - if let Some(index) = self.pairs.iter().position(|value| *value == Some(pair)) { - return Some(index as u32); - } - let index = self.pairs.iter().position(Option::is_none)?; - self.pairs[index] = Some(pair); - Some(index as u32) - } - pub(super) fn payload(&mut self, pair: u8, ordinal: u64) -> Option { - let index = self.get(pair)? as usize; - let origin = *self.origins[index].get_or_insert(ordinal); - Some(ordinal.checked_sub(origin)?.wrapping_add(1) as u8) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/dependency.rs b/drivers/gpu/drm/asahi-neo/g17/dependency.rs deleted file mode 100644 index 5abd494eeb1f58..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/dependency.rs +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Collapse a logical prefix to its newest still-live firmware kick per QID. - -use super::{ - fw::kick::{KickDependency, KICK_BARRIERS_MAX}, - job::Order, -}; -use kernel::prelude::*; - -/// A failed producer does not establish a fault in its consumer's VM. -pub(crate) fn inherited_error(error: Error) -> Error { - if error == EIO || error == ENODEV || error == ETIMEDOUT { - ECANCELED - } else { - error - } -} - -pub(crate) struct Frontier { - dependencies: [KickDependency; KICK_BARRIERS_MAX], - sequences: [u64; KICK_BARRIERS_MAX], - count: usize, -} - -impl Frontier { - pub(crate) const fn new() -> Self { - Self { - dependencies: [KickDependency::ZERO; KICK_BARRIERS_MAX], - sequences: [0; KICK_BARRIERS_MAX], - count: 0, - } - } - - pub(crate) fn as_slice(&self) -> &[KickDependency] { - &self.dependencies[..self.count] - } - - /// Each QID's implicit parent supplies transitive ordering. Choose its newest - /// logical sequence rather than comparing wrapped firmware timestamps. - pub(crate) fn insert(&mut self, sequence: u64, dependency: KickDependency) -> Result { - if let Some(index) = self - .as_slice() - .iter() - .position(|prior| prior.qid() == dependency.qid()) - { - if sequence > self.sequences[index] { - self.dependencies[index] = dependency; - self.sequences[index] = sequence; - } - return Ok(()); - } - if self.count == KICK_BARRIERS_MAX { - return Err(EIO); - } - self.dependencies[self.count] = dependency; - self.sequences[self.count] = sequence; - self.count += 1; - Ok(()) - } - - /// The caller has matched the logical owner and VM. Check every covered - /// predecessor for failure, including one a newer kick would subsume. - pub(crate) fn include( - &mut self, - prefix: Option, - sequence: u64, - status: i32, - word: u64, - ) -> Result { - if !Order::contains(prefix, sequence) { - return Ok(()); - } - if status < 0 { - return Err(ECANCELED); - } - if status > 0 { - return Ok(()); - } - let dependency = KickDependency::from_word(word).ok_or(EIO)?; - self.insert(sequence, dependency) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/event.rs b/drivers/gpu/drm/asahi-neo/g17/event.rs deleted file mode 100644 index 9e4e0253b7c813..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/event.rs +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Event service ordering under the device mutex. A restart ends its role's batch; a worker -//! joins preparation leases and services it on a separate turn before continuing the rings. - -use super::{ - fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, - recovery, Role, -}; -use kernel::prelude::*; - -/// The bounded shared-ring transport; recognized malformed entries return errors after consume. -pub(super) trait Transport { - fn trace(&mut self, role: Role); - fn statistics(&mut self, role: Role); - fn drain( - &mut self, - role: Role, - consume: impl FnMut(&mut Self, Event) -> Result, - ) -> Result; - fn recovery_state(&self) -> Result; -} - -/// Runtime callbacks with the device mutex held. None signals fences or allocates grow backing. -pub(super) trait Queues { - /// Flush the oldest deferred reply/release. False means ring backpressure; each pending - /// transaction retains its own bounded worker-attempt count across service turns. - fn flush_tvb_reply(&mut self) -> Result; - fn flush_freelist_release(&mut self) -> Result; - fn observe_render_releases(&mut self) -> Result; - fn control_backpressured(&self) -> bool; - fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result; - fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result; - fn retain_render_completions(&mut self, masks: [u64; 2]); - fn complete_compute(&mut self, masks: [u64; 2]) -> Result; - fn context_killed(&mut self, cookie: u64); -} - -/// Work requested after releasing the device mutex. -#[derive(Default)] -pub(super) struct Pending { - pub(super) recovery: bool, -} - -/// Access to the single recovery owner while event handlers borrow the device. -pub(super) trait Host: Transport + Queues { - fn recovery(&self) -> &recovery::State; - fn recovery_mut(&mut self) -> &mut recovery::State; -} - -fn batch(host: &mut H, role: Role, masks: &mut [u64; 2]) -> Result { - host.drain(role, |host, event| { - match event { - Event::Stamp(bits) => { - masks[0] |= bits[0]; - masks[1] |= bits[1]; - } - Event::Restart { generation, stamp } => { - if host.recovery_mut().request(generation, stamp)? { - return Ok(false); - } - } - Event::TvbGrow(request) => { - if request.halt_count == host.recovery().generation() { - host.queue_tvb_grow(request)?; - } - if host.control_backpressured() { - return Ok(false); - } - } - Event::FreeListGrow(completion) => host.complete_freelist_grow(completion)?, - Event::ContextKilled(cookie) => host.context_killed(cookie), - Event::Unknown(_) => {} - } - Ok(true) - }) -} - -pub(super) fn service(host: &mut H) -> Result { - if !host.flush_tvb_reply()? || !host.flush_freelist_release()? { - return Ok(Pending::default()); - } - host.observe_render_releases()?; - for role in [Role::Primary, Role::Secondary] { - host.trace(role); - } - for role in [Role::Primary, Role::Secondary] { - host.statistics(role); - } - let mut masks = [0; 2]; - if !host.recovery().pending() { - let mut settled = false; - // The firmware publishes restart before state 1, but its producer can race the snapshot. - for _ in 0..256 { - batch(host, Role::Primary, &mut masks)?; - if host.recovery().pending() || host.control_backpressured() { - settled = true; - break; - } - match host.recovery_state()? { - 0 => { - settled = true; - break; - } - 1 => {} - _ => return Err(EIO), - } - } - if !settled { - return Err(EIO); - } - } - if !host.control_backpressured() { - batch(host, Role::Secondary, &mut masks)?; - } - host.retain_render_completions(masks); - host.complete_compute(masks)?; - Ok(Pending { - recovery: host.recovery().pending(), - }) -} diff --git a/drivers/gpu/drm/asahi-neo/g17/feed.rs b/drivers/gpu/drm/asahi-neo/g17/feed.rs deleted file mode 100644 index 4cd05fde165812..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/feed.rs +++ /dev/null @@ -1,489 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Detects sustained host refill gaps and temporarily requests CPU capacity. -//! Detector updates are lock-free. The worker and policy callbacks serialize -//! frequency requests without nesting their mutex inside a publication lock. -//! Shutdown first closes the detector, joins its producers and the worker, -//! then removes requests. All request storage and policy references are owned -//! by this device, independently of other GPU instances. -//! Policy creation and the initial snapshot take the policy rwsem before the -//! requests mutex. Removal callbacks and the worker take only requests; -//! notifier unregistration joins callbacks without holding that mutex. - -use crate::hw::t8140::qos::CLOCK_HZ; -use core::{ - pin::Pin, - sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}, -}; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, ClockSource, Delta, Monotonic}, -}; - -#[cfg(CONFIG_CPU_FREQ)] -use core::ptr; -#[cfg(CONFIG_CPU_FREQ)] -use kernel::{bindings, new_mutex, sync::Mutex, types::Opaque}; - -const MAX_REFILL_TICKS: u64 = CLOCK_HZ / 500; -const STREAK_ON: u32 = 8; -const STREAK_MAX: u32 = 64; -const LINGER_NS: u64 = 8_000_000; -const POLL_MS: i64 = 2; -#[cfg(CONFIG_CPU_FREQ)] -const FLOOR_PERCENT: u64 = 50; -const WAIT_UTIL_MIN: u32 = 512; -const WAIT_HINT_MS: u32 = 8; -#[cfg(CONFIG_CPU_FREQ)] -const MAX_POLICIES: usize = 8; - -struct Detector { - last_end: AtomicU64, - idle: AtomicU64, - busy: AtomicU64, - streak: AtomicU32, - until: AtomicU64, - active: AtomicBool, - pending: AtomicBool, - stopping: AtomicBool, -} - -impl Detector { - const fn new() -> Self { - Self { - last_end: AtomicU64::new(0), - idle: AtomicU64::new(0), - busy: AtomicU64::new(0), - streak: AtomicU32::new(0), - until: AtomicU64::new(0), - active: AtomicBool::new(false), - pending: AtomicBool::new(false), - stopping: AtomicBool::new(false), - } - } - - fn decay_add(sum: &AtomicU64, add: u64) -> u64 { - let mut next = 0; - let _ = sum.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { - next = value - value / 8 + add; - Some(next) - }); - next - } - - fn note(&self, start: u64, end: u64, now: u64) { - if end <= start { - return; - } - let previous = self.last_end.fetch_max(end, Ordering::AcqRel); - let (idle, busy, paced) = if previous == 0 { - (0, end - start, true) - } else if start >= previous { - let gap = start - previous; - if gap < MAX_REFILL_TICKS { - (gap, end - start, false) - } else { - (0, end - start, true) - } - } else if end > previous { - (0, end - previous, false) - } else { - (0, 0, false) - }; - let idle_sum = Self::decay_add(&self.idle, idle); - let busy_sum = Self::decay_add(&self.busy, busy); - let streak = if paced { - self.streak.store(0, Ordering::Release); - 0 - } else { - let previous = self - .streak - .fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| { - Some((value + 1).min(STREAK_MAX)) - }); - // The closure always returns Some; fetch_update retries contention. - match previous { - Ok(value) => value + 1, - Err(_) => 0, - } - }; - if streak >= STREAK_ON && idle_sum.saturating_mul(4) >= busy_sum { - self.until.store(now + LINGER_NS, Ordering::Release); - if !self.active.swap(true, Ordering::AcqRel) { - self.pending.store(true, Ordering::Release); - } - } - } - - fn wanted(&self, now: u64) -> bool { - !self.stopping.load(Ordering::Acquire) && now < self.until.load(Ordering::Acquire) - } -} - -#[cfg(CONFIG_CPU_FREQ)] -#[pin_data] -struct Requests { - #[pin] - requests: Opaque<[bindings::freq_qos_request; MAX_POLICIES]>, - policies: [*mut bindings::cpufreq_policy; MAX_POLICIES], - maxima: [u32; MAX_POLICIES], - applied: [u32; MAX_POLICIES], -} - -// SAFETY: Each nonnull policy owns a kobject reference. The device requests -// mutex serializes the pinned slots with the worker and policy callbacks. -#[cfg(CONFIG_CPU_FREQ)] -unsafe impl Send for Requests {} - -#[cfg(CONFIG_CPU_FREQ)] -impl Requests { - fn new() -> Self { - Self { - requests: Opaque::zeroed(), - policies: [ptr::null_mut(); MAX_POLICIES], - maxima: [0; MAX_POLICIES], - applied: [0; MAX_POLICIES], - } - } - - /// The caller holds the policy's read or write lock and keeps it alive. - /// Policy removal must subsequently pass through this device's notifier. - unsafe fn add(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { - let this = self.project(); - if this.policies.contains(&policy) { - return; - } - let Some(index) = this.policies.iter().position(|entry| entry.is_null()) else { - return; - }; - // SAFETY: The caller keeps the initialized policy alive. The vacant - // request slot is pinned and exclusively owned under the requests mutex. - let result = unsafe { - bindings::freq_qos_add_request( - ptr::addr_of_mut!((*policy).constraints), - this.requests - .get() - .cast::() - .add(index), - bindings::freq_qos_req_type_FREQ_QOS_MIN, - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as i32, - ) - }; - if result >= 0 { - // SAFETY: The caller's policy reference or CREATE callback keeps the - // kobject alive. Retain it until REMOVE or device shutdown removes - // this request; cpuinfo is protected by the caller's policy lock. - unsafe { - bindings::kobject_get(ptr::addr_of_mut!((*policy).kobj)); - this.maxima[index] = (*policy).cpuinfo.max_freq; - } - this.policies[index] = policy; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; - } - } - - fn apply(self: Pin<&mut Self>, enabled: bool) -> bool { - let this = self.project(); - let mut applied = false; - for index in 0..MAX_POLICIES { - if this.policies[index].is_null() { - continue; - } - let value = if enabled { - (u64::from(this.maxima[index]) * FLOOR_PERCENT / 100) as u32 - } else { - bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32 - }; - if this.applied[index] != value { - // SAFETY: This slot is registered, pinned and exclusively locked; - // the policy reference keeps its constraints alive through REMOVE. - let result = unsafe { - bindings::freq_qos_update_request( - this.requests - .get() - .cast::() - .add(index), - value as i32, - ) - }; - if result >= 0 { - this.applied[index] = value; - } - } - applied |= this.applied[index] != bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; - } - applied - } - - fn remove(self: Pin<&mut Self>, policy: *mut bindings::cpufreq_policy) { - let this = self.project(); - let Some(index) = this.policies.iter().position(|entry| *entry == policy) else { - return; - }; - // SAFETY: This registered pinned slot is exclusively locked. The policy - // cannot finish removal until this reference is dropped. Keep all other - // request slots at their original addresses when releasing this slot. - unsafe { - bindings::freq_qos_remove_request( - this.requests - .get() - .cast::() - .add(index), - ); - bindings::cpufreq_cpu_put(policy); - } - this.policies[index] = ptr::null_mut(); - this.maxima[index] = 0; - this.applied[index] = bindings::FREQ_QOS_MIN_DEFAULT_VALUE as u32; - } - - fn clear(mut self: Pin<&mut Self>) { - for index in 0..MAX_POLICIES { - let policy = self.policies[index]; - if !policy.is_null() { - self.as_mut().remove(policy); - } - } - } -} - -/// Only the blocking notifier core accesses this block after registration. -#[cfg(CONFIG_CPU_FREQ)] -#[pin_data] -struct PolicyNotifier { - #[pin] - block: Opaque, -} - -// SAFETY: The pinned notifier is registered once before publishing Feed and -// unregistered at shutdown. The blocking notifier core serializes its links; -// callbacks use only Feed's atomic state and request mutex. -#[cfg(CONFIG_CPU_FREQ)] -unsafe impl Send for PolicyNotifier {} -// SAFETY: As above; no Rust access mutates the registered block. -#[cfg(CONFIG_CPU_FREQ)] -unsafe impl Sync for PolicyNotifier {} - -#[pin_data(PinnedDrop)] -pub(crate) struct Feed { - detector: Detector, - #[cfg(CONFIG_CPU_FREQ)] - #[pin] - requests: Mutex, - #[cfg(CONFIG_CPU_FREQ)] - #[pin] - notifier: PolicyNotifier, - #[cfg(CONFIG_CPU_FREQ)] - registered: AtomicBool, - applied: AtomicBool, -} - -impl Feed { - pub(crate) fn new() -> Result> { - let feed = Arc::pin_init( - pin_init!(Self { - detector: Detector::new(), - #[cfg(CONFIG_CPU_FREQ)] - requests <- new_mutex!(Requests::new(), "G17 CPU feed"), - #[cfg(CONFIG_CPU_FREQ)] - notifier <- pin_init!(PolicyNotifier { - block: Opaque::new(bindings::notifier_block { - notifier_call: Some(Self::policy_event), - next: ptr::null_mut(), - priority: 0, - }), - }), - #[cfg(CONFIG_CPU_FREQ)] - registered: AtomicBool::new(false), - applied: AtomicBool::new(false), - }), - GFP_KERNEL, - )?; - #[cfg(CONFIG_CPU_FREQ)] - feed.register_policies(); - Ok(feed) - } - - /// Seed existing policies once; later creation/removal is event driven. - #[cfg(CONFIG_CPU_FREQ)] - fn register_policies(&self) { - // SAFETY: Feed is pinned in its Arc and has not been published. The - // notifier block lives until shutdown unregisters and joins callbacks. - let result = unsafe { - bindings::cpufreq_register_notifier( - self.notifier.block.get(), - bindings::CPUFREQ_POLICY_NOTIFIER, - ) - }; - if result < 0 { - // CPUFreq can be disabled at boot independently of this driver. - return; - } - self.registered.store(true, Ordering::Release); - // SAFETY: nr_cpu_ids is initialized before driver probe. - let cpus = unsafe { bindings::nr_cpu_ids }; - for cpu in 0..cpus { - // SAFETY: The bounded CPU index is valid; a nonnull result owns a - // reference even if removal begins before acquiring the policy lock. - let policy = unsafe { bindings::cpufreq_cpu_get(cpu) }; - if policy.is_null() { - continue; - } - // CREATE holds the policy write lock before taking requests. Use - // the same lock order and wait out partial policy initialization. - // SAFETY: The retained reference keeps the policy and rwsem alive. - unsafe { bindings::down_read(ptr::addr_of_mut!((*policy).rwsem)) }; - { - let mut requests = self.requests.lock(); - // Removal unpublishes the CPU pointer before its notifier takes - // requests. Rechecking under this mutex makes either the seed - // skip the policy or the subsequent REMOVE release its slot. - // SAFETY: The CPU index is valid; balance this extra reference below. - let current = unsafe { bindings::cpufreq_cpu_get(cpu) }; - // SAFETY: The policy read lock excludes unfinished CREATE. If - // this policy is still published, REMOVE cannot yet have removed - // the core max request without passing our held requests mutex. - let initialized = - current == policy && unsafe { !(*policy).max_freq_req.qos.is_null() }; - if initialized { - // SAFETY: The reference and read lock keep the initialized - // policy alive; registered REMOVE callbacks cover its lifetime. - unsafe { requests.as_mut().add(policy) }; - } - if !current.is_null() { - // SAFETY: Balances the second lookup, independently of its identity. - unsafe { bindings::cpufreq_cpu_put(current) }; - } - } - // SAFETY: Balance the read lock and first lookup after releasing requests. - unsafe { - bindings::up_read(ptr::addr_of_mut!((*policy).rwsem)); - bindings::cpufreq_cpu_put(policy); - } - } - } - - #[cfg(CONFIG_CPU_FREQ)] - unsafe extern "C" fn policy_event( - block: *mut bindings::notifier_block, - event: c_ulong, - data: *mut core::ffi::c_void, - ) -> core::ffi::c_int { - // SAFETY: This callback is installed only on Feed::notifier.block. Its - // owner remains pinned until unregister joins every callback. - let notifier = - unsafe { kernel::container_of!(Opaque::cast_from(block), PolicyNotifier, block) }; - // SAFETY: The notifier is the embedded, pinned field of this live Feed. - let feed = unsafe { &*kernel::container_of!(notifier, Feed, notifier) }; - let policy = data.cast::(); - let mut requests = feed.requests.lock(); - if event == bindings::CPUFREQ_REMOVE_POLICY as c_ulong { - requests.as_mut().remove(policy); - } else if event == bindings::CPUFREQ_CREATE_POLICY as c_ulong - && !feed.detector.stopping.load(Ordering::Acquire) - { - // SAFETY: CREATE supplies an initialized policy with its write lock - // held and guarantees REMOVE before the core releases its storage. - unsafe { requests.as_mut().add(policy) }; - } else { - return bindings::NOTIFY_DONE as _; - } - let enabled = feed.detector.wanted(Self::now()); - let applied = requests.as_mut().apply(enabled); - feed.applied.store(applied, Ordering::Relaxed); - bindings::NOTIFY_OK as _ - } - - fn now() -> u64 { - Monotonic::ktime_get() as u64 - } - - pub(crate) fn note_render_pass(&self, start: u64, end: u64) { - if cfg!(CONFIG_CPU_FREQ) { - self.detector.note(start, end, Self::now()); - } - } - - /// Called after a firmware event pass, before queuing the single feed work item. - pub(crate) fn take_pending(&self) -> bool { - self.detector.pending.swap(false, Ordering::AcqRel) - } - - #[cfg(CONFIG_CPU_FREQ)] - fn apply(&self, enabled: bool) { - let mut requests = self.requests.lock(); - let enabled = enabled && !self.detector.stopping.load(Ordering::Acquire); - let applied = requests.as_mut().apply(enabled); - self.applied.store(applied, Ordering::Relaxed); - } - - #[cfg(not(CONFIG_CPU_FREQ))] - fn apply(&self, _enabled: bool) {} - - /// Worker context; the linger deadline has no firmware event of its own. - pub(crate) fn run(&self) { - loop { - if self.detector.wanted(Self::now()) { - self.apply(true); - fsleep(Delta::from_millis(POLL_MS)); - continue; - } - self.apply(false); - self.detector.active.store(false, Ordering::SeqCst); - // A detector can extend the deadline after observing the old active - // bit. Reclaim that work unless it already queued another owner. - if self.detector.wanted(Self::now()) - && !self.detector.active.swap(true, Ordering::AcqRel) - { - continue; - } - return; - } - } - - pub(crate) fn wait_hint(&self) -> Option<(u32, u32)> { - if cfg!(CONFIG_UCLAMP_TASK) - && self.detector.wanted(Self::now()) - && self.applied.load(Ordering::Relaxed) - { - Some((WAIT_UTIL_MIN, WAIT_HINT_MS)) - } else { - None - } - } - - /// Call before disabling/joining the event producer and actuator work items. - pub(crate) fn begin_shutdown(&self) { - self.detector.stopping.store(true, Ordering::Release); - } - - /// Call after work items are joined. Removing requests withdraws their floor. - pub(crate) fn shutdown(&self) { - self.begin_shutdown(); - #[cfg(CONFIG_CPU_FREQ)] - { - // Unregister joins callbacks, which take requests: never hold that - // mutex here. Device shutdown has already joined the feed worker. - if self.registered.swap(false, Ordering::AcqRel) { - // SAFETY: The block was registered successfully and remains - // pinned and alive until this synchronous removal returns. - unsafe { - bindings::cpufreq_unregister_notifier( - self.notifier.block.get(), - bindings::CPUFREQ_POLICY_NOTIFIER, - ); - } - } - self.requests.lock().as_mut().clear(); - } - self.applied.store(false, Ordering::Relaxed); - } -} - -#[pinned_drop] -impl PinnedDrop for Feed { - fn drop(self: Pin<&mut Self>) { - self.shutdown(); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fence.rs b/drivers/gpu/drm/asahi-neo/g17/fence.rs deleted file mode 100644 index 9291b310369140..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fence.rs +++ /dev/null @@ -1,257 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware completion fences and the aggregate fence of one submit ioctl. -//! -//! These fences can complete out of order, so each has an independent timeline. -//! Signalling belongs outside the device lock, after timestamp and VM-status writes. - -use super::context::{Context, WorkStateLease}; -use crate::file::SyncItem; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicUsize, Ordering}; -use kernel::{ - bindings, c_str, - dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, - new_mutex, - prelude::*, - str::CStr, - sync::{Arc, Mutex}, -}; - -pub(crate) struct CompletionFence; - -#[vtable] -impl FenceOps for CompletionFence { - fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") - } - - fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("g17-submit") - } -} - -/// Status of an explicitly signalled `CompletionFence`, without taking its lock. -/// This is only for our own fences; foreign dependencies still use the DMA-fence API. -pub(crate) fn completion_status(fence: &Fence) -> i32 { - let raw = fence.raw(); - // SAFETY: the owned reference retains the naturally aligned unsigned-long - // flags. DMA-fence updates published flags with atomic bitops; usize matches - // unsigned long on the supported architectures. Acquire observes the error, - // VM status and host timestamps published before the ordered signal bit. - let flags = unsafe { AtomicUsize::from_ptr(core::ptr::addr_of_mut!((*raw).flags).cast()) } - .load(Ordering::Acquire); - if flags & (1usize << bindings::dma_fence_flag_bits_DMA_FENCE_FLAG_SIGNALED_BIT) == 0 { - return 0; - } - // SAFETY: CompletionFence has no signaled callback. Its error is set before - // signalling and is immutable afterwards. The acquired signal bit makes that - // write visible; read once, as the DMA-fence status API does under its lock. - let error = unsafe { core::ptr::read_volatile(core::ptr::addr_of!((*raw).error)) }; - if error == 0 { 1 } else { error } -} - -/// Give independently retired work a timeline that fence merging cannot subsume. -pub(crate) fn independent(contexts: &FenceContexts) -> Result> { - let fence = contexts.new_fence(0, CompletionFence)?; - // SAFETY: The fence is unique and unpublished: there are no waiters, callbacks or - // shared references. The DMA-fence allocator supplies a globally unique context. - unsafe { (*fence.raw()).context = bindings::dma_fence_context_alloc(1) }; - Ok(fence.into()) -} - -/// An immutable failure fence allocated before the queue is exposed to ioctls. -/// Reusing it for rejected submissions cannot retain or subsume accepted work. -pub(crate) fn allocation_failure(contexts: &FenceContexts) -> Result { - let fence = independent(contexts)?; - fence.set_error(ENOMEM); - fence.signal(); - Ok(Fence::from_fence(&fence)) -} - -/// All command members of an ioctl share its work-state node and aggregate fence. -/// The initial member keeps the aggregate unsignalled until enqueue finishes. -#[pin_data] -pub(crate) struct Submission { - fence: UserFence, - /// Successful retirement drops only this bookkeeping owner; runtime owners - /// and failed submissions keep their work-state allocation. - #[pin] - work_state: Mutex>>, - remaining: AtomicU32, - error: AtomicI32, -} - -impl Submission { - fn new(fence: UserFence, context: Arc) -> Result> { - let work_state = Arc::new(WorkStateLease::deferred(context), GFP_KERNEL)?; - Arc::pin_init( - pin_init!(Self { - fence, - work_state <- new_mutex!(Some(work_state), "G17 submission work state"), - remaining: AtomicU32::new(1), - error: AtomicI32::new(0), - }), - GFP_KERNEL, - ) - } - - pub(crate) fn fence(&self) -> Fence { - Fence::from_fence(&self.fence) - } - - /// Add one command before enqueue ends. The parser bounds the count to 64. - pub(crate) fn member(self: &Arc) -> Member { - self.remaining.fetch_add(1, Ordering::Relaxed); - Member { - submission: self.clone(), - complete: AtomicBool::new(false), - } - } - - fn member_done(&self, result: Result) { - if let Err(error) = result { - let _ = self.error.compare_exchange( - 0, - error.to_errno(), - Ordering::AcqRel, - Ordering::Acquire, - ); - } - if self.remaining.fetch_sub(1, Ordering::AcqRel) == 1 { - let error = self.error.load(Ordering::Acquire); - if error != 0 { - self.fence.set_error(Error::from_errno(error)); - } else { - // Scheduler cleanup can wait behind submission on the same worker. - // Once every member and enqueue have succeeded, retained packets - // must not prevent new submissions from acquiring work-state nodes. - let retired = self.work_state.lock().take(); - drop(retired); - } - self.fence.signal(); - } - } - - fn finish_enqueue(&self, result: Result) { - // A dropped, never-enqueued member reports cancellation. The ioctl's actual - // failure takes precedence, while published members still retain their work. - if let Err(error) = result { - self.error.store(error.to_errno(), Ordering::Release); - } - self.member_done(result); - } -} - -/// One command's contribution to its submission, including construction failure. -pub(crate) struct Member { - submission: Arc, - complete: AtomicBool, -} - -impl Member { - pub(crate) fn work_state(&self) -> Result> { - self.submission - .work_state - .lock() - .as_ref() - .cloned() - .ok_or(ECANCELED) - } - - pub(crate) fn complete(&self, result: Result) { - if !self.complete.swap(true, Ordering::AcqRel) { - self.submission.member_done(result); - } - } -} - -impl Drop for Member { - fn drop(&mut self) { - self.complete(Err(ECANCELED)); - } -} - -fn install(outputs: impl Iterator, fence: &Fence) { - for mut sync in outputs { - if let Some(chain) = sync.chain_fence.take() { - sync.syncobj.add_point(chain, fence, sync.timeline_value); - } else { - sync.syncobj.replace_fence(Some(fence)); - } - } -} - -/// Keeps resolved output syncobjs reachable on every fallible submission path. -/// Input syncobjs stay in the same vector, avoiding a separate output allocation. -pub(crate) struct Outputs { - fallback: UserFence, - syncs: KVec, - input_count: usize, - submission: Option>, - result: Result, -} - -impl Outputs { - pub(crate) fn new( - contexts: &FenceContexts, - mut syncs: KVec, - input_count: usize, - allocation_failure: &Fence, - ) -> Result { - if input_count > syncs.len() { - return Err(EINVAL); - } - let fallback = match independent(contexts) { - Ok(fence) => fence, - Err(error) => { - // Syncobjs and timeline-chain storage are already resolved. - // No work has been accepted; publish the pre-signaled ENOMEM - // fence without another allocation before returning the error. - install(syncs.drain(input_count..), allocation_failure); - return Err(error); - } - }; - Ok(Self { - fallback, - syncs, - input_count, - submission: None, - result: Err(ECANCELED), - }) - } - - pub(crate) fn inputs(&self) -> &[SyncItem] { - &self.syncs[..self.input_count] - } - - /// Publish the actual aggregate before enqueueing any command. - pub(crate) fn publish(&mut self, context: Arc) -> Result> { - let submission = Submission::new(self.fallback.clone(), context)?; - install(self.syncs.drain(self.input_count..), &submission.fence()); - self.submission = Some(submission.clone()); - Ok(submission) - } - - /// The caller first records terminal VM errors and settles prior failed work. - pub(crate) fn finish(mut self, result: Result) -> Result { - self.result = result; - result - } -} - -impl Drop for Outputs { - fn drop(&mut self) { - if let Some(submission) = self.submission.take() { - submission.finish_enqueue(self.result); - return; - } - if let Err(error) = self.result { - self.fallback.set_error(error); - } - install( - self.syncs.drain(self.input_count..), - &Fence::from_fence(&self.fallback), - ); - self.fallback.signal(); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/freelist.rs b/drivers/gpu/drm/asahi-neo/g17/freelist.rs deleted file mode 100644 index c2c9470d10aa3e..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/freelist.rs +++ /dev/null @@ -1,727 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Retained USC private-memory pools and their submission counters. -//! -//! A logical context owns one render pool, shared by its render pairs. A -//! physical compute queue owns a separate pool. All mutation and publication -//! runs under the device mutex. Pool aliases must outlive every firmware -//! reference and are released only after the owner's teardown is acknowledged. - -use super::fw::freelist::*; -use super::fw::initdata::{FreeListArgs, FreeListClass, FreeListControl, FREE_LIST_BLOCK_SIZE}; -use super::object::{Allocator, CpuMap, KernelObject}; -use crate::mmu; -use core::sync::atomic::{fence, AtomicBool, AtomicU64, Ordering}; -use kernel::{ - new_mutex, - prelude::*, - sync::{Arc, Mutex}, -}; - -/// Render pools occupy 129..255; compute uses its QID plus one, in 1..128. -/// First-free allocation keeps the shared firmware namespace deterministic. -pub(crate) struct RenderIds { - used: [AtomicU64; 2], -} -impl RenderIds { - pub(crate) const fn new() -> Self { - Self { - used: [AtomicU64::new(0), AtomicU64::new(1 << 63)], - } - } -} -struct RenderId { - slot: u16, - allocator: Arc, -} -impl RenderId { - fn new(allocator: &Arc) -> Result { - for (bank, used) in allocator.used.iter().enumerate() { - let mut value = used.load(Ordering::Acquire); - loop { - let bit = value.trailing_ones(); - if bit == 64 { - break; - } - match used.compare_exchange_weak( - value, - value | (1u64 << bit), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - return Ok(Self { - slot: 129 + bank as u16 * 64 + bit as u16, - allocator: allocator.clone(), - }) - } - Err(current) => value = current, - } - } - } - Err(ENOSPC) - } -} -impl Drop for RenderId { - fn drop(&mut self) { - let bit = self.slot as usize - 129; - self.allocator.used[bit / 64].fetch_and(!(1u64 << (bit % 64)), Ordering::Release); - } -} - -pub(crate) struct Pages { - backing: KernelObject, - blocks: usize, - layout: RunLayout, -} -impl Pages { - fn new(alloc: &Allocator<'_>, blocks: usize, layout: RunLayout) -> Result { - let mut backing = alloc.lower( - TABLE_SIZE + blocks * FREE_LIST_BLOCK_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; - let base = backing.gpu_va(); - backing.initialize::(0, |tables| tables.init(base, blocks, layout))?; - Ok(Self { - backing, - blocks, - layout, - }) - } - /// Fresh zeroed compute backing, built without the device mutex. - pub(crate) fn compute(alloc: &Allocator<'_>) -> Result { - Self::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute) - } - fn page_list_va(&self) -> u64 { - self.backing.gpu_va() - } - fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 - } - fn repopulate(&self) -> Result { - let table = self.backing.pointer(0, TABLE_SIZE)?.cast::(); - PoolTables::for_each_word( - self.page_list_va(), - self.blocks, - self.layout, - |index, value| { - // SAFETY: The page-aligned backing was validated for the complete - // tables. The enumerator bounds every index inside those tables; - // the owner has proven firmware release before this update. - unsafe { table.add(index).write_volatile(value) }; - }, - ) - } -} - -/// Pool-object identities are independent of recycled firmware IDs and VAs. -static RENDER_POOL_SERIAL: AtomicU64 = AtomicU64::new(0); - -/// The USC backing of a render pool and its two aliases. Field order drops -/// both aliases before the backing. -pub(crate) struct RenderBacking { - _client: mmu::KernelMapping, - _global: mmu::KernelMapping, - pages: Pages, -} - -impl RenderBacking { - /// Fresh populated backing, built without the device mutex. - pub(crate) fn new( - alloc: &Allocator<'_>, - render_global: &mmu::Vm, - client: &mmu::Vm, - ) -> Result { - let pages = Pages::new(alloc, RENDER_BLOCKS, RunLayout::Render)?; - let address = pages.page_list_va(); - let global = pages - .backing - .map_alias(render_global, address, mmu::PROT_GPU_SHARED_RW)?; - let client = pages - .backing - .map_alias(client, address, mmu::PROT_GPU_SHARED_RW)?; - Ok(Self { - _client: client, - _global: global, - pages, - }) - } -} - -/// A render pool retained by a logical context and every graph referencing it. -/// Its backing can be given back while the pool's free-list release has been -/// consumed and the client VM is evicted; a fresh backing is installed before -/// the pool is staged again. -pub(crate) struct RenderPool { - backing: Arc>>, - /// Page-list VA of the installed backing; zero while vacant. - page_list: AtomicU64, - /// Counts installed backings, so a re-armed backing is a new free-list - /// owner even if its addresses repeat. - epoch: AtomicU64, - /// Residency of the client VM the backing is aliased into. - client_residency: Option>, - control: KernelObject, - state: KernelObject, - served: AtomicBool, - serial: u64, - id: RenderId, -} - -impl RenderPool { - pub(crate) fn new( - alloc: &Allocator<'_>, - ids: &Arc, - render_global: &mmu::Vm, - client: &mmu::Vm, - ) -> Result { - let id = RenderId::new(ids)?; - let backing = RenderBacking::new(alloc, render_global, client)?; - let address = backing.pages.page_list_va(); - let run_list_va = backing.pages.run_list_va(); - let state = alloc.kernel( - CONTROL_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let mut control = alloc.kernel( - CONTROL_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - control.write( - 0, - FreeListControl::render( - &FreeListArgs { - buffer_slot: u32::from(id.slot), - class: FreeListClass::Two, - page_list_va: address, - run_list_va, - blocks: RENDER_BLOCKS as u32, - state_va: state.gpu_va(), - }, - 0, - ), - )?; - fence(Ordering::SeqCst); - let serial = RENDER_POOL_SERIAL - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| value.checked_add(1)) - .map_err(|_| EOVERFLOW)? + 1; - let backing = Arc::pin_init( - new_mutex!(Some(backing), "G17 render pool backing"), - GFP_KERNEL, - )?; - Ok(Self { - backing, - page_list: AtomicU64::new(address), - epoch: AtomicU64::new(0), - client_residency: client.residency_gate(), - control, - state, - served: AtomicBool::new(false), - serial, - id, - }) - } - - /// No backing is installed: the pool must be re-armed before staging. - pub(crate) fn vacant(&self) -> bool { - self.page_list.load(Ordering::Acquire) == 0 - } - /// Pressure reclaim evicted the client VM, which is idle. - pub(crate) fn client_evicted(&self) -> bool { - self.client_residency.as_ref().is_some_and(|gate| gate.evicted()) - } - /// Removes the backing. The caller holds the device mutex and observed this - /// pool's free-list release consumed, so no firmware reference remains; the - /// next staging requires a re-armed backing. The caller drops the result - /// after releasing the device mutex. - pub(crate) fn vacate(&self) -> Option { - let backing = self.backing.lock().take(); - if backing.is_some() { - self.page_list.store(0, Ordering::Release); - } - backing - } - /// Installs fresh backing into a vacant pool, unless another caller did. - /// The next staging rewrites the control header and descriptor row for it - /// (its run-list address changes the free-list owner identity). - pub(crate) fn rearm(&self, fresh: RenderBacking) -> Result { - let mut backing = self.backing.lock(); - if backing.is_some() { - return Ok(()); - } - let address = fresh.pages.page_list_va(); - *backing = Some(fresh); - self.epoch.fetch_add(1, Ordering::AcqRel); - self.served.store(true, Ordering::Release); - self.page_list.store(address, Ordering::Release); - Ok(()) - } - /// Re-arms a vacant pool with fresh backing built here, without the device - /// mutex. `client` is the VM of the context that owns this pool. - pub(crate) fn ensure_backing( - &self, - alloc: &Allocator<'_>, - render_global: &mmu::Vm, - client: &mmu::Vm, - ) -> Result { - if !self.vacant() { - return Ok(()); - } - self.rearm(RenderBacking::new(alloc, render_global, client)?) - } - - pub(crate) fn serial(&self) -> u64 { - self.serial - } - pub(crate) fn epoch(&self) -> u64 { - self.epoch.load(Ordering::Acquire) - } - pub(crate) fn id(&self) -> u16 { - self.id.slot - } - pub(crate) fn control_va(&self) -> u64 { - self.control.gpu_va() - } - /// Zero while vacant. - pub(crate) fn page_list_va(&self) -> u64 { - self.page_list.load(Ordering::Acquire) - } - pub(crate) fn run_list_va(&self) -> u64 { - self.page_list_va() + PAGE_LIST_SIZE as u64 - } - pub(crate) fn end_va(&self) -> u64 { - self.run_list_va() - + (RENDER_BLOCKS * size_of::()) as u64 - } - - /// Fresh pools are already populated. Reuse requires the previous release - /// to have been consumed; command completion alone does not establish it. - pub(crate) fn repopulate(&self) -> Result { - if !self.served.swap(true, Ordering::AcqRel) { - return Ok(()); - } - let submitted = self.state.dword(0)?.load(Ordering::Relaxed); - let header = FreeListControl::render( - &FreeListArgs { - buffer_slot: u32::from(self.id()), - class: FreeListClass::Two, - page_list_va: self.page_list_va(), - run_list_va: self.run_list_va(), - blocks: RENDER_BLOCKS as u32, - state_va: self.state.gpu_va(), - }, - submitted, - ); - let mut image = KVVec::from_elem(0u64, CONTROL_SIZE / 8, GFP_KERNEL)?; - // SAFETY: `image` owns a zeroed control page, is large enough for the - // header, and every header byte is initialized integer data. - unsafe { - image - .as_mut_ptr() - .cast::() - .write_unaligned(header) - }; - let control = self.control.pointer(0, CONTROL_SIZE)?.cast::(); - let backing = self.backing.lock(); - let pages = &backing.as_ref().ok_or(EAGAIN)?.pages; - if pages.page_list_va() != self.page_list_va() { - return Err(EIO); - } - pages.repopulate()?; - for (index, value) in image.iter().enumerate() { - // SAFETY: The entire retained page was checked before any store; - // the allocation is page-aligned and release excludes the firmware. - unsafe { control.add(index).write_volatile(*value) }; - } - fence(Ordering::SeqCst); - Ok(()) - } - - /// Submitted is host-owned and aligned. Completed is the packed firmware - /// counter, so retain the byte accesses required by that interface. - pub(crate) fn counters(&self) -> Result<[u64; 2]> { - let submitted = self.state.dword(0)?.load(Ordering::Relaxed); - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; - let mut bytes = [0; 8]; - for (index, byte) in bytes.iter_mut().enumerate() { - // SAFETY: The checked eight-byte range stays mapped; byte accesses - // permit the unaligned counter without widening shared accesses. - *byte = unsafe { completed.add(index).read_volatile() }; - } - Ok([submitted, u64::from_le_bytes(bytes)]) - } - - /// Snapshot the unaligned shared fields before loading their descriptor. - pub(crate) fn page_pool_state(&self) -> Result { - let raw = self.control.pointer(0x14, 0x1c)?; - fence(Ordering::Acquire); - let read = |offset: usize, count: usize| -> u64 { - let mut bytes = [0; 8]; - for (index, byte) in bytes.iter_mut().take(count).enumerate() { - // SAFETY: The complete 0x14..0x30 span was checked. These fixed - // field ranges lie inside it; byte loads permit unaligned words. - *byte = unsafe { raw.add(offset + index).read_volatile() }; - } - u64::from_le_bytes(bytes) - }; - Ok(PagePoolState { - page_list_va: read(0, 8), - unk_1c: read(8, 4) as u32, - unk_20: read(12, 4) as u32, - unk_24: read(16, 4) as u32, - unk_28: read(20, 4) as u32, - page_count: read(24, 4) as u32, - }) - } - - /// Save mutable descriptor state only after both engines and free-list - /// references retire, before the physical slot changes owners. - pub(crate) fn store_page_pool_state(&self, state: PagePoolState) -> Result { - let raw = self.control.pointer(0x20, 0x10)?; - for (word, value) in [state.unk_20, state.unk_24, state.unk_28, state.page_count] - .into_iter() - .enumerate() - { - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: The entire mutable range was checked, and each - // enumerated byte lies inside it. Caller owns the retired pool. - unsafe { raw.add(word * 4 + index).write_volatile(byte) }; - } - } - fence(Ordering::SeqCst); - Ok(()) - } - - /// Prepares the counter before any producer is written. Commit belongs to - /// the publication transaction; dropping this plan does not reserve work. - pub(crate) fn prepare_submission(&self, commands: u32) -> Result> { - if commands > 2 { - return Err(EINVAL); - } - let counter = self.state.dword(0)?; - Ok(Submitted { - counter, - next: counter - .load(Ordering::Relaxed) - .wrapping_add(u64::from(commands)), - }) - } -} - -/// One checked, infallible counter update in a submission transaction. -pub(crate) struct Submitted<'a> { - counter: &'a AtomicU64, - next: u64, -} -impl Submitted<'_> { - pub(crate) fn commit(self) { - self.counter.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - } -} - -/// A pool owned by one physical compute queue, with fixed QID-derived slot. -/// Idle retained queues may give their USC backing back (`pages` is `None`) -/// once the firmware consumed the pool's release; the control and state -/// objects the queue record names stay until processor stop. -pub(crate) struct ComputePool { - pages: Option, - control: KernelObject, - state: KernelObject, - id: u16, -} -impl ComputePool { - pub(crate) fn new(alloc: &Allocator<'_>, qid: u8) -> Result { - if qid > super::fw::kick::QID_MAX { - return Err(EINVAL); - } - let id = u16::from(qid) + 1; - let pages = Pages::new(alloc, COMPUTE_BLOCKS, RunLayout::Compute)?; - let mut control = alloc.kernel( - CONTROL_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let state = alloc.kernel( - CONTROL_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - control.write( - 0, - FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, - state_va: state.gpu_va(), - }), - )?; - fence(Ordering::SeqCst); - Ok(Self { - pages: Some(pages), - control, - state, - id, - }) - } - pub(crate) fn id(&self) -> u16 { - self.id - } - /// EAGAIN while the pool is vacant: the owner must re-arm it first. - pub(crate) fn page_list_va(&self) -> Result { - self.pages.as_ref().map(Pages::page_list_va).ok_or(EAGAIN) - } - pub(crate) fn control_va(&self) -> u64 { - self.control.gpu_va() - } - pub(crate) fn populated(&self) -> bool { - self.pages.is_some() - } - /// The release record for this pool's buffer slot under the free-list - /// generation the queue last published. - pub(crate) fn release_record(&self, generation: u64) -> super::fw::channels::FreeListRelease { - super::fw::channels::FreeListRelease::new(generation, u32::from(self.id)) - } - /// The caller has observed the firmware consume this pool's release and - /// frees the returned backing after releasing the device mutex. - pub(crate) fn take_pages(&mut self) -> Option { - self.pages.take() - } - /// Rewrites the shared control header for fresh backing. The caller then - /// rewrites the page-pool descriptor row before the queue is kicked again. - /// As render repopulation does, the firmware-completed counter is carried - /// over so it stays consistent with the host submitted counter in `state`. - pub(crate) fn rearm(&mut self, pages: Pages) -> Result { - if self.pages.is_some() { - return Err(EBUSY); - } - let completed = self.control.pointer(FreeListControl::COMPLETED, 8)?; - let mut bytes = [0; 8]; - for (index, byte) in bytes.iter_mut().enumerate() { - // SAFETY: The checked eight-byte range stays mapped; byte accesses - // match the unaligned packed counter the firmware writes. - *byte = unsafe { completed.add(index).read_volatile() }; - } - let mut header = FreeListControl::compute(&FreeListArgs { - buffer_slot: u32::from(self.id), - class: FreeListClass::One, - page_list_va: pages.page_list_va(), - run_list_va: pages.run_list_va(), - blocks: COMPUTE_BLOCKS as u32, - state_va: self.state.gpu_va(), - }); - header.unk_54[..8].copy_from_slice(&bytes); - self.control.write(0, header)?; - fence(Ordering::SeqCst); - self.pages = Some(pages); - Ok(()) - } - pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { - let pages = self.pages.as_mut().ok_or(EAGAIN)?; - let address = pages.page_list_va(); - pages - .backing - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) - } - pub(crate) fn reserve_submission(&self) -> Result { - let counter = self.state.dword(0)?; - let next = counter - .load(Ordering::Relaxed) - .checked_add(1) - .ok_or(EOVERFLOW)?; - counter.store(next, Ordering::Relaxed); - fence(Ordering::SeqCst); - Ok(next) - } - /// The caller has proved no pointer from this submission was published. - pub(crate) fn cancel_unpublished(&self, target: u64) -> Result { - let counter = self.state.dword(0)?; - if target == 0 || counter.load(Ordering::Acquire) != target { - return Err(EINVAL); - } - counter.store(target - 1, Ordering::Relaxed); - fence(Ordering::SeqCst); - Ok(()) - } -} - -/// Registration identity held by a free-list lease. The cookie is the end of -/// the run list; it remains stable while overlapping command pairs share it. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct Owner { - /// Identifies the host pool object even when every firmware address recycles. - pub(crate) serial: u64, - /// Identifies the pool's installed backing (render pools are re-armed). - pub(crate) epoch: u64, - pub(crate) buffer_slot: u32, - pub(crate) control_va: u64, - pub(crate) ring_va: u64, - pub(crate) cookie: u64, -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum LeasePhase { - Preparing, - Ready, - ReleaseSent(u32), - Released, -} - -/// References to one prepopulated owner: preparation and two engine commands -/// per render. Published releases do not gate reacquisition; their consumption -/// witnesses are carried into the successor. -#[derive(Copy, Clone, Debug)] -pub(crate) struct Lease { - owner: Owner, - generation: u64, - references: u32, - commands: u32, - phase: LeasePhase, - previous_release: Option, -} - -impl Lease { - /// Acquires an unchanged overlapping owner, or an owner whose previous - /// release was sent. A changed owner additionally requires caller-side - /// proof of old-pool release before its backing can be repopulated. - pub(crate) fn acquire(previous: Option<&Self>, owner: Owner) -> Result { - let Some(previous) = previous else { - return Ok(Self { - owner, - generation: 0, - references: 3, - commands: 2, - phase: LeasePhase::Preparing, - previous_release: None, - }); - }; - if previous.owner.buffer_slot != owner.buffer_slot { - return Err(EINVAL); - } - let generation = previous.generation.checked_add(1).ok_or(EOVERFLOW)?; - let same_owner = previous.owner == owner; - let overlapping = same_owner - && previous.phase == LeasePhase::Ready - && previous.references >= 2 - && previous.commands >= 2; - let reacquire = previous.reacquirable(); - if !overlapping && !reacquire { - return Err(EBUSY); - } - let references = previous.references.checked_add(3).ok_or(EOVERFLOW)?; - let commands = previous.commands.checked_add(2).ok_or(EOVERFLOW)?; - Ok(Self { - owner, - generation: if overlapping { - previous.generation - } else { - generation - }, - references, - commands, - phase: LeasePhase::Preparing, - previous_release: previous.release_witness(), - }) - } - - pub(crate) fn owner(&self) -> Owner { - self.owner - } - pub(crate) fn generation(&self) -> u64 { - self.generation - } - pub(crate) fn reacquirable(&self) -> bool { - matches!( - self.phase, - LeasePhase::ReleaseSent(_) | LeasePhase::Released - ) && self.references == 0 - && self.commands == 0 - } - pub(crate) fn released(&self) -> bool { - self.phase == LeasePhase::Released && self.previous_release.is_none() - } - pub(crate) fn release_witness(&self) -> Option { - match self.phase { - LeasePhase::ReleaseSent(cursor) => Some(cursor), - _ => self.previous_release, - } - } - - /// Publication retires preparation independently of the command references. - pub(crate) fn retire_prepare(&mut self) -> Result { - if self.phase != LeasePhase::Preparing || self.references < 3 { - return Err(EINVAL); - } - self.references -= 1; - self.phase = LeasePhase::Ready; - Ok(()) - } - /// Fixed prepopulated pools never submit an asynchronous growth request. - /// An incoming growth completion therefore cannot name a live request. - pub(crate) fn complete_grow( - &mut self, - _event: super::fw::channels::FreeListCompletion, - ) -> Result { - Err(EIO) - } - /// Both engines have retired this render's exact retained submission. - pub(crate) fn complete_render( - &mut self, - ) -> Result> { - if self.phase != LeasePhase::Ready || self.references < 2 || self.commands < 2 { - return Err(EINVAL); - } - self.references -= 2; - self.commands -= 2; - Ok(self.pending_release()) - } - pub(crate) fn pending_release(&self) -> Option { - (self.phase == LeasePhase::Ready && self.references == 0).then(|| { - super::fw::channels::FreeListRelease::new(self.generation, self.owner.buffer_slot) - }) - } - /// Records the producer-after cursor once the final release was published. - pub(crate) fn mark_release_sent(&mut self, cursor: u32) -> Result { - if self.pending_release().is_none() || cursor >= super::fw::channels::DEVICE_CONTROL_SLOTS { - return Err(EINVAL); - } - self.phase = LeasePhase::ReleaseSent(cursor); - Ok(()) - } - /// Observes all older release records passed by one consumer snapshot. - pub(crate) fn observe_releases(&mut self, consumer: u32, producer: u32) -> Result { - if let Some(cursor) = self.previous_release { - if control_consumed(consumer, producer, cursor)? { - self.previous_release = None; - } - } - if let LeasePhase::ReleaseSent(cursor) = self.phase { - if control_consumed(consumer, producer, cursor)? { - self.phase = LeasePhase::Released; - } - } - Ok(()) - } -} - -/// A publication is retired once its producer-after cursor leaves the -/// current `(consumer, producer]` interval, including wrapped intervals. -pub(crate) fn control_consumed(consumer: u32, producer: u32, cursor: u32) -> Result { - let slots = super::fw::channels::DEVICE_CONTROL_SLOTS; - if consumer >= slots || producer >= slots || cursor >= slots { - return Err(EIO); - } - let pending = producer.wrapping_sub(consumer) & (slots - 1); - let ahead = cursor.wrapping_sub(consumer) & (slots - 1); - Ok(ahead == 0 || ahead > pending) -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/buffer.rs b/drivers/gpu/drm/asahi-neo/g17/fw/buffer.rs deleted file mode 100644 index ec6797637bdbe1..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/buffer.rs +++ /dev/null @@ -1,254 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Parameter-buffer inventory, scene and control records. These objects are -//! initialized before publication; live access uses the host buffer manager's -//! individual aligned words and packed byte fields. - -use super::{clear, offset_va}; -use crate::hw::t8140::resources as cfg; -use kernel::prelude::*; - -pub(crate) const SCENES: usize = 0x50; -pub(crate) const SCRATCH_SLOTS: usize = 0x24; -pub(crate) const SCRATCH_STRIDE: u64 = 0x20; -pub(crate) const FIRST_SCENE: usize = 1; -pub(crate) const PAGE_LIST: usize = 0x8000; -pub(crate) const PAGE_LIST_BYTES: usize = 0x10000; -pub(crate) const CONTROL: usize = 0x20000; -pub(crate) const COUNTER: usize = 0x28000; -pub(crate) const SCENE_TABLE: usize = 0x38000; -pub(crate) const BLOCK_TABLE: usize = 0x40000; -pub(crate) const SIZE: usize = 0x48000; -pub(crate) const METRICS_STRIDE: usize = SCENES * size_of::(); -pub(crate) const METRICS_SLOTS: usize = cfg::METRICS_SIZE as usize / METRICS_STRIDE; -pub(crate) const NORMAL_MAX_BLOCKS: usize = 0x45e; -pub(crate) const GROW_BLOCKS: usize = 10; - -/// Firmware's active parameter-buffer state. Pointers at +0x44, +0x4c, -/// +0x64 and +0x84 are intentionally unaligned. -#[repr(C, packed)] -pub(crate) struct State { - pub(crate) unk_0: [u8; 0xc], - pub(crate) buffer_id: u32, - pub(crate) unk_10: [u8; 0x10], - pub(crate) page_list_fw: u64, - pub(crate) page_list_client: u64, - pub(crate) page_list_size: u32, - pub(crate) pages: u32, - pub(crate) block_capacity: u32, - pub(crate) producer: u32, - pub(crate) consumer: u32, - pub(crate) block_table: u64, - pub(crate) control: u64, - pub(crate) last_page: u32, - pub(crate) block_size: u32, - pub(crate) unk_5c: [u8; 8], - pub(crate) counter: u64, - pub(crate) unk_6c: [u8; 0x10], - pub(crate) maximum_size: u32, - pub(crate) normal_maximum_size: u32, - pub(crate) discard: u64, - pub(crate) unk_8c: [u8; 0x34], -} - -/// One scene's accounting and scratch identity. -#[repr(C, packed)] -pub(crate) struct Scene { - pub(crate) metric_client: u64, - pub(crate) metric_fw: u64, - pub(crate) unk_10: [u8; 0x18], - pub(crate) scratch: u64, - pub(crate) unk_30: [u8; 0x10], - pub(crate) statistics: u64, - pub(crate) used: u32, - pub(crate) generation: u64, - pub(crate) unk_54: [u8; 0x2c], -} - -/// Circular inventory cursors followed by firmware statistics. Firmware owns -/// `read` and the statistics; the host requests reset by release-storing1. -#[repr(C)] -pub(crate) struct Control { - pub(crate) total: u32, - pub(crate) committed: u32, - pub(crate) read: u32, - pub(crate) unk_c: [u32; 13], - pub(crate) max_pages: u32, - pub(crate) unk_44: [u32; 7], - pub(crate) reset: u32, - pub(crate) unk_64: [u32; 7], -} - -#[repr(C)] -pub(crate) struct Block { - pub(crate) first_page: u32, - pub(crate) unk_4: u32, -} - -/// The complete parameter-buffer graph, including reserved spaces between -/// records. The page-list subrange also has a client mapping. -#[repr(C)] -pub(crate) struct Graph { - pub(crate) state: State, - pub(crate) pad_c0: [u8; PAGE_LIST - size_of::()], - pub(crate) pages: [u32; cfg::TVB_BLOCK_SLOTS * cfg::TVB_PAGES_PER_BLOCK], - pub(crate) pad_151a0: [u8; CONTROL - PAGE_LIST - cfg::TVB_PAGE_LIST_SIZE], - pub(crate) control: Control, - pub(crate) pad_20080: [u8; COUNTER - CONTROL - size_of::()], - pub(crate) counter: u32, - pub(crate) pad_28004: [u8; SCENE_TABLE - COUNTER - 4], - pub(crate) scenes: [Scene; SCENES], - pub(crate) trailer: [u8; 0x40], - pub(crate) pad_3a840: [u8; BLOCK_TABLE - SCENE_TABLE - SCENES * size_of::() - 0x40], - pub(crate) blocks: [Block; cfg::TVB_BLOCK_SLOTS], - pub(crate) pad_468d0: [u8; SIZE - BLOCK_TABLE - cfg::TVB_BLOCK_SLOTS * size_of::()], -} - -// SAFETY: Every field of the complete graph consists solely of integers. -unsafe impl Zeroable for Graph {} - -static_assert!(size_of::() == 0xc0); -static_assert!(size_of::() == 0x80); -static_assert!(size_of::() == 0x80); -static_assert!(size_of::() == SIZE); -static_assert!(core::mem::offset_of!(Graph, pages) == PAGE_LIST); -static_assert!(core::mem::offset_of!(Graph, control) == CONTROL); -static_assert!(core::mem::offset_of!(Graph, counter) == COUNTER); -static_assert!(core::mem::offset_of!(Graph, scenes) == SCENE_TABLE); -static_assert!(core::mem::offset_of!(Graph, blocks) == BLOCK_TABLE); -static_assert!(core::mem::offset_of!(State, block_table) == 0x44); -static_assert!(core::mem::offset_of!(State, discard) == 0x84); -static_assert!(core::mem::offset_of!(Scene, generation) == 0x4c); - -/// The same metrics storage in the firmware and application address spaces. -#[derive(Clone, Copy, Default)] -pub(crate) struct MetricsAddresses { - pub(crate) firmware: u64, - pub(crate) client: u64, -} - -pub(crate) struct GraphArgs<'a> { - pub(crate) fw_va: u64, - pub(crate) page_list_client: u64, - pub(crate) metrics: MetricsAddresses, - pub(crate) blocks: &'a [u64], - pub(crate) scratch: u64, - pub(crate) discard: u64, - pub(crate) buffer_id: u32, - pub(crate) generation: u64, -} - -pub(crate) fn compact_page(address: u64) -> Result { - let relative = address.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if relative % cfg::TVB_PAGE != 0 - || address - .checked_add(cfg::TVB_BLOCK_SIZE as u64) - .ok_or(EOVERFLOW)? - > cfg::USER_END - { - return Err(EINVAL); - } - Ok((relative / cfg::TVB_PAGE).try_into()?) -} - -impl Graph { - pub(crate) fn init(&mut self, args: &GraphArgs<'_>) -> Result { - if args.generation == 0 - || args.blocks.len() < cfg::TVB_INITIAL_PAGES.len() - || args.blocks.len() > cfg::TVB_MAX_BLOCKS - { - return Err(EINVAL); - } - let scratch = args.scratch.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - let discard = args.discard.checked_sub(cfg::USER_BASE).ok_or(EINVAL)?; - if scratch > u32::MAX as u64 || discard > u32::MAX as u64 { - return Err(EINVAL); - } - let stats = offset_va(args.fw_va, (CONTROL + 0x40) as u64)?; - clear(self); - for (index, &address) in args.blocks.iter().enumerate() { - let first = compact_page(address)?; - self.blocks[index].first_page = first; - for page in 0..cfg::TVB_PAGES_PER_BLOCK { - self.pages[index * cfg::TVB_PAGES_PER_BLOCK + page] = first + page as u32; - } - } - for (index, scene) in self.scenes.iter_mut().enumerate() { - scene.metric_client = offset_va(args.metrics.client, (index * 4) as u64)?; - scene.metric_fw = offset_va(args.metrics.firmware, (index * 4) as u64)?; - scene.scratch = scratch + (index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE; - scene.statistics = stats; - } - self.scenes[FIRST_SCENE].generation = args.generation; - self.control.total = args.blocks.len() as u32; - self.control.committed = self.control.total; - self.control.reset = 1; - let pages = (args.blocks.len() * cfg::TVB_PAGES_PER_BLOCK) as u32; - self.state.buffer_id = args.buffer_id; - self.state.page_list_fw = offset_va(args.fw_va, PAGE_LIST as u64)?; - self.state.page_list_client = args.page_list_client; - self.state.page_list_size = PAGE_LIST_BYTES as u32; - self.state.pages = pages; - self.state.block_capacity = cfg::TVB_BLOCK_SLOTS as u32; - self.state.producer = args.blocks.len() as u32; - self.state.block_table = offset_va(args.fw_va, BLOCK_TABLE as u64)?; - self.state.control = offset_va(args.fw_va, CONTROL as u64)?; - self.state.last_page = pages - 1; - self.state.block_size = cfg::TVB_BLOCK_SIZE as u32; - self.state.counter = offset_va(args.fw_va, COUNTER as u64)?; - self.state.maximum_size = (cfg::TVB_BLOCK_SLOTS * 4) as u32; - self.state.normal_maximum_size = (NORMAL_MAX_BLOCKS * 4) as u32; - self.state.discard = discard; - Ok(()) - } -} - -/// Rebuilds an idle parameter-buffer descriptor, preserving firmware-owned -/// bits and resetting both ring counters and flags for its rebuilt page list. -pub(crate) fn reload_descriptor(previous: [u32; 4], address: u64, pages: u32) -> [u32; 4] { - const COUNT: u32 = 0x003f_ffff; - let bias = if address & (1u64 << 42) == 0 { - 0x70_0000_0000 - } else { - 0 - }; - let segment = (address.wrapping_add(bias) & 0x70_0000_0000) | ((address >> 3) & 0x80_0000_0000); - [ - ((address >> 4) as u32 & !7) | (previous[0] & 6), - ((segment >> 8) as u32 & 0xf000_0000) | (previous[1] & 0x0fc0_0000) | (pages & COUNT), - previous[2] & !COUNT, - previous[3] & !(COUNT | 0x8000_0000), - ] -} - -/// Compact scratch and tagged metrics addresses carried by scene registers. -pub(crate) fn scene_registers(index: usize, scratch: u64, metrics: u64) -> Result<(u64, u64)> { - const SEGMENT_SELECT: u64 = 1 << 42; - const SEGMENT_BIAS: u64 = 0x70_0000_0000; - const ADDRESS_MASK: u64 = 0x7f_ffff_fffe; - const HIGH_BIT: u64 = 0x80_0000_0000; - if index >= SCENES || metrics & 3 != 0 { - return Err(EINVAL); - } - let scratch = scratch - .checked_sub(cfg::USER_BASE) - .ok_or(EINVAL)? - .checked_add((index % SCRATCH_SLOTS) as u64 * SCRATCH_STRIDE) - .ok_or(EOVERFLOW)?; - if scratch > u32::MAX as u64 || scratch & 0xf != 0 { - return Err(EINVAL); - } - let metric = offset_va(metrics, (index * 4) as u64)?; - let adjusted = offset_va( - metric, - if metric & SEGMENT_SELECT == 0 { - SEGMENT_BIAS - } else { - 0 - }, - )?; - Ok(( - scratch, - (adjusted & ADDRESS_MASK) | ((metric >> 3) & HIGH_BIT) | 1, - )) -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/channels.rs b/drivers/gpu/drm/asahi-neo/g17/fw/channels.rs deleted file mode 100644 index c4cf6960db296d..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/channels.rs +++ /dev/null @@ -1,527 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Device-control records and work-ring slots. -//! -//! The host sends out-of-band requests and replies through the device-control -//! ring, whose 0x40-byte records start with a [`DeviceControlOp`]. It hands -//! hardware queues to the firmware through work rings, one per priority and -//! engine, each holding [`WORK_RING_SLOTS`] [`WorkSlot`]s. -//! -//! Producers and consumers of both rings are live words in the primary state -//! grid; they are not part of these structures. Before publishing a -//! device-control record whose opcode [`DeviceControlOp::asserts_gpu_power`], -//! the host also increments its power-assert tally. - -use kernel::prelude::*; - -use super::kick::QID_MAX; -use super::queue::DataMaster; - -/// Size of a device-control record. -pub(crate) const DEVICE_CONTROL_RECORD_SIZE: usize = 0x40; -/// Number of records in the device-control ring. -pub(crate) const DEVICE_CONTROL_SLOTS: u32 = 256; - -/// Entries in either coprocessor's event ring. -pub(crate) const EVENT_SLOTS: u32 = 256; -/// Entries in either coprocessor's trace ring. -pub(crate) const TRACE_SLOTS: u32 = 512; -/// Entries in either coprocessor's statistics ring. -pub(crate) const STATISTICS_SLOTS: u32 = 256; -/// Words in an event, trace or statistics entry. -pub(crate) const EVENT_WORDS: usize = 18; -/// Byte stride of an event, trace or statistics entry. -pub(crate) const EVENT_RECORD_SIZE: usize = EVENT_WORDS * size_of::(); -/// Producer offset within a firmware-produced ring's state block. -pub(crate) const RX_PRODUCER_OFFSET: usize = 0x20; -/// Trace records use type 5. -pub(crate) const TRACE_TYPES: u32 = 1 << 5; -/// Statistics record types accepted by the host. -pub(crate) const STATISTICS_TYPES: u32 = 0x5fff_0000; - -/// Completion stamp indices signalled by the firmware. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -struct StampEventRecord { - tag: u32, - masks: [u64; 2], - diagnostic_pairs: u16, - unk_16: [u8; 0x32], -} - -/// Restart generation and the affected stamp, or -1 without attribution. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -struct RestartEventRecord { - tag: u32, - generation: u64, - stamp: i32, - unk_10: [u8; 0x38], -} - -/// Parameter-buffer growth request. The reply moves `halt_count` to +0x18. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -struct TvbGrowEventRecord { - tag: u32, - vm_slot: i32, - buffer_slot: u32, - counter: u32, - subpipe: u32, - halt_count: u64, - unk_1c: [u8; 0x2c], -} - -/// Completed asynchronous free-list growth. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -struct FreeListEventRecord { - tag: u32, - stamp: i32, - buffer_slot: u32, - control_va: u64, - ring_va: u64, - cookie: u64, - result: u32, - unk_28: [u8; 0x20], -} - -/// A snapshot of one firmware event. Every variant occupies a complete slot; -/// callers copy the entry before publishing its consumer cursor. -#[repr(C)] -#[derive(Copy, Clone)] -pub(crate) union EventRecord { - words: [u32; EVENT_WORDS], - stamp: StampEventRecord, - restart: RestartEventRecord, - tvb: TvbGrowEventRecord, - free_list: FreeListEventRecord, -} - -static_assert!(size_of::() == EVENT_RECORD_SIZE); -static_assert!(size_of::() == EVENT_RECORD_SIZE); -static_assert!(size_of::() == EVENT_RECORD_SIZE); -static_assert!(size_of::() == EVENT_RECORD_SIZE); -static_assert!(size_of::() == EVENT_RECORD_SIZE); -static_assert!(core::mem::offset_of!(StampEventRecord, masks) == 4); -static_assert!(core::mem::offset_of!(TvbGrowEventRecord, halt_count) == 0x14); -static_assert!(core::mem::offset_of!(FreeListEventRecord, cookie) == 0x1c); - -impl core::fmt::Debug for EventRecord { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("EventRecord") - .field("tag", &self.tag()) - .finish() - } -} - -/// A decoded asynchronous free-list growth completion. Resource ownership and -/// the expected cookie/result are checked by the free-list owner. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct FreeListCompletion { - pub(crate) stamp: i32, - pub(crate) buffer_slot: u32, - pub(crate) control_va: u64, - pub(crate) ring_va: u64, - pub(crate) cookie: u64, - pub(crate) result: u32, -} - -/// Host actions requested by a firmware event. Unknown tags are consumed so -/// one unrecognized record cannot block later completions. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum Event { - Stamp([u64; 2]), - Restart { generation: u64, stamp: i32 }, - TvbGrow(TvbGrowRequest), - FreeListGrow(FreeListCompletion), - ContextKilled(u64), - Unknown(u32), -} - -impl EventRecord { - const STAMP: u32 = 1; - const RESTART: u32 = 4; - const TVB_GROW: u32 = 6; - const FREE_LIST_GROW: u32 = 13; - const CONTEXT_KILLED: u32 = 12; - const DIAGNOSTIC_PAIRS_MAX: u16 = 5; - const TVB_SLOTS: u32 = 127; - const SUBPIPES: u32 = 64; - const FREE_LIST_SLOTS: u32 = 256; - - pub(crate) const fn from_words(words: [u32; EVENT_WORDS]) -> Self { - Self { words } - } - - pub(crate) fn tag(&self) -> u32 { - // SAFETY: All constructors initialize the entire union. Every bit - // pattern is valid for its u32 words, and word zero is the tag. - unsafe { self.words[0] } - } - - /// Validates the record's fields without interpreting resource ownership. - /// A malformed known event is an error, unlike an unknown tag. - pub(crate) fn decode(&self) -> Result { - match self.tag() { - Self::STAMP => { - // SAFETY: The entire slot is initialized, the tag selects - // this layout, and its integer fields have no invalid values. - let record = unsafe { self.stamp }; - if record.diagnostic_pairs > Self::DIAGNOSTIC_PAIRS_MAX { - return Err(EIO); - } - Ok(Event::Stamp(record.masks)) - } - Self::RESTART => { - // SAFETY: As above, for the restart layout. - let record = unsafe { self.restart }; - Ok(Event::Restart { - generation: record.generation, - stamp: record.stamp, - }) - } - Self::TVB_GROW => { - // SAFETY: As above, for the parameter-buffer growth layout. - let record = unsafe { self.tvb }; - if record.vm_slot < -1 - || record.buffer_slot >= Self::TVB_SLOTS - || record.subpipe >= Self::SUBPIPES - { - return Err(EIO); - } - Ok(Event::TvbGrow(TvbGrowRequest { - buffer_slot: record.buffer_slot, - vm_slot: record.vm_slot, - counter: record.counter, - subpipe: record.subpipe, - halt_count: record.halt_count, - })) - } - Self::CONTEXT_KILLED => { - // SAFETY: Every constructor initializes all integer words. - let words = unsafe { self.words }; - Ok(Event::ContextKilled( - u64::from(words[1]) | (u64::from(words[2]) << 32), - )) - } - Self::FREE_LIST_GROW => { - // SAFETY: As above, for the free-list growth layout. - let record = unsafe { self.free_list }; - if record.stamp < -1 - || record.buffer_slot >= Self::FREE_LIST_SLOTS - || record.control_va == 0 - || record.ring_va == 0 - { - return Err(EIO); - } - Ok(Event::FreeListGrow(FreeListCompletion { - stamp: record.stamp, - buffer_slot: record.buffer_slot, - control_va: record.control_va, - ring_va: record.ring_va, - cookie: record.cookie, - result: record.result, - })) - } - tag => Ok(Event::Unknown(tag)), - } - } -} - -/// A complete device-control request. Its variant supplies the power policy; -/// callers construct the contained record with its typed builder. -#[derive(Debug)] -pub(crate) enum ControlRecord { - TvbGrow(TvbGrowReply), - IdlePowerOff(IdlePowerOff), - SchedulerStateRelease(SchedulerStateRelease), - FreeListRelease(FreeListRelease), - ContextKill(ContextKill), -} - -impl ControlRecord { - pub(crate) const fn opcode(&self) -> DeviceControlOp { - match self { - Self::TvbGrow(_) => DeviceControlOp::TvbGrowReply, - Self::IdlePowerOff(_) => DeviceControlOp::IdlePowerOff, - Self::SchedulerStateRelease(_) => DeviceControlOp::SchedulerStateRelease, - Self::FreeListRelease(_) => DeviceControlOp::FreeListRelease, - Self::ContextKill(_) => DeviceControlOp::ContextKill, - } - } - - /// Byte view of the initialized record, excluding the host-only enum tag. - pub(crate) fn bytes(&self) -> &[u8; DEVICE_CONTROL_RECORD_SIZE] { - let record = match self { - Self::TvbGrow(record) => core::ptr::from_ref(record).cast(), - Self::IdlePowerOff(record) => core::ptr::from_ref(record).cast(), - Self::SchedulerStateRelease(record) => core::ptr::from_ref(record).cast(), - Self::FreeListRelease(record) => core::ptr::from_ref(record).cast(), - Self::ContextKill(record) => core::ptr::from_ref(record).cast(), - }; - // SAFETY: Each variant has a statically checked size of 0x40 bytes, - // contains only initialized integer fields with no implicit padding, - // and is borrowed for the lifetime of `self`. The byte view has align 1. - unsafe { &*record } - } -} - -/// Device-control opcodes the driver sends. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum DeviceControlOp { - /// Reply to a parameter-buffer grow request ([`TvbGrowReply`]). - TvbGrowReply = 0x08, - /// Set the idle power-off policy ([`IdlePowerOff`]). - IdlePowerOff = 0x0a, - /// Release a logical queue's scheduler state ([`SchedulerStateRelease`]). - SchedulerStateRelease = 0x14, - /// Release a free list's buffer slot ([`FreeListRelease`]). - FreeListRelease = 0x2e, - /// Stop the queues named by a closed logical scheduler. - ContextKill = 0x29, -} - -impl DeviceControlOp { - /// Tells whether records with this opcode count towards the host - /// power-assert tally. The firmware compares the tally with the counted - /// records it has drained and keeps the GPU cores powered while they - /// differ. - pub(crate) const fn asserts_gpu_power(self) -> bool { - !matches!(self, Self::FreeListRelease) - } -} - -/// The fields of a parameter-buffer grow request event that its reply -/// echoes. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct TvbGrowRequest { - /// Buffer slot of the parameter buffer. - pub(crate) buffer_slot: u32, - /// VM slot of the request, or -1. - pub(crate) vm_slot: i32, - /// Request counter. - pub(crate) counter: u32, - /// Sub-pipe of the request. - pub(crate) subpipe: u32, - /// Firmware halt count the request was made under. - pub(crate) halt_count: u64, -} - -/// Reply to a parameter-buffer grow request. -#[repr(C)] -#[derive(Debug)] -pub(crate) struct TvbGrowReply { - /// [`DeviceControlOp::TvbGrowReply`]. - pub(crate) opcode: u32, - /// Non-zero when the parameter buffer grew. - pub(crate) grew: u32, - /// Buffer slot of the request. - pub(crate) buffer_slot: u32, - /// VM slot of the request. - pub(crate) vm_slot: i32, - /// Request counter. - pub(crate) counter: u32, - /// Sub-pipe of the request. - pub(crate) subpipe: u32, - /// Halt count of the request. - pub(crate) halt_count: u64, - pub(crate) unk_20: [u8; 0x20], -} - -static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); - -impl TvbGrowReply { - /// Replies to `request`; `grew` tells whether the buffer was grown. - pub(crate) const fn new(request: &TvbGrowRequest, grew: bool) -> Self { - Self { - opcode: DeviceControlOp::TvbGrowReply as u32, - grew: grew as u32, - buffer_slot: request.buffer_slot, - vm_slot: request.vm_slot, - counter: request.counter, - subpipe: request.subpipe, - halt_count: request.halt_count, - unk_20: [0; 0x20], - } - } -} - -/// Idle power-off policy argument of [`IdlePowerOff`]. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum IdlePolicy { - /// Keep the GPU cores powered and power them up now. - KeepPowered = 0, - /// Let the firmware power the GPU cores off when idle. - AllowPowerOff = 1, -} - -/// Sets the firmware's idle power-off policy. -#[repr(C)] -#[derive(Debug)] -pub(crate) struct IdlePowerOff { - /// [`DeviceControlOp::IdlePowerOff`]. - pub(crate) opcode: u32, - /// [`IdlePolicy`]. - pub(crate) policy: u32, - pub(crate) unk_08: [u8; 0x38], -} - -static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); - -impl IdlePowerOff { - /// Requests `policy`. - pub(crate) const fn new(policy: IdlePolicy) -> Self { - Self { - opcode: DeviceControlOp::IdlePowerOff as u32, - policy: policy as u32, - unk_08: [0; 0x38], - } - } -} - -/// Releases the scheduler state of a logical queue whose work has drained. -/// -/// The page may be freed once the firmware has consumed the record. -#[repr(C, packed)] -#[derive(Debug)] -pub(crate) struct SchedulerStateRelease { - /// [`DeviceControlOp::SchedulerStateRelease`]. - pub(crate) opcode: u32, - pub(crate) unk_04: u32, - /// The scheduler-page bytes at offsets 0x26, 0, 1 and 4, in record order. - /// Read when the release record is built. - pub(crate) id: [u8; 4], - /// GPU address of the scheduler-state page. - pub(crate) page_va: u64, - pub(crate) unk_14: [u8; 0x2c], -} - -static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); -static_assert!(core::mem::offset_of!(SchedulerStateRelease, page_va) == 0x0c); - -impl SchedulerStateRelease { - /// Releases the scheduler page with its identity bytes in record order. - pub(crate) const fn new(page_va: u64, id: [u8; 4]) -> Self { - Self { - opcode: DeviceControlOp::SchedulerStateRelease as u32, - unk_04: 0, - id, - page_va, - unk_14: [0; 0x2c], - } - } -} - -/// Releases the buffer slot of a free list after its last user. -#[repr(C, packed)] -#[derive(Debug)] -pub(crate) struct FreeListRelease { - /// [`DeviceControlOp::FreeListRelease`]. - pub(crate) opcode: u32, - /// Free-list generation of the registration being released. - pub(crate) generation: u64, - /// Buffer slot of the free list. - pub(crate) buffer_slot: u32, - pub(crate) unk_10: [u8; 0x30], -} - -static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); -static_assert!(core::mem::offset_of!(FreeListRelease, buffer_slot) == 0x0c); - -impl FreeListRelease { - /// Releases `buffer_slot`, registered at `generation`. - pub(crate) const fn new(generation: u64, buffer_slot: u32) -> Self { - Self { - opcode: DeviceControlOp::FreeListRelease as u32, - generation, - buffer_slot, - unk_10: [0; 0x30], - } - } -} - -/// Number of slots in a work ring. -pub(crate) const WORK_RING_SLOTS: u32 = 256; -/// Number of engines with a work ring per priority. -const WORK_RINGS_PER_PRIORITY: usize = 3; - -/// Index in the channel table of the work ring for `data_master` at priority -/// class `priority`, or `None` if `priority` is not a class. -pub(crate) const fn work_ring_index(priority: u8, data_master: DataMaster) -> Option { - if priority >= super::queue::Policy::PRIORITIES { - return None; - } - Some(priority as usize * WORK_RINGS_PER_PRIORITY + data_master as usize) -} - -/// One work-ring slot: a request to run a queue's newly published commands. -#[repr(C)] -#[derive(Debug)] -pub(crate) struct WorkSlot { - pub(crate) unk_00: u64, - /// GPU address of the queue record. - pub(crate) queue_va: u64, - /// [`DataMaster`] of the queue. - pub(crate) data_master: u32, - /// Item-ring producer after the published commands. - pub(crate) items_end: u16, - /// Queue to run. - pub(crate) qid: u8, - /// One in the first slot naming this queue on this ring. Each priority - /// class has its own rings, so a queue whose class changed names the new - /// ring with a first slot again. - pub(crate) first: u8, -} - -static_assert!(size_of::() == 0x18); - -impl WorkSlot { - /// Builds the slot asking the firmware to run queue `qid` up to item-ring - /// position `items_end`. `first` tells whether this is the first slot - /// naming the queue on this ring. - pub(crate) fn new( - data_master: DataMaster, - queue_va: u64, - qid: u8, - items_end: u16, - first: bool, - ) -> Result { - if qid > QID_MAX { - return Err(EINVAL); - } - Ok(Self { - unk_00: 0, - queue_va, - data_master: data_master as u32, - items_end, - qid, - first: first as u8, - }) - } -} - -/// Stop a closed context's scheduler page and echo its cookie through event 12. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct ContextKill { - opcode: u32, - page_va: u64, - cookie: u64, - reserved: [u8; 0x2c], -} -static_assert!(size_of::() == DEVICE_CONTROL_RECORD_SIZE); -static_assert!(core::mem::offset_of!(ContextKill, page_va) == 4); -static_assert!(core::mem::offset_of!(ContextKill, cookie) == 0x0c); -impl ContextKill { - pub(crate) fn new(page_va: u64, cookie: u64) -> Self { - Self { - opcode: DeviceControlOp::ContextKill as u32, - page_va, - cookie, - reserved: [0; 0x2c], - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/compute.rs b/drivers/gpu/drm/asahi-neo/g17/fw/compute.rs deleted file mode 100644 index 5b7489cfe93f4b..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/compute.rs +++ /dev/null @@ -1,442 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Compute work descriptor. -//! -//! A compute kick launches one [`ComputeDescriptor`]. The descriptor embeds -//! the two register arrays its kick entry binds: the main array and a -//! four-entry array that writes the operand-state registers with their -//! regions exchanged. - -use kernel::prelude::*; - -use super::kick::{KickTimestamp, RegisterArray, RegisterArrayBinding, RegisterWrite}; -use super::queue::CommandTag; -use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, - UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, -}; - -/// Size of a compute descriptor. -pub(crate) const COMPUTE_DESCRIPTOR_SIZE: usize = 0x4000; -/// QoS class of compute kicks. -pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; -/// Event mask of compute kicks. -pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; - -const COMPUTE_REGISTERS: usize = 40; -const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; - -/// Validated per-command shader-context scratch request. -#[derive(Clone, Copy)] -pub(crate) struct ScratchRequest { - pages: u16, -} - -impl ScratchRequest { - const FIXED_BYTES: u64 = 0x58000; - const THREADS_PER_CLIQUE: u64 = 32; - const BYTES_PER_STACK_ENTRY: u64 = 8; - const PAGE_SHIFT: u32 = 12; - const PAGE_SIZE: u64 = 1 << Self::PAGE_SHIFT; - const PAGES_PER_MIB: u64 = 1 << (20 - Self::PAGE_SHIFT); - const WINDOW_ENABLED: u64 = 1 << 24; - - /// Payload-free layouts request no scratch even if depth/cliques are set. - /// Otherwise round the fixed and per-thread sections and add a guard page. - pub(crate) fn new(layout: u64) -> Result { - let depth = layout & u16::MAX as u64; - let cliques = (layout >> 24) & u8::MAX as u64; - let user = (layout >> 32) & u16::MAX as u64; - let driver = (layout >> 48) & u16::MAX as u64; - if user == 0 && driver == 0 { - return Ok(Self { pages: 0 }); - } - let threads = cliques.checked_mul(Self::THREADS_PER_CLIQUE).ok_or(EINVAL)?; - let per_thread = depth.checked_mul(Self::BYTES_PER_STACK_ENTRY) - .and_then(|bytes| bytes.checked_add(user)) - .and_then(|bytes| bytes.checked_add(driver)).ok_or(EINVAL)?; - let bytes = threads.checked_mul(per_thread) - .and_then(|bytes| bytes.checked_add(Self::FIXED_BYTES)) - .and_then(|bytes| bytes.checked_add(2 * Self::PAGE_SIZE - 1)) - .ok_or(EINVAL)?; - let pages = bytes >> Self::PAGE_SHIFT; - Ok(Self { pages: u16::try_from(pages).map_err(|_| EINVAL)? }) - } - - pub(crate) fn enabled(self) -> bool { - self.pages != 0 - } - - fn size_word(self) -> u64 { - let pages = u64::from(self.pages); - pages | (pages.div_ceil(Self::PAGES_PER_MIB) << 16) - } - - fn window_word(self) -> u64 { - if self.enabled() { Self::WINDOW_ENABLED } else { UNK_10791 } - } - - fn checkpoints(self) -> [u16; 4] { - CHECKPOINTS.map(|count| count + u16::from(self.enabled())) - } -} -const ALT_REGISTERS: usize = 4; - -/// Object IDs of every compute kick. -const OBJECT_IDS: ObjectIds = ObjectIds { - predecessor: 0x0100_01d7, - current: 0x0200_01dc, -}; -/// Values of the first and second write of register 0x1a440. -const UNK_1A440: u64 = 0x0000_0001_5402_4201; -const UNK_1A440_FINAL: u64 = 0x0000_0001_5402_4209; -/// Value of register 0x1a458: [`UNK_1C8F8`] below bits 31:20 = 0x10c. -const UNK_1A458: u64 = 0x10c0_0000 | UNK_1C8F8; - -/// Offsets in the preempt buffer of the four state words the program names. -const PREEMPT_STATE: [u64; 4] = [0x1480, 0x1488, 0x1490, 0x1498]; -/// Offsets in the operand-state buffer of the regions the program names. -const OPERAND_STATE_10229: u64 = 0x12800; -const OPERAND_STATE_140A8: u64 = 0x13000; -const OPERAND_STATE_10099: u64 = 0x9405; -const OPERAND_STATE_10091: u64 = 0x12400; -const OPERAND_STATE_0A5C1: u64 = 0x0005; -const OPERAND_STATE_0A5C9: u64 = 0x9000; - -/// Checkpoints into the main register array, as entry counts: after the -/// first 0x0a5c9 write, after the final 0x1a440 write (twice), and the full -/// count. -const CHECKPOINTS: [u16; 4] = [38, 39, 39, COMPUTE_REGISTERS as u16]; - -/// Inputs of one compute descriptor. -pub(crate) struct ComputeArgs { - /// Scratch request validated with the userspace command. - pub(crate) scratch: ScratchRequest, - /// Base of the queue's USC window. - pub(crate) usc_base: u64, - /// GPU address of the control stream. - pub(crate) cdm_va: u64, - /// GPU address of the control-stream terminator. - pub(crate) cdm_end_va: u64, - /// GPU address of the sampler heap, or zero. - pub(crate) sampler_heap_va: u64, - /// Number of samplers in the heap. - pub(crate) sampler_count: u32, - /// GPU address of this descriptor, as seen by the work's VM. - pub(crate) descriptor_va: u64, - /// UAT context the work runs in. - pub(crate) context_id: u16, - /// Allocation generation of that context. - pub(crate) context_generation: u8, - /// State word of the logical queue's work-state node. - pub(crate) work_state: u32, - /// GPU address of the logical queue's work-state node. - pub(crate) work_state_va: u64, - /// Queue the work is kicked on. - pub(crate) qid: u8, - /// Number of earlier kicks of the queue. - pub(crate) kick_count: u32, - /// Timestamp the work is kicked at. - pub(crate) kick: KickTimestamp, - /// GPU address of the preempt buffer, as seen by the work's VM. - pub(crate) preempt_va: u64, - /// GPU address of the operand-state buffer, as seen by the work's VM. - pub(crate) operand_state_va: u64, - /// GPU address of the private-memory usage record, as seen by the work's - /// VM. - pub(crate) usage_va: u64, - /// Firmware alias of the private-memory usage record. - pub(crate) usage_fw_va: u64, - /// Buffer slot of the USC private-memory free list. - pub(crate) free_list_slot: u32, - /// GPU address of the free list's shared control object. - pub(crate) free_list_control_va: u64, - /// GPU address the completion stamp value is stored to. - pub(crate) stamp_va: u64, - /// GPU address of the word following the completion stamp. - pub(crate) aux_stamp_va: u64, - /// GPU addresses of the two status words. - pub(crate) status_va: [u64; 2], - /// GPU address of the start/end timestamp pair. - pub(crate) timestamp_va: u64, -} - -fn compute_registers(args: &ComputeArgs) -> Result<[RegisterWrite; COMPUTE_REGISTERS_MAX]> { - let preempt = args.preempt_va; - let operand = args.operand_state_va; - let key = work_key(args.context_id, args.work_state); - let ids = OBJECT_IDS.word(); - - Ok([ - RegisterWrite::new(0x017d9, 1), - RegisterWrite::new(0x1a510, preempt), - RegisterWrite::new(0x1a420, args.cdm_va), - RegisterWrite::new(0x1a4d0, offset_va(preempt, PREEMPT_STATE[0])?), - RegisterWrite::new(0x1a4d8, offset_va(preempt, PREEMPT_STATE[1])?), - RegisterWrite::new(0x1a4e0, offset_va(preempt, PREEMPT_STATE[2])?), - RegisterWrite::new(0x1a4e8, offset_va(preempt, PREEMPT_STATE[3])?), - RegisterWrite::new(0x10071, args.usc_base), - // Helper program binary, data and configuration: none. - RegisterWrite::new(0x11841, 0), - RegisterWrite::new(0x11849, 0), - RegisterWrite::new(0x11f81, 0), - RegisterWrite::new(0x1a440, UNK_1A440), - RegisterWrite::new(0x1a458, UNK_1A458), - RegisterWrite::new(0x101d9, UNK_101D9), - RegisterWrite::new(0x1a089, 0), - RegisterWrite::new(0x1a091, 0), - RegisterWrite::new(0x1a059, 0), - RegisterWrite::new(0x1a061, 0), - RegisterWrite::new(0x1a0b9, 0), - RegisterWrite::new(0x1a0c1, 0), - RegisterWrite::new(0x101d1, args.scratch.size_word()), - RegisterWrite::new(0x0d479, 0), - RegisterWrite::new(0x1a0e9, UNK_1A0E9), - RegisterWrite::new(0x107a1, args.scratch.window_word()), - RegisterWrite::new(0x0a599, UNK_0A599), - RegisterWrite::new(0x0d411, UNK_0D411), - RegisterWrite::new(0x1a540, ids), - RegisterWrite::new(0x014a9, ids), - RegisterWrite::new(0x0a351, ids), - RegisterWrite::new(0x10201, key), - RegisterWrite::new(0x10428, key), - RegisterWrite::new(0x14028, args.free_list_slot as u64), - RegisterWrite::new(0x14070, args.usage_va | 1), - RegisterWrite::new(0x10229, offset_va(operand, OPERAND_STATE_10229)?), - RegisterWrite::new(0x140a8, offset_va(operand, OPERAND_STATE_140A8)?), - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_10091)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x1a440, UNK_1A440_FINAL), - RegisterWrite::new(0x0a599, UNK_0A599), - ]) -} - -/// The second array exchanges the operand-state regions of registers -/// 0x10099/0x0a5c1 and 0x10091/0x0a5c9. -fn alt_registers(args: &ComputeArgs) -> Result<[RegisterWrite; ALT_REGISTERS]> { - let operand = args.operand_state_va; - Ok([ - RegisterWrite::new(0x10099, offset_va(operand, OPERAND_STATE_0A5C1)?), - RegisterWrite::new(0x10091, offset_va(operand, OPERAND_STATE_0A5C9)?), - RegisterWrite::new(0x0a5c1, offset_va(operand, OPERAND_STATE_10099)?), - RegisterWrite::new(0x0a5c9, offset_va(operand, OPERAND_STATE_10091)?), - ]) -} - -/// Compute work descriptor (tag 3). -#[repr(C, packed)] -#[derive(Debug)] -pub(crate) struct ComputeDescriptor { - /// [`CommandTag::Compute`]. - pub(crate) tag: u32, - pub(crate) unk_04: u64, - /// UAT context of the work. - pub(crate) context_id: u32, - /// GPU address of the logical queue's work-state node. - pub(crate) work_state_va: u64, - /// Checkpoints into the main register array. - pub(crate) checkpoints: [u16; 4], - pub(crate) unk_20: [u8; 0x20], - /// Main register array. - pub(crate) registers: RegisterArray, - /// Second register array. - pub(crate) alt_registers: RegisterArray, - pub(crate) unk_e80: [u8; 0x58], - /// GPU address of the preempt buffer. - pub(crate) preempt_va: u64, - /// GPU address of the control-stream terminator. - pub(crate) cdm_end_va: u64, - pub(crate) unk_ee8: [u8; 0x20], - /// First value of register 0x1a440. - pub(crate) unk_f08: u64, - pub(crate) unk_f10: [u8; 0x10], - /// Predecessor object ID. - pub(crate) predecessor_id: u32, - pub(crate) unk_f24: u32, - pub(crate) unk_f28: u32, - /// GPU address of the sampler heap. - pub(crate) sampler_heap_va: u64, - /// Number of samplers. - pub(crate) sampler_count: u32, - /// Number of samplers plus one, or zero without samplers. - pub(crate) sampler_max: u32, - pub(crate) unk_f3c: u32, - /// GPU address the firmware stores `stamp_value` to on completion. - pub(crate) stamp_va: u64, - /// GPU address of the word following the completion stamp. - pub(crate) aux_stamp_va: u64, - /// Completion stamp value of the kick timestamp. - pub(crate) stamp_value: u32, - /// QID of the compute queue. - pub(crate) qid: u32, - pub(crate) unk_f58: u32, - pub(crate) unk_f5c: u32, - /// Kick timestamp. - pub(crate) kick_timestamp: u64, - /// Current object ID. - pub(crate) current_id: u64, - /// Number of earlier kicks of the queue. - pub(crate) kick_count: u32, - pub(crate) unk_f74: [u8; 8], - /// GPU addresses of the two status words. - pub(crate) status_va: [u64; 2], - /// GPU address the start timestamp is written to. - pub(crate) timestamp_start_va: u64, - /// GPU address the end timestamp is written to. - pub(crate) timestamp_end_va: u64, - pub(crate) unk_f9c: [u8; 0x14], - pub(crate) unk_fb0: u16, - /// GPU address of the USC free list's shared control object. - pub(crate) free_list_control_va: u64, - pub(crate) unk_fba: u32, - pub(crate) unk_fbe: [u8; 7], - pub(crate) unk_fc5: u8, - pub(crate) unk_fc6: [u8; 5], - /// Firmware alias of the private-memory usage record. - pub(crate) usage_va: u64, - /// Allocation generation of the UAT context. - pub(crate) context_generation: u8, - pub(crate) unk_fd4: [u8; 0x302c], -} - -static_assert!(size_of::() == COMPUTE_DESCRIPTOR_SIZE); -static_assert!(core::mem::offset_of!(ComputeDescriptor, registers) == 0x40); -static_assert!(core::mem::offset_of!(ComputeDescriptor, alt_registers) == 0x760); -static_assert!(core::mem::offset_of!(ComputeDescriptor, preempt_va) == 0xed8); -static_assert!(core::mem::offset_of!(ComputeDescriptor, unk_f08) == 0xf08); -static_assert!(core::mem::offset_of!(ComputeDescriptor, predecessor_id) == 0xf20); -static_assert!(core::mem::offset_of!(ComputeDescriptor, sampler_heap_va) == 0xf2c); -static_assert!(core::mem::offset_of!(ComputeDescriptor, stamp_va) == 0xf40); -static_assert!(core::mem::offset_of!(ComputeDescriptor, kick_timestamp) == 0xf60); -static_assert!(core::mem::offset_of!(ComputeDescriptor, current_id) == 0xf68); -static_assert!(core::mem::offset_of!(ComputeDescriptor, status_va) == 0xf7c); -static_assert!(core::mem::offset_of!(ComputeDescriptor, free_list_control_va) == 0xfb2); -static_assert!(core::mem::offset_of!(ComputeDescriptor, usage_va) == 0xfcb); -static_assert!(core::mem::offset_of!(ComputeDescriptor, context_generation) == 0xfd3); - -// SAFETY: `ComputeDescriptor` consists of integers and arrays of integers only. -unsafe impl Zeroable for ComputeDescriptor {} - -impl ComputeDescriptor { - const REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; - const ALT_REGISTERS_OFFSET: u64 = core::mem::offset_of!(Self, alt_registers) as u64; - const UNK_F28: u32 = u32::MAX; - const UNK_FB0: u16 = 0x1a; - const UNK_FBA: u32 = 0xe0a0_0001; - const UNK_FC5: u8 = 0x9f; - - /// Writes the complete descriptor. - pub(crate) fn write(&mut self, args: &ComputeArgs) -> Result { - if args.cdm_end_va < args.cdm_va - || (args.sampler_heap_va == 0) != (args.sampler_count == 0) - || args.sampler_heap_va & 7 != 0 - { - return Err(EINVAL); - } - let registers = compute_registers(args)?; - let alt_registers = alt_registers(args)?; - let registers_va = offset_va(args.descriptor_va, Self::REGISTERS_OFFSET)?; - let alt_registers_va = offset_va(args.descriptor_va, Self::ALT_REGISTERS_OFFSET)?; - let timestamp_end_va = timestamp_end_va(args.timestamp_va)?; - let sampler_max = sampler_max(args.sampler_count)?; - - clear(self); - self.tag = CommandTag::Compute as u32; - self.context_id = args.context_id.into(); - self.work_state_va = args.work_state_va; - self.checkpoints = args.scratch.checkpoints(); - // The enabled program prepends one register; disabled descriptors keep - // the same forty writes, trailer count and resume checkpoints. - let first = usize::from(!args.scratch.enabled()); - self.registers.set(registers_va, ®isters[first..])?; - self.alt_registers.set(alt_registers_va, &alt_registers)?; - self.preempt_va = args.preempt_va; - self.cdm_end_va = args.cdm_end_va; - self.unk_f08 = UNK_1A440; - self.predecessor_id = OBJECT_IDS.predecessor; - self.unk_f28 = Self::UNK_F28; - self.sampler_heap_va = args.sampler_heap_va; - self.sampler_count = args.sampler_count; - self.sampler_max = sampler_max; - self.stamp_va = args.stamp_va; - self.aux_stamp_va = args.aux_stamp_va; - self.stamp_value = stamp_value(args.kick.get() as u32); - self.qid = args.qid.into(); - self.kick_timestamp = args.kick.get(); - self.current_id = OBJECT_IDS.current.into(); - self.kick_count = args.kick_count; - self.status_va = args.status_va; - self.timestamp_start_va = args.timestamp_va; - self.timestamp_end_va = timestamp_end_va; - self.unk_fb0 = Self::UNK_FB0; - self.free_list_control_va = args.free_list_control_va; - self.unk_fba = Self::UNK_FBA; - self.unk_fc5 = Self::UNK_FC5; - self.usage_va = args.usage_fw_va; - self.context_generation = args.context_generation; - Ok(()) - } - - /// Returns the register arrays a compute kick binds; `descriptor_va` is - /// the descriptor's address as seen by the work's VM. - pub(crate) fn register_bindings( - descriptor_va: u64, - scratch: ScratchRequest, - ) -> Result<[Option; 4]> { - let registers = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::REGISTERS_OFFSET)?, - scratch.checkpoints()[0] as u8, - )?; - let alt_registers = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ALT_REGISTERS_OFFSET)?, - ALT_REGISTERS as u8, - )?; - Ok([Some(registers), Some(alt_registers), None, None]) - } -} - -/// One retained queue-context record. The descriptor pointer is refreshed for each compute -/// publication; the remaining fields are installed once with the queue. -#[repr(C)] -#[derive(Clone, Copy)] -pub(crate) struct QueueContextItem { - unk_00: u64, - unk_08: u64, - descriptor_va: u64, - queue_va: u64, - unk_20: u64, - qid: u64, - unk_30: [u8; 0x100], - unk_130: u64, - unk_138: u64, - unk_140: [u8; 0x10], - unk_150: u64, - unk_158: u64, - unk_160: [u8; 0x18], - unk_178: u64, - unk_180: [u8; 0x80], -} -impl QueueContextItem { - pub(crate) fn new(descriptor_va: u64, queue_va: u64, qid: u8) -> Self { - Self { - unk_00: 0x1000_1000_0000_0004, - unk_08: 0, - descriptor_va, - queue_va, - unk_20: 0xffff_0801_0000_0001, - qid: u64::from(qid) << 40, - unk_30: [0; 0x100], - unk_130: 0, - unk_138: 4, - unk_140: [0; 0x10], - unk_150: 0x0001_1003_8001_a002, - unk_158: 0x0000_2003_8001_a03b, - unk_160: [0; 0x18], - unk_178: 0x003f_ffff_ffff_ffff, - unk_180: [0; 0x80], - } - } -} -static_assert!(size_of::() == 0x200); diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/context.rs b/drivers/gpu/drm/asahi-neo/g17/fw/context.rs deleted file mode 100644 index 51dbde231e2382..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/context.rs +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Scheduler and timestamp-queue state shared with firmware. Only a freshly -//! leased node may be initialized. After publication, the host writes its -//! separate submitted counter and reads the node completion word individually. - -use kernel::prelude::*; - -#[repr(C)] -pub(crate) struct Scheduler { - unk0: u64, - unk8: [u64; 3], - unk20: u64, - unk28: u64, - unk30: u64, - unk38: [u8; 0x3fc8], -} -impl Scheduler { - /// Called on zeroed storage before its first publication. - pub(crate) fn init(&mut self) { - self.unk0 = 0x0000_0100_0000_ffff; - self.unk20 = 0x0002_0000_0000_0000; - self.unk30 = 0x0000_0000_ff00_0000; - } -} - -#[derive(Clone, Copy)] -#[repr(C)] -pub(crate) struct WorkNode { - pub(crate) submitted: u64, - pub(crate) word: u32, - pub(crate) completed: u32, - pub(crate) threshold: u32, - unk14: [u32; 45], -} -impl WorkNode { - pub(crate) fn new(submitted: u64, word: u32) -> Option { - if submitted == 0 || submitted & 3 != 0 || word & 63 == 63 { - return None; - } - Some(Self { - submitted, - word, - completed: 0, - threshold: 0x50, - unk14: [0; 45], - }) - } -} - -#[derive(Clone, Copy)] -#[repr(C)] -pub(crate) struct WorkHead { - pub(crate) next: u64, - pub(crate) previous: u64, - pub(crate) unk10: u32, - pub(crate) unk14: u32, -} - -const _: () = { - assert!(core::mem::size_of::() == 0x4000); - assert!(core::mem::size_of::() == 0xc8); - assert!(core::mem::offset_of!(WorkNode, completed) == 0xc); - assert!(core::mem::size_of::() == 0x18); -}; - -// SAFETY: These shared records contain only integer fields and arrays. -unsafe impl Zeroable for Scheduler {} -// SAFETY: WorkNode contains only integers; zero is valid for every field. -unsafe impl Zeroable for WorkNode {} -// SAFETY: WorkHead contains only integer addresses and words. -unsafe impl Zeroable for WorkHead {} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/freelist.rs b/drivers/gpu/drm/asahi-neo/g17/fw/freelist.rs deleted file mode 100644 index 2e5708dddd4552..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/freelist.rs +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! USC free-list page addresses and block run tables. - -use super::initdata::{FreeListRunSlot, FREE_LIST_BLOCK_PAGES, FREE_LIST_BLOCK_SIZE}; -use kernel::prelude::*; - -pub(crate) const PAGE_LIST_SIZE: usize = 0x20_0000; -pub(crate) const RUN_LIST_SIZE: usize = 0x1_0000; -pub(crate) const TABLE_SIZE: usize = PAGE_LIST_SIZE + RUN_LIST_SIZE; -pub(crate) const CONTROL_SIZE: usize = 0x4000; -pub(crate) const RENDER_BLOCKS: usize = 22; -pub(crate) const COMPUTE_BLOCKS: usize = 21; - -/// Render reserves eight run words per block; compute packs runs consecutively. -#[derive(Copy, Clone, Debug)] -pub(crate) enum RunLayout { - Render, - Compute, -} - -impl RunLayout { - const fn words(self) -> usize { - match self { - Self::Render => size_of::() / 8, - Self::Compute => 1, - } - } -} - -#[repr(C)] -pub(crate) struct PoolTables { - pages: [u64; PAGE_LIST_SIZE / 8], - runs: [u64; RUN_LIST_SIZE / 8], -} - -// SAFETY: The table consists entirely of integer arrays. -unsafe impl Zeroable for PoolTables {} -static_assert!(size_of::() == TABLE_SIZE); -static_assert!(core::mem::offset_of!(PoolTables, runs) == PAGE_LIST_SIZE); - -impl PoolTables { - /// Enumerates only the initialized run words and page addresses, in their - /// publication order. Reserved run words and unused table space stay zero. - pub(crate) fn for_each_word( - base: u64, - blocks: usize, - layout: RunLayout, - mut put: impl FnMut(usize, u64), - ) -> Result { - if blocks > RUN_LIST_SIZE / (layout.words() * 8) - || blocks > PAGE_LIST_SIZE / (FREE_LIST_BLOCK_PAGES as usize * 8) - { - return Err(EINVAL); - } - let buffers = base.checked_add(TABLE_SIZE as u64).ok_or(EOVERFLOW)?; - buffers - .checked_add((blocks * FREE_LIST_BLOCK_SIZE) as u64) - .ok_or(EOVERFLOW)?; - for block in 0..blocks { - let address = buffers + (block * FREE_LIST_BLOCK_SIZE) as u64; - put( - PAGE_LIST_SIZE / 8 + block * layout.words(), - FreeListRunSlot::new(address).run, - ); - for page in 0..FREE_LIST_BLOCK_PAGES as usize { - put( - block * FREE_LIST_BLOCK_PAGES as usize + page, - address - + (page * (FREE_LIST_BLOCK_SIZE / FREE_LIST_BLOCK_PAGES as usize)) as u64, - ); - } - } - Ok(()) - } - - /// Initializes a zeroed table before its first publication. - pub(crate) fn init(&mut self, base: u64, blocks: usize, layout: RunLayout) -> Result { - Self::for_each_word(base, blocks, layout, |index, value| { - if index < self.pages.len() { - self.pages[index] = value; - } else { - self.runs[index - self.pages.len()] = value; - } - }) - } -} - -/// Mutable page-pool state carried between the retained control record and -/// its global descriptor. Offsets name the source control-record words. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct PagePoolState { - pub(crate) page_list_va: u64, - pub(crate) unk_1c: u32, - pub(crate) unk_20: u32, - pub(crate) unk_24: u32, - pub(crate) unk_28: u32, - pub(crate) page_count: u32, -} - -impl PagePoolState { - /// The fourth descriptor word belongs to firmware and survives every load. - pub(crate) const fn descriptor(self, unk_18: u64) -> super::initdata::PagePoolDescriptor { - super::initdata::PagePoolDescriptor { - page_list: ((self.unk_1c as u64) << 41) | (self.page_list_va >> 7), - counters: ((self.unk_24 as u64) << 33) - | ((self.unk_20 as u64) << 5) - | ((self.unk_28 as u64) << 61), - page_count: self.page_count as u64, - unk_18, - } - } - - /// Address and allocation unit remain properties of the retained pool. - pub(crate) const fn updated(self, descriptor: &super::initdata::PagePoolDescriptor) -> Self { - Self { - unk_20: ((descriptor.counters >> 5) & 0x0fff_ffff) as u32, - unk_24: ((descriptor.counters >> 33) & 0x0fff_ffff) as u32, - unk_28: (descriptor.counters >> 61) as u32, - page_count: descriptor.page_count as u32, - ..self - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/initdata.rs b/drivers/gpu/drm/asahi-neo/g17/fw/initdata.rs deleted file mode 100644 index 94b03ebe284dba..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/initdata.rs +++ /dev/null @@ -1,2369 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! G17 firmware initialization data. -//! -//! Each of the two firmware instances (primary and secondary) is handed one -//! root at boot. The roots name the objects the firmware needs before any -//! work is submitted: -//! -//! - region A and region C, shared by both instances; -//! - one main configuration per instance, holding its channel table; -//! - the hardware-data bundle, one allocation holding the hardware data, the -//! power configuration, the work rings and both main configurations with -//! their device-control rings; -//! - per-instance status blocks inside the private cluster, the fixed-address -//! arena the firmware also writes at fixed offsets. -//! -//! This module defines those layouts and the pure functions that fill them. -//! Allocation, mapping and publication belong to the builder. Initialization -//! requires exclusive access before either firmware instance can access the -//! object. These constructors must not be used to update a published object: -//! its live fields require individual accesses of the widths documented below. -//! -//! All objects start out zeroed. An `unk_` field documents the value -//! the firmware requires there; an `unk_` field without a doc comment is zero. - -use kernel::prelude::*; - -use super::clear; - -/// Interface version at root `+0x00`; the firmware rejects any other. -pub(crate) const INTERFACE_VERSION: [u16; 4] = [0x04c0, 0x0396, 0xa322, 0x0c8a]; - -/// UAT page size, as a shift. -pub(crate) const PAGE_BITS: u32 = 14; -/// UAT page size. -pub(crate) const PAGE_SIZE: usize = 1 << PAGE_BITS; -/// Page-table levels below the root. -const UAT_LEVELS: usize = 3; -/// Address bits resolved by one page-table level. -const BITS_PER_LEVEL: u32 = PAGE_BITS - 3; - -/// Firmware instance, encoded in its root. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum Role { - /// Runs the scheduler, the device-control ring and all work channels. - Primary = 0, - /// Control-only peer of the primary. - Secondary = 1, -} - -// --------------------------------------------------------------------------- -// Roots -// --------------------------------------------------------------------------- - -/// One UAT page-table level, as the firmware walks it. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct LevelDescriptor { - /// Required: 8. - pub(crate) unk_0: u8, - /// Page size shift of the tables at this level. - pub(crate) table_page_bits: u8, - /// Page size shift of the pages they map. - pub(crate) page_bits: u8, - /// Lowest input-address bit indexing this level. - pub(crate) index_shift: u8, - /// Entries per table. - pub(crate) entries: u16, - /// Table size in bytes. - pub(crate) table_size: u16, - /// Required: 1. - pub(crate) unk_8: u64, - /// Output-address bits of an entry. - pub(crate) phys_mask: u64, - /// Input-address bits indexing this level. - pub(crate) index_mask: u64, -} - -// SAFETY: `LevelDescriptor` consists of integers and arrays of integers only. -unsafe impl Zeroable for LevelDescriptor {} - -static_assert!(size_of::() == 0x20); - -impl LevelDescriptor { - /// Returns the descriptor of `level` (0 = top) for a UAT with `ias` - /// input-address and `oas` output-address bits. - const fn new(level: u32, ias: u32, oas: u32) -> Self { - let index_shift = PAGE_BITS + (UAT_LEVELS as u32 - 1 - level) * BITS_PER_LEVEL; - let index_bits = if level == 0 { - ias - index_shift - } else { - BITS_PER_LEVEL - }; - let entries = 1u64 << index_bits; - Self { - unk_0: 8, - table_page_bits: PAGE_BITS as u8, - page_bits: PAGE_BITS as u8, - index_shift: index_shift as u8, - entries: entries as u16, - table_size: PAGE_SIZE as u16, - unk_8: 1, - phys_mask: ((1u64 << oas) - 1) & !(PAGE_SIZE as u64 - 1), - index_mask: (entries - 1) << index_shift, - } - } -} - -/// Addresses and geometry common to both roots. -pub(crate) struct RootArgs { - /// Region A. - pub(crate) region_a_va: u64, - /// Region C. - pub(crate) region_c_va: u64, - /// This instance's main configuration. - pub(crate) main_config_va: u64, - /// This instance's status A block. - pub(crate) status_a_va: u64, - /// UAT input-address bits of the firmware's context. - pub(crate) ias: u32, - /// UAT output-address bits. - pub(crate) oas: u32, -} - -/// Root of the primary instance; the secondary extends it ([`SecondaryRoot`]). -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct Root { - /// [`INTERFACE_VERSION`]. - pub(crate) version: [u16; 4], - /// Region A. - pub(crate) region_a_va: u64, - pub(crate) unk_10: u64, - /// Main configuration of this instance. - pub(crate) main_config_va: u64, - /// Region C. - pub(crate) region_c_va: u64, - /// [`Role`] of this instance. - pub(crate) role: u32, - /// Required: 1. - pub(crate) unk_2c: u32, - /// UAT page size. - pub(crate) page_size: u16, - /// UAT page size shift. - pub(crate) page_bits: u8, - /// Number of page-table levels. - pub(crate) levels: u8, - /// Page-table levels, top first. - pub(crate) level: [LevelDescriptor; UAT_LEVELS], - pub(crate) unk_94: [u8; 0x14], - /// Status A block of this instance. - pub(crate) status_a_va: u64, - /// Primary status B ([`PrimaryStatusB`]); zero for the secondary. - pub(crate) status_b_va: u64, -} - -// SAFETY: `Root` consists of integers and arrays of integers only. -unsafe impl Zeroable for Root {} - -static_assert!(core::mem::offset_of!(Root, role) == 0x28); -static_assert!(core::mem::offset_of!(Root, level) == 0x34); -static_assert!(core::mem::offset_of!(Root, status_a_va) == 0xa8); -static_assert!(size_of::() == 0xb8); - -impl Root { - fn new(role: Role, args: &RootArgs, status_b_va: u64) -> Self { - Self { - version: INTERFACE_VERSION, - region_a_va: args.region_a_va, - unk_10: 0, - main_config_va: args.main_config_va, - region_c_va: args.region_c_va, - role: role as u32, - unk_2c: 1, - page_size: PAGE_SIZE as u16, - page_bits: PAGE_BITS as u8, - levels: UAT_LEVELS as u8, - level: [ - LevelDescriptor::new(0, args.ias, args.oas), - LevelDescriptor::new(1, args.ias, args.oas), - LevelDescriptor::new(2, args.ias, args.oas), - ], - unk_94: [0; 0x14], - status_a_va: args.status_a_va, - status_b_va, - } - } - - /// Returns the primary root. - pub(crate) fn primary(args: &RootArgs, status_b_va: u64) -> Self { - Self::new(Role::Primary, args, status_b_va) - } -} - -/// Root of the secondary instance. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct SecondaryRoot { - /// Common part; the secondary has no status B. - pub(crate) root: Root, - /// The primary's [`PowerState`]. - pub(crate) power_state_va: u64, - /// The secondary's [`SecondaryConfig`]. - pub(crate) config_va: u64, -} - -// SAFETY: `SecondaryRoot` consists of integers and arrays of integers only. -unsafe impl Zeroable for SecondaryRoot {} - -static_assert!(size_of::() == 0xc8); - -impl SecondaryRoot { - /// Returns the secondary root. - pub(crate) fn new(args: &RootArgs, power_state_va: u64, config_va: u64) -> Self { - Self { - root: Root::new(Role::Secondary, args, 0), - power_state_va, - config_va, - } - } -} - -/// Offset of the secondary root from the primary root. Both live in one -/// allocation, and the firmware relies on this distance. -pub(crate) const SECONDARY_ROOT_OFFSET: usize = 0x8000; -/// Size of the allocation holding both roots. -pub(crate) const ROOTS_SIZE: usize = 0xc000; - -// --------------------------------------------------------------------------- -// Channels -// --------------------------------------------------------------------------- - -/// Work channels: TA, 3D and compute at each of four priorities. -pub(crate) const WORK_CHANNELS: usize = 12; -/// Engines with a work channel at each priority (TA, 3D, compute). -const WORK_ENGINES: usize = 3; -/// Work priorities. -const WORK_PRIORITIES: usize = WORK_CHANNELS / WORK_ENGINES; -/// Channel-table index of the device-control channel. -pub(crate) const CONTROL_CHANNEL: usize = 12; -/// Channel-table index of the event and firmware-log rings. -pub(crate) const EVENT_CHANNEL: usize = 13; -/// Channel-table index of the trace and statistics rings. -pub(crate) const TRACE_CHANNEL: usize = 14; -/// Channel-table index of the channel with a single state address. -const CHANNEL_15: usize = 15; -/// Entries in the channel table. -pub(crate) const CHANNELS: usize = 17; - -/// One channel-table entry: four GPU addresses whose meaning depends on the -/// channel. Work and control channels name three counters of their -/// [`ChannelState`] and their ring; the event and trace channels name two -/// state/ring pairs. -#[repr(C)] -pub(crate) struct ChannelEntry(pub(crate) [u64; 4]); - -// SAFETY: `ChannelEntry` is an array of integers. -unsafe impl Zeroable for ChannelEntry {} - -static_assert!(size_of::() == 0x20); - -impl ChannelEntry { - /// Returns the entry of a work or control channel. - const fn ring(state_va: u64, ring_va: u64) -> Self { - Self([ - state_va + ChannelState::CONSUMER as u64, - state_va + ChannelState::UNK_10 as u64, - state_va + ChannelState::PRODUCER as u64, - ring_va, - ]) - } -} - -/// State of a work or control channel in an instance's state grid. -/// -/// All three counters are live words. The firmware advances `consumer` and -/// `unk_10`; the host writes `producer` with single aligned stores, after the -/// records it publishes are complete. At boot the host stores -/// zero to the control channel's consumer, then zero to `unk_10`, then -/// [`OPENING_PRODUCER`] to its producer, primary instance before secondary, -/// after the rest of the initialization data is written. A full barrier follows -/// both instances' stores. These creation-time writes occur before publication; the firmware moves -/// all three to that value once it has drained the opening record. -#[repr(C)] -pub(crate) struct ChannelState { - /// Records the firmware has consumed. - pub(crate) consumer: u32, - pub(crate) unk_4: [u8; 0xc], - /// Firmware-owned. - pub(crate) unk_10: u32, - pub(crate) unk_14: [u8; 0xc], - /// Records the host has published. - pub(crate) producer: u32, - pub(crate) unk_24: [u8; 0x1c], -} - -// SAFETY: `ChannelState` consists of integers and arrays of integers only. -unsafe impl Zeroable for ChannelState {} - -static_assert!(core::mem::offset_of!(ChannelState, consumer) == 0); -static_assert!(core::mem::offset_of!(ChannelState, unk_10) == 0x10); -static_assert!(core::mem::offset_of!(ChannelState, producer) == 0x20); -static_assert!(size_of::() == 0x40); - -impl ChannelState { - pub(crate) const CONSUMER: usize = core::mem::offset_of!(Self, consumer); - pub(crate) const UNK_10: usize = core::mem::offset_of!(Self, unk_10); - /// Offset of [`Self::producer`] in the block. - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); -} - -/// Offset of the device-control channel's state in an instance's state grid. -pub(crate) const CONTROL_STATE_OFFSET: usize = WORK_CHANNELS * size_of::(); - -/// Producer value of the device-control ring of either instance at boot: one -/// opening record. -pub(crate) const OPENING_PRODUCER: u32 = 1; - -/// Position of work channel `channel` in the state grid and in the work-ring -/// array: grouped by engine, priorities in descending order. -const fn work_slot(channel: usize) -> usize { - let engine = channel % WORK_ENGINES; - let priority = channel / WORK_ENGINES; - engine * WORK_PRIORITIES + (WORK_PRIORITIES - 1 - priority) -} - -/// Offset of work channel `channel`'s state in the state grid. -pub(crate) const fn work_state_offset(channel: usize) -> usize { - work_slot(channel) * size_of::() -} - -/// Bundle offset of work channel `channel`'s ring. -pub(crate) const fn work_ring_offset(channel: usize) -> usize { - bundle::WORK_RINGS + work_slot(channel) * bundle::WORK_RING_SIZE -} - -/// Offsets in a status area, the region starting at an instance's status A -/// block, of the rings the firmware fills. -pub(crate) mod status { - /// Event ring state. - pub(crate) const EVENT_STATE: u64 = 0x40; - /// Firmware-log state. - pub(crate) const LOG_STATE: u64 = 0x80; - /// Trace ring state. - pub(crate) const TRACE_STATE: u64 = 0x240; - /// Statistics ring state. - pub(crate) const STATS_STATE: u64 = 0x280; - /// Event ring. - pub(crate) const EVENT_RING: u64 = 0x2c0; - /// Firmware-log ring. - pub(crate) const LOG_RING: u64 = 0x4ac0; - /// The single address in channel 15. - pub(crate) const UNK_2D2C0: u64 = 0x2d2c0; - /// Trace ring. - pub(crate) const TRACE_RING: u64 = 0xa6ac0; - /// Statistics ring. - pub(crate) const STATS_RING: u64 = 0xafac0; -} - -// --------------------------------------------------------------------------- -// Hardware-data bundle layout -// --------------------------------------------------------------------------- - -/// Offsets in the hardware-data bundle. -pub(crate) mod bundle { - /// Size of the bundle allocation. - pub(crate) const SIZE: usize = 0x28000; - /// The [`super::HwData`]. - pub(crate) const HW_DATA: usize = 0; - /// The five views named by the primary main configuration. - pub(crate) const VIEWS: [usize; 5] = [0x2740, 0x3380, 0x4400, 0xbc80, 0xbd00]; - /// The [`super::PowerConfig`], inside view 2. - pub(crate) const POWER_CONFIG: usize = 0x5240; - /// The [`super::HwDataAux`]. - pub(crate) const AUX: usize = 0x8000; - /// A second copy of the [`super::PowerConfigHead`], inside views 3 and 4. - pub(crate) const POWER_CONFIG_COPY: usize = 0xd240; - /// Region the firmware writes, named twice by each main configuration. - pub(crate) const FIRMWARE_AREA: usize = 0xc500; - /// The work rings, see [`super::work_ring_offset`]. - pub(crate) const WORK_RINGS: usize = 0xc5c0; - /// Size of one work ring (256 slots of 0x18 bytes). - pub(crate) const WORK_RING_SIZE: usize = 0x1800; - /// Main configuration of each instance, each followed by its - /// device-control ring. - pub(crate) const MAIN_CONFIG: [usize; 2] = [0x1e5c0, 0x22a80]; - /// Separately allocated pages the firmware finds at fixed offsets from - /// the bundle: a zeroed page named by the primary main configuration. - pub(crate) const ZERO_PAGE: usize = 0x40000; - /// The two completion rings ([`super::CompletionRing`]). - pub(crate) const COMPLETION_RINGS: [usize; 2] = [0x48000, 0x70000]; -} - -// --------------------------------------------------------------------------- -// Main configuration -// --------------------------------------------------------------------------- - -/// Record the host stages in slot 0 of each instance's device-control ring -/// before boot. -#[repr(C)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct OpeningRecord { - /// Opcode: [`Self::PRIMARY`] or [`Self::SECONDARY`]. - pub(crate) opcode: u32, - pub(crate) unk_4: [u8; 0x3c], -} - -// SAFETY: `OpeningRecord` consists of integers and arrays of integers only. -unsafe impl Zeroable for OpeningRecord {} - -static_assert!(size_of::() == 0x40); - -impl OpeningRecord { - const PRIMARY: u32 = 0x16; - const SECONDARY: u32 = 0x2a; - - /// Returns the opening record of `role`. - pub(crate) const fn new(role: Role) -> Self { - Self { - opcode: match role { - Role::Primary => Self::PRIMARY, - Role::Secondary => Self::SECONDARY, - }, - unk_4: [0; 0x3c], - } - } -} - -/// Addresses both main configurations need. -pub(crate) struct MainConfigArgs { - /// Hardware-data bundle. - pub(crate) bundle_va: u64, - /// This main configuration. - pub(crate) main_config_va: u64, - /// This instance's state grid. - pub(crate) state_grid_va: u64, - /// This instance's status A block. - pub(crate) status_a_va: u64, - /// Context-0 alias of the parameter-buffer descriptor table. - pub(crate) pb_descriptors_low_va: u64, - /// Context-0 alias of the page-pool descriptor table. - pub(crate) page_pool_descriptors_low_va: u64, -} - -/// Addresses only the primary main configuration names. -pub(crate) struct PrimaryViews { - /// The zeroed page at [`bundle::ZERO_PAGE`]. - pub(crate) zero_page_va: u64, - /// Parameter-buffer descriptor table. - pub(crate) pb_descriptors_va: u64, - /// page-pool descriptor table. - pub(crate) page_pool_descriptors_va: u64, -} - -/// Main configuration of one instance. Its device-control ring -/// (256 records) follows it directly. -#[repr(C, packed)] -pub(crate) struct MainConfig { - /// Hardware-data bundle ([`HwData`]). - pub(crate) hw_data_va: u64, - /// [`bundle::FIRMWARE_AREA`]. - pub(crate) firmware_area_va: [u64; 2], - pub(crate) unk_18: [u8; 8], - /// Channel table. - pub(crate) channels: [ChannelEntry; CHANNELS], - pub(crate) unk_240: [u8; 0x10], - /// Non-zero enables firmware log output; the driver leaves it off. - pub(crate) log_enable: u32, - /// The bundle views ([`bundle::VIEWS`]); primary only. - pub(crate) view_va: [u64; 5], - pub(crate) unk_27c: [u8; 0x54], - /// [`PrimaryViews::zero_page_va`]; primary only. - pub(crate) zero_page_va: u64, - /// Context-0 alias of the parameter-buffer descriptor table. - pub(crate) pb_descriptors_low_va: u64, - /// Parameter-buffer descriptor table; primary only. - pub(crate) pb_descriptors_va: u64, - /// Context-0 alias of the page-pool descriptor table. - pub(crate) page_pool_descriptors_low_va: u64, - /// page-pool descriptor table; primary only. - pub(crate) page_pool_descriptors_va: u64, - pub(crate) unk_2f8: [u8; 8], - /// Required: 4 for the secondary. - pub(crate) unk_300: u32, - pub(crate) unk_304: [u8; 0x28], - /// Required: 1 for the primary. - pub(crate) unk_32c: u32, - pub(crate) unk_330: [u8; 0x14], - /// Required: 0xabcd_abcd for the primary. - pub(crate) unk_344: u32, - pub(crate) unk_348: [u8; 0x98], - /// Required: 0xff for the primary. - pub(crate) unk_3e0: u32, - pub(crate) unk_3e4: [u8; 0x8d], - /// [`PowerConfig`]; secondary only. - pub(crate) power_config_va: u64, - pub(crate) unk_479: [u8; 0x47], -} - -static_assert!(core::mem::offset_of!(MainConfig, channels) == 0x20); -static_assert!(core::mem::offset_of!(MainConfig, log_enable) == 0x250); -static_assert!(core::mem::offset_of!(MainConfig, view_va) == 0x254); -static_assert!(core::mem::offset_of!(MainConfig, zero_page_va) == 0x2d0); -static_assert!(core::mem::offset_of!(MainConfig, unk_300) == 0x300); -static_assert!(core::mem::offset_of!(MainConfig, unk_3e0) == 0x3e0); -static_assert!(core::mem::offset_of!(MainConfig, power_config_va) == 0x471); -static_assert!(size_of::() == MainConfig::CONTROL_RING); -static_assert!( - bundle::MAIN_CONFIG[0] + MainConfig::CONTROL_RING + 256 * size_of::() - == bundle::MAIN_CONFIG[1] -); - -// SAFETY: `MainConfig` consists of integers and arrays of integers only. -unsafe impl Zeroable for MainConfig {} - -impl MainConfig { - /// Offset of the device-control ring from the main configuration. - pub(crate) const CONTROL_RING: usize = 0x4c0; - - fn init(&mut self, role: Role, args: &MainConfigArgs) { - clear(self); - let bundle = args.bundle_va; - let grid = args.state_grid_va; - let status_a = args.status_a_va; - self.hw_data_va = bundle + bundle::HW_DATA as u64; - self.firmware_area_va = [bundle + bundle::FIRMWARE_AREA as u64; 2]; - if role == Role::Primary { - for channel in 0..WORK_CHANNELS { - self.channels[channel] = ChannelEntry::ring( - grid + work_state_offset(channel) as u64, - bundle + work_ring_offset(channel) as u64, - ); - } - } - self.channels[CONTROL_CHANNEL] = ChannelEntry::ring( - grid + CONTROL_STATE_OFFSET as u64, - args.main_config_va + Self::CONTROL_RING as u64, - ); - self.channels[EVENT_CHANNEL] = ChannelEntry([ - status_a + status::EVENT_STATE, - status_a + status::EVENT_RING, - status_a + status::LOG_STATE, - status_a + status::LOG_RING, - ]); - self.channels[TRACE_CHANNEL] = ChannelEntry([ - status_a + status::TRACE_STATE, - status_a + status::TRACE_RING, - status_a + status::STATS_STATE, - status_a + status::STATS_RING, - ]); - self.channels[CHANNEL_15] = ChannelEntry([status_a + status::UNK_2D2C0, 0, 0, 0]); - self.pb_descriptors_low_va = args.pb_descriptors_low_va; - self.page_pool_descriptors_low_va = args.page_pool_descriptors_low_va; - } - - /// Initializes the primary main configuration. - pub(crate) fn init_primary(&mut self, args: &MainConfigArgs, views: &PrimaryViews) { - self.init(Role::Primary, args); - self.view_va = bundle::VIEWS.map(|offset| args.bundle_va + offset as u64); - self.zero_page_va = views.zero_page_va; - self.pb_descriptors_va = views.pb_descriptors_va; - self.page_pool_descriptors_va = views.page_pool_descriptors_va; - self.unk_32c = 1; - self.unk_344 = 0xabcd_abcd; - self.unk_3e0 = 0xff; - } - - /// Initializes the secondary main configuration. - pub(crate) fn init_secondary(&mut self, args: &MainConfigArgs) { - self.init(Role::Secondary, args); - self.unk_300 = 4; - self.power_config_va = args.bundle_va + bundle::POWER_CONFIG as u64; - } -} - -// --------------------------------------------------------------------------- -// Hardware data -// --------------------------------------------------------------------------- - -/// Slots in the hardware-data register map. -pub(crate) const REGISTER_MAP_SLOTS: usize = 53; - -/// One register window the firmware maps through the hardware data -/// (per-SoC configuration). -#[derive(Copy, Clone, Debug)] -pub(crate) struct RegisterWindow { - /// Register-map slot. - pub(crate) slot: usize, - /// Physical address of the window. - pub(crate) phys: u64, - /// GPU address of the window in the firmware MMIO aperture. - pub(crate) va: u64, - /// Size in bytes; need not be a multiple of the page size. - pub(crate) size: u32, - /// Value of [`RegisterMapEntry::unk_18`]. - pub(crate) unk_18: u64, - /// Value of [`RegisterMapEntry::unk_20`]. - pub(crate) unk_20: u32, -} - -impl RegisterWindow { - /// Returns the page-aligned physical address, GPU address and length the - /// window must be mapped with, or `None` if its physical and GPU - /// addresses differ within a page or the mapped span overflows. - pub(crate) fn mapping(&self) -> Option<(u64, u64, u64)> { - let mask = PAGE_SIZE as u64 - 1; - if self.phys & mask != self.va & mask { - return None; - } - let start = self.va & !mask; - let span = self - .va - .checked_add(self.size as u64)? - .checked_sub(start)? - .checked_add(mask)? - & !mask; - Some((self.phys & !mask, start, span)) - } -} - -/// Value of [`RegisterMapEntry::unk_20`] in a reserved register-map slot. -const RESERVED_SLOT_UNK_20: u32 = 2; - -/// One register-map slot of the hardware data. -#[repr(C)] -pub(crate) struct RegisterMapEntry { - /// Physical address of the window. - pub(crate) phys: u64, - /// GPU address of the window. - pub(crate) va: u64, - /// Window size in bytes. - pub(crate) total_size: u32, - /// Window size in bytes, again. - pub(crate) element_size: u32, - /// Per-window value, see [`RegisterWindow::unk_18`]. - pub(crate) unk_18: u64, - /// Per-window value; [`RESERVED_SLOT_UNK_20`] in a reserved slot. - pub(crate) unk_20: u32, - pub(crate) unk_24: u32, -} - -// SAFETY: `RegisterMapEntry` consists of integers and arrays of integers only. -unsafe impl Zeroable for RegisterMapEntry {} - -static_assert!(size_of::() == 0x28); - -impl RegisterMapEntry { - const fn window(window: &RegisterWindow) -> Self { - Self { - phys: window.phys, - va: window.va, - total_size: window.size, - element_size: window.size, - unk_18: window.unk_18, - unk_20: window.unk_20, - unk_24: 0, - } - } - - const fn reserved() -> Self { - Self { - phys: 0, - va: 0, - total_size: 0, - element_size: 0, - unk_18: 0, - unk_20: RESERVED_SLOT_UNK_20, - unk_24: 0, - } - } -} - -/// Performance states in each performance table. -pub(crate) const PERF_STATES: usize = 11; -/// Voltage lanes of a performance table. -const PERF_LANES: usize = 16; - -/// One GPU operating point, from the device tree, in performance-state -/// order. -#[derive(Copy, Clone, Debug)] -pub(crate) struct Opp { - /// Frequency in Hz. - pub(crate) freq_hz: u64, - /// GPU core voltage in µV. - pub(crate) volt_uv: u32, -} - -/// Per-SoC parts of the performance tables that the operating points do not -/// give. -pub(crate) struct PerfConfig { - /// Operating point of each state of table A. - pub(crate) states_a: [u8; PERF_STATES], - /// Operating point of each state of table B. - pub(crate) states_b: [u8; PERF_STATES], - /// Relative performance of each state of table A, in percent. - pub(crate) relative_a: [u32; PERF_STATES], - /// Relative performance of each state of table B, in percent. - pub(crate) relative_b: [u32; PERF_STATES], - /// Lowest SRAM voltage in mV; a state's SRAM voltage is its core voltage - /// raised to this floor. - pub(crate) sram_min_mv: u32, -} - -/// One performance table of the hardware data. -#[repr(C, packed)] -pub(crate) struct PerfGroup { - /// Required: 10. - pub(crate) unk_0: u32, - /// Frequency of each state in MHz. - pub(crate) freq_mhz: [u32; PERF_STATES], - pub(crate) unk_30: [u8; 0x14], - /// GPU core voltage of each state in mV, per lane. - pub(crate) core_mv: [[u32; PERF_LANES]; PERF_STATES], - pub(crate) unk_304: [u8; 0x140], - /// SRAM voltage of each state in mV, per lane. - pub(crate) sram_mv: [[u32; PERF_LANES]; PERF_STATES], -} - -// SAFETY: `PerfGroup` consists of integers and arrays of integers only. -unsafe impl Zeroable for PerfGroup {} - -static_assert!(core::mem::offset_of!(PerfGroup, core_mv) == 0x44); -static_assert!(core::mem::offset_of!(PerfGroup, sram_mv) == 0x444); -static_assert!(size_of::() == 0x704); - -/// Frequency and voltages of one performance state. -#[derive(Copy, Clone)] -struct PerfState { - freq_mhz: u32, - core_mv: u32, - sram_mv: u32, -} - -impl PerfGroup { - /// Fills the table. `lanes` is the number of voltage lanes holding each - /// state's voltages; the remaining lanes stay zero. - fn fill(&mut self, states: &[PerfState; PERF_STATES], lanes: usize) { - self.unk_0 = 10; - for (index, state) in states.iter().enumerate() { - self.freq_mhz[index] = state.freq_mhz; - for lane in 0..lanes { - self.core_mv[index][lane] = state.core_mv; - self.sram_mv[index][lane] = state.sram_mv; - } - } - } -} - -/// Returns the performance states selected by `map` from `opps`. -fn perf_states( - opps: &[Opp], - map: &[u8; PERF_STATES], - sram_min_mv: u32, -) -> Result<[PerfState; PERF_STATES]> { - let mut states = [PerfState { - freq_mhz: 0, - core_mv: 0, - sram_mv: 0, - }; PERF_STATES]; - for (state, &index) in states.iter_mut().zip(map) { - let opp = opps.get(index as usize).ok_or(EINVAL)?; - let core_mv = opp.volt_uv / 1000; - *state = PerfState { - freq_mhz: u32::try_from(opp.freq_hz / 1_000_000)?, - core_mv, - sram_mv: core_mv.max(sram_min_mv), - }; - } - Ok(states) -} - -/// One completion ring the firmware fills. -#[repr(C)] -pub(crate) struct CompletionRingDescriptor { - /// Context-0 alias of the ring. - pub(crate) low_va: u64, - /// Firmware alias of the ring. - pub(crate) high_va: u64, - /// Entries in the ring. - pub(crate) entries: u32, - /// Size of one entry. - pub(crate) entry_size: u32, - /// Index of this descriptor in the table. - pub(crate) index: u32, - /// Firmware-owned. - pub(crate) unk_1c: u32, -} - -// SAFETY: `CompletionRingDescriptor` consists of integers and arrays of integers only. -unsafe impl Zeroable for CompletionRingDescriptor {} - -static_assert!(size_of::() == 0x20); - -/// Completion-ring descriptors in the hardware data; only -/// [`COMPLETION_RING_INDEX`] are used. -const COMPLETION_RING_DESCRIPTORS: usize = 4; -/// Descriptor index of each completion ring. -pub(crate) const COMPLETION_RING_INDEX: [u32; 2] = [0, 2]; -/// Entries in a completion ring. -const COMPLETION_RING_ENTRIES: u32 = 0x800; -/// Size of a completion-ring entry. -const COMPLETION_RING_ENTRY_SIZE: u32 = 0x40; -/// Size of a completion ring. -pub(crate) const COMPLETION_RING_SIZE: usize = - (COMPLETION_RING_ENTRIES * COMPLETION_RING_ENTRY_SIZE) as usize; - -/// The two aliases of one completion ring. -#[derive(Copy, Clone, Debug)] -pub(crate) struct CompletionRing { - /// Context-0 alias. - pub(crate) low_va: u64, - /// Firmware alias. - pub(crate) high_va: u64, -} - -/// Inputs of the hardware data. -pub(crate) struct HwDataArgs<'a> { - /// Chip identifier. - pub(crate) chip_id: u32, - /// Mapped register windows. - pub(crate) windows: &'a [RegisterWindow], - /// Register-map slots that are reserved but unmapped. - pub(crate) reserved_slots: &'a [usize], - /// GPU operating points, in performance-state order. - pub(crate) opps: &'a [Opp], - /// Per-SoC parts of the performance tables. - pub(crate) perf: &'a PerfConfig, - /// The completion rings, in [`COMPLETION_RING_INDEX`] order. - pub(crate) completion_rings: [CompletionRing; 2], - /// The zeroed QoS object ([`QOS_SIZE`] bytes). - pub(crate) qos_va: u64, -} - -/// Size of the QoS object named by the hardware data. -pub(crate) const QOS_SIZE: usize = 0xe40; - -/// The hardware data, at the base of the bundle and shared by both -/// instances. -#[repr(C, packed)] -pub(crate) struct HwData { - /// Required: 0x6f_0000_0000. - pub(crate) unk_0: u64, - /// Required: 0xffc0_0000. - pub(crate) unk_8: u64, - /// Required: 0x10_0000_0000. - pub(crate) unk_10: u64, - /// Required: 0x10_0000_0000. - pub(crate) unk_18: u64, - /// Required: 0x2ff_ffff_8000. - pub(crate) unk_20: u64, - /// Required: 0xffff_fc21_8140_0000, a GPU address in the firmware MMIO - /// aperture. - pub(crate) unk_28: u64, - pub(crate) unk_30: [u8; 0xb0], - /// Required: see [`HwData::UNK_E0`]. - pub(crate) unk_e0: [u16; 24], - pub(crate) unk_110: [u8; 0x1c8], - /// Required: see [`HwData::UNK_2D8`]. - pub(crate) unk_2d8: [u16; 12], - pub(crate) unk_2f0: [u8; 0xf0], - /// Required: see [`HwData::UNK_3E0`]. - pub(crate) unk_3e0: [u16; 24], - pub(crate) unk_410: [u8; 0x1c8], - /// Required: see [`HwData::UNK_5D8`]. - pub(crate) unk_5d8: [u16; 12], - pub(crate) unk_5f0: [u8; 0x50], - /// Register windows the firmware maps. - pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], - pub(crate) unk_e88: [u8; 8], - /// Chip identifier. - pub(crate) chip_id: u32, - /// Required: 1. - pub(crate) unk_e94: u32, - /// Required: 1. - pub(crate) unk_e98: u32, - pub(crate) unk_e9c: [u8; 4], - /// Required: 1. - pub(crate) unk_ea0: u32, - /// Required: 1. - pub(crate) unk_ea4: u32, - pub(crate) unk_ea8: [u8; 0x10], - /// Required: 1. - pub(crate) unk_eb8: u32, - pub(crate) unk_ebc: [u8; 4], - /// Required: 1. - pub(crate) unk_ec0: u32, - pub(crate) unk_ec4: [u8; 4], - /// Required: 1. - pub(crate) unk_ec8: u32, - pub(crate) unk_ecc: [u8; 4], - /// Required: 0x5dc0 (24000). - pub(crate) unk_ed0: u32, - /// Required: 1. - pub(crate) unk_ed4: u32, - /// Required: 0x10 (16). - pub(crate) unk_ed8: u32, - pub(crate) unk_edc: [u8; 4], - /// Required: 1 in every entry. - pub(crate) unk_ee0: [u32; 3], - pub(crate) unk_eec: [u8; 0x18], - /// Required: 0x1f (31). - pub(crate) unk_f04: u32, - pub(crate) unk_f08: [u8; 0x1c], - /// Required: 4. - pub(crate) unk_f24: u32, - pub(crate) unk_f28: [u8; 0xc], - /// Required: 1. - pub(crate) unk_f34: u32, - /// Required: 1. - pub(crate) unk_f38: u32, - pub(crate) unk_f3c: [u8; 0x10], - /// Required: 0x31 (49). - pub(crate) unk_f4c: u32, - pub(crate) unk_f50: [u8; 0x1c], - /// Required: 1. - pub(crate) unk_f6c: u32, - pub(crate) unk_f70: [u8; 0x18], - /// Required: 6. - pub(crate) unk_f88: u32, - /// Required: 1. - pub(crate) unk_f8c: u32, - pub(crate) unk_f90: [u8; 0x1c], - /// Required: 1. - pub(crate) unk_fac: u32, - pub(crate) unk_fb0: [u8; 8], - /// Required: 0x1e (30). - pub(crate) unk_fb8: u32, - /// Required: 4. - pub(crate) unk_fbc: u32, - /// Required: 6. - pub(crate) unk_fc0: u32, - /// Performance table A. - pub(crate) perf_a: PerfGroup, - pub(crate) unk_16c8: [u8; 0x140], - /// Frequency of each state of table B in MHz. - pub(crate) perf_b_freq_mhz: [u32; PERF_STATES], - pub(crate) unk_1834: [u8; 0x14], - /// Required: 0x3f82_8f5c (1.02 as an IEEE-754 single) in every entry. - pub(crate) unk_1848: [u32; 11], - pub(crate) unk_1874: [u8; 0x54], - /// Relative performance of each state of table A, in percent. - pub(crate) perf_a_relative: [u32; PERF_STATES], - pub(crate) unk_18f4: [u8; 0x14], - /// Relative performance of each state of table B, in percent. - pub(crate) perf_b_relative: [u32; PERF_STATES], - pub(crate) unk_1934: [u8; 0x94], - /// Operating point of each state of table A. - pub(crate) perf_a_states: [u32; PERF_STATES], - pub(crate) unk_19f4: [u8; 0x14], - /// Operating point of each state of table B. - pub(crate) perf_b_states: [u32; PERF_STATES], - pub(crate) unk_1a34: [u8; 0x2a4], - /// Performance table B. Each voltage is stored in lane 0 only. - pub(crate) perf_b: PerfGroup, - pub(crate) unk_23dc: [u8; 0x168], - /// Required: 4. - pub(crate) unk_2544: u32, - /// Required: 6. - pub(crate) unk_2548: u32, - /// Required: 3. - pub(crate) unk_254c: u32, - /// Required: 7. - pub(crate) unk_2550: u32, - /// Required: 7. - pub(crate) unk_2554: u32, - pub(crate) unk_2558: [u8; 0x10], - /// Required: 5. - pub(crate) unk_2568: u32, - pub(crate) unk_256c: [u8; 4], - /// Required: 1. - pub(crate) unk_2570: u32, - pub(crate) unk_2574: [u8; 0x1a], - /// Required: [`WORK_CHANNELS`]. - pub(crate) unk_258e: u16, - /// Required: 0, 1, 1, 1. - pub(crate) unk_2590: [u32; 4], - /// Required: 0, 1, 1, 1. - pub(crate) unk_25a0: [u32; 4], - pub(crate) unk_25b0: [u8; 4], - /// Required: 0xffff_ffff in every entry, one per work channel. - pub(crate) unk_25b4: [u32; WORK_CHANNELS], - pub(crate) unk_25e4: [u8; 0x10], - /// Required: 1. - pub(crate) unk_25f4: u32, - pub(crate) unk_25f8: [u8; 8], - /// Required: 1. - pub(crate) unk_2600: u32, - pub(crate) unk_2604: [u8; 4], - /// Required: 1. - pub(crate) unk_2608: u32, - pub(crate) unk_260c: [u8; 4], - /// Required: 0x100. - pub(crate) unk_2610: u32, - /// Completion-ring descriptors. - pub(crate) completion_rings: [CompletionRingDescriptor; COMPLETION_RING_DESCRIPTORS], - pub(crate) unk_2694: [u8; 0x18], - /// QoS object ([`QOS_SIZE`] bytes, zeroed). - pub(crate) qos_va: u64, - /// Required: 1. - pub(crate) unk_26b4: u32, - /// Required: 1. - pub(crate) unk_26b8: u32, - pub(crate) unk_26bc: [u8; 0x2c], - /// Required: 1. - pub(crate) unk_26e8: u32, - pub(crate) unk_26ec: [u8; 0x16ac], - /// Required: 0xffff_ffff. - pub(crate) unk_3d98: u32, - pub(crate) unk_3d9c: [u8; 0x14], - /// Required: 0xffff_ffff. - pub(crate) unk_3db0: u32, -} - -static_assert!(core::mem::offset_of!(HwData, register_map) == 0x640); -static_assert!(core::mem::offset_of!(HwData, chip_id) == 0xe90); -static_assert!(core::mem::offset_of!(HwData, perf_a) == 0xfc4); -static_assert!(core::mem::offset_of!(HwData, perf_b_freq_mhz) == 0x1808); -static_assert!(core::mem::offset_of!(HwData, perf_b) == 0x1cd8); -static_assert!(core::mem::offset_of!(HwData, unk_258e) == 0x258e); -static_assert!(core::mem::offset_of!(HwData, completion_rings) == 0x2614); -static_assert!(core::mem::offset_of!(HwData, qos_va) == 0x26ac); -static_assert!(size_of::() == 0x3db4); -static_assert!(bundle::HW_DATA + size_of::() <= bundle::VIEWS[2]); - -// SAFETY: `HwData` consists of integers and arrays of integers only. -unsafe impl Zeroable for HwData {} - -impl HwData { - const UNK_E0: [u16; 24] = [ - 0x2008, 0, 0, 0, 0, 0x2008, 0, 0, 0, 0, 0x2008, 0, 0x24cb, 0, 0x2cfa, 0xc98a, 0x24cb, - 0xf4f6, 0xe917, 0x1877, 0x24cb, 0x38d9, 0, 0xbdab, - ]; - const UNK_2D8: [u16; 12] = [ - 0x2000, 0xfff8, 0x2cdb, 0xd32d, 0x2000, 0xf500, 0xe926, 0x21da, 0x2000, 0x38b6, 8, 0xc742, - ]; - const UNK_3E0: [u16; 24] = [ - 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0, 0, 0, 0x7fe0, 0, 0x8000, 0, 0, 0, 0, 0x8000, 0, 0, 0, 0, - 0x8000, 0, - ]; - const UNK_5D8: [u16; 12] = [ - 0x2645, 0x4b23, 0xe98, 0, 0xea5f, 0xd5a2, 0x3fff, 0x4000, 0x4000, 0xca5e, 0xf5a2, 0x4000, - ]; - - /// Initializes the hardware data. - pub(crate) fn init(&mut self, args: &HwDataArgs<'_>) -> Result { - clear(self); - self.unk_0 = 0x6f_0000_0000; - self.unk_8 = 0xffc0_0000; - self.unk_10 = 0x10_0000_0000; - self.unk_18 = 0x10_0000_0000; - self.unk_20 = 0x2ff_ffff_8000; - self.unk_28 = 0xffff_fc21_8140_0000; - self.unk_e0 = Self::UNK_E0; - self.unk_2d8 = Self::UNK_2D8; - self.unk_3e0 = Self::UNK_3E0; - self.unk_5d8 = Self::UNK_5D8; - - let mut used = [false; REGISTER_MAP_SLOTS]; - let mut claim = |slot: usize| -> Result { - match used.get_mut(slot) { - Some(used) if !*used => { - *used = true; - Ok(()) - } - _ => Err(EINVAL), - } - }; - for window in args.windows { - claim(window.slot)?; - self.register_map[window.slot] = RegisterMapEntry::window(window); - } - for &slot in args.reserved_slots { - claim(slot)?; - self.register_map[slot] = RegisterMapEntry::reserved(); - } - - self.chip_id = args.chip_id; - self.unk_e94 = 1; - self.unk_e98 = 1; - self.unk_ea0 = 1; - self.unk_ea4 = 1; - self.unk_eb8 = 1; - self.unk_ec0 = 1; - self.unk_ec8 = 1; - self.unk_ed0 = 0x5dc0; - self.unk_ed4 = 1; - self.unk_ed8 = 0x10; - self.unk_ee0 = [1; 3]; - self.unk_f04 = 0x1f; - self.unk_f24 = 4; - self.unk_f34 = 1; - self.unk_f38 = 1; - self.unk_f4c = 0x31; - self.unk_f6c = 1; - self.unk_f88 = 6; - self.unk_f8c = 1; - self.unk_fac = 1; - self.unk_fb8 = 0x1e; - self.unk_fbc = 4; - self.unk_fc0 = 6; - - let perf = args.perf; - let states_a = perf_states(args.opps, &perf.states_a, perf.sram_min_mv)?; - let states_b = perf_states(args.opps, &perf.states_b, perf.sram_min_mv)?; - self.perf_a.fill(&states_a, PERF_LANES); - self.perf_b.fill(&states_b, 1); - for (index, state) in states_b.iter().enumerate() { - self.perf_b_freq_mhz[index] = state.freq_mhz; - self.perf_a_states[index] = perf.states_a[index].into(); - self.perf_b_states[index] = perf.states_b[index].into(); - } - self.perf_a_relative = perf.relative_a; - self.perf_b_relative = perf.relative_b; - self.unk_1848 = [0x3f82_8f5c; 11]; - self.unk_2544 = 4; - self.unk_2548 = 6; - self.unk_254c = 3; - self.unk_2550 = 7; - self.unk_2554 = 7; - self.unk_2568 = 5; - self.unk_2570 = 1; - - self.unk_258e = WORK_CHANNELS as u16; - self.unk_2590 = [0, 1, 1, 1]; - self.unk_25a0 = [0, 1, 1, 1]; - self.unk_25b4 = [u32::MAX; WORK_CHANNELS]; - self.unk_25f4 = 1; - self.unk_2600 = 1; - self.unk_2608 = 1; - self.unk_2610 = 0x100; - for (ring, index) in args.completion_rings.iter().zip(COMPLETION_RING_INDEX) { - self.completion_rings[index as usize] = CompletionRingDescriptor { - low_va: ring.low_va, - high_va: ring.high_va, - entries: COMPLETION_RING_ENTRIES, - entry_size: COMPLETION_RING_ENTRY_SIZE, - index, - unk_1c: 0, - }; - } - self.qos_va = args.qos_va; - self.unk_26b4 = 1; - self.unk_26b8 = 1; - self.unk_26e8 = 1; - self.unk_3d98 = 0xffff_ffff; - self.unk_3db0 = 0xffff_ffff; - Ok(()) - } -} - -// --------------------------------------------------------------------------- -// Regions A and C -// --------------------------------------------------------------------------- - -/// Region A: a zeroed page both instances read. -#[repr(C)] -pub(crate) struct RegionA { - pub(crate) unk_0: [u8; 0x4000], -} - -// SAFETY: `RegionA` is an array of integers. -unsafe impl Zeroable for RegionA {} - -/// Engines whose work the firmware's context-switch timer may interrupt -/// (bit 0 TA, bit 1 3D, bit 2 compute). No engine is periodically resumed. -pub(crate) const CONTEXT_SWITCH_TIMER_MASK: u32 = 0; -/// Period of the context-switch timer, in units of the firmware's 1 ms -/// scheduling pass. -pub(crate) const CONTEXT_SWITCH_TIMER_MULTIPLIER: u8 = 1; - -/// Region C: firmware policy settings shared by both instances. -/// -/// Three fields are live. The host updates [`Self::effort`] while it submits -/// work: it stores zero to [`Self::unk_e8`], then the effort (0x10000, 1.0 -/// in 16.16 fixed point, while work is streaming; zero once the GPU has been -/// idle for 16 ms), each with one aligned 32-bit store, and then cleans the -/// cache line to the point of coherency. [`Self::power_assert_tally`] is -/// unaligned: for every device-control record whose opcode asserts GPU -/// power, the host adds one to it after writing the record and before -/// advancing the ring producer (with a barrier before the producer store). -/// The firmware powers the GPU cores up while the tally differs from its -/// own count of drained records. -#[repr(C, packed)] -pub(crate) struct RegionC { - /// Engines the context-switch timer applies to ([`CONTEXT_SWITCH_TIMER_MASK`]). - pub(crate) context_switch_timer_mask: u32, - pub(crate) unk_4: [u8; 0x20], - /// Required: 0xbb8 (3000). - pub(crate) unk_24: u32, - pub(crate) unk_28: [u8; 8], - /// Non-zero lets the firmware power the GPU cores off when idle. - pub(crate) idle_power_off: u32, - /// Required: 1. - pub(crate) unk_34: u32, - pub(crate) unk_38: [u8; 4], - /// Required: 0x78 (120). - pub(crate) unk_3c: u32, - pub(crate) unk_40: [u8; 0x14], - /// Required: 0xffff. - pub(crate) unk_54: u16, - /// Required: 0x28 (40). - pub(crate) unk_56: u16, - /// Required: 0xffff. - pub(crate) unk_58: u16, - pub(crate) unk_5a: [u8; 4], - /// Required: 1. - pub(crate) unk_5e: u32, - /// Required: 1. - pub(crate) unk_62: u32, - /// Required: 3. - pub(crate) unk_66: u32, - /// Period of the context-switch timer ([`CONTEXT_SWITCH_TIMER_MULTIPLIER`]). - pub(crate) context_switch_timer_multiplier: u8, - pub(crate) unk_6b: [u8; 0xd], - /// Required: 1 in every entry. - pub(crate) unk_78: [u32; 3], - /// Required: 0x254e (9550). - pub(crate) unk_84: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_88: u32, - /// Required: 0x3e8 (1000). - pub(crate) unk_8c: u32, - pub(crate) unk_90: [u8; 8], - /// Required: 0x3e8 (1000). - pub(crate) unk_98: u32, - /// Required: 0x3000_001d. - pub(crate) unk_9c: u32, - /// Required: 0x3100_0002. - pub(crate) unk_a0: u32, - pub(crate) unk_a4: [u8; 0x40], - /// Performance-floor request (live, see the type documentation). - pub(crate) effort: u32, - /// Live; the host stores zero here whenever it stores [`Self::effort`]. - pub(crate) unk_e8: u32, - pub(crate) unk_ec: [u8; 0x6d8], - /// Required: 0x7d0 (2000). - pub(crate) unk_7c4: u32, - /// Required: 0x3f80_0000 (1.0 as an IEEE-754 single). - pub(crate) unk_7c8: u32, - /// Required: 0x3f4c_cccd (0.8 as an IEEE-754 single). - pub(crate) unk_7cc: u32, - /// Required: 0x3e4c_cccd (0.2 as an IEEE-754 single). - pub(crate) unk_7d0: u32, - /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). - pub(crate) unk_7d4: u32, - /// Required: 0x3dcc_cccd (0.1 as an IEEE-754 single). - pub(crate) unk_7d8: u32, - /// Required: 0x3e80_0000 (0.25 as an IEEE-754 single). - pub(crate) unk_7dc: u32, - /// Required: 0x3f19_999a (0.6 as an IEEE-754 single). - pub(crate) unk_7e0: u32, - /// Required: 0x3f66_6666 (0.9 as an IEEE-754 single). - pub(crate) unk_7e4: u32, - /// Required: 6. - pub(crate) unk_7e8: u32, - /// Required: 1. - pub(crate) unk_7ec: u32, - /// Required: 1. - pub(crate) unk_7f0: u32, - /// Required: 0x64 (100). - pub(crate) unk_7f4: u32, - /// Required: 0x64 (100). - pub(crate) unk_7f8: u32, - pub(crate) unk_7fc: [u8; 0x19c], - /// Required: 0x28 (40). - pub(crate) unk_998: u32, - /// Required: 0xa (10). - pub(crate) unk_99c: u32, - /// Required: 0xfa (250). - pub(crate) unk_9a0: u32, - /// Required: 1. - pub(crate) unk_9a4: u32, - pub(crate) unk_9a8: [u8; 0x10], - /// Required: 2. - pub(crate) unk_9b8: u32, - /// Idle time before the firmware powers the GPU cores off. - pub(crate) idle_power_off_delay: u32, - /// Required: 5. - pub(crate) unk_9c0: u32, - pub(crate) unk_9c4: [u8; 4], - /// Required: 0x28 (40). - pub(crate) unk_9c8: u32, - /// Compute progress watchdog threshold (zero disables the watchdog). - pub(crate) compute_watchdog: u32, - /// Required: 1. - pub(crate) unk_9d0: u32, - pub(crate) unk_9d4: [u8; 0x449], - /// Required: 1. - pub(crate) unk_e1d: u32, - pub(crate) unk_e21: [u8; 0x18], - /// Required: 3. - pub(crate) unk_e39: u32, - /// Required: 1. - pub(crate) unk_e3d: u32, - pub(crate) unk_e41: [u8; 0x10], - /// Host power-assert tally (live, unaligned; see the type documentation). - pub(crate) power_assert_tally: u32, - pub(crate) unk_e55: [u8; 0x1ab], -} - -static_assert!(core::mem::offset_of!(RegionC, idle_power_off) == 0x30); -static_assert!(core::mem::offset_of!(RegionC, context_switch_timer_multiplier) == 0x6a); -static_assert!(core::mem::offset_of!(RegionC, effort) == 0xe4); -static_assert!(core::mem::offset_of!(RegionC, idle_power_off_delay) == 0x9bc); -static_assert!(core::mem::offset_of!(RegionC, compute_watchdog) == 0x9cc); -static_assert!(core::mem::offset_of!(RegionC, power_assert_tally) == 0xe51); -static_assert!(size_of::() == 0x1000); - -// SAFETY: `RegionC` consists of integers and arrays of integers only. -unsafe impl Zeroable for RegionC {} - -impl RegionC { - /// Offset of [`Self::effort`]. - pub(crate) const EFFORT: usize = core::mem::offset_of!(Self, effort); - /// Offset of [`Self::unk_e8`]. - pub(crate) const UNK_E8: usize = core::mem::offset_of!(Self, unk_e8); - /// Offset of [`Self::power_assert_tally`]. - pub(crate) const POWER_ASSERT_TALLY: usize = core::mem::offset_of!(Self, power_assert_tally); - - /// Initializes region C. - pub(crate) fn init(&mut self) { - clear(self); - self.context_switch_timer_mask = CONTEXT_SWITCH_TIMER_MASK; - self.unk_24 = 0xbb8; - self.idle_power_off = 1; - self.unk_34 = 1; - self.unk_3c = 0x78; - self.unk_54 = 0xffff; - self.unk_56 = 0x28; - self.unk_58 = 0xffff; - self.unk_5e = 1; - self.unk_62 = 1; - self.unk_66 = 3; - self.context_switch_timer_multiplier = CONTEXT_SWITCH_TIMER_MULTIPLIER; - self.unk_78 = [1; 3]; - self.unk_84 = 0x254e; - self.unk_88 = 0x3e8; - self.unk_8c = 0x3e8; - self.unk_98 = 0x3e8; - self.unk_9c = 0x3000_001d; - self.unk_a0 = 0x3100_0002; - self.unk_7c4 = 0x7d0; - self.unk_7c8 = 0x3f80_0000; - self.unk_7cc = 0x3f4c_cccd; - self.unk_7d0 = 0x3e4c_cccd; - self.unk_7d4 = 0x3f66_6666; - self.unk_7d8 = 0x3dcc_cccd; - self.unk_7dc = 0x3e80_0000; - self.unk_7e0 = 0x3f19_999a; - self.unk_7e4 = 0x3f66_6666; - self.unk_7e8 = 6; - self.unk_7ec = 1; - self.unk_7f0 = 1; - self.unk_7f4 = 0x64; - self.unk_7f8 = 0x64; - self.unk_998 = 0x28; - self.unk_99c = 0xa; - self.unk_9a0 = 0xfa; - self.unk_9a4 = 1; - self.unk_9b8 = 2; - self.idle_power_off_delay = 40; - self.unk_9c0 = 5; - self.unk_9c8 = 0x28; - self.compute_watchdog = 50; - self.unk_9d0 = 1; - self.unk_e1d = 1; - self.unk_e39 = 3; - self.unk_e3d = 1; - } -} - -// --------------------------------------------------------------------------- -// Status objects -// --------------------------------------------------------------------------- - -/// Status block of an instance (its status A, and the head of the primary -/// status B). -/// -/// At creation only `+0x04` is one. Firmware later owns the status words at -/// `+0x10` and `+0x14`; the host observes each with an aligned 32-bit read. -/// The event-ring state begins at status A `+0x40`, inside this boot view: -/// clearing or copying the whole status block after publication is forbidden. -#[repr(C)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct StatusBlock { - pub(crate) unk_0: u32, - /// Required: 1. - pub(crate) unk_4: u32, - pub(crate) unk_8: [u8; 0x78], -} - -// SAFETY: `StatusBlock` consists of integers and arrays of integers only. -unsafe impl Zeroable for StatusBlock {} - -static_assert!(size_of::() == 0x80); - -impl StatusBlock { - /// Returns a status block in its boot state. - pub(crate) const fn new() -> Self { - Self { - unk_0: 0, - unk_4: 1, - unk_8: [0; 0x78], - } - } -} - - - -/// Size of the firmware-control page named by the primary status B. -pub(crate) const FWCTL_SIZE: usize = 0x4000; -/// Offset of the firmware-control ring in its page. -const FWCTL_RING: u64 = 0x40; - -/// One firmware-written recovery-info entry. The host reads each word separately after the -/// recovery handshake reaches state 3; it never copies the live table as a structure. -#[repr(C)] -pub(crate) struct RecoveryInfoEntry { - pub(crate) flags: u32, - pub(crate) stamp: u32, -} -// SAFETY: Both fields are integers. -unsafe impl Zeroable for RecoveryInfoEntry {} -static_assert!(size_of::() == 8); - -/// Status B of the primary instance, directly after its state grid. -#[repr(C, packed)] -pub(crate) struct PrimaryStatusB { - /// Status block. - pub(crate) status: StatusBlock, - pub(crate) unk_80: [u8; 0x4038], - pub(crate) recovery_info: [RecoveryInfoEntry; 256], - pub(crate) unk_48b8: [u8; 0x20], - pub(crate) recovery_slot_mask: u64, - /// Firmware-control state, at the start of the firmware-control page. - pub(crate) fwctl_state_va: u64, - /// Firmware-control ring, in the firmware-control page. - pub(crate) fwctl_ring_va: u64, - /// Firmware-written epoch, read using an aligned 64-bit load. - pub(crate) recovery_epoch: u64, - pub(crate) unk_48f8: [u8; 8], - /// Firmware writes 1 and 3; host writes 2 and 0 with full barriers around each 32-bit store. - pub(crate) recovery_state: u32, - pub(crate) unk_4904: [u8; 0xc], - /// Must be zero for an event-backed restart; read using an aligned 32-bit load. - pub(crate) host_recovery: u32, - pub(crate) unk_4914: [u8; 0x9b20], - /// Required: 1. - pub(crate) unk_e434: u32, - pub(crate) unk_e438: [u8; 8], - /// Power state, see [`PowerState`]. - pub(crate) power: PowerState, -} - -static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_info) == 0x40b8); -static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_slot_mask) == 0x48d8); -static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_epoch) == 0x48f0); -static_assert!(core::mem::offset_of!(PrimaryStatusB, recovery_state) == 0x4900); -static_assert!(core::mem::offset_of!(PrimaryStatusB, host_recovery) == 0x4910); -static_assert!(core::mem::offset_of!(PrimaryStatusB, fwctl_state_va) == 0x48e0); -static_assert!(core::mem::offset_of!(PrimaryStatusB, unk_e434) == 0xe434); -static_assert!(core::mem::offset_of!(PrimaryStatusB, power) == PrimaryStatusB::POWER_STATE); -static_assert!(size_of::() == 0xeb00); - -// SAFETY: `PrimaryStatusB` consists of integers and arrays of integers only. -unsafe impl Zeroable for PrimaryStatusB {} - -impl PrimaryStatusB { - pub(crate) const RECOVERY_SLOT_MASK: usize = core::mem::offset_of!(Self, recovery_slot_mask); - pub(crate) const RECOVERY_INFO: usize = core::mem::offset_of!(Self, recovery_info); - pub(crate) const RECOVERY_STATE: usize = core::mem::offset_of!(Self, recovery_state); - pub(crate) const HOST_RECOVERY: usize = core::mem::offset_of!(Self, host_recovery); - - /// Offset of the [`PowerState`]. - pub(crate) const POWER_STATE: usize = 0xe440; - - /// Initializes the primary status B; `fwctl_va` is the zeroed - /// firmware-control page ([`FWCTL_SIZE`] bytes). - pub(crate) fn init(&mut self, fwctl_va: u64) { - clear(self); - self.status = StatusBlock::new(); - self.fwctl_state_va = fwctl_va; - self.fwctl_ring_va = fwctl_va + FWCTL_RING; - self.unk_e434 = 1; - self.power.init(); - } -} - - - -/// Offsets in the private cluster. -pub(crate) mod private { - /// Size of the private cluster. - pub(crate) const SIZE: usize = 0x178000; - /// The primary state grid. - pub(crate) const PRIMARY_STATE: usize = 0; - /// The primary status A block. - pub(crate) const PRIMARY_STATUS_A: usize = 0xee40; - /// The secondary status A block. - pub(crate) const SECONDARY_STATUS_A: usize = 0xc3100; - /// The secondary state grid. - pub(crate) const SECONDARY_STATE: usize = 0x1770c0; - /// [`super::PrimaryStatusB`] or [`super::SecondaryConfig`], from an - /// instance's state grid. - pub(crate) const STATUS_B: usize = 0x340; - /// Secondary state named by [`super::HwDataAux`], from the secondary - /// state grid. - pub(crate) const SECONDARY_STATUS: usize = 0x3c0; -} - -static_assert!(private::STATUS_B == CONTROL_STATE_OFFSET + size_of::()); -static_assert!( - private::PRIMARY_STATE + private::STATUS_B + size_of::() - == private::PRIMARY_STATUS_A -); -static_assert!(private::STATUS_B + size_of::() == private::SECONDARY_STATUS); - -// --------------------------------------------------------------------------- -// Power configuration and auxiliary hardware data (bundle) -// --------------------------------------------------------------------------- - -/// Die temperature sensors read by the die temperature limiter. -const TEMP_SENSOR_MASK: u64 = 0x82a; -/// Die temperature the limiter holds the GPU below, in 0.01 °C. -const TEMP_TARGET: u32 = 11_000; -/// Die temperature below which the limiter releases, in 0.01 °C. -const TEMP_RELEASE: u32 = 8_000; -/// Proportional gain of the limiter: 20.34 as an IEEE-754 single. -const TEMP_KP: u32 = 0x41a2_b852; -/// Integral gain of the limiter, per control period: 9.376 as an IEEE-754 -/// single. -const TEMP_KI: u32 = 0x4116_0419; - -/// Nonzero 32-bit words of a firmware object the host builds from constants: (byte offset, value, repeat). -/// Every other word is zero. The table is the sole definition of the object's constant contents. -type Words = &'static [(usize, u32, usize)]; - -fn fill_words(bytes: &mut [u8], words: Words) { - bytes.fill(0); - for &(offset, value, repeat) in words { - for index in 0..repeat { - let at = offset + 4 * index; - bytes[at..at + 4].copy_from_slice(&value.to_le_bytes()); - } - } -} - -const POWER_CONFIG_HEAD_WORDS: Words = &[ - (0x4, 0x5dc00, 2), - (0x10, 4, 1), - (0x14, 0x3f800000, 1), - (0x2c, 1, 2), - (0x40, 0x64, 1), - (0x44, 1, 1), - (0x48, 0x3e8, 1), - (0x50, 0x64, 1), - (0x60, 4, 1), - (0x64, 0x3f800000, 1), - (0x6c, 1, 2), - (0x80, 0x64, 1), - (0x84, 1, 1), - (0x88, 0x3e8, 1), - (0x90, 0x64, 1), - (0xc0, 0x3f828f5c, 11), - (0x9b8, 0x271, 1), - (0x9c4, 0x3f7f2e9f, 1), - (0x9cc, 0x3b516154, 1), - (0x9d4, 0x3ca59586, 1), - (0x9e0, 0x46153800, 1), - (0x9e4, 0x40a90fdb, 1), - (0x9ec, 0xbdd67344, 1), - (0x9f0, 0x28, 1), - (0x9f4, 0x3e8, 2), - (0xa04, 0x254e, 1), - (0xa38, 0x3e8, 1), - (0xa70, 4, 1), - (0xa84, 0x3f800000, 1), - (0xa8c, 0x40cccccd, 1), - (0xa98, 0x47800000, 1), - (0xa9c, 0x3f000000, 1), - (0xaa8, 0x28, 1), - (0xaac, 0x3e8, 1), - (0xabc, 0x254e, 1), - (0xac8, 0x10, 1), - (0xad0, 0x5dc00, 1), - (0xb28, 0x5c, 1), - (0xb30, 0x64, 1), - (0xb34, 0x22, 1), - (0xb38, 6, 1), - (0xb40, 6, 1), - (0xb44, 1, 1), - (0xb4c, 0x3f4ccccd, 1), - (0xb50, 0x3f7df3b6, 1), - (0xb54, 0x3e4ccccd, 1), - (0xb58, 0x3c03126f, 1), - (0xb5c, 0x3f69d4d8, 2), - (0xb68, 0x42be0000, 1), - (0xb6c, 0x4064f5c3, 2), - (0xb74, 0x41179436, 1), - (0xb78, 0x64, 1), - (0xb7c, 0x3e8, 1), - (0xb80, 0x64, 1), - (0xb8c, 0x5c, 1), - (0xbc0, 0x64, 1), - (0xbe0, 0x401, 1), - (0xbe4, 0x6400, 1), - (0xbec, 0x1f4000, 1), - (0xbf0, 0xc800, 1), - (0xbf4, 0xfa000, 1), - (0xbf8, 0xc800, 1), - (0xbfc, 0x7d000, 1), - (0xc00, 0xc800, 1), - (0xc04, 0x3e800, 1), - (0xc08, 0xc800, 1), - (0xc0c, 0x100, 1), - (0xc64, 1, 1), - (0xc90, 1, 1), - (0x1c94, 0xff000000, 1), - (0x1c98, 0xff7fff7f, 74), -]; - -const POWER_CONFIG_TAIL_WORDS: Words = &[ - (0x0, 0xff7fff7f, 181), - (0x2d4, 0x7fff7f, 1), - (0x4d8, 0x47800000, 1), - (0x4dc, 0x42200000, 1), - (0x4e0, 0x447a0000, 1), - (0x4e4, 0x5be, 1), - (0x4f8, 0x28, 1), - (0x500, 0x42c80000, 1), - (0x504, 0x3e8, 1), - (0x50c, 0x3f4ccccd, 1), - (0x510, 0x3e4ccccd, 1), - (0x550, TEMP_SENSOR_MASK as u32, 1), - (0x558, TEMP_RELEASE, 1), - (0x560, 4, 1), - (0x574, 0x3f800000, 1), - (0x57c, TEMP_KI, 1), - (0x588, 0x47800000, 1), - (0x58c, TEMP_KP, 1), - (0x598, 0x28, 1), - (0x59c, 0x3e8, 2), - (0x5ac, TEMP_TARGET, 1), - (0x5e0, 0x3e8, 1), - (0x7fc, 0x42000006, 1), - (0xa10, 0xb8520000, 1), - (0xa14, 0x3f7e, 1), - (0xa18, 0xd70a0000, 1), - (0xa1c, 0x3ba3, 1), - (0xa24, 0x43c8, 1), - (0xa30, 0x4780, 1), - (0xa34, 0x42c8, 1), - (0xa3c, 0x384bac8, 1), - (0xa40, 0x3e80000, 2), - (0xa4c, 0x80000000, 1), - (0xa50, 0x26ac453b, 1), - (0xa84, 0x3e80000, 1), - (0xaa0, 0xc80000, 1), - (0xb58, 0x3e8, 1), - (0xb6c, 1, 1), - (0xba0, 0x101, 1), - (0xba4, 0x400, 1), - (0xbbc, 0x23f00, 1), - (0xbc0, 0x100, 2), - (0xbdc, 0x24000, 1), - (0xbe0, 0x100, 2), - (0xbfc, 0x24100, 1), - (0xc48, 1, 1), - (0xc50, 0x254e, 3), - (0xc68, 0x3f800000, 1), - (0xc6c, 4, 1), - (0xc70, 0x10, 1), - (0xc74, 0x5dc00, 1), - (0xd44, 0x3c, 1), - (0xd50, 0x3f6eeeef, 1), - (0xd58, 0x3d888889, 1), - (0xd60, 0x3f000000, 1), - (0xd6c, 0x47800000, 1), - (0xd70, 0x40880000, 1), - (0xd7c, 0x28, 1), - (0xd80, 0x3e8, 2), - (0xd8c, 0x46153800, 1), - (0xd90, 0x254e, 1), - (0xd9c, 0xf0, 1), - (0xda4, 0x57e400, 1), - (0xdc4, 0x3e8, 1), - (0xe04, 1, 1), - (0xed4, 0x32, 1), - (0xed8, 1, 1), - (0xee0, 0x3f638e39, 1), - (0xee4, 0x3f2aaaab, 1), - (0xee8, 0x3de38e39, 1), - (0xeec, 0x3eaaaaab, 1), - (0xef0, 0xbf4ccccd, 2), - (0xefc, 0x47800000, 1), - (0xf00, 0xc0a00000, 2), - (0xf0c, 0x64, 1), - (0xf10, 0x3e8, 1), - (0xf14, 0x64, 1), - (0xf1c, 0x467a0000, 1), - (0xf20, 0x578, 1), - (0xf2c, 0x90, 1), - (0xf30, 0x30, 1), - (0xf34, 0x34bc00, 1), - (0xf3c, 0x119400, 1), - (0xf48, 0x47800000, 1), - (0xf70, 0x3e80, 1), - (0xf74, 2, 1), - (0xf78, 0x9c4, 1), - (0xf7c, 0x20d, 1), - (0xf80, 2, 1), - (0xf84, 4, 1), - (0xf88, 0x32, 1), - (0xfb8, 0x47800000, 1), - (0xfc8, 0x28, 1), - (0xfcc, 0x3e8, 1), -]; - -const POWER_STATE_WORDS: Words = &[ - (0x10, 0xf, 1), - (0x14, 0x3f000000, 1), - (0x18, 0x40880000, 1), - (0x1c, 0x28, 1), - (0x20, 1, 1), - (0xc8, TEMP_TARGET, 1), - (0xcc, TEMP_RELEASE, 1), - (0xd8, TEMP_KP, 1), - (0xdc, TEMP_KI, 1), - (0xf0, 0x26ac, 1), - (0xf4, 0xc8, 1), - (0x108, 0x42c80000, 1), - (0x10c, 0x43c80000, 1), - (0x110, 0xc8, 1), - (0x128, 1, 1), - (0x12c, 0x2616, 1), - (0x130, 0x3f000000, 1), - (0x134, 0x40cccccd, 1), - (0x14c, 1, 1), - (0x1c0, 1, 2), - (0x1c8, 4, 1), - (0x1cc, 1, 4), - (0x4e4, 1, 1), - (0x4e8, 0x1f4, 1), - (0x4ec, 6, 1), - (0x4f0, 0x30d4, 6), - (0x51c, 0x30d4, 5), - (0x530, 1, 1), - (0x58c, 6, 1), - (0x590, 0xdac, 1), - (0x594, 0x3e8, 1), - (0x598, 0xbb8, 1), - (0x59c, 0x64, 1), - (0x5cc, 1, 1), - (0x5d0, 4, 1), - (0x5dc, 0x1f40, 1), - (0x5e0, 0xc8, 1), - (0x5e4, 0xfa0, 1), - (0x5e8, 0xc8, 1), - (0x5ec, 0x7d0, 1), - (0x5f0, 0xc8, 1), - (0x5f4, 0x3e8, 1), - (0x5f8, 0xc8, 1), - (0x604, 0x41700000, 1), - (0x608, 0x40a00000, 1), - (0x60c, 0x20, 1), - (0x610, 0x3e8, 1), - (0x620, 0x384, 1), -]; - -const HW_DATA_AUX_WORDS: Words = &[ - (0xac, 6, 1), - (0xb8, 1, 1), - (0x174, 0xac000000, 1), - (0x178, 0xe800000d, 1), - (0x17c, 0xb8000003, 1), - (0x180, 0xe800000b, 1), - (0x184, 0xe8000003, 1), - (0x188, 0x64000003, 1), - (0x18c, 0x20000000, 1), - (0x1b4, 4, 1), - (0x1c0, 0x3f780000, 1), - (0x1c8, 0x3d000000, 1), - (0x1d0, 0x40a00000, 1), - (0x1dc, 0x47800000, 1), - (0x1e0, 0x41700000, 1), - (0x1ec, 0x384, 1), - (0x1f0, 0x3e8, 2), - (0x200, 0x3e8, 1), - (0x234, 0x3e8, 1), - (0x274, 0x64, 1), - (0x290, 0x82a, 1), - (0x298, 0x7d, 1), - (0x29c, 1, 1), - (0xee0, 0x18ae, 1), - (0xee8, 1, 1), - (0x2eec, 0x41c8cccd, 2), - (0x35e4, 1, 1), - (0x35e8, 0x1f4, 1), - (0x35ec, 6, 1), - (0x35f0, 0x30d4, 6), - (0x3608, 0xfa0, 1), - (0x361c, 0x30d4, 5), - (0x3630, 1, 1), - (0x37a8, 1, 1), - (0x37b4, 0x424ccccd, 1), - (0x37f4, 0x44068000, 1), - (0x3834, 0x41c9999a, 1), - (0x38e0, 0x186a, 1), -]; - -const SECONDARY_CONFIG_WORDS: Words = &[ - (0x14, 1, 1), - (0x2c, 1, 2), - (0x40, 1, 1), - (0x4c, 1, 1), - (0x50, 0x186a, 1), -]; - -/// Power state the primary status B carries for the secondary instance, -/// named by [`SecondaryRoot::power_state_va`]. -#[repr(C)] -pub(crate) struct PowerState(pub(crate) [u8; 0x6c0]); - -// SAFETY: `PowerState` is a byte array. -unsafe impl Zeroable for PowerState {} - -impl PowerState { - fn init(&mut self) { - fill_words(&mut self.0, POWER_STATE_WORDS); - } -} - -/// Configuration of the secondary instance, named by -/// [`SecondaryRoot::config_va`]. -#[repr(C)] -pub(crate) struct SecondaryConfig(pub(crate) [u8; 0x80]); - -// SAFETY: `SecondaryConfig` is a byte array. -unsafe impl Zeroable for SecondaryConfig {} - -impl SecondaryConfig { - pub(crate) fn init(&mut self) { - fill_words(&mut self.0, SECONDARY_CONFIG_WORDS); - } -} - -/// Leading part of the [`PowerConfig`]. The host writes it twice: in the -/// power configuration and at [`bundle::POWER_CONFIG_COPY`], where the copy -/// ends with the views it lies in. -#[repr(C)] -pub(crate) struct PowerConfigHead(pub(crate) [u8; 0x1dc0]); - -// SAFETY: `PowerConfigHead` is a byte array. -unsafe impl Zeroable for PowerConfigHead {} - -impl PowerConfigHead { - pub(crate) fn init(&mut self) { - fill_words(&mut self.0, POWER_CONFIG_HEAD_WORDS); - } -} - -/// Trailing part of the [`PowerConfig`]. Field names give the offset in the -/// whole power configuration. It carries the die temperature limiter: target -/// and release temperatures in 0.01 °C, proportional and integral gains as -/// IEEE-754 singles. -#[repr(C)] -pub(crate) struct PowerConfigTail(pub(crate) [u8; 0x1000]); - -// SAFETY: `PowerConfigTail` is a byte array. -unsafe impl Zeroable for PowerConfigTail {} - -impl PowerConfigTail { - fn init(&mut self) { - fill_words(&mut self.0, POWER_CONFIG_TAIL_WORDS); - } -} - -/// Auxiliary hardware data the firmware reads at [`bundle::AUX`], up to the -/// start of view 3. No pointer names it. -#[repr(C)] -pub(crate) struct HwDataAux(pub(crate) [u8; 0x3c80]); - -// SAFETY: `HwDataAux` is a byte array. -unsafe impl Zeroable for HwDataAux {} - -impl HwDataAux { - pub(crate) fn init(&mut self, secondary_status_va: u64) { - fill_words(&mut self.0, HW_DATA_AUX_WORDS); - self.0[0xed8..0xee0].copy_from_slice(&secondary_status_va.to_le_bytes()); - } -} - - - - - -/// Power configuration, in view 2 of the bundle. The secondary instance -/// finds it through [`MainConfig::power_config_va`]. -#[repr(C, packed)] -pub(crate) struct PowerConfig { - /// Leading part. - pub(crate) head: PowerConfigHead, - /// Trailing part. - pub(crate) tail: PowerConfigTail, -} - -static_assert!(bundle::POWER_CONFIG + size_of::() == bundle::AUX); -static_assert!(bundle::VIEWS[2] + 0xe40 == bundle::POWER_CONFIG); - -// SAFETY: `PowerConfig` consists of integers and arrays of integers only. -unsafe impl Zeroable for PowerConfig {} - -impl PowerConfig { - /// Initializes the power configuration. - pub(crate) fn init(&mut self) { - self.head.init(); - self.tail.init(); - } -} - - -// --------------------------------------------------------------------------- -// Free lists -// --------------------------------------------------------------------------- - -/// Size of a free-list page. -const FREE_LIST_PAGE_SIZE: u64 = 0x1000; -/// Pages in one free-list block. -pub(crate) const FREE_LIST_BLOCK_PAGES: u32 = 0x100; -/// Size of one free-list block. -pub(crate) const FREE_LIST_BLOCK_SIZE: usize = - FREE_LIST_BLOCK_PAGES as usize * FREE_LIST_PAGE_SIZE as usize; -/// Run-list qwords reserved for each block. -const RUN_SLOT_QWORDS: u32 = 8; -/// Bit position of the page count in a run descriptor. -const RUN_PAGES_SHIFT: u32 = 52; -/// Address bits of a run descriptor. -const RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; - -/// Class of a free list, selecting [`FreeListControl::unk_10`] and -/// [`FreeListControl::unk_60`]. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum FreeListClass { - /// `unk_10` 1, `unk_60` 2: the boot render and compute free lists. - One = 1, - /// `unk_10` 2, `unk_60` 3: the auxiliary boot and runtime render pools. - Two = 2, -} - -impl FreeListClass { - const fn unk_60(self) -> u32 { - match self { - Self::One => 2, - Self::Two => 3, - } - } -} - -/// Description of a free list whose blocks are all populated. -pub(crate) struct FreeListArgs { - /// Buffer slot of the free list. - pub(crate) buffer_slot: u32, - /// Class of the free list. - pub(crate) class: FreeListClass, - /// Page list: one 64-bit address per page of the blocks. - pub(crate) page_list_va: u64, - /// Run list: one [`FreeListRunSlot`] per block. - pub(crate) run_list_va: u64, - /// Blocks in the free list. - pub(crate) blocks: u32, - /// [`FreeListState`]. - pub(crate) state_va: u64, -} - -/// Control object of a USC private-memory free list, at the start of its -/// page. -/// -/// This constructor describes the boot objects. Once a pool is published, its -/// page counters (`+0x24`, `+0x2c`) and run cursor (`+0x48`) are shared state. -/// The firmware-completed operation counter occupies the eight bytes at -/// `+0x54`; it is unaligned and the host reads its bytes individually. -/// A released render pool can be repopulated only after its release is -/// consumed: write page-list/run-list words first, then the complete control -/// page in ascending aligned 64-bit words, then a full barrier. The completed -/// count in that image must equal the retained submitted count, not zero. -/// Publication and reuse belong to the host free-list implementation. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct FreeListControl { - /// Boot value 1, compute value 2, or the render submitted-counter address. - pub(crate) unk_0: u64, - /// Buffer slot of the free list. - pub(crate) buffer_slot: u32, - pub(crate) unk_c: u32, - /// Required: see [`FreeListClass`]. - pub(crate) unk_10: u32, - /// Page list. - pub(crate) page_list_va: u64, - /// Required: 0x40000. - pub(crate) unk_1c: u32, - pub(crate) unk_20: u32, - /// Required: [`Self::page_count`]. - pub(crate) unk_24: u32, - pub(crate) unk_28: u32, - /// Pages in the free list's blocks. - pub(crate) page_count: u32, - /// Run list. - pub(crate) run_list_va: u64, - pub(crate) unk_38: [u8; 8], - /// Required: 4. - pub(crate) unk_40: u32, - pub(crate) unk_44: u32, - /// Run-list qwords in use. - pub(crate) run_cursor: u32, - /// [`FreeListState`]. - pub(crate) state_va: u64, - pub(crate) unk_54: [u8; 0xc], - /// Required: see [`FreeListClass`]. - pub(crate) unk_60: u32, - pub(crate) unk_64: u32, -} - -// SAFETY: `FreeListControl` consists of integers and arrays of integers only. -unsafe impl Zeroable for FreeListControl {} - -static_assert!(core::mem::offset_of!(FreeListControl, page_list_va) == 0x14); -static_assert!(core::mem::offset_of!(FreeListControl, run_cursor) == 0x48); -static_assert!(core::mem::offset_of!(FreeListControl, state_va) == 0x4c); -static_assert!(size_of::() == 0x68); - -impl FreeListControl { - /// Runtime render control; submitted and completed work are retained - /// across release/repopulation of the same pool. - pub(crate) fn render(args: &FreeListArgs, completed: u64) -> Self { - let mut control = Self::new(args); - control.unk_0 = args.state_va; - control.unk_54[..8].copy_from_slice(&completed.to_le_bytes()); - control - } - - /// Runtime compute control, with a compact run list and shared counters. - pub(crate) fn compute(args: &FreeListArgs) -> Self { - let mut control = Self::new(args); - control.unk_0 = 2; - control.unk_10 = 2; - control.unk_54[8..12].copy_from_slice(&1u32.to_le_bytes()); - control - } - - /// Offset of the firmware-completed counter. It must be read bytewise. - pub(crate) const COMPLETED: usize = 0x54; - - /// Returns the control object of the free list `args` describes. - pub(crate) const fn new(args: &FreeListArgs) -> Self { - let pages = args.blocks * FREE_LIST_BLOCK_PAGES; - Self { - unk_0: 1, - buffer_slot: args.buffer_slot, - unk_c: 0, - unk_10: args.class as u32, - page_list_va: args.page_list_va, - unk_1c: 0x4_0000, - unk_20: 0, - unk_24: pages, - unk_28: 0, - page_count: pages, - run_list_va: args.run_list_va, - unk_38: [0; 8], - unk_40: 4, - unk_44: 0, - run_cursor: args.blocks * RUN_SLOT_QWORDS, - state_va: args.state_va, - unk_54: [0; 0xc], - unk_60: args.class.unk_60(), - unk_64: 0, - } - } -} - -/// Shared state of a free list, at the start of its page. -/// -/// Compute creation writes a 32-bit one at `+0x00` in a zeroed page. The -/// render boot object starts with a 64-bit two and the boot pool with zero. -/// In a live render pool this becomes the submitted-operation counter: the -/// host uses one aligned 64-bit store before making work visible. It is -/// retained across release/repopulation, never reset by a struct copy. -#[repr(C)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct FreeListState { - /// [`Self::RENDER`], [`Self::BOOT`] or [`Self::COMPUTE`] at boot. - pub(crate) unk_0: u64, -} - -// SAFETY: `FreeListState` consists of integers and arrays of integers only. -unsafe impl Zeroable for FreeListState {} - -impl FreeListState { - /// Boot state of the render free list. - pub(crate) const RENDER: Self = Self { unk_0: 2 }; - /// Boot state of the boot free list. - pub(crate) const BOOT: Self = Self { unk_0: 0 }; - /// Boot state of a compute free list. - pub(crate) const COMPUTE: Self = Self { unk_0: 1 }; -} - -/// One run-list slot: the run descriptor of one block. -#[repr(C)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct FreeListRunSlot { - /// Block address, with its page count in the top bits. - pub(crate) run: u64, - pub(crate) unk_8: [u8; 0x38], -} - -// SAFETY: `FreeListRunSlot` consists of integers and arrays of integers only. -unsafe impl Zeroable for FreeListRunSlot {} - -static_assert!(size_of::() == RUN_SLOT_QWORDS as usize * size_of::()); - -impl FreeListRunSlot { - /// Returns the slot describing the block at `block_va`. - pub(crate) const fn new(block_va: u64) -> Self { - Self { - run: (block_va & RUN_ADDRESS_MASK) - | ((FREE_LIST_BLOCK_PAGES as u64) << RUN_PAGES_SHIFT), - unk_8: [0; 0x38], - } - } -} - -/// Writes the run descriptors of `blocks` to the start of `slots`. -pub(crate) fn fill_run_list( - slots: &mut [FreeListRunSlot], - blocks: impl IntoIterator, -) -> Result { - let mut slots = slots.iter_mut(); - for block in blocks { - *slots.next().ok_or(EINVAL)? = FreeListRunSlot::new(block); - } - Ok(()) -} - -/// Writes the address of every page of `blocks` to the start of `list`. -pub(crate) fn fill_page_list(list: &mut [u64], blocks: impl IntoIterator) -> Result { - let mut entries = list.iter_mut(); - for block in blocks { - for page in 0..u64::from(FREE_LIST_BLOCK_PAGES) { - *entries.next().ok_or(EINVAL)? = block - .checked_add(page * FREE_LIST_PAGE_SIZE) - .ok_or(EOVERFLOW)?; - } - } - Ok(()) -} - -// --------------------------------------------------------------------------- -// Parameter-buffer and page-pool descriptor tables -// --------------------------------------------------------------------------- - -/// One parameter-buffer descriptor, indexed by buffer slot. -#[repr(C)] -pub(crate) struct PbDescriptor { - pub(crate) unk_0: u32, - pub(crate) unk_4: u32, - pub(crate) unk_8: u32, - pub(crate) unk_c: u32, -} - -// SAFETY: `PbDescriptor` consists of integers and arrays of integers only. -unsafe impl Zeroable for PbDescriptor {} - -static_assert!(size_of::() == 0x10); - -/// Parameter-buffer descriptor table, one page. -#[repr(C)] -pub(crate) struct PbDescriptorTable { - /// Descriptors. - pub(crate) entries: [PbDescriptor; PAGE_SIZE / size_of::()], -} - -// SAFETY: `PbDescriptorTable` consists of integers only. -unsafe impl Zeroable for PbDescriptorTable {} - -impl PbDescriptorTable { - /// Required boot contents of slots 0 and 1. - const BOOT: [PbDescriptor; 2] = [ - PbDescriptor { - unk_0: 0x1_9000, - unk_4: 0x20, - unk_8: 0, - unk_c: 0, - }, - PbDescriptor { - unk_0: 0x7_7000, - unk_4: 0x80, - unk_8: 0x17, - unk_c: 0x17, - }, - ]; - - /// Initializes the table. - pub(crate) fn init(&mut self) { - clear(self); - let [slot0, slot1] = Self::BOOT; - self.entries[0] = slot0; - self.entries[1] = slot1; - } -} - -/// Bit position of the unit size in [`PagePoolDescriptor::page_list`]. -const PAGE_POOL_UNIT_SHIFT: u32 = 41; -/// Required unit size of a page-pool descriptor. -const PAGE_POOL_UNIT: u64 = 0x4_0000; -/// Alignment shift of the page-list address in [`PagePoolDescriptor::page_list`]. -const PAGE_POOL_LIST_SHIFT: u32 = 7; -/// Bit position of the free-page count in [`PagePoolDescriptor::counters`]. -const PAGE_POOL_FREE_PAGES_SHIFT: u32 = 33; - -/// One Page-pool descriptor, indexed by buffer slot. -#[repr(C)] -pub(crate) struct PagePoolDescriptor { - /// Unit size and page-list address. - pub(crate) page_list: u64, - /// Pool counters. - pub(crate) counters: u64, - /// Pages in the pool. - pub(crate) page_count: u64, - /// Firmware-owned. - pub(crate) unk_18: u64, -} - -// SAFETY: `PagePoolDescriptor` consists of integers and arrays of integers only. -unsafe impl Zeroable for PagePoolDescriptor {} - -static_assert!(size_of::() == 0x20); - -impl PagePoolDescriptor { - /// Complete row for a physical compute queue's independent 21-block pool. - pub(crate) const fn compute(page_list_va: u64) -> Self { - let pages = 21 * FREE_LIST_BLOCK_PAGES as u64; - Self::new(page_list_va, pages << PAGE_POOL_FREE_PAGES_SHIFT, pages) - } - - const fn new(page_list_va: u64, counters: u64, page_count: u64) -> Self { - Self { - page_list: (PAGE_POOL_UNIT << PAGE_POOL_UNIT_SHIFT) - | (page_list_va >> PAGE_POOL_LIST_SHIFT), - counters, - page_count, - unk_18: 0, - } - } -} - -/// page-pool descriptor table, one page. -#[repr(C)] -pub(crate) struct PagePoolDescriptorTable { - /// Descriptors. - pub(crate) entries: [PagePoolDescriptor; PAGE_SIZE / size_of::()], -} - -// SAFETY: `PagePoolDescriptorTable` consists of integers only. -unsafe impl Zeroable for PagePoolDescriptorTable {} - -impl PagePoolDescriptorTable { - /// Required counters and page count of slot 0 at boot. - const RENDER_COUNTERS: u64 = 0x3d40_0000_3400; - const RENDER_PAGES: u64 = 0x1d00; - - /// Initializes the table: slot 0 describes the render free list's page - /// list, slot 1 the compute page list. - pub(crate) fn init(&mut self, render_page_list_va: u64, compute_page_list_va: u64) { - clear(self); - self.entries[0] = PagePoolDescriptor::new( - render_page_list_va, - Self::RENDER_COUNTERS, - Self::RENDER_PAGES, - ); - self.entries[1] = PagePoolDescriptor::compute(compute_page_list_va); - } -} - -// --------------------------------------------------------------------------- -// Index table -// --------------------------------------------------------------------------- - -/// Groups of the index table: first index and number of groups. -const INDEX_GROUPS: [(u32, usize); 2] = [(0x11, 6), (0x3c, 2)]; -/// Indices per group. -const INDEX_GROUP_SIZE: usize = 4; -/// Distance between the first indices of consecutive groups. -const INDEX_GROUP_STRIDE: u32 = 5; -/// Entries of the index table. -const INDEX_ENTRIES: usize = (INDEX_GROUPS[0].1 + INDEX_GROUPS[1].1) * INDEX_GROUP_SIZE; - -/// Index table at the start of the index object. -#[repr(C)] -pub(crate) struct IndexTable { - /// Groups of consecutive indices. - pub(crate) index: [u32; INDEX_ENTRIES], - pub(crate) unk_80: [u8; PAGE_SIZE - INDEX_ENTRIES * size_of::()], -} - -static_assert!(size_of::() == PAGE_SIZE); - -// SAFETY: `IndexTable` consists of integers only. -unsafe impl Zeroable for IndexTable {} - -impl IndexTable { - /// Initializes the table. - pub(crate) fn init(&mut self) { - clear(self); - let mut entries = self.index.iter_mut(); - for (first, groups) in INDEX_GROUPS { - for group in 0..groups as u32 { - for member in 0..INDEX_GROUP_SIZE as u32 { - if let Some(entry) = entries.next() { - *entry = first + group * INDEX_GROUP_STRIDE + member; - } - } - } - } - } -} - -/// Device-global compute state. Activation writes only the named fields, once, before the -/// first compute queue is exposed. The padding may contain firmware-owned state. -#[repr(C)] -pub(crate) struct ComputeGlobalState { - pub(crate) unk_0: u8, - unk_1: [u8; 7], - pub(crate) unk_8: u8, - unk_9: [u8; 7], - pub(crate) unk_10: u16, - unk_12: [u8; 6], - pub(crate) unk_18: u16, - unk_1a: [u8; 0x3ee], - pub(crate) unk_408: u32, - pub(crate) unk_40c: u32, - unk_410: [u8; 0x1f4], - pub(crate) unk_604: u32, - unk_608: [u8; 0x200], - pub(crate) scheduler_va: u64, - unk_810: [u8; 0x3f4], - pub(crate) unk_c04: u32, - unk_c08: [u8; 0x1f8], - pub(crate) unk_e00: u64, - pub(crate) unk_e08: u64, - unk_e10: [u8; 0x10], - pub(crate) unk_e20: u64, - pub(crate) unk_e28: u64, -} -static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_408) == 0x408); -static_assert!(core::mem::offset_of!(ComputeGlobalState, scheduler_va) == 0x808); -static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_c04) == 0xc04); -static_assert!(core::mem::offset_of!(ComputeGlobalState, unk_e00) == 0xe00); -static_assert!(size_of::() == 0xe30); - -/// Host-owned recovery report and progress records within the bundle views. -pub(crate) mod recovery { - pub(crate) const REPORT_VIEW: usize = 3; - pub(crate) const REASON: usize = 0x4c; - pub(crate) const PROGRESS_VIEW: usize = 1; - pub(crate) const PROGRESS_START: usize = 0xa40; - pub(crate) const PROGRESS_STRIDE: usize = 0x18; - pub(crate) const VIEW_EXTENTS: [usize; 5] = [0x18c0, 0x0c80, 0x3c00, 0x3380, 0x3300]; -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/kick.rs b/drivers/gpu/drm/asahi-neo/g17/fw/kick.rs deleted file mode 100644 index 47cdce3ca09e5c..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/kick.rs +++ /dev/null @@ -1,549 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Kick-queue entries and the objects they reference. -//! -//! Every hardware queue (QID) owns a kick ring of [`KICK_SLOTS`] entries spaced -//! [`KICK_ENTRY_STRIDE`] bytes apart. A [`KickTimestamp`] names one entry: its -//! low eight bits select the ring slot. To kick, the host writes the complete -//! slot selected by the queue's current timestamp, issues a barrier, advances -//! the timestamp and then announces the entry with a -//! [`super::queue::KickAnnounce`] record. -//! -//! An entry points at a work descriptor and its queue record, lists the kicks -//! it depends on, binds up to four [`RegisterArray`]s embedded in the -//! descriptor and may name a table of [`McacheRange`]s. -//! -//! Constructors of plain values ([`KickTimestamp::new`], -//! [`KickDependency::new`], [`KickDependency::from_word`]) return `None` for -//! values outside their domain; builders of firmware records and of encoded -//! words return `EINVAL`. - -use kernel::prelude::*; - -use super::queue::Policy; - -/// Number of entries in a kick ring. -pub(crate) const KICK_SLOTS: usize = 256; -/// Distance between consecutive kick ring entries. -pub(crate) const KICK_ENTRY_STRIDE: usize = 0x200; -/// Size of a kick ring. -pub(crate) const KICK_RING_SIZE: usize = KICK_SLOTS * KICK_ENTRY_STRIDE; -/// Dependencies one entry can carry: explicit barriers plus the implicit parent. -pub(crate) const KICK_DEPENDENCIES: usize = 32; -/// Explicit barriers one entry can carry. -pub(crate) const KICK_BARRIERS_MAX: usize = KICK_DEPENDENCIES - 1; -/// Highest hardware queue ID. -pub(crate) const QID_MAX: u8 = 0x7f; -/// Highest UAT context ID. -pub(crate) const CONTEXT_MAX: u8 = 0x3f; - -const KICK_TIMESTAMP_BITS: u32 = 40; -const KICK_TIMESTAMP_MASK: u64 = (1 << KICK_TIMESTAMP_BITS) - 1; -const KICK_SLOT_MASK: u64 = KICK_SLOTS as u64 - 1; -/// GPU addresses packed into kick entries are 32-byte aligned and 43 bits wide. -const PACKED_VA_SHIFT: u32 = 5; -const PACKED_VA_LIMIT: u64 = 1 << (38 + PACKED_VA_SHIFT); - -/// Packs a 32-byte aligned GPU address into the low 38 bits of a word. -fn packed_va(va: u64) -> Result { - if va & ((1 << PACKED_VA_SHIFT) - 1) != 0 || va >= PACKED_VA_LIMIT { - return Err(EINVAL); - } - Ok(va >> PACKED_VA_SHIFT) -} - -/// A kick timestamp. -/// -/// The value is a 40-bit counter that advances by one per kick of a queue; -/// bits 7:0 select the kick ring slot. A queue's first kick uses -/// [`KickTimestamp::FIRST`]. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct KickTimestamp(u64); - -impl KickTimestamp { - /// The timestamp preceding a queue's first kick. - pub(crate) const ZERO: Self = Self(0); - /// The timestamp of a queue's first kick. - pub(crate) const FIRST: Self = Self(1); - - /// Returns the timestamp with raw value `value`, if it fits 40 bits. - pub(crate) const fn new(value: u64) -> Option { - if value & !KICK_TIMESTAMP_MASK != 0 { - None - } else { - Some(Self(value)) - } - } - - /// Returns the raw 40-bit value. - pub(crate) const fn get(self) -> u64 { - self.0 - } - - /// Returns the kick ring slot this timestamp selects. - pub(crate) const fn slot(self) -> usize { - (self.0 & KICK_SLOT_MASK) as usize - } - - /// Returns the byte offset of the selected entry within the kick ring. - pub(crate) const fn entry_offset(self) -> usize { - self.slot() * KICK_ENTRY_STRIDE - } - - /// Returns the timestamp of the following kick. - pub(crate) const fn next(self) -> Self { - Self(self.0.wrapping_add(1) & KICK_TIMESTAMP_MASK) - } - - /// Returns the timestamp of the preceding kick. - pub(crate) const fn prev(self) -> Self { - Self(self.0.wrapping_sub(1) & KICK_TIMESTAMP_MASK) - } - - /// Returns the completion value of this kick: the timestamp in bits 39:0 - /// and `pass_seq`, the low byte of the logical queue's render-pass count, - /// in bits 47:40. - pub(crate) const fn completion_value(self, pass_seq: u8) -> u64 { - self.0 | (pass_seq as u64) << KICK_TIMESTAMP_BITS - } -} - -/// A dependency on the completion of one kick. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct KickDependency { - qid: u8, - timestamp: KickTimestamp, -} - -impl KickDependency { - const QID_SHIFT: u32 = KICK_TIMESTAMP_BITS; - - /// Initial value for unused entries in a host-side dependency array. - pub(crate) const ZERO: Self = Self { - qid: 0, - timestamp: KickTimestamp::ZERO, - }; - - /// Returns the dependency on kick `timestamp` of queue `qid`. - pub(crate) const fn new(qid: u8, timestamp: KickTimestamp) -> Option { - if qid > QID_MAX { - None - } else { - Some(Self { qid, timestamp }) - } - } - - /// Returns the queue that was kicked. - pub(crate) const fn qid(self) -> u8 { - self.qid - } - - /// Returns the dependency word: the timestamp in bits 39:0 and the QID in - /// bits 46:40. - pub(crate) const fn word(self) -> u64 { - self.timestamp.0 | (self.qid as u64) << Self::QID_SHIFT - } - - /// Decodes a dependency word; bits 63:47 must be clear. - pub(crate) const fn from_word(word: u64) -> Option { - if word >> (Self::QID_SHIFT + 7) != 0 { - return None; - } - Some(Self { - qid: (word >> Self::QID_SHIFT) as u8, - timestamp: KickTimestamp(word & KICK_TIMESTAMP_MASK), - }) - } -} - -/// One register write of a [`RegisterArray`]. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct RegisterWrite { - /// Register number. - pub(crate) reg: u32, - /// Value written to the register. - pub(crate) value: u64, -} - -impl RegisterWrite { - /// Returns the write of `value` to register `reg`. - pub(crate) const fn new(reg: u32, value: u64) -> Self { - Self { reg, value } - } -} - -/// A buffer that work writes. -#[derive(Copy, Clone, Debug)] -pub(crate) struct WriteRange { - /// GPU address of the buffer. - pub(crate) va: u64, - /// Size of the buffer in bytes. - pub(crate) size: u64, -} - -/// Class of an [`McacheRange`]. -/// -/// The auxiliary framebuffer range takes class 2 and attachment ranges take -/// class 1. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum McacheClass { - /// A render-target attachment. - Attachment = 1, - /// The auxiliary framebuffer. - AuxFramebuffer = 2, -} - -/// One MCache range: a GPU address range that work writes, in 128-byte cache -/// lines. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct McacheRange { - /// First cache line in bits 40:0, UAT context in bits 46:41. - pub(crate) start: u64, - /// End cache line (exclusive) in bits 40:0 and the class above. - pub(crate) end: u64, -} - -impl McacheRange { - const LINE_SIZE: u64 = 0x80; - /// Ranges cover fewer than 2^32 cache lines (512 GiB). - const LINES_MAX: u64 = u32::MAX as u64; - const LINE_LIMIT: u64 = 1 << 41; - const CONTEXT_SHIFT: u32 = 41; - /// Bits 55:47 of the start word hold 0x118 in every range. - const UNK_START: u64 = 0x118 << 47; - /// Bit 41 of the end word is set in every range. - const UNK_END: u64 = 1 << 41; - const CLASS_SHIFT: u32 = 42; - - /// Encodes the cache lines `range` touches, translated in UAT context - /// `context`. - pub(crate) fn new(range: &WriteRange, class: McacheClass, context: u8) -> Result { - if range.size == 0 || context > CONTEXT_MAX { - return Err(EINVAL); - } - let start = range.va / Self::LINE_SIZE; - let end = range - .va - .checked_add(range.size) - .ok_or(EINVAL)? - .div_ceil(Self::LINE_SIZE); - if end >= Self::LINE_LIMIT || end - start > Self::LINES_MAX { - return Err(EINVAL); - } - Ok(Self { - start: start | (context as u64) << Self::CONTEXT_SHIFT | Self::UNK_START, - end: end | Self::UNK_END | 1 << (Self::CLASS_SHIFT + class as u32), - }) - } -} - -/// Register-write entries a [`RegisterArray`] holds. -pub(crate) const REGISTER_ARRAY_ENTRIES: usize = 128; -/// MCache ranges that fit in the tail of a [`RegisterArray`]. -pub(crate) const MCACHE_INLINE_RANGES: usize = 16; - -/// A register array embedded in a work descriptor and bound to its kick entry -/// by a [`RegisterArrayBinding`]. -/// -/// The array ends in a trailer that points back at the array and holds its -/// size. It is only written as part of a cleared descriptor: [`Self::set`] -/// leaves the unused entries and the MCache space untouched. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct RegisterArray { - /// Register writes, in order. - pub(crate) entries: [RegisterWrite; REGISTER_ARRAY_ENTRIES], - /// Space after the entries. The first fragment array keeps its MCache - /// table here when the table fits; it is zero everywhere else. - pub(crate) mcache: [McacheRange; MCACHE_INLINE_RANGES], - /// GPU address of this array, as seen by the work's VM. - pub(crate) self_va: u64, - /// Size of the used entries in bytes (bits 31:16) and their count (15:0). - pub(crate) size: u32, - pub(crate) unk_70c: [u8; 0x14], -} - -impl RegisterArray { - /// Stores `writes` and the trailer into a zeroed array. - pub(crate) fn set(&mut self, self_va: u64, writes: &[RegisterWrite]) -> Result { - let count = writes.len(); - if count > REGISTER_ARRAY_ENTRIES { - return Err(EINVAL); - } - self.entries[..count].copy_from_slice(writes); - self.self_va = self_va; - self.size = (size_of_val(writes) as u32) << 16 | count as u32; - Ok(()) - } -} - -/// Binding of one [`RegisterArray`] to a kick entry. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct RegisterArrayBinding { - word: u64, -} - -impl RegisterArrayBinding { - const CHECKPOINT_SHIFT: u32 = 43; - const CHECKPOINT_MAX: u8 = 0x7f; - - /// Binds the array at `array_va`, as seen by the work's VM, whose first - /// checkpoint is after `checkpoint` entries. - pub(crate) fn new(array_va: u64, checkpoint: u8) -> Result { - if checkpoint > Self::CHECKPOINT_MAX { - return Err(EINVAL); - } - Ok(Self { - word: packed_va(array_va)? | (checkpoint as u64) << Self::CHECKPOINT_SHIFT, - }) - } - - fn word(binding: Option) -> u64 { - binding.map_or(0, |binding| binding.word) - } -} - -/// Location of the MCache table a kick entry names. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct McacheTable { - va: u64, - count: u8, -} - -impl McacheTable { - const VA_SHIFT: u32 = 6; - const COUNT_SHIFT: u32 = 54; - const COUNT_MAX: usize = 64; - - /// Names the `count` ranges at `va`, as seen by the work's VM. - pub(crate) fn new(va: u64, count: usize) -> Result { - if count == 0 || count > Self::COUNT_MAX { - return Err(EINVAL); - } - packed_va(va)?; - Ok(Self { - va, - count: count as u8, - }) - } - - fn word(self) -> u64 { - (self.va >> PACKED_VA_SHIFT) << Self::VA_SHIFT - | ((self.count - 1) as u64) << Self::COUNT_SHIFT - } -} - -/// Quality-of-service identity of the kicked queue. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct KickQos { - /// QoS buffer slot of the logical queue. - pub(crate) slot: u8, - /// Scheduling class of the engine, at most 0x1f. - pub(crate) class: u8, -} - -/// Inputs of one kick entry. -pub(crate) struct KickArgs<'a> { - /// Kicked queue. - pub(crate) qid: u8, - /// Timestamp of this kick; it selects the ring slot. - pub(crate) timestamp: KickTimestamp, - /// Kick this kick implicitly depends on: the previous kick of the same - /// queue. A queue's first kick names `timestamp.prev()` on render queues - /// and [`KickTimestamp::ZERO`] on compute queues. - pub(crate) parent: KickTimestamp, - /// Kicks of other queues this kick must wait for. - pub(crate) barriers: &'a [KickDependency], - /// GPU address of the work descriptor (firmware alias). - pub(crate) descriptor_va: u64, - /// GPU address of the queue record. - pub(crate) queue_va: u64, - /// Queue QoS identity. - pub(crate) qos: KickQos, - /// MCache table of the work, if any. - pub(crate) mcache: Option, - /// Engine-specific event mask. - pub(crate) event_mask: [u64; 4], - /// Register arrays bound to the kick. - pub(crate) register_arrays: [Option; 4], - /// The compute register program includes a shader-context scratch request. - pub(crate) compute_scratch: bool, - /// Priority class of the queue. - pub(crate) priority: u8, -} - -/// One kick ring entry. -/// -/// Live state: none. The host writes the whole entry before announcing it and -/// does not touch it again until the slot is reused. -#[repr(C)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct KickEntry { - /// Timestamp in bits 41:2, QID in bits 48:42, MCache table present in bit - /// 58, queue priority in bits 60:59 and compute scratch enabled in bit 61. - pub(crate) header: u64, - /// MCache table: address >> 5 in bits 43:6, count - 1 in bits 59:54. - pub(crate) mcache: u64, - /// GPU address of the work descriptor. - pub(crate) descriptor_va: u64, - /// GPU address of the queue record. - pub(crate) queue_va: u64, - /// Valid-dependency mask in bits 31:0, QoS slot in bits 39:32, class in - /// bits 47:40; bits 63:48 are all ones. - pub(crate) dependency_control: u64, - /// Explicit barriers followed by the implicit parent. - pub(crate) dependencies: [u64; KICK_DEPENDENCIES], - /// Engine-specific event mask. - pub(crate) event_mask: [u64; 4], - pub(crate) unk_148: u64, - /// Register-array bindings: address >> 5 in bits 37:0, checkpoint in bits - /// 49:43. - pub(crate) register_arrays: [u64; 4], - pub(crate) unk_170: u64, - pub(crate) unk_178: u64, - pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], -} - -static_assert!(size_of::() == KICK_ENTRY_STRIDE); -static_assert!(core::mem::offset_of!(KickEntry, dependency_control) == 0x20); -static_assert!(core::mem::offset_of!(KickEntry, event_mask) == 0x128); -static_assert!(core::mem::offset_of!(KickEntry, register_arrays) == 0x150); -static_assert!(core::mem::offset_of!(KickEntry, unk_178) == 0x178); -static_assert!(size_of::() == 0xc); -static_assert!(size_of::() == 0x10); -static_assert!(size_of::() == 0x720); -static_assert!(core::mem::offset_of!(RegisterArray, mcache) == 0x600); -static_assert!(core::mem::offset_of!(RegisterArray, self_va) == 0x700); - -impl KickEntry { - const TIMESTAMP_SHIFT: u32 = 2; - const QID_SHIFT: u32 = 42; - const MCACHE_PRESENT: u64 = 1 << 58; - const PRIORITY_SHIFT: u32 = 59; - const COMPUTE_SCRATCH: u64 = 1 << 61; - const QOS_SLOT_SHIFT: u32 = 32; - const QOS_CLASS_SHIFT: u32 = 40; - const QOS_CLASS_MAX: u8 = 0x1f; - /// Bits 63:48 of the dependency control word. - const UNK_DEPENDENCY_CONTROL: u64 = 0xffff << 48; - /// Required value of the last word of the entry. - const UNK_178: u64 = 0x003f_ffff_ffff_ffff; - - /// Builds the entry described by `args`. - pub(crate) fn new(args: &KickArgs<'_>) -> Result { - let barriers = args.barriers.len(); - if args.qid > QID_MAX - || barriers > KICK_BARRIERS_MAX - || args.qos.class > Self::QOS_CLASS_MAX - || args.priority >= Policy::PRIORITIES - { - return Err(EINVAL); - } - - let mut header = args.timestamp.get() << Self::TIMESTAMP_SHIFT - | (args.qid as u64) << Self::QID_SHIFT - | (args.priority as u64) << Self::PRIORITY_SHIFT; - if args.compute_scratch { - header |= Self::COMPUTE_SCRATCH; - } - let mcache = match args.mcache { - Some(table) => { - header |= Self::MCACHE_PRESENT; - table.word() - } - None => 0, - }; - - let mut dependencies = [0; KICK_DEPENDENCIES]; - for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { - *slot = barrier.word(); - } - dependencies[barriers] = KickDependency { - qid: args.qid, - timestamp: args.parent, - } - .word(); - let count = barriers + 1; - let mask = if count == KICK_DEPENDENCIES { - u32::MAX - } else { - (1u32 << count) - 1 - }; - - Ok(Self { - header, - mcache, - descriptor_va: args.descriptor_va, - queue_va: args.queue_va, - dependency_control: mask as u64 - | (args.qos.slot as u64) << Self::QOS_SLOT_SHIFT - | (args.qos.class as u64) << Self::QOS_CLASS_SHIFT - | Self::UNK_DEPENDENCY_CONTROL, - dependencies, - event_mask: args.event_mask, - unk_148: 0, - register_arrays: args.register_arrays.map(RegisterArrayBinding::word), - unk_170: 0, - unk_178: Self::UNK_178, - pad_180: [0; KICK_ENTRY_STRIDE - 0x180], - }) - } -} - -/// The complete write-only installation of a fragment/tiling queue pair. -/// Firmware must see both queues configured before either outer command. -pub(crate) struct RenderRegistration { - writes: [(usize, u64); 8], -} - -impl RenderRegistration { - const TAG_PORT: usize = 0x4090; - const DATA_PORT: usize = 0x4098; - const VALID_LOW: usize = 0x21068; - const VALID_HIGH: usize = 0x21070; - const CONFIG_TAG: u64 = 0x0000_0400_0002_1008; - const ADDRESS_MASK: u64 = 0x0000_07ff_ffff_ffe0; - - /// Validates both queues without touching MMIO. The resulting transaction - /// writes fragment configuration and validity before tiling configuration - /// and validity. Each validity mask is stored high before low. Both queues - /// are registered at priority class `priority`. - pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64), priority: u8) -> Result { - if fragment.0 == tiling.0 || priority >= Policy::PRIORITIES { - return Err(EINVAL); - } - let mut writes = [(0, 0); 8]; - for (slot, ((qid, va), engine)) in [ - (fragment, super::queue::DataMaster::Fragment), - (tiling, super::queue::DataMaster::Tiling), - ] - .into_iter() - .enumerate() - { - if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { - return Err(EINVAL); - } - let completion = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); - let payload = ((engine as u64 + 1) << 60) | va | completion; - let (low, high) = if qid < 64 { - (1u64 << qid, 0) - } else { - (0, 1u64 << (qid - 64)) - }; - writes[slot * 4..slot * 4 + 4].copy_from_slice(&[ - (Self::TAG_PORT, Self::CONFIG_TAG | (u64::from(qid) << 43)), - (Self::DATA_PORT, payload), - (Self::VALID_HIGH, high), - (Self::VALID_LOW, low), - ]); - } - Ok(Self { writes }) - } - - pub(crate) fn writes(&self) -> &[(usize, u64); 8] { - &self.writes - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/mod.rs b/drivers/gpu/drm/asahi-neo/g17/fw/mod.rs deleted file mode 100644 index 4f0ae8aa253ed0..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/mod.rs +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! G17 firmware structures. -//! -//! Layouts of the records and descriptors the host shares with the G17 -//! firmware, and the pure builders that fill them. Nothing here allocates, -//! locks or publishes: the owners of the rings and queues do that. -//! -//! This module also holds the few rules shared by the render and compute -//! work descriptors. - -use kernel::prelude::*; - -pub(crate) mod buffer; -pub(crate) mod channels; -pub(crate) mod compute; -pub(crate) mod context; -pub(crate) mod freelist; -pub(crate) mod initdata; -pub(crate) mod kick; -pub(crate) mod queue; -pub(crate) mod render; -pub(crate) mod qos; - -/// Clears a firmware structure in place. -/// -/// Work descriptors are several KiB large and must not be built on the stack. -pub(crate) fn clear(object: &mut T) { - // SAFETY: `T: Zeroable` guarantees that all-zero bytes are a valid `T`, - // and `object` is a valid, exclusive reference to one `T`. - unsafe { core::ptr::write_bytes(object as *mut T, 0, 1) }; -} - -/// Returns `va + offset`, or `EINVAL` if the sum overflows. -pub(crate) fn offset_va(va: u64, offset: u64) -> Result { - va.checked_add(offset).ok_or(EINVAL) -} - -/// Size of one slot of a start/end timestamp pair. -const TIMESTAMP_SIZE: u64 = 8; - -/// Returns the address of the end slot of the timestamp pair at `start_va`. -/// -/// A work item's start and end timestamps are two consecutive 64-bit slots. -pub(crate) fn timestamp_end_va(start_va: u64) -> Result { - offset_va(start_va, TIMESTAMP_SIZE) -} - -/// Returns the work key registers take: the UAT context in bits 15:8 and the -/// low six bits of the work-state word below. -pub(crate) const fn work_key(context_id: u16, state_word: u32) -> u64 { - (context_id as u64) << 8 | (state_word & 0x3f) as u64 -} - -/// Returns the sampler bound descriptors store next to the sampler count: the -/// count plus one, or zero without samplers. -pub(crate) fn sampler_max(count: u32) -> Result { - match count { - 0 => Ok(0), - count => count.checked_add(1).ok_or(EINVAL), - } -} - -/// Returns the completion stamp value of sequence number `sequence`: the -/// value the firmware stores to a queue's completion stamp word when the work -/// completes. -pub(crate) const fn stamp_value(sequence: u32) -> u32 { - sequence.wrapping_shl(8) -} - -/// Object IDs of a work item, written to three registers of its engine and -/// to its descriptor. -/// -/// A render pass takes `predecessor` from its queue pair and a newly -/// allocated `current`; compute work uses fixed IDs. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct ObjectIds { - /// Predecessor object ID. - pub(crate) predecessor: u32, - /// Current object ID. - pub(crate) current: u32, -} - -impl ObjectIds { - /// Returns the register value: `predecessor` in bits 63:32 and `current` - /// in bits 31:0. - pub(crate) const fn word(self) -> u64 { - (self.predecessor as u64) << 32 | self.current as u64 - } -} - -/// Required value of the per-engine registers 0x0a599 (compute), 0x0a5a1 -/// (tiling) and 0x0a5a9 (fragment). -pub(crate) const UNK_0A599: u64 = 0x0000_0060_0040_0020; -/// Required value of the per-engine registers 0x0d411 (compute), 0x0d419 -/// (tiling) and 0x0d429 (fragment). -pub(crate) const UNK_0D411: u64 = 0x0000_0002_0000_0001; -/// Required value of the per-engine registers 0x101d9 (compute), 0x101e1 -/// (tiling) and 0x101e9 (fragment). -pub(crate) const UNK_101D9: u64 = 0x1c; -/// Required value of the per-engine registers 0x1a0e9 (compute), 0x1a0f1 -/// (tiling) and 0x1a0f9 (fragment). -pub(crate) const UNK_1A0E9: u64 = 8; -/// Required value of the per-engine registers 0x107a1 (compute) and 0x10799 -/// (tiling). The fragment register 0x10791 also sets bit 9. -pub(crate) const UNK_10791: u64 = 0xff_0000; -/// Required value of the tiling register 0x1c8f8 and the fragment register -/// 0x100b8; the compute register 0x1a458 holds it in its low bits. -pub(crate) const UNK_1C8F8: u64 = 0x8860; diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/qos.rs b/drivers/gpu/drm/asahi-neo/g17/fw/qos.rs deleted file mode 100644 index d939e012714ee1..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/qos.rs +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! QoS accounting object. Queue rows and submission counts use distinct queue -//! and logical-owner indices. Firmware may observe each live word separately. - -use kernel::prelude::*; - -pub(crate) const QUEUES: usize = 128; -pub(crate) const DATA_MASTERS: usize = 4; - -#[repr(C)] -pub(crate) struct Qos { - pub(crate) rows: [[u8; 8]; QUEUES], - pub(crate) queue_submitted: [u32; QUEUES], - pub(crate) owner_submitted: [u32; QUEUES], - pub(crate) scheduler: [u64; QUEUES], - pub(crate) share: [u32; QUEUES], - pub(crate) intervals: u64, - pub(crate) busy_start: [u64; 3], - pub(crate) dm_submitted: [u64; DATA_MASTERS], -} -// SAFETY: Qos consists entirely of integer fields and arrays. -unsafe impl Zeroable for Qos {} - -const _: () = { - assert!(core::mem::size_of::() == 0xe40); - assert!(core::mem::offset_of!(Qos, queue_submitted) == 0x400); - assert!(core::mem::offset_of!(Qos, owner_submitted) == 0x600); - assert!(core::mem::offset_of!(Qos, scheduler) == 0x800); - assert!(core::mem::offset_of!(Qos, share) == 0xc00); - assert!(core::mem::offset_of!(Qos, intervals) == 0xe00); - assert!(core::mem::offset_of!(Qos, busy_start) == 0xe08); - assert!(core::mem::offset_of!(Qos, dm_submitted) == 0xe20); -}; diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/queue.rs b/drivers/gpu/drm/asahi-neo/g17/fw/queue.rs deleted file mode 100644 index 48b110a4193e3b..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/queue.rs +++ /dev/null @@ -1,611 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Queue records and the commands of a queue's item ring. -//! -//! A hardware queue is described by a [`QueueRecord`]. Its [`PointerBlock`] -//! holds the cursors of the queue's item ring, an array of [`ITEM_RING_ENTRIES`] -//! work pointers: 64-bit GPU addresses of commands. Every command starts with -//! a 32-bit [`CommandTag`]: work descriptors carry the tiling, fragment and -//! compute tags (see `render.rs` and `compute.rs`), and the per-kick records -//! defined here carry the others. -//! -//! One kick appends, in order, the work descriptor, a [`QueueConfig`], a -//! [`KickAnnounce`] record and, when a compute queue is first installed or -//! explicitly waits on another queue, a [`KickPredecessor`]. -//! Compute kicks always carry the [`QueueConfig`]. A render kick carries it -//! only when it installs the queue, updates the context or QoS identity, -//! carries a completion seed, or configures the queue differently from the -//! last [`QueueConfig`] published on it. - -use core::sync::atomic::{AtomicU32, Ordering}; -use kernel::prelude::*; - -use super::kick::{KickTimestamp, QID_MAX}; - -/// Firmware scheduling profile of a queue. -/// -/// Every record that names a queue repeats its priority class: the queue -/// record (twice), [`QueueConfig`], [`KickAnnounce`], the kick entry header, -/// the work ring the queue is published on (`channels.rs`), the doorbell and -/// the render registration token. The queue record and [`QueueConfig`] also -/// carry the scheduling policy word; the QoS row and share word carry the QoS -/// class and share. All of them must agree for one queue. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct Policy { - priority: u8, - policy: u16, - qos_class: u8, - qos_share: u32, -} - -impl Policy { - /// Number of firmware priority classes. - pub(crate) const PRIORITIES: u8 = 4; - - /// Returns the profile with priority class `priority`, policy word - /// `policy`, QoS class `qos_class` and QoS share `qos_share`. - pub(crate) const fn new(priority: u8, policy: u16, qos_class: u8, qos_share: u32) -> Self { - assert!(priority < Self::PRIORITIES); - Self { - priority, - policy, - qos_class, - qos_share, - } - } - - /// Priority class, below [`Self::PRIORITIES`]. - pub(crate) const fn priority(&self) -> u8 { - self.priority - } - - /// Scheduling policy word. - pub(crate) const fn policy(&self) -> u16 { - self.policy - } - - /// QoS class byte of the queue's QoS row. - pub(crate) const fn qos_class(&self) -> u8 { - self.qos_class - } - - /// QoS share word of the owner's QoS slot. - pub(crate) const fn qos_share(&self) -> u32 { - self.qos_share - } -} - -/// Number of work pointers in a queue's item ring. -pub(crate) const ITEM_RING_ENTRIES: u32 = 0x500; - -/// Tag in the first word of every command. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum CommandTag { - /// Tiling (TA) work descriptor. - Tiling = 0, - /// Fragment (3D) work descriptor. - Fragment = 1, - /// Compute work descriptor. - Compute = 3, - /// [`KickAnnounce`]. - KickAnnounce = 0x0e, - /// [`QueueConfig`]. - QueueConfig = 0x0f, - /// [`KickPredecessor`]. - KickPredecessor = 0x10, -} - -/// The engine that executes a queue's work. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[repr(u32)] -pub(crate) enum DataMaster { - /// Tiling (TA). - Tiling = 0, - /// Fragment (3D). - Fragment = 1, - /// Compute (CL). - Compute = 2, -} - -/// Cursors of a queue's item ring. -/// -/// The firmware writes `done` and `unk_30`. The host writes the block on -/// three paths, never with a structure copy once the queue was published: -/// -/// - creation: the whole block, [`PointerBlock::new`], before the queue is -/// first published; -/// - publication: `producer`, with one aligned 32-bit store per publication -/// step, after a barrier that orders the work pointers and commands before -/// it; -/// - reuse of a render queue for a new owner: [`PointerBlock::rebase`]. -#[repr(C)] -#[derive(Debug)] -pub(crate) struct PointerBlock { - /// Item-ring position up to which the firmware has finished commands. - /// Free ring space is measured from it, so it bounds `producer`. - pub(crate) done: AtomicU32, - pub(crate) unk_04: [u32; 11], - /// Written by the firmware. The host treats the queue as idle only when - /// this cursor, `done` and `producer` are equal. - pub(crate) unk_30: AtomicU32, - pub(crate) unk_34: [u32; 3], - /// Item-ring position after the last work pointer the host published. - pub(crate) producer: AtomicU32, - pub(crate) unk_44: [u32; 3], - /// On recovery, the host copies producer here before acknowledging resume. - pub(crate) recovery_target: u32, - pub(crate) unk_54: [u32; 3], - /// Number of entries in the item ring. - pub(crate) entries: u32, - pub(crate) unk_64: [u32; 7], -} - -static_assert!(size_of::() == 0x80); -static_assert!(core::mem::offset_of!(PointerBlock, unk_30) == 0x30); -static_assert!(core::mem::offset_of!(PointerBlock, producer) == 0x40); -static_assert!(core::mem::offset_of!(PointerBlock, entries) == 0x60); - -// SAFETY: PointerBlock contains only integers and integer atomics. -unsafe impl Zeroable for PointerBlock {} - -impl PointerBlock { - pub(crate) const PRODUCER: usize = core::mem::offset_of!(Self, producer); - pub(crate) const RECOVERY_TARGET: usize = core::mem::offset_of!(Self, recovery_target); - - /// Returns the cursors of an empty item ring. - pub(crate) const fn new() -> Self { - Self { - done: AtomicU32::new(0), - unk_04: [0; 11], - unk_30: AtomicU32::new(0), - unk_34: [0; 3], - producer: AtomicU32::new(0), - unk_44: [0; 3], - recovery_target: u32::MAX, - unk_54: [0; 3], - entries: ITEM_RING_ENTRIES, - unk_64: [0; 7], - } - } - - /// Moves the cursors of an empty queue to item-ring position `base`. - /// - /// Used when a render queue pair that was kicked before is handed to a new - /// owner, with `base` the position the host tracks for the new owner's - /// queues. Only valid while the firmware does not run the queue, with all - /// three cursors zero. Writes `done`, `unk_30` and `producer`, in this - /// order, with single stores; the caller issues a full barrier afterwards. - pub(crate) fn rebase(&self, base: u32) { - self.done.store(base, Ordering::Relaxed); - self.unk_30.store(base, Ordering::Relaxed); - self.producer.store(base, Ordering::Relaxed); - } -} - -/// Head of a queue's job list. -/// -/// Live state: the firmware links and unlinks jobs; the host writes the head -/// only at queue creation. -#[repr(C)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct JobListHead { - /// GPU address of the first job, or zero. - pub(crate) first_va: u64, - /// GPU address of the last link: the head itself when the list is empty. - pub(crate) last_va: u64, -} - -impl JobListHead { - /// Returns an empty list whose head is at `self_va`. - pub(crate) const fn new(self_va: u64) -> Self { - Self { - first_va: 0, - last_va: self_va, - } - } -} - -/// Addresses and owner a [`QueueRecord`] names. -pub(crate) struct QueueRecordArgs { - /// GPU address of the queue's [`PointerBlock`]. - pub(crate) pointers_va: u64, - /// GPU address of the queue's item ring. - pub(crate) items_va: u64, - /// GPU address of the job list the queue's work is linked into. - pub(crate) job_list_va: u64, - /// GPU address of the owning logical queue's scheduler-state page. - pub(crate) scheduler_va: u64, - /// Thread-group ID of the owning process (0 while unowned). - pub(crate) owner_pid: u32, - /// Scheduling profile of the owning logical queue. - pub(crate) policy: Policy, -} - -/// Description of one hardware queue, named by its work-ring slots. -/// -/// The host writes it whole at queue creation and changes the owner fields of -/// an idle queue with [`QueueRecord::set_compute_owner`] or, for render -/// queues, field by field (`render/memory.rs`). -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct QueueRecord { - /// GPU address of the [`PointerBlock`]. - pub(crate) pointers_va: u64, - /// GPU address of the item ring. - pub(crate) items_va: u64, - /// GPU address of the job list. - pub(crate) job_list_va: u64, - pub(crate) unk_18: [u8; 0xc], - pub(crate) unk_24: u32, - /// Queue priority, stored twice. - pub(crate) priority: [u32; 2], - pub(crate) unk_30: [u8; 6], - pub(crate) unk_36: u16, - /// Zero at creation; cleared again when a compute queue changes owner. - pub(crate) unk_38: u32, - pub(crate) unk_3c: u32, - /// Scheduling policy. - pub(crate) policy: u32, - pub(crate) unk_44: u32, - /// Thread-group ID of the owning process. - pub(crate) owner_pid: u32, - pub(crate) unk_4c: [u8; 0x50], - /// GPU address of the owner's scheduler-state page. - pub(crate) scheduler_va: u64, - pub(crate) unk_a4: [u8; 0x1c], -} - -static_assert!(size_of::() == 0xc0); -static_assert!(core::mem::offset_of!(QueueRecord, priority) == 0x28); -static_assert!(core::mem::offset_of!(QueueRecord, unk_38) == 0x38); -static_assert!(core::mem::offset_of!(QueueRecord, policy) == 0x40); -static_assert!(core::mem::offset_of!(QueueRecord, scheduler_va) == 0x9c); - -impl QueueRecord { - /// Builds the record of a newly created queue. - pub(crate) fn new(args: &QueueRecordArgs) -> Self { - Self { - pointers_va: args.pointers_va, - items_va: args.items_va, - job_list_va: args.job_list_va, - unk_18: [0; 0xc], - unk_24: u32::MAX, - priority: [args.policy.priority() as u32; 2], - unk_30: [0; 6], - unk_36: u16::MAX, - unk_38: 0, - unk_3c: 0, - policy: args.policy.policy() as u32, - unk_44: u32::MAX, - owner_pid: args.owner_pid, - unk_4c: [0; 0x50], - scheduler_va: args.scheduler_va, - unk_a4: [0; 0x1c], - } - } - - /// Hands an idle compute queue to a new owner. - /// - /// Replaces the job head, process identity and scheduler page, and clears `unk_38`. - /// The priority and policy words are written only when `policy` differs from the - /// profile the record holds. - pub(crate) fn set_compute_owner( - &mut self, - job_list_va: u64, - scheduler_va: u64, - owner_pid: u32, - policy: Policy, - ) { - self.job_list_va = job_list_va; - self.unk_38 = 0; - self.owner_pid = owner_pid; - self.scheduler_va = scheduler_va; - self.set_policy(policy); - } - - /// Replaces the priority and policy words of an idle queue's record when they differ - /// from `policy`. - pub(crate) fn set_policy(&mut self, policy: Policy) { - let priority = [policy.priority() as u32; 2]; - if { self.priority } != priority { - self.priority = priority; - } - if { self.policy } != policy.policy() as u32 { - self.policy = policy.policy() as u32; - } - } -} - -/// Free list a queue's USC private memory is allocated from. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct FreeListBinding { - /// GPU address of the free list's shared control object. - pub(crate) va: u64, - /// Generation of the free list's registration. - pub(crate) generation: u64, - /// Buffer slot of the free list. - pub(crate) slot: u32, -} - -/// Parameter-buffer binding of a tiling queue. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct TilingBinding { - /// GPU address of the parameter-buffer state object. - pub(crate) pb_va: u64, - /// Allocation token of the parameter buffer's buffer slot. - pub(crate) pb_token: u64, - /// Buffer slot of the parameter buffer. - pub(crate) pb_slot: u32, - /// QID of the fragment queue paired with this tiling queue. - pub(crate) fragment_qid: u8, -} - -/// Queue whose configuration a [`QueueConfig`] carries. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum QueueConfigTarget { - /// A tiling queue and its parameter-buffer binding. - Tiling(TilingBinding), - /// A fragment queue. - Fragment, - /// A compute queue. - Compute, -} - -/// Inputs of one [`QueueConfig`]. -pub(crate) struct QueueConfigArgs { - /// Kind of queue. - pub(crate) target: QueueConfigTarget, - /// GPU address of the queue's kick ring. - pub(crate) kick_ring_va: u64, - /// Firmware alias of the queue's kick ring. - pub(crate) kick_ring_fw_va: u64, - /// Configured queue. - pub(crate) qid: u8, - /// The firmware has not yet installed this QID. - pub(crate) install: bool, - /// Completion value the firmware loads for the queue, or `None` to keep - /// the value it holds. - pub(crate) completion_seed: Option, - /// UAT context the queue's work runs in. - pub(crate) context_id: u16, - /// USC private-memory free list. - pub(crate) free_list: FreeListBinding, - /// GPU address of the owner's scheduler-state page. - pub(crate) scheduler_va: u64, - /// Scheduling profile of the owning logical queue. - pub(crate) policy: Policy, - /// QoS buffer slot of the logical queue. - pub(crate) qos_slot: u8, - /// The QoS identity or queue membership changed since the last update. - pub(crate) qos_update: bool, - /// Thread-group ID of the owning process. - pub(crate) owner_pid: u32, - /// The queue's UAT context changed since the last update. - pub(crate) context_update: bool, -} - -/// Configuration of one hardware queue (tag 15). -/// -/// The firmware applies it when it consumes the command, which may be after -/// later kicks were published, so a record must not be rewritten while a -/// work pointer names it. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct QueueConfig { - /// [`CommandTag::QueueConfig`]. - pub(crate) tag: u32, - pub(crate) unk_04: u32, - /// GPU address of the queue's kick ring. - pub(crate) kick_ring_va: u64, - /// Firmware alias of the queue's kick ring. - pub(crate) kick_ring_fw_va: u64, - /// Configured queue. - pub(crate) qid: u16, - /// Non-zero to install the QID. - pub(crate) install: u16, - pub(crate) unk_1c: u16, - /// Queue priority. - pub(crate) priority: u16, - pub(crate) unk_20: u16, - /// [`DataMaster`] of the queue. - pub(crate) data_master: u16, - pub(crate) unk_24: u16, - /// Non-zero when `completion_seed` is valid. - pub(crate) completion_seed_valid: u32, - /// Completion value: kick timestamp in bits 39:0, render-pass sequence in - /// bits 47:40 (see [`KickTimestamp::completion_value`]). - pub(crate) completion_seed: u64, - /// UAT context of the queue's work. - pub(crate) context_id: u16, - pub(crate) unk_34: u16, - /// GPU address of the USC free list's shared control object. - pub(crate) free_list_va: u64, - /// Generation of the USC free list. - pub(crate) free_list_generation: u64, - /// Buffer slot of the USC free list. - pub(crate) free_list_slot: u32, - /// GPU address of the owner's scheduler-state page. - pub(crate) scheduler_va: u64, - /// Non-zero when the QoS identity or membership changed. - pub(crate) qos_update: u32, - /// QoS buffer slot. - pub(crate) qos_slot: u16, - pub(crate) unk_58: u16, - /// Thread-group ID of the owning process. - pub(crate) owner_pid: u32, - /// Scheduling policy. - pub(crate) policy: u16, - pub(crate) unk_60: u16, - /// Non-zero when the UAT context changed. - pub(crate) context_update: u16, - pub(crate) unk_64: u16, - pub(crate) unk_66: u16, - pub(crate) unk_68: [u8; 6], - /// Tiling queues: GPU address of the parameter-buffer state; else zero. - pub(crate) pb_va: u64, - /// Tiling queues: parameter-buffer slot allocation token; else all ones. - pub(crate) pb_token: u64, - /// Tiling queues: parameter-buffer slot; else all ones. - pub(crate) pb_slot: u32, - /// Tiling queues: QID of the paired fragment queue; else all ones. - pub(crate) fragment_qid: u32, - pub(crate) unk_86: [u8; 0x3a], -} - -static_assert!(size_of::() == 0xc0); -static_assert!(core::mem::offset_of!(QueueConfig, completion_seed_valid) == 0x26); -static_assert!(core::mem::offset_of!(QueueConfig, free_list_va) == 0x36); -static_assert!(core::mem::offset_of!(QueueConfig, scheduler_va) == 0x4a); -static_assert!(core::mem::offset_of!(QueueConfig, qos_slot) == 0x56); -static_assert!(core::mem::offset_of!(QueueConfig, owner_pid) == 0x5a); -static_assert!(core::mem::offset_of!(QueueConfig, pb_va) == 0x6e); -static_assert!(core::mem::offset_of!(QueueConfig, fragment_qid) == 0x82); - -impl QueueConfig { - const UNK_66: u16 = 1; - - /// Builds the record described by `args`. - pub(crate) fn new(args: &QueueConfigArgs) -> Result { - if args.qid > QID_MAX { - return Err(EINVAL); - } - let (data_master, pb_va, pb_token, pb_slot, fragment_qid) = match args.target { - QueueConfigTarget::Tiling(binding) => { - if binding.fragment_qid > QID_MAX { - return Err(EINVAL); - } - ( - DataMaster::Tiling, - binding.pb_va, - binding.pb_token, - binding.pb_slot, - binding.fragment_qid as u32, - ) - } - QueueConfigTarget::Fragment => (DataMaster::Fragment, 0, u64::MAX, u32::MAX, u32::MAX), - QueueConfigTarget::Compute => (DataMaster::Compute, 0, u64::MAX, u32::MAX, u32::MAX), - }; - Ok(Self { - tag: CommandTag::QueueConfig as u32, - unk_04: 0, - kick_ring_va: args.kick_ring_va, - kick_ring_fw_va: args.kick_ring_fw_va, - qid: args.qid as u16, - install: args.install as u16, - unk_1c: 0, - priority: args.policy.priority() as u16, - unk_20: 0, - data_master: data_master as u16, - unk_24: 0, - completion_seed_valid: args.completion_seed.is_some() as u32, - completion_seed: args.completion_seed.unwrap_or(0), - context_id: args.context_id, - unk_34: 0, - free_list_va: args.free_list.va, - free_list_generation: args.free_list.generation, - free_list_slot: args.free_list.slot, - scheduler_va: args.scheduler_va, - qos_update: args.qos_update as u32, - qos_slot: args.qos_slot as u16, - unk_58: 0, - owner_pid: args.owner_pid, - policy: args.policy.policy(), - unk_60: 0, - context_update: args.context_update as u16, - unk_64: 0, - unk_66: Self::UNK_66, - unk_68: [0; 6], - pb_va, - pb_token, - pb_slot, - fragment_qid, - unk_86: [0; 0x3a], - }) - } -} - -/// Announcement of a newly written kick entry of a queue (tag 14). -/// -/// A record slot larger than this structure must be zero beyond it. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct KickAnnounce { - /// [`CommandTag::KickAnnounce`]. - pub(crate) tag: u32, - /// Kicked queue. - pub(crate) qid: u16, - /// Number of entries announced. - pub(crate) count: u16, - /// Queue priority. - pub(crate) priority: u8, - /// Timestamp of the announced entry. - pub(crate) timestamp: u64, - /// [`DataMaster`] of the queue. - pub(crate) data_master: u32, - pub(crate) unk_15: [u8; 3], -} - -static_assert!(size_of::() == 0x18); -static_assert!(core::mem::offset_of!(KickAnnounce, timestamp) == 0x09); -static_assert!(core::mem::offset_of!(KickAnnounce, data_master) == 0x11); - -impl KickAnnounce { - /// Announces the entry of queue `qid` at priority class `priority`, - /// written at `timestamp`. - pub(crate) fn new( - qid: u8, - data_master: DataMaster, - timestamp: KickTimestamp, - priority: u8, - ) -> Result { - if qid > QID_MAX || priority >= Policy::PRIORITIES { - return Err(EINVAL); - } - Ok(Self { - tag: CommandTag::KickAnnounce as u32, - qid: qid as u16, - count: 1, - priority, - timestamp: timestamp.get(), - data_master: data_master as u32, - unk_15: [0; 3], - }) - } -} - -/// Names the kick of a compute queue that precedes the announced one -/// (tag 16). -/// -/// A record slot larger than this structure must be zero beyond it. -#[repr(C, packed)] -#[derive(Copy, Clone, Debug)] -pub(crate) struct KickPredecessor { - /// [`CommandTag::KickPredecessor`]. - pub(crate) tag: u32, - /// Kicked queue. - pub(crate) qid: u16, - /// Timestamp of the queue's kick preceding the announced one. - pub(crate) previous: u64, - pub(crate) unk_0e: u32, -} - -static_assert!(size_of::() == 0x12); - -impl KickPredecessor { - const UNK_0E: u32 = 0xfe; - - /// Names `previous` as the kick of queue `qid` preceding the announced - /// one. - pub(crate) fn new(qid: u8, previous: KickTimestamp) -> Result { - if qid > QID_MAX { - return Err(EINVAL); - } - Ok(Self { - tag: CommandTag::KickPredecessor as u32, - qid: qid as u16, - previous: previous.get(), - unk_0e: Self::UNK_0E, - }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/render.rs b/drivers/gpu/drm/asahi-neo/g17/fw/render.rs deleted file mode 100644 index 23eaa7e7dfba9e..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/render.rs +++ /dev/null @@ -1,1329 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Render work descriptors. -//! -//! A render pass is kicked as a tiling (TA) descriptor on one queue and a -//! fragment (3D) descriptor on the queue paired with it. Each descriptor -//! embeds the register arrays its kick entry binds: one for tiling, and for -//! fragment the main array followed by the partial-render store, resume and -//! load arrays. -//! -//! Descriptors are built in place with [`TaDescriptor::write`] and -//! [`FragmentSlot::write`]; the completion link and the fragment MCache table -//! are added just before publication. - -pub(crate) mod storage; - -use kernel::prelude::*; - -use super::kick::{ - KickTimestamp, McacheClass, McacheRange, McacheTable, RegisterArray, RegisterArrayBinding, - RegisterWrite, WriteRange, MCACHE_INLINE_RANGES, -}; -use super::queue::CommandTag; -use super::{ - clear, offset_va, sampler_max, stamp_value, timestamp_end_va, work_key, ObjectIds, UNK_0A599, - UNK_0D411, UNK_101D9, UNK_10791, UNK_1A0E9, UNK_1C8F8, -}; - -/// Size of a tiling descriptor. -pub(crate) const TA_DESCRIPTOR_SIZE: usize = 0x9c0; -/// Size of a [`FragmentSlot`]: the descriptor and its MCache spill table. -pub(crate) const FRAGMENT_SLOT_SIZE: usize = 0x2380; -/// Write ranges of one fragment pass: its attachments and the auxiliary -/// framebuffer. -pub(crate) const MCACHE_RANGES_MAX: usize = 17; -/// Size of the auxiliary framebuffer of a pass. -pub(crate) const AUX_FB_SIZE: u64 = 0x8000; - -/// QoS class of tiling kicks. -pub(crate) const TA_QOS_CLASS: u8 = 0x0c; -/// QoS class of fragment kicks. -pub(crate) const FRAGMENT_QOS_CLASS: u8 = 0x18; -/// Event mask of render kicks. -pub(crate) const RENDER_KICK_EVENT_MASK: [u64; 4] = [0; 4]; - -/// Base of the user address window. Tiling resource registers hold offsets -/// from it rather than addresses. -const USER_BASE: u64 = 0x10_0000_0000; -/// The VDM control-stream register holds a 39-bit offset from [`USER_BASE`]. -const VDM_OFFSET_LIMIT: u64 = 1 << 39; -/// The deflake buffer holds three sub-buffers: at its start and at these -/// offsets. -const DEFLAKE_20: u64 = 0x20; -const DEFLAKE_2A0: u64 = 0x2a0; - -const TA_REGISTERS: usize = 73; -const FRAGMENT_REGISTERS: usize = 89; -const PARTIAL_STORE_REGISTERS: usize = 16; -const PARTIAL_RESUME_REGISTERS: usize = 23; -const PARTIAL_LOAD_REGISTERS: usize = 10; - -/// Checkpoints into the tiling array, as entry counts: before the final two -/// entries, then after the last one (twice). -const TA_CHECKPOINTS: [u16; 3] = [ - TA_REGISTERS as u16 - 2, - TA_REGISTERS as u16, - TA_REGISTERS as u16, -]; -/// Checkpoints into the main fragment array, as entry counts: after the first -/// of the four closing entries, then after the second (twice). -const FRAGMENT_CHECKPOINTS: [u16; 3] = [ - FRAGMENT_REGISTERS as u16 - 3, - FRAGMENT_REGISTERS as u16 - 2, - FRAGMENT_REGISTERS as u16 - 2, -]; - -const TILE_SIZE: u64 = 32; -const MACRO_TILES: u64 = 4; -const REGION_ENTRY_SIZE: u64 = 5; -/// Highest number of samples per pixel, as a power of two. -const SAMPLES_LOG2_MAX: u8 = 2; -/// Bytes of tile buffer per block. -const TIB_BLOCK_SIZE: u64 = 2048; -const TILE_CONFIG_BASE: u64 = 0x280; -const TILE_CONFIG_LAYERED: u64 = 1 << 0; -const TILE_CONFIG_PROCESS_EMPTY_TILES: u64 = 1 << 16; -const AUX_FB_FLAGS_BASE: u64 = 0xc000; -/// The pass rasterizes at most four known primitives (userspace's promise). -const AUX_FB_FLAGS_FEW_PRIMITIVES: u64 = 1 << 0; -const AUX_FB_FLAGS_DBIAS_INT: u64 = 1 << 18; -/// Tiling layer mode: layer count less one, bit 15, and for layered targets -/// bit 14 plus the empty-tile policy. The second copy keeps only the count and -/// bit 14. -const LAYER_MODE_BASE: u64 = 1 << 15; -const LAYER_MODE_LAYERED: u64 = 1 << 14; -const LAYER_MODE_PROCESS_EMPTY: u64 = 1 << 13; -const LAYER_MODE_SKIP_EMPTY: u64 = 1 << 12; -const LAYER_MODE_PIPE_MASK: u64 = 0x47ff; -/// Fragment tile state: required bits 0x3717f and 37, layered targets in bit -/// 32, the utile size bits of the tile-buffer configuration in bits 43:40, -/// tiles in X less one from bit 44 and tiles in Y less one from bit 53. -const TILE_STATE_BASE: u64 = 0x3717f | 1 << 37; -const TILE_STATE_LAYERED: u64 = 1 << 32; -const TILE_STATE_UTILE_SHIFT: u32 = 28; -const TILE_STATE_TILES_X_SHIFT: u32 = 44; -const TILE_STATE_TILES_Y_SHIFT: u32 = 53; -/// Tile-buffer configuration: utile width / 16 in bits 13:12, utile height / -/// 16 in bits 15:14, samples (log2) in bits 1:0. -const UTILE_CONFIG_WIDTH_SHIFT: u32 = 12; -const UTILE_CONFIG_HEIGHT_SHIFT: u32 = 14; -const UTILE_CONFIG_SIZE_MASK: u64 = 0xf000; -/// The partial-render phases load and store every attached depth and stencil -/// surface. -const ZLS_PARTIAL_DEPTH: u64 = 1 << 15 | 1 << 19; -const ZLS_PARTIAL_STENCIL: u64 = 1 << 14 | 1 << 18; -const STENCIL_CLEAR_BASE: u32 = 0x300; -/// Value of the ZLS swizzle registers and descriptor field. -const ZLS_SWIZZLE: u32 = 0x0404_0404; -/// Value of register 0x15359 in the partial store and resume arrays; the -/// main array writes zero. -const UNK_15359_PARTIAL: u64 = 0x20; -/// Required value of register 0x15049 and of fragment descriptor field -/// +0x1f50. -const UNK_15049: u64 = 0x10_0000; -/// Value of the fragment register 0x10791. -const UNK_10791_FRAGMENT: u64 = UNK_10791 | 1 << 9; -/// USC program offsets carry tag bits that are not part of the address. -const USC_OFFSET_TAG_MASK: u32 = 7; - -/// A depth or stencil surface. -#[derive(Copy, Clone, Debug)] -pub(crate) struct ZlsSurface { - /// GPU address of the surface, or zero when it is not attached. - pub(crate) va: u64, - /// GPU address of its compression metadata. - pub(crate) compression_va: u64, - /// Stride of the surface. - pub(crate) stride: u64, - /// Stride of the compression metadata. - pub(crate) compression_stride: u64, -} - -/// A USC program and the resource specifier it is launched with. -#[derive(Copy, Clone, Debug)] -pub(crate) struct UscProgram { - /// Tagged offset of the program in the queue's USC window. - pub(crate) offset: u32, - /// Resource specifier. - pub(crate) rsrc_spec: u64, -} - -/// Render-pass state from the userspace command. -#[derive(Copy, Clone, Debug)] -pub(crate) struct RenderPass { - /// Width of the render target in pixels. - pub(crate) width: u32, - /// Height of the render target in pixels. - pub(crate) height: u32, - /// Number of layers, 1 to 2048. - pub(crate) layers: u16, - /// Utile width: 16 or 32. - pub(crate) utile_width: u8, - /// Utile height: 16 or 32, at most the width. - pub(crate) utile_height: u8, - /// Samples per pixel, as a power of two: 0 to 2. - pub(crate) samples_log2: u8, - /// Bytes of tile buffer per sample. - pub(crate) sample_size: u8, - /// Process tiles no primitive touches. - pub(crate) process_empty_tiles: bool, - /// Multisample control. - pub(crate) msaa_control: u64, - /// PPP control. - pub(crate) ppp_control: u64, - /// GPU address of the VDM control stream. - pub(crate) vdm_va: u64, - /// GPU address of the scissor array. - pub(crate) scissor_va: u64, - /// GPU address of the depth-bias array, or zero. - pub(crate) depth_bias_va: u64, - /// Depth-bias values are integers. - pub(crate) depth_bias_is_int: bool, - /// At most four known primitives; sets the auxiliary framebuffer bit 0. - pub(crate) few_primitives: bool, - /// Host-only: the resolved barriers order the fragment kick, not the tiling kick. - pub(crate) fragment_barriers: bool, - /// GPU address of the occlusion-query results, or zero. - pub(crate) occlusion_query_va: u64, - /// GPU address of the sampler heap, or zero. - pub(crate) sampler_heap_va: u64, - /// Number of samplers in the heap. - pub(crate) sampler_count: u32, - /// ZLS control at the pass boundaries. - pub(crate) zls_control: u64, - /// Depth surface dimensions. - pub(crate) depth_dimensions: u64, - /// Depth surface. - pub(crate) depth: ZlsSurface, - /// Stencil surface. - pub(crate) stencil: ZlsSurface, - /// Depth clear value (IEEE-754 bits). - pub(crate) depth_clear: u32, - /// Stencil clear value (background object register value). - pub(crate) stencil_clear: u32, - /// Merge upper bound in X (IEEE-754 bits). - pub(crate) merge_upper_x: u32, - /// Merge upper bound in Y (IEEE-754 bits). - pub(crate) merge_upper_y: u32, - /// Base of the queue's USC window. - pub(crate) usc_base: u64, - /// Background program. - pub(crate) bg: UscProgram, - /// Partial-render background program. - pub(crate) partial_bg: UscProgram, - /// Tagged USC offset of the end-of-tile program. - pub(crate) eot_offset: u32, - /// Resource specifier of the end-of-tile program. - pub(crate) eot_rsrc_spec: u64, - /// Tagged USC offset of the partial-render end-of-tile program. - pub(crate) partial_eot_offset: u32, - /// Resource specifier of the partial-render end-of-tile program. - pub(crate) partial_eot_rsrc_spec: u64, - /// The resource specifiers have their high 32 bits. - pub(crate) rsrc_spec_hi: bool, -} - -/// Kernel-owned objects and identities a render pass uses. -#[derive(Copy, Clone, Debug)] -pub(crate) struct RenderResources { - /// UAT context the pass runs in. - pub(crate) context_id: u16, - /// Allocation generation of that context. - pub(crate) context_generation: u8, - /// Buffer slot of the parameter buffer. - pub(crate) pb_slot: u32, - /// Buffer slot of the USC private-memory free list. - pub(crate) free_list_slot: u32, - /// GPU address of the free list's shared control object. - pub(crate) free_list_control_va: u64, - /// GPU address of the pass's kick record, in which the firmware counts - /// the pass's completed kicks. - pub(crate) kick_record_va: u64, - /// GPU address of the parameter-buffer state object. - pub(crate) pb_va: u64, - /// GPU address of the parameter-buffer scene entry of the pass. - pub(crate) scene_va: u64, - /// GPU address following the last scene entry. - pub(crate) scene_end_va: u64, - /// Parameter-buffer scratch offset register value. - pub(crate) pm_scratch: u64, - /// Parameter-buffer metric pointer register value. - pub(crate) pm_metric: u64, - /// GPU address of the tile map. - pub(crate) tilemap_va: u64, - /// GPU address of the layer metadata. - pub(crate) layermeta_va: u64, - /// GPU address of the heap metadata. - pub(crate) heapmeta_va: u64, - /// GPU address of the tail-pointer cache. - pub(crate) tpc_va: u64, - /// Size of the tail-pointer cache the target needs. - pub(crate) tpc_size: u64, - /// GPU address of the deflake buffer. - pub(crate) deflake_va: u64, - /// GPU address of the tiling status word. - pub(crate) ta_status_va: u64, - /// GPU address of the fragment status word, as seen by the pass's VM. - pub(crate) fragment_status_va: u64, - /// Firmware alias of the fragment status word. - pub(crate) fragment_status_fw_va: u64, - /// GPU address of the auxiliary framebuffer. - pub(crate) aux_fb_va: u64, - /// GPU address of the pass's start/end timestamp pair. - pub(crate) timestamp_va: u64, - /// Object IDs of the tiling stage. - pub(crate) ta_ids: ObjectIds, - /// Object IDs of the fragment stage. - pub(crate) fragment_ids: ObjectIds, - /// Value of fragment descriptor field +0x2104: bit 0 of GPU register - /// 0xe0141c, read at probe. - pub(crate) unk_2104: u32, - /// Value of fragment descriptor field +0x210c: bits 3:0 of GPU register - /// 0xe01480, read at probe. - pub(crate) unk_210c: u32, -} - -/// Per-stage inputs of a render pass. -#[derive(Copy, Clone, Debug)] -pub(crate) struct RenderStage { - /// Queue the stage is kicked on. - pub(crate) qid: u8, - /// GPU address of the stage's descriptor, as seen by the pass's VM. - pub(crate) descriptor_va: u64, - /// Command-buffer state word of the stage. - pub(crate) state_word: u32, - /// GPU address of the queue's completion stamp word. - pub(crate) stamp_va: u64, - /// GPU address of the queue's word in the auxiliary stamp page. - pub(crate) aux_stamp_va: u64, -} - -/// Inputs of both render descriptors. -pub(crate) struct RenderArgs<'a> { - /// Render-pass state. - pub(crate) pass: &'a RenderPass, - /// Kernel-owned objects. - pub(crate) resources: &'a RenderResources, - /// Submission ordinal of the render queue pair, from zero. - pub(crate) ordinal: u64, - /// Tiling stage. - pub(crate) ta: RenderStage, - /// Fragment stage. - pub(crate) fragment: RenderStage, -} - -impl RenderArgs<'_> { - /// Completion stamp value of this pass: the pass's sequence number is the - /// pair's submission ordinal plus one. - fn stamp_value(&self) -> u32 { - stamp_value((self.ordinal as u32).wrapping_add(1)) - } -} - -/// Tile-grid geometry of a render target, in the units its registers take. -struct Geometry { - tiles_x: u64, - tiles_y: u64, - /// Size of the region array. - region_size: u64, - /// Number of macro tiles. - macro_tiles: u64, - /// Two utile slots per utile of every macro tile. - utile_slots: u64, - /// Macro-tile columns times 3, 2 and 1, in 9-bit fields from bit 0. - x_blocks: u64, - /// Macro-tile rows times 3, 2 and 1, in 9-bit fields from bit 0. - y_blocks: u64, - /// Tiles in Y less one from bit 12, tiles in X less one below. - screen: u64, - /// Height less one from bit 16, width less one below. - pixels: u64, - /// Macro-tile width in utiles from bit 16, height in utiles below. - macro_size: u64, -} - -impl Geometry { - fn new(pass: &RenderPass) -> Result { - if pass.width == 0 || pass.height == 0 || pass.layers == 0 || pass.layers > 2048 { - return Err(EINVAL); - } - if !matches!( - (pass.utile_width, pass.utile_height), - (32, 32) | (32, 16) | (16, 16) - ) { - return Err(EINVAL); - } - let width = pass.width as u64; - let height = pass.height as u64; - let tiles_x = width.div_ceil(TILE_SIZE); - let tiles_y = height.div_ceil(TILE_SIZE); - let utiles_x = TILE_SIZE / pass.utile_width as u64; - let utiles_y = TILE_SIZE / pass.utile_height as u64; - let utiles = utiles_x * utiles_y; - let mtile_x = tiles_x.div_ceil(MACRO_TILES).next_multiple_of(4); - let mtile_y = tiles_y.div_ceil(MACRO_TILES).next_multiple_of(4); - let macro_tiles = mtile_x * mtile_y; - Ok(Self { - tiles_x, - tiles_y, - region_size: (REGION_ENTRY_SIZE * macro_tiles * utiles).div_ceil(4), - macro_tiles, - utile_slots: 2 * macro_tiles * utiles, - x_blocks: (3 * mtile_x) | ((2 * mtile_x) << 9) | (mtile_x << 18), - y_blocks: (3 * mtile_y) | ((2 * mtile_y) << 9) | (mtile_y << 18), - screen: (tiles_y - 1) << 12 | (tiles_x - 1), - pixels: (width - 1) | (height - 1) << 16, - macro_size: (mtile_y * utiles_y) | ((mtile_x * utiles_x) << 16), - }) - } -} - -/// Register and descriptor words derived from the [`RenderPass`]. -struct PassWords { - utile_config: u64, - tile_config: u64, - layer_mode: u64, - tib_blocks: u64, - partial_zls_control: u64, - aux_fb_flags: u64, - bg_rsrc_spec: u64, - bg_va: u64, - partial_bg_rsrc_spec: u64, - partial_bg_va: u64, - eot_va: u64, - eot_rsrc_spec: u64, - partial_eot_va: u64, - partial_eot_rsrc_spec: u64, -} - -impl PassWords { - fn new(pass: &RenderPass) -> Result { - if pass.samples_log2 > SAMPLES_LOG2_MAX { - return Err(EINVAL); - } - let samples = 1u64 << pass.samples_log2; - let layered = pass.layers > 1; - - // The tile configuration carries the empty-tile request for every - // target; the tiling layer mode only for layered ones. - let mut tile_config = TILE_CONFIG_BASE; - let mut layer_mode = LAYER_MODE_BASE | (pass.layers as u64 - 1); - if layered { - tile_config |= TILE_CONFIG_LAYERED; - layer_mode |= LAYER_MODE_LAYERED; - layer_mode |= match pass.process_empty_tiles { - true => LAYER_MODE_PROCESS_EMPTY, - false => LAYER_MODE_SKIP_EMPTY, - }; - } - if pass.process_empty_tiles { - tile_config |= TILE_CONFIG_PROCESS_EMPTY_TILES; - } - - let mut partial_zls_control = pass.zls_control; - if pass.depth.va != 0 { - partial_zls_control |= ZLS_PARTIAL_DEPTH; - } - if pass.stencil.va != 0 { - partial_zls_control |= ZLS_PARTIAL_STENCIL; - } - - let mut aux_fb_flags = AUX_FB_FLAGS_BASE; - if pass.depth_bias_is_int { - aux_fb_flags |= AUX_FB_FLAGS_DBIAS_INT; - } - if pass.few_primitives { - aux_fb_flags |= AUX_FB_FLAGS_FEW_PRIMITIVES; - } - let rsrc_spec = |spec: u64| match pass.rsrc_spec_hi { - true => spec, - false => spec & u32::MAX as u64, - }; - let usc = |offset: u32| offset_va(pass.usc_base, (offset & !USC_OFFSET_TAG_MASK) as u64); - - Ok(Self { - utile_config: (pass.utile_width as u64 / 16) << UTILE_CONFIG_WIDTH_SHIFT - | (pass.utile_height as u64 / 16) << UTILE_CONFIG_HEIGHT_SHIFT - | pass.samples_log2 as u64, - tile_config, - layer_mode, - tib_blocks: (pass.sample_size as u64 - * pass.utile_width as u64 - * pass.utile_height as u64 - * samples) - .div_ceil(TIB_BLOCK_SIZE), - partial_zls_control, - aux_fb_flags, - bg_rsrc_spec: rsrc_spec(pass.bg.rsrc_spec), - bg_va: usc(pass.bg.offset)?, - partial_bg_rsrc_spec: rsrc_spec(pass.partial_bg.rsrc_spec), - partial_bg_va: usc(pass.partial_bg.offset)?, - eot_va: usc(pass.eot_offset)?, - eot_rsrc_spec: rsrc_spec(pass.eot_rsrc_spec), - partial_eot_va: usc(pass.partial_eot_offset)?, - partial_eot_rsrc_spec: rsrc_spec(pass.partial_eot_rsrc_spec), - }) - } -} - -/// Offset of an address from the user window base. -fn user_offset(va: u64) -> Result { - va.checked_sub(USER_BASE).ok_or(EINVAL) -} - -/// Offset of an address from the user window base that fits 32 bits. -fn user_offset32(va: u64) -> Result { - u32::try_from(user_offset(va)?).map_err(|_| EINVAL) -} - -fn ta_registers( - args: &RenderArgs<'_>, - g: &Geometry, - w: &PassWords, -) -> Result<[RegisterWrite; TA_REGISTERS]> { - let p = args.pass; - let r = args.resources; - let tilemap = user_offset(r.tilemap_va)?; - let layermeta = user_offset(r.layermeta_va)?; - let heapmeta = user_offset(r.heapmeta_va)? | 1 << 63; - let tpc = user_offset(r.tpc_va)?; - let deflake = user_offset32(r.deflake_va)? as u64; - let deflake_20 = user_offset32(offset_va(r.deflake_va, DEFLAKE_20)?)? as u64; - let deflake_2a0 = user_offset32(offset_va(r.deflake_va, DEFLAKE_2A0)?)? as u64; - let vdm = user_offset(p.vdm_va)?; - if vdm >= VDM_OFFSET_LIMIT { - return Err(EINVAL); - } - let key = work_key(r.context_id, args.ta.state_word); - let ids = r.ta_ids.word(); - - Ok([ - RegisterWrite::new(0x01748, 1), - RegisterWrite::new(0x10141, 0x200), - RegisterWrite::new(0x1c039, tilemap), - RegisterWrite::new(0x1c9c8, tilemap), - RegisterWrite::new(0x1c0a1, tpc), - RegisterWrite::new(0x1c031, heapmeta), - RegisterWrite::new(0x1c9c0, heapmeta), - RegisterWrite::new(0x1c051, 0x003a_0012_006b_0003), - RegisterWrite::new(0x1c061, 1), - RegisterWrite::new(0x10149, w.utile_config), - RegisterWrite::new(0x10139, p.msaa_control), - RegisterWrite::new(0x10111, deflake_2a0), - RegisterWrite::new(0x1c9b0, deflake_2a0), - RegisterWrite::new(0x10119, deflake_20), - RegisterWrite::new(0x1c9b8, deflake_20), - RegisterWrite::new(0x1c958, 1), - RegisterWrite::new(0x1c950, deflake | 1 << 50), - RegisterWrite::new(0x1c930, 0), - RegisterWrite::new(0x1c880, vdm), - RegisterWrite::new(0x1c079, layermeta), - RegisterWrite::new(0x1c9d8, layermeta), - RegisterWrite::new(0x10151, 0), - RegisterWrite::new(0x1c199, 0), - RegisterWrite::new(0x1c1a1, 0), - RegisterWrite::new(0x1c1a9, 0), - RegisterWrite::new(0x1c1b1, 0), - RegisterWrite::new(0x1c1b9, 0), - RegisterWrite::new(0x1c8f8, UNK_1C8F8), - RegisterWrite::new(0x1c0b1, g.region_size), - RegisterWrite::new(0x1c850, g.region_size), - RegisterWrite::new(0x10131, p.msaa_control), - RegisterWrite::new(0x10121, p.ppp_control), - RegisterWrite::new(0x10129, g.pixels), - RegisterWrite::new(0x101b9, g.screen), - RegisterWrite::new(0x1c069, g.x_blocks), - RegisterWrite::new(0x1c071, g.y_blocks), - RegisterWrite::new(0x1c081, g.macro_tiles), - RegisterWrite::new(0x1c0a9, g.utile_slots), - RegisterWrite::new(0x10171, 0x100), - RegisterWrite::new(0x10169, w.layer_mode), - RegisterWrite::new(0x0a309, 0), - RegisterWrite::new(0x1c8e0, u64::MAX), - RegisterWrite::new(0x1c8e8, u64::MAX), - RegisterWrite::new(0x1c898, 0), - RegisterWrite::new(0x101e1, UNK_101D9), - RegisterWrite::new(0x1c9e8, w.layer_mode & LAYER_MODE_PIPE_MASK), - RegisterWrite::new(0x1a099, 0), - RegisterWrite::new(0x1a0a1, 0), - RegisterWrite::new(0x1a069, 0), - RegisterWrite::new(0x1a071, 0), - RegisterWrite::new(0x1a0c9, 0), - RegisterWrite::new(0x1a0d1, 0), - RegisterWrite::new(0x101c9, 0), - RegisterWrite::new(0x0d471, 0), - RegisterWrite::new(0x1a0f1, UNK_1A0E9), - RegisterWrite::new(0x10799, UNK_10791), - RegisterWrite::new(0x1c830, r.pb_slot as u64), - RegisterWrite::new(0x1ca30, r.pm_scratch), - RegisterWrite::new(0x16c39, r.pm_scratch), - RegisterWrite::new(0x1c910, r.pm_metric), - RegisterWrite::new(0x0a5a1, UNK_0A599), - RegisterWrite::new(0x0d419, UNK_0D411), - RegisterWrite::new(0x1ca10, ids), - RegisterWrite::new(0x014a1, ids), - RegisterWrite::new(0x0a349, ids), - RegisterWrite::new(0x10209, key), - RegisterWrite::new(0x1c9f0, key), - RegisterWrite::new(0x14320, key), - RegisterWrite::new(0x14308, r.free_list_slot as u64), - RegisterWrite::new(0x14318, r.ta_status_va | 1), - RegisterWrite::new(0x01740, 1), - RegisterWrite::new(0x1c880, deflake_2a0), - RegisterWrite::new(0x1c898, 1), - ]) -} - -fn fragment_registers( - args: &RenderArgs<'_>, - g: &Geometry, - w: &PassWords, -) -> [RegisterWrite; FRAGMENT_REGISTERS] { - let p = args.pass; - let r = args.resources; - let mut tile_state = TILE_STATE_BASE - | (w.utile_config & UTILE_CONFIG_SIZE_MASK) << TILE_STATE_UTILE_SHIFT - | (g.tiles_x - 1) << TILE_STATE_TILES_X_SHIFT - | (g.tiles_y - 1) << TILE_STATE_TILES_Y_SHIFT; - if p.layers > 1 { - tile_state |= TILE_STATE_LAYERED; - } - let key = work_key(r.context_id, args.fragment.state_word); - let ids = r.fragment_ids.word(); - - [ - RegisterWrite::new(0x01739, 1), - RegisterWrite::new(0x10009, w.utile_config), - // End-of-tile resource specifier. - RegisterWrite::new(0x15379, w.eot_rsrc_spec), - RegisterWrite::new(0x15381, w.eot_va), - RegisterWrite::new(0x15369, w.bg_rsrc_spec), - RegisterWrite::new(0x15371, w.bg_va), - RegisterWrite::new(0x15131, p.merge_upper_x as u64), - RegisterWrite::new(0x15139, p.merge_upper_y as u64), - RegisterWrite::new(0x100a1, 0), - RegisterWrite::new(0x15069, 0), - RegisterWrite::new(0x15071, 0), - RegisterWrite::new(0x16058, 0), - RegisterWrite::new(0x10019, p.msaa_control), - RegisterWrite::new(0x100b1, g.macro_size), - RegisterWrite::new(0x16030, g.macro_size), - RegisterWrite::new(0x100d9, g.screen), - RegisterWrite::new(0x0a301, 0), - RegisterWrite::new(0x10791, UNK_10791_FRAGMENT), - RegisterWrite::new(0x16098, r.heapmeta_va), - RegisterWrite::new(0x15109, p.scissor_va), - RegisterWrite::new(0x15101, p.depth_bias_va), - RegisterWrite::new(0x15021, w.aux_fb_flags), - RegisterWrite::new(0x15211, (p.height as u64) << 32 | p.width as u64), - RegisterWrite::new(0x15049, UNK_15049), - RegisterWrite::new(0x10051, w.tib_blocks), - RegisterWrite::new(0x15321, p.depth_dimensions), - RegisterWrite::new(0x15301, p.depth_clear as u64), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), - RegisterWrite::new(0x15311, p.occlusion_query_va), - RegisterWrite::new(0x15319, p.zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), - RegisterWrite::new(0x15351, 0), - RegisterWrite::new(0x15329, p.depth.va), - RegisterWrite::new(0x15331, p.depth.va), - RegisterWrite::new(0x15339, p.stencil.va), - RegisterWrite::new(0x15341, p.stencil.va), - RegisterWrite::new(0x15231, 0), - RegisterWrite::new(0x15221, 0), - RegisterWrite::new(0x15239, 0), - RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15401, p.depth.stride), - RegisterWrite::new(0x15421, p.depth.stride), - RegisterWrite::new(0x15409, p.stencil.stride), - RegisterWrite::new(0x15429, p.stencil.stride), - RegisterWrite::new(0x153c1, p.depth.compression_va), - RegisterWrite::new(0x15411, p.depth.compression_stride), - RegisterWrite::new(0x153c9, p.depth.compression_va), - RegisterWrite::new(0x15431, p.depth.compression_stride), - RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15419, p.stencil.compression_stride), - RegisterWrite::new(0x153d9, p.stencil.compression_va), - RegisterWrite::new(0x15439, p.stencil.compression_stride), - RegisterWrite::new(0x16429, r.tilemap_va), - RegisterWrite::new(0x16060, r.layermeta_va), - RegisterWrite::new(0x16431, (4 * g.region_size) << 24), - RegisterWrite::new(0x10039, w.tile_config), - RegisterWrite::new(0x16020, 0), - RegisterWrite::new(0x16451, 0), - RegisterWrite::new(0x15359, 0), - RegisterWrite::new(0x100b8, UNK_1C8F8), - RegisterWrite::new(0x16461, r.aux_fb_va), - RegisterWrite::new(0x16090, r.aux_fb_va), - RegisterWrite::new(0x101e9, UNK_101D9), - RegisterWrite::new(0x160a8, 0), - RegisterWrite::new(0x16068, tile_state), - RegisterWrite::new(0x1a0a9, 0), - RegisterWrite::new(0x1a0b1, 0), - RegisterWrite::new(0x1a079, 0), - RegisterWrite::new(0x1a081, 0), - RegisterWrite::new(0x1a0d9, 0), - RegisterWrite::new(0x1a0e1, 0), - RegisterWrite::new(0x101c1, 0), - RegisterWrite::new(0x0d469, 0), - RegisterWrite::new(0x1a0f9, UNK_1A0E9), - RegisterWrite::new(0x0a5a9, UNK_0A599), - RegisterWrite::new(0x0d429, UNK_0D411), - RegisterWrite::new(0x160e0, ids), - RegisterWrite::new(0x01499, ids), - RegisterWrite::new(0x0a341, ids), - RegisterWrite::new(0x1c838, r.pb_slot as u64), - RegisterWrite::new(0x1ca28, r.pm_scratch), - RegisterWrite::new(0x10211, key), - RegisterWrite::new(0x10420, key), - RegisterWrite::new(0x14048, r.free_list_slot as u64), - RegisterWrite::new(0x14080, r.fragment_status_va | 1), - RegisterWrite::new(0x01731, 1), - RegisterWrite::new(0x16020, 1), - RegisterWrite::new(0x16020, 0), - RegisterWrite::new(0x16068, 0x4_0000), - ] -} - -fn partial_store_registers( - p: &RenderPass, - w: &PassWords, -) -> [RegisterWrite; PARTIAL_STORE_REGISTERS] { - [ - // End-of-tile resource specifier. - RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), - RegisterWrite::new(0x15381, w.partial_eot_va), - RegisterWrite::new(0x10039, w.tile_config), - RegisterWrite::new(0x15359, UNK_15359_PARTIAL), - RegisterWrite::new(0x15331, p.depth.va), - RegisterWrite::new(0x153c9, p.depth.compression_va), - RegisterWrite::new(0x15341, p.stencil.va), - RegisterWrite::new(0x153d9, p.stencil.compression_va), - RegisterWrite::new(0x15421, p.depth.stride), - RegisterWrite::new(0x15431, p.depth.compression_stride), - RegisterWrite::new(0x15429, p.stencil.stride), - RegisterWrite::new(0x15439, p.stencil.compression_stride), - RegisterWrite::new(0x15221, 0), - RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), - ] -} - -fn partial_resume_registers( - p: &RenderPass, - w: &PassWords, -) -> [RegisterWrite; PARTIAL_RESUME_REGISTERS] { - [ - // End-of-tile resource specifier. - RegisterWrite::new(0x15379, w.partial_eot_rsrc_spec), - RegisterWrite::new(0x15381, w.partial_eot_va), - RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), - RegisterWrite::new(0x15371, w.partial_bg_va), - RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15359, UNK_15359_PARTIAL), - RegisterWrite::new(0x15331, p.depth.va), - RegisterWrite::new(0x153c9, p.depth.compression_va), - RegisterWrite::new(0x15341, p.stencil.va), - RegisterWrite::new(0x153d9, p.stencil.compression_va), - RegisterWrite::new(0x15421, p.depth.stride), - RegisterWrite::new(0x15431, p.depth.compression_stride), - RegisterWrite::new(0x15429, p.stencil.stride), - RegisterWrite::new(0x15439, p.stencil.compression_stride), - RegisterWrite::new(0x15221, 0), - RegisterWrite::new(0x15229, 0), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), - RegisterWrite::new(0x15329, p.depth.va), - RegisterWrite::new(0x153c1, p.depth.compression_va), - RegisterWrite::new(0x15339, p.stencil.va), - RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), - ] -} - -fn partial_load_registers( - p: &RenderPass, - w: &PassWords, -) -> [RegisterWrite; PARTIAL_LOAD_REGISTERS] { - [ - RegisterWrite::new(0x15369, w.partial_bg_rsrc_spec), - RegisterWrite::new(0x15371, w.partial_bg_va), - RegisterWrite::new(0x10039, w.tile_config & !TILE_CONFIG_PROCESS_EMPTY_TILES), - RegisterWrite::new(0x15309, (p.stencil_clear | STENCIL_CLEAR_BASE) as u64), - RegisterWrite::new(0x15329, p.depth.va), - RegisterWrite::new(0x153c1, p.depth.compression_va), - RegisterWrite::new(0x15339, p.stencil.va), - RegisterWrite::new(0x153d1, p.stencil.compression_va), - RegisterWrite::new(0x15319, w.partial_zls_control), - RegisterWrite::new(0x15349, ZLS_SWIZZLE as u64), - ] -} - -/// Tiling (TA) work descriptor (tag 0). -#[repr(C, packed)] -#[derive(Debug)] -pub(crate) struct TaDescriptor { - /// [`CommandTag::Tiling`]. - pub(crate) tag: u32, - /// Twice the pair's submission ordinal, plus one. - pub(crate) submit_sequence: u64, - /// UAT context of the pass. - pub(crate) context_id: u32, - /// GPU address of the pass's kick record. - pub(crate) kick_record_va: u64, - /// Buffer slot of the parameter buffer. - pub(crate) pb_slot: u32, - pub(crate) unk_1c: u32, - /// GPU address of the parameter-buffer state object. - pub(crate) pb_va: u64, - /// GPU address of the parameter-buffer scene entry. - pub(crate) scene_va: u64, - /// GPU address following the last scene entry. - pub(crate) scene_end_va: u64, - /// Checkpoints into the register array. - pub(crate) checkpoints: [u16; 3], - pub(crate) unk_3e: [u8; 0xa], - /// The pair's submission ordinal plus half of it. - pub(crate) unk_48: u32, - pub(crate) unk_4c: [u8; 0x14], - /// Register array. - pub(crate) registers: RegisterArray, - /// GPU address of the tail-pointer cache. - pub(crate) tpc_va: u64, - /// Size of the tail-pointer cache the target needs. - pub(crate) tpc_size: u64, - pub(crate) unk_790: [u8; 0xc], - /// QID of the fragment queue that consumes this pass. - pub(crate) fragment_qid: u32, - /// Completion stamp value of the pass. - pub(crate) stamp_value: u32, - /// Non-zero when the fragment queue had earlier kicks outstanding. - pub(crate) fragment_busy: u32, - /// Timestamp of the fragment kick of this pass. - pub(crate) fragment_kick: u64, - /// Render-pass sequence of the pass, stored twice. - pub(crate) pass_seq: [u8; 2], - pub(crate) unk_7b2: [u8; 0x24], - /// Offset from the user base of the deflake sub-buffer at +0x2a0. - pub(crate) deflake_2a0_offset: u32, - pub(crate) unk_7da: [u8; 0x94], - /// Predecessor object ID of the tiling stage. - pub(crate) predecessor_id: u32, - pub(crate) unk_872: u32, - pub(crate) unk_876: u32, - /// GPU address of the sampler heap. - pub(crate) sampler_heap_va: u64, - /// Number of samplers. - pub(crate) sampler_count: u32, - /// Number of samplers plus one, or zero without samplers. - pub(crate) sampler_max: u32, - pub(crate) unk_88a: [u8; 8], - pub(crate) unk_892: u32, - pub(crate) unk_896: [u8; 0x10], - /// GPU address of the tiling queue's completion stamp word. - pub(crate) stamp_va: u64, - /// GPU address of the tiling queue's auxiliary stamp word. - pub(crate) aux_stamp_va: u64, - /// Completion stamp value of the pass. - pub(crate) stamp_value_2: u32, - /// QID of the tiling queue. - pub(crate) qid: u32, - pub(crate) unk_8be: u32, - pub(crate) unk_8c2: u32, - /// Completion value of the tiling kick (see - /// [`KickTimestamp::completion_value`]). - pub(crate) completion: u64, - /// Current object ID of the tiling stage. - pub(crate) current_id: u64, - /// Submission ordinal of the pair. - pub(crate) ordinal: u32, - pub(crate) unk_8da: [u8; 0x24], - /// GPU address the pass start timestamp is written to. - pub(crate) timestamp_start_va: u64, - /// Zero: the tiling stage writes no end timestamp. - pub(crate) timestamp_end_va: u64, - pub(crate) unk_90e: [u8; 0x24], - pub(crate) unk_932: u8, - pub(crate) unk_933: u8, - /// GPU address of the USC free list's shared control object. - pub(crate) free_list_control_va: u64, - pub(crate) unk_93c: u8, - pub(crate) unk_93d: u8, - /// Set on the first kick of a queue pair. - pub(crate) unk_93e: [u8; 2], - pub(crate) unk_940: [u8; 5], - /// GPU address of the tiling status word. - pub(crate) status_va: u64, - /// Allocation generation of the UAT context. - pub(crate) context_generation: u8, - pub(crate) unk_94e: [u8; 0x72], -} - -static_assert!(size_of::() == TA_DESCRIPTOR_SIZE); -static_assert!(core::mem::offset_of!(TaDescriptor, unk_48) == 0x48); -static_assert!(core::mem::offset_of!(TaDescriptor, registers) == 0x60); -static_assert!(core::mem::offset_of!(TaDescriptor, tpc_va) == 0x780); -static_assert!(core::mem::offset_of!(TaDescriptor, fragment_qid) == 0x79c); -static_assert!(core::mem::offset_of!(TaDescriptor, deflake_2a0_offset) == 0x7d6); -static_assert!(core::mem::offset_of!(TaDescriptor, predecessor_id) == 0x86e); -static_assert!(core::mem::offset_of!(TaDescriptor, stamp_va) == 0x8a6); -static_assert!(core::mem::offset_of!(TaDescriptor, completion) == 0x8c6); -static_assert!(core::mem::offset_of!(TaDescriptor, timestamp_start_va) == 0x8fe); -static_assert!(core::mem::offset_of!(TaDescriptor, free_list_control_va) == 0x934); -static_assert!(core::mem::offset_of!(TaDescriptor, status_va) == 0x945); - -// SAFETY: `TaDescriptor` consists of integers and arrays of integers only. -unsafe impl Zeroable for TaDescriptor {} - -impl TaDescriptor { - const ARRAY_OFFSET: u64 = core::mem::offset_of!(Self, registers) as u64; - const UNK_876: u32 = u32::MAX; - const UNK_892: u32 = 1; - const UNK_8C2: u32 = 1; - const UNK_932: u8 = 0x44; - const UNK_93C: u8 = 1; - const UNK_93E_FIRST_KICK: [u8; 2] = [0xd0, 0x91]; - - /// Writes the complete descriptor except the completion link. - pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { - let pass = args.pass; - let resources = args.resources; - let geometry = Geometry::new(pass)?; - let words = PassWords::new(pass)?; - let registers = ta_registers(args, &geometry, &words)?; - let deflake_2a0_offset = user_offset32(offset_va(resources.deflake_va, DEFLAKE_2A0)?)?; - let array_va = offset_va(args.ta.descriptor_va, Self::ARRAY_OFFSET)?; - let sampler_max = sampler_max(pass.sampler_count)?; - let stamp_value = args.stamp_value(); - - clear(self); - self.tag = CommandTag::Tiling as u32; - self.submit_sequence = args.ordinal.wrapping_mul(2).wrapping_add(1); - self.context_id = resources.context_id.into(); - self.kick_record_va = resources.kick_record_va; - self.pb_slot = resources.pb_slot; - self.pb_va = resources.pb_va; - self.scene_va = resources.scene_va; - self.scene_end_va = resources.scene_end_va; - self.checkpoints = TA_CHECKPOINTS; - self.unk_48 = (args.ordinal as u32).wrapping_add((args.ordinal / 2) as u32); - self.registers.set(array_va, ®isters)?; - self.tpc_va = resources.tpc_va; - self.tpc_size = resources.tpc_size; - self.fragment_qid = args.fragment.qid.into(); - self.stamp_value = stamp_value; - self.deflake_2a0_offset = deflake_2a0_offset; - self.predecessor_id = resources.ta_ids.predecessor; - self.unk_876 = Self::UNK_876; - self.sampler_heap_va = pass.sampler_heap_va; - self.sampler_count = pass.sampler_count; - self.sampler_max = sampler_max; - self.unk_892 = Self::UNK_892; - self.stamp_va = args.ta.stamp_va; - self.aux_stamp_va = args.ta.aux_stamp_va; - self.stamp_value_2 = stamp_value; - self.qid = args.ta.qid.into(); - self.unk_8c2 = Self::UNK_8C2; - self.current_id = resources.ta_ids.current.into(); - self.ordinal = args.ordinal as u32; - self.timestamp_start_va = resources.timestamp_va; - self.unk_932 = Self::UNK_932; - self.free_list_control_va = resources.free_list_control_va; - self.unk_93c = Self::UNK_93C; - if args.ordinal == 0 { - self.unk_93e = Self::UNK_93E_FIRST_KICK; - } - self.status_va = resources.ta_status_va; - self.context_generation = resources.context_generation; - Ok(()) - } - - /// Links the pass to the timestamps its two kicks are published at. - /// - /// Must be called before the descriptor is published. `pass_seq` is the - /// low byte of the logical queue's render-pass count (one for its first - /// pass); `fragment_busy` tells whether the fragment queue had not yet - /// completed every earlier kick. - pub(crate) fn set_completion_link( - &mut self, - ta: KickTimestamp, - fragment: KickTimestamp, - pass_seq: u8, - fragment_busy: bool, - ) { - self.fragment_busy = fragment_busy as u32; - self.fragment_kick = fragment.get(); - self.pass_seq = [pass_seq; 2]; - self.completion = ta.completion_value(pass_seq); - } - - /// Returns the register arrays a tiling kick binds; `descriptor_va` is the - /// descriptor's address as seen by the pass's VM. - pub(crate) fn register_bindings( - descriptor_va: u64, - ) -> Result<[Option; 4]> { - let array = RegisterArrayBinding::new( - offset_va(descriptor_va, Self::ARRAY_OFFSET)?, - TA_CHECKPOINTS[0] as u8, - )?; - Ok([Some(array), None, None, None]) - } -} - -/// Fragment (3D) work descriptor (tag 1). -#[repr(C, packed)] -#[derive(Debug)] -pub(crate) struct FragmentDescriptor { - /// [`CommandTag::Fragment`]. - pub(crate) tag: u32, - /// Twice the pair's submission ordinal. - pub(crate) submit_sequence: u64, - /// UAT context of the pass. - pub(crate) context_id: u32, - pub(crate) unk_10: [u8; 0x10], - /// GPU address of the pass's kick record. - pub(crate) kick_record_va: u64, - /// GPU address of the parameter-buffer state object. - pub(crate) pb_va: u64, - /// GPU address of the parameter-buffer scene entry. - pub(crate) scene_va: u64, - /// GPU address following the last scene entry. - pub(crate) scene_end_va: u64, - /// GPU address of the tile map. - pub(crate) tilemap_va: u64, - /// Multisample control. - pub(crate) msaa_control: u64, - /// Samples per pixel. - pub(crate) samples: u32, - /// Macro-tile dimensions in utiles: height in bits 15:0, width above. - pub(crate) macro_size: u32, - pub(crate) unk_58: [u8; 0x10], - /// Merge upper bound in X. - pub(crate) merge_upper_x: u32, - /// Merge upper bound in Y. - pub(crate) merge_upper_y: u32, - pub(crate) unk_70: u64, - /// Number of tiles. - pub(crate) tiles: u64, - /// Checkpoints into the main register array. - pub(crate) checkpoints: [u16; 3], - pub(crate) unk_86: u16, - /// Entry count of the main register array. - pub(crate) entries: u16, - pub(crate) unk_8a: [u8; 6], - /// Render-pass sequence of the pass. - pub(crate) pass_seq: u8, - pub(crate) unk_91: [u8; 0xf], - /// Main, partial-store, partial-resume and partial-load register arrays. - pub(crate) arrays: [RegisterArray; 4], - /// GPU address of the depth-bias array. - pub(crate) depth_bias_va: u64, - pub(crate) unk_1d28: u64, - /// GPU address of the scissor array. - pub(crate) scissor_va: u64, - pub(crate) unk_1d38: u64, - /// GPU address of the occlusion-query results. - pub(crate) occlusion_query_va: u64, - pub(crate) unk_1d48: [u8; 0x130], - /// Background resource specifier. - pub(crate) bg_rsrc_spec: u64, - /// GPU address of the background program. - pub(crate) bg_va: u64, - pub(crate) unk_1e88: [u8; 0x20], - /// Partial-render background resource specifier. - pub(crate) partial_bg_rsrc_spec: u64, - /// GPU address of the partial-render background program. - pub(crate) partial_bg_va: u64, - pub(crate) unk_1eb8: u64, - /// ZLS swizzle. - pub(crate) zls_swizzle: u32, - pub(crate) unk_1ec4: [u8; 0x74], - /// Tile-buffer blocks per utile. - pub(crate) tib_blocks: u64, - /// Auxiliary framebuffer flags. - pub(crate) aux_fb_flags: u64, - /// Width of the render target. - pub(crate) width: u32, - /// Height of the render target. - pub(crate) height: u32, - /// [`UNK_15049`]. - pub(crate) unk_1f50: u64, - /// Tile configuration. - pub(crate) tile_config: u64, - pub(crate) unk_1f60: [u8; 0x18], - /// Low word of the end-of-tile resource specifier. - pub(crate) eot_rsrc_spec: u32, - /// GPU address of the end-of-tile program. - pub(crate) eot_va: u64, - pub(crate) unk_1f84: [u8; 0x14], - /// Low word of the partial-render end-of-tile resource specifier. - pub(crate) partial_eot_rsrc_spec: u32, - /// GPU address of the partial-render end-of-tile program. - pub(crate) partial_eot_va: u64, - pub(crate) unk_1fa4: u32, - /// Depth clear value. - pub(crate) depth_clear: u32, - /// Tile-buffer blocks in bits 63:33 above a constant. - pub(crate) unk_1fac: u64, - pub(crate) unk_1fb4: [u8; 0x150], - /// See [`RenderResources::unk_2104`]. - pub(crate) unk_2104: u32, - /// Predecessor object ID of the fragment stage. - pub(crate) predecessor_id: u32, - /// See [`RenderResources::unk_210c`]. - pub(crate) unk_210c: u32, - pub(crate) unk_2110: u32, - pub(crate) unk_2114: [u8; 0x10], - pub(crate) unk_2124: [u32; 2], - pub(crate) unk_212c: [u8; 0x14], - /// GPU address of the fragment queue's completion stamp word. - pub(crate) stamp_va: u64, - /// GPU address of the fragment queue's auxiliary stamp word. - pub(crate) aux_stamp_va: u64, - /// Completion stamp value of the pass. - pub(crate) stamp_value: u32, - /// QID of the fragment queue. - pub(crate) qid: u32, - pub(crate) unk_2158: u32, - /// Non-zero on the first kick of a queue pair. - pub(crate) first_kick: u32, - /// Completion value of the fragment kick (see - /// [`KickTimestamp::completion_value`]). - pub(crate) completion: u64, - /// Current object ID of the fragment stage. - pub(crate) current_id: u64, - /// Submission ordinal of the pair. - pub(crate) ordinal: u32, - pub(crate) unk_2174: [u8; 0x24], - /// GPU address the pass start timestamp is written to. - pub(crate) timestamp_start_va: u64, - /// GPU address the pass end timestamp is written to. - pub(crate) timestamp_end_va: u64, - pub(crate) unk_21a8: [u8; 0x24], - pub(crate) unk_21cc: u8, - pub(crate) unk_21cd: u8, - /// GPU address of the USC free list's shared control object. - pub(crate) free_list_control_va: u64, - pub(crate) unk_21d6: u8, - /// GPU address of the fragment status word, as seen by the pass's VM. - pub(crate) status_va: u64, - /// Firmware alias of the fragment status word. - pub(crate) status_fw_va: u64, - /// Allocation generation of the UAT context. - pub(crate) context_generation: u8, - pub(crate) unk_21e8: [u8; 0x20], - pub(crate) unk_2208: u8, - pub(crate) unk_2209: [u8; 3], - pub(crate) unk_220c: u8, - pub(crate) unk_220d: [u8; 0x20], - pub(crate) unk_222d: u8, - pub(crate) unk_222e: [u8; 0x12], -} - -static_assert!(size_of::() == 0x2240); -static_assert!(core::mem::offset_of!(FragmentDescriptor, checkpoints) == 0x80); -static_assert!(core::mem::offset_of!(FragmentDescriptor, pass_seq) == 0x90); -static_assert!(core::mem::offset_of!(FragmentDescriptor, arrays) == 0xa0); -static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_bias_va) == 0x1d20); -static_assert!(core::mem::offset_of!(FragmentDescriptor, bg_rsrc_spec) == 0x1e78); -static_assert!(core::mem::offset_of!(FragmentDescriptor, tib_blocks) == 0x1f38); -static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_1f50) == 0x1f50); -static_assert!(core::mem::offset_of!(FragmentDescriptor, eot_va) == 0x1f7c); -static_assert!(core::mem::offset_of!(FragmentDescriptor, depth_clear) == 0x1fa8); -static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_2104) == 0x2104); -static_assert!(core::mem::offset_of!(FragmentDescriptor, stamp_va) == 0x2140); -static_assert!(core::mem::offset_of!(FragmentDescriptor, completion) == 0x2160); -static_assert!(core::mem::offset_of!(FragmentDescriptor, timestamp_start_va) == 0x2198); -static_assert!(core::mem::offset_of!(FragmentDescriptor, free_list_control_va) == 0x21ce); -static_assert!(core::mem::offset_of!(FragmentDescriptor, status_va) == 0x21d7); -static_assert!(core::mem::offset_of!(FragmentDescriptor, unk_222d) == 0x222d); - -// SAFETY: `FragmentDescriptor` consists of integers and arrays of integers only. -unsafe impl Zeroable for FragmentDescriptor {} - -impl FragmentDescriptor { - /// Offset of register array `index` within the descriptor. - const fn array_offset(index: usize) -> u64 { - (core::mem::offset_of!(Self, arrays) + index * size_of::()) as u64 - } - - /// Returns the register arrays a fragment kick binds; `descriptor_va` is - /// the descriptor's address as seen by the pass's VM. - pub(crate) fn register_bindings( - descriptor_va: u64, - ) -> Result<[Option; 4]> { - let array = |index, checkpoint: usize| -> Result> { - let va = offset_va(descriptor_va, Self::array_offset(index))?; - Ok(Some(RegisterArrayBinding::new(va, checkpoint as u8)?)) - }; - Ok([ - array(0, FRAGMENT_CHECKPOINTS[0] as usize)?, - array(1, PARTIAL_STORE_REGISTERS)?, - array(2, PARTIAL_RESUME_REGISTERS)?, - array(3, PARTIAL_LOAD_REGISTERS)?, - ]) - } -} - -/// A fragment descriptor followed by room for an MCache table that does not -/// fit its first register array. -#[repr(C)] -#[derive(Debug)] -pub(crate) struct FragmentSlot { - /// The descriptor. - pub(crate) descriptor: FragmentDescriptor, - /// MCache table of more than [`MCACHE_INLINE_RANGES`] ranges. - pub(crate) mcache: [McacheRange; MCACHE_RANGES_MAX], - pub(crate) pad_2350: [u8; 0x30], -} - -static_assert!(size_of::() == FRAGMENT_SLOT_SIZE); - -impl FragmentSlot { - const INLINE_MCACHE_OFFSET: u64 = - FragmentDescriptor::array_offset(0) + core::mem::offset_of!(RegisterArray, mcache) as u64; - const SPILL_MCACHE_OFFSET: u64 = core::mem::offset_of!(Self, mcache) as u64; - const UNK_2110: u32 = u32::MAX; - const UNK_2124: [u32; 2] = [1, 1]; - const UNK_21CC: u8 = 0x53; - const UNK_21D6: u8 = 1; - const UNK_2208: u8 = 1; - const UNK_220C: u8 = 1; - const UNK_222D: u8 = 1; - const UNK_1FAC_BASE: u64 = 0x300; - const UNK_1FAC_TIB_SHIFT: u32 = 33; - - /// Writes the complete descriptor except the completion link and the - /// MCache table. - /// - /// The spill table is left as it is: a kick entry names it only after - /// [`Self::set_mcache`] rewrote it, so stale bytes there are never read. - pub(crate) fn write(&mut self, args: &RenderArgs<'_>) -> Result { - let pass = args.pass; - let resources = args.resources; - let geometry = Geometry::new(pass)?; - let words = PassWords::new(pass)?; - let base = args.fragment.descriptor_va; - let array_va = |index| offset_va(base, FragmentDescriptor::array_offset(index)); - let arrays = [array_va(0)?, array_va(1)?, array_va(2)?, array_va(3)?]; - let timestamp_end_va = timestamp_end_va(resources.timestamp_va)?; - let desc = &mut self.descriptor; - - clear(desc); - desc.tag = CommandTag::Fragment as u32; - desc.submit_sequence = args.ordinal.wrapping_mul(2); - desc.context_id = resources.context_id.into(); - desc.kick_record_va = resources.kick_record_va; - desc.pb_va = resources.pb_va; - desc.scene_va = resources.scene_va; - desc.scene_end_va = resources.scene_end_va; - desc.tilemap_va = resources.tilemap_va; - desc.msaa_control = pass.msaa_control; - desc.samples = 1 << pass.samples_log2; - desc.macro_size = geometry.macro_size as u32; - desc.merge_upper_x = pass.merge_upper_x; - desc.merge_upper_y = pass.merge_upper_y; - desc.tiles = geometry.tiles_x * geometry.tiles_y; - desc.checkpoints = FRAGMENT_CHECKPOINTS; - desc.entries = FRAGMENT_REGISTERS as u16; - desc.arrays[0].set(arrays[0], &fragment_registers(args, &geometry, &words))?; - desc.arrays[1].set(arrays[1], &partial_store_registers(pass, &words))?; - desc.arrays[2].set(arrays[2], &partial_resume_registers(pass, &words))?; - desc.arrays[3].set(arrays[3], &partial_load_registers(pass, &words))?; - desc.depth_bias_va = pass.depth_bias_va; - desc.scissor_va = pass.scissor_va; - desc.occlusion_query_va = pass.occlusion_query_va; - desc.bg_rsrc_spec = words.bg_rsrc_spec; - desc.bg_va = words.bg_va; - desc.partial_bg_rsrc_spec = words.partial_bg_rsrc_spec; - desc.partial_bg_va = words.partial_bg_va; - desc.zls_swizzle = ZLS_SWIZZLE; - desc.tib_blocks = words.tib_blocks; - desc.aux_fb_flags = words.aux_fb_flags; - desc.width = pass.width; - desc.height = pass.height; - desc.unk_1f50 = UNK_15049; - desc.tile_config = words.tile_config; - desc.eot_rsrc_spec = words.eot_rsrc_spec as u32; - desc.eot_va = words.eot_va; - desc.partial_eot_rsrc_spec = words.partial_eot_rsrc_spec as u32; - desc.partial_eot_va = words.partial_eot_va; - desc.depth_clear = pass.depth_clear; - desc.unk_1fac = words.tib_blocks << Self::UNK_1FAC_TIB_SHIFT | Self::UNK_1FAC_BASE; - desc.unk_2104 = resources.unk_2104; - desc.predecessor_id = resources.fragment_ids.predecessor; - desc.unk_210c = resources.unk_210c; - desc.unk_2110 = Self::UNK_2110; - desc.unk_2124 = Self::UNK_2124; - desc.stamp_va = args.fragment.stamp_va; - desc.aux_stamp_va = args.fragment.aux_stamp_va; - desc.stamp_value = args.stamp_value(); - desc.qid = args.fragment.qid.into(); - desc.first_kick = (args.ordinal == 0) as u32; - desc.current_id = resources.fragment_ids.current.into(); - desc.ordinal = args.ordinal as u32; - desc.timestamp_start_va = resources.timestamp_va; - desc.timestamp_end_va = timestamp_end_va; - desc.unk_21cc = Self::UNK_21CC; - desc.free_list_control_va = resources.free_list_control_va; - desc.unk_21d6 = Self::UNK_21D6; - desc.status_va = resources.fragment_status_va; - desc.status_fw_va = resources.fragment_status_fw_va; - desc.context_generation = resources.context_generation; - desc.unk_2208 = Self::UNK_2208; - desc.unk_220c = Self::UNK_220C; - desc.unk_222d = Self::UNK_222D; - Ok(()) - } - - /// Links the pass to the timestamp of its fragment kick. - /// - /// Must be called before the descriptor is published; see - /// [`TaDescriptor::set_completion_link`]. - pub(crate) fn set_completion_link(&mut self, fragment: KickTimestamp, pass_seq: u8) { - self.descriptor.pass_seq = pass_seq; - self.descriptor.completion = fragment.completion_value(pass_seq); - } - - /// Writes the MCache table of the pass: the auxiliary framebuffer at - /// `aux_fb_va`, then `attachments`, translated in UAT context `context`. - /// `slot_va` is the GPU address of this slot as seen by the pass's VM. - /// - /// Returns the table for the fragment kick entry, or `None` when the pass - /// has no attachments. - pub(crate) fn set_mcache( - &mut self, - slot_va: u64, - aux_fb_va: u64, - attachments: &[WriteRange], - context: u8, - ) -> Result> { - if attachments.is_empty() { - return Ok(None); - } - let count = attachments.len() + 1; - if count > MCACHE_RANGES_MAX { - return Err(EINVAL); - } - let (offset, ranges) = if count <= MCACHE_INLINE_RANGES { - ( - Self::INLINE_MCACHE_OFFSET, - &mut self.descriptor.arrays[0].mcache[..count], - ) - } else { - (Self::SPILL_MCACHE_OFFSET, &mut self.mcache[..count]) - }; - let table = McacheTable::new(offset_va(slot_va, offset)?, count)?; - let aux_fb = WriteRange { - va: aux_fb_va, - size: AUX_FB_SIZE, - }; - ranges[0] = McacheRange::new(&aux_fb, McacheClass::AuxFramebuffer, context)?; - for (range, attachment) in ranges[1..].iter_mut().zip(attachments) { - *range = McacheRange::new(attachment, McacheClass::Attachment, context)?; - } - Ok(Some(table)) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/fw/render/storage.rs b/drivers/gpu/drm/asahi-neo/g17/fw/render/storage.rs deleted file mode 100644 index e1e6a10949fb44..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/fw/render/storage.rs +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Auxiliary render records and their initial images. Live event counters -//! are accessed individually by the queue owner after publication. - -use super::super::{buffer, clear, context, initdata}; -use kernel::prelude::*; - -pub(crate) const SUPPORT_SIZE: usize = 0x40000; -pub(crate) const EVENT_SLOTS: usize = 0x4000; -pub(crate) const EVENT_COUNT: usize = 35; -pub(crate) const STATE_SIZE: usize = 0x18000; -pub(crate) const DEFLAKE: usize = 0x4000; -pub(crate) const TA_STATUS: usize = 0x8000; -pub(crate) const AUXILIARY: usize = 0x10000; -pub(crate) const AUXILIARY_SIZE: usize = 0x8000; - -#[repr(C)] -pub(crate) struct Event { - pub(crate) node: context::WorkNode, - unk_c8: [u8; 0x38], -} - -#[repr(C)] -pub(crate) struct Support { - secondary_index: [u64; 8], - unk_40: [u8; 0x3fc0], - pub(crate) event_slots: [u32; 0x1000], - scene_slots: [u32; 0x1000], - shared_slots: [u32; 0x1000], - flag: u32, - unk_10004: [u8; 0x3ffc], - pub(crate) events: [Event; EVENT_COUNT], - unk_16300: [u8; 0x1d00], - scenes: [buffer::Scene; 79], - unk_1a780: [u8; 0x1880], - shared: buffer::State, - unk_1c0c0: [u8; 0x7f40], - control: initdata::FreeListControl, - unk_24068: [u8; 0x3f98], - submitted: u32, - unk_28004: [u8; 0x3ffc], - scheduler: context::Scheduler, - unk_30000: [u8; 0x10000], -} -// SAFETY: The entire graph is integer storage; all nested types are Zeroable. -unsafe impl Zeroable for Support {} -impl Support { - pub(crate) fn init( - &mut self, - address: u64, - primary_index: u64, - operand_table: u64, - page_list: u64, - word: u32, - ) -> Result { - clear(self); - address.checked_add(SUPPORT_SIZE as u64).ok_or(EOVERFLOW)?; - self.secondary_index = [0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x3c, 0x41]; - self.event_slots[1] = 1; - self.shared_slots[0] = 8; - self.shared_slots[1] = 8; - self.shared_slots[0x60 / 4] = 1; - self.flag = 1; - for (index, event) in self.events.iter_mut().enumerate() { - event.node.submitted = address + EVENT_SLOTS as u64 + index as u64 * 4; - if index == 1 { - event.node.threshold = 0x50; - event.node.word = word; - } - } - for (index, scene) in self.scenes.iter_mut().enumerate() { - scene.metric_client = 0x10_0008_0004 + index as u64 * 4; - scene.metric_fw = address + 0x8004 + index as u64 * 4; - let phase = index % buffer::SCRATCH_SLOTS; - scene.scratch = if phase == buffer::SCRATCH_SLOTS - 1 { - 0x178000 - } else { - 0x178020 + phase as u64 * buffer::SCRATCH_STRIDE - }; - scene.statistics = address + 0xc040; - if index == 0 { - scene.generation = 1; - } - } - let shared = &mut self.shared; - shared.page_list_fw = primary_index; - shared.page_list_client = 0x10_0019_0000; - shared.page_list_size = 0x10000; - shared.pages = 0x20; - shared.block_capacity = 0xc18; - shared.producer = 8; - shared.block_table = address; - shared.control = address + 0xc000; - shared.last_page = 0x1f; - shared.block_size = 0x20000; - shared.counter = address + 0x10000; - shared.maximum_size = 0x3060; - shared.normal_maximum_size = 0x1020; - shared.discard = 0x180000; - self.control = initdata::FreeListControl::new(&initdata::FreeListArgs { - buffer_slot: 0, - class: initdata::FreeListClass::Two, - page_list_va: page_list, - run_list_va: operand_table, - blocks: 25, - state_va: address + 0x28000, - }); - self.control.run_cursor = 0xe0; - self.submitted = 1; - self.scheduler.init(); - Ok(()) - } -} - -/// The viewport words consumed from the deflake page at +0x900. -#[repr(C)] -#[derive(Copy, Clone)] -pub(crate) struct Viewport { - unk_0: u32, - tiles_x: u32, - tiles_y: u32, - unk_c: u32, - scale_x: u32, - translate_x: u32, - scale_y: u32, - translate_y: u32, - unk_20: u32, - scale_z: u32, -} -impl Viewport { - /// Refresh only defined words; firmware-owned gaps retain their contents. - pub(crate) fn apply(&self, destination: &mut Self) { - destination.unk_0 = self.unk_0; - destination.tiles_x = self.tiles_x; - destination.tiles_y = self.tiles_y; - destination.scale_x = self.scale_x; - destination.translate_x = self.translate_x; - destination.scale_y = self.scale_y; - destination.translate_y = self.translate_y; - destination.scale_z = self.scale_z; - } - pub(crate) fn new(width: u32, height: u32) -> Result { - fn half(value: u32) -> Result { - if value == 0 { - return Err(EINVAL); - } - let highest = 31 - value.leading_zeros(); - let shift = 23u32.checked_sub(highest).ok_or(EINVAL)?; - Ok(((highest + 126) << 23) | ((value << shift) & 0x7f_ffff)) - } - let x = half(width)?; - let y = half(height)?; - Ok(Self { - unk_0: 0xc00, - tiles_x: 0x8000_0000 | (width.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1), - tiles_y: height.checked_add(31).ok_or(EOVERFLOW)? / 32 - 1, - unk_c: 0, - scale_x: x, - translate_x: x, - scale_y: y, - translate_y: y | 0x8000_0000, - unk_20: 0, - scale_z: 0x3f80_0000, - }) - } -} - -const _: () = { - assert!(core::mem::size_of::() == 0x100); - assert!(core::mem::size_of::() == SUPPORT_SIZE); - assert!(core::mem::offset_of!(Support, events) == 0x14000); - assert!(core::mem::offset_of!(Support, scenes) == 0x18000); - assert!(core::mem::offset_of!(Support, shared) == 0x1c000); - assert!(core::mem::offset_of!(Support, control) == 0x24000); - assert!(core::mem::offset_of!(Support, scheduler) == 0x2c000); - assert!(core::mem::size_of::() == 0x28); -}; diff --git a/drivers/gpu/drm/asahi-neo/g17/hazard.rs b/drivers/gpu/drm/asahi-neo/g17/hazard.rs deleted file mode 100644 index 20daaab85b8325..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/hazard.rs +++ /dev/null @@ -1,120 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Attachment overlap ordering across physical render pairs and shared images. -//! Tile stores can overlap even when the application's pixel regions do not. -//! Keep independent attachments concurrent; order only conflicting live spans. - -use super::job::Order; - -const MAX_SPANS: usize = 24; -pub(crate) const WAIT_MS: u32 = 1000; - -#[derive(Clone, Copy)] -struct Span { - key: u64, - start: u64, - end: u64, -} - -/// Up to sixteen attachment ranges and eight shared-object ranges. -pub(crate) struct Footprint { - vm: u64, - spans: [Span; MAX_SPANS], - count: usize, - shared: bool, -} - -impl Footprint { - pub(crate) fn new(vm: u64) -> Self { - Self { - vm, - spans: [Span { - key: 0, - start: 0, - end: 0, - }; MAX_SPANS], - count: 0, - shared: false, - } - } - - fn push(&mut self, key: u64, start: u64, end: u64) { - if start < end && self.count < MAX_SPANS { - self.spans[self.count] = Span { key, start, end }; - self.count += 1; - } - } - - pub(crate) fn push_vm(&mut self, start: u64, end: u64) { - self.push(self.vm << 1, start, end); - } - - /// An aligned GEM object address is an identity, never dereferenced here. - pub(crate) fn push_object(&mut self, object: usize, start: u64, end: u64) { - let before = self.count; - self.push(object as u64 | 1, start, end); - self.shared |= self.count != before; - } - - fn overlaps(&self, other: &Self) -> bool { - if self.vm != other.vm && !(self.shared && other.shared) { - return false; - } - self.spans[..self.count].iter().any(|a| { - other.spans[..other.count] - .iter() - .any(|b| a.key == b.key && a.start < b.end && b.start < a.end) - }) - } -} - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum Plan { - Any, - Pair(u8), - Wait, -} - -pub(crate) struct Live<'a> { - pub(crate) owner: u64, - pub(crate) sequence: u64, - pub(crate) slot: Option, - pub(crate) ticket: u64, - pub(crate) footprint: &'a Footprint, - pub(crate) quarantined: bool, -} - -/// A queue prefix already orders covered predecessors. Pending arrivals order -/// different owners fairly without imposing order on unrelated attachments. -pub(crate) fn plan<'a>( - owner: u64, - prefix: Option, - ticket: Option, - footprint: &Footprint, - live: impl IntoIterator>, -) -> Plan { - let mut selected = None; - for entry in live { - if entry.quarantined || !footprint.overlaps(entry.footprint) { - continue; - } - let Some(slot) = entry.slot else { - if entry.owner != owner && ticket.is_none_or(|mine| entry.ticket < mine) { - return Plan::Wait; - } - continue; - }; - if entry.owner != owner { - return Plan::Wait; - } - if Order::contains(prefix, entry.sequence) { - continue; - } - match selected { - None => selected = Some(slot), - Some(prior) if prior == slot => {} - Some(_) => return Plan::Wait, - } - } - selected.map_or(Plan::Any, Plan::Pair) -} diff --git a/drivers/gpu/drm/asahi-neo/g17/ids.rs b/drivers/gpu/drm/asahi-neo/g17/ids.rs deleted file mode 100644 index 6912c6501527ac..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/ids.rs +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Firmware-visible object generations shared by contexts, pools and render work. - -use super::fw::ObjectIds; -use core::sync::atomic::{AtomicU32, Ordering}; -use kernel::prelude::*; - -/// Module-lifetime identities remain unique across retained graphs and device rebinds. -pub(crate) static OBJECTS: Counter = Counter::new(); - -pub(crate) struct Counter(AtomicU32); - -impl Counter { - const fn new() -> Self { - Self(AtomicU32::new(1)) - } - - fn reserve(&self, count: u32) -> Result { - let mut old = self.0.load(Ordering::Acquire); - loop { - let next = old.checked_add(count).ok_or(EOVERFLOW)?; - match self - .0 - .compare_exchange_weak(old, next, Ordering::AcqRel, Ordering::Acquire) - { - Ok(_) => return Ok(old), - Err(current) => old = current, - } - } - } - - pub(crate) fn object(&self) -> Result { - let id = self.reserve(1)?; - if id == 0 { - Err(EOVERFLOW) - } else { - Ok(id) - } - } - - /// Allocate consecutive current IDs atomically, fragment first and tiling - /// second. Both name the same retained predecessor of their render graph. - pub(crate) fn render_pair(&self, predecessor: u32) -> Result<(ObjectIds, ObjectIds)> { - let fragment = self.reserve(2)?; - if predecessor == 0 || fragment == 0 { - return Err(EOVERFLOW); - } - Ok(( - ObjectIds { - predecessor, - current: fragment, - }, - ObjectIds { - predecessor, - current: fragment + 1, - }, - )) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/initdata.rs b/drivers/gpu/drm/asahi-neo/g17/initdata.rs deleted file mode 100644 index 5993be786874e7..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/initdata.rs +++ /dev/null @@ -1,1451 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Allocates, initializes and retains the graph shared by the two firmware instances. -//! -//! The owner is installed in the firmware session before construction starts. Even if an -//! allocation fails, the session stops both coprocessors before releasing any published memory. - -use super::{ - fw::initdata as abi, - object::{CpuMap, KernelObject, Placement}, - Config, Role, -}; -use crate::{driver::AsahiDevice, hw::t8140, mmu}; -use core::ops::Range; -use core::sync::atomic::{fence, AtomicU32, Ordering}; -use kernel::{c_str, prelude::*}; -use t8140::{kernel_window as high, low}; - -const EVENT_RING_RECHECK: u32 = 256; - -const INDEX_SIZE: usize = 0x10000; -const CONTEXT_SIZE: usize = 0x20000; -const REGION_A_SIZE: usize = core::mem::size_of::(); -const REGION_C_SIZE: usize = 0x1000; - -const COMPUTE_BLOCKS_ALLOCATED: usize = 23; -const COMPUTE_BLOCKS_MAPPED: usize = 22; -const COMPUTE_BLOCKS_POPULATED: usize = 21; -const COMPUTE_CONTROL_BLOCKS: u32 = 25; -const COMPUTE_CONTROL_CURSOR: u32 = 0x18; -const COMPUTE_RUN_ADDRESS_MASK: u64 = 0x0000_ffff_ffff_f000; - -/// A borrowed completion slot. Read the identity before requesting its timestamps; -/// unrelated QIDs need only the control word. Each access is an aligned live load. -pub(crate) struct ComputeCompletion<'a> { - object: &'a KernelObject, - offset: usize, -} - -impl ComputeCompletion<'_> { - pub(crate) fn read(&self, word: usize) -> Result { - if word >= 8 { - return Err(EINVAL); - } - // The scanner bounds offset to a 0x40-byte slot in the completion ring. - Ok(self - .object - .dword(self.offset + word * 8)? - .load(Ordering::Relaxed)) - } -} - -/// Unpublished device-wide compute backing, prepared without the device mutex. The cache -/// identity belongs to this allocation, not to any one physical queue or client VM. -pub(super) struct ComputeShared { - id: u64, - pages: KernelObject, - runs: KernelObject, - blocks: KVec, -} - -impl ComputeShared { - pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { - let pages = - KernelObject::backing(alloc.dev, low::RENDER_PAGE_LIST_SIZE, CpuMap::WriteCombined)?; - let runs = - KernelObject::backing(alloc.dev, low::RENDER_RUN_LIST_SIZE, CpuMap::WriteCombined)?; - let mut blocks = KVec::with_capacity(COMPUTE_BLOCKS_ALLOCATED, GFP_KERNEL)?; - for _ in 0..COMPUTE_BLOCKS_ALLOCATED { - blocks.push( - KernelObject::backing(alloc.dev, abi::FREE_LIST_BLOCK_SIZE, CpuMap::WriteCombined)?, - GFP_KERNEL, - )?; - } - let id = u64::from(super::ids::OBJECTS.object()?); - Ok(Self { - id, - pages, - runs, - blocks, - }) - } -} - -/// A device-global scheduler, retained independently of whichever physical queue first -/// needed it. Its generation reservation precedes later graph allocation failures. -pub(super) struct ComputeScheduler { - object: KernelObject, -} - -impl ComputeScheduler { - pub(super) fn new(alloc: &super::object::Allocator<'_>) -> Result { - let mut object = alloc.kernel( - mmu::UAT_PGSZ, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - object.initialize::(0, |state| { - state.init(); - Ok(()) - })?; - fence(Ordering::SeqCst); - // Scheduler creation reserves a generation even when only its address is used by work. - super::ids::OBJECTS.object()?; - Ok(Self { object }) - } -} - -/// Checked views used only within one publication call. Every fallible operation completes -/// before either member of a paired render publication is copied into shared memory. -struct WorkPublication<'a> { - record: &'a super::fw::channels::WorkSlot, - destination: *mut u8, - producer: &'a AtomicU32, - next: u32, -} - -impl WorkPublication<'_> { - fn copy_record(&self) { - // SAFETY: work_publication checked the complete destination range. The record is a - // fully initialized padding-free ABI value, and the device mutex owns publication. - unsafe { - core::ptr::copy_nonoverlapping( - self.record as *const _ as *const u8, - self.destination, - core::mem::size_of_val(self.record), - ); - } - } - - fn commit(self) -> u32 { - self.copy_record(); - fence(Ordering::SeqCst); - self.producer.store(self.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - self.next - } -} - -/// Session-long storage and mappings. Aliases are released before their backing objects. -pub(super) struct InitData { - // Live publication uses interior writes under the device mutex. This owner is Send but not - // Sync, so safe host code cannot issue those writes concurrently through shared references. - _single_publisher: core::marker::PhantomData>, - aliases: KVec, - objects: KVec, - roots: usize, - cluster: usize, - bundle: usize, - region_c: usize, - qos: usize, - pb: usize, - page_pool: usize, - compute_global: usize, - boot_control: usize, - boot_state: usize, - metrics_fw_va: u64, - compute_shared: Option, - compute_scheduler: Option, - compute_active: core::cell::Cell, - event_ring_checked: [core::cell::Cell<(u64, u64, u32)>; 2], - completion_rings: [usize; 2], - stamps: [usize; 2], - render_pages: usize, - render_runs: usize, - render_blocks: core::ops::Range, -} - -/// Placement inputs shared by the build steps. -struct BuildEnv<'a> { - dev: &'a AsahiDevice, - vm: &'a mmu::Vm, - lower: &'a mmu::Vm, - render: &'a mmu::Vm, - base: u64, - dynamic: Range, -} - -impl BuildEnv<'_> { - /// An object at a fixed offset of the kernel window. - fn fixed(&self, offset: u64, size: usize, prot: mmu::Prot, cpu: CpuMap) -> Result { - KernelObject::new( - self.dev, - self.vm, - Placement::At(self.base + offset), - size, - prot, - cpu, - ) - } - - /// An object placed in the dynamic part of the kernel window. - fn dynamic(&self, size: usize, align: u64, prot: mmu::Prot, cpu: CpuMap) -> Result { - KernelObject::new( - self.dev, - self.vm, - Placement::In(self.dynamic.clone(), align), - size, - prot, - cpu, - ) - } -} - -/// Object indices the firmware-object initialization needs from the table step. -struct DescriptorTables { - pb: usize, - page_pool: usize, - fwctl: usize, -} - -/// Object indices and ring addresses the firmware-object initialization needs from the bundle step. -struct BundleObjects { - zero: usize, - completion_rings: [abi::CompletionRing; 2], - region_a: usize, - qos: usize, -} - -impl InitData { - /// Empty construction state, retained even if graph construction fails. - pub(super) fn new() -> Self { - Self { - _single_publisher: core::marker::PhantomData, - aliases: KVec::new(), - objects: KVec::new(), - roots: 0, - cluster: 0, - bundle: 0, - region_c: 0, - qos: 0, - pb: 0, - page_pool: 0, - compute_global: 0, - boot_control: 0, - boot_state: 0, - metrics_fw_va: 0, - compute_shared: None, - compute_scheduler: None, - compute_active: core::cell::Cell::new(false), - event_ring_checked: [const { core::cell::Cell::new((0, 0, 0)) }; 2], - completion_rings: [0; 2], - stamps: [0; 2], - render_pages: 0, - render_runs: 0, - render_blocks: 0..0, - } - } - - pub(super) fn metrics_fw_va(&self) -> Result { - if self.metrics_fw_va == 0 { - return Err(EIO); - } - Ok(self.metrics_fw_va) - } - - pub(super) fn pb_descriptor_table(&self) -> Result<&KernelObject> { - self.object(self.pb) - } - - pub(super) fn page_pool_descriptor_table(&self) -> Result<&KernelObject> { - self.object(self.page_pool) - } - - pub(super) fn stamp_addresses(&self, qid: u8) -> Result<[u64; 2]> { - if qid > super::fw::kick::QID_MAX { - return Err(EINVAL); - } - let offset = u64::from(qid) * core::mem::size_of::() as u64; - Ok([ - self.object(self.stamps[0])?.gpu_va() + offset, - self.object(self.stamps[1])?.gpu_va() + offset, - ]) - } - - /// The caller reads both engine lanes before its acquire fence. Page A is the completed - /// GPU-maintenance witness; page B stays under firmware ownership. - pub(super) fn stamp(&self, qid: u8) -> Result<&AtomicU32> { - if qid > super::fw::kick::QID_MAX { - return Err(EINVAL); - } - self.object(self.stamps[0])? - .word(usize::from(qid) * core::mem::size_of::()) - } - - /// Initialize a fresh physical compute QID's row before exposing its queue. An installed - /// or rebound queue keeps the row's firmware-owned counters and must never call this again. - pub(super) fn initialize_compute_pool(&self, slot: u16, page_list_va: u64) -> Result { - if slot == 0 || slot > u16::from(super::fw::kick::QID_MAX) + 1 { - return Err(EINVAL); - } - let entry = abi::PagePoolDescriptor::compute(page_list_va); - let size = core::mem::size_of::(); - let offset = usize::from(slot) * size; - let object = self.object(self.page_pool)?; - let pointer = object.pointer(offset, size)?; - let page_list = object.dword(offset)?; - let counters = - object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, counters))?; - let count = - object.dword(offset + core::mem::offset_of!(abi::PagePoolDescriptor, page_count))?; - // SAFETY: the whole fresh row was bounds checked, the QID has never been published, - // and the device mutex excludes another host publisher. No other row is touched. - unsafe { - pointer.write_bytes(0, size); - } - page_list.store(entry.page_list, Ordering::Relaxed); - counters.store(entry.counters, Ordering::Relaxed); - count.store(entry.page_count, Ordering::Relaxed); - fence(Ordering::SeqCst); - Ok(()) - } - - pub(super) fn qos(&self) -> Result<&KernelObject> { - self.object(self.qos) - } - - pub(super) fn has_compute_scheduler(&self) -> bool { - self.compute_scheduler.is_some() - } - - /// Only the reserved singleton builder may install a scheduler. Retain it before building - /// the first descriptor graph so later construction failure does not allocate it again. - pub(super) fn install_compute_scheduler(&mut self, scheduler: ComputeScheduler) -> Result { - if self.compute_scheduler.is_some() { - return Err(EBUSY); - } - self.compute_scheduler = Some(scheduler); - Ok(()) - } - - pub(super) fn has_compute_shared(&self) -> bool { - self.compute_shared.is_some() - } - - /// Install a prepared aperture once while holding the device mutex. A competing preparer - /// may have installed its allocation first; the redundant unpublished backing then drops. - pub(super) fn install_compute_shared(&mut self, mut shared: ComputeShared) -> Result { - if self.compute_shared.is_some() { - return Ok(()); - } - let state_va = self.object(self.boot_state)?.gpu_va(); - let mut control = abi::FreeListControl::new(&abi::FreeListArgs { - buffer_slot: u32::MAX, - class: abi::FreeListClass::Two, - page_list_va: low::COMPUTE_POOL_PAGE_LIST, - run_list_va: low::COMPUTE_POOL_RUN_LIST, - blocks: COMPUTE_CONTROL_BLOCKS, - state_va, - }); - control.run_cursor = COMPUTE_CONTROL_CURSOR; - self.object_mut(self.boot_control)?.bytes_mut().fill(0); - self.object_mut(self.boot_control)?.write(0, control)?; - self.object_mut(self.boot_state)?.bytes_mut().fill(0); - shared - .pages - .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |pages| { - abi::fill_page_list(pages, (0..COMPUTE_BLOCKS_POPULATED).map(low::compute_block)) - })?; - shared - .runs - .initialize::<[u64; low::RENDER_RUN_LIST_SIZE / 8]>(0, |runs| { - for (index, run) in runs.iter_mut().take(COMPUTE_BLOCKS_POPULATED).enumerate() { - *run = (low::compute_block(index) & COMPUTE_RUN_ADDRESS_MASK) - | ((abi::FREE_LIST_BLOCK_PAGES as u64) << 52); - } - Ok(()) - })?; - fence(Ordering::SeqCst); - self.compute_shared = Some(shared); - Ok(()) - } - - pub(super) fn cache_compute_vm(&self, vm: &mmu::Vm) -> Result { - let shared = self.compute_shared.as_ref().ok_or(EIO)?; - if vm.compute_pool_mappings_match(shared.id) { - return Ok(()); - } - vm.clear_compute_pool_mappings(); - let mut mappings = KVec::with_capacity(26, GFP_KERNEL)?; - mappings.push( - shared - .pages - .map_alias(vm, low::COMPUTE_POOL_PAGE_LIST, mmu::PROT_GPU_SHARED_RW)?, - GFP_KERNEL, - )?; - mappings.push( - shared - .runs - .map_alias(vm, low::COMPUTE_POOL_RUN_LIST, mmu::PROT_GPU_SHARED_RW)?, - GFP_KERNEL, - )?; - for (index, block) in shared.blocks.iter().take(COMPUTE_BLOCKS_MAPPED).enumerate() { - mappings.push( - block.map_alias(vm, low::compute_block(index), mmu::PROT_GPU_SHARED_RW)?, - GFP_KERNEL, - )?; - } - for slot in 0..2 { - mappings.push( - self.object(self.completion_rings[slot])?.map_alias( - vm, - low::COMPLETION_RINGS[slot], - mmu::PROT_GPU_SHARED_RW, - )?, - GFP_KERNEL, - )?; - } - vm.install_compute_pool_mappings(shared.id, mappings) - } - - /// The descriptor-table views remain immutable for the firmware session. - pub(super) fn validate_descriptor_views(&self) -> Result { - let values = [ - low::PB_DESCRIPTORS, - self.object(self.pb)?.gpu_va(), - low::PAGE_POOL_DESCRIPTORS, - self.object(self.page_pool)?.gpu_va(), - ]; - let base = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, pb_descriptors_low_va); - let bundle = self.object(self.bundle)?; - let mut changed = false; - for (index, value) in values.into_iter().enumerate() { - changed |= bundle.dword(base + index * 8)?.load(Ordering::Relaxed) != value; - } - fence(Ordering::SeqCst); - if changed { - Err(EIO) - } else { - Ok(()) - } - } - - pub(super) fn activate_compute(&self) -> Result { - self.validate_descriptor_views()?; - if self.compute_active.get() { - return Ok(()); - } - let scheduler = self.compute_scheduler.as_ref().ok_or(EIO)?.object.gpu_va(); - let object = self.object(self.compute_global)?; - let state = object - .pointer(0, core::mem::size_of::())? - .cast::(); - // SAFETY: The complete typed view is in the retained page-aligned object. InitData is - // not Sync and its caller holds the sole publication mutex. Each named field is - // naturally aligned and written individually; no reference aliases firmware state. - unsafe { - core::ptr::addr_of_mut!((*state).unk_0).write_volatile(0xff); - core::ptr::addr_of_mut!((*state).unk_8).write_volatile(0xff); - core::ptr::addr_of_mut!((*state).unk_10).write_volatile(0x0f01); - core::ptr::addr_of_mut!((*state).unk_18).write_volatile(0x0f01); - core::ptr::addr_of_mut!((*state).unk_408).write_volatile(2); - core::ptr::addr_of_mut!((*state).unk_40c).write_volatile(2); - core::ptr::addr_of_mut!((*state).unk_604).write_volatile(4); - core::ptr::addr_of_mut!((*state).scheduler_va).write_volatile(scheduler); - core::ptr::addr_of_mut!((*state).unk_c04).write_volatile(0x1000); - core::ptr::addr_of_mut!((*state).unk_e00).write_volatile(7); - core::ptr::addr_of_mut!((*state).unk_e08).write_volatile(0x0000_0001_0e22_618a); - core::ptr::addr_of_mut!((*state).unk_e20).write_volatile(4); - core::ptr::addr_of_mut!((*state).unk_e28).write_volatile(4); - } - self.compute_active.set(true); - Ok(()) - } - - /// The sole host publisher holds the device mutex through the following - /// inner and outer publication. Firmware consumers can only free capacity; - /// neither this check nor waiting for room reserves or limits other queues. - /// `priority` selects the engine's ring of that priority class. - pub(super) fn work_ready( - &self, - priority: u8, - engine: super::fw::queue::DataMaster, - ) -> Result { - let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; - let next = (producer + 1) % super::fw::channels::WORK_RING_SLOTS; - Ok(next != consumer && next != consumer_b) - } - - fn work_publication<'a>( - &'a self, - priority: u8, - record: &'a super::fw::channels::WorkSlot, - ) -> Result> { - use super::fw::{channels, queue::DataMaster}; - let engine = match record.data_master { - 0 => DataMaster::Tiling, - 1 => DataMaster::Fragment, - 2 => DataMaster::Compute, - _ => return Err(EINVAL), - }; - let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let [consumer, consumer_b, producer] = self.work_cursors(priority, engine)?; - let next = (producer + 1) % channels::WORK_RING_SLOTS; - if next == consumer || next == consumer_b { - return Err(EINVAL); - } - let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + producer as usize * core::mem::size_of_val(record), - core::mem::size_of_val(record), - )?; - let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) - + abi::work_state_offset(channel) - + abi::ChannelState::PRODUCER, - )?; - Ok(WorkPublication { - record, - destination, - producer, - next, - }) - } - - /// Two firmware consumers and the host producer, sampled in that order. - pub(super) fn work_cursors( - &self, - priority: u8, - engine: super::fw::queue::DataMaster, - ) -> Result<[u32; 3]> { - use super::fw::channels; - let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let base = Self::state_offset(Role::Primary) + abi::work_state_offset(channel); - let state = self.object(self.cluster)?; - let mut values = [0; 3]; - for (value, offset) in values.iter_mut().zip([ - abi::ChannelState::CONSUMER, - abi::ChannelState::UNK_10, - abi::ChannelState::PRODUCER, - ]) { - *value = state.word(base + offset)?.load(Ordering::Relaxed); - } - if values - .iter() - .any(|&value| value >= channels::WORK_RING_SLOTS) - { - return Err(EINVAL); - } - Ok(values) - } - - /// Inner queue records must be visible before this outer publication. The returned - /// producer is the retained consumer witness for this slot. - pub(super) fn publish_work( - &self, - priority: u8, - record: &super::fw::channels::WorkSlot, - ) -> Result { - Ok(self.work_publication(priority, record)?.commit()) - } - - /// Validate both destinations before exposing fragment, then tiling. The accounting token - /// is committed after copying fragment; late tiling state is published before its slot. - /// Deferred pairs retain both slot bodies without exposing either producer. - pub(super) fn publish_work_pair( - &self, - priority: u8, - fragment: &super::fw::channels::WorkSlot, - tiling: &super::fw::channels::WorkSlot, - deferred: bool, - commit_accounting: impl FnOnce() -> Result, - publish_late_tiling: impl FnOnce(), - ) -> Result<[u32; 2]> { - use super::fw::queue::DataMaster; - if fragment.data_master != DataMaster::Fragment as u32 - || tiling.data_master != DataMaster::Tiling as u32 - { - return Err(EINVAL); - } - let fragment = self.work_publication(priority, fragment)?; - let tiling = self.work_publication(priority, tiling)?; - fragment.copy_record(); - commit_accounting()?; - fence(Ordering::SeqCst); - if !deferred { - fragment.producer.store(fragment.next, Ordering::Relaxed); - } - publish_late_tiling(); - tiling.copy_record(); - fence(Ordering::SeqCst); - if !deferred { - tiling.producer.store(tiling.next, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - Ok([fragment.next, tiling.next]) - } - - fn retained_work_publication<'a>( - &'a self, - priority: u8, - record: &'a super::fw::channels::WorkSlot, - next: u32, - ) -> Result> { - use super::fw::{channels, queue::DataMaster}; - let engine = match record.data_master { - 0 => DataMaster::Tiling, - 1 => DataMaster::Fragment, - _ => return Err(EINVAL), - }; - let channel = channels::work_ring_index(priority, engine).ok_or(EINVAL)?; - let producer = self.object(self.cluster)?.word( - Self::state_offset(Role::Primary) - + abi::work_state_offset(channel) - + abi::ChannelState::PRODUCER, - )?; - let current = producer.load(Ordering::Relaxed); - if current >= channels::WORK_RING_SLOTS || next != (current + 1) % channels::WORK_RING_SLOTS - { - return Err(EBUSY); - } - let destination = self.object(self.bundle)?.pointer( - abi::work_ring_offset(channel) + current as usize * core::mem::size_of_val(record), - core::mem::size_of_val(record), - )?; - Ok(WorkPublication { - record, - destination, - producer, - next, - }) - } - - /// Release a retained pair after its control transaction. Revalidate the original producer - /// witnesses before either copy; both bodies precede the fragment and tiling publications. - pub(super) fn publish_retained_work_pair( - &self, - priority: u8, - fragment: &super::fw::channels::WorkSlot, - tiling: &super::fw::channels::WorkSlot, - next: [u32; 2], - ) -> Result { - use super::fw::queue::DataMaster; - if fragment.data_master != DataMaster::Fragment as u32 - || tiling.data_master != DataMaster::Tiling as u32 - { - return Err(EINVAL); - } - let fragment = self.retained_work_publication(priority, fragment, next[0])?; - let tiling = self.retained_work_publication(priority, tiling, next[1])?; - fragment.copy_record(); - tiling.copy_record(); - fence(Ordering::SeqCst); - fragment.producer.store(fragment.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - tiling.producer.store(tiling.next, Ordering::Relaxed); - fence(Ordering::SeqCst); - Ok(()) - } - - /// Snapshot the shared compute completion ring once per service transaction. The queue - /// scanner latches exact identities; firmware retains sole ownership of these slots. - pub(super) fn scan_compute_completions( - &self, - mut record: impl FnMut(ComputeCompletion<'_>) -> Result, - ) -> Result { - let object = self.object(self.completion_rings[0])?; - fence(Ordering::Acquire); - for slot in 0..(abi::COMPLETION_RING_SIZE / 0x40).min(0x800) { - record(ComputeCompletion { - object, - offset: slot * 0x40, - })?; - } - Ok(()) - } - - fn object(&self, index: usize) -> Result<&KernelObject> { - self.objects.get(index).ok_or(EINVAL) - } - - fn object_mut(&mut self, index: usize) -> Result<&mut KernelObject> { - self.objects.get_mut(index).ok_or(EINVAL) - } - - fn add(&mut self, object: KernelObject) -> Result { - let index = self.objects.len(); - self.objects.push(object, GFP_KERNEL)?; - Ok(index) - } - - fn alias(&mut self, index: usize, vm: &mmu::Vm, va: u64, prot: mmu::Prot) -> Result { - let alias = self.object_mut(index)?.map_alias(vm, va, prot)?; - self.aliases.push(alias, GFP_KERNEL)?; - Ok(()) - } - - /// Constructs the graph after the firmware has published the high-root entries. - pub(super) fn build( - &mut self, - dev: &AsahiDevice, - cfg: &Config, - uat: &mmu::Uat, - render: &mmu::Vm, - metrics: &KernelObject, - ) -> Result { - let range = uat.geometry().kernel_range(); - let env = BuildEnv { - dev, - vm: uat.kernel_vm(), - lower: uat.kernel_lower_vm(), - render, - base: range.start, - dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, - }; - self.map_register_windows(&env)?; - let tables = self.build_descriptor_tables(&env)?; - self.build_render_free_lists(&env)?; - let first_block = self.build_render_pool(&env)?; - self.build_compute_pools(&env, first_block)?; - let bundle = self.build_bundle(&env, cfg, metrics)?; - self.initialize_firmware_objects(&env, cfg, &tables, &bundle) - } - - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io(va, phys.try_into()?, size.try_into()?, mmu::PROT_FW_MMIO_RW)?, - GFP_KERNEL, - )?; - } - Ok(()) - } - - /// The private cluster, zero page and the three descriptor tables with their low aliases. - fn build_descriptor_tables(&mut self, env: &BuildEnv<'_>) -> Result { - use mmu::{ - PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, - PROT_GPU_SHARED_RW as GS, - }; - use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.cluster = self.add(env.fixed(high::PRIVATE_CLUSTER, abi::private::SIZE, FS, WC)?)?; - self.compute_global = self.add(env.fixed(high::ZERO_PAGE, mmu::UAT_PGSZ, FP, WC)?)?; - let pb = self.add(env.fixed(high::PB_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; - self.object_mut(pb)? - .initialize::(0, |table| { - table.init(); - Ok(()) - })?; - let page_pool = self.add(env.fixed(high::PAGE_POOL_DESCRIPTORS, mmu::UAT_PGSZ, FS, WC)?)?; - self.object_mut(page_pool)? - .initialize::(0, |table| { - table.init(low::RENDER_PAGE_LIST, low::COMPUTE_POOL_PAGE_LIST); - Ok(()) - })?; - self.pb = pb; - self.page_pool = page_pool; - self.alias(pb, env.lower, low::PB_DESCRIPTORS, GS)?; - self.alias(page_pool, env.lower, low::PAGE_POOL_DESCRIPTORS, GS)?; - let index = self.add(env.fixed(high::INDEX, INDEX_SIZE, GFS, WB)?)?; - self.object_mut(index)? - .initialize::(0, |table| { - table.init(); - Ok(()) - })?; - self.alias(index, env.lower, low::INDEX, GFS)?; - let fwctl = self.add(env.fixed(high::FWCTL, abi::FWCTL_SIZE, FS, WB)?)?; - Ok(DescriptorTables { - pb, - page_pool, - fwctl, - }) - } - - /// Render and boot free-list controls, descriptor pages and the two queue contexts. - fn build_render_free_lists(&mut self, env: &BuildEnv<'_>) -> Result { - use mmu::{ - PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_PRIV_RW as GFP, - PROT_GPU_FW_SHARED_RW as GFS, - }; - use CpuMap::WriteCombined as WC; - for (offset, state, class, prot, initial) in [ - ( - high::RENDER_FREE_LIST, - high::RENDER_FREE_LIST_STATE, - abi::FreeListClass::One, - FP, - abi::FreeListState::RENDER, - ), - ( - high::BOOT_FREE_LIST, - high::BOOT_FREE_LIST_STATE, - abi::FreeListClass::Two, - GFP, - abi::FreeListState::BOOT, - ), - ] { - let control = self.add(env.fixed(offset, mmu::UAT_PGSZ, prot, WC)?)?; - self.object_mut(control)?.write( - 0, - abi::FreeListControl::new(&abi::FreeListArgs { - buffer_slot: 0, - class, - page_list_va: low::RENDER_PAGE_LIST, - run_list_va: low::RENDER_RUN_LIST, - blocks: low::RENDER_BLOCKS_INITIAL, - state_va: env.base + state, - }), - )?; - let state_prot = if class == abi::FreeListClass::One { - FS - } else { - GFS - }; - let state = self.add(env.fixed(state, mmu::UAT_PGSZ, state_prot, WC)?)?; - self.object_mut(state)?.write(0, initial)?; - if class == abi::FreeListClass::Two { - self.boot_control = control; - self.boot_state = state; - } - } - self.stamps[0] = self.add(env.fixed(high::DESCRIPTOR_PAGE_A, mmu::UAT_PGSZ, GFS, WC)?)?; - self.stamps[1] = self.add(env.fixed(high::DESCRIPTOR_PAGE_B, mmu::UAT_PGSZ, GFP, WC)?)?; - self.add(env.fixed(high::TA_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; - self.add(env.fixed(high::FRAGMENT_CONTEXT, CONTEXT_SIZE, FS, WC)?)?; - Ok(()) - } - - /// Render page list, run list and blocks aliased into the render-global root; returns - /// the index of the first block. - fn build_render_pool(&mut self, env: &BuildEnv<'_>) -> Result { - use mmu::PROT_GPU_SHARED_RW as GS; - use CpuMap::WriteBack as WB; - let pages = self.add(KernelObject::backing(env.dev, low::RENDER_PAGE_LIST_SIZE, WB)?)?; - self.object_mut(pages)? - .initialize::<[u64; low::RENDER_PAGE_LIST_SIZE / 8]>(0, |list| { - abi::fill_page_list( - list, - (0..low::RENDER_BLOCKS_INITIAL as usize).map(low::render_block), - ) - })?; - let runs = self.add(KernelObject::backing(env.dev, low::RENDER_RUN_LIST_SIZE, WB)?)?; - let first_block = self.objects.len(); - for _ in 0..low::RENDER_BLOCKS_ALLOCATED { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; - } - self.render_pages = pages; - self.render_runs = runs; - self.render_blocks = first_block..self.objects.len(); - self.alias(pages, env.render, low::RENDER_PAGE_LIST, GS)?; - self.alias(runs, env.render, low::RENDER_RUN_LIST, GS)?; - for block in 0..low::RENDER_BLOCKS_MAPPED { - self.alias(first_block + block, env.render, low::render_block(block), GS)?; - } - Ok(first_block) - } - - /// Compute free-list controls and states, zero blocks, run list and the compute page - /// list inside the render blocks. - fn build_compute_pools(&mut self, env: &BuildEnv<'_>, first_block: usize) -> Result { - use mmu::PROT_GPU_FW_SHARED_RW as GFS; - use CpuMap::{WriteBack as WB, WriteCombined as WC}; - let compute = self.add(env.fixed(high::COMPUTE_FREE_LIST, mmu::UAT_PGSZ, GFS, WC)?)?; - self.object_mut(compute)?.write( - 0, - abi::FreeListControl::new(&abi::FreeListArgs { - buffer_slot: 0, - class: abi::FreeListClass::One, - page_list_va: low::COMPUTE_PAGE_LIST, - run_list_va: low::COMPUTE_RUN_LIST, - blocks: low::COMPUTE_BLOCKS, - state_va: env.base + high::COMPUTE_FREE_LIST_STATE, - }), - )?; - let compute_state = self.add(KernelObject::backing(env.dev, mmu::UAT_PGSZ, WB)?)?; - self.object_mut(compute_state)? - .write(0, abi::FreeListState::COMPUTE)?; - for _ in &low::COMPUTE_ZERO_BLOCKS { - self.add(KernelObject::backing(env.dev, abi::FREE_LIST_BLOCK_SIZE, WB)?)?; - } - let compute_runs = self.add(KernelObject::new( - env.dev, - env.lower, - Placement::At(low::COMPUTE_RUN_LIST), - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; - self.object_mut(compute_runs)? - .initialize::<[abi::FreeListRunSlot; mmu::UAT_PGSZ / 0x40]>(0, |list| { - abi::fill_run_list(list, low::COMPUTE_FREE_LIST_BLOCKS) - })?; - let ready = self.add(env.fixed( - high::COMPUTE_READY_FREE_LIST, - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; - self.object_mut(ready)?.write( - 0, - abi::FreeListControl::new(&abi::FreeListArgs { - buffer_slot: 1, - class: abi::FreeListClass::One, - page_list_va: low::COMPUTE_POOL_PAGE_LIST, - run_list_va: low::COMPUTE_POOL_RUN_LIST, - blocks: low::COMPUTE_READY_BLOCKS, - state_va: env.base + high::COMPUTE_READY_FREE_LIST_STATE, - }), - )?; - for state in [high::COMPUTE_READY_FREE_LIST_STATE, high::COMPUTE_STATE] { - let obj = self.add(env.fixed(state, mmu::UAT_PGSZ, GFS, WC)?)?; - self.object_mut(obj)? - .write(0, abi::FreeListState::COMPUTE)?; - } - self.object_mut(first_block + low::COMPUTE_PAGE_LIST_BLOCK)? - .initialize::<[u64; abi::FREE_LIST_BLOCK_SIZE / 8]>(0, |list| { - abi::fill_page_list(list, low::COMPUTE_FREE_LIST_BLOCKS) - })?; - fence(Ordering::SeqCst); - Ok(()) - } - - /// The bundle with its zero page and completion rings, the QoS table, the metrics alias, - /// the roots and regions A and C. - fn build_bundle( - &mut self, - env: &BuildEnv<'_>, - cfg: &Config, - metrics: &KernelObject, - ) -> Result { - use mmu::{ - PROT_FW_PRIV_RW as FP, PROT_FW_SHARED_RW as FS, PROT_GPU_FW_SHARED_RW as GFS, - PROT_GPU_SHARED_RW as GS, - }; - use CpuMap::{WriteBack as WB, WriteCombined as WC}; - self.bundle = self.add(env.fixed(high::BUNDLE, abi::bundle::SIZE, FP, WC)?)?; - let zero = self.add(env.fixed( - high::BUNDLE + abi::bundle::ZERO_PAGE as u64, - mmu::UAT_PGSZ, - GFS, - WC, - )?)?; - let mut completion_rings = [abi::CompletionRing { - low_va: 0, - high_va: 0, - }; 2]; - for slot in 0..2 { - let offset = high::BUNDLE + abi::bundle::COMPLETION_RINGS[slot] as u64; - let ring = self.add(env.fixed(offset, abi::COMPLETION_RING_SIZE, FP, WB)?)?; - self.completion_rings[slot] = ring; - self.alias(ring, env.lower, low::COMPLETION_RINGS[slot], GS)?; - completion_rings[slot] = abi::CompletionRing { - low_va: low::COMPLETION_RINGS[slot], - high_va: env.base + offset, - }; - } - let qos = self.add(env.dynamic(abi::QOS_SIZE, mmu::UAT_PGSZ as u64, FS, WC)?)?; - self.qos = qos; - fence(Ordering::SeqCst); - let metrics_alias = - metrics.alias_in(env.vm, env.dynamic.clone(), cfg.pm_metrics.size as u64, GFS)?; - self.metrics_fw_va = metrics_alias.iova(); - self.aliases.push(metrics_alias, GFP_KERNEL)?; - fence(Ordering::SeqCst); - self.roots = self.add(env.dynamic( - abi::ROOTS_SIZE, - abi::SECONDARY_ROOT_OFFSET as u64, - mmu::PROT_FW_SHARED_RO, - WB, - )?)?; - let region_a = self.add(env.dynamic( - REGION_A_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RO, - WB, - )?)?; - self.region_c = self.add(env.dynamic(REGION_C_SIZE, mmu::UAT_PGSZ as u64, FS, WB)?)?; - Ok(BundleObjects { - zero, - completion_rings, - region_a, - qos, - }) - } - - /// Writes every firmware object the host builds from constants and addresses, then the - /// opening control-ring state for both roles. - fn initialize_firmware_objects( - &mut self, - env: &BuildEnv<'_>, - cfg: &Config, - tables: &DescriptorTables, - bundle: &BundleObjects, - ) -> Result { - let base = env.base; - let opps = Self::opps(env.dev)?; - let hw_args = abi::HwDataArgs { - chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, - opps: &opps, - perf: &t8140::PERF, - completion_rings: bundle.completion_rings, - qos_va: self.object(bundle.qos)?.gpu_va(), - }; - self.object_mut(self.bundle)? - .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; - self.object_mut(self.region_c)? - .initialize::(0, |region| { - region.init(); - Ok(()) - })?; - self.object_mut(self.cluster)? - .write(abi::private::PRIMARY_STATUS_A, abi::StatusBlock::new())?; - self.object_mut(self.cluster)? - .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; - let fwctl_va = self.object(tables.fwctl)?.gpu_va(); - self.object_mut(self.cluster)? - .initialize::(abi::private::STATUS_B, |status| { - status.init(fwctl_va); - Ok(()) - })?; - self.object_mut(self.cluster)? - .initialize::( - abi::private::SECONDARY_STATE + abi::private::STATUS_B, - |config| { - config.init(); - Ok(()) - }, - )?; - self.object_mut(self.bundle)? - .initialize::(abi::bundle::POWER_CONFIG, |power| { - power.init(); - Ok(()) - })?; - let secondary_status = base - + high::PRIVATE_CLUSTER - + (abi::private::SECONDARY_STATE + abi::private::SECONDARY_STATUS) as u64; - self.object_mut(self.bundle)? - .initialize::(abi::bundle::AUX, |aux| { - aux.init(secondary_status); - Ok(()) - })?; - self.object_mut(self.bundle)? - .initialize::(abi::bundle::POWER_CONFIG_COPY, |power| { - power.init(); - Ok(()) - })?; - let views = abi::PrimaryViews { - zero_page_va: self.object(bundle.zero)?.gpu_va(), - pb_descriptors_va: self.object(tables.pb)?.gpu_va(), - page_pool_descriptors_va: self.object(tables.page_pool)?.gpu_va(), - }; - for role in [Role::Primary, Role::Secondary] { - let slot = role as usize; - let main_offset = abi::bundle::MAIN_CONFIG[slot]; - let main_va = base + high::BUNDLE + main_offset as u64; - let state_va = base + high::PRIVATE_CLUSTER + Self::state_offset(role) as u64; - let status_va = base + high::PRIVATE_CLUSTER + Self::status_offset(role) as u64; - let args = abi::MainConfigArgs { - bundle_va: base + high::BUNDLE, - main_config_va: main_va, - state_grid_va: state_va, - status_a_va: status_va, - pb_descriptors_low_va: low::PB_DESCRIPTORS, - page_pool_descriptors_low_va: low::PAGE_POOL_DESCRIPTORS, - }; - self.object_mut(self.bundle)? - .initialize::(main_offset, |main| { - match role { - Role::Primary => main.init_primary(&args, &views), - Role::Secondary => main.init_secondary(&args), - }; - Ok(()) - })?; - let abi_role = match role { - Role::Primary => abi::Role::Primary, - Role::Secondary => abi::Role::Secondary, - }; - self.object_mut(self.bundle)?.write( - main_offset + abi::MainConfig::CONTROL_RING, - abi::OpeningRecord::new(abi_role), - )?; - let root = abi::RootArgs { - region_a_va: self.object(bundle.region_a)?.gpu_va(), - region_c_va: self.object(self.region_c)?.gpu_va(), - main_config_va: main_va, - status_a_va: status_va, - ias: cfg.uat_ias, - oas: cfg.uat_oas, - }; - match role { - Role::Primary => self.object_mut(self.roots)?.write( - 0, - abi::Root::primary(&root, state_va + abi::private::STATUS_B as u64), - )?, - Role::Secondary => self.object_mut(self.roots)?.write( - abi::SECONDARY_ROOT_OFFSET, - abi::SecondaryRoot::new( - &root, - base + high::PRIVATE_CLUSTER - + (abi::private::STATUS_B + abi::PrimaryStatusB::POWER_STATE) as u64, - state_va + abi::private::STATUS_B as u64, - ), - )?, - } - } - for role in [Role::Primary, Role::Secondary] { - for (offset, value) in [ - (0, 0), - (0x10, 0), - (abi::ChannelState::PRODUCER, abi::OPENING_PRODUCER), - ] { - self.object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? - .store(value, Ordering::Relaxed); - } - } - fence(Ordering::SeqCst); - Ok(()) - } - - /// Retained fixed aliases for an ordinary client VM. The final block of the render-global - /// pool overlaps completion mappings in a client root and is deliberately not installed. - pub(super) fn user_aliases( - &mut self, - vm: &mmu::Vm, - metrics: &KernelObject, - ) -> Result> { - let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; - mappings.push( - self.object_mut(self.render_pages)?.map_alias( - vm, - low::RENDER_PAGE_LIST, - mmu::PROT_GPU_SHARED_RW, - )?, - GFP_KERNEL, - )?; - mappings.push( - self.object_mut(self.render_runs)?.map_alias( - vm, - low::RENDER_RUN_LIST, - mmu::PROT_GPU_SHARED_RW, - )?, - GFP_KERNEL, - )?; - for (block, index) in self.render_blocks.clone().take(21).enumerate() { - mappings.push( - self.object_mut(index)?.map_alias( - vm, - low::render_block(block), - mmu::PROT_GPU_SHARED_RW, - )?, - GFP_KERNEL, - )?; - } - mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, - GFP_KERNEL, - )?; - Ok(mappings) - } - - /// Writes the effort input in firmware order, then cleans its cache line to coherency. - pub(super) fn publish_effort(&self, effort: u32) -> Result { - let region = self.object(self.region_c)?; - let override_word = region.word(abi::RegionC::UNK_E8)?; - let effort_word = region.word(abi::RegionC::EFFORT)?; - let line = region.pointer(abi::RegionC::EFFORT, 4)?; - override_word.store(0, Ordering::Relaxed); - effort_word.store(effort, Ordering::Relaxed); - // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache - // line and the following system barrier publish the two naturally aligned stores. - unsafe { - core::arch::asm!("dc cvac, {line}", "dsb sy", line = in(reg) line, - options(nostack, preserves_flags)) - }; - Ok(()) - } - - pub(super) fn effort(&self) -> Result { - Ok(self - .object(self.region_c)? - .word(abi::RegionC::EFFORT)? - .load(Ordering::Relaxed)) - } - - fn opps(dev: &AsahiDevice) -> Result> { - let node = dev.as_ref().of_node().ok_or(EIO)?; - let table = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; - let mut opps = KVec::new(); - for opp in table.children() { - opps.push( - abi::Opp { - freq_hz: opp.get_property(c_str!("opp-hz"))?, - volt_uv: opp.get_property(c_str!("opp-microvolt"))?, - }, - GFP_KERNEL, - )?; - } - if opps.len() != t8140::OPPS { - return Err(EINVAL); - } - Ok(opps) - } - - fn state_offset(role: Role) -> usize { - match role { - Role::Primary => abi::private::PRIMARY_STATE, - Role::Secondary => abi::private::SECONDARY_STATE, - } - } - - fn status_offset(role: Role) -> usize { - match role { - Role::Primary => abi::private::PRIMARY_STATUS_A, - Role::Secondary => abi::private::SECONDARY_STATUS_A, - } - } - - /// Recheck the host-authored event pointer pair after a bounded number of drains. - /// The device mutex serializes the cache and all event consumption. - pub(super) fn validate_event_ring(&self, role: Role) -> Result { - let status = self.object(self.cluster)?.gpu_va() + Self::status_offset(role) as u64; - let expected = ( - status + abi::status::EVENT_STATE, - status + abi::status::EVENT_RING, - ); - let cache = &self.event_ring_checked[role as usize]; - let checked = cache.get(); - if checked.2 != 0 && (checked.0, checked.1) == expected { - cache.set((checked.0, checked.1, checked.2 - 1)); - return Ok(()); - } - let offset = abi::bundle::MAIN_CONFIG[role as usize] - + core::mem::offset_of!(abi::MainConfig, channels) - + abi::EVENT_CHANNEL * size_of::(); - let bundle = self.object(self.bundle)?; - let actual = ( - bundle.dword(offset)?.load(Ordering::Relaxed), - bundle - .dword(offset + size_of::())? - .load(Ordering::Relaxed), - ); - if actual != expected { - cache.set((0, 0, 0)); - return Err(EINVAL); - } - cache.set((actual.0, actual.1, EVENT_RING_RECHECK)); - Ok(()) - } - - pub(super) fn status_word(&self, role: Role, offset: usize) -> Result<&AtomicU32> { - self.object(self.cluster)?.word( - Self::status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) - } - - pub(super) fn status_read_words( - &self, - role: Role, - offset: usize, - ) -> Result<[u32; N]> { - self.object(self.cluster)?.read_words( - Self::status_offset(role) - .checked_add(offset) - .ok_or(EINVAL)?, - ) - } - - pub(super) fn control_counters(&self) -> Result<[u32; 3]> { - let state = self.object(self.cluster)?; - let mut counters = [0; 3]; - fence(Ordering::Acquire); - for (counter, offset) in counters.iter_mut().zip([ - abi::ChannelState::CONSUMER, - abi::ChannelState::UNK_10, - abi::ChannelState::PRODUCER, - ]) { - *counter = state - .word(abi::CONTROL_STATE_OFFSET + offset)? - .load(Ordering::Relaxed); - } - Ok(counters) - } - - pub(super) fn control_consumer(&self) -> Result<&AtomicU32> { - self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::CONSUMER) - } - - pub(super) fn control_producer(&self) -> Result<&AtomicU32> { - self.object(self.cluster)? - .word(abi::CONTROL_STATE_OFFSET + abi::ChannelState::PRODUCER) - } - - /// Publishes a control record under the device lock. All checked views are acquired before - /// the first write; record and packed power tally precede the aligned producer store. - pub(super) fn publish_control( - &self, - slot: u32, - record: &[u8; 64], - asserts_power: bool, - next: u32, - ) -> Result { - let offset = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + abi::MainConfig::CONTROL_RING - + (slot as usize).checked_mul(record.len()).ok_or(EINVAL)?; - let ring = self.object(self.bundle)?.pointer(offset, record.len())?; - let power = self - .object(self.region_c)? - .pointer(abi::RegionC::POWER_ASSERT_TALLY, 4)?; - let producer = self.control_producer()?; - // SAFETY: The checked pointers remain valid for the lifetime of this non-Sync owner. - // Firmware only reads records after the producer store and the device mutex serializes - // host publishers. The unaligned power word uses the interface's four-byte copy access. - unsafe { - core::ptr::copy_nonoverlapping(record.as_ptr(), ring, record.len()); - if asserts_power { - let mut tally = [0u8; 4]; - core::ptr::copy_nonoverlapping(power, tally.as_mut_ptr(), 4); - let value = u32::from_le_bytes(tally).wrapping_add(1).to_le_bytes(); - core::ptr::copy_nonoverlapping(value.as_ptr(), power, 4); - } - } - fence(Ordering::SeqCst); - producer.store(next, Ordering::Relaxed); - fence(Ordering::SeqCst); - Ok(()) - } - - pub(super) fn root_va(&self, role: Role) -> Result { - Ok(self.object(self.roots)?.gpu_va() + role as u64 * abi::SECONDARY_ROOT_OFFSET as u64) - } - - pub(super) fn opening_retired(&self, role: Role) -> Result { - for offset in [0, 0x10, abi::ChannelState::PRODUCER] { - if self - .object(self.cluster)? - .word(Self::state_offset(role) + abi::CONTROL_STATE_OFFSET + offset)? - .load(Ordering::SeqCst) - != abi::OPENING_PRODUCER - { - return Ok(false); - } - } - Ok(true) - } -} - -impl InitData { - /// Validate the serialized view before reading its retained host mapping. - fn recovery_view_words(&self, view: usize, offset: usize) -> Result<[u32; N]> { - let bundle = self.object(self.bundle)?; - let base = *abi::bundle::VIEWS.get(view).ok_or(EINVAL)?; - let extent = *abi::recovery::VIEW_EXTENTS.get(view).ok_or(EINVAL)?; - let end = offset.checked_add(N.checked_mul(size_of::()).ok_or(EOVERFLOW)?) - .ok_or(EOVERFLOW)?; - if end > extent { - return Err(EINVAL); - } - let pointer_at = abi::bundle::MAIN_CONFIG[Role::Primary as usize] - + core::mem::offset_of!(abi::MainConfig, view_va) + view * size_of::(); - fence(Ordering::Acquire); - let pointer = bundle.read_words::<2>(pointer_at)?; - let actual = u64::from(pointer[0]) | u64::from(pointer[1]) << 32; - if actual != bundle.gpu_va().checked_add(base as u64).ok_or(EOVERFLOW)? { - return Err(EINVAL); - } - bundle.read_words(base + offset) - } -} - -impl super::recovery::Memory for InitData { - fn fault_sources(&self) -> Result { - let mask = self.object(self.cluster)? - .dword(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_SLOT_MASK)? - .load(Ordering::Relaxed); - let mut sources = super::recovery::Sources::sample(mask as u32, |slot| { - let key = super::recovery::slot_key(slot)?; - let monitor = self.recovery_view_words::<6>(abi::recovery::PROGRESS_VIEW, - abi::recovery::PROGRESS_START + slot * abi::recovery::PROGRESS_STRIDE) - .ok().map(|words| words[0])?; - Some((key, monitor)) - }); - sources.reason = self.recovery_view_words::<1>(abi::recovery::REPORT_VIEW, abi::recovery::REASON) - .ok().map(|words| words[0]); - Ok(sources) - } - - fn recovery_state(&self) -> Result { - fence(Ordering::Acquire); - Ok(self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)? - .load(Ordering::Relaxed)) - } - - fn host_recovery(&self) -> Result { - Ok(self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::HOST_RECOVERY)? - .load(Ordering::Relaxed)) - } - - fn transition(&self, expected: u32, next: u32) -> Result { - let state = self - .object(self.cluster)? - .word(abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_STATE)?; - fence(Ordering::Acquire); - if state.load(Ordering::Relaxed) != expected { - return Err(EBUSY); - } - fence(Ordering::SeqCst); - state.store(next, Ordering::Relaxed); - fence(Ordering::SeqCst); - Ok(()) - } - - fn clear_timestamps(&self) -> Result { - let bundle = self.object(self.bundle)?; - // Validate the complete table before publishing its first live word. - const ENTRIES: usize = 128; - const STRIDE: usize = 0x10; - let offset = abi::bundle::VIEWS[4]; - bundle.pointer(offset, ENTRIES * STRIDE)?; - for qid in 0..ENTRIES { - bundle - .word(offset + qid * STRIDE)? - .store(0, Ordering::Relaxed); - bundle - .dword(offset + qid * STRIDE + 8)? - .store(0, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - Ok(()) - } - - fn entries(&self) -> Result { - let cluster = self.object(self.cluster)?; - let base = abi::private::STATUS_B + abi::PrimaryStatusB::RECOVERY_INFO; - fence(Ordering::Acquire); - let mut entries = super::recovery::Entries::default(); - for index in 0..256 { - let offset = base + index * core::mem::size_of::(); - let flags = cluster.word(offset)?.load(Ordering::Relaxed); - entries.include(flags); - } - Ok(entries) - } - - fn wait_tick(&self) { - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/job.rs b/drivers/gpu/drm/asahi-neo/g17/job.rs deleted file mode 100644 index 9e897ea0c59400..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/job.rs +++ /dev/null @@ -1,396 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Immutable submissions and the DRM scheduler boundary for independent engines. - -use super::{command::Validated, completion::Completion, context::Context}; -use crate::driver::AsahiDevice; -use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU8, Ordering}; -use kernel::{ - bindings, c_str, - dma_fence::{Fence, RawDmaFence}, - drm_neo::sched, - prelude::*, - sync::Arc, -}; - -pub(super) const TIMEOUT_MS: kernel::time::Msecs = 2000; -const RENDER_CREDITS: u32 = 32; -const COMPUTE_CREDITS: u32 = 16; - -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum Engine { - Render, - Compute, -} - -impl Engine { - pub(crate) fn index(self) -> usize { - match self { - Self::Render => 0, - Self::Compute => 1, - } - } -} - -/// Logical sequence numbers do not wrap with firmware timestamps. A zero barrier -/// includes preceding ioctls; the sentinel leaves only an explicit historical prefix. -#[derive(Clone, Copy)] -pub(crate) struct Order { - pub(crate) sequence: u64, - pub(crate) wait_through: [Option; 2], -} - -impl Order { - pub(crate) fn new( - base: [u64; 2], - preceding: [u16; 2], - barriers: [u16; 2], - prior: [u64; 2], - engine: Engine, - ) -> Result { - let mut sequences = [0; 2]; - let mut wait_through = [None; 2]; - for index in 0..2 { - // Earlier commands in this ioctl are not historical commands. - if prior[index] > base[index] { - return Err(EINVAL); - } - sequences[index] = base[index] - .checked_add(u64::from(preceding[index])) - .and_then(|value| value.checked_add(1)) - .ok_or(EOVERFLOW)?; - if barriers[index] != u16::MAX { - if barriers[index] > preceding[index] { - return Err(EINVAL); - } - wait_through[index] = Some( - base[index] - .checked_add(u64::from(barriers[index])) - .ok_or(EOVERFLOW)?, - ); - } - if prior[index] != 0 { - wait_through[index] = Some( - wait_through[index].map_or(prior[index], |last| last.max(prior[index])), - ); - } - } - Ok(Self { - sequence: sequences[engine.index()], - wait_through, - }) - } - - pub(crate) fn contains(prefix: Option, sequence: u64) -> bool { - sequence != 0 && prefix.is_some_and(|last| sequence <= last) - } -} - -pub(crate) struct Packet { - pub(crate) id: u64, - pub(crate) context: Arc, - pub(crate) command: Validated, - pub(crate) order: Order, - pub(crate) completion: Arc, - pub(crate) footprint: Option>, - inputs: KVec, - firmware: KVec, - publication: AtomicU8, - cancelled: AtomicBool, - early: AtomicI32, - backlog: AtomicBool, -} - -impl Packet { - pub(crate) fn new( - id: u64, - context: Arc, - command: Validated, - order: Order, - completion: Arc, - inputs: KVec, - firmware: KVec, - ) -> Result> { - let footprint = match &command { - Validated::Render { attachments, .. } => { - context.vm().render_footprint(&attachments[1])? - } - Validated::Compute { .. } => None, - }; - Ok(Arc::new( - Self { - id, - context, - command, - order, - completion, - footprint, - inputs, - firmware, - publication: AtomicU8::new(0), - cancelled: AtomicBool::new(false), - early: AtomicI32::new(0), - backlog: AtomicBool::new(false), - }, - GFP_KERNEL, - )?) - } - - pub(crate) fn engine(&self) -> Engine { - match &self.command { - Validated::Render { .. } => Engine::Render, - Validated::Compute { .. } => Engine::Compute, - } - } - - /// Scheduler dependency readiness includes failed fences. Consume the actual - /// status before publication, and retain the references through that check. - pub(crate) fn check_dependencies(&self) -> Result { - for fence in &self.inputs { - // SAFETY: The retained fence owns the pointer throughout the wait and read. - let waited = unsafe { - bindings::dma_fence_wait_timeout( - fence.raw(), - true, - kernel::task::MAX_SCHEDULE_TIMEOUT, - ) - }; - if waited < 0 { - return Err(super::dependency::inherited_error(Error::from_errno(waited as i32))); - } - if waited == 0 { - return Err(ECANCELED); - } - check_status(fence)?; - } - for fence in &self.firmware { - check_status(fence)?; - } - Ok(()) - } - - /// Dependency-bearing packets remain on the scheduler publication path. - pub(crate) fn has_dependencies(&self) -> bool { - !self.inputs.is_empty() || !self.firmware.is_empty() - } - - pub(crate) fn begin_publication(&self) -> bool { - self.publication - .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } - pub(crate) fn mark_published(&self) { - self.publication.store(2, Ordering::Release); - } - pub(crate) fn is_published(&self) -> bool { - self.publication.load(Ordering::Acquire) == 2 - } - pub(crate) fn cancel_requested(&self) -> bool { - self.cancelled.load(Ordering::Acquire) - } - pub(crate) fn request_cancel(&self) -> bool { - self.cancelled.store(true, Ordering::Release); - self.publication - .compare_exchange(0, 3, Ordering::AcqRel, Ordering::Acquire) - .is_ok() - } - pub(crate) fn set_early_result(&self, result: Result) { - self.early.store( - result.map_or_else(|error| error.to_errno(), |_| 1), - Ordering::Release, - ); - } - pub(crate) fn early_result(&self) -> Option { - match self.early.load(Ordering::Acquire) { - 0 => None, - 1 => Some(Ok(())), - error => Some(Err(Error::from_errno(error))), - } - } - pub(crate) fn count_backlog(&self) { - self.backlog.store(true, Ordering::Release); - } - pub(crate) fn take_backlog(&self) -> bool { - self.backlog.swap(false, Ordering::AcqRel) - } -} - -pub(crate) fn fence_status(fence: &Fence) -> i32 { - // SAFETY: The owned fence keeps the raw DMA fence alive for this read. - unsafe { bindings::dma_fence_get_status(fence.raw()) } -} - -fn check_status(fence: &Fence) -> Result { - let status = fence_status(fence); - if status < 0 { - Err(super::dependency::inherited_error(Error::from_errno(status))) - } else { - Ok(()) - } -} - -/// The backend retains visible packets through exact retirement or processor stop. -/// It performs all completion signalling after releasing the device mutex. -pub(crate) trait Backend: Send + Sync { - /// Device detector retained by render completions after queue teardown. - fn feed(&self) -> Option> { - None - } - - /// Lazily create or rebind this logical queue's physical compute graph. - fn ensure_compute(&self) -> Result; - /// Settle outstanding work of a terminal VM before an ioctl error is published. - fn fail_vm(&self, error: Error); - /// Called after both scheduler entities have cancelled their queued jobs. - fn release_owner(&self); - fn prepare(&self, packet: &Packet) -> Option; - fn publish(&self, packet: Arc) -> Result; - fn timed_out(&self, packet: Arc) -> sched::Status; - fn cancel(&self, packet: Arc); - /// Transfer a timeout to an outstanding off-lock graph transaction, if any. - fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool; - /// Return true only after taking publication ownership and recording its result. - fn try_early(&self, packet: &Arc) -> bool; - fn count_backlog(&self, packet: &Packet); -} - -struct Job { - backend: Arc, - packet: Arc, -} - -struct Timeout { - backend: Arc, - victim_renewals: u8, - packet: Arc, -} - -impl Drop for Job { - fn drop(&mut self) { - // Entity teardown can free a queued job without invoking cancel. - // Its independent fence may still order another engine. A claimed - // publisher keeps responsibility for visible work and its VM pin. - if self.packet.publication.load(Ordering::Acquire) == 0 { - self.backend.cancel(self.packet.clone()); - } - } -} - -impl sched::JobImpl for Job { - type TimeoutData = Timeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - - fn timeout_data(&self) -> Timeout { - Timeout { - backend: self.backend.clone(), - victim_renewals: 0, - packet: self.packet.clone(), - } - } - - fn false_timeout(job: &mut Timeout, finished: bool) -> bool { - if finished || job.packet.completion.take_replay_timeout() { - return true; - } - let deferred = job.backend.defer_timeout(&job.packet, job.victim_renewals); - if deferred && job.packet.engine() == Engine::Compute { - job.victim_renewals += 1; - } - deferred - } - - fn prepare(job: &mut Self) -> Option { - job.backend.prepare(&job.packet) - } - fn run(job: &mut Self) -> Result> { - job.backend.publish(job.packet.clone()).map(Some) - } - fn timed_out(job: &mut Timeout, finished: bool) -> sched::Status { - // The scheduler retains the detached job throughout this callback. Keep - // its parent callback intact while the driver settles the exact owner; - // reinsertion also handles completion racing with timeout observation. - if !finished { - job.backend.timed_out(job.packet.clone()); - } - sched::Status::NoHang - } - fn cancel(job: &mut Self) { - job.backend.cancel(job.packet.clone()); - } -} - -pub(crate) struct Scheduler { - // Cancel entity jobs before dropping the scheduler's owning reference. - entity: sched::Entity>, - _scheduler: sched::Scheduler>, - backend: Arc, -} - -#[derive(Clone)] -pub(crate) struct Fences { - pub(crate) ready: Fence, - pub(crate) completed: Fence, - pub(crate) publication: Option>, -} - -impl Scheduler { - pub(crate) fn new(dev: &AsahiDevice, backend: Arc, engine: Engine) -> Result { - let (credits, name) = match engine { - Engine::Render => (RENDER_CREDITS, c_str!("asahi_g17_render")), - Engine::Compute => (COMPUTE_CREDITS, c_str!("asahi_g17_compute")), - }; - let scheduler = sched::Scheduler::new(dev.as_ref(), 1, credits, 0, TIMEOUT_MS, name)?; - let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; - Ok(Self { - entity, - _scheduler: scheduler, - backend, - }) - } - - pub(crate) fn enqueue( - &mut self, - packet: Arc, - dependencies: KVec, - early: bool, - publication: Option, - ) -> Result { - let completed = packet.completion.fence(); - let host_timestamps = packet.completion.has_timestamps(); - // Only commands which actually write timestamps become host-prefix - // producers. Ordinary consumers retain the scheduled ready fence and - // their existing firmware dependency path for cross-engine overlap. - let host_publication = if host_timestamps { - packet.completion.publication() - } else { - None - }; - let mut job = self.entity.new_job( - 1, - Job { - backend: self.backend.clone(), - packet: packet.clone(), - }, - )?; - for dependency in dependencies { - job.add_dependency(dependency)?; - } - let mut job = job.arm(); - packet.completion.mark_accepted(); - if let Some(publication) = publication { - publication.activate(); - } - let ready = if host_timestamps { - completed.clone() - } else { - job.fences().scheduled() - }; - // All fallible scheduler setup precedes early hardware publication. The - // scheduler still owns watchdog, credits and cancellation after push. - if packet.engine() == Engine::Render && !(early && self.backend.try_early(&packet)) { - self.backend.count_backlog(&packet); - } - job.push(); - Ok(Fences { ready, completed, publication: host_publication }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/kick.rs b/drivers/gpu/drm/asahi-neo/g17/kick.rs deleted file mode 100644 index f109c8261476e3..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/kick.rs +++ /dev/null @@ -1,355 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Physical kick-ring backing, queue identities and entry publication. -//! -//! The device mutex serializes identities and publication. Installed identities -//! and their ring backing stay owned until both firmware processors stop; -//! command completion alone does not detach a hardware queue. - -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; - -use super::{ - fw::{kick::*, queue::DataMaster}, - object::{Allocator, CpuMap, KernelObject}, -}; -use crate::{hw::t8140, mem, mmu}; - -pub(super) const QID_COUNT: usize = QID_MAX as usize + 1; - -/// QIDs kept free for render pairs when compute queues fill the table. -const COMPUTE_RENDER_RESERVE: usize = 16; - -/// Stable physical-channel identity. The owner must not be reused within a -/// device lifetime, including after an unpublished allocation is cancelled. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) struct Id { - qid: u8, - owner: u64, - engine: DataMaster, -} - -impl Id { - pub(crate) fn qid(self) -> u8 { - self.qid - } - pub(crate) fn engine(self) -> DataMaster { - self.engine - } -} - -#[derive(Copy, Clone)] -struct Identity { - id: Id, - published: bool, -} - -/// One shared namespace for tiling, fragment and compute queues. -pub(crate) struct QueueIds { - entries: [Option; QID_COUNT], - compute_publication: u64, - leased: usize, -} - -impl QueueIds { - pub(crate) fn new() -> impl Init { - kernel::try_init!(Self { - entries <- pin_init::init_array_from_fn(|_| None), - compute_publication: 0, - leased: 0, - }) - } - - /// Reuses only an exact owner/engine match. Preferred placement is for - /// bootstrapping; it does not reserve a partition for an engine. - pub(crate) fn reserve( - &mut self, - owner: u64, - engine: DataMaster, - preferred: Option, - ) -> Result { - if owner == 0 { - return Err(EINVAL); - } - if let Some(entry) = self - .entries - .iter() - .flatten() - .find(|entry| entry.id.owner == owner) - { - return if entry.id.engine == engine { - Ok(entry.id) - } else { - Err(EINVAL) - }; - } - // Healthy existing channels keep their QID. Never recycle a published - // identity to recover capacity after a fault; leave eight render pairs. - if engine == DataMaster::Compute && QID_COUNT - self.leased <= COMPUTE_RENDER_RESERVE { - return Err(ENOSPC); - } - let index = preferred - .map(usize::from) - .filter(|index| self.entries.get(*index).is_some_and(Option::is_none)) - .or_else(|| self.entries.iter().position(Option::is_none)) - .ok_or(ENOSPC)?; - let id = Id { - qid: index as u8, - owner, - engine, - }; - self.entries[index] = Some(Identity { - id, - published: false, - }); - self.leased += 1; - Ok(id) - } - - /// Ordered under the device mutex; no atomic operation on the submission path. - pub(crate) fn next_compute_publication(&mut self) -> Result { - self.compute_publication = self.compute_publication.checked_add(1).ok_or(EOVERFLOW)?; - Ok(self.compute_publication) - } - - /// Call after preflighting the complete installation, before any address - /// becomes firmware-visible. A failed or partial installation retains it. - pub(crate) fn publish(&mut self, id: Id) -> Result { - let entry = self - .entries - .get_mut(usize::from(id.qid)) - .and_then(Option::as_mut) - .ok_or(EINVAL)?; - if entry.id != id { - return Err(EINVAL); - } - entry.published = true; - Ok(()) - } - - pub(crate) fn cancel_unpublished(&mut self, id: Id) -> Result { - let slot = self.entries.get_mut(usize::from(id.qid)).ok_or(EINVAL)?; - match slot { - Some(entry) if entry.id == id && !entry.published => { - *slot = None; - self.leased -= 1; - Ok(()) - } - _ => Err(EINVAL), - } - } -} - -// Aliases drop before their backing. A compute queue is allocated low first; -// a render queue is allocated high first. -enum Backing { - Compute { - high: mmu::KernelMapping, - low: KernelObject, - }, - Render { - low: mmu::KernelMapping, - high: KernelObject, - }, -} - -/// An installed queue's permanent ring. Client aliases are held by the bound -/// context graph and must be dropped before this object. -pub(crate) struct Queue { - backing: Backing, - id: Id, - timestamp: KickTimestamp, - previous: Option, -} - -/// A checked entry and destination held without changing the shared ring. -/// Discarding it leaves both the slot and the next timestamp unchanged. -pub(crate) struct PreparedKick<'a> { - queue: &'a mut Queue, - timestamp: KickTimestamp, - entry: KickEntry, - destination: *mut KickEntry, -} - -impl PreparedKick<'_> { - /// Publish translation updates after the queue configuration pointer and before its entry. - pub(crate) fn prepare_compute_install(&self) -> Result { - self.queue.prepare_compute_install() - } - - /// Writes and fences the complete entry, then advances host state. The - /// caller invokes this between the queue configuration and announcement. - pub(crate) fn commit(self) -> KickTimestamp { - // SAFETY: prepare_entry checked the complete destination range and - // this token exclusively borrows the queue and retains its mapping. - // The caller only prepares retired ring slots; no firmware access to - // the new entry starts before the outer command is published. - unsafe { self.destination.write_unaligned(self.entry) }; - fence(Ordering::SeqCst); - self.queue.previous = Some(self.timestamp); - self.queue.timestamp = self.timestamp.next(); - self.timestamp - } -} - -impl Queue { - pub(crate) fn compute(alloc: &Allocator<'_>, id: Id) -> Result { - if id.engine != DataMaster::Compute { - return Err(EINVAL); - } - let low = alloc.lower( - KICK_RING_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - CpuMap::WriteCombined, - )?; - let mut range = alloc.uat.geometry().kernel_range(); - range.start += t8140::dynamic::KERNEL_OFFSET; - let high = low.alias_in( - alloc.uat.kernel_vm(), - range, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - )?; - fence(Ordering::SeqCst); - Ok(Self { - backing: Backing::Compute { high, low }, - id, - timestamp: KickTimestamp::FIRST, - previous: None, - }) - } - - pub(crate) fn render( - alloc: &Allocator<'_>, - id: Id, - client: &mmu::Vm, - ) -> Result<(Self, mmu::KernelMapping)> { - if id.engine == DataMaster::Compute { - return Err(EINVAL); - } - let high = alloc.kernel( - KICK_RING_SIZE, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let mut range = t8140::dynamic::LOWER; - let (low, client) = loop { - let low = high.alias_in( - alloc.uat.kernel_lower_vm(), - range.clone(), - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - )?; - let end = low - .iova() - .checked_add(KICK_RING_SIZE as u64) - .ok_or(EOVERFLOW)?; - match high.map_alias(client, low.iova(), mmu::PROT_GPU_SHARED_RW) { - Ok(client) => break (low, client), - Err(error) if error == ENOSPC && end < range.end => { - drop(low); - range.start = end; - } - Err(error) => return Err(error), - } - }; - Ok(( - Self { - backing: Backing::Render { low, high }, - id, - timestamp: KickTimestamp::FIRST, - previous: None, - }, - client, - )) - } - - pub(crate) fn id(&self) -> Id { - self.id - } - pub(crate) fn low_va(&self) -> u64 { - match &self.backing { - Backing::Compute { low, .. } => low.gpu_va(), - Backing::Render { low, .. } => low.iova(), - } - } - pub(crate) fn firmware_va(&self) -> u64 { - match &self.backing { - Backing::Compute { high, .. } => high.iova(), - Backing::Render { high, .. } => high.gpu_va(), - } - } - fn object(&mut self) -> &mut KernelObject { - match &mut self.backing { - Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, - } - } - pub(crate) fn map_client(&mut self, vm: &mmu::Vm) -> Result { - let address = self.low_va(); - self.object() - .map_alias(vm, address, mmu::PROT_GPU_SHARED_RW) - } - - /// Reuse a permanent client view only when every page names this ring. - pub(crate) fn prepare_client_alias(&self, vm: &mmu::Vm) -> Result> { - let object = match &self.backing { - Backing::Compute { low, .. } => low, - Backing::Render { high, .. } => high, - }; - object.prepare_alias(vm, self.low_va(), mmu::PROT_GPU_SHARED_RW, true) - } - - /// Both aliases are owned mappings of the same full ring. Publish their - /// page-table updates to each live translation regime before installation. - pub(crate) fn prepare_compute_install(&self) -> Result { - if self.id.engine != DataMaster::Compute { - return Err(EINVAL); - } - fence(Ordering::SeqCst); - for asid in 0..4 { - mem::tlbi_range_or_asid(asid, self.low_va() as usize, KICK_RING_SIZE); - mem::tlbi_range_or_asid(asid, self.firmware_va() as usize, KICK_RING_SIZE); - } - mem::sync(); - Ok(()) - } - - /// Detaches the implicit dependency after firmware recovery, retaining - /// the monotonically advancing 40-bit producer. - pub(crate) fn clear_parent_after_recovery(&mut self) { - self.previous = None; - } - - pub(crate) fn timestamp(&self) -> KickTimestamp { - self.timestamp - } - pub(crate) fn parent(&self) -> KickTimestamp { - self.previous.unwrap_or_else(|| match self.id.engine { - DataMaster::Compute => KickTimestamp::ZERO, - _ => self.timestamp.prev(), - }) - } - pub(crate) fn prepare_entry(&mut self, args: &KickArgs<'_>) -> Result> { - if args.qid != self.id.qid - || args.timestamp != self.timestamp - || args.parent != self.parent() - { - return Err(EINVAL); - } - let entry = KickEntry::new(args)?; - let destination = self - .object() - .pointer(args.timestamp.entry_offset(), size_of::())? - .cast(); - let timestamp = self.timestamp; - Ok(PreparedKick { - queue: self, - timestamp, - entry, - destination, - }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/mod.rs b/drivers/gpu/drm/asahi-neo/g17/mod.rs deleted file mode 100644 index e2f25bf96ffc68..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/mod.rs +++ /dev/null @@ -1,1233 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! G17 GPU support. -//! -//! G17 GPUs run their firmware on two coprocessor instances, the primary and the secondary -//! [`Role`]. Each has its own CPU control registers, RTKit mailbox, crash log buffer and kernel -//! page table root, and both share the UAT, the TTBAT and the firmware data structures that the -//! driver allocates at boot (see [`initdata`]). -//! -//! The firmware notifies the host of new records in its event and trace rings with a message on -//! either coprocessor; the event worker drains them. -//! -//! # Locking -//! -//! `Shared::state` is the device lock. It serializes all firmware-visible publication and all -//! firmware-shared state, including the bounded waits for the firmware that publication needs. -//! Completions are signalled after dropping it. Lock order: `Shared::state` -> UAT shared state -//! -> VM exec lock -> GEM reservation. `RoleState::ack` nests inside nothing. - -mod buffer; -mod channel; -mod command; -pub(crate) use command::Attachments; -mod completion; -pub(crate) mod context; -mod dependency; -mod event; -mod feed; -mod fence; -mod freelist; -pub(crate) mod fw; -mod ids; -pub(crate) mod hazard; -mod initdata; -mod job; -mod kick; -mod object; -mod power; -mod preparation; -mod qos; -mod queue; -mod recovery; -mod runtime; -pub(crate) mod status; -mod teardown; -mod timeout; -mod validation; - -use core::any::Any; -use core::ops::Range; -use core::sync::atomic::{ - AtomicBool, - Ordering, // -}; - -use kernel::{ - c_str, - device::{self, Core}, - drm_neo::gem::shmem, - impl_has_delayed_work, impl_has_work, - iosys_map::IoSysMapRef, - new_condvar, new_delayed_work, new_mutex, new_work, platform, - prelude::*, - soc::apple::rtkit, - sync::{ - aref::ARef, - Arc, - CondVar, - Mutex, // - }, - time::{ - delay::fsleep, - msecs_to_jiffies, - Delta, // - }, - types::{ForeignOwnable, Opaque}, - uapi, - workqueue::{ - self, - DelayedWork, - Work, - WorkItem, // - }, -}; - -use crate::driver::{ - AsahiDevRef, - AsahiDevice, // -}; -use crate::gem::AsahiObject; -use crate::{ - alloc, - gem, - gpu, - hw, - mmu, - regs, // -}; - -use object::{ - CpuMap, - KernelObject, - Placement, // -}; - -/// Expected identification of a G17 GPU. -/// One monotonic domain for recovery grace, idle effort, pool retention and -/// control-ring retirement. -pub(crate) fn now_ns() -> u64 { - ::ktime_get() as u64 -} - -pub(crate) struct Identity { - /// GPU family field of the ID register. - pub(crate) family: u8, - /// GPU variant field of the ID register. - pub(crate) variant: u8, - /// Number of GPU dies. - pub(crate) num_dies: u32, - /// GPU variant reported to userspace. - pub(crate) gpu_variant: hw::GpuVariant, - /// Shader core generation reported to userspace. - pub(crate) usc_generation: u32, - /// Hardware abstraction generation reported to userspace. - pub(crate) hal_generation: u32, -} - -/// A buffer that the firmware expects at a fixed address of the kernel lower root. -pub(crate) struct FixedBuffer { - /// GPU address. - pub(crate) va: u64, - /// Size in bytes. - pub(crate) size: usize, -} - -/// Static configuration of a SoC with a G17 GPU. -pub(crate) struct Config { - /// SoC identifier. - pub(crate) chip_id: u32, - /// Expected GPU identification. - pub(crate) identity: Identity, - /// Input address bits translated by each UAT root. - pub(crate) uat_ias: u32, - /// Output (physical) address bits of the UAT. - pub(crate) uat_oas: u32, - /// Offset of the secondary coprocessor's kernel root in the `pagetables` region. - pub(crate) secondary_root_offset: u64, - /// Crash log buffer of each [`Role`]. - pub(crate) crash_buffers: [FixedBuffer; 2], - /// Parameter buffer page metrics shared with the firmware. - pub(crate) pm_metrics: FixedBuffer, -} - -/// A firmware coprocessor instance. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum Role { - /// The coprocessor that owns the device-control ring. - Primary = 0, - /// The coprocessor that only runs work the primary one hands to it. - Secondary = 1, -} - -/// RTKit endpoint of firmware messages: initialization, its acknowledgement and event -/// notifications. -const EP_FIRMWARE: u8 = 0x20; -/// RTKit endpoint of host notifications to the firmware. -const EP_DOORBELL: u8 = 0x21; - -/// Bit position of the type field of firmware messages. -const MSG_TYPE_SHIFT: u32 = 48; -/// Host message handing a coprocessor the address of its initialization root. -const MSG_INITDATA: u64 = 0x81 << MSG_TYPE_SHIFT; -/// Address bits of an initialization root that [`MSG_INITDATA`] carries. -const MSG_INITDATA_ADDR_MASK: u64 = (1 << 44) - 1; -/// Firmware message acknowledging [`MSG_INITDATA`]. -const MSG_INITDATA_ACK: u64 = 0x09; -/// Firmware message announcing new records in the event and trace rings. -const MSG_EVENTS: u64 = 0x42 << MSG_TYPE_SHIFT; -/// Primary processor notification for newly published device-control records. -const MSG_CONTROL_NOTIFY: u64 = (0x84 << MSG_TYPE_SHIFT) | 0x11; -/// Host message, sent to both coprocessors once both are initialized, ending the power-on -/// transition. The firmware then retires the opening device-control records. -const MSG_POWER_ON_DONE: u64 = 0x89 << MSG_TYPE_SHIFT; - -/// Time the firmware takes at most to acknowledge its initialization. -const INITDATA_ACK_TIMEOUT_MS: u32 = 2000; -/// Bounded mailbox-provider readiness retries, before either coprocessor starts. -const RECEIVER_ATTEMPTS: usize = 3; -const RECEIVER_RETRY_MS: i64 = 20; -/// The firmware does not notify the host when it retires device-control records, so their -/// counters are sampled initially and after at most [`OPENING_RETIRE_POLLS`] waits. -const OPENING_RETIRE_POLL_MS: i64 = 1; -const OPENING_RETIRE_POLLS: usize = 100; - -/// Address bits of a buffer the coprocessor places itself; the coprocessor sets higher bits in -/// its buffer requests. -const RTKIT_BUFFER_ADDR_MASK: u64 = (1 << 40) - 1; -/// Initial contents of crash log buffers, so that a crash log shows what the firmware wrote. -const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; - -/// A crash log buffer handed to RTKit. -pub(crate) struct CrashLog { - iova: usize, - map: shmem::VMap, -} - -impl rtkit::Buffer for CrashLog { - fn iova(&self) -> Result { - Ok(self.iova) - } - - fn buf(&mut self) -> Result> { - Ok(self.map.get()) - } -} - -/// Callback state; initialization ACKs can arrive in hard interrupt context. -#[pin_data] -struct RoleState { - role: Role, - shared: Arc, - crash_log: FixedBuffer, - crash_map: shmem::VMap, - crashed: AtomicBool, - #[pin] - ack: Opaque, -} - -// SAFETY: Completion access uses the kernel completion API; the remaining fields are immutable -// shared references or atomics. The completion is pinned before callbacks can observe it. -unsafe impl Send for RoleState {} -// SAFETY: As above; complete_all is safe concurrently with the single boot waiter. -unsafe impl Sync for RoleState {} - -impl RoleState { - fn signal(&self, crashed: bool) { - if crashed { - self.crashed.store(true, Ordering::Release); - } - // SAFETY: ack is initialized and pinned for the lifetime of the callback state. - unsafe { kernel::bindings::complete_all(self.ack.get()) }; - } - - fn wait_for_ack(&self) -> Result { - // SAFETY: ack is initialized and remains pinned while this reference exists. - let completed = unsafe { - kernel::bindings::wait_for_completion_timeout( - self.ack.get(), - msecs_to_jiffies(INITDATA_ACK_TIMEOUT_MS), - ) - }; - if self.crashed.load(Ordering::Acquire) { - return Err(EIO); - } - if completed == 0 { - return Err(ETIMEDOUT); - } - Ok(()) - } -} - -fn is_events_message(endpoint: u8, message: u64) -> bool { - endpoint == EP_FIRMWARE && message == MSG_EVENTS -} - -/// RTKit callbacks of a coprocessor. -struct RoleOps; - -#[vtable] -impl rtkit::Operations for RoleOps { - type Data = Arc; - type Buffer = CrashLog; - - const COPROC_PLACES_BUFFERS: bool = true; - - fn recv_message_early( - data: ::Borrowed<'_>, - ep: u8, - msg: u64, - ) -> bool { - if ep == EP_FIRMWARE && msg >> MSG_TYPE_SHIFT == MSG_INITDATA_ACK { - data.signal(false); - return true; - } - if !is_events_message(ep, msg) { - return false; - } - data.shared.queue_events(); - true - } - - fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { - data.signal(true); - data.shared.lose_device(data.role, crashlog); - } - - fn shmem_map( - data: ::Borrowed<'_>, - iova: usize, - size: usize, - ) -> Result { - if (iova as u64) & RTKIT_BUFFER_ADDR_MASK != data.crash_log.va || size > data.crash_log.size - { - return Err(EINVAL); - } - Ok(CrashLog { - iova, - map: data.crash_map.clone(), - }) - } -} - -/// A running firmware coprocessor. -/// -/// Dropping it stops the CPU before RTKit and the crash log buffer go away. -struct Coprocessor { - cpu: regs::CpuControl, - running: bool, - rtkit: rtkit::RtKit, - state: Arc, - _crash_buffer: KernelObject, -} - -impl Coprocessor { - /// Sets up a coprocessor's RTKit and crash log buffer, without starting it. - fn new( - pdev: &platform::Device, - dev: &AsahiDevice, - cfg: &Config, - uat: &mmu::Uat, - shared: &Arc, - role: Role, - ) -> Result { - let index = role as usize; - let buffer = &cfg.crash_buffers[index]; - let mut crash_buffer = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(buffer.va), - buffer.size, - mmu::PROT_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - crash_buffer.bytes_mut().fill(CRASH_BUFFER_FILL[index]); - - let crash_map = crash_buffer.kernel_map()?; - let state = Arc::pin_init( - pin_init!(RoleState { - role, - shared: shared.clone(), - crash_log: FixedBuffer { - va: buffer.va, - size: buffer.size, - }, - crash_map, - crashed: AtomicBool::new(false), - ack <- Opaque::ffi_init(|ptr| { - // SAFETY: ptr is valid uninitialized completion storage that is pinned - // before any RTKit callback can use it. - unsafe { kernel::bindings::init_completion(ptr) }; - }), - }), - GFP_KERNEL, - )?; - - Ok(Coprocessor { - cpu: regs::CpuControl::new(pdev, index)?, - running: false, - rtkit: Self::receiver(dev, index, &state)?, - state, - _crash_buffer: crash_buffer, - }) - } - - fn receiver( - dev: &AsahiDevice, - index: usize, - state: &Arc, - ) -> Result> { - for attempt in 0..RECEIVER_ATTEMPTS { - match rtkit::RtKit::::new(dev.as_ref(), None, index, state.clone()) { - Ok(receiver) => return Ok(receiver), - Err(error) if error == EINVAL || error == ETIMEDOUT => { - if attempt + 1 < RECEIVER_ATTEMPTS { - fsleep(Delta::from_millis(RECEIVER_RETRY_MS)); - } - } - Err(error) => return Err(error), - } - } - Err(EPROBE_DEFER) - } - - /// Starts the CPU and waits for RTKit to come up. - fn boot(&mut self) -> Result { - self.cpu.start()?; - self.running = true; - Pin::new(&mut self.rtkit).boot() - } - - /// Starts the firmware endpoints and hands the firmware its initialization root. - fn init(&mut self, root_va: u64) -> Result { - let mut rtkit = Pin::new(&mut self.rtkit); - rtkit.as_mut().start_endpoint(EP_FIRMWARE)?; - rtkit.as_mut().start_endpoint(EP_DOORBELL)?; - rtkit.send_message( - EP_FIRMWARE, - MSG_INITDATA | (root_va & MSG_INITDATA_ADDR_MASK), - )?; - self.state.wait_for_ack() - } - - /// Sends a host notification to the firmware. - fn notify(&mut self, msg: u64) -> Result { - Pin::new(&mut self.rtkit).send_message(EP_DOORBELL, msg) - } - - /// Stops the CPU. The firmware keeps its state, but no longer runs. - fn stop(&mut self) { - if self.running { - self.cpu.stop(); - self.running = false; - } - } -} - -impl Drop for Coprocessor { - fn drop(&mut self) { - self.stop(); - } -} - -/// The GPU-to-fabric bridge, enabled for as long as this exists. -struct Bridge { - regs: regs::Resources, - enabled: bool, -} - -impl Bridge { - fn enable(regs: regs::Resources) -> Result { - let bridge = Bridge { - regs, - enabled: true, - }; - bridge.regs.enable_g17_bridge()?; - Ok(bridge) - } - - fn disable(&mut self) { - if self.enabled { - self.regs.disable_g17_bridge(); - self.enabled = false; - } - } -} - -impl Drop for Bridge { - fn drop(&mut self) { - self.disable(); - } -} - -/// Everything that exists while the firmware runs. -/// -/// Field order is teardown order: both coprocessors stop (in [`Drop`]) before their RTKit is -/// dropped, the bridge is disabled once they are stopped, and the firmware objects and their -/// mappings go before the UAT. Page tables the firmware or the bootloader own are never freed. -struct Firmware { - secondary: Coprocessor, - primary: Coprocessor, - bridge: Bridge, - rings: channel::Rings, - recovery: recovery::State, - effort: power::Effort, - registration_power_release_pending: bool, - queues: runtime::Registry, - pool: Arc, - render_ids: Arc, - init: initdata::InitData, - _render_global: mmu::Vm, - _pm_metrics: Arc, - uat: Arc, -} - -impl Drop for Firmware { - fn drop(&mut self) { - self.secondary.stop(); - self.primary.stop(); - self.pool.stop_after_firmware(); - // RTKit destruction releases the mailbox power references. Clear the bridge while - // those references still keep the register block powered. - self.bridge.disable(); - } -} - -impl Firmware { - /// Publishes controls only while the restart handshake is closed. The device mutex remains - /// held through publication and the following mailbox notification. - fn publish_control(&mut self, record: &fw::channels::ControlRecord) -> Result { - use recovery::Memory; - if self.recovery.pending() || self.init.recovery_state()? != 0 { - return Err(EAGAIN); - } - let target = channel::Rings::publish_control(&self.init, record)?; - core::sync::atomic::fence(Ordering::SeqCst); - self.primary.notify(MSG_CONTROL_NOTIFY)?; - Ok(target) - } - - /// Holds idle descent while a fresh physical render pair is registered. - fn acquire_registration_power(&mut self) -> Result { - power::acquire(self) - } - - fn release_registration_power(&mut self) -> Result { - power::release(self) - } - - /// Called at actual submission publication, not while a queued job is merely being built. - fn note_submission(&mut self) -> Result { - self.effort.submit(); - self.apply_effort(power::FULL_EFFORT) - } - - fn apply_effort(&mut self, effort: u32) -> Result { - Self::apply_shared_effort(&self.init, &mut self.effort, effort) - } - - fn apply_shared_effort( - init: &initdata::InitData, - state: &mut power::Effort, - effort: u32, - ) -> Result { - if state.applied() == effort && init.effort()? == effort { - return Ok(()); - } - init.publish_effort(effort)?; - state.did_apply(effort); - Ok(()) - } - - /// Returns the delay in ns for a follow-up idle check. Queue owners report only submitted, - /// unretired work; quarantined work retaining mappings is excluded. - fn idle_check(&mut self, now_ns: u64, in_flight: bool) -> Result> { - match self.effort.idle(now_ns, in_flight) { - power::IdleAction::None => Ok(None), - power::IdleAction::Clear => { - self.apply_effort(0)?; - Ok(None) - } - power::IdleAction::Recheck(delay) => Ok(Some(delay)), - } - } -} - -impl power::Control for Firmware { - fn crashed(&self) -> bool { - self.primary.state.shared.crashed.load(Ordering::Acquire) - } - fn powered(&self) -> bool { - self.bridge.regs.g17_configuration_powered() - } - fn counters(&self) -> Result<[u32; 3]> { - self.init.control_counters() - } - fn set_idle(&mut self, enabled: bool) -> Result { - use fw::channels::{ControlRecord, IdlePolicy, IdlePowerOff}; - self.publish_control(&ControlRecord::IdlePowerOff(IdlePowerOff::new( - if enabled { - IdlePolicy::AllowPowerOff - } else { - IdlePolicy::KeepPowered - }, - ))) - } - fn wait_tick(&self) { - fsleep(Delta::from_millis(1)); - } - fn registration_release_failed(&mut self) { - self.registration_power_release_pending = true; - } - fn registration_release_succeeded(&mut self) { - self.registration_power_release_pending = false; - } -} - -/// Event work item ID. -const EVENT_WORK: u64 = 0; -/// Short control/retirement polls do not wait behind the longer idle deadline. -const POLL_WORK: u64 = 1; -const IDLE_WORK: u64 = 2; -const GROW_WORK: u64 = 3; -const RECLAIM_WORK: u64 = 4; -const FEED_WORK: u64 = 5; - -/// Device state shared with the coprocessor callbacks and the event worker. -#[pin_data] -struct Shared { - /// Diagnostics may outlive shutdown without retaining the DRM data graph. - dev: ARef, - /// Allocation users take independent references before reserving resources. - /// Shutdown removes this owning backedge after joining device work. - #[pin] - drm_neo: Mutex>, - clusters: u32, - descriptor_flags: [u32; 2], - crashed: AtomicBool, - feed: Arc, - #[pin] - feed_work: Work, - preparations: Arc, - #[pin] - work: Work, - #[pin] - poll: DelayedWork, - #[pin] - idle: DelayedWork, - #[pin] - grow: Work, - #[pin] - reclaim_work: Work, - /// Nests inside the device mutex; detached mappings are dropped after both unlock. - #[pin] - reclaim: Mutex, - /// Serializes detached firmware destruction with unbind. Stop paths only; - /// normal publication and RTKit/fence handlers never acquire this lock. - #[pin] - stop: Mutex<()>, - /// The device lock. `None` before boot completes and after detachment. - #[pin] - state: Mutex>>, - #[pin] - changed: CondVar, -} - -impl_has_work! { - impl HasWork for Shared { self.work } -} - -impl_has_delayed_work! { - impl HasDelayedWork for Shared { self.poll } -} -impl_has_delayed_work! { - impl HasDelayedWork for Shared { self.idle } -} -impl_has_work! { - impl HasWork for Shared { self.grow } -} -impl_has_work! { - impl HasWork for Shared { self.reclaim_work } -} -impl WorkItem for Shared { - type Pointer = Arc; - fn run(this: Arc) { - if let Err(error) = this.grow_render() { - if this.state.lock().is_some() { - dev_err!( - this.dev.as_ref(), - "Render parameter-buffer growth failed: {:?}\n", - error - ); - this.lose_device_quietly(); - } - } - this.queue_events(); - } -} -impl WorkItem for Shared { - type Pointer = Arc; - fn run(this: Arc) { - this.drain_reclaims(); - this.queue_events(); - } -} -impl WorkItem for Shared { - type Pointer = Arc; - fn run(this: Arc) { - let result = { - let mut state = this.state.lock(); - match (*state).as_deref_mut() { - Some(firmware) if firmware.render_control_backpressured() => { - firmware.note_render_control_backpressure() - } - _ => Ok(()), - } - }; - // Only an elapsed one-millisecond timer consumes the retry budget. - // Concurrent event IRQs can request a pass without aging the control. - if result.is_err() { - this.lose_device_quietly(); - } else { - this.queue_events(); - } - } -} -impl WorkItem for Shared { - type Pointer = Arc; - fn run(this: Arc) { - this.queue_events(); - } -} - -impl WorkItem for Shared { - type Pointer = Arc; - - fn run(this: Arc) { - this.run_runtime_events(); - } -} - -impl Shared { - fn drm_neo(&self) -> Result { - self.drm_neo.lock().as_ref().cloned().ok_or(ENODEV) - } - - fn release_drm(&self) { - // The final put may call device-release callbacks. Never run those with - // the reference-slot mutex held. - let retired = self.drm_neo.lock().take(); - drop(retired); - } - - /// Queues the event worker. Callable from any context. - fn queue_events(self: &Arc) { - // Already queued if this fails, which is just as good. - let _ = workqueue::system_highpri().enqueue::, EVENT_WORK>(self.clone()); - } - - fn queue_grow(self: &Arc) { - let _ = workqueue::system_unbound().enqueue::, GROW_WORK>(self.clone()); - } - - fn queue_reclaims(self: &Arc) { - let _ = workqueue::system_unbound().enqueue::, RECLAIM_WORK>(self.clone()); - } - - fn drain_reclaims(self: &Arc) { - loop { - let next = self.reclaim.lock().take_one(); - let Some(next) = next else { - break; - }; - next.finish(); - } - } - - fn disable_work(self: &Arc, work: &Work) { - // SAFETY: Callers pass this Shared's initialized pinned work field. - // Disabling joins a running callback and prevents subsequent enqueue. - let cancelled = unsafe { kernel::bindings::disable_work_sync(Work::raw_get(work)) }; - if cancelled { - // SAFETY: Cancelling the pending callback leaves its one enqueue- - // transferred Arc to reclaim; the callback can no longer consume it. - drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); - } - } - - fn queue_poll(self: &Arc) { - let _ = workqueue::system_highpri() - .enqueue_delayed::, POLL_WORK>(self.clone(), msecs_to_jiffies(1)); - } - - fn queue_idle(self: &Arc, delay_ns: u64) { - // Effort::idle bounds this by the 16 ms idle interval. Round up so a - // timer never changes the firmware effort word before that deadline. - let delay_ms = delay_ns.div_ceil(1_000_000).min(u64::from(u32::MAX)) as u32; - let _ = workqueue::system_highpri() - .enqueue_delayed::, IDLE_WORK>(self.clone(), msecs_to_jiffies(delay_ms)); - } - - fn disable_timer(self: &Arc, timer: &DelayedWork) { - // SAFETY: Every caller passes one of this Shared's initialized pinned - // timer fields. raw_as_work points into its delayed_work; the C call - // disables future enqueues, cancels pending work and joins execution. - let cancelled = unsafe { - let work = Work::raw_get(DelayedWork::raw_as_work(timer)); - let delayed = kernel::container_of!(work, kernel::bindings::delayed_work, work); - kernel::bindings::disable_delayed_work_sync(delayed) - }; - if cancelled { - // SAFETY: The cancelled pending callback owned exactly one Arc - // transferred by enqueue_delayed. It cannot run to reclaim it. - drop(unsafe { Arc::from_raw(Arc::as_ptr(self)) }); - } - } - - /// Marks the device lost without a crash log. - fn lose_device_quietly(self: &Arc) { - self.preparations.fail(); - if !self.crashed.swap(true, Ordering::AcqRel) { - dev_err!(self.dev.as_ref(), "GPU runtime stopped, device lost\n"); - } - self.queue_events(); - } - - /// Marks the device lost after a coprocessor crash and publishes a crash dump. - /// - /// The firmware cannot be restarted: outstanding and future work fails. - fn lose_device(self: &Arc, role: Role, crashlog: Option<&[u8]>) { - self.preparations.fail(); - self.crashed.store(true, Ordering::Release); - dev_err!( - self.dev.as_ref(), - "GPU firmware ({:?}) crashed, device lost\n", - role - ); - - #[cfg(CONFIG_DEV_COREDUMP)] - if let Some(crashlog) = crashlog { - if let Err(e) = self.publish_crashlog(crashlog) { - dev_err!(self.dev.as_ref(), "Could not generate crashdump: {:?}\n", e); - } - } - #[cfg(not(CONFIG_DEV_COREDUMP))] - let _ = crashlog; - self.queue_events(); - } - - #[cfg(CONFIG_DEV_COREDUMP)] - fn publish_crashlog(&self, crashlog: &[u8]) -> Result { - use kernel::{devcoredump, time::msecs_to_jiffies}; - - // The dump does not include firmware memory: its kernel mappings are private to it. - let mut crashdump = crate::crashdump::CrashDumpBuilder::new(KVVec::new())?; - crashdump.add_crashlog(crashlog)?; - let crashdump = KBox::new(crashdump.finalize()?, GFP_KERNEL)?; - devcoredump::dev_coredump( - self.dev.as_ref(), - &crate::THIS_MODULE, - crashdump, - GFP_KERNEL, - msecs_to_jiffies(CRASHDUMP_TIMEOUT_MS), - ); - Ok(()) - } -} - -/// How long a crash dump stays available. -#[cfg(CONFIG_DEV_COREDUMP)] -const CRASHDUMP_TIMEOUT_MS: u32 = 60 * 60 * 1000; - -/// A G17 GPU. -pub(crate) struct Gpu { - cfg: &'static Config, - id: regs::G17Id, - max_frequency_khz: u32, - geometry: mmu::UatGeometry, - ids: gpu::SequenceIDs, - shared: Arc, -} - -impl Gpu { - /// Boots the firmware of a G17 GPU. - pub(crate) fn new( - pdev: &platform::Device, - dev: &AsahiDevice, - cfg: &'static Config, - regs: regs::Resources, - ) -> Result> { - let id = regs.get_g17_id()?; - if id.family != cfg.identity.family - || id.variant != cfg.identity.variant - || id.num_dies != cfg.identity.num_dies - { - dev_err!( - dev.as_ref(), - "Unexpected GPU family {:#x} variant {:#x} with {} dies\n", - id.family, - id.variant, - id.num_dies - ); - return Err(ENODEV); - } - - let geometry = mmu::UatGeometry { - ias: cfg.uat_ias, - oas: cfg.uat_oas, - }; - let max_frequency_khz = Self::max_frequency_khz(dev)?; - let preparations = preparation::Gate::new()?; - let reclaim = runtime::teardown::ReclaimBatch::new()?; - let feed = feed::Feed::new()?; - let shared = Arc::pin_init( - pin_init!(Shared { - drm_neo <- new_mutex!(Some(dev.into()), "g17::Shared::drm_neo"), - dev: dev.as_ref().into(), - clusters: id.num_clusters, - descriptor_flags: [id.perf_control, id.perf_map[0]], - crashed: AtomicBool::new(false), - feed, - feed_work <- new_work!("g17::Shared::feed_work"), - preparations, - work <- new_work!("g17::Shared::work"), - poll <- new_delayed_work!("g17::Shared::poll"), - idle <- new_delayed_work!("g17::Shared::idle"), - grow <- new_work!("g17::Shared::grow"), - reclaim_work <- new_work!("g17::Shared::reclaim_work"), - reclaim <- new_mutex!(reclaim, "g17::Shared::reclaim"), - stop <- new_mutex!((), "g17::Shared::stop"), - state <- new_mutex!(None, "g17::Shared::state"), - changed <- new_condvar!("g17::Shared::changed"), - }), - GFP_KERNEL, - )?; - - let gpu = Arc::new( - Gpu { - cfg, - id, - max_frequency_khz, - geometry, - ids: Default::default(), - shared, - }, - GFP_KERNEL, - )?; - let firmware = Self::boot(pdev, dev, cfg, regs, geometry, &gpu.shared)?; - *gpu.shared.state.lock() = Some(firmware); - gpu.shared.queue_events(); - Ok(gpu) - } - - /// Returns the highest advertised operating-point frequency without changing table order. - fn max_frequency_khz(dev: &AsahiDevice) -> Result { - let node = dev.as_ref().of_node().ok_or(EIO)?; - let opps = node - .parse_phandle(c_str!("operating-points-v2"), 0) - .ok_or(EIO)?; - let mut freq_hz: Option = None; - for opp in opps.children() { - let frequency: u64 = opp.get_property(c_str!("opp-hz"))?; - freq_hz = Some(freq_hz.map_or(frequency, |highest| highest.max(frequency))); - } - Ok((freq_hz.ok_or(EINVAL)? / 1000).try_into()?) - } - - /// Starts both coprocessors and initializes their firmware. - fn boot( - pdev: &platform::Device, - dev: &AsahiDevice, - cfg: &'static Config, - regs: regs::Resources, - geometry: mmu::UatGeometry, - shared: &Arc, - ) -> Result> { - let uat = Arc::new( - mmu::Uat::new( - dev, - geometry, - true, - mmu::UatFirmware::PublishedRoots { - secondary_root_offset: cfg.secondary_root_offset, - }, - )?, - GFP_KERNEL, - )?; - - // The firmware only samples the kernel lower root's top-level entries when it starts, so - // everything it uses there must be mapped first. - let pm_metrics = KernelObject::new( - dev, - uat.kernel_lower_vm(), - Placement::At(cfg.pm_metrics.va), - cfg.pm_metrics.size, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let primary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Primary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Primary coprocessor setup failed: {:?}\n", e))?; - let secondary = Coprocessor::new(pdev, dev, cfg, &uat, shared, Role::Secondary) - .inspect_err(|e| dev_err!(dev.as_ref(), "Secondary coprocessor setup failed: {:?}\n", e))?; - let render_global = uat.new_vm(0, geometry.user_range())?; - - let bridge = Bridge::enable(regs)?; - - // Install all session ownership before either CPU starts. Error unwinding then runs - // Firmware::drop, stopping both CPUs before releasing shared objects or translations. - let mut firmware = KBox::try_init( - kernel::try_init!(Firmware { - secondary, - primary, - bridge, - rings: channel::Rings::new(), - recovery: recovery::State::new(), - effort: power::Effort::new(), - registration_power_release_pending: false, - queues <- runtime::Registry::new(shared.clusters, shared.descriptor_flags), - pool: object::Pool::new()?, - render_ids: Arc::new(freelist::RenderIds::new(), GFP_KERNEL)?, - init: initdata::InitData::new(), - _render_global: render_global, - _pm_metrics: Arc::new(pm_metrics, GFP_KERNEL)?, - uat, - }), - GFP_KERNEL, - )?; - let session = &mut *firmware; - session.primary.boot()?; - session.secondary.boot()?; - let root_entries = session.uat.adopt_firmware_root_entries()?; - session.init.build( - dev, - cfg, - &session.uat, - &session._render_global, - &session._pm_metrics, - )?; - session.uat.confirm_firmware_root_entries(root_entries)?; - session.uat.mirror_secondary_root()?; - session.bridge.regs.enable_g17_gating()?; - session.primary.init(session.init.root_va(Role::Primary)?)?; - session - .secondary - .init(session.init.root_va(Role::Secondary)?)?; - session.primary.notify(MSG_POWER_ON_DONE)?; - session.secondary.notify(MSG_POWER_ON_DONE)?; - Self::wait_for_opening(dev, &session.init, &session.primary, &session.secondary)?; - session - .uat - .activate_render_context(&session._render_global)?; - Ok(firmware) - } - - /// Waits for both coprocessors to retire their opening device-control records. - fn wait_for_opening( - dev: &AsahiDevice, - init: &initdata::InitData, - primary: &Coprocessor, - secondary: &Coprocessor, - ) -> Result { - for poll in 0..=OPENING_RETIRE_POLLS { - if primary.state.crashed.load(Ordering::Acquire) - || secondary.state.crashed.load(Ordering::Acquire) - { - return Err(EIO); - } - if init.opening_retired(Role::Primary)? && init.opening_retired(Role::Secondary)? { - return Ok(()); - } - if poll < OPENING_RETIRE_POLLS { - fsleep(Delta::from_millis(OPENING_RETIRE_POLL_MS)); - } - } - dev_err!( - dev.as_ref(), - "Firmware did not retire its opening records\n" - ); - Err(ETIMEDOUT) - } - - /// Stops the firmware and releases everything it used. - pub(crate) fn shutdown(&self) { - self.shared.feed.begin_shutdown(); - self.shared.preparations.remove(); - self.shared.stop_queues(); - self.shared.disable_timer(&self.shared.poll); - self.shared.disable_timer(&self.shared.idle); - self.shared.disable_work(&self.shared.grow); - self.shared.disable_work(&self.shared.reclaim_work); - self.shared.disable_work(&self.shared.work); - self.shared.disable_work(&self.shared.feed_work); - self.shared.feed.shutdown(); - self.shared.drain_reclaims(); - self.shared.release_drm(); - } -} - -impl gpu::Gpu for Gpu { - fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { - self.shared.feed.wait_hint() - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn arc_as_any(self: Arc) -> Arc { - self as Arc - } - - fn manager(&self) -> Option<&dyn gpu::GpuManager> { - None - } - - fn init(&self) -> Result { - Ok(()) - } - - fn ids(&self) -> &gpu::SequenceIDs { - &self.ids - } - - fn is_crashed(&self) -> bool { - self.shared.crashed.load(Ordering::Acquire) - } - - fn get_params(&self, params: &mut uapi::drm_asahi_neo_params_global) -> Result { - params.features |= - uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 - | uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_EXACT_PRIOR_BARRIERS as u64 - | uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_FEW_PRIMITIVES as u64 - | uapi::drm_asahi_neo_feature_DRM_ASAHI_NEO_FEATURE_FRAGMENT_BARRIERS as u64; - params.gpu_generation = hw::GpuGen::G17 as u32; - params.gpu_variant = self.cfg.identity.gpu_variant as u32; - params.gpu_revision = self.id.gpu_rev as u32; - params.chip_id = self.cfg.chip_id; - params.num_dies = self.id.num_dies; - params.num_clusters_total = self.id.num_clusters; - params.num_cores_per_cluster = self.id.num_cores; - params.max_frequency_khz = self.max_frequency_khz; - params.usc_generation = self.cfg.identity.usc_generation; - params.gpu_hal_generation = self.cfg.identity.hal_generation; - - for (i, mask) in self.id.core_masks.iter().enumerate() { - *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); - } - - Ok(()) - } - - fn base_clock_hz(&self) -> u32 { - let frequency: u64; - // SAFETY: CNTFRQ_EL0 is a read-only register containing the architectural counter rate. - unsafe { core::arch::asm!("mrs {frequency}, CNTFRQ_EL0", frequency = out(reg) frequency) }; - frequency as u32 - } - - fn supports_vm_status(&self) -> bool { - true - } - - fn queue_limits(&self) -> Option { - Some(uapi::drm_asahi_neo_queue_limits { - max_queues: mmu::MAX_EXECUTION_CONTEXTS, - max_in_flight_per_queue: hw::t8140::queues::RENDER_PAIRS_PER_OWNER as u32, - ..Default::default() - }) - } - - fn uat_geometry(&self) -> mmu::UatGeometry { - self.geometry - } - - fn new_vm(&self, kernel_range: Range) -> Result { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_mut().ok_or(ENODEV)?; - let firmware = &mut **firmware; - let vm = firmware - .uat - .new_vm(self.ids.vm.next(), kernel_range)? - .with_status()?; - let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; - let ranges = - hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; - let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; - for range in ranges { - reserved.push(range, GFP_KERNEL)?; - } - vm.install_driver_mappings(mappings, reserved)?; - Ok(vm) - } - - fn new_queue( - &self, - vm: mmu::Vm, - _ualloc: Arc>, - _ualloc_priv: Arc>, - priority: u32, - usc_exec_base: u64, - ) -> Result> { - let dev = self.shared.drm_neo()?; - let owner = self.ids.queue.next(); - let frontend = - queue::Frontend::new(priority, usc_exec_base, self.geometry.user_usable_range())?; - let policy = scheduling_policy(priority)?; - let (backend, slot) = loop { - match runtime::Backend::new(&self.shared, &vm, owner, policy) { - Err(error) if error == EAGAIN => continue, - result => break result?, - } - }; - let frontend = match queue::Queue::new( - &dev, - backend.context.clone(), - backend.clone(), - frontend, - slot, - ) { - Ok(frontend) => frontend, - Err(error) => { - job::Backend::release_owner(&*backend); - return Err(error); - } - }; - Ok(KBox::new(frontend, GFP_KERNEL)?) - } - - fn map_timestamp_buffer( - &self, - mut bo: gem::ObjectRef, - range: Range, - ) -> Result { - let state = self.shared.state.lock(); - let firmware = (*state).as_ref().ok_or(ENODEV)?; - let mut addresses = firmware.uat.geometry().kernel_range(); - addresses.start += hw::t8140::dynamic::KERNEL_OFFSET; - bo.map_range_into_range( - firmware.uat.kernel_vm(), - range, - addresses, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - ) - } -} - -/// Scheduling profile of a new logical queue. `file.rs` passes -/// `DRM_ASAHI_NEO_PRIORITY_REALTIME - priority` and refuses HIGH and REALTIME, so -/// MEDIUM arrives as 2 and LOW as 3. -fn scheduling_policy(priority: u32) -> Result { - match priority { - 2 => Ok(hw::t8140::scheduling::MEDIUM), - 3 => Ok(hw::t8140::scheduling::LOW), - _ => Err(EINVAL), - } -} - -impl Drop for Gpu { - fn drop(&mut self) { - self.shutdown(); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/object.rs b/drivers/gpu/drm/asahi-neo/g17/object.rs deleted file mode 100644 index 4076cd91bff163..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/object.rs +++ /dev/null @@ -1,579 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Kernel objects shared with the G17 firmware. -//! -//! Each firmware object is backed by its own GEM object, mapped into a kernel address space -//! either at an address the firmware interface fixes or anywhere in a given range, and kept -//! mapped into the kernel CPU address space for its whole lifetime. The host builds an object's -//! contents through [`KernelObject::bytes_mut`] and [`KernelObject::write`] before it publishes -//! its address; after that, words the firmware accesses concurrently are only touched through -//! the single-access helpers. - -use core::ops::Range; -use core::ptr::NonNull; -use core::sync::atomic::{ - AtomicU32, - AtomicU64, // -}; - -use kernel::{ - drm_neo::gem::shmem, - prelude::*, // -}; - -use crate::driver::AsahiDevice; -use crate::{ - gem, - mmu, // -}; - -/// CPU caching of the kernel mapping of a [`KernelObject`]. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum CpuMap { - /// Write-back, for memory the firmware only reads through a cached mapping or that the host - /// only reads. - WriteBack, - /// Write-combined, for memory the firmware reads uncached as soon as the host published it. - WriteCombined, -} - -/// Where a [`KernelObject`] is mapped. -pub(crate) enum Placement { - /// At this address, which the firmware interface fixes. - At(u64), - /// Anywhere in this range, at this alignment, followed by an unmapped guard page. - In(Range, u64), -} - -/// A zero-initialized kernel object mapped into a GPU address space and the kernel. -/// -/// # Invariants -/// -/// `ptr` points to the start of the kernel mapping of `obj`, which is `size` bytes long and stays -/// valid as long as `obj` exists, since `obj` holds an owned kernel mapping. -pub(crate) struct KernelObject { - // Unmap from the GPU before dropping the backing object. - mapping: Option, - obj: gem::ObjectRef, - ptr: NonNull, - size: usize, -} - -// SAFETY: The object only refers to kernel memory; concurrent access goes through `&self` -// methods that use atomic or volatile accesses. -unsafe impl Send for KernelObject {} -// SAFETY: See `Send`. -unsafe impl Sync for KernelObject {} - -impl KernelObject { - /// Allocates a zeroed object of `size` bytes and maps it into `vm`. - pub(crate) fn new( - dev: &AsahiDevice, - vm: &mmu::Vm, - placement: Placement, - size: usize, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { - let mut this = Self::backing(dev, size, cpu)?; - let mapping = match placement { - Placement::At(addr) => this.obj.map_at(vm, addr, prot, false)?, - Placement::In(range, align) => this.obj.map_into_range(vm, range, align, prot, true)?, - }; - this.mapping = Some(mapping); - Ok(this) - } - - /// Allocates CPU-accessible backing whose GPU mappings will be installed later. - pub(crate) fn backing(dev: &AsahiDevice, size: usize, cpu: CpuMap) -> Result { - let mut obj = match cpu { - CpuMap::WriteBack => gem::new_kernel_object(dev, size)?, - CpuMap::WriteCombined => gem::new_kernel_object_wc(dev, size)?, - }; - let mut vmap = obj.vmap()?; - vmap.memset(0); - let ptr = NonNull::new(vmap.as_mut_ptr()).ok_or(ENOMEM)?; - core::mem::drop(vmap); - - // INVARIANT: `vmap()` created the owned kernel mapping of `obj` that `ptr` points into, - // and `obj` keeps it until it is dropped. - Ok(Self { - mapping: None, - obj, - ptr, - size, - }) - } - - /// Maps this object into another address space at `addr`, as an alias of the same memory. - pub(crate) fn map_alias( - &self, - vm: &mmu::Vm, - addr: u64, - prot: mmu::Prot, - ) -> Result { - gem::ObjectRef::new(self.obj.gem.clone()).map_at(vm, addr, prot, false) - } - - /// Reuses only a complete alias of this exact retained backing. Permanent - /// queue aliases stay owned by the client VM after a physical pair moves; - /// returning to that VM must not replace or duplicate those mappings. - pub(crate) fn prepare_alias( - &self, - vm: &mmu::Vm, - address: u64, - prot: mmu::Prot, - requires_write: bool, - ) -> Result> { - if self.size == 0 || self.size % mmu::UAT_PGSZ != 0 || address & mmu::UAT_PGMSK as u64 != 0 - { - return Err(EINVAL); - } - address.checked_add(self.size as u64).ok_or(EOVERFLOW)?; - let source = self.mapping.as_ref().ok_or(EINVAL)?; - let mut existing = 0; - for offset in (0..self.size).step_by(mmu::UAT_PGSZ) { - let client = address + offset as u64; - if vm.covers_range(client, mmu::UAT_PGSZ as u64, true, requires_write) { - if vm.translate_iova(client)? != source.translate_offset(offset)? { - return Err(EFAULT); - } - existing += 1; - } - } - if existing != 0 { - if existing != self.size / mmu::UAT_PGSZ { - return Err(EFAULT); - } - return Ok(None); - } - self.map_alias(vm, address, prot).map(Some) - } - - /// Maps this backing into a dynamically placed kernel alias. - pub(crate) fn alias_in( - &self, - vm: &mmu::Vm, - range: Range, - align: u64, - prot: mmu::Prot, - ) -> Result { - gem::ObjectRef::new(self.obj.gem.clone()).map_into_range(vm, range, align, prot, false) - } - - /// Maps an exact backing subrange, retaining an explicit unmapped guard. - pub(crate) fn map_range( - &self, - vm: &mmu::Vm, - source: Range, - range: Range, - align: u64, - prot: mmu::Prot, - guard_size: usize, - ) -> Result { - vm.map_in_range_with_guard_size(&self.obj.gem, source, align, range, prot, guard_size) - } - - /// Initializes a typed view before publishing the object to the firmware. - /// - /// The view is cleared first, making every `Zeroable` representation valid. - pub(crate) fn initialize( - &mut self, - offset: usize, - init: impl FnOnce(&mut T) -> Result, - ) -> Result { - let ptr = self.aligned::(offset)?; - // SAFETY: The checked view lies within the exclusively borrowed object. Zeroable makes - // the cleared bytes a valid T, and the closure cannot retain the reference. - unsafe { - ptr.write_bytes(0, 1); - init(&mut *ptr) - } - } - - /// Returns a reference-counted kernel mapping of the object. - pub(crate) fn kernel_map(&self) -> Result> { - self.obj.gem.owned_vmap() - } - - /// Transfers an unpublished object's GPU mapping to a longer-lived owner. - /// The CPU view remains valid; gpu_va returns zero after this transfer. - pub(crate) fn take_mapping(&mut self) -> Result { - self.mapping.take().ok_or(EINVAL) - } - - /// Returns the GPU address of the object. - pub(crate) fn gpu_va(&self) -> u64 { - self.mapping.as_ref().map_or(0, |mapping| mapping.iova()) - } - - /// Returns the size of the object in bytes. - pub(crate) fn size(&self) -> usize { - self.size - } - - /// Returns the contents of the object. - /// - /// Only for building an object the firmware does not access concurrently, before its address - /// is published or while the firmware is known not to use it. - pub(crate) fn bytes_mut(&mut self) -> &mut [u8] { - // SAFETY: Per the type invariant `ptr` is valid for `size` bytes, and the exclusive borrow - // excludes other host accesses. - unsafe { core::slice::from_raw_parts_mut(self.ptr.as_ptr(), self.size) } - } - - /// Writes `value` at `offset`, with the same restrictions as [`Self::bytes_mut`]. - pub(crate) fn write(&mut self, offset: usize, value: T) -> Result { - let end = offset - .checked_add(core::mem::size_of::()) - .ok_or(EINVAL)?; - if end > self.size { - return Err(EINVAL); - } - // SAFETY: The range was checked to be within the object, and the exclusive borrow excludes - // other host accesses. The destination may be unaligned. - unsafe { - self.ptr - .as_ptr() - .add(offset) - .cast::() - .write_unaligned(value) - }; - Ok(()) - } - - /// Returns a pointer to the naturally aligned `T` at `offset`. - fn aligned(&self, offset: usize) -> Result<*mut T> { - let end = offset - .checked_add(core::mem::size_of::()) - .ok_or(EINVAL)?; - if end > self.size { - return Err(EINVAL); - } - // SAFETY: The checked offset lies within the retained CPU mapping. - let address = unsafe { self.ptr.as_ptr().add(offset) }; - if address as usize % core::mem::align_of::() != 0 { - return Err(EINVAL); - } - // The actual address, not only the offset, satisfies T's alignment. - Ok(address.cast()) - } - - /// Returns a checked raw view; dereferencing it requires external serialization. - pub(super) fn pointer(&self, offset: usize, length: usize) -> Result<*mut u8> { - if offset.checked_add(length).ok_or(EINVAL)? > self.size { - return Err(EINVAL); - } - // SAFETY: The offset was checked against the live object mapping. - Ok(unsafe { self.ptr.as_ptr().add(offset) }) - } - - /// Returns the live 32-bit word at `offset`, which must be naturally aligned. - pub(crate) fn word(&self, offset: usize) -> Result<&AtomicU32> { - // SAFETY: `aligned()` returns a valid, aligned pointer into the object, which lives as long - // as `self`. Other agents only access the word with single 32-bit accesses. - Ok(unsafe { AtomicU32::from_ptr(self.aligned(offset)?) }) - } - - /// Returns the live 64-bit word at `offset`, which must be naturally aligned. - pub(crate) fn dword(&self, offset: usize) -> Result<&AtomicU64> { - // SAFETY: As for `word()`, with 64-bit accesses. - Ok(unsafe { AtomicU64::from_ptr(self.aligned(offset)?) }) - } - - /// Copies `N` 32-bit words that start at `offset` out of the object, one volatile access each. - /// - /// Used for records another agent has published, which it no longer modifies. - pub(crate) fn read_words(&self, offset: usize) -> Result<[u32; N]> { - let base = self.aligned::<[u32; N]>(offset)?.cast::(); - let mut words = [0u32; N]; - for (i, word) in words.iter_mut().enumerate() { - // SAFETY: `aligned()` checked that all `N` words are within the object and aligned. - *word = unsafe { base.add(i).read_volatile() }; - } - Ok(words) - } -} - -/// Allocation domains shared by the G17 resource builders. Dynamic objects -/// retain a guard page; aliases keep the existing backing and its exact VA. -pub(crate) struct Allocator<'a> { - pub(crate) dev: &'a AsahiDevice, - pub(crate) uat: &'a mmu::Uat, -} - -impl Allocator<'_> { - /// Allocates in the dynamic part of the firmware's upper address space. - pub(crate) fn kernel( - &self, - size: usize, - align: u64, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { - let mut range = self.uat.geometry().kernel_range(); - range.start += crate::hw::t8140::dynamic::KERNEL_OFFSET; - KernelObject::new( - self.dev, - self.uat.kernel_vm(), - Placement::In(range, align), - size, - prot, - cpu, - ) - } - - /// Allocates a canonical low mapping, which can be aliased at the same - /// address into an execution context without overlapping client objects. - pub(crate) fn lower( - &self, - size: usize, - align: u64, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { - KernelObject::new( - self.dev, - self.uat.kernel_lower_vm(), - Placement::In(crate::hw::t8140::dynamic::LOWER, align), - size, - prot, - cpu, - ) - } -} - -/// Families whose upper mappings remain cached and installed until firmware -/// stops, even after the logical owner releases the object. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum Family { - Descriptor = 0, - WorkState = 1, - QueueState = 2, - QueueConfig = 3, -} - -const POOL_FAMILIES: usize = 4; -const POOL_SLOTS: usize = 64; -const POOL_BYTES: usize = 128 * 1024 * 1024; - -struct Retained { - object: KernelObject, - prefix: usize, - cpu: CpuMap, - poisoned: bool, -} - -struct PoolInner { - returned: KVec>, - reserved: [usize; POOL_FAMILIES], - bytes: usize, - stopped: bool, -} - -/// Per-device cache-safe object recycling. Every converted mapping reserves -/// one return slot before its page-table attributes change. Active leases and -/// poisoned mappings count against the same capacity as reusable objects. -#[pin_data] -pub(crate) struct Pool { - #[pin] - inner: kernel::sync::Mutex, -} - -struct CacheLease { - pool: kernel::sync::Arc, - family: Family, - prefix: usize, - cpu: CpuMap, - poisoned: bool, -} - -/// A firmware object whose cached mapping is returned to its device pool -/// instead of being unmapped while either firmware processor can reference it. -pub(crate) struct PooledObject { - object: core::mem::ManuallyDrop, - lease: Option, -} - -impl core::ops::Deref for PooledObject { - type Target = KernelObject; - fn deref(&self) -> &Self::Target { - &self.object - } -} -impl core::ops::DerefMut for PooledObject { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.object - } -} -impl Drop for PooledObject { - fn drop(&mut self) { - // SAFETY: This destructor is the only operation that moves `object` - // out; ManuallyDrop prevents a second implicit drop of the field. - let object = unsafe { core::mem::ManuallyDrop::take(&mut self.object) }; - match self.lease.take() { - Some(lease) => lease.pool.return_object(object, &lease), - None => drop(object), - } - } -} - -impl Pool { - pub(crate) fn new() -> Result> { - let mut returned = KVec::with_capacity(POOL_FAMILIES * POOL_SLOTS, GFP_KERNEL)?; - for _ in 0..POOL_FAMILIES * POOL_SLOTS { - returned.push(None, GFP_KERNEL)?; - } - kernel::sync::Arc::pin_init( - pin_init!(Self { - inner <- kernel::new_mutex!(PoolInner { returned, reserved: [0; POOL_FAMILIES], bytes: 0, stopped: false }, "g17::object::Pool"), - }), - GFP_KERNEL, - ) - } - - /// Allocates or reuses a matching object. Capacity pressure leaves a fresh - /// mapping uncached, preserving allocation and submission concurrency. - pub(crate) fn allocate( - self: &kernel::sync::Arc, - alloc: &Allocator<'_>, - family: Family, - size: usize, - prefix: usize, - prot: mmu::Prot, - cpu: CpuMap, - ) -> Result { - let family_index = family as usize; - let begin = family_index * POOL_SLOTS; - let reused = { - let mut inner = self.inner.lock(); - if inner.stopped { - return Err(ENODEV); - } - let slot = inner.returned[begin..begin + POOL_SLOTS] - .iter_mut() - .find(|entry| { - entry.as_ref().is_some_and(|entry| { - !entry.poisoned - && entry.object.size() == size - && entry.prefix == prefix - && entry.cpu == cpu - }) - }); - slot.and_then(Option::take) - }; - let lease = || CacheLease { - pool: self.clone(), - family, - prefix, - cpu, - poisoned: false, - }; - if let Some(mut entry) = reused { - entry.object.bytes_mut().fill(0); - core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); - return Ok(PooledObject { - object: core::mem::ManuallyDrop::new(entry.object), - lease: Some(lease()), - }); - } - let object = alloc.kernel(size, mmu::UAT_PGSZ as u64, prot, cpu)?; - let reserved = { - let mut inner = self.inner.lock(); - if inner.stopped { - return Err(ENODEV); - } - if inner.reserved[family_index] == POOL_SLOTS - || inner - .bytes - .checked_add(size) - .is_none_or(|bytes| bytes > POOL_BYTES) - { - false - } else { - inner.reserved[family_index] += 1; - inner.bytes += size; - true - } - }; - let mut result = PooledObject { - object: core::mem::ManuallyDrop::new(object), - lease: reserved.then(lease), - }; - if !reserved { - return Ok(result); - } - let (proof, mutated) = match result.mapping.as_ref() { - // SAFETY: The allocator just created the private mapping. No - // pointer or alias has been published; its reserved lease retains - // it until processor stop if any attribute change was attempted. - Some(mapping) => unsafe { mapping.set_fw_cached_prefix(prefix) }, - None => (Err(EINVAL), false), - }; - match proof { - Ok(_) => Ok(result), - Err(error) if mutated => { - if let Some(lease) = &mut result.lease { - lease.poisoned = true; - } - // Drop retains the unproven mapping but never makes it reusable. - Err(error) - } - Err(_) => { - result.lease = None; - let mut inner = self.inner.lock(); - inner.reserved[family_index] -= 1; - inner.bytes -= size; - Ok(result) - } - } - } - - fn return_object(&self, object: KernelObject, lease: &CacheLease) { - let mut inner = self.inner.lock(); - let family = lease.family as usize; - if inner.stopped { - inner.reserved[family] -= 1; - inner.bytes -= object.size(); - drop(inner); - drop(object); - return; - } - let begin = family * POOL_SLOTS; - for slot in &mut inner.returned[begin..begin + POOL_SLOTS] { - if slot.is_none() { - *slot = Some(Retained { - object, - prefix: lease.prefix, - cpu: lease.cpu, - poisoned: lease.poisoned, - }); - return; - } - } - // A private CacheLease reserves capacity before conversion, so a - // missing slot means host accounting corruption. Never unmap a cached - // object the firmware may still hold in that failure case. - pr_err!("G17 cached-object return capacity was lost\n"); - core::mem::forget(object); - } - - /// Releases retained mappings after both firmware processors have stopped. - /// Active objects returned later observe the same stop witness. - pub(crate) fn stop_after_firmware(&self) { - self.inner.lock().stopped = true; - for index in 0..POOL_FAMILIES * POOL_SLOTS { - let entry = { - let mut inner = self.inner.lock(); - let entry = inner.returned[index].take(); - if let Some(entry) = &entry { - inner.reserved[index / POOL_SLOTS] -= 1; - inner.bytes -= entry.object.size(); - } - entry - }; - // Mapping teardown must not run while the pool mutex is held. - drop(entry); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/power.rs b/drivers/gpu/drm/asahi-neo/g17/power.rs deleted file mode 100644 index 96b363bc27447e..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/power.rs +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Work-based firmware performance effort. Idle power gating is managed separately by the -//! device-control ring; this input never keeps an otherwise idle GPU powered. - -pub(super) const FULL_EFFORT: u32 = 0x10000; -const IDLE_HOLD_NS: u64 = 16_000_000; - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum IdleAction { - None, - Clear, - Recheck(u64), -} - -pub(super) struct Effort { - streaming: bool, - idle_since: Option, - applied: u32, -} - -impl Effort { - pub(super) const fn new() -> Self { - Self { - streaming: false, - idle_since: None, - applied: 0, - } - } - - pub(super) fn submit(&mut self) { - self.streaming = true; - self.idle_since = None; - } - - pub(super) fn applied(&self) -> u32 { - self.applied - } - pub(super) fn did_apply(&mut self, value: u32) { - self.applied = value; - } - - /// Quarantined jobs do not count as work in flight; they retain mappings but cannot justify - /// retaining the effort input. New submissions restart the continuous-idle interval. - pub(super) fn idle(&mut self, now_ns: u64, in_flight: bool) -> IdleAction { - if !self.streaming { - return IdleAction::None; - } - if in_flight { - self.idle_since = None; - return IdleAction::None; - } - let since = *self.idle_since.get_or_insert(now_ns.max(1)); - let idle = now_ns.saturating_sub(since); - if idle >= IDLE_HOLD_NS { - self.streaming = false; - self.idle_since = None; - IdleAction::Clear - } else { - IdleAction::Recheck(IDLE_HOLD_NS - idle) - } - } -} - -/// Single-producer control path under the device mutex. Event draining must not publish another -/// control between `set_idle` and the exact three-counter retirement witness. -pub(super) trait Control { - fn crashed(&self) -> bool; - fn powered(&self) -> bool; - fn counters(&self) -> kernel::error::Result<[u32; 3]>; - fn set_idle(&mut self, enabled: bool) -> kernel::error::Result; - fn wait_tick(&self); - fn registration_release_failed(&mut self) {} - fn registration_release_succeeded(&mut self) {} -} - -fn wait(control: &C, target: u32, require_powered: bool) -> kernel::error::Result { - use kernel::prelude::*; - const RETIRE_POLLS: usize = 20; - const POWER_POLLS: usize = 250; - let mut retired = false; - for elapsed in 0..RETIRE_POLLS + POWER_POLLS { - if control.crashed() { - return Err(EIO); - } - if !retired { - retired = control.counters()? == [target; 3]; - if !retired && elapsed + 1 >= RETIRE_POLLS { - break; - } - } - if retired && (!require_powered || control.powered()) { - return Ok(()); - } - control.wait_tick(); - } - Err(ETIMEDOUT) -} - -/// Disable idle descent through fresh pair registration, including a post-wake reassertion. -pub(super) fn acquire(control: &mut C) -> kernel::error::Result { - use kernel::prelude::*; - let result = (|| { - for _ in 0..3 { - let powered = control.powered(); - let target = control.set_idle(false)?; - wait(control, target, true)?; - if !powered { - let target = control.set_idle(false)?; - wait(control, target, true)?; - } - if control.powered() { - return Ok(()); - } - } - Err(EAGAIN) - })(); - if result.is_err() && release(control).is_err() { - control.registration_release_failed(); - } - result -} - -pub(super) fn release(control: &mut C) -> kernel::error::Result { - let target = match control.set_idle(true) { - Ok(target) => target, - Err(error) => { - control.registration_release_failed(); - return Err(error); - } - }; - let result = wait(control, target, false); - if result.is_ok() { - control.registration_release_succeeded(); - } else { - control.registration_release_failed(); - } - result -} diff --git a/drivers/gpu/drm/asahi-neo/g17/preparation.rs b/drivers/gpu/drm/asahi-neo/g17/preparation.rs deleted file mode 100644 index 8563beda979e66..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/preparation.rs +++ /dev/null @@ -1,186 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Admission and lifetime of off-lock graph construction. -//! -//! Entering and leaving an open epoch use only atomics. Recovery closes the epoch while holding -//! the device mutex, then drops that mutex before waiting for builders. A builder's lease is -//! stored after its tentative mappings so those mappings are destroyed before the join completes. - -use core::sync::atomic::{AtomicU64, Ordering}; -use kernel::{ - new_condvar, new_mutex, - prelude::*, - sync::{Arc, CondVar, Mutex}, -}; - -const COUNT_BITS: u32 = 20; -const COUNT_MASK: u64 = (1 << COUNT_BITS) - 1; -const CLOSED: u64 = 1 << COUNT_BITS; -const REMOVED: u64 = CLOSED << 1; -const FAILED: u64 = CLOSED << 2; -const EPOCH: u64 = CLOSED << 3; - -#[pin_data] -pub(super) struct Gate { - state: AtomicU64, - #[pin] - wait: Mutex<()>, - #[pin] - changed: CondVar, -} - -impl Gate { - pub(super) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - state: AtomicU64::new(EPOCH), - wait <- new_mutex!((), "g17::Gate::wait"), - changed <- new_condvar!("g17::Gate::changed"), - }), - GFP_KERNEL, - ) - } - - pub(super) fn try_enter(self: &Arc) -> Result { - let mut previous = self.state.load(Ordering::Acquire); - loop { - if previous & (REMOVED | FAILED) != 0 { - return Err(ENODEV); - } - if previous & CLOSED != 0 { - return Err(EAGAIN); - } - if previous & COUNT_MASK == COUNT_MASK { - return Err(ENOSPC); - } - match self.state.compare_exchange_weak( - previous, - previous + 1, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - return Ok(Lease { - gate: self.clone(), - epoch: previous & !COUNT_MASK, - }) - } - Err(current) => previous = current, - } - } - } - - /// Wait only for a temporary recovery closure. Called with no device lock or existing lease. - pub(super) fn enter(self: &Arc, interruptible: bool) -> Result { - match self.try_enter() { - Err(error) if error == EAGAIN => {} - result => return result, - } - let mut wait = self.wait.lock(); - loop { - match self.try_enter() { - Err(error) if error == EAGAIN => {} - result => return result, - } - if interruptible { - if self.changed.wait_interruptible(&mut wait) { - return Err(ERESTARTSYS); - } - } else { - self.changed.wait(&mut wait); - } - } - } - - fn invalidate(&self, flags: u64) { - let mut previous = self.state.load(Ordering::Acquire); - loop { - let mut next = previous | CLOSED | flags; - if previous & CLOSED == 0 { - next = next - .checked_add(EPOCH) - .unwrap_or(previous | CLOSED | REMOVED | flags); - } - match self.state.compare_exchange_weak( - previous, - next, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => return, - Err(current) => previous = current, - } - } - } - - pub(super) fn close(&self) { - self.invalidate(0); - } - - pub(super) fn remove(&self) { - self.invalidate(REMOVED); - let _wait = self.wait.lock(); - self.changed.notify_all(); - } - - pub(super) fn fail(&self) { - self.invalidate(FAILED); - let _wait = self.wait.lock(); - self.changed.notify_all(); - } - - /// Join builders with no device, queue or mapping lock held. - pub(super) fn wait_drained(&self) { - let mut wait = self.wait.lock(); - while self.state.load(Ordering::Acquire) & COUNT_MASK != 0 { - self.changed.wait(&mut wait); - } - } - - /// Reopen only an empty, temporary closure. Failed or removed devices remain closed. - pub(super) fn reopen(&self) -> bool { - let _wait = self.wait.lock(); - let mut previous = self.state.load(Ordering::Acquire); - loop { - if previous & (REMOVED | FAILED | COUNT_MASK) != 0 { - return false; - } - match self.state.compare_exchange_weak( - previous, - previous & !CLOSED, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - self.changed.notify_all(); - return true; - } - Err(current) => previous = current, - } - } - } -} - -/// An off-lock builder's admission epoch. Keep after all of its tentative mappings. -pub(super) struct Lease { - gate: Arc, - epoch: u64, -} - -impl Lease { - /// Must be checked under the device mutex immediately before publishing the built graph. - pub(super) fn is_current(&self) -> bool { - self.gate.state.load(Ordering::Acquire) & !COUNT_MASK == self.epoch - } -} - -impl Drop for Lease { - fn drop(&mut self) { - let previous = self.gate.state.fetch_sub(1, Ordering::AcqRel); - if previous & COUNT_MASK == 1 && previous & CLOSED != 0 { - // Pair the final decrement with the joiner's check/sleep lock to avoid a lost wakeup. - let _wait = self.gate.wait.lock(); - self.gate.changed.notify_all(); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/qos.rs b/drivers/gpu/drm/asahi-neo/g17/qos.rs deleted file mode 100644 index 9a100db4cffdfc..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/qos.rs +++ /dev/null @@ -1,249 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Shared QoS publication and exact-completion accounting for every engine. -//! -//! The device publication lock serializes this host state. Publishing identity -//! precedes submitted counters; retiring a final owner clears only its two row -//! bytes. Cancellation balances host references without rewinding shared counts. - -use super::{ - fw::{qos as fw, queue::Policy}, - object::KernelObject, -}; -use crate::hw::t8140::qos as cfg; -use core::{ - mem::{offset_of, size_of}, - sync::atomic::{fence, AtomicU32, AtomicU64, AtomicU8, Ordering}, -}; -use kernel::prelude::*; - -/// Atomic views retain the allocation borrow and never borrow firmware words -/// as ordinary mutable memory. The arrays are disjoint apart from the explicit -/// data-master time/count indexing in `times_and_counts`. -pub(crate) struct View<'a> { - rows: &'a [[AtomicU8; 8]; fw::QUEUES], - queue_submitted: &'a [AtomicU32; fw::QUEUES], - owner_submitted: &'a [AtomicU32; fw::QUEUES], - scheduler: &'a [AtomicU64; fw::QUEUES], - share: &'a [AtomicU32; fw::QUEUES], - intervals: &'a AtomicU64, - times_and_counts: &'a [AtomicU64; 7], -} -impl<'a> View<'a> { - pub(crate) fn new(object: &'a KernelObject) -> Result { - let ptr = object.pointer(0, size_of::())?; - if ptr as usize & 7 != 0 { - return Err(EINVAL); - } - // SAFETY: pointer spans the complete layout, whose asserted offsets - // and size bound every array below. Integer atomics have their integer - // representation and alignment. The retained object borrow keeps every - // shared reference alive; accesses use each field's fixed width. - Ok(unsafe { - Self { - rows: &*ptr.add(offset_of!(fw::Qos, rows)).cast(), - queue_submitted: &*ptr.add(offset_of!(fw::Qos, queue_submitted)).cast(), - owner_submitted: &*ptr.add(offset_of!(fw::Qos, owner_submitted)).cast(), - scheduler: &*ptr.add(offset_of!(fw::Qos, scheduler)).cast(), - share: &*ptr.add(offset_of!(fw::Qos, share)).cast(), - intervals: &*ptr.add(offset_of!(fw::Qos, intervals)).cast(), - times_and_counts: &*ptr.add(offset_of!(fw::Qos, busy_start)).cast(), - } - }) - } - fn row(&self, qid: usize) -> [u8; 8] { - core::array::from_fn(|index| self.rows[qid][index].load(Ordering::Relaxed)) - } -} - -#[derive(Clone, Copy, Debug)] -pub(crate) struct Owner { - pub(crate) qid: u8, - pub(crate) qos: u8, - pub(crate) data_master: u8, -} -impl Owner { - fn indices(self) -> Result<(usize, usize, usize)> { - let (qid, qos, dm) = ( - usize::from(self.qid), - usize::from(self.qos), - usize::from(self.data_master), - ); - if qid >= fw::QUEUES || qos >= fw::QUEUES || dm >= fw::DATA_MASTERS { - return Err(EINVAL); - } - Ok((qid, qos, dm)) - } -} - -/// Retained until completion, or cancellation before the producer is exposed. -#[derive(Clone, Copy, Debug)] -pub(crate) struct Publication { - pub(crate) record: [u8; 8], - owner: Owner, - dm_submitted: u64, -} - -pub(crate) struct Accounting { - in_flight: [u32; fw::QUEUES], - dm_in_flight: [u64; fw::DATA_MASTERS], - dm_by_qid: [u8; fw::QUEUES], -} -impl Accounting { - pub(crate) const fn new() -> Self { - Self { - in_flight: [0; fw::QUEUES], - dm_in_flight: [0; fw::DATA_MASTERS], - dm_by_qid: [0; fw::QUEUES], - } - } - - /// Called under the device publication lock immediately before exposing - /// the work record. The clock is sampled only when its engine becomes busy. - /// The row's class and the owner's share come from the owner's `policy`. - pub(crate) fn publish( - &mut self, - view: &View<'_>, - owner: Owner, - scheduler: u64, - policy: Policy, - clock: impl FnOnce() -> (u64, u64), - ) -> Result { - let (qid, qos, dm) = owner.indices()?; - if scheduler == 0 { - return Err(EINVAL); - } - let next_qid = self.in_flight[qid].checked_add(1).ok_or(EOVERFLOW)?; - let next_dm = self.dm_in_flight[dm].checked_add(1).ok_or(EOVERFLOW)?; - let starts_busy = self.dm_in_flight[dm] == 0; - let time = if starts_busy { - let (counter, frequency) = clock(); - time_from_host(counter, frequency)? - } else { - 0 - }; - if self.in_flight[qid] != 0 - && (view.rows[qid][0].load(Ordering::Relaxed) != owner.qos - || self.dm_by_qid[qid] != owner.data_master) - { - return Err(EBUSY); - } - view.rows[qid][0].store(owner.qos, Ordering::Relaxed); - view.rows[qid][1].store(policy.qos_class(), Ordering::Relaxed); - let record = view.row(qid); - view.scheduler[qos].store(scheduler, Ordering::Relaxed); - view.share[qos].store(policy.qos_share(), Ordering::Relaxed); - if self.in_flight[qid] == 0 { - fence(Ordering::SeqCst); - } - view.queue_submitted[qid].store( - view.queue_submitted[qid] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - view.owner_submitted[qos].store( - view.owner_submitted[qos] - .load(Ordering::Relaxed) - .wrapping_add(1), - Ordering::Relaxed, - ); - if starts_busy { - view.times_and_counts[dm].store(time, Ordering::Relaxed); - } - let submitted = &view.times_and_counts[3 + dm]; - let dm_submitted = submitted.load(Ordering::Relaxed).wrapping_add(1); - submitted.store(dm_submitted, Ordering::Relaxed); - if starts_busy { - view.intervals.store( - view.intervals.load(Ordering::Relaxed).wrapping_add(1), - Ordering::Relaxed, - ); - } - self.in_flight[qid] = next_qid; - self.dm_in_flight[dm] = next_dm; - self.dm_by_qid[qid] = owner.data_master; - fence(Ordering::SeqCst); - Ok(Publication { - record, - owner, - dm_submitted, - }) - } - - /// A possibly exposed producer must retain its reference until completion. - /// A newer submission on the same engine makes an old ticket uncancellable. - pub(crate) fn cancel(&mut self, view: &View<'_>, ticket: Publication) -> Result { - let (qid, _, dm) = ticket.owner.indices()?; - if self.in_flight[qid] == 0 - || self.dm_in_flight[dm] == 0 - || view.row(qid) != ticket.record - || view.times_and_counts[3 + dm].load(Ordering::Relaxed) != ticket.dm_submitted - { - return Err(EIO); - } - self.complete(view, &[ticket.owner]) - } - - /// Preflights the entire set before clearing any final row. QID membership, - /// scheduler mappings and monotonically increasing counts remain intact. - pub(crate) fn complete(&mut self, view: &View<'_>, owners: &[Owner]) -> Result { - let mut completed = [0u64; fw::DATA_MASTERS]; - for (index, owner) in owners.iter().copied().enumerate() { - let (qid, _, dm) = owner.indices()?; - if owners[..index].iter().any(|other| other.qid == owner.qid) { - return Err(EINVAL); - } - if self.in_flight[qid] == 0 - || view.rows[qid][0].load(Ordering::Relaxed) != owner.qos - || self.dm_by_qid[qid] != owner.data_master - { - return Err(EIO); - } - completed[dm] += 1; - } - for (dm, count) in completed.iter().enumerate() { - if *count > self.dm_in_flight[dm] { - return Err(EIO); - } - } - for owner in owners { - let qid = usize::from(owner.qid); - if self.in_flight[qid] == 1 { - view.rows[qid][0].store(0xff, Ordering::Relaxed); - view.rows[qid][1].store(0, Ordering::Relaxed); - } - } - for owner in owners { - self.in_flight[usize::from(owner.qid)] -= 1; - } - for (dm, count) in completed.iter().enumerate() { - self.dm_in_flight[dm] -= count; - } - fence(Ordering::SeqCst); - Ok(()) - } -} - -/// Firmware accounting uses a 24 MHz domain independent of host timer speed. -pub(crate) fn time_from_host(counter: u64, frequency: u64) -> Result { - if frequency == 0 || frequency > u64::from(u32::MAX) { - return Err(EINVAL); - } - let whole = (counter / frequency) - .checked_mul(cfg::CLOCK_HZ) - .ok_or(EOVERFLOW)?; - let fraction = (counter % frequency) * cfg::CLOCK_HZ / frequency; - whole.checked_add(fraction).ok_or(EOVERFLOW) -} - -pub(crate) fn clock() -> (u64, u64) { - let counter: u64; - let frequency: u64; - // SAFETY: Both timer registers are readable at EL1 and have no memory effects. - unsafe { - core::arch::asm!("mrs {counter}, CNTPCT_EL0", "mrs {frequency}, CNTFRQ_EL0", - counter=out(reg)counter, frequency=out(reg)frequency, options(nomem,nostack,preserves_flags)); - } - (counter, frequency) -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/admission.rs b/drivers/gpu/drm/asahi-neo/g17/queue/admission.rs deleted file mode 100644 index 1c0d6822c395f6..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/admission.rs +++ /dev/null @@ -1,541 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Logical queue and physical render-pair admission. A retained pair belongs -//! to one logical owner even while idle. Closing that owner makes it reusable -//! only after every active lease retires. Quarantine retains these leases until -//! a firmware drain witness or both processors stopping makes reuse safe. -//! -//! The pool lock protects host accounting only and is never held while building -//! a graph or waiting for GPU work. Capacity waiters sleep on the release event. - -use crate::g17::{ - hazard::{self, Footprint, Plan}, - job::Packet, -}; -use crate::{hw::t8140::queues as cfg, mmu::MAX_EXECUTION_CONTEXTS}; -use core::sync::atomic::{AtomicU32, Ordering}; -use kernel::{ - new_condvar, new_mutex, - prelude::*, - sync::{Arc, CondVar, CondVarTimeoutResult, Mutex}, -}; - -#[derive(Clone, Copy)] -pub(crate) struct Request<'a> { - footprint: &'a Arc, - sequence: u64, - prefix: Option, -} - -impl<'a> Request<'a> { - pub(crate) fn of(packet: &'a Packet) -> Option { - packet.footprint.as_ref().map(|footprint| Self { - footprint, - sequence: packet.order.sequence, - prefix: packet.order.wait_through[0], - }) - } -} - -struct InFlight { - owner: u64, - sequence: u64, - slot: Option, - ticket: u64, - footprint: Arc, -} - -const QUARANTINE_WAIT_MS: u32 = 1000; - -struct State { - active: u64, - refs: [u8; cfg::RENDER_SLOTS], - owners: [u64; cfg::RENDER_SLOTS], - epochs: [u64; cfg::RENDER_SLOTS], - quarantined: u64, - kill_hold: u64, - released: u64, - epoch: u64, - footprints: KVec, - next_ticket: u64, -} -impl State { - const MASK: u64 = (u64::MAX >> (64 - cfg::RENDER_SLOTS)) & !1; - const fn new() -> Self { - Self { - active: 0, - refs: [0; cfg::RENDER_SLOTS], - owners: [0; cfg::RENDER_SLOTS], - epochs: [0; cfg::RENDER_SLOTS], - quarantined: 0, - kill_hold: 0, - released: 0, - epoch: 0, - footprints: KVec::new(), - next_ticket: 1, - } - } - fn exhausted(&self) -> bool { - self.quarantined & Self::MASK == Self::MASK - } - fn refusal(&self, owner: u64) -> Result { - if self.exhausted() { - return Err(EIO); - } - match self.owners.iter().position(|id| *id == owner) { - Some(slot) if self.quarantined & (1u64 << slot) != 0 => Err(EIO), - None if !self.owners[1..].contains(&0) - && self.released - & !self.active - & !self.quarantined - & !self.kill_hold - & Self::MASK - == 0 => - { - Err(ENOSPC) - } - _ => Ok(()), - } - } - fn advance(&mut self, slot: usize) { - self.epoch = self.epoch.wrapping_add(1); - self.epochs[slot] = self.epoch; - } - fn take_idle(&mut self, slot: usize, owner: u64) -> u8 { - self.refs[slot] += 1; - self.active |= 1u64 << slot; - self.released &= !(1u64 << slot); - self.owners[slot] = owner; - self.advance(slot); - slot as u8 - } - fn acquire(&mut self, owner: u64) -> Option { - if owner == 0 { - return None; - } - // Prefer the owner's idle pair. A missing sibling can be claimed - // only while this owner has no live kick, before constructing it. - for slot in 0..cfg::RENDER_SLOTS { - if self.owners[slot] == owner - && self.refs[slot] == 0 - && self.quarantined & (1u64 << slot) == 0 - { - if !(0..cfg::RENDER_SLOTS) - .any(|other| self.owners[other] == owner && self.refs[other] != 0) - { - self.claim_sibling(owner); - } - return Some(self.take_idle(slot, owner)); - } - } - if self.owners.contains(&owner) { - // Consecutive kicks are ordered by firmware dependencies. Stack - // onto the least-loaded retained pair without a retirement wait. - let slot = (0..cfg::RENDER_SLOTS) - .filter(|slot| { - self.owners[*slot] == owner - && self.quarantined & (1u64 << slot) == 0 - && self.active & (1u64 << slot) != 0 - && self.refs[*slot] < cfg::RENDER_DEPTH - }) - .min_by_key(|slot| (self.refs[*slot], u64::MAX - self.epochs[*slot]))?; - self.refs[slot] += 1; - self.advance(slot); - return Some(slot as u8); - } - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; - let eligible = |slot: usize| free & (1u64 << slot) != 0; - // Reuse a closed, exactly retired owner before consuming a new graph. - let slot = (0..cfg::RENDER_SLOTS) - .filter(|slot| eligible(*slot) && self.released & (1u64 << slot) != 0) - .min_by_key(|slot| self.epochs[*slot]) - .or_else(|| { - (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) - })?; - let first = self.take_idle(slot, owner); - self.claim_sibling(owner); - Some(first) - } - fn claim_sibling(&mut self, owner: u64) { - if self.owners.iter().filter(|id| **id == owner).count() >= cfg::RENDER_PAIRS_PER_OWNER { - return; - } - let free = !self.active & Self::MASK & !self.quarantined & !self.kill_hold; - let eligible = |slot: usize| free & (1u64 << slot) != 0; - let sibling = (0..cfg::RENDER_SLOTS) - .filter(|slot| { - eligible(*slot) - && self.owners[*slot] != owner - && self.released & (1u64 << slot) != 0 - }) - .min_by_key(|slot| self.epochs[*slot]) - .or_else(|| { - (0..cfg::RENDER_SLOTS).find(|slot| eligible(*slot) && self.owners[*slot] == 0) - }); - if let Some(slot) = sibling { - self.released &= !(1u64 << slot); - self.owners[slot] = owner; - self.advance(slot); - } - } - fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - if self.refs.get(usize::from(publishing)) != Some(&1) { - return None; - } - (0..cfg::RENDER_SLOTS) - .find(|slot| { - *slot != usize::from(publishing) - && self.owners[*slot] == owner - && self.refs[*slot] == 0 - && self.quarantined & (1u64 << slot) == 0 - }) - .map(|slot| slot as u8) - } - fn plan(&self, owner: u64, request: Option>, ticket: Option) -> Plan { - let Some(request) = request else { - return Plan::Any; - }; - hazard::plan( - owner, - request.prefix, - ticket, - request.footprint, - self.footprints.iter().map(|entry| hazard::Live { - owner: entry.owner, - sequence: entry.sequence, - slot: entry.slot, - ticket: entry.ticket, - footprint: &entry.footprint, - quarantined: entry - .slot - .is_some_and(|slot| self.quarantined & (1u64 << slot) != 0), - }), - ) - } - - fn pick(&mut self, owner: u64, plan: Plan) -> Option { - match plan { - Plan::Any => self.acquire(owner), - Plan::Wait => None, - Plan::Pair(slot) => { - let index = usize::from(slot); - if self.owners.get(index) != Some(&owner) - || (self.quarantined | self.kill_hold) & (1u64 << slot) != 0 - || self.refs[index] >= cfg::RENDER_DEPTH - { - return None; - } - if self.refs[index] == 0 { - Some(self.take_idle(index, owner)) - } else { - self.refs[index] += 1; - self.advance(index); - Some(slot) - } - } - } - } - - fn record( - &mut self, - owner: u64, - slot: Option, - ticket: u64, - request: Request<'_>, - ) -> Option<(u64, u64)> { - self.footprints - .push( - InFlight { - owner, - sequence: request.sequence, - slot, - ticket, - footprint: request.footprint.clone(), - }, - GFP_KERNEL, - ) - .ok()?; - Some((owner, request.sequence)) - } - - fn forget_waiter(&mut self, ticket: Option) { - if let Some(ticket) = ticket { - self.footprints - .retain(|entry| entry.slot.is_some() || entry.ticket != ticket); - } - } - - fn release(&mut self, slot: u8) { - let slot = usize::from(slot); - if self.refs[slot] != 0 { - self.refs[slot] -= 1; - } - if self.refs[slot] == 0 { - self.active &= !(1u64 << slot); - } - } -} - -#[pin_data] -pub(crate) struct Pool { - #[pin] - state: Mutex, - #[pin] - released: CondVar, - queues: AtomicU32, -} -impl Pool { - pub(crate) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - state <- new_mutex!(State::new(), "G17 render admission"), - released <- new_condvar!("G17 render pair release"), - queues: AtomicU32::new(0), - }), - GFP_KERNEL, - ) - } - pub(crate) fn reserve_queue(self: &Arc) -> Result { - let mut count = self.queues.load(Ordering::Acquire); - loop { - if count >= MAX_EXECUTION_CONTEXTS { - return Err(ENOSPC); - } - match self.queues.compare_exchange( - count, - count + 1, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => return Ok(QueueSlot { pool: self.clone() }), - Err(current) => count = current, - } - } - } - pub(crate) fn acquire_render( - self: &Arc, - owner: u64, - request: Option>, - ) -> Result { - if owner == 0 { - return Err(EINVAL); - } - let mut state = self.state.lock(); - let mut conflict_budget = kernel::time::msecs_to_jiffies(hazard::WAIT_MS); - let mut quarantine_budget = kernel::time::msecs_to_jiffies(QUARANTINE_WAIT_MS); - let mut expired = false; - let mut ticket = None; - let result = (|| loop { - if state.exhausted() { - return Err(EIO); - } - // The first retained pair's quarantine is not bypassed through its - // healthy sibling. Reclamation wakes this bounded admission wait. - if state - .owners - .iter() - .position(|id| *id == owner) - .is_some_and(|slot| state.quarantined & (1u64 << slot) != 0) - { - if quarantine_budget == 0 { - return Err(EIO); - } - quarantine_budget = match self - .released - .wait_interruptible_timeout(&mut state, quarantine_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; - continue; - } - state.refusal(owner)?; - let plan = if expired { - Plan::Any - } else { - state.plan(owner, request, ticket) - }; - if let Some(slot) = state.pick(owner, plan) { - state.forget_waiter(ticket.take()); - let record = - request.and_then(|request| state.record(owner, Some(slot), 0, request)); - return Ok(RenderLease { - pool: self.clone(), - slot, - record, - }); - } - if plan != Plan::Any { - if ticket.is_none() { - if let Some(request) = request { - let next = state.next_ticket; - state.next_ticket = next.wrapping_add(1); - if state.record(owner, None, next, request).is_some() { - ticket = Some(next); - } - } - } - match self - .released - .wait_interruptible_timeout(&mut state, conflict_budget) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => { - conflict_budget = 0; - expired = true; - } - CondVarTimeoutResult::Woken { jiffies } => conflict_budget = jiffies, - } - } else if self.released.wait_interruptible(&mut state) { - return Err(ERESTARTSYS); - } - })(); - state.forget_waiter(ticket); - result - } - - /// Early publication never waits for overlap; the scheduler retries with - /// the same footprint after its dependencies have become ready. - pub(crate) fn try_acquire_render( - self: &Arc, - owner: u64, - request: Option>, - ) -> Result> { - if owner == 0 { - return Ok(None); - } - let mut state = self.state.lock(); - if state.refusal(owner).is_err() || !state.owners.contains(&owner) { - return Ok(None); - } - let plan = state.plan(owner, request, None); - let Some(slot) = state.pick(owner, plan) else { - return Ok(None); - }; - let record = request.and_then(|request| state.record(owner, Some(slot), 0, request)); - Ok(Some(RenderLease { - pool: self.clone(), - slot, - record, - })) - } - pub(crate) fn warmable_sibling(&self, owner: u64, publishing: u8) -> Option { - self.state.lock().warmable_sibling(owner, publishing) - } - pub(crate) fn release_owner(&self, owner: u64) { - let mut state = self.state.lock(); - for slot in 0..cfg::RENDER_SLOTS { - if state.owners[slot] == owner { - state.released |= 1u64 << slot; - } - } - drop(state); - self.released.notify_all(); - } - /// Active leases survive this transition. Their later drop is still the - /// event which releases capacity, so a stopped owner is never double-freed. - pub(crate) fn release_retained_owners_after_processor_stop(&self) { - let mut state = self.state.lock(); - let released = state.released & !state.active; - for slot in 1..cfg::RENDER_SLOTS { - if released & (1u64 << slot) != 0 { - state.owners[slot] = 0; - state.epochs[slot] = 0; - } - } - state.released &= !released; - state.quarantined &= !released; - state.kill_hold = 0; - drop(state); - self.released.notify_all(); - } -} - -/// Counted once per userspace logical queue, independently of installed graphs. -pub(crate) struct QueueSlot { - pool: Arc, -} -impl Drop for QueueSlot { - fn drop(&mut self) { - self.pool.queues.fetch_sub(1, Ordering::AcqRel); - } -} - -pub(crate) struct RenderLease { - pool: Arc, - slot: u8, - record: Option<(u64, u64)>, -} -impl RenderLease { - pub(crate) fn slot(&self) -> u8 { - self.slot - } - pub(crate) fn quarantine(&self) { - self.pool.state.lock().quarantined |= 1u64 << self.slot; - self.pool.released.notify_all(); - } - /// The VM pin must be retired first. Keep this lease until after clearing - /// quarantine; its drop is what permits waiting publications to proceed. - pub(crate) fn rehabilitate_after_processor_stop(&self) { - self.pool.state.lock().quarantined &= !(1u64 << self.slot); - } - /// Requires the runtime's closed-recovery, exact pair-drain witness. - pub(crate) fn rehabilitate_after_firmware_drain(&self) { - self.rehabilitate_after_processor_stop(); - } -} -impl Drop for RenderLease { - fn drop(&mut self) { - let mut state = self.pool.state.lock(); - if let Some((owner, sequence)) = self.record { - if let Some(index) = state.footprints.iter().position(|entry| { - entry.slot == Some(self.slot) && entry.owner == owner && entry.sequence == sequence - }) { - state.footprints.swap_remove(index); - } - } - state.release(self.slot); - drop(state); - self.pool.released.notify_all(); - } -} - -impl State { - fn hold_for_exit_kill(&mut self, owner: u64, slots: u64) -> bool { - if owner == 0 || slots & !Self::MASK != 0 { - return false; - } - for slot in 0..cfg::RENDER_SLOTS { - let bit = 1u64 << slot; - if slots & bit == 0 { - continue; - } - if self.owners[slot] != owner - || self.refs[slot] != 0 - || self.active & bit != 0 - || self.released & bit == 0 - { - return false; - } - } - self.kill_hold |= slots; - true - } -} -impl Pool { - /// Hold the departed owner's idle pairs before publishing its exit record. - /// Successful publication retains these bits until both processors stop. - pub(crate) fn hold_render_slots_for_exit_kill(&self, owner: u64, slots: u64) -> bool { - self.state.lock().hold_for_exit_kill(owner, slots) - } - - /// Undo only a hold whose control record was not published. - pub(crate) fn release_exit_kill_hold(&self, slots: u64) { - if slots == 0 { - return; - } - self.state.lock().kill_hold &= !slots; - self.released.notify_all(); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/compute.rs b/drivers/gpu/drm/asahi-neo/g17/queue/compute.rs deleted file mode 100644 index 8eb31de5b98b64..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/compute.rs +++ /dev/null @@ -1,1923 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Retained physical compute queues. Logical ownership and command retirement never free an -//! installed graph; the device retains it until both firmware processors have stopped. - -pub(crate) mod exit; -pub(crate) mod retirement; - -use crate::g17::{ - channel::Rings, - context::Context, - fw::{channels::ControlRecord, compute::COMPUTE_DESCRIPTOR_SIZE, queue::*}, - object::{Allocator, CpuMap, KernelObject}, -}; -use crate::{hw::t8140::dynamic, mmu}; -use core::sync::atomic::{fence, Ordering}; -use kernel::{prelude::*, sync::Arc}; -use retirement::{Cursors, RECORD_SLOTS}; - -use crate::g17::now_ns; - -/// Descriptor slots per queue. A kick's descriptor is rewritten when the kick 32 slots later -/// is published; with IN_FLIGHT kicks outstanding at most, that kick has completed by then. -const DESCRIPTORS: usize = 32; -const GRAPH_SIZE: usize = 0x8000; -const CONTEXT_SIZE: usize = 0x20000; -const RECORD_STRIDE: usize = 0x1000; -const CONFIG: usize = 0; -const ANNOUNCE: usize = 0x800; -const PREDECESSOR: usize = 0x400; -const POINTERS: usize = 0x100; -const ITEMS: usize = 0x4000; -const STAMP: usize = 0x300; - -#[repr(C)] -struct GraphLayout { - queue: QueueRecord, - unk_c0: [u8; 0x40], - pointers: PointerBlock, - unk_180: [u8; 0x80], - jobs: JobListHead, - unk_210: [u8; 0x3df0], - items: [u64; ITEM_RING_ENTRIES as usize], - unk_6800: [u8; 0x1800], -} -// SAFETY: The layout contains only integer storage and integer atomics, all valid when zeroed. -unsafe impl Zeroable for GraphLayout {} -static_assert!(core::mem::size_of::() == GRAPH_SIZE); -static_assert!(core::mem::offset_of!(GraphLayout, pointers) == POINTERS); -static_assert!(core::mem::offset_of!(GraphLayout, items) == ITEMS); - -/// Aliases are destroyed before the logical root they name. Superseded bindings remain owned -/// until the first exact successful publication under their replacement has retired. -/// Backing of a vacated pool and descriptor ring, freed outside the device -/// mutex. Field order drops each client alias before the storage it names. -pub(crate) struct Vacated { - _alias: Option, - _pages: Option, - _descriptors: Option, - _ring: Option, -} - -pub(crate) struct Binding { - _kick_alias: mmu::KernelMapping, - /// Client alias of the USC pool pages; dropped when the pool is vacated and - /// recreated when an owner binds a re-armed pool (it pins the backing). - pool_alias: Option, - /// Client alias of the descriptor ring; dropped with the ring when the queue - /// is vacated and recreated when an owner binds a re-armed queue. - descriptors: Option, - preempt: mmu::KernelMapping, - usage: mmu::KernelMapping, - operand: mmu::KernelMapping, - _queue_context: mmu::KernelMapping, - context: Arc, -} - -/// Descriptor ring of one compute QID with its kernel-VM alias. Built without -/// the device mutex; installed into a vacant graph before an owner binds. -pub(crate) struct DescriptorRing { - object: KernelObject, - high: mmu::KernelMapping, -} - -impl DescriptorRing { - pub(crate) fn new(alloc: &Allocator<'_>) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let object = alloc.lower( - DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - page, - prot, - CpuMap::WriteCombined, - )?; - let mut high = alloc.uat.geometry().kernel_range(); - high.start += dynamic::KERNEL_OFFSET; - let high = object.alias_in(alloc.uat.kernel_vm(), high, page, prot)?; - Ok(Self { object, high }) - } -} - -/// Firmware-shared storage of one compute QID, separate from its common kick ring. -/// `ring` is `None` while the queue is vacant (idle, no owner, pool given back). -struct Graph { - ring: Option, - preempt: KernelObject, - usage: KernelObject, - operand: KernelObject, - support: KernelObject, - graph: KernelObject, - context_low: KernelObject, - context_high: KernelObject, - records: KernelObject, - record_ends: [Option; RECORD_SLOTS], - reclaimed: u32, -} - -impl Graph { - fn new(alloc: &Allocator<'_>, context: &Context) -> Result { - let page = mmu::UAT_PGSZ as u64; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let cpu = CpuMap::WriteCombined; - let ring = DescriptorRing::new(alloc)?; - let preempt = alloc.kernel(0x4000, page, prot, cpu)?; - let usage = alloc.kernel(mmu::UAT_PGSZ, page, prot, cpu)?; - let operand = alloc.lower(0x14000, page, prot, cpu)?; - let mut support = alloc.kernel(0xc000, page, prot, cpu)?; - let support_va = support.gpu_va(); - support.write(0, support_va + page)?; - support.write(0x100, support_va + page + 4)?; - support.write(0x110, 0x50u32)?; - support.write(mmu::UAT_PGSZ + 4, 1u32)?; - let mut graph = alloc.kernel(GRAPH_SIZE, page, prot, cpu)?; - let graph_va = graph.gpu_va(); - graph.initialize::(0, |view| { - view.queue = QueueRecord::new(&QueueRecordArgs { - pointers_va: graph_va + POINTERS as u64, - items_va: graph_va + ITEMS as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }); - view.pointers = PointerBlock::new(); - view.jobs = JobListHead::new(graph_va + 0x200); - Ok(()) - })?; - let context_low = alloc.lower(CONTEXT_SIZE, page, prot, cpu)?; - let context_high = alloc.kernel(CONTEXT_SIZE, page, prot, cpu)?; - let records = alloc.kernel(RECORD_SLOTS * RECORD_STRIDE, page, prot, cpu)?; - Ok(Self { - ring: Some(ring), - preempt, - usage, - operand, - support, - graph, - context_low, - context_high, - records, - record_ends: [None; RECORD_SLOTS], - reclaimed: 0, - }) - } - - /// All fallible mapping work finishes before replacing a physical queue's logical owner. - fn bind( - &mut self, - context: Arc, - kick_alias: mmu::KernelMapping, - pool_alias: mmu::KernelMapping, - ) -> Result { - if !context.is_current() { - return Err(EFAULT); - } - let vm = context.vm(); - vm.install_job_context_aliases()?; - let descriptors = self.descriptor_alias(vm)?; - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let page = mmu::UAT_PGSZ as u64; - let preempt = self - .preempt - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let usage = self.usage.alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let operand = self - .operand - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - let queue_context = self - .context_low - .alias_in(vm, dynamic::CLIENT_LOWER, page, prot)?; - Ok(Binding { - _kick_alias: kick_alias, - pool_alias: Some(pool_alias), - descriptors: Some(descriptors), - preempt, - usage, - operand, - _queue_context: queue_context, - context, - }) - } - - /// Caller has excluded publication and proved the retained graph idle. - fn set_owner(&mut self, context: &Context) -> Result { - let pointer = self - .graph - .pointer(0, core::mem::size_of::())? - .cast::(); - // SAFETY: The checked record is in the retained mapping, has byte alignment and consists - // only of integer storage. The device mutex and idle witness exclude simultaneous access. - unsafe { - (*pointer).set_compute_owner( - context.work_head_va(), - context.scheduler_va(), - context.owner_pid(), - context.policy(), - ) - }; - Ok(()) - } - - fn cursors(&self) -> Result { - Ok(Cursors { - producer: self.graph.word(POINTERS + 0x40)?.load(Ordering::Relaxed), - consumer: self.graph.word(POINTERS)?.load(Ordering::Relaxed), - count: self.graph.word(POINTERS + 0x60)?.load(Ordering::Relaxed), - }) - } - fn stamp(&self) -> Result { - fence(Ordering::Acquire); - Ok(self.support.word(STAMP)?.load(Ordering::Relaxed)) - } - /// EAGAIN while the queue is vacant: the owner must re-arm it first. - fn descriptor_vas(&self, slot: u8) -> Result<(u64, u64)> { - let ring = self.ring.as_ref().ok_or(EAGAIN)?; - let offset = (usize::from(slot) % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE) as u64; - Ok((ring.high.iova() + offset, ring.object.gpu_va() + offset)) - } - fn descriptor_alias(&self, vm: &mmu::Vm) -> Result { - let ring = self.ring.as_ref().ok_or(EAGAIN)?; - ring.object - .map_alias(vm, ring.object.gpu_va(), mmu::PROT_GPU_SHARED_RO) - } - /// Detaches the descriptor ring of an idle vacant queue; the caller frees it - /// off-lock. The job list and the context item only name consumed - /// descriptors at this point. - fn vacate_ring(&mut self) -> Option { - self.ring.take() - } - fn rearm_ring(&mut self, ring: DescriptorRing) -> Result { - if self.ring.is_some() { - return Err(EBUSY); - } - self.ring = Some(ring); - Ok(()) - } - fn ensure_record_slot(&self, slot: usize) -> Result { - if let Some(end) = *self.record_ends.get(slot).ok_or(EINVAL)? { - let cursors = self.cursors()?; - if !cursors.consumed(end, cursors.count)? { - return Err(EBUSY); - } - } - Ok(()) - } - - /// A killed queue's ring restarts empty at the firmware's consumer cursor: every - /// item slot is cleared and the host producer is set to the consumer. Only - /// valid after the firmware acknowledged the kill and no command is pending. - fn reset_items(&mut self) -> Result { - let cursors = self.cursors()?; - if cursors.count != ITEM_RING_ENTRIES || cursors.consumer >= cursors.count { - return Err(EIO); - } - for index in 0..ITEM_RING_ENTRIES as usize { - self.graph.dword(ITEMS + index * 8)?.store(0, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - self.graph.word(POINTERS + 0x40)?.store(cursors.consumer, Ordering::Relaxed); - self.reclaimed = cursors.consumer; - fence(Ordering::SeqCst); - Ok(()) - } - /// Reclaim only consumed item pointers, with individual aligned stores and a release fence. - /// This leaves records still named by firmware untouched across arbitrarily many wraps. - fn recycle(&mut self, cursors: Cursors) -> Result { - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.consumer >= cursors.count || self.reclaimed >= cursors.count { - return Err(ERANGE); - } - let done = cursors.consumer; - while self.reclaimed != done { - self.graph - .dword(ITEMS + self.reclaimed as usize * 8)? - .store(0, Ordering::Relaxed); - self.reclaimed = (self.reclaimed + 1) % cursors.count; - } - fence(Ordering::SeqCst); - Ok(()) - } -} - -use crate::g17::{ - completion::Deferred, context::WorkStateLease, freelist::ComputePool, job::Packet, kick, qos, - recovery, status::VmStatus, -}; -use retirement::{Batch, Observation, Recordless, Ticket, IN_FLIGHT}; - -struct Fifo { - entries: [Option; IN_FLIGHT], - head: usize, - len: usize, -} -impl Fifo { - fn new() -> Self { - Self { - entries: core::array::from_fn(|_| None), - head: 0, - len: 0, - } - } - fn front(&self) -> Option<&T> { - self.entries[self.head].as_ref() - } - fn back(&self) -> Option<&T> { - if self.len == 0 { - None - } else { - self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_ref() - } - } - fn back_mut(&mut self) -> Option<&mut T> { - if self.len == 0 { - None - } else { - self.entries[(self.head + self.len - 1) % IN_FLIGHT].as_mut() - } - } - fn push(&mut self, value: T) -> core::result::Result<(), T> { - if self.len == IN_FLIGHT { - return Err(value); - } - self.entries[(self.head + self.len) % IN_FLIGHT] = Some(value); - self.len += 1; - Ok(()) - } - fn pop(&mut self) -> Option { - if self.len == 0 { - return None; - } - let value = self.entries[self.head].take(); - self.head = (self.head + 1) % IN_FLIGHT; - self.len -= 1; - value - } - fn pop_back(&mut self) -> Option { - if self.len == 0 { - return None; - } - self.len -= 1; - self.entries[(self.head + self.len) % IN_FLIGHT].take() - } - fn iter(&self) -> impl Iterator { - Iterator::chain( - self.entries[self.head..].iter(), - self.entries[..self.head].iter(), - ) - .take(self.len) - .filter_map(Option::as_ref) - } - fn iter_mut(&mut self) -> impl Iterator { - let (before, after) = self.entries.split_at_mut(self.head); - Iterator::chain(after.iter_mut(), before.iter_mut()) - .take(self.len) - .filter_map(Option::as_mut) - } -} - -struct Active { - packet: Arc, - published: u64, - work: Arc, - ticket: Option, - observed: Option, - /// Host time the completion record was first observed. - observed_at: Option, - failure_deferred: bool, - qos_pending: bool, -} -struct Previous { - binding: Option, - context: Arc, - publication: Option<(u64, u64)>, -} -/// Per-queue addresses of the bound owner, read under the publication preflight. -struct BindingAddresses { - preempt: u64, - operand: u64, - usage: u64, -} - -#[derive(Copy, Clone, PartialEq, Eq)] -struct ConfigIdentity { - scheduler: u64, - qos: u8, - context: u32, -} - -/// A private physical pool whose descriptor row is installed before allocating its kick ring. -/// The QID remains unpublished throughout construction. -pub(crate) struct PoolBacking { - pool: ComputePool, - id: kick::Id, -} - -impl PoolBacking { - pub(crate) fn new(alloc: &Allocator<'_>, id: kick::Id) -> Result { - let pool = ComputePool::new(alloc, id.qid())?; - Ok(Self { pool, id }) - } - - /// Fresh page-pool row data, installed once while the QID is still unpublished. - pub(crate) fn pool_descriptor(&self) -> Result<(u16, u64)> { - Ok((self.pool.id(), self.pool.page_list_va()?)) - } -} - -/// A physical pool and kick ring prepared before the device-global compute scheduler and -/// descriptor graph. No logical VM aliases or firmware publication exist at this stage. -pub(crate) struct Backing { - pool: ComputePool, - kick: kick::Queue, -} - -impl Backing { - pub(crate) fn new(alloc: &Allocator<'_>, backing: PoolBacking) -> Result { - let PoolBacking { pool, id } = backing; - let kick = kick::Queue::compute(alloc, id)?; - Ok(Self { pool, kick }) - } -} - -/// Installed graph and its exact logical owner. A released idle graph may change owners only -/// after a retirement witness. Failed work retains its pointers independently of fence state. -/// USC pool backing of a retained queue. A release is published only for an idle -/// queue without an owner; the backing is dropped once the firmware consumed it. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum PoolPhase { - Populated, - ReleaseSent(u32), - Vacant, -} - -pub(crate) struct Queue { - owner: Option, - /// Scheduler/work pages of an exited last owner whose context was dropped; - /// the installed queue record still names them until the next owner binds. - retained_owner: Option, - pool_phase: PoolPhase, - /// A killed queue was recycled: its pool/ring must be vacated and re-armed - /// fresh before any owner binds it (set by `recycle_killed`, cleared by `rearm`). - needs_fresh_backing: bool, - /// Monotonic time the last owner released this queue (vacate order). - released_at: u64, - /// Free-list generation of the last published configuration. - free_list_generation: u64, - /// `None` after an exited owner was detached; a new owner binds afresh. - binding: Option, - previous: Option, - kick: kick::Queue, - pool: ComputePool, - graph: Graph, - active: Fifo, - replays: Fifo>, - replay_retirement: bool, - replay_error: Option, - replay_prepared: bool, - exit: Option, - installed: bool, - /// Bit `p` set once a work slot of priority class `p` named this queue. - outer_started: u8, - published_config: Option, - ordinal: u32, - submitted: u32, - last_end: u64, - released: bool, - retirement_ready: bool, - quarantined: bool, - retired_by_teardown: bool, - retire_pending: bool, - awaiting_witness: bool, - spared_quarantine: bool, - spared_deferred: bool, - terminal: Option<(Arc, Error)>, - quarantine_error: Option, - failure_status_pending: bool, - retirement_proved: bool, -} - -impl Queue { - /// The caller reserves a QID first, cancels it on construction failure, and retains every - /// successfully installed queue until processor stop. Shared VM apertures precede this call. - pub(crate) fn new( - alloc: &Allocator<'_>, - backing: Backing, - owner: u64, - context: Arc, - ) -> Result { - let Backing { mut pool, mut kick } = backing; - let mut graph = Graph::new(alloc, &context)?; - let kick_alias = kick.map_client(context.vm())?; - let pool_alias = pool.map_client(context.vm())?; - let binding = graph.bind(context.clone(), kick_alias, pool_alias)?; - context.mark_published(); - Ok(Self { - owner: Some(owner), - retained_owner: None, - pool_phase: PoolPhase::Populated, - needs_fresh_backing: false, - released_at: 0, - free_list_generation: 0, - binding: Some(binding), - previous: None, - kick, - pool, - graph, - active: Fifo::new(), - replays: Fifo::new(), - replay_retirement: false, - replay_error: None, - replay_prepared: false, - exit: None, - installed: false, - outer_started: 0, - published_config: None, - ordinal: 0, - submitted: 0, - last_end: 0, - released: false, - retirement_ready: false, - quarantined: false, - retired_by_teardown: false, - retire_pending: false, - awaiting_witness: false, - spared_quarantine: false, - spared_deferred: false, - terminal: None, - quarantine_error: None, - failure_status_pending: false, - retirement_proved: false, - }) - } - pub(crate) fn qid(&self) -> u8 { - self.kick.id().qid() - } - pub(crate) fn owner(&self) -> Option { - self.owner - } - /// The bound owner's context; `None` once an exited owner was detached. - pub(crate) fn context(&self) -> Option<&Arc> { - self.binding.as_ref().map(|binding| &binding.context) - } - /// Quarantined commands still retain the scheduler until their retirement witness. - pub(crate) fn idle_for_context(&self, context: &Arc) -> bool { - !self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) - || (self.active.len == 0 && self.replays.len == 0) - } - pub(crate) fn in_flight(&self) -> bool { - self.active.len != 0 && !self.quarantined - } - pub(crate) fn room(&self) -> bool { - !self.quarantined && !self.released && self.replays.len == 0 && self.active.len < IN_FLIGHT - } - /// Quarantines without a pending witness no longer describe live pipe order. - pub(crate) fn timeout_head(&self) -> Option { - if self.quarantined && !self.awaiting_witness { - return None; - } - self.active.front().map(|active| crate::g17::timeout::Head { - vm: Arc::as_ptr(active.packet.completion.status()) as usize, - published: active.published, - finished: active.observed.is_some(), - }) - } - - pub(crate) fn can_classify_timeout(&self, packet: &Arc) -> bool { - !self.quarantined && self.owns_packet(packet) - } - - pub(crate) fn oldest_spared(&self) -> Option { - self.active - .front() - .map(|a| a.packet.completion.spared() && a.packet.completion.replay_count() == 0) - } - pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { - for active in self.active.iter() { - visit(active.packet.completion.status()); - } - } - pub(crate) fn classify(&self, class: &dyn Fn(&Arc) -> recovery::Class) { - for active in self.active.iter() { - active - .packet - .completion - .classify(class(active.packet.completion.status())); - } - } - fn qos_owner(&self) -> qos::Owner { - qos::Owner { - qid: self.qid(), - // Accounting completes only for bound owners; a detached queue has no - // pending work and never reaches this. - qos: self.binding.as_ref().map_or(0, |binding| binding.context.qos_id()), - data_master: DataMaster::Compute as u8, - } - } - /// A completion hint can require a second visibility/recordless pass even - /// when firmware has no more events to send. Do not poll idle or failed queues. - pub(crate) fn retirement_polling(&self) -> bool { - !self.quarantined && self.retire_pending - } - - fn selected(&self, masks: Option<[u64; 2]>) -> bool { - self.retired_by_teardown - || self.retire_pending - || masks.is_none_or(|m| m[self.qid() as usize / 64] & (1 << (self.qid() % 64)) != 0) - } - /// Stage every unlatched exact identity before the device's single shared-ring scan. - pub(crate) fn request_observations( - &self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { - return Ok(false); - } - let mut requested = false; - for active in self.active.iter() { - let Some(ticket) = active.ticket else { - break; - }; - if active.observed.is_none() { - batch.request(&ticket)?; - requested = true; - } - } - Ok(requested) - } - pub(crate) fn latch_observations( - &mut self, - batch: &mut Batch, - masks: Option<[u64; 2]>, - ) -> Result { - if self.quarantined || !self.selected(masks) { - return Ok(()); - } - for active in self.active.iter_mut() { - let Some(ticket) = active.ticket else { - break; - }; - if active.observed.is_none() { - active.observed = batch.take(&ticket)?; - if active.observed.is_some() { - active.observed_at = Some(now_ns()); - } - } - } - Ok(()) - } - fn front_finished(&self) -> bool { - self.active.front().is_some_and(|active| { - active.observed.is_some() - || active.ticket.is_some_and(|ticket| { - self.graph - .cursors() - .and_then(|c| c.consumed(ticket.item_producer, ticket.item_count)) - .unwrap_or(false) - }) - }) - } - fn valid_ticket(&self, active: &Active, ticket: &Ticket) -> Result { - if ticket.qid != self.qid() - || ticket.descriptor == 0 - || ticket.item_count == 0 - || ticket.work_node != active.work.node_va()? - || self.submitted == 0 - || self.owner != Some(ticket.owner) - || active.packet.order.sequence != ticket.submission - || active.packet.context.id() != ticket.context_id - || self - .binding - .as_ref() - .is_none_or(|binding| active.packet.context.id() != binding.context.id()) - { - return Err(EIO); - } - Ok(()) - } - - /// Return VM identity for queue-local failure fanout. Deferred callbacks are collected under - /// the mutex and invoked afterwards; failed packets stay retained until their witness. - pub(crate) fn quarantine( - &mut self, - error: Error, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result>> { - if !self.quarantined { - self.quarantined = true; - self.quarantine_error = Some(error); - self.failure_status_pending = true; - self.retirement_proved = false; - self.retired_by_teardown = false; - let all_spared = - self.active.len != 0 && self.active.iter().all(|a| a.packet.completion.spared()); - self.spared_quarantine = all_spared && error == ENODATA; - self.spared_deferred = self.spared_quarantine; - } - let error = self.quarantine_error.ok_or(EIO)?; - let mut terminal = None; - for active in self.active.iter_mut() { - if !self.spared_deferred && !active.failure_deferred { - defer(Deferred::FailedOwned( - active.packet.completion.clone(), - error, - ))?; - active.failure_deferred = true; - } - if self.failure_status_pending - && !active.packet.completion.spared() - && terminal.is_none() - { - terminal = Some(active.packet.completion.status().clone()); - } - } - self.failure_status_pending = false; - self.awaiting_witness = - self.active.len != 0 && self.active.iter().all(|a| a.ticket.is_some()); - Ok(terminal) - } - - /// Poll only the ordered prefix; the next pass may recheck a finished front on another QID's - /// event. A final identity-less scan allows recordless retirement without the one-pass grace. - pub(crate) fn retire( - &mut self, - masks: Option<[u64; 2]>, - epoch: u64, - recovery_state: u32, - mut complete_qos: impl FnMut(qos::Owner) -> Result, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - if self.quarantined || !self.selected(masks) { - return Ok(()); - } - let finished_before = self.retire_pending; - self.retire_pending = false; - let mut oldest = true; - while let Some(front) = self.active.front() { - let result = (|| { - let ticket = front.ticket.ok_or(EIO)?; - self.valid_ticket(front, &ticket)?; - if recovery_state != 0 { - return Err(EAGAIN); - } - let recordless = if epoch != ticket.epoch { - Recordless::CrossedRecovery - } else if masks.is_none() || (finished_before && oldest) { - Recordless::Allowed - } else { - Recordless::NotYet - }; - let next_end = self - .active - .iter() - .skip(1) - .find_map(|a| a.observed.map(|o| o.timestamps[1])); - let stamp = self.graph.stamp()?; - let record_aged = front - .observed_at - .is_some_and(|at| now_ns().saturating_sub(at) >= retirement::STAMP_GRACE_NS); - let timestamps = retirement::poll( - &ticket, - front.observed, - self.graph.cursors()?, - stamp, - recordless, - self.last_end, - next_end, - record_aged, - )?; - if timestamps.is_some() && !retirement::stamp_covers(stamp, ticket.kick) { - retirement::note_stampless(self.qid(), ticket.kick); - } - if timestamps.is_some() - && self.previous.as_ref().is_some_and(|p| { - p.publication != Some((front.packet.order.sequence, ticket.epoch)) - }) - { - return Err(EIO); - } - Ok(timestamps) - })(); - oldest = false; - match result { - Ok(Some(timestamps)) => { - let submitted = self.submitted.checked_sub(1).ok_or(EIO)?; - if front.qos_pending { - complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; - } - let front = self.active.front().ok_or(EIO)?; - self.retire_pending = true; - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok([timestamps[0], timestamps[1], 0, 0]), - ))?; - self.retire_pending = false; - self.submitted = submitted; - let active = self.active.pop().ok_or(EIO)?; - if active.observed.is_some() { - self.last_end = self.last_end.max(timestamps[1]); - } - drop(self.previous.take()); - self.retirement_ready = self.active.len == 0; - if self.released && self.retirement_ready { - self.owner = None; - } - } - Ok(None) | Err(EAGAIN) => { - self.retire_pending = self.front_finished(); - break; - } - Err(error) => { - self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); - return Ok(()); - } - } - } - Ok(()) - } - - /// Exact terminal witness releases command pins, not installed graph ownership. Spared - /// zero-duration work can return its queue to service after settling its accounting. - pub(crate) fn release_witnessed( - &mut self, - recovery_state: u32, - mut complete_qos: impl FnMut(qos::Owner) -> Result, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - self.release_witnessed_inner(recovery_state, false, &mut complete_qos, defer) - } - - pub(crate) fn prepare_replays( - &mut self, - recovery_state: u32, - complete_qos: impl FnMut(qos::Owner) -> Result, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - self.release_witnessed_inner(recovery_state, true, complete_qos, defer) - } - - fn release_witnessed_inner( - &mut self, - recovery_state: u32, - replay: bool, - mut complete_qos: impl FnMut(qos::Owner) -> Result, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - if !self.awaiting_witness || recovery_state != 0 { - return Ok(()); - } - if !self.retirement_proved { - for active in self.active.iter() { - self.valid_ticket(active, &active.ticket.ok_or(EIO)?)?; - } - let newest = self.active.back().and_then(|a| a.ticket).ok_or(EIO)?; - if !retirement::quarantined_retired( - &newest, - self.graph.cursors()?, - self.graph.stamp()?, - self.active.len as u32, - self.submitted, - )? { - return Ok(()); - } - let candidate = self.spared_deferred - && self.spared_quarantine - && !self.released - && self.owner == Some(newest.owner) - && !self - .active - .iter() - .any(|active| active.packet.completion.has_firmware_consumer()) - && self.active.iter().all(|active| { - active - .packet - .completion - .replay_witness_words() - .is_some_and(|words| words[0] == words[1]) - }); - if candidate && !replay { - return Ok(()); - } - self.replay_retirement = candidate; - self.retirement_proved = true; - } - while let Some(front) = self.active.front() { - if front.qos_pending { - complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; - } - let front = self.active.front().ok_or(EIO)?; - let completion = front.packet.completion.clone(); - if self.replay_retirement { - self.replays.push(front.packet.clone()).map_err(|_| EIO)?; - } else if self.spared_deferred { - defer(Deferred::Retired(completion, Err(ENODATA)))?; - } else { - defer(Deferred::Release(completion))?; - } - drop(self.active.pop()); - } - self.awaiting_witness = false; - self.retirement_proved = false; - self.replay_retirement = false; - self.spared_deferred = false; - // The complete retirement witness also covers guilty work. Keep its - // VM status failed, but let logical close return the retained physical - // queue to service instead of consuming another QID for its successor. - self.spared_quarantine = false; - self.quarantine_error = None; - self.submitted = 0; - self.kick.clear_parent_after_recovery(); - self.quarantined = false; - self.retire_pending = false; - self.retirement_ready = true; - if self - .previous - .as_ref() - .is_some_and(|p| p.publication.is_some()) - { - drop(self.previous.take()); - } - if self.released { - self.owner = None; - } - Ok(()) - } - - /// Both FIFOs are bounded by IN_FLIGHT. Count all retained commands of - /// queues which this pass can release, replay or quarantine on failure. - pub(crate) fn replay_commands(&self) -> usize { - if self.awaiting_witness || self.replays.len != 0 { - self.active.len + self.replays.len - } else { - 0 - } - } - - pub(crate) fn replay_front(&self) -> Option> { - self.replays.front().cloned() - } - - /// Drop the witnessed owner only after either publication retained it or - /// the off-lock completion collector accepted its terminal result. - pub(crate) fn finish_replay(&mut self, packet: &Arc) -> Result { - if !self - .replays - .front() - .is_some_and(|front| Arc::ptr_eq(front, packet)) - { - return Err(EIO); - } - drop(self.replays.pop()); - self.replay_prepared = false; - if self.replays.len == 0 { - self.replay_error = None; - } - Ok(()) - } - - pub(crate) fn prepare_replay(&mut self, packet: &Arc) -> Result { - if !self - .replays - .front() - .is_some_and(|front| Arc::ptr_eq(front, packet)) - { - return Err(EIO); - } - if let Some(error) = self.replay_error { - return Err(error); - } - if !self.replay_prepared { - if !packet.completion.begin_replay() { - return Err(EAGAIN); - } - packet.completion.reset_for_replay()?; - self.replay_prepared = true; - } - Ok(()) - } - - pub(crate) fn fail_replays(&mut self, error: Error) { - if self.replay_error.is_none() { - self.replay_error = Some(error); - } - } - - /// Caller already stopped both processors. No firmware owner can access retained mappings. - pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { - while let Some(packet) = self.replays.front() { - defer(Deferred::Retired(packet.completion.clone(), Err(EIO)))?; - drop(self.replays.pop()); - } - while let Some(active) = self.active.front() { - defer(Deferred::Retired( - active.packet.completion.clone(), - Err(EIO), - ))?; - drop(self.active.pop()); - } - Ok(()) - } -} - -impl Queue { - pub(crate) fn matches(&self, owner: u64, qid: u8, context: &Arc) -> Result { - if self.owner != Some(owner) { - return Ok(false); - } - if self.released - || self.quarantined - || self.qid() != qid - || !self - .binding - .as_ref() - .is_some_and(|binding| Arc::ptr_eq(&binding.context, context)) - || !context.is_current() - { - return Err(EIO); - } - Ok(true) - } - pub(crate) fn reusable(&self) -> bool { - self.installed - // A pool release in flight must be witnessed before any successor. - && !matches!(self.pool_phase, PoolPhase::ReleaseSent(_)) - // A recycled killed queue is offered only once its backing is vacant (re-arm path). - && (!self.needs_fresh_backing || self.pool_phase == PoolPhase::Vacant) - // An issued kill still names this graph even if its commands retire - // before the acknowledgement. It cannot acquire a successor owner. - && self.exit.is_none() - && self.owner.is_none() - && self.released - && self.retirement_ready - && !self.quarantined - && self.active.len == 0 - && self.replays.len == 0 - && self.previous.is_none() - } - pub(crate) fn bind_owner(&mut self, owner: u64, context: Arc) -> Result { - if !self.reusable() { - return Err(EBUSY); - } - if !context.is_current() { - return Err(EFAULT); - } - let old = self.binding.as_ref().map(|binding| binding.context.clone()); - let same_vm = old - .as_ref() - .is_some_and(|old| Arc::ptr_eq(old.status(), context.status()) && old.is_current()); - if same_vm { - let binding = self.binding.as_mut().ok_or(EIO)?; - if binding.pool_alias.is_none() { - // The same VM takes back a re-armed pool: alias the new backing. - binding.pool_alias = Some(self.pool.map_client(context.vm())?); - } - if binding.descriptors.is_none() { - binding.descriptors = Some(self.graph.descriptor_alias(context.vm())?); - } - self.graph.set_owner(&context)?; - binding.context = context.clone(); - self.previous = Some(Previous { - binding: None, - context: old.ok_or(EIO)?, - publication: None, - }); - } else { - let kick_alias = self.kick.map_client(context.vm())?; - let pool_alias = self.pool.map_client(context.vm())?; - let binding = self.graph.bind(context.clone(), kick_alias, pool_alias)?; - self.graph.set_owner(&context)?; - let old_binding = self.binding.replace(binding); - // A detached queue (exited owner already dropped) has nothing to supersede; - // the new owner's record fields replace the retained pages' references. - self.previous = old.map(|old| Previous { - binding: old_binding, - context: old, - publication: None, - }); - self.retained_owner = None; - } - context.mark_published(); - self.owner = Some(owner); - self.released = false; - self.retirement_ready = false; - self.retired_by_teardown = false; - Ok(()) - } - /// True allows the registry to cancel the never-published QID and drop this graph. An - /// installed graph remains retained even after the logical owner exits without work. - pub(crate) fn release_owner(&mut self) -> Result { - if !self.installed && self.active.len == 0 && !self.quarantined && self.previous.is_none() { - return Ok(true); - } - self.released = true; - if self.active.len == 0 && !self.quarantined { - if let Some(previous) = self.previous.as_ref().filter(|p| p.publication.is_none()) { - if let Err(error) = self.graph.set_owner(&previous.context) { - self.quarantined = true; - self.retired_by_teardown = true; - return Err(error); - } - let mut previous = self.previous.take().ok_or(EIO)?; - if let Some(binding) = previous.binding.take() { - self.binding = Some(binding); - } else { - self.binding.as_mut().ok_or(EIO)?.context = previous.context; - } - self.retirement_ready = true; - } - if self.retirement_ready { - self.owner = None; - self.released_at = now_ns(); - } - } - Ok(false) - } - /// The last owner's file and VM are gone: its execution root was released - /// after the scheduler-state release and command retirement were witnessed - /// (`Context::release_execution`), and this queue is idle without an owner. - /// Drop the binding (every alias into that VM) and the context, keeping only - /// the scheduler page and work storage the installed queue record names. - /// The returned binding (aliases into the exited VM and the last reference - /// to its context) is dropped by the caller after the device mutex is - /// released: unmapping and VM teardown must not delay other clients. - pub(crate) fn detach_exited_owner(&mut self) -> Option { - if !self.reusable() { - return None; - } - if self.binding.as_ref()?.context.is_current() { - return None; - } - let binding = self.binding.take()?; - self.retained_owner = Some(binding.context.retain_firmware_pages()); - Some(binding) - } - /// An idle retained queue whose USC backing may be given back. - pub(crate) fn vacate_candidate(&self) -> bool { - (self.reusable() || self.recycled_pending()) && self.pool_phase == PoolPhase::Populated && self.pool.populated() - } - /// A recycled killed queue still holding the backing its killed work used. - pub(crate) fn recycled_pending(&self) -> bool { - self.needs_fresh_backing - && self.installed - && self.exit.is_none() - && self.owner.is_none() - && !self.quarantined - && self.active.len == 0 - && self.replays.len == 0 - && self.previous.is_none() - } - /// Recycled queues are vacated regardless of the warm cap. - pub(crate) fn needs_fresh_backing(&self) -> bool { - self.needs_fresh_backing - } - /// The firmware acknowledged the kill of this queue's closed owner (`ContextKilled`) - /// and every killed command was settled: return the QID to service. The item - /// ring is reset to empty at the firmware's consumer cursor; the USC pool and - /// descriptor ring are vacated and re-armed fresh before the next owner binds. - pub(crate) fn recycle_killed(&mut self) -> Result { - if !self.installed - || !self.exit.as_ref().is_some_and(|exit| exit.released()) - || self.active.len != 0 - || self.replays.len != 0 - || self.previous.is_some() - { - return Ok(false); - } - self.graph.reset_items()?; - self.exit = None; - self.quarantined = false; - self.quarantine_error = None; - self.failure_status_pending = false; - self.retired_by_teardown = false; - self.spared_quarantine = false; - self.spared_deferred = false; - self.retire_pending = false; - self.awaiting_witness = false; - self.retirement_proved = false; - self.replay_retirement = false; - self.replay_error = None; - self.replay_prepared = false; - self.submitted = 0; - self.kick.clear_parent_after_recovery(); - self.owner = None; - self.released = true; - self.retirement_ready = true; - self.released_at = now_ns(); - self.needs_fresh_backing = true; - Ok(true) - } - pub(crate) fn released_at(&self) -> u64 { - self.released_at - } - pub(crate) fn pool_vacant(&self) -> bool { - self.pool_phase == PoolPhase::Vacant - } - /// Publishes the release of this idle queue's free list. The caller holds the - /// device mutex and rings the control doorbell afterwards. - pub(crate) fn vacate(&mut self, init: &crate::g17::initdata::InitData) -> Result { - if !self.vacate_candidate() { - return Err(EBUSY); - } - let record = ControlRecord::FreeListRelease( - self.pool.release_record(self.free_list_generation), - ); - let cursor = Rings::publish_control(init, &record)?; - self.pool_phase = PoolPhase::ReleaseSent(cursor); - fence(Ordering::SeqCst); - Ok(()) - } - /// Detaches the USC backing once the firmware's control consumer passed the - /// release. The caller frees the returned backing after releasing the - /// device mutex. - pub(crate) fn observe_vacancy( - &mut self, - consumer: u32, - producer: u32, - ) -> Result> { - let PoolPhase::ReleaseSent(cursor) = self.pool_phase else { - return Ok(None); - }; - if !crate::g17::freelist::control_consumed(consumer, producer, cursor)? { - return Ok(None); - } - // The retained owner binding's client alias pins the backing: detach it - // first (the owner has exited; `reusable()` proved no previous binding). - let alias = self.binding.as_mut().and_then(|binding| binding.pool_alias.take()); - let pages = self.pool.take_pages(); - let descriptors = self.binding.as_mut().and_then(|binding| binding.descriptors.take()); - let ring = self.graph.vacate_ring(); - self.pool_phase = PoolPhase::Vacant; - Ok(Some(Vacated { - _alias: alias, - _pages: pages, - _descriptors: descriptors, - _ring: ring, - })) - } - /// Installs fresh backing into a vacant pool; returns the descriptor row data - /// the caller writes before binding an owner. - pub(crate) fn rearm( - &mut self, - pages: crate::g17::freelist::Pages, - ring: DescriptorRing, - ) -> Result<(u16, u64)> { - if self.pool_phase != PoolPhase::Vacant || !(self.reusable() || self.recycled_pending()) { - return Err(EAGAIN); - } - self.graph.rearm_ring(ring)?; - self.pool.rearm(pages)?; - self.pool_phase = PoolPhase::Populated; - self.needs_fresh_backing = false; - Ok((self.pool.id(), self.pool.page_list_va()?)) - } - /// Exact host identity remains valid even when several commands share an ioctl ID. - pub(crate) fn owns_packet(&self, packet: &Arc) -> bool { - self.active - .iter() - .any(|active| Arc::ptr_eq(&active.packet, packet)) - } - - /// A watchdog settles spared, deferred waiters without changing their VM status. - /// Fully ticketed spared work can still return this queue after its drain witness. - pub(crate) fn timeout( - &mut self, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result>> { - let fresh = !self.quarantined; - let return_spared = self.active.len != 0 - && self - .active - .iter() - .all(|active| active.packet.completion.spared() && active.ticket.is_some()); - if self.spared_deferred { - for active in self.active.iter_mut() { - if !active.failure_deferred { - defer(Deferred::FailedOwned( - active.packet.completion.clone(), - ETIMEDOUT, - ))?; - active.failure_deferred = true; - } - } - self.spared_deferred = false; - } - let result = self.quarantine(ETIMEDOUT, defer); - if fresh { - self.spared_quarantine = return_spared; - } - result - } - - pub(crate) fn cancel( - &mut self, - packet: &Arc, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - if !self.owns_packet(packet) { - return Ok(false); - } - if self.quarantined { - if self.spared_deferred { - for active in self.active.iter_mut() { - if !active.failure_deferred { - defer(Deferred::FailedOwned( - active.packet.completion.clone(), - ECANCELED, - ))?; - active.failure_deferred = true; - } - } - self.spared_deferred = false; - } - return Ok(true); - } - self.released = true; - self.retired_by_teardown = true; - if let Some(active) = self - .active - .iter_mut() - .find(|a| Arc::ptr_eq(&a.packet, packet)) - { - if !active.failure_deferred { - defer(Deferred::FailedOwned( - active.packet.completion.clone(), - ECANCELED, - ))?; - active.failure_deferred = true; - } - } - Ok(true) - } - /// Firmware is halted and publication is closed while its target is copied. - pub(crate) fn publish_recovery_target(&self) -> Result { - let producer = self - .graph - .graph - .word(POINTERS + PointerBlock::PRODUCER)? - .load(Ordering::Relaxed); - self.graph - .graph - .word(POINTERS + PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); - Ok(()) - } - - /// The newest covered kick of a QID subsumes its earlier prefix; later independent work - /// must not become a dependency. Errors are observed before returning any firmware token. - pub(crate) fn dependency( - &self, - owner: u64, - status: &Arc, - through: Option, - recovery_state: u32, - ) -> Result> { - if through.is_none() || self.owner != Some(owner) { - return Ok(None); - } - let Some(active) = self - .active - .iter() - .filter(|a| crate::g17::job::Order::contains(through, a.packet.order.sequence)) - .last() - else { - return Ok(None); - }; - if self.quarantined && self.spared_quarantine { - return Err(ECANCELED); - } - if !Arc::ptr_eq(active.packet.completion.status(), status) - || self.quarantined - || recovery_state != 0 - { - return Err(EIO); - } - let ticket = active.ticket.ok_or(EIO)?; - let state = crate::g17::job::fence_status(&active.packet.completion.fence()); - if state < 0 { - return Err(crate::g17::dependency::inherited_error(Error::from_errno( - state, - ))); - } - if state > 0 { - return Ok(None); - } - active.packet.completion.note_firmware_consumer(); - let kick = crate::g17::fw::kick::KickTimestamp::new(ticket.kick).ok_or(EIO)?; - Ok(Some( - crate::g17::fw::kick::KickDependency::new(self.qid(), kick).ok_or(EIO)?, - )) - } -} - -impl Graph { - fn record( - &mut self, - slot: Option, - offset: usize, - window: usize, - value: T, - ) -> Result { - let (object, start) = if let Some(slot) = slot { - if slot >= RECORD_SLOTS { - return Err(EINVAL); - } - (&mut self.records, slot * RECORD_STRIDE + offset) - } else { - (&mut self.graph, 0x6800 + offset) - }; - if core::mem::size_of::() > window { - return Err(EINVAL); - } - let pointer = object.pointer(start, window)?; - // SAFETY: The checked record window is not named by any unconsumed item pointer. The - // device mutex excludes other host publishers, and no Rust reference to it is formed. - unsafe { pointer.write_bytes(0, window) }; - object.write(start, value)?; - fence(Ordering::SeqCst); - Ok(object.gpu_va() + start as u64) - } - fn append(&self, expected: u32, address: u64, before_publish: impl FnOnce()) -> Result { - let cursors = self.cursors()?; - if cursors.producer != expected - || cursors.count != ITEM_RING_ENTRIES - || expected >= ITEM_RING_ENTRIES - { - return Err(EBUSY); - } - let next = (expected + 1) % ITEM_RING_ENTRIES; - if next == cursors.consumer { - return Err(EBUSY); - } - let slot = self.graph.dword(ITEMS + expected as usize * 8)?; - let producer = self.graph.word(POINTERS + 0x40)?; - if slot.load(Ordering::Relaxed) != 0 { - return Err(EBUSY); - } - before_publish(); - slot.store(address, Ordering::Relaxed); - fence(Ordering::SeqCst); - producer.store(next, Ordering::Relaxed); - Ok(next) - } -} - -/// Borrowed device operations used during one publication. Implementations borrow the initdata, -/// coprocessor and accounting fields separately from the Registry that owns physical queues. -pub(crate) trait Host { - fn next_compute_publication(&mut self) -> Result; - fn epoch(&self) -> Result<(u64, u32)>; - fn prepare_compute_shared(&mut self) -> Result; - fn qos_publish( - &mut self, - owner: qos::Owner, - scheduler: u64, - policy: crate::g17::fw::queue::Policy, - ) -> Result; - fn qos_cancel(&mut self, publication: qos::Publication) -> Result; - fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer(&mut self, priority: u8, slot: &crate::g17::fw::channels::WorkSlot) - -> Result; - fn notify(&mut self, message: u64) -> Result; - fn note_submission(&mut self) -> Result; -} - -impl Queue { - /// Snapshot only while publication is closed and firmware is halted. - pub(crate) fn recovery_diagnostics(&self) -> impl core::fmt::Debug + '_ { - let command = self.active.front().and_then(|active| { - if let crate::g17::command::Validated::Compute { - cdm_va, cdm_end_va, sampler_count, scratch, .. - } = &active.packet.command { - Some((*cdm_va, *cdm_end_va, *sampler_count, scratch.enabled())) - } else { - None - } - }); - ( - self.owner, - self.context().map(|context| (context.id(), context.generation())), - self.ordinal, - self.active.len, - self.graph.cursors(), - self.graph.stamp(), - command, - ) - } - - pub(crate) fn spared_pending(&self) -> bool { - self.quarantined && self.spared_quarantine && self.awaiting_witness - } - - /// Retains the packet before any potentially visible store. An error before publication - /// rolls back its work count; every ambiguous error quarantines the graph and packet. - pub(crate) fn publish( - &mut self, - packet: Arc, - dependencies: &[crate::g17::fw::kick::KickDependency], - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - use crate::g17::fw::{ - channels::WorkSlot, - compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, - kick::{KickArgs, KickQos}, - }; - let (epoch, addresses) = self.publication_preflight(&packet, host)?; - let crate::g17::command::Validated::Compute { - scratch, - usc_base, - cdm_va, - cdm_end_va, - sampler_heap_va, - sampler_count, - .. - } = &packet.command - else { - return Err(EINVAL); - }; - let work = packet.completion.work_state()?; - let work_word = work.word(); - if work_word == u32::MAX { - return Err(EINVAL); - } - let work_node = work.node_va()?; - let timestamp_va = packet.completion.compute_timestamp_va()?; - let kick_timestamp = self.kick.timestamp(); - let kick_parent = self.kick.parent(); - let qid = self.qid(); - let ordinal = self.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let (descriptor_high, descriptor_low) = - self.graph.descriptor_vas(kick_timestamp.slot() as u8)?; - let first = !self.installed && self.ordinal == 0; - let entry_signal = first - || dependencies - .iter() - .any(|dependency| dependency.qid() != qid); - let record_slot = (!first).then_some(self.ordinal as usize % RECORD_SLOTS); - if let Some(slot) = record_slot { - self.graph.ensure_record_slot(slot)?; - } - let args = ComputeArgs { - scratch: *scratch, - usc_base: *usc_base, - cdm_va: *cdm_va, - cdm_end_va: *cdm_end_va, - sampler_heap_va: *sampler_heap_va, - sampler_count: *sampler_count, - descriptor_va: descriptor_low, - context_id: packet.context.id().try_into()?, - context_generation: packet.context.generation(), - work_state: work_word, - work_state_va: work_node, - qid, - kick_count: self.ordinal, - kick: kick_timestamp, - preempt_va: addresses.preempt, - operand_state_va: addresses.operand, - usage_va: addresses.usage, - usage_fw_va: self.graph.usage.gpu_va(), - free_list_slot: u32::from(self.pool.id()), - free_list_control_va: self.pool.control_va(), - stamp_va: self.graph.support.gpu_va() + STAMP as u64, - aux_stamp_va: self.graph.support.gpu_va() + STAMP as u64 + 8, - status_va: [ - self.graph.support.gpu_va() + 0x310, - self.graph.support.gpu_va() + 0x318, - ], - timestamp_va, - }; - let identity = ConfigIdentity { - scheduler: packet.context.scheduler_va(), - qos: packet.context.qos_id(), - context: packet.context.id(), - }; - self.free_list_generation = u64::from(packet.context.scheduler_generation()); - let policy = packet.context.policy(); - let ring = 1u8 << policy.priority(); - let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; - let announce = - KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; - let predecessor = KickPredecessor::new(qid, kick_parent)?; - let kick_args = KickArgs { - qid, - timestamp: kick_timestamp, - parent: kick_parent, - barriers: dependencies, - descriptor_va: descriptor_high, - queue_va: self.graph.graph.gpu_va(), - qos: KickQos { - slot: identity.qos, - class: COMPUTE_QOS_CLASS, - }, - mcache: None, - event_mask: COMPUTE_KICK_EVENT_MASK, - register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, - compute_scratch: scratch.enabled(), - priority: policy.priority(), - }; - let published = host.next_compute_publication()?; - self.active - .push(Active { - packet: packet.clone(), - published, - work: work.clone(), - ticket: None, - observed: None, - observed_at: None, - failure_deferred: false, - qos_pending: false, - }) - .map_err(|_| EBUSY)?; - self.retirement_ready = false; - let mut visible = false; - let mut accounted = false; - let result = (|| -> Result { - work.add_submitted_kicks(1)?; - accounted = true; - self.graph - .ring - .as_mut() - .ok_or(EAGAIN)? - .object - .initialize::( - kick_timestamp.slot() % DESCRIPTORS * COMPUTE_DESCRIPTOR_SIZE, - |descriptor| descriptor.write(&args), - )?; - if first { - self.graph.context_high.write( - 0x200, - crate::g17::fw::compute::QueueContextItem::new( - descriptor_high, - self.graph.graph.gpu_va(), - qid, - ), - )?; - } else { - // The retained context item remains at index one; only its descriptor changes. - self.graph.context_high.write(0x210, descriptor_high)?; - let cursors = self.graph.cursors()?; - let _ = self.graph.recycle(cursors); - } - let config_va = self.graph.record(record_slot, CONFIG, 0xc0, config)?; - let cursors = self.graph.cursors()?; - if cursors.count != ITEM_RING_ENTRIES { - return Err(EIO); - } - if cursors.producer >= cursors.count || cursors.consumer >= cursors.count { - return Err(ERANGE); - } - let free = (cursors.consumer + cursors.count - cursors.producer - 1) % cursors.count; - if free < if entry_signal { 4 } else { 3 } { - return Err(ENOSPC); - } - let prepared_kick = self.kick.prepare_entry(&kick_args)?; - let mut producer; - if first { - producer = self.graph.first_descriptor(descriptor_high)?; - host.prepare_compute_shared()?; - packet.mark_published(); - visible = true; - let qos = host.qos_publish( - qos::Owner { - qid, - qos: identity.qos, - data_master: 2, - }, - identity.scheduler, - policy, - )?; - self.active.back_mut().ok_or(EIO)?.qos_pending = true; - let target = self.pool.reserve_submission().or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - self.pool.cancel_unpublished(target)?; - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; - } else { - let target = self.pool.reserve_submission()?; - producer = self - .graph - .append(cursors.producer, descriptor_high, || { - packet.mark_published() - }) - .or_else(|error| { - self.pool.cancel_unpublished(target)?; - Err(error) - })?; - visible = true; - let qos = host.qos_publish( - qos::Owner { - qid, - qos: identity.qos, - data_master: 2, - }, - identity.scheduler, - policy, - )?; - self.active.back_mut().ok_or(EIO)?.qos_pending = true; - producer = self - .graph - .append(producer, config_va, || {}) - .or_else(|error| { - host.qos_cancel(qos)?; - self.active.back_mut().ok_or(EIO)?.qos_pending = false; - Err(error) - })?; - } - if first { - prepared_kick.prepare_compute_install()?; - } - prepared_kick.commit(); - self.submitted = self.submitted.checked_add(1).ok_or(EOVERFLOW)?; - let announce_va = self.graph.record(record_slot, ANNOUNCE, 0x40, announce)?; - producer = self.graph.append(producer, announce_va, || {})?; - fence(Ordering::SeqCst); - if entry_signal { - let predecessor_va = - self.graph - .record(record_slot, PREDECESSOR, 0x400, predecessor)?; - producer = self.graph.append(producer, predecessor_va, || {})?; - } - if let Some(slot) = record_slot { - self.graph.record_ends[slot] = Some(producer); - } - host.publish_qid(self.kick.id())?; - host.publish_outer( - policy.priority(), - &WorkSlot::new( - DataMaster::Compute, - self.graph.graph.gpu_va(), - qid, - producer.try_into()?, - self.outer_started & ring == 0, - )?, - )?; - let notify_activation = self.outer_started & ring == 0; - self.outer_started |= ring; - self.installed = true; - self.published_config = Some(identity); - self.ordinal = ordinal; - let (accepted, state) = host.epoch()?; - if state != 0 || accepted != epoch { - return Err(EIO); - } - let ticket = Ticket { - owner: self.owner.ok_or(EIO)?, - submission: packet.order.sequence, - context_id: packet.context.id(), - qid, - descriptor: descriptor_high, - work_node, - kick: kick_timestamp.get(), - ordinal, - item_producer: producer, - item_count: ITEM_RING_ENTRIES, - epoch, - }; - self.active.back_mut().ok_or(EIO)?.ticket = Some(ticket); - if let Some(previous) = self.previous.as_mut() { - if previous.publication.is_none() { - previous.publication = Some((packet.order.sequence, epoch)); - } - } - if notify_activation { - fence(Ordering::SeqCst); - // Pipe 2 (compute) in bits 1:0, priority class in bits 3:2. - host.notify((0x83 << 48) | 0x02 | u64::from(policy.priority()) << 2)?; - } - fence(Ordering::SeqCst); - host.notify((0x83 << 48) | 0x10)?; - host.note_submission() - })(); - if let Err(error) = result { - if !visible { - if accounted { - if let Err(error) = work.cancel_unpublished_kicks(1) { - self.terminal = - self.quarantine(error, defer)?.map(|status| (status, error)); - return Err(error); - } - } - drop(self.active.pop_back()); - return Err(error); - } - self.terminal = self.quarantine(error, defer)?.map(|status| (status, error)); - return Err(error); - } - Ok(()) - } - - /// Everything that must hold before a publication changes any state: a healthy device - /// epoch, room in the active ring (or this packet at the replay front), the bound - /// owner's current context and a clean completion status. Returns the epoch and the - /// bound owner's per-queue addresses. - fn publication_preflight( - &self, - packet: &Arc, - host: &mut impl Host, - ) -> Result<(u64, BindingAddresses)> { - let (epoch, state) = host.epoch()?; - if state != 0 { - return Err(EBUSY); - } - let replay_front = self - .replays - .front() - .is_some_and(|front| Arc::ptr_eq(front, packet)); - if !self.room() - && !(replay_front && !self.quarantined && !self.released && self.active.len < IN_FLIGHT) - { - return Err(EBUSY); - } - if !packet.context.is_current() - || !self - .binding - .as_ref() - .is_some_and(|binding| Arc::ptr_eq(&packet.context, &binding.context)) - { - return Err(EFAULT); - } - let binding = self.binding.as_ref().ok_or(EFAULT)?; - if packet.completion.status().get() != 0 { - return Err(EIO); - } - Ok(( - epoch, - BindingAddresses { - preempt: binding.preempt.iova(), - operand: binding.operand.iova(), - usage: binding.usage.iova(), - }, - )) - } - - /// The queue configuration record for this publication: a fresh install on the first - /// one, and QoS/context updates only when the identity changed since the last publication. - fn queue_config( - &self, - identity: &ConfigIdentity, - first: bool, - context_id: u16, - policy: crate::g17::fw::queue::Policy, - packet: &Arc, - ) -> Result { - QueueConfig::new(&QueueConfigArgs { - target: QueueConfigTarget::Compute, - kick_ring_va: self.kick.low_va(), - kick_ring_fw_va: self.kick.firmware_va(), - qid: self.qid(), - install: first, - completion_seed: None, - context_id, - free_list: FreeListBinding { - va: self.pool.control_va(), - generation: u64::from(packet.context.scheduler_generation()), - slot: u32::from(self.pool.id()), - }, - scheduler_va: identity.scheduler, - policy, - qos_slot: identity.qos, - qos_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.qos != identity.qos), - owner_pid: packet.context.owner_pid(), - context_update: self - .published_config - .is_none_or(|p| p.scheduler != identity.scheduler || p.context != identity.context), - }) - } -} - -impl Graph { - /// The queue has never been installed. Retry staging replaces its initial three item slots; - /// no firmware reader can name this graph until its first complete outer publication. - fn first_descriptor(&self, address: u64) -> Result { - let first = self.graph.dword(ITEMS)?; - let second = self.graph.dword(ITEMS + 8)?; - let third = self.graph.dword(ITEMS + 16)?; - let producer = self.graph.word(POINTERS + 0x40)?; - first.store(address, Ordering::Relaxed); - second.store(0, Ordering::Relaxed); - third.store(0, Ordering::Relaxed); - producer.store(1, Ordering::Relaxed); - Ok(1) - } -} -impl Queue { - pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { - self.terminal.take() - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/compute/exit.rs b/drivers/gpu/drm/asahi-neo/g17/queue/compute/exit.rs deleted file mode 100644 index 10d2a372748002..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/compute/exit.rs +++ /dev/null @@ -1,212 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Closed-context kill requests and the command-pin release witness. - -use super::{Context, Deferred, Queue}; -use core::sync::atomic::{fence, AtomicU32, Ordering}; -use kernel::{prelude::*, sync::Arc}; - -pub(crate) const POLL_NS: u64 = 2_500_000_000; -const QUARANTINED_DELAY_NS: u64 = 250_000_000; -const CANCELLED_DELAY_NS: u64 = 2_000_000_000; - -// Firmware-visible cookie namespace persists for the loaded module's lifetime. -static NEXT_COOKIE: AtomicU32 = AtomicU32::new(1); - -pub(crate) fn next_cookie() -> Result { - // Never reuse an identity while a delayed firmware ACK can still exist. - // The final counter value is an exhaustion marker, not a wrapping cookie. - NEXT_COOKIE - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |next| next.checked_add(1)) - .map(u64::from) - .map_err(|_| EOVERFLOW) -} - -pub(crate) struct State { - pub(crate) slot: u8, - pub(crate) qids: u128, - pub(crate) killed: u32, -} -impl State { - pub(crate) fn read(context: &Context) -> Result { - let words = context.scheduler_owner().read_words::<14>(0)?; - let mut bytes = [0u8; 0x38]; - for (destination, word) in bytes.chunks_exact_mut(4).zip(words) { - destination.copy_from_slice(&word.to_le_bytes()); - } - fence(Ordering::Acquire); - Ok(Self::decode(&bytes)) - } - fn decode(bytes: &[u8; 0x38]) -> Self { - let mut low = [0; 8]; - let mut high = [0; 8]; - let mut killed = [0; 4]; - low.copy_from_slice(&bytes[0x12..0x1a]); - high.copy_from_slice(&bytes[0x1a..0x22]); - killed.copy_from_slice(&bytes[0x34..0x38]); - Self { - slot: bytes[1], - qids: u128::from(u64::from_le_bytes(low)) - | (u128::from(u64::from_le_bytes(high)) << 64), - killed: u32::from_le_bytes(killed), - } - } - pub(crate) fn scope(&self, target: u8, own: u128, foreign: u128) -> Option { - if self.slot == 0xff || target >= 128 || self.qids & (1u128 << target) == 0 { - return None; - } - let allowed = (own | (1u128 << target)) & !foreign & !0b11; - (self.qids & !allowed == 0).then_some(self.qids) - } -} - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Phase { - Published, - Acknowledged, - Unproven, - Released, -} - -pub(super) struct Transaction { - cookie: u64, - published: u64, - phase: Phase, - announced: Option, -} - -impl Transaction { - /// The firmware acknowledged the kill and the killed commands were settled. - pub(super) fn released(&self) -> bool { - self.phase == Phase::Released - } -} - -impl Queue { - pub(crate) fn exit_candidate(&self, context: &Arc, age: u64) -> bool { - self.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) - && self.released - && self.active.len != 0 - && self.exit.is_none() - && (self.quarantined || self.retired_by_teardown) - && age - >= if self.quarantined { - QUARANTINED_DELAY_NS - } else { - CANCELLED_DELAY_NS - } - } - pub(crate) fn exit_owner(&self, context: &Arc) -> Option { - if !self - .active - .iter() - .all(|active| Arc::ptr_eq(&active.packet.context, context)) - { - return None; - } - self.owner.or_else(|| { - self.active - .back() - .and_then(|active| active.ticket.map(|ticket| ticket.owner)) - }) - } - pub(crate) fn record_exit(&mut self, cookie: u64, now: u64) { - self.exit = Some(Transaction { - cookie, - published: now, - phase: Phase::Published, - announced: None, - }); - } - /// A failed doorbell does not undo the published kill or consume its retry. - /// The caller retries under the device mutex; only the original record exists. - pub(crate) fn announce_exit(&mut self, now: u64, notify: impl FnOnce() -> Result) -> Result { - let Some(exit) = self.exit.as_mut() else { - return Ok(()); - }; - if exit.phase != Phase::Published || exit.announced.is_some() { - return Ok(()); - } - notify()?; - exit.announced = Some(now); - Ok(()) - } - pub(crate) fn exit_polling(&self, now: u64) -> bool { - self.exit.as_ref().is_some_and(|exit| { - exit.phase == Phase::Published - && now.saturating_sub(exit.announced.unwrap_or(exit.published)) < POLL_NS - }) - } - /// Event consumption and record_exit are serialized by the device mutex. - /// A kill pins this exact queue transaction through its final witness; - /// duplicate, unknown and terminal cookies cannot acknowledge another owner. - pub(crate) fn acknowledge_exit(&mut self, cookie: u64) -> bool { - let Some(exit) = self.exit.as_mut() else { - return false; - }; - if cookie == 0 || exit.cookie != cookie || exit.phase != Phase::Published { - return false; - } - exit.phase = Phase::Acknowledged; - true - } - pub(crate) fn settle_exit( - &mut self, - recovery_closed: bool, - complete_qos: &mut impl FnMut(crate::g17::qos::Owner) -> Result, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - if !self.exit.as_ref().is_some_and(|exit| exit.phase == Phase::Acknowledged) { - return Ok(()); - } - if self.active.len != 0 { - if !recovery_closed { - return Ok(()); - } - let Some(context) = self.context() else { - return Ok(()); - }; - let state = match State::read(context) { - Ok(state) => state, - Err(_) => return Ok(()), - }; - if state.killed != 1 { - self.exit.as_mut().ok_or(EIO)?.phase = Phase::Unproven; - return Ok(()); - } - if !self.active.iter().all(|active| { - active - .packet - .completion - .compute_words() - .is_ok_and(|words| words == [0, 0] || words[1] != 0) - }) { - return Ok(()); - } - } - while let Some(active) = self.active.front() { - if active.qos_pending { - complete_qos(self.qos_owner())?; - self.active.iter_mut().next().ok_or(EIO)?.qos_pending = false; - } - let active = self.active.front().ok_or(EIO)?; - defer(Deferred::Retired( - active.packet.completion.clone(), - Err(if self.spared_deferred { - ENODATA - } else { - ECANCELED - }), - ))?; - drop(self.active.pop()); - } - self.exit.as_mut().ok_or(EIO)?.phase = Phase::Released; - self.quarantined = true; - self.awaiting_witness = false; - self.spared_deferred = false; - self.spared_quarantine = false; - self.retire_pending = false; - self.retirement_ready = false; - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/compute/retirement.rs b/drivers/gpu/drm/asahi-neo/g17/queue/compute/retirement.rs deleted file mode 100644 index b4d4bcef24421f..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/compute/retirement.rs +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Exact identities and visibility witnesses for a physical compute queue. -//! A shared work-state counter can include other engines and cannot prove this queue retired. - -use crate::g17::initdata::ComputeCompletion; -use kernel::prelude::*; - -pub(crate) const IN_FLIGHT: usize = 16; -pub(crate) const RECORD_SLOTS: usize = 32; -const QIDS: usize = 128; -const TIME_MASK: u64 = 0x003f_ffff_ffff_ffff; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Ticket { - pub(crate) owner: u64, - pub(crate) submission: u64, - pub(crate) context_id: u32, - pub(crate) qid: u8, - pub(crate) descriptor: u64, - pub(crate) work_node: u64, - pub(crate) kick: u64, - pub(crate) ordinal: u32, - pub(crate) item_producer: u32, - pub(crate) item_count: u32, - pub(crate) epoch: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Observation { - pub(crate) timestamps: [u64; 2], -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Cursors { - pub(crate) producer: u32, - pub(crate) consumer: u32, - pub(crate) count: u32, -} - -impl Cursors { - /// Whether firmware consumed every record of the given publication, allowing later work. - pub(crate) fn consumed(&self, end: u32, count: u32) -> Result { - if self.count == 0 || self.count != count { - return Err(EIO); - } - let producer = self.producer % count; - let consumer = self.consumer % count; - let unconsumed = (producer + count - consumer) % count; - let following = (producer + count - end % count) % count; - Ok(unconsumed <= following) - } -} - -/// Outstanding kicks span fewer than half the 24-bit stamp sequence. Zero is a valid wrap. -pub(crate) const fn stamp_covers(word: u32, kick: u64) -> bool { - let delta = word.wrapping_sub((kick as u32).wrapping_shl(8)); - delta & 0xff == 0 && delta < 0x8000_0000 -} - -#[derive(Clone, Copy)] -pub(crate) enum Recordless { - Allowed, - NotYet, - CrossedRecovery, -} - -/// How long a command whose exact completion record is present waits for its completion stamp -/// before it retires on the record alone. The firmware stores the stamp after the cache -/// maintenance that follows the record; it occasionally never stores it for a command, most -/// visibly while a render of the same VM is running. -pub(crate) const STAMP_GRACE_NS: u64 = 1_000_000; - -static STAMPLESS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); - -/// Counts a command that retired on its record without its completion stamp; logs the first and -/// every 64th. -pub(crate) fn note_stampless(qid: u8, kick: u64) { - let count = STAMPLESS.fetch_add(1, core::sync::atomic::Ordering::Relaxed) + 1; - if count == 1 || count % 64 == 0 { - pr_info!( - "G17: {} compute commands retired on their completion record without a completion stamp (latest: queue {} kick {:#x})\n", - count, - qid, - kick - ); - } -} - -/// A zero-duration record fails before visibility polling. It proves execution did not occur; -/// ownership still requires a separate retirement witness before mappings can be released. -/// The command is complete when its stamp covers it, or when its exact completion record has -/// been present for [`STAMP_GRACE_NS`] (`record_aged`); in both cases its items must be consumed. -pub(crate) fn poll( - ticket: &Ticket, - observed: Option, - cursors: Cursors, - stamp: u32, - recordless: Recordless, - floor: u64, - next_end: Option, - record_aged: bool, -) -> Result> { - if observed.is_some_and(|record| record.timestamps[0] == record.timestamps[1]) { - return Err(ENODATA); - } - let completed = stamp_covers(stamp, ticket.kick) || (observed.is_some() && record_aged); - if !completed || !cursors.consumed(ticket.item_producer, ticket.item_count)? { - return Ok(None); - } - Ok(Some(match observed { - Some(record) => record.timestamps, - None => match recordless { - Recordless::Allowed => [floor, next_end.unwrap_or(floor).max(floor)], - Recordless::NotYet => return Ok(None), - Recordless::CrossedRecovery => return Err(ENODATA), - }, - })) -} - -/// The newest exact publication covers all preceding kicks, but only a fully consumed ring -/// permits releasing quarantined command pins. The installed graph remains retained. -pub(crate) fn quarantined_retired( - ticket: &Ticket, - cursors: Cursors, - stamp: u32, - pending: u32, - submitted: u32, -) -> Result { - if ticket.item_count == 0 || ticket.descriptor == 0 || pending == 0 { - return Err(EINVAL); - } - if cursors.count == 0 || cursors.count != ticket.item_count { - return Err(EIO); - } - let end = ticket.item_producer % cursors.count; - Ok(stamp_covers(stamp, ticket.kick) - && cursors.producer % cursors.count == end - && cursors.consumer % cursors.count == end - && submitted == pending) -} - -#[derive(Clone, Copy)] -struct Request { - descriptor: u64, - kick: u64, - observed: Option, -} - -/// Reusable heap scratch for one shared-ring scan. QID requests are contiguous and bounded; -/// staging and folding never allocate under the device mutex. -pub(crate) struct Batch { - requests: KVVec, - slots: [(usize, usize); QIDS], - open: Option, -} - -impl Batch { - pub(crate) fn new() -> Result { - Ok(Self { - requests: KVVec::with_capacity(QIDS * IN_FLIGHT, GFP_KERNEL)?, - slots: [(0, 0); QIDS], - open: None, - }) - } - pub(crate) fn clear(&mut self) { - self.requests.clear(); - self.slots.fill((0, 0)); - self.open = None; - } - pub(crate) fn request(&mut self, ticket: &Ticket) -> Result { - if ticket.descriptor == 0 || ticket.kick == 0 { - return Err(EINVAL); - } - let slot = self.slots.get_mut(ticket.qid as usize).ok_or(EINVAL)?; - if slot.1 != 0 && self.open != Some(ticket.qid) { - return Err(EBUSY); - } - if slot.1 >= IN_FLIGHT { - return Err(ENOSPC); - } - let start = if slot.1 == 0 { - self.requests.len() - } else { - slot.0 - }; - self.requests - .push_within_capacity(Request { - descriptor: ticket.descriptor, - kick: ticket.kick, - observed: None, - }) - .map_err(|_| ENOSPC)?; - *slot = (start, slot.1 + 1); - self.open = Some(ticket.qid); - Ok(()) - } - /// Capture the control word once, then read only a requested identity's payload. - /// Records can be overwritten during a scan: incomplete or older identities are skipped. - pub(crate) fn fold(&mut self, record: ComputeCompletion<'_>) -> Result { - let control = record.read(0)?; - let qid = ((control >> 9) & 0x7f) as usize; - let (start, count) = self.slots[qid]; - if count == 0 { - return Ok(()); - } - let descriptor = record.read(6)? & !1; - if descriptor == 0 || record.read(7)? == 0 { - return Ok(()); - } - let kick = (((control >> 8) & 0xffff_ffff_00) | ((control >> 48) & 0xff)) & 0xff_ffff_ffff; - let requests = self.requests.get_mut(start..start + count).ok_or(EIO)?; - let Some(request) = requests - .iter_mut() - .find(|r| r.descriptor == descriptor && r.kick == kick) - else { - return Ok(()); - }; - let timestamps = [record.read(2)? & TIME_MASK, record.read(5)? & TIME_MASK]; - if timestamps[0] == 0 - || timestamps[1] == 0 - || request - .observed - .is_some_and(|old| old.timestamps[1] >= timestamps[1]) - { - return Ok(()); - } - request.observed = Some(Observation { timestamps }); - Ok(()) - } - pub(crate) fn take(&mut self, ticket: &Ticket) -> Result> { - let &(start, count) = self.slots.get(ticket.qid as usize).ok_or(EINVAL)?; - let request = self - .requests - .get_mut(start..start + count) - .ok_or(EIO)? - .iter_mut() - .find(|r| r.descriptor == ticket.descriptor && r.kick == ticket.kick) - .ok_or(EIO)?; - Ok(request.observed.take()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/mod.rs b/drivers/gpu/drm/asahi-neo/g17/queue/mod.rs deleted file mode 100644 index 370f6f5f791488..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/mod.rs +++ /dev/null @@ -1,407 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Userspace submission queues: validate whole ioctls, retain mappings, and enqueue -//! independent engines behind user fences and explicit logical prefixes. - -pub(crate) mod compute; -pub(crate) mod render; - -use super::{ - command::{Parser, UscWindow, Validated}, - completion::{Completion, Destinations, PreparedPublication}, - context::Context, - fence::{self, Outputs, Submission}, - job::{Backend, Engine, Fences, Order, Packet, Scheduler}, - validation::Proof, -}; -use crate::{ - driver::{AsahiDevRef, AsahiDevice}, - file, mmu, -}; -use core::ops::Range; -use kernel::{ - c_str, - dma_fence::{Fence, FenceContexts, RawDmaFence}, - prelude::*, - sync::{Arc, LockClassKey}, - xarray, -}; - -const EARLY_DEPTH: usize = 2; -static FENCE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); - -pub(crate) mod admission; - -/// Host identities for retained physical channels, separate from firmware object generations. -/// The device mutex protects allocation; cancelled preparations never reuse IDs. -pub(crate) struct Identities(u64); - -impl Identities { - pub(crate) const fn new() -> Self { - Self(3) - } - - /// The first two identities remain reserved with QIDs 0 and 1. - pub(crate) fn reserve(&mut self, count: u64) -> Result> { - let first = self.0; - let end = first.checked_add(count).ok_or(EOVERFLOW)?; - self.0 = end; - Ok(first..end) - } -} - -/// Validated frontend state prepared before allocating a logical execution context. -pub(crate) struct Frontend { - fences: FenceContexts, - allocation_failure: Fence, - range: Range, - window: UscWindow, -} - -impl Frontend { - pub(crate) fn new(priority: u32, usc_base: u64, range: Range) -> Result { - if priority > 3 { - return Err(EINVAL); - } - let window = UscWindow::new(usc_base, range.clone())?; - let fences = FenceContexts::new(1, c_str!("asahi_g17_queue"), FENCE_KEY)?; - let allocation_failure = super::fence::allocation_failure(&fences)?; - Ok(Self { - fences, - allocation_failure, - range, - window, - }) - } -} - -pub(crate) struct Queue { - dev: AsahiDevRef, - context: Arc, - backend: Arc, - render: Option>, - compute: Option>, - fences: FenceContexts, - allocation_failure: Fence, - range: Range, - window: UscWindow, - proof: Proof, - sequence: [u64; 2], - frontiers: [KVec<(u64, Fences)>; 2], - failures: [Option<(u64, Fence)>; 2], - // Closing the userspace queue releases admission even if firmware still - // retains one of its execution contexts or installed physical graphs. - _slot: admission::QueueSlot, -} - -impl Queue { - pub(crate) fn new( - dev: &AsahiDevice, - context: Arc, - backend: Arc, - frontend: Frontend, - slot: admission::QueueSlot, - ) -> Result { - let Frontend { - fences, - allocation_failure, - range, - window, - } = frontend; - let render = Some(Scheduler::new(dev, backend.clone(), Engine::Render)?); - Ok(Self { - dev: dev.into(), - context, - backend, - render, - compute: None, - fences, - allocation_failure, - range, - window, - proof: Proof::new(), - sequence: [0; 2], - frontiers: core::array::from_fn(|_| KVec::new()), - failures: [None, None], - _slot: slot, - }) - } - - fn ensure_compute(&mut self) -> Result { - // Admission is checked even when this queue already owns a scheduler. - self.backend.ensure_compute()?; - if self.compute.is_none() { - self.compute = Some(Scheduler::new( - &self.dev, - self.backend.clone(), - Engine::Compute, - )?); - } - Ok(()) - } - - fn prune_frontier(&mut self, index: usize) { - let failure = &mut self.failures[index]; - self.frontiers[index].retain(|(sequence, fences)| { - let status = fence::completion_status(&fences.completed); - if status == 0 { - return true; - } - // One earliest failed ordinal covers every later prefix without - // retaining its packet, mappings, or every failed fence. - if status < 0 && failure.as_ref().is_none_or(|(first, _)| *sequence < *first) { - *failure = Some((*sequence, fences.completed.clone())); - } - false - }); - } - - fn enqueue( - &mut self, - id: u64, - command: Validated, - order: Order, - guard: mmu::VmJobGuard, - aggregate: &Arc, - inputs: &[file::SyncItem], - destinations: Destinations, - ) -> Result { - let engine = match &command { - Validated::Render { .. } => Engine::Render, - Validated::Compute { .. } => Engine::Compute, - }; - let index = engine.index(); - self.prune_frontier(index); - self.frontiers[index].reserve(1, GFP_KERNEL)?; - if engine == Engine::Compute { - self.ensure_compute()?; - } - // Only earlier render ordinals covered by the explicit logical prefix - // can delay host publication. These links never become GPU dependencies. - let host_predecessors = self.frontiers[index] - .iter() - .take_while(|(sequence, _)| { - engine == Engine::Render - && Order::contains(order.wait_through[index], *sequence) - }) - .filter_map(|(_, fences)| fences.publication.as_ref()); - let completion = Completion::new( - &self.dev, - &self.fences, - &self.context, - aggregate.member(), - guard, - destinations, - engine == Engine::Compute, - host_predecessors.clone().next().is_some(), - if engine == Engine::Render { self.backend.feed() } else { None }, - )?; - let publication = completion.publication() - .map(|target| PreparedPublication::new(target, host_predecessors)) - .transpose()?; - if engine == Engine::Compute { - self.prune_frontier(Engine::Render.index()); - } - let other = 1 - index; - let prefix = order.wait_through[other]; - let frontier = if prefix.is_some() { - let frontier = self.frontiers[other].as_slice(); - // Accepted sequences increase from one, and retain preserves their - // order, so all members of a logical prefix form one slice. - let end = frontier.partition_point(|(sequence, _)| Order::contains(prefix, *sequence)); - &frontier[..end] - } else { - &[] - }; - let count = frontier.len(); - let timestamp_prefix = order.wait_through[index]; - let timestamps = if engine == Engine::Compute && timestamp_prefix.is_some() { - let frontier = self.frontiers[index].as_slice(); - let end = frontier - .partition_point(|(sequence, _)| Order::contains(timestamp_prefix, *sequence)); - &frontier[..end] - } else { - &[] - }; - let timestamp_count = timestamps - .iter() - .filter(|(_, fences)| fences.ready.raw() == fences.completed.raw()) - .count(); - let failures: [Option<&Fence>; 2] = core::array::from_fn(|index| { - self.failures[index].as_ref().and_then(|(sequence, fence)| { - Order::contains(order.wait_through[index], *sequence).then_some(fence) - }) - }); - let failure_count = failures.iter().flatten().count(); - let mut dependencies = - KVec::with_capacity(inputs.len() + count + timestamp_count + failure_count, GFP_KERNEL)?; - let mut checked_inputs = - KVec::with_capacity(inputs.len() + timestamp_count + failure_count, GFP_KERNEL)?; - let mut firmware = KVec::with_capacity(count, GFP_KERNEL)?; - for fence in failures.into_iter().flatten() { - checked_inputs.push(fence.clone(), GFP_KERNEL)?; - dependencies.push(fence.clone(), GFP_KERNEL)?; - } - // Same-engine parents order GPU execution, not the host's timestamp - // writes. Only timestamp publishers need these completion waits and - // status checks; ordinary compute keeps its existing queue ordering. - for (_, fences) in timestamps { - if fences.ready.raw() == fences.completed.raw() { - checked_inputs.push(fences.completed.clone(), GFP_KERNEL)?; - dependencies.push(fences.ready.clone(), GFP_KERNEL)?; - } - } - for (_, fences) in frontier { - // Successful completion includes host timestamp publication. Its - // ready fence adds no ordering and would prevent shallow inline - // publication. Failed and pending producers still need checking. - if fence::completion_status(&fences.completed) > 0 { - continue; - } - dependencies.push(fences.ready.clone(), GFP_KERNEL)?; - firmware.push(fences.completed.clone(), GFP_KERNEL)?; - } - // Each command owns every input fence; independent engines may publish in - // either order and must each consume an input's error status before running. - for sync in inputs { - if let Some(fence) = &sync.fence { - dependencies.push(fence.clone(), GFP_KERNEL)?; - checked_inputs.push(fence.clone(), GFP_KERNEL)?; - } - } - let early = engine == Engine::Render - && dependencies.is_empty() - && self.frontiers[index].len() < EARLY_DEPTH; - let packet = Packet::new( - id, - self.context.clone(), - command, - order, - completion, - checked_inputs, - firmware, - )?; - let scheduler = match engine { - Engine::Render => &mut self.render, - Engine::Compute => &mut self.compute, - }; - let fences = scheduler - .as_mut() - .ok_or(ENODEV)? - .enqueue(packet, dependencies, early, publication)?; - self.sequence[index] = order.sequence; - self.frontiers[index] - .push_within_capacity((order.sequence, fences)) - .map_err(|_| ENOMEM)?; - Ok(()) - } - /// Finish parsing, validation and VM pinning before entering publication. - /// Keeping this frame separate bounds the scheduler/ioctl stack depth. - #[inline(never)] - fn prepare_submission( - &mut self, - bytes: &[u8], - objects: Pin<&xarray::XArray>>, - ) -> Result<(KVec<(Validated, Order, Destinations)>, KVec)> { - let parse_guard = self.context.vm().retain_first_job()?; - let mut parser = Parser::new(bytes); - // All historical ordinals use the same entry counters; publication - // starts only after the entire ioctl has been validated and pinned. - let base = self.sequence; - let mut preceding = [0; 2]; - let mut pending = KVec::with_capacity(1, GFP_KERNEL)?; - while let Some(command) = parser.next()? { - let payload = self.proof.validate( - self.context.vm(), - self.range.clone(), - &self.window, - &command.payload, - )?; - let engine = match &payload { - Validated::Render { .. } => Engine::Render, - Validated::Compute { .. } => Engine::Compute, - }; - let order = Order::new( - base, - preceding, - command.barriers, - command.payload.prior(), - engine, - )?; - preceding[engine.index()] += 1; - let destinations = match &payload { - Validated::Render { timestamps, .. } => Destinations::resolve(objects, timestamps)?, - Validated::Compute { timestamps, .. } => { - Destinations::resolve(objects, core::slice::from_ref(timestamps))? - } - }; - pending.push((payload, order, destinations), GFP_KERNEL)?; - } - parser.finish()?; - // Every accepted command pins the VM before any command is enqueued. - let mut guards = KVec::with_capacity(pending.len(), GFP_KERNEL)?; - guards - .push_within_capacity(parse_guard) - .map_err(|_| ENOMEM)?; - while guards.len() < pending.len() { - guards - .push_within_capacity(self.context.vm().retain_job()?) - .map_err(|_| ENOMEM)?; - } - Ok((pending, guards)) - } -} - -impl crate::queue::Queue for Queue { - fn submit( - &mut self, - id: u64, - syncs: KVec, - input_count: usize, - bytes: &[u8], - objects: Pin<&xarray::XArray>>, - ) -> Result { - let mut outputs = Outputs::new( - &self.fences, - syncs, - input_count, - &self.allocation_failure, - )?; - let result = (|| { - if self.context.status().get() != 0 { - return Err(EIO); - } - let (pending, guards) = self.prepare_submission(bytes, objects)?; - let mut guards = guards.into_iter(); - let aggregate = outputs.publish(self.context.clone())?; - for (command, order, destinations) in pending { - let guard = guards.next().ok_or(EIO)?; - self.enqueue( - id, - command, - order, - guard, - &aggregate, - outputs.inputs(), - destinations, - )?; - } - Ok(()) - })(); - if let Err(error) = result { - self.context.status().record_if_device_loss(error); - if [EIO, ETIMEDOUT, ENODEV].contains(&error) || self.context.status().get() != 0 { - self.backend.fail_vm(error); - } - } - outputs.finish(result) - } -} - -impl Drop for Queue { - fn drop(&mut self) { - drop(self.render.take()); - drop(self.compute.take()); - self.backend.release_owner(); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render.rs deleted file mode 100644 index d82ee84c1283ff..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render.rs +++ /dev/null @@ -1,1595 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Physical paired tiling and fragment queues and their retained command storage. - -mod observation; -pub(crate) use observation::Observed; -mod dependencies; -mod memory; -pub(crate) mod parked; -mod publication; -pub(crate) mod retirement; -mod warm; - -use crate::g17::{ - buffer::{self, target}, - command::Validated, - completion::Deferred, - context::Context, - freelist::RenderPool, - fw::{ - channels::WorkSlot, - kick::{KickDependency, KickTimestamp, McacheTable, RenderRegistration}, - queue::DataMaster, - render as abi, - }, - job::Packet, - kick, - object::{Allocator, Pool}, - qos, recovery, - status::VmStatus, -}; -use crate::{hw::t8140::queues, mmu}; -pub(crate) use dependencies::Dependencies; -use kernel::{prelude::*, sync::Arc}; -use retirement::{Ticket, Tracker}; - -/// Firmware dependencies of one render, kept by origin until the kick entries -/// are built: the covered earlier renders (their fragment completions) and the -/// covered compute completion. Which kick waits for the render prefix is the -/// command's choice (`RenderPass::fragment_barriers`); compute always orders -/// the tiling kick, since compute output may feed vertex work. -#[derive(Copy, Clone)] -pub(crate) struct RenderDependencies { - pub(crate) render: [KickDependency; 2], - pub(crate) render_count: usize, - pub(crate) compute: Option, -} - -impl RenderDependencies { - pub(crate) const EMPTY: Self = Self { - render: [KickDependency::ZERO; 2], - render_count: 0, - compute: None, - }; - pub(crate) fn render(&self) -> &[KickDependency] { - &self.render[..self.render_count] - } -} - -#[derive(Copy, Clone)] -pub(crate) struct RenderAddresses { - /// Tiling then fragment; each pair contains the primary and auxiliary stamp. - pub(crate) stamps: [[u64; 2]; 2], -} - -#[derive(Copy, Clone)] -pub(crate) struct PublicationLease { - pub(crate) free_list_generation: u64, - pub(crate) buffer_token: u64, -} - -/// Host services borrow the device-locked firmware state. Resource construction -/// and command geometry preparation do not borrow this interface. -pub(crate) trait Host { - fn epoch(&self) -> Result<(u64, u32)>; - fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]>; - fn qos_publish( - &mut self, - owner: qos::Owner, - scheduler: u64, - policy: crate::g17::fw::queue::Policy, - ) -> Result; - fn qos_cancel(&mut self, publication: qos::Publication) -> Result; - fn install_pair(&mut self, registration: &RenderRegistration) -> Result; - fn publish_qid(&mut self, id: kick::Id) -> Result; - fn publish_outer_pair( - &mut self, - priority: u8, - fragment: &crate::g17::fw::channels::WorkSlot, - tiling: &crate::g17::fw::channels::WorkSlot, - commit: impl FnOnce() -> Result, - late_tiling: impl FnOnce(), - ) -> Result<[u32; 2]>; - fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result; - fn publish_retained_pair( - &mut self, - priority: u8, - fragment: &WorkSlot, - tiling: &WorkSlot, - next: [u32; 2], - ) -> Result; - fn notify_render(&mut self, priority: u8) -> Result; - fn note_submission(&mut self) -> Result; - fn complete_render( - &mut self, - owners: Option<[qos::Owner; 2]>, - pool_id: u16, - generation: u64, - buffer_id: u8, - ) -> Result; -} - -struct Prepared { - packet: Arc, - ordinal: u64, - mcache: Option, -} -struct Active { - qos_completed: bool, - flist_completed: bool, - outer_published: bool, - generation: u64, - packet: Arc, - ticket: Ticket, - kicks: [KickTimestamp; 2], - context_update: bool, - failed: bool, - drained_result: Option>, -} - -/// Installed kick backing remains owned even when this pair has no logical -/// owner. Commands retain their exact descriptor and allocation generations. -pub(crate) struct Pair { - kick_aliases: Option<[Option; 2]>, - memory: KBox, - kicks: [kick::Queue; 2], - free_list: Arc, - pending_aliases: Option>, - prepared: Option, - active: Tracker, - slot: u8, - owner: Option, - ordinal: u64, - initial_packet: u64, - clusters: u32, - descriptor_flags: [u32; 2], - installed: [bool; 2], - configs: [Option; 2], - cursors: [u32; 2], - recovery_generation: u64, - /// Priority class of the pair's installed registration, if any. A pair - /// rebound to an owner of another class is registered again. - registered: Option, - /// Bit `p` set once a work slot of priority class `p` named the pair. - outer_started: u8, - quarantined: bool, - terminal: Option<(Arc, Error)>, - previous: Option, - displaced: Option, - context_update: bool, -} -impl Pair { - #[inline(never)] - pub(crate) fn new( - alloc: &Allocator<'_>, - pool: &Arc, - ids: [kick::Id; 2], - slot: u8, - owner: u64, - context: Arc, - free_list: Arc, - metrics: buffer::MetricsLease, - buffer_id: (u8, u64), - pm_generation: u64, - first: &Arc, - clusters: u32, - descriptor_flags: [u32; 2], - ) -> Result> { - if slot == 0 - || usize::from(slot) >= queues::RENDER_SLOTS - || owner == 0 - || ids[0].engine() != DataMaster::Tiling - || ids[1].engine() != DataMaster::Fragment - || !Arc::ptr_eq(&context, &first.context) - || !context.is_current() - { - return Err(EINVAL); - } - if let Err(error) = alloc.uat.vm_context_mask(context.vm()) { - if error == ENOENT { - context.status().record(error); - } - return Err(error); - } - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; - let word = first.completion.work_state()?.word(); - if word == u32::MAX { - return Err(EINVAL); - } - let layout = target::Layout::new(pass, clusters)?; - let storage = KBox::::new_uninit(GFP_KERNEL)?; - let (tiling, ta_alias) = kick::Queue::render(alloc, ids[0], context.vm())?; - let (fragment, fragment_alias) = kick::Queue::render(alloc, ids[1], context.vm())?; - let (memory, aliases) = memory::Memory::new( - alloc, - pool, - context, - [ids[0].qid(), ids[1].qid()], - &layout, - metrics, - buffer_id, - pm_generation, - 0, - word, - )?; - storage.write_init(kernel::try_init!(Self { - kick_aliases: Some([Some(ta_alias), Some(fragment_alias)]), - memory, - kicks: [tiling, fragment], - free_list, - pending_aliases: Some(aliases), - prepared: None, - active: Tracker::new(), - slot, - owner: Some(owner), - ordinal: 0, - initial_packet: first.id, - clusters, - descriptor_flags, - installed: [false; 2], - configs: [None; 2], - cursors: [0; 2], - recovery_generation: 0, - registered: None, - outer_started: 0, - quarantined: false, - terminal: None, - previous: None, - displaced: None, - context_update: true, - })) - } - pub(crate) fn qids(&self) -> [u8; 2] { - self.kicks.each_ref().map(|kick| kick.id().qid()) - } - pub(crate) fn slot(&self) -> u8 { - self.slot - } - pub(crate) fn owner(&self) -> Option { - self.owner - } - pub(crate) fn context(&self) -> &Arc { - &self.memory.context - } - pub(crate) fn pool(&self) -> &Arc { - &self.free_list - } - pub(crate) fn pool_id(&self) -> u64 { - u64::from(self.memory.lifecycle[0].predecessor) - } - pub(crate) fn buffer_id(&self) -> u8 { - self.memory.manager.buffer_id() - } - pub(crate) fn manager(&mut self) -> &mut buffer::Manager { - &mut self.memory.manager - } - pub(crate) fn in_flight(&self) -> bool { - self.active.len() != 0 - } - pub(crate) fn published_in_flight(&self) -> bool { - !self.quarantined && self.active.iter().any(|active| active.outer_published) - } - pub(crate) fn room(&self) -> bool { - !self.quarantined && self.owner.is_some() && self.prepared.is_none() && self.active.room() - } - pub(crate) fn take_terminal(&mut self) -> Option<(Arc, Error)> { - self.terminal.take() - } - - /// The registry owns this pair exclusively during potentially allocating - /// preparation. No queue producer or shared submission counter changes. - pub(crate) fn prepare( - &mut self, - alloc: &Allocator<'_>, - packet: &Arc, - addresses: RenderAddresses, - ) -> Result { - if !self.room() - || !Arc::ptr_eq(&self.memory.context, &packet.context) - || !packet.context.is_current() - { - return Err(EBUSY); - } - let Validated::Render { - pass, attachments, .. - } = &packet.command - else { - return Err(EINVAL); - }; - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let ordinal = self.ordinal; - ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let work = packet.completion.work_state()?; - let node = work.node_va()?; - let word = work.word(); - let layout = target::Layout::new(pass, self.clusters)?; - let owners = self.memory.owners(); - self.memory - .target - .ensure_capacity(alloc.dev, packet.context.vm(), &layout, &owners)?; - let target = self.memory.target.prepare(ordinal, &layout)?; - let scene = self.memory.manager.prepare_scene(ordinal)?; - self.memory.generation(ordinal)?.prepare(ordinal, pass)?; - let [deflake, ta_status, fragment_status, fragment_fw, auxiliary] = - self.memory.generation(ordinal)?.addresses(); - let lifecycle = if ordinal != 0 || packet.id != self.initial_packet { - let (fragment, tiling) = - crate::g17::ids::OBJECTS.render_pair(self.memory.lifecycle[0].predecessor)?; - [tiling, fragment] - } else { - self.memory.lifecycle - }; - let (scratch, metric) = self.memory.manager.scene_registers(scene)?; - let timestamp_offset = ordinal as usize % memory::RECORDS * 16; - let timestamp = self.memory.timestamps.gpu_va() + timestamp_offset as u64; - let resources = abi::RenderResources { - context_id: packet.context.id().try_into()?, - context_generation: packet.context.generation(), - pb_slot: self.buffer_id().into(), - free_list_slot: self.free_list.id().into(), - free_list_control_va: self.free_list.control_va(), - kick_record_va: node, - pb_va: self.memory.manager.state_va(), - scene_va: self.memory.manager.scene_va(scene)?, - scene_end_va: self.memory.manager.trailer_va(), - pm_scratch: scratch, - pm_metric: metric, - tilemap_va: target.tilemap, - layermeta_va: target.layer, - heapmeta_va: target.heap, - tpc_va: self.memory.target.tpc_va(), - tpc_size: layout.tpc_size as u64, - deflake_va: deflake, - ta_status_va: ta_status, - fragment_status_va: fragment_status, - fragment_status_fw_va: fragment_fw, - aux_fb_va: auxiliary, - timestamp_va: timestamp, - ta_ids: lifecycle[0], - fragment_ids: lifecycle[1], - unk_2104: self.descriptor_flags[0], - unk_210c: self.descriptor_flags[1], - }; - let qids = self.qids(); - let args = abi::RenderArgs { - pass, - resources: &resources, - ordinal, - ta: abi::RenderStage { - qid: qids[0], - descriptor_va: self.memory.graph.descriptor_client(0, ordinal), - state_word: word, - stamp_va: addresses.stamps[0][0], - aux_stamp_va: addresses.stamps[0][1], - }, - fragment: abi::RenderStage { - qid: qids[1], - descriptor_va: self.memory.graph.descriptor_client(1, ordinal), - state_word: word, - stamp_va: addresses.stamps[1][0], - aux_stamp_va: addresses.stamps[1][1], - }, - }; - self.memory.graph.claim(ordinal)?; - let result = self.memory.write_descriptors( - &args, - attachments[1].as_slice(), - packet.context.id().try_into()?, - ); - let mcache = match result { - Ok(mcache) => mcache, - Err(error) => { - self.memory.graph.release(ordinal)?; - self.memory.generation(ordinal)?.ordinal = None; - self.memory.target.cancel(ordinal); - return Err(error); - } - }; - self.memory.timestamps.write(timestamp_offset, [0u64; 2])?; - self.memory.manager.claim_scene(ordinal, scene)?; - self.prepared = Some(Prepared { - packet: packet.clone(), - ordinal, - mcache, - }); - Ok(()) - } -} - -/// Kick barriers of one render. The fragment kick always waits for its own tiling kick; -/// with the fragment-barriers flag the resolved prefix orders the fragment kick too and the -/// tiling kick keeps only its implicit parent, otherwise the prefix orders the tiling kick. -/// A compute dependency always orders the tiling kick. -struct RenderBarriers { - tiling: [KickDependency; 3], - tiling_count: usize, - fragment: [KickDependency; 3], - fragment_count: usize, -} - -fn render_barriers( - fragment_stage: bool, - dependencies: &RenderDependencies, - own_tiling: KickDependency, -) -> RenderBarriers { - let mut tiling_barriers = [KickDependency::ZERO; 3]; - let mut tiling_count = 0; - let mut fragment_barriers = [own_tiling; 3]; - let mut fragment_count = 1; - for dependency in dependencies.render() { - if fragment_stage { - fragment_barriers[fragment_count] = *dependency; - fragment_count += 1; - } else { - tiling_barriers[tiling_count] = *dependency; - tiling_count += 1; - } - } - if let Some(compute) = dependencies.compute { - tiling_barriers[tiling_count] = compute; - tiling_count += 1; - } - RenderBarriers { - tiling: tiling_barriers, - tiling_count, - fragment: fragment_barriers, - fragment_count, - } -} - -impl Pair { - /// All descriptor bytes and retained allocations are ready before entry. - /// The first inner producer establishes firmware ownership, including on - /// later errors. Such failures retain the active packet for recovery. - pub(crate) fn publish( - &mut self, - packet: Arc, - dependencies: &RenderDependencies, - lease: PublicationLease, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - use crate::g17::fw::{ - channels::WorkSlot, - kick::{KickArgs, KickQos}, - queue::{ - self, FreeListBinding, QueueConfig, QueueConfigArgs, QueueConfigTarget, - TilingBinding, - }, - }; - use core::sync::atomic::{fence, Ordering}; - let (epoch, mcache, policy) = self.publication_preflight(&packet, host)?; - let ordinal = self.ordinal; - let next_ordinal = ordinal.checked_add(1).ok_or(EOVERFLOW)?; - let qids = self.qids(); - let kicks = self.kicks.each_ref().map(kick::Queue::timestamp); - let parents = self.kicks.each_ref().map(kick::Queue::parent); - let work = packet.completion.work_state()?; - let work_slot = work.render_slot(self.slot)?; - self.memory.bind_work_slot(ordinal, qids, work_slot)?; - let payload = work.channel_payload(self.slot, ordinal)?; - let stamps = host.stamps(qids)?; - fence(Ordering::Acquire); - let covered = stamps.map(|stamp| retirement::stamp_covers(stamp, ordinal)); - self.memory - .completion_links(ordinal, kicks, payload, !covered[1])?; - let context = &packet.context; - let ring = 1u8 << policy.priority(); - let qos = context.qos_id(); - let qos_owners = [ - qos::Owner { - qid: qids[0], - qos, - data_master: DataMaster::Tiling as u8, - }, - qos::Owner { - qid: qids[1], - qos, - data_master: DataMaster::Fragment as u8, - }, - ]; - let mut configs = [None; 2]; - for stage in 0..2 { - configs[stage] = Some(QueueConfig::new(&QueueConfigArgs { - target: if stage == 0 { - QueueConfigTarget::Tiling(TilingBinding { - pb_va: self.memory.manager.state_va(), - pb_token: lease.buffer_token, - pb_slot: self.buffer_id().into(), - fragment_qid: qids[1], - }) - } else { - QueueConfigTarget::Fragment - }, - kick_ring_va: self.kicks[stage].low_va(), - kick_ring_fw_va: self.kicks[stage].firmware_va(), - qid: qids[stage], - install: !self.installed[stage], - completion_seed: covered[stage].then(|| { - kicks[stage] - .prev() - .completion_value(payload.wrapping_sub(1)) - }), - context_id: context.id().try_into()?, - free_list: FreeListBinding { - va: self.free_list.control_va(), - generation: lease.free_list_generation, - slot: self.free_list.id().into(), - }, - scheduler_va: context.scheduler_va(), - policy, - qos_slot: qos, - qos_update: self.context_update, - owner_pid: context.owner_pid(), - context_update: self.context_update, - })?); - } - let configs = [configs[0].ok_or(EIO)?, configs[1].ok_or(EIO)?]; - let cache = if self.recovery_generation == epoch { - self.configs - } else { - [None; 2] - }; - let present = [ - publication::required(cache[0], configs[0]), - publication::required(cache[1], configs[1]), - ]; - let cursors = self.memory.graph.cursors()?; - if cursors[0].write != self.cursors[0] || cursors[1].write != self.cursors[1] { - return Err(EBUSY); - } - let queues = qids.map(|qid| { - self.memory.graph.queues.gpu_va() - + u64::from(qid) * size_of::() as u64 - }); - let Validated::Render { pass, .. } = &packet.command else { - return Err(EINVAL); - }; - let own_tiling = KickDependency::new(qids[0], kicks[0]).ok_or(EINVAL)?; - let barriers = render_barriers(pass.fragment_barriers, dependencies, own_tiling); - let tiling_barriers = &barriers.tiling[..barriers.tiling_count]; - let args = |stage: usize, barriers, arrays, mcache| KickArgs { - qid: qids[stage], - timestamp: kicks[stage], - parent: parents[stage], - barriers, - descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), - queue_va: queues[stage], - qos: KickQos { - slot: qos, - class: if stage == 0 { - abi::TA_QOS_CLASS - } else { - abi::FRAGMENT_QOS_CLASS - }, - }, - mcache, - event_mask: abi::RENDER_KICK_EVENT_MASK, - register_arrays: arrays, - compute_scratch: false, - priority: policy.priority(), - }; - let ta_args = args( - 0, - tiling_barriers, - abi::TaDescriptor::register_bindings(self.memory.graph.descriptor_client(0, ordinal))?, - None, - ); - let fragment_args = args( - 1, - &barriers.fragment[..barriers.fragment_count], - abi::FragmentDescriptor::register_bindings( - self.memory.graph.descriptor_client(1, ordinal), - )?, - mcache, - ); - let registration = RenderRegistration::new( - (qids[1], self.kicks[1].low_va()), - (qids[0], self.kicks[0].low_va()), - policy.priority(), - )?; - let [tiling_kick, fragment_kick] = &mut self.kicks; - let tiling_kick = tiling_kick.prepare_entry(&ta_args)?; - let fragment_kick = fragment_kick.prepare_entry(&fragment_args)?; - let memory = &mut *self.memory; - let lanes = [ - publication::Lane::new( - &memory.graph, - 0, - ordinal, - self.cursors[0], - configs[0], - present[0], - kicks[0], - )?, - publication::Lane::new( - &memory.graph, - 1, - ordinal, - self.cursors[1], - configs[1], - present[1], - kicks[1], - )?, - ]; - let windows = [lanes[0].window, lanes[1].window]; - let previous = self.active.front().map_or(cursors, |active| { - active.ticket.windows.map(|window| { - let target = window.target(); - retirement::Cursors { - done: target, - read: target, - write: target, - } - }) - }); - let fragment_slot = WorkSlot::new( - DataMaster::Fragment, - queues[1], - qids[1], - windows[1].target().try_into()?, - self.outer_started & ring == 0, - )?; - let tiling_slot = WorkSlot::new( - DataMaster::Tiling, - queues[0], - qids[0], - windows[0].target().try_into()?, - self.outer_started & ring == 0, - )?; - let event_slot = memory - .graph - .support - .word(abi::storage::EVENT_SLOTS + 4)?; - let next_event = event_slot.load(Ordering::Relaxed).wrapping_add(1); - let inner = memory.graph.support.word(0x28000)?; - let next_inner = (next_ordinal as u32).wrapping_mul(2); - let flist_submission = self.free_list.prepare_submission(2)?; - let pm_submission = memory.manager.prepare_submission()?; - self.active - .push(Active { - qos_completed: false, - flist_completed: false, - outer_published: false, - generation: lease.free_list_generation, - packet: packet.clone(), - ticket: Ticket { - ordinal, - windows, - previous, - }, - kicks, - context_update: self.context_update, - failed: false, - drained_result: None, - }) - .map_err(|_| EBUSY)?; - self.prepared = None; - packet.mark_published(); - lanes[1].publish_prefix(); - fragment_kick.commit(); - lanes[1].publish_kick(); - tiling_kick.commit(); - let mut qos_publications = [None; 2]; - let result = (|| { - qos_publications[1] = - Some(host.qos_publish(qos_owners[1], context.scheduler_va(), policy)?); - qos_publications[0] = - Some(host.qos_publish(qos_owners[0], context.scheduler_va(), policy)?); - // A class change happens only at a rebind, after the pair drained. - if self.registered != Some(policy.priority()) { - host.install_pair(®istration)?; - self.registered = Some(policy.priority()); - } - for kick in &self.kicks { - host.publish_qid(kick.id())?; - } - host.publish_outer_pair( - policy.priority(), - &fragment_slot, - &tiling_slot, - || { - work.add_submitted_kicks(2)?; - flist_submission.commit(); - pm_submission.commit(); - Ok(()) - }, - || lanes[0].publish_tiling(event_slot, next_event, inner, next_inner), - ) - })(); - let outer_next = match result { - Ok(next) => next, - Err(error) => { - self.quarantined = true; - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - self.active - .back() - .ok_or(EIO)? - .packet - .completion - .quarantine_render(); - for publication in qos_publications.into_iter().flatten() { - host.qos_cancel(publication)?; - } - self.active.back_mut().ok_or(EIO)?.qos_completed = true; - self.quarantined = true; - self.record_publication_failure(&packet, error, defer)?; - return Err(error); - } - }; - self.ordinal = next_ordinal; - self.cursors = windows.map(|window| window.target()); - for stage in 0..2 { - if present[stage] { - self.configs[stage] = Some(configs[stage]); - } - } - self.recovery_generation = epoch; - self.installed = [true; 2]; - self.outer_started |= ring; - if let Some(previous) = self.previous.as_mut().filter(|p| p.publication.is_none()) { - previous.publication = Some(ordinal); - } - packet.context.mark_published(); - self.announce_publication( - host, - (&fragment_slot, &tiling_slot, outer_next), - lease.free_list_generation, - policy.priority(), - &packet, - defer, - ) - } - - /// Everything that must hold before a publication changes any state: no alias work - /// pending, a healthy device epoch, room in the active ring, this slot's packet, the - /// prepared descriptors for this ordinal, a clean completion status and a registration - /// class the pair can still take. Returns the epoch, the prepared memory-cache setting - /// and the owner's policy. - fn publication_preflight( - &self, - packet: &Arc, - host: &mut impl Host, - ) -> Result<(u64, Option, crate::g17::fw::queue::Policy)> { - if self.pending_aliases.is_some() || self.kick_aliases.is_some() { - return Err(EBUSY); - } - let (epoch, recovery_state) = host.epoch()?; - if recovery_state != 0 - || self.quarantined - || !self.active.room() - || packet.completion.render_slot() != Some(self.slot) - { - return Err(EBUSY); - } - let prepared = self.prepared.as_ref().ok_or(EINVAL)?; - if !Arc::ptr_eq(&prepared.packet, packet) - || prepared.ordinal != self.ordinal - || !self.memory.owns(self.ordinal) - || !packet.context.is_current() - { - return Err(EIO); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let policy = packet.context.policy(); - // A registration of another class can only follow a rebind, which - // requires the pair to have drained. - if self.registered.is_some_and(|class| class != policy.priority()) && self.in_flight() { - return Err(EIO); - } - Ok((epoch, prepared.mcache, policy)) - } - - /// Records a failed publication on the retained active entry: the terminal status - /// (once), the deferred failure and the entry's failed flag. - fn record_publication_failure( - &mut self, - packet: &Arc, - error: Error, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - if !packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((packet.completion.status().clone(), error)); - } - defer(Deferred::FailedOwned(packet.completion.clone(), error))?; - self.active.back_mut().ok_or(EIO)?.failed = true; - Ok(()) - } - - /// Activates the pool lease and announces the retained pair to the firmware; a failure - /// here quarantines the pair with the packet still retained for recovery. - fn announce_publication( - &mut self, - host: &mut impl Host, - (fragment_slot, tiling_slot, outer_next): (&WorkSlot, &WorkSlot, [u32; 2]), - free_list_generation: u64, - priority: u8, - packet: &Arc, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - let publish = (|| { - host.activate_render_pool(self.free_list.id(), free_list_generation)?; - host.publish_retained_pair(priority, fragment_slot, tiling_slot, outer_next)?; - self.active.back_mut().ok_or(EIO)?.outer_published = true; - host.notify_render(priority)?; - host.note_submission() - })(); - if let Err(error) = publish { - self.quarantined = true; - self.active - .back() - .ok_or(EIO)? - .packet - .completion - .quarantine_render(); - self.record_publication_failure(packet, error, defer)?; - return Err(error); - } - Ok(()) - } -} - -struct Observer<'a, H> { - memory: &'a memory::Memory, - qids: [u8; 2], - host: &'a mut H, -} -impl retirement::Memory for Observer<'_, H> { - fn snapshot(&mut self, ordinal: u64) -> Result { - observation::snapshot(self.memory, ordinal) - } - - fn stamps(&mut self) -> Result<[u32; 2]> { - self.host.stamps(self.qids) - } - fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]> { - self.memory.pass(ordinal) - } -} - -impl Pair { - fn qos_owners(&self) -> [qos::Owner; 2] { - let qids = self.qids(); - let qos = self.context().qos_id(); - [ - qos::Owner { - qid: qids[0], - qos, - data_master: DataMaster::Tiling as u8, - }, - qos::Owner { - qid: qids[1], - qos, - data_master: DataMaster::Fragment as u8, - }, - ] - } - fn selected(&self, masks: Option<[u64; 2]>) -> bool { - masks.is_none_or(|masks| { - self.qids() - .into_iter() - .all(|qid| masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) - }) - } - pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { - self.active - .iter() - .any(|active| Arc::ptr_eq(&active.packet, packet)) - || self - .prepared - .as_ref() - .is_some_and(|prepared| Arc::ptr_eq(&prepared.packet, packet)) - } - pub(crate) fn take_deferred_timeout(&self) -> Option<(Arc, Error)> { - self.active.iter().find_map(|active| { - active - .packet - .completion - .take_deferred_timeout() - .map(|error| (active.packet.clone(), error)) - }) - } - - pub(crate) fn oldest_packet(&self) -> Option> { - self.active.front().map(|active| active.packet.clone()) - } - /// Sample PM feedback only after this pair's exact retirement. Busy pairs - /// keep their current capacity and let firmware request growth as needed. - pub(crate) fn host_growth_target(&mut self, packet: &Packet) -> Result> { - let Validated::Render { pass, .. } = &packet.command else { - return Err(EINVAL); - }; - let required = target::Layout::new(pass, self.clusters)?.tvb_blocks; - if self.in_flight() { - return Ok(None); - } - if !self.idle_for_context(self.context())? { - return Err(EBUSY); - } - let feedback = if self.ordinal != 0 { - self.memory.manager.automatic_target()?.unwrap_or(0) - } else { - 0 - }; - let required = required.max(feedback); - Ok((required > self.memory.manager.blocks()).then_some(required)) - } - - pub(crate) fn needs_drain(&self) -> bool { - self.quarantined - } - - pub(crate) fn oldest_spared(&self) -> Option { - self.active - .front() - .map(|active| active.packet.completion.spared()) - } - pub(crate) fn visit_owners(&self, visit: &mut dyn FnMut(&Arc)) { - for active in self.active.iter() { - visit(active.packet.completion.status()); - } - } - pub(crate) fn classify(&self, classify: &dyn Fn(&Arc) -> recovery::Class) { - for active in self.active.iter() { - active - .packet - .completion - .classify(classify(active.packet.completion.status())); - } - } - /// A slot stamp must name retained work before its execution markers are meaningful. - pub(crate) fn pass_started(&self, stamp: u64) -> Option { - const MASK: u64 = (1 << 40) - 1; - let ordinal = self.active.iter().map(|active| active.ticket.ordinal) - .find(|ordinal| ordinal.wrapping_add(1) & MASK == stamp & MASK)?; - Some(observation::snapshot(&self.memory, ordinal).ok()?.pass[0] != 0) - } - - /// Firmware is halted and all graph borrowers have joined before this copy. - pub(crate) fn publish_recovery_targets(&self) -> Result { - self.memory.graph.publish_recovery_targets() - } - - pub(crate) fn resync(&mut self) { - self.configs = [None; 2]; - for kick in &mut self.kicks { - kick.clear_parent_after_recovery(); - } - } - pub(crate) fn matches(&self, owner: u64, context: &Arc) -> Result { - Ok(self.owner == Some(owner) - && Arc::ptr_eq(self.context(), context) - && context.is_current()) - } - /// Classification controls VM poisoning. Every failed render fence settles, - /// while command memory remains owned until paired retirement or stop. - pub(crate) fn quarantine( - &mut self, - error: Error, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - self.quarantined = true; - // Successors published after recovery can still fail behind its spared - // head. Classify the whole failing group before any fence can settle. - if self.active.iter().any(|active| active.packet.completion.spared()) { - for active in self.active.iter() { - active.packet.completion.classify(recovery::Class::Spared); - } - } - for active in self.active.iter_mut() { - active.packet.completion.quarantine_render(); - if !active.failed { - defer(Deferred::FailedOwned( - active.packet.completion.clone(), - error, - ))?; - active.failed = true; - if !active.packet.completion.spared() && self.terminal.is_none() { - self.terminal = Some((active.packet.completion.status().clone(), error)); - } - } - } - Ok(()) - } - pub(crate) fn retire( - &mut self, - masks: Option<[u64; 2]>, - _epoch: u64, - state: u32, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - self.retire_front(masks, state, None, host, defer) - } - - pub(crate) fn retire_observed( - &mut self, - observed: Observed, - state: u32, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - self.retire_front(None, state, Some(observed), host, defer) - } - - fn retire_front( - &mut self, - masks: Option<[u64; 2]>, - state: u32, - observed: Option, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - if self.quarantined || state != 0 || !self.selected(masks) { - return Ok(false); - } - let qids = self.qids(); - let qos = self.qos_owners(); - let pool = self.free_list.id(); - let buffer = self.buffer_id(); - loop { - let Some(front) = self.active.front_mut() else { - return Ok(false); - }; - if !front.outer_published { - return Ok(false); - } - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) - || !self.memory.owns(front.ticket.ordinal) - { - return Err(EIO); - } - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = if let Some(observed) = observed { - if observed.ticket != front.ticket { - return Err(EIO); - } - retirement::poll_observed(&mut front.ticket, &mut observer, observed.snapshot) - } else { - retirement::poll(&mut front.ticket, &mut observer) - }; - let timestamps = match result { - Ok(retirement::Poll::Complete(timestamps)) => timestamps, - Ok(retirement::Poll::Retry | retirement::Poll::Pending) => return Ok(false), - Err(error) => { - self.quarantine(error, defer)?; - return Ok(false); - } - }; - let ordinal = front.ticket.ordinal; - let context_update = front.context_update; - self.note_retirement(ordinal, context_update); - let front = self.active.front_mut().ok_or(EIO)?; - if !front.flist_completed { - host.complete_render( - (!front.qos_completed).then_some(qos), - pool, - front.generation, - buffer, - )?; - front.qos_completed = true; - front.flist_completed = true; - } - defer(Deferred::Retired( - front.packet.completion.clone(), - Ok(timestamps), - ))?; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - return Ok(true); - } - } - - /// A closed preparation epoch invalidates only this unpublished descriptor. - /// Its completion remains available for a retry under the next epoch. - pub(crate) fn discard_prepared(&mut self, packet: &Arc) -> Result { - let Some(prepared) = self.prepared.as_ref() else { - return Ok(()); - }; - if !Arc::ptr_eq(&prepared.packet, packet) { - return Err(EBUSY); - } - self.memory.retire(prepared.ordinal)?; - self.prepared = None; - Ok(()) - } -} - -impl Pair { - /// Call before and again after servicing firmware events. FList completion - /// counters are full-width, including beyond the low-word wrap boundary. - pub(crate) fn firmware_drained(&self, host: &impl Host) -> Result { - if self.prepared.is_some() || self.active.iter().any(|a| !a.outer_published) { - return Ok(false); - } - let counters = self.free_list.counters()?; - let stamps = host.stamps(self.qids())?; - let (_, state) = host.epoch()?; - Ok(retirement::drained(self.ordinal, stamps, counters, state)) - } - - /// Account one witnessed tracker, preserving all completion ownership. - /// The caller services its pending FList release before accounting another - /// tracker and invokes release_witnessed only after that control stage. - pub(crate) fn account_drained(&mut self, host: &mut impl Host) -> Result { - use retirement::Memory; - if !self.firmware_drained(host)? { - return Ok(false); - } - let qids = self.qids(); - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let Some(active) = self.active.iter_mut().find(|a| !a.flist_completed) else { - return Ok(false); - }; - let mut observer = Observer { - memory: &self.memory, - qids, - host, - }; - let result = match retirement::poll(&mut active.ticket, &mut observer)? { - retirement::Poll::Complete(ticks) => Ok(ticks), - retirement::Poll::Pending | retirement::Poll::Retry => { - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - if !retirement::abandoned(&active.ticket, snapshot, stamps, state, true)? { - return Ok(false); - } - Err(EIO) - } - }; - let ordinal = active.ticket.ordinal; - let context_update = result.is_ok() && active.context_update; - self.note_retirement(ordinal, context_update); - let active = self - .active - .iter_mut() - .find(|active| active.ticket.ordinal == ordinal) - .ok_or(EIO)?; - host.complete_render( - (!active.qos_completed).then_some(qos), - self.free_list.id(), - active.generation, - buffer, - )?; - active.qos_completed = true; - active.flist_completed = true; - active.drained_result = Some(result); - Ok(true) - } - - /// Fresh proof after asynchronous control publication is the ownership - /// boundary. A refused deferred item leaves the precise owner available. - pub(crate) fn release_witnessed( - &mut self, - host: &impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - if self.active.iter().any(|a| !a.flist_completed) || !self.firmware_drained(host)? { - return Ok(()); - } - while let Some(active) = self.active.front() { - let result = active.drained_result.ok_or(EIO)?; - defer(Deferred::Drained(active.packet.completion.clone(), result))?; - let ordinal = active.ticket.ordinal; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - } - self.quarantined = false; - self.resync(); - Ok(()) - } - - /// The registry first polls the completed prefix through this exact Arc. - /// With successors, source policy quarantines every remaining group. A - /// single unexecuted group may use the strict ownership witness instead. - pub(crate) fn fail_after_observation( - &mut self, - packet: &Arc, - error: Error, - host: &mut impl Host, - defer: &mut impl FnMut(Deferred) -> Result, - ) -> Result { - use retirement::Memory; - if !self.active.iter().any(|a| Arc::ptr_eq(&a.packet, packet)) { - return Ok(false); - } - if !self.quarantined && self.active.len() == 1 { - let active = self.active.front().ok_or(EIO)?; - let mut observer = Observer { - memory: &self.memory, - qids: self.qids(), - host, - }; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let snapshot = observer.snapshot(active.ticket.ordinal)?; - let stamps = observer.stamps()?; - let (_, state) = observer.host.epoch()?; - if active.outer_published - && retirement::abandoned(&active.ticket, snapshot, stamps, state, false)? - { - let qos = self.qos_owners(); - let buffer = self.buffer_id(); - let active = self.active.front_mut().ok_or(EIO)?; - if !active.flist_completed { - host.complete_render( - (!active.qos_completed).then_some(qos), - self.free_list.id(), - active.generation, - buffer, - )?; - active.flist_completed = true; - active.qos_completed = true; - } - if !active.failed && !active.packet.completion.spared() { - active.packet.completion.status().record(EIO); - if self.terminal.is_none() { - self.terminal = Some((active.packet.completion.status().clone(), error)); - } - } - defer(Deferred::Retired( - active.packet.completion.clone(), - Err(EIO), - ))?; - let ordinal = active.ticket.ordinal; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - return Ok(true); - } - } - self.quarantine(error, defer)?; - Ok(true) - } - - /// Both processors have stopped before the caller reaches this method. - /// VM ownership settles before quarantined admission is rehabilitated. - pub(crate) fn stopped(&mut self, defer: &mut impl FnMut(Deferred) -> Result) -> Result { - if let Some(prepared) = self.prepared.as_ref() { - defer(Deferred::Unpublished( - prepared.packet.completion.clone(), - EIO, - ))?; - self.memory.retire(prepared.ordinal)?; - self.prepared = None; - } - while let Some(active) = self.active.front() { - defer(Deferred::Drained( - active.packet.completion.clone(), - Err(EIO), - ))?; - let ordinal = active.ticket.ordinal; - self.memory.retire(ordinal)?; - self.active.pop().ok_or(EIO)?; - } - if let Some(previous) = self.previous.as_mut() { - previous.ready = true; - } - Ok(()) - } -} - -impl Pair { - /// Commit a cold graph only after the registry revalidates its preparation - /// lease. The original PB row is restored on every later failure. - pub(crate) fn finish_install( - &mut self, - alloc: &Allocator<'_>, - descriptors: &crate::g17::object::KernelObject, - ) -> Result { - install_memory( - alloc, - descriptors, - self.slot, - &mut self.memory, - &mut self.pending_aliases, - &mut self.kick_aliases, - ) - } -} - -fn install_memory( - alloc: &Allocator<'_>, - descriptors: &crate::g17::object::KernelObject, - slot: u8, - memory: &mut memory::Memory, - pending_aliases: &mut Option>, - kick_aliases: &mut Option<[Option; 2]>, -) -> Result { - use core::sync::atomic::{fence, Ordering}; - if pending_aliases.is_none() && kick_aliases.is_none() { - return Ok(()); - } - if !memory.context.is_current() || kick_aliases.is_none() { - return Err(EIO); - } - let mut driver = KVec::new(); - driver.reserve(2, GFP_KERNEL)?; - let aliases = memory.take_persistent(pending_aliases.as_mut().ok_or(EIO)?)?; - let context = memory.context.clone(); - let vm = context.vm(); - let pool_id = u64::from(memory.lifecycle[0].predecessor); - let mut staged: Option<[u32; 4]> = None; - let offset = usize::from(memory.manager.buffer_id()) * 16; - let result: Result = (|| { - vm.install_render_pool_mappings(slot, pool_id, memory.manager.blocks(), aliases)?; - memory.installed_pool = Some(slot); - alloc.uat.flush_vm_contexts(vm)?; - let words = [ - descriptors.word(offset)?, - descriptors.word(offset + 4)?, - descriptors.word(offset + 8)?, - descriptors.word(offset + 12)?, - ]; - let previous = words.map(|word| word.load(Ordering::Relaxed)); - let pages: u32 = (memory.manager.blocks() - * crate::hw::t8140::resources::TVB_PAGES_PER_BLOCK) - .try_into()?; - let next = crate::g17::fw::buffer::reload_descriptor( - previous, - memory.manager.page_list_va(), - pages, - ); - for (word, value) in words.into_iter().zip(next) { - word.store(value, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - staged = Some(previous); - for mapping in kick_aliases.take().ok_or(EIO)?.into_iter().flatten() { - driver.push(mapping, GFP_KERNEL)?; - } - vm.append_driver_mappings(driver)?; - Ok(()) - })(); - if let Err(error) = result { - if error == ENOENT { - context.status().record(error); - } - if let Some(previous) = staged { - for (index, value) in previous.into_iter().enumerate() { - descriptors - .word(offset + index * 4)? - .store(value, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - } - vm.clear_render_pool_mappings_if_owner(slot, pool_id); - memory.installed_pool = None; - return Err(error); - } - *pending_aliases = None; - Ok(()) -} - -/// A binding prepared while the registry exclusively lends the idle pair. -/// Its frontier prevents a stale off-lock result from replacing another owner. -pub(crate) struct PreparedBinding { - previous_context: Arc, - previous_owner: Option, - ordinal: u64, - owner: u64, - context: Arc, - pool: Arc, - memory: Option>, - aliases: Option>, - kicks: Option<[Option; 2]>, - initial_packet: u64, -} - -/// The registry retains displaced backing until this exact context's release -/// is acknowledged. A warm binding retains the old context without its graph. -pub(crate) struct RetiredBinding { - memory: Option>, - context: Arc, - pool: Arc, - buffer_id: u8, - publication: Option, - ready: bool, -} -impl RetiredBinding { - pub(crate) fn context(&self) -> &Arc { - &self.context - } - pub(crate) fn pool(&self) -> &Arc { - &self.pool - } - pub(crate) fn buffer_id(&self) -> Option { - self.memory.as_ref().map(|_| self.buffer_id) - } -} - -impl Pair { - fn note_retirement(&mut self, ordinal: u64, context_update: bool) { - if !context_update { - return; - } - self.context_update = false; - if let Some(previous) = self.previous.as_mut() { - if previous - .publication - .is_some_and(|published| published <= ordinal) - { - previous.ready = true; - } - } - } - pub(crate) fn take_retired_binding(&mut self) -> Option { - if self.displaced.is_some() { - return self.displaced.take(); - } - if self.previous.as_ref().is_some_and(|p| p.ready) { - self.previous.take() - } else { - None - } - } - pub(crate) fn idle_for_context(&self, context: &Arc) -> Result { - if !Arc::ptr_eq(context, self.context()) { - return Ok(true); - } - if self.prepared.is_some() || self.in_flight() || self.quarantined { - return Ok(false); - } - if self.ordinal == 0 { - return Ok(true); - } - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - Ok(self - .memory - .graph - .cursors()? - .into_iter() - .all(|c| c.done == c.read && c.read == c.write)) - } - pub(crate) fn reusable(&self) -> Result { - let previous_ready = self - .previous - .as_ref() - .is_none_or(|p| p.memory.is_none() && self.context_update && !p.ready); - Ok(self.owner.is_none() - && self.displaced.is_none() - && previous_ready - && self.idle_for_context(self.context())?) - } - /// Logical close leaves the physical channel identity and backing retained. - pub(crate) fn release_owner(&mut self) { - self.owner = None; - } - #[inline(never)] - pub(crate) fn prepare_bind( - &mut self, - alloc: &Allocator<'_>, - pool: &Arc, - owner: u64, - context: Arc, - free_list: Arc, - metrics: Option, - buffer_id: Option<(u8, u64)>, - pm_generation: Option, - first: &Arc, - ) -> Result { - if !self.reusable()? - || owner == 0 - || !Arc::ptr_eq(&context, &first.context) - || !context.is_current() - { - return Err(EBUSY); - } - alloc.uat.vm_context_mask(context.vm())?; - let same_vm = Arc::ptr_eq(self.context().status(), context.status()); - let (memory, aliases, kicks) = if same_vm { - if metrics.is_some() || buffer_id.is_some() || pm_generation.is_some() { - return Err(EINVAL); - } - (None, None, None) - } else { - let metrics = metrics.ok_or(EINVAL)?; - let buffer_id = buffer_id.ok_or(EINVAL)?; - let pm_generation = pm_generation.ok_or(EINVAL)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; - let layout = target::Layout::new(pass, self.clusters)?; - let aliases = [ - self.kicks[0].prepare_client_alias(context.vm())?, - self.kicks[1].prepare_client_alias(context.vm())?, - ]; - let (memory, persistent) = memory::Memory::new( - alloc, - pool, - context.clone(), - self.qids(), - &layout, - metrics, - buffer_id, - pm_generation, - self.ordinal, - first.completion.work_state()?.word(), - )?; - (Some(memory), Some(persistent), Some(aliases)) - }; - Ok(PreparedBinding { - previous_context: self.context().clone(), - previous_owner: self.owner, - ordinal: self.ordinal, - owner, - context, - pool: free_list, - memory, - aliases, - kicks, - initial_packet: first.id, - }) - } - pub(crate) fn bind_owner( - &mut self, - binding: &mut PreparedBinding, - alloc: &Allocator<'_>, - descriptors: &crate::g17::object::KernelObject, - ) -> Result { - if !self.reusable()? - || binding.ordinal != self.ordinal - || binding.previous_owner != self.owner - || !Arc::ptr_eq(&binding.previous_context, self.context()) - || !binding.context.is_current() - { - return Err(EBUSY); - } - let foreign = binding.memory.is_some(); - if let Some(memory) = binding.memory.as_mut() { - install_memory( - alloc, - descriptors, - self.slot, - memory, - &mut binding.aliases, - &mut binding.kicks, - )?; - } else { - self.memory.set_owner(self.qids(), &binding.context)?; - } - let old_context = self.context().clone(); - let old_buffer = self.buffer_id(); - let old_pool = core::mem::replace(&mut self.free_list, binding.pool.clone()); - let old_memory = binding - .memory - .take() - .map(|memory| core::mem::replace(&mut self.memory, memory)); - let retired = RetiredBinding { - memory: old_memory, - context: old_context, - pool: old_pool, - buffer_id: old_buffer, - publication: None, - ready: foreign, - }; - if foreign { - self.displaced = Some(retired); - } else if self.previous.is_none() { - self.previous = Some(retired); - } - self.memory.context = binding.context.clone(); - self.owner = Some(binding.owner); - self.initial_packet = binding.initial_packet; - self.context_update = true; - if foreign { - self.cursors = [0; 2]; - self.configs = [None; 2]; - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render/dependencies.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render/dependencies.rs deleted file mode 100644 index be1d9fc9c902ce..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render/dependencies.rs +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Render predecessors retained while a physical pair is prepared off-lock. - -use super::Pair; -use crate::g17::{ - dependency::Frontier, - fw::kick::KickDependency, - job::{self, Packet}, - status::VmStatus, -}; -use crate::hw::t8140::queues::RENDER_DEPTH; -use kernel::{prelude::*, sync::Arc}; - -struct Predecessor { - packet: Arc, - word: u64, -} - -/// A loan of the graph cannot hide work already published on that pair. Fence -/// status remains live through the retained Packet, even while its storage is away. -pub(crate) struct Dependencies { - owner: Option, - entries: [Option; RENDER_DEPTH as usize], -} - -fn include( - frontier: &mut Frontier, - status: &Arc, - prefix: Option, - packet: &Arc, - word: u64, -) -> Result { - if !Arc::ptr_eq(packet.completion.status(), status) { - return Ok(()); - } - if !job::Order::contains(prefix, packet.order.sequence) { - return Ok(()); - } - frontier.include( - prefix, - packet.order.sequence, - job::fence_status(&packet.completion.scheduler_fence()), - word, - ) -} - -impl Dependencies { - pub(crate) fn contains_packet(&self, packet: &Arc) -> bool { - self.entries - .iter() - .flatten() - .any(|entry| Arc::ptr_eq(&entry.packet, packet)) - } - - pub(crate) fn collect( - &self, - owner: u64, - status: &Arc, - prefix: Option, - frontier: &mut Frontier, - ) -> Result { - if self.owner != Some(owner) { - return Ok(()); - } - for entry in self.entries.iter().flatten() { - include(frontier, status, prefix, &entry.packet, entry.word)?; - } - Ok(()) - } -} - -impl Pair { - /// The fixed tracker bounds this snapshot; no allocation or packet can be - /// lost between capture and the registry moving the pair under the same lock. - pub(crate) fn dependency_snapshot(&self) -> Dependencies { - let qid = self.qids()[1]; - let mut active = self.active.iter(); - Dependencies { - owner: self.owner, - entries: core::array::from_fn(|_| { - active.next().map(|entry| Predecessor { - packet: entry.packet.clone(), - word: KickDependency::new(qid, entry.kicks[1]) - .map_or(u64::MAX, KickDependency::word), - }) - }), - } - } - - pub(crate) fn collect_dependencies( - &self, - owner: u64, - status: &Arc, - prefix: Option, - frontier: &mut Frontier, - ) -> Result { - if self.owner != Some(owner) { - return Ok(()); - } - let qid = self.qids()[1]; - for entry in self.active.iter() { - let word = - KickDependency::new(qid, entry.kicks[1]).map_or(u64::MAX, KickDependency::word); - include(frontier, status, prefix, &entry.packet, word)?; - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render/memory.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render/memory.rs deleted file mode 100644 index 85319585b3d135..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render/memory.rs +++ /dev/null @@ -1,723 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Storage of one render pair and its per-command generations. Client aliases -//! precede their backing owners so teardown cannot leave a dangling GPU view. - -use super::retirement::Cursors; -use crate::g17::{ - buffer::{self, target}, - context::Context, - fw::{ - self, queue, - render::{self as abi, storage}, - }, - object::{Allocator, CpuMap, Family, KernelObject, Pool, PooledObject}, -}; -use crate::{ - hw::t8140::{dynamic, kernel_window, low, queues, resources as cfg}, - mmu, -}; -use core::sync::atomic::Ordering; -use kernel::{prelude::*, sync::Arc}; - -pub(super) const RECORDS: usize = 32; -pub(super) const TA_BYTES: usize = 0x14000; -pub(super) const FRAGMENT_BYTES: usize = 0x48000; -pub(super) const GRAPH_SIZE: usize = 0x1c000; -pub(super) const POINTERS: [usize; 2] = [0x9000, 0x9080]; -pub(super) const ITEMS: [usize; 2] = [0xc000, 0x10000]; -pub(super) const ANNOUNCE_STRIDE: usize = 0x80; -pub(super) const ANNOUNCE: [usize; 2] = [0x17000, 0x1b000]; -pub(super) const JOB_LIST: usize = 0xa000; -pub(super) const CONFIG: [usize; 2] = [0, 0x4000]; -pub(super) const CONFIG_STRIDE: usize = 0x180; -const DEPTH: usize = queues::RENDER_DEPTH as usize; -/// Offsets of the priority pair and the policy word in a queue record. -const PRIORITY: usize = core::mem::offset_of!(queue::QueueRecord, priority); -const POLICY: usize = core::mem::offset_of!(queue::QueueRecord, policy); - -struct Aliases { - client: mmu::KernelMapping, - _global: mmu::KernelMapping, -} -impl Aliases { - fn new( - object: &KernelObject, - alloc: &Allocator<'_>, - vm: &mmu::Vm, - source: core::ops::Range, - ) -> Result { - let mut range = dynamic::LOWER; - let size = (source.end - source.start) as u64; - loop { - // Descriptor aliases are read-only to GPU consumers; the canonical - // firmware mapping remains writable for command state updates. - let global = object.map_range( - alloc.uat.kernel_lower_vm(), - source.clone(), - range.clone(), - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RO, - 0, - )?; - let address = global.iova(); - let end = address.checked_add(size).ok_or(EOVERFLOW)?; - match object.map_range( - vm, - source.clone(), - address..end, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RO, - 0, - ) { - Ok(client) => { - return Ok(Self { - client, - _global: global, - }) - } - Err(error) if error == ENOSPC && end < range.end => { - drop(global); - range.start = end; - } - Err(error) => return Err(error), - } - } - } -} - -/// Stable queue records and rotating descriptors/configuration records. -pub(super) struct Graph { - aliases: [Aliases; 2], - pub(super) descriptors: PooledObject, - pub(super) queues: PooledObject, - pub(super) config: PooledObject, - pub(super) support: KernelObject, - pub(super) owners: [Option; RECORDS], -} -impl Graph { - pub(super) fn new( - alloc: &Allocator<'_>, - pool: &Arc, - context: &Context, - qids: [u8; 2], - ) -> Result { - let prot = mmu::PROT_GPU_FW_SHARED_RW; - let descriptors = pool.allocate( - alloc, - Family::Descriptor, - TA_BYTES + FRAGMENT_BYTES, - TA_BYTES + FRAGMENT_BYTES, - prot, - CpuMap::WriteBack, - )?; - let ta = Aliases::new(&descriptors, alloc, context.vm(), 0..TA_BYTES)?; - let fragment = Aliases::new( - &descriptors, - alloc, - context.vm(), - TA_BYTES..TA_BYTES + FRAGMENT_BYTES, - )?; - let mut queues = pool.allocate( - alloc, - Family::QueueState, - GRAPH_SIZE, - 0x8000, - prot, - CpuMap::WriteCombined, - )?; - let config = pool.allocate( - alloc, - Family::QueueConfig, - 0x8000, - 0x8000, - prot, - CpuMap::WriteBack, - )?; - let support = alloc.kernel( - storage::SUPPORT_SIZE, - mmu::UAT_PGSZ as u64, - prot, - CpuMap::WriteCombined, - )?; - let address = queues.gpu_va(); - for stage in 0..2 { - queues.write( - qids[stage] as usize * size_of::(), - queue::QueueRecord::new(&queue::QueueRecordArgs { - pointers_va: address + POINTERS[stage] as u64, - items_va: address + ITEMS[stage] as u64, - job_list_va: context.work_head_va(), - scheduler_va: context.scheduler_va(), - owner_pid: context.owner_pid(), - policy: context.policy(), - }), - )?; - queues.initialize::(POINTERS[stage], |view| { - *view = queue::PointerBlock::new(); - Ok(()) - })?; - } - queues.write(JOB_LIST, queue::JobListHead::new(address + JOB_LIST as u64))?; - Ok(Self { - aliases: [ta, fragment], - descriptors, - queues, - config, - support, - owners: [None; RECORDS], - }) - } - pub(super) fn offset(stage: usize, ordinal: u64) -> usize { - let slot = ordinal as usize % RECORDS; - if stage == 0 { - slot * abi::TA_DESCRIPTOR_SIZE - } else { - TA_BYTES + slot * abi::FRAGMENT_SLOT_SIZE - } - } - pub(super) fn descriptor_va(&self, stage: usize, ordinal: u64) -> u64 { - self.descriptors.gpu_va() + Self::offset(stage, ordinal) as u64 - } - pub(super) fn descriptor_client(&self, stage: usize, ordinal: u64) -> u64 { - let stride = if stage == 0 { - abi::TA_DESCRIPTOR_SIZE - } else { - abi::FRAGMENT_SLOT_SIZE - }; - self.aliases[stage].client.iova() + (ordinal as usize % RECORDS * stride) as u64 - } - pub(super) fn publish_recovery_targets(&self) -> Result { - for at in POINTERS { - let producer = self.queues.word(at + queue::PointerBlock::PRODUCER)? - .load(Ordering::Relaxed); - self.queues.word(at + queue::PointerBlock::RECOVERY_TARGET)? - .store(producer, Ordering::Relaxed); - } - Ok(()) - } - - pub(super) fn cursors(&self) -> Result<[Cursors; 2]> { - let mut values = [Cursors { - done: 0, - read: 0, - write: 0, - }; 2]; - for stage in 0..2 { - let at = POINTERS[stage]; - values[stage] = Cursors { - done: self.queues.word(at)?.load(Ordering::Relaxed), - read: self.queues.word(at + 0x30)?.load(Ordering::Relaxed), - write: self.queues.word(at + 0x40)?.load(Ordering::Relaxed), - }; - } - Ok(values) - } - pub(super) fn claim(&mut self, ordinal: u64) -> Result { - let owner = &mut self.owners[ordinal as usize % RECORDS]; - if owner.is_some_and(|old| old != ordinal) { - return Err(EBUSY); - } - *owner = Some(ordinal); - Ok(()) - } - pub(super) fn owns(&self, ordinal: u64) -> bool { - self.owners[ordinal as usize % RECORDS] == Some(ordinal) - } - pub(super) fn release(&mut self, ordinal: u64) -> Result { - if !self.owns(ordinal) { - return Err(EIO); - } - self.owners[ordinal as usize % RECORDS] = None; - Ok(()) - } -} - -struct Compact { - mapping: Option, - address: u64, - _backing: KernelObject, -} -impl Compact { - fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, size: usize) -> Result { - let backing = KernelObject::backing(alloc.dev, size, CpuMap::WriteCombined)?; - let mapping = buffer::compact_alias( - &backing, - vm, - 0..size, - cfg::TVB_PAGE, - mmu::PROT_GPU_SHARED_RW, - mmu::UAT_PGSZ, - )?; - Ok(Self { - address: mapping.iova(), - mapping: Some(mapping), - _backing: backing, - }) - } -} - -pub(super) struct Generation { - deflake: mmu::KernelMapping, - fragment_client: mmu::KernelMapping, - state: KernelObject, - fragment: KernelObject, - pub(super) ordinal: Option, -} -impl Generation { - fn allocate_state(alloc: &Allocator<'_>, vm: &mmu::Vm) -> Result { - KernelObject::new( - alloc.dev, - vm, - crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, cfg::TVB_PAGE), - storage::STATE_SIZE, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - ) - } - fn new(alloc: &Allocator<'_>, vm: &mmu::Vm, state: KernelObject) -> Result> { - let deflake = buffer::compact_alias( - &state, - vm, - storage::DEFLAKE..storage::DEFLAKE + mmu::UAT_PGSZ, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - mmu::UAT_PGSZ, - )?; - let fragment = alloc.kernel( - mmu::UAT_PGSZ, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let fragment_client = buffer::compact_alias( - &fragment, - vm, - 0..mmu::UAT_PGSZ, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RW, - mmu::UAT_PGSZ, - )?; - KBox::new( - Self { - deflake, - fragment_client, - state, - fragment, - ordinal: None, - }, - GFP_KERNEL, - ) - .map_err(Into::into) - } - pub(super) fn prepare(&mut self, ordinal: u64, pass: &abi::RenderPass) -> Result { - if self.ordinal.is_some_and(|old| old != ordinal) { - return Err(EBUSY); - } - let viewport = storage::Viewport::new(pass.width, pass.height)?; - self.state.bytes_mut()[storage::AUXILIARY..storage::AUXILIARY + storage::AUXILIARY_SIZE] - .fill(0); - let destination = self - .state - .pointer(storage::DEFLAKE + 0x900, size_of::())? - .cast::(); - if destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); - } - // SAFETY: The complete typed integer view is in the retained CPU mapping - // and naturally aligned. This command generation is exclusively owned - // and unpublished; no prior command can access it until reassigned. - unsafe { viewport.apply(&mut *destination) }; - self.fragment.bytes_mut().fill(0); - self.ordinal = Some(ordinal); - Ok(()) - } - pub(super) fn addresses(&self) -> [u64; 5] { - [ - self.deflake.iova(), - self.state.gpu_va() + storage::TA_STATUS as u64, - self.fragment_client.iova(), - self.fragment.gpu_va(), - // The retained state already maps these pages into the client VM. - // The auxiliary framebuffer accepts a full GPU address; reusing this - // view avoids exhausting the fixed 4 MiB alias arena across pairs. - self.state.gpu_va() + storage::AUXILIARY as u64, - ] - } -} - -pub(super) struct Memory { - _fragment_view: mmu::KernelMapping, - pub(super) graph: Graph, - scratch: Compact, - discard: Compact, - pub(super) target: KBox, - generations: [Option>; DEPTH], - generation_count: usize, - pub(super) timestamps: KernelObject, - pub(super) manager: buffer::Manager, - pub(super) lifecycle: [fw::ObjectIds; 2], - // The extra client views retain their allocation positions even though - // kick entries consume the register arrays embedded in the descriptors. - registers: KernelObject, - pub(super) context: Arc, - pub(super) installed_pool: Option, -} -impl Memory { - #[inline(never)] - pub(super) fn new( - alloc: &Allocator<'_>, - pool: &Arc, - context: Arc, - qids: [u8; 2], - layout: &target::Layout, - metrics: buffer::MetricsLease, - buffer_id: (u8, u64), - pm_generation: u64, - ordinal: u64, - word: u32, - ) -> Result<(KBox, KVec)> { - let storage = KBox::::new_uninit(GFP_KERNEL)?; - let vm = context.vm(); - let mut graph = Graph::new(alloc, pool, &context, qids)?; - let mut target = target::Target::new(alloc.dev, vm, layout)?; - target.prepare(ordinal, layout)?; - let state = Generation::allocate_state(alloc, vm)?; - let scratch = Compact::new(alloc, vm, cfg::SCENE_SIZE as usize)?; - let discard = Compact::new(alloc, vm, cfg::DISCARD_SIZE as usize)?; - let (tvb, mappings) = buffer::Tvb::new(alloc.dev, vm, layout.tvb_blocks)?; - let count = DEPTH; - let initial = if ordinal == 0 { - 0 - } else { - (ordinal ^ 1) as usize % count - }; - let mut generations = core::array::from_fn(|_| None); - generations[initial] = Some(Generation::new(alloc, vm, state)?); - for offset in 1..count { - let index = (initial + offset) % count; - let state = Generation::allocate_state(alloc, vm)?; - generations[index] = Some(Generation::new(alloc, vm, state)?); - } - let timestamps = alloc.kernel( - mmu::UAT_PGSZ, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_FW_SHARED_RW, - CpuMap::WriteCombined, - )?; - let manager = buffer::Manager::new( - alloc, - vm, - tvb, - metrics, - buffer::ManagerArgs { - scratch: scratch.address, - discard: discard.address, - buffer_id: buffer_id.0, - generation: pm_generation, - first_ordinal: ordinal, - }, - )?; - let predecessor = crate::g17::ids::OBJECTS.object()?; - let (fragment, tiling) = crate::g17::ids::OBJECTS.render_pair(predecessor)?; - let support_va = graph.support.gpu_va(); - graph.support.initialize::(0, |view| { - view.init( - support_va, - alloc.uat.geometry().kernel_range().start + kernel_window::INDEX, - low::RENDER_RUN_LIST, - low::RENDER_PAGE_LIST, - word, - ) - })?; - let registers = KernelObject::new( - alloc.dev, - vm, - crate::g17::object::Placement::In(dynamic::CLIENT_LOWER, mmu::UAT_PGSZ as u64), - mmu::UAT_PGSZ, - mmu::PROT_GPU_SHARED_RO, - CpuMap::WriteCombined, - )?; - let fragment_view = graph.descriptors.map_range( - vm, - TA_BYTES..TA_BYTES + mmu::UAT_PGSZ, - dynamic::CLIENT_LOWER, - mmu::UAT_PGSZ as u64, - mmu::PROT_GPU_SHARED_RO, - 0, - )?; - Ok(( - storage.write_init(kernel::try_init!(Self { - graph, - scratch, - discard: discard, - target, - generations, - generation_count: count, - timestamps, - manager, - lifecycle: [tiling, fragment], - _fragment_view: fragment_view, - registers: registers, - context, - installed_pool: None, - }))?, - mappings, - )) - } - pub(super) fn generation(&mut self, ordinal: u64) -> Result<&mut Generation> { - self.generations[ordinal as usize % self.generation_count] - .as_deref_mut() - .ok_or(EIO) - } - pub(super) fn owns(&self, ordinal: u64) -> bool { - self.graph.owns(ordinal) - && self.generations[ordinal as usize % self.generation_count] - .as_ref() - .is_some_and(|generation| generation.ordinal == Some(ordinal)) - } - pub(super) fn pass(&self, ordinal: u64) -> Result<[u64; 2]> { - if !self.owns(ordinal) { - return Err(EIO); - } - let offset = ordinal as usize % RECORDS * 16; - Ok([ - self.timestamps.dword(offset)?.load(Ordering::Relaxed), - self.timestamps.dword(offset + 8)?.load(Ordering::Relaxed), - ]) - } - pub(super) fn owners(&self) -> [Option; DEPTH] { - core::array::from_fn(|index| self.generations[index].as_ref().and_then(|g| g.ordinal)) - } - pub(super) fn retire(&mut self, ordinal: u64) -> Result { - if !self.owns(ordinal) { - return Err(EIO); - } - self.target.retire(ordinal)?; - self.manager.retire_scene(ordinal)?; - self.graph.release(ordinal)?; - self.generation(ordinal)?.ordinal = None; - Ok(()) - } -} - -const _: () = { - assert!(abi::TA_DESCRIPTOR_SIZE * RECORDS <= TA_BYTES); - assert!(abi::FRAGMENT_SLOT_SIZE * RECORDS <= FRAGMENT_BYTES); - assert!(RECORDS * CONFIG_STRIDE <= 0x4000); -}; - -impl Memory { - pub(super) fn write_descriptors( - &mut self, - args: &abi::RenderArgs<'_>, - attachments: &[fw::kick::WriteRange], - context: u8, - ) -> Result> { - let tiling = self - .graph - .descriptors - .pointer( - Graph::offset(0, args.ordinal), - size_of::(), - )? - .cast::(); - let fragment = self - .graph - .descriptors - .pointer( - Graph::offset(1, args.ordinal), - size_of::(), - )? - .cast::(); - if tiling.align_offset(align_of::()) != 0 - || fragment.align_offset(align_of::()) != 0 - { - return Err(EINVAL); - } - // SAFETY: Both complete integer-only views lie in the retained mapping, - // are aligned and disjoint. The full ordinal has an exclusive slot - // claim and no producer refers to this command yet. - unsafe { - (&mut *tiling).write(args)?; - let fragment = &mut *fragment; - fragment.write(args)?; - self.registers.bytes_mut().fill(0); - for (index, array) in fragment.descriptor.arrays.iter().enumerate() { - for (entry, value) in array - .entries - .get(..(array.size & 0xffff) as usize) - .ok_or(EINVAL)? - .iter() - .enumerate() - { - self.registers.write( - index * size_of::() - + entry * size_of::(), - *value, - )?; - } - } - fragment.set_mcache( - args.fragment.descriptor_va, - args.resources.aux_fb_va, - attachments, - context, - ) - } - } -} - -impl Memory { - pub(super) fn completion_links( - &mut self, - ordinal: u64, - kicks: [fw::kick::KickTimestamp; 2], - payload: u8, - fragment_busy: bool, - ) -> Result { - if !self.owns(ordinal) { - return Err(EIO); - } - let tiling = self - .graph - .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? - .cast::(); - let fragment = self - .graph - .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? - .cast::(); - // SAFETY: The complete views were bounds/alignment checked during - // preparation. The unchanged full ordinal still exclusively owns both - // slots and neither descriptor has been published. - unsafe { - (&mut *tiling).set_completion_link(kicks[0], kicks[1], payload, fragment_busy); - (&mut *fragment).set_completion_link(kicks[1], payload); - } - Ok(()) - } -} - -impl Memory { - /// Preserve the initial mapping order while moving VM-lifetime mappings - /// out of the graph. Reserve all host storage before moving any mapping. - pub(super) fn take_persistent( - &mut self, - tvb: &mut KVec, - ) -> Result> { - if self.scratch.mapping.is_none() - || self.discard.mapping.is_none() - || self.registers.gpu_va() == 0 - { - return Err(EIO); - } - let mut aliases = KVec::new(); - aliases.reserve(tvb.len().checked_add(3).ok_or(EOVERFLOW)?, GFP_KERNEL)?; - aliases.push(self.scratch.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; - aliases.push(self.discard.mapping.take().ok_or(EIO)?, GFP_KERNEL)?; - aliases.push(self.registers.take_mapping()?, GFP_KERNEL)?; - for mapping in core::mem::take(tvb) { - aliases.push(mapping, GFP_KERNEL)?; - } - Ok(aliases) - } -} - -impl Memory { - pub(super) fn bind_work_slot(&mut self, ordinal: u64, qids: [u8; 2], slot: u32) -> Result { - if slot >= 4 || !self.owns(ordinal) { - return Err(EINVAL); - } - let queues = [ - self.graph - .queues - .word(usize::from(qids[1]) * size_of::() + 0x38)?, - self.graph - .queues - .word(usize::from(qids[0]) * size_of::() + 0x38)?, - ]; - let ta = self - .graph - .descriptors - .pointer(Graph::offset(0, ordinal), size_of::())? - .cast::(); - let fragment = self - .graph - .descriptors - .pointer(Graph::offset(1, ordinal), size_of::())? - .cast::(); - for word in queues { - word.store(slot, Ordering::Relaxed); - } - // SAFETY: The generation owns these prepared, unpublished POD views. - // Packed fields are assigned by value without an unaligned reference. - unsafe { - (*ta).unk_8be = slot; - (*fragment).descriptor.unk_2158 = slot; - } - core::sync::atomic::fence(Ordering::Release); - Ok(()) - } - - /// Retargets both retired queue records to `context`. A context of another - /// scheduling profile also replaces the priority and policy words, after - /// the owner fields; the same profile leaves them untouched. - pub(super) fn set_owner(&self, qids: [u8; 2], context: &Context) -> Result { - let offsets = qids.map(|qid| usize::from(qid) * size_of::()); - let base = self.graph.queues.pointer(0, GRAPH_SIZE)?; - // Check every field before the first update, retaining the required - // job-head pair, PID pair, then packed scheduler-pointer pair order. - for at in offsets { - self.graph.queues.pointer(at + 0x10, 8)?; - self.graph.queues.pointer(at + 0x48, 4)?; - self.graph.queues.pointer(at + 0x9c, 8)?; - self.graph.queues.pointer(at + PRIORITY, 8)?; - self.graph.queues.pointer(at + POLICY, 4)?; - } - let policy = context.policy(); - // SAFETY: Both queues are retired before retargeting. Complete byte - // views were checked above; packed scheduler words use unaligned stores. - unsafe { - for at in offsets { - base.add(at + 0x10) - .cast::() - .write_unaligned(context.work_head_va()); - } - for at in offsets { - base.add(at + 0x48) - .cast::() - .write_unaligned(context.owner_pid()); - } - for at in offsets { - base.add(at + 0x9c) - .cast::() - .write_unaligned(context.scheduler_va()); - } - for at in offsets { - let words = base.add(at + PRIORITY).cast::<[u32; 2]>(); - let priority = [u32::from(policy.priority()); 2]; - if words.read_unaligned() != priority { - words.write_unaligned(priority); - } - let word = base.add(at + POLICY).cast::(); - if word.read_unaligned() != u32::from(policy.policy()) { - word.write_unaligned(u32::from(policy.policy())); - } - } - } - Ok(()) - } -} - -impl Drop for Memory { - fn drop(&mut self) { - if let Some(slot) = self.installed_pool { - self.context.vm().clear_render_pool_mappings_if_owner( - slot, - u64::from(self.lifecycle[0].predecessor), - ); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render/observation.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render/observation.rs deleted file mode 100644 index 3e853115e4f75a..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render/observation.rs +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Read-only cursor and timestamp snapshots while a physical pair is borrowed. -//! Final stamps, timestamp refresh and ownership changes belong to commit. - -use super::{memory, retirement, Pair}; -use crate::g17::job::Packet; -use core::sync::atomic::{fence, Ordering}; -use kernel::{prelude::*, sync::Arc}; - -pub(crate) struct Observed { - pub(super) ticket: retirement::Ticket, - pub(super) snapshot: retirement::Snapshot, -} - -impl Pair { - /// Reject incomplete event/stamp prefixes before lending the graph. This - /// is only a prefilter; commit repeats the final stamp and timestamp reads. - pub(crate) fn observation_packet( - &self, - masks: [u64; 2], - stamps: [u32; 2], - state: u32, - ) -> Result>> { - if self.quarantined || self.prepared.is_some() || state != 0 || !self.selected(Some(masks)) - { - return Ok(None); - } - let Some(front) = self.active.front() else { - return Ok(None); - }; - if !front.outer_published { - return Ok(None); - } - let published = front.ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - if !stamps - .into_iter() - .all(|stamp| retirement::stamp_covers(stamp, published)) - { - return Ok(None); - } - Ok(Some(front.packet.clone())) - } - - pub(crate) fn observe_front(&self) -> Result { - let front = self.active.front().ok_or(EIO)?; - if !Arc::ptr_eq(&front.packet.context, &self.memory.context) { - return Err(EIO); - } - let ticket = front.ticket; - for _ in 0..2 { - fence(Ordering::Acquire); - let snapshot = snapshot(&self.memory, ticket.ordinal)?; - if retirement::observe(&ticket, snapshot).is_ok() { - return Ok(Observed { ticket, snapshot }); - } - } - Err(EAGAIN) - } -} - -pub(super) fn snapshot(memory: &memory::Memory, ordinal: u64) -> Result { - if !memory.owns(ordinal) { - return Err(EIO); - } - let cursors = memory.graph.cursors()?; - // Sample the private head before the logical shared head, whose membership - // may include other pairs. Individual completion does not require it empty. - let _first = memory - .graph - .queues - .dword(memory::JOB_LIST)? - .load(Ordering::Relaxed); - let _last = memory - .graph - .queues - .dword(memory::JOB_LIST + 8)? - .load(Ordering::Relaxed); - let head = memory.context.work_head()?; - Ok(retirement::Snapshot { - cursors, - job_list_empty: head == [0, memory.context.work_head_va()], - pass: memory.pass(ordinal)?, - }) -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render/parked.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render/parked.rs deleted file mode 100644 index f712bb7b1b87dd..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render/parked.rs +++ /dev/null @@ -1,243 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Installed kick storage retained after a logical scheduler release is acknowledged. - -use super::{install_memory, memory, target, Pair, RetiredBinding, Tracker}; -use crate::g17::{ - buffer, - command::Validated, - context::Context, - freelist::RenderPool, - job::Packet, - kick, - object::{Allocator, KernelObject, Pool}, -}; -use crate::mmu; -use kernel::{prelude::*, sync::Arc}; - -/// The empty physical pair retains installed addresses and the last scheduler page, -/// without retaining that scheduler's execution context or client root. A prior -/// context awaiting an executed replacement keeps its separate retirement owner. -pub(crate) struct ParkedPair { - kicks: Option<[kick::Queue; 2]>, - _scheduler: Arc, - ids: [kick::Id; 2], - slot: u8, - ordinal: u64, - clusters: u32, - descriptor_flags: [u32; 2], - installed: [bool; 2], - recovery_generation: u64, - registered: Option, - outer_started: u8, - previous: Option, -} - -/// All new graph ownership stays separate from the installed kick storage until commit. -pub(crate) struct Prepared { - memory: Option>, - pool: Arc, - aliases: Option>, - kicks: Option<[Option; 2]>, - ids: [kick::Id; 2], - slot: u8, - ordinal: u64, - owner: u64, - initial_packet: u64, - installed: bool, -} - -impl Pair { - /// The registry also checks the shared free-list release and scheduler-release witness. - /// Drain displaced owners before this preflight; they have their own logical releases. - pub(crate) fn can_park(&self, context: &Arc) -> Result { - Ok(Arc::ptr_eq(context, self.context()) - && self.owner.is_none() - && self - .previous - .as_ref() - .is_none_or(|previous| previous.memory.is_none() && !previous.ready) - && self.displaced.is_none() - && self.pending_aliases.is_none() - && self.kick_aliases.is_none() - && self.idle_for_context(context)?) - } - - /// Consumes the pair only after can_park and all collector capacity checks succeeded - /// under the same device mutex. Moving these owners performs no firmware operation. - pub(crate) fn park(self) -> (ParkedPair, RetiredBinding) { - let context = self.memory.context.clone(); - let scheduler = context.scheduler_owner(); - let buffer_id = self.buffer_id(); - let parked = ParkedPair { - ids: self.kicks.each_ref().map(|queue| queue.id()), - kicks: Some(self.kicks), - _scheduler: scheduler, - slot: self.slot, - ordinal: self.ordinal, - clusters: self.clusters, - descriptor_flags: self.descriptor_flags, - installed: self.installed, - recovery_generation: self.recovery_generation, - registered: self.registered, - outer_started: self.outer_started, - previous: self.previous, - }; - let retired = RetiredBinding { - memory: Some(self.memory), - context, - pool: self.free_list, - buffer_id, - publication: None, - ready: true, - }; - (parked, retired) - } -} - -impl ParkedPair { - pub(crate) fn qids(&self) -> [u8; 2] { - self.ids.map(kick::Id::qid) - } - - #[inline(never)] - pub(crate) fn prepare( - &self, - alloc: &Allocator<'_>, - pool: &Arc, - owner: u64, - context: Arc, - free_list: Arc, - metrics: buffer::MetricsLease, - buffer_id: (u8, u64), - pm_generation: u64, - first: &Arc, - ) -> Result { - if owner == 0 || !Arc::ptr_eq(&context, &first.context) || !context.is_current() { - return Err(EINVAL); - } - let kicks = self.kicks.as_ref().ok_or(EBUSY)?; - let Validated::Render { pass, .. } = &first.command else { - return Err(EINVAL); - }; - let word = first.completion.work_state()?.word(); - if word == u32::MAX { - return Err(EINVAL); - } - alloc.uat.vm_context_mask(context.vm())?; - let layout = target::Layout::new(pass, self.clusters)?; - let aliases = [ - kicks[0].prepare_client_alias(context.vm())?, - kicks[1].prepare_client_alias(context.vm())?, - ]; - let (memory, persistent) = memory::Memory::new( - alloc, - pool, - context, - self.qids(), - &layout, - metrics, - buffer_id, - pm_generation, - self.ordinal, - word, - )?; - Ok(Prepared { - memory: Some(memory), - pool: free_list, - aliases: Some(persistent), - kicks: Some(aliases), - ids: self.ids, - slot: self.slot, - ordinal: self.ordinal, - owner, - initial_packet: first.id, - installed: false, - }) - } - - fn matches(&self, prepared: &Prepared) -> bool { - self.kicks.is_some() - && self.ids == prepared.ids - && self.slot == prepared.slot - && self.ordinal == prepared.ordinal - && prepared - .memory - .as_ref() - .is_some_and(|memory| memory.context.is_current()) - } - - /// All fallible graph installation precedes taking the installed kicks. Keep the empty - /// parked owner until the caller drops it off-lock, including its old scheduler page. - pub(crate) fn commit( - &mut self, - prepared: &mut Option, - storage: KBox>, - ) -> Result> { - let candidate = prepared.as_ref().ok_or(EINVAL)?; - if !candidate.installed || !self.matches(candidate) { - return Err(EBUSY); - } - let kicks = self.kicks.take().ok_or(EIO)?; - // The candidate was validated above and cannot change through the exclusive borrow. - let Some(prepared) = prepared.take() else { - self.kicks = Some(kicks); - return Err(EIO); - }; - let mut next = prepared; - let Some(memory) = next.memory.take() else { - self.kicks = Some(kicks); - return Err(EIO); - }; - storage.write_init(kernel::try_init!(Pair { - kick_aliases: None, - memory, - kicks, - free_list: next.pool.clone(), - pending_aliases: None, - prepared: None, - active: Tracker::new(), - slot: self.slot, - owner: Some(next.owner), - ordinal: self.ordinal, - initial_packet: next.initial_packet, - clusters: self.clusters, - descriptor_flags: self.descriptor_flags, - installed: self.installed, - configs: [None; 2], - cursors: [0; 2], - recovery_generation: self.recovery_generation, - registered: self.registered, - outer_started: self.outer_started, - quarantined: false, - terminal: None, - previous: self.previous.take(), - displaced: None, - context_update: true, - })) - } -} - -impl Prepared { - /// Called after the registry revalidates its preparation epoch and parked pair identity. - pub(crate) fn finish_install( - &mut self, - parked: &ParkedPair, - alloc: &Allocator<'_>, - descriptors: &KernelObject, - ) -> Result { - if !parked.matches(self) { - return Err(EBUSY); - } - install_memory( - alloc, - descriptors, - self.slot, - self.memory.as_mut().ok_or(EIO)?, - &mut self.aliases, - &mut self.kicks, - )?; - self.installed = true; - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render/publication.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render/publication.rs deleted file mode 100644 index 0fc9c1d9a15dcf..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render/publication.rs +++ /dev/null @@ -1,165 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Preflighted item-ring publications. Each lane retains an independent cursor -//! and omits its configuration record only when its durable identity matches. - -use super::{memory, retirement::Window}; -use crate::g17::fw::{ - kick::KickTimestamp, - queue::{DataMaster, KickAnnounce, QueueConfig, ITEM_RING_ENTRIES}, -}; -use core::sync::atomic::{fence, AtomicU32, AtomicU64, Ordering}; -use kernel::prelude::*; - -fn identity(mut config: QueueConfig) -> QueueConfig { - config.install = 0; - config.context_update = 0; - config.completion_seed_valid = 0; - config.completion_seed = 0; - config.qos_update = 0; - config -} -pub(super) fn required(previous: Option, config: QueueConfig) -> bool { - config.install != 0 - || config.context_update != 0 - || config.completion_seed_valid != 0 - || config.qos_update != 0 - || previous.map(identity) != Some(identity(config)) -} - -pub(super) struct Lane<'a> { - items: [&'a AtomicU64; 3], - producer: &'a AtomicU32, - values: [u64; 3], - config: QueueConfig, - config_destination: *mut u64, - config_present: bool, - announce: KickAnnounce, - announce_destination: *mut KickAnnounce, - prefix: usize, - prefix_end: u32, - pub(super) window: Window, -} -impl<'a> Lane<'a> { - pub(super) fn new( - graph: &'a memory::Graph, - stage: usize, - ordinal: u64, - base: u32, - config: QueueConfig, - present: bool, - timestamp: KickTimestamp, - ) -> Result { - let count = 2 + u32::from(present); - let window = Window::new(base, count, ITEM_RING_ENTRIES)?; - let offset = ordinal as usize % memory::RECORDS; - let config_offset = memory::CONFIG[stage] + offset * memory::CONFIG_STRIDE; - let announce_offset = memory::ANNOUNCE[stage] + offset * memory::ANNOUNCE_STRIDE; - let config_destination = graph - .config - .pointer(config_offset, size_of::())? - .cast::(); - if config_destination.align_offset(align_of::()) != 0 { - return Err(EINVAL); - } - let announce_destination = graph - .queues - .pointer(announce_offset, size_of::())? - .cast::(); - let index = - |delta: u32| memory::ITEMS[stage] + ((base + delta) % ITEM_RING_ENTRIES) as usize * 8; - let items = [ - graph.queues.dword(index(0))?, - graph.queues.dword(index(1))?, - graph.queues.dword(index(2))?, - ]; - let announce_va = graph.queues.gpu_va() + announce_offset as u64; - let descriptor = graph.descriptor_va(stage, ordinal); - let values = if present { - [ - descriptor, - graph.config.gpu_va() + config_offset as u64, - announce_va, - ] - } else { - [descriptor, announce_va, 0] - }; - Ok(Self { - items, - producer: graph.queues.word(memory::POINTERS[stage] + 0x40)?, - values, - config, - config_destination, - config_present: present, - announce: KickAnnounce::new( - config.qid.try_into()?, - if stage == 0 { - DataMaster::Tiling - } else { - DataMaster::Fragment - }, - timestamp, - config.priority as u8, - )?, - announce_destination, - prefix: count as usize - 1, - prefix_end: (base + count - 1) % ITEM_RING_ENTRIES, - window, - }) - } - fn items(&self, range: core::ops::Range) { - for index in range { - self.items[index].store(self.values[index], Ordering::Relaxed); - } - } - fn config(&self) { - if !self.config_present { - return; - } - let source = (&self.config as *const QueueConfig).cast::(); - for index in 0..size_of::() / 8 { - // SAFETY: QueueConfig is packed integer storage with no padding; - // every source qword is read unaligned. The destination is a - // checked, aligned record owned by this unpublished ordinal. - unsafe { - self.config_destination - .add(index) - .write(source.add(index).read_unaligned()); - } - } - } - fn announce(&self) { - // SAFETY: The full record is inside the retained queue graph and is - // exclusively owned by this ordinal. Publication follows its copy. - unsafe { - self.announce_destination.write_unaligned(self.announce); - } - } - pub(super) fn publish_prefix(&self) { - self.items(0..self.prefix); - self.config(); - fence(Ordering::SeqCst); - self.producer.store(self.prefix_end, Ordering::Relaxed); - } - pub(super) fn publish_kick(&self) { - self.announce(); - self.items(self.prefix..self.prefix + 1); - fence(Ordering::SeqCst); - self.producer.store(self.window.target(), Ordering::Relaxed); - } - pub(super) fn publish_tiling( - &self, - event_slot: &AtomicU32, - next_slot: u32, - inner: &AtomicU32, - next_inner: u32, - ) { - self.items(0..self.prefix + 1); - self.config(); - self.announce(); - fence(Ordering::SeqCst); - self.producer.store(self.window.target(), Ordering::Relaxed); - event_slot.store(next_slot, Ordering::Relaxed); - inner.store(next_inner, Ordering::Relaxed); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render/retirement.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render/retirement.rs deleted file mode 100644 index d0a2d566bdc8d3..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render/retirement.rs +++ /dev/null @@ -1,348 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Paired render retirement evidence and bounded ownership of in-flight work. -//! Queue progress and execution timestamps precede the completion stamps which -//! authorize releasing command memory. A signalled error is not that witness. - -use crate::hw::t8140::queues; -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; - -const DEPTH: usize = queues::RENDER_DEPTH as usize; -const MAX_GROUP_ITEMS: u32 = 4; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Cursors { - pub(crate) done: u32, - pub(crate) read: u32, - pub(crate) write: u32, -} - -impl Cursors { - fn valid(self) -> bool { - self.done <= self.write && self.read <= self.write - } - - fn idle(self) -> bool { - self.done == self.read && self.read == self.write - } -} - -/// Each engine may omit its configuration update independently. Preserve both -/// exact windows; neither target can be reconstructed from a render ordinal. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Window { - base: u32, - count: u32, - capacity: u32, -} - -impl Window { - pub(crate) fn new(base: u32, count: u32, capacity: u32) -> Result { - if base >= capacity || !(2..=MAX_GROUP_ITEMS).contains(&count) || count >= capacity { - return Err(EINVAL); - } - Ok(Self { - base, - count, - capacity, - }) - } - - pub(crate) fn target(self) -> u32 { - ((u64::from(self.base) + u64::from(self.count)) % u64::from(self.capacity)) as u32 - } - - fn normalize(self, cursors: Cursors) -> Result { - // Earlier and later groups can coexist in a snapshot. Anchor behind - // this group's base so a backwards observation cannot look like wrap. - let back = MAX_GROUP_ITEMS * (DEPTH as u32 - 1); - let limit = MAX_GROUP_ITEMS * DEPTH as u32 + back; - if limit >= self.capacity { - return Err(EINVAL); - } - let base = (u64::from(self.base) + u64::from(self.capacity) - u64::from(back)) - % u64::from(self.capacity); - let distance = |value: u32| -> Result { - if value >= self.capacity { - return Err(EIO); - } - let distance = - (u64::from(value) + u64::from(self.capacity) - base) % u64::from(self.capacity); - if distance > u64::from(limit) { - return Err(EIO); - } - Ok(distance as u32) - }; - Ok(Cursors { - done: distance(cursors.done)?, - read: distance(cursors.read)?, - write: distance(cursors.write)?, - }) - } - - fn progress(self, previous: Cursors, current: Cursors) -> Result<(bool, bool, bool)> { - let previous = self.normalize(previous)?; - let current = self.normalize(current)?; - let prefix = MAX_GROUP_ITEMS * (DEPTH as u32 - 1) + self.count; - if !previous.valid() - || !current.valid() - || current.done < previous.done - || current.read < previous.read - || current.write < previous.write - || prefix > current.write - { - return Err(EIO); - } - Ok(( - current.read >= prefix, - current.done >= prefix, - current.idle(), - )) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Ticket { - /// Full physical-pair ordinal, retained through owner switches and slot wrap. - pub(crate) ordinal: u64, - pub(crate) windows: [Window; 2], - pub(crate) previous: [Cursors; 2], -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Snapshot { - pub(crate) cursors: [Cursors; 2], - pub(crate) job_list_empty: bool, - /// The tiler writes the start and fragment processing writes the end. - pub(crate) pass: [u64; 2], -} - -impl Snapshot { - fn executed(self) -> bool { - self.pass[0] != 0 && self.pass[1] != 0 - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Observation { - pub(crate) accepted: bool, - pub(crate) retired: bool, - pub(crate) complete: bool, - pub(crate) idle: bool, -} - -pub(crate) fn observe(ticket: &Ticket, snapshot: Snapshot) -> Result { - let ta = ticket.windows[0].progress(ticket.previous[0], snapshot.cursors[0])?; - let fragment = ticket.windows[1].progress(ticket.previous[1], snapshot.cursors[1])?; - let retired = ta.1 && fragment.1; - Ok(Observation { - accepted: ta.0 && fragment.0, - retired, - complete: retired && snapshot.executed(), - idle: ta.2 && fragment.2 && snapshot.job_list_empty, - }) -} - -/// Outstanding publications span less than half the 24-bit stamp sequence. -/// A zero stamp is valid when that sequence wraps. -pub(crate) const fn stamp_covers(stamp: u32, published: u64) -> bool { - let delta = stamp.wrapping_sub((published as u32).wrapping_shl(8)); - delta & 0xff == 0 && delta < 0x8000_0000 -} - -fn visible(ticket: &Ticket, stamps: [u32; 2]) -> Result { - let published = ticket.ordinal.checked_add(1).ok_or(EOVERFLOW)?; - Ok(stamps - .into_iter() - .all(|stamp| stamp_covers(stamp, published))) -} - -/// Reads the graph owned by one physical pair. The owner validates the full -/// ordinal before reading reused descriptor/timestamp slots. Word accessors -/// perform aligned atomic loads; the observer supplies the acquire barriers. -pub(crate) trait Memory { - fn snapshot(&mut self, ordinal: u64) -> Result; - fn stamps(&mut self) -> Result<[u32; 2]>; - fn pass(&mut self, ordinal: u64) -> Result<[u64; 2]>; -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Poll { - Retry, - Pending, - Complete([u64; 4]), -} - -/// Observe the ordered front. Individually sampled cursor words can straddle -/// a firmware update, so an invalid snapshot receives one immediate reread. -/// Shared work-state membership can include other engines and cannot gate a -/// successful individual member's completion. -pub(crate) fn poll(ticket: &mut Ticket, memory: &mut impl Memory) -> Result { - fence(Ordering::Acquire); - let mut snapshot = memory.snapshot(ticket.ordinal)?; - let observation = match observe(ticket, snapshot) { - Ok(observation) => observation, - Err(_) => { - fence(Ordering::Acquire); - snapshot = memory.snapshot(ticket.ordinal)?; - match observe(ticket, snapshot) { - Ok(observation) => observation, - Err(_) => return Ok(Poll::Retry), - } - } - }; - complete_poll(ticket, memory, snapshot, observation) -} - -/// Finish an observation sampled while its exact pair was exclusively borrowed. -/// The caller restores that pair and validates its epoch and front ticket first. -/// Completion stamps and the final timestamp refresh remain commit-time reads. -pub(crate) fn poll_observed( - ticket: &mut Ticket, - memory: &mut impl Memory, - snapshot: Snapshot, -) -> Result { - let observation = observe(ticket, snapshot)?; - complete_poll(ticket, memory, snapshot, observation) -} - -fn complete_poll( - ticket: &mut Ticket, - memory: &mut impl Memory, - snapshot: Snapshot, - observation: Observation, -) -> Result { - if observation.complete { - if !visible(ticket, memory.stamps()?)? { - return Ok(Poll::Pending); - } - // Refresh this generation's pass after observing the post-maintenance - // stamps. The pre-stamp snapshot alone cannot authorize export. - fence(Ordering::Acquire); - let [start, end] = memory.pass(ticket.ordinal)?; - return Ok(Poll::Complete([start, start, start, end])); - } - ticket.previous = snapshot.cursors; - Ok(Poll::Pending) -} - -/// Strict timeout settlement accepts only a nonexecuted, retired and unlinked -/// group. Drain reclamation may also settle executed groups, but retains both -/// ownership proofs and the exact post-maintenance stamps in either case. -pub(crate) fn abandoned( - ticket: &Ticket, - snapshot: Snapshot, - stamps: [u32; 2], - recovery_state: u32, - accept_executed: bool, -) -> Result { - let completion = observe(ticket, snapshot)?; - if !completion.retired || !snapshot.job_list_empty || (snapshot.executed() && !accept_executed) - { - return Ok(false); - } - Ok(visible(ticket, stamps)? && recovery_state == 0) -} - -/// Pair-wide proof, checked before and after servicing free-list events and -/// again after retiring all trackers. A proof from before event service must -/// not authorize releasing guards after a new recovery began. -pub(crate) fn drained( - published: u64, - stamps: [u32; 2], - free_list: [u64; 2], - recovery_state: u32, -) -> bool { - published == 0 - || (stamps - .into_iter() - .all(|stamp| stamp_covers(stamp, published)) - && free_list[0] == free_list[1] - && recovery_state == 0) -} - -/// Fixed storage keeps admission, rollback and retirement allocation-free. -/// Each element retains its packet and resource leases independently of fences. -pub(crate) struct Tracker { - entries: [Option; DEPTH], - head: usize, - len: usize, -} - -impl Tracker { - pub(crate) const fn new() -> Self { - Self { - entries: [const { None }; DEPTH], - head: 0, - len: 0, - } - } - - pub(crate) fn len(&self) -> usize { - self.len - } - pub(crate) fn room(&self) -> bool { - self.len < DEPTH - } - pub(crate) fn front(&self) -> Option<&T> { - if self.len == 0 { - None - } else { - self.entries[self.head].as_ref() - } - } - pub(crate) fn front_mut(&mut self) -> Option<&mut T> { - if self.len == 0 { - None - } else { - self.entries[self.head].as_mut() - } - } - pub(crate) fn back(&self) -> Option<&T> { - if self.len == 0 { - None - } else { - self.entries[(self.head + self.len - 1) % DEPTH].as_ref() - } - } - pub(crate) fn back_mut(&mut self) -> Option<&mut T> { - if self.len == 0 { - None - } else { - self.entries[(self.head + self.len - 1) % DEPTH].as_mut() - } - } - pub(crate) fn push(&mut self, value: T) -> core::result::Result<(), T> { - if !self.room() { - return Err(value); - } - self.entries[(self.head + self.len) % DEPTH] = Some(value); - self.len += 1; - Ok(()) - } - pub(crate) fn pop(&mut self) -> Option { - if self.len == 0 { - return None; - } - let value = self.entries[self.head].take(); - self.head = (self.head + 1) % DEPTH; - self.len -= 1; - value - } - - pub(crate) fn iter(&self) -> impl Iterator { - Iterator::chain( - self.entries[self.head..].iter(), - self.entries[..self.head].iter(), - ) - .take(self.len) - .filter_map(Option::as_ref) - } - pub(crate) fn iter_mut(&mut self) -> impl Iterator { - let (before, after) = self.entries.split_at_mut(self.head); - Iterator::chain(after.iter_mut(), before.iter_mut()) - .take(self.len) - .filter_map(Option::as_mut) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/queue/render/warm.rs b/drivers/gpu/drm/asahi-neo/g17/queue/render/warm.rs deleted file mode 100644 index 4db471f254ec61..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/queue/render/warm.rs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Fresh render graph cursor initialization at a retained physical ordinal. - -use super::{memory, Pair}; -use crate::g17::fw::queue::PointerBlock; -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; - -impl Pair { - /// A new graph at a retained ordinal starts its variable item rings at zero. - /// The old graph has retired; firmware has not received this graph's queues. - pub(crate) fn rebase_fresh_graph(&mut self) -> Result { - if self.ordinal == 0 { - return Ok(()); - } - fence(Ordering::Acquire); - if self - .memory - .graph - .cursors()? - .into_iter() - .any(|cursor| cursor.done != 0 || cursor.read != 0 || cursor.write != 0) - { - return Err(EBUSY); - } - let mut blocks = [core::ptr::null::(); 2]; - for (block, offset) in blocks.iter_mut().zip(memory::POINTERS) { - let pointer = self - .memory - .graph - .queues - .pointer(offset, size_of::())?; - if pointer.align_offset(align_of::()) != 0 { - return Err(EINVAL); - } - *block = pointer.cast(); - } - for block in blocks { - // SAFETY: Both complete pointer blocks were checked above. This fresh - // graph is unpublished; only the live atomic cursors are accessed. - unsafe { &*block }.rebase(0); - } - fence(Ordering::SeqCst); - self.cursors = [0; 2]; - Ok(()) - } - -} diff --git a/drivers/gpu/drm/asahi-neo/g17/recovery.rs b/drivers/gpu/drm/asahi-neo/g17/recovery.rs deleted file mode 100644 index 74bb9ca7567eac..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/recovery.rs +++ /dev/null @@ -1,301 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Firmware restart handshake and per-VM failure classification. -//! -//! The device mutex serializes this state with publication. Preparations are closed and joined -//! without that mutex before `service`; no queue callback may signal a fence here. Queue -//! owners retain failed work until its retirement witness, a complete pair-drain witness, or -//! processor stop. A failed fence alone never releases mappings. - -mod attribution; -mod slot; -pub(crate) use attribution::Sources; -use attribution::Attribution; - -pub(super) fn slot_key(slot: usize) -> Option { - slot::key(slot).ok() -} - -use super::status::VmStatus; -use kernel::{prelude::*, sync::Arc}; - -const QIDS: u8 = 128; -const GUILTY: u8 = 1 << 1; -const CANCELLED: u8 = 1 << 2; -const MAX_GUILTY_VMS: usize = 8; -const RECOVERY_POLLS: usize = 500; - -/// Per-QID classifications from the complete firmware recovery table. -/// Multiple records for a queue contribute all of their classification bits. -#[derive(Default)] -pub(super) struct Entries { - guilty: u128, - cancelled: u128, -} - -impl Entries { - pub(super) fn include(&mut self, flags: u32) { - if flags & 1 == 0 { - return; - } - let qid = (flags >> 4) & 0x7f; - let bit = 1u128 << qid; - if flags & u32::from(GUILTY) != 0 { - self.guilty |= bit; - } - if flags & u32::from(CANCELLED) != 0 { - self.cancelled |= bit; - } - } -} - -/// How an error completion affects its VM's sticky status and other jobs. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum Class { - /// Only this work's fence fails; VM status and unrelated work remain usable. - Spared, - /// This VM owns guilty work. - Guilty, -} - -/// Queue operations under the device mutex. Completion signalling belongs to the caller after -/// that mutex is dropped. `oldest_spared` returns None if the QID has no in-flight owner. -pub(super) trait Queues { - fn render_scheduler_active(&self) -> bool; - fn render_pass_started(&self, qid: u8, stamp: u64) -> Option; - /// Copy every retained queue's live producer to its recovery target, then - /// issue a full barrier. A failed graph must not skip subsequent queues. - fn publish_recovery_targets(&self) -> Option; - fn oldest_spared(&self, qid: u8) -> Option; - fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)); - /// Classify every in-flight completion; return QIDs of render slots with spared work. - fn classify_work(&mut self, class: &dyn Fn(&Arc) -> Class) -> u128; - fn settle_blamed_render(&mut self, qid: u8, error: Error); - fn settle_cancelled_renders(&mut self, qids: u128); - fn rescan_compute(&mut self) -> Result; - fn release_retired_compute(&mut self); -} - -/// Live shared-memory operations. The handshake word uses aligned 32-bit accesses; transitions -/// check the expected value, issue a full barrier, store the next value, then another barrier. -pub(super) trait Memory { - fn recovery_state(&self) -> Result; - fn host_recovery(&self) -> Result; - fn fault_sources(&self) -> Result; - fn transition(&self, expected: u32, next: u32) -> Result; - fn clear_timestamps(&self) -> Result; - fn entries(&self) -> Result; - /// One millisecond wait; this is polled because the firmware sends no state-3 notification. - fn wait_tick(&self); -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -struct Request { - generation: u64, - blamed: Option, -} - -/// Generation accepted by the host and the coalesced request from either coprocessor. -pub(super) struct State { - accepted: u64, - pending: Option, -} - -/// Non-fatal post-recovery failures. The firmware handshake has already closed successfully. -pub(super) struct ServiceResult { - pub(super) scan_error: Option, - pub(super) target_error: Option, -} - -impl State { - pub(super) const fn new() -> Self { - Self { - accepted: 0, - pending: None, - } - } - pub(super) fn generation(&self) -> u64 { - self.accepted - } - pub(super) fn pending(&self) -> bool { - self.pending.is_some() - } - - /// Returns false for a consumed duplicate from an older generation, true when the event - /// drain must stop until the current request is serviced. The first blamed QID wins. - pub(super) fn request(&mut self, generation: u64, stamp: i32) -> Result { - if generation < self.accepted { - return Ok(false); - } - if generation != self.accepted { - return Err(EIO); - } - if stamp != -1 && !(0..i32::from(QIDS)).contains(&stamp) { - return Err(EIO); - } - let blamed = if (0..i32::from(QIDS)).contains(&stamp) { - Some(stamp as u8) - } else { - None - }; - if let Some(pending) = self.pending.as_mut() { - if pending.generation != generation { - return Err(EBUSY); - } - pending.blamed = pending.blamed.or(blamed); - } else { - self.pending = Some(Request { generation, blamed }); - } - Ok(true) - } -} - -/// The firmware owner implements memory and queue access without splitting its -/// mutable borrow or duplicating the authoritative restart generation. -pub(super) trait Host: Memory + Queues { - fn recovery(&self) -> &State; - fn recovery_mut(&mut self) -> &mut State; - fn report_recovery(&self, _generation: u64, _blamed: Option, _sources: &Sources) {} -} - -/// Performs one pending restart. The caller has closed preparation admission and joined all -/// builders off-lock. Work remains mapped throughout every callback, including on error. -pub(super) fn service( - host: &mut H, - crashed: impl Fn() -> bool, -) -> Result { - let request = host.recovery().pending.ok_or(EINVAL)?; - if request.generation != host.recovery().accepted || host.host_recovery()? != 0 { - return Err(EIO); - } - // The restart record is published before firmware announces state 1. - // Its ring cursor is already consumed; wait only for that readiness word, - // without advancing the host generation or acknowledging an early event. - let mut ready = false; - for _ in 0..RECOVERY_POLLS { - if crashed() { - return Err(EIO); - } - match host.recovery_state()? { - 1 => { - ready = true; - break; - } - 0 => host.wait_tick(), - _ => return Err(EIO), - } - } - if !ready { - return Err(ETIMEDOUT); - } - // Recheck ownership after the off-CPU waits, before any handshake mutation. - if host.host_recovery()? != 0 { - return Err(EIO); - } - host.recovery_mut().accepted = host.recovery().accepted.wrapping_add(1); - if request.generation == 0 && !host.render_scheduler_active() { - host.clear_timestamps()?; - } - let mut sources = host.fault_sources()?; - host.report_recovery(request.generation, request.blamed, &sources); - let target_error = host.publish_recovery_targets(); - host.transition(1, 2)?; - let mut recovered = false; - for _ in 0..RECOVERY_POLLS { - if host.recovery_state()? == 3 { - recovered = true; - break; - } - if crashed() { - return Err(EIO); - } - host.wait_tick(); - } - if !recovered { - return Err(ETIMEDOUT); - } - // A failed table read leaves only the explicit blamed QID as evidence of guilt. - let entries = host.entries().unwrap_or_default(); - host.transition(3, 0)?; - if host.recovery_state()? != 0 { - return Err(EIO); - } - host.recovery_mut().pending = None; - - let cancelled = classify(host, entries, request.blamed, &mut sources); - if let Some(qid) = request.blamed { - host.settle_blamed_render(qid, EIO); - } - if cancelled != 0 { - host.settle_cancelled_renders(cancelled); - } - let scan_error = host.rescan_compute().err(); - host.release_retired_compute(); - Ok(ServiceResult { scan_error, target_error }) -} - -/// Resolves guilty QIDs before changing any completion's classification. Unknown ownership or -/// too many distinct guilty VMs leaves all prior classifications intact. The oldest spared work -/// of a QID makes later blame collateral to its previous recovery, rather than new VM guilt. -fn classify(queues: &mut Q, entries: Entries, blamed: Option, sources: &mut Sources) -> u128 { - let mut guilty_qids = entries.guilty | blamed.map_or(0, |qid| 1u128 << qid); - let cancelled = entries.cancelled; - let attribution = sources.attribute(blamed, |qid, stamp| queues.render_pass_started(qid, stamp)); - match attribution { - Attribution::Firmware => {} - Attribution::Queues(qids) => guilty_qids = qids, - Attribution::NoCulprit => guilty_qids = 0, - } - let no_culprit = attribution == Attribution::NoCulprit; - if guilty_qids == 0 && !no_culprit { - return 0; - } - let mut guilty: [Option>; MAX_GUILTY_VMS] = Default::default(); - let mut count = 0; - let mut collateral = 0u128; - let mut unresolved = false; - for qid in 0..QIDS { - if guilty_qids & (1u128 << qid) == 0 { - continue; - } - match queues.oldest_spared(qid) { - None => unresolved = true, - Some(true) => collateral |= 1u128 << qid, - Some(false) => { - let mut found = false; - queues.visit_owners(qid, &mut |status| { - found = true; - if guilty[..count] - .iter() - .flatten() - .any(|known| Arc::ptr_eq(known, status)) - { - return; - } - if let Some(slot) = guilty.get_mut(count) { - *slot = Some(status.clone()); - count += 1; - } else { - unresolved = true; - } - }); - unresolved |= !found; - } - } - } - if unresolved || (count == 0 && collateral == 0 && !no_culprit) { - return 0; - } - let spared = queues.classify_work(&|status| { - guilty[..count] - .iter() - .find_map(|known| { - known - .as_ref() - .filter(|known| Arc::ptr_eq(known, status)) - .map(|_| Class::Guilty) - }) - .unwrap_or(Class::Spared) - }); - cancelled & !(guilty_qids & !collateral) & spared -} diff --git a/drivers/gpu/drm/asahi-neo/g17/recovery/attribution.rs b/drivers/gpu/drm/asahi-neo/g17/recovery/attribution.rs deleted file mode 100644 index a580a860ee3c46..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/recovery/attribution.rs +++ /dev/null @@ -1,110 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Resolve ambiguous recovery blame using slot identity and render execution evidence. - -pub(super) const SLOTS: usize = 16; -const STAMP_MASK: u64 = (1 << 40) - 1; -const PAGE_FAULT: u32 = 3; - -#[derive(Clone, Copy, Debug, Default)] -pub(super) struct Source { - slot: u8, - qid: Option, - stamp: u64, - monitor: u32, - started: Option, -} - -pub(crate) struct Sources { - pub(crate) reason: Option, - mask: u32, - records: [Source; SLOTS], - count: usize, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum Attribution { - Firmware, - Queues(u128), - NoCulprit, -} - -impl Sources { - /// Selected slot identities only; unused records carry no diagnostic evidence. - pub(crate) fn diagnostics(&self) -> impl core::fmt::Debug + '_ { - (self.mask, &self.records[..self.count]) - } - - /// Firmware is halted for every selected key and progress-record read. - /// Failed reads stay unknown; incomplete evidence cannot override firmware blame. - pub(crate) fn sample(mask: u32, mut read: impl FnMut(usize) -> Option<(u64, u32)>) -> Self { - let mut sources = Self { - reason: None, - mask, - records: [Source::default(); SLOTS], - count: 0, - }; - for slot in 0..SLOTS { - if mask & (1 << slot) == 0 { - continue; - } - let Some((key, monitor)) = read(slot) else { - continue; - }; - let qid = (key >> 40) & 0xff; - sources.records[sources.count] = Source { - slot: slot as u8, - qid: (key != 0 && qid < 128).then_some(qid as u8), - stamp: key & STAMP_MASK, - monitor, - started: None, - }; - sources.count += 1; - } - sources - } - - pub(super) fn attribute( - &mut self, - blamed: Option, - mut started: impl FnMut(u8, u64) -> Option, - ) -> Attribution { - if self.reason.is_none() || self.count as u32 != self.mask.count_ones() { - return Attribution::Firmware; - } - for source in &mut self.records[..self.count] { - if let Some(qid) = source.qid { - source.started = started(qid, source.stamp); - } - } - if self.reason == Some(PAGE_FAULT) || self.mask.count_ones() < 2 { - return Attribution::Firmware; - } - let Some(blamed) = blamed else { - return Attribution::Firmware; - }; - let Some(source) = self.records[..self.count] - .iter() - .find(|source| source.qid == Some(blamed)) - else { - return Attribution::Firmware; - }; - if source.started != Some(false) || source.monitor != 0 { - return Attribution::Firmware; - } - let slot = source.slot; - let mut guilty = 0; - for source in &self.records[..self.count] { - if source.slot != slot && source.started == Some(true) { - if let Some(qid) = source.qid { - guilty |= 1u128 << qid; - } - } - } - if guilty == 0 { - Attribution::NoCulprit - } else { - Attribution::Queues(guilty) - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/recovery/slot.rs b/drivers/gpu/drm/asahi-neo/g17/recovery/slot.rs deleted file mode 100644 index 307534708e922f..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/recovery/slot.rs +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Bounded read-only slot identity sampling while firmware is halted. - -use crate::{hw::t8140, mmu}; -use core::{ - ffi::c_void, - ptr::NonNull, - sync::atomic::{AtomicU64, Ordering}, -}; -use kernel::{bindings, prelude::*}; - -struct Mapping(NonNull); - -impl Drop for Mapping { - fn drop(&mut self) { - // SAFETY: This is the successful memremap result owned exclusively by this guard. - unsafe { bindings::memunmap(self.0.as_ptr()) }; - } -} - -pub(super) fn key(slot: usize) -> Result { - if slot >= super::attribution::SLOTS { - return Err(EINVAL); - } - let physical = t8140::recovery::SLOT_KEYS + slot as u64 * size_of::() as u64; - let offset = physical as usize & (mmu::UAT_PGSZ - 1); - // SAFETY: The fixed recovery-key range is firmware RAM, page aligned here - // for mapping. Only the selected aligned eight-byte word is read below. - let raw = unsafe { - bindings::memremap( - physical - offset as u64, - mmu::UAT_PGSZ, - bindings::MEMREMAP_WB as _, - ) - }; - let mapping = Mapping(NonNull::new(raw).ok_or(ENOMEM)?); - let ctr: u64; - // SAFETY: CTR_EL0 is readable at EL1. DminLine specifies the cache maintenance granule. - unsafe { - core::arch::asm!("mrs {ctr}, ctr_el0", ctr = out(reg) ctr, - options(nomem, nostack, preserves_flags)); - } - let line = 4usize << ((ctr >> 16) & 0xf); - if line > mmu::UAT_PGSZ { - return Err(EINVAL); - } - // SAFETY: The range and offset are eight-byte aligned and lie within the mapped page. - let pointer = unsafe { - mapping - .0 - .as_ptr() - .cast::() - .add(offset) - .cast::() - }; - let mut address = pointer as usize & !(line - 1); - let end = (pointer as usize) - .checked_add(size_of::()) - .ok_or(EOVERFLOW)?; - // SAFETY: The mapping stays live and is never written by the host. The - // invalidated cache line lies within its page; barriers complete the - // invalidation before the aligned load of the selected firmware-owned word. - unsafe { - core::arch::asm!("dsb osh", options(nostack, preserves_flags)); - while address < end { - core::arch::asm!("dc ivac, {address}", address = in(reg) address, - options(nostack, preserves_flags)); - address = address.checked_add(line).ok_or(EOVERFLOW)?; - } - core::arch::asm!("dsb osh", options(nostack, preserves_flags)); - Ok((*pointer).load(Ordering::Acquire)) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime.rs b/drivers/gpu/drm/asahi-neo/g17/runtime.rs deleted file mode 100644 index 029dd590148544..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime.rs +++ /dev/null @@ -1,1181 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Device ownership of physical queues and off-lock construction reservations. -//! -//! The device mutex protects this registry. Logical queues may disappear while -//! their published graphs remain here; only exact retirement permits rebinding, -//! and only processor stop permits dropping installed firmware addresses. -//! A preparation lease keeps Shared::state populated: shutdown and loss join -//! every lease before taking the session, including builders asleep on changed. - -mod backend; -mod compute_control; -mod compute_replay; -mod compute_exit; -mod events; -mod feed; -mod render; -pub(super) mod teardown; - -pub(super) use super::now_ns; -use super::{ - buffer::{BufferIds, MetricsIds}, - context::{Context, QosIds}, - fw::queue::DataMaster, - kick, qos, - queue::{self, admission, compute}, -}; -use core::sync::atomic::{fence, Ordering}; -use kernel::{prelude::*, sync::Arc}; - -const QIDS: usize = 128; -const FIRST_COMPUTE_QID: u8 = 4; -const RECOVERY_SETTLE_POLLS: usize = 200; -const WORK_RING_WAIT_NS: u64 = super::job::TIMEOUT_MS as u64 * 1_000_000; - -/// A reservation remains identifiable while its backing is built without the -/// device mutex. Failed construction returns the slot, never its host identity. -#[derive(Clone, Copy)] -pub(super) struct ComputeReservation { - slot: usize, - pub(super) id: kick::Id, - owner: u64, -} - -struct ComputeEntry { - reservation: ComputeReservation, - queue: Option>, -} - -pub(super) struct Registry { - pub(super) qids: kick::QueueIds, - pub(super) buffers: BufferIds, - pub(super) accounting: qos::Accounting, - pub(super) metrics: Arc, - pub(super) qos_ids: Arc, - pub(super) admission: Arc, - render: render::RenderState, - identities: queue::Identities, - compute: KVec>, - pub(super) compute_building: bool, - compute_scheduler_building: bool, - replay_outer_wait: Option, - pub(super) observations: compute::retirement::Batch, - worker_deferred: Option, - teardown: super::teardown::Pending, -} - -impl Registry { - /// Initialize inside the firmware allocation instead of staging the whole - /// registry on the probe stack. No additional allocation is required. - pub(super) fn new( - clusters: u32, - descriptor_flags: [u32; 2], - ) -> impl Init { - kernel::try_init!(Self { - qids <- kick::QueueIds::new().chain(|qids| { - qids.reserve(1, DataMaster::Tiling, Some(0))?; - qids.reserve(2, DataMaster::Fragment, Some(1))?; - Ok(()) - }), - buffers <- BufferIds::new(), - accounting: qos::Accounting::new(), - metrics: Arc::new(MetricsIds::new(), GFP_KERNEL)?, - qos_ids: QosIds::new()?, - admission: admission::Pool::new()?, - render: render::RenderState::new(clusters, descriptor_flags)?, - identities: queue::Identities::new(), - compute: { - let mut compute = KVec::with_capacity(QIDS, GFP_KERNEL)?; - for _ in 0..QIDS { - compute.push(None, GFP_KERNEL)?; - } - compute - }, - compute_building: false, - compute_scheduler_building: false, - replay_outer_wait: None, - observations: compute::retirement::Batch::new()?, - worker_deferred: Some(DeferredBatch::worker()?), - teardown: super::teardown::Pending::new()?, - }) - } - - pub(super) fn compute(&self, owner: u64) -> Option<&compute::Queue> { - self.compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .find(|queue| queue.owner() == Some(owner)) - } - - pub(super) fn compute_mut(&mut self, owner: u64) -> Option<&mut compute::Queue> { - self.compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) - } - - /// An installed idle queue keeps its original QID and backing. Mapping a - /// replacement context must succeed before its old binding is displaced. - /// A vacant queue (USC backing given back) is offered for re-arming only - /// when no populated idle queue exists. - pub(super) fn rebind_compute(&mut self, owner: u64, context: &Arc) -> Result { - if let Some(queue) = self.compute(owner) { - return Ok(if queue.matches(owner, queue.qid(), context)? { - Rebind::Done - } else { - Rebind::Fresh - }); - } - let mut vacant = None; - for queue in self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .filter(|queue| queue.reusable()) - { - if queue.pool_vacant() { - vacant.get_or_insert(queue.qid()); - continue; - } - queue.bind_owner(owner, context.clone())?; - return Ok(Rebind::Done); - } - Ok(vacant.map_or(Rebind::Fresh, Rebind::Rearm)) - } - - /// Installs fresh USC backing into the vacant queue `qid`; the caller writes - /// the returned descriptor row before binding. EAGAIN if the queue was taken. - pub(super) fn rearm_compute( - &mut self, - qid: u8, - pages: crate::g17::freelist::Pages, - ring: compute::DescriptorRing, - ) -> Result<(u16, u64)> { - let queue = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == qid) - .ok_or(EAGAIN)?; - queue.rearm(pages, ring) - } - - pub(super) fn bind_rearmed(&mut self, qid: u8, owner: u64, context: &Arc) -> Result { - let queue = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == qid && queue.reusable()) - .ok_or(EAGAIN)?; - queue.bind_owner(owner, context.clone()) - } - - /// Publishes releases for the oldest idle retained pools beyond `cap`, at - /// most four per pass. Returns the number published. - pub(super) fn vacate_idle_pools( - &mut self, - init: &super::initdata::InitData, - cap: usize, - ) -> Result { - // Count first: event turns are frequent and the steady state has no excess. - let forced = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.vacate_candidate() && queue.needs_fresh_backing()) - .count(); - let count = self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.vacate_candidate()) - .count(); - if count <= cap && forced == 0 { - return Ok(0); - } - let mut candidates: KVec<(u64, u8)> = KVec::with_capacity(count, GFP_KERNEL)?; - for queue in self - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.vacate_candidate()) - { - // Recycled killed queues sort first (time 0) and count beyond the cap. - let key = if queue.needs_fresh_backing() { 0 } else { queue.released_at() }; - candidates.push((key, queue.qid()), GFP_KERNEL)?; - } - candidates.sort_unstable(); - let excess = candidates.len().saturating_sub(cap).max(forced); - let mut published = 0; - for (_, qid) in candidates.iter().take(excess.min(4)) { - let queue = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.qid() == *qid) - .ok_or(EIO)?; - match queue.vacate(init) { - Err(error) if error == EAGAIN => break, - result => result?, - } - published += 1; - } - Ok(published) - } - - /// Frees the backing of every pool whose release the firmware consumed. - pub(super) fn observe_pool_releases( - &mut self, - init: &super::initdata::InitData, - deferred: &mut DeferredBatch, - ) -> Result { - fence(Ordering::Acquire); - let consumer = init.control_consumer()?.load(Ordering::Relaxed); - let producer = init.control_producer()?.load(Ordering::Relaxed); - let mut freed = 0; - for queue in self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - { - if let Some(vacated) = queue.observe_vacancy(consumer, producer)? { - deferred.defer_vacated(vacated); - freed += 1; - } - } - Ok(freed) - } - - pub(super) fn reserve_compute(&mut self, owner: u64) -> Result { - if owner == 0 - || self.compute.iter().flatten().any(|entry| { - entry - .queue - .as_ref() - .map_or(entry.reservation.owner == owner, |queue| { - queue.owner() == Some(owner) - }) - }) - { - return Err(EINVAL); - } - let slot = self - .compute - .iter() - .position(Option::is_none) - .ok_or(ENOSPC)?; - let identity = self.identities.reserve(1)?.start; - let id = self - .qids - .reserve(identity, DataMaster::Compute, Some(FIRST_COMPUTE_QID))?; - let reservation = ComputeReservation { slot, id, owner }; - self.compute[slot] = Some(ComputeEntry { - reservation, - queue: None, - }); - Ok(reservation) - } - - fn reserved(&mut self, reservation: ComputeReservation) -> Result<&mut ComputeEntry> { - let entry = self - .compute - .get_mut(reservation.slot) - .and_then(Option::as_mut) - .ok_or(EIO)?; - if entry.reservation.id != reservation.id - || entry.reservation.owner != reservation.owner - || entry.queue.is_some() - { - return Err(EIO); - } - Ok(entry) - } - - /// The caller checked its preparation epoch and retained the same device - /// session before placing this graph where publication can find it. - pub(super) fn install_compute( - &mut self, - reservation: ComputeReservation, - queue: &mut Option>, - ) -> Result { - let built = queue.as_ref().ok_or(EINVAL)?; - if built.qid() != reservation.id.qid() || built.owner() != Some(reservation.owner) { - return Err(EIO); - } - let entry = self.reserved(reservation)?; - entry.queue = queue.take(); - Ok(()) - } - - pub(super) fn cancel_compute(&mut self, reservation: ComputeReservation) -> Result { - self.reserved(reservation)?; - self.qids.cancel_unpublished(reservation.id)?; - self.compute[reservation.slot] = None; - Ok(()) - } - - pub(super) fn release_compute(&mut self, owner: u64) -> Result { - let Some(slot) = self.compute.iter().position(|entry| { - entry - .as_ref() - .and_then(|entry| entry.queue.as_ref()) - .is_some_and(|queue| queue.owner() == Some(owner)) - }) else { - return Ok(()); - }; - let entry = self.compute[slot].as_mut().ok_or(EIO)?; - if entry.queue.as_mut().ok_or(EIO)?.release_owner()? { - self.qids.cancel_unpublished(entry.reservation.id)?; - self.compute[slot] = None; - } - Ok(()) - } -} - -/// Outcome of offering an owner a retained compute queue. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(super) enum Rebind { - /// The owner now holds an installed, populated queue. - Done, - /// Only a vacant queue is available; re-arm `qid` with fresh backing. - Rearm(u8), - /// No retained queue suits the owner: reserve a fresh QID. - Fresh, -} - -/// An immutable logical owner shared by its two scheduler entities. Installed -/// physical graphs retain the context independently of this frontend lifetime. -pub(super) struct Backend { - shared: Arc, - pub(super) context: Arc, - owner: u64, - backlog: core::sync::atomic::AtomicU32, -} - -impl Backend { - pub(super) fn new( - shared: &Arc, - vm: &crate::mmu::Vm, - owner: u64, - policy: super::fw::queue::Policy, - ) -> Result<(Arc, admission::QueueSlot)> { - let lease = shared.preparations.enter(true)?; - let dev = shared.drm_neo()?; - let (uat, pool, qos, execution, slot) = { - let mut state = shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if shared.crashed.load(core::sync::atomic::Ordering::Acquire) { - return Err(EIO); - } - let slot = firmware.queues.admission.reserve_queue()?; - let execution = firmware.uat.new_execution_context(vm)?; - ( - firmware.uat.clone(), - firmware.pool.clone(), - firmware.queues.qos_ids.clone(), - execution, - slot, - ) - }; - let alloc = super::object::Allocator { - dev: &dev, - uat: &uat, - }; - let owner_pid = - u32::try_from(kernel::current!().group_leader().pid()).map_err(|_| EOVERFLOW)?; - let context = Arc::new( - Context::new(&alloc, &pool, &qos, execution, owner_pid, policy)?, - GFP_KERNEL, - )?; - let state = shared.state.lock(); - if !lease.is_current() || state.is_none() { - return Err(EAGAIN); - } - drop(state); - Ok(( - Arc::new( - Self { - shared: shared.clone(), - context, - owner, - backlog: core::sync::atomic::AtomicU32::new(0), - }, - GFP_KERNEL, - )?, - slot, - )) - } - - /// The shared aperture has exactly one builder. Other queues sleep without - /// holding the device mutex; recovery joins its lease before closing roots. - fn shared_compute(&self) -> Result { - loop { - let lease = self.shared.preparations.enter(false)?; - let dev = self.shared.drm_neo()?; - let uat = { - let mut state = self.shared.state.lock(); - loop { - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if self - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Err(EIO); - } - if !lease.is_current() { - return Err(EAGAIN); - } - if firmware.init.has_compute_shared() { - return firmware.init.cache_compute_vm(self.context.vm()); - } - if !firmware.queues.compute_building { - firmware.queues.compute_building = true; - break firmware.uat.clone(); - } - self.shared.changed.wait(&mut state); - } - }; - let alloc = super::object::Allocator { - dev: &dev, - uat: &uat, - }; - let mut built = Some(super::initdata::ComputeShared::new(&alloc)); - let result = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - Err(EAGAIN) - } else { - built - .take() - .ok_or(EIO)? - .and_then(|built| firmware.init.install_compute_shared(built)) - .and_then(|()| firmware.init.cache_compute_vm(self.context.vm())) - } - }; - drop(built); - { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .compute_building = false; - } - self.shared.changed.notify_all(); - drop(lease); - if result != Err(EAGAIN) { - return result; - } - } - } - - /// Uses the queue builder's existing lease. Entering another closed epoch - /// here would make recovery wait for the very builder blocked on reopening. - fn compute_scheduler( - &self, - alloc: &super::object::Allocator<'_>, - lease: &super::preparation::Lease, - ) -> Result { - { - let mut state = self.shared.state.lock(); - loop { - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - if firmware.init.has_compute_scheduler() { - return Ok(()); - } - if !firmware.queues.compute_scheduler_building { - firmware.queues.compute_scheduler_building = true; - break; - } - self.shared.changed.wait(&mut state); - } - } - let mut built = Some(super::initdata::ComputeScheduler::new(alloc)); - let result = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - Err(EAGAIN) - } else { - built - .take() - .ok_or(EIO)? - .and_then(|built| firmware.init.install_compute_scheduler(built)) - } - }; - drop(built); - { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .compute_scheduler_building = false; - } - self.shared.changed.notify_all(); - result - } - - fn build_compute(&self) -> Result { - loop { - match self.shared_compute() { - Err(error) if error == EAGAIN => continue, - result => { - result?; - break; - } - } - } - loop { - let lease = self.shared.preparations.enter(false)?; - let dev = self.shared.drm_neo()?; - let (uat, reservation) = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if self - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Err(EIO); - } - match firmware.queues.rebind_compute(self.owner, &self.context)? { - Rebind::Done => return Ok(()), - Rebind::Rearm(qid) => (firmware.uat.clone(), Err(qid)), - Rebind::Fresh => { - let reservation = firmware.queues.reserve_compute(self.owner)?; - (firmware.uat.clone(), Ok(reservation)) - } - } - }; - let alloc = super::object::Allocator { - dev: &dev, - uat: &uat, - }; - let reservation = match reservation { - Ok(reservation) => reservation, - Err(qid) => { - // Fresh zeroed backing is built without the device mutex; the - // queue may have been taken or re-armed by another owner since. - let pages = crate::g17::freelist::Pages::compute(&alloc)?; - let ring = compute::DescriptorRing::new(&alloc)?; - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - continue; - } - let (slot, page_list) = match firmware.queues.rearm_compute(qid, pages, ring) { - Err(error) if error == EAGAIN => continue, - result => result?, - }; - // The firmware consumed this slot's release before the backing - // was dropped; the row is rewritten before the queue is kicked. - firmware.init.initialize_compute_pool(slot, page_list)?; - firmware.queues.bind_rearmed(qid, self.owner, &self.context)?; - return Ok(()); - } - }; - let built = (|| { - let pool = compute::PoolBacking::new(&alloc, reservation.id)?; - { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - firmware.queues.reserved(reservation)?; - let (slot, page_list) = pool.pool_descriptor()?; - firmware.init.initialize_compute_pool(slot, page_list)?; - } - let backing = compute::Backing::new(&alloc, pool)?; - // The global scheduler is created after pool/kick backing and - // retained even if this queue's later descriptor graph fails. - self.compute_scheduler(&alloc, &lease)?; - let queue = compute::Queue::new(&alloc, backing, self.owner, self.context.clone())?; - Ok(KBox::new(queue, GFP_KERNEL)?) - })(); - let (mut built, build_error) = match built { - Ok(queue) => (Some(queue), None), - Err(error) => (None, Some(error)), - }; - let result = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - Err(EAGAIN) - } else if let Some(error) = build_error { - Err(error) - } else { - firmware.queues.install_compute(reservation, &mut built) - } - }; - // The old aliases must disappear before a waiting constructor can - // reuse the QID and its queue-local free-list identity. Keep the - // preparation lease until both mappings and reservation are gone. - drop(built); - if result.is_err() { - let mut state = self.shared.state.lock(); - (*state) - .as_mut() - .ok_or(ENODEV)? - .queues - .cancel_compute(reservation)?; - } - drop(lease); - if result != Err(EAGAIN) { - return result; - } - } - } -} - -/// One device transaction can fail each retained kick once and then retire it. -/// The physical namespace bounds the batch independently of open logical queues. -/// Collection failure leaves the engine's Active owner intact for a stopped retry. -pub(super) struct DeferredBatch { - values: KVVec, - /// Exited owners detached from idle retained compute queues and the backing - /// of vacated pools: unmapping and freeing happen in `finish`, off-lock. - bindings: KVVec, - vacated: KVVec, - /// Backing of render pools vacated for evicted idle clients. - render: KVVec, -} -impl DeferredBatch { - const DEPTH: usize = if crate::hw::t8140::queues::RENDER_DEPTH as usize - > crate::hw::t8140::queues::COMPUTE_DEPTH - { - crate::hw::t8140::queues::RENDER_DEPTH as usize - } else { - crate::hw::t8140::queues::COMPUTE_DEPTH - }; - - /// Reused by the sole event work item; no allocation on a completion pass. - fn worker() -> Result { - Self::with_capacity(2 * QIDS * Self::DEPTH, QIDS) - } - /// One publication can quarantine only its selected physical queue/pair. - /// VM-wide settlement is a separate transaction before publishing its error. - fn publication() -> Result { - Self::with_capacity(2 * Self::DEPTH, 0) - } - fn with_capacity(capacity: usize, detached: usize) -> Result { - Ok(Self { - values: KVVec::with_capacity(capacity, GFP_KERNEL)?, - bindings: KVVec::with_capacity(detached, GFP_KERNEL)?, - vacated: KVVec::with_capacity(detached, GFP_KERNEL)?, - render: KVVec::with_capacity( - if detached != 0 { crate::hw::t8140::queues::RENDER_SLOTS } else { 0 }, - GFP_KERNEL, - )?, - }) - } - /// Keeps the detached owner's unmapping and VM teardown out of the device - /// mutex. Without reserved room the binding is dropped here instead. - pub(super) fn defer_binding(&mut self, binding: super::queue::compute::Binding) { - if let Err(binding) = self.bindings.push_within_capacity(binding) { - drop(binding); - } - } - /// Room for one more vacated render backing; the worker vacates no more. - pub(super) fn render_backing_room(&self) -> bool { - self.render.len() < self.render.capacity() - } - pub(super) fn defer_render_backing(&mut self, backing: super::freelist::RenderBacking) { - if let Err(backing) = self.render.push_within_capacity(backing) { - drop(backing); - } - } - pub(super) fn defer_vacated(&mut self, vacated: super::queue::compute::Vacated) { - if let Err(vacated) = self.vacated.push_within_capacity(vacated) { - drop(vacated); - } - } - /// A recovery pass can fail and retire each retained command at most once. - /// Reserve before changing queues, plus a full publication for its caller. - /// Healthy submissions keep the small publication collector unchanged. - fn reserve_replays(&mut self, commands: usize) -> Result { - self.values - .reserve(2 * commands + 2 * Self::DEPTH, GFP_KERNEL)?; - Ok(()) - } - pub(super) fn push(&mut self, value: super::completion::Deferred) -> Result { - use super::completion::Deferred; - // Publishers may acquire the device mutex as soon as this pass drops - // it. Expose terminal VM status now, before off-lock fence callbacks. - // Recovery-spared and cancelled commands never poison their VM. - if let Deferred::FailedOwned(completion, error) = &value { - if *error != ECANCELED && !completion.spared() { - completion.status().record(*error); - } - } - self.values.push_within_capacity(value).map_err(|_| EIO)?; - // A later publisher can borrow this pair before finish signals its old - // fence. Publish the exact completion's witness only after collection - // succeeds; failure fences and rejected events are not retirement. - if let Some( - Deferred::Retired(completion, _) - | Deferred::Drained(completion, _) - | Deferred::Release(completion), - ) = self.values.last() - { - completion.note_retirement_queued(); - } - Ok(()) - } - /// Called only after releasing the device mutex. - pub(super) fn finish(&mut self) { - for value in self.values.drain_all() { - value.finish(); - } - for binding in self.bindings.drain_all() { - drop(binding); - } - for vacated in self.vacated.drain_all() { - drop(vacated); - } - for backing in self.render.drain_all() { - drop(backing); - } - } -} - -struct ComputeHost<'a> { - init: &'a super::initdata::InitData, - primary: &'a mut super::Coprocessor, - effort: &'a mut super::power::Effort, - qids: &'a mut kick::QueueIds, - accounting: &'a mut qos::Accounting, - generation: u64, -} -impl compute::Host for ComputeHost<'_> { - fn next_compute_publication(&mut self) -> Result { - self.qids.next_compute_publication() - } - fn epoch(&self) -> Result<(u64, u32)> { - Ok(( - self.generation, - super::recovery::Memory::recovery_state(self.init)?, - )) - } - fn prepare_compute_shared(&mut self) -> Result { - self.init.activate_compute() - } - fn qos_publish( - &mut self, - owner: qos::Owner, - scheduler: u64, - policy: super::fw::queue::Policy, - ) -> Result { - self.accounting.publish( - &qos::View::new(self.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) - } - fn qos_cancel(&mut self, publication: qos::Publication) -> Result { - self.accounting - .cancel(&qos::View::new(self.init.qos()?)?, publication) - } - fn publish_qid(&mut self, id: kick::Id) -> Result { - self.qids.publish(id) - } - fn publish_outer(&mut self, priority: u8, slot: &super::fw::channels::WorkSlot) -> Result { - self.init.publish_work(priority, slot).map(|_| ()) - } - fn notify(&mut self, message: u64) -> Result { - self.primary.notify(message) - } - fn note_submission(&mut self) -> Result { - self.effort.submit(); - super::Firmware::apply_shared_effort(self.init, self.effort, super::power::FULL_EFFORT) - } -} - -impl super::Firmware { - fn publish_compute( - &mut self, - owner: u64, - packet: Arc, - dependencies: &[super::fw::kick::KickDependency], - deferred: &mut DeferredBatch, - ) -> Result { - if self.recovery.pending() { - return Err(EAGAIN); - } - // No descriptor, accounting or inner producer changes precede this - // capacity check. The same device lock covers queue.publish below. - if !self - .init - .work_ready(packet.context.policy().priority(), DataMaster::Compute)? - { - return Err(EAGAIN); - } - let queues = &mut self.queues; - let queue = queues - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) - .ok_or(EIO)?; - let mut host = ComputeHost { - init: &self.init, - primary: &mut self.primary, - effort: &mut self.effort, - qids: &mut queues.qids, - accounting: &mut queues.accounting, - generation: self.recovery.generation(), - }; - queue.publish(packet, dependencies, &mut host, &mut |event| { - deferred.push(event) - }) - } -} - -impl Registry { - fn release_compute_witness( - &mut self, - init: &super::initdata::InitData, - owner: u64, - deferred: &mut DeferredBatch, - ) -> Result { - let Some(queue) = self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - .find(|queue| queue.owner() == Some(owner)) - else { - return Ok(false); - }; - if !queue.spared_pending() { - return Ok(false); - } - let view = qos::View::new(init.qos()?)?; - queue.release_witnessed( - super::recovery::Memory::recovery_state(init)?, - |owner| self.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; - Ok(queue.spared_pending()) - } - - fn fail_compute_vm( - &mut self, - status: &Arc, - error: Error, - deferred: &mut DeferredBatch, - ) -> Result { - for queue in self - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - { - if queue.in_flight() - && queue - .context() - .is_some_and(|context| Arc::ptr_eq(context.status(), status)) - { - queue.quarantine(error, &mut |event| deferred.push(event))?; - } - } - Ok(()) - } -} - -impl super::Firmware { - /// Latch one shared snapshot for all selected queues before settling any VM. - /// A later queue's exact success in this batch wins over a peer's failure. - fn scan_compute(&mut self, masks: Option<[u64; 2]>, deferred: &mut DeferredBatch) -> Result { - let state = super::recovery::Memory::recovery_state(&self.init)?; - let generation = self.recovery.generation(); - let registry = &mut self.queues; - registry.observations.clear(); - let mut requested = false; - for queue in registry - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - { - requested |= queue.request_observations(&mut registry.observations, masks)?; - } - if requested { - self.init - .scan_compute_completions(|record| registry.observations.fold(record))?; - } - let view = qos::View::new(self.init.qos()?)?; - for queue in registry - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - { - queue.latch_observations(&mut registry.observations, masks)?; - queue.retire( - masks, - generation, - state, - |owner| registry.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; - } - // All compute observations in this batch are latched before a VM - // failure can settle its render peers. Exact successes win first. - for index in 0..self.queues.compute.len() { - let terminal = self.queues.compute[index] - .as_mut() - .and_then(|entry| entry.queue.as_deref_mut()) - .and_then(compute::Queue::take_terminal); - if let Some((status, error)) = terminal { - status.record(error); - self.settle_render_vm(&status, error, deferred)?; - self.queues.fail_compute_vm(&status, error, deferred)?; - } - } - let view = qos::View::new(self.init.qos()?)?; - let registry = &mut self.queues; - for queue in registry - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - { - queue.release_witnessed( - state, - |owner| registry.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; - } - Ok(()) - } -} - -impl Backend { - fn ensure_compute(&self) -> Result { - use kernel::time::{delay::fsleep, Delta}; - const SPARED_WAIT_MS: usize = 200; - let mut deferred = DeferredBatch::publication()?; - for _ in 0..SPARED_WAIT_MS { - let pending = (|| { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - firmware.replay_compute(&mut deferred)?; - firmware - .queues - .release_compute_witness(&firmware.init, self.owner, &mut deferred) - })(); - deferred.finish(); - let pending = pending?; - if !pending { - return self.build_compute(); - } - // Other owners continue submitting while this owner gives its - // queue-local drain witness time to arrive. - fsleep(Delta::from_millis(1)); - } - // No new command has been accepted. Keep the spared queue and its - // retained work intact until a later admission observes its witness. - Err(EBUSY) - } - - /// Called after taking this packet's publication claim. All waits and work - /// state allocation precede the device mutex; only the checked epoch commits. - fn compute_packet( - &self, - packet: &Arc, - deferred: &mut DeferredBatch, - ) -> Result { - packet.check_dependencies()?; - let mut recovery_polls = 0; - let mut outer_wait = None; - loop { - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let lease = self.shared.preparations.enter(false)?; - packet.completion.work_state()?.acquire()?; - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - drop(state); - continue; - } - if packet.cancel_requested() { - return Err(ECANCELED); - } - if self - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Err(EIO); - } - let recovery_state = super::recovery::Memory::recovery_state(&firmware.init)?; - if firmware.recovery.pending() || recovery_state != 0 { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; - } - firmware.replay_compute(deferred)?; - let dependencies = firmware.queues.render_prefix(self.owner, packet)?; - let result = firmware.publish_compute( - self.owner, - packet.clone(), - dependencies.as_slice(), - deferred, - ); - if matches!(result, Err(EBUSY) | Err(EAGAIN)) - && !packet.is_published() - && (firmware.recovery.pending() - || super::recovery::Memory::recovery_state(&firmware.init)? != 0) - { - if recovery_polls == RECOVERY_SETTLE_POLLS { - return Err(EBUSY); - } - recovery_polls += 1; - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; - } - if matches!(result, Err(EAGAIN)) && !packet.is_published() { - let now = now_ns(); - let since = *outer_wait.get_or_insert(now); - if now.saturating_sub(since) >= WORK_RING_WAIT_NS { - return Err(EBUSY); - } - // Backpressure owns no firmware-visible attempt. Other queues - // can submit and the event worker can retire while we sleep. - drop(state); - drop(lease); - self.shared.queue_events(); - kernel::time::delay::fsleep(kernel::time::Delta::from_millis(1)); - continue; - } - return result; - } - } -} - -impl super::Firmware { - /// Both processors are stopped and their owner detached before this call. - /// Error fences release their command mappings without the device mutex - /// or an allocation-dependent collector. - fn finish_stopped_queues(&mut self) { - self.stop_render(); - for queue in self - .queues - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - { - let result = queue.stopped(&mut |event| { - event.finish(); - Ok(()) - }); - if let Err(error) = result { - dev_err!( - self.primary.state.shared.dev.as_ref(), - "Could not settle stopped compute queue: {:?}\n", - error - ); - } - } - self.queues - .admission - .release_retained_owners_after_processor_stop(); - } -} - -impl super::Shared { - /// Joins construction and complete host teardown, including RTKit drops. - /// The stop-only mutex outlives detached firmware destruction, so unbind - /// cannot return while another stop caller still owns mailbox clients. - /// Completion callbacks remain outside the device mutex. Direct handlers - /// do not synchronously request or join stop; this path does not join - /// scheduler workers. - pub(super) fn stop_queues(&self) { - self.preparations.wait_drained(); - let _stopping = self.stop.lock(); - let firmware = { - let mut state = self.state.lock(); - if let Some(firmware) = (*state).as_deref_mut() { - firmware.secondary.stop(); - firmware.primary.stop(); - } - state.take() - }; - if let Some(mut firmware) = firmware { - firmware.finish_stopped_queues(); - // RTKit destruction still dereferences the mailbox provider. - drop(firmware); - } - } -} - -/// Idle retained compute queues that keep their USC pool populated; older idle -/// pools give their backing back. -const COMPUTE_WARM_CAP: usize = 4; - -impl super::Firmware { - /// Event-worker service step: witness consumed pool releases, then release the - /// oldest idle retained pools beyond the warm cap. - /// Detached owner bindings and vacated pool backing go to `deferred`, which - /// the worker finishes after releasing the device mutex. - pub(in crate::g17) fn service_compute_pools( - &mut self, - deferred: &mut DeferredBatch, - ) -> Result { - for queue in self - .queues - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - { - queue.recycle_killed()?; - if let Some(binding) = queue.detach_exited_owner() { - deferred.defer_binding(binding); - } - } - self.queues.observe_pool_releases(&self.init, deferred)?; - if self.render_control_backpressured() { - return Ok(()); - } - let published = self.queues.vacate_idle_pools(&self.init, COMPUTE_WARM_CAP)?; - if published != 0 { - fence(Ordering::SeqCst); - self.primary.notify(crate::g17::MSG_CONTROL_NOTIFY)?; - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/backend.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/backend.rs deleted file mode 100644 index 63a8a0db669c12..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/backend.rs +++ /dev/null @@ -1,190 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Scheduler callbacks and the off-lock fence boundary of a logical queue. - -use super::{Backend, DeferredBatch}; -use crate::g17::{ - completion::Deferred, - job::{self, Engine, Packet}, -}; -use core::sync::atomic::Ordering; -use kernel::{dma_fence::Fence, drm_neo::sched, prelude::*, sync::Arc}; - -impl Backend { - pub(super) fn release_backlog(&self, packet: &Packet) { - if packet.take_backlog() { - self.backlog.fetch_sub(1, Ordering::AcqRel); - } - } - - fn stop_after_error(&self) { - self.shared.lose_device_quietly(); - self.shared.stop_queues(); - } - - /// Complete all failure fanout before signalling this packet's unpublished - /// error. Visible failures keep the engine's retained owner through a witness. - pub(super) fn finish_publication( - &self, - packet: &Arc, - result: Result, - deferred: &mut DeferredBatch, - ) -> Result { - if let Err(error) = result { - if [EIO, ETIMEDOUT, ENODEV].contains(&error) || packet.completion.status().get() != 0 { - ::fail_vm(self, error); - } - deferred.finish(); - if !packet.is_published() { - packet.completion.fail_unpublished(error); - } - Err(error) - } else { - deferred.finish(); - Ok(packet.completion.scheduler_fence()) - } - } - - fn release_queue(&self) -> Result { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - // Retain a published scheduler before a never-kicked physical graph is - // removed. The teardown entry owns its root until control consumption. - firmware - .queues - .teardown - .close(&self.context, super::now_ns())?; - firmware.queues.release_compute(self.owner)?; - firmware.queues.release_render(self.owner)?; - firmware.queues.admission.release_owner(self.owner); - drop(state); - self.shared.queue_events(); - Ok(()) - } -} - -impl job::Backend for Backend { - fn feed(&self) -> Option> { - Some(self.shared.feed.clone()) - } - - fn ensure_compute(&self) -> Result { - Backend::ensure_compute(self) - } - - fn fail_vm(&self, error: Error) { - self.context.status().record(error); - let mut deferred = match DeferredBatch::worker() { - Ok(batch) => batch, - Err(_) => { - self.stop_after_error(); - return; - } - }; - let result = (|| { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - // Final observations win over this VM's terminal fanout. - firmware.scan_compute(None, &mut deferred)?; - firmware.settle_render_vm(self.context.status(), error, &mut deferred)?; - firmware - .queues - .fail_compute_vm(self.context.status(), error, &mut deferred) - })(); - deferred.finish(); - if result.is_err() { - self.stop_after_error(); - } - } - - fn release_owner(&self) { - if self.release_queue().is_err() { - self.stop_after_error(); - } - } - - /// Firmware kick parents order the same engine. Explicit cross-engine and - /// input dependencies were already attached to the DRM scheduled job. - fn prepare(&self, _packet: &Packet) -> Option { - None - } - - fn publish(&self, packet: Arc) -> Result { - if let Some(result) = packet.early_result() { - return result.map(|()| packet.completion.scheduler_fence()); - } - let mut deferred = match DeferredBatch::publication() { - Ok(batch) => batch, - Err(error) => { - self.release_backlog(&packet); - if packet.completion.status().get() != 0 { - ::fail_vm(self, error); - } - packet.completion.fail_unpublished(error); - return Err(error); - } - }; - let result = if !packet.begin_publication() { - Err(ECANCELED) - } else if packet.completion.status().get() != 0 { - Err(EIO) - } else { - match packet.engine() { - Engine::Compute => self.compute_packet(&packet, &mut deferred), - Engine::Render => self.render_packet(&packet, &mut deferred), - } - }; - self.release_backlog(&packet); - self.finish_publication(&packet, result, &mut deferred) - } - - fn timed_out(&self, packet: Arc) -> sched::Status { - self.release_backlog(&packet); - let result = match packet.engine() { - Engine::Compute => self.timeout_compute(&packet), - Engine::Render => self.timeout_render(&packet), - }; - if result.is_err() { - self.stop_after_error(); - } - sched::Status::NoHang - } - - fn cancel(&self, packet: Arc) { - self.release_backlog(&packet); - if packet.request_cancel() { - Deferred::Unpublished(packet.completion.clone(), ECANCELED).finish(); - return; - } - let result = match packet.engine() { - Engine::Compute => self.cancel_compute(&packet), - Engine::Render => self.cancel_render(&packet), - }; - if result.is_err() { - self.stop_after_error(); - } - } - - fn defer_timeout(&self, packet: &Arc, renewals: u8) -> bool { - if packet.engine() == Engine::Compute { - return self.defer_compute_timeout(packet, renewals); - } - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .is_some_and(|firmware| firmware.queues.defer_render_timeout(packet, ETIMEDOUT)) - } - - fn try_early(&self, packet: &Arc) -> bool { - self.try_render_early(packet) - } - - fn count_backlog(&self, packet: &Packet) { - packet.count_backlog(); - self.backlog.fetch_add(1, Ordering::AcqRel); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/compute_control.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/compute_control.rs deleted file mode 100644 index dac4ca48a34aec..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/compute_control.rs +++ /dev/null @@ -1,96 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Scheduler cancellation and final retirement checks for retained compute work. - -use super::{Backend, DeferredBatch}; -use crate::g17::job::{self, Packet}; -use kernel::{prelude::*, sync::Arc}; - -impl Backend { - /// A final retirement pass precedes bounded cross-VM timeout attribution. - /// All completion callbacks run after releasing the device mutex. - pub(super) fn defer_compute_timeout(&self, packet: &Arc, renewals: u8) -> bool { - use crate::g17::{recovery::Class, timeout::{self, Decision}}; - let Ok(mut deferred) = DeferredBatch::worker() else { - return false; - }; - let result = (|| -> Result { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(Decision::Own); - }; - firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute(self.owner) - .filter(|queue| queue.can_classify_timeout(packet)) else { - return Ok(Decision::Own); - }; - let Some(own) = queue.timeout_head() else { - return Ok(Decision::Own); - }; - let others = firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|other| other.owner() != Some(self.owner)) - .filter_map(|other| other.timeout_head()); - let decision = timeout::classify(own, renewals, others); - if decision == Decision::Spare { - firmware.queues.compute(self.owner).ok_or(EIO)? - .classify(&|_| Class::Spared); - } - if decision != Decision::Own && (renewals == 0 || decision == Decision::Spare) { - dev_warn!(self.shared.dev.as_ref(), - "Compute watchdog owner={} publication={} renewals={} decision={:?} behind earlier work of another VM\n", - self.owner, own.published, renewals, decision); - } - Ok(decision) - })(); - deferred.finish(); - matches!(result, Ok(Decision::Defer)) - } - - /// The caller owns cancellation of a packet that never claimed publication. - /// A missing retained packet may already have a healthy callback waiting off-lock. - pub(super) fn timeout_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; - let result = (|| { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - // Teardown owns the detached queues and will settle their callbacks. - return Ok(None); - }; - firmware.scan_compute(None, &mut deferred)?; - let Some(queue) = firmware.queues.compute_mut(self.owner) else { - return Ok(None); - }; - if !queue.owns_packet(packet) { - return Ok(None); - } - queue.timeout(&mut |event| deferred.push(event)) - })(); - if result.as_ref().is_ok_and(|status| status.is_some()) { - // VM status and peer settlement precede signalling this timeout. The backend - // owns the same logical VM as the retained packet matched under the mutex. - ::fail_vm(self, ETIMEDOUT); - } - deferred.finish(); - result.map(|_| ()) - } - - /// Cancel only a scheduler waiter. Visible mappings stay pinned until their witness; - /// cancellation neither quarantines hardware admission nor fans out a VM failure. - pub(super) fn cancel_compute(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; - let result = (|| { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - firmware.scan_compute(None, &mut deferred)?; - if let Some(queue) = firmware.queues.compute_mut(self.owner) { - queue.cancel(packet, &mut |event| deferred.push(event))?; - } - Ok(()) - })(); - deferred.finish(); - result - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/compute_exit.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/compute_exit.rs deleted file mode 100644 index 984b6c1f643ebc..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/compute_exit.rs +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Process-exit controls for closed contexts whose compute commands remain owned. - -use super::{now_ns, DeferredBatch}; -use crate::g17::{ - channel::Rings, - context::Context, - fw::channels::{ContextKill, ControlRecord}, - queue::compute::exit, - recovery::Memory as _, - Firmware, MSG_CONTROL_NOTIFY, -}; -use core::sync::atomic::{fence, Ordering}; -use kernel::{prelude::*, sync::Arc}; - -impl super::Registry { - /// ACK storage belongs to the already-retained transaction, so a full event - /// batch cannot overflow a separate cache. No allocation or wait is needed. - pub(super) fn acknowledge_compute_exit(&mut self, cookie: u64) { - for entry in self.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { - if queue.acknowledge_exit(cookie) { - return; - } - } - } - } -} - -impl Firmware { - pub(in crate::g17) fn service_compute_exit(&mut self, deferred: &mut DeferredBatch) -> Result { - let recovery_closed = self.init.recovery_state()? == 0; - let view = crate::g17::qos::View::new(self.init.qos()?)?; - for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { - queue.settle_exit( - recovery_closed, - &mut |owner| self.queues.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; - } - } - if self.recovery.pending() || !recovery_closed { - return Ok(()); - } - let now = now_ns(); - // Publication has already retained each kill. Retry only the doorbell; - // failed attempts do not extend its bounded polling window. - for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { - let _ = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)); - } - } - for index in 0..self.queues.teardown.len() { - let Some((context, age)) = self.queues.teardown.closed(index, now) else { - continue; - }; - if !self.queues.render.teardown_idle(&context)? - || !self.queues.render.teardown_pools_released(&context) - { - continue; - } - if let Err(error) = self.publish_compute_exit(&context, age, now) { - dev_err!( - self.primary.state.shared.dev.as_ref(), - "Scheduler {} exit request failed: {:?}\n", - context.id(), - error - ); - } - } - Ok(()) - } - - fn publish_compute_exit(&mut self, context: &Arc, age: u64, now: u64) -> Result { - let Some(index) = self.queues.compute.iter().position(|entry| { - entry - .as_ref() - .and_then(|entry| entry.queue.as_deref()) - .is_some_and(|queue| queue.exit_candidate(context, age)) - }) else { - return Ok(()); - }; - if self.render_control_backpressured() { - return Ok(()); - } - let queue = self.queues.compute[index] - .as_ref() - .and_then(|entry| entry.queue.as_deref()) - .ok_or(EIO)?; - let Some(owner) = queue.exit_owner(context) else { - return Ok(()); - }; - let target = queue.qid(); - let (mut own, mut foreign, slots) = self.queues.render.exit_scope(context); - for entry in self.queues.compute.iter().flatten() { - if let Some(queue) = entry.queue.as_deref() { - let bit = 1u128 << queue.qid(); - if queue.context().is_some_and(|bound| Arc::ptr_eq(bound, context)) { - own |= bit; - } else { - foreign |= bit; - } - } - } - let state = exit::State::read(context)?; - let Some(qids) = state.scope(target, own, foreign) else { - return Ok(()); - }; - let mut hold = 0u64; - for slot in 0..crate::hw::t8140::queues::RENDER_SLOTS as u8 { - if slots & (1u64 << slot) != 0 && self.queues.render.qids_for_slot(slot) & qids != 0 { - hold |= 1u64 << slot; - } - } - let cookie = exit::next_cookie()?; - if hold != 0 - && !self - .queues - .admission - .hold_render_slots_for_exit_kill(owner, hold) - { - return Ok(()); - } - let record = ControlRecord::ContextKill(ContextKill::new(context.scheduler_va(), cookie)); - if let Err(error) = Rings::publish_control(&self.init, &record) { - self.queues.admission.release_exit_kill_hold(hold); - return if error == EAGAIN { Ok(()) } else { Err(error) }; - } - // Publication is irrevocable even if its doorbell fails. Keep both - // the killed graph and the held render identities until processor stop. - let queue = self.queues.compute[index] - .as_mut() - .and_then(|entry| entry.queue.as_deref_mut()) - .ok_or(EIO)?; - queue.record_exit(cookie, now); - fence(Ordering::SeqCst); - if let Err(error) = queue.announce_exit(now, || self.primary.notify(MSG_CONTROL_NOTIFY)) { - dev_warn!( - self.primary.state.shared.dev.as_ref(), - "Scheduler {} exit announcement failed: {:?}\n", - context.id(), - error - ); - } - Ok(()) - } - - pub(in crate::g17) fn compute_exit_polling(&self, now: u64) -> bool { - if self - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_polling(now)) - { - return true; - } - (0..self.queues.teardown.len()).any(|index| { - self.queues - .teardown - .closed(index, now) - .is_some_and(|(context, age)| { - age < exit::POLL_NS - && self - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.exit_candidate(&context, u64::MAX)) - }) - }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/compute_replay.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/compute_replay.rs deleted file mode 100644 index 0d4a29e779b4a8..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/compute_replay.rs +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Ordered republication of witnessed, unexecuted compute commands. - -use super::DeferredBatch; -use crate::g17::{completion::Deferred, qos, recovery, Firmware}; -use kernel::prelude::*; - -impl Firmware { - /// Called only from an outer worker or client submission, never from event - /// dispatch or a recovery callback nested within publication. All candidate - /// queues are observed before any retained packet is published again. - pub(in crate::g17) fn replay_compute(&mut self, deferred: &mut DeferredBatch) -> Result { - if self.recovery.pending() { - return Ok(()); - } - let state = recovery::Memory::recovery_state(&self.init)?; - if state != 0 { - return Ok(()); - } - let commands = self - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .map(|queue| queue.replay_commands()) - .sum::(); - if commands == 0 { - self.queues.replay_outer_wait = None; - return Ok(()); - } - deferred.reserve_replays(commands)?; - let view = qos::View::new(self.init.qos()?)?; - for entry in self.queues.compute.iter_mut().flatten() { - if let Some(queue) = entry.queue.as_deref_mut() { - queue.prepare_replays( - state, - |owner| self.queues.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; - } - } - for index in 0..self.queues.compute.len() { - loop { - if self.recovery.pending() || recovery::Memory::recovery_state(&self.init)? != 0 { - return Ok(()); - } - let Some(queue) = self.queues.compute[index] - .as_ref() - .and_then(|entry| entry.queue.as_deref()) - else { - break; - }; - let Some(packet) = queue.replay_front() else { - break; - }; - let owner = queue.owner(); - let outer_ready = self.init.work_ready( - packet.context.policy().priority(), - crate::g17::fw::queue::DataMaster::Compute, - )?; - if !outer_ready { - let now = super::now_ns(); - let since = *self.queues.replay_outer_wait.get_or_insert(now); - if now.saturating_sub(since) < super::WORK_RING_WAIT_NS { - // Keep the FIFO and its replay credit untouched. The - // event worker schedules the existing delayed poll. - return Ok(()); - } - } else { - self.queues.replay_outer_wait = None; - } - - let result = if !outer_ready { - // This replay has no live firmware ownership. Persistent - // backpressure uses the existing failed-replay retirement. - Err(EBUSY) - } else { (|| { - self.queues.compute[index] - .as_mut() - .and_then(|entry| entry.queue.as_deref_mut()) - .ok_or(EIO)? - .prepare_replay(&packet)?; - let owner = owner.ok_or(EIO)?; - packet.check_dependencies()?; - let dependencies = self.queues.render_prefix(owner, &packet)?; - self.publish_compute(owner, packet.clone(), dependencies.as_slice(), deferred) - })() }; - let recovery_wait = matches!(result, Err(EAGAIN) | Err(EBUSY)) - && (self.recovery.pending() - || recovery::Memory::recovery_state(&self.init)? != 0); - let queue = self.queues.compute[index] - .as_mut() - .and_then(|entry| entry.queue.as_deref_mut()) - .ok_or(EIO)?; - if let Err(error) = result { - if recovery_wait && !queue.owns_packet(&packet) { - // Keep this prepared attempt and its one timeout credit - // while the outer worker closes the new recovery. - return Ok(()); - } - queue.fail_replays(error); - if !queue.owns_packet(&packet) { - deferred - .push(Deferred::Retired(packet.completion.clone(), Err(ENODATA)))?; - } - } - queue.finish_replay(&packet)?; - } - } - self.queues.replay_outer_wait = None; - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/events.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/events.rs deleted file mode 100644 index 8944327b84c5c3..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/events.rs +++ /dev/null @@ -1,475 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Event and restart callbacks borrow the complete firmware owner. The reusable -//! completion collector belongs to the worker and is drained outside its mutex. - -use super::DeferredBatch; -use crate::g17::{ - channel, event, - fw::channels::{Event, FreeListCompletion, TvbGrowRequest}, - recovery, - status::VmStatus, - Firmware, Role, -}; -use kernel::{prelude::*, sync::Arc}; - -struct Service<'a> { - firmware: &'a mut Firmware, - deferred: &'a mut DeferredBatch, - render_masks: [u64; 2], - settlement_error: Option, -} - -impl event::Transport for Service<'_> { - fn trace(&mut self, role: Role) { - let firmware = &mut self.firmware; - firmware - .rings - .drain_trace(&firmware.primary.state.shared.dev, &firmware.init, role); - } - fn statistics(&mut self, role: Role) { - let firmware = &mut self.firmware; - firmware - .rings - .drain_statistics(&firmware.primary.state.shared.dev, &firmware.init, role); - } - fn drain( - &mut self, - role: Role, - mut consume: impl FnMut(&mut Self, Event) -> Result, - ) -> Result { - let mut events = channel::Events::new(role); - while let Some(event) = - events.next(&self.firmware.primary.state.shared.dev, &self.firmware.init)? - { - if !consume(self, event)? { - break; - } - } - Ok(()) - } - fn recovery_state(&self) -> Result { - recovery::Memory::recovery_state(&self.firmware.init) - } -} - -impl event::Host for Service<'_> { - fn recovery(&self) -> &recovery::State { - &self.firmware.recovery - } - fn recovery_mut(&mut self) -> &mut recovery::State { - &mut self.firmware.recovery - } -} - -impl event::Queues for Service<'_> { - fn context_killed(&mut self, cookie: u64) { - self.firmware.queues.acknowledge_compute_exit(cookie); - } - fn flush_tvb_reply(&mut self) -> Result { - self.firmware.flush_tvb_reply() - } - fn flush_freelist_release(&mut self) -> Result { - self.firmware.flush_freelist_release() - } - fn observe_render_releases(&mut self) -> Result { - self.firmware.observe_render_releases() - } - fn control_backpressured(&self) -> bool { - self.firmware.render_control_backpressured() - } - fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { - self.firmware.queue_tvb_grow(request) - } - fn complete_freelist_grow(&mut self, completion: FreeListCompletion) -> Result { - self.firmware.complete_freelist_grow(completion) - } - fn retain_render_completions(&mut self, masks: [u64; 2]) { - // Inline timeout and drain callers may discard the worker hints. The - // queue owner retains stamps until its borrowed pair returns to scan. - self.firmware.queues.retain_render_completions(masks); - self.render_masks[0] |= masks[0]; - self.render_masks[1] |= masks[1]; - } - fn complete_compute(&mut self, masks: [u64; 2]) -> Result { - // A batch without a queue identity still observes completions whose - // notification was coalesced; identified batches retain QID filtering. - let selected = (masks != [0; 2]).then_some(masks); - self.firmware.scan_compute(selected, self.deferred) - } -} - -impl recovery::Host for Service<'_> { - fn recovery(&self) -> &recovery::State { - &self.firmware.recovery - } - fn recovery_mut(&mut self) -> &mut recovery::State { - &mut self.firmware.recovery - } - fn report_recovery(&self, generation: u64, blamed: Option, sources: &recovery::Sources) { - dev_warn!( - self.firmware.primary.state.shared.dev.as_ref(), - "G17 firmware recovery: generation={} qid={:?} reason={:?} slots={:?}\n", - generation, - blamed, - sources.reason, - sources.diagnostics() - ); - let Some(qid) = blamed else { - return; - }; - for queue in self.firmware.queues.compute.iter().flatten() - .filter_map(|entry| entry.queue.as_deref()) - .filter(|queue| queue.qid() == qid) - { - dev_warn!( - self.firmware.primary.state.shared.dev.as_ref(), - "G17 recovery compute: qid={} (owner, context/generation, ordinal, active, cursors, stamp, CDM/samplers/scratch)={:?}\n", - qid, - queue.recovery_diagnostics() - ); - } - if let Some((qids, owner, context, generation)) = - self.firmware.queues.render_recovery_diagnostics(qid) - { - dev_warn!( - self.firmware.primary.state.shared.dev.as_ref(), - "G17 recovery render: qids={:?} owner={:?} context={} generation={}\n", - qids, - owner, - context, - generation - ); - } - } -} - -impl recovery::Memory for Service<'_> { - fn recovery_state(&self) -> Result { - self.firmware.init.recovery_state() - } - fn host_recovery(&self) -> Result { - self.firmware.init.host_recovery() - } - fn fault_sources(&self) -> Result { - self.firmware.init.fault_sources() - } - - fn transition(&self, expected: u32, next: u32) -> Result { - self.firmware.init.transition(expected, next) - } - fn clear_timestamps(&self) -> Result { - self.firmware.init.clear_timestamps() - } - fn entries(&self) -> Result { - self.firmware.init.entries() - } - fn wait_tick(&self) { - self.firmware.init.wait_tick(); - } -} - -impl recovery::Queues for Service<'_> { - fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { - self.firmware.queues.render_pass_started(qid, stamp) - } - fn publish_recovery_targets(&self) -> Option { - let mut failure = self.firmware.queues.publish_render_recovery_targets(); - for queue in self - .firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - { - if let Err(error) = queue.publish_recovery_target() { - failure = failure.or(Some(error)); - } - } - core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); - failure - } - fn render_scheduler_active(&self) -> bool { - self.firmware.queues.render_scheduler_active() || !self.firmware.queues.teardown.is_empty() - } - fn oldest_spared(&self, qid: u8) -> Option { - self.firmware.queues.render_oldest_spared(qid).or_else(|| { - self.firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .find(|queue| queue.qid() == qid) - .and_then(|queue| queue.oldest_spared()) - }) - } - fn visit_owners(&self, qid: u8, visit: &mut dyn FnMut(&Arc)) { - self.firmware.queues.visit_render_owners(qid, visit); - for queue in self - .firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - { - if queue.qid() == qid { - queue.visit_owners(visit); - } - } - } - fn classify_work(&mut self, class: &dyn Fn(&Arc) -> recovery::Class) -> u128 { - for queue in self - .firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - { - queue.classify(class); - } - self.firmware.queues.classify_render(class) - } - fn settle_blamed_render(&mut self, qid: u8, error: Error) { - let result = self - .firmware - .settle_blamed_render(qid, error, self.deferred); - self.settlement_error = self.settlement_error.or(result.err()); - } - fn settle_cancelled_renders(&mut self, qids: u128) { - let result = self.firmware.settle_cancelled_renders(qids, self.deferred); - self.settlement_error = self.settlement_error.or(result.err()); - } - fn rescan_compute(&mut self) -> Result { - self.firmware.scan_compute(None, self.deferred) - } - fn release_retired_compute(&mut self) { - let result = self.firmware.release_compute_witnesses(self.deferred); - self.settlement_error = self.settlement_error.or(result.err()); - } -} - -impl Firmware { - fn release_compute_witnesses(&mut self, deferred: &mut DeferredBatch) -> Result { - let state = recovery::Memory::recovery_state(&self.init)?; - let view = crate::g17::qos::View::new(self.init.qos()?)?; - let registry = &mut self.queues; - for queue in registry - .compute - .iter_mut() - .flatten() - .filter_map(|entry| entry.queue.as_deref_mut()) - { - queue.release_witnessed( - state, - |owner| registry.accounting.complete(&view, &[owner]), - &mut |event| deferred.push(event), - )?; - } - Ok(()) - } - - pub(super) fn service_runtime_events( - &mut self, - deferred: &mut DeferredBatch, - ) -> Result<(event::Pending, [u64; 2])> { - let mut service = Service { - firmware: self, - deferred, - render_masks: [0; 2], - settlement_error: None, - }; - let pending = event::service(&mut service)?; - if pending.recovery { - // Inline timeout/drain service can consume the restart record too. - // Close publication in this same critical section and ensure the - // event worker will join builders before the restart handshake. - let shared = &service.firmware.primary.state.shared; - shared.preparations.close(); - shared.queue_events(); - } - Ok((pending, service.render_masks)) - } - - /// Preparation admission is already closed and all builders have joined. - pub(super) fn service_runtime_recovery(&mut self, deferred: &mut DeferredBatch) -> Result { - let shared = self.primary.state.shared.clone(); - let mut service = Service { - firmware: self, - deferred, - render_masks: [0; 2], - settlement_error: None, - }; - let result = recovery::service(&mut service, || { - shared.crashed.load(core::sync::atomic::Ordering::Acquire) - })?; - if let Some(error) = result.target_error { - dev_warn!( - shared.dev.as_ref(), - "Recovery target copy failed: {:?}\n", - error - ); - } - if let Some(error) = result.scan_error { - dev_warn!( - shared.dev.as_ref(), - "Post-recovery compute scan failed: {:?}\n", - error - ); - } - if let Some(error) = service.settlement_error { - return Err(error); - } - Ok(()) - } -} - -impl crate::g17::Shared { - /// The single event work item owns this preallocated collector until all - /// callbacks are finished. Shutdown can detach Firmware during an off-lock - /// boundary; the collector itself still owns every deferred completion. - pub(in crate::g17) fn run_runtime_events(self: &Arc) { - if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - self.stop_queues(); - return; - } - let mut deferred = { - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return; - }; - let Some(deferred) = firmware.queues.worker_deferred.take() else { - drop(state); - self.lose_device_quietly(); - return; - }; - deferred - }; - let result = self.runtime_event_turn(&mut deferred); - deferred.finish(); - self.queue_feed(); - if let Some(firmware) = (*self.state.lock()).as_deref_mut() { - firmware.queues.worker_deferred = Some(deferred); - } - if let Err(error) = result { - dev_err!(self.dev.as_ref(), "GPU event service failed: {:?}\n", error); - self.lose_device_quietly(); - self.stop_queues(); - } - } - - fn runtime_event_turn(self: &Arc, deferred: &mut DeferredBatch) -> Result { - let (recover, masks) = { - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - let (pending, masks) = firmware.service_runtime_events(deferred)?; - let recover = pending.recovery || firmware.recovery.pending(); - if recover { - // Epoch closure and event consumption are serialized with every - // publisher. Joining builders must happen after unlocking. - self.preparations.close(); - } - (recover, masks) - }; - deferred.finish(); - if recover { - self.preparations.wait_drained(); - } - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - return Err(EIO); - } - if recover { - firmware.service_runtime_recovery(deferred)?; - if !firmware.recovery.pending() { - if !self.preparations.reopen() { - return Ok(()); - } - // Restart stopped consumption before its following record. - // Resume builders before observing pairs outside this lock. - self.queue_events(); - } - } - // Replay only at this outer boundary, after recovery has settled. - // Publication may service events but must never recurse into replay. - firmware.replay_compute(deferred)?; - drop(state); - self.observe_renders(masks, deferred)?; - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - if self.crashed.load(core::sync::atomic::Ordering::Acquire) { - return Err(EIO); - } - let now = super::now_ns(); - let compute = firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.in_flight()); - let in_flight = compute || firmware.queues.render_in_flight(); - let idle = firmware.idle_check(now, in_flight)?; - firmware.reclaim_drained_renders(deferred)?; - firmware.service_compute_exit(deferred)?; - firmware.service_compute_pools(deferred)?; - if !firmware.recovery.pending() { - firmware - .queues - .vacate_evicted_render_pools(&firmware.init, deferred)?; - } - { - let mut reclaim = self.reclaim.lock(); - firmware.service_teardown(&mut reclaim)?; - if !reclaim.is_empty() { - self.queue_reclaims(); - } - } - let backpressured = firmware.render_control_backpressured(); - let polling = backpressured - || firmware.queues.replay_outer_wait.is_some() - || firmware.queues.teardown.polling(now) - || firmware.compute_exit_polling(now) - || firmware - .queues - .compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_deref()) - .any(|queue| queue.retirement_polling()); - if polling { - self.queue_poll(); - } - if let Some(delay) = idle { - self.queue_idle(delay); - } - if firmware.recovery.pending() { - self.preparations.close(); - self.queue_events(); - } else if !recover - && !polling - && (channel::Events::pending(&firmware.init, Role::Primary)? - || channel::Events::pending(&firmware.init, Role::Secondary)?) - { - // A full bounded pass must not depend on a fresh notification for - // records that arrived while it was consuming the same ring. - self.queue_events(); - } - let grow_ready = firmware.render_grow_ready(); - drop(state); - if grow_ready { - self.queue_grow(); - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/feed.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/feed.rs deleted file mode 100644 index f3faffb89df764..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/feed.rs +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Schedules CPU capacity updates after completed firmware event passes. The -//! worker owns frequency requests independently of the device publication lock. - -use crate::g17::{Shared, FEED_WORK}; -use kernel::{ - impl_has_work, - sync::Arc, - workqueue::{self, WorkItem}, -}; - -impl_has_work! { - impl HasWork for Shared { self.feed_work } -} - -impl WorkItem for Shared { - type Pointer = Arc; - - fn run(this: Arc) { - this.feed.run(); - } -} - -impl Shared { - /// The completion callbacks have finished outside the device mutex before - /// the event worker hands the detector's request to the actuator. - pub(in crate::g17) fn queue_feed(self: &Arc) { - if self.feed.take_pending() { - let _ = workqueue::system_unbound().enqueue::, FEED_WORK>(self.clone()); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render.rs deleted file mode 100644 index 71b21f98479a6c..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render.rs +++ /dev/null @@ -1,744 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Device ownership of retained render pairs and their shared free-list leases. -//! The device mutex protects identities and publication. A preparation gate -//! lease permits graph construction outside that mutex without session teardown. - -mod exit; -mod dependencies; -mod grow; -mod observe; -mod pools; -mod prepare; -mod recovery; -mod settle; -mod submit; -pub(super) mod teardown; - -use super::{DeferredBatch, Registry}; -use crate::g17::{context::Context, fw::queue::DataMaster, job::Packet, kick, queue::render::Pair}; -use crate::hw::t8140::queues::RENDER_SLOTS; -use kernel::{prelude::*, sync::Arc}; - -#[derive(Copy, Clone)] -struct Config { - clusters: u32, - descriptor_flags: [u32; 2], -} - -#[derive(Copy, Clone)] -struct Reservation { - slot: u8, - ids: [kick::Id; 2], - owner: u64, - buffer: (u8, u64), -} - -struct Entry { - reservation: Reservation, - pair: Option>, - parked: Option, - preparing: Option>, - context: Option>, - pool: Option>, - pending_owner: Option, - pending_owner_closed: bool, - borrowed_active: bool, - borrowed_dependencies: Option, - event_masks: [u64; 2], - closed: bool, - pending_error: Option<(Arc, Error)>, - pending_timeout: bool, - constructing: bool, - growing: bool, - published: bool, -} - -pub(super) struct RenderState { - config: Config, - entries: KVec>, - pools: pools::Pools, - grow: grow::Grow, - retired: KVec, - next_pm_generation: u64, -} - -impl RenderState { - pub(super) fn new(clusters: u32, descriptor_flags: [u32; 2]) -> Result { - if clusters == 0 { - return Err(EINVAL); - } - let mut entries = KVec::with_capacity(RENDER_SLOTS, GFP_KERNEL)?; - for _ in 0..RENDER_SLOTS { - entries.push(None, GFP_KERNEL)?; - } - Ok(Self { - config: Config { - clusters, - descriptor_flags, - }, - entries, - pools: pools::Pools::new()?, - grow: grow::Grow::new()?, - retired: KVec::with_capacity(128, GFP_KERNEL)?, - next_pm_generation: 1, - }) - } - - fn reserve_pm_generation(&mut self) -> Result { - let generation = self.next_pm_generation; - self.next_pm_generation = generation.checked_add(1).ok_or(EOVERFLOW)?; - Ok(generation) - } - - fn collect_retired(&mut self, slot: usize) -> Result { - loop { - if self.retired.len() == self.retired.capacity() { - return Ok(()); - } - let retired = self.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - .and_then(|pair| pair.take_retired_binding()); - let Some(retired) = retired else { - return Ok(()); - }; - self.retired - .push_within_capacity(retired) - .map_err(|_| EIO)?; - } - } - - fn entry(&self, slot: u8) -> Result<&Entry> { - self.entries - .get(usize::from(slot)) - .and_then(Option::as_ref) - .ok_or(EIO) - } - - fn entry_mut(&mut self, slot: u8) -> Result<&mut Entry> { - self.entries - .get_mut(usize::from(slot)) - .and_then(Option::as_mut) - .ok_or(EIO) - } - - fn reserved(&mut self, reservation: Reservation) -> Result<&mut Entry> { - let entry = self.entry_mut(reservation.slot)?; - if entry.reservation.ids != reservation.ids - || entry.reservation.owner != reservation.owner - || entry.reservation.buffer != reservation.buffer - || entry.pair.is_some() - || entry.preparing.is_some() - || entry.context.is_some() - || !entry.constructing - { - return Err(EIO); - } - Ok(entry) - } - - /// Keep metadata while graph preparation owns the pair outside the mutex. - /// Completion scans skip it; cancellation is recorded and applied on return. - fn take_pair(&mut self, slot: u8, owner: u64, packet: &Arc) -> Result> { - let entry = self.entry_mut(slot)?; - if entry.closed - || entry.reservation.owner != owner - || entry.preparing.is_some() - || entry.pending_error.is_some() - || entry.pending_timeout - { - return Err(EBUSY); - } - let pair = entry.pair.as_ref().ok_or(EIO)?; - if !pair.matches(owner, &packet.context)? { - return Err(EIO); - } - entry.borrowed_active = pair.published_in_flight(); - entry.borrowed_dependencies = Some(pair.dependency_snapshot()); - entry.preparing = Some(packet.clone()); - entry.pair.take().ok_or(EIO) - } - - /// Validate before taking the owner from the caller. Even rejected returns - /// keep their backing alive until the caller resolves the reservation. - fn return_pair(&mut self, slot: u8, pair: &mut Option>) -> Result { - let returning = pair.as_ref().ok_or(EINVAL)?; - let entry = self.entry_mut(slot)?; - if entry.pair.is_some() - || entry.preparing.is_none() - || returning.slot() != slot - || returning.qids() != entry.reservation.ids.map(kick::Id::qid) - || returning.owner() != Some(entry.reservation.owner) - { - return Err(EIO); - } - entry.pair = pair.take(); - entry.borrowed_dependencies = None; - entry.preparing = None; - Ok(()) - } - - fn save_detaching_pool(&self, init: &crate::g17::initdata::InitData, slot: u8) -> Result { - let pool = self.entry(slot)?.pool.as_ref().ok_or(EIO)?; - let shared = self.entries.iter().enumerate().any(|(index, entry)| { - index != usize::from(slot) - && entry.as_ref().is_some_and(|entry| { - entry.pool.as_ref().is_some_and(|other| { - other.id() == pool.id() && other.control_va() == pool.control_va() - }) - }) - }); - if !shared { - self.pools.save(init, pool)?; - } - Ok(()) - } -} - -impl Registry { - fn ensure_render_ids(&mut self, slot: u8, owner: u64) -> Result { - if owner == 0 || slot == 0 || usize::from(slot) >= RENDER_SLOTS { - return Err(EINVAL); - } - if self.render.entries[usize::from(slot)].is_none() { - let identity = self.identities.reserve(2)?.start; - let tiling = self.qids.reserve(identity, DataMaster::Tiling, None)?; - let fragment = match self.qids.reserve(identity + 1, DataMaster::Fragment, None) { - Ok(id) => id, - Err(error) => { - self.qids.cancel_unpublished(tiling)?; - return Err(error); - } - }; - self.render.entries[usize::from(slot)] = Some(Entry { - reservation: Reservation { - slot, - ids: [tiling, fragment], - owner, - buffer: (0, 0), - }, - pair: None, - parked: None, - preparing: None, - context: None, - pool: None, - pending_owner: None, - pending_owner_closed: false, - borrowed_active: false, - borrowed_dependencies: None, - event_masks: [0; 2], - closed: false, - pending_error: None, - pending_timeout: false, - constructing: false, - growing: false, - published: false, - }); - } - Ok(()) - } - - /// Only an optional sibling's unpublished QID-capacity refusal is soft. - /// ensure_render_ids cancels a first ID if the second cannot be reserved; - /// validation, identity overflow, and cancellation errors still propagate. - fn prepare_render_ids(&mut self, slot: u8, owner: u64, optional: bool) -> Result { - match self.ensure_render_ids(slot, owner) { - Ok(()) => Ok(true), - Err(error) if optional && error == ENOSPC => Ok(false), - Err(error) => Err(error), - } - } - - fn reserve_render(&mut self, slot: u8, owner: u64) -> Result { - self.ensure_render_ids(slot, owner)?; - let entry = self.render.entry(slot)?; - if entry.constructing - || entry.pair.is_some() - || entry.parked.is_some() - || entry.preparing.is_some() - || entry.growing - { - return Err(EBUSY); - } - let ids = self.render.entry(slot)?.reservation.ids; - // Physical queue identities survive failed graph construction. Only - // failure to acquire the second identity can return the first one. - for id in ids { - self.qids.publish(id)?; - } - let buffer = self.buffers.allocate_render()?; - let entry = self.render.entry_mut(slot)?; - entry.reservation.owner = owner; - entry.reservation.buffer = buffer; - entry.constructing = true; - entry.closed = false; - Ok(entry.reservation) - } - - fn install_render( - &mut self, - reservation: Reservation, - pair: &mut Option>, - ) -> Result { - let built = pair.as_ref().ok_or(EINVAL)?; - if built.slot() != reservation.slot - || built.qids() != reservation.ids.map(kick::Id::qid) - || built.owner() != Some(reservation.owner) - || built.buffer_id() != reservation.buffer.0 - { - return Err(EIO); - } - if self.render.reserved(reservation)?.closed { - return Err(ECANCELED); - } - // Construction holds one PB reference; retain the owner independently - // and return that reference before the first command acquires its own. - self.buffers.reserve_owner(reservation.buffer.0)?; - self.buffers.release(reservation.buffer.0)?; - let entry = self.render.reserved(reservation)?; - entry.context = Some(built.context().clone()); - entry.pool = Some(built.pool().clone()); - entry.pair = pair.take(); - entry.constructing = false; - Ok(()) - } - - /// Drop the rejected graph outside the device mutex before returning its - /// temporary PB reference. The physical queue identities remain reserved. - fn cancel_render_reservation(&mut self, reservation: Reservation) -> Result { - self.render.reserved(reservation)?; - self.buffers.release(reservation.buffer.0)?; - self.render.entry_mut(reservation.slot)?.constructing = false; - Ok(()) - } - - pub(super) fn fail_render_vm( - &mut self, - status: &Arc, - error: Error, - deferred: &mut DeferredBatch, - ) -> Result { - for entry in self.render.entries.iter_mut().flatten() { - if !entry - .context - .as_ref() - .is_some_and(|context| Arc::ptr_eq(context.status(), status)) - { - continue; - } - if let Some(pair) = entry.pair.as_mut() { - pair.quarantine(error, &mut |event| deferred.push(event))?; - } else if entry.preparing.is_some() || entry.growing { - entry.pending_error = Some((status.clone(), error)); - } - } - Ok(()) - } -} - -struct Host<'a> { - firmware: &'a mut crate::g17::Firmware, - registration_powered: bool, -} -impl Host<'_> { - fn release_registration(&mut self) { - if !self.registration_powered && !self.firmware.registration_power_release_pending { - return; - } - if let Err(error) = self.firmware.release_registration_power() { - dev_warn!( - self.firmware.primary.state.shared.dev.as_ref(), - "Could not restore render idle policy: {:?}\n", - error - ); - } else { - self.registration_powered = false; - } - } -} -impl crate::g17::queue::render::Host for Host<'_> { - fn epoch(&self) -> Result<(u64, u32)> { - Ok(( - self.firmware.recovery.generation(), - crate::g17::recovery::Memory::recovery_state(&self.firmware.init)?, - )) - } - fn stamps(&self, qids: [u8; 2]) -> Result<[u32; 2]> { - use core::sync::atomic::Ordering; - let stamps = [ - self.firmware.init.stamp(qids[0])?.load(Ordering::Relaxed), - self.firmware.init.stamp(qids[1])?.load(Ordering::Relaxed), - ]; - core::sync::atomic::fence(Ordering::Acquire); - Ok(stamps) - } - fn qos_publish( - &mut self, - owner: crate::g17::qos::Owner, - scheduler: u64, - policy: crate::g17::fw::queue::Policy, - ) -> Result { - use crate::g17::qos; - self.firmware.queues.accounting.publish( - &qos::View::new(self.firmware.init.qos()?)?, - owner, - scheduler, - policy, - qos::clock, - ) - } - fn qos_cancel(&mut self, publication: crate::g17::qos::Publication) -> Result { - use crate::g17::qos; - self.firmware - .queues - .accounting - .cancel(&qos::View::new(self.firmware.init.qos()?)?, publication) - } - /// Idle descent is held for the whole first install. A point-in-time power - /// check let the firmware gate the cores between the host's configuration - /// writes while several clients installed their first pairs at once. - fn install_pair(&mut self, registration: &crate::g17::fw::kick::RenderRegistration) -> Result { - if !self.registration_powered { - self.firmware.acquire_registration_power()?; - self.registration_powered = true; - } - for _ in 0..3 { - if self - .firmware - .bridge - .regs - .install_render_pair(registration)? - { - return Ok(()); - } - } - Err(EAGAIN) - } - fn publish_qid(&mut self, id: kick::Id) -> Result { - self.firmware.queues.qids.publish(id) - } - fn publish_outer_pair( - &mut self, - priority: u8, - fragment: &crate::g17::fw::channels::WorkSlot, - tiling: &crate::g17::fw::channels::WorkSlot, - commit: impl FnOnce() -> Result, - late_tiling: impl FnOnce(), - ) -> Result<[u32; 2]> { - self.firmware - .init - .publish_work_pair(priority, fragment, tiling, true, commit, late_tiling) - } - fn activate_render_pool(&mut self, id: u16, generation: u64) -> Result { - self.release_registration(); - let result = self.firmware.queues.render.pools.activate(id, generation); - if result.is_err() { - self.firmware.primary.state.shared.lose_device_quietly(); - } - result - } - fn publish_retained_pair( - &mut self, - priority: u8, - fragment: &crate::g17::fw::channels::WorkSlot, - tiling: &crate::g17::fw::channels::WorkSlot, - next: [u32; 2], - ) -> Result { - let result = self - .firmware - .init - .publish_retained_work_pair(priority, fragment, tiling, next); - if result.is_err() { - self.firmware.primary.state.shared.lose_device_quietly(); - } - result - } - /// Pipe 0 (tiling) in bits 1:0, priority class in bits 3:2. - fn notify_render(&mut self, priority: u8) -> Result { - self.release_registration(); - let result = self - .firmware - .primary - .notify((0x83 << crate::g17::MSG_TYPE_SHIFT) | u64::from(priority) << 2); - if result.is_err() { - self.firmware.primary.state.shared.lose_device_quietly(); - } - result - } - fn note_submission(&mut self) -> Result { - self.firmware.note_submission() - } - fn complete_render( - &mut self, - owners: Option<[crate::g17::qos::Owner; 2]>, - pool_id: u16, - generation: u64, - buffer_id: u8, - ) -> Result { - use crate::g17::qos; - let queues = &mut self.firmware.queues; - let next = queues.render.pools.completed(pool_id, generation)?; - queues.buffers.can_release(buffer_id)?; - let view = qos::View::new(self.firmware.init.qos()?)?; - if let Some(owners) = owners { - queues.accounting.complete(&view, &owners)?; - } - queues.render.pools.commit_completed(pool_id, next); - queues.buffers.release(buffer_id) - } -} - -impl crate::g17::Firmware { - /// The pair is returned even when transport, accounting or collection fails. - /// No off-lock operation or callback which can re-enter the device runs here. - fn with_render( - &mut self, - slot: usize, - operation: impl FnOnce(&mut Pair, &mut Host<'_>) -> Result, - ) -> Result { - let mut pair = self - .queues - .render - .entries - .get_mut(slot) - .and_then(Option::as_mut) - .and_then(|entry| entry.pair.take()) - .ok_or(EBUSY)?; - let mut host = Host { - firmware: self, - registration_powered: false, - }; - let result = operation(&mut pair, &mut host); - host.release_registration(); - self.queues.render.entries[slot].as_mut().ok_or(EIO)?.pair = Some(pair); - result - } - - fn release_render_lists(&mut self) -> Result { - if self.queues.render.grow.reply.is_some() { - return Err(EAGAIN); - } - let primary = &mut self.primary; - self.queues.render.pools.publish_releases(&self.init, || { - primary.notify(crate::g17::MSG_CONTROL_NOTIFY) - }) - } - - /// Events have already been consumed by the caller. Each healthy completion - /// services its final pool release before examining the next retained job. - fn retire_render_slot( - &mut self, - slot: usize, - masks: Option<[u64; 2]>, - deferred: &mut DeferredBatch, - ) -> Result { - let generation = self.recovery.generation(); - let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - loop { - let progressed = self.with_render(slot, |pair, host| { - pair.retire(masks, generation, state, host, &mut |event| { - deferred.push(event) - }) - })?; - if !progressed { - break; - } - match self.release_render_lists() { - Err(error) if error == EAGAIN => break, - result => result?, - } - } - Ok(()) - } - - pub(in crate::g17) fn settle_render_vm( - &mut self, - status: &Arc, - error: Error, - deferred: &mut DeferredBatch, - ) -> Result { - for slot in 1..self.queues.render.entries.len() { - if self.queues.render.entries[slot] - .as_ref() - .is_some_and(|entry| { - entry.pair.is_some() - && entry - .context - .as_ref() - .is_some_and(|context| Arc::ptr_eq(context.status(), status)) - }) - { - self.retire_render_slot(slot, None, deferred)?; - } - } - self.queues.fail_render_vm(status, error, deferred) - } - - pub(in crate::g17) fn stop_render(&mut self) { - for entry in self.queues.render.entries.iter_mut().flatten() { - if let Some(pair) = entry.pair.as_mut() { - if let Err(error) = pair.stopped(&mut |event| { - event.finish(); - Ok(()) - }) { - dev_err!( - self.primary.state.shared.dev.as_ref(), - "Could not settle stopped render pair: {:?}\n", - error - ); - } - } - } - } -} - -impl Registry { - pub(in crate::g17) fn render_oldest_spared(&self, qid: u8) -> Option { - self.render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.qids().contains(&qid)) - .and_then(|pair| pair.oldest_spared()) - } - pub(in crate::g17) fn visit_render_owners( - &self, - qid: u8, - visit: &mut dyn FnMut(&Arc), - ) { - for pair in self - .render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - { - if pair.qids().contains(&qid) { - pair.visit_owners(visit); - } - } - } - pub(in crate::g17) fn classify_render( - &self, - classify: &dyn Fn(&Arc) -> crate::g17::recovery::Class, - ) -> u128 { - let mut spared = 0; - for pair in self - .render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - { - pair.classify(classify); - if pair.oldest_spared() == Some(true) { - for qid in pair.qids() { - spared |= 1u128 << qid; - } - } - } - spared - } - - pub(in crate::g17) fn render_scheduler_active(&self) -> bool { - self.render.entries.iter().any(Option::is_some) - || !self.render.retired.is_empty() - || !self.teardown.is_empty() - } - /// Gives back the USC backing of render pools whose client VM pressure - /// reclaim evicted (an idle VM) and whose final free-list release was - /// consumed. The vacated backings go to the worker's deferred batch, which - /// frees them after the device mutex; a later job of that client re-arms - /// the pool first. - pub(in crate::g17) fn vacate_evicted_render_pools( - &mut self, - init: &crate::g17::initdata::InitData, - deferred: &mut super::DeferredBatch, - ) -> Result { - let render = &mut self.render; - render.pools.observe(init)?; - for entry in render.entries.iter().flatten() { - if !deferred.render_backing_room() { - break; - } - if entry.constructing || entry.growing || entry.preparing.is_some() { - continue; - } - let Some(pool) = entry.pool.as_ref() else { - continue; - }; - if pool.vacant() || !render.pools.idle(pool.id()) || !pool.client_evicted() { - continue; - } - if let Some(backing) = pool.vacate() { - deferred.defer_render_backing(backing); - } - } - Ok(()) - } - pub(in crate::g17) fn render_in_flight(&self) -> bool { - self.render.entries.iter().flatten().any(|entry| { - entry.pair.as_ref().map_or( - (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, - |pair| pair.published_in_flight(), - ) - }) - } - pub(in crate::g17) fn retain_render_completions(&mut self, masks: [u64; 2]) { - for entry in self.render.entries.iter_mut().flatten() { - for id in entry.reservation.ids { - let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] |= masks[qid / 64] & (1u64 << (qid % 64)); - } - } - } - pub(in crate::g17) fn render_needs_rescan(&self) -> bool { - self.render.entries.iter().flatten().any(|entry| { - let active = entry - .pair - .as_ref() - .map_or( - (entry.preparing.is_some() || entry.growing) && entry.borrowed_active, - |pair| pair.published_in_flight(), - ); - active - && entry.reservation.ids.into_iter().all(|id| { - let qid = usize::from(id.qid()); - entry.event_masks[qid / 64] & (1u64 << (qid % 64)) != 0 - }) - }) - } - pub(in crate::g17) fn defer_render_timeout( - &mut self, - packet: &Arc, - error: Error, - ) -> bool { - // Its tracker may already be gone while off-lock completion still - // owns the fence and admission lease. A loan for a subsequent packet - // cannot turn this exact accepted retirement into a missing-owner error. - if packet.completion.retirement_queued() { - return true; - } - let Some(slot) = packet.completion.render_slot() else { - return false; - }; - let Ok(entry) = self.render.entry_mut(slot) else { - return false; - }; - if entry.preparing.is_none() && !entry.growing { - return false; - } - if !entry - .borrowed_dependencies - .as_ref() - .is_some_and(|dependencies| dependencies.contains_packet(packet)) - { - return false; - } - packet.completion.defer_timeout(error); - entry.pending_timeout = true; - true - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/dependencies.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/dependencies.rs deleted file mode 100644 index 09e4322bf42cef..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/dependencies.rs +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Both engines preserve the covered render prefix while physical graphs are borrowed. - -use crate::g17::{dependency::Frontier, job::Packet, runtime::Registry}; -use kernel::{prelude::*, sync::Arc}; - -impl Registry { - pub(in crate::g17::runtime) fn render_prefix( - &self, - owner: u64, - packet: &Arc, - ) -> Result { - let mut frontier = Frontier::new(); - for entry in self.render.entries.iter().flatten() { - if let Some(pair) = entry.pair.as_ref() { - pair.collect_dependencies( - owner, - packet.completion.status(), - packet.order.wait_through[0], - &mut frontier, - )?; - } else if let Some(dependencies) = entry.borrowed_dependencies.as_ref() { - dependencies.collect( - owner, - packet.completion.status(), - packet.order.wait_through[0], - &mut frontier, - )?; - } - } - Ok(frontier) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/exit.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/exit.rs deleted file mode 100644 index c47110cb76fadf..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/exit.rs +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Physical render queue scope for an exiting logical context. A graph loan -//! remains foreign to the transaction until its exact owner is restored. - -use super::RenderState; -use crate::g17::context::Context; -use kernel::sync::Arc; - -impl RenderState { - pub(in crate::g17::runtime) fn exit_scope(&self, context: &Arc) -> (u128, u128, u64) { - let (mut own, mut foreign, mut slots) = (0, 0, 0); - for (slot, entry) in self.entries.iter().enumerate() { - let Some(entry) = entry else { - continue; - }; - let qids = self.qids_for_slot(slot as u8); - let preparing = entry.preparing.is_some() || entry.constructing || entry.growing; - if !preparing && entry.context.as_ref().is_some_and(|owner| Arc::ptr_eq(owner, context)) { - own |= qids; - slots |= 1u64 << slot; - } else if preparing || entry.context.is_some() { - foreign |= qids; - } - } - (own, foreign, slots) - } - - pub(in crate::g17::runtime) fn qids_for_slot(&self, slot: u8) -> u128 { - self.entries - .get(usize::from(slot)) - .and_then(Option::as_ref) - .map_or(0, |entry| { - entry.reservation.ids.into_iter().fold(0, |mask, id| { - mask | (1u128 << id.qid()) - }) - }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/grow.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/grow.rs deleted file mode 100644 index 0ab673b79be265..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/grow.rs +++ /dev/null @@ -1,323 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Deferred parameter-buffer growth. Event ingress retains the entire request; -//! allocation borrows one pair under a preparation lease and runs off-lock. - -use super::RenderState; -use crate::g17::{ - channel::Rings, - fw::channels::{ControlRecord, FreeListCompletion, TvbGrowReply, TvbGrowRequest, EVENT_SLOTS}, -}; -use core::sync::atomic::{fence, Ordering}; -use kernel::prelude::*; - -const RETRY_LIMIT: u16 = 1000; - -pub(super) struct Grow { - requests: KVec, - active: Option, - pub(super) reply: Option<(TvbGrowRequest, bool, u16)>, - release_attempts: u16, -} -impl Grow { - pub(super) fn new() -> Result { - Ok(Self { - requests: KVec::with_capacity(2 * EVENT_SLOTS as usize, GFP_KERNEL)?, - active: None, - reply: None, - release_attempts: 0, - }) - } -} - -impl RenderState { - pub(in crate::g17::runtime) fn control_backpressured(&self) -> bool { - self.grow.reply.is_some() || self.pools.pending() - } - fn grow_owner(&self, request: TvbGrowRequest) -> Option { - self.entries.iter().position(|entry| { - entry.as_ref().is_some_and(|entry| { - entry.published - && u32::from(entry.reservation.buffer.0) == request.buffer_slot - && entry.context.as_ref().is_some_and(|context| { - request.vm_slot == -1 || request.vm_slot as u32 == u32::from(context.id()) - }) - }) - }) - } - fn grow_owner_busy(&self, request: TvbGrowRequest) -> bool { - self.grow_owner(request).is_some_and(|slot| { - self.entries[slot] - .as_ref() - .is_some_and(|entry| entry.preparing.is_some() || entry.growing) - }) - } -} - -impl crate::g17::Firmware { - pub(in crate::g17) fn queue_tvb_grow(&mut self, request: TvbGrowRequest) -> Result { - let render = &mut self.queues.render; - if request.halt_count != self.recovery.generation() - || render.grow.active == Some(request) - || render.grow.requests.contains(&request) - { - return Ok(()); - } - render - .grow - .requests - .push_within_capacity(request) - .map_err(|_| EOVERFLOW) - } - pub(in crate::g17) fn render_grow_pending(&self) -> bool { - !self.queues.render.grow.requests.is_empty() - } - /// Borrowed owners and control publication each have their own retry wake. - /// A stale request is runnable so the worker can discard it immediately. - pub(in crate::g17) fn render_grow_ready(&self) -> bool { - let render = &self.queues.render; - !render.grow.requests.is_empty() - && !self.recovery.pending() - && !self.render_control_backpressured() - && render.grow.active.is_none() - && render.grow.requests.iter().any(|request| { - request.halt_count != self.recovery.generation() - || !render.grow_owner_busy(*request) - }) - } - pub(in crate::g17) fn render_control_backpressured(&self) -> bool { - self.queues.render.grow.reply.is_some() || self.queues.render.pools.pending() - } - pub(in crate::g17) fn complete_freelist_grow(&mut self, event: FreeListCompletion) -> Result { - self.queues.render.pools.complete_grow(event) - } - pub(in crate::g17) fn observe_render_releases(&mut self) -> Result { - self.queues.render.pools.observe(&self.init) - } - pub(in crate::g17) fn flush_tvb_reply(&mut self) -> Result { - let Some((request, grew, _)) = self.queues.render.grow.reply else { - return Ok(true); - }; - if request.halt_count != self.recovery.generation() { - return Err(EIO); - } - let record = ControlRecord::TvbGrow(TvbGrowReply::new(&request, grew)); - match Rings::publish_control(&self.init, &record) { - Err(error) if error == EAGAIN => return Ok(false), - result => { - result?; - } - } - // Producer ownership is irreversible, even if announcement fails. - self.queues.render.grow.reply = None; - fence(Ordering::SeqCst); - self.primary - .notify((0x84 << crate::g17::MSG_TYPE_SHIFT) | 0x11)?; - Ok(true) - } - pub(in crate::g17) fn flush_freelist_release(&mut self) -> Result { - if self.queues.render.grow.reply.is_some() { - return Ok(false); - } - match self.release_render_lists() { - Err(error) if error == EAGAIN => Ok(false), - result => { - result?; - self.queues.render.grow.release_attempts = 0; - Ok(true) - } - } - } - /// Only the sleeping event worker consumes this budget. Inline submission - /// probes cannot turn a transient full control ring into a rapid timeout. - pub(in crate::g17) fn note_render_control_backpressure(&mut self) -> Result { - let render = &mut self.queues.render; - let attempts = if let Some((_, _, attempts)) = render.grow.reply.as_mut() { - attempts - } else if render.pools.pending() { - &mut render.grow.release_attempts - } else { - return Err(EINVAL); - }; - *attempts = attempts.checked_add(1).ok_or(ETIMEDOUT)?; - if *attempts > RETRY_LIMIT { - return Err(ETIMEDOUT); - } - Ok(()) - } -} - -impl crate::g17::Firmware { - pub(super) fn commit_render_growth( - &mut self, - slot: u8, - prepared: &mut Option, - ) -> Result { - if prepared.is_none() { - return Ok(()); - } - let pair = self - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)?; - let vm = pair.context().vm().clone(); - let pool = pair.pool_id(); - let buffer_id = pair.buffer_id(); - // Preflight every destination before the graph starts publishing. - let table = self.init.pb_descriptor_table()?; - let offset = usize::from(buffer_id) * 16; - let words = [ - table.word(offset)?, - table.word(offset + 4)?, - table.word(offset + 8)?, - table.word(offset + 12)?, - ]; - let descriptor = pair - .manager() - .commit_growth(prepared, |previous, blocks, mappings| { - vm.append_render_pool_mappings_prepared(slot, pool, previous, blocks, mappings) - })?; - if let Some(descriptor) = descriptor { - for (word, value) in words.into_iter().zip(descriptor) { - word.store(value, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - } - Ok(()) - } -} - -impl crate::g17::Shared { - pub(in crate::g17) fn grow_render(&self) -> Result { - loop { - let lease = self.preparations.enter(false)?; - let dev = self.drm_neo()?; - let (request, slot, mut pair) = { - let mut state = self.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if firmware.render_control_backpressured() { - return Ok(()); - } - let generation = firmware.recovery.generation(); - let render = &mut firmware.queues.render; - if render.grow.active.is_some() { - return Ok(()); - } - let Some(index) = render.grow.requests.iter().position(|request| { - request.halt_count != generation || !render.grow_owner_busy(*request) - }) else { - return Ok(()); - }; - let request = render.grow.requests.remove(index).map_err(|_| EIO)?; - if request.halt_count != generation { - continue; - } - let slot = render.grow_owner(request); - let pair = match slot { - Some(slot) => { - let entry = render.entries[slot].as_mut().ok_or(EIO)?; - if entry.preparing.is_some() || entry.growing { - return Err(EIO); - } - entry.borrowed_active = entry - .pair - .as_ref() - .is_some_and(|pair| pair.published_in_flight()); - entry.borrowed_dependencies = - entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); - entry.growing = true; - Some(entry.pair.take().ok_or(EIO)?) - } - None => None, - }; - render.grow.active = Some(request); - (request, slot, pair) - }; - let prepared = match pair.as_mut() { - Some(pair) => { - let vm = pair.context().vm().clone(); - let pool = pair.pool_id(); - let slot = pair.slot(); - let manager = pair.manager(); - let before = manager.blocks(); - if vm.render_pool_tvb_blocks(slot, pool) != Some(before) { - Err(EFAULT) - } else { - manager.prepare_growth( - &dev, - &vm, - crate::g17::buffer::firmware_growth_target(before), - None, - ) - } - } - None => Err(ENOENT), - }; - let (mut prepared, prepare_error) = match prepared { - Ok(plan) => (plan, None), - Err(error) => (None, Some(error)), - }; - let mut state = self.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if let Some(slot) = slot { - let entry = firmware.queues.render.entries[slot].as_mut().ok_or(EIO)?; - if entry.pair.is_some() || !entry.growing { - return Err(EIO); - } - entry.pair = pair.take(); - entry.borrowed_dependencies = None; - entry.growing = false; - } - if firmware.queues.render.grow.active != Some(request) { - return Err(EIO); - } - firmware.queues.render.grow.active = None; - self.changed.notify_all(); - if !lease.is_current() || request.halt_count != firmware.recovery.generation() { - continue; - } - let result = if let Some(error) = prepare_error { - Err(error) - } else if prepared.is_none() { - Ok(false) - } else if let Some(slot) = slot { - let pair = firmware.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - .ok_or(EIO)?; - let vm = pair.context().vm().clone(); - let pool = pair.pool_id(); - let slot = pair.slot(); - let grew = prepared.is_some(); - pair.manager() - .commit_growth(&mut prepared, |previous, blocks, mappings| { - vm.append_render_pool_mappings_prepared( - slot, pool, previous, blocks, mappings, - ) - }) - .map(|_| grew) - } else { - Err(ENOENT) - }; - let grew = match result { - Ok(grew) => grew, - Err(error) => { - dev_warn!( - self.dev.as_ref(), - "Could not grow render parameter buffer {}: {:?}\n", - request.buffer_slot, - error - ); - false - } - }; - firmware.queues.render.grow.reply = Some((request, grew, 0)); - if !firmware.flush_tvb_reply()? { - return Ok(()); - } - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/observe.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/observe.rs deleted file mode 100644 index 31bcb688c80018..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/observe.rs +++ /dev/null @@ -1,235 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Event-driven render snapshots outside the device mutex. The preparation -//! epoch and exact pair owner survive every read, cancellation and retry. - -use super::super::DeferredBatch; -use crate::g17::queue::render::Pair; -use core::sync::atomic::Ordering; -use kernel::{prelude::*, sync::Arc}; - -impl crate::g17::Firmware { - fn take_render_observation(&mut self, start: usize) -> Result)>> { - if self.recovery.pending() { - return Ok(None); - } - let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - for slot in start..self.queues.render.entries.len() { - let Some(entry) = self.queues.render.entries[slot].as_mut() else { - continue; - }; - if entry.preparing.is_some() || entry.growing { - continue; - } - let Some(pair) = entry.pair.as_ref() else { - continue; - }; - let qids = pair.qids(); - if !pair.published_in_flight() - || !qids - .into_iter() - .all(|qid| entry.event_masks[usize::from(qid) / 64] & (1u64 << (qid % 64)) != 0) - { - continue; - } - let stamps = [ - self.init.stamp(qids[0])?.load(Ordering::Relaxed), - self.init.stamp(qids[1])?.load(Ordering::Relaxed), - ]; - core::sync::atomic::fence(Ordering::Acquire); - let Some(packet) = pair.observation_packet(entry.event_masks, stamps, state)? else { - continue; - }; - entry.borrowed_active = pair.published_in_flight(); - entry.borrowed_dependencies = Some(pair.dependency_snapshot()); - entry.preparing = Some(packet); - return Ok(Some((slot, entry.pair.take().ok_or(EIO)?))); - } - Ok(None) - } - - fn return_render_observation(&mut self, slot: usize, pair: &mut Option>) -> Result { - let returning = pair.as_ref().ok_or(EINVAL)?; - let entry = self.queues.render.entry_mut(slot as u8)?; - if entry.pair.is_some() - || entry.preparing.is_none() - || returning.slot() as usize != slot - || returning.qids() != entry.reservation.ids.map(crate::g17::kick::Id::qid) - { - return Err(EIO); - } - entry.pair = pair.take(); - entry.borrowed_dependencies = None; - entry.preparing = None; - Ok(()) - } - - pub(super) fn settle_render_terminal( - &mut self, - slot: usize, - deferred: &mut DeferredBatch, - ) -> Result { - let terminal = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - .and_then(|pair| pair.take_terminal()); - if let Some((status, error)) = terminal { - status.record(error); - self.scan_compute(None, deferred)?; - self.settle_render_vm(&status, error, deferred)?; - self.queues.fail_compute_vm(&status, error, deferred)?; - } - Ok(()) - } -} - -impl crate::g17::Shared { - pub(in crate::g17) fn observe_renders( - self: &Arc, - masks: [u64; 2], - deferred: &mut DeferredBatch, - ) -> Result { - { - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - firmware.queues.retain_render_completions(masks); - // Returned preparers can have deferred a VM failure or timeout. - // Only this full worker collector runs their final observations. - for slot in 1..firmware.queues.render.entries.len() { - if firmware.queues.render.entries[slot] - .as_ref() - .is_some_and(|entry| { - entry.pair.is_some() - && (entry.pending_error.is_some() - || entry.pending_timeout - || entry.closed) - }) - { - firmware.apply_render_deferred(slot as u8, deferred)?; - } - } - } - deferred.finish(); - let mut cursor = 1; - let mut remaining = crate::hw::t8140::queues::RENDER_DEPTH; - let mut retries = 0; - loop { - let epoch = match self.preparations.try_enter() { - Err(error) if error == EAGAIN || error == ENODEV => break, - result => result?, - }; - let loan = { - let mut state = self.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - if !epoch.is_current() || self.crashed.load(Ordering::Acquire) { - break; - } - firmware.take_render_observation(cursor)? - }; - let Some((slot, pair)) = loan else { - break; - }; - if slot != cursor { - remaining = crate::hw::t8140::queues::RENDER_DEPTH; - retries = 0; - } - let observed = pair.observe_front(); - let mut pair = Some(pair); - let mut state = self.state.lock(); - // Return installed ownership before examining read errors or epoch. - let restored = (*state) - .as_deref_mut() - .ok_or(ENODEV) - .and_then(|firmware| firmware.return_render_observation(slot, &mut pair)); - if let Err(error) = restored { - // A failed invariant must not drop firmware-owned storage. Join - // the other preparers only after releasing our own epoch, and - // keep this detached owner until both processors have stopped. - drop(state); - drop(epoch); - self.lose_device_quietly(); - self.stop_queues(); - if let Some(mut pair) = pair { - pair.stopped(&mut |event| { - event.finish(); - Ok(()) - })?; - } - return Err(error); - } - let firmware = (*state).as_deref_mut().ok_or(EIO)?; - self.changed.notify_all(); - if firmware.render_grow_pending() { - self.queue_events(); - } - if !epoch.is_current() - || firmware.recovery.pending() - || self.crashed.load(Ordering::Acquire) - { - break; - } - let result = (|| { - let state = crate::g17::recovery::Memory::recovery_state(&firmware.init)?; - let progressed = match observed { - Ok(observed) => firmware.with_render(slot, |pair, host| { - pair.retire_observed(observed, state, host, &mut |event| { - deferred.push(event) - }) - })?, - Err(error) if error == EAGAIN => false, - Err(error) => return Err(error), - }; - if progressed { - match firmware.release_render_lists() { - Err(error) if error == EAGAIN => {} - result => result?, - } - } - firmware.queues.render.collect_retired(slot)?; - // An observation of the old front is committed before a queued - // exact timeout is permitted to advance that same tracker. - firmware.apply_render_deferred(slot as u8, deferred)?; - firmware.settle_render_terminal(slot, deferred)?; - if firmware - .queues - .render - .entry(slot as u8)? - .pair - .as_ref() - .is_some_and(|pair| !pair.in_flight()) - { - firmware.queues.render.entry_mut(slot as u8)?.event_masks = [0; 2]; - } - Ok(progressed) - })(); - drop(state); - drop(epoch); - deferred.finish(); - if result? { - remaining -= 1; - retries = 0; - cursor = if remaining == 0 { slot + 1 } else { slot }; - } else { - retries += 1; - cursor = if retries >= 2 { slot + 1 } else { slot }; - } - if cursor != slot { - remaining = crate::hw::t8140::queues::RENDER_DEPTH; - retries = 0; - } - } - { - let state = self.state.lock(); - if (*state).as_deref().is_some_and(|firmware| { - !firmware.recovery.pending() && firmware.queues.render_needs_rescan() - }) { - self.queue_poll(); - } - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/pools.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/pools.rs deleted file mode 100644 index f7f48e853f2252..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/pools.rs +++ /dev/null @@ -1,241 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Device-wide references to render free lists. Two physical pairs bound to -//! one logical context share one registration and one final release witness. -//! The device mutex serializes every transition and descriptor-table access. - -use crate::g17::{ - channel::Rings, - freelist::{Lease, Owner, RenderPool}, - fw::{ - channels::{ControlRecord, FreeListCompletion}, - initdata::PagePoolDescriptor, - }, - initdata::InitData, - object::KernelObject, -}; -use core::sync::atomic::{fence, AtomicU64, Ordering}; -use kernel::prelude::*; - -const POOLS: usize = 256; -const FIRST_RENDER_POOL: usize = 129; - -pub(super) struct Pools { - leases: KVec>, -} - -/// Undo state cannot leave the device-locked publication transaction. Rollback -/// is permitted only before the packet exposes its first firmware reference. -pub(super) struct Preparation { - previous: Option, - slot: usize, - pub(super) generation: u64, -} - -impl Pools { - pub(super) fn new() -> Result { - let mut leases = KVec::with_capacity(POOLS, GFP_KERNEL)?; - for _ in 0..POOLS { - leases.push(None, GFP_KERNEL)?; - } - Ok(Self { leases }) - } - - fn slot(id: u16) -> Result { - let slot = usize::from(id); - if !(FIRST_RENDER_POOL..POOLS).contains(&slot) { - return Err(EINVAL); - } - Ok(slot) - } - - fn owner(pool: &RenderPool) -> Owner { - Owner { - serial: pool.serial(), - epoch: pool.epoch(), - buffer_slot: u32::from(pool.id()), - control_va: pool.control_va(), - ring_va: pool.run_list_va(), - cookie: pool.end_va(), - } - } - - /// The pool's registration has a consumed final release and no references: - /// the firmware holds nothing of its backing. - pub(super) fn idle(&self, id: u16) -> bool { - Self::slot(id) - .ok() - .and_then(|slot| self.leases[slot]) - .is_some_and(|lease| lease.released()) - } - - pub(super) fn released(&self, id: u32, _control: u64) -> bool { - self.leases - .get(id as usize) - .is_none_or(|lease| lease.as_ref().is_none_or(|lease| lease.released())) - } - - /// Install both engine references before either inner queue becomes - /// visible. Warm overlap keeps the existing generation and pool contents. - pub(super) fn prepare(&mut self, init: &InitData, pool: &RenderPool) -> Result { - // A vacated pool is re-armed by the submission path before it retries. - if pool.vacant() { - return Err(EAGAIN); - } - self.observe(init)?; - let slot = Self::slot(pool.id())?; - let owner = Self::owner(pool); - let previous = self.leases[slot]; - if previous.is_some_and(|lease| lease.owner() != owner && !lease.released()) { - return Err(EBUSY); - } - // Pools arrive fully populated. There is no asynchronous initial grow. - let next = Lease::acquire(previous.as_ref(), owner)?; - if previous.is_none_or(|lease| lease.owner() != owner) { - self.stage(init, pool)?; - } - let generation = next.generation(); - self.leases[slot] = Some(next); - Ok(Preparation { - previous, - slot, - generation, - }) - } - - /// Warming validates the incoming owner and stages its descriptor without - /// acquiring command references or publishing an asynchronous request. - pub(super) fn warm(&self, init: &InitData, pool: &RenderPool) -> Result { - if pool.vacant() { - return Err(EAGAIN); - } - let previous = self.leases[Self::slot(pool.id())?]; - let owner = Self::owner(pool); - if previous.is_some_and(|lease| lease.owner() == owner) { - return Ok(()); - } - if previous.is_some_and(|lease| !lease.released()) { - return Err(EBUSY); - } - Lease::acquire(previous.as_ref(), owner)?; - self.stage(init, pool) - } - - fn stage(&self, init: &InitData, pool: &RenderPool) -> Result { - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; - pool.repopulate()?; - let descriptor = pool - .page_pool_state()? - .descriptor(words[3].load(Ordering::Relaxed)); - for (word, value) in words.into_iter().zip([ - descriptor.page_list, - descriptor.counters, - descriptor.page_count, - descriptor.unk_18, - ]) { - word.store(value, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - Ok(()) - } - - /// Retire preparation only after both deferred outer bodies and their - /// inner commands exist, before publishing either outer producer. - pub(super) fn activate(&mut self, id: u16, generation: u64) -> Result { - let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; - if lease.generation() != generation { - return Err(EIO); - } - lease.retire_prepare() - } - - pub(super) fn rollback(&mut self, preparation: Preparation) { - self.leases[preparation.slot] = preparation.previous; - } - - /// Preflight a paired retirement before QoS or PB reference accounting. - pub(super) fn completed(&self, id: u16, generation: u64) -> Result { - let mut next = self.leases[Self::slot(id)?].ok_or(EIO)?; - if next.generation() != generation { - return Err(EIO); - } - next.complete_render()?; - Ok(next) - } - - /// The matching completed() result and other accounting preflights were - /// accepted under the same device mutex, so this assignment cannot fail. - pub(super) fn commit_completed(&mut self, id: u16, next: Lease) { - self.leases[usize::from(id)] = Some(next); - } - - pub(super) fn pending(&self) -> bool { - self.leases - .iter() - .flatten() - .any(|lease| lease.pending_release().is_some()) - } - - pub(super) fn complete_grow(&mut self, event: FreeListCompletion) -> Result { - let id = u16::try_from(event.buffer_slot).map_err(|_| EINVAL)?; - let lease = self.leases[Self::slot(id)?].as_mut().ok_or(EIO)?; - lease.complete_grow(event) - } - - pub(super) fn observe(&mut self, init: &InitData) -> Result { - fence(Ordering::Acquire); - let consumer = init.control_consumer()?.load(Ordering::Relaxed); - let producer = init.control_producer()?.load(Ordering::Relaxed); - for lease in self.leases.iter_mut().flatten() { - lease.observe_releases(consumer, producer)?; - } - Ok(()) - } - - /// Store the release witness immediately after the ring producer. A failed - /// mailbox announcement must never cause the same release to be replayed. - pub(super) fn publish_releases( - &mut self, - init: &InitData, - mut announce: impl FnMut() -> Result, - ) -> Result { - for lease in self.leases.iter_mut().flatten() { - let Some(release) = lease.pending_release() else { - continue; - }; - let cursor = Rings::publish_control(init, &ControlRecord::FreeListRelease(release))?; - lease.mark_release_sent(cursor)?; - fence(Ordering::SeqCst); - announce()?; - } - self.observe(init) - } - - /// A detached pool may capture firmware counters only after its last pair - /// stopped using this registration. Callers check all retained pair owners. - pub(super) fn save(&self, init: &InitData, pool: &RenderPool) -> Result { - let state = pool.page_pool_state()?; - let words = descriptor_words(init.page_pool_descriptor_table()?, pool.id())?; - fence(Ordering::Acquire); - let descriptor = PagePoolDescriptor { - page_list: words[0].load(Ordering::Relaxed), - counters: words[1].load(Ordering::Relaxed), - page_count: words[2].load(Ordering::Relaxed), - unk_18: words[3].load(Ordering::Relaxed), - }; - if descriptor.page_list != state.descriptor(descriptor.unk_18).page_list { - return Err(EIO); - } - pool.store_page_pool_state(state.updated(&descriptor)) - } -} - -fn descriptor_words(table: &KernelObject, id: u16) -> Result<[&AtomicU64; 4]> { - let offset = Pools::slot(id)? * size_of::(); - Ok([ - table.dword(offset)?, - table.dword(offset + 8)?, - table.dword(offset + 16)?, - table.dword(offset + 24)?, - ]) -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/prepare.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/prepare.rs deleted file mode 100644 index 0d1a3ec72093bc..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/prepare.rs +++ /dev/null @@ -1,717 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Off-lock render graph construction and command preparation. Every borrowed -//! installed pair returns to its device owner before its preparation lease ends. - -use super::{Config, Reservation}; -use crate::g17::{ - buffer::MetricsLease, - preparation::Lease, - job::Packet, - object::{Allocator, KernelObject, Pool}, - queue::render::{Pair, RenderAddresses}, -}; -use kernel::{prelude::*, sync::Arc}; - -struct BuildResources { - uat: Arc, - objects: Arc, - ids: Arc, - global: crate::mmu::Vm, - metrics: Arc, - metrics_ids: Arc, - metrics_bases: crate::g17::fw::buffer::MetricsAddresses, - config: Config, -} -impl BuildResources { - fn new(firmware: &crate::g17::Firmware) -> Result { - Ok(Self { - uat: firmware.uat.clone(), - objects: firmware.pool.clone(), - ids: firmware.render_ids.clone(), - global: firmware._render_global.clone(), - metrics: firmware._pm_metrics.clone(), - metrics_ids: firmware.queues.metrics.clone(), - metrics_bases: crate::g17::fw::buffer::MetricsAddresses { - firmware: firmware.init.metrics_fw_va()?, - client: crate::hw::t8140::CONFIG.pm_metrics.va, - }, - config: firmware.queues.render.config, - }) - } - fn metrics(&self) -> Result { - MetricsLease::new(&self.metrics_ids, &self.metrics, self.metrics_bases) - } -} - -enum Build { - Fresh(Reservation, u64, MetricsLease), - Parked { - parked: crate::g17::queue::render::parked::ParkedPair, - buffer: (u8, u64), - pm_generation: u64, - metrics: MetricsLease, - }, - Rebind { - pair: KBox, - buffer: Option<(u8, u64)>, - pm_generation: Option, - metrics: Option, - }, -} - -/// What every build step needs besides its variant: the preparation lease, the allocator, -/// the device resources, the slot and the first packet. -#[derive(Clone, Copy)] -struct BuildContext<'a> { - lease: &'a Lease, - alloc: &'a Allocator<'a>, - resources: &'a BuildResources, - slot: u8, - first: &'a Arc, -} - -impl super::super::Backend { - fn build_render_pair(&self, slot: u8, first: &Arc, optional: bool) -> Result { - let lease = self.shared.preparations.enter(false)?; - let dev = self.shared.drm_neo()?; - let (resources, build) = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - if !firmware.queues.prepare_render_ids(slot, self.owner, optional)? { - return Ok(()); - } - let resources = BuildResources::new(firmware)?; - let build = - if let Some(entry) = firmware.queues.render.entries[usize::from(slot)].as_ref() { - if entry.pending_error.is_some() || entry.pending_timeout { - self.shared.queue_events(); - return Err(EBUSY); - } - if entry.constructing || entry.preparing.is_some() || entry.growing { - return Err(EBUSY); - } - if entry.parked.is_some() { - let metrics = resources.metrics()?; - let pm_generation = firmware.queues.render.reserve_pm_generation()?; - let buffer = firmware.queues.buffers.allocate_render()?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = Some(first.clone()); - entry.pending_owner = Some(self.owner); - entry.borrowed_active = false; - Build::Parked { - parked: entry.parked.take().ok_or(EIO)?, - buffer, - pm_generation, - metrics, - } - } else if entry.pair.is_none() { - let metrics = resources.metrics()?; - let pm_generation = firmware.queues.render.reserve_pm_generation()?; - Build::Fresh( - firmware.queues.reserve_render(slot, self.owner)?, - pm_generation, - metrics, - ) - } else { - let pair = entry.pair.as_ref().ok_or(EBUSY)?; - if pair.matches(self.owner, &self.context)? { - if !pair.pool().vacant() { - return Ok(()); - } - let pool = pair.pool().clone(); - drop(state); - let alloc = Allocator { - dev: &dev, - uat: &resources.uat, - }; - return pool.ensure_backing(&alloc, &resources.global, self.context.vm()); - } - if !pair.reusable()? { - return Err(EBUSY); - } - let foreign = !Arc::ptr_eq(pair.context().status(), self.context.status()); - if foreign { - firmware - .queues - .render - .save_detaching_pool(&firmware.init, slot)?; - } - let metrics = if foreign { - Some(resources.metrics()?) - } else { - None - }; - let pm_generation = if foreign { - Some(firmware.queues.render.reserve_pm_generation()?) - } else { - None - }; - let buffer = if foreign { - Some(firmware.queues.buffers.allocate_render()?) - } else { - None - }; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = Some(first.clone()); - entry.pending_owner = Some(self.owner); - entry.borrowed_active = entry - .pair - .as_ref() - .is_some_and(|pair| pair.published_in_flight()); - entry.borrowed_dependencies = - entry.pair.as_ref().map(|pair| pair.dependency_snapshot()); - Build::Rebind { - pair: entry.pair.take().ok_or(EIO)?, - buffer, - pm_generation, - metrics, - } - } - } else { - let metrics = resources.metrics()?; - let pm_generation = firmware.queues.render.reserve_pm_generation()?; - Build::Fresh( - firmware.queues.reserve_render(slot, self.owner)?, - pm_generation, - metrics, - ) - }; - (resources, build) - }; - let alloc = Allocator { - dev: &dev, - uat: &resources.uat, - }; - let fresh_graph = match &build { - Build::Fresh(..) | Build::Parked { .. } => true, - Build::Rebind { buffer, .. } => buffer.is_some(), - }; - let ctx = BuildContext { - lease: &lease, - alloc: &alloc, - resources: &resources, - slot, - first, - }; - match build { - Build::Fresh(reservation, pm_generation, metrics) => { - self.install_fresh_pair(&ctx, reservation, pm_generation, metrics)? - } - Build::Parked { - parked, - buffer, - pm_generation, - metrics, - } => self.install_parked_pair(&ctx, parked, buffer, pm_generation, metrics)?, - Build::Rebind { - pair, - buffer, - pm_generation, - metrics, - } => self.rebind_pair(&ctx, pair, buffer, pm_generation, metrics)?, - } - self.finish_render_build(&lease, slot, fresh_graph) - } - - /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler - /// run_job stack path, which is close to its budget. - #[inline(always)] - /// Off-lock construction of a fresh pair, then its device-locked install. - fn install_fresh_pair( - &self, - ctx: &BuildContext<'_>, - reservation: Reservation, - pm_generation: u64, - metrics: MetricsLease, - ) -> Result { - let BuildContext { - lease, - alloc, - resources, - slot, - first, - } = *ctx; - let built = (|| -> Result> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - Pair::new( - &alloc, - &resources.objects, - reservation.ids, - slot, - self.owner, - self.context.clone(), - free_list, - metrics, - reservation.buffer, - pm_generation, - first, - resources.config.clusters, - resources.config.descriptor_flags, - ) - })(); - let (mut built, error) = match built { - Ok(pair) => (Some(pair), None), - Err(error) => (None, Some(error)), - }; - let result = (|| { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - if firmware.queues.render.reserved(reservation)?.closed { - return Err(ECANCELED); - } - built - .as_mut() - .ok_or(EIO)? - .finish_install(&alloc, firmware.init.pb_descriptor_table()?)?; - firmware.queues.install_render(reservation, &mut built) - })(); - // Fresh backing disappears before its temporary PB reference returns. - drop(built); - if result.is_err() { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .cancel_render_reservation(reservation)?; - } - result - } - - /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler - /// run_job stack path, which is close to its budget. - #[inline(always)] - /// Off-lock preparation of a parked pair for its new owner, then its device-locked - /// commit; the pair returns to its parked state on failure. - fn install_parked_pair( - &self, - ctx: &BuildContext<'_>, - mut parked: crate::g17::queue::render::parked::ParkedPair, - buffer: (u8, u64), - pm_generation: u64, - metrics: MetricsLease, - ) -> Result { - let BuildContext { - lease, - alloc, - resources, - slot, - first, - } = *ctx; - let built = (|| -> Result<_> { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - let prepared = parked.prepare( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - )?; - let storage = KBox::::new_uninit(GFP_KERNEL)?; - Ok((prepared, storage)) - })(); - let (mut prepared, mut storage, error) = match built { - Ok((prepared, storage)) => (Some(prepared), Some(storage), None), - Err(error) => (None, None, Some(error)), - }; - let result = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) - || entry.pending_owner_closed - || entry.pair.is_some() - || entry.parked.is_some() - { - return Err(ECANCELED); - } - let storage = storage.take().ok_or(EIO)?; - firmware.queues.buffers.can_release(buffer.0)?; - prepared.as_mut().ok_or(EIO)?.finish_install( - &parked, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - let pair = parked.commit(&mut prepared, storage)?; - firmware.queues.buffers.reserve_owner(buffer.0)?; - firmware.queues.buffers.release(buffer.0)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - entry.reservation.buffer = buffer; - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.pair = Some(pair); - entry.closed = false; - entry.published = false; - Ok(()) - })() - }; - // Rejected candidate mappings disappear before the PB identity - // returns, while the original installed kick owner stays live. - drop(prepared); - drop(storage); - let grow_ready = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if result.is_err() { - firmware.queues.render.entry_mut(slot)?.parked = Some(parked); - firmware.queues.buffers.release(buffer.0)?; - } else { - drop(state); - drop(parked); - state = self.shared.state.lock(); - } - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - firmware.render_grow_ready() - }; - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); - } - result - } - - /// Inlined into build_render_pair(): as a separate frame this step deepens the scheduler - /// run_job stack path, which is close to its budget. - #[inline(always)] - /// Off-lock preparation of an installed pair's rebinding, then its device-locked - /// owner change; the original pair stays installed on failure. - fn rebind_pair( - &self, - ctx: &BuildContext<'_>, - mut pair: KBox, - buffer: Option<(u8, u64)>, - pm_generation: Option, - metrics: Option, - ) -> Result { - let BuildContext { - lease, - alloc, - resources, - slot, - first, - } = *ctx; - let prepared = (|| { - let free_list = - self.context - .render_pool(&alloc, &resources.ids, &resources.global)?; - pair.prepare_bind( - &alloc, - &resources.objects, - self.owner, - self.context.clone(), - free_list, - metrics, - buffer, - pm_generation, - first, - ) - })(); - let (mut prepared, error) = match prepared { - Ok(binding) => (Some(binding), None), - Err(error) => (None, Some(error)), - }; - let (result, grow_ready) = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - let result = (|| { - if !lease.is_current() { - return Err(EAGAIN); - } - if let Some(error) = error { - return Err(error); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_owner != Some(self.owner) || entry.pending_owner_closed { - return Err(ECANCELED); - } - if firmware.queues.render.retired.len() - == firmware.queues.render.retired.capacity() - { - return Err(ENOSPC); - } - firmware.queues.teardown.displace( - pair.context(), - true, - super::super::now_ns(), - )?; - pair.bind_owner( - prepared.as_mut().ok_or(EIO)?, - &alloc, - firmware.init.pb_descriptor_table()?, - )?; - if let Some((id, _)) = buffer { - firmware.queues.buffers.reserve_owner(id)?; - firmware.queues.buffers.release(id)?; - } - let entry = firmware.queues.render.entry_mut(slot)?; - entry.reservation.owner = self.owner; - if let Some(buffer) = buffer { - entry.reservation.buffer = buffer; - } - entry.context = Some(self.context.clone()); - entry.pool = Some(pair.pool().clone()); - entry.closed = false; - Ok(()) - })(); - // The original installed pair remains owned on every error. - let entry = firmware.queues.render.entry_mut(slot)?; - entry.preparing = None; - entry.pending_owner = None; - entry.pending_owner_closed = false; - entry.pair = Some(pair); - entry.borrowed_dependencies = None; - (result, firmware.render_grow_ready()) - }; - drop(prepared); - if result.is_err() { - if let Some((id, _)) = buffer { - let mut state = self.shared.state.lock(); - (*state) - .as_deref_mut() - .ok_or(ENODEV)? - .queues - .buffers - .release(id)?; - } - } - self.shared.changed.notify_all(); - if grow_ready { - self.shared.queue_grow(); - } - result - } - - /// Under the device mutex: the graph is complete and retained; rebase a fresh graph's - /// cursors and warm its pool. - fn finish_render_build(&self, lease: &Lease, slot: u8, fresh_graph: bool) -> Result { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - if firmware.queues.render.entry(slot)?.closed { - return Err(ECANCELED); - } - if !self.context.is_current() || self.context.status().get() != 0 { - return Err(EFAULT); - } - // From this point every error leaves the complete graph retained for - // retry. Physical queue identities and installed mappings stay owned. - if fresh_graph { - firmware - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)? - .rebase_fresh_graph()?; - } - let pool = firmware - .queues - .render - .entry(slot)? - .pool - .clone() - .ok_or(EIO)?; - firmware.queues.render.pools.warm(&firmware.init, &pool) - } - - pub(super) fn ensure_render(&self, packet: &Arc, slot: u8) -> Result { - self.build_render_pair(slot, packet, false)?; - let sibling = { - let state = self.shared.state.lock(); - (*state) - .as_deref() - .ok_or(ENODEV)? - .queues - .admission - .warmable_sibling(self.owner, slot) - }; - if let Some(sibling) = sibling { - // An optional sibling may be borrowed or waiting for recovery. - // The selected pair is revalidated by prepare_render before publication. - match self.build_render_pair(sibling, packet, true) { - Err(error) if error == EBUSY || error == EAGAIN => {} - result => result?, - } - } - Ok(()) - } - - pub(super) fn prepare_render( - &self, - slot: u8, - packet: &Arc, - deferred: &mut super::super::DeferredBatch, - ) -> Result { - let lease = self.shared.preparations.enter(false)?; - let dev = self.shared.drm_neo()?; - let (uat, mut pair, addresses, growth_target) = { - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - if !lease.is_current() { - return Err(EAGAIN); - } - let qids = firmware - .queues - .render - .entry(slot)? - .pair - .as_ref() - .ok_or(EBUSY)? - .qids(); - let addresses = RenderAddresses { - stamps: [ - firmware.init.stamp_addresses(qids[0])?, - firmware.init.stamp_addresses(qids[1])?, - ], - }; - let pair = firmware - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)?; - if !pair.matches(self.owner, &packet.context)? { - return Err(EIO); - } - // The selected scheduler is retained through release acknowledgement - // even if later private command allocation fails. Sibling warming - // does not cross this publication boundary. - packet.context.mark_published(); - let target = pair.host_growth_target(packet)?; - let buffer_id = pair.buffer_id(); - let growth_target = if let Some(target) = target { - let table = firmware.init.pb_descriptor_table()?; - let offset = usize::from(buffer_id) * 16; - core::sync::atomic::fence(core::sync::atomic::Ordering::Acquire); - let mut descriptor = [0; 4]; - for (index, word) in descriptor.iter_mut().enumerate() { - *word = table - .word(offset + index * 4)? - .load(core::sync::atomic::Ordering::Relaxed); - } - Some((target, descriptor)) - } else { - None - }; - let pair = firmware.queues.render.take_pair(slot, self.owner, packet)?; - (firmware.uat.clone(), pair, addresses, growth_target) - }; - let alloc = Allocator { - dev: &dev, - uat: &uat, - }; - let mut growth = None; - let result = (|| { - if let Some((target, descriptor)) = growth_target { - let vm = pair.context().vm().clone(); - growth = pair.manager().prepare_growth( - &dev, - &vm, - target, - Some(descriptor), - )?; - } - pair.prepare(&alloc, packet, addresses) - })(); - let mut returned = Some(pair); - let mut state = self.shared.state.lock(); - let firmware = (*state).as_deref_mut().ok_or(ENODEV)?; - firmware.queues.render.return_pair(slot, &mut returned)?; - if firmware.queues.render.entry(slot)?.closed { - firmware - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)? - .release_owner(); - } - self.shared.changed.notify_all(); - let published = (|| { - let entry = firmware.queues.render.entry(slot)?; - if entry.pending_error.is_some() || entry.pending_timeout { - // Final timeout observations can settle unrelated queues. The - // event worker owns the full collector needed for that sweep. - self.shared.queue_events(); - return Err(EBUSY); - } - if !lease.is_current() - || firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 - { - return Err(EAGAIN); - } - // Private restaging can fail after preparing a larger inventory. - // Finish its descriptor before discarding the command preparation. - firmware.commit_render_growth(slot, &mut growth)?; - result?; - if packet.cancel_requested() || firmware.queues.render.entry(slot)?.closed { - return Err(ECANCELED); - } - firmware.publish_render(self.owner, slot, packet, deferred) - })(); - let result = (|| { - if published.is_err() && !packet.is_published() { - firmware - .queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)? - .discard_prepared(packet)?; - } - published - })(); - let grow_ready = firmware.render_grow_ready(); - drop(state); - if grow_ready { - self.shared.queue_grow(); - } - result - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/recovery.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/recovery.rs deleted file mode 100644 index 7a897a7afca939..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/recovery.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Recovery targets for retained render graphs while firmware is halted. - -use crate::g17::runtime::Registry; -use kernel::prelude::*; - -impl Registry { - pub(in crate::g17::runtime) fn render_recovery_diagnostics( - &self, - qid: u8, - ) -> Option<([u8; 2], Option, u32, u8)> { - let pair = self.render.entries.iter().flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.qids().contains(&qid))?; - Some((pair.qids(), pair.owner(), pair.context().id(), pair.context().generation())) - } - - pub(in crate::g17::runtime) fn render_pass_started(&self, qid: u8, stamp: u64) -> Option { - self.render.entries.iter().flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.qids().contains(&qid))? - .pass_started(stamp) - } - - pub(in crate::g17::runtime) fn publish_render_recovery_targets(&self) -> Option { - let mut failure = None; - for pair in self.render.entries.iter().flatten() - .filter_map(|entry| entry.pair.as_ref()) - { - if let Err(error) = pair.publish_recovery_targets() { - failure = failure.or(Some(error)); - } - } - failure - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/settle.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/settle.rs deleted file mode 100644 index 415d1f391f601b..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/settle.rs +++ /dev/null @@ -1,235 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Exact packet timeout and paired firmware-drain settlement. Every fence and -//! admission release is collected under the device lock and finished outside it. - -use crate::g17::{ - job::Packet, - runtime::{Backend, DeferredBatch}, -}; -use kernel::{dma_fence::RawDmaFence, prelude::*, sync::Arc}; - -impl Backend { - pub(in crate::g17) fn timeout_render(&self, packet: &Arc) -> Result { - let mut deferred = DeferredBatch::worker()?; - let result = { - let mut state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref_mut() else { - return Ok(()); - }; - if firmware.queues.defer_render_timeout(packet, ETIMEDOUT) { - return Ok(()); - } - firmware.fail_render_packet(packet, ETIMEDOUT, &mut deferred) - }; - deferred.finish(); - result - } - pub(in crate::g17) fn cancel_render(&self, packet: &Arc) -> Result { - let fence = packet.completion.scheduler_fence(); - // SAFETY: The retained fence remains alive across this bounded, - // uninterruptible queue-teardown wait, without holding a device lock. - let completed = unsafe { - kernel::bindings::dma_fence_wait_timeout( - fence.raw(), - false, - kernel::time::msecs_to_jiffies(2000) as _, - ) - }; - if completed > 0 { - return Ok(()); - } - if packet.is_published() { - self.timeout_render(packet) - } else { - // The outer callback already completed a successful 0 -> 3 - // cancellation. A claimed publisher still owns its VM pin and - // observes cancel_requested before committing or finishing. - Ok(()) - } - } -} - -impl crate::g17::Firmware { - pub(super) fn fail_render_packet( - &mut self, - packet: &Arc, - error: Error, - deferred: &mut DeferredBatch, - ) -> Result { - let Some(slot) = packet.completion.render_slot() else { - return Ok(()); - }; - let slot = usize::from(slot); - loop { - let pair = self - .queues - .render - .entries - .get(slot) - .and_then(Option::as_ref) - .and_then(|entry| entry.pair.as_ref()) - .ok_or(EIO)?; - if !pair.contains_packet(packet) { - return Ok(()); - } - let generation = self.recovery.generation(); - let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - let progressed = self.with_render(slot, |pair, host| { - pair.retire(None, generation, state, host, &mut |event| { - deferred.push(event) - }) - })?; - if !progressed { - break; - } - match self.release_render_lists() { - Err(error) if error == EAGAIN => {} - result => result?, - } - } - self.with_render(slot, |pair, host| { - pair.fail_after_observation(packet, error, host, &mut |event| deferred.push(event)) - })?; - self.queues.render.collect_retired(slot)?; - let terminal = self.queues.render.entries[slot] - .as_mut() - .and_then(|entry| entry.pair.as_mut()) - .and_then(|pair| pair.take_terminal()); - if let Some((status, error)) = terminal { - status.record(error); - self.scan_compute(None, deferred)?; - self.settle_render_vm(&status, error, deferred)?; - self.queues.fail_compute_vm(&status, error, deferred)?; - } - Ok(()) - } - - pub(in crate::g17) fn settle_blamed_render( - &mut self, - qid: u8, - error: Error, - deferred: &mut DeferredBatch, - ) -> Result { - let packet = self - .queues - .render - .entries - .iter() - .flatten() - .filter_map(|entry| entry.pair.as_ref()) - .find(|pair| pair.qids().contains(&qid)) - .and_then(|pair| pair.oldest_packet()); - if let Some(packet) = packet { - self.fail_render_packet(&packet, error, deferred)?; - } - Ok(()) - } - - pub(in crate::g17) fn settle_cancelled_renders( - &mut self, - mask: u128, - deferred: &mut DeferredBatch, - ) -> Result { - for slot in 1..self.queues.render.entries.len() { - let packet = self.queues.render.entries[slot] - .as_ref() - .and_then(|entry| entry.pair.as_ref()) - .filter(|pair| { - pair.qids() - .into_iter() - .any(|qid| mask & (1u128 << qid) != 0) - && pair.oldest_spared() == Some(true) - }) - .and_then(|pair| pair.oldest_packet()); - if let Some(packet) = packet { - self.fail_render_packet(&packet, ECANCELED, deferred)?; - } - } - Ok(()) - } - - pub(in crate::g17) fn reclaim_drained_renders( - &mut self, - deferred: &mut DeferredBatch, - ) -> Result { - if self.recovery.pending() - || self - .primary - .state - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Ok(()); - } - let mut serviced = false; - for slot in 1..self.queues.render.entries.len() { - if !self.queues.render.entries[slot] - .as_ref() - .and_then(|entry| entry.pair.as_ref()) - .is_some_and(|pair| pair.needs_drain()) - { - continue; - } - if let Err(error) = self.drain_render_slot(slot, &mut serviced, deferred) { - dev_warn!( - self.primary.state.shared.dev.as_ref(), - "Could not settle drained render pair {}: {:?}\n", - slot, - error - ); - } - } - Ok(()) - } - - fn drain_render_slot( - &mut self, - slot: usize, - serviced: &mut bool, - deferred: &mut DeferredBatch, - ) -> Result { - if !self.with_render(slot, |pair, host| pair.firmware_drained(host))? { - return Ok(()); - } - if !*serviced { - self.service_runtime_events(deferred)?; - *serviced = true; - } - if self.recovery.pending() - || self - .primary - .state - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Ok(()); - } - loop { - let accounted = self.with_render(slot, |pair, host| pair.account_drained(host))?; - if !accounted { - break; - } - match self.release_render_lists() { - Err(error) if error == EAGAIN => {} - result => result?, - } - } - if self.recovery.pending() - || self - .primary - .state - .shared - .crashed - .load(core::sync::atomic::Ordering::Acquire) - { - return Ok(()); - } - self.with_render(slot, |pair, host| { - pair.release_witnessed(host, &mut |event| deferred.push(event)) - })?; - self.queues.render.collect_retired(slot) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/submit.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/submit.rs deleted file mode 100644 index dbf7175c5cd4bd..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/submit.rs +++ /dev/null @@ -1,324 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Submission admission, preparation and the firmware visibility boundary. - -use super::super::{Backend, DeferredBatch, Registry}; -use crate::g17::{ - job::Packet, - queue::admission::{RenderLease, Request}, -}; -use core::sync::atomic::Ordering; -use kernel::{ - prelude::*, - sync::Arc, - time::{delay::fsleep, Delta}, -}; - -impl Backend { - pub(in crate::g17) fn render_packet( - &self, - packet: &Arc, - deferred: &mut DeferredBatch, - ) -> Result { - self.render_with_lease(packet, None, deferred) - } - - fn render_with_lease( - &self, - packet: &Arc, - early: Option, - deferred: &mut DeferredBatch, - ) -> Result { - packet.check_dependencies()?; - if packet.cancel_requested() { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - let interruptible = early.is_none(); - let admission = match early { - Some(lease) => lease, - None => { - let pool = { - let state = self.shared.state.lock(); - (*state).as_deref().ok_or(ENODEV)?.queues.admission.clone() - }; - pool.acquire_render(self.owner, Request::of(packet))? - } - }; - let slot = admission.slot(); - packet.completion.attach_render_lease(admission)?; - let mut state_polls = 0; - let mut state_delay = 20i64; - let mut takeover_polls = 0; - let mut recovery_polls = 0; - loop { - if packet.cancel_requested() { - return Err(ECANCELED); - } - if packet.completion.status().get() != 0 || self.shared.crashed.load(Ordering::Acquire) - { - return Err(EIO); - } - if !self.context.is_current() { - return Err(EFAULT); - } - let work_state = { - let preparation = self.shared.preparations.enter(interruptible)?; - let result = packet.completion.work_state().and_then(|state| state.acquire()); - if preparation.is_current() { - result - } else { - Err(EAGAIN) - } - }; - match work_state { - Err(error) if error == EAGAIN => continue, - Err(error) if error == EBUSY => { - if state_polls == 1000 { - return Err(ETIMEDOUT); - } - state_polls += 1; - fsleep(Delta::from_micros(state_delay)); - state_delay = (state_delay * 2).min(1000); - continue; - } - result => { - result?; - } - } - match self.ensure_render(packet, slot) { - Err(error) if error == EAGAIN => continue, - Err(error) if error == EBUSY => { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); - continue; - } - result => result?, - } - match self.prepare_render(slot, packet, deferred) { - Err(error) if !packet.is_published() && [EAGAIN, EBUSY].contains(&error) => { - let recovering = { - let state = self.shared.state.lock(); - let firmware = (*state).as_deref().ok_or(ENODEV)?; - firmware.recovery.pending() - || crate::g17::recovery::Memory::recovery_state(&firmware.init)? != 0 - }; - if recovering { - if recovery_polls == super::super::RECOVERY_SETTLE_POLLS { - return Err(ETIMEDOUT); - } - recovery_polls += 1; - self.shared.queue_events(); - fsleep(Delta::from_millis(1)); - } else if error == EBUSY { - if takeover_polls == 5000 { - return Err(ETIMEDOUT); - } - takeover_polls += 1; - fsleep(Delta::from_micros(200)); - } - } - result => return result, - } - } - } - - pub(in crate::g17) fn try_render_early(&self, packet: &Arc) -> bool { - if self.backlog.load(Ordering::Acquire) != 0 || packet.has_dependencies() { - return false; - } - let admission = { - let state = self.shared.state.lock(); - let Some(firmware) = (*state).as_deref() else { - return false; - }; - match firmware - .queues - .admission - .try_acquire_render(self.owner, Request::of(packet)) - { - Ok(Some(lease)) => lease, - _ => return false, - } - }; - if !packet.begin_publication() { - packet.completion.fail_unpublished(ECANCELED); - packet.set_early_result(Err(ECANCELED)); - return true; - } - let mut deferred = match DeferredBatch::publication() { - Ok(deferred) => deferred, - Err(error) => { - if packet.completion.status().get() != 0 { - ::fail_vm(self, error); - } - packet.completion.fail_unpublished(error); - packet.set_early_result(Err(error)); - return true; - } - }; - let result = self.render_with_lease(packet, Some(admission), &mut deferred); - let result = self - .finish_publication(packet, result, &mut deferred) - .map(|_| ()); - packet.set_early_result(result); - true - } -} - -impl Registry { - fn render_dependencies( - &self, - owner: u64, - packet: &Arc, - state: u32, - ) -> Result { - let mut dependencies = crate::g17::queue::render::RenderDependencies::EMPTY; - let frontier = self.render_prefix(owner, packet)?; - for dependency in frontier.as_slice() { - *dependencies - .render - .get_mut(dependencies.render_count) - .ok_or(EOVERFLOW)? = *dependency; - dependencies.render_count += 1; - } - if let Some(queue) = self.compute(owner) { - dependencies.compute = queue.dependency( - owner, - packet.completion.status(), - packet.order.wait_through[1], - state, - )?; - } - Ok(dependencies) - } -} - -impl crate::g17::Firmware { - pub(super) fn apply_render_deferred( - &mut self, - slot: u8, - deferred: &mut DeferredBatch, - ) -> Result { - let observe_before_failure = { - let entry = self.queues.render.entry_mut(slot)?; - let pair = entry.pair.as_ref().ok_or(EIO)?; - entry.pending_error.as_ref().is_some_and(|(status, _)| { - Arc::ptr_eq(pair.context().status(), status) - }) - }; - if observe_before_failure { - // VM failure could not observe this pair while preparation owned - // it. Restore the same final-observation boundary used by - // settle_render_vm before quarantining any still-owned commands. - // Keep pending_error intact if observation itself fails. - self.retire_render_slot(usize::from(slot), None, deferred)?; - } - let entry = self.queues.render.entry_mut(slot)?; - let pair = entry.pair.as_mut().ok_or(EIO)?; - if let Some((status, error)) = entry.pending_error.take() { - if Arc::ptr_eq(pair.context().status(), &status) { - pair.quarantine(error, &mut |event| deferred.push(event))?; - } - } - if entry.closed { - pair.release_owner(); - } - if !entry.pending_timeout { - return Ok(()); - } - for _ in 0..crate::hw::t8140::queues::RENDER_DEPTH { - if self.recovery.pending() || self.primary.state.shared.crashed.load(Ordering::Acquire) - { - return Ok(()); - } - let Some((packet, error)) = self - .queues - .render - .entry(slot)? - .pair - .as_ref() - .ok_or(EIO)? - .take_deferred_timeout() - else { - self.queues.render.entry_mut(slot)?.pending_timeout = false; - return Ok(()); - }; - if let Err(failure) = self.fail_render_packet(&packet, error, deferred) { - packet.completion.defer_timeout(error); - return Err(failure); - } - } - self.queues.render.entry_mut(slot)?.pending_timeout = false; - Ok(()) - } - - pub(super) fn publish_render( - &mut self, - owner: u64, - slot: u8, - packet: &Arc, - deferred: &mut DeferredBatch, - ) -> Result { - use crate::g17::queue::render::PublicationLease; - let state = crate::g17::recovery::Memory::recovery_state(&self.init)?; - if self.recovery.pending() || state != 0 { - return Err(EAGAIN); - } - if packet.completion.status().get() != 0 { - return Err(EIO); - } - // Both outer destinations must have room before pool accounting or - // either inner queue becomes visible. All host producers share this lock. - let priority = packet.context.policy().priority(); - if !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Fragment)? - || !self.init.work_ready(priority, crate::g17::fw::queue::DataMaster::Tiling)? - { - return Err(EBUSY); - } - let dependencies = self.queues.render_dependencies(owner, packet, state)?; - let pair = self.queues.render.entry(slot)?.pair.as_ref().ok_or(EIO)?; - let pool = pair.pool().clone(); - let buffer_id = pair.buffer_id(); - let (buffer_id, buffer_token) = self.queues.buffers.acquire_owner(buffer_id)?; - let preparation = match self.queues.render.pools.prepare(&self.init, &pool) { - Ok(preparation) => preparation, - Err(error) => { - self.queues.buffers.release(buffer_id)?; - return Err(error); - } - }; - let lease = PublicationLease { - free_list_generation: preparation.generation, - buffer_token, - }; - let result = self.with_render(usize::from(slot), |pair, host| { - pair.publish( - packet.clone(), - &dependencies, - lease, - host, - &mut |event| deferred.push(event), - ) - }); - if packet.is_published() { - self.queues.render.entry_mut(slot)?.published = true; - } else if result.is_err() { - self.queues.render.pools.rollback(preparation); - self.queues.buffers.release(buffer_id)?; - self.queues - .render - .entry_mut(slot)? - .pair - .as_mut() - .ok_or(EIO)? - .discard_prepared(packet)?; - } - result - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/render/teardown.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/render/teardown.rs deleted file mode 100644 index 759c0c2e43dc0c..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/render/teardown.rs +++ /dev/null @@ -1,164 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Exact logical ownership gates for scheduler release and physical-pair parking. - -use super::RenderState; -use crate::g17::{ - buffer::BufferIds, - context::Context, - initdata::InitData, - runtime::{teardown::ReclaimBatch, Registry}, -}; -use kernel::{prelude::*, sync::Arc}; - -impl RenderState { - pub(in crate::g17::runtime) fn observe_teardown_releases(&mut self, init: &InitData) -> Result { - self.pools.observe(init) - } - - pub(in crate::g17::runtime) fn teardown_idle(&self, context: &Arc) -> Result { - for entry in self.entries.iter().flatten() { - if !entry - .context - .as_ref() - .is_some_and(|owner| Arc::ptr_eq(owner, context)) - { - continue; - } - if entry.preparing.is_some() || entry.growing { - return Ok(false); - } - if let Some(pair) = entry.pair.as_ref() { - if !pair.idle_for_context(context)? { - return Ok(false); - } - } - } - Ok(true) - } - - pub(in crate::g17::runtime) fn teardown_pools_released(&self, context: &Arc) -> bool { - context.render_pools_released(|id, control| self.pools.released(id, control)) - && self - .retired - .iter() - .filter(|old| Arc::ptr_eq(old.context(), context)) - .all(|old| { - self.pools - .released(u32::from(old.pool().id()), old.pool().control_va()) - }) - } - - /// Warm superseded roots need only their exact executed replacement witness. - /// Graphs with their own memory remain until that Context's scheduler ack. - pub(in crate::g17::runtime) fn collect_reclaimable( - &mut self, - out: &mut ReclaimBatch, - ) -> Result { - for slot in 0..self.entries.len() { - self.collect_retired(slot)?; - } - let mut index = 0; - while index < self.retired.len() { - if self.retired[index].buffer_id().is_some() { - index += 1; - continue; - } - let Ok(target) = out.binding_slot() else { - break; - }; - *target = Some(self.retired.remove(index).map_err(|_| EIO)?); - } - Ok(()) - } - - /// Preflight the entire Context before detaching its first graph. Successor - /// graphs on the same VM have a different Arc and remain installed. - pub(in crate::g17::runtime) fn detach_context( - &mut self, - context: &Arc, - buffers: &mut BufferIds, - out: &mut ReclaimBatch, - ) -> Result { - let mut count = self - .retired - .iter() - .filter(|old| Arc::ptr_eq(old.context(), context)) - .count(); - for entry in self.entries.iter().flatten() { - if !entry - .context - .as_ref() - .is_some_and(|owner| Arc::ptr_eq(owner, context)) - { - continue; - } - if entry.preparing.is_some() || entry.growing || entry.parked.is_some() { - return Err(EBUSY); - } - if let Some(pair) = entry.pair.as_ref() { - if !pair.can_park(context)? { - return Err(EBUSY); - } - count += 1; - } - } - if !out.has_room(count, 1) { - return Err(EBUSY); - } - for entry in self.entries.iter_mut().flatten() { - if !entry - .context - .as_ref() - .is_some_and(|owner| Arc::ptr_eq(owner, context)) - { - continue; - } - let Some(pair) = entry.pair.as_ref() else { - continue; - }; - buffers.unreserve_owner(pair.buffer_id())?; - let target = out.binding_slot()?; - let pair = entry.pair.take().ok_or(EIO)?; - let (parked, retired) = KBox::into_inner(pair).park(); - *target = Some(retired); - entry.parked = Some(parked); - // Detached backing retains these references until the worker runs. - entry.context = None; - entry.pool = None; - } - let mut index = 0; - while index < self.retired.len() { - if !Arc::ptr_eq(self.retired[index].context(), context) { - index += 1; - continue; - } - if let Some(id) = self.retired[index].buffer_id() { - buffers.unreserve_owner(id)?; - } - let target = out.binding_slot()?; - *target = Some(self.retired.remove(index).map_err(|_| EIO)?); - } - Ok(()) - } -} - -impl Registry { - /// Close while a builder owns a pair records intent. Its return path applies - /// the release before admitting further commands from either logical owner. - pub(in crate::g17::runtime) fn release_render(&mut self, owner: u64) -> Result { - for entry in self.render.entries.iter_mut().flatten() { - if entry.pending_owner == Some(owner) { - entry.pending_owner_closed = true; - } - if entry.reservation.owner != owner { - continue; - } - entry.closed = true; - if let Some(pair) = entry.pair.as_mut() { - pair.release_owner(); - } - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/runtime/teardown.rs b/drivers/gpu/drm/asahi-neo/g17/runtime/teardown.rs deleted file mode 100644 index 8a637e0b3ed01c..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/runtime/teardown.rs +++ /dev/null @@ -1,199 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Scheduler release and off-lock destruction of acknowledged logical queue resources. - -use super::{ComputeEntry, Registry}; -use crate::g17::{ - buffer::BufferIds, - channel::Rings, - context::Context, - fw::channels::{ControlRecord, SchedulerStateRelease}, - initdata::InitData, - queue::render::RetiredBinding, - recovery::Memory as _, - teardown::{Failure, Host}, - Coprocessor, Firmware, MSG_CONTROL_NOTIFY, -}; -use crate::hw::t8140::queues::RENDER_SLOTS; -use core::sync::atomic::Ordering; -use kernel::{prelude::*, sync::Arc}; - -const CONTEXTS: usize = 128; - -/// Pop under the reclaim mutex, then finish after dropping every device lock. -pub(in crate::g17) enum Reclaim { - Binding(RetiredBinding), - Context(Arc), -} - -impl Reclaim { - pub(in crate::g17) fn finish(self) { - match self { - Self::Binding(binding) => drop(binding), - Self::Context(context) => { - context.release_execution(); - drop(context.take_render_pool()); - drop(context); - } - } - } -} - -/// All slots are allocated at probe. A full collector postpones detach. Empty -/// slots let a validated transfer move ownership without a fallible vector push. -pub(in crate::g17) struct ReclaimBatch { - bindings: KVVec>, - contexts: KVec>>, -} - -impl ReclaimBatch { - pub(in crate::g17) fn new() -> Result { - let mut bindings = KVVec::with_capacity(CONTEXTS + RENDER_SLOTS, GFP_KERNEL)?; - for _ in 0..CONTEXTS + RENDER_SLOTS { - bindings.push(None, GFP_KERNEL)?; - } - let mut contexts = KVec::with_capacity(CONTEXTS, GFP_KERNEL)?; - for _ in 0..CONTEXTS { - contexts.push(None, GFP_KERNEL)?; - } - Ok(Self { bindings, contexts }) - } - - pub(in crate::g17) fn is_empty(&self) -> bool { - self.bindings.iter().all(Option::is_none) && self.contexts.iter().all(Option::is_none) - } - - /// Graphs go before the Context owners that armed their scheduler releases. - pub(in crate::g17) fn take_one(&mut self) -> Option { - if let Some(slot) = self.bindings.iter_mut().find(|slot| slot.is_some()) { - return slot.take().map(Reclaim::Binding); - } - self.contexts - .iter_mut() - .find(|slot| slot.is_some()) - .and_then(Option::take) - .map(Reclaim::Context) - } - - pub(in crate::g17::runtime) fn has_room(&self, bindings: usize, contexts: usize) -> bool { - self.bindings.iter().filter(|slot| slot.is_none()).count() >= bindings - && self.contexts.iter().filter(|slot| slot.is_none()).count() >= contexts - } - - pub(in crate::g17::runtime) fn binding_slot(&mut self) -> Result<&mut Option> { - self.bindings - .iter_mut() - .find(|slot| slot.is_none()) - .ok_or(EBUSY) - } - - fn context(&mut self, context: &Arc) -> Result { - let slot = self - .contexts - .iter_mut() - .find(|slot| slot.is_none()) - .ok_or(EBUSY)?; - *slot = Some(context.clone()); - Ok(()) - } -} - -struct Service<'a> { - render: &'a mut super::render::RenderState, - compute: &'a [Option], - buffers: &'a mut BufferIds, - init: &'a InitData, - primary: &'a mut Coprocessor, - reclaim: &'a mut ReclaimBatch, -} - -impl Host for Service<'_> { - fn render_idle(&mut self, context: &Arc) -> Result { - self.render.teardown_idle(context) - } - fn compute_idle(&self, context: &Arc) -> bool { - self.compute - .iter() - .flatten() - .filter_map(|entry| entry.queue.as_ref()) - .all(|queue| queue.idle_for_context(context)) - } - fn flists_released(&self, context: &Arc) -> bool { - self.render.teardown_pools_released(context) - } - fn control_backpressured(&self) -> bool { - self.render.control_backpressured() - } - fn publish(&mut self, record: &SchedulerStateRelease) -> Result { - Rings::publish_control( - self.init, - &ControlRecord::SchedulerStateRelease(SchedulerStateRelease::new( - record.page_va, - record.id, - )), - ) - } - fn notify(&mut self) -> Result { - self.primary.notify(MSG_CONTROL_NOTIFY) - } - fn detach(&mut self, context: &Arc) -> Result { - self.render - .detach_context(context, self.buffers, self.reclaim)?; - // detach_context preflights this slot together with every graph slot. - // Retain it before Pending removes its reference under the device lock. - self.reclaim.context(context) - } - fn report(&mut self, context: &Arc, failure: Failure) { - let dev = self.primary.state.shared.dev.as_ref(); - match failure { - Failure::Publication(error) => dev_err!( - dev, - "Scheduler {} release failed: {:?}\n", - context.id(), - error - ), - Failure::Announcement(error) => dev_warn!( - dev, - "Scheduler {} release announcement failed: {:?}\n", - context.id(), - error - ), - } - } -} - -impl Firmware { - /// The event worker has already reclaimed drained pairs and reannounced - /// pending controls. Recovery blocks new releases, never an observed ack. - pub(in crate::g17) fn service_teardown(&mut self, reclaim: &mut ReclaimBatch) -> Result { - self.queues.render.collect_reclaimable(reclaim)?; - if self.queues.teardown.is_empty() { - return Ok(()); - } - self.queues.render.observe_teardown_releases(&self.init)?; - let recovery = self.recovery.pending() || self.init.recovery_state()? != 0; - let Registry { - render, - compute, - buffers, - teardown, - .. - } = &mut self.queues; - let mut service = Service { - render, - compute, - buffers, - init: &self.init, - primary: &mut self.primary, - reclaim, - }; - teardown.publish_ready(&mut service, recovery, super::now_ns())?; - let consumer = self.init.control_consumer()?.load(Ordering::Relaxed); - let producer = self.init.control_producer()?.load(Ordering::Relaxed); - while teardown - .take_consumed(consumer, producer, &mut service)? - .is_some() - {} - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/status.rs b/drivers/gpu/drm/asahi-neo/g17/status.rs deleted file mode 100644 index 92192d7e20d569..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/status.rs +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! VM admission errors and client failure reporting. No firmware state is accessed here. - -use crate::gem; -use core::sync::atomic::{AtomicI32, Ordering}; -use kernel::{new_mutex, prelude::*, sync::Mutex}; - -const MIRROR_MAX_SIZE: usize = 64 * 1024; - -/// A VM and its accepted jobs share this status, including after queue destruction. -/// Client failure is separate from hardware admission: recovery may preserve the -/// VM while failed accepted work must still be reported before signalling fences. -#[pin_data] -pub(crate) struct VmStatus { - error: AtomicI32, - reported_error: AtomicI32, - /// The client's mapped status word, retained with its object only while the - /// client's VM handle exists; released by `release_mirror` at VM close. - #[pin] - mirror: Mutex>, -} - -/// One aligned word inside the retained, vmapped mirror object. -struct Mirror { - word: *mut i32, - _owner: gem::ObjectRef, -} -// SAFETY: The word points into the mapping the retained owner keeps alive; it is -// only accessed atomically through this pointer while the mutex is held. -unsafe impl Send for Mirror {} - -impl VmStatus { - pub(crate) fn new() -> impl PinInit { - pin_init!(Self { - error: AtomicI32::new(0), - reported_error: AtomicI32::new(0), - mirror <- new_mutex!(None, "Asahi VM status mirror"), - }) - } - - pub(crate) fn get(&self) -> i32 { - self.error.load(Ordering::Acquire) - } - - /// Record permanent admission failure and report it to the client. - #[track_caller] - pub(crate) fn record(&self, error: Error) { - self.report_failure(error); - let _ = self.error.compare_exchange( - 0, - error.to_errno(), - Ordering::SeqCst, - Ordering::SeqCst, - ); - } - - pub(crate) fn reported_error(&self) -> i32 { - self.reported_error.load(Ordering::Acquire) - } - - /// Report failed accepted work without preventing recovery of the VM. - /// Every reporter publishes the first error before returning: a losing - /// reporter may signal its fence before the winning reporter resumes. - /// - /// The first failure reported to a VM is logged once, with the reporting site: the client treats - /// any reported failure as a lost device, and nothing else in the kernel log names it. - #[track_caller] - pub(crate) fn report_failure(&self, error: Error) { - let error = match self.reported_error.compare_exchange( - 0, - error.to_errno(), - Ordering::SeqCst, - Ordering::SeqCst, - ) { - Ok(_) => { - let at = core::panic::Location::caller(); - pr_warn!( - "GPU work of a client failed ({:?}, reported at {}:{}); the client sees a lost device\n", - error, - at.file().rsplit("asahi/").next().unwrap_or(at.file()), - at.line() - ); - error.to_errno() - } - Err(first) => first, - }; - let mirror = self.mirror.lock(); - if let Some(mirror) = mirror.as_ref() { - // SAFETY: attach_mirror stored only an aligned word within a mapping the - // retained owner keeps alive while this entry exists under the mutex. - unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst); - } - } - - /// Rejected submissions and contention leave the VM usable. - #[track_caller] - pub(crate) fn record_if_device_loss(&self, error: Error) { - if error == EIO || error == ETIMEDOUT || error == ENODEV { - self.record(error); - } - } - - /// Retain one mapped GEM word until the client's VM handle is dropped. - pub(crate) fn attach_mirror(&self, mut owner: gem::ObjectRef, offset: usize) -> Result { - if owner.size() > MIRROR_MAX_SIZE - || offset % core::mem::align_of::() != 0 - || offset - .checked_add(core::mem::size_of::()) - .is_none_or(|end| end > owner.size()) - { - return Err(EINVAL); - } - let base = owner.vmap()?.as_mut_ptr(); - // SAFETY: The checked range is inside the retained, page-aligned mapping. - let word = unsafe { base.add(offset) }.cast::(); - let mut mirror = self.mirror.lock(); - if mirror.is_some() { - return Err(EBUSY); - } - // No reporter runs while the mutex is held, so the word holds the first - // reported error (or zero) when it becomes visible. - let error = self.reported_error.load(Ordering::SeqCst); - // SAFETY: The retained owner keeps this checked, aligned word mapped. - unsafe { AtomicI32::from_ptr(word) }.store(error, Ordering::SeqCst); - *mirror = Some(Mirror { - word, - _owner: owner, - }); - Ok(()) - } - - /// The client's VM handle is gone and nobody reads the mirror any more. The - /// mapping and object reference drop outside the mutex (GEM release may sleep). - pub(crate) fn release_mirror(&self) { - let taken = self.mirror.lock().take(); - drop(taken); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/teardown.rs b/drivers/gpu/drm/asahi-neo/g17/teardown.rs deleted file mode 100644 index 1a25cd7e3fa3b1..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/teardown.rs +++ /dev/null @@ -1,229 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Retained scheduler-state releases. Logical close arms a release; render and compute drain, -//! private-memory release and control-ring consumption are separate ownership witnesses. -//! The device mutex serializes this state with the Registry's displaced graph ownership. - -use super::{context::Context, freelist::control_consumed, fw::channels::SchedulerStateRelease}; -use core::sync::atomic::{fence, Ordering}; -use kernel::{prelude::*, sync::Arc}; - -/// Entries preallocated at device start; a burst of closes beyond it grows the table. -const CAPACITY: usize = 128; -const POLL_NS: u64 = 2_000_000_000; - -#[derive(Clone, Copy, PartialEq, Eq)] -enum Phase { - Deferred, - Draining, - Released(u32), -} - -struct Entry { - context: Arc, - phase: Phase, - armed: u64, - announce_pending: bool, -} - -/// Publication and announcement errors retain the entry; no timeout frees -/// memory the firmware may still own. -#[derive(Clone, Copy)] -pub(crate) enum Failure { - Publication(Error), - Announcement(Error), -} - -/// Borrowed device operations. Readiness covers every matching installed and displaced graph, -/// including pools kept by the Context after a same-VM successor took over its physical pair. -/// `detach` must retain all detached backing for off-lock destruction before returning success. -/// It compares logical Arc identity so a successor sharing an execution ID remains installed. -pub(crate) trait Host { - fn render_idle(&mut self, context: &Arc) -> Result; - fn compute_idle(&self, context: &Arc) -> bool; - fn flists_released(&self, context: &Arc) -> bool; - fn control_backpressured(&self) -> bool; - fn publish(&mut self, record: &SchedulerStateRelease) -> Result; - fn notify(&mut self) -> Result; - fn detach(&mut self, context: &Arc) -> Result; - fn report(&mut self, context: &Arc, failure: Failure); -} - -pub(crate) struct Pending { - entries: KVec, -} - -impl Pending { - pub(crate) fn new() -> Result { - Ok(Self { - entries: KVec::with_capacity(CAPACITY, GFP_KERNEL)?, - }) - } - - pub(crate) fn closed(&self, index: usize, now: u64) -> Option<(Arc, u64)> { - self.entries - .get(index) - .filter(|entry| entry.phase == Phase::Draining) - .map(|entry| (entry.context.clone(), now.saturating_sub(entry.armed))) - } - - pub(crate) fn len(&self) -> usize { - self.entries.len() - } - - pub(crate) fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - fn find(&self, context: &Arc) -> Option { - self.entries - .iter() - .position(|entry| Arc::ptr_eq(&entry.context, context)) - } - - /// Closes outpacing firmware release consumption grow the table instead of - /// failing the close: a failed close stops the device for every client. - fn insert(&mut self, context: &Arc, phase: Phase, now: u64) -> Result { - let entry = Entry { - context: context.clone(), - phase, - armed: now, - announce_pending: false, - }; - self.entries.push(entry, GFP_KERNEL)?; - Ok(()) - } - - /// Call before replacing a graph's logical owner. Failure leaves that graph with its - /// original owner. The Registry, rather than this state machine, retains displaced backing. - pub(crate) fn displace( - &mut self, - context: &Arc, - owner_closed: bool, - now: u64, - ) -> Result { - if self.find(context).is_some() { - return Ok(()); - } - let phase = if owner_closed { - Phase::Draining - } else { - Phase::Deferred - }; - self.insert(context, phase, now) - } - - /// No allocation is needed at close. A scheduler page never exposed to firmware requires - /// no release, unless the Registry already retained one of its displaced graphs. - pub(crate) fn close(&mut self, context: &Arc, now: u64) -> Result { - if let Some(index) = self.find(context) { - let entry = &mut self.entries[index]; - if entry.phase == Phase::Deferred { - entry.phase = Phase::Draining; - entry.armed = now; - } - return Ok(()); - } - if context.is_published() { - self.insert(context, Phase::Draining, now)?; - } - Ok(()) - } - - /// Publish in insertion order. Mandatory control replies and a full control ring stop this - /// pass; an error for one page leaves it retained while other ready releases may progress. - pub(crate) fn publish_ready(&mut self, host: &mut impl Host, recovery: bool, now: u64) -> Result { - if !recovery { - for entry in self.entries.iter_mut() { - // The ring record is already published. Retry only its doorbell; - // never duplicate a scheduler release after a transport error. - if matches!(entry.phase, Phase::Released(_)) { - if entry.announce_pending && host.notify().is_ok() { - entry.announce_pending = false; - // One successful retry starts its consumption window. - // Failed retries never slide the polling deadline. - entry.armed = now; - } - continue; - } - if entry.phase != Phase::Draining - || !host.render_idle(&entry.context)? - || !host.flists_released(&entry.context) - || !host.compute_idle(&entry.context) - { - continue; - } - if host.control_backpressured() { - break; - } - let result = (|| { - let id = entry.context.release_identity()?; - let record = SchedulerStateRelease::new( - entry.context.scheduler_va(), - [id[3], id[0], id[1], id[2]], - ); - host.publish(&record) - })(); - let next = match result { - Ok(next) => next, - Err(EAGAIN) => break, - Err(error) => { - host.report(&entry.context, Failure::Publication(error)); - continue; - } - }; - entry.phase = Phase::Released(next); - // A long command drain must not consume the acknowledgement - // polling window before this release exists on the control ring. - entry.armed = now; - entry.announce_pending = true; - fence(Ordering::SeqCst); - match host.notify() { - Ok(()) => entry.announce_pending = false, - Err(error) => host.report(&entry.context, Failure::Announcement(error)), - } - } - } - fence(Ordering::SeqCst); - Ok(()) - } - - /// The caller snapshots both control cursors once for this settlement pass and repeatedly - /// takes consumed entries. It retains each returned Context until its detached graphs and - /// VM aliases can be destroyed outside the device mutex. - pub(crate) fn take_consumed( - &mut self, - consumer: u32, - producer: u32, - host: &mut impl Host, - ) -> Result>> { - for index in 0..self.entries.len() { - let entry = &self.entries[index]; - let Phase::Released(target) = entry.phase else { - continue; - }; - if !control_consumed(consumer, producer, target)? - || !host.flists_released(&entry.context) - { - continue; - } - match host.detach(&entry.context) { - Ok(()) => {} - Err(EBUSY) => continue, - Err(error) => return Err(error), - } - let entry = self.entries.remove(index).map_err(|_| EIO)?; - return Ok(Some(entry.context)); - } - Ok(None) - } - - /// Poll for a bounded window at close and again at actual release publication. - /// Failed announcements remain retryable on later service even after polling - /// expires. Time passage never substitutes for a firmware-consumption witness. - pub(crate) fn polling(&self, now: u64) -> bool { - self.entries.iter().any(|entry| { - entry.phase != Phase::Deferred && now.saturating_sub(entry.armed) < POLL_NS - }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/timeout.rs b/drivers/gpu/drm/asahi-neo/g17/timeout.rs deleted file mode 100644 index e76dab07f56db7..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/timeout.rs +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Bounded isolation for a compute watchdog blocked behind another VM's older kick. -//! Publication order is evidence of a possible blocker, not proof of a firmware -//! dependency. Renewals are bounded; the oldest unfinished queue is never excused. - -pub(crate) const MAX_RENEWALS: u8 = 8; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct Head { - pub(crate) vm: usize, - pub(crate) published: u64, - pub(crate) finished: bool, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Decision { - Own, - Defer, - Spare, -} - -pub(crate) fn classify(own: Head, renewals: u8, others: impl Iterator) -> Decision { - if own.finished || !others.into_iter().any(|other| { - other.vm != own.vm && !other.finished && other.published < own.published - }) { - Decision::Own - } else if renewals < MAX_RENEWALS { - Decision::Defer - } else { - Decision::Spare - } -} - -#[cfg(test)] -mod tests { - use super::*; - fn head(vm: usize, published: u64, finished: bool) -> Head { - Head { vm, published, finished } - } - #[test] - fn earlier_other_vm_gets_bounded_grace() { - let own = head(5, 101, false); - for count in 0..MAX_RENEWALS { - assert_eq!(classify(own, count, [head(7, 100, false)].into_iter()), Decision::Defer); - } - assert_eq!(classify(own, MAX_RENEWALS, [head(7, 100, false)].into_iter()), Decision::Spare); - assert_eq!(classify(own, 2, [].into_iter()), Decision::Own); - } - #[test] - fn oldest_unfinished_queue_is_never_excused() { - assert_eq!(classify(head(7, 100, false), 0, [head(5, 101, false)].into_iter()), Decision::Own); - } - #[test] - fn same_vm_finished_and_later_work_do_not_block() { - let own = head(5, 101, false); - assert_eq!(classify(own, 0, [head(5, 100, false), head(7, 100, true), head(7, 101, false), head(7, 102, false)].into_iter()), Decision::Own); - assert_eq!(classify(head(5, 101, true), 0, [head(7, 100, false)].into_iter()), Decision::Own); - } -} diff --git a/drivers/gpu/drm/asahi-neo/g17/validation.rs b/drivers/gpu/drm/asahi-neo/g17/validation.rs deleted file mode 100644 index da43c10cd3bef5..00000000000000 --- a/drivers/gpu/drm/asahi-neo/g17/validation.rs +++ /dev/null @@ -1,145 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Reuse address-coverage proofs only while the VM's page tables are unchanged. - -use super::command::{Access, AddressSpace, Payload, UscWindow, Validated}; -use crate::mmu; -use core::{cell::RefCell, ops::Range}; -use kernel::prelude::*; - -const MAX_RANGES: usize = 48; -type Request = (u64, u64, u8); - -#[derive(Clone, Copy)] -struct Requests { - ranges: [Request; MAX_RANGES], - count: usize, - overflow: bool, -} - -impl Requests { - const fn new() -> Self { - Self { - ranges: [(0, 0, 0); MAX_RANGES], - count: 0, - overflow: false, - } - } - - fn as_slice(&self) -> &[Request] { - &self.ranges[..self.count] - } - - fn insert(&mut self, request: Request) { - if self.as_slice().contains(&request) { - return; - } - if self.count == MAX_RANGES { - self.overflow = true; - return; - } - self.ranges[self.count] = request; - self.count += 1; - } -} - -/// One logical queue's bounded cache of exact address, size and permission tuples. -/// Mapping mutations make the epoch unavailable until their final page-table write. -pub(crate) struct Proof { - epoch: Option, - requests: Requests, -} - -impl Proof { - pub(crate) const fn new() -> Self { - Self { - epoch: None, - requests: Requests::new(), - } - } - - fn covers(&self, epoch: Option, requests: &Requests) -> bool { - self.epoch.is_some() - && self.epoch == epoch - && requests - .as_slice() - .iter() - .all(|request| self.requests.as_slice().contains(request)) - } - - fn record(&mut self, epoch: Option, requests: Requests) { - // Alternating in-flight buffers must not evict each other's proofs. - if epoch.is_some() && self.epoch == epoch && !requests.overflow { - for &request in requests.as_slice() { - self.requests.insert(request); - } - if !self.requests.overflow { - return; - } - } - // An incomplete union cannot be reused; retain the latest complete set. - self.epoch = if requests.overflow { None } else { epoch }; - self.requests = requests; - } - - /// The VM-job guard must already pin the mappings for this command. Keep the - /// range arrays off the caller's frame before descending into publication. - #[inline(never)] - pub(crate) fn validate( - &mut self, - vm: &mmu::Vm, - range: Range, - window: &UscWindow, - payload: &Payload, - ) -> Result { - let requests = RefCell::new(Requests::new()); - let optimistic = payload.validate(window, &Space::Record(&range, &requests)); - let requests = requests.into_inner(); - if optimistic.is_ok() && !requests.overflow { - if self.covers(vm.mapping_validation_epoch(), &requests) { - return optimistic; - } - let (covered, epoch) = vm.covers_ranges_batch(requests.as_slice()); - if covered { - self.record(epoch, requests); - return optimistic; - } - self.epoch = None; - } - // Preserve per-range validation's failure and concurrent-mutation behavior. - payload.validate(window, &Space::Checked(&range, vm)) - } -} - -enum Space<'a> { - Record(&'a Range, &'a RefCell), - Checked(&'a Range, &'a mmu::Vm), -} - -impl AddressSpace for Space<'_> { - fn covers(&self, address: u64, size: u64, access: Access) -> bool { - let range = match self { - Self::Record(range, _) | Self::Checked(range, _) => range, - }; - let Some(end) = address.checked_add(size) else { - return false; - }; - if size == 0 || address < range.start || end > range.end { - return false; - } - let permissions = match access { - Access::Read => 1, - Access::Write => 2, - Access::ReadWrite => 3, - }; - match self { - Self::Record(_, requests) => { - requests.borrow_mut().insert((address, size, permissions)); - true - } - Self::Checked(_, vm) => { - vm.covers_range(address, size, permissions & 1 != 0, permissions & 2 != 0) - } - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/gem.rs b/drivers/gpu/drm/asahi-neo/gem.rs deleted file mode 100644 index 1edd3a16ebddaa..00000000000000 --- a/drivers/gpu/drm/asahi-neo/gem.rs +++ /dev/null @@ -1,315 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Asahi driver GEM object implementation -//! -//! Basic wrappers and adaptations between generic GEM shmem objects and this driver's -//! view of what a GPU buffer object is. It is in charge of keeping track of all mappings for -//! each GEM object so we can remove them when a client (File) or a Vm are destroyed, as well as -//! implementing RTKit buffers on top of GEM objects for firmware use. - -use kernel::{ - drm_neo, - drm_neo::gem::{ - shmem, - shmem::VMap, - BaseObject, - DriverObject, // - }, - error::Result, - prelude::*, - sync::aref::ARef, - uapi, // -}; - -use core::ops::Range; -use core::sync::atomic::{ - AtomicU64, - Ordering, // -}; - -use crate::{ - debug::*, - driver::{ - AsahiDevice, - AsahiDriver, // - }, - file, - mmu, - util::*, // -}; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Gem; - -/// Represents the inner data of a GEM object for this driver. -#[pin_data] -pub(crate) struct AsahiObject { - /// ID for debug - id: u64, - /// Object size retained for live-object accounting. - size: usize, - /// Object creation flags. - flags: u32, - /// Whether this object can be exported. - exportable: bool, - /// Whether this is a kernel-created object. - kernel: bool, - /// Driver aliases outside the user GPUVA tree; they pin the backing. - kernel_mappings: AtomicU64, -} - -/// Type alias for the shmem GEM object type for this driver. -pub(crate) type Object = shmem::Object; - -/// VM-private BOs share their assigned VM's reservation object. Validate this -/// before acquiring mapping state; generic GPUVM accepts external shareable BOs. -pub(crate) fn validate_vm_binding(gem: &Object, vm: &mmu::Vm) -> Result { - if gem.flags & uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_VM_PRIVATE != 0 - && vm.is_extobj(gem) - { - return Err(EINVAL); - } - Ok(()) -} - -unsafe impl Send for AsahiObject {} -unsafe impl Sync for AsahiObject {} - -// /// Type alias for the SGTable type for this driver. -// pub(crate) type SGTable = shmem::SGTable; - -/// A shared reference to a GEM object for this driver. -pub(crate) struct ObjectRef { - /// The underlying GEM object reference - pub(crate) gem: ARef, - /// The kernel-side VMap of this object, if needed - vmap: Option>, -} - -crate::no_debug!(ObjectRef); - -static GEM_ID: AtomicU64 = AtomicU64::new(0); - -impl ObjectRef { - /// Create a new wrapper for a raw GEM object reference. - pub(crate) fn new(gem: ARef) -> ObjectRef { - ObjectRef { gem, vmap: None } - } - - /// Return the `VMap` for this object, creating it if necessary. - pub(crate) fn vmap(&mut self) -> Result> { - if self.vmap.is_none() { - self.vmap = Some(self.gem.owned_vmap()?); - } - self.gem.vmap() - } - - /// Returns the size of an object in bytes - pub(crate) fn size(&self) -> usize { - self.gem.size() - } - - /// Maps an object into a given `Vm` at any free address within a given range. - pub(crate) fn map_into_range( - &mut self, - vm: &crate::mmu::Vm, - range: Range, - alignment: u64, - prot: mmu::Prot, - guard: bool, - ) -> Result { - // Only used for kernel objects now - if !self.gem.kernel { - return Err(EINVAL); - } - vm.map_in_range(&self.gem, 0..self.gem.size(), alignment, range, prot, guard) - } - - /// Maps a range within an object into a given `Vm` at any free address within a given range. - pub(crate) fn map_range_into_range( - &mut self, - vm: &crate::mmu::Vm, - obj_range: Range, - range: Range, - alignment: u64, - prot: mmu::Prot, - guard: bool, - ) -> Result { - if obj_range.end > self.gem.size() { - return Err(EINVAL); - } - validate_vm_binding(&self.gem, vm)?; - vm.map_in_range(&self.gem, obj_range, alignment, range, prot, guard) - } - - /// Maps an object into a given `Vm` at a specific address. - /// - /// Returns Err(ENOSPC) if the requested address is already busy. - pub(crate) fn map_at( - &mut self, - vm: &crate::mmu::Vm, - addr: u64, - prot: mmu::Prot, - guard: bool, - ) -> Result { - validate_vm_binding(&self.gem, vm)?; - - vm.map_at(addr, self.gem.size(), self.gem.clone(), prot, guard) - } -} - -pub(crate) struct AsahiObjConfig { - flags: u32, - exportable: bool, - kernel: bool, -} - -/// Create a new kernel-owned GEM object. -pub(crate) fn new_kernel_object(dev: &AsahiDevice, size: usize) -> Result { - new_kernel_object_mapped(dev, size, false) -} - -/// Create a new kernel-owned GEM object whose kernel mapping is write-combined. -/// -/// Used for memory that a non-coherent agent reads as soon as the host has stored to it and -/// issued a barrier, without cache maintenance. -pub(crate) fn new_kernel_object_wc(dev: &AsahiDevice, size: usize) -> Result { - new_kernel_object_mapped(dev, size, true) -} - -fn new_kernel_object_mapped(dev: &AsahiDevice, size: usize, map_wc: bool) -> Result { - let gem = shmem::Object::::new( - dev, - align(size, mmu::UAT_PGSZ), - shmem::ObjectConfig:: { - reclaimable_cpu_mappings: false, - map_wc, - parent_resv_obj: None, - }, - AsahiObjConfig { - flags: 0, - exportable: false, - kernel: true, - }, - )?; - - mod_pr_debug!("AsahiObject new kernel object id={}\n", gem.id); - Ok(ObjectRef::new(gem)) -} - -/// Create a new user-owned GEM object with the given flags. -pub(crate) fn new_object( - dev: &AsahiDevice, - size: usize, - flags: u32, - parent_object: Option<&shmem::Object>, - reclaim_cpu: bool, -) -> Result> { - if (flags & uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_VM_PRIVATE != 0) != parent_object.is_some() - { - return Err(EINVAL); - } - - let gem = shmem::Object::::new( - dev, - align(size, mmu::UAT_PGSZ), - shmem::ObjectConfig:: { - reclaimable_cpu_mappings: parent_object.is_some() && reclaim_cpu, - map_wc: flags & uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_WRITEBACK == 0, - parent_resv_obj: parent_object, - }, - AsahiObjConfig { - flags, - exportable: parent_object.is_none(), - kernel: false, - }, - )?; - - mod_pr_debug!("AsahiObject new user object: id={}\n", gem.id); - Ok(gem) -} - -#[vtable] -impl DriverObject for AsahiObject { - type Driver = AsahiDriver; - type Args = AsahiObjConfig; - - const HAS_EXPORT: bool = true; - - /// Callback to create the inner data of a GEM object - fn new(_dev: &AsahiDevice, size: usize, args: Self::Args) -> impl PinInit { - let id = GEM_ID.fetch_add(1, Ordering::Relaxed); - mod_pr_debug!("AsahiObject::new id={}\n", id); - try_pin_init!(AsahiObject { - id, - size, - flags: args.flags, - exportable: args.exportable, - kernel: args.kernel, - kernel_mappings: AtomicU64::new(0), - }) - } - - /// Callback to drop all mappings for a GEM object owned by a given `File` - fn close(obj: &::Object, file: &drm_neo::gem::DriverFile) { - // fn close(obj: &Object, file: &DrmFile) { - mod_pr_debug!("AsahiObject::close id={}\n", obj.id); - if file::File::unbind_gem_object(file, obj).is_err() { - pr_err!("AsahiObject::close: Failed to unbind GEM object\n"); - } - } - - /// Optional handle for exporting a gem object. - fn export( - obj: &::Object, - flags: u32, - ) -> Result> { - if !obj.exportable { - return Err(EINVAL); - } - - obj.prime_export(flags) - } -} - -/// A driver-created alias of a GEM object, possibly in a root the user GPUVA -/// tree does not describe. It keeps the object out of idle reclaim. -pub(crate) struct KernelMappingPin { - object: ARef, -} - -impl KernelMappingPin { - pub(crate) fn new(object: &Object) -> Self { - object.kernel_mappings.fetch_add(1, Ordering::AcqRel); - Self { - object: object.into(), - } - } -} - -impl core::ops::Deref for KernelMappingPin { - type Target = Object; - fn deref(&self) -> &Object { - &self.object - } -} - -impl Drop for KernelMappingPin { - fn drop(&mut self) { - self.object.kernel_mappings.fetch_sub(1, Ordering::Release); - } -} - -impl AsahiObject { - pub(crate) fn idle_reclaim_candidate(&self) -> bool { - !self.kernel && !self.exportable && self.kernel_mappings.load(Ordering::Acquire) == 0 - } -} - -/// Purge advice for a VM-private, non-exportable object. Returns whether the -/// contents are retained (0 once purged). -pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { - if gem.flags & uapi::drm_asahi_neo_gem_flags_DRM_ASAHI_NEO_GEM_VM_PRIVATE == 0 || gem.exportable { - return Err(EINVAL); - } - Ok(u32::from(gem.madvise(i32::from(dontneed)))) -} diff --git a/drivers/gpu/drm/asahi-neo/gpu.rs b/drivers/gpu/drm/asahi-neo/gpu.rs deleted file mode 100644 index 7ca3c505e30d37..00000000000000 --- a/drivers/gpu/drm/asahi-neo/gpu.rs +++ /dev/null @@ -1,1733 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Top-level GPU manager -//! -//! This module is the root of all GPU firmware management for a given driver instance. It is -//! responsible for initialization, owning the top-level managers (events, UAT, etc.), and -//! communicating with the raw RtKit endpoints to send and receive messages to/from the GPU -//! firmware. -//! -//! It is also the point where diverging driver firmware/GPU variants (using the versions macro) -//! are unified, so that the top level of the driver itself (in `driver`) does not have to concern -//! itself with version dependence. - -use core::any::Any; -use core::ops::Range; -use core::slice; -use core::sync::atomic::{ - AtomicBool, - AtomicU64, - Ordering, // -}; - -use kernel::{ - c_str, - drm_neo::gem::shmem, - error::code::*, - io::mem::{ - Mem, - MemFlag, // - }, - iosys_map::IoSysMapRef, - macros::versions, - new_mutex, - prelude::*, - soc::apple::rtkit, - sync::{ - lock::{ - mutex::MutexBackend, - Guard, // - }, - Arc, - Mutex, - UniqueArc, // - }, - time::{ - Delta, - Instant, - Monotonic, // - }, - types::ForeignOwnable, - uapi, // -}; -#[cfg(CONFIG_DEV_COREDUMP)] -use kernel::{ - devcoredump, - time::msecs_to_jiffies, // -}; - -use crate::alloc::Allocator; -use crate::debug::*; -use crate::driver::{ - AsahiDevRef, - AsahiDevice, // -}; -use crate::fw::channels::{ - ChannelErrorType, - PipeType, // -}; -use crate::fw::types::{ - U32, - U64, // -}; -use crate::{ - alloc, - buffer, - channel, - event, - fw, - gem, - hw, - initdata, - mem, - mmu, - queue, - regs, - stats, - workqueue, // -}; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Gpu; - -/// Firmware endpoint for init & incoming notifications. -const EP_FIRMWARE: u8 = 0x20; - -/// Doorbell endpoint for work/message submissions. -const EP_DOORBELL: u8 = 0x21; - -/// Initialize the GPU firmware. -const MSG_INIT: u64 = 0x81 << 48; -const INIT_DATA_MASK: u64 = (1 << 44) - 1; - -/// TX channel doorbell. -const MSG_TX_DOORBELL: u64 = 0x83 << 48; -/// Firmware control channel doorbell. -const MSG_FWCTL: u64 = 0x84 << 48; -// /// Halt the firmware (?). -// const MSG_HALT: u64 = 0x85 << 48; - -/// Receive channel doorbell notification. -const MSG_RX_DOORBELL: u64 = 0x42 << 48; - -/// Doorbell number for firmware kicks/wakeups. -const DOORBELL_KICKFW: u64 = 0x10; -/// Doorbell number for device control channel kicks. -const DOORBELL_DEVCTRL: u64 = 0x11; - -// Upper kernel half VA address ranges. -/// Private (cached) firmware structure VA range base. -const IOVA_KERN_PRIV_RANGE: Range = 0xffffffa000000000..0xffffffa600000000; -/// Private (cached) GPU-RO firmware structure VA range base. -const IOVA_KERN_GPU_RO_RANGE: Range = 0xffffffa600000000..0xffffffa800000000; -/// Shared (uncached) firmware structure VA range base. -const IOVA_KERN_SHARED_RANGE: Range = 0xffffffa800000000..0xffffffaa00000000; -/// Shared (uncached) read-only firmware structure VA range base. -const IOVA_KERN_SHARED_RO_RANGE: Range = 0xffffffaa00000000..0xffffffac00000000; -/// GPU/FW shared structure VA range base. -const IOVA_KERN_GPU_RANGE: Range = 0xffffffac00000000..0xffffffae00000000; -/// GPU/FW shared structure VA range base. -const IOVA_KERN_RTKIT_RANGE: Range = 0xffffffae00000000..0xffffffae10000000; -/// Shared (uncached) timestamp region. -pub(crate) const IOVA_KERN_TIMESTAMP_RANGE: Range = 0xffffffae10000000..0xffffffae14000000; -/// FW MMIO VA range base. -const IOVA_KERN_MMIO_RANGE: Range = 0xffffffaf00000000..0xffffffb000000000; - -/// GPU/FW buffer manager control address (context 0 low) -pub(crate) const IOVA_KERN_GPU_BUFMGR_LOW: u64 = 0x20_0000_0000; -/// GPU/FW buffer manager control address (context 0 high) -pub(crate) const IOVA_KERN_GPU_BUFMGR_HIGH: u64 = 0xffffffaeffff0000; - -/// Timeout for entering the halt state after a fault or request. -const HALT_ENTER_TIMEOUT: Delta = Delta::from_millis(100); - -/// Maximum amount of firmware-private memory garbage allowed before collection. -/// Collection flushes the FW cache and is expensive, so this needs to be -/// reasonably high. -const MAX_FW_ALLOC_GARBAGE_BYTES: usize = 16 * 1024 * 1024; -/// Maximum count of firmware-private memory garbage objects allowed before collection. -/// This works out to 16K of memory in the garbage list (8 bytes each), which keeps us -/// within the safe range for kmalloc (on 16K page systems). -const MAX_FW_ALLOC_GARBAGE_OBJECTS: usize = 2048; - -/// Global allocators used for kernel-half structures. -pub(crate) struct KernelAllocators { - pub(crate) private: alloc::DefaultAllocator, - pub(crate) shared: alloc::DefaultAllocator, - pub(crate) shared_ro: alloc::DefaultAllocator, - #[allow(dead_code)] - pub(crate) gpu: alloc::DefaultAllocator, - pub(crate) gpu_ro: alloc::DefaultAllocator, -} - -/// Receive (GPU->driver) ring buffer channels. -#[versions(AGX_NEO)] -#[pin_data] -struct RxChannels { - event: channel::EventChannel::ver, - fw_log: channel::FwLogChannel, - ktrace: channel::KTraceChannel, - stats: channel::StatsChannel::ver, -} - -/// GPU work submission pipe channels (driver->GPU). -#[versions(AGX_NEO)] -struct PipeChannels { - pub(crate) vtx: KVec>>>, - pub(crate) frag: KVec>>>, - pub(crate) comp: KVec>>>, -} - -/// Misc command transmit (driver->GPU) channels. -#[versions(AGX_NEO)] -#[pin_data] -struct TxChannels { - pub(crate) device_control: channel::DeviceControlChannel::ver, -} - -/// Number of work submission pipes per type, one for each priority level. -const NUM_PIPES: usize = 4; - -/// A generic monotonically incrementing ID used to uniquely identify object instances within the -/// driver. -pub(crate) struct ID(AtomicU64); - -impl ID { - /// Create a new ID counter with a given value. - fn new(val: u64) -> ID { - ID(AtomicU64::new(val)) - } - - /// Fetch the next unique ID. - pub(crate) fn next(&self) -> u64 { - self.0.fetch_add(1, Ordering::Relaxed) - } -} - -impl Default for ID { - /// IDs default to starting at 2, as 0/1 are considered reserved for the system. - fn default() -> Self { - Self::new(2) - } -} - -/// A guard representing one active submission on the GPU. When dropped, decrements the active -/// submission count. -pub(crate) struct OpGuard(Arc); - -impl Drop for OpGuard { - fn drop(&mut self) { - self.0.end_op(); - } -} - -/// Set of global sequence IDs used in the driver. -#[derive(Default)] -pub(crate) struct SequenceIDs { - /// `File` instance ID. - pub(crate) file: ID, - /// `Vm` instance ID. - pub(crate) vm: ID, - /// Submission instance ID. - pub(crate) submission: ID, - /// `Queue` instance ID. - pub(crate) queue: ID, -} - -/// Top-level GPU manager that owns all the global state relevant to the driver instance. -#[versions(AGX_NEO)] -#[pin_data] -pub(crate) struct GpuManager { - dev: AsahiDevRef, - cfg: &'static hw::HwConfig, - dyncfg: hw::DynConfig, - pub(crate) initdata: fw::types::GpuObject, - uat: mmu::Uat, - crashed: AtomicBool, - #[pin] - alloc: Mutex, - io_mappings: KVec, - next_mmio_iova: u64, - #[pin] - rtkit: Mutex>>, - #[pin] - rx_channels: Mutex, - #[pin] - tx_channels: Mutex, - #[pin] - fwctl_channel: Mutex, - pipes: PipeChannels::ver, - event_manager: Arc, - stats: Arc, - buffer_mgr: buffer::BufferManager::ver, - ids: SequenceIDs, - #[allow(clippy::vec_box)] - #[pin] - garbage_contexts: Mutex>>>, -} - -/// Operations the DRM driver performs on a GPU, independent of its firmware interface. -pub(crate) trait Gpu: Send + Sync { - /// Return a snapshot only for firmware families with a supported decoder. - fn stats_snapshot(&self) -> Option> { - None - } - - /// Cast as an Any type. - fn as_any(&self) -> &dyn Any; - /// Cast Arc as an Any type. - fn arc_as_any(self: Arc) -> Arc; - /// Return the G13/G14 firmware manager, if this GPU uses that firmware interface. - fn manager(&self) -> Option<&dyn GpuManager>; - /// Initialize the GPU. - fn init(&self) -> Result; - /// Return a reference to the global `SequenceIDs` instance. - fn ids(&self) -> &SequenceIDs; - /// Check whether the GPU is crashed - fn is_crashed(&self) -> bool; - /// Fill in the hardware description reported to userspace. - fn get_params(&self, params: &mut uapi::drm_asahi_neo_params_global) -> Result; - /// Whether user VMs retain a sticky error and support a userspace mirror. - fn supports_vm_status(&self) -> bool { - false - } - /// Limits for independently executing queues, if supported by this GPU. - fn queue_limits(&self) -> Option { - None - } - /// Temporary capacity hint and validity in milliseconds for subsequent syncobj waits. - fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { - None - } - /// Frequency of the CPU timer that GPU timestamps are derived from. - fn base_clock_hz(&self) -> u32; - /// Return the GPU address space geometry. - fn uat_geometry(&self) -> mmu::UatGeometry; - /// Create a new `Vm` with the given range reserved for kernel-managed objects. - fn new_vm(&self, kernel_range: Range) -> Result; - /// Create a new user command queue. - fn new_queue( - &self, - vm: mmu::Vm, - ualloc: Arc>, - ualloc_priv: Arc>, - priority: u32, - usc_exec_base: u64, - ) -> Result>; - /// Map a BO as a timestamp buffer - fn map_timestamp_buffer( - &self, - bo: gem::ObjectRef, - range: Range, - ) -> Result; -} - -/// Trait used to abstract the firmware/GPU-dependent variants of the GpuManager. -pub(crate) trait GpuManager: Send + Sync { - /// Update the GPU globals from global info - /// - /// TODO: Unclear what can and cannot be updated like this. - fn update_globals(&self); - /// Get a reference to the KernelAllocators. - fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend>; - /// Bind a `Vm` to an available slot and return the `VmBind`. - fn bind_vm(&self, vm: &mmu::Vm) -> Result; - /// Kick the firmware (wake it up if asleep). - /// - /// This should be useful to reduce latency on work submission, so we can ask the firmware to - /// wake up while we do some preparatory work for the work submission. - fn kick_firmware(&self) -> Result; - /// Flush the entire firmware cache. - /// - /// TODO: Does this actually work? - fn flush_fw_cache(&self) -> Result; - /// Handle a GPU work timeout event. - fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32); - /// Handle a GPU fault event. - fn handle_fault(&self); - /// Handle a channel error event. - fn handle_channel_error( - &self, - error_type: ChannelErrorType, - pipe_type: u32, - event_slot: u32, - event_value: u32, - ); - /// Acknowledge a Buffer grow op. - fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32); - /// Send a firmware control command (secure cache flush). - fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result; - /// Get the static GPU configuration for this SoC. - fn get_cfg(&self) -> &'static hw::HwConfig; - /// Get the dynamic GPU configuration for this SoC. - fn get_dyncfg(&self) -> &hw::DynConfig; - /// Register an unused context as garbage - fn free_context(&self, data: KBox>); -} - -/// Private generic trait for functions that don't need to escape this module. -trait GpuManagerPriv { - /// Decrement the pending submission counter. - fn end_op(&self); -} - -pub(crate) struct RtkitObject { - vmap: shmem::VMap, - mapping: mmu::KernelMapping, -} - -impl rtkit::Buffer for RtkitObject { - fn iova(&self) -> Result { - Ok(self.mapping.iova() as usize) - } - fn buf(&mut self) -> Result> { - Ok(self.vmap.get()) - } -} - -#[versions(AGX_NEO)] -#[vtable] -impl rtkit::Operations for GpuManager::ver { - type Data = Arc; - type Buffer = RtkitObject; - - fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { - let dev = &data.dev; - //dev_info!(dev.as_ref(), "RtKit message: {:#x}:{:#x}\n", ep, msg); - - if ep != EP_FIRMWARE || msg != MSG_RX_DOORBELL { - dev_err!(dev.as_ref(), "Unknown message: {:#x}:{:#x}\n", ep, msg); - return; - } - - let mut ch = data.rx_channels.lock(); - - ch.fw_log.poll(); - ch.ktrace.poll(); - ch.stats.poll(); - ch.event.poll(); - } - - fn crashed(data: ::Borrowed<'_>, crashlog: Option<&[u8]>) { - let dev = &data.dev; - - data.crashed.store(true, Ordering::Relaxed); - - #[cfg(CONFIG_DEV_COREDUMP)] - if let Err(e) = data.generate_crashdump(crashlog) { - dev_err!(dev.as_ref(), "Could not generate crashdump: {:?}\n", e); - } - #[cfg(not(CONFIG_DEV_COREDUMP))] - let _ = crashlog; - - if debug_enabled(DebugFlags::OopsOnGpuCrash) { - panic!("GPU firmware crashed"); - } else { - dev_err!(dev.as_ref(), "GPU firmware crashed, failing all jobs\n"); - data.event_manager.fail_all(workqueue::WorkError::NoDevice); - } - } - - fn shmem_alloc( - data: ::Borrowed<'_>, - size: usize, - ) -> Result { - let dev = &data.dev; - mod_dev_dbg!(dev, "shmem_alloc() {:#x} bytes\n", size); - - let mut obj = gem::new_kernel_object(dev, size)?; - let vmap = obj.gem.owned_vmap()?; - let mapping = obj.map_into_range( - data.uat.kernel_vm(), - IOVA_KERN_RTKIT_RANGE, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - true, - )?; - mod_dev_dbg!(dev, "shmem_alloc() -> VA {:#x}\n", mapping.iova()); - Ok(RtkitObject { vmap, mapping }) - } -} - -#[versions(AGX_NEO)] -impl GpuManager::ver { - /// Create a new GpuManager of this version/GPU combination. - #[inline(never)] - pub(crate) fn new( - dev: &AsahiDevice, - res: ®s::Resources, - cfg: &'static hw::HwConfig, - ) -> Result> { - let uat = Self::make_uat(dev, cfg)?; - let dyncfg = Self::make_dyncfg(dev, res, cfg, &uat)?; - - let mut alloc = KernelAllocators { - private: alloc::DefaultAllocator::new( - dev, - uat.kernel_vm(), - IOVA_KERN_PRIV_RANGE, - 0x80, - mmu::PROT_FW_PRIV_RW, - 1024 * 1024, - true, - fmt!("Kernel Private"), - true, - )?, - shared: alloc::DefaultAllocator::new( - dev, - uat.kernel_vm(), - IOVA_KERN_SHARED_RANGE, - 0x80, - mmu::PROT_FW_SHARED_RW, - 1024 * 1024, - true, - fmt!("Kernel Shared"), - false, - )?, - shared_ro: alloc::DefaultAllocator::new( - dev, - uat.kernel_vm(), - IOVA_KERN_SHARED_RO_RANGE, - 0x80, - mmu::PROT_FW_SHARED_RO, - 64 * 1024, - true, - fmt!("Kernel RO Shared"), - false, - )?, - gpu: alloc::DefaultAllocator::new( - dev, - uat.kernel_vm(), - IOVA_KERN_GPU_RANGE, - 0x80, - mmu::PROT_GPU_FW_SHARED_RW, - 64 * 1024, - true, - fmt!("Kernel GPU Shared"), - false, - )?, - gpu_ro: alloc::DefaultAllocator::new( - dev, - uat.kernel_vm(), - IOVA_KERN_GPU_RO_RANGE, - 0x80, - mmu::PROT_GPU_RO_FW_PRIV_RW, - 1024 * 1024, - true, - fmt!("Kernel GPU RO Shared"), - true, - )?, - }; - - let event_manager = Self::make_event_manager(&mut alloc)?; - let mut initdata = Self::make_initdata(dev, cfg, &dyncfg, &mut alloc)?; - - initdata.runtime_pointers.buffer_mgr_ctl_low_mapping = - Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( - uat.kernel_lower_vm(), - IOVA_KERN_GPU_BUFMGR_LOW, - mmu::PROT_GPU_SHARED_RW, - false, - )?); - initdata.runtime_pointers.buffer_mgr_ctl_high_mapping = - Some(initdata.runtime_pointers.buffer_mgr_ctl.map_at( - uat.kernel_vm(), - IOVA_KERN_GPU_BUFMGR_HIGH, - mmu::PROT_FW_SHARED_RW, - false, - )?); - - let mut mgr = Self::make_mgr(dev, cfg, dyncfg, uat, alloc, event_manager, initdata)?; - - { - let fwctl = mgr.fwctl_channel.lock(); - let p_fwctl = fwctl.to_raw(); - core::mem::drop(fwctl); - - mgr.as_mut() - .initdata_mut() - .fw_status - .with_mut(|raw, _inner| { - raw.fwctl_channel = p_fwctl; - }); - } - - { - let txc = mgr.tx_channels.lock(); - let p_device_control = txc.device_control.to_raw(); - core::mem::drop(txc); - - let rxc = mgr.rx_channels.lock(); - let p_event = rxc.event.to_raw(); - let p_fw_log = rxc.fw_log.to_raw(); - let p_ktrace = rxc.ktrace.to_raw(); - let p_stats = rxc.stats.to_raw(); - let p_fwlog_buf = rxc.fw_log.get_buf(); - core::mem::drop(rxc); - - mgr.as_mut() - .initdata_mut() - .runtime_pointers - .with_mut(|raw, _inner| { - raw.device_control = p_device_control; - raw.event = p_event; - raw.fw_log = p_fw_log; - raw.ktrace = p_ktrace; - raw.stats = p_stats; - raw.fwlog_buf = Some(p_fwlog_buf); - }); - } - - let mut p_pipes: KVec = KVec::new(); - - for ((v, f), c) in mgr - .pipes - .vtx - .iter() - .zip(&mgr.pipes.frag) - .zip(&mgr.pipes.comp) - { - p_pipes.push( - fw::initdata::raw::PipeChannels::ver { - vtx: v.lock().to_raw(), - frag: f.lock().to_raw(), - comp: c.lock().to_raw(), - }, - GFP_KERNEL, - )?; - } - - mgr.as_mut() - .initdata_mut() - .runtime_pointers - .with_mut(|raw, _inner| { - for (i, p) in p_pipes.into_iter().enumerate() { - raw.pipes[i].vtx = p.vtx; - raw.pipes[i].frag = p.frag; - raw.pipes[i].comp = p.comp; - } - }); - - for (i, map) in cfg.io_mappings.iter().enumerate() { - if let Some(map) = map.as_ref() { - Self::iomap(&mut mgr, cfg, i, map)?; - } - } - - #[ver(V >= V13_0B4)] - if let Some(base) = cfg.sram_base { - let size = cfg.sram_size.unwrap(); - let iova = mgr.as_mut().alloc_mmio_iova(size); - - let mapping = mgr - .uat - .kernel_vm() - .map_io(iova, base, size, mmu::PROT_FW_SHARED_RW)?; - - mgr.as_mut() - .initdata_mut() - .runtime_pointers - .hwdata_b - .with_mut(|raw, _| { - raw.sgx_sram_ptr = U64(mapping.iova()); - }); - - mgr.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; - } - - let mgr = Arc::from(mgr); - - let rtkit = rtkit::RtKit::::new(dev.as_ref(), None, 0, mgr.clone())?; - - *mgr.rtkit.lock() = Some(rtkit); - - { - let mut rxc = mgr.rx_channels.lock(); - rxc.event.set_manager(mgr.clone()); - } - - Ok(mgr) - } - - /// Return a mutable reference to the initdata member - fn initdata_mut( - self: Pin<&mut Self>, - ) -> &mut fw::types::GpuObject { - // SAFETY: initdata does not require structural pinning. - unsafe { &mut self.get_unchecked_mut().initdata } - } - - /// Return a mutable reference to the io_mappings member - fn io_mappings_mut(self: Pin<&mut Self>) -> &mut KVec { - // SAFETY: io_mappings does not require structural pinning. - unsafe { &mut self.get_unchecked_mut().io_mappings } - } - - /// Allocate an MMIO iova range - fn alloc_mmio_iova(self: Pin<&mut Self>, size: usize) -> u64 { - // SAFETY: next_mmio_iova does not require structural pinning. - let next_ref = unsafe { &mut self.get_unchecked_mut().next_mmio_iova }; - - let addr = *next_ref; - let next = addr + (size + mmu::UAT_PGSZ) as u64; - - assert!(next <= IOVA_KERN_MMIO_RANGE.end); - - *next_ref = next; - - addr - } - - /// Build the entire GPU InitData structure tree and return it as a boxed GpuObject. - fn make_initdata( - dev: &AsahiDevice, - cfg: &'static hw::HwConfig, - dyncfg: &hw::DynConfig, - alloc: &mut KernelAllocators, - ) -> Result>> { - let mut builder = initdata::InitDataBuilder::ver::new(dev, alloc, cfg, dyncfg); - builder.build() - } - - /// Create a fresh boxed Uat instance. - /// - /// Force disable inlining to avoid blowing up the stack. - #[inline(never)] - fn make_uat(dev: &AsahiDevice, cfg: &'static hw::HwConfig) -> Result> { - // G14X has a new thing in the Scene structure that unfortunately requires - // write access from user contexts. Hopefully it's not security-sensitive. - #[ver(G >= G14X)] - let map_kernel_to_user = true; - #[ver(G < G14X)] - let map_kernel_to_user = false; - - Ok(KBox::new( - mmu::Uat::new( - dev, - mmu::UatGeometry { - ias: cfg.uat_ias, - oas: cfg.uat_oas, - }, - map_kernel_to_user, - mmu::UatFirmware::Handoff, - )?, - GFP_KERNEL, - )?) - } - - /// Actually create the final GpuManager instance, as a UniqueArc. - /// - /// Force disable inlining to avoid blowing up the stack. - #[inline(never)] - fn make_mgr( - dev: &AsahiDevice, - cfg: &'static hw::HwConfig, - dyncfg: KBox, - uat: KBox, - mut alloc: KernelAllocators, - event_manager: Arc, - initdata: KBox>, - ) -> Result>> { - let mut pipes = PipeChannels::ver { - vtx: KVec::new(), - frag: KVec::new(), - comp: KVec::new(), - }; - - for _i in 0..=NUM_PIPES - 1 { - pipes.vtx.push( - KBox::pin_init( - new_mutex!(channel::PipeChannel::ver::new(dev, &mut alloc)?, "pipe_vtx",), - GFP_KERNEL, - )?, - GFP_KERNEL, - )?; - pipes.frag.push( - KBox::pin_init( - new_mutex!( - channel::PipeChannel::ver::new(dev, &mut alloc)?, - "pipe_frag", - ), - GFP_KERNEL, - )?, - GFP_KERNEL, - )?; - pipes.comp.push( - KBox::pin_init( - new_mutex!( - channel::PipeChannel::ver::new(dev, &mut alloc)?, - "pipe_comp", - ), - GFP_KERNEL, - )?, - GFP_KERNEL, - )?; - } - - let fwctl_channel = channel::FwCtlChannel::new(dev, &mut alloc)?; - - let buffer_mgr = buffer::BufferManager::ver::new()?; - let event_manager_clone = event_manager.clone(); - let buffer_mgr_clone = buffer_mgr.clone(); - let stats_snap = Arc::new(stats::StatsSnapshot::default(), GFP_KERNEL)?; - let stats_snap_clone = stats_snap.clone(); - let alloc_ref = &mut alloc; - let rx_channels = KBox::init( - try_init!(RxChannels::ver { - event: channel::EventChannel::ver::new( - dev, - alloc_ref, - event_manager_clone, - buffer_mgr_clone, - )?, - fw_log: channel::FwLogChannel::new(dev, alloc_ref)?, - ktrace: channel::KTraceChannel::new(dev, alloc_ref)?, - stats: channel::StatsChannel::ver::new( - dev, - alloc_ref, - stats_snap_clone, - u64::from(cfg.base_clock_hz), - )?, - }), - GFP_KERNEL, - )?; - - let alloc_ref = &mut alloc; - let tx_channels = KBox::init( - try_init!(TxChannels::ver { - device_control: channel::DeviceControlChannel::ver::new(dev, alloc_ref)?, - }), - GFP_KERNEL, - )?; - - let x = UniqueArc::pin_init( - try_pin_init!(GpuManager::ver { - dev: dev.into(), - cfg, - dyncfg: KBox::::into_inner(dyncfg), - initdata: KBox::>::into_inner(initdata), - uat: KBox::::into_inner(uat), - io_mappings: KVec::new(), - next_mmio_iova: IOVA_KERN_MMIO_RANGE.start, - rtkit <- new_mutex!(None, "rtkit"), - crashed: AtomicBool::new(false), - event_manager, - stats: stats_snap.clone(), - alloc <- new_mutex!(alloc, "alloc"), - fwctl_channel <- new_mutex!(fwctl_channel, "fwctl_channel"), - rx_channels <- new_mutex!(KBox::::into_inner(rx_channels), "rx_channels"), - tx_channels <- new_mutex!(KBox::::into_inner(tx_channels), "tx_channels"), - pipes, - buffer_mgr, - ids: Default::default(), - garbage_contexts <- new_mutex!(KVec::new(), "garbage_contexts"), - }), - GFP_KERNEL, - )?; - - Ok(x) - } - - fn load_hwdata_blob(dev: &AsahiDevice, name: &CStr, size_name: &CStr) -> Result> { - let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; - let size: usize = dev - .as_ref() - .fwnode() - .ok_or(ENOENT)? - .property_read::(size_name) - .or(0) - .try_into()?; - let res = of_node.reserved_mem_region_to_resource_byname(name)?; - // SAFETY: No dma here, just loading init data. - let mem = unsafe { Mem::try_new(res, (MemFlag::WB).into())? }; - if size > mem.size() { - return Err(ENOENT); - } - // SAFETY: trusting the bootloader to fill it out correctly - let blob_sl = unsafe { slice::from_raw_parts(mem.ptr(), size) }; - let mut blob = KVVec::new(); - blob.extend_from_slice(blob_sl, GFP_KERNEL)?; - Ok(blob) - } - - /// Fetch and validate the GPU dynamic configuration from the device tree and hardware. - /// - /// Force disable inlining to avoid blowing up the stack. - #[inline(never)] - fn make_dyncfg( - dev: &AsahiDevice, - res: ®s::Resources, - cfg: &'static hw::HwConfig, - uat: &mmu::Uat, - ) -> Result> { - let gpu_id = res.get_gpu_id()?; - - dev_info!(dev.as_ref(), "GPU Information:\n"); - dev_info!( - dev.as_ref(), - " Type: {:?}{:?}\n", - gpu_id.gpu_gen, - gpu_id.gpu_variant - ); - dev_info!(dev.as_ref(), " Clusters: {}\n", gpu_id.num_clusters); - dev_info!( - dev.as_ref(), - " Cores: {} ({})\n", - gpu_id.num_cores, - gpu_id.num_cores * gpu_id.num_clusters - ); - dev_info!( - dev.as_ref(), - " Frags: {} ({})\n", - gpu_id.num_frags, - gpu_id.num_frags * gpu_id.num_clusters - ); - dev_info!( - dev.as_ref(), - " GPs: {} ({})\n", - gpu_id.num_gps, - gpu_id.num_gps * gpu_id.num_clusters - ); - dev_info!(dev.as_ref(), " Core masks: {:#x?}\n", gpu_id.core_masks); - dev_info!( - dev.as_ref(), - " Active cores: {}\n", - gpu_id.total_active_cores - ); - - dev_info!(dev.as_ref(), "Getting configuration from device tree...\n"); - let pwr_cfg = hw::PwrConfig::load(dev, cfg)?; - dev_info!(dev.as_ref(), "Dynamic configuration fetched\n"); - - if gpu_id.gpu_gen != cfg.gpu_gen || gpu_id.gpu_variant != cfg.gpu_variant { - dev_err!( - dev.as_ref(), - "GPU type mismatch (expected {:?}{:?}, found {:?}{:?})\n", - cfg.gpu_gen, - cfg.gpu_variant, - gpu_id.gpu_gen, - gpu_id.gpu_variant - ); - return Err(EIO); - } - if gpu_id.num_clusters > cfg.max_num_clusters { - dev_err!( - dev.as_ref(), - "Too many clusters ({} > {})\n", - gpu_id.num_clusters, - cfg.max_num_clusters - ); - return Err(EIO); - } - if gpu_id.num_cores > cfg.max_num_cores { - dev_err!( - dev.as_ref(), - "Too many cores ({} > {})\n", - gpu_id.num_cores, - cfg.max_num_cores - ); - return Err(EIO); - } - if gpu_id.num_frags > cfg.max_num_frags { - dev_err!( - dev.as_ref(), - "Too many frags ({} > {})\n", - gpu_id.num_frags, - cfg.max_num_frags - ); - return Err(EIO); - } - if gpu_id.num_gps > cfg.max_num_gps { - dev_err!( - dev.as_ref(), - "Too many GPs ({} > {})\n", - gpu_id.num_gps, - cfg.max_num_gps - ); - return Err(EIO); - } - - let fwnode = dev.as_ref().fwnode().ok_or(ENOENT)?; - - Ok(KBox::new( - hw::DynConfig { - pwr: pwr_cfg, - uat_ttb_base: uat.ttb_base(), - id: gpu_id, - firmware_version: fwnode - .property_read_array_vec(c_str!("apple,firmware-version"), 3)? - .or(kernel::kvec![0; 3]?), - - hw_data_a: Self::load_hwdata_blob( - dev, - c_str!("hw-cal-a"), - c_str!("debug,hw-cal-a-size"), - ) - .unwrap_or(KVVec::new()), - hw_data_b: Self::load_hwdata_blob( - dev, - c_str!("hw-cal-b"), - c_str!("debug,hw-cal-b-size"), - ) - .unwrap_or(KVVec::new()), - hw_globals: Self::load_hwdata_blob( - dev, - c_str!("globals"), - c_str!("debug,globals-size"), - ) - .unwrap_or(KVVec::new()), - }, - GFP_KERNEL, - )?) - } - - /// Create the global GPU event manager, and return an `Arc<>` to it. - fn make_event_manager(alloc: &mut KernelAllocators) -> Result> { - Ok(Arc::new(event::EventManager::new(alloc)?, GFP_KERNEL)?) - } - - /// Create a new MMIO mapping and add it to the mappings list in initdata at the specified - /// index. - fn iomap( - this: &mut Pin>, - cfg: &'static hw::HwConfig, - index: usize, - map: &hw::IOMapping, - ) -> Result { - let dies = if map.per_die { - cfg.num_dies as usize - } else { - 1 - }; - - let off = map.base & mmu::UAT_PGMSK; - let base = map.base - off; - let end = (map.base + map.size + mmu::UAT_PGMSK) & !mmu::UAT_PGMSK; - let map_size = end - base; - - // Array mappings must be aligned - assert!((off == 0 && map_size == map.size) || (map.count == 1 && !map.per_die)); - assert!(map.count > 0); - - let iova = this.as_mut().alloc_mmio_iova(map_size * map.count * dies); - let mut cur_iova = iova; - - for die in 0..dies { - for i in 0..map.count { - let phys_off = die * 0x20_0000_0000 + i * map.stride; - - let mapping = this.uat.kernel_vm().map_io( - cur_iova, - base + phys_off, - map_size, - if map.writable { - mmu::PROT_FW_MMIO_RW - } else { - mmu::PROT_FW_MMIO_RO - }, - )?; - - this.as_mut().io_mappings_mut().push(mapping, GFP_KERNEL)?; - cur_iova += map_size as u64; - } - } - - this.as_mut() - .initdata_mut() - .runtime_pointers - .hwdata_b - .with_mut(|raw, _| { - raw.io_mappings[index] = fw::initdata::raw::IOMapping { - phys_addr: U64(map.base as u64), - virt_addr: U64(iova + off as u64), - total_size: (map.size * map.count * dies) as u32, - element_size: map.size as u32, - readwrite: U64(map.writable as u64), - }; - }); - - Ok(()) - } - - /// Mark work associated with currently in-progress event slots as failed, after a fault or - /// timeout. - fn mark_pending_events(&self, culprit_slot: Option, error: workqueue::WorkError) { - dev_err!(self.dev.as_ref(), " Pending events:\n"); - - self.initdata.globals.with(|raw, _inner| { - for (index, i) in raw.pending_stamps.iter().enumerate() { - let info = i.info.load(Ordering::Relaxed); - let wait_value = i.wait_value.load(Ordering::Relaxed); - - if info & 1 != 0 { - #[ver(V >= V13_5)] - let slot = (info >> 4) & 0x7f; - #[ver(V < V13_5)] - let slot = (info >> 3) & 0x7f; - #[ver(V >= V13_5)] - let flags = info & 0xf; - #[ver(V < V13_5)] - let flags = info & 0x7; - dev_err!( - self.dev.as_ref(), - " [{}:{}] flags={} value={:#x}\n", - index, - slot, - flags, - wait_value - ); - let error = if culprit_slot.is_some() && culprit_slot != Some(slot) { - workqueue::WorkError::Killed - } else { - error - }; - self.event_manager.mark_error(slot, wait_value, error); - i.info.store(0, Ordering::Relaxed); - i.wait_value.store(0, Ordering::Relaxed); - } - } - }); - } - - /// Fetch the GPU MMU fault information from the hardware registers. - fn get_fault_info(&self) -> Option { - let res = &(*self.dev).resources; - - let info = res.get_fault_info(self.cfg); - if info.is_some() { - dev_err!( - self.dev.as_ref(), - " Fault info: {:#x?}\n", - info.as_ref().unwrap() - ); - } - info - } - - /// Resume the GPU firmware after it halts (due to a timeout, fault, or request). - fn recover(&self) { - self.initdata.fw_status.with(|raw, _inner| { - let halt_count = raw.flags.halt_count.load(Ordering::Relaxed); - let mut halted = raw.flags.halted.load(Ordering::Relaxed); - dev_err!(self.dev.as_ref(), " Halt count: {}\n", halt_count); - dev_err!(self.dev.as_ref(), " Halted: {}\n", halted); - - if halted == 0 { - let start = Instant::::now(); - while start.elapsed() < HALT_ENTER_TIMEOUT { - halted = raw.flags.halted.load(Ordering::Relaxed); - if halted != 0 { - break; - } - mem::sync(); - } - halted = raw.flags.halted.load(Ordering::Relaxed); - } - - if debug_enabled(DebugFlags::NoGpuRecovery) { - dev_crit!( - self.dev.as_ref(), - " GPU recovery is disabled, wedging forever!\n" - ); - } else if halted != 0 { - dev_err!(self.dev.as_ref(), " Attempting recovery...\n"); - raw.flags.halted.store(0, Ordering::SeqCst); - raw.flags.resume.store(1, Ordering::SeqCst); - } else { - dev_err!(self.dev.as_ref(), " Cannot recover.\n"); - } - }); - } - - /// Return the packed GPU enabled core masks. - // Only used for some versions - #[allow(dead_code)] - pub(crate) fn core_masks_packed(&self) -> &[u32] { - self.dyncfg.id.core_masks_packed.as_slice() - } - - /// Kick a submission pipe for a submitted job to tell the firmware to start processing it. - pub(crate) fn run_job(&self, job: workqueue::JobSubmission::ver<'_>) -> Result { - mod_dev_dbg!(self.dev, "GPU: run_job\n"); - - let pipe_type = job.pipe_type(); - mod_dev_dbg!(self.dev, "GPU: run_job: pipe_type={:?}\n", pipe_type); - - let pipes = match pipe_type { - PipeType::Vertex => &self.pipes.vtx, - PipeType::Fragment => &self.pipes.frag, - PipeType::Compute => &self.pipes.comp, - }; - - let index: usize = job.priority() as usize; - let mut pipe = pipes.get(index).ok_or(EIO)?.lock(); - - mod_dev_dbg!(self.dev, "GPU: run_job: run()\n"); - job.run(&mut pipe); - mod_dev_dbg!(self.dev, "GPU: run_job: ring doorbell\n"); - - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().as_pin_mut().unwrap(); - rtk.send_message( - EP_DOORBELL, - MSG_TX_DOORBELL | pipe_type as u64 | ((index as u64) << 2), - )?; - mod_dev_dbg!(self.dev, "GPU: run_job: done\n"); - - Ok(()) - } - - pub(crate) fn start_op(self: &Arc) -> Result { - if self.is_crashed() { - return Err(ENODEV); - } - - let val = self - .initdata - .globals - .with(|raw, _inner| raw.pending_submissions.fetch_add(1, Ordering::Acquire)); - - mod_dev_dbg!(self.dev, "OP start (pending: {})\n", val + 1); - let guard = OpGuard(self.clone()); - self.kick_firmware()?; - Ok(guard) - } - - fn invalidate_context( - &self, - context: &fw::types::GpuObject, - ) -> Result { - mod_dev_dbg!( - self.dev, - "Invalidating GPU context @ {:?}\n", - context.weak_pointer() - ); - - if self.is_crashed() { - return Err(ENODEV); - } - - let mut guard = self.alloc.lock(); - let (garbage_count, _) = guard.private.garbage(); - let (garbage_count_gpuro, _) = guard.gpu_ro.garbage(); - - let dc = context.with( - |raw, _inner| fw::channels::DeviceControlMsg::ver::DestroyContext { - unk_4: 0, - ctx_23: raw.unk_23, - #[ver(V < V13_3)] - __pad0: Default::default(), - unk_c: U32(0), - unk_10: U32(0), - ctx_0: raw.unk_0, - ctx_1: raw.unk_1, - ctx_4: raw.unk_4, - #[ver(V < V13_3)] - __pad1: Default::default(), - #[ver(V < V13_3)] - unk_18: 0, - gpu_context: Some(context.weak_pointer()), - __pad2: Default::default(), - }, - ); - - mod_dev_dbg!(self.dev, "Context invalidation command: {:?}\n", &dc); - - let mut txch = self.tx_channels.lock(); - - let token = txch.device_control.send(&dc); - - { - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().as_pin_mut().unwrap(); - rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; - } - - txch.device_control.wait_for(token)?; - - mod_dev_dbg!( - self.dev, - "GPU context invalidated: {:?}\n", - context.weak_pointer() - ); - - // The invalidation does a cache flush, so it is okay to collect garbage - guard.private.collect_garbage(garbage_count); - guard.gpu_ro.collect_garbage(garbage_count_gpuro); - - Ok(()) - } - - #[cfg(CONFIG_DEV_COREDUMP)] - fn generate_crashdump(&self, crashlog: Option<&[u8]>) -> Result { - // Lock the allocators, to block kernel/FW memory mutations (mostly) - let kalloc = self.alloc(); - let pages = self.uat.dump_kernel_pages()?; - core::mem::drop(kalloc); - - let mut crashdump = crate::crashdump::CrashDumpBuilder::new(pages)?; - let initdata_addr = self.initdata.gpu_va().get(); - crashdump.add_agx_info(self.cfg, &self.dyncfg, initdata_addr)?; - if let Some(crashlog) = crashlog { - crashdump.add_crashlog(crashlog)?; - } - let crashdump = KBox::new(crashdump.finalize()?, GFP_KERNEL)?; - - devcoredump::dev_coredump( - self.dev.as_ref(), - &crate::THIS_MODULE, - crashdump, - GFP_KERNEL, - msecs_to_jiffies(60 * 60 * 1000), - ); - - Ok(()) - } -} - -#[versions(AGX_NEO)] -impl Gpu for GpuManager::ver { - fn stats_snapshot(&self) -> Option> { - Some(self.stats.clone()) - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn arc_as_any(self: Arc) -> Arc { - self as Arc - } - - fn manager(&self) -> Option<&dyn GpuManager> { - Some(self) - } - - fn init(&self) -> Result { - self.tx_channels.lock().device_control.send( - &fw::channels::DeviceControlMsg::ver::Initialize(Default::default()), - ); - - let initdata = self.initdata.gpu_va().get(); - let mut guard = self.rtkit.lock(); - let mut rtk = guard.as_mut().as_pin_mut().unwrap(); - - rtk.as_mut().boot()?; - rtk.as_mut().start_endpoint(EP_FIRMWARE)?; - rtk.as_mut().start_endpoint(EP_DOORBELL)?; - rtk.as_mut() - .send_message(EP_FIRMWARE, MSG_INIT | (initdata & INIT_DATA_MASK))?; - rtk.as_mut() - .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; - core::mem::drop(guard); - - self.kick_firmware()?; - Ok(()) - } - - fn ids(&self) -> &SequenceIDs { - &self.ids - } - - fn is_crashed(&self) -> bool { - self.crashed.load(Ordering::Relaxed) - } - - fn get_params(&self, params: &mut uapi::drm_asahi_neo_params_global) -> Result { - params.gpu_generation = self.dyncfg.id.gpu_gen as u32; - params.gpu_variant = self.dyncfg.id.gpu_variant as u32; - params.gpu_revision = self.dyncfg.id.gpu_rev as u32; - params.chip_id = self.cfg.chip_id; - params.num_dies = self.cfg.num_dies; - params.num_clusters_total = self.dyncfg.id.num_clusters; - params.num_cores_per_cluster = self.dyncfg.id.num_cores; - params.max_frequency_khz = self.dyncfg.pwr.max_frequency_khz(); - // G13 and G14 share the second shader ISA generation and predate the numbered command - // stream generations. - params.usc_generation = 2; - params.gpu_hal_generation = uapi::drm_asahi_neo_gpu_hal_generation_DRM_ASAHI_NEO_GPU_HAL_LEGACY; - - for (i, mask) in self.dyncfg.id.core_masks.iter().enumerate() { - *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); - } - - Ok(()) - } - - fn base_clock_hz(&self) -> u32 { - self.cfg.base_clock_hz - } - - fn uat_geometry(&self) -> mmu::UatGeometry { - self.uat.geometry() - } - - fn new_vm(&self, kernel_range: Range) -> Result { - self.uat.new_vm(self.ids.vm.next(), kernel_range) - } - - fn new_queue( - &self, - vm: mmu::Vm, - ualloc: Arc>, - ualloc_priv: Arc>, - priority: u32, - usc_exec_base: u64, - ) -> Result> { - let mut kalloc = self.alloc(); - let id = self.ids.queue.next(); - Ok(KBox::new( - queue::Queue::ver::new( - &self.dev, - self, - vm, - &mut kalloc, - ualloc, - ualloc_priv, - self.event_manager.clone(), - self.stats.clone(), - &self.buffer_mgr, - id, - priority, - usc_exec_base, - )?, - GFP_KERNEL, - )?) - } - - fn map_timestamp_buffer( - &self, - mut bo: gem::ObjectRef, - range: Range, - ) -> Result { - bo.map_range_into_range( - self.uat.kernel_vm(), - range, - IOVA_KERN_TIMESTAMP_RANGE, - mmu::UAT_PGSZ as u64, - mmu::PROT_FW_SHARED_RW, - false, - ) - } -} - -#[versions(AGX_NEO)] -impl GpuManager for GpuManager::ver { - fn update_globals(&self) { - let mut timeout: u32 = 2; - if debug_enabled(DebugFlags::WaitForPowerOff) { - timeout = 0; - } else if debug_enabled(DebugFlags::KeepGpuPowered) { - timeout = 5000; - } - - self.initdata.globals.with(|raw, _inner| { - raw.idle_off_delay_ms.store(timeout, Ordering::Relaxed); - }); - } - - fn alloc(&self) -> Guard<'_, KernelAllocators, MutexBackend> { - /* Clean up idle contexts */ - let mut garbage_ctx = KVec::new(); - core::mem::swap(&mut *self.garbage_contexts.lock(), &mut garbage_ctx); - - for ctx in garbage_ctx { - if self.invalidate_context(&ctx).is_err() { - dev_err!( - self.dev.as_ref(), - "GpuContext: Failed to invalidate GPU context!\n" - ); - if debug_enabled(DebugFlags::OopsOnGpuCrash) { - panic!("GPU firmware timed out"); - } - } - } - - let mut guard = self.alloc.lock(); - let (garbage_count, garbage_bytes) = guard.private.garbage(); - let (ro_garbage_count, ro_garbage_bytes) = guard.gpu_ro.garbage(); - - if garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES - || ro_garbage_bytes > MAX_FW_ALLOC_GARBAGE_BYTES - || garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS - || ro_garbage_count > MAX_FW_ALLOC_GARBAGE_OBJECTS - { - mod_dev_dbg!( - self.dev, - "Collecting kalloc garbage (private: {} objects, {} bytes, gpuro: {} objects, {} bytes)\n", - garbage_count, - garbage_bytes, - ro_garbage_count, - ro_garbage_bytes - ); - if self.flush_fw_cache().is_err() { - dev_err!(self.dev.as_ref(), "Failed to flush FW cache\n"); - } else { - guard.private.collect_garbage(garbage_count); - guard.gpu_ro.collect_garbage(ro_garbage_count); - } - } - - guard - } - - fn bind_vm(&self, vm: &mmu::Vm) -> Result { - self.uat.bind(vm) - } - - fn kick_firmware(&self) -> Result { - if self.is_crashed() { - return Err(ENODEV); - } - - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().as_pin_mut().unwrap(); - rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_KICKFW)?; - - Ok(()) - } - - fn flush_fw_cache(&self) -> Result { - mod_dev_dbg!(self.dev, "Flushing coprocessor data cache\n"); - - if self.is_crashed() { - return Err(ENODEV); - } - - // ctx_0 == 0xff or ctx_1 == 0xff cause no effect on context, - // but this command does a full cache flush too, so abuse it - // for that. - - let dc = fw::channels::DeviceControlMsg::ver::DestroyContext { - unk_4: 0, - - ctx_23: 0, - #[ver(V < V13_3)] - __pad0: Default::default(), - unk_c: U32(0), - unk_10: U32(0), - ctx_0: 0xff, - ctx_1: 0xff, - ctx_4: 0, - #[ver(V < V13_3)] - __pad1: Default::default(), - #[ver(V < V13_3)] - unk_18: 0, - gpu_context: None, - __pad2: Default::default(), - }; - - let mut txch = self.tx_channels.lock(); - - let token = txch.device_control.send(&dc); - { - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().as_pin_mut().unwrap(); - rtk.send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL)?; - } - - txch.device_control.wait_for(token)?; - Ok(()) - } - - fn handle_timeout(&self, counter: u32, event_slot: i32, unk: u32) { - dev_err!(self.dev.as_ref(), " (\\________/) \n"); - dev_err!(self.dev.as_ref(), " | | \n"); - dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); - dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); - dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); - dev_err!(self.dev.as_ref(), " |________| \n"); - dev_err!(self.dev.as_ref(), "** GPU timeout nya~!!!!! **\n"); - dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); - dev_err!(self.dev.as_ref(), " Timeout count: {}\n", counter); - dev_err!(self.dev.as_ref(), " Unk: {}\n", unk); - - // If we have fault info, consider it a fault. - let error = match self.get_fault_info() { - Some(info) => workqueue::WorkError::Fault(info), - None => workqueue::WorkError::Timeout, - }; - self.mark_pending_events(event_slot.try_into().ok(), error); - self.recover(); - } - - fn handle_fault(&self) { - dev_err!(self.dev.as_ref(), " (\\________/) \n"); - dev_err!(self.dev.as_ref(), " | | \n"); - dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); - dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); - dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); - dev_err!(self.dev.as_ref(), " |________| \n"); - dev_err!(self.dev.as_ref(), "GPU fault nya~!!!!!\n"); - let error = match self.get_fault_info() { - Some(info) => workqueue::WorkError::Fault(info), - None => workqueue::WorkError::Unknown, - }; - self.mark_pending_events(None, error); - self.recover(); - } - - fn handle_channel_error( - &self, - error_type: ChannelErrorType, - pipe_type: u32, - event_slot: u32, - event_value: u32, - ) { - dev_err!(self.dev.as_ref(), " (\\________/) \n"); - dev_err!(self.dev.as_ref(), " | | \n"); - dev_err!(self.dev.as_ref(), "'.| \\ , / |.'\n"); - dev_err!(self.dev.as_ref(), "--| / (( \\ |--\n"); - dev_err!(self.dev.as_ref(), ".'| _-_- |'.\n"); - dev_err!(self.dev.as_ref(), " |________| \n"); - dev_err!(self.dev.as_ref(), "GPU channel error nya~!!!!!\n"); - dev_err!(self.dev.as_ref(), " Error type: {:?}\n", error_type); - dev_err!(self.dev.as_ref(), " Pipe type: {}\n", pipe_type); - dev_err!(self.dev.as_ref(), " Event slot: {}\n", event_slot); - dev_err!(self.dev.as_ref(), " Event value: {:#x?}\n", event_value); - - self.event_manager.mark_error( - event_slot, - event_value, - workqueue::WorkError::ChannelError(error_type), - ); - - let wq = match self.event_manager.get_owner(event_slot) { - Some(wq) => wq, - None => { - dev_err!( - self.dev.as_ref(), - "Workqueue not found for this event slot!\n" - ); - return; - } - }; - - let wq = match wq.as_any().downcast_ref::() { - Some(wq) => wq, - None => { - dev_crit!(self.dev.as_ref(), "GpuManager mismatched with WorkQueue!\n"); - return; - } - }; - - if debug_enabled(DebugFlags::VerboseFaults) { - wq.dump_info(); - } - - let dc = fw::channels::DeviceControlMsg::ver::RecoverChannel { - pipe_type, - work_queue: wq.info_pointer(), - event_value, - __pad: Default::default(), - }; - - mod_dev_dbg!(self.dev, "Recover Channel command: {:?}\n", &dc); - let mut txch = self.tx_channels.lock(); - - let token = txch.device_control.send(&dc); - { - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().as_pin_mut().unwrap(); - if rtk - .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) - .is_err() - { - dev_err!( - self.dev.as_ref(), - "Failed to send Recover Channel command\n" - ); - } - } - - if txch.device_control.wait_for(token).is_err() { - dev_err!( - self.dev.as_ref(), - "Timed out waiting for Recover Channel command\n" - ); - } - - if debug_enabled(DebugFlags::VerboseFaults) { - wq.dump_info(); - } - } - - fn ack_grow(&self, buffer_slot: u32, vm_slot: u32, counter: u32) { - let halt_count = self - .initdata - .fw_status - .with(|raw, _inner| raw.flags.halt_count.load(Ordering::Relaxed)); - - let dc = fw::channels::DeviceControlMsg::ver::GrowTVBAck { - unk_4: 1, - buffer_slot, - vm_slot, - counter, - subpipe: 0, // TODO - halt_count: U64(halt_count), - __pad: Default::default(), - }; - - mod_dev_dbg!(self.dev, "TVB Grow Ack command: {:?}\n", &dc); - - let mut txch = self.tx_channels.lock(); - - txch.device_control.send(&dc); - { - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().as_pin_mut().unwrap(); - if rtk - .send_message(EP_DOORBELL, MSG_TX_DOORBELL | DOORBELL_DEVCTRL) - .is_err() - { - dev_err!(self.dev.as_ref(), "Failed to send TVB Grow Ack command\n"); - } - } - } - - fn fwctl(&self, msg: fw::channels::FwCtlMsg) -> Result { - if self.is_crashed() { - return Err(ENODEV); - } - - let mut fwctl = self.fwctl_channel.lock(); - let token = fwctl.send(&msg); - { - let mut guard = self.rtkit.lock(); - let rtk = guard.as_mut().as_pin_mut().unwrap(); - rtk.send_message(EP_DOORBELL, MSG_FWCTL)?; - } - fwctl.wait_for(token)?; - Ok(()) - } - - fn get_cfg(&self) -> &'static hw::HwConfig { - self.cfg - } - - fn get_dyncfg(&self) -> &hw::DynConfig { - &self.dyncfg - } - - fn free_context(&self, ctx: KBox>) { - let mut garbage = self.garbage_contexts.lock(); - - if garbage.push(ctx, GFP_KERNEL).is_err() { - dev_err!( - self.dev.as_ref(), - "Failed to reserve space for freed context, deadlock possible.\n" - ); - } - } -} - -#[versions(AGX_NEO)] -impl GpuManagerPriv for GpuManager::ver { - fn end_op(&self) { - let val = self - .initdata - .globals - .with(|raw, _inner| raw.pending_submissions.fetch_sub(1, Ordering::Release)); - - mod_dev_dbg!(self.dev, "OP end (pending: {})\n", val - 1); - } -} diff --git a/drivers/gpu/drm/asahi-neo/hw/mod.rs b/drivers/gpu/drm/asahi-neo/hw/mod.rs deleted file mode 100644 index 8ed9c8625243a8..00000000000000 --- a/drivers/gpu/drm/asahi-neo/hw/mod.rs +++ /dev/null @@ -1,660 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Per-SoC hardware configuration structures -//! -//! This module contains the definitions used to store per-GPU and per-SoC configuration data. - -use crate::driver::AsahiDevice; -use crate::fw::types::*; -use kernel::c_str; -use kernel::prelude::*; - -const MAX_POWERZONES: usize = 5; - -pub(crate) mod t600x; -pub(crate) mod t602x; -pub(crate) mod t8103; -pub(crate) mod t8112; -pub(crate) mod t8140; - -/// GPU generation enumeration. Note: Part of the UABI. -#[derive(Debug, PartialEq, Copy, Clone)] -#[repr(u32)] -pub(crate) enum GpuGen { - G13 = 13, - G14 = 14, - G17 = 17, -} - -/// GPU variant enumeration. Note: Part of the UABI. -#[derive(Debug, PartialEq, Copy, Clone)] -#[repr(u32)] -pub(crate) enum GpuVariant { - P = 'P' as u32, - G = 'G' as u32, - S = 'S' as u32, - C = 'C' as u32, - D = 'D' as u32, -} - -/// GPU revision enumeration. Note: Part of the UABI. -#[derive(Debug, PartialEq, Copy, Clone)] -#[repr(u32)] -pub(crate) enum GpuRevision { - A0 = 0x00, - A1 = 0x01, - B0 = 0x10, - B1 = 0x11, - C0 = 0x20, - C1 = 0x21, -} - -/// GPU core type enumeration. Note: Part of the firmware ABI. -#[derive(Debug, Copy, Clone)] -#[repr(u32)] -pub(crate) enum GpuCore { - // Unknown = 0, - // G5P = 1, - // G5G = 2, - // G9P = 3, - // G9G = 4, - // G10P = 5, - // G11P = 6, - // G11M = 7, - // G11G = 8, - // G12P = 9, - // G13P = 10, - G13G = 11, - G13S = 12, - G13C = 13, - // G14P = 14, - G14G = 15, - G14S = 16, - G14C = 17, - G14D = 18, // Split out, unlike G13D -} - -/// GPU revision ID. Note: Part of the firmware ABI. -#[derive(Debug, PartialEq, Copy, Clone)] -#[repr(u32)] -pub(crate) enum GpuRevisionID { - // Unknown = 0, - A0 = 1, - A1 = 2, - B0 = 3, - B1 = 4, - C0 = 5, - C1 = 6, -} - -/// A single performance state of the GPU. -#[derive(Debug)] -pub(crate) struct PState { - /// Voltage in millivolts, per GPU cluster. - pub(crate) volt_mv: KVec, - /// Frequency in hertz. - pub(crate) freq_hz: u32, - /// Maximum power consumption of the GPU at this pstate, in milliwatts. - pub(crate) pwr_mw: u32, -} - -impl PState { - pub(crate) fn max_volt_mv(&self) -> u32 { - *self.volt_mv.iter().max().expect("No voltages") - } -} - -/// A power zone definition (we have no idea what this is but Apple puts them in the DT). -#[allow(missing_docs)] -#[derive(Debug, Copy, Clone)] -pub(crate) struct PowerZone { - pub(crate) target: u32, - pub(crate) target_offset: u32, - pub(crate) filter_tc: u32, -} - -/// An MMIO mapping used by the firmware. -#[derive(Debug, Copy, Clone)] -pub(crate) struct IOMapping { - /// Base physical address of the mapping. - pub(crate) base: usize, - /// Whether this mapping should be replicated to all dies - pub(crate) per_die: bool, - /// Number of mappings. - pub(crate) count: usize, - /// Size of one mapping. - pub(crate) size: usize, - /// Stride between mappings. - pub(crate) stride: usize, - /// Whether the mapping should be writable. - pub(crate) writable: bool, -} - -impl IOMapping { - /// Convenience constructor for a new IOMapping. - pub(crate) const fn new( - base: usize, - per_die: bool, - count: usize, - size: usize, - stride: usize, - writable: bool, - ) -> IOMapping { - IOMapping { - base, - per_die, - count, - size, - stride, - writable, - } - } -} - -/// Unknown HwConfigA fields that vary from SoC to SoC. -#[allow(missing_docs)] -#[derive(Debug, Copy, Clone)] -pub(crate) struct HwConfigA { - pub(crate) unk_87c: i32, - pub(crate) unk_8cc: u32, - pub(crate) unk_e24: u32, -} - -/// Unknown HwConfigB fields that vary from SoC to SoC. -#[allow(missing_docs)] -#[derive(Debug, Copy, Clone)] -pub(crate) struct HwConfigB { - pub(crate) unk_454: u32, - pub(crate) unk_4e0: u64, - pub(crate) unk_534: u32, - pub(crate) unk_ab8: u32, - pub(crate) unk_abc: u32, - pub(crate) unk_b30: u32, -} - -/// Render command configs that vary from SoC to SoC. -#[derive(Debug, Copy, Clone)] -pub(crate) struct HwRenderConfig { - /// Vertex/tiling-related configuration register (lsb: disable clustering) - pub(crate) tiling_control: u32, -} - -#[derive(Debug)] -pub(crate) struct HwConfigShared2Curves { - pub(crate) t1_coef: u32, - pub(crate) t2: &'static [i16], - pub(crate) t3_coefs: &'static [u32], - pub(crate) t3_scales: &'static [u32], -} - -/// Static hardware clustering configuration for multi-cluster SoCs. -#[derive(Debug)] -pub(crate) struct HwClusteringConfig { - pub(crate) meta1_blocksize: usize, - pub(crate) meta2_size: usize, - pub(crate) meta3_size: usize, - pub(crate) meta4_size: usize, - pub(crate) max_splits: usize, -} - -/// Static hardware configuration for a given SoC model. -#[derive(Debug)] -pub(crate) struct HwConfig { - /// Chip ID in hex format (e.g. 0x8103 for t8103). - pub(crate) chip_id: u32, - /// GPU generation. - pub(crate) gpu_gen: GpuGen, - /// GPU variant type. - pub(crate) gpu_variant: GpuVariant, - /// GPU core type ID (as known by the firmware). - pub(crate) gpu_core: GpuCore, - - /// Base clock used used for timekeeping. - pub(crate) base_clock_hz: u32, - /// Input address bits translated by each UAT page table root on this SoC. - pub(crate) uat_ias: u32, - /// Output address space for the UAT on this SoC. - pub(crate) uat_oas: u32, - /// Number of dies on this SoC. - pub(crate) num_dies: u32, - /// Maximum number of clusters on this SoC. - pub(crate) max_num_clusters: u32, - /// Maximum number of cores per cluster for this GPU. - pub(crate) max_num_cores: u32, - /// Maximum number of frags per cluster for this GPU. - pub(crate) max_num_frags: u32, - /// Maximum number of GPs per cluster for this GPU. - pub(crate) max_num_gps: u32, - - /// Required size of the first preemption buffer. - pub(crate) preempt1_size: usize, - /// Required size of the second preemption buffer. - pub(crate) preempt2_size: usize, - /// Required size of the third preemption buffer. - pub(crate) preempt3_size: usize, - - /// Required size of the compute preemption buffer. - pub(crate) compute_preempt1_size: usize, - - pub(crate) clustering: Option, - - /// Rendering-relevant configuration. - pub(crate) render: HwRenderConfig, - - /// Misc HWDataA field values. - pub(crate) da: HwConfigA, - /// Misc HWDataB field values. - pub(crate) db: HwConfigB, - /// HwDataShared1.table. - pub(crate) shared1_tab: &'static [i32], - /// HwDataShared1.unk_a4. - pub(crate) shared1_a4: u32, - /// HwDataShared2.table. - pub(crate) shared2_tab: &'static [i32], - /// HwDataShared2.unk_508. - pub(crate) shared2_unk_508: u32, - /// HwDataShared2.unk_508. - pub(crate) shared2_curves: Option, - - /// HwDataShared3.unk_8. - pub(crate) shared3_unk: u32, - /// HwDataShared3.table. - pub(crate) shared3_tab: &'static [u32], - - /// Globals.idle_off_standby_timer. - pub(crate) idle_off_standby_timer_default: u32, - /// Globals.unk_hws2_4. - pub(crate) unk_hws2_4: Option<[F32; 8]>, - /// Globals.unk_hws2_24. - pub(crate) unk_hws2_24: u32, - /// Globals.unk_54 - pub(crate) global_unk_54: u16, - - /// Constant related to SRAM voltages. - pub(crate) sram_k: F32, - /// Unknown per-cluster coefficients 1. - pub(crate) unk_coef_a: &'static [&'static [F32]], - /// Unknown per-cluster coefficients 2. - pub(crate) unk_coef_b: &'static [&'static [F32]], - /// Unknown table in Global struct. - pub(crate) global_tab: Option<&'static [u8]>, - /// Whether this GPU has CS/AFR performance states - pub(crate) has_csafr: bool, - - /// Temperature sensor list (8 bits per sensor). - pub(crate) fast_sensor_mask: [u64; 2], - /// Temperature sensor list (alternate). - pub(crate) fast_sensor_mask_alt: [u64; 2], - /// Temperature sensor present bitmask. - pub(crate) fast_die0_sensor_present: u32, - /// Required MMIO mappings for this GPU/firmware. - pub(crate) io_mappings: &'static [Option], - /// SRAM base - pub(crate) sram_base: Option, - /// SRAM size - pub(crate) sram_size: Option, -} - -/// Dynamic (fetched from hardware/DT) configuration. -#[derive(Debug)] -pub(crate) struct DynConfig { - /// Base physical address of the UAT TTB (from DT reserved memory region). - pub(crate) uat_ttb_base: u64, - /// GPU ID configuration read from hardware. - pub(crate) id: GpuIdConfig, - /// Power calibration configuration for this specific chip/device. - pub(crate) pwr: PwrConfig, - /// Firmware version. - #[allow(dead_code)] - pub(crate) firmware_version: KVec, - - pub(crate) hw_data_a: KVVec, - pub(crate) hw_data_b: KVVec, - pub(crate) hw_globals: KVVec, -} - -/// Specific GPU ID configuration fetched from SGX MMIO registers. -#[derive(Debug)] -pub(crate) struct GpuIdConfig { - /// GPU generation (should match static config). - pub(crate) gpu_gen: GpuGen, - /// GPU variant type (should match static config). - pub(crate) gpu_variant: GpuVariant, - /// GPU silicon revision. - pub(crate) gpu_rev: GpuRevision, - /// GPU silicon revision ID (firmware enum). - pub(crate) gpu_rev_id: GpuRevisionID, - /// Total number of GPU clusters. - pub(crate) num_clusters: u32, - /// Maximum number of GPU cores per cluster. - pub(crate) num_cores: u32, - /// Number of frags per cluster. - pub(crate) num_frags: u32, - /// Number of GPs per cluster. - pub(crate) num_gps: u32, - /// Total number of active cores for the whole GPU. - pub(crate) total_active_cores: u32, - /// Mask of active cores per cluster. - pub(crate) core_masks: KVec, - /// Packed mask of all active cores. - pub(crate) core_masks_packed: KVec, -} - -/// Configurable CS/AFR GPU power settings from the device tree. -#[derive(Debug)] -pub(crate) struct CsAfrPwrConfig { - /// GPU CS performance state list. - pub(crate) perf_states_cs: KVec, - /// GPU AFR performance state list. - pub(crate) perf_states_afr: KVec, - - /// CS leakage coefficient per die. - pub(crate) leak_coef_cs: KVec, - /// AFR leakage coefficient per die. - pub(crate) leak_coef_afr: KVec, - - /// Minimum voltage for the CS/AFR SRAM power domain in microvolts. - pub(crate) min_sram_microvolt: u32, -} - -/// Configurable GPU power settings from the device tree. -#[derive(Debug)] -pub(crate) struct PwrConfig { - /// GPU performance state list. - pub(crate) perf_states: KVec, - /// GPU power zone list. - pub(crate) power_zones: KVec, - - /// Core leakage coefficient per cluster. - pub(crate) core_leak_coef: KVec, - /// SRAM leakage coefficient per cluster. - pub(crate) sram_leak_coef: KVec, - - pub(crate) csafr: Option, - - /// Maximum total power of the GPU in milliwatts. - pub(crate) max_power_mw: u32, - /// Maximum frequency of the GPU in megahertz. - pub(crate) max_freq_mhz: u32, - - /// Minimum performance state to start at. - pub(crate) perf_base_pstate: u32, - /// Maximum enabled performance state. - pub(crate) perf_max_pstate: u32, - - /// Minimum voltage for the SRAM power domain in microvolts. - pub(crate) min_sram_microvolt: u32, - - // Most of these fields are just named after Apple ADT property names and we don't fully - // understand them. They configure various power-related PID loops and filters. - /// Average power filter time constant in milliseconds. - pub(crate) avg_power_filter_tc_ms: u32, - /// Average power filter PID integral gain? - pub(crate) avg_power_ki_only: F32, - /// Average power filter PID proportional gain? - pub(crate) avg_power_kp: F32, - pub(crate) avg_power_min_duty_cycle: u32, - /// Average power target filter time constant in periods. - pub(crate) avg_power_target_filter_tc: u32, - /// "Fast die0" (temperature?) PID integral gain. - pub(crate) fast_die0_integral_gain: F32, - /// "Fast die0" (temperature?) PID proportional gain. - pub(crate) fast_die0_proportional_gain: F32, - pub(crate) fast_die0_prop_tgt_delta: u32, - pub(crate) fast_die0_release_temp: u32, - /// Delay from the fender (?) becoming idle to powerdown - pub(crate) fender_idle_off_delay_ms: u32, - /// Timeout from firmware early wake to sleep if no work was submitted (?) - pub(crate) fw_early_wake_timeout_ms: u32, - /// Delay from the GPU becoming idle to powerdown - pub(crate) idle_off_delay_ms: u32, - /// Related to the above? - pub(crate) idle_off_standby_timer: u32, - /// Percent? - pub(crate) perf_boost_ce_step: u32, - /// Minimum utilization before performance state is increased in %. - pub(crate) perf_boost_min_util: u32, - pub(crate) perf_filter_drop_threshold: u32, - /// Performance PID filter time constant? (periods?) - pub(crate) perf_filter_time_constant: u32, - /// Performance PID filter time constant 2? (periods?) - pub(crate) perf_filter_time_constant2: u32, - /// Performance PID integral gain. - pub(crate) perf_integral_gain: F32, - /// Performance PID integral gain 2 (?). - pub(crate) perf_integral_gain2: F32, - pub(crate) perf_integral_min_clamp: u32, - /// Performance PID proportional gain. - pub(crate) perf_proportional_gain: F32, - /// Performance PID proportional gain 2 (?). - pub(crate) perf_proportional_gain2: F32, - pub(crate) perf_reset_iters: u32, - /// Target GPU utilization for the performance controller in %. - pub(crate) perf_tgt_utilization: u32, - /// Power sampling period in milliseconds. - pub(crate) power_sample_period: u32, - /// PPM (?) filter time constant in milliseconds. - pub(crate) ppm_filter_time_constant_ms: u32, - /// PPM (?) filter PID integral gain. - pub(crate) ppm_ki: F32, - /// PPM (?) filter PID proportional gain. - pub(crate) ppm_kp: F32, - /// Power consumption filter time constant (periods?) - pub(crate) pwr_filter_time_constant: u32, - /// Power consumption filter PID integral gain. - pub(crate) pwr_integral_gain: F32, - pub(crate) pwr_integral_min_clamp: u32, - pub(crate) pwr_min_duty_cycle: u32, - pub(crate) pwr_proportional_gain: F32, - /// Power sample period in base clocks, used when not an integer number of ms - pub(crate) pwr_sample_period_aic_clks: u32, - - pub(crate) se_engagement_criteria: i32, - pub(crate) se_filter_time_constant: u32, - pub(crate) se_filter_time_constant_1: u32, - pub(crate) se_inactive_threshold: u32, - pub(crate) se_ki: F32, - pub(crate) se_ki_1: F32, - pub(crate) se_kp: F32, - pub(crate) se_kp_1: F32, - pub(crate) se_reset_criteria: u32, -} - -impl PwrConfig { - fn load_opp( - dev: &AsahiDevice, - name: &CStr, - cfg: &HwConfig, - is_main: bool, - ) -> Result> { - let mut perf_states = KVec::new(); - - let node = dev.as_ref().of_node().ok_or(EIO)?; - let opps = node.parse_phandle(name, 0).ok_or(EIO)?; - - for opp in opps.children() { - let freq_hz: u64 = opp.get_property(c_str!("opp-hz"))?; - let mut volt_uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; - let pwr_uw: u32 = if is_main { - opp.get_property(c_str!("opp-microwatt"))? - } else { - 0 - }; - - let voltage_count = if is_main { - cfg.max_num_clusters - } else { - cfg.num_dies - }; - - if volt_uv.len() != voltage_count as usize { - dev_err!( - dev.as_ref(), - "Invalid opp-microvolt length (expected {}, got {})\n", - voltage_count, - volt_uv.len() - ); - return Err(EINVAL); - } - - volt_uv.iter_mut().for_each(|a| *a /= 1000); - let volt_mv = volt_uv; - - let pwr_mw = pwr_uw / 1000; - - perf_states.push( - PState { - freq_hz: freq_hz.try_into()?, - volt_mv, - pwr_mw, - }, - GFP_KERNEL, - )?; - } - - if perf_states.is_empty() { - Err(EINVAL) - } else { - Ok(perf_states) - } - } - - /// Load the GPU power configuration from the device tree. - pub(crate) fn load(dev: &AsahiDevice, cfg: &HwConfig) -> Result { - let perf_states = Self::load_opp(dev, c_str!("operating-points-v2"), cfg, true)?; - let node = dev.as_ref().of_node().ok_or(EIO)?; - - macro_rules! prop { - ($prop:expr, $default:expr) => {{ - node.get_opt_property(c_str!($prop)) - .map_err(|e| { - dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); - e - })? - .unwrap_or($default) - }}; - ($prop:expr) => {{ - node.get_property(c_str!($prop)).map_err(|e| { - dev_err!(dev.as_ref(), "Error reading property {}: {:?}\n", $prop, e); - e - })? - }}; - } - - let pz_data = prop!("apple,power-zones", KVec::new()); - - if pz_data.len() > 3 * MAX_POWERZONES || pz_data.len() % 3 != 0 { - dev_err!(dev.as_ref(), "Invalid apple,power-zones value\n"); - return Err(EINVAL); - } - - let mut power_zones = KVec::new(); - for zone in pz_data.chunks_exact(3) { - power_zones.push( - PowerZone { - target: zone[0], - target_offset: zone[1], - filter_tc: zone[2], - }, - GFP_KERNEL, - )?; - } - - let core_leak_coef: KVec = prop!("apple,core-leak-coef"); - let sram_leak_coef: KVec = prop!("apple,sram-leak-coef"); - - if core_leak_coef.len() != cfg.max_num_clusters as usize { - dev_err!(dev.as_ref(), "Invalid apple,core-leak-coef\n"); - return Err(EINVAL); - } - if sram_leak_coef.len() != cfg.max_num_clusters as usize { - dev_err!(dev.as_ref(), "Invalid apple,sram_leak_coef\n"); - return Err(EINVAL); - } - - let csafr = if cfg.has_csafr { - Some(CsAfrPwrConfig { - perf_states_cs: Self::load_opp(dev, c_str!("apple,cs-opp"), cfg, false)?, - perf_states_afr: Self::load_opp(dev, c_str!("apple,afr-opp"), cfg, false)?, - leak_coef_cs: prop!("apple,cs-leak-coef"), - leak_coef_afr: prop!("apple,afr-leak-coef"), - min_sram_microvolt: prop!("apple,csafr-min-sram-microvolt"), - }) - } else { - None - }; - - let power_sample_period: u32 = prop!("apple,power-sample-period"); - - Ok(PwrConfig { - core_leak_coef, - sram_leak_coef, - - max_power_mw: perf_states.iter().map(|a| a.pwr_mw).max().unwrap(), - max_freq_mhz: perf_states.iter().map(|a| a.freq_hz).max().unwrap() / 1_000_000, - - perf_base_pstate: prop!("apple,perf-base-pstate", 1), - perf_max_pstate: perf_states.len() as u32 - 1, - min_sram_microvolt: prop!("apple,min-sram-microvolt"), - - avg_power_filter_tc_ms: prop!("apple,avg-power-filter-tc-ms"), - avg_power_ki_only: prop!("apple,avg-power-ki-only"), - avg_power_kp: prop!("apple,avg-power-kp"), - avg_power_min_duty_cycle: prop!("apple,avg-power-min-duty-cycle"), - avg_power_target_filter_tc: prop!("apple,avg-power-target-filter-tc"), - fast_die0_integral_gain: prop!("apple,fast-die0-integral-gain"), - fast_die0_proportional_gain: prop!("apple,fast-die0-proportional-gain"), - fast_die0_prop_tgt_delta: prop!("apple,fast-die0-prop-tgt-delta", 0), - fast_die0_release_temp: prop!("apple,fast-die0-release-temp", 80), - fender_idle_off_delay_ms: prop!("apple,fender-idle-off-delay-ms", 40), - fw_early_wake_timeout_ms: prop!("apple,fw-early-wake-timeout-ms", 5), - idle_off_delay_ms: prop!("apple,idle-off-delay-ms", 2), - idle_off_standby_timer: prop!( - "apple,idleoff-standby-timer", - cfg.idle_off_standby_timer_default - ), - perf_boost_ce_step: prop!("apple,perf-boost-ce-step", 25), - perf_boost_min_util: prop!("apple,perf-boost-min-util", 100), - perf_filter_drop_threshold: prop!("apple,perf-filter-drop-threshold"), - perf_filter_time_constant2: prop!("apple,perf-filter-time-constant2"), - perf_filter_time_constant: prop!("apple,perf-filter-time-constant"), - perf_integral_gain2: prop!("apple,perf-integral-gain2"), - perf_integral_gain: prop!("apple,perf-integral-gain", f32!(7.8956833)), - perf_integral_min_clamp: prop!("apple,perf-integral-min-clamp"), - perf_proportional_gain2: prop!("apple,perf-proportional-gain2"), - perf_proportional_gain: prop!("apple,perf-proportional-gain", f32!(14.707963)), - perf_reset_iters: prop!("apple,perf-reset-iters", 6), - perf_tgt_utilization: prop!("apple,perf-tgt-utilization"), - power_sample_period, - ppm_filter_time_constant_ms: prop!("apple,ppm-filter-time-constant-ms"), - ppm_ki: prop!("apple,ppm-ki"), - ppm_kp: prop!("apple,ppm-kp"), - pwr_filter_time_constant: prop!("apple,pwr-filter-time-constant", 313), - pwr_integral_gain: prop!("apple,pwr-integral-gain", f32!(0.0202129)), - pwr_integral_min_clamp: prop!("apple,pwr-integral-min-clamp", 0), - pwr_min_duty_cycle: prop!("apple,pwr-min-duty-cycle"), - pwr_proportional_gain: prop!("apple,pwr-proportional-gain", f32!(5.2831855)), - pwr_sample_period_aic_clks: prop!( - "apple,pwr-sample-period-aic-clks", - cfg.base_clock_hz / 1000 * power_sample_period - ), - se_engagement_criteria: prop!("apple,se-engagement-criteria", -1), - se_filter_time_constant: prop!("apple,se-filter-time-constant", 9), - se_filter_time_constant_1: prop!("apple,se-filter-time-constant-1", 3), - se_inactive_threshold: prop!("apple,se-inactive-threshold", 2500), - se_ki: prop!("apple,se-ki", f32!(-50.0)), - se_ki_1: prop!("apple,se-ki-1", f32!(-100.0)), - se_kp: prop!("apple,se-kp", f32!(-5.0)), - se_kp_1: prop!("apple,se-kp-1", f32!(-10.0)), - se_reset_criteria: prop!("apple,se-reset-criteria", 50), - - perf_states, - power_zones, - csafr, - }) - } - - pub(crate) fn max_frequency_khz(&self) -> u32 { - self.perf_states[self.perf_max_pstate as usize].freq_hz / 1000 - } -} diff --git a/drivers/gpu/drm/asahi-neo/hw/t600x.rs b/drivers/gpu/drm/asahi-neo/hw/t600x.rs deleted file mode 100644 index 85cf88054e44c5..00000000000000 --- a/drivers/gpu/drm/asahi-neo/hw/t600x.rs +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. - -use crate::f32; - -use super::*; - -const fn iomaps(mcc_count: usize, has_die1: bool) -> [Option; 20] { - [ - Some(IOMapping::new(0x404d00000, false, 1, 0x1c000, 0, true)), // Fender - Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer - Some(IOMapping::new(0x28e104000, false, 1, 0x4000, 0, true)), // AICSWInt - Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX - None, // UVD - None, // unused - None, // DisplayUnderrunWA - Some(IOMapping::new(0x28e494000, true, 1, 0x4000, 0, false)), // AnalogTempSensorControllerRegs - None, // PMPDoorbell - Some(IOMapping::new(0x404d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs - Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs - Some(IOMapping::new( - 0x200000000, - true, - mcc_count, - 0xd8000, - 0x1000000, - true, - )), // MCache registers - None, // AICBankedRegisters - None, // PMGRScratch - Some(IOMapping::new(0x2643c4000, false, 1, 0x1000, 0, true)), // NIA Special agent idle register die 0 - if has_die1 { - // NIA Special agent idle register die 1 - Some(IOMapping::new(0x22643c4000, false, 1, 0x1000, 0, true)) - } else { - None - }, - None, // CRE registers - None, // Streaming codec registers - Some(IOMapping::new(0x28e3d0000, false, 1, 0x1000, 0, true)), // ? - Some(IOMapping::new(0x28e3c0000, false, 1, 0x2000, 0, false)), // ? - ] -} - -pub(crate) const HWCONFIG_T6002: super::HwConfig = HwConfig { - chip_id: 0x6002, - gpu_gen: GpuGen::G13, - gpu_variant: GpuVariant::D, - gpu_core: GpuCore::G13C, - - base_clock_hz: 24_000_000, - uat_ias: 39, - uat_oas: 42, - num_dies: 2, - max_num_clusters: 8, - max_num_cores: 8, - max_num_frags: 8, - max_num_gps: 4, - - preempt1_size: 0x540, - preempt2_size: 0x280, - preempt3_size: 0x20, - compute_preempt1_size: 0x3bd00, - clustering: Some(HwClusteringConfig { - meta1_blocksize: 0x44, - meta2_size: 0xc0 * 8, - meta3_size: 0x280 * 8, - meta4_size: 0x30 * 16, - max_splits: 16, - }), - - render: HwRenderConfig { - tiling_control: 0xa540, - }, - - da: HwConfigA { - unk_87c: 900, - unk_8cc: 11000, - unk_e24: 125, - }, - db: HwConfigB { - unk_454: 1, - unk_4e0: 4, - unk_534: 1, - unk_ab8: 0x2084, - unk_abc: 0x80, - unk_b30: 0, - }, - shared1_tab: &[ - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - ], - shared1_a4: 0xffff, - shared2_tab: &[-1, -1, -1, -1, 0x2aa, 0xaaa, -1, -1, 0, 0], - shared2_unk_508: 0xcc00001, - shared2_curves: None, - shared3_unk: 0, - shared3_tab: &[], - idle_off_standby_timer_default: 0, - unk_hws2_4: None, - unk_hws2_24: 0, - global_unk_54: 0xffff, - sram_k: f32!(1.02), - unk_coef_a: &[ - &f32!([9.838]), - &f32!([9.819]), - &f32!([9.826]), - &f32!([9.799]), - &f32!([9.799]), - &f32!([9.826]), - &f32!([9.819]), - &f32!([9.838]), - ], - unk_coef_b: &[ - &f32!([13.0]), - &f32!([13.0]), - &f32!([13.0]), - &f32!([13.0]), - &f32!([13.0]), - &f32!([13.0]), - &f32!([13.0]), - &f32!([13.0]), - ], - global_tab: Some(&[ - 0, 1, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, - ]), - has_csafr: false, - fast_sensor_mask: [0x8080808080808080, 0], - fast_sensor_mask_alt: [0x9090909090909090, 0], - fast_die0_sensor_present: 0xff, - io_mappings: &iomaps(8, true), - sram_base: None, - sram_size: None, -}; - -pub(crate) const HWCONFIG_T6001: super::HwConfig = HwConfig { - chip_id: 0x6001, - gpu_variant: GpuVariant::C, - gpu_core: GpuCore::G13C, - - num_dies: 1, - max_num_clusters: 4, - fast_sensor_mask: [0x80808080, 0], - fast_sensor_mask_alt: [0x90909090, 0], - fast_die0_sensor_present: 0x0f, - io_mappings: &iomaps(8, false), - ..HWCONFIG_T6002 -}; - -pub(crate) const HWCONFIG_T6000: super::HwConfig = HwConfig { - chip_id: 0x6000, - gpu_variant: GpuVariant::S, - gpu_core: GpuCore::G13S, - - max_num_clusters: 2, - fast_sensor_mask: [0x8080, 0], - fast_sensor_mask_alt: [0x9090, 0], - fast_die0_sensor_present: 0x03, - io_mappings: &iomaps(4, false), - ..HWCONFIG_T6001 -}; diff --git a/drivers/gpu/drm/asahi-neo/hw/t602x.rs b/drivers/gpu/drm/asahi-neo/hw/t602x.rs deleted file mode 100644 index 6e2033ecb50ddd..00000000000000 --- a/drivers/gpu/drm/asahi-neo/hw/t602x.rs +++ /dev/null @@ -1,180 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware configuration for t600x (M1 Pro/Max/Ultra) platforms. - -use crate::f32; - -use super::*; - -const fn iomaps(chip_id: u32, mcc_count: usize) -> [Option; 24] { - [ - Some(IOMapping::new(0x404d00000, false, 1, 0x144000, 0, true)), // Fender - Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer - Some(IOMapping::new(0x28e106000, false, 1, 0x4000, 0, true)), // AICSWInt - Some(IOMapping::new(0x404000000, false, 1, 0x20000, 0, true)), // RGX - None, // UVD - None, // unused - None, // DisplayUnderrunWA - Some(match chip_id { - 0x6020 => IOMapping::new(0x28e460000, true, 1, 0x4000, 0, false), - _ => IOMapping::new(0x28e478000, true, 1, 0x4000, 0, false), - }), // AnalogTempSensorControllerRegs - None, // PMPDoorbell - Some(IOMapping::new(0x404e08000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs - None, // GMGIFAFRegs - Some(IOMapping::new( - 0x200000000, - true, - mcc_count, - 0xd8000, - 0x1000000, - true, - )), // MCache registers - Some(IOMapping::new(0x28e118000, false, 1, 0x4000, 0, false)), // AICBankedRegisters - None, // PMGRScratch - None, // NIA Special agent idle register die 0 - None, // NIA Special agent idle register die 1 - None, // CRE registers - None, // Streaming codec registers - Some(IOMapping::new(0x28e3d0000, false, 1, 0x4000, 0, true)), // ? - Some(IOMapping::new(0x28e3c0000, false, 1, 0x4000, 0, false)), // ? - Some(IOMapping::new(0x28e3d8000, false, 1, 0x4000, 0, true)), // ? - Some(IOMapping::new(0x404eac000, true, 1, 0x4000, 0, true)), // ? - None, - None, - ] -} - -// TODO: Tentative -pub(crate) const HWCONFIG_T6022: super::HwConfig = HwConfig { - chip_id: 0x6022, - gpu_gen: GpuGen::G14, - gpu_variant: GpuVariant::D, - gpu_core: GpuCore::G14D, - - base_clock_hz: 24_000_000, - uat_ias: 39, - uat_oas: 42, - num_dies: 2, - max_num_clusters: 8, - max_num_cores: 10, - max_num_frags: 10, - max_num_gps: 4, - - preempt1_size: 0x540, - preempt2_size: 0x280, - preempt3_size: 0x40, - compute_preempt1_size: 0x25980 * 2, // Conservative guess - clustering: Some(HwClusteringConfig { - meta1_blocksize: 0x44, - meta2_size: 0xc0 * 16, - meta3_size: 0x280 * 16, - meta4_size: 0x10 * 128, - max_splits: 64, - }), - - render: HwRenderConfig { - tiling_control: 0x180340, - }, - - da: HwConfigA { - unk_87c: 500, - unk_8cc: 11000, - unk_e24: 125, - }, - db: HwConfigB { - unk_454: 1, - unk_4e0: 4, - unk_534: 0, - unk_ab8: 0, // Unused - unk_abc: 0, // Unused - unk_b30: 0, - }, - shared1_tab: &[ - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - ], - shared1_a4: 0, - shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0xaaaaa, 0], - shared2_unk_508: 0xc00007, - shared2_curves: Some(HwConfigShared2Curves { - t1_coef: 11000, - t2: &[ - 0xf07, 0x4c0, 0x680, 0x8c0, 0xa80, 0xc40, 0xd80, 0xec0, 0xf40, - ], - t3_coefs: &[0, 20, 27, 36, 43, 50, 55, 60, 62], - t3_scales: &[9, 3209, 10400], - }), - shared3_unk: 8, - shared3_tab: &[ - 125, 125, 125, 125, 125, 125, 125, 125, 7500, 125, 125, 125, 125, 125, 125, 125, - ], - idle_off_standby_timer_default: 700, - unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.5, 0.9])), - unk_hws2_24: 6, - global_unk_54: 4000, - sram_k: f32!(1.02), - unk_coef_a: &[ - &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), - &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), - &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), - &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), - &f32!([0.0, 0.0, 0.0, 6.9, 0.0]), - &f32!([0.0, 8.2, 0.0, 6.9, 0.0]), - &f32!([0.0, 0.0, 0.0, 6.9, 6.9]), - &f32!([0.0, 8.2, 0.0, 6.9, 6.9]), - ], - unk_coef_b: &[ - &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), - &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), - &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), - &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), - &f32!([0.0, 0.0, 0.0, 8.0, 0.0]), - &f32!([0.0, 9.0, 0.0, 8.0, 0.0]), - &f32!([0.0, 0.0, 0.0, 8.0, 8.0]), - &f32!([0.0, 9.0, 0.0, 8.0, 8.0]), - ], - global_tab: Some(&[ - 0, 2, 2, 1, 1, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 2, 90, 75, 1, 1, 1, 1, 90, 75, 1, 1, - ]), - has_csafr: true, - fast_sensor_mask: [0x40005000c000d00, 0xd000c0005000400], - // Apple typo? Should probably be 0x140015001c001d00 - fast_sensor_mask_alt: [0x140015001d001d00, 0x1d001c0015001400], - fast_die0_sensor_present: 0, // Unused - io_mappings: &iomaps(0x6022, 8), - sram_base: Some(0x404d60000), - sram_size: Some(0x20000), -}; - -pub(crate) const HWCONFIG_T6021: super::HwConfig = HwConfig { - chip_id: 0x6021, - gpu_variant: GpuVariant::C, - gpu_core: GpuCore::G14C, - - num_dies: 1, - max_num_clusters: 4, - compute_preempt1_size: 0x25980, - unk_hws2_4: Some(f32!([1.0, 0.8, 0.2, 0.9, 0.1, 0.25, 0.7, 0.9])), - fast_sensor_mask: [0x40005000c000d00, 0], - fast_sensor_mask_alt: [0x140015001d001d00, 0], - io_mappings: &iomaps(0x6021, 8), - ..HWCONFIG_T6022 -}; - -pub(crate) const HWCONFIG_T6020: super::HwConfig = HwConfig { - chip_id: 0x6020, - gpu_variant: GpuVariant::S, - gpu_core: GpuCore::G14S, - - db: HwConfigB { - unk_454: 0, - ..HWCONFIG_T6021.db - }, - - max_num_clusters: 2, - fast_sensor_mask: [0xc000d00, 0], - fast_sensor_mask_alt: [0x1d001d00, 0], - io_mappings: &iomaps(0x6020, 4), - ..HWCONFIG_T6021 -}; diff --git a/drivers/gpu/drm/asahi-neo/hw/t8103.rs b/drivers/gpu/drm/asahi-neo/hw/t8103.rs deleted file mode 100644 index a2a76600c0a3bb..00000000000000 --- a/drivers/gpu/drm/asahi-neo/hw/t8103.rs +++ /dev/null @@ -1,93 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware configuration for t8103 platforms (M1). - -use crate::f32; - -use super::*; - -pub(crate) const HWCONFIG: super::HwConfig = HwConfig { - chip_id: 0x8103, - gpu_gen: GpuGen::G13, - gpu_variant: GpuVariant::G, - gpu_core: GpuCore::G13G, - - base_clock_hz: 24_000_000, - uat_ias: 39, - uat_oas: 40, - num_dies: 1, - max_num_clusters: 1, - max_num_cores: 8, - max_num_frags: 8, - max_num_gps: 4, - - preempt1_size: 0x540, - preempt2_size: 0x280, - preempt3_size: 0x20, - compute_preempt1_size: 0x7f80, - clustering: None, - - render: HwRenderConfig { - // bit 0: disable clustering (always) - tiling_control: 0xa041, - }, - - da: HwConfigA { - unk_87c: -220, - unk_8cc: 9880, - unk_e24: 112, - }, - db: HwConfigB { - unk_454: 1, - unk_4e0: 0, - unk_534: 0, - unk_ab8: 0x48, - unk_abc: 0x8, - unk_b30: 0, - }, - shared1_tab: &[ - -1, 0x7282, 0x50ea, 0x370a, 0x25be, 0x1c1f, 0x16fb, -1, -1, -1, -1, -1, -1, -1, -1, -1, - ], - shared1_a4: 0xffff, - shared2_tab: &[0x800, 0x1555, -1, -1, -1, -1, -1, -1, 0, 0], - shared2_unk_508: 0xc00007, - shared2_curves: None, - shared3_unk: 0, - shared3_tab: &[], - idle_off_standby_timer_default: 0, - unk_hws2_4: None, - unk_hws2_24: 0, - global_unk_54: 0xffff, - sram_k: f32!(1.02), - unk_coef_a: &[], - unk_coef_b: &[], - global_tab: None, - has_csafr: false, - fast_sensor_mask: [0x12, 0], - fast_sensor_mask_alt: [0x12, 0], - fast_die0_sensor_present: 0x01, - io_mappings: &[ - Some(IOMapping::new(0x204d00000, false, 1, 0x1c000, 0, true)), // Fender - Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer - Some(IOMapping::new(0x23b104000, false, 1, 0x4000, 0, true)), // AICSWInt - Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX - None, // UVD - None, // unused - None, // DisplayUnderrunWA - Some(IOMapping::new(0x23b2e8000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs - Some(IOMapping::new(0x23bc00000, false, 1, 0x1000, 0, true)), // PMPDoorbell - Some(IOMapping::new(0x204d80000, false, 1, 0x5000, 0, true)), // MetrologySensorRegs - Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs - Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers - None, // AICBankedRegisters - Some(IOMapping::new(0x23b738000, false, 1, 0x1000, 0, true)), // PMGRScratch - None, // NIA Special agent idle register die 0 - None, // NIA Special agent idle register die 1 - None, // CRE registers - None, // Streaming codec registers - None, // - None, // - ], - sram_base: None, - sram_size: None, -}; diff --git a/drivers/gpu/drm/asahi-neo/hw/t8112.rs b/drivers/gpu/drm/asahi-neo/hw/t8112.rs deleted file mode 100644 index bac2757175f2c2..00000000000000 --- a/drivers/gpu/drm/asahi-neo/hw/t8112.rs +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware configuration for t8112 platforms (M2). - -use crate::f32; - -use super::*; - -pub(crate) const HWCONFIG: super::HwConfig = HwConfig { - chip_id: 0x8112, - gpu_gen: GpuGen::G14, - gpu_variant: GpuVariant::G, - gpu_core: GpuCore::G14G, - - base_clock_hz: 24_000_000, - uat_ias: 39, - uat_oas: 40, - num_dies: 1, - max_num_clusters: 1, - max_num_cores: 10, - max_num_frags: 10, - max_num_gps: 4, - - preempt1_size: 0x540, - preempt2_size: 0x280, - preempt3_size: 0x20, - compute_preempt1_size: 0x10000, // TODO: Check - clustering: None, - - render: HwRenderConfig { - // TODO: this is unused here, may be present in newer FW - tiling_control: 0xa041, - }, - - da: HwConfigA { - unk_87c: 900, - unk_8cc: 11000, - unk_e24: 125, - }, - db: HwConfigB { - unk_454: 1, - unk_4e0: 4, - unk_534: 0, - unk_ab8: 0x2048, - unk_abc: 0x4000, - unk_b30: 1, - }, - shared1_tab: &[ - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, - ], - shared1_a4: 0, - shared2_tab: &[-1, -1, -1, -1, -1, -1, -1, -1, 0xaa5aa, 0], - shared2_unk_508: 0xc00000, - shared2_curves: Some(HwConfigShared2Curves { - t1_coef: 7200, - t2: &[ - 0xf07, 0x4c0, 0x6c0, 0x8c0, 0xac0, 0xc40, 0xdc0, 0xec0, 0xf80, - ], - t3_coefs: &[0, 20, 28, 36, 44, 50, 56, 60, 63], - t3_scales: &[9, 3209, 10400], - }), - shared3_unk: 5, - shared3_tab: &[ - 10700, 10700, 10700, 10700, 10700, 6000, 1000, 1000, 1000, 10700, 10700, 10700, 10700, - 10700, 10700, 10700, - ], - idle_off_standby_timer_default: 0, - unk_hws2_4: None, - unk_hws2_24: 0, - global_unk_54: 0xffff, - - sram_k: f32!(1.02), - // 13.2: last coef changed from 6.6 to 5.3, assuming that was a fix we can backport - unk_coef_a: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], - unk_coef_b: &[&f32!([0.0, 0.0, 0.0, 0.0, 5.3, 0.0, 5.3, /*6.6*/ 5.3])], - global_tab: None, - has_csafr: false, - fast_sensor_mask: [0x6800, 0], - fast_sensor_mask_alt: [0x6800, 0], - fast_die0_sensor_present: 0x02, - io_mappings: &[ - Some(IOMapping::new(0x204d00000, false, 1, 0x14000, 0, true)), // Fender - Some(IOMapping::new(0x20e100000, false, 1, 0x4000, 0, false)), // AICTimer - Some(IOMapping::new(0x23b0c4000, false, 1, 0x4000, 0, true)), // AICSWInt - Some(IOMapping::new(0x204000000, false, 1, 0x20000, 0, true)), // RGX - None, // UVD - None, // unused - None, // DisplayUnderrunWA - Some(IOMapping::new(0x23b2c0000, false, 1, 0x1000, 0, false)), // AnalogTempSensorControllerRegs - None, // PMPDoorbell - Some(IOMapping::new(0x204d80000, false, 1, 0x8000, 0, true)), // MetrologySensorRegs - Some(IOMapping::new(0x204d61000, false, 1, 0x1000, 0, true)), // GMGIFAFRegs - Some(IOMapping::new(0x200000000, false, 1, 0xd6400, 0, true)), // MCache registers - None, // AICBankedRegisters - None, // PMGRScratch - None, // NIA Special agent idle register die 0 - None, // NIA Special agent idle register die 1 - Some(IOMapping::new(0x204e00000, false, 1, 0x10000, 0, true)), // CRE registers - Some(IOMapping::new(0x27d050000, false, 1, 0x4000, 0, true)), // Streaming codec registers - Some(IOMapping::new(0x23b3d0000, false, 1, 0x1000, 0, true)), // - Some(IOMapping::new(0x23b3c0000, false, 1, 0x1000, 0, false)), // - ], - sram_base: None, - sram_size: None, -}; diff --git a/drivers/gpu/drm/asahi-neo/hw/t8140.rs b/drivers/gpu/drm/asahi-neo/hw/t8140.rs deleted file mode 100644 index 9772d318ea819a..00000000000000 --- a/drivers/gpu/drm/asahi-neo/hw/t8140.rs +++ /dev/null @@ -1,403 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Hardware configuration for t8140 platforms (A18 Pro). - -use crate::g17::{ - Config, - FixedBuffer, - Identity, // -}; -use crate::hw::GpuVariant; - -/// Static configuration of the T8140 GPU. -pub(crate) const CONFIG: Config = Config { - chip_id: CHIP_ID, - identity: Identity { - family: 0xa, - variant: 0, - num_dies: 1, - gpu_variant: GpuVariant::P, - usc_generation: 3, - hal_generation: 200, - }, - uat_ias: 42, - uat_oas: 42, - // The secondary coprocessor's kernel root follows the primary one's 256 KiB of page tables. - secondary_root_offset: 0x40000, - // The coprocessors request their crash log buffers at these addresses, in this order, and - // expect them to be mapped before they start. - crash_buffers: [ - FixedBuffer { - va: 0x01e1_c000, - size: 0x8000, - }, - FixedBuffer { - va: 0x01f5_0000, - size: 0x8000, - }, - ], - // Read by the firmware through the kernel lower root, whose top-level entries it only - // samples when it starts. - pm_metrics: FixedBuffer { - va: 0x10_0008_0000, - size: 0x8000, - }, -}; - -use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; - -/// Chip identifier written to the hardware data. -pub(crate) const CHIP_ID: u32 = 0x8140; - -/// Base of the firmware MMIO aperture in the kernel window. -const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; - -const fn window( - slot: usize, - phys: u64, - offset: u64, - size: u32, - unk_18: u64, - unk_20: u32, -) -> RegisterWindow { - RegisterWindow { - slot, - phys, - va: MMIO_APERTURE + offset, - size, - unk_18, - unk_20, - } -} - -/// Register windows the firmware maps, by register-map slot. -pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 18] = [ - window(0, 0x3_0101_4000, 0x0_0000, 0x4000, 0, 2), - window(3, 0x2_2010_4000, 0x0_8000, 0x1_8000, 0, 2), - window(9, 0x3_003d_0000, 0x2_8000, 0x1000, 0, 2), - window(10, 0x3_003c_0000, 0x3_0000, 0x2000, 0, 0), - window(12, 0x4_0165_c000, 0x3_8000, 0x4000, 0, 2), - window(14, 0x3_0028_0000, 0x4_0000, 0x8000, 0, 0), - window(17, 0x4_8000_0000, 0x5_0000, 0x2_1400, 0, 2), - window(22, 0x4_8100_0000, 0x7_8000, 0x8000, 0, 2), - window(26, 0x4_80d0_4000, 0x8_8000, 0x8000, 0xd0_4000, 2), - window(27, 0x4_80d0_d000, 0x9_9000, 0x1000, 0xd0_d000, 2), - window(28, 0x4_80d5_8000, 0xa_0000, 0x8000, 0xd5_8000, 2), - window(29, 0x4_80d1_0000, 0xb_0000, 0x4000, 0xd1_0000, 2), - window(31, 0x4_80d4_0000, 0xb_8000, 0x4000, 0xd4_0000, 2), - window(32, 0x4_80d6_0000, 0xc_0000, 0x4000, 0xd6_0000, 2), - window(35, 0x4_80e0_0000, 0xc_8000, 0x4000, 0xe0_0000, 2), - window(39, 0x4_80e0_8000, 0xd_0000, 0x8000, 0, 2), - window(40, 0x4_80e1_c000, 0xe_0000, 0x4000, 0xe1_c000, 2), - window(41, 0x4_80e1_f800, 0xe_b800, 0x4000, 0, 2), -]; - -/// Register-map slots that are reserved but have no window. -pub(crate) const RESERVED_REGISTER_SLOTS: [usize; 17] = [ - 2, 5, 6, 7, 8, 30, 33, 34, 37, 38, 42, 43, 46, 47, 48, 49, 52, -]; - -/// Performance-table configuration. The device tree lists the operating -/// points in performance-state order; table A uses the odd states above -/// state 5, table B the even ones. -pub(crate) const PERF: PerfConfig = PerfConfig { - states_a: [0, 1, 2, 3, 4, 5, 7, 9, 11, 13, 15], - states_b: [0, 1, 2, 3, 4, 5, 6, 8, 10, 12, 14], - relative_a: [0, 15, 22, 29, 42, 56, 74, 86, 90, 92, 100], - relative_b: [0, 0, 13, 24, 40, 52, 63, 73, 83, 88, 100], - sram_min_mv: 765, -}; - -/// Operating points the performance tables reference. -pub(crate) const OPPS: usize = 16; - -const _: () = { - let mut index = 0; - while index < PERF_STATES { - assert!((PERF.states_a[index] as usize) < OPPS); - assert!((PERF.states_b[index] as usize) < OPPS); - index += 1; - } -}; - -/// Offsets of the fixed objects in the kernel window, in the order the -/// builder creates them. -pub(crate) mod kernel_window { - /// Private cluster (`initdata::private`). - pub(crate) const PRIVATE_CLUSTER: u64 = 0x2_0000; - /// Zeroed firmware-private page. - pub(crate) const ZERO_PAGE: u64 = 0x15d_8000; - /// Parameter-buffer descriptor table. - pub(crate) const PB_DESCRIPTORS: u64 = 0x15e_0000; - /// page-pool descriptor table. - pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x15e_8000; - /// Index object (64 KiB, `initdata::IndexTable` in its first page). - pub(crate) const INDEX: u64 = 0xc084_8000; - /// Firmware-control page. - pub(crate) const FWCTL: u64 = 0x1c_0000; - /// Render free-list control and state pages. - pub(crate) const RENDER_FREE_LIST: u64 = 0xc083_0000; - pub(crate) const RENDER_FREE_LIST_STATE: u64 = 0x160_8000; - /// Boot free-list control and state pages. - pub(crate) const BOOT_FREE_LIST: u64 = 0xc082_8000; - pub(crate) const BOOT_FREE_LIST_STATE: u64 = 0x160_0000; - /// Zeroed descriptor pages. - pub(crate) const DESCRIPTOR_PAGE_A: u64 = 0x1c_8000; - pub(crate) const DESCRIPTOR_PAGE_B: u64 = 0xc07c_0000; - /// Zeroed TA and 3D context pages (128 KiB each). - pub(crate) const TA_CONTEXT: u64 = 0x1d_8000; - pub(crate) const FRAGMENT_CONTEXT: u64 = 0x20_0000; - /// Compute free-list control page and the state page it names (the - /// state page is not mapped at boot). - pub(crate) const COMPUTE_FREE_LIST: u64 = 0xc086_8000; - pub(crate) const COMPUTE_FREE_LIST_STATE: u64 = 0x163_0000; - /// Second compute free-list control page and its state page. - pub(crate) const COMPUTE_READY_FREE_LIST: u64 = 0xc087_8000; - pub(crate) const COMPUTE_READY_FREE_LIST_STATE: u64 = 0x164_8000; - /// Further compute free-list state page. - pub(crate) const COMPUTE_STATE: u64 = 0x165_0000; - /// Hardware-data bundle (`initdata::bundle`). - pub(crate) const BUNDLE: u64 = 0xc078_8000; -} - -/// Arenas used by dynamically allocated firmware objects. -pub(crate) mod dynamic { - /// First dynamic byte, relative to the upper kernel window. - pub(crate) const KERNEL_OFFSET: u64 = 0x168_4000; - /// Client-private compute and render backing after the shared queue arena. - pub(crate) const CLIENT_LOWER: core::ops::Range = 0x70_8000_0000..0x71_0000_0000; - /// Canonical low mappings shared by hardware queues and their clients. - /// Client-private allocations start after this arena. - pub(crate) const LOWER: core::ops::Range = 0x70_0400_0000..0x70_8000_0000; -} - -/// Fixed addresses in the firmware's context-0 and context-1 address spaces. -pub(crate) mod low { - /// Context-0 alias of the parameter-buffer descriptor table. - pub(crate) const PB_DESCRIPTORS: u64 = 0x70_0183_8000; - /// Context-0 alias of the page-pool descriptor table. - pub(crate) const PAGE_POOL_DESCRIPTORS: u64 = 0x70_0184_0000; - /// Context-0 alias of the index object. - pub(crate) const INDEX: u64 = 0x10_0019_0000; - /// Context-0 aliases of the completion rings. - pub(crate) const COMPLETION_RINGS: [u64; 2] = [0x70_03f0_0000, 0x70_03f2_8000]; - /// Context-0 run list of the compute free list. - pub(crate) const COMPUTE_RUN_LIST: u64 = 0x70_03ad_8000; - - /// Render free list (context 1): page list. - pub(crate) const RENDER_PAGE_LIST: u64 = 0x70_0000_0000; - /// Size of the render page list. - pub(crate) const RENDER_PAGE_LIST_SIZE: usize = 0x20_0000; - /// Render free list: run list (64 KiB, zero at boot). - pub(crate) const RENDER_RUN_LIST: u64 = 0x70_0020_8000; - /// Size of the render run list. - pub(crate) const RENDER_RUN_LIST_SIZE: usize = 0x1_0000; - /// Render free list: first block. - pub(crate) const RENDER_BLOCKS: u64 = 0x70_0022_0000; - /// Distance between consecutive blocks. - pub(crate) const BLOCK_STRIDE: u64 = 0x10_8000; - /// Render blocks allocated at boot. - pub(crate) const RENDER_BLOCKS_ALLOCATED: usize = 28; - /// Render blocks mapped at boot. - pub(crate) const RENDER_BLOCKS_MAPPED: usize = 22; - /// Render blocks in the free list at boot. - pub(crate) const RENDER_BLOCKS_INITIAL: u32 = 17; - /// Render block holding the compute page list. - pub(crate) const COMPUTE_PAGE_LIST_BLOCK: usize = 22; - - /// Distance of the compute pool from the render pool. - const COMPUTE: u64 = 0x200_0000; - /// Compute pool: page list. - pub(crate) const COMPUTE_POOL_PAGE_LIST: u64 = RENDER_PAGE_LIST + COMPUTE; - /// Compute pool: run list. - pub(crate) const COMPUTE_POOL_RUN_LIST: u64 = RENDER_RUN_LIST + COMPUTE; - /// Compute pool: first block. - pub(crate) const COMPUTE_POOL_BLOCKS: u64 = RENDER_BLOCKS + COMPUTE; - /// Blocks the compute free list declares. - pub(crate) const COMPUTE_BLOCKS: u32 = 18; - /// Compute pool blocks in the second compute free list. - pub(crate) const COMPUTE_READY_BLOCKS: u32 = 17; - /// Page list of the compute free list (inside the block - /// [`COMPUTE_PAGE_LIST_BLOCK`] holds, relocated to the compute pool). - pub(crate) const COMPUTE_PAGE_LIST: u64 = - RENDER_BLOCKS + COMPUTE_PAGE_LIST_BLOCK as u64 * BLOCK_STRIDE + COMPUTE; - /// Two zeroed compute blocks. - pub(crate) const COMPUTE_ZERO_BLOCKS: [u64; 2] = [ - RENDER_PAGE_LIST + COMPUTE, - RENDER_PAGE_LIST + BLOCK_STRIDE + COMPUTE, - ]; - - /// Returns block `index` of the render pool. - pub(crate) const fn render_block(index: usize) -> u64 { - RENDER_BLOCKS + index as u64 * BLOCK_STRIDE - } - - /// Returns block `index` of the compute pool. - pub(crate) const fn compute_block(index: usize) -> u64 { - COMPUTE_POOL_BLOCKS + index as u64 * BLOCK_STRIDE - } - - /// Blocks of the compute free list, in run-list order: pool blocks 0 to - /// 17, the two zeroed blocks, then pool block 18. - pub(crate) const COMPUTE_FREE_LIST_BLOCKS: [u64; 21] = { - let mut blocks = [0; 21]; - let mut index = 0; - while index < 18 { - blocks[index] = compute_block(index); - index += 1; - } - blocks[18] = COMPUTE_ZERO_BLOCKS[0]; - blocks[19] = COMPUTE_ZERO_BLOCKS[1]; - blocks[20] = compute_block(18); - blocks - }; -} - -/// Client-root reservations and parameter-buffer geometry. -pub(crate) mod resources { - use core::ops::Range; - - pub(crate) const UAT_PAGE: u64 = 0x4000; - pub(crate) const USER_BASE: u64 = 0x10_0000_0000; - pub(crate) const TIMESTAMPS: Range = 0x70_0000_0000..0x71_0000_0000; - pub(crate) const COMMAND_TIMESTAMP_HZ: u64 = 24_000_000; - pub(crate) const USER_END: u64 = USER_BASE + (1 << 32); - pub(crate) const FIXED_PAIRS: usize = 2; - pub(crate) const FIXED_PAIR_STRIDE: u64 = 0x8_0000; - pub(crate) const SCENE: u64 = USER_BASE + 0x17_8000; - pub(crate) const SCENE_SIZE: u64 = 0x8000; - pub(crate) const DISCARD: u64 = USER_BASE + 0x18_0000; - pub(crate) const DISCARD_SIZE: u64 = 0x8000; - pub(crate) const PAGE_LIST: u64 = USER_BASE + 0x19_0000; - pub(crate) const FRAGMENT_STATUS: u64 = USER_BASE + 0x1a_8000; - pub(crate) const RENDER_ALIASES: Range = 0x10_0200_0000..0x10_0220_0000; - pub(crate) const TARGETS: Range = 0x10_5000_0000..0x10_6400_0000; - pub(crate) const TARGET_RECORD_MIN: usize = 0x4_0000; - pub(crate) const TARGET_RECORD_MAX: usize = 0x80_0000; - pub(crate) const TPC_MIN: usize = 0x3_0000; - pub(crate) const TPC_MAX: usize = 0x200_0000; - pub(crate) const AUXILIARY: Range = 0x100_0200_0000..0x100_0240_0000; - pub(crate) const METRICS_SIZE: u64 = 0x8000; - pub(crate) const TVB_PAGE: u64 = 0x8000; - pub(crate) const TVB_BLOCK_SIZE: usize = 0x2_0000; - pub(crate) const TVB_BLOCK_STRIDE: u64 = 0x2_8000; - pub(crate) const TVB_BLOCK_SLOTS: usize = 0xd1a; - pub(crate) const TVB_MAX_BLOCKS: usize = TVB_BLOCK_SLOTS - 1; - pub(crate) const TVB_PAGES_PER_BLOCK: usize = 4; - pub(crate) const TVB_PAGE_LIST_SIZE: usize = TVB_BLOCK_SLOTS * TVB_PAGES_PER_BLOCK * 4; - pub(crate) const TVB_INITIAL_PAGES: [u32; 32] = [ - 0x11, 0x16, 0x1b, 0x20, 0x25, 0x2a, 0x4a, 0x4f, 0x54, 0x59, 0x5e, 0x63, 0x68, 0x6d, 0x72, - 0x77, 0x7c, 0x81, 0x86, 0x8b, 0x90, 0x95, 0x9a, 0x9f, 0xa4, 0xa9, 0xae, 0xb3, 0xb8, 0xbd, - 0xc2, 0xc7, - ]; - pub(crate) const TVB_GROWTH: u64 = USER_BASE + 0x0400_0000; - pub(crate) const TVB_PAIR_STRIDE: u64 = 0x2800_0000; - pub(crate) const RESERVED_RANGES: usize = 48; - - /// Compact allocations search these shared client arenas in this order. - pub(crate) fn compact_ranges() -> [Range; 4] { - let second = TVB_GROWTH + TVB_PAIR_STRIDE; - [ - RENDER_ALIASES, - TVB_GROWTH - ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE, - second..second + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE, - TARGETS, - ] - } - - /// All driver-owned windows reserved before the first userspace bind. - /// Reserving them does not install pages or allocate backing memory. - pub(crate) fn client_reserved_ranges(metrics_va: u64) -> Option<[Range; RESERVED_RANGES]> { - let mut ranges = core::array::from_fn(|_| 0..0); - let mut index = 0; - let mut add = |range| { - ranges[index] = range; - index += 1; - }; - add(0..0x1_0000); - add(TIMESTAMPS); - let page_list_size = (TVB_PAGE_LIST_SIZE as u64 + UAT_PAGE - 1) & !(UAT_PAGE - 1); - for pair in 0..FIXED_PAIRS { - let offset = pair as u64 * FIXED_PAIR_STRIDE; - add(SCENE + offset..SCENE + offset + SCENE_SIZE); - add(DISCARD + offset..DISCARD + offset + DISCARD_SIZE); - add(PAGE_LIST + offset..PAGE_LIST + offset + page_list_size); - add(FRAGMENT_STATUS + offset..FRAGMENT_STATUS + offset + UAT_PAGE); - } - add(RENDER_ALIASES); - add(TARGETS); - for page in TVB_INITIAL_PAGES { - let start = USER_BASE + u64::from(page) * TVB_PAGE; - add(start..start + TVB_BLOCK_STRIDE); - } - add(TVB_GROWTH - ..TVB_GROWTH + (TVB_MAX_BLOCKS - TVB_INITIAL_PAGES.len()) as u64 * TVB_BLOCK_STRIDE); - for pair in 1..FIXED_PAIRS { - let start = TVB_GROWTH + pair as u64 * TVB_PAIR_STRIDE; - add(start..start + TVB_MAX_BLOCKS as u64 * TVB_BLOCK_STRIDE); - } - add(AUXILIARY); - add(metrics_va..metrics_va.checked_add(METRICS_SIZE)?); - Some(ranges) - } -} - -/// Canonical userspace resource and its presentation in an execution context. -#[derive(Clone, Copy)] -pub(crate) struct ContextView { - pub(crate) source: u64, - pub(crate) low: u64, - pub(crate) size: usize, - pub(crate) write: bool, -} - -const fn context_view(low: u64, size: usize, write: bool) -> ContextView { - ContextView { - source: 0x10_0000_0000 + low, - low, - size, - write, - } -} - -pub(crate) const CONTEXT_VIEWS: [ContextView; 7] = [ - context_view(0, 0x10000, false), - context_view(0x18000, 0x8000, false), - context_view(0x28000, 0x8000, false), - context_view(0x38000, 0x8000, false), - context_view(0x48000, 0x8000, false), - context_view(0x58000, 0x8000, true), - context_view(0x68000, 0xc000, true), -]; - -/// Scheduler accounting policy and its firmware clock domain. -pub(crate) mod qos { - pub(crate) const CLASS: u8 = 0x0f; - pub(crate) const SHARE: u32 = 0x2000; - pub(crate) const CLOCK_HZ: u64 = 24_000_000; -} - -/// Firmware scheduling profiles of the DRM queue priorities. Native user -/// queues use priority class 2 with policy 2. -pub(crate) mod scheduling { - use crate::g17::fw::queue::Policy; - - /// `DRM_ASAHI_NEO_PRIORITY_MEDIUM`. - pub(crate) const MEDIUM: Policy = Policy::new(2, 2, super::qos::CLASS, super::qos::SHARE); - /// `DRM_ASAHI_NEO_PRIORITY_LOW`. - pub(crate) const LOW: Policy = Policy::new(3, 2, super::qos::CLASS, super::qos::SHARE); -} - -/// Physical workqueue capacity and logical-owner admission limits. -pub(crate) mod queues { - pub(crate) const RENDER_SLOTS: usize = 64; - pub(crate) const RENDER_PAIRS_PER_OWNER: usize = 2; - pub(crate) const RENDER_DEPTH: u8 = 16; - pub(crate) const COMPUTE_DEPTH: usize = 16; -} - -/// Read-only recovery evidence, sampled only for selected halted slots. -pub(crate) mod recovery { - pub(crate) const SLOT_KEYS: u64 = 0x0000_0100_01ea_74b8; -} diff --git a/drivers/gpu/drm/asahi-neo/initdata.rs b/drivers/gpu/drm/asahi-neo/initdata.rs deleted file mode 100644 index 91a5bd73d97e9c..00000000000000 --- a/drivers/gpu/drm/asahi-neo/initdata.rs +++ /dev/null @@ -1,1025 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#![allow(clippy::unusual_byte_groupings)] - -//! GPU initialization data builder. -//! -//! The root of all interaction between the GPU firmware and the host driver is a complex set of -//! nested structures that we call InitData. This includes both GPU hardware/firmware configuration -//! and the pointers to the ring buffers and global data fields that are used for communication at -//! runtime. -//! -//! Many of these structures are poorly understood, so there are lots of hardcoded unknown values -//! derived from observing the InitData structures that macOS generates. - -use crate::f32; -use crate::fw::initdata::*; -use crate::fw::types::*; -use crate::module_parameters; -use crate::{ - driver::AsahiDevice, - gem, - gpu, - hw, - mmu, // -}; -use kernel::error::{ - Error, - Result, // -}; -use kernel::macros::versions; -use kernel::prelude::*; -use kernel::try_init; - -use ::pin_init; -use ::pin_init::Init; - -/// Builder helper for the global GPU InitData. -#[versions(AGX_NEO)] -pub(crate) struct InitDataBuilder<'a> { - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, -} - -#[versions(AGX_NEO)] -impl<'a> InitDataBuilder::ver<'a> { - /// Create a new InitData builder - pub(crate) fn new( - dev: &'a AsahiDevice, - alloc: &'a mut gpu::KernelAllocators, - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - ) -> InitDataBuilder::ver<'a> { - InitDataBuilder::ver { - dev, - alloc, - cfg, - dyncfg, - } - } - - /// Create the HwDataShared1 structure, which is used in two places in InitData. - fn hw_shared1(cfg: &'static hw::HwConfig) -> impl Init { - init!(raw::HwDataShared1 { - unk_a4: cfg.shared1_a4, - ..Zeroable::init_zeroed() - }) - .chain(|ret| { - for (i, val) in cfg.shared1_tab.iter().enumerate() { - ret.table[i] = *val; - } - Ok(()) - }) - } - - fn init_curve( - curve: &mut raw::HwDataShared2Curve, - unk_0: u32, - unk_4: u32, - t1: &[u16], - t2: &[i16], - t3: &[KVec], - ) { - curve.unk_0 = unk_0; - curve.unk_4 = unk_4; - (*curve.t1)[..t1.len()].copy_from_slice(t1); - (*curve.t1)[t1.len()..].fill(t1[0]); - (*curve.t2)[..t2.len()].copy_from_slice(t2); - (*curve.t2)[t2.len()..].fill(t2[0]); - for (i, a) in curve.t3.iter_mut().enumerate() { - a.fill(0x3ffffff); - if i < t3.len() { - let b = &t3[i]; - (**a)[..b.len()].copy_from_slice(b); - } - } - } - - /// Create the HwDataShared2 structure, which is used in two places in InitData. - fn hw_shared2( - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - ) -> impl Init + 'a { - try_init!(raw::HwDataShared2 { - unk_28: Array::new([0xff; 16]), - g14: Default::default(), - unk_508: cfg.shared2_unk_508, - ..Zeroable::init_zeroed() - }) - .chain(|ret| { - for (i, val) in cfg.shared2_tab.iter().enumerate() { - ret.table[i] = *val; - } - - let curve_cfg = match cfg.shared2_curves.as_ref() { - None => return Ok(()), - Some(a) => a, - }; - - let mut t1 = KVec::new(); - let mut t3 = KVec::new(); - - for _ in 0..curve_cfg.t3_scales.len() { - t3.push(KVec::new(), GFP_KERNEL)?; - } - - for (i, ps) in dyncfg.pwr.perf_states.iter().enumerate() { - let t3_coef = curve_cfg.t3_coefs[i]; - if t3_coef == 0 { - t1.push(0xffff, GFP_KERNEL)?; - for j in t3.iter_mut() { - j.push(0x3ffffff, GFP_KERNEL)?; - } - continue; - } - - let f_khz = (ps.freq_hz / 1000) as u64; - let v_max = ps.max_volt_mv() as u64; - - t1.push( - (1000000000 * (curve_cfg.t1_coef as u64) / (f_khz * v_max)) - .try_into() - .unwrap(), - GFP_KERNEL, - )?; - - for (j, scale) in curve_cfg.t3_scales.iter().enumerate() { - t3[j].push( - (t3_coef as u64 * 1000000100 * *scale as u64 / (f_khz * v_max * 6)) - .try_into() - .unwrap(), - GFP_KERNEL, - )?; - } - } - - ret.g14.unk_14 = 0x6000000; - Self::init_curve( - &mut ret.g14.curve1, - 0, - 0x20000000, - &[0xffff], - &[0x0f07], - &[], - ); - Self::init_curve(&mut ret.g14.curve2, 7, 0x80000000, &t1, curve_cfg.t2, &t3); - - Ok(()) - }) - } - - /// Create the HwDataShared3 structure, which is used in two places in InitData. - fn hw_shared3(cfg: &'static hw::HwConfig) -> impl Init { - pin_init::init_zeroed::().chain(|ret| { - if !cfg.shared3_tab.is_empty() { - ret.unk_0 = 1; - ret.unk_4 = 500; - ret.unk_8 = cfg.shared3_unk; - ret.table.copy_from_slice(cfg.shared3_tab); - ret.unk_4c = 1; - } - Ok(()) - }) - } - - /// Create an unknown T81xx-specific data structure. - fn t81xx_data( - cfg: &'static hw::HwConfig, - dyncfg: &'a hw::DynConfig, - ) -> impl Init { - let _perf_max_pstate = dyncfg.pwr.perf_max_pstate; - - pin_init::init_zeroed::().chain(move |_ret| { - match cfg.chip_id { - 0x8103 | 0x8112 => { - #[ver(V < V13_3)] - { - _ret.unk_d8c = 0x80000000; - _ret.unk_d90 = 4; - _ret.unk_d9c = f32!(0.6); - _ret.unk_da4 = f32!(0.4); - _ret.unk_dac = f32!(0.38552); - _ret.unk_db8 = f32!(65536.0); - _ret.unk_dbc = f32!(13.56); - _ret.max_pstate_scaled = 100 * _perf_max_pstate; - } - } - _ => (), - } - Ok(()) - }) - } - - /// Create the HwDataA structure. This mostly contains power-related configuration. - fn hwdata_a(&mut self) -> Result> { - let pwr = &self.dyncfg.pwr; - let period_ms = pwr.power_sample_period; - let period_s = F32::from(period_ms) / f32!(1000.0); - let ppm_filter_tc_periods = pwr.ppm_filter_time_constant_ms / period_ms; - #[ver(V >= V13_0B4)] - let ppm_filter_tc_ms_rounded = ppm_filter_tc_periods * period_ms; - let ppm_filter_a = f32!(1.0) / ppm_filter_tc_periods.into(); - let perf_filter_a = f32!(1.0) / pwr.perf_filter_time_constant.into(); - let perf_filter_a2 = f32!(1.0) / pwr.perf_filter_time_constant2.into(); - let avg_power_target_filter_a = f32!(1.0) / pwr.avg_power_target_filter_tc.into(); - let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; - #[ver(V >= V13_0B4)] - let avg_power_filter_tc_ms_rounded = avg_power_filter_tc_periods * period_ms; - let avg_power_filter_a = f32!(1.0) / avg_power_filter_tc_periods.into(); - let pwr_filter_a = f32!(1.0) / pwr.pwr_filter_time_constant.into(); - - let base_ps = pwr.perf_base_pstate; - let base_ps_scaled = 100 * base_ps; - let max_ps = pwr.perf_max_pstate; - let max_ps_scaled = 100 * max_ps; - let boost_ps_count = max_ps - base_ps; - - #[allow(unused_variables)] - let base_clock_khz = self.cfg.base_clock_hz / 1000; - let v_clocks_per_period = pwr.pwr_sample_period_aic_clks; - - #[allow(unused_variables)] - let clocks_per_period_coarse = self.cfg.base_clock_hz / 1000 * pwr.power_sample_period; - - self.alloc - .private - .new_init(pin_init::init_zeroed(), |_inner, _ptr| { - let cfg = &self.cfg; - let dyncfg = &self.dyncfg; - try_init!(raw::HwDataA::ver { - clocks_per_period: v_clocks_per_period, - #[ver(V >= V13_0B4)] - clocks_per_period_2: v_clocks_per_period, - pwr_status: AtomicU32::new(4), - unk_10: f32!(1.0), - actual_pstate: 1, - tgt_pstate: 1, - base_pstate_scaled: base_ps_scaled, - unk_40: 1, - max_pstate_scaled: max_ps_scaled, - min_pstate_scaled: 100, - unk_64c: 625, - pwr_filter_a_neg: f32!(1.0) - pwr_filter_a, - pwr_filter_a: pwr_filter_a, - pwr_integral_gain: pwr.pwr_integral_gain, - pwr_integral_min_clamp: pwr.pwr_integral_min_clamp.into(), - max_power_1: pwr.max_power_mw.into(), - pwr_proportional_gain: pwr.pwr_proportional_gain, - pwr_pstate_related_k: -F32::from(max_ps_scaled) / pwr.max_power_mw.into(), - pwr_pstate_max_dc_offset: pwr.pwr_min_duty_cycle as i32 - max_ps_scaled as i32, - max_pstate_scaled_2: max_ps_scaled, - max_power_2: pwr.max_power_mw, - max_pstate_scaled_3: max_ps_scaled, - ppm_filter_tc_periods_x4: ppm_filter_tc_periods * 4, - ppm_filter_a_neg: f32!(1.0) - ppm_filter_a, - ppm_filter_a: ppm_filter_a, - ppm_ki_dt: pwr.ppm_ki * period_s, - unk_6fc: f32!(65536.0), - ppm_kp: pwr.ppm_kp, - pwr_min_duty_cycle: pwr.pwr_min_duty_cycle, - max_pstate_scaled_4: max_ps_scaled, - unk_71c: f32!(0.0), - max_power_3: pwr.max_power_mw, - cur_power_mw_2: 0x0, - ppm_filter_tc_ms: pwr.ppm_filter_time_constant_ms, - #[ver(V >= V13_0B4)] - ppm_filter_tc_clks: ppm_filter_tc_ms_rounded * base_clock_khz, - perf_tgt_utilization: pwr.perf_tgt_utilization, - perf_boost_min_util: pwr.perf_boost_min_util, - perf_boost_ce_step: pwr.perf_boost_ce_step, - perf_reset_iters: pwr.perf_reset_iters, - unk_774: 6, - unk_778: 1, - perf_filter_drop_threshold: pwr.perf_filter_drop_threshold, - perf_filter_a_neg: f32!(1.0) - perf_filter_a, - perf_filter_a2_neg: f32!(1.0) - perf_filter_a2, - perf_filter_a: perf_filter_a, - perf_filter_a2: perf_filter_a2, - perf_ki: pwr.perf_integral_gain, - perf_ki2: pwr.perf_integral_gain2, - perf_integral_min_clamp: pwr.perf_integral_min_clamp.into(), - unk_79c: f32!(95.0), - perf_kp: pwr.perf_proportional_gain, - perf_kp2: pwr.perf_proportional_gain2, - boost_state_unk_k: F32::from(boost_ps_count) / f32!(0.95), - base_pstate_scaled_2: base_ps_scaled, - max_pstate_scaled_5: max_ps_scaled, - base_pstate_scaled_3: base_ps_scaled, - perf_tgt_utilization_2: pwr.perf_tgt_utilization, - base_pstate_scaled_4: base_ps_scaled, - unk_7fc: f32!(65536.0), - pwr_min_duty_cycle_2: pwr.pwr_min_duty_cycle.into(), - max_pstate_scaled_6: max_ps_scaled.into(), - max_freq_mhz: pwr.max_freq_mhz, - pwr_min_duty_cycle_3: pwr.pwr_min_duty_cycle, - min_pstate_scaled_4: f32!(100.0), - max_pstate_scaled_7: max_ps_scaled, - unk_alpha_neg: f32!(0.8), - unk_alpha: f32!(0.2), - fast_die0_sensor_mask: U64(cfg.fast_sensor_mask[0]), - #[ver(G >= G14X)] - fast_die1_sensor_mask: U64(cfg.fast_sensor_mask[1]), - fast_die0_release_temp_cc: 100 * pwr.fast_die0_release_temp, - unk_87c: cfg.da.unk_87c, - unk_880: 0x4, - unk_894: f32!(1.0), - - fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, - unk_8a8: f32!(65536.0), - fast_die0_kp: pwr.fast_die0_proportional_gain, - pwr_min_duty_cycle_4: pwr.pwr_min_duty_cycle, - max_pstate_scaled_8: max_ps_scaled, - max_pstate_scaled_9: max_ps_scaled, - fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, - unk_8cc: cfg.da.unk_8cc, - max_pstate_scaled_10: max_ps_scaled, - max_pstate_scaled_11: max_ps_scaled, - unk_c2c: 1, - power_zone_count: pwr.power_zones.len() as u32, - max_power_4: pwr.max_power_mw, - max_power_5: pwr.max_power_mw, - max_power_6: pwr.max_power_mw, - avg_power_target_filter_a_neg: f32!(1.0) - avg_power_target_filter_a, - avg_power_target_filter_a: avg_power_target_filter_a, - avg_power_target_filter_tc_x4: 4 * pwr.avg_power_target_filter_tc, - avg_power_target_filter_tc_xperiod: period_ms * pwr.avg_power_target_filter_tc, - #[ver(V >= V13_0B4)] - avg_power_target_filter_tc_clks: period_ms - * pwr.avg_power_target_filter_tc - * base_clock_khz, - avg_power_filter_tc_periods_x4: 4 * avg_power_filter_tc_periods, - avg_power_filter_a_neg: f32!(1.0) - avg_power_filter_a, - avg_power_filter_a: avg_power_filter_a, - avg_power_ki_dt: pwr.avg_power_ki_only * period_s, - unk_d20: f32!(65536.0), - avg_power_kp: pwr.avg_power_kp, - avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, - max_pstate_scaled_12: max_ps_scaled, - max_pstate_scaled_13: max_ps_scaled, - max_power_7: pwr.max_power_mw.into(), - max_power_8: pwr.max_power_mw, - avg_power_filter_tc_ms: pwr.avg_power_filter_tc_ms, - #[ver(V >= V13_0B4)] - avg_power_filter_tc_clks: avg_power_filter_tc_ms_rounded * base_clock_khz, - max_pstate_scaled_14: max_ps_scaled, - t81xx_data <- Self::t81xx_data(cfg, dyncfg), - #[ver(V >= V13_0B4)] - unk_e10_0 <- { - let filter_a = f32!(1.0) / pwr.se_filter_time_constant.into(); - let filter_1_a = f32!(1.0) / pwr.se_filter_time_constant_1.into(); - try_init!(raw::HwDataA130Extra { - unk_38: 4, - unk_3c: 8000, - gpu_se_inactive_threshold: pwr.se_inactive_threshold, - gpu_se_engagement_criteria: pwr.se_engagement_criteria, - gpu_se_reset_criteria: pwr.se_reset_criteria, - unk_54: 50, - unk_58: 0x1, - gpu_se_filter_a_neg: f32!(1.0) - filter_a, - gpu_se_filter_1_a_neg: f32!(1.0) - filter_1_a, - gpu_se_filter_a: filter_a, - gpu_se_filter_1_a: filter_1_a, - gpu_se_ki_dt: pwr.se_ki * period_s, - gpu_se_ki_1_dt: pwr.se_ki_1 * period_s, - unk_7c: f32!(65536.0), - gpu_se_kp: pwr.se_kp, - gpu_se_kp_1: pwr.se_kp_1, - - #[ver(V >= V13_3)] - unk_8c: 100, - #[ver(V < V13_3)] - unk_8c: 40, - - max_pstate_scaled_1: max_ps_scaled, - unk_9c: f32!(8000.0), - unk_a0: 1400, - gpu_se_filter_time_constant_ms: pwr.se_filter_time_constant * period_ms, - gpu_se_filter_time_constant_1_ms: pwr.se_filter_time_constant_1 - * period_ms, - gpu_se_filter_time_constant_clks: U64((pwr.se_filter_time_constant - * clocks_per_period_coarse) - .into()), - gpu_se_filter_time_constant_1_clks: U64((pwr - .se_filter_time_constant_1 - * clocks_per_period_coarse) - .into()), - unk_c4: f32!(65536.0), - unk_114: f32!(65536.0), - unk_124: 40, - max_pstate_scaled_2: max_ps_scaled, - ..Zeroable::init_zeroed() - }) - }, - fast_die0_sensor_mask_2: U64(cfg.fast_sensor_mask[0]), - #[ver(G >= G14X)] - fast_die1_sensor_mask_2: U64(cfg.fast_sensor_mask[1]), - unk_e24: cfg.da.unk_e24, - unk_e28: 1, - fast_die0_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[0]), - #[ver(G >= G14X)] - fast_die1_sensor_mask_alt: U64(cfg.fast_sensor_mask_alt[1]), - #[ver(V < V13_0B4)] - fast_die0_sensor_present: U64(cfg.fast_die0_sensor_present as u64), - unk_163c: 1, - unk_3644: 0, - hws1 <- Self::hw_shared1(cfg), - hws2 <- Self::hw_shared2(cfg, dyncfg), - hws3 <- Self::hw_shared3(cfg), - unk_3ce8: 1, - ..Zeroable::init_zeroed() - }) - .chain(|raw| { - for i in 0..self.dyncfg.pwr.perf_states.len() { - raw.sram_k[i] = self.cfg.sram_k; - } - - for (i, coef) in pwr.core_leak_coef.iter().enumerate() { - raw.core_leak_coef[i] = *coef; - } - - for (i, coef) in pwr.sram_leak_coef.iter().enumerate() { - raw.sram_leak_coef[i] = *coef; - } - - #[ver(V >= V13_0B4)] - if let Some(csafr) = pwr.csafr.as_ref() { - for (i, coef) in csafr.leak_coef_afr.iter().enumerate() { - raw.aux_leak_coef.cs_1[i] = *coef; - raw.aux_leak_coef.cs_2[i] = *coef; - } - - for (i, coef) in csafr.leak_coef_cs.iter().enumerate() { - raw.aux_leak_coef.afr_1[i] = *coef; - raw.aux_leak_coef.afr_2[i] = *coef; - } - } - - for i in 0..self.dyncfg.id.num_clusters as usize { - if let Some(coef_a) = self.cfg.unk_coef_a.get(i) { - (*raw.unk_coef_a1[i])[..coef_a.len()].copy_from_slice(coef_a); - (*raw.unk_coef_a2[i])[..coef_a.len()].copy_from_slice(coef_a); - } - if let Some(coef_b) = self.cfg.unk_coef_b.get(i) { - (*raw.unk_coef_b1[i])[..coef_b.len()].copy_from_slice(coef_b); - (*raw.unk_coef_b2[i])[..coef_b.len()].copy_from_slice(coef_b); - } - } - - for (i, pz) in pwr.power_zones.iter().enumerate() { - raw.power_zones[i].target = pz.target; - raw.power_zones[i].target_off = pz.target - pz.target_offset; - raw.power_zones[i].filter_tc_x4 = 4 * pz.filter_tc; - raw.power_zones[i].filter_tc_xperiod = period_ms * pz.filter_tc; - let filter_a = f32!(1.0) / pz.filter_tc.into(); - raw.power_zones[i].filter_a = filter_a; - raw.power_zones[i].filter_a_neg = f32!(1.0) - filter_a; - #[ver(V >= V13_0B4)] - raw.power_zones[i].unk_10 = 1320000000; - } - - #[ver(V >= V13_0B4 && G >= G14X)] - for (i, j) in raw.hws2.g14.curve2.t1.iter().enumerate() { - raw.unk_hws2[i] = if *j == 0xffff { 0 } else { j / 2 }; - } - - if !dyncfg.hw_data_b.is_empty() { - unsafe { - let mut matches: bool = true; - let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataA::ver as *const u8, - core::mem::size_of::(), - ); - if sla.len() != dyncfg.hw_data_a.len() { - matches = false; - dev_err!( - self.dev.as_ref(), - "!!! Hwdata A size mismatch: {} {}", - sla.len(), - dyncfg.hw_data_a.len(), - ); - } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_a.len()) { - if sla[i] != dyncfg.hw_data_a[i] { - matches = false; - dev_err!(self.dev.as_ref(), "!!! Hwdata A first mismatch: {i}"); - break; - } - } - if matches { - dev_info!(self.dev.as_ref(), "!!! Hwdata A match"); - } - } - } - - Ok(()) - }) - }) - } - - /// Create the HwDataB structure. This mostly contains GPU-related configuration. - fn hwdata_b(&mut self) -> Result> { - self.alloc - .private - .new_init(pin_init::init_zeroed(), |_inner, _ptr| { - let cfg = &self.cfg; - let dyncfg = &self.dyncfg; - try_init!(raw::HwDataB::ver { - // Userspace VA map related - #[ver(V < V13_0B4)] - unk_0: U64(0x13_00000000), - unk_8: U64(0x14_00000000), - #[ver(V < V13_0B4)] - unk_10: U64(0x1_00000000), - unk_18: U64(0xffc00000), - // USC start - unk_20: U64(0), // U64(0x11_00000000), - unk_28: U64(0), // U64(0x11_00000000), - // Unknown page - //unk_30: U64(0x6f_ffff8000), - unk_30: U64(mmu::UatGeometry { - ias: cfg.uat_ias, - oas: cfg.uat_oas, - } - .unk_page()), - timestamp_area_base: U64(gpu::IOVA_KERN_TIMESTAMP_RANGE.start), - // TODO: yuv matrices - chip_id: cfg.chip_id, - unk_454: cfg.db.unk_454, - unk_458: 0x1, - unk_460: 0x1, - unk_464: 0x1, - unk_468: 0x1, - unk_47c: 0x1, - unk_484: 0x1, - unk_48c: 0x1, - base_clock_khz: cfg.base_clock_hz / 1000, - power_sample_period: dyncfg.pwr.power_sample_period, - unk_49c: 0x1, - unk_4a0: 0x1, - unk_4a4: 0x1, - unk_4c0: 0x1f, - unk_4e0: U64(cfg.db.unk_4e0), - unk_4f0: 0x1, - unk_4f4: 0x1, - unk_504: 0x31, - unk_524: 0x1, // use_secure_cache_flush - unk_534: cfg.db.unk_534, - num_frags: dyncfg.id.num_frags * dyncfg.id.num_clusters, - unk_554: 0x1, - uat_ttb_base: U64(dyncfg.uat_ttb_base), - gpu_core_id: cfg.gpu_core as u32, - gpu_rev_id: dyncfg.id.gpu_rev_id as u32, - num_cores: dyncfg.id.num_cores * dyncfg.id.num_clusters, - max_pstate: dyncfg.pwr.perf_states.len() as u32 - 1, - #[ver(V < V13_0B4)] - num_pstates: dyncfg.pwr.perf_states.len() as u32, - #[ver(V < V13_0B4)] - min_sram_volt: dyncfg.pwr.min_sram_microvolt / 1000, - #[ver(V < V13_0B4)] - unk_ab8: cfg.db.unk_ab8, - #[ver(V < V13_0B4)] - unk_abc: cfg.db.unk_abc, - #[ver(V < V13_0B4)] - unk_ac0: 0x1020, - - #[ver(V >= V13_0B4)] - unk_ae4: Array::new([0x0, 0x3, 0x7, 0x7]), - #[ver(V < V13_0B4)] - unk_ae4: Array::new([0x0, 0xf, 0x3f, 0x3f]), - unk_b10: 0x1, - timer_offset: U64(0), - unk_b24: 0x1, - unk_b28: 0x1, - unk_b2c: 0x1, - unk_b30: cfg.db.unk_b30, - #[ver(V >= V13_0B4)] - unk_b38_0: 1, - #[ver(V >= V13_0B4)] - unk_b38_4: 1, - unk_b38: Array::new([0xffffffff; 12]), - #[ver(V >= V13_0B4 && V < V13_3)] - unk_c3c: 0x19, - #[ver(V >= V13_3)] - unk_c3c: 0x1a, - ..Zeroable::init_zeroed() - }) - .chain(|raw| { - #[ver(V >= V13_3)] - for i in 0..16 { - raw.unk_arr_0[i] = i as u32; - } - - let base_ps = self.dyncfg.pwr.perf_base_pstate as usize; - let max_ps = self.dyncfg.pwr.perf_max_pstate as usize; - let base_freq = self.dyncfg.pwr.perf_states[base_ps].freq_hz; - let max_freq = self.dyncfg.pwr.perf_states[max_ps].freq_hz; - - for (i, ps) in self.dyncfg.pwr.perf_states.iter().enumerate() { - raw.frequencies[i] = ps.freq_hz / 1000000; - for (j, mv) in ps.volt_mv.iter().enumerate() { - let sram_mv = (*mv).max(self.dyncfg.pwr.min_sram_microvolt / 1000); - raw.voltages[i][j] = *mv; - raw.voltages_sram[i][j] = sram_mv; - } - for j in ps.volt_mv.len()..raw.voltages[i].len() { - raw.voltages[i][j] = raw.voltages[i][0]; - raw.voltages_sram[i][j] = raw.voltages_sram[i][0]; - } - raw.sram_k[i] = self.cfg.sram_k; - raw.rel_max_powers[i] = ps.pwr_mw * 100 / self.dyncfg.pwr.max_power_mw; - raw.rel_boost_freqs[i] = if i > base_ps { - (ps.freq_hz - base_freq) / ((max_freq - base_freq) / 100) - } else { - 0 - }; - } - - #[ver(V >= V13_0B4)] - if let Some(csafr) = self.dyncfg.pwr.csafr.as_ref() { - let aux = &mut raw.aux_ps; - aux.cs_max_pstate = (csafr.perf_states_cs.len() - 1).try_into()?; - aux.afr_max_pstate = (csafr.perf_states_afr.len() - 1).try_into()?; - - for (i, ps) in csafr.perf_states_cs.iter().enumerate() { - aux.cs_frequencies[i] = ps.freq_hz / 1000000; - for (j, mv) in ps.volt_mv.iter().enumerate() { - let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); - aux.cs_voltages[i][j] = *mv; - aux.cs_voltages_sram[i][j] = sram_mv; - } - } - - for (i, ps) in csafr.perf_states_afr.iter().enumerate() { - aux.afr_frequencies[i] = ps.freq_hz / 1000000; - for (j, mv) in ps.volt_mv.iter().enumerate() { - let sram_mv = (*mv).max(csafr.min_sram_microvolt / 1000); - aux.afr_voltages[i][j] = *mv; - aux.afr_voltages_sram[i][j] = sram_mv; - } - } - } - - // Special case override for T602x - #[ver(G == G14X)] - if dyncfg.id.gpu_rev_id == hw::GpuRevisionID::B1 { - raw.gpu_rev_id = hw::GpuRevisionID::B0 as u32; - } - - if !dyncfg.hw_data_b.is_empty() { - unsafe { - let mut matches: bool = true; - let sla = core::slice::from_raw_parts( - raw as *const raw::HwDataB::ver as *const u8, - core::mem::size_of::(), - ); - if sla.len() != dyncfg.hw_data_b.len() { - matches = false; - dev_err!( - self.dev.as_ref(), - "!!! Hwdata B size mismatch: {} {}", - sla.len(), - dyncfg.hw_data_b.len(), - ); - } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_data_b.len()) { - if sla[i] != dyncfg.hw_data_b[i] { - matches = false; - dev_err!(self.dev.as_ref(), "!!! Hwdata B first mismatch: {i}"); - break; - } - } - if matches { - dev_info!(self.dev.as_ref(), "!!! Hwdata B match"); - } - } - } - - Ok(()) - }) - }) - } - - /// Create the Globals structure, which contains global firmware config including more power - /// configuration data and globals used to exchange state between the firmware and driver. - fn globals(&mut self) -> Result> { - self.alloc - .private - .new_init(pin_init::init_zeroed(), |_inner, _ptr| { - let cfg = &self.cfg; - let dyncfg = &self.dyncfg; - let pwr = &dyncfg.pwr; - let period_ms = pwr.power_sample_period; - let period_s = F32::from(period_ms) / f32!(1000.0); - let avg_power_filter_tc_periods = pwr.avg_power_filter_tc_ms / period_ms; - - let max_ps = pwr.perf_max_pstate; - let max_ps_scaled = 100 * max_ps; - - try_init!(raw::Globals::ver { - //ktrace_enable: 0xffffffff, - ktrace_enable: 0, - #[ver(V >= V13_2)] - unk_24_0: 3000, - unk_24: 0, - #[ver(V >= V13_0B4)] - debug: 0, - unk_28: 1, - #[ver(G >= G14X)] - unk_2c_0: 1, - #[ver(V >= V13_0B4 && G < G14X)] - unk_2c_0: 0, - unk_2c: 1, - unk_30: 0, - unk_34: 120, - // sub <- try_init!(raw::GlobalsSub::ver { - unk_54: cfg.global_unk_54, - unk_56: 40, - unk_58: 0xffff, - unk_5e: U32(1), - unk_66: U32(1), - // ..Zeroable::init_zeroed() - // }), - unk_8900: 1, - pending_submissions: AtomicU32::new(0), - max_power: pwr.max_power_mw, - max_pstate_scaled: max_ps_scaled, - max_pstate_scaled_2: max_ps_scaled, - max_pstate_scaled_3: max_ps_scaled, - power_zone_count: pwr.power_zones.len() as u32, - avg_power_filter_tc_periods: avg_power_filter_tc_periods, - avg_power_ki_dt: pwr.avg_power_ki_only * period_s, - avg_power_kp: pwr.avg_power_kp, - avg_power_min_duty_cycle: pwr.avg_power_min_duty_cycle, - avg_power_target_filter_tc: pwr.avg_power_target_filter_tc, - unk_89bc: cfg.da.unk_8cc, - fast_die0_release_temp: 100 * pwr.fast_die0_release_temp, - unk_89c4: cfg.da.unk_87c, - fast_die0_prop_tgt_delta: 100 * pwr.fast_die0_prop_tgt_delta, - fast_die0_kp: pwr.fast_die0_proportional_gain, - fast_die0_ki_dt: pwr.fast_die0_integral_gain * period_s, - unk_89e0: 1, - max_power_2: pwr.max_power_mw, - ppm_kp: pwr.ppm_kp, - ppm_ki_dt: pwr.ppm_ki * period_s, - #[ver(V >= V13_0B4)] - unk_89f4_8: 1, - unk_89f4: 0, - hws1 <- Self::hw_shared1(cfg), - hws2 <- Self::hw_shared2(cfg, dyncfg), - hws3 <- Self::hw_shared3(cfg), - #[ver(V >= V13_0B4)] - idle_off_standby_timer: pwr.idle_off_standby_timer, - #[ver(V >= V13_0B4)] - unk_hws2_4: cfg.unk_hws2_4.map(Array::new).unwrap_or_default(), - #[ver(V >= V13_0B4)] - unk_hws2_24: cfg.unk_hws2_24, - unk_900c: 1, - #[ver(V >= V13_0B4)] - unk_9010_0: 1, - #[ver(V >= V13_0B4)] - unk_903c: 1, - #[ver(V < V13_0B4)] - unk_903c: 0, - fault_control: *module_parameters::fault_control.value(), - do_init: 1, - progress_check_interval_3d: 40, - progress_check_interval_ta: 10, - progress_check_interval_cl: 250, - #[ver(V >= V13_0B4)] - unk_1102c_0: 1, - #[ver(V >= V13_0B4)] - unk_1102c_4: 1, - #[ver(V >= V13_0B4)] - unk_1102c_8: 100, - #[ver(V >= V13_0B4)] - unk_1102c_c: 1, - idle_off_delay_ms: AtomicU32::new(pwr.idle_off_delay_ms), - fender_idle_off_delay_ms: pwr.fender_idle_off_delay_ms, - fw_early_wake_timeout_ms: pwr.fw_early_wake_timeout_ms, - cl_context_switch_timeout_ms: 40, - #[ver(V >= V13_0B4)] - cl_kill_timeout_ms: 50, - #[ver(V >= V13_0B4)] - unk_11edc: 0, - #[ver(V >= V13_0B4)] - unk_11efc: 0, - ..Zeroable::init_zeroed() - }) - .chain(|raw| { - for (i, pz) in self.dyncfg.pwr.power_zones.iter().enumerate() { - raw.power_zones[i].target = pz.target; - raw.power_zones[i].target_off = pz.target - pz.target_offset; - raw.power_zones[i].filter_tc = pz.filter_tc; - } - - if let Some(tab) = self.cfg.global_tab.as_ref() { - for (i, x) in tab.iter().enumerate() { - raw.unk_118ec[i] = *x; - } - raw.unk_118e8 = 1; - } - - if !dyncfg.hw_globals.is_empty() { - unsafe { - let mut matches: bool = true; - let sla = core::slice::from_raw_parts( - raw as *const raw::Globals::ver as *const u8, - core::mem::size_of::(), - ); - if sla.len() != dyncfg.hw_globals.len() { - matches = false; - dev_err!( - self.dev.as_ref(), - "!!! Globals size mismatch: {} {}", - sla.len(), - dyncfg.hw_globals.len(), - ); - } - for i in 0..core::cmp::min(sla.len(), dyncfg.hw_globals.len()) { - if sla[i] != dyncfg.hw_globals[i] { - matches = false; - dev_err!(self.dev.as_ref(), "!!! Globals first mismatch: {i}"); - break; - } - } - if matches { - dev_info!(self.dev.as_ref(), "!!! Globals match"); - } - } - } - - Ok(()) - }) - }) - } - - /// Create the RuntimePointers structure, which contains pointers to most of the other - /// structures including the ring buffer channels, statistics structures, and HwDataA/HwDataB. - fn runtime_pointers(&mut self) -> Result> { - let hwa = self.hwdata_a()?; - let hwb = self.hwdata_b()?; - - let mut buffer_mgr_ctl = gem::new_kernel_object(self.dev, 0x4000)?; - buffer_mgr_ctl.vmap()?.memset(0); - - GpuObject::new_init_prealloc( - self.alloc.private.alloc_object()?, - |_ptr| { - let alloc = &mut *self.alloc; - try_init!(RuntimePointers::ver { - stats <- { - let alloc = &mut *alloc; - try_init!(Stats::ver { - vtx: alloc.private.new_default::()?, - frag: alloc.private.new_init( - pin_init::init_zeroed::(), - |_inner, _ptr| { - try_init!(raw::GpuGlobalStatsFrag::ver { - total_cmds: 0, - unk_4: 0, - stats: Default::default(), - }) - } - )?, - comp: alloc.private.new_default::()?, - }) - }, - - hwdata_a: hwa, - unkptr_190: alloc.private.array_empty_tagged(0x80, b"I190")?, - unkptr_198: alloc.private.array_empty_tagged(0xc0, b"I198")?, - hwdata_b: hwb, - - unkptr_1b8: alloc.private.array_empty_tagged(0x1000, b"I1B8")?, - unkptr_1c0: alloc.private.array_empty_tagged(0x300, b"I1C0")?, - unkptr_1c8: alloc.private.array_empty_tagged(0x1000, b"I1C8")?, - - buffer_mgr_ctl, - buffer_mgr_ctl_low_mapping: None, - buffer_mgr_ctl_high_mapping: None, - }) - }, - |inner, _ptr| { - try_init!(raw::RuntimePointers::ver { - pipes: Default::default(), - device_control: Default::default(), - event: Default::default(), - fw_log: Default::default(), - ktrace: Default::default(), - stats: Default::default(), - - stats_vtx: inner.stats.vtx.gpu_pointer(), - stats_frag: inner.stats.frag.gpu_pointer(), - stats_comp: inner.stats.comp.gpu_pointer(), - - hwdata_a: inner.hwdata_a.gpu_pointer(), - unkptr_190: inner.unkptr_190.gpu_pointer(), - unkptr_198: inner.unkptr_198.gpu_pointer(), - hwdata_b: inner.hwdata_b.gpu_pointer(), - hwdata_b_2: inner.hwdata_b.gpu_pointer(), - - fwlog_buf: None, - - unkptr_1b8: inner.unkptr_1b8.gpu_pointer(), - - #[ver(G < G14X)] - unkptr_1c0: inner.unkptr_1c0.gpu_pointer(), - #[ver(G < G14X)] - unkptr_1c8: inner.unkptr_1c8.gpu_pointer(), - - buffer_mgr_ctl_gpu_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_LOW), - buffer_mgr_ctl_fw_addr: U64(gpu::IOVA_KERN_GPU_BUFMGR_HIGH), - - __pad0: Default::default(), - unk_160: U64(0), - unk_168: U64(0), - unk_1d0: 0, - unk_1d4: 0, - unk_1d8: Default::default(), - - __pad1: Default::default(), - gpu_scratch: raw::RuntimeScratch::ver { - unk_6b38: 0xff, - ..Default::default() - }, - }) - }, - ) - } - - /// Create the FwStatus structure, which is used to coordinate the firmware halt state between - /// the firmware and the driver. - fn fw_status(&mut self) -> Result> { - self.alloc - .shared - .new_object(Default::default(), |_inner| Default::default()) - } - - /// Create one UatLevelInfo structure, which describes one level of translation for the UAT MMU. - fn uat_level_info( - cfg: &'static hw::HwConfig, - index_shift: usize, - num_entries: usize, - ) -> raw::UatLevelInfo { - raw::UatLevelInfo { - index_shift: index_shift as _, - unk_1: 14, - unk_2: 14, - unk_3: 8, - unk_4: 0x4000, - num_entries: num_entries as _, - unk_8: U64(1), - unk_10: U64(((1u64 << cfg.uat_oas) - 1) & !(mmu::UAT_PGMSK as u64)), - index_mask: U64(((num_entries - 1) << index_shift) as u64), - } - } - - /// Build the top-level InitData object. - #[inline(never)] - pub(crate) fn build(&mut self) -> Result>> { - let runtime_pointers = self.runtime_pointers()?; - let globals = self.globals()?; - let fw_status = self.fw_status()?; - let shared_ro = &mut self.alloc.shared_ro; - - let obj = self.alloc.private.new_init( - try_init!(InitData::ver { - unk_buf: shared_ro.array_empty_tagged(0x4000, b"IDTA")?, - runtime_pointers, - globals, - fw_status, - }), - |inner, _ptr| { - let cfg = &self.cfg; - try_init!(raw::InitData::ver { - #[ver(V == V13_5 && G != G14X)] - ver_info: Array::new([0x6ba0, 0x1f28, 0x601, 0xb0]), - #[ver(V == V13_5 && G == G14X)] - ver_info: Array::new([0xb390, 0x70f8, 0x601, 0xb0]), - unk_buf: inner.unk_buf.gpu_pointer(), - unk_8: 0, - unk_c: 0, - runtime_pointers: inner.runtime_pointers.gpu_pointer(), - globals: inner.globals.gpu_pointer(), - fw_status: inner.fw_status.gpu_pointer(), - uat_page_size: 0x4000, - uat_page_bits: 14, - uat_num_levels: 3, - uat_level_info: Array::new([ - Self::uat_level_info(cfg, 36, 8), - Self::uat_level_info(cfg, 25, 2048), - Self::uat_level_info(cfg, 14, 2048), - ]), - __pad0: Default::default(), - host_mapped_fw_allocations: 1, - unk_ac: 0, - unk_b0: 0, - unk_b4: 0, - unk_b8: 0, - }) - }, - )?; - Ok(KBox::new(obj, GFP_KERNEL)?) - } -} diff --git a/drivers/gpu/drm/asahi-neo/mem.rs b/drivers/gpu/drm/asahi-neo/mem.rs deleted file mode 100644 index 0e99ab33fd3fc4..00000000000000 --- a/drivers/gpu/drm/asahi-neo/mem.rs +++ /dev/null @@ -1,211 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! ARM64 low level memory operations. -//! -//! This GPU uses CPU-side `tlbi` outer-shareable instructions to manage its TLBs. -//! Yes, really. Even though the VA address spaces are unrelated. -//! -//! Right now we pick our own ASIDs and don't coordinate with the CPU. This might result -//! in needless TLB shootdowns on the CPU side... TODO: fix this. - -use core::arch::asm; -use core::cmp::min; - -use kernel::page::PAGE_SIZE; - -use crate::debug::*; -use crate::mmu; - -type Asid = u8; - -/// Invalidate the entire GPU TLB. -#[inline(always)] -pub(crate) fn tlbi_all() { - // SAFETY: tlbi is always safe by definition - unsafe { - asm!(".arch armv8.4-a", "tlbi vmalle1os",); - } -} - -/// Invalidate all TLB entries for a given ASID. -#[inline(always)] -pub(crate) fn tlbi_asid(asid: Asid) { - if debug_enabled(DebugFlags::ConservativeTlbi) { - tlbi_all(); - sync(); - return; - } - - // SAFETY: tlbi is always safe by definition - unsafe { - asm!( - ".arch armv8.4-a", - "tlbi aside1os, {x}", - x = in(reg) ((asid as u64) << 48) - ); - } -} - -/// Invalidate a single page for a given ASID. -#[inline(always)] -pub(crate) fn tlbi_page(asid: Asid, va: usize) { - if debug_enabled(DebugFlags::ConservativeTlbi) { - tlbi_all(); - sync(); - return; - } - - let val: u64 = ((asid as u64) << 48) | ((va as u64 >> 12) & 0xffffffffffc); - // SAFETY: tlbi is always safe by definition - unsafe { - asm!( - ".arch armv8.4-a", - "tlbi vae1os, {x}", - x = in(reg) val - ); - } -} - -/// Invalidate a range of pages for a given ASID. -#[inline(always)] -pub(crate) fn tlbi_range(asid: Asid, va: usize, len: usize) { - if debug_enabled(DebugFlags::ConservativeTlbi) { - tlbi_all(); - sync(); - return; - } - - if len == 0 { - return; - } - - let start_pg = va >> mmu::UAT_PGBIT; - let end_pg = (va + len + mmu::UAT_PGMSK) >> mmu::UAT_PGBIT; - - const BADDR_MASK: u64 = (1 << 37) - 1; - let mut val: u64 = ((asid as u64) << 48) | (2 << 46) | (start_pg as u64 & BADDR_MASK); - let pages = end_pg - start_pg; - - // Guess? It's possible that the page count is in terms of 4K pages - // when the CPU is in 4K mode... - #[cfg(CONFIG_ARM64_4K_PAGES)] - let pages = 4 * pages; - - if pages == 1 { - tlbi_page(asid, va); - return; - } - - // Page count is always in units of 2 - let num = ((pages + 1) >> 1) as u64; - // base: 5 bits - // exp: 2 bits - // pages = (base + 1) << (5 * exp + 1) - // 0:00000 -> 2 pages = 2 << 0 - // 0:11111 -> 32 * 2 pages = 2 << 5 - // 1:00000 -> 1 * 32 * 2 pages = 2 << 5 - // 1:11111 -> 32 * 32 * 2 pages = 2 << 10 - // 2:00000 -> 1 * 32 * 32 * 2 pages = 2 << 10 - // 2:11111 -> 32 * 32 * 32 * 2 pages = 2 << 15 - // 3:00000 -> 1 * 32 * 32 * 32 * 2 pages = 2 << 15 - // 3:11111 -> 32 * 32 * 32 * 32 * 2 pages = 2 << 20 - let exp = min(3, (64 - num.leading_zeros()) / 5); - let bits = 5 * exp; - let mut base = (num + (1 << bits) - 1) >> bits; - - val |= (exp as u64) << 44; - - while base > 32 { - // SAFETY: tlbi is always safe by definition - unsafe { - asm!( - ".arch armv8.4-a", - "tlbi rvae1os, {x}", - x = in(reg) val | (31 << 39) - ); - } - base -= 32; - // Each full operation covers 32 units of 2 << bits GPU granules. - // Advance only the base-address field, preserving the ASID and scale. - let next = (val & BADDR_MASK) + (32u64 << (bits + 1)); - val = (val & !BADDR_MASK) | (next & BADDR_MASK); - } - - // SAFETY: tlbi is always safe by definition - unsafe { - asm!( - ".arch armv8.4-a", - "tlbi rvae1os, {x}", - x = in(reg) val | ((base - 1) << 39) - ); - } -} - -/// Range length from which [`tlbi_range_or_asid`] invalidates the whole ASID. -const TLBI_RANGE_MAX: usize = 0x200_0000; -/// Range length from which [`tlbi_range_or_asid`] counts in 1 MiB units (scale 1) instead of -/// 32 KiB units (scale 0). -const TLBI_RANGE_SCALE1_MIN: usize = 0x10_0000; -/// Translation granule field value for 16 KiB pages. -const TLBI_TG_16K: u64 = 2; -/// Width of the base address field of a range TLBI operand. -const TLBI_RANGE_BADDR_MASK: u64 = (1 << 38) - 1; - -/// Invalidate a range of pages for a given ASID with a single range operation. -/// -/// Unlike [`tlbi_range`], which issues as many range operations as needed, this invalidates -/// ranges that do not fit one operation (and ranges starting at VA 0) by ASID. Single pages use a -/// page operation, at the CPU page size. -#[inline(always)] -pub(crate) fn tlbi_range_or_asid(asid: Asid, va: usize, len: usize) { - if debug_enabled(DebugFlags::ConservativeTlbi) { - tlbi_all(); - sync(); - return; - } - - if len == 0 { - return; - } - - if va == 0 || len >= TLBI_RANGE_MAX { - tlbi_asid(asid); - return; - } - if len <= PAGE_SIZE { - tlbi_page(asid, va); - return; - } - - // The range covers (NUM + 1) << (5 * SCALE + 1) granules of 16 KiB. - let (scale, unit_shift) = if len >= TLBI_RANGE_SCALE1_MIN { - (1u64, 20) - } else { - (0u64, 15) - }; - let num = ((len - 1) >> unit_shift) as u64; - - let val = ((asid as u64) << 48) - | (TLBI_TG_16K << 46) - | (scale << 44) - | (num << 39) - | (((va >> mmu::UAT_PGBIT) as u64) & TLBI_RANGE_BADDR_MASK); - - // SAFETY: tlbi is always safe by definition - unsafe { - asm!( - ".arch armv8.4-a", - "tlbi rvae1os, {x}", - x = in(reg) val - ); - } -} - -/// Issue a memory barrier (`dsb sy`). -#[inline(always)] -pub(crate) fn sync() { - // SAFETY: Barriers are always safe - unsafe { - asm!("dsb sy"); - } -} diff --git a/drivers/gpu/drm/asahi-neo/microseq.rs b/drivers/gpu/drm/asahi-neo/microseq.rs deleted file mode 100644 index cbdb5de62e9218..00000000000000 --- a/drivers/gpu/drm/asahi-neo/microseq.rs +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU Micro operation sequence builder -//! -//! As part of a single job submisssion to the GPU, the GPU firmware interprets a sequence of -//! commands that we call a "microsequence". These are responsible for setting up the job execution, -//! timestamping the process, waiting for completion, tearing up any resources, and signaling -//! completion to the driver via the event stamp mechanism. -//! -//! Although the microsequences used by the macOS driver are usually quite uniform and simple, the -//! firmware actually implements enough operations to make this interpreter Turing-complete (!). -//! Most of those aren't implemented yet, since we don't need them, but they could come in handy in -//! the future to do strange things or work around firmware bugs... -//! -//! This module simply implements a collection of microsequence operations that can be appended to -//! and later concatenated into one buffer, ready for firmware execution. - -use crate::fw::microseq; -pub(crate) use crate::fw::microseq::*; -use crate::fw::types::*; -use kernel::prelude::*; - -/// MicroSequence object type, which is just an opaque byte array. -pub(crate) type MicroSequence = GpuArray; - -/// MicroSequence builder. -pub(crate) struct Builder { - ops: KVec, -} - -impl Builder { - /// Create a new Builder object - pub(crate) fn new() -> Builder { - Builder { ops: KVec::new() } - } - - /// Get the relative offset from the current pointer to a given target offset. - /// - /// Used for relative jumps. - pub(crate) fn offset_to(&self, target: i32) -> i32 { - target - self.ops.len() as i32 - } - - /// Add an operation to the end of the sequence. - pub(crate) fn add(&mut self, op: T) -> Result { - let off = self.ops.len(); - let p: *const T = &op; - let p: *const u8 = p as *const u8; - // SAFETY: Microseq operations always have no padding bytes, so it is safe to - // access them as a byte slice. - let s: &[u8] = unsafe { core::slice::from_raw_parts(p, core::mem::size_of::()) }; - self.ops.extend_from_slice(s, GFP_KERNEL)?; - Ok(off as i32) - } - - /// Collect all submitted operations into a finalized GPU object. - pub(crate) fn build(self, alloc: &mut Allocator) -> Result { - let mut array = alloc.array_empty::(self.ops.len())?; - - array.as_mut_slice().clone_from_slice(self.ops.as_slice()); - Ok(array) - } -} diff --git a/drivers/gpu/drm/asahi-neo/mmu.rs b/drivers/gpu/drm/asahi-neo/mmu.rs deleted file mode 100644 index 86576e4d4dc15b..00000000000000 --- a/drivers/gpu/drm/asahi-neo/mmu.rs +++ /dev/null @@ -1,2675 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU UAT (MMU) management -//! -//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table -//! format. This module manages the global MMU structures, including a shared handoff structure -//! that is used to coordinate VM management operations with the firmware, the TTBAT which points -//! to currently active GPU VM contexts, as well as the individual `Vm` operations to map and -//! unmap buffer objects into a single user or kernel address space. -//! -//! The actual page table management is in the `pt` module. -//! -//! G17 firmware does not take part in the handoff protocol. Instead, each of its two coprocessor -//! instances publishes the top-level entries it owns into its own kernel root while it boots, and -//! TTBAT contexts other than a VM's bind slot may name a VM's root. See [`UatFirmware`]. - -use core::fmt::Debug; -use core::mem::size_of; -use core::num::NonZeroUsize; -use core::ops::Range; -use core::sync::atomic::{ - fence, - AtomicBool, - AtomicU32, - AtomicU64, - AtomicU8, - Ordering, // -}; - -use core::cmp; - -use kernel::{ - addr::PhysicalAddr, - bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, - c_str, - device, - drm_neo::{ - gem::shmem, - gpuvm, - mm, // - }, - error::Result, - io, - new_mutex, - prelude::*, - static_lock_class, - sync::{ - aref::ARef, - lock::{ - mutex::MutexBackend, - Guard, // - }, - Arc, - Mutex, // - }, - time::{ - delay::fsleep, - Delta, - Instant, - Monotonic, // - }, // -}; - -use crate::debug::*; -use crate::module_parameters; -use crate::no_debug; -use crate::{ - driver, - fw, - gem, - mem, - pgtable, - slotalloc, - util::RangeExt, // -}; - -// KernelMapping protection types -pub(crate) use crate::pgtable::Prot; -pub(crate) use pgtable::prot::*; -pub(crate) use pgtable::{ - UatPageTable, - UAT_PGBIT, - UAT_PGMSK, - UAT_PGSZ, - UAT_TOP_LEVEL_SHIFT, // -}; - -use pin_init; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; - -mod bind; -mod context; -mod shared; -mod lifetime; -mod residency; -pub(crate) use residency::{Gate as ResidencyGate, VmShrinker}; -pub(crate) use bind::PreparedUserBindBatch; -pub(crate) use lifetime::VmJobGuard; - -/// PPL magic number for the handoff region -const PPL_MAGIC: u64 = 0x4b1d000000000002; - -/// Number of supported context entries in the TTBAT -const UAT_NUM_CTX: usize = 64; - -/// Application contexts owned by independent logical queues. Contexts 0..4 -/// and 63 remain reserved for the firmware and device-global resources. -const EXECUTION_CONTEXT_MASK: u64 = ((1u64 << 63) - 1) & !((1u64 << 5) - 1); -/// Maximum simultaneously retained logical queue contexts. -pub(crate) const MAX_EXECUTION_CONTEXTS: u32 = EXECUTION_CONTEXT_MASK.count_ones(); -/// First context available for users -const UAT_USER_CTX_START: usize = 1; -/// Number of available user contexts -const UAT_USER_CTX: usize = UAT_NUM_CTX - UAT_USER_CTX_START; - -/// Lower/user base VA -pub(crate) const IOVA_USER_BASE: u64 = UAT_PGSZ as u64; - -/// Top-level page table entry of the upper half that holds the driver-managed kernel VA range. -const IOVA_KERN_TOP_LEVEL_INDEX: u64 = 2; - -/// TTBAT context that starts out as an alias of the kernel context (with its own ASID) with -/// [`UatFirmware::PublishedRoots`]. -const KERNEL_ALIAS_CTX: usize = 1; - -/// How the GPU firmware shares UAT management with the driver. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub(crate) enum UatFirmware { - /// One coprocessor that publishes its magic number in the handoff region, arbitrates the - /// handoff lock with the driver, and acknowledges cache flush requests (G13/G14). - Handoff, - /// Two coprocessor instances that never touch the handoff region (G17). - /// - /// Each instance has its own kernel root and writes the top-level entries below and including - /// the driver-managed kernel window into it while it boots: the lower ones are private to the - /// instance, and the kernel window entry points at a table that the bootloader reserved. The - /// primary instance uses the `pagetables` root, the secondary one the root at - /// `secondary_root_offset` in the same region. TTBAT context 1 starts as an alias of the kernel - /// context. - PublishedRoots { - /// Offset of the secondary instance's kernel root in the `pagetables` region. - secondary_root_offset: u64, - }, -} - -const TTBR_VALID: u64 = 0x1; // BIT(0) -const TTBR_ASID_SHIFT: usize = 48; - -/// UAT address space geometry of a given SoC. -#[derive(Debug, Copy, Clone)] -pub(crate) struct UatGeometry { - /// Input address bits translated by each page table root. - pub(crate) ias: u32, - /// Output (physical) address bits. - pub(crate) oas: u32, -} - -impl UatGeometry { - /// Lower/user top VA - const fn user_top(&self) -> u64 { - 1 << self.ias - } - - /// Lower/user VA range - pub(crate) const fn user_range(&self) -> Range { - IOVA_USER_BASE..self.user_top() - } - - /// Address of a special dummy page? - pub(crate) const fn unk_page(&self) -> u64 { - self.user_top() - 2 * UAT_PGSZ as u64 - } - - /// User VA range excluding the unk page - pub(crate) const fn user_usable_range(&self) -> Range { - IOVA_USER_BASE..self.unk_page() - } - - /// Upper/kernel base VA - const fn upper_base(&self) -> u64 { - !(self.user_top() - 1) - } - - /// Driver-managed kernel VA range - pub(crate) const fn kernel_range(&self) -> Range { - let start = self.upper_base() + (IOVA_KERN_TOP_LEVEL_INDEX << UAT_TOP_LEVEL_SHIFT); - start..(start + (1 << UAT_TOP_LEVEL_SHIFT)) - } - - /// Full kernel VA range - #[cfg(CONFIG_DEV_COREDUMP)] - const fn upper_range(&self) -> Range { - self.upper_base()..(!UAT_PGMSK as u64) - } -} - -/// A pre-allocated memory region for UAT management -struct UatRegion { - base: PhysicalAddr, - map: io::mem::Mem, -} - -/// SAFETY: It's safe to share UAT region records across threads. -unsafe impl Send for UatRegion {} -/// SAFETY: It's safe to share UAT region records across threads. -unsafe impl Sync for UatRegion {} - -/// Handoff region flush info structure -#[repr(C)] -struct FlushInfo { - state: AtomicU64, - addr: AtomicU64, - size: AtomicU64, -} - -/// UAT Handoff region layout -#[repr(C)] -struct Handoff { - magic_ap: AtomicU64, - magic_fw: AtomicU64, - - lock_ap: AtomicU8, - lock_fw: AtomicU8, - // Implicit padding: 2 bytes - turn: AtomicU32, - cur_slot: AtomicU32, - // Implicit padding: 4 bytes - flush: [FlushInfo; UAT_NUM_CTX + 1], - - unk2: AtomicU8, - // Implicit padding: 7 bytes - unk3: AtomicU64, -} - -const HANDOFF_SIZE: usize = size_of::(); - -/// One VM slot in the TTBAT -#[repr(C)] -struct SlotTTBS { - ttb0: AtomicU64, - ttb1: AtomicU64, -} - -const SLOTS_SIZE: usize = UAT_NUM_CTX * size_of::(); - -// We need at least page 0 (ttb0) -const PAGETABLES_SIZE: usize = UAT_PGSZ; - -/// Lock-free visibility of the mapping epoch. Writers run under the GPUVM -/// exec lock; nested mutations keep cached validation unavailable until the -/// outermost mutation has finished its leaf writes and invalidations. -struct MappingEpoch { - visible: AtomicU64, - pending: AtomicU64, - writers: AtomicU32, -} - -impl MappingEpoch { - fn new() -> Self { - Self { - visible: AtomicU64::new(0), - pending: AtomicU64::new(0), - writers: AtomicU32::new(0), - } - } - fn snapshot(&self) -> Option { - let value = self.visible.load(Ordering::Acquire); - (value != u64::MAX).then_some(value) - } -} - -#[must_use = "retain until the mapping mutation and its invalidations finish"] -struct MappingMutation(Option>); - -impl Drop for MappingMutation { - fn drop(&mut self) { - if let Some(epoch) = self.0.as_ref() { - if epoch.writers.fetch_sub(1, Ordering::Relaxed) == 1 { - epoch - .visible - .store(epoch.pending.load(Ordering::Relaxed), Ordering::Release); - } - } - } -} - -/// Inner data for a Vm instance. This is reference-counted by the outer Vm object. -struct VmInner { - dev: driver::AsahiDevRef, - is_kernel: bool, - va_range: Range, - page_table: UatPageTable, - mapping_epoch: Option>, - epoch: Option, - mm: mm::Allocator<(), KernelMappingInner>, - uat_inner: Arc, - binding: Arc>, - id: u64, -} - -/// Slot binding-related inner data for a Vm instance. -struct VmBinding { - active_users: usize, - binding: Option>, - bind_token: Option, - ttb: u64, -} - -struct VmBoInner { - sgt: Option>, - sg_vec: Option)>>, -} - -/// Data associated with a VM <=> BO pairing -#[pin_data] -struct VmBo { - #[pin] - inner: Mutex, -} - -impl gpuvm::DriverGpuVmBo for VmBo { - fn new() -> impl PinInit { - pin_init!(VmBo { - inner <- new_mutex!(VmBoInner { - sgt: None, - sg_vec: None, - }, "VmBinding"), - }) - } -} - -/// Per-mapping metadata that survives backing eviction and GPUVA splits. -#[derive(Default, Clone)] -struct VmGpuVa { - prot: Option, -} -impl gpuvm::DriverGpuVa for VmGpuVa {} - -#[derive(Default)] -struct StepContext { - new_va: Option>>>, - prev_va: Option>>>, - next_va: Option>>>, - vm_bo: Option>>, - prot: Prot, -} - -impl gpuvm::DriverGpuVm for VmInner { - type Driver = driver::AsahiDriver; - type GpuVmBo = VmBo; - type GpuVa = VmGpuVa; - type StepContext = StepContext; - - fn step_map( - self: &mut gpuvm::UpdatingGpuVm<'_, Self>, - op: &mut gpuvm::OpMap, - ctx: &mut Self::StepContext, - ) -> Result { - let _mutation = self.mapping_mutation(); - let mut iova = op.addr(); - let mut left = op.range() as usize; - let mut offset = op.offset() as usize; - - let bo = ctx.vm_bo.as_ref().expect("step_map with no BO"); - - let one_page = op.flags().contains(gpuvm::GpuVaFlags::REPEAT); - - let mut do_map = |mut addr: usize, mut len: usize, offset: &mut usize| -> Result { - if left == 0 { - return Ok(false); - } - - if *offset > 0 { - let skip = len.min(*offset); - addr += skip; - len -= skip; - *offset -= skip; - } - if len == 0 { - return Ok(true); - } - assert!(*offset == 0); - - if one_page { - len = left; - } else { - len = len.min(left); - } - - mod_dev_dbg!( - self.dev, - "MMU: map: {:#x}:{:#x} -> {:#x} [OP={}]\n", - addr, - len, - iova, - one_page - ); - - if let Err(error) = self.page_table.map_pages( - iova..(iova + len as u64), - addr as PhysicalAddr, - ctx.prot, - one_page, - ) { - self.rollback_failed_map(op.addr(), op.range()); - return Err(error); - } - - left -= len; - iova += len as u64; - Ok(true) - }; - - let guard = bo.inner().inner.lock(); - if let Some(sg_vec) = guard.sg_vec.as_ref() { - let start_idx = sg_vec - .binary_search_by(|range| { - if range.0 > offset { - cmp::Ordering::Greater - } else if (range.0 + range.1.len()) <= offset { - cmp::Ordering::Less - } else { - cmp::Ordering::Equal - } - }) - .expect("sg_vec does not contain offset???"); - - offset -= sg_vec[start_idx].0 as usize; - - for cur in start_idx..sg_vec.len() { - let addr = sg_vec[cur].1.start as usize; - let len: usize = sg_vec[cur].1.len() as usize; - if do_map(addr, len, &mut offset)? == false { - break; - } - } - } else { - for range in guard.sgt.as_ref().expect("step_map with no SGT").iter() { - // TODO: proper DMA address/length handling - let addr = range.dma_address() as usize; - let len: usize = range.dma_len() as usize; - if do_map(addr, len, &mut offset)? == false { - break; - } - } - } - - let mut gpuva = ctx.new_va.take().expect("Multiple step_map calls"); - gpuva.as_mut().inner_mut().get_mut().prot = Some(ctx.prot); - - if op - .map_and_link_va( - self, - gpuva, - ctx.vm_bo.as_ref().expect("step_map with no BO"), - ) - .is_err() - { - dev_err!( - self.dev.as_ref(), - "map_and_link_va failed: {:#x} [{:#x}] -> {:#x}\n", - op.offset(), - op.range(), - op.addr() - ); - self.rollback_failed_map(op.addr(), op.range()); - return Err(EINVAL); - } - self.tlbi_contexts_naming_root(op.addr(), op.range() as usize); - Ok(()) - } - fn step_unmap( - self: &mut gpuvm::UpdatingGpuVm<'_, Self>, - op: &mut gpuvm::OpUnMap, - _ctx: &mut Self::StepContext, - ) -> Result { - let va = op.va().expect("step_unmap: missing VA"); - - self.unmap_gpuva(va)?; - - if op.unmap_and_unlink_va_defer().is_none() { - dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); - } - Ok(()) - } - fn step_remap( - self: &mut gpuvm::UpdatingGpuVm<'_, Self>, - op: &mut gpuvm::OpReMap, - vm_bo: &gpuvm::GpuVmBo, - ctx: &mut Self::StepContext, - ) -> Result { - let va = op.unmap().va().expect("No previous VA"); - let orig_addr = va.addr(); - let orig_range = va.range(); - let orig_prot = va.inner().prot.ok_or(EINVAL)?; - let orig_invalidated = va.flags().contains(gpuvm::GpuVaFlags::INVALIDATED); - - // Only unmap the hole between prev/next, if they exist - let unmap_start = if let Some(op) = op.prev_map() { - op.addr() + op.range() - } else { - orig_addr - }; - - let unmap_end = if let Some(op) = op.next_map() { - op.addr() - } else { - orig_addr + orig_range - }; - - mod_dev_dbg!( - self.dev, - "MMU: unmap for remap: {:#x}..{:#x} (from {:#x}:{:#x})\n", - unmap_start, - unmap_end, - orig_addr, - orig_range - ); - - let unmap_range = unmap_end - unmap_start; - - let _mutation = self.mapping_mutation(); - if orig_invalidated { - self.page_table.discard_partial_map(unmap_start..unmap_end)?; - } else { - self.page_table.unmap_pages(unmap_start..unmap_end)?; - } - - if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); - self.tlbi_range(asid as u8, unmap_start, unmap_range as usize); - mod_dev_dbg!( - self.dev, - "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", - asid, - unmap_start, - unmap_range, - ); - mem::sync(); - } - - self.tlbi_contexts_naming_root(unmap_start, unmap_range as usize); - - if op.unmap().unmap_and_unlink_va_defer().is_none() { - dev_err!(self.dev.as_ref(), "step_unmap: could not unlink gpuva"); - } - - if let Some(prev_op) = op.prev_map() { - let mut prev_gpuva = ctx - .prev_va - .take() - .expect("Multiple step_remap calls with prev_op"); - prev_gpuva.as_mut().inner_mut().get_mut().prot = Some(orig_prot); - if prev_op.map_and_link_va(self, prev_gpuva, vm_bo).is_err() { - dev_err!(self.dev.as_ref(), "step_remap: could not relink prev gpuva"); - return Err(EINVAL); - } - } - - if let Some(next_op) = op.next_map() { - let mut next_gpuva = ctx - .next_va - .take() - .expect("Multiple step_remap calls with next_op"); - next_gpuva.as_mut().inner_mut().get_mut().prot = Some(orig_prot); - if next_op.map_and_link_va(self, next_gpuva, vm_bo).is_err() { - dev_err!(self.dev.as_ref(), "step_remap: could not relink next gpuva"); - return Err(EINVAL); - } - } - - Ok(()) - } -} - -impl VmInner { - /// Remove hardware leaves for one complete user GPUVA without allocating. - fn unmap_gpuva(&mut self, va: &gpuvm::GpuVa) -> Result { - mod_dev_dbg!(self.dev, "MMU: unmap: {:#x}:{:#x}\n", va.addr(), va.range()); - - let _mutation = self.mapping_mutation(); - if va.flags().contains(gpuvm::GpuVaFlags::INVALIDATED) { - self.page_table.discard_partial_map(va.addr()..(va.addr() + va.range()))?; - } else { - self.page_table.unmap_pages(va.addr()..(va.addr() + va.range()))?; - } - - if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); - self.tlbi_range(asid as u8, va.addr(), va.range() as usize); - mod_dev_dbg!( - self.dev, - "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", - asid, - va.addr(), - va.range(), - ); - mem::sync(); - } - - self.tlbi_contexts_naming_root(va.addr(), va.range() as usize); - - Ok(()) - } - - /// A failed map has no GPUVA owner to retain its backing or remove its leaves. - /// GPUVM has already split or removed overlaps throughout this operation's range. - fn rollback_failed_map(&mut self, addr: u64, size: u64) { - // Include the current segment: map_pages can fail after writing some of its leaves. - // The checked GPUVM operation supplies an aligned range without overflow. - let result = self.page_table.discard_partial_map(addr..addr + size); - if self.uat_inner.firmware == UatFirmware::Handoff { - if let Some(asid) = self.slot() { - fence(Ordering::SeqCst); - self.tlbi_range(asid as u8, addr, size as usize); - mem::sync(); - } - } - self.tlbi_contexts_naming_root(addr, size as usize); - if let Err(error) = result { - dev_err!( - self.dev.as_ref(), - "Could not discard failed mapping: {:?}\n", - error - ); - } - } - - fn mapping_mutation(&mut self) -> MappingMutation { - let state = self.mapping_epoch.clone(); - if let Some(state) = state.as_ref() { - state.writers.fetch_add(1, Ordering::Relaxed); - state.visible.swap(u64::MAX, Ordering::AcqRel); - self.epoch = self.epoch.and_then(|epoch| epoch.checked_add(1)); - state - .pending - .store(self.epoch.unwrap_or(u64::MAX), Ordering::Relaxed); - } - MappingMutation(state) - } - - /// Returns the slot index, if this VM is bound. - fn slot(&self) -> Option { - if self.is_kernel { - // The GFX ASC does not care about the ASID. Pick an arbitrary one. - // TODO: This needs to be a persistently reserved ASID once we integrate - // with the ARM64 kernel ASID machinery to avoid overlap. - Some(0) - } else { - // We don't check whether we lost the slot, which could cause unnecessary - // invalidations against another Vm. However, this situation should be very - // rare (e.g. a Vm lost its slot, which means 63 other Vms bound in the - // interim, and then it gets killed / drops its mappings without doing any - // final rendering). Anything doing active maps/unmaps is probably also - // rendering and therefore likely bound. - self.binding - .lock() - .bind_token - .as_ref() - .map(|token| token.last_slot() + UAT_USER_CTX_START as u32) - } - } - - /// Returns the translation table base for this Vm - fn ttb(&self) -> u64 { - self.page_table.ttb() - } - - /// Invalidate a VA range of this Vm's address space in the TLB of one ASID. - fn tlbi_range(&self, asid: u8, iova: u64, size: usize) { - match self.uat_inner.firmware { - UatFirmware::Handoff => mem::tlbi_range(asid, iova as usize, size), - UatFirmware::PublishedRoots { .. } => { - mem::tlbi_range_or_asid(asid, iova as usize, size) - } - } - } - - /// Invalidate a VA range in every TTBAT context that names this Vm's root as its lower half. - /// - /// With [`UatFirmware::PublishedRoots`], contexts other than a Vm's bind slot name its root - /// (the kernel lower root sits in contexts 0 and 1 and has no slot at all), so a mapping - /// change must reach all of them. Other firmware only uses the bind slot, which the callers - /// already invalidate. - fn tlbi_contexts_naming_root(&self, iova: u64, size: usize) { - if self.uat_inner.firmware == UatFirmware::Handoff { - return; - } - - let contexts = self.uat_inner.lock().contexts_naming_root(self.ttb()); - self.tlbi_context_mask(iova, size, contexts); - } - - fn tlbi_context_mask(&self, iova: u64, size: usize, contexts: u64) { - if contexts == 0 { - return; - } - - fence(Ordering::SeqCst); - for ctx in 0..UAT_NUM_CTX { - if contexts & (1 << ctx) != 0 { - mem::tlbi_range_or_asid(ctx as u8, iova as usize, size); - } - } - mem::sync(); - } - - /// Map an `mm::Node` representing an mapping in VA space. - fn map_node(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { - let end = node - .start() - .checked_add(node.mapped_size as u64) - .ok_or(EOVERFLOW)?; - let _mutation = self.mapping_mutation(); - self.page_table.prepare_map(node.start()..end)?; - let result = self.map_node_prepared(node, prot); - if result.is_err() { - self.rollback_failed_map(node.start(), node.mapped_size as u64); - } - result - } - - /// Install only into the empty range reserved by `map_node`. - fn map_node_prepared(&mut self, node: &mm::Node<(), KernelMappingInner>, prot: Prot) -> Result { - let mut iova = node.start(); - let guard = node.bo.as_ref().ok_or(EINVAL)?.get()?.inner().inner.lock(); - let sgt = guard.sgt.as_ref().ok_or(EINVAL)?; - let mut offset = node.offset; - let mut left = node.mapped_size; - - for range in sgt.iter() { - if left == 0 { - break; - } - - // TODO: proper DMA address/length handling - let mut addr = range.dma_address() as usize; - let mut len: usize = range.dma_len() as usize; - - if (offset | addr | len | iova as usize) & UAT_PGMSK != 0 { - dev_err!( - self.dev.as_ref(), - "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", - addr, - len, - iova - ); - return Err(EINVAL); - } - - if offset > 0 { - let skip = len.min(offset); - addr += skip; - len -= skip; - offset -= skip; - } - - len = len.min(left); - - if len == 0 { - continue; - } - - mod_dev_dbg!( - self.dev, - "MMU: map: {:#x}:{:#x} -> {:#x}\n", - addr, - len, - iova - ); - - self.page_table.map_pages( - iova..(iova + len as u64), - addr as PhysicalAddr, - prot, - false, - )?; - - iova += len as u64; - left -= len; - } - - // A context naming this root may be live and hold a negative translation for the range. - self.tlbi_contexts_naming_root(node.start(), node.mapped_size); - Ok(()) - } -} - -/// Shared reference to a virtual memory address space ([`Vm`]). -#[derive(Clone)] -pub(crate) struct Vm { - id: u64, - mapping_epoch: Option>, - lifetime: Option>, - residency: Option>, - context_bindings: Option>, - shared_bindings: Option>, - status: Option>, - inner: ARef>, - dummy_obj: ARef, - binding: Arc>, -} -no_debug!(Vm); - -/// An application TTBAT identity retained by a logical queue and its jobs. -/// The root lease is dropped before the VM, so no page table can be freed -/// while an active context still names it. Installed firmware queues retain -/// this object through replacement retirement or processor stop; a consumed -/// closed-scheduler release may return its root identity while keeping storage. -pub(crate) struct ExecutionContext { - root: ExecutionRoot, - vm: Vm, -} - -struct ExecutionRoot { - released: AtomicBool, - inner: Arc, - id: u8, - generation: u8, - low: u64, - high: u64, -} - -impl ExecutionContext { - pub(crate) fn id(&self) -> u32 { - self.root.id.into() - } - pub(crate) fn generation(&self) -> u8 { - self.root.generation - } - pub(crate) fn vm(&self) -> &Vm { - &self.vm - } - - /// Called only after the closed scheduler's release was consumed and all - /// its commands retired. Installed storage may outlive this root lease. - pub(crate) fn release(&self) { - self.root.release(); - } - - pub(crate) fn is_current(&self) -> bool { - let inner = self.root.inner.lock(); - if self.root.released.load(Ordering::Acquire) { - return false; - } - let roots = &inner.ttbs()[usize::from(self.root.id)]; - roots.ttb0.load(Ordering::Acquire) == self.root.low - && roots.ttb1.load(Ordering::Acquire) == self.root.high - } -} - -impl ExecutionRoot { - fn release(&self) { - if self.released.swap(true, Ordering::AcqRel) { - return; - } - let id = usize::from(self.id); - let release = { - let inner = self.inner.lock(); - inner.handoff().lock(); - let roots = &inner.ttbs()[id]; - let low = roots.ttb0.load(Ordering::Acquire); - let high = roots.ttb1.load(Ordering::Acquire); - let owned = low == self.low && high == self.high; - if owned { - roots.ttb0.store(0, Ordering::Release); - roots.ttb1.store(0, Ordering::Release); - } - inner.handoff().unlock(); - owned || (low == 0 && high == 0) - }; - if release { - fence(Ordering::SeqCst); - mem::tlbi_asid(self.id); - mem::sync(); - self.inner.lock().execution_contexts &= !(1u64 << id); - } else { - pr_err!( - "MMU: application context {} changed owners while leased\n", - id - ); - } - } -} - -impl Drop for ExecutionRoot { - fn drop(&mut self) { - self.release(); - } -} - -/// Slot data for a [`Vm`] slot (nothing, we only care about the indices). -pub(crate) struct SlotInner(); - -impl slotalloc::SlotItem for SlotInner { - type Data = (); -} - -/// Represents a single user of a binding of a [`Vm`] to a slot. -/// -/// The number of users is counted, and the slot will be freed when it drops to 0. -#[derive(Debug)] -pub(crate) struct VmBind(Vm, u32); - -impl VmBind { - /// Returns the slot that this `Vm` is bound to. - pub(crate) fn slot(&self) -> u32 { - self.1 - } -} - -impl Drop for VmBind { - fn drop(&mut self) { - let mut binding = self.0.binding.lock(); - - assert_ne!(binding.active_users, 0); - binding.active_users -= 1; - mod_pr_debug!( - "MMU: slot {} active users {}\n", - self.1, - binding.active_users - ); - if binding.active_users == 0 { - binding.binding = None; - } - } -} - -impl Clone for VmBind { - fn clone(&self) -> VmBind { - let mut binding = self.0.binding.lock(); - - binding.active_users += 1; - mod_pr_debug!( - "MMU: slot {} active users {}\n", - self.1, - binding.active_users - ); - VmBind(self.0.clone(), self.1) - } -} - -/// A mapping BO reference released under the immediate-mode GPUVA lock. -/// Retain the owner until deferred cleanup completes, including its final BO. -struct MappingBo { - bo: Option>>, - owner: ARef>, -} -impl MappingBo { - fn new( - bo: ARef>, - owner: ARef>, - ) -> Self { - Self { - bo: Some(bo), - owner, - } - } - fn get(&self) -> Result<&gpuvm::GpuVmBo> { - self.bo.as_deref().ok_or(EINVAL) - } -} -impl Drop for MappingBo { - fn drop(&mut self) { - let Some(bo) = self.bo.take() else { - return; - }; - let raw = ARef::into_raw(bo); - // SAFETY: GpuVmBo is repr(C) with drm_gpuvm_bo first. into_raw - // transfers one live reference. Every Vm here uses IMMEDIATE_MODE; - // deferred put selects its GPUVA lock and retains cleanup ownership. - unsafe { - kernel::bindings::drm_gpuvm_bo_put_deferred( - raw.as_ptr().cast::(), - ) - }; - self.owner.bo_deferred_cleanup(); - } -} - -/// Inner data required for an object mapping into a [`Vm`]. -pub(crate) struct KernelMappingInner { - // Drop order matters: - // - Deferred-put the GpuVmBo under its immediate-mode GPUVA lock - // - Drop the GEM BO next, since BO free can take the resv lock itself - // - Drop the owner GpuVm last, since that again can take resv locks when the refcount drops to 0 - bo: Option, - _gem: Option, - owner: ARef>, - uat_inner: Arc, - prot: Prot, - offset: usize, - mapped_size: usize, -} - -/// Prepared CPU timestamp access; destruction happens after completion signalling. -pub(crate) struct TimestampMapping { - mapping: shmem::VMap, - offset: usize, -} - -impl TimestampMapping { - pub(crate) fn write(&self, value: u64) { - let map = self.mapping.get(); - for (index, byte) in value.to_le_bytes().into_iter().enumerate() { - // SAFETY: timestamp_mapping checked the complete eight-byte extent. - // The owned vmap pins its backing; byte atomics permit unaligned and - // overlapping timestamp destinations without mutable slice aliases. - let destination = unsafe { - &*map.as_mut_ptr().add(self.offset + index).cast::() - }; - destination.store(byte, Ordering::Relaxed); - } - } -} - -/// An object mapping into a [`Vm`], which reserves the address range from use by other mappings. -pub(crate) struct KernelMapping(mm::Node<(), KernelMappingInner>); - -impl KernelMapping { - /// Returns the IOVA base of this mapping - pub(crate) fn iova(&self) -> u64 { - self.0.start() - } - - /// Returns the size of this mapping in bytes - pub(crate) fn size(&self) -> usize { - self.0.mapped_size - } - - /// Returns the IOVA base of this mapping - pub(crate) fn iova_range(&self) -> Range { - self.0.start()..(self.0.start() + self.0.mapped_size as u64) - } - - /// Resolves a byte in this retained mapping without creating page tables. - pub(crate) fn translate_offset(&self, offset: usize) -> Result { - if offset >= self.size() { - return Err(EINVAL); - } - let address = self.iova().checked_add(offset as u64).ok_or(EOVERFLOW)?; - self.0.owner.exec_lock(None, false)?.page_table.translate_iova(address) - } - - /// Writes a userspace timestamp through the retained GEM mapping. Byte - /// stores support unaligned destinations and concurrent completion writes - /// without creating mutable aliases. The complete timestamp is not atomic. - /// Retain CPU access before this destination's completion fence is published. - pub(crate) fn timestamp_mapping(&self, offset: usize) -> Result { - use kernel::drm_neo::gem::BaseObject; - let gem = self.0._gem.as_ref().ok_or(EINVAL)?; - if offset.checked_add(size_of::()).ok_or(EOVERFLOW)? > self.size() { - return Err(ERANGE); - } - let start = self.0.offset.checked_add(offset).ok_or(EOVERFLOW)?; - let end = start.checked_add(size_of::()).ok_or(EOVERFLOW)?; - if end > gem.size() { - return Err(ERANGE); - } - Ok(TimestampMapping { mapping: gem.owned_vmap::()?, offset: start }) - } - - /// Gives a fresh kernel mapping a firmware-cached prefix, preserving its tail. - /// The result includes whether any page-table entry may have changed; after - /// mutation the caller must retain the mapping until processor stop even if - /// the readback fails. The original protection is retained for teardown. - /// - /// # Safety - /// The mapping must be unpublished. After mutation it must not be unmapped - /// while a firmware instance can retain a cached translation or contents. - pub(crate) unsafe fn set_fw_cached_prefix(&self, prefix: usize) -> (Result<[u64; 2]>, bool) { - const MAX_PAGES: usize = 32; - const ATTRIBUTE_SHIFT: u32 = 2; - const ATTRIBUTE_MASK: u64 = 7; - const UNCACHED: u64 = 2; - const CACHE_ATTRIBUTE_CHANGE: u64 = UNCACHED << ATTRIBUTE_SHIFT; - let mut mutated = false; - let result = (|| { - if self.0.uat_inner.firmware == UatFirmware::Handoff - || prefix == 0 - || prefix > self.size() - || (prefix | self.size()) & UAT_PGMSK != 0 - || self.size() > MAX_PAGES * UAT_PGSZ - { - return Err(EINVAL); - } - let mut owner = self.0.owner.exec_lock(None, false)?; - if !owner.is_kernel { - return Err(EINVAL); - } - let pages = self.size() / UAT_PGSZ; - let cached_pages = prefix / UAT_PGSZ; - let mut before = [0; MAX_PAGES]; - for (index, value) in before[..pages].iter_mut().enumerate() { - *value = owner - .page_table - .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; - if (*value >> ATTRIBUTE_SHIFT) & ATTRIBUTE_MASK != UNCACHED { - return Err(EINVAL); - } - } - let _mutation = owner.mapping_mutation(); - mutated = true; - owner.page_table.reprot_pages( - self.iova()..self.iova() + prefix as u64, - PROT_GPU_FW_PRIV_RW, - )?; - fence(Ordering::SeqCst); - mem::tlbi_all(); - mem::sync(); - let mut after = [0; MAX_PAGES]; - for (index, value) in after[..pages].iter_mut().enumerate() { - *value = owner - .page_table - .leaf_entry(self.iova() + (index * UAT_PGSZ) as u64)?; - let change = if index < cached_pages { - CACHE_ATTRIBUTE_CHANGE - } else { - 0 - }; - if before[index] ^ *value != change { - return Err(EFAULT); - } - } - Ok([after[0], after[cached_pages.min(pages - 1)]]) - })(); - (result, mutated) - } - - /// Remap a cached mapping as uncached, then synchronously flush that range of VAs from the - /// coprocessor cache. This is required to safely unmap cached/private mappings. - fn remap_uncached_and_flush(&mut self) { - let mut owner = self.0.owner.lock_inner(); - - mod_dev_dbg!( - owner.dev, - "MMU: remap as uncached {:#x}:{:#x}\n", - self.iova(), - self.size() - ); - - // Remap in-place as uncached. - // Do not try to unmap the guard page (-1) - let prot = self.0.prot.as_uncached(); - let _mutation = owner.mapping_mutation(); - if owner - .page_table - .reprot_pages(self.iova_range(), prot) - .is_err() - { - dev_err!( - owner.dev.as_ref(), - "MMU: remap {:#x}:{:#x} failed\n", - self.iova(), - self.size() - ); - } - fence(Ordering::SeqCst); - - // If we don't have (and have never had) a VM slot, just return - let slot = match owner.slot() { - None => return, - Some(slot) => slot, - }; - - let flush_slot = if owner.is_kernel { - // If this is a kernel mapping, always flush on index 64 - UAT_NUM_CTX as u32 - } else { - // Otherwise, check if this slot is the active one, otherwise return - // Also check that we actually own this slot - let ttb = owner.ttb() | TTBR_VALID | (slot as u64) << TTBR_ASID_SHIFT; - - let uat_inner = self.0.uat_inner.lock(); - uat_inner.handoff().lock(); - let cur_slot = uat_inner.handoff().current_slot(); - let ttb_cur = uat_inner.ttbs()[slot as usize].ttb0.load(Ordering::Relaxed); - uat_inner.handoff().unlock(); - if cur_slot == Some(slot) && ttb_cur == ttb { - slot - } else { - return; - } - }; - - // FIXME: There is a race here, though it'll probably never happen in practice. - // In theory, it's possible for the ASC to finish using our slot, whatever command - // it was processing to complete, the slot to be lost to another context, and the ASC - // to begin using it again with a different page table, thus faulting when it gets a - // flush request here. In practice, the chance of this happening is probably vanishingly - // small, as all 62 other slots would have to be recycled or in use before that slot can - // be reused, and the ASC using user contexts at all is very rare. - - // Still, the locking around UAT/Handoff/TTBs should probably be redesigned to better - // model the interactions with the firmware and avoid these races. - // Possibly TTB changes should be tied to slot locks: - - // Flush: - // - Can early check handoff here (no need to lock). - // If user slot and it doesn't match the active ASC slot, - // we can elide the flush as the ASC guarantees it flushes - // TLBs/caches when it switches context. We just need a - // barrier to ensure ordering. - // - Lock TTB slot - // - If user ctx: - // - Lock handoff AP-side - // - Lock handoff dekker - // - Check TTB & handoff cur ctx - // - Perform flush if necessary - // - This implies taking the fwring lock - // - // TTB change: - // - lock TTB slot - // - lock handoff AP-side - // - lock handoff dekker - // change TTB - - // Lock this flush slot, and write the range to it - let flush = self.0.uat_inner.lock_flush(flush_slot); - let pages = self.size() >> UAT_PGBIT; - flush.begin_flush(self.iova(), self.size() as u64); - if pages >= 0x10000 { - dev_err!( - owner.dev.as_ref(), - "MMU: Flush too big ({:#x} pages))\n", - pages - ); - } - - let cmd = fw::channels::FwCtlMsg { - addr: fw::types::U64(self.iova()), - unk_8: 0, - slot: flush_slot, - page_count: pages as u16, - unk_12: 2, // ? - }; - - // Tell the firmware to do a cache flush - let ret = match (*owner.dev).gpu.manager() { - Some(gpu) => gpu.fwctl(cmd), - None => Err(ENODEV), - }; - if let Err(e) = ret { - dev_err!( - owner.dev.as_ref(), - "MMU: ASC cache flush {:#x}:{:#x} failed (err: {:?})\n", - self.iova(), - self.size(), - e - ); - } - - // Finish the flush - flush.end_flush(); - - // Slot is unlocked here - } -} -no_debug!(KernelMapping); - -impl Drop for KernelMapping { - fn drop(&mut self) { - // This is the main unmap function for UAT mappings. - // The sequence of operations here is finicky, due to the interaction - // between cached GFX ASC mappings and the page tables. These mappings - // always have to be flushed from the cache before being unmapped. - - // For uncached mappings, just unmapping and flushing the TLB is sufficient. - - // For cached mappings, this is the required sequence: - // 1. Remap it as uncached - // 2. Flush the TLB range - // 3. If kernel VA mapping OR user VA mapping and handoff.current_slot() == slot: - // a. Take a lock for this slot - // b. Write the flush range to the right context slot in handoff area - // c. Issue a cache invalidation request via FwCtl queue - // d. Poll for completion via queue - // e. Check for completion flag in the handoff area - // f. Drop the lock - // 4. Unmap - // 5. Flush the TLB range again - - // Firmware that does not use the handoff protocol cannot be asked to flush its cache. - // Such firmware only caches kernel mappings that live until its coprocessors stop. - if self.0.prot.is_cached_noncoherent() && self.0.uat_inner.firmware == UatFirmware::Handoff - { - mod_pr_debug!( - "MMU: remap as uncached {:#x}:{:#x}\n", - self.iova(), - self.size() - ); - self.remap_uncached_and_flush(); - } - - // Only VM page tables/binding state are touched here. The shared - // reservation is sufficient and avoids fallible exec allocation in Drop. - let mut owner = self.0.owner.lock_inner(); - mod_dev_dbg!( - owner.dev, - "MMU: unmap {:#x}:{:#x}\n", - self.iova(), - self.size() - ); - - let _mutation = owner.mapping_mutation(); - if owner.page_table.unmap_pages(self.iova_range()).is_err() { - dev_err!( - owner.dev.as_ref(), - "MMU: unmap {:#x}:{:#x} failed\n", - self.iova(), - self.size() - ); - } - - if let Some(asid) = owner.slot() { - fence(Ordering::SeqCst); - owner.tlbi_range(asid as u8, self.iova(), self.size()); - mod_dev_dbg!( - owner.dev, - "MMU: flush range: asid={:#x} start={:#x} len={:#x}\n", - asid, - self.iova(), - self.size() - ); - mem::sync(); - } - - // The range returns to the VA allocator once this mapping is gone, so no context that - // names this root may keep a stale translation for it. - owner.tlbi_contexts_naming_root(self.iova(), self.size()); - } -} - -/// Shared UAT global data structures -struct UatShared { - kernel_ttb1: u64, - execution_contexts: u64, - context_generations: [u8; UAT_NUM_CTX], - /// Kernel root of the secondary firmware instance, if there is one. - secondary_ttb1: Option, - map_kernel_to_user: bool, - handoff_rgn: UatRegion, - ttbs_rgn: UatRegion, -} - -impl UatShared { - /// Returns the handoff region area - fn handoff(&self) -> &Handoff { - // SAFETY: pointer is non-null per the type invariant - unsafe { (self.handoff_rgn.map.ptr() as *mut Handoff).as_ref() }.unwrap() - } - - /// Returns the TTBAT area - fn ttbs(&self) -> &[SlotTTBS; UAT_NUM_CTX] { - // SAFETY: pointer is non-null per the type invariant - unsafe { (self.ttbs_rgn.map.ptr() as *mut [SlotTTBS; UAT_NUM_CTX]).as_ref() }.unwrap() - } - - /// Returns the mask of TTBAT contexts whose lower half is the root `ttb`. - fn contexts_naming_root(&self, ttb: u64) -> u64 { - self.ttbs() - .iter() - .enumerate() - .filter(|(ctx, slot)| slot.ttb0.load(Ordering::Acquire) == tagged_root(ttb, *ctx)) - .fold(0, |mask, (ctx, _)| mask | (1 << ctx)) - } -} - -/// Returns the TTBAT value naming root `ttb` in context `ctx`. -const fn tagged_root(ttb: u64, ctx: usize) -> u64 { - ttb | TTBR_VALID | (ctx as u64) << TTBR_ASID_SHIFT -} - -// SAFETY: Nothing here is unsafe to send across threads. -unsafe impl Send for UatShared {} - -/// Inner data for the top-level UAT instance. -#[pin_data] -struct UatInner { - firmware: UatFirmware, - #[pin] - shared: Mutex, - #[pin] - handoff_flush: [Mutex; UAT_NUM_CTX + 1], -} - -impl UatInner { - /// Take the lock on the shared data and return the guard. - fn lock(&self) -> Guard<'_, UatShared, MutexBackend> { - self.shared.lock() - } - - /// Take a lock on a handoff flush slot and return the guard. - fn lock_flush(&self, slot: u32) -> Guard<'_, HandoffFlush, MutexBackend> { - self.handoff_flush[slot as usize].lock() - } -} - -/// Top-level UAT manager object -pub(crate) struct Uat { - dev: driver::AsahiDevRef, - geometry: UatGeometry, - - inner: Arc, - slots: slotalloc::SlotAllocator, - - kernel_vm: Vm, - kernel_lower_vm: Vm, -} - -impl Handoff { - /// Lock the handoff region from firmware access - fn lock(&self) { - self.lock_ap.store(1, Ordering::Relaxed); - fence(Ordering::SeqCst); - - while self.lock_fw.load(Ordering::Relaxed) != 0 { - if self.turn.load(Ordering::Relaxed) != 0 { - self.lock_ap.store(0, Ordering::Relaxed); - while self.turn.load(Ordering::Relaxed) != 0 {} - self.lock_ap.store(1, Ordering::Relaxed); - fence(Ordering::SeqCst); - } - } - fence(Ordering::Acquire); - } - - /// Unlock the handoff region, allowing firmware access - fn unlock(&self) { - self.turn.store(1, Ordering::Relaxed); - self.lock_ap.store(0, Ordering::Release); - } - - /// Returns the current Vm slot mapped by the firmware for lower/unprivileged access, if any. - fn current_slot(&self) -> Option { - let slot = self.cur_slot.load(Ordering::Relaxed); - if slot == 0 || slot == u32::MAX { - None - } else { - Some(slot) - } - } - - /// Initialize the handoff region - fn init(&self, firmware: UatFirmware) -> Result { - self.magic_ap.store(PPL_MAGIC, Ordering::Relaxed); - self.cur_slot.store(0, Ordering::Relaxed); - self.unk3.store(0, Ordering::Relaxed); - fence(Ordering::SeqCst); - - if firmware == UatFirmware::Handoff { - self.wait_for_firmware()?; - } - - for i in 0..=UAT_NUM_CTX { - self.flush[i].state.store(0, Ordering::Relaxed); - self.flush[i].addr.store(0, Ordering::Relaxed); - self.flush[i].size.store(0, Ordering::Relaxed); - } - fence(Ordering::SeqCst); - Ok(()) - } - - /// Wait for the firmware to publish its half of the handoff region. - fn wait_for_firmware(&self) -> Result { - let start = Instant::::now(); - const TIMEOUT: Delta = Delta::from_millis(1000); - - self.lock(); - while start.elapsed() < TIMEOUT { - if self.magic_fw.load(Ordering::Relaxed) == PPL_MAGIC { - break; - } else { - self.unlock(); - fsleep(Delta::from_millis(10)); - self.lock(); - } - } - - if self.magic_fw.load(Ordering::Relaxed) != PPL_MAGIC { - self.unlock(); - pr_err!("Handoff: Failed to initialize (firmware not running?)\n"); - return Err(EIO); - } - - self.unlock(); - Ok(()) - } -} - -/// Represents a single flush info slot in the handoff region. -/// -/// # Invariants -/// The pointer is valid and there is no aliasing HandoffFlush instance. -struct HandoffFlush(*const FlushInfo); - -// SAFETY: These pointers are safe to send across threads. -unsafe impl Send for HandoffFlush {} - -impl HandoffFlush { - /// Set up a flush operation for the coprocessor - fn begin_flush(&self, start: u64, size: u64) { - // SAFETY: Per the type invariant, this is safe - let flush = unsafe { self.0.as_ref().unwrap() }; - - let state = flush.state.load(Ordering::Relaxed); - if state != 0 { - pr_err!("Handoff: expected flush state 0, got {}\n", state); - } - flush.addr.store(start, Ordering::Relaxed); - flush.size.store(size, Ordering::Relaxed); - flush.state.store(1, Ordering::Relaxed); - } - - /// Complete a flush operation for the coprocessor - fn end_flush(&self) { - // SAFETY: Per the type invariant, this is safe - let flush = unsafe { self.0.as_ref().unwrap() }; - let state = flush.state.load(Ordering::Relaxed); - if state != 2 { - pr_err!("Handoff: expected flush state 2, got {}\n", state); - } - flush.state.store(0, Ordering::Relaxed); - } -} - -impl Vm { - /// Create a new virtual memory address space - fn new( - dev: &driver::AsahiDevice, - uat_inner: Arc, - kernel_range: Range, - geometry: UatGeometry, - ttb: Option, - id: u64, - ) -> Result { - let dummy_obj = gem::new_kernel_object(dev, UAT_PGSZ)?; - let is_kernel = ttb.is_some(); - - let page_table = if let Some(ttb) = ttb { - UatPageTable::new_with_ttb( - ttb, - geometry.kernel_range(), - geometry.ias, - geometry.oas, - uat_inner.firmware != UatFirmware::Handoff, - )? - } else { - UatPageTable::new(geometry.ias, geometry.oas)? - }; - - let (va_range, gpuvm_range) = if is_kernel { - (geometry.kernel_range(), kernel_range.clone()) - } else { - let mut range = geometry.user_range(); - if uat_inner.firmware != UatFirmware::Handoff { - range.start = 0; - } - (range, geometry.user_usable_range()) - }; - - let mm = mm::Allocator::new(va_range.start, va_range.range(), ())?; - - let binding = Arc::pin_init( - new_mutex!( - VmBinding { - binding: None, - bind_token: None, - active_users: 0, - ttb: page_table.ttb(), - }, - "VmBinding", - ), - GFP_KERNEL, - )?; - - let mapping_epoch = if uat_inner.firmware != UatFirmware::Handoff { - Some(Arc::new(MappingEpoch::new(), GFP_KERNEL)?) - } else { - None - }; - let binding_clone = binding.clone(); - Ok(Vm { - id, - mapping_epoch: mapping_epoch.clone(), - lifetime: None, - residency: None, - context_bindings: None, - shared_bindings: None, - status: None, - dummy_obj: dummy_obj.gem.clone(), - inner: gpuvm::GpuVm::new( - c_str!("Asahi::GpuVm"), - // TODO: should we using DRM_GPUVM_RESV_PROTECTED as well? - drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE, - dev, - dummy_obj.gem.clone(), - gpuvm_range, - kernel_range, - init!(VmInner { - dev: dev.into(), - va_range, - is_kernel, - page_table, - mapping_epoch, - epoch: Some(0), - mm, - uat_inner, - binding: binding_clone, - id, - }), - )?, - binding, - }) - } - - /// Attach a status before sharing a newly created user VM with queues. - pub(crate) fn with_status(mut self) -> Result { - self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); - self.lifetime = Some(lifetime::VmLifetime::new()?); - self.residency = Some(residency::Gate::new()?); - self.context_bindings = Some(context::ContextBindings::new()?); - self.shared_bindings = Some(shared::SharedBindings::new()?); - Ok(self) - } - - /// Status shared by every queue using this VM, if enabled by its GPU. - pub(crate) fn status(&self) -> Option<&Arc> { - self.status.as_ref() - } - - fn covers_range_locked( - inner: &mut VmInner, - address: u64, - size: u64, - read: bool, - write: bool, - ) -> bool { - if size == 0 { - return false; - } - let Some(end) = address.checked_add(size) else { - return false; - }; - let mask = UAT_PGMSK as u64; - let start = address & !mask; - let end = if end & mask == 0 { - end - } else { - let Some(end) = (end | mask).checked_add(1) else { - return false; - }; - end - }; - if start < inner.va_range.start || end > inner.va_range.end { - return false; - } - inner - .page_table - .covers_range(start..end, read, write) - .unwrap_or(false) - } - - pub(crate) fn covers_range(&self, address: u64, size: u64, read: bool, write: bool) -> bool { - let Ok(mut inner) = self.inner.exec_lock(None, false) else { - return false; - }; - Self::covers_range_locked(&mut inner, address, size, read, write) - } - - /// Checks all writable sparse blocks and their guard coverage under one - /// GPUVM lock. A fully mapped guard is not a valid sparse allocation. - pub(crate) fn covers_sparse_blocks_with_guards( - &self, - addresses: &[u64], - block_size: u64, - stride: u64, - ) -> bool { - if stride <= block_size { - return false; - } - let Ok(mut inner) = self.inner.exec_lock(None, false) else { - return false; - }; - addresses.iter().all(|&address| { - let Some(guard) = address.checked_add(block_size) else { - return false; - }; - Self::covers_range_locked(&mut inner, address, block_size, true, true) - && !Self::covers_range_locked( - &mut inner, guard, stride - block_size, false, false, - ) - }) - } - - /// Checks all exact request tuples under one GPUVM exec lock. Access bit0 - /// requests reading and bit1 writing. The epoch names this same snapshot. - pub(crate) fn covers_ranges_batch(&self, ranges: &[(u64, u64, u8)]) -> (bool, Option) { - let Ok(mut inner) = self.inner.exec_lock(None, false) else { - return (false, None); - }; - let epoch = inner.epoch; - let covered = ranges.iter().all(|&(address, size, access)| { - Self::covers_range_locked(&mut inner, address, size, access & 1 != 0, access & 2 != 0) - }); - (covered, epoch) - } - - /// Returns None while a leaf mutation is in progress or after epoch overflow. - pub(crate) fn mapping_validation_epoch(&self) -> Option { - self.mapping_epoch.as_ref()?.snapshot() - } - - /// Get the translation table base for this Vm - fn ttb(&self) -> u64 { - self.binding.lock().ttb - } - - /// Map a GEM object (using its `SGTable`) into this Vm at a free address in a given range. - #[allow(clippy::too_many_arguments)] - pub(crate) fn map_in_range( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard: bool, - ) -> Result { - self.map_in_range_with_guard_size( - gem, - object_range, - alignment, - range, - prot, - if guard { UAT_PGSZ } else { 0 }, - ) - } - - /// Reserves a whole-page unmapped guard without adding it to the mapped backing. - pub(crate) fn map_in_range_with_guard_size( - &self, - gem: &gem::Object, - object_range: Range, - alignment: u64, - range: Range, - prot: Prot, - guard_size: usize, - ) -> Result { - use kernel::drm_neo::gem::BaseObject; - if guard_size % UAT_PGSZ != 0 { - return Err(EINVAL); - } - let size = object_range - .end - .checked_sub(object_range.start) - .ok_or(EINVAL)?; - if size == 0 || object_range.end > gem.size() { - return Err(EINVAL); - } - let reserved_size = size.checked_add(guard_size).ok_or(EOVERFLOW)?; - let _residency = self.enter_residency()?; - let sgt = gem.owned_sg_table()?; - let mut inner = self.inner.exec_lock(Some(gem), false)?; - let vm_bo = self.inner.obtain_bo(gem)?; - - let mut vm_bo_guard = vm_bo.inner().inner.lock(); - if vm_bo_guard.sgt.is_none() { - vm_bo_guard.sgt.replace(sgt); - } - core::mem::drop(vm_bo_guard); - - let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); - let uat_inner = inner.uat_inner.clone(); - // A failed reservation drops its payload while exec still owns GEM - // reservations. Attach BO/GEM references only after success, since - // their destructors may acquire the same reservation locks. - let result = inner.mm.insert_node_in_range( - KernelMappingInner { - owner: self.inner.clone(), - uat_inner, - prot, - bo: None, - _gem: None, - offset: object_range.start, - mapped_size: size, - }, - reserved_size as u64, - alignment, - 0, - range.start, - range.end, - mm::InsertMode::Best, - ); - let mut node = match result { - Ok(node) => node, - Err(error) => { - drop(inner); - return Err(error); - } - }; - { - let payload = node.as_mut().inner_mut(); - payload.bo = vm_bo.take(); - payload._gem = Some(gem::KernelMappingPin::new(gem)); - } - - let ret = inner.map_node(&node, prot); - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. - core::mem::drop(inner); - ret?; - Ok(KernelMapping(node)) - } - - /// Map a GEM object into this Vm at a specific address. - #[allow(clippy::too_many_arguments)] - pub(crate) fn map_at( - &self, - addr: u64, - size: usize, - gem: ARef, - prot: Prot, - guard: bool, - ) -> Result { - let reserved_size = size - .checked_add(if guard { UAT_PGSZ } else { 0 }) - .ok_or(EOVERFLOW)?; - let _residency = self.enter_residency()?; - let sgt = gem.owned_sg_table()?; - let mut inner = self.inner.exec_lock(Some(&gem), false)?; - - let vm_bo = self.inner.obtain_bo(&gem)?; - - let mut vm_bo_guard = vm_bo.inner().inner.lock(); - if vm_bo_guard.sgt.is_none() { - vm_bo_guard.sgt.replace(sgt); - } - core::mem::drop(vm_bo_guard); - - let mut vm_bo = Some(MappingBo::new(vm_bo, self.inner.clone())); - let uat_inner = inner.uat_inner.clone(); - // Keep reservation-locking destructors out of the allocator payload - // until reservation succeeds. - let result = inner.mm.reserve_node( - KernelMappingInner { - owner: self.inner.clone(), - uat_inner, - prot, - bo: None, - _gem: None, - offset: 0, - mapped_size: size, - }, - addr, - reserved_size as u64, - 0, - ); - let mut node = match result { - Ok(node) => node, - Err(error) => { - drop(inner); - return Err(error); - } - }; - { - let payload = node.as_mut().inner_mut(); - payload.bo = vm_bo.take(); - payload._gem = Some(gem::KernelMappingPin::new(&gem)); - } - - let ret = inner.map_node(&node, prot); - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. - core::mem::drop(inner); - ret?; - Ok(KernelMapping(node)) - } - - /// Map a range of a GEM object into this Vm using GPUVM. - #[allow(clippy::too_many_arguments)] - pub(crate) fn bind_object( - &self, - gem: &gem::Object, - addr: u64, - size: u64, - offset: u64, - prot: Prot, - single_page: bool, - ) -> Result { - let _residency = self.enter_residency_metadata()?; - gem::validate_vm_binding(gem, self)?; - self.wait_for_user_map_admission()?; - // Mapping needs a complete context - let mut ctx = StepContext { - new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - prot, - ..Default::default() - }; - - let vm_bo = self.inner.obtain_bo(gem)?; - { - let needs_sgt = vm_bo.inner().inner.lock().sgt.is_none(); - if needs_sgt { - // Pin without the BO mutex: alias construction can hold the GEM - // reservation while acquiring this mutex. - let sgt = gem.owned_sg_table()?; - let mut vm_bo_guard = vm_bo.inner().inner.lock(); - if vm_bo_guard.sgt.is_none() { - if vm_bo_guard.sg_vec.is_none() { - let mut sg_vec = KVVec::new(); - let mut offset = 0; - for range in sgt.iter() { - let addr = range.dma_address() as usize; - let len = range.dma_len() as usize; - sg_vec.push((offset, addr..(addr + len)), GFP_KERNEL)?; - offset += len; - } - vm_bo_guard.sg_vec.replace(sg_vec); - } - vm_bo_guard.sgt.replace(sgt); - } - // Release the mutex before a losing initializer drops its SG - // owner. The declaration order also preserves this on errors. - drop(vm_bo_guard); - } - } - - let mut inner = self.inner.exec_lock(Some(gem), true)?; - - // Preallocate the page tables, to fail early if we ENOMEM - inner.page_table.alloc_pages(addr..(addr + size))?; - - ctx.vm_bo = Some(vm_bo); - - if (addr | size | offset) & (UAT_PGMSK as u64) != 0 { - dev_err!( - inner.dev.as_ref(), - "MMU: Map step {:#x} [{:#x}] -> {:#x} is not page-aligned\n", - offset, - size, - addr - ); - return Err(EINVAL); - } - - let (flags, gem_range) = if single_page { - (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) - } else { - (gpuvm::GpuVaFlags::NONE, 0u32) - }; - - mod_dev_dbg!( - inner.dev, - "MMU: sm_map: {:#x} [{:#x}] -> {:#x}\n", - offset, - size, - addr - ); - let _commit = self.mapping_commit()?; - inner.sm_map(&mut ctx, addr, size, offset, gem_range, flags) - } - - /// Add a direct MMIO mapping to this Vm at a free address. - pub(crate) fn map_io( - &self, - iova: u64, - phys: usize, - size: usize, - prot: Prot, - ) -> Result { - let mut inner = self.inner.exec_lock(None, false)?; - - if (iova as usize | phys | size) & UAT_PGMSK != 0 { - dev_err!( - inner.dev.as_ref(), - "MMU: KernelMapping {:#x}:{:#x} -> {:#x} is not page-aligned\n", - phys, - size, - iova - ); - return Err(EINVAL); - } - - dev_info!( - inner.dev.as_ref(), - "MMU: IO map: {:#x}:{:#x} -> {:#x}\n", - phys, - size, - iova - ); - - let uat_inner = inner.uat_inner.clone(); - let node = inner.mm.reserve_node( - KernelMappingInner { - owner: self.inner.clone(), - uat_inner, - prot, - bo: None, - _gem: None, - offset: 0, - mapped_size: size, - }, - iova, - size as u64, - 0, - )?; - - let end = iova.checked_add(size as u64).ok_or(EOVERFLOW)?; - inner.page_table.prepare_map(iova..end)?; - let mutation = inner.mapping_mutation(); - let ret = inner.page_table.map_pages( - iova..end, - phys as PhysicalAddr, - prot, - false, - ); - if ret.is_err() { - inner.rollback_failed_map(iova, size as u64); - } - // Drop the exec_lock first, so that if map_node failed the - // KernelMappingInner destructur does not deadlock. - drop(mutation); - core::mem::drop(inner); - ret?; - Ok(KernelMapping(node)) - } - - /// Unmap everything in an address range. - pub(crate) fn unmap_range(&self, iova: u64, size: u64) -> Result { - if self.defer_unmap(iova, size)? { - return Ok(()); - } - self.unmap_range_commit(iova, size, true) - } - - fn unmap_prepared_cleanup(&self, unmap: &mut bind::PreparedUserUnmap) -> Result { - let mut inner = self.inner.lock_inner(); - // SAFETY: step_unmap/step_remap modify this VM's translations and - // immediate GPUVA metadata only, using the retained split nodes. They - // do not access external BO backing and defer all BO reference puts. - let result = unsafe { inner.sm_unmap_inner(&mut unmap.ctx, unmap.iova, unmap.size) }; - if result.is_ok() { - self.untrack_shared_range(unmap.iova, unmap.size); - } - drop(inner); - self.bo_deferred_cleanup(); - result - } - - fn unmap_range_commit(&self, iova: u64, size: u64, user: bool) -> Result { - let _residency = self.enter_retirement_residency(); - // Unmapping a range can only do a single split, so just preallocate - // the prev and next GpuVas - let mut ctx = StepContext { - prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - ..Default::default() - }; - - let mut inner = self.inner.exec_lock(None, false)?; - - mod_dev_dbg!(inner.dev, "MMU: sm_unmap: {:#x}:{:#x}\n", iova, size); - let _commit = if user { - match self.mapping_commit() { - Ok(guard) => guard, - Err(error) => { - drop(inner); - if self.defer_unmap(iova, size)? { - return Ok(()); - } - return Err(error); - } - } - } else { - None - }; - inner.sm_unmap(&mut ctx, iova, size)?; - self.untrack_shared_range(iova, size); - Ok(()) - } - - fn drop_mappings_now(&self, gem: &gem::Object) -> Result { - let _residency = self.enter_retirement_residency(); - // Removing whole mappings only does unmaps, so no preallocated VAs - let mut ctx = Default::default(); - - let inner = match self.inner.exec_lock(Some(gem), false) { - Err(error) if error == ENOMEM => return self.drop_mappings_low_memory(gem), - result => result?, - }; - - if let Some(bo) = self.inner.find_bo(gem) { - mod_dev_dbg!(inner.dev, "MMU: bo_unmap\n"); - let result = self.inner.bo_unmap(&mut ctx, &bo); - if result.is_ok() { - // The close callback or deferred object list retains this GEM, - // so clearing its candidate cannot release its final reference. - self.untrack_context_object(gem); - self.untrack_shared_object(gem); - } - mod_dev_dbg!(inner.dev, "MMU: bo_unmap done\n"); - // BO release can drain unrelated deferred GEMs, including on an - // unmap error. Always release reservations before dropping it. - core::mem::drop(inner); - core::mem::drop(bo); - if result == Err(ENOMEM) { - return self.drop_mappings_low_memory(gem); - } - result?; - } - - Ok(()) - } - - /// Preserve the indexed fast path normally; memory pressure must not lose - /// accepted object cleanup just because an exec or operation list failed. - fn drop_mappings_low_memory(&self, gem: &gem::Object) -> Result { - let mut inner = self.inner.lock_inner(); - let result = inner.unmap_object(gem, VmInner::unmap_gpuva); - if result.is_ok() { - self.untrack_context_object(gem); - self.untrack_shared_object(gem); - } - drop(inner); - self.bo_deferred_cleanup(); - result - } - - /// Returns the dummy GEM object used to hold the shared DMA reservation locks - pub(crate) fn get_resv_obj(&self) -> ARef { - self.dummy_obj.clone() - } - - /// Check whether an object is external to this GpuVm - pub(crate) fn is_extobj(&self, gem: &gem::Object) -> bool { - self.inner.is_extobj(gem) - } - - /// Check whether an object is external to this GpuVm - pub(crate) fn bo_deferred_cleanup(&self) { - self.inner.bo_deferred_cleanup() - } -} - -impl Drop for VmInner { - fn drop(&mut self) { - let mut binding = self.binding.lock(); - assert_eq!(binding.active_users, 0); - - mod_pr_debug!( - "VmInner::Drop [{}]: bind_token={:?}\n", - self.id, - binding.bind_token - ); - - // Make sure this VM is not mapped to a TTB if it was - if let Some(token) = binding.bind_token.take() { - let idx = (token.last_slot() as usize) + UAT_USER_CTX_START; - let ttb = self.ttb() | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - - let uat_inner = self.uat_inner.lock(); - uat_inner.handoff().lock(); - let handoff_cur = uat_inner.handoff().current_slot(); - let ttb_cur = uat_inner.ttbs()[idx].ttb0.load(Ordering::SeqCst); - let inval = ttb_cur == ttb; - if inval { - if handoff_cur == Some(idx as u32) { - pr_err!( - "VmInner::drop owning slot {}, but it is currently in use by the ASC?\n", - idx - ); - } - uat_inner.ttbs()[idx].ttb0.store(0, Ordering::SeqCst); - uat_inner.ttbs()[idx].ttb1.store(0, Ordering::SeqCst); - } - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); - - // In principle we dropped all the KernelMappings already, but we might as - // well play it safe and invalidate the whole ASID. - if inval { - mod_pr_debug!( - "VmInner::Drop [{}]: need inval for ASID {:#x}\n", - self.id, - idx - ); - mem::tlbi_asid(idx as u8); - mem::sync(); - } - } - } -} - -impl Uat { - /// Map a bootloader-preallocated memory region - fn map_region( - dev: &device::Device, - name: &CStr, - size: usize, - cached: bool, - ) -> Result { - let of_node = dev.of_node().ok_or(EINVAL)?; - let res = of_node.reserved_mem_region_to_resource_byname(name)?; - let base = res.start(); - let res_size = res.size().try_into()?; - - if size > res_size { - dev_err!( - dev, - "Region {} is too small (expected {}, got {})\n", - name, - size, - res_size - ); - return Err(ENOMEM); - } - - let flags = if cached { - io::mem::MemFlag::WB - } else { - io::mem::MemFlag::WC - }; - - // SAFETY: The safety of this operation hinges on the correctness of - // much of this file and also the `pgtable` module, so it is difficult - // to prove in a single safety comment. Such is life with raw GPU - // page table management... - let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }.inspect_err(|_| { - dev_err!(dev, "Failed to remap {} mem resource\n", name); - })?; - - Ok(UatRegion { base, map }) - } - - /// Returns a reference to the global kernel (upper half) `Vm` - pub(crate) fn kernel_vm(&self) -> &Vm { - &self.kernel_vm - } - - /// Returns a reference to the local kernel (lower half) `Vm` - pub(crate) fn kernel_lower_vm(&self) -> &Vm { - &self.kernel_lower_vm - } - - /// Installed descriptor and bind contexts rooted in this user VM. A graph - /// constructor checks this before changing a retained pool's mappings. - pub(crate) fn vm_context_mask(&self, vm: &Vm) -> Result { - if !matches!(self.inner.firmware, UatFirmware::PublishedRoots { .. }) { - return Err(EINVAL); - } - let root = { - let owner = vm.inner.exec_lock(None, false)?; - if owner.is_kernel || !Arc::ptr_eq(&owner.uat_inner, &self.inner) { - return Err(EINVAL); - } - owner.ttb() - }; - Ok(self.inner.lock().contexts_naming_root(root)) - } - - /// Publish a completed alias set before its descriptors become visible. - /// Match actual roots, including independently allocated execution contexts. - pub(crate) fn flush_vm_contexts(&self, vm: &Vm) -> Result { - let contexts = self.vm_context_mask(vm)?; - if contexts == 0 { - return Err(ENOENT); - } - fence(Ordering::SeqCst); - for context in 0..UAT_NUM_CTX { - if contexts & (1u64 << context) != 0 { - mem::tlbi_asid(context as u8); - } - } - mem::sync(); - Ok(contexts) - } - - #[cfg(CONFIG_DEV_COREDUMP)] - pub(crate) fn dump_kernel_pages(&self) -> Result> { - let mut inner = self.kernel_vm.inner.exec_lock(None, false)?; - inner.page_table.dump_pages(self.geometry.upper_range()) - } - - /// Returns the base physical address of the TTBAT region. - pub(crate) fn ttb_base(&self) -> u64 { - let inner = self.inner.lock(); - - inner.ttbs_rgn.base - } - - /// Binds a `Vm` to a slot, preferring the last used one. - pub(crate) fn bind(&self, vm: &Vm) -> Result { - let mut binding = vm.binding.lock(); - - if binding.binding.is_none() { - assert_eq!(binding.active_users, 0); - - let isolation = *module_parameters::robust_isolation.value() != 0; - - self.slots.set_limit(if isolation { - NonZeroUsize::new(1) - } else { - None - }); - - let slot = self.slots.get(binding.bind_token)?; - if slot.changed() { - mod_pr_debug!("Vm Bind [{}]: bind_token={:?}\n", vm.id, slot.token(),); - let idx = (slot.slot() as usize) + UAT_USER_CTX_START; - let ttb = binding.ttb | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT; - - let uat_inner = self.inner.lock(); - - let ttb1 = if uat_inner.map_kernel_to_user { - uat_inner.kernel_ttb1 | TTBR_VALID | (idx as u64) << TTBR_ASID_SHIFT - } else { - 0 - }; - - let ttbs = uat_inner.ttbs(); - uat_inner.handoff().lock(); - if uat_inner.handoff().current_slot() == Some(idx as u32) { - pr_err!( - "Vm::bind to slot {}, but it is currently in use by the ASC?\n", - idx - ); - } - ttbs[idx].ttb0.store(ttb, Ordering::Release); - ttbs[idx].ttb1.store(ttb1, Ordering::Release); - uat_inner.handoff().unlock(); - core::mem::drop(uat_inner); - - // Make sure all TLB entries from the previous owner of this ASID are gone - mem::tlbi_asid(idx as u8); - mem::sync(); - } - - binding.bind_token = Some(slot.token()); - binding.binding = Some(slot); - } - - binding.active_users += 1; - - let slot = binding.binding.as_ref().unwrap().slot() + UAT_USER_CTX_START as u32; - mod_pr_debug!("MMU: slot {} active users {}\n", slot, binding.active_users); - Ok(VmBind(vm.clone(), slot)) - } - - /// Allocates a stable application context without changing the VM bind slot. - /// Both roots and the generation remain unchanged for the lease's lifetime. - pub(crate) fn new_execution_context(&self, vm: &Vm) -> Result> { - if self.inner.firmware == UatFirmware::Handoff { - return Err(EINVAL); - } - { - let owner = vm.inner.exec_lock(None, false)?; - if owner.is_kernel - || !core::ptr::eq(Arc::as_ptr(&owner.uat_inner), Arc::as_ptr(&self.inner)) - { - return Err(EINVAL); - } - } - let (id, generation) = { - let mut inner = self.inner.lock(); - let free = !inner.execution_contexts & EXECUTION_CONTEXT_MASK; - if free == 0 { - return Err(ENOSPC); - } - let id = free.trailing_zeros() as usize; - inner.execution_contexts |= 1u64 << id; - inner.context_generations[id] = inner.context_generations[id].wrapping_add(1); - (id, inner.context_generations[id]) - }; - let low = tagged_root(vm.ttb(), id); - let high = tagged_root(self.kernel_vm.ttb(), id); - // Allocate the complete host owner before either TTBAT half is published. - // ExecutionRoot releases an empty claimed slot if allocation fails. - let context = Arc::new( - ExecutionContext { - root: ExecutionRoot { - released: AtomicBool::new(false), - inner: self.inner.clone(), - id: id as u8, - generation, - low, - high, - }, - vm: vm.clone(), - }, - GFP_KERNEL, - )?; - { - let inner = self.inner.lock(); - let roots = &inner.ttbs()[id]; - if roots.ttb0.load(Ordering::Acquire) != 0 || roots.ttb1.load(Ordering::Acquire) != 0 { - return Err(EBUSY); - } - roots.ttb0.store(low, Ordering::Release); - roots.ttb1.store(high, Ordering::Release); - } - fence(Ordering::SeqCst); - mem::tlbi_asid(id as u8); - mem::sync(); - Ok(context) - } - - /// Creates a new `Vm` linked to this UAT. - pub(crate) fn new_vm(&self, id: u64, kernel_range: Range) -> Result { - Vm::new( - &self.dev, - self.inner.clone(), - kernel_range, - self.geometry, - None, - id, - ) - } - - /// Returns the address space geometry of this UAT. - pub(crate) fn geometry(&self) -> UatGeometry { - self.geometry - } - - /// Adopts the top-level entries that the primary firmware instance published in the kernel - /// root while booting ([`UatFirmware::PublishedRoots`]). - /// - /// The entries below the kernel window belong to the firmware, and the kernel window entry - /// points at a table that the bootloader reserved, so the driver maps kernel objects beneath - /// it and never frees it. Entries above the kernel window do not belong to anyone and are - /// cleared. Must run after the primary instance booted and before anything is mapped into the - /// kernel `Vm`. - pub(crate) fn adopt_firmware_root_entries(&self) -> Result<[u64; 3]> { - let owned = IOVA_KERN_TOP_LEVEL_INDEX as usize + 1; - let mut snapshot = [0; 3]; - // SAFETY: The kernel root is the bootloader-reserved `pagetables` region, which outlives - // this `Uat`. - unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), UAT_PGSZ / size_of::(), |ptes| { - for (value, pte) in snapshot.iter_mut().zip(ptes) { - *value = pte.load(Ordering::Relaxed); - } - for pte in &ptes[owned..] { - pte.store(0, Ordering::Relaxed); - } - }) - }?; - - fence(Ordering::SeqCst); - mem::tlbi_all(); - mem::sync(); - Ok(snapshot) - } - - /// Checks that allocation under the shared subtree did not replace a firmware-owned entry. - pub(crate) fn confirm_firmware_root_entries(&self, snapshot: [u64; 3]) -> Result { - // SAFETY: The root is a bootloader-reserved page retained for this UAT's lifetime. - let matches = unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), snapshot.len(), |ptes| { - ptes.iter() - .zip(snapshot) - .all(|(pte, expected)| pte.load(Ordering::Relaxed) == expected) - }) - }?; - if !matches { - return Err(EIO); - } - Ok(()) - } - - /// Copies the kernel root's top-level entries from the kernel window up into the root of the - /// secondary firmware instance ([`UatFirmware::PublishedRoots`]), whose lower entries are - /// private to that instance. - pub(crate) fn mirror_secondary_root(&self) -> Result { - let secondary = self.inner.lock().secondary_ttb1.ok_or(EINVAL)?; - let count = UAT_PGSZ / size_of::(); - let first = IOVA_KERN_TOP_LEVEL_INDEX as usize; - - // SAFETY: Both roots are in the bootloader-reserved `pagetables` region, which outlives - // this `Uat`. - unsafe { - pgtable::with_root_entries(self.kernel_vm.ttb(), count, |primary| { - pgtable::with_root_entries(secondary, count, |secondary| { - for (src, dst) in primary.iter().zip(secondary).skip(first) { - let pte = src.load(Ordering::Relaxed); - if pte != 0 && dst.load(Ordering::Relaxed) != pte { - dst.store(pte, Ordering::Relaxed); - } - } - }) - }) - }??; - - fence(Ordering::SeqCst); - mem::tlbi_all(); - mem::sync(); - Ok(()) - } - - /// Activates the device-global render root in context 1 after the boot records retire. - /// Both contexts must still contain their original kernel roots. - pub(crate) fn activate_render_context(&self, vm: &Vm) -> Result { - let inner = self.inner.lock(); - inner.handoff().lock(); - let ttbs = inner.ttbs(); - let ready = [0, KERNEL_ALIAS_CTX].into_iter().all(|ctx| { - ttbs[ctx].ttb0.load(Ordering::Acquire) == tagged_root(self.kernel_lower_vm.ttb(), ctx) - && ttbs[ctx].ttb1.load(Ordering::Acquire) == tagged_root(self.kernel_vm.ttb(), ctx) - }); - if !ready { - inner.handoff().unlock(); - return Err(EIO); - } - ttbs[KERNEL_ALIAS_CTX] - .ttb0 - .store(tagged_root(vm.ttb(), KERNEL_ALIAS_CTX), Ordering::Release); - inner.handoff().unlock(); - core::mem::drop(inner); - fence(Ordering::SeqCst); - mem::tlbi_asid(KERNEL_ALIAS_CTX as u8); - mem::sync(); - Ok(()) - } - - /// Creates the reference-counted inner data for a new `Uat` instance. - #[inline(never)] - fn make_inner(dev: &driver::AsahiDevice, firmware: UatFirmware) -> Result> { - let handoff_rgn = Self::map_region(dev.as_ref(), c_str!("handoff"), HANDOFF_SIZE, true)?; - let ttbs_rgn = Self::map_region(dev.as_ref(), c_str!("ttbs"), SLOTS_SIZE, true)?; - - // SAFETY: The Handoff struct layout matches the firmware's view of memory at this address, - // and the region is at least large enough per the size specified above. - let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; - - dev_info!(dev.as_ref(), "MMU: Initializing kernel page table\n"); - - Arc::pin_init( - try_pin_init!(UatInner { - firmware, - handoff_flush <- pin_init::pin_init_array_from_fn(|i| { - new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") - }), - shared <- new_mutex!( - UatShared { - kernel_ttb1: 0, - execution_contexts: !EXECUTION_CONTEXT_MASK, - context_generations: [0; UAT_NUM_CTX], - secondary_ttb1: None, - map_kernel_to_user: false, - handoff_rgn, - ttbs_rgn, - }, - "uat_shared" - ), - }), - GFP_KERNEL, - ) - } - - /// Creates a new `Uat` instance for the given address space geometry and firmware. - #[inline(never)] - pub(crate) fn new( - dev: &driver::AsahiDevice, - geometry: UatGeometry, - map_kernel_to_user: bool, - firmware: UatFirmware, - ) -> Result { - dev_info!(dev.as_ref(), "MMU: Initializing...\n"); - - let inner = Self::make_inner(dev, firmware)?; - - let of_node = dev.as_ref().of_node().ok_or(EINVAL)?; - let res = of_node.reserved_mem_region_to_resource_byname(c_str!("pagetables"))?; - let ttb1 = res.start(); - let ttb1size: usize = res.size().try_into()?; - - let (pagetables_size, secondary_ttb1) = match firmware { - UatFirmware::Handoff => (PAGETABLES_SIZE, None), - UatFirmware::PublishedRoots { - secondary_root_offset, - } => ( - secondary_root_offset as usize + UAT_PGSZ, - Some(ttb1 + secondary_root_offset), - ), - }; - if ttb1size < pagetables_size { - dev_err!(dev.as_ref(), "MMU: Pagetables region is too small\n"); - return Err(ENOMEM); - } - - dev_info!(dev.as_ref(), "MMU: Creating kernel page tables\n"); - let kernel_lower_vm = - Vm::new(dev, inner.clone(), geometry.user_range(), geometry, None, 1)?; - let kernel_vm = Vm::new( - dev, - inner.clone(), - geometry.kernel_range(), - geometry, - Some(ttb1), - 0, - )?; - - dev_info!(dev.as_ref(), "MMU: Kernel page tables created\n"); - - let ttb0 = kernel_lower_vm.ttb(); - - let uat = Self { - dev: dev.into(), - geometry, - kernel_vm, - kernel_lower_vm, - inner, - slots: slotalloc::SlotAllocator::new( - UAT_USER_CTX as u32, - (), - |_inner, _slot| Some(SlotInner()), - c_str!("Uat::SlotAllocator"), - static_lock_class!(), - static_lock_class!(), - )?, - }; - - let mut inner = uat.inner.lock(); - - inner.map_kernel_to_user = map_kernel_to_user; - inner.kernel_ttb1 = ttb1; - inner.secondary_ttb1 = secondary_ttb1; - - inner.handoff().init(firmware)?; - - dev_info!(dev.as_ref(), "MMU: Initializing TTBs\n"); - - inner.handoff().lock(); - - let ttbs = inner.ttbs(); - - ttbs[0].ttb0.store(ttb0 | TTBR_VALID, Ordering::SeqCst); - ttbs[0].ttb1.store(ttb1 | TTBR_VALID, Ordering::SeqCst); - - for ctx in &ttbs[1..] { - ctx.ttb0.store(0, Ordering::Relaxed); - ctx.ttb1.store(0, Ordering::Relaxed); - } - - if secondary_ttb1.is_some() { - let alias = &ttbs[KERNEL_ALIAS_CTX]; - alias - .ttb0 - .store(tagged_root(ttb0, KERNEL_ALIAS_CTX), Ordering::SeqCst); - alias - .ttb1 - .store(tagged_root(ttb1, KERNEL_ALIAS_CTX), Ordering::SeqCst); - } - - inner.handoff().unlock(); - - core::mem::drop(inner); - - dev_info!(dev.as_ref(), "MMU: initialized\n"); - - Ok(uat) - } -} - -impl Drop for Uat { - fn drop(&mut self) { - // Make sure we flush the TLBs - fence(Ordering::SeqCst); - mem::tlbi_all(); - mem::sync(); - } -} diff --git a/drivers/gpu/drm/asahi-neo/mmu/bind.rs b/drivers/gpu/drm/asahi-neo/mmu/bind.rs deleted file mode 100644 index adbb609d5a88ce..00000000000000 --- a/drivers/gpu/drm/asahi-neo/mmu/bind.rs +++ /dev/null @@ -1,224 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Prepared userspace mapping transactions. Host callback resources are reserved for the -//! complete ioctl before its first visible mapping changes; leaf commits retain their -//! individual interruptible GPUVM reservation acquisition and quarantine admission check. - -use super::*; - -pub(crate) struct PreparedUserMap { - gem: ARef, - ctx: StepContext, - addr: u64, - size: u64, - offset: u64, - single_page: bool, - context: context::PreparedBinding, - _residency: Option, -} - -pub(crate) struct PreparedUserUnmap { - _residency: Option, - pub(super) iova: u64, - pub(super) size: u64, - pub(super) ctx: StepContext, -} - -// SAFETY: This owns a thread-safe residency lease, a numeric range and unused, -// unlinked split-node allocations; step_remap moves each node out of ctx before -// linking it, so a retained ctx never owns a linked GPUVA. -unsafe impl Send for PreparedUserUnmap {} - -pub(super) enum PreparedUserBindOp { - Map(PreparedUserMap), - Unmap(PreparedUserUnmap), -} - -pub(crate) struct PreparedUserBindBatch { - pub(super) ops: KVec, -} - -impl PreparedUserBindBatch { - pub(crate) fn new(capacity: usize) -> Result { - Ok(Self { - ops: KVec::with_capacity(capacity, GFP_KERNEL)?, - }) - } - pub(crate) fn push_map(&mut self, map: PreparedUserMap) -> Result { - Ok(self.ops.push(PreparedUserBindOp::Map(map), GFP_KERNEL)?) - } - pub(crate) fn push_unmap(&mut self, unmap: PreparedUserUnmap) -> Result { - Ok(self - .ops - .push(PreparedUserBindOp::Unmap(unmap), GFP_KERNEL)?) - } -} - -impl Vm { - /// Prepares host state without taking DMA reservations or changing any leaf. - pub(crate) fn prepare_bind_object( - &self, - gem: &gem::Object, - addr: u64, - size: u64, - offset: u64, - prot: Prot, - single_page: bool, - ) -> Result { - gem::validate_vm_binding(gem, self)?; - let residency = self.enter_residency_metadata()?; - let context = self.prepare_context_binding(addr, size, offset)?; - let mut ctx = StepContext { - new_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - prot, - ..Default::default() - }; - let vm_bo = self.inner.obtain_bo(gem)?; - { - let needs_sgt = vm_bo.inner().inner.lock().sgt.is_none(); - if needs_sgt { - // Pin without the BO mutex: alias construction can hold the GEM - // reservation while acquiring this mutex. - let sgt = gem.owned_sg_table()?; - let mut bo = vm_bo.inner().inner.lock(); - if bo.sgt.is_none() { - if bo.sg_vec.is_none() { - let mut segments = KVVec::new(); - let mut offset = 0; - for range in sgt.iter() { - let address = range.dma_address() as usize; - let length = range.dma_len() as usize; - segments.push((offset, address..address + length), GFP_KERNEL)?; - offset += length; - } - bo.sg_vec = Some(segments); - } - bo.sgt = Some(sgt); - } - // Release the mutex before a losing initializer drops its SG - // owner. The declaration order also preserves this on errors. - drop(bo); - } - } - ctx.vm_bo = Some(vm_bo); - if (addr | size | offset) & UAT_PGMSK as u64 != 0 { - return Err(EINVAL); - } - Ok(PreparedUserMap { - _residency: residency, - gem: gem.into(), - ctx, - addr, - size, - offset, - single_page, - context, - }) - } - - pub(crate) fn prepare_user_unmap(&self, iova: u64, size: u64) -> Result { - if (iova | size) & UAT_PGMSK as u64 != 0 || !self.inner.range_valid(iova, size) { - return Err(EINVAL); - } - let residency = self.enter_residency_metadata()?; - Ok(PreparedUserUnmap { - _residency: residency, - iova, - size, - ctx: StepContext { - prev_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - next_va: Some(gpuvm::GpuVa::::new(pin_init::default())?), - ..Default::default() - }, - }) - } - - /// Commits in ioctl order. Quarantine may defer an all-unmap batch, but can - /// never allow a map to replace a mapping still owned by failed work. - pub(crate) fn commit_prepared_user_bind_batch( - &self, - batch: &mut PreparedUserBindBatch, - ) -> Result { - if self.defer_bind_batch(batch, 0)? { - return Ok(()); - } - let mut index = 0; - while index < batch.ops.len() { - let result = match &mut batch.ops[index] { - PreparedUserBindOp::Map(map) => { - let mut inner = self.inner.exec_lock(Some(&map.gem), true)?; - match self.mapping_commit() { - Ok(commit) => { - let (flags, repeat) = if map.single_page { - (gpuvm::GpuVaFlags::REPEAT, UAT_PGSZ as u32) - } else { - (gpuvm::GpuVaFlags::NONE, 0) - }; - let result = inner.sm_map( - &mut map.ctx, - map.addr, - map.size, - map.offset, - repeat, - flags, - ); - let retired = result.is_ok().then(|| { - let retired = self.commit_context_binding(&map.gem, &map.context); - self.track_shared_binding( - &map.gem, - map.addr, - map.size, - map.offset, - map.single_page, - ); - retired - }); - drop(inner); - drop(commit); - drop(retired); - Some(result) - } - Err(error) if error == EIO => { - drop(inner); - None - } - Err(error) => return Err(error), - } - } - PreparedUserBindOp::Unmap(unmap) => { - let mut inner = self.inner.exec_lock(None, true)?; - match self.mapping_commit() { - Ok(commit) => { - let result = inner.sm_unmap(&mut unmap.ctx, unmap.iova, unmap.size); - let retired = result.is_ok().then(|| { - let retired = self.untrack_context_ranges(core::iter::once( - unmap.iova..unmap.iova + unmap.size, - )); - self.untrack_shared_range(unmap.iova, unmap.size); - retired - }); - drop(inner); - drop(commit); - drop(retired); - Some(result) - } - Err(error) if error == EIO => { - drop(inner); - None - } - Err(error) => return Err(error), - } - } - }; - if let Some(result) = result { - result?; - index += 1; - } else if self.defer_bind_batch(batch, index)? { - return Ok(()); - } - } - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/mmu/context.rs b/drivers/gpu/drm/asahi-neo/mmu/context.rs deleted file mode 100644 index e41c73ce295af7..00000000000000 --- a/drivers/gpu/drm/asahi-neo/mmu/context.rs +++ /dev/null @@ -1,287 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Tracks canonical context resources and retains their fixed low presentation. -//! -//! Userspace owns low views 1 through 6. Only the reserved null view is created -//! here. Its installation is serialized per VM and retained through VM teardown. -//! Commits take bindings after the GPUVM exec lock; deferred unmaps take it -//! after the VM lifetime lock. Readers release bindings before taking either. - -use super::*; -use crate::hw::t8140::CONTEXT_VIEWS; -use kernel::drm_neo::gem::BaseObject; - -struct Binding { - gem: ARef, - offset: usize, -} -struct Bindings { - generation: u64, - slots: [Option; CONTEXT_VIEWS.len()], -} -/// All arithmetic for a canonical map is checked before its page-table commit. -pub(super) struct PreparedBinding { - address: u64, - object_offset: u64, - touched: u8, - covered: u8, -} - -/// Final GEM release can acquire DMA reservations. Keep replaced references -/// until the caller has released its GPUVM and mapping-admission locks. -pub(super) struct RetiredBindings([Option; CONTEXT_VIEWS.len()]); - -impl RetiredBindings { - fn new() -> Self { - Self(core::array::from_fn(|_| None)) - } -} - -#[pin_data] -pub(super) struct ContextBindings { - #[pin] - bindings: Mutex, - #[pin] - install: Mutex<()>, -} -impl ContextBindings { - pub(super) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - bindings <- new_mutex!(Bindings { generation: 0, - slots: core::array::from_fn(|_| None) }, "VM context resources"), - install <- new_mutex!((), "VM context alias installation"), - }), - GFP_KERNEL, - ) - } -} - -impl Vm { - pub(crate) fn validate_context_binding( - &self, - address: u64, - size: u64, - single_page: bool, - ) -> Result { - if self.context_bindings.is_none() || !single_page { - return Ok(()); - } - let end = address.checked_add(size).ok_or(EOVERFLOW)?; - if CONTEXT_VIEWS - .iter() - .any(|view| address < view.source + view.size as u64 && view.source < end) - { - return Err(EINVAL); - } - Ok(()) - } - - pub(super) fn prepare_context_binding( - &self, - address: u64, - size: u64, - object_offset: u64, - ) -> Result { - let mut update = PreparedBinding { - address, - object_offset, - touched: 0, - covered: 0, - }; - if self.context_bindings.is_none() { - return Ok(update); - } - let end = address.checked_add(size).ok_or(EOVERFLOW)?; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if address < view.source + view.size as u64 && view.source < end { - update.touched |= 1 << index; - } - if address <= view.source && view.source + view.size as u64 <= end { - update.covered |= 1 << index; - usize::try_from( - object_offset - .checked_add(view.source - address) - .ok_or(EOVERFLOW)?, - )?; - } - } - Ok(update) - } - - /// Called after a successful leaf commit, while its exec lock still orders - /// mapping changes. The prepared update cannot fail after visibility. - pub(super) fn commit_context_binding( - &self, - gem: &ARef, - update: &PreparedBinding, - ) -> RetiredBindings { - let mut retired = RetiredBindings::new(); - if update.touched == 0 { - return retired; - } - let Some(context) = self.context_bindings.as_ref() else { - return retired; - }; - let mut bindings = context.bindings.lock(); - for (index, slot) in bindings.slots.iter_mut().enumerate() { - if update.touched & (1 << index) != 0 { - retired.0[index] = slot.take(); - } - if update.covered & (1 << index) != 0 { - // Coverage proves source >= address. Preparation checked this - // exact sum and its usize conversion against the same view. - let offset = (update.object_offset - + (CONTEXT_VIEWS[index].source - update.address)) as usize; - *slot = Some(Binding { - gem: gem.clone(), - offset, - }); - } - } - bindings.generation = bindings.generation.wrapping_add(1); - retired - } - - /// Successful user unmaps and accepted deferred unmaps invalidate the - /// matching candidates. Returning their owners avoids destruction under - /// the caller's exec or lifetime lock. - pub(super) fn untrack_context_ranges( - &self, - ranges: impl Iterator>, - ) -> RetiredBindings { - let mut retired = RetiredBindings::new(); - let Some(context) = self.context_bindings.as_ref() else { - return retired; - }; - let mut bindings = context.bindings.lock(); - for range in ranges { - let mut touched = false; - for (index, view) in CONTEXT_VIEWS.iter().enumerate() { - if range.start < view.source + view.size as u64 && view.source < range.end { - if let Some(binding) = bindings.slots[index].take() { - retired.0[index] = Some(binding); - } - touched = true; - } - } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } - } - retired - } - - pub(crate) fn untrack_context_object(&self, gem: &gem::Object) { - let Some(context) = self.context_bindings.as_ref() else { - return; - }; - let mut bindings = context.bindings.lock(); - let mut touched = false; - for slot in &mut bindings.slots { - if slot - .as_ref() - .is_some_and(|binding| core::ptr::eq(&*binding.gem, gem)) - { - *slot = None; - touched = true; - } - } - if touched { - bindings.generation = bindings.generation.wrapping_add(1); - } - } - - pub(crate) fn translate_iova(&self, address: u64) -> Result { - self.inner - .exec_lock(None, false)? - .page_table - .translate_iova(address) - } - - /// Requires a job mapping guard before validating or installing these views. - pub(crate) fn install_job_context_aliases(&self) -> Result { - let context = self.context_bindings.as_ref().ok_or(EINVAL)?; - let (generation, snapshot) = { - let bindings = context.bindings.lock(); - let mut snapshot = KVec::with_capacity(CONTEXT_VIEWS.len(), GFP_KERNEL)?; - for slot in &bindings.slots { - let binding = slot.as_ref().ok_or(ENOENT)?; - snapshot.push((binding.gem.clone(), binding.offset), GFP_KERNEL)?; - } - (bindings.generation, snapshot) - }; - for (index, (gem, offset)) in snapshot.iter().enumerate().skip(1) { - let view = CONTEXT_VIEWS[index]; - if !self.covers_range(view.source, view.size as u64, true, view.write) - || !self.covers_range(view.low, view.size as u64, true, view.write) - { - return Err(EFAULT); - } - if offset.checked_add(view.size).ok_or(EOVERFLOW)? > gem.size() { - return Err(ERANGE); - } - for page in (0..view.size).step_by(UAT_PGSZ) { - if self.translate_iova(view.source + page as u64)? - != self.translate_iova(view.low + page as u64)? - { - return Err(EFAULT); - } - } - } - let _install = context.install.lock(); - let view = CONTEXT_VIEWS[0]; - let (gem, offset) = &snapshot[0]; - // Null presentation requires writes in addition to the canonical view's - // ordinary read requirement. - if !self.covers_range(view.source, view.size as u64, true, true) { - return Err(EFAULT); - } - let end = offset.checked_add(view.size).ok_or(EOVERFLOW)?; - if end > gem.size() { - return Err(ERANGE); - } - let mut existing = 0; - for page in (0..view.size).step_by(UAT_PGSZ) { - if self.covers_range(view.low + page as u64, UAT_PGSZ as u64, true, true) { - existing += 1; - if self.translate_iova(view.source + page as u64)? - != self.translate_iova(view.low + page as u64)? - { - return Err(EFAULT); - } - } - } - if existing != 0 { - if existing != view.size / UAT_PGSZ { - return Err(EFAULT); - } - return if context.bindings.lock().generation == generation { - Ok(()) - } else { - Err(EAGAIN) - }; - } - let mut mappings = KVec::with_capacity(1, GFP_KERNEL)?; - let mapping = self.map_in_range( - gem, - *offset..end, - UAT_PGSZ as u64, - view.low..view.low + view.size as u64, - PROT_GPU_SHARED_RW, - false, - )?; - for page in (0..view.size).step_by(UAT_PGSZ) { - if self.translate_iova(view.source + page as u64)? - != self.translate_iova(view.low + page as u64)? - { - return Err(EFAULT); - } - } - mappings.push(mapping, GFP_KERNEL)?; - if context.bindings.lock().generation != generation { - return Err(EAGAIN); - } - self.append_driver_mappings(mappings) - } -} diff --git a/drivers/gpu/drm/asahi-neo/mmu/lifetime.rs b/drivers/gpu/drm/asahi-neo/mmu/lifetime.rs deleted file mode 100644 index d53c19b93bfdcd..00000000000000 --- a/drivers/gpu/drm/asahi-neo/mmu/lifetime.rs +++ /dev/null @@ -1,634 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Keeps user mappings alive until accepted work has relinquished firmware ownership. -//! -//! The state lock is taken after GPUVM's exec lock when committing user mappings. Waiting for -//! that lock's commits never holds a GPUVM reservation. Draining takes no state lock while it -//! unmaps memory. Driver aliases retain only the inner GPUVM, so they create no cycle with Vm. - -use super::*; -use kernel::{ - new_condvar, - sync::{CondVar, CondVarTimeoutResult}, - time::msecs_to_jiffies, -}; - -const ADMISSION_WAIT_MS: u32 = 1000; - -pub(super) struct DriverMappings { - mappings: KVec, - reserved: KVec>, - compute: Option, - render: [Option; crate::hw::t8140::queues::RENDER_SLOTS], -} - -struct PoolMappings { - id: u64, - blocks: usize, - mappings: KVec, -} - -impl DriverMappings { - fn ranges(&self) -> impl Iterator> + '_ { - Iterator::chain( - Iterator::chain( - self.mappings.iter(), - Iterator::chain( - self.compute.iter().flat_map(|pool| pool.mappings.iter()), - self.render - .iter() - .flatten() - .flat_map(|pool| pool.mappings.iter()), - ), - ) - .map(KernelMapping::iova_range), - self.reserved.iter().cloned(), - ) - } -} - -struct State { - active: usize, - commits: usize, - blocked: bool, - quarantined: bool, - draining: bool, - closed: bool, - cleanup_failed: bool, - close_ranges: Option<(Range, Range)>, - unmaps: KVec, - objects: KVec>, -} - -#[pin_data] -pub(super) struct VmLifetime { - #[pin] - state: Mutex, - #[pin] - changed: CondVar, - #[pin] - driver: Mutex>, -} - -impl VmLifetime { - pub(super) fn can_reclaim(&self) -> bool { - self.state.try_lock().is_some_and(|state| { - state.active == 0 && state.commits == 0 - && !state.blocked && !state.quarantined && !state.draining - && !state.closed && !state.cleanup_failed - }) - } - - pub(super) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - state <- new_mutex!(State { active:0, commits:0, blocked:false, - quarantined:false, draining:false, closed:false, cleanup_failed:false, close_ranges:None, - unmaps:KVec::new(), objects:KVec::new() }, "VM job lifetime"), - changed <- new_condvar!("VM mapping admission"), - driver <- new_mutex!(None, "VM driver mappings"), - }), - GFP_KERNEL, - ) - } -} - -/// Retains a VM's mappings through a command's exact retirement. An error fence is not -/// retirement: potentially visible work must retain its guard through quarantine or stop. -pub(crate) struct VmJobGuard { - vm: Vm, - _residency: Option, -} - -impl VmJobGuard { - /// Blocks mapping commits before the error fence can become visible to userspace. - pub(crate) fn quarantine(&self) { - if let Some(lifetime) = self.vm.lifetime.as_ref() { - let mut state = lifetime.state.lock(); - state.blocked = true; - while state.commits != 0 { - lifetime.changed.wait(&mut state); - } - state.quarantined = true; - } - } -} - -pub(super) struct MappingCommit { - lifetime: Arc, -} -impl Drop for MappingCommit { - fn drop(&mut self) { - let mut state = self.lifetime.state.lock(); - state.commits -= 1; - if state.commits == 0 { - self.lifetime.changed.notify_all(); - } - } -} - -impl Vm { - /// Attaches aliases while constructing this VM, before exposing it to userspace. - pub(crate) fn install_driver_mappings( - &self, - mappings: KVec, - reserved: KVec>, - ) -> Result { - let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; - let mut driver = lifetime.driver.lock(); - if driver.is_some() { - return Err(EBUSY); - } - *driver = Some(DriverMappings { - mappings, - reserved, - compute: None, - render: core::array::from_fn(|_| None), - }); - Ok(()) - } - - /// Reserves the complete append before moving aliases into VM ownership. - pub(crate) fn append_driver_mappings(&self, mappings: KVec) -> Result { - let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; - let mut driver = lifetime.driver.lock(); - let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; - driver.mappings.reserve(mappings.len(), GFP_KERNEL)?; - for mapping in mappings { - driver.mappings.push(mapping, GFP_KERNEL)?; - } - Ok(()) - } - - pub(crate) fn render_pool_tvb_blocks(&self, slot: u8, id: u64) -> Option { - if id == 0 { - return None; - } - self.lifetime - .as_ref()? - .driver - .lock() - .as_ref()? - .render - .get(usize::from(slot))? - .as_ref() - .filter(|pool| pool.id == id) - .map(|pool| pool.blocks) - } - - /// Retains a new generation after all references to a replaced pool retire. - pub(crate) fn install_render_pool_mappings( - &self, - slot: u8, - id: u64, - blocks: usize, - mappings: KVec, - ) -> Result { - if id == 0 { - return Err(EINVAL); - } - let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; - let mut driver = lifetime.driver.lock(); - let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; - let pool = driver.render.get_mut(usize::from(slot)).ok_or(EINVAL)?; - if pool.is_some() { - return Err(EBUSY); - } - *pool = Some(PoolMappings { - id, - blocks, - mappings, - }); - Ok(()) - } - - /// Consumes aliases only after the exact cached owner and complete append - /// capacity have been checked. Failed preparation retains its mappings for - /// destruction after the caller releases the device mutex. - pub(crate) fn append_render_pool_mappings_prepared( - &self, - slot: u8, - id: u64, - previous: usize, - blocks: usize, - mappings: &mut KVec, - ) -> Result { - if id == 0 || blocks <= previous || mappings.len() != blocks - previous { - return Err(EINVAL); - } - let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; - let mut driver = lifetime.driver.lock(); - let pool = Option::as_mut(&mut *driver) - .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .and_then(Option::as_mut) - .filter(|pool| pool.id == id) - .ok_or(EINVAL)?; - if pool.blocks != previous { - return Err(EBUSY); - } - pool.mappings - .len() - .checked_add(mappings.len()) - .ok_or(EOVERFLOW)?; - pool.mappings.reserve(mappings.len(), GFP_KERNEL)?; - for mapping in mappings.drain_all() { - pool.mappings.push(mapping, GFP_KERNEL)?; - } - pool.blocks = blocks; - Ok(()) - } - - /// A delayed teardown must not remove a successor generation. Remove - /// under the cache lock, then unmap after dropping that lock. - pub(crate) fn clear_render_pool_mappings_if_owner(&self, slot: u8, id: u64) { - if id == 0 { - return; - } - let previous = self.lifetime.as_ref().and_then(|lifetime| { - let mut driver = lifetime.driver.lock(); - Option::as_mut(&mut *driver) - .and_then(|driver| driver.render.get_mut(usize::from(slot))) - .filter(|pool| pool.as_ref().is_some_and(|pool| pool.id == id)) - .and_then(Option::take) - }); - drop(previous); - } - - pub(crate) fn compute_pool_mappings_match(&self, id: u64) -> bool { - id != 0 - && self.lifetime.as_ref().is_some_and(|lifetime| { - lifetime - .driver - .lock() - .as_ref() - .is_some_and(|driver| driver.compute.as_ref().is_some_and(|pool| pool.id == id)) - }) - } - - /// The device publication lock serializes first-use alias construction. - /// Never replace a live generation; its jobs must retire before clearing. - pub(crate) fn install_compute_pool_mappings( - &self, - id: u64, - mappings: KVec, - ) -> Result { - if id == 0 { - return Err(EINVAL); - } - let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; - let mut driver = lifetime.driver.lock(); - let driver = Option::as_mut(&mut *driver).ok_or(EINVAL)?; - if driver.compute.is_some() { - return Err(EBUSY); - } - driver.compute = Some(PoolMappings { - id, - blocks: 0, - mappings, - }); - Ok(()) - } - - /// Called under the publication lock after all obsolete pool work retires. - /// Drop outside the driver lock because mapping teardown acquires GPUVM locks. - pub(crate) fn clear_compute_pool_mappings(&self) { - let previous = self.lifetime.as_ref().and_then(|lifetime| { - let mut driver = lifetime.driver.lock(); - Option::as_mut(&mut *driver).and_then(|driver| driver.compute.take()) - }); - drop(previous); - } - - /// Tests mappings and reservations which userspace must never replace. - pub(crate) fn driver_range_overlaps(&self, range: Range) -> bool { - let Some(lifetime) = self.lifetime.as_ref() else { - return false; - }; - let driver = lifetime.driver.lock(); - driver.as_ref().is_some_and(|driver| { - driver - .ranges() - .any(|reserved| range.start < reserved.end && reserved.start < range.end) - }) - } - - /// Waits before reservation acquisition, allowing healthy retirement to drain. - pub(crate) fn wait_for_user_map_admission(&self) -> Result { - let Some(lifetime) = self.lifetime.as_ref() else { - return Ok(()); - }; - let mut state = lifetime.state.lock(); - let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); - loop { - let failed = self.status.as_ref().is_some_and(|status| status.get() != 0); - if !failed - && !state.closed - && (state.blocked || (state.active != 0 && state.quarantined)) - && remaining != 0 - { - remaining = match lifetime - .changed - .wait_interruptible_timeout(&mut state, remaining) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; - continue; - } - if state.closed { - return Err(ENOENT); - } - if failed || state.blocked || (state.active != 0 && state.quarantined) { - return Err(EIO); - } - if !state.draining { - return Ok(()); - } - if lifetime.changed.wait_interruptible(&mut state) { - return Err(ERESTARTSYS); - } - } - } - - /// Acquires a job reference without imposing a VM-wide limit on healthy work. - pub(crate) fn retain_job(&self) -> Result { - let residency = self.enter_residency()?; - let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; - let mut state = lifetime.state.lock(); - loop { - if state.closed { - return Err(ENOENT); - } - if state.blocked { - return Err(EIO); - } - if !state.draining { - state.active = state.active.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(VmJobGuard { vm: self.clone(), _residency: residency }); - } - lifetime.changed.wait(&mut state); - } - } - - /// The first job of a submission may wait for a clean VM's quarantine drain. - pub(crate) fn retain_first_job(&self) -> Result { - match self.retain_job() { - Err(error) - if error == EIO && self.status.as_ref().is_some_and(|status| status.get() == 0) => { - } - result => return result, - } - let lifetime = self.lifetime.as_ref().ok_or(EINVAL)?; - let mut remaining = msecs_to_jiffies(ADMISSION_WAIT_MS); - let mut state = lifetime.state.lock(); - while state.blocked - && remaining != 0 - && self.status.as_ref().is_some_and(|status| status.get() == 0) - { - remaining = match lifetime - .changed - .wait_interruptible_timeout(&mut state, remaining) - { - CondVarTimeoutResult::Signal { .. } => return Err(ERESTARTSYS), - CondVarTimeoutResult::Timeout => 0, - CondVarTimeoutResult::Woken { jiffies } => jiffies, - }; - } - drop(state); - self.retain_job() - } - - /// Called only after the GPUVM exec/reservation locks have been acquired. - pub(super) fn mapping_commit(&self) -> Result> { - let Some(lifetime) = self.lifetime.as_ref() else { - return Ok(None); - }; - let mut state = lifetime.state.lock(); - if state.closed { - return Err(ENOENT); - } - if state.blocked { - return Err(EIO); - } - state.commits = state.commits.checked_add(1).ok_or(EOVERFLOW)?; - Ok(Some(MappingCommit { - lifetime: lifetime.clone(), - })) - } - - /// Reserves cleanup bookkeeping before a batch starts committing leaves. - pub(crate) fn reserve_deferred_user_unmaps(&self, count: usize) -> Result { - if let Some(lifetime) = self.lifetime.as_ref() { - lifetime.state.lock().unmaps.reserve(count, GFP_KERNEL)?; - } - Ok(()) - } - - pub(super) fn defer_bind_batch(&self, batch: &mut PreparedUserBindBatch, start: usize) -> Result { - use super::bind::PreparedUserBindOp; - let Some(lifetime) = self.lifetime.as_ref() else { - return Ok(false); - }; - let mut state = lifetime.state.lock(); - if !state.blocked { - return Ok(false); - } - if state.closed { - return Err(ENOENT); - } - if batch - .ops - .iter().skip(start) - .any(|op| matches!(op, PreparedUserBindOp::Map(_))) - { - return Err(EIO); - } - if !state.quarantined { - return Err(EBUSY); - } - // Reserve before accepting any range; a partial deferred batch must - // not leave canonical metadata ahead of the accepted cleanup. - state.unmaps.reserve(batch.ops.len() - start, GFP_KERNEL)?; - let retired = self.untrack_context_ranges(batch.ops.iter().skip(start).filter_map(|op| match op { - PreparedUserBindOp::Unmap(unmap) => Some(unmap.iova..unmap.iova + unmap.size), - PreparedUserBindOp::Map(_) => None, - })); - // Transfer the preallocated split nodes along with each accepted range. - // Already-committed prefix operations must not be replayed: their nodes - // may have been consumed and another binding may now occupy the range. - for op in batch.ops.drain(start..) { - if let PreparedUserBindOp::Unmap(unmap) = op { - if !state.unmaps.iter().any(|old| old.iova == unmap.iova && old.size == unmap.size) { - state.unmaps.push(unmap, GFP_KERNEL)?; - } - } - } - drop(state); - drop(retired); - Ok(true) - } - - /// Returns true if a quarantined owner requires this unmap to be deferred. - pub(super) fn defer_unmap(&self, iova: u64, size: u64) -> Result { - let Some(lifetime) = self.lifetime.as_ref() else { - return Ok(false); - }; - let state = lifetime.state.lock(); - if state.closed { - return Err(ENOENT); - } - if !state.blocked { - return Ok(false); - } - if !state.quarantined { - return Err(EBUSY); - } - if state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { - return Ok(true); - } - drop(state); - let unmap = self.prepare_user_unmap(iova, size)?; - let mut state = lifetime.state.lock(); - if state.closed { return Err(ENOENT); } - if !state.blocked { return Ok(false); } - if !state.quarantined { return Err(EBUSY); } - if !state.unmaps.iter().any(|old| old.iova == iova && old.size == size) { - state.unmaps.push(unmap, GFP_KERNEL)?; - } - Ok(true) - } - - /// Defers handle-close cleanup while any accepted job can still use the BO. - pub(crate) fn drop_mappings(&self, gem: &gem::Object) -> Result { - let Some(lifetime) = self.lifetime.as_ref() else { - return self.drop_mappings_now(gem); - }; - let mut state = lifetime.state.lock(); - if !state - .objects - .iter() - .any(|object| core::ptr::eq(&**object, gem)) - { - state.objects.push(gem.into(), GFP_KERNEL)?; - } - if state.active != 0 || state.draining { - // Stop new jobs from selecting this accepted cleanup candidate. - // The queued object retains its GEM while the candidate is removed. - // Actual unmap repeats this under exec to order any intervening bind. - self.untrack_context_object(gem); - return Ok(()); - } - state.draining = true; - drop(state); - self.drain_mappings(lifetime); - Ok(()) - } - - /// Closes admission permanently and preserves mappings referenced by accepted jobs. - pub(crate) fn unmap_user_ranges(&self, user: Range, kernel: Range) -> Result { - if let Some(lifetime) = self.lifetime.as_ref() { - let mut state = lifetime.state.lock(); - state.blocked = true; - while state.commits != 0 { - lifetime.changed.wait(&mut state); - } - state.closed = true; - if state.active != 0 { - state.close_ranges = Some((user, kernel)); - return Ok(()); - } - } - self.unmap_user_ranges_now(user, kernel) - } - - fn unmap_user_ranges_now(&self, user: Range, _kernel: Range) -> Result { - let _residency = self.enter_retirement_residency(); - // User mappings alone have GPUVA nodes. Kernel and driver aliases use - // mm::Node and are untouched; GPUVM's embedded kernel cutout is skipped. - // Closed admission and zero accepted jobs make whole-node teardown safe. - let mut inner = self.inner.lock_inner(); - let result = inner.unmap_all(VmInner::unmap_gpuva); - if result.is_ok() { - self.untrack_shared_range(user.start, user.end - user.start); - } - drop(inner); - // Final GEM/SG-table destruction may acquire reservations itself. - self.bo_deferred_cleanup(); - result - } -} - -impl Drop for VmJobGuard { - fn drop(&mut self) { - let Some(lifetime) = self.vm.lifetime.as_ref() else { - return; - }; - let mut state = lifetime.state.lock(); - state.active -= 1; - if state.active != 0 { - return; - } - state.draining = true; - drop(state); - self.vm.drain_mappings(lifetime); - } -} - -impl Vm { - /// Admission stays closed to new jobs while the exec lock is acquired for cleanup. - /// Recheck queued cleanup and reopen under the same lock used to append it. - fn drain_mappings(&self, lifetime: &VmLifetime) { - let _residency = self.enter_retirement_residency(); - let mut state = lifetime.state.lock(); - let mut failure = None; - loop { - let close = state.close_ranges.take(); - let unmaps = core::mem::replace(&mut state.unmaps, KVec::new()); - let objects = core::mem::replace(&mut state.objects, KVec::new()); - drop(state); - if let Some((user, kernel)) = close { - if let Err(error) = self.unmap_user_ranges_now(user, kernel) { - failure.get_or_insert(error); - pr_err!("MMU: deferred VM close failed\n"); - } - } else { - for mut unmap in unmaps { - if let Err(error) = self.unmap_prepared_cleanup(&mut unmap) { - failure.get_or_insert(error); - pr_err!("MMU: deferred user unmap failed\n"); - } - } - } - for object in objects { - if let Err(error) = self.drop_mappings_now(&object) { - failure.get_or_insert(error); - pr_err!("MMU: deferred object unmap failed\n"); - } - } - state = lifetime.state.lock(); - if state.close_ranges.is_some() || !state.unmaps.is_empty() || !state.objects.is_empty() - { - continue; - } - state.quarantined = false; - state.draining = false; - if let Some(error) = failure { - // Cleanup is allocation-free for accepted ranges and has a - // whole-node low-memory fallback for objects. An unexpected - // invariant error must not reopen admission over stale leaves. - // GPUVA owners retain backing until final VM close; report the - // terminal failure instead of silently discarding its outcome. - state.blocked = true; - state.cleanup_failed = true; - if let Some(status) = &self.status { status.record(error); } - } else if !state.closed && !state.cleanup_failed { - state.blocked = false; - } - drop(state); - lifetime.changed.notify_all(); - break; - } - self.bo_deferred_cleanup(); - } -} diff --git a/drivers/gpu/drm/asahi-neo/mmu/residency.rs b/drivers/gpu/drm/asahi-neo/mmu/residency.rs deleted file mode 100644 index 3a76f07577d2b6..00000000000000 --- a/drivers/gpu/drm/asahi-neo/mmu/residency.rs +++ /dev/null @@ -1,647 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Pressure-only backing residency for idle G17 user VMs. -//! -//! This gate protects backing/address preparation as well as accepted jobs. -//! Its writer never waits for active readers: reclaim skips the VM instead. - -use super::*; -use kernel::drm_neo::gem::BaseObject; -use kernel::{new_condvar, sync::CondVar}; - -/// Re-evicting backing that a job just had to restore only repeats the -/// restoration. After a restore, reclaim of this VM pauses for a backoff -/// proportional to what the restore cost, within these bounds. -const RESTORE_BACKOFF_MIN_NS: u64 = 1_000_000_000; -const RESTORE_BACKOFF_MAX_NS: u64 = 30_000_000_000; -const RESTORE_BACKOFF_FACTOR: u64 = 16; -/// A VM whose last job, binding or other residency user ended (or which was -/// created) more recently than this is in use: a submitting or loading client -/// would restore whatever reclaim released, synchronously in its next job. -const RECLAIM_QUIET_NS: u64 = 2000000000; - -fn now_ns() -> u64 { - ::ktime_get() as u64 -} - -#[derive(Default)] -struct State { - users: usize, - busy: bool, - evicted: bool, - closed: bool, - /// Monotonic time the last restoration finished and its duration. - restored_at: u64, - restore_cost: u64, - /// Monotonic time the last residency user (job, binding, retirement) - /// released its lease; the gate's creation time before the first one. - last_active: u64, -} - -impl State { - /// Reclaim skips a VM that is in use, was recently used or recently restored. - fn reclaim_deferred(&self, now: u64) -> bool { - if now.saturating_sub(self.last_active) < RECLAIM_QUIET_NS { - return true; - } - if self.restored_at == 0 { - return false; - } - let backoff = self - .restore_cost - .saturating_mul(RESTORE_BACKOFF_FACTOR) - .clamp(RESTORE_BACKOFF_MIN_NS, RESTORE_BACKOFF_MAX_NS); - now.saturating_sub(self.restored_at) < backoff - } -} - -#[pin_data] -pub(crate) struct Gate { - #[pin] - state: Mutex, - #[pin] - ready: CondVar, -} - -pub(crate) struct Lease { - gate: Arc, -} - -pub(crate) struct Reclaim { - gate: Arc, - changed: bool, -} - -impl Gate { - pub(crate) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - state <- new_mutex!( - State { last_active: now_ns(), ..State::default() }, - "Asahi residency" - ), - ready <- new_condvar!("Asahi residency ready"), - }), - GFP_KERNEL, - ) - } - - /// Only a previous pressure eviction can cause restoration or waiting. - pub(crate) fn enter(gate: &Arc, mut restore: impl FnMut() -> Result) -> Result { - loop { - let mut state = gate.state.lock(); - if state.closed { - return Err(ENOENT); - } - while state.busy { - if gate.ready.wait_interruptible(&mut state) { - return Err(ERESTARTSYS); - } - } - if state.closed { - return Err(ENOENT); - } - if !state.evicted { - state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; - return Ok(Lease { gate: gate.clone() }); - } - // Binding edits can add mappings or retire invalidated GPUVAs - // without restoring the rest of the VM. Wait for those mutations - // before walking the authoritative logical bindings for restoration. - if state.users != 0 { - if gate.ready.wait_interruptible(&mut state) { - return Err(ERESTARTSYS); - } - continue; - } - // No new reader can enter until every retained mapping is restored. - state.busy = true; - drop(state); - let started = now_ns(); - let result = restore(); - let mut state = gate.state.lock(); - state.busy = false; - if result.is_ok() { - state.evicted = false; - let now = now_ns(); - state.restored_at = now; - state.restore_cost = now.saturating_sub(started); - } - drop(state); - gate.ready.notify_all(); - result?; - } - } - - pub(crate) fn enter_metadata(gate: &Arc) -> Result { - let mut state = gate.state.lock(); - while state.busy { - if gate.ready.wait_interruptible(&mut state) { - return Err(ERESTARTSYS); - } - } - if state.closed { - return Err(ENOENT); - } - state.users = state.users.checked_add(1).ok_or(EOVERFLOW)?; - Ok(Lease { gate: gate.clone() }) - } - - /// Retirement of an already closed handle may outlive the file's VM. - /// A closed gate excludes every new reclaimer and restorer, so no lease - /// is necessary then. Otherwise serialize metadata removal with restoration. - pub(super) fn enter_retirement(gate: &Arc) -> Option { - let mut state = gate.state.lock(); - while state.busy { - gate.ready.wait(&mut state); - } - if state.closed { - return None; - } - state.users += 1; - Some(Lease { gate: gate.clone() }) - } - - /// Direct reclaim cannot block on a job, another reclaimer or restoration. - /// A prior partial eviction does not exclude another scan: the VM may - /// still retain eligible BOs after satisfying a previous scan target. - /// The busy gate continues to exclude restoration and metadata changes. - pub(crate) fn try_reclaim(gate: &Arc) -> Option { - let mut state = gate.state.try_lock()?; - if state.users != 0 || state.busy || state.closed || state.reclaim_deferred(now_ns()) { - return None; - } - state.busy = true; - Some(Reclaim { - gate: gate.clone(), - changed: false, - }) - } - - pub(crate) fn close(&self) { - let mut state = self.state.lock(); - state.closed = true; - while state.busy { - self.ready.wait(&mut state); - } - } - - pub(crate) fn idle(&self) -> bool { - self.state.try_lock().is_some_and(|state| { - state.users == 0 - && !state.busy - && !state.closed - && !state.reclaim_deferred(now_ns()) - }) - } - - /// Pressure reclaim released this VM's backing and no user has entered - /// since: its next job restores first. Never blocks; a contended gate - /// reads as not evicted. - pub(crate) fn evicted(&self) -> bool { - self.state - .try_lock() - .is_some_and(|state| state.evicted && !state.busy && state.users == 0) - } - - /// A page-table mutation happened outside a `Reclaim` (the purge path runs - /// on busy VMs). The next `enter` restores every INVALIDATED mapping. - pub(crate) fn note_evicted(&self) { - self.state.lock().evicted = true; - } -} - -impl Reclaim { - /// Call before the first page-table mutation, including fallible unmaps. - pub(crate) fn mark_changed(&mut self) { - self.changed = true; - } -} - -impl Drop for Reclaim { - fn drop(&mut self) { - let mut state = self.gate.state.lock(); - state.evicted |= self.changed; - state.busy = false; - drop(state); - self.gate.ready.notify_all(); - } -} - -impl Drop for Lease { - fn drop(&mut self) { - let mut state = self.gate.state.lock(); - debug_assert!(state.users != 0); - state.users -= 1; - state.last_active = now_ns(); - if state.users == 0 { - drop(state); - self.gate.ready.notify_all(); - } - } -} - -/// Registration lifetime belongs to the file's VM handle, not the VM clone -/// retained by the callback. Unregister synchronously before dropping the clone. -pub(crate) struct VmShrinker { - vm: crate::mmu::Vm, - raw: *mut kernel::bindings::shrinker, -} - -// SAFETY: shrinker core serializes registration lifetime; callback VM operations -// use its residency/reservation locks. raw is never dereferenced after free. -unsafe impl Send for VmShrinker {} -unsafe impl Sync for VmShrinker {} - -impl VmShrinker { - pub(crate) fn new(vm: &crate::mmu::Vm) -> Result> { - let mut owner = KBox::new( - Self { - vm: vm.clone(), - raw: core::ptr::null_mut(), - }, - GFP_KERNEL, - )?; - // SAFETY: The format is static and has no format arguments. - let raw = - unsafe { kernel::bindings::shrinker_alloc(0, c_str!("asahi-idle-vm").as_char_ptr()) }; - if raw.is_null() { - return Err(ENOMEM); - } - owner.raw = raw; - // SAFETY: Newly allocated/unregistered shrinker is exclusively owned. - unsafe { - (*raw).count_objects = Some(Self::count); - (*raw).scan_objects = Some(Self::scan); - (*raw).private_data = (&*owner as *const Self).cast_mut().cast(); - (*raw).seeks = 2; - (*raw).batch = 256; - kernel::bindings::shrinker_register(raw); - } - Ok(owner) - } - - unsafe extern "C" fn count( - raw: *mut kernel::bindings::shrinker, - _sc: *mut kernel::bindings::shrink_control, - ) -> kernel::ffi::c_ulong { - // SAFETY: The registration owns this stable allocation until callbacks drain. - let owner = unsafe { &*((*raw).private_data as *const Self) }; - owner.vm.idle_reclaim_count() as _ - } - - unsafe extern "C" fn scan( - raw: *mut kernel::bindings::shrinker, - sc: *mut kernel::bindings::shrink_control, - ) -> kernel::ffi::c_ulong { - // SAFETY: Both pointers are supplied by the registered shrinker core. - let owner = unsafe { &*((*raw).private_data as *const Self) }; - let target = unsafe { (*sc).nr_to_scan } as usize; - owner.vm.reclaim_idle_pages(target) as _ - } -} - -impl Drop for VmShrinker { - fn drop(&mut self) { - if !self.raw.is_null() { - // SAFETY: This owns the sole registration; free drains callbacks - // before the embedded VM and callback private_data can disappear. - unsafe { kernel::bindings::shrinker_free(self.raw) }; - } - } -} - -impl Vm { - pub(crate) fn close_idle_reclaim(&self) { - if let Some(gate) = self.residency.as_ref() { - gate.close(); - } - } - - pub(crate) fn residency_reclaim_enabled(&self) -> bool { - self.residency.is_some() - } - - /// The residency gate of a user VM, for observers that must not keep the VM. - pub(crate) fn residency_gate(&self) -> Option> { - self.residency.clone() - } - - pub(super) fn enter_residency_metadata(&self) -> Result> { - self.residency - .as_ref() - .map(Gate::enter_metadata) - .transpose() - } - - pub(super) fn enter_residency(&self) -> Result> { - self.residency - .as_ref() - .map(|gate| Gate::enter(gate, || self.restore_idle_pages())) - .transpose() - } - - pub(crate) fn idle_reclaim_count(&self) -> usize { - let Some(gate) = self.residency.as_ref() else { - return 0; - }; - if self - .status - .as_ref() - .map_or(true, |status| status.get() != 0) - { - return 0; - } - let idle = gate.idle(); - let Some(mut inner) = self.inner.try_lock_private() else { - return 0; - }; - let mut pages = 0usize; - // SAFETY: Counting changes no logical mapping/list. Private BOs share - // this reservation; linked aliases retain their borrowed BO and GEM. - let _ = unsafe { inner.for_each_private_bo(|_, bo| { - let object = bo.object(); - // Purgeable objects count on busy VMs too; idle backing only on idle ones. - if object.idle_reclaim_candidate() - && (object.madv_locked() > 0 || idle) - && object.sole_sg_lease_releasable_locked() - { - pages = pages.saturating_add(object.size() / kernel::page::PAGE_SIZE); - } - Ok(true) - }) }; - - pages - } - - /// Discard the backing of objects userspace marked DONTNEED. This needs - /// only the VM reservation: binding edits and restoration hold it too, - /// and the advice guarantees no pending or future GPU reference until - /// the next WILLNEED. Purged mappings stay INVALIDATED until unbound. - fn purge_advised_pages(&self, target: usize) -> usize { - let Some(gate) = self.residency.as_ref() else { - return 0; - }; - if self - .status - .as_ref() - .map_or(true, |status| status.get() != 0) - { - return 0; - } - let Some(mut inner) = self.inner.try_lock_private() else { - return 0; - }; - let uat = inner.uat_inner.clone(); - let Some(shared) = uat.shared.try_lock() else { - return 0; - }; - let contexts = shared.contexts_naming_root(inner.ttb()); - let mut purged = 0usize; - // SAFETY: The private reservation excludes binding edits and restoration. - // Callbacks change PTEs/flags only; all aliases stay linked. - let _ = unsafe { inner.for_each_private_bo(|inner, bo| { - if purged >= target { return Ok(false); } - let object = bo.object(); - if !object.idle_reclaim_candidate() || object.madv_locked() <= 0 { - return Ok(true); - } - { - let Some(bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; - if bo_inner.sgt.is_some() && !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } - } - // Any mutation that does not end in a completed purge leaves - // INVALIDATED aliases with cleared PTEs while madv stays >= 0, so - // the next WILLNEED would report the contents retained. Record the - // eviction so the gate restores those aliases before the next job. - let mut mutated = false; - let invalidated = inner.invalidate_private_bo_mappings(bo, |inner, start, range| { - mutated = true; - let end = start.checked_add(range).ok_or(EOVERFLOW)?; - let _change = inner.mapping_mutation(); - inner.page_table.discard_partial_map(start..end)?; - inner.tlbi_context_mask(start, range as usize, contexts); - Ok(()) - }); - if let Err(error) = invalidated { - if mutated { - gate.note_evicted(); - } - return Err(error); - } - { - let Some(mut bo_inner) = bo.inner().inner.try_lock() else { - gate.note_evicted(); - return Ok(true); - }; - bo_inner.sg_vec = None; - bo_inner.sgt = None; - } - if object.purge_locked().is_ok() { - purged = purged.saturating_add(object.size() / kernel::page::PAGE_SIZE); - } else { - gate.note_evicted(); - } - Ok(purged < target) - }) }; - drop(shared); - drop(inner); - purged - } - - pub(crate) fn reclaim_idle_pages(&self, target: usize) -> usize { - let Some(gate) = self.residency.as_ref() else { - return 0; - }; - let purged = self.purge_advised_pages(target); - if purged >= target { - return purged; - } - let target = target - purged; - let Some(mut reclaim) = Gate::try_reclaim(gate) else { - return purged; - }; - // Error fences cannot prove firmware ownership ended. - if self - .status - .as_ref() - .map_or(true, |status| status.get() != 0) - || !self - .lifetime - .as_ref() - .is_some_and(|lifetime| lifetime.can_reclaim()) - { - return purged; - } - let Some(mut inner) = self.inner.try_lock_private() else { - return purged; - }; - let uat = inner.uat_inner.clone(); - let Some(shared) = uat.shared.try_lock() else { - return purged; - }; - let contexts = shared.contexts_naming_root(inner.ttb()); - let mut unpinned = 0usize; - // SAFETY: Exclusive residency and the private reservation exclude jobs, - // binding edits and retirement. Callbacks change PTEs/flags only; all - // logical aliases stay linked and retain their borrowed BO/GEM. - let _ = unsafe { inner.for_each_private_bo(|inner, bo| { - if unpinned >= target { return Ok(false); } - let object = bo.object(); - if !object.idle_reclaim_candidate() { return Ok(true); } - { - // Avoid inversion against preparation (VmBoInner -> resv). - let Some(bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; - if bo_inner.sgt.is_none() || !object.sole_sg_lease_releasable_locked() { - return Ok(true); - } - } - // The association retains SG backing until *all* aliases have - // been invalidated and their device translations flushed. A - // contended cursor or PTE failure retains backing and the touched - // INVALIDATED prefix for restoration or a later reclaim attempt. - inner.invalidate_private_bo_mappings(bo, |inner, start, range| { - reclaim.mark_changed(); - let end = start.checked_add(range).ok_or(EOVERFLOW)?; - let _change = inner.mapping_mutation(); - inner.page_table.discard_partial_map(start..end)?; - inner.tlbi_context_mask(start, range as usize, contexts); - Ok(()) - })?; - { - let Some(mut bo_inner) = bo.inner().inner.try_lock() else { return Ok(true); }; - bo_inner.sg_vec = None; - bo_inner.sgt = None; - } - // Every private GPUVA alias was removed from the device page - // tables above. Kernel mappings and other SG owners are excluded; - // the C/Rust backing helper rechecks exclusion before release. - if object.release_idle_pages_locked().is_ok() { - unpinned = unpinned.saturating_add(object.size() / kernel::page::PAGE_SIZE); - } - Ok(unpinned < target) - }) }; - drop(shared); - drop(inner); - - purged + unpinned - } - - fn restore_idle_pages(&self) -> Result { - // Unbind/free can remove or split invalidated mappings while the VM - // stays evicted. Rebuild solely from the authoritative logical GPUVAs, - // never from stale pre-eviction records that could resurrect a BO. - // Gate::enter holds exclusive residency ownership for this entire - // walk: metadata edits, retirement and reclaim cannot change the tree. - // Retain one invalidated mapping at a time, avoiding a VM-sized KVec - // allocation precisely when memory pressure requires restoration. - let mut cursor = 0u64; - loop { - let map = loop { - if let Some(inner) = self.inner.try_lock_private() { - let map = inner.next_private_mapping( - &mut cursor, gpuvm::GpuVaFlags::INVALIDATED, - )?; - // Purged (DONTNEED) objects have no contents to restore; - // their mappings stay INVALIDATED until unbound. - // SAFETY: The private guard holds the VM reservation. - let purged = map.as_ref().is_some_and(|map| unsafe { map.object.madv_locked() < 0 }); - break (map, purged); - } - // Wait for reservation ownership without retaining a mapping. - drop(self.inner.exec_lock(None, true)?); - }; - // The reservation guard has dropped before pinning backing, and - // before any returned object reference can be released on failure. - let (Some(map), purged) = map else { break; }; - if purged { - continue; - } - let mut restored = 0u64; - let bo_owner = self.inner.find_bo(&map.object).ok_or(EFAULT)?; - { - let needs_sgt = bo_owner.inner().inner.lock().sgt.is_none(); - if needs_sgt { - // Preserve review1's reservation/BO lock order: the getter - // may acquire the object reservation; never hold BO here. - let sgt = map.object.owned_sg_table()?; - let mut bo = bo_owner.inner().inner.lock(); - if bo.sgt.is_none() { bo.sgt = Some(sgt); } - drop(bo); - } - } - let mut inner = self.inner.exec_lock(None, true)?; - let bo = bo_owner.inner().inner.lock(); - let sgt = bo.sgt.as_ref().ok_or(EFAULT)?; - let result = (|| -> Result { - let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; - let _mutation = inner.mapping_mutation(); - inner.page_table.discard_partial_map(map.addr..end)?; - inner.tlbi_contexts_naming_root(map.addr, map.range as usize); - let prot = map.inner.prot.ok_or(EINVAL)?; - let repeat = map.flags.contains(gpuvm::GpuVaFlags::REPEAT); - let mut skip = map - .offset - .checked_add(if repeat { 0 } else { restored }) - .ok_or(EOVERFLOW)? as usize; - let mut left = map.range.checked_sub(restored).ok_or(EOVERFLOW)? as usize; - for entry in sgt.iter() { - if left == 0 { - break; - } - let len = entry.dma_len() as usize; - if skip >= len { - skip -= len; - continue; - } - let physical = entry.dma_address() as usize + skip; - let bytes = if repeat { left } else { left.min(len - skip) }; - let start = map.addr.checked_add(restored).ok_or(EOVERFLOW)?; - let end = start.checked_add(bytes as u64).ok_or(EOVERFLOW)?; - let _change = inner.mapping_mutation(); - inner.page_table.map_pages( - start..end, - physical as PhysicalAddr, - prot, - repeat, - )?; - restored += bytes as u64; - left -= bytes; - skip = 0; - } - if left != 0 { - return Err(EFAULT); - } - // The exclusive residency gate excludes submissions until - // every invalidated mapping is restored. Publish this logical - // range once, instead of issuing a synchronized TLBI for each - // physical SG segment. Keep the preceding break-before-make - // invalidation; the error path clears and flushes the entire - // range if any map operation (including a partial one) fails. - inner.tlbi_contexts_naming_root(map.addr, map.range as usize); - inner.set_invalidated(map.addr, map.range, false)?; - - Ok(()) - })(); - if let Err(error) = result { - // Keep an invalidated GPUVA wholly unmapped even on partial - // restoration failure, so later unbind/close cannot skip a - // prefix that was already reinstalled. Backing stays retained. - // map_pages may fail inside its current SG segment. Clear the - // complete logical range, including that partially written segment. - let end = map.addr.checked_add(map.range).ok_or(EOVERFLOW)?; - let _mutation = inner.mapping_mutation(); - inner.page_table.discard_partial_map(map.addr..end)?; - inner.tlbi_contexts_naming_root(map.addr, map.range as usize); - return Err(error); - } - } - Ok(()) - } - - pub(super) fn enter_retirement_residency(&self) -> Option { - self.residency.as_ref().and_then(Gate::enter_retirement) - } -} diff --git a/drivers/gpu/drm/asahi-neo/mmu/shared.rs b/drivers/gpu/drm/asahi-neo/mmu/shared.rs deleted file mode 100644 index 5b87c55a06b72b..00000000000000 --- a/drivers/gpu/drm/asahi-neo/mmu/shared.rs +++ /dev/null @@ -1,134 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Shared-object identities for attachment ordering across user address spaces. -//! These records do not own mappings. The VM's job guard protects the mappings -//! while submission captures the intersecting object offsets. Mapping commits -//! hold the GPUVM exec lock through metadata updates, then release it. The lock -//! order is GPUVM exec followed by bindings; footprint readers take only bindings. - -use super::*; -use crate::g17::{hazard::Footprint, Attachments}; - -struct Binding { - start: u64, - end: u64, - object: usize, - offset: u64, -} - -#[pin_data] -pub(super) struct SharedBindings { - #[pin] - bindings: Mutex>, - count: AtomicU32, -} - -impl SharedBindings { - pub(super) fn new() -> Result> { - Arc::pin_init( - pin_init!(Self { - bindings <- new_mutex!(KVec::new(), "Shared attachment bindings"), - count: AtomicU32::new(0), - }), - GFP_KERNEL, - ) - } -} - -impl Vm { - pub(super) fn track_shared_binding( - &self, - gem: &gem::Object, - address: u64, - size: u64, - offset: u64, - single_page: bool, - ) { - let Some(shared) = &self.shared_bindings else { - return; - }; - let Some(end) = address.checked_add(size) else { - return; - }; - let external = !single_page && self.is_extobj(gem); - if !external && shared.count.load(Ordering::Relaxed) == 0 { - return; - } - let mut bindings = shared.bindings.lock(); - bindings.retain(|entry| entry.end <= address || end <= entry.start); - if external { - let _ = bindings.push( - Binding { - start: address, - end, - object: gem as *const gem::Object as usize, - offset, - }, - GFP_KERNEL, - ); - } - shared.count.store(bindings.len() as u32, Ordering::Relaxed); - } - - pub(super) fn untrack_shared_range(&self, address: u64, size: u64) { - let Some(shared) = &self.shared_bindings else { - return; - }; - if shared.count.load(Ordering::Relaxed) == 0 { - return; - } - let Some(end) = address.checked_add(size) else { - return; - }; - let mut bindings = shared.bindings.lock(); - bindings.retain(|entry| entry.end <= address || end <= entry.start); - shared.count.store(bindings.len() as u32, Ordering::Relaxed); - } - - pub(crate) fn untrack_shared_object(&self, gem: &gem::Object) { - let Some(shared) = &self.shared_bindings else { - return; - }; - if shared.count.load(Ordering::Relaxed) == 0 { - return; - } - let object = gem as *const gem::Object as usize; - let mut bindings = shared.bindings.lock(); - bindings.retain(|entry| entry.object != object); - shared.count.store(bindings.len() as u32, Ordering::Relaxed); - } - - pub(crate) fn render_footprint( - &self, - attachments: &Attachments, - ) -> Result>> { - if attachments.as_slice().is_empty() { - return Ok(None); - } - let mut footprint = Footprint::new(self.id); - let bindings = self.shared_bindings.as_ref().and_then(|shared| { - (shared.count.load(Ordering::Relaxed) != 0).then(|| shared.bindings.lock()) - }); - for attachment in attachments.as_slice() { - let end = attachment.va.saturating_add(attachment.size); - footprint.push_vm(attachment.va, end); - let Some(bindings) = &bindings else { - continue; - }; - for entry in bindings.iter() { - let first = entry.start.max(attachment.va); - let last = entry.end.min(end); - if first < last { - let offset = entry.offset.saturating_add(first - entry.start); - footprint.push_object( - entry.object, - offset, - offset.saturating_add(last - first), - ); - } - } - } - drop(bindings); - Ok(Some(Arc::new(footprint, GFP_KERNEL)?)) - } -} diff --git a/drivers/gpu/drm/asahi-neo/object.rs b/drivers/gpu/drm/asahi-neo/object.rs deleted file mode 100644 index 38a2268137effb..00000000000000 --- a/drivers/gpu/drm/asahi-neo/object.rs +++ /dev/null @@ -1,733 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Asahi GPU object model -//! -//! The AGX GPU includes a coprocessor that uses a large number of shared memory structures to -//! communicate with the driver. These structures contain GPU VA pointers to each other, which are -//! directly dereferenced by the firmware and are expected to always be valid for the usage -//! lifetime of the containing struct (which is an implicit contract, not explicitly managed). -//! Any faults cause an unrecoverable firmware crash, requiring a full system reboot. -//! -//! In order to manage this complexity safely, we implement a GPU object model using Rust's type -//! system to enforce GPU object lifetime relationships. GPU objects represent an allocated piece -//! of memory of a given type, mapped to the GPU (and usually also the CPU). On the CPU side, -//! these objects are associated with a pure Rust structure that contains the objects it depends -//! on (or references to them). This allows us to map Rust lifetimes into the GPU object model -//! system. Then, GPU VA pointers also inherit those lifetimes, which means the Rust borrow checker -//! can ensure that all pointers are assigned an address that is guaranteed to outlive the GPU -//! object it points to. -//! -//! Since the firmware object model does have self-referencing pointers (and there is of course no -//! underlying revocability mechanism to make it safe), we must have an escape hatch. GPU pointers -//! can be weak pointers, which do not enforce lifetimes. In those cases, it is the user's -//! responsibility to ensure that lifetime requirements are met. -//! -//! In other words, the model is necessarily leaky and there is no way to fully map Rust safety to -//! GPU firmware object safety. The goal of the model is to make it easy to model the lifetimes of -//! GPU objects and have the compiler help in avoiding mistakes, rather than to guarantee safety -//! 100% of the time as would be the case for CPU-side Rust code. - -// TODO: There is a fundamental soundness issue with sharing memory with the GPU (that even affects -// C code too). Since the GPU is free to mutate that memory at any time, normal reference invariants -// cannot be enforced on the CPU side. For example, the compiler could perform an optimization that -// assumes that a given memory location does not change between two reads, and causes UB otherwise, -// and then the GPU could mutate that memory out from under the CPU. -// -// For cases where we *expect* this to happen, we use atomic types, which avoid this issue. However, -// doing so for every single field of every type is a non-starter. Right now, there seems to be no -// good solution for this that does not come with significant performance or ergonomics downsides. -// -// In *practice* we are almost always only writing GPU memory, and only reading from atomics, so the -// chances of this actually triggering UB (e.g. a security issue that can be triggered from the GPU -// side) due to a compiler optimization are very slim. -// -// Further discussion: https://github.com/rust-lang/unsafe-code-guidelines/issues/152 - -use kernel::{ - error::code::*, - prelude::*, - sync::Arc, // -}; - -use core::fmt; -use core::fmt::Debug; -use core::fmt::Formatter; -use core::marker::PhantomData; -use core::mem::MaybeUninit; -use core::num::NonZeroU64; -use core::ops::{ - Deref, - DerefMut, - Index, - IndexMut, // -}; -use core::{mem, ptr, slice}; - -use crate::alloc::Allocation; -use crate::debug::*; -use crate::fw::types::Zeroable; -use crate::mmu; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Object; - -/// A GPU-side strong pointer, which is a 64-bit non-zero VA with an associated lifetime. -/// -/// In rare cases these pointers are not aligned, so this is `packed(1)`. -#[repr(C, packed(1))] -pub(crate) struct GpuPointer<'a, T: ?Sized>(NonZeroU64, PhantomData<&'a T>); - -impl<'a, T: ?Sized> GpuPointer<'a, T> { - /// Logical OR the pointer with an arbitrary `u64`. This is used when GPU struct fields contain - /// misc flag fields in the upper bits. The lifetime is retained. This is GPU-unsafe in - /// principle, but we assert that only non-implemented address bits are touched, which is safe - /// for pointers used by the GPU (not by firmware). - pub(crate) fn or(&self, other: u64) -> GpuPointer<'a, T> { - // This will fail for kernel-half pointers, which should not be ORed. - assert_eq!(self.0.get() & other, 0); - // Assert that we only touch the high bits. - assert_eq!(other & 0xffffffffff, 0); - GpuPointer(self.0 | other, PhantomData) - } - - /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and - /// should only be used via the `inner_ptr` macro to get pointers to inner fields, hence we mark - /// it `unsafe` to discourage direct use. - /// - /// # Safety - /// Do not use directly, only via `inner_ptr`. - // NOTE: The third argument is a type inference hack. - pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuPointer<'a, U> { - GpuPointer::<'a, U>( - NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), - PhantomData, - ) - } -} - -impl<'a, T> GpuPointer<'a, T> { - /// Create a GPU pointer from a KernelMapping and an offset. - /// TODO: Change all GPU pointers to point to the raw types so size_of here is GPU-sound. - pub(crate) fn from_mapping( - mapping: &'a Arc, - offset: usize, - ) -> Result> { - let addr = mapping.iova().checked_add(offset as u64).ok_or(EINVAL)?; - let end = offset - .checked_add(core::mem::size_of::()) - .ok_or(EINVAL)?; - if end > mapping.size() { - Err(ERANGE) - } else { - Ok(Self(addr.try_into().unwrap(), PhantomData)) - } - } -} - -impl<'a, T: ?Sized> Debug for GpuPointer<'a, T> { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let val = self.0; - f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) - } -} - -impl<'a, T: ?Sized> From> for u64 { - fn from(value: GpuPointer<'a, T>) -> Self { - value.0.get() - } -} - -/// Take a pointer to a sub-field within a structure pointed to by a GpuPointer, keeping the -/// lifetime. -#[macro_export] -macro_rules! inner_ptr { - ($gpuva:expr, $($f:tt)*) => ({ - // This mirrors kernel::offset_of(), except we use type inference to avoid having to know - // the type of the pointer explicitly. - fn uninit_from(_: GpuPointer<'_, T>) -> core::mem::MaybeUninit> { - core::mem::MaybeUninit::uninit() - } - let tmp = uninit_from($gpuva); - let outer = tmp.as_ptr(); - // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that - // we don't actually read from `outer` (which would be UB) nor create an intermediate - // reference. - let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; - // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; - // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer - // object. - unsafe { $gpuva.offset(off.try_into().unwrap(), p) } - }) -} - -/// A GPU-side weak pointer, which is a 64-bit non-zero VA with no lifetime. -/// -/// In rare cases these pointers are not aligned, so this is `packed(1)`. -#[repr(C, packed(1))] -pub(crate) struct GpuWeakPointer(NonZeroU64, PhantomData<*const T>); - -/// SAFETY: GPU weak pointers are always safe to share between threads. -unsafe impl Send for GpuWeakPointer {} -/// SAFETY: GPU weak pointers are always safe to share between threads. -unsafe impl Sync for GpuWeakPointer {} - -// Weak pointers can be copied/cloned regardless of their target type. -impl Copy for GpuWeakPointer {} - -impl Clone for GpuWeakPointer { - fn clone(&self) -> Self { - *self - } -} - -impl GpuWeakPointer { - /// Add an arbitrary offset to the pointer. This is not safe (from the GPU perspective), and - /// should only be used via the `inner_weak_ptr` macro to get pointers to inner fields, hence we - /// mark it `unsafe` to discourage direct use. - /// - /// # Safety - /// Do not use directly, only via `inner_weak_ptr`. - // NOTE: The third argument is a type inference hack. - pub(crate) unsafe fn offset(&self, off: usize, _: *const U) -> GpuWeakPointer { - GpuWeakPointer::( - NonZeroU64::new(self.0.get() + (off as u64)).unwrap(), - PhantomData, - ) - } - - /// Upgrade a weak pointer into a strong pointer. This is not considered safe from the GPU - /// perspective. - /// - /// # Safety - /// The caller must ensure tht the data pointed to lives in the GPU at least as long as the - /// returned lifetime. - pub(crate) unsafe fn upgrade<'a>(&self) -> GpuPointer<'a, T> { - GpuPointer(self.0, PhantomData) - } -} - -impl From> for u64 { - fn from(value: GpuWeakPointer) -> Self { - value.0.get() - } -} - -impl Debug for GpuWeakPointer { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let val = self.0; - f.write_fmt(format_args!("{:#x} ({})", val, core::any::type_name::())) - } -} - -/// Take a pointer to a sub-field within a structure pointed to by a GpuWeakPointer. -#[macro_export] -macro_rules! inner_weak_ptr { - ($gpuva:expr, $($f:tt)*) => ({ - // See inner_ptr() - fn uninit_from(_: GpuWeakPointer) -> core::mem::MaybeUninit> { - core::mem::MaybeUninit::uninit() - } - let tmp = uninit_from($gpuva); - let outer = tmp.as_ptr(); - // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that - // we don't actually read from `outer` (which would be UB) nor create an intermediate - // reference. - let p: *const _ = unsafe { core::ptr::addr_of!((*outer).$($f)*) }; - let inner = p as *const u8; - // SAFETY: The two pointers are within the same allocation block. - let off = unsafe { inner.offset_from(outer as *const u8) }; - // SAFETY: The resulting pointer is guaranteed to point to valid memory within the outer - // object. - unsafe { $gpuva.offset(off.try_into().unwrap(), p) } - }) -} - -/// Types that implement this trait represent a GPU structure from the CPU side. -/// -/// The `Raw` type represents the actual raw structure definition on the GPU side. -/// -/// Types implementing [`GpuStruct`] must have fields owning any objects (or strong references -/// to them) that GPU pointers in the `Raw` structure point to. This mechanism is used to enforce -/// lifetimes. -pub(crate) trait GpuStruct: 'static { - /// The type of the GPU-side structure definition representing the firmware struct layout. - type Raw<'a>; -} - -/// An instance of a GPU object in memory. -/// -/// # Invariants -/// `raw` must point to a valid mapping of the `T::Raw` type associated with the `alloc` allocation. -/// `gpu_ptr` must be the GPU address of the same object. -pub(crate) struct GpuObject> { - raw: *mut T::Raw<'static>, - alloc: U, - gpu_ptr: GpuWeakPointer, - inner: KBox, -} - -impl> GpuObject { - /// Create a new GpuObject given an allocator and the inner data (a type implementing - /// GpuStruct). - /// - /// The caller passes a closure that constructs the `T::Raw` type given a reference to the - /// `GpuStruct`. This is the mechanism used to enforce lifetimes. - pub(crate) fn new( - alloc: U, - inner: T, - callback: impl for<'a> FnOnce(&'a T) -> T::Raw<'a>, - ) -> Result { - let size = mem::size_of::>(); - if size > 0x1000 { - dev_crit!( - alloc.device().as_ref(), - "Allocating {} of size {:#x}, with new, please use new_boxed!\n", - core::any::type_name::(), - size - ); - } - if alloc.size() < size { - return Err(ENOMEM); - } - let gpu_ptr = - GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); - mod_dev_dbg!( - alloc.device(), - "Allocating {} @ {:#x}\n", - core::any::type_name::(), - alloc.gpu_ptr() - ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'static>; - let mut raw = callback(&inner); - // SAFETY: `p` is guaranteed to be valid per the Allocation invariant, and the type is - // identical to the type of `raw` other than the lifetime. - unsafe { p.copy_from(&mut raw as *mut _ as *mut u8 as *mut _, 1) }; - mem::forget(raw); - Ok(Self { - raw: p, - gpu_ptr, - alloc, - inner: KBox::new(inner, GFP_KERNEL)?, - }) - } - - /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing - /// GpuStruct). - /// - /// The caller passes a closure that initializes the `T::Raw` type given a reference to the - /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() - /// macro to avoid constructing the whole `T::Raw` object on the stack. - pub(crate) fn new_boxed( - alloc: U, - inner: KBox, - callback: impl for<'a> FnOnce( - &'a T, - &'a mut MaybeUninit>, - ) -> Result<&'a mut T::Raw<'a>>, - ) -> Result { - if alloc.size() < mem::size_of::>() { - return Err(ENOMEM); - } - let gpu_ptr = - GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); - mod_dev_dbg!( - alloc.device(), - "Allocating {} @ {:#x}\n", - core::any::type_name::(), - alloc.gpu_ptr() - ); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut MaybeUninit>; - // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - let raw = callback(&inner, unsafe { &mut *p })?; - if p as *mut T::Raw<'_> != raw as *mut _ { - dev_err!( - alloc.device().as_ref(), - "Allocation callback returned a mismatched reference ({})\n", - core::any::type_name::(), - ); - return Err(EINVAL); - } - Ok(Self { - raw: p as *mut u8 as *mut T::Raw<'static>, - gpu_ptr, - alloc, - inner, - }) - } - - /// Create a new GpuObject given an allocator and the inner data (a type implementing - /// GpuStruct). - /// - /// The caller passes a closure that initializes the `T::Raw` type given a reference to the - /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() - /// macro to avoid constructing the whole `T::Raw` object on the stack. - pub(crate) fn new_inplace( - alloc: U, - inner: T, - callback: impl for<'a> FnOnce( - &'a T, - &'a mut MaybeUninit>, - ) -> Result<&'a mut T::Raw<'a>>, - ) -> Result { - GpuObject::::new_boxed(alloc, KBox::new(inner, GFP_KERNEL)?, callback) - } - - /// Create a new GpuObject given an allocator and the boxed inner data (a type implementing - /// GpuStruct). - /// - /// The caller passes a closure that initializes the `T::Raw` type given a reference to the - /// `GpuStruct` and a `MaybeUninit`. This is intended to be used with the place!() - /// macro to avoid constructing the whole `T::Raw` object on the stack. - pub(crate) fn new_init_prealloc<'a, I: Init, R: PinInit, F>, E, F>( - alloc: U, - inner_init: impl FnOnce(GpuWeakPointer) -> I, - raw_init: impl FnOnce(&'a T, GpuWeakPointer) -> R, - ) -> Result - where - kernel::error::Error: core::convert::From, - kernel::error::Error: core::convert::From, - { - if alloc.size() < mem::size_of::>() { - return Err(ENOMEM); - } - let gpu_ptr = - GpuWeakPointer::(NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?, PhantomData); - mod_dev_dbg!( - alloc.device(), - "Allocating {} @ {:#x}\n", - core::any::type_name::(), - alloc.gpu_ptr() - ); - let inner = inner_init(gpu_ptr); - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr() as *mut T::Raw<'_>; - let ret = Self { - raw: p as *mut u8 as *mut T::Raw<'static>, - gpu_ptr, - alloc, - inner: KBox::init(inner, GFP_KERNEL)?, - }; - let q = &*ret.inner as *const T; - // SAFETY: `p` is guaranteed to be valid per the Allocation invariant. - unsafe { raw_init(&*q, gpu_ptr).__pinned_init(p) }?; - Ok(ret) - } - - /// Returns the GPU VA of this object (as a raw [`NonZeroU64`]) - pub(crate) fn gpu_va(&self) -> NonZeroU64 { - self.gpu_ptr.0 - } - - /// Returns a strong GPU pointer to this object, with a lifetime. - pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, T> { - GpuPointer(self.gpu_ptr.0, PhantomData) - } - - /// Returns a weak GPU pointer to this object, with no lifetime. - pub(crate) fn weak_pointer(&self) -> GpuWeakPointer { - GpuWeakPointer(self.gpu_ptr.0, PhantomData) - } - - /// Perform a mutation to the inner `Raw` data given a user-supplied callback. - /// - /// The callback gets a mutable reference to the `GpuStruct` type. - pub(crate) fn with_mut( - &mut self, - callback: impl for<'a> FnOnce(&'a mut ::Raw<'a>, &'a mut T) -> RetVal, - ) -> RetVal { - // SAFETY: `self.raw` is valid per the type invariant, and the second half is just - // converting lifetimes. - unsafe { callback(&mut *self.raw, &mut *(&mut *self.inner as *mut _)) } - } - - /// Access the inner `Raw` data given a user-supplied callback. - /// - /// The callback gets a reference to the `GpuStruct` type. - pub(crate) fn with( - &self, - callback: impl for<'a> FnOnce(&'a ::Raw<'a>, &'a T) -> RetVal, - ) -> RetVal { - // SAFETY: `self.raw` is valid per the type invariant, and the second half is just - // converting lifetimes. - unsafe { callback(&*self.raw, &*(&*self.inner as *const _)) } - } -} - -impl> Deref for GpuObject { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl> DerefMut for GpuObject { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} - -impl> Debug for GpuObject -where - ::Raw<'static>: Debug, -{ - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct(core::any::type_name::()) - // SAFETY: `self.raw` is valid per the type invariant. - .field("raw", &format_args!("{:#X?}", unsafe { &*self.raw })) - .field("inner", &format_args!("{:#X?}", &self.inner)) - .field("alloc", &format_args!("{:?}", &self.alloc)) - .finish() - } -} - -impl> GpuObject -where - for<'a> ::Raw<'a>: Default + Zeroable, -{ - /// Create a new GpuObject with default data. `T` must implement `Default` and `T::Raw` must - /// implement `Zeroable`, since the GPU-side memory is initialized by zeroing. - pub(crate) fn new_default(alloc: U) -> Result { - GpuObject::::new_inplace(alloc, Default::default(), |_inner, raw| { - // SAFETY: `raw` is valid here, and `T::Raw` implements `Zeroable`. - Ok(unsafe { - ptr::write_bytes(raw, 0, 1); - (*raw).assume_init_mut() - }) - }) - } -} - -impl> Drop for GpuObject { - fn drop(&mut self) { - mod_dev_dbg!( - self.alloc.device(), - "Dropping {} @ {:?}\n", - core::any::type_name::(), - self.gpu_pointer() - ); - } -} - -// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send -unsafe impl> Send for GpuObject {} -// SAFETY: GpuObjects are Send as long as the GpuStruct itself is Send -unsafe impl> Sync for GpuObject {} - -/// Trait used to erase the type of a GpuObject, used when we need to keep a list of heterogenous -/// objects around. -pub(crate) trait OpaqueGpuObject: Send + Sync { - fn gpu_va(&self) -> NonZeroU64; -} - -impl> OpaqueGpuObject for GpuObject { - fn gpu_va(&self) -> NonZeroU64 { - Self::gpu_va(self) - } -} - -/// An array of raw GPU objects that is only accessible to the GPU (no CPU-side mapping required). -/// -/// This must necessarily be uninitialized as far as the GPU is concerned, so it cannot be used -/// when initialization is required. -/// -/// # Invariants -/// -/// `alloc` is valid and at least as large as `len` times the size of one `T`. -/// `gpu_ptr` is valid and points to the allocation start. -pub(crate) struct GpuOnlyArray> { - len: usize, - alloc: U, - gpu_ptr: NonZeroU64, - _p: PhantomData, -} - -impl> GpuOnlyArray { - /// Allocate a new GPU-only array with the given length. - pub(crate) fn new(alloc: U, count: usize) -> Result> { - let bytes = count * mem::size_of::(); - let gpu_ptr = NonZeroU64::new(alloc.gpu_ptr()).ok_or(EINVAL)?; - if alloc.size() < bytes { - return Err(ENOMEM); - } - Ok(Self { - len: count, - alloc, - gpu_ptr, - _p: PhantomData, - }) - } - - /// Returns the GPU VA of this arraw (as a raw [`NonZeroU64`]) - pub(crate) fn gpu_va(&self) -> NonZeroU64 { - self.gpu_ptr - } - - /// Returns a strong GPU pointer to this array, with a lifetime. - pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, &'_ [T]> { - GpuPointer(self.gpu_ptr, PhantomData) - } - - /// Returns a weak GPU pointer to this array, with no lifetime. - pub(crate) fn weak_pointer(&self) -> GpuWeakPointer<[T]> { - GpuWeakPointer(self.gpu_ptr, PhantomData) - } - - /// Returns a pointer to an offset within the array (as a subslice). - pub(crate) fn gpu_offset_pointer(&self, offset: usize) -> GpuPointer<'_, &'_ [T]> { - if offset > self.len { - panic!("Index {} out of bounds (len: {})", offset, self.len); - } - GpuPointer( - NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), - PhantomData, - ) - } - - /* Not used yet - /// Returns a weak pointer to an offset within the array (as a subslice). - pub(crate) fn weak_offset_pointer(&self, offset: usize) -> GpuWeakPointer<[T]> { - if offset > self.len { - panic!("Index {} out of bounds (len: {})", offset, self.len); - } - GpuWeakPointer( - NonZeroU64::new(self.gpu_ptr.get() + (offset * mem::size_of::()) as u64).unwrap(), - PhantomData, - ) - } - - /// Returns a pointer to an element within the array. - pub(crate) fn gpu_item_pointer(&self, index: usize) -> GpuPointer<'_, &'_ T> { - if index >= self.len { - panic!("Index {} out of bounds (len: {})", index, self.len); - } - GpuPointer( - NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), - PhantomData, - ) - } - */ - - /// Returns a weak pointer to an element within the array. - pub(crate) fn weak_item_pointer(&self, index: usize) -> GpuWeakPointer { - if index >= self.len { - panic!("Index {} out of bounds (len: {})", index, self.len); - } - GpuWeakPointer( - NonZeroU64::new(self.gpu_ptr.get() + (index * mem::size_of::()) as u64).unwrap(), - PhantomData, - ) - } - - /// Returns the length of the array. - pub(crate) fn len(&self) -> usize { - self.len - } -} - -impl> Debug for GpuOnlyArray { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct(core::any::type_name::()) - .field("len", &format_args!("{:#X?}", self.len())) - .finish() - } -} - -impl> Drop for GpuOnlyArray { - fn drop(&mut self) { - mod_dev_dbg!( - self.alloc.device(), - "Dropping {} @ {:?}\n", - core::any::type_name::(), - self.gpu_pointer() - ); - } -} - -/// An array of raw GPU objects that is also CPU-accessible. -/// -/// # Invariants -/// -/// `raw` is valid and points to the CPU-side view of the array (which must have one). -pub(crate) struct GpuArray> { - raw: *mut T, - array: GpuOnlyArray, -} - -impl> GpuArray { - /// Allocate a new GPU array, initializing each element to its default. - pub(crate) fn empty(alloc: U, count: usize) -> Result> { - let p = alloc.ptr().ok_or(EINVAL)?.as_ptr(); - let inner = GpuOnlyArray::new(alloc, count)?; - let mut pi = p; - for _i in 0..count { - // SAFETY: `pi` is valid per the Allocation type invariant, and GpuOnlyArray guarantees - // that it can never iterate beyond the buffer length. - unsafe { - pi.write(Default::default()); - pi = pi.add(1); - } - } - Ok(Self { - raw: p, - array: inner, - }) - } -} - -impl> GpuArray { - /// Get a slice view of the array contents. - pub(crate) fn as_slice(&self) -> &[T] { - // SAFETY: self.raw / self.len are valid per the type invariant - unsafe { slice::from_raw_parts(self.raw, self.len) } - } - - /// Get a mutable slice view of the array contents. - pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { - // SAFETY: self.raw / self.len are valid per the type invariant - unsafe { slice::from_raw_parts_mut(self.raw, self.len) } - } -} - -impl> Deref for GpuArray { - type Target = GpuOnlyArray; - - fn deref(&self) -> &GpuOnlyArray { - &self.array - } -} - -impl> Index for GpuArray { - type Output = T; - - fn index(&self, index: usize) -> &T { - if index >= self.len { - panic!("Index {} out of bounds (len: {})", index, self.len); - } - // SAFETY: This is bounds checked above - unsafe { &*(self.raw.add(index)) } - } -} - -impl> IndexMut for GpuArray { - fn index_mut(&mut self, index: usize) -> &mut T { - if index >= self.len { - panic!("Index {} out of bounds (len: {})", index, self.len); - } - // SAFETY: This is bounds checked above - unsafe { &mut *(self.raw.add(index)) } - } -} - -// SAFETY: GpuArray are Send as long as the contained type itself is Send -unsafe impl> Send for GpuArray {} -// SAFETY: GpuArray are Sync as long as the contained type itself is Sync -unsafe impl> Sync for GpuArray {} - -impl> Debug for GpuArray { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct(core::any::type_name::()) - .field("array", &format_args!("{:#X?}", self.as_slice())) - .finish() - } -} diff --git a/drivers/gpu/drm/asahi-neo/pgtable.rs b/drivers/gpu/drm/asahi-neo/pgtable.rs deleted file mode 100644 index 03001892d98bb4..00000000000000 --- a/drivers/gpu/drm/asahi-neo/pgtable.rs +++ /dev/null @@ -1,892 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! UAT Page Table management -//! -//! AGX GPUs use an MMU called the UAT, which is largely compatible with the ARM64 page table -//! format. This module manages the actual page tables by allocating raw memory pages from -//! the kernel page allocator. - -use core::fmt::Debug; -use core::mem::size_of; -use core::ops::Range; -use core::sync::atomic::{ - AtomicU64, - Ordering, // -}; - -use kernel::{ - addr::PhysicalAddr, - error::Result, - page::Page, - prelude::*, // -}; -#[cfg(CONFIG_DEV_COREDUMP)] -use kernel::{ - types::Owned, - uapi::{ - PF_R, - PF_W, - PF_X, // - }, -}; - -use crate::debug::*; -use crate::util::align; - -const DEBUG_CLASS: DebugFlags = DebugFlags::PgTable; - -/// Number of bits in a page offset. -pub(crate) const UAT_PGBIT: usize = 14; -/// UAT page size. -pub(crate) const UAT_PGSZ: usize = 1 << UAT_PGBIT; -/// UAT page offset mask. -pub(crate) const UAT_PGMSK: usize = UAT_PGSZ - 1; - -type Pte = AtomicU64; - -const PTE_BIT: usize = 3; // log2(sizeof(Pte)) -const PTE_SIZE: usize = 1 << PTE_BIT; - -/// Number of PTEs per page. -const UAT_NPTE: usize = UAT_PGSZ / size_of::(); - -/// Number of address bits to address a level -const UAT_LVBIT: usize = UAT_PGBIT - PTE_BIT; -/// Number of entries per level -const UAT_LVSZ: usize = UAT_NPTE; -/// Mask of level bits -const UAT_LVMSK: u64 = (UAT_LVSZ - 1) as u64; - -const UAT_LEVELS: usize = 3; - -/// Number of address bits below the index of a top-level page table entry. -pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; - -/// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. -/// -/// # Safety -/// -/// `ttb` must be the physical address of a page table root in RAM that stays allocated while `f` -/// runs. Other agents (the GPU firmware) may access the entries concurrently, which is why they -/// are only exposed as atomics. -pub(crate) unsafe fn with_root_entries( - ttb: PhysicalAddr, - count: usize, - f: impl FnOnce(&[AtomicU64]) -> R, -) -> Result { - if count > UAT_NPTE { - return Err(EINVAL); - } - // SAFETY: The caller guarantees that `ttb` is an allocated page in RAM. - let page = unsafe { Page::borrow_phys(&ttb) }.ok_or(EIO)?; - page.with_pointer_into_page(0, count * PTE_SIZE, |p| { - // SAFETY: with_pointer_into_page() ensures the pointer is valid for `count` PTEs, and page - // table entries are naturally aligned. - Ok(f(unsafe { - core::slice::from_raw_parts(p as *const AtomicU64, count) - })) - }) -} - -const PTE_TYPE_BITS: u64 = 3; -const PTE_TYPE_LEAF_TABLE: u64 = 3; - -const UAT_NON_GLOBAL: u64 = 1 << 11; -const UAT_AP_SHIFT: u32 = 6; -const UAT_AP_BITS: u64 = 3 << UAT_AP_SHIFT; -const UAT_HIGH_BITS_SHIFT: u32 = 52; -const UAT_HIGH_BITS: u64 = 0xfff << UAT_HIGH_BITS_SHIFT; -const UAT_MEMATTR_SHIFT: u32 = 2; -const UAT_MEMATTR_BITS: u64 = 7 << UAT_MEMATTR_SHIFT; - -const UAT_PROT_BITS: u64 = UAT_AP_BITS | UAT_MEMATTR_BITS | UAT_HIGH_BITS; - -const UAT_AF: u64 = 1 << 10; - -const MEMATTR_CACHED: u8 = 0; -const MEMATTR_DEV: u8 = 1; -const MEMATTR_UNCACHED: u8 = 2; - -const AP_FW_GPU: u8 = 0; -const AP_FW: u8 = 1; -const AP_GPU: u8 = 2; - -const HIGH_BITS_PXN: u16 = 1 << 1; -const HIGH_BITS_UXN: u16 = 1 << 2; -const HIGH_BITS_GPU_ACCESS: u16 = 1 << 3; - -#[cfg(CONFIG_DEV_COREDUMP)] -pub(crate) const PTE_ADDR_BITS: u64 = (!UAT_PGMSK as u64) & (!UAT_HIGH_BITS); - -#[derive(Debug, Copy, Clone)] -pub(crate) struct Prot { - memattr: u8, - ap: u8, - high_bits: u16, -} - -// Firmware + GPU access -const PROT_FW_GPU_NA: Prot = Prot::from_bits(AP_FW_GPU, 0, 0); -const _PROT_FW_GPU_RO: Prot = Prot::from_bits(AP_FW_GPU, 0, 1); -const _PROT_FW_GPU_WO: Prot = Prot::from_bits(AP_FW_GPU, 1, 0); -const PROT_FW_GPU_RW: Prot = Prot::from_bits(AP_FW_GPU, 1, 1); - -// Firmware only access -const PROT_FW_RO: Prot = Prot::from_bits(AP_FW, 0, 0); -const _PROT_FW_NA: Prot = Prot::from_bits(AP_FW, 0, 1); -const PROT_FW_RW: Prot = Prot::from_bits(AP_FW, 1, 0); -const PROT_FW_RW_GPU_RO: Prot = Prot::from_bits(AP_FW, 1, 1); - -// GPU only access -const PROT_GPU_RO: Prot = Prot::from_bits(AP_GPU, 0, 0); -const PROT_GPU_WO: Prot = Prot::from_bits(AP_GPU, 0, 1); -const PROT_GPU_RW: Prot = Prot::from_bits(AP_GPU, 1, 0); -const _PROT_GPU_NA: Prot = Prot::from_bits(AP_GPU, 1, 1); - -#[cfg(CONFIG_DEV_COREDUMP)] -const PF_RW: u32 = PF_R | PF_W; -#[cfg(CONFIG_DEV_COREDUMP)] -const PF_RX: u32 = PF_R | PF_X; - -// For crash dumps -#[cfg(CONFIG_DEV_COREDUMP)] -const PROT_TO_PERMS_FW: [[u32; 4]; 4] = [ - [0, 0, 0, PF_RW], - [0, PF_RW, 0, PF_RW], - [PF_RX, PF_RX, 0, PF_R], - [PF_RX, PF_RW, 0, PF_R], -]; -#[cfg(CONFIG_DEV_COREDUMP)] -const PROT_TO_PERMS_OS: [[u32; 4]; 4] = [ - [0, PF_R, PF_W, PF_RW], - [PF_R, 0, PF_RW, PF_RW], - [0, 0, 0, 0], - [0, 0, 0, 0], -]; - -pub(crate) mod prot { - pub(crate) use super::Prot; - use super::*; - - /// Firmware MMIO R/W - pub(crate) const PROT_FW_MMIO_RW: Prot = PROT_FW_RW.memattr(MEMATTR_DEV); - /// Firmware MMIO R/O - pub(crate) const PROT_FW_MMIO_RO: Prot = PROT_FW_RO.memattr(MEMATTR_DEV); - /// Firmware shared (uncached) RW - pub(crate) const PROT_FW_SHARED_RW: Prot = PROT_FW_RW.memattr(MEMATTR_UNCACHED); - /// Firmware shared (uncached) RO - pub(crate) const PROT_FW_SHARED_RO: Prot = PROT_FW_RO.memattr(MEMATTR_UNCACHED); - /// Firmware private (cached) RW - pub(crate) const PROT_FW_PRIV_RW: Prot = PROT_FW_RW.memattr(MEMATTR_CACHED); - /// Firmware/GPU shared (uncached) RW - pub(crate) const PROT_GPU_FW_SHARED_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_UNCACHED); - /// Firmware/GPU shared (private) RW - pub(crate) const PROT_GPU_FW_PRIV_RW: Prot = PROT_FW_GPU_RW.memattr(MEMATTR_CACHED); - /// Firmware-RW/GPU-RO shared (private) RW - pub(crate) const PROT_GPU_RO_FW_PRIV_RW: Prot = PROT_FW_RW_GPU_RO.memattr(MEMATTR_CACHED); - /// GPU shared/coherent RW - pub(crate) const PROT_GPU_SHARED_RW: Prot = PROT_GPU_RW.memattr(MEMATTR_UNCACHED); - /// GPU shared/coherent RO - pub(crate) const PROT_GPU_SHARED_RO: Prot = PROT_GPU_RO.memattr(MEMATTR_UNCACHED); - /// GPU shared/coherent WO - pub(crate) const PROT_GPU_SHARED_WO: Prot = PROT_GPU_WO.memattr(MEMATTR_UNCACHED); -} - -impl Prot { - const fn from_bits(ap: u8, uxn: u16, pxn: u16) -> Self { - assert!(uxn <= 1); - assert!(pxn <= 1); - assert!(ap <= 3); - - Prot { - high_bits: HIGH_BITS_GPU_ACCESS | (pxn * HIGH_BITS_PXN) | (uxn * HIGH_BITS_UXN), - memattr: 0, - ap, - } - } - - pub(crate) const fn from_pte(pte: u64) -> Self { - Prot { - high_bits: (pte >> UAT_HIGH_BITS_SHIFT) as u16, - ap: ((pte & UAT_AP_BITS) >> UAT_AP_SHIFT) as u8, - memattr: ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8, - } - } - - #[cfg(CONFIG_DEV_COREDUMP)] - pub(crate) const fn elf_flags(&self) -> u32 { - let ap = (self.ap & 3) as usize; - let uxn = if self.high_bits & HIGH_BITS_UXN != 0 { - 1 - } else { - 0 - }; - let pxn = if self.high_bits & HIGH_BITS_PXN != 0 { - 1 - } else { - 0 - }; - let gpu = self.high_bits & HIGH_BITS_GPU_ACCESS != 0; - - // Format: - // [12 top bits of PTE] [12 bottom bits of PTE] [5 bits pad] [ELF RWX] - let mut perms = if gpu { - PROT_TO_PERMS_OS[ap][(uxn << 1) | pxn] - } else { - PROT_TO_PERMS_FW[ap][(uxn << 1) | pxn] - }; - - perms |= ((self.as_pte() >> 52) << 20) as u32; - perms |= ((self.as_pte() & 0xfff) << 8) as u32; - - perms - } - - const fn memattr(&self, memattr: u8) -> Self { - Self { memattr, ..*self } - } - - const fn as_pte(&self) -> u64 { - (self.ap as u64) << UAT_AP_SHIFT - | (self.high_bits as u64) << UAT_HIGH_BITS_SHIFT - | (self.memattr as u64) << UAT_MEMATTR_SHIFT - | UAT_AF - } - - /// Whether this leaf allows the requested GPU accesses. - pub(crate) const fn allows_gpu(&self, need_read: bool, need_write: bool) -> bool { - let uxn = self.high_bits & HIGH_BITS_UXN != 0; - let pxn = self.high_bits & HIGH_BITS_PXN != 0; - let (readable, writable) = match self.ap { - AP_FW_GPU => (pxn, uxn), - AP_FW => (uxn && pxn, false), - AP_GPU => (!pxn, uxn != pxn), - _ => (false, false), - }; - (!need_read || readable) && (!need_write || writable) - } - - pub(crate) const fn is_cached_noncoherent(&self) -> bool { - self.ap != AP_GPU && self.memattr == MEMATTR_CACHED - } - - pub(crate) const fn as_uncached(&self) -> Self { - self.memattr(MEMATTR_UNCACHED) - } -} - -impl Default for Prot { - fn default() -> Self { - PROT_FW_GPU_NA - } -} - -#[cfg(CONFIG_DEV_COREDUMP)] -pub(crate) struct DumpedPage { - pub(crate) iova: u64, - pub(crate) pte: u64, - pub(crate) data: Option>, -} - -pub(crate) struct UatPageTable { - ttb: PhysicalAddr, - ttb_owned: bool, - /// Tables below a root the driver did not allocate may belong to the bootloader or the - /// firmware, so only the tables recorded here are ever freed. `None` if the driver owns the - /// tables below the root, as with the handoff firmware interface. - driver_tables: Option>, - va_range: Range, - ias_mask: u64, - oas_mask: u64, -} - -impl UatPageTable { - pub(crate) fn new(ias: u32, oas: u32) -> Result { - mod_pr_debug!("UATPageTable::new: ias={} oas={}\n", ias, oas); - let ttb_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - let ttb = Page::into_phys(ttb_page); - Ok(UatPageTable { - ttb, - ttb_owned: true, - driver_tables: None, - va_range: 0..(1u64 << ias), - ias_mask: (1u64 << ias) - 1, - oas_mask: (1u64 << oas) - 1, - }) - } - - /// Borrow a root; track individual child ownership only when other agents - /// can install tables below it. Handoff roots have driver-owned children. - pub(crate) fn new_with_ttb( - ttb: PhysicalAddr, - va_range: Range, - ias: u32, - oas: u32, - shared_tables: bool, - ) -> Result { - mod_pr_debug!( - "UATPageTable::new_with_ttb: ttb={:#x} range={:#x?} ias={} oas={}\n", - ttb, - va_range, - ias, - oas - ); - if ttb & (UAT_PGMSK as PhysicalAddr) != 0 { - return Err(EINVAL); - } - if (va_range.start | va_range.end) & (UAT_PGMSK as u64) != 0 { - return Err(EINVAL); - } - // SAFETY: The bootloader-managed root stays valid while this page table exists. - if unsafe { Page::borrow_phys(&ttb) }.is_none() { - pr_err!( - "UATPageTable::new_with_ttb: ttb at {:#x} is not mapped (DT using no-map?)\n", - ttb - ); - return Err(EIO); - } - - Ok(UatPageTable { - ttb, - ttb_owned: false, - driver_tables: shared_tables.then(KVec::new), - va_range, - ias_mask: (1u64 << ias) - 1, - oas_mask: (1u64 << oas) - 1, - }) - } - - pub(crate) fn ttb(&self) -> PhysicalAddr { - self.ttb - } - - fn with_pages( - &mut self, - iova_range: Range, - alloc: bool, - free: bool, - mut cb: F, - ) -> Result - where - F: FnMut(u64, &[Pte]) -> Result, - { - mod_pr_debug!( - "UATPageTable::with_pages: {:#x?} alloc={} free={}\n", - iova_range, - alloc, - free - ); - if (iova_range.start | iova_range.end) & (UAT_PGMSK as u64) != 0 { - pr_err!( - "UATPageTable::with_pages: iova range not aligned: {:#x?}\n", - iova_range - ); - return Err(EINVAL); - } - - if iova_range.is_empty() { - return Ok(()); - } - - let mut iova = iova_range.start & self.ias_mask; - let mut last_iova = iova; - // Handle the case where iova_range.end is just at the top boundary of the IAS - let end = ((iova_range.end - 1) & self.ias_mask) + 1; - - let mut pt_addr: [Option; UAT_LEVELS] = Default::default(); - // For every child table, retain the parent page and entry that names it. - // Children are detached before their parents are freed, so these borrowed - // parent pages remain valid while the descriptor is cleared. - let mut pt_parent: [Option<(PhysicalAddr, usize)>; UAT_LEVELS - 1] = Default::default(); - pt_addr[UAT_LEVELS - 1] = Some(self.ttb); - - 'outer: while iova < end { - mod_pr_debug!("UATPageTable::with_pages: iova={:#x}\n", iova); - let addr_diff = last_iova ^ iova; - // Detach leaves before intermediate tables. The old order freed a - // parent first and could leave its descriptor naming a freed child. - for level in 0..UAT_LEVELS - 1 { - // If the iova has changed at this level or above, invalidate the physaddr - if addr_diff & !((1 << (UAT_PGBIT + (level + 1) * UAT_LVBIT)) - 1) != 0 { - if let Some(phys) = pt_addr[level].take() { - if free { - mod_pr_debug!( - "UATPageTable::with_pages: free level {} {:#x?}\n", - level, - phys - ); - let parent = pt_parent[level].take(); - self.free_table(phys, parent)?; - } else { - pt_parent[level] = None; - } - mod_pr_debug!("UATPageTable::with_pages: invalidate level {}\n", level); - } - } - } - last_iova = iova; - for level in (0..UAT_LEVELS - 1).rev() { - // Fetch the page table base address for this level - if pt_addr[level].is_none() { - let phys = pt_addr[level + 1].unwrap(); - mod_pr_debug!( - "UATPageTable::with_pages: need level {}, parent phys {:#x}\n", - level, - phys - ); - let upidx = ((iova >> (UAT_PGBIT + (level + 1) * UAT_LVBIT) as u64) & UAT_LVMSK) - as usize; - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let upt = unsafe { Page::borrow_phys_unchecked(&phys) }; - mod_pr_debug!("UATPageTable::with_pages: borrowed phys {:#x}\n", phys); - pt_addr[level] = - upt.with_pointer_into_page(upidx * PTE_SIZE, PTE_SIZE, |p| { - let uptep = p as *const _ as *const Pte; - // SAFETY: with_pointer_into_page() ensures the pointer is valid, - // and our index is aligned so it is safe to deref as an AtomicU64. - let upte = unsafe { &*uptep }; - let mut upte_val = upte.load(Ordering::Relaxed); - // Allocate if requested - if upte_val == 0 && alloc { - let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; - mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); - if let Some(tables) = self.driver_tables.as_mut() { - tables.push(pt_page.phys(), GFP_KERNEL)?; - } - let pt_paddr = Page::into_phys(pt_page); - upte_val = pt_paddr | PTE_TYPE_LEAF_TABLE; - // Complete child-table initialization before a - // concurrent device walk can follow its parent. - // A later mapping/TLBI barrier cannot order an - // already-visible parent against earlier clears. - crate::mem::sync(); - upte.store(upte_val, Ordering::Relaxed); - } - if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { - Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) - } else if upte_val == 0 || (!alloc && !free) { - mod_pr_debug!("UATPageTable::with_pages: no level {}\n", level); - Ok(None) - } else { - pr_err!("UATPageTable::with_pages: Unexpected Table PTE value {:#x} at iova {:#x} index {} phys {:#x}\n", upte_val, - iova, level + 1, phys + ((upidx * PTE_SIZE) as PhysicalAddr)); - Ok(None) - } - })?; - if pt_addr[level].is_some() { - pt_parent[level] = Some((phys, upidx)); - } - mod_pr_debug!( - "UATPageTable::with_pages: level {} PT {:#x?}\n", - level, - pt_addr[level] - ); - } - // If we don't have a page table, skip this entire level - if pt_addr[level].is_none() { - let block = 1 << (UAT_PGBIT + UAT_LVBIT * (level + 1)); - let old = iova; - iova = align(iova + 1, block); - mod_pr_debug!( - "UATPageTable::with_pages: skip {:#x} {:#x} -> {:#x}\n", - block, - old, - iova - ); - continue 'outer; - } - } - - let idx = ((iova >> UAT_PGBIT as u64) & UAT_LVMSK) as usize; - let max_count = UAT_NPTE - idx; - let count = (((end - iova) >> UAT_PGBIT) as usize).min(max_count); - let phys = pt_addr[0].unwrap(); - mod_pr_debug!( - "UATPageTable::with_pages: leaf PT at {:#x} idx {:#x} count {:#x} iova {:#x}\n", - phys, - idx, - count, - iova - ); - // SAFETY: Page table addresses are either allocated by us, or - // firmware-managed and safe to borrow a struct page from. - let pt = unsafe { Page::borrow_phys_unchecked(&phys) }; - pt.with_pointer_into_page(idx * PTE_SIZE, count * PTE_SIZE, |p| { - let ptep = p as *const _ as *const Pte; - // SAFETY: We know this is a valid pointer to PTEs and the range is valid and - // checked by with_pointer_into_page(). - let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; - cb(iova, ptes)?; - Ok(()) - })?; - - let block = 1 << (UAT_PGBIT + UAT_LVBIT); - iova = align(iova + 1, block); - } - - if free { - // The final path is detached bottom-up for the same reason as the - // boundary transitions above. - for level in 0..UAT_LEVELS - 1 { - if let Some(phys) = pt_addr[level] { - mod_pr_debug!( - "UATPageTable::with_pages: free level {} {:#x?}\n", - level, - phys - ); - self.free_table(phys, pt_parent[level].take())?; - } - } - } - - Ok(()) - } - - /// Free a page table, unless it was not allocated by the driver. - fn free_table( - &mut self, - phys: PhysicalAddr, - parent: Option<(PhysicalAddr, usize)>, - ) -> Result { - let owned = if let Some(tables) = self.driver_tables.as_ref() { - match tables.iter().position(|&table| table == phys) { - Some(_) => true, - None => false, - } - } else { - true - }; - if !owned { - return Ok(()); - } - if let Some((parent_phys, parent_index)) = parent { - // SAFETY: The bottom-up traversal keeps the parent allocated while - // its child descriptor is detached. A changed descriptor indicates - // an ownership violation; fail closed instead of freeing a table - // that may still be reachable. - let page = unsafe { Page::borrow_phys(&parent_phys) }.ok_or(EIO)?; - page.with_pointer_into_page(parent_index * PTE_SIZE, PTE_SIZE, |p| { - let pte = unsafe { &*(p as *const AtomicU64) }; - let current = pte.load(Ordering::Acquire); - let expected = phys & self.oas_mask & (!UAT_PGMSK as u64); - if current & self.oas_mask & (!UAT_PGMSK as u64) != expected { - return Err(EIO); - } - pte.store(0, Ordering::Release); - Ok(()) - })?; - } - if let Some(tables) = self.driver_tables.as_mut() { - let index = tables - .iter() - .position(|&table| table == phys) - .ok_or(EIO)?; - tables.swap_remove(index); - } - // SAFETY: Page tables allocated by the driver always come from Page::into_phys(). Without - // `driver_tables`, every child table belongs to the driver. - unsafe { Page::from_phys(phys) }; - Ok(()) - } - - /// Checks every page, including holes skipped by the page-table walker. - pub(crate) fn covers_range( - &mut self, - range: Range, - read: bool, - write: bool, - ) -> Result { - if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { - return Ok(false); - } - let expected = (range.end - range.start) >> UAT_PGBIT; - let mut visited = 0u64; - let mut permitted = true; - self.with_pages(range, false, false, |_, ptes| { - visited = visited.checked_add(ptes.len() as u64).ok_or(EOVERFLOW)?; - for pte in ptes { - let value = pte.load(Ordering::Acquire); - if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE - || !Prot::from_pte(value).allows_gpu(read, write) - { - permitted = false; - } - } - Ok(()) - })?; - Ok(expected == visited && permitted) - } - - /// Resolves an existing leaf without creating page tables. - pub(crate) fn translate_iova(&mut self, iova: u64) -> Result { - let entry = self.leaf_entry(iova & !(UAT_PGMSK as u64))?; - Ok((entry & self.oas_mask & !(UAT_PGMSK as u64)) | (iova & UAT_PGMSK as u64)) - } - - /// Reads one mapped leaf entry without allocating missing tables. - pub(crate) fn leaf_entry(&mut self, iova: u64) -> Result { - if iova & UAT_PGMSK as u64 != 0 { - return Err(EINVAL); - } - let end = iova.checked_add(UAT_PGSZ as u64).ok_or(EINVAL)?; - let mut value = 0; - self.with_pages(iova..end, false, false, |_, ptes| { - if let Some(pte) = ptes.first() { - value = pte.load(Ordering::Relaxed); - } - Ok(()) - })?; - if value & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { - return Err(EFAULT); - } - Ok(value) - } - - pub(crate) fn alloc_pages(&mut self, iova_range: Range) -> Result { - mod_pr_debug!("UATPageTable::alloc_pages: {:#x?}\n", iova_range); - self.with_pages(iova_range, true, false, |_, _| Ok(())) - } - - /// Reserve all tables for a fixed mapping before publishing any leaves. - /// GPUVM and the fixed-address allocator may share a page table, so check - /// that neither owns a destination leaf before allowing failure rollback. - pub(crate) fn prepare_map(&mut self, range: Range) -> Result { - if range.is_empty() || (range.start | range.end) & UAT_PGMSK as u64 != 0 { - return Err(EINVAL); - } - self.with_pages(range.clone(), false, false, |_, ptes| { - if ptes.iter().any(|pte| pte.load(Ordering::Acquire) != 0) { - return Err(EBUSY); - } - Ok(()) - })?; - self.alloc_pages(range) - } - - fn pte_bits(&self) -> u64 { - if self.ttb_owned { - // Owned page tables are userspace, so non-global - PTE_TYPE_LEAF_TABLE | UAT_NON_GLOBAL - } else { - // The sole non-owned page table is kernelspace, so global - PTE_TYPE_LEAF_TABLE - } - } - - pub(crate) fn map_pages( - &mut self, - iova_range: Range, - mut phys: PhysicalAddr, - prot: Prot, - one_page: bool, - ) -> Result { - mod_pr_debug!( - "UATPageTable::map_pages: {:#x?} {:#x?} {:?}\n", - iova_range, - phys, - prot - ); - if phys & (UAT_PGMSK as PhysicalAddr) != 0 { - pr_err!("UATPageTable::map_pages: phys not aligned: {:#x?}\n", phys); - return Err(EINVAL); - } - - let pte_bits = self.pte_bits(); - - self.with_pages(iova_range, true, false, |iova, ptes| { - for (idx, pte) in ptes.iter().enumerate() { - let ptev = pte.load(Ordering::Relaxed); - if ptev != 0 { - pr_err!( - "UATPageTable::map_pages: Page at IOVA {:#x} is mapped (PTE: {:#x})\n", - iova + (idx * UAT_PGSZ) as u64, - ptev - ); - } - pte.store(phys | prot.as_pte() | pte_bits, Ordering::Relaxed); - if !one_page { - phys += UAT_PGSZ as PhysicalAddr; - } - } - Ok(()) - }) - } - - pub(crate) fn reprot_pages(&mut self, iova_range: Range, prot: Prot) -> Result { - mod_pr_debug!( - "UATPageTable::reprot_pages: {:#x?} {:?}\n", - iova_range, - prot - ); - self.with_pages(iova_range, true, false, |iova, ptes| { - for (idx, pte) in ptes.iter().enumerate() { - let ptev = pte.load(Ordering::Relaxed); - if ptev & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { - pr_err!( - "UATPageTable::reprot_pages: Page at IOVA {:#x} is unmapped (PTE: {:#x})\n", - iova + (idx * UAT_PGSZ) as u64, - ptev - ); - continue; - } - pte.store((ptev & !UAT_PROT_BITS) | prot.as_pte(), Ordering::Relaxed); - } - Ok(()) - }) - } - - pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { - mod_pr_debug!("UATPageTable::unmap_pages: {:#x?}\n", iova_range); - self.with_pages(iova_range, false, false, |iova, ptes| { - for (idx, pte) in ptes.iter().enumerate() { - if pte.load(Ordering::Relaxed) & PTE_TYPE_LEAF_TABLE == 0 { - pr_err!( - "UATPageTable::unmap_pages: Page at IOVA {:#x} already unmapped\n", - iova + (idx * UAT_PGSZ) as u64 - ); - } - pte.store(0, Ordering::Relaxed); - } - Ok(()) - }) - } - - /// Discards an aligned map operation that failed before acquiring a GPUVA owner. - /// Missing tables and leaves are expected; existing table storage stays allocated. - pub(crate) fn discard_partial_map(&mut self, iova_range: Range) -> Result { - self.with_pages(iova_range, false, false, |_, ptes| { - for pte in ptes { - pte.store(0, Ordering::Relaxed); - } - Ok(()) - }) - } - - #[cfg(CONFIG_DEV_COREDUMP)] - pub(crate) fn dump_pages(&mut self, iova_range: Range) -> Result> { - let mut pages = KVVec::new(); - let oas_mask = self.oas_mask; - let iova_base = self.va_range.start & !self.ias_mask; - self.with_pages(iova_range, false, false, |iova, ptes| { - let iova = iova | iova_base; - for (idx, ppte) in ptes.iter().enumerate() { - let pte = ppte.load(Ordering::Relaxed); - if (pte & PTE_TYPE_LEAF_TABLE) != PTE_TYPE_LEAF_TABLE { - continue; - } - let memattr = ((pte & UAT_MEMATTR_BITS) >> UAT_MEMATTR_SHIFT) as u8; - - if !(memattr == MEMATTR_CACHED || memattr == MEMATTR_UNCACHED) { - pages.push( - DumpedPage { - iova: iova + (idx * UAT_PGSZ) as u64, - pte, - data: None, - }, - GFP_KERNEL, - )?; - continue; - } - let phys = pte & oas_mask & (!UAT_PGMSK as u64); - // SAFETY: GPU pages are either firmware/preallocated pages - // (which the kernel isn't concerned with and are either in - // the page map or not, and if they aren't, borrow_phys() - // will fail), or GPU page table pages (which we own), - // or GEM buffer pages (which are locked while they are - // mapped in the page table), so they should be safe to - // borrow. - // - // This does trust the firmware not to have any weird - // mappings in its own internal page tables, but since - // those are managed by the uPPL which is privileged anyway, - // this trust does not actually extend any trust boundary. - let src_page = match unsafe { Page::borrow_phys(&phys) } { - Some(page) => page, - None => { - pages.push( - DumpedPage { - iova: iova + (idx * UAT_PGSZ) as u64, - pte, - data: None, - }, - GFP_KERNEL, - )?; - continue; - } - }; - let dst_page = Page::alloc_page(GFP_KERNEL)?; - src_page.with_page_mapped(|psrc| -> Result { - // SAFETY: This could technically still have a data race with the firmware - // or other driver code (or even userspace with timestamp buffers), but while - // the Rust language technically says this is UB, in the real world, using - // atomic reads for this is guaranteed to never cause any harmful effects - // other than possibly reading torn/unreliable data. At least on ARM64 anyway. - // - // (Yes, I checked with Rust people about this. ~~ Lina) - // - let src_items = unsafe { - core::slice::from_raw_parts( - psrc as *const AtomicU64, - UAT_PGSZ / core::mem::size_of::(), - ) - }; - dst_page.with_page_mapped(|pdst| -> Result { - // SAFETY: We own the destination page, so it is safe to view its contents - // as a u64 slice. - let dst_items = unsafe { - core::slice::from_raw_parts_mut( - pdst as *mut u64, - UAT_PGSZ / core::mem::size_of::(), - ) - }; - for (si, di) in src_items.iter().zip(dst_items.iter_mut()) { - *di = si.load(Ordering::Relaxed); - } - Ok(()) - })?; - Ok(()) - })?; - pages.push( - DumpedPage { - iova: iova + (idx * UAT_PGSZ) as u64, - pte, - data: Some(dst_page), - }, - GFP_KERNEL, - )?; - } - Ok(()) - })?; - Ok(pages) - } -} - -impl Drop for UatPageTable { - fn drop(&mut self) { - mod_pr_debug!("UATPageTable::drop range: {:#x?}\n", &self.va_range); - if self - .with_pages(self.va_range.clone(), false, true, |iova, ptes| { - for (idx, pte) in ptes.iter().enumerate() { - if pte.load(Ordering::Relaxed) != 0 { - pr_err!( - "UATPageTable::drop: Leaked page at IOVA {:#x}\n", - iova + (idx * UAT_PGSZ) as u64 - ); - } - } - Ok(()) - }) - .is_err() - { - pr_err!("UATPageTable::drop failed to free page tables\n",); - } - if self.ttb_owned { - mod_pr_debug!("UATPageTable::drop: Free TTB {:#x}\n", self.ttb); - // SAFETY: If we own the ttb, it was allocated with Page::into_phys(). - unsafe { - Page::from_phys(self.ttb); - } - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/queue/common.rs b/drivers/gpu/drm/asahi-neo/queue/common.rs deleted file mode 100644 index 067624cd66734f..00000000000000 --- a/drivers/gpu/drm/asahi-neo/queue/common.rs +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Common queue functionality. -//! -//! Shared helpers used by the submission logic for multiple command types. - -use crate::file; -use crate::fw::job::UserTimestamp; - -use kernel::prelude::*; -use kernel::uapi; -use kernel::xarray; - -pub(super) fn get_timestamp_object( - objects: Pin<&xarray::XArray>>, - timestamp: uapi::drm_asahi_neo_timestamp, -) -> Result> { - if timestamp.handle == 0 { - return Ok(None); - } - - let guard = objects.lock(); - let object = guard - .get(timestamp.handle.try_into()?) - .ok_or(ENOENT)? - .clone(); - core::mem::drop(guard); - - #[allow(irrefutable_let_patterns)] - if let file::Object::TimestampBuffer(mapping) = object { - let offset = timestamp.offset; - if (offset.checked_add(8).ok_or(EINVAL)?) as usize > mapping.size() { - return Err(ERANGE); - } - Ok(Some(UserTimestamp { - mapping: mapping.clone(), - offset: offset as usize, - })) - } else { - Err(EINVAL) - } -} diff --git a/drivers/gpu/drm/asahi-neo/queue/compute.rs b/drivers/gpu/drm/asahi-neo/queue/compute.rs deleted file mode 100644 index df9e4c97e32480..00000000000000 --- a/drivers/gpu/drm/asahi-neo/queue/compute.rs +++ /dev/null @@ -1,385 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#![allow(clippy::unusual_byte_groupings)] - -//! Compute work queue. -//! -//! A compute queue consists of one underlying WorkQueue. -//! This module is in charge of creating all of the firmware structures required to submit compute -//! work to the GPU, based on the userspace command buffer. - -use super::common; -use crate::alloc::Allocator; -use crate::debug::*; -use crate::fw::types::*; -use crate::gpu::GpuManager; -use crate::{ - file, - fw, - gpu, - microseq, // -}; -use crate::{ - inner_ptr, - inner_weak_ptr, // -}; -use core::sync::atomic::Ordering; -use kernel::dma_fence::RawDmaFence; -use kernel::drm_neo::sched::Job; -use kernel::prelude::*; -use kernel::sync::Arc; -use kernel::uapi; -use kernel::xarray; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Compute; - -#[versions(AGX_NEO)] -impl super::QueueInner::ver { - /// Submit work to a compute queue. - pub(super) fn submit_compute( - &self, - job: &mut Job, - cmdbuf: &uapi::drm_asahi_neo_cmd_compute, - attachments: µseq::Attachments, - objects: Pin<&xarray::XArray>>, - id: u64, - flush_stamps: bool, - ) -> Result { - let gpu = match (*self.dev) - .gpu - .as_any() - .downcast_ref::() - { - Some(gpu) => gpu, - None => { - dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); - return Err(EIO); - } - }; - - let mut alloc = gpu.alloc(); - let kalloc = &mut *alloc; - - mod_dev_dbg!(self.dev, "[Submission {}] Compute!\n", id); - - if cmdbuf.flags != 0 { - return Err(EINVAL); - } - - let mut user_timestamps: fw::job::UserTimestamps = Default::default(); - user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts.start)?; - user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts.end)?; - - // This sequence number increases per new client/VM? assigned to some slot, - // but it's unclear *which* slot... - let slot_client_seq: u8 = (self.id & 0xff) as u8; - - let vm_bind = job.vm_bind.clone(); - - mod_dev_dbg!( - self.dev, - "[Submission {}] VM slot = {}\n", - id, - vm_bind.slot() - ); - - let notifier = self.notifier.clone(); - - let fence = job.fence.clone(); - let comp_job = job.get_comp()?; - let ev_comp = comp_job.event_info(); - - let preempt2_off = gpu.get_cfg().compute_preempt1_size; - let preempt3_off = preempt2_off + 8; - let preempt4_off = preempt3_off + 8; - let preempt5_off = preempt4_off + 8; - let preempt_size = preempt5_off + 8; - - let preempt_buf = self - .ualloc - .lock() - .array_empty_tagged(preempt_size, b"CPMT")?; - - mod_dev_dbg!( - self.dev, - "[Submission {}] Event #{} {:#x?} -> {:#x?}\n", - id, - ev_comp.slot, - ev_comp.value, - ev_comp.value.next(), - ); - - let timestamps = Arc::new( - kalloc.shared.new_default::()?, - GFP_KERNEL, - )?; - - let uuid = 0; - mod_dev_dbg!(self.dev, "[Submission {}] UUID = {:#x?}\n", id, uuid); - - // TODO: check - #[ver(V >= V13_0B4)] - let count = self.counter.fetch_add(1, Ordering::Relaxed); - - let comp = GpuObject::new_init_prealloc( - kalloc.gpu_ro.alloc_object()?, - |ptr: GpuWeakPointer| { - let notifier = notifier.clone(); - let vm_bind = vm_bind.clone(); - try_init!(fw::compute::RunCompute::ver { - preempt_buf: preempt_buf, - micro_seq: { - let mut builder = microseq::Builder::new(); - - let stats = gpu.initdata.runtime_pointers.stats.comp.weak_pointer(); - - let start_comp = builder.add(microseq::StartCompute::ver { - header: microseq::op::StartCompute::HEADER, - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - #[ver(G < G14X)] - job_params1: Some(inner_weak_ptr!(ptr, job_params1)), - #[ver(G >= G14X)] - job_params1: None, - #[ver(G >= G14X)] - registers: inner_weak_ptr!(ptr, registers), - stats, - work_queue: ev_comp.info_ptr, - vm_slot: vm_bind.slot(), - unk_28: 0x1, - event_generation: self.id as u32, - event_seq: U64(ev_comp.event_seq), - unk_38: 0x0, - job_params2: inner_weak_ptr!(ptr, job_params2), - unk_44: 0x0, - uuid, - attachments: *attachments, - padding: Default::default(), - #[ver(V >= V13_0B4)] - unk_flag: inner_weak_ptr!(ptr, unk_flag), - #[ver(V >= V13_0B4)] - counter: U64(count), - #[ver(V >= V13_0B4)] - notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), - })?; - - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } - - #[ver(G < G14X)] - builder.add(microseq::WaitForIdle { - header: microseq::op::WaitForIdle::new(microseq::Pipe::Compute), - })?; - #[ver(G >= G14X)] - builder.add(microseq::WaitForIdle2 { - header: microseq::op::WaitForIdle2::HEADER, - })?; - - if user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_comp.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, context_store_req), - uuid, - unk_30_padding: 0, - })?; - } - - let off = builder.offset_to(start_comp); - builder.add(microseq::FinalizeCompute::ver { - header: microseq::op::FinalizeCompute::HEADER, - stats, - work_queue: ev_comp.info_ptr, - vm_slot: vm_bind.slot(), - #[ver(V < V13_0B4)] - unk_18: 0, - job_params2: inner_weak_ptr!(ptr, job_params2), - unk_24: 0, - uuid, - fw_stamp: ev_comp.fw_stamp_pointer, - stamp_value: ev_comp.value.next(), - unk_38: 0, - unk_3c: 0, - unk_40: 0, - unk_44: 0, - unk_48: 0, - unk_4c: 0, - unk_50: 0, - unk_54: 0, - unk_58: 0, - #[ver(G == G14 && V < V13_0B4)] - unk_5c_g14: U64(0), - restart_branch_offset: off, - has_attachments: (attachments.count > 0) as u32, - #[ver(V >= V13_0B4)] - unk_64: Default::default(), - #[ver(V >= V13_0B4)] - unk_flag: inner_weak_ptr!(ptr, unk_flag), - #[ver(V >= V13_0B4)] - unk_79: Default::default(), - })?; - - builder.add(microseq::RetireStamp { - header: microseq::op::RetireStamp::HEADER, - })?; - builder.build(&mut kalloc.private)? - }, - notifier, - vm_bind, - timestamps, - user_timestamps, - }) - }, - |inner, _ptr| { - let vm_slot = vm_bind.slot(); - try_init!(fw::compute::raw::RunCompute::ver { - tag: fw::workqueue::CommandType::RunCompute, - #[ver(V >= V13_0B4)] - counter: U64(count), - unk_4: 0, - vm_slot, - notifier: inner.notifier.gpu_pointer(), - unk_pointee: Default::default(), - #[ver(G < G14X)] - __pad0: Default::default(), - #[ver(G < G14X)] - job_params1 <- try_init!(fw::compute::raw::JobParameters1 { - preempt_buf1: inner.preempt_buf.gpu_pointer(), - cdm_ctrl_stream_base: U64(cmdbuf.cdm_ctrl_stream_base), - // buf2-5 Only if internal program is used - preempt_buf2: inner.preempt_buf.gpu_offset_pointer(preempt2_off), - preempt_buf3: inner.preempt_buf.gpu_offset_pointer(preempt3_off), - preempt_buf4: inner.preempt_buf.gpu_offset_pointer(preempt4_off), - preempt_buf5: inner.preempt_buf.gpu_offset_pointer(preempt5_off), - usc_exec_base_cp: U64(self.usc_exec_base), - unk_38: U64(0x8c60), - helper_program: cmdbuf.helper.binary, // Internal program addr | 1 - unk_44: 0, - helper_arg: U64(cmdbuf.helper.data), // Only if internal program used - helper_cfg: cmdbuf.helper.cfg, // 0x40 if internal program used - unk_54: 0, - unk_58: 1, - unk_5c: 0, - iogpu_unk_40: 0, // 0x1c if internal program used - __pad: Default::default(), - }), - #[ver(G >= G14X)] - registers: fw::job::raw::RegisterArray::new( - inner_weak_ptr!(_ptr, registers.registers), - |r| { - r.add(0x1a510, inner.preempt_buf.gpu_pointer().into()); - r.add(0x1a420, cmdbuf.cdm_ctrl_stream_base); - // buf2-5 Only if internal program is used - r.add(0x1a4d0, inner.preempt_buf.gpu_offset_pointer(preempt2_off).into()); - r.add(0x1a4d8, inner.preempt_buf.gpu_offset_pointer(preempt3_off).into()); - r.add(0x1a4e0, inner.preempt_buf.gpu_offset_pointer(preempt4_off).into()); - r.add(0x1a4e8, inner.preempt_buf.gpu_offset_pointer(preempt5_off).into()); - r.add(0x10071, self.usc_exec_base); // USC_EXEC_BASE_CP - r.add(0x11841, cmdbuf.helper.binary.into()); - r.add(0x11849, cmdbuf.helper.data); - r.add(0x11f81, cmdbuf.helper.cfg.into()); - r.add(0x1a440, 0x24201); - r.add(0x12091, 0 /* iogpu_unk_40 */); - /* - r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? - r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? - */ - } - ), - __pad1: Default::default(), - microsequence: inner.micro_seq.gpu_pointer(), - microsequence_size: inner.micro_seq.len() as u32, - job_params2 <- try_init!(fw::compute::raw::JobParameters2::ver { - #[ver(V >= V13_0B4)] - unk_0_0: 0, - unk_0: Default::default(), - preempt_buf1: inner.preempt_buf.gpu_pointer(), - cdm_ctrl_stream_end: U64(cmdbuf.cdm_ctrl_stream_end), - unk_34: Default::default(), - #[ver(G < G14X)] - unk_g14x: 0, - #[ver(G >= G14X)] - unk_g14x: 0x24201, - unk_58: 0, - #[ver(V < V13_0B4)] - unk_5c: 0, - }), - encoder_params <- try_init!(fw::job::raw::EncoderParams { - unk_8: 0x0, // fixed - sync_grow: 0x0, // check! - unk_10: 0x0, // fixed - encoder_id: 0, - unk_18: 0x0, // fixed - unk_mask: 0xffffffff, - sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, - }), - meta <- try_init!(fw::job::raw::JobMeta { - unk_0: 0, - unk_2: 0, - no_preemption: 0, - stamp: ev_comp.stamp_pointer, - fw_stamp: ev_comp.fw_stamp_pointer, - stamp_value: ev_comp.value.next(), - stamp_slot: ev_comp.slot, - evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, - uuid, - event_seq: ev_comp.event_seq as u32, - }), - command_time: U64(0), - timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { - start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), start)), - end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), end)), - }), - user_timestamp_pointers: inner.user_timestamps.pointers()?, - client_sequence: slot_client_seq, - pad_2d1: Default::default(), - unk_2d4: 0, - unk_2d8: 0, - #[ver(V >= V13_0B4)] - context_store_req: U64(0), - #[ver(V >= V13_0B4)] - context_store_compl: U64(0), - #[ver(V >= V13_0B4)] - unk_2e9: Default::default(), - #[ver(V >= V13_0B4)] - unk_flag: U32(0), - #[ver(V >= V13_0B4)] - unk_pad: Default::default(), - }) - }, - )?; - - core::mem::drop(alloc); - - fence.add_command(); - comp_job.add_cb(comp, vm_bind.slot(), move |error| { - if let Some(err) = error { - fence.set_error(err.into()) - } - - fence.command_complete(); - })?; - - comp_job.next_seq(); - - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/queue/mod.rs b/drivers/gpu/drm/asahi-neo/queue/mod.rs deleted file mode 100644 index 03926179c44db3..00000000000000 --- a/drivers/gpu/drm/asahi-neo/queue/mod.rs +++ /dev/null @@ -1,958 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Submission queue management -//! -//! This module implements the userspace view of submission queues and the logic to map userspace -//! submissions to firmware queues. - -use kernel::dma_fence::*; -use kernel::prelude::*; -use kernel::{ - c_str, - dma_fence, - drm_neo::sched, - macros::versions, - sync::{ - Arc, - LockClassKey, - Mutex, // - }, - uapi, - xarray, // -}; - -use crate::alloc::Allocator; -use crate::debug::*; -use crate::driver::{AsahiDevRef, AsahiDevice}; -use crate::file::MAX_COMMANDS_PER_SUBMISSION; -use crate::fw::types::*; -use crate::gpu::{ - Gpu, - GpuManager, // -}; -use crate::inner_weak_ptr; -use crate::microseq; -use crate::module_parameters; -use crate::stats; -use crate::util::{ - AnyBitPattern, - Reader, // -}; -use crate::{ - alloc, - buffer, - channel, - event, - file, - fw, - gpu, - mmu, - workqueue, // -}; - -use core::sync::atomic::{ - AtomicU64, - Ordering, // -}; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Queue; - -const WQ_SIZE: u32 = 0x500; - -mod common; -mod compute; -mod render; - -/// Trait implemented by all versioned queues. -pub(crate) trait Queue: Send + Sync { - fn submit( - &mut self, - id: u64, - syncs: KVec, - in_sync_count: usize, - cmdbuf_raw: &[u8], - objects: Pin<&xarray::XArray>>, - ) -> Result; -} - -#[versions(AGX_NEO)] -struct SubQueue { - wq: Arc, -} - -#[versions(AGX_NEO)] -impl SubQueue::ver { - fn new_job(&mut self, fence: dma_fence::Fence) -> SubQueueJob::ver { - SubQueueJob::ver { - wq: self.wq.clone(), - fence: Some(fence), - job: None, - } - } -} - -#[versions(AGX_NEO)] -struct SubQueueJob { - wq: Arc, - job: Option, - fence: Option, -} - -#[versions(AGX_NEO)] -impl SubQueueJob::ver { - fn get(&mut self) -> Result<&mut workqueue::Job::ver> { - if self.job.is_none() { - mod_pr_debug!("SubQueueJob: Creating {:?} job\n", self.wq.pipe_type()); - self.job - .replace(self.wq.new_job(self.fence.take().unwrap())?); - } - Ok(self.job.as_mut().expect("expected a Job")) - } - - fn commit(&mut self) -> Result { - match self.job.as_mut() { - Some(job) => job.commit(), - None => Ok(()), - } - } - - fn can_submit(&self) -> Option { - self.job.as_ref().and_then(|job| job.can_submit()) - } -} - -#[versions(AGX_NEO)] -pub(crate) struct Queue { - dev: AsahiDevRef, - _sched: sched::Scheduler, - entity: sched::Entity, - vm: mmu::Vm, - q_vtx: Option, - q_frag: Option, - q_comp: Option, - fence_ctx: FenceContexts, - inner: QueueInner::ver, - stats: Arc, -} - -#[versions(AGX_NEO)] -pub(crate) struct QueueInner { - dev: AsahiDevRef, - ualloc: Arc>, - buffer: buffer::Buffer::ver, - gpu_context: Arc, - notifier_list: Arc>, - notifier: Arc>, - usc_exec_base: u64, - id: u64, - #[ver(V >= V13_0B4)] - counter: AtomicU64, -} - -#[versions(AGX_NEO)] -#[derive(Default)] -pub(crate) struct JobFence { - id: u64, - pending: AtomicU64, - stats: Option>, -} - -#[versions(AGX_NEO)] -impl JobFence::ver { - fn add_command(self: &FenceObject) { - self.pending.fetch_add(1, Ordering::Relaxed); - } - - fn command_complete(self: &FenceObject) { - let remain = self.pending.fetch_sub(1, Ordering::Relaxed) - 1; - mod_pr_debug!( - "JobFence[{}]: Command complete (remain: {})\n", - self.id, - remain - ); - if remain == 0 { - if let Some(stats) = &self.stats { - stats.note_job(); - } - mod_pr_debug!("JobFence[{}]: Signaling\n", self.id); - self.signal(); - } - } -} - -#[versions(AGX_NEO)] -#[vtable] -impl dma_fence::FenceOps for JobFence::ver { - fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("asahi") - } - fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { - c_str!("queue") - } -} - -#[versions(AGX_NEO)] -pub(crate) struct QueueJob { - dev: AsahiDevRef, - vm_bind: mmu::VmBind, - op_guard: Option, - sj_vtx: Option, - sj_frag: Option, - sj_comp: Option, - fence: UserFence, - notifier: Arc>, - notification_count: u32, - id: u64, -} - -pub(crate) struct QueueTimeout { - dev: AsahiDevRef, - id: u64, -} - -#[versions(AGX_NEO)] -impl QueueJob::ver { - fn get_vtx(&mut self) -> Result<&mut workqueue::Job::ver> { - self.sj_vtx - .as_mut() - .ok_or_else(|| { - cls_pr_debug!(Errors, "No vertex queue\n"); - EINVAL - })? - .get() - } - fn get_frag(&mut self) -> Result<&mut workqueue::Job::ver> { - self.sj_frag - .as_mut() - .ok_or_else(|| { - cls_pr_debug!(Errors, "No fragment queue\n"); - EINVAL - })? - .get() - } - fn get_comp(&mut self) -> Result<&mut workqueue::Job::ver> { - self.sj_comp - .as_mut() - .ok_or_else(|| { - cls_pr_debug!(Errors, "No compute queue\n"); - EINVAL - })? - .get() - } - - fn commit(&mut self) -> Result { - mod_dev_dbg!(self.dev, "QueueJob {}: Committing\n", self.id); - - self.sj_vtx.as_mut().map(|a| a.commit()).unwrap_or(Ok(()))?; - self.sj_frag - .as_mut() - .map(|a| a.commit()) - .unwrap_or(Ok(()))?; - self.sj_comp.as_mut().map(|a| a.commit()).unwrap_or(Ok(())) - } -} - -#[versions(AGX_NEO)] -impl sched::JobImpl for QueueJob::ver { - type TimeoutData = QueueTimeout; - const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); - - fn timeout_data(&self) -> QueueTimeout { - QueueTimeout { dev: self.dev.clone(), id: self.id } - } - - fn prepare(job: &mut Self) -> Option { - mod_dev_dbg!(job.dev, "QueueJob {}: Checking runnability\n", job.id); - - if let Some(sj) = job.sj_vtx.as_ref() { - if let Some(fence) = sj.can_submit() { - mod_dev_dbg!( - job.dev, - "QueueJob {}: Blocking due to vertex queue full\n", - job.id - ); - return Some(fence); - } - } - if let Some(sj) = job.sj_frag.as_ref() { - if let Some(fence) = sj.can_submit() { - mod_dev_dbg!( - job.dev, - "QueueJob {}: Blocking due to fragment queue full\n", - job.id - ); - return Some(fence); - } - } - if let Some(sj) = job.sj_comp.as_ref() { - if let Some(fence) = sj.can_submit() { - mod_dev_dbg!( - job.dev, - "QueueJob {}: Blocking due to compute queue full\n", - job.id - ); - return Some(fence); - } - } - None - } - - #[allow(unused_assignments)] - fn run(job: &mut Self) -> Result> { - mod_dev_dbg!(job.dev, "QueueJob {}: Running Job\n", job.id); - - // We can only increase the notifier threshold here, now that we are - // actually running the job. We cannot increase it while queueing the - // job without introducing subtle race conditions. Suppose we did, as - // early versions of drm_neo/asahi did: - // - // 1. When processing the ioctl submit, a job is queued to drm_sched. - // Incorrectly, the notifier threshold is increased, gating firmware - // events. - // 2. When DRM schedules an event, the hardware is kicked. - // 3. When the number of processed jobs equals the threshold, the - // firmware signals the complete event to the kernel - // 4. When the kernel gets a complete event, we signal the out-syncs. - // - // Does that work? There are a few scenarios. - // - // 1. There is nothing else ioctl submitted before the job completes. - // The job is scheduled, completes, and signals immediately. - // Everything works. - // 2. There is nontrivial sync across different queues. Since each queue - // has a separate own notifier threshold, submitting one does not - // block scheduling of the other. Everything works the way you'd - // expect. drm_neo/sched handles the wait/signal ordering. - // 3. Two ioctls are submitted back-to-back. The first signals a fence - // that the second waits on. Due to the notifier threshold increment, - // the first job's completion event is deferred. But in good - // conditions, drm_neo/sched will schedule the second submit anyway - // because it kills the pointless intra-queue sync. Then both - // commands execute and are signalled together. - // 4. Two ioctls are submitted back-to-back as above, but conditions are - // bad. Reporting completion of the first job is still masked by the - // notifier threshold, but the intra-queue fences are not optimized - // out in drm_neo/sched... drm_neo/sched doesn't schedule the second job - // until the first is signalled, but the first isn't signalled until - // the second is completed, but the second can't complete until it's - // scheduled. We hang! - // - // In good conditions, everything works properly and/or we win the race - // to mask the issue. So the issue here is challenging to hit. - // Nevertheless, we do need to get it right. - // - // The intention with drm_neo/sched is that jobs that are not yet scheduled - // are "invisible" to the firmware. Incrementing the notifier threshold - // earlier than this violates that which leads to circles like the - // above. Deferring the increment to submit solves the race. - let gpu = match (*job.dev) - .gpu - .clone() - .arc_as_any() - .downcast::() - { - Ok(gpu) => gpu, - Err(_) => { - dev_crit!(job.dev.as_ref(), "GpuManager mismatched with QueueJob!\n"); - return Err(EIO); - } - }; - - if job.op_guard.is_none() { - job.op_guard = Some(gpu.start_op()?); - } - - // First submit all the commands for each queue. This can fail. - - let mut frag_job = None; - let mut frag_sub = None; - if let Some(sj) = job.sj_frag.as_mut() { - frag_job = sj.job.take(); - if let Some(wqjob) = frag_job.as_mut() { - mod_dev_dbg!(job.dev, "QueueJob {}: Submit fragment\n", job.id); - frag_sub = Some(wqjob.submit()?); - } - } - - let mut vtx_job = None; - let mut vtx_sub = None; - if let Some(sj) = job.sj_vtx.as_mut() { - vtx_job = sj.job.take(); - if let Some(wqjob) = vtx_job.as_mut() { - mod_dev_dbg!(job.dev, "QueueJob {}: Submit vertex\n", job.id); - vtx_sub = Some(wqjob.submit()?); - } - } - - let mut comp_job = None; - let mut comp_sub = None; - if let Some(sj) = job.sj_comp.as_mut() { - comp_job = sj.job.take(); - if let Some(wqjob) = comp_job.as_mut() { - mod_dev_dbg!(job.dev, "QueueJob {}: Submit compute\n", job.id); - comp_sub = Some(wqjob.submit()?); - } - } - - // Prepared submissions still hold their workqueue locks and have not - // published cpu_wptr. Only count them after every fallible submit() - // succeeded, before the first run_job exposes commands to firmware. - job.notifier.threshold.with(|raw, _inner| { - raw.increase(job.notification_count); - }); - - // Now we fully commit to running the job - mod_dev_dbg!(job.dev, "QueueJob {}: Run fragment\n", job.id); - frag_sub.map(|a| gpu.run_job(a)).transpose()?; - - mod_dev_dbg!(job.dev, "QueueJob {}: Run vertex\n", job.id); - vtx_sub.map(|a| gpu.run_job(a)).transpose()?; - - mod_dev_dbg!(job.dev, "QueueJob {}: Run compute\n", job.id); - comp_sub.map(|a| gpu.run_job(a)).transpose()?; - - mod_dev_dbg!(job.dev, "QueueJob {}: Drop compute job\n", job.id); - core::mem::drop(comp_job); - mod_dev_dbg!(job.dev, "QueueJob {}: Drop vertex job\n", job.id); - core::mem::drop(vtx_job); - mod_dev_dbg!(job.dev, "QueueJob {}: Drop fragment job\n", job.id); - core::mem::drop(frag_job); - - Ok(Some(Fence::from_fence(&job.fence))) - } - - fn timed_out(job: &mut QueueTimeout, _finished: bool) -> sched::Status { - // FIXME: Handle timeouts properly - dev_err!( - job.dev.as_ref(), - "QueueJob {}: Job timed out on the DRM scheduler, things will probably break\n", - job.id - ); - sched::Status::NoDevice - } - - fn cancel(job: &mut Self) { - dev_info!( - job.dev.as_ref(), - "QueueJob {}: Job canceled on DRM scheduler teardown\n", - job.id - ); - } -} - -#[versions(AGX_NEO)] -impl Drop for QueueJob::ver { - fn drop(&mut self) { - mod_dev_dbg!(self.dev, "QueueJob {}: Dropping\n", self.id); - } -} - -static QUEUE_NAME: &CStr = c_str!("asahi_fence"); -static QUEUE_CLASS_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); - -#[versions(AGX_NEO)] -impl Queue::ver { - /// Create a new user queue. - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - dev: &AsahiDevice, - gpu: &dyn GpuManager, - vm: mmu::Vm, - alloc: &mut gpu::KernelAllocators, - ualloc: Arc>, - ualloc_priv: Arc>, - event_manager: Arc, - stats: Arc, - mgr: &buffer::BufferManager::ver, - id: u64, - priority: u32, - usc_exec_base: u64, - ) -> Result { - mod_dev_dbg!(dev, "[Queue {}] Creating queue\n", id); - - // Must be shared, no cache management on this one! - let mut notifier_list = alloc.shared.new_default::()?; - - let self_ptr = notifier_list.weak_pointer(); - notifier_list.with_mut(|raw, _inner| { - raw.list_head.next = Some(inner_weak_ptr!(self_ptr, list_head)); - }); - - let threshold = alloc.shared.new_default::()?; - - let notifier: Arc> = Arc::new( - alloc.private.new_init( - /*try_*/ init!(fw::event::Notifier::ver { threshold }), - |inner, _p| { - try_init!(fw::event::raw::Notifier::ver { - threshold: inner.threshold.gpu_pointer(), - generation: AtomicU32::new(id as u32), - cur_count: AtomicU32::new(0), - unk_10: AtomicU32::new(0x50), - state: Default::default() - }) - }, - )?, - GFP_KERNEL, - )?; - - // Priorities are handled by the AGX scheduler, there is no meaning within a - // per-queue scheduler. Use a single run queue wth Kernel priority. - let sched = - sched::Scheduler::new(dev.as_ref(), 1, WQ_SIZE, 0, 100000, c_str!("asahi_sched"))?; - let entity = sched::Entity::new(&sched, sched::Priority::Kernel)?; - - let buffer = buffer::Buffer::ver::new(gpu, alloc, ualloc.clone(), ualloc_priv, mgr)?; - - let mut ret = Queue::ver { - dev: dev.into(), - _sched: sched, - entity, - vm, - q_vtx: None, - q_frag: None, - q_comp: None, - fence_ctx: FenceContexts::new(1, QUEUE_NAME, QUEUE_CLASS_KEY)?, - stats, - inner: QueueInner::ver { - dev: dev.into(), - ualloc, - gpu_context: Arc::new( - workqueue::GpuContext::new(dev, alloc, buffer.any_ref())?, - GFP_KERNEL, - )?, - - buffer, - notifier_list: Arc::new(notifier_list, GFP_KERNEL)?, - notifier, - usc_exec_base, - id, - #[ver(V >= V13_0B4)] - counter: AtomicU64::new(0), - }, - }; - - // Rendering structures - let tvb_blocks = *module_parameters::initial_tvb_size.value(); - - ret.inner.buffer.ensure_blocks(tvb_blocks)?; - - ret.q_vtx = Some(SubQueue::ver { - wq: workqueue::WorkQueue::ver::new( - dev, - alloc, - event_manager.clone(), - ret.inner.gpu_context.clone(), - ret.inner.notifier_list.clone(), - channel::PipeType::Vertex, - id, - priority, - WQ_SIZE, - )?, - }); - - ret.q_frag = Some(SubQueue::ver { - wq: workqueue::WorkQueue::ver::new( - dev, - alloc, - event_manager.clone(), - ret.inner.gpu_context.clone(), - ret.inner.notifier_list.clone(), - channel::PipeType::Fragment, - id, - priority, - WQ_SIZE, - )?, - }); - - // Compute structures - ret.q_comp = Some(SubQueue::ver { - wq: workqueue::WorkQueue::ver::new( - dev, - alloc, - event_manager, - ret.inner.gpu_context.clone(), - ret.inner.notifier_list.clone(), - channel::PipeType::Compute, - id, - priority, - WQ_SIZE, - )?, - }); - - mod_dev_dbg!(dev, "[Queue {}] Queue created\n", id); - Ok(ret) - } -} - -const SQ_RENDER: usize = 0; -const SQ_COMPUTE: usize = 1; -const SQ_COUNT: usize = 2; - -// SAFETY: All bit patterns are valid by construction. -unsafe impl AnyBitPattern for uapi::drm_asahi_neo_cmd_header {} -unsafe impl AnyBitPattern for uapi::drm_asahi_neo_cmd_render {} -unsafe impl AnyBitPattern for uapi::drm_asahi_neo_cmd_compute {} -unsafe impl AnyBitPattern for uapi::drm_asahi_neo_attachment {} - -fn build_attachments(reader: &mut Reader<'_>, size: usize) -> Result { - const STRIDE: usize = core::mem::size_of::(); - let count = size / STRIDE; - - if count > microseq::MAX_ATTACHMENTS { - return Err(EINVAL); - } - - let mut attachments: microseq::Attachments = Default::default(); - attachments.count = count as u32; - - for i in 0..count { - let att: uapi::drm_asahi_neo_attachment = reader.read()?; - - if att.flags != 0 || att.pad != 0 { - return Err(EINVAL); - } - - // Some kind of power-of-2 exponent related to attachment size, in - // bounds [1, 6]? We don't know what this is exactly yet. - let unk_e = 1; - - let cache_lines = (att.size + 127) >> 7; - attachments.list[i as usize] = microseq::Attachment { - address: U64(att.pointer), - size: cache_lines.try_into()?, - unk_c: 0x17, - unk_e: unk_e as u16, - }; - } - - Ok(attachments) -} - -#[versions(AGX_NEO)] -impl Queue for Queue::ver { - fn submit( - &mut self, - id: u64, - mut syncs: KVec, - in_sync_count: usize, - cmdbuf_raw: &[u8], - objects: Pin<&xarray::XArray>>, - ) -> Result { - let gpu = match (*self.dev) - .gpu - .clone() - .arc_as_any() - .downcast::() - { - Ok(gpu) => gpu, - Err(_) => { - dev_crit!(self.dev.as_ref(), "GpuManager mismatched with JobImpl!\n"); - return Err(EIO); - } - }; - - mod_dev_dbg!(self.dev, "[Submission {}] Submit job\n", id); - - if gpu.is_crashed() { - dev_err!( - self.dev.as_ref(), - "[Submission {}] GPU is crashed, cannot submit\n", - id - ); - return Err(ENODEV); - } - - let op_guard = if in_sync_count > 0 { - Some(gpu.start_op()?) - } else { - None - }; - - let mut events: [KVec>; SQ_COUNT] = - Default::default(); - - events[SQ_RENDER].push( - self.q_frag.as_ref().and_then(|a| a.wq.event_info()), - GFP_KERNEL, - )?; - events[SQ_COMPUTE].push( - self.q_comp.as_ref().and_then(|a| a.wq.event_info()), - GFP_KERNEL, - )?; - - let vm_bind = gpu.bind_vm(&self.vm)?; - let vm_slot = vm_bind.slot(); - - mod_dev_dbg!(self.dev, "[Submission {}] Creating job\n", id); - - // FIXME: I think this can violate the fence seqno ordering contract. - // If we have e.g. a render submission with no barriers and then a compute submission - // with no barriers, it's possible for the compute submission to complete first, and - // therefore its fence. Maybe we should have separate fence contexts for render - // and compute, and then do a ? (Vert+frag should be fine since there is no vert - // without frag, and frag always serializes.) - let fence: UserFence = self - .fence_ctx - .new_fence::( - 0, - JobFence::ver { - id, - pending: Default::default(), - stats: Some(self.stats.clone()), - }, - )? - .into(); - - let mut cmdbuf = Reader::new(cmdbuf_raw); - - // First, parse the headers to determine the number of compute/render - // commands. This will be used to determine when to flush stamps. - // - // We also use it to determine how many notifications the job will - // generate. We could calculate that in the second pass since we don't - // need until much later, but it's convenient to gather everything at - // the same time. - let mut nr_commands = 0; - let mut last_compute = 0; - let mut last_render = 0; - let mut nr_render = 0; - let mut nr_compute = 0; - - while !cmdbuf.is_empty() { - let header: uapi::drm_asahi_neo_cmd_header = cmdbuf.read()?; - cmdbuf.skip(header.size as usize); - nr_commands += 1; - - match header.cmd_type as u32 { - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_RENDER => { - last_render = nr_commands; - nr_render += 1; - } - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_COMPUTE => { - last_compute = nr_commands; - nr_compute += 1; - } - _ => {} - } - } - - let mut job = self.entity.new_job( - 1, - QueueJob::ver { - dev: self.dev.clone(), - vm_bind, - op_guard, - sj_vtx: self - .q_vtx - .as_mut() - .map(|a| a.new_job(Fence::from_fence(&fence))), - sj_frag: self - .q_frag - .as_mut() - .map(|a| a.new_job(Fence::from_fence(&fence))), - sj_comp: self - .q_comp - .as_mut() - .map(|a| a.new_job(Fence::from_fence(&fence))), - fence, - notifier: self.inner.notifier.clone(), - - // Each render command generates 2 notifications: 1 for the - // vertex part, 1 for the fragment part. Each compute command - // generates 1 notification. Sum up to calculate the total - // notification count for the job. - notification_count: (2 * nr_render) + nr_compute, - - id, - }, - )?; - - mod_dev_dbg!( - self.dev, - "[Submission {}] Adding {} in_syncs\n", - id, - in_sync_count - ); - for sync in syncs.drain(0..in_sync_count) { - if let Some(fence) = sync.fence { - job.add_dependency(fence)?; - } - } - - // Validate the number of hardware commands, ignoring software commands - let nr_hw_commands = nr_render + nr_compute; - if nr_hw_commands == 0 || nr_hw_commands > MAX_COMMANDS_PER_SUBMISSION { - cls_pr_debug!( - Errors, - "submit: Command count {} out of valid range [1, {}]\n", - nr_hw_commands, - MAX_COMMANDS_PER_SUBMISSION - 1 - ); - return Err(EINVAL); - } - - cmdbuf.rewind(); - - let mut command_index = 0; - let mut vertex_attachments: microseq::Attachments = Default::default(); - let mut fragment_attachments: microseq::Attachments = Default::default(); - let mut compute_attachments: microseq::Attachments = Default::default(); - - // Parse the full command buffer submitting as we go - while !cmdbuf.is_empty() { - let header: uapi::drm_asahi_neo_cmd_header = cmdbuf.read()?; - let header_size = header.size as usize; - - // Pre-increment command index to match last_compute/last_render - command_index += 1; - - for (queue_idx, index) in [header.vdm_barrier, header.cdm_barrier].iter().enumerate() { - if *index == uapi::DRM_ASAHI_NEO_BARRIER_NONE as u16 { - continue; - } - if let Some(event) = events[queue_idx].get(*index as usize).ok_or_else(|| { - cls_pr_debug!(Errors, "Invalid barrier #{}: {}\n", queue_idx, index); - EINVAL - })? { - let mut alloc = gpu.alloc(); - let queue_job = match header.cmd_type as u32 { - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_RENDER => job.get_vtx()?, - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_COMPUTE => job.get_comp()?, - _ => return Err(EINVAL), - }; - mod_dev_dbg!(self.dev, "[Submission {}] Create Explicit Barrier\n", id); - let barrier = alloc.private.new_init( - pin_init::zeroed::(), - |_inner, _p| { - let queue_job = &queue_job; - try_init!(fw::workqueue::raw::Barrier { - tag: fw::workqueue::CommandType::Barrier, - wait_stamp: event.fw_stamp_pointer, - wait_value: event.value, - wait_slot: event.slot, - stamp_self: queue_job.event_info().value.next(), - uuid: 0xffffbbbb, - external_barrier: 0, - internal_barrier_type: 1, - padding: Default::default(), - }) - }, - )?; - mod_dev_dbg!(self.dev, "[Submission {}] Add Explicit Barrier\n", id); - queue_job.add(barrier, vm_slot)?; - } else { - assert!(*index == 0); - } - } - - match header.cmd_type as u32 { - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_RENDER => { - let render: uapi::drm_asahi_neo_cmd_render = cmdbuf.read_up_to(header_size)?; - - self.inner.submit_render( - &mut job, - &render, - &vertex_attachments, - &fragment_attachments, - objects, - id, - command_index == last_render, - )?; - events[SQ_RENDER].push( - Some( - job.sj_frag - .as_ref() - .expect("No frag queue?") - .job - .as_ref() - .expect("No frag job?") - .event_info(), - ), - GFP_KERNEL, - )?; - } - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_CMD_COMPUTE => { - let compute: uapi::drm_asahi_neo_cmd_compute = cmdbuf.read_up_to(header_size)?; - - self.inner.submit_compute( - &mut job, - &compute, - &compute_attachments, - objects, - id, - command_index == last_compute, - )?; - events[SQ_COMPUTE].push( - Some( - job.sj_comp - .as_ref() - .expect("No comp queue?") - .job - .as_ref() - .expect("No comp job?") - .event_info(), - ), - GFP_KERNEL, - )?; - } - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_SET_VERTEX_ATTACHMENTS => { - vertex_attachments = build_attachments(&mut cmdbuf, header_size)?; - } - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_SET_FRAGMENT_ATTACHMENTS => { - fragment_attachments = build_attachments(&mut cmdbuf, header_size)?; - } - uapi::drm_asahi_neo_cmd_type_DRM_ASAHI_NEO_SET_COMPUTE_ATTACHMENTS => { - compute_attachments = build_attachments(&mut cmdbuf, header_size)?; - } - _ => { - cls_pr_debug!(Errors, "Unknown command type {}\n", header.cmd_type); - return Err(EINVAL); - } - } - } - - mod_dev_dbg!( - self.dev, - "Queue {}: Committing job {}\n", - self.inner.id, - job.id - ); - job.commit()?; - - mod_dev_dbg!(self.dev, "Queue {}: Arming job {}\n", self.inner.id, job.id); - let mut job = job.arm(); - let out_fence = job.fences().finished(); - mod_dev_dbg!( - self.dev, - "Queue {}: Pushing job {}\n", - self.inner.id, - job.id - ); - job.push(); - - mod_dev_dbg!( - self.dev, - "Queue {}: Adding {} out_syncs\n", - self.inner.id, - syncs.len() - ); - for mut sync in syncs { - if let Some(chain) = sync.chain_fence.take() { - sync.syncobj - .add_point(chain, &out_fence, sync.timeline_value); - } else { - sync.syncobj.replace_fence(Some(&out_fence)); - } - } - - Ok(()) - } -} - -#[versions(AGX_NEO)] -impl Drop for Queue::ver { - fn drop(&mut self) { - mod_dev_dbg!(self.dev, "[Queue {}] Dropping queue\n", self.inner.id); - } -} diff --git a/drivers/gpu/drm/asahi-neo/queue/render.rs b/drivers/gpu/drm/asahi-neo/queue/render.rs deleted file mode 100644 index 586a11f4708a4e..00000000000000 --- a/drivers/gpu/drm/asahi-neo/queue/render.rs +++ /dev/null @@ -1,1400 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -#![allow(clippy::unusual_byte_groupings)] - -//! Render work queue. -//! -//! A render queue consists of two underlying WorkQueues, one for vertex and one for fragment work. -//! This module is in charge of creating all of the firmware structures required to submit 3D -//! rendering work to the GPU, based on the userspace command buffer. - -use super::common; -use crate::alloc::Allocator; -use crate::debug::*; -use crate::fw::types::*; -use crate::gpu::GpuManager; -use crate::util::*; -use crate::{ - buffer, - file, - fw, - gpu, - microseq, // -}; -use crate::{ - inner_ptr, - inner_weak_ptr, // -}; -use core::sync::atomic::Ordering; -use kernel::dma_fence::RawDmaFence; -use kernel::drm_neo::sched::Job; -use kernel::prelude::*; -use kernel::sync::Arc; -use kernel::uapi; -use kernel::xarray; - -const DEBUG_CLASS: DebugFlags = DebugFlags::Render; - -/// Tiling/Vertex control bit to disable using more than one GPU cluster. This results in decreased -/// throughput but also less latency, which is probably desirable for light vertex loads where the -/// overhead of clustering/merging would exceed the time it takes to just run the job on one -/// cluster. -const TILECTL_DISABLE_CLUSTERING: u32 = 1u32 << 0; - -#[versions(AGX_NEO)] -impl super::QueueInner::ver { - /// Get the appropriate tiling parameters for a given userspace command buffer. - fn get_tiling_params( - cmdbuf: &uapi::drm_asahi_neo_cmd_render, - num_clusters: u32, - ) -> Result { - let width: u32 = cmdbuf.width_px as u32; - let height: u32 = cmdbuf.height_px as u32; - let layers: u32 = cmdbuf.layers as u32; - - if layers == 0 || layers > 2048 { - cls_pr_debug!(Errors, "Layer count invalid ({})\n", layers); - return Err(EINVAL); - } - - // This is overflow safe: all these calculations are done in u32. - // At 64Kx64K max dimensions above, this is 2**32 pixels max. - // In terms of tiles that are always larger than one pixel, - // this can never overflow. Note that real actual dimensions - // are limited to 16K * 16K below anyway. - // - // Once we multiply by the layer count, then we need to check - // for overflow or use u64. - - let tile_width = 32u32; - let tile_height = 32u32; - - let utile_width = cmdbuf.utile_width_px as u32; - let utile_height = cmdbuf.utile_height_px as u32; - - match (utile_width, utile_height) { - (32, 32) | (32, 16) | (16, 16) => (), - _ => { - cls_pr_debug!( - Errors, - "uTile size invalid ({} x {})\n", - utile_width, - utile_height - ); - return Err(EINVAL); - } - }; - - let utiles_per_tile_x = tile_width / utile_width; - let utiles_per_tile_y = tile_height / utile_height; - - let utiles_per_tile = utiles_per_tile_x * utiles_per_tile_y; - - let tiles_x = width.div_ceil(tile_width); - let tiles_y = height.div_ceil(tile_height); - let tiles = tiles_x * tiles_y; - - let mtiles_x = 4u32; - let mtiles_y = 4u32; - let mtiles = mtiles_x * mtiles_y; - - let tiles_per_mtile_x = align(tiles_x.div_ceil(mtiles_x), 4); - let tiles_per_mtile_y = align(tiles_y.div_ceil(mtiles_y), 4); - let tiles_per_mtile = tiles_per_mtile_x * tiles_per_mtile_y; - - let mtile_x1 = tiles_per_mtile_x; - let mtile_x2 = 2 * tiles_per_mtile_x; - let mtile_x3 = 3 * tiles_per_mtile_x; - - let mtile_y1 = tiles_per_mtile_y; - let mtile_y2 = 2 * tiles_per_mtile_y; - let mtile_y3 = 3 * tiles_per_mtile_y; - - let rgn_entry_size = 5; - // Macrotile stride in 32-bit words - let rgn_size = align(rgn_entry_size * tiles_per_mtile * utiles_per_tile, 4) / 4; - let tilemap_size = (4 * rgn_size * mtiles) as usize * layers as usize; - - let tpc_entry_size = 8; - // TPC stride in 32-bit words - let tpc_mtile_stride = tpc_entry_size * utiles_per_tile * tiles_per_mtile / 4; - let tpc_size = - (4 * tpc_mtile_stride * mtiles) as usize * layers as usize * num_clusters as usize; - - // No idea where this comes from, but it fits what macOS does... - // GUESS: Number of 32K heap blocks to fit a 5-byte region header/pointer per tile? - // That would make a ton of sense... - let meta1_layer_stride = if num_clusters > 1 { - (align(tiles_x, 2) * align(tiles_y, 4) * utiles_per_tile).div_ceil(0x1980) - } else { - 0 - }; - - let mut min_tvb_blocks = align((tiles_x * tiles_y).div_ceil(128), 8); - - if num_clusters > 1 { - min_tvb_blocks = min_tvb_blocks.max(7 + 2 * layers); - } - - Ok(buffer::TileInfo { - tiles_x, - tiles_y, - tiles, - utile_width, - utile_height, - //mtiles_x, - //mtiles_y, - tiles_per_mtile_x, - tiles_per_mtile_y, - //tiles_per_mtile, - utiles_per_mtile_x: tiles_per_mtile_x * utiles_per_tile_x, - utiles_per_mtile_y: tiles_per_mtile_y * utiles_per_tile_y, - //utiles_per_mtile: tiles_per_mtile * utiles_per_tile, - tilemap_size, - tpc_size, - meta1_layer_stride, - #[ver(G < G14X)] - meta1_blocks: meta1_layer_stride * (cmdbuf.layers as u32), - #[ver(G >= G14X)] - meta1_blocks: meta1_layer_stride, - layermeta_size: if layers > 1 { 0x100 } else { 0 }, - min_tvb_blocks: min_tvb_blocks as usize, - params: fw::vertex::raw::TilingParameters { - rgn_size, - unk_4: 0x88, - ppp_ctrl: cmdbuf.ppp_ctrl, - x_max: (width - 1) as u16, - y_max: (height - 1) as u16, - te_screen: ((tiles_y - 1) << 12) | (tiles_x - 1), - te_mtile1: mtile_x3 | (mtile_x2 << 9) | (mtile_x1 << 18), - te_mtile2: mtile_y3 | (mtile_y2 << 9) | (mtile_y1 << 18), - tiles_per_mtile, - tpc_stride: tpc_mtile_stride, - unk_24: 0x100, - unk_28: if layers > 1 { - 0xe000 | (layers - 1) - } else { - 0x8000 - }, - helper_cfg: cmdbuf.vertex_helper.cfg, - __pad: Default::default(), - }, - }) - } - - /// Submit work to a render queue. - pub(super) fn submit_render( - &self, - job: &mut Job, - cmdbuf: &uapi::drm_asahi_neo_cmd_render, - vertex_attachments: µseq::Attachments, - fragment_attachments: µseq::Attachments, - objects: Pin<&xarray::XArray>>, - id: u64, - flush_stamps: bool, - ) -> Result { - mod_dev_dbg!(self.dev, "[Submission {}] Render!\n", id); - - if cmdbuf.flags - & !(uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_VERTEX_SCRATCH - | uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_PROCESS_EMPTY_TILES - | uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_NO_VERTEX_CLUSTERING - | uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_DBIAS_IS_INT) as u32 - != 0 - { - cls_pr_debug!(Errors, "Invalid flags ({:#x})\n", cmdbuf.flags); - return Err(EINVAL); - } - - if cmdbuf.width_px == 0 - || cmdbuf.height_px == 0 - || cmdbuf.width_px > 16384 - || cmdbuf.height_px > 16384 - { - cls_pr_debug!( - Errors, - "Invalid dimensions ({}x{})\n", - cmdbuf.width_px, - cmdbuf.height_px - ); - return Err(EINVAL); - } - - let mut vtx_user_timestamps: fw::job::UserTimestamps = Default::default(); - let mut frg_user_timestamps: fw::job::UserTimestamps = Default::default(); - - vtx_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_vtx.start)?; - vtx_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_vtx.end)?; - frg_user_timestamps.start = common::get_timestamp_object(objects, cmdbuf.ts_frag.start)?; - frg_user_timestamps.end = common::get_timestamp_object(objects, cmdbuf.ts_frag.end)?; - - let gpu = match (*self.dev) - .gpu - .as_any() - .downcast_ref::() - { - Some(gpu) => gpu, - None => { - dev_crit!(self.dev.as_ref(), "GpuManager mismatched with Queue!\n"); - return Err(EIO); - } - }; - - let nclusters = gpu.get_dyncfg().id.num_clusters; - - // Can be set to false to disable clustering (for simpler jobs), but then the - // core masks below should be adjusted to cover a single rolling cluster. - let mut clustering = nclusters > 1; - - if debug_enabled(debug::DebugFlags::DisableClustering) - || cmdbuf.flags - & uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_NO_VERTEX_CLUSTERING as u32 - != 0 - { - clustering = false; - } - - #[ver(G != G14)] - let tiling_control = { - let render_cfg = gpu.get_cfg().render; - let mut tiling_control = render_cfg.tiling_control; - - if !clustering { - tiling_control |= TILECTL_DISABLE_CLUSTERING; - } - tiling_control - }; - - let mut alloc = gpu.alloc(); - let kalloc = &mut *alloc; - - // This sequence number increases per new client/VM? assigned to some slot, - // but it's unclear *which* slot... - let slot_client_seq: u8 = (self.id & 0xff) as u8; - - let tile_info = Self::get_tiling_params(&cmdbuf, if clustering { nclusters } else { 1 })?; - - let buffer = &self.buffer; - let notifier = self.notifier.clone(); - - let tvb_autogrown = buffer.auto_grow()?; - if tvb_autogrown { - let new_size = buffer.block_count() as usize; - cls_dev_dbg!( - TVBStats, - &self.dev, - "[Submission {}] TVB grew to {} bytes ({} blocks) due to overflows\n", - id, - new_size * buffer::BLOCK_SIZE, - new_size, - ); - } - - let tvb_grown = buffer.ensure_blocks(tile_info.min_tvb_blocks)?; - if tvb_grown { - cls_dev_dbg!( - TVBStats, - &self.dev, - "[Submission {}] TVB grew to {} bytes ({} blocks) due to dimensions ({}x{})\n", - id, - tile_info.min_tvb_blocks * buffer::BLOCK_SIZE, - tile_info.min_tvb_blocks, - cmdbuf.width_px, - cmdbuf.height_px - ); - } - - let scene = Arc::new(buffer.new_scene(kalloc, &tile_info)?, GFP_KERNEL)?; - - let vm_bind = job.vm_bind.clone(); - - mod_dev_dbg!( - self.dev, - "[Submission {}] VM slot = {}\n", - id, - vm_bind.slot() - ); - - let ev_vtx = job.get_vtx()?.event_info(); - let ev_frag = job.get_frag()?.event_info(); - - mod_dev_dbg!( - self.dev, - "[Submission {}] Vert event #{} -> {:#x?}\n", - id, - ev_vtx.slot, - ev_vtx.value.next(), - ); - mod_dev_dbg!( - self.dev, - "[Submission {}] Frag event #{} -> {:#x?}\n", - id, - ev_frag.slot, - ev_frag.value.next(), - ); - - let uuid_3d = 0; - let uuid_ta = 0; - - mod_dev_dbg!( - self.dev, - "[Submission {}] Vert UUID = {:#x?}\n", - id, - uuid_ta - ); - mod_dev_dbg!( - self.dev, - "[Submission {}] Frag UUID = {:#x?}\n", - id, - uuid_3d - ); - - let fence = job.fence.clone(); - let frag_job = job.get_frag()?; - - mod_dev_dbg!(self.dev, "[Submission {}] Create Barrier\n", id); - let barrier = kalloc.private.new_init( - pin_init::zeroed::(), - |_inner, _p| { - try_init!(fw::workqueue::raw::Barrier { - tag: fw::workqueue::CommandType::Barrier, - wait_stamp: ev_vtx.fw_stamp_pointer, - wait_value: ev_vtx.value.next(), - wait_slot: ev_vtx.slot, - stamp_self: ev_frag.value.next(), - uuid: uuid_3d, - external_barrier: 0, - internal_barrier_type: 0, - padding: Default::default(), - }) - }, - )?; - - mod_dev_dbg!(self.dev, "[Submission {}] Add Barrier\n", id); - frag_job.add(barrier, vm_bind.slot())?; - - let timestamps = Arc::new( - kalloc.shared.new_default::()?, - GFP_KERNEL, - )?; - - let unk1 = false; - - let mut tile_config: u64 = 0; - if !unk1 { - tile_config |= 0x280; - } - if cmdbuf.layers > 1 { - tile_config |= 1; - } - if cmdbuf.flags & uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_PROCESS_EMPTY_TILES as u32 - != 0 - { - tile_config |= 0x10000; - } - - let samples_log2 = match cmdbuf.samples { - 1 => 0, - 2 => 1, - 4 => 2, - _ => { - cls_pr_debug!(Errors, "Invalid sample count {}\n", cmdbuf.samples); - return Err(EINVAL); - } - }; - - let utile_config = ((tile_info.utile_width / 16) << 12) - | ((tile_info.utile_height / 16) << 14) - | samples_log2; - - // Calculate the number of 2KiB blocks to allocate per utile. This is - // just a bit of dimensional analysis. - let pixels_per_utile: u32 = - (cmdbuf.utile_width_px as u32) * (cmdbuf.utile_height_px as u32); - let samples_per_utile: u32 = pixels_per_utile << samples_log2; - let utile_size_bytes: u32 = (cmdbuf.sample_size_B as u32) * samples_per_utile; - let block_size_bytes: u32 = 2048; - let blocks_per_utile: u32 = utile_size_bytes.div_ceil(block_size_bytes); - - #[ver(G >= G14X)] - let frg_tilecfg = 0x0000000_00036011 - | (((tile_info.tiles_x - 1) as u64) << 44) - | (((tile_info.tiles_y - 1) as u64) << 53) - | (if unk1 { 0 } else { 0x20_00000000 }) - | (if cmdbuf.layers > 1 { 0x1_00000000 } else { 0 }) - | ((utile_config as u64 & 0xf000) << 28); - - // TODO: check - #[ver(V >= V13_0B4)] - let count_frag = self.counter.fetch_add(2, Ordering::Relaxed); - #[ver(V >= V13_0B4)] - let count_vtx = count_frag + 1; - - // Unknowns handling - - #[ver(G >= G14)] - let g14_unk = 0x4040404; - #[ver(G < G14)] - let g14_unk = 0; - #[ver(G < G14X)] - let frg_unk_140 = 0x8c60; - let frg_unk_158 = 0x1c; - #[ver(G >= G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64; - #[ver(G < G14)] - let load_bgobjvals = cmdbuf.isp_bgobjvals as u64 | 0x400; - let reload_zlsctrl = cmdbuf.zls_ctrl; - let iogpu_unk54 = 0x3a0012006b0003; - let iogpu_unk56 = 1; - #[ver(G < G14)] - let tiling_control_2 = 0; - #[ver(G >= G14X)] - let tiling_control_2 = 4; - #[ver(G >= G14X)] - let vtx_unk_f0 = 0x1c; - #[ver(G < G14)] - let vtx_unk_f0 = 0x1c + (align(tile_info.meta1_blocks, 4) as u64); - let vtx_unk_118 = 0x1c; - - // DRM_ASAHI_NEO_RENDER_DBIAS_IS_INT chosen to match hardware bit. - let isp_ctl = 0xc000u32 - | (cmdbuf.flags & uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_DBIAS_IS_INT as u32); - - // Always allow preemption at the UAPI level - let no_preemption = false; - - mod_dev_dbg!(self.dev, "[Submission {}] Create Frag\n", id); - let frag = GpuObject::new_init_prealloc( - kalloc.gpu_ro.alloc_object()?, - |ptr: GpuWeakPointer| { - let scene = scene.clone(); - let notifier = notifier.clone(); - let vm_bind = vm_bind.clone(); - let timestamps = timestamps.clone(); - let private = &mut kalloc.private; - try_init!(fw::fragment::RunFragment::ver { - micro_seq: { - let mut builder = microseq::Builder::new(); - - let stats = inner_weak_ptr!( - gpu.initdata.runtime_pointers.stats.frag.weak_pointer(), - stats - ); - - let start_frag = builder.add(microseq::StartFragment::ver { - header: microseq::op::StartFragment::HEADER, - #[ver(G < G14X)] - job_params2: Some(inner_weak_ptr!(ptr, job_params2)), - #[ver(G < G14X)] - job_params1: Some(inner_weak_ptr!(ptr, job_params1)), - #[ver(G >= G14X)] - job_params1: None, - #[ver(G >= G14X)] - job_params2: None, - #[ver(G >= G14X)] - registers: inner_weak_ptr!(ptr, registers), - scene: scene.gpu_pointer(), - stats, - busy_flag: inner_weak_ptr!(ptr, busy_flag), - tvb_overflow_count: inner_weak_ptr!(ptr, tvb_overflow_count), - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - work_queue: ev_frag.info_ptr, - work_item: ptr, - vm_slot: vm_bind.slot(), - unk_50: 0x1, // fixed - event_generation: self.id as u32, - buffer_slot: scene.slot(), - sync_grow: 0, - event_seq: U64(ev_frag.event_seq), - unk_68: 0, - unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), - unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), - #[ver(V >= V13_3)] - unk_7c_0: U64(0), - unk_7c: 0, - unk_80: 0, - unk_84: unk1.into(), - uuid: uuid_3d, - attachments: *fragment_attachments, - padding: 0, - #[ver(V >= V13_0B4)] - counter: U64(count_frag), - #[ver(V >= V13_0B4)] - notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), - })?; - - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } - - #[ver(G < G14X)] - builder.add(microseq::WaitForIdle { - header: microseq::op::WaitForIdle::new(microseq::Pipe::Fragment), - })?; - #[ver(G >= G14X)] - builder.add(microseq::WaitForIdle2 { - header: microseq::op::WaitForIdle2::HEADER, - })?; - - if frg_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_frag.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_3d, - unk_30_padding: 0, - })?; - } - - let off = builder.offset_to(start_frag); - builder.add(microseq::FinalizeFragment::ver { - header: microseq::op::FinalizeFragment::HEADER, - uuid: uuid_3d, - unk_8: 0, - fw_stamp: ev_frag.fw_stamp_pointer, - stamp_value: ev_frag.value.next(), - unk_18: 0, - scene: scene.weak_pointer(), - buffer: scene.weak_buffer_pointer(), - unk_2c: U64(1), - stats, - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - busy_flag: inner_weak_ptr!(ptr, busy_flag), - work_queue: ev_frag.info_ptr, - work_item: ptr, - vm_slot: vm_bind.slot(), - unk_60: 0, - unk_758_flag: inner_weak_ptr!(ptr, unk_758_flag), - #[ver(V >= V13_3)] - unk_6c_0: U64(0), - unk_6c: U64(0), - unk_74: U64(0), - unk_7c: U64(0), - unk_84: U64(0), - unk_8c: U64(0), - #[ver(G == G14 && V < V13_0B4)] - unk_8c_g14: U64(0), - restart_branch_offset: off, - has_attachments: (fragment_attachments.count > 0) as u32, - #[ver(V >= V13_0B4)] - unk_9c: Default::default(), - })?; - - builder.add(microseq::RetireStamp { - header: microseq::op::RetireStamp::HEADER, - })?; - - builder.build(private)? - }, - notifier, - scene, - vm_bind, - aux_fb: self.ualloc.lock().array_empty_tagged(0x8000, b"AXFB")?, - timestamps, - user_timestamps: frg_user_timestamps, - }) - }, - |inner, _ptr| { - let vm_slot = vm_bind.slot(); - let aux_fb_info = fw::fragment::raw::AuxFBInfo::ver { - isp_ctl: isp_ctl, - unk2: 0, - width: cmdbuf.width_px as u32, - height: cmdbuf.height_px as u32, - #[ver(V >= V13_0B4)] - unk3: U64(0x100000), - }; - - try_init!(fw::fragment::raw::RunFragment::ver { - tag: fw::workqueue::CommandType::RunFragment, - #[ver(V >= V13_0B4)] - counter: U64(count_frag), - vm_slot, - unk_8: 0, - microsequence: inner.micro_seq.gpu_pointer(), - microsequence_size: inner.micro_seq.len() as u32, - notifier: inner.notifier.gpu_pointer(), - buffer: inner.scene.buffer_pointer(), - scene: inner.scene.gpu_pointer(), - unk_buffer_buf: inner.scene.kernel_buffer_pointer(), - tvb_tilemap: inner.scene.tvb_tilemap_pointer(), - ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), - samples: cmdbuf.samples as u32, - tiles_per_mtile_y: tile_info.tiles_per_mtile_y as u16, - tiles_per_mtile_x: tile_info.tiles_per_mtile_x as u16, - unk_50: U64(0), - unk_58: U64(0), - isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), - isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), - unk_68: U64(0), - tile_count: U64(tile_info.tiles as u64), - #[ver(G < G14X)] - job_params1 <- try_init!(fw::fragment::raw::JobParameters1::ver { - utile_config, - unk_4: 0, - bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.bg.rsrc_spec as u64), - address: U64(cmdbuf.bg.usc as u64), - }, - ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), - isp_scissor_base: U64(cmdbuf.isp_scissor_base), - isp_dbias_base: U64(cmdbuf.isp_dbias_base), - isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), - aux_fb_info, - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), - zls_ctrl: U64(cmdbuf.zls_ctrl), - #[ver(G >= G14)] - unk_58_g14_0: U64(g14_unk), - #[ver(G >= G14)] - unk_58_g14_8: U64(0), - z_load: U64(cmdbuf.depth.base), - z_store: U64(cmdbuf.depth.base), - s_load: U64(cmdbuf.stencil.base), - s_store: U64(cmdbuf.stencil.base), - #[ver(G >= G14)] - unk_68_g14_0: Default::default(), - z_load_stride: U64(cmdbuf.depth.stride as u64), - z_store_stride: U64(cmdbuf.depth.stride as u64), - s_load_stride: U64(cmdbuf.stencil.stride as u64), - s_store_stride: U64(cmdbuf.stencil.stride as u64), - z_load_comp: U64(cmdbuf.depth.comp_base), - z_load_comp_stride: U64(cmdbuf.depth.comp_stride as u64), - z_store_comp: U64(cmdbuf.depth.comp_base), - z_store_comp_stride: U64(cmdbuf.depth.comp_stride as u64), - s_load_comp: U64(cmdbuf.stencil.comp_base), - s_load_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), - s_store_comp: U64(cmdbuf.stencil.comp_base), - s_store_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), - tvb_tilemap: inner.scene.tvb_tilemap_pointer(), - tvb_layermeta: inner.scene.tvb_layermeta_pointer(), - mtile_stride_dwords: U64((4 * tile_info.params.rgn_size as u64) << 24), - tvb_heapmeta: inner.scene.tvb_heapmeta_pointer(), - tile_config: U64(tile_config), - aux_fb: inner.aux_fb.gpu_pointer(), - unk_108: Default::default(), - usc_exec_base_isp: U64(self.usc_exec_base), - unk_140: U64(frg_unk_140), - helper_program: cmdbuf.fragment_helper.binary, - unk_14c: 0, - helper_arg: U64(cmdbuf.fragment_helper.data), - unk_158: U64(frg_unk_158), - unk_160: U64(0), - __pad: Default::default(), - #[ver(V < V13_0B4)] - __pad1: Default::default(), - }), - #[ver(G < G14X)] - job_params2 <- try_init!(fw::fragment::raw::JobParameters2 { - eot_rsrc_spec: cmdbuf.eot.rsrc_spec, - eot_usc: cmdbuf.eot.usc, - unk_8: 0x0, - unk_c: 0x0, - isp_merge_upper_x: F32::from_bits(cmdbuf.isp_merge_upper_x), - isp_merge_upper_y: F32::from_bits(cmdbuf.isp_merge_upper_y), - unk_18: U64(0x0), - utiles_per_mtile_y: tile_info.utiles_per_mtile_y as u16, - utiles_per_mtile_x: tile_info.utiles_per_mtile_x as u16, - unk_24: 0x0, - tile_counts: ((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1), - tib_blocks: blocks_per_utile, - isp_bgobjdepth: cmdbuf.isp_bgobjdepth, - // TODO: does this flag need to be exposed to userspace? - isp_bgobjvals: load_bgobjvals as u32, - unk_38: 0x0, - unk_3c: 0x1, - helper_cfg: cmdbuf.fragment_helper.cfg, - __pad: Default::default(), - }), - #[ver(G >= G14X)] - registers: fw::job::raw::RegisterArray::new( - inner_weak_ptr!(_ptr, registers.registers), - |r| { - r.add(0x1739, 1); - r.add(0x10009, utile_config.into()); - r.add(0x15379, cmdbuf.eot.rsrc_spec.into()); - r.add(0x15381, cmdbuf.eot.usc.into()); - r.add(0x15369, cmdbuf.bg.rsrc_spec.into()); - r.add(0x15371, cmdbuf.bg.usc.into()); - r.add(0x15131, cmdbuf.isp_merge_upper_x.into()); - r.add(0x15139, cmdbuf.isp_merge_upper_y.into()); - r.add(0x100a1, 0); - r.add(0x15069, 0); - r.add(0x15071, 0); // pointer - r.add(0x16058, 0); - r.add(0x10019, cmdbuf.ppp_multisamplectl); - let isp_mtile_size = (tile_info.utiles_per_mtile_y - | (tile_info.utiles_per_mtile_x << 16)) - .into(); - r.add(0x100b1, isp_mtile_size); // ISP_MTILE_SIZE - r.add(0x16030, isp_mtile_size); // ISP_MTILE_SIZE - r.add( - 0x100d9, - (((tile_info.tiles_y - 1) << 12) | (tile_info.tiles_x - 1)).into(), - ); // TE_SCREEN - r.add(0x16098, inner.scene.tvb_heapmeta_pointer().into()); - r.add(0x15109, cmdbuf.isp_scissor_base); // ISP_SCISSOR_BASE - r.add(0x15101, cmdbuf.isp_dbias_base); // ISP_DBIAS_BASE - r.add(0x15021, isp_ctl.into()); // aux_fb_info.unk_1 - r.add( - 0x15211, - ((cmdbuf.height_px as u64) << 32) | cmdbuf.width_px as u64, - ); // aux_fb_info.{width, heigh - r.add(0x15049, 0x100000); // s2.aux_fb_info.unk3 - r.add(0x10051, blocks_per_utile.into()); // s1.unk_2c - r.add(0x15321, cmdbuf.isp_zls_pixels.into()); // ISP_ZLS_PIXELS - r.add(0x15301, cmdbuf.isp_bgobjdepth.into()); // ISP_BGOBJDEPTH - r.add(0x15309, load_bgobjvals); // ISP_BGOBJVALS - r.add(0x15311, cmdbuf.isp_oclqry_base); // ISP_OCLQRY_BASE - r.add(0x15319, cmdbuf.zls_ctrl); // ISP_ZLSCTL - r.add(0x15349, g14_unk); // s2.unk_58_g14_0 - r.add(0x15351, 0); // s2.unk_58_g14_8 - r.add(0x15329, cmdbuf.depth.base); // ISP_ZLOAD_BASE - r.add(0x15331, cmdbuf.depth.base); // ISP_ZSTORE_BASE - r.add(0x15339, cmdbuf.stencil.base); // ISP_STENCIL_LOAD_BASE - r.add(0x15341, cmdbuf.stencil.base); // ISP_STENCIL_STORE_BASE - r.add(0x15231, 0); - r.add(0x15221, 0); - r.add(0x15239, 0); - r.add(0x15229, 0); - r.add(0x15401, cmdbuf.depth.stride as u64); // load - r.add(0x15421, cmdbuf.depth.stride as u64); // store - r.add(0x15409, cmdbuf.stencil.stride as u64); // load - r.add(0x15429, cmdbuf.stencil.stride as u64); - r.add(0x153c1, cmdbuf.depth.comp_base); // load - r.add(0x15411, cmdbuf.depth.comp_stride as u64); // load - r.add(0x153c9, cmdbuf.depth.comp_base); // store - r.add(0x15431, cmdbuf.depth.comp_stride as u64); // store - r.add(0x153d1, cmdbuf.stencil.comp_base); // load - r.add(0x15419, cmdbuf.stencil.comp_stride as u64); // load - r.add(0x153d9, cmdbuf.stencil.comp_base); // store - r.add(0x15439, cmdbuf.stencil.comp_stride as u64); // store - r.add(0x16429, inner.scene.tvb_tilemap_pointer().into()); - r.add(0x16060, inner.scene.tvb_layermeta_pointer().into()); - r.add(0x16431, (4 * tile_info.params.rgn_size as u64) << 24); // ISP_RGN? - r.add(0x10039, tile_config); // tile_config ISP_CTL? - r.add(0x16451, 0x0); // ISP_RENDER_ORIGIN - r.add(0x11821, cmdbuf.fragment_helper.binary.into()); - r.add(0x11829, cmdbuf.fragment_helper.data); - r.add(0x11f79, cmdbuf.fragment_helper.cfg.into()); - r.add(0x15359, 0); - r.add(0x10069, self.usc_exec_base); // frag; USC_EXEC_BASE_ISP - r.add(0x16020, 0); - r.add(0x16461, inner.aux_fb.gpu_pointer().into()); - r.add(0x16090, inner.aux_fb.gpu_pointer().into()); - r.add(0x120a1, frg_unk_158); - r.add(0x160a8, 0); - r.add(0x16068, frg_tilecfg); - r.add(0x160b8, 0x0); - /* - r.add(0x10201, 0x100); // Some kind of counter?? Does this matter? - r.add(0x10428, 0x100); // Some kind of counter?? Does this matter? - r.add(0x1c838, 1); // ? - r.add(0x1ca28, 0x1502960f00); // ?? - r.add(0x1731, 0x1); // ?? - */ - } - ), - job_params3 <- try_init!(fw::fragment::raw::JobParameters3::ver { - isp_dbias_base: fw::fragment::raw::ArrayAddr { - ptr: U64(cmdbuf.isp_dbias_base), - unk_padding: U64(0), - }, - isp_scissor_base: fw::fragment::raw::ArrayAddr { - ptr: U64(cmdbuf.isp_scissor_base), - unk_padding: U64(0), - }, - isp_oclqry_base: U64(cmdbuf.isp_oclqry_base), - unk_118: U64(0x0), - unk_120: Default::default(), - unk_partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), - }, - unk_258: U64(0), - unk_260: U64(0), - unk_268: U64(0), - unk_270: U64(0), - partial_bg: fw::fragment::raw::BackgroundProgram { - rsrc_spec: U64(cmdbuf.partial_bg.rsrc_spec as u64), - address: U64(cmdbuf.partial_bg.usc as u64), - }, - zls_ctrl: U64(reload_zlsctrl), - unk_290: U64(g14_unk), - z_load: U64(cmdbuf.depth.base), - z_partial_stride: U64(cmdbuf.depth.stride as u64), - z_partial_comp_stride: U64(cmdbuf.depth.comp_stride as u64), - z_store: U64(cmdbuf.depth.base), - z_partial: U64(cmdbuf.depth.base), - z_partial_comp: U64(cmdbuf.depth.comp_base), - s_load: U64(cmdbuf.stencil.base), - s_partial_stride: U64(cmdbuf.stencil.stride as u64), - s_partial_comp_stride: U64(cmdbuf.stencil.comp_stride as u64), - s_store: U64(cmdbuf.stencil.base), - s_partial: U64(cmdbuf.stencil.base), - s_partial_comp: U64(cmdbuf.stencil.comp_base), - unk_2f8: Default::default(), - tib_blocks: blocks_per_utile, - unk_30c: 0x0, - aux_fb_info, - tile_config: U64(tile_config), - unk_328_padding: Default::default(), - unk_partial_eot: fw::fragment::raw::EotProgram::new( - cmdbuf.partial_eot.rsrc_spec, - cmdbuf.partial_eot.usc - ), - partial_eot: fw::fragment::raw::EotProgram::new( - cmdbuf.partial_eot.rsrc_spec, - cmdbuf.partial_eot.usc - ), - isp_bgobjdepth: cmdbuf.isp_bgobjdepth, - isp_bgobjvals: cmdbuf.isp_bgobjvals, - sample_size: cmdbuf.sample_size_B as u32, - unk_37c: 0x0, - unk_380: U64(0x0), - unk_388: U64(0x0), - #[ver(V >= V13_0B4)] - unk_390_0: U64(0x0), - isp_zls_pixels: U64(cmdbuf.isp_zls_pixels as u64), - }), - unk_758_flag: 0, - unk_75c_flag: 0, - unk_buf: Default::default(), - busy_flag: 0, - tvb_overflow_count: 0, - unk_878: 0, - encoder_params <- try_init!(fw::job::raw::EncoderParams { - // Maybe set when reloading z/s? - unk_8: 0, - sync_grow: 0, - unk_10: 0x0, // fixed - encoder_id: 0, - unk_18: 0x0, // fixed - unk_mask: 0xffffffffu32, - sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, - }), - process_empty_tiles: (cmdbuf.flags - & uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_PROCESS_EMPTY_TILES as u32 - != 0) as u32, - // TODO: needs to be investigated - no_clear_pipeline_textures: 1, - // TODO: needs to be investigated - msaa_zs: 0, - unk_pointee: 0, - #[ver(V >= V13_3)] - unk_v13_3: 0, - meta <- try_init!(fw::job::raw::JobMeta { - unk_0: 0, - unk_2: 0, - no_preemption: no_preemption as u8, - stamp: ev_frag.stamp_pointer, - fw_stamp: ev_frag.fw_stamp_pointer, - stamp_value: ev_frag.value.next(), - stamp_slot: ev_frag.slot, - evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, - uuid: uuid_3d, - event_seq: ev_frag.event_seq as u32, - }), - unk_after_meta: unk1.into(), - unk_buf_0: U64(0), - unk_buf_8: U64(0), - #[ver(G < G14X)] - unk_buf_10: U64(1), - #[ver(G >= G14X)] - unk_buf_10: U64(0), - command_time: U64(0), - timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { - start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.start)), - end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), frag.end)), - }), - user_timestamp_pointers: inner.user_timestamps.pointers()?, - client_sequence: slot_client_seq, - pad_925: Default::default(), - unk_928: 0, - unk_92c: 0, - #[ver(V >= V13_0B4)] - unk_ts: U64(0), - #[ver(V >= V13_0B4)] - unk_92d_8: Default::default(), - }) - }, - )?; - - mod_dev_dbg!(self.dev, "[Submission {}] Add Frag\n", id); - fence.add_command(); - - frag_job.add_cb(frag, vm_bind.slot(), move |error| { - if let Some(err) = error { - fence.set_error(err.into()); - } - - fence.command_complete(); - })?; - - let fence = job.fence.clone(); - let vtx_job = job.get_vtx()?; - - if scene.rebind() || tvb_grown || tvb_autogrown { - mod_dev_dbg!(self.dev, "[Submission {}] Create Bind Buffer\n", id); - let bind_buffer = kalloc.private.new_init( - { - let scene = scene.clone(); - try_init!(fw::buffer::InitBuffer::ver { scene }) - }, - |inner, _ptr| { - let vm_slot = vm_bind.slot(); - try_init!(fw::buffer::raw::InitBuffer::ver { - tag: fw::workqueue::CommandType::InitBuffer, - vm_slot, - buffer_slot: inner.scene.slot(), - unk_c: 0, - block_count: buffer.block_count(), - buffer: inner.scene.buffer_pointer(), - stamp_value: ev_vtx.value.next(), - }) - }, - )?; - - mod_dev_dbg!(self.dev, "[Submission {}] Add Bind Buffer\n", id); - vtx_job.add(bind_buffer, vm_bind.slot())?; - } - - mod_dev_dbg!(self.dev, "[Submission {}] Create Vertex\n", id); - let vtx = GpuObject::new_init_prealloc( - kalloc.gpu_ro.alloc_object()?, - |ptr: GpuWeakPointer| { - let scene = scene.clone(); - let vm_bind = vm_bind.clone(); - let timestamps = timestamps.clone(); - let private = &mut kalloc.private; - try_init!(fw::vertex::RunVertex::ver { - micro_seq: { - let mut builder = microseq::Builder::new(); - - let stats = inner_weak_ptr!( - gpu.initdata.runtime_pointers.stats.vtx.weak_pointer(), - stats - ); - - let start_vtx = builder.add(microseq::StartVertex::ver { - header: microseq::op::StartVertex::HEADER, - #[ver(G < G14X)] - tiling_params: Some(inner_weak_ptr!(ptr, tiling_params)), - #[ver(G < G14X)] - job_params1: Some(inner_weak_ptr!(ptr, job_params1)), - #[ver(G >= G14X)] - tiling_params: None, - #[ver(G >= G14X)] - job_params1: None, - #[ver(G >= G14X)] - registers: inner_weak_ptr!(ptr, registers), - buffer: scene.weak_buffer_pointer(), - scene: scene.weak_pointer(), - stats, - work_queue: ev_vtx.info_ptr, - vm_slot: vm_bind.slot(), - unk_38: 1, // fixed - event_generation: self.id as u32, - buffer_slot: scene.slot(), - unk_44: 0, - event_seq: U64(ev_vtx.event_seq), - unk_50: 0, - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - unk_job_buf: inner_weak_ptr!(ptr, unk_buf_0), - unk_64: 0x0, // fixed - unk_68: unk1.into(), - uuid: uuid_ta, - attachments: *vertex_attachments, - padding: 0, - #[ver(V >= V13_0B4)] - counter: U64(count_vtx), - #[ver(V >= V13_0B4)] - notifier_buf: inner_weak_ptr!(notifier.weak_pointer(), state.unk_buf), - #[ver(V < V13_0B4)] - unk_178: 0x0, // padding? - #[ver(V >= V13_0B4)] - unk_178: (!clustering) as u32, - })?; - - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(true), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.start_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } - - #[ver(G < G14X)] - builder.add(microseq::WaitForIdle { - header: microseq::op::WaitForIdle::new(microseq::Pipe::Vertex), - })?; - #[ver(G >= G14X)] - builder.add(microseq::WaitForIdle2 { - header: microseq::op::WaitForIdle2::HEADER, - })?; - - if vtx_user_timestamps.any() { - builder.add(microseq::Timestamp::ver { - header: microseq::op::Timestamp::new(false), - command_time: inner_weak_ptr!(ptr, command_time), - ts_pointers: inner_weak_ptr!(ptr, timestamp_pointers), - update_ts: inner_weak_ptr!(ptr, timestamp_pointers.end_addr), - work_queue: ev_vtx.info_ptr, - user_ts_pointers: inner_weak_ptr!(ptr, user_timestamp_pointers), - #[ver(V >= V13_0B4)] - unk_ts: inner_weak_ptr!(ptr, unk_ts), - uuid: uuid_ta, - unk_30_padding: 0, - })?; - } - - let off = builder.offset_to(start_vtx); - builder.add(microseq::FinalizeVertex::ver { - header: microseq::op::FinalizeVertex::HEADER, - scene: scene.weak_pointer(), - buffer: scene.weak_buffer_pointer(), - stats, - work_queue: ev_vtx.info_ptr, - vm_slot: vm_bind.slot(), - unk_28: 0x0, // fixed - unk_pointer: inner_weak_ptr!(ptr, unk_pointee), - unk_34: 0x0, // fixed - uuid: uuid_ta, - fw_stamp: ev_vtx.fw_stamp_pointer, - stamp_value: ev_vtx.value.next(), - unk_48: U64(0x0), // fixed - unk_50: 0x0, // fixed - unk_54: 0x0, // fixed - unk_58: U64(0x0), // fixed - unk_60: 0x0, // fixed - unk_64: 0x0, // fixed - unk_68: 0x0, // fixed - #[ver(G >= G14 && V < V13_0B4)] - unk_68_g14: U64(0), - restart_branch_offset: off, - has_attachments: (vertex_attachments.count > 0) as u32, - #[ver(V >= V13_0B4)] - unk_74: Default::default(), // Ventura - })?; - - builder.add(microseq::RetireStamp { - header: microseq::op::RetireStamp::HEADER, - })?; - builder.build(private)? - }, - notifier, - scene, - vm_bind, - timestamps, - user_timestamps: vtx_user_timestamps, - }) - }, - |inner, _ptr| { - let vm_slot = vm_bind.slot(); - #[ver(G < G14)] - let core_masks = gpu.core_masks_packed(); - - try_init!(fw::vertex::raw::RunVertex::ver { - tag: fw::workqueue::CommandType::RunVertex, - #[ver(V >= V13_0B4)] - counter: U64(count_vtx), - vm_slot, - unk_8: 0, - notifier: inner.notifier.gpu_pointer(), - buffer_slot: inner.scene.slot(), - unk_1c: 0, - buffer: inner.scene.buffer_pointer(), - scene: inner.scene.gpu_pointer(), - unk_buffer_buf: inner.scene.kernel_buffer_pointer(), - unk_34: 0, - #[ver(G < G14X)] - job_params1 <- try_init!(fw::vertex::raw::JobParameters1::ver { - unk_0: U64(if unk1 { 0 } else { 0x200 }), // sometimes 0 - unk_8: f32!(1e-20), // fixed - unk_c: f32!(1e-20), // fixed - tvb_tilemap: inner.scene.tvb_tilemap_pointer(), - #[ver(G < G14)] - tvb_cluster_tilemaps: inner.scene.cluster_tilemaps_pointer(), - tpc: inner.scene.tpc_pointer(), - tvb_heapmeta: inner.scene.tvb_heapmeta_pointer().or(0x8000_0000_0000_0000), - iogpu_unk_54: U64(iogpu_unk54), // fixed - iogpu_unk_56: U64(iogpu_unk56), // fixed - #[ver(G < G14)] - tvb_cluster_meta1: inner - .scene - .meta_1_pointer() - .map(|x| x.or((tile_info.meta1_layer_stride as u64) << 50)), - utile_config, - unk_4c: 0, - ppp_multisamplectl: U64(cmdbuf.ppp_multisamplectl), // fixed - tvb_layermeta: inner.scene.tvb_layermeta_pointer(), - #[ver(G < G14)] - tvb_cluster_layermeta: inner.scene.tvb_cluster_layermeta_pointer(), - #[ver(G < G14)] - core_mask: Array::new([ - *core_masks.first().unwrap_or(&0), - *core_masks.get(1).unwrap_or(&0), - ]), - preempt_buf1: inner.scene.preempt_buf_1_pointer(), - preempt_buf2: inner.scene.preempt_buf_2_pointer(), - unk_80: U64(0x1), // fixed - preempt_buf3: inner.scene.preempt_buf_3_pointer().or(0x4_0000_0000_0000), // check - vdm_ctrl_stream_base: U64(cmdbuf.vdm_ctrl_stream_base), - #[ver(G < G14)] - tvb_cluster_meta2: inner.scene.meta_2_pointer(), - #[ver(G < G14)] - tvb_cluster_meta3: inner.scene.meta_3_pointer(), - #[ver(G < G14)] - tiling_control, - #[ver(G < G14)] - unk_ac: tiling_control_2 as u32, // fixed - unk_b0: Default::default(), // fixed - usc_exec_base_ta: U64(self.usc_exec_base), - #[ver(G < G14)] - tvb_cluster_meta4: inner - .scene - .meta_4_pointer() - .map(|x| x.or(0x3000_0000_0000_0000)), - #[ver(G < G14)] - unk_f0: U64(vtx_unk_f0), - unk_f8: U64(0x8c60), // fixed - helper_program: cmdbuf.vertex_helper.binary, - unk_104: 0, - helper_arg: U64(cmdbuf.vertex_helper.data), - unk_110: Default::default(), // fixed - unk_118: vtx_unk_118 as u32, // fixed - __pad: Default::default(), - }), - #[ver(G < G14X)] - tiling_params: tile_info.params, - #[ver(G >= G14X)] - registers: fw::job::raw::RegisterArray::new( - inner_weak_ptr!(_ptr, registers.registers), - |r| { - r.add(0x10141, if unk1 { 0 } else { 0x200 }); // s2.unk_0 - r.add(0x1c039, inner.scene.tvb_tilemap_pointer().into()); - r.add(0x1c9c8, inner.scene.tvb_tilemap_pointer().into()); - - let cl_tilemaps_ptr = inner - .scene - .cluster_tilemaps_pointer() - .map_or(0, |a| a.into()); - r.add(0x1c041, cl_tilemaps_ptr); - r.add(0x1c9d0, cl_tilemaps_ptr); - r.add(0x1c0a1, inner.scene.tpc_pointer().into()); // TE_TPC_ADDR - - let tvb_heapmeta_ptr = inner - .scene - .tvb_heapmeta_pointer() - .or(0x8000_0000_0000_0000) - .into(); - r.add(0x1c031, tvb_heapmeta_ptr); - r.add(0x1c9c0, tvb_heapmeta_ptr); - r.add(0x1c051, iogpu_unk54); // iogpu_unk_54/55 - r.add(0x1c061, iogpu_unk56); // iogpu_unk_56 - r.add(0x10149, utile_config.into()); // s2.unk_48 utile_config - r.add(0x10139, cmdbuf.ppp_multisamplectl); // PPP_MULTISAMPLECTL - r.add(0x10111, inner.scene.preempt_buf_1_pointer().into()); - r.add(0x1c9b0, inner.scene.preempt_buf_1_pointer().into()); - r.add(0x10119, inner.scene.preempt_buf_2_pointer().into()); - r.add(0x1c9b8, inner.scene.preempt_buf_2_pointer().into()); - r.add(0x1c958, 1); // s2.unk_80 - r.add( - 0x1c950, - inner - .scene - .preempt_buf_3_pointer() - .or(0x4_0000_0000_0000) - .into(), - ); - r.add(0x1c930, 0); // VCE related addr, lsb to enable - r.add(0x1c880, cmdbuf.vdm_ctrl_stream_base); // VDM_CTRL_STREAM_BASE - r.add(0x1c898, 0x0); // if lsb set, faults in UL1C0, possibly missing addr. - r.add( - 0x1c948, - inner.scene.meta_2_pointer().map_or(0, |a| a.into()), - ); // tvb_cluster_meta2 - r.add( - 0x1c888, - inner.scene.meta_3_pointer().map_or(0, |a| a.into()), - ); // tvb_cluster_meta3 - r.add(0x1c890, tiling_control.into()); // tvb_tiling_control - r.add(0x1c918, tiling_control_2); - r.add(0x1c079, inner.scene.tvb_layermeta_pointer().into()); - r.add(0x1c9d8, inner.scene.tvb_layermeta_pointer().into()); - let cl_layermeta_pointer = - inner.scene.tvb_cluster_layermeta_pointer().map_or(0, |a| a.into()); - r.add(0x1c089, cl_layermeta_pointer); - r.add(0x1c9e0, cl_layermeta_pointer); - let cl_meta_4_pointer = - inner.scene.meta_4_pointer().map_or(0, |a| a.into()); - r.add(0x16c41, cl_meta_4_pointer); // tvb_cluster_meta4 - r.add(0x1ca40, cl_meta_4_pointer); // tvb_cluster_meta4 - r.add(0x1c9a8, vtx_unk_f0); // + meta1_blocks? min_free_tvb_pages? - r.add( - 0x1c920, - inner.scene.meta_1_pointer().map_or(0, |a| a.into()), - ); // ??? | meta1_blocks? - r.add(0x10151, 0); - r.add(0x1c199, 0); - r.add(0x1c1a1, 0); - r.add(0x1c1a9, 0); // 0x10151 bit 1 enables - r.add(0x1c1b1, 0); - r.add(0x1c1b9, 0); - r.add(0x10061, self.usc_exec_base); // USC_EXEC_BASE_TA - r.add(0x11801, cmdbuf.vertex_helper.binary.into()); - r.add(0x11809, cmdbuf.vertex_helper.data); - r.add(0x11f71, cmdbuf.vertex_helper.cfg.into()); - r.add(0x1c0b1, tile_info.params.rgn_size.into()); // TE_PSG - r.add(0x1c850, tile_info.params.rgn_size.into()); - r.add(0x10131, tile_info.params.unk_4.into()); - r.add(0x10121, tile_info.params.ppp_ctrl.into()); // PPP_CTRL - r.add( - 0x10129, - tile_info.params.x_max as u64 - | ((tile_info.params.y_max as u64) << 16), - ); // PPP_SCREEN - r.add(0x101b9, tile_info.params.te_screen.into()); // TE_SCREEN - r.add(0x1c069, tile_info.params.te_mtile1.into()); // TE_MTILE1 - r.add(0x1c071, tile_info.params.te_mtile2.into()); // TE_MTILE2 - r.add(0x1c081, tile_info.params.tiles_per_mtile.into()); // TE_MTILE - r.add(0x1c0a9, tile_info.params.tpc_stride.into()); // TE_TPC - r.add(0x10171, tile_info.params.unk_24.into()); - r.add(0x10169, tile_info.params.unk_28.into()); // TA_RENDER_TARGET_MAX - r.add(0x12099, vtx_unk_118); - r.add(0x1c9e8, (tile_info.params.unk_28 & 0x4fff).into()); - /* - r.add(0x10209, 0x100); // Some kind of counter?? Does this matter? - r.add(0x1c9f0, 0x100); // Some kind of counter?? Does this matter? - r.add(0x1c830, 1); // ? - r.add(0x1ca30, 0x1502960e60); // ? - r.add(0x16c39, 0x1502960e60); // ? - r.add(0x1c910, 0xa0000b011d); // ? - r.add(0x1c8e0, 0xff); // cluster mask - r.add(0x1c8e8, 0); // ? - */ - } - ), - tpc: inner.scene.tpc_pointer(), - tpc_size: U64(tile_info.tpc_size as u64), - microsequence: inner.micro_seq.gpu_pointer(), - microsequence_size: inner.micro_seq.len() as u32, - fragment_stamp_slot: ev_frag.slot, - fragment_stamp_value: ev_frag.value.next(), - unk_pointee: 0, - unk_pad: 0, - job_params2 <- try_init!(fw::vertex::raw::JobParameters2 { - unk_480: Default::default(), // fixed - unk_498: U64(0x0), // fixed - unk_4a0: 0x0, // fixed - preempt_buf1: inner.scene.preempt_buf_1_pointer(), - unk_4ac: 0x0, // fixed - unk_4b0: U64(0x0), // fixed - unk_4b8: 0x0, // fixed - unk_4bc: U64(0x0), // fixed - unk_4c4_padding: Default::default(), - unk_50c: 0x0, // fixed - unk_510: U64(0x0), // fixed - unk_518: U64(0x0), // fixed - unk_520: U64(0x0), // fixed - }), - encoder_params <- try_init!(fw::job::raw::EncoderParams { - unk_8: 0x0, // fixed - sync_grow: 0x0, // fixed - unk_10: 0x0, // fixed - encoder_id: 0, - unk_18: 0x0, // fixed - unk_mask: 0xffffffffu32, - sampler_array: U64(cmdbuf.sampler_heap), - sampler_count: cmdbuf.sampler_count as u32, - sampler_max: (cmdbuf.sampler_count as u32) + 1, - }), - unk_55c: 0, - unk_560: 0, - sync_grow: 0, - unk_568: 0, - uses_scratch: (cmdbuf.flags - & uapi::drm_asahi_neo_render_flags_DRM_ASAHI_NEO_RENDER_VERTEX_SCRATCH as u32 - != 0) as u32, - meta <- try_init!(fw::job::raw::JobMeta { - unk_0: 0, - unk_2: 0, - no_preemption: no_preemption as u8, - stamp: ev_vtx.stamp_pointer, - fw_stamp: ev_vtx.fw_stamp_pointer, - stamp_value: ev_vtx.value.next(), - stamp_slot: ev_vtx.slot, - evctl_index: 0, // fixed - flush_stamps: flush_stamps as u32, - uuid: uuid_ta, - event_seq: ev_vtx.event_seq as u32, - }), - unk_after_meta: unk1.into(), - unk_buf_0: U64(0), - unk_buf_8: U64(0), - unk_buf_10: U64(0), - command_time: U64(0), - timestamp_pointers <- try_init!(fw::job::raw::TimestampPointers { - start_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.start)), - end_addr: Some(inner_ptr!(inner.timestamps.gpu_pointer(), vtx.end)), - }), - user_timestamp_pointers: inner.user_timestamps.pointers()?, - client_sequence: slot_client_seq, - pad_5d5: Default::default(), - unk_5d8: 0, - unk_5dc: 0, - #[ver(V >= V13_0B4)] - unk_ts: U64(0), - #[ver(V >= V13_0B4)] - unk_5dd_8: Default::default(), - }) - }, - )?; - - core::mem::drop(alloc); - - mod_dev_dbg!(self.dev, "[Submission {}] Add Vertex\n", id); - fence.add_command(); - vtx_job.add_cb(vtx, vm_bind.slot(), move |error| { - if let Some(err) = error { - fence.set_error(err.into()) - } - - fence.command_complete(); - })?; - - mod_dev_dbg!(self.dev, "[Submission {}] Increment counters\n", id); - - // TODO: handle rollbacks, move to job submit? - buffer.increment(); - - job.get_vtx()?.next_seq(); - job.get_frag()?.next_seq(); - - Ok(()) - } -} diff --git a/drivers/gpu/drm/asahi-neo/regs.rs b/drivers/gpu/drm/asahi-neo/regs.rs deleted file mode 100644 index 2bfd754864358e..00000000000000 --- a/drivers/gpu/drm/asahi-neo/regs.rs +++ /dev/null @@ -1,742 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU MMIO register abstraction -//! -//! Since the vast majority of the interactions with the GPU are brokered through the firmware, -//! there is very little need to interact directly with GPU MMIO register. This module abstracts -//! the few operations that require that, mainly reading the MMU fault status, reading GPU ID -//! information, and starting the GPU firmware coprocessor. -//! -//! On G17, the driver also enables the bridge between the GPU and the fabric before the -//! coprocessors start, and enables dynamic power gating before the firmware is initialized. - -use crate::hw; -use core::mem::MaybeUninit; -use kernel::{ - bindings, c_str, - device::Core, - devres::Devres, - error::{from_err_ptr, to_result}, - io::{ - mem::IoMem, // - Io, - }, - platform, - prelude::*, - sync::{aref::ARef, Arc}, // -}; - -/// Size of the ASC control MMIO region. -pub(crate) const ASC_CTL_SIZE: usize = 0x4000; - -/// Size of the SGX MMIO region. -pub(crate) const SGX_SIZE: usize = 0x1000000; - -const CPU_CONTROL: usize = 0x44; -const CPU_RUN: u32 = 0x1 << 4; // BIT(4) - -const FAULT_INFO: usize = 0x17030; - -/// A nonzero low nibble permits writes through the G17 queue configuration port. -const G17_HOST_IRQ_SUMMARY: usize = 0xe01000; -const G17_HOST_POWERED: u32 = 0xf; - -const ID_VERSION: usize = 0xd04000; -const ID_UNK08: usize = 0xd04008; -const ID_COUNTS_1: usize = 0xd04010; -const ID_COUNTS_2: usize = 0xd04014; -const ID_UNK18: usize = 0xd04018; -const ID_CLUSTERS: usize = 0xd0401c; - -const CORE_MASK_0: usize = 0xd01500; -const CORE_MASK_1: usize = 0xd01514; - -const CORE_MASKS_G14X: usize = 0xe01500; -const FAULT_INFO_G14X: usize = 0xd8c0; -const FAULT_ADDR_G14X: usize = 0xd8c8; - -/// G17 GPU-to-fabric bridge control words. Bit 0 enables the bridge; the other bits are status. -const G17_BRIDGE_CONTROL: [usize; 2] = [0x1000104, 0x1000108]; -const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) - -/// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. -const G17_GATING_CONTROL: usize = 0xd06030; -const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) - -/// These performance selectors must be sampled before firmware can gate the core registers. -const G17_PERF_CONTROL: usize = 0xe0141c; -const G17_PERF_MAP: usize = 0xe01480; -const G17_CORE_POWER: usize = 0xe01000; - -/// Enum representing the unit that caused an MMU fault. -#[allow(non_camel_case_types)] -#[allow(clippy::upper_case_acronyms)] -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub(crate) enum FaultUnit { - /// Decompress / pixel fetch - DCMP(u8), - /// USC L1 Cache (device loads/stores) - UL1C(u8), - /// Compress / pixel store - CMP(u8), - GSL1(u8), - IAP(u8), - VCE(u8), - /// Tiling Engine - TE(u8), - RAS(u8), - /// Vertex Data Master - VDM(u8), - PPP(u8), - /// ISP Parameter Fetch - IPF(u8), - IPF_CPF(u8), - VF(u8), - VF_CPF(u8), - /// Depth/Stencil load/store - ZLS(u8), - - /// Parameter Management - dPM, - /// Compute Data Master - dCDM_KS(u8), - dIPP, - dIPP_CS, - // Vertex Data Master - dVDM_CSD, - dVDM_SSD, - dVDM_ILF, - dVDM_ILD, - dRDE(u8), - FC, - GSL2, - - /// Graphics L2 Cache Control? - GL2CC_META(u8), - GL2CC_MB, - - /// Parameter Management - gPM_SP(u8), - /// Vertex Data Master - CSD - gVDM_CSD_SP(u8), - gVDM_SSD_SP(u8), - gVDM_ILF_SP(u8), - gVDM_TFP_SP(u8), - gVDM_MMB_SP(u8), - /// Compute Data Master - gCDM_CS_KS0_SP(u8), - gCDM_CS_KS1_SP(u8), - gCDM_CS_KS2_SP(u8), - gCDM_KS0_SP(u8), - gCDM_KS1_SP(u8), - gCDM_KS2_SP(u8), - gIPP_SP(u8), - gIPP_CS_SP(u8), - gRDE0_SP(u8), - gRDE1_SP(u8), - - gCDM_CS, - gCDM_ID, - gCDM_CSR, - gCDM_CSW, - gCDM_CTXR, - gCDM_CTXW, - gIPP, - gIPP_CS, - gKSM_RCE, - - Unknown(u8), -} - -/// Reason for an MMU fault. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub(crate) enum FaultReason { - Unmapped, - AfFault, - WriteOnly, - ReadOnly, - NoAccess, - Unknown(u8), -} - -/// Collection of information about an MMU fault. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub(crate) struct FaultInfo { - pub(crate) address: u64, - pub(crate) sideband: u8, - pub(crate) vm_slot: u32, - pub(crate) unit_code: u8, - pub(crate) unit: FaultUnit, - pub(crate) level: u8, - pub(crate) unk_5: u8, - pub(crate) read: bool, - pub(crate) reason: FaultReason, -} - -/// Identification of a G17 GPU. -pub(crate) struct G17Id { - /// GPU family field of the ID register. - pub(crate) family: u8, - /// GPU variant field of the ID register. - pub(crate) variant: u8, - /// Silicon revision. - pub(crate) gpu_rev: hw::GpuRevision, - /// Number of dies. - pub(crate) num_dies: u32, - /// Number of clusters, over all dies. - pub(crate) num_clusters: u32, - /// Number of cores per cluster. - pub(crate) num_cores: u32, - /// Enabled cores of each cluster. - pub(crate) core_masks: KVec, - /// Performance selector control, sampled before core power gating starts. - pub(crate) perf_control: u32, - /// Performance selector table indices, low and high nibbles. - pub(crate) perf_map: [u32; 2], -} - -/// Provider-owned lifecycle of one G17 firmware coprocessor. -/// -/// The mailbox provider owns the wrapper mappings and runtime-PM references. -/// The consumer device link keeps it alive until this GPU has stopped using it. -pub(crate) struct CpuControl { - dev: ARef, - mbox: *mut bindings::apple_mbox, -} - -// SAFETY: The live provider serializes CPU transitions with its lifecycle mutex; -// the device link orders provider removal after this consumer's teardown. -unsafe impl Send for CpuControl {} -// SAFETY: All accesses use the same provider-serialized lifecycle API. -unsafe impl Sync for CpuControl {} - -impl CpuControl { - /// Resolve the mailbox phandle and validate its firmware role before use. - pub(crate) fn new(pdev: &platform::Device, index: usize) -> Result { - let expected = match index { - 0 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX, - 1 => bindings::apple_mbox_ascwrap_v6_role_APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1, - _ => return Err(EINVAL), - }; - // SAFETY: pdev is live; the provider establishes the consumer device link. - let mbox = unsafe { - from_err_ptr(bindings::apple_mbox_get(pdev.as_ref().as_raw(), index.try_into()?))? - }; - let mut lifecycle = MaybeUninit::uninit(); - // SAFETY: mbox is live, and lifecycle is writable storage for the C result. - to_result(unsafe { - bindings::apple_mbox_ascwrap_v6_get_lifecycle(mbox, lifecycle.as_mut_ptr()) - })?; - // SAFETY: A successful provider query initialized every field. - if unsafe { lifecycle.assume_init() }.role != expected { - return Err(EINVAL); - } - // SAFETY: This only checks the provider's matching start/stop capability. - to_result(unsafe { bindings::apple_mbox_ascwrap_v6_require_safe_cpu_lifecycle(mbox) })?; - Ok(Self { dev: pdev.into(), mbox }) - } - - /// Start through the owner of the CPU registers and runtime-PM reference. - pub(crate) fn start(&self) -> Result { - // SAFETY: The constructor established the live, role-checked provider. - to_result(unsafe { bindings::apple_mbox_ascwrap_v6_start_cpu(self.mbox) }) - } - - /// Stop before releasing the transport or any firmware-visible memory. - pub(crate) fn stop(&self) { - // SAFETY: The provider remains live during consumer teardown. Removal - // stops the CPU before setting the fence that can return ENODEV here. - let result = to_result(unsafe { bindings::apple_mbox_ascwrap_v6_stop_cpu(self.mbox) }); - if let Err(error) = result { - if error != ENODEV { - dev_err!(self.dev.as_ref(), "Could not stop GPU coprocessor: {:?}\n", error); - } - } - } -} - -/// Device resources for this GPU instance. -#[derive(Clone)] -pub(crate) struct Resources { - dev: ARef, - sgx: Arc>>, -} - -impl Resources { - /// Map the required resources given our platform device. - pub(crate) fn new(pdev: &platform::Device) -> Result { - let sgx_req = pdev.io_request_by_name(c_str!("sgx")).ok_or(EINVAL)?; - let sgx_iomem = Arc::pin_init(sgx_req.iomap_sized::(), GFP_KERNEL)?; - - Ok(Resources { - // SAFETY: This device does DMA via the UAT IOMMU. - dev: pdev.into(), - sgx: sgx_iomem, - }) - } - - /// Installs both render queues while the caller holds the device mutex. - /// No other port publisher may interleave between this power witness and - /// the final validity strobe. False means no write was attempted. - pub(crate) fn install_render_pair( - &self, - registration: &crate::g17::fw::kick::RenderRegistration, - ) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { - return Ok(false); - } - for &(offset, value) in registration.writes() { - sgx.try_write64(value, offset)?; - } - Ok(true) - } - - fn sgx_read32(&self) -> u32 { - if let Some(sgx) = self.sgx.try_access() { - sgx.relaxed().read32(OFF) - } else { - 0 - } - } - - /* Not yet used - fn sgx_write32(&self, val: u32) { - if let Some(sgx) = self.sgx.try_access() { - sgx.write32_relaxed(val, OFF) - } - } - */ - - fn sgx_read64(&self) -> u64 { - if let Some(sgx) = self.sgx.try_access() { - sgx.relaxed().read64(OFF) - } else { - 0 - } - } - - /* Not yet used - fn sgx_write64(&self, val: u64) { - if let Some(sgx) = self.sgx.try_access() { - sgx.write64_relaxed(val, OFF) - } - } - */ - - /// Initialize the MMIO registers for the GPU. - pub(crate) fn init_mmio(&self) -> Result { - // Nothing to do for now... - - Ok(()) - } - - /// Start the ASC coprocessor CPU. - pub(crate) fn start_cpu(pdev: &platform::Device) -> Result { - let asc_req = pdev.io_request_by_name(c_str!("asc")).ok_or(EINVAL)?; - let asc_iomem = KBox::pin_init(asc_req.iomap_sized::(), GFP_KERNEL)?; - let res = asc_iomem.access(pdev.as_ref())?.relaxed(); - - let val = res.read32(CPU_CONTROL); - res.write32(val | CPU_RUN, CPU_CONTROL); - Ok(()) - } - - /// Get the GPU identification info from registers. - /// - /// See [`hw::GpuIdConfig`] for the result. - pub(crate) fn get_gpu_id(&self) -> Result { - let id_version = self.sgx_read32::(); - let id_unk08 = self.sgx_read32::(); - let id_counts_1 = self.sgx_read32::(); - let id_counts_2 = self.sgx_read32::(); - let id_unk18 = self.sgx_read32::(); - let id_clusters = self.sgx_read32::(); - - dev_info!( - self.dev.as_ref(), - "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", - id_version, - id_unk08, - id_counts_1, - id_counts_2, - id_unk18, - id_clusters - ); - - let gpu_gen = (id_version >> 24) & 0xff; - - let mut core_mask_regs = KVec::new(); - - let num_clusters = match gpu_gen { - 4 | 5 => { - // G13 | G14G - core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; - core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; - (id_clusters >> 12) & 0xff - } - 6 => { - // G14X - core_mask_regs.push(self.sgx_read32::(), GFP_KERNEL)?; - core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), GFP_KERNEL)?; - core_mask_regs.push(self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), GFP_KERNEL)?; - // Clusters per die * num dies - ((id_counts_1 >> 8) & 0xff) * ((id_counts_1 >> 16) & 0xf) - } - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); - return Err(ENODEV); - } - }; - - let mut core_masks_packed = KVec::new(); - core_masks_packed.extend_from_slice(&core_mask_regs, GFP_KERNEL)?; - - dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_masks_packed); - - let num_cores = id_counts_1 & 0xff; - - if num_cores > 32 { - dev_err!( - self.dev.as_ref(), - "Too many cores per cluster ({} > 32)\n", - num_cores - ); - return Err(ENODEV); - } - - if num_cores * num_clusters > (core_mask_regs.len() * 32) as u32 { - dev_err!( - self.dev.as_ref(), - "Too many total cores ({} x {} > {})\n", - num_clusters, - num_cores, - core_mask_regs.len() * 32 - ); - return Err(ENODEV); - } - - let (core_masks, total_active_cores) = - self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; - let (gpu_rev, gpu_rev_id) = self.gpu_revision(id_version)?; - - Ok(hw::GpuIdConfig { - gpu_gen: match (id_version >> 24) & 0xff { - 4 => hw::GpuGen::G13, - 5 => hw::GpuGen::G14, - 6 => hw::GpuGen::G14, // G14X has a separate ID - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU generation {}\n", a); - return Err(ENODEV); - } - }, - gpu_variant: match (id_version >> 16) & 0xff { - 1 => hw::GpuVariant::P, // Guess - 2 => hw::GpuVariant::G, - 3 => hw::GpuVariant::S, - 4 => { - if num_clusters > 4 { - hw::GpuVariant::D - } else { - hw::GpuVariant::C - } - } - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU variant {}\n", a); - return Err(ENODEV); - } - }, - gpu_rev, - gpu_rev_id, - num_clusters, - num_cores, - num_frags: num_cores, // Used to be id_counts_1[15:8] but does not work for G14X - num_gps: (id_counts_2 >> 16) & 0xff, - total_active_cores, - core_masks, - core_masks_packed, - }) - } - - /// Get the identification of a G17 GPU from its registers. - pub(crate) fn get_g17_id(&self) -> Result { - let id_version = self.sgx_read32::(); - let id_unk08 = self.sgx_read32::(); - let id_counts_1 = self.sgx_read32::(); - let id_counts_2 = self.sgx_read32::(); - let id_unk18 = self.sgx_read32::(); - let id_clusters = self.sgx_read32::(); - - dev_info!( - self.dev.as_ref(), - "GPU ID registers: {:#x} {:#x} {:#x} {:#x} {:#x} {:#x}\n", - id_version, - id_unk08, - id_counts_1, - id_counts_2, - id_unk18, - id_clusters - ); - - // Later reads can fault while the firmware has gated the cores. Submission uses only - // these saved values and never re-reads the registers. - let perf_control = self.sgx_read32::() & 1; - let perf_map = self.sgx_read32::(); - let mut core_mask_regs = [ - self.sgx_read32::(), - self.sgx_read32::<{ CORE_MASKS_G14X + 4 }>(), - self.sgx_read32::<{ CORE_MASKS_G14X + 8 }>(), - ]; - dev_info!(self.dev.as_ref(), "Core masks: {:#x?}\n", core_mask_regs); - - let num_dies = (id_counts_1 >> 16) & 0xf; - let num_clusters = ((id_counts_1 >> 8) & 0xff) * num_dies; - let num_cores = id_counts_1 & 0xff; - if num_cores == 0 || num_cores > 32 || num_cores * num_clusters > 3 * 32 { - dev_err!( - self.dev.as_ref(), - "Invalid core counts ({} x {})\n", - num_clusters, - num_cores - ); - return Err(ENODEV); - } - let (core_masks, _) = - self.split_core_masks(&mut core_mask_regs, num_clusters, num_cores)?; - let (gpu_rev, _) = self.gpu_revision(id_version)?; - - Ok(G17Id { - family: ((id_version >> 24) & 0xff) as u8, - variant: ((id_version >> 16) & 0xff) as u8, - gpu_rev, - num_dies, - num_clusters, - num_cores, - core_masks, - perf_control, - perf_map: [perf_map & 0xf, (perf_map >> 16) & 0xf], - }) - } - - /// Enable the G17 GPU-to-fabric bridge and check that it took effect. - /// - /// The GPU power domain must be on. The coprocessors cannot reach memory until this is done. - pub(crate) fn enable_g17_bridge(&self) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - for offset in G17_BRIDGE_CONTROL { - let val = sgx.try_read32(offset)?; - sgx.try_write32(val | G17_BRIDGE_ENABLE, offset)?; - if sgx.try_read32(offset)? & G17_BRIDGE_ENABLE == 0 { - dev_err!( - self.dev.as_ref(), - "GPU bridge {:#x} did not enable\n", - offset - ); - return Err(EIO); - } - } - Ok(()) - } - - /// Disable the G17 GPU-to-fabric bridge. Both coprocessors must be stopped. - pub(crate) fn disable_g17_bridge(&self) { - if let Some(sgx) = self.sgx.try_access() { - for offset in G17_BRIDGE_CONTROL { - if let Ok(val) = sgx.try_read32(offset) { - // Cannot fail: the offset was read just before. - let _ = sgx.try_write32(val & !G17_BRIDGE_ENABLE, offset); - } - } - } - } - - /// Power witness for the queue-configuration port, accessible while the cores are gated. - pub(crate) fn g17_configuration_powered(&self) -> bool { - self.sgx_read32::() & 0xf != 0 - } - - /// Enable dynamic power gating of idle G17 GPU cores. The firmware expects this before it is - /// initialized. - pub(crate) fn enable_g17_gating(&self) -> Result { - let sgx = self.sgx.try_access().ok_or(ENODEV)?; - let val = sgx.try_read32(G17_GATING_CONTROL)?; - sgx.try_write32(val & !G17_GATING_DISABLE, G17_GATING_CONTROL) - } - - /// Split the packed per-cluster core masks into one mask per cluster. - /// - /// Returns the masks and the total number of enabled cores. - fn split_core_masks( - &self, - core_mask_regs: &mut [u32], - num_clusters: u32, - num_cores: u32, - ) -> Result<(KVec, u32)> { - let mut core_masks = KVec::new(); - let mut total_active_cores: u32 = 0; - - let max_core_mask = ((1u64 << num_cores) - 1) as u32; - for _ in 0..num_clusters { - let mask = core_mask_regs[0] & max_core_mask; - core_masks.push(mask, GFP_KERNEL)?; - for i in 0..core_mask_regs.len() { - core_mask_regs[i] = core_mask_regs[i].checked_shr(num_cores).unwrap_or(0); - if i < (core_mask_regs.len() - 1) { - core_mask_regs[i] |= core_mask_regs[i + 1] - .checked_shl(32 - num_cores) - .unwrap_or(0); - } - } - total_active_cores += mask.count_ones(); - } - - if core_mask_regs.iter().any(|a| *a != 0) { - dev_err!( - self.dev.as_ref(), - "Leftover core mask: {:#x?}\n", - core_mask_regs - ); - return Err(EIO); - } - - Ok((core_masks, total_active_cores)) - } - - /// Decode the silicon revision from the ID version register. - fn gpu_revision(&self, id_version: u32) -> Result<(hw::GpuRevision, hw::GpuRevisionID)> { - Ok(match (id_version >> 8) & 0xff { - 0x00 => (hw::GpuRevision::A0, hw::GpuRevisionID::A0), - 0x01 => (hw::GpuRevision::A1, hw::GpuRevisionID::A1), - 0x10 => (hw::GpuRevision::B0, hw::GpuRevisionID::B0), - 0x11 => (hw::GpuRevision::B1, hw::GpuRevisionID::B1), - 0x20 => (hw::GpuRevision::C0, hw::GpuRevisionID::C0), - 0x21 => (hw::GpuRevision::C1, hw::GpuRevisionID::C1), - a => { - dev_err!(self.dev.as_ref(), "Unknown GPU revision {}\n", a); - return Err(ENODEV); - } - }) - } - - /// Get the fault information from the MMU status register, if one occurred. - pub(crate) fn get_fault_info(&self, cfg: &'static hw::HwConfig) -> Option { - let g14x = cfg.gpu_core as u32 >= hw::GpuCore::G14S as u32; - - let fault_info = if g14x { - self.sgx_read64::() - } else { - self.sgx_read64::() - }; - - if fault_info & 1 == 0 { - return None; - } - - let fault_addr = if g14x { - self.sgx_read64::() - } else { - fault_info >> 30 - }; - - let unit_code = ((fault_info >> 9) & 0xff) as u8; - let unit = match unit_code { - 0x00..=0x9f => match unit_code & 0xf { - 0x0 => FaultUnit::DCMP(unit_code >> 4), - 0x1 => FaultUnit::UL1C(unit_code >> 4), - 0x2 => FaultUnit::CMP(unit_code >> 4), - 0x3 => FaultUnit::GSL1(unit_code >> 4), - 0x4 => FaultUnit::IAP(unit_code >> 4), - 0x5 => FaultUnit::VCE(unit_code >> 4), - 0x6 => FaultUnit::TE(unit_code >> 4), - 0x7 => FaultUnit::RAS(unit_code >> 4), - 0x8 => FaultUnit::VDM(unit_code >> 4), - 0x9 => FaultUnit::PPP(unit_code >> 4), - 0xa => FaultUnit::IPF(unit_code >> 4), - 0xb => FaultUnit::IPF_CPF(unit_code >> 4), - 0xc => FaultUnit::VF(unit_code >> 4), - 0xd => FaultUnit::VF_CPF(unit_code >> 4), - 0xe => FaultUnit::ZLS(unit_code >> 4), - _ => FaultUnit::Unknown(unit_code), - }, - 0xa1 => FaultUnit::dPM, - 0xa2 => FaultUnit::dCDM_KS(0), - 0xa3 => FaultUnit::dCDM_KS(1), - 0xa4 => FaultUnit::dCDM_KS(2), - 0xa5 => FaultUnit::dIPP, - 0xa6 => FaultUnit::dIPP_CS, - 0xa7 => FaultUnit::dVDM_CSD, - 0xa8 => FaultUnit::dVDM_SSD, - 0xa9 => FaultUnit::dVDM_ILF, - 0xaa => FaultUnit::dVDM_ILD, - 0xab => FaultUnit::dRDE(0), - 0xac => FaultUnit::dRDE(1), - 0xad => FaultUnit::FC, - 0xae => FaultUnit::GSL2, - 0xb0..=0xb7 => FaultUnit::GL2CC_META(unit_code & 0xf), - 0xb8 => FaultUnit::GL2CC_MB, - 0xd0..=0xdf if g14x => match unit_code & 0xf { - 0x0 => FaultUnit::gCDM_CS, - 0x1 => FaultUnit::gCDM_ID, - 0x2 => FaultUnit::gCDM_CSR, - 0x3 => FaultUnit::gCDM_CSW, - 0x4 => FaultUnit::gCDM_CTXR, - 0x5 => FaultUnit::gCDM_CTXW, - 0x6 => FaultUnit::gIPP, - 0x7 => FaultUnit::gIPP_CS, - 0x8 => FaultUnit::gKSM_RCE, - _ => FaultUnit::Unknown(unit_code), - }, - 0xe0..=0xff if g14x => match unit_code & 0xf { - 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), - 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), - 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), - 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), - 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), - 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), - 0x6 => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), - _ => FaultUnit::Unknown(unit_code), - }, - 0xe0..=0xff if !g14x => match unit_code & 0xf { - 0x0 => FaultUnit::gPM_SP((unit_code >> 4) & 1), - 0x1 => FaultUnit::gVDM_CSD_SP((unit_code >> 4) & 1), - 0x2 => FaultUnit::gVDM_SSD_SP((unit_code >> 4) & 1), - 0x3 => FaultUnit::gVDM_ILF_SP((unit_code >> 4) & 1), - 0x4 => FaultUnit::gVDM_TFP_SP((unit_code >> 4) & 1), - 0x5 => FaultUnit::gVDM_MMB_SP((unit_code >> 4) & 1), - 0x6 => FaultUnit::gCDM_CS_KS0_SP((unit_code >> 4) & 1), - 0x7 => FaultUnit::gCDM_CS_KS1_SP((unit_code >> 4) & 1), - 0x8 => FaultUnit::gCDM_CS_KS2_SP((unit_code >> 4) & 1), - 0x9 => FaultUnit::gCDM_KS0_SP((unit_code >> 4) & 1), - 0xa => FaultUnit::gCDM_KS1_SP((unit_code >> 4) & 1), - 0xb => FaultUnit::gCDM_KS2_SP((unit_code >> 4) & 1), - 0xc => FaultUnit::gIPP_SP((unit_code >> 4) & 1), - 0xd => FaultUnit::gIPP_CS_SP((unit_code >> 4) & 1), - 0xe => FaultUnit::gRDE0_SP((unit_code >> 4) & 1), - 0xf => FaultUnit::gRDE1_SP((unit_code >> 4) & 1), - _ => FaultUnit::Unknown(unit_code), - }, - _ => FaultUnit::Unknown(unit_code), - }; - - let reason = match (fault_info >> 1) & 0x7 { - 0 => FaultReason::Unmapped, - 1 => FaultReason::AfFault, - 2 => FaultReason::WriteOnly, - 3 => FaultReason::ReadOnly, - 4 => FaultReason::NoAccess, - a => FaultReason::Unknown(a as u8), - }; - - Some(FaultInfo { - address: fault_addr << 6, - sideband: ((fault_info >> 23) & 0x7f) as u8, - vm_slot: ((fault_info >> 17) & 0x3f) as u32, - unit_code, - unit, - level: ((fault_info >> 7) & 3) as u8, - unk_5: ((fault_info >> 5) & 3) as u8, - read: (fault_info & (1 << 4)) != 0, - reason, - }) - } -} diff --git a/drivers/gpu/drm/asahi-neo/slotalloc.rs b/drivers/gpu/drm/asahi-neo/slotalloc.rs deleted file mode 100644 index fde7470fe57791..00000000000000 --- a/drivers/gpu/drm/asahi-neo/slotalloc.rs +++ /dev/null @@ -1,323 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Generic slot allocator -//! -//! This is a simple allocator to manage fixed-size pools of GPU resources that are transiently -//! required during command execution. Each item resides in a "slot" at a given index. Users borrow -//! and return free items from the available pool. -//! -//! Allocations are "sticky", and return a token that callers can use to request the same slot -//! again later. This allows slots to be lazily invalidated, so that multiple uses by the same user -//! avoid any actual cleanup work. -//! -//! The allocation policy is currently a simple LRU mechanism, doing a full linear scan over the -//! slots when no token was previously provided. This is probably good enough, since in the absence -//! of serious system contention most allocation requests will be immediately fulfilled from the -//! previous slot without doing an LRU scan. - -use core::num::NonZeroUsize; -use core::ops::{ - Deref, - DerefMut, // -}; -use kernel::{ - error::{ - code::*, - Result, // - }, - prelude::*, - str::CStr, - sync::{ - Arc, - CondVar, - LockClassKey, - Mutex, // - }, -}; - -/// Trait representing a single item within a slot. -pub(crate) trait SlotItem { - /// Arbitrary user data associated with the SlotAllocator. - type Data; - - /// Called eagerly when this item is released back into the available pool. - fn release(&mut self, _data: &mut Self::Data, _slot: u32) {} -} - -/// Trivial implementation for users which do not require any slot data nor any allocator data. -impl SlotItem for () { - type Data = (); -} - -/// Represents a current or previous allocation of an item from a slot. Users keep `SlotToken`s -/// around across allocations to request that, if possible, the same slot be reused. -#[derive(Copy, Clone, Debug)] -pub(crate) struct SlotToken { - time: u64, - slot: u32, -} - -impl SlotToken { - /// Returns the slot index that this token represents a past assignment to. - pub(crate) fn last_slot(&self) -> u32 { - self.slot - } -} - -/// A guard representing active ownership of a slot. -pub(crate) struct Guard { - item: Option, - changed: bool, - token: SlotToken, - alloc: Arc>, -} - -impl Guard { - /// Returns the active slot owned by this `Guard`. - pub(crate) fn slot(&self) -> u32 { - self.token.slot - } - - /// Returns `true` if the slot changed since the last allocation (or no `SlotToken` was - /// provided), or `false` if the previously allocated slot was successfully re-acquired with - /// no other users in the interim. - pub(crate) fn changed(&self) -> bool { - self.changed - } - - /// Returns a `SlotToken` that can be used to re-request the same slot at a later time, after - /// this `Guard` is dropped. - pub(crate) fn token(&self) -> SlotToken { - self.token - } -} - -impl Deref for Guard { - type Target = T; - - fn deref(&self) -> &Self::Target { - self.item.as_ref().expect("SlotItem Guard lost our item!") - } -} - -impl DerefMut for Guard { - fn deref_mut(&mut self) -> &mut Self::Target { - self.item.as_mut().expect("SlotItem Guard lost our item!") - } -} - -/// A slot item that is currently free. -struct Entry { - item: T, - get_time: u64, - drop_time: u64, -} - -/// Inner data for the `SlotAllocator`, protected by a `Mutex`. -struct SlotAllocatorInner { - data: T::Data, - slots: KVec>>, - get_count: u64, - drop_count: u64, - slot_limit: usize, -} - -/// A single slot allocator instance. -#[pin_data] -struct SlotAllocatorOuter { - #[pin] - inner: Mutex>, - #[pin] - cond: CondVar, -} - -/// A shared reference to a slot allocator instance. -pub(crate) struct SlotAllocator(Arc>); - -impl SlotAllocator { - /// Creates a new `SlotAllocator`, with a fixed number of slots and arbitrary associated data. - /// - /// The caller provides a constructor callback which takes a reference to the `T::Data` and - /// creates a single slot. This is called during construction to create all the initial - /// items, which then live the lifetime of the `SlotAllocator`. - pub(crate) fn new( - num_slots: u32, - mut data: T::Data, - mut constructor: impl FnMut(&mut T::Data, u32) -> Option, - name: &'static CStr, - lock_key1: Pin<&'static LockClassKey>, - lock_key2: Pin<&'static LockClassKey>, - ) -> Result> { - let mut slots = KVec::with_capacity(num_slots as usize, GFP_KERNEL)?; - - for i in 0..num_slots { - slots - .push( - constructor(&mut data, i).map(|item| Entry { - item, - get_time: 0, - drop_time: 0, - }), - GFP_KERNEL, - ) - .expect("try_push() failed after reservation"); - } - - let inner = SlotAllocatorInner { - data, - slots, - get_count: 0, - drop_count: 0, - slot_limit: usize::MAX, - }; - - let alloc = Arc::pin_init( - pin_init!(SlotAllocatorOuter { - // SAFETY: `mutex_init!` is called below. - inner <- Mutex::new(inner, name, lock_key1), - // SAFETY: `condvar_init!` is called below. - cond <- CondVar::new(name, lock_key2), - }), - GFP_KERNEL, - )?; - - Ok(SlotAllocator(alloc)) - } - - /// Calls a callback on the inner data associated with this allocator, taking the lock. - pub(crate) fn with_inner(&self, cb: impl FnOnce(&mut T::Data) -> RetVal) -> RetVal { - let mut inner = self.0.inner.lock(); - cb(&mut inner.data) - } - - /// Set the slot limit for this allocator. New bindings will not use slots above - /// this threshold. - pub(crate) fn set_limit(&self, limit: Option) { - let mut inner = self.0.inner.lock(); - inner.slot_limit = limit.unwrap_or(NonZeroUsize::MAX).get(); - } - - /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. - /// - /// Blocks if no slots are free. - pub(crate) fn get(&self, token: Option) -> Result> { - self.get_inner(token, |_a, _b| Ok(())) - } - - /// Gets a fresh slot, optionally reusing a previous allocation if a `SlotToken` is provided. - /// - /// Blocks if no slots are free. - /// - /// This version allows the caller to pass in a callback that gets a mutable reference to the - /// user data for the allocator and the freshly acquired slot, which is called before the - /// allocator lock is released. This can be used to perform bookkeeping associated with - /// specific slots (such as tracking their current owner). - pub(crate) fn get_inner( - &self, - token: Option, - cb: impl FnOnce(&mut T::Data, &mut Guard) -> Result<()>, - ) -> Result> { - let mut inner = self.0.inner.lock(); - - if let Some(token) = token { - if (token.slot as usize) < inner.slot_limit { - let slot = &mut inner.slots[token.slot as usize]; - if slot.is_some() { - let count = slot.as_ref().unwrap().get_time; - if count == token.time { - let mut guard = Guard { - item: Some(slot.take().unwrap().item), - token, - changed: false, - alloc: self.0.clone(), - }; - cb(&mut inner.data, &mut guard)?; - return Ok(guard); - } - } - } - } - - let mut first = true; - let slot = loop { - let mut oldest_time = u64::MAX; - let mut oldest_slot = 0u32; - - for (i, slot) in inner.slots.iter().enumerate() { - if i >= inner.slot_limit { - break; - } - if let Some(slot) = slot.as_ref() { - if slot.drop_time < oldest_time { - oldest_slot = i as u32; - oldest_time = slot.drop_time; - } - } - } - - if oldest_time == u64::MAX { - if first && inner.slot_limit == usize::MAX { - pr_warn!( - "{}: out of slots, blocking\n", - core::any::type_name::() - ); - } - first = false; - if self.0.cond.wait_interruptible(&mut inner) { - return Err(ERESTARTSYS); - } - } else { - break oldest_slot; - } - }; - - inner.get_count += 1; - - let item = inner.slots[slot as usize] - .take() - .expect("Someone stole our slot?") - .item; - - let mut guard = Guard { - item: Some(item), - changed: true, - token: SlotToken { - time: inner.get_count, - slot, - }, - alloc: self.0.clone(), - }; - - cb(&mut inner.data, &mut guard)?; - Ok(guard) - } -} - -impl Clone for SlotAllocator { - fn clone(&self) -> Self { - SlotAllocator(self.0.clone()) - } -} - -impl Drop for Guard { - fn drop(&mut self) { - let mut inner = self.alloc.inner.lock(); - if inner.slots[self.token.slot as usize].is_some() { - pr_crit!( - "{}: tried to return an item into a full slot ({})\n", - core::any::type_name::(), - self.token.slot - ); - } else { - inner.drop_count += 1; - let mut item = self.item.take().expect("Guard lost its item"); - item.release(&mut inner.data, self.token.slot); - inner.slots[self.token.slot as usize] = Some(Entry { - item, - get_time: self.token.time, - drop_time: inner.drop_count, - }); - self.alloc.cond.notify_one(); - } - } -} diff --git a/drivers/gpu/drm/asahi-neo/stats.rs b/drivers/gpu/drm/asahi-neo/stats.rs deleted file mode 100644 index 772d969d9baff9..00000000000000 --- a/drivers/gpu/drm/asahi-neo/stats.rs +++ /dev/null @@ -1,78 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Snapshot of AGX firmware statistics. -//! -//! [`StatsSnapshot`] accumulates the values decoded by `StatsChannel::poll` -//! (see `fw::channels.rs::StatsMsg`) and exposes them to the sysfs file -//! `agx_stats` (registered from `sysfs.c`). -//! -//! The snapshot belongs to one GPU manager. The firmware-stat -//! decoder that feeds it lives in `StatsChannel::poll` (`channel.rs`), inside -//! the versioned context where `StatsMsg` is nameable; `note_job()` is called -//! from the `recv_message` rtkit callback (so the firmware mailbox IRQ -//! context); the sysfs `show` callback runs in arbitrary process context. All -//! fields use `AtomicU*` so neither side takes a lock. -//! -//! Field semantics (firmware units, late-validated): -//! - `util1..util4`: raw `Utilization` u32s from the firmware. Meaning -//! (firmware tick or %?) is validated against a controlled MLX load; we -//! publish them unscaled per the producer contract. -//! - `pstate`: current performance state index from `PowerState`. -//! - `avg_power_mw`: average power in milliwatts from `AvgPower` (firmware -//! field is u32 — units not validated for every chip rev; treat as raw). -//! - `temperature`: raw `Temperature` value, scale, tmin, tmax. -//! - `busy_ns`: cumulative utilization-weighted busy time in nanoseconds, -//! derived from successive `Utilization` windows (busiest subqueue -//! percentage x window duration; the firmware timestamps are base-clock -//! ticks, 24 MHz, converted with `HwConfig::base_clock_hz`). Monotonic and -//! bounded by elapsed time. Saturates at u64::MAX. -//! - `jobs`: completed submissions since boot, counted at submission -//! completion (`JobFence::command_complete`). -//! -//! This module is read-only with respect to scheduling or power behaviour: -//! the firmware already sends the messages, the channel already polls them, -//! we only retain the latest values. - -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; - -/// Per-device snapshot of AGX firmware stats. -/// -/// The manager and bound driver retain Arcs. The driver's Arc outlives the -/// per-device C sysfs registration; repr(C) establishes the shared layout. -#[derive(Default)] -#[repr(C)] -pub(crate) struct StatsSnapshot { - pub(crate) util1: AtomicU32, - pub(crate) util2: AtomicU32, - pub(crate) util3: AtomicU32, - pub(crate) util4: AtomicU32, - pub(crate) pstate: AtomicU32, - pub(crate) avg_power_mw: AtomicU32, - pub(crate) temperature_raw: AtomicU32, - pub(crate) temperature_scale: AtomicU32, - pub(crate) temperature_tmin: AtomicU32, - pub(crate) temperature_tmax: AtomicU32, - /// Cumulative utilization-weighted busy time. - pub(crate) busy_ns: AtomicU64, - /// Completed submissions since boot, counted at fence signal in - /// `JobFence::command_complete` when the last command of a submission - /// completes. - pub(crate) jobs: AtomicU64, -} - -// Keep the Rust half of the C sysfs layout contract checked as well. -const _: () = { - assert!(core::mem::offset_of!(StatsSnapshot, pstate) == 16); - assert!(core::mem::offset_of!(StatsSnapshot, busy_ns) == 40); - assert!(core::mem::offset_of!(StatsSnapshot, jobs) == 48); - assert!(core::mem::size_of::() == 56); -}; - -impl StatsSnapshot { - /// Bump the completed-submission counter (called from the queue - /// completion path in `JobFence::command_complete`, NOT from the stats - /// channel). - pub(crate) fn note_job(&self) { - self.jobs.fetch_add(1, Ordering::Relaxed); - } -} diff --git a/drivers/gpu/drm/asahi-neo/sysfs.c b/drivers/gpu/drm/asahi-neo/sysfs.c deleted file mode 100644 index 505cd8e1ac4a4b..00000000000000 --- a/drivers/gpu/drm/asahi-neo/sysfs.c +++ /dev/null @@ -1,131 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT -// -// Per-device sysfs shim. Rust owns the snapshot through an Arc in the bound -// driver. The attribute wrapper is removed and its readers drained before -// that Arc is dropped. Atomic fields may reflect different sample instants. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "sysfs.h" - -/* Mirror of repr(C) Rust atomics; READ_ONCE accesses the current values. */ -struct asahi_stats_snapshot { - u32 util1; - u32 util2; - u32 util3; - u32 util4; - u32 pstate; - u32 avg_power_mw; - u32 temperature_raw; - u32 temperature_scale; - u32 temperature_tmin; - u32 temperature_tmax; - u64 busy_ns; - u64 jobs; -}; - -struct asahi_stats_attribute { - struct device_attribute attr; - const struct asahi_stats_snapshot *snapshot; - bool export_enabled; -}; - -static ssize_t agx_stats_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct asahi_stats_attribute *stats = - container_of(attr, struct asahi_stats_attribute, attr); - const struct asahi_stats_snapshot *snap = stats->snapshot; - ssize_t n = 0; - - if (!stats->export_enabled) - return sysfs_emit(buf, "unsupported\n"); - n += scnprintf(buf + n, PAGE_SIZE - n, "busy_ns %llu\n", - (unsigned long long)READ_ONCE(snap->busy_ns)); - n += scnprintf(buf + n, PAGE_SIZE - n, "jobs %llu\n", - (unsigned long long)READ_ONCE(snap->jobs)); - n += scnprintf(buf + n, PAGE_SIZE - n, "pstate %u\n", - (u32)READ_ONCE(snap->pstate)); - n += scnprintf(buf + n, PAGE_SIZE - n, "power_mw %u\n", - (u32)READ_ONCE(snap->avg_power_mw)); - n += scnprintf(buf + n, PAGE_SIZE - n, "util1 %u\n", - (u32)READ_ONCE(snap->util1)); - n += scnprintf(buf + n, PAGE_SIZE - n, "util2 %u\n", - (u32)READ_ONCE(snap->util2)); - n += scnprintf(buf + n, PAGE_SIZE - n, "util3 %u\n", - (u32)READ_ONCE(snap->util3)); - n += scnprintf(buf + n, PAGE_SIZE - n, "util4 %u\n", - (u32)READ_ONCE(snap->util4)); - n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_raw %u\n", - (u32)READ_ONCE(snap->temperature_raw)); - n += scnprintf(buf + n, PAGE_SIZE - n, "temperature_scale %u\n", - (u32)READ_ONCE(snap->temperature_scale)); - - return n; -} - - -/* - * Called from Rust's `AsahiDriver::probe` after the DRM device is - * registered. Returns 0 on success, or a negative errno. - */ -int asahi_neo_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle) -{ - struct asahi_stats_attribute *stats; - int ret; - - if (!dev || !handle || (export_enabled && !snapshot)) - return -ENODEV; - - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, pstate) != 16); - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, busy_ns) != 40); - BUILD_BUG_ON(offsetof(struct asahi_stats_snapshot, jobs) != 48); - BUILD_BUG_ON(sizeof(struct asahi_stats_snapshot) != 56); - - stats = kzalloc_obj(*stats); - if (!stats) - return -ENOMEM; - - sysfs_attr_init(&stats->attr.attr); - stats->attr.attr.name = "agx_stats"; - stats->attr.attr.mode = 0444; - stats->attr.show = agx_stats_show; - stats->snapshot = snapshot; - stats->export_enabled = export_enabled; - - ret = device_create_file(dev, &stats->attr); - if (ret) { - kfree(stats); - return ret; - } - - *handle = stats; - return 0; -} -EXPORT_SYMBOL_GPL(asahi_neo_sysfs_register); - -void asahi_neo_sysfs_unregister(struct device *dev, void *handle) -{ - struct asahi_stats_attribute *stats = handle; - - if (!dev || !stats) - return; - - /* Removal drains active show callbacks before either allocation dies. */ - device_remove_file(dev, &stats->attr); - kfree(stats); -} -EXPORT_SYMBOL_GPL(asahi_neo_sysfs_unregister); - -MODULE_AUTHOR("AGX driver maintainer"); -MODULE_LICENSE("Dual MIT/GPL"); -MODULE_DESCRIPTION("Asahi AGX firmware stats sysfs shim"); diff --git a/drivers/gpu/drm/asahi-neo/sysfs.h b/drivers/gpu/drm/asahi-neo/sysfs.h deleted file mode 100644 index 2693d05a84f23f..00000000000000 --- a/drivers/gpu/drm/asahi-neo/sysfs.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ - -#ifndef _ASAHI_SYSFS_H -#define _ASAHI_SYSFS_H - -struct device; - -int asahi_neo_sysfs_register(struct device *dev, const void *snapshot, - int export_enabled, void **handle); -void asahi_neo_sysfs_unregister(struct device *dev, void *handle); - -#endif /* _ASAHI_SYSFS_H */ diff --git a/drivers/gpu/drm/asahi-neo/sysfs_exports.rs b/drivers/gpu/drm/asahi-neo/sysfs_exports.rs deleted file mode 100644 index 895d4c75eb2003..00000000000000 --- a/drivers/gpu/drm/asahi-neo/sysfs_exports.rs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Per-device sysfs registration. The caller retains the snapshot Arc until -//! unregister has drained active readers and freed the C attribute wrapper. - -use core::ffi::{c_int, c_void}; -use crate::stats::StatsSnapshot; - -extern "C" { - fn asahi_neo_sysfs_register( - dev: *mut c_void, - snapshot: *const StatsSnapshot, - export_enabled: c_int, - handle: *mut *mut c_void, - ) -> c_int; - fn asahi_neo_sysfs_unregister(dev: *mut c_void, handle: *mut c_void); -} - -/// Register only after initialization succeeds, keeping the snapshot Arc alive. -pub(crate) fn register( - dev: *mut kernel::bindings::device, - snapshot: *const StatsSnapshot, - export_enabled: bool, -) -> kernel::error::Result<*mut c_void> { - let mut handle = core::ptr::null_mut(); - let ret = unsafe { asahi_neo_sysfs_register(dev.cast(), snapshot, export_enabled as c_int, &mut handle) }; - if ret < 0 { - Err(kernel::error::Error::from_errno(ret)) - } else { - Ok(handle) - } -} - -/// Drain readers before the caller drops its snapshot Arc. -pub(crate) fn unregister(dev: *mut kernel::bindings::device, handle: *mut c_void) { - unsafe { asahi_neo_sysfs_unregister(dev.cast(), handle) }; -} diff --git a/drivers/gpu/drm/asahi-neo/util.rs b/drivers/gpu/drm/asahi-neo/util.rs deleted file mode 100644 index 00246e1f64ab78..00000000000000 --- a/drivers/gpu/drm/asahi-neo/util.rs +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Miscellaneous utility functions - -use core::ops::{ - Add, - BitAnd, - Div, - Not, - Sub, // -}; -use kernel::prelude::*; - -/// Aligns an integer type to a power of two. -pub(crate) fn align(a: T, b: T) -> T -where - T: Copy - + Default - + BitAnd - + Not - + Add - + Sub - + Div - + core::cmp::PartialEq, -{ - let def: T = Default::default(); - #[allow(clippy::eq_op)] - let one: T = !def / !def; - - assert!((b & (b - one)) == def); - - (a + b - one) & !(b - one) -} - -/// Aligns an integer type down to a power of two. -pub(crate) fn align_down(a: T, b: T) -> T -where - T: Copy - + Default - + BitAnd - + Not - + Sub - + Div - + core::cmp::PartialEq, -{ - let def: T = Default::default(); - #[allow(clippy::eq_op)] - let one: T = !def / !def; - - assert!((b & (b - one)) == def); - - a & !(b - one) -} - -pub(crate) trait RangeExt { - fn overlaps(&self, other: Self) -> bool; - fn is_superset(&self, other: Self) -> bool; - // fn len(&self) -> usize; - fn range(&self) -> T; -} - -impl + Default + Copy + Sub> RangeExt for core::ops::Range -where - usize: core::convert::TryFrom, - >::Error: core::fmt::Debug, -{ - fn overlaps(&self, other: Self) -> bool { - !(self.is_empty() || other.is_empty() || self.end <= other.start || other.end <= self.start) - } - fn is_superset(&self, other: Self) -> bool { - !self.is_empty() - && (other.is_empty() || (other.start >= self.start && other.end <= self.end)) - } - fn range(&self) -> T { - if self.is_empty() { - Default::default() - } else { - self.end - self.start - } - } - // fn len(&self) -> usize { - // self.range().try_into().unwrap() - // } -} - -pub(crate) fn gcd(in_n: u64, in_m: u64) -> u64 { - let mut n = in_n; - let mut m = in_m; - - while n != 0 { - let remainder = m % n; - m = n; - n = remainder; - } - - m -} - -pub(crate) unsafe trait AnyBitPattern: Default + Sized + Copy + 'static {} - -pub(crate) struct Reader<'a> { - buffer: &'a [u8], - offset: usize, -} - -impl<'a> Reader<'a> { - pub(crate) fn new(buffer: &'a [u8]) -> Self { - Reader { buffer, offset: 0 } - } - - pub(crate) fn read_up_to(&mut self, declared_size: usize) -> Result { - let mut obj: T = Default::default(); - let size: usize = core::mem::size_of::().min(declared_size); - let end = self.offset.checked_add(declared_size).ok_or(EINVAL)?; - let range = self.offset..end; - let src = self.buffer.get(range).ok_or(EINVAL)?; - - // SAFETY: The output pointer is valid, and the size does not exceed - // the type size, and all bit patterns are valid. - let dst = unsafe { core::slice::from_raw_parts_mut(&mut obj as *mut _ as *mut u8, size) }; - - dst.copy_from_slice(&src[..size]); - self.offset = end; - Ok(obj) - } - - pub(crate) fn read(&mut self) -> Result { - self.read_up_to(core::mem::size_of::()) - } - - pub(crate) fn is_empty(&self) -> bool { - self.offset >= self.buffer.len() - } - - pub(crate) fn skip(&mut self, size: usize) { - self.offset += size - } - - pub(crate) fn rewind(&mut self) { - self.offset = 0 - } -} diff --git a/drivers/gpu/drm/asahi-neo/vm/mod.rs b/drivers/gpu/drm/asahi-neo/vm/mod.rs deleted file mode 100644 index 63cf8a76cd5ce4..00000000000000 --- a/drivers/gpu/drm/asahi-neo/vm/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! Virtual address space management - -mod range; diff --git a/drivers/gpu/drm/asahi-neo/vm/range.rs b/drivers/gpu/drm/asahi-neo/vm/range.rs deleted file mode 100644 index 727d29a5cccf02..00000000000000 --- a/drivers/gpu/drm/asahi-neo/vm/range.rs +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 or MIT - -// Copied from tyr's mmu/vm/range.rs - -//! Range allocator. -//! -//! This module allows you to search for unused ranges to store GEM objects. - -use kernel::alloc::Flags; -use kernel::maple_tree::MapleTreeAlloc; -use kernel::prelude::*; -use kernel::sync::Arc; - -use core::ops::Range; - -/// The actual storage for the ranges. -/// -/// All ranges must fit within the `range` field. -/// -/// The implementation is different on 32-bit and 64-bit cpus. On 64-bit, the 64-bit addresses are -/// stored directly in the maple tree, but on 32-bit, the maple tree stores the ranges translated -/// in the range zero until `range.end-range.start`. This is done because the maple tree uses -/// unsigned long as its address type, which is too small to store the 64-bit address directly on -/// 32-bit machines. -#[pin_data] -struct RangeAllocInner { - #[pin] - maple: MapleTreeAlloc<()>, - range: Range, -} - -/// This object allows you to allocate ranges on the inner maple tree. -pub(crate) struct RangeAlloc { - inner: Arc, -} - -/// Represents a live range in the maple tree. -/// -/// The destructor removes the range from the maple tree, allowing others to allocate it in the -/// future. -pub(crate) struct LiveRange { - inner: Arc, - offset: u64, - size: usize, -} - -impl RangeAlloc { - pub(crate) fn new(start: u64, end: u64, gfp: Flags) -> Result { - if end < start { - return Err(EINVAL); - } - - let inner = Arc::pin_init( - try_pin_init!(RangeAllocInner { - maple <- MapleTreeAlloc::new(), - range: start..end, - }), - gfp, - )?; - - Ok(RangeAlloc { inner }) - } - - pub(crate) fn allocate(&self, size: usize, gfp: Flags) -> Result { - let maple_start = self.inner.range.start as usize; - let maple_end = self.inner.range.end as usize; - - let offset = self - .inner - .maple - .alloc_range(size, (), maple_start..maple_end, gfp)?; - - Ok(LiveRange { - inner: self.inner.clone(), - offset: offset as u64, - size, - }) - } - - pub(crate) fn insert(&self, start: u64, end: u64, gfp: Flags) -> Result { - if end < start { - return Err(EINVAL); - } - if start < self.inner.range.start { - return Err(EINVAL); - } - if end > self.inner.range.end { - return Err(EINVAL); - } - - self.inner - .maple - .insert_range(start as usize..end as usize, (), gfp)?; - - Ok(LiveRange { - inner: self.inner.clone(), - offset: start, - size: (end - start) as usize, - }) - } -} - -impl LiveRange { - pub(crate) fn size(&self) -> usize { - self.size - } - - pub(crate) fn start(&self) -> u64 { - self.offset - } - - pub(crate) fn end(&self) -> u64 { - self.offset + self.size as u64 - } - - pub(crate) fn range(&self) -> Range { - self.start()..self.end() - } -} - -impl Drop for LiveRange { - fn drop(&mut self) { - self.inner.maple.erase(self.offset as usize); - } -} diff --git a/drivers/gpu/drm/asahi-neo/workqueue.rs b/drivers/gpu/drm/asahi-neo/workqueue.rs deleted file mode 100644 index 0eabec8d410c77..00000000000000 --- a/drivers/gpu/drm/asahi-neo/workqueue.rs +++ /dev/null @@ -1,1019 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only OR MIT - -//! GPU command execution queues -//! -//! The AGX GPU firmware schedules GPU work commands out of work queues, which are ring buffers of -//! pointers to work commands. There can be an arbitrary number of work queues. Work queues have an -//! associated type (vertex, fragment, or compute) and may only contain generic commands or commands -//! specific to that type. -//! -//! This module manages queueing work commands into a work queue and submitting them for execution -//! by the firmware. An active work queue needs an event to signal completion of its work, which is -//! owned by what we call a batch. This event then notifies the work queue when work is completed, -//! and that triggers freeing of all resources associated with that work. An idle work queue gives -//! up its associated event. - -use crate::debug::*; -use crate::fw::channels::{ - ChannelErrorType, - PipeType, // -}; -use crate::fw::types::*; -use crate::fw::workqueue::*; -use crate::no_debug; -use crate::object::OpaqueGpuObject; -use crate::{ - channel, - driver, - event, - fw, - gpu, - regs, // -}; -use core::any::Any; -use core::num::NonZeroU64; -use core::sync::atomic::Ordering; -use kernel::{ - dma_fence, - error::code::*, - new_mutex, - prelude::*, - sync::{ - lock::{ - mutex::MutexBackend, - Guard, // - }, - Arc, - Mutex, // - }, - workqueue::{ - self, - impl_has_work, - new_work, - Work, - WorkItem, // - }, // -}; - -pub(crate) trait OpaqueCommandObject: OpaqueGpuObject {} - -impl OpaqueCommandObject for GpuObject where T: Command {} - -const DEBUG_CLASS: DebugFlags = DebugFlags::WorkQueue; - -const MAX_JOB_SLOTS: u32 = 127; - -/// An enum of possible errors that might cause a piece of work to fail execution. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub(crate) enum WorkError { - /// GPU timeout (command execution took too long). - Timeout, - /// GPU MMU fault (invalid access). - Fault(regs::FaultInfo), - /// Work failed due to an error caused by other concurrent GPU work. - Killed, - /// Channel error - ChannelError(ChannelErrorType), - /// The GPU crashed. - NoDevice, - /// Unknown reason. - Unknown, -} - -impl From for kernel::error::Error { - fn from(err: WorkError) -> Self { - match err { - WorkError::Timeout => ETIMEDOUT, - // Not EFAULT because that's for userspace faults - WorkError::Fault(_) => EIO, - WorkError::Unknown => ENODATA, - WorkError::Killed => ECANCELED, - WorkError::NoDevice => ENODEV, - WorkError::ChannelError(_) => EIO, - } - } -} - -/// A GPU context tracking structure, which must be explicitly invalidated when dropped. -pub(crate) struct GpuContext { - dev: driver::AsahiDevRef, - data: Option>>, -} -no_debug!(GpuContext); - -impl GpuContext { - /// Allocate a new GPU context. - pub(crate) fn new( - dev: &driver::AsahiDevice, - alloc: &mut gpu::KernelAllocators, - buffer: Arc, - ) -> Result { - Ok(GpuContext { - dev: dev.into(), - data: Some(KBox::new( - alloc.shared.new_object( - fw::workqueue::GpuContextData { _buffer: buffer }, - |_inner| Default::default(), - )?, - GFP_KERNEL, - )?), - }) - } - - /// Returns the GPU pointer to the inner GPU context data structure. - pub(crate) fn gpu_pointer(&self) -> GpuPointer<'_, fw::workqueue::GpuContextData> { - self.data.as_ref().unwrap().gpu_pointer() - } -} - -impl Drop for GpuContext { - fn drop(&mut self) { - mod_dev_dbg!(self.dev, "GpuContext: Freeing GPU context\n"); - let data = self.data.take().unwrap(); - // A context only exists for the manager whose allocators created it, so the manager is - // always present. - if let Some(gpu) = (*self.dev).gpu.manager() { - gpu.free_context(data); - } - } -} - -struct SubmittedWork -where - O: OpaqueCommandObject, - C: FnOnce(Option) + Send + Sync + 'static, -{ - object: O, - value: EventValue, - error: Option, - wptr: u32, - vm_slot: u32, - callback: Option, - fence: dma_fence::Fence, -} - -pub(crate) trait GenSubmittedWork: Send + Sync { - fn gpu_va(&self) -> NonZeroU64; - fn value(&self) -> event::EventValue; - fn wptr(&self) -> u32; - fn set_wptr(&mut self, wptr: u32); - fn mark_error(&mut self, error: WorkError); - fn complete(&mut self); - fn get_fence(&self) -> dma_fence::Fence; -} - -#[pin_data] -struct SubmittedWorkContainer { - #[pin] - work: Work, - inner: KBox, -} - -impl_has_work! { - impl HasWork for SubmittedWorkContainer { self.work } -} - -impl WorkItem for SubmittedWorkContainer { - type Pointer = Pin>; - - fn run(this: Pin>) { - mod_pr_debug!("WorkQueue: Freeing command @ {:?}\n", this.inner.gpu_va()); - } -} - -impl SubmittedWorkContainer { - fn inner_mut(self: Pin<&mut Self>) -> &mut KBox { - // SAFETY: inner does not require structural pinning. - unsafe { &mut self.get_unchecked_mut().inner } - } -} - -impl) + Send + Sync> GenSubmittedWork - for SubmittedWork -{ - fn gpu_va(&self) -> NonZeroU64 { - self.object.gpu_va() - } - - fn value(&self) -> event::EventValue { - self.value - } - - fn wptr(&self) -> u32 { - self.wptr - } - - fn set_wptr(&mut self, wptr: u32) { - self.wptr = wptr; - } - - fn complete(&mut self) { - if let Some(cb) = self.callback.take() { - cb(self.error); - } - } - - fn mark_error(&mut self, error: WorkError) { - mod_pr_debug!("WorkQueue: Command at value {:#x?} failed\n", self.value); - self.error = Some(match error { - WorkError::Fault(info) if info.vm_slot != self.vm_slot => WorkError::Killed, - err => err, - }); - } - - fn get_fence(&self) -> dma_fence::Fence { - self.fence.clone() - } -} - -/// Inner data for managing a single work queue. -#[versions(AGX_NEO)] -struct WorkQueueInner { - dev: driver::AsahiDevRef, - event_manager: Arc, - info: GpuObject, - new: bool, - pipe_type: PipeType, - size: u32, - wptr: u32, - pending: KVec>>, - last_token: Option, - pending_jobs: usize, - last_submitted: Option, - last_completed: Option, - event: Option<(event::Event, event::EventValue)>, - priority: u32, - commit_seq: u64, - submit_seq: u64, - event_seq: u64, -} - -/// An instance of a work queue. -#[versions(AGX_NEO)] -#[pin_data] -pub(crate) struct WorkQueue { - info_pointer: GpuWeakPointer, - #[pin] - inner: Mutex, -} - -#[versions(AGX_NEO)] -impl WorkQueueInner::ver { - /// Return the GPU done pointer, representing how many work items have been completed by the - /// GPU. - fn doneptr(&self) -> u32 { - self.info - .state - .with(|raw, _inner| raw.gpu_doneptr.load(Ordering::Acquire)) - } -} - -#[versions(AGX_NEO)] -#[derive(Copy, Clone)] -pub(crate) struct QueueEventInfo { - pub(crate) stamp_pointer: GpuWeakPointer, - pub(crate) fw_stamp_pointer: GpuWeakPointer, - pub(crate) slot: u32, - pub(crate) value: event::EventValue, - pub(crate) cmd_seq: u64, - pub(crate) event_seq: u64, - pub(crate) info_ptr: GpuWeakPointer, -} - -#[versions(AGX_NEO)] -pub(crate) struct Job { - wq: Arc, - event_info: QueueEventInfo::ver, - start_value: EventValue, - pending: KVec>>, - committed: bool, - submitted: bool, - event_count: usize, - fence: dma_fence::Fence, -} - -#[versions(AGX_NEO)] -pub(crate) struct JobSubmission<'a> { - inner: Option>, - wptr: u32, - event_count: usize, - command_count: usize, -} - -#[versions(AGX_NEO)] -impl Job::ver { - pub(crate) fn event_info(&self) -> QueueEventInfo::ver { - let mut info = self.event_info; - info.cmd_seq += self.pending.len() as u64; - info.event_seq += self.event_count as u64; - - info - } - - pub(crate) fn next_seq(&mut self) { - self.event_count += 1; - self.event_info.value.increment(); - } - - pub(crate) fn add( - &mut self, - command: O, - vm_slot: u32, - ) -> Result { - self.add_cb(command, vm_slot, |_| {}) - } - - pub(crate) fn add_cb( - &mut self, - command: O, - vm_slot: u32, - callback: impl FnOnce(Option) + Sync + Send + 'static, - ) -> Result { - if self.committed { - pr_err!("WorkQueue: Tried to mutate committed Job\n"); - return Err(EINVAL); - } - - let fence = self.fence.clone(); - let value = self.event_info.value.next(); - - self.pending.push( - KBox::try_pin_init( - try_pin_init!(SubmittedWorkContainer { - work <- new_work!("SubmittedWorkWrapper::work"), - inner: KBox::new(SubmittedWork::<_, _> { - object: command, - value, - error: None, - callback: Some(callback), - wptr: 0, - vm_slot, - fence, - }, GFP_KERNEL)? - }), - GFP_KERNEL, - )?, - GFP_KERNEL, - )?; - - Ok(()) - } - - pub(crate) fn commit(&mut self) -> Result { - if self.committed { - pr_err!("WorkQueue: Tried to commit committed Job\n"); - return Err(EINVAL); - } - - if self.pending.is_empty() { - pr_err!("WorkQueue: Job::commit() with no commands\n"); - return Err(EINVAL); - } - - let mut inner = self.wq.inner.lock(); - - let ev = inner.event.as_mut().expect("WorkQueue: Job lost its event"); - - if ev.1 != self.start_value { - pr_err!( - "WorkQueue: Job::commit() out of order (event slot {} {:?} != {:?}\n", - ev.0.slot(), - ev.1, - self.start_value - ); - return Err(EINVAL); - } - - ev.1 = self.event_info.value; - inner.commit_seq += self.pending.len() as u64; - inner.event_seq += self.event_count as u64; - self.committed = true; - - Ok(()) - } - - pub(crate) fn can_submit(&self) -> Option { - let inner = self.wq.inner.lock(); - if inner.free_slots() > self.event_count && inner.free_space() > self.pending.len() { - None - } else if let Some(work) = inner.pending.first() { - Some(work.inner.get_fence()) - } else { - pr_err!( - "WorkQueue: Cannot submit, but queue is empty? {} > {}, {} > {} (pend={} ls={:#x?} lc={:#x?}) ev={:#x?} cur={:#x?} slot {:?}\n", - inner.free_slots(), - self.event_count, - inner.free_space(), - self.pending.len(), - inner.pending.len(), - inner.last_submitted, - inner.last_completed, - inner.event.as_ref().map(|a| a.1), - inner.event.as_ref().map(|a| a.0.current()), - inner.event.as_ref().map(|a| a.0.slot()), - ); - None - } - } - - pub(crate) fn submit(&mut self) -> Result> { - if !self.committed { - pr_err!("WorkQueue: Tried to submit uncommitted Job\n"); - return Err(EINVAL); - } - - if self.submitted { - pr_err!("WorkQueue: Tried to submit Job twice\n"); - return Err(EINVAL); - } - - if self.pending.is_empty() { - pr_err!("WorkQueue: Job::submit() with no commands\n"); - return Err(EINVAL); - } - - let mut inner = self.wq.inner.lock(); - - if inner.submit_seq != self.event_info.cmd_seq { - pr_err!( - "WorkQueue: Job::submit() out of order (submit_seq {} != {})\n", - inner.submit_seq, - self.event_info.cmd_seq - ); - return Err(EINVAL); - } - - if inner.commit_seq < (self.event_info.cmd_seq + self.pending.len() as u64) { - pr_err!( - "WorkQueue: Job::submit() out of order (commit_seq {} != {})\n", - inner.commit_seq, - (self.event_info.cmd_seq + self.pending.len() as u64) - ); - return Err(EINVAL); - } - - let mut wptr = inner.wptr; - let command_count = self.pending.len(); - - if inner.free_space() <= command_count { - pr_err!("WorkQueue: Job does not fit in ring buffer\n"); - return Err(EBUSY); - } - - inner.pending.reserve(command_count, GFP_KERNEL)?; - - inner.last_submitted = Some(self.event_info.value); - mod_dev_dbg!( - inner.dev, - "WorkQueue: submitting {} cmds at {:#x?}, lc {:#x?}, cur {:#x?}, pending {}, events {}\n", - self.pending.len(), - inner.last_submitted, - inner.last_completed, - inner.event.as_ref().map(|a| a.0.current()), - inner.pending.len(), - self.event_count, - ); - - for mut command in self.pending.drain(..) { - command.as_mut().inner_mut().set_wptr(wptr); - - let next_wptr = (wptr + 1) % inner.size; - assert!(inner.doneptr() != next_wptr); - inner.info.ring[wptr as usize] = command.inner.gpu_va().get(); - wptr = next_wptr; - - // Cannot fail, since we did a reserve(1) above - inner - .pending - .push(command, GFP_KERNEL) - .expect("push() failed after reserve()"); - } - - self.submitted = true; - - Ok(JobSubmission::ver { - inner: Some(inner), - wptr, - command_count, - event_count: self.event_count, - }) - } -} - -#[versions(AGX_NEO)] -impl<'a> JobSubmission::ver<'a> { - pub(crate) fn run(mut self, channel: &mut channel::PipeChannel::ver) { - let command_count = self.command_count; - let mut inner = self.inner.take().expect("No inner?"); - let wptr = self.wptr; - core::mem::forget(self); - - inner - .info - .state - .with(|raw, _inner| raw.cpu_wptr.store(wptr, Ordering::Release)); - - inner.wptr = wptr; - - let event = inner.event.as_mut().expect("JobSubmission lost its event"); - - let event_slot = event.0.slot(); - - let msg = fw::channels::RunWorkQueueMsg::ver { - pipe_type: inner.pipe_type, - work_queue: Some(inner.info.weak_pointer()), - wptr: inner.wptr, - event_slot, - is_new: inner.new, - __pad: Default::default(), - }; - channel.send(&msg); - inner.new = false; - - inner.submit_seq += command_count as u64; - } - - pub(crate) fn pipe_type(&self) -> PipeType { - self.inner.as_ref().expect("No inner?").pipe_type - } - - pub(crate) fn priority(&self) -> u32 { - self.inner.as_ref().expect("No inner?").priority - } -} - -#[versions(AGX_NEO)] -impl Drop for Job::ver { - fn drop(&mut self) { - mod_pr_debug!("WorkQueue: Dropping Job\n"); - let mut inner = self.wq.inner.lock(); - - if !self.committed { - pr_info!( - "WorkQueue: Dropping uncommitted job with {} events\n", - self.event_count - ); - } - - if self.committed && !self.submitted { - let pipe_type = inner.pipe_type; - let event = inner.event.as_mut().expect("Job lost its event"); - pr_info!( - "WorkQueue({:?}): Roll back {} events (slot {} val {:#x?}) and {} commands\n", - pipe_type, - self.event_count, - event.0.slot(), - event.1, - self.pending.len() - ); - event.1.sub(self.event_count as u32); - inner.commit_seq -= self.pending.len() as u64; - inner.event_seq -= self.event_count as u64; - } - - inner.pending_jobs -= 1; - - if inner.pending.is_empty() && inner.pending_jobs == 0 { - mod_pr_debug!("WorkQueue({:?}): Dropping event\n", inner.pipe_type); - inner.event = None; - inner.last_submitted = None; - inner.last_completed = None; - } - mod_pr_debug!("WorkQueue({:?}): Dropped Job\n", inner.pipe_type); - } -} - -#[versions(AGX_NEO)] -impl<'a> Drop for JobSubmission::ver<'a> { - fn drop(&mut self) { - let inner = self.inner.as_mut().expect("No inner?"); - mod_pr_debug!("WorkQueue({:?}): Dropping JobSubmission\n", inner.pipe_type); - - let new_len = inner.pending.len() - self.command_count; - inner.pending.truncate(new_len); - - let pipe_type = inner.pipe_type; - let event = inner.event.as_mut().expect("JobSubmission lost its event"); - pr_info!( - "WorkQueue({:?}): JobSubmission: Roll back {} events (slot {} val {:#x?}) and {} commands\n", - pipe_type, - self.event_count, - event.0.slot(), - event.1, - self.command_count - ); - event.1.sub(self.event_count as u32); - let val = event.1; - inner.commit_seq -= self.command_count as u64; - inner.event_seq -= self.event_count as u64; - inner.last_submitted = Some(val); - mod_pr_debug!("WorkQueue({:?}): Dropped JobSubmission\n", inner.pipe_type); - } -} - -#[versions(AGX_NEO)] -impl WorkQueueInner::ver { - /// Return the number of free entries in the workqueue - pub(crate) fn free_space(&self) -> usize { - self.size as usize - self.pending.len() - 1 - } - - pub(crate) fn free_slots(&self) -> usize { - let busy_slots = if let Some(ls) = self.last_submitted { - let lc = self - .last_completed - .expect("last_submitted but not completed?"); - ls.delta(&lc) - } else { - 0 - }; - - ((MAX_JOB_SLOTS as i32) - busy_slots).max(0) as usize - } -} - -#[versions(AGX_NEO)] -impl WorkQueue::ver { - /// Create a new WorkQueue of a given type and priority. - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - dev: &driver::AsahiDevice, - alloc: &mut gpu::KernelAllocators, - event_manager: Arc, - gpu_context: Arc, - notifier_list: Arc>, - pipe_type: PipeType, - id: u64, - priority: u32, - size: u32, - ) -> Result> { - let gpu_buf = alloc.private.array_empty_tagged(0x2c18, b"GPBF")?; - let mut state = alloc.shared.new_default::()?; - let ring = alloc.shared.array_empty(size as usize)?; - let mut prio = *raw::PRIORITY.get(priority as usize).ok_or(EINVAL)?; - - if pipe_type == PipeType::Compute && !debug_enabled(DebugFlags::Debug0) { - // Hack to disable compute preemption until we fix it - prio.0 = 0; - prio.5 = 1; - } - - let inner = WorkQueueInner::ver { - dev: dev.into(), - event_manager, - // Use shared (coherent) state with verbose faults so we can dump state correctly - info: if debug_enabled(DebugFlags::VerboseFaults) { - &mut alloc.shared - } else { - &mut alloc.private - } - .new_init( - try_init!(QueueInfo::ver { - state: { - state.with_mut(|raw, _inner| { - raw.rb_size = size; - }); - state - }, - ring, - gpu_buf, - notifier_list: notifier_list, - gpu_context: gpu_context, - }), - |inner, _p| { - try_init!(raw::QueueInfo::ver { - state: inner.state.gpu_pointer(), - ring: inner.ring.gpu_pointer(), - notifier_list: inner.notifier_list.gpu_pointer(), - gpu_buf: inner.gpu_buf.gpu_pointer(), - gpu_rptr1: Default::default(), - gpu_rptr2: Default::default(), - gpu_rptr3: Default::default(), - event_id: AtomicI32::new(-1), - priority: prio, - unk_4c: -1, - uuid: id as u32, - unk_54: -1, - unk_58: Default::default(), - busy: Default::default(), - __pad: Default::default(), - #[ver(V >= V13_2 && G < G14X)] - unk_84_0: 0, - unk_84_state: Default::default(), - error_count: Default::default(), - unk_8c: 0, - unk_90: 0, - unk_94: 0, - pending: Default::default(), - unk_9c: 0, - gpu_context: inner.gpu_context.gpu_pointer(), - unk_a8: Default::default(), - #[ver(V >= V13_2 && G < G14X)] - unk_b0: 0, - }) - }, - )?, - new: true, - pipe_type, - size, - wptr: 0, - pending: KVec::new(), - last_token: None, - event: None, - priority, - pending_jobs: 0, - commit_seq: 0, - submit_seq: 0, - event_seq: 0, - last_completed: None, - last_submitted: None, - }; - - let info_pointer = inner.info.weak_pointer(); - - Arc::pin_init( - pin_init!(Self { - info_pointer, - inner <- match pipe_type { - PipeType::Vertex => new_mutex!(inner, "WorkQueue::inner (Vertex)"), - PipeType::Fragment => new_mutex!(inner, "WorkQueue::inner (Fragment)"), - PipeType::Compute => new_mutex!(inner, "WorkQueue::inner (Compute)"), - }, - }), - GFP_KERNEL, - ) - } - - pub(crate) fn event_info(&self) -> Option { - let inner = self.inner.lock(); - - inner.event.as_ref().map(|ev| QueueEventInfo::ver { - stamp_pointer: ev.0.stamp_pointer(), - fw_stamp_pointer: ev.0.fw_stamp_pointer(), - slot: ev.0.slot(), - value: ev.1, - cmd_seq: inner.commit_seq, - event_seq: inner.event_seq, - info_ptr: self.info_pointer, - }) - } - - pub(crate) fn new_job(self: &Arc, fence: dma_fence::Fence) -> Result { - let mut inner = self.inner.lock(); - - if inner.event.is_none() { - mod_pr_debug!("WorkQueue({:?}): Grabbing event\n", inner.pipe_type); - let event = inner.event_manager.get(inner.last_token, self.clone())?; - let cur = event.current(); - inner.last_token = Some(event.token()); - mod_pr_debug!( - "WorkQueue({:?}): Grabbed event slot {}: {:#x?}\n", - inner.pipe_type, - event.slot(), - cur - ); - inner.event = Some((event, cur)); - inner.last_submitted = Some(cur); - inner.last_completed = Some(cur); - } - - inner.pending_jobs += 1; - - let ev = &inner.event.as_ref().unwrap(); - - mod_pr_debug!( - "WorkQueue({:?}): New job at value {:#x?} slot {}\n", - inner.pipe_type, - ev.1, - ev.0.slot() - ); - Ok(Job::ver { - wq: self.clone(), - event_info: QueueEventInfo::ver { - stamp_pointer: ev.0.stamp_pointer(), - fw_stamp_pointer: ev.0.fw_stamp_pointer(), - slot: ev.0.slot(), - value: ev.1, - cmd_seq: inner.commit_seq, - event_seq: inner.event_seq, - info_ptr: self.info_pointer, - }, - start_value: ev.1, - pending: KVec::new(), - event_count: 0, - committed: false, - submitted: false, - fence, - }) - } - - pub(crate) fn pipe_type(&self) -> PipeType { - self.inner.lock().pipe_type - } - - pub(crate) fn dump_info(&self) { - pr_info!("WorkQueue @ {:?}:", self.info_pointer); - self.inner.lock().info.with(|raw, _inner| { - pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr1.load(Ordering::Relaxed)); - pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr2.load(Ordering::Relaxed)); - pr_info!(" GPU rptr1: {:#x}", raw.gpu_rptr3.load(Ordering::Relaxed)); - pr_info!(" Event ID: {:#x}", raw.event_id.load(Ordering::Relaxed)); - pr_info!(" Busy: {:#x}", raw.busy.load(Ordering::Relaxed)); - pr_info!(" Unk 84: {:#x}", raw.unk_84_state.load(Ordering::Relaxed)); - pr_info!( - " Error count: {:#x}", - raw.error_count.load(Ordering::Relaxed) - ); - pr_info!(" Pending: {:#x}", raw.pending.load(Ordering::Relaxed)); - }); - } - - pub(crate) fn info_pointer(&self) -> GpuWeakPointer { - self.info_pointer - } -} - -/// Trait used to erase the version-specific type of WorkQueues, to avoid leaking -/// version-specificity into the event module. -pub(crate) trait WorkQueue { - /// Cast as an Any type. - fn as_any(&self) -> &dyn Any; - - fn signal(&self) -> bool; - fn mark_error(&self, value: event::EventValue, error: WorkError); - fn fail_all(&self, error: WorkError); -} - -#[versions(AGX_NEO)] -impl WorkQueue for WorkQueue::ver { - fn as_any(&self) -> &dyn Any { - self - } - - /// Signal a workqueue that some work was completed. - /// - /// This will check the event stamp value to find out exactly how many commands were processed. - fn signal(&self) -> bool { - let mut inner = self.inner.lock(); - let event = inner.event.as_ref(); - let value = match event { - None => { - mod_pr_debug!("WorkQueue: signal() called but no event?\n"); - - if inner.pending_jobs > 0 || !inner.pending.is_empty() { - pr_crit!("WorkQueue: signal() called with no event and pending jobs.\n"); - } - return true; - } - Some(event) => event.0.current(), - }; - - if let Some(lc) = inner.last_completed { - if value < lc { - pr_err!( - "WorkQueue: event rolled back? cur {:#x?}, lc {:#x?}, ls {:#x?}", - value, - inner.last_completed, - inner.last_submitted - ); - } - } else { - pr_crit!("WorkQueue: signal() called with no last_completed.\n"); - } - inner.last_completed = Some(value); - - mod_pr_debug!( - "WorkQueue({:?}): Signaling event {:?} value {:#x?}\n", - inner.pipe_type, - inner.last_token, - value - ); - - let mut completed_commands: usize = 0; - - for cmd in inner.pending.iter() { - if cmd.inner.value() <= value { - mod_pr_debug!( - "WorkQueue({:?}): Command at value {:#x?} complete\n", - inner.pipe_type, - cmd.inner.value() - ); - completed_commands += 1; - } else { - break; - } - } - - if completed_commands == 0 { - return inner.pending.is_empty(); - } - - let last_wptr = inner.pending[completed_commands - 1].inner.wptr(); - let pipe_type = inner.pipe_type; - - for mut cmd in inner.pending.drain(..completed_commands) { - mod_pr_debug!( - "WorkQueue({:?}): Queueing command @ {:?} for cleanup\n", - pipe_type, - cmd.inner.gpu_va() - ); - cmd.as_mut().inner_mut().complete(); - workqueue::system().enqueue(cmd); - } - - mod_pr_debug!( - "WorkQueue({:?}): Completed {} commands, left pending {}, ls {:#x?}, lc {:#x?}\n", - inner.pipe_type, - completed_commands, - inner.pending.len(), - inner.last_submitted, - inner.last_completed, - ); - - inner - .info - .state - .with(|raw, _inner| raw.cpu_freeptr.store(last_wptr, Ordering::Release)); - - let empty = inner.pending.is_empty(); - if empty && inner.pending_jobs == 0 { - inner.event = None; - inner.last_submitted = None; - inner.last_completed = None; - } - - empty - } - - /// Mark this queue's work up to a certain stamp value as having failed. - fn mark_error(&self, value: event::EventValue, error: WorkError) { - // If anything is marked completed, we can consider it successful - // at this point, even if we didn't get the signal event yet. - self.signal(); - - let mut inner = self.inner.lock(); - - if inner.event.is_none() { - mod_pr_debug!("WorkQueue: signal_fault() called but no event?\n"); - - if inner.pending_jobs > 0 || !inner.pending.is_empty() { - pr_crit!("WorkQueue: signal_fault() called with no event and pending jobs.\n"); - } - return; - } - - mod_pr_debug!( - "WorkQueue({:?}): Signaling fault for event {:?} at value {:#x?}\n", - inner.pipe_type, - inner.last_token, - value - ); - - for cmd in inner.pending.iter_mut() { - if cmd.inner.value() <= value { - cmd.as_mut().inner_mut().mark_error(error); - } else { - break; - } - } - } - - /// Mark all of this queue's work as having failed, and complete it. - fn fail_all(&self, error: WorkError) { - // If anything is marked completed, we can consider it successful - // at this point, even if we didn't get the signal event yet. - self.signal(); - - let mut inner = self.inner.lock(); - - if inner.event.is_none() { - mod_pr_debug!("WorkQueue: fail_all() called but no event?\n"); - - if inner.pending_jobs > 0 || !inner.pending.is_empty() { - pr_crit!("WorkQueue: fail_all() called with no event and pending jobs.\n"); - } - return; - } - - mod_pr_debug!( - "WorkQueue({:?}): Failing all jobs {:?}\n", - inner.pipe_type, - error - ); - - let mut cmds = KVec::new(); - - core::mem::swap(&mut inner.pending, &mut cmds); - - if inner.pending_jobs == 0 { - inner.event = None; - } - - core::mem::drop(inner); - - for mut cmd in cmds { - cmd.as_mut().inner_mut().mark_error(error); - cmd.as_mut().inner_mut().complete(); - } - } -} diff --git a/include/uapi/drm/asahi_neo_drm.h b/include/uapi/drm/asahi_neo_drm.h deleted file mode 100644 index 8af13919eae333..00000000000000 --- a/include/uapi/drm/asahi_neo_drm.h +++ /dev/null @@ -1,1524 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright (C) The Asahi Linux Contributors - * Copyright (C) 2018-2023 Collabora Ltd. - * Copyright (C) 2014-2018 Broadcom - */ -#ifndef _ASAHI_NEO_DRM_H_ -#define _ASAHI_NEO_DRM_H_ - -#include "drm.h" - -#if defined(__cplusplus) -extern "C" { -#endif - -/** - * DOC: Introduction to the Asahi UAPI - * - * This documentation describes the Asahi IOCTLs. - * - * Just a few generic rules about the data passed to the Asahi IOCTLs (cribbed - * from Panthor): - * - * - Structures must be aligned on 64-bit/8-byte. If the object is not - * naturally aligned, a padding field must be added. - * - Fields must be explicitly aligned to their natural type alignment with - * pad[0..N] fields. - * - All padding fields will be checked by the driver to make sure they are - * zeroed. - * - Flags can be added, but not removed/replaced. - * - New fields can be added to the main structures (the structures - * directly passed to the ioctl). Those fields can be added at the end of - * the structure, or replace existing padding fields. Any new field being - * added must preserve the behavior that existed before those fields were - * added when a value of zero is passed. - * - New fields can be added to indirect objects (objects pointed by the - * main structure), iff those objects are passed a size to reflect the - * size known by the userspace driver (see - * drm_asahi_neo_cmd_header::size). - * - If the kernel driver is too old to know some fields, those will be - * ignored if zero, and otherwise rejected (and so will be zero on output). - * - If userspace is too old to know some fields, those will be zeroed - * (input) before the structure is parsed by the kernel driver. - * - Each new flag/field addition must come with a driver version update so - * the userspace driver doesn't have to guess which flags are supported. - * - Structures should not contain unions, as this would defeat the - * extensibility of such structures. - * - IOCTLs can't be removed or replaced. New IOCTL IDs should be placed - * at the end of the drm_asahi_neo_ioctl_id enum. - */ - -/** - * enum drm_asahi_neo_ioctl_id - IOCTL IDs - * - * Place new ioctls at the end, don't re-order, don't replace or remove entries. - * - * These IDs are not meant to be used directly. Use the DRM_IOCTL_ASAHI_NEO_xxx - * definitions instead. - */ -enum drm_asahi_neo_ioctl_id { - /** @DRM_ASAHI_NEO_GET_PARAMS: Query device properties. */ - DRM_ASAHI_NEO_GET_PARAMS = 0, - - /** @DRM_ASAHI_NEO_GET_TIME: Query device time. */ - DRM_ASAHI_NEO_GET_TIME, - - /** @DRM_ASAHI_NEO_VM_CREATE: Create a GPU VM address space. */ - DRM_ASAHI_NEO_VM_CREATE, - - /** @DRM_ASAHI_NEO_VM_DESTROY: Destroy a VM. */ - DRM_ASAHI_NEO_VM_DESTROY, - - /** @DRM_ASAHI_NEO_VM_BIND: Bind/unbind memory to a VM. */ - DRM_ASAHI_NEO_VM_BIND, - - /** @DRM_ASAHI_NEO_GEM_CREATE: Create a buffer object. */ - DRM_ASAHI_NEO_GEM_CREATE, - - /** - * @DRM_ASAHI_NEO_GEM_MMAP_OFFSET: Get offset to pass to mmap() to map a - * given GEM handle. - */ - DRM_ASAHI_NEO_GEM_MMAP_OFFSET, - - /** @DRM_ASAHI_NEO_GEM_BIND_OBJECT: Bind memory as a special object */ - DRM_ASAHI_NEO_GEM_BIND_OBJECT, - - /** @DRM_ASAHI_NEO_QUEUE_CREATE: Create a scheduling queue. */ - DRM_ASAHI_NEO_QUEUE_CREATE, - - /** @DRM_ASAHI_NEO_QUEUE_DESTROY: Destroy a scheduling queue. */ - DRM_ASAHI_NEO_QUEUE_DESTROY, - - /** @DRM_ASAHI_NEO_SUBMIT: Submit commands to a queue. */ - DRM_ASAHI_NEO_SUBMIT, - - /** @DRM_ASAHI_NEO_GEM_MADVISE: Mark a VM-private object purgeable or needed. */ - DRM_ASAHI_NEO_GEM_MADVISE, -}; - -#define DRM_ASAHI_NEO_MAX_CLUSTERS 64 - -/** - * struct drm_asahi_neo_params_global - Global parameters. - * - * This struct may be queried by drm_asahi_neo_get_params. - */ -struct drm_asahi_neo_params_global { - /** @features: Feature bits from drm_asahi_neo_feature */ - __u64 features; - - /** @gpu_generation: GPU generation, e.g. 13 for G13G */ - __u32 gpu_generation; - - /** @gpu_variant: GPU variant as a character, e.g. 'C' for G13C */ - __u32 gpu_variant; - - /** - * @gpu_revision: GPU revision in BCD, e.g. 0x00 for 'A0' or - * 0x21 for 'C1' - */ - __u32 gpu_revision; - - /** @chip_id: Chip ID in BCD, e.g. 0x8103 for T8103 */ - __u32 chip_id; - - /** @num_dies: Number of dies in the SoC */ - __u32 num_dies; - - /** @num_clusters_total: Number of GPU clusters (across all dies) */ - __u32 num_clusters_total; - - /** - * @num_cores_per_cluster: Number of logical cores per cluster - * (including inactive/nonexistent) - */ - __u32 num_cores_per_cluster; - - /** @max_frequency_khz: Maximum GPU core clock frequency */ - __u32 max_frequency_khz; - - /** @core_masks: Bitmask of present/enabled cores per cluster */ - __u64 core_masks[DRM_ASAHI_NEO_MAX_CLUSTERS]; - - /** - * @vm_start: VM range start VMA. Together with @vm_end, this defines - * the window of valid GPU VAs. Userspace is expected to subdivide VAs - * out of this window. - * - * This window contains all virtual addresses that userspace needs to - * know about. There may be kernel-internal GPU VAs outside this range, - * but that detail is not relevant here. - */ - __u64 vm_start; - - /** @vm_end: VM range end VMA */ - __u64 vm_end; - - /** - * @vm_kernel_min_size: Minimum kernel VMA window size. - * - * When creating a VM, userspace is required to carve out a section of - * virtual addresses (within the range given by @vm_start and - * @vm_end). The kernel will allocate various internal structures - * within the specified VA range. - * - * Allowing userspace to choose the VA range for the kernel, rather than - * the kernel reserving VAs and requiring userspace to cope, can assist - * in implementing SVM. - */ - __u64 vm_kernel_min_size; - - /** - * @max_commands_per_submission: Maximum number of supported commands - * per submission. This mirrors firmware limits. Userspace must split up - * larger command buffers, which may require inserting additional - * synchronization. - */ - __u32 max_commands_per_submission; - - /** - * @max_attachments: Maximum number of drm_asahi_neo_attachment's per - * command - */ - __u32 max_attachments; - - /** - * @command_timestamp_frequency_hz: Timebase frequency for timestamps - * written during command execution, specified via drm_asahi_neo_timestamp - * structures. As this rate is controlled by the firmware, it is a - * queryable parameter. - * - * Userspace must divide by this frequency to convert timestamps to - * seconds, rather than hardcoding a particular firmware's rate. - */ - __u64 command_timestamp_frequency_hz; - - /** - * @usc_generation: Instruction set generation of the unified shader - * cores: 2 for G13 and G14, 3 for G15 and newer. - */ - __u32 usc_generation; - - /** - * @gpu_hal_generation: Command stream generation, from - * enum drm_asahi_neo_gpu_hal_generation. - */ - __u32 gpu_hal_generation; -}; - -/** - * enum drm_asahi_neo_gpu_hal_generation - GPU command stream generation - * - * The command stream generation determines the layout of the control streams - * and descriptors that userspace builds for the GPU. - */ -enum drm_asahi_neo_gpu_hal_generation { - /** @DRM_ASAHI_NEO_GPU_HAL_LEGACY: Unnumbered generation of G13 to G15. */ - DRM_ASAHI_NEO_GPU_HAL_LEGACY = 0, - - /** @DRM_ASAHI_NEO_GPU_HAL_200: Generation 200, used by G17P. */ - DRM_ASAHI_NEO_GPU_HAL_200 = 200, - - /** @DRM_ASAHI_NEO_GPU_HAL_300: Generation 300. */ - DRM_ASAHI_NEO_GPU_HAL_300 = 300, -}; - -/** - * enum drm_asahi_neo_feature - Feature bits - * - * This covers only features that userspace cannot infer from the architecture - * version. Most features don't need to be here. - */ -enum drm_asahi_neo_feature { - /** - * @DRM_ASAHI_NEO_FEATURE_SOFT_FAULTS: GPU has "soft fault" enabled. Shader - * loads of unmapped memory will return zero. Shader stores to unmapped - * memory will be silently discarded. Note that only shader load/store - * is affected. Other hardware units are not affected, notably including - * texture sampling. - * - * Soft fault is set when initializing the GPU and cannot be runtime - * toggled. Therefore, it is exposed as a feature bit and not a - * userspace-settable flag on the VM. When soft fault is enabled, - * userspace can speculate memory accesses more aggressively. - */ - DRM_ASAHI_NEO_FEATURE_SOFT_FAULTS = (1UL) << 0, - - /** - * @DRM_ASAHI_NEO_FEATURE_VM_STATUS: GET_PARAMS supports - * DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS. - */ - DRM_ASAHI_NEO_FEATURE_VM_STATUS = (1UL) << 1, - - /** - * @DRM_ASAHI_NEO_FEATURE_INDEPENDENT_QUEUES: Each queue reserves its own - * hardware execution state when it is created, and GET_PARAMS - * supports DRM_ASAHI_NEO_PARAM_GROUP_QUEUE_LIMITS. - */ - DRM_ASAHI_NEO_FEATURE_INDEPENDENT_QUEUES = (1UL) << 2, - - /** - * @DRM_ASAHI_NEO_FEATURE_PER_COMMAND_INPUT_SYNCS: Every hardware command of - * a submission waits for all of the submission's input syncs, including - * their errors, so independent commands need no extra dependency on the - * first command. Userspace that splits a batch across several - * submissions and relies on this must pass its input syncs to each of - * them. - */ - DRM_ASAHI_NEO_FEATURE_PER_COMMAND_INPUT_SYNCS = (1UL) << 3, - - /** - * @DRM_ASAHI_NEO_FEATURE_VM_STATUS_MIRROR: GET_PARAMS supports - * DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS_MIRROR, which registers a userspace - * word that always holds the VM status that - * DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS would return. The kernel stores a - * failure there before it signals any fence of the failed job, so after - * a successful wait a load of the word replaces the VM_STATUS query. - * Registration may still be refused. - */ - DRM_ASAHI_NEO_FEATURE_VM_STATUS_MIRROR = (1UL) << 4, - - /** - * @DRM_ASAHI_NEO_FEATURE_EXACT_PRIOR_BARRIERS: Render and compute payloads - * support exact queue-local historical prefixes in prior_vdm/prior_cdm. - * These fields supplement the header barriers. - */ - DRM_ASAHI_NEO_FEATURE_EXACT_PRIOR_BARRIERS = (1UL) << 5, - - /** - * @DRM_ASAHI_NEO_FEATURE_FEW_PRIMITIVES: Render commands accept - * DRM_ASAHI_NEO_RENDER_FEW_PRIMITIVES. - */ - DRM_ASAHI_NEO_FEATURE_FEW_PRIMITIVES = (1UL) << 6, - - /** - * @DRM_ASAHI_NEO_FEATURE_FRAGMENT_BARRIERS: Render commands accept - * DRM_ASAHI_NEO_RENDER_FRAGMENT_BARRIERS. - */ - DRM_ASAHI_NEO_FEATURE_FRAGMENT_BARRIERS = (1UL) << 7, -}; - -/** - * define DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS - GET_PARAMS group for - * struct drm_asahi_neo_vm_status - */ -#define DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS 1 - -/** - * define DRM_ASAHI_NEO_PARAM_GROUP_QUEUE_LIMITS - GET_PARAMS group for - * struct drm_asahi_neo_queue_limits - */ -#define DRM_ASAHI_NEO_PARAM_GROUP_QUEUE_LIMITS 2 - -/** - * define DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS_MIRROR - GET_PARAMS group for - * struct drm_asahi_neo_vm_status_mirror - */ -#define DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS_MIRROR 3 - -/** - * struct drm_asahi_neo_vm_status - VM status, GET_PARAMS group - * DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS - * - * This is an in/out query on a VM of the calling file. Userspace sets - * &drm_asahi_neo_get_params.pointer to this structure and - * &drm_asahi_neo_get_params.size to its size. - * - * The kernel records the failure of an accepted job before it signals the - * job's fences, so userspace must query the status after a successful syncobj - * wait before it treats the work as successfully executed. A signalled - * syncobj alone does not indicate successful execution. An execution failure - * also marks other VMs whose work can no longer make progress. A query never - * resets or recovers the GPU. - * - * Invalid requests fail with EINVAL (non-zero @flags or @pad, wrong size), - * ENOENT (@vm_id unknown to the calling file) or EFAULT, and write nothing. - */ -struct drm_asahi_neo_vm_status { - /** @vm_id: VM to query */ - __u32 vm_id; - - /** @flags: MBZ */ - __u32 flags; - - /** - * @error: Output: 0 if no submission error was observed, otherwise the - * first error as a negative errno. The error is sticky for the lifetime - * of the VM: neither creating queues nor waiting on other fences clears - * it. - */ - __s32 error; - - /** @pad: MBZ */ - __u32 pad; -}; - -/** - * struct drm_asahi_neo_queue_limits - Queue limits, GET_PARAMS group - * DRM_ASAHI_NEO_PARAM_GROUP_QUEUE_LIMITS - * - * The query neither allocates nor resets anything. - */ -struct drm_asahi_neo_queue_limits { - /** - * @max_queues: Maximum number of queues on the device, shared by all - * files and VMs. This is a hardware bound, not a promise that as many - * are currently available: QUEUE_CREATE reserves a queue's hardware - * state and fails once it runs out. - */ - __u32 max_queues; - - /** - * @max_in_flight_per_queue: Number of jobs of a queue that the - * hardware may run at once. Further jobs wait behind the preceding work - * and their dependencies. - */ - __u32 max_in_flight_per_queue; - - /** @flags: Reserved, zero */ - __u32 flags; - - /** @pad: Reserved, zero */ - __u32 pad; -}; - -/** - * struct drm_asahi_neo_vm_status_mirror - VM status mirror registration, GET_PARAMS - * group DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS_MIRROR - * - * Registers a 32-bit word in a GEM object of the calling file. The kernel - * keeps its own reference to the object for the lifetime of the VM status, - * zeroes the word, and from then on keeps it equal to the VM status error - * (see &drm_asahi_neo_vm_status.error). Each VM accepts one registration. - * EOPNOTSUPP means that userspace must keep using - * DRM_ASAHI_NEO_PARAM_GROUP_VM_STATUS. - */ -struct drm_asahi_neo_vm_status_mirror { - /** @vm_id: VM whose status is mirrored */ - __u32 vm_id; - - /** @handle: GEM object holding the word, at most 64 KiB in size */ - __u32 handle; - - /** @offset: Byte offset of the word in the object, 4-byte aligned */ - __u32 offset; - - /** @flags: MBZ */ - __u32 flags; -}; - -/** - * struct drm_asahi_neo_get_params - Arguments passed to DRM_IOCTL_ASAHI_NEO_GET_PARAMS - */ -struct drm_asahi_neo_get_params { - /** - * @param_group: Parameter group: 0 for struct drm_asahi_neo_params_global, - * otherwise one of DRM_ASAHI_NEO_PARAM_GROUP_*. - */ - __u32 param_group; - - /** @pad: MBZ */ - __u32 pad; - - /** - * @pointer: User pointer to the parameter struct. The VM status groups - * also read their input from it. - */ - __u64 pointer; - - /** - * @size: Size of the user buffer. In case of older userspace, this may - * be less than sizeof(struct drm_asahi_neo_params_global). The kernel will - * not write past the length specified here, allowing extensibility. - */ - __u64 size; -}; - -/** - * struct drm_asahi_neo_vm_create - Arguments passed to DRM_IOCTL_ASAHI_NEO_VM_CREATE - */ -struct drm_asahi_neo_vm_create { - /** - * @kernel_start: Start of the kernel-reserved address range. See - * drm_asahi_neo_params_global::vm_kernel_min_size. - * - * Both @kernel_start and @kernel_end must be within the range of - * valid VAs given by drm_asahi_neo_params_global::vm_start and - * drm_asahi_neo_params_global::vm_end. The size of the kernel range - * (@kernel_end - @kernel_start) must be at least - * drm_asahi_neo_params_global::vm_kernel_min_size. - * - * Userspace must not bind any memory on this VM into this reserved - * range, it is for kernel use only. - */ - __u64 kernel_start; - - /** - * @kernel_end: End of the kernel-reserved address range. See - * @kernel_start. - */ - __u64 kernel_end; - - /** @vm_id: Returned VM ID */ - __u32 vm_id; - - /** @pad: MBZ */ - __u32 pad; -}; - -/** - * struct drm_asahi_neo_vm_destroy - Arguments passed to DRM_IOCTL_ASAHI_NEO_VM_DESTROY - */ -struct drm_asahi_neo_vm_destroy { - /** @vm_id: VM ID to be destroyed */ - __u32 vm_id; - - /** @pad: MBZ */ - __u32 pad; -}; - -/** - * enum drm_asahi_neo_gem_flags - Flags for GEM creation - */ -enum drm_asahi_neo_gem_flags { - /** - * @DRM_ASAHI_NEO_GEM_WRITEBACK: BO should be CPU-mapped as writeback. - * - * Map as writeback instead of write-combine. This optimizes for CPU - * reads. - */ - DRM_ASAHI_NEO_GEM_WRITEBACK = (1L << 0), - - /** - * @DRM_ASAHI_NEO_GEM_VM_PRIVATE: BO is private to this GPU VM (no exports). - */ - DRM_ASAHI_NEO_GEM_VM_PRIVATE = (1L << 1), -}; - -/** - * struct drm_asahi_neo_gem_create - Arguments passed to DRM_IOCTL_ASAHI_NEO_GEM_CREATE - */ -struct drm_asahi_neo_gem_create { - /** @size: Size of the BO */ - __u64 size; - - /** @flags: Combination of drm_asahi_neo_gem_flags flags. */ - __u32 flags; - - /** - * @vm_id: VM ID to assign to the BO, if DRM_ASAHI_NEO_GEM_VM_PRIVATE is set - */ - __u32 vm_id; - - /** @handle: Returned GEM handle for the BO */ - __u32 handle; - - /** @pad: MBZ */ - __u32 pad; -}; - -/** - * struct drm_asahi_neo_gem_mmap_offset - Arguments passed to - * DRM_IOCTL_ASAHI_NEO_GEM_MMAP_OFFSET - */ -struct drm_asahi_neo_gem_mmap_offset { - /** @handle: Handle for the object being mapped. */ - __u32 handle; - - /** @flags: Must be zero */ - __u32 flags; - - /** @offset: The fake offset to use for subsequent mmap call */ - __u64 offset; -}; - -/** - * enum drm_asahi_neo_gem_madv - Advice for DRM_IOCTL_ASAHI_NEO_GEM_MADVISE - */ -enum drm_asahi_neo_gem_madv { - /** @DRM_ASAHI_NEO_GEM_MADV_WILLNEED: The object's contents are needed. */ - DRM_ASAHI_NEO_GEM_MADV_WILLNEED = 0, - /** - * @DRM_ASAHI_NEO_GEM_MADV_DONTNEED: The object is not referenced by any - * pending or future GPU work until the next WILLNEED. The kernel may - * discard its contents under memory pressure. - */ - DRM_ASAHI_NEO_GEM_MADV_DONTNEED = 1, -}; - -/** - * struct drm_asahi_neo_gem_madvise - Arguments passed to - * DRM_IOCTL_ASAHI_NEO_GEM_MADVISE - * - * Only VM-private objects accept advice. A purged object keeps its handle - * and mappings but has no contents; userspace must not use it again. - */ -struct drm_asahi_neo_gem_madvise { - /** @handle: Handle of the object. */ - __u32 handle; - - /** @madv: One of enum drm_asahi_neo_gem_madv. */ - __u32 madv; - - /** @retained: Set by the kernel: 1 if the contents are retained, 0 if purged. */ - __u32 retained; - - /** @pad: Must be zero. */ - __u32 pad; -}; - -/** - * enum drm_asahi_neo_bind_flags - Flags for GEM binding - */ -enum drm_asahi_neo_bind_flags { - /** - * @DRM_ASAHI_NEO_BIND_UNBIND: Instead of binding a GEM object to the range, - * simply unbind the GPU VMA range. - */ - DRM_ASAHI_NEO_BIND_UNBIND = (1L << 0), - - /** @DRM_ASAHI_NEO_BIND_READ: Map BO with GPU read permission */ - DRM_ASAHI_NEO_BIND_READ = (1L << 1), - - /** @DRM_ASAHI_NEO_BIND_WRITE: Map BO with GPU write permission */ - DRM_ASAHI_NEO_BIND_WRITE = (1L << 2), - - /** - * @DRM_ASAHI_NEO_BIND_SINGLE_PAGE: Map a single page of the BO repeatedly - * across the VA range. - * - * This is useful to fill a VA range with scratch pages or zero pages. - * It is intended as a mechanism to accelerate sparse. - */ - DRM_ASAHI_NEO_BIND_SINGLE_PAGE = (1L << 3), -}; - -/** - * struct drm_asahi_neo_gem_bind_op - Description of a single GEM bind operation. - */ -struct drm_asahi_neo_gem_bind_op { - /** @flags: Combination of drm_asahi_neo_bind_flags flags. */ - __u32 flags; - - /** @handle: GEM object to bind (except for UNBIND) */ - __u32 handle; - - /** - * @offset: Offset into the object (except for UNBIND). - * - * For a regular bind, this is the beginning of the region of the GEM - * object to bind. - * - * For a single-page bind, this is the offset to the single page that - * will be repeatedly bound. - * - * Must be page-size aligned. - */ - __u64 offset; - - /** - * @range: Number of bytes to bind/unbind to @addr. - * - * Must be page-size aligned. - */ - __u64 range; - - /** - * @addr: Address to bind to. - * - * Must be page-size aligned. - */ - __u64 addr; -}; - -/** - * struct drm_asahi_neo_vm_bind - Arguments passed to - * DRM_IOCTL_ASAHI_NEO_VM_BIND - */ -struct drm_asahi_neo_vm_bind { - /** @vm_id: The ID of the VM to bind to */ - __u32 vm_id; - - /** @num_binds: number of binds in this IOCTL. */ - __u32 num_binds; - - /** - * @stride: Stride in bytes between consecutive binds. This allows - * extensibility of drm_asahi_neo_gem_bind_op. - */ - __u32 stride; - - /** @pad: MBZ */ - __u32 pad; - - /** - * @userptr: User pointer to an array of @num_binds structures of type - * @drm_asahi_neo_gem_bind_op and size @stride bytes. - */ - __u64 userptr; -}; - -/** - * enum drm_asahi_neo_bind_object_op - Special object bind operation - */ -enum drm_asahi_neo_bind_object_op { - /** @DRM_ASAHI_NEO_BIND_OBJECT_OP_BIND: Bind a BO as a special GPU object */ - DRM_ASAHI_NEO_BIND_OBJECT_OP_BIND = 0, - - /** @DRM_ASAHI_NEO_BIND_OBJECT_OP_UNBIND: Unbind a special GPU object */ - DRM_ASAHI_NEO_BIND_OBJECT_OP_UNBIND = 1, -}; - -/** - * enum drm_asahi_neo_bind_object_flags - Special object bind flags - */ -enum drm_asahi_neo_bind_object_flags { - /** - * @DRM_ASAHI_NEO_BIND_OBJECT_USAGE_TIMESTAMPS: Map a BO as a timestamp - * buffer. - */ - DRM_ASAHI_NEO_BIND_OBJECT_USAGE_TIMESTAMPS = (1L << 0), -}; - -/** - * struct drm_asahi_neo_gem_bind_object - Arguments passed to - * DRM_IOCTL_ASAHI_NEO_GEM_BIND_OBJECT - */ -struct drm_asahi_neo_gem_bind_object { - /** @op: Bind operation (enum drm_asahi_neo_bind_object_op) */ - __u32 op; - - /** @flags: Combination of drm_asahi_neo_bind_object_flags flags. */ - __u32 flags; - - /** @handle: GEM object to bind/unbind (BIND) */ - __u32 handle; - - /** @vm_id: The ID of the VM to operate on (MBZ currently) */ - __u32 vm_id; - - /** @offset: Offset into the object (BIND only) */ - __u64 offset; - - /** @range: Number of bytes to bind/unbind (BIND only) */ - __u64 range; - - /** @object_handle: Object handle (out for BIND, in for UNBIND) */ - __u32 object_handle; - - /** @pad: MBZ */ - __u32 pad; -}; - -/** - * enum drm_asahi_neo_cmd_type - Command type - */ -enum drm_asahi_neo_cmd_type { - /** - * @DRM_ASAHI_NEO_CMD_RENDER: Render command, executing on the render - * subqueue. Combined vertex and fragment operation. - * - * Followed by a @drm_asahi_neo_cmd_render payload. - */ - DRM_ASAHI_NEO_CMD_RENDER = 0, - - /** - * @DRM_ASAHI_NEO_CMD_COMPUTE: Compute command on the compute subqueue. - * - * Followed by a @drm_asahi_neo_cmd_compute payload. - */ - DRM_ASAHI_NEO_CMD_COMPUTE = 1, - - /** - * @DRM_ASAHI_NEO_SET_VERTEX_ATTACHMENTS: Software command to set - * attachments for subsequent vertex shaders in the same submit. - * - * Followed by (possibly multiple) @drm_asahi_neo_attachment payloads. - */ - DRM_ASAHI_NEO_SET_VERTEX_ATTACHMENTS = 2, - - /** - * @DRM_ASAHI_NEO_SET_FRAGMENT_ATTACHMENTS: Software command to set - * attachments for subsequent fragment shaders in the same submit. - * - * Followed by (possibly multiple) @drm_asahi_neo_attachment payloads. - */ - DRM_ASAHI_NEO_SET_FRAGMENT_ATTACHMENTS = 3, - - /** - * @DRM_ASAHI_NEO_SET_COMPUTE_ATTACHMENTS: Software command to set - * attachments for subsequent compute shaders in the same submit. - * - * Followed by (possibly multiple) @drm_asahi_neo_attachment payloads. - */ - DRM_ASAHI_NEO_SET_COMPUTE_ATTACHMENTS = 4, -}; - -/** - * enum drm_asahi_neo_priority - Scheduling queue priority. - * - * These priorities are forwarded to the firmware to influence firmware - * scheduling. The exact policy is ultimately decided by firmware, but - * these enums allow userspace to communicate the intentions. - */ -enum drm_asahi_neo_priority { - /** @DRM_ASAHI_NEO_PRIORITY_LOW: Low priority queue. */ - DRM_ASAHI_NEO_PRIORITY_LOW = 0, - - /** @DRM_ASAHI_NEO_PRIORITY_MEDIUM: Medium priority queue. */ - DRM_ASAHI_NEO_PRIORITY_MEDIUM = 1, - - /** - * @DRM_ASAHI_NEO_PRIORITY_HIGH: High priority queue. - * - * Reserved for future extension. - */ - DRM_ASAHI_NEO_PRIORITY_HIGH = 2, - - /** - * @DRM_ASAHI_NEO_PRIORITY_REALTIME: Real-time priority queue. - * - * Reserved for future extension. - */ - DRM_ASAHI_NEO_PRIORITY_REALTIME = 3, -}; - -/** - * struct drm_asahi_neo_queue_create - Arguments passed to - * DRM_IOCTL_ASAHI_NEO_QUEUE_CREATE - */ -struct drm_asahi_neo_queue_create { - /** @flags: MBZ */ - __u32 flags; - - /** @vm_id: The ID of the VM this queue is bound to */ - __u32 vm_id; - - /** @priority: One of drm_asahi_neo_priority */ - __u32 priority; - - /** @queue_id: The returned queue ID */ - __u32 queue_id; - - /** - * @usc_exec_base: GPU base address for all USC binaries (shaders) on - * this queue. USC addresses are 32-bit relative to this 64-bit base. - * - * This sets the following registers on all queue commands: - * - * USC_EXEC_BASE_TA (vertex) - * USC_EXEC_BASE_ISP (fragment) - * USC_EXEC_BASE_CP (compute) - * - * While the hardware lets us configure these independently per command, - * we do not have a use case for this. Instead, we expect userspace to - * fix a 4GiB VA carveout for USC memory and pass its base address here. - */ - __u64 usc_exec_base; -}; - -/** - * struct drm_asahi_neo_queue_destroy - Arguments passed to - * DRM_IOCTL_ASAHI_NEO_QUEUE_DESTROY - */ -struct drm_asahi_neo_queue_destroy { - /** @queue_id: The queue ID to be destroyed */ - __u32 queue_id; - - /** @pad: MBZ */ - __u32 pad; -}; - -/** - * enum drm_asahi_neo_sync_type - Sync item type - */ -enum drm_asahi_neo_sync_type { - /** @DRM_ASAHI_NEO_SYNC_SYNCOBJ: Binary sync object */ - DRM_ASAHI_NEO_SYNC_SYNCOBJ = 0, - - /** @DRM_ASAHI_NEO_SYNC_TIMELINE_SYNCOBJ: Timeline sync object */ - DRM_ASAHI_NEO_SYNC_TIMELINE_SYNCOBJ = 1, -}; - -/** - * struct drm_asahi_neo_sync - Sync item - */ -struct drm_asahi_neo_sync { - /** @sync_type: One of drm_asahi_neo_sync_type */ - __u32 sync_type; - - /** @handle: The sync object handle */ - __u32 handle; - - /** @timeline_value: Timeline value for timeline sync objects */ - __u64 timeline_value; -}; - -/** - * define DRM_ASAHI_NEO_BARRIER_NONE - Command index for no barrier - * - * This special value may be passed in to drm_asahi_neo_command::vdm_barrier or - * drm_asahi_neo_command::cdm_barrier to indicate that the respective subqueue - * should not wait on any previous work. - */ -#define DRM_ASAHI_NEO_BARRIER_NONE (0xFFFFu) - -/** - * struct drm_asahi_neo_cmd_header - Top level command structure - * - * This struct is core to the command buffer definition and therefore is not - * extensible. - */ -struct drm_asahi_neo_cmd_header { - /** @cmd_type: One of drm_asahi_neo_cmd_type */ - __u16 cmd_type; - - /** - * @size: Size of this command, not including this header. - * - * For hardware commands, this enables extensibility of commands without - * requiring extra command types. Passing a command that is shorter - * than expected is explicitly allowed for backwards-compatibility. - * Truncated fields will be zeroed. - * - * For the synthetic attachment setting commands, this implicitly - * encodes the number of attachments. These commands take multiple - * fixed-size @drm_asahi_neo_attachment structures as their payload, so size - * equals number of attachments * sizeof(struct drm_asahi_neo_attachment). - */ - __u16 size; - - /** - * @vdm_barrier: VDM (render) command index to wait on. - * - * Barriers are indices relative to the beginning of a given submit. A - * barrier of 0 waits on commands submitted to the respective subqueue - * in previous submit ioctls. A barrier of N waits on N previous - * commands on the subqueue within the current submit ioctl. As a - * special case, passing @DRM_ASAHI_NEO_BARRIER_NONE avoids waiting on any - * commands in the subqueue. - * - * Examples: - * - * 0: This waits on all previous work. - * - * NONE: This does not wait for anything on this subqueue. - * - * 1: This waits on the first render command in the submit. - * This is valid only if there are multiple render commands in the - * same submit. - * - * Barriers are valid only for hardware commands. Synthetic software - * commands to set attachments must pass NONE here. - */ - __u16 vdm_barrier; - - /** - * @cdm_barrier: CDM (compute) command index to wait on. - * - * See @vdm_barrier, and replace VDM/render with CDM/compute. - */ - __u16 cdm_barrier; -}; - -/** - * struct drm_asahi_neo_submit - Arguments passed to DRM_IOCTL_ASAHI_NEO_SUBMIT - */ -struct drm_asahi_neo_submit { - /** - * @syncs: An optional pointer to an array of drm_asahi_neo_sync. The first - * @in_sync_count elements are in-syncs, then the remaining - * @out_sync_count elements are out-syncs. Using a single array with - * explicit partitioning simplifies handling. - */ - __u64 syncs; - - /** - * @cmdbuf: Pointer to the command buffer to submit. - * - * This is a flat command buffer. By design, it contains no CPU - * pointers, which makes it suitable for a virtgpu wire protocol without - * requiring any serializing/deserializing step. - * - * It consists of a series of commands. Each command begins with a - * fixed-size @drm_asahi_neo_cmd_header header and is followed by a - * variable-length payload according to the type and size in the header. - * - * The combined count of "real" hardware commands must be nonzero and at - * most drm_asahi_neo_params_global::max_commands_per_submission. - */ - __u64 cmdbuf; - - /** @flags: Flags for command submission (MBZ) */ - __u32 flags; - - /** @queue_id: The queue ID to be submitted to */ - __u32 queue_id; - - /** - * @in_sync_count: Number of sync objects to wait on before starting - * this job. - */ - __u32 in_sync_count; - - /** - * @out_sync_count: Number of sync objects to signal upon completion of - * this job. - */ - __u32 out_sync_count; - - /** @cmdbuf_size: Command buffer size in bytes */ - __u32 cmdbuf_size; - - /** @pad: MBZ */ - __u32 pad; -}; - -/** - * struct drm_asahi_neo_attachment - Describe an "attachment". - * - * Attachments are any memory written by shaders, notably including render - * target attachments written by the end-of-tile program. This is purely a hint - * about the accessed memory regions. It is optional to specify, which is - * fortunate as it cannot be specified precisely with bindless access anyway. - * But where possible, it's probably a good idea for userspace to include these - * hints, forwarded to the firmware. - * - * This struct is implicitly sized and therefore is not extensible. - */ -struct drm_asahi_neo_attachment { - /** @pointer: Base address of the attachment */ - __u64 pointer; - - /** @size: Size of the attachment in bytes */ - __u64 size; - - /** @pad: MBZ */ - __u32 pad; - - /** @flags: MBZ */ - __u32 flags; -}; - -enum drm_asahi_neo_render_flags { - /** - * @DRM_ASAHI_NEO_RENDER_VERTEX_SCRATCH: A vertex stage shader uses scratch - * memory. - */ - DRM_ASAHI_NEO_RENDER_VERTEX_SCRATCH = (1U << 0), - - /** - * @DRM_ASAHI_NEO_RENDER_PROCESS_EMPTY_TILES: Process even empty tiles. - * This must be set when clearing render targets. - */ - DRM_ASAHI_NEO_RENDER_PROCESS_EMPTY_TILES = (1U << 1), - - /** - * @DRM_ASAHI_NEO_RENDER_NO_VERTEX_CLUSTERING: Run vertex stage on a single - * cluster (on multi-cluster GPUs) - * - * This harms performance but can workaround certain sync/coherency - * bugs, and therefore is useful for debugging. - */ - DRM_ASAHI_NEO_RENDER_NO_VERTEX_CLUSTERING = (1U << 2), - - /** - * @DRM_ASAHI_NEO_RENDER_RSRC_SPEC_HI: The command carries the high halves - * of the background and end-of-tile resource specifiers - * (&drm_asahi_neo_cmd_render.bg_rsrc_spec_hi and the following fields). - * Kernels that do not know the flag reject the command instead of - * silently truncating the specifiers. - */ - DRM_ASAHI_NEO_RENDER_RSRC_SPEC_HI = (1U << 4), - - /** - * @DRM_ASAHI_NEO_RENDER_DBIAS_IS_INT: Use integer depth bias formula. - * - * Graphics specifications contain two alternate formulas for depth - * bias, a float formula used with floating-point depth buffers and an - * integer formula using with unorm depth buffers. This flag specifies - * that the integer formula should be used. If omitted, the float - * formula is used instead. - * - * This corresponds to bit 18 of the relevant hardware control register, - * so we match that here for efficiency. - */ - DRM_ASAHI_NEO_RENDER_DBIAS_IS_INT = (1U << 18), - - /** - * @DRM_ASAHI_NEO_RENDER_FEW_PRIMITIVES: The render pass is known to rasterize - * at most four primitives in total. - * - * The count includes every primitive the pass generates: topology-derived - * primitive counts of all draws multiplied by their instance counts, plus - * internal work (clears, visibility, scissor/state generated work). Passes - * with indirect draws, mesh work or any work whose primitive count is not - * known at record time must not set this flag. The flag sets bit 0 of the - * auxiliary framebuffer control word for the fragment stage; the hardware - * effect is an optimization for small known passes and its omission is - * always safe. Requires DRM_ASAHI_NEO_FEATURE_FEW_PRIMITIVES. - */ - DRM_ASAHI_NEO_RENDER_FEW_PRIMITIVES = (1U << 19), - - /** - * @DRM_ASAHI_NEO_RENDER_FRAGMENT_BARRIERS: Only the fragment stage waits for - * this command's barriers. - * - * The render dependencies resolved from @vdm_barrier and @prior_vdm are - * attached to the fragment half of the render instead of its vertex - * (tiling) half, so vertex processing may start while the renders this - * command depends on are still executing their fragment work. The - * fragment half still waits for its own vertex half and for the previous - * fragment work of its physical queue; the compute dependencies from - * @cdm_barrier and @prior_cdm keep ordering the vertex half. Set this - * only when every render dependency into the command is consumed by - * fragment-stage or attachment accesses alone: nothing the vertex stage - * reads (vertex/index/indirect buffers, vertex, tessellation or geometry - * shader resources, transform feedback, conditional-rendering predicates) - * may depend on it. Requires DRM_ASAHI_NEO_FEATURE_FRAGMENT_BARRIERS. - */ - DRM_ASAHI_NEO_RENDER_FRAGMENT_BARRIERS = (1U << 20), -}; - -/** - * struct drm_asahi_neo_zls_buffer - Describe a depth or stencil buffer. - * - * These fields correspond to hardware registers in the ZLS (Z Load/Store) unit. - * There are three hardware registers for each field respectively for loads, - * stores, and partial renders. In practice, it makes sense to set all to the - * same values, except in exceptional cases not yet implemented in userspace, so - * we do not duplicate here for simplicity/efficiency. - * - * This struct is embedded in other structs and therefore is not extensible. - */ -struct drm_asahi_neo_zls_buffer { - /** @base: Base address of the buffer */ - __u64 base; - - /** - * @comp_base: If the load buffer is compressed, address of the - * compression metadata section. - */ - __u64 comp_base; - - /** - * @stride: If layered rendering is enabled, the number of bytes - * between each layer of the buffer. - */ - __u32 stride; - - /** - * @comp_stride: If layered rendering is enabled, the number of bytes - * between each layer of the compression metadata. - */ - __u32 comp_stride; -}; - -/** - * struct drm_asahi_neo_timestamp - Describe a timestamp write. - * - * The firmware can optionally write the GPU timestamp at render pass - * granularities, but it needs to be mapped specially via - * DRM_IOCTL_ASAHI_NEO_GEM_BIND_OBJECT. This structure therefore describes where to - * write as a handle-offset pair, rather than a GPU address like normal. - * - * This struct is embedded in other structs and therefore is not extensible. - */ -struct drm_asahi_neo_timestamp { - /** - * @handle: Handle of the timestamp buffer, or 0 to skip this - * timestamp. If nonzero, this must equal the value returned in - * drm_asahi_neo_gem_bind_object::object_handle. - */ - __u32 handle; - - /** @offset: Offset to write into the timestamp buffer */ - __u32 offset; -}; - -/** - * struct drm_asahi_neo_timestamps - Describe timestamp writes. - * - * Each operation that can be timestamped, can be timestamped at the start and - * end. Therefore, drm_asahi_neo_timestamp structs always come in pairs, bundled - * together into drm_asahi_neo_timestamps. - * - * This struct is embedded in other structs and therefore is not extensible. - */ -struct drm_asahi_neo_timestamps { - /** @start: Timestamp recorded at the start of the operation */ - struct drm_asahi_neo_timestamp start; - - /** @end: Timestamp recorded at the end of the operation */ - struct drm_asahi_neo_timestamp end; -}; - -/** - * struct drm_asahi_neo_helper_program - Describe helper program configuration. - * - * The helper program is a compute-like kernel required for various hardware - * functionality. Its most important role is dynamically allocating - * scratch/stack memory for individual subgroups, by partitioning a static - * allocation shared for the whole device. It is supplied by userspace via - * drm_asahi_neo_helper_program and internally dispatched by the hardware as needed. - * - * This struct is embedded in other structs and therefore is not extensible. - */ -struct drm_asahi_neo_helper_program { - /** - * @binary: USC address to the helper program binary. This is a tagged - * pointer with configuration in the bottom bits. - */ - __u32 binary; - - /** @cfg: Additional configuration bits for the helper program. */ - __u32 cfg; - - /** - * @data: Data passed to the helper program. This value is not - * interpreted by the kernel, firmware, or hardware in any way. It is - * simply a sideband for userspace, set with the submit ioctl and read - * via special registers inside the helper program. - * - * In practice, userspace will pass a 64-bit GPU VA here pointing to the - * actual arguments, which presumably don't fit in 64-bits. - */ - __u64 data; -}; - -/** - * struct drm_asahi_neo_bg_eot - Describe a background or end-of-tile program. - * - * The background and end-of-tile programs are dispatched by the hardware at the - * beginning and end of rendering. As the hardware "tilebuffer" is simply local - * memory, these programs are necessary to implement API-level render targets. - * The fragment-like background program is responsible for loading either the - * clear colour or the existing render target contents, while the compute-like - * end-of-tile program stores the tilebuffer contents to memory. - * - * This struct is embedded in other structs and therefore is not extensible. - */ -struct drm_asahi_neo_bg_eot { - /** - * @usc: USC address of the hardware USC words binding resources - * (including images and uniforms) and the program itself. Note this is - * an additional layer of indirection compared to the helper program, - * avoiding the need for a sideband for data. This is a tagged pointer - * with additional configuration in the bottom bits. - */ - __u32 usc; - - /** - * @rsrc_spec: Resource specifier for the program. This is a packed - * hardware data structure describing the required number of registers, - * uniforms, bound textures, and bound samplers. - */ - __u32 rsrc_spec; -}; - -/** - * struct drm_asahi_neo_cmd_render - Command to submit 3D - * - * This command submits a single render pass. The hardware control stream may - * include many draws and subpasses, but within the command, the framebuffer - * dimensions and attachments are fixed. - * - * The hardware requires the firmware to set a large number of Control Registers - * setting up state at render pass granularity before each command rendering 3D. - * The firmware bundles this state into data structures. Unfortunately, we - * cannot expose either any of that directly to userspace, because the - * kernel-firmware ABI is not stable. Although we can guarantee the firmware - * updates in tandem with the kernel, we cannot break old userspace when - * upgrading the firmware and kernel. Therefore, we need to abstract well the - * data structures to avoid tying our hands with future firmwares. - * - * The bulk of drm_asahi_neo_cmd_render therefore consists of values of hardware - * control registers, marshalled via the firmware interface. - * - * The framebuffer/tilebuffer dimensions are also specified here. In addition to - * being passed to the firmware/hardware, the kernel requires these dimensions - * to calculate various essential tiling-related data structures. It is - * unfortunate that our submits are heavier than on vendors with saner - * hardware-software interfaces. The upshot is all of this information is - * readily available to userspace with all current APIs. - * - * It looks odd - but it's not overly burdensome and it ensures we can remain - * compatible with old userspace. - */ -struct drm_asahi_neo_cmd_render { - /** @flags: Combination of drm_asahi_neo_render_flags flags. */ - __u32 flags; - - /** - * @isp_zls_pixels: ISP_ZLS_PIXELS register value. This contains the - * depth/stencil width/height, which may differ from the framebuffer - * width/height. - */ - __u32 isp_zls_pixels; - - /** - * @vdm_ctrl_stream_base: VDM_CTRL_STREAM_BASE register value. GPU - * address to the beginning of the VDM control stream. - */ - __u64 vdm_ctrl_stream_base; - - /** @vertex_helper: Helper program used for the vertex shader */ - struct drm_asahi_neo_helper_program vertex_helper; - - /** @fragment_helper: Helper program used for the fragment shader */ - struct drm_asahi_neo_helper_program fragment_helper; - - /** - * @isp_scissor_base: ISP_SCISSOR_BASE register value. GPU address of an - * array of scissor descriptors indexed in the render pass. - */ - __u64 isp_scissor_base; - - /** - * @isp_dbias_base: ISP_DBIAS_BASE register value. GPU address of an - * array of depth bias values indexed in the render pass. - */ - __u64 isp_dbias_base; - - /** - * @isp_oclqry_base: ISP_OCLQRY_BASE register value. GPU address of an - * array of occlusion query results written by the render pass. - */ - __u64 isp_oclqry_base; - - /** @depth: Depth buffer */ - struct drm_asahi_neo_zls_buffer depth; - - /** @stencil: Stencil buffer */ - struct drm_asahi_neo_zls_buffer stencil; - - /** @zls_ctrl: ZLS_CTRL register value */ - __u64 zls_ctrl; - - /** @ppp_multisamplectl: PPP_MULTISAMPLECTL register value */ - __u64 ppp_multisamplectl; - - /** - * @sampler_heap: Base address of the sampler heap. This heap is used - * for both vertex shaders and fragment shaders. The registers are - * per-stage, but there is no known use case for separate heaps. - */ - __u64 sampler_heap; - - /** @ppp_ctrl: PPP_CTRL register value */ - __u32 ppp_ctrl; - - /** @width_px: Framebuffer width in pixels */ - __u16 width_px; - - /** @height_px: Framebuffer height in pixels */ - __u16 height_px; - - /** @layers: Number of layers in the framebuffer */ - __u16 layers; - - /** @sampler_count: Number of samplers in the sampler heap. */ - __u16 sampler_count; - - /** @utile_width_px: Width of a logical tilebuffer tile in pixels */ - __u8 utile_width_px; - - /** @utile_height_px: Height of a logical tilebuffer tile in pixels */ - __u8 utile_height_px; - - /** @samples: # of samples in the framebuffer. Must be 1, 2, or 4. */ - __u8 samples; - - /** @sample_size_B: # of bytes in the tilebuffer required per sample. */ - __u8 sample_size_B; - - /** - * @isp_merge_upper_x: 32-bit float used in the hardware triangle - * merging. Calculate as: tan(60 deg) * width. - * - * Making these values UAPI avoids requiring floating-point calculations - * in the kernel in the hot path. - */ - __u32 isp_merge_upper_x; - - /** - * @isp_merge_upper_y: 32-bit float. Calculate as: tan(60 deg) * height. - * See @isp_merge_upper_x. - */ - __u32 isp_merge_upper_y; - - /** @bg: Background program run for each tile at the start */ - struct drm_asahi_neo_bg_eot bg; - - /** @eot: End-of-tile program ran for each tile at the end */ - struct drm_asahi_neo_bg_eot eot; - - /** - * @partial_bg: Background program ran at the start of each tile when - * resuming the render pass during a partial render. - */ - struct drm_asahi_neo_bg_eot partial_bg; - - /** - * @partial_eot: End-of-tile program ran at the end of each tile when - * pausing the render pass during a partial render. - */ - struct drm_asahi_neo_bg_eot partial_eot; - - /** - * @isp_bgobjdepth: ISP_BGOBJDEPTH register value. This is the depth - * buffer clear value, encoded in the depth buffer's format: either a - * 32-bit float or a 16-bit unorm (with upper bits zeroed). - */ - __u32 isp_bgobjdepth; - - /** - * @isp_bgobjvals: ISP_BGOBJVALS register value. The bottom 8-bits - * contain the stencil buffer clear value. - */ - __u32 isp_bgobjvals; - - /** @ts_vtx: Timestamps for the vertex portion of the render */ - struct drm_asahi_neo_timestamps ts_vtx; - - /** @ts_frag: Timestamps for the fragment portion of the render */ - struct drm_asahi_neo_timestamps ts_frag; - - /** @bg_eot_rsrc_spec_hi: High half of @eot.rsrc_spec */ - __u32 bg_eot_rsrc_spec_hi; - - /** @bg_eot_partial_rsrc_spec_hi: High half of @partial_eot.rsrc_spec */ - __u32 bg_eot_partial_rsrc_spec_hi; - - /** @bg_rsrc_spec_hi: High half of @bg.rsrc_spec */ - __u32 bg_rsrc_spec_hi; - - /** @bg_partial_rsrc_spec_hi: High half of @partial_bg.rsrc_spec */ - __u32 bg_partial_rsrc_spec_hi; - - /** - * @prior_vdm: Historical render prefix, or zero for no added dependency. - * - * Requires DRM_ASAHI_NEO_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render - * commands accepted on this queue since creation, starting at one. A - * nonzero value must not exceed the render count at this ioctl's entry; - * it waits for every render command through that ordinal. The effective - * dependency is the maximum of this prefix and the header's resolved - * vdm_barrier. Header NONE does not suppress this field. - */ - __u64 prior_vdm; - - /** - * @prior_cdm: Historical compute prefix; see @prior_vdm, using the - * queue's compute command count and the header's cdm_barrier instead. - */ - __u64 prior_cdm; -}; - -/** - * struct drm_asahi_neo_cmd_compute - Command to submit compute - * - * This command submits a control stream consisting of compute dispatches. There - * is essentially no limit on how many compute dispatches may be included in a - * single compute command, although timestamps are at command granularity. - */ -struct drm_asahi_neo_cmd_compute { - /** @flags: MBZ */ - __u32 flags; - - /** @sampler_count: Number of samplers in the sampler heap. */ - __u32 sampler_count; - - /** - * @cdm_ctrl_stream_base: CDM_CTRL_STREAM_BASE register value. GPU - * address to the beginning of the CDM control stream. - */ - __u64 cdm_ctrl_stream_base; - - /** - * @cdm_ctrl_stream_end: GPU base address to the end of the hardware - * control stream. Note this only considers the first contiguous segment - * of the control stream, as the stream might jump elsewhere. - */ - __u64 cdm_ctrl_stream_end; - - /** @sampler_heap: Base address of the sampler heap. */ - __u64 sampler_heap; - - /** @helper: Helper program used for this compute command */ - struct drm_asahi_neo_helper_program helper; - - /** @ts: Timestamps for the compute command */ - struct drm_asahi_neo_timestamps ts; - - /** @scs_layout: Reserved; must be zero. */ - __u64 scs_layout; - - /** - * @prior_vdm: Historical render prefix, or zero for no added dependency. - * - * Requires DRM_ASAHI_NEO_FEATURE_EXACT_PRIOR_BARRIERS. Counts hardware render - * commands accepted on this queue since creation, starting at one. A - * nonzero value must not exceed the render count at this ioctl's entry; - * it waits for every render command through that ordinal. The effective - * dependency is the maximum of this prefix and the header's resolved - * vdm_barrier. Header NONE does not suppress this field. - */ - __u64 prior_vdm; - - /** - * @prior_cdm: Historical compute prefix; see @prior_vdm, using the - * queue's compute command count and the header's cdm_barrier instead. - */ - __u64 prior_cdm; -}; - -/** - * struct drm_asahi_neo_get_time - Arguments passed to DRM_IOCTL_ASAHI_NEO_GET_TIME - */ -struct drm_asahi_neo_get_time { - /** @flags: MBZ. */ - __u64 flags; - - /** @gpu_timestamp: On return, the GPU timestamp in nanoseconds. */ - __u64 gpu_timestamp; -}; - -/** - * DRM_IOCTL_ASAHI_NEO() - Build an Asahi IOCTL number - * @__access: Access type. Must be R, W or RW. - * @__id: One of the DRM_ASAHI_NEO_xxx id. - * @__type: Suffix of the type being passed to the IOCTL. - * - * Don't use this macro directly, use the DRM_IOCTL_ASAHI_NEO_xxx - * values instead. - * - * Return: An IOCTL number to be passed to ioctl() from userspace. - */ -#define DRM_IOCTL_ASAHI_NEO(__access, __id, __type) \ - DRM_IO ## __access(DRM_COMMAND_BASE + DRM_ASAHI_NEO_ ## __id, \ - struct drm_asahi_neo_ ## __type) - -/* Note: this is an enum so that it can be resolved by Rust bindgen. */ -enum { - DRM_IOCTL_ASAHI_NEO_GET_PARAMS = DRM_IOCTL_ASAHI_NEO(W, GET_PARAMS, get_params), - DRM_IOCTL_ASAHI_NEO_GET_TIME = DRM_IOCTL_ASAHI_NEO(WR, GET_TIME, get_time), - DRM_IOCTL_ASAHI_NEO_VM_CREATE = DRM_IOCTL_ASAHI_NEO(WR, VM_CREATE, vm_create), - DRM_IOCTL_ASAHI_NEO_VM_DESTROY = DRM_IOCTL_ASAHI_NEO(W, VM_DESTROY, vm_destroy), - DRM_IOCTL_ASAHI_NEO_VM_BIND = DRM_IOCTL_ASAHI_NEO(W, VM_BIND, vm_bind), - DRM_IOCTL_ASAHI_NEO_GEM_CREATE = DRM_IOCTL_ASAHI_NEO(WR, GEM_CREATE, gem_create), - DRM_IOCTL_ASAHI_NEO_GEM_MMAP_OFFSET = DRM_IOCTL_ASAHI_NEO(WR, GEM_MMAP_OFFSET, gem_mmap_offset), - DRM_IOCTL_ASAHI_NEO_GEM_BIND_OBJECT = DRM_IOCTL_ASAHI_NEO(WR, GEM_BIND_OBJECT, gem_bind_object), - DRM_IOCTL_ASAHI_NEO_QUEUE_CREATE = DRM_IOCTL_ASAHI_NEO(WR, QUEUE_CREATE, queue_create), - DRM_IOCTL_ASAHI_NEO_QUEUE_DESTROY = DRM_IOCTL_ASAHI_NEO(W, QUEUE_DESTROY, queue_destroy), - DRM_IOCTL_ASAHI_NEO_SUBMIT = DRM_IOCTL_ASAHI_NEO(W, SUBMIT, submit), - DRM_IOCTL_ASAHI_NEO_GEM_MADVISE = DRM_IOCTL_ASAHI_NEO(WR, GEM_MADVISE, gem_madvise), -}; - -#if defined(__cplusplus) -} -#endif - -#endif /* _ASAHI_NEO_DRM_H_ */ diff --git a/rust/kernel/drm_neo/device.rs b/rust/kernel/drm_neo/device.rs deleted file mode 100644 index 8786b91f2aad6c..00000000000000 --- a/rust/kernel/drm_neo/device.rs +++ /dev/null @@ -1,338 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM device. -//! -//! C header: [`include/drm_neo/drm_device.h`](srctree/include/drm_neo/drm_device.h) - -use crate::{ - alloc::allocator::Kmalloc, - bindings, device, - drm_neo::{ - self, - driver::AllocImpl, // - }, - error::from_err_ptr, - prelude::*, - sync::aref::{ - ARef, - AlwaysRefCounted, // - }, - types::Opaque, - workqueue::{ - HasDelayedWork, - HasWork, - Work, - WorkItem, // - }, -}; -use core::{ - alloc::Layout, - cell::UnsafeCell, - mem::{self, MaybeUninit}, - ops::Deref, - ptr::{ - self, - NonNull, // - }, - sync::atomic::{AtomicBool, Ordering}, -}; - -#[cfg(CONFIG_DRM_LEGACY)] -macro_rules! drm_legacy_fields { - ( $($field:ident: $val:expr),* $(,)? ) => { - bindings::drm_driver { - $( $field: $val ),*, - firstopen: None, - preclose: None, - dma_ioctl: None, - dma_quiescent: None, - context_dtor: None, - irq_handler: None, - irq_preinstall: None, - irq_postinstall: None, - irq_uninstall: None, - get_vblank_counter: None, - enable_vblank: None, - disable_vblank: None, - dev_priv_size: 0, - } - } -} - -#[cfg(not(CONFIG_DRM_LEGACY))] -macro_rules! drm_legacy_fields { - ( $($field:ident: $val:expr),* $(,)? ) => { - bindings::drm_driver { - $( $field: $val ),* - } - } -} - -/// A typed DRM device with a specific `drm_neo::Driver` implementation. -/// -/// The device is always reference-counted. -/// -/// # Invariants -/// -/// `self.dev` is a valid instance of a `struct device`. -/// When `data_initialized` is true, `data` owns initialized, pinned driver data. -/// Otherwise, driver data must not be accessed. -#[repr(C)] -pub struct Device { - dev: Opaque, - data: UnsafeCell>, - data_initialized: AtomicBool, -} - -impl Device { - const VTABLE: bindings::drm_driver = drm_legacy_fields! { - load: None, - open: Some(drm_neo::File::::open_callback), - postclose: Some(drm_neo::File::::postclose_callback), - unload: None, - release: Some(Self::release), - master_set: None, - master_drop: None, - debugfs_init: None, - gem_create_object: T::Object::ALLOC_OPS.gem_create_object, - prime_handle_to_fd: T::Object::ALLOC_OPS.prime_handle_to_fd, - prime_fd_to_handle: T::Object::ALLOC_OPS.prime_fd_to_handle, - gem_prime_import: T::Object::ALLOC_OPS.gem_prime_import, - gem_prime_import_sg_table: T::Object::ALLOC_OPS.gem_prime_import_sg_table, - dumb_create: T::Object::ALLOC_OPS.dumb_create, - dumb_map_offset: T::Object::ALLOC_OPS.dumb_map_offset, - show_fdinfo: None, - fbdev_probe: None, - - major: T::INFO.major, - minor: T::INFO.minor, - patchlevel: T::INFO.patchlevel, - name: crate::str::as_char_ptr_in_const_context(T::INFO.name).cast_mut(), - desc: crate::str::as_char_ptr_in_const_context(T::INFO.desc).cast_mut(), - - driver_features: T::FEATURES, - ioctls: T::IOCTLS.as_ptr(), - num_ioctls: T::IOCTLS.len() as i32, - fops: &Self::GEM_FOPS, - }; - - const ALLOC_VTABLE: bindings::drm_driver = bindings::drm_driver { - release: None, - ..Self::VTABLE - }; - - const GEM_FOPS: bindings::file_operations = drm_neo::gem::create_fops(T::MODULE); - - /// Create a new `drm_neo::Device` for a `drm_neo::Driver`. - pub fn new(dev: &device::Device, data: impl PinInit) -> Result> { - // SAFETY: The device remains private until its data is initialized below. - let drm_neo = unsafe { Self::new_uninit(dev)? }; - // SAFETY: This is the only initializer, and no references to the data exist yet. - unsafe { drm_neo.init_data(data)? }; - Ok(drm_neo) - } - - /// Allocate a DRM device whose driver data will be initialized separately. - /// - /// # Safety - /// - /// The caller must not access driver data, publish the device to userspace, or - /// schedule work that accesses driver data before calling [`Self::init_data`]. - pub unsafe fn new_uninit(dev: &device::Device) -> Result> { - let layout = Kmalloc::aligned_layout(Layout::new::()); - // Allocation failure must not inspect fields that we have not initialized yet. - let raw_drm: *mut Self = unsafe { - bindings::__drm_dev_alloc( - dev.as_raw(), - const { &Self::ALLOC_VTABLE }, - layout.size(), - mem::offset_of!(Self, dev), - ) - } - .cast(); - let raw_drm = NonNull::new(from_err_ptr(raw_drm)?).ok_or(ENOMEM)?; - // SAFETY: Allocation succeeded, and these fields have no previous value. - unsafe { - ptr::addr_of_mut!((*raw_drm.as_ptr()).data) - .write(UnsafeCell::new(MaybeUninit::uninit())); - ptr::addr_of_mut!((*raw_drm.as_ptr()).data_initialized).write(AtomicBool::new(false)); - } - // SAFETY: The allocation is private and its release state is now initialized. - let drm_dev = unsafe { Self::into_drm_device(raw_drm) }; - unsafe { (*drm_dev).driver = const { &Self::VTABLE } }; - // SAFETY: We own the initial DRM reference; uninitialized data is behind UnsafeCell. - Ok(unsafe { ARef::from_raw(raw_drm) }) - } - - /// Initialize the pinned driver data of a device allocated by [`Self::new_uninit`]. - /// - /// # Safety - /// - /// The caller must be the only initializer, the data must be uninitialized, and - /// no driver-data users may run until this method returns successfully. - pub unsafe fn init_data(&self, data: impl PinInit) -> Result { - let raw_data = self.data.get().cast::(); - // SAFETY: The caller guarantees exclusive initialization of this pinned slot. - unsafe { data.__pinned_init(raw_data) }?; - // Publish destructor ownership only after the real initializer succeeded. - self.data_initialized.store(true, Ordering::Release); - Ok(()) - } - - pub(crate) fn as_raw(&self) -> *mut bindings::drm_device { - self.dev.get() - } - - /// # Safety - /// - /// `ptr` must be a valid pointer to a `struct device` embedded in `Self`. - unsafe fn from_drm_device(ptr: *const bindings::drm_device) -> *mut Self { - // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a - // `struct drm_device` embedded in `Self`. - unsafe { crate::container_of!(Opaque::cast_from(ptr), Self, dev) }.cast_mut() - } - - /// # Safety - /// - /// `ptr` must be a valid pointer to `Self`. - unsafe fn into_drm_device(ptr: NonNull) -> *mut bindings::drm_device { - // SAFETY: By the safety requirements of this function, `ptr` is a valid pointer to `Self`. - unsafe { &raw mut (*ptr.as_ptr()).dev }.cast() - } - - /// Not intended to be called externally, except via declare_drm_neo_ioctls!() - /// - /// # Safety - /// - /// Callers must ensure that `ptr` is valid, non-null, and has a non-zero reference count, - /// i.e. it must be ensured that the reference count of the C `struct drm_device` `ptr` points - /// to can't drop to zero, for the duration of this function call and the entire duration when - /// the returned reference exists. - /// - /// Additionally, callers must ensure that the `struct device`, `ptr` is pointing to, is - /// embedded in `Self`. - #[doc(hidden)] - pub unsafe fn from_raw<'a>(ptr: *const bindings::drm_device) -> &'a Self { - // SAFETY: By the safety requirements of this function `ptr` is a valid pointer to a - // `struct drm_device` embedded in `Self`. - let ptr = unsafe { Self::from_drm_device(ptr) }; - - // SAFETY: `ptr` is valid by the safety requirements of this function. - unsafe { &*ptr.cast() } - } - - extern "C" fn release(ptr: *mut bindings::drm_device) { - // SAFETY: `ptr` is a valid pointer to a `struct drm_device` and embedded in `Self`. - let this = unsafe { Self::from_drm_device(ptr) }; - - // SAFETY: new_uninit() initialized this flag before enabling this callback. - // A failed initializer unwinds its own partial fields and never sets the flag. - if unsafe { (*this).data_initialized.load(Ordering::Acquire) } { - // SAFETY: Successful initialization published ownership of pinned driver data. - // DRM still owns the embedded C device and its final managed cleanup/kfree. - unsafe { ptr::drop_in_place((*this).data.get().cast::()) }; - } - } -} - -impl Deref for Device { - type Target = T::Data; - - fn deref(&self) -> &Self::Target { - debug_assert!(self.data_initialized.load(Ordering::Acquire)); - // SAFETY: Safe constructors initialize data; new_uninit() requires callers - // to prevent data access until init_data() succeeds. - unsafe { &*self.data.get().cast::() } - } -} - -// SAFETY: DRM device objects are always reference counted and the get/put functions -// satisfy the requirements. -unsafe impl AlwaysRefCounted for Device { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is non-zero. - unsafe { bindings::drm_dev_get(self.as_raw()) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: `obj` is a valid pointer to `Self`. - let drm_dev = unsafe { Self::into_drm_device(obj) }; - - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_dev_put(drm_dev) }; - } -} - -impl AsRef for Device { - fn as_ref(&self) -> &device::Device { - // SAFETY: `bindings::drm_device::dev` is valid as long as the DRM device itself is valid, - // which is guaranteed by the type invariant. - unsafe { device::Device::from_raw((*self.as_raw()).dev) } - } -} - -// SAFETY: A `drm_neo::Device` can be released from any thread. -unsafe impl Send for Device {} - -// SAFETY: A `drm_neo::Device` can be shared among threads because all immutable methods are protected -// by the synchronization in `struct drm_device`. -unsafe impl Sync for Device {} - -impl WorkItem for Device -where - T: drm_neo::Driver, - T::Data: WorkItem>>, - T::Data: HasWork, ID>, -{ - type Pointer = ARef>; - - fn run(ptr: ARef>) { - T::Data::run(ptr); - } -} - -// SAFETY: -// -// - `raw_get_work` and `work_container_of` return valid pointers by relying on -// `T::Data::raw_get_work` and `container_of`. In particular, `T::Data` is -// stored inline in `drm_neo::Device`, so the `container_of` call is valid. -// -// - The two methods are true inverses of each other: given `ptr: *mut -// Device`, `raw_get_work` will return a `*mut Work, ID>` through -// `T::Data::raw_get_work` and given a `ptr: *mut Work, ID>`, -// `work_container_of` will return a `*mut Device` through `container_of`. -unsafe impl HasWork, ID> for Device -where - T: drm_neo::Driver, - T::Data: HasWork, ID>, -{ - unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work, ID> { - // SAFETY: The caller promises that `ptr` points to a valid `Device`. - let data_ptr = unsafe { (&raw mut (*ptr).data).cast::() }; - - // SAFETY: `data_ptr` is a valid pointer to `T::Data`. - unsafe { T::Data::raw_get_work(data_ptr) } - } - - unsafe fn work_container_of(ptr: *mut Work, ID>) -> *mut Self { - // SAFETY: The caller promises that `ptr` points at a `Work` field in - // `T::Data`. - let data_ptr = unsafe { T::Data::work_container_of(ptr) } - .cast::>>(); - - // SAFETY: `T::Data` is stored as the `data` field in `Device`. - unsafe { crate::container_of!(data_ptr, Self, data) } - } -} - -// SAFETY: Our `HasWork` implementation returns a `work_struct` that is -// stored in the `work` field of a `delayed_work` with the same access rules as -// the `work_struct` owing to the bound on `T::Data: HasDelayedWork, -// ID>`, which requires that `T::Data::raw_get_work` return a `work_struct` that -// is inside a `delayed_work`. -unsafe impl HasDelayedWork, ID> for Device -where - T: drm_neo::Driver, - T::Data: HasDelayedWork, ID>, -{ -} diff --git a/rust/kernel/drm_neo/driver.rs b/rust/kernel/drm_neo/driver.rs deleted file mode 100644 index 1e0838f78ea7d9..00000000000000 --- a/rust/kernel/drm_neo/driver.rs +++ /dev/null @@ -1,188 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM driver core. -//! -//! C header: [`include/drm_neo/drm_drv.h`](srctree/include/drm_neo/drm_drv.h) - -use crate::{ - bindings, - device, - devres, - drm_neo, - error::to_result, - prelude::*, - sync::aref::ARef, // -}; - -/// Driver use the GEM memory manager. This should be set for all modern drivers. -pub const FEAT_GEM: u32 = bindings::drm_driver_feature_DRIVER_GEM; -/// Driver supports dedicated render nodes. -pub const FEAT_RENDER: u32 = bindings::drm_driver_feature_DRIVER_RENDER; -/// Driver supports DRM sync objects for explicit synchronization of command submission. -pub const FEAT_SYNCOBJ: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ; -/// Driver supports the timeline flavor of DRM sync objects for explicit synchronization of command -/// submission. -pub const FEAT_SYNCOBJ_TIMELINE: u32 = bindings::drm_driver_feature_DRIVER_SYNCOBJ_TIMELINE; -/// Driver supports user defined GPU VA bindings for GEM objects. -pub const FEAT_GEM_GPUVA: u32 = bindings::drm_driver_feature_DRIVER_GEM_GPUVA; - -/// Information data for a DRM Driver. -pub struct DriverInfo { - /// Driver major version. - pub major: i32, - /// Driver minor version. - pub minor: i32, - /// Driver patchlevel version. - pub patchlevel: i32, - /// Driver name. - pub name: &'static CStr, - /// Driver description. - pub desc: &'static CStr, -} - -/// Internal memory management operation set, normally created by memory managers (e.g. GEM). -pub struct AllocOps { - pub(crate) gem_create_object: Option< - unsafe extern "C" fn( - dev: *mut bindings::drm_device, - size: usize, - ) -> *mut bindings::drm_gem_object, - >, - pub(crate) prime_handle_to_fd: Option< - unsafe extern "C" fn( - dev: *mut bindings::drm_device, - file_priv: *mut bindings::drm_file, - handle: u32, - flags: u32, - prime_fd: *mut core::ffi::c_int, - ) -> core::ffi::c_int, - >, - pub(crate) prime_fd_to_handle: Option< - unsafe extern "C" fn( - dev: *mut bindings::drm_device, - file_priv: *mut bindings::drm_file, - prime_fd: core::ffi::c_int, - handle: *mut u32, - ) -> core::ffi::c_int, - >, - pub(crate) gem_prime_import: Option< - unsafe extern "C" fn( - dev: *mut bindings::drm_device, - dma_buf: *mut bindings::dma_buf, - ) -> *mut bindings::drm_gem_object, - >, - pub(crate) gem_prime_import_sg_table: Option< - unsafe extern "C" fn( - dev: *mut bindings::drm_device, - attach: *mut bindings::dma_buf_attachment, - sgt: *mut bindings::sg_table, - ) -> *mut bindings::drm_gem_object, - >, - pub(crate) dumb_create: Option< - unsafe extern "C" fn( - file_priv: *mut bindings::drm_file, - dev: *mut bindings::drm_device, - args: *mut bindings::drm_mode_create_dumb, - ) -> core::ffi::c_int, - >, - pub(crate) dumb_map_offset: Option< - unsafe extern "C" fn( - file_priv: *mut bindings::drm_file, - dev: *mut bindings::drm_device, - handle: u32, - offset: *mut u64, - ) -> core::ffi::c_int, - >, -} - -/// Trait for memory manager implementations. Implemented internally. -pub trait AllocImpl: super::private::Sealed + drm_neo::gem::IntoGEMObject { - /// The [`Driver`] implementation for this [`AllocImpl`]. - type Driver: drm_neo::Driver; - - /// The C callback operations for this memory manager. - const ALLOC_OPS: AllocOps; -} - -/// The DRM `Driver` trait. -/// -/// This trait must be implemented by drivers in order to create a `struct drm_device` and `struct -/// drm_driver` to be registered in the DRM subsystem. -#[vtable] -pub trait Driver { - /// Context data associated with the DRM driver - type Data: Sync + Send; - - /// The type used to manage memory for this driver. - type Object: AllocImpl; - - /// The type used to represent a DRM File (client) - type File: drm_neo::file::DriverFile; - - /// Driver metadata - const INFO: DriverInfo; - - /// Module implementing the driver callbacks. Loadable drivers should provide - /// their `THIS_MODULE` so open files and exported DMA buffers retain the code. - const MODULE: Option<&'static crate::ThisModule> = None; - - /// Feature flags - const FEATURES: u32; - - /// IOCTL list. See `kernel::drm_neo::ioctl::declare_drm_neo_ioctls!{}`. - const IOCTLS: &'static [drm_neo::ioctl::DrmIoctlDescriptor]; -} - -/// The registration type of a `drm_neo::Device`. -/// -/// Once the `Registration` structure is dropped, the device is unregistered. -pub struct Registration(ARef>); - -impl Registration { - fn new(drm_neo: &drm_neo::Device, flags: usize) -> Result { - // SAFETY: `drm_neo.as_raw()` is valid by the invariants of `drm_neo::Device`. - to_result(unsafe { bindings::drm_dev_register(drm_neo.as_raw(), flags) })?; - - Ok(Self(drm_neo.into())) - } - - /// Registers a new [`Device`](drm_neo::Device) with userspace. - /// - /// Ownership of the [`Registration`] object is passed to [`devres::register`]. - pub fn new_foreign_owned( - drm_neo: &drm_neo::Device, - dev: &device::Device, - flags: usize, - ) -> Result - where - T: 'static, - { - if drm_neo.as_ref().as_raw() != dev.as_raw() { - return Err(EINVAL); - } - - let reg = Registration::::new(drm_neo, flags)?; - - devres::register(dev, reg, GFP_KERNEL) - } - - /// Returns a reference to the `Device` instance for this registration. - pub fn device(&self) -> &drm_neo::Device { - &self.0 - } -} - -// SAFETY: `Registration` doesn't offer any methods or access to fields when shared between -// threads, hence it's safe to share it. -unsafe impl Sync for Registration {} - -// SAFETY: Registration with and unregistration from the DRM subsystem can happen from any thread. -unsafe impl Send for Registration {} - -impl Drop for Registration { - fn drop(&mut self) { - // SAFETY: Safe by the invariant of `ARef>`. The existence of this - // `Registration` also guarantees the this `drm_neo::Device` is actually registered. - unsafe { bindings::drm_dev_unregister(self.0.as_raw()) }; - } -} diff --git a/rust/kernel/drm_neo/file.rs b/rust/kernel/drm_neo/file.rs deleted file mode 100644 index 9bb81f21556f0b..00000000000000 --- a/rust/kernel/drm_neo/file.rs +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM File objects. -//! -//! C header: [`include/drm_neo/drm_file.h`](srctree/include/drm_neo/drm_file.h) - -use crate::{ - bindings, - drm_neo, - prelude::*, - types::Opaque, // -}; -use core::marker::PhantomData; - -/// Trait that must be implemented by DRM drivers to represent a DRM File (a client instance). -pub trait DriverFile { - /// The parent `Driver` implementation for this `DriverFile`. - type Driver: drm_neo::Driver; - - /// Open a new file (called when a client opens the DRM device). - fn open(device: &drm_neo::Device) -> Result>>; - - /// Get raw drm_file pointer - fn as_raw(&self) -> *mut bindings::drm_file; -} - -/// An open DRM File. -/// -/// # Invariants -/// -/// `self.0` is a valid instance of a `struct drm_file`. -#[repr(transparent)] -pub struct File(Opaque, PhantomData); - -impl File { - #[doc(hidden)] - /// Not intended to be called externally, except via declare_drm_neo_ioctls!() - /// - /// # Safety - /// - /// `raw_file` must be a valid pointer to an open `struct drm_file`, opened through `T::open`. - pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_file) -> &'a File { - // SAFETY: `raw_file` is valid by the safety requirements of this function. - unsafe { &*ptr.cast() } - } - - pub(super) fn as_raw(&self) -> *mut bindings::drm_file { - self.0.get() - } - - /// Tag recent feeder work for temporary capacity hints during syncobj waits. - /// User scheduling attributes and CPU affinity remain unchanged. - pub fn set_syncobj_wait_hint(&self, util_min: u32, duration_ms: u32) { - // SAFETY: this File keeps the opened drm_file alive; the C helper - // bounds both arguments and serializes the per-file hint internally. - unsafe { bindings::drm_syncobj_set_wait_hint(self.as_raw(), util_min, duration_ms) }; - } - - fn driver_priv(&self) -> *mut T { - // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. - unsafe { (*self.as_raw()).driver_priv }.cast() - } - - /// Return a pinned reference to the driver file structure. - pub fn inner(&self) -> Pin<&T> { - // SAFETY: By the type invariant the pointer `self.as_raw()` points to a valid and opened - // `struct drm_file`, hence `driver_priv` has been properly initialized by `open_callback`. - unsafe { Pin::new_unchecked(&*(self.driver_priv())) } - } - - /// The open callback of a `struct drm_file`. - pub(crate) extern "C" fn open_callback( - raw_dev: *mut bindings::drm_device, - raw_file: *mut bindings::drm_file, - ) -> core::ffi::c_int { - // SAFETY: A callback from `struct drm_driver::open` guarantees that - // - `raw_dev` is valid pointer to a `struct drm_device`, - // - the corresponding `struct drm_device` has been registered. - let drm_neo = unsafe { drm_neo::Device::from_raw(raw_dev) }; - - // SAFETY: `raw_file` is a valid pointer to a `struct drm_file`. - let file = unsafe { File::::from_raw(raw_file) }; - - let inner = match T::open(drm_neo) { - Err(e) => { - return e.to_errno(); - } - Ok(i) => i, - }; - - // SAFETY: This pointer is treated as pinned, and the Drop guarantee is upheld in - // `postclose_callback()`. - let driver_priv = KBox::into_raw(unsafe { Pin::into_inner_unchecked(inner) }); - - // SAFETY: By the type invariants of `Self`, `self.as_raw()` is always valid. - unsafe { (*file.as_raw()).driver_priv = driver_priv.cast() }; - - 0 - } - - /// The postclose callback of a `struct drm_file`. - pub(crate) extern "C" fn postclose_callback( - _raw_dev: *mut bindings::drm_device, - raw_file: *mut bindings::drm_file, - ) { - // SAFETY: This reference won't escape this function - let file = unsafe { File::::from_raw(raw_file) }; - - // SAFETY: `file.driver_priv` has been created in `open_callback` through `KBox::into_raw`. - let _ = unsafe { KBox::from_raw(file.driver_priv()) }; - } -} - -impl super::private::Sealed for File {} diff --git a/rust/kernel/drm_neo/gem/mod.rs b/rust/kernel/drm_neo/gem/mod.rs deleted file mode 100644 index bf986fe3ece6f0..00000000000000 --- a/rust/kernel/drm_neo/gem/mod.rs +++ /dev/null @@ -1,497 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM GEM API -//! -//! C header: [`include/drm_neo/drm_gem.h`](srctree/include/drm_neo/drm_gem.h) - -use crate::{ - bindings, - dma_buf, - drm_neo::{ - self, - driver::{ - AllocImpl, - AllocOps, // - }, - }, - error::{ - from_err_ptr, - to_result, // - }, - prelude::*, - sync::aref::{ - ARef, - AlwaysRefCounted, // - }, - types::Opaque, // -}; -use core::{ - marker::PhantomData, - ops::Deref, - ptr::NonNull, // -}; - -#[cfg(CONFIG_RUST_DRM_GEM_SHMEM_HELPER)] -pub mod shmem; - -/// A macro for implementing [`AlwaysRefCounted`] for any GEM object type. -/// -/// Since all GEM objects use the same refcounting scheme. -#[macro_export] -macro_rules! impl_aref_for_neo_gem_obj { - ( - impl $( <$( $tparam_id:ident ),+> )? for $type:ty - $( - where - $( $bind_param:path : $bind_trait:path ),+ - )? - ) => { - // SAFETY: All GEM objects are refcounted. - unsafe impl $( <$( $tparam_id ),+> )? $crate::sync::aref::AlwaysRefCounted for $type - where - Self: IntoGEMObject, - $( $( $bind_param : $bind_trait ),+ )? - { - fn inc_ref(&self) { - // SAFETY: The existence of a shared reference guarantees that the refcount is - // non-zero. - unsafe { bindings::drm_gem_object_get(self.as_raw()) }; - } - - unsafe fn dec_ref(obj: core::ptr::NonNull) { - // SAFETY: `obj` is a valid pointer to an `Object`. - let obj = unsafe { obj.as_ref() }.as_raw(); - - // SAFETY: The safety requirements guarantee that the refcount is non-zero. - unsafe { bindings::drm_gem_object_put(obj) }; - } - } - }; -} -#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), allow(unused))] -pub(crate) use impl_aref_for_neo_gem_obj; - -/// A type alias for retrieving a [`Driver`]s [`DriverFile`] implementation from its -/// [`DriverObject`] implementation. -/// -/// [`Driver`]: drm_neo::Driver -/// [`DriverFile`]: drm_neo::file::DriverFile -pub type DriverFile = drm_neo::File<<::Driver as drm_neo::Driver>::File>; - -/// GEM object functions, which must be implemented by drivers. -#[vtable] -pub trait DriverObject: Sync + Send + Sized { - /// Parent `Driver` for this object. - type Driver: drm_neo::Driver; - - /// The data type to use for passing arguments to [`DriverObject::new`]. - type Args; - - /// Create a new driver data object for a GEM object of a given size. - fn new( - dev: &drm_neo::Device, - size: usize, - args: Self::Args, - ) -> impl PinInit; - - /// Open a new handle to an existing object, associated with a File. - fn open(_obj: &::Object, _file: &DriverFile) -> Result { - Ok(()) - } - - /// Close a handle to an existing object, associated with a File. - fn close(_obj: &::Object, _file: &DriverFile) {} - - /// Optional handle for exporting a gem object. - fn export( - _obj: &::Object, - _flags: u32, - ) -> Result::Object>> { - unimplemented!() - } -} - -/// Trait that represents a GEM object subtype -pub trait IntoGEMObject: Sized + super::private::Sealed + AlwaysRefCounted { - /// Returns a reference to the raw `drm_gem_object` structure, which must be valid as long as - /// this owning object is valid. - fn as_raw(&self) -> *mut bindings::drm_gem_object; - - /// Converts a pointer to a `struct drm_gem_object` into a reference to `Self`. - /// - /// # Safety - /// - /// - `self_ptr` must be a valid pointer to `Self`. - /// - The caller promises that holding the immutable reference returned by this function does - /// not violate rust's data aliasing rules and remains valid throughout the lifetime of `'a`. - unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self; -} - -extern "C" fn open_callback( - raw_obj: *mut bindings::drm_gem_object, - raw_file: *mut bindings::drm_file, -) -> core::ffi::c_int { - // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. - let file = unsafe { DriverFile::::from_raw(raw_file) }; - - // SAFETY: `open_callback` is specified in the AllocOps structure for `DriverObject`, - // ensuring that `raw_obj` is contained within a `DriverObject` - let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; - - match T::open(obj, file) { - Err(e) => e.to_errno(), - Ok(()) => 0, - } -} - -extern "C" fn close_callback( - raw_obj: *mut bindings::drm_gem_object, - raw_file: *mut bindings::drm_file, -) { - // SAFETY: `open_callback` is only ever called with a valid pointer to a `struct drm_file`. - let file = unsafe { DriverFile::::from_raw(raw_file) }; - - // SAFETY: `close_callback` is specified in the AllocOps structure for `Object`, ensuring - // that `raw_obj` is indeed contained within a `Object`. - let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; - - T::close(obj, file); -} - -extern "C" fn export_callback( - raw_obj: *mut bindings::drm_gem_object, - flags: i32, -) -> *mut bindings::dma_buf { - // SAFETY: `export_callback` is specified in the AllocOps structure for `Object`, ensuring - // that `raw_obj` is contained within a `Object`. - let obj = unsafe { <::Object as IntoGEMObject>::from_raw(raw_obj) }; - - match T::export(obj, flags as _) { - // DRM takes a hold of the reference - Ok(buf) => buf.into_raw(), - Err(e) => e.to_ptr(), - } -} - -impl IntoGEMObject for Object { - fn as_raw(&self) -> *mut bindings::drm_gem_object { - self.obj.get() - } - - unsafe fn from_raw<'a>(self_ptr: *mut bindings::drm_gem_object) -> &'a Self { - // SAFETY: `obj` is guaranteed to be in an `Object` via the safety contract of this - // function - unsafe { &*crate::container_of!(Opaque::cast_from(self_ptr), Object, obj) } - } -} - -/// Base operations shared by all GEM object classes -pub trait BaseObject: IntoGEMObject { - /// Returns the size of the object in bytes. - fn size(&self) -> usize { - // SAFETY: `self.as_raw()` is guaranteed to be a pointer to a valid `struct drm_gem_object`. - unsafe { (*self.as_raw()).size } - } - - /// Creates a new handle for the object associated with a given `File` - /// (or returns an existing one). - fn create_handle(&self, file: &drm_neo::File) -> Result - where - Self: AllocImpl, - D: drm_neo::Driver, - F: drm_neo::file::DriverFile, - { - let mut handle: u32 = 0; - // SAFETY: The arguments are all valid per the type invariants. - to_result(unsafe { - bindings::drm_gem_handle_create(file.as_raw().cast(), self.as_raw(), &mut handle) - })?; - Ok(handle) - } - - /// Looks up an object by its handle for a given `File`. - fn lookup_handle(file: &drm_neo::File, handle: u32) -> Result> - where - Self: AllocImpl, - D: drm_neo::Driver, - F: drm_neo::file::DriverFile, - { - // SAFETY: The arguments are all valid per the type invariants. - let ptr = unsafe { bindings::drm_gem_object_lookup(file.as_raw().cast(), handle) }; - if ptr.is_null() { - return Err(ENOENT); - } - - // SAFETY: - // - A `drm_neo::Driver` can only have a single `File` implementation. - // - `file` uses the same `drm_neo::Driver` as `Self`. - // - Therefore, we're guaranteed that `ptr` must be a gem object embedded within `Self`. - // - And we check if the pointer is null befoe calling from_raw(), ensuring that `ptr` is a - // valid pointer to an initialized `Self`. - let obj = unsafe { Self::from_raw(ptr) }; - - // SAFETY: - // - We take ownership of the reference of `drm_gem_object_lookup()`. - // - Our `NonNull` comes from an immutable reference, thus ensuring it is a valid pointer to - // `Self`. - Ok(unsafe { ARef::from_raw(obj.into()) }) - } - - /// Export a [`DmaBuf`] for this GEM object using the DRM prime helper library. - /// - /// `flags` should be a set of flags from [`fs::file::flags`](kernel::fs::file::flags). - fn prime_export(&self, flags: u32) -> Result> { - // SAFETY: - // - `as_raw()` always returns a valid pointer to a `drm_gem_object`. - // - `drm_gem_prime_export()` returns either an error pointer, or a valid pointer to an - // initialized `dma_buf` on success. - let dma_ptr = - from_err_ptr(unsafe { bindings::drm_gem_prime_export(self.as_raw(), flags as _) })?; - - // SAFETY: - // - We checked that dma_ptr is not an error, so it must point to an initialized dma_buf - // - We used drm_gem_prime_export(), so `dma_ptr` will remain valid until a call to - // `drm_gem_prime_release()` which we don't call here. - let dma_buf = unsafe { dma_buf::DmaBuf::as_ref(dma_ptr) }; - - // INVARIANT: We used drm_gem_prime_export() to create this dma_buf, fulfilling the - // invariant that this dma_buf came from a GEM object of type `Self`. - Ok(DmaBuf(dma_buf.into(), PhantomData)) - } - - /// Creates an mmap offset to map the object from userspace. - fn create_mmap_offset(&self) -> Result { - // SAFETY: The arguments are valid per the type invariant. - to_result(unsafe { bindings::drm_gem_create_mmap_offset(self.as_raw()) })?; - - // SAFETY: The arguments are valid per the type invariant. - Ok(unsafe { bindings::drm_vma_node_offset_addr(&raw mut (*self.as_raw()).vma_node) }) - } - - /// Lock the gpuva lock - fn lock_gpuva(&self) { - unsafe { - bindings::mutex_lock(&raw mut (*self.as_raw()).gpuva.lock); - } - } - - /// Lock the gpuva lock - fn unlock_gpuva(&self) { - unsafe { - bindings::mutex_unlock(&raw mut (*self.as_raw()).gpuva.lock); - } - } -} - -impl BaseObject for T {} - -/// Crate-private base operations shared by all GEM object classes. -#[cfg_attr(not(CONFIG_RUST_DRM_GEM_SHMEM_HELPER), expect(unused))] -pub(crate) trait BaseObjectPrivate: IntoGEMObject { - /// Return a pointer to this object's dma_resv. - fn raw_dma_resv(&self) -> *mut bindings::dma_resv { - // SAFETY: `self.as_raw()` always returns a valid pointer to the base DRM GEM object. - unsafe { (*self.as_raw()).resv } - } -} - -impl BaseObjectPrivate for T {} - -/// A base GEM object. -/// -/// # Invariants -/// -/// - `self.obj` is a valid instance of a `struct drm_gem_object`. -#[repr(C)] -#[pin_data] -pub struct Object { - obj: Opaque, - #[pin] - data: T, -} - -impl Object { - const OBJECT_FUNCS: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { - free: Some(Self::free_callback), - open: Some(open_callback::), - close: Some(close_callback::), - print_info: None, - export: if T::HAS_EXPORT { - Some(export_callback::) - } else { - None - }, - pin: None, - unpin: None, - get_sg_table: None, - vmap: None, - vunmap: None, - mmap: None, - status: None, - vm_ops: core::ptr::null_mut(), - evict: None, - rss: None, - }; - - /// Create a new GEM object. - pub fn new(dev: &drm_neo::Device, size: usize, args: T::Args) -> Result> { - let obj: Pin> = KBox::pin_init( - try_pin_init!(Self { - obj: Opaque::new(bindings::drm_gem_object::default()), - data <- T::new(dev, size, args), - }), - GFP_KERNEL, - )?; - - // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. - unsafe { (*obj.as_raw()).funcs = &Self::OBJECT_FUNCS }; - - if let Err(err) = - // SAFETY: The arguments are all valid per the type invariants. - to_result(unsafe { - bindings::drm_gem_object_init(dev.as_raw(), obj.obj.get(), size) - }) - { - // SAFETY: `drm_gem_object_init()` initializes the private GEM object state before - // failing, so `drm_gem_private_object_fini()` is the matching cleanup. - unsafe { bindings::drm_gem_private_object_fini(obj.obj.get()) }; - return Err(err); - } - - // SAFETY: We will never move out of `Self` as `ARef` is always treated as pinned. - let ptr = KBox::into_raw(unsafe { Pin::into_inner_unchecked(obj) }); - - // SAFETY: `ptr` comes from `KBox::into_raw` and hence can't be NULL. - let ptr = unsafe { NonNull::new_unchecked(ptr) }; - - // SAFETY: We take over the initial reference count from `drm_gem_object_init()`. - Ok(unsafe { ARef::from_raw(ptr) }) - } - - /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &drm_neo::Device { - // SAFETY: - // - `struct drm_gem_object.dev` is initialized and valid for as long as the GEM - // object lives. - // - The device we used for creating the gem object is passed as &drm_neo::Device to - // Object::::new(), so we know that `T::Driver` is the right generic parameter to use - // here. - unsafe { drm_neo::Device::from_raw((*self.as_raw()).dev) } - } - - fn as_raw(&self) -> *mut bindings::drm_gem_object { - self.obj.get() - } - - extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { - let ptr: *mut Opaque = obj.cast(); - - // SAFETY: All of our objects are of type `Object`. - let this = unsafe { crate::container_of!(ptr, Self, obj) }; - - // SAFETY: The C code only ever calls this callback with a valid pointer to a `struct - // drm_gem_object`. - unsafe { bindings::drm_gem_object_release(obj) }; - - // SAFETY: All of our objects are allocated via `KBox`, and we're in the - // free callback which guarantees this object has zero remaining references, - // so we can drop it. - let _ = unsafe { KBox::from_raw(this) }; - } -} - -impl_aref_for_neo_gem_obj!(impl for Object where T: DriverObject); - -impl super::private::Sealed for Object {} - -impl Deref for Object { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.data - } -} - -impl AllocImpl for Object { - type Driver = T::Driver; - - const ALLOC_OPS: AllocOps = AllocOps { - gem_create_object: None, - prime_handle_to_fd: None, - prime_fd_to_handle: None, - gem_prime_import: None, - gem_prime_import_sg_table: None, - dumb_create: None, - dumb_map_offset: None, - }; -} - -/// A [`dma_buf::DmaBuf`] which has been exported from a GEM object. -/// -/// The [`dma_buf::DmaBuf`] will be released when this type is dropped. -/// -/// # Invariants -/// -/// - `self.0` points to a valid initialized [`dma_buf::DmaBuf`] for the lifetime of this object. -/// - The GEM object from which this [`dma_buf::DmaBuf`] was exported from is guaranteed to be of -/// type `T`. -pub struct DmaBuf(NonNull, PhantomData); - -impl Deref for DmaBuf { - type Target = dma_buf::DmaBuf; - - #[inline] - fn deref(&self) -> &Self::Target { - // SAFETY: This pointer is guaranteed to be valid by our type invariants. - unsafe { self.0.as_ref() } - } -} - -impl Drop for DmaBuf { - #[inline] - fn drop(&mut self) { - // SAFETY: - // - `dma_buf::DmaBuf` is guaranteed to have an identical layout to `struct dma_buf` - // by its type invariants. - // - We hold the last reference to this `DmaBuf`, making it safe to destroy. - unsafe { bindings::drm_gem_dmabuf_release(self.0.cast().as_ptr()) } - } -} - -impl DmaBuf { - /// Leak the reference for this [`DmaBuf`] and return a raw pointer to it. - #[inline] - pub(crate) fn into_raw(self) -> *mut bindings::dma_buf { - let dma_ptr = self.as_raw(); - - core::mem::forget(self); - dma_ptr - } -} - -pub(super) const fn create_fops( - module: Option<&'static crate::ThisModule>, -) -> bindings::file_operations { - // SAFETY: As by the type invariant, it is safe to initialize `bindings::file_operations` - // zeroed. - let mut fops: bindings::file_operations = unsafe { core::mem::zeroed() }; - - fops.owner = match module { - Some(module) => module.as_ptr(), - None => core::ptr::null_mut(), - }; - fops.open = Some(bindings::drm_open); - fops.release = Some(bindings::drm_release); - fops.unlocked_ioctl = Some(bindings::drm_ioctl); - #[cfg(CONFIG_COMPAT)] - { - fops.compat_ioctl = Some(bindings::drm_compat_ioctl); - } - fops.poll = Some(bindings::drm_poll); - fops.read = Some(bindings::drm_read); - fops.llseek = Some(bindings::noop_llseek); - fops.mmap = Some(bindings::drm_gem_mmap); - fops.fop_flags = bindings::FOP_UNSIGNED_OFFSET; - - fops -} diff --git a/rust/kernel/drm_neo/gem/shmem.rs b/rust/kernel/drm_neo/gem/shmem.rs deleted file mode 100644 index 91e75ae396d9b6..00000000000000 --- a/rust/kernel/drm_neo/gem/shmem.rs +++ /dev/null @@ -1,584 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! DRM GEM shmem helper objects -//! -//! C header: [`include/linux/drm_neo/drm_gem_shmem_helper.h`](srctree/include/drm_neo/drm_gem_shmem_helper.h) - -// TODO: -// - There are a number of spots here that manually acquire/release the DMA reservation lock using -// dma_resv_(un)lock(). In the future we should add support for ww mutex, expose a method to -// acquire a reference to the WwMutex, and then use that directly instead of the C functions here. - -use crate::{ - container_of, - drm_neo::{ - device, - driver, - gem, - private::Sealed, // - }, - error::{ - from_err_ptr, - to_result, // - }, - iosys_map::*, - prelude::*, - scatterlist, - sync::aref::ARef, - transmute::*, - types::Opaque, // -}; -use core::{ - mem::{ - self, - MaybeUninit, // - }, - ops::{ - Deref, - DerefMut, // - }, - ptr::NonNull, - sync::atomic::{AtomicUsize, Ordering}, -}; -use gem::{ - BaseObject, - BaseObjectPrivate, - DriverObject, - IntoGEMObject, // -}; - -/// A struct for controlling the creation of shmem-backed GEM objects. -/// -/// This is used with [`Object::new()`] to control various properties that can only be set when -/// initially creating a shmem-backed GEM object. -#[derive(Default)] -pub struct ObjectConfig<'a, T: DriverObject> { - /// Whether to set the write-combine map flag. - pub map_wc: bool, - - /// Opt in to revocable CPU VMAs, with driver-controlled idle eviction. - pub reclaimable_cpu_mappings: bool, - - /// Reuse the DMA reservation from another GEM object. - /// - /// The newly created [`Object`] will hold an owned refcount to `parent_resv_obj` if specified. - pub parent_resv_obj: Option<&'a Object>, -} - -const SGT_PERMANENT: usize = 1 << (usize::BITS - 1); -const SGT_RECLAIMING: usize = 1 << (usize::BITS - 2); -const SGT_USERS: usize = SGT_RECLAIMING - 1; - -/// A shmem-backed GEM object. -/// -/// # Invariants -/// -/// `obj` contains a valid initialized `struct drm_gem_shmem_object` for the lifetime of this -/// object. -#[repr(C)] -#[pin_data] -pub struct Object { - #[pin] - obj: Opaque, - /// Parent object that owns this object's DMA reservation object. - parent_resv_obj: Option>>, - /// Owned SGT readers, or permanent borrowed access, exclude backing eviction. - sgt_access: AtomicUsize, - #[pin] - inner: T, -} - -super::impl_aref_for_neo_gem_obj!(impl for Object where T: DriverObject); - -// SAFETY: All GEM objects are thread-safe. -unsafe impl Send for Object {} - -// SAFETY: All GEM objects are thread-safe. -unsafe impl Sync for Object {} - -impl Object { - /// `drm_gem_object_funcs` vtable suitable for GEM shmem objects. - const VTABLE: bindings::drm_gem_object_funcs = bindings::drm_gem_object_funcs { - free: Some(Self::free_callback), - open: Some(super::open_callback::), - close: Some(super::close_callback::), - print_info: Some(bindings::drm_gem_shmem_object_print_info), - export: if T::HAS_EXPORT { - Some(super::export_callback::) - } else { - None - }, - pin: Some(bindings::drm_gem_shmem_object_pin), - unpin: Some(bindings::drm_gem_shmem_object_unpin), - get_sg_table: Some(bindings::drm_gem_shmem_object_get_sg_table), - vmap: Some(bindings::drm_gem_shmem_object_vmap), - vunmap: Some(bindings::drm_gem_shmem_object_vunmap), - mmap: Some(bindings::drm_gem_shmem_object_mmap), - status: None, - rss: None, - #[allow(unused_unsafe, reason = "Safe since Rust 1.82.0")] - // SAFETY: `drm_gem_shmem_vm_ops` is a valid, static const on the C side. - vm_ops: unsafe { &raw const bindings::drm_gem_shmem_vm_ops }, - evict: None, - }; - - /// Return a raw pointer to the embedded drm_gem_shmem_object. - fn as_raw_shmem(&self) -> *mut bindings::drm_gem_shmem_object { - self.obj.get() - } - - /// Create a new shmem-backed DRM object of the given size. - /// - /// Additional config options can be specified using `config`. - pub fn new( - dev: &device::Device, - size: usize, - config: ObjectConfig<'_, T>, - args: T::Args, - ) -> Result> { - let new: Pin> = KBox::try_pin_init( - try_pin_init!(Self { - obj <- Opaque::init_zeroed(), - parent_resv_obj: config.parent_resv_obj.map(|p| p.into()), - sgt_access: AtomicUsize::new(0), - inner <- T::new(dev, size, args), - }), - GFP_KERNEL, - )?; - - // SAFETY: `obj.as_raw()` is guaranteed to be valid by the initialization above. - unsafe { (*new.as_raw()).funcs = &Self::VTABLE }; - - // SAFETY: The arguments are all valid via the type invariants. - to_result(unsafe { bindings::drm_gem_shmem_init(dev.as_raw(), new.as_raw_shmem(), size) })?; - - // SAFETY: We never move out of `self`. - let new = KBox::into_raw(unsafe { Pin::into_inner_unchecked(new) }); - - // SAFETY: We're taking over the owned refcount from `drm_gem_shmem_init`. - let obj = unsafe { ARef::from_raw(NonNull::new_unchecked(new)) }; - - // Start filling out values from `config` - if let Some(parent_resv) = config.parent_resv_obj { - // SAFETY: We have yet to expose the new gem object outside of this function, so it is - // safe to modify this field. - unsafe { (*obj.obj.get()).base.resv = parent_resv.raw_dma_resv() }; - } - - // SAFETY: We have yet to expose this object outside of this function, so we're guaranteed - // to have exclusive access - thus making this safe to hold a mutable reference to. - let shmem = unsafe { &mut *obj.as_raw_shmem() }; - shmem.set_map_wc(config.map_wc); - shmem.reclaimable_cpu_mappings = config.reclaimable_cpu_mappings; - - Ok(obj) - } - - /// Returns the `Device` that owns this GEM object. - pub fn dev(&self) -> &device::Device { - // SAFETY: `dev` will have been initialized in `Self::new()` by `drm_gem_shmem_init()`. - unsafe { device::Device::from_raw((*self.as_raw()).dev) } - } - - extern "C" fn free_callback(obj: *mut bindings::drm_gem_object) { - // SAFETY: - // - DRM always passes a valid gem object here - // - We used drm_gem_shmem_create() in our create_gem_object callback, so we know that - // `obj` is contained within a drm_gem_shmem_object - let this = unsafe { container_of!(obj, bindings::drm_gem_shmem_object, base) }; - - // SAFETY: - // - We're in free_callback - so this function is safe to call. - // - We won't be using the gem resources on `this` after this call. - unsafe { bindings::drm_gem_shmem_release(this) }; - - // SAFETY: - // - We verified above that `obj` is valid, which makes `this` valid - // - This function is set in AllocOps, so we know that `this` is contained within a - // `Object` - let this = unsafe { container_of!(Opaque::cast_from(this), Self, obj) }.cast_mut(); - - // SAFETY: We're recovering the Kbox<> we created in gem_create_object() - let _ = unsafe { KBox::from_raw(this) }; - } - - /// Creates (if necessary) and returns an immutable reference to a scatter-gather table of DMA - /// pages for this object. - /// - /// This will pin the object in memory. - #[inline] - pub fn sg_table(&self) -> Result<&scatterlist::SGTable> { - // The borrowed reference can outlive any lock, so pin permanently. - self.acquire_sgt_access(true)?; - self.sg_table_resident() - } - - fn acquire_sgt_access(&self, permanent: bool) -> Result { - let mut state = self.sgt_access.load(Ordering::Acquire); - loop { - if state & SGT_RECLAIMING != 0 { - return Err(EBUSY); - } - let next = if permanent { - state | SGT_PERMANENT - } else { - if state & SGT_USERS == SGT_USERS { return Err(EOVERFLOW); } - state + 1 - }; - match self.sgt_access.compare_exchange_weak( - state, next, Ordering::AcqRel, Ordering::Acquire, - ) { - Ok(_) => return Ok(()), - Err(current) => state = current, - } - } - } - - fn sg_table_resident(&self) -> Result<&scatterlist::SGTable> { - // SAFETY: - // - drm_gem_shmem_get_pages_sgt is thread-safe. - // - drm_gem_shmem_get_pages_sgt returns either a valid pointer to a scatterlist, or an - // error pointer. - let sgt = from_err_ptr(unsafe { bindings::drm_gem_shmem_get_pages_sgt(self.as_raw_shmem()) })?; - - // SAFETY: We checked above that `sgt` is not an error pointer, so it must be a valid - // pointer to a scatterlist - Ok(unsafe { scatterlist::SGTable::from_raw(sgt) }) - } - - /// Creates (if necessary) and returns an owned reference to a scatter-gather table of DMA pages - /// for this object. - /// - /// This is the same as [`sg_table`](Self::sg_table), except that it instead returns an - /// [`shmem::SGTable`] which holds a reference to the associated gem object, instead of a - /// reference to an [`scatterlist::SGTable`]. - /// - /// This will pin the object in memory. - /// - /// [`shmem::SGTable`]: SGTable - pub fn owned_sg_table(&self) -> Result> { - self.acquire_sgt_access(false)?; - let sgt = match self.sg_table_resident() { - Ok(sgt) => sgt.into(), - Err(error) => { - self.sgt_access.fetch_sub(1, Ordering::Release); - return Err(error); - } - }; - Ok(SGTable { sgt, _owner: self.into() }) - } - - /// Whether the caller's sole SG lease is the last user of the backing. - /// - /// # Safety - /// The caller holds the object's DMA reservation lock. - pub unsafe fn sole_sg_lease_releasable_locked(&self) -> bool { - self.sgt_access.load(Ordering::Acquire) == 1 && - // SAFETY: The caller establishes the reservation-lock contract. - unsafe { bindings::drm_gem_shmem_idle_pages_releasable_locked(self.as_raw_shmem()) } - } - - /// Release idle backing without discarding contents. A later - /// `owned_sg_table()` may return different DMA addresses. - /// - /// # Safety - /// - /// The caller holds this object's DMA reservation lock, has removed and - /// invalidated every device mapping, and excludes new device access. - pub unsafe fn release_idle_pages_locked(&self) -> Result { - self.sgt_access.compare_exchange( - 0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire, - ).map_err(|_| EBUSY)?; - // SAFETY: The caller's contract plus the exclusive state above. - let result = to_result(unsafe { - bindings::drm_gem_shmem_release_idle_pages_locked(self.as_raw_shmem()) - }); - self.sgt_access.store(0, Ordering::Release); - result - } - - /// Apply purge advice (0 = needed, >0 = may be purged) under the - /// object's reservation lock. Returns false if the object was already - /// purged; the advice is then ignored. - pub fn madvise(&self, madv: i32) -> bool { - // SAFETY: The object retains its reservation; the C helper requires it held. - unsafe { - bindings::dma_resv_lock(self.raw_dma_resv(), core::ptr::null_mut()); - let retained = bindings::drm_gem_shmem_madvise_locked(self.as_raw_shmem(), madv) != 0; - bindings::dma_resv_unlock(self.raw_dma_resv()); - retained - } - } - - /// Current purge advice: negative once purged. - /// - /// # Safety - /// The caller holds the object's DMA reservation lock. - pub unsafe fn madv_locked(&self) -> i32 { - // SAFETY: Reservation ownership serializes the field. - unsafe { (*self.as_raw_shmem()).madv } - } - - /// Discard the backing of a purgeable object. The handle and logical - /// mappings remain; the contents are gone and `madv_locked()` turns negative. - /// - /// # Safety - /// - /// The caller holds this object's DMA reservation lock, has removed and - /// invalidated every device mapping, and no Rust SG lease remains. - pub unsafe fn purge_locked(&self) -> Result { - // SAFETY: The caller's contract; the exclusive state below excludes new leases. - if !unsafe { bindings::drm_gem_shmem_is_purgeable(self.as_raw_shmem()) } { - return Err(EBUSY); - } - self.sgt_access.compare_exchange( - 0, SGT_RECLAIMING, Ordering::AcqRel, Ordering::Acquire, - ).map_err(|_| EBUSY)?; - // SAFETY: See above. - unsafe { bindings::drm_gem_shmem_purge_locked(self.as_raw_shmem()) }; - self.sgt_access.store(0, Ordering::Release); - Ok(()) - } - - /// Attempt to create a [`RawIoSysMap`] from the gem object. - fn raw_vmap(&self) -> Result> { - build_assert!( - mem::size_of::() > 0, - "It doesn't make sense for the mapping type to be a ZST" - ); - - let mut map: MaybeUninit = MaybeUninit::uninit(); - - // SAFETY: drm_gem_shmem_vmap can be called with the DMA reservation lock held - to_result(unsafe { - // TODO: see top of file - bindings::dma_resv_lock(self.raw_dma_resv(), core::ptr::null_mut()); - let ret = bindings::drm_gem_shmem_vmap_locked(self.as_raw_shmem(), map.as_mut_ptr()); - bindings::dma_resv_unlock(self.raw_dma_resv()); - ret - })?; - - // SAFETY: if drm_gem_shmem_vmap did not fail, map is initialized now - Ok(unsafe { RawIoSysMap::from_raw(map.assume_init()) }) - } - - /// Unmap a [`RawIoSysMap`] from the gem object. - /// - /// # Safety - /// - /// - The caller promises that `map` came from a prior call to [`Self::raw_vmap`] on this gem - /// object. - /// - The caller promises that the memory pointed to by `map` will no longer be accesed through - /// this instance. - unsafe fn raw_vunmap(&self, map: &mut RawIoSysMap) { - let resv = self.raw_dma_resv(); - - // SAFETY: - // - This function is safe to call with the DMA reservation lock held - // - Our `ARef` is proof that the underlying gem object here is initialized and thus safe to - // dereference. - unsafe { - // TODO: see top of file - bindings::dma_resv_lock(resv, core::ptr::null_mut()); - bindings::drm_gem_shmem_vunmap_locked(self.as_raw_shmem(), map.as_raw_mut()); - bindings::dma_resv_unlock(resv); - } - } - - /// Creates and returns a virtual kernel memory mapping for this object. - pub fn vmap(&self) -> Result> { - let map = self.raw_vmap()?; - - Ok(VMapRef { - // SAFETY: - // - The size of the vmap is the same as the size of the gem - // - The vmap will remain alive until this object is dropped. - map: unsafe { IoSysMapRef::new(map, self.size()) }, - owner: self, - }) - } - - /// Creates and returns an owned reference to a virtual kernel memory mapping for this object. - pub fn owned_vmap(&self) -> Result> { - Ok(VMap { - map: self.raw_vmap()?, - owner: self.into(), - }) - } -} - -impl Deref for Object { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl DerefMut for Object { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} - -impl Sealed for Object {} - -impl gem::IntoGEMObject for Object { - fn as_raw(&self) -> *mut bindings::drm_gem_object { - // SAFETY: - // - Our immutable reference is proof that this is safe to dereference. - // - `obj` is always a valid drm_gem_shmem_object via our type invariants. - unsafe { &raw mut (*self.obj.get()).base } - } - - unsafe fn from_raw<'a>(obj: *mut bindings::drm_gem_object) -> &'a Object { - // SAFETY: The safety contract of from_gem_obj() guarantees that `obj` is contained within - // `Self` - unsafe { - let obj = Opaque::cast_from(container_of!(obj, bindings::drm_gem_shmem_object, base)); - - &*container_of!(obj, Object, obj) - } - } -} - -impl driver::AllocImpl for Object { - type Driver = T::Driver; - - const ALLOC_OPS: driver::AllocOps = driver::AllocOps { - gem_create_object: None, - prime_handle_to_fd: None, - prime_fd_to_handle: None, - gem_prime_import: None, - gem_prime_import_sg_table: Some(bindings::drm_gem_shmem_prime_import_sg_table), - dumb_create: Some(bindings::drm_gem_shmem_dumb_create), - dumb_map_offset: None, - }; -} - -/// A borrowed reference to a virtual mapping for a shmem-based GEM object in kernel address space. -pub struct VMapRef<'a, D: DriverObject, T: AsBytes + FromBytes> { - map: IoSysMapRef<'a, T>, - owner: &'a Object, -} - -impl<'a, D: DriverObject, T: AsBytes + FromBytes> Clone for VMapRef<'a, D, T> { - fn clone(&self) -> Self { - // SAFETY: We have a successful vmap already, so this can't fail - unsafe { self.owner.vmap().unwrap_unchecked() } - } -} - -impl<'a, D: DriverObject, T: AsBytes + FromBytes> Deref for VMapRef<'a, D, T> { - type Target = IoSysMapRef<'a, T>; - - fn deref(&self) -> &Self::Target { - &self.map - } -} - -impl<'a, D: DriverObject, T: AsBytes + FromBytes> DerefMut for VMapRef<'a, D, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.map - } -} - -impl<'a, D: DriverObject, T: AsBytes + FromBytes> Drop for VMapRef<'a, D, T> { - fn drop(&mut self) { - // SAFETY: Our existence is proof that this map was previously created using self.owner. - unsafe { self.owner.raw_vunmap(&mut self.map) }; - } -} - -/// An owned reference to a virtual mapping for a shmem-based GEM object in kernel address space. -/// -/// # Invariants -/// -/// - The memory pointed to by `map` is at least as large as `T`. -/// - The memory pointed to by `map` remains valid at least until this object is dropped. -pub struct VMap { - map: RawIoSysMap, - owner: ARef>, -} - -impl Clone for VMap { - fn clone(&self) -> Self { - // SAFETY: We have a successful vmap already, so this can't fail - unsafe { self.owner.owned_vmap().unwrap_unchecked() } - } -} - -impl<'a, D: DriverObject, T: AsBytes + FromBytes> From> for VMap { - fn from(value: VMapRef<'a, D, T>) -> Self { - let this = Self { - map: value.map.clone(), - owner: value.owner.into(), - }; - - mem::forget(value); - this - } -} - -impl VMap { - /// Return a reference to the iosys map for this `VMap`. - pub fn get(&self) -> IoSysMapRef<'_, T> { - // SAFETY: The size of the iosys_map is equivalent to the size of the gem object. - unsafe { IoSysMapRef::new(self.map.clone(), self.owner.size()) } - } - - /// Borrows a reference to the object that owns this virtual mapping. - pub fn owner(&self) -> &Object { - &self.owner - } -} - -impl Drop for VMap { - fn drop(&mut self) { - // SAFETY: Our existence is proof that this map was previously created using self.owner - unsafe { self.owner.raw_vunmap(&mut self.map) }; - } -} - -/// SAFETY: `iosys_map` objects are safe to send across threads. -unsafe impl Send for VMap {} -/// SAFETY: `iosys_map` objects are safe to send across threads. -unsafe impl Sync for VMap {} - -/// An owned reference to a scatter-gather table of DMA address spans for a GEM shmem object. -/// -/// This object holds an owned reference to the underlying GEM shmem object, ensuring that the -/// [`scatterlist::SGTable`] referenced by this type remains valid for the lifetime of this object. -/// -/// # Invariants -/// -/// - `_owner` retains both the GEM object and one counted SG-table lease; eviction -/// cannot begin until every lease has been released. -/// - `sgt` corresponds to the owned object in `_owner`. -/// - This object is only exposed in situations where we know the underlying `SGTable` will not be -/// modified for the lifetime of this object. Thus, it is safe to send/access this type across -/// threads. -pub struct SGTable { - sgt: NonNull, - _owner: ARef>, -} - -// SAFETY: This object is thread-safe via our type invariants. -unsafe impl Send for SGTable {} -// SAFETY: This object is thread-safe via our type invariants. -unsafe impl Sync for SGTable {} - -impl Drop for SGTable { - fn drop(&mut self) { - self._owner.sgt_access.fetch_sub(1, Ordering::Release); - } -} - -impl Deref for SGTable { - type Target = scatterlist::SGTable; - - fn deref(&self) -> &Self::Target { - // SAFETY: Creating an immutable reference to this is safe via our type invariants. - unsafe { self.sgt.as_ref() } - } -} diff --git a/rust/kernel/drm_neo/gpuvm.rs b/rust/kernel/drm_neo/gpuvm.rs deleted file mode 100644 index ac82eaf1955d3e..00000000000000 --- a/rust/kernel/drm_neo/gpuvm.rs +++ /dev/null @@ -1,1375 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM Sync Objects -//! -//! C header: [`include/drm_neo/drm_gpuvm.h`](../../../../include/drm_neo/drm_gpuvm.h) - -#![allow(missing_docs)] - -use crate::{ - bindings, - drm_neo::{ - self, - device, - gem::{ - BaseObject, - IntoGEMObject, // - }, // - }, - error::{ - code::{ - EINVAL, - ENOMEM, // - }, - from_err_ptr, - from_result, - to_result, - Error, - Result, // - }, - prelude::*, - sync::aref::{ - ARef, - AlwaysRefCounted, // - }, - types::{NotThreadSafe, Opaque}, // -}; - -use core::cell::UnsafeCell; -use core::marker::{PhantomData, PhantomPinned}; -use core::mem::MaybeUninit; -use core::ops::{Deref, DerefMut, Range}; -use core::ptr::NonNull; -use pin_init; - -/// GpuVaFlags to be used for a GpuVa. -/// -/// They can be combined with the operators `|`, `&`, and `!`. -#[derive(Clone, Copy, PartialEq, Default)] -pub struct GpuVaFlags(u32); - -impl GpuVaFlags { - /// No GpuVaFlags (zero) - pub const NONE: GpuVaFlags = GpuVaFlags(0); - - /// The backing GEM is invalidated. - pub const INVALIDATED: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_INVALIDATED); - - /// The GpuVa is a sparse mapping. - pub const SPARSE: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_SPARSE); - - /// The GpuVa is a repeat mapping. - pub const REPEAT: GpuVaFlags = GpuVaFlags(bindings::drm_gpuva_flags_DRM_GPUVA_REPEAT); - - /// Construct a driver-specific GpuVaFlag. - /// - /// The argument must be a flag index in the range [0..28]. - pub const fn user_flag(index: u32) -> GpuVaFlags { - let flags = bindings::drm_gpuva_flags_DRM_GPUVA_USERBITS << index; - assert!(flags != 0); - GpuVaFlags(flags) - } - - /// Get the raw representation of this flag. - pub(crate) fn as_raw(self) -> u32 { - self.0 - } - - /// Check whether `flags` is contained in `self`. - pub fn contains(self, flags: GpuVaFlags) -> bool { - (self & flags) == flags - } -} - -impl core::ops::BitOr for GpuVaFlags { - type Output = Self; - fn bitor(self, rhs: Self) -> Self::Output { - Self(self.0 | rhs.0) - } -} - -impl core::ops::BitAnd for GpuVaFlags { - type Output = Self; - fn bitand(self, rhs: Self) -> Self::Output { - Self(self.0 & rhs.0) - } -} - -impl core::ops::Not for GpuVaFlags { - type Output = Self; - fn not(self) -> Self::Output { - Self(!self.0) - } -} - -/// Trait that must be implemented by DRM drivers to represent a DRM GpuVm (a GPU address space). -pub trait DriverGpuVm: Sized { - /// The parent `Driver` implementation for this `DriverGpuVm`. - type Driver: drm_neo::Driver; - type GpuVa: DriverGpuVa = (); - type GpuVmBo: DriverGpuVmBo = (); - type StepContext = (); - - fn step_map( - self: &mut UpdatingGpuVm<'_, Self>, - op: &mut OpMap, - ctx: &mut Self::StepContext, - ) -> Result; - fn step_unmap( - self: &mut UpdatingGpuVm<'_, Self>, - op: &mut OpUnMap, - ctx: &mut Self::StepContext, - ) -> Result; - fn step_remap( - self: &mut UpdatingGpuVm<'_, Self>, - op: &mut OpReMap, - vm_bo: &GpuVmBo, - ctx: &mut Self::StepContext, - ) -> Result; -} - -struct StepContext<'a, T: DriverGpuVm> { - gpuvm: &'a GpuVm, - ctx: &'a mut T::StepContext, -} - -/// Trait that must be implemented by DRM drivers to represent a DRM GpuVa (a mapping in GPU address space). -pub trait DriverGpuVa: Sized {} - -impl DriverGpuVa for () {} - -/// Trait that must be implemented by DRM drivers to represent a DRM GpuVmBo (a connection between a BO and a VM). -pub trait DriverGpuVmBo: Sized { - fn new() -> impl PinInit; -} - -/// Provide a default implementation for trivial types -impl DriverGpuVmBo for T { - fn new() -> impl PinInit { - pin_init::default() - } -} - -/// A convenience type for the driver's GEM object. -type Object = <::Driver as drm_neo::driver::Driver>::Object; - -#[repr(transparent)] -pub struct OpMap(bindings::drm_gpuva_op_map, PhantomData); -#[repr(transparent)] -pub struct OpUnMap(bindings::drm_gpuva_op_unmap, PhantomData); -#[repr(transparent)] -pub struct OpReMap(bindings::drm_gpuva_op_remap, PhantomData); - -impl OpMap { - pub fn addr(&self) -> u64 { - self.0.va.addr - } - pub fn range(&self) -> u64 { - self.0.va.range - } - pub fn offset(&self) -> u64 { - self.0.gem.offset - } - pub fn flags(&self) -> GpuVaFlags { - GpuVaFlags(self.0.flags) - } - pub fn object(&self) -> &Object { - let p = unsafe { as IntoGEMObject>::from_raw(self.0.gem.obj) }; - // SAFETY: The GEM object has an active reference for the lifetime of this op - &*p - } - pub fn map_and_link_va( - &mut self, - gpuvm: &mut UpdatingGpuVm<'_, T>, - gpuva: Pin>>, - gpuvmbo: &GpuVmBo, - ) -> Result<(), Pin>>> { - // SAFETY: We are handing off the GpuVa ownership and it will not be moved. - let p = KBox::leak(unsafe { Pin::into_inner_unchecked(gpuva) }); - // SAFETY: These C functions are called with the correct invariants - unsafe { - bindings::drm_gpuva_init_from_op(&mut p.gpuva, &mut self.0); - if bindings::drm_gpuva_insert(gpuvm.0.gpuvm() as *mut _, &mut p.gpuva) != 0 { - // EEXIST, return the GpuVa to the caller as an error - return Err(Pin::new_unchecked(KBox::from_raw(p))); - }; - // SAFETY: This takes a new reference to the gpuvmbo. - gpuvmbo.lock_gpuva(); - bindings::drm_gpuva_link(&mut p.gpuva, &gpuvmbo.bo as *const _ as *mut _); - gpuvmbo.unlock_gpuva(); - } - Ok(()) - } -} - -impl OpUnMap { - pub fn va(&self) -> Option<&GpuVa> { - if self.0.va.is_null() { - return None; - } - // SAFETY: Container invariant is guaranteed for ops structs created for our types. - let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; - // SAFETY: The GpuVa object reference is valid per the op_unmap contract - Some(unsafe { &*p }) - } - pub fn unmap_and_unlink_va(&mut self) -> Option>>> { - self.do_unmap_and_unlink_va(false) - } - pub fn unmap_and_unlink_va_defer(&mut self) -> Option>>> { - self.do_unmap_and_unlink_va(true) - } - fn do_unmap_and_unlink_va(&mut self, defer: bool) -> Option>>> { - if self.0.va.is_null() { - return None; - } - // SAFETY: Container invariant is guaranteed for ops structs created for our types. - let p = unsafe { crate::container_of!(self.0.va, GpuVa, gpuva) as *mut GpuVa }; - - // SAFETY: The GpuVa object reference is valid per the op_unmap contract - unsafe { - bindings::drm_gpuva_unmap(&mut self.0); - if defer { - bindings::drm_gpuva_unlink_defer(self.0.va); - } else { - bindings::drm_gpuva_unlink(self.0.va); - } - } - - // Unlinking/unmapping relinquishes ownership of the GpuVa object, - // so clear the pointer - self.0.va = core::ptr::null_mut(); - // SAFETY: The GpuVa object reference is valid per the op_unmap contract - Some(unsafe { Pin::new_unchecked(KBox::from_raw(p)) }) - } -} - -impl OpReMap { - pub fn prev_map(&mut self) -> Option<&mut OpMap> { - // SAFETY: The prev pointer must be valid if not-NULL per the op_remap contract - unsafe { (self.0.prev as *mut OpMap).as_mut() } - } - pub fn next_map(&mut self) -> Option<&mut OpMap> { - // SAFETY: The next pointer must be valid if not-NULL per the op_remap contract - unsafe { (self.0.next as *mut OpMap).as_mut() } - } - pub fn unmap(&mut self) -> &mut OpUnMap { - // SAFETY: The unmap pointer is always valid per the op_remap contract - unsafe { (self.0.unmap as *mut OpUnMap).as_mut().unwrap() } - } -} - -/// A base GPU VA. -#[repr(C)] -#[pin_data] -pub struct GpuVa { - #[pin] - gpuva: bindings::drm_gpuva, - #[pin] - inner: T::GpuVa, - #[pin] - _p: PhantomPinned, -} - -impl GpuVa { - pub fn new(inner: impl PinInit) -> Result>>> - where - Error: From, - { - KBox::try_pin_init( - try_pin_init!(Self { - gpuva <- pin_init::init_zeroed(), - inner <- inner, - _p: PhantomPinned - }), - GFP_KERNEL, - ) - } - - /// Driver metadata retained independently of the resident page tables. - pub fn inner(&self) -> &T::GpuVa { &self.inner } - - /// Update metadata before linking a newly allocated GPUVA. - pub fn inner_mut(self: Pin<&mut Self>) -> Pin<&mut T::GpuVa> { - // SAFETY: inner is structurally pinned with its enclosing GPUVA. - unsafe { self.map_unchecked_mut(|va| &mut va.inner) } - } - - pub fn addr(&self) -> u64 { - self.gpuva.va.addr - } - pub fn range(&self) -> u64 { - self.gpuva.va.range - } - pub fn offset(&self) -> u64 { - self.gpuva.gem.offset - } - pub fn flags(&self) -> GpuVaFlags { - GpuVaFlags(self.gpuva.flags) - } -} - -/// A base GpuVm BO. -#[repr(C)] -#[pin_data] -pub struct GpuVmBo { - #[pin] - bo: bindings::drm_gpuvm_bo, - #[pin] - inner: T::GpuVmBo, - #[pin] - _p: PhantomPinned, -} - -impl GpuVmBo { - /// The GEM object retained by this VM/BO association. - pub fn object(&self) -> &Object { - // SAFETY: drm_gpuvm_bo owns this GEM reference until its destructor; - // DriverGpuVm fixes the GEM type for every association in this VM. - unsafe { &* as IntoGEMObject>::from_raw(self.bo.obj) } - } - - /// Return a reference to the inner driver data for this GpuVmBo - pub fn inner(&self) -> &T::GpuVmBo { - &self.inner - } - /// Lock the GpuVmBo's gem boject gpuva lock - pub fn lock_gpuva(&self) { - unsafe { - let lock = &raw mut (*self.bo.obj).gpuva.lock; - bindings::mutex_lock(lock); - } - } - /// Unlock the GpuVmBo's gem boject gpuva lock - pub fn unlock_gpuva(&self) { - unsafe { - let lock = &raw mut (*self.bo.obj).gpuva.lock; - bindings::mutex_unlock(lock); - } - } -} - -// SAFETY: DRM GpuVmBo objects are always reference counted and the get/put functions -// satisfy the requirements. -unsafe impl AlwaysRefCounted for GpuVmBo { - fn inc_ref(&self) { - // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). - unsafe { bindings::drm_gpuvm_bo_get(&self.bo as *const _ as *mut _) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: This owns a live BO reference. Immediate-mode GPUVA lists - // use the GEM's gpuva mutex, not its reservation. The deferred put - // releases that mutex before final GEM destruction. Keep the GPUVM - // alive across draining, which can drop its last BO-owned reference. - unsafe { - let bo = &raw mut (*obj.as_ptr()).bo; - let vm = (*bo).vm; - if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE != 0 { - bindings::drm_gpuvm_get(vm); - bindings::drm_gpuvm_bo_put_deferred(bo); - bindings::drm_gpuvm_bo_deferred_cleanup(vm); - bindings::drm_gpuvm_put(vm); - return; - } - - // A final BO put also puts its GEM. Retain the GEM until after - // unlocking its reservation, which may be embedded in that GEM. - let gem = (*bo).obj; - bindings::drm_gem_object_get(gem); - let resv = (*gem).resv; - bindings::dma_resv_lock(resv, core::ptr::null_mut()); - bindings::drm_gpuvm_bo_put(bo); - bindings::dma_resv_unlock(resv); - bindings::drm_gem_object_put(gem); - } - } -} - -/// A base GPU VM. -#[repr(C)] -#[pin_data] -pub struct GpuVm { - #[pin] - gpuvm: Opaque, - #[pin] - inner: UnsafeCell, - #[pin] - _p: PhantomPinned, -} - -pub(super) unsafe extern "C" fn vm_free_callback( - raw_gpuvm: *mut bindings::drm_gpuvm, -) { - // SAFETY: Container invariant is guaranteed for objects using our callback. - let p = unsafe { - crate::container_of!( - raw_gpuvm as *mut Opaque, - GpuVm, - gpuvm - ) as *mut GpuVm - }; - - // SAFETY: p is guaranteed to be valid for drm_gpuvm objects using this callback. - unsafe { drop(KBox::from_raw(p)) }; -} - -pub(super) unsafe extern "C" fn vm_bo_alloc_callback() -> *mut bindings::drm_gpuvm_bo -{ - let obj: Result>>> = KBox::try_pin_init( - try_pin_init!(GpuVmBo:: { - // The bindgen `Default` implementation is exactly a zero fill, but its - // out-of-line symbol is not exported to loadable modules. `drm_gpuvm_bo` - // is `Zeroable`, so initialize it directly without a cross-crate call. - bo <- pin_init::zeroed::(), - inner <- T::GpuVmBo::new(), - _p: PhantomPinned - }), - GFP_KERNEL, - ); - - match obj { - Ok(obj) => - // SAFETY: The DRM core will keep this object pinned - unsafe { - let p = KBox::leak(Pin::into_inner_unchecked(obj)); - &mut p.bo - }, - Err(_) => core::ptr::null_mut(), - } -} - -pub(super) unsafe extern "C" fn vm_bo_free_callback( - raw_vm_bo: *mut bindings::drm_gpuvm_bo, -) { - // SAFETY: Container invariant is guaranteed for objects using this callback. - let p = unsafe { crate::container_of!(raw_vm_bo, GpuVmBo, bo) as *mut GpuVmBo }; - - // SAFETY: p is guaranteed to be valid for drm_gpuvm_bo objects using this callback. - unsafe { drop(KBox::from_raw(p)) }; -} - -pub(super) unsafe extern "C" fn step_map_callback( - op: *mut bindings::drm_gpuva_op, - _priv: *mut core::ffi::c_void, -) -> core::ffi::c_int { - // SAFETY: We know this is a map op, and OpMap is a transparent wrapper. - let map = unsafe { &mut *((&mut (*op).__bindgen_anon_1.map) as *mut _ as *mut OpMap) }; - // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is - // guaranteed to outlive this function. - let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; - - from_result(|| { - UpdatingGpuVm(ctx.gpuvm).step_map(map, ctx.ctx)?; - Ok(0) - }) -} - -pub(super) unsafe extern "C" fn step_remap_callback( - op: *mut bindings::drm_gpuva_op, - _priv: *mut core::ffi::c_void, -) -> core::ffi::c_int { - // SAFETY: We know this is a map op, and OpReMap is a transparent wrapper. - let remap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.remap) as *mut _ as *mut OpReMap) }; - // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is - // guaranteed to outlive this function. - let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; - - let p_vm_bo = remap.unmap().va().unwrap().gpuva.vm_bo; - - let res = { - // SAFETY: vm_bo pointer must be valid and non-null by the step_remap invariants. - // Since we grab a ref, this reference's lifetime is until the decref. - let vm_bo_ref = unsafe { - bindings::drm_gpuvm_bo_get(p_vm_bo); - &*(crate::container_of!(p_vm_bo, GpuVmBo, bo) as *mut GpuVmBo) - }; - - from_result(|| { - UpdatingGpuVm(ctx.gpuvm).step_remap(remap, vm_bo_ref, ctx.ctx)?; - Ok(0) - }) - }; - - // SAFETY: We incremented the refcount above, and the Rust reference we took is - // no longer in scope. - unsafe { bindings::drm_gpuvm_bo_put_deferred(p_vm_bo) }; - - res -} -pub(super) unsafe extern "C" fn step_unmap_callback( - op: *mut bindings::drm_gpuva_op, - _priv: *mut core::ffi::c_void, -) -> core::ffi::c_int { - // SAFETY: We know this is a map op, and OpUnMap is a transparent wrapper. - let unmap = unsafe { &mut *((&mut (*op).__bindgen_anon_1.unmap) as *mut _ as *mut OpUnMap) }; - // SAFETY: This is a pointer to a StepContext created inline in sm_map(), which is - // guaranteed to outlive this function. - let ctx = unsafe { &mut *(_priv as *mut StepContext<'_, T>) }; - - from_result(|| { - UpdatingGpuVm(ctx.gpuvm).step_unmap(unmap, ctx.ctx)?; - Ok(0) - }) -} - -pub(super) unsafe extern "C" fn exec_lock_gem_object( - vm_exec: *mut bindings::drm_gpuvm_exec, -) -> core::ffi::c_int { - // SAFETY: The gpuvm_exec object is valid and priv_ is a GEM object pointer - // when this callback is used - unsafe { bindings::drm_exec_lock_obj(&mut (*vm_exec).exec, (*vm_exec).extra.priv_ as *mut _) } -} - -impl GpuVm { - const OPS: bindings::drm_gpuvm_ops = bindings::drm_gpuvm_ops { - vm_free: Some(vm_free_callback::), - op_alloc: None, - op_free: None, - vm_bo_alloc: Some(vm_bo_alloc_callback::), - vm_bo_free: Some(vm_bo_free_callback::), - vm_bo_validate: None, - sm_step_map: Some(step_map_callback::), - sm_step_remap: Some(step_remap_callback::), - sm_step_unmap: Some(step_unmap_callback::), - sm_can_merge_flags: None, - }; - - fn gpuvm(&self) -> *const bindings::drm_gpuvm { - self.gpuvm.get() - } - - pub fn new( - name: &'static CStr, - flags: bindings::drm_gpuvm_flags, - dev: &device::Device, - r_obj: ARef>, - range: Range, - reserve_range: Range, - inner: impl PinInit, - ) -> Result>> - where - Error: From, - { - let obj: Pin> = KBox::try_pin_init( - try_pin_init!(Self { - // SAFETY: drm_gpuvm_init cannot fail and always initializes the member - gpuvm <- unsafe { - pin_init::pin_init_from_closure(move |slot: *mut Opaque | { - // Zero-init required by drm_gpuvm_init - *slot = Opaque::zeroed(); - bindings::drm_gpuvm_init( - Opaque::cast_into(slot), - name.as_char_ptr(), - flags, - dev.as_raw(), - r_obj.as_raw() as *const _ as *mut _, - range.start, - range.end - range.start, - reserve_range.start, - reserve_range.end - reserve_range.start, - &Self::OPS - ); - Ok(()) - }) - }, - // SAFETY: Just passing through to the initializer argument - inner <- unsafe { - pin_init::pin_init_from_closure(move |slot: *mut UnsafeCell | { - inner.__pinned_init(slot as *mut _) - }) - }, - _p: PhantomPinned - }), - GFP_KERNEL, - )?; - - // SAFETY: We never move out of the object - let vm_ref = unsafe { - ARef::from_raw(NonNull::new_unchecked(KBox::leak( - Pin::into_inner_unchecked(obj), - ))) - }; - - Ok(vm_ref) - } - - /// Lock only the VM's driver data, without allocating an exec context. - /// - /// This takes the same shared reservation as `exec_lock`, but does not lock - /// external BO reservations. The returned guard exposes only the inner - /// driver data, not GPUVA operations requiring those other reservations. - /// Callers must not already hold a reservation lock. - pub fn lock_inner(&self) -> GpuVmInnerGuard<'_, T> { - // SAFETY: The borrowed GPUVM retains r_obj and its reservation for the - // guard's lifetime. A non-interruptible single-reservation lock with no - // ww context cannot fail or allocate. No other reservation is acquired. - unsafe { - bindings::dma_resv_lock((*(*self.gpuvm()).r_obj).resv, core::ptr::null_mut()); - } - GpuVmInnerGuard { - gpuvm: self, - _not_send: NotThreadSafe, - } - } - - // The caller owns the VM reservation and keeps the exact logical VA linked. - unsafe fn set_invalidated_locked(&self, addr: u64, range: u64, invalidated: bool) -> Result { - if range == 0 || addr.checked_add(range).is_none() { return Err(EINVAL); } - // SAFETY: The caller owns the VM reservation and keeps the VA linked. - let raw = unsafe { bindings::drm_gpuva_find(self.gpuvm() as *mut _, addr, range) }; - if raw.is_null() { return Err(EINVAL); } - // SAFETY: Reservation ownership excludes all concurrent VA mutation. - unsafe { - if (*raw).va.addr != addr || (*raw).va.range != range { return Err(EINVAL); } - if invalidated { (*raw).flags |= GpuVaFlags::INVALIDATED.as_raw(); } - else { (*raw).flags &= !GpuVaFlags::INVALIDATED.as_raw(); } - } - Ok(()) - } - - /// Try to lock only the VM reservation, without allocation or waiting. - /// The guard protects VM metadata and private BOs, not external backing. - /// Callers must not already hold a reservation lock. - pub fn try_lock_private(&self) -> Option> { - // SAFETY: r_obj and its reservation are retained by this GPUVM. - let resv = unsafe { (*(*self.gpuvm()).r_obj).resv }; - // SAFETY: The reservation outlives the returned borrow and guard. - if !unsafe { bindings::dma_resv_trylock(resv) } { return None; } - Some(PrivateGpuVmGuard { gpuvm: self, resv, _not_send: NotThreadSafe }) - } - - pub fn exec_lock<'a, 'b>( - &'a self, - obj: Option<&'b Object>, - interruptible: bool, - ) -> Result> { - // Do not try to lock the object if it is internal (since it is already locked). - let is_ext = obj.map(|a| self.is_extobj(a)).unwrap_or(false); - - // vm_exec needs a stable address while locking. Construct the unlock - // guard only after success; C already finalizes the exec on error. - let mut vm_exec = KBox::init( - init!(bindings::drm_gpuvm_exec { - vm: self.gpuvm() as *mut _, - flags: if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 - }, - // SAFETY: bindgen's `Default` for this C structure is exactly - // `MaybeUninit::zeroed().assume_init()`. Spell that out here so - // external modules do not import its unexported trait method. - exec: unsafe { MaybeUninit::::zeroed().assume_init() }, - extra: match (is_ext, obj) { - (true, Some(obj)) => bindings::drm_gpuvm_exec__bindgen_ty_1 { - fn_: Some(exec_lock_gem_object), - priv_: obj.as_raw() as *const _ as *mut _, - }, - // SAFETY: as above, this bindgen C structure is zero-valid. - _ => unsafe { - MaybeUninit::::zeroed() - .assume_init() - }, - }, - num_fences: 0, - }), - GFP_KERNEL, - )?; - - // SAFETY: The object is valid and was initialized above - to_result(unsafe { bindings::drm_gpuvm_exec_lock(&mut *vm_exec) })?; - - Ok(LockedGpuVm { - gpuvm: self, - vm_exec, - objects: LockedObjects::Single(obj), - }) - } - - /// Lock the GPUVM and an array of additional GEM objects in one - /// wound/wait transaction. - pub fn exec_lock_array<'a, 'b>( - &'a self, - objects: &'b [ARef>], - interruptible: bool, - ) -> Result> { - let count = u32::try_from(objects.len()).map_err(|_| EINVAL)?; - let mut raw_objects = KVec::with_capacity(objects.len(), GFP_KERNEL)?; - for object in objects { - raw_objects.push(object.as_raw(), GFP_KERNEL)?; - } - - // vm_exec needs a stable address while locking. Construct the unlock - // guard only after success; C already finalizes the exec on error. - let mut vm_exec = KBox::init( - init!(bindings::drm_gpuvm_exec { - vm: self.gpuvm() as *mut _, - flags: (if interruptible { - bindings::DRM_EXEC_INTERRUPTIBLE_WAIT - } else { - 0 - }) | bindings::DRM_EXEC_IGNORE_DUPLICATES, - // SAFETY: both bindgen C structures are zero-valid; their generated - // `Default` methods perform the same zero initialization but are not - // exported from the kernel crate to loadable modules. - exec: unsafe { MaybeUninit::::zeroed().assume_init() }, - extra: unsafe { - MaybeUninit::::zeroed() - .assume_init() - }, - num_fences: 0, - }), - GFP_KERNEL, - )?; - - // SAFETY: Every raw pointer is backed by an ARef in `objects`, which - // outlives the returned guard. The helper consumes the temporary - // pointer array before returning and keeps its own object references. - to_result(unsafe { - bindings::drm_gpuvm_exec_lock_array( - &mut *vm_exec, - raw_objects.as_mut_ptr(), - count, - ) - })?; - - Ok(LockedGpuVm { - gpuvm: self, - vm_exec, - objects: LockedObjects::Array(objects), - }) - } - - /// Returns true if the given object is external to the GPUVM - /// (that is, if it does not share the DMA reservation object of the GPUVM). - pub fn is_extobj(&self, obj: &impl IntoGEMObject) -> bool { - let gem = obj.as_raw() as *const _ as *mut _; - // SAFETY: This is safe to call as long as the arguments are valid pointers. - unsafe { bindings::drm_gpuvm_is_extobj(self.gpuvm() as *mut _, gem) } - } - - /// Validate against the immutable address-space bounds and kernel cutout. - pub fn range_valid(&self, addr: u64, range: u64) -> bool { - // SAFETY: the borrowed VM retains its immutable geometry. - unsafe { bindings::drm_gpuvm_range_valid(self.gpuvm() as *mut _, addr, range) } - } - - pub fn bo_deferred_cleanup(&self) { - unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.gpuvm() as *mut _) } - } - - pub fn find_bo(&self, obj: &Object) -> Option>> { - obj.lock_gpuva(); - // SAFETY: drm_gem_object.gpuva.lock was just locked. - let p = unsafe { - bindings::drm_gpuvm_bo_find(self.gpuvm() as *mut _, obj.as_raw() as *const _ as *mut _) - }; - obj.unlock_gpuva(); - if p.is_null() { - None - } else { - // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. - let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; - // SAFETY: We checked for NULL above, and the types ensure that - // this object was created by vm_bo_alloc_callback. - Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) - } - } - - pub fn obtain_bo(&self, obj: &Object) -> Result>> { - obj.lock_gpuva(); - // SAFETY: drm_gem_object.gpuva.lock was just locked. - let p = unsafe { - bindings::drm_gpuvm_bo_obtain_locked( - self.gpuvm() as *mut _, - obj.as_raw() as *const _ as *mut _, - ) - }; - obj.unlock_gpuva(); - let p = from_err_ptr(p)?; - if p.is_null() { - Err(ENOMEM) - } else { - // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. - let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; - // SAFETY: We checked for NULL above, and the types ensure that - // this object was created by vm_bo_alloc_callback. - Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) - } - } - - pub fn bo_unmap(&self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { - let mut ctx = StepContext { ctx, gpuvm: self }; - // SAFETY: LockedGpuVm implies the right locks are held. - to_result(unsafe { - bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) - }) - } -} - -// SAFETY: DRM GpuVm objects are always reference counted and the get/put functions -// satisfy the requirements. -unsafe impl AlwaysRefCounted for GpuVm { - fn inc_ref(&self) { - // SAFETY: The drm_gpuvm_get function satisfies the requirements for inc_ref(). - unsafe { bindings::drm_gpuvm_get(&self.gpuvm as *const _ as *mut _) }; - } - - unsafe fn dec_ref(obj: NonNull) { - // SAFETY: The drm_gpuvm_put function satisfies the requirements for dec_ref(). - unsafe { bindings::drm_gpuvm_put(Opaque::cast_into(&(*obj.as_ptr()).gpuvm)) }; - } -} - -enum LockedObjects<'a, T: DriverGpuVm> { - Single(Option<&'a Object>), - Array(&'a [ARef>]), -} - -impl LockedObjects<'_, T> { - fn single(&self) -> Option<&Object> { - match self { - Self::Single(object) => *object, - Self::Array(_) => None, - } - } - - fn indexed(&self, index: usize) -> Option<&Object> { - match self { - Self::Array(objects) => objects.get(index).map(|object| &**object), - Self::Single(_) => None, - } - } -} - -/// Access to VM driver data under its shared reservation, without external BO locks. -pub struct GpuVmInnerGuard<'a, T: DriverGpuVm> { - gpuvm: &'a GpuVm, - // Reservation locks must be released by the task that acquired them. - _not_send: NotThreadSafe, -} - -impl GpuVmInnerGuard<'_, T> { - /// Run immediate-mode unmap steps without allocating an exec context. - /// - /// # Safety - /// The driver's unmap/remap callbacks must touch only VM metadata and - /// translations protected by this reservation. They must not access - /// external BO backing that requires its own reservation. Any split-node - /// storage must already be owned by `ctx`; callbacks must defer BO puts. - pub unsafe fn sm_unmap_inner( - &mut self, ctx: &mut T::StepContext, addr: u64, range: u64, - ) -> Result { - let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; - // SAFETY: this guard retains the VM and its shared reservation. - if unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { - return Err(EINVAL); - } - let mut ctx = StepContext { gpuvm: self.gpuvm, ctx }; - // SAFETY: caller supplies the callback contract above. Immediate-mode - // link/unlink acquire the separate GEM GPUVA-list mutex themselves. - to_result(unsafe { - bindings::drm_gpuvm_sm_unmap(vm, &mut ctx as *mut _ as *mut _, addr, range) - }) - } - - /// Remove every driver GPUVA in an immediate-mode VM, preserving its kernel cutout. - /// - /// `unmap` must remove the GPU translation for each complete GPUVA before - /// returning success. Only the VM's shared reservation is held. No external - /// BO reservations are acquired and no split nodes or operation lists are - /// allocated. The callback cannot retain the borrowed GPUVA. - /// - /// Call `GpuVm::bo_deferred_cleanup` after releasing this guard, including - /// when a callback fails after earlier mappings have already been removed. - pub fn unmap_all( - &mut self, - unmap: impl FnMut(&mut T, &GpuVa) -> Result, - ) -> Result { - self.unmap_matching(None, unmap) - } - - /// Allocation-free whole-node removal for one GEM in an immediate-mode VM. - /// Uses the same translation-removal and deferred-put contract as unmap_all. - /// This scans the VM list and is intended for low-memory cleanup fallback. - pub fn unmap_object( - &mut self, - object: &Object, - unmap: impl FnMut(&mut T, &GpuVa) -> Result, - ) -> Result { - self.unmap_matching(Some(object), unmap) - } - - fn unmap_matching( - &mut self, - object: Option<&Object>, - mut unmap: impl FnMut(&mut T, &GpuVa) -> Result, - ) -> Result { - let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; - // SAFETY: The guard owns this live VM's shared reservation. Immediate - // mode lets unlink take the GEM GPUVA mutex independently of its resv. - unsafe { - if (*vm).flags & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { - return Err(EINVAL); - } - let head = &raw mut (*vm).rb.list; - let mut entry = (*head).next; - while entry != head { - let va = crate::container_of!(entry, bindings::drm_gpuva, rb.entry) as *mut _; - // Save the successor before removing and freeing this GPUVA. - entry = (*entry).next; - // This node is embedded in the C GPUVM, not a driver GpuVa. - if core::ptr::eq(va, &raw const (*vm).kernel_alloc_node) { - continue; - } - if object.is_some_and(|object| (*va).gem.obj != object.as_raw()) { - continue; - } - let driver_va = crate::container_of!(va, GpuVa, gpuva); - unmap(&mut *self.gpuvm.inner.get(), &*driver_va)?; - let mut op = OpUnMap( - bindings::drm_gpuva_op_unmap { va, keep: false }, - PhantomData::, - ); - drop(op.unmap_and_unlink_va_defer()); - } - } - Ok(()) - } -} - -impl Deref for GpuVmInnerGuard<'_, T> { - type Target = T; - - fn deref(&self) -> &T { - // SAFETY: The shared reservation excludes every other inner-data guard, - // including LockedGpuVm and its UpdatingGpuVm callbacks. - unsafe { &*self.gpuvm.inner.get() } - } -} - -impl DerefMut for GpuVmInnerGuard<'_, T> { - fn deref_mut(&mut self) -> &mut T { - // SAFETY: As above, with exclusive access to this guard. - unsafe { &mut *self.gpuvm.inner.get() } - } -} - -impl Drop for GpuVmInnerGuard<'_, T> { - fn drop(&mut self) { - // SAFETY: This non-Send guard owns the reservation on this task, and - // its GPUVM borrow keeps r_obj alive until after the unlock. - unsafe { bindings::dma_resv_unlock((*(*self.gpuvm.gpuvm()).r_obj).resv) }; - } -} - -/// Logical binding retained independently of hardware page-table residency. -/// Owns a GEM reference and copied metadata, never a GpuVmBo reference. -/// Driver metadata cloning must not introduce a reservation-taking destructor. -pub struct MappingSnapshot { - pub object: ARef>, - pub addr: u64, - pub range: u64, - pub offset: u64, - pub flags: GpuVaFlags, - pub inner: T::GpuVa, -} - -/// Nonblocking reservation ownership for pressure scans of private objects. -pub struct PrivateGpuVmGuard<'a, T: DriverGpuVm> { - gpuvm: &'a GpuVm, - resv: *mut bindings::dma_resv, - // DMA reservation ownership must remain on the acquiring task. - _not_send: NotThreadSafe, -} - -impl PrivateGpuVmGuard<'_, T> { - /// Visit each private BO with linked logical mappings once, without allocating or retaining - /// references. Returning false stops the walk. Contended GEM lists are skipped. - /// - /// # Safety - /// The callback must not insert/remove logical mappings or modify either GPUVA list. This - /// reservation keeps their linked BO/GEM references alive for the complete callback. - /// The callback must not retain a borrowed BO or acquire its GEM GPUVA mutex recursively. - pub unsafe fn for_each_private_bo( - &mut self, - mut visit: impl FnMut(&mut Self, &GpuVmBo) -> Result, - ) -> Result { - let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; - if unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 { - return Err(EINVAL); - } - let head = unsafe { &raw const (*vm).rb.list }; - // SAFETY: The VM reservation protects this ordered mapping list. The callback does not - // mutate its links, and no borrowed or owned reference outlives the guard. - let mut node = unsafe { (*head).next }; - while node != head.cast_mut() { - let (next, raw) = unsafe { - ((*node).next, crate::container_of!(node, bindings::drm_gpuva, rb.entry)) - }; - let va = unsafe { &*raw }; - if !va.gem.obj.is_null() && !va.vm_bo.is_null() { - let object = unsafe { &*Object::::from_raw(va.gem.obj) }; - if !self.gpuvm.is_extobj(object) { - let lock = unsafe { &raw mut (*va.gem.obj).gpuva.lock }; - // Never block direct reclaim on a GEM-list owner. The canonical alias is - // the list's first-linked node, not its lowest address or current residency. - if unsafe { bindings::mutex_trylock(lock) } != 0 { - let canonical = unsafe { - (*va.vm_bo).list.gpuva.next == (&raw const va.gem.entry).cast_mut() - }; - unsafe { bindings::mutex_unlock(lock) }; - if canonical { - let bo = unsafe { - &*crate::container_of!(va.vm_bo, GpuVmBo, bo) - }; - if !visit(self, bo)? { break; } - } - } - } - } - node = next; - } - Ok(()) - } - - /// Mark and visit every logical alias of this private BO without a snapshot allocation. - /// GEM-list locking is nonblocking and held only while reading a cursor, never during the - /// callback. Failure (including contention) can leave a prefix marked INVALIDATED; callers - /// must retain backing unless this method succeeds for the entire association. - /// - /// # Safety - /// The callback must not insert/remove logical mappings, alter list links, or release the - /// BO's SG/backing lease. The BO must remain linked throughout the walk. Exclude new jobs and - /// binding edits for the whole invalidation and backing-release transaction. - pub unsafe fn invalidate_private_bo_mappings( - &mut self, - bo: &GpuVmBo, - mut visit: impl FnMut(&mut Self, u64, u64) -> Result, - ) -> Result { - let vm = self.gpuvm.gpuvm() as *mut bindings::drm_gpuvm; - if bo.bo.vm != vm || self.gpuvm.is_extobj(bo.object()) - || unsafe { (*vm).flags } & bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE == 0 - { return Err(EINVAL); } - let head = &raw const bo.bo.list.gpuva; - let lock = unsafe { &raw mut (*bo.bo.obj).gpuva.lock }; - if unsafe { bindings::mutex_trylock(lock) } == 0 { return Err(EAGAIN); } - let mut node = unsafe { (*head).next }; - unsafe { bindings::mutex_unlock(lock) }; - while node != head.cast_mut() { - if unsafe { bindings::mutex_trylock(lock) } == 0 { return Err(EAGAIN); } - // SAFETY: The list mutex protects link reads; this reservation and the caller's - // no-unlink contract keep current/next and their association alive after unlocking. - let (next, raw) = unsafe { - ((*node).next, crate::container_of!(node, bindings::drm_gpuva, gem.entry)) - }; - unsafe { bindings::mutex_unlock(lock) }; - let va = unsafe { &mut *raw }; - if va.vm != vm || va.vm_bo != (&raw const bo.bo).cast_mut() - || va.va.range == 0 || va.va.addr.checked_add(va.va.range).is_none() - { return Err(EINVAL); } - // Flags are serialized by the VM reservation, independently of GEM's list mutex. - va.flags |= GpuVaFlags::INVALIDATED.as_raw(); - visit(self, va.va.addr, va.va.range)?; - node = next; - } - Ok(()) - } - - /// Retain the next private mapping containing all `required_flags`. - /// - /// `cursor` is an exclusive end address and advances past skipped mappings - /// as well. No collection is allocated; only the returned object is retained. - /// Callers must drop this reservation guard before releasing that reference - /// or acquiring backing. To cover a stable set across calls, separately - /// exclude logical binding edits (including retirement) for the whole walk. - pub fn next_private_mapping( - &self, - cursor: &mut u64, - required_flags: GpuVaFlags, - ) -> Result>> - where T::GpuVa: Clone { - while *cursor != u64::MAX { - // SAFETY: The VM reservation serializes mutation of its VA tree. - let raw = unsafe { - bindings::drm_gpuva_find_first( - self.gpuvm.gpuvm() as *mut _, *cursor, u64::MAX - *cursor, - ) - }; - if raw.is_null() { return Ok(None); } - // SAFETY: raw stays linked and alive while this guard is held. - let va = unsafe { &*raw }; - let end = va.va.addr.checked_add(va.va.range).ok_or(EINVAL)?; - if end <= *cursor { return Err(EINVAL); } - *cursor = end; - if !GpuVaFlags(va.flags).contains(required_flags) - || va.gem.obj.is_null() || va.vm_bo.is_null() { - continue; - } - // SAFETY: The linked VA retains this driver's object. - let obj = unsafe { &*Object::::from_raw(va.gem.obj) }; - if self.gpuvm.is_extobj(obj) { continue; } - // SAFETY: Driver-owned mappings have this allocation layout. The - // special kernel cutout has no GEM object and was skipped above. - let typed = unsafe { &*crate::container_of!(raw, GpuVa, gpuva) }; - return Ok(Some(MappingSnapshot { - object: obj.into(), addr: va.va.addr, - range: va.va.range, offset: va.gem.offset, - flags: GpuVaFlags(va.flags), inner: typed.inner.clone(), - })); - } - Ok(None) - } - - -} - -impl Deref for PrivateGpuVmGuard<'_, T> { - type Target = T; - fn deref(&self) -> &T { - // SAFETY: The reservation serializes all access to the VM's inner data. - unsafe { &*self.gpuvm.inner.get() } - } -} -impl DerefMut for PrivateGpuVmGuard<'_, T> { - fn deref_mut(&mut self) -> &mut T { - // SAFETY: This guard has exclusive reservation ownership. - unsafe { &mut *self.gpuvm.inner.get() } - } -} -impl Drop for PrivateGpuVmGuard<'_, T> { - fn drop(&mut self) { - // SAFETY: try_lock_private acquired exactly this reservation. - unsafe { bindings::dma_resv_unlock(self.resv) }; - } -} - -pub struct LockedGpuVm<'a, 'b, T: DriverGpuVm> { - gpuvm: &'a GpuVm, - vm_exec: KBox, - objects: LockedObjects<'b, T>, -} - -impl LockedGpuVm<'_, '_, T> { - /// Mark the exact retained logical mapping after page-table restoration. - pub fn set_invalidated(&mut self, addr: u64, range: u64, invalidated: bool) -> Result { - // SAFETY: Both guard types own the VM reservation. - unsafe { self.gpuvm.set_invalidated_locked(addr, range, invalidated) } - } - - pub fn find_bo(&mut self) -> Option>> { - let obj = self.objects.single()?; - // SAFETY: LockedGpuVm implies the right locks are held. - let p = unsafe { - bindings::drm_gpuvm_bo_find( - self.gpuvm.gpuvm() as *mut _, - obj.as_raw() as *const _ as *mut _, - ) - }; - if p.is_null() { - None - } else { - // SAFETY: All the drm_gpuvm_bo objects in this GpuVm are always allocated by us as GpuVmBo. - let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; - // SAFETY: We checked for NULL above, and the types ensure that - // this object was created by vm_bo_alloc_callback. - Some(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) - } - } - - pub fn obtain_bo(&mut self) -> Result>> { - let obj = self.objects.single().ok_or(EINVAL)?; - // SAFETY: LockedGpuVm implies the right locks are held. - let p = unsafe { - bindings::drm_gpuvm_bo_obtain_locked( - self.gpuvm.gpuvm() as *mut _, - obj.as_raw() as *const _ as *mut _, - ) - }; - let p = from_err_ptr(p)?; - if p.is_null() { - Err(ENOMEM) - } else { - // SAFETY: Container invariant is guaranteed for GpuVmBo objects for this GpuVm. - let p = unsafe { crate::container_of!(p, GpuVmBo, bo) as *mut GpuVmBo }; - // SAFETY: We checked for NULL above, and the types ensure that - // this object was created by vm_bo_alloc_callback. - Ok(unsafe { ARef::from_raw(NonNull::new_unchecked(p)) }) - } - } - - pub fn sm_map( - &mut self, - ctx: &mut T::StepContext, - req_addr: u64, - req_range: u64, - req_offset: u64, - req_gem_range: u32, - flags: GpuVaFlags, - ) -> Result { - let obj = self.objects.single().ok_or(EINVAL)?.as_raw(); - self.sm_map_raw( - obj, - ctx, - req_addr, - req_range, - req_offset, - req_gem_range, - flags, - ) - } - - /// Map with an object acquired by [`GpuVm::exec_lock_array`]. - pub fn sm_map_indexed( - &mut self, - object_index: usize, - ctx: &mut T::StepContext, - req_addr: u64, - req_range: u64, - req_offset: u64, - req_gem_range: u32, - flags: GpuVaFlags, - ) -> Result { - let obj = self - .objects - .indexed(object_index) - .ok_or(EINVAL)? - .as_raw(); - self.sm_map_raw( - obj, - ctx, - req_addr, - req_range, - req_offset, - req_gem_range, - flags, - ) - } - - fn sm_map_raw( - &mut self, - obj: *mut bindings::drm_gem_object, - ctx: &mut T::StepContext, - req_addr: u64, - req_range: u64, - req_offset: u64, - req_gem_range: u32, - flags: GpuVaFlags, - ) -> Result { - let mut ctx = StepContext { - ctx, - gpuvm: self.gpuvm, - }; - - let req = bindings::drm_gpuvm_map_req { - map: bindings::drm_gpuva_op_map { - va: bindings::drm_gpuva_op_map__bindgen_ty_1 { - addr: req_addr, - range: req_range, - }, - gem: bindings::drm_gpuva_op_map__bindgen_ty_2 { - offset: req_offset, - range: req_gem_range, - obj, - }, - flags: flags.as_raw(), - }, - }; - - // SAFETY: The object pointer came from the reference set retained by - // this guard, and LockedGpuVm implies the required locks are held. - to_result(unsafe { - bindings::drm_gpuvm_sm_map( - self.gpuvm.gpuvm() as *mut _, - &mut ctx as *mut _ as *mut _, - &raw const req, - ) - }) - } - - pub fn sm_unmap(&mut self, ctx: &mut T::StepContext, req_addr: u64, req_range: u64) -> Result { - let mut ctx = StepContext { - ctx, - gpuvm: self.gpuvm, - }; - // SAFETY: LockedGpuVm implies the right locks are held. - to_result(unsafe { - bindings::drm_gpuvm_sm_unmap( - self.gpuvm.gpuvm() as *mut _, - &mut ctx as *mut _ as *mut _, - req_addr, - req_range, - ) - }) - } - - pub fn bo_unmap(&mut self, ctx: &mut T::StepContext, bo: &GpuVmBo) -> Result { - let mut ctx = StepContext { - ctx, - gpuvm: self.gpuvm, - }; - // SAFETY: LockedGpuVm implies the right locks are held. - to_result(unsafe { - bindings::drm_gpuvm_bo_unmap(&bo.bo as *const _ as *mut _, &mut ctx as *mut _ as *mut _) - }) - } -} - -impl Deref for LockedGpuVm<'_, '_, T> { - type Target = T; - - fn deref(&self) -> &T { - // SAFETY: The existence of this LockedGpuVm implies the lock is held, - // so this is the only reference - unsafe { &*self.gpuvm.inner.get() } - } -} - -impl DerefMut for LockedGpuVm<'_, '_, T> { - fn deref_mut(&mut self) -> &mut T { - // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, - // so this is the only reference - unsafe { &mut *self.gpuvm.inner.get() } - } -} - -impl Drop for LockedGpuVm<'_, '_, T> { - fn drop(&mut self) { - // SAFETY: We hold the lock, so it's safe to unlock - unsafe { - bindings::drm_gpuvm_exec_unlock(&mut *self.vm_exec); - } - } -} - -pub struct UpdatingGpuVm<'a, T: DriverGpuVm>(&'a GpuVm); - -impl UpdatingGpuVm<'_, T> {} - -impl Deref for UpdatingGpuVm<'_, T> { - type Target = T; - - fn deref(&self) -> &T { - // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, - // so this is the only reference - unsafe { &*self.0.inner.get() } - } -} - -impl DerefMut for UpdatingGpuVm<'_, T> { - fn deref_mut(&mut self) -> &mut T { - // SAFETY: The existence of this UpdatingGpuVm implies the lock is held, - // so this is the only reference - unsafe { &mut *self.0.inner.get() } - } -} - -// SAFETY: All our trait methods take locks -unsafe impl Sync for GpuVm {} -// SAFETY: All our trait methods take locks -unsafe impl Send for GpuVm {} - -// SAFETY: All our trait methods take locks -unsafe impl Sync for GpuVmBo {} -// SAFETY: All our trait methods take locks -unsafe impl Send for GpuVmBo {} diff --git a/rust/kernel/drm_neo/ioctl.rs b/rust/kernel/drm_neo/ioctl.rs deleted file mode 100644 index a7c7bf586e87c6..00000000000000 --- a/rust/kernel/drm_neo/ioctl.rs +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM IOCTL definitions. -//! -//! C header: [`include/drm_neo/drm_ioctl.h`](srctree/include/drm_neo/drm_ioctl.h) - -use crate::ioctl; - -const BASE: u32 = uapi::DRM_IOCTL_BASE as u32; - -/// Construct a DRM ioctl number with no argument. -#[allow(non_snake_case)] -#[inline(always)] -pub const fn IO(nr: u32) -> u32 { - ioctl::_IO(BASE, nr) -} - -/// Construct a DRM ioctl number with a read-only argument. -#[allow(non_snake_case)] -#[inline(always)] -pub const fn IOR(nr: u32) -> u32 { - ioctl::_IOR::(BASE, nr) -} - -/// Construct a DRM ioctl number with a write-only argument. -#[allow(non_snake_case)] -#[inline(always)] -pub const fn IOW(nr: u32) -> u32 { - ioctl::_IOW::(BASE, nr) -} - -/// Construct a DRM ioctl number with a read-write argument. -#[allow(non_snake_case)] -#[inline(always)] -pub const fn IOWR(nr: u32) -> u32 { - ioctl::_IOWR::(BASE, nr) -} - -/// Descriptor type for DRM ioctls. Use the `declare_drm_neo_ioctls!{}` macro to construct them. -pub type DrmIoctlDescriptor = bindings::drm_ioctl_desc; - -/// This is for ioctl which are used for rendering, and require that the file descriptor is either -/// for a render node, or if it’s a legacy/primary node, then it must be authenticated. -pub const AUTH: u32 = bindings::drm_ioctl_flags_DRM_AUTH; - -/// This must be set for any ioctl which can change the modeset or display state. Userspace must -/// call the ioctl through a primary node, while it is the active master. -/// -/// Note that read-only modeset ioctl can also be called by unauthenticated clients, or when a -/// master is not the currently active one. -pub const MASTER: u32 = bindings::drm_ioctl_flags_DRM_MASTER; - -/// Anything that could potentially wreak a master file descriptor needs to have this flag set. -/// -/// Current that’s only for the SETMASTER and DROPMASTER ioctl, which e.g. logind can call to -/// force a non-behaving master (display compositor) into compliance. -/// -/// This is equivalent to callers with the SYSADMIN capability. -pub const ROOT_ONLY: u32 = bindings::drm_ioctl_flags_DRM_ROOT_ONLY; - -/// This is used for all ioctl needed for rendering only, for drivers which support render nodes. -/// This should be all new render drivers, and hence it should be always set for any ioctl with -/// `AUTH` set. Note though that read-only query ioctl might have this set, but have not set -/// DRM_AUTH because they do not require authentication. -pub const RENDER_ALLOW: u32 = bindings::drm_ioctl_flags_DRM_RENDER_ALLOW; - -/// Internal structures used by the `declare_drm_neo_ioctls!{}` macro. Do not use directly. -#[doc(hidden)] -pub mod internal { - pub use bindings::drm_device; - pub use bindings::drm_file; - pub use bindings::drm_ioctl_desc; -} - -/// Declare the DRM ioctls for a driver. -/// -/// Each entry in the list should have the form: -/// -/// `(ioctl_number, argument_type, flags, user_callback),` -/// -/// `argument_type` is the type name within the `bindings` crate. -/// `user_callback` should have the following prototype: -/// -/// ```ignore -/// fn foo(device: &kernel::drm_neo::Device, -/// data: &mut uapi::argument_type, -/// file: &kernel::drm_neo::File, -/// ) -> Result -/// ``` -/// where `Self` is the drm_neo::drv::Driver implementation these ioctls are being declared within. -/// -/// # Examples -/// -/// ```ignore -/// kernel::declare_drm_neo_ioctls! { -/// (FOO_GET_PARAM, drm_foo_get_param, ioctl::RENDER_ALLOW, my_get_param_handler), -/// } -/// ``` -/// -#[macro_export] -macro_rules! declare_drm_neo_ioctls { - ( $(($cmd:ident, $struct:ident, $flags:expr, $func:expr)),* $(,)? ) => { - const IOCTLS: &'static [$crate::drm_neo::ioctl::DrmIoctlDescriptor] = { - use $crate::uapi::*; - const _:() = { - let i: u32 = $crate::uapi::DRM_COMMAND_BASE; - // Assert that all the IOCTLs are in the right order and there are no gaps, - // and that the size of the specified type is correct. - $( - let cmd: u32 = $crate::macros::concat_idents!(DRM_IOCTL_, $cmd); - ::core::assert!(i == $crate::ioctl::_IOC_NR(cmd)); - ::core::assert!(core::mem::size_of::<$crate::uapi::$struct>() == - $crate::ioctl::_IOC_SIZE(cmd)); - let i: u32 = i + 1; - )* - }; - - let ioctls = &[$( - $crate::drm_neo::ioctl::internal::drm_ioctl_desc { - cmd: $crate::macros::concat_idents!(DRM_IOCTL_, $cmd) as u32, - func: { - #[allow(non_snake_case)] - unsafe extern "C" fn $cmd( - raw_dev: *mut $crate::drm_neo::ioctl::internal::drm_device, - raw_data: *mut ::core::ffi::c_void, - raw_file: *mut $crate::drm_neo::ioctl::internal::drm_file, - ) -> core::ffi::c_int { - // SAFETY: - // - The DRM core ensures the device lives while callbacks are being - // called. - // - The DRM device must have been registered when we're called through - // an IOCTL. - // - // FIXME: Currently there is nothing enforcing that the types of the - // dev/file match the current driver these ioctls are being declared - // for, and it's not clear how to enforce this within the type system. - let dev = $crate::drm_neo::device::Device::from_raw(raw_dev); - // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we - // asserted above matches the size of this type, and all bit patterns of - // UAPI structs must be valid. - // The `ioctl` argument is exclusively owned by the handler - // and guaranteed by the C implementation (`drm_ioctl()`) to remain - // valid for the entire lifetime of the reference taken here. - // There is no concurrent access or aliasing; no other references - // to this object exist during this call. - let data = unsafe { &mut *(raw_data.cast::<$crate::uapi::$struct>()) }; - // SAFETY: This is just the DRM file structure - let file = unsafe { $crate::drm_neo::File::from_raw(raw_file) }; - - match $func(dev, data, file) { - Err(e) => e.to_errno(), - Ok(i) => i.try_into() - .unwrap_or($crate::error::code::ERANGE.to_errno()), - } - } - Some($cmd) - }, - flags: $flags, - name: $crate::str::as_char_ptr_in_const_context( - $crate::c_str!(::core::stringify!($cmd)), - ), - } - ),*]; - ioctls - }; - }; -} diff --git a/rust/kernel/drm_neo/mm.rs b/rust/kernel/drm_neo/mm.rs deleted file mode 100644 index daa05abbad1580..00000000000000 --- a/rust/kernel/drm_neo/mm.rs +++ /dev/null @@ -1,344 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM MM range allocator -//! -//! C header: [`include/drm_neo/drm_mm.h`](../../../../include/drm_neo/drm_mm.h) - -use crate::{ - alloc::flags::*, - bindings, - error::{code::EINVAL, to_result, Result}, - sync::{new_mutex, Arc, Mutex, UniqueArc}, - types::Opaque, -}; - -use crate::init::InPlaceInit; -use crate::prelude::KBox; - -use core::{ - marker::{PhantomData, PhantomPinned}, - ops::{Deref, Range}, - pin::Pin, -}; - -/// Type alias representing a DRM MM node. -pub type Node = Pin>>; - -/// Trait which must be implemented by the inner allocator state type provided by the user. -pub trait AllocInner { - /// Notification that a node was dropped from the allocator. - fn drop_object(&mut self, _start: u64, _size: u64, _color: usize, _object: &mut T) {} -} - -impl AllocInner for () {} - -/// Wrapper type for a `struct drm_mm` plus user AllocInner object. -/// -/// # Invariants -/// The `drm_mm` struct is valid and initialized. -struct MmInner, T>(Opaque, A, PhantomData); - -/// Represents a single allocated node in the MM allocator -pub struct NodeData, T> { - node: bindings::drm_mm_node, - mm: Arc>>, - valid: bool, - /// A drm_mm_node needs to be pinned because nodes reference each other in a linked list. - _pin: PhantomPinned, - inner: T, -} - -// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. -unsafe impl, T: Send> Send for NodeData {} -// SAFETY: Allocator ops take the mutex, and there are no mutable actions on the node. -unsafe impl, T: Sync> Sync for NodeData {} - -/// Available MM node insertion modes -#[repr(u32)] -pub enum InsertMode { - /// Search for the smallest hole (within the search range) that fits the desired node. - /// - /// Allocates the node from the bottom of the found hole. - Best = bindings::drm_mm_insert_mode_DRM_MM_INSERT_BEST, - - /// Search for the lowest hole (address closest to 0, within the search range) that fits the - /// desired node. - /// - /// Allocates the node from the bottom of the found hole. - Low = bindings::drm_mm_insert_mode_DRM_MM_INSERT_LOW, - - /// Search for the highest hole (address closest to U64_MAX, within the search range) that fits - /// the desired node. - /// - /// Allocates the node from the top of the found hole. The specified alignment for the node is - /// applied to the base of the node (`Node.start()`). - High = bindings::drm_mm_insert_mode_DRM_MM_INSERT_HIGH, - - /// Search for the most recently evicted hole (within the search range) that fits the desired - /// node. This is appropriate for use immediately after performing an eviction scan and removing - /// the selected nodes to form a hole. - /// - /// Allocates the node from the bottom of the found hole. - Evict = bindings::drm_mm_insert_mode_DRM_MM_INSERT_EVICT, -} - -/// A clonable, interlocked reference to the allocator state. -/// -/// This is useful to perform actions on the user-supplied `AllocInner` type given just a Node, -/// without immediately taking the lock. -#[derive(Clone)] -pub struct InnerRef, T>(Arc>>); - -impl, T> InnerRef { - /// Operate on the user `AllocInner` implementation, taking the lock. - pub fn with(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { - let mut l = self.0.lock(); - cb(&mut l.1) - } -} - -impl, T> NodeData { - /// Returns the color of the node (an opaque value) - pub fn color(&self) -> usize { - self.node.color as usize - } - - /// Returns the start address of the node - pub fn start(&self) -> u64 { - self.node.start - } - - /// Returns the size of the node in bytes - pub fn size(&self) -> u64 { - self.node.size - } - - /// Operate on the user `AllocInner` implementation associated with this node's allocator. - pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { - let mut l = self.mm.lock(); - cb(&mut l.1) - } - - /// Return a clonable, detached reference to the allocator inner data. - pub fn alloc_ref(&self) -> InnerRef { - InnerRef(self.mm.clone()) - } - - /// Return a mutable reference to the inner data. - pub fn inner_mut(self: Pin<&mut Self>) -> &mut T { - // SAFETY: This is okay because inner is not structural - unsafe { &mut self.get_unchecked_mut().inner } - } -} - -impl, T> Deref for NodeData { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl, T> Drop for NodeData { - fn drop(&mut self) { - if self.valid { - let mut guard = self.mm.lock(); - - // Inform the user allocator that a node is being dropped. - guard - .1 - .drop_object(self.start(), self.size(), self.color(), &mut self.inner); - // SAFETY: The MM lock is still taken, so we can safely remove the node. - unsafe { bindings::drm_mm_remove_node(&mut self.node) }; - } - } -} - -/// An instance of a DRM MM range allocator. -pub struct Allocator, T> { - mm: Arc>>, - _p: PhantomData, -} - -impl, T> Allocator { - /// Return the first allocator reservation intersecting an address range. - /// Unlike a page-table walk, this includes unmapped guard space retained - /// inside a node. The interval query and its result are protected by the - /// allocator mutex; no probe node is inserted into the live VA space. - pub fn first_overlap(&self, range: Range) -> Result>> { - if range.start >= range.end { - return Err(EINVAL); - } - let guard = self.mm.lock(); - // The DRM interval query requires both endpoints to lie inside its - // allocator. head_node is the end sentinel; its wrapping size is the - // negated allocator span. - let head = unsafe { &(*guard.0.get()).head_node }; - let allocator_start = head.start.wrapping_add(head.size); - if range.start < allocator_start || range.end > head.start { - return Err(EINVAL); - } - // SAFETY: The allocator is initialized for the lifetime of `self`. - // Its mutex prevents node removal while the returned pointer is read. - // drm_mm returns its end sentinel when no interval intersects. - let node = unsafe { - bindings::__drm_mm_interval_first( - guard.0.get(), range.start, range.end - 1, - ) - }; - // SAFETY: drm_mm_interval_first returns a valid node or the allocator - // end sentinel, both of which remain live while `guard` is held. - let node = unsafe { node.as_ref().ok_or(EINVAL)? }; - if node.start >= range.end { - return Ok(None); - } - Ok(Some(node.start..node.start.saturating_add(node.size))) - } - - /// Create a new range allocator for the given start and size range of addresses. - /// - /// The user may optionally provide an inner object representing allocator state, which will - /// be protected by the same lock. If not required, `()` can be used. - #[track_caller] - pub fn new(start: u64, size: u64, inner: A) -> Result> { - // SAFETY: We call `Mutex::init_lock` below. - let mm = UniqueArc::pin_init( - new_mutex!(MmInner(Opaque::uninit(), inner, PhantomData)), - GFP_KERNEL, - )?; - - // SAFETY: The Opaque instance provides a valid pointer, and it is initialized after - // this call. - unsafe { - bindings::drm_mm_init(mm.lock().0.get(), start, size); - } - - Ok(Allocator { - mm: mm.into(), - _p: PhantomData, - }) - } - - /// Insert a new node into the allocator of a given size. - /// - /// `node` is the user `T` type data to store into the node. - pub fn insert_node(&mut self, node: T, size: u64) -> Result> { - self.insert_node_generic(node, size, 0, 0, InsertMode::Best) - } - - /// Insert a new node into the allocator of a given size, with configurable alignment, - /// color, and insertion mode. - /// - /// `node` is the user `T` type data to store into the node. - pub fn insert_node_generic( - &mut self, - node: T, - size: u64, - alignment: u64, - color: usize, - mode: InsertMode, - ) -> Result> { - self.insert_node_in_range(node, size, alignment, color, 0, u64::MAX, mode) - } - - /// Insert a new node into the allocator of a given size, with configurable alignment, - /// color, insertion mode, and sub-range to allocate from. - /// - /// `node` is the user `T` type data to store into the node. - #[allow(clippy::too_many_arguments)] - pub fn insert_node_in_range( - &mut self, - node: T, - size: u64, - alignment: u64, - color: usize, - start: u64, - end: u64, - mode: InsertMode, - ) -> Result> { - let mut mm_node = KBox::new( - NodeData { - // SAFETY: This C struct should be zero-initialized. - node: unsafe { core::mem::zeroed() }, - valid: false, - inner: node, - mm: self.mm.clone(), - _pin: PhantomPinned, - }, - GFP_KERNEL, - )?; - - let guard = self.mm.lock(); - // SAFETY: We hold the lock and all pointers are valid. - to_result(unsafe { - bindings::drm_mm_insert_node_in_range( - guard.0.get(), - &mut mm_node.node, - size, - alignment, - color, - start, - end, - mode as u32, - ) - })?; - - mm_node.valid = true; - - Ok(Pin::from(mm_node)) - } - - /// Insert a node into the allocator at a fixed start address. - /// - /// `node` is the user `T` type data to store into the node. - pub fn reserve_node( - &mut self, - node: T, - start: u64, - size: u64, - color: usize, - ) -> Result> { - let mut mm_node = KBox::new( - NodeData { - // SAFETY: This C struct should be zero-initialized. - node: unsafe { core::mem::zeroed() }, - valid: false, - inner: node, - mm: self.mm.clone(), - _pin: PhantomPinned, - }, - GFP_KERNEL, - )?; - - mm_node.node.start = start; - mm_node.node.size = size; - mm_node.node.color = color as crate::ffi::c_ulong; - - let guard = self.mm.lock(); - // SAFETY: We hold the lock and all pointers are valid. - to_result(unsafe { bindings::drm_mm_reserve_node(guard.0.get(), &mut mm_node.node) })?; - - mm_node.valid = true; - - Ok(Pin::from(mm_node)) - } - - /// Operate on the inner user type `A`, taking the allocator lock - pub fn with_inner(&self, cb: impl FnOnce(&mut A) -> RetVal) -> RetVal { - let mut guard = self.mm.lock(); - cb(&mut guard.1) - } -} - -impl, T> Drop for MmInner { - fn drop(&mut self) { - // SAFETY: If the MmInner is dropped then all nodes are gone (since they hold references), - // so it is safe to tear down the allocator. - unsafe { - bindings::drm_mm_takedown(self.0.get()); - } - } -} - -// SAFETY: MmInner is safely Send if the AllocInner user type is Send. -unsafe impl, T> Send for MmInner {} diff --git a/rust/kernel/drm_neo/mod.rs b/rust/kernel/drm_neo/mod.rs deleted file mode 100644 index 882841415aa414..00000000000000 --- a/rust/kernel/drm_neo/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM subsystem abstractions. - -pub mod device; -pub mod driver; -pub mod file; -pub mod gem; -#[cfg(CONFIG_DRM_GPUVM = "y")] -pub mod gpuvm; -pub mod ioctl; -pub mod mm; -pub mod sched; -pub mod syncobj; - -pub use self::device::Device; -pub use self::driver::Driver; -pub use self::driver::DriverInfo; -pub use self::driver::Registration; -pub use self::file::File; - -pub(crate) mod private { - pub trait Sealed {} -} diff --git a/rust/kernel/drm_neo/sched.rs b/rust/kernel/drm_neo/sched.rs deleted file mode 100644 index 910385f16cb556..00000000000000 --- a/rust/kernel/drm_neo/sched.rs +++ /dev/null @@ -1,549 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM Scheduler -//! -//! C header: [`include/drm_neo/gpu_scheduler.h`](../../../../include/drm_neo/gpu_scheduler.h) - -use crate::{ - bindings, device, - dma_fence::*, - error::{to_result, Result}, - prelude::*, - sync::{Arc, UniqueArc}, - time::{self, msecs_to_jiffies}, -}; -use core::marker::PhantomData; -use core::mem::MaybeUninit; -use core::ops::{Deref, DerefMut}; -use core::ptr::{addr_of, addr_of_mut}; - -/// Scheduler status after timeout recovery -#[repr(u32)] -pub enum Status { - /// Device recovered from the timeout and can execute jobs again - Nominal = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_RESET, - /// Device is no longer available - NoDevice = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_ENODEV, - /// Reinsert the job and rearm its timeout without resetting the scheduler. - NoHang = bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG, -} - -/// Scheduler priorities -#[repr(u32)] -pub enum Priority { - /// Low userspace priority - Low = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_LOW, - /// Normal userspace priority - Normal = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_NORMAL, - /// High userspace priority - High = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_HIGH, - /// Kernel priority (highest) - Kernel = bindings::drm_sched_priority_DRM_SCHED_PRIORITY_KERNEL, -} - -/// Trait to be implemented by driver job objects. -pub trait JobImpl: Sized { - /// State owned exclusively by the timeout worker. It must not borrow the - /// mutable submission state: timeout can run before `run` returns. - type TimeoutData; - - /// Snapshot or independently retain timeout state before scheduler ownership. - fn timeout_data(&self) -> Self::TimeoutData; - - /// Module implementing the callbacks. Its caller must retain the module - /// through entity destruction; the core pins detached kill-job callbacks. - const MODULE: Option<&'static crate::ThisModule> = None; - - /// Handle a scoped timeout without resetting the C scheduler. - /// - /// The backend must settle/quarantine its exact hardware work, or retain - /// that owner until an in-progress host transaction can classify it. It - /// must not invoke drm_sched_stop/start or replay scheduler jobs. The core - /// reinserts the detached guilty job through NO_HANG, preserving every - /// parent callback and the backend's actual success/error fence result. - const MANAGED_TIMEOUT_RECOVERY: bool = false; - - /// Reconsider a timeout before stopping the scheduler or settling fences. - /// Returning true asks the core to reinsert the retained job and rearm. - /// The backend must not stop/start the scheduler from this hook. - fn false_timeout(_data: &mut Self::TimeoutData, _finished: bool) -> bool { false } - - - /// Called when the scheduler is considering scheduling this job next, to get another Fence - /// for this job to block on. Once it returns None, run() may be called. - fn prepare(_job: &mut Self) -> Option { - None // Equivalent to NULL function pointer - } - - /// Called to execute the job once all of the dependencies have been resolved. This may be - /// called multiple times, if timed_out() has happened and drm_sched_job_recovery() decides - /// to try it again. - fn run(job: &mut Self) -> Result>; - - /// Called when a job has taken too long to execute, to trigger GPU recovery. - /// - /// This method is called in a workqueue context. - fn timed_out(data: &mut Self::TimeoutData, finished: bool) -> Status; - - /// Called for remaining jobs in drm_sched_fini() to ensure the job's fences - /// get signalled before the scheduler is torn down. - fn cancel(job: &mut Self); -} - -unsafe extern "C" fn prepare_job_cb( - sched_job: *mut bindings::drm_sched_job, - _s_entity: *mut bindings::drm_sched_entity, -) -> *mut bindings::dma_fence { - // SAFETY: All of our jobs are Job. - let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - - // SAFETY: prepare/run are serialized by submit work. Only borrow inner; - // the timeout worker independently owns timeout_data and C owns job. - match T::prepare(unsafe { &mut *addr_of_mut!((*p).inner) }) { - None => core::ptr::null_mut(), - Some(fence) => fence.into_raw(), - } -} - -unsafe extern "C" fn run_job_cb( - sched_job: *mut bindings::drm_sched_job, -) -> *mut bindings::dma_fence { - // SAFETY: All of our jobs are Job. - let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - - // SAFETY: submit work owns inner until return, even while timeout work - // accesses the disjoint timeout_data field. Do not borrow the whole Job. - match T::run(unsafe { &mut *addr_of_mut!((*p).inner) }) { - Err(e) => e.to_ptr(), - Ok(None) => core::ptr::null_mut(), - Ok(Some(fence)) => fence.into_raw(), - } -} - -unsafe extern "C" fn timedout_job_cb( - sched_job: *mut bindings::drm_sched_job, -) -> bindings::drm_gpu_sched_stat { - // SAFETY: All of our jobs are Job. - let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - - // The C timeout worker owns this job outside pending_list until return. - // Free work cannot release it even if its parent signals concurrently. - let finished = unsafe { addr_of_mut!((*(*sched_job).s_fence).finished) }; - let is_finished = || unsafe { bindings::dma_fence_get_status(finished) != 0 }; - if T::MANAGED_TIMEOUT_RECOVERY { - if is_finished() { - return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; - } - } - // SAFETY: the C core retains the detached job and serializes its timeout - // worker. This field is disjoint from inner, which run may still mutate. - let data = unsafe { &mut *addr_of_mut!((*p).timeout_data) }; - if T::false_timeout(data, is_finished()) { - return bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG; - } - let status = T::timed_out(data, is_finished()); - if T::MANAGED_TIMEOUT_RECOVERY { - // This is scoped fence handling, not C reset recovery. stop would - // detach and discard healthy parents; start would cancel those jobs. - // NO_HANG reinserts the exact job and queues free work if it completed - // during the callback. It does not change the parent's error or replay - // work. A deferred exact owner remains eligible for its real callback. - bindings::drm_gpu_sched_stat_DRM_GPU_SCHED_STAT_NO_HANG - } else { - status as bindings::drm_gpu_sched_stat - } -} - -unsafe extern "C" fn free_job_cb(sched_job: *mut bindings::drm_sched_job) { - // SAFETY: All of our jobs are Job. - let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - - // Fault-only ownership witness: this callback is where the core returns - // ownership of an errored job. A signalled fence alone is not that proof. - // SAFETY: A job reaching free_job has a live scheduler fence; its error - // was fixed before signal and the core's completion callback has finished. - if T::MANAGED_TIMEOUT_RECOVERY - && unsafe { (*(*sched_job).s_fence).finished.error } == ETIMEDOUT.to_errno() - { - // Use numeric identity: the fmt adapter borrows arguments, and - // Pointer formatting of that reference can print a stack address. - pr_info!("DRM Rust scheduler: released timed-out job={:#x} context={} seqno={}\n", - sched_job as usize, - unsafe { (*(*sched_job).s_fence).finished.context }, - unsafe { (*(*sched_job).s_fence).finished.seqno }); - } - // Convert the job back to a Box and drop it - // SAFETY: All of our Jobs are created inside a box. - unsafe { drop(KBox::from_raw(p)) }; -} - -unsafe extern "C" fn cancel_job_cb(sched_job: *mut bindings::drm_sched_job) { - // SAFETY: All of our jobs are Job. - let p = unsafe { crate::container_of!(sched_job, Job, job) as *mut Job }; - - // fini has stopped/synchronized submission and timeout workers. A parent - // callback can still run on the device completion thread. The scheduler - // fence retains the parent reference while we remove its callback under - // the parent's lock; that also waits for an already-running callback. - let s_fence = unsafe { (*sched_job).s_fence }; - let parent = unsafe { (*s_fence).parent }; - let removed_parent_callback = if !parent.is_null() { - // SAFETY: run_job installed (or attempted to install on an already - // signaled parent) this callback before fini synchronized submit work. - unsafe { bindings::dma_fence_remove_callback(parent, addr_of_mut!((*sched_job).cb)) } - } else { false }; - let finish_here = removed_parent_callback || (parent.is_null() - && unsafe { bindings::dma_fence_get_status(addr_of_mut!((*s_fence).finished)) == 0 }); - // Null/error run_job calls job_done before submit work returns, so its - // finished fence is already signaled. NULL+unfinished can instead survive - // a legacy backend's prior reset stop: C stop dropped the parent and its - // credits, but not its score. Do not subtract those credits twice. - - // SAFETY: cancellation cannot race a scheduler callback using this job - // after the removal/parent-lock synchronization above. The backend must - // retain any still-hardware-owned resources independently of Job storage. - T::cancel(unsafe { &mut *addr_of_mut!((*p).inner) }); - - if finish_here { - // Match drm_sched_job_done's accounting exactly once, only when its - // callback did not do it. Submission/free work is stopped, so fini - // itself unlinks and frees this job; no workqueue rearm is appropriate. - let sched = unsafe { (*sched_job).sched }; - unsafe { - if removed_parent_callback { - bindings::atomic_sub((*sched_job).credits as i32, - addr_of_mut!((*sched).credit_count)); - } - bindings::atomic_sub(1, (*sched).score); - } - let fence = unsafe { Fence::get_raw(addr_of_mut!((*s_fence).finished)) }; - fence.set_error(ECANCELED); - let _ = fence.signal(); - } - // Do not clear/put parent here: s_fence still owns that reference and its - // normal release path drops it. A completed callback's result is retained. -} - -/// A DRM scheduler job. -pub struct Job { - job: bindings::drm_sched_job, - inner: T, - timeout_data: T::TimeoutData, - // Entity destruction can leave kill_jobs_work queued on the system - // workqueue or waiting for a dependency. That work still dereferences - // job->sched->ops after drm_sched_fini has stopped the scheduler workers. - // Retain only the allocation: retaining SchedulerOwner here would keep - // fini from running while pending jobs still need its cancellation. - _scheduler: Arc>, -} - -impl Job { - /// Returns whether an armed job has finished, successfully or not. - pub fn is_finished(&self) -> bool { - if self.job.sched.is_null() { - return false; - } - // SAFETY: An armed job has an initialized scheduler fence, retained - // for its lifetime, including while the timeout worker owns the job. - unsafe { bindings::dma_fence_get_status(addr_of_mut!((*self.job.s_fence).finished)) != 0 } - } -} - -impl Deref for Job { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl DerefMut for Job { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.inner - } -} - -impl Drop for Job { - fn drop(&mut self) { - // SAFETY: At this point the job has either been submitted and this is being called from - // `free_job_cb` above, or it hasn't and it is safe to call `drm_sched_job_cleanup`. - unsafe { bindings::drm_sched_job_cleanup(&mut self.job) }; - } -} - -/// A pending DRM scheduler job (not yet armed) -pub struct PendingJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); - -impl<'a, T: JobImpl> PendingJob<'a, T> { - /// Add a fence as a dependency to the job - pub fn add_dependency(&mut self, fence: Fence) -> Result { - // SAFETY: C call with correct arguments - to_result(unsafe { - bindings::drm_sched_job_add_dependency(&mut self.0.job, fence.into_raw()) - }) - } - - /// Arm the job to make it ready for execution - pub fn arm(mut self) -> ArmedJob<'a, T> { - // SAFETY: C call with correct arguments - unsafe { bindings::drm_sched_job_arm(&mut self.0.job) }; - ArmedJob(self.0, PhantomData) - } -} - -impl<'a, T: JobImpl> Deref for PendingJob<'a, T> { - type Target = Job; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl<'a, T: JobImpl> DerefMut for PendingJob<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -/// An armed DRM scheduler job (not yet submitted) -pub struct ArmedJob<'a, T: JobImpl>(KBox>, PhantomData<&'a T>); - -impl<'a, T: JobImpl> ArmedJob<'a, T> { - /// Returns the job fences - pub fn fences(&mut self) -> JobFences<'_> { - // SAFETY: s_fence is always a valid drm_sched_fence pointer - JobFences(unsafe { &mut *self.0.job.s_fence }) - } - - /// Push the job for execution into the scheduler - pub fn push(self) { - // After this point, the job is submitted and owned by the scheduler - let ptr = match self { - ArmedJob(job, _) => KBox::>::into_raw(job), - }; - - // SAFETY: We are passing in ownership of a valid Box raw pointer. - unsafe { bindings::drm_sched_entity_push_job(addr_of_mut!((*ptr).job)) }; - } -} -impl<'a, T: JobImpl> Deref for ArmedJob<'a, T> { - type Target = Job; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl<'a, T: JobImpl> DerefMut for ArmedJob<'a, T> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -/// Reference to the bundle of fences attached to a DRM scheduler job -pub struct JobFences<'a>(&'a mut bindings::drm_sched_fence); - -impl<'a> JobFences<'a> { - /// Returns a new reference to the job scheduled fence. - pub fn scheduled(&mut self) -> Fence { - // SAFETY: self.0.scheduled is always a valid fence - unsafe { Fence::get_raw(&mut self.0.scheduled) } - } - - /// Returns a new reference to the job finished fence. - pub fn finished(&mut self) -> Fence { - // SAFETY: self.0.finished is always a valid fence - unsafe { Fence::get_raw(&mut self.0.finished) } - } -} - -struct EntityInner { - entity: bindings::drm_sched_entity, - // TODO: Allow users to share guilty flag between entities - sched: Arc>, - guilty: bindings::atomic_t, - _p: PhantomData, -} - -impl Drop for EntityInner { - fn drop(&mut self) { - // SAFETY: The EntityInner is initialized. Jobs removed from the entity - // may still await kill_jobs_work; each retains the scheduler allocation. - unsafe { bindings::drm_sched_entity_destroy(&mut self.entity) }; - } -} - -// SAFETY: TODO -unsafe impl Sync for EntityInner {} -// SAFETY: TODO -unsafe impl Send for EntityInner {} - -/// A DRM scheduler entity. -pub struct Entity(Pin>>); - -impl Entity { - /// Create a new scheduler entity. - pub fn new(sched: &Scheduler, priority: Priority) -> Result { - let mut entity: KBox>> = - KBox::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - - let mut sched_ptr = &sched.0.allocation.sched as *const _ as *mut _; - - // SAFETY: The Box is allocated above and valid. - unsafe { - bindings::drm_sched_entity_init( - addr_of_mut!((*entity.as_mut_ptr()).entity), - priority as _, - &mut sched_ptr, - 1, - addr_of_mut!((*entity.as_mut_ptr()).guilty), - ) - }; - - // SAFETY: The Box is allocated above and valid. - unsafe { addr_of_mut!((*entity.as_mut_ptr()).sched).write(sched.0.clone()) }; - - // SAFETY: entity is now initialized. - Ok(Self(Pin::from(unsafe { entity.assume_init() }))) - } - - /// Create a new job on this entity. - /// - /// The entity must outlive the pending job until it transitions into the submitted state, - /// after which the scheduler owns it. Since jobs must be submitted in creation order, - /// this requires a mutable reference to the entity, ensuring that only one new job can be - /// in flight at once. - pub fn new_job(&mut self, credits: u32, inner: T) -> Result> { - let mut job: KBox>> = Box::new_uninit(GFP_KERNEL | __GFP_ZERO)?; - let timeout_data = inner.timeout_data(); - - // SAFETY: We hold a reference to the entity (which is a valid pointer), - // and the job object was just allocated above. - to_result(unsafe { - bindings::drm_sched_job_init( - addr_of_mut!((*job.as_mut_ptr()).job), - &self.0.as_ref().get_ref().entity as *const _ as *mut _, - credits, - core::ptr::null_mut(), - 0, - ) - })?; - - // SAFETY: The Box pointer is valid, and this initializes the inner member. - unsafe { addr_of_mut!((*job.as_mut_ptr()).inner).write(inner) }; - // SAFETY: initialize the independent timeout worker's owned state. - unsafe { addr_of_mut!((*job.as_mut_ptr()).timeout_data).write(timeout_data) }; - - // SAFETY: The job allocation is valid and this initializes its last - // field. Arc::clone is infallible and the entity still owns the live - // scheduler, so every successfully initialized job retains its C - // callback target before it can be armed or submitted. - unsafe { - addr_of_mut!((*job.as_mut_ptr())._scheduler) - .write(self.0.as_ref().get_ref().sched.allocation.clone()) - }; - - // SAFETY: All fields of the Job are now initialized. - Ok(PendingJob(unsafe { job.assume_init() }, PhantomData)) - } -} - -/// DRM scheduler allocation, retained by live owners and deferred job callbacks. -pub struct SchedulerInner { - sched: bindings::drm_gpu_scheduler, - _p: PhantomData, -} - -// Only public scheduler/entity handles keep this owner alive. Jobs retain -// SchedulerInner instead, so outstanding work cannot prevent finalization. -struct SchedulerOwner { - allocation: Arc>, -} - -impl Drop for SchedulerOwner { - fn drop(&mut self) { - // SAFETY: fini stops submission, synchronizes the timeout worker and - // cancels/frees remaining jobs through our synchronized cancel callback. - // The C contract explicitly prohibits drm_sched_stop before fini. - // Deferred entity-kill callbacks are on a different workqueue and may - // survive this call. Their job guards retain allocation through cleanup. - unsafe { - bindings::drm_sched_fini(addr_of!(self.allocation.sched).cast_mut()) - }; - } -} - -// SAFETY: TODO -unsafe impl Sync for SchedulerInner {} -// SAFETY: TODO -unsafe impl Send for SchedulerInner {} - -/// A DRM Scheduler -pub struct Scheduler(Arc>); - -impl Scheduler { - const OPS: bindings::drm_sched_backend_ops = bindings::drm_sched_backend_ops { - prepare_job: Some(prepare_job_cb::), - run_job: Some(run_job_cb::), - timedout_job: Some(timedout_job_cb::), - free_job: Some(free_job_cb::), - cancel_job: Some(cancel_job_cb::), - // A terminal backend result must not orphan the detached Job. The - // core restores ownership after all Rust callback borrows end. - retain_job_on_enodev: true, - owner: match T::MODULE { - Some(module) => module.as_ptr(), - None => core::ptr::null_mut(), - }, - }; - /// Creates a new DRM Scheduler object - // TODO: Shared timeout workqueues & scores - pub fn new( - device: &device::Device, - num_rqs: u32, - credit_limit: u32, - hang_limit: u32, - timeout_ms: time::Msecs, - name: &'static CStr, - ) -> Result> { - let mut sched: UniqueArc>> = - UniqueArc::new_uninit(GFP_KERNEL)?; - // Allocate the owner before starting C workers: no fallible allocation - // may abandon an initialized scheduler without running drm_sched_fini. - let owner: UniqueArc>> = - UniqueArc::new_uninit(GFP_KERNEL)?; - - // SAFETY: zero sched->sched_rq as drm_sched_init() uses it to exit early withoput initialisation - // TODO: allocate sched zzeroed instead - unsafe { - (*sched.as_mut_ptr()).sched.sched_rq = core::ptr::null_mut(); - }; - - let init_ops = bindings::drm_sched_init_args { - ops: &Self::OPS, - submit_wq: core::ptr::null_mut(), - timeout_wq: core::ptr::null_mut(), - num_rqs, - credit_limit, - hang_limit, - timeout: msecs_to_jiffies(timeout_ms).try_into()?, - score: core::ptr::null_mut(), - name: name.as_char_ptr(), - dev: device.as_raw(), - }; - - // SAFETY: The drm_sched pointer is valid and pinned as it was just allocated above. - // `device` is valid by its type invarants - to_result(unsafe { - bindings::drm_sched_init( - addr_of_mut!((*sched.as_mut_ptr()).sched), - addr_of!(init_ops), - ) - })?; - - // SAFETY: C initialized sched and PhantomData has no stored state. - // No fallible operation follows successful initialization. - let allocation = unsafe { sched.assume_init() }.into(); - Ok(Scheduler(owner.write(SchedulerOwner { allocation }).into())) - } -} diff --git a/rust/kernel/drm_neo/syncobj.rs b/rust/kernel/drm_neo/syncobj.rs deleted file mode 100644 index 1aff064f43ecc5..00000000000000 --- a/rust/kernel/drm_neo/syncobj.rs +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT - -//! DRM Sync Objects -//! -//! C header: [`include/drm_neo/drm_syncobj.h`](../../../../include/drm_neo/drm_syncobj.h) - -use crate::{bindings, dma_fence::*, drm_neo, error::Result, prelude::*}; - -/// A DRM Sync Object -/// -/// # Invariants -/// ptr is a valid pointer to a drm_syncobj and we own a reference to it. -pub struct SyncObj { - ptr: *mut bindings::drm_syncobj, -} - -impl SyncObj { - /// Looks up a sync object by its handle for a given `File`. - pub fn lookup_handle( - file: &drm_neo::File, - handle: u32, - ) -> Result { - // SAFETY: The arguments are all valid per the type invariants. - let ptr = unsafe { bindings::drm_syncobj_find(file.as_raw() as *mut _, handle) }; - - if ptr.is_null() { - Err(ENOENT) - } else { - Ok(SyncObj { ptr }) - } - } - - /// Returns the DMA fence associated with this sync object, if any. - pub fn fence_get(&self) -> Option { - // SAFETY: self.ptr is always valid - let fence = unsafe { bindings::drm_syncobj_fence_get(self.ptr) }; - if fence.is_null() { - None - } else { - // SAFETY: The pointer is non-NULL and drm_syncobj_fence_get acquired an - // additional reference. - Some(unsafe { Fence::from_raw(fence) }) - } - } - - /// Replaces the DMA fence with a new one, or removes it if fence is None. - pub fn replace_fence(&self, fence: Option<&Fence>) { - // SAFETY: All arguments should be valid per the respective type invariants. - unsafe { - bindings::drm_syncobj_replace_fence( - self.ptr, - fence.map_or(core::ptr::null_mut(), |a| a.raw()), - ) - }; - } - - /// Adds a new timeline point to the syncobj. - pub fn add_point(&self, chain: FenceChain, fence: &Fence, point: u64) { - // SAFETY: All arguments should be valid per the respective type invariants. - // This takes over the FenceChain ownership. - unsafe { bindings::drm_syncobj_add_point(self.ptr, chain.into_raw(), fence.raw(), point) }; - } -} - -impl Drop for SyncObj { - fn drop(&mut self) { - // SAFETY: We own a reference to this syncobj. - unsafe { bindings::drm_syncobj_put(self.ptr) }; - } -} - -impl Clone for SyncObj { - fn clone(&self) -> Self { - // SAFETY: `ptr` is valid per the type invariant and we own a reference to it. - unsafe { bindings::drm_syncobj_get(self.ptr) }; - SyncObj { ptr: self.ptr } - } -} - -// SAFETY: drm_syncobj operations are internally locked. -unsafe impl Sync for SyncObj {} -// SAFETY: drm_syncobj operations are internally locked. -unsafe impl Send for SyncObj {} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 20b12c1497d988..a40585a222baab 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -86,8 +86,6 @@ pub mod dma_fence; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; -#[cfg(CONFIG_DRM_ASAHI_NEO)] -pub mod drm_neo; pub mod error; pub mod faux; #[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)] diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 3f1bf3477224dc..512ac0aea08fde 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -8,7 +8,6 @@ #include #include -#include #include #include #include diff --git a/tools/asahi/neo-registration-tests.py b/tools/asahi/neo-registration-tests.py deleted file mode 100644 index e5e276640d8f4e..00000000000000 --- a/tools/asahi/neo-registration-tests.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0-only OR MIT -"""Exercise display registration, SoC admission and registration failure cleanup.""" -import argparse -from pathlib import Path -import re -import subprocess -import tempfile - -ROOT = Path(__file__).resolve().parents[2] - - -def function(source, name): - match = re.search(r'^(?:static )?(?:int|void) __init ' + name + r'\(void\)', source, re.M) - assert match, name - end = source.index('\n}', match.end()) + 2 - return source[match.start():end] - - -def driver_name(source, symbol): - start = source.index('static struct platform_driver ' + symbol + ' =') - end = source.index('\n};', start) - return re.search(r'\.name\s*=\s*"([^"]+)"', source[start:end])[1] - - -def check(root): - legacy = root / 'drivers/gpu/drm/apple' - neo = root / 'drivers/gpu/drm/apple-neo' - files = {str(p): p.read_text() for d in (legacy, neo) - for p in (d/'audio.c', d/'dcp.c', d/'apple_drv.c')} - code = r''' -#include -#include -#include -#include -#include -#define __init -#define IS_ENABLED(x) x -#define CONFIG_DRM_APPLE_AUDIO AUDIO -#define CONFIG_DRM_APPLE_NEO_AUDIO AUDIO -struct platform_driver { const char *name; }; -static struct platform_driver drivers[6]; -static struct platform_driver *registered[6]; -static unsigned count, attempts, unregisters; -static int fail_at = -1; -static bool t8140, firmware_only; -static bool of_machine_is_compatible(const char *name) { - assert(!strcmp(name,"apple,t8140")); return t8140; -} -static bool drm_firmware_drivers_only(void) { return firmware_only; } -static int platform_driver_register(struct platform_driver *driver) { - if ((int)attempts++ == fail_at) return -EIO; - for (unsigned i=0;iname,registered[i]->name)) return -EBUSY; - assert(count<6);registered[count++]=driver;return 0; -} -static void platform_driver_unregister(struct platform_driver *driver) { - for (unsigned i=0;i Date: Thu, 15 Oct 2026 08:49:52 +0200 Subject: [PATCH 566/631] drm/apple: av: recycle AV endpoint service slots The AV endpoint (0x29) announces a fresh DCPAVSimpleVideoInterface and DCPAVAudioInterface on new channels for every DPTX connection, so every display replug or DPMS wake that drops HPD uses two more slots. Since "drm/apple: pin DP service owners before recycling retired AFK slots" a torn-down slot is only reclaimed if its ops opt in with .reusable, which the AV services never did. After eight connections the 16-slot table is full, every later announcement fails with AFK[ep:29]: no reusable service slots and the audio service of that port is not re-registered until reboot. Convert the AV owners and opt in to reuse: - DCPAVSimpleVideoInterface keeps no pointer to its service. - asrv->srv now holds an owner pin taken in init and dropped in teardown. A teardown of an older generation no longer clears the pointer or unplugs the jack of its successor. - av_service_connect() and av_service_disconnect() call into the service after dropping srv_rwsem, so they pin it for the duration. Add afk_service_pin() for this; unlike afk_service_get() it also succeeds after teardown, so the close is still sent. Add a KUnit case for a close pinned across teardown. Signed-off-by: Thomas Glanzmann --- drivers/gpu/drm/apple/afk-service-test.c | 18 ++++++++++ drivers/gpu/drm/apple/afk.h | 19 ++++++++++ drivers/gpu/drm/apple/av.c | 46 ++++++++++++++++++------ 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/apple/afk-service-test.c b/drivers/gpu/drm/apple/afk-service-test.c index 1cf589ae7a4172..75a352bec0481c 100644 --- a/drivers/gpu/drm/apple/afk-service-test.c +++ b/drivers/gpu/drm/apple/afk-service-test.c @@ -101,6 +101,23 @@ static void afk_service_completed_reader_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); } +static void afk_service_close_pin_test(struct kunit *test) +{ + struct apple_epic_service *service = test->priv; + + KUNIT_EXPECT_PTR_EQ(test, afk_service_pin(NULL), NULL); + KUNIT_ASSERT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 1)); + KUNIT_ASSERT_PTR_EQ(test, afk_service_get(service), service); /* owner */ + afk_test_retire(service); + /* A close issued after teardown still pins the owner's service. */ + KUNIT_ASSERT_PTR_EQ(test, afk_service_pin(service), service); + afk_service_put(service); /* owner detaches */ + KUNIT_EXPECT_FALSE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); + KUNIT_EXPECT_EQ(test, service->channel, 1U); + afk_service_put(service); + KUNIT_EXPECT_TRUE(test, afk_service_reinit(service, NULL, &reusable_ops, 2)); +} + static void afk_service_tag_sequence_test(struct kunit *test) { struct apple_epic_service *service = test->priv; @@ -226,6 +243,7 @@ static struct kunit_case afk_service_cases[] = { KUNIT_CASE(afk_service_reader_pin_test), KUNIT_CASE(afk_service_pending_reply_test), KUNIT_CASE(afk_service_completed_reader_test), + KUNIT_CASE(afk_service_close_pin_test), KUNIT_CASE(afk_service_tag_sequence_test), KUNIT_CASE(afk_service_unconverted_owner_test), { } diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 94926860ba33f6..1186ec19f2f758 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -159,6 +159,25 @@ static inline void afk_service_put(struct apple_epic_service *service) afk_service_retry(service->ep); } +/* + * Take another pin on a service the caller already keeps pinned, e.g. an + * owner pointer read under the owner's lock. Unlike afk_service_get() this + * also succeeds after teardown, so a close can still be issued. + */ +static inline struct apple_epic_service *afk_service_pin(struct apple_epic_service *service) +{ + unsigned long flags; + + if (!service) + return NULL; + + spin_lock_irqsave(&service->lock, flags); + service->users++; + spin_unlock_irqrestore(&service->lock, flags); + + return service; +} + static inline void afk_service_disable(struct apple_epic_service *service) { unsigned long flags; diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 563a02bb4e5519..f79fbd8e32614d 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -96,12 +96,21 @@ static void av_audiosrv_init(struct apple_epic_service *service, const char *nam { struct apple_dcp *dcp = service->ep->dcp; struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *old; + + /* The owner pin keeps the slot from being recycled while published. */ + service = afk_service_get(service); + if (!service) + return; mutex_lock(&asrv->plug_lock); down_write(&asrv->srv_rwsem); + old = asrv->srv; asrv->srv = service; up_write(&asrv->srv_rwsem); + if (old) + afk_service_put(old); asrv->plugged = true; mutex_unlock(&asrv->plug_lock); @@ -112,18 +121,27 @@ static void av_audiosrv_teardown(struct apple_epic_service *service) { struct apple_dcp *dcp = service->ep->dcp; struct audiosrv_data *asrv = dcp->audiosrv; + bool owned; mutex_lock(&asrv->plug_lock); down_write(&asrv->srv_rwsem); - asrv->srv = NULL; + owned = asrv->srv == service; + if (owned) + asrv->srv = NULL; up_write(&asrv->srv_rwsem); - asrv->plugged = false; - if (asrv->audio_dev) - dcpaud_disconnect(asrv->audio_dev); + /* A stale generation must not unplug its successor. */ + if (owned) { + asrv->plugged = false; + if (asrv->audio_dev) + dcpaud_disconnect(asrv->audio_dev); + } mutex_unlock(&asrv->plug_lock); + + if (owned) + afk_service_put(service); } int dcp_audiosrv_prepare(struct device *dev, struct dcp_sound_cookie *cookie) @@ -279,11 +297,13 @@ static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, static const struct apple_epic_service_ops avep_ops[] = { { .name = "DCPAVSimpleVideoInterface", + .reusable = true, .init = av_interface_init, .teardown = av_interface_teardown, }, { .name = "DCPAVAudioInterface", + .reusable = true, .init = av_audiosrv_init, .report = av_audiosrv_report, .teardown = av_audiosrv_teardown, @@ -298,21 +318,21 @@ void av_service_connect(struct apple_dcp *dcp) int ret; scoped_guard(rwsem_write, &asrv->srv_rwsem) { - if (!asrv->srv) + service = afk_service_pin(asrv->srv); + if (!service) return; - service = asrv->srv; } /* open AV audio service */ dev_info(dcp->dev, "%s: starting audio service, plugged:%d\n", __func__, asrv->plugged); if (asrv->is_open) - return; + goto out; ret = afk_service_call(service, 0, asrv->cmds.open, NULL, 0, 32, NULL, 0, 32); if (ret) { dev_err(dcp->dev, "error opening audio service: %d\n", ret); - return; + goto out; } mutex_lock(&asrv->plug_lock); asrv->is_open = true; @@ -320,6 +340,8 @@ void av_service_connect(struct apple_dcp *dcp) if (asrv->audio_dev) dcpaud_connect(asrv->audio_dev, asrv->plugged); mutex_unlock(&asrv->plug_lock); +out: + afk_service_put(service); } void av_service_disconnect(struct apple_dcp *dcp) @@ -329,15 +351,15 @@ void av_service_disconnect(struct apple_dcp *dcp) int ret; scoped_guard(rwsem_write, &asrv->srv_rwsem) { - if (!asrv->srv) + service = afk_service_pin(asrv->srv); + if (!service) return; - service = asrv->srv; } /* close AV audio service */ dev_info(dcp->dev, "%s: stopping audio service\n", __func__); if (!asrv->is_open) - return; + goto out; mutex_lock(&asrv->plug_lock); @@ -354,6 +376,8 @@ void av_service_disconnect(struct apple_dcp *dcp) if (service->torndown) afk_service_disable(service); asrv->is_open = false; +out: + afk_service_put(service); } static void av_work_service_start(struct work_struct *work) From efebc610521532a281fafec86a8ffac03cf6cd81 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Fri, 16 Oct 2026 00:18:02 -0400 Subject: [PATCH 567/631] treewide: finish the merge with integration/aurora Replaying aurora-wip onto integration/aurora carries each aurora-wip change only where it applies to integration's code. These files still differ from the resolved merge of the two branches, so bring them to it. These are the remaining files, mostly duplicate hunks left by the replay and entries that both branches added in the same place. Assisted-by: LLM Signed-off-by: Diljit Singh --- MAINTAINERS | 29 +------- .../arm64/boot/dts/apple/t602x-j414-j416.dtsi | 2 +- .../boot/dts/apple/t8142-j813-display.dtsi | 2 +- drivers/gpu/drm/drm_gem_shmem_helper.c | 72 ------------------- drivers/media/platform/apple/isp/Kconfig | 2 + drivers/media/platform/apple/isp/isp-v4l2.c | 2 +- drivers/reset/reset-apple-cio.c | 3 +- 7 files changed, 9 insertions(+), 103 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index c0f155b3949711..dacf240284a9bf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1952,14 +1952,9 @@ L: dri-devel@lists.freedesktop.org S: Maintained T: git git://anongit.freedesktop.org/drm/drm-misc F: Documentation/devicetree/bindings/display/apple,dcp-typec-routes.yaml -F: Documentation/devicetree/bindings/chosen/apple,dcp-rtkit-quiesced.yaml -F: Documentation/devicetree/bindings/display/apple,dcp.yaml -F: Documentation/devicetree/bindings/display/apple,display-subsystem.yaml -F: Documentation/gpu/apple-dcp-backlight.rst F: drivers/gpu/drm/apple/ F: drivers/gpu/drm/apple/dcp-fabric* F: include/linux/soc/apple/dp-tunnel.h -F: tools/testing/apple/ F: tools/testing/selftests/drivers/gpu/apple-afkv2/ APPLE PCIE CONTROLLER DRIVER @@ -2016,32 +2011,14 @@ F: drivers/hid/dockchannel-hid/dockchannel-hid-protocol* F: drivers/iio/common/aop_sensors/ F: drivers/phy/apple/atc-j700-dp.h F: drivers/rtc/rtc-apple-pmu.c -F: drivers/pci/controller/pcie-apple-piodma* F: drivers/soc/apple/aop.rs F: drivers/soc/apple/mcc.c F: drivers/tty/serial/apple_dockchannel* +F: drivers/pci/controller/pcie-apple-piodma* F: include/linux/soc/apple/pci-apple-piodma.h -F: include/uapi/linux/apple-sep.h -F: include/linux/soc/apple/smc-thermal.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs -F: tools/diagnostics/apple-dptx-training-test.c -F: tools/diagnostics/apple-j700-dp-test.c -F: tools/diagnostics/display-attributes-fixture.json -F: tools/diagnostics/test-apple-afk-admission.py -F: tools/diagnostics/test-apple-atc-unwind.py -F: tools/diagnostics/test-apple-dcp-readiness.py -F: tools/diagnostics/test-apple-dptx-readiness.py -F: tools/diagnostics/test-apple-external-lifecycle.py -F: tools/diagnostics/test-apple-external-sdr.py -F: tools/diagnostics/test-apple-j700-crossbar.py -F: tools/diagnostics/test-apple-j700-d134.py -F: tools/diagnostics/test-apple-j700-dp.py -F: tools/diagnostics/test-apple-j700-opaque-scope.py -F: tools/diagnostics/test-apple-j700-provider.py -F: tools/diagnostics/test-apple-j700-tuning.py -F: tools/diagnostics/wayland-output-frame-test.py F: tools/testing/apple/ F: tools/testing/selftests/drivers/gpu/apple-dcp/ @@ -2651,9 +2628,6 @@ F: drivers/cpufreq/apple-soc-cpufreq.c F: drivers/dma/apple-admac.c F: drivers/gpio/gpio-macsmc.c F: drivers/hwmon/macsmc-hwmon*.c -F: drivers/hwmon/macsmc-hwmon.c -F: drivers/pmdomain/apple/ -F: drivers/pmdomain/apple/.kunitconfig.apple-pmp F: drivers/i2c/busses/i2c-pasemi-core.c F: drivers/i2c/busses/i2c-pasemi-platform.c F: drivers/input/misc/macsmc-input.c @@ -2670,6 +2644,7 @@ F: drivers/phy/apple/ F: drivers/phy/apple/.kunitconfig.apple-atc F: drivers/pinctrl/pinctrl-apple-gpio.c F: drivers/pmdomain/apple/ +F: drivers/pmdomain/apple/.kunitconfig.apple-pmp F: drivers/power/reset/macsmc-reboot.c F: drivers/power/supply/macsmc-power.c F: drivers/pwm/pwm-apple.c diff --git a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi index 98301e37872bb0..34e5fce427854a 100644 --- a/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi +++ b/arch/arm64/boot/dts/apple/t602x-j414-j416.dtsi @@ -25,8 +25,8 @@ disp0 = &display; disp0_piodma = &disp0_piodma; keyboard = &keyboard; - sio = &sio; sep = &sep; + sio = &sio; /* m1n1 writes this machine's Thunderbolt calibration onto these * aliases. usb4-N-* is current m1n1; usb4_N_* is the 1.6.1 bootloader. */ diff --git a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi index a50fa38fd9265f..9bd549a5784cc0 100644 --- a/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi +++ b/arch/arm64/boot/dts/apple/t8142-j813-display.dtsi @@ -16,7 +16,7 @@ }; display: display-subsystem { - compatible = "apple,display-subsystem"; + compatible = "apple,t8142-display-subsystem", "apple,display-subsystem"; /* disp0_dart must be first since it is locked */ iommus = <&disp0_dart 0>; /* generate phandle explicitly for use in loader */ diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index 5d8b8f1fde05bd..eab3abedcc8741 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -600,64 +600,6 @@ bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shm } EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); -/** - * drm_gem_shmem_release_idle_pages_locked - Release idle backing without purging - * @shmem: shmem GEM object whose device mappings have been removed - * - * Drop the cached SG table and the page array of an object that no device - * mapping, pin, vmap or SG-table borrower uses, so ordinary shmem reclaim can - * swap its contents. Revocable CPU VMAs are zapped first and refault later. - * The caller holds the reservation lock, has removed every device mapping - * including cached translations, and excludes new device access. A later - * drm_gem_shmem_get_pages_sgt() may return different DMA addresses. - * - * Returns: 0 on success, -EBUSY if backing is still in use, -EINVAL for an - * imported, exported or purged object. - */ -int drm_gem_shmem_release_idle_pages_locked(struct drm_gem_shmem_object *shmem) -{ - struct drm_gem_object *obj = &shmem->base; - - dma_resv_assert_held(obj->resv); - - if (drm_gem_is_imported(obj) || obj->dma_buf || shmem->madv < 0) - return -EINVAL; - - if (refcount_read(&shmem->pages_pin_count) || - refcount_read(&shmem->vmap_use_count) || - refcount_read(&shmem->pages_use_count) != !!shmem->sgt || - !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP)) - return -EBUSY; - - /* Zap revocable PFNs under resv so no fault can reinsert them. */ - if (shmem->reclaimable_cpu_mappings && refcount_read(&shmem->cpu_vma_count)) { - drm_vma_node_unmap(&obj->vma_node, obj->dev->anon_inode->i_mapping); - drm_gem_shmem_put_cpu_pages_locked(shmem); - } - -static struct page *drm_gem_shmem_present_page(struct drm_gem_shmem_object *shmem, - pgoff_t index) -{ - if (shmem->pages && shmem->pages[index]) - return shmem->pages[index]; - if (shmem->cpu_pages) - return shmem->cpu_pages[index]; - return NULL; -} - -bool drm_gem_shmem_idle_pages_releasable_locked(struct drm_gem_shmem_object *shmem) -{ - struct drm_gem_object *obj = &shmem->base; - - dma_resv_assert_held(obj->resv); - return shmem->sgt && !drm_gem_is_imported(obj) && !obj->dma_buf && - shmem->madv >= 0 && !refcount_read(&shmem->pages_pin_count) && - !refcount_read(&shmem->vmap_use_count) && - refcount_read(&shmem->pages_use_count) == 1 && - dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_BOOKKEEP); -} -EXPORT_SYMBOL_GPL(drm_gem_shmem_idle_pages_releasable_locked); - /** * drm_gem_shmem_release_idle_pages_locked - Release idle backing without purging * @shmem: shmem GEM object whose device mappings have been removed @@ -717,8 +659,6 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); - drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); - /* Revoke PFNs and complete the TLB flush while backing is still pinned. */ shmem->madv = -1; drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); @@ -783,14 +723,6 @@ EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) { return pgoff - drm_vma_node_start(&obj->vma_node); -/* GEM and PRIME add the object's fake offset; VMA splits retain it. - * Direct helper callers may instead supply an object-relative offset. - */ -static pgoff_t drm_gem_shmem_object_offset(struct drm_gem_object *obj, pgoff_t pgoff) -{ - pgoff_t base = drm_vma_node_start(&obj->vma_node); - - return pgoff >= base ? pgoff - base : pgoff; } static void drm_gem_shmem_record_mkwrite(struct vm_fault *vmf) @@ -1027,10 +959,6 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (is_cow_mapping(vma->vm_flags)) return -EINVAL; - /* Revocation zaps the device anon mapping only. */ - if (shmem->reclaimable_cpu_mappings && - (!vma->vm_file || vma->vm_file->f_mapping != obj->dev->anon_inode->i_mapping)) - return -EINVAL; /* Validate bytes, including a partial native-page offset from PRIME. */ { pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index 79a7a075bd54e8..2b026dba15f08c 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -27,6 +27,8 @@ config VIDEO_APPLE_ISP_KUNIT_TEST overflow, alignment and overlap with the channel table or other rings. These tests run without starting or accessing the ISP and do not require a supported Apple device or the ISP driver itself. + +config VIDEO_APPLE_ISP_V4L2_KUNIT_TEST bool "KUnit tests for the Apple ISP driver" if !KUNIT_ALL_TESTS depends on VIDEO_APPLE_ISP && KUNIT depends on KUNIT=y || VIDEO_APPLE_ISP=m diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index f321cbcb3368d9..01cd82122201a0 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -1165,6 +1165,6 @@ void apple_isp_free_video(struct apple_isp *isp) isp_free_meta_surfaces(isp); } -#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_V4L2_KUNIT_TEST) #include "isp-v4l2-test.c" #endif diff --git a/drivers/reset/reset-apple-cio.c b/drivers/reset/reset-apple-cio.c index 252d57cd59bb1b..1154272a19c07b 100644 --- a/drivers/reset/reset-apple-cio.c +++ b/drivers/reset/reset-apple-cio.c @@ -37,7 +37,8 @@ * separate 16 KiB page at 0x350780000, so the register is 0x350780008. The * device tree describes that page as its own syscon. t8122 is unresolved. * - * t8132 and t8142 use a different, per-port layout and are not supported. + * On t8142, selector 0x62 is a separate MMIO region at 0x380780000, using a + * per-port layout. The t8132 layout has not been decoded. * * Reference firmware polls the busy bits, stores the request, then polls * again, with a 192 ms budget. It discards the first poll's result. The From 3d255038f3b13757b7f3f035a58b23e4c8630f2b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:30:38 -0700 Subject: [PATCH 568/631] aurora-sep: neo: accept both T8140 GPU and display driver names The T8140 GPU and display drivers were folded into the shared asahi and apple-drm drivers, so their platform driver names changed from asahi_neo and apple-drm-neo to asahi and apple-drm. The session helper was switched to the new names only. A MacBook Neo running a kernel with the separate drivers (the released kernel, or a fallback boot entry) then fails the native display check and drops to software rendering, and the helper packaged with the released Neo Mesa fails the same way on the merged kernel. Accept either spelling for each driver. Admission stays T8140-only: the root compatible must be apple,t8140 with board apple,j700, the GPU card must carry apple,agx-t8140 and be the admitted render device, and the KMS card must carry apple,t8140-display-subsystem. M1, M2 and M3 Macs, which also use the asahi and apple-drm names, never reach the driver-name check. Extend the Neo helper tests to cover both name pairs on J700, refuse other drivers (simple framebuffer, unbound GPU, look-alike names), and check that J313, J414s, J514s and J613 keep their environment with either name pair. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- tools/aurora-sep/neo/mesa-neo-env | 15 ++++++-- tools/aurora-sep/tests/test_neo_gpu.py | 48 ++++++++++++++++++++------ 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/tools/aurora-sep/neo/mesa-neo-env b/tools/aurora-sep/neo/mesa-neo-env index 0ea9eb32902e7a..8ac121f257bc78 100644 --- a/tools/aurora-sep/neo/mesa-neo-env +++ b/tools/aurora-sep/neo/mesa-neo-env @@ -47,8 +47,14 @@ mesa_neo_display_admit() { case "${_neo_card##*/}" in card*[!0-9]*) continue;; esac [ -e "$_neo_card" ] || continue if mesa_neo_has_compatible "$_neo_card/device/of_node/compatible" apple,agx-t8140; then - [ "$(readlink -f "$_neo_card/device" 2>/dev/null)" = "$_neo_gpu_device" ] && - [ "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" = asahi ] || return 1 + [ "$(readlink -f "$_neo_card/device" 2>/dev/null)" = "$_neo_gpu_device" ] || return 1 + # Kernels with separate T8140 drivers name them asahi_neo and + # apple-drm-neo; the shared drivers are asahi and apple-drm. + # The compatible checks keep both spellings T8140-only. + case "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" in + asahi|asahi_neo) ;; + *) return 1;; + esac # The GPU primary node has no KMS connectors. for _neo_connector in "$_neo_card"-*; do [ ! -e "$_neo_connector" ] || return 1 @@ -57,7 +63,10 @@ mesa_neo_display_admit() { fi _neo_cards=$((_neo_cards+1)) # A firmware framebuffer cannot use the Asahi render-only KMS route. - [ "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" = apple-drm ] || return 1 + case "$(basename "$(readlink -f "$_neo_card/device/driver" 2>/dev/null)")" in + apple-drm|apple-drm-neo) ;; + *) return 1;; + esac mesa_neo_has_compatible "$_neo_card/device/of_node/compatible" apple,t8140-display-subsystem || return 1 [ -r "$_neo_root/dev/dri/${_neo_card##*/}" ] && [ -w "$_neo_root/dev/dri/${_neo_card##*/}" ] || return 1 for _neo_connector in "$_neo_card"-*; do diff --git a/tools/aurora-sep/tests/test_neo_gpu.py b/tools/aurora-sep/tests/test_neo_gpu.py index a9dddb2488f977..5c73190b103bef 100644 --- a/tools/aurora-sep/tests/test_neo_gpu.py +++ b/tools/aurora-sep/tests/test_neo_gpu.py @@ -127,7 +127,8 @@ def test_transaction_restores_boot_config_and_intent(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(self.original_m1n1.read_bytes(), b'original own stage2') - def hook(self, intent=True, profile='j700-g17p-hal200', abi=0, display=True): + def hook(self, intent=True, profile='j700-g17p-hal200', abi=0, display=True, + gpu_driver='asahi', kms_driver='apple-drm'): for path in ['libexec', 'share/mesa-neo']: (self.prefix / path).mkdir(parents=True, exist_ok=True) (self.prefix / 'share/mesa-neo/profile').write_text('j700-g17p-hal200\n') @@ -144,9 +145,11 @@ def hook(self, intent=True, profile='j700-g17p-hal200', abi=0, display=True): (self.tmp / 'dev').mkdir(exist_ok=True) (self.tmp / 'dev/renderD128').touch() gpu = render.parent - driver = self.tmp / 'drivers/asahi' - driver.mkdir(parents=True, exist_ok=True) - if not (gpu / 'driver').is_symlink(): + if (gpu / 'driver').is_symlink(): + (gpu / 'driver').unlink() + if gpu_driver is not None: + driver = self.tmp / 'drivers' / gpu_driver + driver.mkdir(parents=True, exist_ok=True) (gpu / 'driver').symlink_to(driver) primary = self.tmp / 'sys/card1' primary.mkdir(exist_ok=True) @@ -156,10 +159,11 @@ def hook(self, intent=True, profile='j700-g17p-hal200', abi=0, display=True): kms = self.tmp / 'sys/card2/device/of_node' kms.mkdir(parents=True, exist_ok=True) (kms / 'compatible').write_bytes(b'apple,t8140-display-subsystem\0') - dcp = self.tmp / 'drivers/apple-drm' + dcp = self.tmp / 'drivers' / kms_driver dcp.mkdir(parents=True, exist_ok=True) - if not (kms.parent / 'driver').is_symlink(): - (kms.parent / 'driver').symlink_to(dcp) + if (kms.parent / 'driver').is_symlink(): + (kms.parent / 'driver').unlink() + (kms.parent / 'driver').symlink_to(dcp) (self.tmp / 'dev/card2').touch() connector = self.tmp / 'sys/card2-eDP-1' connector.mkdir(exist_ok=True) @@ -208,11 +212,33 @@ def test_hook_without_native_display_keeps_software_fallback(self): self.assertNotIn('MESA_NEO_PROFILE', env) self.assertEqual(env['LD_LIBRARY_PATH'], '/keep/lib') + def test_hook_accepts_separate_and_shared_driver_names(self): + for gpu_driver, kms_driver in [('asahi_neo', 'apple-drm-neo'), ('asahi', 'apple-drm')]: + with self.subTest(gpu=gpu_driver, kms=kms_driver): + env = self.hook(gpu_driver=gpu_driver, kms_driver=kms_driver) + self.assertEqual(env['GALLIUM_DRIVER'], 'asahi') + self.assertEqual(env['MESA_NEO_PROFILE'], 'j700-g17p-hal200') + self.assertNotIn('LIBGL_ALWAYS_SOFTWARE', env) + + def test_hook_refuses_other_drivers(self): + for args in [dict(kms_driver='simple-framebuffer'), dict(kms_driver='apple-drm-neox'), + dict(gpu_driver='asahi-neo'), dict(gpu_driver=None)]: + with self.subTest(args=args): + env = self.hook(**args) + self.assertEqual(env['LIBGL_ALWAYS_SOFTWARE'], '1') + self.assertEqual(env['MESA_NEO_FALLBACK_REASON'], 'native-display-unavailable') + self.assertNotIn('MESA_NEO_PROFILE', env) + def test_hook_leaves_other_macs_unchanged(self): - self.mac('j613', 't8122') - env = self.hook() - self.assertEqual(env['GALLIUM_DRIVER'], 'zink') - self.assertEqual(env['VK_DRIVER_FILES'], '/stale/m3.json') + for board, soc in [('j613', 't8122'), ('j514s', 't6030'), ('j313', 't8103'), ('j414s', 't6020')]: + for gpu_driver, kms_driver in [('asahi', 'apple-drm'), ('asahi_neo', 'apple-drm-neo')]: + with self.subTest(board=board, gpu=gpu_driver): + self.mac(board, soc) + env = self.hook(gpu_driver=gpu_driver, kms_driver=kms_driver) + self.assertEqual(env['GALLIUM_DRIVER'], 'zink') + self.assertEqual(env['VK_DRIVER_FILES'], '/stale/m3.json') + self.assertNotIn('MESA_NEO_PROFILE', env) + self.assertNotIn('LIBGL_ALWAYS_SOFTWARE', env) if __name__ == '__main__': From 2bf01adf4862725d602e9d7fe38248f240624c29 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:40:36 -0700 Subject: [PATCH 569/631] pmdomain: apple: pmp-report: match T8140 report entries by compatible The T8140 PMP report moved into the shared pmp-report driver, but its entry driver only matches apple,t6000-pmp-v2-report-entry. A J700 device tree whose GPU report entry carries only apple,t8140-pmp-v2-report-entry, as in the released kernel's tree, then leaves the pmp-gfx power domain without a provider. The GPU and its PMC agent sit below that domain, so they never probe. Match the T8140 entry compatible too. Its probe already goes to the T8140 path whenever the parent report is a published (T8140) one, so the entry is driven the same way whichever compatible matched. No other SoC uses this compatible. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/pmdomain/apple/pmp-report.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index 9d585f5a0f712d..809d8fc650eba3 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -1719,6 +1719,8 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) static const struct of_device_id apple_pmp_report_entry_of_match[] = { { .compatible = "apple,t6000-pmp-v2-report-entry" }, + /* Probed through apple_pmp_t8140_entry_probe() by its report. */ + { .compatible = "apple,t8140-pmp-v2-report-entry" }, {} }; From 731e926b1912a2184feb886f6977d8a28d70abe7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:40:37 -0700 Subject: [PATCH 570/631] dt-bindings: power: apple,t8140-pmp-report: drop the t6000 entry fallback A T8140 report entry is programmed differently from a T6000 one: it is reported through the T8140 published report, not through the T6000 request bits. Listing apple,t6000-pmp-v2-report-entry as a fallback says a T6000 entry driver can drive it, which is not true. A kernel whose T6000 entry driver registers before its T8140 one, such as the released kernel with separate T8140 drivers, binds the T6000 driver to the entry and then treats the T8140 report's data as a T6000 report. Describe the entry with its own compatible only. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../devicetree/bindings/power/apple,t8140-pmp-report.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml index 781afa1ad9d9f1..2df7b3e018a3e1 100644 --- a/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml +++ b/Documentation/devicetree/bindings/power/apple,t8140-pmp-report.yaml @@ -45,9 +45,7 @@ patternProperties: properties: compatible: - items: - - const: apple,t8140-pmp-v2-report-entry - - const: apple,t6000-pmp-v2-report-entry + const: apple,t8140-pmp-v2-report-entry reg: maxItems: 1 @@ -91,8 +89,7 @@ examples: #size-cells = <0>; report@9 { - compatible = "apple,t8140-pmp-v2-report-entry", - "apple,t6000-pmp-v2-report-entry"; + compatible = "apple,t8140-pmp-v2-report-entry"; reg = <0x9>; label = "pmp-gfx"; #power-domain-cells = <0>; From df016a38440e7121cddc13e38016c5bdde5905c9 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:40:37 -0700 Subject: [PATCH 571/631] arm64: dts: apple: t8140: drop the t6000 fallback from the GFX report The GFX report entry listed apple,t6000-pmp-v2-report-entry after its own compatible. On a MacBook Neo the boot loader carries the device tree of the installed kernel package, and the previous kernel boot entry uses that same tree. The released kernel registers its T6000 entry driver before its T8140 one, so with this tree it binds the T6000 driver to the GFX entry. That driver then reads the T8140 report's data as a T6000 report: the field it takes as its spinlock holds a list pointer, so the first power-on of the always-on GFX domain at boot spins with interrupts off. Use the T8140 compatible only, which the shared pmp-report driver now matches. The entry is then the same as in the released tree, and either kernel binds it to its T8140 entry driver. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- arch/arm64/boot/dts/apple/t8140.dtsi | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/apple/t8140.dtsi b/arch/arm64/boot/dts/apple/t8140.dtsi index 89a5a609817e42..a2752e59c07478 100644 --- a/arch/arm64/boot/dts/apple/t8140.dtsi +++ b/arch/arm64/boot/dts/apple/t8140.dtsi @@ -491,8 +491,7 @@ /* PMP soc-device 9, which the PMP acknowledges */ pmp_report_gfx: report@9 { - compatible = "apple,t8140-pmp-v2-report-entry", - "apple,t6000-pmp-v2-report-entry"; + compatible = "apple,t8140-pmp-v2-report-entry"; reg = <0x9>; label = "pmp-gfx"; #power-domain-cells = <0>; From d4b2c4ac469856efbfdb127b02ac6fb3d9515f73 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:49:26 -0700 Subject: [PATCH 572/631] spi: apple-qspi: keep the T8140 boot NOR read-only outside nvram The released kernel kept the MacBook Neo (T8140) boot NOR behind the driver's read-only policy. The merged T8140 path skips that policy and passes every spi-mem request to the flash: status and configuration register writes, chip erase, address-mode changes and programs or erases anywhere on the chip. The boot NOR holds the system firmware, so a bad request from SPI-NOR, or from a future change to the flash description, can leave the machine unable to boot. Only reads were tried on this path on hardware. Gate the T8140 path. Data reads still reach the whole NOR. Write enable and disable are admitted. A page program, or a 4, 32 or 64 KiB erase, is admitted only when every byte it can change lies inside the flash's "nvram" partition from the device tree, which stays writable for boot-volume selection. A program counts the whole 4 KiB blocks it touches, so a write that wraps within its page cannot leave the partition either. Everything else is refused with -EACCES and a rate-limited warning. The partition must be the only one labelled "nvram", 4 KiB aligned and below 4 GiB; otherwise no program or erase is admitted and the whole NOR is read-only. An address that does not fit in the request's address bytes is refused, since the controller sends only those bytes. T8122 and T8132 keep their read_only policy and bounded J613 writes unchanged. Add KUnit tests for the T8140 rules. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/spi/Kconfig | 13 ++- drivers/spi/Makefile | 1 + drivers/spi/spi-apple-qspi-policy-test.c | 133 +++++++++++++++++++++++ drivers/spi/spi-apple-qspi-policy.h | 54 +++++++++ drivers/spi/spi-apple-qspi.c | 104 +++++++++++++++++- 5 files changed, 298 insertions(+), 7 deletions(-) create mode 100644 drivers/spi/spi-apple-qspi-policy-test.c diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index f700acbcf8a168..f869f0b880d0e7 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -140,7 +140,18 @@ config SPI_APPLE_QSPI on T8122 (J613 25G83), T8132 (M4) and T8140 systems. On T8122 and T8132 the driver defaults read-only, and only the pinned J613 NVRAM layout can enable bounded writes through an explicit root runtime - opt-in. T8140 is not restricted. + opt-in. On T8140 only the nvram partition accepts programs and + erases; the rest of the boot NOR is read-only. + +config SPI_APPLE_QSPI_KUNIT_TEST + tristate "KUnit tests for the Apple QSPIMC boot NOR write window" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on SPI_APPLE_QSPI || COMPILE_TEST + default KUNIT_ALL_TESTS + help + Tests which T8140 boot NOR requests the QSPIMC driver admits: write + enable control, and programs and erases inside the nvram partition + only. No hardware is accessed. config SPI_AR934X tristate "Qualcomm Atheros AR934X/QCA95XX SPI controller driver" diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile index b422044ca08a25..4925623c8bcdb9 100644 --- a/drivers/spi/Makefile +++ b/drivers/spi/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_SPI_AMLOGIC_SPIFC_A4) += spi-amlogic-spifc-a4.o obj-$(CONFIG_SPI_AMLOGIC_SPISG) += spi-amlogic-spisg.o obj-$(CONFIG_SPI_APPLE) += spi-apple.o obj-$(CONFIG_SPI_APPLE_QSPI) += spi-apple-qspi.o +obj-$(CONFIG_SPI_APPLE_QSPI_KUNIT_TEST) += spi-apple-qspi-policy-test.o obj-$(CONFIG_SPI_AR934X) += spi-ar934x.o obj-$(CONFIG_SPI_ARMADA_3700) += spi-armada-3700.o obj-$(CONFIG_SPI_ASPEED_SMC) += spi-aspeed-smc.o diff --git a/drivers/spi/spi-apple-qspi-policy-test.c b/drivers/spi/spi-apple-qspi-policy-test.c new file mode 100644 index 00000000000000..9ec58d587c60f2 --- /dev/null +++ b/drivers/spi/spi-apple-qspi-policy-test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for the T8140 boot NOR write window of the Apple QSPIMC driver. + */ + +#include + +#include "spi-apple-qspi-policy.h" + +/* The J700 nvram partition. */ +#define NV_START 0x24000ULL +#define NV_END 0x1f4000ULL + +static int nv(unsigned int opcode, unsigned int addr_bytes, + unsigned long long addr, unsigned int len, int data_out) +{ + return apple_qspi_window_allowed(opcode, addr_bytes, addr, len, + data_out, NV_START, NV_END); +} + +static void qspi_window_write_enable(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, nv(0x06, 0, 0, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0x04, 0, 0, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x06, 3, NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x06, 0, 0, 1, 1)); + /* Write-enable control needs no window. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x06, 0, 0, 0, 0, 0, 0)); +} + +static void qspi_window_program(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_START, 256, 1)); + KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_START + 0x1234, 1, 1)); + KUNIT_EXPECT_TRUE(test, nv(0x02, 3, NV_END - 256, 256, 1)); + KUNIT_EXPECT_TRUE(test, nv(0x12, 4, NV_START, 256, 1)); + /* Any byte outside, or a wrap within a page outside, is refused. */ + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START - 256, 256, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START - 1, 2, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_END, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_END - 1, 2, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, 0, 256, 1)); + /* A program needs data out, and at most one 4 KiB block's worth. */ + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 0, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 256, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, NV_START, 4097, 1)); +} + +static void qspi_window_erase(struct kunit *test) +{ + KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_START, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_START + 0x123, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0x20, 3, NV_END - 1, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0x21, 4, NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START - 1, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_END, 0, 0)); + /* 32 and 64 KiB erases cover the whole aligned block. */ + KUNIT_EXPECT_TRUE(test, nv(0x52, 3, 0x28000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x52, 3, NV_START, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0xd8, 3, 0x30000, 0, 0)); + KUNIT_EXPECT_TRUE(test, nv(0xdc, 4, 0x1e0000, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0xd8, 3, 0x1f0000, 0, 0)); + /* An erase carries no data. */ + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START, 1, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, NV_START, 0, 1)); +} + +static void qspi_window_refused_commands(struct kunit *test) +{ + static const u8 refused[] = { + 0x01, 0x31, 0x11, /* status register writes */ + 0x60, 0xc7, /* chip erase */ + 0xb7, 0xe9, 0xc5, /* address mode, extended address */ + 0x66, 0x99, 0xf0, /* reset */ + 0x42, 0x44, 0x98, /* OTP program and erase, global unlock */ + 0x32, 0x34, 0x38, /* multi-line page programs */ + 0x7a, 0x75, 0xb9, 0xab, /* suspend, resume, power down */ + }; + int i; + + for (i = 0; i < ARRAY_SIZE(refused); i++) { + KUNIT_EXPECT_FALSE(test, nv(refused[i], 0, 0, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(refused[i], 3, NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(refused[i], 3, NV_START, 1, 1)); + } +} + +static void qspi_window_address_width(struct kunit *test) +{ + /* Three address bytes cannot carry an address above 16 MiB. */ + KUNIT_EXPECT_FALSE(test, nv(0x02, 3, (1ULL << 24) + NV_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 3, (1ULL << 24) + NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x12, 4, (1ULL << 32) + NV_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x02, 2, NV_START, 1, 1)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 0, NV_START, 0, 0)); + KUNIT_EXPECT_FALSE(test, nv(0x20, 5, NV_START, 0, 0)); + /* A window that reaches the top of 24-bit addressing. */ + KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x20, 3, 0xfff000, 0, 0, + 0xff0000, 1ULL << 24)); + /* A program that crosses 16 MiB cannot use three address bytes. */ + KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x02, 3, 0xffffff, 2, 1, + 0xff0000, 1ULL << 25)); + KUNIT_EXPECT_TRUE(test, apple_qspi_window_allowed(0x12, 4, 0xffffff, 2, 1, + 0xff0000, 1ULL << 25)); +} + +static void qspi_window_empty(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x02, 3, 0, 1, 1, 0, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x20, 3, 0, 0, 0, 0, 0)); + KUNIT_EXPECT_FALSE(test, apple_qspi_window_allowed(0x20, 3, NV_START, 0, 0, + NV_START, NV_START)); +} + +static struct kunit_case apple_qspi_window_cases[] = { + KUNIT_CASE(qspi_window_write_enable), + KUNIT_CASE(qspi_window_program), + KUNIT_CASE(qspi_window_erase), + KUNIT_CASE(qspi_window_refused_commands), + KUNIT_CASE(qspi_window_address_width), + KUNIT_CASE(qspi_window_empty), + {} +}; + +static struct kunit_suite apple_qspi_window_suite = { + .name = "apple-qspi-nvram-window", + .test_cases = apple_qspi_window_cases, +}; +kunit_test_suite(apple_qspi_window_suite); + +MODULE_DESCRIPTION("KUnit tests for the Apple QSPIMC boot NOR write window"); +MODULE_LICENSE("GPL"); diff --git a/drivers/spi/spi-apple-qspi-policy.h b/drivers/spi/spi-apple-qspi-policy.h index 6c877fa59854de..5d788d69f4f8ba 100644 --- a/drivers/spi/spi-apple-qspi-policy.h +++ b/drivers/spi/spi-apple-qspi-policy.h @@ -35,6 +35,60 @@ static inline int apple_qspi_mutation_allowed(unsigned int opcode, } } +/* + * T8140 boot NOR: may a request that does not read data reach the flash? + * Write-enable control is admitted. A page program, or a 4, 32 or 64 KiB + * erase, is admitted only when every byte it can change lies inside + * [start, end), the nvram partition. A program counts the whole 4 KiB + * blocks it touches, which covers a write that wraps within its page. Status + * and configuration register writes, chip erase, address-mode, reset and OTP + * commands are refused. The address must fit in addr_bytes: the controller + * sends only those bytes, so a wider value would alias a lower address. + */ +static inline int apple_qspi_window_allowed(unsigned int opcode, + unsigned int addr_bytes, unsigned long long addr, + unsigned int len, int data_out, + unsigned long long start, unsigned long long end) +{ + unsigned long long block, first, last; + + switch (opcode) { + case 0x06: /* WREN */ + case 0x04: /* WRDI */ + return !addr_bytes && !len && !data_out; + case 0x02: /* PP */ + case 0x12: /* PP_4B */ + if (!data_out || !len || len > APPLE_QSPI_ERASE_SIZE) + return 0; + block = APPLE_QSPI_ERASE_SIZE; + break; + case 0x20: /* BE_4K */ + case 0x21: /* BE_4K_4B */ + block = 0x1000; + break; + case 0x52: /* BE_32K */ + case 0x5c: /* BE_32K_4B */ + block = 0x8000; + break; + case 0xd8: /* SE */ + case 0xdc: /* SE_4B */ + block = 0x10000; + break; + default: + return 0; + } + if (opcode != 0x02 && opcode != 0x12 && (len || data_out)) + return 0; + if ((addr_bytes != 3 && addr_bytes != 4) || addr >> (8 * addr_bytes)) + return 0; + + /* [first, last) are the blocks the flash can change. */ + first = addr & ~(block - 1); + last = ((addr + (len ? len : 1) - 1) | (block - 1)) + 1; + return last <= 1ULL << (8 * addr_bytes) && + start <= first && last <= end; +} + /* Protection and activity bits must permit a write; JEDEC ID is checked separately. */ static inline int apple_qspi_status_allows_write(unsigned int sr1, unsigned int sr2, unsigned int sr3) diff --git a/drivers/spi/spi-apple-qspi.c b/drivers/spi/spi-apple-qspi.c index 0367b63ae8aaee..39f019f8367d97 100644 --- a/drivers/spi/spi-apple-qspi.c +++ b/drivers/spi/spi-apple-qspi.c @@ -14,8 +14,12 @@ * 4 KiB erases are admitted. A power loss can interrupt a NOR update within * the admitted range. * - * T8140 keeps unrestricted spi-mem access with dual and quad transfers; the - * read_only policy does not apply to it. + * On T8140 the read_only policy does not apply. Reads reach the whole NOR, + * but a program or erase is admitted only inside the flash's "nvram" + * partition, which stays writable for boot-volume selection. Everything + * else on the boot NOR is read-only: status and configuration register + * writes, chip erase, address-mode, reset and OTP commands are refused. + * Without a usable nvram partition the whole NOR is read-only. */ #include @@ -158,11 +162,13 @@ static bool apple_qspi_opcode_is_read(u8 opcode) /* Per-compatible controller behaviour. */ struct apple_qspi_hw { + /* Gate requests through the read_only/J613 NVRAM write policy. */ + bool nvram_policy; /* - * Gate requests through the read_only/J613 NVRAM write policy. Without - * it, every supported spi-mem operation reaches the controller. + * Admit reads anywhere, but programs and erases only inside the nvram + * partition (apple_qspi_window_allowed()). */ - bool nvram_policy; + bool nvram_window; /* * Send TX-only operations as one payload stream: zero header counts and * a length word that covers opcode, address and data. Otherwise the @@ -189,6 +195,7 @@ static const struct apple_qspi_hw apple_qspi_t8122_hw = { }; static const struct apple_qspi_hw apple_qspi_t8140_hw = { + .nvram_window = true, .multi_io = true, .us_poll = true, .exact_clock = true, @@ -200,6 +207,9 @@ struct apple_qspi { /* Only the pinned J613 controller/partition may opt in. */ bool nvram_write_candidate; bool write_fault; + /* With hw->nvram_window: the nvram partition, empty when none. */ + u64 window_start; + u64 window_end; void __iomem *base; void __iomem *p; void __iomem *c; @@ -900,11 +910,36 @@ static int apple_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) return ret; } -/* Controllers without the NVRAM policy pass every supported request through. */ +/* T8140: data reads pass; anything else must stay inside the nvram window. */ +static int apple_qspi_window_exec_op(struct spi_mem *mem, + const struct spi_mem_op *op) +{ + struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); + + if (op->data.dir == SPI_MEM_DATA_IN && op->data.nbytes) + return apple_qspi_transfer_op(mem, op); + if (!op->dummy.nbytes && + apple_qspi_window_allowed(op->cmd.opcode, op->addr.nbytes, + op->addr.val, op->data.nbytes, + op->data.dir == SPI_MEM_DATA_OUT, + qspi->window_start, qspi->window_end)) + return apple_qspi_transfer_op(mem, op); + + dev_warn_ratelimited(&mem->spi->dev, + "boot NOR request denied: opcode=%02x addr=%#llx addr_bytes=%u len=%u dir=%u nvram=[%#llx, %#llx)\n", + op->cmd.opcode, (unsigned long long)op->addr.val, + op->addr.nbytes, op->data.nbytes, op->data.dir, + (unsigned long long)qspi->window_start, + (unsigned long long)qspi->window_end); + return -EACCES; +} + static int apple_qspi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) { struct apple_qspi *qspi = spi_controller_get_devdata(mem->spi->controller); + if (qspi->hw->nvram_window) + return apple_qspi_window_exec_op(mem, op); if (!qspi->hw->nvram_policy) return apple_qspi_transfer_op(mem, op); return apple_qspi_exec_op(mem, op); @@ -952,6 +987,61 @@ static bool apple_qspi_nvram_layout(struct device_node *node) return found; } +/* + * The one "nvram" partition of the flash on CS0, 4 KiB aligned and within + * 32-bit addressing. Without it, no program or erase is admitted. + */ +static void apple_qspi_nvram_window(struct device *dev, struct apple_qspi *qspi) +{ + struct device_node *flash, *parts, *part; + const char *label; + u32 reg[2], cells[2], cs; + u64 start = 0, end = 0; + int found = 0; + + flash = of_get_child_by_name(dev->of_node, "flash"); + if (!flash) + goto out; + if (!of_device_is_compatible(flash, "jedec,spi-nor") || + of_property_read_u32(flash, "reg", &cs) || cs) { + of_node_put(flash); + goto out; + } + parts = of_get_child_by_name(flash, "partitions"); + of_node_put(flash); + if (!parts) + goto out; + if (of_device_is_compatible(parts, "fixed-partitions") && + !of_property_read_u32(parts, "#address-cells", &cells[0]) && + !of_property_read_u32(parts, "#size-cells", &cells[1]) && + cells[0] == 1 && cells[1] == 1) { + for_each_available_child_of_node(parts, part) { + if (of_property_read_string(part, "label", &label) || + strcmp(label, "nvram")) + continue; + found++; + if (of_property_count_u32_elems(part, "reg") == 2 && + !of_property_read_u32_array(part, "reg", reg, 2)) { + start = reg[0]; + end = start + reg[1]; + } + } + } + of_node_put(parts); + if (found != 1 || start >= end || end > 1ULL << 32 || + !IS_ALIGNED(start, APPLE_QSPI_ERASE_SIZE) || + !IS_ALIGNED(end, APPLE_QSPI_ERASE_SIZE)) + start = end = 0; +out: + qspi->window_start = start; + qspi->window_end = end; + if (start < end) + dev_dbg(dev, "boot NOR read-only outside nvram [%#llx, %#llx)\n", + start, end); + else + dev_dbg(dev, "boot NOR read-only: no usable nvram partition\n"); +} + static int apple_qspi_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -985,6 +1075,8 @@ static int apple_qspi_probe(struct platform_device *pdev) if (qspi->hw->nvram_policy) dev_info(dev, "NVRAM layout candidate: %u\n", qspi->nvram_write_candidate); + if (qspi->hw->nvram_window) + apple_qspi_nvram_window(dev, qspi); qspi->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(qspi->base)) return PTR_ERR(qspi->base); From 1d66b81ec0ea1d51c26fc259532b36fbe3420f06 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:45:30 -0700 Subject: [PATCH 573/631] soc: apple: sep: Do not autoload the driver through a module alias Exporting the apple,sep OF table as a module alias lets udev load apple_sep on its own on every board whose device tree enables the SEP node (J293, J313, J314s/c, J316s/c, J456, J457, J700). It then loads with the module defaults, xart_writes=1 and provision_keybag=1, as soon as udev processes the platform device. That bypasses the loader units that have loaded the driver until now. apple-sep.path starts apple-sep.service only once /dev/disk/by-partlabel/iBootSystemContainer exists, and the installer enables them only after it has shown the write notice and applied the owner's --read-only choice or an existing block. A udev load runs before either: on a Mac where the owner never enabled or later disabled the units, the driver provisions an identity keybag and writes to the shared xART store anyway. And probe opens the iBoot system container by its partition-label link; if udev has not created that link yet, probe fails, nothing retries it, and the service's later modprobe finds the module already loaded, so the SEP stays unbound for that boot. Keep the OF table for driver matching but do not export it, as before. Loading stays with apple-sep.service (the installer's aurora-touchid package, used on every Mac including the MacBook Neo) and aurora-sep.service (tools/aurora-sep), which both load the driver by name. Update the README, which described the alias, and the package checksum that covers it. Verified on the production config: the rebuilt apple-sep.ko has no alias= entries in .modinfo; the PR head's module had of:N*T*Capple,sep and of:N*T*Capple,sepC*. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/soc/apple/sep.rs | 12 ++++++------ tools/aurora-sep/README.md | 5 +++-- tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/drivers/soc/apple/sep.rs b/drivers/soc/apple/sep.rs index 8ff12dfefab8ec..4cae3993f6ef9d 100644 --- a/drivers/soc/apple/sep.rs +++ b/drivers/soc/apple/sep.rs @@ -2321,12 +2321,12 @@ impl WorkItem for SepData { struct SepDriver(Arc); -kernel::of_device_table!( - OF_TABLE, - MODULE_OF_TABLE, - (), - [(of::DeviceId::new(c"apple,sep"), ())] -); +// Deliberately not exported as a module alias: udev must not load the driver +// on its own. The loader service loads it once the iBoot system container's +// partition link exists, which the xART store needs at probe, and only after +// the owner has enabled it (or chosen read-only) for this Mac. +const OF_TABLE: kernel::device_id::IdArray = + kernel::device_id::IdArray::new([(of::DeviceId::new(c"apple,sep"), ())]); impl platform::Driver for SepDriver { type IdInfo = (); diff --git a/tools/aurora-sep/README.md b/tools/aurora-sep/README.md index 5de174ba4c7994..354def8d4f5b8b 100644 --- a/tools/aurora-sep/README.md +++ b/tools/aurora-sep/README.md @@ -45,8 +45,9 @@ for example with `systemctl edit aurora-sep.service`: Environment=AURORA_SEP_READY_TIMEOUT=180 ``` -The driver defaults to `xart_writes=1` and exports the `apple,sep` module -alias for automatic loading. Writes are accepted only when the APFS `.gl` +The driver defaults to `xart_writes=1`. It exports no module alias, so udev +never loads it on its own: the service above loads it, once the iBoot system +container is reachable. Writes are accepted only when the APFS `.gl` extent passes the driver's narrow ownership checks: one physical extent, no snapshots or revert state, and a single extent reference owned by the `.gl` inode. The kernel resolves that extent from checksum-validated APFS metadata; unsupported layouts diff --git a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD index bcb0917d589869..4a939a4876313a 100644 --- a/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD +++ b/tools/aurora-sep/packaging/arch/aurora-sep/PKGBUILD @@ -7,7 +7,7 @@ pkgname=aurora-sep pkgver=1 -pkgrel=5 +pkgrel=6 pkgdesc="Loader and desktop integration for the Apple SEP (Touch ID) driver" url="https://github.com/aurora-silicon/linux/tree/aurora-wip/tools/aurora-sep" arch=(any) @@ -40,7 +40,7 @@ sha256sums=('abea60bb264ced349f7cf671d8d26e1609a81a60ea4c34d66ec6f7f73b847dbb' '0695bd9ce4c207241db13c3ea07819a032d8213417021378f5015ab9317cde12' '5936eb76db666496a8700dade7ac6bcb3785b517958dcea10685690f1822c86f' 'ffab97b20bfd5f0eb1ab641179a131843349cf16890fff0cf2d893f0b5681b37' - 'a87584f87b8b49c19333fd9c5a8e469401756ec08519fd6cc655bef74b5d284f' + 'b16402fe4d08358ecf862868cbc9b51a2e5a5e310b6ce52b04f4a8550a1cd4b5' 'a950153f3f4e600f29b8e1976bb7de71f96bbf6b922cb88512c40409a8d1b6ef') # A unit and the helper it runs ($1, installed as $3), pointed at the From c80c151a1851ea49535777086adc6020d70e372a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:53:06 -0700 Subject: [PATCH 574/631] soc: apple: sep: Initialise an empty host-state store file again The host-state store is now initialised only when this open created the file. An existing file that is shorter than the store or lacks a valid header is refused, including one that is empty or all zero. That is exactly what a creation interrupted before its superblock was written leaves: the driver creates the file, zero-fills its 72 blocks and only then writes the superblock. A power loss or crash in that window, or a create() that fails partway (out of space, I/O error), leaves such a file, and every later boot then logs "host-state store failed validation" and keeps Touch ID off until someone deletes the file by hand. The .19 release initialised such a file again on the next open. Such a file holds no host state, so re-initialising it loses nothing. Do that again, with a warning, for an existing file that is empty or all zero (the same test the refusal already used to describe it). Keep refusing, without writing to it, any file with other contents that is short or has a bad header, as well as the slot-length and write-intent damage: those may be what is left of real host state. Add a KUnit suite that opens files in the test kernel's root file system: an existing empty file and short or full-size all-zero files are initialised; a short file or a bad header with other contents is refused and left byte-for-byte as it was; a valid store keeps its records across a reopen. Fixes: 4ecd4330795c ("soc: apple: sep: Refuse a damaged host-state store instead of erasing it") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/soc/apple/store.rs | 150 +++++++++++++++++++++++++++++++------ 1 file changed, 126 insertions(+), 24 deletions(-) diff --git a/drivers/soc/apple/store.rs b/drivers/soc/apple/store.rs index 03bd6f1bf0a01d..417b2514f3821c 100644 --- a/drivers/soc/apple/store.rs +++ b/drivers/soc/apple/store.rs @@ -133,7 +133,12 @@ fn describe(file: &shim::StoreFile, size: u64) -> Option<&'static str> { impl Store { pub(crate) fn open() -> Result { - let (file, created) = shim::StoreFile::open_host(STORE_PATH)?; + Self::open_at(STORE_PATH) + } + + /// [`Store::open`] on the file at `path`. + fn open_at(path: &CStr) -> Result { + let (file, created) = shim::StoreFile::open_host(path)?; let mut store = Store { file, @@ -143,9 +148,10 @@ impl Store { fresh: false, }; - // Only a new, empty file is initialised. An existing file may hold the - // only copy of the host state, so one that this driver cannot use is - // refused and left as it is. + // A new file is initialised, and so is an existing one that holds + // nothing (see `init_blank`). Any other file may hold the only copy of + // the host state, so one that this driver cannot use is refused and + // left as it is. let size = store.file.size()?; if created && size == 0 { store.create()?; @@ -153,7 +159,7 @@ impl Store { return Ok(store); } if size < STORE_SIZE as u64 { - return Err(store.refuse(Damage::Size, size)); + return store.init_blank(Damage::Size, size); } let mut sb = KVec::with_capacity(BLOCK_SIZE, GFP_KERNEL)?; @@ -165,7 +171,7 @@ impl Store { || le32(&sb, SB_SLOT_COUNT) as usize != SLOT_COUNT || le32(&sb, SB_BLOCK_SIZE) as usize != BLOCK_SIZE { - return Err(store.refuse(Damage::Header, size)); + return store.init_blank(Damage::Header, size); } store.generation = le64(&sb, SB_GENERATION); @@ -181,7 +187,7 @@ impl Store { }; // A longer record would be read from the blocks of the next slots. if store.slots[i].used && store.slots[i].len as usize > MAX_VALUE { - return Err(store.refuse(Damage::Slot(i), size)); + return Err(store.refuse(Damage::Slot(i))); } } @@ -195,7 +201,7 @@ impl Store { // that names a committed record is damage, and freeing that // block would discard the record. if store.slots[slot].used { - return Err(store.refuse(Damage::Intent, size)); + return Err(store.refuse(Damage::Intent)); } store.slots[slot] = Slot::FREE; } @@ -205,24 +211,32 @@ impl Store { Ok(store) } + /// An existing file that is too short or has no valid header. If it is + /// empty or all zero it holds no host state: that is all a creation + /// interrupted before its superblock was written leaves, and every boot + /// after one would otherwise find Touch ID off until the file is removed + /// by hand. Such a file is initialised like a new one, as earlier + /// releases did. Anything else may be what is left of a store, and is + /// refused without writing to it. + fn init_blank(mut self, damage: Damage, size: u64) -> Result { + let Some(what) = describe(&self.file, size) else { + return Err(self.refuse(damage)); + }; + pr_warn!( + "host-state store file is {} and holds no state; initialising it\n", + what + ); + self.create()?; + Ok(self) + } + /// Logs why the existing file is refused, and returns the error. Nothing /// is written to the file. - fn refuse(&self, damage: Damage, size: u64) -> Error { - let what = match damage { - Damage::Size | Damage::Header => describe(&self.file, size), - _ => None, - }; - match what { - Some(what) => pr_err!( - "host-state store failed validation ({:?}, {}), refusing it\n", - damage, - what - ), - None => pr_err!( - "host-state store failed validation ({:?}), refusing it\n", - damage - ), - } + fn refuse(&self, damage: Damage) -> Error { + pr_err!( + "host-state store failed validation ({:?}), refusing it\n", + damage + ); EIO } @@ -363,3 +377,91 @@ pub(crate) const fn crc16_ccitt_false(data: &[u8]) -> u16 { // Canonical CCITT-FALSE check value; guards against a reflected/XOR variant. static_assert!(crc16_ccitt_false(b"123456789") == 0x29B1); + +#[kernel::macros::kunit_tests(apple_sep_host_store)] +mod tests { + use super::*; + + // In the root of the test kernel's in-memory root file system. + const PATH: &CStr = c"/apple-sep-host-store-test.bin"; + + /// Replaces the test file's contents with `len` bytes of `byte`. + fn fill(len: usize, byte: u8) -> Result { + let file = shim::StoreFile::open_trunc(PATH)?; + let mut buf = KVec::new(); + buf.resize(BLOCK_SIZE, byte, GFP_KERNEL)?; + let mut off = 0; + while off < len { + let n = (len - off).min(BLOCK_SIZE); + file.write_all(off as u64, &buf[..n])?; + off += n; + } + file.sync() + } + + /// The test file's size and its first byte (0 when empty). + fn peek() -> Result<(u64, u8)> { + let file = shim::StoreFile::open_readonly(PATH)?; + let size = file.size()?; + let mut first = [0u8; 1]; + if size > 0 { + file.read_exact(0, &mut first)?; + } + Ok((size, first[0])) + } + + /// Opens the test file as a store, expecting a newly initialised one. + fn opens_fresh() -> Result { + let store = Store::open_at(PATH)?; + Ok(store.fresh && store.slots.iter().all(|slot| !slot.used)) + } + + #[test] + fn an_existing_empty_file_is_initialised() -> Result { + fill(0, 0)?; + assert!(opens_fresh()?); + assert_eq!(peek()?, (STORE_SIZE as u64, MAGIC[0])); + Ok(()) + } + + #[test] + fn an_all_zero_file_is_initialised() -> Result { + // Short, as a creation interrupted while zero-filling leaves it. + fill(BLOCK_SIZE + 100, 0)?; + assert!(opens_fresh()?); + assert_eq!(peek()?, (STORE_SIZE as u64, MAGIC[0])); + // Full size, interrupted before the superblock was written. + fill(STORE_SIZE, 0)?; + assert!(opens_fresh()?); + assert_eq!(peek()?, (STORE_SIZE as u64, MAGIC[0])); + Ok(()) + } + + #[test] + fn a_short_file_with_data_is_refused_untouched() -> Result { + fill(100, 0x5a)?; + assert!(Store::open_at(PATH).err() == Some(EIO)); + assert_eq!(peek()?, (100, 0x5a)); + Ok(()) + } + + #[test] + fn a_bad_header_with_data_is_refused_untouched() -> Result { + fill(STORE_SIZE, 0x5a)?; + assert!(Store::open_at(PATH).err() == Some(EIO)); + assert_eq!(peek()?, (STORE_SIZE as u64, 0x5a)); + Ok(()) + } + + #[test] + fn a_valid_store_keeps_its_records() -> Result { + let key = Key::root(0xf0); + fill(0, 0)?; + Store::open_at(PATH)?.write(&key, b"kept")?; + let mut store = Store::open_at(PATH)?; + assert!(!store.fresh); + let value = store.read(&key)?.ok_or(ENOENT)?; + assert_eq!(&value[..], b"kept"); + Ok(()) + } +} From bf80deeb634ab95a865b6d08b3d234b65e9d88d6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:53:51 -0700 Subject: [PATCH 575/631] soc: apple: sep: Let suspend proceed when a capture is slow to stop Before suspend the driver ends any Touch ID enrol or verify and waits up to 6 s for its work item to stop. If the work item is still running after that, the sleep notifier now fails with EBUSY, which aborts the suspend. On a laptop, a lid close on battery then leaves the machine awake with the lid shut. The work item can outlast the 6 s when it waits for the SBIO exchange lock behind another caller's exchange, or when it is in a sensor bring-up whose enclave requests each wait up to 5 s for a reply. The .19 release logged the slow capture and let the suspend continue, on every Mac this driver supports. Continuing writes nothing itself: the notifier only stops waiting. The enclave's power domains stay on across sleep, so it keeps its state, and a Catacomb or xART save still under way is ordinary file and block I/O that carries on after resume, like any other I/O in flight at suspend. What continuing does leave is the case the drain was added for, now only after 6 s: a work item that may still talk to the sensor while devices go down. Every capture starts by power-cycling and registering the sensor again, so the next one starts clean. Restore the .19 behaviour: wait the same 6 s, log, and continue. The notifier cannot fail again, so the error return added for it goes too. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/soc/apple/pm.rs | 13 ++++--------- drivers/soc/apple/pm_shim.c | 9 +++++---- drivers/soc/apple/sbio.rs | 11 +++++++---- drivers/soc/apple/shim.h | 2 +- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/drivers/soc/apple/pm.rs b/drivers/soc/apple/pm.rs index a517e6e448732b..c2f59228a76e04 100644 --- a/drivers/soc/apple/pm.rs +++ b/drivers/soc/apple/pm.rs @@ -21,7 +21,7 @@ use crate::SepData; static SEP: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); extern "C" { - fn sep_pm_register(event: unsafe extern "C" fn(bool) -> c_int) -> c_int; + fn sep_pm_register(event: unsafe extern "C" fn(bool)) -> c_int; fn sep_pm_unregister(); fn sep_pm_keep_domains(dev: *mut c_void) -> c_int; } @@ -41,25 +41,20 @@ pub(crate) fn keep_domains(dev: &kernel::device::Device) -> Result<()> { /// # Safety /// Called by `pm_shim.c` from the PM notifier chain, in process context. -unsafe extern "C" fn pm_event(entering: bool) -> c_int { +unsafe extern "C" fn pm_event(entering: bool) { let ptr = SEP.load(Ordering::Acquire); if ptr.is_null() { - return 0; + return; } // SAFETY: produced by `into_foreign` in `register`, and reclaimed by // `unregister` only after `sep_pm_unregister` has waited out any running // notifier call, so it is live for this borrow. let sep = unsafe { as ForeignOwnable>::borrow(ptr) }; if entering { - if let Err(error) = sep.sleep_prepare() { - // A failing notifier is not included in the PM chain's rollback. - sep.sleep_finished(); - return error.to_errno(); - } + sep.sleep_prepare(); } else { sep.sleep_finished(); } - 0 } pub(crate) fn register(sep: Arc) -> Result<()> { diff --git a/drivers/soc/apple/pm_shim.c b/drivers/soc/apple/pm_shim.c index 2ef96c5fd96ebe..884ecbb354a062 100644 --- a/drivers/soc/apple/pm_shim.c +++ b/drivers/soc/apple/pm_shim.c @@ -14,12 +14,12 @@ #include "shim.h" -static int (*sep_pm_event)(bool entering); +static void (*sep_pm_event)(bool entering); static int sep_pm_notify(struct notifier_block *nb, unsigned long action, void *data) { - int (*event)(bool entering) = READ_ONCE(sep_pm_event); + void (*event)(bool entering) = READ_ONCE(sep_pm_event); if (!event) return NOTIFY_DONE; @@ -28,7 +28,8 @@ static int sep_pm_notify(struct notifier_block *nb, unsigned long action, case PM_SUSPEND_PREPARE: case PM_HIBERNATION_PREPARE: case PM_RESTORE_PREPARE: - return notifier_from_errno(event(true)); + event(true); + return NOTIFY_OK; case PM_POST_SUSPEND: case PM_POST_HIBERNATION: case PM_POST_RESTORE: @@ -43,7 +44,7 @@ static struct notifier_block sep_pm_nb = { .notifier_call = sep_pm_notify, }; -int sep_pm_register(int (*event)(bool entering)) +int sep_pm_register(void (*event)(bool entering)) { int ret; diff --git a/drivers/soc/apple/sbio.rs b/drivers/soc/apple/sbio.rs index 434a8b42871495..200bae32a4938a 100644 --- a/drivers/soc/apple/sbio.rs +++ b/drivers/soc/apple/sbio.rs @@ -3081,7 +3081,7 @@ impl SepData { /// The system is about to suspend. End any capture in progress, hold new /// ones, and wait for the enrol or verify work to stop touching the sensor /// before devices go down. - pub(crate) fn sleep_prepare(&self) -> Result<()> { + pub(crate) fn sleep_prepare(&self) { let woke = { let mut session = self.bio_session.lock(); self.suspending.store(true, Relaxed); @@ -3098,17 +3098,20 @@ impl SepData { let mut waited: u32 = 0; while self.captures_running.load(Relaxed) != 0 { if waited >= SLEEP_DRAIN_MS { + // Suspend anyway, as before: failing here would leave a + // laptop awake with its lid shut. The work item has been told + // to stop, and the next capture power-cycles and registers + // the sensor again. dev_warn!( self.dev, - "Touch ID: capture did not drain within {} ms; aborting suspend\n", + "Touch ID: a capture was still running {} ms after the system started to sleep; continuing\n", SLEEP_DRAIN_MS ); - return Err(EBUSY); + return; } kernel::time::delay::fsleep(kernel::time::Delta::from_millis(10)); waited += 10; } - Ok(()) } /// The system has resumed, or the suspend was aborted. diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 83839b9353691e..c8b118c582e6d7 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -25,7 +25,7 @@ void sep_cancel_delayed_work_sync(void *work); /* -- pm_shim.c ---------------------------------------------------------- */ -int sep_pm_register(int (*event)(bool entering)); +int sep_pm_register(void (*event)(bool entering)); void sep_pm_unregister(void); int sep_pm_keep_domains(void *dev); From 418c346f87ebdf46936d0e35055d0f62ee879732 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:12:11 -0700 Subject: [PATCH 576/631] tools/aurora-sep: tests: Let the T8112 SEP DMA harness build again test_t8112_sep_dma.py compiles the DART driver's DMA-window parser, map and lookup paths and firmware-slot checks against small mocks and runs the T8112 SEP DMA round trips on the result. The DART code in this tree gates its firmware-slot checks on the new fw_mirror flag and calls two new firmware-only helpers, which the mocks lack, so the harness no longer compiled and none of its three tests ran. Add fw_mirror to the mock struct apple_dart (zero, as for a T8112 DART, where the driver sets it only on locked T8140 and T8142 DARTs) and stub apple_dart_fw_mapping_matches() and apple_dart_fw_handoff_page_busy() with abort(), like the other firmware-only helpers: the SEP domain must never reach them. No assertion changes. All three tests pass on this tree, and the updated harness also passes against the .19 base, whose DART code does not use the new names. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- tools/aurora-sep/tests/test_t8112_sep_dma.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/aurora-sep/tests/test_t8112_sep_dma.py b/tools/aurora-sep/tests/test_t8112_sep_dma.py index fb39aead6a3aad..ab4ed74a16bbbb 100644 --- a/tools/aurora-sep/tests/test_t8112_sep_dma.py +++ b/tools/aurora-sep/tests/test_t8112_sep_dma.py @@ -72,7 +72,7 @@ def setUpClass(cls): struct apple_dart { u32 ias, pgsize, num_streams; u64 dma_min, dma_max, dma_offset; - bool locked, fw_handoff; + bool locked, fw_mirror, fw_handoff; struct apple_dart_hw *hw; int lock; const u64 *locked_ttbr[1][DART_MAX_TTBR]; @@ -108,7 +108,8 @@ def setUpClass(cls): struct io_pgtable { struct io_pgtable_ops ops; struct io_pgtable_cfg cfg; }; #define io_pgtable_ops_to_pgtable(ops) ((struct io_pgtable *)(ops)) #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) -/* This T8112 SEP domain has no firmware handoff; firmware-only calls are errors. */ +/* This T8112 SEP domain has no firmware mirror or handoff; firmware-only calls + * are errors. */ #define spin_lock_irqsave(lock, flags) abort() #define spin_unlock_irqrestore(lock, flags) abort() #define READ_ONCE(x) (x) @@ -126,6 +127,14 @@ def setUpClass(cls): static bool apple_dart_fw_leaf_decode(u64 pte, dma_addr_t iova, phys_addr_t *phys, int *prot) { abort(); } +static bool apple_dart_fw_mapping_matches(phys_addr_t phys, int prot, + phys_addr_t expected, int required) +{ abort(); } +static bool apple_dart_fw_handoff_page_busy(struct apple_dart *dart, + struct apple_dart_domain *domain, int i, + struct apple_dart_atomic_stream_map *map, + u64 addr) +{ abort(); } enum { APPLE_DART, APPLE_DART2 }; struct dart_io_pgtable { struct { int fmt; } iop; }; #define GENMASK_ULL(h, l) ((~0ULL >> (63 - (h))) & (~0ULL << (l))) From 0378c54a442e13c8e3f94cefb5b17dc11d12d203 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:43:13 -0700 Subject: [PATCH 577/631] usb: typec: tipd: send each IRQ_HPD request to the display route once Commit 3842a7424b69 ("usb: typec: tipd: report a coalesced sink HPD pulse as IRQ_HPD") and commit 095df56f983d ("drm: apple: relink a Type-C sink that lost its link to a hub's USB3") made the CD321x update and the DCP display route pass on every DP status that has IRQ_HPD set, even an unchanged one. That has three effects: - A partner's DP status keeps the IRQ_HPD bit of its last request. Every later update of the port, such as a power status event from a monitor that charges the Mac, sent that request again, and the display side handled each copy as a new one. - A port with more than one display route (M1 Pro/Max, M2 Pro/Max, M3 Pro) has each route called with the same state, and only the first is meant to act on it. With the route's check bypassed, every route acted on the same request. - A port with a "displayport" reference (one on the M1 Ultra and the M2 Ultra Mac Studio) already reports a coalesced HPD pulse out of band, as a disconnect and a connect. The added IRQ_HPD made the display handle the pulse a second time. Restore both checks, and let only the coalesced HPD pulse through as a new request. It is reported as IRQ_HPD only on ports without the out-of-band report. When the display route still holds IRQ_HPD from an earlier request, the status goes out without IRQ_HPD first, so that the route sees a change, as cd321x_dp_sid_replay() does for an SN201202x partner's request. That replay's edge no longer counts as the pulse itself, so a pulse that comes with such a request is sent once, by the update that follows. The cached status stays the one the route holds, which both checks rely on. Add a KUnit case for the decision. Fixes: 3842a7424b69 ("usb: typec: tipd: report a coalesced sink HPD pulse as IRQ_HPD") Fixes: 095df56f983d ("drm: apple: relink a Type-C sink that lost its link to a hub's USB3") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/dcp-fabric.c | 6 ++-- drivers/usb/typec/tipd/cd321x-pm-test.c | 18 +++++++++++ drivers/usb/typec/tipd/cd321x-pm.h | 25 ++++++++++++++++ drivers/usb/typec/tipd/core.c | 40 ++++++++++++++++++++----- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index 80c205b8a1f67f..e40f2a5059789d 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1432,14 +1432,16 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, /* * Every candidate route for this port is notified with the same state, * so only the first to arrive does the work; the others return early. + * That includes a sink's IRQ_HPD request: the port controller sends a + * new one as a changed status, clearing IRQ_HPD first when the route + * still holds the previous request (cd321x_dp_send()). * * Deliberately not a nominated coordinator: fwnode_typec_mux_get() * caps the providers one connector may have and drops the remainder * without a word, so a nominated route might never be called at all -- * and the port would then never be routed. */ - if (port->applied_valid && !(dp_status & DP_STATUS_IRQ_HPD) && - port->applied_alt == state->alt && + if (port->applied_valid && port->applied_alt == state->alt && port->applied_mode == state->mode && port->applied_status == dp_status && port->applied_conf == dp_conf) return 0; diff --git a/drivers/usb/typec/tipd/cd321x-pm-test.c b/drivers/usb/typec/tipd/cd321x-pm-test.c index 0e86ff1f42bd13..fc416af2500201 100644 --- a/drivers/usb/typec/tipd/cd321x-pm-test.c +++ b/drivers/usb/typec/tipd/cd321x-pm-test.c @@ -770,6 +770,23 @@ static void cd321x_display_taken_test(struct kunit *test) KUNIT_EXPECT_FALSE(test, cd321x_display_taken(0, false)); } +static void cd321x_dp_send_test(struct kunit *test) +{ + const u32 hpd = DP_STATUS_HPD_STATE | DP_STATUS_CON_UFP_D; + const u32 irq = hpd | DP_STATUS_IRQ_HPD; + + /* An unchanged status is not sent, also one that carries IRQ_HPD. */ + KUNIT_EXPECT_EQ(test, cd321x_dp_send(hpd, hpd, false), CD321X_DP_SEND_NONE); + KUNIT_EXPECT_EQ(test, cd321x_dp_send(irq, irq, false), CD321X_DP_SEND_NONE); + /* A new IRQ_HPD from the partner is a changed status. */ + KUNIT_EXPECT_EQ(test, cd321x_dp_send(hpd, irq, false), CD321X_DP_SEND); + KUNIT_EXPECT_EQ(test, cd321x_dp_send(irq, hpd, false), CD321X_DP_SEND); + KUNIT_EXPECT_EQ(test, cd321x_dp_send(hpd, 0, false), CD321X_DP_SEND); + /* An HPD pulse is sent once, as an edge if the route holds IRQ_HPD. */ + KUNIT_EXPECT_EQ(test, cd321x_dp_send(hpd, irq, true), CD321X_DP_SEND); + KUNIT_EXPECT_EQ(test, cd321x_dp_send(irq, irq, true), CD321X_DP_SEND_EDGE); +} + static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_reset_first_irq_before_ready_test), KUNIT_CASE(cd321x_reset_first_irq_during_prepare_test), @@ -807,6 +824,7 @@ static struct kunit_case cd321x_pm_cases[] = { KUNIT_CASE(cd321x_runtime_failure_revalidates_test), KUNIT_CASE(cd321x_restore_ignores_fault_budget_test), KUNIT_CASE(cd321x_display_taken_test), + KUNIT_CASE(cd321x_dp_send_test), {} }; diff --git a/drivers/usb/typec/tipd/cd321x-pm.h b/drivers/usb/typec/tipd/cd321x-pm.h index 7f7cae8166e776..e499e694a7d4a8 100644 --- a/drivers/usb/typec/tipd/cd321x-pm.h +++ b/drivers/usb/typec/tipd/cd321x-pm.h @@ -4,6 +4,7 @@ #include #include +#include #define CD321X_RESUME_ATTEMPTS 3 #define CD321X_LINK_EVENTS 3 @@ -225,4 +226,28 @@ static inline bool cd321x_display_taken(int ret, bool enospc) return ret == (enospc ? -ENOSPC : -EBUSY); } +enum cd321x_dp_send { + CD321X_DP_SEND_NONE, /* the display route holds this status */ + CD321X_DP_SEND, /* send the status */ + CD321X_DP_SEND_EDGE, /* send it without IRQ_HPD first, then with it */ +}; + +/* + * What a DP alt mode update with DP status @status sends to the display + * route, which holds @applied for the same pin assignment and configuration. + * The route drops a status equal to the one it holds, and so does this: an + * IRQ_HPD that the partner's last status still carries is not a new request. + * @pulse is a new request: the sink dropped HPD and raised it again since the + * last update (@status has IRQ_HPD set for it). When the route holds IRQ_HPD + * already, the request is sent as an edge, so that the route sees it. + */ +static inline enum cd321x_dp_send cd321x_dp_send(u32 applied, u32 status, + bool pulse) +{ + if (pulse) + return applied & DP_STATUS_IRQ_HPD ? CD321X_DP_SEND_EDGE : + CD321X_DP_SEND; + return applied == status ? CD321X_DP_SEND_NONE : CD321X_DP_SEND; +} + #endif /* __CD321X_PM_H__ */ diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 6c90eb3221adf0..2ceadd2d8c8c72 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -671,7 +671,9 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); } else if (st->data_status & TPS_DATA_STATUS_DP_CONNECTION) { struct typec_displayport_data dp_data; + enum cd321x_dp_send send; unsigned long mode; + bool hpd_pulse; switch (TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT(st->data_status)) { case TPS_DATA_STATUS_DP_SPEC_PIN_ASSIGNMENT_A: @@ -708,18 +710,23 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, * The sink dropped HPD and raised it again within the debounce * window, as a monitor does after losing the link. The coalesced * state looks unchanged, so the display side would never hear of - * it: report the pulse as IRQ_HPD. + * it: report the pulse as IRQ_HPD. A port with a "displayport" + * reference reports the pulse out of band instead, as a + * disconnect and a connect (see cd321x_update_work()). */ - if ((st->data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL) && - (dp_data.status & DP_STATUS_HPD_STATE) && - (cd321x->dp_status & DP_STATUS_HPD_STATE)) + hpd_pulse = !(IS_REACHABLE(CONFIG_DRM) && cd321x->connector_fwnode) && + (st->data_status_changed & CD321X_DATA_STATUS_HPD_LEVEL) && + (dp_data.status & DP_STATUS_HPD_STATE) && + (cd321x->dp_status & DP_STATUS_HPD_STATE); + if (hpd_pulse) dp_data.status |= DP_STATUS_IRQ_HPD; if (cd321x->state_valid && cd321x->state.alt == cd321x->port_altmode_dp && - !(dp_data.status & DP_STATUS_IRQ_HPD) && - cd321x->state.mode == mode && - cd321x->dp_status == dp_data.status && - cd321x->dp_conf == dp_data.conf) + cd321x->state.mode == mode && cd321x->dp_conf == dp_data.conf) + send = cd321x_dp_send(cd321x->dp_status, dp_data.status, hpd_pulse); + else + send = CD321X_DP_SEND; + if (send == CD321X_DP_SEND_NONE) return 0; cd321x->state.alt = cd321x->port_altmode_dp; @@ -728,6 +735,21 @@ static int cd321x_typec_update_mode(struct tps6598x *tps, ret = typec_thunderbolt_switch_set(cd321x->tbt_switch, &tbt_switch_data); if (ret) goto out; + if (send == CD321X_DP_SEND_EDGE) { + struct typec_displayport_data edge = { + .status = dp_data.status & ~DP_STATUS_IRQ_HPD, + .conf = dp_data.conf, + }; + + cd321x->state.data = &edge; + ret = typec_mux_set(cd321x->mux, &cd321x->state); + cd321x->state.data = &dp_data; + if (ret) { + *display_busy = cd321x_display_taken(ret, tps->h17_display); + goto out; + } + cd321x->dp_status = edge.status; + } ret = typec_mux_set(cd321x->mux, &cd321x->state); *display_busy = cd321x_display_taken(ret, tps->h17_display); if (!ret) { @@ -792,6 +814,8 @@ static void cd321x_dp_sid_replay(struct cd321x *cd321x, const struct cd321x_stat } else { return; } + /* Not the HPD pulse: the update that follows reports that, once. */ + edge.data_status_changed &= ~CD321X_DATA_STATUS_HPD_LEVEL; cd321x_typec_update_mode(&cd321x->tps, &edge, &display_busy); } From 07fc9fe06c29194561a3e5fceb846cde39fea569 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:54:13 -0700 Subject: [PATCH 578/631] drm: apple: relink a Type-C sink only when DCP drops it at link-up Commit 095df56f983d ("drm: apple: relink a Type-C sink that lost its link to a hub's USB3") answers every IRQ_HPD of a DP alt mode sink on the M1, M2 and T8122 DCPs with a relink: the sink is unplugged and plugged in again a second later, up to three times per attachment. Before, the driver retrained the link by applying the active mode again, and the display stayed connected. Sinks raise IRQ_HPD for many reasons, such as a DP-to-HDMI adapter whose TV changed state, so each request now made the display go away for a second, with hotplug events to userspace. A request within 4 s after a relink was ignored, with no retrain. The relink also started for any unplug DCP reported while HPD stayed high. That includes the HPD drop the placeholder EDID handling makes itself, and a TV behind an adapter that goes to standby long after it was set up. The relink work attached the sink again through dcp_dptx_connect_oob() after checking for a real unplug in a separate hold of hpd_mutex. An unplug in between left the output marked as attached to a cable that was gone. Its retry state was updated without a lock from the DCP callback, the Type-C mux path and dcp_poweron(). The relink's own unplug and replug changed the generation, which reset the retry count and let a second relink start while the first one was still running. Keep the relink for the failure it was written for only: - Every IRQ_HPD gets a retrain, as before. The one exception is a request that comes while a relink is queued or running: the relink connects the sink anew. Handled as before, it would find the cable marked detached during the relink's unplug and fail the Type-C mux update, which makes the port controller reconnect the whole port. - A relink starts only when DCP drops a sink that still asserts HPD within 10 s of the driver asserting HPD for that attachment (its connect, a DPMS wake or a relink), at most three times per attachment or wake, one at a time, and while the DRM connector is connected. It does not start while the driver itself holds HPD low or moves the display to another pipeline. Any other drop is handled as an unplug, as before. - The decisions move to dcp-relink.h, and their state is under a spinlock. The relink attaches the sink again in the same hold of hpd_mutex that checks for a real unplug or replug. It keeps its retry count across its own generation changes, and is dropped when Type-C work is disabled. Add KUnit cases for the decisions. Fixes: 095df56f983d ("drm: apple: relink a Type-C sink that lost its link to a hub's USB3") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/dcp-fabric-test.c | 136 ++++++++++++++++ drivers/gpu/drm/apple/dcp-fabric.c | 4 +- drivers/gpu/drm/apple/dcp-internal.h | 9 +- drivers/gpu/drm/apple/dcp-relink.h | 120 ++++++++++++++ drivers/gpu/drm/apple/dcp.c | 208 +++++++++++++++--------- 5 files changed, 396 insertions(+), 81 deletions(-) create mode 100644 drivers/gpu/drm/apple/dcp-relink.h diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index 24e4c53bdf92e5..fe17f76d9354ae 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -9,6 +9,7 @@ #include "dcp-fabric-core.h" #include "dcp-fabric-effects.h" #include "dcp-fabric-session.h" +#include "dcp-relink.h" struct fabric_fixture { struct dcp_fabric_pipeline pipeline[2]; @@ -2138,7 +2139,142 @@ static void fabric_t6030_follow_test(struct kunit *test) NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); } +static unsigned long relink_ms(unsigned long t, unsigned int ms) +{ + return t + msecs_to_jiffies(ms); +} + +/* An enabled state for attachment @generation, whose link came up @t. */ +static void relink_fresh(struct dcp_relink_state *s, u64 generation, + unsigned long t) +{ + *s = (struct dcp_relink_state){}; + dcp_relink_enable(s, true); + dcp_relink_attach(s, generation); + dcp_relink_link_up(s, generation, t); +} + +/* DCP dropping a sink right after its link came up, and only then. */ +static void relink_unplug_test(struct kunit *test) +{ + unsigned long t = 100000; + struct dcp_relink_state s; + + relink_fresh(&s, 5, t); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 5, relink_ms(t, 2000))); + KUNIT_EXPECT_EQ(test, s.tries, 1U); + KUNIT_EXPECT_TRUE(test, s.active); + + /* a sink dropped long after its link came up is unplugged */ + relink_fresh(&s, 5, t); + KUNIT_EXPECT_FALSE(test, dcp_relink_unplug(&s, 5, + relink_ms(t, DCP_RELINK_LINKUP_MS + 1))); + /* also one of an attachment whose link-up was not seen */ + KUNIT_EXPECT_FALSE(test, dcp_relink_unplug(&s, 6, relink_ms(t, 10))); + KUNIT_EXPECT_FALSE(test, s.active); + KUNIT_EXPECT_EQ(test, s.tries, 0U); + + /* a link that came up late, after a failed first connect */ + relink_fresh(&s, 5, t); + dcp_relink_link_up(&s, 5, relink_ms(t, 30000)); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 5, relink_ms(t, 32000))); + + /* a reconnect under a new generation is no link-up of the attachment */ + relink_fresh(&s, 5, t); + dcp_relink_link_up(&s, 6, relink_ms(t, 30000)); + KUNIT_EXPECT_FALSE(test, dcp_relink_unplug(&s, 6, relink_ms(t, 32000))); + dcp_relink_attach(&s, 7); + KUNIT_EXPECT_FALSE(test, dcp_relink_unplug(&s, 7, relink_ms(t, 32000))); + + /* a DPMS wake brings the link up again */ + dcp_relink_wake(&s, 7, relink_ms(t, 60000)); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 7, relink_ms(t, 62000))); +} + +/* One relink at a time; the replay keeps the retries of its attachment. */ +static void relink_running_test(struct kunit *test) +{ + unsigned long t = 100000; + struct dcp_relink_state s; + + relink_fresh(&s, 5, t); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 5, t)); + /* another drop meanwhile is an unplug, and starts nothing */ + KUNIT_EXPECT_FALSE(test, dcp_relink_unplug(&s, 5, t)); + KUNIT_EXPECT_EQ(test, s.tries, 1U); + KUNIT_EXPECT_TRUE(test, s.active); + + /* its own unplug and replug change the generation */ + dcp_relink_reattach(&s, 7, relink_ms(t, 1000)); + dcp_relink_link_up(&s, 7, relink_ms(t, 1200)); + dcp_relink_end(&s); + KUNIT_EXPECT_FALSE(test, s.active); + KUNIT_EXPECT_EQ(test, s.tries, 1U); + + /* dropped again right after the relink: the next try */ + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 7, relink_ms(t, 3200))); + KUNIT_EXPECT_EQ(test, s.tries, 2U); +} + +/* Three relinks per attachment or wake; a new attachment starts afresh. */ +static void relink_budget_test(struct kunit *test) +{ + unsigned long t = 100000; + struct dcp_relink_state s; + unsigned int i; + + relink_fresh(&s, 5, t); + for (i = 0; i < DCP_RELINK_RETRIES; i++) { + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 5, t)); + dcp_relink_reattach(&s, 5, t); + dcp_relink_end(&s); + } + KUNIT_EXPECT_FALSE(test, dcp_relink_unplug(&s, 5, t)); + KUNIT_EXPECT_EQ(test, s.tries, (u32)DCP_RELINK_RETRIES); + + dcp_relink_wake(&s, 5, t); + KUNIT_EXPECT_EQ(test, s.tries, 0U); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 5, t)); + dcp_relink_end(&s); + + dcp_relink_attach(&s, 9); + dcp_relink_link_up(&s, 9, t); + KUNIT_EXPECT_EQ(test, s.tries, 0U); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 9, t)); + KUNIT_EXPECT_EQ(test, s.tries, 1U); +} + +/* Disabled relinks are never queued, and a queued one is dropped. */ +static void relink_disabled_test(struct kunit *test) +{ + struct dcp_relink_state s; + + relink_fresh(&s, 5, 1000); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 5, 1000)); + dcp_relink_enable(&s, false); + KUNIT_EXPECT_FALSE(test, s.active); + KUNIT_EXPECT_FALSE(test, dcp_relink_unplug(&s, 5, 1000)); + KUNIT_EXPECT_FALSE(test, s.active); + dcp_relink_enable(&s, true); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 5, 1000)); +} + +/* A jiffies value of 0 still marks the link-up time. */ +static void relink_stamp_test(struct kunit *test) +{ + struct dcp_relink_state s; + + relink_fresh(&s, 1, 0); + KUNIT_EXPECT_NE(test, s.linked, 0UL); + KUNIT_EXPECT_TRUE(test, dcp_relink_unplug(&s, 1, 3)); +} + static struct kunit_case fabric_tests[] = { + KUNIT_CASE(relink_unplug_test), + KUNIT_CASE(relink_running_test), + KUNIT_CASE(relink_budget_test), + KUNIT_CASE(relink_disabled_test), + KUNIT_CASE(relink_stamp_test), KUNIT_CASE(fabric_t6030_clock_routes_test), KUNIT_CASE(fabric_t6030_plan_test), KUNIT_CASE(fabric_t6030_follow_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index e40f2a5059789d..6f33e318ae02fc 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -1568,7 +1568,9 @@ static int dcp_typec_route_set(struct typec_mux_dev *mux, /* * A T6030 external processor services the request itself, and * reads the display again if it changed; elsewhere the active - * mode is applied again. H17P always applies it again. + * mode is applied again. H17P always applies it again. While a + * relink connects the sink anew, there is nothing to apply, see + * dcp_typec_sink_relink(). */ if (dcp->typec_connector && dcp->fw_compat == DCP_FIRMWARE_H17P) dcp_retrain_oob(dcp->typec_connector); diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 4d706327793c04..f2edaed1e6a60b 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -18,6 +18,7 @@ #include "dcp-fabric.h" #include "dcp-fabric-session.h" +#include "dcp-relink.h" #include "dptxep.h" #include "dcp_backlight.h" #include "dcp-display-metadata.h" @@ -383,11 +384,11 @@ struct apple_dcp { u64 placeholder_generation; struct delayed_work placeholder_edid_wq; /* replugs replayed for a Type-C sink that lost its link */ - u32 relink_tries; - bool relink_active; - unsigned long relink_done; - u64 relink_generation; + spinlock_t relink_lock; + struct dcp_relink_state relink; struct delayed_work typec_relink_wq; + /* the driver holds the sink's HPD low itself; hpd_mutex */ + bool typec_hpd_held; bool use_timestamps; bool vrr_enabled; struct dcp_set_digital_out_mode_req mode; diff --git a/drivers/gpu/drm/apple/dcp-relink.h b/drivers/gpu/drm/apple/dcp-relink.h new file mode 100644 index 00000000000000..d00db0d6c3487e --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-relink.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef __APPLE_DCP_RELINK_H__ +#define __APPLE_DCP_RELINK_H__ + +#include +#include + +/* + * Relinking a Type-C sink: replaying an unplug and a replug of a sink that + * DCP dropped right after its link came up, although it still asserts HPD, + * see dcp_typec_relink(). These decisions take the state under the DCP's + * relink lock. + */ + +/* relinks per attachment or DPMS wake */ +#define DCP_RELINK_RETRIES 3 +#define DCP_RELINK_UNPLUG_MS 1000 +/* + * DCP drops such a sink about 2 s after HPD was asserted for it. A sink it + * drops later, with HPD still high (a TV behind a DP-to-HDMI adapter that + * goes to standby), is handled as unplugged, as before. + */ +#define DCP_RELINK_LINKUP_MS 10000 + +struct dcp_relink_state { + u64 generation; /* the Type-C attachment described */ + unsigned long linked; /* when HPD was last asserted for it, 0: unknown */ + u32 tries; + bool active; /* a relink is queued or runs */ + bool enabled; /* relinks may be queued */ +}; + +static inline unsigned long dcp_relink_stamp(unsigned long now) +{ + return now ?: 1; +} + +/* + * Another attachment starts afresh. A running relink stays marked: its own + * unplug and replug change the generation. + */ +static inline void dcp_relink_sync(struct dcp_relink_state *s, u64 generation) +{ + if (s->generation == generation) + return; + s->generation = generation; + s->linked = 0; + s->tries = 0; +} + +/* A display was attached as @generation. */ +static inline void dcp_relink_attach(struct dcp_relink_state *s, u64 generation) +{ + dcp_relink_sync(s, generation); +} + +/* + * HPD is asserted for attachment @generation, @now: its link comes up. Not + * for a reconnect that made a new generation without an attachment. + */ +static inline void dcp_relink_link_up(struct dcp_relink_state *s, + u64 generation, unsigned long now) +{ + if (s->generation == generation) + s->linked = dcp_relink_stamp(now); +} + +/* + * DCP dropped a sink that still asserts HPD, @now. True: queue a relink. + * False: handle the drop as an unplug, as before. One relink runs at a time. + */ +static inline bool dcp_relink_unplug(struct dcp_relink_state *s, u64 generation, + unsigned long now) +{ + if (s->active || !s->enabled) + return false; + dcp_relink_sync(s, generation); + if (!s->linked || + !time_before(now, s->linked + msecs_to_jiffies(DCP_RELINK_LINKUP_MS)) || + s->tries >= DCP_RELINK_RETRIES) + return false; + s->tries++; + s->active = true; + return true; +} + +/* + * The relink attached the sink again, as @generation, @now. It keeps the + * retries of the attachment it replays. + */ +static inline void dcp_relink_reattach(struct dcp_relink_state *s, + u64 generation, unsigned long now) +{ + s->generation = generation; + s->linked = dcp_relink_stamp(now); +} + +static inline void dcp_relink_end(struct dcp_relink_state *s) +{ + s->active = false; +} + +/* A DPMS wake of @generation, @now, trains the link anew as a replug does. */ +static inline void dcp_relink_wake(struct dcp_relink_state *s, u64 generation, + unsigned long now) +{ + dcp_relink_sync(s, generation); + s->tries = 0; + s->linked = dcp_relink_stamp(now); +} + +/* Relinks may (not) be queued; nothing runs once they may not. */ +static inline void dcp_relink_enable(struct dcp_relink_state *s, bool enable) +{ + s->enabled = enable; + if (!enable) + s->active = false; +} + +#endif /* __APPLE_DCP_RELINK_H__ */ diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index 690b488cb1fbeb..0100df7bb43dca 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -751,6 +751,13 @@ static int dcp_dptx_connect_session(struct apple_dcp *dcp, u32 port, if (dcp->avep) av_service_connect(dcp); + /* The sink's link comes up now, see dcp_typec_relink(). */ + if (!port && dcp_is_typec_output(dcp)) { + scoped_guard(spinlock_irqsave, &dcp->relink_lock) + dcp_relink_link_up(&dcp->relink, dcp->typec_generation, + jiffies); + } + ret = 0; goto out_unlock; @@ -1071,7 +1078,9 @@ static void dcp_placeholder_edid_work(struct work_struct *work) * Some adapters answer the first connection with a 1024x768 * placeholder and publish the panel EDID only after HPD drops * and returns. One pulse; a second placeholder is left alone. + * DCP reports the drop as an unplug, which is not a lost link. */ + WRITE_ONCE(dcp->typec_hpd_held, true); ret = dptxport_set_hpd(service, false); if (ret) { dev_info(dcp->dev, "placeholder EDID: HPD drop failed: %d\n", @@ -1092,6 +1101,7 @@ static void dcp_placeholder_edid_work(struct work_struct *work) dev_info(dcp->dev, "placeholder EDID: HPD assert failed: %d\n", ret); out_unlock: + WRITE_ONCE(dcp->typec_hpd_held, false); mutex_unlock(&dcp->hpd_mutex); } @@ -1129,78 +1139,112 @@ static void dcp_typec_reconnect_work(struct work_struct *work) dcp->typec_reconnect_tries, ret); } -#define DPTX_RELINK_RETRIES 3 -#define DPTX_RELINK_UNPLUG_MS 1000 -/* a monitor pulses HPD up to about 4 s after it lost the link */ -#define DPTX_RELINK_SETTLE_MS 4000 +/* A display is attached to the Type-C output: a new attachment. */ +static void dcp_typec_attach_locked(struct apple_dcp *dcp) +{ + lockdep_assert_held(&dcp->hpd_mutex); + dcp->typec_generation++; + WRITE_ONCE(dcp->typec_cable_connected, true); + dcp->typec_reconnect_tries = 0; + dcp->placeholder_retried = false; + cancel_delayed_work(&dcp->typec_reconnect_wq); + /* A newly attached display gets its own retries. */ + if (dcp->external_native) { + atomic_set(&dcp->external_retries, 0); + cancel_delayed_work(&dcp->external_retry_wq); + } +} + +/* Connect the link of an attached display, retrying later if that fails. */ +static int dcp_dptx_connect_attached(struct apple_dcp *dcp, u32 port) +{ + int ret = dcp_dptx_connect(dcp, port); + + if (ret && ret != -ESHUTDOWN && dcp_is_typec_output(dcp)) + dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); + + return ret; +} /* * A USB-C monitor with a built-in hub can lose the DP link when the hub * brings its SuperSpeed link up right around DCP's link training for the - * first modeset. DCP may then report the sink as unplugged two seconds - * later although it still asserts HPD, or keep driving the dead link - * without noticing while the monitor pulses HPD to be detected again. - * Neither is recovered by anything else: replay an unplug and replug of - * the sink, which retrains the link once the hub has settled, as a cable - * replug does. + * first modeset. DCP may then report the sink as unplugged about two + * seconds after HPD was asserted for it, although the sink still asserts + * HPD, and nothing else connects it again: replay an unplug and replug of + * the sink ("relink"), which trains the link once the hub has settled, as + * a cable replug does. A sink's IRQ_HPD gets a retrain, as before. * - * The retries cover one connection or one DPMS wake: the replay itself - * does not reset them, dcp_poweron() does. + * The retries cover one attachment or one DPMS wake: the replay itself + * does not reset them, dcp_poweron() does. dcp-relink.h holds the + * decisions; relink_lock protects their state. */ -static bool dcp_typec_relink_start(struct apple_dcp *dcp, const char *why) +static bool dcp_typec_relink_eligible(struct apple_dcp *dcp) { - u64 generation = READ_ONCE(dcp->typec_generation); + struct apple_connector *connector = READ_ONCE(dcp->connector); /* H17P and native external pipes recover through their own lifecycle */ if (dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->external_native || !dcp_is_typec_output(dcp) || dcp_is_usb4_output(dcp) || - dcp_is_external_h17p(dcp) || - !READ_ONCE(dcp->typec_cable_connected) || - READ_ONCE(dcp->typec_crtc_off) || !dcp->typec_connector) - return false; - - if (dcp->relink_generation != generation) { - dcp->relink_generation = generation; - dcp->relink_tries = 0; - WRITE_ONCE(dcp->relink_active, false); - } - /* the unplug a running relink replays makes DCP report one too */ - if (READ_ONCE(dcp->relink_active)) - return true; - if (dcp->relink_tries >= DPTX_RELINK_RETRIES) + dcp_is_external_h17p(dcp) || !dcp->typec_connector) return false; - - dcp->relink_tries++; - WRITE_ONCE(dcp->relink_active, true); - dev_info(dcp->dev, "%s, relinking the Type-C sink (%u/%u)\n", why, - dcp->relink_tries, DPTX_RELINK_RETRIES); - mod_delayed_work(system_freezable_wq, &dcp->typec_relink_wq, 0); - - return true; + /* + * Only a sink that is attached, shown and still connected for DRM: + * not one the driver itself let go, holds HPD low for (see + * dcp_placeholder_edid_work()) or moves to another pipeline. + */ + return READ_ONCE(dcp->typec_cable_connected) && + !READ_ONCE(dcp->typec_crtc_off) && + !READ_ONCE(dcp->typec_hpd_held) && + !READ_ONCE(dcp->tb_retiring) && + !READ_ONCE(dcp->typec_follow_retiring) && + connector && READ_ONCE(connector->connected); } /* DCP reported an unplug of a Type-C sink that still asserts HPD */ bool dcp_typec_relink(struct apple_dcp *dcp) { - return dcp_typec_relink_start(dcp, "DCP dropped a sink that asserts HPD"); + unsigned long flags; + bool start; + u32 tries; + + if (!dcp_typec_relink_eligible(dcp)) + return false; + + spin_lock_irqsave(&dcp->relink_lock, flags); + start = dcp_relink_unplug(&dcp->relink, READ_ONCE(dcp->typec_generation), + jiffies); + if (start) + mod_delayed_work(system_freezable_wq, &dcp->typec_relink_wq, 0); + tries = dcp->relink.tries; + spin_unlock_irqrestore(&dcp->relink_lock, flags); + + if (start) + dev_info(dcp->dev, + "DCP dropped a sink that asserts HPD, relinking the Type-C sink (%u/%u)\n", + tries, DCP_RELINK_RETRIES); + return start; } /* - * The sink asked for attention: IRQ_HPD or a coalesced HPD pulse. Right - * after a relink that is the sink's late reaction to the loss the relink - * already handled; another relink would only interrupt the new link. + * The sink asked for attention: IRQ_HPD or a coalesced HPD pulse. The + * caller retrains the link, as for any such request, unless a relink is + * queued or runs: that connects the sink anew, and until it does, the + * sink has no link to retrain. */ bool dcp_typec_sink_relink(struct apple_dcp *dcp) { - unsigned long done = READ_ONCE(dcp->relink_done); + unsigned long flags; + bool active; - if (done && time_before(jiffies, done + msecs_to_jiffies(DPTX_RELINK_SETTLE_MS)) && - READ_ONCE(dcp->relink_generation) == READ_ONCE(dcp->typec_generation)) { - dev_info(dcp->dev, "Type-C sink requested attention right after a relink, ignored\n"); - return true; - } + if (dcp->fw_compat == DCP_FIRMWARE_H17P) + return false; + + spin_lock_irqsave(&dcp->relink_lock, flags); + active = dcp->relink.active; + spin_unlock_irqrestore(&dcp->relink_lock, flags); - return dcp_typec_relink_start(dcp, "Type-C sink requested attention"); + return active; } static void dcp_typec_relink_work(struct work_struct *work) @@ -1209,29 +1253,42 @@ static void dcp_typec_relink_work(struct work_struct *work) container_of(to_delayed_work(work), struct apple_dcp, typec_relink_wq); struct platform_device *pdev = to_platform_device(dcp->dev); + unsigned long flags; u64 generation; + bool run; - scoped_guard(mutex, &dcp->hpd_mutex) + scoped_guard(mutex, &dcp->hpd_mutex) { generation = dcp->typec_generation; - if (generation != dcp->relink_generation || - !READ_ONCE(dcp->typec_cable_connected)) + spin_lock_irqsave(&dcp->relink_lock, flags); + run = dcp->relink.active && dcp->relink.generation == generation && + dcp->typec_cable_connected; + spin_unlock_irqrestore(&dcp->relink_lock, flags); + } + if (!run) goto out; dcp_dptx_disconnect_oob(pdev, 0); - msleep(DPTX_RELINK_UNPLUG_MS); + msleep(DCP_RELINK_UNPLUG_MS); - /* a real unplug or replug meanwhile takes over */ + /* + * Attach the sink again as a replug does, unless a real unplug or + * replug came meanwhile, which then takes over. Checked and done + * under one hold of hpd_mutex, so that neither can come in between. + */ + cancel_delayed_work_sync(&dcp->placeholder_edid_wq); scoped_guard(mutex, &dcp->hpd_mutex) { if (dcp->typec_generation != generation + 1) goto out; + dcp_typec_attach_locked(dcp); + spin_lock_irqsave(&dcp->relink_lock, flags); + dcp_relink_reattach(&dcp->relink, dcp->typec_generation, jiffies); + spin_unlock_irqrestore(&dcp->relink_lock, flags); } - dcp_dptx_connect_oob(pdev, 0); - - scoped_guard(mutex, &dcp->hpd_mutex) - dcp->relink_generation = dcp->typec_generation; - WRITE_ONCE(dcp->relink_done, jiffies); + dcp_dptx_connect_attached(dcp, 0); out: - WRITE_ONCE(dcp->relink_active, false); + spin_lock_irqsave(&dcp->relink_lock, flags); + dcp_relink_end(&dcp->relink); + spin_unlock_irqrestore(&dcp->relink_lock, flags); } static void disconnected_hpd_event(struct apple_connector *con) @@ -1302,7 +1359,6 @@ int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port) int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) { struct apple_dcp *dcp = platform_get_drvdata(pdev); - int ret; if (dcp->fw_compat == DCP_FIRMWARE_H17P) return dcp_dptx_connect_oob_h17p(pdev, port); @@ -1310,23 +1366,12 @@ int dcp_dptx_connect_oob(struct platform_device *pdev, u32 port) if (dcp_is_typec_output(dcp)) { cancel_delayed_work_sync(&dcp->placeholder_edid_wq); guard(mutex)(&dcp->hpd_mutex); - dcp->typec_generation++; - WRITE_ONCE(dcp->typec_cable_connected, true); - dcp->typec_reconnect_tries = 0; - dcp->placeholder_retried = false; - cancel_delayed_work(&dcp->typec_reconnect_wq); - /* A newly attached display gets its own retries. */ - if (dcp->external_native) { - atomic_set(&dcp->external_retries, 0); - cancel_delayed_work(&dcp->external_retry_wq); - } + dcp_typec_attach_locked(dcp); + scoped_guard(spinlock_irqsave, &dcp->relink_lock) + dcp_relink_attach(&dcp->relink, dcp->typec_generation); } - ret = dcp_dptx_connect(dcp, port); - if (ret && ret != -ESHUTDOWN && dcp_is_typec_output(dcp)) - dcp_queue_typec_reconnect(dcp, DPTX_RECONNECT_DELAY); - - return ret; + return dcp_dptx_connect_attached(dcp, port); } int dcp_dptx_disconnect_oob(struct platform_device *pdev, u32 port) @@ -1761,8 +1806,14 @@ void dcp_poweron(struct platform_device *pdev) * A DPMS wake retrains the link like a replug: give it a fresh * relink budget. No relink runs while the CRTC was off. */ - if (wait_for_typec_hpd) - dcp->relink_tries = 0; + if (wait_for_typec_hpd) { + unsigned long flags; + + spin_lock_irqsave(&dcp->relink_lock, flags); + dcp_relink_wake(&dcp->relink, + READ_ONCE(dcp->typec_generation), jiffies); + spin_unlock_irqrestore(&dcp->relink_lock, flags); + } /* * A Type-C CRTC disable releases its DPTX session. Re-establish it @@ -2272,6 +2323,8 @@ static void dcp_disable_typec_work(struct apple_dcp *dcp, bool release_cable) /* Block new enqueues as well as draining users of the AFK endpoints. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); disable_delayed_work_sync(&dcp->placeholder_edid_wq); + scoped_guard(spinlock_irqsave, &dcp->relink_lock) + dcp_relink_enable(&dcp->relink, false); disable_delayed_work_sync(&dcp->typec_relink_wq); disable_delayed_work_sync(&dcp->hdmi_settle_wq); disable_delayed_work_sync(&dcp->hdmi_recheck_wq); @@ -2285,6 +2338,8 @@ static void dcp_enable_typec_work(struct apple_dcp *dcp) enable_delayed_work(&dcp->typec_reconnect_wq); enable_delayed_work(&dcp->placeholder_edid_wq); enable_delayed_work(&dcp->typec_relink_wq); + scoped_guard(spinlock_irqsave, &dcp->relink_lock) + dcp_relink_enable(&dcp->relink, true); enable_delayed_work(&dcp->hdmi_settle_wq); enable_delayed_work(&dcp->hdmi_recheck_wq); dcp_hdmi_enable(dcp); @@ -2731,6 +2786,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, dcp_placeholder_edid_work); INIT_DELAYED_WORK(&dcp->typec_relink_wq, dcp_typec_relink_work); + spin_lock_init(&dcp->relink_lock); /* Balanced by enable at successful component bind. */ disable_delayed_work(&dcp->typec_reconnect_wq); disable_delayed_work(&dcp->placeholder_edid_wq); From d25e17cc3e469a4bb0b51aa423bcd3689a40a638 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:04:38 -0700 Subject: [PATCH 579/631] drm: apple: answer DPTX group 1 calls 3 and 4 on 13.5 firmware again On the 13.5 firmware, DCP's DPTX port sends AP calls of group 1, commands 3 and 4, around link (re)configuration, audio prepare included. The DPTX port handler takes only a command, so these calls reach the group 0 handlers of the same numbers, SET and GET_DRIVE_SETTINGS, which store addresses from the payload as drive settings. On an M2 Pro (j416s, 13.5 firmware) DCP then refused, or never confirmed, the audio start for a TV behind a USB-C to HDMI adapter. Commit 589c9a3ea798 answered these calls, and commit 8be1719a3e4c ("drm/apple: serve M1/M2/M3, T8140 and T8142 from one driver") kept that only for the H17 firmware. Answer exactly the tested calls again: on 13.5 firmware, group 1 calls 3 and 4 get their request back with the fields that dptxport_call_group1() clears. Every other call, on 12.3 and 13.5 alike, still goes to the same handler by its command, whatever its group, as in the released kernel. The 12.3 firmware is untested and therefore left as it was, and so is the 14.7 port. Reword the comment on the reply to say what the driver does. Fixes: 8be1719a3e4c ("drm/apple: serve M1/M2/M3, T8140 and T8142 from one driver") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/dptxep.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index 350a186b8ed037..a9a78aafe0caf1 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -1048,13 +1048,12 @@ static int dptxport_v2_group_call(struct apple_epic_service *service, u16 group, /* * Group 1 calls 3 and 4 come around link (re)configuration, audio prepare - * included. They carry a lane count at +0x10 in (1, 3) and at +0x00 in - * (1, 4); the J700 code reads (1, 3) as drive settings with levels per - * lane from +0x20, and its group 0 GET (0, 4) has the count at +0x00. - * Both are acked with the request, (1, 3) with +0x00..0x03 - * cleared and (1, 4) with +0x10..0x13 and +0x20 to the end cleared, so - * what DCP left there does not go back. Tested on t6020 with 13.5 - * firmware: TV through a USB-C to HDMI adapter, Dell over TB. + * included. Handled as group 0's SET and GET_DRIVE_SETTINGS, which have + * the same numbers, they stored addresses from their payload as drive + * settings, and DCP then refused to start audio to a TV behind a USB-C to + * HDMI adapter (M2 Pro, 13.5 firmware). Answer each with its request, with + * +0x00..0x03 cleared for (1, 3), and +0x10..0x13 and everything from +0x20 + * cleared for (1, 4). */ static int dptxport_call_group1(u32 command, const void *data, size_t data_size, void *reply, size_t reply_size) @@ -1114,12 +1113,28 @@ static void dptxport_teardown(struct apple_epic_service *service) dptx_service_unbind(service); } +/* + * 12.3 and 13.5 firmware: every call goes to dptxport_call() by its + * command, whatever its group, except group 1 calls 3 and 4 on 13.5, see + * dptxport_call_group1(). Only that has been tested; 12.3 has not. + */ +static int dptxport_legacy_call(struct apple_epic_service *service, u16 group, + u32 command, const void *data, size_t size, + void *reply, size_t room) +{ + if (group == 1 && (command == 3 || command == 4) && + service->ep->dcp->fw_compat == DCP_FIRMWARE_V_13_5) + return dptxport_call_group1(command, data, size, reply, room); + return dptxport_call(service, command, data, size, reply, room); +} + /* 12.3 and 13.5 firmware. */ static const struct apple_epic_service_ops dptxep_ops[] = { { .name = "AppleDCPDPTXRemotePort", .init = dptxport_init, .call = dptxport_call, + .call_v2 = dptxport_legacy_call, }, {} }; From ba5563baba3f612a31861269b3b224191a23e9d9 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:56:01 -0700 Subject: [PATCH 580/631] drm/apple: expect a moved tunnel to start its route in the parser test Commit afbcf8083bd5 ("drm/apple: let a Thunderbolt route follow the CRTC a modeset pairs it with") made dcp_crtc_needs_route_start() true for a Thunderbolt tunnel too, so that a moved tunnel starts its link before its CRTC powers on, as a moved direct route does. The parser KUnit case parser_route_programming_guard still expected false for a tunnel and has failed since. Expect true. Fixes: afbcf8083bd5 ("drm/apple: let a Thunderbolt route follow the CRTC a modeset pairs it with") Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/apple/parser-test.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index 96a119bff535e0..5defa832be7f44 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -819,8 +819,9 @@ static void parser_route_programming_guard(struct kunit *test) dcp->typec_follow_gen = 4; KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); dcp->typec_follow_gen = 5; + /* A moved Thunderbolt tunnel starts its link as a direct route does. */ dcp->active_typec_route->tunnel = true; - KUNIT_EXPECT_FALSE(test, dcp_crtc_needs_route_start(dcp)); + KUNIT_EXPECT_TRUE(test, dcp_crtc_needs_route_start(dcp)); dcp->active_typec_route->tunnel = false; KUNIT_EXPECT_FALSE(test, dcp_crtc_route_ready(&crtc->base, &state, true)); crtc_state.connector_mask = 0; From 1b50cbd1e008f61375b34e09d121f0bff5aec2a6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:49:26 -0700 Subject: [PATCH 581/631] drm/asahi: reject unsupported command fields on the M1/M2 queue path drm_asahi_cmd_render and drm_asahi_cmd_compute gained fields that only other runtimes implement: the exact prior barriers and per-stage ray query state of a render command, and the scratch layout, prior barrier, auxiliary allocation and intersection fields of a compute command. The M1/M2 queue path reads them into the command and then ignores them, so a submission that asks for one of them runs without it instead of failing, and the fields could never gain a meaning on this path without changing what such a submission does. Read render and compute commands through a helper that fails with EINVAL when any byte of the declared command from the end of the layout this path implements (320 and 128 bytes) is nonzero. That covers the new fields and any tail beyond the kernel's own structure, the same way the feature-gated render flags are already rejected. Commands that stop before that point are zero-extended as before. Upstream Mesa sends 240-byte render and 64-byte compute commands, and the M3 Mesa 320 and 128 bytes, so neither sends a byte this check looks at. Host unit tests cover the helper. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/queue/mod.rs | 19 ++++++++- drivers/gpu/drm/asahi/util.rs | 62 ++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/queue/mod.rs b/drivers/gpu/drm/asahi/queue/mod.rs index 862e6ad92e752f..83eb7acc84f0e1 100644 --- a/drivers/gpu/drm/asahi/queue/mod.rs +++ b/drivers/gpu/drm/asahi/queue/mod.rs @@ -590,6 +590,13 @@ const SQ_RENDER: usize = 0; const SQ_COMPUTE: usize = 1; const SQ_COUNT: usize = 2; +/// Bytes of a render command this queue implements. The fields from `prior_vdm` on belong to +/// other runtimes and must be zero here, as must any bytes beyond the kernel's structure. +const RENDER_SUPPORTED_SIZE: usize = core::mem::offset_of!(uapi::drm_asahi_cmd_render, prior_vdm); +/// Bytes of a compute command this queue implements; see [`RENDER_SUPPORTED_SIZE`]. +const COMPUTE_SUPPORTED_SIZE: usize = + core::mem::offset_of!(uapi::drm_asahi_cmd_compute, scs_layout); + // SAFETY: All bit patterns are valid by construction. unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_header {} unsafe impl AnyBitPattern for uapi::drm_asahi_cmd_render {} @@ -850,7 +857,11 @@ impl Queue for Queue::ver { match header.cmd_type as u32 { uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER => { - let render: uapi::drm_asahi_cmd_render = cmdbuf.read_up_to(header_size)?; + let render: uapi::drm_asahi_cmd_render = cmdbuf + .read_supported(header_size, RENDER_SUPPORTED_SIZE) + .inspect_err(|_| { + cls_pr_debug!(Errors, "Unsupported render command fields\n") + })?; self.inner.submit_render( &mut job, @@ -875,7 +886,11 @@ impl Queue for Queue::ver { )?; } uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE => { - let compute: uapi::drm_asahi_cmd_compute = cmdbuf.read_up_to(header_size)?; + let compute: uapi::drm_asahi_cmd_compute = cmdbuf + .read_supported(header_size, COMPUTE_SUPPORTED_SIZE) + .inspect_err(|_| { + cls_pr_debug!(Errors, "Unsupported compute command fields\n") + })?; self.inner.submit_compute( &mut job, diff --git a/drivers/gpu/drm/asahi/util.rs b/drivers/gpu/drm/asahi/util.rs index 046d360996184a..121e8bf8c1ed42 100644 --- a/drivers/gpu/drm/asahi/util.rs +++ b/drivers/gpu/drm/asahi/util.rs @@ -126,6 +126,23 @@ impl<'a> Reader<'a> { Ok(obj) } + /// Reads a record like [`Self::read_up_to`], for a path that implements only the first + /// `supported` bytes of `T`. Any nonzero byte of the declared record from `supported` on, + /// whether in a field this path does not implement or in a tail beyond `T`, fails with + /// `EINVAL` instead of being ignored. + pub(crate) fn read_supported( + &mut self, + declared_size: usize, + supported: usize, + ) -> Result { + let end = self.offset.checked_add(declared_size).ok_or(EINVAL)?; + let record = self.buffer.get(self.offset..end).ok_or(EINVAL)?; + if record.iter().skip(supported).any(|byte| *byte != 0) { + return Err(EINVAL); + } + self.read_up_to(declared_size) + } + pub(crate) fn read(&mut self) -> Result { self.read_up_to(core::mem::size_of::()) } @@ -179,4 +196,49 @@ mod tests { assert_eq!(pair.second, 0); assert!(reader.is_empty()); } + + #[test] + fn read_supported_accepts_zero_unsupported_fields_and_tail() { + let raw = [5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0]; + let mut reader = Reader::new(&raw); + let pair: Pair = reader.read_supported(12, 4).unwrap(); + assert_eq!(pair.first, 5); + assert_eq!(pair.second, 0); + assert_eq!(reader.read::().unwrap(), 6); + assert!(reader.is_empty()); + } + + #[test] + fn read_supported_accepts_a_record_shorter_than_the_supported_part() { + let raw = [7, 0]; + let mut reader = Reader::new(&raw); + let pair: Pair = reader.read_supported(2, 4).unwrap(); + assert_eq!(pair.first, 7); + assert_eq!(pair.second, 0); + assert!(reader.is_empty()); + } + + #[test] + fn read_supported_rejects_an_unsupported_field() { + let raw = [1, 0, 0, 0, 0, 1, 0, 0]; + let mut reader = Reader::new(&raw); + assert!(reader.read_supported::(8, 4).is_err()); + } + + #[test] + fn read_supported_rejects_a_nonzero_tail_beyond_the_type() { + let raw = [1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0x80]; + let mut reader = Reader::new(&raw); + assert!(reader.read_supported::(12, 8).is_err()); + let mut reader = Reader::new(&raw); + let pair: Pair = reader.read_supported(8, 8).unwrap(); + assert_eq!(pair.second, 2); + } + + #[test] + fn read_supported_rejects_a_truncated_record() { + let raw = [1, 0, 0, 0]; + let mut reader = Reader::new(&raw); + assert!(reader.read_supported::(8, 8).is_err()); + } } From 8af856032038b1794b40e59941d7aab6e762ecf6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:49:27 -0700 Subject: [PATCH 582/631] drm/asahi: keep the range TLBI base address in its field A range TLBI operand carries the base address in bits 36:0 and the translation table level hint in bits 38:37. tlbi_range_or_asid() masked the page number to 38 bits, so a page number with bit 37 set also set a level hint. Mask it to 37 bits, as tlbi_range() already does. Only the G17P (T8140) runtime uses this helper. Its kernel VM sits in the upper half of the GPU address space, where page-number bit 37 (VA bit 51) is always set, so every multi-page range invalidation of a kernel mapping carried a level-1 hint instead of none. Without the hint the invalidation covers entries at every table level, a superset of what the hinted one covered, so no invalidation is narrowed. User VM addresses are below that bit and their operands do not change. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/asahi/mem.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/asahi/mem.rs b/drivers/gpu/drm/asahi/mem.rs index f354e4bb1f71ab..988040dd15bffb 100644 --- a/drivers/gpu/drm/asahi/mem.rs +++ b/drivers/gpu/drm/asahi/mem.rs @@ -143,8 +143,8 @@ const TLBI_RANGE_MAX: usize = 0x200_0000; const TLBI_RANGE_SCALE1_MIN: usize = 0x10_0000; /// Translation granule field value for 16 KiB pages. const TLBI_TG_16K: u64 = 2; -/// Width of the base address field of a range TLBI operand. -const TLBI_RANGE_BADDR_MASK: u64 = (1 << 38) - 1; +/// Base address field of a range TLBI operand, bits 36:0. Bits 38:37 hold the TTL hint. +const TLBI_RANGE_BADDR_MASK: u64 = (1 << 37) - 1; /// Invalidate a range of pages for a given ASID with a single range operation. /// From 0a934a7b3115db29a4ad2ce638353c3ccb240d72 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:49:27 -0700 Subject: [PATCH 583/631] wifi: brcmfmac: msgbuf: look up the traced flowring under its lock brcmf_msgbuf_process_txstatus() read the completed frame's flowring for every tx completion on every chip, to filter the awdl_trace log line, even with the trace off. The read checked flow->rings[flowid] and then dereferenced it again in brcmf_flowring_ifidx_get() without a lock, while brcmf_msgbuf_delete_flowring() can free the same ring from process context on its error paths (bus down, a full control ring, a failed submit). A completion handled in that window reads freed memory. Look the ring up only when awdl_trace is set, and under the flowring's commonring lock, the lock brcmf_msgbuf_delete_flowring() and the tx flow worker already use for that ring. Free the host side of a flowring under the same lock, through one helper used by every msgbuf path that deletes a flowring. A commonring that was never configured has no initialized lock and never carried a frame, so both sides skip the lock for it. With the trace off, tx completion no longer touches the flowring at all, as before the AWDL trace was added. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../broadcom/brcm80211/brcmfmac/msgbuf.c | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c index d841d9e80637d2..3d79b6ed471865 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c @@ -601,6 +601,45 @@ static void brcmf_msgbuf_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb) { } +/* Free the host state of a flowring. The ring's commonring lock, which the tx + * flow worker holds while it uses the ring, also orders this with the tx + * status lookup in brcmf_msgbuf_flowring_ifidx(). A commonring that was never + * configured has no initialized lock; no frame was ever posted on it, so no tx + * status can refer to it. + */ +static void +brcmf_msgbuf_flowring_delete(struct brcmf_msgbuf *msgbuf, u16 flowid) +{ + struct brcmf_commonring *commonring = msgbuf->flowrings[flowid]; + + if (!commonring->inited) { + brcmf_flowring_delete(msgbuf->flow, flowid); + return; + } + + brcmf_commonring_lock(commonring); + brcmf_flowring_delete(msgbuf->flow, flowid); + brcmf_commonring_unlock(commonring); +} + +/* The interface index the host gave a flowring, or 0 once the ring is gone. */ +static u8 +brcmf_msgbuf_flowring_ifidx(struct brcmf_msgbuf *msgbuf, u16 flowid) +{ + struct brcmf_commonring *commonring = msgbuf->flowrings[flowid]; + u8 ifidx = 0; + + if (!commonring->inited) + return 0; + + brcmf_commonring_lock(commonring); + if (msgbuf->flow->rings[flowid]) + ifidx = brcmf_flowring_ifidx_get(msgbuf->flow, flowid); + brcmf_commonring_unlock(commonring); + + return ifidx; +} + static void brcmf_msgbuf_remove_flowring(struct brcmf_msgbuf *msgbuf, u16 flowid) { @@ -614,7 +653,7 @@ brcmf_msgbuf_remove_flowring(struct brcmf_msgbuf *msgbuf, u16 flowid) dma_free_coherent(msgbuf->drvr->bus_if->dev, dma_sz, dma_buf, msgbuf->flowring_dma_handle[flowid]); - brcmf_flowring_delete(msgbuf->flow, flowid); + brcmf_msgbuf_flowring_delete(msgbuf, flowid); } @@ -657,7 +696,7 @@ brcmf_msgbuf_flowring_create_worker(struct brcmf_msgbuf *msgbuf, GFP_KERNEL); if (!dma_buf) { bphy_err(drvr, "dma_alloc_coherent failed\n"); - brcmf_flowring_delete(msgbuf->flow, flowid); + brcmf_msgbuf_flowring_delete(msgbuf, flowid); return BRCMF_FLOWRING_INVALID_ID; } @@ -982,18 +1021,20 @@ brcmf_msgbuf_process_txstatus(struct brcmf_msgbuf *msgbuf, void *buf) * (0x0006 EXPIRED) after the ring was freed -- awdl0's VO multicast * never completes while the ring is open (observed as a NULL deref * here on `awdl-if destroy`). The skb and its pktid are unaffected. + * Only the trace looks the ring up, under the ring's lock, so that a + * ring removed from another context is not read while it is freed. */ - ring_ifidx = msgbuf->flow->rings[flowid] ? - brcmf_flowring_ifidx_get(msgbuf->flow, flowid) : 0; - if (ring_ifidx && READ_ONCE(brcmf_awdl_trace) && - __ratelimit(&brcmf_awdl_txs_rs)) - pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", - ring_ifidx, tx_status->msg.ifidx, - ifp ? ifp->is_awdl : -1, flowid, - (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), - le16_to_cpu(tx_status->tx_status), - le16_to_cpu(tx_status->metadata_len), - skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + if (READ_ONCE(brcmf_awdl_trace)) { + ring_ifidx = brcmf_msgbuf_flowring_ifidx(msgbuf, flowid); + if (ring_ifidx && __ratelimit(&brcmf_awdl_txs_rs)) + pr_info("brcmfmac: awdl txstatus ring_ifidx=%u msg_ifidx=%u awdl=%d flow=%u status=%d tx_status=0x%04x meta=%u dst=%pM\n", + ring_ifidx, tx_status->msg.ifidx, + ifp ? ifp->is_awdl : -1, flowid, + (int)(s16)le16_to_cpu(tx_status->compl_hdr.status), + le16_to_cpu(tx_status->tx_status), + le16_to_cpu(tx_status->metadata_len), + skb->len >= ETH_ALEN ? skb->data : (u8 *)"\0\0\0\0\0\0"); + } brcmf_txfinalize(ifp, skb, true); } @@ -1374,7 +1415,7 @@ brcmf_msgbuf_process_flow_ring_delete_response(struct brcmf_msgbuf *msgbuf, if (status) { bphy_err(drvr, "Flowring deletion failed, code %d\n", status); - brcmf_flowring_delete(msgbuf->flow, flowid); + brcmf_msgbuf_flowring_delete(msgbuf, flowid); return; } brcmf_dbg(MSGBUF, "Flowring %d Delete response status %d\n", flowid, From d4f767b6f7d373fc00d8e9824e6b5e00f7dc43a6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:49:28 -0700 Subject: [PATCH 584/631] wifi: brcmfmac: drop the runtime AWDL interface_create parameters awdl_create_flags, awdl_if_index and awdl_bssid were writable at runtime (0644) so the values could be swept during bring-up. That sweep is over, and a runtime write of awdl_bssid frees the old string while brcmf_cfg80211_request_if_internal() may be parsing it. Use constants for the flags (MAC_USE | IF_INDEX_USE | BSSID_USE, the 0x1a default) and the interface index (2, the default), and make awdl_bssid read-only after load (0444). Interface creation sends the same values as before when no parameter was set. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../brcm80211/brcmfmac/interface_create.c | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index 3a4f6b3be44708..bfdf87409870ad 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -25,20 +25,12 @@ /* Creating a usable AWDL interface requires interface_create flags * MAC_USE | IF_INDEX_USE | BSSID_USE together with an explicit BSSID and - * an interface index; brcmfmac otherwise sends MAC_USE alone, which the - * firmware accepts but leaves the AWDL iovars unusable. These parameters - * exist so the three values can be swept without rebuilding the module. + * an interface index (BRCMF_AWDL_CREATE_FLAGS, BRCMF_AWDL_IF_INDEX); + * brcmfmac otherwise sends MAC_USE alone, which the firmware accepts but + * leaves the AWDL iovars unusable. The BSSID is set at module load only. */ -static uint brcmf_awdl_create_flags = 0x1a; -module_param_named(awdl_create_flags, brcmf_awdl_create_flags, uint, 0644); -MODULE_PARM_DESC(awdl_create_flags, "interface_create flags for AWDL"); - -static uint brcmf_awdl_if_index = 2; -module_param_named(awdl_if_index, brcmf_awdl_if_index, uint, 0644); -MODULE_PARM_DESC(awdl_if_index, "interface_create if_index for AWDL"); - static char *brcmf_awdl_bssid = "00:25:00:ff:94:73"; -module_param_named(awdl_bssid, brcmf_awdl_bssid, charp, 0644); +module_param_named(awdl_bssid, brcmf_awdl_bssid, charp, 0444); MODULE_PARM_DESC(awdl_bssid, "interface_create BSSID for AWDL"); #define BRCMF_INTERFACE_CREATE_VER_1 1 @@ -64,6 +56,12 @@ MODULE_PARM_DESC(awdl_bssid, "interface_create BSSID for AWDL"); #define BRCMF_INTERFACE_BSSID_DONT_USE (0 << 4) #define BRCMF_INTERFACE_BSSID_USE (1 << 4) +/* interface_create flags and interface index of an AWDL interface */ +#define BRCMF_AWDL_CREATE_FLAGS (BRCMF_INTERFACE_MAC_USE | \ + BRCMF_INTERFACE_IF_INDEX_USE | \ + BRCMF_INTERFACE_BSSID_USE) +#define BRCMF_AWDL_IF_INDEX 2 + /* * From revision >= 2 Bit 0 of flags field will not be used for STA or AP interface creation. * "iftype" field shall be used for identifying the interface type. @@ -204,16 +202,15 @@ static int brcmf_cfg80211_request_if_internal(struct brcmf_if *ifp, u32 version, iface_v3.mac_addr); } if (awdl) { - flags = brcmf_awdl_create_flags; + flags = BRCMF_AWDL_CREATE_FLAGS; /* brcmf_awdl_add_vif() passes NULL, so derive a MAC * here rather than send MAC_USE with an all-zero * address. */ - if ((flags & BRCMF_INTERFACE_MAC_USE) && - is_zero_ether_addr(iface_v3.mac_addr)) + if (is_zero_ether_addr(iface_v3.mac_addr)) brcmf_set_vif_sta_macaddr(ifp, iface_v3.mac_addr); - iface_v3.if_index = brcmf_awdl_if_index; + iface_v3.if_index = BRCMF_AWDL_IF_INDEX; if (mac_pton(brcmf_awdl_bssid, iface_v3.bssid)) { brcmf_info("awdl interface_create: flags 0x%x, if_index %u, bssid %pM, mac %pM\n", flags, iface_v3.if_index, From e75cc822fa9cac1727309129def86f5207c74a7c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:50:34 -0700 Subject: [PATCH 585/631] wifi: brcmfmac: add an off-switch for AWDL interface creation AWDL interface creation through the AWDL vendor command was tested on BCM4387, the only chip that Brent Kearney's AWDL commits ("wifi: brcmfmac: add AWDL interface creation for BCM4387", "wifi: brcmfmac: create the AWDL interface") name. The command is registered on every brcmfmac chip, so per our device trees it is available on: - BCM4387 (tested): the M1 Pro/Max and M3 Pro/Max MacBook Pros, the M2 MacBook Airs, the M1 Max/Ultra Mac Studios and the M2 Ultra Mac Pro. - BCM4378 (untested): the M1 MacBook Air, 13" MacBook Pro, Mac mini and iMacs, and the M2 13" MacBook Pro. - BCM4388 (untested): the M2 Pro/Max MacBook Pros, the M2 and M2 Pro Mac minis, the M2 Max/Ultra Mac Studios, and the M3 14" MacBook Pro, MacBook Airs and iMacs. The feature stays on by default, so the omdrop AirDrop tool works with no module options. Add an awdl module parameter as its off-switch: awdl=1, the default, registers the vendor command as before; awdl=0 leaves it out of the wiphy's vendor commands, so a request for it fails with EOPNOTSUPP like any unknown vendor command, and nothing in the driver creates or drives an AWDL interface. The parameter is read-only after load (0444). The default awdl_bssid stays the fixed BSSID that every AWDL node uses, which peers need in order to accept the interface's frames. awdl_bssid still overrides it at load. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c | 6 ++++-- .../net/wireless/broadcom/brcm80211/brcmfmac/common.c | 9 +++++++++ .../net/wireless/broadcom/brcm80211/brcmfmac/common.h | 1 + .../broadcom/brcm80211/brcmfmac/interface_create.c | 6 ++++-- .../net/wireless/broadcom/brcm80211/brcmfmac/vendor.h | 1 + 5 files changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d1b2aa550ee5a0..8d20847cb7fc9a 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -8281,9 +8281,11 @@ static int brcmf_setup_wiphy(struct wiphy *wiphy, struct brcmf_if *ifp) gscan = brcmf_feat_is_enabled(ifp, BRCMF_FEAT_GSCAN); brcmf_pno_wiphy_params(wiphy, gscan); } - /* vendor commands/events support */ + /* vendor commands/events support; awdl=0 leaves out the AWDL command */ wiphy->vendor_commands = brcmf_vendor_cmds; - wiphy->n_vendor_commands = BRCMF_VNDR_CMDS_LAST - 1; + wiphy->n_vendor_commands = ifp->drvr->settings->awdl_enable ? + BRCMF_VNDR_CMDS_LAST - 1 : + BRCMF_VNDR_CMDS_AWDL - 1; if (brcmf_feat_is_enabled(ifp, BRCMF_FEAT_WOWL)) brcmf_wiphy_wowl_params(wiphy, ifp); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c index 2c99a990315682..f7bd81af34d13c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c @@ -56,6 +56,14 @@ static int brcmf_p2p_enable; module_param_named(p2pon, brcmf_p2p_enable, int, 0); MODULE_PARM_DESC(p2pon, "Enable legacy p2p management functionality"); +/* AWDL interface creation through the vendor command is on by default. + * awdl=0 at module load turns it off: the AWDL vendor command is then not + * registered, and nothing in the driver creates or drives an AWDL interface. + */ +static int brcmf_awdl_enable = 1; +module_param_named(awdl, brcmf_awdl_enable, int, 0444); +MODULE_PARM_DESC(awdl, "AWDL interface creation through the vendor command (default 1, 0 turns it off)"); + static int brcmf_feature_disable; module_param_named(feature_disable, brcmf_feature_disable, int, 0); MODULE_PARM_DESC(feature_disable, "Disable features"); @@ -514,6 +522,7 @@ struct brcmf_mp_device *brcmf_get_module_param(struct device *dev, /* start by using the module parameters */ settings->p2p_enable = !!brcmf_p2p_enable; + settings->awdl_enable = !!brcmf_awdl_enable; settings->feature_disable = brcmf_feature_disable; settings->fcmode = brcmf_fcmode; settings->roamoff = !!brcmf_roamoff; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h index 3bdb6984b2dd4b..21488a791c384c 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h @@ -44,6 +44,7 @@ extern struct brcmf_mp_global_t brcmf_mp_global; */ struct brcmf_mp_device { bool p2p_enable; + bool awdl_enable; unsigned int feature_disable; int fcmode; bool roamoff; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index bfdf87409870ad..98933b588fb79f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -27,11 +27,13 @@ * MAC_USE | IF_INDEX_USE | BSSID_USE together with an explicit BSSID and * an interface index (BRCMF_AWDL_CREATE_FLAGS, BRCMF_AWDL_IF_INDEX); * brcmfmac otherwise sends MAC_USE alone, which the firmware accepts but - * leaves the AWDL iovars unusable. The BSSID is set at module load only. + * leaves the AWDL iovars unusable. The default BSSID is the fixed one that + * every AWDL node uses, so peers accept the interface's frames; awdl_bssid + * can override it at module load only. */ static char *brcmf_awdl_bssid = "00:25:00:ff:94:73"; module_param_named(awdl_bssid, brcmf_awdl_bssid, charp, 0444); -MODULE_PARM_DESC(awdl_bssid, "interface_create BSSID for AWDL"); +MODULE_PARM_DESC(awdl_bssid, "interface_create BSSID for AWDL (default: the fixed AWDL BSSID)"); #define BRCMF_INTERFACE_CREATE_VER_1 1 #define BRCMF_INTERFACE_CREATE_VER_2 2 diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h index 74886de2ba78de..55e792b0b2c9c4 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/vendor.h @@ -11,6 +11,7 @@ enum brcmf_vndr_cmds { BRCMF_VNDR_CMDS_UNSPEC, BRCMF_VNDR_CMDS_DCMD, + /* Last in brcmf_vendor_cmds[]: not registered with awdl=0. */ BRCMF_VNDR_CMDS_AWDL, BRCMF_VNDR_CMDS_LAST }; From 347adeccb0c8853d32bfca1ac7cf414bf0939bc9 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:09:13 -0700 Subject: [PATCH 586/631] wifi: brcmfmac: refuse AWDL on interface_create v1 and v2 Only the version 3 interface_create request carries the BSSID and interface index an AWDL interface needs, and only that branch fills them in. On firmware that reports version 1 or 2, an AWDL request went out without them, and on version 1 with the interface type folded into the flags. On firmware that reports no version, the fallback calls pass the interface type and the version in each other's place, so after a version 2 AWDL request the second attempt asked for an AP interface. Refuse an AWDL request with EOPNOTSUPP, and log why, unless the firmware reports interface_create version 3. The AWDL vendor command returns that error to userspace. Station and AP requests are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../broadcom/brcm80211/brcmfmac/interface_create.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c index 98933b588fb79f..c25782b6e1c080 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/interface_create.c @@ -244,6 +244,17 @@ static int brcmf_cfg80211_request_if(struct brcmf_if *ifp, iface_create_ver = 0; err = brcmf_fil_bsscfg_int_query(ifp, "interface_create", &iface_create_ver); + /* Only the version 3 request can describe an AWDL interface. Refuse it + * on older firmware, including firmware too old to report a version, + * rather than send a request that does not say what is asked for. + */ + if (if_type == BRCMF_INTERFACE_TYPE_AWDL && + (err || iface_create_ver != BRCMF_INTERFACE_CREATE_VER_3)) { + bphy_err(ifp->drvr, + "AWDL needs interface_create version 3 (firmware: %d)\n", + err ? -1 : (int)iface_create_ver); + return -EOPNOTSUPP; + } if (!err) { err = brcmf_cfg80211_request_if_internal(ifp, iface_create_ver, if_type, macaddr); From 4bbc184bb298f3459f0cf3c5a695cd3b18ed0669 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:09:56 -0700 Subject: [PATCH 587/631] wifi: brcmfmac: run queued firmware events before stopping their worker brcmf_detach() now stops the bus and the firmware event worker before it removes the interfaces. brcmf_fweh_detach() stopped the worker with cancel_work_sync() alone, so when an event had just been queued and its worker scheduled but not yet started, the work was cancelled, the WARN_ON on a non-empty event queue fired, and the queued events leaked when the event state was freed. brcmf_detach() runs on module unload, PCIe removal, resume after the chip lost power and reset after a firmware crash, on every Mac. Flush the worker before cancelling it, so the events still queued are handled while the interfaces exist, as brcmf_detach() intends ("Drain IF events before their interfaces and protocol state go away"), and the queue is empty when the worker is cancelled. The WARN_ON still flags an event queued by a producer that was not stopped. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index 3e06c23defcb3f..f990757183b9b8 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -385,8 +385,13 @@ void brcmf_fweh_detach(struct brcmf_pub *drvr) if (!fweh) return; - /* cancel the worker if initialized */ + /* Stop the worker if initialized. brcmf_detach() calls this with the + * event producers stopped, before its interfaces go away, so first run + * the events still queued: a worker that was queued but had not started + * would otherwise be cancelled with its events left on the queue. + */ if (fweh->event_work.func) { + flush_work(&fweh->event_work); cancel_work_sync(&fweh->event_work); WARN_ON(!list_empty(&fweh->event_q)); } From 4c56f9cd3033e5f65ef9c61bc7b396d50bce23f7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:33:57 -0700 Subject: [PATCH 588/631] rust: restore --no-layout-tests, make layout checks opt-in Commit 97655bb70bc8 ("rust: check generated layouts and provide corrected private bindgen") dropped the upstream --no-layout-tests flag from cmd_bindgen. The generated bindings then carry bindgen's size, alignment and offset assertions, and stock bindgen cannot build the kernel any more: with the release configuration, `make prepare` fails on the size of ns_common, pid_namespace, user_namespace and cgroup_namespace with bindgen 0.73.2, and additionally task_struct, zone and pglist_data with 0.73.1. Our release recipe builds with the distribution's stock rust-bindgen, so the package build breaks. These are size-only mismatches at the end of the types: every offset and alignment assertion passes. The .19 release builds the same mismatches, with the assertions not generated. Pass --no-layout-tests again by default, as upstream does, so the bindgen command line matches .19 again. Building with `make BINDGEN_LAYOUT_TESTS=1` drops the flag and keeps the assertions, so the private generator in tools/aurora-bindgen can still be used to check a configuration. Update the documentation and the recipe's hint to describe that. Verified with the release configuration (olddefconfig), ARCH=arm64 LLVM=1 W=1: `make prepare` and `make rust/` succeed with no warnings using /usr/bin/bindgen 0.73.1 and using unpatched upstream bindgen 0.73.2. With BINDGEN_LAYOUT_TESTS=1 both stop with E0080 size errors on the types above; with the patched generator from tools/aurora-bindgen, all 36943 assertions build. Switching the variable regenerates the bindings. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- Documentation/rust/bindgen-layout-checks.rst | 107 +++++++++++-------- rust/Makefile | 15 ++- tools/aurora-bindgen/README.rst | 28 +++-- tools/aurora-bindgen/build.py | 2 +- 4 files changed, 93 insertions(+), 59 deletions(-) diff --git a/Documentation/rust/bindgen-layout-checks.rst b/Documentation/rust/bindgen-layout-checks.rst index 8ad8c24526b672..aaac5b24fb742e 100644 --- a/Documentation/rust/bindgen-layout-checks.rst +++ b/Documentation/rust/bindgen-layout-checks.rst @@ -3,47 +3,66 @@ Checking generated binding layouts ================================== -Kernel bindings retain bindgen's layout checks. With the configured Rust target, -these are unconditional constant assertions, checked while the bindings crate -is compiled. They compare Rust sizes, alignments and field offsets against the -values obtained from Clang, without executing kernel code. - -Do not disable these checks to make an incompatible binding generator compile. -A type can have the right size and still have incorrect member offsets. For -example, an empty GNU C struct may have size zero and alignment 64; losing that -alignment moves later fields in an enclosing struct without necessarily -changing the enclosing size. - -Known generator requirements ----------------------------- - -Unmodified bindgen 0.73.2 loses explicit alignment on defined zero-sized C -structs. It also omits the named anonymous record member accepted by Clang's -``-fms-extensions`` in a declaration such as:: - - struct payload { unsigned long words[20]; }; - union container { - struct payload; - unsigned long other; - }; - -The generator used for this tree must preserve these layouts. Use a corrected -bindgen build through the normal ``BINDGEN=/path/to/bindgen`` make variable. -``tools/aurora-bindgen/`` contains pinned patches, a private build recipe and -compile-only valid-input tests. It never installs a system-wide tool. The -associated build provenance must retain the generator source revision, -patches, binary digest, and valid-input C/Rust layout checks. A version string -alone does not identify a locally corrected tool. Do not add dummy kernel -storage, hardcode task offsets, or change scheduler configuration to conceal a -binding generator error. - -After a binding layout correction, rebuild the kernel Image and all matching -modules. The generated bindings are consumed by the Rust kernel crate and by -multiple drivers; rebuilding just one module can leave incompatible layouts in -the same kernel. Keep the old Image and its own modules together as a fallback. - -The minimum accepted bindgen version is 0.71.1. Its Rust 1.77 ``offset_of`` -feature enables unconditional const layout assertions; the configured Rust -target 1.85 selects that path rather than ``#[test]``-only functions. The pinned -0.73.2 tool has the same behavior. See the source links and explicit target -accounting in ``tools/aurora-bindgen/README.rst``. +bindgen can emit compile-time assertions that compare the size, alignment and +field offsets of every generated Rust type with the values Clang computes for +the C type. As upstream, the kernel build passes ``--no-layout-tests`` to +bindgen, so the generated bindings do not contain these assertions and any +supported bindgen version can build the kernel. + +To check the layouts of a configuration, build with ``BINDGEN_LAYOUT_TESTS=1``:: + + make LLVM=1 BINDGEN_LAYOUT_TESTS=1 prepare + +The bindings are then generated with the assertions, which are checked while +the ``bindings`` and ``uapi`` crates are compiled; nothing is executed. A +mismatch stops the build with an ``E0080`` error naming the type and the +property (``Size of``, ``Alignment of`` or ``Offset of``). Changing the +variable regenerates the bindings. The check covers only the configuration +being built. + +Known generator limitations +--------------------------- + +Unmodified bindgen 0.73 does not reproduce every C layout used by the kernel: + +* It omits a named anonymous record member, which Clang accepts with + ``-fms-extensions``, in a declaration such as:: + + struct payload { unsigned long words[20]; }; + union container { + struct payload; + unsigned long other; + }; + +* It can drop the explicit alignment of a defined zero-sized C struct, such + as ``struct empty {} __attribute__((aligned(64)));``. + +With the assertions enabled and stock bindgen 0.73.2, an arm64 Apple +silicon configuration of this tree fails on the size of ``ns_common`` and of +the namespace structures that embed it; bindgen 0.73.1 also fails on +``task_struct``, ``zone`` and ``pglist_data``. In those builds every offset +and alignment assertion passes: bindgen pads each field to the offset Clang +reports, so only the end of the Rust type is short. Rust code that reaches +such a type through a pointer and reads its fields gets the right data, but +must not rely on ``size_of`` for it, copy it by value or index an array of +it. The affected types depend on the bindgen version and the configuration, +which is what the check is for. + +Corrected generator +------------------- + +``tools/aurora-bindgen/`` contains pinned patches for bindgen 0.73.2 that +repair both cases, a private build recipe and compile-only tests; see +``tools/aurora-bindgen/README.rst``. It installs nothing system-wide. To check +the layouts with it:: + + make LLVM=1 BINDGEN=/path/to/bindgen BINDGEN_LAYOUT_TESTS=1 ... + +The generated bindings are consumed by the Rust kernel crate and by several +drivers. After changing the bindings generator, rebuild the kernel Image and +all modules together. + +The assertions need bindgen 0.71.1 or later (the kernel minimum) together with +the configured ``--rust-target 1.85``: that selects unconditional constant +assertions rather than ``#[test]`` functions, so a build with +``BINDGEN_LAYOUT_TESTS=1`` checks them without a Rust test configuration. diff --git a/rust/Makefile b/rust/Makefile index 2fee22bdb0e146..915d3a6f9e0b25 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -443,14 +443,23 @@ endif # architecture instead of generating `usize`. bindgen_c_flags_final = $(bindgen_c_flags_lto) -fno-builtin -D__BINDGEN__ +# Layout assertions are off by default, as upstream: stock bindgen 0.73 +# computes a different total size for a few kernel types, so the bindings +# would not build with it. `make BINDGEN_LAYOUT_TESTS=1` keeps them, for +# checking a bindings generator; see +# Documentation/rust/bindgen-layout-checks.rst. +ifeq ($(BINDGEN_LAYOUT_TESTS),1) +bindgen_layout_flags := +else +bindgen_layout_flags := --no-layout-tests +endif + # `--rust-target` points to our minimum supported Rust version. -# Keep bindgen layout assertions enabled: a lost alignment can change field -# offsets without changing the containing type size. See Documentation/rust/ -# bindgen-layout-checks.rst for generator requirements. quiet_cmd_bindgen = BINDGEN $@ cmd_bindgen = \ $(BINDGEN) $< $(bindgen_target_flags) --rust-target 1.85 \ --use-core --with-derive-default --ctypes-prefix ffi \ + $(bindgen_layout_flags) \ --no-debug '.*' --enable-function-attribute-detection \ -o $@ -- $(bindgen_c_flags_final) -DMODULE \ $(bindgen_target_cflags) $(bindgen_target_extra) diff --git a/tools/aurora-bindgen/README.rst b/tools/aurora-bindgen/README.rst index 2c560aa13f96e4..75d4c3a2480e74 100644 --- a/tools/aurora-bindgen/README.rst +++ b/tools/aurora-bindgen/README.rst @@ -25,7 +25,10 @@ locked dependency graph is used, with at most two build jobs. ``--archive`` can supply an already downloaded source archive; its digest is still checked. The script builds and runs compile-only valid-input checks, then prints the exact ``BINDGEN=/absolute/path/to/bindgen`` make argument. It installs nothing globally. -Use that argument for the complete kernel Image and matching modules build. +The kernel build passes ``--no-layout-tests`` to bindgen unless +``BINDGEN_LAYOUT_TESTS=1`` is given, so add that variable to check layouts +with the private tool, and build the complete kernel Image and matching +modules with the same tool. Both scripts reject optimized Python (``-O``, ``-OO`` or ``PYTHONOPTIMIZE``) before creating output, since their validation assertions must remain enabled. @@ -67,14 +70,16 @@ generator, and its changes and both tool digests belong in the build record. Repository guard and supported tool versions -------------------------------------------- -The kernel keeps bindgen layout checks enabled. Its minimum bindgen is 0.71.1, -and its fixed ``--rust-target 1.85`` enables the ``offset_of`` feature (available -from Rust 1.77). In both official 0.71.1 and the pinned 0.73.2 generator, that -feature selects unconditional const size/alignment/offset assertions rather -than ``#[test]`` functions. A normal bindings crate compile therefore checks -layouts without requiring a Rust unit-test configuration. This is source-backed -for the accepted minimum and pinned tool; future generator changes must retain -that capability. Never remove the assertions to accept an incompatible tool. +By default the kernel passes ``--no-layout-tests``, as upstream, so stock +bindgen builds it; ``make BINDGEN_LAYOUT_TESTS=1`` keeps the layout checks +(see ``Documentation/rust/bindgen-layout-checks.rst``). Its minimum bindgen is +0.71.1, and its fixed ``--rust-target 1.85`` enables the ``offset_of`` feature +(available from Rust 1.77). In both official 0.71.1 and the pinned 0.73.2 +generator, that feature selects unconditional const size/alignment/offset +assertions rather than ``#[test]`` functions. A bindings crate compile with +``BINDGEN_LAYOUT_TESTS=1`` therefore checks layouts without requiring a Rust +unit-test configuration. This is source-backed for the accepted minimum and +pinned tool; future generator changes must retain that capability. Primary sources: @@ -82,5 +87,6 @@ Primary sources: * https://github.com/rust-lang/rust-bindgen/blob/v0.71.1/bindgen/codegen/mod.rs * https://github.com/rust-lang/rust-bindgen/tree/v0.73.2 -The complete kernel build remains the configuration-specific layout gate; these -small generator fixtures do not replace it or establish runtime GPU stability. +A complete kernel build with ``BINDGEN_LAYOUT_TESTS=1`` remains the +configuration-specific layout check; these small generator fixtures do not +replace it or establish runtime GPU stability. diff --git a/tools/aurora-bindgen/build.py b/tools/aurora-bindgen/build.py index 5895a84a7b5c44..7aa222cc028bbe 100644 --- a/tools/aurora-bindgen/build.py +++ b/tools/aurora-bindgen/build.py @@ -22,4 +22,4 @@ cmd=['cargo','build','--locked','--release','-p','bindgen-cli','-j'+str(a.jobs)] with (O/'build.log').open('w') as log:subprocess.run(cmd,cwd=S,stdout=log,stderr=subprocess.STDOUT,check=True) binary=S/'target/release/bindgen';subprocess.run([sys.executable,str(P/'test.py'),'--bindgen',str(binary),'--output-dir',str(O/'tests')],check=True) -r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' ...; keep full kernel layout assertions enabled.') +r={'status':'BUILT_AND_COMPILE_TESTED','binary':str(binary),'binary_sha256':sha(binary),'source_manifest_sha256':sha(P/'sources.json'),'build_command':cmd,'rustc_version':subprocess.check_output(['rustc','-vV'],text=True),'cargo_version':subprocess.check_output(['cargo','-vV'],text=True),'tests_manifest_sha256':sha(O/'tests/result.json'),'installed_globally':False};(O/'result.json').write_text(json.dumps(r,indent=2)+'\n');print(json.dumps(r,indent=2));print('Use make BINDGEN='+str(binary)+' BINDGEN_LAYOUT_TESTS=1 ... to build the kernel with layout assertions.') From b8c3c61294648b9263f208f7203a8bed8ba235e7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:34:54 -0700 Subject: [PATCH 589/631] usb: typec: tipd: put the debugfs command file behind a Kconfig option Commit 83e0357fcc61 ("usb: typec: tipd: accept Apple "DBMa" mode, add debugfs command access") creates usb/tipd//command whenever CONFIG_DEBUG_FS is set. Writing to it sends LOCK, DBMa, VDMs, DVEn and DISC to the USB-C PD controller, which can disconnect the port, put the controller into its debug mode or reboot a Mac connected to the port. The release configuration has DEBUG_FS=y, so every Mac with a CD321x or SN201202x controller would ship this interface, which .19 does not have. Add TYPEC_TPS6598X_DEBUG_COMMANDS, a bool that depends on the tipd core and DEBUG_FS and defaults to n, and create the command file only when it is set. The read-only regs file and the acceptance of a controller found in DBMa at probe are unchanged. Built W=1 with the release configuration (olddefconfig): with the option off, tps6598x-core.o has no command file operations; with it on, the command file builds as before. No new warnings either way. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/usb/typec/tipd/Kconfig | 16 ++++++++++++++++ drivers/usb/typec/tipd/debugfs.c | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/tipd/Kconfig b/drivers/usb/typec/tipd/Kconfig index a447a4622ea765..0422366342e8d6 100644 --- a/drivers/usb/typec/tipd/Kconfig +++ b/drivers/usb/typec/tipd/Kconfig @@ -28,6 +28,22 @@ config TYPEC_SN201202X If you choose to build this driver as a dynamically linked module, the module will be called sn201202x.ko. +config TYPEC_TPS6598X_DEBUG_COMMANDS + bool "Debugfs command access to TPS6598x/CD321x controllers" + depends on TYPEC_TPS6598X_CORE && DEBUG_FS + help + Add a root-only debugfs file, usb/tipd//command, that sends + a few controller commands (LOCK, DBMa, VDMs, DVEn and DISC) to the + USB Power Delivery controller. Tools such as tuxvdmtool use it to + send Apple VDMs (reboot, DFU, serial, Debug USB) to a Mac connected + to the port, or to enable the local end of a debug route. + + These commands can disconnect the port or reboot the connected + device. The read-only register file, usb/tipd//regs, is + available with DEBUG_FS whether or not this is set. + + Say N unless you use the controller's debug mode for development. + config TYPEC_TIPD_KUNIT_TEST tristate "KUnit tests for CD321x resume state" if !KUNIT_ALL_TESTS depends on KUNIT && TYPEC_TPS6598X_CORE diff --git a/drivers/usb/typec/tipd/debugfs.c b/drivers/usb/typec/tipd/debugfs.c index 8f35394624fe69..77252b3ed3c900 100644 --- a/drivers/usb/typec/tipd/debugfs.c +++ b/drivers/usb/typec/tipd/debugfs.c @@ -3,6 +3,7 @@ * TI TPS6598x / Apple CD321x / SN201202x debugfs interface * * /sys/kernel/debug/usb/tipd//command + * Only with CONFIG_TYPEC_TPS6598X_DEBUG_COMMANDS. * write: 4CC command followed by 0..64 bytes for DATA1. The command is * executed as a whole under the driver's lock. Only a small set * of commands used to talk to a connected Apple device (as @@ -269,8 +270,9 @@ void tipd_debugfs_register(struct tps6598x *tps) dbg->dentry = debugfs_create_dir(dev_name(tps->dev), tipd_debugfs_root); tps->debugfs = dbg; - debugfs_create_file("command", 0600, dbg->dentry, tps, - &tipd_debugfs_command_fops); + if (IS_ENABLED(CONFIG_TYPEC_TPS6598X_DEBUG_COMMANDS)) + debugfs_create_file("command", 0600, dbg->dentry, tps, + &tipd_debugfs_command_fops); debugfs_create_file("regs", 0400, dbg->dentry, tps, &tipd_debugfs_regs_fops); } From 52841d18939ab9cb79b0b12ae012fdca6c43a80f Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:41:06 -0700 Subject: [PATCH 590/631] vfio/type1: make the retained-fragment KUnit tests depend on TMPFS The mixed-granule retained-fragment tests, built into VFIO_TYPE1_ACCOUNT_KUNIT_TEST when MM_SUBPAGE is set, fill their source pages with kernel_write() to a shmem file. Without CONFIG_TMPFS shmem files have no write_iter, the write fails and 9 of the 10 vfio-type1-fragments cases fail with -EIO. Nothing in Kconfig asked for TMPFS, so a kunitconfig that enables the tests but not TMPFS builds and then fails at run time. Make VFIO_TYPE1_ACCOUNT_KUNIT_TEST depend on TMPFS when MM_SUBPAGE is set, and make the two arm64 harness options that select it depend on TMPFS, so the select cannot bypass the dependency. Configurations without MM_SUBPAGE (all non-user4k builds) are unchanged. KUnit, arm64, user4k-vfio kunitconfig with TMPFS=y: 12/12 pass (vfio-type1-account 2/2, vfio-type1-fragments 10/10). Without TMPFS, kunit.py now reports the test options as unsatisfiable instead of building a kernel whose tests fail. The non-user4k vfio kunitconfig still configures as before. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- arch/arm64/Kconfig.debug | 2 ++ drivers/vfio/Kconfig | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 52fced2ab8355d..9331b85d04aa17 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -82,6 +82,7 @@ config ARM64_USER4K_VFIO_ACCOUNT_TEST bool "VFIO type1 pinning harness for mixed userspace granules" depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y depends on IOMMUFD != m + depends on TMPFS select VFIO select VFIO_GROUP select VFIO_CONTAINER @@ -99,6 +100,7 @@ config ARM64_USER4K_VFIO_IOCTL_TEST depends on ARM64_USER4K_EXPERIMENTAL && KUNIT=y && DEBUG_KERNEL depends on RUNTIME_TESTING_MENU depends on IOMMUFD != m + depends on TMPFS select ARM64_USER4K_VFIO_ACCOUNT_TEST default n help diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index 33f6c7e9608d5b..92103812d8fcfb 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -57,6 +57,8 @@ config VFIO_IOMMU_TYPE1 config VFIO_TYPE1_ACCOUNT_KUNIT_TEST bool "Test VFIO type1 pin accounting and retained fragments" if !KUNIT_ALL_TESTS depends on KUNIT=y && VFIO_IOMMU_TYPE1=y + # The retained-fragment tests write their source pages to a shmem file + depends on TMPFS || !MM_SUBPAGE default KUNIT_ALL_TESTS help Exercise locked-memory refunds after source-mm exit and DMA owner From 0765e6acfd1ff2bd9250bfeb8dc9bee9f4742030 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:26:41 -0700 Subject: [PATCH 591/631] drm/apple: build the AFK service test only with the display driver afk_service_put() in afk.h now ends with a call to afk_service_retry() in afk.c, so the AFK service lifetime test needs appledrm. It was still built from drivers/gpu/drm/Makefile whenever DRM_APPLE_KUNIT_TEST was set, for running without the display driver, and the Kconfig entry allowed the test to be built in while the driver is a module. Two configurations that Kconfig accepts then fail to build: - DRM_APPLE=n with DRM_APPLE_KUNIT_TEST=y or m (for example KUNIT_ALL_TESTS on a kernel without the Apple display driver): afk_service_retry is undefined. - DRM_APPLE=m with DRM_APPLE_KUNIT_TEST=y: the built-in test refers to a function in appledrm.ko and vmlinux does not link. Build afk-service-test.o from drivers/gpu/drm/apple/Makefile next to the other tests that use the driver, and limit DRM_APPLE_KUNIT_TEST to m when DRM_APPLE is m. The modeset state test still builds without the driver, as before. Checked with arm64 kunit.py builds (DRM_APPLE / DRM_APPLE_KUNIT_TEST): before this change n/y and m/y fail to link and n/m fails modpost on afk_service_retry; after it n/y builds and its 9 tests pass, n/m, y/m and m/m build, and m/y is reduced to m/m by Kconfig. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/Makefile | 1 - drivers/gpu/drm/apple/Kconfig | 17 ++++++++++------- drivers/gpu/drm/apple/Makefile | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 846f2da7c81a08..da772e16b4a9f3 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -231,7 +231,6 @@ obj-$(CONFIG_DRM_APPLE_FABRIC_CORE) += apple/dcp-fabric-core.o obj-$(CONFIG_DRM_APPLE_FABRIC_KUNIT_TEST) += apple/dcp-fabric-test.o obj-$(CONFIG_DRM_APPLE) += apple/ obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/iomfb-state-test.o -obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += apple/afk-service-test.o obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ obj-$(CONFIG_DRM_MCDE) += mcde/ obj-$(CONFIG_DRM_TIDSS) += tidss/ diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 32173fef7127f3..c50d6ba4a5f8af 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -37,19 +37,22 @@ config DRM_APPLE_DEBUG config DRM_APPLE_KUNIT_TEST tristate "KUnit tests for Apple display controller" if !KUNIT_ALL_TESTS depends on KUNIT + depends on DRM_APPLE || !DRM_APPLE select DRM_KUNIT_TEST_HELPERS if DRM_APPLE default KUNIT_ALL_TESTS help Test modeset and hotplug state transitions without display hardware, including link changes that race with a modeset and recovery after failed modesets. These tests can run on other architectures and do - not require the Apple display driver to be enabled. Also test service - ownership and pending commands across disconnect and reannouncement. - With the Apple display driver enabled, also build tests for its AFK - ring parser (header strides used by older firmware and larger rings, - malformed ring size combinations), the H17P backlight policy - (takeover, DPMS, suspend, fades and present completion) and the - system resume of a CRTC whose mode is no longer admissible. + not require the Apple display driver to be enabled. + With the Apple display driver enabled, also build tests for AFK + service ownership and pending commands across disconnect and + reannouncement, for its AFK ring parser (header strides used by + older firmware and larger rings, malformed ring size combinations), + the H17P backlight policy (takeover, DPMS, suspend, fades and + present completion) and the system resume of a CRTC whose mode is + no longer admissible. These tests use the driver, so with the + driver built as a module they can only be built as modules too. config DRM_APPLE_DISPLAY_KUNIT_TEST bool "KUnit tests for Apple display mode probing" if !KUNIT_ALL_TESTS diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index dfd6fde66c387f..2f268b192655e6 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -34,6 +34,6 @@ appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += hotplug-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += external-hold-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += v14-board-test.o -# afk-service-test.o is built from drivers/gpu/drm/Makefile. +obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk-service-test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += afk_test.o obj-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_backlight_test.o From d0565a858891cbeb7a5db56fe8aa33ea3ec03052 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:26:41 -0700 Subject: [PATCH 592/631] docs: index the new Apple documents and fix their title markup Several documents added or changed in this series break the htmldocs build with warnings: - arch/arm64/apple-m6-cpu.rst, gpu/apple-dcp-backlight.rst and soc/apple/t8140-pmp-dvfs.rst are not in any toctree, so they are built but cannot be reached from the documentation index. There is no Documentation/soc/ section to hold the last one. - The title over- or underlines of those documents and of admin-guide/media/apple-ave.rst, arch/arm64/user4k.rst, arch/arm64/apple-j813.rst and PCI/controller/apple-t8140.rst are shorter than the titles. Add apple-m6-cpu to the arm64 index and apple-dcp-backlight to the GPU driver index. Move t8140-pmp-dvfs.rst to arch/arm64/apple-t8140-pmp-dvfs.rst, next to the other Apple platform documents, index it there and update MAINTAINERS. Lengthen the short title lines. The text of the documents is unchanged. Checked with Sphinx 9.1 (`make SPHINXDIRS="arch/arm64 gpu mm rust admin-guide PCI networking" htmldocs`): the warnings above are gone, and the only remaining ones are present on the base tree as well. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- Documentation/PCI/controller/apple-t8140.rst | 2 +- Documentation/admin-guide/media/apple-ave.rst | 2 +- Documentation/arch/arm64/apple-j813.rst | 2 +- Documentation/arch/arm64/apple-m6-cpu.rst | 4 ++-- .../arm64/apple-t8140-pmp-dvfs.rst} | 2 +- Documentation/arch/arm64/index.rst | 2 ++ Documentation/arch/arm64/user4k.rst | 8 ++++---- Documentation/gpu/apple-dcp-backlight.rst | 4 ++-- Documentation/gpu/drivers.rst | 1 + MAINTAINERS | 2 +- 10 files changed, 16 insertions(+), 13 deletions(-) rename Documentation/{soc/apple/t8140-pmp-dvfs.rst => arch/arm64/apple-t8140-pmp-dvfs.rst} (97%) diff --git a/Documentation/PCI/controller/apple-t8140.rst b/Documentation/PCI/controller/apple-t8140.rst index 496e9b94d40f60..7d342d1c1b4d1d 100644 --- a/Documentation/PCI/controller/apple-t8140.rst +++ b/Documentation/PCI/controller/apple-t8140.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only T8140/J700 experimental radio PCIe bootstrap -========================================== +============================================ The J700 path trains port 0, validates a bounded PIODMA bootstrap request, and enables native ECAM for the MediaTek Wi-Fi and Bluetooth functions. diff --git a/Documentation/admin-guide/media/apple-ave.rst b/Documentation/admin-guide/media/apple-ave.rst index 100b86e69b2d58..ee275bc6d455f6 100644 --- a/Documentation/admin-guide/media/apple-ave.rst +++ b/Documentation/admin-guide/media/apple-ave.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only Experimental T8140 video encoding -================================ +================================= The J700 AVE driver exposes a root-only debugfs firmware transport for hardware encoding experiments. It is not a V4L2 encoder, has no stable diff --git a/Documentation/arch/arm64/apple-j813.rst b/Documentation/arch/arm64/apple-j813.rst index ff02f1cb91ac94..2f132ea1a11dbc 100644 --- a/Documentation/arch/arm64/apple-j813.rst +++ b/Documentation/arch/arm64/apple-j813.rst @@ -179,7 +179,7 @@ separate checks below. The disabled Samsung UART node records its measured resources; the qualified console path is DockChannel. Temperature monitoring and built-in input -======================================== +========================================= Temperature monitoring enumerates the running SMC's readable, non-function ``T`` keys with four-byte floating-point values. It exposes their identifiers diff --git a/Documentation/arch/arm64/apple-m6-cpu.rst b/Documentation/arch/arm64/apple-m6-cpu.rst index 046872e83e09a7..20f08a57b2a167 100644 --- a/Documentation/arch/arm64/apple-m6-cpu.rst +++ b/Documentation/arch/arm64/apple-m6-cpu.rst @@ -1,5 +1,5 @@ M6 CPU bring-up -============== +=============== J873g has six E CPUs and a shared performance domain containing two P CPUs and four M CPUs. The T8152 ACC request uses a five-bit hardware state, with @@ -7,7 +7,7 @@ state 2 as the first ADT operating point. Do not use the older voltage or PLL initialization sequences. Failure cases and qualification ------------------------------- +------------------------------- Before testing, account for incorrect MPIDRs, incomplete spin-table release, broken cross-CPU interrupts, a stalled architectural timer, mismatched OPP diff --git a/Documentation/soc/apple/t8140-pmp-dvfs.rst b/Documentation/arch/arm64/apple-t8140-pmp-dvfs.rst similarity index 97% rename from Documentation/soc/apple/t8140-pmp-dvfs.rst rename to Documentation/arch/arm64/apple-t8140-pmp-dvfs.rst index 7ec4ae7366ed90..53067571c8e3aa 100644 --- a/Documentation/soc/apple/t8140-pmp-dvfs.rst +++ b/Documentation/arch/arm64/apple-t8140-pmp-dvfs.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only T8140 PMP DVFS diagnostics -========================= +========================== The existing PMP report provider exposes a read-only ``dvfs`` file under ``/sys/kernel/debug/apple-pmp-/`` on T8140. The provider must be diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index 23afe611f30398..9066f2a0815c86 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -10,6 +10,8 @@ ARM64 Architecture acpi_object_usage amu apple-j813 + apple-m6-cpu + apple-t8140-pmp-dvfs arm-acpi arm-cca asymmetric-32bit diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index d483c565d853b1..70e39f084e4172 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -1,8 +1,8 @@ .. SPDX-License-Identifier: GPL-2.0-only -=========================================== +============================================ Experimental per-process page sizes on arm64 -=========================================== +============================================ The mixed-page implementation lets an address space use a different user translation granule from the kernel's native physical page size. The kernel @@ -112,7 +112,7 @@ token through TLB invalidation and the required RCU grace period, including allocation-failure fallback paths. Translation context and invalidation -=================================== +==================================== The arm64 backend supplies per-mm page-table geometry. TTBR1 retains native kernel tables, while TTBR0 uses the selected user geometry. A granule change @@ -132,7 +132,7 @@ Independent TG0/TG1 fields do not make all other translation controls independent; the LPA2 exclusion must not be removed merely by changing TG0. Sharing, pinning and interface limits -==================================== +===================================== Mixed-granule sharing is permitted. Shared mappings refer to one coherent backing object, with permissions belonging to each virtual mapping. Shared diff --git a/Documentation/gpu/apple-dcp-backlight.rst b/Documentation/gpu/apple-dcp-backlight.rst index e486fcf4840df4..d9a87fc4ea58ef 100644 --- a/Documentation/gpu/apple-dcp-backlight.rst +++ b/Documentation/gpu/apple-dcp-backlight.rst @@ -1,7 +1,7 @@ .. SPDX-License-Identifier: GPL-2.0-only OR MIT Apple DCP H17P backlight integration -=================================== +==================================== The backlight policy implements the clean display specification's BACKLIGHT-1 through BACKLIGHT-4, with IOMFB-3 and LIFETIME-1 through LIFETIME-3 constraints. @@ -129,7 +129,7 @@ Hardware acceptance requires O-BACKLIGHT, O-BACKLIGHT-TAKEOVER (10 Hz PBwo with no jump above 0.05 W), O-BACKLIGHT-CRC, O-DPMS (twenty cycles), and O-SUSPEND. Measured activation and software replay --------------------------------------- +--------------------------------------- Linux-boundary measurements on the default J700 driver establish an 18-byte fragment at A408 offsets 0x354 through 0x365. Brightness-enabled presents use diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst index 2e13e0ad7e88bb..fe6870fc962549 100644 --- a/Documentation/gpu/drivers.rst +++ b/Documentation/gpu/drivers.rst @@ -6,6 +6,7 @@ GPU Driver Documentation :maxdepth: 3 amdgpu/index + apple-dcp-backlight i915 imagination/index mcde diff --git a/MAINTAINERS b/MAINTAINERS index dacf240284a9bf..fe20121dbe8940 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2001,7 +2001,7 @@ F: Documentation/devicetree/bindings/sound/apple,t8140-aop-audio.yaml F: Documentation/devicetree/bindings/usb/vialab,vl122.yaml F: Documentation/gpu/apple-dcp-backlight.rst F: Documentation/hid/apple-j700-multitouch.rst -F: Documentation/soc/apple/t8140-pmp-dvfs.rst +F: Documentation/arch/arm64/apple-t8140-pmp-dvfs.rst F: Documentation/networking/device_drivers/wifi/mt7932-neo.rst F: arch/arm64/boot/dts/apple/t8140* F: arch/arm64/configs/j700*.config From b1d8dfecce30887bb4fe71b418ad5b87627f5234 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:43 -0700 Subject: [PATCH 593/631] mm: keep the legacy bad-p4d handling for native-only geometry p4d_none_or_clear_bad_mm() reports and clears a bad top-level entry with p4d_ERROR() and p4d_clear(). The legacy helper it replaced in the page table walkers uses p4d_clear_bad(), which does nothing when the p4d level is folded, as on arm64 with four-level 16K tables. A corrupt top-level entry therefore went from being skipped to being logged and zeroed in mprotect, mremap, the page walker, swapoff and the zap, copy and free paths, even with the experimental user granule disabled. Use the per-mm accessors only when the architecture provides target-mm page table geometry, and fall back to the legacy behaviour otherwise. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- include/linux/pgtable.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 5c67d9b5d2f8df..2c5e581d610a8f 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -1572,6 +1572,7 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) static inline int p4d_none_or_clear_bad_mm(struct mm_struct *mm, p4d_t *p4d) { +#ifdef CONFIG_ARCH_HAS_MM_PGTABLE if (p4d_none_mm(mm, *p4d)) return 1; if (unlikely(p4d_bad_mm(mm, *p4d))) { @@ -1580,6 +1581,16 @@ static inline int p4d_none_or_clear_bad_mm(struct mm_struct *mm, p4d_t *p4d) return 1; } return 0; +#else + /* Native geometry only: identical to p4d_none_or_clear_bad(). */ + if (p4d_none(*p4d)) + return 1; + if (unlikely(p4d_bad(*p4d))) { + p4d_clear_bad(p4d); + return 1; + } + return 0; +#endif } static inline int p4d_none_or_clear_bad(p4d_t *p4d) From 5264e477f5d111d0e9ed9d54c67067c0c68663a2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:44 -0700 Subject: [PATCH 594/631] mm: restore native vm_insert_pages() and vm_map_pages() without user4k The mixed-granule rework routed vm_insert_pages(), vm_map_pages(), vm_map_pages_zero() and the map_kernel_pages helpers through a new byte-range inserter on every kernel. With ARM64_USER4K_EXPERIMENTAL off this still changed their behaviour: - vm_map_pages() now fails with -EINVAL on a VM_PFNMAP|VM_MIXEDMAP VMA and returns -EINVAL instead of -EFAULT for a range outside the VMA; - vm_insert_pages() and map_kernel_pages_prepare() reject an address that is not page aligned and add overflow checks that return new error codes; - insert_pages() returns -EOVERFLOW for large counts. These are the mmap paths of many drivers (dma-buf heaps, io_uring rings, media and GPU buffers). Keep the byte-range inserter and the new checks under CONFIG_MM_SUBPAGE, and build the previous implementations otherwise. vm_insert_pages_range() stays available as a new interface. With the option off, insert_pages(), vm_insert_pages(), vm_map_pages() and map_kernel_pages_prepare() compile to the .19 code apart from the equivalent phys-based PTE construction in insert_page_into_pte_locked(). Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/memory.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/mm/memory.c b/mm/memory.c index 8e58ea71b88c0f..6064016c0e1b80 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2834,6 +2834,7 @@ static int insert_page_range(struct vm_area_struct *vma, unsigned long addr, return ret; } +#ifdef CONFIG_MM_SUBPAGE static int insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num, pgprot_t prot) { @@ -2847,6 +2848,79 @@ static int insert_pages(struct vm_area_struct *vma, unsigned long addr, *num = DIV_ROUND_UP(length, PAGE_SIZE); return ret; } +#else +static int insert_page_in_batch_locked(struct vm_area_struct *vma, pte_t *pte, + unsigned long addr, struct page *page, pgprot_t prot) +{ + int err; + + err = validate_page_before_insert(vma, page); + if (err) + return err; + return insert_page_into_pte_locked(vma, pte, addr, page, prot, false, 0); +} + +/* insert_pages() amortizes the cost of spinlock operations + * when inserting pages in a loop. + */ +static int insert_pages(struct vm_area_struct *vma, unsigned long addr, + struct page **pages, unsigned long *num, pgprot_t prot) +{ + pmd_t *pmd = NULL; + pte_t *start_pte, *pte; + spinlock_t *pte_lock; + struct mm_struct *const mm = vma->vm_mm; + unsigned long curr_page_idx = 0; + unsigned long remaining_pages_total = *num; + unsigned long pages_to_write_in_pmd; + int ret; +more: + ret = -EFAULT; + pmd = walk_to_pmd(mm, addr); + if (!pmd) + goto out; + + pages_to_write_in_pmd = min_t(unsigned long, + remaining_pages_total, PTRS_PER_PTE - pte_index(addr)); + + /* Allocate the PTE if necessary; takes PMD lock once only. */ + ret = -ENOMEM; + if (pte_alloc(mm, pmd)) + goto out; + + while (pages_to_write_in_pmd) { + int pte_idx = 0; + const int batch_size = min_t(int, pages_to_write_in_pmd, 8); + + start_pte = pte_offset_map_lock(mm, pmd, addr, &pte_lock); + if (!start_pte) { + ret = -EFAULT; + goto out; + } + for (pte = start_pte; pte_idx < batch_size; ++pte, ++pte_idx) { + int err = insert_page_in_batch_locked(vma, pte, + addr, pages[curr_page_idx], prot); + if (unlikely(err)) { + pte_unmap_unlock(start_pte, pte_lock); + ret = err; + remaining_pages_total -= pte_idx; + goto out; + } + addr += PAGE_SIZE; + ++curr_page_idx; + } + pte_unmap_unlock(start_pte, pte_lock); + pages_to_write_in_pmd -= batch_size; + remaining_pages_total -= batch_size; + } + if (remaining_pages_total) + goto more; + ret = 0; +out: + *num = remaining_pages_total; + return ret; +} +#endif /** * vm_insert_pages_range - map a byte range from a native-page array @@ -2905,6 +2979,7 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num) { const unsigned long nr_pages = *num; +#ifdef CONFIG_MM_SUBPAGE unsigned long length, end; if (!IS_ALIGNED(addr, mm_page_size(vma->vm_mm)) || @@ -2912,6 +2987,12 @@ int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, check_add_overflow(addr, length, &end) || !range_in_vma(vma, addr, end)) return -EFAULT; +#else + const unsigned long end = addr + PAGE_SIZE * nr_pages; + + if (!range_in_vma(vma, addr, end)) + return -EFAULT; +#endif if (!(vma->vm_flags & VM_MIXEDMAP)) { VM_WARN_ON_ONCE(mmap_read_trylock(vma->vm_mm)); VM_WARN_ON_ONCE(vma->vm_flags & VM_PFNMAP); @@ -2935,11 +3016,17 @@ int map_kernel_pages_prepare(struct vm_area_desc *desc) } nr_pages = action->map_kernel.nr_pages; +#ifdef CONFIG_MM_SUBPAGE if (!IS_ALIGNED(addr, mm_page_size(desc->mm)) || check_mul_overflow(nr_pages, PAGE_SIZE, &end) || check_add_overflow(addr, end, &end) || !range_in_vma_desc(desc, addr, end)) return -EFAULT; +#else + end = addr + PAGE_SIZE * nr_pages; + if (!range_in_vma_desc(desc, addr, end)) + return -EFAULT; +#endif return 0; } @@ -3043,6 +3130,7 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num, unsigned long offset, unsigned int subpage_offset) { +#ifdef CONFIG_MM_SUBPAGE unsigned long bytes, available; if (offset >= num || @@ -3056,6 +3144,20 @@ static int __vm_map_pages(struct vm_area_struct *vma, struct page **pages, return -ENXIO; return vm_insert_pages_range(vma, vma->vm_start, pages, num, bytes, vma->vm_end - vma->vm_start); +#else + unsigned long count = vma_pages(vma); + unsigned long uaddr = vma->vm_start; + + /* Fail if the user requested offset is beyond the end of the object */ + if (offset >= num) + return -ENXIO; + + /* Fail if the user requested size exceeds available object size */ + if (count > num - offset) + return -ENXIO; + + return vm_insert_pages(vma, uaddr, pages + offset, &count); +#endif } /** From aeb594e25c1c5142fd60795a3ad1bede6a825fa7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:45 -0700 Subject: [PATCH 595/631] mm: keep native remap and I/O memory range checks without user4k The I/O mapping helpers were rewritten in byte units for small user granules, which also changed them on kernels without ARM64_USER4K_EXPERIMENTAL: - remap_pfn_range() and io_remap_pfn_range() reject sizes within a page of ULONG_MAX with -EINVAL before tracking the range; - vm_iomap_memory() and simple_ioremap_prepare() validate the physical range with different overflow and bounds arithmetic, so the accepted set of (start, length, vm_pgoff) inputs changed at the edges. vm_iomap_memory() is used by fbdev and many drivers' mmap handlers. Build the byte-based range validation only with CONFIG_MM_SUBPAGE and keep the previous pfn-based __simple_ioremap_prep(), simple_ioremap_prepare(), vm_iomap_memory() and remap range tracking otherwise. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/memory.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 6064016c0e1b80..aa0f8401ae4a28 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3655,10 +3655,14 @@ static int remap_pfn_range_track(struct vm_area_struct *vma, unsigned long addr, int err; size = mm_page_align(vma->vm_mm, size); +#ifdef CONFIG_MM_SUBPAGE if (check_add_overflow(size, (unsigned long)offset, &track_size) || track_size > ULONG_MAX - (PAGE_SIZE - 1)) return -EINVAL; track_size = PAGE_ALIGN(track_size); +#else + track_size = size; +#endif /* * If we cover the full VMA, we'll perform actual tracking, and @@ -3808,6 +3812,7 @@ int remap_pfn_range_complete(struct vm_area_struct *vma, return do_remap_pfn_range(vma, start, pfn, mmap_action_remap_offset(action), size, prot); } +#ifdef CONFIG_MM_SUBPAGE static int __simple_ioremap_prep(struct mm_struct *mm, unsigned long vm_len, struct vm_page_offset pos, phys_addr_t start_phys, unsigned long size, @@ -3844,18 +3849,52 @@ static int __simple_ioremap_prep(struct mm_struct *mm, unsigned long vm_len, return -EINVAL; return 0; } +#else +static int __simple_ioremap_prep(unsigned long vm_len, pgoff_t vm_pgoff, + phys_addr_t start_phys, unsigned long size, + unsigned long *pfnp) +{ + unsigned long pfn, pages; + + /* Check that the physical memory area passed in looks valid */ + if (start_phys + size < start_phys) + return -EINVAL; + /* + * You *really* shouldn't map things that aren't page-aligned, + * but we've historically allowed it because IO memory might + * just have smaller alignment. + */ + size += start_phys & ~PAGE_MASK; + pfn = start_phys >> PAGE_SHIFT; + pages = (size + ~PAGE_MASK) >> PAGE_SHIFT; + if (pfn + pages < pfn) + return -EINVAL; + + /* We start the mapping 'vm_pgoff' pages into the area */ + if (vm_pgoff > pages) + return -EINVAL; + pfn += vm_pgoff; + pages -= vm_pgoff; + + /* Can we fit all of the mapping? */ + if ((vm_len >> PAGE_SHIFT) > pages) + return -EINVAL; + + *pfnp = pfn; + return 0; +} +#endif int simple_ioremap_prepare(struct vm_area_desc *desc) { struct mmap_action *action = &desc->action; const phys_addr_t start = action->simple_ioremap.start_phys_addr; const unsigned long size = action->simple_ioremap.size; +#ifdef CONFIG_MM_SUBPAGE struct vm_page_offset physical, pos = { .index = desc->pgoff }; int err; -#ifdef CONFIG_MM_SUBPAGE pos.offset = desc->subpage_offset; -#endif err = __simple_ioremap_prep(desc->mm, vma_desc_size(desc), pos, start, size, &physical); if (err) @@ -3863,8 +3902,18 @@ int simple_ioremap_prepare(struct vm_area_desc *desc) /* The I/O remap logic does the heavy lifting. */ mmap_action_ioremap_full(desc, physical.index); -#ifdef CONFIG_MM_SUBPAGE action->remap.offset = physical.offset; +#else + unsigned long pfn; + int err; + + err = __simple_ioremap_prep(vma_desc_size(desc), desc->pgoff, + start, size, &pfn); + if (err) + return err; + + /* The I/O remap logic does the heavy lifting. */ + mmap_action_ioremap_full(desc, pfn); #endif return io_remap_pfn_range_prepare(desc); } @@ -3889,6 +3938,7 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long const unsigned long vm_start = vma->vm_start; const unsigned long vm_end = vma->vm_end; const unsigned long vm_len = vm_end - vm_start; +#ifdef CONFIG_MM_SUBPAGE struct vm_page_offset physical; int err; @@ -3900,6 +3950,17 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long /* Ok, let it rip */ return io_remap_pfn_range_offset(vma, vm_start, physical.index, physical.offset, vm_len, vma->vm_page_prot); +#else + unsigned long pfn; + int err; + + err = __simple_ioremap_prep(vm_len, vma->vm_pgoff, start, len, &pfn); + if (err) + return err; + + /* Ok, let it rip */ + return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); +#endif } EXPORT_SYMBOL(vm_iomap_memory); From 996067c67d6f09b666f5b6621af0e91219b98c27 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:46 -0700 Subject: [PATCH 596/631] mm: keep generic_access_phys() lengths unchanged without user4k generic_access_phys() now returns 0 for a non-positive length, rejects an address outside the VMA and clips each access to the end of the current user leaf. On kernels without ARM64_USER4K_EXPERIMENTAL the clip makes ptrace and /proc/PID/mem accesses to a VM_IO/VM_PFNMAP mapping map and copy one page-table leaf at a time, where the .19 kernel mapped the whole requested range with a single ioremap(). Apply the new checks and the clip only with CONFIG_MM_SUBPAGE. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/memory.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/memory.c b/mm/memory.c index aa0f8401ae4a28..270b959502fa87 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -8991,6 +8991,7 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, .allow_subpage = true }; +#ifdef CONFIG_MM_SUBPAGE if (len <= 0) return 0; if (addr < vma->vm_start || addr >= vma->vm_end) @@ -8999,6 +9000,7 @@ int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, len = min_t(unsigned long, len, vma->vm_end - addr); len = min_t(unsigned long, len, mm_page_size(vma->vm_mm) - (addr & ~mm_page_mask(vma->vm_mm))); +#endif retry: if (follow_pfnmap_start(&args)) return -EINVAL; From 78404cee8c7659c6cf6370d5f32db1a38e4874a5 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:46 -0700 Subject: [PATCH 597/631] mm: skip zero-count folio reference adds on COW copies copy_present_page() and wp_page_copy() add mm_pte_native_pages() - 1 extra references to the new folio, so that a user leaf larger than a native page holds one reference per native page. For native and smaller user granules that count is zero, and every anonymous COW copy paid for an atomic add of zero on the folio refcount, including on kernels without ARM64_USER4K_EXPERIMENTAL. Only add the references when a leaf covers more than one native page. No functional change. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/memory.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index 270b959502fa87..ae28b2d3454d99 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1098,7 +1098,9 @@ copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma } *prealloc = NULL; - folio_ref_add(new_folio, mm_pte_native_pages(dst_vma->vm_mm) - 1); + /* Only a larger user granule maps more than one native page. */ + if (mm_pte_native_pages(dst_vma->vm_mm) > 1) + folio_ref_add(new_folio, mm_pte_native_pages(dst_vma->vm_mm) - 1); __folio_mark_uptodate(new_folio); folio_add_new_anon_rmap(new_folio, dst_vma, addr, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, dst_vma); @@ -4610,7 +4612,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) * some TLBs while the old PTE remains in others. */ ptep_clear_flush(vma, vmf->address, vmf->pte); - folio_ref_add(new_folio, mm_pte_native_pages(mm) - 1); + if (mm_pte_native_pages(mm) > 1) + folio_ref_add(new_folio, mm_pte_native_pages(mm) - 1); folio_add_new_anon_rmap(new_folio, vma, vmf->address, RMAP_EXCLUSIVE); folio_add_lru_vma(new_folio, vma); BUG_ON(unshare && pte_write(entry)); From 89337e112c8b333ea7fb7fe84de93586fa50f23b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:47 -0700 Subject: [PATCH 598/631] mm: keep the native locked_vm accounting when user4k is off __account_locked_vm() gained overflow checks and a new error return for an unbalanced decrement: an over-large decrement now warns and returns -EINVAL without touching mm->locked_vm, and increments that would wrap now fail with -ENOMEM. These only matter for the mixed granule accounting units, but they also changed the behaviour of the exported helper on every kernel built without the feature. Build the reworked helper only with CONFIG_MM_SUBPAGE and keep the previous implementation otherwise, so kernels without the experimental user granule keep the existing locked_vm accounting exactly. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/util.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/mm/util.c b/mm/util.c index 2adb3d08660d68..b37a3e8bc63109 100644 --- a/mm/util.c +++ b/mm/util.c @@ -516,6 +516,7 @@ EXPORT_SYMBOL_IF_KUNIT(arch_pick_mmap_layout); * * 0 on success * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded. */ +#ifdef CONFIG_MM_SUBPAGE int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, const struct task_struct *task, bool bypass_rlim) { @@ -553,6 +554,37 @@ int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, return ret; } +#else +int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc, + const struct task_struct *task, bool bypass_rlim) +{ + unsigned long locked_vm, limit; + int ret = 0; + + mmap_assert_write_locked(mm); + + locked_vm = mm->locked_vm; + if (inc) { + if (!bypass_rlim) { + limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT; + if (locked_vm + pages > limit) + ret = -ENOMEM; + } + if (!ret) + mm->locked_vm = locked_vm + pages; + } else { + WARN_ON_ONCE(pages > locked_vm); + mm->locked_vm = locked_vm - pages; + } + + pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid, + (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT, + locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK), + ret ? " - exceeded" : ""); + + return ret; +} +#endif EXPORT_SYMBOL_GPL(__account_locked_vm); /** From 7607f149a410f426d115d4cd9b0a5531854bf2b4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:48 -0700 Subject: [PATCH 599/631] mm: release exit_mmap commitment after the mmap lock again exit_mmap() now returns the address space's committed memory before dropping the mmap lock, because the per-mm commitment used by the experimental user granule is protected by that lock. Without the feature there is no per-mm state to protect; keep the previous order of unlocking first and then returning the global commitment. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/mmap.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mm/mmap.c b/mm/mmap.c index 097161e3e61efd..f82ed9c9204f20 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1371,8 +1371,14 @@ void exit_mmap(struct mm_struct *mm) destroy: __mt_destroy(&mm->mm_mt); trace_exit_mmap(mm); +#ifdef CONFIG_MM_SUBPAGE + /* The per-mm commitment is updated under the mmap lock. */ mm_unacct_memory(mm, nr_accounted); mmap_write_unlock(mm); +#else + mmap_write_unlock(mm); + vm_unacct_memory(nr_accounted); +#endif } /* From 01996cd22edad30486432d12d07d8aeb2e0fcad3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:48 -0700 Subject: [PATCH 600/631] mm: keep the single-page rmap walk bound for native geometry vma_address_end() now aligns the walk address to the mapping granule and clamps the end of a single-page walk to the VMA end. For native mappings the address is already page aligned and inside the VMA, so the result is unchanged, but the extra work is compiled into every rmap walk. Keep the previous single-page bound when the experimental user granule is not built. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/internal.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm/internal.h b/mm/internal.h index bd953128ed6f1e..c9b44ed39a810a 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1323,9 +1323,14 @@ static inline unsigned long vma_address_end(struct page_vma_mapped_walk *pvmw) #endif /* Common case, plus ->pgoff is invalid for KSM */ +#ifdef CONFIG_MM_SUBPAGE if (pvmw->nr_pages == 1 && mm_page_size(vma->vm_mm) >= PAGE_SIZE) return min((pvmw->address & mm_page_mask(vma->vm_mm)) + mm_page_size(vma->vm_mm), vma->vm_end); +#else + if (pvmw->nr_pages == 1) + return pvmw->address + PAGE_SIZE; +#endif pgoff = pvmw->pgoff + pvmw->nr_pages; address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT); From ccd751939c152daa753e8c05c83dd2bd6d3e4d26 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:49 -0700 Subject: [PATCH 601/631] shmem: do not evaluate shmem_uffd_forget() arguments when unused Without the subpage userfaultfd tracking, shmem_uffd_forget() is an empty inline, but its folio_nr_pages() argument is still evaluated and leaves a dead read of the folio flags in the page cache deletion paths. Make the stub a statement macro that does not evaluate its arguments, so those paths compile exactly as before. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- include/linux/shmem_fs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index bf3a0129547001..b9376f8c430e1d 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -165,8 +165,8 @@ static inline bool shmem_uffd_range_missing(struct folio *folio, pgoff_t index, unsigned int offset, unsigned long length) { return false; } static inline void shmem_uffd_mark_mapped(struct folio *folio, unsigned long offset, unsigned long length) { } -static inline void shmem_uffd_forget(struct address_space *mapping, - pgoff_t start, unsigned long nr) { } +/* Do not evaluate the arguments: folio_nr_pages() would still read flags. */ +#define shmem_uffd_forget(mapping, start, nr) do { } while (0) #endif void shmem_unlock_mapping(struct address_space *mapping); From 5812ea2970fd22b805e943803ef2a53217f7e8b7 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:50 -0700 Subject: [PATCH 602/631] mm/shmem: skip i_pages lock round trip without uffd subpage data When folio_alloc_swap() fails, shmem_writeout() now takes and drops the mapping's i_pages lock with interrupts disabled to forget userfaultfd subpage metadata. Without CONFIG_MM_SUBPAGE and CONFIG_USERFAULTFD that metadata does not exist and the call is an empty stub, so the lock round trip is pure overhead on the swap-full path. Compile the block only when the metadata exists. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/shmem.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/shmem.c b/mm/shmem.c index 705765dddf1d22..e1270b7a16e4ea 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2045,10 +2045,12 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, swap_cache_del_folio(folio); goto redirty; } +#if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) /* Allocation failed: no swap entry owns the reserved inode metadata. */ xa_lock_irq(&mapping->i_pages); shmem_uffd_forget(mapping, folio->index, folio_nr_pages(folio)); xa_unlock_irq(&mapping->i_pages); +#endif if (nr_pages > 1) goto try_split; redirty: From 058eb531de3984b2b694f069fdae2496329c55f3 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:50 -0700 Subject: [PATCH 603/631] mm/swap: keep native swap I/O unchanged without subpage support Every swapon of a swap file on a block-backed filesystem now set up a global bio_set with a rescuer workqueue that is never freed, and could fail swapon with -ENOMEM, even though the reserve is only used to split the I/O of large base-page folios, which only exist with subpage support. Swap reads and writes also went through bio_await() with a private submit callback instead of submit_bio()/submit_bio_wait(), and the per-CPU allocation fast path gained a check that cannot fail for native folios. Build the extent-splitting reserve and submit helper only with CONFIG_MM_SUBPAGE, use plain submit_bio()/submit_bio_wait() otherwise, and skip the fast-path check and the reserve setup when the option is off. With CONFIG_MM_SUBPAGE=y the code is unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/page_io.c | 22 ++++++++++++++++++++-- mm/swapfile.c | 6 ++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mm/page_io.c b/mm/page_io.c index 290706f931bdcd..a6a6e7ee21bab3 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -412,6 +412,7 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug) *swap_plug = sio; } +#ifdef CONFIG_MM_SUBPAGE static DEFINE_MUTEX(swap_extent_io_mutex); static struct bio_set swap_extent_bioset; @@ -464,6 +465,23 @@ static void swap_submit_bio(struct bio *bio, void *private) submit_bio(bio); } +static void swap_submit_bio_wait(struct bio *bio, struct folio *folio) +{ + bio_await(bio, folio, swap_submit_bio); +} +#else +/* Without subpage support, large folios never reach a swap file. */ +static inline void swap_submit_bio(struct bio *bio, void *private) +{ + submit_bio(bio); +} + +static inline void swap_submit_bio_wait(struct bio *bio, struct folio *folio) +{ + submit_bio_wait(bio); +} +#endif + static void swap_writepage_bdev_sync(struct folio *folio, struct swap_info_struct *sis) { @@ -480,7 +498,7 @@ static void swap_writepage_bdev_sync(struct folio *folio, folio_start_writeback(folio); folio_unlock(folio); - bio_await(&bio, folio, swap_submit_bio); + swap_submit_bio_wait(&bio, folio); __end_swap_bio_write(&bio); } @@ -643,7 +661,7 @@ static void swap_read_folio_bdev_sync(struct folio *folio, count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN); count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio)); count_vm_events(PSWPIN, folio_nr_pages(folio)); - bio_await(&bio, folio, swap_submit_bio); + swap_submit_bio_wait(&bio, folio); __end_swap_bio_read(&bio); put_task_struct(current); } diff --git a/mm/swapfile.c b/mm/swapfile.c index ca11d95c2d0371..3f863dd41bf62d 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1387,7 +1387,8 @@ static bool swap_alloc_fast(struct folio *folio, bool mandatory) offset = this_cpu_read(percpu_swap_cluster.offset[order]); if (!si || !offset || !get_swap_device_info(si)) return false; - if (!swap_folio_alloc_allowed(si, folio, mandatory)) { + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && + !swap_folio_alloc_allowed(si, folio, mandatory)) { put_swap_device(si); return false; } @@ -3119,7 +3120,8 @@ static int setup_swap_extents(struct swap_info_struct *sis, } else { ret = generic_swapfile_activate(sis, swap_file, span); } - if (ret >= 0 && sis->bdev && + /* Only subpage swap needs the reserve for splitting large folio I/O. */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && ret >= 0 && sis->bdev && !(sis->flags & SWP_FS_OPS) && swap_extent_io_init()) { destroy_swap_extents(sis, swap_file); return -ENOMEM; From 0b87330a6e1e1fcbcc40b6786e05956fec1e8c30 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:51 -0700 Subject: [PATCH 604/631] mm/vmscan: gate MGLRU legacy-memcg writeback change on subpage The mixed-granule series also changed MGLRU reclaim for cgroup v1 memcgs: folios under writeback are no longer rotated to a younger generation and keep PG_reclaim when isolated. This alters reclaim behaviour for every kernel with CONFIG_LRU_GEN, independent of the mixed-granule option, and it is not an upstream change. Apply it only with CONFIG_MM_SUBPAGE so kernels without the option keep the existing MGLRU behaviour. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/vmscan.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index fe28f70fd51b17..c09b30a5a96065 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4682,7 +4682,8 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c * before I/O completes. Let the common path apply its GFP and mapping * deadlock checks instead of hiding those folios from it. */ - if (writeback && !writeback_throttling_sane(sc)) + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && writeback && + !writeback_throttling_sane(sc)) return false; /* waiting for writeback */ @@ -4719,8 +4720,12 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca if (!folio_test_referenced(folio)) set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); - /* Preserve legacy memcg's repeated-writeback marker for shrink_folio_list(). */ - if (writeback_throttling_sane(sc) || !folio_test_writeback(folio)) + /* + * For shrink_folio_list(). Subpage kernels keep legacy memcg's + * repeated-writeback marker. + */ + if (!IS_ENABLED(CONFIG_MM_SUBPAGE) || writeback_throttling_sane(sc) || + !folio_test_writeback(folio)) folio_clear_reclaim(folio); success = lru_gen_del_folio(lruvec, folio, true); From d2883a8fee2b670934471db3cb64f42c0403f450 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:52 -0700 Subject: [PATCH 605/631] mm/pagewalk: keep native walk_page_mapping() clip without subpage walk_page_mapping() gained an early return for nr == 0, an -EINVAL return when first_index + nr overflows, and a new byte-exact clipping calculation. For native VMAs the clip is equivalent, but the new early returns change the result for degenerate requests. Move the clip into walk_mapping_clip() with the byte-exact version for CONFIG_MM_SUBPAGE and the previous index-based version otherwise, and only apply the new argument checks with the option on. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/pagewalk.c | 50 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 54bc7ac30befa5..daa40cf248a775 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -771,6 +771,36 @@ int walk_page_vma(struct vm_area_struct *vma, const struct mm_walk_ops *ops, return __walk_page_range(vma->vm_start, vma->vm_end, &walk); } +#ifdef CONFIG_MM_SUBPAGE +/* Clip native file indices to the VMA's exact byte interval. */ +static bool walk_mapping_clip(struct vm_area_struct *vma, pgoff_t first_index, + pgoff_t last_index, unsigned long *start_addr, + unsigned long *end_addr) +{ + *start_addr = first_index <= vma->vm_pgoff ? vma->vm_start : + vma_address_at_offset(vma, (struct vm_page_offset) { first_index, 0 }); + *end_addr = last_index >= vma_last_pgoff(vma) ? vma->vm_end : + vma_address_at_offset(vma, (struct vm_page_offset) { last_index + 1, 0 }); + return !IS_ERR_VALUE(*start_addr) && !IS_ERR_VALUE(*end_addr) && + *start_addr < *end_addr; +} +#else +/* Clip to the vma */ +static bool walk_mapping_clip(struct vm_area_struct *vma, pgoff_t first_index, + pgoff_t last_index, unsigned long *start_addr, + unsigned long *end_addr) +{ + pgoff_t vba = vma->vm_pgoff; + pgoff_t vea = vba + vma_pages(vma); + pgoff_t cba = max(first_index, vba); + pgoff_t cea = min(last_index + 1, vea); + + *start_addr = ((cba - vba) << PAGE_SHIFT) + vma->vm_start; + *end_addr = ((cea - vba) << PAGE_SHIFT) + vma->vm_start; + return *start_addr < *end_addr; +} +#endif + /** * walk_page_mapping - walk all memory areas mapped into a struct address_space. * @mapping: Pointer to the struct address_space @@ -816,20 +846,20 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, if (!check_ops_safe(ops)) return -EINVAL; - if (!nr) - return 0; - if (check_add_overflow(first_index, nr - 1, &last_index)) - return -EINVAL; + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) { + if (!nr) + return 0; + if (check_add_overflow(first_index, nr - 1, &last_index)) + return -EINVAL; + } else { + last_index = first_index + nr - 1; + } lockdep_assert_held(&mapping->i_mmap_rwsem); vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, last_index) { - /* Clip native file indices to the VMA's exact byte interval. */ - start_addr = first_index <= vma->vm_pgoff ? vma->vm_start : - vma_address_at_offset(vma, (struct vm_page_offset) { first_index, 0 }); - end_addr = last_index >= vma_last_pgoff(vma) ? vma->vm_end : - vma_address_at_offset(vma, (struct vm_page_offset) { last_index + 1, 0 }); - if (IS_ERR_VALUE(start_addr) || IS_ERR_VALUE(end_addr) || start_addr >= end_addr) + if (!walk_mapping_clip(vma, first_index, last_index, + &start_addr, &end_addr)) continue; walk.vma = vma; From 2406596fa2c843abddb04418676b19b278caaaa2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:52 -0700 Subject: [PATCH 606/631] mm/process_vm_access: keep allocation order without subpage process_vm_rw_core() now allocates the page array after looking up the task and its mm, so that subpage mms, which do not use the array, skip the allocation. Without subpage support this only changed which error wins when the allocation fails (-ENOMEM versus -ESRCH/-EPERM) and moved the allocation under mm_access(). Allocate before the lookup, as before, when CONFIG_MM_SUBPAGE is off. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/process_vm_access.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c index f178b5fdb65890..48af2c9c5a4306 100644 --- a/mm/process_vm_access.c +++ b/mm/process_vm_access.c @@ -242,6 +242,18 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, if (nr_pages == 0) return 0; + /* Subpage kernels allocate only after choosing the page-array path. */ + if (!IS_ENABLED(CONFIG_MM_SUBPAGE) && nr_pages > PVM_MAX_PP_ARRAY_COUNT) { + /* For reliability don't try to kmalloc more than + 2 pages worth */ + process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, + sizeof(struct page *)*nr_pages), + GFP_KERNEL); + + if (!process_pages) + return -ENOMEM; + } + /* Get process information */ task = find_get_task_by_vpid(pid); if (!task) { @@ -269,7 +281,7 @@ static ssize_t process_vm_rw_core(pid_t pid, struct iov_iter *iter, goto copied; } #endif - if (nr_pages > PVM_MAX_PP_ARRAY_COUNT) { + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && nr_pages > PVM_MAX_PP_ARRAY_COUNT) { /* For reliability don't try to kmalloc more than 2 pages worth */ process_pages = kmalloc(min_t(size_t, PVM_MAX_KMALLOC_PAGES * PAGE_SIZE, From 1c76bc45a6a80d578118746495e0d76b898650bd Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:53 -0700 Subject: [PATCH 607/631] net: keep MSG_ZEROCOPY realloc semantics without user4k msg_zerocopy_realloc() now allocates a fresh completion record for a stream socket whose tail skb carries another kind of ubuf_info, instead of returning NULL. This is needed when a retained-subpage splice owner sits on the skb, but it also changed kernels without ARM64_USER4K_EXPERIMENTAL: a MSG_ZEROCOPY send on a TCP socket whose last skb belongs to another zerocopy user, such as io_uring send zerocopy, used to fail with ENOBUFS and now succeeds on a new segment. Only take the new path with CONFIG_MM_SUBPAGE, which is the only configuration that creates subpage splice owners. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- net/core/skbuff.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 090a26ed9f2db2..8c97fcd2d0e7ad 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -1768,7 +1768,9 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, /* there might be non MSG_ZEROCOPY users */ if (uarg->ops != &msg_zerocopy_ubuf_ops) { - if (sk->sk_type == SOCK_STREAM) + /* A retained-subpage owner may sit on a stream skb. */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && + sk->sk_type == SOCK_STREAM) goto new_alloc; return NULL; } From 9a16d02736b70119652a7b273bbee7b3be614c64 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:54 -0700 Subject: [PATCH 608/631] prctl: build the page-size requests only with selectable granules The four PR_AURORA_{GET,SET}_{EXEC,DEFAULT}_PAGE_SIZE prctl cases were compiled into every kernel. On a kernel without CONFIG_ARCH_HAS_USER_PAGE_SIZE they returned -EINVAL directly, which skipped the unknown-option path that every other unsupported prctl takes, so the task_prctl_unknown tracepoint no longer fired for those option numbers. Compile the cases and their helpers only when the architecture supports selecting the user page size. Kernels without it handle the option numbers exactly as before: the default case emits the tracepoint and returns -EINVAL. Kernels with the feature are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- kernel/sys.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/kernel/sys.c b/kernel/sys.c index 09702e8f773ba0..f41014735ade45 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2532,22 +2532,18 @@ static int prctl_set_thp_disable(bool thp_disable, unsigned long flags, return 0; } +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE static int prctl_page_size_get(struct task_struct *task, bool persistent) { -#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE unsigned int shift = persistent ? task->default_exec_page_shift : task->exec_page_shift; return shift ? 1U << shift : 0; -#else - return -EINVAL; -#endif } static int prctl_page_size_set(struct task_struct *task, unsigned long size, bool persistent) { -#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE int shift = arch_exec_page_size_shift(size); if (shift < 0) @@ -2561,10 +2557,8 @@ static int prctl_page_size_set(struct task_struct *task, unsigned long size, task->exec_page_shift = shift; } return 0; -#else - return -EINVAL; -#endif } +#endif int __weak arch_prctl_mem_model_get(struct task_struct *t) { @@ -2589,6 +2583,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, error = 0; switch (option) { +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE case PR_AURORA_GET_EXEC_PAGE_SIZE: if (arg2 || arg3 || arg4 || arg5) return -EINVAL; @@ -2605,6 +2600,7 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, if (arg3 || arg4 || arg5) return -EINVAL; return prctl_page_size_set(me, arg2, true); +#endif case PR_GET_MEM_MODEL: if (arg2 || arg3 || arg4 || arg5) return -EINVAL; From ecc0c29cc3d0d5a177b92be3decf5c90652b5868 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:54 -0700 Subject: [PATCH 609/631] ocfs2: keep the native truncate unmap sequence without user4k ocfs2_truncate_file() was switched to truncate_pagecache() so that the mixed-granule tail invalidation runs for small userspace leaves. truncate_pagecache() also repeats unmap_mapping_range() after the pagecache truncation, so kernels built without ARM64_USER4K_EXPERIMENTAL got an extra unmap pass that the .19 kernel and upstream ocfs2 do not perform. Use truncate_pagecache() only when CONFIG_MM_SUBPAGE is enabled and keep the original unmap_mapping_range() + truncate_inode_pages() sequence otherwise, so ocfs2 truncation is unchanged in production builds. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- fs/ocfs2/file.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 21153bd53f66ea..846fbb50c36794 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -504,7 +504,12 @@ int ocfs2_truncate_file(struct inode *inode, goto bail_unlock_sem; } - truncate_pagecache(inode, new_i_size); + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) { + truncate_pagecache(inode, new_i_size); + } else { + unmap_mapping_range(inode->i_mapping, new_i_size + PAGE_SIZE - 1, 0, 1); + truncate_inode_pages(inode->i_mapping, new_i_size); + } status = ocfs2_commit_truncate(osb, inode, di_bh); if (status < 0) { From 71a6bc1bec57b14bebe076aad5fb769e023ebccd Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:55 -0700 Subject: [PATCH 610/631] block: size user passthrough bios as before without user4k bio_map_user_iov() now sizes its bio with bio_iov_vecs_to_alloc(), which returns 0 for bvec-backed iterators and only differs from iov_iter_npages() so that user4k can count user fragments. Kernels built without ARM64_USER4K_EXPERIMENTAL therefore allocated passthrough bios differently from .19 for bvec iterators (for example io_uring fixed buffers on NVMe passthrough). Use bio_iov_vecs_to_alloc() only with CONFIG_MM_SUBPAGE and keep iov_iter_npages() otherwise. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- block/blk-map.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/block/blk-map.c b/block/blk-map.c index b98eed276aea25..dbbf84f44a2854 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -260,7 +260,9 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { - unsigned int nr_vecs = bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS); + unsigned int nr_vecs = IS_ENABLED(CONFIG_MM_SUBPAGE) ? + bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS) : + iov_iter_npages(iter, BIO_MAX_VECS); struct bio *bio; int ret; From c5a6961ce15a57575a07b0170a5c892c08119cab Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:56 -0700 Subject: [PATCH 611/631] io_uring/rsrc: limit the fragment-owner account skip to user4k headpage_already_acct() was changed to skip registered buffers whose release callback is not io_release_ubuf(), so that fragment-backed buffers with their own charge lifetime are not matched. Without ARM64_USER4K_EXPERIMENTAL this also skipped kernel-registered bvec buffers, changing how a compound head page already present in such a buffer affects locked-memory accounting compared with .19. Apply the skip only when CONFIG_MM_SUBPAGE is enabled. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- io_uring/rsrc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index b442d5cf6c7f5d..0e0550f151ec8d 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -645,7 +645,7 @@ static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages, continue; imu = node->buf; /* Other backing owners have independent charge lifetimes. */ - if (imu->release != io_release_ubuf) + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && imu->release != io_release_ubuf) continue; for (j = 0; j < imu->nr_bvecs; j++) { if (!PageCompound(imu->bvec[j].bv_page)) From a07e98c6181d4fe90de13812fc5e9f1c531fdf30 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:57 -0700 Subject: [PATCH 612/631] drm/shmem-helper: keep native fault and mmap paths without user4k The mixed-granule work changed the shmem GEM helper in two places that also run on kernels built without ARM64_USER4K_EXPERIMENTAL: - single-page faults go through vmf_insert_pfn_prot_mkwrite_offset() with a run-time computed sub-page offset instead of vmf_insert_pfn_mkwrite(); - drm_gem_shmem_mmap() gained an extra byte-range check that can return -EINVAL for callers whose vm_pgoff does not carry the GEM fake offset. These are the CPU mapping paths of every shmem-backed GPU buffer. Build both only with CONFIG_MM_SUBPAGE and keep the .19 code otherwise, so the production fault and mmap paths are unchanged. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/gpu/drm/drm_gem_shmem_helper.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c index eab3abedcc8741..f72952aa996cd0 100644 --- a/drivers/gpu/drm/drm_gem_shmem_helper.c +++ b/drivers/gpu/drm/drm_gem_shmem_helper.c @@ -763,10 +763,14 @@ static vm_fault_t try_insert_pfn(struct vm_fault *vmf, unsigned int order, vm_fault_t ret = VM_FAULT_FALLBACK; if (!order) { +#ifdef CONFIG_MM_SUBPAGE struct vm_page_offset pos = vma_page_offset_at(vmf->vma, vmf->address); ret = vmf_insert_pfn_prot_mkwrite_offset(vmf->vma, vmf->address, pfn, pos.offset, vmf->vma->vm_page_prot, write); +#else + ret = vmf_insert_pfn_mkwrite(vmf->vma, vmf->address, pfn, write); +#endif #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP } else if (order == PMD_ORDER) { unsigned long paddr = pfn << PAGE_SHIFT; @@ -959,6 +963,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct if (is_cow_mapping(vma->vm_flags)) return -EINVAL; +#ifdef CONFIG_MM_SUBPAGE /* Validate bytes, including a partial native-page offset from PRIME. */ { pgoff_t index = drm_gem_shmem_object_offset(obj, vma->vm_pgoff); @@ -969,6 +974,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct offset > obj->size || length > obj->size - offset) return -EINVAL; } +#endif /* Revocation zaps the device anon mapping only. */ if (shmem->reclaimable_cpu_mappings && From 55a5ef43af232e5ee879de616bcde38c301ee228 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:58 -0700 Subject: [PATCH 613/631] userfaultfd: keep the UFFDIO_COPY source check without user4k UFFDIO_COPY now validates the source range against current->mm instead of the registered mm, which matters only when the pager and target can use different user granules. On kernels built without ARM64_USER4K_EXPERIMENTAL it changed which task size bounds the source range when the uffd is driven from another process (for example a compat monitor). Validate the source against current->mm only with CONFIG_MM_SUBPAGE and keep the original validate_unaligned_range(ctx->mm, ...) otherwise. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- fs/userfaultfd.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 3174e3b6edb414..64b32cce273750 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1663,9 +1663,15 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx, /* * COPY reads the pager's address space, which can have a different * task size and granule from the registered target. Only the target - * range below requires a page-aligned length. + * range below requires a page-aligned length. Without mixed granules, + * keep validating the source against the registered mm. */ - ret = validate_byte_range(current->mm, uffdio_copy.src, uffdio_copy.len); + if (IS_ENABLED(CONFIG_MM_SUBPAGE)) + ret = validate_byte_range(current->mm, uffdio_copy.src, + uffdio_copy.len); + else + ret = validate_unaligned_range(ctx->mm, uffdio_copy.src, + uffdio_copy.len); if (ret) goto out; ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len); From fb16e8779391b801e22226f3faf68520c1dd69fb Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:58 -0700 Subject: [PATCH 614/631] tomoyo: keep the page-based argv dump without user4k tomoyo_dump_page() was rewritten to copy exec arguments through access_remote_vm() with a byte-position cache, so that small user leaves are read correctly. Kernels built without ARM64_USER4K_EXPERIMENTAL do not need this and got a different lookup, locking and caching path from .19. Keep the rewritten helper and its position-based cache under CONFIG_MM_SUBPAGE and restore the page-reference implementation otherwise. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- security/tomoyo/common.h | 5 +++++ security/tomoyo/domain.c | 48 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h index b859f099ae9eac..ea55bcf222c36b 100644 --- a/security/tomoyo/common.h +++ b/security/tomoyo/common.h @@ -575,8 +575,13 @@ struct tomoyo_mini_stat { /* Structure for dumping argv[] and envp[] of "struct linux_binprm". */ struct tomoyo_page_dump { +#ifdef CONFIG_MM_SUBPAGE unsigned long pos; /* First cached virtual byte, or ULONG_MAX. */ char *data; /* Buffer for one native page of virtual bytes. */ +#else + struct page *page; /* Previously dumped page. */ + char *data; /* Contents of "page". Size is PAGE_SIZE. */ +#endif }; /* Structure for attribute checks in addition to pathname checks. */ diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c index 2ff8d783f54cc2..c0bb9305d05d52 100644 --- a/security/tomoyo/domain.c +++ b/security/tomoyo/domain.c @@ -903,6 +903,7 @@ int tomoyo_find_next_domain(struct linux_binprm *bprm) * * Returns true on success, false otherwise. */ +#ifdef CONFIG_MM_SUBPAGE bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, struct tomoyo_page_dump *dump) { @@ -940,3 +941,50 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, dump->pos = pos; return true; } +#else +bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos, + struct tomoyo_page_dump *dump) +{ + struct page *page; +#ifdef CONFIG_MMU + int ret; +#endif + + /* dump->data is released by tomoyo_find_next_domain(). */ + if (!dump->data) { + dump->data = kzalloc(PAGE_SIZE, GFP_NOFS); + if (!dump->data) + return false; + } + /* Same with get_arg_page(bprm, pos, 0) in fs/exec.c */ +#ifdef CONFIG_MMU + /* + * This is called at execve() time in order to dig around + * in the argv/environment of the new process + * (represented by bprm). + */ + mmap_read_lock(bprm->mm); + ret = get_user_pages_remote(bprm->mm, pos, 1, + FOLL_FORCE, &page, NULL); + mmap_read_unlock(bprm->mm); + if (ret <= 0) + return false; +#else + page = bprm->page[pos / PAGE_SIZE]; +#endif + if (page != dump->page) { + const unsigned int offset = pos % PAGE_SIZE; + char *kaddr = kmap_local_page(page); + + dump->page = page; + memcpy(dump->data + offset, kaddr + offset, + PAGE_SIZE - offset); + kunmap_local(kaddr); + } + /* Same with put_arg_page(page) in fs/exec.c */ +#ifdef CONFIG_MMU + put_page(page); +#endif + return true; +} +#endif From c21d8e24a99ea2e3e7ba7465d7def777096bfee6 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:00:59 -0700 Subject: [PATCH 615/631] proc: fold the PAGEMAP_SCAN page size without user4k PAGEMAP_SCAN keeps the target mm's leaf size in pagemap_scan_private so mixed-granule address spaces are walked correctly. Without ARM64_USER4K_EXPERIMENTAL that field always holds PAGE_SIZE, but it is loaded at run time, so the walk uses divides and multiplies instead of shifts and the private structure grows. Keep the field only with CONFIG_MM_SUBPAGE and read it through a helper that returns PAGE_SIZE otherwise. No functional change; production builds compile the scan paths as before. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- fs/proc/task_mmu.c | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 8938ae67581001..22210197753aa0 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -2491,13 +2491,25 @@ static int pagemap_release(struct inode *inode, struct file *file) struct pagemap_scan_private { struct pm_scan_arg arg; +#ifdef CONFIG_MM_SUBPAGE unsigned long page_size; +#endif unsigned long masks_of_interest, cur_vma_category; struct page_region *vec_buf; unsigned long vec_buf_len, vec_buf_index, found_pages; struct page_region __user *vec_out; }; +/* Only mixed-granule kernels carry the target mm's leaf size at run time. */ +static inline unsigned long pagemap_scan_page_size(const struct pagemap_scan_private *p) +{ +#ifdef CONFIG_MM_SUBPAGE + return p->page_size; +#else + return PAGE_SIZE; +#endif +} + static unsigned long pagemap_page_category(struct pagemap_scan_private *p, struct vm_area_struct *vma, unsigned long addr, pte_t pte) @@ -2718,7 +2730,7 @@ static void pagemap_scan_backout_range(struct pagemap_scan_private *p, else cur_buf->start = cur_buf->end = 0; - p->found_pages -= (end - addr) / p->page_size; + p->found_pages -= (end - addr) / pagemap_scan_page_size(p); } #endif @@ -2830,11 +2842,11 @@ static int pagemap_scan_output(unsigned long categories, categories &= p->arg.return_mask; - n_pages = (*end - addr) / p->page_size; + n_pages = (*end - addr) / pagemap_scan_page_size(p); if (check_add_overflow(p->found_pages, n_pages, &total_pages) || total_pages > p->arg.max_pages) { size_t n_too_much = total_pages - p->arg.max_pages; - *end -= n_too_much * p->page_size; + *end -= n_too_much * pagemap_scan_page_size(p); n_pages -= n_too_much; ret = -ENOSPC; } @@ -2928,7 +2940,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { /* Fast path for performing exclusive WP */ - for (addr = start; addr != end; pte++, addr += p->page_size) { + for (addr = start; addr != end; pte++, addr += pagemap_scan_page_size(p)) { pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2937,7 +2949,7 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, make_uffd_wp_pte(vma, addr, pte, ptent); if (!flush_end) start = addr; - flush_end = addr + p->page_size; + flush_end = addr + pagemap_scan_page_size(p); } goto flush_and_return; } @@ -2945,8 +2957,8 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, if (!p->arg.category_anyof_mask && !p->arg.category_inverted && p->arg.category_mask == PAGE_IS_WRITTEN && p->arg.return_mask == PAGE_IS_WRITTEN) { - for (addr = start; addr < end; pte++, addr += p->page_size) { - unsigned long next = addr + p->page_size; + for (addr = start; addr < end; pte++, addr += pagemap_scan_page_size(p)) { + unsigned long next = addr + pagemap_scan_page_size(p); pte_t ptent = ptep_get(pte); if ((pte_present(ptent) && pte_uffd_wp(ptent)) || @@ -2966,11 +2978,11 @@ static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, goto flush_and_return; } - for (addr = start; addr != end; pte++, addr += p->page_size) { + for (addr = start; addr != end; pte++, addr += pagemap_scan_page_size(p)) { pte_t ptent = ptep_get(pte); unsigned long categories = p->cur_vma_category | pagemap_page_category(p, vma, addr, ptent); - unsigned long next = addr + p->page_size; + unsigned long next = addr + pagemap_scan_page_size(p); if (!pagemap_scan_is_interesting_page(categories, p)) continue; @@ -3266,7 +3278,11 @@ static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) { - struct pagemap_scan_private p = { .page_size = mm_page_size(mm) }; + struct pagemap_scan_private p = { +#ifdef CONFIG_MM_SUBPAGE + .page_size = mm_page_size(mm), +#endif + }; unsigned long walk_start; size_t n_ranges_out = 0; int ret; From e0e270cd746e23a63170ba28f3f8f8e1e69af174 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:01:00 -0700 Subject: [PATCH 616/631] vfio/type1: compile out fragment provider paths without MM_SUBPAGE The mixed-granule work added branches on vfio_dma::fragments to the unpin, unmap, map, replay, re-account, owner-change and dma_rw paths. Without CONFIG_MM_SUBPAGE no fragment provider is ever installed, so those branches are dead at run time but are still compiled into every kernel. Test them through vfio_dma_has_fragments(), which folds to false when MM_SUBPAGE is off, so a native-only build carries only the previous code paths. No functional change. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/vfio/vfio_iommu_type1.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index df079de9ee5de5..f7a49214d44bc8 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -255,6 +255,12 @@ static size_t vfio_dirty_pgsize(struct vfio_iommu *iommu) return IS_ENABLED(CONFIG_MM_SUBPAGE) ? SZ_4K : 1UL << __ffs(iommu->pgsize_bitmap); } +/* Fragment providers exist only when mixed user granules are built in. */ +static inline bool vfio_dma_has_fragments(struct vfio_dma *dma) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE) && dma->fragments; +} + static int vfio_dma_bitmap_alloc(struct vfio_dma *dma, size_t pgsize) { uint64_t npages = dma->size / pgsize; @@ -440,7 +446,7 @@ static int vfio_iova_put_vfio_pfn(struct vfio_dma *dma, struct vfio_pfn *vpfn) vpfn->ref_count--; if (!vpfn->ref_count) { - if (dma->fragments) { + if (vfio_dma_has_fragments(dma)) { unsigned long offset = vpfn->iova - dma->iova; unsigned long length = vpfn->length; @@ -973,7 +979,7 @@ static int vfio_pin_external_segment(struct vfio_iommu *iommu, return -EINVAL; } *pfn = vpfn->pfn; - } else if (dma->fragments) { + } else if (vfio_dma_has_fragments(dma)) { ret = vfio_fragments_external(dma, iova, length, pfn); if (ret) return ret; @@ -1207,7 +1213,7 @@ static long vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, if (!dma->size) return 0; - if (dma->fragments) { + if (vfio_dma_has_fragments(dma)) { vfio_fragments_unmap(iommu, dma); return 0; } @@ -1643,7 +1649,7 @@ static int vfio_pin_map_dma(struct vfio_iommu *iommu, struct vfio_dma *dma, unsigned long pfn, limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; int ret = 0; - if (dma->fragments) { + if (vfio_dma_has_fragments(dma)) { dma->size = map_size; ret = vfio_fragments_map_all(iommu, dma); if (ret) @@ -1755,8 +1761,8 @@ static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, struct vfio_fragment_provider *provider; int ret; - if (dma->fragments || (mm_page_size(current->mm) == PAGE_SIZE && - IS_ALIGNED(vaddr, PAGE_SIZE))) + if (vfio_dma_has_fragments(dma) || + (mm_page_size(current->mm) == PAGE_SIZE && IS_ALIGNED(vaddr, PAGE_SIZE))) return vfio_change_dma_mm(dma, current->group_leader, current->mm, capable(CAP_IPC_LOCK)); @@ -1922,7 +1928,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, size_t pos = 0; dma = rb_entry(n, struct vfio_dma, node); - if (dma->fragments) { + if (vfio_dma_has_fragments(dma)) { ret = vfio_fragments_acquire(dma, 0, dma->size); if (!ret) ret = vfio_fragments_map_domain(dma, domain->domain); @@ -2014,7 +2020,7 @@ static int vfio_iommu_replay(struct vfio_iommu *iommu, struct vfio_dma *dma = rb_entry(n, struct vfio_dma, node); size_t pos = 0; - if (dma->fragments) { + if (vfio_dma_has_fragments(dma)) { iommu_unmap(domain->domain, dma->iova, dma->size); vfio_fragments_release_unused(dma); continue; @@ -2555,7 +2561,7 @@ static void vfio_iommu_unmap_unpin_reaccount(struct vfio_iommu *iommu) long locked = 0, unlocked = 0; dma = rb_entry(n, struct vfio_dma, node); - if (dma->fragments) { + if (vfio_dma_has_fragments(dma)) { vfio_unmap_unpin(iommu, dma, true); continue; } @@ -3332,7 +3338,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, return -EPERM; mm = dma->mm; - if (dma->fragments) { + if (vfio_dma_has_fragments(dma)) { int ret; if (!kthread && current->mm != mm) From 2bccca3035db9e41410fde2ede59864c3bf431dc Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:01:01 -0700 Subject: [PATCH 617/631] vfio/type1: keep native external pinning when MM_SUBPAGE is off The external pin/unpin rewrite for retained fragments also changed kernels without mixed user granules: - vfio_iommu_type1_pin_pages() and _unpin_pages() round the start IOVA down to a page instead of using the caller's IOVA; - pinning a page that is already pinned now sets its dirty bit again; - dirty bits are set as length >> pgshift, which is zero when the smallest IOMMU page is larger than PAGE_SIZE, so pinned pages can be missing from the dirty bitmap, both at pin time and when the bitmap is repopulated. Build the fragment-aware pin_pages()/unpin_pages() and their helpers only with CONFIG_MM_SUBPAGE and keep the previous native implementations otherwise. Repopulate the dirty bitmap with one bit per pinned page as before. Drop the vfio_fragments_external() stub, which no longer has a caller without MM_SUBPAGE. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/vfio/vfio_iommu_type1.c | 169 +++++++++++++++++++++- drivers/vfio/vfio_iommu_type1_fragments.c | 5 - 2 files changed, 168 insertions(+), 6 deletions(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index f7a49214d44bc8..86b87b6f2a0dc1 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -296,7 +296,7 @@ static void vfio_dma_populate_bitmap(struct vfio_dma *dma, size_t pgsize) struct vfio_pfn *vpfn = rb_entry(p, struct vfio_pfn, node); bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, - vpfn->length >> pgshift); + IS_ENABLED(CONFIG_MM_SUBPAGE) ? vpfn->length >> pgshift : 1); } } @@ -940,6 +940,7 @@ static int vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, return unlocked; } +#ifdef CONFIG_MM_SUBPAGE /* A native page may contain independently owned fine-grained DMA areas. */ static unsigned long vfio_unpin_external_range(struct vfio_iommu *iommu, dma_addr_t start, unsigned long length, @@ -1111,6 +1112,172 @@ static void vfio_iommu_type1_unpin_pages(void *iommu_data, mutex_unlock(&iommu->lock); WARN_ON(unpinned != iova_size); } +#else +static int vfio_iommu_type1_pin_pages(void *iommu_data, + struct iommu_group *iommu_group, + dma_addr_t user_iova, + int npage, int prot, + struct page **pages) +{ + struct vfio_iommu *iommu = iommu_data; + struct vfio_iommu_group *group; + int i, j, ret; + unsigned long remote_vaddr; + struct vfio_dma *dma; + bool do_accounting; + dma_addr_t iova_end; + size_t iova_size; + + if (!iommu || !pages || npage <= 0) + return -EINVAL; + + /* Supported for v2 version only */ + if (!iommu->v2) + return -EACCES; + + if (check_mul_overflow(npage, PAGE_SIZE, &iova_size) || + check_add_overflow(user_iova, iova_size - 1, &iova_end)) + return -EOVERFLOW; + + mutex_lock(&iommu->lock); + + if (WARN_ONCE(iommu->vaddr_invalid_count, + "vfio_pin_pages not allowed with VFIO_UPDATE_VADDR\n")) { + ret = -EBUSY; + goto pin_done; + } + + /* Fail if no dma_umap notifier is registered */ + if (list_empty(&iommu->device_list)) { + ret = -EINVAL; + goto pin_done; + } + + /* + * If iommu capable domain exist in the container then all pages are + * already pinned and accounted. Accounting should be done if there is no + * iommu capable domain in the container. + */ + do_accounting = list_empty(&iommu->domain_list); + + for (i = 0; i < npage; i++) { + unsigned long phys_pfn; + dma_addr_t iova; + struct vfio_pfn *vpfn; + + iova = user_iova + PAGE_SIZE * i; + dma = vfio_find_dma(iommu, iova, PAGE_SIZE); + if (!dma) { + ret = -EINVAL; + goto pin_unwind; + } + + if ((dma->prot & prot) != prot) { + ret = -EPERM; + goto pin_unwind; + } + + vpfn = vfio_iova_get_vfio_pfn(dma, iova); + if (vpfn) { + pages[i] = pfn_to_page(vpfn->pfn); + continue; + } + + remote_vaddr = dma->vaddr + (iova - dma->iova); + ret = vfio_pin_page_external(dma, remote_vaddr, &phys_pfn, + do_accounting); + if (ret) + goto pin_unwind; + + if (!pfn_valid(phys_pfn)) { + ret = -EINVAL; + goto pin_unwind; + } + + ret = vfio_add_to_pfn_list(dma, iova, phys_pfn, PAGE_SIZE); + if (ret) { + if (put_pfn(phys_pfn, dma->prot) && do_accounting) + vfio_lock_acct(dma, -1, true); + goto pin_unwind; + } + + pages[i] = pfn_to_page(phys_pfn); + + if (iommu->dirty_page_tracking) { + unsigned long pgshift = __ffs(iommu->pgsize_bitmap); + + /* + * Bitmap populated with the smallest supported page + * size + */ + bitmap_set(dma->bitmap, + (iova - dma->iova) >> pgshift, 1); + } + } + ret = i; + + group = vfio_iommu_find_iommu_group(iommu, iommu_group); + if (!group->pinned_page_dirty_scope) { + group->pinned_page_dirty_scope = true; + iommu->num_non_pinned_groups--; + } + + goto pin_done; + +pin_unwind: + pages[i] = NULL; + for (j = 0; j < i; j++) { + dma_addr_t iova; + + iova = user_iova + PAGE_SIZE * j; + dma = vfio_find_dma(iommu, iova, PAGE_SIZE); + vfio_unpin_page_external(dma, iova, do_accounting); + pages[j] = NULL; + } +pin_done: + mutex_unlock(&iommu->lock); + return ret; +} + +static void vfio_iommu_type1_unpin_pages(void *iommu_data, + dma_addr_t user_iova, int npage) +{ + struct vfio_iommu *iommu = iommu_data; + bool do_accounting; + dma_addr_t iova_end; + size_t iova_size; + int i; + + /* Supported for v2 version only */ + if (WARN_ON(!iommu->v2)) + return; + + if (WARN_ON(npage <= 0)) + return; + + if (WARN_ON(check_mul_overflow(npage, PAGE_SIZE, &iova_size) || + check_add_overflow(user_iova, iova_size - 1, &iova_end))) + return; + + mutex_lock(&iommu->lock); + + do_accounting = list_empty(&iommu->domain_list); + for (i = 0; i < npage; i++) { + dma_addr_t iova = user_iova + PAGE_SIZE * i; + struct vfio_dma *dma; + + dma = vfio_find_dma(iommu, iova, PAGE_SIZE); + if (!dma) + break; + + vfio_unpin_page_external(dma, iova, do_accounting); + } + + mutex_unlock(&iommu->lock); + + WARN_ON(i != npage); +} +#endif static long vfio_sync_unpin(struct vfio_dma *dma, struct vfio_domain *domain, struct list_head *regions, diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c index 22d46a1b805a7b..22bfb611cab875 100644 --- a/drivers/vfio/vfio_iommu_type1_fragments.c +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -596,11 +596,6 @@ static void vfio_fragments_unmap(struct vfio_iommu *iommu, struct vfio_dma *dma) static void vfio_fragments_destroy(struct vfio_dma *dma) { } -static int vfio_fragments_external(struct vfio_dma *dma, dma_addr_t iova, - unsigned long length, unsigned long *pfn) -{ - return -EOPNOTSUPP; -} static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *data, unsigned long length, bool write, size_t *copied) { From d698033b97a2179ac547ae9d6009f134e4cc615c Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:01:01 -0700 Subject: [PATCH 618/631] vfio/type1: skip the CAP_IPC_LOCK check when the DMA owner is unchanged vfio_change_dma_owner() now calls capable(CAP_IPC_LOCK) for VFIO_DMA_MAP_FLAG_VADDR updates of native-page mappings, including updates issued by the process that already owns the mapping. capable() has side effects: it marks the task as having used a privilege and can emit LSM audit records. The previous code returned before the check when the mm did not change. Return early in that case again. The result is unchanged: moving the charge of an unchanged owner onto itself was already a no-op. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/vfio/vfio_iommu_type1.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 86b87b6f2a0dc1..26c567100d9936 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -1929,9 +1929,13 @@ static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, int ret; if (vfio_dma_has_fragments(dma) || - (mm_page_size(current->mm) == PAGE_SIZE && IS_ALIGNED(vaddr, PAGE_SIZE))) + (mm_page_size(current->mm) == PAGE_SIZE && IS_ALIGNED(vaddr, PAGE_SIZE))) { + /* An unchanged owner needs neither a capability check nor a recharge. */ + if (current->mm == dma->mm) + return 0; return vfio_change_dma_mm(dma, current->group_leader, current->mm, capable(CAP_IPC_LOCK)); + } /* Prepare only metadata: failure must preserve every legacy pin/charge. */ provider = vfio_fragments_prepare_native(iommu, dma); From 3d100cb5a7670f62c9fe807d111a26ee86edeca4 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:01:02 -0700 Subject: [PATCH 619/631] vfio/type1: keep the IOMMU page size warnings without MM_SUBPAGE The map and unmap paths warned when the smallest IOMMU page size is larger than PAGE_SIZE, a layout that the native pinning and dirty tracking code does not handle. The mixed-granule work removed both warnings for every kernel. Keep them when CONFIG_MM_SUBPAGE is off. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/vfio/vfio_iommu_type1.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 26c567100d9936..9c08dac3fa66b6 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -1671,6 +1671,7 @@ static int vfio_dma_do_unmap(struct vfio_iommu *iommu, goto unlock; } + WARN_ON(!IS_ENABLED(CONFIG_MM_SUBPAGE) && ((pgsize - 1) & PAGE_MASK)); again: /* * vfio-iommu-type1 (v1) - User mappings were coalesced together to @@ -1987,6 +1988,8 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, pgsize = (size_t)1 << __ffs(iommu->pgsize_bitmap); + WARN_ON(!IS_ENABLED(CONFIG_MM_SUBPAGE) && ((pgsize - 1) & PAGE_MASK)); + if ((size | iova | vaddr) & (pgsize - 1)) { ret = -EINVAL; goto out_unlock; From 2b1ceeb10e4736f733bbea2f233a03f4a0d1b542 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:01:03 -0700 Subject: [PATCH 620/631] vfio/type1: keep the previous locked_vm refund rules without MM_SUBPAGE The mixed-granule work changed locked_vm accounting for every kernel: refunds take the owner's mmap_lock without allowing a fatal signal to interrupt them, refunds are applied to an mm whose users have all exited, a zero-page charge no longer takes the lock, and a DMA owner change refunds the old mm even after it has exited and undoes the new charge if that refund fails. These keep vfio_dma::locked_vm consistent with the pins that were actually released, which retained fragments rely on, but they also change kernels without mixed user granules. Keep the previous rules when CONFIG_MM_SUBPAGE is off, and build the accounting KUnit suite, which checks the new rules, only with MM_SUBPAGE. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- drivers/vfio/Kconfig | 4 +-- drivers/vfio/vfio_iommu_type1.c | 41 +++++++++++++++++++++-- drivers/vfio/vfio_iommu_type1_fragments.c | 16 --------- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index 92103812d8fcfb..c0f91dad66c2dd 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -56,9 +56,9 @@ config VFIO_IOMMU_TYPE1 config VFIO_TYPE1_ACCOUNT_KUNIT_TEST bool "Test VFIO type1 pin accounting and retained fragments" if !KUNIT_ALL_TESTS - depends on KUNIT=y && VFIO_IOMMU_TYPE1=y + depends on KUNIT=y && VFIO_IOMMU_TYPE1=y && MM_SUBPAGE # The retained-fragment tests write their source pages to a shmem file - depends on TMPFS || !MM_SUBPAGE + depends on TMPFS default KUNIT_ALL_TESTS help Exercise locked-memory refunds after source-mm exit and DMA owner diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 9c08dac3fa66b6..9da7326f684d20 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -470,11 +470,11 @@ static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, { int ret; - if (!npage) + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && !npage) return 0; /* A released pin must be refunded even if the caller is interrupted. */ - if (npage < 0) { + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && npage < 0) { mmap_write_lock(mm); } else { ret = mmap_write_lock_killable(mm); @@ -490,7 +490,7 @@ static int mm_lock_acct(struct task_struct *task, struct mm_struct *mm, static int vfio_lock_acct(struct vfio_dma *dma, long npage, bool async) { struct mm_struct *mm; - bool get_mm = async && npage > 0; + bool get_mm = async && (!IS_ENABLED(CONFIG_MM_SUBPAGE) || npage > 0); int ret; if (!npage) @@ -1881,6 +1881,7 @@ static bool vfio_iommu_iova_dma_valid(struct vfio_iommu *iommu, return list_empty(iova); } +#ifdef CONFIG_MM_SUBPAGE static int vfio_change_dma_mm_pages(struct vfio_dma *dma, struct task_struct *task, struct mm_struct *mm, bool lock_cap, unsigned long pages) { @@ -1950,6 +1951,40 @@ static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, vfio_fragments_commit_native(dma, provider); return ret; } +#else +static int vfio_change_dma_owner(struct vfio_iommu *iommu, struct vfio_dma *dma, + unsigned long vaddr) +{ + struct task_struct *task = current->group_leader; + struct mm_struct *mm = current->mm; + long npage = dma->locked_vm; + bool lock_cap; + int ret; + + if (mm == dma->mm) + return 0; + + lock_cap = capable(CAP_IPC_LOCK); + ret = mm_lock_acct(task, mm, lock_cap, npage); + if (ret) + return ret; + + if (mmget_not_zero(dma->mm)) { + mm_lock_acct(dma->task, dma->mm, dma->lock_cap, -npage); + mmput(dma->mm); + } + + if (dma->task != task) { + put_task_struct(dma->task); + dma->task = get_task_struct(task); + } + mmdrop(dma->mm); + dma->mm = mm; + mmgrab(dma->mm); + dma->lock_cap = lock_cap; + return 0; +} +#endif static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) diff --git a/drivers/vfio/vfio_iommu_type1_fragments.c b/drivers/vfio/vfio_iommu_type1_fragments.c index 22bfb611cab875..659c7b075efa28 100644 --- a/drivers/vfio/vfio_iommu_type1_fragments.c +++ b/drivers/vfio/vfio_iommu_type1_fragments.c @@ -547,22 +547,6 @@ static int vfio_fragments_rw(struct vfio_dma *dma, unsigned long offset, void *d return ret; } #else -static struct vfio_fragment_provider * -vfio_fragments_prepare_native(struct vfio_iommu *iommu, struct vfio_dma *dma) -{ - return ERR_PTR(-EOPNOTSUPP); -} -static void vfio_fragments_abort_native(struct vfio_fragment_provider *provider) -{ -} -static unsigned long vfio_fragments_native_pages(struct vfio_fragment_provider *provider) -{ - return 0; -} -static void vfio_fragments_commit_native(struct vfio_dma *dma, - struct vfio_fragment_provider *provider) -{ -} static void vfio_fragments_put_native_pin(struct vfio_dma *dma, unsigned long pfn) { } From 4f088212736f57544e4785cfe8a3392facd24790 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:01:04 -0700 Subject: [PATCH 621/631] exec: drop the user page-size preference on privilege-gaining exec The experimental exec page-size preference is set by an unprivileged prctl and survives exec: PR_AURORA_SET_EXEC_PAGE_SIZE selects the granule for the next exec and PR_AURORA_SET_DEFAULT_PAGE_SIZE for all later ones. Nothing cleared it for a set-user-ID or set-group-ID binary, so an unprivileged user could start a privileged program with a 4K user address space and code paths it was never tested with, and the preference was also inherited by everything that program executed. Treat the preference like the personality flags in PER_CLEAR_ON_SETID. arm64_bprm_prepare_mm() now also records the granule the image gets without a preference (native for AArch64, 4K for AArch32 when available). Once begin_new_exec() has computed the new credentials, an exec that gains privilege (set-user-ID, set-group-ID, file capabilities or an LSM secure-exec transition) is restaged at that granule before the point of no return, and both the one-shot and the default preference are cleared for the new image and its children. Add KUnit cases that stage arguments in a 4K mm and check that a privileged exec is restaged at the native granule with its arguments intact, that an unprivileged one keeps 4K, and that the arch hook records the preference-free granule for AArch64 and AArch32. Only affects kernels built with ARM64_USER4K_EXPERIMENTAL. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- arch/arm64/kernel/process.c | 9 ++- arch/arm64/mm/user4k-test.c | 116 ++++++++++++++++++++++++++++++++++++ fs/exec.c | 41 +++++++++++++ include/linux/binfmts.h | 8 +++ include/uapi/linux/prctl.h | 5 +- 5 files changed, 175 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index a326135b164d4f..6d54e364810343 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -891,10 +891,13 @@ int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) { unsigned int requested = current->exec_page_shift ?: current->default_exec_page_shift; - unsigned int shift = arm64_exec_page_shift(compat, - cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE), PAGE_SHIFT, - requested); + bool has_4k = cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE); + unsigned int shift = arm64_exec_page_shift(compat, has_4k, PAGE_SHIFT, + requested); + /* begin_new_exec() falls back to this if the exec gains privilege. */ + bprm->secure_page_shift = arm64_exec_page_shift(compat, has_4k, + PAGE_SHIFT, 0); return bprm_set_page_shift(bprm, shift); } diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index d5349f54f6d7c4..a78c03c7d8c967 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -5377,6 +5378,119 @@ static void user4k_exec_arguments_test(struct kunit *test) } } +/* Stage two strings at the top of a fresh exec stack in @mm. */ +static unsigned long test_stage_exec_strings(struct kunit *test, + struct linux_binprm *bprm, struct mm_struct *mm) +{ + unsigned long initial; + + bprm->mm = mm; + KUNIT_ASSERT_EQ(test, create_init_stack_vma(mm, &bprm->vma, &bprm->p), 0); + initial = bprm->p; + bprm->argmin = initial - SZ_2M; + bprm->rlim_stack.rlim_cur = SZ_8M; + bprm->rlim_stack.rlim_max = RLIM_INFINITY; + KUNIT_ASSERT_EQ(test, copy_string_kernel("arg", bprm), 0); + KUNIT_ASSERT_EQ(test, copy_string_kernel("setid-image", bprm), 0); + return initial - bprm->p; +} + +static void test_expect_exec_strings(struct kunit *test, + struct linux_binprm *bprm, unsigned long copied) +{ + char buf[16] = {}; + + KUNIT_ASSERT_EQ(test, copied, sizeof("setid-image") + sizeof("arg")); + KUNIT_ASSERT_EQ(test, access_remote_vm(bprm->mm, bprm->p, buf, copied, 0), + (int)copied); + KUNIT_EXPECT_STREQ(test, buf, "setid-image"); + KUNIT_EXPECT_STREQ(test, buf + sizeof("setid-image"), "arg"); +} + +/* + * A set-user-ID, set-group-ID or secure-exec image must not run with the + * caller's unprivileged granule preference: it is restaged at the granule + * it gets without one, with the staged arguments intact. + */ +static void user4k_exec_privileged_test(struct kunit *test) +{ + unsigned int privileged; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm stack setup requires hardware opt-in"); + return; + } + for (privileged = 0; privileged < 3; privileged++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + struct mm_struct *mm = test_vma_mm(test); + unsigned long copied; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + KUNIT_ASSERT_NOT_NULL(test, mm); + copied = test_stage_exec_strings(test, bprm, mm); + bprm->secure_page_shift = PAGE_SHIFT; + if (privileged == 1) + bprm->per_clear = PER_CLEAR_ON_SETID; + else if (privileged == 2) + bprm->secureexec = 1; + /* A replacement drops the staging mm's reference. */ + mmget(mm); + KUNIT_ASSERT_EQ(test, bprm_secure_page_size(bprm), 0); + if (bprm->mm != mm) + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, + user4k_vma_mm_free, bprm->mm), 0); + else + mmput(mm); + KUNIT_EXPECT_EQ(test, mm_page_shift(bprm->mm), + privileged ? PAGE_SHIFT : 12U); + KUNIT_EXPECT_PTR_EQ(test, bprm->vma->vm_mm, bprm->mm); + test_expect_exec_strings(test, bprm, copied); + } +} + +/* The arch selection records the preference-free granule for begin_new_exec(). */ +static void user4k_exec_secure_granule_test(struct kunit *test) +{ + unsigned int saved = current->exec_page_shift; + unsigned int compat; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm stack setup requires hardware opt-in"); + return; + } + if (!cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE)) { + kunit_skip(test, "no 4K translation granule"); + return; + } + for (compat = 0; compat < 2; compat++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + struct mm_struct *mm = mm_alloc_exec_page_shift(PAGE_SHIFT); + unsigned long copied; + int ret; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + KUNIT_ASSERT_NOT_NULL(test, mm); + copied = test_stage_exec_strings(test, bprm, mm); + current->exec_page_shift = 12; + ret = arm64_bprm_prepare_mm(bprm, compat); + current->exec_page_shift = saved; + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_EQ(test, mm_page_shift(bprm->mm), 12U); + /* AArch32 always uses 4K when available; AArch64 defaults to native. */ + KUNIT_EXPECT_EQ(test, bprm->secure_page_shift, compat ? 12U : PAGE_SHIFT); + bprm->per_clear = PER_CLEAR_ON_SETID; + mm = bprm->mm; + /* Keep the staging mm alive across a possible replacement. */ + mmget(mm); + KUNIT_ASSERT_EQ(test, bprm_secure_page_size(bprm), 0); + mmput(mm); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, + bprm->mm), 0); + KUNIT_EXPECT_EQ(test, mm_page_shift(bprm->mm), compat ? 12U : PAGE_SHIFT); + test_expect_exec_strings(test, bprm, copied); + } +} + static struct vm_area_struct *test_special_vma(struct mm_struct *mm, const struct vm_special_mapping *sm, unsigned long start, unsigned long len, vm_flags_t flags) @@ -14324,6 +14438,8 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_mremap_resize_test), KUNIT_CASE(user4k_stack_relocate_test), KUNIT_CASE(user4k_exec_arguments_test), + KUNIT_CASE(user4k_exec_privileged_test), + KUNIT_CASE(user4k_exec_secure_granule_test), KUNIT_CASE(user4k_special_mapping_test), KUNIT_CASE(native_callback_mapping_test), KUNIT_CASE(user4k_callback_mapping_test), diff --git a/fs/exec.c b/fs/exec.c index c9e6ecf187b7b6..cde9ae10c7f2a8 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1253,6 +1253,33 @@ void __set_task_comm(struct task_struct *tsk, const char *buf, bool exec) perf_event_comm(tsk, exec); } +#if defined(CONFIG_MM_SUBPAGE) || defined(CONFIG_ARCH_HAS_USER_PAGE_SIZE) +/* + * The user page-size preference is set by an unprivileged prctl. Like the + * personality bits in PER_CLEAR_ON_SETID, it must not reach an image that + * gains privilege through set-user-ID, set-group-ID, file capabilities or + * an LSM secure-exec transition. + */ +static bool bprm_drops_page_size_policy(const struct linux_binprm *bprm) +{ + return (bprm->per_clear & PER_CLEAR_ON_SETID) || bprm->secureexec; +} +#endif + +#ifdef CONFIG_MM_SUBPAGE +/* + * Stage an image that gains privilege at the granule it gets without the + * caller's preference. Credentials must already be computed, and this + * must run before the point of no return. + */ +int bprm_secure_page_size(struct linux_binprm *bprm) +{ + if (!bprm_drops_page_size_policy(bprm)) + return 0; + return bprm_set_page_shift(bprm, bprm->secure_page_shift ?: PAGE_SHIFT); +} +#endif + /* * Calling this is the point of no return. None of the failures will be * seen by userspace since either the process is already taking a fatal @@ -1269,6 +1296,13 @@ int begin_new_exec(struct linux_binprm * bprm) if (retval) return retval; +#ifdef CONFIG_MM_SUBPAGE + /* Still before the point of no return: a failure fails the exec. */ + retval = bprm_secure_page_size(bprm); + if (retval) + return retval; +#endif + /* * This tracepoint marks the point before flushing the old exec where * the current task is still unchanged, but errors are fatal (point of @@ -1345,6 +1379,13 @@ int begin_new_exec(struct linux_binprm * bprm) PF_NOFREEZE | PF_NO_SETAFFINITY); flush_thread(); me->personality &= ~bprm->per_clear; +#ifdef CONFIG_ARCH_HAS_USER_PAGE_SIZE + /* A privileged image and its children start without a preference. */ + if (bprm_drops_page_size_policy(bprm)) { + me->exec_page_shift = 0; + me->default_exec_page_shift = 0; + } +#endif clear_syscall_work_syscall_user_dispatch(me); diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index c242a5e9cc758f..c39594705bc607 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -67,6 +67,13 @@ struct linux_binprm { unsigned long exec; struct rlimit rlim_stack; /* Saved RLIMIT_STACK used during exec. */ +#ifdef CONFIG_MM_SUBPAGE + /* + * User page shift the image gets without the caller's page-size + * preference, used when the exec gains privilege. Zero: native. + */ + unsigned int secure_page_shift; +#endif char buf[BINPRM_BUF_SIZE]; } __randomize_layout; @@ -125,6 +132,7 @@ extern void unregister_binfmt(struct linux_binfmt *); #ifdef CONFIG_MM_SUBPAGE int bprm_set_page_shift(struct linux_binprm *bprm, unsigned int page_shift); +int bprm_secure_page_size(struct linux_binprm *bprm); #endif extern int __must_check remove_arg_zero(struct linux_binprm *); diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index 6e08471fbe822f..a46e4de4fff318 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -430,7 +430,10 @@ struct prctl_mm_map { * SET_DEFAULT selects the fallback for subsequent execs; zero restores native. * GET_DEFAULT returns the alternative size in bytes, or zero for native. * Fork inherits both policies; exec preserves DEFAULT. Failed exec preserves - * both. Neither setter changes the live address space or other threads. + * both. An exec that gains privilege (set-user-ID, set-group-ID, file + * capabilities or a secure-exec transition) ignores both, runs the image at + * its default granule and clears both for the new image. + * Neither setter changes the live address space or other threads. * AArch32 always selects 4K when supported, otherwise the native granule, * independently of these preferences. Sizes are capability-checked. */ From 5fc7e1d1fb9dfe1d1d9b78a7abf050c634cfafcd Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:01:04 -0700 Subject: [PATCH 622/631] mm: refuse HugeTLB mappings in mms with a non-native user granule HugeTLB mappings are built with the native page table geometry, and PMD tables can be shared between processes mapping the same file (huge_pmd_share() and pt_share_count). An mm with 4K user tables on a 16K kernel uses different table formats, and its table descriptors reuse the same ptdesc word for fragment counts, but nothing stopped a 4K-mode process from mapping hugetlbfs, MAP_HUGETLB, SHM_HUGETLB or MFD_HUGETLB memory. Refuse such mappings with -EINVAL in do_mmap() when the mm's user granule is not the native page size. Every HugeTLB VMA is created through do_mmap(), and fork and exec cannot hand one to such an mm, so this covers all entry points. Applications that probe for huge pages fall back as they do when none are configured. Add a KUnit case that maps a hugetlbfs file and MAP_HUGETLB memory into a 4K mm (refused) and the same file into a native mm (accepted). Only affects kernels built with ARM64_USER4K_EXPERIMENTAL. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- arch/arm64/mm/user4k-test.c | 49 +++++++++++++++++++++++++++++++++++++ mm/mmap.c | 10 ++++++++ 2 files changed, 59 insertions(+) diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index a78c03c7d8c967..9d7cbd4aae3d5b 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #include #include @@ -5491,6 +5493,52 @@ static void user4k_exec_secure_granule_test(struct kunit *test) } } +/* HugeTLB tables use native geometry; a small-granule mm must not map them. */ +static void user4k_hugetlb_refused_test(struct kunit *test) +{ +#ifdef CONFIG_HUGETLBFS + const unsigned long prot = PROT_READ | PROT_WRITE; + const unsigned long flags = MAP_SHARED | MAP_NORESERVE | MAP_FIXED_NOREPLACE; + const unsigned long addr = SZ_1G; + struct mm_struct *small, *native; + unsigned long len, result[3]; + struct file *file; + + if (!user4k_test_enabled && !IS_ENABLED(CONFIG_KASAN)) { + kunit_skip(test, "borrowed-mm mapping fixture requires hardware opt-in"); + return; + } + len = huge_page_size(&default_hstate); + file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, mk_vma_flags(VMA_NORESERVE_BIT), + HUGETLB_ANONHUGE_INODE, 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, file); + small = test_vma_mm(test); + native = mm_alloc(); + KUNIT_ASSERT_NOT_NULL(test, small); + KUNIT_ASSERT_NOT_NULL(test, native); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, user4k_vma_mm_free, native), 0); + + kthread_use_mm(small); + result[0] = vm_mmap(file, addr, len, prot, flags, 0); + result[1] = ksys_mmap_pgoff(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | + MAP_HUGETLB | MAP_NORESERVE | MAP_FIXED_NOREPLACE, + -1, 0); + kthread_unuse_mm(small); + kthread_use_mm(native); + result[2] = vm_mmap(file, addr, len, prot, flags, 0); + kthread_unuse_mm(native); + fput(file); + + KUNIT_EXPECT_EQ(test, result[0], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, result[1], (unsigned long)-EINVAL); + KUNIT_EXPECT_EQ(test, small->map_count, 0); + /* The same mapping is accepted with native tables. */ + KUNIT_EXPECT_EQ(test, result[2], addr); +#else + kunit_skip(test, "requires CONFIG_HUGETLBFS"); +#endif +} + static struct vm_area_struct *test_special_vma(struct mm_struct *mm, const struct vm_special_mapping *sm, unsigned long start, unsigned long len, vm_flags_t flags) @@ -14440,6 +14488,7 @@ static struct kunit_case user4k_test_cases[] = { KUNIT_CASE(user4k_exec_arguments_test), KUNIT_CASE(user4k_exec_privileged_test), KUNIT_CASE(user4k_exec_secure_granule_test), + KUNIT_CASE(user4k_hugetlb_refused_test), KUNIT_CASE(user4k_special_mapping_test), KUNIT_CASE(native_callback_mapping_test), KUNIT_CASE(user4k_callback_mapping_test), diff --git a/mm/mmap.c b/mm/mmap.c index f82ed9c9204f20..d80c52188d293a 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -352,6 +352,16 @@ static unsigned long do_mmap_internal(struct file *file, unsigned long addr, !IS_ALIGNED(pos.offset, mm_page_size(mm))) return -EINVAL; + /* + * HugeTLB mappings use native page table geometry and can share PMD + * tables with other processes. An mm with another user granule has + * different tables, so refuse them for every hugetlbfs, MAP_HUGETLB, + * SHM_HUGETLB and MFD_HUGETLB file. + */ + if (IS_ENABLED(CONFIG_MM_SUBPAGE) && file && is_file_hugepages(file) && + mm_page_size(mm) != PAGE_SIZE) + return -EINVAL; + /* * Does the application expect PROT_READ to imply PROT_EXEC? * From 632c8fade2c3cea620e02696905b867c4ed92c8e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:45:29 -0700 Subject: [PATCH 623/631] mm: move the vmf_insert_pfn_prot_mkwrite() kernel-doc back The mixed-granule rework split vmf_insert_pfn_prot_mkwrite() into a static __vmf_insert_pfn_prot_mkwrite() helper and a thin exported wrapper, but left the kernel-doc comment on the helper. kernel-doc now warns that the comment describes the wrong prototype and that the helper's offset parameter is undocumented, and the exported function lost its documentation. Move the comment back onto vmf_insert_pfn_prot_mkwrite(). No code change: the prod-config mm/memory.o is identical before and after. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/memory.c | 53 +++++++++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index ae28b2d3454d99..a688ae9b44cf6f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3258,6 +3258,33 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, return VM_FAULT_NOPAGE; } +static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn, pgprot_t pgprot, + bool mkwrite, unsigned int offset) +{ + /* + * Technically, architectures with pte_special can avoid all these + * restrictions (same for remap_pfn_range). However we would like + * consistency in testing and feature parity among all, so we should + * try to keep these invariants in place for everybody. + */ + BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); + BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == + (VM_PFNMAP|VM_MIXEDMAP)); + BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); + BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); + + if (addr < vma->vm_start || addr >= vma->vm_end) + return VM_FAULT_SIGBUS; + + if (!pfn_modify_allowed(pfn, pgprot)) + return VM_FAULT_SIGBUS; + + pfnmap_setup_cachemode_pfn(pfn, &pgprot); + + return insert_pfn(vma, addr, pfn, pgprot, mkwrite, offset); +} + /** * vmf_insert_pfn_prot_mkwrite - insert single pfn into user vma with specified pgprot * @vma: user vma to map to @@ -3305,32 +3332,6 @@ static vm_fault_t insert_pfn(struct vm_area_struct *vma, unsigned long addr, * Context: Process context. May allocate using %GFP_KERNEL. * Return: vm_fault_t value. */ -static vm_fault_t __vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, - unsigned long addr, unsigned long pfn, pgprot_t pgprot, - bool mkwrite, unsigned int offset) -{ - /* - * Technically, architectures with pte_special can avoid all these - * restrictions (same for remap_pfn_range). However we would like - * consistency in testing and feature parity among all, so we should - * try to keep these invariants in place for everybody. - */ - BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); - BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == - (VM_PFNMAP|VM_MIXEDMAP)); - BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); - BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn)); - - if (addr < vma->vm_start || addr >= vma->vm_end) - return VM_FAULT_SIGBUS; - - if (!pfn_modify_allowed(pfn, pgprot)) - return VM_FAULT_SIGBUS; - - pfnmap_setup_cachemode_pfn(pfn, &pgprot); - - return insert_pfn(vma, addr, pfn, pgprot, mkwrite, offset); -} vm_fault_t vmf_insert_pfn_prot_mkwrite(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn, pgprot_t pgprot, bool mkwrite) From 4154c885c0b9938379ffbeb6795260eb004db3b8 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:46:08 -0700 Subject: [PATCH 624/631] perf/core: restore the speculation bound on AUX page lookups When the mmap path was converted to byte offsets for mixed user granules, the AUX page lookup moved from perf_mmap_to_page() into map_range(), and the array_index_nospec() clamp on the AUX page index was dropped. The index is still steered by userspace: map_range() walks every page of the AUX mapping, whose length the caller chooses, and only a conditional branch keeps pagenum below rb->aux_nr_pages before rb->aux_pages[] is read. Perf events are built into every kernel, so this weakened the Spectre-v1 hardening on all configurations. Clamp the index with array_index_nospec() again, as perf_mmap_to_page() did before. The architectural behaviour is unchanged; map_range() again contains the cmp/ngc/csdb/and mask sequence that the .19 kernel had in perf_mmap_to_page(). Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- kernel/events/core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index 4b1b0f8b15cf29..098f05936d7a68 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #include "internal.h" @@ -7215,7 +7216,9 @@ static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) err = -EINVAL; break; } - page = virt_to_page(rb->aux_pages[pagenum]); + /* The mapping length drives pagenum: bound it under speculation. */ + page = virt_to_page(rb->aux_pages[array_index_nospec(pagenum, + rb->aux_nr_pages)]); } else { page = perf_mmap_to_page(rb, pagenum); if (!pagenum) From e0bf77e8a3166a29a256995646e40e2db8b3beaa Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:46:43 -0700 Subject: [PATCH 625/631] arm64: make the mixed-granule KUnit tests depend on TMPFS Several ARM64_USER4K_KUNIT_TEST cases create shmem files with shmem_file_setup() and fill them with kernel_write(): the shared cache-file fixture used by many file-mapping cases, the 64K-granule shmem fault allocation-failure case and the 4K shmem partial-hole case. Without CONFIG_TMPFS shmem files have no write_iter, so those writes fail and the cases fail at run time. Nothing in Kconfig asked for TMPFS, so a kunitconfig that enables the tests but not TMPFS builds a kernel whose tests fail. Make ARM64_USER4K_KUNIT_TEST depend on TMPFS, as was done for the VFIO type1 harness options. The in-tree arm64_user4k kunitconfigs already set TMPFS=y and configure as before; without TMPFS, kunit.py now reports the test option as unsatisfiable. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- arch/arm64/Kconfig.debug | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 9331b85d04aa17..afed4f763c6158 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -47,6 +47,7 @@ config ARM64_USER4K_KUNIT_TEST bool "Test mixed userspace granules with larger kernel pages" depends on KUNIT=y && ARM64_USER4K_EXPERIMENTAL depends on ARM64_PA_BITS_48 + depends on TMPFS select ARM64_USER4K_DATA_KUNIT_TEST select IOMMUFD select IOMMUFD_DRIVER From d97563b36cd13a215367a44db6f8f4952b97f508 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:12:42 -0700 Subject: [PATCH 626/631] arm64: build mixed-granule userspace by default Turn ARM64_USER4K_EXPERIMENTAL on by default, so release kernels configured with olddefconfig include it. A process that doesn't use the PR_AURORA_SET_EXEC_PAGE_SIZE prctl keeps the native granule. The mixed-granule paths in core mm, net, vfio, block and io_uring are then built in, though, and every process runs through them. So far it has run on the J700 (MacBook Neo) with j700.config, and in the arm64_user4k KUnit configurations under QEMU. It has not been run on an M1, M2 or M3 machine. Replace the help text's advice to say N, which no longer matches the default. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- arch/arm64/Kconfig.debug | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index afed4f763c6158..1f779db2342d26 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -32,7 +32,7 @@ config ARM64_USER4K_EXPERIMENTAL select ARCH_SUPPORTS_PT_RECLAIM depends on (ARM64_16K_PAGES || ARM64_64K_PAGES) && ARM64_VA_BITS_48 && ARM64_PA_BITS_48 depends on !ARM64_LPA2 && !NO_PAGE_MAPCOUNT - default n + default y help Build the in-development MM support for alternative 4K user tables on 16K/64K kernels and 16K user tables on 64K kernels. @@ -41,7 +41,7 @@ config ARM64_USER4K_EXPERIMENTAL This is a development ABI: several MM interfaces and device mappings still require integration and validation. The private prctl numbers are not upstream assignments or a stable ABI promise. - Say N for normal kernels. + Programs that do not use the prctl keep the native granule. config ARM64_USER4K_KUNIT_TEST bool "Test mixed userspace granules with larger kernel pages" From 249829c32c880ddba07771db7c9c95a2baa0dbd2 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:47:42 -0700 Subject: [PATCH 627/631] Documentation: arm64: user4k: say the option is on by default The previous commit made ARM64_USER4K_EXPERIMENTAL default to y, so the configuration section no longer matches when it says the option defaults off. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- Documentation/arch/arm64/user4k.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index 70e39f084e4172..f6278555f680da 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -19,7 +19,7 @@ Configuration and supported combinations ======================================== Enable ``CONFIG_ARM64_USER4K_EXPERIMENTAL`` with 48-bit VA and PA and a native -16K or 64K page size. The option defaults off and selects the generic +16K or 64K page size. The option is on by default and selects the generic ``MM_SUBPAGE`` backing and ``ARCH_HAS_USER_PAGE_SIZE`` exec policy support. LPA2 and ``NO_PAGE_MAPCOUNT`` configurations are excluded by Kconfig. From 2a56089e5baf2bdd9d95760ca5b60e3ab76a9d0a Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:51:10 -0700 Subject: [PATCH 628/631] mm: pick the mmap layout from the mm being set up With ARM64_USER4K_EXPERIMENTAL, arch_pick_mmap_layout() takes the mmap randomisation and the mmap_base alignment from current->mm, not from the mm it was given. Exec only calls it once the new mm is installed, so the two are the same there. KUnit's kunit_vm_mmap() also calls it, from a kernel thread with no mm, and every usercopy test then oopses on a NULL pointer dereference in arch_pick_mmap_layout(). Pass the mm through to the randomisation and mmap_base helpers. arch_mmap_rnd() keeps using current->mm for its other callers. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- mm/util.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/mm/util.c b/mm/util.c index b37a3e8bc63109..a4b9ddc5368594 100644 --- a/mm/util.c +++ b/mm/util.c @@ -406,7 +406,7 @@ unsigned long __weak arch_randomize_brk(struct mm_struct *mm) return randomize_page_shift(mm->brk, SZ_1G, mm_page_shift(mm)); } -unsigned long arch_mmap_rnd(void) +static unsigned long mmap_rnd(const struct mm_struct *mm) { unsigned int bits; unsigned long mask; @@ -418,8 +418,13 @@ unsigned long arch_mmap_rnd(void) #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */ bits = mmap_rnd_bits; - mask = mm_aslr_mask(current->mm, (1UL << bits) - 1); - return (get_random_long() & mask) << mm_page_shift(current->mm); + mask = mm_aslr_mask(mm, (1UL << bits) - 1); + return (get_random_long() & mask) << mm_page_shift(mm); +} + +unsigned long arch_mmap_rnd(void) +{ + return mmap_rnd(current->mm); } static int mmap_is_legacy(const struct rlimit *rlim_stack) @@ -443,7 +448,8 @@ static int mmap_is_legacy(const struct rlimit *rlim_stack) #define MIN_GAP (SZ_128M) #define MAX_GAP (STACK_TOP / 6 * 5) -static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rlim_stack) +static unsigned long mmap_base(const struct mm_struct *mm, const unsigned long rnd, + const struct rlimit *rlim_stack) { #ifdef CONFIG_STACK_GROWSUP /* @@ -452,15 +458,15 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli * task. mmap_base starts directly below the stack and grows * downwards. */ - return (mmap_upper_limit(rlim_stack) - rnd) & mm_page_mask(current->mm); + return (mmap_upper_limit(rlim_stack) - rnd) & mm_page_mask(mm); #else unsigned long gap = rlim_stack->rlim_cur; unsigned long pad = stack_guard_gap; /* Account for stack randomization if necessary */ if (current->flags & PF_RANDOMIZE) - pad += mm_aslr_mask(current->mm, STACK_RND_MASK) << - mm_page_shift(current->mm); + pad += mm_aslr_mask(mm, STACK_RND_MASK) << + mm_page_shift(mm); /* Values close to RLIM_INFINITY can overflow. */ if (gap + pad > gap) @@ -471,7 +477,7 @@ static unsigned long mmap_base(const unsigned long rnd, const struct rlimit *rli else if (gap > MAX_GAP) gap = MAX_GAP; - return mm_page_align(current->mm, STACK_TOP - gap - rnd); + return mm_page_align(mm, STACK_TOP - gap - rnd); #endif } @@ -480,13 +486,13 @@ void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack unsigned long random_factor = 0UL; if (current->flags & PF_RANDOMIZE) - random_factor = arch_mmap_rnd(); + random_factor = mmap_rnd(mm); if (mmap_is_legacy(rlim_stack)) { mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; mm_flags_clear(MMF_TOPDOWN, mm); } else { - mm->mmap_base = mmap_base(random_factor, rlim_stack); + mm->mmap_base = mmap_base(mm, random_factor, rlim_stack); mm_flags_set(MMF_TOPDOWN, mm); } } From 17059686aaa39e3892df7df6af6b5ca074f3e411 Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:41:13 -0700 Subject: [PATCH 629/631] arm64: user4k: keep 4K userspace off unless booted with user4k=on ARM64_USER4K_EXPERIMENTAL is now built into the release kernel by default. With it, any local user can ask for 4K pages through the private PR_AURORA_SET_{EXEC,DEFAULT}_PAGE_SIZE prctls on every CPU that supports the 4K translation granule, M1-M3 included. The feature's hardware testing so far is DJ's J700 (MacBook Neo) only; it has never run on an M1, M2 or M3. Make it an experimental opt-in. The code stays compiled in, but 4K userspace is now off unless the kernel is booted with user4k=on (parsed with kstrtobool, so 1/0/y/n work too). user4k_enabled() reports the choice from a __ro_after_init bool set by an early parameter, so it is fixed before CPU capabilities are detected. While it is off: - arch_exec_page_size_shift(), the validator behind both SET prctls, refuses every non-native size with EINVAL. A native or zero size still succeeds, and the GET prctls report the native default. - arm64_bprm_prepare_mm() ignores any exec preference and no longer gives AArch32 images 4K, so every ELF exec, and its fallback when the exec gains privilege, gets the native granule. - arch_mm_init_exec() never builds an alternative-granule mm, so no other exec path can create one either. - The 4K and 16K user granule capabilities are not detected, so nothing reports 4K support and the boot log has no "4K userspace translation granule" line. Fork copies the mm and both preferences, and neither can be set while off, so nothing non-native is inherited. The shared MM paths that every process runs stay compiled in and do not change. Add an arm64-user4k-policy KUnit suite that runs with or without the parameter. It checks the request policy for both states, and that the real prctls, exec preparation and exec mm allocation follow the boot switch. The other user4k suites now need user4k=on. Document the parameter in kernel-parameters.txt, the Kconfig help and user4k.rst. Tested under QEMU KUnit: the policy suite passes 3/3 without the parameter and 3/3 with it, and with user4k=on the arm64_user4k config gives the same per-test results as before (103 pass, 94 skip) plus the new cases. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../admin-guide/kernel-parameters.txt | 10 ++ Documentation/arch/arm64/user4k.rst | 56 +++++-- Documentation/mm/user_page_size.rst | 4 +- arch/arm64/Kconfig.debug | 29 ++-- arch/arm64/include/asm/mmu.h | 38 +++++ arch/arm64/include/asm/mmu_context.h | 3 + arch/arm64/kernel/cpufeature.c | 5 + arch/arm64/kernel/process.c | 31 ++-- arch/arm64/mm/context.c | 15 ++ arch/arm64/mm/user4k-test.c | 138 +++++++++++++++++- 10 files changed, 288 insertions(+), 41 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2ebdd37817b2e8..b359c9c231e359 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -8305,6 +8305,16 @@ Kernel parameters not on uas) Example: quirks=0419:aaf5:rl,0421:0433:rc + user4k= [ARM64,EARLY] + Format: + Enable the experimental per-process user page sizes + (4K userspace pages on a 16K or 64K kernel) built + with CONFIG_ARM64_USER4K_EXPERIMENTAL. + Default: off. While off, the private page-size + prctls refuse every non-native size with EINVAL + and every exec uses the native page size. + See Documentation/arch/arm64/user4k.rst. + user_debug= [KNL,ARM] Format: See arch/arm/Kconfig.debug help text. diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index f6278555f680da..704c84199f7236 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -11,14 +11,16 @@ fixed for its lifetime; exec can create a new address space with another supported granule. See :doc:`../../mm/user_page_size` for the generic launch policy and architecture integration contract. -This is an experimental ABI. The launch commands use private prctl numbers, -not assigned upstream UAPI. The implementation and tests do not establish -complete application, device or hardware qualification. +This is an experimental feature and ABI. It is built into the kernel but +stays off unless the kernel is booted with ``user4k=on``. The launch +commands use private prctl numbers, not assigned upstream UAPI. The +implementation and tests do not establish complete application, device or +hardware qualification. Configuration and supported combinations ======================================== -Enable ``CONFIG_ARM64_USER4K_EXPERIMENTAL`` with 48-bit VA and PA and a native +``CONFIG_ARM64_USER4K_EXPERIMENTAL`` requires 48-bit VA and PA and a native 16K or 64K page size. The option is on by default and selects the generic ``MM_SUBPAGE`` backing and ``ARCH_HAS_USER_PAGE_SIZE`` exec policy support. LPA2 and ``NO_PAGE_MAPCOUNT`` configurations are excluded by Kconfig. @@ -33,11 +35,12 @@ Native kernel size AArch64 default AArch64 alternatives 64K 64K 4K and 16K, subject to support =================== ================== ============================ -Capability detection uses the sanitized system-wide granule intersection. -AArch32 executables select 4K when that capability is available and retain -the native granule otherwise, regardless of launch preferences. The latter -case still enforces native mapping/protection alignment. AArch32 mmap2 file -offsets retain their 4K ABI unit in either case. +Capability detection uses the sanitized system-wide granule intersection, +and only runs with ``user4k=on``. AArch32 executables select 4K when that +capability is available and retain the native granule otherwise, regardless +of launch preferences. The latter case still enforces native +mapping/protection alignment. AArch32 mmap2 file offsets retain their 4K +ABI unit in either case. An AArch32 executable running under an arm64 kernel is distinct from an ``arch/arm`` kernel. The latter retains its existing native ABI; this series @@ -49,10 +52,34 @@ Larger-than-native process granules are not exposed. Internal 64K-on-16K fixtures exercise allocation, mappings and argument staging, but those fixtures do not enable a public 64K process ABI on a 16K kernel. +Enabling at boot +================ + +Building the option in does not enable it. Add ``user4k=on`` to the kernel +command line to enable it for that boot; ``user4k=off`` or no parameter +leaves it off. The value is parsed with ``kstrtobool()``, so ``1``/``0`` +and ``y``/``n`` also work, and it cannot be changed after boot. + +While it is off: + +* ``PR_AURORA_SET_EXEC_PAGE_SIZE`` and ``PR_AURORA_SET_DEFAULT_PAGE_SIZE`` + fail with ``EINVAL`` for every non-native size; a native or zero size + still succeeds. The GET commands report the native default (zero). +* Every exec, including AArch32 executables, gets the native granule. +* The CPUs report no alternative user granule: the boot log has no + ``4K userspace translation granule`` capability line. + +The shared MM code paths that serve every process are compiled in either +way and do not change with the parameter. + +With ``user4k=on`` the CPU capability line appears on CPUs that support +the 4K translation granule, and the launcher below can select 4K pages. + Selecting an executable's page size =================================== -Build the development launcher with:: +The kernel must be booted with ``user4k=on``. Build the development +launcher with:: make -C tools/aurora-userpages @@ -163,14 +190,19 @@ Test coverage ============= The arm64 selftests are in ``tools/testing/selftests/arm64/user4k``. The -``page-contract`` executable checks the exposed userspace contract. Extended +fixtures that select another page size need a guest booted with +``user4k=on``. The ``page-contract`` executable checks the exposed +userspace contract. Extended fixtures cover exec policy, AArch32, shared mappings, accounting, reclaim, I/O and device-facing interfaces. Several are disposable-guest PID1 programs with companion executables or storage requirements; they are not ordinary host test commands. ``CONFIG_ARM64_USER4K_KUNIT_TEST`` and the data/TLB suites exercise internal -geometry and ownership. Some fixtures require ``arm64.user4k_test=1`` and +geometry and ownership. Boot them with ``user4k=on``, for example +``kunit.py run --kernel_args user4k=on``. The ``arm64-user4k-policy`` suite +also runs without it and checks that the feature stays closed while off. +Some fixtures additionally require ``arm64.user4k_test=1`` and some hardware translation tests are disabled under KASAN. Check individual KTAP results and skips rather than treating boot success as a suite pass. Private IOMMUFD/VFIO test-device options are separate and should remain off diff --git a/Documentation/mm/user_page_size.rst b/Documentation/mm/user_page_size.rst index 62ef6ae39e0aca..bda5591fb8541e 100644 --- a/Documentation/mm/user_page_size.rst +++ b/Documentation/mm/user_page_size.rst @@ -59,8 +59,8 @@ An architecture selecting ``ARCH_HAS_USER_PAGE_SIZE`` must implement: * ``arch_exec_page_size_shift(size)``: return zero for a native request, a validated alternative shift, or a negative errno. Validation includes - the supported hardware and kernel configuration, not just whether the - requested byte size is a power of two. + the supported hardware, the kernel configuration and any boot-time + switch, not just whether the requested byte size is a power of two. * ``arch_mm_init_exec(mm, shift)``: initialize the private address space's geometry before allocating its page-table root. Zero denotes native geometry. The hook receives the request explicitly and does not read diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 1f779db2342d26..f9bf9f071ff54a 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -36,12 +36,20 @@ config ARM64_USER4K_EXPERIMENTAL help Build the in-development MM support for alternative 4K user tables on 16K/64K kernels and 16K user tables on 64K kernels. - Ordinary address spaces and kernel tables retain the native granule. An experimental - one-shot prctl selects a supported granule for the next exec. - This is a development ABI: several MM interfaces and device mappings - still require integration and validation. The private prctl numbers - are not upstream assignments or a stable ABI promise. - Programs that do not use the prctl keep the native granule. + + This is experimental. The code is built in, but the feature is off + at run time unless the kernel is booted with user4k=on. While it is + off, the page-size prctls refuse every non-native size with EINVAL, + every exec gets the native granule and the CPUs report no + alternative user granule. The shared MM code paths remain compiled + in either way. + + With user4k=on, an experimental prctl selects a supported granule + for the next exec. Ordinary address spaces and kernel tables retain + the native granule. This is a development ABI: several MM interfaces + and device mappings still require integration and validation. The + private prctl numbers are not upstream assignments or a stable ABI + promise. Programs that do not use the prctl keep the native granule. config ARM64_USER4K_KUNIT_TEST bool "Test mixed userspace granules with larger kernel pages" @@ -55,9 +63,11 @@ config ARM64_USER4K_KUNIT_TEST help Exercise independent TTBR0 translation granules, subpage permissions and invalidation while TTBR1 retains the normal native kernel tables. - Hardware translation/uaccess fixtures require arm64.user4k_test=1 - as a second opt-in and are disabled with KASAN. Software MM fixtures - can run with KASAN without borrowing or replacing TTBR0. + Boot the test kernel with user4k=on (kunit.py --kernel_args + user4k=on); only the arm64-user4k-policy suite is meant to run + without it. Hardware translation/uaccess fixtures also require + arm64.user4k_test=1 and are disabled with KASAN. Software MM + fixtures can run with KASAN without borrowing or replacing TTBR0. Run only in a disposable test VM. This regression harness does not establish hardware qualification or production ABI readiness. Say N for normal kernels. @@ -118,3 +128,4 @@ config ARM64_USER4K_DATA_KUNIT_TEST Exercise packed allocation, slot reuse, copying and deferred lifetime. These tests use native kernel addresses without changing TTBR0, and can run with KASAN even when the architectural tests are disabled. + Boot the test kernel with user4k=on. diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index da82ed5f0c41ff..55719dc00a413c 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -17,9 +17,47 @@ #ifndef __ASSEMBLER__ +#include #include +#include #include +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +extern bool arm64_user4k_on; + +/* + * Mixed-granule userspace is built in but experimental: it stays off unless + * the kernel is booted with user4k=on. Fixed once early parameters are parsed. + */ +static inline bool user4k_enabled(void) +{ + return arm64_user4k_on; +} +#else +static inline bool user4k_enabled(void) +{ + return false; +} +#endif + +/* + * Validate a page-size request for a later exec. Zero selects the native + * granule; an alternative needs user4k=on and CPU support. Pure for KUnit. + */ +static inline int arm64_exec_page_size_request(unsigned long size, bool enabled, + bool has_4k, bool has_16k) +{ + if (!size || size == PAGE_SIZE) + return 0; + if (!enabled) + return -EINVAL; + if (size == SZ_4K) + return has_4k ? 12 : -EOPNOTSUPP; + if (PAGE_SIZE > SZ_16K && size == SZ_16K) + return has_16k ? 14 : -EOPNOTSUPP; + return -EINVAL; +} + /* AArch32 falls back to the native ABI when 4K translation is unavailable. */ static inline unsigned int arm64_exec_page_shift(bool compat, bool has_4k, unsigned int native_shift, unsigned int requested_shift) diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index db27ff606555be..a73d4d8d992e9b 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -30,6 +30,9 @@ #define arch_mm_init_exec arch_mm_init_exec static inline void arch_mm_init_exec(struct mm_struct *mm, unsigned int page_shift) { + /* Without user4k=on every exec gets the native granule. */ + if (!user4k_enabled()) + return; if (page_shift == 12) mm->context.flags |= MMCF_USER_4K; else if (page_shift == 14 && PAGE_SHIFT > 14) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index e30ef03e87b3ef..4d69fa0fd3c76a 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -1695,6 +1695,9 @@ early_param("arm64.user16k_deny_late_cpu", user16k_deny_late_cpu_setup); static bool has_user4k_granule(const struct arm64_cpu_capabilities *entry, int scope) { + /* Without user4k=on no alternative granule is offered or reported. */ + if (!user4k_enabled()) + return false; #ifdef CONFIG_ARM64_USER4K_KUNIT_TEST if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && smp_processor_id() == user4k_deny_late_cpu) @@ -1707,6 +1710,8 @@ static bool has_user4k_granule(const struct arm64_cpu_capabilities *entry, static bool has_user16k_granule(const struct arm64_cpu_capabilities *entry, int scope) { + if (!user4k_enabled()) + return false; #ifdef CONFIG_ARM64_USER4K_KUNIT_TEST if (scope == SCOPE_LOCAL_CPU && system_capabilities_finalized() && smp_processor_id() == user16k_deny_late_cpu) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 6d54e364810343..3ef4660dc1787a 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -889,11 +889,15 @@ unsigned long arch_align_stack(unsigned long sp) #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) { - unsigned int requested = current->exec_page_shift ?: - current->default_exec_page_shift; - bool has_4k = cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE); - unsigned int shift = arm64_exec_page_shift(compat, has_4k, PAGE_SHIFT, - requested); + /* Without user4k=on every image, AArch32 included, stays native. */ + bool enabled = user4k_enabled(); + bool has_4k = enabled && cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE); + unsigned int requested = 0, shift; + + if (enabled) + requested = current->exec_page_shift ?: + current->default_exec_page_shift; + shift = arm64_exec_page_shift(compat, has_4k, PAGE_SHIFT, requested); /* begin_new_exec() falls back to this if the exec gains privilege. */ bprm->secure_page_shift = arm64_exec_page_shift(compat, has_4k, @@ -901,18 +905,15 @@ int arm64_bprm_prepare_mm(struct linux_binprm *bprm, bool compat) return bprm_set_page_shift(bprm, shift); } -/* Return zero for the native default, or a validated alternative shift. */ +/* + * Return zero for the native default, or a validated alternative shift. + * Every alternative is refused with -EINVAL unless booted with user4k=on. + */ int arch_exec_page_size_shift(unsigned long size) { - if (!size || size == PAGE_SIZE) - return 0; - if (size == SZ_4K) - return cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE) ? - 12 : -EOPNOTSUPP; - if (PAGE_SIZE > SZ_16K && size == SZ_16K) - return cpus_have_final_cap(ARM64_HAS_USER16K_GRANULE) ? - 14 : -EOPNOTSUPP; - return -EINVAL; + return arm64_exec_page_size_request(size, user4k_enabled(), + cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE), + cpus_have_final_cap(ARM64_HAS_USER16K_GRANULE)); } #endif diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index cd3f56b9427083..d7dd35722c6656 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -39,6 +39,21 @@ static unsigned long *pinned_asid_map; #define ctxid2asid(asid) ((asid) & ~ASID_MASK) #define asid2ctxid(asid, genid) ((asid) | (genid)) +#ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL +/* + * Experimental mixed-granule userspace is off unless booted with user4k=on. + * Early parameters are parsed in setup_arch(), before CPU capability + * detection reads this. + */ +bool arm64_user4k_on __ro_after_init; + +static int __init user4k_setup(char *str) +{ + return kstrtobool(str, &arm64_user4k_on); +} +early_param("user4k", user4k_setup); +#endif + #ifdef CONFIG_ARM64_USER4K_KUNIT_TEST /* Test-only capacity reduction; hardware ASID width and normal boots are unchanged. */ static bool user4k_test_small_asids; diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 9d7cbd4aae3d5b..a8c9859ef3986c 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -13743,6 +13744,123 @@ static void user4k_exec_granule_policy_test(struct kunit *test) } } +/* Exec page-size requests are refused unless booted with user4k=on. */ +static void user4k_page_size_request_test(struct kunit *test) +{ + static const struct { + unsigned long size; + int off, on, on_no4k; + } cases[] = { + { 0, 0, 0, 0 }, + { PAGE_SIZE, 0, 0, 0 }, + { SZ_4K, -EINVAL, 12, -EOPNOTSUPP }, + { SZ_8K, -EINVAL, -EINVAL, -EINVAL }, + { SZ_2M, -EINVAL, -EINVAL, -EINVAL }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + unsigned long size = cases[i].size; + + KUNIT_EXPECT_EQ(test, arm64_exec_page_size_request(size, false, true, true), + cases[i].off); + KUNIT_EXPECT_EQ(test, arm64_exec_page_size_request(size, false, false, false), + cases[i].off); + KUNIT_EXPECT_EQ(test, arm64_exec_page_size_request(size, true, true, true), + cases[i].on); + KUNIT_EXPECT_EQ(test, arm64_exec_page_size_request(size, true, false, false), + cases[i].on_no4k); + } + /* Larger-than-native process granules are never offered. */ + if (PAGE_SIZE == SZ_16K) { + KUNIT_EXPECT_EQ(test, arm64_exec_page_size_request(SZ_64K, false, true, true), + -EINVAL); + KUNIT_EXPECT_EQ(test, arm64_exec_page_size_request(SZ_64K, true, true, true), + -EINVAL); + } +} + +asmlinkage long __arm64_sys_prctl(const struct pt_regs *regs); + +static long test_prctl(unsigned long option, unsigned long arg2) +{ + struct pt_regs regs = { .regs = { option, arg2 } }; + + return __arm64_sys_prctl(®s); +} + +/* Both SET prctls follow the boot switch; native requests always succeed. */ +static void user4k_boot_switch_prctl_test(struct kunit *test) +{ + static const unsigned long options[][2] = { + { PR_AURORA_SET_EXEC_PAGE_SIZE, PR_AURORA_GET_EXEC_PAGE_SIZE }, + { PR_AURORA_SET_DEFAULT_PAGE_SIZE, PR_AURORA_GET_DEFAULT_PAGE_SIZE }, + }; + unsigned int saved_exec = current->exec_page_shift; + unsigned int saved_default = current->default_exec_page_shift; + bool has_4k = cpus_have_final_cap(ARM64_HAS_USER4K_GRANULE); + long expect = !user4k_enabled() ? -EINVAL : has_4k ? 0 : -EOPNOTSUPP; + unsigned int i; + + /* Without user4k=on the CPUs report no 4K user granule either. */ + if (!user4k_enabled()) + KUNIT_EXPECT_FALSE(test, has_4k); + for (i = 0; i < ARRAY_SIZE(options); i++) { + unsigned long set = options[i][0], get = options[i][1]; + + current->exec_page_shift = 0; + current->default_exec_page_shift = 0; + KUNIT_EXPECT_EQ(test, test_prctl(set, SZ_4K), expect); + KUNIT_EXPECT_EQ(test, test_prctl(get, 0), expect ? 0L : (long)SZ_4K); + if (PAGE_SIZE == SZ_16K) { + KUNIT_EXPECT_EQ(test, test_prctl(set, SZ_64K), (long)-EINVAL); + KUNIT_EXPECT_EQ(test, test_prctl(get, 0), expect ? 0L : (long)SZ_4K); + } + KUNIT_EXPECT_EQ(test, test_prctl(set, PAGE_SIZE), 0L); + KUNIT_EXPECT_EQ(test, test_prctl(set, 0), 0L); + KUNIT_EXPECT_EQ(test, test_prctl(get, 0), 0L); + } + current->exec_page_shift = saved_exec; + current->default_exec_page_shift = saved_default; +} + +/* With the switch off, exec ignores any preference and stays native. */ +static void user4k_boot_switch_exec_test(struct kunit *test) +{ + unsigned int saved_exec = current->exec_page_shift; + unsigned int saved_default = current->default_exec_page_shift; + struct mm_struct *mm; + unsigned int compat; + + mm = mm_alloc_exec_page_shift(12); + KUNIT_ASSERT_NOT_NULL(test, mm); + KUNIT_EXPECT_EQ(test, mm_page_shift(mm), user4k_enabled() ? 12U : PAGE_SHIFT); + mmput(mm); + /* With user4k=on, user4k_exec_secure_granule_test covers staging. */ + if (user4k_enabled()) + return; + for (compat = 0; compat < 2; compat++) { + struct linux_binprm *bprm = kunit_kzalloc(test, sizeof(*bprm), GFP_KERNEL); + int ret; + + KUNIT_ASSERT_NOT_NULL(test, bprm); + mm = mm_alloc_exec_page_shift(PAGE_SHIFT); + KUNIT_ASSERT_NOT_NULL(test, mm); + bprm->mm = mm; + current->exec_page_shift = 12; + current->default_exec_page_shift = 12; + ret = arm64_bprm_prepare_mm(bprm, compat); + current->exec_page_shift = saved_exec; + current->default_exec_page_shift = saved_default; + KUNIT_EXPECT_EQ(test, ret, 0); + KUNIT_EXPECT_PTR_EQ(test, bprm->mm, mm); + KUNIT_EXPECT_EQ(test, mm_page_shift(bprm->mm), PAGE_SHIFT); + /* AArch32 also keeps the native granule. */ + KUNIT_EXPECT_EQ(test, bprm->secure_page_shift, PAGE_SHIFT); + mmput(mm); + } +} + static void user_granule_zero_region_test(struct kunit *test) { unsigned long base_pfn = zero_pfn(0); @@ -14604,12 +14722,26 @@ static struct kunit_suite user4k_test_suite = { .test_cases = user4k_test_cases, }; +/* Safe with or without user4k=on; the other suites need user4k=on. */ +static struct kunit_case user4k_policy_cases[] = { + KUNIT_CASE(user4k_page_size_request_test), + KUNIT_CASE(user4k_boot_switch_prctl_test), + KUNIT_CASE(user4k_boot_switch_exec_test), + {} +}; + +static struct kunit_suite user4k_policy_suite = { + .name = "arm64-user4k-policy", + .test_cases = user4k_policy_cases, +}; + #if defined(CONFIG_SHMEM) && defined(CONFIG_MM_SUBPAGE) && defined(CONFIG_USERFAULTFD) -kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_shmem_swap_suite, - &user4k_coarse_shmem_suite, +kunit_test_suites(&user4k_policy_suite, &user4k_test_suite, &user4k_swap_suite, + &user4k_shmem_swap_suite, &user4k_coarse_shmem_suite, &user4k_nonlinear_swap_suite); #else -kunit_test_suites(&user4k_test_suite, &user4k_swap_suite, &user4k_nonlinear_swap_suite); +kunit_test_suites(&user4k_policy_suite, &user4k_test_suite, &user4k_swap_suite, + &user4k_nonlinear_swap_suite); #endif MODULE_LICENSE("GPL"); From 2e6535a04c4efe07ecf041705e6e17cb81b17c7b Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:41:20 -0700 Subject: [PATCH 630/631] mm: user-subpage: skip the 4K page_ext record when user4k is off MM_SUBPAGE registers a per-page 4K ownership record in page_ext whose need() callback always returned true. A kernel with user4k built in therefore reserved 32 bytes for every native page at boot (2 MiB per GiB of RAM with 16K pages, about 0.2%), although 4K userspace is now off unless the kernel is booted with user4k=on and no 4K mm can exist without it. The feature's hardware testing so far is DJ's J700 only. Add arch_mm_alt_granule_enabled(), which arm64 maps to user4k_enabled(), and return it from need(). Early parameters are parsed in setup_arch(), before mm_core_init() and page_alloc_init_late() size page_ext, so the answer is known in time. Without the record, page_ext can still be allocated for another user such as page_owner or page_table_check, and mm_subpage_ext_ops then has no offset of its own: reading it would alias that user's data. Route every lookup in user-subpage.c through helpers that return no record while user4k is off. Callers that run for all memory already treat a missing record as "no 4K owner", which is the right answer then. The shmem userfaultfd mask helpers report no missing 4K units and ignore updates, and the smaps/pagemap mapcount helper returns zero counts without warning. Add a KUnit case for the lookups on a native page in both states. Verified under QEMU KUnit (1 GiB guest): the arm64_user4k config allocates 4 MiB of page_ext with user4k=on and 2 MiB (page_table_check only) without it; the mm-core config allocates 2 MiB with it and none without it, and passes 39/39 both ways. With user4k=on the user4k configs match the earlier per-test results plus the new cases: arm64_user4k 107 pass/94 skip, mm_kasan 154/47, data_kasan 16/8, VFIO 12/12. Without it the policy suite passes 4/4. A production W=1 build (Image, modules, dtbs) has no new warnings. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .../admin-guide/kernel-parameters.txt | 5 +- Documentation/arch/arm64/user4k.rst | 4 + Documentation/mm/user_page_size.rst | 5 ++ arch/arm64/Kconfig.debug | 6 +- arch/arm64/include/asm/mmu.h | 1 + arch/arm64/mm/context.c | 4 +- arch/arm64/mm/user4k-test.c | 19 +++++ include/linux/mm_granule.h | 11 +++ mm/user-subpage.c | 74 ++++++++++++------- 9 files changed, 97 insertions(+), 32 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b359c9c231e359..5b084af09e3fa0 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -8311,8 +8311,9 @@ Kernel parameters (4K userspace pages on a 16K or 64K kernel) built with CONFIG_ARM64_USER4K_EXPERIMENTAL. Default: off. While off, the private page-size - prctls refuse every non-native size with EINVAL - and every exec uses the native page size. + prctls refuse every non-native size with EINVAL, + every exec uses the native page size and the 4K + ownership record is not reserved in page_ext. See Documentation/arch/arm64/user4k.rst. user_debug= [KNL,ARM] diff --git a/Documentation/arch/arm64/user4k.rst b/Documentation/arch/arm64/user4k.rst index 704c84199f7236..4f6a02932e3d86 100644 --- a/Documentation/arch/arm64/user4k.rst +++ b/Documentation/arch/arm64/user4k.rst @@ -68,6 +68,10 @@ While it is off: * Every exec, including AArch32 executables, gets the native granule. * The CPUs report no alternative user granule: the boot log has no ``4K userspace translation granule`` capability line. +* The 4K ownership record is not reserved in ``page_ext``. With it on, + the record takes 32 bytes for every native page of RAM (about 0.2% with + 16K pages). ``page_ext`` itself is still allocated if another user, such + as page_owner, needs it. The shared MM code paths that serve every process are compiled in either way and do not change with the parameter. diff --git a/Documentation/mm/user_page_size.rst b/Documentation/mm/user_page_size.rst index bda5591fb8541e..654581c85929d7 100644 --- a/Documentation/mm/user_page_size.rst +++ b/Documentation/mm/user_page_size.rst @@ -65,6 +65,11 @@ An architecture selecting ``ARCH_HAS_USER_PAGE_SIZE`` must implement: geometry before allocating its page-table root. Zero denotes native geometry. The hook receives the request explicitly and does not read a separate architecture thread-policy field. +* ``arch_mm_alt_granule_enabled()``, optionally: report whether this boot + may create alternative-granule address spaces. It must be fixed before + memory management initialization. When it is false the ``MM_SUBPAGE`` + page_ext record is not reserved. arm64 returns true only when booted + with ``user4k=on``. * Binary-loader integration that resolves the final executable policy before exposing the address space. ``bprm_set_page_shift()`` transfers staged argument bytes when the final choice changes the initial granule. diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index f9bf9f071ff54a..f1978b9dff2bbd 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -40,9 +40,9 @@ config ARM64_USER4K_EXPERIMENTAL This is experimental. The code is built in, but the feature is off at run time unless the kernel is booted with user4k=on. While it is off, the page-size prctls refuse every non-native size with EINVAL, - every exec gets the native granule and the CPUs report no - alternative user granule. The shared MM code paths remain compiled - in either way. + every exec gets the native granule, the CPUs report no alternative + user granule and the per-page 4K ownership record in page_ext is not + reserved. The shared MM code paths remain compiled in either way. With user4k=on, an experimental prctl selects a supported granule for the next exec. Ordinary address spaces and kernel tables retain diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 55719dc00a413c..db721e655bb4b1 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -33,6 +33,7 @@ static inline bool user4k_enabled(void) { return arm64_user4k_on; } +#define arch_mm_alt_granule_enabled user4k_enabled #else static inline bool user4k_enabled(void) { diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index d7dd35722c6656..cac6b968a6888b 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -42,8 +42,8 @@ static unsigned long *pinned_asid_map; #ifdef CONFIG_ARM64_USER4K_EXPERIMENTAL /* * Experimental mixed-granule userspace is off unless booted with user4k=on. - * Early parameters are parsed in setup_arch(), before CPU capability - * detection reads this. + * Early parameters are parsed in setup_arch(), before page_ext sizing and + * CPU capability detection read this. */ bool arm64_user4k_on __ro_after_init; diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index a8c9859ef3986c..dfc7086ae05383 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -13861,6 +13861,24 @@ static void user4k_boot_switch_exec_test(struct kunit *test) } } +/* Native pages never have a 4K owner; without user4k=on there is no record. */ +static void user4k_boot_switch_page_ext_test(struct kunit *test) +{ + struct mm_subpage_mapcounts counts; + struct page *page = alloc_page(GFP_KERNEL); + unsigned int i; + + KUNIT_ASSERT_NOT_NULL(test, page); + memset(&counts, 0xff, sizeof(counts)); + KUNIT_EXPECT_EQ(test, mm_subpage_file_mapcounts(page, &counts), 0U); + for (i = 0; i < MM_SUBPAGES_PER_PAGE; i++) + KUNIT_EXPECT_EQ(test, counts.slots[i], 0U); + KUNIT_EXPECT_NULL(test, mm_subpage_get_from_phys(page_to_phys(page))); + if (!user4k_enabled()) + KUNIT_EXPECT_EQ(test, mm_subpage_shmem_missing(page), 0UL); + __free_page(page); +} + static void user_granule_zero_region_test(struct kunit *test) { unsigned long base_pfn = zero_pfn(0); @@ -14727,6 +14745,7 @@ static struct kunit_case user4k_policy_cases[] = { KUNIT_CASE(user4k_page_size_request_test), KUNIT_CASE(user4k_boot_switch_prctl_test), KUNIT_CASE(user4k_boot_switch_exec_test), + KUNIT_CASE(user4k_boot_switch_page_ext_test), {} }; diff --git a/include/linux/mm_granule.h b/include/linux/mm_granule.h index 6e73216edcf96d..ceb676d822db82 100644 --- a/include/linux/mm_granule.h +++ b/include/linux/mm_granule.h @@ -18,6 +18,17 @@ static inline unsigned int arch_mm_page_shift(const struct mm_struct *mm) } #endif +/* + * Whether this boot may create an mm with a non-native user granule. An + * architecture with a boot-time switch fixes it before mm initialisation. + */ +#ifndef arch_mm_alt_granule_enabled +static inline bool arch_mm_alt_granule_enabled(void) +{ + return IS_ENABLED(CONFIG_MM_SUBPAGE); +} +#endif + static inline unsigned int mm_page_shift(const struct mm_struct *mm) { return arch_mm_page_shift(mm); diff --git a/mm/user-subpage.c b/mm/user-subpage.c index c9101990099df9..62638337ee40d3 100644 --- a/mm/user-subpage.c +++ b/mm/user-subpage.c @@ -69,12 +69,41 @@ static_assert(SUBPAGES_PER_FOLIO == 1 || sizeof(struct mm_subpage_ext) == sizeof(void *))); static_assert(SUBPAGES_PER_FOLIO <= sizeof(u16) * BITS_PER_BYTE); +/* + * The record is reserved only when this boot may create alternative-granule + * mms (on arm64, user4k=on). Otherwise page_ext may still exist for other + * users, but mm_subpage_ext_ops has no offset in it and must not be read. + */ +static bool __init subpage_ext_needed(void) +{ + return arch_mm_alt_granule_enabled(); +} + +struct page_ext_operations mm_subpage_ext_ops = { + .size = sizeof(struct mm_subpage_ext), + .need = subpage_ext_needed, +}; + +static struct page_ext *subpage_ext_get(const struct page *page) +{ + return arch_mm_alt_granule_enabled() ? page_ext_get(page) : NULL; +} + +static struct page_ext *subpage_ext_from_phys(phys_addr_t phys) +{ + return arch_mm_alt_granule_enabled() ? page_ext_from_phys(phys) : NULL; +} + unsigned long mm_subpage_shmem_missing(struct page *page) { - struct page_ext *ext = page_ext_get(page); + struct page_ext *ext; struct mm_subpage_ext *subext; unsigned long missing; + /* Without alternative granules no 4K unit is ever missing. */ + if (!arch_mm_alt_granule_enabled()) + return 0; + ext = subpage_ext_get(page); /* A referenced native page keeps its initialized section online. */ VM_BUG_ON(!ext); subext = page_ext_data(ext, &mm_subpage_ext_ops); @@ -85,26 +114,19 @@ unsigned long mm_subpage_shmem_missing(struct page *page) void mm_subpage_shmem_set_missing(struct page *page, unsigned long missing) { - struct page_ext *ext = page_ext_get(page); + struct page_ext *ext; struct mm_subpage_ext *subext; - VM_BUG_ON(!ext); VM_BUG_ON(missing & ~GENMASK(SUBPAGES_PER_FOLIO - 1, 0)); + if (!arch_mm_alt_granule_enabled()) + return; + ext = subpage_ext_get(page); + VM_BUG_ON(!ext); subext = page_ext_data(ext, &mm_subpage_ext_ops); subext->shmem_missing = missing; page_ext_put(ext); } -static bool __init subpage_ext_needed(void) -{ - return true; -} - -struct page_ext_operations mm_subpage_ext_ops = { - .size = sizeof(struct mm_subpage_ext), - .need = subpage_ext_needed, -}; - /* Only alternative PTE mappings contribute; native rmap counts stay unchanged. */ static void subpage_file_map_update(phys_addr_t phys, unsigned long size, int delta) { @@ -115,7 +137,7 @@ static void subpage_file_map_update(phys_addr_t phys, unsigned long size, int de VM_BUG_ON(!is_power_of_2(size) || size < MM_SUBPAGE_SIZE || size >= PAGE_SIZE || !IS_ALIGNED(phys, size)); - ext = page_ext_from_phys(phys); + ext = subpage_ext_from_phys(phys); if (WARN_ON_ONCE(!ext)) return; subext = page_ext_data(ext, &mm_subpage_ext_ops); @@ -144,11 +166,13 @@ void __mm_subpage_file_map_del(phys_addr_t phys, unsigned long size) /* Snapshot for statistics only, never an ownership or pinning decision. */ unsigned int mm_subpage_file_mapcounts(struct page *page, struct mm_subpage_mapcounts *counts) { - struct page_ext *ext = page_ext_get(page); + struct page_ext *ext = subpage_ext_get(page); struct mm_subpage_ext *subext; unsigned int i, leaves; - if (WARN_ON_ONCE(!ext)) { + /* Without alternative granules there are no alternative leaves. */ + if (!ext) { + WARN_ON_ONCE(arch_mm_alt_granule_enabled()); memset(counts, 0, sizeof(*counts)); return 0; } @@ -162,7 +186,7 @@ unsigned int mm_subpage_file_mapcounts(struct page *page, struct mm_subpage_mapc static int owner_bind(struct mm_subpage_owner *owner) { - struct page_ext *ext = page_ext_get(&owner->folio->page); + struct page_ext *ext = subpage_ext_get(&owner->folio->page); struct mm_subpage_ext *subext; int ret; @@ -176,7 +200,7 @@ static int owner_bind(struct mm_subpage_owner *owner) static void owner_unbind(struct mm_subpage_owner *owner) { - struct page_ext *ext = page_ext_get(&owner->folio->page); + struct page_ext *ext = subpage_ext_get(&owner->folio->page); struct mm_subpage_ext *subext; /* The allocated, referenced folio prevents its section going offline. */ @@ -197,7 +221,7 @@ struct mm_subpage *mm_subpage_get_from_phys(phys_addr_t phys) if (!IS_ALIGNED(phys, MM_SUBPAGE_SIZE)) return NULL; - ext = page_ext_from_phys(phys); + ext = subpage_ext_from_phys(phys); if (!ext) return NULL; subext = page_ext_data(ext, &mm_subpage_ext_ops); @@ -906,7 +930,7 @@ void mm_subpage_move_finish(struct mm_subpage_move *move) int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps) { - struct page_ext *ext = page_ext_get(&folio->page); + struct page_ext *ext = subpage_ext_get(&folio->page); struct mm_subpage_ext *subext; struct mm_subpage_owner *owner; unsigned long flags; @@ -941,7 +965,7 @@ int mm_subpage_snapshot_rmap(struct folio *folio, struct mm_subpage_rmap *maps) bool mm_subpage_address_in_vma(const struct folio *folio, const struct vm_area_struct *vma, unsigned long *address) { - struct page_ext *ext = page_ext_get(&folio->page); + struct page_ext *ext = subpage_ext_get(&folio->page); struct mm_subpage_ext *subext; struct mm_subpage_owner *owner; unsigned long flags; @@ -1491,7 +1515,7 @@ struct mm_subpage *mm_subpage_copy_locked(struct mm_subpage_pool *pool, static struct mm_subpage *restore_bound_slot(struct folio *folio, unsigned int offset, unsigned int shift) { - struct page_ext *ext = page_ext_get(&folio->page); + struct page_ext *ext = subpage_ext_get(&folio->page); struct mm_subpage_owner *owner; struct mm_subpage_pool *pool; struct mm_subpage_ext *subext; @@ -1639,7 +1663,7 @@ int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) VM_BUG_ON_FOLIO(!folio_test_locked(src), src); VM_BUG_ON_FOLIO(!folio_test_locked(dst), dst); - ext = page_ext_get(&src->page); + ext = subpage_ext_get(&src->page); if (!ext) return 0; subext = page_ext_data(ext, &mm_subpage_ext_ops); @@ -1716,7 +1740,7 @@ int mm_subpage_migrate_prepare(struct folio *src, struct folio *dst) bool mm_subpage_migrating(struct folio *folio) { - struct page_ext *ext = page_ext_get(&folio->page); + struct page_ext *ext = subpage_ext_get(&folio->page); struct mm_subpage_owner *owner; struct mm_subpage_ext *subext; bool active = false; @@ -1756,7 +1780,7 @@ struct mm_subpage *mm_subpage_migrate_restore(struct folio *folio, unsigned int void mm_subpage_migrate_finish(struct folio *folio) { - struct page_ext *ext = page_ext_get(&folio->page); + struct page_ext *ext = subpage_ext_get(&folio->page); struct mm_subpage_owner *owner; struct mm_subpage_ext *subext; struct mm_subpage_pool *pool; From 7638540a67d8a8fd1a0f228ba119e14193702f9e Mon Sep 17 00:00:00 2001 From: Chris Kearney <303316+iconidentify@users.noreply.github.com> Date: Sat, 17 Oct 2026 09:41:20 -0700 Subject: [PATCH 631/631] tools/aurora-userpages: say that 4K pages need user4k=on 4K userspace is now off unless the kernel is booted with user4k=on, and without it the kernel refuses a 4K request with EINVAL. The launcher then only printed "Invalid argument", which does not say what is missing. Say in the README that the kernel must be booted with user4k=on, and print a hint when a smaller-than-native size is refused with EINVAL. errno is saved before the first message is printed. Signed-off-by: Chris Kearney <303316+iconidentify@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- tools/aurora-userpages/README.rst | 4 +++- tools/aurora-userpages/userpages.c | 21 +++++++++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tools/aurora-userpages/README.rst b/tools/aurora-userpages/README.rst index 64f4fcb0dd12ac..92b8a8c9224120 100644 --- a/tools/aurora-userpages/README.rst +++ b/tools/aurora-userpages/README.rst @@ -4,7 +4,9 @@ Per-process userspace page-size launcher ======================================= Build with ``make`` in this directory. The running arm64 kernel must provide -Aurora's experimental per-process granule API. +Aurora's experimental per-process granule API and be booted with +``user4k=on``. The feature is off by default; without that parameter every +non-native size is refused with ``EINVAL`` and ``native`` still works. Examples:: diff --git a/tools/aurora-userpages/userpages.c b/tools/aurora-userpages/userpages.c index 02d9e4357224de..51993fbc92fc0a 100644 --- a/tools/aurora-userpages/userpages.c +++ b/tools/aurora-userpages/userpages.c @@ -19,6 +19,15 @@ static int usage(const char *name, int status) return status; } +/* Without user4k=on the kernel refuses every smaller size with EINVAL. */ +static int off_hint(unsigned long size, int error) +{ + if (error == EINVAL && size && size < getauxval(AT_PAGESZ)) + fputs("Smaller page sizes need a kernel booted with user4k=on.\n", + stderr); + return 1; +} + int main(int argc, char **argv) { unsigned long size; @@ -66,14 +75,18 @@ int main(int argc, char **argv) return usage(argv[0], 2); if (!once && prctl(PR_AURORA_SET_DEFAULT_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + int error = errno; + fprintf(stderr, "Cannot set inherited %s pages: %s\n", - argv[first], strerror(errno)); - return 1; + argv[first], strerror(error)); + return off_hint(size, error); } if (prctl(PR_AURORA_SET_EXEC_PAGE_SIZE, size, 0UL, 0UL, 0UL)) { + int error = errno; + fprintf(stderr, "Cannot select %s userspace pages: %s\n", - argv[first], strerror(errno)); - return 1; + argv[first], strerror(error)); + return off_hint(size, error); } execvp(argv[first + 1], &argv[first + 1]); int error = errno;